New upstream version 18.11-rc1upstream/18.11-rc1

Change-Id: Iaa71986dd6332e878d8f4bf493101b2bbc6313bb
Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>

684 files changed, 81726 insertions, 21408 deletions

diff --git a/drivers/Makefile b/drivers/Makefile
index 75660765..7d5da5d9 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -5,6 +5,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 
 DIRS-y += common
 DIRS-y += bus
+DEPDIRS-bus := common
 DIRS-y += mempool
 DEPDIRS-mempool := common bus
 DIRS-y += net
diff --git a/drivers/baseband/meson.build b/drivers/baseband/meson.build
new file mode 100644
index 00000000..52489df3
--- /dev/null
+++ b/drivers/baseband/meson.build
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
+
+drivers = ['null']
+
+config_flag_fmt = 'RTE_LIBRTE_@0@_PMD'
+driver_name_fmt = 'rte_pmd_@0@'
diff --git a/drivers/baseband/null/meson.build b/drivers/baseband/null/meson.build
new file mode 100644
index 00000000..64c29d86
--- /dev/null
+++ b/drivers/baseband/null/meson.build
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
+
+deps += ['bbdev', 'bus_vdev', 'ring']
+name = 'bbdev_null'
+allow_experimental_apis = true
+sources = files('bbdev_null.c')
diff --git a/drivers/bus/dpaa/Makefile b/drivers/bus/dpaa/Makefile
index bffaa9d9..800e5cd2 100644
--- a/drivers/bus/dpaa/Makefile
+++ b/drivers/bus/dpaa/Makefile
@@ -14,7 +14,6 @@ CFLAGS := -I$(SRCDIR) $(CFLAGS)
 CFLAGS += -O3 $(WERROR_FLAGS)
 CFLAGS += -Wno-pointer-arith
 CFLAGS += -Wno-cast-qual
-CFLAGS += -D _GNU_SOURCE
 CFLAGS += -I$(RTE_BUS_DPAA)/
 CFLAGS += -I$(RTE_BUS_DPAA)/include
 CFLAGS += -I$(RTE_BUS_DPAA)/base/qbman
@@ -24,7 +23,7 @@ CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include
 # versioning export map
 EXPORT_MAP := rte_bus_dpaa_version.map
 
-LIBABIVER := 1
+LIBABIVER := 2
 
 # all source are stored in SRCS-y
 #
@@ -48,5 +47,6 @@ SRCS-$(CONFIG_RTE_LIBRTE_DPAA_BUS) += \
 LDLIBS += -lpthread
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev
+LDLIBS += -lrte_common_dpaax
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/bus/dpaa/base/fman/netcfg_layer.c b/drivers/bus/dpaa/base/fman/netcfg_layer.c
index 031c6f1a..6b522420 100644
--- a/drivers/bus/dpaa/base/fman/netcfg_layer.c
+++ b/drivers/bus/dpaa/base/fman/netcfg_layer.c
@@ -21,7 +21,7 @@
 /* This data structure contaings all configurations information
  * related to usages of DPA devices.
  */
-struct netcfg_info *netcfg;
+static struct netcfg_info *netcfg;
 /* fd to open a socket for making ioctl request to disable/enable shared
  *  interfaces.
  */
diff --git a/drivers/bus/dpaa/base/qbman/bman_driver.c b/drivers/bus/dpaa/base/qbman/bman_driver.c
index b14b5905..750b756b 100644
--- a/drivers/bus/dpaa/base/qbman/bman_driver.c
+++ b/drivers/bus/dpaa/base/qbman/bman_driver.c
@@ -23,7 +23,7 @@ static void *bman_ccsr_map;
 /* Portal driver */
 /*****************/
 
-static __thread int fd = -1;
+static __thread int bmfd = -1;
 static __thread struct bm_portal_config pcfg;
 static __thread struct dpaa_ioctl_portal_map map = {
 	.type = dpaa_portal_bman
@@ -70,14 +70,14 @@ static int fsl_bman_portal_init(uint32_t idx, int is_shared)
 	pcfg.index = map.index;
 	bman_depletion_fill(&pcfg.mask);
 
-	fd = open(BMAN_PORTAL_IRQ_PATH, O_RDONLY);
-	if (fd == -1) {
+	bmfd = open(BMAN_PORTAL_IRQ_PATH, O_RDONLY);
+	if (bmfd == -1) {
 		pr_err("BMan irq init failed");
 		process_portal_unmap(&map.addr);
 		return -EBUSY;
 	}
 	/* Use the IRQ FD as a unique IRQ number */
-	pcfg.irq = fd;
+	pcfg.irq = bmfd;
 
 	portal = bman_create_affine_portal(&pcfg);
 	if (!portal) {
@@ -90,7 +90,7 @@ static int fsl_bman_portal_init(uint32_t idx, int is_shared)
 	/* Set the IRQ number */
 	irq_map.type = dpaa_portal_bman;
 	irq_map.portal_cinh = map.addr.cinh;
-	process_portal_irq_map(fd, &irq_map);
+	process_portal_irq_map(bmfd, &irq_map);
 	return 0;
 }
 
@@ -99,7 +99,7 @@ static int fsl_bman_portal_finish(void)
 	__maybe_unused const struct bm_portal_config *cfg;
 	int ret;
 
-	process_portal_irq_unmap(fd);
+	process_portal_irq_unmap(bmfd);
 
 	cfg = bman_destroy_affine_portal();
 	DPAA_BUG_ON(cfg != &pcfg);
@@ -109,6 +109,11 @@ static int fsl_bman_portal_finish(void)
 	return ret;
 }
 
+int bman_thread_fd(void)
+{
+	return bmfd;
+}
+
 int bman_thread_init(void)
 {
 	/* Convert from contiguous/virtual cpu numbering to real cpu when
diff --git a/drivers/bus/dpaa/base/qbman/qman.c b/drivers/bus/dpaa/base/qbman/qman.c
index 7c17027f..dc64d089 100644
--- a/drivers/bus/dpaa/base/qbman/qman.c
+++ b/drivers/bus/dpaa/base/qbman/qman.c
@@ -852,11 +852,9 @@ mr_loop:
 			case QM_MR_VERB_FQPN:
 				/* Parked */
 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-				fq = get_fq_table_entry(
-					be32_to_cpu(msg->fq.contextB));
+				fq = get_fq_table_entry(msg->fq.contextB);
 #else
-				fq = (void *)(uintptr_t)
-					be32_to_cpu(msg->fq.contextB);
+				fq = (void *)(uintptr_t)msg->fq.contextB;
 #endif
 				fq_state_change(p, fq, msg, verb);
 				if (fq->cb.fqs)
@@ -967,7 +965,6 @@ static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 		*shadow = *dq;
 		dq = shadow;
 		shadow->fqid = be32_to_cpu(shadow->fqid);
-		shadow->contextB = be32_to_cpu(shadow->contextB);
 		shadow->seqnum = be16_to_cpu(shadow->seqnum);
 		hw_fd_to_cpu(&shadow->fd);
 #endif
@@ -1040,6 +1037,50 @@ static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 	return limit;
 }
 
+int qman_irqsource_add(u32 bits)
+{
+	struct qman_portal *p = get_affine_portal();
+
+	bits = bits & QM_PIRQ_VISIBLE;
+
+	/* Clear any previously remaining interrupt conditions in
+	 * QCSP_ISR. This prevents raising a false interrupt when
+	 * interrupt conditions are enabled in QCSP_IER.
+	 */
+	qm_isr_status_clear(&p->p, bits);
+	dpaa_set_bits(bits, &p->irq_sources);
+	qm_isr_enable_write(&p->p, p->irq_sources);
+
+
+	return 0;
+}
+
+int qman_irqsource_remove(u32 bits)
+{
+	struct qman_portal *p = get_affine_portal();
+	u32 ier;
+
+	/* Our interrupt handler only processes+clears status register bits that
+	 * are in p->irq_sources. As we're trimming that mask, if one of them
+	 * were to assert in the status register just before we remove it from
+	 * the enable register, there would be an interrupt-storm when we
+	 * release the IRQ lock. So we wait for the enable register update to
+	 * take effect in h/w (by reading it back) and then clear all other bits
+	 * in the status register. Ie. we clear them from ISR once it's certain
+	 * IER won't allow them to reassert.
+	 */
+
+	bits &= QM_PIRQ_VISIBLE;
+	dpaa_clear_bits(bits, &p->irq_sources);
+	qm_isr_enable_write(&p->p, p->irq_sources);
+	ier = qm_isr_enable_read(&p->p);
+	/* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
+	 * data-dependency, ie. to protect against re-ordering.
+	 */
+	qm_isr_status_clear(&p->p, ~ier);
+	return 0;
+}
+
 u16 qman_affine_channel(int cpu)
 {
 	if (cpu < 0) {
@@ -1092,9 +1133,9 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit,
 
 		/* SDQCR: context_b points to the FQ */
 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-		fq = qman_fq_lookup_table[be32_to_cpu(dq[rx_number]->contextB)];
+		fq = qman_fq_lookup_table[dq[rx_number]->contextB];
 #else
-		fq = (void *)be32_to_cpu(dq[rx_number]->contextB);
+		fq = (void *)dq[rx_number]->contextB;
 #endif
 		if (fq->cb.dqrr_prepare)
 			fq->cb.dqrr_prepare(shadow[rx_number],
@@ -1114,6 +1155,14 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit,
 	return rx_number;
 }
 
+void qman_clear_irq(void)
+{
+	struct qman_portal *p = get_affine_portal();
+	u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
+		~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
+	qm_isr_status_clear(&p->p, clear);
+}
+
 u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit,
 			void **bufs)
 {
@@ -1143,7 +1192,6 @@ u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit,
 		*shadow = *dq;
 		dq = shadow;
 		shadow->fqid = be32_to_cpu(shadow->fqid);
-		shadow->contextB = be32_to_cpu(shadow->contextB);
 		shadow->seqnum = be16_to_cpu(shadow->seqnum);
 		hw_fd_to_cpu(&shadow->fd);
 #endif
@@ -1208,7 +1256,6 @@ struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq)
 	*shadow = *dq;
 	dq = shadow;
 	shadow->fqid = be32_to_cpu(shadow->fqid);
-	shadow->contextB = be32_to_cpu(shadow->contextB);
 	shadow->seqnum = be16_to_cpu(shadow->seqnum);
 	hw_fd_to_cpu(&shadow->fd);
 #endif
@@ -1504,7 +1551,7 @@ int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
 
 		mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB;
 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-		mcc->initfq.fqd.context_b = fq->key;
+		mcc->initfq.fqd.context_b = cpu_to_be32(fq->key);
 #else
 		mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq;
 #endif
@@ -2186,11 +2233,6 @@ int qman_enqueue_multi(struct qman_fq *fq,
 	/* try to send as many frames as possible */
 	while (eqcr->available && frames_to_send--) {
 		eq->fqid = fq->fqid_le;
-#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-		eq->tag = cpu_to_be32(fq->key);
-#else
-		eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
-#endif
 		eq->fd.opaque_addr = fd->opaque_addr;
 		eq->fd.addr = cpu_to_be40(fd->addr);
 		eq->fd.status = cpu_to_be32(fd->status);
diff --git a/drivers/bus/dpaa/base/qbman/qman_driver.c b/drivers/bus/dpaa/base/qbman/qman_driver.c
index f6ecd6b2..ba153396 100644
--- a/drivers/bus/dpaa/base/qbman/qman_driver.c
+++ b/drivers/bus/dpaa/base/qbman/qman_driver.c
@@ -113,6 +113,11 @@ static int fsl_qman_portal_finish(void)
 	return ret;
 }
 
+int qman_thread_fd(void)
+{
+	return qmfd;
+}
+
 int qman_thread_init(void)
 {
 	/* Convert from contiguous/virtual cpu numbering to real cpu when
@@ -135,7 +140,7 @@ void qman_thread_irq(void)
 	 * rather than breaking that encapsulation I am simply hard-coding the
 	 * offset to the inhibit register here.
 	 */
-	out_be32(qpcfg.addr_virt[DPAA_PORTAL_CI] + 0xe0c, 0);
+	out_be32(qpcfg.addr_virt[DPAA_PORTAL_CI] + 0x36C0, 0);
 }
 
 struct qman_portal *fsl_qman_portal_create(void)
diff --git a/drivers/bus/dpaa/dpaa_bus.c b/drivers/bus/dpaa/dpaa_bus.c
index 16fabd1b..203f60dc 100644
--- a/drivers/bus/dpaa/dpaa_bus.c
+++ b/drivers/bus/dpaa/dpaa_bus.c
@@ -34,6 +34,7 @@
 
 #include <rte_dpaa_bus.h>
 #include <rte_dpaa_logs.h>
+#include <dpaax_iova_table.h>
 
 #include <fsl_usd.h>
 #include <fsl_qman.h>
@@ -46,7 +47,7 @@ int dpaa_logtype_mempool;
 int dpaa_logtype_pmd;
 int dpaa_logtype_eventdev;
 
-struct rte_dpaa_bus rte_dpaa_bus;
+static struct rte_dpaa_bus rte_dpaa_bus;
 struct netcfg_info *dpaa_netcfg;
 
 /* define a variable to hold the portal_key, once created.*/
@@ -165,6 +166,8 @@ dpaa_create_device_list(void)
 			goto cleanup;
 		}
 
+		dev->device.bus = &rte_dpaa_bus.bus;
+
 		cfg = &dpaa_netcfg->port_cfg[i];
 		fman_intf = cfg->fman_if;
 
@@ -546,6 +549,9 @@ rte_dpaa_bus_probe(void)
 		fclose(svr_file);
 	}
 
+	/* And initialize the PA->VA translation table */
+	dpaax_iova_table_populate();
+
 	/* For each registered driver, and device, call the driver->probe */
 	TAILQ_FOREACH(dev, &rte_dpaa_bus.device_list, next) {
 		TAILQ_FOREACH(drv, &rte_dpaa_bus.driver_list, next) {
@@ -553,6 +559,9 @@ rte_dpaa_bus_probe(void)
 			if (ret)
 				continue;
 
+			if (rte_dev_is_probed(&dev->device))
+				continue;
+
 			if (!drv->probe ||
 			    (dev->device.devargs &&
 			    dev->device.devargs->policy == RTE_DEV_BLACKLISTED))
@@ -563,8 +572,12 @@ rte_dpaa_bus_probe(void)
 			    dev->device.devargs->policy ==
 			    RTE_DEV_WHITELISTED)) {
 				ret = drv->probe(drv, dev);
-				if (ret)
+				if (ret) {
 					DPAA_BUS_ERR("Unable to probe.\n");
+				} else {
+					dev->driver = drv;
+					dev->device.driver = &drv->driver;
+				}
 			}
 			break;
 		}
@@ -611,7 +624,7 @@ rte_dpaa_get_iommu_class(void)
 	return RTE_IOVA_PA;
 }
 
-struct rte_dpaa_bus rte_dpaa_bus = {
+static struct rte_dpaa_bus rte_dpaa_bus = {
 	.bus = {
 		.scan = rte_dpaa_bus_scan,
 		.probe = rte_dpaa_bus_probe,
diff --git a/drivers/bus/dpaa/include/compat.h b/drivers/bus/dpaa/include/compat.h
index 92241d23..41226577 100644
--- a/drivers/bus/dpaa/include/compat.h
+++ b/drivers/bus/dpaa/include/compat.h
@@ -57,8 +57,9 @@
 #ifndef __packed
 #define __packed	__rte_packed
 #endif
+#ifndef noinline
 #define noinline	__attribute__((noinline))
-
+#endif
 #define L1_CACHE_BYTES 64
 #define ____cacheline_aligned __attribute__((aligned(L1_CACHE_BYTES)))
 #define __stringify_1(x) #x
@@ -75,20 +76,25 @@
 		printf(fmt, ##args); \
 		fflush(stdout); \
 	} while (0)
-
+#ifndef pr_crit
 #define pr_crit(fmt, args...)	 prflush("CRIT:" fmt, ##args)
+#endif
+#ifndef pr_err
 #define pr_err(fmt, args...)	 prflush("ERR:" fmt, ##args)
+#endif
+#ifndef pr_warn
 #define pr_warn(fmt, args...)	 prflush("WARN:" fmt, ##args)
+#endif
+#ifndef pr_info
 #define pr_info(fmt, args...)	 prflush(fmt, ##args)
-
-#ifdef RTE_LIBRTE_DPAA_DEBUG_BUS
-#ifdef pr_debug
-#undef pr_debug
 #endif
+#ifndef pr_debug
+#ifdef RTE_LIBRTE_DPAA_DEBUG_BUS
 #define pr_debug(fmt, args...)	printf(fmt, ##args)
 #else
 #define pr_debug(fmt, args...) {}
 #endif
+#endif
 
 #define DPAA_BUG_ON(x) RTE_ASSERT(x)
 
@@ -256,7 +262,9 @@ __bswap_24(uint32_t x)
 #define be16_to_cpu(x) rte_be_to_cpu_16(x)
 
 #define cpu_to_be64(x) rte_cpu_to_be_64(x)
+#if !defined(cpu_to_be32)
 #define cpu_to_be32(x) rte_cpu_to_be_32(x)
+#endif
 #define cpu_to_be16(x) rte_cpu_to_be_16(x)
 
 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
diff --git a/drivers/bus/dpaa/include/fsl_fman_crc64.h b/drivers/bus/dpaa/include/fsl_fman_crc64.h
index bf162f3a..08ad6304 100644
--- a/drivers/bus/dpaa/include/fsl_fman_crc64.h
+++ b/drivers/bus/dpaa/include/fsl_fman_crc64.h
@@ -42,9 +42,9 @@ struct fman_crc64_t {
 	uint64_t initial;
 	uint64_t table[1 << 8];
 };
-extern struct fman_crc64_t FMAN_CRC64_ECMA_182;
+extern struct fman_crc64_t fman_crc64_ecma_182;
 #define DECLARE_FMAN_CRC64_TABLE() \
-struct fman_crc64_t FMAN_CRC64_ECMA_182 = { \
+struct fman_crc64_t fman_crc64_ecma_182 = { \
 	0xFFFFFFFFFFFFFFFFULL, \
 	{ \
 		0x0000000000000000ULL, 0xb32e4cbe03a75f6fULL, \
@@ -183,7 +183,7 @@ struct fman_crc64_t FMAN_CRC64_ECMA_182 = { \
  */
 static inline uint64_t fman_crc64_init(void)
 {
-	return FMAN_CRC64_ECMA_182.initial;
+	return fman_crc64_ecma_182.initial;
 }
 
 /* Updates the CRC with arbitrary data */
@@ -192,7 +192,7 @@ static inline uint64_t fman_crc64_update(uint64_t crc,
 {
 	uint8_t *p = data;
 	while (len--)
-		crc = FMAN_CRC64_ECMA_182.table[(crc ^ *(p++)) & 0xff] ^
+		crc = fman_crc64_ecma_182.table[(crc ^ *(p++)) & 0xff] ^
 				(crc >> 8);
 	return crc;
 }
diff --git a/drivers/bus/dpaa/include/fsl_qman.h b/drivers/bus/dpaa/include/fsl_qman.h
index b18cf037..e4384149 100644
--- a/drivers/bus/dpaa/include/fsl_qman.h
+++ b/drivers/bus/dpaa/include/fsl_qman.h
@@ -1316,6 +1316,26 @@ u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit,
 			void **bufs);
 
 /**
+ * qman_irqsource_add - add processing sources to be interrupt-driven
+ * @bits: bitmask of QM_PIRQ_**I processing sources
+ *
+ * Adds processing sources that should be interrupt-driven (rather than
+ * processed via qman_poll_***() functions). Returns zero for success, or
+ * -EINVAL if the current CPU is sharing a portal hosted on another CPU.
+ */
+int qman_irqsource_add(u32 bits);
+
+/**
+ * qman_irqsource_remove - remove processing sources from being interrupt-driven
+ * @bits: bitmask of QM_PIRQ_**I processing sources
+ *
+ * Removes processing sources from being interrupt-driven, so that they will
+ * instead be processed via qman_poll_***() functions. Returns zero for success,
+ * or -EINVAL if the current CPU is sharing a portal hosted on another CPU.
+ */
+int qman_irqsource_remove(u32 bits);
+
+/**
  * qman_affine_channel - return the channel ID of an portal
  * @cpu: the cpu whose affine portal is the subject of the query
  *
diff --git a/drivers/bus/dpaa/include/fsl_usd.h b/drivers/bus/dpaa/include/fsl_usd.h
index e1836175..ec1ab7ce 100644
--- a/drivers/bus/dpaa/include/fsl_usd.h
+++ b/drivers/bus/dpaa/include/fsl_usd.h
@@ -55,6 +55,10 @@ int qman_free_raw_portal(struct dpaa_raw_portal *portal);
 int bman_allocate_raw_portal(struct dpaa_raw_portal *portal);
 int bman_free_raw_portal(struct dpaa_raw_portal *portal);
 
+/* Obtain thread-local UIO file-descriptors */
+int qman_thread_fd(void);
+int bman_thread_fd(void);
+
 /* Post-process interrupts. NB, the kernel IRQ handler disables the interrupt
  * line before notifying us, and this post-processing re-enables it once
  * processing is complete. As such, it is essential to call this before going
@@ -63,6 +67,8 @@ int bman_free_raw_portal(struct dpaa_raw_portal *portal);
 void qman_thread_irq(void);
 void bman_thread_irq(void);
 
+void qman_clear_irq(void);
+
 /* Global setup */
 int qman_global_init(void);
 int bman_global_init(void);
diff --git a/drivers/bus/dpaa/meson.build b/drivers/bus/dpaa/meson.build
index d10b62c0..1fcb4e91 100644
--- a/drivers/bus/dpaa/meson.build
+++ b/drivers/bus/dpaa/meson.build
@@ -1,11 +1,13 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright 2018 NXP
 
+version = 2
+
 if host_machine.system() != 'linux'
         build = false
 endif
 
-deps += ['eventdev']
+deps += ['common_dpaax', 'eventdev']
 sources = files('base/fman/fman.c',
 		'base/fman/fman_hw.c',
 		'base/fman/netcfg_layer.c',
@@ -26,4 +28,3 @@ if cc.has_argument('-Wno-cast-qual')
 endif
 
 includes += include_directories('include', 'base/qbman')
-cflags += ['-D_GNU_SOURCE']
diff --git a/drivers/bus/dpaa/rte_bus_dpaa_version.map b/drivers/bus/dpaa/rte_bus_dpaa_version.map
index 7d6d6243..70076c7a 100644
--- a/drivers/bus/dpaa/rte_bus_dpaa_version.map
+++ b/drivers/bus/dpaa/rte_bus_dpaa_version.map
@@ -95,10 +95,24 @@ DPDK_18.02 {
 
 DPDK_18.08 {
 	global:
-
 	fman_if_get_sg_enable;
 	fman_if_set_sg;
 	of_get_mac_address;
 
 	local: *;
 } DPDK_18.02;
+
+DPDK_18.11 {
+	global:
+	bman_thread_irq;
+	fman_if_get_sg_enable;
+	fman_if_set_sg;
+	qman_clear_irq;
+
+	qman_irqsource_add;
+	qman_irqsource_remove;
+	qman_thread_fd;
+	qman_thread_irq;
+
+	local: *;
+} DPDK_18.08;
diff --git a/drivers/bus/dpaa/rte_dpaa_bus.h b/drivers/bus/dpaa/rte_dpaa_bus.h
index 15dc6a4a..1d580a00 100644
--- a/drivers/bus/dpaa/rte_dpaa_bus.h
+++ b/drivers/bus/dpaa/rte_dpaa_bus.h
@@ -8,6 +8,7 @@
 
 #include <rte_bus.h>
 #include <rte_mempool.h>
+#include <dpaax_iova_table.h>
 
 #include <fsl_usd.h>
 #include <fsl_qman.h>
@@ -110,6 +111,11 @@ extern struct dpaa_memseg_list rte_dpaa_memsegs;
 static inline void *rte_dpaa_mem_ptov(phys_addr_t paddr)
 {
 	struct dpaa_memseg *ms;
+	void *va;
+
+	va = dpaax_iova_table_get_va(paddr);
+	if (likely(va != NULL))
+		return va;
 
 	/* Check if the address is already part of the memseg list internally
 	 * maintained by the dpaa driver.
diff --git a/drivers/bus/fslmc/Makefile b/drivers/bus/fslmc/Makefile
index 515d0f53..218d9bd2 100644
--- a/drivers/bus/fslmc/Makefile
+++ b/drivers/bus/fslmc/Makefile
@@ -19,12 +19,13 @@ CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include
 CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev
+LDLIBS += -lrte_common_dpaax
 
 # versioning export map
 EXPORT_MAP := rte_bus_fslmc_version.map
 
 # library version
-LIBABIVER := 1
+LIBABIVER := 2
 
 SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += \
         qbman/qbman_portal.c \
diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index d2900edc..89af9385 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -20,6 +20,8 @@
 #include <fslmc_vfio.h>
 #include "fslmc_logs.h"
 
+#include <dpaax_iova_table.h>
+
 int dpaa2_logtype_bus;
 
 #define VFIO_IOMMU_GROUP_PATH "/sys/kernel/iommu_groups"
@@ -161,6 +163,8 @@ scan_one_fslmc_device(char *dev_name)
 		return -ENOMEM;
 	}
 
+	dev->device.bus = &rte_fslmc_bus.bus;
+
 	/* Parse the device name and ID */
 	t_ptr = strtok(dup_dev_name, ".");
 	if (!t_ptr) {
@@ -375,6 +379,19 @@ rte_fslmc_probe(void)
 
 	probe_all = rte_fslmc_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST;
 
+	/* In case of PA, the FD addresses returned by qbman APIs are physical
+	 * addresses, which need conversion into equivalent VA address for
+	 * rte_mbuf. For that, a table (a serial array, in memory) is used to
+	 * increase translation efficiency.
+	 * This has to be done before probe as some device initialization
+	 * (during) probe allocate memory (dpaa2_sec) which needs to be pinned
+	 * to this table.
+	 *
+	 * Error is ignored as relevant logs are handled within dpaax and
+	 * handling for unavailable dpaax table too is transparent to caller.
+	 */
+	dpaax_iova_table_populate();
+
 	TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) {
 		TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
 			ret = rte_fslmc_match(drv, dev);
@@ -384,6 +401,9 @@ rte_fslmc_probe(void)
 			if (!drv->probe)
 				continue;
 
+			if (rte_dev_is_probed(&dev->device))
+				continue;
+
 			if (dev->device.devargs &&
 			  dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
 				DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
@@ -396,8 +416,12 @@ rte_fslmc_probe(void)
 			   dev->device.devargs->policy ==
 			   RTE_DEV_WHITELISTED)) {
 				ret = drv->probe(drv, dev);
-				if (ret)
+				if (ret) {
 					DPAA2_BUS_ERR("Unable to probe");
+				} else {
+					dev->driver = drv;
+					dev->device.driver = &drv->driver;
+				}
 			}
 			break;
 		}
@@ -450,6 +474,11 @@ rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver)
 
 	fslmc_bus = driver->fslmc_bus;
 
+	/* Cleanup the PA->VA Translation table; From whereever this function
+	 * is called from.
+	 */
+	dpaax_iova_table_depopulate();
+
 	TAILQ_REMOVE(&fslmc_bus->driver_list, driver, next);
 	/* Update Bus references */
 	driver->fslmc_bus = NULL;
@@ -490,6 +519,10 @@ rte_dpaa2_get_iommu_class(void)
 	if (TAILQ_EMPTY(&rte_fslmc_bus.device_list))
 		return RTE_IOVA_DC;
 
+#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
+	return RTE_IOVA_PA;
+#endif
+
 	/* check if all devices on the bus support Virtual addressing or not */
 	has_iova_va = fslmc_all_device_support_iova();
 
diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index 4c2cd2a8..493b6e5b 100644
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -221,6 +221,13 @@ fslmc_memevent_cb(enum rte_mem_event type, const void *addr, size_t len,
 					"alloc" : "dealloc",
 				va, virt_addr, iova_addr, map_len);
 
+		/* iova_addr may be set to RTE_BAD_IOVA */
+		if (iova_addr == RTE_BAD_IOVA) {
+			DPAA2_BUS_DEBUG("Segment has invalid iova, skipping\n");
+			cur_len += map_len;
+			continue;
+		}
+
 		if (type == RTE_MEM_EVENT_ALLOC)
 			ret = fslmc_map_dma(virt_addr, iova_addr, map_len);
 		else
@@ -318,11 +325,15 @@ fslmc_unmap_dma(uint64_t vaddr, uint64_t iovaddr __rte_unused, size_t len)
 
 static int
 fslmc_dmamap_seg(const struct rte_memseg_list *msl __rte_unused,
-		 const struct rte_memseg *ms, void *arg)
+		const struct rte_memseg *ms, void *arg)
 {
 	int *n_segs = arg;
 	int ret;
 
+	/* if IOVA address is invalid, skip */
+	if (ms->iova == RTE_BAD_IOVA)
+		return 0;
+
 	ret = fslmc_map_dma(ms->addr_64, ms->iova, ms->len);
 	if (ret)
 		DPAA2_BUS_ERR("Unable to VFIO map (addr=%p, len=%zu)",
diff --git a/drivers/bus/fslmc/mc/dpbp.c b/drivers/bus/fslmc/mc/dpbp.c
index 0215d22d..d9103409 100644
--- a/drivers/bus/fslmc/mc/dpbp.c
+++ b/drivers/bus/fslmc/mc/dpbp.c
@@ -248,6 +248,16 @@ int dpbp_reset(struct fsl_mc_io *mc_io,
 	/* send command to mc*/
 	return mc_send_command(mc_io, &cmd);
 }
+/**
+ * dpbp_get_attributes - Retrieve DPBP attributes.
+ *
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPBP object
+ * @attr:	Returned object's attributes
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
 int dpbp_get_attributes(struct fsl_mc_io *mc_io,
 			uint32_t cmd_flags,
 			uint16_t token,
diff --git a/drivers/bus/fslmc/mc/dpci.c b/drivers/bus/fslmc/mc/dpci.c
index ff366bfa..95edae9d 100644
--- a/drivers/bus/fslmc/mc/dpci.c
+++ b/drivers/bus/fslmc/mc/dpci.c
@@ -265,6 +265,15 @@ int dpci_reset(struct fsl_mc_io *mc_io,
 	return mc_send_command(mc_io, &cmd);
 }
 
+/**
+ * dpci_get_attributes() - Retrieve DPCI attributes.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPCI object
+ * @attr:	Returned object's attributes
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
 int dpci_get_attributes(struct fsl_mc_io *mc_io,
 			uint32_t cmd_flags,
 			uint16_t token,
@@ -292,6 +301,94 @@ int dpci_get_attributes(struct fsl_mc_io *mc_io,
 	return 0;
 }
 
+/**
+ * dpci_get_peer_attributes() - Retrieve peer DPCI attributes.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPCI object
+ * @attr:	Returned peer attributes
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dpci_get_peer_attributes(struct fsl_mc_io *mc_io,
+			     uint32_t cmd_flags,
+			     uint16_t token,
+			     struct dpci_peer_attr *attr)
+{
+	struct dpci_rsp_get_peer_attr *rsp_params;
+	struct mc_command cmd = { 0 };
+	int err;
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_PEER_ATTR,
+					  cmd_flags,
+					  token);
+
+	/* send command to mc*/
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	/* retrieve response parameters */
+	rsp_params = (struct dpci_rsp_get_peer_attr *)cmd.params;
+	attr->peer_id = le32_to_cpu(rsp_params->id);
+	attr->num_of_priorities = rsp_params->num_of_priorities;
+
+	return 0;
+}
+
+/**
+ * dpci_get_link_state() - Retrieve the DPCI link state.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPCI object
+ * @up:		Returned link state; returns '1' if link is up, '0' otherwise
+ *
+ * DPCI can be connected to another DPCI, together they
+ * create a 'link'. In order to use the DPCI Tx and Rx queues,
+ * both objects must be enabled.
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dpci_get_link_state(struct fsl_mc_io *mc_io,
+			uint32_t cmd_flags,
+			uint16_t token,
+			int *up)
+{
+	struct dpci_rsp_get_link_state *rsp_params;
+	struct mc_command cmd = { 0 };
+	int err;
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_LINK_STATE,
+					  cmd_flags,
+					  token);
+
+	/* send command to mc*/
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	/* retrieve response parameters */
+	rsp_params = (struct dpci_rsp_get_link_state *)cmd.params;
+	*up = dpci_get_field(rsp_params->up, UP);
+
+	return 0;
+}
+
+/**
+ * dpci_set_rx_queue() - Set Rx queue configuration
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPCI object
+ * @priority:	Select the queue relative to number of
+ *			priorities configured at DPCI creation; use
+ *			DPCI_ALL_QUEUES to configure all Rx queues
+ *			identically.
+ * @cfg:	Rx queue configuration
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
 int dpci_set_rx_queue(struct fsl_mc_io *mc_io,
 		      uint32_t cmd_flags,
 		      uint16_t token,
@@ -314,6 +411,9 @@ int dpci_set_rx_queue(struct fsl_mc_io *mc_io,
 	dpci_set_field(cmd_params->dest_type,
 		       DEST_TYPE,
 		       cfg->dest_cfg.dest_type);
+	dpci_set_field(cmd_params->dest_type,
+		       ORDER_PRESERVATION,
+		       cfg->order_preservation_en);
 
 	/* send command to mc*/
 	return mc_send_command(mc_io, &cmd);
@@ -438,3 +538,100 @@ int dpci_get_api_version(struct fsl_mc_io *mc_io,
 
 	return 0;
 }
+
+/**
+ * dpci_set_opr() - Set Order Restoration configuration.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPCI object
+ * @index:	The queue index
+ * @options:	Configuration mode options
+ *		can be OPR_OPT_CREATE or OPR_OPT_RETIRE
+ * @cfg:	Configuration options for the OPR
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dpci_set_opr(struct fsl_mc_io *mc_io,
+		 uint32_t cmd_flags,
+		 uint16_t token,
+		 uint8_t index,
+		 uint8_t options,
+		 struct opr_cfg *cfg)
+{
+	struct dpci_cmd_set_opr *cmd_params;
+	struct mc_command cmd = { 0 };
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPCI_CMDID_SET_OPR,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dpci_cmd_set_opr *)cmd.params;
+	cmd_params->index = index;
+	cmd_params->options = options;
+	cmd_params->oloe = cfg->oloe;
+	cmd_params->oeane = cfg->oeane;
+	cmd_params->olws = cfg->olws;
+	cmd_params->oa = cfg->oa;
+	cmd_params->oprrws = cfg->oprrws;
+
+	/* send command to mc*/
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dpci_get_opr() - Retrieve Order Restoration config and query.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPCI object
+ * @index:	The queue index
+ * @cfg:	Returned OPR configuration
+ * @qry:	Returned OPR query
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dpci_get_opr(struct fsl_mc_io *mc_io,
+		 uint32_t cmd_flags,
+		 uint16_t token,
+		 uint8_t index,
+		 struct opr_cfg *cfg,
+		 struct opr_qry *qry)
+{
+	struct dpci_rsp_get_opr *rsp_params;
+	struct dpci_cmd_get_opr *cmd_params;
+	struct mc_command cmd = { 0 };
+	int err;
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_OPR,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dpci_cmd_get_opr *)cmd.params;
+	cmd_params->index = index;
+
+	/* send command to mc*/
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	/* retrieve response parameters */
+	rsp_params = (struct dpci_rsp_get_opr *)cmd.params;
+	cfg->oloe = rsp_params->oloe;
+	cfg->oeane = rsp_params->oeane;
+	cfg->olws = rsp_params->olws;
+	cfg->oa = rsp_params->oa;
+	cfg->oprrws = rsp_params->oprrws;
+	qry->rip = dpci_get_field(rsp_params->flags, RIP);
+	qry->enable = dpci_get_field(rsp_params->flags, OPR_ENABLE);
+	qry->nesn = le16_to_cpu(rsp_params->nesn);
+	qry->ndsn = le16_to_cpu(rsp_params->ndsn);
+	qry->ea_tseq = le16_to_cpu(rsp_params->ea_tseq);
+	qry->tseq_nlis = dpci_get_field(rsp_params->tseq_nlis, TSEQ_NLIS);
+	qry->ea_hseq = le16_to_cpu(rsp_params->ea_hseq);
+	qry->hseq_nlis = dpci_get_field(rsp_params->hseq_nlis, HSEQ_NLIS);
+	qry->ea_hptr = le16_to_cpu(rsp_params->ea_hptr);
+	qry->ea_tptr = le16_to_cpu(rsp_params->ea_tptr);
+	qry->opr_vid = le16_to_cpu(rsp_params->opr_vid);
+	qry->opr_id = le16_to_cpu(rsp_params->opr_id);
+
+	return 0;
+}
diff --git a/drivers/bus/fslmc/mc/dpcon.c b/drivers/bus/fslmc/mc/dpcon.c
index 3f6e04b9..92bd2651 100644
--- a/drivers/bus/fslmc/mc/dpcon.c
+++ b/drivers/bus/fslmc/mc/dpcon.c
@@ -296,6 +296,36 @@ int dpcon_get_attributes(struct fsl_mc_io *mc_io,
 }
 
 /**
+ * dpcon_set_notification() - Set DPCON notification destination
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPCON object
+ * @cfg:	Notification parameters
+ *
+ * Return:	'0' on Success; Error code otherwise
+ */
+int dpcon_set_notification(struct fsl_mc_io *mc_io,
+			   uint32_t cmd_flags,
+			   uint16_t token,
+			   struct dpcon_notification_cfg *cfg)
+{
+	struct dpcon_cmd_set_notification *dpcon_cmd;
+	struct mc_command cmd = { 0 };
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPCON_CMDID_SET_NOTIFICATION,
+					  cmd_flags,
+					  token);
+	dpcon_cmd = (struct dpcon_cmd_set_notification *)cmd.params;
+	dpcon_cmd->dpio_id = cpu_to_le32(cfg->dpio_id);
+	dpcon_cmd->priority = cfg->priority;
+	dpcon_cmd->user_ctx = cpu_to_le64(cfg->user_ctx);
+
+	/* send command to mc*/
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
  * dpcon_get_api_version - Get Data Path Concentrator API version
  * @mc_io:	Pointer to MC portal's DPCON object
  * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
diff --git a/drivers/bus/fslmc/mc/dpdmai.c b/drivers/bus/fslmc/mc/dpdmai.c
index 528889df..dcb9d516 100644
--- a/drivers/bus/fslmc/mc/dpdmai.c
+++ b/drivers/bus/fslmc/mc/dpdmai.c
@@ -113,6 +113,7 @@ int dpdmai_create(struct fsl_mc_io *mc_io,
 					  cmd_flags,
 					  dprc_token);
 	cmd_params = (struct dpdmai_cmd_create *)cmd.params;
+	cmd_params->num_queues = cfg->num_queues;
 	cmd_params->priorities[0] = cfg->priorities[0];
 	cmd_params->priorities[1] = cfg->priorities[1];
 
@@ -297,6 +298,7 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io,
 	rsp_params = (struct dpdmai_rsp_get_attr *)cmd.params;
 	attr->id = le32_to_cpu(rsp_params->id);
 	attr->num_of_priorities = rsp_params->num_of_priorities;
+	attr->num_of_queues = rsp_params->num_of_queues;
 
 	return 0;
 }
@@ -306,6 +308,8 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io,
  * @mc_io:	Pointer to MC portal's I/O object
  * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
  * @token:	Token of DPDMAI object
+ * @queue_idx: Rx queue index. Accepted values are form 0 to num_queues
+ *		parameter provided in dpdmai_create
  * @priority:	Select the queue relative to number of
  *		priorities configured at DPDMAI creation; use
  *		DPDMAI_ALL_QUEUES to configure all Rx queues
@@ -317,6 +321,7 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io,
 int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
 			uint32_t cmd_flags,
 			uint16_t token,
+			uint8_t queue_idx,
 			uint8_t priority,
 			const struct dpdmai_rx_queue_cfg *cfg)
 {
@@ -331,6 +336,7 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
 	cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id);
 	cmd_params->dest_priority = cfg->dest_cfg.priority;
 	cmd_params->priority = priority;
+	cmd_params->queue_idx = queue_idx;
 	cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx);
 	cmd_params->options = cpu_to_le32(cfg->options);
 	dpdmai_set_field(cmd_params->dest_type,
@@ -346,6 +352,8 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
  * @mc_io:	Pointer to MC portal's I/O object
  * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
  * @token:	Token of DPDMAI object
+ * @queue_idx: Rx queue index. Accepted values are form 0 to num_queues
+ *		parameter provided in dpdmai_create
  * @priority:	Select the queue relative to number of
  *		priorities configured at DPDMAI creation
  * @attr:	Returned Rx queue attributes
@@ -355,6 +363,7 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
 int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
 			uint32_t cmd_flags,
 			uint16_t token,
+			uint8_t queue_idx,
 			uint8_t priority,
 			struct dpdmai_rx_queue_attr *attr)
 {
@@ -369,6 +378,7 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
 					  token);
 	cmd_params = (struct dpdmai_cmd_get_queue *)cmd.params;
 	cmd_params->priority = priority;
+	cmd_params->queue_idx = queue_idx;
 
 	/* send command to mc*/
 	err = mc_send_command(mc_io, &cmd);
@@ -392,6 +402,8 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
  * @mc_io:	Pointer to MC portal's I/O object
  * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
  * @token:	Token of DPDMAI object
+ * @queue_idx: Tx queue index. Accepted values are form 0 to num_queues
+ *		parameter provided in dpdmai_create
  * @priority:	Select the queue relative to number of
  *		priorities configured at DPDMAI creation
  * @attr:	Returned Tx queue attributes
@@ -401,6 +413,7 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
 int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io,
 			uint32_t cmd_flags,
 			uint16_t token,
+			uint8_t queue_idx,
 			uint8_t priority,
 			struct dpdmai_tx_queue_attr *attr)
 {
@@ -415,6 +428,7 @@ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io,
 					  token);
 	cmd_params = (struct dpdmai_cmd_get_queue *)cmd.params;
 	cmd_params->priority = priority;
+	cmd_params->queue_idx = queue_idx;
 
 	/* send command to mc*/
 	err = mc_send_command(mc_io, &cmd);
diff --git a/drivers/bus/fslmc/mc/dpio.c b/drivers/bus/fslmc/mc/dpio.c
index 966277cc..a3382ed1 100644
--- a/drivers/bus/fslmc/mc/dpio.c
+++ b/drivers/bus/fslmc/mc/dpio.c
@@ -268,6 +268,15 @@ int dpio_reset(struct fsl_mc_io *mc_io,
 	return mc_send_command(mc_io, &cmd);
 }
 
+/**
+ * dpio_get_attributes() - Retrieve DPIO attributes
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPIO object
+ * @attr:	Returned object's attributes
+ *
+ * Return:	'0' on Success; Error code otherwise
+ */
 int dpio_get_attributes(struct fsl_mc_io *mc_io,
 			uint32_t cmd_flags,
 			uint16_t token,
diff --git a/drivers/bus/fslmc/mc/fsl_dpbp.h b/drivers/bus/fslmc/mc/fsl_dpbp.h
index 11183626..9d405b42 100644
--- a/drivers/bus/fslmc/mc/fsl_dpbp.h
+++ b/drivers/bus/fslmc/mc/fsl_dpbp.h
@@ -82,6 +82,7 @@ int dpbp_get_attributes(struct fsl_mc_io *mc_io,
 /**
  * BPSCN write will attempt to allocate into a cache (coherent write)
  */
+#define DPBP_NOTIF_OPT_COHERENT_WRITE	0x00000001
 int dpbp_get_api_version(struct fsl_mc_io *mc_io,
 			 uint32_t cmd_flags,
 			 uint16_t *major_ver,
diff --git a/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h b/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h
index 18402ced..55c9fc9b 100644
--- a/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h
+++ b/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h
@@ -9,13 +9,15 @@
 
 /* DPBP Version */
 #define DPBP_VER_MAJOR				3
-#define DPBP_VER_MINOR				3
+#define DPBP_VER_MINOR				4
 
 /* Command versioning */
 #define DPBP_CMD_BASE_VERSION			1
+#define DPBP_CMD_VERSION_2			2
 #define DPBP_CMD_ID_OFFSET			4
 
 #define DPBP_CMD(id)	((id << DPBP_CMD_ID_OFFSET) | DPBP_CMD_BASE_VERSION)
+#define DPBP_CMD_V2(id)	((id << DPBP_CMD_ID_OFFSET) | DPBP_CMD_VERSION_2)
 
 /* Command IDs */
 #define DPBP_CMDID_CLOSE		DPBP_CMD(0x800)
@@ -37,8 +39,8 @@
 #define DPBP_CMDID_GET_IRQ_STATUS	DPBP_CMD(0x016)
 #define DPBP_CMDID_CLEAR_IRQ_STATUS	DPBP_CMD(0x017)
 
-#define DPBP_CMDID_SET_NOTIFICATIONS	DPBP_CMD(0x1b0)
-#define DPBP_CMDID_GET_NOTIFICATIONS	DPBP_CMD(0x1b1)
+#define DPBP_CMDID_SET_NOTIFICATIONS	DPBP_CMD_V2(0x1b0)
+#define DPBP_CMDID_GET_NOTIFICATIONS	DPBP_CMD_V2(0x1b1)
 
 #define DPBP_CMDID_GET_FREE_BUFFERS_NUM	DPBP_CMD(0x1b2)
 
@@ -68,8 +70,8 @@ struct dpbp_cmd_set_notifications {
 	uint32_t depletion_exit;
 	uint32_t surplus_entry;
 	uint32_t surplus_exit;
-	uint16_t options;
-	uint16_t pad[3];
+	uint32_t options;
+	uint16_t pad[2];
 	uint64_t message_ctx;
 	uint64_t message_iova;
 };
@@ -79,8 +81,8 @@ struct dpbp_rsp_get_notifications {
 	uint32_t depletion_exit;
 	uint32_t surplus_entry;
 	uint32_t surplus_exit;
-	uint16_t options;
-	uint16_t pad[3];
+	uint32_t options;
+	uint16_t pad[2];
 	uint64_t message_ctx;
 	uint64_t message_iova;
 };
diff --git a/drivers/bus/fslmc/mc/fsl_dpci.h b/drivers/bus/fslmc/mc/fsl_dpci.h
index f69ed3f3..9af9097e 100644
--- a/drivers/bus/fslmc/mc/fsl_dpci.h
+++ b/drivers/bus/fslmc/mc/fsl_dpci.h
@@ -6,6 +6,8 @@
 #ifndef __FSL_DPCI_H
 #define __FSL_DPCI_H
 
+#include <fsl_dpopr.h>
+
 /* Data Path Communication Interface API
  * Contains initialization APIs and runtime control APIs for DPCI
  */
@@ -17,7 +19,7 @@ struct fsl_mc_io;
 /**
  * Maximum number of Tx/Rx priorities per DPCI object
  */
-#define DPCI_PRIO_NUM		2
+#define DPCI_PRIO_NUM		4
 
 /**
  * Indicates an invalid frame queue
@@ -107,6 +109,27 @@ int dpci_get_attributes(struct fsl_mc_io *mc_io,
 			struct dpci_attr *attr);
 
 /**
+ * struct dpci_peer_attr - Structure representing the peer DPCI attributes
+ * @peer_id:		DPCI peer id; if no peer is connected returns (-1)
+ * @num_of_priorities:	The pper's number of receive priorities; determines the
+ *			number of transmit priorities for the local DPCI object
+ */
+struct dpci_peer_attr {
+	int peer_id;
+	uint8_t num_of_priorities;
+};
+
+int dpci_get_peer_attributes(struct fsl_mc_io *mc_io,
+			     uint32_t cmd_flags,
+			     uint16_t token,
+			     struct dpci_peer_attr *attr);
+
+int dpci_get_link_state(struct fsl_mc_io *mc_io,
+			uint32_t cmd_flags,
+			uint16_t token,
+			int *up);
+
+/**
  * enum dpci_dest - DPCI destination types
  * @DPCI_DEST_NONE:	Unassigned destination; The queue is set in parked mode
  *			and does not generate FQDAN notifications; user is
@@ -154,6 +177,11 @@ struct dpci_dest_cfg {
 #define DPCI_QUEUE_OPT_DEST		0x00000002
 
 /**
+ * Set the queue to hold active mode.
+ */
+#define DPCI_QUEUE_OPT_HOLD_ACTIVE	0x00000004
+
+/**
  * struct dpci_rx_queue_cfg - Structure representing RX queue configuration
  * @options:	Flags representing the suggested modifications to the queue;
  *		Use any combination of 'DPCI_QUEUE_OPT_<X>' flags
@@ -163,11 +191,14 @@ struct dpci_dest_cfg {
  *		'options'
  * @dest_cfg:	Queue destination parameters;
  *		valid only if 'DPCI_QUEUE_OPT_DEST' is contained in 'options'
+ * @order_preservation_en: order preservation configuration for the rx queue
+ * valid only if 'DPCI_QUEUE_OPT_HOLD_ACTIVE' is contained in 'options'
  */
 struct dpci_rx_queue_cfg {
 	uint32_t options;
 	uint64_t user_ctx;
 	struct dpci_dest_cfg dest_cfg;
+	int order_preservation_en;
 };
 
 int dpci_set_rx_queue(struct fsl_mc_io *mc_io,
@@ -217,4 +248,18 @@ int dpci_get_api_version(struct fsl_mc_io *mc_io,
 			 uint16_t *major_ver,
 			 uint16_t *minor_ver);
 
+int dpci_set_opr(struct fsl_mc_io *mc_io,
+		 uint32_t cmd_flags,
+		 uint16_t token,
+		 uint8_t index,
+		 uint8_t options,
+		 struct opr_cfg *cfg);
+
+int dpci_get_opr(struct fsl_mc_io *mc_io,
+		 uint32_t cmd_flags,
+		 uint16_t token,
+		 uint8_t index,
+		 struct opr_cfg *cfg,
+		 struct opr_qry *qry);
+
 #endif /* __FSL_DPCI_H */
diff --git a/drivers/bus/fslmc/mc/fsl_dpci_cmd.h b/drivers/bus/fslmc/mc/fsl_dpci_cmd.h
index 634248ac..92b85a82 100644
--- a/drivers/bus/fslmc/mc/fsl_dpci_cmd.h
+++ b/drivers/bus/fslmc/mc/fsl_dpci_cmd.h
@@ -8,7 +8,7 @@
 
 /* DPCI Version */
 #define DPCI_VER_MAJOR			3
-#define DPCI_VER_MINOR			3
+#define DPCI_VER_MINOR			4
 
 #define DPCI_CMD_BASE_VERSION		1
 #define DPCI_CMD_BASE_VERSION_V2	2
@@ -35,6 +35,8 @@
 #define DPCI_CMDID_GET_PEER_ATTR	DPCI_CMD_V1(0x0e2)
 #define DPCI_CMDID_GET_RX_QUEUE		DPCI_CMD_V1(0x0e3)
 #define DPCI_CMDID_GET_TX_QUEUE		DPCI_CMD_V1(0x0e4)
+#define DPCI_CMDID_SET_OPR		DPCI_CMD_V1(0x0e5)
+#define DPCI_CMDID_GET_OPR		DPCI_CMD_V1(0x0e6)
 
 /* Macros for accessing command fields smaller than 1byte */
 #define DPCI_MASK(field)        \
@@ -90,6 +92,8 @@ struct dpci_rsp_get_link_state {
 
 #define DPCI_DEST_TYPE_SHIFT	0
 #define DPCI_DEST_TYPE_SIZE	4
+#define DPCI_ORDER_PRESERVATION_SHIFT	4
+#define DPCI_ORDER_PRESERVATION_SIZE	1
 
 struct dpci_cmd_set_rx_queue {
 	uint32_t dest_id;
@@ -128,5 +132,61 @@ struct dpci_rsp_get_api_version {
 	uint16_t minor;
 };
 
+struct dpci_cmd_set_opr {
+	uint16_t pad0;
+	uint8_t index;
+	uint8_t options;
+	uint8_t pad1[7];
+	uint8_t oloe;
+	uint8_t oeane;
+	uint8_t olws;
+	uint8_t oa;
+	uint8_t oprrws;
+};
+
+struct dpci_cmd_get_opr {
+	uint16_t pad;
+	uint8_t index;
+};
+
+#define DPCI_RIP_SHIFT		0
+#define DPCI_RIP_SIZE		1
+#define DPCI_OPR_ENABLE_SHIFT	1
+#define DPCI_OPR_ENABLE_SIZE	1
+#define DPCI_TSEQ_NLIS_SHIFT	0
+#define DPCI_TSEQ_NLIS_SIZE	1
+#define DPCI_HSEQ_NLIS_SHIFT	0
+#define DPCI_HSEQ_NLIS_SIZE	1
+
+struct dpci_rsp_get_opr {
+	uint64_t pad0;
+	/* from LSB: rip:1 enable:1 */
+	uint8_t flags;
+	uint16_t pad1;
+	uint8_t oloe;
+	uint8_t oeane;
+	uint8_t olws;
+	uint8_t oa;
+	uint8_t oprrws;
+	uint16_t nesn;
+	uint16_t pad8;
+	uint16_t ndsn;
+	uint16_t pad2;
+	uint16_t ea_tseq;
+	/* only the LSB */
+	uint8_t tseq_nlis;
+	uint8_t pad3;
+	uint16_t ea_hseq;
+	/* only the LSB */
+	uint8_t hseq_nlis;
+	uint8_t pad4;
+	uint16_t ea_hptr;
+	uint16_t pad5;
+	uint16_t ea_tptr;
+	uint16_t pad6;
+	uint16_t opr_vid;
+	uint16_t pad7;
+	uint16_t opr_id;
+};
 #pragma pack(pop)
 #endif /* _FSL_DPCI_CMD_H */
diff --git a/drivers/bus/fslmc/mc/fsl_dpcon.h b/drivers/bus/fslmc/mc/fsl_dpcon.h
index 36dd5f3c..fc0430dc 100644
--- a/drivers/bus/fslmc/mc/fsl_dpcon.h
+++ b/drivers/bus/fslmc/mc/fsl_dpcon.h
@@ -81,6 +81,25 @@ int dpcon_get_attributes(struct fsl_mc_io *mc_io,
 			 uint16_t token,
 			 struct dpcon_attr *attr);
 
+/**
+ * struct dpcon_notification_cfg - Structure representing notification params
+ * @dpio_id:	DPIO object ID; must be configured with a notification channel;
+ *		to disable notifications set it to 'DPCON_INVALID_DPIO_ID';
+ * @priority:	Priority selection within the DPIO channel; valid values
+ *		are 0-7, depending on the number of priorities in that channel
+ * @user_ctx:	User context value provided with each CDAN message
+ */
+struct dpcon_notification_cfg {
+	int dpio_id;
+	uint8_t priority;
+	uint64_t user_ctx;
+};
+
+int dpcon_set_notification(struct fsl_mc_io *mc_io,
+			   uint32_t cmd_flags,
+			   uint16_t token,
+			   struct dpcon_notification_cfg *cfg);
+
 int dpcon_get_api_version(struct fsl_mc_io *mc_io,
 			  uint32_t cmd_flags,
 			  uint16_t *major_ver,
diff --git a/drivers/bus/fslmc/mc/fsl_dpdmai.h b/drivers/bus/fslmc/mc/fsl_dpdmai.h
index 03e46ec1..40469cc1 100644
--- a/drivers/bus/fslmc/mc/fsl_dpdmai.h
+++ b/drivers/bus/fslmc/mc/fsl_dpdmai.h
@@ -39,6 +39,7 @@ int dpdmai_close(struct fsl_mc_io *mc_io,
  *	should be configured with 0
  */
 struct dpdmai_cfg {
+	uint8_t num_queues;
 	uint8_t priorities[DPDMAI_PRIO_NUM];
 };
 
@@ -78,6 +79,7 @@ int dpdmai_reset(struct fsl_mc_io *mc_io,
 struct dpdmai_attr {
 	int id;
 	uint8_t num_of_priorities;
+	uint8_t num_of_queues;
 };
 
 int dpdmai_get_attributes(struct fsl_mc_io *mc_io,
@@ -149,6 +151,7 @@ struct dpdmai_rx_queue_cfg {
 int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
 			uint32_t cmd_flags,
 			uint16_t token,
+			uint8_t queue_idx,
 			uint8_t priority,
 			const struct dpdmai_rx_queue_cfg *cfg);
 
@@ -168,6 +171,7 @@ struct dpdmai_rx_queue_attr {
 int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
 			uint32_t cmd_flags,
 			uint16_t token,
+			uint8_t queue_idx,
 			uint8_t priority,
 			struct dpdmai_rx_queue_attr *attr);
 
@@ -183,6 +187,7 @@ struct dpdmai_tx_queue_attr {
 int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io,
 			uint32_t cmd_flags,
 			uint16_t token,
+			uint8_t queue_idx,
 			uint8_t priority,
 			struct dpdmai_tx_queue_attr *attr);
 
diff --git a/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h b/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h
index 618e19ea..7e122de4 100644
--- a/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h
+++ b/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h
@@ -7,30 +7,32 @@
 
 /* DPDMAI Version */
 #define DPDMAI_VER_MAJOR		3
-#define DPDMAI_VER_MINOR		2
+#define DPDMAI_VER_MINOR		3
 
 /* Command versioning */
 #define DPDMAI_CMD_BASE_VERSION		1
+#define DPDMAI_CMD_VERSION_2		2
 #define DPDMAI_CMD_ID_OFFSET		4
 
 #define DPDMAI_CMD(id)	((id << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
+#define DPDMAI_CMD_V2(id) ((id << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_VERSION_2)
 
 /* Command IDs */
 #define DPDMAI_CMDID_CLOSE		DPDMAI_CMD(0x800)
 #define DPDMAI_CMDID_OPEN		DPDMAI_CMD(0x80E)
-#define DPDMAI_CMDID_CREATE		DPDMAI_CMD(0x90E)
+#define DPDMAI_CMDID_CREATE		DPDMAI_CMD_V2(0x90E)
 #define DPDMAI_CMDID_DESTROY		DPDMAI_CMD(0x98E)
 #define DPDMAI_CMDID_GET_API_VERSION	DPDMAI_CMD(0xa0E)
 
 #define DPDMAI_CMDID_ENABLE		DPDMAI_CMD(0x002)
 #define DPDMAI_CMDID_DISABLE		DPDMAI_CMD(0x003)
-#define DPDMAI_CMDID_GET_ATTR		DPDMAI_CMD(0x004)
+#define DPDMAI_CMDID_GET_ATTR		DPDMAI_CMD_V2(0x004)
 #define DPDMAI_CMDID_RESET		DPDMAI_CMD(0x005)
 #define DPDMAI_CMDID_IS_ENABLED		DPDMAI_CMD(0x006)
 
-#define DPDMAI_CMDID_SET_RX_QUEUE	DPDMAI_CMD(0x1A0)
-#define DPDMAI_CMDID_GET_RX_QUEUE	DPDMAI_CMD(0x1A1)
-#define DPDMAI_CMDID_GET_TX_QUEUE	DPDMAI_CMD(0x1A2)
+#define DPDMAI_CMDID_SET_RX_QUEUE	DPDMAI_CMD_V2(0x1A0)
+#define DPDMAI_CMDID_GET_RX_QUEUE	DPDMAI_CMD_V2(0x1A1)
+#define DPDMAI_CMDID_GET_TX_QUEUE	DPDMAI_CMD_V2(0x1A2)
 
 /* Macros for accessing command fields smaller than 1byte */
 #define DPDMAI_MASK(field)        \
@@ -47,7 +49,7 @@ struct dpdmai_cmd_open {
 };
 
 struct dpdmai_cmd_create {
-	uint8_t pad;
+	uint8_t num_queues;
 	uint8_t priorities[2];
 };
 
@@ -66,6 +68,7 @@ struct dpdmai_rsp_is_enabled {
 struct dpdmai_rsp_get_attr {
 	uint32_t id;
 	uint8_t num_of_priorities;
+	uint8_t num_of_queues;
 };
 
 #define DPDMAI_DEST_TYPE_SHIFT	0
@@ -77,7 +80,7 @@ struct dpdmai_cmd_set_rx_queue {
 	uint8_t priority;
 	/* from LSB: dest_type:4 */
 	uint8_t dest_type;
-	uint8_t pad;
+	uint8_t queue_idx;
 	uint64_t user_ctx;
 	uint32_t options;
 };
@@ -85,6 +88,7 @@ struct dpdmai_cmd_set_rx_queue {
 struct dpdmai_cmd_get_queue {
 	uint8_t pad[5];
 	uint8_t priority;
+	uint8_t queue_idx;
 };
 
 struct dpdmai_rsp_get_rx_queue {
diff --git a/drivers/bus/fslmc/mc/fsl_dpmng.h b/drivers/bus/fslmc/mc/fsl_dpmng.h
index afaf9b71..8559bef8 100644
--- a/drivers/bus/fslmc/mc/fsl_dpmng.h
+++ b/drivers/bus/fslmc/mc/fsl_dpmng.h
@@ -18,7 +18,7 @@ struct fsl_mc_io;
  * Management Complex firmware version information
  */
 #define MC_VER_MAJOR 10
-#define MC_VER_MINOR 3
+#define MC_VER_MINOR 10
 
 /**
  * struct mc_version
diff --git a/drivers/bus/fslmc/mc/fsl_dpopr.h b/drivers/bus/fslmc/mc/fsl_dpopr.h
new file mode 100644
index 00000000..fd727e01
--- /dev/null
+++ b/drivers/bus/fslmc/mc/fsl_dpopr.h
@@ -0,0 +1,85 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+ *
+ * Copyright 2013-2015 Freescale Semiconductor Inc.
+ * Copyright 2018 NXP
+ *
+ */
+#ifndef __FSL_DPOPR_H_
+#define __FSL_DPOPR_H_
+
+/** @addtogroup dpopr Data Path Order Restoration API
+ * Contains initialization APIs and runtime APIs for the Order Restoration
+ * @{
+ */
+
+/** Order Restoration properties */
+
+/**
+ * Create a new Order Point Record option
+ */
+#define OPR_OPT_CREATE 0x1
+/**
+ * Retire an existing Order Point Record option
+ */
+#define OPR_OPT_RETIRE 0x2
+
+/**
+ * struct opr_cfg - Structure representing OPR configuration
+ * @oprrws: Order point record (OPR) restoration window size (0 to 5)
+ *			0 - Window size is 32 frames.
+ *			1 - Window size is 64 frames.
+ *			2 - Window size is 128 frames.
+ *			3 - Window size is 256 frames.
+ *			4 - Window size is 512 frames.
+ *			5 - Window size is 1024 frames.
+ *@oa: OPR auto advance NESN window size (0 disabled, 1 enabled)
+ *@olws: OPR acceptable late arrival window size (0 to 3)
+ *			0 - Disabled. Late arrivals are always rejected.
+ *			1 - Window size is 32 frames.
+ *			2 - Window size is the same as the OPR restoration
+ *			window size configured in the OPRRWS field.
+ *			3 - Window size is 8192 frames.
+ *			Late arrivals are always accepted.
+ *@oeane: Order restoration list (ORL) resource exhaustion
+ *			advance NESN enable (0 disabled, 1 enabled)
+ *@oloe: OPR loose ordering enable (0 disabled, 1 enabled)
+ */
+struct opr_cfg {
+	uint8_t oprrws;
+	uint8_t oa;
+	uint8_t olws;
+	uint8_t oeane;
+	uint8_t oloe;
+};
+
+/**
+ * struct opr_qry - Structure representing OPR configuration
+ * @enable: Enabled state
+ * @rip: Retirement In Progress
+ * @ndsn: Next dispensed sequence number
+ * @nesn: Next expected sequence number
+ * @ea_hseq: Early arrival head sequence number
+ * @hseq_nlis: HSEQ not last in sequence
+ * @ea_tseq: Early arrival tail sequence number
+ * @tseq_nlis: TSEQ not last in sequence
+ * @ea_tptr: Early arrival tail pointer
+ * @ea_hptr: Early arrival head pointer
+ * @opr_id: Order Point Record ID
+ * @opr_vid: Order Point Record Virtual ID
+ */
+struct opr_qry {
+	char enable;
+	char rip;
+	uint16_t ndsn;
+	uint16_t nesn;
+	uint16_t ea_hseq;
+	char hseq_nlis;
+	uint16_t ea_tseq;
+	char tseq_nlis;
+	uint16_t ea_tptr;
+	uint16_t ea_hptr;
+	uint16_t opr_id;
+	uint16_t opr_vid;
+};
+
+#endif /* __FSL_DPOPR_H_ */
diff --git a/drivers/bus/fslmc/meson.build b/drivers/bus/fslmc/meson.build
index 22a56a6f..4b052157 100644
--- a/drivers/bus/fslmc/meson.build
+++ b/drivers/bus/fslmc/meson.build
@@ -1,11 +1,13 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright 2018 NXP
 
+version = 2
+
 if host_machine.system() != 'linux'
         build = false
 endif
 
-deps += ['eventdev', 'kvargs']
+deps += ['common_dpaax', 'eventdev', 'kvargs']
 sources = files('fslmc_bus.c',
 		'fslmc_vfio.c',
 		'mc/dpbp.c',
@@ -24,4 +26,3 @@ sources = files('fslmc_bus.c',
 allow_experimental_apis = true
 
 includes += include_directories('mc', 'qbman/include', 'portal')
-cflags += ['-D_GNU_SOURCE']
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
index 39c5adf9..db49d637 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
@@ -28,6 +28,13 @@
 #include "portal/dpaa2_hw_pvt.h"
 #include "portal/dpaa2_hw_dpio.h"
 
+/* List of all the memseg information locally maintained in dpaa2 driver. This
+ * is to optimize the PA_to_VA searches until a better mechanism (algo) is
+ * available.
+ */
+struct dpaa2_memseg_list rte_dpaa2_memsegs
+	= TAILQ_HEAD_INITIALIZER(rte_dpaa2_memsegs);
+
 TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
 static struct dpbp_dev_list dpbp_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
index 99f70be1..ce069984 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016 NXP
+ *   Copyright 2016-2018 NXP
  *
  */
 #include <unistd.h>
@@ -53,6 +53,11 @@ static uint32_t io_space_count;
 /* Variable to store DPAA2 platform type */
 uint32_t dpaa2_svr_family;
 
+/* Variable to store DPAA2 DQRR size */
+uint8_t dpaa2_dqrr_size;
+/* Variable to store DPAA2 EQCR size */
+uint8_t dpaa2_eqcr_size;
+
 /*Stashing Macros default for LS208x*/
 static int dpaa2_core_cluster_base = 0x04;
 static int dpaa2_cluster_sz = 2;
@@ -125,7 +130,7 @@ static void dpaa2_affine_dpio_intr_to_respective_core(int32_t dpio_id)
 		 cpu_mask, token);
 	ret = system(command);
 	if (ret < 0)
-		DPAA2_BUS_WARN(
+		DPAA2_BUS_DEBUG(
 			"Failed to affine interrupts on respective core");
 	else
 		DPAA2_BUS_DEBUG(" %s command is executed", command);
@@ -178,68 +183,6 @@ static int dpaa2_dpio_intr_init(struct dpaa2_dpio_dev *dpio_dev)
 #endif
 
 static int
-configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev)
-{
-	struct qbman_swp_desc p_des;
-	struct dpio_attr attr;
-
-	dpio_dev->dpio = malloc(sizeof(struct fsl_mc_io));
-	if (!dpio_dev->dpio) {
-		DPAA2_BUS_ERR("Memory allocation failure");
-		return -1;
-	}
-
-	dpio_dev->dpio->regs = dpio_dev->mc_portal;
-	if (dpio_open(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->hw_id,
-		      &dpio_dev->token)) {
-		DPAA2_BUS_ERR("Failed to allocate IO space");
-		free(dpio_dev->dpio);
-		return -1;
-	}
-
-	if (dpio_reset(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) {
-		DPAA2_BUS_ERR("Failed to reset dpio");
-		dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token);
-		free(dpio_dev->dpio);
-		return -1;
-	}
-
-	if (dpio_enable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) {
-		DPAA2_BUS_ERR("Failed to Enable dpio");
-		dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token);
-		free(dpio_dev->dpio);
-		return -1;
-	}
-
-	if (dpio_get_attributes(dpio_dev->dpio, CMD_PRI_LOW,
-				dpio_dev->token, &attr)) {
-		DPAA2_BUS_ERR("DPIO Get attribute failed");
-		dpio_disable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token);
-		dpio_close(dpio_dev->dpio, CMD_PRI_LOW,  dpio_dev->token);
-		free(dpio_dev->dpio);
-		return -1;
-	}
-
-	/* Configure & setup SW portal */
-	p_des.block = NULL;
-	p_des.idx = attr.qbman_portal_id;
-	p_des.cena_bar = (void *)(dpio_dev->qbman_portal_ce_paddr);
-	p_des.cinh_bar = (void *)(dpio_dev->qbman_portal_ci_paddr);
-	p_des.irq = -1;
-	p_des.qman_version = attr.qbman_version;
-
-	dpio_dev->sw_portal = qbman_swp_init(&p_des);
-	if (dpio_dev->sw_portal == NULL) {
-		DPAA2_BUS_ERR("QBMan SW Portal Init failed");
-		dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token);
-		free(dpio_dev->dpio);
-		return -1;
-	}
-
-	return 0;
-}
-
-static int
 dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev, int cpu_id)
 {
 	int sdest, ret;
@@ -402,15 +345,17 @@ dpaa2_create_dpio_device(int vdev_fd,
 			 struct vfio_device_info *obj_info,
 			 int object_id)
 {
-	struct dpaa2_dpio_dev *dpio_dev;
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
 	struct vfio_region_info reg_info = { .argsz = sizeof(reg_info)};
+	struct qbman_swp_desc p_des;
+	struct dpio_attr attr;
 
 	if (obj_info->num_regions < NUM_DPIO_REGIONS) {
 		DPAA2_BUS_ERR("Not sufficient number of DPIO regions");
 		return -1;
 	}
 
-	dpio_dev = rte_malloc(NULL, sizeof(struct dpaa2_dpio_dev),
+	dpio_dev = rte_zmalloc(NULL, sizeof(struct dpaa2_dpio_dev),
 			      RTE_CACHE_LINE_SIZE);
 	if (!dpio_dev) {
 		DPAA2_BUS_ERR("Memory allocation failed for DPIO Device");
@@ -423,45 +368,33 @@ dpaa2_create_dpio_device(int vdev_fd,
 	/* Using single portal  for all devices */
 	dpio_dev->mc_portal = rte_mcp_ptr_list[MC_PORTAL_INDEX];
 
-	reg_info.index = 0;
-	if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
-		DPAA2_BUS_ERR("vfio: error getting region info");
-		rte_free(dpio_dev);
-		return -1;
+	dpio_dev->dpio = malloc(sizeof(struct fsl_mc_io));
+	if (!dpio_dev->dpio) {
+		DPAA2_BUS_ERR("Memory allocation failure");
+		goto err;
 	}
 
-	dpio_dev->ce_size = reg_info.size;
-	dpio_dev->qbman_portal_ce_paddr = (size_t)mmap(NULL, reg_info.size,
-				PROT_WRITE | PROT_READ, MAP_SHARED,
-				vdev_fd, reg_info.offset);
-
-	reg_info.index = 1;
-	if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
-		DPAA2_BUS_ERR("vfio: error getting region info");
-		rte_free(dpio_dev);
-		return -1;
+	dpio_dev->dpio->regs = dpio_dev->mc_portal;
+	if (dpio_open(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->hw_id,
+		      &dpio_dev->token)) {
+		DPAA2_BUS_ERR("Failed to allocate IO space");
+		goto err;
 	}
 
-	dpio_dev->ci_size = reg_info.size;
-	dpio_dev->qbman_portal_ci_paddr = (size_t)mmap(NULL, reg_info.size,
-				PROT_WRITE | PROT_READ, MAP_SHARED,
-				vdev_fd, reg_info.offset);
-
-	if (configure_dpio_qbman_swp(dpio_dev)) {
-		DPAA2_BUS_ERR(
-			     "Fail to configure the dpio qbman portal for %d",
-			     dpio_dev->hw_id);
-		rte_free(dpio_dev);
-		return -1;
+	if (dpio_reset(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) {
+		DPAA2_BUS_ERR("Failed to reset dpio");
+		goto err;
 	}
 
-	io_space_count++;
-	dpio_dev->index = io_space_count;
+	if (dpio_enable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) {
+		DPAA2_BUS_ERR("Failed to Enable dpio");
+		goto err;
+	}
 
-	if (rte_dpaa2_vfio_setup_intr(&dpio_dev->intr_handle, vdev_fd, 1)) {
-		DPAA2_BUS_ERR("Fail to setup interrupt for %d",
-			      dpio_dev->hw_id);
-		rte_free(dpio_dev);
+	if (dpio_get_attributes(dpio_dev->dpio, CMD_PRI_LOW,
+				dpio_dev->token, &attr)) {
+		DPAA2_BUS_ERR("DPIO Get attribute failed");
+		goto err;
 	}
 
 	/* find the SoC type for the first time */
@@ -481,11 +414,77 @@ dpaa2_create_dpio_device(int vdev_fd,
 			DPAA2_BUS_DEBUG("LX2160 Platform Detected");
 		}
 		dpaa2_svr_family = (mc_plat_info.svr & 0xffff0000);
+
+		if (dpaa2_svr_family == SVR_LX2160A) {
+			dpaa2_dqrr_size = DPAA2_LX2_DQRR_RING_SIZE;
+			dpaa2_eqcr_size = DPAA2_LX2_EQCR_RING_SIZE;
+		} else {
+			dpaa2_dqrr_size = DPAA2_DQRR_RING_SIZE;
+			dpaa2_eqcr_size = DPAA2_EQCR_RING_SIZE;
+		}
+	}
+
+	if (dpaa2_svr_family == SVR_LX2160A)
+		reg_info.index = DPAA2_SWP_CENA_MEM_REGION;
+	else
+		reg_info.index = DPAA2_SWP_CENA_REGION;
+
+	if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
+		DPAA2_BUS_ERR("vfio: error getting region info");
+		goto err;
+	}
+
+	dpio_dev->ce_size = reg_info.size;
+	dpio_dev->qbman_portal_ce_paddr = (size_t)mmap(NULL, reg_info.size,
+				PROT_WRITE | PROT_READ, MAP_SHARED,
+				vdev_fd, reg_info.offset);
+
+	reg_info.index = DPAA2_SWP_CINH_REGION;
+	if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
+		DPAA2_BUS_ERR("vfio: error getting region info");
+		goto err;
+	}
+
+	dpio_dev->ci_size = reg_info.size;
+	dpio_dev->qbman_portal_ci_paddr = (size_t)mmap(NULL, reg_info.size,
+				PROT_WRITE | PROT_READ, MAP_SHARED,
+				vdev_fd, reg_info.offset);
+
+	/* Configure & setup SW portal */
+	p_des.block = NULL;
+	p_des.idx = attr.qbman_portal_id;
+	p_des.cena_bar = (void *)(dpio_dev->qbman_portal_ce_paddr);
+	p_des.cinh_bar = (void *)(dpio_dev->qbman_portal_ci_paddr);
+	p_des.irq = -1;
+	p_des.qman_version = attr.qbman_version;
+
+	dpio_dev->sw_portal = qbman_swp_init(&p_des);
+	if (dpio_dev->sw_portal == NULL) {
+		DPAA2_BUS_ERR("QBMan SW Portal Init failed");
+		goto err;
+	}
+
+	io_space_count++;
+	dpio_dev->index = io_space_count;
+
+	if (rte_dpaa2_vfio_setup_intr(&dpio_dev->intr_handle, vdev_fd, 1)) {
+		DPAA2_BUS_ERR("Fail to setup interrupt for %d",
+			      dpio_dev->hw_id);
+		goto err;
 	}
 
 	TAILQ_INSERT_TAIL(&dpio_dev_list, dpio_dev, next);
 
 	return 0;
+
+err:
+	if (dpio_dev->dpio) {
+		dpio_disable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token);
+		dpio_close(dpio_dev->dpio, CMD_PRI_LOW,  dpio_dev->token);
+		free(dpio_dev->dpio);
+	}
+	rte_free(dpio_dev);
+	return -1;
 }
 
 void
@@ -506,7 +505,7 @@ dpaa2_alloc_dq_storage(struct queue_storage_info_t *q_storage)
 
 	for (i = 0; i < NUM_DQS_PER_QUEUE; i++) {
 		q_storage->dq_storage[i] = rte_malloc(NULL,
-			DPAA2_DQRR_RING_SIZE * sizeof(struct qbman_result),
+			dpaa2_dqrr_size * sizeof(struct qbman_result),
 			RTE_CACHE_LINE_SIZE);
 		if (!q_storage->dq_storage[i])
 			goto fail;
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h
index d593eea7..462501a2 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h
@@ -30,6 +30,10 @@ RTE_DECLARE_PER_LCORE(struct dpaa2_io_portal_t, _dpaa2_io);
 
 /* Variable to store DPAA2 platform type */
 extern uint32_t dpaa2_svr_family;
+/* Variable to store DPAA2 DQRR size */
+extern uint8_t dpaa2_dqrr_size;
+/* Variable to store DPAA2 EQCR size */
+extern uint8_t dpaa2_eqcr_size;
 
 extern struct dpaa2_io_portal_t dpaa2_io_portal[RTE_MAX_LCORE];
 
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
index 82075936..efbeebef 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016 NXP
+ *   Copyright 2016-2018 NXP
  *
  */
 
@@ -9,6 +9,7 @@
 #define _DPAA2_HW_PVT_H_
 
 #include <rte_eventdev.h>
+#include <dpaax_iova_table.h>
 
 #include <mc/fsl_mc_sys.h>
 #include <fsl_qbman_portal.h>
@@ -31,11 +32,27 @@
 #define VLAN_TAG_SIZE   4 /** < Vlan Header Length */
 #endif
 
-#define MAX_TX_RING_SLOTS	8
-	/** <Maximum number of slots available in TX ring*/
+/* Maximum number of slots available in TX ring */
+#define MAX_TX_RING_SLOTS			32
 
-#define DPAA2_DQRR_RING_SIZE	16
-	/** <Maximum number of slots available in RX ring*/
+/* Maximum number of slots available in RX ring */
+#define DPAA2_EQCR_RING_SIZE		8
+/* Maximum number of slots available in RX ring on LX2 */
+#define DPAA2_LX2_EQCR_RING_SIZE	32
+
+/* Maximum number of slots available in RX ring */
+#define DPAA2_DQRR_RING_SIZE		16
+/* Maximum number of slots available in RX ring on LX2 */
+#define DPAA2_LX2_DQRR_RING_SIZE	32
+
+/* EQCR shift to get EQCR size (2 >> 3) = 8 for LS2/LS2 */
+#define DPAA2_EQCR_SHIFT		3
+/* EQCR shift to get EQCR size for LX2 (2 >> 5) = 32 for LX2 */
+#define DPAA2_LX2_EQCR_SHIFT		5
+
+#define DPAA2_SWP_CENA_REGION		0
+#define DPAA2_SWP_CINH_REGION		1
+#define DPAA2_SWP_CENA_MEM_REGION	2
 
 #define MC_PORTAL_INDEX		0
 #define NUM_DPIO_REGIONS	2
@@ -193,6 +210,12 @@ enum qbman_fd_format {
 #define DPAA2_RESET_FD_CTRL(fd)	 ((fd)->simple.ctrl = 0)
 
 #define	DPAA2_SET_FD_ASAL(fd, asal)	((fd)->simple.ctrl |= (asal << 16))
+
+#define DPAA2_RESET_FD_FLC(fd)	do {	\
+	(fd)->simple.flc_lo = 0;	\
+	(fd)->simple.flc_hi = 0;	\
+} while (0)
+
 #define DPAA2_SET_FD_FLC(fd, addr)	do { \
 	(fd)->simple.flc_lo = lower_32_bits((size_t)(addr));	\
 	(fd)->simple.flc_hi = upper_32_bits((uint64_t)(addr));	\
@@ -275,28 +298,26 @@ extern struct dpaa2_memseg_list rte_dpaa2_memsegs;
 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 extern uint8_t dpaa2_virt_mode;
 static void *dpaa2_mem_ptov(phys_addr_t paddr) __attribute__((unused));
-/* todo - this is costly, need to write a fast coversion routine */
+
 static void *dpaa2_mem_ptov(phys_addr_t paddr)
 {
-	struct dpaa2_memseg *ms;
+	void *va;
 
 	if (dpaa2_virt_mode)
 		return (void *)(size_t)paddr;
 
-	/* Check if the address is already part of the memseg list internally
-	 * maintained by the dpaa2 driver.
-	 */
-	TAILQ_FOREACH(ms, &rte_dpaa2_memsegs, next) {
-		if (paddr >= ms->iova && paddr <
-			ms->iova + ms->len)
-			return RTE_PTR_ADD(ms->vaddr, (uintptr_t)(paddr - ms->iova));
-	}
+	va = (void *)dpaax_iova_table_get_va(paddr);
+	if (likely(va != NULL))
+		return va;
 
 	/* If not, Fallback to full memseg list searching */
-	return rte_mem_iova2virt(paddr);
+	va = rte_mem_iova2virt(paddr);
+
+	return va;
 }
 
 static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) __attribute__((unused));
+
 static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr)
 {
 	const struct rte_memseg *memseg;
diff --git a/drivers/bus/fslmc/qbman/include/compat.h b/drivers/bus/fslmc/qbman/include/compat.h
index 7be8f54c..655bff4b 100644
--- a/drivers/bus/fslmc/qbman/include/compat.h
+++ b/drivers/bus/fslmc/qbman/include/compat.h
@@ -78,13 +78,14 @@ do { \
 #define lower_32_bits(x) ((uint32_t)(x))
 #define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16))
 
-
 #define __iomem
 
 #define __raw_readb(p)	(*(const volatile unsigned char *)(p))
 #define __raw_readl(p)	(*(const volatile unsigned int *)(p))
 #define __raw_writel(v, p) {*(volatile unsigned int *)(p) = (v); }
 
+#define dma_wmb()		rte_smp_mb()
+
 #define atomic_t                rte_atomic32_t
 #define atomic_read(v)          rte_atomic32_read(v)
 #define atomic_set(v, i)        rte_atomic32_set(v, i)
diff --git a/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h b/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h
index 3e63db3a..10c72e04 100644
--- a/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h
+++ b/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h
@@ -43,6 +43,15 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d);
 void qbman_swp_finish(struct qbman_swp *p);
 
 /**
+ * qbman_swp_invalidate() - Invalidate the cache enabled area of the QBMan
+ * portal. This is required to be called if a portal moved to another core
+ * because the QBMan portal area is non coherent
+ * @p: the qbman_swp object to be invalidated
+ *
+ */
+void qbman_swp_invalidate(struct qbman_swp *p);
+
+/**
  * qbman_swp_get_desc() - Get the descriptor of the given portal object.
  * @p: the given portal object.
  *
@@ -172,7 +181,7 @@ void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit);
 /**
  * struct qbman_result - structure for qbman dequeue response and/or
  * notification.
- * @donot_manipulate_directly: the 16 32bit data to represent the whole
+ * @dont_manipulate_directly: the 16 32bit data to represent the whole
  * possible qbman dequeue result.
  */
 struct qbman_result {
@@ -262,7 +271,7 @@ void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable);
  */
 struct qbman_pull_desc {
 	union {
-		uint32_t donot_manipulate_directly[16];
+		uint32_t dont_manipulate_directly[16];
 		struct pull {
 			uint8_t verb;
 			uint8_t numf;
@@ -356,6 +365,14 @@ void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid,
 				 enum qbman_pull_type_e dct);
 
 /**
+ * qbman_pull_desc_set_rad() - Decide whether reschedule the fq after dequeue
+ *
+ * @rad: 1 = Reschedule the FQ after dequeue.
+ *	 0 = Allow the FQ to remain active after dequeue.
+ */
+void qbman_pull_desc_set_rad(struct qbman_pull_desc *d, int rad);
+
+/**
  * qbman_swp_pull() - Issue the pull dequeue command
  * @s: the software portal object.
  * @d: the software portal descriptor which has been configured with
@@ -775,7 +792,7 @@ uint64_t qbman_result_cgcu_icnt(const struct qbman_result *scn);
 /* struct qbman_eq_desc - structure of enqueue descriptor */
 struct qbman_eq_desc {
 	union {
-		uint32_t donot_manipulate_directly[8];
+		uint32_t dont_manipulate_directly[8];
 		struct eq {
 			uint8_t verb;
 			uint8_t dca;
@@ -796,11 +813,11 @@ struct qbman_eq_desc {
 
 /**
  * struct qbman_eq_response - structure of enqueue response
- * @donot_manipulate_directly: the 16 32bit data to represent the whole
+ * @dont_manipulate_directly: the 16 32bit data to represent the whole
  * enqueue response.
  */
 struct qbman_eq_response {
-	uint32_t donot_manipulate_directly[16];
+	uint32_t dont_manipulate_directly[16];
 };
 
 /**
@@ -958,6 +975,7 @@ int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d,
  * @s: the software portal used for enqueue.
  * @d: the enqueue descriptor.
  * @fd: the frame descriptor to be enqueued.
+ * @flags: bit-mask of QBMAN_ENQUEUE_FLAG_*** options
  * @num_frames: the number of the frames to be enqueued.
  *
  * Return the number of enqueued frames, -EBUSY if the EQCR is not ready.
@@ -973,7 +991,6 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s,
  * @s: the software portal used for enqueue.
  * @d: the enqueue descriptor.
  * @fd: the frame descriptor to be enqueued.
- * @flags: bit-mask of QBMAN_ENQUEUE_FLAG_*** options
  * @num_frames: the number of the frames to be enqueued.
  *
  * Return the number of enqueued frames, -EBUSY if the EQCR is not ready.
@@ -998,12 +1015,12 @@ int qbman_swp_enqueue_thresh(struct qbman_swp *s, unsigned int thresh);
 	/*******************/
 /**
  * struct qbman_release_desc - The structure for buffer release descriptor
- * @donot_manipulate_directly: the 32bit data to represent the whole
+ * @dont_manipulate_directly: the 32bit data to represent the whole
  * possible settings of qbman release descriptor.
  */
 struct qbman_release_desc {
 	union {
-		uint32_t donot_manipulate_directly[16];
+		uint32_t dont_manipulate_directly[16];
 		struct br {
 			uint8_t verb;
 			uint8_t reserved;
diff --git a/drivers/bus/fslmc/qbman/qbman_portal.c b/drivers/bus/fslmc/qbman/qbman_portal.c
index 07145005..3380e54f 100644
--- a/drivers/bus/fslmc/qbman/qbman_portal.c
+++ b/drivers/bus/fslmc/qbman/qbman_portal.c
@@ -1,39 +1,17 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
+ * Copyright 2018 NXP
  *
  */
 
+#include "qbman_sys.h"
 #include "qbman_portal.h"
 
 /* QBMan portal management command codes */
 #define QBMAN_MC_ACQUIRE       0x30
 #define QBMAN_WQCHAN_CONFIGURE 0x46
 
-/* CINH register offsets */
-#define QBMAN_CINH_SWP_EQCR_PI 0x800
-#define QBMAN_CINH_SWP_EQCR_CI 0x840
-#define QBMAN_CINH_SWP_EQAR    0x8c0
-#define QBMAN_CINH_SWP_DQPI    0xa00
-#define QBMAN_CINH_SWP_DCAP    0xac0
-#define QBMAN_CINH_SWP_SDQCR   0xb00
-#define QBMAN_CINH_SWP_RAR     0xcc0
-#define QBMAN_CINH_SWP_ISR     0xe00
-#define QBMAN_CINH_SWP_IER     0xe40
-#define QBMAN_CINH_SWP_ISDR    0xe80
-#define QBMAN_CINH_SWP_IIR     0xec0
-#define QBMAN_CINH_SWP_DQRR_ITR    0xa80
-#define QBMAN_CINH_SWP_ITPR    0xf40
-
-/* CENA register offsets */
-#define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6))
-#define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((uint32_t)(n) << 6))
-#define QBMAN_CENA_SWP_RCR(n)  (0x400 + ((uint32_t)(n) << 6))
-#define QBMAN_CENA_SWP_CR      0x600
-#define QBMAN_CENA_SWP_RR(vb)  (0x700 + ((uint32_t)(vb) >> 1))
-#define QBMAN_CENA_SWP_VDQCR   0x780
-#define QBMAN_CENA_SWP_EQCR_CI 0x840
-
 /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */
 #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)p & 0x1ff) >> 6)
 
@@ -83,6 +61,102 @@ enum qbman_sdqcr_fc {
 #define MAX_QBMAN_PORTALS  64
 static struct qbman_swp *portal_idx_map[MAX_QBMAN_PORTALS];
 
+/* Internal Function declaration */
+static int
+qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd);
+static int
+qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd);
+
+static int
+qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd);
+static int
+qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd);
+
+static int
+qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd,
+		uint32_t *flags,
+		int num_frames);
+static int
+qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd,
+		uint32_t *flags,
+		int num_frames);
+
+static int
+qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd,
+		int num_frames);
+static int
+qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd,
+		int num_frames);
+
+static int
+qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d);
+static int
+qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d);
+
+const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s);
+const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s);
+
+static int
+qbman_swp_release_direct(struct qbman_swp *s,
+		const struct qbman_release_desc *d,
+		const uint64_t *buffers, unsigned int num_buffers);
+static int
+qbman_swp_release_mem_back(struct qbman_swp *s,
+		const struct qbman_release_desc *d,
+		const uint64_t *buffers, unsigned int num_buffers);
+
+/* Function pointers */
+static int (*qbman_swp_enqueue_array_mode_ptr)(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd)
+	= qbman_swp_enqueue_array_mode_direct;
+
+static int (*qbman_swp_enqueue_ring_mode_ptr)(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd)
+	= qbman_swp_enqueue_ring_mode_direct;
+
+static int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd,
+		uint32_t *flags,
+		int num_frames)
+	= qbman_swp_enqueue_multiple_direct;
+
+static int (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s,
+		const struct qbman_eq_desc *d,
+		const struct qbman_fd *fd,
+		int num_frames)
+	= qbman_swp_enqueue_multiple_desc_direct;
+
+static int (*qbman_swp_pull_ptr)(struct qbman_swp *s,
+		struct qbman_pull_desc *d)
+	= qbman_swp_pull_direct;
+
+const struct qbman_result *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s)
+		= qbman_swp_dqrr_next_direct;
+
+static int (*qbman_swp_release_ptr)(struct qbman_swp *s,
+			const struct qbman_release_desc *d,
+			const uint64_t *buffers, unsigned int num_buffers)
+			= qbman_swp_release_direct;
+
 /*********************************/
 /* Portal constructor/destructor */
 /*********************************/
@@ -104,25 +178,30 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
 {
 	int ret;
 	uint32_t eqcr_pi;
+	uint32_t mask_size;
 	struct qbman_swp *p = malloc(sizeof(*p));
 
 	if (!p)
 		return NULL;
+
+	memset(p, 0, sizeof(struct qbman_swp));
+
 	p->desc = *d;
 #ifdef QBMAN_CHECKING
 	p->mc.check = swp_mc_can_start;
 #endif
 	p->mc.valid_bit = QB_VALID_BIT;
-	p->sdq = 0;
 	p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
 	p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
 	p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
+	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000)
+		p->mr.valid_bit = QB_VALID_BIT;
 
 	atomic_set(&p->vdq.busy, 1);
 	p->vdq.valid_bit = QB_VALID_BIT;
-	p->dqrr.next_idx = 0;
 	p->dqrr.valid_bit = QB_VALID_BIT;
-	if ((p->desc.qman_version & 0xFFFF0000) < QMAN_REV_4100) {
+	qman_version = p->desc.qman_version;
+	if ((qman_version & 0xFFFF0000) < QMAN_REV_4100) {
 		p->dqrr.dqrr_size = 4;
 		p->dqrr.reset_bug = 1;
 	} else {
@@ -136,18 +215,54 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
 		pr_err("qbman_swp_sys_init() failed %d\n", ret);
 		return NULL;
 	}
+
+	/* Verify that the DQRRPI is 0 - if it is not the portal isn't
+	 * in default state which is an error
+	 */
+	if (qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQPI) & 0xF) {
+		pr_err("qbman DQRR PI is not zero, portal is not clean\n");
+		free(p);
+		return NULL;
+	}
+
 	/* SDQCR needs to be initialized to 0 when no channels are
 	 * being dequeued from or else the QMan HW will indicate an
 	 * error.  The values that were calculated above will be
 	 * applied when dequeues from a specific channel are enabled.
 	 */
 	qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_SDQCR, 0);
+
+	p->eqcr.pi_ring_size = 8;
+	if ((qman_version & 0xFFFF0000) >= QMAN_REV_5000) {
+		p->eqcr.pi_ring_size = 32;
+		qbman_swp_enqueue_array_mode_ptr =
+				qbman_swp_enqueue_array_mode_mem_back;
+		qbman_swp_enqueue_ring_mode_ptr =
+				qbman_swp_enqueue_ring_mode_mem_back;
+		qbman_swp_enqueue_multiple_ptr =
+				qbman_swp_enqueue_multiple_mem_back;
+		qbman_swp_enqueue_multiple_desc_ptr =
+				qbman_swp_enqueue_multiple_desc_mem_back;
+		qbman_swp_pull_ptr = qbman_swp_pull_mem_back;
+		qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back;
+		qbman_swp_release_ptr = qbman_swp_release_mem_back;
+	}
+
+	for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
+		p->eqcr.pi_mask = (p->eqcr.pi_mask<<1) + 1;
 	eqcr_pi = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_PI);
-	p->eqcr.pi = eqcr_pi & 0xF;
+	p->eqcr.pi = eqcr_pi & p->eqcr.pi_mask;
 	p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT;
-	p->eqcr.ci = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_CI) & 0xF;
-	p->eqcr.available = QBMAN_EQCR_SIZE - qm_cyc_diff(QBMAN_EQCR_SIZE,
-						p->eqcr.ci, p->eqcr.pi);
+	if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
+		p->eqcr.ci = qbman_cinh_read(&p->sys,
+				QBMAN_CINH_SWP_EQCR_CI) & p->eqcr.pi_mask;
+	else
+		p->eqcr.ci = qbman_cinh_read(&p->sys,
+				QBMAN_CINH_SWP_EQCR_PI) & p->eqcr.pi_mask;
+	p->eqcr.available = p->eqcr.pi_ring_size -
+				qm_cyc_diff(p->eqcr.pi_ring_size,
+				p->eqcr.ci & (p->eqcr.pi_mask<<1),
+				p->eqcr.pi & (p->eqcr.pi_mask<<1));
 
 	portal_idx_map[p->desc.idx] = p;
 	return p;
@@ -229,7 +344,8 @@ int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p)
 
 void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit)
 {
-	qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0);
+	qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR,
+			 inhibit ? 0xffffffff : 0);
 }
 
 /***********************/
@@ -246,7 +362,10 @@ void *qbman_swp_mc_start(struct qbman_swp *p)
 #ifdef QBMAN_CHECKING
 	QBMAN_BUG_ON(p->mc.check != swp_mc_can_start);
 #endif
-	ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR);
+	if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
+		ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR);
+	else
+		ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR_MEM);
 #ifdef QBMAN_CHECKING
 	if (!ret)
 		p->mc.check = swp_mc_can_submit;
@@ -266,8 +385,17 @@ void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint8_t cmd_verb)
 	 * caller wants to OR but has forgotten to do so.
 	 */
 	QBMAN_BUG_ON((*v & cmd_verb) != *v);
-	*v = cmd_verb | p->mc.valid_bit;
-	qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd);
+	if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
+		dma_wmb();
+		*v = cmd_verb | p->mc.valid_bit;
+		qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd);
+		clean(cmd);
+	} else {
+		*v = cmd_verb | p->mr.valid_bit;
+		qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR_MEM, cmd);
+		dma_wmb();
+		qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE);
+	}
 #ifdef QBMAN_CHECKING
 	p->mc.check = swp_mc_can_poll;
 #endif
@@ -279,17 +407,34 @@ void *qbman_swp_mc_result(struct qbman_swp *p)
 #ifdef QBMAN_CHECKING
 	QBMAN_BUG_ON(p->mc.check != swp_mc_can_poll);
 #endif
-	qbman_cena_invalidate_prefetch(&p->sys,
-				       QBMAN_CENA_SWP_RR(p->mc.valid_bit));
-	ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR(p->mc.valid_bit));
-	/* Remove the valid-bit - command completed if the rest is non-zero */
-	verb = ret[0] & ~QB_VALID_BIT;
-	if (!verb)
-		return NULL;
+	if ((p->desc.qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
+		qbman_cena_invalidate_prefetch(&p->sys,
+				QBMAN_CENA_SWP_RR(p->mc.valid_bit));
+		ret = qbman_cena_read(&p->sys,
+				QBMAN_CENA_SWP_RR(p->mc.valid_bit));
+		/* Remove the valid-bit -
+		 * command completed iff the rest is non-zero
+		 */
+		verb = ret[0] & ~QB_VALID_BIT;
+		if (!verb)
+			return NULL;
+		p->mc.valid_bit ^= QB_VALID_BIT;
+	} else {
+		ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR_MEM);
+		/* Command completed if the valid bit is toggled */
+		if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT))
+			return NULL;
+		/* Remove the valid-bit -
+		 * command completed iff the rest is non-zero
+		 */
+		verb = ret[0] & ~QB_VALID_BIT;
+		if (!verb)
+			return NULL;
+		p->mr.valid_bit ^= QB_VALID_BIT;
+	}
 #ifdef QBMAN_CHECKING
 	p->mc.check = swp_mc_can_start;
 #endif
-	p->mc.valid_bit ^= QB_VALID_BIT;
 	return ret;
 }
 
@@ -417,13 +562,26 @@ void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable,
 	}
 }
 
-#define EQAR_IDX(eqar)     ((eqar) & 0x7)
+#define EQAR_IDX(eqar)     ((eqar) & 0x1f)
 #define EQAR_VB(eqar)      ((eqar) & 0x80)
 #define EQAR_SUCCESS(eqar) ((eqar) & 0x100)
 
-static int qbman_swp_enqueue_array_mode(struct qbman_swp *s,
-					const struct qbman_eq_desc *d,
-					const struct qbman_fd *fd)
+static inline void qbman_write_eqcr_am_rt_register(struct qbman_swp *p,
+						   uint8_t idx)
+{
+	if (idx < 16)
+		qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT + idx * 4,
+				     QMAN_RT_MODE);
+	else
+		qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT2 +
+				     (idx - 16) * 4,
+				     QMAN_RT_MODE);
+}
+
+
+static int qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s,
+					       const struct qbman_eq_desc *d,
+					       const struct qbman_fd *fd)
 {
 	uint32_t *p;
 	const uint32_t *cl = qb_cl(d);
@@ -433,39 +591,69 @@ static int qbman_swp_enqueue_array_mode(struct qbman_swp *s,
 	if (!EQAR_SUCCESS(eqar))
 		return -EBUSY;
 	p = qbman_cena_write_start_wo_shadow(&s->sys,
-					QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
+			QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
 	memcpy(&p[1], &cl[1], 28);
 	memcpy(&p[8], fd, sizeof(*fd));
+
 	/* Set the verb byte, have to substitute in the valid-bit */
-	lwsync();
+	dma_wmb();
 	p[0] = cl[0] | EQAR_VB(eqar);
 	qbman_cena_write_complete_wo_shadow(&s->sys,
-					QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
+				QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
 	return 0;
 }
+static int qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s,
+						 const struct qbman_eq_desc *d,
+						 const struct qbman_fd *fd)
+{
+	uint32_t *p;
+	const uint32_t *cl = qb_cl(d);
+	uint32_t eqar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_EQAR);
 
-static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s,
-				       const struct qbman_eq_desc *d,
-				       const struct qbman_fd *fd)
+	pr_debug("EQAR=%08x\n", eqar);
+	if (!EQAR_SUCCESS(eqar))
+		return -EBUSY;
+	p = qbman_cena_write_start_wo_shadow(&s->sys,
+			QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
+	memcpy(&p[1], &cl[1], 28);
+	memcpy(&p[8], fd, sizeof(*fd));
+
+	/* Set the verb byte, have to substitute in the valid-bit */
+	p[0] = cl[0] | EQAR_VB(eqar);
+	dma_wmb();
+	qbman_write_eqcr_am_rt_register(s, EQAR_IDX(eqar));
+	return 0;
+}
+
+static inline int qbman_swp_enqueue_array_mode(struct qbman_swp *s,
+					       const struct qbman_eq_desc *d,
+					       const struct qbman_fd *fd)
+{
+	return qbman_swp_enqueue_array_mode_ptr(s, d, fd);
+}
+
+static int qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s,
+					      const struct qbman_eq_desc *d,
+					      const struct qbman_fd *fd)
 {
 	uint32_t *p;
 	const uint32_t *cl = qb_cl(d);
-	uint32_t eqcr_ci;
-	uint8_t diff;
+	uint32_t eqcr_ci, full_mask, half_mask;
 
+	half_mask = (s->eqcr.pi_mask>>1);
+	full_mask = s->eqcr.pi_mask;
 	if (!s->eqcr.available) {
 		eqcr_ci = s->eqcr.ci;
 		s->eqcr.ci = qbman_cena_read_reg(&s->sys,
-				QBMAN_CENA_SWP_EQCR_CI) & 0xF;
-		diff = qm_cyc_diff(QBMAN_EQCR_SIZE,
-				   eqcr_ci, s->eqcr.ci);
-		s->eqcr.available += diff;
-		if (!diff)
+				QBMAN_CENA_SWP_EQCR_CI) & full_mask;
+		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+				eqcr_ci, s->eqcr.ci);
+		if (!s->eqcr.available)
 			return -EBUSY;
 	}
 
 	p = qbman_cena_write_start_wo_shadow(&s->sys,
-					QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7));
+			QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask));
 	memcpy(&p[1], &cl[1], 28);
 	memcpy(&p[8], fd, sizeof(*fd));
 	lwsync();
@@ -473,16 +661,61 @@ static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s,
 	/* Set the verb byte, have to substitute in the valid-bit */
 	p[0] = cl[0] | s->eqcr.pi_vb;
 	qbman_cena_write_complete_wo_shadow(&s->sys,
-					QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7));
+			QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask));
 	s->eqcr.pi++;
-	s->eqcr.pi &= 0xF;
+	s->eqcr.pi &= full_mask;
 	s->eqcr.available--;
-	if (!(s->eqcr.pi & 7))
+	if (!(s->eqcr.pi & half_mask))
 		s->eqcr.pi_vb ^= QB_VALID_BIT;
 
 	return 0;
 }
 
+static int qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s,
+						const struct qbman_eq_desc *d,
+						const struct qbman_fd *fd)
+{
+	uint32_t *p;
+	const uint32_t *cl = qb_cl(d);
+	uint32_t eqcr_ci, full_mask, half_mask;
+
+	half_mask = (s->eqcr.pi_mask>>1);
+	full_mask = s->eqcr.pi_mask;
+	if (!s->eqcr.available) {
+		eqcr_ci = s->eqcr.ci;
+		s->eqcr.ci = qbman_cinh_read(&s->sys,
+				QBMAN_CENA_SWP_EQCR_CI) & full_mask;
+		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+				eqcr_ci, s->eqcr.ci);
+		if (!s->eqcr.available)
+			return -EBUSY;
+	}
+
+	p = qbman_cena_write_start_wo_shadow(&s->sys,
+			QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask));
+	memcpy(&p[1], &cl[1], 28);
+	memcpy(&p[8], fd, sizeof(*fd));
+
+	/* Set the verb byte, have to substitute in the valid-bit */
+	p[0] = cl[0] | s->eqcr.pi_vb;
+	s->eqcr.pi++;
+	s->eqcr.pi &= full_mask;
+	s->eqcr.available--;
+	if (!(s->eqcr.pi & half_mask))
+		s->eqcr.pi_vb ^= QB_VALID_BIT;
+	dma_wmb();
+	qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI,
+				(QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
+	return 0;
+}
+
+static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s,
+				       const struct qbman_eq_desc *d,
+				       const struct qbman_fd *fd)
+{
+	return qbman_swp_enqueue_ring_mode_ptr(s, d, fd);
+}
+
 int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d,
 		      const struct qbman_fd *fd)
 {
@@ -492,27 +725,27 @@ int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d,
 		return qbman_swp_enqueue_ring_mode(s, d, fd);
 }
 
-int qbman_swp_enqueue_multiple(struct qbman_swp *s,
-			       const struct qbman_eq_desc *d,
-			       const struct qbman_fd *fd,
-			       uint32_t *flags,
-			       int num_frames)
+static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
+					     const struct qbman_eq_desc *d,
+					     const struct qbman_fd *fd,
+					     uint32_t *flags,
+					     int num_frames)
 {
-	uint32_t *p;
+	uint32_t *p = NULL;
 	const uint32_t *cl = qb_cl(d);
-	uint32_t eqcr_ci, eqcr_pi;
-	uint8_t diff;
+	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
 	int i, num_enqueued = 0;
 	uint64_t addr_cena;
 
+	half_mask = (s->eqcr.pi_mask>>1);
+	full_mask = s->eqcr.pi_mask;
 	if (!s->eqcr.available) {
 		eqcr_ci = s->eqcr.ci;
 		s->eqcr.ci = qbman_cena_read_reg(&s->sys,
-				QBMAN_CENA_SWP_EQCR_CI) & 0xF;
-		diff = qm_cyc_diff(QBMAN_EQCR_SIZE,
-				   eqcr_ci, s->eqcr.ci);
-		s->eqcr.available += diff;
-		if (!diff)
+				QBMAN_CENA_SWP_EQCR_CI) & full_mask;
+		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+				eqcr_ci, s->eqcr.ci);
+		if (!s->eqcr.available)
 			return 0;
 	}
 
@@ -523,11 +756,10 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s,
 	/* Fill in the EQCR ring */
 	for (i = 0; i < num_enqueued; i++) {
 		p = qbman_cena_write_start_wo_shadow(&s->sys,
-					QBMAN_CENA_SWP_EQCR(eqcr_pi & 7));
+				QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 		memcpy(&p[1], &cl[1], 28);
 		memcpy(&p[8], &fd[i], sizeof(*fd));
 		eqcr_pi++;
-		eqcr_pi &= 0xF;
 	}
 
 	lwsync();
@@ -536,7 +768,7 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s,
 	eqcr_pi = s->eqcr.pi;
 	for (i = 0; i < num_enqueued; i++) {
 		p = qbman_cena_write_start_wo_shadow(&s->sys,
-					QBMAN_CENA_SWP_EQCR(eqcr_pi & 7));
+				QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 		p[0] = cl[0] | s->eqcr.pi_vb;
 		if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
 			struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
@@ -545,8 +777,7 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s,
 				((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
 		}
 		eqcr_pi++;
-		eqcr_pi &= 0xF;
-		if (!(eqcr_pi & 7))
+		if (!(eqcr_pi & half_mask))
 			s->eqcr.pi_vb ^= QB_VALID_BIT;
 	}
 
@@ -554,35 +785,104 @@ int qbman_swp_enqueue_multiple(struct qbman_swp *s,
 	eqcr_pi = s->eqcr.pi;
 	addr_cena = (size_t)s->sys.addr_cena;
 	for (i = 0; i < num_enqueued; i++) {
-		dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)));
+		dcbf((uintptr_t)(addr_cena +
+			QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)));
 		eqcr_pi++;
-		eqcr_pi &= 0xF;
 	}
-	s->eqcr.pi = eqcr_pi;
+	s->eqcr.pi = eqcr_pi & full_mask;
 
 	return num_enqueued;
 }
 
-int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s,
-				    const struct qbman_eq_desc *d,
-				    const struct qbman_fd *fd,
-				    int num_frames)
+static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
+					       const struct qbman_eq_desc *d,
+					       const struct qbman_fd *fd,
+					       uint32_t *flags,
+					       int num_frames)
+{
+	uint32_t *p = NULL;
+	const uint32_t *cl = qb_cl(d);
+	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
+	int i, num_enqueued = 0;
+
+	half_mask = (s->eqcr.pi_mask>>1);
+	full_mask = s->eqcr.pi_mask;
+	if (!s->eqcr.available) {
+		eqcr_ci = s->eqcr.ci;
+		s->eqcr.ci = qbman_cinh_read(&s->sys,
+				QBMAN_CENA_SWP_EQCR_CI) & full_mask;
+		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+					eqcr_ci, s->eqcr.ci);
+		if (!s->eqcr.available)
+			return 0;
+	}
+
+	eqcr_pi = s->eqcr.pi;
+	num_enqueued = (s->eqcr.available < num_frames) ?
+			s->eqcr.available : num_frames;
+	s->eqcr.available -= num_enqueued;
+	/* Fill in the EQCR ring */
+	for (i = 0; i < num_enqueued; i++) {
+		p = qbman_cena_write_start_wo_shadow(&s->sys,
+				QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+		memcpy(&p[1], &cl[1], 28);
+		memcpy(&p[8], &fd[i], sizeof(*fd));
+		eqcr_pi++;
+	}
+
+	/* Set the verb byte, have to substitute in the valid-bit */
+	eqcr_pi = s->eqcr.pi;
+	for (i = 0; i < num_enqueued; i++) {
+		p = qbman_cena_write_start_wo_shadow(&s->sys,
+				QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+		p[0] = cl[0] | s->eqcr.pi_vb;
+		if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
+			struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
+
+			d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
+				((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
+		}
+		eqcr_pi++;
+		if (!(eqcr_pi & half_mask))
+			s->eqcr.pi_vb ^= QB_VALID_BIT;
+	}
+	s->eqcr.pi = eqcr_pi & full_mask;
+
+	dma_wmb();
+	qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI,
+				(QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
+	return num_enqueued;
+}
+
+inline int qbman_swp_enqueue_multiple(struct qbman_swp *s,
+				      const struct qbman_eq_desc *d,
+				      const struct qbman_fd *fd,
+				      uint32_t *flags,
+				      int num_frames)
+{
+	return qbman_swp_enqueue_multiple_ptr(s, d, fd, flags, num_frames);
+}
+
+static int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
+					const struct qbman_eq_desc *d,
+					const struct qbman_fd *fd,
+					int num_frames)
 {
 	uint32_t *p;
 	const uint32_t *cl;
-	uint32_t eqcr_ci, eqcr_pi;
-	uint8_t diff;
+	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
 	int i, num_enqueued = 0;
 	uint64_t addr_cena;
 
+	half_mask = (s->eqcr.pi_mask>>1);
+	full_mask = s->eqcr.pi_mask;
 	if (!s->eqcr.available) {
 		eqcr_ci = s->eqcr.ci;
 		s->eqcr.ci = qbman_cena_read_reg(&s->sys,
-				QBMAN_CENA_SWP_EQCR_CI) & 0xF;
-		diff = qm_cyc_diff(QBMAN_EQCR_SIZE,
-				   eqcr_ci, s->eqcr.ci);
-		s->eqcr.available += diff;
-		if (!diff)
+				QBMAN_CENA_SWP_EQCR_CI) & full_mask;
+		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+					eqcr_ci, s->eqcr.ci);
+		if (!s->eqcr.available)
 			return 0;
 	}
 
@@ -593,12 +893,11 @@ int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s,
 	/* Fill in the EQCR ring */
 	for (i = 0; i < num_enqueued; i++) {
 		p = qbman_cena_write_start_wo_shadow(&s->sys,
-					QBMAN_CENA_SWP_EQCR(eqcr_pi & 7));
+				QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 		cl = qb_cl(&d[i]);
 		memcpy(&p[1], &cl[1], 28);
 		memcpy(&p[8], &fd[i], sizeof(*fd));
 		eqcr_pi++;
-		eqcr_pi &= 0xF;
 	}
 
 	lwsync();
@@ -607,12 +906,11 @@ int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s,
 	eqcr_pi = s->eqcr.pi;
 	for (i = 0; i < num_enqueued; i++) {
 		p = qbman_cena_write_start_wo_shadow(&s->sys,
-					QBMAN_CENA_SWP_EQCR(eqcr_pi & 7));
+				QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 		cl = qb_cl(&d[i]);
 		p[0] = cl[0] | s->eqcr.pi_vb;
 		eqcr_pi++;
-		eqcr_pi &= 0xF;
-		if (!(eqcr_pi & 7))
+		if (!(eqcr_pi & half_mask))
 			s->eqcr.pi_vb ^= QB_VALID_BIT;
 	}
 
@@ -620,14 +918,78 @@ int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s,
 	eqcr_pi = s->eqcr.pi;
 	addr_cena = (size_t)s->sys.addr_cena;
 	for (i = 0; i < num_enqueued; i++) {
-		dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)));
+		dcbf((uintptr_t)(addr_cena +
+			QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)));
+		eqcr_pi++;
+	}
+	s->eqcr.pi = eqcr_pi & full_mask;
+
+	return num_enqueued;
+}
+
+static int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
+					const struct qbman_eq_desc *d,
+					const struct qbman_fd *fd,
+					int num_frames)
+{
+	uint32_t *p;
+	const uint32_t *cl;
+	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
+	int i, num_enqueued = 0;
+
+	half_mask = (s->eqcr.pi_mask>>1);
+	full_mask = s->eqcr.pi_mask;
+	if (!s->eqcr.available) {
+		eqcr_ci = s->eqcr.ci;
+		s->eqcr.ci = qbman_cinh_read(&s->sys,
+				QBMAN_CENA_SWP_EQCR_CI) & full_mask;
+		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
+					eqcr_ci, s->eqcr.ci);
+		if (!s->eqcr.available)
+			return 0;
+	}
+
+	eqcr_pi = s->eqcr.pi;
+	num_enqueued = (s->eqcr.available < num_frames) ?
+			s->eqcr.available : num_frames;
+	s->eqcr.available -= num_enqueued;
+	/* Fill in the EQCR ring */
+	for (i = 0; i < num_enqueued; i++) {
+		p = qbman_cena_write_start_wo_shadow(&s->sys,
+				QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+		cl = qb_cl(&d[i]);
+		memcpy(&p[1], &cl[1], 28);
+		memcpy(&p[8], &fd[i], sizeof(*fd));
+		eqcr_pi++;
+	}
+
+	/* Set the verb byte, have to substitute in the valid-bit */
+	eqcr_pi = s->eqcr.pi;
+	for (i = 0; i < num_enqueued; i++) {
+		p = qbman_cena_write_start_wo_shadow(&s->sys,
+				QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
+		cl = qb_cl(&d[i]);
+		p[0] = cl[0] | s->eqcr.pi_vb;
 		eqcr_pi++;
-		eqcr_pi &= 0xF;
+		if (!(eqcr_pi & half_mask))
+			s->eqcr.pi_vb ^= QB_VALID_BIT;
 	}
-	s->eqcr.pi = eqcr_pi;
+
+	s->eqcr.pi = eqcr_pi & full_mask;
+
+	dma_wmb();
+	qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI,
+				(QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
 
 	return num_enqueued;
 }
+inline int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s,
+					   const struct qbman_eq_desc *d,
+					   const struct qbman_fd *fd,
+					   int num_frames)
+{
+	return qbman_swp_enqueue_multiple_desc_ptr(s, d, fd, num_frames);
+}
 
 /*************************/
 /* Static (push) dequeue */
@@ -670,6 +1032,7 @@ void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable)
 #define QB_VDQCR_VERB_DT_SHIFT     2
 #define QB_VDQCR_VERB_RLS_SHIFT    4
 #define QB_VDQCR_VERB_WAE_SHIFT    5
+#define QB_VDQCR_VERB_RAD_SHIFT    6
 
 enum qb_pull_dt_e {
 	qb_pull_dt_channel,
@@ -702,7 +1065,8 @@ void qbman_pull_desc_set_storage(struct qbman_pull_desc *d,
 	d->pull.rsp_addr = storage_phys;
 }
 
-void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, uint8_t numframes)
+void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d,
+				   uint8_t numframes)
 {
 	d->pull.numf = numframes - 1;
 }
@@ -735,7 +1099,20 @@ void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid,
 	d->pull.dq_src = chid;
 }
 
-int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
+void qbman_pull_desc_set_rad(struct qbman_pull_desc *d, int rad)
+{
+	if (d->pull.verb & (1 << QB_VDQCR_VERB_RLS_SHIFT)) {
+		if (rad)
+			d->pull.verb |= 1 << QB_VDQCR_VERB_RAD_SHIFT;
+		else
+			d->pull.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT);
+	} else {
+		printf("The RAD feature is not valid when RLS = 0\n");
+	}
+}
+
+static int qbman_swp_pull_direct(struct qbman_swp *s,
+				 struct qbman_pull_desc *d)
 {
 	uint32_t *p;
 	uint32_t *cl = qb_cl(d);
@@ -759,6 +1136,36 @@ int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
 	return 0;
 }
 
+static int qbman_swp_pull_mem_back(struct qbman_swp *s,
+				   struct qbman_pull_desc *d)
+{
+	uint32_t *p;
+	uint32_t *cl = qb_cl(d);
+
+	if (!atomic_dec_and_test(&s->vdq.busy)) {
+		atomic_inc(&s->vdq.busy);
+		return -EBUSY;
+	}
+
+	d->pull.tok = s->sys.idx + 1;
+	s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt;
+	p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR_MEM);
+	memcpy(&p[1], &cl[1], 12);
+
+	/* Set the verb byte, have to substitute in the valid-bit */
+	p[0] = cl[0] | s->vdq.valid_bit;
+	s->vdq.valid_bit ^= QB_VALID_BIT;
+	dma_wmb();
+	qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE);
+
+	return 0;
+}
+
+inline int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
+{
+	return qbman_swp_pull_ptr(s, d);
+}
+
 /****************/
 /* Polling DQRR */
 /****************/
@@ -791,7 +1198,12 @@ void qbman_swp_prefetch_dqrr_next(struct qbman_swp *s)
  * only once, so repeated calls can return a sequence of DQRR entries, without
  * requiring they be consumed immediately or in any particular order.
  */
-const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s)
+inline const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s)
+{
+	return qbman_swp_dqrr_next_ptr(s);
+}
+
+const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s)
 {
 	uint32_t verb;
 	uint32_t response_verb;
@@ -801,7 +1213,7 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s)
 	/* Before using valid-bit to detect if something is there, we have to
 	 * handle the case of the DQRR reset bug...
 	 */
-	if (unlikely(s->dqrr.reset_bug)) {
+	if (s->dqrr.reset_bug) {
 		/* We pick up new entries by cache-inhibited producer index,
 		 * which means that a non-coherent mapping would require us to
 		 * invalidate and read *only* once that PI has indicated that
@@ -833,7 +1245,8 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s)
 					QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
 	}
 	p = qbman_cena_read_wo_shadow(&s->sys,
-				      QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
+			QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
+
 	verb = p->dq.verb;
 
 	/* If the valid-bit isn't of the expected polarity, nothing there. Note,
@@ -867,11 +1280,54 @@ const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s)
 	return p;
 }
 
+const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s)
+{
+	uint32_t verb;
+	uint32_t response_verb;
+	uint32_t flags;
+	const struct qbman_result *p;
+
+	p = qbman_cena_read_wo_shadow(&s->sys,
+			QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx));
+
+	verb = p->dq.verb;
+
+	/* If the valid-bit isn't of the expected polarity, nothing there. Note,
+	 * in the DQRR reset bug workaround, we shouldn't need to skip these
+	 * check, because we've already determined that a new entry is available
+	 * and we've invalidated the cacheline before reading it, so the
+	 * valid-bit behaviour is repaired and should tell us what we already
+	 * knew from reading PI.
+	 */
+	if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit)
+		return NULL;
+
+	/* There's something there. Move "next_idx" attention to the next ring
+	 * entry (and prefetch it) before returning what we found.
+	 */
+	s->dqrr.next_idx++;
+	if (s->dqrr.next_idx == s->dqrr.dqrr_size) {
+		s->dqrr.next_idx = 0;
+		s->dqrr.valid_bit ^= QB_VALID_BIT;
+	}
+	/* If this is the final response to a volatile dequeue command
+	 * indicate that the vdq is no longer busy
+	 */
+	flags = p->dq.stat;
+	response_verb = verb & QBMAN_RESPONSE_VERB_MASK;
+	if ((response_verb == QBMAN_RESULT_DQ) &&
+	    (flags & QBMAN_DQ_STAT_VOLATILE) &&
+	    (flags & QBMAN_DQ_STAT_EXPIRED))
+		atomic_inc(&s->vdq.busy);
+	return p;
+}
+
 /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */
 void qbman_swp_dqrr_consume(struct qbman_swp *s,
 			    const struct qbman_result *dq)
 {
-	qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq));
+	qbman_cinh_write(&s->sys,
+			QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq));
 }
 
 /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */
@@ -884,6 +1340,7 @@ void qbman_swp_dqrr_idx_consume(struct qbman_swp *s,
 /*********************************/
 /* Polling user-provided storage */
 /*********************************/
+
 int qbman_result_has_new_result(struct qbman_swp *s,
 				struct qbman_result *dq)
 {
@@ -898,11 +1355,11 @@ int qbman_result_has_new_result(struct qbman_swp *s,
 	((struct qbman_result *)dq)->dq.tok = 0;
 
 	/*
-	 * VDQCR "no longer busy" hook - not quite the same as DQRR, because the
-	 * fact "VDQCR" shows busy doesn't mean that we hold the result that
-	 * makes it available. Eg. we may be looking at our 10th dequeue result,
-	 * having released VDQCR after the 1st result and it is now busy due to
-	 * some other command!
+	 * VDQCR "no longer busy" hook - not quite the same as DQRR, because
+	 * the fact "VDQCR" shows busy doesn't mean that we hold the result
+	 * that makes it available. Eg. we may be looking at our 10th dequeue
+	 * result, having released VDQCR after the 1st result and it is now
+	 * busy due to some other command!
 	 */
 	if (s->vdq.storage == dq) {
 		s->vdq.storage = NULL;
@@ -936,11 +1393,11 @@ int qbman_check_command_complete(struct qbman_result *dq)
 
 	s = portal_idx_map[dq->dq.tok - 1];
 	/*
-	 * VDQCR "no longer busy" hook - not quite the same as DQRR, because the
-	 * fact "VDQCR" shows busy doesn't mean that we hold the result that
-	 * makes it available. Eg. we may be looking at our 10th dequeue result,
-	 * having released VDQCR after the 1st result and it is now busy due to
-	 * some other command!
+	 * VDQCR "no longer busy" hook - not quite the same as DQRR, because
+	 * the fact "VDQCR" shows busy doesn't mean that we hold the result
+	 * that makes it available. Eg. we may be looking at our 10th dequeue
+	 * result, having released VDQCR after the 1st result and it is now
+	 * busy due to some other command!
 	 */
 	if (s->vdq.storage == dq) {
 		s->vdq.storage = NULL;
@@ -1142,8 +1599,10 @@ void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable)
 #define RAR_VB(rar)      ((rar) & 0x80)
 #define RAR_SUCCESS(rar) ((rar) & 0x100)
 
-int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d,
-		      const uint64_t *buffers, unsigned int num_buffers)
+static int qbman_swp_release_direct(struct qbman_swp *s,
+				    const struct qbman_release_desc *d,
+				    const uint64_t *buffers,
+				    unsigned int num_buffers)
 {
 	uint32_t *p;
 	const uint32_t *cl = qb_cl(d);
@@ -1157,22 +1616,63 @@ int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d,
 
 	/* Start the release command */
 	p = qbman_cena_write_start_wo_shadow(&s->sys,
-					     QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
+				     QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
 
 	/* Copy the caller's buffer pointers to the command */
 	u64_to_le32_copy(&p[2], buffers, num_buffers);
 
-	/* Set the verb byte, have to substitute in the valid-bit and the number
-	 * of buffers.
+	/* Set the verb byte, have to substitute in the valid-bit and the
+	 * number of buffers.
 	 */
 	lwsync();
 	p[0] = cl[0] | RAR_VB(rar) | num_buffers;
 	qbman_cena_write_complete_wo_shadow(&s->sys,
-					    QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
+				    QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
 
 	return 0;
 }
 
+static int qbman_swp_release_mem_back(struct qbman_swp *s,
+				      const struct qbman_release_desc *d,
+				      const uint64_t *buffers,
+				      unsigned int num_buffers)
+{
+	uint32_t *p;
+	const uint32_t *cl = qb_cl(d);
+	uint32_t rar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_RAR);
+
+	pr_debug("RAR=%08x\n", rar);
+	if (!RAR_SUCCESS(rar))
+		return -EBUSY;
+
+	QBMAN_BUG_ON(!num_buffers || (num_buffers > 7));
+
+	/* Start the release command */
+	p = qbman_cena_write_start_wo_shadow(&s->sys,
+		QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar)));
+
+	/* Copy the caller's buffer pointers to the command */
+	u64_to_le32_copy(&p[2], buffers, num_buffers);
+
+	/* Set the verb byte, have to substitute in the valid-bit and the
+	 * number of buffers.
+	 */
+	p[0] = cl[0] | RAR_VB(rar) | num_buffers;
+	lwsync();
+	qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_RCR_AM_RT +
+		RAR_IDX(rar) * 4, QMAN_RT_MODE);
+
+	return 0;
+}
+
+inline int qbman_swp_release(struct qbman_swp *s,
+			     const struct qbman_release_desc *d,
+			     const uint64_t *buffers,
+			     unsigned int num_buffers)
+{
+	return qbman_swp_release_ptr(s, d, buffers, num_buffers);
+}
+
 /*******************/
 /* Buffer acquires */
 /*******************/
@@ -1214,7 +1714,7 @@ int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers,
 
 	/* Complete the management command */
 	r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE);
-	if (unlikely(!r)) {
+	if (!r) {
 		pr_err("qbman: acquire from BPID %d failed, no response\n",
 		       bpid);
 		return -EIO;
@@ -1224,7 +1724,7 @@ int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers,
 	QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_MC_ACQUIRE);
 
 	/* Determine success or failure */
-	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
+	if (r->rslt != QBMAN_MC_RSLT_OK) {
 		pr_err("Acquire buffers from BPID 0x%x failed, code=0x%02x\n",
 		       bpid, r->rslt);
 		return -EIO;
@@ -1271,7 +1771,7 @@ static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid,
 
 	/* Complete the management command */
 	r = qbman_swp_mc_complete(s, p, alt_fq_verb);
-	if (unlikely(!r)) {
+	if (!r) {
 		pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n",
 		       alt_fq_verb);
 		return -EIO;
@@ -1281,7 +1781,7 @@ static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid,
 	QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != alt_fq_verb);
 
 	/* Determine success or failure */
-	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
+	if (r->rslt != QBMAN_MC_RSLT_OK) {
 		pr_err("ALT FQID %d failed: verb = 0x%08x, code = 0x%02x\n",
 		       fqid, alt_fq_verb, r->rslt);
 		return -EIO;
@@ -1362,7 +1862,7 @@ static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid,
 
 	/* Complete the management command */
 	r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE);
-	if (unlikely(!r)) {
+	if (!r) {
 		pr_err("qbman: wqchan config failed, no response\n");
 		return -EIO;
 	}
@@ -1372,7 +1872,7 @@ static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid,
 		     != QBMAN_WQCHAN_CONFIGURE);
 
 	/* Determine success or failure */
-	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
+	if (r->rslt != QBMAN_MC_RSLT_OK) {
 		pr_err("CDAN cQID %d failed: code = 0x%02x\n",
 		       channelid, r->rslt);
 		return -EIO;
diff --git a/drivers/bus/fslmc/qbman/qbman_portal.h b/drivers/bus/fslmc/qbman/qbman_portal.h
index dbea22a1..3b0fc540 100644
--- a/drivers/bus/fslmc/qbman/qbman_portal.h
+++ b/drivers/bus/fslmc/qbman/qbman_portal.h
@@ -1,12 +1,17 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
+ * Copyright 2018 NXP
  *
  */
 
+#ifndef _QBMAN_PORTAL_H_
+#define _QBMAN_PORTAL_H_
+
 #include "qbman_sys.h"
 #include <fsl_qbman_portal.h>
 
+uint32_t qman_version;
 #define QMAN_REV_4000   0x04000000
 #define QMAN_REV_4100   0x04010000
 #define QMAN_REV_4101   0x04010001
@@ -14,13 +19,14 @@
 /* All QBMan command and result structures use this "valid bit" encoding */
 #define QB_VALID_BIT ((uint32_t)0x80)
 
+/* All QBMan command use this "Read trigger bit" encoding */
+#define QB_RT_BIT ((uint32_t)0x100)
+
 /* Management command result codes */
 #define QBMAN_MC_RSLT_OK      0xf0
 
 /* QBMan DQRR size is set at runtime in qbman_portal.c */
 
-#define QBMAN_EQCR_SIZE 8
-
 static inline uint8_t qm_cyc_diff(uint8_t ringsize, uint8_t first,
 				  uint8_t last)
 {
@@ -51,6 +57,10 @@ struct qbman_swp {
 #endif
 		uint32_t valid_bit; /* 0x00 or 0x80 */
 	} mc;
+	/* Management response */
+	struct {
+		uint32_t valid_bit; /* 0x00 or 0x80 */
+	} mr;
 	/* Push dequeues */
 	uint32_t sdq;
 	/* Volatile dequeues */
@@ -87,6 +97,8 @@ struct qbman_swp {
 	struct {
 		uint32_t pi;
 		uint32_t pi_vb;
+		uint32_t pi_ring_size;
+		uint32_t pi_mask;
 		uint32_t ci;
 		int available;
 	} eqcr;
@@ -141,4 +153,16 @@ static inline void *qbman_swp_mc_complete(struct qbman_swp *swp, void *cmd,
  * an inline) is necessary to work with different descriptor types and to work
  * correctly with const and non-const inputs (and similarly-qualified outputs).
  */
-#define qb_cl(d) (&(d)->donot_manipulate_directly[0])
+#define qb_cl(d) (&(d)->dont_manipulate_directly[0])
+
+#ifdef RTE_ARCH_ARM64
+	#define clean(p) \
+			{ asm volatile("dc cvac, %0;" : : "r" (p) : "memory"); }
+	#define invalidate(p) \
+			{ asm volatile("dc ivac, %0" : : "r"(p) : "memory"); }
+#else
+	#define clean(p)
+	#define invalidate(p)
+#endif
+
+#endif
diff --git a/drivers/bus/fslmc/qbman/qbman_sys.h b/drivers/bus/fslmc/qbman/qbman_sys.h
index 2bd33ea5..d41af835 100644
--- a/drivers/bus/fslmc/qbman/qbman_sys.h
+++ b/drivers/bus/fslmc/qbman/qbman_sys.h
@@ -18,11 +18,51 @@
  * *not* to provide linux compatibility.
  */
 
+#ifndef _QBMAN_SYS_H_
+#define _QBMAN_SYS_H_
+
 #include "qbman_sys_decl.h"
 
 #define CENA_WRITE_ENABLE 0
 #define CINH_WRITE_ENABLE 1
 
+/* CINH register offsets */
+#define QBMAN_CINH_SWP_EQCR_PI      0x800
+#define QBMAN_CINH_SWP_EQCR_CI      0x840
+#define QBMAN_CINH_SWP_EQAR         0x8c0
+#define QBMAN_CINH_SWP_CR_RT        0x900
+#define QBMAN_CINH_SWP_VDQCR_RT     0x940
+#define QBMAN_CINH_SWP_EQCR_AM_RT   0x980
+#define QBMAN_CINH_SWP_RCR_AM_RT    0x9c0
+#define QBMAN_CINH_SWP_DQPI         0xa00
+#define QBMAN_CINH_SWP_DQRR_ITR     0xa80
+#define QBMAN_CINH_SWP_DCAP         0xac0
+#define QBMAN_CINH_SWP_SDQCR        0xb00
+#define QBMAN_CINH_SWP_EQCR_AM_RT2  0xb40
+#define QBMAN_CINH_SWP_RCR_PI       0xc00
+#define QBMAN_CINH_SWP_RAR          0xcc0
+#define QBMAN_CINH_SWP_ISR          0xe00
+#define QBMAN_CINH_SWP_IER          0xe40
+#define QBMAN_CINH_SWP_ISDR         0xe80
+#define QBMAN_CINH_SWP_IIR          0xec0
+#define QBMAN_CINH_SWP_ITPR         0xf40
+
+/* CENA register offsets */
+#define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6))
+#define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((uint32_t)(n) << 6))
+#define QBMAN_CENA_SWP_RCR(n)  (0x400 + ((uint32_t)(n) << 6))
+#define QBMAN_CENA_SWP_CR      0x600
+#define QBMAN_CENA_SWP_RR(vb)  (0x700 + ((uint32_t)(vb) >> 1))
+#define QBMAN_CENA_SWP_VDQCR   0x780
+#define QBMAN_CENA_SWP_EQCR_CI 0x840
+
+/* CENA register offsets in memory-backed mode */
+#define QBMAN_CENA_SWP_DQRR_MEM(n)  (0x800 + ((uint32_t)(n) << 6))
+#define QBMAN_CENA_SWP_RCR_MEM(n)   (0x1400 + ((uint32_t)(n) << 6))
+#define QBMAN_CENA_SWP_CR_MEM       0x1600
+#define QBMAN_CENA_SWP_RR_MEM       0x1680
+#define QBMAN_CENA_SWP_VDQCR_MEM    0x1780
+
 /* Debugging assists */
 static inline void __hexdump(unsigned long start, unsigned long end,
 			     unsigned long p, size_t sz, const unsigned char *c)
@@ -125,8 +165,8 @@ struct qbman_swp_sys {
 	 * place-holder.
 	 */
 	uint8_t *cena;
-	uint8_t __iomem *addr_cena;
-	uint8_t __iomem *addr_cinh;
+	uint8_t *addr_cena;
+	uint8_t *addr_cinh;
 	uint32_t idx;
 	enum qbman_eqcr_mode eqcr_mode;
 };
@@ -292,13 +332,16 @@ static inline void qbman_cena_prefetch(struct qbman_swp_sys *s,
  * qbman_portal.c. So use of it is declared locally here.
  */
 #define QBMAN_CINH_SWP_CFG   0xd00
-#define QBMAN_CINH_SWP_CFG   0xd00
+
 #define SWP_CFG_DQRR_MF_SHIFT 20
 #define SWP_CFG_EST_SHIFT     16
+#define SWP_CFG_CPBS_SHIFT    15
 #define SWP_CFG_WN_SHIFT      14
 #define SWP_CFG_RPM_SHIFT     12
 #define SWP_CFG_DCM_SHIFT     10
 #define SWP_CFG_EPM_SHIFT     8
+#define SWP_CFG_VPM_SHIFT     7
+#define SWP_CFG_CPM_SHIFT     6
 #define SWP_CFG_SD_SHIFT      5
 #define SWP_CFG_SP_SHIFT      4
 #define SWP_CFG_SE_SHIFT      3
@@ -329,11 +372,20 @@ static inline uint32_t qbman_set_swp_cfg(uint8_t max_fill, uint8_t wn,
 	return reg;
 }
 
+#define QMAN_RT_MODE	0x00000100
+
+#define QMAN_REV_4000	0x04000000
+#define QMAN_REV_4100	0x04010000
+#define QMAN_REV_4101	0x04010001
+#define QMAN_REV_5000	0x05000000
+#define QMAN_REV_MASK	0xffff0000
+
 static inline int qbman_swp_sys_init(struct qbman_swp_sys *s,
 				     const struct qbman_swp_desc *d,
 				     uint8_t dqrr_size)
 {
 	uint32_t reg;
+	int i;
 #ifdef RTE_ARCH_64
 	uint8_t wn = CENA_WRITE_ENABLE;
 #else
@@ -343,7 +395,7 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s,
 	s->addr_cena = d->cena_bar;
 	s->addr_cinh = d->cinh_bar;
 	s->idx = (uint32_t)d->idx;
-	s->cena = malloc(4096);
+	s->cena = malloc(64*1024);
 	if (!s->cena) {
 		pr_err("Could not allocate page for cena shadow\n");
 		return -1;
@@ -358,12 +410,34 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s,
 	reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG);
 	QBMAN_BUG_ON(reg);
 #endif
+	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000)
+		memset(s->addr_cena, 0, 64*1024);
+	else {
+		/* Invalidate the portal memory.
+		 * This ensures no stale cache lines
+		 */
+		for (i = 0; i < 0x1000; i += 64)
+			dccivac(s->addr_cena + i);
+	}
+
 	if (s->eqcr_mode == qman_eqcr_vb_array)
-		reg = qbman_set_swp_cfg(dqrr_size, wn, 0, 3, 2, 3, 1, 1, 1, 1,
-					1, 1);
-	else
-		reg = qbman_set_swp_cfg(dqrr_size, wn, 1, 3, 2, 2, 1, 1, 1, 1,
-					1, 1);
+		reg = qbman_set_swp_cfg(dqrr_size, wn,
+					0, 3, 2, 3, 1, 1, 1, 1, 1, 1);
+	else {
+		if ((d->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
+			reg = qbman_set_swp_cfg(dqrr_size, wn,
+						1, 3, 2, 2, 1, 1, 1, 1, 1, 1);
+		else
+			reg = qbman_set_swp_cfg(dqrr_size, wn,
+						1, 3, 2, 0, 1, 1, 1, 1, 1, 1);
+	}
+
+	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
+		reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */
+		       1 << SWP_CFG_VPM_SHIFT |  /* VDQCR read triggered mode */
+		       1 << SWP_CFG_CPM_SHIFT;   /* CR read triggered mode */
+	}
+
 	qbman_cinh_write(s, QBMAN_CINH_SWP_CFG, reg);
 	reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG);
 	if (!reg) {
@@ -371,6 +445,12 @@ static inline int qbman_swp_sys_init(struct qbman_swp_sys *s,
 		free(s->cena);
 		return -1;
 	}
+
+	if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
+		qbman_cinh_write(s, QBMAN_CINH_SWP_EQCR_PI, QMAN_RT_MODE);
+		qbman_cinh_write(s, QBMAN_CINH_SWP_RCR_PI, QMAN_RT_MODE);
+	}
+
 	return 0;
 }
 
@@ -378,3 +458,5 @@ static inline void qbman_swp_sys_finish(struct qbman_swp_sys *s)
 {
 	free(s->cena);
 }
+
+#endif /* _QBMAN_SYS_H_ */
diff --git a/drivers/bus/fslmc/qbman/qbman_sys_decl.h b/drivers/bus/fslmc/qbman/qbman_sys_decl.h
index fa6977fe..a29f5b46 100644
--- a/drivers/bus/fslmc/qbman/qbman_sys_decl.h
+++ b/drivers/bus/fslmc/qbman/qbman_sys_decl.h
@@ -3,6 +3,9 @@
  * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
  *
  */
+#ifndef _QBMAN_SYS_DECL_H_
+#define _QBMAN_SYS_DECL_H_
+
 #include <compat.h>
 #include <fsl_qbman_base.h>
 
@@ -51,3 +54,4 @@ static inline void prefetch_for_store(void *p)
 	RTE_SET_USED(p);
 }
 #endif
+#endif /* _QBMAN_SYS_DECL_H_ */
diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
index fe45a113..dcc4e082 100644
--- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
+++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
@@ -114,5 +114,18 @@ DPDK_18.05 {
 	dpdmai_open;
 	dpdmai_set_rx_queue;
 	rte_dpaa2_free_dpci_dev;
+	rte_dpaa2_memsegs;
 
 } DPDK_18.02;
+
+DPDK_18.11 {
+	global:
+
+	dpaa2_dqrr_size;
+	dpaa2_eqcr_size;
+	dpci_get_link_state;
+	dpci_get_opr;
+	dpci_get_peer_attributes;
+	dpci_set_opr;
+
+} DPDK_18.05;
diff --git a/drivers/bus/ifpga/Makefile b/drivers/bus/ifpga/Makefile
index 3ff3bdb8..514452b3 100644
--- a/drivers/bus/ifpga/Makefile
+++ b/drivers/bus/ifpga/Makefile
@@ -19,7 +19,7 @@ LDLIBS += -lrte_kvargs
 EXPORT_MAP := rte_bus_ifpga_version.map
 
 # library version
-LIBABIVER := 1
+LIBABIVER := 2
 
 SRCS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += ifpga_bus.c
 SRCS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += ifpga_common.c
diff --git a/drivers/bus/ifpga/ifpga_bus.c b/drivers/bus/ifpga/ifpga_bus.c
index b324872e..5f23ed8b 100644
--- a/drivers/bus/ifpga/ifpga_bus.c
+++ b/drivers/bus/ifpga/ifpga_bus.c
@@ -142,6 +142,7 @@ ifpga_scan_one(struct rte_rawdev *rawdev,
 	if (!afu_dev)
 		goto end;
 
+	afu_dev->device.bus = &rte_ifpga_bus;
 	afu_dev->device.devargs = devargs;
 	afu_dev->device.numa_node = SOCKET_ID_ANY;
 	afu_dev->device.name = devargs->name;
@@ -279,14 +280,13 @@ ifpga_probe_one_driver(struct rte_afu_driver *drv,
 
 	/* reference driver structure */
 	afu_dev->driver = drv;
-	afu_dev->device.driver = &drv->driver;
 
 	/* call the driver probe() function */
 	ret = drv->probe(afu_dev);
-	if (ret) {
+	if (ret)
 		afu_dev->driver = NULL;
-		afu_dev->device.driver = NULL;
-	}
+	else
+		afu_dev->device.driver = &drv->driver;
 
 	return ret;
 }
@@ -301,8 +301,11 @@ ifpga_probe_all_drivers(struct rte_afu_device *afu_dev)
 		return -1;
 
 	/* Check if a driver is already loaded */
-	if (afu_dev->driver != NULL)
-		return 0;
+	if (rte_dev_is_probed(&afu_dev->device)) {
+		IFPGA_BUS_DEBUG("Device %s is already probed\n",
+				rte_ifpga_device_name(afu_dev));
+		return -EEXIST;
+	}
 
 	TAILQ_FOREACH(drv, &ifpga_afu_drv_list, next) {
 		if (ifpga_probe_one_driver(drv, afu_dev)) {
@@ -325,14 +328,13 @@ ifpga_probe(void)
 	int ret = 0;
 
 	TAILQ_FOREACH(afu_dev, &ifpga_afu_dev_list, next) {
-		if (afu_dev->device.driver)
-			continue;
-
 		ret = ifpga_probe_all_drivers(afu_dev);
+		if (ret == -EEXIST)
+			continue;
 		if (ret < 0)
 			IFPGA_BUS_ERR("failed to initialize %s device\n",
 				rte_ifpga_device_name(afu_dev));
-		}
+	}
 
 	return ret;
 }
@@ -347,23 +349,20 @@ static int
 ifpga_remove_driver(struct rte_afu_device *afu_dev)
 {
 	const char *name;
-	const struct rte_afu_driver *driver;
 
 	name = rte_ifpga_device_name(afu_dev);
-	if (!afu_dev->device.driver) {
+	if (afu_dev->driver == NULL) {
 		IFPGA_BUS_DEBUG("no driver attach to device %s\n", name);
 		return 1;
 	}
 
-	driver = RTE_DRV_TO_AFU_CONST(afu_dev->device.driver);
-	return driver->remove(afu_dev);
+	return afu_dev->driver->remove(afu_dev);
 }
 
 static int
 ifpga_unplug(struct rte_device *dev)
 {
 	struct rte_afu_device *afu_dev = NULL;
-	struct rte_devargs *devargs = NULL;
 	int ret;
 
 	if (dev == NULL)
@@ -373,15 +372,13 @@ ifpga_unplug(struct rte_device *dev)
 	if (!afu_dev)
 		return -ENOENT;
 
-	devargs = dev->devargs;
-
 	ret = ifpga_remove_driver(afu_dev);
 	if (ret)
 		return ret;
 
 	TAILQ_REMOVE(&ifpga_afu_dev_list, afu_dev, next);
 
-	rte_devargs_remove(devargs->bus->name, devargs->name);
+	rte_devargs_remove(dev->devargs);
 	free(afu_dev);
 	return 0;
 
diff --git a/drivers/bus/ifpga/meson.build b/drivers/bus/ifpga/meson.build
index c9b08c86..0b5c38d5 100644
--- a/drivers/bus/ifpga/meson.build
+++ b/drivers/bus/ifpga/meson.build
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2010-2018 Intel Corporation
 
+version = 2
+
 deps += ['pci', 'kvargs', 'rawdev']
 install_headers('rte_bus_ifpga.h')
 sources = files('ifpga_common.c', 'ifpga_bus.c')
diff --git a/drivers/bus/ifpga/rte_bus_ifpga.h b/drivers/bus/ifpga/rte_bus_ifpga.h
index 51d5ae0d..d53c0f48 100644
--- a/drivers/bus/ifpga/rte_bus_ifpga.h
+++ b/drivers/bus/ifpga/rte_bus_ifpga.h
@@ -83,9 +83,6 @@ struct rte_afu_device {
 #define RTE_DEV_TO_AFU(ptr) \
 	container_of(ptr, struct rte_afu_device, device)
 
-#define RTE_DRV_TO_AFU_CONST(ptr) \
-	container_of(ptr, const struct rte_afu_driver, driver)
-
 /**
  * Initialization function for the driver called during FPGA BUS probing.
  */
diff --git a/drivers/bus/pci/Makefile b/drivers/bus/pci/Makefile
index cf373068..f33e0120 100644
--- a/drivers/bus/pci/Makefile
+++ b/drivers/bus/pci/Makefile
@@ -4,7 +4,7 @@
 include $(RTE_SDK)/mk/rte.vars.mk
 
 LIB = librte_bus_pci.a
-LIBABIVER := 1
+LIBABIVER := 2
 EXPORT_MAP := rte_bus_pci_version.map
 
 CFLAGS := -I$(SRCDIR) $(CFLAGS)
@@ -26,10 +26,11 @@ CFLAGS += -I$(RTE_SDK)/lib/librte_eal/$(SYSTEM)app/eal
 CFLAGS += -DALLOW_EXPERIMENTAL_API
 
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
-LDLIBS += -lrte_ethdev -lrte_pci
+LDLIBS += -lrte_ethdev -lrte_pci -lrte_kvargs
 
 include $(RTE_SDK)/drivers/bus/pci/$(SYSTEM)/Makefile
 SRCS-$(CONFIG_RTE_LIBRTE_PCI_BUS) := $(addprefix $(SYSTEM)/,$(SRCS))
+SRCS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci_params.c
 SRCS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci_common.c
 SRCS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci_common_uio.c
 
diff --git a/drivers/bus/pci/bsd/pci.c b/drivers/bus/pci/bsd/pci.c
index 655b34b7..d09f8ee5 100644
--- a/drivers/bus/pci/bsd/pci.c
+++ b/drivers/bus/pci/bsd/pci.c
@@ -223,6 +223,8 @@ pci_scan_one(int dev_pci_fd, struct pci_conf *conf)
 	}
 
 	memset(dev, 0, sizeof(*dev));
+	dev->device.bus = &rte_pci_bus.bus;
+
 	dev->addr.domain = conf->pc_sel.pc_domain;
 	dev->addr.bus = conf->pc_sel.pc_bus;
 	dev->addr.devid = conf->pc_sel.pc_dev;
@@ -439,6 +441,8 @@ int rte_pci_read_config(const struct rte_pci_device *dev,
 {
 	int fd = -1;
 	int size;
+	/* Copy Linux implementation's behaviour */
+	const int return_len = len;
 	struct pci_io pi = {
 		.pi_sel = {
 			.pc_domain = dev->addr.domain,
@@ -469,7 +473,7 @@ int rte_pci_read_config(const struct rte_pci_device *dev,
 	}
 	close(fd);
 
-	return 0;
+	return return_len;
 
  error:
 	if (fd >= 0)
diff --git a/drivers/bus/pci/linux/Makefile b/drivers/bus/pci/linux/Makefile
index 96ea1d54..90404468 100644
--- a/drivers/bus/pci/linux/Makefile
+++ b/drivers/bus/pci/linux/Makefile
@@ -4,5 +4,3 @@
 SRCS += pci.c
 SRCS += pci_uio.c
 SRCS += pci_vfio.c
-
-CFLAGS += -D_GNU_SOURCE
diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c
index 04648ac9..45c24ef7 100644
--- a/drivers/bus/pci/linux/pci.c
+++ b/drivers/bus/pci/linux/pci.c
@@ -119,7 +119,7 @@ rte_pci_unmap_device(struct rte_pci_device *dev)
 static int
 find_max_end_va(const struct rte_memseg_list *msl, void *arg)
 {
-	size_t sz = msl->memseg_arr.len * msl->page_sz;
+	size_t sz = msl->len;
 	void *end_va = RTE_PTR_ADD(msl->base_va, sz);
 	void **max_va = arg;
 
@@ -228,6 +228,7 @@ pci_scan_one(const char *dirname, const struct rte_pci_addr *addr)
 		return -1;
 
 	memset(dev, 0, sizeof(*dev));
+	dev->device.bus = &rte_pci_bus.bus;
 	dev->addr = *addr;
 
 	/* get vendor id */
@@ -588,10 +589,8 @@ pci_one_device_iommu_support_va(struct rte_pci_device *dev)
 	fclose(fp);
 
 	mgaw = ((vtd_cap_reg & VTD_CAP_MGAW_MASK) >> VTD_CAP_MGAW_SHIFT) + 1;
-	if (mgaw < X86_VA_WIDTH)
-		return false;
 
-	return true;
+	return rte_eal_check_dma_mask(mgaw) == 0 ? true : false;
 }
 #elif defined(RTE_ARCH_PPC_64)
 static bool
@@ -620,8 +619,11 @@ pci_devices_iommu_support_va(void)
 		FOREACH_DEVICE_ON_PCIBUS(dev) {
 			if (!rte_pci_match(drv, dev))
 				continue;
-			if (!pci_one_device_iommu_support_va(dev))
-				return false;
+			/*
+			 * just one PCI device needs to be checked out because
+			 * the IOMMU hardware is the same for all of them.
+			 */
+			return pci_one_device_iommu_support_va(dev);
 		}
 	}
 	return true;
@@ -672,23 +674,21 @@ rte_pci_get_iommu_class(void)
 int rte_pci_read_config(const struct rte_pci_device *device,
 		void *buf, size_t len, off_t offset)
 {
+	char devname[RTE_DEV_NAME_MAX_LEN] = "";
 	const struct rte_intr_handle *intr_handle = &device->intr_handle;
 
-	switch (intr_handle->type) {
-	case RTE_INTR_HANDLE_UIO:
-	case RTE_INTR_HANDLE_UIO_INTX:
+	switch (device->kdrv) {
+	case RTE_KDRV_IGB_UIO:
 		return pci_uio_read_config(intr_handle, buf, len, offset);
-
 #ifdef VFIO_PRESENT
-	case RTE_INTR_HANDLE_VFIO_MSIX:
-	case RTE_INTR_HANDLE_VFIO_MSI:
-	case RTE_INTR_HANDLE_VFIO_LEGACY:
+	case RTE_KDRV_VFIO:
 		return pci_vfio_read_config(intr_handle, buf, len, offset);
 #endif
 	default:
+		rte_pci_device_name(&device->addr, devname,
+				    RTE_DEV_NAME_MAX_LEN);
 		RTE_LOG(ERR, EAL,
-			"Unknown handle type of fd %d\n",
-					intr_handle->fd);
+			"Unknown driver type for %s\n", devname);
 		return -1;
 	}
 }
@@ -697,23 +697,21 @@ int rte_pci_read_config(const struct rte_pci_device *device,
 int rte_pci_write_config(const struct rte_pci_device *device,
 		const void *buf, size_t len, off_t offset)
 {
+	char devname[RTE_DEV_NAME_MAX_LEN] = "";
 	const struct rte_intr_handle *intr_handle = &device->intr_handle;
 
-	switch (intr_handle->type) {
-	case RTE_INTR_HANDLE_UIO:
-	case RTE_INTR_HANDLE_UIO_INTX:
+	switch (device->kdrv) {
+	case RTE_KDRV_IGB_UIO:
 		return pci_uio_write_config(intr_handle, buf, len, offset);
-
 #ifdef VFIO_PRESENT
-	case RTE_INTR_HANDLE_VFIO_MSIX:
-	case RTE_INTR_HANDLE_VFIO_MSI:
-	case RTE_INTR_HANDLE_VFIO_LEGACY:
+	case RTE_KDRV_VFIO:
 		return pci_vfio_write_config(intr_handle, buf, len, offset);
 #endif
 	default:
+		rte_pci_device_name(&device->addr, devname,
+				    RTE_DEV_NAME_MAX_LEN);
 		RTE_LOG(ERR, EAL,
-			"Unknown handle type of fd %d\n",
-					intr_handle->fd);
+			"Unknown driver type for %s\n", devname);
 		return -1;
 	}
 }
diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 686386d6..305cc060 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -17,6 +17,8 @@
 #include <rte_eal_memconfig.h>
 #include <rte_malloc.h>
 #include <rte_vfio.h>
+#include <rte_eal.h>
+#include <rte_bus.h>
 
 #include "eal_filesystem.h"
 
@@ -35,7 +37,9 @@
 
 #ifdef VFIO_PRESENT
 
+#ifndef PAGE_SIZE
 #define PAGE_SIZE   (sysconf(_SC_PAGESIZE))
+#endif
 #define PAGE_MASK   (~(PAGE_SIZE - 1))
 
 static struct rte_tailq_elem rte_vfio_tailq = {
@@ -277,6 +281,114 @@ pci_vfio_setup_interrupts(struct rte_pci_device *dev, int vfio_dev_fd)
 	return -1;
 }
 
+#ifdef HAVE_VFIO_DEV_REQ_INTERFACE
+static void
+pci_vfio_req_handler(void *param)
+{
+	struct rte_bus *bus;
+	int ret;
+	struct rte_device *device = (struct rte_device *)param;
+
+	bus = rte_bus_find_by_device(device);
+	if (bus == NULL) {
+		RTE_LOG(ERR, EAL, "Cannot find bus for device (%s)\n",
+			device->name);
+		return;
+	}
+
+	/*
+	 * vfio kernel module request user space to release allocated
+	 * resources before device be deleted in kernel, so it can directly
+	 * call the vfio bus hot-unplug handler to process it.
+	 */
+	ret = bus->hot_unplug_handler(device);
+	if (ret)
+		RTE_LOG(ERR, EAL,
+			"Can not handle hot-unplug for device (%s)\n",
+			device->name);
+}
+
+/* enable notifier (only enable req now) */
+static int
+pci_vfio_enable_notifier(struct rte_pci_device *dev, int vfio_dev_fd)
+{
+	int ret;
+	int fd = -1;
+
+	/* set up an eventfd for req notifier */
+	fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+	if (fd < 0) {
+		RTE_LOG(ERR, EAL, "Cannot set up eventfd, error %i (%s)\n",
+			errno, strerror(errno));
+		return -1;
+	}
+
+	dev->vfio_req_intr_handle.fd = fd;
+	dev->vfio_req_intr_handle.type = RTE_INTR_HANDLE_VFIO_REQ;
+	dev->vfio_req_intr_handle.vfio_dev_fd = vfio_dev_fd;
+
+	ret = rte_intr_callback_register(&dev->vfio_req_intr_handle,
+					 pci_vfio_req_handler,
+					 (void *)&dev->device);
+	if (ret) {
+		RTE_LOG(ERR, EAL, "Fail to register req notifier handler.\n");
+		goto error;
+	}
+
+	ret = rte_intr_enable(&dev->vfio_req_intr_handle);
+	if (ret) {
+		RTE_LOG(ERR, EAL, "Fail to enable req notifier.\n");
+		ret = rte_intr_callback_unregister(&dev->vfio_req_intr_handle,
+						 pci_vfio_req_handler,
+						 (void *)&dev->device);
+		if (ret < 0)
+			RTE_LOG(ERR, EAL,
+				"Fail to unregister req notifier handler.\n");
+		goto error;
+	}
+
+	return 0;
+error:
+	close(fd);
+
+	dev->vfio_req_intr_handle.fd = -1;
+	dev->vfio_req_intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
+	dev->vfio_req_intr_handle.vfio_dev_fd = -1;
+
+	return -1;
+}
+
+/* disable notifier (only disable req now) */
+static int
+pci_vfio_disable_notifier(struct rte_pci_device *dev)
+{
+	int ret;
+
+	ret = rte_intr_disable(&dev->vfio_req_intr_handle);
+	if (ret) {
+		RTE_LOG(ERR, EAL, "fail to disable req notifier.\n");
+		return -1;
+	}
+
+	ret = rte_intr_callback_unregister(&dev->vfio_req_intr_handle,
+					   pci_vfio_req_handler,
+					   (void *)&dev->device);
+	if (ret < 0) {
+		RTE_LOG(ERR, EAL,
+			 "fail to unregister req notifier handler.\n");
+		return -1;
+	}
+
+	close(dev->vfio_req_intr_handle.fd);
+
+	dev->vfio_req_intr_handle.fd = -1;
+	dev->vfio_req_intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
+	dev->vfio_req_intr_handle.vfio_dev_fd = -1;
+
+	return 0;
+}
+#endif
+
 static int
 pci_vfio_is_ioport_bar(int vfio_dev_fd, int bar_index)
 {
@@ -415,6 +527,93 @@ pci_vfio_mmap_bar(int vfio_dev_fd, struct mapped_pci_resource *vfio_res,
 	return 0;
 }
 
+/*
+ * region info may contain capability headers, so we need to keep reallocating
+ * the memory until we match allocated memory size with argsz.
+ */
+static int
+pci_vfio_get_region_info(int vfio_dev_fd, struct vfio_region_info **info,
+		int region)
+{
+	struct vfio_region_info *ri;
+	size_t argsz = sizeof(*ri);
+	int ret;
+
+	ri = malloc(sizeof(*ri));
+	if (ri == NULL) {
+		RTE_LOG(ERR, EAL, "Cannot allocate memory for region info\n");
+		return -1;
+	}
+again:
+	memset(ri, 0, argsz);
+	ri->argsz = argsz;
+	ri->index = region;
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, ri);
+	if (ret < 0) {
+		free(ri);
+		return ret;
+	}
+	if (ri->argsz != argsz) {
+		struct vfio_region_info *tmp;
+
+		argsz = ri->argsz;
+		tmp = realloc(ri, argsz);
+
+		if (tmp == NULL) {
+			/* realloc failed but the ri is still there */
+			free(ri);
+			RTE_LOG(ERR, EAL, "Cannot reallocate memory for region info\n");
+			return -1;
+		}
+		ri = tmp;
+		goto again;
+	}
+	*info = ri;
+
+	return 0;
+}
+
+static struct vfio_info_cap_header *
+pci_vfio_info_cap(struct vfio_region_info *info, int cap)
+{
+	struct vfio_info_cap_header *h;
+	size_t offset;
+
+	if ((info->flags & RTE_VFIO_INFO_FLAG_CAPS) == 0) {
+		/* VFIO info does not advertise capabilities */
+		return NULL;
+	}
+
+	offset = VFIO_CAP_OFFSET(info);
+	while (offset != 0) {
+		h = RTE_PTR_ADD(info, offset);
+		if (h->id == cap)
+			return h;
+		offset = h->next;
+	}
+	return NULL;
+}
+
+static int
+pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
+{
+	struct vfio_region_info *info;
+	int ret;
+
+	ret = pci_vfio_get_region_info(vfio_dev_fd, &info, msix_region);
+	if (ret < 0)
+		return -1;
+
+	ret = pci_vfio_info_cap(info, RTE_VFIO_CAP_MSIX_MAPPABLE) != NULL;
+
+	/* cleanup */
+	free(info);
+
+	return ret;
+}
+
+
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
@@ -430,6 +629,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	struct pci_map *maps;
 
 	dev->intr_handle.fd = -1;
+#ifdef HAVE_VFIO_DEV_REQ_INTERFACE
+	dev->vfio_req_intr_handle.fd = -1;
+#endif
 
 	/* store PCI address string */
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
@@ -464,56 +666,75 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	if (ret < 0) {
 		RTE_LOG(ERR, EAL, "  %s cannot get MSI-X BAR number!\n",
 				pci_addr);
-		goto err_vfio_dev_fd;
+		goto err_vfio_res;
+	}
+	/* if we found our MSI-X BAR region, check if we can mmap it */
+	if (vfio_res->msix_table.bar_index != -1) {
+		int ret = pci_vfio_msix_is_mappable(vfio_dev_fd,
+				vfio_res->msix_table.bar_index);
+		if (ret < 0) {
+			RTE_LOG(ERR, EAL, "Couldn't check if MSI-X BAR is mappable\n");
+			goto err_vfio_res;
+		} else if (ret != 0) {
+			/* we can map it, so we don't care where it is */
+			RTE_LOG(DEBUG, EAL, "VFIO reports MSI-X BAR as mappable\n");
+			vfio_res->msix_table.bar_index = -1;
+		}
 	}
 
 	for (i = 0; i < (int) vfio_res->nb_maps; i++) {
-		struct vfio_region_info reg = { .argsz = sizeof(reg) };
+		struct vfio_region_info *reg = NULL;
 		void *bar_addr;
 
-		reg.index = i;
-
-		ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg);
-		if (ret) {
+		ret = pci_vfio_get_region_info(vfio_dev_fd, &reg, i);
+		if (ret < 0) {
 			RTE_LOG(ERR, EAL, "  %s cannot get device region info "
-					"error %i (%s)\n", pci_addr, errno, strerror(errno));
+				"error %i (%s)\n", pci_addr, errno,
+				strerror(errno));
 			goto err_vfio_res;
 		}
 
 		/* chk for io port region */
 		ret = pci_vfio_is_ioport_bar(vfio_dev_fd, i);
-		if (ret < 0)
+		if (ret < 0) {
+			free(reg);
 			goto err_vfio_res;
-		else if (ret) {
+		} else if (ret) {
 			RTE_LOG(INFO, EAL, "Ignore mapping IO port bar(%d)\n",
 					i);
+			free(reg);
 			continue;
 		}
 
 		/* skip non-mmapable BARs */
-		if ((reg.flags & VFIO_REGION_INFO_FLAG_MMAP) == 0)
+		if ((reg->flags & VFIO_REGION_INFO_FLAG_MMAP) == 0) {
+			free(reg);
 			continue;
+		}
 
 		/* try mapping somewhere close to the end of hugepages */
 		if (pci_map_addr == NULL)
 			pci_map_addr = pci_find_max_end_va();
 
 		bar_addr = pci_map_addr;
-		pci_map_addr = RTE_PTR_ADD(bar_addr, (size_t) reg.size);
+		pci_map_addr = RTE_PTR_ADD(bar_addr, (size_t) reg->size);
 
 		maps[i].addr = bar_addr;
-		maps[i].offset = reg.offset;
-		maps[i].size = reg.size;
+		maps[i].offset = reg->offset;
+		maps[i].size = reg->size;
 		maps[i].path = NULL; /* vfio doesn't have per-resource paths */
 
 		ret = pci_vfio_mmap_bar(vfio_dev_fd, vfio_res, i, 0);
 		if (ret < 0) {
 			RTE_LOG(ERR, EAL, "  %s mapping BAR%i failed: %s\n",
 					pci_addr, i, strerror(errno));
+			free(reg);
 			goto err_vfio_res;
 		}
 
 		dev->mem_resource[i].addr = maps[i].addr;
+
+		free(reg);
 	}
 
 	if (pci_rte_vfio_setup_device(dev, vfio_dev_fd) < 0) {
@@ -521,6 +742,13 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 		goto err_vfio_res;
 	}
 
+#ifdef HAVE_VFIO_DEV_REQ_INTERFACE
+	if (pci_vfio_enable_notifier(dev, vfio_dev_fd) != 0) {
+		RTE_LOG(ERR, EAL, "Error setting up notifier!\n");
+		goto err_vfio_res;
+	}
+
+#endif
 	TAILQ_INSERT_TAIL(vfio_res_list, vfio_res, next);
 
 	return 0;
@@ -546,6 +774,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 	struct pci_map *maps;
 
 	dev->intr_handle.fd = -1;
+#ifdef HAVE_VFIO_DEV_REQ_INTERFACE
+	dev->vfio_req_intr_handle.fd = -1;
+#endif
 
 	/* store PCI address string */
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
@@ -586,6 +817,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 
 	/* we need save vfio_dev_fd, so it can be used during release */
 	dev->intr_handle.vfio_dev_fd = vfio_dev_fd;
+#ifdef HAVE_VFIO_DEV_REQ_INTERFACE
+	dev->vfio_req_intr_handle.vfio_dev_fd = vfio_dev_fd;
+#endif
 
 	return 0;
 err_vfio_dev_fd:
@@ -658,6 +892,14 @@ pci_vfio_unmap_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+#ifdef HAVE_VFIO_DEV_REQ_INTERFACE
+	ret = pci_vfio_disable_notifier(dev);
+	if (ret) {
+		RTE_LOG(ERR, EAL, "fail to disable req notifier.\n");
+		return -1;
+	}
+
+#endif
 	if (close(dev->intr_handle.fd) < 0) {
 		RTE_LOG(INFO, EAL, "Error when closing eventfd file descriptor for %s\n",
 			pci_addr);
diff --git a/drivers/bus/pci/meson.build b/drivers/bus/pci/meson.build
index 72939e59..a3140ff9 100644
--- a/drivers/bus/pci/meson.build
+++ b/drivers/bus/pci/meson.build
@@ -1,15 +1,18 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
+version = 2
+
 deps += ['pci']
 install_headers('rte_bus_pci.h')
-sources = files('pci_common.c', 'pci_common_uio.c')
+sources = files('pci_common.c',
+	'pci_common_uio.c',
+	'pci_params.c')
 if host_machine.system() == 'linux'
 	sources += files('linux/pci.c',
 			'linux/pci_uio.c',
 			'linux/pci_vfio.c')
 	includes += include_directories('linux')
-	cflags += ['-D_GNU_SOURCE']
 else
 	sources += files('bsd/pci.c')
 	includes += include_directories('bsd')
@@ -17,3 +20,5 @@ endif
 
 # memseg walk is not part of stable API yet
 allow_experimental_apis = true
+
+deps += ['kvargs']
diff --git a/drivers/bus/pci/pci_common.c b/drivers/bus/pci/pci_common.c
index 7736b3f9..6276e5d6 100644
--- a/drivers/bus/pci/pci_common.c
+++ b/drivers/bus/pci/pci_common.c
@@ -6,6 +6,7 @@
 #include <string.h>
 #include <inttypes.h>
 #include <stdint.h>
+#include <stdbool.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <sys/queue.h>
@@ -23,12 +24,11 @@
 #include <rte_string_fns.h>
 #include <rte_common.h>
 #include <rte_devargs.h>
+#include <rte_vfio.h>
 
 #include "private.h"
 
 
-extern struct rte_pci_bus rte_pci_bus;
-
 #define SYSFS_PCI_DEVICES "/sys/bus/pci/devices"
 
 const char *rte_pci_get_sysfs_path(void)
@@ -123,6 +123,7 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr,
 			 struct rte_pci_device *dev)
 {
 	int ret;
+	bool already_probed;
 	struct rte_pci_addr *loc;
 
 	if ((dr == NULL) || (dev == NULL))
@@ -153,6 +154,13 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr,
 		dev->device.numa_node = 0;
 	}
 
+	already_probed = rte_dev_is_probed(&dev->device);
+	if (already_probed && !(dr->drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) {
+		RTE_LOG(DEBUG, EAL, "Device %s is already probed\n",
+				dev->device.name);
+		return -EEXIST;
+	}
+
 	RTE_LOG(INFO, EAL, "  probe driver: %x:%x %s\n", dev->id.vendor_id,
 		dev->id.device_id, dr->driver.name);
 
@@ -161,24 +169,24 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr,
 	 * This needs to be before rte_pci_map_device(), as it enables to use
 	 * driver flags for adjusting configuration.
 	 */
-	dev->driver = dr;
-	dev->device.driver = &dr->driver;
+	if (!already_probed)
+		dev->driver = dr;
 
-	if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) {
+	if (!already_probed && (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING)) {
 		/* map resources for devices that use igb_uio */
 		ret = rte_pci_map_device(dev);
 		if (ret != 0) {
 			dev->driver = NULL;
-			dev->device.driver = NULL;
 			return ret;
 		}
 	}
 
 	/* call the driver probe() function */
 	ret = dr->probe(dr, dev);
+	if (already_probed)
+		return ret; /* no rollback if already succeeded earlier */
 	if (ret) {
 		dev->driver = NULL;
-		dev->device.driver = NULL;
 		if ((dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) &&
 			/* Don't unmap if device is unsupported and
 			 * driver needs mapped resources.
@@ -186,6 +194,8 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr,
 			!(ret > 0 &&
 				(dr->drv_flags & RTE_PCI_DRV_KEEP_MAPPED_RES)))
 			rte_pci_unmap_device(dev);
+	} else {
+		dev->device.driver = &dr->driver;
 	}
 
 	return ret;
@@ -233,7 +243,7 @@ rte_pci_detach_dev(struct rte_pci_device *dev)
 
 /*
  * If vendor/device ID match, call the probe() function of all
- * registered driver for the given device. Return -1 if initialization
+ * registered driver for the given device. Return < 0 if initialization
  * failed, return 1 if no driver is found for this device.
  */
 static int
@@ -243,17 +253,13 @@ pci_probe_all_drivers(struct rte_pci_device *dev)
 	int rc = 0;
 
 	if (dev == NULL)
-		return -1;
-
-	/* Check if a driver is already loaded */
-	if (dev->driver != NULL)
-		return 0;
+		return -EINVAL;
 
 	FOREACH_DRIVER_ON_PCIBUS(dr) {
 		rc = rte_pci_probe_one_driver(dr, dev);
 		if (rc < 0)
 			/* negative value is an error */
-			return -1;
+			return rc;
 		if (rc > 0)
 			/* positive value means driver doesn't support it */
 			continue;
@@ -290,11 +296,14 @@ rte_pci_probe(void)
 			devargs->policy == RTE_DEV_WHITELISTED)
 			ret = pci_probe_all_drivers(dev);
 		if (ret < 0) {
-			RTE_LOG(ERR, EAL, "Requested device " PCI_PRI_FMT
-				 " cannot be used\n", dev->addr.domain, dev->addr.bus,
-				 dev->addr.devid, dev->addr.function);
-			rte_errno = errno;
-			failed++;
+			if (ret != -EEXIST) {
+				RTE_LOG(ERR, EAL, "Requested device "
+					PCI_PRI_FMT " cannot be used\n",
+					dev->addr.domain, dev->addr.bus,
+					dev->addr.devid, dev->addr.function);
+				rte_errno = errno;
+				failed++;
+			}
 			ret = 0;
 		}
 	}
@@ -405,6 +414,98 @@ pci_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
 	return NULL;
 }
 
+/*
+ * find the device which encounter the failure, by iterate over all device on
+ * PCI bus to check if the memory failure address is located in the range
+ * of the BARs of the device.
+ */
+static struct rte_pci_device *
+pci_find_device_by_addr(const void *failure_addr)
+{
+	struct rte_pci_device *pdev = NULL;
+	uint64_t check_point, start, end, len;
+	int i;
+
+	check_point = (uint64_t)(uintptr_t)failure_addr;
+
+	FOREACH_DEVICE_ON_PCIBUS(pdev) {
+		for (i = 0; i != RTE_DIM(pdev->mem_resource); i++) {
+			start = (uint64_t)(uintptr_t)pdev->mem_resource[i].addr;
+			len = pdev->mem_resource[i].len;
+			end = start + len;
+			if (check_point >= start && check_point < end) {
+				RTE_LOG(DEBUG, EAL, "Failure address %16.16"
+					PRIx64" belongs to device %s!\n",
+					check_point, pdev->device.name);
+				return pdev;
+			}
+		}
+	}
+	return NULL;
+}
+
+static int
+pci_hot_unplug_handler(struct rte_device *dev)
+{
+	struct rte_pci_device *pdev = NULL;
+	int ret = 0;
+
+	pdev = RTE_DEV_TO_PCI(dev);
+	if (!pdev)
+		return -1;
+
+	switch (pdev->kdrv) {
+#ifdef HAVE_VFIO_DEV_REQ_INTERFACE
+	case RTE_KDRV_VFIO:
+		/*
+		 * vfio kernel module guaranty the pci device would not be
+		 * deleted until the user space release the resource, so no
+		 * need to remap BARs resource here, just directly notify
+		 * the req event to the user space to handle it.
+		 */
+		rte_dev_event_callback_process(dev->name,
+					       RTE_DEV_EVENT_REMOVE);
+		break;
+#endif
+	case RTE_KDRV_IGB_UIO:
+	case RTE_KDRV_UIO_GENERIC:
+	case RTE_KDRV_NIC_UIO:
+		/* BARs resource is invalid, remap it to be safe. */
+		ret = pci_uio_remap_resource(pdev);
+		break;
+	default:
+		RTE_LOG(DEBUG, EAL,
+			"Not managed by a supported kernel driver, skipped\n");
+		ret = -1;
+		break;
+	}
+
+	return ret;
+}
+
+static int
+pci_sigbus_handler(const void *failure_addr)
+{
+	struct rte_pci_device *pdev = NULL;
+	int ret = 0;
+
+	pdev = pci_find_device_by_addr(failure_addr);
+	if (!pdev) {
+		/* It is a generic sigbus error, no bus would handle it. */
+		ret = 1;
+	} else {
+		/* The sigbus error is caused of hot-unplug. */
+		ret = pci_hot_unplug_handler(&pdev->device);
+		if (ret) {
+			RTE_LOG(ERR, EAL,
+				"Failed to handle hot-unplug for device %s",
+				pdev->name);
+			ret = -1;
+		}
+	}
+	return ret;
+}
+
 static int
 pci_plug(struct rte_device *dev)
 {
@@ -421,6 +522,7 @@ pci_unplug(struct rte_device *dev)
 	ret = rte_pci_detach_dev(pdev);
 	if (ret == 0) {
 		rte_pci_remove_device(pdev);
+		rte_devargs_remove(dev->devargs);
 		free(pdev);
 	}
 	return ret;
@@ -435,6 +537,9 @@ struct rte_pci_bus rte_pci_bus = {
 		.unplug = pci_unplug,
 		.parse = pci_parse,
 		.get_iommu_class = rte_pci_get_iommu_class,
+		.dev_iterate = rte_pci_dev_iterate,
+		.hot_unplug_handler = pci_hot_unplug_handler,
+		.sigbus_handler = pci_sigbus_handler,
 	},
 	.device_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.device_list),
 	.driver_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.driver_list),
diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c
index 54bc20b5..7ea73dbc 100644
--- a/drivers/bus/pci/pci_common_uio.c
+++ b/drivers/bus/pci/pci_common_uio.c
@@ -146,6 +146,39 @@ pci_uio_unmap(struct mapped_pci_resource *uio_res)
 	}
 }
 
+/* remap the PCI resource of a PCI device in anonymous virtual memory */
+int
+pci_uio_remap_resource(struct rte_pci_device *dev)
+{
+	int i;
+	void *map_address;
+
+	if (dev == NULL)
+		return -1;
+
+	/* Remap all BARs */
+	for (i = 0; i != PCI_MAX_RESOURCE; i++) {
+		/* skip empty BAR */
+		if (dev->mem_resource[i].phys_addr == 0)
+			continue;
+		map_address = mmap(dev->mem_resource[i].addr,
+				(size_t)dev->mem_resource[i].len,
+				PROT_READ | PROT_WRITE,
+				MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
+		if (map_address == MAP_FAILED) {
+			RTE_LOG(ERR, EAL,
+				"Cannot remap resource for device %s\n",
+				dev->name);
+			return -1;
+		}
+		RTE_LOG(INFO, EAL,
+			"Successful remap resource for device %s\n",
+			dev->name);
+	}
+
+	return 0;
+}
+
 static struct mapped_pci_resource *
 pci_uio_find_resource(struct rte_pci_device *dev)
 {
diff --git a/drivers/bus/pci/pci_params.c b/drivers/bus/pci/pci_params.c
new file mode 100644
index 00000000..3192e9c9
--- /dev/null
+++ b/drivers/bus/pci/pci_params.c
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Gaëtan Rivet
+ */
+
+#include <rte_bus.h>
+#include <rte_bus_pci.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_pci.h>
+
+#include "private.h"
+
+enum pci_params {
+	RTE_PCI_PARAM_ADDR,
+	RTE_PCI_PARAM_MAX,
+};
+
+static const char * const pci_params_keys[] = {
+	[RTE_PCI_PARAM_ADDR] = "addr",
+	[RTE_PCI_PARAM_MAX] = NULL,
+};
+
+static int
+pci_addr_kv_cmp(const char *key __rte_unused,
+		const char *value,
+		void *_addr2)
+{
+	struct rte_pci_addr _addr1;
+	struct rte_pci_addr *addr1 = &_addr1;
+	struct rte_pci_addr *addr2 = _addr2;
+
+	if (rte_pci_addr_parse(value, addr1))
+		return -1;
+	return -abs(rte_pci_addr_cmp(addr1, addr2));
+}
+
+static int
+pci_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	const struct rte_kvargs *kvlist = _kvlist;
+	const struct rte_pci_device *pdev;
+
+	if (kvlist == NULL)
+		/* Empty string matches everything. */
+		return 0;
+	pdev = RTE_DEV_TO_PCI_CONST(dev);
+	/* if any field does not match. */
+	if (rte_kvargs_process(kvlist, pci_params_keys[RTE_PCI_PARAM_ADDR],
+			       &pci_addr_kv_cmp,
+			       (void *)(intptr_t)&pdev->addr))
+		return 1;
+	return 0;
+}
+
+void *
+rte_pci_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, pci_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = rte_pci_bus.bus.find_device;
+	dev = find_device(start, pci_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/pci/private.h b/drivers/bus/pci/private.h
index 8ddd03e1..13c3324b 100644
--- a/drivers/bus/pci/private.h
+++ b/drivers/bus/pci/private.h
@@ -10,9 +10,13 @@
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
 
+extern struct rte_pci_bus rte_pci_bus;
+
 struct rte_pci_driver;
 struct rte_pci_device;
 
+extern struct rte_pci_bus rte_pci_bus;
+
 /**
  * Probe the PCI bus
  *
@@ -123,6 +127,18 @@ void pci_uio_free_resource(struct rte_pci_device *dev,
 		struct mapped_pci_resource *uio_res);
 
 /**
+ * Remap the PCI resource of a PCI device in anonymous virtual memory.
+ *
+ * @param dev
+ *   Point to the struct rte pci device.
+ * @return
+ *   - On success, zero.
+ *   - On failure, a negative value.
+ */
+int
+pci_uio_remap_resource(struct rte_pci_device *dev);
+
+/**
  * Map device memory to uio resource
  *
  * This function is private to EAL.
@@ -166,4 +182,27 @@ rte_pci_match(const struct rte_pci_driver *pci_drv,
 enum rte_iova_mode
 rte_pci_get_iommu_class(void);
 
+/*
+ * Iterate over internal devices,
+ * matching any device against the provided
+ * string.
+ *
+ * @param start
+ *   Iteration starting point.
+ *
+ * @param str
+ *   Device string to match against.
+ *
+ * @param it
+ *   (unused) iterator structure.
+ *
+ * @return
+ *   A pointer to the next matching device if any.
+ *   NULL otherwise.
+ */
+void *
+rte_pci_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it);
+
 #endif /* _PCI_PRIVATE_H_ */
diff --git a/drivers/bus/pci/rte_bus_pci.h b/drivers/bus/pci/rte_bus_pci.h
index 0d1955ff..f0d6d81c 100644
--- a/drivers/bus/pci/rte_bus_pci.h
+++ b/drivers/bus/pci/rte_bus_pci.h
@@ -62,10 +62,12 @@ struct rte_pci_device {
 	struct rte_mem_resource mem_resource[PCI_MAX_RESOURCE];
 					    /**< PCI Memory Resource */
 	struct rte_intr_handle intr_handle; /**< Interrupt handle */
-	struct rte_pci_driver *driver;      /**< Associated driver */
+	struct rte_pci_driver *driver;      /**< PCI driver used in probing */
 	uint16_t max_vfs;                   /**< sriov enable if not zero */
 	enum rte_kernel_driver kdrv;        /**< Kernel driver passthrough */
 	char name[PCI_PRI_STR_SIZE+1];      /**< PCI location (ASCII) */
+	struct rte_intr_handle vfio_req_intr_handle;
+				/**< Handler of VFIO request interrupt */
 };
 
 /**
@@ -121,7 +123,7 @@ struct rte_pci_driver {
 	pci_probe_t *probe;                /**< Device Probe function. */
 	pci_remove_t *remove;              /**< Device Remove function. */
 	const struct rte_pci_id *id_table; /**< ID table, NULL terminated. */
-	uint32_t drv_flags;                /**< Flags contolling handling of device. */
+	uint32_t drv_flags;                /**< Flags RTE_PCI_DRV_*. */
 };
 
 /**
@@ -137,6 +139,8 @@ struct rte_pci_bus {
 #define RTE_PCI_DRV_NEED_MAPPING 0x0001
 /** Device needs PCI BAR mapping with enabled write combining (wc) */
 #define RTE_PCI_DRV_WC_ACTIVATE 0x0002
+/** Device already probed can be probed again to check for new ports. */
+#define RTE_PCI_DRV_PROBE_AGAIN 0x0004
 /** Device driver supports link state interrupt */
 #define RTE_PCI_DRV_INTR_LSC	0x0008
 /** Device driver supports device removal interrupt */
@@ -219,6 +223,8 @@ void rte_pci_unregister(struct rte_pci_driver *driver);
  *   The length of the data buffer.
  * @param offset
  *   The offset into PCI config space
+ * @return
+ *  Number of bytes read on success, negative on error.
  */
 int rte_pci_read_config(const struct rte_pci_device *device,
 		void *buf, size_t len, off_t offset);
diff --git a/drivers/bus/vdev/Makefile b/drivers/bus/vdev/Makefile
index bd0bb895..803b8ea7 100644
--- a/drivers/bus/vdev/Makefile
+++ b/drivers/bus/vdev/Makefile
@@ -16,11 +16,12 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 EXPORT_MAP := rte_bus_vdev_version.map
 
 # library version
-LIBABIVER := 1
+LIBABIVER := 2
 
 SRCS-y += vdev.c
+SRCS-y += vdev_params.c
 
-LDLIBS += -lrte_eal
+LDLIBS += -lrte_eal -lrte_kvargs
 
 #
 # Export include files
diff --git a/drivers/bus/vdev/meson.build b/drivers/bus/vdev/meson.build
index 2ee648b4..803785f1 100644
--- a/drivers/bus/vdev/meson.build
+++ b/drivers/bus/vdev/meson.build
@@ -1,7 +1,12 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-sources = files('vdev.c')
+version = 2
+
+sources = files('vdev.c',
+	'vdev_params.c')
 install_headers('rte_bus_vdev.h')
 
 allow_experimental_apis = true
+
+deps += ['kvargs']
diff --git a/drivers/bus/vdev/vdev.c b/drivers/bus/vdev/vdev.c
index 6139dd55..9c66bdc7 100644
--- a/drivers/bus/vdev/vdev.c
+++ b/drivers/bus/vdev/vdev.c
@@ -23,6 +23,7 @@
 
 #include "rte_bus_vdev.h"
 #include "vdev_logs.h"
+#include "vdev_private.h"
 
 #define VDEV_MP_KEY	"bus_vdev_mp"
 
@@ -40,7 +41,7 @@ static struct vdev_device_list vdev_device_list =
 static rte_spinlock_recursive_t vdev_device_list_lock =
 	RTE_SPINLOCK_RECURSIVE_INITIALIZER;
 
-struct vdev_driver_list vdev_driver_list =
+static struct vdev_driver_list vdev_driver_list =
 	TAILQ_HEAD_INITIALIZER(vdev_driver_list);
 
 struct vdev_custom_scan {
@@ -149,10 +150,9 @@ vdev_probe_all_drivers(struct rte_vdev_device *dev)
 
 	if (vdev_parse(name, &driver))
 		return -1;
-	dev->device.driver = &driver->driver;
 	ret = driver->probe(dev);
-	if (ret)
-		dev->device.driver = NULL;
+	if (ret == 0)
+		dev->device.driver = &driver->driver;
 	return ret;
 }
 
@@ -202,7 +202,9 @@ alloc_devargs(const char *name, const char *args)
 }
 
 static int
-insert_vdev(const char *name, const char *args, struct rte_vdev_device **p_dev)
+insert_vdev(const char *name, const char *args,
+		struct rte_vdev_device **p_dev,
+		bool init)
 {
 	struct rte_vdev_device *dev;
 	struct rte_devargs *devargs;
@@ -221,17 +223,24 @@ insert_vdev(const char *name, const char *args, struct rte_vdev_device **p_dev)
 		goto fail;
 	}
 
+	dev->device.bus = &rte_vdev_bus;
 	dev->device.devargs = devargs;
 	dev->device.numa_node = SOCKET_ID_ANY;
 	dev->device.name = devargs->name;
 
 	if (find_vdev(name)) {
+		/*
+		 * A vdev is expected to have only one port.
+		 * So there is no reason to try probing again,
+		 * even with new arguments.
+		 */
 		ret = -EEXIST;
 		goto fail;
 	}
 
 	TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
-	rte_devargs_insert(devargs);
+	if (init)
+		rte_devargs_insert(devargs);
 
 	if (p_dev)
 		*p_dev = dev;
@@ -248,20 +257,18 @@ int
 rte_vdev_init(const char *name, const char *args)
 {
 	struct rte_vdev_device *dev;
-	struct rte_devargs *devargs;
 	int ret;
 
 	rte_spinlock_recursive_lock(&vdev_device_list_lock);
-	ret = insert_vdev(name, args, &dev);
+	ret = insert_vdev(name, args, &dev, true);
 	if (ret == 0) {
 		ret = vdev_probe_all_drivers(dev);
 		if (ret) {
 			if (ret > 0)
 				VDEV_LOG(ERR, "no driver found for %s", name);
 			/* If fails, remove it from vdev list */
-			devargs = dev->device.devargs;
 			TAILQ_REMOVE(&vdev_device_list, dev, next);
-			rte_devargs_remove(devargs->bus->name, devargs->name);
+			rte_devargs_remove(dev->device.devargs);
 			free(dev);
 		}
 	}
@@ -289,7 +296,6 @@ int
 rte_vdev_uninit(const char *name)
 {
 	struct rte_vdev_device *dev;
-	struct rte_devargs *devargs;
 	int ret;
 
 	if (name == NULL)
@@ -308,8 +314,7 @@ rte_vdev_uninit(const char *name)
 		goto unlock;
 
 	TAILQ_REMOVE(&vdev_device_list, dev, next);
-	devargs = dev->device.devargs;
-	rte_devargs_remove(devargs->bus->name, devargs->name);
+	rte_devargs_remove(dev->device.devargs);
 	free(dev);
 
 unlock:
@@ -346,6 +351,7 @@ vdev_action(const struct rte_mp_msg *mp_msg, const void *peer)
 	const struct vdev_param *in = (const struct vdev_param *)mp_msg->param;
 	const char *devname;
 	int num;
+	int ret;
 
 	strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name));
 	mp_resp.len_param = sizeof(*ou);
@@ -380,7 +386,10 @@ vdev_action(const struct rte_mp_msg *mp_msg, const void *peer)
 		break;
 	case VDEV_SCAN_ONE:
 		VDEV_LOG(INFO, "receive vdev, %s", in->name);
-		if (insert_vdev(in->name, NULL, NULL) < 0)
+		ret = insert_vdev(in->name, NULL, NULL, false);
+		if (ret == -EEXIST)
+			VDEV_LOG(DEBUG, "device already exist, %s", in->name);
+		else if (ret < 0)
 			VDEV_LOG(ERR, "failed to add vdev, %s", in->name);
 		break;
 	default:
@@ -419,6 +428,7 @@ vdev_scan(void)
 			mp_rep = &mp_reply.msgs[0];
 			resp = (struct vdev_param *)mp_rep->param;
 			VDEV_LOG(INFO, "Received %d vdevs", resp->num);
+			free(mp_reply.msgs);
 		} else
 			VDEV_LOG(ERR, "Failed to request vdev from primary");
 
@@ -455,6 +465,7 @@ vdev_scan(void)
 			continue;
 		}
 
+		dev->device.bus = &rte_vdev_bus;
 		dev->device.devargs = devargs;
 		dev->device.numa_node = SOCKET_ID_ANY;
 		dev->device.name = devargs->name;
@@ -480,7 +491,7 @@ vdev_probe(void)
 		 * we call each driver probe.
 		 */
 
-		if (dev->device.driver)
+		if (rte_dev_is_probed(&dev->device))
 			continue;
 
 		if (vdev_probe_all_drivers(dev)) {
@@ -493,9 +504,9 @@ vdev_probe(void)
 	return ret;
 }
 
-static struct rte_device *
-vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
-		 const void *data)
+struct rte_device *
+rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		     const void *data)
 {
 	const struct rte_vdev_device *vstart;
 	struct rte_vdev_device *dev;
@@ -532,10 +543,11 @@ vdev_unplug(struct rte_device *dev)
 static struct rte_bus rte_vdev_bus = {
 	.scan = vdev_scan,
 	.probe = vdev_probe,
-	.find_device = vdev_find_device,
+	.find_device = rte_vdev_find_device,
 	.plug = vdev_plug,
 	.unplug = vdev_unplug,
 	.parse = vdev_parse,
+	.dev_iterate = rte_vdev_dev_iterate,
 };
 
 RTE_REGISTER_BUS(vdev, rte_vdev_bus);
diff --git a/drivers/bus/vdev/vdev_params.c b/drivers/bus/vdev/vdev_params.c
new file mode 100644
index 00000000..6f74704d
--- /dev/null
+++ b/drivers/bus/vdev/vdev_params.c
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Gaëtan Rivet
+ */
+
+#include <string.h>
+
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+#include <rte_errno.h>
+
+#include "vdev_logs.h"
+#include "vdev_private.h"
+
+enum vdev_params {
+	RTE_VDEV_PARAM_NAME,
+	RTE_VDEV_PARAM_MAX,
+};
+
+static const char * const vdev_params_keys[] = {
+	[RTE_VDEV_PARAM_NAME] = "name",
+	[RTE_VDEV_PARAM_MAX] = NULL,
+};
+
+static int
+vdev_dev_match(const struct rte_device *dev,
+	       const void *_kvlist)
+{
+	int ret;
+	const struct rte_kvargs *kvlist = _kvlist;
+	char *name;
+
+	/* cannot pass const dev->name to rte_kvargs_process() */
+	name = strdup(dev->name);
+	if (name == NULL)
+		return -1;
+	ret = rte_kvargs_process(kvlist,
+		vdev_params_keys[RTE_VDEV_PARAM_NAME],
+		rte_kvargs_strcmp, name);
+	free(name);
+	if (ret != 0)
+		return -1;
+
+	return 0;
+}
+
+void *
+rte_vdev_dev_iterate(const void *start,
+		     const char *str,
+		     const struct rte_dev_iterator *it __rte_unused)
+{
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, vdev_params_keys);
+		if (kvargs == NULL) {
+			VDEV_LOG(ERR, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	dev = rte_vdev_find_device(start, vdev_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/vdev/vdev_private.h b/drivers/bus/vdev/vdev_private.h
new file mode 100644
index 00000000..ba6dc48f
--- /dev/null
+++ b/drivers/bus/vdev/vdev_private.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Gaëtan Rivet
+ */
+
+#ifndef _VDEV_PRIVATE_H_
+#define _VDEV_PRIVATE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rte_device *
+rte_vdev_find_device(const struct rte_device *start,
+		     rte_dev_cmp_t cmp,
+		     const void *data);
+
+void *
+rte_vdev_dev_iterate(const void *start,
+		     const char *str,
+		     const struct rte_dev_iterator *it);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _VDEV_PRIVATE_H_ */
diff --git a/drivers/bus/vmbus/Makefile b/drivers/bus/vmbus/Makefile
index deee9dd1..e54c557c 100644
--- a/drivers/bus/vmbus/Makefile
+++ b/drivers/bus/vmbus/Makefile
@@ -3,7 +3,7 @@
 include $(RTE_SDK)/mk/rte.vars.mk
 
 LIB = librte_bus_vmbus.a
-LIBABIVER := 1
+LIBABIVER := 2
 EXPORT_MAP := rte_bus_vmbus_version.map
 
 CFLAGS += -I$(SRCDIR)
diff --git a/drivers/bus/vmbus/linux/vmbus_bus.c b/drivers/bus/vmbus/linux/vmbus_bus.c
index 52d6a3c0..a4755a38 100644
--- a/drivers/bus/vmbus/linux/vmbus_bus.c
+++ b/drivers/bus/vmbus/linux/vmbus_bus.c
@@ -229,6 +229,7 @@ vmbus_scan_one(const char *name)
 	if (dev == NULL)
 		return -1;
 
+	dev->device.bus = &rte_vmbus_bus.bus;
 	dev->device.name = strdup(name);
 	if (!dev->device.name)
 		goto error;
@@ -276,6 +277,8 @@ vmbus_scan_one(const char *name)
 		dev->device.numa_node = SOCKET_ID_ANY;
 	}
 
+	dev->device.devargs = vmbus_devargs_lookup(dev);
+
 	/* device is valid, add in list (sorted) */
 	VMBUS_LOG(DEBUG, "Adding vmbus device %s", name);
 
diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build
index 18daabec..0e4d058e 100644
--- a/drivers/bus/vmbus/meson.build
+++ b/drivers/bus/vmbus/meson.build
@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
+version = 2
+
 allow_experimental_apis = true
 
 install_headers('rte_bus_vmbus.h','rte_vmbus_reg.h')
diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h
index 9964fc42..211127dd 100644
--- a/drivers/bus/vmbus/private.h
+++ b/drivers/bus/vmbus/private.h
@@ -10,11 +10,14 @@
 #include <sys/uio.h>
 #include <rte_log.h>
 #include <rte_vmbus_reg.h>
+#include <rte_bus_vmbus.h>
 
 #ifndef PAGE_SIZE
 #define PAGE_SIZE	4096
 #endif
 
+extern struct rte_vmbus_bus rte_vmbus_bus;
+
 extern int vmbus_logtype_bus;
 #define VMBUS_LOG(level, fmt, args...) \
 	rte_log(RTE_LOG_ ## level, vmbus_logtype_bus, "%s(): " fmt "\n", \
@@ -66,6 +69,9 @@ struct vmbus_channel {
 
 #define VMBUS_MAX_CHANNELS	64
 
+struct rte_devargs *
+vmbus_devargs_lookup(struct rte_vmbus_device *dev);
+
 int vmbus_chan_create(const struct rte_vmbus_device *device,
 		      uint16_t relid, uint16_t subid, uint8_t monitor_id,
 		      struct vmbus_channel **new_chan);
diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h b/drivers/bus/vmbus/rte_bus_vmbus.h
index 4a2c1f6f..2839fef5 100644
--- a/drivers/bus/vmbus/rte_bus_vmbus.h
+++ b/drivers/bus/vmbus/rte_bus_vmbus.h
@@ -365,6 +365,21 @@ void rte_vmbus_chan_signal_read(struct vmbus_channel *chan, uint32_t bytes_read)
 uint16_t rte_vmbus_sub_channel_index(const struct vmbus_channel *chan);
 
 /**
+ * Set the host monitor latency hint
+ *
+ * @param dev
+ *    VMBUS device
+ * @param chan
+ *	Pointer to vmbus_channel structure.
+ * @param latency
+ *	Approximate wait period between hypervisor examinations of
+ *	the trigger page (in nanoseconds).
+ */
+void rte_vmbus_set_latency(const struct rte_vmbus_device *dev,
+			   const struct vmbus_channel *chan,
+			   uint32_t latency);
+
+/**
  * Register a VMBUS driver.
  *
  * @param driver
diff --git a/drivers/bus/vmbus/rte_bus_vmbus_version.map b/drivers/bus/vmbus/rte_bus_vmbus_version.map
index dabb9203..ae231ad3 100644
--- a/drivers/bus/vmbus/rte_bus_vmbus_version.map
+++ b/drivers/bus/vmbus/rte_bus_vmbus_version.map
@@ -27,3 +27,10 @@ DPDK_18.08 {
 
 	local: *;
 };
+
+DPDK_18.11 {
+	global:
+
+	rte_vmbus_set_latency;
+
+} DPDK_18.08;
diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c
index cc5f3e83..bd14c066 100644
--- a/drivers/bus/vmbus/vmbus_channel.c
+++ b/drivers/bus/vmbus/vmbus_channel.c
@@ -60,6 +60,32 @@ vmbus_set_event(const struct rte_vmbus_device *dev,
 }
 
 /*
+ * Set the wait between when hypervisor examines the trigger.
+ */
+void
+rte_vmbus_set_latency(const struct rte_vmbus_device *dev,
+		      const struct vmbus_channel *chan,
+		      uint32_t latency)
+{
+	uint32_t trig_idx = chan->monitor_id / VMBUS_MONTRIG_LEN;
+	uint32_t trig_offs = chan->monitor_id % VMBUS_MONTRIG_LEN;
+
+	if (latency >= UINT16_MAX * 100) {
+		VMBUS_LOG(ERR, "invalid latency value %u", latency);
+		return;
+	}
+
+	if (trig_idx >= VMBUS_MONTRIGS_MAX) {
+		VMBUS_LOG(ERR, "invalid monitor trigger %u",
+			  trig_idx);
+		return;
+	}
+
+	/* Host value is expressed in 100 nanosecond units */
+	dev->monitor_page->lat[trig_idx][trig_offs] = latency / 100;
+}
+
+/*
  * Notify host that there are data pending on our TX bufring.
  *
  * Since this in userspace, rely on the monitor page.
diff --git a/drivers/bus/vmbus/vmbus_common.c b/drivers/bus/vmbus/vmbus_common.c
index c7165ad5..48a219f7 100644
--- a/drivers/bus/vmbus/vmbus_common.c
+++ b/drivers/bus/vmbus/vmbus_common.c
@@ -85,7 +85,6 @@ vmbus_match(const struct rte_vmbus_driver *dr,
 
 	return false;
 }
-
 /*
  * If device ID match, call the devinit() function of the driver.
  */
@@ -112,7 +111,6 @@ vmbus_probe_one_driver(struct rte_vmbus_driver *dr,
 
 	/* reference driver structure */
 	dev->driver = dr;
-	dev->device.driver = &dr->driver;
 
 	if (dev->device.numa_node < 0) {
 		VMBUS_LOG(WARNING, "  Invalid NUMA socket, default to 0");
@@ -125,6 +123,8 @@ vmbus_probe_one_driver(struct rte_vmbus_driver *dr,
 	if (ret) {
 		dev->driver = NULL;
 		rte_vmbus_unmap_device(dev);
+	} else {
+		dev->device.driver = &dr->driver;
 	}
 
 	return ret;
@@ -143,7 +143,7 @@ vmbus_probe_all_drivers(struct rte_vmbus_device *dev)
 	int rc;
 
 	/* Check if a driver is already loaded */
-	if (dev->driver != NULL) {
+	if (rte_dev_is_probed(&dev->device)) {
 		VMBUS_LOG(DEBUG, "VMBUS driver already loaded");
 		return 0;
 	}
@@ -204,6 +204,27 @@ vmbus_parse(const char *name, void *addr)
 	return ret;
 }
 
+/*
+ * scan for matching device args on command line
+ * example:
+ *	-w 'vmbus:635a7ae3-091e-4410-ad59-667c4f8c04c3,latency=20'
+ */
+struct rte_devargs *
+vmbus_devargs_lookup(struct rte_vmbus_device *dev)
+{
+	struct rte_devargs *devargs;
+	rte_uuid_t addr;
+
+	RTE_EAL_DEVARGS_FOREACH("vmbus", devargs) {
+		vmbus_parse(devargs->name, &addr);
+
+		if (rte_uuid_compare(dev->device_id, addr) == 0)
+			return devargs;
+	}
+	return NULL;
+
+}
+
 /* register vmbus driver */
 void
 rte_vmbus_register(struct rte_vmbus_driver *driver)
diff --git a/drivers/common/Makefile b/drivers/common/Makefile
index 0fd22376..87b8a59a 100644
--- a/drivers/common/Makefile
+++ b/drivers/common/Makefile
@@ -4,8 +4,23 @@
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
+ifeq ($(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO),y)
+DIRS-y += cpt
+endif
+
 ifeq ($(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF)$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL),yy)
 DIRS-y += octeontx
 endif
 
+MVEP-y := $(CONFIG_RTE_LIBRTE_MVPP2_PMD)
+MVEP-y += $(CONFIG_RTE_LIBRTE_MVNETA_PMD)
+MVEP-y += $(CONFIG_RTE_LIBRTE_PMD_MVSAM_CRYPTO)
+ifneq (,$(findstring y,$(MVEP-y)))
+DIRS-y += mvep
+endif
+
+ifeq ($(CONFIG_RTE_LIBRTE_COMMON_DPAAX),y)
+DIRS-y += dpaax
+endif
+
 include $(RTE_SDK)/mk/rte.subdir.mk
diff --git a/drivers/common/cpt/Makefile b/drivers/common/cpt/Makefile
new file mode 100644
index 00000000..2340aa96
--- /dev/null
+++ b/drivers/common/cpt/Makefile
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Cavium, Inc
+#
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_common_cpt.a
+
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -I$(RTE_SDK)/drivers/bus/pci
+EXPORT_MAP := rte_common_cpt_version.map
+
+LIBABIVER := 1
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-y += cpt_pmd_ops_helper.c
+
+LDLIBS += -lrte_eal
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/common/cpt/cpt_common.h b/drivers/common/cpt/cpt_common.h
new file mode 100644
index 00000000..8461cd60
--- /dev/null
+++ b/drivers/common/cpt/cpt_common.h
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_COMMON_H_
+#define _CPT_COMMON_H_
+
+/*
+ * This file defines common macros and structs
+ */
+
+/*
+ * Macros to determine CPT model. Driver makefile will define CPT_MODEL
+ * accordingly
+ */
+#define CRYPTO_OCTEONTX		0x1
+
+#define TIME_IN_RESET_COUNT	5
+
+/* Default command timeout in seconds */
+#define DEFAULT_COMMAND_TIMEOUT	4
+
+#define CPT_COUNT_THOLD		32
+#define CPT_TIMER_THOLD		0x3F
+
+#define AE_TYPE 1
+#define SE_TYPE 2
+
+#ifndef ROUNDUP4
+#define ROUNDUP4(val)	(((val) + 3) & 0xfffffffc)
+#endif
+
+#ifndef ROUNDUP8
+#define ROUNDUP8(val)	(((val) + 7) & 0xfffffff8)
+#endif
+
+#ifndef ROUNDUP16
+#define ROUNDUP16(val)	(((val) + 15) & 0xfffffff0)
+#endif
+
+#ifndef __hot
+#define __hot __attribute__((hot))
+#endif
+
+#define MOD_INC(i, l)   ((i) == (l - 1) ? (i) = 0 : (i)++)
+
+struct cptvf_meta_info {
+	void *cptvf_meta_pool;
+	int cptvf_op_mlen;
+	int cptvf_op_sb_mlen;
+};
+
+struct rid {
+	/** Request id of a crypto operation */
+	uintptr_t rid;
+};
+
+/*
+ * Pending queue structure
+ *
+ */
+struct pending_queue {
+	/** Tail of queue to be used for enqueue */
+	uint16_t enq_tail;
+	/** Head of queue to be used for dequeue */
+	uint16_t deq_head;
+	/** Array of pending requests */
+	struct rid *rid_queue;
+	/** Pending requests count */
+	uint64_t pending_count;
+};
+
+struct cpt_request_info {
+	/** Data path fields */
+	uint64_t comp_baddr;
+	volatile uint64_t *completion_addr;
+	volatile uint64_t *alternate_caddr;
+	void *op;
+	struct {
+		uint64_t ei0;
+		uint64_t ei1;
+		uint64_t ei2;
+		uint64_t ei3;
+	} ist;
+
+	/** Control path fields */
+	uint64_t time_out;
+	uint8_t extra_time;
+};
+
+#endif /* _CPT_COMMON_H_ */
diff --git a/drivers/common/cpt/cpt_hw_types.h b/drivers/common/cpt/cpt_hw_types.h
new file mode 100644
index 00000000..cff59c79
--- /dev/null
+++ b/drivers/common/cpt/cpt_hw_types.h
@@ -0,0 +1,522 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_HW_TYPES_H_
+#define _CPT_HW_TYPES_H_
+
+#include <rte_byteorder.h>
+
+/*
+ * This file defines HRM specific structs.
+ *
+ */
+
+#define CPT_VF_INTR_MBOX_MASK   (1<<0)
+#define CPT_VF_INTR_DOVF_MASK   (1<<1)
+#define CPT_VF_INTR_IRDE_MASK   (1<<2)
+#define CPT_VF_INTR_NWRP_MASK   (1<<3)
+#define CPT_VF_INTR_SWERR_MASK  (1<<4)
+#define CPT_VF_INTR_HWERR_MASK  (1<<5)
+#define CPT_VF_INTR_FAULT_MASK  (1<<6)
+
+#define CPT_INST_SIZE           (64)
+#define CPT_NEXT_CHUNK_PTR_SIZE (8)
+
+/*
+ * CPT_INST_S software command definitions
+ * Words EI (0-3)
+ */
+typedef union {
+	uint64_t u64;
+	struct {
+		uint16_t opcode;
+		uint16_t param1;
+		uint16_t param2;
+		uint16_t dlen;
+	} s;
+} vq_cmd_word0_t;
+
+typedef union {
+	uint64_t u64;
+	struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+		uint64_t grp	: 3;
+		uint64_t cptr	: 61;
+#else
+		uint64_t cptr	: 61;
+		uint64_t grp	: 3;
+#endif
+	} s;
+} vq_cmd_word3_t;
+
+typedef struct cpt_vq_command {
+	vq_cmd_word0_t cmd;
+	uint64_t dptr;
+	uint64_t rptr;
+	vq_cmd_word3_t cptr;
+} cpt_vq_cmd_t;
+
+/**
+ * Structure cpt_inst_s
+ *
+ * CPT Instruction Structure
+ * This structure specifies the instruction layout.
+ * Instructions are stored in memory as little-endian unless
+ * CPT()_PF_Q()_CTL[INST_BE] is set.
+ */
+typedef union cpt_inst_s {
+	uint64_t u[8];
+	struct cpt_inst_s_8s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+		uint64_t reserved_17_63        : 47;
+		/* [ 16: 16] Done interrupt.
+		 * 0 = No interrupts related to this instruction.
+		 * 1 = When the instruction completes,CPT()_VQ()_DONE[DONE]
+		 * will be incremented, and based on the rules described
+		 * there an interrupt may occur.
+		 */
+		uint64_t doneint               : 1;
+		uint64_t reserved_0_15         : 16;
+#else /* Word 0 - Little Endian */
+		uint64_t reserved_0_15         : 16;
+		uint64_t doneint               : 1;
+		uint64_t reserved_17_63        : 47;
+#endif /* Word 0 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 1 - Big Endian */
+		/* [127: 64] Result IOVA.
+		 * If nonzero, specifies where to write CPT_RES_S.
+		 * If zero, no result structure will be written.
+		 * Address must be 16-byte aligned.
+		 *
+		 * Bits <63:49> are ignored by hardware; software should
+		 * use a sign-extended bit <48> for forward compatibility.
+		 */
+		uint64_t res_addr              : 64;
+#else /* Word 1 - Little Endian */
+		uint64_t res_addr              : 64;
+#endif /* Word 1 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 2 - Big Endian */
+		uint64_t reserved_172_191      : 20;
+		/* [171:162] If [WQ_PTR] is nonzero, the SSO guest-group to
+		 * use when CPT submits work to SSO.
+		 * For the SSO to not discard the add-work request, FPA_PF_MAP()
+		 * must map [GRP] and CPT()_PF_Q()_GMCTL[GMID] as valid.
+		 */
+		uint64_t grp                   : 10;
+		/* [161:160] If [WQ_PTR] is nonzero, the SSO tag type to use
+		 * when CPT submits work to SSO.
+		 */
+		uint64_t tt                    : 2;
+		/* [159:128] If [WQ_PTR] is nonzero, the SSO tag to use when
+		 * CPT submits work to SSO.
+		 */
+		uint64_t tag                   : 32;
+#else /* Word 2 - Little Endian */
+		uint64_t tag                   : 32;
+		uint64_t tt                    : 2;
+		uint64_t grp                   : 10;
+		uint64_t reserved_172_191      : 20;
+#endif /* Word 2 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 3 - Big Endian */
+		/** [255:192] If [WQ_PTR] is nonzero, it is a pointer to a
+		 * work-queue entry that CPT submits work to SSO after all
+		 * context, output data, and result write operations are
+		 * visible to other CNXXXX units and the cores.
+		 * Bits <2:0> must be zero.
+		 * Bits <63:49> are ignored by hardware; software should use a
+		 * sign-extended bit <48> for forward compatibility.
+		 * Internal:Bits <63:49>, <2:0> are ignored by hardware,
+		 * treated as always 0x0.
+		 **/
+		uint64_t wq_ptr                : 64;
+#else /* Word 3 - Little Endian */
+		uint64_t wq_ptr                : 64;
+#endif /* Word 3 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 4 - Big Endian */
+		union {
+			/** [319:256] Engine instruction word 0. Passed to the
+			 * AE/SE.
+			 **/
+			uint64_t ei0                   : 64;
+			vq_cmd_word0_t vq_cmd_w0;
+		};
+#else /* Word 4 - Little Endian */
+		union {
+			uint64_t ei0                   : 64;
+			vq_cmd_word0_t vq_cmd_w0;
+		};
+#endif /* Word 4 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 5 - Big Endian */
+		union {
+			/** [383:320] Engine instruction word 1. Passed to the
+			 * AE/SE.
+			 **/
+			uint64_t ei1                   : 64;
+			uint64_t dptr;
+		};
+#else /* Word 5 - Little Endian */
+		union {
+			uint64_t ei1                   : 64;
+			uint64_t dptr;
+		};
+#endif /* Word 5 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 6 - Big Endian */
+		union {
+			/** [447:384] Engine instruction word 2. Passed to the
+			 * AE/SE.
+			 **/
+			uint64_t ei2                   : 64;
+			uint64_t rptr;
+		};
+#else /* Word 6 - Little Endian */
+		union {
+			uint64_t ei2                   : 64;
+			uint64_t rptr;
+		};
+#endif /* Word 6 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 7 - Big Endian */
+		union {
+			/** [511:448] Engine instruction word 3. Passed to the
+			 * AE/SE.
+			 **/
+			uint64_t ei3                   : 64;
+			vq_cmd_word3_t vq_cmd_w3;
+		};
+#else /* Word 7 - Little Endian */
+		union {
+			uint64_t ei3                   : 64;
+			vq_cmd_word3_t vq_cmd_w3;
+		};
+#endif /* Word 7 - End */
+	} s8x;
+} cpt_inst_s_t;
+
+/**
+ * Structure cpt_res_s
+ *
+ * CPT Result Structure
+ * The CPT coprocessor writes the result structure after it completes a
+ * CPT_INST_S instruction. The result structure is exactly 16 bytes, and each
+ * instruction completion produces exactly one result structure.
+ *
+ * This structure is stored in memory as little-endian unless
+ * CPT()_PF_Q()_CTL[INST_BE] is set.
+ */
+typedef union cpt_res_s {
+	uint64_t u[2];
+	struct cpt_res_s_8s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+		uint64_t reserved_17_63        : 47;
+		/** [ 16: 16] Done interrupt. This bit is copied from the
+		 * corresponding instruction's CPT_INST_S[DONEINT].
+		 **/
+		uint64_t doneint               : 1;
+		uint64_t reserved_8_15         : 8;
+		/** [  7:  0] Indicates completion/error status of the CPT
+		 * coprocessor for the associated instruction, as enumerated by
+		 * CPT_COMP_E. Core software may write the memory location
+		 * containing [COMPCODE] to 0x0 before ringing the doorbell, and
+		 * then poll for completion by checking for a nonzero value.
+		 *
+		 * Once the core observes a nonzero [COMPCODE] value in this
+		 * case, the CPT coprocessor will have also completed L2/DRAM
+		 * write operations.
+		 **/
+		uint64_t compcode              : 8;
+#else /* Word 0 - Little Endian */
+		uint64_t compcode              : 8;
+		uint64_t reserved_8_15         : 8;
+		uint64_t doneint               : 1;
+		uint64_t reserved_17_63        : 47;
+#endif /* Word 0 - End */
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 1 - Big Endian */
+		uint64_t reserved_64_127       : 64;
+#else /* Word 1 - Little Endian */
+		uint64_t reserved_64_127       : 64;
+#endif /* Word 1 - End */
+	} s8x;
+} cpt_res_s_t;
+
+/**
+ * Register (NCB) cpt#_vq#_ctl
+ *
+ * CPT VF Queue Control Registers
+ * This register configures queues. This register should be changed (other than
+ * clearing [ENA]) only when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]).
+ */
+typedef union {
+	uint64_t u;
+	struct cptx_vqx_ctl_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+		uint64_t reserved_1_63         : 63;
+		/** [  0:  0](R/W/H) Enables the logical instruction queue.
+		 * See also CPT()_PF_Q()_CTL[CONT_ERR] and
+		 * CPT()_VQ()_INPROG[INFLIGHT].
+		 * 1 = Queue is enabled.
+		 * 0 = Queue is disabled.
+		 **/
+		uint64_t ena                   : 1;
+#else /* Word 0 - Little Endian */
+		uint64_t ena                   : 1;
+		uint64_t reserved_1_63         : 63;
+#endif /* Word 0 - End */
+	} s;
+} cptx_vqx_ctl_t;
+
+/**
+ * Register (NCB) cpt#_vq#_done
+ *
+ * CPT Queue Done Count Registers
+ * These registers contain the per-queue instruction done count.
+ */
+typedef union {
+	uint64_t u;
+	struct cptx_vqx_done_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+		uint64_t reserved_20_63        : 44;
+		/** [ 19:  0](R/W/H) Done count. When CPT_INST_S[DONEINT] set
+		 * and that instruction completes,CPT()_VQ()_DONE[DONE] is
+		 * incremented when the instruction finishes. Write to this
+		 * field are for diagnostic use only; instead software writes
+		 * CPT()_VQ()_DONE_ACK with the number of decrements for this
+		 * field.
+		 *
+		 * Interrupts are sent as follows:
+		 *
+		 * When CPT()_VQ()_DONE[DONE] = 0, then no results are pending,
+		 * the interrupt coalescing timer is held to zero, and an
+		 * interrupt is not sent.
+		 *
+		 * When CPT()_VQ()_DONE[DONE] != 0, then the interrupt
+		 * coalescing timer counts. If the counter is >= CPT()_VQ()_DONE
+		 * _WAIT[TIME_WAIT]*1024, or CPT()_VQ()_DONE[DONE] >= CPT()_VQ()
+		 * _DONE_WAIT[NUM_WAIT], i.e. enough time has passed or enough
+		 * results have arrived, then the interrupt is sent.  Otherwise,
+		 * it is not sent due to coalescing.
+		 *
+		 * When CPT()_VQ()_DONE_ACK is written (or CPT()_VQ()_DONE is
+		 * written but this is not typical), the interrupt coalescing
+		 * timer restarts.  Note after decrementing this interrupt
+		 * equation is recomputed, for example if CPT()_VQ()_DONE[DONE]
+		 * >= CPT()_VQ()_DONE_WAIT[NUM_WAIT] and because the timer is
+		 * zero, the interrupt will be resent immediately.  (This covers
+		 * the race case between software acknowledging an interrupt and
+		 * a result returning.)
+		 *
+		 * When CPT()_VQ()_DONE_ENA_W1S[DONE] = 0, interrupts are not
+		 * sent, but the counting described above still occurs.
+		 *
+		 * Since CPT instructions complete out-of-order, if software is
+		 * using completion interrupts the suggested scheme is to
+		 * request a DONEINT on each request, and when an interrupt
+		 * arrives perform a "greedy" scan for completions; even if a
+		 * later command is acknowledged first this will not result in
+		 * missing a completion.
+		 *
+		 * Software is responsible for making sure [DONE] does not
+		 * overflow; for example by insuring there are not more than
+		 * 2^20-1 instructions in flight that may request interrupts.
+		 **/
+		uint64_t done                  : 20;
+#else /* Word 0 - Little Endian */
+		uint64_t done                  : 20;
+		uint64_t reserved_20_63        : 44;
+#endif /* Word 0 - End */
+	} s;
+} cptx_vqx_done_t;
+
+/**
+ * Register (NCB) cpt#_vq#_done_ack
+ *
+ * CPT Queue Done Count Ack Registers
+ * This register is written by software to acknowledge interrupts.
+ */
+typedef union {
+	uint64_t u;
+	struct cptx_vqx_done_ack_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+		uint64_t reserved_20_63        : 44;
+		/** [ 19:  0](R/W/H) Number of decrements to CPT()_VQ()_DONE
+		 * [DONE]. Reads CPT()_VQ()_DONE[DONE].
+		 *
+		 * Written by software to acknowledge interrupts. If CPT()_VQ()_
+		 * DONE[DONE] is still nonzero the interrupt will be re-sent if
+		 * the conditions described in CPT()_VQ()_DONE[DONE] are
+		 * satisfied.
+		 **/
+		uint64_t done_ack              : 20;
+#else /* Word 0 - Little Endian */
+		uint64_t done_ack              : 20;
+		uint64_t reserved_20_63        : 44;
+#endif /* Word 0 - End */
+	} s;
+} cptx_vqx_done_ack_t;
+
+/**
+ * Register (NCB) cpt#_vq#_done_wait
+ *
+ * CPT Queue Done Interrupt Coalescing Wait Registers
+ * Specifies the per queue interrupt coalescing settings.
+ */
+typedef union {
+	uint64_t u;
+	struct cptx_vqx_done_wait_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+		uint64_t reserved_48_63        : 16;
+		/** [ 47: 32](R/W) Time hold-off. When CPT()_VQ()_DONE[DONE] =
+		 * 0, or CPT()_VQ()_DONE_ACK is written a timer is cleared. When
+		 * the timer reaches [TIME_WAIT]*1024 then interrupt coalescing
+		 * ends; see CPT()_VQ()_DONE[DONE]. If 0x0, time coalescing is
+		 * disabled.
+		 **/
+		uint64_t time_wait             : 16;
+		uint64_t reserved_20_31        : 12;
+		/** [ 19:  0](R/W) Number of messages hold-off. When
+		 * CPT()_VQ()_DONE[DONE] >= [NUM_WAIT] then interrupt coalescing
+		 * ends; see CPT()_VQ()_DONE[DONE]. If 0x0, same behavior as
+		 * 0x1.
+		 **/
+		uint64_t num_wait              : 20;
+#else /* Word 0 - Little Endian */
+		uint64_t num_wait              : 20;
+		uint64_t reserved_20_31        : 12;
+		uint64_t time_wait             : 16;
+		uint64_t reserved_48_63        : 16;
+#endif /* Word 0 - End */
+	} s;
+} cptx_vqx_done_wait_t;
+
+/**
+ * Register (NCB) cpt#_vq#_doorbell
+ *
+ * CPT Queue Doorbell Registers
+ * Doorbells for the CPT instruction queues.
+ */
+typedef union {
+	uint64_t u;
+	struct cptx_vqx_doorbell_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+		uint64_t reserved_20_63        : 44;
+		uint64_t dbell_cnt             : 20;
+		/** [ 19:  0](R/W/H) Number of instruction queue 64-bit words
+		 * to add to the CPT instruction doorbell count. Readback value
+		 * is the the current number of pending doorbell requests.
+		 *
+		 * If counter overflows CPT()_VQ()_MISC_INT[DBELL_DOVF] is set.
+		 *
+		 * To reset the count back to zero, write one to clear
+		 * CPT()_VQ()_MISC_INT_ENA_W1C[DBELL_DOVF], then write a value
+		 * of 2^20 minus the read [DBELL_CNT], then write one to
+		 * CPT()_VQ()_MISC_INT_W1C[DBELL_DOVF] and
+		 * CPT()_VQ()_MISC_INT_ENA_W1S[DBELL_DOVF].
+		 *
+		 * Must be a multiple of 8.  All CPT instructions are 8 words
+		 * and require a doorbell count of multiple of 8.
+		 **/
+#else /* Word 0 - Little Endian */
+		uint64_t dbell_cnt             : 20;
+		uint64_t reserved_20_63        : 44;
+#endif /* Word 0 - End */
+	} s;
+} cptx_vqx_doorbell_t;
+
+/**
+ * Register (NCB) cpt#_vq#_inprog
+ *
+ * CPT Queue In Progress Count Registers
+ * These registers contain the per-queue instruction in flight registers.
+ */
+typedef union {
+	uint64_t u;
+	struct cptx_vqx_inprog_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+		uint64_t reserved_8_63         : 56;
+		/** [  7:  0](RO/H) Inflight count. Counts the number of
+		 * instructions for the VF for which CPT is fetching, executing
+		 * or responding to instructions. However this does not include
+		 * any interrupts that are awaiting software handling
+		 * (CPT()_VQ()_DONE[DONE] != 0x0).
+		 *
+		 * A queue may not be reconfigured until:
+		 *  1. CPT()_VQ()_CTL[ENA] is cleared by software.
+		 *  2. [INFLIGHT] is polled until equals to zero.
+		 **/
+		uint64_t inflight              : 8;
+#else /* Word 0 - Little Endian */
+		uint64_t inflight              : 8;
+		uint64_t reserved_8_63         : 56;
+#endif /* Word 0 - End */
+	} s;
+} cptx_vqx_inprog_t;
+
+/**
+ * Register (NCB) cpt#_vq#_misc_int
+ *
+ * CPT Queue Misc Interrupt Register
+ * These registers contain the per-queue miscellaneous interrupts.
+ */
+typedef union {
+	uint64_t u;
+	struct cptx_vqx_misc_int_s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+		uint64_t reserved_7_63         : 57;
+		/** [  6:  6](R/W1C/H) Translation fault detected. */
+		uint64_t fault		       : 1;
+		/** [  5:  5](R/W1C/H) Hardware error from engines. */
+		uint64_t hwerr		       : 1;
+		/** [  4:  4](R/W1C/H) Software error from engines. */
+		uint64_t swerr                 : 1;
+		/** [  3:  3](R/W1C/H) NCB result write response error. */
+		uint64_t nwrp                  : 1;
+		/** [  2:  2](R/W1C/H) Instruction NCB read response error. */
+		uint64_t irde                  : 1;
+		/** [  1:  1](R/W1C/H) Doorbell overflow. */
+		uint64_t dovf                  : 1;
+		/** [  0:  0](R/W1C/H) PF to VF mailbox interrupt. Set when
+		 * CPT()_VF()_PF_MBOX(0) is written.
+		 **/
+		uint64_t mbox                  : 1;
+#else /* Word 0 - Little Endian */
+		uint64_t mbox                  : 1;
+		uint64_t dovf                  : 1;
+		uint64_t irde                  : 1;
+		uint64_t nwrp                  : 1;
+		uint64_t swerr                 : 1;
+		uint64_t hwerr		       : 1;
+		uint64_t fault		       : 1;
+		uint64_t reserved_5_63         : 59;
+#endif /* Word 0 - End */
+	} s;
+} cptx_vqx_misc_int_t;
+
+/**
+ * Register (NCB) cpt#_vq#_saddr
+ *
+ * CPT Queue Starting Buffer Address Registers
+ * These registers set the instruction buffer starting address.
+ */
+typedef union {
+	uint64_t u;
+	struct cptx_vqx_saddr_s	{
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+		uint64_t reserved_49_63        : 15;
+		/** [ 48:  6](R/W/H) Instruction buffer IOVA <48:6>
+		 * (64-byte aligned). When written, it is the initial buffer
+		 * starting address; when read, it is the next read pointer to
+		 * be requested from L2C. The PTR field is overwritten with the
+		 * next pointer each time that the command buffer segment is
+		 * exhausted. New commands will then be read from the newly
+		 * specified command buffer pointer.
+		 **/
+		uint64_t ptr                   : 43;
+		uint64_t reserved_0_5          : 6;
+#else /* Word 0 - Little Endian */
+		uint64_t reserved_0_5          : 6;
+		uint64_t ptr                   : 43;
+		uint64_t reserved_49_63        : 15;
+#endif /* Word 0 - End */
+	} s;
+} cptx_vqx_saddr_t;
+
+#endif /*_CPT_HW_TYPES_H_ */
diff --git a/drivers/common/cpt/cpt_mcode_defines.h b/drivers/common/cpt/cpt_mcode_defines.h
new file mode 100644
index 00000000..becc14f4
--- /dev/null
+++ b/drivers/common/cpt/cpt_mcode_defines.h
@@ -0,0 +1,386 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_MCODE_DEFINES_H_
+#define _CPT_MCODE_DEFINES_H_
+
+#include <rte_byteorder.h>
+#include <rte_memory.h>
+
+/*
+ * This file defines macros and structures according to microcode spec
+ *
+ */
+/* SE opcodes */
+#define CPT_MAJOR_OP_FC		0x33
+#define CPT_MAJOR_OP_HASH	0x34
+#define CPT_MAJOR_OP_HMAC	0x35
+#define CPT_MAJOR_OP_ZUC_SNOW3G	0x37
+#define CPT_MAJOR_OP_KASUMI	0x38
+#define CPT_MAJOR_OP_MISC	0x01
+
+#define CPT_BYTE_16		16
+#define CPT_BYTE_24		24
+#define CPT_BYTE_32		32
+#define CPT_MAX_SG_IN_OUT_CNT	32
+#define CPT_MAX_SG_CNT		(CPT_MAX_SG_IN_OUT_CNT/2)
+
+#define COMPLETION_CODE_SIZE	8
+#define COMPLETION_CODE_INIT	0
+
+#define SG_LIST_HDR_SIZE	(8u)
+#define SG_ENTRY_SIZE		sizeof(sg_comp_t)
+
+#define CPT_DMA_MODE		(1 << 7)
+
+#define CPT_FROM_CTX		0
+#define CPT_FROM_DPTR		1
+
+#define FC_GEN			0x1
+#define ZUC_SNOW3G		0x2
+#define KASUMI			0x3
+#define HASH_HMAC		0x4
+
+#define ZS_EA			0x1
+#define ZS_IA			0x2
+#define K_F8			0x4
+#define K_F9			0x8
+
+#define CPT_OP_CIPHER_ENCRYPT	0x1
+#define CPT_OP_CIPHER_DECRYPT	0x2
+#define CPT_OP_CIPHER_MASK	0x3
+
+#define CPT_OP_AUTH_VERIFY	0x4
+#define CPT_OP_AUTH_GENERATE	0x8
+#define CPT_OP_AUTH_MASK	0xC
+
+#define CPT_OP_ENCODE	(CPT_OP_CIPHER_ENCRYPT | CPT_OP_AUTH_GENERATE)
+#define CPT_OP_DECODE	(CPT_OP_CIPHER_DECRYPT | CPT_OP_AUTH_VERIFY)
+
+/* #define CPT_ALWAYS_USE_SG_MODE */
+#define CPT_ALWAYS_USE_SEPARATE_BUF
+
+/*
+ * Parameters for Flexi Crypto
+ * requests
+ */
+#define VALID_AAD_BUF 0x01
+#define VALID_MAC_BUF 0x02
+#define VALID_IV_BUF 0x04
+#define SINGLE_BUF_INPLACE 0x08
+#define SINGLE_BUF_HEADTAILROOM 0x10
+
+#define ENCR_IV_OFFSET(__d_offs) ((__d_offs >> 32) & 0xffff)
+#define ENCR_OFFSET(__d_offs) ((__d_offs >> 16) & 0xffff)
+#define AUTH_OFFSET(__d_offs) (__d_offs & 0xffff)
+#define ENCR_DLEN(__d_lens) (__d_lens >> 32)
+#define AUTH_DLEN(__d_lens) (__d_lens & 0xffffffff)
+
+/* FC offset_control at start of DPTR in bytes */
+#define OFF_CTRL_LEN  8 /**< bytes */
+
+typedef enum {
+	MD5_TYPE        = 1,
+	SHA1_TYPE       = 2,
+	SHA2_SHA224     = 3,
+	SHA2_SHA256     = 4,
+	SHA2_SHA384     = 5,
+	SHA2_SHA512     = 6,
+	GMAC_TYPE       = 7,
+	XCBC_TYPE       = 8,
+	SHA3_SHA224     = 10,
+	SHA3_SHA256     = 11,
+	SHA3_SHA384     = 12,
+	SHA3_SHA512     = 13,
+	SHA3_SHAKE256   = 14,
+	SHA3_SHAKE512   = 15,
+
+	/* These are only for software use */
+	ZUC_EIA3        = 0x90,
+	SNOW3G_UIA2     = 0x91,
+	KASUMI_F9_CBC   = 0x92,
+	KASUMI_F9_ECB   = 0x93,
+} mc_hash_type_t;
+
+typedef enum {
+	/* To support passthrough */
+	PASSTHROUGH  = 0x0,
+	/*
+	 * These are defined by MC for Flexi crypto
+	 * for field of 4 bits
+	 */
+	DES3_CBC    = 0x1,
+	DES3_ECB    = 0x2,
+	AES_CBC     = 0x3,
+	AES_ECB     = 0x4,
+	AES_CFB     = 0x5,
+	AES_CTR     = 0x6,
+	AES_GCM     = 0x7,
+	AES_XTS     = 0x8,
+
+	/* These are only for software use */
+	ZUC_EEA3        = 0x90,
+	SNOW3G_UEA2     = 0x91,
+	KASUMI_F8_CBC   = 0x92,
+	KASUMI_F8_ECB   = 0x93,
+} mc_cipher_type_t;
+
+typedef enum {
+	AES_128_BIT = 0x1,
+	AES_192_BIT = 0x2,
+	AES_256_BIT = 0x3
+} mc_aes_type_t;
+
+typedef enum {
+	/* Microcode errors */
+	NO_ERR = 0x00,
+	ERR_OPCODE_UNSUPPORTED = 0x01,
+
+	/* SCATTER GATHER */
+	ERR_SCATTER_GATHER_WRITE_LENGTH = 0x02,
+	ERR_SCATTER_GATHER_LIST = 0x03,
+	ERR_SCATTER_GATHER_NOT_SUPPORTED = 0x04,
+
+	/* SE GC */
+	ERR_GC_LENGTH_INVALID = 0x41,
+	ERR_GC_RANDOM_LEN_INVALID = 0x42,
+	ERR_GC_DATA_LEN_INVALID = 0x43,
+	ERR_GC_DRBG_TYPE_INVALID = 0x44,
+	ERR_GC_CTX_LEN_INVALID = 0x45,
+	ERR_GC_CIPHER_UNSUPPORTED = 0x46,
+	ERR_GC_AUTH_UNSUPPORTED = 0x47,
+	ERR_GC_OFFSET_INVALID = 0x48,
+	ERR_GC_HASH_MODE_UNSUPPORTED = 0x49,
+	ERR_GC_DRBG_ENTROPY_LEN_INVALID = 0x4a,
+	ERR_GC_DRBG_ADDNL_LEN_INVALID = 0x4b,
+	ERR_GC_ICV_MISCOMPARE = 0x4c,
+	ERR_GC_DATA_UNALIGNED = 0x4d,
+
+	/* API Layer */
+	ERR_BAD_ALT_CCODE = 0xfd,
+	ERR_REQ_PENDING = 0xfe,
+	ERR_REQ_TIMEOUT = 0xff,
+
+	ERR_BAD_INPUT_LENGTH = (0x40000000 | 384),    /* 0x40000180 */
+	ERR_BAD_KEY_LENGTH,
+	ERR_BAD_KEY_HANDLE,
+	ERR_BAD_CONTEXT_HANDLE,
+	ERR_BAD_SCALAR_LENGTH,
+	ERR_BAD_DIGEST_LENGTH,
+	ERR_BAD_INPUT_ARG,
+	ERR_BAD_RECORD_PADDING,
+	ERR_NB_REQUEST_PENDING,
+	ERR_EIO,
+	ERR_ENODEV,
+} mc_error_code_t;
+
+/**
+ * Enumeration cpt_comp_e
+ *
+ * CPT Completion Enumeration
+ * Enumerates the values of CPT_RES_S[COMPCODE].
+ */
+typedef enum {
+	CPT_8X_COMP_E_NOTDONE    = (0x00),
+	CPT_8X_COMP_E_GOOD       = (0x01),
+	CPT_8X_COMP_E_FAULT      = (0x02),
+	CPT_8X_COMP_E_SWERR      = (0x03),
+	CPT_8X_COMP_E_HWERR      = (0x04),
+	CPT_8X_COMP_E_LAST_ENTRY = (0xFF)
+} cpt_comp_e_t;
+
+typedef struct sglist_comp {
+	union {
+		uint64_t len;
+		struct {
+			uint16_t len[4];
+		} s;
+	} u;
+	uint64_t ptr[4];
+} sg_comp_t;
+
+struct cpt_sess_misc {
+	/** CPT opcode */
+	uint16_t cpt_op:4;
+	/** ZUC, SNOW3G &  KASUMI flags */
+	uint16_t zsk_flag:4;
+	/** Flag for AES GCM */
+	uint16_t aes_gcm:1;
+	/** Flag for AES CTR */
+	uint16_t aes_ctr:1;
+	/** Flag for NULL cipher/auth */
+	uint16_t is_null:1;
+	/** Flag for GMAC */
+	uint16_t is_gmac:1;
+	/** AAD length */
+	uint16_t aad_length;
+	/** MAC len in bytes */
+	uint8_t mac_len;
+	/** IV length in bytes */
+	uint8_t iv_length;
+	/** Auth IV length in bytes */
+	uint8_t auth_iv_length;
+	/** Reserved field */
+	uint8_t rsvd1;
+	/** IV offset in bytes */
+	uint16_t iv_offset;
+	/** Auth IV offset in bytes */
+	uint16_t auth_iv_offset;
+	/** Salt */
+	uint32_t salt;
+	/** Context DMA address */
+	phys_addr_t ctx_dma_addr;
+};
+
+typedef union {
+	uint64_t flags;
+	struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+		uint64_t enc_cipher   : 4;
+		uint64_t reserved1    : 1;
+		uint64_t aes_key      : 2;
+		uint64_t iv_source    : 1;
+		uint64_t hash_type    : 4;
+		uint64_t reserved2    : 3;
+		uint64_t auth_input_type : 1;
+		uint64_t mac_len      : 8;
+		uint64_t reserved3    : 8;
+		uint64_t encr_offset  : 16;
+		uint64_t iv_offset    : 8;
+		uint64_t auth_offset  : 8;
+#else
+		uint64_t auth_offset  : 8;
+		uint64_t iv_offset    : 8;
+		uint64_t encr_offset  : 16;
+		uint64_t reserved3    : 8;
+		uint64_t mac_len      : 8;
+		uint64_t auth_input_type : 1;
+		uint64_t reserved2    : 3;
+		uint64_t hash_type    : 4;
+		uint64_t iv_source    : 1;
+		uint64_t aes_key      : 2;
+		uint64_t reserved1    : 1;
+		uint64_t enc_cipher   : 4;
+#endif
+	} e;
+} encr_ctrl_t;
+
+typedef struct {
+	encr_ctrl_t enc_ctrl;
+	uint8_t  encr_key[32];
+	uint8_t  encr_iv[16];
+} mc_enc_context_t;
+
+typedef struct {
+	uint8_t  ipad[64];
+	uint8_t  opad[64];
+} mc_fc_hmac_context_t;
+
+typedef struct {
+	mc_enc_context_t     enc;
+	mc_fc_hmac_context_t hmac;
+} mc_fc_context_t;
+
+typedef struct {
+	uint8_t encr_auth_iv[16];
+	uint8_t ci_key[16];
+	uint8_t zuc_const[32];
+} mc_zuc_snow3g_ctx_t;
+
+typedef struct {
+	uint8_t reg_A[8];
+	uint8_t ci_key[16];
+} mc_kasumi_ctx_t;
+
+struct cpt_ctx {
+	/* Below fields are accessed by sw */
+	uint64_t enc_cipher	:8;
+	uint64_t hash_type	:8;
+	uint64_t mac_len	:8;
+	uint64_t auth_key_len	:8;
+	uint64_t fc_type	:4;
+	uint64_t hmac		:1;
+	uint64_t zsk_flags	:3;
+	uint64_t k_ecb		:1;
+	uint64_t snow3g		:1;
+	uint64_t rsvd		:22;
+	/* Below fields are accessed by hardware */
+	union {
+		mc_fc_context_t fctx;
+		mc_zuc_snow3g_ctx_t zs_ctx;
+		mc_kasumi_ctx_t k_ctx;
+	};
+	uint8_t  auth_key[64];
+};
+
+/* Buffer pointer */
+typedef struct buf_ptr {
+	void *vaddr;
+	phys_addr_t dma_addr;
+	uint32_t size;
+	uint32_t resv;
+} buf_ptr_t;
+
+/* IOV Pointer */
+typedef struct{
+	int buf_cnt;
+	buf_ptr_t bufs[0];
+} iov_ptr_t;
+
+typedef union opcode_info {
+	uint16_t flags;
+	struct {
+		uint8_t major;
+		uint8_t minor;
+	} s;
+} opcode_info_t;
+
+typedef struct fc_params {
+	/* 0th cache line */
+	union {
+		buf_ptr_t bufs[1];
+		struct {
+			iov_ptr_t *src_iov;
+			iov_ptr_t *dst_iov;
+		};
+	};
+	void *iv_buf;
+	void *auth_iv_buf;
+	buf_ptr_t meta_buf;
+	buf_ptr_t ctx_buf;
+	uint64_t rsvd2;
+
+	/* 1st cache line */
+	buf_ptr_t aad_buf;
+	buf_ptr_t mac_buf;
+
+} fc_params_t;
+
+/*
+ * Parameters for digest
+ * generate requests
+ * Only src_iov, op, ctx_buf, mac_buf, prep_req
+ * meta_buf, auth_data_len are used for digest gen.
+ */
+typedef struct fc_params digest_params_t;
+
+/* Cipher Algorithms */
+typedef mc_cipher_type_t cipher_type_t;
+
+/* Auth Algorithms */
+typedef mc_hash_type_t auth_type_t;
+
+/* Helper macros */
+
+#define CPT_P_ENC_CTRL(fctx)  fctx->enc.enc_ctrl.e
+
+#define SRC_IOV_SIZE \
+	(sizeof(iov_ptr_t) + (sizeof(buf_ptr_t) * CPT_MAX_SG_CNT))
+#define DST_IOV_SIZE \
+	(sizeof(iov_ptr_t) + (sizeof(buf_ptr_t) * CPT_MAX_SG_CNT))
+
+#define SESS_PRIV(__sess) \
+	(void *)((uint8_t *)__sess + sizeof(struct cpt_sess_misc))
+
+#endif /* _CPT_MCODE_DEFINES_H_ */
diff --git a/drivers/common/cpt/cpt_pmd_logs.h b/drivers/common/cpt/cpt_pmd_logs.h
new file mode 100644
index 00000000..4cbec4e3
--- /dev/null
+++ b/drivers/common/cpt/cpt_pmd_logs.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_PMD_LOGS_H_
+#define _CPT_PMD_LOGS_H_
+
+#include <rte_log.h>
+
+/*
+ * This file defines log macros
+ */
+
+#define CPT_PMD_DRV_LOG_RAW(level, fmt, args...) \
+		rte_log(RTE_LOG_ ## level, cpt_logtype, \
+			"cpt: %s(): " fmt "\n", __func__, ##args)
+
+#define CPT_PMD_INIT_FUNC_TRACE() CPT_PMD_DRV_LOG_RAW(DEBUG, " >>")
+
+#define CPT_LOG_INFO(fmt, args...) \
+	CPT_PMD_DRV_LOG_RAW(INFO, fmt, ## args)
+#define CPT_LOG_WARN(fmt, args...) \
+	CPT_PMD_DRV_LOG_RAW(WARNING, fmt, ## args)
+#define CPT_LOG_ERR(fmt, args...) \
+	CPT_PMD_DRV_LOG_RAW(ERR, fmt, ## args)
+
+/*
+ * DP logs, toggled out at compile time if level lower than current level.
+ * DP logs would be logged under 'PMD' type. So for dynamic logging, the
+ * level of 'pmd' has to be used.
+ */
+#define CPT_LOG_DP(level, fmt, args...) \
+	RTE_LOG_DP(level, PMD, fmt "\n", ## args)
+
+#define CPT_LOG_DP_DEBUG(fmt, args...) \
+	CPT_LOG_DP(DEBUG, fmt, ## args)
+#define CPT_LOG_DP_INFO(fmt, args...) \
+	CPT_LOG_DP(INFO, fmt, ## args)
+#define CPT_LOG_DP_WARN(fmt, args...) \
+	CPT_LOG_DP(WARNING, fmt, ## args)
+#define CPT_LOG_DP_ERR(fmt, args...) \
+	CPT_LOG_DP(ERR, fmt, ## args)
+
+/*
+ * cpt_logtype will be used for common logging. This field would be initialized
+ * by otx_* driver routines during PCI probe.
+ */
+int cpt_logtype;
+
+#endif /* _CPT_PMD_LOGS_H_ */
diff --git a/drivers/common/cpt/cpt_pmd_ops_helper.c b/drivers/common/cpt/cpt_pmd_ops_helper.c
new file mode 100644
index 00000000..1c18180f
--- /dev/null
+++ b/drivers/common/cpt/cpt_pmd_ops_helper.c
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#include <rte_common.h>
+
+#include "cpt_common.h"
+#include "cpt_hw_types.h"
+#include "cpt_mcode_defines.h"
+#include "cpt_pmd_ops_helper.h"
+
+#define CPT_MAX_IV_LEN 16
+#define CPT_OFFSET_CONTROL_BYTES 8
+
+int32_t
+cpt_pmd_ops_helper_get_mlen_direct_mode(void)
+{
+	uint32_t len = 0;
+
+	/* Request structure */
+	len = sizeof(struct cpt_request_info);
+
+	/* CPT HW result structure plus extra as it is aligned */
+	len += 2*sizeof(cpt_res_s_t);
+
+	return len;
+}
+
+int
+cpt_pmd_ops_helper_get_mlen_sg_mode(void)
+{
+	uint32_t len = 0;
+
+	len += sizeof(struct cpt_request_info);
+	len += CPT_OFFSET_CONTROL_BYTES + CPT_MAX_IV_LEN;
+	len += ROUNDUP8(SG_LIST_HDR_SIZE +
+			(ROUNDUP4(CPT_MAX_SG_IN_OUT_CNT) >> 2) * SG_ENTRY_SIZE);
+	len += 2 * COMPLETION_CODE_SIZE;
+	len += 2 * sizeof(cpt_res_s_t);
+	return len;
+}
diff --git a/drivers/common/cpt/cpt_pmd_ops_helper.h b/drivers/common/cpt/cpt_pmd_ops_helper.h
new file mode 100644
index 00000000..dd32f9a4
--- /dev/null
+++ b/drivers/common/cpt/cpt_pmd_ops_helper.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_PMD_OPS_HELPER_H_
+#define _CPT_PMD_OPS_HELPER_H_
+
+/*
+ * This file defines the agreement between the common layer and the individual
+ * crypto drivers for OCTEON TX series. Control path in otx* directory can
+ * directly call functions declared here.
+ */
+
+/*
+ * Get meta length required when operating in direct mode (single buffer
+ * in-place)
+ *
+ * @return
+ *   - length
+ */
+
+int32_t
+cpt_pmd_ops_helper_get_mlen_direct_mode(void);
+
+/*
+ * Get size of contiguous meta buffer to be allocated when working in scatter
+ * gather mode.
+ *
+ * @return
+ *   - length
+ */
+int
+cpt_pmd_ops_helper_get_mlen_sg_mode(void);
+#endif /* _CPT_PMD_OPS_HELPER_H_ */
diff --git a/drivers/common/cpt/cpt_request_mgr.h b/drivers/common/cpt/cpt_request_mgr.h
new file mode 100644
index 00000000..4463cfbe
--- /dev/null
+++ b/drivers/common/cpt/cpt_request_mgr.h
@@ -0,0 +1,185 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_REQUEST_MGR_H_
+#define _CPT_REQUEST_MGR_H_
+
+#include <rte_branch_prediction.h>
+#include <rte_cycles.h>
+
+#include "cpt_common.h"
+#include "cpt_mcode_defines.h"
+
+#if CPT_MODEL == CRYPTO_OCTEONTX
+#include "../../crypto/octeontx/otx_cryptodev_hw_access.h"
+#endif
+
+/*
+ * This file defines the agreement between the common layer and the individual
+ * crypto drivers for OCTEON TX series. Datapath in otx* directory include this
+ * file and all these functions are static inlined for better performance.
+ *
+ */
+
+/*
+ * Get the session size
+ *
+ * This function is used in the data path.
+ *
+ * @return
+ *   - session size
+ */
+static __rte_always_inline unsigned int
+cpt_get_session_size(void)
+{
+	unsigned int ctx_len = sizeof(struct cpt_ctx);
+	return (sizeof(struct cpt_sess_misc) + RTE_ALIGN_CEIL(ctx_len, 8));
+}
+
+static __rte_always_inline int32_t __hot
+cpt_enqueue_req(struct cpt_instance *instance, struct pending_queue *pqueue,
+		void *req)
+{
+	struct cpt_request_info *user_req = (struct cpt_request_info *)req;
+	int32_t ret = 0;
+
+	if (unlikely(!req))
+		return 0;
+
+	if (unlikely(pqueue->pending_count >= DEFAULT_CMD_QLEN))
+		return -EAGAIN;
+
+	fill_cpt_inst(instance, req);
+
+	CPT_LOG_DP_DEBUG("req: %p op: %p ", req, user_req->op);
+
+	/* Fill time_out cycles */
+	user_req->time_out = rte_get_timer_cycles() +
+			DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
+	user_req->extra_time = 0;
+
+	/* Default mode of software queue */
+	mark_cpt_inst(instance);
+
+	pqueue->rid_queue[pqueue->enq_tail].rid =
+		(uintptr_t)user_req;
+	/* We will use soft queue length here to limit
+	 * requests
+	 */
+	MOD_INC(pqueue->enq_tail, DEFAULT_CMD_QLEN);
+	pqueue->pending_count += 1;
+
+	CPT_LOG_DP_DEBUG("Submitted NB cmd with request: %p "
+			 "op: %p", user_req, user_req->op);
+
+	return ret;
+}
+
+static __rte_always_inline int __hot
+cpt_pmd_crypto_operation(struct cpt_instance *instance,
+		struct rte_crypto_op *op, struct pending_queue *pqueue,
+		uint8_t cpt_driver_id)
+{
+	struct cpt_sess_misc *sess = NULL;
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	void *prep_req = NULL, *mdata = NULL;
+	int ret = 0;
+	uint64_t cpt_op;
+	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
+
+	if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
+		int sess_len;
+
+		sess_len = cpt_get_session_size();
+
+		sess = rte_calloc(__func__, 1, sess_len, 8);
+		if (!sess)
+			return -ENOMEM;
+
+		sess->ctx_dma_addr =  rte_malloc_virt2iova(sess) +
+			sizeof(struct cpt_sess_misc);
+
+		ret = instance_session_cfg(sym_op->xform, (void *)sess);
+		if (unlikely(ret))
+			return -EINVAL;
+	} else {
+		sess = (struct cpt_sess_misc *)
+		get_sym_session_private_data(sym_op->session,
+		cpt_driver_id);
+	}
+
+	cpt_op = sess->cpt_op;
+
+	mdata = &(cptvf->meta_info);
+
+	if (likely(cpt_op & CPT_OP_CIPHER_MASK))
+		prep_req = fill_fc_params(op, sess, &mdata, &ret);
+	else
+		prep_req = fill_digest_params(op, sess, &mdata, &ret);
+
+	if (unlikely(!prep_req)) {
+		CPT_LOG_DP_ERR("prep cryto req : op %p, cpt_op 0x%x "
+			       "ret 0x%x", op, (unsigned int)cpt_op, ret);
+		goto req_fail;
+	}
+
+	/* Enqueue prepared instruction to HW */
+	ret = cpt_enqueue_req(instance, pqueue, prep_req);
+
+	if (unlikely(ret)) {
+		if (unlikely(ret == -EAGAIN))
+			goto req_fail;
+		CPT_LOG_DP_ERR("Error enqueing crypto request : error "
+			       "code %d", ret);
+		goto req_fail;
+	}
+
+	return 0;
+
+req_fail:
+	if (mdata)
+		free_op_meta(mdata, cptvf->meta_info.cptvf_meta_pool);
+	return ret;
+}
+
+static __rte_always_inline int32_t __hot
+cpt_dequeue_burst(struct cpt_instance *instance, uint16_t cnt,
+		  void *resp[], uint8_t cc[], struct pending_queue *pqueue)
+{
+	struct cpt_request_info *user_req;
+	struct rid *rid_e;
+	int i, count, pcount;
+	uint8_t ret;
+
+	pcount = pqueue->pending_count;
+	count = (cnt > pcount) ? pcount : cnt;
+
+	for (i = 0; i < count; i++) {
+		rid_e = &pqueue->rid_queue[pqueue->deq_head];
+		user_req = (struct cpt_request_info *)(rid_e->rid);
+
+		if (likely((i+1) < count))
+			rte_prefetch_non_temporal((void *)rid_e[1].rid);
+
+		ret = check_nb_command_id(user_req, instance);
+
+		if (unlikely(ret == ERR_REQ_PENDING)) {
+			/* Stop checking for completions */
+			break;
+		}
+
+		/* Return completion code and op handle */
+		cc[i] = (uint8_t)ret;
+		resp[i] = user_req->op;
+		CPT_LOG_DP_DEBUG("Request %p Op %p completed with code %d",
+			   user_req, user_req->op, ret);
+
+		MOD_INC(pqueue->deq_head, DEFAULT_CMD_QLEN);
+		pqueue->pending_count -= 1;
+	}
+
+	return i;
+}
+
+#endif /* _CPT_REQUEST_MGR_H_ */
diff --git a/drivers/common/cpt/cpt_ucode.h b/drivers/common/cpt/cpt_ucode.h
new file mode 100644
index 00000000..c5a9f34b
--- /dev/null
+++ b/drivers/common/cpt/cpt_ucode.h
@@ -0,0 +1,3648 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _CPT_UCODE_H_
+#define _CPT_UCODE_H_
+#include <stdbool.h>
+
+#include "cpt_common.h"
+#include "cpt_hw_types.h"
+#include "cpt_mcode_defines.h"
+
+/*
+ * This file defines functions that are interfaces to microcode spec.
+ *
+ */
+
+static uint8_t zuc_d[32] = {
+	0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E,
+	0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF,
+	0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1,
+	0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC
+};
+
+static __rte_always_inline int
+cpt_is_algo_supported(struct rte_crypto_sym_xform *xform)
+{
+	/*
+	 * Microcode only supports the following combination.
+	 * Encryption followed by authentication
+	 * Authentication followed by decryption
+	 */
+	if (xform->next) {
+		if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) &&
+		    (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
+		    (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)) {
+			/* Unsupported as of now by microcode */
+			CPT_LOG_DP_ERR("Unsupported combination");
+			return -1;
+		}
+		if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
+		    (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) &&
+		    (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT)) {
+			/* For GMAC auth there is no cipher operation */
+			if (xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM ||
+			    xform->next->auth.algo !=
+			    RTE_CRYPTO_AUTH_AES_GMAC) {
+				/* Unsupported as of now by microcode */
+				CPT_LOG_DP_ERR("Unsupported combination");
+				return -1;
+			}
+		}
+	}
+	return 0;
+}
+
+static __rte_always_inline void
+gen_key_snow3g(uint8_t *ck, uint32_t *keyx)
+{
+	int i, base;
+
+	for (i = 0; i < 4; i++) {
+		base = 4 * i;
+		keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) |
+			(ck[base + 2] << 8) | (ck[base + 3]);
+		keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]);
+	}
+}
+
+static __rte_always_inline void
+cpt_fc_salt_update(void *ctx,
+		   uint8_t *salt)
+{
+	struct cpt_ctx *cpt_ctx = ctx;
+	memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4);
+}
+
+static __rte_always_inline int
+cpt_fc_ciph_validate_key_aes(uint16_t key_len)
+{
+	switch (key_len) {
+	case CPT_BYTE_16:
+	case CPT_BYTE_24:
+	case CPT_BYTE_32:
+		return 0;
+	default:
+		return -1;
+	}
+}
+
+static __rte_always_inline int
+cpt_fc_ciph_validate_key(cipher_type_t type, struct cpt_ctx *cpt_ctx,
+		uint16_t key_len)
+{
+	int fc_type = 0;
+	switch (type) {
+	case PASSTHROUGH:
+		fc_type = FC_GEN;
+		break;
+	case DES3_CBC:
+	case DES3_ECB:
+		fc_type = FC_GEN;
+		break;
+	case AES_CBC:
+	case AES_ECB:
+	case AES_CFB:
+	case AES_CTR:
+	case AES_GCM:
+		if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0))
+			return -1;
+		fc_type = FC_GEN;
+		break;
+	case AES_XTS:
+		key_len = key_len / 2;
+		if (unlikely(key_len == CPT_BYTE_24)) {
+			CPT_LOG_DP_ERR("Invalid AES key len for XTS");
+			return -1;
+		}
+		if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0))
+			return -1;
+		fc_type = FC_GEN;
+		break;
+	case ZUC_EEA3:
+	case SNOW3G_UEA2:
+		if (unlikely(key_len != 16))
+			return -1;
+		/* No support for AEAD yet */
+		if (unlikely(cpt_ctx->hash_type))
+			return -1;
+		fc_type = ZUC_SNOW3G;
+		break;
+	case KASUMI_F8_CBC:
+	case KASUMI_F8_ECB:
+		if (unlikely(key_len != 16))
+			return -1;
+		/* No support for AEAD yet */
+		if (unlikely(cpt_ctx->hash_type))
+			return -1;
+		fc_type = KASUMI;
+		break;
+	default:
+		return -1;
+	}
+	return fc_type;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx)
+{
+	cpt_ctx->enc_cipher = 0;
+	CPT_P_ENC_CTRL(fctx).enc_cipher = 0;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len)
+{
+	mc_aes_type_t aes_key_type = 0;
+	switch (key_len) {
+	case CPT_BYTE_16:
+		aes_key_type = AES_128_BIT;
+		break;
+	case CPT_BYTE_24:
+		aes_key_type = AES_192_BIT;
+		break;
+	case CPT_BYTE_32:
+		aes_key_type = AES_256_BIT;
+		break;
+	default:
+		/* This should not happen */
+		CPT_LOG_DP_ERR("Invalid AES key len");
+		return;
+	}
+	CPT_P_ENC_CTRL(fctx).aes_key = aes_key_type;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, uint8_t *key,
+		uint16_t key_len)
+{
+	uint32_t keyx[4];
+	cpt_ctx->snow3g = 1;
+	gen_key_snow3g(key, keyx);
+	memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len);
+	cpt_ctx->fc_type = ZUC_SNOW3G;
+	cpt_ctx->zsk_flags = 0;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, uint8_t *key,
+		uint16_t key_len)
+{
+	cpt_ctx->snow3g = 0;
+	memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len);
+	memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32);
+	cpt_ctx->fc_type = ZUC_SNOW3G;
+	cpt_ctx->zsk_flags = 0;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, uint8_t *key,
+		uint16_t key_len)
+{
+	cpt_ctx->k_ecb = 1;
+	memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
+	cpt_ctx->zsk_flags = 0;
+	cpt_ctx->fc_type = KASUMI;
+}
+
+static __rte_always_inline void
+cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, uint8_t *key,
+		uint16_t key_len)
+{
+	memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
+	cpt_ctx->zsk_flags = 0;
+	cpt_ctx->fc_type = KASUMI;
+}
+
+static __rte_always_inline int
+cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, uint8_t *key,
+		    uint16_t key_len, uint8_t *salt)
+{
+	struct cpt_ctx *cpt_ctx = ctx;
+	mc_fc_context_t *fctx = &cpt_ctx->fctx;
+	uint64_t *ctrl_flags = NULL;
+	int fc_type;
+
+	/* Validate key before proceeding */
+	fc_type = cpt_fc_ciph_validate_key(type, cpt_ctx, key_len);
+	if (unlikely(fc_type == -1))
+		return -1;
+
+	if (fc_type == FC_GEN) {
+		cpt_ctx->fc_type = FC_GEN;
+		ctrl_flags = (uint64_t *)&(fctx->enc.enc_ctrl.flags);
+		*ctrl_flags = rte_be_to_cpu_64(*ctrl_flags);
+		/*
+		 * We need to always say IV is from DPTR as user can
+		 * sometimes iverride IV per operation.
+		 */
+		CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_DPTR;
+	}
+
+	switch (type) {
+	case PASSTHROUGH:
+		cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx);
+		goto fc_success;
+	case DES3_CBC:
+		/* CPT performs DES using 3DES with the 8B DES-key
+		 * replicated 2 more times to match the 24B 3DES-key.
+		 * Eg. If org. key is "0x0a 0x0b", then new key is
+		 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b"
+		 */
+		if (key_len == 8) {
+			/* Skipping the first 8B as it will be copied
+			 * in the regular code flow
+			 */
+			memcpy(fctx->enc.encr_key+key_len, key, key_len);
+			memcpy(fctx->enc.encr_key+2*key_len, key, key_len);
+		}
+		break;
+	case DES3_ECB:
+		/* For DES3_ECB IV need to be from CTX. */
+		CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_CTX;
+		break;
+	case AES_CBC:
+	case AES_ECB:
+	case AES_CFB:
+	case AES_CTR:
+		cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
+		break;
+	case AES_GCM:
+		/* Even though iv source is from dptr,
+		 * aes_gcm salt is taken from ctx
+		 */
+		if (salt) {
+			memcpy(fctx->enc.encr_iv, salt, 4);
+			/* Assuming it was just salt update
+			 * and nothing else
+			 */
+			if (!key)
+				goto fc_success;
+		}
+		cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
+		break;
+	case AES_XTS:
+		key_len = key_len / 2;
+		cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
+
+		/* Copy key2 for XTS into ipad */
+		memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
+		memcpy(fctx->hmac.ipad, &key[key_len], key_len);
+		break;
+	case SNOW3G_UEA2:
+		cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len);
+		goto success;
+	case ZUC_EEA3:
+		cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len);
+		goto success;
+	case KASUMI_F8_ECB:
+		cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len);
+		goto success;
+	case KASUMI_F8_CBC:
+		cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len);
+		goto success;
+	default:
+		break;
+	}
+
+	/* Only for FC_GEN case */
+
+	/* For GMAC auth, cipher must be NULL */
+	if (cpt_ctx->hash_type != GMAC_TYPE)
+		CPT_P_ENC_CTRL(fctx).enc_cipher = type;
+
+	memcpy(fctx->enc.encr_key, key, key_len);
+
+fc_success:
+	*ctrl_flags = rte_cpu_to_be_64(*ctrl_flags);
+
+success:
+	cpt_ctx->enc_cipher = type;
+
+	return 0;
+}
+
+static __rte_always_inline uint32_t
+fill_sg_comp(sg_comp_t *list,
+	     uint32_t i,
+	     phys_addr_t dma_addr,
+	     uint32_t size)
+{
+	sg_comp_t *to = &list[i>>2];
+
+	to->u.s.len[i%4] = rte_cpu_to_be_16(size);
+	to->ptr[i%4] = rte_cpu_to_be_64(dma_addr);
+	i++;
+	return i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg_comp_from_buf(sg_comp_t *list,
+		      uint32_t i,
+		      buf_ptr_t *from)
+{
+	sg_comp_t *to = &list[i>>2];
+
+	to->u.s.len[i%4] = rte_cpu_to_be_16(from->size);
+	to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr);
+	i++;
+	return i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg_comp_from_buf_min(sg_comp_t *list,
+			  uint32_t i,
+			  buf_ptr_t *from,
+			  uint32_t *psize)
+{
+	sg_comp_t *to = &list[i >> 2];
+	uint32_t size = *psize;
+	uint32_t e_len;
+
+	e_len = (size > from->size) ? from->size : size;
+	to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+	to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr);
+	*psize -= e_len;
+	i++;
+	return i;
+}
+
+/*
+ * This fills the MC expected SGIO list
+ * from IOV given by user.
+ */
+static __rte_always_inline uint32_t
+fill_sg_comp_from_iov(sg_comp_t *list,
+		      uint32_t i,
+		      iov_ptr_t *from, uint32_t from_offset,
+		      uint32_t *psize, buf_ptr_t *extra_buf,
+		      uint32_t extra_offset)
+{
+	int32_t j;
+	uint32_t extra_len = extra_buf ? extra_buf->size : 0;
+	uint32_t size = *psize - extra_len;
+	buf_ptr_t *bufs;
+
+	bufs = from->bufs;
+	for (j = 0; (j < from->buf_cnt) && size; j++) {
+		phys_addr_t e_dma_addr;
+		uint32_t e_len;
+		sg_comp_t *to = &list[i >> 2];
+
+		if (!bufs[j].size)
+			continue;
+
+		if (unlikely(from_offset)) {
+			if (from_offset >= bufs[j].size) {
+				from_offset -= bufs[j].size;
+				continue;
+			}
+			e_dma_addr = bufs[j].dma_addr + from_offset;
+			e_len = (size > (bufs[j].size - from_offset)) ?
+				(bufs[j].size - from_offset) : size;
+			from_offset = 0;
+		} else {
+			e_dma_addr = bufs[j].dma_addr;
+			e_len = (size > bufs[j].size) ?
+				bufs[j].size : size;
+		}
+
+		to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+		to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr);
+
+		if (extra_len && (e_len >= extra_offset)) {
+			/* Break the data at given offset */
+			uint32_t next_len = e_len - extra_offset;
+			phys_addr_t next_dma = e_dma_addr + extra_offset;
+
+			if (!extra_offset) {
+				i--;
+			} else {
+				e_len = extra_offset;
+				size -= e_len;
+				to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+			}
+
+			/* Insert extra data ptr */
+			if (extra_len) {
+				i++;
+				to = &list[i >> 2];
+				to->u.s.len[i % 4] =
+					rte_cpu_to_be_16(extra_buf->size);
+				to->ptr[i % 4] =
+					rte_cpu_to_be_64(extra_buf->dma_addr);
+
+				/* size already decremented by extra len */
+			}
+
+			/* insert the rest of the data */
+			if (next_len) {
+				i++;
+				to = &list[i >> 2];
+				to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);
+				to->ptr[i % 4] = rte_cpu_to_be_64(next_dma);
+				size -= next_len;
+			}
+			extra_len = 0;
+
+		} else {
+			size -= e_len;
+		}
+		if (extra_offset)
+			extra_offset -= size;
+		i++;
+	}
+
+	*psize = size;
+	return (uint32_t)i;
+}
+
+static __rte_always_inline int
+cpt_digest_gen_prep(uint32_t flags,
+		    uint64_t d_lens,
+		    digest_params_t *params,
+		    void *op,
+		    void **prep_req)
+{
+	struct cpt_request_info *req;
+	uint32_t size, i;
+	int32_t m_size;
+	uint16_t data_len, mac_len, key_len;
+	auth_type_t hash_type;
+	buf_ptr_t *meta_p;
+	struct cpt_ctx *ctx;
+	sg_comp_t *gather_comp;
+	sg_comp_t *scatter_comp;
+	uint8_t *in_buffer;
+	uint32_t g_size_bytes, s_size_bytes;
+	uint64_t dptr_dma, rptr_dma;
+	vq_cmd_word0_t vq_cmd_w0;
+	vq_cmd_word3_t vq_cmd_w3;
+	void *c_vaddr, *m_vaddr;
+	uint64_t c_dma, m_dma;
+	opcode_info_t opcode;
+
+	if (!params || !params->ctx_buf.vaddr)
+		return ERR_BAD_INPUT_ARG;
+
+	ctx = params->ctx_buf.vaddr;
+	meta_p = &params->meta_buf;
+
+	if (!meta_p->vaddr || !meta_p->dma_addr)
+		return ERR_BAD_INPUT_ARG;
+
+	if (meta_p->size < sizeof(struct cpt_request_info))
+		return ERR_BAD_INPUT_ARG;
+
+	m_vaddr = meta_p->vaddr;
+	m_dma = meta_p->dma_addr;
+	m_size = meta_p->size;
+
+	/*
+	 * Save initial space that followed app data for completion code &
+	 * alternate completion code to fall in same cache line as app data
+	 */
+	m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+	m_dma += COMPLETION_CODE_SIZE;
+	size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+		(uint8_t *)m_vaddr;
+	c_vaddr = (uint8_t *)m_vaddr + size;
+	c_dma = m_dma + size;
+	size += sizeof(cpt_res_s_t);
+
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	req = m_vaddr;
+
+	size = sizeof(struct cpt_request_info);
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	hash_type = ctx->hash_type;
+	mac_len = ctx->mac_len;
+	key_len = ctx->auth_key_len;
+	data_len = AUTH_DLEN(d_lens);
+
+	/*GP op header */
+	vq_cmd_w0.u64 = 0;
+	vq_cmd_w0.s.param2 = rte_cpu_to_be_16(((uint16_t)hash_type << 8));
+	if (ctx->hmac) {
+		opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE;
+		vq_cmd_w0.s.param1 = rte_cpu_to_be_16(key_len);
+		vq_cmd_w0.s.dlen =
+			rte_cpu_to_be_16((data_len + ROUNDUP8(key_len)));
+	} else {
+		opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE;
+		vq_cmd_w0.s.param1 = 0;
+		vq_cmd_w0.s.dlen = rte_cpu_to_be_16(data_len);
+	}
+
+	opcode.s.minor = 0;
+
+	/* Null auth only case enters the if */
+	if (unlikely(!hash_type && !ctx->enc_cipher)) {
+		opcode.s.major = CPT_MAJOR_OP_MISC;
+		/* Minor op is passthrough */
+		opcode.s.minor = 0x03;
+		/* Send out completion code only */
+		vq_cmd_w0.s.param2 = 0x1;
+	}
+
+	vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+	/* DPTR has SG list */
+	in_buffer = m_vaddr;
+	dptr_dma = m_dma;
+
+	((uint16_t *)in_buffer)[0] = 0;
+	((uint16_t *)in_buffer)[1] = 0;
+
+	/* TODO Add error check if space will be sufficient */
+	gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+	/*
+	 * Input gather list
+	 */
+
+	i = 0;
+
+	if (ctx->hmac) {
+		uint64_t k_dma = params->ctx_buf.dma_addr +
+			offsetof(struct cpt_ctx, auth_key);
+		/* Key */
+		i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len));
+	}
+
+	/* input data */
+	size = data_len;
+	if (size) {
+		i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov,
+					  0, &size, NULL, 0);
+		if (size) {
+			CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short"
+					 " by %dB", size);
+			return ERR_BAD_INPUT_ARG;
+		}
+	} else {
+		/*
+		 * Looks like we need to support zero data
+		 * gather ptr in case of hash & hmac
+		 */
+		i++;
+	}
+	((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+	g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+	/*
+	 * Output Gather list
+	 */
+
+	i = 0;
+	scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+	if (flags & VALID_MAC_BUF) {
+		if (params->mac_buf.size < mac_len)
+			return ERR_BAD_INPUT_ARG;
+
+		size = mac_len;
+		i = fill_sg_comp_from_buf_min(scatter_comp, i,
+					      &params->mac_buf, &size);
+	} else {
+		size = mac_len;
+		i = fill_sg_comp_from_iov(scatter_comp, i,
+					  params->src_iov, data_len,
+					  &size, NULL, 0);
+		if (size) {
+			CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short by"
+					 " %dB", size);
+			return ERR_BAD_INPUT_ARG;
+		}
+	}
+
+	((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+	s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+	size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+	/* This is DPTR len incase of SG mode */
+	vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	/* cpt alternate completion address saved earlier */
+	req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+	*req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+	rptr_dma = c_dma - 8;
+
+	req->ist.ei1 = dptr_dma;
+	req->ist.ei2 = rptr_dma;
+	/* First 16-bit swap then 64-bit swap */
+	/* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+	 * to eliminate all the swapping
+	 */
+	vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+	/* vq command w3 */
+	vq_cmd_w3.u64 = 0;
+
+	/* 16 byte aligned cpt res address */
+	req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+	*req->completion_addr = COMPLETION_CODE_INIT;
+	req->comp_baddr  = c_dma;
+
+	/* Fill microcode part of instruction */
+	req->ist.ei0 = vq_cmd_w0.u64;
+	req->ist.ei3 = vq_cmd_w3.u64;
+
+	req->op = op;
+
+	*prep_req = req;
+	return 0;
+}
+
+static __rte_always_inline int
+cpt_enc_hmac_prep(uint32_t flags,
+		  uint64_t d_offs,
+		  uint64_t d_lens,
+		  fc_params_t *fc_params,
+		  void *op,
+		  void **prep_req)
+{
+	uint32_t iv_offset = 0;
+	int32_t inputlen, outputlen, enc_dlen, auth_dlen;
+	struct cpt_ctx *cpt_ctx;
+	uint32_t cipher_type, hash_type;
+	uint32_t mac_len, size;
+	uint8_t iv_len = 16;
+	struct cpt_request_info *req;
+	buf_ptr_t *meta_p, *aad_buf = NULL;
+	uint32_t encr_offset, auth_offset;
+	uint32_t encr_data_len, auth_data_len, aad_len = 0;
+	uint32_t passthrough_len = 0;
+	void *m_vaddr, *offset_vaddr;
+	uint64_t m_dma, offset_dma, ctx_dma;
+	vq_cmd_word0_t vq_cmd_w0;
+	vq_cmd_word3_t vq_cmd_w3;
+	void *c_vaddr;
+	uint64_t c_dma;
+	int32_t m_size;
+	opcode_info_t opcode;
+
+	meta_p = &fc_params->meta_buf;
+	m_vaddr = meta_p->vaddr;
+	m_dma = meta_p->dma_addr;
+	m_size = meta_p->size;
+
+	encr_offset = ENCR_OFFSET(d_offs);
+	auth_offset = AUTH_OFFSET(d_offs);
+	encr_data_len = ENCR_DLEN(d_lens);
+	auth_data_len = AUTH_DLEN(d_lens);
+	if (unlikely(flags & VALID_AAD_BUF)) {
+		/*
+		 * We dont support both aad
+		 * and auth data separately
+		 */
+		auth_data_len = 0;
+		auth_offset = 0;
+		aad_len = fc_params->aad_buf.size;
+		aad_buf = &fc_params->aad_buf;
+	}
+	cpt_ctx = fc_params->ctx_buf.vaddr;
+	cipher_type = cpt_ctx->enc_cipher;
+	hash_type = cpt_ctx->hash_type;
+	mac_len = cpt_ctx->mac_len;
+
+	/*
+	 * Save initial space that followed app data for completion code &
+	 * alternate completion code to fall in same cache line as app data
+	 */
+	m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+	m_dma += COMPLETION_CODE_SIZE;
+	size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+		(uint8_t *)m_vaddr;
+
+	c_vaddr = (uint8_t *)m_vaddr + size;
+	c_dma = m_dma + size;
+	size += sizeof(cpt_res_s_t);
+
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	/* start cpt request info struct at 8 byte boundary */
+	size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) -
+		(uint8_t *)m_vaddr;
+
+	req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size);
+
+	size += sizeof(struct cpt_request_info);
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	if (hash_type == GMAC_TYPE)
+		encr_data_len = 0;
+
+	if (unlikely(!(flags & VALID_IV_BUF))) {
+		iv_len = 0;
+		iv_offset = ENCR_IV_OFFSET(d_offs);
+	}
+
+	if (unlikely(flags & VALID_AAD_BUF)) {
+		/*
+		 * When AAD is given, data above encr_offset is pass through
+		 * Since AAD is given as separate pointer and not as offset,
+		 * this is a special case as we need to fragment input data
+		 * into passthrough + encr_data and then insert AAD in between.
+		 */
+		if (hash_type != GMAC_TYPE) {
+			passthrough_len = encr_offset;
+			auth_offset = passthrough_len + iv_len;
+			encr_offset = passthrough_len + aad_len + iv_len;
+			auth_data_len = aad_len + encr_data_len;
+		} else {
+			passthrough_len = 16 + aad_len;
+			auth_offset = passthrough_len + iv_len;
+			auth_data_len = aad_len;
+		}
+	} else {
+		encr_offset += iv_len;
+		auth_offset += iv_len;
+	}
+
+	/* Encryption */
+	opcode.s.major = CPT_MAJOR_OP_FC;
+	opcode.s.minor = 0;
+
+	auth_dlen = auth_offset + auth_data_len;
+	enc_dlen = encr_data_len + encr_offset;
+	if (unlikely(encr_data_len & 0xf)) {
+		if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB))
+			enc_dlen = ROUNDUP8(encr_data_len) + encr_offset;
+		else if (likely((cipher_type == AES_CBC) ||
+				(cipher_type == AES_ECB)))
+			enc_dlen = ROUNDUP16(encr_data_len) + encr_offset;
+	}
+
+	if (unlikely(hash_type == GMAC_TYPE)) {
+		encr_offset = auth_dlen;
+		enc_dlen = 0;
+	}
+
+	if (unlikely(auth_dlen > enc_dlen)) {
+		inputlen = auth_dlen;
+		outputlen = auth_dlen + mac_len;
+	} else {
+		inputlen = enc_dlen;
+		outputlen = enc_dlen + mac_len;
+	}
+
+	/* GP op header */
+	vq_cmd_w0.u64 = 0;
+	vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+	vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len);
+	/*
+	 * In 83XX since we have a limitation of
+	 * IV & Offset control word not part of instruction
+	 * and need to be part of Data Buffer, we check if
+	 * head room is there and then only do the Direct mode processing
+	 */
+	if (likely((flags & SINGLE_BUF_INPLACE) &&
+		   (flags & SINGLE_BUF_HEADTAILROOM))) {
+		void *dm_vaddr = fc_params->bufs[0].vaddr;
+		uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr;
+		/*
+		 * This flag indicates that there is 24 bytes head room and
+		 * 8 bytes tail room available, so that we get to do
+		 * DIRECT MODE with limitation
+		 */
+
+		offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len;
+		offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
+
+		/* DPTR */
+		req->ist.ei1 = offset_dma;
+		/* RPTR should just exclude offset control word */
+		req->ist.ei2 = dm_dma_addr - iv_len;
+		req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
+						    + outputlen - iv_len);
+
+		vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN);
+
+		vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+		if (likely(iv_len)) {
+			uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
+						      + OFF_CTRL_LEN);
+			uint64_t *src = fc_params->iv_buf;
+			dest[0] = src[0];
+			dest[1] = src[1];
+		}
+
+		*(uint64_t *)offset_vaddr =
+			rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
+				((uint64_t)iv_offset << 8) |
+				((uint64_t)auth_offset));
+
+	} else {
+		uint32_t i, g_size_bytes, s_size_bytes;
+		uint64_t dptr_dma, rptr_dma;
+		sg_comp_t *gather_comp;
+		sg_comp_t *scatter_comp;
+		uint8_t *in_buffer;
+
+		/* This falls under strict SG mode */
+		offset_vaddr = m_vaddr;
+		offset_dma = m_dma;
+		size = OFF_CTRL_LEN + iv_len;
+
+		m_vaddr = (uint8_t *)m_vaddr + size;
+		m_dma += size;
+		m_size -= size;
+
+		opcode.s.major |= CPT_DMA_MODE;
+
+		vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+		if (likely(iv_len)) {
+			uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
+						      + OFF_CTRL_LEN);
+			uint64_t *src = fc_params->iv_buf;
+			dest[0] = src[0];
+			dest[1] = src[1];
+		}
+
+		*(uint64_t *)offset_vaddr =
+			rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
+				((uint64_t)iv_offset << 8) |
+				((uint64_t)auth_offset));
+
+		/* DPTR has SG list */
+		in_buffer = m_vaddr;
+		dptr_dma = m_dma;
+
+		((uint16_t *)in_buffer)[0] = 0;
+		((uint16_t *)in_buffer)[1] = 0;
+
+		/* TODO Add error check if space will be sufficient */
+		gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+		/*
+		 * Input Gather List
+		 */
+
+		i = 0;
+
+		/* Offset control word that includes iv */
+		i = fill_sg_comp(gather_comp, i, offset_dma,
+				 OFF_CTRL_LEN + iv_len);
+
+		/* Add input data */
+		size = inputlen - iv_len;
+		if (likely(size)) {
+			uint32_t aad_offset = aad_len ? passthrough_len : 0;
+
+			if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+				i = fill_sg_comp_from_buf_min(gather_comp, i,
+							      fc_params->bufs,
+							      &size);
+			} else {
+				i = fill_sg_comp_from_iov(gather_comp, i,
+							  fc_params->src_iov,
+							  0, &size,
+							  aad_buf, aad_offset);
+			}
+
+			if (unlikely(size)) {
+				CPT_LOG_DP_ERR("Insufficient buffer space,"
+					       " size %d needed", size);
+				return ERR_BAD_INPUT_ARG;
+			}
+		}
+		((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+		g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+		/*
+		 * Output Scatter list
+		 */
+		i = 0;
+		scatter_comp =
+			(sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+		/* Add IV */
+		if (likely(iv_len)) {
+			i = fill_sg_comp(scatter_comp, i,
+					 offset_dma + OFF_CTRL_LEN,
+					 iv_len);
+		}
+
+		/* output data or output data + digest*/
+		if (unlikely(flags & VALID_MAC_BUF)) {
+			size = outputlen - iv_len - mac_len;
+			if (size) {
+				uint32_t aad_offset =
+					aad_len ? passthrough_len : 0;
+
+				if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+					i = fill_sg_comp_from_buf_min(
+							scatter_comp,
+							i,
+							fc_params->bufs,
+							&size);
+				} else {
+					i = fill_sg_comp_from_iov(scatter_comp,
+							i,
+							fc_params->dst_iov,
+							0,
+							&size,
+							aad_buf,
+							aad_offset);
+				}
+				if (size)
+					return ERR_BAD_INPUT_ARG;
+			}
+			/* mac_data */
+			if (mac_len) {
+				i = fill_sg_comp_from_buf(scatter_comp, i,
+							  &fc_params->mac_buf);
+			}
+		} else {
+			/* Output including mac */
+			size = outputlen - iv_len;
+			if (likely(size)) {
+				uint32_t aad_offset =
+					aad_len ? passthrough_len : 0;
+
+				if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+					i = fill_sg_comp_from_buf_min(
+							scatter_comp,
+							i,
+							fc_params->bufs,
+							&size);
+				} else {
+					i = fill_sg_comp_from_iov(scatter_comp,
+							i,
+							fc_params->dst_iov,
+							0,
+							&size,
+							aad_buf,
+							aad_offset);
+				}
+				if (unlikely(size)) {
+					CPT_LOG_DP_ERR("Insufficient buffer"
+						       " space, size %d needed",
+						       size);
+					return ERR_BAD_INPUT_ARG;
+				}
+			}
+		}
+		((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+		s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+		size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+		/* This is DPTR len incase of SG mode */
+		vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+		m_vaddr = (uint8_t *)m_vaddr + size;
+		m_dma += size;
+		m_size -= size;
+
+		/* cpt alternate completion address saved earlier */
+		req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+		*req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+		rptr_dma = c_dma - 8;
+
+		req->ist.ei1 = dptr_dma;
+		req->ist.ei2 = rptr_dma;
+	}
+
+	/* First 16-bit swap then 64-bit swap */
+	/* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+	 * to eliminate all the swapping
+	 */
+	vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+	ctx_dma = fc_params->ctx_buf.dma_addr +
+		offsetof(struct cpt_ctx, fctx);
+	/* vq command w3 */
+	vq_cmd_w3.u64 = 0;
+	vq_cmd_w3.s.grp = 0;
+	vq_cmd_w3.s.cptr = ctx_dma;
+
+	/* 16 byte aligned cpt res address */
+	req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+	*req->completion_addr = COMPLETION_CODE_INIT;
+	req->comp_baddr  = c_dma;
+
+	/* Fill microcode part of instruction */
+	req->ist.ei0 = vq_cmd_w0.u64;
+	req->ist.ei3 = vq_cmd_w3.u64;
+
+	req->op  = op;
+
+	*prep_req = req;
+	return 0;
+}
+
+static __rte_always_inline int
+cpt_dec_hmac_prep(uint32_t flags,
+		  uint64_t d_offs,
+		  uint64_t d_lens,
+		  fc_params_t *fc_params,
+		  void *op,
+		  void **prep_req)
+{
+	uint32_t iv_offset = 0, size;
+	int32_t inputlen, outputlen, enc_dlen, auth_dlen;
+	struct cpt_ctx *cpt_ctx;
+	int32_t hash_type, mac_len, m_size;
+	uint8_t iv_len = 16;
+	struct cpt_request_info *req;
+	buf_ptr_t *meta_p, *aad_buf = NULL;
+	uint32_t encr_offset, auth_offset;
+	uint32_t encr_data_len, auth_data_len, aad_len = 0;
+	uint32_t passthrough_len = 0;
+	void *m_vaddr, *offset_vaddr;
+	uint64_t m_dma, offset_dma, ctx_dma;
+	opcode_info_t opcode;
+	vq_cmd_word0_t vq_cmd_w0;
+	vq_cmd_word3_t vq_cmd_w3;
+	void *c_vaddr;
+	uint64_t c_dma;
+
+	meta_p = &fc_params->meta_buf;
+	m_vaddr = meta_p->vaddr;
+	m_dma = meta_p->dma_addr;
+	m_size = meta_p->size;
+
+	encr_offset = ENCR_OFFSET(d_offs);
+	auth_offset = AUTH_OFFSET(d_offs);
+	encr_data_len = ENCR_DLEN(d_lens);
+	auth_data_len = AUTH_DLEN(d_lens);
+
+	if (unlikely(flags & VALID_AAD_BUF)) {
+		/*
+		 * We dont support both aad
+		 * and auth data separately
+		 */
+		auth_data_len = 0;
+		auth_offset = 0;
+		aad_len = fc_params->aad_buf.size;
+		aad_buf = &fc_params->aad_buf;
+	}
+
+	cpt_ctx = fc_params->ctx_buf.vaddr;
+	hash_type = cpt_ctx->hash_type;
+	mac_len = cpt_ctx->mac_len;
+
+	if (hash_type == GMAC_TYPE)
+		encr_data_len = 0;
+
+	if (unlikely(!(flags & VALID_IV_BUF))) {
+		iv_len = 0;
+		iv_offset = ENCR_IV_OFFSET(d_offs);
+	}
+
+	if (unlikely(flags & VALID_AAD_BUF)) {
+		/*
+		 * When AAD is given, data above encr_offset is pass through
+		 * Since AAD is given as separate pointer and not as offset,
+		 * this is a special case as we need to fragment input data
+		 * into passthrough + encr_data and then insert AAD in between.
+		 */
+		if (hash_type != GMAC_TYPE) {
+			passthrough_len = encr_offset;
+			auth_offset = passthrough_len + iv_len;
+			encr_offset = passthrough_len + aad_len + iv_len;
+			auth_data_len = aad_len + encr_data_len;
+		} else {
+			passthrough_len = 16 + aad_len;
+			auth_offset = passthrough_len + iv_len;
+			auth_data_len = aad_len;
+		}
+	} else {
+		encr_offset += iv_len;
+		auth_offset += iv_len;
+	}
+
+	/*
+	 * Save initial space that followed app data for completion code &
+	 * alternate completion code to fall in same cache line as app data
+	 */
+	m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+	m_dma += COMPLETION_CODE_SIZE;
+	size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+	       (uint8_t *)m_vaddr;
+	c_vaddr = (uint8_t *)m_vaddr + size;
+	c_dma = m_dma + size;
+	size += sizeof(cpt_res_s_t);
+
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	/* start cpt request info structure at 8 byte alignment */
+	size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) -
+		(uint8_t *)m_vaddr;
+
+	req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size);
+
+	size += sizeof(struct cpt_request_info);
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	/* Decryption */
+	opcode.s.major = CPT_MAJOR_OP_FC;
+	opcode.s.minor = 1;
+
+	enc_dlen = encr_offset + encr_data_len;
+	auth_dlen = auth_offset + auth_data_len;
+
+	if (auth_dlen > enc_dlen) {
+		inputlen = auth_dlen + mac_len;
+		outputlen = auth_dlen;
+	} else {
+		inputlen = enc_dlen + mac_len;
+		outputlen = enc_dlen;
+	}
+
+	if (hash_type == GMAC_TYPE)
+		encr_offset = inputlen;
+
+	vq_cmd_w0.u64 = 0;
+	vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+	vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len);
+
+	/*
+	 * In 83XX since we have a limitation of
+	 * IV & Offset control word not part of instruction
+	 * and need to be part of Data Buffer, we check if
+	 * head room is there and then only do the Direct mode processing
+	 */
+	if (likely((flags & SINGLE_BUF_INPLACE) &&
+		   (flags & SINGLE_BUF_HEADTAILROOM))) {
+		void *dm_vaddr = fc_params->bufs[0].vaddr;
+		uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr;
+		/*
+		 * This flag indicates that there is 24 bytes head room and
+		 * 8 bytes tail room available, so that we get to do
+		 * DIRECT MODE with limitation
+		 */
+
+		offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len;
+		offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
+		req->ist.ei1 = offset_dma;
+
+		/* RPTR should just exclude offset control word */
+		req->ist.ei2 = dm_dma_addr - iv_len;
+
+		req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr +
+					outputlen - iv_len);
+		/* since this is decryption,
+		 * don't touch the content of
+		 * alternate ccode space as it contains
+		 * hmac.
+		 */
+
+		vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN);
+
+		vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+		if (likely(iv_len)) {
+			uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
+						      OFF_CTRL_LEN);
+			uint64_t *src = fc_params->iv_buf;
+			dest[0] = src[0];
+			dest[1] = src[1];
+		}
+
+		*(uint64_t *)offset_vaddr =
+			rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
+				((uint64_t)iv_offset << 8) |
+				((uint64_t)auth_offset));
+
+	} else {
+		uint64_t dptr_dma, rptr_dma;
+		uint32_t g_size_bytes, s_size_bytes;
+		sg_comp_t *gather_comp;
+		sg_comp_t *scatter_comp;
+		uint8_t *in_buffer;
+		uint8_t i = 0;
+
+		/* This falls under strict SG mode */
+		offset_vaddr = m_vaddr;
+		offset_dma = m_dma;
+		size = OFF_CTRL_LEN + iv_len;
+
+		m_vaddr = (uint8_t *)m_vaddr + size;
+		m_dma += size;
+		m_size -= size;
+
+		opcode.s.major |= CPT_DMA_MODE;
+
+		vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+		if (likely(iv_len)) {
+			uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
+						      OFF_CTRL_LEN);
+			uint64_t *src = fc_params->iv_buf;
+			dest[0] = src[0];
+			dest[1] = src[1];
+		}
+
+		*(uint64_t *)offset_vaddr =
+			rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
+				((uint64_t)iv_offset << 8) |
+				((uint64_t)auth_offset));
+
+		/* DPTR has SG list */
+		in_buffer = m_vaddr;
+		dptr_dma = m_dma;
+
+		((uint16_t *)in_buffer)[0] = 0;
+		((uint16_t *)in_buffer)[1] = 0;
+
+		/* TODO Add error check if space will be sufficient */
+		gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+		/*
+		 * Input Gather List
+		 */
+		i = 0;
+
+		/* Offset control word that includes iv */
+		i = fill_sg_comp(gather_comp, i, offset_dma,
+				 OFF_CTRL_LEN + iv_len);
+
+		/* Add input data */
+		if (flags & VALID_MAC_BUF) {
+			size = inputlen - iv_len - mac_len;
+			if (size) {
+				/* input data only */
+				if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+					i = fill_sg_comp_from_buf_min(
+							gather_comp, i,
+							fc_params->bufs,
+							&size);
+				} else {
+					uint32_t aad_offset = aad_len ?
+						passthrough_len : 0;
+
+					i = fill_sg_comp_from_iov(gather_comp,
+							i,
+							fc_params->src_iov,
+							0, &size,
+							aad_buf,
+							aad_offset);
+				}
+				if (size)
+					return ERR_BAD_INPUT_ARG;
+			}
+
+			/* mac data */
+			if (mac_len) {
+				i = fill_sg_comp_from_buf(gather_comp, i,
+							  &fc_params->mac_buf);
+			}
+		} else {
+			/* input data + mac */
+			size = inputlen - iv_len;
+			if (size) {
+				if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+					i = fill_sg_comp_from_buf_min(
+							gather_comp, i,
+							fc_params->bufs,
+							&size);
+				} else {
+					uint32_t aad_offset = aad_len ?
+						passthrough_len : 0;
+
+					if (!fc_params->src_iov)
+						return ERR_BAD_INPUT_ARG;
+
+					i = fill_sg_comp_from_iov(
+							gather_comp, i,
+							fc_params->src_iov,
+							0, &size,
+							aad_buf,
+							aad_offset);
+				}
+
+				if (size)
+					return ERR_BAD_INPUT_ARG;
+			}
+		}
+		((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+		g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+		/*
+		 * Output Scatter List
+		 */
+
+		i = 0;
+		scatter_comp =
+			(sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+		/* Add iv */
+		if (iv_len) {
+			i = fill_sg_comp(scatter_comp, i,
+					 offset_dma + OFF_CTRL_LEN,
+					 iv_len);
+		}
+
+		/* Add output data */
+		size = outputlen - iv_len;
+		if (size) {
+			if (unlikely(flags & SINGLE_BUF_INPLACE)) {
+				/* handle single buffer here */
+				i = fill_sg_comp_from_buf_min(scatter_comp, i,
+							      fc_params->bufs,
+							      &size);
+			} else {
+				uint32_t aad_offset = aad_len ?
+					passthrough_len : 0;
+
+				if (!fc_params->dst_iov)
+					return ERR_BAD_INPUT_ARG;
+
+				i = fill_sg_comp_from_iov(scatter_comp, i,
+							  fc_params->dst_iov, 0,
+							  &size, aad_buf,
+							  aad_offset);
+			}
+
+			if (unlikely(size))
+				return ERR_BAD_INPUT_ARG;
+		}
+
+		((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+		s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+		size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+		/* This is DPTR len incase of SG mode */
+		vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+		m_vaddr = (uint8_t *)m_vaddr + size;
+		m_dma += size;
+		m_size -= size;
+
+		/* cpt alternate completion address saved earlier */
+		req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+		*req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+		rptr_dma = c_dma - 8;
+		size += COMPLETION_CODE_SIZE;
+
+		req->ist.ei1 = dptr_dma;
+		req->ist.ei2 = rptr_dma;
+	}
+
+	/* First 16-bit swap then 64-bit swap */
+	/* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+	 * to eliminate all the swapping
+	 */
+	vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+	ctx_dma = fc_params->ctx_buf.dma_addr +
+		offsetof(struct cpt_ctx, fctx);
+	/* vq command w3 */
+	vq_cmd_w3.u64 = 0;
+	vq_cmd_w3.s.grp = 0;
+	vq_cmd_w3.s.cptr = ctx_dma;
+
+	/* 16 byte aligned cpt res address */
+	req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+	*req->completion_addr = COMPLETION_CODE_INIT;
+	req->comp_baddr  = c_dma;
+
+	/* Fill microcode part of instruction */
+	req->ist.ei0 = vq_cmd_w0.u64;
+	req->ist.ei3 = vq_cmd_w3.u64;
+
+	req->op = op;
+
+	*prep_req = req;
+	return 0;
+}
+
+static __rte_always_inline int
+cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
+			uint64_t d_offs,
+			uint64_t d_lens,
+			fc_params_t *params,
+			void *op,
+			void **prep_req)
+{
+	uint32_t size;
+	int32_t inputlen, outputlen;
+	struct cpt_ctx *cpt_ctx;
+	uint32_t mac_len = 0;
+	uint8_t snow3g, j;
+	struct cpt_request_info *req;
+	buf_ptr_t *buf_p;
+	uint32_t encr_offset = 0, auth_offset = 0;
+	uint32_t encr_data_len = 0, auth_data_len = 0;
+	int flags, iv_len = 16, m_size;
+	void *m_vaddr, *c_vaddr;
+	uint64_t m_dma, c_dma, offset_ctrl;
+	uint64_t *offset_vaddr, offset_dma;
+	uint32_t *iv_s, iv[4];
+	vq_cmd_word0_t vq_cmd_w0;
+	vq_cmd_word3_t vq_cmd_w3;
+	opcode_info_t opcode;
+
+	buf_p = &params->meta_buf;
+	m_vaddr = buf_p->vaddr;
+	m_dma = buf_p->dma_addr;
+	m_size = buf_p->size;
+
+	cpt_ctx = params->ctx_buf.vaddr;
+	flags = cpt_ctx->zsk_flags;
+	mac_len = cpt_ctx->mac_len;
+	snow3g = cpt_ctx->snow3g;
+
+	/*
+	 * Save initial space that followed app data for completion code &
+	 * alternate completion code to fall in same cache line as app data
+	 */
+	m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+	m_dma += COMPLETION_CODE_SIZE;
+	size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+		(uint8_t *)m_vaddr;
+
+	c_vaddr = (uint8_t *)m_vaddr + size;
+	c_dma = m_dma + size;
+	size += sizeof(cpt_res_s_t);
+
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	/* Reserve memory for cpt request info */
+	req = m_vaddr;
+
+	size = sizeof(struct cpt_request_info);
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G;
+
+	/* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
+	opcode.s.minor = ((1 << 6) | (snow3g << 5) | (0 << 4) |
+			  (0 << 3) | (flags & 0x7));
+
+	if (flags == 0x1) {
+		/*
+		 * Microcode expects offsets in bytes
+		 * TODO: Rounding off
+		 */
+		auth_data_len = AUTH_DLEN(d_lens);
+
+		/* EIA3 or UIA2 */
+		auth_offset = AUTH_OFFSET(d_offs);
+		auth_offset = auth_offset / 8;
+
+		/* consider iv len */
+		auth_offset += iv_len;
+
+		inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8);
+		outputlen = mac_len;
+
+		offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset);
+
+	} else {
+		/* EEA3 or UEA2 */
+		/*
+		 * Microcode expects offsets in bytes
+		 * TODO: Rounding off
+		 */
+		encr_data_len = ENCR_DLEN(d_lens);
+
+		encr_offset = ENCR_OFFSET(d_offs);
+		encr_offset = encr_offset / 8;
+		/* consider iv len */
+		encr_offset += iv_len;
+
+		inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
+		outputlen = inputlen;
+
+		/* iv offset is 0 */
+		offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+	}
+
+	/* IV */
+	iv_s = (flags == 0x1) ? params->auth_iv_buf :
+		params->iv_buf;
+
+	if (snow3g) {
+		/*
+		 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
+		 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
+		 */
+
+		for (j = 0; j < 4; j++)
+			iv[j] = iv_s[3 - j];
+	} else {
+		/* ZUC doesn't need a swap */
+		for (j = 0; j < 4; j++)
+			iv[j] = iv_s[j];
+	}
+
+	/*
+	 * GP op header, lengths are expected in bits.
+	 */
+	vq_cmd_w0.u64 = 0;
+	vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+	vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len);
+
+	/*
+	 * In 83XX since we have a limitation of
+	 * IV & Offset control word not part of instruction
+	 * and need to be part of Data Buffer, we check if
+	 * head room is there and then only do the Direct mode processing
+	 */
+	if (likely((req_flags & SINGLE_BUF_INPLACE) &&
+		   (req_flags & SINGLE_BUF_HEADTAILROOM))) {
+		void *dm_vaddr = params->bufs[0].vaddr;
+		uint64_t dm_dma_addr = params->bufs[0].dma_addr;
+		/*
+		 * This flag indicates that there is 24 bytes head room and
+		 * 8 bytes tail room available, so that we get to do
+		 * DIRECT MODE with limitation
+		 */
+
+		offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
+					    OFF_CTRL_LEN - iv_len);
+		offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
+
+		/* DPTR */
+		req->ist.ei1 = offset_dma;
+		/* RPTR should just exclude offset control word */
+		req->ist.ei2 = dm_dma_addr - iv_len;
+		req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
+						    + outputlen - iv_len);
+
+		vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN);
+
+		vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+		if (likely(iv_len)) {
+			uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
+						      + OFF_CTRL_LEN);
+			memcpy(iv_d, iv, 16);
+		}
+
+		*offset_vaddr = offset_ctrl;
+	} else {
+		uint32_t i, g_size_bytes, s_size_bytes;
+		uint64_t dptr_dma, rptr_dma;
+		sg_comp_t *gather_comp;
+		sg_comp_t *scatter_comp;
+		uint8_t *in_buffer;
+		uint32_t *iv_d;
+
+		/* save space for iv */
+		offset_vaddr = m_vaddr;
+		offset_dma = m_dma;
+
+		m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
+		m_dma += OFF_CTRL_LEN + iv_len;
+		m_size -= OFF_CTRL_LEN + iv_len;
+
+		opcode.s.major |= CPT_DMA_MODE;
+
+		vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+		/* DPTR has SG list */
+		in_buffer = m_vaddr;
+		dptr_dma = m_dma;
+
+		((uint16_t *)in_buffer)[0] = 0;
+		((uint16_t *)in_buffer)[1] = 0;
+
+		/* TODO Add error check if space will be sufficient */
+		gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+		/*
+		 * Input Gather List
+		 */
+		i = 0;
+
+		/* Offset control word followed by iv */
+
+		i = fill_sg_comp(gather_comp, i, offset_dma,
+				 OFF_CTRL_LEN + iv_len);
+
+		/* iv offset is 0 */
+		*offset_vaddr = offset_ctrl;
+
+		iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN);
+		memcpy(iv_d, iv, 16);
+
+		/* input data */
+		size = inputlen - iv_len;
+		if (size) {
+			i = fill_sg_comp_from_iov(gather_comp, i,
+						  params->src_iov,
+						  0, &size, NULL, 0);
+			if (size)
+				return ERR_BAD_INPUT_ARG;
+		}
+		((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+		g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+		/*
+		 * Output Scatter List
+		 */
+
+		i = 0;
+		scatter_comp =
+			(sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+		if (flags == 0x1) {
+			/* IV in SLIST only for EEA3 & UEA2 */
+			iv_len = 0;
+		}
+
+		if (iv_len) {
+			i = fill_sg_comp(scatter_comp, i,
+					 offset_dma + OFF_CTRL_LEN, iv_len);
+		}
+
+		/* Add output data */
+		if (req_flags & VALID_MAC_BUF) {
+			size = outputlen - iv_len - mac_len;
+			if (size) {
+				i = fill_sg_comp_from_iov(scatter_comp, i,
+							  params->dst_iov, 0,
+							  &size, NULL, 0);
+
+				if (size)
+					return ERR_BAD_INPUT_ARG;
+			}
+
+			/* mac data */
+			if (mac_len) {
+				i = fill_sg_comp_from_buf(scatter_comp, i,
+							  &params->mac_buf);
+			}
+		} else {
+			/* Output including mac */
+			size = outputlen - iv_len;
+			if (size) {
+				i = fill_sg_comp_from_iov(scatter_comp, i,
+							  params->dst_iov, 0,
+							  &size, NULL, 0);
+
+				if (size)
+					return ERR_BAD_INPUT_ARG;
+			}
+		}
+		((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+		s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+		size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+		/* This is DPTR len incase of SG mode */
+		vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+		m_vaddr = (uint8_t *)m_vaddr + size;
+		m_dma += size;
+		m_size -= size;
+
+		/* cpt alternate completion address saved earlier */
+		req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+		*req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+		rptr_dma = c_dma - 8;
+
+		req->ist.ei1 = dptr_dma;
+		req->ist.ei2 = rptr_dma;
+	}
+
+	/* First 16-bit swap then 64-bit swap */
+	/* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+	 * to eliminate all the swapping
+	 */
+	vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+	/* vq command w3 */
+	vq_cmd_w3.u64 = 0;
+	vq_cmd_w3.s.grp = 0;
+	vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
+		offsetof(struct cpt_ctx, zs_ctx);
+
+	/* 16 byte aligned cpt res address */
+	req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+	*req->completion_addr = COMPLETION_CODE_INIT;
+	req->comp_baddr  = c_dma;
+
+	/* Fill microcode part of instruction */
+	req->ist.ei0 = vq_cmd_w0.u64;
+	req->ist.ei3 = vq_cmd_w3.u64;
+
+	req->op = op;
+
+	*prep_req = req;
+	return 0;
+}
+
+static __rte_always_inline int
+cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
+			uint64_t d_offs,
+			uint64_t d_lens,
+			fc_params_t *params,
+			void *op,
+			void **prep_req)
+{
+	uint32_t size;
+	int32_t inputlen = 0, outputlen;
+	struct cpt_ctx *cpt_ctx;
+	uint8_t snow3g, iv_len = 16;
+	struct cpt_request_info *req;
+	buf_ptr_t *buf_p;
+	uint32_t encr_offset;
+	uint32_t encr_data_len;
+	int flags, m_size;
+	void *m_vaddr, *c_vaddr;
+	uint64_t m_dma, c_dma;
+	uint64_t *offset_vaddr, offset_dma;
+	uint32_t *iv_s, iv[4], j;
+	vq_cmd_word0_t vq_cmd_w0;
+	vq_cmd_word3_t vq_cmd_w3;
+	opcode_info_t opcode;
+
+	buf_p = &params->meta_buf;
+	m_vaddr = buf_p->vaddr;
+	m_dma = buf_p->dma_addr;
+	m_size = buf_p->size;
+
+	/*
+	 * Microcode expects offsets in bytes
+	 * TODO: Rounding off
+	 */
+	encr_offset = ENCR_OFFSET(d_offs) / 8;
+	encr_data_len = ENCR_DLEN(d_lens);
+
+	cpt_ctx = params->ctx_buf.vaddr;
+	flags = cpt_ctx->zsk_flags;
+	snow3g = cpt_ctx->snow3g;
+	/*
+	 * Save initial space that followed app data for completion code &
+	 * alternate completion code to fall in same cache line as app data
+	 */
+	m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+	m_dma += COMPLETION_CODE_SIZE;
+	size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+		(uint8_t *)m_vaddr;
+
+	c_vaddr = (uint8_t *)m_vaddr + size;
+	c_dma = m_dma + size;
+	size += sizeof(cpt_res_s_t);
+
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	/* Reserve memory for cpt request info */
+	req = m_vaddr;
+
+	size = sizeof(struct cpt_request_info);
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G;
+
+	/* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
+	opcode.s.minor = ((1 << 6) | (snow3g << 5) | (0 << 4) |
+			  (0 << 3) | (flags & 0x7));
+
+	/* consider iv len */
+	encr_offset += iv_len;
+
+	inputlen = encr_offset +
+		(RTE_ALIGN(encr_data_len, 8) / 8);
+	outputlen = inputlen;
+
+	/* IV */
+	iv_s = params->iv_buf;
+	if (snow3g) {
+		/*
+		 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
+		 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
+		 */
+
+		for (j = 0; j < 4; j++)
+			iv[j] = iv_s[3 - j];
+	} else {
+		/* ZUC doesn't need a swap */
+		for (j = 0; j < 4; j++)
+			iv[j] = iv_s[j];
+	}
+
+	/*
+	 * GP op header, lengths are expected in bits.
+	 */
+	vq_cmd_w0.u64 = 0;
+	vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+
+	/*
+	 * In 83XX since we have a limitation of
+	 * IV & Offset control word not part of instruction
+	 * and need to be part of Data Buffer, we check if
+	 * head room is there and then only do the Direct mode processing
+	 */
+	if (likely((req_flags & SINGLE_BUF_INPLACE) &&
+		   (req_flags & SINGLE_BUF_HEADTAILROOM))) {
+		void *dm_vaddr = params->bufs[0].vaddr;
+		uint64_t dm_dma_addr = params->bufs[0].dma_addr;
+		/*
+		 * This flag indicates that there is 24 bytes head room and
+		 * 8 bytes tail room available, so that we get to do
+		 * DIRECT MODE with limitation
+		 */
+
+		offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
+					    OFF_CTRL_LEN - iv_len);
+		offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
+
+		/* DPTR */
+		req->ist.ei1 = offset_dma;
+		/* RPTR should just exclude offset control word */
+		req->ist.ei2 = dm_dma_addr - iv_len;
+		req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
+						    + outputlen - iv_len);
+
+		vq_cmd_w0.s.dlen = rte_cpu_to_be_16(inputlen + OFF_CTRL_LEN);
+
+		vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+		if (likely(iv_len)) {
+			uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
+						      + OFF_CTRL_LEN);
+			memcpy(iv_d, iv, 16);
+		}
+
+		/* iv offset is 0 */
+		*offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+	} else {
+		uint32_t i, g_size_bytes, s_size_bytes;
+		uint64_t dptr_dma, rptr_dma;
+		sg_comp_t *gather_comp;
+		sg_comp_t *scatter_comp;
+		uint8_t *in_buffer;
+		uint32_t *iv_d;
+
+		/* save space for offset and iv... */
+		offset_vaddr = m_vaddr;
+		offset_dma = m_dma;
+
+		m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
+		m_dma += OFF_CTRL_LEN + iv_len;
+		m_size -= OFF_CTRL_LEN + iv_len;
+
+		opcode.s.major |= CPT_DMA_MODE;
+
+		vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+		/* DPTR has SG list */
+		in_buffer = m_vaddr;
+		dptr_dma = m_dma;
+
+		((uint16_t *)in_buffer)[0] = 0;
+		((uint16_t *)in_buffer)[1] = 0;
+
+		/* TODO Add error check if space will be sufficient */
+		gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+		/*
+		 * Input Gather List
+		 */
+		i = 0;
+
+		/* Offset control word */
+
+		/* iv offset is 0 */
+		*offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+
+		i = fill_sg_comp(gather_comp, i, offset_dma,
+				 OFF_CTRL_LEN + iv_len);
+
+		iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN);
+		memcpy(iv_d, iv, 16);
+
+		/* Add input data */
+		size = inputlen - iv_len;
+		if (size) {
+			i = fill_sg_comp_from_iov(gather_comp, i,
+						  params->src_iov,
+						  0, &size, NULL, 0);
+			if (size)
+				return ERR_BAD_INPUT_ARG;
+		}
+		((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+		g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+		/*
+		 * Output Scatter List
+		 */
+
+		i = 0;
+		scatter_comp =
+			(sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+		/* IV */
+		i = fill_sg_comp(scatter_comp, i,
+				 offset_dma + OFF_CTRL_LEN,
+				 iv_len);
+
+		/* Add output data */
+		size = outputlen - iv_len;
+		if (size) {
+			i = fill_sg_comp_from_iov(scatter_comp, i,
+						  params->dst_iov, 0,
+						  &size, NULL, 0);
+
+			if (size)
+				return ERR_BAD_INPUT_ARG;
+		}
+		((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+		s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+		size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+		/* This is DPTR len incase of SG mode */
+		vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+		m_vaddr = (uint8_t *)m_vaddr + size;
+		m_dma += size;
+		m_size -= size;
+
+		/* cpt alternate completion address saved earlier */
+		req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+		*req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+		rptr_dma = c_dma - 8;
+
+		req->ist.ei1 = dptr_dma;
+		req->ist.ei2 = rptr_dma;
+	}
+
+	/* First 16-bit swap then 64-bit swap */
+	/* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+	 * to eliminate all the swapping
+	 */
+	vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+	/* vq command w3 */
+	vq_cmd_w3.u64 = 0;
+	vq_cmd_w3.s.grp = 0;
+	vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
+		offsetof(struct cpt_ctx, zs_ctx);
+
+	/* 16 byte aligned cpt res address */
+	req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+	*req->completion_addr = COMPLETION_CODE_INIT;
+	req->comp_baddr  = c_dma;
+
+	/* Fill microcode part of instruction */
+	req->ist.ei0 = vq_cmd_w0.u64;
+	req->ist.ei3 = vq_cmd_w3.u64;
+
+	req->op = op;
+
+	*prep_req = req;
+	return 0;
+}
+
+static __rte_always_inline int
+cpt_kasumi_enc_prep(uint32_t req_flags,
+		    uint64_t d_offs,
+		    uint64_t d_lens,
+		    fc_params_t *params,
+		    void *op,
+		    void **prep_req)
+{
+	uint32_t size;
+	int32_t inputlen = 0, outputlen = 0;
+	struct cpt_ctx *cpt_ctx;
+	uint32_t mac_len = 0;
+	uint8_t i = 0;
+	struct cpt_request_info *req;
+	buf_ptr_t *buf_p;
+	uint32_t encr_offset, auth_offset;
+	uint32_t encr_data_len, auth_data_len;
+	int flags, m_size;
+	uint8_t *iv_s, *iv_d, iv_len = 8;
+	uint8_t dir = 0;
+	void *m_vaddr, *c_vaddr;
+	uint64_t m_dma, c_dma;
+	uint64_t *offset_vaddr, offset_dma;
+	vq_cmd_word0_t vq_cmd_w0;
+	vq_cmd_word3_t vq_cmd_w3;
+	opcode_info_t opcode;
+	uint8_t *in_buffer;
+	uint32_t g_size_bytes, s_size_bytes;
+	uint64_t dptr_dma, rptr_dma;
+	sg_comp_t *gather_comp;
+	sg_comp_t *scatter_comp;
+
+	buf_p = &params->meta_buf;
+	m_vaddr = buf_p->vaddr;
+	m_dma = buf_p->dma_addr;
+	m_size = buf_p->size;
+
+	encr_offset = ENCR_OFFSET(d_offs) / 8;
+	auth_offset = AUTH_OFFSET(d_offs) / 8;
+	encr_data_len = ENCR_DLEN(d_lens);
+	auth_data_len = AUTH_DLEN(d_lens);
+
+	cpt_ctx = params->ctx_buf.vaddr;
+	flags = cpt_ctx->zsk_flags;
+	mac_len = cpt_ctx->mac_len;
+
+	if (flags == 0x0)
+		iv_s = params->iv_buf;
+	else
+		iv_s = params->auth_iv_buf;
+
+	dir = iv_s[8] & 0x1;
+
+	/*
+	 * Save initial space that followed app data for completion code &
+	 * alternate completion code to fall in same cache line as app data
+	 */
+	m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+	m_dma += COMPLETION_CODE_SIZE;
+	size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+		(uint8_t *)m_vaddr;
+
+	c_vaddr = (uint8_t *)m_vaddr + size;
+	c_dma = m_dma + size;
+	size += sizeof(cpt_res_s_t);
+
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	/* Reserve memory for cpt request info */
+	req = m_vaddr;
+
+	size = sizeof(struct cpt_request_info);
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
+
+	/* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
+	opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
+			  (dir << 4) | (0 << 3) | (flags & 0x7));
+
+	/*
+	 * GP op header, lengths are expected in bits.
+	 */
+	vq_cmd_w0.u64 = 0;
+	vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+	vq_cmd_w0.s.param2 = rte_cpu_to_be_16(auth_data_len);
+	vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+	/* consider iv len */
+	if (flags == 0x0) {
+		encr_offset += iv_len;
+		auth_offset += iv_len;
+	}
+
+	/* save space for offset ctrl and iv */
+	offset_vaddr = m_vaddr;
+	offset_dma = m_dma;
+
+	m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
+	m_dma += OFF_CTRL_LEN + iv_len;
+	m_size -= OFF_CTRL_LEN + iv_len;
+
+	/* DPTR has SG list */
+	in_buffer = m_vaddr;
+	dptr_dma = m_dma;
+
+	((uint16_t *)in_buffer)[0] = 0;
+	((uint16_t *)in_buffer)[1] = 0;
+
+	/* TODO Add error check if space will be sufficient */
+	gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+	/*
+	 * Input Gather List
+	 */
+	i = 0;
+
+	/* Offset control word followed by iv */
+
+	if (flags == 0x0) {
+		inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
+		outputlen = inputlen;
+		/* iv offset is 0 */
+		*offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+	} else {
+		inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8);
+		outputlen = mac_len;
+		/* iv offset is 0 */
+		*offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset);
+	}
+
+	i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len);
+
+	/* IV */
+	iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN;
+	memcpy(iv_d, iv_s, iv_len);
+
+	/* input data */
+	size = inputlen - iv_len;
+	if (size) {
+		i = fill_sg_comp_from_iov(gather_comp, i,
+					  params->src_iov, 0,
+					  &size, NULL, 0);
+
+		if (size)
+			return ERR_BAD_INPUT_ARG;
+	}
+	((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+	g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+	/*
+	 * Output Scatter List
+	 */
+
+	i = 0;
+	scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+	if (flags == 0x1) {
+		/* IV in SLIST only for F8 */
+		iv_len = 0;
+	}
+
+	/* IV */
+	if (iv_len) {
+		i = fill_sg_comp(scatter_comp, i,
+				 offset_dma + OFF_CTRL_LEN,
+				 iv_len);
+	}
+
+	/* Add output data */
+	if (req_flags & VALID_MAC_BUF) {
+		size = outputlen - iv_len - mac_len;
+		if (size) {
+			i = fill_sg_comp_from_iov(scatter_comp, i,
+						  params->dst_iov, 0,
+						  &size, NULL, 0);
+
+			if (size)
+				return ERR_BAD_INPUT_ARG;
+		}
+
+		/* mac data */
+		if (mac_len) {
+			i = fill_sg_comp_from_buf(scatter_comp, i,
+						  &params->mac_buf);
+		}
+	} else {
+		/* Output including mac */
+		size = outputlen - iv_len;
+		if (size) {
+			i = fill_sg_comp_from_iov(scatter_comp, i,
+						  params->dst_iov, 0,
+						  &size, NULL, 0);
+
+			if (size)
+				return ERR_BAD_INPUT_ARG;
+		}
+	}
+	((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+	s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+	size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+	/* This is DPTR len incase of SG mode */
+	vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	/* cpt alternate completion address saved earlier */
+	req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+	*req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+	rptr_dma = c_dma - 8;
+
+	req->ist.ei1 = dptr_dma;
+	req->ist.ei2 = rptr_dma;
+
+	/* First 16-bit swap then 64-bit swap */
+	/* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+	 * to eliminate all the swapping
+	 */
+	vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+	/* vq command w3 */
+	vq_cmd_w3.u64 = 0;
+	vq_cmd_w3.s.grp = 0;
+	vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
+		offsetof(struct cpt_ctx, k_ctx);
+
+	/* 16 byte aligned cpt res address */
+	req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+	*req->completion_addr = COMPLETION_CODE_INIT;
+	req->comp_baddr  = c_dma;
+
+	/* Fill microcode part of instruction */
+	req->ist.ei0 = vq_cmd_w0.u64;
+	req->ist.ei3 = vq_cmd_w3.u64;
+
+	req->op = op;
+
+	*prep_req = req;
+	return 0;
+}
+
+static __rte_always_inline int
+cpt_kasumi_dec_prep(uint64_t d_offs,
+		    uint64_t d_lens,
+		    fc_params_t *params,
+		    void *op,
+		    void **prep_req)
+{
+	uint32_t size;
+	int32_t inputlen = 0, outputlen;
+	struct cpt_ctx *cpt_ctx;
+	uint8_t i = 0, iv_len = 8;
+	struct cpt_request_info *req;
+	buf_ptr_t *buf_p;
+	uint32_t encr_offset;
+	uint32_t encr_data_len;
+	int flags, m_size;
+	uint8_t dir = 0;
+	void *m_vaddr, *c_vaddr;
+	uint64_t m_dma, c_dma;
+	uint64_t *offset_vaddr, offset_dma;
+	vq_cmd_word0_t vq_cmd_w0;
+	vq_cmd_word3_t vq_cmd_w3;
+	opcode_info_t opcode;
+	uint8_t *in_buffer;
+	uint32_t g_size_bytes, s_size_bytes;
+	uint64_t dptr_dma, rptr_dma;
+	sg_comp_t *gather_comp;
+	sg_comp_t *scatter_comp;
+
+	buf_p = &params->meta_buf;
+	m_vaddr = buf_p->vaddr;
+	m_dma = buf_p->dma_addr;
+	m_size = buf_p->size;
+
+	encr_offset = ENCR_OFFSET(d_offs) / 8;
+	encr_data_len = ENCR_DLEN(d_lens);
+
+	cpt_ctx = params->ctx_buf.vaddr;
+	flags = cpt_ctx->zsk_flags;
+	/*
+	 * Save initial space that followed app data for completion code &
+	 * alternate completion code to fall in same cache line as app data
+	 */
+	m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+	m_dma += COMPLETION_CODE_SIZE;
+	size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+		(uint8_t *)m_vaddr;
+
+	c_vaddr = (uint8_t *)m_vaddr + size;
+	c_dma = m_dma + size;
+	size += sizeof(cpt_res_s_t);
+
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	/* Reserve memory for cpt request info */
+	req = m_vaddr;
+
+	size = sizeof(struct cpt_request_info);
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
+
+	/* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
+	opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
+			  (dir << 4) | (0 << 3) | (flags & 0x7));
+
+	/*
+	 * GP op header, lengths are expected in bits.
+	 */
+	vq_cmd_w0.u64 = 0;
+	vq_cmd_w0.s.param1 = rte_cpu_to_be_16(encr_data_len);
+	vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+	/* consider iv len */
+	encr_offset += iv_len;
+
+	inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8);
+	outputlen = inputlen;
+
+	/* save space for offset ctrl & iv */
+	offset_vaddr = m_vaddr;
+	offset_dma = m_dma;
+
+	m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
+	m_dma += OFF_CTRL_LEN + iv_len;
+	m_size -= OFF_CTRL_LEN + iv_len;
+
+	/* DPTR has SG list */
+	in_buffer = m_vaddr;
+	dptr_dma = m_dma;
+
+	((uint16_t *)in_buffer)[0] = 0;
+	((uint16_t *)in_buffer)[1] = 0;
+
+	/* TODO Add error check if space will be sufficient */
+	gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+	/*
+	 * Input Gather List
+	 */
+	i = 0;
+
+	/* Offset control word followed by iv */
+	*offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+
+	i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len);
+
+	/* IV */
+	memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN,
+	       params->iv_buf, iv_len);
+
+	/* Add input data */
+	size = inputlen - iv_len;
+	if (size) {
+		i = fill_sg_comp_from_iov(gather_comp, i,
+					  params->src_iov,
+					  0, &size, NULL, 0);
+		if (size)
+			return ERR_BAD_INPUT_ARG;
+	}
+	((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+	g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+	/*
+	 * Output Scatter List
+	 */
+
+	i = 0;
+	scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+	/* IV */
+	i = fill_sg_comp(scatter_comp, i,
+			 offset_dma + OFF_CTRL_LEN,
+			 iv_len);
+
+	/* Add output data */
+	size = outputlen - iv_len;
+	if (size) {
+		i = fill_sg_comp_from_iov(scatter_comp, i,
+					  params->dst_iov, 0,
+					  &size, NULL, 0);
+		if (size)
+			return ERR_BAD_INPUT_ARG;
+	}
+	((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+	s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+	size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+	/* This is DPTR len incase of SG mode */
+	vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+	m_vaddr = (uint8_t *)m_vaddr + size;
+	m_dma += size;
+	m_size -= size;
+
+	/* cpt alternate completion address saved earlier */
+	req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+	*req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+	rptr_dma = c_dma - 8;
+
+	req->ist.ei1 = dptr_dma;
+	req->ist.ei2 = rptr_dma;
+
+	/* First 16-bit swap then 64-bit swap */
+	/* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+	 * to eliminate all the swapping
+	 */
+	vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+	/* vq command w3 */
+	vq_cmd_w3.u64 = 0;
+	vq_cmd_w3.s.grp = 0;
+	vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
+		offsetof(struct cpt_ctx, k_ctx);
+
+	/* 16 byte aligned cpt res address */
+	req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+	*req->completion_addr = COMPLETION_CODE_INIT;
+	req->comp_baddr  = c_dma;
+
+	/* Fill microcode part of instruction */
+	req->ist.ei0 = vq_cmd_w0.u64;
+	req->ist.ei3 = vq_cmd_w3.u64;
+
+	req->op = op;
+
+	*prep_req = req;
+	return 0;
+}
+
+static __rte_always_inline void *
+cpt_fc_dec_hmac_prep(uint32_t flags,
+		     uint64_t d_offs,
+		     uint64_t d_lens,
+		     fc_params_t *fc_params,
+		     void *op, int *ret_val)
+{
+	struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
+	uint8_t fc_type;
+	void *prep_req = NULL;
+	int ret;
+
+	fc_type = ctx->fc_type;
+
+	if (likely(fc_type == FC_GEN)) {
+		ret = cpt_dec_hmac_prep(flags, d_offs, d_lens,
+					fc_params, op, &prep_req);
+	} else if (fc_type == ZUC_SNOW3G) {
+		ret = cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens,
+					      fc_params, op, &prep_req);
+	} else if (fc_type == KASUMI) {
+		ret = cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op,
+					  &prep_req);
+	} else {
+		/*
+		 * For AUTH_ONLY case,
+		 * MC only supports digest generation and verification
+		 * should be done in software by memcmp()
+		 */
+
+		ret = ERR_EIO;
+	}
+
+	if (unlikely(!prep_req))
+		*ret_val = ret;
+	return prep_req;
+}
+
+static __rte_always_inline void *__hot
+cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
+		     fc_params_t *fc_params, void *op, int *ret_val)
+{
+	struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
+	uint8_t fc_type;
+	void *prep_req = NULL;
+	int ret;
+
+	fc_type = ctx->fc_type;
+
+	/* Common api for rest of the ops */
+	if (likely(fc_type == FC_GEN)) {
+		ret = cpt_enc_hmac_prep(flags, d_offs, d_lens,
+					fc_params, op, &prep_req);
+	} else if (fc_type == ZUC_SNOW3G) {
+		ret = cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens,
+					      fc_params, op, &prep_req);
+	} else if (fc_type == KASUMI) {
+		ret = cpt_kasumi_enc_prep(flags, d_offs, d_lens,
+					  fc_params, op, &prep_req);
+	} else if (fc_type == HASH_HMAC) {
+		ret = cpt_digest_gen_prep(flags, d_lens, fc_params, op,
+					  &prep_req);
+	} else {
+		ret = ERR_EIO;
+	}
+
+	if (unlikely(!prep_req))
+		*ret_val = ret;
+	return prep_req;
+}
+
+static __rte_always_inline int
+cpt_fc_auth_set_key(void *ctx, auth_type_t type, uint8_t *key,
+		    uint16_t key_len, uint16_t mac_len)
+{
+	struct cpt_ctx *cpt_ctx = ctx;
+	mc_fc_context_t *fctx = &cpt_ctx->fctx;
+	uint64_t *ctrl_flags = NULL;
+
+	if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) {
+		uint32_t keyx[4];
+
+		if (key_len != 16)
+			return -1;
+		/* No support for AEAD yet */
+		if (cpt_ctx->enc_cipher)
+			return -1;
+		/* For ZUC/SNOW3G/Kasumi */
+		switch (type) {
+		case SNOW3G_UIA2:
+			cpt_ctx->snow3g = 1;
+			gen_key_snow3g(key, keyx);
+			memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len);
+			cpt_ctx->fc_type = ZUC_SNOW3G;
+			cpt_ctx->zsk_flags = 0x1;
+			break;
+		case ZUC_EIA3:
+			cpt_ctx->snow3g = 0;
+			memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len);
+			memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32);
+			cpt_ctx->fc_type = ZUC_SNOW3G;
+			cpt_ctx->zsk_flags = 0x1;
+			break;
+		case KASUMI_F9_ECB:
+			/* Kasumi ECB mode */
+			cpt_ctx->k_ecb = 1;
+			memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
+			cpt_ctx->fc_type = KASUMI;
+			cpt_ctx->zsk_flags = 0x1;
+			break;
+		case KASUMI_F9_CBC:
+			memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
+			cpt_ctx->fc_type = KASUMI;
+			cpt_ctx->zsk_flags = 0x1;
+			break;
+		default:
+			return -1;
+		}
+		cpt_ctx->mac_len = 4;
+		cpt_ctx->hash_type = type;
+		return 0;
+	}
+
+	if (!(cpt_ctx->fc_type == FC_GEN && !type)) {
+		if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher)
+			cpt_ctx->fc_type = HASH_HMAC;
+	}
+
+	ctrl_flags = (uint64_t *)&fctx->enc.enc_ctrl.flags;
+	*ctrl_flags = rte_be_to_cpu_64(*ctrl_flags);
+
+	/* For GMAC auth, cipher must be NULL */
+	if (type == GMAC_TYPE)
+		CPT_P_ENC_CTRL(fctx).enc_cipher = 0;
+
+	CPT_P_ENC_CTRL(fctx).hash_type = cpt_ctx->hash_type = type;
+	CPT_P_ENC_CTRL(fctx).mac_len = cpt_ctx->mac_len = mac_len;
+
+	if (key_len) {
+		cpt_ctx->hmac = 1;
+		memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key));
+		memcpy(cpt_ctx->auth_key, key, key_len);
+		cpt_ctx->auth_key_len = key_len;
+		memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
+		memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad));
+		memcpy(fctx->hmac.opad, key, key_len);
+		CPT_P_ENC_CTRL(fctx).auth_input_type = 1;
+	}
+	*ctrl_flags = rte_cpu_to_be_64(*ctrl_flags);
+	return 0;
+}
+
+static __rte_always_inline int
+fill_sess_aead(struct rte_crypto_sym_xform *xform,
+		 struct cpt_sess_misc *sess)
+{
+	struct rte_crypto_aead_xform *aead_form;
+	cipher_type_t enc_type = 0; /* NULL Cipher type */
+	auth_type_t auth_type = 0; /* NULL Auth type */
+	uint32_t cipher_key_len = 0;
+	uint8_t zsk_flag = 0, aes_gcm = 0;
+	aead_form = &xform->aead;
+	void *ctx;
+
+	if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT &&
+	   aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) {
+		sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
+		sess->cpt_op |= CPT_OP_AUTH_GENERATE;
+	} else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT &&
+		aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) {
+		sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
+		sess->cpt_op |= CPT_OP_AUTH_VERIFY;
+	} else {
+		CPT_LOG_DP_ERR("Unknown cipher operation\n");
+		return -1;
+	}
+	switch (aead_form->algo) {
+	case RTE_CRYPTO_AEAD_AES_GCM:
+		enc_type = AES_GCM;
+		cipher_key_len = 16;
+		aes_gcm = 1;
+		break;
+	case RTE_CRYPTO_AEAD_AES_CCM:
+		CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u",
+			       aead_form->algo);
+		return -1;
+	default:
+		CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
+			       aead_form->algo);
+		return -1;
+	}
+	if (aead_form->key.length < cipher_key_len) {
+		CPT_LOG_DP_ERR("Invalid cipher params keylen %lu",
+			       (unsigned int long)aead_form->key.length);
+		return -1;
+	}
+	sess->zsk_flag = zsk_flag;
+	sess->aes_gcm = aes_gcm;
+	sess->mac_len = aead_form->digest_length;
+	sess->iv_offset = aead_form->iv.offset;
+	sess->iv_length = aead_form->iv.length;
+	sess->aad_length = aead_form->aad_length;
+	ctx = (void *)((uint8_t *)sess + sizeof(struct cpt_sess_misc)),
+
+	cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data,
+			aead_form->key.length, NULL);
+
+	cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, aead_form->digest_length);
+
+	return 0;
+}
+
+static __rte_always_inline int
+fill_sess_cipher(struct rte_crypto_sym_xform *xform,
+		 struct cpt_sess_misc *sess)
+{
+	struct rte_crypto_cipher_xform *c_form;
+	cipher_type_t enc_type = 0; /* NULL Cipher type */
+	uint32_t cipher_key_len = 0;
+	uint8_t zsk_flag = 0, aes_gcm = 0, aes_ctr = 0, is_null = 0;
+
+	if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER)
+		return -1;
+
+	c_form = &xform->cipher;
+
+	if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+		sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
+	else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT)
+		sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
+	else {
+		CPT_LOG_DP_ERR("Unknown cipher operation\n");
+		return -1;
+	}
+
+	switch (c_form->algo) {
+	case RTE_CRYPTO_CIPHER_AES_CBC:
+		enc_type = AES_CBC;
+		cipher_key_len = 16;
+		break;
+	case RTE_CRYPTO_CIPHER_3DES_CBC:
+		enc_type = DES3_CBC;
+		cipher_key_len = 24;
+		break;
+	case RTE_CRYPTO_CIPHER_DES_CBC:
+		/* DES is implemented using 3DES in hardware */
+		enc_type = DES3_CBC;
+		cipher_key_len = 8;
+		break;
+	case RTE_CRYPTO_CIPHER_AES_CTR:
+		enc_type = AES_CTR;
+		cipher_key_len = 16;
+		aes_ctr = 1;
+		break;
+	case RTE_CRYPTO_CIPHER_NULL:
+		enc_type = 0;
+		is_null = 1;
+		break;
+	case RTE_CRYPTO_CIPHER_KASUMI_F8:
+		enc_type = KASUMI_F8_ECB;
+		cipher_key_len = 16;
+		zsk_flag = K_F8;
+		break;
+	case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
+		enc_type = SNOW3G_UEA2;
+		cipher_key_len = 16;
+		zsk_flag = ZS_EA;
+		break;
+	case RTE_CRYPTO_CIPHER_ZUC_EEA3:
+		enc_type = ZUC_EEA3;
+		cipher_key_len = 16;
+		zsk_flag = ZS_EA;
+		break;
+	case RTE_CRYPTO_CIPHER_AES_XTS:
+		enc_type = AES_XTS;
+		cipher_key_len = 16;
+		break;
+	case RTE_CRYPTO_CIPHER_3DES_ECB:
+		enc_type = DES3_ECB;
+		cipher_key_len = 24;
+		break;
+	case RTE_CRYPTO_CIPHER_AES_ECB:
+		enc_type = AES_ECB;
+		cipher_key_len = 16;
+		break;
+	case RTE_CRYPTO_CIPHER_3DES_CTR:
+	case RTE_CRYPTO_CIPHER_AES_F8:
+	case RTE_CRYPTO_CIPHER_ARC4:
+		CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u",
+			       c_form->algo);
+		return -1;
+	default:
+		CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
+			       c_form->algo);
+		return -1;
+	}
+
+	if (c_form->key.length < cipher_key_len) {
+		CPT_LOG_DP_ERR("Invalid cipher params keylen %lu",
+			       (unsigned long) c_form->key.length);
+		return -1;
+	}
+
+	sess->zsk_flag = zsk_flag;
+	sess->aes_gcm = aes_gcm;
+	sess->aes_ctr = aes_ctr;
+	sess->iv_offset = c_form->iv.offset;
+	sess->iv_length = c_form->iv.length;
+	sess->is_null = is_null;
+
+	cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, c_form->key.data,
+			    c_form->key.length, NULL);
+
+	return 0;
+}
+
+static __rte_always_inline int
+fill_sess_auth(struct rte_crypto_sym_xform *xform,
+	       struct cpt_sess_misc *sess)
+{
+	struct rte_crypto_auth_xform *a_form;
+	auth_type_t auth_type = 0; /* NULL Auth type */
+	uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0;
+
+	if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH)
+		goto error_out;
+
+	a_form = &xform->auth;
+
+	if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
+		sess->cpt_op |= CPT_OP_AUTH_VERIFY;
+	else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE)
+		sess->cpt_op |= CPT_OP_AUTH_GENERATE;
+	else {
+		CPT_LOG_DP_ERR("Unknown auth operation");
+		return -1;
+	}
+
+	if (a_form->key.length > 64) {
+		CPT_LOG_DP_ERR("Auth key length is big");
+		return -1;
+	}
+
+	switch (a_form->algo) {
+	case RTE_CRYPTO_AUTH_SHA1_HMAC:
+		/* Fall through */
+	case RTE_CRYPTO_AUTH_SHA1:
+		auth_type = SHA1_TYPE;
+		break;
+	case RTE_CRYPTO_AUTH_SHA256_HMAC:
+	case RTE_CRYPTO_AUTH_SHA256:
+		auth_type = SHA2_SHA256;
+		break;
+	case RTE_CRYPTO_AUTH_SHA512_HMAC:
+	case RTE_CRYPTO_AUTH_SHA512:
+		auth_type = SHA2_SHA512;
+		break;
+	case RTE_CRYPTO_AUTH_AES_GMAC:
+		auth_type = GMAC_TYPE;
+		aes_gcm = 1;
+		break;
+	case RTE_CRYPTO_AUTH_SHA224_HMAC:
+	case RTE_CRYPTO_AUTH_SHA224:
+		auth_type = SHA2_SHA224;
+		break;
+	case RTE_CRYPTO_AUTH_SHA384_HMAC:
+	case RTE_CRYPTO_AUTH_SHA384:
+		auth_type = SHA2_SHA384;
+		break;
+	case RTE_CRYPTO_AUTH_MD5_HMAC:
+	case RTE_CRYPTO_AUTH_MD5:
+		auth_type = MD5_TYPE;
+		break;
+	case RTE_CRYPTO_AUTH_KASUMI_F9:
+		auth_type = KASUMI_F9_ECB;
+		/*
+		 * Indicate that direction needs to be taken out
+		 * from end of src
+		 */
+		zsk_flag = K_F9;
+		break;
+	case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
+		auth_type = SNOW3G_UIA2;
+		zsk_flag = ZS_IA;
+		break;
+	case RTE_CRYPTO_AUTH_ZUC_EIA3:
+		auth_type = ZUC_EIA3;
+		zsk_flag = ZS_IA;
+		break;
+	case RTE_CRYPTO_AUTH_NULL:
+		auth_type = 0;
+		is_null = 1;
+		break;
+	case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
+	case RTE_CRYPTO_AUTH_AES_CMAC:
+	case RTE_CRYPTO_AUTH_AES_CBC_MAC:
+		CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u",
+			       a_form->algo);
+		goto error_out;
+	default:
+		CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified",
+			       a_form->algo);
+		goto error_out;
+	}
+
+	sess->zsk_flag = zsk_flag;
+	sess->aes_gcm = aes_gcm;
+	sess->mac_len = a_form->digest_length;
+	sess->is_null = is_null;
+	if (zsk_flag) {
+		sess->auth_iv_offset = a_form->iv.offset;
+		sess->auth_iv_length = a_form->iv.length;
+	}
+	cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, a_form->key.data,
+			    a_form->key.length, a_form->digest_length);
+
+	return 0;
+
+error_out:
+	return -1;
+}
+
+static __rte_always_inline int
+fill_sess_gmac(struct rte_crypto_sym_xform *xform,
+		 struct cpt_sess_misc *sess)
+{
+	struct rte_crypto_auth_xform *a_form;
+	cipher_type_t enc_type = 0; /* NULL Cipher type */
+	auth_type_t auth_type = 0; /* NULL Auth type */
+	uint8_t zsk_flag = 0, aes_gcm = 0;
+	void *ctx;
+
+	if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH)
+		return -1;
+
+	a_form = &xform->auth;
+
+	if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE)
+		sess->cpt_op |= CPT_OP_ENCODE;
+	else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
+		sess->cpt_op |= CPT_OP_DECODE;
+	else {
+		CPT_LOG_DP_ERR("Unknown auth operation");
+		return -1;
+	}
+
+	switch (a_form->algo) {
+	case RTE_CRYPTO_AUTH_AES_GMAC:
+		enc_type = AES_GCM;
+		auth_type = GMAC_TYPE;
+		break;
+	default:
+		CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
+			       a_form->algo);
+		return -1;
+	}
+
+	sess->zsk_flag = zsk_flag;
+	sess->aes_gcm = aes_gcm;
+	sess->is_gmac = 1;
+	sess->iv_offset = a_form->iv.offset;
+	sess->iv_length = a_form->iv.length;
+	sess->mac_len = a_form->digest_length;
+	ctx = (void *)((uint8_t *)sess + sizeof(struct cpt_sess_misc)),
+
+	cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data,
+			a_form->key.length, NULL);
+	cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, a_form->digest_length);
+
+	return 0;
+}
+
+static __rte_always_inline void *
+alloc_op_meta(struct rte_mbuf *m_src,
+	      buf_ptr_t *buf,
+	      int32_t len,
+	      struct rte_mempool *cpt_meta_pool)
+{
+	uint8_t *mdata;
+
+#ifndef CPT_ALWAYS_USE_SEPARATE_BUF
+	if (likely(m_src && (m_src->nb_segs == 1))) {
+		int32_t tailroom;
+		phys_addr_t mphys;
+
+		/* Check if tailroom is sufficient to hold meta data */
+		tailroom = rte_pktmbuf_tailroom(m_src);
+		if (likely(tailroom > len + 8)) {
+			mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len;
+			mphys = m_src->buf_physaddr + m_src->buf_len;
+			mdata -= len;
+			mphys -= len;
+			buf->vaddr = mdata;
+			buf->dma_addr = mphys;
+			buf->size = len;
+			/* Indicate that this is a mbuf allocated mdata */
+			mdata = (uint8_t *)((uint64_t)mdata | 1ull);
+			return mdata;
+		}
+	}
+#else
+	RTE_SET_USED(m_src);
+#endif
+
+	if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))
+		return NULL;
+
+	buf->vaddr = mdata;
+	buf->dma_addr = rte_mempool_virt2iova(mdata);
+	buf->size = len;
+
+	return mdata;
+}
+
+/**
+ * cpt_free_metabuf - free metabuf to mempool.
+ * @param instance: pointer to instance.
+ * @param objp: pointer to the metabuf.
+ */
+static __rte_always_inline void
+free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool)
+{
+	bool nofree = ((uintptr_t)mdata & 1ull);
+
+	if (likely(nofree))
+		return;
+	rte_mempool_put(cpt_meta_pool, mdata);
+}
+
+static __rte_always_inline uint32_t
+prepare_iov_from_pkt(struct rte_mbuf *pkt,
+		     iov_ptr_t *iovec, uint32_t start_offset)
+{
+	uint16_t index = 0;
+	void *seg_data = NULL;
+	phys_addr_t seg_phys;
+	int32_t seg_size = 0;
+
+	if (!pkt) {
+		iovec->buf_cnt = 0;
+		return 0;
+	}
+
+	if (!start_offset) {
+		seg_data = rte_pktmbuf_mtod(pkt, void *);
+		seg_phys = rte_pktmbuf_mtophys(pkt);
+		seg_size = pkt->data_len;
+	} else {
+		while (start_offset >= pkt->data_len) {
+			start_offset -= pkt->data_len;
+			pkt = pkt->next;
+		}
+
+		seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset);
+		seg_phys = rte_pktmbuf_mtophys_offset(pkt, start_offset);
+		seg_size = pkt->data_len - start_offset;
+		if (!seg_size)
+			return 1;
+	}
+
+	/* first seg */
+	iovec->bufs[index].vaddr = seg_data;
+	iovec->bufs[index].dma_addr = seg_phys;
+	iovec->bufs[index].size = seg_size;
+	index++;
+	pkt = pkt->next;
+
+	while (unlikely(pkt != NULL)) {
+		seg_data = rte_pktmbuf_mtod(pkt, void *);
+		seg_phys = rte_pktmbuf_mtophys(pkt);
+		seg_size = pkt->data_len;
+		if (!seg_size)
+			break;
+
+		iovec->bufs[index].vaddr = seg_data;
+		iovec->bufs[index].dma_addr = seg_phys;
+		iovec->bufs[index].size = seg_size;
+
+		index++;
+
+		pkt = pkt->next;
+	}
+
+	iovec->buf_cnt = index;
+	return 0;
+}
+
+static __rte_always_inline uint32_t
+prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt,
+			     fc_params_t *param,
+			     uint32_t *flags)
+{
+	uint16_t index = 0;
+	void *seg_data = NULL;
+	phys_addr_t seg_phys;
+	uint32_t seg_size = 0;
+	iov_ptr_t *iovec;
+
+	seg_data = rte_pktmbuf_mtod(pkt, void *);
+	seg_phys = rte_pktmbuf_mtophys(pkt);
+	seg_size = pkt->data_len;
+
+	/* first seg */
+	if (likely(!pkt->next)) {
+		uint32_t headroom, tailroom;
+
+		*flags |= SINGLE_BUF_INPLACE;
+		headroom = rte_pktmbuf_headroom(pkt);
+		tailroom = rte_pktmbuf_tailroom(pkt);
+		if (likely((headroom >= 24) &&
+		    (tailroom >= 8))) {
+			/* In 83XX this is prerequivisit for Direct mode */
+			*flags |= SINGLE_BUF_HEADTAILROOM;
+		}
+		param->bufs[0].vaddr = seg_data;
+		param->bufs[0].dma_addr = seg_phys;
+		param->bufs[0].size = seg_size;
+		return 0;
+	}
+	iovec = param->src_iov;
+	iovec->bufs[index].vaddr = seg_data;
+	iovec->bufs[index].dma_addr = seg_phys;
+	iovec->bufs[index].size = seg_size;
+	index++;
+	pkt = pkt->next;
+
+	while (unlikely(pkt != NULL)) {
+		seg_data = rte_pktmbuf_mtod(pkt, void *);
+		seg_phys = rte_pktmbuf_mtophys(pkt);
+		seg_size = pkt->data_len;
+
+		if (!seg_size)
+			break;
+
+		iovec->bufs[index].vaddr = seg_data;
+		iovec->bufs[index].dma_addr = seg_phys;
+		iovec->bufs[index].size = seg_size;
+
+		index++;
+
+		pkt = pkt->next;
+	}
+
+	iovec->buf_cnt = index;
+	return 0;
+}
+
+static __rte_always_inline void *
+fill_fc_params(struct rte_crypto_op *cop,
+	       struct cpt_sess_misc *sess_misc,
+	       void **mdata_ptr,
+	       int *op_ret)
+{
+	uint32_t space = 0;
+	struct rte_crypto_sym_op *sym_op = cop->sym;
+	void *mdata;
+	uintptr_t *op;
+	uint32_t mc_hash_off;
+	uint32_t flags = 0;
+	uint64_t d_offs, d_lens;
+	void *prep_req = NULL;
+	struct rte_mbuf *m_src, *m_dst;
+	uint8_t cpt_op = sess_misc->cpt_op;
+	uint8_t zsk_flag = sess_misc->zsk_flag;
+	uint8_t aes_gcm = sess_misc->aes_gcm;
+	uint16_t mac_len = sess_misc->mac_len;
+#ifdef CPT_ALWAYS_USE_SG_MODE
+	uint8_t inplace = 0;
+#else
+	uint8_t inplace = 1;
+#endif
+	fc_params_t fc_params;
+	char src[SRC_IOV_SIZE];
+	char dst[SRC_IOV_SIZE];
+	uint32_t iv_buf[4];
+	struct cptvf_meta_info *cpt_m_info =
+				(struct cptvf_meta_info *)(*mdata_ptr);
+
+	if (likely(sess_misc->iv_length)) {
+		flags |= VALID_IV_BUF;
+		fc_params.iv_buf = rte_crypto_op_ctod_offset(cop,
+				   uint8_t *, sess_misc->iv_offset);
+		if (sess_misc->aes_ctr &&
+		    unlikely(sess_misc->iv_length != 16)) {
+			memcpy((uint8_t *)iv_buf,
+				rte_crypto_op_ctod_offset(cop,
+				uint8_t *, sess_misc->iv_offset), 12);
+			iv_buf[3] = rte_cpu_to_be_32(0x1);
+			fc_params.iv_buf = iv_buf;
+		}
+	}
+
+	if (zsk_flag) {
+		fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop,
+					uint8_t *,
+					sess_misc->auth_iv_offset);
+		if (zsk_flag == K_F9) {
+			CPT_LOG_DP_ERR("Should not reach here for "
+			"kasumi F9\n");
+		}
+		if (zsk_flag != ZS_EA)
+			inplace = 0;
+	}
+	m_src = sym_op->m_src;
+	m_dst = sym_op->m_dst;
+
+	if (aes_gcm) {
+		uint8_t *salt;
+		uint8_t *aad_data;
+		uint16_t aad_len;
+
+		d_offs = sym_op->aead.data.offset;
+		d_lens = sym_op->aead.data.length;
+		mc_hash_off = sym_op->aead.data.offset +
+			      sym_op->aead.data.length;
+
+		aad_data = sym_op->aead.aad.data;
+		aad_len = sess_misc->aad_length;
+		if (likely((aad_data + aad_len) ==
+			   rte_pktmbuf_mtod_offset(m_src,
+				uint8_t *,
+				sym_op->aead.data.offset))) {
+			d_offs = (d_offs - aad_len) | (d_offs << 16);
+			d_lens = (d_lens + aad_len) | (d_lens << 32);
+		} else {
+			fc_params.aad_buf.vaddr = sym_op->aead.aad.data;
+			fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr;
+			fc_params.aad_buf.size = aad_len;
+			flags |= VALID_AAD_BUF;
+			inplace = 0;
+			d_offs = d_offs << 16;
+			d_lens = d_lens << 32;
+		}
+
+		salt = fc_params.iv_buf;
+		if (unlikely(*(uint32_t *)salt != sess_misc->salt)) {
+			cpt_fc_salt_update(SESS_PRIV(sess_misc), salt);
+			sess_misc->salt = *(uint32_t *)salt;
+		}
+		fc_params.iv_buf = salt + 4;
+		if (likely(mac_len)) {
+			struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst :
+					     m_src;
+
+			if (!m)
+				m = m_src;
+
+			/* hmac immediately following data is best case */
+			if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
+			    mc_hash_off !=
+			    (uint8_t *)sym_op->aead.digest.data)) {
+				flags |= VALID_MAC_BUF;
+				fc_params.mac_buf.size = sess_misc->mac_len;
+				fc_params.mac_buf.vaddr =
+				  sym_op->aead.digest.data;
+				fc_params.mac_buf.dma_addr =
+				 sym_op->aead.digest.phys_addr;
+				inplace = 0;
+			}
+		}
+	} else {
+		d_offs = sym_op->cipher.data.offset;
+		d_lens = sym_op->cipher.data.length;
+		mc_hash_off = sym_op->cipher.data.offset +
+			      sym_op->cipher.data.length;
+		d_offs = (d_offs << 16) | sym_op->auth.data.offset;
+		d_lens = (d_lens << 32) | sym_op->auth.data.length;
+
+		if (mc_hash_off < (sym_op->auth.data.offset +
+				   sym_op->auth.data.length)){
+			mc_hash_off = (sym_op->auth.data.offset +
+				       sym_op->auth.data.length);
+		}
+		/* for gmac, salt should be updated like in gcm */
+		if (unlikely(sess_misc->is_gmac)) {
+			uint8_t *salt;
+			salt = fc_params.iv_buf;
+			if (unlikely(*(uint32_t *)salt != sess_misc->salt)) {
+				cpt_fc_salt_update(SESS_PRIV(sess_misc), salt);
+				sess_misc->salt = *(uint32_t *)salt;
+			}
+			fc_params.iv_buf = salt + 4;
+		}
+		if (likely(mac_len)) {
+			struct rte_mbuf *m;
+
+			m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src;
+			if (!m)
+				m = m_src;
+
+			/* hmac immediately following data is best case */
+			if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
+			    mc_hash_off !=
+			     (uint8_t *)sym_op->auth.digest.data)) {
+				flags |= VALID_MAC_BUF;
+				fc_params.mac_buf.size =
+					sess_misc->mac_len;
+				fc_params.mac_buf.vaddr =
+					sym_op->auth.digest.data;
+				fc_params.mac_buf.dma_addr =
+				sym_op->auth.digest.phys_addr;
+				inplace = 0;
+			}
+		}
+	}
+	fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc);
+	fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr;
+
+	if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE))
+		inplace = 0;
+
+	if (likely(!m_dst && inplace)) {
+		/* Case of single buffer without AAD buf or
+		 * separate mac buf in place and
+		 * not air crypto
+		 */
+		fc_params.dst_iov = fc_params.src_iov = (void *)src;
+
+		if (unlikely(prepare_iov_from_pkt_inplace(m_src,
+							  &fc_params,
+							  &flags))) {
+			CPT_LOG_DP_ERR("Prepare inplace src iov failed");
+			*op_ret = -1;
+			return NULL;
+		}
+
+	} else {
+		/* Out of place processing */
+		fc_params.src_iov = (void *)src;
+		fc_params.dst_iov = (void *)dst;
+
+		/* Store SG I/O in the api for reuse */
+		if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) {
+			CPT_LOG_DP_ERR("Prepare src iov failed");
+			*op_ret = -1;
+			return NULL;
+		}
+
+		if (unlikely(m_dst != NULL)) {
+			uint32_t pkt_len;
+
+			/* Try to make room as much as src has */
+			m_dst = sym_op->m_dst;
+			pkt_len = rte_pktmbuf_pkt_len(m_dst);
+
+			if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) {
+				pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len;
+				if (!rte_pktmbuf_append(m_dst, pkt_len)) {
+					CPT_LOG_DP_ERR("Not enough space in "
+						       "m_dst %p, need %u"
+						       " more",
+						       m_dst, pkt_len);
+					return NULL;
+				}
+			}
+
+			if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) {
+				CPT_LOG_DP_ERR("Prepare dst iov failed for "
+					       "m_dst %p", m_dst);
+				return NULL;
+			}
+		} else {
+			fc_params.dst_iov = (void *)src;
+		}
+	}
+
+	if (likely(flags & SINGLE_BUF_HEADTAILROOM))
+		mdata = alloc_op_meta(m_src,
+				      &fc_params.meta_buf,
+				      cpt_m_info->cptvf_op_sb_mlen,
+				      cpt_m_info->cptvf_meta_pool);
+	else
+		mdata = alloc_op_meta(NULL,
+				      &fc_params.meta_buf,
+				      cpt_m_info->cptvf_op_mlen,
+				      cpt_m_info->cptvf_meta_pool);
+
+	if (unlikely(mdata == NULL)) {
+		CPT_LOG_DP_ERR("Error allocating meta buffer for request");
+		return NULL;
+	}
+
+	op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull);
+	op[0] = (uintptr_t)mdata;
+	op[1] = (uintptr_t)cop;
+	op[2] = op[3] = 0; /* Used to indicate auth verify */
+	space += 4 * sizeof(uint64_t);
+
+	fc_params.meta_buf.vaddr = (uint8_t *)op + space;
+	fc_params.meta_buf.dma_addr += space;
+	fc_params.meta_buf.size -= space;
+
+	/* Finally prepare the instruction */
+	if (cpt_op & CPT_OP_ENCODE)
+		prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens,
+						&fc_params, op, op_ret);
+	else
+		prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens,
+						&fc_params, op, op_ret);
+
+	if (unlikely(!prep_req))
+		free_op_meta(mdata, cpt_m_info->cptvf_meta_pool);
+	*mdata_ptr = mdata;
+	return prep_req;
+}
+
+static __rte_always_inline void
+compl_auth_verify(struct rte_crypto_op *op,
+		      uint8_t *gen_mac,
+		      uint64_t mac_len)
+{
+	uint8_t *mac;
+	struct rte_crypto_sym_op *sym_op = op->sym;
+
+	if (sym_op->auth.digest.data)
+		mac = sym_op->auth.digest.data;
+	else
+		mac = rte_pktmbuf_mtod_offset(sym_op->m_src,
+					      uint8_t *,
+					      sym_op->auth.data.length +
+					      sym_op->auth.data.offset);
+	if (!mac) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return;
+	}
+
+	if (memcmp(mac, gen_mac, mac_len))
+		op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+	else
+		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+}
+
+static __rte_always_inline int
+instance_session_cfg(struct rte_crypto_sym_xform *xform, void *sess)
+{
+	struct rte_crypto_sym_xform *chain;
+
+	CPT_PMD_INIT_FUNC_TRACE();
+
+	if (cpt_is_algo_supported(xform))
+		goto err;
+
+	chain = xform;
+	while (chain) {
+		switch (chain->type) {
+		case RTE_CRYPTO_SYM_XFORM_AEAD:
+			if (fill_sess_aead(chain, sess))
+				goto err;
+			break;
+		case RTE_CRYPTO_SYM_XFORM_CIPHER:
+			if (fill_sess_cipher(chain, sess))
+				goto err;
+			break;
+		case RTE_CRYPTO_SYM_XFORM_AUTH:
+			if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
+				if (fill_sess_gmac(chain, sess))
+					goto err;
+			} else {
+				if (fill_sess_auth(chain, sess))
+					goto err;
+			}
+			break;
+		default:
+			CPT_LOG_DP_ERR("Invalid crypto xform type");
+			break;
+		}
+		chain = chain->next;
+	}
+
+	return 0;
+
+err:
+	return -1;
+}
+
+static __rte_always_inline void
+find_kasumif9_direction_and_length(uint8_t *src,
+				   uint32_t counter_num_bytes,
+				   uint32_t *addr_length_in_bits,
+				   uint8_t *addr_direction)
+{
+	uint8_t found = 0;
+	while (!found && counter_num_bytes > 0) {
+		counter_num_bytes--;
+		if (src[counter_num_bytes] == 0x00)
+			continue;
+		if (src[counter_num_bytes] == 0x80) {
+			*addr_direction  =  src[counter_num_bytes - 1] & 0x1;
+			*addr_length_in_bits = counter_num_bytes * 8  - 1;
+			found = 1;
+		} else {
+			int i = 0;
+			uint8_t last_byte = src[counter_num_bytes];
+			for (i = 0; i < 8 && found == 0; i++) {
+				if (last_byte & (1 << i)) {
+					*addr_direction = (last_byte >> (i+1))
+							  & 0x1;
+					if (i != 6)
+						*addr_length_in_bits =
+							counter_num_bytes * 8
+							+ (8 - (i + 2));
+					else
+						*addr_length_in_bits =
+							counter_num_bytes * 8;
+					found = 1;
+					}
+				}
+			}
+	}
+}
+
+/*
+ * This handles all auth only except AES_GMAC
+ */
+static __rte_always_inline void *
+fill_digest_params(struct rte_crypto_op *cop,
+		   struct cpt_sess_misc *sess,
+		   void **mdata_ptr,
+		   int *op_ret)
+{
+	uint32_t space = 0;
+	struct rte_crypto_sym_op *sym_op = cop->sym;
+	void *mdata;
+	phys_addr_t mphys;
+	uint64_t *op;
+	uint32_t auth_range_off;
+	uint32_t flags = 0;
+	uint64_t d_offs = 0, d_lens;
+	void *prep_req = NULL;
+	struct rte_mbuf *m_src, *m_dst;
+	uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK;
+	uint8_t zsk_flag = sess->zsk_flag;
+	uint16_t mac_len = sess->mac_len;
+	fc_params_t params;
+	char src[SRC_IOV_SIZE];
+	uint8_t iv_buf[16];
+	memset(&params, 0, sizeof(fc_params_t));
+	struct cptvf_meta_info *cpt_m_info =
+				(struct cptvf_meta_info *)(*mdata_ptr);
+
+	m_src = sym_op->m_src;
+
+	/* For just digest lets force mempool alloc */
+	mdata = alloc_op_meta(NULL, &params.meta_buf, cpt_m_info->cptvf_op_mlen,
+			      cpt_m_info->cptvf_meta_pool);
+	if (mdata == NULL) {
+		CPT_LOG_DP_ERR("Error allocating meta buffer for request");
+		*op_ret = -ENOMEM;
+		return NULL;
+	}
+
+	mphys = params.meta_buf.dma_addr;
+
+	op = mdata;
+	op[0] = (uintptr_t)mdata;
+	op[1] = (uintptr_t)cop;
+	op[2] = op[3] = 0; /* Used to indicate auth verify */
+	space += 4 * sizeof(uint64_t);
+
+	auth_range_off = sym_op->auth.data.offset;
+
+	flags = VALID_MAC_BUF;
+	params.src_iov = (void *)src;
+	if (unlikely(zsk_flag)) {
+		/*
+		 * Since for Zuc, Kasumi, Snow3g offsets are in bits
+		 * we will send pass through even for auth only case,
+		 * let MC handle it
+		 */
+		d_offs = auth_range_off;
+		auth_range_off = 0;
+		params.auth_iv_buf = rte_crypto_op_ctod_offset(cop,
+					uint8_t *, sess->auth_iv_offset);
+		if (zsk_flag == K_F9) {
+			uint32_t length_in_bits, num_bytes;
+			uint8_t *src, direction = 0;
+			uint32_t counter_num_bytes;
+
+			memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src,
+							uint8_t *), 8);
+			/*
+			 * This is kasumi f9, take direction from
+			 * source buffer
+			 */
+			length_in_bits = cop->sym->auth.data.length;
+			num_bytes = (length_in_bits >> 3);
+			counter_num_bytes = num_bytes;
+			src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *);
+			find_kasumif9_direction_and_length(src,
+						counter_num_bytes,
+						&length_in_bits,
+						&direction);
+			length_in_bits -= 64;
+			cop->sym->auth.data.offset += 64;
+			d_offs = cop->sym->auth.data.offset;
+			auth_range_off = d_offs / 8;
+			cop->sym->auth.data.length = length_in_bits;
+
+			/* Store it at end of auth iv */
+			iv_buf[8] = direction;
+			params.auth_iv_buf = iv_buf;
+		}
+	}
+
+	d_lens = sym_op->auth.data.length;
+
+	params.ctx_buf.vaddr = SESS_PRIV(sess);
+	params.ctx_buf.dma_addr = sess->ctx_dma_addr;
+
+	if (auth_op == CPT_OP_AUTH_GENERATE) {
+		if (sym_op->auth.digest.data) {
+			/*
+			 * Digest to be generated
+			 * in separate buffer
+			 */
+			params.mac_buf.size =
+				sess->mac_len;
+			params.mac_buf.vaddr =
+				sym_op->auth.digest.data;
+			params.mac_buf.dma_addr =
+				sym_op->auth.digest.phys_addr;
+		} else {
+			uint32_t off = sym_op->auth.data.offset +
+				sym_op->auth.data.length;
+			int32_t dlen, space;
+
+			m_dst = sym_op->m_dst ?
+				sym_op->m_dst : sym_op->m_src;
+			dlen = rte_pktmbuf_pkt_len(m_dst);
+
+			space = off + mac_len - dlen;
+			if (space > 0)
+				if (!rte_pktmbuf_append(m_dst, space)) {
+					CPT_LOG_DP_ERR("Failed to extend "
+						       "mbuf by %uB", space);
+					goto err;
+				}
+
+			params.mac_buf.vaddr =
+				rte_pktmbuf_mtod_offset(m_dst, void *, off);
+			params.mac_buf.dma_addr =
+				rte_pktmbuf_mtophys_offset(m_dst, off);
+			params.mac_buf.size = mac_len;
+		}
+	} else {
+		/* Need space for storing generated mac */
+		params.mac_buf.vaddr = (uint8_t *)mdata + space;
+		params.mac_buf.dma_addr = mphys + space;
+		params.mac_buf.size = mac_len;
+		space += RTE_ALIGN_CEIL(mac_len, 8);
+		op[2] = (uintptr_t)params.mac_buf.vaddr;
+		op[3] = mac_len;
+	}
+
+	params.meta_buf.vaddr = (uint8_t *)mdata + space;
+	params.meta_buf.dma_addr = mphys + space;
+	params.meta_buf.size -= space;
+
+	/* Out of place processing */
+	params.src_iov = (void *)src;
+
+	/*Store SG I/O in the api for reuse */
+	if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) {
+		CPT_LOG_DP_ERR("Prepare src iov failed");
+		*op_ret = -1;
+		goto err;
+	}
+
+	prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens,
+					&params, op, op_ret);
+	*mdata_ptr = mdata;
+	return prep_req;
+err:
+	if (unlikely(!prep_req))
+		free_op_meta(mdata, cpt_m_info->cptvf_meta_pool);
+	return NULL;
+}
+
+#endif /*_CPT_UCODE_H_ */
diff --git a/drivers/common/cpt/meson.build b/drivers/common/cpt/meson.build
new file mode 100644
index 00000000..0a905aa4
--- /dev/null
+++ b/drivers/common/cpt/meson.build
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Cavium, Inc
+
+sources = files('cpt_pmd_ops_helper.c')
+
+deps = ['kvargs', 'pci', 'cryptodev']
+includes += include_directories('../../crypto/octeontx')
+allow_experimental_apis = true
diff --git a/drivers/common/cpt/rte_common_cpt_version.map b/drivers/common/cpt/rte_common_cpt_version.map
new file mode 100644
index 00000000..dec614f0
--- /dev/null
+++ b/drivers/common/cpt/rte_common_cpt_version.map
@@ -0,0 +1,6 @@
+DPDK_18.11 {
+	global:
+
+	cpt_pmd_ops_helper_get_mlen_direct_mode;
+	cpt_pmd_ops_helper_get_mlen_sg_mode;
+};
diff --git a/drivers/common/dpaax/Makefile b/drivers/common/dpaax/Makefile
new file mode 100644
index 00000000..94d2cf0c
--- /dev/null
+++ b/drivers/common/dpaax/Makefile
@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2018 NXP
+#
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_common_dpaax.a
+
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+
+# versioning export map
+EXPORT_MAP := rte_common_dpaax_version.map
+
+# library version
+LIBABIVER := 1
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-y += dpaax_iova_table.c
+
+LDLIBS += -lrte_eal
+
+SYMLINK-y-include += dpaax_iova_table.h
+
+include $(RTE_SDK)/mk/rte.lib.mk
\ No newline at end of file
diff --git a/drivers/common/dpaax/dpaax_iova_table.c b/drivers/common/dpaax/dpaax_iova_table.c
new file mode 100644
index 00000000..2dd38a92
--- /dev/null
+++ b/drivers/common/dpaax/dpaax_iova_table.c
@@ -0,0 +1,465 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#include <rte_memory.h>
+
+#include "dpaax_iova_table.h"
+#include "dpaax_logs.h"
+
+/* Global dpaax logger identifier */
+int dpaax_logger;
+
+/* Global table reference */
+struct dpaax_iova_table *dpaax_iova_table_p;
+
+static int dpaax_handle_memevents(void);
+
+/* A structure representing the device-tree node available in /proc/device-tree.
+ */
+struct reg_node {
+	phys_addr_t addr;
+	size_t len;
+};
+
+/* A ntohll equivalent routine
+ * XXX: This is only applicable for 64 bit environment.
+ */
+static void
+rotate_8(unsigned char *arr)
+{
+	uint32_t temp;
+	uint32_t *first_half;
+	uint32_t *second_half;
+
+	first_half = (uint32_t *)(arr);
+	second_half = (uint32_t *)(arr + 4);
+
+	temp = *first_half;
+	*first_half = *second_half;
+	*second_half = temp;
+
+	*first_half = ntohl(*first_half);
+	*second_half = ntohl(*second_half);
+}
+
+/* read_memory_nodes
+ * Memory layout for DPAAx platforms (LS1043, LS1046, LS1088, LS2088, LX2160)
+ * are populated by Uboot and available in device tree:
+ * /proc/device-tree/memory@<address>/reg <= register.
+ * Entries are of the form:
+ *  (<8 byte start addr><8 byte length>)(..more similar blocks of start,len>)..
+ *
+ * @param count
+ *    OUT populate number of entries found in memory node
+ * @return
+ *    Pointer to array of reg_node elements, count size
+ */
+static struct reg_node *
+read_memory_node(unsigned int *count)
+{
+	int fd, ret, i;
+	unsigned int j;
+	glob_t result = {0};
+	struct stat statbuf = {0};
+	char file_data[MEM_NODE_FILE_LEN];
+	struct reg_node *nodes = NULL;
+
+	*count = 0;
+
+	ret = glob(MEM_NODE_PATH_GLOB, 0, NULL, &result);
+	if (ret != 0) {
+		DPAAX_DEBUG("Unable to glob device-tree memory node: (%s)(%d)",
+			    MEM_NODE_PATH_GLOB, ret);
+		goto out;
+	}
+
+	if (result.gl_pathc != 1) {
+		/* Either more than one memory@<addr> node found, or none.
+		 * In either case, cannot work ahead.
+		 */
+		DPAAX_DEBUG("Found (%zu) entries in device-tree. Not supported!",
+			    result.gl_pathc);
+		goto out;
+	}
+
+	DPAAX_DEBUG("Opening and parsing device-tree node: (%s)",
+		    result.gl_pathv[0]);
+	fd = open(result.gl_pathv[0], O_RDONLY);
+	if (fd < 0) {
+		DPAAX_DEBUG("Unable to open the device-tree node: (%s)(fd=%d)",
+			    MEM_NODE_PATH_GLOB, fd);
+		goto cleanup;
+	}
+
+	/* Stat to get the file size */
+	ret = fstat(fd, &statbuf);
+	if (ret != 0) {
+		DPAAX_DEBUG("Unable to get device-tree memory node size.");
+		goto cleanup;
+	}
+
+	DPAAX_DEBUG("Size of device-tree mem node: %lu", statbuf.st_size);
+	if (statbuf.st_size > MEM_NODE_FILE_LEN) {
+		DPAAX_DEBUG("More memory nodes available than assumed.");
+		DPAAX_DEBUG("System may not work properly!");
+	}
+
+	ret = read(fd, file_data, statbuf.st_size > MEM_NODE_FILE_LEN ?
+				  MEM_NODE_FILE_LEN : statbuf.st_size);
+	if (ret <= 0) {
+		DPAAX_DEBUG("Unable to read device-tree memory node: (%d)",
+			    ret);
+		goto cleanup;
+	}
+
+	/* The reg node should be multiple of 16 bytes, 8 bytes each for addr
+	 * and len.
+	 */
+	*count = (statbuf.st_size / 16);
+	if ((*count) <= 0 || (statbuf.st_size % 16 != 0)) {
+		DPAAX_DEBUG("Invalid memory node values or count. (size=%lu)",
+			    statbuf.st_size);
+		goto cleanup;
+	}
+
+	/* each entry is of 16 bytes, and size/16 is total count of entries */
+	nodes = malloc(sizeof(struct reg_node) * (*count));
+	if (!nodes) {
+		DPAAX_DEBUG("Failure in allocating working memory.");
+		goto cleanup;
+	}
+	memset(nodes, 0, sizeof(struct reg_node) * (*count));
+
+	for (i = 0, j = 0; i < (statbuf.st_size) && j < (*count); i += 16, j++) {
+		memcpy(&nodes[j], file_data + i, 16);
+		/* Rotate (ntohl) each 8 byte entry */
+		rotate_8((unsigned char *)(&(nodes[j].addr)));
+		rotate_8((unsigned char *)(&(nodes[j].len)));
+	}
+
+	DPAAX_DEBUG("Device-tree memory node data:");
+	do {
+		DPAAX_DEBUG("\n    %08" PRIx64 " %08zu", nodes[j].addr, nodes[j].len);
+	} while (--j);
+
+cleanup:
+	close(fd);
+	globfree(&result);
+out:
+	return nodes;
+}
+
+int
+dpaax_iova_table_populate(void)
+{
+	int ret;
+	unsigned int i, node_count;
+	size_t tot_memory_size, total_table_size;
+	struct reg_node *nodes;
+	struct dpaax_iovat_element *entry;
+
+	/* dpaax_iova_table_p is a singleton - only one instance should be
+	 * created.
+	 */
+	if (dpaax_iova_table_p) {
+		DPAAX_DEBUG("Multiple allocation attempt for IOVA Table (%p)",
+			    dpaax_iova_table_p);
+		/* This can be an error case as well - some path not cleaning
+		 * up table - but, for now, it is assumed that if IOVA Table
+		 * pointer is valid, table is allocated.
+		 */
+		return 0;
+	}
+
+	nodes = read_memory_node(&node_count);
+	if (nodes == NULL) {
+		DPAAX_WARN("PA->VA translation not available;");
+		DPAAX_WARN("Expect performance impact.");
+		return -1;
+	}
+
+	tot_memory_size = 0;
+	for (i = 0; i < node_count; i++)
+		tot_memory_size += nodes[i].len;
+
+	DPAAX_DEBUG("Total available PA memory size: %zu", tot_memory_size);
+
+	/* Total table size = meta data + tot_memory_size/8 */
+	total_table_size = sizeof(struct dpaax_iova_table) +
+			   (sizeof(struct dpaax_iovat_element) * node_count) +
+			   ((tot_memory_size / DPAAX_MEM_SPLIT) * sizeof(uint64_t));
+
+	/* TODO: This memory doesn't need to shared but needs to be always
+	 * pinned to RAM (no swap out) - using hugepage rather than malloc
+	 */
+	dpaax_iova_table_p = rte_zmalloc(NULL, total_table_size, 0);
+	if (dpaax_iova_table_p == NULL) {
+		DPAAX_WARN("Unable to allocate memory for PA->VA Table;");
+		DPAAX_WARN("PA->VA translation not available;");
+		DPAAX_WARN("Expect performance impact.");
+		free(nodes);
+		return -1;
+	}
+
+	/* Initialize table */
+	dpaax_iova_table_p->count = node_count;
+	entry = dpaax_iova_table_p->entries;
+
+	DPAAX_DEBUG("IOVA Table entries: (entry start = %p)", (void *)entry);
+	DPAAX_DEBUG("\t(entry),(start),(len),(next)");
+
+	for (i = 0; i < node_count; i++) {
+		/* dpaax_iova_table_p
+		 * |   dpaax_iova_table_p->entries
+		 * |      |
+		 * |      |
+		 * V      V
+		 * +------+------+-------+---+----------+---------+---
+		 * |iova_ |entry | entry |   | pages    | pages   |
+		 * |table | 1    |  2    |...| entry 1  | entry2  |
+		 * +-----'+.-----+-------+---+;---------+;--------+---
+		 *         \      \          /          /
+		 *          `~~~~~~|~~~~~>pages        /
+		 *                  \                 /
+		 *                   `~~~~~~~~~~~>pages
+		 */
+		entry[i].start = nodes[i].addr;
+		entry[i].len = nodes[i].len;
+		if (i > 0)
+			entry[i].pages = entry[i-1].pages +
+				((entry[i-1].len/DPAAX_MEM_SPLIT));
+		else
+			entry[i].pages = (uint64_t *)((unsigned char *)entry +
+					 (sizeof(struct dpaax_iovat_element) *
+					 node_count));
+
+		DPAAX_DEBUG("\t(%u),(%8"PRIx64"),(%8zu),(%8p)",
+			    i, entry[i].start, entry[i].len, entry[i].pages);
+	}
+
+	/* Release memory associated with nodes array - not required now */
+	free(nodes);
+
+	DPAAX_DEBUG("Adding mem-event handler\n");
+	ret = dpaax_handle_memevents();
+	if (ret) {
+		DPAAX_ERR("Unable to add mem-event handler");
+		DPAAX_WARN("Cases with non-buffer pool mem won't work!");
+	}
+
+	return 0;
+}
+
+void
+dpaax_iova_table_depopulate(void)
+{
+	if (dpaax_iova_table_p == NULL)
+		return;
+
+	rte_free(dpaax_iova_table_p->entries);
+	dpaax_iova_table_p = NULL;
+
+	DPAAX_DEBUG("IOVA Table cleanedup");
+}
+
+int
+dpaax_iova_table_update(phys_addr_t paddr, void *vaddr, size_t length)
+{
+	int found = 0;
+	unsigned int i;
+	size_t req_length = length, e_offset;
+	struct dpaax_iovat_element *entry;
+	uintptr_t align_vaddr;
+	phys_addr_t align_paddr;
+
+	if (unlikely(dpaax_iova_table_p == NULL))
+		return -1;
+
+	align_paddr = paddr & DPAAX_MEM_SPLIT_MASK;
+	align_vaddr = ((uintptr_t)vaddr & DPAAX_MEM_SPLIT_MASK);
+
+	/* Check if paddr is available in table */
+	entry = dpaax_iova_table_p->entries;
+	for (i = 0; i < dpaax_iova_table_p->count; i++) {
+		if (align_paddr < entry[i].start) {
+			/* Address lower than start, but not found in previous
+			 * iteration shouldn't exist.
+			 */
+			DPAAX_ERR("Add: Incorrect entry for PA->VA Table"
+				  "(%"PRIu64")", paddr);
+			DPAAX_ERR("Add: Lowest address: %"PRIu64"",
+				  entry[i].start);
+			return -1;
+		}
+
+		if (align_paddr > (entry[i].start + entry[i].len))
+			continue;
+
+		/* align_paddr >= start && align_paddr < (start + len) */
+		found = 1;
+
+		do {
+			e_offset = ((align_paddr - entry[i].start) / DPAAX_MEM_SPLIT);
+			/* TODO: Whatif something already exists at this
+			 * location - is that an error? For now, ignoring the
+			 * case.
+			 */
+			entry[i].pages[e_offset] = align_vaddr;
+			DPAAX_DEBUG("Added: vaddr=%zu for Phy:%"PRIu64" at %zu"
+				    " remaining len %zu", align_vaddr,
+				    align_paddr, e_offset, req_length);
+
+			/* Incoming request can be larger than the
+			 * DPAAX_MEM_SPLIT size - in which case, multiple
+			 * entries in entry->pages[] are filled up.
+			 */
+			if (req_length <= DPAAX_MEM_SPLIT)
+				break;
+			align_paddr += DPAAX_MEM_SPLIT;
+			align_vaddr += DPAAX_MEM_SPLIT;
+			req_length -= DPAAX_MEM_SPLIT;
+		} while (1);
+
+		break;
+	}
+
+	if (!found) {
+		/* There might be case where the incoming physical address is
+		 * beyond the address discovered in the memory node of
+		 * device-tree. Specially if some malloc'd area is used by EAL
+		 * and the memevent handlers passes that across. But, this is
+		 * not necessarily an error.
+		 */
+		DPAAX_DEBUG("Add: Unable to find slot for vaddr:(%p),"
+			    " phy(%"PRIu64")",
+			    vaddr, paddr);
+		return -1;
+	}
+
+	DPAAX_DEBUG("Add: Found slot at (%"PRIu64")[(%zu)] for vaddr:(%p),"
+		    " phy(%"PRIu64"), len(%zu)", entry[i].start, e_offset,
+		    vaddr, paddr, length);
+	return 0;
+}
+
+/* dpaax_iova_table_dump
+ * Dump the table, with its entries, on screen. Only works in Debug Mode
+ * Not for weak hearted - the tables can get quite large
+ */
+void
+dpaax_iova_table_dump(void)
+{
+	unsigned int i, j;
+	struct dpaax_iovat_element *entry;
+
+	/* In case DEBUG is not enabled, some 'if' conditions might misbehave
+	 * as they have nothing else in them  except a DPAAX_DEBUG() which if
+	 * tuned out would leave 'if' naked.
+	 */
+	if (rte_log_get_global_level() < RTE_LOG_DEBUG) {
+		DPAAX_ERR("Set log level to Debug for PA->Table dump!");
+		return;
+	}
+
+	DPAAX_DEBUG(" === Start of PA->VA Translation Table ===");
+	if (dpaax_iova_table_p == NULL)
+		DPAAX_DEBUG("\tNULL");
+
+	entry = dpaax_iova_table_p->entries;
+	for (i = 0; i < dpaax_iova_table_p->count; i++) {
+		DPAAX_DEBUG("\t(%16i),(%16"PRIu64"),(%16zu),(%16p)",
+			    i, entry[i].start, entry[i].len, entry[i].pages);
+		DPAAX_DEBUG("\t\t          (PA),          (VA)");
+		for (j = 0; j < (entry->len/DPAAX_MEM_SPLIT); j++) {
+			if (entry[i].pages[j] == 0)
+				continue;
+			DPAAX_DEBUG("\t\t(%16"PRIx64"),(%16"PRIx64")",
+				    (entry[i].start + (j * sizeof(uint64_t))),
+				    entry[i].pages[j]);
+		}
+	}
+	DPAAX_DEBUG(" === End of PA->VA Translation Table ===");
+}
+
+static void
+dpaax_memevent_cb(enum rte_mem_event type, const void *addr, size_t len,
+		  void *arg __rte_unused)
+{
+	struct rte_memseg_list *msl;
+	struct rte_memseg *ms;
+	size_t cur_len = 0, map_len = 0;
+	phys_addr_t phys_addr;
+	void *virt_addr;
+	int ret;
+
+	DPAAX_DEBUG("Called with addr=%p, len=%zu", addr, len);
+
+	msl = rte_mem_virt2memseg_list(addr);
+
+	while (cur_len < len) {
+		const void *va = RTE_PTR_ADD(addr, cur_len);
+
+		ms = rte_mem_virt2memseg(va, msl);
+		phys_addr = rte_mem_virt2phy(ms->addr);
+		virt_addr = ms->addr;
+		map_len = ms->len;
+
+		DPAAX_DEBUG("Request for %s, va=%p, virt_addr=%p,"
+			    "iova=%"PRIu64", map_len=%zu",
+			    type == RTE_MEM_EVENT_ALLOC ?
+			    "alloc" : "dealloc",
+			    va, virt_addr, phys_addr, map_len);
+
+		if (type == RTE_MEM_EVENT_ALLOC)
+			ret = dpaax_iova_table_update(phys_addr, virt_addr,
+						      map_len);
+		else
+			/* In case of mem_events for MEM_EVENT_FREE, complete
+			 * hugepage is released and its PA entry is set to 0.
+			 */
+			ret = dpaax_iova_table_update(phys_addr, 0, map_len);
+
+		if (ret != 0) {
+			DPAAX_DEBUG("PA-Table entry update failed. "
+				    "Map=%d, addr=%p, len=%zu, err:(%d)",
+				    type, va, map_len, ret);
+			return;
+		}
+
+		cur_len += map_len;
+	}
+}
+
+static int
+dpaax_memevent_walk_memsegs(const struct rte_memseg_list *msl __rte_unused,
+			    const struct rte_memseg *ms, size_t len,
+			    void *arg __rte_unused)
+{
+	DPAAX_DEBUG("Walking for %p (pa=%"PRIu64") and len %zu",
+		    ms->addr, ms->phys_addr, len);
+	dpaax_iova_table_update(rte_mem_virt2phy(ms->addr), ms->addr, len);
+	return 0;
+}
+
+static int
+dpaax_handle_memevents(void)
+{
+	/* First, walk through all memsegs and pin them, before installing
+	 * handler. This assures that all memseg which have already been
+	 * identified/allocated by EAL, are already part of PA->VA Table. This
+	 * is especially for cases where application allocates memory before
+	 * the EAL or this is an externally allocated memory passed to EAL.
+	 */
+	rte_memseg_contig_walk_thread_unsafe(dpaax_memevent_walk_memsegs, NULL);
+
+	return rte_mem_event_callback_register("dpaax_memevents_cb",
+					       dpaax_memevent_cb, NULL);
+}
+
+RTE_INIT(dpaax_log)
+{
+	dpaax_logger = rte_log_register("pmd.common.dpaax");
+	if (dpaax_logger >= 0)
+		rte_log_set_level(dpaax_logger, RTE_LOG_ERR);
+}
diff --git a/drivers/common/dpaax/dpaax_iova_table.h b/drivers/common/dpaax/dpaax_iova_table.h
new file mode 100644
index 00000000..138827e7
--- /dev/null
+++ b/drivers/common/dpaax/dpaax_iova_table.h
@@ -0,0 +1,105 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#ifndef _DPAAX_IOVA_TABLE_H_
+#define _DPAAX_IOVA_TABLE_H_
+
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <inttypes.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <glob.h>
+#include <errno.h>
+#include <arpa/inet.h>
+
+#include <rte_eal.h>
+#include <rte_branch_prediction.h>
+#include <rte_memory.h>
+#include <rte_malloc.h>
+
+struct dpaax_iovat_element {
+	phys_addr_t start; /**< Start address of block of physical pages */
+	size_t len; /**< Difference of end-start for quick access */
+	uint64_t *pages; /**< VA for each physical page in this block */
+};
+
+struct dpaax_iova_table {
+	unsigned int count; /**< No. of blocks of contiguous physical pages */
+	struct dpaax_iovat_element entries[0];
+};
+
+/* Pointer to the table, which is common for DPAA/DPAA2 and only a single
+ * instance is required across net/crypto/event drivers. This table is
+ * populated iff devices are found on the bus.
+ */
+extern struct dpaax_iova_table *dpaax_iova_table_p;
+
+/* Device tree file for memory layout is named 'memory@<addr>' where the 'addr'
+ * is SoC dependent, or even Uboot fixup dependent.
+ */
+#define MEM_NODE_PATH_GLOB "/proc/device-tree/memory[@0-9]*/reg"
+/* Device file should be multiple of 16 bytes, each containing 8 byte of addr
+ * and its length. Assuming max of 5 entries.
+ */
+#define MEM_NODE_FILE_LEN ((16 * 5) + 1)
+
+/* Table is made up of DPAAX_MEM_SPLIT elements for each contiguous zone. This
+ * helps avoid separate handling for cases where more than one size of hugepage
+ * is supported.
+ */
+#define DPAAX_MEM_SPLIT (1<<21)
+#define DPAAX_MEM_SPLIT_MASK ~(DPAAX_MEM_SPLIT - 1) /**< Floor aligned */
+#define DPAAX_MEM_SPLIT_MASK_OFF (DPAAX_MEM_SPLIT - 1) /**< Offset */
+
+/* APIs exposed */
+int dpaax_iova_table_populate(void);
+void dpaax_iova_table_depopulate(void);
+int dpaax_iova_table_update(phys_addr_t paddr, void *vaddr, size_t length);
+void dpaax_iova_table_dump(void);
+
+static inline void *dpaax_iova_table_get_va(phys_addr_t paddr) __attribute__((hot));
+
+static inline void *
+dpaax_iova_table_get_va(phys_addr_t paddr) {
+	unsigned int i = 0, index;
+	void *vaddr = 0;
+	phys_addr_t paddr_align = paddr & DPAAX_MEM_SPLIT_MASK;
+	size_t offset = paddr & DPAAX_MEM_SPLIT_MASK_OFF;
+	struct dpaax_iovat_element *entry;
+
+	if (unlikely(dpaax_iova_table_p == NULL))
+		return NULL;
+
+	entry = dpaax_iova_table_p->entries;
+
+	do {
+		if (unlikely(i > dpaax_iova_table_p->count))
+			break;
+
+		if (paddr_align < entry[i].start) {
+			/* Incorrect paddr; Not in memory range */
+			return NULL;
+		}
+
+		if (paddr_align > (entry[i].start + entry[i].len)) {
+			i++;
+			continue;
+		}
+
+		/* paddr > entry->start && paddr <= entry->(start+len) */
+		index = (paddr_align - entry[i].start)/DPAAX_MEM_SPLIT;
+		vaddr = (void *)((uintptr_t)entry[i].pages[index] + offset);
+		break;
+	} while (1);
+
+	return vaddr;
+}
+
+#endif /* _DPAAX_IOVA_TABLE_H_ */
diff --git a/drivers/common/dpaax/dpaax_logs.h b/drivers/common/dpaax/dpaax_logs.h
new file mode 100644
index 00000000..bf1b27cc
--- /dev/null
+++ b/drivers/common/dpaax/dpaax_logs.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#ifndef _DPAAX_LOGS_H_
+#define _DPAAX_LOGS_H_
+
+#include <rte_log.h>
+
+extern int dpaax_logger;
+
+#define DPAAX_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, dpaax_logger, "dpaax: " fmt "\n", \
+		##args)
+
+/* Debug logs are with Function names */
+#define DPAAX_DEBUG(fmt, args...) \
+	rte_log(RTE_LOG_DEBUG, dpaax_logger, "dpaax: %s():	 " fmt "\n", \
+		__func__, ##args)
+
+#define DPAAX_INFO(fmt, args...) \
+	DPAAX_LOG(INFO, fmt, ## args)
+#define DPAAX_ERR(fmt, args...) \
+	DPAAX_LOG(ERR, fmt, ## args)
+#define DPAAX_WARN(fmt, args...) \
+	DPAAX_LOG(WARNING, fmt, ## args)
+
+/* DP Logs, toggled out at compile time if level lower than current level */
+#define DPAAX_DP_LOG(level, fmt, args...) \
+	RTE_LOG_DP(level, PMD, fmt, ## args)
+
+#define DPAAX_DP_DEBUG(fmt, args...) \
+	DPAAX_DP_LOG(DEBUG, fmt, ## args)
+#define DPAAX_DP_INFO(fmt, args...) \
+	DPAAX_DP_LOG(INFO, fmt, ## args)
+#define DPAAX_DP_WARN(fmt, args...) \
+	DPAAX_DP_LOG(WARNING, fmt, ## args)
+
+#endif /* _DPAAX_LOGS_H_ */
diff --git a/drivers/common/dpaax/meson.build b/drivers/common/dpaax/meson.build
new file mode 100644
index 00000000..98a1bdd4
--- /dev/null
+++ b/drivers/common/dpaax/meson.build
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 NXP
+
+allow_experimental_apis = true
+
+if host_machine.system() != 'linux'
+        build = false
+endif
+
+sources = files('dpaax_iova_table.c')
+
+cflags += ['-D_GNU_SOURCE']
diff --git a/drivers/common/dpaax/rte_common_dpaax_version.map b/drivers/common/dpaax/rte_common_dpaax_version.map
new file mode 100644
index 00000000..8131c9e3
--- /dev/null
+++ b/drivers/common/dpaax/rte_common_dpaax_version.map
@@ -0,0 +1,11 @@
+DPDK_18.11 {
+	global:
+
+	dpaax_iova_table_update;
+	dpaax_iova_table_depopulate;
+	dpaax_iova_table_dump;
+	dpaax_iova_table_p;
+	dpaax_iova_table_populate;
+
+	local: *;
+};
diff --git a/drivers/common/meson.build b/drivers/common/meson.build
index d7b7d8cf..a5093410 100644
--- a/drivers/common/meson.build
+++ b/drivers/common/meson.build
@@ -2,6 +2,6 @@
 # Copyright(c) 2018 Cavium, Inc
 
 std_deps = ['eal']
-drivers = ['octeontx', 'qat']
+drivers = ['cpt', 'dpaax', 'mvep', 'octeontx', 'qat']
 config_flag_fmt = 'RTE_LIBRTE_@0@_COMMON'
 driver_name_fmt = 'rte_common_@0@'
diff --git a/drivers/common/mvep/Makefile b/drivers/common/mvep/Makefile
new file mode 100644
index 00000000..1f5f005d
--- /dev/null
+++ b/drivers/common/mvep/Makefile
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Marvell International Ltd.
+#
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(MAKECMDGOALS),config)
+ifeq ($(LIBMUSDK_PATH),)
+$(error "Please define LIBMUSDK_PATH environment variable")
+endif
+endif
+endif
+
+# library name
+LIB = librte_common_mvep.a
+
+# library version
+LIBABIVER := 1
+
+# versioning export map
+EXPORT_MAP := rte_common_mvep_version.map
+
+# external library dependencies
+CFLAGS += -I$($RTE_SDK)/drivers/common/mvep
+CFLAGS += -I$(LIBMUSDK_PATH)/include
+CFLAGS += -DMVCONF_TYPES_PUBLIC
+CFLAGS += -DMVCONF_DMA_PHYS_ADDR_T_PUBLIC
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -O3
+LDLIBS += -L$(LIBMUSDK_PATH)/lib
+LDLIBS += -lmusdk
+LDLIBS += -lrte_eal -lrte_kvargs
+
+# library source files
+SRCS-y += mvep_common.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/common/mvep/meson.build b/drivers/common/mvep/meson.build
new file mode 100644
index 00000000..8ccfacb3
--- /dev/null
+++ b/drivers/common/mvep/meson.build
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Marvell International Ltd.
+# Copyright(c) 2018 Semihalf.
+# All rights reserved.
+#
+path = get_option('lib_musdk_dir')
+lib_dir = path + '/lib'
+inc_dir = path + '/include'
+
+lib = cc.find_library('libmusdk', dirs: [lib_dir], required: false)
+if not lib.found()
+	build = false
+else
+	ext_deps += lib
+	includes += include_directories(inc_dir)
+	cflags += ['-DMVCONF_TYPES_PUBLIC', '-DMVCONF_DMA_PHYS_ADDR_T_PUBLIC']
+endif
+
+sources = files('mvep_common.c')
diff --git a/drivers/common/mvep/mvep_common.c b/drivers/common/mvep/mvep_common.c
new file mode 100644
index 00000000..67fa65b5
--- /dev/null
+++ b/drivers/common/mvep/mvep_common.c
@@ -0,0 +1,45 @@
+/*  SPDX-License-Identifier: BSD-3-Clause
+ *  Copyright(c) 2018 Marvell International Ltd.
+ */
+
+#include <rte_common.h>
+
+#include <env/mv_autogen_comp_flags.h>
+#include <env/mv_sys_dma.h>
+
+#include "rte_mvep_common.h"
+
+/* Memory size (in bytes) for MUSDK dma buffers */
+#define MRVL_MUSDK_DMA_MEMSIZE (40 * 1024 * 1024)
+
+struct mvep {
+	uint32_t ref_count;
+};
+
+static struct mvep mvep;
+
+int rte_mvep_init(enum mvep_module_type module __rte_unused,
+		  struct rte_kvargs *kvlist __rte_unused)
+{
+	int ret;
+
+	if (!mvep.ref_count) {
+		ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE);
+		if (ret)
+			return ret;
+	}
+
+	mvep.ref_count++;
+
+	return 0;
+}
+
+int rte_mvep_deinit(enum mvep_module_type module __rte_unused)
+{
+	mvep.ref_count--;
+
+	if (!mvep.ref_count)
+		mv_sys_dma_mem_destroy();
+
+	return 0;
+}
diff --git a/drivers/common/mvep/rte_common_mvep_version.map b/drivers/common/mvep/rte_common_mvep_version.map
new file mode 100644
index 00000000..c71722d7
--- /dev/null
+++ b/drivers/common/mvep/rte_common_mvep_version.map
@@ -0,0 +1,6 @@
+DPDK_18.11 {
+	global:
+
+	rte_mvep_init;
+	rte_mvep_deinit;
+};
diff --git a/drivers/common/mvep/rte_mvep_common.h b/drivers/common/mvep/rte_mvep_common.h
new file mode 100644
index 00000000..0593cefc
--- /dev/null
+++ b/drivers/common/mvep/rte_mvep_common.h
@@ -0,0 +1,21 @@
+/*  SPDX-License-Identifier: BSD-3-Clause
+ *  Copyright(c) 2018 Marvell International Ltd.
+ */
+
+#ifndef __RTE_MVEP_COMMON_H__
+#define __RTE_MVEP_COMMON_H__
+
+#include <rte_kvargs.h>
+
+enum mvep_module_type {
+	MVEP_MOD_T_NONE = 0,
+	MVEP_MOD_T_PP2,
+	MVEP_MOD_T_SAM,
+	MVEP_MOD_T_NETA,
+	MVEP_MOD_T_LAST
+};
+
+int rte_mvep_init(enum mvep_module_type module, struct rte_kvargs *kvlist);
+int rte_mvep_deinit(enum mvep_module_type module);
+
+#endif /* __RTE_MVEP_COMMON_H__ */
diff --git a/drivers/common/qat/qat_common.h b/drivers/common/qat/qat_common.h
index d4bef539..de9a3ba5 100644
--- a/drivers/common/qat/qat_common.h
+++ b/drivers/common/qat/qat_common.h
@@ -17,7 +17,8 @@
  */
 enum qat_device_gen {
 	QAT_GEN1 = 1,
-	QAT_GEN2
+	QAT_GEN2,
+	QAT_GEN3
 };
 
 enum qat_service_type {
diff --git a/drivers/common/qat/qat_device.c b/drivers/common/qat/qat_device.c
index f32d7235..2a1cf3e1 100644
--- a/drivers/common/qat/qat_device.c
+++ b/drivers/common/qat/qat_device.c
@@ -7,6 +7,7 @@
 #include "qat_device.h"
 #include "adf_transport_access_macros.h"
 #include "qat_sym_pmd.h"
+#include "qat_comp_pmd.h"
 
 /* Hardware device information per generation */
 __extension__
@@ -14,11 +15,18 @@ struct qat_gen_hw_data qat_gen_config[] =  {
 	[QAT_GEN1] = {
 		.dev_gen = QAT_GEN1,
 		.qp_hw_data = qat_gen1_qps,
+		.comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1
 	},
 	[QAT_GEN2] = {
 		.dev_gen = QAT_GEN2,
 		.qp_hw_data = qat_gen1_qps,
 		/* gen2 has same ring layout as gen1 */
+		.comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2
+	},
+	[QAT_GEN3] = {
+		.dev_gen = QAT_GEN3,
+		.qp_hw_data = qat_gen3_qps,
+		.comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
 	},
 };
 
@@ -43,10 +51,12 @@ static const struct rte_pci_id pci_id_qat_map[] = {
 		{
 			RTE_PCI_DEVICE(0x8086, 0x6f55),
 		},
+		{
+			RTE_PCI_DEVICE(0x8086, 0x18a1),
+		},
 		{.device_id = 0},
 };
 
-
 static struct qat_pci_device *
 qat_pci_get_dev(uint8_t dev_id)
 {
@@ -130,6 +140,9 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev)
 	case 0x6f55:
 		qat_dev->qat_dev_gen = QAT_GEN2;
 		break;
+	case 0x18a1:
+		qat_dev->qat_dev_gen = QAT_GEN3;
+		break;
 	default:
 		QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
 		return NULL;
@@ -187,6 +200,7 @@ static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 		struct rte_pci_device *pci_dev)
 {
 	int ret = 0;
+	int num_pmds_created = 0;
 	struct qat_pci_device *qat_pci_dev;
 
 	QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
@@ -199,23 +213,33 @@ static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 		return -ENODEV;
 
 	ret = qat_sym_dev_create(qat_pci_dev);
-	if (ret != 0)
-		goto error_out;
+	if (ret == 0)
+		num_pmds_created++;
+	else
+		QAT_LOG(WARNING,
+				"Failed to create QAT SYM PMD on device %s",
+				qat_pci_dev->name);
 
 	ret = qat_comp_dev_create(qat_pci_dev);
-	if (ret != 0)
-		goto error_out;
+	if (ret == 0)
+		num_pmds_created++;
+	else
+		QAT_LOG(WARNING,
+				"Failed to create QAT COMP PMD on device %s",
+				qat_pci_dev->name);
 
 	ret = qat_asym_dev_create(qat_pci_dev);
-	if (ret != 0)
-		goto error_out;
-
-	return 0;
+	if (ret == 0)
+		num_pmds_created++;
+	else
+		QAT_LOG(WARNING,
+				"Failed to create QAT ASYM PMD on device %s",
+				qat_pci_dev->name);
 
-error_out:
-	qat_pci_dev_destroy(qat_pci_dev, pci_dev);
-	return ret;
+	if (num_pmds_created == 0)
+		qat_pci_dev_destroy(qat_pci_dev, pci_dev);
 
+	return 0;
 }
 
 static int qat_pci_remove(struct rte_pci_device *pci_dev)
@@ -239,37 +263,37 @@ static struct rte_pci_driver rte_qat_pmd = {
 	.remove = qat_pci_remove
 };
 
-__attribute__((weak)) int
+__rte_weak int
 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
 {
 	return 0;
 }
 
-__attribute__((weak)) int
+__rte_weak int
 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
 {
 	return 0;
 }
 
-__attribute__((weak)) int
+__rte_weak int
 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 {
 	return 0;
 }
 
-__attribute__((weak)) int
+__rte_weak int
 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 {
 	return 0;
 }
 
-__attribute__((weak)) int
+__rte_weak int
 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
 {
 	return 0;
 }
 
-__attribute__((weak)) int
+__rte_weak int
 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 {
 	return 0;
diff --git a/drivers/common/qat/qat_device.h b/drivers/common/qat/qat_device.h
index 9599fc59..eb81c78f 100644
--- a/drivers/common/qat/qat_device.h
+++ b/drivers/common/qat/qat_device.h
@@ -16,6 +16,12 @@
 
 #define QAT_DEV_NAME_MAX_LEN	64
 
+enum qat_comp_num_im_buffers {
+	QAT_NUM_INTERM_BUFS_GEN1 = 12,
+	QAT_NUM_INTERM_BUFS_GEN2 = 20,
+	QAT_NUM_INTERM_BUFS_GEN3 = 20
+};
+
 /*
  * This struct holds all the data about a QAT pci device
  * including data about all services it supports.
@@ -59,6 +65,11 @@ struct qat_pci_device {
 	/* Data relating to compression service */
 	struct qat_comp_dev_private *comp_dev;
 	/**< link back to compressdev private data */
+	struct rte_device comp_rte_dev;
+	/**< This represents the compression subset of this pci device.
+	 * Register with this rather than with the one in
+	 * pci_dev so that its driver can have a compression-specific name
+	 */
 
 	/* Data relating to asymmetric crypto service */
 
@@ -67,6 +78,7 @@ struct qat_pci_device {
 struct qat_gen_hw_data {
 	enum qat_device_gen dev_gen;
 	const struct qat_qp_hw_data (*qp_hw_data)[ADF_MAX_QPS_ON_ANY_SERVICE];
+	enum qat_comp_num_im_buffers comp_num_im_bufs_required;
 };
 
 extern struct qat_gen_hw_data qat_gen_config[];
diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c
index 7ca7a45e..79f6a01b 100644
--- a/drivers/common/qat/qat_qp.c
+++ b/drivers/common/qat/qat_qp.c
@@ -90,6 +90,44 @@ const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
 	}
 };
 
+__extension__
+const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES]
+					 [ADF_MAX_QPS_ON_ANY_SERVICE] = {
+	/* queue pairs which provide an asymmetric crypto service */
+	[QAT_SERVICE_ASYMMETRIC] = {
+		{
+			.service_type = QAT_SERVICE_ASYMMETRIC,
+			.hw_bundle_num = 0,
+			.tx_ring_num = 0,
+			.rx_ring_num = 4,
+			.tx_msg_size = 64,
+			.rx_msg_size = 32,
+		}
+	},
+	/* queue pairs which provide a symmetric crypto service */
+	[QAT_SERVICE_SYMMETRIC] = {
+		{
+			.service_type = QAT_SERVICE_SYMMETRIC,
+			.hw_bundle_num = 0,
+			.tx_ring_num = 1,
+			.rx_ring_num = 5,
+			.tx_msg_size = 128,
+			.rx_msg_size = 32,
+		}
+	},
+	/* queue pairs which provide a compression service */
+	[QAT_SERVICE_COMPRESSION] = {
+		{
+			.service_type = QAT_SERVICE_COMPRESSION,
+			.hw_bundle_num = 0,
+			.tx_ring_num = 3,
+			.rx_ring_num = 7,
+			.tx_msg_size = 128,
+			.rx_msg_size = 32,
+		}
+	}
+};
+
 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 	uint32_t queue_size_bytes);
 static void qat_queue_delete(struct qat_queue *queue);
@@ -596,15 +634,23 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 	uint32_t head;
 	uint32_t resp_counter = 0;
 	uint8_t *resp_msg;
+	uint8_t hdr_flags;
 
 	rx_queue = &(tmp_qp->rx_q);
 	tx_queue = &(tmp_qp->tx_q);
 	head = rx_queue->head;
 	resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
+	hdr_flags = ((struct icp_qat_fw_comn_resp_hdr *)resp_msg)->hdr_flags;
 
 	while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 			resp_counter != nb_ops) {
 
+		if (unlikely(!ICP_QAT_FW_COMN_VALID_FLAG_GET(hdr_flags))) {
+			/* Fatal firmware error */
+			QAT_LOG(ERR, "QAT Firmware returned invalid response");
+			return 0;
+		}
+
 		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 			qat_sym_process_response(ops, resp_msg);
 		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
@@ -635,7 +681,7 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 	return resp_counter;
 }
 
-__attribute__((weak)) int
+__rte_weak int
 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused)
 {
 	return  0;
diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h
index 69f8a613..6f1525e1 100644
--- a/drivers/common/qat/qat_qp.h
+++ b/drivers/common/qat/qat_qp.h
@@ -85,6 +85,7 @@ struct qat_qp {
 } __rte_cache_aligned;
 
 extern const struct qat_qp_hw_data qat_gen1_qps[][ADF_MAX_QPS_ON_ANY_SERVICE];
+extern const struct qat_qp_hw_data qat_gen3_qps[][ADF_MAX_QPS_ON_ANY_SERVICE];
 
 uint16_t
 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops);
diff --git a/drivers/compress/octeontx/include/zip_regs.h b/drivers/compress/octeontx/include/zip_regs.h
index 1e74db43..04c3d75e 100644
--- a/drivers/compress/octeontx/include/zip_regs.h
+++ b/drivers/compress/octeontx/include/zip_regs.h
@@ -12,12 +12,12 @@
  * ZIP compression coding Enumeration
  * Enumerates ZIP_INST_S[CC].
  */
-enum {
+enum zip_cc {
 	ZIP_CC_DEFAULT = 0,
 	ZIP_CC_DYN_HUFF,
 	ZIP_CC_FIXED_HUFF,
 	ZIP_CC_LZS
-} zip_cc;
+};
 
 /**
  * Register (NCB) zip_vq#_ena
diff --git a/drivers/compress/octeontx/otx_zip.h b/drivers/compress/octeontx/otx_zip.h
index 99a38d00..3abefd1d 100644
--- a/drivers/compress/octeontx/otx_zip.h
+++ b/drivers/compress/octeontx/otx_zip.h
@@ -79,7 +79,7 @@ int octtx_zip_logtype_driver;
 	ZIP_PMD_LOG(ERR, fmt, ## args)
 
 /* resources required to process stream */
-enum {
+enum NUM_BUFS_PER_STREAM {
 	RES_BUF = 0,
 	CMD_BUF,
 	HASH_CTX_BUF,
@@ -88,7 +88,7 @@ enum {
 	OUT_DATA_BUF,
 	HISTORY_DATA_BUF,
 	MAX_BUFS_PER_STREAM
-} NUM_BUFS_PER_STREAM;
+};
 
 struct zip_stream;
 struct zipvf_qp;
@@ -106,7 +106,7 @@ struct zip_stream {
 	comp_func_t func;
 	/* function to process comp operation */
 	void *bufs[MAX_BUFS_PER_STREAM];
-} _rte_cache_aligned;
+} __rte_cache_aligned;
 
 
 /**
diff --git a/drivers/compress/octeontx/otx_zip_pmd.c b/drivers/compress/octeontx/otx_zip_pmd.c
index 9d13f933..67ff5066 100644
--- a/drivers/compress/octeontx/otx_zip_pmd.c
+++ b/drivers/compress/octeontx/otx_zip_pmd.c
@@ -533,7 +533,7 @@ zip_pmd_dequeue_burst_sync(void *queue_pair,
 	return nb_dequeued;
 }
 
-struct rte_compressdev_ops octtx_zip_pmd_ops = {
+static struct rte_compressdev_ops octtx_zip_pmd_ops = {
 		.dev_configure		= zip_pmd_config,
 		.dev_start		= zip_pmd_start,
 		.dev_stop		= zip_pmd_stop,
diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c
index 38c8a5b8..d70c5949 100644
--- a/drivers/compress/qat/qat_comp.c
+++ b/drivers/compress/qat/qat_comp.c
@@ -145,7 +145,6 @@ qat_comp_process_response(void **op, uint8_t *resp)
 		rx_op->debug_status =
 			*((uint16_t *)(&resp_msg->comn_resp.comn_error));
 	} else {
-		struct qat_comp_xform *qat_xform = rx_op->private_xform;
 		struct icp_qat_fw_resp_comp_pars *comp_resp =
 		  (struct icp_qat_fw_resp_comp_pars *)&resp_msg->comp_resp_pars;
 
@@ -193,7 +192,7 @@ static void qat_comp_create_req_hdr(struct icp_qat_fw_comn_req_hdr *header,
 }
 
 static int qat_comp_create_templates(struct qat_comp_xform *qat_xform,
-			const struct rte_memzone *interm_buff_mz __rte_unused,
+			const struct rte_memzone *interm_buff_mz,
 			const struct rte_comp_xform *xform)
 {
 	struct icp_qat_fw_comp_req *comp_req;
@@ -281,10 +280,20 @@ static int qat_comp_create_templates(struct qat_comp_xform *qat_xform,
 		ICP_QAT_FW_COMN_CURR_ID_SET(&comp_req->comp_cd_ctrl,
 					    ICP_QAT_FW_SLICE_COMP);
 	} else if (qat_xform->qat_comp_request_type ==
-		   QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS) {
+			QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS) {
 
-		QAT_LOG(ERR, "Dynamic huffman encoding not supported");
-		return -EINVAL;
+		ICP_QAT_FW_COMN_NEXT_ID_SET(&comp_req->comp_cd_ctrl,
+				ICP_QAT_FW_SLICE_XLAT);
+		ICP_QAT_FW_COMN_CURR_ID_SET(&comp_req->comp_cd_ctrl,
+				ICP_QAT_FW_SLICE_COMP);
+
+		ICP_QAT_FW_COMN_NEXT_ID_SET(&comp_req->u2.xlt_cd_ctrl,
+				ICP_QAT_FW_SLICE_DRAM_WR);
+		ICP_QAT_FW_COMN_CURR_ID_SET(&comp_req->u2.xlt_cd_ctrl,
+				ICP_QAT_FW_SLICE_XLAT);
+
+		comp_req->u1.xlt_pars.inter_buff_ptr =
+				interm_buff_mz->phys_addr;
 	}
 
 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
@@ -335,27 +344,35 @@ qat_comp_private_xform_create(struct rte_compressdev *dev,
 			(struct qat_comp_xform *)*private_xform;
 
 	if (xform->type == RTE_COMP_COMPRESS) {
-		if (xform->compress.deflate.huffman ==
-				RTE_COMP_HUFFMAN_DYNAMIC) {
-			QAT_LOG(ERR,
-			"QAT device doesn't support dynamic compression");
-			return -ENOTSUP;
-		}
 
 		if (xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED ||
 		  ((xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_DEFAULT)
 				   && qat->interm_buff_mz == NULL))
-
 			qat_xform->qat_comp_request_type =
 					QAT_COMP_REQUEST_FIXED_COMP_STATELESS;
 
+		else if ((xform->compress.deflate.huffman ==
+				RTE_COMP_HUFFMAN_DYNAMIC ||
+				xform->compress.deflate.huffman ==
+						RTE_COMP_HUFFMAN_DEFAULT) &&
+				qat->interm_buff_mz != NULL)
+
+			qat_xform->qat_comp_request_type =
+					QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS;
+
+		else {
+			QAT_LOG(ERR,
+					"IM buffers needed for dynamic deflate. Set size in config file");
+			return -EINVAL;
+		}
+
+		qat_xform->checksum_type = xform->compress.chksum;
 
 	} else {
 		qat_xform->qat_comp_request_type = QAT_COMP_REQUEST_DECOMPRESS;
+		qat_xform->checksum_type = xform->decompress.chksum;
 	}
 
-	qat_xform->checksum_type = xform->compress.chksum;
-
 	if (qat_comp_create_templates(qat_xform, qat->interm_buff_mz, xform)) {
 		QAT_LOG(ERR, "QAT: Problem with setting compression");
 		return -EINVAL;
diff --git a/drivers/compress/qat/qat_comp.h b/drivers/compress/qat/qat_comp.h
index 8d315efb..99a4462e 100644
--- a/drivers/compress/qat/qat_comp.h
+++ b/drivers/compress/qat/qat_comp.h
@@ -15,6 +15,10 @@
 #include "icp_qat_fw_comp.h"
 #include "icp_qat_fw_la.h"
 
+#define QAT_64_BYTE_ALIGN_MASK (~0x3f)
+#define QAT_64_BYTE_ALIGN (64)
+#define QAT_NUM_BUFS_IN_IM_SGL 1
+
 #define ERR_CODE_QAT_COMP_WRONG_FW -99
 
 enum qat_comp_request_type {
@@ -24,6 +28,15 @@ enum qat_comp_request_type {
 	REQ_COMP_END
 };
 
+struct array_of_ptrs {
+	phys_addr_t pointer[0];
+};
+
+struct qat_inter_sgl {
+	qat_sgl_hdr;
+	struct qat_flat_buf buffers[QAT_NUM_BUFS_IN_IM_SGL];
+} __rte_packed __rte_cache_aligned;
+
 struct qat_comp_sgl {
 	qat_sgl_hdr;
 	struct qat_flat_buf buffers[RTE_PMD_QAT_COMP_SGL_MAX_SEGMENTS];
diff --git a/drivers/compress/qat/qat_comp_pmd.c b/drivers/compress/qat/qat_comp_pmd.c
index b89975fc..01dd7361 100644
--- a/drivers/compress/qat/qat_comp_pmd.c
+++ b/drivers/compress/qat/qat_comp_pmd.c
@@ -14,6 +14,7 @@ static const struct rte_compressdev_capabilities qat_comp_gen_capabilities[] = {
 				RTE_COMP_FF_CRC32_ADLER32_CHECKSUM |
 				RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
 				RTE_COMP_FF_HUFFMAN_FIXED |
+				RTE_COMP_FF_HUFFMAN_DYNAMIC |
 				RTE_COMP_FF_OOP_SGL_IN_SGL_OUT |
 				RTE_COMP_FF_OOP_SGL_IN_LB_OUT |
 				RTE_COMP_FF_OOP_LB_IN_SGL_OUT,
@@ -112,7 +113,7 @@ qat_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
 
 	/* store a link to the qp in the qat_pci_device */
 	qat_private->qat_dev->qps_in_use[QAT_SERVICE_COMPRESSION][qp_id]
-							= *qp_addr;
+								= *qp_addr;
 
 	qp = (struct qat_qp *)*qp_addr;
 
@@ -135,6 +136,103 @@ qat_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
 	return ret;
 }
 
+
+#define QAT_IM_BUFFER_DEBUG 0
+static const struct rte_memzone *
+qat_comp_setup_inter_buffers(struct qat_comp_dev_private *comp_dev,
+			      uint32_t buff_size)
+{
+	char inter_buff_mz_name[RTE_MEMZONE_NAMESIZE];
+	const struct rte_memzone *memzone;
+	uint8_t *mz_start = NULL;
+	rte_iova_t mz_start_phys = 0;
+	struct array_of_ptrs *array_of_pointers;
+	int size_of_ptr_array;
+	uint32_t full_size;
+	uint32_t offset_of_sgls, offset_of_flat_buffs = 0;
+	int i;
+	int num_im_sgls = qat_gen_config[
+		comp_dev->qat_dev->qat_dev_gen].comp_num_im_bufs_required;
+
+	QAT_LOG(DEBUG, "QAT COMP device %s needs %d sgls",
+				comp_dev->qat_dev->name, num_im_sgls);
+	snprintf(inter_buff_mz_name, RTE_MEMZONE_NAMESIZE,
+				"%s_inter_buff", comp_dev->qat_dev->name);
+	memzone = rte_memzone_lookup(inter_buff_mz_name);
+	if (memzone != NULL) {
+		QAT_LOG(DEBUG, "QAT COMP im buffer memzone created already");
+		return memzone;
+	}
+
+	/* Create a memzone to hold intermediate buffers and associated
+	 * meta-data needed by the firmware. The memzone contains:
+	 *  - a list of num_im_sgls physical pointers to sgls
+	 *  - the num_im_sgl sgl structures, each pointing to 2 flat buffers
+	 *  - the flat buffers: num_im_sgl * 2
+	 * where num_im_sgls depends on the hardware generation of the device
+	 */
+
+	size_of_ptr_array = num_im_sgls * sizeof(phys_addr_t);
+	offset_of_sgls = (size_of_ptr_array + (~QAT_64_BYTE_ALIGN_MASK))
+			& QAT_64_BYTE_ALIGN_MASK;
+	offset_of_flat_buffs =
+	    offset_of_sgls + num_im_sgls * sizeof(struct qat_inter_sgl);
+	full_size = offset_of_flat_buffs +
+			num_im_sgls * buff_size * QAT_NUM_BUFS_IN_IM_SGL;
+
+	memzone = rte_memzone_reserve_aligned(inter_buff_mz_name, full_size,
+			comp_dev->compressdev->data->socket_id,
+			RTE_MEMZONE_2MB, QAT_64_BYTE_ALIGN);
+	if (memzone == NULL) {
+		QAT_LOG(ERR, "Can't allocate intermediate buffers"
+				" for device %s", comp_dev->qat_dev->name);
+		return NULL;
+	}
+
+	mz_start = (uint8_t *)memzone->addr;
+	mz_start_phys = memzone->phys_addr;
+	QAT_LOG(DEBUG, "Memzone %s: addr = %p, phys = 0x%"PRIx64
+			", size required %d, size created %zu",
+			inter_buff_mz_name, mz_start, mz_start_phys,
+			full_size, memzone->len);
+
+	array_of_pointers = (struct array_of_ptrs *)mz_start;
+	for (i = 0; i < num_im_sgls; i++) {
+		uint32_t curr_sgl_offset =
+		    offset_of_sgls + i * sizeof(struct qat_inter_sgl);
+		struct qat_inter_sgl *sgl =
+		    (struct qat_inter_sgl *)(mz_start +	curr_sgl_offset);
+		array_of_pointers->pointer[i] = mz_start_phys + curr_sgl_offset;
+
+		sgl->num_bufs = QAT_NUM_BUFS_IN_IM_SGL;
+		sgl->num_mapped_bufs = 0;
+		sgl->resrvd = 0;
+		sgl->buffers[0].addr = mz_start_phys + offset_of_flat_buffs +
+			((i * QAT_NUM_BUFS_IN_IM_SGL) * buff_size);
+		sgl->buffers[0].len = buff_size;
+		sgl->buffers[0].resrvd = 0;
+		sgl->buffers[1].addr = mz_start_phys + offset_of_flat_buffs +
+			(((i * QAT_NUM_BUFS_IN_IM_SGL) + 1) * buff_size);
+		sgl->buffers[1].len = buff_size;
+		sgl->buffers[1].resrvd = 0;
+
+#if QAT_IM_BUFFER_DEBUG
+		QAT_LOG(DEBUG, "  : phys addr of sgl[%i] in array_of_pointers"
+			    "= 0x%"PRIx64, i, array_of_pointers->pointer[i]);
+		QAT_LOG(DEBUG, "  : virt address of sgl[%i] = %p", i, sgl);
+		QAT_LOG(DEBUG, "  : sgl->buffers[0].addr = 0x%"PRIx64", len=%d",
+			sgl->buffers[0].addr, sgl->buffers[0].len);
+		QAT_LOG(DEBUG, "  : sgl->buffers[1].addr = 0x%"PRIx64", len=%d",
+			sgl->buffers[1].addr, sgl->buffers[1].len);
+#endif
+		}
+#if QAT_IM_BUFFER_DEBUG
+	QAT_DP_HEXDUMP_LOG(DEBUG,  "IM buffer memzone start:",
+			mz_start, offset_of_flat_buffs + 32);
+#endif
+	return memzone;
+}
+
 static struct rte_mempool *
 qat_comp_create_xform_pool(struct qat_comp_dev_private *comp_dev,
 			      uint32_t num_elements)
@@ -176,6 +274,12 @@ qat_comp_create_xform_pool(struct qat_comp_dev_private *comp_dev,
 static void
 _qat_comp_dev_config_clear(struct qat_comp_dev_private *comp_dev)
 {
+	/* Free intermediate buffers */
+	if (comp_dev->interm_buff_mz) {
+		rte_memzone_free(comp_dev->interm_buff_mz);
+		comp_dev->interm_buff_mz = NULL;
+	}
+
 	/* Free private_xform pool */
 	if (comp_dev->xformpool) {
 		/* Free internal mempool for private xforms */
@@ -197,6 +301,21 @@ qat_comp_dev_config(struct rte_compressdev *dev,
 		return -EINVAL;
 	}
 
+	if (RTE_PMD_QAT_COMP_IM_BUFFER_SIZE == 0) {
+		QAT_LOG(WARNING,
+			"RTE_PMD_QAT_COMP_IM_BUFFER_SIZE = 0 in config file, so"
+			" QAT device can't be used for Dynamic Deflate. "
+			"Did you really intend to do this?");
+	} else {
+		comp_dev->interm_buff_mz =
+				qat_comp_setup_inter_buffers(comp_dev,
+					RTE_PMD_QAT_COMP_IM_BUFFER_SIZE);
+		if (comp_dev->interm_buff_mz == NULL) {
+			ret = -ENOMEM;
+			goto error_out;
+		}
+	}
+
 	comp_dev->xformpool = qat_comp_create_xform_pool(comp_dev,
 					config->max_nb_priv_xforms);
 	if (comp_dev->xformpool == NULL) {
@@ -348,6 +467,16 @@ static struct rte_compressdev_ops compress_qat_ops = {
 	.private_xform_free	= qat_comp_private_xform_free
 };
 
+/* An rte_driver is needed in the registration of the device with compressdev.
+ * The actual qat pci's rte_driver can't be used as its name represents
+ * the whole pci device with all services. Think of this as a holder for a name
+ * for the compression part of the pci device.
+ */
+static const char qat_comp_drv_name[] = RTE_STR(COMPRESSDEV_NAME_QAT_PMD);
+static const struct rte_driver compdev_qat_driver = {
+	.name = qat_comp_drv_name,
+	.alias = qat_comp_drv_name
+};
 int
 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
 {
@@ -355,6 +484,10 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
 		QAT_LOG(ERR, "Compression PMD not supported on QAT dh895xcc");
 		return 0;
 	}
+	if (qat_pci_dev->qat_dev_gen == QAT_GEN3) {
+		QAT_LOG(ERR, "Compression PMD not supported on QAT c4xxx");
+		return 0;
+	}
 
 	struct rte_compressdev_pmd_init_params init_params = {
 		.name = "",
@@ -368,8 +501,14 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
 			qat_pci_dev->name, "comp");
 	QAT_LOG(DEBUG, "Creating QAT COMP device %s", name);
 
+	/* Populate subset device to use in compressdev device creation */
+	qat_pci_dev->comp_rte_dev.driver = &compdev_qat_driver;
+	qat_pci_dev->comp_rte_dev.numa_node =
+					qat_pci_dev->pci_dev->device.numa_node;
+	qat_pci_dev->comp_rte_dev.devargs = NULL;
+
 	compressdev = rte_compressdev_pmd_create(name,
-			&qat_pci_dev->pci_dev->device,
+			&(qat_pci_dev->comp_rte_dev),
 			sizeof(struct qat_comp_dev_private),
 			&init_params);
 
@@ -391,6 +530,7 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
 	switch (qat_pci_dev->qat_dev_gen) {
 	case QAT_GEN1:
 	case QAT_GEN2:
+	case QAT_GEN3:
 		comp_dev->qat_dev_capabilities = qat_comp_gen_capabilities;
 		break;
 	default:
diff --git a/drivers/compress/qat/qat_comp_pmd.h b/drivers/compress/qat/qat_comp_pmd.h
index 9ad2a283..b8299d43 100644
--- a/drivers/compress/qat/qat_comp_pmd.h
+++ b/drivers/compress/qat/qat_comp_pmd.h
@@ -12,6 +12,9 @@
 
 #include "qat_device.h"
 
+/**< Intel(R) QAT Compression PMD driver name */
+#define COMPRESSDEV_NAME_QAT_PMD	compress_qat
+
 /** private data structure for a QAT compression device.
  * This QAT device is a device offering only a compression service,
  * there can be one of these on each qat_pci_device (VF).
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index c480cbd3..009f8443 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -7,6 +7,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += aesni_gcm
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += aesni_mb
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO) += armv8
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_CCP) += ccp
+DIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += octeontx
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_OPENSSL) += openssl
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += snow3g
@@ -14,12 +15,15 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += kasumi
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_ZUC) += zuc
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_MVSAM_CRYPTO) += mvsam
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += null
+ifeq ($(CONFIG_RTE_LIBRTE_SECURITY),y)
 ifeq ($(CONFIG_RTE_EAL_VFIO)$(CONFIG_RTE_LIBRTE_FSLMC_BUS),yy)
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += dpaa2_sec
-endif
+endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 ifeq ($(CONFIG_RTE_LIBRTE_DPAA_BUS),y)
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA_SEC) += dpaa_sec
-endif
+DIRS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr
+endif # CONFIG_RTE_LIBRTE_PMD_DPAA_SEC
+endif # CONFIG_RTE_LIBRTE_SECURITY
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_VIRTIO_CRYPTO) += virtio
 
 include $(RTE_SDK)/mk/rte.subdir.mk
diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c
index 752e0cd6..ebdf7c35 100644
--- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c
+++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c
@@ -23,7 +23,6 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops,
 {
 	const struct rte_crypto_sym_xform *auth_xform;
 	const struct rte_crypto_sym_xform *aead_xform;
-	uint16_t digest_length;
 	uint8_t key_length;
 	uint8_t *key;
 
@@ -47,7 +46,7 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops,
 
 		key_length = auth_xform->auth.key.length;
 		key = auth_xform->auth.key.data;
-		digest_length = auth_xform->auth.digest_length;
+		sess->req_digest_length = auth_xform->auth.digest_length;
 
 	/* AES-GCM */
 	} else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
@@ -73,7 +72,7 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops,
 		key = aead_xform->aead.key.data;
 
 		sess->aad_length = aead_xform->aead.aad_length;
-		digest_length = aead_xform->aead.digest_length;
+		sess->req_digest_length = aead_xform->aead.digest_length;
 	} else {
 		AESNI_GCM_LOG(ERR, "Wrong xform type, has to be AEAD or authentication");
 		return -ENOTSUP;
@@ -106,13 +105,28 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops,
 	gcm_ops[sess->key].precomp(key, &sess->gdata_key);
 
 	/* Digest check */
-	if (digest_length != 16 &&
-			digest_length != 12 &&
-			digest_length != 8) {
+	if (sess->req_digest_length > 16) {
 		AESNI_GCM_LOG(ERR, "Invalid digest length");
 		return -EINVAL;
 	}
-	sess->digest_length = digest_length;
+	/*
+	 * Multi-buffer lib supports digest sizes from 4 to 16 bytes
+	 * in version 0.50 and sizes of 8, 12 and 16 bytes,
+	 * in version 0.49.
+	 * If size requested is different, generate the full digest
+	 * (16 bytes) in a temporary location and then memcpy
+	 * the requested number of bytes.
+	 */
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+	if (sess->req_digest_length < 4)
+#else
+	if (sess->req_digest_length != 16 &&
+			sess->req_digest_length != 12 &&
+			sess->req_digest_length != 8)
+#endif
+		sess->gen_digest_length = 16;
+	else
+		sess->gen_digest_length = sess->req_digest_length;
 
 	return 0;
 }
@@ -180,6 +194,7 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op,
 	struct rte_mbuf *m_src = sym_op->m_src;
 	uint32_t offset, data_offset, data_length;
 	uint32_t part_len, total_len, data_len;
+	uint8_t *tag;
 
 	if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION ||
 			session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) {
@@ -225,17 +240,8 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op,
 
 	iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 				session->iv.offset);
-	/*
-	 * GCM working in 12B IV mode => 16B pre-counter block we need
-	 * to set BE LSB to 1, driver expects that 16B is allocated
-	 */
-	if (session->iv.length == 12) {
-		uint32_t *iv_padd = (uint32_t *)&(iv_ptr[12]);
-		*iv_padd = rte_bswap32(1);
-	}
 
 	if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION) {
-
 		qp->ops[session->key].init(&session->gdata_key,
 				&qp->gdata_ctx,
 				iv_ptr,
@@ -263,13 +269,16 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op,
 			total_len -= part_len;
 		}
 
+		if (session->req_digest_length != session->gen_digest_length)
+			tag = qp->temp_digest;
+		else
+			tag = sym_op->aead.digest.data;
+
 		qp->ops[session->key].finalize(&session->gdata_key,
 				&qp->gdata_ctx,
-				sym_op->aead.digest.data,
-				(uint64_t)session->digest_length);
+				tag,
+				session->gen_digest_length);
 	} else if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) {
-		uint8_t *auth_tag = qp->temp_digest;
-
 		qp->ops[session->key].init(&session->gdata_key,
 				&qp->gdata_ctx,
 				iv_ptr,
@@ -298,33 +307,41 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op,
 			total_len -= part_len;
 		}
 
+		tag = qp->temp_digest;
 		qp->ops[session->key].finalize(&session->gdata_key,
 				&qp->gdata_ctx,
-				auth_tag,
-				(uint64_t)session->digest_length);
+				tag,
+				session->gen_digest_length);
 	} else if (session->op == AESNI_GMAC_OP_GENERATE) {
 		qp->ops[session->key].init(&session->gdata_key,
 				&qp->gdata_ctx,
 				iv_ptr,
 				src,
 				(uint64_t)data_length);
+		if (session->req_digest_length != session->gen_digest_length)
+			tag = qp->temp_digest;
+		else
+			tag = sym_op->auth.digest.data;
 		qp->ops[session->key].finalize(&session->gdata_key,
 				&qp->gdata_ctx,
-				sym_op->auth.digest.data,
-				(uint64_t)session->digest_length);
+				tag,
+				session->gen_digest_length);
 	} else { /* AESNI_GMAC_OP_VERIFY */
-		uint8_t *auth_tag = qp->temp_digest;
-
 		qp->ops[session->key].init(&session->gdata_key,
 				&qp->gdata_ctx,
 				iv_ptr,
 				src,
 				(uint64_t)data_length);
 
+		/*
+		 * Generate always 16 bytes and later compare only
+		 * the bytes passed.
+		 */
+		tag = qp->temp_digest;
 		qp->ops[session->key].finalize(&session->gdata_key,
 				&qp->gdata_ctx,
-				auth_tag,
-				(uint64_t)session->digest_length);
+				tag,
+				session->gen_digest_length);
 	}
 
 	return 0;
@@ -361,13 +378,22 @@ post_process_gcm_crypto_op(struct aesni_gcm_qp *qp,
 
 #ifdef RTE_LIBRTE_PMD_AESNI_GCM_DEBUG
 		rte_hexdump(stdout, "auth tag (orig):",
-				digest, session->digest_length);
+				digest, session->req_digest_length);
 		rte_hexdump(stdout, "auth tag (calc):",
-				tag, session->digest_length);
+				tag, session->req_digest_length);
 #endif
 
-		if (memcmp(tag, digest,	session->digest_length) != 0)
+		if (memcmp(tag, digest,	session->req_digest_length) != 0)
 			op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+	} else {
+		if (session->req_digest_length != session->gen_digest_length) {
+			if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION)
+				memcpy(op->sym->aead.digest.data, qp->temp_digest,
+						session->req_digest_length);
+			else
+				memcpy(op->sym->auth.digest.data, qp->temp_digest,
+						session->req_digest_length);
+		}
 	}
 }
 
diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c
index b6b4dd02..c343a393 100644
--- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c
+++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c
@@ -24,9 +24,9 @@ static const struct rte_cryptodev_capabilities aesni_gcm_pmd_capabilities[] = {
 					.increment = 8
 				},
 				.digest_size = {
-					.min = 8,
+					.min = 1,
 					.max = 16,
-					.increment = 4
+					.increment = 1
 				},
 				.iv_size = {
 					.min = 12,
@@ -49,9 +49,9 @@ static const struct rte_cryptodev_capabilities aesni_gcm_pmd_capabilities[] = {
 					.increment = 8
 				},
 				.digest_size = {
-					.min = 8,
+					.min = 1,
 					.max = 16,
-					.increment = 4
+					.increment = 1
 				},
 				.aad_size = {
 					.min = 0,
diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h
index c13a12a5..92b04135 100644
--- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h
+++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h
@@ -76,8 +76,10 @@ struct aesni_gcm_session {
 	/**< IV parameters */
 	uint16_t aad_length;
 	/**< AAD length */
-	uint16_t digest_length;
-	/**< Digest length */
+	uint16_t req_digest_length;
+	/**< Requested digest length */
+	uint16_t gen_digest_length;
+	/**< Generated digest length */
 	enum aesni_gcm_operation op;
 	/**< GCM operation type */
 	enum aesni_gcm_key key;
diff --git a/drivers/crypto/aesni_gcm/meson.build b/drivers/crypto/aesni_gcm/meson.build
new file mode 100644
index 00000000..a02da1ef
--- /dev/null
+++ b/drivers/crypto/aesni_gcm/meson.build
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+lib = cc.find_library('IPSec_MB', required: false)
+if not lib.found()
+	build = false
+else
+	ext_deps += lib
+endif
+
+sources = files('aesni_gcm_pmd.c', 'aesni_gcm_pmd_ops.c')
+deps += ['bus_vdev']
diff --git a/drivers/crypto/aesni_mb/aesni_mb_ops.h b/drivers/crypto/aesni_mb/aesni_mb_ops.h
index 5a1cba6c..575d6a5b 100644
--- a/drivers/crypto/aesni_mb/aesni_mb_ops.h
+++ b/drivers/crypto/aesni_mb/aesni_mb_ops.h
@@ -11,6 +11,15 @@
 
 #include <intel-ipsec-mb.h>
 
+/*
+ * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50,
+ * so if macro is not defined, it means that the version is 0.49.
+ */
+#if !defined(IMB_VERSION_NUM)
+#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
+#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0)
+#endif
+
 enum aesni_mb_vector_mode {
 	RTE_AESNI_MB_NOT_SUPPORTED = 0,
 	RTE_AESNI_MB_SSE,
@@ -39,6 +48,8 @@ typedef void (*aes_cmac_sub_key_gen_t)
 		(const void *exp_key, void *k2, void *k3);
 typedef void (*aes_cmac_keyexp_t)
 		(const void *key, void *keyexp);
+typedef void (*aes_gcm_keyexp_t)
+		(const void *key, struct gcm_key_data *keyexp);
 
 /** Multi-buffer library function pointer table */
 struct aesni_mb_op_fns {
@@ -86,8 +97,24 @@ struct aesni_mb_op_fns {
 			/**< AES CMAC subkey expansions */
 			aes_cmac_keyexp_t aes_cmac_expkey;
 			/**< AES CMAC key expansions */
+			aes_gcm_keyexp_t aes_gcm_128;
+			/**< AES GCM 128 key expansions */
+			aes_gcm_keyexp_t aes_gcm_192;
+			/**< AES GCM 192 key expansions */
+			aes_gcm_keyexp_t aes_gcm_256;
+			/**< AES GCM 256 key expansions */
 		} keyexp;
 		/**< Key expansion functions */
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		struct {
+			hash_fn_t sha1;
+			hash_fn_t sha224;
+			hash_fn_t sha256;
+			hash_fn_t sha384;
+			hash_fn_t sha512;
+		} multi_block;
+		/** multi block hash functions */
+#endif
 	} aux;
 	/**< Auxiliary functions */
 };
@@ -104,7 +131,13 @@ static const struct aesni_mb_op_fns job_ops[] = {
 				},
 				.keyexp = {
 					NULL
+				},
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+				.multi_block = {
+					NULL
 				}
+#endif
+
 			}
 		},
 		[RTE_AESNI_MB_SSE] = {
@@ -130,8 +163,20 @@ static const struct aesni_mb_op_fns job_ops[] = {
 					aes_keyexp_256_sse,
 					aes_xcbc_expand_key_sse,
 					aes_cmac_subkey_gen_sse,
-					aes_keyexp_128_enc_sse
+					aes_keyexp_128_enc_sse,
+					aes_gcm_pre_128_sse,
+					aes_gcm_pre_192_sse,
+					aes_gcm_pre_256_sse
+				},
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+				.multi_block = {
+					sha1_sse,
+					sha224_sse,
+					sha256_sse,
+					sha384_sse,
+					sha512_sse
 				}
+#endif
 			}
 		},
 		[RTE_AESNI_MB_AVX] = {
@@ -157,8 +202,20 @@ static const struct aesni_mb_op_fns job_ops[] = {
 					aes_keyexp_256_avx,
 					aes_xcbc_expand_key_avx,
 					aes_cmac_subkey_gen_avx,
-					aes_keyexp_128_enc_avx
+					aes_keyexp_128_enc_avx,
+					aes_gcm_pre_128_avx_gen2,
+					aes_gcm_pre_192_avx_gen2,
+					aes_gcm_pre_256_avx_gen2
+				},
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+				.multi_block = {
+					sha1_avx,
+					sha224_avx,
+					sha256_avx,
+					sha384_avx,
+					sha512_avx
 				}
+#endif
 			}
 		},
 		[RTE_AESNI_MB_AVX2] = {
@@ -184,8 +241,20 @@ static const struct aesni_mb_op_fns job_ops[] = {
 					aes_keyexp_256_avx2,
 					aes_xcbc_expand_key_avx2,
 					aes_cmac_subkey_gen_avx2,
-					aes_keyexp_128_enc_avx2
+					aes_keyexp_128_enc_avx2,
+					aes_gcm_pre_128_avx_gen4,
+					aes_gcm_pre_192_avx_gen4,
+					aes_gcm_pre_256_avx_gen4
+				},
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+				.multi_block = {
+					sha1_avx2,
+					sha224_avx2,
+					sha256_avx2,
+					sha384_avx2,
+					sha512_avx2
 				}
+#endif
 			}
 		},
 		[RTE_AESNI_MB_AVX512] = {
@@ -211,8 +280,20 @@ static const struct aesni_mb_op_fns job_ops[] = {
 					aes_keyexp_256_avx512,
 					aes_xcbc_expand_key_avx512,
 					aes_cmac_subkey_gen_avx512,
-					aes_keyexp_128_enc_avx512
+					aes_keyexp_128_enc_avx512,
+					aes_gcm_pre_128_avx_gen4,
+					aes_gcm_pre_192_avx_gen4,
+					aes_gcm_pre_256_avx_gen4
+				},
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+				.multi_block = {
+					sha1_avx512,
+					sha224_avx512,
+					sha256_avx512,
+					sha384_avx512,
+					sha512_avx512
 				}
+#endif
 			}
 		}
 };
diff --git a/drivers/crypto/aesni_mb/meson.build b/drivers/crypto/aesni_mb/meson.build
new file mode 100644
index 00000000..aae0995e
--- /dev/null
+++ b/drivers/crypto/aesni_mb/meson.build
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+lib = cc.find_library('IPSec_MB', required: false)
+if not lib.found()
+	build = false
+else
+	ext_deps += lib
+endif
+
+sources = files('rte_aesni_mb_pmd.c', 'rte_aesni_mb_pmd_ops.c')
+deps += ['bus_vdev']
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
index 93dc7a44..83250e32 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
@@ -14,6 +14,9 @@
 
 #include "rte_aesni_mb_pmd_private.h"
 
+#define AES_CCM_DIGEST_MIN_LEN 4
+#define AES_CCM_DIGEST_MAX_LEN 16
+#define HMAC_MAX_BLOCK_SIZE 128
 static uint8_t cryptodev_driver_id;
 
 typedef void (*hash_one_block_t)(const void *data, void *digest);
@@ -83,7 +86,8 @@ aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
 	}
 
 	if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
-		if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
+		if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||
+				xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
 			if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 				return AESNI_MB_OP_AEAD_CIPHER_HASH;
 			else
@@ -101,6 +105,8 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 		const struct rte_crypto_sym_xform *xform)
 {
 	hash_one_block_t hash_oneblock_fn;
+	unsigned int key_larger_block_size = 0;
+	uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };
 
 	if (xform == NULL) {
 		sess->auth.algo = NULL_HASH;
@@ -112,12 +118,23 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 		return -1;
 	}
 
+	/* Set the request digest size */
+	sess->auth.req_digest_len = xform->auth.digest_length;
+
 	/* Select auth generate/verify */
 	sess->auth.operation = xform->auth.op;
 
 	/* Set Authentication Parameters */
 	if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
 		sess->auth.algo = AES_XCBC;
+
+		uint16_t xcbc_mac_digest_len =
+			get_truncated_digest_byte_length(AES_XCBC);
+		if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
+			AESNI_MB_LOG(ERR, "Invalid digest size\n");
+			return -EINVAL;
+		}
+		sess->auth.gen_digest_len = sess->auth.req_digest_len;
 		(*mb_ops->aux.keyexp.aes_xcbc)(xform->auth.key.data,
 				sess->auth.xcbc.k1_expanded,
 				sess->auth.xcbc.k2, sess->auth.xcbc.k3);
@@ -126,6 +143,32 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 
 	if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
 		sess->auth.algo = AES_CMAC;
+
+		uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
+
+		if (sess->auth.req_digest_len > cmac_digest_len) {
+			AESNI_MB_LOG(ERR, "Invalid digest size\n");
+			return -EINVAL;
+		}
+		/*
+		 * Multi-buffer lib supports digest sizes from 4 to 16 bytes
+		 * in version 0.50 and sizes of 12 and 16 bytes,
+		 * in version 0.49.
+		 * If size requested is different, generate the full digest
+		 * (16 bytes) in a temporary location and then memcpy
+		 * the requested number of bytes.
+		 */
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		if (sess->auth.req_digest_len < 4)
+#else
+		uint16_t cmac_trunc_digest_len =
+				get_truncated_digest_byte_length(AES_CMAC);
+		if (sess->auth.req_digest_len != cmac_digest_len &&
+				sess->auth.req_digest_len != cmac_trunc_digest_len)
+#endif
+			sess->auth.gen_digest_len = cmac_digest_len;
+		else
+			sess->auth.gen_digest_len = sess->auth.req_digest_len;
 		(*mb_ops->aux.keyexp.aes_cmac_expkey)(xform->auth.key.data,
 				sess->auth.cmac.expkey);
 
@@ -134,7 +177,6 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 		return 0;
 	}
 
-
 	switch (xform->auth.algo) {
 	case RTE_CRYPTO_AUTH_MD5_HMAC:
 		sess->auth.algo = MD5;
@@ -143,34 +185,107 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 	case RTE_CRYPTO_AUTH_SHA1_HMAC:
 		sess->auth.algo = SHA1;
 		hash_oneblock_fn = mb_ops->aux.one_block.sha1;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
+			mb_ops->aux.multi_block.sha1(
+				xform->auth.key.data,
+				xform->auth.key.length,
+				hashed_key);
+			key_larger_block_size = 1;
+		}
+#endif
 		break;
 	case RTE_CRYPTO_AUTH_SHA224_HMAC:
 		sess->auth.algo = SHA_224;
 		hash_oneblock_fn = mb_ops->aux.one_block.sha224;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
+			mb_ops->aux.multi_block.sha224(
+				xform->auth.key.data,
+				xform->auth.key.length,
+				hashed_key);
+			key_larger_block_size = 1;
+		}
+#endif
 		break;
 	case RTE_CRYPTO_AUTH_SHA256_HMAC:
 		sess->auth.algo = SHA_256;
 		hash_oneblock_fn = mb_ops->aux.one_block.sha256;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
+			mb_ops->aux.multi_block.sha256(
+				xform->auth.key.data,
+				xform->auth.key.length,
+				hashed_key);
+			key_larger_block_size = 1;
+		}
+#endif
 		break;
 	case RTE_CRYPTO_AUTH_SHA384_HMAC:
 		sess->auth.algo = SHA_384;
 		hash_oneblock_fn = mb_ops->aux.one_block.sha384;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
+			mb_ops->aux.multi_block.sha384(
+				xform->auth.key.data,
+				xform->auth.key.length,
+				hashed_key);
+			key_larger_block_size = 1;
+		}
+#endif
 		break;
 	case RTE_CRYPTO_AUTH_SHA512_HMAC:
 		sess->auth.algo = SHA_512;
 		hash_oneblock_fn = mb_ops->aux.one_block.sha512;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
+			mb_ops->aux.multi_block.sha512(
+				xform->auth.key.data,
+				xform->auth.key.length,
+				hashed_key);
+			key_larger_block_size = 1;
+		}
+#endif
 		break;
 	default:
 		AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
 		return -ENOTSUP;
 	}
+	uint16_t trunc_digest_size =
+			get_truncated_digest_byte_length(sess->auth.algo);
+	uint16_t full_digest_size =
+			get_digest_byte_length(sess->auth.algo);
+
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+	if (sess->auth.req_digest_len > full_digest_size ||
+			sess->auth.req_digest_len == 0) {
+#else
+	if (sess->auth.req_digest_len != trunc_digest_size) {
+#endif
+		AESNI_MB_LOG(ERR, "Invalid digest size\n");
+		return -EINVAL;
+	}
+
+	if (sess->auth.req_digest_len != trunc_digest_size &&
+			sess->auth.req_digest_len != full_digest_size)
+		sess->auth.gen_digest_len = full_digest_size;
+	else
+		sess->auth.gen_digest_len = sess->auth.req_digest_len;
 
 	/* Calculate Authentication precomputes */
-	calculate_auth_precomputes(hash_oneblock_fn,
+	if (key_larger_block_size) {
+		calculate_auth_precomputes(hash_oneblock_fn,
+			sess->auth.pads.inner, sess->auth.pads.outer,
+			hashed_key,
+			xform->auth.key.length,
+			get_auth_algo_blocksize(sess->auth.algo));
+	} else {
+		calculate_auth_precomputes(hash_oneblock_fn,
 			sess->auth.pads.inner, sess->auth.pads.outer,
 			xform->auth.key.data,
 			xform->auth.key.length,
 			get_auth_algo_blocksize(sess->auth.algo));
+	}
 
 	return 0;
 }
@@ -330,7 +445,10 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
 		struct aesni_mb_session *sess,
 		const struct rte_crypto_sym_xform *xform)
 {
-	aes_keyexp_t aes_keyexp_fn;
+	union {
+		aes_keyexp_t aes_keyexp_fn;
+		aes_gcm_keyexp_t aes_gcm_keyexp_fn;
+	} keyexp;
 
 	switch (xform->aead.op) {
 	case RTE_CRYPTO_AEAD_OP_ENCRYPT:
@@ -350,7 +468,53 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
 	case RTE_CRYPTO_AEAD_AES_CCM:
 		sess->cipher.mode = CCM;
 		sess->auth.algo = AES_CCM;
+
+		/* Check key length and choose key expansion function for AES */
+		switch (xform->aead.key.length) {
+		case AES_128_BYTES:
+			sess->cipher.key_length_in_bytes = AES_128_BYTES;
+			keyexp.aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
+			break;
+		default:
+			AESNI_MB_LOG(ERR, "Invalid cipher key length");
+			return -EINVAL;
+		}
+
+		/* Expanded cipher keys */
+		(*keyexp.aes_keyexp_fn)(xform->aead.key.data,
+				sess->cipher.expanded_aes_keys.encode,
+				sess->cipher.expanded_aes_keys.decode);
+		break;
+
+	case RTE_CRYPTO_AEAD_AES_GCM:
+		sess->cipher.mode = GCM;
+		sess->auth.algo = AES_GMAC;
+
+		switch (xform->aead.key.length) {
+		case AES_128_BYTES:
+			sess->cipher.key_length_in_bytes = AES_128_BYTES;
+			keyexp.aes_gcm_keyexp_fn =
+					mb_ops->aux.keyexp.aes_gcm_128;
+			break;
+		case AES_192_BYTES:
+			sess->cipher.key_length_in_bytes = AES_192_BYTES;
+			keyexp.aes_gcm_keyexp_fn =
+					mb_ops->aux.keyexp.aes_gcm_192;
+			break;
+		case AES_256_BYTES:
+			sess->cipher.key_length_in_bytes = AES_256_BYTES;
+			keyexp.aes_gcm_keyexp_fn =
+					mb_ops->aux.keyexp.aes_gcm_256;
+			break;
+		default:
+			AESNI_MB_LOG(ERR, "Invalid cipher key length");
+			return -EINVAL;
+		}
+
+		(keyexp.aes_gcm_keyexp_fn)(xform->aead.key.data,
+				&sess->cipher.gcm_key);
 		break;
+
 	default:
 		AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
 		return -ENOTSUP;
@@ -360,22 +524,15 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
 	sess->iv.offset = xform->aead.iv.offset;
 	sess->iv.length = xform->aead.iv.length;
 
-	/* Check key length and choose key expansion function for AES */
-
-	switch (xform->aead.key.length) {
-	case AES_128_BYTES:
-		sess->cipher.key_length_in_bytes = AES_128_BYTES;
-		aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
-		break;
-	default:
-		AESNI_MB_LOG(ERR, "Invalid cipher key length");
+	sess->auth.req_digest_len = xform->aead.digest_length;
+	/* CCM digests must be between 4 and 16 and an even number */
+	if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
+			sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
+			(sess->auth.req_digest_len & 1) == 1) {
+		AESNI_MB_LOG(ERR, "Invalid digest size\n");
 		return -EINVAL;
 	}
-
-	/* Expanded cipher keys */
-	(*aes_keyexp_fn)(xform->aead.key.data,
-			sess->cipher.expanded_aes_keys.encode,
-			sess->cipher.expanded_aes_keys.decode);
+	sess->auth.gen_digest_len = sess->auth.req_digest_len;
 
 	return 0;
 }
@@ -397,19 +554,16 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
 		sess->chain_order = HASH_CIPHER;
 		auth_xform = xform;
 		cipher_xform = xform->next;
-		sess->auth.digest_len = xform->auth.digest_length;
 		break;
 	case AESNI_MB_OP_CIPHER_HASH:
 		sess->chain_order = CIPHER_HASH;
 		auth_xform = xform->next;
 		cipher_xform = xform;
-		sess->auth.digest_len = xform->auth.digest_length;
 		break;
 	case AESNI_MB_OP_HASH_ONLY:
 		sess->chain_order = HASH_CIPHER;
 		auth_xform = xform;
 		cipher_xform = NULL;
-		sess->auth.digest_len = xform->auth.digest_length;
 		break;
 	case AESNI_MB_OP_CIPHER_ONLY:
 		/*
@@ -428,13 +582,11 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
 	case AESNI_MB_OP_AEAD_CIPHER_HASH:
 		sess->chain_order = CIPHER_HASH;
 		sess->aead.aad_len = xform->aead.aad_length;
-		sess->auth.digest_len = xform->aead.digest_length;
 		aead_xform = xform;
 		break;
 	case AESNI_MB_OP_AEAD_HASH_CIPHER:
 		sess->chain_order = HASH_CIPHER;
 		sess->aead.aad_len = xform->aead.aad_length;
-		sess->auth.digest_len = xform->aead.digest_length;
 		aead_xform = xform;
 		break;
 	case AESNI_MB_OP_NOT_SUPPORTED:
@@ -573,38 +725,62 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
 
 	job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
 
-	if (job->cipher_mode == DES3) {
-		job->aes_enc_key_expanded =
-			session->cipher.exp_3des_keys.ks_ptr;
-		job->aes_dec_key_expanded =
-			session->cipher.exp_3des_keys.ks_ptr;
-	} else {
-		job->aes_enc_key_expanded =
-			session->cipher.expanded_aes_keys.encode;
-		job->aes_dec_key_expanded =
-			session->cipher.expanded_aes_keys.decode;
-	}
-
-
-
-
 	/* Set authentication parameters */
 	job->hash_alg = session->auth.algo;
-	if (job->hash_alg == AES_XCBC) {
+
+	switch (job->hash_alg) {
+	case AES_XCBC:
 		job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
 		job->u.XCBC._k2 = session->auth.xcbc.k2;
 		job->u.XCBC._k3 = session->auth.xcbc.k3;
-	} else if (job->hash_alg == AES_CCM) {
+
+		job->aes_enc_key_expanded =
+				session->cipher.expanded_aes_keys.encode;
+		job->aes_dec_key_expanded =
+				session->cipher.expanded_aes_keys.decode;
+		break;
+
+	case AES_CCM:
 		job->u.CCM.aad = op->sym->aead.aad.data + 18;
 		job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
-	} else if (job->hash_alg == AES_CMAC) {
+		job->aes_enc_key_expanded =
+				session->cipher.expanded_aes_keys.encode;
+		job->aes_dec_key_expanded =
+				session->cipher.expanded_aes_keys.decode;
+		break;
+
+	case AES_CMAC:
 		job->u.CMAC._key_expanded = session->auth.cmac.expkey;
 		job->u.CMAC._skey1 = session->auth.cmac.skey1;
 		job->u.CMAC._skey2 = session->auth.cmac.skey2;
+		job->aes_enc_key_expanded =
+				session->cipher.expanded_aes_keys.encode;
+		job->aes_dec_key_expanded =
+				session->cipher.expanded_aes_keys.decode;
+		break;
 
-	} else {
+	case AES_GMAC:
+		job->u.GCM.aad = op->sym->aead.aad.data;
+		job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
+		job->aes_enc_key_expanded = &session->cipher.gcm_key;
+		job->aes_dec_key_expanded = &session->cipher.gcm_key;
+		break;
+
+	default:
 		job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
 		job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
+
+		if (job->cipher_mode == DES3) {
+			job->aes_enc_key_expanded =
+				session->cipher.exp_3des_keys.ks_ptr;
+			job->aes_dec_key_expanded =
+				session->cipher.exp_3des_keys.ks_ptr;
+		} else {
+			job->aes_enc_key_expanded =
+				session->cipher.expanded_aes_keys.encode;
+			job->aes_dec_key_expanded =
+				session->cipher.expanded_aes_keys.decode;
+		}
 	}
 
 	/* Mutable crypto operation parameters */
@@ -625,7 +801,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
 				rte_pktmbuf_data_len(op->sym->m_src));
 	} else {
 		m_dst = m_src;
-		if (job->hash_alg == AES_CCM)
+		if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC)
 			m_offset = op->sym->aead.data.offset;
 		else
 			m_offset = op->sym->cipher.data.offset;
@@ -637,32 +813,33 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
 		job->auth_tag_output = qp->temp_digests[*digest_idx];
 		*digest_idx = (*digest_idx + 1) % MAX_JOBS;
 	} else {
-		if (job->hash_alg == AES_CCM)
+		if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC)
 			job->auth_tag_output = op->sym->aead.digest.data;
 		else
 			job->auth_tag_output = op->sym->auth.digest.data;
-	}
 
+		if (session->auth.req_digest_len != session->auth.gen_digest_len) {
+			job->auth_tag_output = qp->temp_digests[*digest_idx];
+			*digest_idx = (*digest_idx + 1) % MAX_JOBS;
+		}
+	}
 	/*
 	 * Multi-buffer library current only support returning a truncated
 	 * digest length as specified in the relevant IPsec RFCs
 	 */
-	if (job->hash_alg != AES_CCM && job->hash_alg != AES_CMAC)
-		job->auth_tag_output_len_in_bytes =
-				get_truncated_digest_byte_length(job->hash_alg);
-	else
-		job->auth_tag_output_len_in_bytes = session->auth.digest_len;
 
+	/* Set digest length */
+	job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
 
 	/* Set IV parameters */
-
 	job->iv_len_in_bytes = session->iv.length;
 
 	/* Data  Parameter */
 	job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
 	job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
 
-	if (job->hash_alg == AES_CCM) {
+	switch (job->hash_alg) {
+	case AES_CCM:
 		job->cipher_start_src_offset_in_bytes =
 				op->sym->aead.data.offset;
 		job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
@@ -671,7 +848,19 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
 
 		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 			session->iv.offset + 1);
-	} else {
+		break;
+
+	case AES_GMAC:
+		job->cipher_start_src_offset_in_bytes =
+				op->sym->aead.data.offset;
+		job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
+		job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
+		job->msg_len_to_hash_in_bytes = job->msg_len_to_cipher_in_bytes;
+		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+				session->iv.offset);
+		break;
+
+	default:
 		job->cipher_start_src_offset_in_bytes =
 				op->sym->cipher.data.offset;
 		job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
@@ -690,20 +879,37 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
 }
 
 static inline void
-verify_digest(struct aesni_mb_qp *qp __rte_unused, JOB_AES_HMAC *job,
-		struct rte_crypto_op *op) {
+verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
+		struct aesni_mb_session *sess)
+{
 	/* Verify digest if required */
-	if (job->hash_alg == AES_CCM) {
+	if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) {
 		if (memcmp(job->auth_tag_output, op->sym->aead.digest.data,
-				job->auth_tag_output_len_in_bytes) != 0)
+				sess->auth.req_digest_len) != 0)
 			op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 	} else {
 		if (memcmp(job->auth_tag_output, op->sym->auth.digest.data,
-				job->auth_tag_output_len_in_bytes) != 0)
+				sess->auth.req_digest_len) != 0)
 			op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 	}
 }
 
+static inline void
+generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
+		struct aesni_mb_session *sess)
+{
+	/* No extra copy neeed */
+	if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
+		return;
+
+	/*
+	 * This can only happen for HMAC, so only digest
+	 * for authentication algos is required
+	 */
+	memcpy(op->sym->auth.digest.data, job->auth_tag_output,
+			sess->auth.req_digest_len);
+}
+
 /**
  * Process a completed job and return rte_mbuf which job processed
  *
@@ -730,7 +936,9 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
 			if (job->hash_alg != NULL_HASH) {
 				if (sess->auth.operation ==
 						RTE_CRYPTO_AUTH_OP_VERIFY)
-					verify_digest(qp, job, op);
+					verify_digest(job, op, sess);
+				else
+					generate_digest(job, op, sess);
 			}
 			break;
 		default:
@@ -833,22 +1041,30 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 
 	uint8_t digest_idx = qp->digest_idx;
 	do {
-		/* Get next operation to process from ingress queue */
-		retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
-		if (retval < 0)
-			break;
-
 		/* Get next free mb job struct from mb manager */
 		job = (*qp->op_fns->job.get_next)(qp->mb_mgr);
 		if (unlikely(job == NULL)) {
 			/* if no free mb job structs we need to flush mb_mgr */
 			processed_jobs += flush_mb_mgr(qp,
 					&ops[processed_jobs],
-					(nb_ops - processed_jobs) - 1);
+					nb_ops - processed_jobs);
+
+			if (nb_ops == processed_jobs)
+				break;
 
 			job = (*qp->op_fns->job.get_next)(qp->mb_mgr);
 		}
 
+		/*
+		 * Get next operation to process from ingress queue.
+		 * There is no need to return the job to the MB_MGR
+		 * if there are no more operations to process, since the MB_MGR
+		 * can use that pointer again in next get_next calls.
+		 */
+		retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
+		if (retval < 0)
+			break;
+
 		retval = set_mb_job_params(job, qp, op, &digest_idx);
 		if (unlikely(retval != 0)) {
 			qp->stats.dequeue_err_count++;
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
index ab26e5ae..43f6c26e 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
@@ -25,9 +25,15 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
 					.increment = 1
 				},
 				.digest_size = {
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+					.min = 1,
+					.max = 16,
+					.increment = 1
+#else
 					.min = 12,
 					.max = 12,
 					.increment = 0
+#endif
 				},
 				.iv_size = { 0 }
 			}, }
@@ -42,13 +48,23 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
 				.block_size = 64,
 				.key_size = {
 					.min = 1,
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+					.max = 65535,
+#else
 					.max = 64,
+#endif
 					.increment = 1
 				},
 				.digest_size = {
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+					.min = 1,
+					.max = 20,
+					.increment = 1
+#else
 					.min = 12,
 					.max = 12,
 					.increment = 0
+#endif
 				},
 				.iv_size = { 0 }
 			}, }
@@ -63,13 +79,23 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
 				.block_size = 64,
 				.key_size = {
 					.min = 1,
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+					.max = 65535,
+#else
 					.max = 64,
+#endif
 					.increment = 1
 				},
 				.digest_size = {
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+					.min = 1,
+					.max = 28,
+					.increment = 1
+#else
 					.min = 14,
 					.max = 14,
 					.increment = 0
+#endif
 				},
 				.iv_size = { 0 }
 			}, }
@@ -84,13 +110,23 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
 				.block_size = 64,
 				.key_size = {
 					.min = 1,
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+					.max = 65535,
+#else
 					.max = 64,
+#endif
 					.increment = 1
 				},
 				.digest_size = {
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+					.min = 1,
+					.max = 32,
+					.increment = 1
+#else
 					.min = 16,
 					.max = 16,
 					.increment = 0
+#endif
 				},
 				.iv_size = { 0 }
 			}, }
@@ -105,13 +141,23 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
 				.block_size = 128,
 				.key_size = {
 					.min = 1,
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+					.max = 65535,
+#else
 					.max = 128,
+#endif
 					.increment = 1
 				},
 				.digest_size = {
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+					.min = 1,
+					.max = 48,
+					.increment = 1
+#else
 					.min = 24,
 					.max = 24,
 					.increment = 0
+#endif
 				},
 				.iv_size = { 0 }
 			}, }
@@ -126,13 +172,23 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
 				.block_size = 128,
 				.key_size = {
 					.min = 1,
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+					.max = 65535,
+#else
 					.max = 128,
+#endif
 					.increment = 1
 				},
 				.digest_size = {
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+					.min = 1,
+					.max = 64,
+					.increment = 1
+#else
 					.min = 32,
 					.max = 32,
 					.increment = 0
+#endif
 				},
 				.iv_size = { 0 }
 			}, }
@@ -322,14 +378,44 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
 					.increment = 0
 				},
 				.digest_size = {
-					.min = 12,
+					.min = 1,
 					.max = 16,
-					.increment = 4
+					.increment = 1
 				},
 				.iv_size = { 0 }
 			}, }
 		}, }
 	},
+	{	/* AES GCM */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+			{.aead = {
+				.algo = RTE_CRYPTO_AEAD_AES_GCM,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 8
+				},
+				.digest_size = {
+					.min = 8,
+					.max = 16,
+					.increment = 4
+				},
+				.aad_size = {
+					.min = 0,
+					.max = 65535,
+					.increment = 1
+				},
+				.iv_size = {
+					.min = 12,
+					.max = 12,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
 	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 };
 
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
index 70e9d18e..d8021cda 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
@@ -7,15 +7,6 @@
 
 #include "aesni_mb_ops.h"
 
-/*
- * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50,
- * so if macro is not defined, it means that the version is 0.49.
- */
-#if !defined(IMB_VERSION_NUM)
-#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
-#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0)
-#endif
-
 #define CRYPTODEV_NAME_AESNI_MB_PMD	crypto_aesni_mb
 /**< AES-NI Multi buffer PMD device name */
 
@@ -31,8 +22,8 @@ int aesni_mb_logtype_driver;
 #define HMAC_IPAD_VALUE			(0x36)
 #define HMAC_OPAD_VALUE			(0x5C)
 
-/* Maximum length for digest (SHA-512 truncated needs 32 bytes) */
-#define DIGEST_LENGTH_MAX 32
+/* Maximum length for digest */
+#define DIGEST_LENGTH_MAX 64
 static const unsigned auth_blocksize[] = {
 		[MD5]		= 64,
 		[SHA1]		= 64,
@@ -64,7 +55,7 @@ static const unsigned auth_truncated_digest_byte_lengths[] = {
 		[SHA_384]	= 24,
 		[SHA_512]	= 32,
 		[AES_XCBC]	= 12,
-		[AES_CMAC]	= 16,
+		[AES_CMAC]	= 12,
 		[AES_CCM]	= 8,
 		[NULL_HASH]	= 0
 };
@@ -91,11 +82,13 @@ static const unsigned auth_digest_byte_lengths[] = {
 		[SHA_512]	= 64,
 		[AES_XCBC]	= 16,
 		[AES_CMAC]	= 16,
+		[AES_GMAC]	= 12,
 		[NULL_HASH]		= 0
 };
 
 /**
- * Get the output digest size in bytes for a specified authentication algorithm
+ * Get the full digest size in bytes for a specified authentication algorithm
+ * (if available in the Multi-buffer library)
  *
  * @Note: this function will not return a valid value for a non-valid
  * authentication algorithm
@@ -180,6 +173,8 @@ struct aesni_mb_session {
 				const void *ks_ptr[3];
 				uint64_t key[3][16];
 			} exp_3des_keys;
+
+			struct gcm_key_data gcm_key;
 		};
 		/**< Expanded AES keys - Allocating space to
 		 * contain the maximum expanded key size which
@@ -226,8 +221,10 @@ struct aesni_mb_session {
 			} cmac;
 			/**< Expanded XCBC authentication keys */
 		};
-	/** digest size */
-	uint16_t digest_len;
+	/** Generated digest size by the Multi-buffer library */
+	uint16_t gen_digest_len;
+	/** Requested digest size from Cryptodev */
+	uint16_t req_digest_len;
 
 	} auth;
 	struct {
diff --git a/drivers/crypto/caam_jr/Makefile b/drivers/crypto/caam_jr/Makefile
new file mode 100644
index 00000000..88cdf741
--- /dev/null
+++ b/drivers/crypto/caam_jr/Makefile
@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2017 NXP
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_pmd_caam_jr.a
+
+# build flags
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+CFLAGS += -D _GNU_SOURCE
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+
+CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa/include
+CFLAGS += -I$(RTE_SDK)/drivers/crypto/caam_jr
+#sharing the hw flib headers from dpaa2_sec pmd
+CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
+
+# versioning export map
+EXPORT_MAP := rte_pmd_caam_jr_version.map
+
+# library version
+LIBABIVER := 1
+
+# library source files
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr_capabilities.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr_hw.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr_uio.c
+# library dependencies
+
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_cryptodev
+LDLIBS += -lrte_bus_dpaa
+LDLIBS += -lrte_bus_vdev
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/crypto/caam_jr/caam_jr.c b/drivers/crypto/caam_jr/caam_jr.c
new file mode 100644
index 00000000..f505adf6
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr.c
@@ -0,0 +1,2508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <sched.h>
+#include <net/if.h>
+
+#include <rte_byteorder.h>
+#include <rte_common.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_crypto.h>
+#include <rte_cryptodev.h>
+#include <rte_bus_vdev.h>
+#include <rte_malloc.h>
+#include <rte_security_driver.h>
+#include <rte_hexdump.h>
+
+#include <caam_jr_capabilities.h>
+#include <caam_jr_config.h>
+#include <caam_jr_hw_specific.h>
+#include <caam_jr_pvt.h>
+#include <caam_jr_desc.h>
+#include <caam_jr_log.h>
+
+/* RTA header files */
+#include <hw/desc/common.h>
+#include <hw/desc/algo.h>
+#include <of.h>
+
+#define CAAM_JR_DBG	0
+#define CRYPTODEV_NAME_CAAM_JR_PMD	crypto_caam_jr
+static uint8_t cryptodev_driver_id;
+int caam_jr_logtype;
+
+enum rta_sec_era rta_sec_era;
+
+/* Lists the states possible for the SEC user space driver. */
+enum sec_driver_state_e {
+	SEC_DRIVER_STATE_IDLE,		/* Driver not initialized */
+	SEC_DRIVER_STATE_STARTED,	/* Driver initialized and can be used*/
+	SEC_DRIVER_STATE_RELEASE,	/* Driver release is in progress */
+};
+
+/* Job rings used for communication with SEC HW */
+static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];
+
+/* The current state of SEC user space driver */
+static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;
+
+/* The number of job rings used by SEC user space driver */
+static int g_job_rings_no;
+static int g_job_rings_max;
+
+struct sec_outring_entry {
+	phys_addr_t desc;	/* Pointer to completed descriptor */
+	uint32_t status;	/* Status for completed descriptor */
+} __rte_packed;
+
+/* virtual address conversin when mempool support is available for ctx */
+static inline phys_addr_t
+caam_jr_vtop_ctx(struct caam_jr_op_ctx *ctx, void *vaddr)
+{
+	PMD_INIT_FUNC_TRACE();
+	return (size_t)vaddr - ctx->vtop_offset;
+}
+
+static inline void
+caam_jr_op_ending(struct caam_jr_op_ctx *ctx)
+{
+	PMD_INIT_FUNC_TRACE();
+	/* report op status to sym->op and then free the ctx memeory  */
+	rte_mempool_put(ctx->ctx_pool, (void *)ctx);
+}
+
+static inline struct caam_jr_op_ctx *
+caam_jr_alloc_ctx(struct caam_jr_session *ses)
+{
+	struct caam_jr_op_ctx *ctx;
+	int ret;
+
+	PMD_INIT_FUNC_TRACE();
+	ret = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
+	if (!ctx || ret) {
+		CAAM_JR_DP_WARN("Alloc sec descriptor failed!");
+		return NULL;
+	}
+	/*
+	 * Clear SG memory. There are 16 SG entries of 16 Bytes each.
+	 * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
+	 * to clear all the SG entries. caam_jr_alloc_ctx() is called for
+	 * each packet, memset is costlier than dcbz_64().
+	 */
+	dcbz_64(&ctx->sg[SG_CACHELINE_0]);
+	dcbz_64(&ctx->sg[SG_CACHELINE_1]);
+	dcbz_64(&ctx->sg[SG_CACHELINE_2]);
+	dcbz_64(&ctx->sg[SG_CACHELINE_3]);
+
+	ctx->ctx_pool = ses->ctx_pool;
+	ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
+
+	return ctx;
+}
+
+static
+void caam_jr_stats_get(struct rte_cryptodev *dev,
+			struct rte_cryptodev_stats *stats)
+{
+	struct caam_jr_qp **qp = (struct caam_jr_qp **)
+					dev->data->queue_pairs;
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+	if (stats == NULL) {
+		CAAM_JR_ERR("Invalid stats ptr NULL");
+		return;
+	}
+	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
+		if (qp[i] == NULL) {
+			CAAM_JR_WARN("Uninitialised queue pair");
+			continue;
+		}
+
+		stats->enqueued_count += qp[i]->tx_pkts;
+		stats->dequeued_count += qp[i]->rx_pkts;
+		stats->enqueue_err_count += qp[i]->tx_errs;
+		stats->dequeue_err_count += qp[i]->rx_errs;
+		CAAM_JR_INFO("extra stats:\n\tRX Poll ERR = %" PRIu64
+			     "\n\tTX Ring Full = %" PRIu64,
+			     qp[i]->rx_poll_err,
+			     qp[i]->tx_ring_full);
+	}
+}
+
+static
+void caam_jr_stats_reset(struct rte_cryptodev *dev)
+{
+	int i;
+	struct caam_jr_qp **qp = (struct caam_jr_qp **)
+				   (dev->data->queue_pairs);
+
+	PMD_INIT_FUNC_TRACE();
+	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
+		if (qp[i] == NULL) {
+			CAAM_JR_WARN("Uninitialised queue pair");
+			continue;
+		}
+		qp[i]->rx_pkts = 0;
+		qp[i]->rx_errs = 0;
+		qp[i]->rx_poll_err = 0;
+		qp[i]->tx_pkts = 0;
+		qp[i]->tx_errs = 0;
+		qp[i]->tx_ring_full = 0;
+	}
+}
+
+static inline int
+is_cipher_only(struct caam_jr_session *ses)
+{
+	PMD_INIT_FUNC_TRACE();
+	return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
+		(ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
+}
+
+static inline int
+is_auth_only(struct caam_jr_session *ses)
+{
+	PMD_INIT_FUNC_TRACE();
+	return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
+		(ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
+}
+
+static inline int
+is_aead(struct caam_jr_session *ses)
+{
+	PMD_INIT_FUNC_TRACE();
+	return ((ses->cipher_alg == 0) &&
+		(ses->auth_alg == 0) &&
+		(ses->aead_alg != 0));
+}
+
+static inline int
+is_auth_cipher(struct caam_jr_session *ses)
+{
+	PMD_INIT_FUNC_TRACE();
+	return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
+		(ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
+		(ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
+}
+
+static inline int
+is_proto_ipsec(struct caam_jr_session *ses)
+{
+	PMD_INIT_FUNC_TRACE();
+	return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
+}
+
+static inline int
+is_encode(struct caam_jr_session *ses)
+{
+	PMD_INIT_FUNC_TRACE();
+	return ses->dir == DIR_ENC;
+}
+
+static inline int
+is_decode(struct caam_jr_session *ses)
+{
+	PMD_INIT_FUNC_TRACE();
+	return ses->dir == DIR_DEC;
+}
+
+static inline void
+caam_auth_alg(struct caam_jr_session *ses, struct alginfo *alginfo_a)
+{
+	PMD_INIT_FUNC_TRACE();
+	switch (ses->auth_alg) {
+	case RTE_CRYPTO_AUTH_NULL:
+		ses->digest_length = 0;
+		break;
+	case RTE_CRYPTO_AUTH_MD5_HMAC:
+		alginfo_a->algtype =
+			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+			OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
+		alginfo_a->algmode = OP_ALG_AAI_HMAC;
+		break;
+	case RTE_CRYPTO_AUTH_SHA1_HMAC:
+		alginfo_a->algtype =
+			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+			OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
+		alginfo_a->algmode = OP_ALG_AAI_HMAC;
+		break;
+	case RTE_CRYPTO_AUTH_SHA224_HMAC:
+		alginfo_a->algtype =
+			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+			OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
+		alginfo_a->algmode = OP_ALG_AAI_HMAC;
+		break;
+	case RTE_CRYPTO_AUTH_SHA256_HMAC:
+		alginfo_a->algtype =
+			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+			OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
+		alginfo_a->algmode = OP_ALG_AAI_HMAC;
+		break;
+	case RTE_CRYPTO_AUTH_SHA384_HMAC:
+		alginfo_a->algtype =
+			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+			OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
+		alginfo_a->algmode = OP_ALG_AAI_HMAC;
+		break;
+	case RTE_CRYPTO_AUTH_SHA512_HMAC:
+		alginfo_a->algtype =
+			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+			OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
+		alginfo_a->algmode = OP_ALG_AAI_HMAC;
+		break;
+	default:
+		CAAM_JR_DEBUG("unsupported auth alg %u", ses->auth_alg);
+	}
+}
+
+static inline void
+caam_cipher_alg(struct caam_jr_session *ses, struct alginfo *alginfo_c)
+{
+	PMD_INIT_FUNC_TRACE();
+	switch (ses->cipher_alg) {
+	case RTE_CRYPTO_CIPHER_NULL:
+		break;
+	case RTE_CRYPTO_CIPHER_AES_CBC:
+		alginfo_c->algtype =
+			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+			OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
+		alginfo_c->algmode = OP_ALG_AAI_CBC;
+		break;
+	case RTE_CRYPTO_CIPHER_3DES_CBC:
+		alginfo_c->algtype =
+			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+			OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
+		alginfo_c->algmode = OP_ALG_AAI_CBC;
+		break;
+	case RTE_CRYPTO_CIPHER_AES_CTR:
+		alginfo_c->algtype =
+			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+			OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
+		alginfo_c->algmode = OP_ALG_AAI_CTR;
+		break;
+	default:
+		CAAM_JR_DEBUG("unsupported cipher alg %d", ses->cipher_alg);
+	}
+}
+
+static inline void
+caam_aead_alg(struct caam_jr_session *ses, struct alginfo *alginfo)
+{
+	PMD_INIT_FUNC_TRACE();
+	switch (ses->aead_alg) {
+	case RTE_CRYPTO_AEAD_AES_GCM:
+		alginfo->algtype = OP_ALG_ALGSEL_AES;
+		alginfo->algmode = OP_ALG_AAI_GCM;
+		break;
+	default:
+		CAAM_JR_DEBUG("unsupported AEAD alg %d", ses->aead_alg);
+	}
+}
+
+/* prepare command block of the session */
+static int
+caam_jr_prep_cdb(struct caam_jr_session *ses)
+{
+	struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
+	int32_t shared_desc_len = 0;
+	struct sec_cdb *cdb;
+	int err;
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+	int swap = false;
+#else
+	int swap = true;
+#endif
+
+	PMD_INIT_FUNC_TRACE();
+	if (ses->cdb)
+		caam_jr_dma_free(ses->cdb);
+
+	cdb = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, sizeof(struct sec_cdb));
+	if (!cdb) {
+		CAAM_JR_ERR("failed to allocate memory for cdb\n");
+		return -1;
+	}
+
+	ses->cdb = cdb;
+
+	memset(cdb, 0, sizeof(struct sec_cdb));
+
+	if (is_cipher_only(ses)) {
+		caam_cipher_alg(ses, &alginfo_c);
+		if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
+			CAAM_JR_ERR("not supported cipher alg");
+			rte_free(cdb);
+			return -ENOTSUP;
+		}
+
+		alginfo_c.key = (size_t)ses->cipher_key.data;
+		alginfo_c.keylen = ses->cipher_key.length;
+		alginfo_c.key_enc_flags = 0;
+		alginfo_c.key_type = RTA_DATA_IMM;
+
+		shared_desc_len = cnstr_shdsc_blkcipher(
+						cdb->sh_desc, true,
+						swap, &alginfo_c,
+						NULL,
+						ses->iv.length,
+						ses->dir);
+	} else if (is_auth_only(ses)) {
+		caam_auth_alg(ses, &alginfo_a);
+		if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
+			CAAM_JR_ERR("not supported auth alg");
+			rte_free(cdb);
+			return -ENOTSUP;
+		}
+
+		alginfo_a.key = (size_t)ses->auth_key.data;
+		alginfo_a.keylen = ses->auth_key.length;
+		alginfo_a.key_enc_flags = 0;
+		alginfo_a.key_type = RTA_DATA_IMM;
+
+		shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
+						   swap, &alginfo_a,
+						   !ses->dir,
+						   ses->digest_length);
+	} else if (is_aead(ses)) {
+		caam_aead_alg(ses, &alginfo);
+		if (alginfo.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
+			CAAM_JR_ERR("not supported aead alg");
+			rte_free(cdb);
+			return -ENOTSUP;
+		}
+		alginfo.key = (size_t)ses->aead_key.data;
+		alginfo.keylen = ses->aead_key.length;
+		alginfo.key_enc_flags = 0;
+		alginfo.key_type = RTA_DATA_IMM;
+
+		if (ses->dir == DIR_ENC)
+			shared_desc_len = cnstr_shdsc_gcm_encap(
+					cdb->sh_desc, true, swap,
+					&alginfo,
+					ses->iv.length,
+					ses->digest_length);
+		else
+			shared_desc_len = cnstr_shdsc_gcm_decap(
+					cdb->sh_desc, true, swap,
+					&alginfo,
+					ses->iv.length,
+					ses->digest_length);
+	} else {
+		caam_cipher_alg(ses, &alginfo_c);
+		if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
+			CAAM_JR_ERR("not supported cipher alg");
+			rte_free(cdb);
+			return -ENOTSUP;
+		}
+
+		alginfo_c.key = (size_t)ses->cipher_key.data;
+		alginfo_c.keylen = ses->cipher_key.length;
+		alginfo_c.key_enc_flags = 0;
+		alginfo_c.key_type = RTA_DATA_IMM;
+
+		caam_auth_alg(ses, &alginfo_a);
+		if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
+			CAAM_JR_ERR("not supported auth alg");
+			rte_free(cdb);
+			return -ENOTSUP;
+		}
+
+		alginfo_a.key = (size_t)ses->auth_key.data;
+		alginfo_a.keylen = ses->auth_key.length;
+		alginfo_a.key_enc_flags = 0;
+		alginfo_a.key_type = RTA_DATA_IMM;
+
+		cdb->sh_desc[0] = alginfo_c.keylen;
+		cdb->sh_desc[1] = alginfo_a.keylen;
+		err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
+				       MIN_JOB_DESC_SIZE,
+				       (unsigned int *)cdb->sh_desc,
+				       &cdb->sh_desc[2], 2);
+
+		if (err < 0) {
+			CAAM_JR_ERR("Crypto: Incorrect key lengths");
+			rte_free(cdb);
+			return err;
+		}
+		if (cdb->sh_desc[2] & 1)
+			alginfo_c.key_type = RTA_DATA_IMM;
+		else {
+			alginfo_c.key = (size_t)caam_jr_mem_vtop(
+						(void *)(size_t)alginfo_c.key);
+			alginfo_c.key_type = RTA_DATA_PTR;
+		}
+		if (cdb->sh_desc[2] & (1<<1))
+			alginfo_a.key_type = RTA_DATA_IMM;
+		else {
+			alginfo_a.key = (size_t)caam_jr_mem_vtop(
+						(void *)(size_t)alginfo_a.key);
+			alginfo_a.key_type = RTA_DATA_PTR;
+		}
+		cdb->sh_desc[0] = 0;
+		cdb->sh_desc[1] = 0;
+		cdb->sh_desc[2] = 0;
+		if (is_proto_ipsec(ses)) {
+			if (ses->dir == DIR_ENC) {
+				shared_desc_len = cnstr_shdsc_ipsec_new_encap(
+						cdb->sh_desc,
+						true, swap, SHR_SERIAL,
+						&ses->encap_pdb,
+						(uint8_t *)&ses->ip4_hdr,
+						&alginfo_c, &alginfo_a);
+			} else if (ses->dir == DIR_DEC) {
+				shared_desc_len = cnstr_shdsc_ipsec_new_decap(
+						cdb->sh_desc,
+						true, swap, SHR_SERIAL,
+						&ses->decap_pdb,
+						&alginfo_c, &alginfo_a);
+			}
+		} else {
+			/* Auth_only_len is set as 0 here and it will be
+			 * overwritten in fd for each packet.
+			 */
+			shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
+					true, swap, &alginfo_c, &alginfo_a,
+					ses->iv.length, 0,
+					ses->digest_length, ses->dir);
+		}
+	}
+
+	if (shared_desc_len < 0) {
+		CAAM_JR_ERR("error in preparing command block");
+		return shared_desc_len;
+	}
+
+#if CAAM_JR_DBG
+	SEC_DUMP_DESC(cdb->sh_desc);
+#endif
+
+	cdb->sh_hdr.hi.field.idlen = shared_desc_len;
+
+	return 0;
+}
+
+/* @brief Poll the HW for already processed jobs in the JR
+ * and silently discard the available jobs or notify them to UA
+ * with indicated error code.
+ *
+ * @param [in,out]  job_ring        The job ring to poll.
+ * @param [in]  do_notify           Can be #TRUE or #FALSE. Indicates if
+ *				    descriptors are to be discarded
+ *                                  or notified to UA with given error_code.
+ * @param [out] notified_descs    Number of notified descriptors. Can be NULL
+ *					if do_notify is #FALSE
+ */
+static void
+hw_flush_job_ring(struct sec_job_ring_t *job_ring,
+		  uint32_t do_notify,
+		  uint32_t *notified_descs)
+{
+	int32_t jobs_no_to_discard = 0;
+	int32_t discarded_descs_no = 0;
+
+	PMD_INIT_FUNC_TRACE();
+	CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
+		job_ring, job_ring->pidx, job_ring->cidx, do_notify);
+
+	jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);
+
+	/* Discard all jobs */
+	CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
+		  job_ring, job_ring->pidx, job_ring->cidx,
+		  jobs_no_to_discard);
+
+	while (jobs_no_to_discard > discarded_descs_no) {
+		discarded_descs_no++;
+		/* Now increment the consumer index for the current job ring,
+		 * AFTER saving job in temporary location!
+		 * Increment the consumer index for the current job ring
+		 */
+		job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
+					 SEC_JOB_RING_SIZE);
+
+		hw_remove_entries(job_ring, 1);
+	}
+
+	if (do_notify == true) {
+		ASSERT(notified_descs != NULL);
+		*notified_descs = discarded_descs_no;
+	}
+}
+
+/* @brief Poll the HW for already processed jobs in the JR
+ * and notify the available jobs to UA.
+ *
+ * @param [in]  job_ring	The job ring to poll.
+ * @param [in]  limit           The maximum number of jobs to notify.
+ *                              If set to negative value, all available jobs are
+ *				notified.
+ *
+ * @retval >=0 for No of jobs notified to UA.
+ * @retval -1 for error
+ */
+static int
+hw_poll_job_ring(struct sec_job_ring_t *job_ring,
+		 struct rte_crypto_op **ops, int32_t limit,
+		 struct caam_jr_qp *jr_qp)
+{
+	int32_t jobs_no_to_notify = 0; /* the number of done jobs to notify*/
+	int32_t number_of_jobs_available = 0;
+	int32_t notified_descs_no = 0;
+	uint32_t sec_error_code = 0;
+	struct job_descriptor *current_desc;
+	phys_addr_t current_desc_addr;
+	phys_addr_t *temp_addr;
+	struct caam_jr_op_ctx *ctx;
+
+	PMD_INIT_FUNC_TRACE();
+	/* TODO check for ops have memory*/
+	/* check here if any JR error that cannot be written
+	 * in the output status word has occurred
+	 */
+	if (JR_REG_JRINT_JRE_EXTRACT(GET_JR_REG(JRINT, job_ring))) {
+		CAAM_JR_INFO("err received");
+		sec_error_code = JR_REG_JRINT_ERR_TYPE_EXTRACT(
+					GET_JR_REG(JRINT, job_ring));
+		if (unlikely(sec_error_code)) {
+			hw_job_ring_error_print(job_ring, sec_error_code);
+			return -1;
+		}
+	}
+	/* compute the number of jobs available in the job ring based on the
+	 * producer and consumer index values.
+	 */
+	number_of_jobs_available = hw_get_no_finished_jobs(job_ring);
+	/* Compute the number of notifications that need to be raised to UA
+	 * If limit > total number of done jobs -> notify all done jobs
+	 * If limit = 0 -> error
+	 * If limit < total number of done jobs -> notify a number
+	 * of done jobs equal with limit
+	 */
+	jobs_no_to_notify = (limit > number_of_jobs_available) ?
+				number_of_jobs_available : limit;
+	CAAM_JR_DP_DEBUG(
+		"Jr[%p] pi[%d] ci[%d].limit =%d Available=%d.Jobs to notify=%d",
+		job_ring, job_ring->pidx, job_ring->cidx,
+		limit, number_of_jobs_available, jobs_no_to_notify);
+
+	rte_smp_rmb();
+
+	while (jobs_no_to_notify > notified_descs_no) {
+		static uint64_t false_alarm;
+		static uint64_t real_poll;
+
+		/* Get job status here */
+		sec_error_code = job_ring->output_ring[job_ring->cidx].status;
+		/* Get completed descriptor */
+		temp_addr = &(job_ring->output_ring[job_ring->cidx].desc);
+		current_desc_addr = (phys_addr_t)sec_read_addr(temp_addr);
+
+		real_poll++;
+		/* todo check if it is false alarm no desc present */
+		if (!current_desc_addr) {
+			false_alarm++;
+			printf("false alarm %" PRIu64 "real %" PRIu64
+				" sec_err =0x%x cidx Index =0%d\n",
+				false_alarm, real_poll,
+				sec_error_code, job_ring->cidx);
+			rte_panic("CAAM JR descriptor NULL");
+			return notified_descs_no;
+		}
+		current_desc = (struct job_descriptor *)
+				caam_jr_dma_ptov(current_desc_addr);
+		/* now increment the consumer index for the current job ring,
+		 * AFTER saving job in temporary location!
+		 */
+		job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
+				 SEC_JOB_RING_SIZE);
+		/* Signal that the job has been processed and the slot is free*/
+		hw_remove_entries(job_ring, 1);
+		/*TODO for multiple ops, packets*/
+		ctx = container_of(current_desc, struct caam_jr_op_ctx, jobdes);
+		if (unlikely(sec_error_code)) {
+			CAAM_JR_ERR("desc at cidx %d generated error 0x%x\n",
+				job_ring->cidx, sec_error_code);
+			hw_handle_job_ring_error(job_ring, sec_error_code);
+			//todo improve with exact errors
+			ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			jr_qp->rx_errs++;
+		} else {
+			ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+#if CAAM_JR_DBG
+			if (ctx->op->sym->m_dst) {
+				rte_hexdump(stdout, "PROCESSED",
+				rte_pktmbuf_mtod(ctx->op->sym->m_dst, void *),
+				rte_pktmbuf_data_len(ctx->op->sym->m_dst));
+			} else {
+				rte_hexdump(stdout, "PROCESSED",
+				rte_pktmbuf_mtod(ctx->op->sym->m_src, void *),
+				rte_pktmbuf_data_len(ctx->op->sym->m_src));
+			}
+#endif
+		}
+		if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
+			struct ip *ip4_hdr;
+
+			if (ctx->op->sym->m_dst) {
+				/*TODO check for ip header or other*/
+				ip4_hdr = (struct ip *)
+				rte_pktmbuf_mtod(ctx->op->sym->m_dst, char*);
+				ctx->op->sym->m_dst->pkt_len =
+					rte_be_to_cpu_16(ip4_hdr->ip_len);
+				ctx->op->sym->m_dst->data_len =
+					rte_be_to_cpu_16(ip4_hdr->ip_len);
+			} else {
+				ip4_hdr = (struct ip *)
+				rte_pktmbuf_mtod(ctx->op->sym->m_src, char*);
+				ctx->op->sym->m_src->pkt_len =
+					rte_be_to_cpu_16(ip4_hdr->ip_len);
+				ctx->op->sym->m_src->data_len =
+					rte_be_to_cpu_16(ip4_hdr->ip_len);
+			}
+		}
+		*ops = ctx->op;
+		caam_jr_op_ending(ctx);
+		ops++;
+		notified_descs_no++;
+	}
+	return notified_descs_no;
+}
+
+static uint16_t
+caam_jr_dequeue_burst(void *qp, struct rte_crypto_op **ops,
+		       uint16_t nb_ops)
+{
+	struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
+	struct sec_job_ring_t *ring = jr_qp->ring;
+	int num_rx;
+	int ret;
+
+	PMD_INIT_FUNC_TRACE();
+	CAAM_JR_DP_DEBUG("Jr[%p]Polling. limit[%d]", ring, nb_ops);
+
+	/* Poll job ring
+	 * If nb_ops < 0 -> poll JR until no more notifications are available.
+	 * If nb_ops > 0 -> poll JR until limit is reached.
+	 */
+
+	/* Run hw poll job ring */
+	num_rx = hw_poll_job_ring(ring, ops, nb_ops, jr_qp);
+	if (num_rx < 0) {
+		CAAM_JR_ERR("Error polling SEC engine (%d)", num_rx);
+		return 0;
+	}
+
+	CAAM_JR_DP_DEBUG("Jr[%p].Jobs notified[%d]. ", ring, num_rx);
+
+	if (ring->jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
+		if (num_rx < nb_ops) {
+			ret = caam_jr_enable_irqs(ring->irq_fd);
+			SEC_ASSERT(ret == 0, ret,
+			"Failed to enable irqs for job ring %p", ring);
+		}
+	} else if (ring->jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
+
+		/* Always enable IRQ generation when in pure IRQ mode */
+		ret = caam_jr_enable_irqs(ring->irq_fd);
+		SEC_ASSERT(ret == 0, ret,
+			"Failed to enable irqs for job ring %p", ring);
+	}
+
+	jr_qp->rx_pkts += num_rx;
+
+	return num_rx;
+}
+
+/**
+ * packet looks like:
+ *		|<----data_len------->|
+ *    |ip_header|ah_header|icv|payload|
+ *              ^
+ *		|
+ *	   mbuf->pkt.data
+ */
+static inline struct caam_jr_op_ctx *
+build_auth_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+	struct rte_crypto_sym_op *sym = op->sym;
+	struct rte_mbuf *mbuf = sym->m_src;
+	struct caam_jr_op_ctx *ctx;
+	struct sec4_sg_entry *sg;
+	int	length;
+	struct sec_cdb *cdb;
+	uint64_t sdesc_offset;
+	struct sec_job_descriptor_t *jobdescr;
+	uint8_t extra_segs;
+
+	PMD_INIT_FUNC_TRACE();
+	if (is_decode(ses))
+		extra_segs = 2;
+	else
+		extra_segs = 1;
+
+	if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
+		CAAM_JR_DP_ERR("Auth: Max sec segs supported is %d",
+				MAX_SG_ENTRIES);
+		return NULL;
+	}
+
+	ctx = caam_jr_alloc_ctx(ses);
+	if (!ctx)
+		return NULL;
+
+	ctx->op = op;
+
+	cdb = ses->cdb;
+	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+	SEC_JD_INIT(jobdescr);
+	SEC_JD_SET_SD(jobdescr,
+		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+		cdb->sh_hdr.hi.field.idlen);
+
+	/* output */
+	SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
+			0, ses->digest_length);
+
+	/*input */
+	sg = &ctx->sg[0];
+	length = sym->auth.data.length;
+	sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + sym->auth.data.offset);
+	sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
+
+	/* Successive segs */
+	mbuf = mbuf->next;
+	while (mbuf) {
+		sg++;
+		sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
+		sg->len = cpu_to_caam32(mbuf->data_len);
+		mbuf = mbuf->next;
+	}
+
+	if (is_decode(ses)) {
+		/* digest verification case */
+		sg++;
+		/* hash result or digest, save digest first */
+		rte_memcpy(ctx->digest, sym->auth.digest.data,
+			   ses->digest_length);
+#if CAAM_JR_DBG
+		rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
+#endif
+		sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
+		sg->len = cpu_to_caam32(ses->digest_length);
+		length += ses->digest_length;
+	} else {
+		length -= ses->digest_length;
+	}
+
+	/* last element*/
+	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+	SEC_JD_SET_IN_PTR(jobdescr,
+		(uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, length);
+	/* enabling sg list */
+	(jobdescr)->seq_in.command.word  |= 0x01000000;
+
+	return ctx;
+}
+
+static inline struct caam_jr_op_ctx *
+build_auth_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+	struct rte_crypto_sym_op *sym = op->sym;
+	struct caam_jr_op_ctx *ctx;
+	struct sec4_sg_entry *sg;
+	rte_iova_t start_addr;
+	struct sec_cdb *cdb;
+	uint64_t sdesc_offset;
+	struct sec_job_descriptor_t *jobdescr;
+
+	PMD_INIT_FUNC_TRACE();
+	ctx = caam_jr_alloc_ctx(ses);
+	if (!ctx)
+		return NULL;
+
+	ctx->op = op;
+
+	cdb = ses->cdb;
+	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+	start_addr = rte_pktmbuf_iova(sym->m_src);
+
+	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+	SEC_JD_INIT(jobdescr);
+	SEC_JD_SET_SD(jobdescr,
+		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+		cdb->sh_hdr.hi.field.idlen);
+
+	/* output */
+	SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
+			0, ses->digest_length);
+
+	/*input */
+	if (is_decode(ses)) {
+		sg = &ctx->sg[0];
+		SEC_JD_SET_IN_PTR(jobdescr,
+			(uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
+			(sym->auth.data.length + ses->digest_length));
+		/* enabling sg list */
+		(jobdescr)->seq_in.command.word  |= 0x01000000;
+
+		/* hash result or digest, save digest first */
+		rte_memcpy(ctx->digest, sym->auth.digest.data,
+			   ses->digest_length);
+		sg->ptr = cpu_to_caam64(start_addr + sym->auth.data.offset);
+		sg->len = cpu_to_caam32(sym->auth.data.length);
+
+#if CAAM_JR_DBG
+		rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
+#endif
+		/* let's check digest by hw */
+		sg++;
+		sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
+		sg->len = cpu_to_caam32(ses->digest_length);
+		/* last element*/
+		sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+	} else {
+		SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)start_addr,
+			sym->auth.data.offset, sym->auth.data.length);
+	}
+	return ctx;
+}
+
+static inline struct caam_jr_op_ctx *
+build_cipher_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+	struct rte_crypto_sym_op *sym = op->sym;
+	struct rte_mbuf *mbuf = sym->m_src;
+	struct caam_jr_op_ctx *ctx;
+	struct sec4_sg_entry *sg, *in_sg;
+	int length;
+	struct sec_cdb *cdb;
+	uint64_t sdesc_offset;
+	uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
+			ses->iv.offset);
+	struct sec_job_descriptor_t *jobdescr;
+	uint8_t reg_segs;
+
+	PMD_INIT_FUNC_TRACE();
+	if (sym->m_dst) {
+		mbuf = sym->m_dst;
+		reg_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
+	} else {
+		mbuf = sym->m_src;
+		reg_segs = mbuf->nb_segs * 2 + 2;
+	}
+
+	if (reg_segs > MAX_SG_ENTRIES) {
+		CAAM_JR_DP_ERR("Cipher: Max sec segs supported is %d",
+				MAX_SG_ENTRIES);
+		return NULL;
+	}
+
+	ctx = caam_jr_alloc_ctx(ses);
+	if (!ctx)
+		return NULL;
+
+	ctx->op = op;
+	cdb = ses->cdb;
+	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+	SEC_JD_INIT(jobdescr);
+	SEC_JD_SET_SD(jobdescr,
+		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+		cdb->sh_hdr.hi.field.idlen);
+
+#if CAAM_JR_DBG
+	CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
+			sym->m_src->data_off, sym->cipher.data.offset,
+			sym->cipher.data.length, ses->iv.length);
+#endif
+	/* output */
+	if (sym->m_dst)
+		mbuf = sym->m_dst;
+	else
+		mbuf = sym->m_src;
+
+	sg = &ctx->sg[0];
+	length = sym->cipher.data.length;
+
+	sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
+		+ sym->cipher.data.offset);
+	sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
+
+	/* Successive segs */
+	mbuf = mbuf->next;
+	while (mbuf) {
+		sg++;
+		sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
+		sg->len = cpu_to_caam32(mbuf->data_len);
+		mbuf = mbuf->next;
+	}
+	/* last element*/
+	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+	SEC_JD_SET_OUT_PTR(jobdescr,
+			(uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0,
+			length);
+	/*enabling sg bit */
+	(jobdescr)->seq_out.command.word  |= 0x01000000;
+
+	/*input */
+	sg++;
+	mbuf = sym->m_src;
+	in_sg = sg;
+
+	length = sym->cipher.data.length + ses->iv.length;
+
+	/* IV */
+	sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
+	sg->len = cpu_to_caam32(ses->iv.length);
+
+	/* 1st seg */
+	sg++;
+	sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
+				+ sym->cipher.data.offset);
+	sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
+
+	/* Successive segs */
+	mbuf = mbuf->next;
+	while (mbuf) {
+		sg++;
+		sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
+		sg->len = cpu_to_caam32(mbuf->data_len);
+		mbuf = mbuf->next;
+	}
+	/* last element*/
+	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+
+	SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, in_sg), 0,
+				length);
+	/*enabling sg bit */
+	(jobdescr)->seq_in.command.word  |= 0x01000000;
+
+	return ctx;
+}
+
+static inline struct caam_jr_op_ctx *
+build_cipher_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+	struct rte_crypto_sym_op *sym = op->sym;
+	struct caam_jr_op_ctx *ctx;
+	struct sec4_sg_entry *sg;
+	rte_iova_t src_start_addr, dst_start_addr;
+	struct sec_cdb *cdb;
+	uint64_t sdesc_offset;
+	uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
+			ses->iv.offset);
+	struct sec_job_descriptor_t *jobdescr;
+
+	PMD_INIT_FUNC_TRACE();
+	ctx = caam_jr_alloc_ctx(ses);
+	if (!ctx)
+		return NULL;
+
+	ctx->op = op;
+	cdb = ses->cdb;
+	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+	src_start_addr = rte_pktmbuf_iova(sym->m_src);
+	if (sym->m_dst)
+		dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
+	else
+		dst_start_addr = src_start_addr;
+
+	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+	SEC_JD_INIT(jobdescr);
+	SEC_JD_SET_SD(jobdescr,
+		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+		cdb->sh_hdr.hi.field.idlen);
+
+#if CAAM_JR_DBG
+	CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
+			sym->m_src->data_off, sym->cipher.data.offset,
+			sym->cipher.data.length, ses->iv.length);
+#endif
+	/* output */
+	SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr,
+			sym->cipher.data.offset,
+			sym->cipher.data.length + ses->iv.length);
+
+	/*input */
+	sg = &ctx->sg[0];
+	SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
+				sym->cipher.data.length + ses->iv.length);
+	/*enabling sg bit */
+	(jobdescr)->seq_in.command.word  |= 0x01000000;
+
+	sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
+	sg->len = cpu_to_caam32(ses->iv.length);
+
+	sg = &ctx->sg[1];
+	sg->ptr = cpu_to_caam64(src_start_addr + sym->cipher.data.offset);
+	sg->len = cpu_to_caam32(sym->cipher.data.length);
+	/* last element*/
+	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+	return ctx;
+}
+
+/* For decapsulation:
+ *     Input:
+ * +----+----------------+--------------------------------+-----+
+ * | IV | Auth-only data | Authenticated & Encrypted data | ICV |
+ * +----+----------------+--------------------------------+-----+
+ *     Output:
+ * +----+--------------------------+
+ * | Decrypted & authenticated data |
+ * +----+--------------------------+
+ */
+
+static inline struct caam_jr_op_ctx *
+build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+	struct rte_crypto_sym_op *sym = op->sym;
+	struct caam_jr_op_ctx *ctx;
+	struct sec4_sg_entry *sg, *out_sg, *in_sg;
+	struct rte_mbuf *mbuf;
+	uint32_t length = 0;
+	struct sec_cdb *cdb;
+	uint64_t sdesc_offset;
+	uint8_t req_segs;
+	uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
+			ses->iv.offset);
+	struct sec_job_descriptor_t *jobdescr;
+	uint32_t auth_only_len;
+
+	PMD_INIT_FUNC_TRACE();
+	auth_only_len = op->sym->auth.data.length -
+				op->sym->cipher.data.length;
+
+	if (sym->m_dst) {
+		mbuf = sym->m_dst;
+		req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
+	} else {
+		mbuf = sym->m_src;
+		req_segs = mbuf->nb_segs * 2 + 3;
+	}
+
+	if (req_segs > MAX_SG_ENTRIES) {
+		CAAM_JR_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
+				MAX_SG_ENTRIES);
+		return NULL;
+	}
+
+	ctx = caam_jr_alloc_ctx(ses);
+	if (!ctx)
+		return NULL;
+
+	ctx->op = op;
+	cdb = ses->cdb;
+	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+	SEC_JD_INIT(jobdescr);
+	SEC_JD_SET_SD(jobdescr,
+		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+		cdb->sh_hdr.hi.field.idlen);
+
+	/* output */
+	if (sym->m_dst)
+		mbuf = sym->m_dst;
+	else
+		mbuf = sym->m_src;
+
+	out_sg = &ctx->sg[0];
+	if (is_encode(ses))
+		length = sym->auth.data.length + ses->digest_length;
+	else
+		length = sym->auth.data.length;
+
+	sg = &ctx->sg[0];
+
+	/* 1st seg */
+	sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
+		+ sym->auth.data.offset);
+	sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
+
+	/* Successive segs */
+	mbuf = mbuf->next;
+	while (mbuf) {
+		sg++;
+		sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
+		sg->len = cpu_to_caam32(mbuf->data_len);
+		mbuf = mbuf->next;
+	}
+
+	if (is_encode(ses)) {
+		/* set auth output */
+		sg++;
+		sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
+		sg->len = cpu_to_caam32(ses->digest_length);
+	}
+	/* last element*/
+	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+	SEC_JD_SET_OUT_PTR(jobdescr,
+			   (uint64_t)caam_jr_dma_vtop(out_sg), 0, length);
+	/* set sg bit */
+	(jobdescr)->seq_out.command.word  |= 0x01000000;
+
+	/* input */
+	sg++;
+	mbuf = sym->m_src;
+	in_sg = sg;
+	if (is_encode(ses))
+		length = ses->iv.length + sym->auth.data.length;
+	else
+		length = ses->iv.length + sym->auth.data.length
+						+ ses->digest_length;
+
+	sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
+	sg->len = cpu_to_caam32(ses->iv.length);
+
+	sg++;
+	/* 1st seg */
+	sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
+		+ sym->auth.data.offset);
+	sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
+
+	/* Successive segs */
+	mbuf = mbuf->next;
+	while (mbuf) {
+		sg++;
+		sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
+		sg->len = cpu_to_caam32(mbuf->data_len);
+		mbuf = mbuf->next;
+	}
+
+	if (is_decode(ses)) {
+		sg++;
+		rte_memcpy(ctx->digest, sym->auth.digest.data,
+		       ses->digest_length);
+		sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
+		sg->len = cpu_to_caam32(ses->digest_length);
+	}
+	/* last element*/
+	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+	SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(in_sg), 0,
+				length);
+	/* set sg bit */
+	(jobdescr)->seq_in.command.word  |= 0x01000000;
+	/* Auth_only_len is set as 0 in descriptor and it is
+	 * overwritten here in the jd which will update
+	 * the DPOVRD reg.
+	 */
+	if (auth_only_len)
+		/* set sg bit */
+		(jobdescr)->dpovrd = 0x80000000 | auth_only_len;
+
+	return ctx;
+}
+
+static inline struct caam_jr_op_ctx *
+build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+	struct rte_crypto_sym_op *sym = op->sym;
+	struct caam_jr_op_ctx *ctx;
+	struct sec4_sg_entry *sg;
+	rte_iova_t src_start_addr, dst_start_addr;
+	uint32_t length = 0;
+	struct sec_cdb *cdb;
+	uint64_t sdesc_offset;
+	uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
+			ses->iv.offset);
+	struct sec_job_descriptor_t *jobdescr;
+	uint32_t auth_only_len;
+
+	PMD_INIT_FUNC_TRACE();
+	auth_only_len = op->sym->auth.data.length -
+				op->sym->cipher.data.length;
+
+	src_start_addr = rte_pktmbuf_iova(sym->m_src);
+	if (sym->m_dst)
+		dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
+	else
+		dst_start_addr = src_start_addr;
+
+	ctx = caam_jr_alloc_ctx(ses);
+	if (!ctx)
+		return NULL;
+
+	ctx->op = op;
+	cdb = ses->cdb;
+	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+	SEC_JD_INIT(jobdescr);
+	SEC_JD_SET_SD(jobdescr,
+		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+		cdb->sh_hdr.hi.field.idlen);
+
+	/* input */
+	sg = &ctx->sg[0];
+	if (is_encode(ses)) {
+		sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
+		sg->len = cpu_to_caam32(ses->iv.length);
+		length += ses->iv.length;
+
+		sg++;
+		sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
+		sg->len = cpu_to_caam32(sym->auth.data.length);
+		length += sym->auth.data.length;
+		/* last element*/
+		sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+	} else {
+		sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
+		sg->len = cpu_to_caam32(ses->iv.length);
+		length += ses->iv.length;
+
+		sg++;
+		sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
+		sg->len = cpu_to_caam32(sym->auth.data.length);
+		length += sym->auth.data.length;
+
+		rte_memcpy(ctx->digest, sym->auth.digest.data,
+		       ses->digest_length);
+		sg++;
+		sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
+		sg->len = cpu_to_caam32(ses->digest_length);
+		length += ses->digest_length;
+		/* last element*/
+		sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+	}
+
+	SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(&ctx->sg[0]), 0,
+				length);
+	/* set sg bit */
+	(jobdescr)->seq_in.command.word  |= 0x01000000;
+
+	/* output */
+	sg = &ctx->sg[6];
+
+	sg->ptr = cpu_to_caam64(dst_start_addr + sym->cipher.data.offset);
+	sg->len = cpu_to_caam32(sym->cipher.data.length);
+	length = sym->cipher.data.length;
+
+	if (is_encode(ses)) {
+		/* set auth output */
+		sg++;
+		sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
+		sg->len = cpu_to_caam32(ses->digest_length);
+		length += ses->digest_length;
+	}
+	/* last element*/
+	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+	SEC_JD_SET_OUT_PTR(jobdescr,
+			   (uint64_t)caam_jr_dma_vtop(&ctx->sg[6]), 0, length);
+	/* set sg bit */
+	(jobdescr)->seq_out.command.word  |= 0x01000000;
+
+	/* Auth_only_len is set as 0 in descriptor and it is
+	 * overwritten here in the jd which will update
+	 * the DPOVRD reg.
+	 */
+	if (auth_only_len)
+		/* set sg bit */
+		(jobdescr)->dpovrd = 0x80000000 | auth_only_len;
+
+	return ctx;
+}
+
+static inline struct caam_jr_op_ctx *
+build_proto(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+	struct rte_crypto_sym_op *sym = op->sym;
+	struct caam_jr_op_ctx *ctx = NULL;
+	phys_addr_t src_start_addr, dst_start_addr;
+	struct sec_cdb *cdb;
+	uint64_t sdesc_offset;
+	struct sec_job_descriptor_t *jobdescr;
+
+	PMD_INIT_FUNC_TRACE();
+	ctx = caam_jr_alloc_ctx(ses);
+	if (!ctx)
+		return NULL;
+	ctx->op = op;
+
+	src_start_addr = rte_pktmbuf_iova(sym->m_src);
+	if (sym->m_dst)
+		dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
+	else
+		dst_start_addr = src_start_addr;
+
+	cdb = ses->cdb;
+	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+	SEC_JD_INIT(jobdescr);
+	SEC_JD_SET_SD(jobdescr,
+		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+			cdb->sh_hdr.hi.field.idlen);
+
+	/* output */
+	SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, 0,
+			sym->m_src->buf_len - sym->m_src->data_off);
+	/* input */
+	SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)src_start_addr, 0,
+			sym->m_src->pkt_len);
+	sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
+
+	return ctx;
+}
+
+static int
+caam_jr_enqueue_op(struct rte_crypto_op *op, struct caam_jr_qp *qp)
+{
+	struct sec_job_ring_t *ring = qp->ring;
+	struct caam_jr_session *ses;
+	struct caam_jr_op_ctx *ctx = NULL;
+	struct sec_job_descriptor_t *jobdescr __rte_unused;
+
+	PMD_INIT_FUNC_TRACE();
+	switch (op->sess_type) {
+	case RTE_CRYPTO_OP_WITH_SESSION:
+		ses = (struct caam_jr_session *)
+		get_sym_session_private_data(op->sym->session,
+					cryptodev_driver_id);
+		break;
+	case RTE_CRYPTO_OP_SECURITY_SESSION:
+		ses = (struct caam_jr_session *)
+			get_sec_session_private_data(
+					op->sym->sec_session);
+		break;
+	default:
+		CAAM_JR_DP_ERR("sessionless crypto op not supported");
+		qp->tx_errs++;
+		return -1;
+	}
+
+	if (unlikely(!ses->qp || ses->qp != qp)) {
+		CAAM_JR_DP_DEBUG("Old:sess->qp=%p New qp = %p\n", ses->qp, qp);
+		ses->qp = qp;
+		caam_jr_prep_cdb(ses);
+	}
+
+	if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
+		if (is_auth_cipher(ses))
+			ctx = build_cipher_auth(op, ses);
+		else if (is_aead(ses))
+			goto err1;
+		else if (is_auth_only(ses))
+			ctx = build_auth_only(op, ses);
+		else if (is_cipher_only(ses))
+			ctx = build_cipher_only(op, ses);
+		else if (is_proto_ipsec(ses))
+			ctx = build_proto(op, ses);
+	} else {
+		if (is_auth_cipher(ses))
+			ctx = build_cipher_auth_sg(op, ses);
+		else if (is_aead(ses))
+			goto err1;
+		else if (is_auth_only(ses))
+			ctx = build_auth_only_sg(op, ses);
+		else if (is_cipher_only(ses))
+			ctx = build_cipher_only_sg(op, ses);
+	}
+err1:
+	if (unlikely(!ctx)) {
+		qp->tx_errs++;
+		CAAM_JR_ERR("not supported sec op");
+		return -1;
+	}
+#if CAAM_JR_DBG
+	if (is_decode(ses))
+		rte_hexdump(stdout, "DECODE",
+			rte_pktmbuf_mtod(op->sym->m_src, void *),
+			rte_pktmbuf_data_len(op->sym->m_src));
+	else
+		rte_hexdump(stdout, "ENCODE",
+			rte_pktmbuf_mtod(op->sym->m_src, void *),
+			rte_pktmbuf_data_len(op->sym->m_src));
+
+	printf("\n JD before conversion\n");
+	for (int i = 0; i < 12; i++)
+		printf("\n 0x%08x", ctx->jobdes.desc[i]);
+#endif
+
+	CAAM_JR_DP_DEBUG("Jr[%p] pi[%d] ci[%d].Before sending desc",
+		      ring, ring->pidx, ring->cidx);
+
+	/* todo - do we want to retry */
+	if (SEC_JOB_RING_IS_FULL(ring->pidx, ring->cidx,
+			 SEC_JOB_RING_SIZE, SEC_JOB_RING_SIZE)) {
+		CAAM_JR_DP_DEBUG("Ring FULL Jr[%p] pi[%d] ci[%d].Size = %d",
+			      ring, ring->pidx, ring->cidx, SEC_JOB_RING_SIZE);
+		caam_jr_op_ending(ctx);
+		qp->tx_ring_full++;
+		return -EBUSY;
+	}
+
+#if CORE_BYTE_ORDER != CAAM_BYTE_ORDER
+	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+	jobdescr->deschdr.command.word =
+		cpu_to_caam32(jobdescr->deschdr.command.word);
+	jobdescr->sd_ptr = cpu_to_caam64(jobdescr->sd_ptr);
+	jobdescr->seq_out.command.word =
+		cpu_to_caam32(jobdescr->seq_out.command.word);
+	jobdescr->seq_out_ptr = cpu_to_caam64(jobdescr->seq_out_ptr);
+	jobdescr->out_ext_length = cpu_to_caam32(jobdescr->out_ext_length);
+	jobdescr->seq_in.command.word =
+		cpu_to_caam32(jobdescr->seq_in.command.word);
+	jobdescr->seq_in_ptr = cpu_to_caam64(jobdescr->seq_in_ptr);
+	jobdescr->in_ext_length = cpu_to_caam32(jobdescr->in_ext_length);
+	jobdescr->load_dpovrd.command.word =
+		cpu_to_caam32(jobdescr->load_dpovrd.command.word);
+	jobdescr->dpovrd = cpu_to_caam32(jobdescr->dpovrd);
+#endif
+
+	/* Set ptr in input ring to current descriptor	*/
+	sec_write_addr(&ring->input_ring[ring->pidx],
+			(phys_addr_t)caam_jr_vtop_ctx(ctx, ctx->jobdes.desc));
+	rte_smp_wmb();
+
+	/* Notify HW that a new job is enqueued */
+	hw_enqueue_desc_on_job_ring(ring);
+
+	/* increment the producer index for the current job ring */
+	ring->pidx = SEC_CIRCULAR_COUNTER(ring->pidx, SEC_JOB_RING_SIZE);
+
+	return 0;
+}
+
+static uint16_t
+caam_jr_enqueue_burst(void *qp, struct rte_crypto_op **ops,
+		       uint16_t nb_ops)
+{
+	/* Function to transmit the frames to given device and queuepair */
+	uint32_t loop;
+	int32_t ret;
+	struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
+	uint16_t num_tx = 0;
+
+	PMD_INIT_FUNC_TRACE();
+	/*Prepare each packet which is to be sent*/
+	for (loop = 0; loop < nb_ops; loop++) {
+		ret = caam_jr_enqueue_op(ops[loop], jr_qp);
+		if (!ret)
+			num_tx++;
+	}
+
+	jr_qp->tx_pkts += num_tx;
+
+	return num_tx;
+}
+
+/* Release queue pair */
+static int
+caam_jr_queue_pair_release(struct rte_cryptodev *dev,
+			   uint16_t qp_id)
+{
+	struct sec_job_ring_t *internals;
+	struct caam_jr_qp *qp = NULL;
+
+	PMD_INIT_FUNC_TRACE();
+	CAAM_JR_DEBUG("dev =%p, queue =%d", dev, qp_id);
+
+	internals = dev->data->dev_private;
+	if (qp_id >= internals->max_nb_queue_pairs) {
+		CAAM_JR_ERR("Max supported qpid %d",
+			     internals->max_nb_queue_pairs);
+		return -EINVAL;
+	}
+
+	qp = &internals->qps[qp_id];
+	qp->ring = NULL;
+	dev->data->queue_pairs[qp_id] = NULL;
+
+	return 0;
+}
+
+/* Setup a queue pair */
+static int
+caam_jr_queue_pair_setup(
+		struct rte_cryptodev *dev, uint16_t qp_id,
+		__rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
+		__rte_unused int socket_id,
+		__rte_unused struct rte_mempool *session_pool)
+{
+	struct sec_job_ring_t *internals;
+	struct caam_jr_qp *qp = NULL;
+
+	PMD_INIT_FUNC_TRACE();
+	CAAM_JR_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
+
+	internals = dev->data->dev_private;
+	if (qp_id >= internals->max_nb_queue_pairs) {
+		CAAM_JR_ERR("Max supported qpid %d",
+			     internals->max_nb_queue_pairs);
+		return -EINVAL;
+	}
+
+	qp = &internals->qps[qp_id];
+	qp->ring = internals;
+	dev->data->queue_pairs[qp_id] = qp;
+
+	return 0;
+}
+
+/* Return the number of allocated queue pairs */
+static uint32_t
+caam_jr_queue_pair_count(struct rte_cryptodev *dev)
+{
+	PMD_INIT_FUNC_TRACE();
+
+	return dev->data->nb_queue_pairs;
+}
+
+/* Returns the size of the aesni gcm session structure */
+static unsigned int
+caam_jr_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	PMD_INIT_FUNC_TRACE();
+
+	return sizeof(struct caam_jr_session);
+}
+
+static int
+caam_jr_cipher_init(struct rte_cryptodev *dev __rte_unused,
+		    struct rte_crypto_sym_xform *xform,
+		    struct caam_jr_session *session)
+{
+	PMD_INIT_FUNC_TRACE();
+	session->cipher_alg = xform->cipher.algo;
+	session->iv.length = xform->cipher.iv.length;
+	session->iv.offset = xform->cipher.iv.offset;
+	session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
+					       RTE_CACHE_LINE_SIZE);
+	if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
+		CAAM_JR_ERR("No Memory for cipher key\n");
+		return -ENOMEM;
+	}
+	session->cipher_key.length = xform->cipher.key.length;
+
+	memcpy(session->cipher_key.data, xform->cipher.key.data,
+	       xform->cipher.key.length);
+	session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
+			DIR_ENC : DIR_DEC;
+
+	return 0;
+}
+
+static int
+caam_jr_auth_init(struct rte_cryptodev *dev __rte_unused,
+		  struct rte_crypto_sym_xform *xform,
+		  struct caam_jr_session *session)
+{
+	PMD_INIT_FUNC_TRACE();
+	session->auth_alg = xform->auth.algo;
+	session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
+					     RTE_CACHE_LINE_SIZE);
+	if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
+		CAAM_JR_ERR("No Memory for auth key\n");
+		return -ENOMEM;
+	}
+	session->auth_key.length = xform->auth.key.length;
+	session->digest_length = xform->auth.digest_length;
+
+	memcpy(session->auth_key.data, xform->auth.key.data,
+	       xform->auth.key.length);
+	session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
+			DIR_ENC : DIR_DEC;
+
+	return 0;
+}
+
+static int
+caam_jr_aead_init(struct rte_cryptodev *dev __rte_unused,
+		  struct rte_crypto_sym_xform *xform,
+		  struct caam_jr_session *session)
+{
+	PMD_INIT_FUNC_TRACE();
+	session->aead_alg = xform->aead.algo;
+	session->iv.length = xform->aead.iv.length;
+	session->iv.offset = xform->aead.iv.offset;
+	session->auth_only_len = xform->aead.aad_length;
+	session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
+					     RTE_CACHE_LINE_SIZE);
+	if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
+		CAAM_JR_ERR("No Memory for aead key\n");
+		return -ENOMEM;
+	}
+	session->aead_key.length = xform->aead.key.length;
+	session->digest_length = xform->aead.digest_length;
+
+	memcpy(session->aead_key.data, xform->aead.key.data,
+	       xform->aead.key.length);
+	session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
+			DIR_ENC : DIR_DEC;
+
+	return 0;
+}
+
+static int
+caam_jr_set_session_parameters(struct rte_cryptodev *dev,
+			       struct rte_crypto_sym_xform *xform, void *sess)
+{
+	struct sec_job_ring_t *internals = dev->data->dev_private;
+	struct caam_jr_session *session = sess;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (unlikely(sess == NULL)) {
+		CAAM_JR_ERR("invalid session struct");
+		return -EINVAL;
+	}
+
+	/* Default IV length = 0 */
+	session->iv.length = 0;
+
+	/* Cipher Only */
+	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
+		session->auth_alg = RTE_CRYPTO_AUTH_NULL;
+		caam_jr_cipher_init(dev, xform, session);
+
+	/* Authentication Only */
+	} else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
+		   xform->next == NULL) {
+		session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
+		caam_jr_auth_init(dev, xform, session);
+
+	/* Cipher then Authenticate */
+	} else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
+		   xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
+		if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
+			caam_jr_cipher_init(dev, xform, session);
+			caam_jr_auth_init(dev, xform->next, session);
+		} else {
+			CAAM_JR_ERR("Not supported: Auth then Cipher");
+			goto err1;
+		}
+
+	/* Authenticate then Cipher */
+	} else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
+		   xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
+		if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
+			caam_jr_auth_init(dev, xform, session);
+			caam_jr_cipher_init(dev, xform->next, session);
+		} else {
+			CAAM_JR_ERR("Not supported: Auth then Cipher");
+			goto err1;
+		}
+
+	/* AEAD operation for AES-GCM kind of Algorithms */
+	} else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
+		   xform->next == NULL) {
+		caam_jr_aead_init(dev, xform, session);
+
+	} else {
+		CAAM_JR_ERR("Invalid crypto type");
+		return -EINVAL;
+	}
+	session->ctx_pool = internals->ctx_pool;
+
+	return 0;
+
+err1:
+	rte_free(session->cipher_key.data);
+	rte_free(session->auth_key.data);
+	memset(session, 0, sizeof(struct caam_jr_session));
+
+	return -EINVAL;
+}
+
+static int
+caam_jr_sym_session_configure(struct rte_cryptodev *dev,
+			      struct rte_crypto_sym_xform *xform,
+			      struct rte_cryptodev_sym_session *sess,
+			      struct rte_mempool *mempool)
+{
+	void *sess_private_data;
+	int ret;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rte_mempool_get(mempool, &sess_private_data)) {
+		CAAM_JR_ERR("Couldn't get object from session mempool");
+		return -ENOMEM;
+	}
+
+	memset(sess_private_data, 0, sizeof(struct caam_jr_session));
+	ret = caam_jr_set_session_parameters(dev, xform, sess_private_data);
+	if (ret != 0) {
+		CAAM_JR_ERR("failed to configure session parameters");
+		/* Return session to mempool */
+		rte_mempool_put(mempool, sess_private_data);
+		return ret;
+	}
+
+	set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
+
+	return 0;
+}
+
+/* Clear the memory of session so it doesn't leave key material behind */
+static void
+caam_jr_sym_session_clear(struct rte_cryptodev *dev,
+		struct rte_cryptodev_sym_session *sess)
+{
+	uint8_t index = dev->driver_id;
+	void *sess_priv = get_sym_session_private_data(sess, index);
+	struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (sess_priv) {
+		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+
+		rte_free(s->cipher_key.data);
+		rte_free(s->auth_key.data);
+		memset(s, 0, sizeof(struct caam_jr_session));
+		set_sym_session_private_data(sess, index, NULL);
+		rte_mempool_put(sess_mp, sess_priv);
+	}
+}
+
+static int
+caam_jr_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
+			  struct rte_security_session_conf *conf,
+			  void *sess)
+{
+	struct sec_job_ring_t *internals = dev->data->dev_private;
+	struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
+	struct rte_crypto_auth_xform *auth_xform;
+	struct rte_crypto_cipher_xform *cipher_xform;
+	struct caam_jr_session *session = (struct caam_jr_session *)sess;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
+		cipher_xform = &conf->crypto_xform->cipher;
+		auth_xform = &conf->crypto_xform->next->auth;
+	} else {
+		auth_xform = &conf->crypto_xform->auth;
+		cipher_xform = &conf->crypto_xform->next->cipher;
+	}
+	session->proto_alg = conf->protocol;
+	session->cipher_key.data = rte_zmalloc(NULL,
+					       cipher_xform->key.length,
+					       RTE_CACHE_LINE_SIZE);
+	if (session->cipher_key.data == NULL &&
+			cipher_xform->key.length > 0) {
+		CAAM_JR_ERR("No Memory for cipher key\n");
+		return -ENOMEM;
+	}
+
+	session->cipher_key.length = cipher_xform->key.length;
+	session->auth_key.data = rte_zmalloc(NULL,
+					auth_xform->key.length,
+					RTE_CACHE_LINE_SIZE);
+	if (session->auth_key.data == NULL &&
+			auth_xform->key.length > 0) {
+		CAAM_JR_ERR("No Memory for auth key\n");
+		rte_free(session->cipher_key.data);
+		return -ENOMEM;
+	}
+	session->auth_key.length = auth_xform->key.length;
+	memcpy(session->cipher_key.data, cipher_xform->key.data,
+			cipher_xform->key.length);
+	memcpy(session->auth_key.data, auth_xform->key.data,
+			auth_xform->key.length);
+
+	switch (auth_xform->algo) {
+	case RTE_CRYPTO_AUTH_SHA1_HMAC:
+		session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
+		break;
+	case RTE_CRYPTO_AUTH_MD5_HMAC:
+		session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
+		break;
+	case RTE_CRYPTO_AUTH_SHA256_HMAC:
+		session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
+		break;
+	case RTE_CRYPTO_AUTH_SHA384_HMAC:
+		session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
+		break;
+	case RTE_CRYPTO_AUTH_SHA512_HMAC:
+		session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
+		break;
+	case RTE_CRYPTO_AUTH_AES_CMAC:
+		session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
+		break;
+	case RTE_CRYPTO_AUTH_NULL:
+		session->auth_alg = RTE_CRYPTO_AUTH_NULL;
+		break;
+	case RTE_CRYPTO_AUTH_SHA224_HMAC:
+	case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
+	case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
+	case RTE_CRYPTO_AUTH_SHA1:
+	case RTE_CRYPTO_AUTH_SHA256:
+	case RTE_CRYPTO_AUTH_SHA512:
+	case RTE_CRYPTO_AUTH_SHA224:
+	case RTE_CRYPTO_AUTH_SHA384:
+	case RTE_CRYPTO_AUTH_MD5:
+	case RTE_CRYPTO_AUTH_AES_GMAC:
+	case RTE_CRYPTO_AUTH_KASUMI_F9:
+	case RTE_CRYPTO_AUTH_AES_CBC_MAC:
+	case RTE_CRYPTO_AUTH_ZUC_EIA3:
+		CAAM_JR_ERR("Crypto: Unsupported auth alg %u\n",
+			auth_xform->algo);
+		goto out;
+	default:
+		CAAM_JR_ERR("Crypto: Undefined Auth specified %u\n",
+			auth_xform->algo);
+		goto out;
+	}
+
+	switch (cipher_xform->algo) {
+	case RTE_CRYPTO_CIPHER_AES_CBC:
+		session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
+		break;
+	case RTE_CRYPTO_CIPHER_3DES_CBC:
+		session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
+		break;
+	case RTE_CRYPTO_CIPHER_AES_CTR:
+		session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
+		break;
+	case RTE_CRYPTO_CIPHER_NULL:
+	case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
+	case RTE_CRYPTO_CIPHER_3DES_ECB:
+	case RTE_CRYPTO_CIPHER_AES_ECB:
+	case RTE_CRYPTO_CIPHER_KASUMI_F8:
+		CAAM_JR_ERR("Crypto: Unsupported Cipher alg %u\n",
+			cipher_xform->algo);
+		goto out;
+	default:
+		CAAM_JR_ERR("Crypto: Undefined Cipher specified %u\n",
+			cipher_xform->algo);
+		goto out;
+	}
+
+	if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
+		memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
+				sizeof(session->ip4_hdr));
+		session->ip4_hdr.ip_v = IPVERSION;
+		session->ip4_hdr.ip_hl = 5;
+		session->ip4_hdr.ip_len = rte_cpu_to_be_16(
+						sizeof(session->ip4_hdr));
+		session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
+		session->ip4_hdr.ip_id = 0;
+		session->ip4_hdr.ip_off = 0;
+		session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
+		session->ip4_hdr.ip_p = (ipsec_xform->proto ==
+				RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
+				: IPPROTO_AH;
+		session->ip4_hdr.ip_sum = 0;
+		session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
+		session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
+		session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
+						(void *)&session->ip4_hdr,
+						sizeof(struct ip));
+
+		session->encap_pdb.options =
+			(IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
+			PDBOPTS_ESP_OIHI_PDB_INL |
+			PDBOPTS_ESP_IVSRC |
+			PDBHMO_ESP_ENCAP_DTTL;
+		session->encap_pdb.spi = ipsec_xform->spi;
+		session->encap_pdb.ip_hdr_len = sizeof(struct ip);
+
+		session->dir = DIR_ENC;
+	} else if (ipsec_xform->direction ==
+			RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
+		memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
+		session->decap_pdb.options = sizeof(struct ip) << 16;
+		session->dir = DIR_DEC;
+	} else
+		goto out;
+	session->ctx_pool = internals->ctx_pool;
+
+	return 0;
+out:
+	rte_free(session->auth_key.data);
+	rte_free(session->cipher_key.data);
+	memset(session, 0, sizeof(struct caam_jr_session));
+	return -1;
+}
+
+static int
+caam_jr_security_session_create(void *dev,
+				struct rte_security_session_conf *conf,
+				struct rte_security_session *sess,
+				struct rte_mempool *mempool)
+{
+	void *sess_private_data;
+	struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
+	int ret;
+
+	PMD_INIT_FUNC_TRACE();
+	if (rte_mempool_get(mempool, &sess_private_data)) {
+		CAAM_JR_ERR("Couldn't get object from session mempool");
+		return -ENOMEM;
+	}
+
+	switch (conf->protocol) {
+	case RTE_SECURITY_PROTOCOL_IPSEC:
+		ret = caam_jr_set_ipsec_session(cdev, conf,
+				sess_private_data);
+		break;
+	case RTE_SECURITY_PROTOCOL_MACSEC:
+		return -ENOTSUP;
+	default:
+		return -EINVAL;
+	}
+	if (ret != 0) {
+		CAAM_JR_ERR("failed to configure session parameters");
+		/* Return session to mempool */
+		rte_mempool_put(mempool, sess_private_data);
+		return ret;
+	}
+
+	set_sec_session_private_data(sess, sess_private_data);
+
+	return ret;
+}
+
+/* Clear the memory of session so it doesn't leave key material behind */
+static int
+caam_jr_security_session_destroy(void *dev __rte_unused,
+				 struct rte_security_session *sess)
+{
+	PMD_INIT_FUNC_TRACE();
+	void *sess_priv = get_sec_session_private_data(sess);
+
+	struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
+
+	if (sess_priv) {
+		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+
+		rte_free(s->cipher_key.data);
+		rte_free(s->auth_key.data);
+		memset(sess, 0, sizeof(struct caam_jr_session));
+		set_sec_session_private_data(sess, NULL);
+		rte_mempool_put(sess_mp, sess_priv);
+	}
+	return 0;
+}
+
+
+static int
+caam_jr_dev_configure(struct rte_cryptodev *dev,
+		       struct rte_cryptodev_config *config __rte_unused)
+{
+	char str[20];
+	struct sec_job_ring_t *internals;
+
+	PMD_INIT_FUNC_TRACE();
+
+	internals = dev->data->dev_private;
+	sprintf(str, "ctx_pool_%d", dev->data->dev_id);
+	if (!internals->ctx_pool) {
+		internals->ctx_pool = rte_mempool_create((const char *)str,
+						CTX_POOL_NUM_BUFS,
+						sizeof(struct caam_jr_op_ctx),
+						CTX_POOL_CACHE_SIZE, 0,
+						NULL, NULL, NULL, NULL,
+						SOCKET_ID_ANY, 0);
+		if (!internals->ctx_pool) {
+			CAAM_JR_ERR("%s create failed\n", str);
+			return -ENOMEM;
+		}
+	} else
+		CAAM_JR_INFO("mempool already created for dev_id : %d",
+				dev->data->dev_id);
+
+	return 0;
+}
+
+static int
+caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused)
+{
+	PMD_INIT_FUNC_TRACE();
+	return 0;
+}
+
+static void
+caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused)
+{
+	PMD_INIT_FUNC_TRACE();
+}
+
+static int
+caam_jr_dev_close(struct rte_cryptodev *dev)
+{
+	struct sec_job_ring_t *internals;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (dev == NULL)
+		return -ENOMEM;
+
+	internals = dev->data->dev_private;
+	rte_mempool_free(internals->ctx_pool);
+	internals->ctx_pool = NULL;
+
+	return 0;
+}
+
+static void
+caam_jr_dev_infos_get(struct rte_cryptodev *dev,
+		       struct rte_cryptodev_info *info)
+{
+	struct sec_job_ring_t *internals = dev->data->dev_private;
+
+	PMD_INIT_FUNC_TRACE();
+	if (info != NULL) {
+		info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
+		info->feature_flags = dev->feature_flags;
+		info->capabilities = caam_jr_get_cryptodev_capabilities();
+		info->sym.max_nb_sessions = internals->max_nb_sessions;
+		info->driver_id = cryptodev_driver_id;
+	}
+}
+
+static struct rte_cryptodev_ops caam_jr_ops = {
+	.dev_configure	      = caam_jr_dev_configure,
+	.dev_start	      = caam_jr_dev_start,
+	.dev_stop	      = caam_jr_dev_stop,
+	.dev_close	      = caam_jr_dev_close,
+	.dev_infos_get        = caam_jr_dev_infos_get,
+	.stats_get	      = caam_jr_stats_get,
+	.stats_reset	      = caam_jr_stats_reset,
+	.queue_pair_setup     = caam_jr_queue_pair_setup,
+	.queue_pair_release   = caam_jr_queue_pair_release,
+	.queue_pair_count     = caam_jr_queue_pair_count,
+	.sym_session_get_size = caam_jr_sym_session_get_size,
+	.sym_session_configure = caam_jr_sym_session_configure,
+	.sym_session_clear    = caam_jr_sym_session_clear
+};
+
+static struct rte_security_ops caam_jr_security_ops = {
+	.session_create = caam_jr_security_session_create,
+	.session_update = NULL,
+	.session_stats_get = NULL,
+	.session_destroy = caam_jr_security_session_destroy,
+	.set_pkt_metadata = NULL,
+	.capabilities_get = caam_jr_get_security_capabilities
+};
+
+/* @brief Flush job rings of any processed descs.
+ * The processed descs are silently dropped,
+ * WITHOUT being notified to UA.
+ */
+static void
+close_job_ring(struct sec_job_ring_t *job_ring)
+{
+	PMD_INIT_FUNC_TRACE();
+	if (job_ring->irq_fd) {
+		/* Producer index is frozen. If consumer index is not equal
+		 * with producer index, then we have descs to flush.
+		 */
+		while (job_ring->pidx != job_ring->cidx)
+			hw_flush_job_ring(job_ring, false, NULL);
+
+		/* free the uio job ring */
+		free_job_ring(job_ring->irq_fd);
+		job_ring->irq_fd = 0;
+		caam_jr_dma_free(job_ring->input_ring);
+		caam_jr_dma_free(job_ring->output_ring);
+		g_job_rings_no--;
+	}
+}
+
+/** @brief Release the software and hardware resources tied to a job ring.
+ * @param [in] job_ring The job ring
+ *
+ * @retval  0 for success
+ * @retval  -1 for error
+ */
+static int
+shutdown_job_ring(struct sec_job_ring_t *job_ring)
+{
+	int ret = 0;
+
+	PMD_INIT_FUNC_TRACE();
+	ASSERT(job_ring != NULL);
+	ret = hw_shutdown_job_ring(job_ring);
+	SEC_ASSERT(ret == 0, ret,
+		"Failed to shutdown hardware job ring %p",
+		job_ring);
+
+	if (job_ring->coalescing_en)
+		hw_job_ring_disable_coalescing(job_ring);
+
+	if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
+		ret = caam_jr_disable_irqs(job_ring->irq_fd);
+		SEC_ASSERT(ret == 0, ret,
+		"Failed to disable irqs for job ring %p",
+		job_ring);
+	}
+
+	return ret;
+}
+
+/*
+ * @brief Release the resources used by the SEC user space driver.
+ *
+ * Reset and release SEC's job rings indicated by the User Application at
+ * init_job_ring() and free any memory allocated internally.
+ * Call once during application tear down.
+ *
+ * @note In case there are any descriptors in-flight (descriptors received by
+ * SEC driver for processing and for which no response was yet provided to UA),
+ * the descriptors are discarded without any notifications to User Application.
+ *
+ * @retval ::0			is returned for a successful execution
+ * @retval ::-1		is returned if SEC driver release is in progress
+ */
+static int
+caam_jr_dev_uninit(struct rte_cryptodev *dev)
+{
+	struct sec_job_ring_t *internals;
+
+	PMD_INIT_FUNC_TRACE();
+	if (dev == NULL)
+		return -ENODEV;
+
+	internals = dev->data->dev_private;
+	rte_free(dev->security_ctx);
+
+	/* If any descriptors in flight , poll and wait
+	 * until all descriptors are received and silently discarded.
+	 */
+	if (internals) {
+		shutdown_job_ring(internals);
+		close_job_ring(internals);
+		rte_mempool_free(internals->ctx_pool);
+	}
+
+	CAAM_JR_INFO("Closing crypto device %s", dev->data->name);
+
+	/* last caam jr instance) */
+	if (g_job_rings_no == 0)
+		g_driver_state = SEC_DRIVER_STATE_IDLE;
+
+	return SEC_SUCCESS;
+}
+
+/* @brief Initialize the software and hardware resources tied to a job ring.
+ * @param [in] jr_mode;		Model to be used by SEC Driver to receive
+ *				notifications from SEC.  Can be either
+ *				of the three: #SEC_NOTIFICATION_TYPE_NAPI
+ *				#SEC_NOTIFICATION_TYPE_IRQ or
+ *				#SEC_NOTIFICATION_TYPE_POLL
+ * @param [in] NAPI_mode	The NAPI work mode to configure a job ring at
+ *				startup. Used only when #SEC_NOTIFICATION_TYPE
+ *				is set to #SEC_NOTIFICATION_TYPE_NAPI.
+ * @param [in] irq_coalescing_timer This value determines the maximum
+ *					amount of time after processing a
+ *					descriptor before raising an interrupt.
+ * @param [in] irq_coalescing_count This value determines how many
+ *					descriptors are completed before
+ *					raising an interrupt.
+ * @param [in] reg_base_addr,	The job ring base address register
+ * @param [in] irq_id		The job ring interrupt identification number.
+ * @retval  job_ring_handle for successful job ring configuration
+ * @retval  NULL on error
+ *
+ */
+static void *
+init_job_ring(void *reg_base_addr, uint32_t irq_id)
+{
+	struct sec_job_ring_t *job_ring = NULL;
+	int i, ret = 0;
+	int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
+	int napi_mode = 0;
+	int irq_coalescing_timer = 0;
+	int irq_coalescing_count = 0;
+
+	for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
+		if (g_job_rings[i].irq_fd == 0) {
+			job_ring = &g_job_rings[i];
+			g_job_rings_no++;
+			break;
+		}
+	}
+	if (job_ring == NULL) {
+		CAAM_JR_ERR("No free job ring\n");
+		return NULL;
+	}
+
+	job_ring->register_base_addr = reg_base_addr;
+	job_ring->jr_mode = jr_mode;
+	job_ring->napi_mode = 0;
+	job_ring->irq_fd = irq_id;
+
+	/* Allocate mem for input and output ring */
+
+	/* Allocate memory for input ring */
+	job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
+				SEC_DMA_MEM_INPUT_RING_SIZE);
+	memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);
+
+	/* Allocate memory for output ring */
+	job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
+				SEC_DMA_MEM_OUTPUT_RING_SIZE);
+	memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);
+
+	/* Reset job ring in SEC hw and configure job ring registers */
+	ret = hw_reset_job_ring(job_ring);
+	if (ret != 0) {
+		CAAM_JR_ERR("Failed to reset hardware job ring");
+		goto cleanup;
+	}
+
+	if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
+	/* When SEC US driver works in NAPI mode, the UA can select
+	 * if the driver starts with IRQs on or off.
+	 */
+		if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
+			CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
+				job_ring);
+			ret = caam_jr_enable_irqs(job_ring->irq_fd);
+			if (ret != 0) {
+				CAAM_JR_ERR("Failed to enable irqs for job ring");
+				goto cleanup;
+			}
+		}
+	} else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
+	/* When SEC US driver works in pure interrupt mode,
+	 * IRQ's are always enabled.
+	 */
+		CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
+			 job_ring);
+		ret = caam_jr_enable_irqs(job_ring->irq_fd);
+		if (ret != 0) {
+			CAAM_JR_ERR("Failed to enable irqs for job ring");
+			goto cleanup;
+		}
+	}
+	if (irq_coalescing_timer || irq_coalescing_count) {
+		hw_job_ring_set_coalescing_param(job_ring,
+			 irq_coalescing_timer,
+			 irq_coalescing_count);
+
+		hw_job_ring_enable_coalescing(job_ring);
+		job_ring->coalescing_en = 1;
+	}
+
+	job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
+	job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
+	job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;
+
+	return job_ring;
+cleanup:
+	caam_jr_dma_free(job_ring->output_ring);
+	caam_jr_dma_free(job_ring->input_ring);
+	return NULL;
+}
+
+
+static int
+caam_jr_dev_init(const char *name,
+		 struct rte_vdev_device *vdev,
+		 struct rte_cryptodev_pmd_init_params *init_params)
+{
+	struct rte_cryptodev *dev;
+	struct rte_security_ctx *security_instance;
+	struct uio_job_ring *job_ring;
+	char str[RTE_CRYPTODEV_NAME_MAX_LEN];
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* Validate driver state */
+	if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
+		g_job_rings_max = sec_configure();
+		if (!g_job_rings_max) {
+			CAAM_JR_ERR("No job ring detected on UIO !!!!");
+			return -1;
+		}
+		/* Update driver state */
+		g_driver_state = SEC_DRIVER_STATE_STARTED;
+	}
+
+	if (g_job_rings_no >= g_job_rings_max) {
+		CAAM_JR_ERR("No more job rings available max=%d!!!!",
+				g_job_rings_max);
+		return -1;
+	}
+
+	job_ring = config_job_ring();
+	if (job_ring == NULL) {
+		CAAM_JR_ERR("failed to create job ring");
+		goto init_error;
+	}
+
+	snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);
+
+	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
+	if (dev == NULL) {
+		CAAM_JR_ERR("failed to create cryptodev vdev");
+		goto cleanup;
+	}
+	/*TODO free it during teardown*/
+	dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
+						job_ring->uio_fd);
+
+	if (!dev->data->dev_private) {
+		CAAM_JR_ERR("Ring memory allocation failed\n");
+		goto cleanup2;
+	}
+
+	dev->driver_id = cryptodev_driver_id;
+	dev->dev_ops = &caam_jr_ops;
+
+	/* register rx/tx burst functions for data path */
+	dev->dequeue_burst = caam_jr_dequeue_burst;
+	dev->enqueue_burst = caam_jr_enqueue_burst;
+	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+			RTE_CRYPTODEV_FF_HW_ACCELERATED |
+			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
+			RTE_CRYPTODEV_FF_SECURITY |
+			RTE_CRYPTODEV_FF_IN_PLACE_SGL |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
+
+	/* For secondary processes, we don't initialise any further as primary
+	 * has already done this work. Only check we don't need a different
+	 * RX function
+	 */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		CAAM_JR_WARN("Device already init by primary process");
+		return 0;
+	}
+
+	/*TODO free it during teardown*/
+	security_instance = rte_malloc("caam_jr",
+				sizeof(struct rte_security_ctx), 0);
+	if (security_instance == NULL) {
+		CAAM_JR_ERR("memory allocation failed\n");
+		//todo error handling.
+		goto cleanup2;
+	}
+
+	security_instance->device = (void *)dev;
+	security_instance->ops = &caam_jr_security_ops;
+	security_instance->sess_cnt = 0;
+	dev->security_ctx = security_instance;
+
+	RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name);
+
+	return 0;
+
+cleanup2:
+	caam_jr_dev_uninit(dev);
+	rte_cryptodev_pmd_release_device(dev);
+cleanup:
+	free_job_ring(job_ring->uio_fd);
+init_error:
+	CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
+			init_params->name);
+
+	return -ENXIO;
+}
+
+/** Initialise CAAM JR crypto device */
+static int
+cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
+{
+	struct rte_cryptodev_pmd_init_params init_params = {
+		"",
+		sizeof(struct sec_job_ring_t),
+		rte_socket_id(),
+		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
+	};
+	const char *name;
+	const char *input_args;
+
+	name = rte_vdev_device_name(vdev);
+	if (name == NULL)
+		return -EINVAL;
+
+	input_args = rte_vdev_device_args(vdev);
+	rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
+
+	/* if sec device version is not configured */
+	if (!rta_get_sec_era()) {
+		const struct device_node *caam_node;
+
+		for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
+			const uint32_t *prop = of_get_property(caam_node,
+					"fsl,sec-era",
+					NULL);
+			if (prop) {
+				rta_set_sec_era(
+					INTL_SEC_ERA(cpu_to_caam32(*prop)));
+				break;
+			}
+		}
+	}
+#ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
+	if (rta_get_sec_era() > RTA_SEC_ERA_8) {
+		RTE_LOG(ERR, PMD,
+		"CAAM is compiled in BE mode for device with sec era > 8???\n");
+		return -EINVAL;
+	}
+#endif
+
+	return caam_jr_dev_init(name, vdev, &init_params);
+}
+
+/** Uninitialise CAAM JR crypto device */
+static int
+cryptodev_caam_jr_remove(struct rte_vdev_device *vdev)
+{
+	struct rte_cryptodev *cryptodev;
+	const char *name;
+
+	name = rte_vdev_device_name(vdev);
+	if (name == NULL)
+		return -EINVAL;
+
+	cryptodev = rte_cryptodev_pmd_get_named_dev(name);
+	if (cryptodev == NULL)
+		return -ENODEV;
+
+	caam_jr_dev_uninit(cryptodev);
+
+	return rte_cryptodev_pmd_destroy(cryptodev);
+}
+
+static struct rte_vdev_driver cryptodev_caam_jr_drv = {
+	.probe = cryptodev_caam_jr_probe,
+	.remove = cryptodev_caam_jr_remove
+};
+
+static struct cryptodev_driver caam_jr_crypto_drv;
+
+RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv);
+RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD,
+	"max_nb_queue_pairs=<int>"
+	"socket_id=<int>");
+RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver,
+		cryptodev_driver_id);
+
+RTE_INIT(caam_jr_init_log)
+{
+	caam_jr_logtype = rte_log_register("pmd.crypto.caam");
+	if (caam_jr_logtype >= 0)
+		rte_log_set_level(caam_jr_logtype, RTE_LOG_NOTICE);
+}
diff --git a/drivers/crypto/caam_jr/caam_jr_capabilities.c b/drivers/crypto/caam_jr/caam_jr_capabilities.c
new file mode 100644
index 00000000..c51593c4
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_capabilities.c
@@ -0,0 +1,266 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#include <caam_jr_capabilities.h>
+
+static const struct rte_cryptodev_capabilities caam_jr_capabilities[] = {
+	{	/* MD5 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_MD5_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 1,
+					.max = 64,
+					.increment = 1
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 16,
+					.increment = 1
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* SHA1 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 1,
+					.max = 64,
+					.increment = 1
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 20,
+					.increment = 1
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* SHA224 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 1,
+					.max = 64,
+					.increment = 1
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 28,
+					.increment = 1
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* SHA256 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 1,
+					.max = 64,
+					.increment = 1
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 32,
+					.increment = 1
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* SHA384 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
+				.block_size = 128,
+				.key_size = {
+					.min = 1,
+					.max = 128,
+					.increment = 1
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 48,
+					.increment = 1
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* SHA512 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
+				.block_size = 128,
+				.key_size = {
+					.min = 1,
+					.max = 128,
+					.increment = 1
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 64,
+					.increment = 1
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* AES GCM */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+			{.aead = {
+				.algo = RTE_CRYPTO_AEAD_AES_GCM,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 8
+				},
+				.digest_size = {
+					.min = 8,
+					.max = 16,
+					.increment = 4
+				},
+				.aad_size = {
+					.min = 0,
+					.max = 240,
+					.increment = 1
+				},
+				.iv_size = {
+					.min = 12,
+					.max = 12,
+					.increment = 0
+				},
+			}, }
+		}, }
+	},
+	{	/* AES CBC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_CBC,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 8
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* AES CTR */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_CTR,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 8
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* 3DES CBC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_3DES_CBC,
+				.block_size = 8,
+				.key_size = {
+					.min = 16,
+					.max = 24,
+					.increment = 8
+				},
+				.iv_size = {
+					.min = 8,
+					.max = 8,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+
+	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+static const struct rte_security_capability caam_jr_security_cap[] = {
+	{ /* IPsec Lookaside Protocol offload ESP Transport Egress */
+		.action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
+		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+		.ipsec = {
+			.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+			.mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
+			.direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
+			.options = { 0 }
+		},
+		.crypto_capabilities = caam_jr_capabilities
+	},
+	{ /* IPsec Lookaside Protocol offload ESP Tunnel Ingress */
+		.action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
+		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+		.ipsec = {
+			.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+			.mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
+			.direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
+			.options = { 0 }
+		},
+		.crypto_capabilities = caam_jr_capabilities
+	},
+	{
+		.action = RTE_SECURITY_ACTION_TYPE_NONE
+	}
+};
+
+const struct rte_cryptodev_capabilities *
+caam_jr_get_cryptodev_capabilities(void)
+{
+	return caam_jr_capabilities;
+}
+
+const struct rte_security_capability *
+caam_jr_get_security_capabilities(void *device __rte_unused)
+{
+	return caam_jr_security_cap;
+}
diff --git a/drivers/crypto/caam_jr/caam_jr_capabilities.h b/drivers/crypto/caam_jr/caam_jr_capabilities.h
new file mode 100644
index 00000000..c1e3f305
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_capabilities.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef CAAM_JR_CAPABILITIES_H
+#define CAAM_JR_CAPABILITIES_H
+
+#include <rte_cryptodev.h>
+#include <rte_security.h>
+
+/* Get cryptodev capabilities */
+const struct rte_cryptodev_capabilities *
+caam_jr_get_cryptodev_capabilities(void);
+/* Get security capabilities */
+const struct rte_security_capability *
+caam_jr_get_security_capabilities(void *device);
+
+#endif
diff --git a/drivers/crypto/caam_jr/caam_jr_config.h b/drivers/crypto/caam_jr/caam_jr_config.h
new file mode 100644
index 00000000..041187a8
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_config.h
@@ -0,0 +1,207 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef CAAM_JR_CONFIG_H
+#define CAAM_JR_CONFIG_H
+
+#include <rte_byteorder.h>
+
+#include <compat.h>
+
+#ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
+#define CAAM_BYTE_ORDER __BIG_ENDIAN
+#else
+#define CAAM_BYTE_ORDER __LITTLE_ENDIAN
+#endif
+
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+#define CORE_BYTE_ORDER __BIG_ENDIAN
+#else
+#define CORE_BYTE_ORDER __LITTLE_ENDIAN
+#endif
+
+#if CORE_BYTE_ORDER != CAAM_BYTE_ORDER
+
+#define cpu_to_caam64 rte_cpu_to_be_64
+#define cpu_to_caam32 rte_cpu_to_be_32
+#else
+#define cpu_to_caam64
+#define cpu_to_caam32
+
+#endif
+
+/*
+ * SEC is configured to start work in polling mode,
+ * when configured for NAPI notification style.
+ */
+#define SEC_STARTUP_POLLING_MODE     0
+/*
+ * SEC is configured to start work in interrupt mode,
+ * when configured for NAPI notification style.
+ */
+#define SEC_STARTUP_INTERRUPT_MODE   1
+
+/*
+ * SEC driver will use NAPI model to receive notifications
+ * for processed packets from SEC engine hardware:
+ * - IRQ for low traffic
+ * - polling for high traffic.
+ */
+#define SEC_NOTIFICATION_TYPE_NAPI  0
+/*
+ * SEC driver will use ONLY interrupts to receive notifications
+ * for processed packets from SEC engine hardware.
+ */
+#define SEC_NOTIFICATION_TYPE_IRQ   1
+/*
+ * SEC driver will use ONLY polling to receive notifications
+ * for processed packets from SEC engine hardware.
+ */
+#define SEC_NOTIFICATION_TYPE_POLL  2
+
+/*
+ * SEC USER SPACE DRIVER related configuration.
+ */
+
+/*
+ * Determines how SEC user space driver will receive notifications
+ * for processed packets from SEC engine.
+ * Valid values are: #SEC_NOTIFICATION_TYPE_POLL, #SEC_NOTIFICATION_TYPE_IRQ
+ * and #SEC_NOTIFICATION_TYPE_NAPI.
+ */
+#define SEC_NOTIFICATION_TYPE   SEC_NOTIFICATION_TYPE_POLL
+
+/* Maximum number of job rings supported by SEC hardware */
+#define MAX_SEC_JOB_RINGS         4
+
+/* Maximum number of QP per job ring */
+#define RTE_CAAM_MAX_NB_SEC_QPS    1
+
+/*
+ * Size of cryptographic context that is used directly in communicating
+ * with SEC device. SEC device works only with physical addresses. This
+ * is the maximum size for a SEC descriptor ( = 64 words).
+ */
+#define SEC_CRYPTO_DESCRIPTOR_SIZE  256
+
+/*
+ * Size of job descriptor submitted to SEC device for each packet to
+ * be processed.
+ * Job descriptor contains 3 DMA address pointers:
+ *	- to shared descriptor, to input buffer and to output buffer.
+ * The job descriptor contains other SEC specific commands as well:
+ *	- HEADER command, SEQ IN PTR command SEQ OUT PTR command and opaque data
+ *      each measuring 4 bytes.
+ * Job descriptor size, depending on physical address representation:
+ *	- 32 bit - size is 28 bytes - cacheline-aligned size is 64 bytes
+ *	- 36 bit - size is 40 bytes - cacheline-aligned size is 64 bytes
+ * @note: Job descriptor must be cacheline-aligned to ensure efficient
+ *	memory access.
+ * @note: If other format is used for job descriptor, then the size must be
+ *	revised.
+ */
+#define SEC_JOB_DESCRIPTOR_SIZE     64
+
+/*
+ * Size of one entry in the input ring of a job ring.
+ * Input ring contains pointers to job descriptors.
+ * The memory used for an input ring and output ring must be physically
+ * contiguous.
+ */
+#define SEC_JOB_INPUT_RING_ENTRY_SIZE  sizeof(dma_addr_t)
+
+/*
+ * Size of one entry in the output ring of a job ring.
+ * Output ring entry is a pointer to a job descriptor followed by a 4 byte
+ * status word.
+ * The memory used for an input ring and output ring must be physically
+ * contiguous.
+ * @note If desired to use also the optional SEQ OUT indication in output ring
+ * entries,
+ * then 4 more bytes must be added to the size.
+ */
+#define SEC_JOB_OUTPUT_RING_ENTRY_SIZE  (SEC_JOB_INPUT_RING_ENTRY_SIZE + 4)
+
+/*
+ * DMA memory required for an input ring of a job ring.
+ */
+#define SEC_DMA_MEM_INPUT_RING_SIZE     ((SEC_JOB_INPUT_RING_ENTRY_SIZE) * \
+					(SEC_JOB_RING_SIZE))
+
+/*
+ * DMA memory required for an output ring of a job ring.
+ *  Required extra 4 byte for status word per each entry.
+ */
+#define SEC_DMA_MEM_OUTPUT_RING_SIZE    ((SEC_JOB_OUTPUT_RING_ENTRY_SIZE) * \
+					(SEC_JOB_RING_SIZE))
+
+/* DMA memory required for a job ring, including both input and output rings. */
+#define SEC_DMA_MEM_JOB_RING_SIZE       ((SEC_DMA_MEM_INPUT_RING_SIZE) + \
+					(SEC_DMA_MEM_OUTPUT_RING_SIZE))
+
+/*
+ * When calling sec_init() UA will provide an area of virtual memory
+ *  of size #SEC_DMA_MEMORY_SIZE to be  used internally by the driver
+ *  to allocate data (like SEC descriptors) that needs to be passed to
+ *  SEC device in physical addressing and later on retrieved from SEC device.
+ *  At initialization the UA provides specialized ptov/vtop functions/macros to
+ *  translate addresses allocated from this memory area.
+ */
+#define SEC_DMA_MEMORY_SIZE          ((SEC_DMA_MEM_JOB_RING_SIZE) * \
+					(MAX_SEC_JOB_RINGS))
+
+#define L1_CACHE_BYTES 64
+
+/* SEC JOB RING related configuration. */
+
+/*
+ * Configure the size of the JOB RING.
+ * The maximum size of the ring in hardware limited to 1024.
+ * However the number of packets in flight in a time interval of 1ms can
+ * be calculated from the traffic rate (Mbps) and packet size.
+ * Here it was considered a packet size of 64 bytes.
+ *
+ * @note Round up to nearest power of 2 for optimized update
+ * of producer/consumer indexes of each job ring
+ */
+#define SEC_JOB_RING_SIZE     512
+
+/*
+ * Interrupt coalescing related configuration.
+ * NOTE: SEC hardware enabled interrupt
+ * coalescing is not supported on SEC version 3.1!
+ * SEC version 4.4 has support for interrupt
+ * coalescing.
+ */
+
+#if SEC_NOTIFICATION_TYPE != SEC_NOTIFICATION_TYPE_POLL
+
+#define SEC_INT_COALESCING_ENABLE   1
+/*
+ * Interrupt Coalescing Descriptor Count Threshold.
+ * While interrupt coalescing is enabled (ICEN=1), this value determines
+ * how many Descriptors are completed before raising an interrupt.
+ *
+ * Valid values for this field are from 0 to 255.
+ * Note that a value of 1 functionally defeats the advantages of interrupt
+ * coalescing since the threshold value is reached each time that a
+ * Job Descriptor is completed. A value of 0 is treated in the same
+ * manner as a value of 1.
+ */
+#define SEC_INTERRUPT_COALESCING_DESCRIPTOR_COUNT_THRESH  10
+
+/*
+ * Interrupt Coalescing Timer Threshold.
+ * While interrupt coalescing is enabled (ICEN=1), this value determines the
+ * maximum amount of time after processing a Descriptor before raising an
+ * interrupt.
+ * The threshold value is represented in units equal to 64 CAAM interface
+ * clocks. Valid values for this field are from 1 to 65535.
+ * A value of 0 results in behavior identical to that when interrupt
+ * coalescing is disabled.
+ */
+#define SEC_INTERRUPT_COALESCING_TIMER_THRESH  100
+#endif /* SEC_NOTIFICATION_TYPE_POLL */
+
+#endif /* CAAM_JR_CONFIG_H */
diff --git a/drivers/crypto/caam_jr/caam_jr_desc.h b/drivers/crypto/caam_jr/caam_jr_desc.h
new file mode 100644
index 00000000..6683ea83
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_desc.h
@@ -0,0 +1,285 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef CAAM_JR_DESC_H
+#define CAAM_JR_DESC_H
+
+#define CMD_HDR_CTYPE_SD		0x16
+#define CMD_HDR_CTYPE_JD		0x17
+
+/* The maximum size of a SEC descriptor, in WORDs (32 bits). */
+#define MAX_DESC_SIZE_WORDS                     64
+
+/*
+ * Macros manipulating descriptors
+ */
+/* Macro for setting the SD pointer in a JD. Common for all protocols
+ * supported by the SEC driver.
+ */
+#define SEC_JD_SET_SD(descriptor, ptr, len)	   {	  \
+	(descriptor)->sd_ptr = (ptr);			       \
+	(descriptor)->deschdr.command.jd.shr_desc_len = (len);      \
+}
+
+/* Macro for setting a pointer to the job which this descriptor processes.
+ * It eases the lookup procedure for identifying the descriptor that has
+ * completed.
+ */
+#define SEC_JD_SET_JOB_PTR(descriptor, ptr) \
+	((descriptor)->job_ptr = (ptr))
+
+/* Macro for setting up a JD. The structure of the JD is common across all
+ * supported protocols, thus its structure is identical.
+ */
+#define SEC_JD_INIT(descriptor)	      ({ \
+	/* CTYPE = job descriptor			       \
+	 * RSMS, DNR = 0
+	 * ONE = 1
+	 * Start Index = 0
+	 * ZRO,TD, MTD = 0
+	 * SHR = 1 (there's a shared descriptor referenced
+	 *	  by this job descriptor,pointer in next word)
+	 * REO = 1 (execute job descr. first, shared descriptor
+	 *	  after)
+	 * SHARE = DEFER
+	 * Descriptor Length = 0 ( to be completed @ runtime ) */ \
+	(descriptor)->deschdr.command.word = 0xB0801C0D;	\
+	/*
+	 * CTYPE = SEQ OUT command * Scater Gather Flag = 0
+	 * (can be updated @ runtime) PRE = 0 * EXT = 1
+	 * (data length is in next word, following the * command)
+	 * RTO = 0 */						\
+	(descriptor)->seq_out.command.word = 0xF8400000; /**/	\
+	/*
+	 * CTYPE = SEQ IN command
+	 * Scater Gather Flag = 0 (can be updated @ runtime)
+	 * PRE = 0
+	 * EXT = 1 ( data length is in next word, following the
+	 *	   command)
+	 * RTO = 0 */						\
+	(descriptor)->seq_in.command.word  = 0xF0400000; /**/	\
+	/*
+	 * In order to be compatible with QI scenarios, the DPOVRD value
+	 * loaded must be formated like this:
+	 * DPOVRD_EN (1b) | Res| DPOVRD Value (right aligned). */ \
+	(descriptor)->load_dpovrd.command.word = 0x16870004;	\
+	/* By default, DPOVRD mechanism is disabled, thus the value to be
+	 * LOAD-ed through the above descriptor command will be
+	 * 0x0000_0000. */					\
+	(descriptor)->dpovrd = 0x00000000;			\
+})
+
+/* Macro for setting the pointer to the input buffer in the JD, according to
+ * the parameters set by the user in the ::sec_packet_t structure.
+ */
+#define SEC_JD_SET_IN_PTR(descriptor, phys_addr, offset, length) {     \
+	(descriptor)->seq_in_ptr = (phys_addr) + (offset);	      \
+	(descriptor)->in_ext_length = (length);			 \
+}
+
+/* Macro for setting the pointer to the output buffer in the JD, according to
+ * the parameters set by the user in the ::sec_packet_t structure.
+ */
+#define SEC_JD_SET_OUT_PTR(descriptor, phys_addr, offset, length) {    \
+	(descriptor)->seq_out_ptr = (phys_addr) + (offset);	     \
+	(descriptor)->out_ext_length = (length);			\
+}
+
+/* Macro for setting the Scatter-Gather flag in the SEQ IN command. Used in
+ * case the input buffer is split in multiple buffers, according to the user
+ * specification.
+ */
+#define SEC_JD_SET_SG_IN(descriptor) \
+	((descriptor)->seq_in.command.field.sgf =  1)
+
+/* Macro for setting the Scatter-Gather flag in the SEQ OUT command. Used in
+ * case the output buffer is split in multiple buffers, according to the user
+ * specification.
+ */
+#define SEC_JD_SET_SG_OUT(descriptor) \
+	((descriptor)->seq_out.command.field.sgf = 1)
+
+#define SEC_JD_SET_DPOVRD(descriptor) \
+
+/* Macro for retrieving a descriptor's length. Works for both SD and JD. */
+#define SEC_GET_DESC_LEN(descriptor)					\
+	(((struct descriptor_header_s *)(descriptor))->command.sd.ctype == \
+	CMD_HDR_CTYPE_SD ? ((struct descriptor_header_s *) \
+	(descriptor))->command.sd.desclen :	\
+	((struct descriptor_header_s *)(descriptor))->command.jd.desclen)
+
+/* Helper macro for dumping the hex representation of a descriptor */
+#define SEC_DUMP_DESC(descriptor) {					\
+	int __i;							\
+	CAAM_JR_INFO("Des@ 0x%08x\n", (uint32_t)((uint32_t *)(descriptor)));\
+	for (__i = 0;						\
+		__i < SEC_GET_DESC_LEN(descriptor);			\
+		__i++) {						\
+		printf("0x%08x: 0x%08x\n",			\
+			(uint32_t)(((uint32_t *)(descriptor)) + __i),	\
+			*(((uint32_t *)(descriptor)) + __i));		\
+	}								\
+}
+/* Union describing a descriptor header.
+ */
+struct descriptor_header_s {
+	union {
+		uint32_t word;
+		struct {
+			/* 4  */ unsigned int ctype:5;
+			/* 5  */ unsigned int res1:2;
+			/* 7  */ unsigned int dnr:1;
+			/* 8  */ unsigned int one:1;
+			/* 9  */ unsigned int res2:1;
+			/* 10 */ unsigned int start_idx:6;
+			/* 16 */ unsigned int res3:2;
+			/* 18 */ unsigned int cif:1;
+			/* 19 */ unsigned int sc:1;
+			/* 20 */ unsigned int pd:1;
+			/* 21 */ unsigned int res4:1;
+			/* 22 */ unsigned int share:2;
+			/* 24 */ unsigned int res5:2;
+			/* 26 */ unsigned int desclen:6;
+		} sd;
+		struct {
+			/* TODO only below struct members are corrected,
+			 * all others also need to be reversed please verify it
+			 */
+			/* 0 */ unsigned int desclen:7;
+			/* 7 */ unsigned int res4:1;
+			/* 8 */ unsigned int share:3;
+			/* 11 */ unsigned int reo:1;
+			/* 12 */ unsigned int shr:1;
+			/* 13 */ unsigned int mtd:1;
+			/* 14 */ unsigned int td:1;
+			/* 15 */ unsigned int zero:1;
+			/* 16 */ unsigned int shr_desc_len:6;
+			/* 22  */ unsigned int res2:1;
+			/* 23  */ unsigned int one:1;
+			/* 24  */ unsigned int dnr:1;
+			/* 25  */ unsigned int rsms:1;
+			/* 26  */ unsigned int res1:1;
+			/* 27  */ unsigned int ctype:5;
+		} jd;
+	} __rte_packed command;
+} __rte_packed;
+
+/* Union describing a KEY command in a descriptor.
+ */
+struct key_command_s {
+	union {
+		uint32_t word;
+		struct {
+			unsigned int ctype:5;
+			unsigned int cls:2;
+			unsigned int sgf:1;
+			unsigned int imm:1;
+			unsigned int enc:1;
+			unsigned int nwb:1;
+			unsigned int ekt:1;
+			unsigned int kdest:4;
+			unsigned int tk:1;
+			unsigned int rsvd1:5;
+			unsigned int length:10;
+		} __rte_packed field;
+	} __rte_packed command;
+} __rte_packed;
+
+/* Union describing a PROTOCOL command
+ * in a descriptor.
+ */
+struct protocol_operation_command_s {
+	union {
+		uint32_t word;
+		struct {
+			unsigned int ctype:5;
+			unsigned int optype:3;
+			unsigned char protid;
+			unsigned short protinfo;
+		} __rte_packed field;
+	} __rte_packed command;
+} __rte_packed;
+
+/* Union describing a SEQIN command in a
+ * descriptor.
+ */
+struct seq_in_command_s {
+	union {
+		uint32_t word;
+		struct {
+			unsigned int ctype:5;
+			unsigned int res1:1;
+			unsigned int inl:1;
+			unsigned int sgf:1;
+			unsigned int pre:1;
+			unsigned int ext:1;
+			unsigned int rto:1;
+			unsigned int rjd:1;
+			unsigned int res2:4;
+			unsigned int length:16;
+		} field;
+	} __rte_packed command;
+} __rte_packed;
+
+/* Union describing a SEQOUT command in a
+ * descriptor.
+ */
+struct seq_out_command_s {
+	union {
+		uint32_t word;
+		struct {
+			unsigned int ctype:5;
+			unsigned int res1:2;
+			unsigned int sgf:1;
+			unsigned int pre:1;
+			unsigned int ext:1;
+			unsigned int rto:1;
+			unsigned int res2:5;
+			unsigned int length:16;
+		} field;
+	} __rte_packed command;
+} __rte_packed;
+
+struct load_command_s {
+	union {
+		uint32_t word;
+		struct {
+			unsigned int ctype:5;
+			unsigned int class:2;
+			unsigned int sgf:1;
+			unsigned int imm:1;
+			unsigned int dst:7;
+			unsigned char offset;
+			unsigned char length;
+		} fields;
+	} __rte_packed command;
+} __rte_packed;
+
+/* Structure encompassing a general shared descriptor of maximum
+ * size (64 WORDs). Usually, other specific shared descriptor structures
+ * will be type-casted to this one
+ * this one.
+ */
+struct sec_sd_t {
+	uint32_t rsvd[MAX_DESC_SIZE_WORDS];
+} __attribute__((packed, aligned(64)));
+
+/* Structure encompassing a job descriptor which processes
+ * a single packet from a context. The job descriptor references
+ * a shared descriptor from a SEC context.
+ */
+struct sec_job_descriptor_t {
+	struct descriptor_header_s deschdr;
+	dma_addr_t sd_ptr;
+	struct seq_out_command_s seq_out;
+	dma_addr_t seq_out_ptr;
+	uint32_t out_ext_length;
+	struct seq_in_command_s seq_in;
+	dma_addr_t seq_in_ptr;
+	uint32_t in_ext_length;
+	struct load_command_s load_dpovrd;
+	uint32_t dpovrd;
+} __attribute__((packed, aligned(64)));
+
+#endif
diff --git a/drivers/crypto/caam_jr/caam_jr_hw.c b/drivers/crypto/caam_jr/caam_jr_hw.c
new file mode 100644
index 00000000..4a2b0899
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_hw.c
@@ -0,0 +1,367 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <rte_common.h>
+#include <rte_memory.h>
+#include <rte_malloc.h>
+#include <rte_crypto.h>
+#include <rte_security.h>
+
+#include <caam_jr_config.h>
+#include <caam_jr_hw_specific.h>
+#include <caam_jr_pvt.h>
+#include <caam_jr_log.h>
+
+/* RTA header files */
+#include <hw/desc/common.h>
+#include <hw/desc/algo.h>
+#include <hw/desc/ipsec.h>
+
+/* Used to retry resetting a job ring in SEC hardware. */
+#define SEC_TIMEOUT 100000
+
+/* @brief Process Jump Halt Condition related errors
+ *
+ * @param [in]  error_code        The error code in the descriptor status word
+ */
+static inline void
+hw_handle_jmp_halt_cond_err(union hw_error_code error_code)
+{
+	CAAM_JR_DEBUG("JMP: %d, Descriptor Index: 0x%x, Condition: 0x%x",
+			error_code.error_desc.jmp_halt_cond_src.jmp,
+			error_code.error_desc.jmp_halt_cond_src.desc_idx,
+			error_code.error_desc.jmp_halt_cond_src.cond);
+	(void)error_code;
+}
+
+/* @brief Process DECO related errors
+ *
+ * @param [in]  error_code        The error code in the descriptor status word
+ */
+static inline void
+hw_handle_deco_err(union hw_error_code error_code)
+{
+	CAAM_JR_DEBUG("JMP: %d, Descriptor Index: 0x%x",
+			error_code.error_desc.deco_src.jmp,
+			error_code.error_desc.deco_src.desc_idx);
+
+	switch (error_code.error_desc.deco_src.desc_err) {
+	case SEC_HW_ERR_DECO_HFN_THRESHOLD:
+		CAAM_JR_DEBUG(" Warning: Descriptor completed normally,"
+			"but 3GPP HFN matches or exceeds the Threshold ");
+		break;
+	default:
+		CAAM_JR_DEBUG("Error 0x%04x not implemented",
+				error_code.error_desc.deco_src.desc_err);
+		break;
+	}
+}
+
+/* @brief Process  Jump Halt User Status related errors
+ *
+ * @param [in]  error_code        The error code in the descriptor status word
+ */
+static inline void
+hw_handle_jmp_halt_user_err(union hw_error_code error_code __rte_unused)
+{
+	CAAM_JR_DEBUG(" Not implemented");
+}
+
+/* @brief Process CCB related errors
+ *
+ * @param [in]  error_code        The error code in the descriptor status word
+ */
+static inline void
+hw_handle_ccb_err(union hw_error_code hw_error_code __rte_unused)
+{
+	CAAM_JR_DEBUG(" Not implemented");
+}
+
+/* @brief Process Job Ring related errors
+ *
+ * @param [in]  error_code        The error code in the descriptor status word
+ */
+static inline void
+hw_handle_jr_err(union hw_error_code hw_error_code __rte_unused)
+{
+	CAAM_JR_DEBUG(" Not implemented");
+}
+
+int
+hw_reset_job_ring(struct sec_job_ring_t *job_ring)
+{
+	int ret = 0;
+
+	ASSERT(job_ring->register_base_addr != NULL);
+
+	/* First reset the job ring in hw */
+	ret = hw_shutdown_job_ring(job_ring);
+	SEC_ASSERT(ret == 0, ret, "Failed resetting job ring in hardware");
+
+	/* In order to have the HW JR in a workable state
+	 * after a reset, I need to re-write the input
+	 * queue size, input start address, output queue
+	 * size and output start address
+	 */
+	/* Write the JR input queue size to the HW register */
+	hw_set_input_ring_size(job_ring, SEC_JOB_RING_SIZE);
+
+	/* Write the JR output queue size to the HW register */
+	hw_set_output_ring_size(job_ring, SEC_JOB_RING_SIZE);
+
+	/* Write the JR input queue start address */
+	hw_set_input_ring_start_addr(job_ring,
+			caam_jr_dma_vtop(job_ring->input_ring));
+	CAAM_JR_DEBUG(" Set input ring base address to : Virtual: 0x%" PRIx64
+		      ",Physical: 0x%" PRIx64 ", Read from HW: 0x%" PRIx64,
+		      (uint64_t)(uintptr_t)job_ring->input_ring,
+		      caam_jr_dma_vtop(job_ring->input_ring),
+		      hw_get_inp_queue_base(job_ring));
+
+	/* Write the JR output queue start address */
+	hw_set_output_ring_start_addr(job_ring,
+			caam_jr_dma_vtop(job_ring->output_ring));
+	CAAM_JR_DEBUG(" Set output ring base address to: Virtual: 0x%" PRIx64
+		      ",Physical: 0x%" PRIx64 ", Read from HW: 0x%" PRIx64,
+		      (uint64_t)(uintptr_t)job_ring->output_ring,
+		      caam_jr_dma_vtop(job_ring->output_ring),
+		      hw_get_out_queue_base(job_ring));
+	return ret;
+}
+
+int
+hw_shutdown_job_ring(struct sec_job_ring_t *job_ring)
+{
+	unsigned int timeout = SEC_TIMEOUT;
+	uint32_t tmp = 0;
+	int usleep_interval = 10;
+
+	if (job_ring->register_base_addr == NULL) {
+		CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+			job_ring);
+		return 0;
+	}
+
+	CAAM_JR_INFO("Resetting Job ring %p", job_ring);
+
+	/*
+	 * Mask interrupts since we are going to poll
+	 * for reset completion status
+	 * Also, at POR, interrupts are ENABLED on a JR, thus
+	 * this is the point where I can disable them without
+	 * changing the code logic too much
+	 */
+	caam_jr_disable_irqs(job_ring->irq_fd);
+
+	/* initiate flush (required prior to reset) */
+	SET_JR_REG(JRCR, job_ring, JR_REG_JRCR_VAL_RESET);
+
+	/* dummy read */
+	tmp = GET_JR_REG(JRCR, job_ring);
+
+	do {
+		tmp = GET_JR_REG(JRINT, job_ring);
+		usleep(usleep_interval);
+	} while (((tmp & JRINT_ERR_HALT_MASK) ==
+			JRINT_ERR_HALT_INPROGRESS) && --timeout);
+
+	CAAM_JR_INFO("JRINT is %x", tmp);
+	if ((tmp & JRINT_ERR_HALT_MASK) != JRINT_ERR_HALT_COMPLETE ||
+		timeout == 0) {
+		CAAM_JR_ERR("0x%x, %d", tmp, timeout);
+		/* unmask interrupts */
+		if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL)
+			caam_jr_enable_irqs(job_ring->irq_fd);
+		return -1;
+	}
+
+	/* Initiate reset */
+	timeout = SEC_TIMEOUT;
+	SET_JR_REG(JRCR, job_ring, JR_REG_JRCR_VAL_RESET);
+
+	do {
+		tmp = GET_JR_REG(JRCR, job_ring);
+		usleep(usleep_interval);
+	} while ((tmp & JR_REG_JRCR_VAL_RESET) && --timeout);
+
+	CAAM_JR_DEBUG("JRCR is %x", tmp);
+	if (timeout == 0) {
+		CAAM_JR_ERR("Failed to reset hw job ring %p", job_ring);
+		/* unmask interrupts */
+		if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL)
+			caam_jr_enable_irqs(job_ring->irq_fd);
+		return -1;
+	}
+	/* unmask interrupts */
+	if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL)
+		caam_jr_enable_irqs(job_ring->irq_fd);
+	return 0;
+
+}
+
+void
+hw_handle_job_ring_error(struct sec_job_ring_t *job_ring __rte_unused,
+			 uint32_t error_code)
+{
+	union hw_error_code hw_err_code;
+
+	hw_err_code.error = error_code;
+	switch (hw_err_code.error_desc.value.ssrc) {
+	case SEC_HW_ERR_SSRC_NO_SRC:
+		ASSERT(hw_err_code.error_desc.no_status_src.res == 0);
+		CAAM_JR_ERR("No Status Source ");
+		break;
+	case SEC_HW_ERR_SSRC_CCB_ERR:
+		CAAM_JR_ERR("CCB Status Source");
+		hw_handle_ccb_err(hw_err_code);
+		break;
+	case SEC_HW_ERR_SSRC_JMP_HALT_U:
+		CAAM_JR_ERR("Jump Halt User Status Source");
+		hw_handle_jmp_halt_user_err(hw_err_code);
+		break;
+	case SEC_HW_ERR_SSRC_DECO:
+		CAAM_JR_ERR("DECO Status Source");
+		hw_handle_deco_err(hw_err_code);
+		break;
+	case SEC_HW_ERR_SSRC_JR:
+		CAAM_JR_ERR("Job Ring Status Source");
+		hw_handle_jr_err(hw_err_code);
+		break;
+	case SEC_HW_ERR_SSRC_JMP_HALT_COND:
+		CAAM_JR_ERR("Jump Halt Condition Codes");
+		hw_handle_jmp_halt_cond_err(hw_err_code);
+		break;
+	default:
+		ASSERT(0);
+		CAAM_JR_ERR("Unknown SSRC");
+		break;
+	}
+}
+
+void
+hw_job_ring_error_print(struct sec_job_ring_t *job_ring, int code)
+{
+	switch (code) {
+	case JRINT_ERR_WRITE_STATUS:
+		CAAM_JR_ERR("Error writing status to Output Ring ");
+		break;
+	case JRINT_ERR_BAD_INPUT_BASE:
+		CAAM_JR_ERR(
+		"Bad Input Ring Base (%p) (not on a 4-byte boundary) ",
+		(void *)job_ring);
+		break;
+	case JRINT_ERR_BAD_OUTPUT_BASE:
+		CAAM_JR_ERR(
+		"Bad Output Ring Base (%p) (not on a 4-byte boundary) ",
+		(void *)job_ring);
+		break;
+	case JRINT_ERR_WRITE_2_IRBA:
+		CAAM_JR_ERR(
+		"Invalid write to Input Ring Base Address Register ");
+		break;
+	case JRINT_ERR_WRITE_2_ORBA:
+		CAAM_JR_ERR(
+		"Invalid write to Output Ring Base Address Register ");
+		break;
+	case JRINT_ERR_RES_B4_HALT:
+		CAAM_JR_ERR(
+		"Job Ring [%p] released before Job Ring is halted",
+		(void *)job_ring);
+		break;
+	case JRINT_ERR_REM_TOO_MANY:
+		CAAM_JR_ERR("Removed too many jobs from job ring [%p]",
+			(void *)job_ring);
+		break;
+	case JRINT_ERR_ADD_TOO_MANY:
+		CAAM_JR_ERR("Added too many jobs on job ring [%p]", job_ring);
+		break;
+	default:
+		CAAM_JR_ERR(" Unknown SEC JR Error :%d",
+				code);
+		break;
+	}
+}
+
+int
+hw_job_ring_set_coalescing_param(struct sec_job_ring_t *job_ring,
+				 uint16_t irq_coalescing_timer,
+				 uint8_t irq_coalescing_count)
+{
+	uint32_t reg_val = 0;
+
+	ASSERT(job_ring != NULL);
+	if (job_ring->register_base_addr == NULL) {
+		CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+			job_ring);
+		return -1;
+	}
+	/* Set descriptor count coalescing */
+	reg_val |= (irq_coalescing_count << JR_REG_JRCFG_LO_ICDCT_SHIFT);
+
+	/* Set coalescing timer value */
+	reg_val |= (irq_coalescing_timer << JR_REG_JRCFG_LO_ICTT_SHIFT);
+
+	/* Update parameters in HW */
+	SET_JR_REG_LO(JRCFG, job_ring, reg_val);
+	CAAM_JR_DEBUG("Set coalescing params on jr %p timer:%d, desc count: %d",
+			job_ring, irq_coalescing_timer, irq_coalescing_timer);
+
+	return 0;
+}
+
+int
+hw_job_ring_enable_coalescing(struct sec_job_ring_t *job_ring)
+{
+	uint32_t reg_val = 0;
+
+	ASSERT(job_ring != NULL);
+	if (job_ring->register_base_addr == NULL) {
+		CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+			job_ring);
+		return -1;
+	}
+
+	/* Get the current value of the register */
+	reg_val = GET_JR_REG_LO(JRCFG, job_ring);
+
+	/* Enable coalescing */
+	reg_val |= JR_REG_JRCFG_LO_ICEN_EN;
+
+	/* Write in hw */
+	SET_JR_REG_LO(JRCFG, job_ring, reg_val);
+
+	CAAM_JR_DEBUG("Enabled coalescing on jr %p ",
+			job_ring);
+
+	return 0;
+}
+
+int
+hw_job_ring_disable_coalescing(struct sec_job_ring_t *job_ring)
+{
+	uint32_t reg_val = 0;
+
+	ASSERT(job_ring != NULL);
+
+	if (job_ring->register_base_addr == NULL) {
+		CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+			job_ring);
+		return -1;
+	}
+
+	/* Get the current value of the register */
+	reg_val = GET_JR_REG_LO(JRCFG, job_ring);
+
+	/* Disable coalescing */
+	reg_val &= ~JR_REG_JRCFG_LO_ICEN_EN;
+
+	/* Write in hw */
+	SET_JR_REG_LO(JRCFG, job_ring, reg_val);
+	CAAM_JR_DEBUG("Disabled coalescing on jr %p ", job_ring);
+
+	return 0;
+}
diff --git a/drivers/crypto/caam_jr/caam_jr_hw_specific.h b/drivers/crypto/caam_jr/caam_jr_hw_specific.h
new file mode 100644
index 00000000..5f58a585
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_hw_specific.h
@@ -0,0 +1,503 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 NXP
+ */
+
+#ifndef CAAM_JR_HW_SPECIFIC_H
+#define CAAM_JR_HW_SPECIFIC_H
+
+#include <caam_jr_config.h>
+
+/*
+ * Offset to the registers of a job ring.
+ * Is different for each job ring.
+ */
+#define CHAN_BASE(jr)   ((size_t)(jr)->register_base_addr)
+
+#define SEC_JOB_RING_IS_FULL(pi, ci, ring_max_size, ring_threshold) \
+		((((pi) + 1 + ((ring_max_size) - (ring_threshold))) & \
+		  (ring_max_size - 1))  == ((ci)))
+
+#define SEC_CIRCULAR_COUNTER(x, max)   (((x) + 1) & (max - 1))
+
+/*
+ * Assert that cond is true. If !cond is true, display str and the vararg list
+ * in a printf-like syntax. also, if !cond is true, return altRet.
+ *
+ * \param cond          A boolean expression to be asserted true
+ * \param altRet        The value to be returned if cond doesn't hold true
+ * \param str           A quoted char string
+ *
+ * E.g.:
+ *      SEC_ASSERT(ret > 0, 0, "ERROR initializing app: code = %d\n", ret);
+ */
+#define SEC_ASSERT(cond, altRet, ...) do {\
+	if (unlikely(!(cond))) {\
+		CAAM_JR_ERR(__VA_ARGS__); \
+		return altRet; \
+	} \
+} while (0)
+
+#define SEC_DP_ASSERT(cond, altRet, ...) do {\
+	if (unlikely(!(cond))) {\
+		CAAM_JR_DP_ERR(__VA_ARGS__); \
+		return altRet; \
+	} \
+} while (0)
+
+#define ASSERT(x)
+
+/*
+ * Constants representing various job ring registers
+ */
+#if CAAM_BYTE_ORDER == __BIG_ENDIAN
+#define JR_REG_IRBA_OFFSET		0x0000
+#define JR_REG_IRBA_OFFSET_LO		0x0004
+#else
+#define JR_REG_IRBA_OFFSET		0x0004
+#define JR_REG_IRBA_OFFSET_LO		0x0000
+#endif
+
+#define JR_REG_IRSR_OFFSET		0x000C
+#define JR_REG_IRSA_OFFSET		0x0014
+#define JR_REG_IRJA_OFFSET		0x001C
+
+#if CAAM_BYTE_ORDER == __BIG_ENDIAN
+#define JR_REG_ORBA_OFFSET		0x0020
+#define JR_REG_ORBA_OFFSET_LO		0x0024
+#else
+#define JR_REG_ORBA_OFFSET		0x0024
+#define JR_REG_ORBA_OFFSET_LO		0x0020
+#endif
+
+#define JR_REG_ORSR_OFFSET		0x002C
+#define JR_REG_ORJR_OFFSET		0x0034
+#define JR_REG_ORSFR_OFFSET		0x003C
+#define JR_REG_JROSR_OFFSET		0x0044
+#define JR_REG_JRINT_OFFSET		0x004C
+
+#define JR_REG_JRCFG_OFFSET		0x0050
+#define JR_REG_JRCFG_OFFSET_LO		0x0054
+
+#define JR_REG_IRRI_OFFSET		0x005C
+#define JR_REG_ORWI_OFFSET		0x0064
+#define JR_REG_JRCR_OFFSET		0x006C
+
+/*
+ * Constants for error handling on job ring
+ */
+#define JR_REG_JRINT_ERR_TYPE_SHIFT	8
+#define JR_REG_JRINT_ERR_ORWI_SHIFT	16
+#define JR_REG_JRINIT_JRE_SHIFT		1
+
+#define JRINT_JRE			(1 << JR_REG_JRINIT_JRE_SHIFT)
+#define JRINT_ERR_WRITE_STATUS		(1 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_BAD_INPUT_BASE	(3 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_BAD_OUTPUT_BASE	(4 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_WRITE_2_IRBA		(5 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_WRITE_2_ORBA		(6 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_RES_B4_HALT		(7 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_REM_TOO_MANY		(8 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_ADD_TOO_MANY		(9 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_HALT_MASK		0x0C
+#define JRINT_ERR_HALT_INPROGRESS	0x04
+#define JRINT_ERR_HALT_COMPLETE		0x08
+
+#define JR_REG_JRCR_VAL_RESET		0x00000001
+
+#define JR_REG_JRCFG_LO_ICTT_SHIFT	0x10
+#define JR_REG_JRCFG_LO_ICDCT_SHIFT	0x08
+#define JR_REG_JRCFG_LO_ICEN_EN		0x02
+
+/*
+ * Constants for Descriptor Processing errors
+ */
+#define SEC_HW_ERR_SSRC_NO_SRC		0x00
+#define SEC_HW_ERR_SSRC_CCB_ERR		0x02
+#define SEC_HW_ERR_SSRC_JMP_HALT_U	0x03
+#define SEC_HW_ERR_SSRC_DECO		0x04
+#define SEC_HW_ERR_SSRC_JR		0x06
+#define SEC_HW_ERR_SSRC_JMP_HALT_COND   0x07
+
+#define SEC_HW_ERR_DECO_HFN_THRESHOLD   0xF1
+#define SEC_HW_ERR_CCB_ICV_CHECK_FAIL   0x0A
+
+/*
+ * Constants for descriptors
+ */
+/* Return higher 32 bits of physical address */
+#define PHYS_ADDR_HI(phys_addr) \
+	    (uint32_t)(((uint64_t)phys_addr) >> 32)
+
+/* Return lower 32 bits of physical address */
+#define PHYS_ADDR_LO(phys_addr) \
+	    (uint32_t)(((uint64_t)phys_addr) & 0xFFFFFFFF)
+
+/*
+ * Macros for extracting error codes for the job ring
+ */
+#define JR_REG_JRINT_ERR_TYPE_EXTRACT(value)      ((value) & 0x00000F00)
+#define JR_REG_JRINT_ERR_ORWI_EXTRACT(value)     \
+	(((value) & 0x3FFF0000) >> JR_REG_JRINT_ERR_ORWI_SHIFT)
+#define JR_REG_JRINT_JRE_EXTRACT(value)	   ((value) & JRINT_JRE)
+
+/*
+ * Macros for managing the job ring
+ */
+/* Read pointer to job ring input ring start address */
+#if defined(RTE_ARCH_ARM64)
+#define hw_get_inp_queue_base(jr) ((((dma_addr_t)GET_JR_REG(IRBA, \
+				      (jr))) << 32) | \
+				      (GET_JR_REG_LO(IRBA, (jr))))
+
+/* Read pointer to job ring output ring start address */
+#define hw_get_out_queue_base(jr) (((dma_addr_t)(GET_JR_REG(ORBA, \
+				     (jr))) << 32) | \
+				     (GET_JR_REG_LO(ORBA, (jr))))
+#else
+#define hw_get_inp_queue_base(jr)   ((dma_addr_t)(GET_JR_REG_LO(IRBA, (jr))))
+
+#define hw_get_out_queue_base(jr)   ((dma_addr_t)(GET_JR_REG_LO(ORBA, (jr))))
+#endif
+
+/*
+ * IRJA - Input Ring Jobs Added Register shows
+ * how many new jobs were added to the Input Ring.
+ */
+#define hw_enqueue_desc_on_job_ring(job_ring) SET_JR_REG(IRJA, (job_ring), 1)
+
+#define hw_set_input_ring_size(job_ring, size) SET_JR_REG(IRSR, job_ring, \
+							 (size))
+
+#define hw_set_output_ring_size(job_ring, size) SET_JR_REG(ORSR, job_ring, \
+							  (size))
+
+#if defined(RTE_ARCH_ARM64)
+#define hw_set_input_ring_start_addr(job_ring, start_addr)	\
+{								\
+	SET_JR_REG(IRBA, job_ring, PHYS_ADDR_HI(start_addr));	\
+	SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+
+#define hw_set_output_ring_start_addr(job_ring, start_addr) \
+{								\
+	SET_JR_REG(ORBA, job_ring, PHYS_ADDR_HI(start_addr));	\
+	SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+
+#else
+#define hw_set_input_ring_start_addr(job_ring, start_addr)	\
+{								\
+	SET_JR_REG(IRBA, job_ring, 0);	\
+	SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+
+#define hw_set_output_ring_start_addr(job_ring, start_addr) \
+{								\
+	SET_JR_REG(ORBA, job_ring, 0);	\
+	SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+#endif
+
+/* ORJR - Output Ring Jobs Removed Register shows how many jobs were
+ * removed from the Output Ring for processing by software. This is done after
+ * the software has processed the entries.
+ */
+#define hw_remove_entries(jr, no_entries) SET_JR_REG(ORJR, (jr), (no_entries))
+
+/* IRSA - Input Ring Slots Available register holds the number of entries in
+ * the Job Ring's input ring. Once a job is enqueued, the value returned is
+ * decremented by the hardware by the number of jobs enqueued.
+ */
+#define hw_get_available_slots(jr)		GET_JR_REG(IRSA, jr)
+
+/* ORSFR - Output Ring Slots Full register holds the number of jobs which were
+ * processed by the SEC and can be retrieved by the software. Once a job has
+ * been processed by software, the user will call hw_remove_one_entry in order
+ * to notify the SEC that the entry was processed.
+ */
+#define hw_get_no_finished_jobs(jr)		GET_JR_REG(ORSFR, jr)
+
+/*
+ * Macros for manipulating JR registers
+ */
+#if CORE_BYTE_ORDER == CAAM_BYTE_ORDER
+#define sec_read_32(addr)	(*(volatile unsigned int *)(addr))
+#define sec_write_32(addr, val)	(*(volatile unsigned int *)(addr) = (val))
+
+#else
+#define sec_read_32(addr)	rte_bswap32((*(volatile unsigned int *)(addr)))
+#define sec_write_32(addr, val) \
+			(*(volatile unsigned int *)(addr) = rte_bswap32(val))
+#endif
+
+#if CAAM_BYTE_ORDER == __LITTLE_ENDIAN
+#define sec_read_64(addr)	(((u64)sec_read_32((u32 *)(addr) + 1) << 32) | \
+				(sec_read_32((u32 *)(addr))))
+
+#define sec_write_64(addr, val) {				\
+	sec_write_32((u32 *)(addr) + 1, (u32)((val) >> 32));	\
+	sec_write_32((u32 *)(addr), (u32)(val));		\
+}
+#else /* CAAM_BYTE_ORDER == __BIG_ENDIAN */
+#define sec_read_64(addr)	(((u64)sec_read_32((u32 *)(addr)) << 32) | \
+				(sec_read_32((u32 *)(addr) + 1)))
+
+#define sec_write_64(addr, val) {				\
+	sec_write_32((u32 *)(addr), (u32)((val) >> 32));	\
+	sec_write_32((u32 *)(addr) + 1, (u32)(val));		\
+}
+#endif
+
+#if defined(RTE_ARCH_ARM64)
+#define sec_read_addr(a)	sec_read_64((a))
+#define sec_write_addr(a, v)	sec_write_64((a), (v))
+#else
+#define sec_read_addr(a)	sec_read_32((a))
+#define sec_write_addr(a, v)	sec_write_32((a), (v))
+#endif
+
+#define JR_REG(name, jr)	(CHAN_BASE(jr) + JR_REG_##name##_OFFSET)
+#define JR_REG_LO(name, jr)	(CHAN_BASE(jr) + JR_REG_##name##_OFFSET_LO)
+
+#define GET_JR_REG(name, jr)	(sec_read_32(JR_REG(name, (jr))))
+#define GET_JR_REG_LO(name, jr)	(sec_read_32(JR_REG_LO(name, (jr))))
+
+#define SET_JR_REG(name, jr, value) \
+				(sec_write_32(JR_REG(name, (jr)), value))
+#define SET_JR_REG_LO(name, jr, value) \
+				(sec_write_32(JR_REG_LO(name, (jr)), value))
+
+/* Lists the possible states for a job ring. */
+typedef enum sec_job_ring_state_e {
+	SEC_JOB_RING_STATE_STARTED,	/* Job ring is initialized */
+	SEC_JOB_RING_STATE_RESET,	/* Job ring reset is in progress */
+} sec_job_ring_state_t;
+
+/* code or cmd block to caam */
+struct sec_cdb {
+	struct {
+		union {
+			uint32_t word;
+			struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+				uint16_t rsvd63_48;
+				unsigned int rsvd47_39:9;
+				unsigned int idlen:7;
+#else
+				unsigned int idlen:7;
+				unsigned int rsvd47_39:9;
+				uint16_t rsvd63_48;
+#endif
+			} field;
+		} __rte_packed hi;
+
+		union {
+			uint32_t word;
+			struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+				unsigned int rsvd31_30:2;
+				unsigned int fsgt:1;
+				unsigned int lng:1;
+				unsigned int offset:2;
+				unsigned int abs:1;
+				unsigned int add_buf:1;
+				uint8_t pool_id;
+				uint16_t pool_buffer_size;
+#else
+				uint16_t pool_buffer_size;
+				uint8_t pool_id;
+				unsigned int add_buf:1;
+				unsigned int abs:1;
+				unsigned int offset:2;
+				unsigned int lng:1;
+				unsigned int fsgt:1;
+				unsigned int rsvd31_30:2;
+#endif
+			} field;
+		} __rte_packed lo;
+	} __rte_packed sh_hdr;
+
+	uint32_t sh_desc[SEC_JOB_DESCRIPTOR_SIZE];
+};
+
+struct caam_jr_qp {
+	struct sec_job_ring_t *ring;
+	uint64_t rx_pkts;
+	uint64_t rx_errs;
+	uint64_t rx_poll_err;
+	uint64_t tx_pkts;
+	uint64_t tx_errs;
+	uint64_t tx_ring_full;
+};
+
+struct sec_job_ring_t {
+	/* TODO: Add wrapper macro to make it obvious this is the consumer index
+	 * on the output ring
+	 */
+	uint32_t cidx;		/* Consumer index for job ring (jobs array).
+				 * @note: cidx and pidx are accessed from
+				 * different threads. Place the cidx and pidx
+				 * inside the structure so that they lay on
+				 * different cachelines, to avoid false sharing
+				 * between threads when the threads run on
+				 * different cores!
+				 */
+	/* TODO: Add wrapper macro to make it obvious this is the producer index
+	 * on the input ring
+	 */
+	uint32_t pidx;		/* Producer index for job ring (jobs array) */
+
+	phys_addr_t *input_ring;/* Ring of output descriptors received from SEC.
+				 * Size of array is power of 2 to allow fast
+				 * update of producer/consumer indexes with
+				 * bitwise operations.
+				 */
+
+	struct sec_outring_entry *output_ring;
+				/* Ring of output descriptors received from SEC.
+				 * Size of array is power of 2 to allow fast
+				 * update of producer/consumer indexes with
+				 * bitwise operations.
+				 */
+
+	uint32_t irq_fd;	/* The file descriptor used for polling from
+				 * user space for interrupts notifications
+				 */
+	uint32_t jr_mode;	/* Model used by SEC Driver to receive
+				 * notifications from SEC.  Can be either
+				 * of the three: #SEC_NOTIFICATION_TYPE_NAPI
+				 * #SEC_NOTIFICATION_TYPE_IRQ or
+				 * #SEC_NOTIFICATION_TYPE_POLL
+				 */
+	uint32_t napi_mode;	/* Job ring mode if NAPI mode is chosen
+				 * Used only when jr_mode is set to
+				 * #SEC_NOTIFICATION_TYPE_NAPI
+				 */
+	void *register_base_addr;	/* Base address for SEC's
+					 * register memory for this job ring.
+					 */
+	uint8_t coalescing_en;		/* notifies if coelescing is
+					 * enabled for the job ring
+					 */
+	sec_job_ring_state_t jr_state;	/* The state of this job ring */
+
+	struct rte_mempool *ctx_pool;   /* per dev mempool for caam_jr_op_ctx */
+	unsigned int max_nb_queue_pairs;
+	unsigned int max_nb_sessions;
+	struct caam_jr_qp qps[RTE_CAAM_MAX_NB_SEC_QPS]; /* i/o queue for sec */
+};
+
+/* Union describing the possible error codes that
+ * can be set in the descriptor status word
+ */
+union hw_error_code {
+	uint32_t error;
+	union {
+		struct {
+			uint32_t ssrc:4;
+			uint32_t ssed_val:28;
+		} __rte_packed value;
+		struct {
+			uint32_t ssrc:4;
+			uint32_t res:28;
+		} __rte_packed no_status_src;
+		struct {
+			uint32_t ssrc:4;
+			uint32_t jmp:1;
+			uint32_t res:11;
+			uint32_t desc_idx:8;
+			uint32_t cha_id:4;
+			uint32_t err_id:4;
+		} __rte_packed ccb_status_src;
+		struct {
+			uint32_t ssrc:4;
+			uint32_t jmp:1;
+			uint32_t res:11;
+			uint32_t desc_idx:8;
+			uint32_t offset:8;
+		} __rte_packed jmp_halt_user_src;
+		struct {
+			uint32_t ssrc:4;
+			uint32_t jmp:1;
+			uint32_t res:11;
+			uint32_t desc_idx:8;
+			uint32_t desc_err:8;
+		} __rte_packed deco_src;
+		struct {
+			uint32_t ssrc:4;
+			uint32_t res:17;
+			uint32_t naddr:3;
+			uint32_t desc_err:8;
+		} __rte_packed jr_src;
+		struct {
+			uint32_t ssrc:4;
+			uint32_t jmp:1;
+			uint32_t res:11;
+			uint32_t desc_idx:8;
+			uint32_t cond:8;
+		} __rte_packed jmp_halt_cond_src;
+	} __rte_packed error_desc;
+} __rte_packed;
+
+/* @brief Initialize a job ring/channel in SEC device.
+ * Write configuration register/s to properly initialize a job ring.
+ *
+ * @param [in] job_ring     The job ring
+ *
+ * @retval 0 for success
+ * @retval other for error
+ */
+int hw_reset_job_ring(struct sec_job_ring_t *job_ring);
+
+/* @brief Reset a job ring/channel in SEC device.
+ * Write configuration register/s to reset a job ring.
+ *
+ * @param [in] job_ring     The job ring
+ *
+ * @retval 0 for success
+ * @retval -1 in case job ring reset failed
+ */
+int hw_shutdown_job_ring(struct sec_job_ring_t *job_ring);
+
+/* @brief Handle a job ring/channel error in SEC device.
+ * Identify the error type and clear error bits if required.
+ *
+ * @param [in]  job_ring	The job ring
+ * @param [in]  sec_error_code  The job ring's error code
+ */
+void hw_handle_job_ring_error(struct sec_job_ring_t *job_ring,
+			      uint32_t sec_error_code);
+
+/* @brief Handle a job ring error in the device.
+ * Identify the error type and printout a explanatory
+ * messages.
+ *
+ * @param [in]  job_ring	The job ring
+ *
+ */
+void hw_job_ring_error_print(struct sec_job_ring_t *job_ring, int code);
+
+/* @brief Set interrupt coalescing parameters on the Job Ring.
+ * @param [in]  job_ring		The job ring
+ * @param [in]  irq_coalesing_timer     Interrupt coalescing timer threshold.
+ *					This value determines the maximum
+ *					amount of time after processing a
+ *					descriptor before raising an interrupt.
+ * @param [in]  irq_coalescing_count    Interrupt coalescing descriptor count
+ *					threshold.
+ */
+int hw_job_ring_set_coalescing_param(struct sec_job_ring_t *job_ring,
+				     uint16_t irq_coalescing_timer,
+				     uint8_t irq_coalescing_count);
+
+/* @brief Enable interrupt coalescing on a job ring
+ * @param [in]  job_ring		The job ring
+ */
+int hw_job_ring_enable_coalescing(struct sec_job_ring_t *job_ring);
+
+/* @brief Disable interrupt coalescing on a job ring
+ * @param [in]  job_ring		The job ring
+ */
+int hw_job_ring_disable_coalescing(struct sec_job_ring_t *job_ring);
+
+#endif /* CAAM_JR_HW_SPECIFIC_H */
diff --git a/drivers/crypto/caam_jr/caam_jr_log.h b/drivers/crypto/caam_jr/caam_jr_log.h
new file mode 100644
index 00000000..106ff07a
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_log.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef _CAAM_JR_LOG_H_
+#define _CAAM_JR_LOG_H_
+
+#include <rte_log.h>
+
+extern int caam_jr_logtype;
+
+#define CAAM_JR_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, caam_jr_logtype, "caam_jr: " \
+		fmt "\n", ##args)
+
+#define CAAM_JR_DEBUG(fmt, args...) \
+	rte_log(RTE_LOG_DEBUG, caam_jr_logtype, "caam_jr: %s(): " \
+		fmt "\n", __func__, ##args)
+
+#define PMD_INIT_FUNC_TRACE() CAAM_JR_DEBUG(" >>")
+
+#define CAAM_JR_INFO(fmt, args...) \
+	CAAM_JR_LOG(INFO, fmt, ## args)
+#define CAAM_JR_ERR(fmt, args...) \
+	CAAM_JR_LOG(ERR, fmt, ## args)
+#define CAAM_JR_WARN(fmt, args...) \
+	CAAM_JR_LOG(WARNING, fmt, ## args)
+
+/* DP Logs, toggled out at compile time if level lower than current level */
+#define CAAM_JR_DP_LOG(level, fmt, args...) \
+	RTE_LOG_DP(level, PMD, fmt "\n", ## args)
+
+#define CAAM_JR_DP_DEBUG(fmt, args...) \
+	CAAM_JR_DP_LOG(DEBUG, fmt, ## args)
+#define CAAM_JR_DP_INFO(fmt, args...) \
+	CAAM_JR_DP_LOG(INFO, fmt, ## args)
+#define CAAM_JR_DP_WARN(fmt, args...) \
+	CAAM_JR_DP_LOG(WARNING, fmt, ## args)
+#define CAAM_JR_DP_ERR(fmt, args...) \
+	CAAM_JR_DP_LOG(ERR, fmt, ## args)
+
+#endif /* _CAAM_JR_LOG_H_ */
diff --git a/drivers/crypto/caam_jr/caam_jr_pvt.h b/drivers/crypto/caam_jr/caam_jr_pvt.h
new file mode 100644
index 00000000..9f1adabc
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_pvt.h
@@ -0,0 +1,291 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef CAAM_JR_PVT_H
+#define CAAM_JR_PVT_H
+
+#include <hw/desc/ipsec.h>
+
+/* NXP CAAM JR PMD device name */
+
+#define CAAM_JR_ALG_UNSUPPORT	(-1)
+
+/* Minimum job descriptor consists of a oneword job descriptor HEADER and
+ * a pointer to the shared descriptor.
+ */
+#define MIN_JOB_DESC_SIZE	(CAAM_CMD_SZ + CAAM_PTR_SZ)
+#define CAAM_JOB_DESC_SIZE	13
+
+/* CTX_POOL_NUM_BUFS is set as per the ipsec-secgw application */
+#define CTX_POOL_NUM_BUFS	32000
+#define CTX_POOL_CACHE_SIZE	512
+
+#define DIR_ENC                 1
+#define DIR_DEC                 0
+
+#define JR_MAX_NB_MAX_DIGEST	32
+
+#define RTE_CAAM_JR_PMD_MAX_NB_SESSIONS 2048
+
+
+/* Return codes for SEC user space driver APIs */
+enum sec_return_code_e {
+	SEC_SUCCESS = 0,	       /* Operation executed successfully.*/
+	SEC_INVALID_INPUT_PARAM,       /* API received an invalid input
+					* parameter
+					*/
+	SEC_OUT_OF_MEMORY,	       /* Memory allocation failed. */
+	SEC_DESCRIPTOR_IN_FLIGHT,      /* API function indicates there are
+					* descriptors in flight
+					* for SEC to process.
+					*/
+	SEC_LAST_DESCRIPTOR_IN_FLIGHT, /* API function indicates there is one
+					* last descriptor in flight
+					* for SEC to process that.
+					*/
+	SEC_PROCESSING_ERROR,	       /* Indicates a SEC processing error
+					* occurred on a Job Ring which requires
+					* a SEC user space driver shutdown. Can
+					* be returned from sec_poll_job_ring().
+					* Then the only other API that can be
+					* called after this error is
+					* sec_release().
+					*/
+	SEC_DESC_PROCESSING_ERROR,     /* Indicates a SEC descriptor processing
+					* error occurred on a Job Ring. Can be
+					* returned from sec_poll_job_ring().
+					* The driver was able to reset job ring
+					* and job ring can be used like in a
+					* normal case.
+					*/
+	SEC_JR_IS_FULL,			/* Job Ring is full. There is no more
+					 * room in the JR for new descriptors.
+					 * This can happen if the descriptor RX
+					 * rate is higher than SEC's capacity.
+					 */
+	SEC_DRIVER_RELEASE_IN_PROGRESS, /* SEC driver shutdown is in progress,
+					 * descriptors processing or polling is
+					 * allowed.
+					 */
+	SEC_DRIVER_ALREADY_INITIALIZED, /* SEC driver is already initialized.*/
+	SEC_DRIVER_NOT_INITIALIZED,	/* SEC driver is NOT initialized. */
+	SEC_JOB_RING_RESET_IN_PROGRESS, /* Job ring is resetting due to a
+					 * per-descriptor SEC processing error
+					 * ::SEC_desc_PROCESSING_ERROR. Reset is
+					 * finished when sec_poll_job_ring()
+					 * return. Then the job ring can be used
+					 * again.
+					 */
+	SEC_RESET_ENGINE_FAILED,	/* Resetting of SEC Engine by SEC Kernel
+					 * Driver Failed
+					 */
+	SEC_ENABLE_IRQS_FAILED,		/* Enabling of IRQs in SEC Kernel Driver
+					 * Failed
+					 */
+	SEC_DISABLE_IRQS_FAILED,	/* Disabling of IRQs in SEC Kernel
+					 * Driver Failed
+					 */
+	/* END OF VALID VALUES */
+
+	SEC_RETURN_CODE_MAX_VALUE,	/* Invalid value for return code. It is
+					 * used to mark the end of the return
+					 * code values. @note ALL new return
+					 * code values MUST be added before
+					 * ::SEC_RETURN_CODE_MAX_VALUE!
+					 */
+};
+
+enum caam_jr_op_type {
+	CAAM_JR_NONE,  /* No Cipher operations*/
+	CAAM_JR_CIPHER,/* CIPHER operations */
+	CAAM_JR_AUTH,  /* Authentication Operations */
+	CAAM_JR_AEAD,  /* Authenticated Encryption with associated data */
+	CAAM_JR_IPSEC, /* IPSEC protocol operations*/
+	CAAM_JR_PDCP,  /* PDCP protocol operations*/
+	CAAM_JR_PKC,   /* Public Key Cryptographic Operations */
+	CAAM_JR_MAX
+};
+
+struct caam_jr_session {
+	uint8_t dir;         /* Operation Direction */
+	enum rte_crypto_cipher_algorithm cipher_alg; /* Cipher Algorithm*/
+	enum rte_crypto_auth_algorithm auth_alg; /* Authentication Algorithm*/
+	enum rte_crypto_aead_algorithm aead_alg; /* AEAD Algorithm*/
+	enum rte_security_session_protocol proto_alg; /* Security Algorithm*/
+	union {
+		struct {
+			uint8_t *data;	/* pointer to key data */
+			size_t length;	/* key length in bytes */
+		} aead_key;
+		struct {
+			struct {
+				uint8_t *data;	/* pointer to key data */
+				size_t length;	/* key length in bytes */
+			} cipher_key;
+			struct {
+				uint8_t *data;	/* pointer to key data */
+				size_t length;	/* key length in bytes */
+			} auth_key;
+		};
+	};
+	struct {
+		uint16_t length;
+		uint16_t offset;
+	} iv;	/* Initialisation vector parameters */
+	uint16_t auth_only_len; /* Length of data for Auth only */
+	uint32_t digest_length;
+	struct ipsec_encap_pdb encap_pdb;
+	struct ip ip4_hdr;
+	struct ipsec_decap_pdb decap_pdb;
+	struct caam_jr_qp *qp;
+	struct sec_cdb *cdb;	/* cmd block associated with qp */
+	struct rte_mempool *ctx_pool; /* session mempool for caam_jr_op_ctx */
+};
+
+/*
+ * 16-byte hardware scatter/gather table
+ */
+
+#define SEC4_SG_LEN_EXT		0x80000000	/* Entry points to table */
+#define SEC4_SG_LEN_FIN		0x40000000	/* Last ent in table */
+#define SEC4_SG_BPID_MASK	0x000000ff
+#define SEC4_SG_BPID_SHIFT	16
+#define SEC4_SG_LEN_MASK	0x3fffffff	/* Excludes EXT and FINAL */
+#define SEC4_SG_OFFSET_MASK	0x00001fff
+
+struct sec4_sg_entry {
+	uint64_t ptr;
+	uint32_t len;
+	uint32_t bpid_offset;
+};
+
+#define MAX_SG_ENTRIES		16
+#define SG_CACHELINE_0		0
+#define SG_CACHELINE_1		4
+#define SG_CACHELINE_2		8
+#define SG_CACHELINE_3		12
+
+/* Structure encompassing a job descriptor which is to be processed
+ * by SEC. User should also initialise this structure with the callback
+ * function pointer which will be called by driver after recieving proccessed
+ * descriptor from SEC. User data is also passed in this data structure which
+ * will be sent as an argument to the user callback function.
+ */
+struct job_descriptor {
+	uint32_t desc[CAAM_JOB_DESC_SIZE];
+};
+
+struct caam_jr_op_ctx {
+	struct job_descriptor jobdes;
+	/* sg[0] output, sg[1] input, others are possible sub frames */
+	struct sec4_sg_entry sg[MAX_SG_ENTRIES];
+	struct rte_crypto_op *op;
+	struct rte_mempool *ctx_pool; /* mempool pointer for caam_jr_op_ctx */
+	int64_t vtop_offset;
+	uint8_t digest[JR_MAX_NB_MAX_DIGEST];
+};
+
+/**
+ * Checksum
+ *
+ * @param buffer calculate chksum for buffer
+ * @param len    buffer length
+ *
+ * @return checksum value in host cpu order
+ */
+static inline uint16_t
+calc_chksum(void *buffer, int len)
+{
+	uint16_t *buf = (uint16_t *)buffer;
+	uint32_t sum = 0;
+	uint16_t result;
+
+	for (sum = 0; len > 1; len -= 2)
+		sum += *buf++;
+
+	if (len == 1)
+		sum += *(unsigned char *)buf;
+
+	sum = (sum >> 16) + (sum & 0xFFFF);
+	sum += (sum >> 16);
+	result = ~sum;
+
+	return  result;
+}
+struct uio_job_ring {
+	uint32_t jr_id;
+	uint32_t uio_fd;
+	void *register_base_addr;
+	int map_size;
+	int uio_minor_number;
+};
+
+int sec_cleanup(void);
+int sec_configure(void);
+struct uio_job_ring *config_job_ring(void);
+void free_job_ring(uint32_t uio_fd);
+
+/* For Dma memory allocation of specified length and alignment */
+static inline void *
+caam_jr_dma_mem_alloc(size_t align, size_t len)
+{
+	return rte_malloc("mem_alloc", len, align);
+}
+
+/* For freeing dma memory */
+static inline void
+caam_jr_dma_free(void *ptr)
+{
+	rte_free(ptr);
+}
+
+static inline rte_iova_t
+caam_jr_mem_vtop(void *vaddr)
+{
+	const struct rte_memseg *ms;
+
+	ms = rte_mem_virt2memseg(vaddr, NULL);
+	if (ms)
+		return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
+	return (size_t)NULL;
+}
+
+static inline void *
+caam_jr_dma_ptov(rte_iova_t paddr)
+{
+	return rte_mem_iova2virt(paddr);
+}
+
+/* Virtual to physical address conversion */
+static inline rte_iova_t caam_jr_dma_vtop(void *ptr)
+{
+	return caam_jr_mem_vtop(ptr);
+}
+
+/** @brief Request to SEC kernel driver to enable interrupts for
+ *         descriptor finished processing
+ *  Use UIO to communicate with SEC kernel driver: write command
+ *  value that indicates an IRQ enable action into UIO file descriptor
+ *  of this job ring.
+ *
+ * @param [in]  uio_fd     Job Ring UIO File descriptor
+ * @retval 0 for success
+ * @retval -1 value for error
+ */
+uint32_t caam_jr_enable_irqs(uint32_t uio_fd);
+
+/** @brief Request to SEC kernel driver to disable interrupts for descriptor
+ *  finished processing
+ *  Use UIO to communicate with SEC kernel driver: write command
+ *  value that indicates an IRQ disable action into UIO file descriptor
+ *  of this job ring.
+ *
+ * @param [in]  uio_fd    UIO File descripto
+ * @retval 0 for success
+ * @retval -1 value for error
+ *
+ */
+uint32_t caam_jr_disable_irqs(uint32_t uio_fd);
+
+#endif
diff --git a/drivers/crypto/caam_jr/caam_jr_uio.c b/drivers/crypto/caam_jr/caam_jr_uio.c
new file mode 100644
index 00000000..c07d9db0
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_uio.c
@@ -0,0 +1,501 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <dirent.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <errno.h>
+#include <fcntl.h>
+
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_crypto.h>
+#include <rte_security.h>
+
+#include <caam_jr_config.h>
+#include <caam_jr_hw_specific.h>
+#include <caam_jr_pvt.h>
+#include <caam_jr_log.h>
+
+/* RTA header files */
+#include <hw/desc/common.h>
+#include <hw/desc/algo.h>
+#include <hw/desc/ipsec.h>
+
+/* Prefix path to sysfs directory where UIO device attributes are exported.
+ * Path for UIO device X is /sys/class/uio/uioX
+ */
+#define SEC_UIO_DEVICE_SYS_ATTR_PATH    "/sys/class/uio"
+
+/* Subfolder in sysfs where mapping attributes are exported
+ * for each UIO device. Path for mapping Y for device X is:
+ *      /sys/class/uio/uioX/maps/mapY
+ */
+#define SEC_UIO_DEVICE_SYS_MAP_ATTR     "maps/map"
+
+/* Name of UIO device file prefix. Each UIO device will have a device file
+ * /dev/uioX, where X is the minor device number.
+ */
+#define SEC_UIO_DEVICE_FILE_NAME    "/dev/uio"
+
+/*
+ * Name of UIO device. Each user space SEC job ring will have a corresponding
+ * UIO device with the name sec-channelX, where X is the job ring id.
+ * Maximum length is #SEC_UIO_MAX_DEVICE_NAME_LENGTH.
+ *
+ * @note  Must be kept in synch with SEC kernel driver
+ * define #SEC_UIO_DEVICE_NAME !
+ */
+#define SEC_UIO_DEVICE_NAME     "fsl-jr"
+
+/* Maximum length for the name of an UIO device file.
+ * Device file name format is: /dev/uioX.
+ */
+#define SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH 30
+
+/* Maximum length for the name of an attribute file for an UIO device.
+ * Attribute files are exported in sysfs and have the name formatted as:
+ *      /sys/class/uio/uioX/<attribute_file_name>
+ */
+#define SEC_UIO_MAX_ATTR_FILE_NAME  100
+
+/* Command that is used by SEC user space driver and SEC kernel driver
+ *  to signal a request from the former to the later to disable job DONE
+ *  and error IRQs on a certain job ring.
+ *  The configuration is done at SEC Controller's level.
+ *  @note   Need to be kept in synch with #SEC_UIO_DISABLE_IRQ_CMD from
+ *          linux/drivers/crypto/talitos.c !
+ */
+#define SEC_UIO_DISABLE_IRQ_CMD     0
+
+/* Command that is used by SEC user space driver and SEC kernel driver
+ *  to signal a request from the former to the later to enable job DONE
+ *  and error IRQs on a certain job ring.
+ *  The configuration is done at SEC Controller's level.
+ *  @note   Need to be kept in synch with #SEC_UIO_ENABLE_IRQ_CMD from
+ *          linux/drivers/crypto/talitos.c !
+ */
+#define SEC_UIO_ENABLE_IRQ_CMD      1
+
+/** Command that is used by SEC user space driver and SEC kernel driver
+ *  to signal a request from the former to the later to do a SEC engine reset.
+ *  @note   Need to be kept in synch with #SEC_UIO_RESET_SEC_ENGINE_CMD from
+ *          linux/drivers/crypto/talitos.c !
+ */
+#define SEC_UIO_RESET_SEC_ENGINE_CMD    3
+
+/* The id for the mapping used to export SEC's registers to
+ * user space through UIO devices.
+ */
+#define SEC_UIO_MAP_ID              0
+
+static struct uio_job_ring g_uio_job_ring[MAX_SEC_JOB_RINGS];
+static int g_uio_jr_num;
+
+/** @brief Checks if a file name contains a certain substring.
+ * If so, it extracts the number following the substring.
+ * This function assumes a filename format of: [text][number].
+ * @param [in]  filename    File name
+ * @param [in]  match       String to match in file name
+ * @param [out] number      The number extracted from filename
+ *
+ * @retval true if file name matches the criteria
+ * @retval false if file name does not match the criteria
+ */
+static bool
+file_name_match_extract(const char filename[], const char match[], int *number)
+{
+	char *substr = NULL;
+
+	substr = strstr(filename, match);
+	if (substr == NULL)
+		return false;
+
+	/* substring <match> was found in <filename>
+	 * read number following <match> substring in <filename>
+	 */
+	if (sscanf(filename + strlen(match), "%d", number) <= 0)
+		return false;
+
+	return true;
+}
+
+/** @brief Reads first line from a file.
+ * Composes file name as: root/subdir/filename
+ *
+ * @param [in]  root     Root path
+ * @param [in]  subdir   Subdirectory name
+ * @param [in]  filename File name
+ * @param [out] line     The first line read from file.
+ *
+ * @retval 0 for succes
+ * @retval other value for error
+ */
+static int
+file_read_first_line(const char root[], const char subdir[],
+		     const char filename[], char *line)
+{
+	char absolute_file_name[SEC_UIO_MAX_ATTR_FILE_NAME];
+	int fd = 0, ret = 0;
+
+	/*compose the file name: root/subdir/filename */
+	memset(absolute_file_name, 0, sizeof(absolute_file_name));
+	snprintf(absolute_file_name, SEC_UIO_MAX_ATTR_FILE_NAME,
+		 "%s/%s/%s", root, subdir, filename);
+
+	fd = open(absolute_file_name, O_RDONLY);
+	SEC_ASSERT(fd > 0, fd, "Error opening file %s",
+			absolute_file_name);
+
+	/* read UIO device name from first line in file */
+	ret = read(fd, line, SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH);
+	close(fd);
+
+	/* NULL-ify string */
+	line[SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH - 1] = '\0';
+
+	if (ret <= 0) {
+		CAAM_JR_ERR("Error reading from file %s", absolute_file_name);
+		return ret;
+	}
+
+	return 0;
+}
+
+/** @brief Uses UIO control to send commands to SEC kernel driver.
+ * The mechanism is to write a command word into the file descriptor
+ * that the user-space driver obtained for each user-space SEC job ring.
+ * Both user-space driver and kernel driver must have the same understanding
+ * about the command codes.
+ *
+ * @param [in]  UIO FD		    The UIO file descriptor
+ * @param [in]  uio_command         Command word
+ *
+ * @retval Result of write operation on the job ring's UIO file descriptor.
+ *         Should be sizeof(int) for success operations.
+ *         Other values can be returned and used, if desired to add special
+ *         meaning to return values, but this has to be programmed in SEC
+ *         kernel driver as well. No special return values are used.
+ */
+static int
+sec_uio_send_command(uint32_t uio_fd, int32_t uio_command)
+{
+	int ret;
+
+	/* Use UIO file descriptor we have for this job ring.
+	 * Writing a command code to this file descriptor will make the
+	 * SEC kernel driver execute the desired command.
+	 */
+	ret = write(uio_fd, &uio_command, sizeof(int));
+	return ret;
+}
+
+/** @brief Request to SEC kernel driver to enable interrupts for
+ *         descriptor finished processing
+ *  Use UIO to communicate with SEC kernel driver: write command
+ *  value that indicates an IRQ enable action into UIO file descriptor
+ *  of this job ring.
+ *
+ * @param [in]  uio_fd     Job Ring UIO File descriptor
+ * @retval 0 for success
+ * @retval -1 value for error
+ */
+uint32_t
+caam_jr_enable_irqs(uint32_t uio_fd)
+{
+	int ret;
+
+	/* Use UIO file descriptor we have for this job ring.
+	 * Writing a command code to this file descriptor will make the
+	 * SEC kernel driver enable DONE and Error IRQs for this job ring,
+	 * at Controller level.
+	 */
+	ret = sec_uio_send_command(uio_fd, SEC_UIO_ENABLE_IRQ_CMD);
+	SEC_ASSERT(ret == sizeof(int), -1,
+		"Failed to request SEC engine to enable job done and "
+		"error IRQs through UIO control. UIO FD %d. Reset SEC driver!",
+		uio_fd);
+	CAAM_JR_DEBUG("Enabled IRQs on jr with uio_fd %d", uio_fd);
+	return 0;
+}
+
+
+/** @brief Request to SEC kernel driver to disable interrupts for descriptor
+ *  finished processing
+ *  Use UIO to communicate with SEC kernel driver: write command
+ *  value that indicates an IRQ disable action into UIO file descriptor
+ *  of this job ring.
+ *
+ * @param [in]  uio_fd    UIO File descripto
+ * @retval 0 for success
+ * @retval -1 value for error
+ *
+ */
+uint32_t
+caam_jr_disable_irqs(uint32_t uio_fd)
+{
+	int ret;
+
+	/* Use UIO file descriptor we have for this job ring.
+	 * Writing a command code to this file descriptor will make the
+	 * SEC kernel driver disable IRQs for this job ring,
+	 * at Controller level.
+	 */
+
+	ret = sec_uio_send_command(uio_fd, SEC_UIO_DISABLE_IRQ_CMD);
+	SEC_ASSERT(ret == sizeof(int), -1,
+		"Failed to request SEC engine to disable job done and "
+		"IRQs through UIO control. UIO_FD %d Reset SEC driver!",
+		uio_fd);
+	CAAM_JR_DEBUG("Disabled IRQs on jr with uio_fd %d", uio_fd);
+	return 0;
+}
+
+/** @brief Maps register range assigned for a job ring.
+ *
+ * @param [in] uio_device_fd    UIO device file descriptor
+ * @param [in] uio_device_id    UIO device id
+ * @param [in] uio_map_id       UIO allows maximum 5 different mapping for
+				each device. Maps start with id 0.
+ * @param [out] map_size        Map size.
+ * @retval  NULL if failed to map registers
+ * @retval  Virtual address for mapped register address range
+ */
+static void *
+uio_map_registers(int uio_device_fd, int uio_device_id,
+		  int uio_map_id, int *map_size)
+{
+	void *mapped_address = NULL;
+	unsigned int uio_map_size = 0;
+	char uio_sys_root[SEC_UIO_MAX_ATTR_FILE_NAME];
+	char uio_sys_map_subdir[SEC_UIO_MAX_ATTR_FILE_NAME];
+	char uio_map_size_str[32];
+	int ret = 0;
+
+	/* compose the file name: root/subdir/filename */
+	memset(uio_sys_root, 0, sizeof(uio_sys_root));
+	memset(uio_sys_map_subdir, 0, sizeof(uio_sys_map_subdir));
+	memset(uio_map_size_str, 0, sizeof(uio_map_size_str));
+
+	/* Compose string: /sys/class/uio/uioX */
+	sprintf(uio_sys_root, "%s/%s%d", SEC_UIO_DEVICE_SYS_ATTR_PATH,
+		"uio", uio_device_id);
+	/* Compose string: maps/mapY */
+	sprintf(uio_sys_map_subdir, "%s%d", SEC_UIO_DEVICE_SYS_MAP_ATTR,
+		uio_map_id);
+
+	/* Read first (and only) line from file
+	 * /sys/class/uio/uioX/maps/mapY/size
+	 */
+	ret = file_read_first_line(uio_sys_root, uio_sys_map_subdir,
+				 "size", uio_map_size_str);
+	SEC_ASSERT(ret == 0, NULL, "file_read_first_line() failed");
+
+	/* Read mapping size, expressed in hexa(base 16) */
+	uio_map_size = strtol(uio_map_size_str, NULL, 16);
+
+	/* Map the region in user space */
+	mapped_address = mmap(0, /*dynamically choose virtual address */
+		uio_map_size, PROT_READ | PROT_WRITE,
+		MAP_SHARED, uio_device_fd, 0);
+	/* offset = 0 because UIO device has only one mapping
+	 * for the entire SEC register memory
+	 */
+	if (mapped_address == MAP_FAILED) {
+		CAAM_JR_ERR(
+			"Failed to map registers! errno = %d job ring fd  = %d,"
+			"uio device id = %d, uio map id = %d", errno,
+			uio_device_fd, uio_device_id, uio_map_id);
+		return NULL;
+	}
+
+	/*
+	 * Save the map size to use it later on for munmap-ing.
+	 */
+	*map_size = uio_map_size;
+
+	CAAM_JR_INFO("UIO dev[%d] mapped region [id =%d] size 0x%x at %p",
+		uio_device_id, uio_map_id, uio_map_size, mapped_address);
+
+	return mapped_address;
+}
+
+void
+free_job_ring(uint32_t uio_fd)
+{
+	struct uio_job_ring *job_ring = NULL;
+	int i;
+
+	if (!job_ring->uio_fd)
+		return;
+
+	for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
+		if (g_uio_job_ring[i].uio_fd == uio_fd) {
+			job_ring = &g_uio_job_ring[i];
+			break;
+		}
+	}
+
+	if (job_ring == NULL) {
+		CAAM_JR_ERR("JR not available for fd = %x\n", uio_fd);
+		return;
+	}
+
+	/* Open device file */
+	CAAM_JR_INFO("Closed device file for job ring %d , fd = %d",
+			job_ring->jr_id, job_ring->uio_fd);
+	close(job_ring->uio_fd);
+	g_uio_jr_num--;
+	job_ring->uio_fd = 0;
+	if (job_ring->register_base_addr == NULL)
+		return;
+
+	/* Unmap the PCI memory resource of device */
+	if (munmap(job_ring->register_base_addr, job_ring->map_size)) {
+		CAAM_JR_INFO("cannot munmap(%p, 0x%lx): %s",
+			job_ring->register_base_addr,
+			(unsigned long)job_ring->map_size, strerror(errno));
+	} else
+		CAAM_JR_DEBUG("  JR UIO memory unmapped at %p",
+				job_ring->register_base_addr);
+	job_ring->register_base_addr = NULL;
+}
+
+struct
+uio_job_ring *config_job_ring(void)
+{
+	char uio_device_file_name[32];
+	struct uio_job_ring *job_ring = NULL;
+	int i;
+
+	for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
+		if (g_uio_job_ring[i].uio_fd == 0) {
+			job_ring = &g_uio_job_ring[i];
+			g_uio_jr_num++;
+			break;
+		}
+	}
+
+	if (job_ring == NULL) {
+		CAAM_JR_ERR("No free job ring\n");
+		return NULL;
+	}
+
+	/* Find UIO device created by SEC kernel driver for this job ring. */
+	memset(uio_device_file_name, 0, sizeof(uio_device_file_name));
+
+	sprintf(uio_device_file_name, "%s%d", SEC_UIO_DEVICE_FILE_NAME,
+		job_ring->uio_minor_number);
+
+	/* Open device file */
+	job_ring->uio_fd = open(uio_device_file_name, O_RDWR);
+	SEC_ASSERT(job_ring->uio_fd > 0, NULL,
+		"Failed to open UIO device file for job ring %d",
+		job_ring->jr_id);
+
+	CAAM_JR_INFO("Open device(%s) file for job ring=%d , uio_fd = %d",
+		uio_device_file_name, job_ring->jr_id, job_ring->uio_fd);
+
+	ASSERT(job_ring->register_base_addr == NULL);
+	job_ring->register_base_addr = uio_map_registers(
+			job_ring->uio_fd, job_ring->uio_minor_number,
+			SEC_UIO_MAP_ID, &job_ring->map_size);
+
+	SEC_ASSERT(job_ring->register_base_addr != NULL, NULL,
+		"Failed to map SEC registers");
+	return job_ring;
+}
+
+int
+sec_configure(void)
+{
+	char uio_name[32];
+	int config_jr_no = 0, jr_id = -1;
+	int uio_minor_number = -1;
+	int ret;
+	DIR *d = NULL;
+	struct dirent *dir;
+
+	d = opendir(SEC_UIO_DEVICE_SYS_ATTR_PATH);
+	if (d == NULL) {
+		printf("\nError opening directory '%s': %s\n",
+			SEC_UIO_DEVICE_SYS_ATTR_PATH, strerror(errno));
+		return -1;
+	}
+
+	/* Iterate through all subdirs */
+	while ((dir = readdir(d)) != NULL) {
+		if (!strncmp(dir->d_name, ".", 1) ||
+				!strncmp(dir->d_name, "..", 2))
+			continue;
+
+		if (file_name_match_extract
+			(dir->d_name, "uio", &uio_minor_number)) {
+		/*
+		 * Open file uioX/name and read first line which contains
+		 * the name for the device. Based on the name check if this
+		 * UIO device is UIO device for job ring with id jr_id.
+		 */
+			memset(uio_name, 0, sizeof(uio_name));
+			ret = file_read_first_line(SEC_UIO_DEVICE_SYS_ATTR_PATH,
+					dir->d_name, "name", uio_name);
+			CAAM_JR_INFO("sec device uio name: %s", uio_name);
+			SEC_ASSERT(ret == 0, -1, "file_read_first_line failed");
+
+			if (file_name_match_extract(uio_name,
+						SEC_UIO_DEVICE_NAME,
+						&jr_id)) {
+				g_uio_job_ring[config_jr_no].jr_id = jr_id;
+				g_uio_job_ring[config_jr_no].uio_minor_number =
+							uio_minor_number;
+				CAAM_JR_INFO("Detected logical JRID:%d", jr_id);
+				config_jr_no++;
+
+				/* todo  find the actual ring id
+				 * OF_FULLNAME=/soc/crypto@1700000/jr@20000
+				 */
+			}
+		}
+	}
+	closedir(d);
+
+	if (config_jr_no == 0) {
+		CAAM_JR_ERR("! No SEC Job Rings assigned for userspace usage!");
+		return 0;
+	}
+	CAAM_JR_INFO("Total JR detected =%d", config_jr_no);
+	return config_jr_no;
+}
+
+int
+sec_cleanup(void)
+{
+	int i;
+	struct uio_job_ring *job_ring;
+
+	for (i = 0; i < g_uio_jr_num; i++) {
+		job_ring = &g_uio_job_ring[i];
+		/* munmap SEC's register memory */
+		if (job_ring->register_base_addr) {
+			munmap(job_ring->register_base_addr,
+				job_ring->map_size);
+			job_ring->register_base_addr = NULL;
+		}
+		/* I need to close the fd after shutdown UIO commands need to be
+		 * sent using the fd
+		 */
+		if (job_ring->uio_fd != 0) {
+			CAAM_JR_INFO(
+			"Closed device file for job ring %d , fd = %d",
+			job_ring->jr_id, job_ring->uio_fd);
+			close(job_ring->uio_fd);
+		}
+	}
+	return 0;
+}
diff --git a/drivers/crypto/caam_jr/meson.build b/drivers/crypto/caam_jr/meson.build
new file mode 100644
index 00000000..99b71aef
--- /dev/null
+++ b/drivers/crypto/caam_jr/meson.build
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2018 NXP
+
+if host_machine.system() != 'linux'
+        build = false
+endif
+
+deps += ['bus_vdev', 'bus_dpaa', 'security']
+sources = files('caam_jr_capabilities.c',
+		'caam_jr_hw.c',
+		'caam_jr_uio.c',
+		'caam_jr.c')
+
+allow_experimental_apis = true
+
+includes += include_directories('../dpaa2_sec/')
+includes += include_directories('../../bus/dpaa/include/')
diff --git a/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map b/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map
new file mode 100644
index 00000000..521e51f4
--- /dev/null
+++ b/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map
@@ -0,0 +1,4 @@
+DPDK_18.11 {
+
+	local: *;
+};
diff --git a/drivers/crypto/dpaa2_sec/Makefile b/drivers/crypto/dpaa2_sec/Makefile
index da3d8f84..f537f76a 100644
--- a/drivers/crypto/dpaa2_sec/Makefile
+++ b/drivers/crypto/dpaa2_sec/Makefile
@@ -4,13 +4,6 @@
 #
 
 include $(RTE_SDK)/mk/rte.vars.mk
-
-ifneq ($(MAKECMDGOALS),clean)
-ifneq ($(CONFIG_RTE_LIBRTE_SECURITY),y)
-$(error "RTE_LIBRTE_SECURITY is required to build RTE_LIBRTE_PMD_DPAA2_SEC")
-endif
-endif
-
 #
 # library name
 #
@@ -20,7 +13,6 @@ LIB = librte_pmd_dpaa2_sec.a
 CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
-CFLAGS += -D _GNU_SOURCE
 
 ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y)
 ifeq ($(shell test $(GCC_VERSION) -gt 70 && echo 1), 1)
@@ -41,7 +33,7 @@ CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
 EXPORT_MAP := rte_pmd_dpaa2_sec_version.map
 
 # library version
-LIBABIVER := 1
+LIBABIVER := 2
 
 # library source files
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += dpaa2_sec_dpseci.c
@@ -51,5 +43,6 @@ LDLIBS += -lrte_bus_fslmc
 LDLIBS += -lrte_mempool_dpaa2
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_cryptodev
+LDLIBS += -lrte_common_dpaax
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
index 2a3c61c6..6095c602 100644
--- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
+++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016 NXP
+ *   Copyright 2016-2018 NXP
  *
  */
 
@@ -10,7 +10,6 @@
 
 #include <rte_mbuf.h>
 #include <rte_cryptodev.h>
-#include <rte_security_driver.h>
 #include <rte_malloc.h>
 #include <rte_memcpy.h>
 #include <rte_string_fns.h>
@@ -24,10 +23,12 @@
 #include <dpaa2_hw_pvt.h>
 #include <dpaa2_hw_dpio.h>
 #include <dpaa2_hw_mempool.h>
+#include <fsl_dpopr.h>
 #include <fsl_dpseci.h>
 #include <fsl_mc_sys.h>
 
 #include "dpaa2_sec_priv.h"
+#include "dpaa2_sec_event.h"
 #include "dpaa2_sec_logs.h"
 
 /* Required types */
@@ -35,6 +36,7 @@ typedef uint64_t	dma_addr_t;
 
 /* RTA header files */
 #include <hw/desc/ipsec.h>
+#include <hw/desc/pdcp.h>
 #include <hw/desc/algo.h>
 
 /* Minimum job descriptor consists of a oneword job descriptor HEADER and
@@ -62,11 +64,87 @@ static uint8_t cryptodev_driver_id;
 int dpaa2_logtype_sec;
 
 static inline int
+build_proto_compound_fd(dpaa2_sec_session *sess,
+	       struct rte_crypto_op *op,
+	       struct qbman_fd *fd, uint16_t bpid)
+{
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	struct ctxt_priv *priv = sess->ctxt;
+	struct qbman_fle *fle, *ip_fle, *op_fle;
+	struct sec_flow_context *flc;
+	struct rte_mbuf *src_mbuf = sym_op->m_src;
+	struct rte_mbuf *dst_mbuf = sym_op->m_dst;
+	int retval;
+
+	if (!dst_mbuf)
+		dst_mbuf = src_mbuf;
+
+	/* Save the shared descriptor */
+	flc = &priv->flc_desc[0].flc;
+
+	/* we are using the first FLE entry to store Mbuf */
+	retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
+	if (retval) {
+		DPAA2_SEC_ERR("Memory alloc failed");
+		return -1;
+	}
+	memset(fle, 0, FLE_POOL_BUF_SIZE);
+	DPAA2_SET_FLE_ADDR(fle, (size_t)op);
+	DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
+
+	op_fle = fle + 1;
+	ip_fle = fle + 2;
+
+	if (likely(bpid < MAX_BPID)) {
+		DPAA2_SET_FD_BPID(fd, bpid);
+		DPAA2_SET_FLE_BPID(op_fle, bpid);
+		DPAA2_SET_FLE_BPID(ip_fle, bpid);
+	} else {
+		DPAA2_SET_FD_IVP(fd);
+		DPAA2_SET_FLE_IVP(op_fle);
+		DPAA2_SET_FLE_IVP(ip_fle);
+	}
+
+	/* Configure FD as a FRAME LIST */
+	DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
+	DPAA2_SET_FD_COMPOUND_FMT(fd);
+	DPAA2_SET_FD_FLC(fd, (ptrdiff_t)flc);
+
+	/* Configure Output FLE with dst mbuf data  */
+	DPAA2_SET_FLE_ADDR(op_fle, DPAA2_MBUF_VADDR_TO_IOVA(dst_mbuf));
+	DPAA2_SET_FLE_OFFSET(op_fle, dst_mbuf->data_off);
+	DPAA2_SET_FLE_LEN(op_fle, dst_mbuf->buf_len);
+
+	/* Configure Input FLE with src mbuf data */
+	DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_MBUF_VADDR_TO_IOVA(src_mbuf));
+	DPAA2_SET_FLE_OFFSET(ip_fle, src_mbuf->data_off);
+	DPAA2_SET_FLE_LEN(ip_fle, src_mbuf->pkt_len);
+
+	DPAA2_SET_FD_LEN(fd, ip_fle->length);
+	DPAA2_SET_FLE_FIN(ip_fle);
+
+#ifdef ENABLE_HFN_OVERRIDE
+	if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
+		/*enable HFN override override */
+		DPAA2_SET_FLE_INTERNAL_JD(ip_fle, sess->pdcp.hfn_ovd);
+		DPAA2_SET_FLE_INTERNAL_JD(op_fle, sess->pdcp.hfn_ovd);
+		DPAA2_SET_FD_INTERNAL_JD(fd, sess->pdcp.hfn_ovd);
+	}
+#endif
+
+	return 0;
+
+}
+
+static inline int
 build_proto_fd(dpaa2_sec_session *sess,
 	       struct rte_crypto_op *op,
 	       struct qbman_fd *fd, uint16_t bpid)
 {
 	struct rte_crypto_sym_op *sym_op = op->sym;
+	if (sym_op->m_dst)
+		return build_proto_compound_fd(sess, op, fd, bpid);
+
 	struct ctxt_priv *priv = sess->ctxt;
 	struct sec_flow_context *flc;
 	struct rte_mbuf *mbuf = sym_op->m_src;
@@ -1124,6 +1202,9 @@ build_sec_fd(struct rte_crypto_op *op,
 		case DPAA2_SEC_IPSEC:
 			ret = build_proto_fd(sess, op, fd, bpid);
 			break;
+		case DPAA2_SEC_PDCP:
+			ret = build_proto_compound_fd(sess, op, fd, bpid);
+			break;
 		case DPAA2_SEC_HASH_CIPHER:
 		default:
 			DPAA2_SEC_ERR("error: Unsupported session");
@@ -1145,6 +1226,7 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
 	struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
 	struct qbman_swp *swp;
 	uint16_t num_tx = 0;
+	uint32_t flags[MAX_TX_RING_SLOTS] = {0};
 	/*todo - need to support multiple buffer pools */
 	uint16_t bpid;
 	struct rte_mempool *mb_pool;
@@ -1172,9 +1254,19 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
 	swp = DPAA2_PER_LCORE_PORTAL;
 
 	while (nb_ops) {
-		frames_to_send = (nb_ops >> 3) ? MAX_TX_RING_SLOTS : nb_ops;
+		frames_to_send = (nb_ops > dpaa2_eqcr_size) ?
+			dpaa2_eqcr_size : nb_ops;
 
 		for (loop = 0; loop < frames_to_send; loop++) {
+			if ((*ops)->sym->m_src->seqn) {
+			 uint8_t dqrr_index = (*ops)->sym->m_src->seqn - 1;
+
+			 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA | dqrr_index;
+			 DPAA2_PER_LCORE_DQRR_SIZE--;
+			 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
+			 (*ops)->sym->m_src->seqn = DPAA2_INVALID_MBUF_SEQN;
+			}
+
 			/*Clear the unused FD fields before sending*/
 			memset(&fd_arr[loop], 0, sizeof(struct qbman_fd));
 			mb_pool = (*ops)->sym->m_src->pool;
@@ -1191,7 +1283,7 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
 		while (loop < frames_to_send) {
 			loop += qbman_swp_enqueue_multiple(swp, &eqdesc,
 							&fd_arr[loop],
-							NULL,
+							&flags[loop],
 							frames_to_send - loop);
 		}
 
@@ -1216,6 +1308,9 @@ sec_simple_fd_to_mbuf(const struct qbman_fd *fd, __rte_unused uint8_t id)
 		DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)),
 		rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 
+	diff = len - mbuf->pkt_len;
+	mbuf->pkt_len += diff;
+	mbuf->data_len += diff;
 	op = (struct rte_crypto_op *)(size_t)mbuf->buf_iova;
 	mbuf->buf_iova = op->sym->aead.digest.phys_addr;
 	op->sym->aead.digest.phys_addr = 0L;
@@ -1226,9 +1321,6 @@ sec_simple_fd_to_mbuf(const struct qbman_fd *fd, __rte_unused uint8_t id)
 		mbuf->data_off += SEC_FLC_DHR_OUTBOUND;
 	else
 		mbuf->data_off += SEC_FLC_DHR_INBOUND;
-	diff = len - mbuf->pkt_len;
-	mbuf->pkt_len += diff;
-	mbuf->data_len += diff;
 
 	return op;
 }
@@ -1273,6 +1365,16 @@ sec_fd_to_mbuf(const struct qbman_fd *fd, uint8_t driver_id)
 	} else
 		dst = src;
 
+	if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
+		dpaa2_sec_session *sess = (dpaa2_sec_session *)
+			get_sec_session_private_data(op->sym->sec_session);
+		if (sess->ctxt_type == DPAA2_SEC_IPSEC) {
+			uint16_t len = DPAA2_GET_FD_LEN(fd);
+			dst->pkt_len = len;
+			dst->data_len = len;
+		}
+	}
+
 	DPAA2_SEC_DP_DEBUG("mbuf %p BMAN buf addr %p,"
 		" fdaddr =%" PRIx64 " bpid =%d meta =%d off =%d, len =%d\n",
 		(void *)dst,
@@ -1321,8 +1423,8 @@ dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 
 	qbman_pull_desc_clear(&pulldesc);
 	qbman_pull_desc_set_numframes(&pulldesc,
-				      (nb_ops > DPAA2_DQRR_RING_SIZE) ?
-				      DPAA2_DQRR_RING_SIZE : nb_ops);
+				      (nb_ops > dpaa2_dqrr_size) ?
+				      dpaa2_dqrr_size : nb_ops);
 	qbman_pull_desc_set_fq(&pulldesc, fqid);
 	qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 				    (dma_addr_t)DPAA2_VADDR_TO_IOVA(dq_storage),
@@ -2099,6 +2201,7 @@ dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev,
 		return -1;
 	}
 
+	memset(session, 0, sizeof(dpaa2_sec_session));
 	/* Default IV length = 0 */
 	session->iv.length = 0;
 
@@ -2139,107 +2242,127 @@ dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev,
 }
 
 static int
-dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
-			    struct rte_security_session_conf *conf,
-			    void *sess)
+dpaa2_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform,
+			dpaa2_sec_session *session,
+			struct alginfo *aeaddata)
 {
-	struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
-	struct rte_crypto_auth_xform *auth_xform;
-	struct rte_crypto_cipher_xform *cipher_xform;
-	dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
-	struct ctxt_priv *priv;
-	struct ipsec_encap_pdb encap_pdb;
-	struct ipsec_decap_pdb decap_pdb;
-	struct alginfo authdata, cipherdata;
-	int bufsize;
-	struct sec_flow_context *flc;
-
 	PMD_INIT_FUNC_TRACE();
 
-	if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
-		cipher_xform = &conf->crypto_xform->cipher;
-		auth_xform = &conf->crypto_xform->next->auth;
-	} else {
-		auth_xform = &conf->crypto_xform->auth;
-		cipher_xform = &conf->crypto_xform->next->cipher;
+	session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
+					       RTE_CACHE_LINE_SIZE);
+	if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
+		DPAA2_SEC_ERR("No Memory for aead key");
+		return -1;
 	}
-	priv = (struct ctxt_priv *)rte_zmalloc(NULL,
-				sizeof(struct ctxt_priv) +
-				sizeof(struct sec_flc_desc),
-				RTE_CACHE_LINE_SIZE);
+	memcpy(session->aead_key.data, aead_xform->key.data,
+	       aead_xform->key.length);
 
-	if (priv == NULL) {
-		DPAA2_SEC_ERR("No memory for priv CTXT");
-		return -ENOMEM;
-	}
+	session->digest_length = aead_xform->digest_length;
+	session->aead_key.length = aead_xform->key.length;
 
-	flc = &priv->flc_desc[0].flc;
+	aeaddata->key = (size_t)session->aead_key.data;
+	aeaddata->keylen = session->aead_key.length;
+	aeaddata->key_enc_flags = 0;
+	aeaddata->key_type = RTA_DATA_IMM;
 
-	session->ctxt_type = DPAA2_SEC_IPSEC;
-	session->cipher_key.data = rte_zmalloc(NULL,
-					       cipher_xform->key.length,
-					       RTE_CACHE_LINE_SIZE);
-	if (session->cipher_key.data == NULL &&
-			cipher_xform->key.length > 0) {
-		DPAA2_SEC_ERR("No Memory for cipher key");
-		rte_free(priv);
-		return -ENOMEM;
+	switch (aead_xform->algo) {
+	case RTE_CRYPTO_AEAD_AES_GCM:
+		aeaddata->algtype = OP_ALG_ALGSEL_AES;
+		aeaddata->algmode = OP_ALG_AAI_GCM;
+		session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
+		break;
+	case RTE_CRYPTO_AEAD_AES_CCM:
+		aeaddata->algtype = OP_ALG_ALGSEL_AES;
+		aeaddata->algmode = OP_ALG_AAI_CCM;
+		session->aead_alg = RTE_CRYPTO_AEAD_AES_CCM;
+		break;
+	default:
+		DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
+			      aead_xform->algo);
+		return -1;
 	}
+	session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
+				DIR_ENC : DIR_DEC;
 
-	session->cipher_key.length = cipher_xform->key.length;
-	session->auth_key.data = rte_zmalloc(NULL,
-					auth_xform->key.length,
-					RTE_CACHE_LINE_SIZE);
-	if (session->auth_key.data == NULL &&
-			auth_xform->key.length > 0) {
-		DPAA2_SEC_ERR("No Memory for auth key");
-		rte_free(session->cipher_key.data);
-		rte_free(priv);
-		return -ENOMEM;
+	return 0;
+}
+
+static int
+dpaa2_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform,
+	struct rte_crypto_auth_xform *auth_xform,
+	dpaa2_sec_session *session,
+	struct alginfo *cipherdata,
+	struct alginfo *authdata)
+{
+	if (cipher_xform) {
+		session->cipher_key.data = rte_zmalloc(NULL,
+						       cipher_xform->key.length,
+						       RTE_CACHE_LINE_SIZE);
+		if (session->cipher_key.data == NULL &&
+				cipher_xform->key.length > 0) {
+			DPAA2_SEC_ERR("No Memory for cipher key");
+			return -ENOMEM;
+		}
+
+		session->cipher_key.length = cipher_xform->key.length;
+		memcpy(session->cipher_key.data, cipher_xform->key.data,
+				cipher_xform->key.length);
+		session->cipher_alg = cipher_xform->algo;
+	} else {
+		session->cipher_key.data = NULL;
+		session->cipher_key.length = 0;
+		session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
+	}
+
+	if (auth_xform) {
+		session->auth_key.data = rte_zmalloc(NULL,
+						auth_xform->key.length,
+						RTE_CACHE_LINE_SIZE);
+		if (session->auth_key.data == NULL &&
+				auth_xform->key.length > 0) {
+			DPAA2_SEC_ERR("No Memory for auth key");
+			return -ENOMEM;
+		}
+		session->auth_key.length = auth_xform->key.length;
+		memcpy(session->auth_key.data, auth_xform->key.data,
+				auth_xform->key.length);
+		session->auth_alg = auth_xform->algo;
+	} else {
+		session->auth_key.data = NULL;
+		session->auth_key.length = 0;
+		session->auth_alg = RTE_CRYPTO_AUTH_NULL;
 	}
-	session->auth_key.length = auth_xform->key.length;
-	memcpy(session->cipher_key.data, cipher_xform->key.data,
-			cipher_xform->key.length);
-	memcpy(session->auth_key.data, auth_xform->key.data,
-			auth_xform->key.length);
 
-	authdata.key = (size_t)session->auth_key.data;
-	authdata.keylen = session->auth_key.length;
-	authdata.key_enc_flags = 0;
-	authdata.key_type = RTA_DATA_IMM;
-	switch (auth_xform->algo) {
+	authdata->key = (size_t)session->auth_key.data;
+	authdata->keylen = session->auth_key.length;
+	authdata->key_enc_flags = 0;
+	authdata->key_type = RTA_DATA_IMM;
+	switch (session->auth_alg) {
 	case RTE_CRYPTO_AUTH_SHA1_HMAC:
-		authdata.algtype = OP_PCL_IPSEC_HMAC_SHA1_96;
-		authdata.algmode = OP_ALG_AAI_HMAC;
-		session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
+		authdata->algtype = OP_PCL_IPSEC_HMAC_SHA1_96;
+		authdata->algmode = OP_ALG_AAI_HMAC;
 		break;
 	case RTE_CRYPTO_AUTH_MD5_HMAC:
-		authdata.algtype = OP_PCL_IPSEC_HMAC_MD5_96;
-		authdata.algmode = OP_ALG_AAI_HMAC;
-		session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
+		authdata->algtype = OP_PCL_IPSEC_HMAC_MD5_96;
+		authdata->algmode = OP_ALG_AAI_HMAC;
 		break;
 	case RTE_CRYPTO_AUTH_SHA256_HMAC:
-		authdata.algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128;
-		authdata.algmode = OP_ALG_AAI_HMAC;
-		session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
+		authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128;
+		authdata->algmode = OP_ALG_AAI_HMAC;
 		break;
 	case RTE_CRYPTO_AUTH_SHA384_HMAC:
-		authdata.algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192;
-		authdata.algmode = OP_ALG_AAI_HMAC;
-		session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
+		authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192;
+		authdata->algmode = OP_ALG_AAI_HMAC;
 		break;
 	case RTE_CRYPTO_AUTH_SHA512_HMAC:
-		authdata.algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256;
-		authdata.algmode = OP_ALG_AAI_HMAC;
-		session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
+		authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256;
+		authdata->algmode = OP_ALG_AAI_HMAC;
 		break;
 	case RTE_CRYPTO_AUTH_AES_CMAC:
-		authdata.algtype = OP_PCL_IPSEC_AES_CMAC_96;
-		session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
+		authdata->algtype = OP_PCL_IPSEC_AES_CMAC_96;
 		break;
 	case RTE_CRYPTO_AUTH_NULL:
-		authdata.algtype = OP_PCL_IPSEC_HMAC_NULL;
-		session->auth_alg = RTE_CRYPTO_AUTH_NULL;
+		authdata->algtype = OP_PCL_IPSEC_HMAC_NULL;
 		break;
 	case RTE_CRYPTO_AUTH_SHA224_HMAC:
 	case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
@@ -2255,50 +2378,119 @@ dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
 	case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 	case RTE_CRYPTO_AUTH_ZUC_EIA3:
 		DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
-			      auth_xform->algo);
-		goto out;
+			      session->auth_alg);
+		return -1;
 	default:
 		DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
-			      auth_xform->algo);
-		goto out;
+			      session->auth_alg);
+		return -1;
 	}
-	cipherdata.key = (size_t)session->cipher_key.data;
-	cipherdata.keylen = session->cipher_key.length;
-	cipherdata.key_enc_flags = 0;
-	cipherdata.key_type = RTA_DATA_IMM;
+	cipherdata->key = (size_t)session->cipher_key.data;
+	cipherdata->keylen = session->cipher_key.length;
+	cipherdata->key_enc_flags = 0;
+	cipherdata->key_type = RTA_DATA_IMM;
 
-	switch (cipher_xform->algo) {
+	switch (session->cipher_alg) {
 	case RTE_CRYPTO_CIPHER_AES_CBC:
-		cipherdata.algtype = OP_PCL_IPSEC_AES_CBC;
-		cipherdata.algmode = OP_ALG_AAI_CBC;
-		session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
+		cipherdata->algtype = OP_PCL_IPSEC_AES_CBC;
+		cipherdata->algmode = OP_ALG_AAI_CBC;
 		break;
 	case RTE_CRYPTO_CIPHER_3DES_CBC:
-		cipherdata.algtype = OP_PCL_IPSEC_3DES;
-		cipherdata.algmode = OP_ALG_AAI_CBC;
-		session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
+		cipherdata->algtype = OP_PCL_IPSEC_3DES;
+		cipherdata->algmode = OP_ALG_AAI_CBC;
 		break;
 	case RTE_CRYPTO_CIPHER_AES_CTR:
-		cipherdata.algtype = OP_PCL_IPSEC_AES_CTR;
-		cipherdata.algmode = OP_ALG_AAI_CTR;
-		session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
+		cipherdata->algtype = OP_PCL_IPSEC_AES_CTR;
+		cipherdata->algmode = OP_ALG_AAI_CTR;
 		break;
 	case RTE_CRYPTO_CIPHER_NULL:
-		cipherdata.algtype = OP_PCL_IPSEC_NULL;
+		cipherdata->algtype = OP_PCL_IPSEC_NULL;
 		break;
 	case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 	case RTE_CRYPTO_CIPHER_3DES_ECB:
 	case RTE_CRYPTO_CIPHER_AES_ECB:
 	case RTE_CRYPTO_CIPHER_KASUMI_F8:
 		DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
-			      cipher_xform->algo);
-		goto out;
+			      session->cipher_alg);
+		return -1;
 	default:
 		DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
-			      cipher_xform->algo);
+			      session->cipher_alg);
+		return -1;
+	}
+
+	return 0;
+}
+
+#ifdef RTE_LIBRTE_SECURITY_TEST
+static uint8_t aes_cbc_iv[] = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f };
+#endif
+
+static int
+dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
+			    struct rte_security_session_conf *conf,
+			    void *sess)
+{
+	struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
+	struct rte_crypto_cipher_xform *cipher_xform = NULL;
+	struct rte_crypto_auth_xform *auth_xform = NULL;
+	struct rte_crypto_aead_xform *aead_xform = NULL;
+	dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
+	struct ctxt_priv *priv;
+	struct ipsec_encap_pdb encap_pdb;
+	struct ipsec_decap_pdb decap_pdb;
+	struct alginfo authdata, cipherdata;
+	int bufsize;
+	struct sec_flow_context *flc;
+	struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
+	int ret = -1;
+
+	PMD_INIT_FUNC_TRACE();
+
+	priv = (struct ctxt_priv *)rte_zmalloc(NULL,
+				sizeof(struct ctxt_priv) +
+				sizeof(struct sec_flc_desc),
+				RTE_CACHE_LINE_SIZE);
+
+	if (priv == NULL) {
+		DPAA2_SEC_ERR("No memory for priv CTXT");
+		return -ENOMEM;
+	}
+
+	priv->fle_pool = dev_priv->fle_pool;
+	flc = &priv->flc_desc[0].flc;
+
+	memset(session, 0, sizeof(dpaa2_sec_session));
+
+	if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
+		cipher_xform = &conf->crypto_xform->cipher;
+		if (conf->crypto_xform->next)
+			auth_xform = &conf->crypto_xform->next->auth;
+		ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
+					session, &cipherdata, &authdata);
+	} else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
+		auth_xform = &conf->crypto_xform->auth;
+		if (conf->crypto_xform->next)
+			cipher_xform = &conf->crypto_xform->next->cipher;
+		ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
+					session, &cipherdata, &authdata);
+	} else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
+		aead_xform = &conf->crypto_xform->aead;
+		ret = dpaa2_sec_ipsec_aead_init(aead_xform,
+					session, &cipherdata);
+	} else {
+		DPAA2_SEC_ERR("XFORM not specified");
+		ret = -EINVAL;
+		goto out;
+	}
+	if (ret) {
+		DPAA2_SEC_ERR("Failed to process xform");
 		goto out;
 	}
 
+	session->ctxt_type = DPAA2_SEC_IPSEC;
 	if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
 		struct ip ip4_hdr;
 
@@ -2310,7 +2502,7 @@ dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
 		ip4_hdr.ip_id = 0;
 		ip4_hdr.ip_off = 0;
 		ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
-		ip4_hdr.ip_p = 0x32;
+		ip4_hdr.ip_p = IPPROTO_ESP;
 		ip4_hdr.ip_sum = 0;
 		ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
 		ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
@@ -2322,13 +2514,14 @@ dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
 		encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
 			PDBOPTS_ESP_OIHI_PDB_INL |
 			PDBOPTS_ESP_IVSRC |
-			PDBHMO_ESP_ENCAP_DTTL;
+			PDBHMO_ESP_ENCAP_DTTL |
+			PDBHMO_ESP_SNR;
 		encap_pdb.spi = ipsec_xform->spi;
 		encap_pdb.ip_hdr_len = sizeof(struct ip);
 
 		session->dir = DIR_ENC;
 		bufsize = cnstr_shdsc_ipsec_new_encap(priv->flc_desc[0].desc,
-				1, 0, &encap_pdb,
+				1, 0, SHR_SERIAL, &encap_pdb,
 				(uint8_t *)&ip4_hdr,
 				&cipherdata, &authdata);
 	} else if (ipsec_xform->direction ==
@@ -2338,7 +2531,8 @@ dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
 		decap_pdb.options = sizeof(struct ip) << 16;
 		session->dir = DIR_DEC;
 		bufsize = cnstr_shdsc_ipsec_new_decap(priv->flc_desc[0].desc,
-				1, 0, &decap_pdb, &cipherdata, &authdata);
+				1, 0, SHR_SERIAL,
+				&decap_pdb, &cipherdata, &authdata);
 	} else
 		goto out;
 
@@ -2372,6 +2566,244 @@ out:
 	rte_free(session->auth_key.data);
 	rte_free(session->cipher_key.data);
 	rte_free(priv);
+	return ret;
+}
+
+static int
+dpaa2_sec_set_pdcp_session(struct rte_cryptodev *dev,
+			   struct rte_security_session_conf *conf,
+			   void *sess)
+{
+	struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp;
+	struct rte_crypto_sym_xform *xform = conf->crypto_xform;
+	struct rte_crypto_auth_xform *auth_xform = NULL;
+	struct rte_crypto_cipher_xform *cipher_xform;
+	dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
+	struct ctxt_priv *priv;
+	struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
+	struct alginfo authdata, cipherdata;
+	int bufsize = -1;
+	struct sec_flow_context *flc;
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+	int swap = true;
+#else
+	int swap = false;
+#endif
+
+	PMD_INIT_FUNC_TRACE();
+
+	memset(session, 0, sizeof(dpaa2_sec_session));
+
+	priv = (struct ctxt_priv *)rte_zmalloc(NULL,
+				sizeof(struct ctxt_priv) +
+				sizeof(struct sec_flc_desc),
+				RTE_CACHE_LINE_SIZE);
+
+	if (priv == NULL) {
+		DPAA2_SEC_ERR("No memory for priv CTXT");
+		return -ENOMEM;
+	}
+
+	priv->fle_pool = dev_priv->fle_pool;
+	flc = &priv->flc_desc[0].flc;
+
+	/* find xfrm types */
+	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
+		cipher_xform = &xform->cipher;
+	} else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
+		   xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
+		session->ext_params.aead_ctxt.auth_cipher_text = true;
+		cipher_xform = &xform->cipher;
+		auth_xform = &xform->next->auth;
+	} else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
+		   xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
+		session->ext_params.aead_ctxt.auth_cipher_text = false;
+		cipher_xform = &xform->next->cipher;
+		auth_xform = &xform->auth;
+	} else {
+		DPAA2_SEC_ERR("Invalid crypto type");
+		return -EINVAL;
+	}
+
+	session->ctxt_type = DPAA2_SEC_PDCP;
+	if (cipher_xform) {
+		session->cipher_key.data = rte_zmalloc(NULL,
+					       cipher_xform->key.length,
+					       RTE_CACHE_LINE_SIZE);
+		if (session->cipher_key.data == NULL &&
+				cipher_xform->key.length > 0) {
+			DPAA2_SEC_ERR("No Memory for cipher key");
+			rte_free(priv);
+			return -ENOMEM;
+		}
+		session->cipher_key.length = cipher_xform->key.length;
+		memcpy(session->cipher_key.data, cipher_xform->key.data,
+			cipher_xform->key.length);
+		session->dir =
+			(cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
+					DIR_ENC : DIR_DEC;
+		session->cipher_alg = cipher_xform->algo;
+	} else {
+		session->cipher_key.data = NULL;
+		session->cipher_key.length = 0;
+		session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
+		session->dir = DIR_ENC;
+	}
+
+	session->pdcp.domain = pdcp_xform->domain;
+	session->pdcp.bearer = pdcp_xform->bearer;
+	session->pdcp.pkt_dir = pdcp_xform->pkt_dir;
+	session->pdcp.sn_size = pdcp_xform->sn_size;
+#ifdef ENABLE_HFN_OVERRIDE
+	session->pdcp.hfn_ovd = pdcp_xform->hfn_ovd;
+#endif
+	session->pdcp.hfn = pdcp_xform->hfn;
+	session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
+
+	cipherdata.key = (size_t)session->cipher_key.data;
+	cipherdata.keylen = session->cipher_key.length;
+	cipherdata.key_enc_flags = 0;
+	cipherdata.key_type = RTA_DATA_IMM;
+
+	switch (session->cipher_alg) {
+	case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
+		cipherdata.algtype = PDCP_CIPHER_TYPE_SNOW;
+		break;
+	case RTE_CRYPTO_CIPHER_ZUC_EEA3:
+		cipherdata.algtype = PDCP_CIPHER_TYPE_ZUC;
+		break;
+	case RTE_CRYPTO_CIPHER_AES_CTR:
+		cipherdata.algtype = PDCP_CIPHER_TYPE_AES;
+		break;
+	case RTE_CRYPTO_CIPHER_NULL:
+		cipherdata.algtype = PDCP_CIPHER_TYPE_NULL;
+		break;
+	default:
+		DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
+			      session->cipher_alg);
+		goto out;
+	}
+
+	/* Auth is only applicable for control mode operation. */
+	if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
+		if (pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_5) {
+			DPAA2_SEC_ERR(
+				"PDCP Seq Num size should be 5 bits for cmode");
+			goto out;
+		}
+		if (auth_xform) {
+			session->auth_key.data = rte_zmalloc(NULL,
+							auth_xform->key.length,
+							RTE_CACHE_LINE_SIZE);
+			if (session->auth_key.data == NULL &&
+					auth_xform->key.length > 0) {
+				DPAA2_SEC_ERR("No Memory for auth key");
+				rte_free(session->cipher_key.data);
+				rte_free(priv);
+				return -ENOMEM;
+			}
+			session->auth_key.length = auth_xform->key.length;
+			memcpy(session->auth_key.data, auth_xform->key.data,
+					auth_xform->key.length);
+			session->auth_alg = auth_xform->algo;
+		} else {
+			session->auth_key.data = NULL;
+			session->auth_key.length = 0;
+			session->auth_alg = RTE_CRYPTO_AUTH_NULL;
+		}
+		authdata.key = (size_t)session->auth_key.data;
+		authdata.keylen = session->auth_key.length;
+		authdata.key_enc_flags = 0;
+		authdata.key_type = RTA_DATA_IMM;
+
+		switch (session->auth_alg) {
+		case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
+			authdata.algtype = PDCP_AUTH_TYPE_SNOW;
+			break;
+		case RTE_CRYPTO_AUTH_ZUC_EIA3:
+			authdata.algtype = PDCP_AUTH_TYPE_ZUC;
+			break;
+		case RTE_CRYPTO_AUTH_AES_CMAC:
+			authdata.algtype = PDCP_AUTH_TYPE_AES;
+			break;
+		case RTE_CRYPTO_AUTH_NULL:
+			authdata.algtype = PDCP_AUTH_TYPE_NULL;
+			break;
+		default:
+			DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
+				      session->auth_alg);
+			goto out;
+		}
+
+		if (session->dir == DIR_ENC)
+			bufsize = cnstr_shdsc_pdcp_c_plane_encap(
+					priv->flc_desc[0].desc, 1, swap,
+					pdcp_xform->hfn,
+					pdcp_xform->bearer,
+					pdcp_xform->pkt_dir,
+					pdcp_xform->hfn_threshold,
+					&cipherdata, &authdata,
+					0);
+		else if (session->dir == DIR_DEC)
+			bufsize = cnstr_shdsc_pdcp_c_plane_decap(
+					priv->flc_desc[0].desc, 1, swap,
+					pdcp_xform->hfn,
+					pdcp_xform->bearer,
+					pdcp_xform->pkt_dir,
+					pdcp_xform->hfn_threshold,
+					&cipherdata, &authdata,
+					0);
+	} else {
+		if (session->dir == DIR_ENC)
+			bufsize = cnstr_shdsc_pdcp_u_plane_encap(
+					priv->flc_desc[0].desc, 1, swap,
+					(enum pdcp_sn_size)pdcp_xform->sn_size,
+					pdcp_xform->hfn,
+					pdcp_xform->bearer,
+					pdcp_xform->pkt_dir,
+					pdcp_xform->hfn_threshold,
+					&cipherdata, 0);
+		else if (session->dir == DIR_DEC)
+			bufsize = cnstr_shdsc_pdcp_u_plane_decap(
+					priv->flc_desc[0].desc, 1, swap,
+					(enum pdcp_sn_size)pdcp_xform->sn_size,
+					pdcp_xform->hfn,
+					pdcp_xform->bearer,
+					pdcp_xform->pkt_dir,
+					pdcp_xform->hfn_threshold,
+					&cipherdata, 0);
+	}
+
+	if (bufsize < 0) {
+		DPAA2_SEC_ERR("Crypto: Invalid buffer length");
+		goto out;
+	}
+
+	/* Enable the stashing control bit */
+	DPAA2_SET_FLC_RSC(flc);
+	flc->word2_rflc_31_0 = lower_32_bits(
+			(size_t)&(((struct dpaa2_sec_qp *)
+			dev->data->queue_pairs[0])->rx_vq) | 0x14);
+	flc->word3_rflc_63_32 = upper_32_bits(
+			(size_t)&(((struct dpaa2_sec_qp *)
+			dev->data->queue_pairs[0])->rx_vq));
+
+	flc->word1_sdl = (uint8_t)bufsize;
+
+	/* Set EWS bit i.e. enable write-safe */
+	DPAA2_SET_FLC_EWS(flc);
+	/* Set BS = 1 i.e reuse input buffers as output buffers */
+	DPAA2_SET_FLC_REUSE_BS(flc);
+	/* Set FF = 10; reuse input buffers if they provide sufficient space */
+	DPAA2_SET_FLC_REUSE_FF(flc);
+
+	session->ctxt = priv;
+
+	return 0;
+out:
+	rte_free(session->auth_key.data);
+	rte_free(session->cipher_key.data);
+	rte_free(priv);
 	return -1;
 }
 
@@ -2397,6 +2829,10 @@ dpaa2_sec_security_session_create(void *dev,
 		break;
 	case RTE_SECURITY_PROTOCOL_MACSEC:
 		return -ENOTSUP;
+	case RTE_SECURITY_PROTOCOL_PDCP:
+		ret = dpaa2_sec_set_pdcp_session(cdev, conf,
+				sess_private_data);
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -2686,6 +3122,129 @@ void dpaa2_sec_stats_reset(struct rte_cryptodev *dev)
 	}
 }
 
+static void __attribute__((hot))
+dpaa2_sec_process_parallel_event(struct qbman_swp *swp,
+				 const struct qbman_fd *fd,
+				 const struct qbman_result *dq,
+				 struct dpaa2_queue *rxq,
+				 struct rte_event *ev)
+{
+	/* Prefetching mbuf */
+	rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
+		rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
+
+	/* Prefetching ipsec crypto_op stored in priv data of mbuf */
+	rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
+
+	ev->flow_id = rxq->ev.flow_id;
+	ev->sub_event_type = rxq->ev.sub_event_type;
+	ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
+	ev->op = RTE_EVENT_OP_NEW;
+	ev->sched_type = rxq->ev.sched_type;
+	ev->queue_id = rxq->ev.queue_id;
+	ev->priority = rxq->ev.priority;
+	ev->event_ptr = sec_fd_to_mbuf(fd, ((struct rte_cryptodev *)
+				(rxq->dev))->driver_id);
+
+	qbman_swp_dqrr_consume(swp, dq);
+}
+static void
+dpaa2_sec_process_atomic_event(struct qbman_swp *swp __attribute__((unused)),
+				 const struct qbman_fd *fd,
+				 const struct qbman_result *dq,
+				 struct dpaa2_queue *rxq,
+				 struct rte_event *ev)
+{
+	uint8_t dqrr_index;
+	struct rte_crypto_op *crypto_op = (struct rte_crypto_op *)ev->event_ptr;
+	/* Prefetching mbuf */
+	rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
+		rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
+
+	/* Prefetching ipsec crypto_op stored in priv data of mbuf */
+	rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
+
+	ev->flow_id = rxq->ev.flow_id;
+	ev->sub_event_type = rxq->ev.sub_event_type;
+	ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
+	ev->op = RTE_EVENT_OP_NEW;
+	ev->sched_type = rxq->ev.sched_type;
+	ev->queue_id = rxq->ev.queue_id;
+	ev->priority = rxq->ev.priority;
+
+	ev->event_ptr = sec_fd_to_mbuf(fd, ((struct rte_cryptodev *)
+				(rxq->dev))->driver_id);
+	dqrr_index = qbman_get_dqrr_idx(dq);
+	crypto_op->sym->m_src->seqn = dqrr_index + 1;
+	DPAA2_PER_LCORE_DQRR_SIZE++;
+	DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
+	DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = crypto_op->sym->m_src;
+}
+
+int
+dpaa2_sec_eventq_attach(const struct rte_cryptodev *dev,
+		int qp_id,
+		uint16_t dpcon_id,
+		const struct rte_event *event)
+{
+	struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
+	struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
+	struct dpaa2_sec_qp *qp = dev->data->queue_pairs[qp_id];
+	struct dpseci_rx_queue_cfg cfg;
+	int ret;
+
+	if (event->sched_type == RTE_SCHED_TYPE_PARALLEL)
+		qp->rx_vq.cb = dpaa2_sec_process_parallel_event;
+	else if (event->sched_type == RTE_SCHED_TYPE_ATOMIC)
+		qp->rx_vq.cb = dpaa2_sec_process_atomic_event;
+	else
+		return -EINVAL;
+
+	memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
+	cfg.options = DPSECI_QUEUE_OPT_DEST;
+	cfg.dest_cfg.dest_type = DPSECI_DEST_DPCON;
+	cfg.dest_cfg.dest_id = dpcon_id;
+	cfg.dest_cfg.priority = event->priority;
+
+	cfg.options |= DPSECI_QUEUE_OPT_USER_CTX;
+	cfg.user_ctx = (size_t)(qp);
+	if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) {
+		cfg.options |= DPSECI_QUEUE_OPT_ORDER_PRESERVATION;
+		cfg.order_preservation_en = 1;
+	}
+	ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
+				  qp_id, &cfg);
+	if (ret) {
+		RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
+		return ret;
+	}
+
+	memcpy(&qp->rx_vq.ev, event, sizeof(struct rte_event));
+
+	return 0;
+}
+
+int
+dpaa2_sec_eventq_detach(const struct rte_cryptodev *dev,
+			int qp_id)
+{
+	struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
+	struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
+	struct dpseci_rx_queue_cfg cfg;
+	int ret;
+
+	memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
+	cfg.options = DPSECI_QUEUE_OPT_DEST;
+	cfg.dest_cfg.dest_type = DPSECI_DEST_NONE;
+
+	ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
+				  qp_id, &cfg);
+	if (ret)
+		RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
+
+	return ret;
+}
+
 static struct rte_cryptodev_ops crypto_ops = {
 	.dev_configure	      = dpaa2_sec_dev_configure,
 	.dev_start	      = dpaa2_sec_dev_start,
@@ -2708,7 +3267,7 @@ dpaa2_sec_capabilities_get(void *device __rte_unused)
 	return dpaa2_sec_security_cap;
 }
 
-struct rte_security_ops dpaa2_sec_security_ops = {
+static const struct rte_security_ops dpaa2_sec_security_ops = {
 	.session_create = dpaa2_sec_security_session_create,
 	.session_update = NULL,
 	.session_stats_get = NULL,
@@ -2843,7 +3402,7 @@ init_error:
 }
 
 static int
-cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv,
+cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused,
 			  struct rte_dpaa2_device *dpaa2_dev)
 {
 	struct rte_cryptodev *cryptodev;
@@ -2871,7 +3430,6 @@ cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv,
 
 	dpaa2_dev->cryptodev = cryptodev;
 	cryptodev->device = &dpaa2_dev->device;
-	cryptodev->device->driver = &dpaa2_drv->driver;
 
 	/* init user callbacks */
 	TAILQ_INIT(&(cryptodev->link_intr_cbs));
diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_event.h b/drivers/crypto/dpaa2_sec/dpaa2_sec_event.h
new file mode 100644
index 00000000..97709942
--- /dev/null
+++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_event.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ *
+ */
+
+#ifndef _DPAA2_SEC_EVENT_H_
+#define _DPAA2_SEC_EVENT_H_
+
+int
+dpaa2_sec_eventq_attach(const struct rte_cryptodev *dev,
+		int qp_id,
+		uint16_t dpcon_id,
+		const struct rte_event *event);
+
+int dpaa2_sec_eventq_detach(const struct rte_cryptodev *dev,
+		int qp_id);
+
+#endif /* _DPAA2_SEC_EVENT_H_ */
diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h
index d015be1e..51751103 100644
--- a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h
+++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h
@@ -8,6 +8,8 @@
 #ifndef _RTE_DPAA2_SEC_PMD_PRIVATE_H_
 #define _RTE_DPAA2_SEC_PMD_PRIVATE_H_
 
+#include <rte_security_driver.h>
+
 #define CRYPTODEV_NAME_DPAA2_SEC_PMD	crypto_dpaa2_sec
 /**< NXP DPAA2 - SEC PMD device name */
 
@@ -135,6 +137,19 @@ struct dpaa2_sec_aead_ctxt {
 	uint8_t auth_cipher_text;       /**< Authenticate/cipher ordering */
 };
 
+/*
+ * The structure is to be filled by user for PDCP Protocol
+ */
+struct dpaa2_pdcp_ctxt {
+	enum rte_security_pdcp_domain domain; /*!< Data/Control mode*/
+	int8_t bearer;	/*!< PDCP bearer ID */
+	int8_t pkt_dir;/*!< PDCP Frame Direction 0:UL 1:DL*/
+	int8_t hfn_ovd;/*!< Overwrite HFN per packet*/
+	uint32_t hfn;	/*!< Hyper Frame Number */
+	uint32_t hfn_threshold;	/*!< HFN Threashold for key renegotiation */
+	uint8_t sn_size;	/*!< Sequence number size, 7/12/15 */
+};
+
 typedef struct dpaa2_sec_session_entry {
 	void *ctxt;
 	uint8_t ctxt_type;
@@ -158,15 +173,20 @@ typedef struct dpaa2_sec_session_entry {
 			} auth_key;
 		};
 	};
-	struct {
-		uint16_t length; /**< IV length in bytes */
-		uint16_t offset; /**< IV offset in bytes */
-	} iv;
-	uint16_t digest_length;
-	uint8_t status;
 	union {
-		struct dpaa2_sec_aead_ctxt aead_ctxt;
-	} ext_params;
+		struct {
+			struct {
+				uint16_t length; /**< IV length in bytes */
+				uint16_t offset; /**< IV offset in bytes */
+			} iv;
+			uint16_t digest_length;
+			uint8_t status;
+			union {
+				struct dpaa2_sec_aead_ctxt aead_ctxt;
+			} ext_params;
+		};
+		struct dpaa2_pdcp_ctxt pdcp;
+	};
 } dpaa2_sec_session;
 
 static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = {
@@ -390,6 +410,162 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = {
 	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 };
 
+static const struct rte_cryptodev_capabilities dpaa2_pdcp_capabilities[] = {
+	{	/* SNOW 3G (UIA2) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 4,
+					.max = 4,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* SNOW 3G (UEA2) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* AES CTR */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_CTR,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 8
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* NULL (AUTH) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_NULL,
+				.block_size = 1,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.iv_size = { 0 }
+			}, },
+		}, },
+	},
+	{	/* NULL (CIPHER) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_NULL,
+				.block_size = 1,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				}
+			}, },
+		}, }
+	},
+	{	/* ZUC (EEA3) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_ZUC_EEA3,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* ZUC (EIA3) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_ZUC_EIA3,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 4,
+					.max = 4,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+
+	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
 static const struct rte_security_capability dpaa2_sec_security_cap[] = {
 	{ /* IPsec Lookaside Protocol offload ESP Transport Egress */
 		.action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
@@ -413,6 +589,24 @@ static const struct rte_security_capability dpaa2_sec_security_cap[] = {
 		},
 		.crypto_capabilities = dpaa2_sec_capabilities
 	},
+	{ /* PDCP Lookaside Protocol offload Data */
+		.action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
+		.protocol = RTE_SECURITY_PROTOCOL_PDCP,
+		.pdcp = {
+			.domain = RTE_SECURITY_PDCP_MODE_DATA,
+			.capa_flags = 0
+		},
+		.crypto_capabilities = dpaa2_pdcp_capabilities
+	},
+	{ /* PDCP Lookaside Protocol offload Control */
+		.action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
+		.protocol = RTE_SECURITY_PROTOCOL_PDCP,
+		.pdcp = {
+			.domain = RTE_SECURITY_PDCP_MODE_CONTROL,
+			.capa_flags = 0
+		},
+		.crypto_capabilities = dpaa2_pdcp_capabilities
+	},
 	{
 		.action = RTE_SECURITY_ACTION_TYPE_NONE
 	}
diff --git a/drivers/crypto/dpaa2_sec/hw/desc.h b/drivers/crypto/dpaa2_sec/hw/desc.h
index e9255832..5d99dd8a 100644
--- a/drivers/crypto/dpaa2_sec/hw/desc.h
+++ b/drivers/crypto/dpaa2_sec/hw/desc.h
@@ -588,7 +588,7 @@
 #define OP_PCLID_TLS10_PRF	(0x09 << OP_PCLID_SHIFT)
 #define OP_PCLID_TLS11_PRF	(0x0a << OP_PCLID_SHIFT)
 #define OP_PCLID_TLS12_PRF	(0x0b << OP_PCLID_SHIFT)
-#define OP_PCLID_DTLS10_PRF	(0x0c << OP_PCLID_SHIFT)
+#define OP_PCLID_DTLS_PRF	(0x0c << OP_PCLID_SHIFT)
 #define OP_PCLID_PUBLICKEYPAIR	(0x14 << OP_PCLID_SHIFT)
 #define OP_PCLID_DSASIGN	(0x15 << OP_PCLID_SHIFT)
 #define OP_PCLID_DSAVERIFY	(0x16 << OP_PCLID_SHIFT)
@@ -612,7 +612,7 @@
 #define OP_PCLID_TLS10		(0x09 << OP_PCLID_SHIFT)
 #define OP_PCLID_TLS11		(0x0a << OP_PCLID_SHIFT)
 #define OP_PCLID_TLS12		(0x0b << OP_PCLID_SHIFT)
-#define OP_PCLID_DTLS10		(0x0c << OP_PCLID_SHIFT)
+#define OP_PCLID_DTLS		(0x0c << OP_PCLID_SHIFT)
 #define OP_PCLID_BLOB		(0x0d << OP_PCLID_SHIFT)
 #define OP_PCLID_IPSEC_NEW	(0x11 << OP_PCLID_SHIFT)
 #define OP_PCLID_3G_DCRC	(0x31 << OP_PCLID_SHIFT)
@@ -665,643 +665,179 @@
 
 #define OP_PCL_SRTP_HMAC_SHA1_160		 0x0007
 
-/* For SSL 3.0 - OP_PCLID_SSL30 */
-#define OP_PCL_SSL30_AES_128_CBC_SHA		 0x002f
-#define OP_PCL_SSL30_AES_128_CBC_SHA_2		 0x0030
-#define OP_PCL_SSL30_AES_128_CBC_SHA_3		 0x0031
-#define OP_PCL_SSL30_AES_128_CBC_SHA_4		 0x0032
-#define OP_PCL_SSL30_AES_128_CBC_SHA_5		 0x0033
-#define OP_PCL_SSL30_AES_128_CBC_SHA_6		 0x0034
-#define OP_PCL_SSL30_AES_128_CBC_SHA_7		 0x008c
-#define OP_PCL_SSL30_AES_128_CBC_SHA_8		 0x0090
-#define OP_PCL_SSL30_AES_128_CBC_SHA_9		 0x0094
-#define OP_PCL_SSL30_AES_128_CBC_SHA_10		 0xc004
-#define OP_PCL_SSL30_AES_128_CBC_SHA_11		 0xc009
-#define OP_PCL_SSL30_AES_128_CBC_SHA_12		 0xc00e
-#define OP_PCL_SSL30_AES_128_CBC_SHA_13		 0xc013
-#define OP_PCL_SSL30_AES_128_CBC_SHA_14		 0xc018
-#define OP_PCL_SSL30_AES_128_CBC_SHA_15		 0xc01d
-#define OP_PCL_SSL30_AES_128_CBC_SHA_16		 0xc01e
-#define OP_PCL_SSL30_AES_128_CBC_SHA_17		 0xc01f
-
-#define OP_PCL_SSL30_AES_256_CBC_SHA		 0x0035
-#define OP_PCL_SSL30_AES_256_CBC_SHA_2		 0x0036
-#define OP_PCL_SSL30_AES_256_CBC_SHA_3		 0x0037
-#define OP_PCL_SSL30_AES_256_CBC_SHA_4		 0x0038
-#define OP_PCL_SSL30_AES_256_CBC_SHA_5		 0x0039
-#define OP_PCL_SSL30_AES_256_CBC_SHA_6		 0x003a
-#define OP_PCL_SSL30_AES_256_CBC_SHA_7		 0x008d
-#define OP_PCL_SSL30_AES_256_CBC_SHA_8		 0x0091
-#define OP_PCL_SSL30_AES_256_CBC_SHA_9		 0x0095
-#define OP_PCL_SSL30_AES_256_CBC_SHA_10		 0xc005
-#define OP_PCL_SSL30_AES_256_CBC_SHA_11		 0xc00a
-#define OP_PCL_SSL30_AES_256_CBC_SHA_12		 0xc00f
-#define OP_PCL_SSL30_AES_256_CBC_SHA_13		 0xc014
-#define OP_PCL_SSL30_AES_256_CBC_SHA_14		 0xc019
-#define OP_PCL_SSL30_AES_256_CBC_SHA_15		 0xc020
-#define OP_PCL_SSL30_AES_256_CBC_SHA_16		 0xc021
-#define OP_PCL_SSL30_AES_256_CBC_SHA_17		 0xc022
-
-#define OP_PCL_SSL30_AES_128_GCM_SHA256_1	 0x009C
-#define OP_PCL_SSL30_AES_256_GCM_SHA384_1	 0x009D
-#define OP_PCL_SSL30_AES_128_GCM_SHA256_2	 0x009E
-#define OP_PCL_SSL30_AES_256_GCM_SHA384_2	 0x009F
-#define OP_PCL_SSL30_AES_128_GCM_SHA256_3	 0x00A0
-#define OP_PCL_SSL30_AES_256_GCM_SHA384_3	 0x00A1
-#define OP_PCL_SSL30_AES_128_GCM_SHA256_4	 0x00A2
-#define OP_PCL_SSL30_AES_256_GCM_SHA384_4	 0x00A3
-#define OP_PCL_SSL30_AES_128_GCM_SHA256_5	 0x00A4
-#define OP_PCL_SSL30_AES_256_GCM_SHA384_5	 0x00A5
-#define OP_PCL_SSL30_AES_128_GCM_SHA256_6	 0x00A6
-
-#define OP_PCL_TLS_DH_ANON_AES_256_GCM_SHA384	 0x00A7
-#define OP_PCL_TLS_PSK_AES_128_GCM_SHA256	 0x00A8
-#define OP_PCL_TLS_PSK_AES_256_GCM_SHA384	 0x00A9
-#define OP_PCL_TLS_DHE_PSK_AES_128_GCM_SHA256	 0x00AA
-#define OP_PCL_TLS_DHE_PSK_AES_256_GCM_SHA384	 0x00AB
-#define OP_PCL_TLS_RSA_PSK_AES_128_GCM_SHA256	 0x00AC
-#define OP_PCL_TLS_RSA_PSK_AES_256_GCM_SHA384	 0x00AD
-#define OP_PCL_TLS_PSK_AES_128_CBC_SHA256	 0x00AE
-#define OP_PCL_TLS_PSK_AES_256_CBC_SHA384	 0x00AF
-#define OP_PCL_TLS_DHE_PSK_AES_128_CBC_SHA256	 0x00B2
-#define OP_PCL_TLS_DHE_PSK_AES_256_CBC_SHA384	 0x00B3
-#define OP_PCL_TLS_RSA_PSK_AES_128_CBC_SHA256	 0x00B6
-#define OP_PCL_TLS_RSA_PSK_AES_256_CBC_SHA384	 0x00B7
-
-#define OP_PCL_SSL30_3DES_EDE_CBC_MD5		 0x0023
-
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA		 0x001f
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_2		 0x008b
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_3		 0x008f
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_4		 0x0093
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_5		 0x000a
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_6		 0x000d
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_7		 0x0010
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_8		 0x0013
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_9		 0x0016
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_10	 0x001b
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_11	 0xc003
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_12	 0xc008
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_13	 0xc00d
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_14	 0xc012
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_15	 0xc017
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_16	 0xc01a
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_17	 0xc01b
-#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_18	 0xc01c
-
-#define OP_PCL_SSL30_DES40_CBC_MD5		 0x0029
-
-#define OP_PCL_SSL30_DES_CBC_MD5		 0x0022
-
-#define OP_PCL_SSL30_DES40_CBC_SHA		 0x0008
-#define OP_PCL_SSL30_DES40_CBC_SHA_2		 0x000b
-#define OP_PCL_SSL30_DES40_CBC_SHA_3		 0x000e
-#define OP_PCL_SSL30_DES40_CBC_SHA_4		 0x0011
-#define OP_PCL_SSL30_DES40_CBC_SHA_5		 0x0014
-#define OP_PCL_SSL30_DES40_CBC_SHA_6		 0x0019
-#define OP_PCL_SSL30_DES40_CBC_SHA_7		 0x0026
-
-#define OP_PCL_SSL30_DES_CBC_SHA		 0x001e
-#define OP_PCL_SSL30_DES_CBC_SHA_2		 0x0009
-#define OP_PCL_SSL30_DES_CBC_SHA_3		 0x000c
-#define OP_PCL_SSL30_DES_CBC_SHA_4		 0x000f
-#define OP_PCL_SSL30_DES_CBC_SHA_5		 0x0012
-#define OP_PCL_SSL30_DES_CBC_SHA_6		 0x0015
-#define OP_PCL_SSL30_DES_CBC_SHA_7		 0x001a
-
-#define OP_PCL_SSL30_RC4_128_MD5		 0x0024
-#define OP_PCL_SSL30_RC4_128_MD5_2		 0x0004
-#define OP_PCL_SSL30_RC4_128_MD5_3		 0x0018
-
-#define OP_PCL_SSL30_RC4_40_MD5			 0x002b
-#define OP_PCL_SSL30_RC4_40_MD5_2		 0x0003
-#define OP_PCL_SSL30_RC4_40_MD5_3		 0x0017
-
-#define OP_PCL_SSL30_RC4_128_SHA		 0x0020
-#define OP_PCL_SSL30_RC4_128_SHA_2		 0x008a
-#define OP_PCL_SSL30_RC4_128_SHA_3		 0x008e
-#define OP_PCL_SSL30_RC4_128_SHA_4		 0x0092
-#define OP_PCL_SSL30_RC4_128_SHA_5		 0x0005
-#define OP_PCL_SSL30_RC4_128_SHA_6		 0xc002
-#define OP_PCL_SSL30_RC4_128_SHA_7		 0xc007
-#define OP_PCL_SSL30_RC4_128_SHA_8		 0xc00c
-#define OP_PCL_SSL30_RC4_128_SHA_9		 0xc011
-#define OP_PCL_SSL30_RC4_128_SHA_10		 0xc016
-
-#define OP_PCL_SSL30_RC4_40_SHA			 0x0028
-
-/* For TLS 1.0 - OP_PCLID_TLS10 */
-#define OP_PCL_TLS10_AES_128_CBC_SHA		 0x002f
-#define OP_PCL_TLS10_AES_128_CBC_SHA_2		 0x0030
-#define OP_PCL_TLS10_AES_128_CBC_SHA_3		 0x0031
-#define OP_PCL_TLS10_AES_128_CBC_SHA_4		 0x0032
-#define OP_PCL_TLS10_AES_128_CBC_SHA_5		 0x0033
-#define OP_PCL_TLS10_AES_128_CBC_SHA_6		 0x0034
-#define OP_PCL_TLS10_AES_128_CBC_SHA_7		 0x008c
-#define OP_PCL_TLS10_AES_128_CBC_SHA_8		 0x0090
-#define OP_PCL_TLS10_AES_128_CBC_SHA_9		 0x0094
-#define OP_PCL_TLS10_AES_128_CBC_SHA_10		 0xc004
-#define OP_PCL_TLS10_AES_128_CBC_SHA_11		 0xc009
-#define OP_PCL_TLS10_AES_128_CBC_SHA_12		 0xc00e
-#define OP_PCL_TLS10_AES_128_CBC_SHA_13		 0xc013
-#define OP_PCL_TLS10_AES_128_CBC_SHA_14		 0xc018
-#define OP_PCL_TLS10_AES_128_CBC_SHA_15		 0xc01d
-#define OP_PCL_TLS10_AES_128_CBC_SHA_16		 0xc01e
-#define OP_PCL_TLS10_AES_128_CBC_SHA_17		 0xc01f
-
-#define OP_PCL_TLS10_AES_256_CBC_SHA		 0x0035
-#define OP_PCL_TLS10_AES_256_CBC_SHA_2		 0x0036
-#define OP_PCL_TLS10_AES_256_CBC_SHA_3		 0x0037
-#define OP_PCL_TLS10_AES_256_CBC_SHA_4		 0x0038
-#define OP_PCL_TLS10_AES_256_CBC_SHA_5		 0x0039
-#define OP_PCL_TLS10_AES_256_CBC_SHA_6		 0x003a
-#define OP_PCL_TLS10_AES_256_CBC_SHA_7		 0x008d
-#define OP_PCL_TLS10_AES_256_CBC_SHA_8		 0x0091
-#define OP_PCL_TLS10_AES_256_CBC_SHA_9		 0x0095
-#define OP_PCL_TLS10_AES_256_CBC_SHA_10		 0xc005
-#define OP_PCL_TLS10_AES_256_CBC_SHA_11		 0xc00a
-#define OP_PCL_TLS10_AES_256_CBC_SHA_12		 0xc00f
-#define OP_PCL_TLS10_AES_256_CBC_SHA_13		 0xc014
-#define OP_PCL_TLS10_AES_256_CBC_SHA_14		 0xc019
-#define OP_PCL_TLS10_AES_256_CBC_SHA_15		 0xc020
-#define OP_PCL_TLS10_AES_256_CBC_SHA_16		 0xc021
-#define OP_PCL_TLS10_AES_256_CBC_SHA_17		 0xc022
-
-#define OP_PCL_TLS_ECDHE_ECDSA_AES_128_CBC_SHA256  0xC023
-#define OP_PCL_TLS_ECDHE_ECDSA_AES_256_CBC_SHA384  0xC024
-#define OP_PCL_TLS_ECDH_ECDSA_AES_128_CBC_SHA256   0xC025
-#define OP_PCL_TLS_ECDH_ECDSA_AES_256_CBC_SHA384   0xC026
-#define OP_PCL_TLS_ECDHE_RSA_AES_128_CBC_SHA256	   0xC027
-#define OP_PCL_TLS_ECDHE_RSA_AES_256_CBC_SHA384	   0xC028
-#define OP_PCL_TLS_ECDH_RSA_AES_128_CBC_SHA256	   0xC029
-#define OP_PCL_TLS_ECDH_RSA_AES_256_CBC_SHA384	   0xC02A
-#define OP_PCL_TLS_ECDHE_ECDSA_AES_128_GCM_SHA256  0xC02B
-#define OP_PCL_TLS_ECDHE_ECDSA_AES_256_GCM_SHA384  0xC02C
-#define OP_PCL_TLS_ECDH_ECDSA_AES_128_GCM_SHA256   0xC02D
-#define OP_PCL_TLS_ECDH_ECDSA_AES_256_GCM_SHA384   0xC02E
-#define OP_PCL_TLS_ECDHE_RSA_AES_128_GCM_SHA256	   0xC02F
-#define OP_PCL_TLS_ECDHE_RSA_AES_256_GCM_SHA384	   0xC030
-#define OP_PCL_TLS_ECDH_RSA_AES_128_GCM_SHA256	   0xC031
-#define OP_PCL_TLS_ECDH_RSA_AES_256_GCM_SHA384	   0xC032
-#define OP_PCL_TLS_ECDHE_PSK_RC4_128_SHA	   0xC033
-#define OP_PCL_TLS_ECDHE_PSK_3DES_EDE_CBC_SHA	   0xC034
-#define OP_PCL_TLS_ECDHE_PSK_AES_128_CBC_SHA	   0xC035
-#define OP_PCL_TLS_ECDHE_PSK_AES_256_CBC_SHA	   0xC036
-#define OP_PCL_TLS_ECDHE_PSK_AES_128_CBC_SHA256	   0xC037
-#define OP_PCL_TLS_ECDHE_PSK_AES_256_CBC_SHA384	   0xC038
-
-/* #define OP_PCL_TLS10_3DES_EDE_CBC_MD5	0x0023 */
-
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA		 0x001f
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_2		 0x008b
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_3		 0x008f
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_4		 0x0093
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_5		 0x000a
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_6		 0x000d
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_7		 0x0010
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_8		 0x0013
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_9		 0x0016
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_10	 0x001b
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_11	 0xc003
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_12	 0xc008
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_13	 0xc00d
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_14	 0xc012
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_15	 0xc017
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_16	 0xc01a
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_17	 0xc01b
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_18	 0xc01c
-
-#define OP_PCL_TLS10_DES40_CBC_MD5		 0x0029
-
-#define OP_PCL_TLS10_DES_CBC_MD5		 0x0022
-
-#define OP_PCL_TLS10_DES40_CBC_SHA		 0x0008
-#define OP_PCL_TLS10_DES40_CBC_SHA_2		 0x000b
-#define OP_PCL_TLS10_DES40_CBC_SHA_3		 0x000e
-#define OP_PCL_TLS10_DES40_CBC_SHA_4		 0x0011
-#define OP_PCL_TLS10_DES40_CBC_SHA_5		 0x0014
-#define OP_PCL_TLS10_DES40_CBC_SHA_6		 0x0019
-#define OP_PCL_TLS10_DES40_CBC_SHA_7		 0x0026
-
-#define OP_PCL_TLS10_DES_CBC_SHA		 0x001e
-#define OP_PCL_TLS10_DES_CBC_SHA_2		 0x0009
-#define OP_PCL_TLS10_DES_CBC_SHA_3		 0x000c
-#define OP_PCL_TLS10_DES_CBC_SHA_4		 0x000f
-#define OP_PCL_TLS10_DES_CBC_SHA_5		 0x0012
-#define OP_PCL_TLS10_DES_CBC_SHA_6		 0x0015
-#define OP_PCL_TLS10_DES_CBC_SHA_7		 0x001a
-
-#define OP_PCL_TLS10_RC4_128_MD5		 0x0024
-#define OP_PCL_TLS10_RC4_128_MD5_2		 0x0004
-#define OP_PCL_TLS10_RC4_128_MD5_3		 0x0018
-
-#define OP_PCL_TLS10_RC4_40_MD5			 0x002b
-#define OP_PCL_TLS10_RC4_40_MD5_2		 0x0003
-#define OP_PCL_TLS10_RC4_40_MD5_3		 0x0017
-
-#define OP_PCL_TLS10_RC4_128_SHA		 0x0020
-#define OP_PCL_TLS10_RC4_128_SHA_2		 0x008a
-#define OP_PCL_TLS10_RC4_128_SHA_3		 0x008e
-#define OP_PCL_TLS10_RC4_128_SHA_4		 0x0092
-#define OP_PCL_TLS10_RC4_128_SHA_5		 0x0005
-#define OP_PCL_TLS10_RC4_128_SHA_6		 0xc002
-#define OP_PCL_TLS10_RC4_128_SHA_7		 0xc007
-#define OP_PCL_TLS10_RC4_128_SHA_8		 0xc00c
-#define OP_PCL_TLS10_RC4_128_SHA_9		 0xc011
-#define OP_PCL_TLS10_RC4_128_SHA_10		 0xc016
-
-#define OP_PCL_TLS10_RC4_40_SHA			 0x0028
-
-#define OP_PCL_TLS10_3DES_EDE_CBC_MD5		 0xff23
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA160	 0xff30
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA224	 0xff34
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA256	 0xff36
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA384	 0xff33
-#define OP_PCL_TLS10_3DES_EDE_CBC_SHA512	 0xff35
-#define OP_PCL_TLS10_AES_128_CBC_SHA160		 0xff80
-#define OP_PCL_TLS10_AES_128_CBC_SHA224		 0xff84
-#define OP_PCL_TLS10_AES_128_CBC_SHA256		 0xff86
-#define OP_PCL_TLS10_AES_128_CBC_SHA384		 0xff83
-#define OP_PCL_TLS10_AES_128_CBC_SHA512		 0xff85
-#define OP_PCL_TLS10_AES_192_CBC_SHA160		 0xff20
-#define OP_PCL_TLS10_AES_192_CBC_SHA224		 0xff24
-#define OP_PCL_TLS10_AES_192_CBC_SHA256		 0xff26
-#define OP_PCL_TLS10_AES_192_CBC_SHA384		 0xff23
-#define OP_PCL_TLS10_AES_192_CBC_SHA512		 0xff25
-#define OP_PCL_TLS10_AES_256_CBC_SHA160		 0xff60
-#define OP_PCL_TLS10_AES_256_CBC_SHA224		 0xff64
-#define OP_PCL_TLS10_AES_256_CBC_SHA256		 0xff66
-#define OP_PCL_TLS10_AES_256_CBC_SHA384		 0xff63
-#define OP_PCL_TLS10_AES_256_CBC_SHA512		 0xff65
-
-#define OP_PCL_TLS_PVT_AES_192_CBC_SHA160	 0xff90
-#define OP_PCL_TLS_PVT_AES_192_CBC_SHA384	 0xff93
-#define OP_PCL_TLS_PVT_AES_192_CBC_SHA224	 0xff94
-#define OP_PCL_TLS_PVT_AES_192_CBC_SHA512	 0xff95
-#define OP_PCL_TLS_PVT_AES_192_CBC_SHA256	 0xff96
-#define OP_PCL_TLS_PVT_MASTER_SECRET_PRF_FE	 0xfffe
-#define OP_PCL_TLS_PVT_MASTER_SECRET_PRF_FF	 0xffff
-
-/* For TLS 1.1 - OP_PCLID_TLS11 */
-#define OP_PCL_TLS11_AES_128_CBC_SHA		 0x002f
-#define OP_PCL_TLS11_AES_128_CBC_SHA_2		 0x0030
-#define OP_PCL_TLS11_AES_128_CBC_SHA_3		 0x0031
-#define OP_PCL_TLS11_AES_128_CBC_SHA_4		 0x0032
-#define OP_PCL_TLS11_AES_128_CBC_SHA_5		 0x0033
-#define OP_PCL_TLS11_AES_128_CBC_SHA_6		 0x0034
-#define OP_PCL_TLS11_AES_128_CBC_SHA_7		 0x008c
-#define OP_PCL_TLS11_AES_128_CBC_SHA_8		 0x0090
-#define OP_PCL_TLS11_AES_128_CBC_SHA_9		 0x0094
-#define OP_PCL_TLS11_AES_128_CBC_SHA_10		 0xc004
-#define OP_PCL_TLS11_AES_128_CBC_SHA_11		 0xc009
-#define OP_PCL_TLS11_AES_128_CBC_SHA_12		 0xc00e
-#define OP_PCL_TLS11_AES_128_CBC_SHA_13		 0xc013
-#define OP_PCL_TLS11_AES_128_CBC_SHA_14		 0xc018
-#define OP_PCL_TLS11_AES_128_CBC_SHA_15		 0xc01d
-#define OP_PCL_TLS11_AES_128_CBC_SHA_16		 0xc01e
-#define OP_PCL_TLS11_AES_128_CBC_SHA_17		 0xc01f
-
-#define OP_PCL_TLS11_AES_256_CBC_SHA		 0x0035
-#define OP_PCL_TLS11_AES_256_CBC_SHA_2		 0x0036
-#define OP_PCL_TLS11_AES_256_CBC_SHA_3		 0x0037
-#define OP_PCL_TLS11_AES_256_CBC_SHA_4		 0x0038
-#define OP_PCL_TLS11_AES_256_CBC_SHA_5		 0x0039
-#define OP_PCL_TLS11_AES_256_CBC_SHA_6		 0x003a
-#define OP_PCL_TLS11_AES_256_CBC_SHA_7		 0x008d
-#define OP_PCL_TLS11_AES_256_CBC_SHA_8		 0x0091
-#define OP_PCL_TLS11_AES_256_CBC_SHA_9		 0x0095
-#define OP_PCL_TLS11_AES_256_CBC_SHA_10		 0xc005
-#define OP_PCL_TLS11_AES_256_CBC_SHA_11		 0xc00a
-#define OP_PCL_TLS11_AES_256_CBC_SHA_12		 0xc00f
-#define OP_PCL_TLS11_AES_256_CBC_SHA_13		 0xc014
-#define OP_PCL_TLS11_AES_256_CBC_SHA_14		 0xc019
-#define OP_PCL_TLS11_AES_256_CBC_SHA_15		 0xc020
-#define OP_PCL_TLS11_AES_256_CBC_SHA_16		 0xc021
-#define OP_PCL_TLS11_AES_256_CBC_SHA_17		 0xc022
-
-/* #define OP_PCL_TLS11_3DES_EDE_CBC_MD5	0x0023 */
-
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA		 0x001f
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_2		 0x008b
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_3		 0x008f
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_4		 0x0093
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_5		 0x000a
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_6		 0x000d
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_7		 0x0010
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_8		 0x0013
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_9		 0x0016
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_10	 0x001b
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_11	 0xc003
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_12	 0xc008
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_13	 0xc00d
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_14	 0xc012
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_15	 0xc017
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_16	 0xc01a
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_17	 0xc01b
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_18	 0xc01c
-
-#define OP_PCL_TLS11_DES40_CBC_MD5		 0x0029
-
-#define OP_PCL_TLS11_DES_CBC_MD5		 0x0022
-
-#define OP_PCL_TLS11_DES40_CBC_SHA		 0x0008
-#define OP_PCL_TLS11_DES40_CBC_SHA_2		 0x000b
-#define OP_PCL_TLS11_DES40_CBC_SHA_3		 0x000e
-#define OP_PCL_TLS11_DES40_CBC_SHA_4		 0x0011
-#define OP_PCL_TLS11_DES40_CBC_SHA_5		 0x0014
-#define OP_PCL_TLS11_DES40_CBC_SHA_6		 0x0019
-#define OP_PCL_TLS11_DES40_CBC_SHA_7		 0x0026
-
-#define OP_PCL_TLS11_DES_CBC_SHA		 0x001e
-#define OP_PCL_TLS11_DES_CBC_SHA_2		 0x0009
-#define OP_PCL_TLS11_DES_CBC_SHA_3		 0x000c
-#define OP_PCL_TLS11_DES_CBC_SHA_4		 0x000f
-#define OP_PCL_TLS11_DES_CBC_SHA_5		 0x0012
-#define OP_PCL_TLS11_DES_CBC_SHA_6		 0x0015
-#define OP_PCL_TLS11_DES_CBC_SHA_7		 0x001a
-
-#define OP_PCL_TLS11_RC4_128_MD5		 0x0024
-#define OP_PCL_TLS11_RC4_128_MD5_2		 0x0004
-#define OP_PCL_TLS11_RC4_128_MD5_3		 0x0018
-
-#define OP_PCL_TLS11_RC4_40_MD5			 0x002b
-#define OP_PCL_TLS11_RC4_40_MD5_2		 0x0003
-#define OP_PCL_TLS11_RC4_40_MD5_3		 0x0017
-
-#define OP_PCL_TLS11_RC4_128_SHA		 0x0020
-#define OP_PCL_TLS11_RC4_128_SHA_2		 0x008a
-#define OP_PCL_TLS11_RC4_128_SHA_3		 0x008e
-#define OP_PCL_TLS11_RC4_128_SHA_4		 0x0092
-#define OP_PCL_TLS11_RC4_128_SHA_5		 0x0005
-#define OP_PCL_TLS11_RC4_128_SHA_6		 0xc002
-#define OP_PCL_TLS11_RC4_128_SHA_7		 0xc007
-#define OP_PCL_TLS11_RC4_128_SHA_8		 0xc00c
-#define OP_PCL_TLS11_RC4_128_SHA_9		 0xc011
-#define OP_PCL_TLS11_RC4_128_SHA_10		 0xc016
-
-#define OP_PCL_TLS11_RC4_40_SHA			 0x0028
-
-#define OP_PCL_TLS11_3DES_EDE_CBC_MD5		 0xff23
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA160	 0xff30
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA224	 0xff34
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA256	 0xff36
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA384	 0xff33
-#define OP_PCL_TLS11_3DES_EDE_CBC_SHA512	 0xff35
-#define OP_PCL_TLS11_AES_128_CBC_SHA160		 0xff80
-#define OP_PCL_TLS11_AES_128_CBC_SHA224		 0xff84
-#define OP_PCL_TLS11_AES_128_CBC_SHA256		 0xff86
-#define OP_PCL_TLS11_AES_128_CBC_SHA384		 0xff83
-#define OP_PCL_TLS11_AES_128_CBC_SHA512		 0xff85
-#define OP_PCL_TLS11_AES_192_CBC_SHA160		 0xff20
-#define OP_PCL_TLS11_AES_192_CBC_SHA224		 0xff24
-#define OP_PCL_TLS11_AES_192_CBC_SHA256		 0xff26
-#define OP_PCL_TLS11_AES_192_CBC_SHA384		 0xff23
-#define OP_PCL_TLS11_AES_192_CBC_SHA512		 0xff25
-#define OP_PCL_TLS11_AES_256_CBC_SHA160		 0xff60
-#define OP_PCL_TLS11_AES_256_CBC_SHA224		 0xff64
-#define OP_PCL_TLS11_AES_256_CBC_SHA256		 0xff66
-#define OP_PCL_TLS11_AES_256_CBC_SHA384		 0xff63
-#define OP_PCL_TLS11_AES_256_CBC_SHA512		 0xff65
-
-
-/* For TLS 1.2 - OP_PCLID_TLS12 */
-#define OP_PCL_TLS12_AES_128_CBC_SHA		 0x002f
-#define OP_PCL_TLS12_AES_128_CBC_SHA_2		 0x0030
-#define OP_PCL_TLS12_AES_128_CBC_SHA_3		 0x0031
-#define OP_PCL_TLS12_AES_128_CBC_SHA_4		 0x0032
-#define OP_PCL_TLS12_AES_128_CBC_SHA_5		 0x0033
-#define OP_PCL_TLS12_AES_128_CBC_SHA_6		 0x0034
-#define OP_PCL_TLS12_AES_128_CBC_SHA_7		 0x008c
-#define OP_PCL_TLS12_AES_128_CBC_SHA_8		 0x0090
-#define OP_PCL_TLS12_AES_128_CBC_SHA_9		 0x0094
-#define OP_PCL_TLS12_AES_128_CBC_SHA_10		 0xc004
-#define OP_PCL_TLS12_AES_128_CBC_SHA_11		 0xc009
-#define OP_PCL_TLS12_AES_128_CBC_SHA_12		 0xc00e
-#define OP_PCL_TLS12_AES_128_CBC_SHA_13		 0xc013
-#define OP_PCL_TLS12_AES_128_CBC_SHA_14		 0xc018
-#define OP_PCL_TLS12_AES_128_CBC_SHA_15		 0xc01d
-#define OP_PCL_TLS12_AES_128_CBC_SHA_16		 0xc01e
-#define OP_PCL_TLS12_AES_128_CBC_SHA_17		 0xc01f
-
-#define OP_PCL_TLS12_AES_256_CBC_SHA		 0x0035
-#define OP_PCL_TLS12_AES_256_CBC_SHA_2		 0x0036
-#define OP_PCL_TLS12_AES_256_CBC_SHA_3		 0x0037
-#define OP_PCL_TLS12_AES_256_CBC_SHA_4		 0x0038
-#define OP_PCL_TLS12_AES_256_CBC_SHA_5		 0x0039
-#define OP_PCL_TLS12_AES_256_CBC_SHA_6		 0x003a
-#define OP_PCL_TLS12_AES_256_CBC_SHA_7		 0x008d
-#define OP_PCL_TLS12_AES_256_CBC_SHA_8		 0x0091
-#define OP_PCL_TLS12_AES_256_CBC_SHA_9		 0x0095
-#define OP_PCL_TLS12_AES_256_CBC_SHA_10		 0xc005
-#define OP_PCL_TLS12_AES_256_CBC_SHA_11		 0xc00a
-#define OP_PCL_TLS12_AES_256_CBC_SHA_12		 0xc00f
-#define OP_PCL_TLS12_AES_256_CBC_SHA_13		 0xc014
-#define OP_PCL_TLS12_AES_256_CBC_SHA_14		 0xc019
-#define OP_PCL_TLS12_AES_256_CBC_SHA_15		 0xc020
-#define OP_PCL_TLS12_AES_256_CBC_SHA_16		 0xc021
-#define OP_PCL_TLS12_AES_256_CBC_SHA_17		 0xc022
-
-/* #define OP_PCL_TLS12_3DES_EDE_CBC_MD5	0x0023 */
-
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA		 0x001f
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_2		 0x008b
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_3		 0x008f
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_4		 0x0093
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_5		 0x000a
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_6		 0x000d
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_7		 0x0010
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_8		 0x0013
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_9		 0x0016
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_10	 0x001b
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_11	 0xc003
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_12	 0xc008
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_13	 0xc00d
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_14	 0xc012
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_15	 0xc017
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_16	 0xc01a
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_17	 0xc01b
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_18	 0xc01c
-
-#define OP_PCL_TLS12_DES40_CBC_MD5		 0x0029
-
-#define OP_PCL_TLS12_DES_CBC_MD5		 0x0022
-
-#define OP_PCL_TLS12_DES40_CBC_SHA		 0x0008
-#define OP_PCL_TLS12_DES40_CBC_SHA_2		 0x000b
-#define OP_PCL_TLS12_DES40_CBC_SHA_3		 0x000e
-#define OP_PCL_TLS12_DES40_CBC_SHA_4		 0x0011
-#define OP_PCL_TLS12_DES40_CBC_SHA_5		 0x0014
-#define OP_PCL_TLS12_DES40_CBC_SHA_6		 0x0019
-#define OP_PCL_TLS12_DES40_CBC_SHA_7		 0x0026
-
-#define OP_PCL_TLS12_DES_CBC_SHA		 0x001e
-#define OP_PCL_TLS12_DES_CBC_SHA_2		 0x0009
-#define OP_PCL_TLS12_DES_CBC_SHA_3		 0x000c
-#define OP_PCL_TLS12_DES_CBC_SHA_4		 0x000f
-#define OP_PCL_TLS12_DES_CBC_SHA_5		 0x0012
-#define OP_PCL_TLS12_DES_CBC_SHA_6		 0x0015
-#define OP_PCL_TLS12_DES_CBC_SHA_7		 0x001a
-
-#define OP_PCL_TLS12_RC4_128_MD5		 0x0024
-#define OP_PCL_TLS12_RC4_128_MD5_2		 0x0004
-#define OP_PCL_TLS12_RC4_128_MD5_3		 0x0018
-
-#define OP_PCL_TLS12_RC4_40_MD5			 0x002b
-#define OP_PCL_TLS12_RC4_40_MD5_2		 0x0003
-#define OP_PCL_TLS12_RC4_40_MD5_3		 0x0017
-
-#define OP_PCL_TLS12_RC4_128_SHA		 0x0020
-#define OP_PCL_TLS12_RC4_128_SHA_2		 0x008a
-#define OP_PCL_TLS12_RC4_128_SHA_3		 0x008e
-#define OP_PCL_TLS12_RC4_128_SHA_4		 0x0092
-#define OP_PCL_TLS12_RC4_128_SHA_5		 0x0005
-#define OP_PCL_TLS12_RC4_128_SHA_6		 0xc002
-#define OP_PCL_TLS12_RC4_128_SHA_7		 0xc007
-#define OP_PCL_TLS12_RC4_128_SHA_8		 0xc00c
-#define OP_PCL_TLS12_RC4_128_SHA_9		 0xc011
-#define OP_PCL_TLS12_RC4_128_SHA_10		 0xc016
-
-#define OP_PCL_TLS12_RC4_40_SHA			 0x0028
-
-/* #define OP_PCL_TLS12_AES_128_CBC_SHA256	0x003c */
-#define OP_PCL_TLS12_AES_128_CBC_SHA256_2	 0x003e
-#define OP_PCL_TLS12_AES_128_CBC_SHA256_3	 0x003f
-#define OP_PCL_TLS12_AES_128_CBC_SHA256_4	 0x0040
-#define OP_PCL_TLS12_AES_128_CBC_SHA256_5	 0x0067
-#define OP_PCL_TLS12_AES_128_CBC_SHA256_6	 0x006c
-
-/* #define OP_PCL_TLS12_AES_256_CBC_SHA256	0x003d */
-#define OP_PCL_TLS12_AES_256_CBC_SHA256_2	 0x0068
-#define OP_PCL_TLS12_AES_256_CBC_SHA256_3	 0x0069
-#define OP_PCL_TLS12_AES_256_CBC_SHA256_4	 0x006a
-#define OP_PCL_TLS12_AES_256_CBC_SHA256_5	 0x006b
-#define OP_PCL_TLS12_AES_256_CBC_SHA256_6	 0x006d
-
-/* AEAD_AES_xxx_CCM/GCM remain to be defined... */
-
-#define OP_PCL_TLS12_3DES_EDE_CBC_MD5		 0xff23
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA160	 0xff30
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA224	 0xff34
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA256	 0xff36
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA384	 0xff33
-#define OP_PCL_TLS12_3DES_EDE_CBC_SHA512	 0xff35
-#define OP_PCL_TLS12_AES_128_CBC_SHA160		 0xff80
-#define OP_PCL_TLS12_AES_128_CBC_SHA224		 0xff84
-#define OP_PCL_TLS12_AES_128_CBC_SHA256		 0xff86
-#define OP_PCL_TLS12_AES_128_CBC_SHA384		 0xff83
-#define OP_PCL_TLS12_AES_128_CBC_SHA512		 0xff85
-#define OP_PCL_TLS12_AES_192_CBC_SHA160		 0xff20
-#define OP_PCL_TLS12_AES_192_CBC_SHA224		 0xff24
-#define OP_PCL_TLS12_AES_192_CBC_SHA256		 0xff26
-#define OP_PCL_TLS12_AES_192_CBC_SHA384		 0xff23
-#define OP_PCL_TLS12_AES_192_CBC_SHA512		 0xff25
-#define OP_PCL_TLS12_AES_256_CBC_SHA160		 0xff60
-#define OP_PCL_TLS12_AES_256_CBC_SHA224		 0xff64
-#define OP_PCL_TLS12_AES_256_CBC_SHA256		 0xff66
-#define OP_PCL_TLS12_AES_256_CBC_SHA384		 0xff63
-#define OP_PCL_TLS12_AES_256_CBC_SHA512		 0xff65
-
-/* For DTLS - OP_PCLID_DTLS */
-
-#define OP_PCL_DTLS_AES_128_CBC_SHA		 0x002f
-#define OP_PCL_DTLS_AES_128_CBC_SHA_2		 0x0030
-#define OP_PCL_DTLS_AES_128_CBC_SHA_3		 0x0031
-#define OP_PCL_DTLS_AES_128_CBC_SHA_4		 0x0032
-#define OP_PCL_DTLS_AES_128_CBC_SHA_5		 0x0033
-#define OP_PCL_DTLS_AES_128_CBC_SHA_6		 0x0034
-#define OP_PCL_DTLS_AES_128_CBC_SHA_7		 0x008c
-#define OP_PCL_DTLS_AES_128_CBC_SHA_8		 0x0090
-#define OP_PCL_DTLS_AES_128_CBC_SHA_9		 0x0094
-#define OP_PCL_DTLS_AES_128_CBC_SHA_10		 0xc004
-#define OP_PCL_DTLS_AES_128_CBC_SHA_11		 0xc009
-#define OP_PCL_DTLS_AES_128_CBC_SHA_12		 0xc00e
-#define OP_PCL_DTLS_AES_128_CBC_SHA_13		 0xc013
-#define OP_PCL_DTLS_AES_128_CBC_SHA_14		 0xc018
-#define OP_PCL_DTLS_AES_128_CBC_SHA_15		 0xc01d
-#define OP_PCL_DTLS_AES_128_CBC_SHA_16		 0xc01e
-#define OP_PCL_DTLS_AES_128_CBC_SHA_17		 0xc01f
-
-#define OP_PCL_DTLS_AES_256_CBC_SHA		 0x0035
-#define OP_PCL_DTLS_AES_256_CBC_SHA_2		 0x0036
-#define OP_PCL_DTLS_AES_256_CBC_SHA_3		 0x0037
-#define OP_PCL_DTLS_AES_256_CBC_SHA_4		 0x0038
-#define OP_PCL_DTLS_AES_256_CBC_SHA_5		 0x0039
-#define OP_PCL_DTLS_AES_256_CBC_SHA_6		 0x003a
-#define OP_PCL_DTLS_AES_256_CBC_SHA_7		 0x008d
-#define OP_PCL_DTLS_AES_256_CBC_SHA_8		 0x0091
-#define OP_PCL_DTLS_AES_256_CBC_SHA_9		 0x0095
-#define OP_PCL_DTLS_AES_256_CBC_SHA_10		 0xc005
-#define OP_PCL_DTLS_AES_256_CBC_SHA_11		 0xc00a
-#define OP_PCL_DTLS_AES_256_CBC_SHA_12		 0xc00f
-#define OP_PCL_DTLS_AES_256_CBC_SHA_13		 0xc014
-#define OP_PCL_DTLS_AES_256_CBC_SHA_14		 0xc019
-#define OP_PCL_DTLS_AES_256_CBC_SHA_15		 0xc020
-#define OP_PCL_DTLS_AES_256_CBC_SHA_16		 0xc021
-#define OP_PCL_DTLS_AES_256_CBC_SHA_17		 0xc022
-
-/* #define OP_PCL_DTLS_3DES_EDE_CBC_MD5		0x0023 */
-
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA		 0x001f
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_2		 0x008b
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_3		 0x008f
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_4		 0x0093
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_5		 0x000a
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_6		 0x000d
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_7		 0x0010
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_8		 0x0013
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_9		 0x0016
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_10		 0x001b
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_11		 0xc003
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_12		 0xc008
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_13		 0xc00d
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_14		 0xc012
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_15		 0xc017
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_16		 0xc01a
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_17		 0xc01b
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_18		 0xc01c
-
-#define OP_PCL_DTLS_DES40_CBC_MD5		 0x0029
-
-#define OP_PCL_DTLS_DES_CBC_MD5			 0x0022
-
-#define OP_PCL_DTLS_DES40_CBC_SHA		 0x0008
-#define OP_PCL_DTLS_DES40_CBC_SHA_2		 0x000b
-#define OP_PCL_DTLS_DES40_CBC_SHA_3		 0x000e
-#define OP_PCL_DTLS_DES40_CBC_SHA_4		 0x0011
-#define OP_PCL_DTLS_DES40_CBC_SHA_5		 0x0014
-#define OP_PCL_DTLS_DES40_CBC_SHA_6		 0x0019
-#define OP_PCL_DTLS_DES40_CBC_SHA_7		 0x0026
-
-
-#define OP_PCL_DTLS_DES_CBC_SHA			 0x001e
-#define OP_PCL_DTLS_DES_CBC_SHA_2		 0x0009
-#define OP_PCL_DTLS_DES_CBC_SHA_3		 0x000c
-#define OP_PCL_DTLS_DES_CBC_SHA_4		 0x000f
-#define OP_PCL_DTLS_DES_CBC_SHA_5		 0x0012
-#define OP_PCL_DTLS_DES_CBC_SHA_6		 0x0015
-#define OP_PCL_DTLS_DES_CBC_SHA_7		 0x001a
-
-#define OP_PCL_DTLS_3DES_EDE_CBC_MD5		 0xff23
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA160		 0xff30
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA224		 0xff34
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA256		 0xff36
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA384		 0xff33
-#define OP_PCL_DTLS_3DES_EDE_CBC_SHA512		 0xff35
-#define OP_PCL_DTLS_AES_128_CBC_SHA160		 0xff80
-#define OP_PCL_DTLS_AES_128_CBC_SHA224		 0xff84
-#define OP_PCL_DTLS_AES_128_CBC_SHA256		 0xff86
-#define OP_PCL_DTLS_AES_128_CBC_SHA384		 0xff83
-#define OP_PCL_DTLS_AES_128_CBC_SHA512		 0xff85
-#define OP_PCL_DTLS_AES_192_CBC_SHA160		 0xff20
-#define OP_PCL_DTLS_AES_192_CBC_SHA224		 0xff24
-#define OP_PCL_DTLS_AES_192_CBC_SHA256		 0xff26
-#define OP_PCL_DTLS_AES_192_CBC_SHA384		 0xff23
-#define OP_PCL_DTLS_AES_192_CBC_SHA512		 0xff25
-#define OP_PCL_DTLS_AES_256_CBC_SHA160		 0xff60
-#define OP_PCL_DTLS_AES_256_CBC_SHA224		 0xff64
-#define OP_PCL_DTLS_AES_256_CBC_SHA256		 0xff66
-#define OP_PCL_DTLS_AES_256_CBC_SHA384		 0xff63
-#define OP_PCL_DTLS_AES_256_CBC_SHA512		 0xff65
+/*
+ * For SSL/TLS/DTLS - OP_PCL_TLS
+ * For more details see IANA TLS Cipher Suite registry:
+ * https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml
+ * Note: for private/internal use (reserved by IANA) - OP_PCL_PVT_TLS
+ */
+#define OP_PCL_TLS_RSA_EXPORT_WITH_RC4_40_MD5		0x0003
+#define OP_PCL_TLS_RSA_WITH_RC4_128_MD5			0x0004
+#define OP_PCL_TLS_RSA_WITH_RC4_128_SHA			0x0005
+#define OP_PCL_TLS_RSA_EXPORT_WITH_DES40_CBC_SHA	0x0008
+#define OP_PCL_TLS_RSA_WITH_DES_CBC_SHA			0x0009
+#define OP_PCL_TLS_RSA_WITH_3DES_EDE_CBC_SHA		0x000a
+#define OP_PCL_TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA	0x000b
+#define OP_PCL_TLS_DH_DSS_WITH_DES_CBC_SHA		0x000c
+#define OP_PCL_TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA		0x000d
+#define OP_PCL_TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA	0x000e
+#define OP_PCL_TLS_DH_RSA_WITH_DES_CBC_SHA		0x000f
+#define OP_PCL_TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA		0x0010
+#define OP_PCL_TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA	0x0011
+#define OP_PCL_TLS_DHE_DSS_WITH_DES_CBC_SHA		0x0012
+#define OP_PCL_TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA	0x0013
+#define OP_PCL_TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA	0x0014
+#define OP_PCL_TLS_DHE_RSA_WITH_DES_CBC_SHA		0x0015
+#define OP_PCL_TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA	0x0016
+#define OP_PCL_TLS_DH_anon_EXPORT_WITH_RC4_40_MD5	0x0017
+#define OP_PCL_TLS_DH_anon_WITH_RC4_128_MD5		0x0018
+#define OP_PCL_TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA	0x0019
+#define OP_PCL_TLS_DH_anon_WITH_DES_CBC_SHA		0x001a
+#define OP_PCL_TLS_DH_anon_WITH_3DES_EDE_CBC_SHA	0x001b
+#define OP_PCL_TLS_KRB5_WITH_DES_CBC_SHA		0x001e
+#define OP_PCL_TLS_KRB5_WITH_3DES_EDE_CBC_SHA		0x001f
+#define OP_PCL_TLS_KRB5_WITH_RC4_128_SHA		0x0020
+#define OP_PCL_TLS_KRB5_WITH_3DES_EDE_CBC_MD5		0x0023
+#define OP_PCL_TLS_KRB5_WITH_DES_CBC_MD5		0x0022
+#define OP_PCL_TLS_KRB5_WITH_RC4_128_MD5		0x0024
+#define OP_PCL_TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA	0x0026
+#define OP_PCL_TLS_KRB5_EXPORT_WITH_RC4_40_SHA		0x0028
+#define OP_PCL_TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5	0x0029
+#define OP_PCL_TLS_KRB5_EXPORT_WITH_RC4_40_MD5		0x002b
+#define OP_PCL_TLS_RSA_WITH_AES_128_CBC_SHA		0x002f
+#define OP_PCL_TLS_DH_DSS_WITH_AES_128_CBC_SHA		0x0030
+#define OP_PCL_TLS_DH_RSA_WITH_AES_128_CBC_SHA		0x0031
+#define OP_PCL_TLS_DHE_DSS_WITH_AES_128_CBC_SHA		0x0032
+#define OP_PCL_TLS_DHE_RSA_WITH_AES_128_CBC_SHA		0x0033
+#define OP_PCL_TLS_DH_anon_WITH_AES_128_CBC_SHA		0x0034
+#define OP_PCL_TLS_RSA_WITH_AES_256_CBC_SHA		0x0035
+#define OP_PCL_TLS_DH_DSS_WITH_AES_256_CBC_SHA		0x0036
+#define OP_PCL_TLS_DH_RSA_WITH_AES_256_CBC_SHA		0x0037
+#define OP_PCL_TLS_DHE_DSS_WITH_AES_256_CBC_SHA		0x0038
+#define OP_PCL_TLS_DHE_RSA_WITH_AES_256_CBC_SHA		0x0039
+#define OP_PCL_TLS_DH_anon_WITH_AES_256_CBC_SHA		0x003a
+#define OP_PCL_TLS_RSA_WITH_AES_128_CBC_SHA256		0x003c
+#define OP_PCL_TLS_RSA_WITH_AES_256_CBC_SHA256		0x003d
+#define OP_PCL_TLS_DH_DSS_WITH_AES_128_CBC_SHA256	0x003e
+#define OP_PCL_TLS_DH_RSA_WITH_AES_128_CBC_SHA256	0x003f
+#define OP_PCL_TLS_DHE_DSS_WITH_AES_128_CBC_SHA256	0x0040
+#define OP_PCL_TLS_DHE_RSA_WITH_AES_128_CBC_SHA256	0x0067
+#define OP_PCL_TLS_DH_DSS_WITH_AES_256_CBC_SHA256	0x0068
+#define OP_PCL_TLS_DH_RSA_WITH_AES_256_CBC_SHA256	0x0069
+#define OP_PCL_TLS_DHE_DSS_WITH_AES_256_CBC_SHA256	0x006a
+#define OP_PCL_TLS_DHE_RSA_WITH_AES_256_CBC_SHA256	0x006b
+#define OP_PCL_TLS_DH_anon_WITH_AES_128_CBC_SHA256	0x006c
+#define OP_PCL_TLS_DH_anon_WITH_AES_256_CBC_SHA256	0x006d
+#define OP_PCL_TLS_PSK_WITH_RC4_128_SHA			0x008a
+#define OP_PCL_TLS_PSK_WITH_3DES_EDE_CBC_SHA		0x008b
+#define OP_PCL_TLS_PSK_WITH_AES_128_CBC_SHA		0x008c
+#define OP_PCL_TLS_PSK_WITH_AES_256_CBC_SHA		0x008d
+#define OP_PCL_TLS_DHE_PSK_WITH_RC4_128_SHA		0x008e
+#define OP_PCL_TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA	0x008f
+#define OP_PCL_TLS_DHE_PSK_WITH_AES_128_CBC_SHA		0x0090
+#define OP_PCL_TLS_DHE_PSK_WITH_AES_256_CBC_SHA		0x0091
+#define OP_PCL_TLS_RSA_PSK_WITH_RC4_128_SHA		0x0092
+#define OP_PCL_TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA	0x0093
+#define OP_PCL_TLS_RSA_PSK_WITH_AES_128_CBC_SHA		0x0094
+#define OP_PCL_TLS_RSA_PSK_WITH_AES_256_CBC_SHA		0x0095
+#define OP_PCL_TLS_RSA_WITH_AES_128_GCM_SHA256		0x009c
+#define OP_PCL_TLS_RSA_WITH_AES_256_GCM_SHA384		0x009d
+#define OP_PCL_TLS_DHE_RSA_WITH_AES_128_GCM_SHA256	0x009e
+#define OP_PCL_TLS_DHE_RSA_WITH_AES_256_GCM_SHA384	0x009f
+#define OP_PCL_TLS_DH_RSA_WITH_AES_128_GCM_SHA256	0x00a0
+#define OP_PCL_TLS_DH_RSA_WITH_AES_256_GCM_SHA384	0x00a1
+#define OP_PCL_TLS_DHE_DSS_WITH_AES_128_GCM_SHA256	0x00a2
+#define OP_PCL_TLS_DHE_DSS_WITH_AES_256_GCM_SHA384	0x00a3
+#define OP_PCL_TLS_DH_DSS_WITH_AES_128_GCM_SHA256	0x00a4
+#define OP_PCL_TLS_DH_DSS_WITH_AES_256_GCM_SHA384	0x00a5
+#define OP_PCL_TLS_DH_anon_WITH_AES_128_GCM_SHA256	0x00a6
+#define OP_PCL_TLS_DH_anon_WITH_AES_256_GCM_SHA384	0x00a7
+#define OP_PCL_TLS_PSK_WITH_AES_128_GCM_SHA256		0x00a8
+#define OP_PCL_TLS_PSK_WITH_AES_256_GCM_SHA384		0x00a9
+#define OP_PCL_TLS_DHE_PSK_WITH_AES_128_GCM_SHA256	0x00aa
+#define OP_PCL_TLS_DHE_PSK_WITH_AES_256_GCM_SHA384	0x00ab
+#define OP_PCL_TLS_RSA_PSK_WITH_AES_128_GCM_SHA256	0x00ac
+#define OP_PCL_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384	0x00ad
+#define OP_PCL_TLS_PSK_WITH_AES_128_CBC_SHA256		0x00ae
+#define OP_PCL_TLS_PSK_WITH_AES_256_CBC_SHA384		0x00af
+#define OP_PCL_TLS_DHE_PSK_WITH_AES_128_CBC_SHA256	0x00b2
+#define OP_PCL_TLS_DHE_PSK_WITH_AES_256_CBC_SHA384	0x00b3
+#define OP_PCL_TLS_RSA_PSK_WITH_AES_128_CBC_SHA256	0x00b6
+#define OP_PCL_TLS_RSA_PSK_WITH_AES_256_CBC_SHA384	0x00b7
+#define OP_PCL_TLS_ECDH_ECDSA_WITH_RC4_128_SHA		0xc002
+#define OP_PCL_TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA	0xc003
+#define OP_PCL_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA	0xc004
+#define OP_PCL_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA	0xc005
+#define OP_PCL_TLS_ECDHE_ECDSA_WITH_RC4_128_SHA		0xc007
+#define OP_PCL_TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA	0xc008
+#define OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA	0xc009
+#define OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA	0xc00a
+#define OP_PCL_TLS_ECDH_RSA_WITH_RC4_128_SHA		0xc00c
+#define OP_PCL_TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA	0xc00d
+#define OP_PCL_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA	0xc00e
+#define OP_PCL_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA	0xc00f
+#define OP_PCL_TLS_ECDHE_RSA_WITH_RC4_128_SHA		0xc011
+#define OP_PCL_TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA	0xc012
+#define OP_PCL_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA	0xc013
+#define OP_PCL_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA	0xc014
+#define OP_PCL_TLS_ECDH_anon_WITH_RC4_128_SHA		0xc016
+#define OP_PCL_TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA	0xc017
+#define OP_PCL_TLS_ECDH_anon_WITH_AES_128_CBC_SHA	0xc018
+#define OP_PCL_TLS_ECDH_anon_WITH_AES_256_CBC_SHA	0xc019
+#define OP_PCL_TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA	0xc01a
+#define OP_PCL_TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA	0xc01b
+#define OP_PCL_TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA	0xc01c
+#define OP_PCL_TLS_SRP_SHA_WITH_AES_128_CBC_SHA		0xc01d
+#define OP_PCL_TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA	0xc01e
+#define OP_PCL_TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA	0xc01f
+#define OP_PCL_TLS_SRP_SHA_WITH_AES_256_CBC_SHA		0xc020
+#define OP_PCL_TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA	0xc021
+#define OP_PCL_TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA	0xc022
+#define OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256	0xc023
+#define OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384	0xc024
+#define OP_PCL_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256	0xc025
+#define OP_PCL_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384	0xc026
+#define OP_PCL_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256	0xc027
+#define OP_PCL_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384	0xc028
+#define OP_PCL_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256	0xc029
+#define OP_PCL_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384	0xc02a
+#define OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256	0xc02b
+#define OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384	0xc02c
+#define OP_PCL_TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256	0xc02d
+#define OP_PCL_TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384	0xc02e
+#define OP_PCL_TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256	0xc02f
+#define OP_PCL_TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384	0xc030
+#define OP_PCL_TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256	0xc031
+#define OP_PCL_TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384	0xc032
+#define OP_PCL_TLS_ECDHE_PSK_WITH_RC4_128_SHA		0xc033
+#define OP_PCL_TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA	0xc034
+#define OP_PCL_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA	0xc035
+#define OP_PCL_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA	0xc036
+#define OP_PCL_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256	0xc037
+#define OP_PCL_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384	0xc038
+#define OP_PCL_PVT_TLS_3DES_EDE_CBC_MD5			0xff23
+#define OP_PCL_PVT_TLS_3DES_EDE_CBC_SHA160		0xff30
+#define OP_PCL_PVT_TLS_3DES_EDE_CBC_SHA384		0xff33
+#define OP_PCL_PVT_TLS_3DES_EDE_CBC_SHA224		0xff34
+#define OP_PCL_PVT_TLS_3DES_EDE_CBC_SHA512		0xff35
+#define OP_PCL_PVT_TLS_3DES_EDE_CBC_SHA256		0xff36
+#define OP_PCL_PVT_TLS_AES_256_CBC_SHA160		0xff60
+#define OP_PCL_PVT_TLS_AES_256_CBC_SHA384		0xff63
+#define OP_PCL_PVT_TLS_AES_256_CBC_SHA224		0xff64
+#define OP_PCL_PVT_TLS_AES_256_CBC_SHA512		0xff65
+#define OP_PCL_PVT_TLS_AES_256_CBC_SHA256		0xff66
+#define OP_PCL_PVT_TLS_AES_128_CBC_SHA160		0xff80
+#define OP_PCL_PVT_TLS_AES_128_CBC_SHA384		0xff83
+#define OP_PCL_PVT_TLS_AES_128_CBC_SHA224		0xff84
+#define OP_PCL_PVT_TLS_AES_128_CBC_SHA512		0xff85
+#define OP_PCL_PVT_TLS_AES_128_CBC_SHA256		0xff86
+#define OP_PCL_PVT_TLS_AES_192_CBC_SHA160		0xff90
+#define OP_PCL_PVT_TLS_AES_192_CBC_SHA384		0xff93
+#define OP_PCL_PVT_TLS_AES_192_CBC_SHA224		0xff94
+#define OP_PCL_PVT_TLS_AES_192_CBC_SHA512		0xff95
+#define OP_PCL_PVT_TLS_AES_192_CBC_SHA256		0xff96
+#define OP_PCL_PVT_TLS_MASTER_SECRET_PRF_FE		0xfffe
+#define OP_PCL_PVT_TLS_MASTER_SECRET_PRF_FF		0xffff
 
 /* 802.16 WiMAX protinfos */
 #define OP_PCL_WIMAX_OFDM			 0x0201
@@ -1332,7 +868,7 @@
 #define OP_PCL_LTE_MIXED_AUTH_SHIFT	0
 #define OP_PCL_LTE_MIXED_AUTH_MASK	(3 << OP_PCL_LTE_MIXED_AUTH_SHIFT)
 #define OP_PCL_LTE_MIXED_ENC_SHIFT	8
-#define OP_PCL_LTE_MIXED_ENC_MASK	(3 < OP_PCL_LTE_MIXED_ENC_SHIFT)
+#define OP_PCL_LTE_MIXED_ENC_MASK	(3 << OP_PCL_LTE_MIXED_ENC_SHIFT)
 #define OP_PCL_LTE_MIXED_AUTH_NULL	(OP_PCL_LTE_NULL << \
 					 OP_PCL_LTE_MIXED_AUTH_SHIFT)
 #define OP_PCL_LTE_MIXED_AUTH_SNOW	(OP_PCL_LTE_SNOW << \
diff --git a/drivers/crypto/dpaa2_sec/hw/desc/algo.h b/drivers/crypto/dpaa2_sec/hw/desc/algo.h
index 91f3e067..febcb6d0 100644
--- a/drivers/crypto/dpaa2_sec/hw/desc/algo.h
+++ b/drivers/crypto/dpaa2_sec/hw/desc/algo.h
@@ -410,6 +410,35 @@ cnstr_shdsc_kasumi_f9(uint32_t *descbuf, bool ps, bool swap,
 }
 
 /**
+ * cnstr_shdsc_crc - CRC32 Accelerator (IEEE 802 CRC32 protocol mode)
+ * @descbuf: pointer to descriptor-under-construction buffer
+ * @swap: must be true when core endianness doesn't match SEC endianness
+ *
+ * Return: size of descriptor written in words or negative number on error
+ */
+static inline int
+cnstr_shdsc_crc(uint32_t *descbuf, bool swap)
+{
+	struct program prg;
+	struct program *p = &prg;
+
+	PROGRAM_CNTXT_INIT(p, descbuf, 0);
+	if (swap)
+		PROGRAM_SET_BSWAP(p);
+
+	SHR_HDR(p, SHR_ALWAYS, 1, 0);
+
+	MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0);
+	ALG_OPERATION(p, OP_ALG_ALGSEL_CRC,
+		      OP_ALG_AAI_802 | OP_ALG_AAI_DOC,
+		      OP_ALG_AS_FINALIZE, 0, DIR_ENC);
+	SEQFIFOLOAD(p, MSG2, 0, VLF | LAST2);
+	SEQSTORE(p, CONTEXT2, 0, 4, 0);
+
+	return PROGRAM_FINALIZE(p);
+}
+
+/**
  * cnstr_shdsc_gcm_encap - AES-GCM encap as a shared descriptor
  * @descbuf: pointer to descriptor-under-construction buffer
  * @ps: if 36/40bit addressing is desired, this parameter must be true
@@ -614,33 +643,4 @@ cnstr_shdsc_gcm_decap(uint32_t *descbuf, bool ps, bool swap,
 	return PROGRAM_FINALIZE(p);
 }
 
-/**
- * cnstr_shdsc_crc - CRC32 Accelerator (IEEE 802 CRC32 protocol mode)
- * @descbuf: pointer to descriptor-under-construction buffer
- * @swap: must be true when core endianness doesn't match SEC endianness
- *
- * Return: size of descriptor written in words or negative number on error
- */
-static inline int
-cnstr_shdsc_crc(uint32_t *descbuf, bool swap)
-{
-	struct program prg;
-	struct program *p = &prg;
-
-	PROGRAM_CNTXT_INIT(p, descbuf, 0);
-	if (swap)
-		PROGRAM_SET_BSWAP(p);
-
-	SHR_HDR(p, SHR_ALWAYS, 1, 0);
-
-	MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0);
-	ALG_OPERATION(p, OP_ALG_ALGSEL_CRC,
-		      OP_ALG_AAI_802 | OP_ALG_AAI_DOC,
-		      OP_ALG_AS_FINALIZE, 0, DIR_ENC);
-	SEQFIFOLOAD(p, MSG2, 0, VLF | LAST2);
-	SEQSTORE(p, CONTEXT2, 0, 4, 0);
-
-	return PROGRAM_FINALIZE(p);
-}
-
 #endif /* __DESC_ALGO_H__ */
diff --git a/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h b/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h
index 35cc02a6..d256a391 100644
--- a/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h
+++ b/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h
@@ -522,44 +522,133 @@ enum ipsec_icv_size {
 
 /*
  * IPSec ESP Datapath Protocol Override Register (DPOVRD)
+ * IPSEC_N_* defines are for IPsec new mode.
  */
 
-#define IPSEC_DECO_DPOVRD_USE		0x80
+/**
+ * IPSEC_DPOVRD_USE - DPOVRD will override values specified in the PDB
+ */
+#define IPSEC_DPOVRD_USE	BIT(31)
 
-struct ipsec_deco_dpovrd {
-	uint8_t ovrd_ecn;
-	uint8_t ip_hdr_len;
-	uint8_t nh_offset;
-	union {
-		uint8_t next_header;	/* next header if encap */
-		uint8_t rsvd;		/* reserved if decap */
-	};
-};
+/**
+ * IPSEC_DPOVRD_ECN_SHIFT - Explicit Congestion Notification
+ *
+ * If set, MSB of the 4 bits indicates that the 2 LSBs will replace the ECN bits
+ * in the IP header.
+ */
+#define IPSEC_DPOVRD_ECN_SHIFT		24
 
-struct ipsec_new_encap_deco_dpovrd {
-#define IPSEC_NEW_ENCAP_DECO_DPOVRD_USE	0x8000
-	uint16_t ovrd_ip_hdr_len;	/* OVRD + outer IP header material
-					 * length
-					 */
-#define IPSEC_NEW_ENCAP_OIMIF		0x80
-	uint8_t oimif_aoipho;		/* OIMIF + actual outer IP header
-					 * offset
-					 */
-	uint8_t rsvd;
-};
+/**
+ * IPSEC_DPOVRD_ECN_MASK - See IPSEC_DPOVRD_ECN_SHIFT
+ */
+#define IPSEC_DPOVRD_ECN_MASK		(0xf << IPSEC_ENCAP_DPOVRD_ECN_SHIFT)
 
-struct ipsec_new_decap_deco_dpovrd {
-	uint8_t ovrd;
-	uint8_t aoipho_hi;		/* upper nibble of actual outer IP
-					 * header
-					 */
-	uint16_t aoipho_lo_ip_hdr_len;	/* lower nibble of actual outer IP
-					 * header + outer IP header material
-					 */
-};
+/**
+ * IPSEC_DPOVRD_IP_HDR_LEN_SHIFT - The length (in bytes) of the portion of the
+ *                                 IP header that is not encrypted
+ */
+#define IPSEC_DPOVRD_IP_HDR_LEN_SHIFT	16
+
+/**
+ * IPSEC_DPOVRD_IP_HDR_LEN_MASK - See IPSEC_DPOVRD_IP_HDR_LEN_SHIFT
+ */
+#define IPSEC_DPOVRD_IP_HDR_LEN_MASK	(0xff << IPSEC_DPOVRD_IP_HDR_LEN_SHIFT)
+
+/**
+ * IPSEC_DPOVRD_NH_OFFSET_SHIFT - The location of the next header field within
+ *                                the IP header of the transport mode packet
+ *
+ * Encap:
+ *	ESP_Trailer_NH <-- IP_Hdr[DPOVRD[NH_OFFSET]]
+ *	IP_Hdr[DPOVRD[NH_OFFSET]] <-- DPOVRD[NH]
+ *Decap:
+ *	IP_Hdr[DPOVRD[NH_OFFSET]] <-- ESP_Trailer_NH
+ */
+#define IPSEC_DPOVRD_NH_OFFSET_SHIFT	8
+
+/**
+ * IPSEC_DPOVRD_NH_OFFSET_MASK - See IPSEC_DPOVRD_NH_OFFSET_SHIFT
+ */
+#define IPSEC_DPOVRD_NH_OFFSET_MASK	(0xff << IPSEC_DPOVRD_NH_OFFSET_SHIFT)
+
+/**
+ * IPSEC_DPOVRD_NH_MASK - See IPSEC_DPOVRD_NH_OFFSET_SHIFT
+ *                        Valid only for encapsulation.
+ */
+#define IPSEC_DPOVRD_NH_MASK		0xff
+
+/**
+ * IPSEC_N_ENCAP_DPOVRD_OIM_LEN_SHIFT - Outer IP header Material length (encap)
+ *                                      Valid only if L2_COPY is not set.
+ */
+#define IPSEC_N_ENCAP_DPOVRD_OIM_LEN_SHIFT	16
+
+/**
+ * IPSEC_N_ENCAP_DPOVRD_OIM_LEN_MASK - See IPSEC_N_ENCAP_DPOVRD_OIM_LEN_SHIFT
+ */
+#define IPSEC_N_ENCAP_DPOVRD_OIM_LEN_MASK \
+	(0xfff << IPSEC_N_ENCAP_DPOVRD_OIM_LEN_SHIFT)
+
+/**
+ * IPSEC_N_ENCAP_DPOVRD_L2_LEN_SHIFT - L2 header length
+ *                                     Valid only if L2_COPY is set.
+ */
+#define IPSEC_N_ENCAP_DPOVRD_L2_LEN_SHIFT	16
 
-static inline void
-__gen_auth_key(struct program *program, struct alginfo *authdata)
+/**
+ * IPSEC_N_ENCAP_DPOVRD_L2_LEN_MASK - See IPSEC_N_ENCAP_DPOVRD_L2_LEN_SHIFT
+ */
+#define IPSEC_N_ENCAP_DPOVRD_L2_LEN_MASK \
+	(0xff << IPSEC_N_ENCAP_DPOVRD_L2_LEN_SHIFT)
+
+/**
+ * IPSEC_N_ENCAP_DPOVRD_OIMIF -  Outer IP header Material in Input Frame
+ */
+#define IPSEC_N_ENCAP_DPOVRD_OIMIF		BIT(15)
+
+/**
+ * IPSEC_N_ENCAP_DPOVRD_L2_COPY - L2 header present in input frame
+ *
+ * Note: For Era <= 8, this bit is reserved (not used) by HW.
+ */
+#define IPSEC_N_ENCAP_DPOVRD_L2_COPY		BIT(14)
+
+/**
+ * IPSEC_N_ENCAP_DPOVRD_AOIPHO_SHIFT - Actual Outer IP Header Offset (encap)
+ */
+#define IPSEC_N_ENCAP_DPOVRD_AOIPHO_SHIFT	8
+
+/**
+ * IPSEC_N_ENCAP_DPOVRD_AOIPHO_MASK - See IPSEC_N_ENCAP_DPOVRD_AOIPHO_SHIFT
+ */
+#define IPSEC_N_ENCAP_DPOVRD_AOIPHO_MASK \
+	(0x3c << IPSEC_N_ENCAP_DPOVRD_AOIPHO_SHIFT)
+
+/**
+ * IPSEC_N_ENCAP_DPOVRD_NH_MASK -  Next Header
+ *
+ * Used in the Next Header field of the encapsulated payload.
+ */
+#define IPSEC_N_ENCAP_DPOVRD_NH_MASK		0xff
+
+/**
+ * IPSEC_N_DECAP_DPOVRD_AOIPHO_SHIFT - Actual Outer IP Header Offset (decap)
+ */
+#define IPSEC_N_DECAP_DPOVRD_AOIPHO_SHIFT	12
+
+/**
+ * IPSEC_N_DECAP_DPOVRD_AOIPHO_MASK - See IPSEC_N_DECAP_DPOVRD_AOIPHO_SHIFT
+ */
+#define IPSEC_N_DECAP_DPOVRD_AOIPHO_MASK \
+	(0xff << IPSEC_N_DECAP_DPOVRD_AOIPHO_SHIFT)
+
+/**
+ * IPSEC_N_DECAP_DPOVRD_OIM_LEN_MASK - Outer IP header Material length (decap)
+ */
+#define IPSEC_N_DECAP_DPOVRD_OIM_LEN_MASK	0xfff
+
+static inline void __gen_auth_key(struct program *program,
+				  struct alginfo *authdata)
 {
 	uint32_t dkp_protid;
 
@@ -603,6 +692,7 @@ __gen_auth_key(struct program *program, struct alginfo *authdata)
  * @descbuf: pointer to buffer used for descriptor construction
  * @ps: if 36/40bit addressing is desired, this parameter must be true
  * @swap: if true, perform descriptor byte swapping on a 4-byte boundary
+ * @share: sharing type of shared descriptor
  * @pdb: pointer to the PDB to be used with this descriptor
  *       This structure will be copied inline to the descriptor under
  *       construction. No error checking will be made. Refer to the
@@ -621,6 +711,7 @@ __gen_auth_key(struct program *program, struct alginfo *authdata)
  */
 static inline int
 cnstr_shdsc_ipsec_encap(uint32_t *descbuf, bool ps, bool swap,
+					  enum rta_share_type share,
 			struct ipsec_encap_pdb *pdb,
 			struct alginfo *cipherdata,
 			struct alginfo *authdata)
@@ -638,7 +729,7 @@ cnstr_shdsc_ipsec_encap(uint32_t *descbuf, bool ps, bool swap,
 		PROGRAM_SET_BSWAP(p);
 	if (ps)
 		PROGRAM_SET_36BIT_ADDR(p);
-	phdr = SHR_HDR(p, SHR_SERIAL, hdr, 0);
+	phdr = SHR_HDR(p, share, hdr, 0);
 	__rta_copy_ipsec_encap_pdb(p, pdb, cipherdata->algtype);
 	COPY_DATA(p, pdb->ip_hdr, pdb->ip_hdr_len);
 	SET_LABEL(p, hdr);
@@ -669,6 +760,7 @@ cnstr_shdsc_ipsec_encap(uint32_t *descbuf, bool ps, bool swap,
  * @descbuf: pointer to buffer used for descriptor construction
  * @ps: if 36/40bit addressing is desired, this parameter must be true
  * @swap: if true, perform descriptor byte swapping on a 4-byte boundary
+ * @share: sharing type of shared descriptor
  * @pdb: pointer to the PDB to be used with this descriptor
  *       This structure will be copied inline to the descriptor under
  *       construction. No error checking will be made. Refer to the
@@ -687,6 +779,7 @@ cnstr_shdsc_ipsec_encap(uint32_t *descbuf, bool ps, bool swap,
  */
 static inline int
 cnstr_shdsc_ipsec_decap(uint32_t *descbuf, bool ps, bool swap,
+			enum rta_share_type share,
 			struct ipsec_decap_pdb *pdb,
 			struct alginfo *cipherdata,
 			struct alginfo *authdata)
@@ -704,7 +797,7 @@ cnstr_shdsc_ipsec_decap(uint32_t *descbuf, bool ps, bool swap,
 		PROGRAM_SET_BSWAP(p);
 	if (ps)
 		PROGRAM_SET_36BIT_ADDR(p);
-	phdr = SHR_HDR(p, SHR_SERIAL, hdr, 0);
+	phdr = SHR_HDR(p, share, hdr, 0);
 	__rta_copy_ipsec_decap_pdb(p, pdb, cipherdata->algtype);
 	SET_LABEL(p, hdr);
 	pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, BOTH|SHRD);
@@ -1040,7 +1133,7 @@ cnstr_shdsc_ipsec_decap_des_aes_xcbc(uint32_t *descbuf,
  * layers to determine whether Outer IP Header and/or keys can be inlined or
  * not. To be used as first parameter of rta_inline_query().
  */
-#define IPSEC_NEW_ENC_BASE_DESC_LEN	(5 * CAAM_CMD_SZ + \
+#define IPSEC_NEW_ENC_BASE_DESC_LEN	(12 * CAAM_CMD_SZ + \
 					 sizeof(struct ipsec_encap_pdb))
 
 /**
@@ -1052,7 +1145,7 @@ cnstr_shdsc_ipsec_decap_des_aes_xcbc(uint32_t *descbuf,
  * layers to determine whether Outer IP Header and/or key can be inlined or
  * not. To be used as first parameter of rta_inline_query().
  */
-#define IPSEC_NEW_NULL_ENC_BASE_DESC_LEN	(4 * CAAM_CMD_SZ + \
+#define IPSEC_NEW_NULL_ENC_BASE_DESC_LEN	(11 * CAAM_CMD_SZ + \
 						 sizeof(struct ipsec_encap_pdb))
 
 /**
@@ -1061,6 +1154,7 @@ cnstr_shdsc_ipsec_decap_des_aes_xcbc(uint32_t *descbuf,
  * @descbuf: pointer to buffer used for descriptor construction
  * @ps: if 36/40bit addressing is desired, this parameter must be true
  * @swap: must be true when core endianness doesn't match SEC endianness
+ * @share: sharing type of shared descriptor
  * @pdb: pointer to the PDB to be used with this descriptor
  *       This structure will be copied inline to the descriptor under
  *       construction. No error checking will be made. Refer to the
@@ -1080,11 +1174,21 @@ cnstr_shdsc_ipsec_decap_des_aes_xcbc(uint32_t *descbuf,
  *            compute MDHA on the fly in HW.
  *            Valid algorithm values - one of OP_PCL_IPSEC_*
  *
+ * Note: L2 header copy functionality is implemented assuming that bits 14
+ * (currently reserved) and 16-23 (part of Outer IP Header Material Length)
+ * in DPOVRD register are not used (which is usually the case when L3 header
+ * is provided in PDB).
+ * When DPOVRD[14] is set, frame starts with an L2 header; in this case, the
+ * L2 header length is found at DPOVRD[23:16]. SEC uses this length to copy
+ * the header and then it deletes DPOVRD[23:16] (so there is no side effect
+ * when later running IPsec protocol).
+ *
  * Return: size of descriptor written in words or negative number on error
  */
 static inline int
 cnstr_shdsc_ipsec_new_encap(uint32_t *descbuf, bool ps,
 			    bool swap,
+					      enum rta_share_type share,
 			    struct ipsec_encap_pdb *pdb,
 			    uint8_t *opt_ip_hdr,
 			    struct alginfo *cipherdata,
@@ -1097,6 +1201,8 @@ cnstr_shdsc_ipsec_new_encap(uint32_t *descbuf, bool ps,
 	REFERENCE(pkeyjmp);
 	LABEL(hdr);
 	REFERENCE(phdr);
+	LABEL(l2copy);
+	REFERENCE(pl2copy);
 
 	if (rta_sec_era < RTA_SEC_ERA_8) {
 		pr_err("IPsec new mode encap: available only for Era %d or above\n",
@@ -1109,7 +1215,7 @@ cnstr_shdsc_ipsec_new_encap(uint32_t *descbuf, bool ps,
 		PROGRAM_SET_BSWAP(p);
 	if (ps)
 		PROGRAM_SET_36BIT_ADDR(p);
-	phdr = SHR_HDR(p, SHR_SERIAL, hdr, 0);
+	phdr = SHR_HDR(p, share, hdr, 0);
 
 	__rta_copy_ipsec_encap_pdb(p, pdb, cipherdata->algtype);
 
@@ -1128,6 +1234,16 @@ cnstr_shdsc_ipsec_new_encap(uint32_t *descbuf, bool ps,
 	}
 	SET_LABEL(p, hdr);
 
+	MATHB(p, DPOVRD, AND, IPSEC_N_ENCAP_DPOVRD_L2_COPY, NONE, 4, IMMED2);
+	pl2copy = JUMP(p, l2copy, LOCAL_JUMP, ALL_TRUE, MATH_Z);
+	MATHI(p, DPOVRD, RSHIFT, IPSEC_N_ENCAP_DPOVRD_L2_LEN_SHIFT, VSEQOUTSZ,
+	      1, 0);
+	MATHB(p, DPOVRD, AND, ~IPSEC_N_ENCAP_DPOVRD_L2_LEN_MASK, DPOVRD, 4,
+	      IMMED2);
+	/* TODO: CLASS2 corresponds to AUX=2'b10; add more intuitive defines */
+	SEQFIFOSTORE(p, METADATA, 0, 0, CLASS2 | VLF);
+	SET_LABEL(p, l2copy);
+
 	pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SHRD);
 	if (authdata->keylen)
 		__gen_auth_key(p, authdata);
@@ -1138,6 +1254,7 @@ cnstr_shdsc_ipsec_new_encap(uint32_t *descbuf, bool ps,
 	PROTOCOL(p, OP_TYPE_ENCAP_PROTOCOL,
 		 OP_PCLID_IPSEC_NEW,
 		 (uint16_t)(cipherdata->algtype | authdata->algtype));
+	PATCH_JUMP(p, pl2copy, l2copy);
 	PATCH_JUMP(p, pkeyjmp, keyjmp);
 	PATCH_HDR(p, phdr, hdr);
 	return PROGRAM_FINALIZE(p);
@@ -1171,6 +1288,7 @@ cnstr_shdsc_ipsec_new_encap(uint32_t *descbuf, bool ps,
  * @descbuf: pointer to buffer used for descriptor construction
  * @ps: if 36/40bit addressing is desired, this parameter must be true
  * @swap: must be true when core endianness doesn't match SEC endianness
+ * @share: sharing type of shared descriptor
  * @pdb: pointer to the PDB to be used with this descriptor
  *       This structure will be copied inline to the descriptor under
  *       construction. No error checking will be made. Refer to the
@@ -1188,6 +1306,7 @@ cnstr_shdsc_ipsec_new_encap(uint32_t *descbuf, bool ps,
 static inline int
 cnstr_shdsc_ipsec_new_decap(uint32_t *descbuf, bool ps,
 			    bool swap,
+					      enum rta_share_type share,
 			    struct ipsec_decap_pdb *pdb,
 			    struct alginfo *cipherdata,
 			    struct alginfo *authdata)
@@ -1211,7 +1330,7 @@ cnstr_shdsc_ipsec_new_decap(uint32_t *descbuf, bool ps,
 		PROGRAM_SET_BSWAP(p);
 	if (ps)
 		PROGRAM_SET_36BIT_ADDR(p);
-	phdr = SHR_HDR(p, SHR_SERIAL, hdr, 0);
+	phdr = SHR_HDR(p, share, hdr, 0);
 	__rta_copy_ipsec_decap_pdb(p, pdb, cipherdata->algtype);
 	SET_LABEL(p, hdr);
 	pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SHRD);
diff --git a/drivers/crypto/dpaa2_sec/hw/desc/pdcp.h b/drivers/crypto/dpaa2_sec/hw/desc/pdcp.h
new file mode 100644
index 00000000..719ef605
--- /dev/null
+++ b/drivers/crypto/dpaa2_sec/hw/desc/pdcp.h
@@ -0,0 +1,2796 @@
+/*
+ * Copyright 2008-2013 Freescale Semiconductor, Inc.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause or GPL-2.0+
+ */
+
+#ifndef __DESC_PDCP_H__
+#define __DESC_PDCP_H__
+
+#include "hw/rta.h"
+#include "common.h"
+
+/**
+ * DOC: PDCP Shared Descriptor Constructors
+ *
+ * Shared descriptors for PDCP protocol.
+ */
+
+/**
+ * PDCP_NULL_MAX_FRAME_LEN - The maximum frame frame length that is supported by
+ *                           PDCP NULL protocol.
+ */
+#define PDCP_NULL_MAX_FRAME_LEN		0x00002FFF
+
+/**
+ * PDCP_MAC_I_LEN - The length of the MAC-I for PDCP protocol operation
+ */
+#define PDCP_MAC_I_LEN			0x00000004
+
+/**
+ * PDCP_MAX_FRAME_LEN_STATUS - The status returned in FD status/command field in
+ *                             case the input frame is larger than
+ *                             PDCP_NULL_MAX_FRAME_LEN.
+ */
+#define PDCP_MAX_FRAME_LEN_STATUS	0xF1
+
+/**
+ * PDCP_C_PLANE_SN_MASK - This mask is used in the PDCP descriptors for
+ *                        extracting the sequence number (SN) from the PDCP
+ *                        Control Plane header. For PDCP Control Plane, the SN
+ *                        is constant (5 bits) as opposed to PDCP Data Plane
+ *                        (7/12/15 bits).
+ */
+#define PDCP_C_PLANE_SN_MASK		0x1F000000
+#define PDCP_C_PLANE_SN_MASK_BE		0x0000001F
+
+/**
+ * PDCP_U_PLANE_15BIT_SN_MASK - This mask is used in the PDCP descriptors for
+ *                              extracting the sequence number (SN) from the
+ *                              PDCP User Plane header. For PDCP Control Plane,
+ *                              the SN is constant (5 bits) as opposed to PDCP
+ *                              Data Plane (7/12/15 bits).
+ */
+#define PDCP_U_PLANE_15BIT_SN_MASK	0xFF7F0000
+#define PDCP_U_PLANE_15BIT_SN_MASK_BE	0x00007FFF
+
+/**
+ * PDCP_BEARER_MASK - This mask is used masking out the bearer for PDCP
+ *                    processing with SNOW f9 in LTE.
+ *
+ * The value on which this mask is applied is formatted as below:
+ *     Count-C (32 bit) | Bearer (5 bit) | Direction (1 bit) | 0 (26 bits)
+ *
+ * Applying this mask is done for creating the upper 64 bits of the IV needed
+ * for SNOW f9.
+ *
+ * The lower 32 bits of the mask are used for masking the direction for AES
+ * CMAC IV.
+ */
+#define PDCP_BEARER_MASK		0x00000004FFFFFFFFull
+#define PDCP_BEARER_MASK_BE		0xFFFFFFFF04000000ull
+
+/**
+ * PDCP_DIR_MASK - This mask is used masking out the direction for PDCP
+ *                 processing with SNOW f9 in LTE.
+ *
+ * The value on which this mask is applied is formatted as below:
+ *     Bearer (5 bit) | Direction (1 bit) | 0 (26 bits)
+ *
+ * Applying this mask is done for creating the lower 32 bits of the IV needed
+ * for SNOW f9.
+ *
+ * The upper 32 bits of the mask are used for masking the direction for AES
+ * CMAC IV.
+ */
+#define PDCP_DIR_MASK			0x00000000000000F8ull
+#define PDCP_DIR_MASK_BE			0xF800000000000000ull
+
+/**
+ * PDCP_NULL_INT_MAC_I_VAL - The value of the PDCP PDU MAC-I in case NULL
+ *                           integrity is used.
+ */
+
+#define PDCP_NULL_INT_MAC_I_VAL		0x00000000
+
+/**
+ * PDCP_NULL_INT_ICV_CHECK_FAILED_STATUS - The status used to report ICV check
+ *                                         failed in case of NULL integrity
+ *                                         Control Plane processing.
+ */
+#define PDCP_NULL_INT_ICV_CHECK_FAILED_STATUS	0x0A
+/**
+ * PDCP_DPOVRD_HFN_OV_EN - Value to be used in the FD status/cmd field to
+ *                         indicate the HFN override mechanism is active for the
+ *                         frame.
+ */
+#define PDCP_DPOVRD_HFN_OV_EN		0x80000000
+
+/**
+ * PDCP_P4080REV2_HFN_OV_BUFLEN - The length in bytes of the supplementary space
+ *                                that must be provided by the user at the
+ *                                beginning of the input frame buffer for
+ *                                P4080 REV 2.
+ *
+ * The format of the frame buffer is the following:
+ *
+ *  |<---PDCP_P4080REV2_HFN_OV_BUFLEN-->|
+ * //===================================||============||==============\\
+ * || PDCP_DPOVRD_HFN_OV_EN | HFN value || PDCP Header|| PDCP Payload ||
+ * \\===================================||============||==============//
+ *
+ * If HFN override mechanism is not desired, then the MSB of the first 4 bytes
+ * must be set to 0b.
+ */
+#define PDCP_P4080REV2_HFN_OV_BUFLEN	4
+
+/**
+ * enum cipher_type_pdcp - Type selectors for cipher types in PDCP protocol OP
+ *                         instructions.
+ * @PDCP_CIPHER_TYPE_NULL: NULL
+ * @PDCP_CIPHER_TYPE_SNOW: SNOW F8
+ * @PDCP_CIPHER_TYPE_AES: AES
+ * @PDCP_CIPHER_TYPE_ZUC: ZUCE
+ * @PDCP_CIPHER_TYPE_INVALID: invalid option
+ */
+enum cipher_type_pdcp {
+	PDCP_CIPHER_TYPE_NULL,
+	PDCP_CIPHER_TYPE_SNOW,
+	PDCP_CIPHER_TYPE_AES,
+	PDCP_CIPHER_TYPE_ZUC,
+	PDCP_CIPHER_TYPE_INVALID
+};
+
+/**
+ * enum auth_type_pdcp - Type selectors for integrity types in PDCP protocol OP
+ *                       instructions.
+ * @PDCP_AUTH_TYPE_NULL: NULL
+ * @PDCP_AUTH_TYPE_SNOW: SNOW F9
+ * @PDCP_AUTH_TYPE_AES: AES CMAC
+ * @PDCP_AUTH_TYPE_ZUC: ZUCA
+ * @PDCP_AUTH_TYPE_INVALID: invalid option
+ */
+enum auth_type_pdcp {
+	PDCP_AUTH_TYPE_NULL,
+	PDCP_AUTH_TYPE_SNOW,
+	PDCP_AUTH_TYPE_AES,
+	PDCP_AUTH_TYPE_ZUC,
+	PDCP_AUTH_TYPE_INVALID
+};
+
+/**
+ * enum pdcp_dir - Type selectors for direction for PDCP protocol
+ * @PDCP_DIR_UPLINK: uplink direction
+ * @PDCP_DIR_DOWNLINK: downlink direction
+ * @PDCP_DIR_INVALID: invalid option
+ */
+enum pdcp_dir {
+	PDCP_DIR_UPLINK = 0,
+	PDCP_DIR_DOWNLINK = 1,
+	PDCP_DIR_INVALID
+};
+
+/**
+ * enum pdcp_plane - PDCP domain selectors
+ * @PDCP_CONTROL_PLANE: Control Plane
+ * @PDCP_DATA_PLANE: Data Plane
+ * @PDCP_SHORT_MAC: Short MAC
+ */
+enum pdcp_plane {
+	PDCP_CONTROL_PLANE,
+	PDCP_DATA_PLANE,
+	PDCP_SHORT_MAC
+};
+
+/**
+ * enum pdcp_sn_size - Sequence Number Size selectors for PDCP protocol
+ * @PDCP_SN_SIZE_5: 5bit sequence number
+ * @PDCP_SN_SIZE_7: 7bit sequence number
+ * @PDCP_SN_SIZE_12: 12bit sequence number
+ * @PDCP_SN_SIZE_15: 15bit sequence number
+ * @PDCP_SN_SIZE_18: 18bit sequence number
+ */
+enum pdcp_sn_size {
+	PDCP_SN_SIZE_5 = 5,
+	PDCP_SN_SIZE_7 = 7,
+	PDCP_SN_SIZE_12 = 12,
+	PDCP_SN_SIZE_15 = 15
+};
+
+/*
+ * PDCP Control Plane Protocol Data Blocks
+ */
+#define PDCP_C_PLANE_PDB_HFN_SHIFT		5
+#define PDCP_C_PLANE_PDB_BEARER_SHIFT		27
+#define PDCP_C_PLANE_PDB_DIR_SHIFT		26
+#define PDCP_C_PLANE_PDB_HFN_THR_SHIFT		5
+
+#define PDCP_U_PLANE_PDB_OPT_SHORT_SN		0x2
+#define PDCP_U_PLANE_PDB_OPT_15B_SN		0x4
+#define PDCP_U_PLANE_PDB_SHORT_SN_HFN_SHIFT	7
+#define PDCP_U_PLANE_PDB_LONG_SN_HFN_SHIFT	12
+#define PDCP_U_PLANE_PDB_15BIT_SN_HFN_SHIFT	15
+#define PDCP_U_PLANE_PDB_BEARER_SHIFT		27
+#define PDCP_U_PLANE_PDB_DIR_SHIFT		26
+#define PDCP_U_PLANE_PDB_SHORT_SN_HFN_THR_SHIFT	7
+#define PDCP_U_PLANE_PDB_LONG_SN_HFN_THR_SHIFT	12
+#define PDCP_U_PLANE_PDB_15BIT_SN_HFN_THR_SHIFT	15
+
+struct pdcp_pdb {
+	union {
+		uint32_t opt;
+		uint32_t rsvd;
+	} opt_res;
+	uint32_t hfn_res;	/* HyperFrame number,(27, 25 or 21 bits),
+				 * left aligned & right-padded with zeros.
+				 */
+	uint32_t bearer_dir_res;/* Bearer(5 bits), packet direction (1 bit),
+				 * left aligned & right-padded with zeros.
+				 */
+	uint32_t hfn_thr_res;	/* HyperFrame number threshold (27, 25 or 21
+				 * bits), left aligned & right-padded with
+				 * zeros.
+				 */
+};
+
+/*
+ * PDCP internal PDB types
+ */
+enum pdb_type_e {
+	PDCP_PDB_TYPE_NO_PDB,
+	PDCP_PDB_TYPE_FULL_PDB,
+	PDCP_PDB_TYPE_REDUCED_PDB,
+	PDCP_PDB_TYPE_INVALID
+};
+
+/*
+ * Function for appending the portion of a PDCP Control Plane shared descriptor
+ * which performs NULL encryption and integrity (i.e. copies the input frame
+ * to the output frame, appending 32 bits of zeros at the end (MAC-I for
+ * NULL integrity).
+ */
+static inline int
+pdcp_insert_cplane_null_op(struct program *p,
+			   bool swap __maybe_unused,
+			   struct alginfo *cipherdata __maybe_unused,
+			   struct alginfo *authdata __maybe_unused,
+			   unsigned int dir,
+			   unsigned char era_2_sw_hfn_ovrd __maybe_unused)
+{
+	LABEL(local_offset);
+	REFERENCE(move_cmd_read_descbuf);
+	REFERENCE(move_cmd_write_descbuf);
+
+	if (rta_sec_era > RTA_SEC_ERA_2) {
+		MATHB(p, SEQINSZ, ADD, ZERO, VSEQINSZ, 4, 0);
+		if (dir == OP_TYPE_ENCAP_PROTOCOL)
+			MATHB(p, SEQINSZ, ADD, PDCP_MAC_I_LEN, VSEQOUTSZ, 4,
+			      IMMED2);
+		else
+			MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, VSEQOUTSZ, 4,
+			      IMMED2);
+	} else {
+		MATHB(p, SEQINSZ, ADD, ONE, VSEQINSZ, 4, 0);
+		MATHB(p, VSEQINSZ, SUB, ONE, VSEQINSZ, 4, 0);
+
+		if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+			MATHB(p, SEQINSZ, ADD, PDCP_MAC_I_LEN, VSEQOUTSZ, 4,
+			      IMMED2);
+			MATHB(p, VSEQINSZ, SUB, ONE, MATH0, 4, 0);
+		} else {
+			MATHB(p, VSEQINSZ, SUB, PDCP_MAC_I_LEN, VSEQINSZ, 4,
+			      IMMED2);
+			MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, VSEQOUTSZ, 4,
+			      IMMED2);
+			MATHB(p, VSEQOUTSZ, SUB, ONE, MATH0, 4, 0);
+		}
+
+		MATHB(p, MATH0, ADD, ONE, MATH0, 4, 0);
+
+		/*
+		 * Since MOVELEN is available only starting with
+		 * SEC ERA 3, use poor man's MOVELEN: create a MOVE
+		 * command dynamically by writing the length from M1 by
+		 * OR-ing the command in the M1 register and MOVE the
+		 * result into the descriptor buffer. Care must be taken
+		 * wrt. the location of the command because of SEC
+		 * pipelining. The actual MOVEs are written at the end
+		 * of the descriptor due to calculations needed on the
+		 * offset in the descriptor for the MOVE command.
+		 */
+		move_cmd_read_descbuf = MOVE(p, DESCBUF, 0, MATH0, 0, 6,
+					     IMMED);
+		move_cmd_write_descbuf = MOVE(p, MATH0, 0, DESCBUF, 0, 8,
+					      WAITCOMP | IMMED);
+	}
+	MATHB(p, VSEQINSZ, SUB, PDCP_NULL_MAX_FRAME_LEN, NONE, 4,
+	      IMMED2);
+	JUMP(p, PDCP_MAX_FRAME_LEN_STATUS, HALT_STATUS, ALL_FALSE, MATH_N);
+
+	if (rta_sec_era > RTA_SEC_ERA_2) {
+		if (dir == OP_TYPE_ENCAP_PROTOCOL)
+			MATHB(p, VSEQINSZ, ADD, ZERO, MATH0, 4, 0);
+		else
+			MATHB(p, VSEQOUTSZ, ADD, ZERO, MATH0, 4, 0);
+	}
+	SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+	SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | FLUSH1);
+
+	if (rta_sec_era > RTA_SEC_ERA_2) {
+		MOVE(p, AB1, 0, OFIFO, 0, MATH0, 0);
+	} else {
+		SET_LABEL(p, local_offset);
+
+		/* Shut off automatic Info FIFO entries */
+		LOAD(p, 0, DCTRL, LDOFF_DISABLE_AUTO_NFIFO, 0, IMMED);
+		/* Placeholder for MOVE command with length from M1 register */
+		MOVE(p, IFIFOAB1, 0, OFIFO, 0, 0, IMMED);
+		/* Enable automatic Info FIFO entries */
+		LOAD(p, 0, DCTRL, LDOFF_ENABLE_AUTO_NFIFO, 0, IMMED);
+	}
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		MATHB(p, MATH1, XOR, MATH1, MATH0, 8, 0);
+		MOVE(p, MATH0, 0, OFIFO, 0, 4, IMMED);
+	}
+
+	if (rta_sec_era < RTA_SEC_ERA_3) {
+		PATCH_MOVE(p, move_cmd_read_descbuf, local_offset);
+		PATCH_MOVE(p, move_cmd_write_descbuf, local_offset);
+	}
+
+	return 0;
+}
+
+static inline int
+insert_copy_frame_op(struct program *p,
+		     struct alginfo *cipherdata __maybe_unused,
+		     unsigned int dir __maybe_unused)
+{
+	LABEL(local_offset);
+	REFERENCE(move_cmd_read_descbuf);
+	REFERENCE(move_cmd_write_descbuf);
+
+	if (rta_sec_era > RTA_SEC_ERA_2) {
+		MATHB(p, SEQINSZ, ADD, ZERO, VSEQINSZ,  4, 0);
+		MATHB(p, SEQINSZ, ADD, ZERO, VSEQOUTSZ,  4, 0);
+	} else {
+		MATHB(p, SEQINSZ, ADD, ONE, VSEQINSZ,  4, 0);
+		MATHB(p, VSEQINSZ, SUB, ONE, VSEQINSZ,  4, 0);
+		MATHB(p, SEQINSZ, ADD, ONE, VSEQOUTSZ,  4, 0);
+		MATHB(p, VSEQOUTSZ, SUB, ONE, VSEQOUTSZ,  4, 0);
+		MATHB(p, VSEQINSZ, SUB, ONE, MATH0,  4, 0);
+		MATHB(p, MATH0, ADD, ONE, MATH0,  4, 0);
+
+		/*
+		 * Since MOVELEN is available only starting with
+		 * SEC ERA 3, use poor man's MOVELEN: create a MOVE
+		 * command dynamically by writing the length from M1 by
+		 * OR-ing the command in the M1 register and MOVE the
+		 * result into the descriptor buffer. Care must be taken
+		 * wrt. the location of the command because of SEC
+		 * pipelining. The actual MOVEs are written at the end
+		 * of the descriptor due to calculations needed on the
+		 * offset in the descriptor for the MOVE command.
+		 */
+		move_cmd_read_descbuf = MOVE(p, DESCBUF, 0, MATH0, 0, 6,
+					     IMMED);
+		move_cmd_write_descbuf = MOVE(p, MATH0, 0, DESCBUF, 0, 8,
+					      WAITCOMP | IMMED);
+	}
+	MATHB(p, SEQINSZ, SUB, PDCP_NULL_MAX_FRAME_LEN, NONE,  4,
+	      IFB | IMMED2);
+	JUMP(p, PDCP_MAX_FRAME_LEN_STATUS, HALT_STATUS, ALL_FALSE, MATH_N);
+
+	if (rta_sec_era > RTA_SEC_ERA_2)
+		MATHB(p, VSEQINSZ, ADD, ZERO, MATH0,  4, 0);
+
+	SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | FLUSH1);
+	if (rta_sec_era > RTA_SEC_ERA_2) {
+		MOVE(p, AB1, 0, OFIFO, 0, MATH0, 0);
+	} else {
+		SET_LABEL(p, local_offset);
+
+		/* Shut off automatic Info FIFO entries */
+		LOAD(p, 0, DCTRL, LDOFF_DISABLE_AUTO_NFIFO, 0, IMMED);
+
+		/* Placeholder for MOVE command with length from M0 register */
+		MOVE(p, IFIFOAB1, 0, OFIFO, 0, 0, IMMED);
+
+		/* Enable automatic Info FIFO entries */
+		LOAD(p, 0, DCTRL, LDOFF_ENABLE_AUTO_NFIFO, 0, IMMED);
+	}
+
+	SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+
+	if (rta_sec_era < RTA_SEC_ERA_3) {
+		PATCH_MOVE(p, move_cmd_read_descbuf, local_offset);
+		PATCH_MOVE(p, move_cmd_write_descbuf, local_offset);
+	}
+	return 0;
+}
+
+static inline int
+pdcp_insert_cplane_int_only_op(struct program *p,
+			       bool swap __maybe_unused,
+			       struct alginfo *cipherdata __maybe_unused,
+			       struct alginfo *authdata, unsigned int dir,
+			       unsigned char era_2_sw_hfn_ovrd)
+{
+	LABEL(local_offset);
+	REFERENCE(move_cmd_read_descbuf);
+	REFERENCE(move_cmd_write_descbuf);
+
+	switch (authdata->algtype) {
+	case PDCP_AUTH_TYPE_SNOW:
+		/* Insert Auth Key */
+		KEY(p, KEY2, authdata->key_enc_flags, authdata->key,
+		    authdata->keylen, INLINE_KEY(authdata));
+		SEQLOAD(p, MATH0, 7, 1, 0);
+		JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+
+		if (rta_sec_era > RTA_SEC_ERA_2 ||
+		    (rta_sec_era == RTA_SEC_ERA_2 &&
+				   era_2_sw_hfn_ovrd == 0)) {
+			SEQINPTR(p, 0, 1, RTO);
+		} else {
+			SEQINPTR(p, 0, 5, RTO);
+			SEQFIFOLOAD(p, SKIP, 4, 0);
+		}
+
+		if (swap == false) {
+			MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1,  8,
+			      IFB | IMMED2);
+			MATHB(p, MATH1, SHLD, MATH1, MATH1,  8, 0);
+
+			MOVEB(p, DESCBUF, 8, MATH2, 0, 8, WAITCOMP | IMMED);
+
+			MATHB(p, MATH2, AND, PDCP_BEARER_MASK, MATH2, 8,
+			      IMMED2);
+			MOVEB(p, DESCBUF, 0x0C, MATH3, 0, 4, WAITCOMP | IMMED);
+			MATHB(p, MATH3, AND, PDCP_DIR_MASK, MATH3, 8, IMMED2);
+			MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+			MOVEB(p, MATH2, 0, CONTEXT2, 0, 0x0C, WAITCOMP | IMMED);
+		} else {
+			MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK_BE, MATH1,  8,
+			      IFB | IMMED2);
+			MATHB(p, MATH1, SHLD, MATH1, MATH1,  8, 0);
+
+			MOVE(p, DESCBUF, 8, MATH2, 0, 8, WAITCOMP | IMMED);
+			MATHB(p, MATH2, AND, PDCP_BEARER_MASK_BE, MATH2, 8,
+			      IMMED2);
+
+			MOVE(p, DESCBUF, 0x0C, MATH3, 0, 4, WAITCOMP | IMMED);
+			MATHB(p, MATH3, AND, PDCP_DIR_MASK_BE, MATH3, 8,
+			      IMMED2);
+			MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+			MOVE(p, MATH2, 0, CONTEXT2, 0, 0x0C, WAITCOMP | IMMED);
+		}
+
+		if (dir == OP_TYPE_DECAP_PROTOCOL) {
+			MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, MATH1, 4,
+			      IMMED2);
+		} else {
+			if (rta_sec_era > RTA_SEC_ERA_2) {
+				MATHB(p, SEQINSZ, SUB, ZERO, MATH1, 4,
+				      0);
+			} else {
+				MATHB(p, SEQINSZ, ADD, ONE, MATH1, 4,
+				      0);
+				MATHB(p, MATH1, SUB, ONE, MATH1, 4,
+				      0);
+			}
+		}
+
+		if (rta_sec_era > RTA_SEC_ERA_2) {
+			MATHB(p, MATH1, SUB, ZERO, VSEQINSZ, 4, 0);
+			MATHB(p, MATH1, SUB, ZERO, VSEQOUTSZ, 4, 0);
+		} else {
+			MATHB(p, ZERO, ADD, MATH1, VSEQINSZ, 4, 0);
+			MATHB(p, ZERO, ADD, MATH1, VSEQOUTSZ, 4, 0);
+
+			/*
+			 * Since MOVELEN is available only starting with
+			 * SEC ERA 3, use poor man's MOVELEN: create a MOVE
+			 * command dynamically by writing the length from M1 by
+			 * OR-ing the command in the M1 register and MOVE the
+			 * result into the descriptor buffer. Care must be taken
+			 * wrt. the location of the command because of SEC
+			 * pipelining. The actual MOVEs are written at the end
+			 * of the descriptor due to calculations needed on the
+			 * offset in the descriptor for the MOVE command.
+			 */
+			move_cmd_read_descbuf = MOVE(p, DESCBUF, 0, MATH1, 0, 6,
+						     IMMED);
+			move_cmd_write_descbuf = MOVE(p, MATH1, 0, DESCBUF, 0,
+						      8, WAITCOMP | IMMED);
+		}
+
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_SNOW_F9, OP_ALG_AAI_F9,
+			      OP_ALG_AS_INITFINAL,
+			      dir == OP_TYPE_ENCAP_PROTOCOL ?
+				     ICV_CHECK_DISABLE : ICV_CHECK_ENABLE,
+			      DIR_ENC);
+
+		if (rta_sec_era > RTA_SEC_ERA_2) {
+			SEQFIFOLOAD(p, MSGINSNOOP, 0,
+				    VLF | LAST1 | LAST2 | FLUSH1);
+			MOVE(p, AB1, 0, OFIFO, 0, MATH1, 0);
+		} else {
+			SEQFIFOLOAD(p, MSGINSNOOP, 0,
+				    VLF | LAST1 | LAST2 | FLUSH1);
+			SET_LABEL(p, local_offset);
+
+			/* Shut off automatic Info FIFO entries */
+			LOAD(p, 0, DCTRL, LDOFF_DISABLE_AUTO_NFIFO, 0, IMMED);
+			/*
+			 * Placeholder for MOVE command with length from M1
+			 * register
+			 */
+			MOVE(p, IFIFOAB1, 0, OFIFO, 0, 0, IMMED);
+			/* Enable automatic Info FIFO entries */
+			LOAD(p, 0, DCTRL, LDOFF_ENABLE_AUTO_NFIFO, 0, IMMED);
+		}
+
+		if (dir == OP_TYPE_DECAP_PROTOCOL)
+			SEQFIFOLOAD(p, ICV2, 4, LAST2);
+		else
+			SEQSTORE(p, CONTEXT2, 0, 4, 0);
+
+		break;
+
+	case PDCP_AUTH_TYPE_AES:
+		/* Insert Auth Key */
+		KEY(p, KEY1, authdata->key_enc_flags, authdata->key,
+		    authdata->keylen, INLINE_KEY(authdata));
+		SEQLOAD(p, MATH0, 7, 1, 0);
+		JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+		if (rta_sec_era > RTA_SEC_ERA_2 ||
+		    (rta_sec_era == RTA_SEC_ERA_2 &&
+		     era_2_sw_hfn_ovrd == 0)) {
+			SEQINPTR(p, 0, 1, RTO);
+		} else {
+			SEQINPTR(p, 0, 5, RTO);
+			SEQFIFOLOAD(p, SKIP, 4, 0);
+		}
+
+		if (swap == false) {
+			MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			      IFB | IMMED2);
+			MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+
+			MOVEB(p, DESCBUF, 8, MATH2, 0, 8, WAITCOMP | IMMED);
+			MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+			MOVEB(p, MATH2, 0, IFIFOAB1, 0, 8, IMMED);
+		} else {
+			MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK_BE, MATH1, 8,
+			      IFB | IMMED2);
+			MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+
+			MOVE(p, DESCBUF, 8, MATH2, 0, 8, WAITCOMP | IMMED);
+			MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+			MOVE(p, MATH2, 0, IFIFOAB1, 0, 8, IMMED);
+		}
+
+		if (dir == OP_TYPE_DECAP_PROTOCOL) {
+			MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, MATH1, 4,
+			      IMMED2);
+		} else {
+			if (rta_sec_era > RTA_SEC_ERA_2) {
+				MATHB(p, SEQINSZ, SUB, ZERO, MATH1, 4,
+				      0);
+			} else {
+				MATHB(p, SEQINSZ, ADD, ONE, MATH1, 4,
+				      0);
+				MATHB(p, MATH1, SUB, ONE, MATH1, 4,
+				      0);
+			}
+		}
+
+		if (rta_sec_era > RTA_SEC_ERA_2) {
+			MATHB(p, MATH1, SUB, ZERO, VSEQINSZ, 4, 0);
+			MATHB(p, MATH1, SUB, ZERO, VSEQOUTSZ, 4, 0);
+		} else {
+			MATHB(p, ZERO, ADD, MATH1, VSEQINSZ, 4, 0);
+			MATHB(p, ZERO, ADD, MATH1, VSEQOUTSZ, 4, 0);
+
+			/*
+			 * Since MOVELEN is available only starting with
+			 * SEC ERA 3, use poor man's MOVELEN: create a MOVE
+			 * command dynamically by writing the length from M1 by
+			 * OR-ing the command in the M1 register and MOVE the
+			 * result into the descriptor buffer. Care must be taken
+			 * wrt. the location of the command because of SEC
+			 * pipelining. The actual MOVEs are written at the end
+			 * of the descriptor due to calculations needed on the
+			 * offset in the descriptor for the MOVE command.
+			 */
+			move_cmd_read_descbuf = MOVE(p, DESCBUF, 0, MATH1, 0, 6,
+						     IMMED);
+			move_cmd_write_descbuf = MOVE(p, MATH1, 0, DESCBUF, 0,
+						      8, WAITCOMP | IMMED);
+		}
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_AES,
+			      OP_ALG_AAI_CMAC,
+			      OP_ALG_AS_INITFINAL,
+			      dir == OP_TYPE_ENCAP_PROTOCOL ?
+				     ICV_CHECK_DISABLE : ICV_CHECK_ENABLE,
+			      DIR_ENC);
+
+		if (rta_sec_era > RTA_SEC_ERA_2) {
+			MOVE(p, AB2, 0, OFIFO, 0, MATH1, 0);
+			SEQFIFOLOAD(p, MSGINSNOOP, 0,
+				    VLF | LAST1 | LAST2 | FLUSH1);
+		} else {
+			SEQFIFOLOAD(p, MSGINSNOOP, 0,
+				    VLF | LAST1 | LAST2 | FLUSH1);
+			SET_LABEL(p, local_offset);
+
+			/* Shut off automatic Info FIFO entries */
+			LOAD(p, 0, DCTRL, LDOFF_DISABLE_AUTO_NFIFO, 0, IMMED);
+
+			/*
+			 * Placeholder for MOVE command with length from
+			 * M1 register
+			 */
+			MOVE(p, IFIFOAB2, 0, OFIFO, 0, 0, IMMED);
+
+			/* Enable automatic Info FIFO entries */
+			LOAD(p, 0, DCTRL, LDOFF_ENABLE_AUTO_NFIFO, 0, IMMED);
+		}
+
+		if (dir == OP_TYPE_DECAP_PROTOCOL)
+			SEQFIFOLOAD(p, ICV1, 4, LAST1 | FLUSH1);
+		else
+			SEQSTORE(p, CONTEXT1, 0, 4, 0);
+
+		break;
+
+	case PDCP_AUTH_TYPE_ZUC:
+		if (rta_sec_era < RTA_SEC_ERA_5) {
+			pr_err("Invalid era for selected algorithm\n");
+			return -ENOTSUP;
+		}
+		/* Insert Auth Key */
+		KEY(p, KEY2, authdata->key_enc_flags, authdata->key,
+		    authdata->keylen, INLINE_KEY(authdata));
+		SEQLOAD(p, MATH0, 7, 1, 0);
+		JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+		SEQINPTR(p, 0, 1, RTO);
+		if (swap == false) {
+			MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			      IFB | IMMED2);
+			MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+
+			MOVEB(p, DESCBUF, 8, MATH2, 0, 8, WAITCOMP | IMMED);
+			MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+			MOVEB(p, MATH2, 0, CONTEXT2, 0, 8, IMMED);
+
+		} else {
+			MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK_BE, MATH1, 8,
+			      IFB | IMMED2);
+			MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+
+			MOVE(p, DESCBUF, 8, MATH2, 0, 8, WAITCOMP | IMMED);
+			MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+			MOVE(p, MATH2, 0, CONTEXT2, 0, 8, IMMED);
+		}
+		if (dir == OP_TYPE_DECAP_PROTOCOL)
+			MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, MATH1, 4,
+			      IMMED2);
+		else
+			MATHB(p, SEQINSZ, SUB, ZERO, MATH1, 4, 0);
+
+		MATHB(p, MATH1, SUB, ZERO, VSEQINSZ, 4, 0);
+		MATHB(p, MATH1, SUB, ZERO, VSEQOUTSZ, 4, 0);
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_ZUCA,
+			      OP_ALG_AAI_F9,
+			      OP_ALG_AS_INITFINAL,
+			      dir == OP_TYPE_ENCAP_PROTOCOL ?
+				     ICV_CHECK_DISABLE : ICV_CHECK_ENABLE,
+			      DIR_ENC);
+		SEQFIFOLOAD(p, MSGINSNOOP, 0, VLF | LAST1 | LAST2 | FLUSH1);
+		MOVE(p, AB1, 0, OFIFO, 0, MATH1, 0);
+
+		if (dir == OP_TYPE_DECAP_PROTOCOL)
+			SEQFIFOLOAD(p, ICV2, 4, LAST2);
+		else
+			SEQSTORE(p, CONTEXT2, 0, 4, 0);
+
+		break;
+
+	default:
+		pr_err("%s: Invalid integrity algorithm selected: %d\n",
+		       "pdcp_insert_cplane_int_only_op", authdata->algtype);
+		return -EINVAL;
+	}
+
+	if (rta_sec_era < RTA_SEC_ERA_3) {
+		PATCH_MOVE(p, move_cmd_read_descbuf, local_offset);
+		PATCH_MOVE(p, move_cmd_write_descbuf, local_offset);
+	}
+
+	return 0;
+}
+
+static inline int
+pdcp_insert_cplane_enc_only_op(struct program *p,
+			       bool swap __maybe_unused,
+			       struct alginfo *cipherdata,
+			       struct alginfo *authdata __maybe_unused,
+			       unsigned int dir,
+			       unsigned char era_2_sw_hfn_ovrd __maybe_unused)
+{
+	/* Insert Cipher Key */
+	KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+	    cipherdata->keylen, INLINE_KEY(cipherdata));
+
+	if (rta_sec_era >= RTA_SEC_ERA_8) {
+		PROTOCOL(p, dir, OP_PCLID_LTE_PDCP_CTRL_MIXED,
+				(uint16_t)cipherdata->algtype << 8);
+		return 0;
+	}
+
+	SEQLOAD(p, MATH0, 7, 1, 0);
+	JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+	if (swap == false)
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			IFB | IMMED2);
+	else
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK_BE, MATH1, 8,
+			IFB | IMMED2);
+	SEQSTORE(p, MATH0, 7, 1, 0);
+	MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+	MOVE(p, DESCBUF, 8, MATH2, 0, 8, WAITCOMP | IMMED);
+	MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+
+	switch (cipherdata->algtype) {
+	case PDCP_CIPHER_TYPE_SNOW:
+		MOVE(p, MATH2, 0, CONTEXT1, 0, 8, WAITCOMP | IMMED);
+
+		if (rta_sec_era > RTA_SEC_ERA_2) {
+			MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+		} else {
+			MATHB(p, SEQINSZ, SUB, ONE, MATH1, 4, 0);
+			MATHB(p, MATH1, ADD, ONE, VSEQINSZ, 4, 0);
+		}
+
+		if (dir == OP_TYPE_ENCAP_PROTOCOL)
+			MATHB(p, SEQINSZ, ADD, PDCP_MAC_I_LEN, VSEQOUTSZ, 4,
+			      IMMED2);
+		else
+			MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, VSEQOUTSZ, 4,
+			      IMMED2);
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF | CONT);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_SNOW_F8,
+			      OP_ALG_AAI_F8,
+			      OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE,
+			      dir == OP_TYPE_ENCAP_PROTOCOL ?
+					DIR_ENC : DIR_DEC);
+		break;
+
+	case PDCP_CIPHER_TYPE_AES:
+		MOVE(p, MATH2, 0, CONTEXT1, 0x10, 0x10, WAITCOMP | IMMED);
+
+		if (rta_sec_era > RTA_SEC_ERA_2) {
+			MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+		} else {
+			MATHB(p, SEQINSZ, SUB, ONE, MATH1, 4, 0);
+			MATHB(p, MATH1, ADD, ONE, VSEQINSZ, 4, 0);
+		}
+
+		if (dir == OP_TYPE_ENCAP_PROTOCOL)
+			MATHB(p, SEQINSZ, ADD, PDCP_MAC_I_LEN, VSEQOUTSZ, 4,
+			      IMMED2);
+		else
+			MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, VSEQOUTSZ, 4,
+			      IMMED2);
+
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF | CONT);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_AES,
+			      OP_ALG_AAI_CTR,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      dir == OP_TYPE_ENCAP_PROTOCOL ?
+					DIR_ENC : DIR_DEC);
+		break;
+
+	case PDCP_CIPHER_TYPE_ZUC:
+		if (rta_sec_era < RTA_SEC_ERA_5) {
+			pr_err("Invalid era for selected algorithm\n");
+			return -ENOTSUP;
+		}
+
+		MOVE(p, MATH2, 0, CONTEXT1, 0, 0x08, IMMED);
+		MOVE(p, MATH2, 0, CONTEXT1, 0x08, 0x08, WAITCOMP | IMMED);
+		MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+		if (dir == OP_TYPE_ENCAP_PROTOCOL)
+			MATHB(p, SEQINSZ, ADD, PDCP_MAC_I_LEN, VSEQOUTSZ, 4,
+			      IMMED2);
+		else
+			MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, VSEQOUTSZ, 4,
+			      IMMED2);
+
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF | CONT);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_ZUCE,
+			      OP_ALG_AAI_F8,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      dir == OP_TYPE_ENCAP_PROTOCOL ?
+					DIR_ENC : DIR_DEC);
+		break;
+
+	default:
+		pr_err("%s: Invalid encrypt algorithm selected: %d\n",
+		       "pdcp_insert_cplane_enc_only_op", cipherdata->algtype);
+		return -EINVAL;
+	}
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		SEQFIFOLOAD(p, MSG1, 0, VLF);
+		FIFOLOAD(p, MSG1, PDCP_NULL_INT_MAC_I_VAL, 4,
+			 LAST1 | FLUSH1 | IMMED);
+	} else {
+		SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | FLUSH1);
+		MOVE(p, OFIFO, 0, MATH1, 4, PDCP_MAC_I_LEN, WAITCOMP | IMMED);
+		MATHB(p, MATH1, XOR, PDCP_NULL_INT_MAC_I_VAL, NONE, 4, IMMED2);
+		JUMP(p, PDCP_NULL_INT_ICV_CHECK_FAILED_STATUS,
+		     HALT_STATUS, ALL_FALSE, MATH_Z);
+	}
+
+	return 0;
+}
+
+static inline int
+pdcp_insert_cplane_acc_op(struct program *p,
+			  bool swap __maybe_unused,
+			  struct alginfo *cipherdata,
+			  struct alginfo *authdata,
+			  unsigned int dir,
+			  unsigned char era_2_hfn_ovrd __maybe_unused)
+{
+	/* Insert Auth Key */
+	KEY(p, KEY2, authdata->key_enc_flags, authdata->key, authdata->keylen,
+	    INLINE_KEY(authdata));
+
+	/* Insert Cipher Key */
+	KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+	    cipherdata->keylen, INLINE_KEY(cipherdata));
+	PROTOCOL(p, dir, OP_PCLID_LTE_PDCP_CTRL, (uint16_t)cipherdata->algtype);
+
+	return 0;
+}
+
+static inline int
+pdcp_insert_cplane_snow_aes_op(struct program *p,
+			       bool swap __maybe_unused,
+			       struct alginfo *cipherdata,
+			       struct alginfo *authdata,
+			       unsigned int dir,
+			       unsigned char era_2_sw_hfn_ovrd)
+{
+	LABEL(back_to_sd_offset);
+	LABEL(end_desc);
+	LABEL(local_offset);
+	LABEL(jump_to_beginning);
+	LABEL(fifo_load_mac_i_offset);
+	REFERENCE(seqin_ptr_read);
+	REFERENCE(seqin_ptr_write);
+	REFERENCE(seq_out_read);
+	REFERENCE(jump_back_to_sd_cmd);
+	REFERENCE(move_mac_i_to_desc_buf);
+
+	if (rta_sec_era >= RTA_SEC_ERA_8) {
+		KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+				cipherdata->keylen, INLINE_KEY(cipherdata));
+		KEY(p, KEY2, authdata->key_enc_flags, authdata->key,
+				authdata->keylen, INLINE_KEY(authdata));
+
+		PROTOCOL(p, dir, OP_PCLID_LTE_PDCP_CTRL_MIXED,
+			 ((uint16_t)cipherdata->algtype << 8) |
+			 (uint16_t)authdata->algtype);
+
+		return 0;
+	}
+
+	SEQLOAD(p, MATH0, 7, 1, 0);
+	JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+	if (swap == false)
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			IFB | IMMED2);
+	else
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK_BE, MATH1, 8,
+			IFB | IMMED2);
+	MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+	MOVE(p, DESCBUF, 4, MATH2, 0, 0x08, WAITCOMP | IMMED);
+	MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+	SEQSTORE(p, MATH0, 7, 1, 0);
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		if (rta_sec_era > RTA_SEC_ERA_2 ||
+		    (rta_sec_era == RTA_SEC_ERA_2 &&
+				   era_2_sw_hfn_ovrd == 0)) {
+			SEQINPTR(p, 0, 1, RTO);
+		} else {
+			SEQINPTR(p, 0, 5, RTO);
+			SEQFIFOLOAD(p, SKIP, 4, 0);
+		}
+		KEY(p, KEY1, authdata->key_enc_flags, authdata->key,
+		    authdata->keylen, INLINE_KEY(authdata));
+		MOVE(p, MATH2, 0, IFIFOAB1, 0, 0x08, IMMED);
+
+		if (rta_sec_era > RTA_SEC_ERA_2) {
+			MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+			MATHB(p, SEQINSZ, SUB, ZERO, MATH1, 4, 0);
+			MATHB(p, VSEQINSZ, ADD, PDCP_MAC_I_LEN - 1, VSEQOUTSZ,
+			      4, IMMED2);
+		} else {
+			MATHB(p, SEQINSZ, SUB, MATH3, VSEQINSZ, 4, 0);
+			MATHB(p, VSEQINSZ, ADD, PDCP_MAC_I_LEN - 1, VSEQOUTSZ,
+			      4, IMMED2);
+			/*
+			 * Note: Although the calculations below might seem a
+			 * little off, the logic is the following:
+			 *
+			 * - SEQ IN PTR RTO below needs the full length of the
+			 *   frame; in case of P4080_REV_2_HFN_OV_WORKAROUND,
+			 *   this means the length of the frame to be processed
+			 *   + 4 bytes (the HFN override flag and value).
+			 *   The length of the frame to be processed minus 1
+			 *   byte is in the VSIL register (because
+			 *   VSIL = SIL + 3, due to 1 byte, the header being
+			 *   already written by the SEQ STORE above). So for
+			 *   calculating the length to use in RTO, I add one
+			 *   to the VSIL value in order to obtain the total
+			 *   frame length. This helps in case of P4080 which
+			 *   can have the value 0 as an operand in a MATH
+			 *   command only as SRC1 When the HFN override
+			 *   workaround is not enabled, the length of the
+			 *   frame is given by the SIL register; the
+			 *   calculation is similar to the one in the SEC 4.2
+			 *   and SEC 5.3 cases.
+			 */
+			if (era_2_sw_hfn_ovrd)
+				MATHB(p, VSEQOUTSZ, ADD, ONE, MATH1, 4,
+				      0);
+			else
+				MATHB(p, SEQINSZ, ADD, MATH3, MATH1, 4,
+				      0);
+		}
+		/*
+		 * Placeholder for filling the length in
+		 * SEQIN PTR RTO below
+		 */
+		seqin_ptr_read = MOVE(p, DESCBUF, 0, MATH1, 0, 6, IMMED);
+		seqin_ptr_write = MOVE(p, MATH1, 0, DESCBUF, 0, 8,
+				       WAITCOMP | IMMED);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_AES,
+			      OP_ALG_AAI_CMAC,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      DIR_DEC);
+		SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | FLUSH1);
+		MOVE(p, CONTEXT1, 0, MATH3, 0, 4, WAITCOMP | IMMED);
+		if (rta_sec_era <= RTA_SEC_ERA_3)
+			LOAD(p, CLRW_CLR_C1KEY |
+			     CLRW_CLR_C1CTX |
+			     CLRW_CLR_C1ICV |
+			     CLRW_CLR_C1DATAS |
+			     CLRW_CLR_C1MODE,
+			     CLRW, 0, 4, IMMED);
+		else
+			LOAD(p, CLRW_RESET_CLS1_CHA |
+			     CLRW_CLR_C1KEY |
+			     CLRW_CLR_C1CTX |
+			     CLRW_CLR_C1ICV |
+			     CLRW_CLR_C1DATAS |
+			     CLRW_CLR_C1MODE,
+			     CLRW, 0, 4, IMMED);
+
+		if (rta_sec_era <= RTA_SEC_ERA_3)
+			LOAD(p, CCTRL_RESET_CHA_ALL, CCTRL, 0, 4, IMMED);
+
+		KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+		    cipherdata->keylen, INLINE_KEY(cipherdata));
+		SET_LABEL(p, local_offset);
+		MOVE(p, MATH2, 0, CONTEXT1, 0, 8, IMMED);
+		SEQINPTR(p, 0, 0, RTO);
+
+		if (rta_sec_era == RTA_SEC_ERA_2 && era_2_sw_hfn_ovrd) {
+			SEQFIFOLOAD(p, SKIP, 5, 0);
+			MATHB(p, SEQINSZ, ADD, ONE, SEQINSZ, 4, 0);
+		}
+
+		MATHB(p, SEQINSZ, SUB, ONE, VSEQINSZ, 4, 0);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_SNOW_F8,
+			      OP_ALG_AAI_F8,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      DIR_ENC);
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+
+		if (rta_sec_era > RTA_SEC_ERA_2 ||
+		    (rta_sec_era == RTA_SEC_ERA_2 &&
+				   era_2_sw_hfn_ovrd == 0))
+			SEQFIFOLOAD(p, SKIP, 1, 0);
+
+		SEQFIFOLOAD(p, MSG1, 0, VLF);
+		MOVE(p, MATH3, 0, IFIFOAB1, 0, 4, LAST1 | FLUSH1 | IMMED);
+		PATCH_MOVE(p, seqin_ptr_read, local_offset);
+		PATCH_MOVE(p, seqin_ptr_write, local_offset);
+	} else {
+		MOVE(p, MATH2, 0, CONTEXT1, 0, 8, IMMED);
+
+		if (rta_sec_era >= RTA_SEC_ERA_5)
+			MOVE(p, CONTEXT1, 0, CONTEXT2, 0, 8, IMMED);
+
+		if (rta_sec_era > RTA_SEC_ERA_2)
+			MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+		else
+			MATHB(p, SEQINSZ, SUB, MATH3, VSEQINSZ, 4, 0);
+
+		MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, VSEQOUTSZ, 4, IMMED2);
+/*
+ * TODO: To be changed when proper support is added in RTA (can't load a
+ * command that is also written by RTA (or patch it for that matter).
+ * Change when proper RTA support is added.
+ */
+		if (p->ps)
+			WORD(p, 0x168B0004);
+		else
+			WORD(p, 0x16880404);
+
+		jump_back_to_sd_cmd = JUMP(p, 0, LOCAL_JUMP, ALL_TRUE, 0);
+		/*
+		 * Placeholder for command reading  the SEQ OUT command in
+		 * JD. Done for rereading the decrypted data and performing
+		 * the integrity check
+		 */
+/*
+ * TODO: RTA currently doesn't support patching of length of a MOVE command
+ * Thus, it is inserted as a raw word, as per PS setting.
+ */
+		if (p->ps)
+			seq_out_read = MOVE(p, DESCBUF, 0, MATH1, 0, 20,
+					    WAITCOMP | IMMED);
+		else
+			seq_out_read = MOVE(p, DESCBUF, 0, MATH1, 0, 16,
+					    WAITCOMP | IMMED);
+
+		MATHB(p, MATH1, XOR, CMD_SEQ_IN_PTR ^ CMD_SEQ_OUT_PTR, MATH1, 4,
+		      IMMED2);
+		/* Placeholder for overwriting the SEQ IN  with SEQ OUT */
+/*
+ * TODO: RTA currently doesn't support patching of length of a MOVE command
+ * Thus, it is inserted as a raw word, as per PS setting.
+ */
+		if (p->ps)
+			MOVE(p, MATH1, 0, DESCBUF, 0, 24, IMMED);
+		else
+			MOVE(p, MATH1, 0, DESCBUF, 0, 20, IMMED);
+
+		KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+		    cipherdata->keylen, INLINE_KEY(cipherdata));
+
+		if (rta_sec_era >= RTA_SEC_ERA_4)
+			MOVE(p, CONTEXT1, 0, CONTEXT2, 0, 8, IMMED);
+		else
+			MOVE(p, CONTEXT1, 0, MATH3, 0, 8, IMMED);
+
+		ALG_OPERATION(p, OP_ALG_ALGSEL_SNOW_F8,
+			      OP_ALG_AAI_F8,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      DIR_DEC);
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF | CONT);
+		SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | FLUSH1);
+
+		if (rta_sec_era <= RTA_SEC_ERA_3)
+			move_mac_i_to_desc_buf = MOVE(p, OFIFO, 0, DESCBUF, 0,
+						      4, WAITCOMP | IMMED);
+		else
+			MOVE(p, OFIFO, 0, MATH3, 0, 4, IMMED);
+
+		if (rta_sec_era <= RTA_SEC_ERA_3)
+			LOAD(p, CCTRL_RESET_CHA_ALL, CCTRL, 0, 4, IMMED);
+		else
+			LOAD(p, CLRW_RESET_CLS1_CHA |
+			     CLRW_CLR_C1KEY |
+			     CLRW_CLR_C1CTX |
+			     CLRW_CLR_C1ICV |
+			     CLRW_CLR_C1DATAS |
+			     CLRW_CLR_C1MODE,
+			     CLRW, 0, 4, IMMED);
+
+		KEY(p, KEY1, authdata->key_enc_flags, authdata->key,
+		    authdata->keylen, INLINE_KEY(authdata));
+		/*
+		 * Placeholder for jump in SD for executing the new SEQ IN PTR
+		 * command (which is actually the old SEQ OUT PTR command
+		 * copied over from JD.
+		 */
+		SET_LABEL(p, jump_to_beginning);
+		JUMP(p, 1 - jump_to_beginning, LOCAL_JUMP, ALL_TRUE, 0);
+		SET_LABEL(p, back_to_sd_offset);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_AES,
+			      OP_ALG_AAI_CMAC,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_ENABLE,
+			      DIR_DEC);
+
+		/* Read the # of bytes written in the output buffer + 1 (HDR) */
+		MATHB(p, VSEQOUTSZ, ADD, ONE, VSEQINSZ, 4, 0);
+
+		if (rta_sec_era <= RTA_SEC_ERA_3)
+			MOVE(p, MATH3, 0, IFIFOAB1, 0, 8, IMMED);
+		else
+			MOVE(p, CONTEXT2, 0, IFIFOAB1, 0, 8, IMMED);
+
+		if (rta_sec_era == RTA_SEC_ERA_2 && era_2_sw_hfn_ovrd)
+			SEQFIFOLOAD(p, SKIP, 4, 0);
+
+		SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | FLUSH1);
+
+		if (rta_sec_era >= RTA_SEC_ERA_4) {
+			LOAD(p, NFIFOENTRY_STYPE_ALTSOURCE |
+			     NFIFOENTRY_DEST_CLASS1 |
+			     NFIFOENTRY_DTYPE_ICV |
+			     NFIFOENTRY_LC1 |
+			     NFIFOENTRY_FC1 | 4, NFIFO_SZL, 0, 4, IMMED);
+			MOVE(p, MATH3, 0, ALTSOURCE, 0, 4, IMMED);
+		} else {
+			SET_LABEL(p, fifo_load_mac_i_offset);
+			FIFOLOAD(p, ICV1, fifo_load_mac_i_offset, 4,
+				 LAST1 | FLUSH1 | IMMED);
+		}
+
+		SET_LABEL(p, end_desc);
+
+		if (!p->ps) {
+			PATCH_MOVE(p, seq_out_read, end_desc + 1);
+			PATCH_JUMP(p, jump_back_to_sd_cmd,
+				   back_to_sd_offset + jump_back_to_sd_cmd - 5);
+
+			if (rta_sec_era <= RTA_SEC_ERA_3)
+				PATCH_MOVE(p, move_mac_i_to_desc_buf,
+					   fifo_load_mac_i_offset + 1);
+		} else {
+			PATCH_MOVE(p, seq_out_read, end_desc + 2);
+			PATCH_JUMP(p, jump_back_to_sd_cmd,
+				   back_to_sd_offset + jump_back_to_sd_cmd - 5);
+
+			if (rta_sec_era <= RTA_SEC_ERA_3)
+				PATCH_MOVE(p, move_mac_i_to_desc_buf,
+					   fifo_load_mac_i_offset + 1);
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+pdcp_insert_cplane_aes_snow_op(struct program *p,
+			       bool swap __maybe_unused,
+			       struct alginfo *cipherdata,
+			       struct alginfo *authdata,
+			       unsigned int dir,
+			       unsigned char era_2_sw_hfn_ovrd __maybe_unused)
+{
+	KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+	    cipherdata->keylen, INLINE_KEY(cipherdata));
+	KEY(p, KEY2, authdata->key_enc_flags, authdata->key, authdata->keylen,
+	    INLINE_KEY(authdata));
+
+	if (rta_sec_era >= RTA_SEC_ERA_8) {
+		PROTOCOL(p, dir, OP_PCLID_LTE_PDCP_CTRL_MIXED,
+			 ((uint16_t)cipherdata->algtype << 8) |
+			 (uint16_t)authdata->algtype);
+
+		return 0;
+	}
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL)
+		MATHB(p, SEQINSZ, SUB, ONE, VSEQINSZ, 4, 0);
+
+	SEQLOAD(p, MATH0, 7, 1, 0);
+	JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+	MOVE(p, MATH0, 7, IFIFOAB2, 0, 1, IMMED);
+	if (swap == false)
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			IFB | IMMED2);
+	else
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK_BE, MATH1, 8,
+			IFB | IMMED2);
+
+	SEQSTORE(p, MATH0, 7, 1, 0);
+	MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+	MOVE(p, DESCBUF, 4, MATH2, 0, 8, WAITCOMP | IMMED);
+	MATHB(p, MATH1, OR, MATH2, MATH1, 8, 0);
+	MOVE(p, MATH1, 0, CONTEXT1, 16, 8, IMMED);
+	MOVE(p, MATH1, 0, CONTEXT2, 0, 4, IMMED);
+	if (swap == false) {
+		MATHB(p, MATH1, AND, lower_32_bits(PDCP_BEARER_MASK), MATH2, 4,
+			IMMED2);
+		MATHB(p, MATH1, AND, upper_32_bits(PDCP_DIR_MASK), MATH3, 4,
+			IMMED2);
+	} else {
+		MATHB(p, MATH1, AND, lower_32_bits(PDCP_BEARER_MASK_BE), MATH2,
+			4, IMMED2);
+		MATHB(p, MATH1, AND, upper_32_bits(PDCP_DIR_MASK_BE), MATH3,
+			4, IMMED2);
+	}
+	MATHB(p, MATH3, SHLD, MATH3, MATH3, 8, 0);
+	MOVE(p, MATH2, 4, OFIFO, 0, 12, IMMED);
+	MOVE(p, OFIFO, 0, CONTEXT2, 4, 12, IMMED);
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		MATHB(p, SEQINSZ, ADD, PDCP_MAC_I_LEN, VSEQOUTSZ, 4, IMMED2);
+	} else {
+		MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, MATH1, 4, IMMED2);
+
+		MATHB(p, ZERO, ADD, MATH1, VSEQOUTSZ, 4, 0);
+		MATHB(p, ZERO, ADD, MATH1, VSEQINSZ, 4, 0);
+	}
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL)
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+	else
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF | CONT);
+
+	ALG_OPERATION(p, OP_ALG_ALGSEL_SNOW_F9,
+		      OP_ALG_AAI_F9,
+		      OP_ALG_AS_INITFINAL,
+		      dir == OP_TYPE_ENCAP_PROTOCOL ?
+			     ICV_CHECK_DISABLE : ICV_CHECK_ENABLE,
+		      DIR_DEC);
+	ALG_OPERATION(p, OP_ALG_ALGSEL_AES,
+		      OP_ALG_AAI_CTR,
+		      OP_ALG_AS_INITFINAL,
+		      ICV_CHECK_DISABLE,
+		      dir == OP_TYPE_ENCAP_PROTOCOL ? DIR_ENC : DIR_DEC);
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		SEQFIFOLOAD(p, MSGINSNOOP, 0, VLF | LAST2);
+		MOVE(p, CONTEXT2, 0, IFIFOAB1, 0, 4, LAST1 | FLUSH1 | IMMED);
+	} else {
+		SEQFIFOLOAD(p, MSGOUTSNOOP, 0, VLF | LAST2);
+		SEQFIFOLOAD(p, MSG1, 4, LAST1 | FLUSH1);
+		JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CLASS1 | NOP | NIFP);
+
+		if (rta_sec_era >= RTA_SEC_ERA_6)
+			LOAD(p, 0, DCTRL, 0, LDLEN_RST_CHA_OFIFO_PTR, IMMED);
+
+		MOVE(p, OFIFO, 0, MATH0, 0, 4, WAITCOMP | IMMED);
+
+		NFIFOADD(p, IFIFO, ICV2, 4, LAST2);
+
+		if (rta_sec_era <= RTA_SEC_ERA_2) {
+			/* Shut off automatic Info FIFO entries */
+			LOAD(p, 0, DCTRL, LDOFF_DISABLE_AUTO_NFIFO, 0, IMMED);
+			MOVE(p, MATH0, 0, IFIFOAB2, 0, 4, WAITCOMP | IMMED);
+		} else {
+			MOVE(p, MATH0, 0, IFIFO, 0, 4, WAITCOMP | IMMED);
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+pdcp_insert_cplane_snow_zuc_op(struct program *p,
+			       bool swap __maybe_unused,
+			       struct alginfo *cipherdata,
+			       struct alginfo *authdata,
+			       unsigned int dir,
+			       unsigned char era_2_sw_hfn_ovrd __maybe_unused)
+{
+	LABEL(keyjump);
+	REFERENCE(pkeyjump);
+
+	if (rta_sec_era < RTA_SEC_ERA_5) {
+		pr_err("Invalid era for selected algorithm\n");
+		return -ENOTSUP;
+	}
+
+	pkeyjump = JUMP(p, keyjump, LOCAL_JUMP, ALL_TRUE, SHRD | SELF | BOTH);
+	KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+	    cipherdata->keylen, INLINE_KEY(cipherdata));
+	KEY(p, KEY2, authdata->key_enc_flags, authdata->key, authdata->keylen,
+	    INLINE_KEY(authdata));
+
+	SET_LABEL(p, keyjump);
+
+	if (rta_sec_era >= RTA_SEC_ERA_8) {
+		PROTOCOL(p, dir, OP_PCLID_LTE_PDCP_CTRL_MIXED,
+			 ((uint16_t)cipherdata->algtype << 8) |
+			 (uint16_t)authdata->algtype);
+		return 0;
+	}
+
+	SEQLOAD(p, MATH0, 7, 1, 0);
+	JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+	MOVE(p, MATH0, 7, IFIFOAB2, 0, 1, IMMED);
+	if (swap == false)
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			IFB | IMMED2);
+	else
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			IFB | IMMED2);
+
+	MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+	MOVE(p, DESCBUF, 4, MATH2, 0, 8, WAITCOMP | IMMED);
+	MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+	MOVE(p, MATH2, 0, CONTEXT1, 0, 8, IMMED);
+	MOVE(p, MATH2, 0, CONTEXT2, 0, 8, WAITCOMP | IMMED);
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL)
+		MATHB(p, SEQINSZ, ADD, PDCP_MAC_I_LEN, VSEQOUTSZ, 4, IMMED2);
+	else
+		MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, VSEQOUTSZ, 4, IMMED2);
+
+	MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+	SEQSTORE(p, MATH0, 7, 1, 0);
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+		SEQFIFOLOAD(p, MSGINSNOOP, 0, VLF | LAST2);
+	} else {
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF | CONT);
+		SEQFIFOLOAD(p, MSGOUTSNOOP, 0, VLF | LAST1 | FLUSH1);
+	}
+
+	ALG_OPERATION(p, OP_ALG_ALGSEL_ZUCA,
+		      OP_ALG_AAI_F9,
+		      OP_ALG_AS_INITFINAL,
+		      dir == OP_TYPE_ENCAP_PROTOCOL ?
+			     ICV_CHECK_DISABLE : ICV_CHECK_ENABLE,
+		      DIR_ENC);
+
+	ALG_OPERATION(p, OP_ALG_ALGSEL_SNOW_F8,
+		      OP_ALG_AAI_F8,
+		      OP_ALG_AS_INITFINAL,
+		      ICV_CHECK_DISABLE,
+		      dir == OP_TYPE_ENCAP_PROTOCOL ? DIR_ENC : DIR_DEC);
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		MOVE(p, CONTEXT2, 0, IFIFOAB1, 0, 4, LAST1 | FLUSH1 | IMMED);
+	} else {
+		/* Save ICV */
+		MOVE(p, OFIFO, 0, MATH0, 0, 4, IMMED);
+		LOAD(p, NFIFOENTRY_STYPE_ALTSOURCE |
+		     NFIFOENTRY_DEST_CLASS2 |
+		     NFIFOENTRY_DTYPE_ICV |
+		     NFIFOENTRY_LC2 | 4, NFIFO_SZL, 0, 4, IMMED);
+		MOVE(p, MATH0, 0, ALTSOURCE, 0, 4, WAITCOMP | IMMED);
+	}
+
+	/* Reset ZUCA mode and done interrupt */
+	LOAD(p, CLRW_CLR_C2MODE, CLRW, 0, 4, IMMED);
+	LOAD(p, CIRQ_ZADI, ICTRL, 0, 4, IMMED);
+
+	PATCH_JUMP(p, pkeyjump, keyjump);
+	return 0;
+}
+
+static inline int
+pdcp_insert_cplane_aes_zuc_op(struct program *p,
+			      bool swap __maybe_unused,
+			      struct alginfo *cipherdata,
+			      struct alginfo *authdata,
+			      unsigned int dir,
+			      unsigned char era_2_sw_hfn_ovrd __maybe_unused)
+{
+	LABEL(keyjump);
+	REFERENCE(pkeyjump);
+
+	if (rta_sec_era < RTA_SEC_ERA_5) {
+		pr_err("Invalid era for selected algorithm\n");
+		return -ENOTSUP;
+	}
+
+	pkeyjump = JUMP(p, keyjump, LOCAL_JUMP, ALL_TRUE, SHRD | SELF | BOTH);
+	KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+	    cipherdata->keylen, INLINE_KEY(cipherdata));
+	KEY(p, KEY2, authdata->key_enc_flags, authdata->key, authdata->keylen,
+	    INLINE_KEY(authdata));
+
+	if (rta_sec_era >= RTA_SEC_ERA_8) {
+		PROTOCOL(p, dir, OP_PCLID_LTE_PDCP_CTRL_MIXED,
+			 ((uint16_t)cipherdata->algtype << 8) |
+			 (uint16_t)authdata->algtype);
+
+		return 0;
+	}
+
+	SET_LABEL(p, keyjump);
+	SEQLOAD(p, MATH0, 7, 1, 0);
+	JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+	MOVE(p, MATH0, 7, IFIFOAB2, 0, 1, IMMED);
+	if (swap == false)
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			IFB | IMMED2);
+	else
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			IFB | IMMED2);
+
+	MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+	MOVE(p, DESCBUF, 4, MATH2, 0, 8, WAITCOMP | IMMED);
+	MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+	MOVE(p, MATH2, 0, CONTEXT1, 16, 8, IMMED);
+	MOVE(p, MATH2, 0, CONTEXT2, 0, 8, WAITCOMP | IMMED);
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL)
+		MATHB(p, SEQINSZ, ADD, PDCP_MAC_I_LEN, VSEQOUTSZ, 4, IMMED2);
+	else
+		MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, VSEQOUTSZ, 4, IMMED2);
+
+	MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+	SEQSTORE(p, MATH0, 7, 1, 0);
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+		SEQFIFOLOAD(p, MSGINSNOOP, 0, VLF | LAST2);
+	} else {
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF | CONT);
+		SEQFIFOLOAD(p, MSGOUTSNOOP, 0, VLF | LAST1 | FLUSH1);
+	}
+
+	ALG_OPERATION(p, OP_ALG_ALGSEL_ZUCA,
+		      OP_ALG_AAI_F9,
+		      OP_ALG_AS_INITFINAL,
+		      dir == OP_TYPE_ENCAP_PROTOCOL ?
+			     ICV_CHECK_DISABLE : ICV_CHECK_ENABLE,
+		      DIR_ENC);
+
+	ALG_OPERATION(p, OP_ALG_ALGSEL_AES,
+		      OP_ALG_AAI_CTR,
+		      OP_ALG_AS_INITFINAL,
+		      ICV_CHECK_DISABLE,
+		      dir == OP_TYPE_ENCAP_PROTOCOL ? DIR_ENC : DIR_DEC);
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		MOVE(p, CONTEXT2, 0, IFIFOAB1, 0, 4, LAST1 | FLUSH1 | IMMED);
+	} else {
+		/* Save ICV */
+		MOVE(p, OFIFO, 0, MATH0, 0, 4, IMMED);
+
+		LOAD(p, NFIFOENTRY_STYPE_ALTSOURCE |
+		     NFIFOENTRY_DEST_CLASS2 |
+		     NFIFOENTRY_DTYPE_ICV |
+		     NFIFOENTRY_LC2 | 4, NFIFO_SZL, 0, 4, IMMED);
+		MOVE(p, MATH0, 0, ALTSOURCE, 0, 4, WAITCOMP | IMMED);
+	}
+
+	/* Reset ZUCA mode and done interrupt */
+	LOAD(p, CLRW_CLR_C2MODE, CLRW, 0, 4, IMMED);
+	LOAD(p, CIRQ_ZADI, ICTRL, 0, 4, IMMED);
+
+	PATCH_JUMP(p, pkeyjump, keyjump);
+
+	return 0;
+}
+
+static inline int
+pdcp_insert_cplane_zuc_snow_op(struct program *p,
+			       bool swap __maybe_unused,
+			       struct alginfo *cipherdata,
+			       struct alginfo *authdata,
+			       unsigned int dir,
+			       unsigned char era_2_sw_hfn_ovrd __maybe_unused)
+{
+	LABEL(keyjump);
+	REFERENCE(pkeyjump);
+
+	if (rta_sec_era < RTA_SEC_ERA_5) {
+		pr_err("Invalid era for selected algorithm\n");
+		return -ENOTSUP;
+	}
+
+	pkeyjump = JUMP(p, keyjump, LOCAL_JUMP, ALL_TRUE, SHRD | SELF | BOTH);
+	KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+	    cipherdata->keylen, INLINE_KEY(cipherdata));
+	KEY(p, KEY2, authdata->key_enc_flags, authdata->key, authdata->keylen,
+	    INLINE_KEY(authdata));
+
+	if (rta_sec_era >= RTA_SEC_ERA_8) {
+		PROTOCOL(p, dir, OP_PCLID_LTE_PDCP_CTRL_MIXED,
+			 ((uint16_t)cipherdata->algtype << 8) |
+			 (uint16_t)authdata->algtype);
+
+		return 0;
+	}
+
+	SET_LABEL(p, keyjump);
+	SEQLOAD(p, MATH0, 7, 1, 0);
+	JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+	MOVE(p, MATH0, 7, IFIFOAB2, 0, 1, IMMED);
+	if (swap == false)
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			IFB | IMMED2);
+	else
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			IFB | IMMED2);
+
+	MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+	MOVE(p, DESCBUF, 4, MATH2, 0, 8, WAITCOMP | IMMED);
+	MATHB(p, MATH1, OR, MATH2, MATH1, 8, 0);
+	MOVE(p, MATH1, 0, CONTEXT1, 0, 8, IMMED);
+	MOVE(p, MATH1, 0, CONTEXT2, 0, 4, IMMED);
+	if (swap == false) {
+		MATHB(p, MATH1, AND, lower_32_bits(PDCP_BEARER_MASK), MATH2,
+			4, IMMED2);
+		MATHB(p, MATH1, AND, upper_32_bits(PDCP_DIR_MASK), MATH3,
+			4, IMMED2);
+	} else {
+		MATHB(p, MATH1, AND, lower_32_bits(PDCP_BEARER_MASK_BE), MATH2,
+			4, IMMED2);
+		MATHB(p, MATH1, AND, upper_32_bits(PDCP_DIR_MASK_BE), MATH3,
+			4, IMMED2);
+	}
+	MATHB(p, MATH3, SHLD, MATH3, MATH3, 8, 0);
+	MOVE(p, MATH2, 4, OFIFO, 0, 12, IMMED);
+	MOVE(p, OFIFO, 0, CONTEXT2, 4, 12, IMMED);
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		MATHB(p, SEQINSZ, ADD, PDCP_MAC_I_LEN, VSEQOUTSZ, 4, IMMED2);
+		MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+	} else {
+		MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, VSEQOUTSZ, 4, IMMED2);
+		MATHB(p, VSEQOUTSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+	}
+
+	SEQSTORE(p, MATH0, 7, 1, 0);
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+		SEQFIFOLOAD(p, MSGINSNOOP, 0, VLF | LAST2);
+	} else {
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF | CONT);
+		SEQFIFOLOAD(p, MSGOUTSNOOP, 0, VLF | LAST2);
+	}
+
+	ALG_OPERATION(p, OP_ALG_ALGSEL_SNOW_F9,
+		      OP_ALG_AAI_F9,
+		      OP_ALG_AS_INITFINAL,
+		      dir == OP_TYPE_ENCAP_PROTOCOL ?
+			     ICV_CHECK_DISABLE : ICV_CHECK_ENABLE,
+		      DIR_DEC);
+
+	ALG_OPERATION(p, OP_ALG_ALGSEL_ZUCE,
+		      OP_ALG_AAI_F8,
+		      OP_ALG_AS_INITFINAL,
+		      ICV_CHECK_DISABLE,
+		      dir == OP_TYPE_ENCAP_PROTOCOL ? DIR_ENC : DIR_DEC);
+
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		MOVE(p, CONTEXT2, 0, IFIFOAB1, 0, 4, LAST1 | FLUSH1 | IMMED);
+	} else {
+		SEQFIFOLOAD(p, MSG1, 4, LAST1 | FLUSH1);
+		JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CLASS1 | NOP | NIFP);
+
+		if (rta_sec_era >= RTA_SEC_ERA_6)
+			/*
+			 * For SEC ERA 6, there's a problem with the OFIFO
+			 * pointer, and thus it needs to be reset here before
+			 * moving to M0.
+			 */
+			LOAD(p, 0, DCTRL, 0, LDLEN_RST_CHA_OFIFO_PTR, IMMED);
+
+		/* Put ICV to M0 before sending it to C2 for comparison. */
+		MOVE(p, OFIFO, 0, MATH0, 0, 4, WAITCOMP | IMMED);
+
+		LOAD(p, NFIFOENTRY_STYPE_ALTSOURCE |
+		     NFIFOENTRY_DEST_CLASS2 |
+		     NFIFOENTRY_DTYPE_ICV |
+		     NFIFOENTRY_LC2 | 4, NFIFO_SZL, 0, 4, IMMED);
+		MOVE(p, MATH0, 0, ALTSOURCE, 0, 4, IMMED);
+	}
+
+	PATCH_JUMP(p, pkeyjump, keyjump);
+	return 0;
+}
+
+static inline int
+pdcp_insert_cplane_zuc_aes_op(struct program *p,
+			      bool swap __maybe_unused,
+			      struct alginfo *cipherdata,
+			      struct alginfo *authdata,
+			      unsigned int dir,
+			      unsigned char era_2_sw_hfn_ovrd __maybe_unused)
+{
+	if (rta_sec_era < RTA_SEC_ERA_5) {
+		pr_err("Invalid era for selected algorithm\n");
+		return -ENOTSUP;
+	}
+
+	if (rta_sec_era >= RTA_SEC_ERA_8) {
+		KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+				cipherdata->keylen, INLINE_KEY(cipherdata));
+		KEY(p, KEY2, authdata->key_enc_flags, authdata->key,
+				authdata->keylen, INLINE_KEY(authdata));
+
+		PROTOCOL(p, dir, OP_PCLID_LTE_PDCP_CTRL_MIXED,
+			 ((uint16_t)cipherdata->algtype << 8) |
+			 (uint16_t)authdata->algtype);
+		return 0;
+	}
+
+	SEQLOAD(p, MATH0, 7, 1, 0);
+	JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+	if (swap == false)
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			IFB | IMMED2);
+	else
+		MATHB(p, MATH0, AND, PDCP_C_PLANE_SN_MASK, MATH1, 8,
+			IFB | IMMED2);
+
+	MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+	MOVE(p, DESCBUF, 4, MATH2, 0, 0x08, WAITCOMP | IMMED);
+	MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+	SEQSTORE(p, MATH0, 7, 1, 0);
+	if (dir == OP_TYPE_ENCAP_PROTOCOL) {
+		KEY(p, KEY1, authdata->key_enc_flags, authdata->key,
+		    authdata->keylen, INLINE_KEY(authdata));
+		MOVE(p, MATH2, 0, IFIFOAB1, 0, 0x08, IMMED);
+		MOVE(p, MATH0, 7, IFIFOAB1, 0, 1, IMMED);
+
+		MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+		MATHB(p, VSEQINSZ, ADD, PDCP_MAC_I_LEN, VSEQOUTSZ, 4, IMMED2);
+
+		ALG_OPERATION(p, OP_ALG_ALGSEL_AES,
+			      OP_ALG_AAI_CMAC,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      DIR_DEC);
+		SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | FLUSH1);
+		MOVE(p, CONTEXT1, 0, MATH3, 0, 4, WAITCOMP | IMMED);
+		LOAD(p, CLRW_RESET_CLS1_CHA |
+		     CLRW_CLR_C1KEY |
+		     CLRW_CLR_C1CTX |
+		     CLRW_CLR_C1ICV |
+		     CLRW_CLR_C1DATAS |
+		     CLRW_CLR_C1MODE,
+		     CLRW, 0, 4, IMMED);
+
+		KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+		    cipherdata->keylen, INLINE_KEY(cipherdata));
+
+		MOVE(p, MATH2, 0, CONTEXT1, 0, 8, IMMED);
+		SEQINPTR(p, 0, PDCP_NULL_MAX_FRAME_LEN, RTO);
+
+		ALG_OPERATION(p, OP_ALG_ALGSEL_ZUCE,
+			      OP_ALG_AAI_F8,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      DIR_ENC);
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+
+		SEQFIFOLOAD(p, SKIP, 1, 0);
+
+		SEQFIFOLOAD(p, MSG1, 0, VLF);
+		MOVE(p, MATH3, 0, IFIFOAB1, 0, 4, LAST1 | FLUSH1 | IMMED);
+	} else {
+		MOVE(p, MATH2, 0, CONTEXT1, 0, 8, IMMED);
+
+		MOVE(p, CONTEXT1, 0, CONTEXT2, 0, 8, IMMED);
+
+		MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+
+		MATHB(p, SEQINSZ, SUB, PDCP_MAC_I_LEN, VSEQOUTSZ, 4, IMMED2);
+
+		KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+		    cipherdata->keylen, INLINE_KEY(cipherdata));
+
+		MOVE(p, CONTEXT1, 0, CONTEXT2, 0, 8, IMMED);
+
+		ALG_OPERATION(p, OP_ALG_ALGSEL_ZUCE,
+			      OP_ALG_AAI_F8,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      DIR_DEC);
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF | CONT);
+		SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | FLUSH1);
+
+		MOVE(p, OFIFO, 0, MATH3, 0, 4, IMMED);
+
+		LOAD(p, CLRW_RESET_CLS1_CHA |
+		     CLRW_CLR_C1KEY |
+		     CLRW_CLR_C1CTX |
+		     CLRW_CLR_C1ICV |
+		     CLRW_CLR_C1DATAS |
+		     CLRW_CLR_C1MODE,
+		     CLRW, 0, 4, IMMED);
+
+		KEY(p, KEY1, authdata->key_enc_flags, authdata->key,
+		    authdata->keylen, INLINE_KEY(authdata));
+
+		SEQINPTR(p, 0, 0, SOP);
+
+		ALG_OPERATION(p, OP_ALG_ALGSEL_AES,
+			      OP_ALG_AAI_CMAC,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_ENABLE,
+			      DIR_DEC);
+
+		MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+
+		MOVE(p, CONTEXT2, 0, IFIFOAB1, 0, 8, IMMED);
+
+		SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | FLUSH1);
+
+		LOAD(p, NFIFOENTRY_STYPE_ALTSOURCE |
+		     NFIFOENTRY_DEST_CLASS1 |
+		     NFIFOENTRY_DTYPE_ICV |
+		     NFIFOENTRY_LC1 |
+		     NFIFOENTRY_FC1 | 4, NFIFO_SZL, 0, 4, IMMED);
+		MOVE(p, MATH3, 0, ALTSOURCE, 0, 4, IMMED);
+	}
+
+	return 0;
+}
+
+static inline int
+pdcp_insert_uplane_15bit_op(struct program *p,
+			    bool swap __maybe_unused,
+			    struct alginfo *cipherdata,
+			    unsigned int dir)
+{
+	int op;
+	/* Insert Cipher Key */
+	KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key,
+	    cipherdata->keylen, INLINE_KEY(cipherdata));
+
+	if (rta_sec_era >= RTA_SEC_ERA_8) {
+		PROTOCOL(p, dir, OP_PCLID_LTE_PDCP_USER,
+			 (uint16_t)cipherdata->algtype);
+		return 0;
+	}
+
+	SEQLOAD(p, MATH0, 6, 2, 0);
+	JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+	if (swap == false)
+		MATHB(p, MATH0, AND, PDCP_U_PLANE_15BIT_SN_MASK, MATH1, 8,
+		      IFB | IMMED2);
+	else
+		MATHB(p, MATH0, AND, PDCP_U_PLANE_15BIT_SN_MASK_BE, MATH1, 8,
+		      IFB | IMMED2);
+	SEQSTORE(p, MATH0, 6, 2, 0);
+	MATHB(p, MATH1, SHLD, MATH1, MATH1, 8, 0);
+	MOVE(p, DESCBUF, 8, MATH2, 0, 8, WAITCOMP | IMMED);
+	MATHB(p, MATH1, OR, MATH2, MATH2, 8, 0);
+
+	MATHB(p, SEQINSZ, SUB, MATH3, VSEQINSZ, 4, 0);
+	MATHB(p, SEQINSZ, SUB, MATH3, VSEQOUTSZ, 4, 0);
+
+	SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+
+	op = dir == OP_TYPE_ENCAP_PROTOCOL ? DIR_ENC : DIR_DEC;
+	switch (cipherdata->algtype) {
+	case PDCP_CIPHER_TYPE_SNOW:
+		MOVE(p, MATH2, 0, CONTEXT1, 0, 8, WAITCOMP | IMMED);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_SNOW_F8,
+			      OP_ALG_AAI_F8,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      op);
+		break;
+
+	case PDCP_CIPHER_TYPE_AES:
+		MOVE(p, MATH2, 0, CONTEXT1, 0x10, 0x10, WAITCOMP | IMMED);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_AES,
+			      OP_ALG_AAI_CTR,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      op);
+		break;
+
+	case PDCP_CIPHER_TYPE_ZUC:
+		if (rta_sec_era < RTA_SEC_ERA_5) {
+			pr_err("Invalid era for selected algorithm\n");
+			return -ENOTSUP;
+		}
+		MOVE(p, MATH2, 0, CONTEXT1, 0, 0x08, IMMED);
+		MOVE(p, MATH2, 0, CONTEXT1, 0x08, 0x08, WAITCOMP | IMMED);
+
+		ALG_OPERATION(p, OP_ALG_ALGSEL_ZUCE,
+			      OP_ALG_AAI_F8,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      op);
+		break;
+
+	default:
+		pr_err("%s: Invalid encrypt algorithm selected: %d\n",
+		       "pdcp_insert_uplane_15bit_op", cipherdata->algtype);
+		return -EINVAL;
+	}
+
+	SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | FLUSH1);
+
+	return 0;
+}
+
+/*
+ * Function for inserting the snippet of code responsible for creating
+ * the HFN override code via either DPOVRD or via the input frame.
+ */
+static inline int
+insert_hfn_ov_op(struct program *p,
+		 uint32_t shift,
+		 enum pdb_type_e pdb_type,
+		 unsigned char era_2_sw_hfn_ovrd)
+{
+	uint32_t imm = PDCP_DPOVRD_HFN_OV_EN;
+	uint16_t hfn_pdb_offset;
+
+	if (rta_sec_era == RTA_SEC_ERA_2 && !era_2_sw_hfn_ovrd)
+		return 0;
+
+	switch (pdb_type) {
+	case PDCP_PDB_TYPE_NO_PDB:
+		/*
+		 * If there is no PDB, then HFN override mechanism does not
+		 * make any sense, thus in this case the function will
+		 * return the pointer to the current position in the
+		 * descriptor buffer
+		 */
+		return 0;
+
+	case PDCP_PDB_TYPE_REDUCED_PDB:
+		hfn_pdb_offset = 4;
+		break;
+
+	case PDCP_PDB_TYPE_FULL_PDB:
+		hfn_pdb_offset = 8;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	if (rta_sec_era > RTA_SEC_ERA_2) {
+		MATHB(p, DPOVRD, AND, imm, NONE, 8, IFB | IMMED2);
+	} else {
+		SEQLOAD(p, MATH0, 4, 4, 0);
+		JUMP(p, 1, LOCAL_JUMP, ALL_TRUE, CALM);
+		MATHB(p, MATH0, AND, imm, NONE, 8, IFB | IMMED2);
+		SEQSTORE(p, MATH0, 4, 4, 0);
+	}
+
+	if (rta_sec_era >= RTA_SEC_ERA_8)
+		JUMP(p, 6, LOCAL_JUMP, ALL_TRUE, MATH_Z);
+	else
+		JUMP(p, 5, LOCAL_JUMP, ALL_TRUE, MATH_Z);
+
+	if (rta_sec_era > RTA_SEC_ERA_2)
+		MATHB(p, DPOVRD, LSHIFT, shift, MATH0, 4, IMMED2);
+	else
+		MATHB(p, MATH0, LSHIFT, shift, MATH0, 4, IMMED2);
+
+	MATHB(p, MATH0, SHLD, MATH0, MATH0, 8, 0);
+	MOVE(p, MATH0, 0, DESCBUF, hfn_pdb_offset, 4, IMMED);
+
+	if (rta_sec_era >= RTA_SEC_ERA_8)
+		/*
+		 * For ERA8, DPOVRD could be handled by the PROTOCOL command
+		 * itself. For now, this is not done. Thus, clear DPOVRD here
+		 * to alleviate any side-effects.
+		 */
+		MATHB(p, DPOVRD, AND, ZERO, DPOVRD, 4, STL);
+
+	return 0;
+}
+
+/*
+ * PDCP Control PDB creation function
+ */
+static inline enum pdb_type_e
+cnstr_pdcp_c_plane_pdb(struct program *p,
+		       uint32_t hfn,
+		       unsigned char bearer,
+		       unsigned char direction,
+		       uint32_t hfn_threshold,
+		       struct alginfo *cipherdata,
+		       struct alginfo *authdata)
+{
+	struct pdcp_pdb pdb;
+	enum pdb_type_e
+		pdb_mask[PDCP_CIPHER_TYPE_INVALID][PDCP_AUTH_TYPE_INVALID] = {
+			{	/* NULL */
+				PDCP_PDB_TYPE_NO_PDB,		/* NULL */
+				PDCP_PDB_TYPE_FULL_PDB,		/* SNOW f9 */
+				PDCP_PDB_TYPE_FULL_PDB,		/* AES CMAC */
+				PDCP_PDB_TYPE_FULL_PDB		/* ZUC-I */
+			},
+			{	/* SNOW f8 */
+				PDCP_PDB_TYPE_FULL_PDB,		/* NULL */
+				PDCP_PDB_TYPE_FULL_PDB,		/* SNOW f9 */
+				PDCP_PDB_TYPE_REDUCED_PDB,	/* AES CMAC */
+				PDCP_PDB_TYPE_REDUCED_PDB	/* ZUC-I */
+			},
+			{	/* AES CTR */
+				PDCP_PDB_TYPE_FULL_PDB,		/* NULL */
+				PDCP_PDB_TYPE_REDUCED_PDB,	/* SNOW f9 */
+				PDCP_PDB_TYPE_FULL_PDB,		/* AES CMAC */
+				PDCP_PDB_TYPE_REDUCED_PDB	/* ZUC-I */
+			},
+			{	/* ZUC-E */
+				PDCP_PDB_TYPE_FULL_PDB,		/* NULL */
+				PDCP_PDB_TYPE_REDUCED_PDB,	/* SNOW f9 */
+				PDCP_PDB_TYPE_REDUCED_PDB,	/* AES CMAC */
+				PDCP_PDB_TYPE_FULL_PDB		/* ZUC-I */
+			},
+	};
+
+	if (rta_sec_era >= RTA_SEC_ERA_8) {
+		memset(&pdb, 0x00, sizeof(struct pdcp_pdb));
+
+		/* This is a HW issue. Bit 2 should be set to zero,
+		 * but it does not work this way. Override here.
+		 */
+		pdb.opt_res.rsvd = 0x00000002;
+
+		/* Copy relevant information from user to PDB */
+		pdb.hfn_res = hfn << PDCP_C_PLANE_PDB_HFN_SHIFT;
+		pdb.bearer_dir_res = (uint32_t)
+				((bearer << PDCP_C_PLANE_PDB_BEARER_SHIFT) |
+				 (direction << PDCP_C_PLANE_PDB_DIR_SHIFT));
+		pdb.hfn_thr_res =
+				hfn_threshold << PDCP_C_PLANE_PDB_HFN_THR_SHIFT;
+
+		/* copy PDB in descriptor*/
+		__rta_out32(p, pdb.opt_res.opt);
+		__rta_out32(p, pdb.hfn_res);
+		__rta_out32(p, pdb.bearer_dir_res);
+		__rta_out32(p, pdb.hfn_thr_res);
+
+		return PDCP_PDB_TYPE_FULL_PDB;
+	}
+
+	switch (pdb_mask[cipherdata->algtype][authdata->algtype]) {
+	case PDCP_PDB_TYPE_NO_PDB:
+		break;
+
+	case PDCP_PDB_TYPE_REDUCED_PDB:
+		__rta_out32(p, (hfn << PDCP_C_PLANE_PDB_HFN_SHIFT));
+		__rta_out32(p,
+			    (uint32_t)((bearer <<
+					PDCP_C_PLANE_PDB_BEARER_SHIFT) |
+					(direction <<
+					 PDCP_C_PLANE_PDB_DIR_SHIFT)));
+		break;
+
+	case PDCP_PDB_TYPE_FULL_PDB:
+		memset(&pdb, 0x00, sizeof(struct pdcp_pdb));
+
+		/* This is a HW issue. Bit 2 should be set to zero,
+		 * but it does not work this way. Override here.
+		 */
+		pdb.opt_res.rsvd = 0x00000002;
+
+		/* Copy relevant information from user to PDB */
+		pdb.hfn_res = hfn << PDCP_C_PLANE_PDB_HFN_SHIFT;
+		pdb.bearer_dir_res = (uint32_t)
+			((bearer << PDCP_C_PLANE_PDB_BEARER_SHIFT) |
+			 (direction << PDCP_C_PLANE_PDB_DIR_SHIFT));
+		pdb.hfn_thr_res =
+			hfn_threshold << PDCP_C_PLANE_PDB_HFN_THR_SHIFT;
+
+		/* copy PDB in descriptor*/
+		__rta_out32(p, pdb.opt_res.opt);
+		__rta_out32(p, pdb.hfn_res);
+		__rta_out32(p, pdb.bearer_dir_res);
+		__rta_out32(p, pdb.hfn_thr_res);
+
+		break;
+
+	default:
+		return PDCP_PDB_TYPE_INVALID;
+	}
+
+	return pdb_mask[cipherdata->algtype][authdata->algtype];
+}
+
+/*
+ * PDCP UPlane PDB creation function
+ */
+static inline int
+cnstr_pdcp_u_plane_pdb(struct program *p,
+		       enum pdcp_sn_size sn_size,
+		       uint32_t hfn, unsigned short bearer,
+		       unsigned short direction,
+		       uint32_t hfn_threshold)
+{
+	struct pdcp_pdb pdb;
+	/* Read options from user */
+	/* Depending on sequence number length, the HFN and HFN threshold
+	 * have different lengths.
+	 */
+	memset(&pdb, 0x00, sizeof(struct pdcp_pdb));
+
+	switch (sn_size) {
+	case PDCP_SN_SIZE_7:
+		pdb.opt_res.opt |= PDCP_U_PLANE_PDB_OPT_SHORT_SN;
+		pdb.hfn_res = hfn << PDCP_U_PLANE_PDB_SHORT_SN_HFN_SHIFT;
+		pdb.hfn_thr_res =
+			hfn_threshold<<PDCP_U_PLANE_PDB_SHORT_SN_HFN_THR_SHIFT;
+		break;
+
+	case PDCP_SN_SIZE_12:
+		pdb.opt_res.opt &= (uint32_t)(~PDCP_U_PLANE_PDB_OPT_SHORT_SN);
+		pdb.hfn_res = hfn << PDCP_U_PLANE_PDB_LONG_SN_HFN_SHIFT;
+		pdb.hfn_thr_res =
+			hfn_threshold<<PDCP_U_PLANE_PDB_LONG_SN_HFN_THR_SHIFT;
+		break;
+
+	case PDCP_SN_SIZE_15:
+		pdb.opt_res.opt = (uint32_t)(PDCP_U_PLANE_PDB_OPT_15B_SN);
+		pdb.hfn_res = hfn << PDCP_U_PLANE_PDB_15BIT_SN_HFN_SHIFT;
+		pdb.hfn_thr_res =
+			hfn_threshold<<PDCP_U_PLANE_PDB_15BIT_SN_HFN_THR_SHIFT;
+		break;
+
+	default:
+		pr_err("Invalid Sequence Number Size setting in PDB\n");
+		return -EINVAL;
+	}
+
+	pdb.bearer_dir_res = (uint32_t)
+				((bearer << PDCP_U_PLANE_PDB_BEARER_SHIFT) |
+				 (direction << PDCP_U_PLANE_PDB_DIR_SHIFT));
+
+	/* copy PDB in descriptor*/
+	__rta_out32(p, pdb.opt_res.opt);
+	__rta_out32(p, pdb.hfn_res);
+	__rta_out32(p, pdb.bearer_dir_res);
+	__rta_out32(p, pdb.hfn_thr_res);
+
+	return 0;
+}
+/**
+ * cnstr_shdsc_pdcp_c_plane_encap - Function for creating a PDCP Control Plane
+ *                                  encapsulation descriptor.
+ * @descbuf: pointer to buffer for descriptor construction
+ * @ps: if 36/40bit addressing is desired, this parameter must be true
+ * @swap: must be true when core endianness doesn't match SEC endianness
+ * @hfn: starting Hyper Frame Number to be used together with the SN from the
+ *       PDCP frames.
+ * @bearer: radio bearer ID
+ * @direction: the direction of the PDCP frame (UL/DL)
+ * @hfn_threshold: HFN value that once reached triggers a warning from SEC that
+ *                 keys should be renegotiated at the earliest convenience.
+ * @cipherdata: pointer to block cipher transform definitions
+ *              Valid algorithm values are those from cipher_type_pdcp enum.
+ * @authdata: pointer to authentication transform definitions
+ *            Valid algorithm values are those from auth_type_pdcp enum.
+ * @era_2_sw_hfn_ovrd: if software HFN override mechanism is desired for
+ *                     this descriptor. Note: Can only be used for
+ *                     SEC ERA 2.
+ * Return: size of descriptor written in words or negative number on error.
+ *         Once the function returns, the value of this parameter can be used
+ *         for reclaiming the space that wasn't used for the descriptor.
+ *
+ * Note: descbuf must be large enough to contain a full 256 byte long
+ * descriptor; after the function returns, by subtracting the actual number of
+ * bytes used, the user can reuse the remaining buffer space for other purposes.
+ */
+static inline int
+cnstr_shdsc_pdcp_c_plane_encap(uint32_t *descbuf,
+			       bool ps,
+			       bool swap,
+			       uint32_t hfn,
+			       unsigned char bearer,
+			       unsigned char direction,
+			       uint32_t hfn_threshold,
+			       struct alginfo *cipherdata,
+			       struct alginfo *authdata,
+			       unsigned char era_2_sw_hfn_ovrd)
+{
+	static int
+		(*pdcp_cp_fp[PDCP_CIPHER_TYPE_INVALID][PDCP_AUTH_TYPE_INVALID])
+			(struct program*, bool swap, struct alginfo *,
+			 struct alginfo *, unsigned int,
+			unsigned char __maybe_unused) = {
+		{	/* NULL */
+			pdcp_insert_cplane_null_op,	/* NULL */
+			pdcp_insert_cplane_int_only_op,	/* SNOW f9 */
+			pdcp_insert_cplane_int_only_op,	/* AES CMAC */
+			pdcp_insert_cplane_int_only_op	/* ZUC-I */
+		},
+		{	/* SNOW f8 */
+			pdcp_insert_cplane_enc_only_op,	/* NULL */
+			pdcp_insert_cplane_acc_op,	/* SNOW f9 */
+			pdcp_insert_cplane_snow_aes_op,	/* AES CMAC */
+			pdcp_insert_cplane_snow_zuc_op	/* ZUC-I */
+		},
+		{	/* AES CTR */
+			pdcp_insert_cplane_enc_only_op,	/* NULL */
+			pdcp_insert_cplane_aes_snow_op,	/* SNOW f9 */
+			pdcp_insert_cplane_acc_op,	/* AES CMAC */
+			pdcp_insert_cplane_aes_zuc_op	/* ZUC-I */
+		},
+		{	/* ZUC-E */
+			pdcp_insert_cplane_enc_only_op,	/* NULL */
+			pdcp_insert_cplane_zuc_snow_op,	/* SNOW f9 */
+			pdcp_insert_cplane_zuc_aes_op,	/* AES CMAC */
+			pdcp_insert_cplane_acc_op	/* ZUC-I */
+		},
+	};
+	static enum rta_share_type
+		desc_share[PDCP_CIPHER_TYPE_INVALID][PDCP_AUTH_TYPE_INVALID] = {
+		{	/* NULL */
+			SHR_WAIT,	/* NULL */
+			SHR_ALWAYS,	/* SNOW f9 */
+			SHR_ALWAYS,	/* AES CMAC */
+			SHR_ALWAYS	/* ZUC-I */
+		},
+		{	/* SNOW f8 */
+			SHR_ALWAYS,	/* NULL */
+			SHR_ALWAYS,	/* SNOW f9 */
+			SHR_WAIT,	/* AES CMAC */
+			SHR_WAIT	/* ZUC-I */
+		},
+		{	/* AES CTR */
+			SHR_ALWAYS,	/* NULL */
+			SHR_ALWAYS,	/* SNOW f9 */
+			SHR_ALWAYS,	/* AES CMAC */
+			SHR_WAIT	/* ZUC-I */
+		},
+		{	/* ZUC-E */
+			SHR_ALWAYS,	/* NULL */
+			SHR_WAIT,	/* SNOW f9 */
+			SHR_WAIT,	/* AES CMAC */
+			SHR_ALWAYS	/* ZUC-I */
+		},
+	};
+	enum pdb_type_e pdb_type;
+	struct program prg;
+	struct program *p = &prg;
+	int err;
+	LABEL(pdb_end);
+
+	if (rta_sec_era != RTA_SEC_ERA_2 && era_2_sw_hfn_ovrd) {
+		pr_err("Cannot select SW HFN override for other era than 2");
+		return -EINVAL;
+	}
+
+	PROGRAM_CNTXT_INIT(p, descbuf, 0);
+	if (swap)
+		PROGRAM_SET_BSWAP(p);
+	if (ps)
+		PROGRAM_SET_36BIT_ADDR(p);
+
+	SHR_HDR(p, desc_share[cipherdata->algtype][authdata->algtype], 0, 0);
+
+	pdb_type = cnstr_pdcp_c_plane_pdb(p,
+			hfn,
+			bearer,
+			direction,
+			hfn_threshold,
+			cipherdata,
+			authdata);
+
+	SET_LABEL(p, pdb_end);
+
+	err = insert_hfn_ov_op(p, PDCP_SN_SIZE_5, pdb_type,
+			       era_2_sw_hfn_ovrd);
+	if (err)
+		return err;
+
+	err = pdcp_cp_fp[cipherdata->algtype][authdata->algtype](p,
+		swap,
+		cipherdata,
+		authdata,
+		OP_TYPE_ENCAP_PROTOCOL,
+		era_2_sw_hfn_ovrd);
+	if (err)
+		return err;
+
+	PATCH_HDR(p, 0, pdb_end);
+
+	return PROGRAM_FINALIZE(p);
+}
+
+/**
+ * cnstr_shdsc_pdcp_c_plane_decap - Function for creating a PDCP Control Plane
+ *                                  decapsulation descriptor.
+ * @descbuf: pointer to buffer for descriptor construction
+ * @ps: if 36/40bit addressing is desired, this parameter must be true
+ * @swap: must be true when core endianness doesn't match SEC endianness
+ * @hfn: starting Hyper Frame Number to be used together with the SN from the
+ *       PDCP frames.
+ * @bearer: radio bearer ID
+ * @direction: the direction of the PDCP frame (UL/DL)
+ * @hfn_threshold: HFN value that once reached triggers a warning from SEC that
+ *                 keys should be renegotiated at the earliest convenience.
+ * @cipherdata: pointer to block cipher transform definitions
+ *              Valid algorithm values are those from cipher_type_pdcp enum.
+ * @authdata: pointer to authentication transform definitions
+ *            Valid algorithm values are those from auth_type_pdcp enum.
+ * @era_2_sw_hfn_ovrd: if software HFN override mechanism is desired for
+ *                     this descriptor. Note: Can only be used for
+ *                     SEC ERA 2.
+ *
+ * Return: size of descriptor written in words or negative number on error.
+ *         Once the function returns, the value of this parameter can be used
+ *         for reclaiming the space that wasn't used for the descriptor.
+ *
+ * Note: descbuf must be large enough to contain a full 256 byte long
+ * descriptor; after the function returns, by subtracting the actual number of
+ * bytes used, the user can reuse the remaining buffer space for other purposes.
+ */
+static inline int
+cnstr_shdsc_pdcp_c_plane_decap(uint32_t *descbuf,
+			       bool ps,
+			       bool swap,
+			       uint32_t hfn,
+			       unsigned char bearer,
+			       unsigned char direction,
+			       uint32_t hfn_threshold,
+			       struct alginfo *cipherdata,
+			       struct alginfo *authdata,
+			       unsigned char era_2_sw_hfn_ovrd)
+{
+	static int
+		(*pdcp_cp_fp[PDCP_CIPHER_TYPE_INVALID][PDCP_AUTH_TYPE_INVALID])
+			(struct program*, bool swap, struct alginfo *,
+			 struct alginfo *, unsigned int, unsigned char) = {
+		{	/* NULL */
+			pdcp_insert_cplane_null_op,	/* NULL */
+			pdcp_insert_cplane_int_only_op,	/* SNOW f9 */
+			pdcp_insert_cplane_int_only_op,	/* AES CMAC */
+			pdcp_insert_cplane_int_only_op	/* ZUC-I */
+		},
+		{	/* SNOW f8 */
+			pdcp_insert_cplane_enc_only_op,	/* NULL */
+			pdcp_insert_cplane_acc_op,	/* SNOW f9 */
+			pdcp_insert_cplane_snow_aes_op,	/* AES CMAC */
+			pdcp_insert_cplane_snow_zuc_op	/* ZUC-I */
+		},
+		{	/* AES CTR */
+			pdcp_insert_cplane_enc_only_op,	/* NULL */
+			pdcp_insert_cplane_aes_snow_op,	/* SNOW f9 */
+			pdcp_insert_cplane_acc_op,	/* AES CMAC */
+			pdcp_insert_cplane_aes_zuc_op	/* ZUC-I */
+		},
+		{	/* ZUC-E */
+			pdcp_insert_cplane_enc_only_op,	/* NULL */
+			pdcp_insert_cplane_zuc_snow_op,	/* SNOW f9 */
+			pdcp_insert_cplane_zuc_aes_op,	/* AES CMAC */
+			pdcp_insert_cplane_acc_op	/* ZUC-I */
+		},
+	};
+	static enum rta_share_type
+		desc_share[PDCP_CIPHER_TYPE_INVALID][PDCP_AUTH_TYPE_INVALID] = {
+		{	/* NULL */
+			SHR_WAIT,	/* NULL */
+			SHR_ALWAYS,	/* SNOW f9 */
+			SHR_ALWAYS,	/* AES CMAC */
+			SHR_ALWAYS	/* ZUC-I */
+		},
+		{	/* SNOW f8 */
+			SHR_ALWAYS,	/* NULL */
+			SHR_ALWAYS,	/* SNOW f9 */
+			SHR_WAIT,	/* AES CMAC */
+			SHR_WAIT	/* ZUC-I */
+		},
+		{	/* AES CTR */
+			SHR_ALWAYS,	/* NULL */
+			SHR_ALWAYS,	/* SNOW f9 */
+			SHR_ALWAYS,	/* AES CMAC */
+			SHR_WAIT	/* ZUC-I */
+		},
+		{	/* ZUC-E */
+			SHR_ALWAYS,	/* NULL */
+			SHR_WAIT,	/* SNOW f9 */
+			SHR_WAIT,	/* AES CMAC */
+			SHR_ALWAYS	/* ZUC-I */
+		},
+	};
+	enum pdb_type_e pdb_type;
+	struct program prg;
+	struct program *p = &prg;
+	int err;
+	LABEL(pdb_end);
+
+	if (rta_sec_era != RTA_SEC_ERA_2 && era_2_sw_hfn_ovrd) {
+		pr_err("Cannot select SW HFN override for other era than 2");
+		return -EINVAL;
+	}
+
+	PROGRAM_CNTXT_INIT(p, descbuf, 0);
+	if (swap)
+		PROGRAM_SET_BSWAP(p);
+	if (ps)
+		PROGRAM_SET_36BIT_ADDR(p);
+
+	SHR_HDR(p, desc_share[cipherdata->algtype][authdata->algtype], 0, 0);
+
+	pdb_type = cnstr_pdcp_c_plane_pdb(p,
+			hfn,
+			bearer,
+			direction,
+			hfn_threshold,
+			cipherdata,
+			authdata);
+
+	SET_LABEL(p, pdb_end);
+
+	err = insert_hfn_ov_op(p, PDCP_SN_SIZE_5, pdb_type,
+			       era_2_sw_hfn_ovrd);
+	if (err)
+		return err;
+
+	err = pdcp_cp_fp[cipherdata->algtype][authdata->algtype](p,
+		swap,
+		cipherdata,
+		authdata,
+		OP_TYPE_DECAP_PROTOCOL,
+		era_2_sw_hfn_ovrd);
+	if (err)
+		return err;
+
+	PATCH_HDR(p, 0, pdb_end);
+
+	return PROGRAM_FINALIZE(p);
+}
+
+/**
+ * cnstr_shdsc_pdcp_u_plane_encap - Function for creating a PDCP User Plane
+ *                                  encapsulation descriptor.
+ * @descbuf: pointer to buffer for descriptor construction
+ * @ps: if 36/40bit addressing is desired, this parameter must be true
+ * @swap: must be true when core endianness doesn't match SEC endianness
+ * @sn_size: selects Sequence Number Size: 7/12/15 bits
+ * @hfn: starting Hyper Frame Number to be used together with the SN from the
+ *       PDCP frames.
+ * @bearer: radio bearer ID
+ * @direction: the direction of the PDCP frame (UL/DL)
+ * @hfn_threshold: HFN value that once reached triggers a warning from SEC that
+ *                 keys should be renegotiated at the earliest convenience.
+ * @cipherdata: pointer to block cipher transform definitions
+ *              Valid algorithm values are those from cipher_type_pdcp enum.
+ * @era_2_sw_hfn_ovrd: if software HFN override mechanism is desired for
+ *                     this descriptor. Note: Can only be used for
+ *                     SEC ERA 2.
+ *
+ * Return: size of descriptor written in words or negative number on error.
+ *         Once the function returns, the value of this parameter can be used
+ *         for reclaiming the space that wasn't used for the descriptor.
+ *
+ * Note: descbuf must be large enough to contain a full 256 byte long
+ * descriptor; after the function returns, by subtracting the actual number of
+ * bytes used, the user can reuse the remaining buffer space for other purposes.
+ */
+static inline int
+cnstr_shdsc_pdcp_u_plane_encap(uint32_t *descbuf,
+			       bool ps,
+			       bool swap,
+			       enum pdcp_sn_size sn_size,
+			       uint32_t hfn,
+			       unsigned short bearer,
+			       unsigned short direction,
+			       uint32_t hfn_threshold,
+			       struct alginfo *cipherdata,
+			       unsigned char era_2_sw_hfn_ovrd)
+{
+	struct program prg;
+	struct program *p = &prg;
+	int err;
+	LABEL(pdb_end);
+
+	if (rta_sec_era != RTA_SEC_ERA_2 && era_2_sw_hfn_ovrd) {
+		pr_err("Cannot select SW HFN ovrd for other era than 2");
+		return -EINVAL;
+	}
+
+	PROGRAM_CNTXT_INIT(p, descbuf, 0);
+	if (swap)
+		PROGRAM_SET_BSWAP(p);
+	if (ps)
+		PROGRAM_SET_36BIT_ADDR(p);
+
+	SHR_HDR(p, SHR_ALWAYS, 0, 0);
+	if (cnstr_pdcp_u_plane_pdb(p, sn_size, hfn, bearer, direction,
+				   hfn_threshold)) {
+		pr_err("Error creating PDCP UPlane PDB\n");
+		return -EINVAL;
+	}
+	SET_LABEL(p, pdb_end);
+
+	err = insert_hfn_ov_op(p, sn_size, PDCP_PDB_TYPE_FULL_PDB,
+			       era_2_sw_hfn_ovrd);
+	if (err)
+		return err;
+
+	switch (sn_size) {
+	case PDCP_SN_SIZE_7:
+	case PDCP_SN_SIZE_12:
+		switch (cipherdata->algtype) {
+		case PDCP_CIPHER_TYPE_ZUC:
+			if (rta_sec_era < RTA_SEC_ERA_5) {
+				pr_err("Invalid era for selected algorithm\n");
+				return -ENOTSUP;
+			}
+		case PDCP_CIPHER_TYPE_AES:
+		case PDCP_CIPHER_TYPE_SNOW:
+			/* Insert Cipher Key */
+			KEY(p, KEY1, cipherdata->key_enc_flags,
+			    (uint64_t)cipherdata->key, cipherdata->keylen,
+			    INLINE_KEY(cipherdata));
+			PROTOCOL(p, OP_TYPE_ENCAP_PROTOCOL,
+				 OP_PCLID_LTE_PDCP_USER,
+				 (uint16_t)cipherdata->algtype);
+			break;
+		case PDCP_CIPHER_TYPE_NULL:
+			insert_copy_frame_op(p,
+					     cipherdata,
+					     OP_TYPE_ENCAP_PROTOCOL);
+			break;
+		default:
+			pr_err("%s: Invalid encrypt algorithm selected: %d\n",
+			       "cnstr_pcl_shdsc_pdcp_u_plane_decap",
+			       cipherdata->algtype);
+			return -EINVAL;
+		}
+		break;
+
+	case PDCP_SN_SIZE_15:
+		switch (cipherdata->algtype) {
+		case PDCP_CIPHER_TYPE_NULL:
+			insert_copy_frame_op(p,
+					     cipherdata,
+					     OP_TYPE_ENCAP_PROTOCOL);
+			break;
+
+		default:
+			err = pdcp_insert_uplane_15bit_op(p, swap, cipherdata,
+				OP_TYPE_ENCAP_PROTOCOL);
+			if (err)
+				return err;
+			break;
+		}
+		break;
+
+	case PDCP_SN_SIZE_5:
+	default:
+		pr_err("Invalid SN size selected\n");
+		return -ENOTSUP;
+	}
+
+	PATCH_HDR(p, 0, pdb_end);
+	return PROGRAM_FINALIZE(p);
+}
+
+/**
+ * cnstr_shdsc_pdcp_u_plane_decap - Function for creating a PDCP User Plane
+ *                                  decapsulation descriptor.
+ * @descbuf: pointer to buffer for descriptor construction
+ * @ps: if 36/40bit addressing is desired, this parameter must be true
+ * @swap: must be true when core endianness doesn't match SEC endianness
+ * @sn_size: selects Sequence Number Size: 7/12/15 bits
+ * @hfn: starting Hyper Frame Number to be used together with the SN from the
+ *       PDCP frames.
+ * @bearer: radio bearer ID
+ * @direction: the direction of the PDCP frame (UL/DL)
+ * @hfn_threshold: HFN value that once reached triggers a warning from SEC that
+ *                 keys should be renegotiated at the earliest convenience.
+ * @cipherdata: pointer to block cipher transform definitions
+ *              Valid algorithm values are those from cipher_type_pdcp enum.
+ * @era_2_sw_hfn_ovrd: if software HFN override mechanism is desired for
+ *                     this descriptor. Note: Can only be used for
+ *                     SEC ERA 2.
+ *
+ * Return: size of descriptor written in words or negative number on error.
+ *         Once the function returns, the value of this parameter can be used
+ *         for reclaiming the space that wasn't used for the descriptor.
+ *
+ * Note: descbuf must be large enough to contain a full 256 byte long
+ * descriptor; after the function returns, by subtracting the actual number of
+ * bytes used, the user can reuse the remaining buffer space for other purposes.
+ */
+static inline int
+cnstr_shdsc_pdcp_u_plane_decap(uint32_t *descbuf,
+			       bool ps,
+			       bool swap,
+			       enum pdcp_sn_size sn_size,
+			       uint32_t hfn,
+			       unsigned short bearer,
+			       unsigned short direction,
+			       uint32_t hfn_threshold,
+			       struct alginfo *cipherdata,
+			       unsigned char era_2_sw_hfn_ovrd)
+{
+	struct program prg;
+	struct program *p = &prg;
+	int err;
+	LABEL(pdb_end);
+
+	if (rta_sec_era != RTA_SEC_ERA_2 && era_2_sw_hfn_ovrd) {
+		pr_err("Cannot select SW HFN override for other era than 2");
+		return -EINVAL;
+	}
+
+	PROGRAM_CNTXT_INIT(p, descbuf, 0);
+	if (swap)
+		PROGRAM_SET_BSWAP(p);
+	if (ps)
+		PROGRAM_SET_36BIT_ADDR(p);
+
+	SHR_HDR(p, SHR_ALWAYS, 0, 0);
+	if (cnstr_pdcp_u_plane_pdb(p, sn_size, hfn, bearer, direction,
+				   hfn_threshold)) {
+		pr_err("Error creating PDCP UPlane PDB\n");
+		return -EINVAL;
+	}
+	SET_LABEL(p, pdb_end);
+
+	err = insert_hfn_ov_op(p, sn_size, PDCP_PDB_TYPE_FULL_PDB,
+			       era_2_sw_hfn_ovrd);
+	if (err)
+		return err;
+
+	switch (sn_size) {
+	case PDCP_SN_SIZE_7:
+	case PDCP_SN_SIZE_12:
+		switch (cipherdata->algtype) {
+		case PDCP_CIPHER_TYPE_ZUC:
+			if (rta_sec_era < RTA_SEC_ERA_5) {
+				pr_err("Invalid era for selected algorithm\n");
+				return -ENOTSUP;
+			}
+		case PDCP_CIPHER_TYPE_AES:
+		case PDCP_CIPHER_TYPE_SNOW:
+			/* Insert Cipher Key */
+			KEY(p, KEY1, cipherdata->key_enc_flags,
+			    cipherdata->key, cipherdata->keylen,
+			    INLINE_KEY(cipherdata));
+			PROTOCOL(p, OP_TYPE_DECAP_PROTOCOL,
+				 OP_PCLID_LTE_PDCP_USER,
+				 (uint16_t)cipherdata->algtype);
+			break;
+		case PDCP_CIPHER_TYPE_NULL:
+			insert_copy_frame_op(p,
+					     cipherdata,
+					     OP_TYPE_DECAP_PROTOCOL);
+			break;
+		default:
+			pr_err("%s: Invalid encrypt algorithm selected: %d\n",
+			       "cnstr_pcl_shdsc_pdcp_u_plane_decap",
+			       cipherdata->algtype);
+			return -EINVAL;
+		}
+		break;
+
+	case PDCP_SN_SIZE_15:
+		switch (cipherdata->algtype) {
+		case PDCP_CIPHER_TYPE_NULL:
+			insert_copy_frame_op(p,
+					     cipherdata,
+					     OP_TYPE_DECAP_PROTOCOL);
+			break;
+
+		default:
+			err = pdcp_insert_uplane_15bit_op(p, swap, cipherdata,
+				OP_TYPE_DECAP_PROTOCOL);
+			if (err)
+				return err;
+			break;
+		}
+		break;
+
+	case PDCP_SN_SIZE_5:
+	default:
+		pr_err("Invalid SN size selected\n");
+		return -ENOTSUP;
+	}
+
+	PATCH_HDR(p, 0, pdb_end);
+	return PROGRAM_FINALIZE(p);
+}
+
+/**
+ * cnstr_shdsc_pdcp_short_mac - Function for creating a PDCP Short MAC
+ *                              descriptor.
+ * @descbuf: pointer to buffer for descriptor construction
+ * @ps: if 36/40bit addressing is desired, this parameter must be true
+ * @swap: must be true when core endianness doesn't match SEC endianness
+ * @authdata: pointer to authentication transform definitions
+ *            Valid algorithm values are those from auth_type_pdcp enum.
+ *
+ * Return: size of descriptor written in words or negative number on error.
+ *         Once the function returns, the value of this parameter can be used
+ *         for reclaiming the space that wasn't used for the descriptor.
+ *
+ * Note: descbuf must be large enough to contain a full 256 byte long
+ * descriptor; after the function returns, by subtracting the actual number of
+ * bytes used, the user can reuse the remaining buffer space for other purposes.
+ */
+static inline int
+cnstr_shdsc_pdcp_short_mac(uint32_t *descbuf,
+			   bool ps,
+			   bool swap,
+			   struct alginfo *authdata)
+{
+	struct program prg;
+	struct program *p = &prg;
+	uint32_t iv[3] = {0, 0, 0};
+	LABEL(local_offset);
+	REFERENCE(move_cmd_read_descbuf);
+	REFERENCE(move_cmd_write_descbuf);
+
+	PROGRAM_CNTXT_INIT(p, descbuf, 0);
+	if (swap)
+		PROGRAM_SET_BSWAP(p);
+	if (ps)
+		PROGRAM_SET_36BIT_ADDR(p);
+
+	SHR_HDR(p, SHR_ALWAYS, 1, 0);
+
+	if (rta_sec_era > RTA_SEC_ERA_2) {
+		MATHB(p, SEQINSZ, SUB, ZERO, VSEQINSZ, 4, 0);
+		MATHB(p, SEQINSZ, SUB, ZERO, MATH1, 4, 0);
+	} else {
+		MATHB(p, SEQINSZ, ADD, ONE, MATH1, 4, 0);
+		MATHB(p, MATH1, SUB, ONE, MATH1, 4, 0);
+		MATHB(p, ZERO, ADD, MATH1, VSEQINSZ, 4, 0);
+		MOVE(p, MATH1, 0, MATH0, 0, 8, IMMED);
+
+		/*
+		 * Since MOVELEN is available only starting with
+		 * SEC ERA 3, use poor man's MOVELEN: create a MOVE
+		 * command dynamically by writing the length from M1 by
+		 * OR-ing the command in the M1 register and MOVE the
+		 * result into the descriptor buffer. Care must be taken
+		 * wrt. the location of the command because of SEC
+		 * pipelining. The actual MOVEs are written at the end
+		 * of the descriptor due to calculations needed on the
+		 * offset in the descriptor for the MOVE command.
+		 */
+		move_cmd_read_descbuf = MOVE(p, DESCBUF, 0, MATH0, 0, 6,
+					     IMMED);
+		move_cmd_write_descbuf = MOVE(p, MATH0, 0, DESCBUF, 0, 8,
+					      WAITCOMP | IMMED);
+	}
+	MATHB(p, ZERO, ADD, MATH1, VSEQOUTSZ, 4, 0);
+
+	switch (authdata->algtype) {
+	case PDCP_AUTH_TYPE_NULL:
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+		if (rta_sec_era > RTA_SEC_ERA_2) {
+			MOVE(p, AB1, 0, OFIFO, 0, MATH1, 0);
+		} else {
+			SET_LABEL(p, local_offset);
+
+			/* Shut off automatic Info FIFO entries */
+			LOAD(p, 0, DCTRL, LDOFF_DISABLE_AUTO_NFIFO, 0, IMMED);
+
+			/* Placeholder for MOVE command with length from M1
+			 * register
+			 */
+			MOVE(p, IFIFOAB1, 0, OFIFO, 0, 0, IMMED);
+
+			/* Enable automatic Info FIFO entries */
+			LOAD(p, 0, DCTRL, LDOFF_ENABLE_AUTO_NFIFO, 0, IMMED);
+		}
+
+		LOAD(p, (uintptr_t)iv, MATH0, 0, 8, IMMED | COPY);
+		SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1 | LAST2 | FLUSH1);
+		SEQSTORE(p, MATH0, 0, 4, 0);
+
+		break;
+
+	case PDCP_AUTH_TYPE_SNOW:
+		iv[0] = 0xFFFFFFFF;
+		iv[1] = swap ? swab32(0x04000000) : 0x04000000;
+		iv[2] = swap ? swab32(0xF8000000) : 0xF8000000;
+
+		KEY(p, KEY2, authdata->key_enc_flags, authdata->key,
+		    authdata->keylen, INLINE_KEY(authdata));
+		LOAD(p, (uintptr_t)&iv, CONTEXT2, 0, 12, IMMED | COPY);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_SNOW_F9,
+			      OP_ALG_AAI_F9,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      DIR_ENC);
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+
+		if (rta_sec_era > RTA_SEC_ERA_2) {
+			MOVE(p, AB1, 0, OFIFO, 0, MATH1, 0);
+		} else {
+			SET_LABEL(p, local_offset);
+
+
+			/* Shut off automatic Info FIFO entries */
+			LOAD(p, 0, DCTRL, LDOFF_DISABLE_AUTO_NFIFO, 0, IMMED);
+
+			/* Placeholder for MOVE command with length from M1
+			 * register
+			 */
+			MOVE(p, IFIFOAB1, 0, OFIFO, 0, 0, IMMED);
+
+			/* Enable automatic Info FIFO entries */
+			LOAD(p, 0, DCTRL, LDOFF_ENABLE_AUTO_NFIFO, 0, IMMED);
+		}
+		SEQFIFOLOAD(p, MSGINSNOOP, 0, VLF | LAST1 | LAST2 | FLUSH1);
+		SEQSTORE(p, CONTEXT2, 0, 4, 0);
+
+		break;
+
+	case PDCP_AUTH_TYPE_AES:
+		iv[0] = 0xFFFFFFFF;
+		iv[1] = swap ? swab32(0xFC000000) : 0xFC000000;
+		iv[2] = 0x00000000; /* unused */
+
+		KEY(p, KEY1, authdata->key_enc_flags, authdata->key,
+		    authdata->keylen, INLINE_KEY(authdata));
+		LOAD(p, (uintptr_t)&iv, MATH0, 0, 8, IMMED | COPY);
+		MOVE(p, MATH0, 0, IFIFOAB1, 0, 8, IMMED);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_AES,
+			      OP_ALG_AAI_CMAC,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      DIR_ENC);
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+
+		if (rta_sec_era > RTA_SEC_ERA_2) {
+			MOVE(p, AB2, 0, OFIFO, 0, MATH1, 0);
+		} else {
+			SET_LABEL(p, local_offset);
+
+			/* Shut off automatic Info FIFO entries */
+			LOAD(p, 0, DCTRL, LDOFF_DISABLE_AUTO_NFIFO, 0, IMMED);
+
+			/* Placeholder for MOVE command with length from M1
+			 * register
+			 */
+			MOVE(p, IFIFOAB2, 0, OFIFO, 0, 0, IMMED);
+
+			/* Enable automatic Info FIFO entries */
+			LOAD(p, 0, DCTRL, LDOFF_ENABLE_AUTO_NFIFO, 0, IMMED);
+		}
+		SEQFIFOLOAD(p, MSGINSNOOP, 0, VLF | LAST1 | LAST2 | FLUSH1);
+		SEQSTORE(p, CONTEXT1, 0, 4, 0);
+
+		break;
+
+	case PDCP_AUTH_TYPE_ZUC:
+		if (rta_sec_era < RTA_SEC_ERA_5) {
+			pr_err("Invalid era for selected algorithm\n");
+			return -ENOTSUP;
+		}
+		iv[0] = 0xFFFFFFFF;
+		iv[1] = swap ? swab32(0xFC000000) : 0xFC000000;
+		iv[2] = 0x00000000; /* unused */
+
+		KEY(p, KEY2, authdata->key_enc_flags, authdata->key,
+		    authdata->keylen, INLINE_KEY(authdata));
+		LOAD(p, (uintptr_t)&iv, CONTEXT2, 0, 12, IMMED | COPY);
+		ALG_OPERATION(p, OP_ALG_ALGSEL_ZUCA,
+			      OP_ALG_AAI_F9,
+			      OP_ALG_AS_INITFINAL,
+			      ICV_CHECK_DISABLE,
+			      DIR_ENC);
+		SEQFIFOSTORE(p, MSG, 0, 0, VLF);
+		MOVE(p, AB1, 0, OFIFO, 0, MATH1, 0);
+		SEQFIFOLOAD(p, MSGINSNOOP, 0, VLF | LAST1 | LAST2 | FLUSH1);
+		SEQSTORE(p, CONTEXT2, 0, 4, 0);
+
+		break;
+
+	default:
+		pr_err("%s: Invalid integrity algorithm selected: %d\n",
+		       "cnstr_shdsc_pdcp_short_mac", authdata->algtype);
+		return -EINVAL;
+	}
+
+
+	if (rta_sec_era < RTA_SEC_ERA_3) {
+		PATCH_MOVE(p, move_cmd_read_descbuf, local_offset);
+		PATCH_MOVE(p, move_cmd_write_descbuf, local_offset);
+	}
+
+	return PROGRAM_FINALIZE(p);
+}
+
+#endif /* __DESC_PDCP_H__ */
diff --git a/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h
index d9a5b0e5..cf8dfb91 100644
--- a/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h
+++ b/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h
@@ -14,178 +14,176 @@ static inline int
 __rta_ssl_proto(uint16_t protoinfo)
 {
 	switch (protoinfo) {
-	case OP_PCL_SSL30_RC4_40_MD5_2:
-	case OP_PCL_SSL30_RC4_128_MD5_2:
-	case OP_PCL_SSL30_RC4_128_SHA_5:
-	case OP_PCL_SSL30_RC4_40_MD5_3:
-	case OP_PCL_SSL30_RC4_128_MD5_3:
-	case OP_PCL_SSL30_RC4_128_SHA:
-	case OP_PCL_SSL30_RC4_128_MD5:
-	case OP_PCL_SSL30_RC4_40_SHA:
-	case OP_PCL_SSL30_RC4_40_MD5:
-	case OP_PCL_SSL30_RC4_128_SHA_2:
-	case OP_PCL_SSL30_RC4_128_SHA_3:
-	case OP_PCL_SSL30_RC4_128_SHA_4:
-	case OP_PCL_SSL30_RC4_128_SHA_6:
-	case OP_PCL_SSL30_RC4_128_SHA_7:
-	case OP_PCL_SSL30_RC4_128_SHA_8:
-	case OP_PCL_SSL30_RC4_128_SHA_9:
-	case OP_PCL_SSL30_RC4_128_SHA_10:
-	case OP_PCL_TLS_ECDHE_PSK_RC4_128_SHA:
+	case OP_PCL_TLS_RSA_EXPORT_WITH_RC4_40_MD5:
+	case OP_PCL_TLS_RSA_WITH_RC4_128_MD5:
+	case OP_PCL_TLS_RSA_WITH_RC4_128_SHA:
+	case OP_PCL_TLS_DH_anon_EXPORT_WITH_RC4_40_MD5:
+	case OP_PCL_TLS_DH_anon_WITH_RC4_128_MD5:
+	case OP_PCL_TLS_KRB5_WITH_RC4_128_SHA:
+	case OP_PCL_TLS_KRB5_WITH_RC4_128_MD5:
+	case OP_PCL_TLS_KRB5_EXPORT_WITH_RC4_40_SHA:
+	case OP_PCL_TLS_KRB5_EXPORT_WITH_RC4_40_MD5:
+	case OP_PCL_TLS_PSK_WITH_RC4_128_SHA:
+	case OP_PCL_TLS_DHE_PSK_WITH_RC4_128_SHA:
+	case OP_PCL_TLS_RSA_PSK_WITH_RC4_128_SHA:
+	case OP_PCL_TLS_ECDH_ECDSA_WITH_RC4_128_SHA:
+	case OP_PCL_TLS_ECDHE_ECDSA_WITH_RC4_128_SHA:
+	case OP_PCL_TLS_ECDH_RSA_WITH_RC4_128_SHA:
+	case OP_PCL_TLS_ECDHE_RSA_WITH_RC4_128_SHA:
+	case OP_PCL_TLS_ECDH_anon_WITH_RC4_128_SHA:
+	case OP_PCL_TLS_ECDHE_PSK_WITH_RC4_128_SHA:
 		if (rta_sec_era == RTA_SEC_ERA_7)
 			return -EINVAL;
 		/* fall through if not Era 7 */
-	case OP_PCL_SSL30_DES40_CBC_SHA:
-	case OP_PCL_SSL30_DES_CBC_SHA_2:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_5:
-	case OP_PCL_SSL30_DES40_CBC_SHA_2:
-	case OP_PCL_SSL30_DES_CBC_SHA_3:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_6:
-	case OP_PCL_SSL30_DES40_CBC_SHA_3:
-	case OP_PCL_SSL30_DES_CBC_SHA_4:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_7:
-	case OP_PCL_SSL30_DES40_CBC_SHA_4:
-	case OP_PCL_SSL30_DES_CBC_SHA_5:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_8:
-	case OP_PCL_SSL30_DES40_CBC_SHA_5:
-	case OP_PCL_SSL30_DES_CBC_SHA_6:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_9:
-	case OP_PCL_SSL30_DES40_CBC_SHA_6:
-	case OP_PCL_SSL30_DES_CBC_SHA_7:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_10:
-	case OP_PCL_SSL30_DES_CBC_SHA:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA:
-	case OP_PCL_SSL30_DES_CBC_MD5:
-	case OP_PCL_SSL30_3DES_EDE_CBC_MD5:
-	case OP_PCL_SSL30_DES40_CBC_SHA_7:
-	case OP_PCL_SSL30_DES40_CBC_MD5:
-	case OP_PCL_SSL30_AES_128_CBC_SHA:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_2:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_3:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_4:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_5:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_6:
-	case OP_PCL_SSL30_AES_256_CBC_SHA:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_2:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_3:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_4:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_5:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_6:
-	case OP_PCL_TLS12_AES_128_CBC_SHA256_2:
-	case OP_PCL_TLS12_AES_128_CBC_SHA256_3:
-	case OP_PCL_TLS12_AES_128_CBC_SHA256_4:
-	case OP_PCL_TLS12_AES_128_CBC_SHA256_5:
-	case OP_PCL_TLS12_AES_256_CBC_SHA256_2:
-	case OP_PCL_TLS12_AES_256_CBC_SHA256_3:
-	case OP_PCL_TLS12_AES_256_CBC_SHA256_4:
-	case OP_PCL_TLS12_AES_256_CBC_SHA256_5:
-	case OP_PCL_TLS12_AES_128_CBC_SHA256_6:
-	case OP_PCL_TLS12_AES_256_CBC_SHA256_6:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_2:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_7:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_7:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_3:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_8:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_8:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_4:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_9:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_9:
-	case OP_PCL_SSL30_AES_128_GCM_SHA256_1:
-	case OP_PCL_SSL30_AES_256_GCM_SHA384_1:
-	case OP_PCL_SSL30_AES_128_GCM_SHA256_2:
-	case OP_PCL_SSL30_AES_256_GCM_SHA384_2:
-	case OP_PCL_SSL30_AES_128_GCM_SHA256_3:
-	case OP_PCL_SSL30_AES_256_GCM_SHA384_3:
-	case OP_PCL_SSL30_AES_128_GCM_SHA256_4:
-	case OP_PCL_SSL30_AES_256_GCM_SHA384_4:
-	case OP_PCL_SSL30_AES_128_GCM_SHA256_5:
-	case OP_PCL_SSL30_AES_256_GCM_SHA384_5:
-	case OP_PCL_SSL30_AES_128_GCM_SHA256_6:
-	case OP_PCL_TLS_DH_ANON_AES_256_GCM_SHA384:
-	case OP_PCL_TLS_PSK_AES_128_GCM_SHA256:
-	case OP_PCL_TLS_PSK_AES_256_GCM_SHA384:
-	case OP_PCL_TLS_DHE_PSK_AES_128_GCM_SHA256:
-	case OP_PCL_TLS_DHE_PSK_AES_256_GCM_SHA384:
-	case OP_PCL_TLS_RSA_PSK_AES_128_GCM_SHA256:
-	case OP_PCL_TLS_RSA_PSK_AES_256_GCM_SHA384:
-	case OP_PCL_TLS_PSK_AES_128_CBC_SHA256:
-	case OP_PCL_TLS_PSK_AES_256_CBC_SHA384:
-	case OP_PCL_TLS_DHE_PSK_AES_128_CBC_SHA256:
-	case OP_PCL_TLS_DHE_PSK_AES_256_CBC_SHA384:
-	case OP_PCL_TLS_RSA_PSK_AES_128_CBC_SHA256:
-	case OP_PCL_TLS_RSA_PSK_AES_256_CBC_SHA384:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_11:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_10:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_10:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_12:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_11:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_11:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_12:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_13:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_12:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_14:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_13:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_13:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_15:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_14:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_14:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_16:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_17:
-	case OP_PCL_SSL30_3DES_EDE_CBC_SHA_18:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_15:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_16:
-	case OP_PCL_SSL30_AES_128_CBC_SHA_17:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_15:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_16:
-	case OP_PCL_SSL30_AES_256_CBC_SHA_17:
-	case OP_PCL_TLS_ECDHE_ECDSA_AES_128_CBC_SHA256:
-	case OP_PCL_TLS_ECDHE_ECDSA_AES_256_CBC_SHA384:
-	case OP_PCL_TLS_ECDH_ECDSA_AES_128_CBC_SHA256:
-	case OP_PCL_TLS_ECDH_ECDSA_AES_256_CBC_SHA384:
-	case OP_PCL_TLS_ECDHE_RSA_AES_128_CBC_SHA256:
-	case OP_PCL_TLS_ECDHE_RSA_AES_256_CBC_SHA384:
-	case OP_PCL_TLS_ECDH_RSA_AES_128_CBC_SHA256:
-	case OP_PCL_TLS_ECDH_RSA_AES_256_CBC_SHA384:
-	case OP_PCL_TLS_ECDHE_ECDSA_AES_128_GCM_SHA256:
-	case OP_PCL_TLS_ECDHE_ECDSA_AES_256_GCM_SHA384:
-	case OP_PCL_TLS_ECDH_ECDSA_AES_128_GCM_SHA256:
-	case OP_PCL_TLS_ECDH_ECDSA_AES_256_GCM_SHA384:
-	case OP_PCL_TLS_ECDHE_RSA_AES_128_GCM_SHA256:
-	case OP_PCL_TLS_ECDHE_RSA_AES_256_GCM_SHA384:
-	case OP_PCL_TLS_ECDH_RSA_AES_128_GCM_SHA256:
-	case OP_PCL_TLS_ECDH_RSA_AES_256_GCM_SHA384:
-	case OP_PCL_TLS_ECDHE_PSK_3DES_EDE_CBC_SHA:
-	case OP_PCL_TLS_ECDHE_PSK_AES_128_CBC_SHA:
-	case OP_PCL_TLS_ECDHE_PSK_AES_256_CBC_SHA:
-	case OP_PCL_TLS_ECDHE_PSK_AES_128_CBC_SHA256:
-	case OP_PCL_TLS_ECDHE_PSK_AES_256_CBC_SHA384:
-	case OP_PCL_TLS12_3DES_EDE_CBC_MD5:
-	case OP_PCL_TLS12_3DES_EDE_CBC_SHA160:
-	case OP_PCL_TLS12_3DES_EDE_CBC_SHA224:
-	case OP_PCL_TLS12_3DES_EDE_CBC_SHA256:
-	case OP_PCL_TLS12_3DES_EDE_CBC_SHA384:
-	case OP_PCL_TLS12_3DES_EDE_CBC_SHA512:
-	case OP_PCL_TLS12_AES_128_CBC_SHA160:
-	case OP_PCL_TLS12_AES_128_CBC_SHA224:
-	case OP_PCL_TLS12_AES_128_CBC_SHA256:
-	case OP_PCL_TLS12_AES_128_CBC_SHA384:
-	case OP_PCL_TLS12_AES_128_CBC_SHA512:
-	case OP_PCL_TLS12_AES_192_CBC_SHA160:
-	case OP_PCL_TLS12_AES_192_CBC_SHA224:
-	case OP_PCL_TLS12_AES_192_CBC_SHA256:
-	case OP_PCL_TLS12_AES_192_CBC_SHA512:
-	case OP_PCL_TLS12_AES_256_CBC_SHA160:
-	case OP_PCL_TLS12_AES_256_CBC_SHA224:
-	case OP_PCL_TLS12_AES_256_CBC_SHA256:
-	case OP_PCL_TLS12_AES_256_CBC_SHA384:
-	case OP_PCL_TLS12_AES_256_CBC_SHA512:
-	case OP_PCL_TLS_PVT_AES_192_CBC_SHA160:
-	case OP_PCL_TLS_PVT_AES_192_CBC_SHA384:
-	case OP_PCL_TLS_PVT_AES_192_CBC_SHA224:
-	case OP_PCL_TLS_PVT_AES_192_CBC_SHA512:
-	case OP_PCL_TLS_PVT_AES_192_CBC_SHA256:
-	case OP_PCL_TLS_PVT_MASTER_SECRET_PRF_FE:
-	case OP_PCL_TLS_PVT_MASTER_SECRET_PRF_FF:
+	case OP_PCL_TLS_RSA_EXPORT_WITH_DES40_CBC_SHA:
+	case OP_PCL_TLS_RSA_WITH_DES_CBC_SHA:
+	case OP_PCL_TLS_RSA_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA:
+	case OP_PCL_TLS_DH_DSS_WITH_DES_CBC_SHA:
+	case OP_PCL_TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA:
+	case OP_PCL_TLS_DH_RSA_WITH_DES_CBC_SHA:
+	case OP_PCL_TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA:
+	case OP_PCL_TLS_DHE_DSS_WITH_DES_CBC_SHA:
+	case OP_PCL_TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA:
+	case OP_PCL_TLS_DHE_RSA_WITH_DES_CBC_SHA:
+	case OP_PCL_TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA:
+	case OP_PCL_TLS_DH_anon_WITH_DES_CBC_SHA:
+	case OP_PCL_TLS_DH_anon_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_KRB5_WITH_DES_CBC_SHA:
+	case OP_PCL_TLS_KRB5_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_KRB5_WITH_DES_CBC_MD5:
+	case OP_PCL_TLS_KRB5_WITH_3DES_EDE_CBC_MD5:
+	case OP_PCL_TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA:
+	case OP_PCL_TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5:
+	case OP_PCL_TLS_RSA_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_DH_DSS_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_DH_RSA_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_DHE_DSS_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_DHE_RSA_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_DH_anon_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_RSA_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_DH_DSS_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_DH_RSA_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_DHE_DSS_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_DHE_RSA_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_DH_anon_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_DH_DSS_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_DH_RSA_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_DHE_DSS_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_DHE_RSA_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_DH_DSS_WITH_AES_256_CBC_SHA256:
+	case OP_PCL_TLS_DH_RSA_WITH_AES_256_CBC_SHA256:
+	case OP_PCL_TLS_DHE_DSS_WITH_AES_256_CBC_SHA256:
+	case OP_PCL_TLS_DHE_RSA_WITH_AES_256_CBC_SHA256:
+	case OP_PCL_TLS_DH_anon_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_DH_anon_WITH_AES_256_CBC_SHA256:
+	case OP_PCL_TLS_PSK_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_PSK_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_PSK_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_DHE_PSK_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_DHE_PSK_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_RSA_PSK_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_RSA_PSK_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_RSA_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_RSA_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_DHE_RSA_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_DHE_RSA_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_DH_RSA_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_DH_RSA_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_DHE_DSS_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_DHE_DSS_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_DH_DSS_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_DH_DSS_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_DH_anon_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_DH_anon_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_PSK_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_PSK_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_DHE_PSK_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_DHE_PSK_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_RSA_PSK_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_PSK_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_PSK_WITH_AES_256_CBC_SHA384:
+	case OP_PCL_TLS_DHE_PSK_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_DHE_PSK_WITH_AES_256_CBC_SHA384:
+	case OP_PCL_TLS_RSA_PSK_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_RSA_PSK_WITH_AES_256_CBC_SHA384:
+	case OP_PCL_TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_ECDH_anon_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_ECDH_anon_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_SRP_SHA_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_SRP_SHA_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384:
+	case OP_PCL_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384:
+	case OP_PCL_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384:
+	case OP_PCL_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384:
+	case OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256:
+	case OP_PCL_TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384:
+	case OP_PCL_TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA:
+	case OP_PCL_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA:
+	case OP_PCL_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA:
+	case OP_PCL_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384:
+	case OP_PCL_TLS_RSA_WITH_AES_128_CBC_SHA256:
+	case OP_PCL_TLS_RSA_WITH_AES_256_CBC_SHA256:
+	case OP_PCL_PVT_TLS_3DES_EDE_CBC_MD5:
+	case OP_PCL_PVT_TLS_3DES_EDE_CBC_SHA160:
+	case OP_PCL_PVT_TLS_3DES_EDE_CBC_SHA224:
+	case OP_PCL_PVT_TLS_3DES_EDE_CBC_SHA256:
+	case OP_PCL_PVT_TLS_3DES_EDE_CBC_SHA384:
+	case OP_PCL_PVT_TLS_3DES_EDE_CBC_SHA512:
+	case OP_PCL_PVT_TLS_AES_128_CBC_SHA160:
+	case OP_PCL_PVT_TLS_AES_128_CBC_SHA224:
+	case OP_PCL_PVT_TLS_AES_128_CBC_SHA256:
+	case OP_PCL_PVT_TLS_AES_128_CBC_SHA384:
+	case OP_PCL_PVT_TLS_AES_128_CBC_SHA512:
+	case OP_PCL_PVT_TLS_AES_192_CBC_SHA160:
+	case OP_PCL_PVT_TLS_AES_192_CBC_SHA224:
+	case OP_PCL_PVT_TLS_AES_192_CBC_SHA256:
+	case OP_PCL_PVT_TLS_AES_192_CBC_SHA512:
+	case OP_PCL_PVT_TLS_AES_256_CBC_SHA160:
+	case OP_PCL_PVT_TLS_AES_256_CBC_SHA224:
+	case OP_PCL_PVT_TLS_AES_256_CBC_SHA384:
+	case OP_PCL_PVT_TLS_AES_256_CBC_SHA512:
+	case OP_PCL_PVT_TLS_AES_256_CBC_SHA256:
+	case OP_PCL_PVT_TLS_AES_192_CBC_SHA384:
+	case OP_PCL_PVT_TLS_MASTER_SECRET_PRF_FE:
+	case OP_PCL_PVT_TLS_MASTER_SECRET_PRF_FF:
 		return 0;
 	}
 
@@ -323,6 +321,12 @@ static const uint32_t proto_blob_flags[] = {
 	OP_PCL_BLOB_FORMAT_MASK | OP_PCL_BLOB_BLACK | OP_PCL_BLOB_TKEK |
 		OP_PCL_BLOB_EKT | OP_PCL_BLOB_REG_MASK | OP_PCL_BLOB_SEC_MEM,
 	OP_PCL_BLOB_FORMAT_MASK | OP_PCL_BLOB_BLACK | OP_PCL_BLOB_TKEK |
+		OP_PCL_BLOB_EKT | OP_PCL_BLOB_REG_MASK | OP_PCL_BLOB_SEC_MEM,
+	OP_PCL_BLOB_FORMAT_MASK | OP_PCL_BLOB_BLACK | OP_PCL_BLOB_TKEK |
+		OP_PCL_BLOB_EKT | OP_PCL_BLOB_REG_MASK | OP_PCL_BLOB_SEC_MEM,
+	OP_PCL_BLOB_FORMAT_MASK | OP_PCL_BLOB_BLACK | OP_PCL_BLOB_TKEK |
+		OP_PCL_BLOB_EKT | OP_PCL_BLOB_REG_MASK | OP_PCL_BLOB_SEC_MEM,
+	OP_PCL_BLOB_FORMAT_MASK | OP_PCL_BLOB_BLACK | OP_PCL_BLOB_TKEK |
 		OP_PCL_BLOB_EKT | OP_PCL_BLOB_REG_MASK | OP_PCL_BLOB_SEC_MEM
 };
 
@@ -556,7 +560,7 @@ static const struct proto_map proto_table[] = {
 	{OP_TYPE_UNI_PROTOCOL,   OP_PCLID_TLS10_PRF,	 __rta_ssl_proto},
 	{OP_TYPE_UNI_PROTOCOL,   OP_PCLID_TLS11_PRF,	 __rta_ssl_proto},
 	{OP_TYPE_UNI_PROTOCOL,   OP_PCLID_TLS12_PRF,	 __rta_ssl_proto},
-	{OP_TYPE_UNI_PROTOCOL,   OP_PCLID_DTLS10_PRF,	 __rta_ssl_proto},
+	{OP_TYPE_UNI_PROTOCOL,   OP_PCLID_DTLS_PRF,	 __rta_ssl_proto},
 	{OP_TYPE_UNI_PROTOCOL,   OP_PCLID_IKEV1_PRF,	 __rta_ike_proto},
 	{OP_TYPE_UNI_PROTOCOL,   OP_PCLID_IKEV2_PRF,	 __rta_ike_proto},
 	{OP_TYPE_UNI_PROTOCOL,   OP_PCLID_PUBLICKEYPAIR, __rta_dlc_proto},
@@ -568,7 +572,7 @@ static const struct proto_map proto_table[] = {
 	{OP_TYPE_DECAP_PROTOCOL, OP_PCLID_TLS10,	 __rta_ssl_proto},
 	{OP_TYPE_DECAP_PROTOCOL, OP_PCLID_TLS11,	 __rta_ssl_proto},
 	{OP_TYPE_DECAP_PROTOCOL, OP_PCLID_TLS12,	 __rta_ssl_proto},
-	{OP_TYPE_DECAP_PROTOCOL, OP_PCLID_DTLS10,	 __rta_ssl_proto},
+	{OP_TYPE_DECAP_PROTOCOL, OP_PCLID_DTLS,		 __rta_ssl_proto},
 	{OP_TYPE_DECAP_PROTOCOL, OP_PCLID_MACSEC,        __rta_macsec_proto},
 	{OP_TYPE_DECAP_PROTOCOL, OP_PCLID_WIFI,          __rta_wifi_proto},
 	{OP_TYPE_DECAP_PROTOCOL, OP_PCLID_WIMAX,         __rta_wimax_proto},
diff --git a/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h b/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h
index 6e666108..5357187f 100644
--- a/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h
+++ b/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h
@@ -497,6 +497,28 @@ __rta_out64(struct program *program, bool is_ext, uint64_t val)
 	}
 }
 
+static inline void __rta_out_be64(struct program *program, bool is_ext,
+				  uint64_t val)
+{
+	if (is_ext) {
+		__rta_out_be32(program, upper_32_bits(val));
+		__rta_out_be32(program, lower_32_bits(val));
+	} else {
+		__rta_out_be32(program, lower_32_bits(val));
+	}
+}
+
+static inline void __rta_out_le64(struct program *program, bool is_ext,
+				  uint64_t val)
+{
+	if (is_ext) {
+		__rta_out_le32(program, lower_32_bits(val));
+		__rta_out_le32(program, upper_32_bits(val));
+	} else {
+		__rta_out_le32(program, lower_32_bits(val));
+	}
+}
+
 static inline unsigned int
 rta_word(struct program *program, uint32_t val)
 {
diff --git a/drivers/crypto/dpaa2_sec/mc/dpseci.c b/drivers/crypto/dpaa2_sec/mc/dpseci.c
index de8ca970..87e0defd 100644
--- a/drivers/crypto/dpaa2_sec/mc/dpseci.c
+++ b/drivers/crypto/dpaa2_sec/mc/dpseci.c
@@ -6,6 +6,7 @@
  */
 #include <fsl_mc_sys.h>
 #include <fsl_mc_cmd.h>
+#include <fsl_dpopr.h>
 #include <fsl_dpseci.h>
 #include <fsl_dpseci_cmd.h>
 
@@ -116,11 +117,13 @@ int dpseci_create(struct fsl_mc_io *mc_io,
 					  cmd_flags,
 					  dprc_token);
 	cmd_params = (struct dpseci_cmd_create *)cmd.params;
-	for (i = 0; i < DPSECI_PRIO_NUM; i++)
+	for (i = 0; i < 8; i++)
 		cmd_params->priorities[i] = cfg->priorities[i];
+	for (i = 0; i < 8; i++)
+		cmd_params->priorities2[i] = cfg->priorities[8 + i];
 	cmd_params->num_tx_queues = cfg->num_tx_queues;
 	cmd_params->num_rx_queues = cfg->num_rx_queues;
-	cmd_params->options = cfg->options;
+	cmd_params->options = cpu_to_le32(cfg->options);
 
 	/* send command to mc*/
 	err = mc_send_command(mc_io, &cmd);
@@ -302,7 +305,7 @@ int dpseci_get_attributes(struct fsl_mc_io *mc_io,
 	/* retrieve response parameters */
 	rsp_params = (struct dpseci_rsp_get_attr *)cmd.params;
 	attr->id = le32_to_cpu(rsp_params->id);
-	attr->options = rsp_params->options;
+	attr->options = le32_to_cpu(rsp_params->options);
 	attr->num_tx_queues = rsp_params->num_tx_queues;
 	attr->num_rx_queues = rsp_params->num_rx_queues;
 
@@ -490,6 +493,8 @@ int dpseci_get_sec_attr(struct fsl_mc_io *mc_io,
 	attr->arc4_acc_num = rsp_params->arc4_acc_num;
 	attr->des_acc_num = rsp_params->des_acc_num;
 	attr->aes_acc_num = rsp_params->aes_acc_num;
+	attr->ccha_acc_num = rsp_params->ccha_acc_num;
+	attr->ptha_acc_num = rsp_params->ptha_acc_num;
 
 	return 0;
 }
@@ -569,6 +574,113 @@ int dpseci_get_api_version(struct fsl_mc_io *mc_io,
 	return 0;
 }
 
+/**
+ * dpseci_set_opr() - Set Order Restoration configuration.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPSECI object
+ * @index:	The queue index
+ * @options:	Configuration mode options
+ *			can be OPR_OPT_CREATE or OPR_OPT_RETIRE
+ * @cfg:	Configuration options for the OPR
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dpseci_set_opr(struct fsl_mc_io *mc_io,
+		   uint32_t cmd_flags,
+		   uint16_t token,
+		   uint8_t index,
+		   uint8_t options,
+		   struct opr_cfg *cfg)
+{
+	struct dpseci_cmd_set_opr *cmd_params;
+	struct mc_command cmd = { 0 };
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_OPR,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dpseci_cmd_set_opr *)cmd.params;
+	cmd_params->index = index;
+	cmd_params->options = options;
+	cmd_params->oloe = cfg->oloe;
+	cmd_params->oeane = cfg->oeane;
+	cmd_params->olws = cfg->olws;
+	cmd_params->oa = cfg->oa;
+	cmd_params->oprrws = cfg->oprrws;
+
+	/* send command to mc*/
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dpseci_get_opr() - Retrieve Order Restoration config and query.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPSECI object
+ * @index:	The queue index
+ * @cfg:	Returned OPR configuration
+ * @qry:	Returned OPR query
+ *
+ * Return:     '0' on Success; Error code otherwise.
+ */
+int dpseci_get_opr(struct fsl_mc_io *mc_io,
+		   uint32_t cmd_flags,
+		   uint16_t token,
+		   uint8_t index,
+		   struct opr_cfg *cfg,
+		   struct opr_qry *qry)
+{
+	struct dpseci_rsp_get_opr *rsp_params;
+	struct dpseci_cmd_get_opr *cmd_params;
+	struct mc_command cmd = { 0 };
+	int err;
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_OPR,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dpseci_cmd_get_opr *)cmd.params;
+	cmd_params->index = index;
+
+	/* send command to mc*/
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	/* retrieve response parameters */
+	rsp_params = (struct dpseci_rsp_get_opr *)cmd.params;
+	cfg->oloe = rsp_params->oloe;
+	cfg->oeane = rsp_params->oeane;
+	cfg->olws = rsp_params->olws;
+	cfg->oa = rsp_params->oa;
+	cfg->oprrws = rsp_params->oprrws;
+	qry->rip = dpseci_get_field(rsp_params->flags, RIP);
+	qry->enable = dpseci_get_field(rsp_params->flags, OPR_ENABLE);
+	qry->nesn = le16_to_cpu(rsp_params->nesn);
+	qry->ndsn = le16_to_cpu(rsp_params->ndsn);
+	qry->ea_tseq = le16_to_cpu(rsp_params->ea_tseq);
+	qry->tseq_nlis = dpseci_get_field(rsp_params->tseq_nlis, TSEQ_NLIS);
+	qry->ea_hseq = le16_to_cpu(rsp_params->ea_hseq);
+	qry->hseq_nlis = dpseci_get_field(rsp_params->hseq_nlis, HSEQ_NLIS);
+	qry->ea_hptr = le16_to_cpu(rsp_params->ea_hptr);
+	qry->ea_tptr = le16_to_cpu(rsp_params->ea_tptr);
+	qry->opr_vid = le16_to_cpu(rsp_params->opr_vid);
+	qry->opr_id = le16_to_cpu(rsp_params->opr_id);
+
+	return 0;
+}
+
+/**
+ * dpseci_set_congestion_notification() - Set congestion group
+ *	notification configuration
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPSECI object
+ * @cfg:	congestion notification configuration
+ *
+ * Return:	'0' on success, error code otherwise
+ */
 int dpseci_set_congestion_notification(
 			struct fsl_mc_io *mc_io,
 			uint32_t cmd_flags,
@@ -604,6 +716,16 @@ int dpseci_set_congestion_notification(
 	return mc_send_command(mc_io, &cmd);
 }
 
+/**
+ * dpseci_get_congestion_notification() - Get congestion group
+ *	notification configuration
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPSECI object
+ * @cfg:	congestion notification configuration
+ *
+ * Return:	'0' on success, error code otherwise
+ */
 int dpseci_get_congestion_notification(
 				struct fsl_mc_io *mc_io,
 				uint32_t cmd_flags,
diff --git a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h
index 12ac005a..279e8f4d 100644
--- a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h
+++ b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h
@@ -20,7 +20,7 @@ struct fsl_mc_io;
 /**
  * Maximum number of Tx/Rx priorities per DPSECI object
  */
-#define DPSECI_PRIO_NUM		8
+#define DPSECI_MAX_QUEUE_NUM		16
 
 /**
  * All queues considered; see dpseci_set_rx_queue()
@@ -58,7 +58,7 @@ struct dpseci_cfg {
 	uint32_t options;
 	uint8_t num_tx_queues;
 	uint8_t num_rx_queues;
-	uint8_t priorities[DPSECI_PRIO_NUM];
+	uint8_t priorities[DPSECI_MAX_QUEUE_NUM];
 };
 
 int dpseci_create(struct fsl_mc_io *mc_io,
@@ -259,6 +259,10 @@ int dpseci_get_tx_queue(struct fsl_mc_io *mc_io,
  *			implemented in this version of SEC.
  * @aes_acc_num:	The number of copies of the AES module that are
  *			implemented in this version of SEC.
+ * @ccha_acc_num:	The number of copies of the ChaCha20 module that are
+ *			implemented in this version of SEC.
+ * @ptha_acc_num:	The number of copies of the Poly1305 module that are
+ *			implemented in this version of SEC.
  **/
 
 struct dpseci_sec_attr {
@@ -279,6 +283,8 @@ struct dpseci_sec_attr {
 	uint8_t arc4_acc_num;
 	uint8_t des_acc_num;
 	uint8_t aes_acc_num;
+	uint8_t ccha_acc_num;
+	uint8_t ptha_acc_num;
 };
 
 int dpseci_get_sec_attr(struct fsl_mc_io *mc_io,
@@ -316,6 +322,21 @@ int dpseci_get_api_version(struct fsl_mc_io *mc_io,
 			   uint32_t cmd_flags,
 			   uint16_t *major_ver,
 			   uint16_t *minor_ver);
+
+int dpseci_set_opr(struct fsl_mc_io *mc_io,
+		   uint32_t cmd_flags,
+		   uint16_t token,
+		   uint8_t index,
+		   uint8_t options,
+		   struct opr_cfg *cfg);
+
+int dpseci_get_opr(struct fsl_mc_io *mc_io,
+		   uint32_t cmd_flags,
+		   uint16_t token,
+		   uint8_t index,
+		   struct opr_cfg *cfg,
+		   struct opr_qry *qry);
+
 /**
  * enum dpseci_congestion_unit - DPSECI congestion units
  * @DPSECI_CONGESTION_UNIT_BYTES: bytes units
diff --git a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h
index 26cef0f7..af3518a0 100644
--- a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h
+++ b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h
@@ -9,22 +9,25 @@
 
 /* DPSECI Version */
 #define DPSECI_VER_MAJOR		5
-#define DPSECI_VER_MINOR		1
+#define DPSECI_VER_MINOR		3
 
 /* Command versioning */
 #define DPSECI_CMD_BASE_VERSION		1
 #define DPSECI_CMD_BASE_VERSION_V2	2
+#define DPSECI_CMD_BASE_VERSION_V3	3
 #define DPSECI_CMD_ID_OFFSET		4
 
 #define DPSECI_CMD_V1(id) \
 	((id << DPSECI_CMD_ID_OFFSET) | DPSECI_CMD_BASE_VERSION)
 #define DPSECI_CMD_V2(id) \
 	((id << DPSECI_CMD_ID_OFFSET) | DPSECI_CMD_BASE_VERSION_V2)
+#define DPSECI_CMD_V3(id) \
+	((id << DPSECI_CMD_ID_OFFSET) | DPSECI_CMD_BASE_VERSION_V3)
 
 /* Command IDs */
 #define DPSECI_CMDID_CLOSE		DPSECI_CMD_V1(0x800)
 #define DPSECI_CMDID_OPEN		DPSECI_CMD_V1(0x809)
-#define DPSECI_CMDID_CREATE		DPSECI_CMD_V2(0x909)
+#define DPSECI_CMDID_CREATE		DPSECI_CMD_V3(0x909)
 #define DPSECI_CMDID_DESTROY		DPSECI_CMD_V1(0x989)
 #define DPSECI_CMDID_GET_API_VERSION	DPSECI_CMD_V1(0xa09)
 
@@ -37,9 +40,10 @@
 #define DPSECI_CMDID_SET_RX_QUEUE	DPSECI_CMD_V1(0x194)
 #define DPSECI_CMDID_GET_RX_QUEUE	DPSECI_CMD_V1(0x196)
 #define DPSECI_CMDID_GET_TX_QUEUE	DPSECI_CMD_V1(0x197)
-#define DPSECI_CMDID_GET_SEC_ATTR	DPSECI_CMD_V1(0x198)
+#define DPSECI_CMDID_GET_SEC_ATTR	DPSECI_CMD_V2(0x198)
 #define DPSECI_CMDID_GET_SEC_COUNTERS	DPSECI_CMD_V1(0x199)
-
+#define DPSECI_CMDID_SET_OPR		DPSECI_CMD_V1(0x19A)
+#define DPSECI_CMDID_GET_OPR		DPSECI_CMD_V1(0x19B)
 #define DPSECI_CMDID_SET_CONGESTION_NOTIFICATION	DPSECI_CMD_V1(0x170)
 #define DPSECI_CMDID_GET_CONGESTION_NOTIFICATION	DPSECI_CMD_V1(0x171)
 
@@ -63,6 +67,8 @@ struct dpseci_cmd_create {
 	uint8_t num_rx_queues;
 	uint8_t pad[6];
 	uint32_t options;
+	uint32_t pad2;
+	uint8_t priorities2[8];
 };
 
 struct dpseci_cmd_destroy {
@@ -152,6 +158,8 @@ struct dpseci_rsp_get_sec_attr {
 	uint8_t arc4_acc_num;
 	uint8_t des_acc_num;
 	uint8_t aes_acc_num;
+	uint8_t ccha_acc_num;
+	uint8_t ptha_acc_num;
 };
 
 struct dpseci_rsp_get_sec_counters {
@@ -169,6 +177,63 @@ struct dpseci_rsp_get_api_version {
 	uint16_t minor;
 };
 
+struct dpseci_cmd_set_opr {
+	uint16_t pad0;
+	uint8_t index;
+	uint8_t options;
+	uint8_t pad1[7];
+	uint8_t oloe;
+	uint8_t oeane;
+	uint8_t olws;
+	uint8_t oa;
+	uint8_t oprrws;
+};
+
+struct dpseci_cmd_get_opr {
+	uint16_t pad;
+	uint8_t index;
+};
+
+#define DPSECI_RIP_SHIFT	0
+#define DPSECI_RIP_SIZE		1
+#define DPSECI_OPR_ENABLE_SHIFT	1
+#define DPSECI_OPR_ENABLE_SIZE	1
+#define DPSECI_TSEQ_NLIS_SHIFT	0
+#define DPSECI_TSEQ_NLIS_SIZE	1
+#define DPSECI_HSEQ_NLIS_SHIFT	0
+#define DPSECI_HSEQ_NLIS_SIZE	1
+
+struct dpseci_rsp_get_opr {
+	uint64_t pad0;
+	/* from LSB: rip:1 enable:1 */
+	uint8_t flags;
+	uint16_t pad1;
+	uint8_t oloe;
+	uint8_t oeane;
+	uint8_t olws;
+	uint8_t oa;
+	uint8_t oprrws;
+	uint16_t nesn;
+	uint16_t pad8;
+	uint16_t ndsn;
+	uint16_t pad2;
+	uint16_t ea_tseq;
+	/* only the LSB */
+	uint8_t tseq_nlis;
+	uint8_t pad3;
+	uint16_t ea_hseq;
+	/* only the LSB */
+	uint8_t hseq_nlis;
+	uint8_t pad4;
+	uint16_t ea_hptr;
+	uint16_t pad5;
+	uint16_t ea_tptr;
+	uint16_t pad6;
+	uint16_t opr_vid;
+	uint16_t pad7;
+	uint16_t opr_id;
+};
+
 #define DPSECI_DEST_TYPE_SHIFT		0
 #define DPSECI_DEST_TYPE_SIZE		4
 #define DPSECI_CG_UNITS_SHIFT		4
diff --git a/drivers/crypto/dpaa2_sec/meson.build b/drivers/crypto/dpaa2_sec/meson.build
index 01afc587..8fa4827e 100644
--- a/drivers/crypto/dpaa2_sec/meson.build
+++ b/drivers/crypto/dpaa2_sec/meson.build
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright 2018 NXP
 
+version = 2
+
 if host_machine.system() != 'linux'
         build = false
 endif
diff --git a/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map b/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map
index 8591cc0b..0bfb986d 100644
--- a/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map
+++ b/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map
@@ -2,3 +2,11 @@ DPDK_17.05 {
 
 	local: *;
 };
+
+DPDK_18.11 {
+	global:
+
+	dpaa2_sec_eventq_attach;
+	dpaa2_sec_eventq_detach;
+
+} DPDK_17.05;
diff --git a/drivers/crypto/dpaa_sec/Makefile b/drivers/crypto/dpaa_sec/Makefile
index 9be44704..5ce95c23 100644
--- a/drivers/crypto/dpaa_sec/Makefile
+++ b/drivers/crypto/dpaa_sec/Makefile
@@ -11,7 +11,6 @@ LIB = librte_pmd_dpaa_sec.a
 
 # build flags
 CFLAGS += -DALLOW_EXPERIMENTAL_API
-CFLAGS += -D _GNU_SOURCE
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 
@@ -38,5 +37,6 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA_SEC) += dpaa_sec.c
 
 LDLIBS += -lrte_bus_dpaa
 LDLIBS += -lrte_mempool_dpaa
+LDLIBS += -lrte_common_dpaax
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.c b/drivers/crypto/dpaa_sec/dpaa_sec.c
index f571050b..d83e7454 100644
--- a/drivers/crypto/dpaa_sec/dpaa_sec.c
+++ b/drivers/crypto/dpaa_sec/dpaa_sec.c
@@ -23,6 +23,7 @@
 #include <rte_mbuf.h>
 #include <rte_memcpy.h>
 #include <rte_string_fns.h>
+#include <rte_spinlock.h>
 
 #include <fsl_usd.h>
 #include <fsl_qman.h>
@@ -106,6 +107,12 @@ dpaa_mem_vtop(void *vaddr)
 static inline void *
 dpaa_mem_ptov(rte_iova_t paddr)
 {
+	void *va;
+
+	va = (void *)dpaax_iova_table_get_va(paddr);
+	if (likely(va))
+		return va;
+
 	return rte_mem_iova2virt(paddr);
 }
 
@@ -274,6 +281,9 @@ caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a)
 {
 	switch (ses->auth_alg) {
 	case RTE_CRYPTO_AUTH_NULL:
+		alginfo_a->algtype =
+			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+			OP_PCL_IPSEC_HMAC_NULL : 0;
 		ses->digest_length = 0;
 		break;
 	case RTE_CRYPTO_AUTH_MD5_HMAC:
@@ -322,6 +332,9 @@ caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c)
 {
 	switch (ses->cipher_alg) {
 	case RTE_CRYPTO_CIPHER_NULL:
+		alginfo_c->algtype =
+			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+			OP_PCL_IPSEC_NULL : 0;
 		break;
 	case RTE_CRYPTO_CIPHER_AES_CBC:
 		alginfo_c->algtype =
@@ -359,6 +372,87 @@ caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo)
 	}
 }
 
+/* prepare ipsec proto command block of the session */
+static int
+dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses)
+{
+	struct alginfo cipherdata = {0}, authdata = {0};
+	struct sec_cdb *cdb = &ses->cdb;
+	int32_t shared_desc_len = 0;
+	int err;
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+	int swap = false;
+#else
+	int swap = true;
+#endif
+
+	caam_cipher_alg(ses, &cipherdata);
+	if (cipherdata.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
+		DPAA_SEC_ERR("not supported cipher alg");
+		return -ENOTSUP;
+	}
+
+	cipherdata.key = (size_t)ses->cipher_key.data;
+	cipherdata.keylen = ses->cipher_key.length;
+	cipherdata.key_enc_flags = 0;
+	cipherdata.key_type = RTA_DATA_IMM;
+
+	caam_auth_alg(ses, &authdata);
+	if (authdata.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
+		DPAA_SEC_ERR("not supported auth alg");
+		return -ENOTSUP;
+	}
+
+	authdata.key = (size_t)ses->auth_key.data;
+	authdata.keylen = ses->auth_key.length;
+	authdata.key_enc_flags = 0;
+	authdata.key_type = RTA_DATA_IMM;
+
+	cdb->sh_desc[0] = cipherdata.keylen;
+	cdb->sh_desc[1] = authdata.keylen;
+	err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
+			       MIN_JOB_DESC_SIZE,
+			       (unsigned int *)cdb->sh_desc,
+			       &cdb->sh_desc[2], 2);
+
+	if (err < 0) {
+		DPAA_SEC_ERR("Crypto: Incorrect key lengths");
+		return err;
+	}
+	if (cdb->sh_desc[2] & 1)
+		cipherdata.key_type = RTA_DATA_IMM;
+	else {
+		cipherdata.key = (size_t)dpaa_mem_vtop(
+					(void *)(size_t)cipherdata.key);
+		cipherdata.key_type = RTA_DATA_PTR;
+	}
+	if (cdb->sh_desc[2] & (1<<1))
+		authdata.key_type = RTA_DATA_IMM;
+	else {
+		authdata.key = (size_t)dpaa_mem_vtop(
+					(void *)(size_t)authdata.key);
+		authdata.key_type = RTA_DATA_PTR;
+	}
+
+	cdb->sh_desc[0] = 0;
+	cdb->sh_desc[1] = 0;
+	cdb->sh_desc[2] = 0;
+	if (ses->dir == DIR_ENC) {
+		shared_desc_len = cnstr_shdsc_ipsec_new_encap(
+				cdb->sh_desc,
+				true, swap, SHR_SERIAL,
+				&ses->encap_pdb,
+				(uint8_t *)&ses->ip4_hdr,
+				&cipherdata, &authdata);
+	} else if (ses->dir == DIR_DEC) {
+		shared_desc_len = cnstr_shdsc_ipsec_new_decap(
+				cdb->sh_desc,
+				true, swap, SHR_SERIAL,
+				&ses->decap_pdb,
+				&cipherdata, &authdata);
+	}
+	return shared_desc_len;
+}
 
 /* prepare command block of the session */
 static int
@@ -376,7 +470,9 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses)
 
 	memset(cdb, 0, sizeof(struct sec_cdb));
 
-	if (is_cipher_only(ses)) {
+	if (is_proto_ipsec(ses)) {
+		shared_desc_len = dpaa_sec_prep_ipsec_cdb(ses);
+	} else if (is_cipher_only(ses)) {
 		caam_cipher_alg(ses, &alginfo_c);
 		if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 			DPAA_SEC_ERR("not supported cipher alg");
@@ -484,28 +580,13 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses)
 		cdb->sh_desc[0] = 0;
 		cdb->sh_desc[1] = 0;
 		cdb->sh_desc[2] = 0;
-		if (is_proto_ipsec(ses)) {
-			if (ses->dir == DIR_ENC) {
-				shared_desc_len = cnstr_shdsc_ipsec_new_encap(
-						cdb->sh_desc,
-						true, swap, &ses->encap_pdb,
-						(uint8_t *)&ses->ip4_hdr,
-						&alginfo_c, &alginfo_a);
-			} else if (ses->dir == DIR_DEC) {
-				shared_desc_len = cnstr_shdsc_ipsec_new_decap(
-						cdb->sh_desc,
-						true, swap, &ses->decap_pdb,
-						&alginfo_c, &alginfo_a);
-			}
-		} else {
-			/* Auth_only_len is set as 0 here and it will be
-			 * overwritten in fd for each packet.
-			 */
-			shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
-					true, swap, &alginfo_c, &alginfo_a,
-					ses->iv.length, 0,
-					ses->digest_length, ses->dir);
-		}
+		/* Auth_only_len is set as 0 here and it will be
+		 * overwritten in fd for each packet.
+		 */
+		shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
+				true, swap, &alginfo_c, &alginfo_a,
+				ses->iv.length, 0,
+				ses->digest_length, ses->dir);
 	}
 
 	if (shared_desc_len < 0) {
@@ -1445,20 +1526,26 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
 				nb_ops = loop;
 				goto send_pkts;
 			}
-			if (unlikely(!ses->qp || ses->qp != qp)) {
-				DPAA_SEC_DP_ERR("sess->qp - %p qp %p",
-					     ses->qp, qp);
+			if (unlikely(!ses->qp)) {
 				if (dpaa_sec_attach_sess_q(qp, ses)) {
 					frames_to_send = loop;
 					nb_ops = loop;
 					goto send_pkts;
 				}
+			} else if (unlikely(ses->qp != qp)) {
+				DPAA_SEC_DP_ERR("Old:sess->qp = %p"
+					" New qp = %p\n", ses->qp, qp);
+				frames_to_send = loop;
+				nb_ops = loop;
+				goto send_pkts;
 			}
 
 			auth_only_len = op->sym->auth.data.length -
 						op->sym->cipher.data.length;
 			if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
-				if (is_auth_only(ses)) {
+				if (is_proto_ipsec(ses)) {
+					cf = build_proto(op, ses);
+				} else if (is_auth_only(ses)) {
 					cf = build_auth_only(op, ses);
 				} else if (is_cipher_only(ses)) {
 					cf = build_cipher_only(op, ses);
@@ -1467,8 +1554,6 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
 					auth_only_len = ses->auth_only_len;
 				} else if (is_auth_cipher(ses)) {
 					cf = build_cipher_auth(op, ses);
-				} else if (is_proto_ipsec(ses)) {
-					cf = build_proto(op, ses);
 				} else {
 					DPAA_SEC_DP_ERR("not supported ops");
 					frames_to_send = loop;
@@ -1760,6 +1845,7 @@ dpaa_sec_set_session_parameters(struct rte_cryptodev *dev,
 		DPAA_SEC_ERR("invalid session struct");
 		return -EINVAL;
 	}
+	memset(session, 0, sizeof(dpaa_sec_session));
 
 	/* Default IV length = 0 */
 	session->iv.length = 0;
@@ -1807,7 +1893,9 @@ dpaa_sec_set_session_parameters(struct rte_cryptodev *dev,
 		return -EINVAL;
 	}
 	session->ctx_pool = internals->ctx_pool;
+	rte_spinlock_lock(&internals->lock);
 	session->inq = dpaa_sec_attach_rxq(internals);
+	rte_spinlock_unlock(&internals->lock);
 	if (session->inq == NULL) {
 		DPAA_SEC_ERR("unable to attach sec queue");
 		goto err1;
@@ -1888,111 +1976,86 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
 {
 	struct dpaa_sec_dev_private *internals = dev->data->dev_private;
 	struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
-	struct rte_crypto_auth_xform *auth_xform;
-	struct rte_crypto_cipher_xform *cipher_xform;
+	struct rte_crypto_auth_xform *auth_xform = NULL;
+	struct rte_crypto_cipher_xform *cipher_xform = NULL;
 	dpaa_sec_session *session = (dpaa_sec_session *)sess;
 
 	PMD_INIT_FUNC_TRACE();
 
+	memset(session, 0, sizeof(dpaa_sec_session));
 	if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
 		cipher_xform = &conf->crypto_xform->cipher;
-		auth_xform = &conf->crypto_xform->next->auth;
+		if (conf->crypto_xform->next)
+			auth_xform = &conf->crypto_xform->next->auth;
 	} else {
 		auth_xform = &conf->crypto_xform->auth;
-		cipher_xform = &conf->crypto_xform->next->cipher;
+		if (conf->crypto_xform->next)
+			cipher_xform = &conf->crypto_xform->next->cipher;
 	}
 	session->proto_alg = conf->protocol;
-	session->cipher_key.data = rte_zmalloc(NULL,
-					       cipher_xform->key.length,
-					       RTE_CACHE_LINE_SIZE);
-	if (session->cipher_key.data == NULL &&
-			cipher_xform->key.length > 0) {
-		DPAA_SEC_ERR("No Memory for cipher key");
-		return -ENOMEM;
-	}
 
-	session->cipher_key.length = cipher_xform->key.length;
-	session->auth_key.data = rte_zmalloc(NULL,
-					auth_xform->key.length,
-					RTE_CACHE_LINE_SIZE);
-	if (session->auth_key.data == NULL &&
-			auth_xform->key.length > 0) {
-		DPAA_SEC_ERR("No Memory for auth key");
-		rte_free(session->cipher_key.data);
-		return -ENOMEM;
+	if (cipher_xform && cipher_xform->algo != RTE_CRYPTO_CIPHER_NULL) {
+		session->cipher_key.data = rte_zmalloc(NULL,
+						       cipher_xform->key.length,
+						       RTE_CACHE_LINE_SIZE);
+		if (session->cipher_key.data == NULL &&
+				cipher_xform->key.length > 0) {
+			DPAA_SEC_ERR("No Memory for cipher key");
+			return -ENOMEM;
+		}
+		memcpy(session->cipher_key.data, cipher_xform->key.data,
+				cipher_xform->key.length);
+		session->cipher_key.length = cipher_xform->key.length;
+
+		switch (cipher_xform->algo) {
+		case RTE_CRYPTO_CIPHER_AES_CBC:
+		case RTE_CRYPTO_CIPHER_3DES_CBC:
+		case RTE_CRYPTO_CIPHER_AES_CTR:
+			break;
+		default:
+			DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u",
+				cipher_xform->algo);
+			goto out;
+		}
+		session->cipher_alg = cipher_xform->algo;
+	} else {
+		session->cipher_key.data = NULL;
+		session->cipher_key.length = 0;
+		session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
 	}
-	session->auth_key.length = auth_xform->key.length;
-	memcpy(session->cipher_key.data, cipher_xform->key.data,
-			cipher_xform->key.length);
-	memcpy(session->auth_key.data, auth_xform->key.data,
-			auth_xform->key.length);
 
-	switch (auth_xform->algo) {
-	case RTE_CRYPTO_AUTH_SHA1_HMAC:
-		session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
-		break;
-	case RTE_CRYPTO_AUTH_MD5_HMAC:
-		session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
-		break;
-	case RTE_CRYPTO_AUTH_SHA256_HMAC:
-		session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
-		break;
-	case RTE_CRYPTO_AUTH_SHA384_HMAC:
-		session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
-		break;
-	case RTE_CRYPTO_AUTH_SHA512_HMAC:
-		session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
-		break;
-	case RTE_CRYPTO_AUTH_AES_CMAC:
-		session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
-		break;
-	case RTE_CRYPTO_AUTH_NULL:
+	if (auth_xform && auth_xform->algo != RTE_CRYPTO_AUTH_NULL) {
+		session->auth_key.data = rte_zmalloc(NULL,
+						auth_xform->key.length,
+						RTE_CACHE_LINE_SIZE);
+		if (session->auth_key.data == NULL &&
+				auth_xform->key.length > 0) {
+			DPAA_SEC_ERR("No Memory for auth key");
+			rte_free(session->cipher_key.data);
+			return -ENOMEM;
+		}
+		memcpy(session->auth_key.data, auth_xform->key.data,
+				auth_xform->key.length);
+		session->auth_key.length = auth_xform->key.length;
+
+		switch (auth_xform->algo) {
+		case RTE_CRYPTO_AUTH_SHA1_HMAC:
+		case RTE_CRYPTO_AUTH_MD5_HMAC:
+		case RTE_CRYPTO_AUTH_SHA256_HMAC:
+		case RTE_CRYPTO_AUTH_SHA384_HMAC:
+		case RTE_CRYPTO_AUTH_SHA512_HMAC:
+		case RTE_CRYPTO_AUTH_AES_CMAC:
+			break;
+		default:
+			DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
+				auth_xform->algo);
+			goto out;
+		}
+		session->auth_alg = auth_xform->algo;
+	} else {
+		session->auth_key.data = NULL;
+		session->auth_key.length = 0;
 		session->auth_alg = RTE_CRYPTO_AUTH_NULL;
-		break;
-	case RTE_CRYPTO_AUTH_SHA224_HMAC:
-	case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
-	case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
-	case RTE_CRYPTO_AUTH_SHA1:
-	case RTE_CRYPTO_AUTH_SHA256:
-	case RTE_CRYPTO_AUTH_SHA512:
-	case RTE_CRYPTO_AUTH_SHA224:
-	case RTE_CRYPTO_AUTH_SHA384:
-	case RTE_CRYPTO_AUTH_MD5:
-	case RTE_CRYPTO_AUTH_AES_GMAC:
-	case RTE_CRYPTO_AUTH_KASUMI_F9:
-	case RTE_CRYPTO_AUTH_AES_CBC_MAC:
-	case RTE_CRYPTO_AUTH_ZUC_EIA3:
-		DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
-			auth_xform->algo);
-		goto out;
-	default:
-		DPAA_SEC_ERR("Crypto: Undefined Auth specified %u",
-			auth_xform->algo);
-		goto out;
-	}
-
-	switch (cipher_xform->algo) {
-	case RTE_CRYPTO_CIPHER_AES_CBC:
-		session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
-		break;
-	case RTE_CRYPTO_CIPHER_3DES_CBC:
-		session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
-		break;
-	case RTE_CRYPTO_CIPHER_AES_CTR:
-		session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
-		break;
-	case RTE_CRYPTO_CIPHER_NULL:
-	case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
-	case RTE_CRYPTO_CIPHER_3DES_ECB:
-	case RTE_CRYPTO_CIPHER_AES_ECB:
-	case RTE_CRYPTO_CIPHER_KASUMI_F8:
-		DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u",
-			cipher_xform->algo);
-		goto out;
-	default:
-		DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
-			cipher_xform->algo);
-		goto out;
 	}
 
 	if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
@@ -2020,7 +2083,8 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
 			(IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
 			PDBOPTS_ESP_OIHI_PDB_INL |
 			PDBOPTS_ESP_IVSRC |
-			PDBHMO_ESP_ENCAP_DTTL;
+			PDBHMO_ESP_ENCAP_DTTL |
+			PDBHMO_ESP_SNR;
 		session->encap_pdb.spi = ipsec_xform->spi;
 		session->encap_pdb.ip_hdr_len = sizeof(struct ip);
 
@@ -2033,7 +2097,9 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
 	} else
 		goto out;
 	session->ctx_pool = internals->ctx_pool;
+	rte_spinlock_lock(&internals->lock);
 	session->inq = dpaa_sec_attach_rxq(internals);
+	rte_spinlock_unlock(&internals->lock);
 	if (session->inq == NULL) {
 		DPAA_SEC_ERR("unable to attach sec queue");
 		goto out;
@@ -2204,7 +2270,7 @@ dpaa_sec_capabilities_get(void *device __rte_unused)
 	return dpaa_sec_security_cap;
 }
 
-struct rte_security_ops dpaa_sec_security_ops = {
+static const struct rte_security_ops dpaa_sec_security_ops = {
 	.session_create = dpaa_sec_security_session_create,
 	.session_update = NULL,
 	.session_stats_get = NULL,
@@ -2284,6 +2350,7 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
 	security_instance->sess_cnt = 0;
 	cryptodev->security_ctx = security_instance;
 
+	rte_spinlock_init(&internals->lock);
 	for (i = 0; i < internals->max_nb_queue_pairs; i++) {
 		/* init qman fq for queue pair */
 		qp = &internals->qps[i];
@@ -2316,7 +2383,7 @@ init_error:
 }
 
 static int
-cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv,
+cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused,
 				struct rte_dpaa_device *dpaa_dev)
 {
 	struct rte_cryptodev *cryptodev;
@@ -2344,7 +2411,6 @@ cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv,
 
 	dpaa_dev->crypto_dev = cryptodev;
 	cryptodev->device = &dpaa_dev->device;
-	cryptodev->device->driver = &dpaa_drv->driver;
 
 	/* init user callbacks */
 	TAILQ_INIT(&(cryptodev->link_intr_cbs));
diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.h b/drivers/crypto/dpaa_sec/dpaa_sec.h
index ac6c00a6..f4b87844 100644
--- a/drivers/crypto/dpaa_sec/dpaa_sec.h
+++ b/drivers/crypto/dpaa_sec/dpaa_sec.h
@@ -137,7 +137,7 @@ struct dpaa_sec_qp {
 	int tx_errs;
 };
 
-#define RTE_DPAA_MAX_NB_SEC_QPS 8
+#define RTE_DPAA_MAX_NB_SEC_QPS 2
 #define RTE_DPAA_MAX_RX_QUEUE RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS
 #define DPAA_MAX_DEQUEUE_NUM_FRAMES 63
 
@@ -150,6 +150,7 @@ struct dpaa_sec_dev_private {
 	unsigned char inq_attach[RTE_DPAA_MAX_RX_QUEUE];
 	unsigned int max_nb_queue_pairs;
 	unsigned int max_nb_sessions;
+	rte_spinlock_t lock;
 };
 
 #define MAX_SG_ENTRIES		16
diff --git a/drivers/crypto/kasumi/meson.build b/drivers/crypto/kasumi/meson.build
new file mode 100644
index 00000000..a09b0e25
--- /dev/null
+++ b/drivers/crypto/kasumi/meson.build
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+lib = cc.find_library('libsso_kasumi', required: false)
+if not lib.found()
+	build = false
+else
+	ext_deps += lib
+endif
+
+sources = files('rte_kasumi_pmd.c', 'rte_kasumi_pmd_ops.c')
+deps += ['bus_vdev']
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index d64ca418..bf1bd928 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -1,8 +1,9 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['ccp', 'dpaa_sec', 'dpaa2_sec', 'mvsam',
-	'null', 'openssl', 'qat', 'virtio']
+drivers = ['aesni_gcm', 'aesni_mb', 'caam_jr', 'ccp', 'dpaa_sec', 'dpaa2_sec',
+	'kasumi', 'mvsam', 'null', 'octeontx', 'openssl', 'qat', 'scheduler',
+	'virtio', 'zuc']
 
 std_deps = ['cryptodev'] # cryptodev pulls in all other needed deps
 config_flag_fmt = 'RTE_LIBRTE_@0@_PMD'
diff --git a/drivers/crypto/mvsam/Makefile b/drivers/crypto/mvsam/Makefile
index c3dc72c1..2b4d036c 100644
--- a/drivers/crypto/mvsam/Makefile
+++ b/drivers/crypto/mvsam/Makefile
@@ -19,6 +19,7 @@ LIB = librte_pmd_mvsam_crypto.a
 # build flags
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -I$(RTE_SDK)/drivers/common/mvep
 CFLAGS += -I$(LIBMUSDK_PATH)/include
 CFLAGS += -DMVCONF_TYPES_PUBLIC
 CFLAGS += -DMVCONF_DMA_PHYS_ADDR_T_PUBLIC
@@ -31,9 +32,9 @@ EXPORT_MAP := rte_pmd_mvsam_version.map
 
 # external library dependencies
 LDLIBS += -L$(LIBMUSDK_PATH)/lib -lmusdk
-LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_kvargs
 LDLIBS += -lrte_cryptodev
-LDLIBS += -lrte_bus_vdev
+LDLIBS += -lrte_bus_vdev -lrte_common_mvep
 
 # library source files
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_MVSAM_CRYPTO) += rte_mrvl_pmd.c
diff --git a/drivers/crypto/mvsam/meson.build b/drivers/crypto/mvsam/meson.build
index 3c8ea3cf..f1c87966 100644
--- a/drivers/crypto/mvsam/meson.build
+++ b/drivers/crypto/mvsam/meson.build
@@ -18,4 +18,4 @@ endif
 
 sources = files('rte_mrvl_pmd.c', 'rte_mrvl_pmd_ops.c')
 
-deps += ['bus_vdev']
+deps += ['bus_vdev', 'common_mvep']
diff --git a/drivers/crypto/mvsam/rte_mrvl_pmd.c b/drivers/crypto/mvsam/rte_mrvl_pmd.c
index 73eff757..c2ae82a2 100644
--- a/drivers/crypto/mvsam/rte_mrvl_pmd.c
+++ b/drivers/crypto/mvsam/rte_mrvl_pmd.c
@@ -11,11 +11,11 @@
 #include <rte_bus_vdev.h>
 #include <rte_malloc.h>
 #include <rte_cpuflags.h>
+#include <rte_kvargs.h>
+#include <rte_mvep_common.h>
 
 #include "rte_mrvl_pmd_private.h"
 
-#define MRVL_MUSDK_DMA_MEMSIZE 41943040
-
 #define MRVL_PMD_MAX_NB_SESS_ARG		("max_nb_sessions")
 #define MRVL_PMD_DEFAULT_MAX_NB_SESSIONS	2048
 
@@ -68,6 +68,9 @@ __rte_aligned(32);
  */
 static const
 struct cipher_params_mapping cipher_map[RTE_CRYPTO_CIPHER_LIST_END] = {
+	[RTE_CRYPTO_CIPHER_NULL] = {
+		.supported = ALGO_SUPPORTED,
+		.cipher_alg = SAM_CIPHER_NONE },
 	[RTE_CRYPTO_CIPHER_3DES_CBC] = {
 		.supported = ALGO_SUPPORTED,
 		.cipher_alg = SAM_CIPHER_3DES,
@@ -93,6 +96,11 @@ struct cipher_params_mapping cipher_map[RTE_CRYPTO_CIPHER_LIST_END] = {
 		.cipher_alg = SAM_CIPHER_AES,
 		.cipher_mode = SAM_CIPHER_CTR,
 		.max_key_len = BITS2BYTES(256) },
+	[RTE_CRYPTO_CIPHER_AES_ECB] = {
+		.supported = ALGO_SUPPORTED,
+		.cipher_alg = SAM_CIPHER_AES,
+		.cipher_mode = SAM_CIPHER_ECB,
+		.max_key_len = BITS2BYTES(256) },
 };
 
 /**
@@ -100,6 +108,9 @@ struct cipher_params_mapping cipher_map[RTE_CRYPTO_CIPHER_LIST_END] = {
  */
 static const
 struct auth_params_mapping auth_map[RTE_CRYPTO_AUTH_LIST_END] = {
+	[RTE_CRYPTO_AUTH_NULL] = {
+		.supported = ALGO_SUPPORTED,
+		.auth_alg = SAM_AUTH_NONE },
 	[RTE_CRYPTO_AUTH_MD5_HMAC] = {
 		.supported = ALGO_SUPPORTED,
 		.auth_alg = SAM_AUTH_HMAC_MD5 },
@@ -112,6 +123,9 @@ struct auth_params_mapping auth_map[RTE_CRYPTO_AUTH_LIST_END] = {
 	[RTE_CRYPTO_AUTH_SHA1] = {
 		.supported = ALGO_SUPPORTED,
 		.auth_alg = SAM_AUTH_HASH_SHA1 },
+	[RTE_CRYPTO_AUTH_SHA224_HMAC] = {
+		.supported = ALGO_SUPPORTED,
+		.auth_alg = SAM_AUTH_HMAC_SHA2_224 },
 	[RTE_CRYPTO_AUTH_SHA224] = {
 		.supported = ALGO_SUPPORTED,
 		.auth_alg = SAM_AUTH_HASH_SHA2_224 },
@@ -210,7 +224,7 @@ mrvl_crypto_set_cipher_session_parameters(struct mrvl_crypto_session *sess,
 {
 	/* Make sure we've got proper struct */
 	if (cipher_xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
-		MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!");
+		MRVL_LOG(ERR, "Wrong xform struct provided!");
 		return -EINVAL;
 	}
 
@@ -218,7 +232,7 @@ mrvl_crypto_set_cipher_session_parameters(struct mrvl_crypto_session *sess,
 	if ((cipher_xform->cipher.algo > RTE_DIM(cipher_map)) ||
 		(cipher_map[cipher_xform->cipher.algo].supported
 			!= ALGO_SUPPORTED)) {
-		MRVL_CRYPTO_LOG_ERR("Cipher algorithm not supported!");
+		MRVL_LOG(ERR, "Cipher algorithm not supported!");
 		return -EINVAL;
 	}
 
@@ -238,7 +252,7 @@ mrvl_crypto_set_cipher_session_parameters(struct mrvl_crypto_session *sess,
 	/* Get max key length. */
 	if (cipher_xform->cipher.key.length >
 		cipher_map[cipher_xform->cipher.algo].max_key_len) {
-		MRVL_CRYPTO_LOG_ERR("Wrong key length!");
+		MRVL_LOG(ERR, "Wrong key length!");
 		return -EINVAL;
 	}
 
@@ -261,14 +275,14 @@ mrvl_crypto_set_auth_session_parameters(struct mrvl_crypto_session *sess,
 {
 	/* Make sure we've got proper struct */
 	if (auth_xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
-		MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!");
+		MRVL_LOG(ERR, "Wrong xform struct provided!");
 		return -EINVAL;
 	}
 
 	/* See if map data is present and valid */
 	if ((auth_xform->auth.algo > RTE_DIM(auth_map)) ||
 		(auth_map[auth_xform->auth.algo].supported != ALGO_SUPPORTED)) {
-		MRVL_CRYPTO_LOG_ERR("Auth algorithm not supported!");
+		MRVL_LOG(ERR, "Auth algorithm not supported!");
 		return -EINVAL;
 	}
 
@@ -300,7 +314,7 @@ mrvl_crypto_set_aead_session_parameters(struct mrvl_crypto_session *sess,
 {
 	/* Make sure we've got proper struct */
 	if (aead_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD) {
-		MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!");
+		MRVL_LOG(ERR, "Wrong xform struct provided!");
 		return -EINVAL;
 	}
 
@@ -308,7 +322,7 @@ mrvl_crypto_set_aead_session_parameters(struct mrvl_crypto_session *sess,
 	if ((aead_xform->aead.algo > RTE_DIM(aead_map)) ||
 		(aead_map[aead_xform->aead.algo].supported
 			!= ALGO_SUPPORTED)) {
-		MRVL_CRYPTO_LOG_ERR("AEAD algorithm not supported!");
+		MRVL_LOG(ERR, "AEAD algorithm not supported!");
 		return -EINVAL;
 	}
 
@@ -326,7 +340,7 @@ mrvl_crypto_set_aead_session_parameters(struct mrvl_crypto_session *sess,
 	/* Get max key length. */
 	if (aead_xform->aead.key.length >
 		aead_map[aead_xform->aead.algo].max_key_len) {
-		MRVL_CRYPTO_LOG_ERR("Wrong key length!");
+		MRVL_LOG(ERR, "Wrong key length!");
 		return -EINVAL;
 	}
 
@@ -391,21 +405,21 @@ mrvl_crypto_set_session_parameters(struct mrvl_crypto_session *sess,
 	if ((cipher_xform != NULL) &&
 		(mrvl_crypto_set_cipher_session_parameters(
 			sess, cipher_xform) < 0)) {
-		MRVL_CRYPTO_LOG_ERR("Invalid/unsupported cipher parameters");
+		MRVL_LOG(ERR, "Invalid/unsupported cipher parameters!");
 		return -EINVAL;
 	}
 
 	if ((auth_xform != NULL) &&
 		(mrvl_crypto_set_auth_session_parameters(
 			sess, auth_xform) < 0)) {
-		MRVL_CRYPTO_LOG_ERR("Invalid/unsupported auth parameters");
+		MRVL_LOG(ERR, "Invalid/unsupported auth parameters!");
 		return -EINVAL;
 	}
 
 	if ((aead_xform != NULL) &&
 		(mrvl_crypto_set_aead_session_parameters(
 			sess, aead_xform) < 0)) {
-		MRVL_CRYPTO_LOG_ERR("Invalid/unsupported aead parameters");
+		MRVL_LOG(ERR, "Invalid/unsupported aead parameters!");
 		return -EINVAL;
 	}
 
@@ -437,12 +451,14 @@ mrvl_request_prepare(struct sam_cio_op_params *request,
 		struct rte_crypto_op *op)
 {
 	struct mrvl_crypto_session *sess;
-	struct rte_mbuf *dst_mbuf;
+	struct rte_mbuf *src_mbuf, *dst_mbuf;
+	uint16_t segments_nb;
 	uint8_t *digest;
+	int i;
 
 	if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
-		MRVL_CRYPTO_LOG_ERR("MRVL CRYPTO PMD only supports session "
-				"oriented requests, op (%p) is sessionless.",
+		MRVL_LOG(ERR, "MRVL CRYPTO PMD only supports session "
+				"oriented requests, op (%p) is sessionless!",
 				op);
 		return -EINVAL;
 	}
@@ -450,39 +466,56 @@ mrvl_request_prepare(struct sam_cio_op_params *request,
 	sess = (struct mrvl_crypto_session *)get_sym_session_private_data(
 			op->sym->session, cryptodev_driver_id);
 	if (unlikely(sess == NULL)) {
-		MRVL_CRYPTO_LOG_ERR("Session was not created for this device");
+		MRVL_LOG(ERR, "Session was not created for this device!");
 		return -EINVAL;
 	}
 
-	/*
+	request->sa = sess->sam_sess;
+	request->cookie = op;
+
+	src_mbuf = op->sym->m_src;
+	segments_nb = src_mbuf->nb_segs;
+	/* The following conditions must be met:
+	 * - Destination buffer is required when segmented source buffer
+	 * - Segmented destination buffer is not supported
+	 */
+	if ((segments_nb > 1) && (!op->sym->m_dst)) {
+		MRVL_LOG(ERR, "op->sym->m_dst = NULL!");
+		return -1;
+	}
+	/* For non SG case:
 	 * If application delivered us null dst buffer, it means it expects
 	 * us to deliver the result in src buffer.
 	 */
 	dst_mbuf = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
 
-	request->sa = sess->sam_sess;
-	request->cookie = op;
-
-	/* Single buffers only, sorry. */
-	request->num_bufs = 1;
-	request->src = src_bd;
-	src_bd->vaddr = rte_pktmbuf_mtod(op->sym->m_src, void *);
-	src_bd->paddr = rte_pktmbuf_iova(op->sym->m_src);
-	src_bd->len = rte_pktmbuf_data_len(op->sym->m_src);
-
-	/* Empty source. */
-	if (rte_pktmbuf_data_len(op->sym->m_src) == 0) {
-		/* EIP does not support 0 length buffers. */
-		MRVL_CRYPTO_LOG_ERR("Buffer length == 0 not supported!");
+	if (!rte_pktmbuf_is_contiguous(dst_mbuf)) {
+		MRVL_LOG(ERR, "Segmented destination buffer not supported!");
 		return -1;
 	}
 
+	request->num_bufs = segments_nb;
+	for (i = 0; i < segments_nb; i++) {
+		/* Empty source. */
+		if (rte_pktmbuf_data_len(src_mbuf) == 0) {
+			/* EIP does not support 0 length buffers. */
+			MRVL_LOG(ERR, "Buffer length == 0 not supported!");
+			return -1;
+		}
+		src_bd[i].vaddr = rte_pktmbuf_mtod(src_mbuf, void *);
+		src_bd[i].paddr = rte_pktmbuf_iova(src_mbuf);
+		src_bd[i].len = rte_pktmbuf_data_len(src_mbuf);
+
+		src_mbuf = src_mbuf->next;
+	}
+	request->src = src_bd;
+
 	/* Empty destination. */
 	if (rte_pktmbuf_data_len(dst_mbuf) == 0) {
 		/* Make dst buffer fit at least source data. */
 		if (rte_pktmbuf_append(dst_mbuf,
 			rte_pktmbuf_data_len(op->sym->m_src)) == NULL) {
-			MRVL_CRYPTO_LOG_ERR("Unable to set big enough dst buffer!");
+			MRVL_LOG(ERR, "Unable to set big enough dst buffer!");
 			return -1;
 		}
 	}
@@ -527,7 +560,7 @@ mrvl_request_prepare(struct sam_cio_op_params *request,
 
 	/*
 	 * EIP supports only scenarios where ICV(digest buffer) is placed at
-	 * auth_icv_offset. Any other placement means risking errors.
+	 * auth_icv_offset.
 	 */
 	if (sess->sam_sess_params.dir == SAM_DIR_ENCRYPT) {
 		/*
@@ -536,17 +569,36 @@ mrvl_request_prepare(struct sam_cio_op_params *request,
 		 */
 		if (rte_pktmbuf_mtod_offset(
 				dst_mbuf, uint8_t *,
-				request->auth_icv_offset) == digest) {
+				request->auth_icv_offset) == digest)
 			return 0;
-		}
 	} else {/* sess->sam_sess_params.dir == SAM_DIR_DECRYPT */
 		/*
 		 * EIP will look for digest at auth_icv_offset
-		 * offset in SRC buffer.
+		 * offset in SRC buffer. It must be placed in the last
+		 * segment and the offset must be set to reach digest
+		 * in the last segment
 		 */
-		if (rte_pktmbuf_mtod_offset(
-				op->sym->m_src, uint8_t *,
-				request->auth_icv_offset) == digest) {
+		struct rte_mbuf *last_seg =  op->sym->m_src;
+		uint32_t d_offset = request->auth_icv_offset;
+		u32 d_size = sess->sam_sess_params.u.basic.auth_icv_len;
+		unsigned char *d_ptr;
+
+		/* Find the last segment and the offset for the last segment */
+		while ((last_seg->next != NULL) &&
+				(d_offset >= last_seg->data_len)) {
+			d_offset -= last_seg->data_len;
+			last_seg = last_seg->next;
+		}
+
+		if (rte_pktmbuf_mtod_offset(last_seg, uint8_t *,
+					    d_offset) == digest)
+			return 0;
+
+		/* copy digest to last segment */
+		if (last_seg->buf_len >= (d_size + d_offset)) {
+			d_ptr = (unsigned char *)last_seg->buf_addr +
+				 d_offset;
+			rte_memcpy(d_ptr, digest, d_size);
 			return 0;
 		}
 	}
@@ -582,11 +634,10 @@ mrvl_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	int ret;
 	struct sam_cio_op_params requests[nb_ops];
 	/*
-	 * DPDK uses single fragment buffers, so we can KISS descriptors.
 	 * SAM does not store bd pointers, so on-stack scope will be enough.
 	 */
-	struct sam_buf_info src_bd[nb_ops];
-	struct sam_buf_info dst_bd[nb_ops];
+	struct mrvl_crypto_src_table src_bd[nb_ops];
+	struct sam_buf_info          dst_bd[nb_ops];
 	struct mrvl_crypto_qp *qp = (struct mrvl_crypto_qp *)queue_pair;
 
 	if (nb_ops == 0)
@@ -594,15 +645,17 @@ mrvl_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 
 	/* Prepare the burst. */
 	memset(&requests, 0, sizeof(requests));
+	memset(&src_bd, 0, sizeof(src_bd));
 
 	/* Iterate through */
 	for (; iter_ops < nb_ops; ++iter_ops) {
+		/* store the op id for debug */
+		src_bd[iter_ops].iter_ops = iter_ops;
 		if (mrvl_request_prepare(&requests[iter_ops],
-					&src_bd[iter_ops],
+					src_bd[iter_ops].src_bd,
 					&dst_bd[iter_ops],
 					ops[iter_ops]) < 0) {
-			MRVL_CRYPTO_LOG_ERR(
-				"Error while parameters preparation!");
+			MRVL_LOG(ERR, "Error while preparing parameters!");
 			qp->stats.enqueue_err_count++;
 			ops[iter_ops]->status = RTE_CRYPTO_OP_STATUS_ERROR;
 
@@ -680,12 +733,12 @@ mrvl_crypto_pmd_dequeue_burst(void *queue_pair,
 			ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 			break;
 		case SAM_CIO_ERR_ICV:
-			MRVL_CRYPTO_LOG_DBG("CIO returned SAM_CIO_ERR_ICV.");
+			MRVL_LOG(DEBUG, "CIO returned SAM_CIO_ERR_ICV.");
 			ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 			break;
 		default:
-			MRVL_CRYPTO_LOG_DBG(
-				"CIO returned Error: %d", results[i].status);
+			MRVL_LOG(DEBUG,
+				"CIO returned Error: %d.", results[i].status);
 			ops[i]->status = RTE_CRYPTO_OP_STATUS_ERROR;
 			break;
 		}
@@ -711,12 +764,12 @@ cryptodev_mrvl_crypto_create(const char *name,
 	struct rte_cryptodev *dev;
 	struct mrvl_crypto_private *internals;
 	struct sam_init_params	sam_params;
-	int ret;
+	int ret = -EINVAL;
 
 	dev = rte_cryptodev_pmd_create(name, &vdev->device,
 			&init_params->common);
 	if (dev == NULL) {
-		MRVL_CRYPTO_LOG_ERR("failed to create cryptodev vdev");
+		MRVL_LOG(ERR, "Failed to create cryptodev vdev!");
 		goto init_error;
 	}
 
@@ -729,7 +782,9 @@ cryptodev_mrvl_crypto_create(const char *name,
 
 	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
-			RTE_CRYPTODEV_FF_HW_ACCELERATED;
+			RTE_CRYPTODEV_FF_HW_ACCELERATED |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
 
 	/* Set vector instructions mode supported */
 	internals = dev->data->dev_private;
@@ -737,29 +792,26 @@ cryptodev_mrvl_crypto_create(const char *name,
 	internals->max_nb_qpairs = init_params->common.max_nb_queue_pairs;
 	internals->max_nb_sessions = init_params->max_nb_sessions;
 
-	/*
-	 * ret == -EEXIST is correct, it means DMA
-	 * has been already initialized.
-	 */
-	ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE);
-	if (ret < 0) {
-		if (ret != -EEXIST)
-			return ret;
-
-		MRVL_CRYPTO_LOG_INFO(
-			"DMA memory has been already initialized by a different driver.");
-	}
+	ret = rte_mvep_init(MVEP_MOD_T_SAM, NULL);
+	if (ret)
+		goto init_error;
 
 	sam_params.max_num_sessions = internals->max_nb_sessions;
 
-	return sam_init(&sam_params);
+	/* sam_set_debug_flags(3); */
+
+	ret = sam_init(&sam_params);
+	if (ret)
+		goto init_error;
+
+	return 0;
 
 init_error:
-	MRVL_CRYPTO_LOG_ERR(
-		"driver %s: %s failed", init_params->common.name, __func__);
+	MRVL_LOG(ERR,
+		"Driver %s: %s failed!", init_params->common.name, __func__);
 
 	cryptodev_mrvl_crypto_uninit(vdev);
-	return -EFAULT;
+	return ret;
 }
 
 /** Parse integer from integer argument */
@@ -771,7 +823,7 @@ parse_integer_arg(const char *key __rte_unused,
 
 	*i = atoi(value);
 	if (*i < 0) {
-		MRVL_CRYPTO_LOG_ERR("Argument has to be positive.\n");
+		MRVL_LOG(ERR, "Argument has to be positive!");
 		return -EINVAL;
 	}
 
@@ -786,9 +838,8 @@ parse_name_arg(const char *key __rte_unused,
 	struct rte_cryptodev_pmd_init_params *params = extra_args;
 
 	if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
-		MRVL_CRYPTO_LOG_ERR("Invalid name %s, should be less than "
-				"%u bytes.\n", value,
-				RTE_CRYPTODEV_NAME_MAX_LEN - 1);
+		MRVL_LOG(ERR, "Invalid name %s, should be less than %u bytes!",
+			 value, RTE_CRYPTODEV_NAME_MAX_LEN - 1);
 		return -EINVAL;
 	}
 
@@ -864,7 +915,7 @@ cryptodev_mrvl_crypto_init(struct rte_vdev_device *vdev)
 			.private_data_size =
 				sizeof(struct mrvl_crypto_private),
 			.max_nb_queue_pairs =
-				sam_get_num_inst() * SAM_HW_RING_NUM,
+				sam_get_num_inst() * sam_get_num_cios(0),
 			.socket_id = rte_socket_id()
 		},
 		.max_nb_sessions = MRVL_PMD_DEFAULT_MAX_NB_SESSIONS
@@ -880,9 +931,8 @@ cryptodev_mrvl_crypto_init(struct rte_vdev_device *vdev)
 
 	ret = mrvl_pmd_parse_input_args(&init_params, args);
 	if (ret) {
-		RTE_LOG(ERR, PMD,
-			"Failed to parse initialisation arguments[%s]\n",
-			args);
+		MRVL_LOG(ERR, "Failed to parse initialisation arguments[%s]!",
+			 args);
 		return -EINVAL;
 	}
 
@@ -904,11 +954,11 @@ cryptodev_mrvl_crypto_uninit(struct rte_vdev_device *vdev)
 	if (name == NULL)
 		return -EINVAL;
 
-	RTE_LOG(INFO, PMD,
-		"Closing Marvell crypto device %s on numa socket %u\n",
-		name, rte_socket_id());
+	MRVL_LOG(INFO, "Closing Marvell crypto device %s on numa socket %u.",
+		 name, rte_socket_id());
 
 	sam_deinit();
+	rte_mvep_deinit(MVEP_MOD_T_SAM);
 
 	cryptodev = rte_cryptodev_pmd_get_named_dev(name);
 	if (cryptodev == NULL)
@@ -935,3 +985,8 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_MRVL_PMD,
 	"socket_id=<int>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(mrvl_crypto_drv, cryptodev_mrvl_pmd_drv.driver,
 		cryptodev_driver_id);
+
+RTE_INIT(crypto_mrvl_init_log)
+{
+	mrvl_logtype_driver = rte_log_register("pmd.crypto.mvsam");
+}
diff --git a/drivers/crypto/mvsam/rte_mrvl_pmd_ops.c b/drivers/crypto/mvsam/rte_mrvl_pmd_ops.c
index c045562c..9956f051 100644
--- a/drivers/crypto/mvsam/rte_mrvl_pmd_ops.c
+++ b/drivers/crypto/mvsam/rte_mrvl_pmd_ops.c
@@ -30,9 +30,9 @@ static const struct rte_cryptodev_capabilities
 					.increment = 1
 				},
 				.digest_size = {
-					.min = 16,
+					.min = 12,
 					.max = 16,
-					.increment = 0
+					.increment = 4
 				},
 			}, }
 		}, }
@@ -50,9 +50,9 @@ static const struct rte_cryptodev_capabilities
 						.increment = 0
 					},
 					.digest_size = {
-						.min = 16,
+						.min = 12,
 						.max = 16,
-						.increment = 0
+						.increment = 4
 					},
 				}, }
 			}, }
@@ -70,9 +70,9 @@ static const struct rte_cryptodev_capabilities
 						.increment = 1
 					},
 					.digest_size = {
-						.min = 20,
+						.min = 12,
 						.max = 20,
-						.increment = 0
+						.increment = 4
 					},
 				}, }
 			}, }
@@ -90,8 +90,29 @@ static const struct rte_cryptodev_capabilities
 					.increment = 0
 				},
 				.digest_size = {
-					.min = 20,
+					.min = 12,
 					.max = 20,
+					.increment = 4
+				},
+			}, }
+		}, }
+	},
+	{
+		/* SHA224 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 1,
+					.max = 64,
+					.increment = 1
+				},
+				.digest_size = {
+					.min = 28,
+					.max = 28,
 					.increment = 0
 				},
 			}, }
@@ -110,9 +131,9 @@ static const struct rte_cryptodev_capabilities
 					.increment = 0
 				},
 				.digest_size = {
-					.min = 28,
+					.min = 12,
 					.max = 28,
-					.increment = 0
+					.increment = 4
 				},
 			}, }
 		}, }
@@ -130,9 +151,9 @@ static const struct rte_cryptodev_capabilities
 						.increment = 1
 					},
 					.digest_size = {
-						.min = 32,
+						.min = 12,
 						.max = 32,
-						.increment = 0
+						.increment = 4
 					},
 				}, }
 			}, }
@@ -150,9 +171,9 @@ static const struct rte_cryptodev_capabilities
 						.increment = 0
 					},
 					.digest_size = {
-						.min = 32,
+						.min = 12,
 						.max = 32,
-						.increment = 0
+						.increment = 4
 					},
 				}, }
 			}, }
@@ -170,9 +191,9 @@ static const struct rte_cryptodev_capabilities
 					.increment = 1
 				},
 				.digest_size = {
-					.min = 48,
+					.min = 12,
 					.max = 48,
-					.increment = 0
+					.increment = 4
 				},
 			}, }
 		}, }
@@ -190,9 +211,9 @@ static const struct rte_cryptodev_capabilities
 					.increment = 0
 				},
 				.digest_size = {
-					.min = 48,
+					.min = 12,
 					.max = 48,
-					.increment = 0
+					.increment = 4
 				},
 			}, }
 		}, }
@@ -210,9 +231,9 @@ static const struct rte_cryptodev_capabilities
 					.increment = 1
 				},
 				.digest_size = {
-					.min = 64,
-					.max = 64,
-					.increment = 0
+					.min = 12,
+					.max = 48,
+					.increment = 4
 				},
 			}, }
 		}, }
@@ -230,8 +251,8 @@ static const struct rte_cryptodev_capabilities
 					.increment = 0
 				},
 				.digest_size = {
-					.min = 64,
-					.max = 64,
+					.min = 12,
+					.max = 48,
 					.increment = 0
 				},
 			}, }
@@ -277,6 +298,26 @@ static const struct rte_cryptodev_capabilities
 			}, }
 		}, }
 	},
+	{	/* AES ECB */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_ECB,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 8
+				},
+				.iv_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
 	{	/* AES GCM */
 		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 		{.sym = {
@@ -372,6 +413,71 @@ static const struct rte_cryptodev_capabilities
 			}, }
 		}, }
 	},
+	{	/* 3DES ECB */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_3DES_ECB,
+				.block_size = 8,
+				.key_size = {
+					.min = 24,
+					.max = 24,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* NULL (AUTH) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_NULL,
+				.block_size = 1,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				}
+			}, },
+		}, },
+	},
+	{	/* NULL (CIPHER) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_NULL,
+				.block_size = 1,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				}
+			}, },
+		}, }
+	},
 
 	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 };
@@ -551,7 +657,7 @@ mrvl_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		 */
 		int num = sam_get_num_inst();
 		if (num == 0) {
-			MRVL_CRYPTO_LOG_ERR("No crypto engines detected.\n");
+			MRVL_LOG(ERR, "No crypto engines detected!");
 			return -1;
 		}
 
@@ -635,7 +741,7 @@ mrvl_crypto_pmd_sym_session_configure(__rte_unused struct rte_cryptodev *dev,
 	int ret;
 
 	if (sess == NULL) {
-		MRVL_CRYPTO_LOG_ERR("Invalid session struct.");
+		MRVL_LOG(ERR, "Invalid session struct!");
 		return -EINVAL;
 	}
 
@@ -646,7 +752,7 @@ mrvl_crypto_pmd_sym_session_configure(__rte_unused struct rte_cryptodev *dev,
 
 	ret = mrvl_crypto_set_session_parameters(sess_private_data, xform);
 	if (ret != 0) {
-		MRVL_CRYPTO_LOG_ERR("Failed to configure session parameters.");
+		MRVL_LOG(ERR, "Failed to configure session parameters!");
 
 		/* Return session to mempool */
 		rte_mempool_put(mp, sess_private_data);
@@ -658,7 +764,7 @@ mrvl_crypto_pmd_sym_session_configure(__rte_unused struct rte_cryptodev *dev,
 	mrvl_sess = (struct mrvl_crypto_session *)sess_private_data;
 	if (sam_session_create(&mrvl_sess->sam_sess_params,
 				&mrvl_sess->sam_sess) < 0) {
-		MRVL_CRYPTO_LOG_DBG("Failed to create session!");
+		MRVL_LOG(DEBUG, "Failed to create session!");
 		return -EIO;
 	}
 
@@ -686,7 +792,7 @@ mrvl_crypto_pmd_sym_session_clear(struct rte_cryptodev *dev,
 
 		if (mrvl_sess->sam_sess &&
 		    sam_session_destroy(mrvl_sess->sam_sess) < 0) {
-			MRVL_CRYPTO_LOG_INFO("Error while destroying session!");
+			MRVL_LOG(ERR, "Error while destroying session!");
 		}
 
 		memset(sess, 0, sizeof(struct mrvl_crypto_session));
diff --git a/drivers/crypto/mvsam/rte_mrvl_pmd_private.h b/drivers/crypto/mvsam/rte_mrvl_pmd_private.h
index c16d95b4..6f8cf562 100644
--- a/drivers/crypto/mvsam/rte_mrvl_pmd_private.h
+++ b/drivers/crypto/mvsam/rte_mrvl_pmd_private.h
@@ -12,32 +12,21 @@
 #define CRYPTODEV_NAME_MRVL_PMD crypto_mvsam
 /**< Marvell PMD device name */
 
-#define MRVL_CRYPTO_LOG_ERR(fmt, args...) \
-	RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",  \
-			RTE_STR(CRYPTODEV_NAME_MRVL_PMD), \
-			__func__, __LINE__, ## args)
-
-#ifdef RTE_LIBRTE_PMD_MRVL_CRYPTO_DEBUG
-#define MRVL_CRYPTO_LOG_INFO(fmt, args...) \
-	RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_MRVL_PMD), \
-			__func__, __LINE__, ## args)
-
-#define MRVL_CRYPTO_LOG_DBG(fmt, args...) \
-	RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_MRVL_PMD), \
-			__func__, __LINE__, ## args)
-
-#else
-#define MRVL_CRYPTO_LOG_INFO(fmt, args...)
-#define MRVL_CRYPTO_LOG_DBG(fmt, args...)
-#endif
+/** MRVL PMD LOGTYPE DRIVER */
+int mrvl_logtype_driver;
+
+#define MRVL_LOG(level, fmt, ...) \
+	rte_log(RTE_LOG_ ## level, mrvl_logtype_driver, \
+			"%s() line %u: " fmt "\n", __func__, __LINE__, \
+					## __VA_ARGS__)
 
 /**
  * Handy bits->bytes conversion macro.
  */
 #define BITS2BYTES(x) ((x) >> 3)
 
+#define MRVL_MAX_SEGMENTS 16
+
 /** The operation order mode enumerator. */
 enum mrvl_crypto_chain_order {
 	MRVL_CRYPTO_CHAIN_CIPHER_ONLY,
@@ -84,6 +73,11 @@ struct mrvl_crypto_session {
 	uint16_t cipher_iv_offset;
 } __rte_cache_aligned;
 
+struct mrvl_crypto_src_table {
+	uint16_t iter_ops;
+	struct sam_buf_info src_bd[MRVL_MAX_SEGMENTS];
+} __rte_cache_aligned;
+
 /** Set and validate MRVL crypto session parameters */
 extern int
 mrvl_crypto_set_session_parameters(struct mrvl_crypto_session *sess,
diff --git a/drivers/crypto/null/null_crypto_pmd_ops.c b/drivers/crypto/null/null_crypto_pmd_ops.c
index bb2b6e14..2bdcd019 100644
--- a/drivers/crypto/null/null_crypto_pmd_ops.c
+++ b/drivers/crypto/null/null_crypto_pmd_ops.c
@@ -308,7 +308,7 @@ null_crypto_pmd_sym_session_clear(struct rte_cryptodev *dev,
 	}
 }
 
-struct rte_cryptodev_ops pmd_ops = {
+static struct rte_cryptodev_ops pmd_ops = {
 		.dev_configure		= null_crypto_pmd_config,
 		.dev_start		= null_crypto_pmd_start,
 		.dev_stop		= null_crypto_pmd_stop,
diff --git a/drivers/crypto/octeontx/Makefile b/drivers/crypto/octeontx/Makefile
new file mode 100644
index 00000000..2e78e69b
--- /dev/null
+++ b/drivers/crypto/octeontx/Makefile
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Cavium, Inc
+#
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_pmd_octeontx_crypto.a
+
+# library version
+LIBABIVER := 1
+
+# build flags
+CFLAGS += $(WERROR_FLAGS)
+
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_cryptodev
+LDLIBS += -lrte_pci -lrte_bus_pci
+LDLIBS += -lrte_common_cpt
+
+VPATH += $(RTE_SDK)/drivers/crypto/octeontx
+
+CFLAGS += -O3 -DCPT_MODEL=CRYPTO_OCTEONTX
+CFLAGS += -I$(RTE_SDK)/drivers/common/cpt
+
+# PMD code
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += otx_cryptodev.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += otx_cryptodev_capabilities.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += otx_cryptodev_hw_access.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += otx_cryptodev_mbox.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += otx_cryptodev_ops.c
+
+# export include files
+SYMLINK-y-include +=
+
+# versioning export map
+EXPORT_MAP := rte_pmd_octeontx_crypto_version.map
+
+# library dependencies
+DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += lib/librte_eal
+DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += lib/librte_cryptodev
+DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += lib/librte_mempool
+DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += lib/librte_mbuf
+DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += lib/librte_malloc
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/crypto/octeontx/meson.build b/drivers/crypto/octeontx/meson.build
new file mode 100644
index 00000000..6511b402
--- /dev/null
+++ b/drivers/crypto/octeontx/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Cavium, Inc
+if host_machine.system() != 'linux'
+	build = false
+endif
+
+deps += ['bus_pci']
+deps += ['common_cpt']
+name = 'octeontx_crypto'
+
+sources = files('otx_cryptodev.c',
+		'otx_cryptodev_capabilities.c',
+		'otx_cryptodev_hw_access.c',
+		'otx_cryptodev_mbox.c',
+		'otx_cryptodev_ops.c')
+
+includes += include_directories('../../common/cpt')
+cflags += '-DCPT_MODEL=CRYPTO_OCTEONTX'
diff --git a/drivers/crypto/octeontx/otx_cryptodev.c b/drivers/crypto/octeontx/otx_cryptodev.c
new file mode 100644
index 00000000..269f0456
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev.c
@@ -0,0 +1,133 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#include <rte_bus_pci.h>
+#include <rte_common.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_log.h>
+#include <rte_pci.h>
+
+/* CPT common headers */
+#include "cpt_pmd_logs.h"
+
+#include "otx_cryptodev.h"
+#include "otx_cryptodev_ops.h"
+
+static int otx_cryptodev_logtype;
+
+static struct rte_pci_id pci_id_cpt_table[] = {
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID),
+	},
+	/* sentinel */
+	{
+		.device_id = 0
+	},
+};
+
+static void
+otx_cpt_logtype_init(void)
+{
+	cpt_logtype = otx_cryptodev_logtype;
+}
+
+static int
+otx_cpt_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct rte_cryptodev *cryptodev;
+	char name[RTE_CRYPTODEV_NAME_MAX_LEN];
+	int retval;
+
+	if (pci_drv == NULL)
+		return -ENODEV;
+
+	rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
+
+	cryptodev = rte_cryptodev_pmd_allocate(name, rte_socket_id());
+	if (cryptodev == NULL)
+		return -ENOMEM;
+
+	cryptodev->device = &pci_dev->device;
+	cryptodev->device->driver = &pci_drv->driver;
+	cryptodev->driver_id = otx_cryptodev_driver_id;
+
+	/* init user callbacks */
+	TAILQ_INIT(&(cryptodev->link_intr_cbs));
+
+	/* init logtype used in common */
+	otx_cpt_logtype_init();
+
+	/* Invoke PMD device initialization function */
+	retval = otx_cpt_dev_create(cryptodev);
+	if (retval == 0)
+		return 0;
+
+	CPT_LOG_ERR("[DRV %s]: Failed to create device "
+			"(vendor_id: 0x%x device_id: 0x%x",
+			pci_drv->driver.name,
+			(unsigned int) pci_dev->id.vendor_id,
+			(unsigned int) pci_dev->id.device_id);
+
+	cryptodev->attached = RTE_CRYPTODEV_DETACHED;
+
+	return -ENXIO;
+}
+
+static int
+otx_cpt_pci_remove(struct rte_pci_device *pci_dev)
+{
+	struct rte_cryptodev *cryptodev;
+	char name[RTE_CRYPTODEV_NAME_MAX_LEN];
+
+	if (pci_dev == NULL)
+		return -EINVAL;
+
+	rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
+
+	cryptodev = rte_cryptodev_pmd_get_named_dev(name);
+	if (cryptodev == NULL)
+		return -ENODEV;
+
+	if (pci_dev->driver == NULL)
+		return -ENODEV;
+
+	/* free crypto device */
+	rte_cryptodev_pmd_release_device(cryptodev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+		rte_free(cryptodev->data->dev_private);
+
+	cryptodev->device = NULL;
+	cryptodev->device->driver = NULL;
+	cryptodev->data = NULL;
+
+	/* free metapool memory */
+	cleanup_global_resources();
+
+	return 0;
+}
+
+static struct rte_pci_driver otx_cryptodev_pmd = {
+	.id_table = pci_id_cpt_table,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = otx_cpt_pci_probe,
+	.remove = otx_cpt_pci_remove,
+};
+
+static struct cryptodev_driver otx_cryptodev_drv;
+
+RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_OCTEONTX_PMD, otx_cryptodev_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(CRYPTODEV_NAME_OCTEONTX_PMD, pci_id_cpt_table);
+RTE_PMD_REGISTER_CRYPTO_DRIVER(otx_cryptodev_drv, otx_cryptodev_pmd.driver,
+		otx_cryptodev_driver_id);
+
+RTE_INIT(otx_cpt_init_log)
+{
+	/* Bus level logs */
+	otx_cryptodev_logtype = rte_log_register("pmd.crypto.octeontx");
+	if (otx_cryptodev_logtype >= 0)
+		rte_log_set_level(otx_cryptodev_logtype, RTE_LOG_NOTICE);
+}
diff --git a/drivers/crypto/octeontx/otx_cryptodev.h b/drivers/crypto/octeontx/otx_cryptodev.h
new file mode 100644
index 00000000..6c2871d7
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _OTX_CRYPTODEV_H_
+#define _OTX_CRYPTODEV_H_
+
+/* Cavium OCTEON TX crypto PMD device name */
+#define CRYPTODEV_NAME_OCTEONTX_PMD	crypto_octeontx
+
+/* Device ID */
+#define PCI_VENDOR_ID_CAVIUM		0x177d
+#define CPT_81XX_PCI_VF_DEVICE_ID	0xa041
+
+/*
+ * Crypto device driver ID
+ */
+uint8_t otx_cryptodev_driver_id;
+
+#endif /* _OTX_CRYPTODEV_H_ */
diff --git a/drivers/crypto/octeontx/otx_cryptodev_capabilities.c b/drivers/crypto/octeontx/otx_cryptodev_capabilities.c
new file mode 100644
index 00000000..946571cf
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev_capabilities.c
@@ -0,0 +1,604 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#include <rte_cryptodev.h>
+
+#include "otx_cryptodev_capabilities.h"
+
+static const struct rte_cryptodev_capabilities otx_capabilities[] = {
+	/* Symmetric capabilities */
+	{	/* NULL (AUTH) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_NULL,
+				.block_size = 1,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+			}, },
+		}, },
+	},
+	{	/* AES GMAC (AUTH) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_AES_GMAC,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 8
+				},
+				.digest_size = {
+					.min = 8,
+					.max = 16,
+					.increment = 4
+				},
+				.iv_size = {
+					.min = 12,
+					.max = 12,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* KASUMI (F9) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_KASUMI_F9,
+				.block_size = 8,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 4,
+					.max = 4,
+					.increment = 0
+				},
+			}, }
+		}, }
+	},
+	{	/* MD5 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_MD5,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 16,
+					.increment = 1
+				},
+			}, }
+		}, }
+	},
+	{	/* MD5 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_MD5_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 8,
+					.max = 64,
+					.increment = 8
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 16,
+					.increment = 1
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA1 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA1,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 20,
+					.increment = 1
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA1 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 64,
+					.max = 64,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 20,
+					.increment = 1
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA224 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA224,
+				.block_size = 64,
+					.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 28,
+					.increment = 1
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA224 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
+				.block_size = 64,
+					.key_size = {
+					.min = 64,
+					.max = 64,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 28,
+					.increment = 1
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA256 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA256,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 32,
+					.increment = 1
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA256 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 64,
+					.max = 64,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 32,
+					.increment = 1
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA384 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA384,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 48,
+					.increment = 1
+					},
+			}, }
+		}, }
+	},
+	{	/* SHA384 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 64,
+					.max = 64,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 48,
+					.increment = 1
+					},
+			}, }
+		}, }
+	},
+	{	/* SHA512 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA512,
+				.block_size = 128,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 64,
+					.increment = 1
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA512 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
+				.block_size = 128,
+				.key_size = {
+					.min = 64,
+					.max = 64,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 1,
+					.max = 64,
+					.increment = 1
+				},
+			}, }
+		}, }
+	},
+	{	/* SNOW 3G (UIA2) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 4,
+					.max = 4,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* ZUC (EIA3) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_ZUC_EIA3,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 4,
+					.max = 4,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* NULL (CIPHER) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_NULL,
+				.block_size = 1,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				}
+			}, },
+		}, }
+	},
+	{	/* 3DES CBC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_3DES_CBC,
+				.block_size = 8,
+				.key_size = {
+					.min = 24,
+					.max = 24,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 8,
+					.max = 16,
+					.increment = 8
+				}
+			}, }
+		}, }
+	},
+	{	/* 3DES ECB */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_3DES_ECB,
+				.block_size = 8,
+				.key_size = {
+					.min = 24,
+					.max = 24,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* AES CBC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_CBC,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 8
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* AES CTR */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_CTR,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 8
+				},
+				.iv_size = {
+					.min = 12,
+					.max = 16,
+					.increment = 4
+				}
+			}, }
+		}, }
+	},
+	{	/* AES XTS */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_XTS,
+				.block_size = 16,
+				.key_size = {
+					.min = 32,
+					.max = 64,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* DES CBC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_DES_CBC,
+				.block_size = 8,
+				.key_size = {
+					.min = 8,
+					.max = 8,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 8,
+					.max = 8,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* KASUMI (F8) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_KASUMI_F8,
+				.block_size = 8,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 8,
+					.max = 8,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* SNOW 3G (UEA2) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* ZUC (EEA3) */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_ZUC_EEA3,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	{	/* AES GCM */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+			{.aead = {
+				.algo = RTE_CRYPTO_AEAD_AES_GCM,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 8
+				},
+				.digest_size = {
+					.min = 8,
+					.max = 16,
+					.increment = 4
+				},
+				.aad_size = {
+					.min = 0,
+					.max = 1024,
+					.increment = 1
+				},
+				.iv_size = {
+					.min = 12,
+					.max = 12,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+	/* End of symmetric capabilities */
+	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+const struct rte_cryptodev_capabilities *
+otx_get_capabilities(void)
+{
+	return otx_capabilities;
+}
diff --git a/drivers/crypto/octeontx/otx_cryptodev_capabilities.h b/drivers/crypto/octeontx/otx_cryptodev_capabilities.h
new file mode 100644
index 00000000..fc62821b
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev_capabilities.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _OTX_CRYPTODEV_CAPABILITIES_H_
+#define _OTX_CRYPTODEV_CAPABILITIES_H_
+
+#include <rte_cryptodev.h>
+
+/*
+ * Get capabilities list for the device
+ *
+ */
+const struct rte_cryptodev_capabilities *
+otx_get_capabilities(void);
+
+#endif /* _OTX_CRYPTODEV_CAPABILITIES_H_ */
diff --git a/drivers/crypto/octeontx/otx_cryptodev_hw_access.c b/drivers/crypto/octeontx/otx_cryptodev_hw_access.c
new file mode 100644
index 00000000..5e705a83
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev_hw_access.c
@@ -0,0 +1,598 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+#include <assert.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <rte_branch_prediction.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_memzone.h>
+
+#include "otx_cryptodev_hw_access.h"
+#include "otx_cryptodev_mbox.h"
+
+#include "cpt_pmd_logs.h"
+#include "cpt_hw_types.h"
+
+/*
+ * VF HAL functions
+ * Access its own BAR0/4 registers by passing VF number as 0.
+ * OS/PCI maps them accordingly.
+ */
+
+static int
+otx_cpt_vf_init(struct cpt_vf *cptvf)
+{
+	int ret = 0;
+
+	/* Check ready with PF */
+	/* Gets chip ID / device Id from PF if ready */
+	ret = otx_cpt_check_pf_ready(cptvf);
+	if (ret) {
+		CPT_LOG_ERR("%s: PF not responding to READY msg",
+				cptvf->dev_name);
+		ret = -EBUSY;
+		goto exit;
+	}
+
+	CPT_LOG_DP_DEBUG("%s: %s done", cptvf->dev_name, __func__);
+
+exit:
+	return ret;
+}
+
+/*
+ * Read Interrupt status of the VF
+ *
+ * @param   cptvf	cptvf structure
+ */
+static uint64_t
+otx_cpt_read_vf_misc_intr_status(struct cpt_vf *cptvf)
+{
+	return CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), CPTX_VQX_MISC_INT(0, 0));
+}
+
+/*
+ * Clear mailbox interrupt of the VF
+ *
+ * @param   cptvf	cptvf structure
+ */
+static void
+otx_cpt_clear_mbox_intr(struct cpt_vf *cptvf)
+{
+	cptx_vqx_misc_int_t vqx_misc_int;
+
+	vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				      CPTX_VQX_MISC_INT(0, 0));
+	/* W1C for the VF */
+	vqx_misc_int.s.mbox = 1;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
+}
+
+/*
+ * Clear instruction NCB read error interrupt of the VF
+ *
+ * @param   cptvf	cptvf structure
+ */
+static void
+otx_cpt_clear_irde_intr(struct cpt_vf *cptvf)
+{
+	cptx_vqx_misc_int_t vqx_misc_int;
+
+	vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				      CPTX_VQX_MISC_INT(0, 0));
+	/* W1C for the VF */
+	vqx_misc_int.s.irde = 1;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
+}
+
+/*
+ * Clear NCB result write response error interrupt of the VF
+ *
+ * @param   cptvf	cptvf structure
+ */
+static void
+otx_cpt_clear_nwrp_intr(struct cpt_vf *cptvf)
+{
+	cptx_vqx_misc_int_t vqx_misc_int;
+
+	vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				      CPTX_VQX_MISC_INT(0, 0));
+	/* W1C for the VF */
+	vqx_misc_int.s.nwrp = 1;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
+}
+
+/*
+ * Clear swerr interrupt of the VF
+ *
+ * @param   cptvf	cptvf structure
+ */
+static void
+otx_cpt_clear_swerr_intr(struct cpt_vf *cptvf)
+{
+	cptx_vqx_misc_int_t vqx_misc_int;
+
+	vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				      CPTX_VQX_MISC_INT(0, 0));
+	/* W1C for the VF */
+	vqx_misc_int.s.swerr = 1;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
+}
+
+/*
+ * Clear hwerr interrupt of the VF
+ *
+ * @param   cptvf	cptvf structure
+ */
+static void
+otx_cpt_clear_hwerr_intr(struct cpt_vf *cptvf)
+{
+	cptx_vqx_misc_int_t vqx_misc_int;
+
+	vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				      CPTX_VQX_MISC_INT(0, 0));
+	/* W1C for the VF */
+	vqx_misc_int.s.hwerr = 1;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
+}
+
+/*
+ * Clear translation fault interrupt of the VF
+ *
+ * @param   cptvf	cptvf structure
+ */
+static void
+otx_cpt_clear_fault_intr(struct cpt_vf *cptvf)
+{
+	cptx_vqx_misc_int_t vqx_misc_int;
+
+	vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				CPTX_VQX_MISC_INT(0, 0));
+	/* W1C for the VF */
+	vqx_misc_int.s.fault = 1;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
+}
+
+/*
+ * Clear doorbell overflow interrupt of the VF
+ *
+ * @param   cptvf	cptvf structure
+ */
+static void
+otx_cpt_clear_dovf_intr(struct cpt_vf *cptvf)
+{
+	cptx_vqx_misc_int_t vqx_misc_int;
+
+	vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				      CPTX_VQX_MISC_INT(0, 0));
+	/* W1C for the VF */
+	vqx_misc_int.s.dovf = 1;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
+}
+
+/* Write to VQX_CTL register
+ */
+static void
+otx_cpt_write_vq_ctl(struct cpt_vf *cptvf, bool val)
+{
+	cptx_vqx_ctl_t vqx_ctl;
+
+	vqx_ctl.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				 CPTX_VQX_CTL(0, 0));
+	vqx_ctl.s.ena = val;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_CTL(0, 0), vqx_ctl.u);
+}
+
+/* Write to VQX_INPROG register
+ */
+static void
+otx_cpt_write_vq_inprog(struct cpt_vf *cptvf, uint8_t val)
+{
+	cptx_vqx_inprog_t vqx_inprg;
+
+	vqx_inprg.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				   CPTX_VQX_INPROG(0, 0));
+	vqx_inprg.s.inflight = val;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
+}
+
+/* Write to VQX_DONE_WAIT NUMWAIT register
+ */
+static void
+otx_cpt_write_vq_done_numwait(struct cpt_vf *cptvf, uint32_t val)
+{
+	cptx_vqx_done_wait_t vqx_dwait;
+
+	vqx_dwait.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				   CPTX_VQX_DONE_WAIT(0, 0));
+	vqx_dwait.s.num_wait = val;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_DONE_WAIT(0, 0), vqx_dwait.u);
+}
+
+/* Write to VQX_DONE_WAIT NUM_WAIT register
+ */
+static void
+otx_cpt_write_vq_done_timewait(struct cpt_vf *cptvf, uint16_t val)
+{
+	cptx_vqx_done_wait_t vqx_dwait;
+
+	vqx_dwait.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				   CPTX_VQX_DONE_WAIT(0, 0));
+	vqx_dwait.s.time_wait = val;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_DONE_WAIT(0, 0), vqx_dwait.u);
+}
+
+/* Write to VQX_SADDR register
+ */
+static void
+otx_cpt_write_vq_saddr(struct cpt_vf *cptvf, uint64_t val)
+{
+	cptx_vqx_saddr_t vqx_saddr;
+
+	vqx_saddr.u = val;
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
+}
+
+static void
+otx_cpt_vfvq_init(struct cpt_vf *cptvf)
+{
+	uint64_t base_addr = 0;
+
+	/* Disable the VQ */
+	otx_cpt_write_vq_ctl(cptvf, 0);
+
+	/* Reset the doorbell */
+	otx_cpt_write_vq_doorbell(cptvf, 0);
+	/* Clear inflight */
+	otx_cpt_write_vq_inprog(cptvf, 0);
+
+	/* Write VQ SADDR */
+	base_addr = (uint64_t)(cptvf->cqueue.chead[0].dma_addr);
+	otx_cpt_write_vq_saddr(cptvf, base_addr);
+
+	/* Configure timerhold / coalescence */
+	otx_cpt_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
+	otx_cpt_write_vq_done_numwait(cptvf, CPT_COUNT_THOLD);
+
+	/* Enable the VQ */
+	otx_cpt_write_vq_ctl(cptvf, 1);
+}
+
+static int
+cpt_vq_init(struct cpt_vf *cptvf, uint8_t group)
+{
+	int err;
+
+	/* Convey VQ LEN to PF */
+	err = otx_cpt_send_vq_size_msg(cptvf);
+	if (err) {
+		CPT_LOG_ERR("%s: PF not responding to QLEN msg",
+			    cptvf->dev_name);
+		err = -EBUSY;
+		goto cleanup;
+	}
+
+	/* CPT VF device initialization */
+	otx_cpt_vfvq_init(cptvf);
+
+	/* Send msg to PF to assign currnet Q to required group */
+	cptvf->vfgrp = group;
+	err = otx_cpt_send_vf_grp_msg(cptvf, group);
+	if (err) {
+		CPT_LOG_ERR("%s: PF not responding to VF_GRP msg",
+			    cptvf->dev_name);
+		err = -EBUSY;
+		goto cleanup;
+	}
+
+	CPT_LOG_DP_DEBUG("%s: %s done", cptvf->dev_name, __func__);
+	return 0;
+
+cleanup:
+	return err;
+}
+
+void
+otx_cpt_poll_misc(struct cpt_vf *cptvf)
+{
+	uint64_t intr;
+
+	intr = otx_cpt_read_vf_misc_intr_status(cptvf);
+
+	if (!intr)
+		return;
+
+	/* Check for MISC interrupt types */
+	if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
+		CPT_LOG_DP_DEBUG("%s: Mailbox interrupt 0x%lx on CPT VF %d",
+			cptvf->dev_name, (unsigned int long)intr, cptvf->vfid);
+		otx_cpt_handle_mbox_intr(cptvf);
+		otx_cpt_clear_mbox_intr(cptvf);
+	} else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
+		otx_cpt_clear_irde_intr(cptvf);
+		CPT_LOG_DP_DEBUG("%s: Instruction NCB read error interrupt "
+				"0x%lx on CPT VF %d", cptvf->dev_name,
+				(unsigned int long)intr, cptvf->vfid);
+	} else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) {
+		otx_cpt_clear_nwrp_intr(cptvf);
+		CPT_LOG_DP_DEBUG("%s: NCB response write error interrupt 0x%lx"
+				" on CPT VF %d", cptvf->dev_name,
+				(unsigned int long)intr, cptvf->vfid);
+	} else if (unlikely(intr & CPT_VF_INTR_SWERR_MASK)) {
+		otx_cpt_clear_swerr_intr(cptvf);
+		CPT_LOG_DP_DEBUG("%s: Software error interrupt 0x%lx on CPT VF "
+				"%d", cptvf->dev_name, (unsigned int long)intr,
+				cptvf->vfid);
+	} else if (unlikely(intr & CPT_VF_INTR_HWERR_MASK)) {
+		otx_cpt_clear_hwerr_intr(cptvf);
+		CPT_LOG_DP_DEBUG("%s: Hardware error interrupt 0x%lx on CPT VF "
+				"%d", cptvf->dev_name, (unsigned int long)intr,
+				cptvf->vfid);
+	} else if (unlikely(intr & CPT_VF_INTR_FAULT_MASK)) {
+		otx_cpt_clear_fault_intr(cptvf);
+		CPT_LOG_DP_DEBUG("%s: Translation fault interrupt 0x%lx on CPT VF "
+				"%d", cptvf->dev_name, (unsigned int long)intr,
+				cptvf->vfid);
+	} else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) {
+		otx_cpt_clear_dovf_intr(cptvf);
+		CPT_LOG_DP_DEBUG("%s: Doorbell overflow interrupt 0x%lx on CPT VF "
+				"%d", cptvf->dev_name, (unsigned int long)intr,
+				cptvf->vfid);
+	} else
+		CPT_LOG_DP_ERR("%s: Unhandled interrupt 0x%lx in CPT VF %d",
+				cptvf->dev_name, (unsigned int long)intr,
+				cptvf->vfid);
+}
+
+int
+otx_cpt_hw_init(struct cpt_vf *cptvf, void *pdev, void *reg_base, char *name)
+{
+	memset(cptvf, 0, sizeof(struct cpt_vf));
+
+	/* Bar0 base address */
+	cptvf->reg_base = reg_base;
+	strncpy(cptvf->dev_name, name, 32);
+
+	cptvf->pdev = pdev;
+
+	/* To clear if there are any pending mbox msgs */
+	otx_cpt_poll_misc(cptvf);
+
+	if (otx_cpt_vf_init(cptvf)) {
+		CPT_LOG_ERR("Failed to initialize CPT VF device");
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+otx_cpt_deinit_device(void *dev)
+{
+	struct cpt_vf *cptvf = (struct cpt_vf *)dev;
+
+	/* Do misc work one last time */
+	otx_cpt_poll_misc(cptvf);
+
+	return 0;
+}
+
+int
+otx_cpt_get_resource(void *dev, uint8_t group, struct cpt_instance **instance)
+{
+	int ret = -ENOENT, len, qlen, i;
+	int chunk_len, chunks, chunk_size;
+	struct cpt_vf *cptvf = (struct cpt_vf *)dev;
+	struct cpt_instance *cpt_instance;
+	struct command_chunk *chunk_head = NULL, *chunk_prev = NULL;
+	struct command_chunk *chunk = NULL;
+	uint8_t *mem;
+	const struct rte_memzone *rz;
+	uint64_t dma_addr = 0, alloc_len, used_len;
+	uint64_t *next_ptr;
+	uint64_t pg_sz = sysconf(_SC_PAGESIZE);
+
+	CPT_LOG_DP_DEBUG("Initializing cpt resource %s", cptvf->dev_name);
+
+	cpt_instance = &cptvf->instance;
+
+	memset(&cptvf->cqueue, 0, sizeof(cptvf->cqueue));
+	memset(&cptvf->pqueue, 0, sizeof(cptvf->pqueue));
+
+	/* Chunks are of fixed size buffers */
+	chunks = DEFAULT_CMD_QCHUNKS;
+	chunk_len = DEFAULT_CMD_QCHUNK_SIZE;
+
+	qlen = chunks * chunk_len;
+	/* Chunk size includes 8 bytes of next chunk ptr */
+	chunk_size = chunk_len * CPT_INST_SIZE + CPT_NEXT_CHUNK_PTR_SIZE;
+
+	/* For command chunk structures */
+	len = chunks * RTE_ALIGN(sizeof(struct command_chunk), 8);
+
+	/* For pending queue */
+	len += qlen * RTE_ALIGN(sizeof(struct rid), 8);
+
+	/* So that instruction queues start as pg size aligned */
+	len = RTE_ALIGN(len, pg_sz);
+
+	/* For Instruction queues */
+	len += chunks * RTE_ALIGN(chunk_size, 128);
+
+	/* Wastage after instruction queues */
+	len = RTE_ALIGN(len, pg_sz);
+
+	rz = rte_memzone_reserve_aligned(cptvf->dev_name, len, cptvf->node,
+					 RTE_MEMZONE_SIZE_HINT_ONLY |
+					 RTE_MEMZONE_256MB,
+					 RTE_CACHE_LINE_SIZE);
+	if (!rz) {
+		ret = rte_errno;
+		goto cleanup;
+	}
+
+	mem = rz->addr;
+	dma_addr = rz->phys_addr;
+	alloc_len = len;
+
+	memset(mem, 0, len);
+
+	cpt_instance->rsvd = (uintptr_t)rz;
+
+	/* Pending queue setup */
+	cptvf->pqueue.rid_queue = (struct rid *)mem;
+	cptvf->pqueue.enq_tail = 0;
+	cptvf->pqueue.deq_head = 0;
+	cptvf->pqueue.pending_count = 0;
+
+	mem +=  qlen * RTE_ALIGN(sizeof(struct rid), 8);
+	len -=  qlen * RTE_ALIGN(sizeof(struct rid), 8);
+	dma_addr += qlen * RTE_ALIGN(sizeof(struct rid), 8);
+
+	/* Alignment wastage */
+	used_len = alloc_len - len;
+	mem += RTE_ALIGN(used_len, pg_sz) - used_len;
+	len -= RTE_ALIGN(used_len, pg_sz) - used_len;
+	dma_addr += RTE_ALIGN(used_len, pg_sz) - used_len;
+
+	/* Init instruction queues */
+	chunk_head = &cptvf->cqueue.chead[0];
+	i = qlen;
+
+	chunk_prev = NULL;
+	for (i = 0; i < DEFAULT_CMD_QCHUNKS; i++) {
+		int csize;
+
+		chunk = &cptvf->cqueue.chead[i];
+		chunk->head = mem;
+		chunk->dma_addr = dma_addr;
+
+		csize = RTE_ALIGN(chunk_size, 128);
+		mem += csize;
+		dma_addr += csize;
+		len -= csize;
+
+		if (chunk_prev) {
+			next_ptr = (uint64_t *)(chunk_prev->head +
+						chunk_size - 8);
+			*next_ptr = (uint64_t)chunk->dma_addr;
+		}
+		chunk_prev = chunk;
+	}
+	/* Circular loop */
+	next_ptr = (uint64_t *)(chunk_prev->head + chunk_size - 8);
+	*next_ptr = (uint64_t)chunk_head->dma_addr;
+
+	assert(!len);
+
+	/* This is used for CPT(0)_PF_Q(0..15)_CTL.size config */
+	cptvf->qsize = chunk_size / 8;
+	cptvf->cqueue.qhead = chunk_head->head;
+	cptvf->cqueue.idx = 0;
+	cptvf->cqueue.cchunk = 0;
+
+	if (cpt_vq_init(cptvf, group)) {
+		CPT_LOG_ERR("Failed to initialize CPT VQ of device %s",
+			    cptvf->dev_name);
+		ret = -EBUSY;
+		goto cleanup;
+	}
+
+	*instance = cpt_instance;
+
+	CPT_LOG_DP_DEBUG("Crypto device (%s) initialized", cptvf->dev_name);
+
+	return 0;
+cleanup:
+	rte_memzone_free(rz);
+	*instance = NULL;
+	return ret;
+}
+
+int
+otx_cpt_put_resource(struct cpt_instance *instance)
+{
+	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
+	struct rte_memzone *rz;
+
+	if (!cptvf) {
+		CPT_LOG_ERR("Invalid CPTVF handle");
+		return -EINVAL;
+	}
+
+	CPT_LOG_DP_DEBUG("Releasing cpt device %s", cptvf->dev_name);
+
+	rz = (struct rte_memzone *)instance->rsvd;
+	rte_memzone_free(rz);
+	return 0;
+}
+
+int
+otx_cpt_start_device(void *dev)
+{
+	int rc;
+	struct cpt_vf *cptvf = (struct cpt_vf *)dev;
+
+	rc = otx_cpt_send_vf_up(cptvf);
+	if (rc) {
+		CPT_LOG_ERR("Failed to mark CPT VF device %s UP, rc = %d",
+			    cptvf->dev_name, rc);
+		return -EFAULT;
+	}
+
+	if ((cptvf->vftype != SE_TYPE) && (cptvf->vftype != AE_TYPE)) {
+		CPT_LOG_ERR("Fatal error, unexpected vf type %u, for CPT VF "
+			    "device %s", cptvf->vftype, cptvf->dev_name);
+		return -ENOENT;
+	}
+
+	return 0;
+}
+
+void
+otx_cpt_stop_device(void *dev)
+{
+	int rc;
+	uint32_t pending, retries = 5;
+	struct cpt_vf *cptvf = (struct cpt_vf *)dev;
+
+	/* Wait for pending entries to complete */
+	pending = otx_cpt_read_vq_doorbell(cptvf);
+	while (pending) {
+		CPT_LOG_DP_DEBUG("%s: Waiting for pending %u cmds to complete",
+				 cptvf->dev_name, pending);
+		sleep(1);
+		pending = otx_cpt_read_vq_doorbell(cptvf);
+		retries--;
+		if (!retries)
+			break;
+	}
+
+	if (!retries && pending) {
+		CPT_LOG_ERR("%s: Timeout waiting for commands(%u)",
+			    cptvf->dev_name, pending);
+		return;
+	}
+
+	rc = otx_cpt_send_vf_down(cptvf);
+	if (rc) {
+		CPT_LOG_ERR("Failed to bring down vf %s, rc %d",
+			    cptvf->dev_name, rc);
+		return;
+	}
+}
diff --git a/drivers/crypto/octeontx/otx_cryptodev_hw_access.h b/drivers/crypto/octeontx/otx_cryptodev_hw_access.h
new file mode 100644
index 00000000..82b15eea
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev_hw_access.h
@@ -0,0 +1,320 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+#ifndef _OTX_CRYPTODEV_HW_ACCESS_H_
+#define _OTX_CRYPTODEV_HW_ACCESS_H_
+
+#include <stdbool.h>
+
+#include <rte_branch_prediction.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+#include <rte_prefetch.h>
+
+#include "cpt_common.h"
+#include "cpt_hw_types.h"
+#include "cpt_mcode_defines.h"
+#include "cpt_pmd_logs.h"
+
+#define CPT_INTR_POLL_INTERVAL_MS	(50)
+
+/* Default command queue length */
+#define DEFAULT_CMD_QCHUNKS		2
+#define DEFAULT_CMD_QCHUNK_SIZE		1023
+#define DEFAULT_CMD_QLEN \
+		(DEFAULT_CMD_QCHUNK_SIZE * DEFAULT_CMD_QCHUNKS)
+
+#define CPT_CSR_REG_BASE(cpt)		((cpt)->reg_base)
+
+/* Read hw register */
+#define CPT_READ_CSR(__hw_addr, __offset) \
+	rte_read64_relaxed((uint8_t *)__hw_addr + __offset)
+
+/* Write hw register */
+#define CPT_WRITE_CSR(__hw_addr, __offset, __val) \
+	rte_write64_relaxed((__val), ((uint8_t *)__hw_addr + __offset))
+
+/* cpt instance */
+struct cpt_instance {
+	uint32_t queue_id;
+	uintptr_t rsvd;
+};
+
+struct command_chunk {
+	/** 128-byte aligned real_vaddr */
+	uint8_t *head;
+	/** 128-byte aligned real_dma_addr */
+	phys_addr_t dma_addr;
+};
+
+/**
+ * Command queue structure
+ */
+struct command_queue {
+	/** Command queue host write idx */
+	uint32_t idx;
+	/** Command queue chunk */
+	uint32_t cchunk;
+	/** Command queue head; instructions are inserted here */
+	uint8_t *qhead;
+	/** Command chunk list head */
+	struct command_chunk chead[DEFAULT_CMD_QCHUNKS];
+};
+
+/**
+ * CPT VF device structure
+ */
+struct cpt_vf {
+	/** CPT instance */
+	struct cpt_instance instance;
+	/** Register start address */
+	uint8_t *reg_base;
+	/** Command queue information */
+	struct command_queue cqueue;
+	/** Pending queue information */
+	struct pending_queue pqueue;
+	/** Meta information per vf */
+	struct cptvf_meta_info meta_info;
+
+	/** Below fields are accessed only in control path */
+
+	/** Env specific pdev representing the pci dev */
+	void *pdev;
+	/** Calculated queue size */
+	uint32_t qsize;
+	/** Device index (0...CPT_MAX_VQ_NUM)*/
+	uint8_t  vfid;
+	/** VF type of cpt_vf_type_t (SE_TYPE(2) or AE_TYPE(1) */
+	uint8_t  vftype;
+	/** VF group (0 - 8) */
+	uint8_t  vfgrp;
+	/** Operating node: Bits (46:44) in BAR0 address */
+	uint8_t  node;
+
+	/** VF-PF mailbox communication */
+
+	/** Flag if acked */
+	bool pf_acked;
+	/** Flag if not acked */
+	bool pf_nacked;
+
+	/** Device name */
+	char dev_name[32];
+} __rte_cache_aligned;
+
+/*
+ * CPT Registers map for 81xx
+ */
+
+/* VF registers */
+#define CPTX_VQX_CTL(a, b)		(0x0000100ll + 0x1000000000ll * \
+					 ((a) & 0x0) + 0x100000ll * (b))
+#define CPTX_VQX_SADDR(a, b)		(0x0000200ll + 0x1000000000ll * \
+					 ((a) & 0x0) + 0x100000ll * (b))
+#define CPTX_VQX_DONE_WAIT(a, b)	(0x0000400ll + 0x1000000000ll * \
+					 ((a) & 0x0) + 0x100000ll * (b))
+#define CPTX_VQX_INPROG(a, b)		(0x0000410ll + 0x1000000000ll * \
+					 ((a) & 0x0) + 0x100000ll * (b))
+#define CPTX_VQX_DONE(a, b)		(0x0000420ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b))
+#define CPTX_VQX_DONE_ACK(a, b)		(0x0000440ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b))
+#define CPTX_VQX_DONE_INT_W1S(a, b)	(0x0000460ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b))
+#define CPTX_VQX_DONE_INT_W1C(a, b)	(0x0000468ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b))
+#define CPTX_VQX_DONE_ENA_W1S(a, b)	(0x0000470ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b))
+#define CPTX_VQX_DONE_ENA_W1C(a, b)	(0x0000478ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b))
+#define CPTX_VQX_MISC_INT(a, b)		(0x0000500ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b))
+#define CPTX_VQX_MISC_INT_W1S(a, b)	(0x0000508ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b))
+#define CPTX_VQX_MISC_ENA_W1S(a, b)	(0x0000510ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b))
+#define CPTX_VQX_MISC_ENA_W1C(a, b)	(0x0000518ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b))
+#define CPTX_VQX_DOORBELL(a, b)		(0x0000600ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b))
+#define CPTX_VFX_PF_MBOXX(a, b, c)	(0x0001000ll + 0x1000000000ll * \
+					 ((a) & 0x1) + 0x100000ll * (b) + \
+					 8ll * ((c) & 0x1))
+
+/* VF HAL functions */
+
+void
+otx_cpt_poll_misc(struct cpt_vf *cptvf);
+
+int
+otx_cpt_hw_init(struct cpt_vf *cptvf, void *pdev, void *reg_base, char *name);
+
+int
+otx_cpt_deinit_device(void *dev);
+
+int
+otx_cpt_get_resource(void *dev, uint8_t group, struct cpt_instance **instance);
+
+int
+otx_cpt_put_resource(struct cpt_instance *instance);
+
+int
+otx_cpt_start_device(void *cptvf);
+
+void
+otx_cpt_stop_device(void *cptvf);
+
+/* Write to VQX_DOORBELL register
+ */
+static __rte_always_inline void
+otx_cpt_write_vq_doorbell(struct cpt_vf *cptvf, uint32_t val)
+{
+	cptx_vqx_doorbell_t vqx_dbell;
+
+	vqx_dbell.u = 0;
+	vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VQX_DOORBELL(0, 0), vqx_dbell.u);
+}
+
+static __rte_always_inline uint32_t
+otx_cpt_read_vq_doorbell(struct cpt_vf *cptvf)
+{
+	cptx_vqx_doorbell_t vqx_dbell;
+
+	vqx_dbell.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				   CPTX_VQX_DOORBELL(0, 0));
+	return vqx_dbell.s.dbell_cnt;
+}
+
+static __rte_always_inline void
+otx_cpt_ring_dbell(struct cpt_instance *instance, uint16_t count)
+{
+	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
+	/* Memory barrier to flush pending writes */
+	rte_smp_wmb();
+	otx_cpt_write_vq_doorbell(cptvf, count);
+}
+
+static __rte_always_inline void *
+get_cpt_inst(struct command_queue *cqueue)
+{
+	CPT_LOG_DP_DEBUG("CPT queue idx %u\n", cqueue->idx);
+	return &cqueue->qhead[cqueue->idx * CPT_INST_SIZE];
+}
+
+static __rte_always_inline void
+fill_cpt_inst(struct cpt_instance *instance, void *req)
+{
+	struct command_queue *cqueue;
+	cpt_inst_s_t *cpt_ist_p;
+	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
+	struct cpt_request_info *user_req = (struct cpt_request_info *)req;
+	cqueue = &cptvf->cqueue;
+	cpt_ist_p = get_cpt_inst(cqueue);
+	rte_prefetch_non_temporal(cpt_ist_p);
+
+	/* EI0, EI1, EI2, EI3 are already prepared */
+	/* HW W0 */
+	cpt_ist_p->u[0] = 0;
+	/* HW W1 */
+	cpt_ist_p->s8x.res_addr = user_req->comp_baddr;
+	/* HW W2 */
+	cpt_ist_p->u[2] = 0;
+	/* HW W3 */
+	cpt_ist_p->s8x.wq_ptr = 0;
+
+	/* MC EI0 */
+	cpt_ist_p->s8x.ei0 = user_req->ist.ei0;
+	/* MC EI1 */
+	cpt_ist_p->s8x.ei1 = user_req->ist.ei1;
+	/* MC EI2 */
+	cpt_ist_p->s8x.ei2 = user_req->ist.ei2;
+	/* MC EI3 */
+	cpt_ist_p->s8x.ei3 = user_req->ist.ei3;
+}
+
+static __rte_always_inline void
+mark_cpt_inst(struct cpt_instance *instance)
+{
+	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
+	struct command_queue *queue = &cptvf->cqueue;
+	if (unlikely(++queue->idx >= DEFAULT_CMD_QCHUNK_SIZE)) {
+		uint32_t cchunk = queue->cchunk;
+		MOD_INC(cchunk, DEFAULT_CMD_QCHUNKS);
+		queue->qhead = queue->chead[cchunk].head;
+		queue->idx = 0;
+		queue->cchunk = cchunk;
+	}
+}
+
+static __rte_always_inline uint8_t
+check_nb_command_id(struct cpt_request_info *user_req,
+		struct cpt_instance *instance)
+{
+	uint8_t ret = ERR_REQ_PENDING;
+	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
+	volatile cpt_res_s_t *cptres;
+
+	cptres = (volatile cpt_res_s_t *)user_req->completion_addr;
+
+	if (unlikely(cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE)) {
+		/*
+		 * Wait for some time for this command to get completed
+		 * before timing out
+		 */
+		if (rte_get_timer_cycles() < user_req->time_out)
+			return ret;
+		/*
+		 * TODO: See if alternate caddr can be used to not loop
+		 * longer than needed.
+		 */
+		if ((cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE) &&
+		    (user_req->extra_time < TIME_IN_RESET_COUNT)) {
+			user_req->extra_time++;
+			return ret;
+		}
+
+		if (cptres->s8x.compcode != CPT_8X_COMP_E_NOTDONE)
+			goto complete;
+
+		ret = ERR_REQ_TIMEOUT;
+		CPT_LOG_DP_ERR("Request %p timedout", user_req);
+		otx_cpt_poll_misc(cptvf);
+		goto exit;
+	}
+
+complete:
+	if (likely(cptres->s8x.compcode == CPT_8X_COMP_E_GOOD)) {
+		ret = 0; /* success */
+		if (unlikely((uint8_t)*user_req->alternate_caddr)) {
+			ret = (uint8_t)*user_req->alternate_caddr;
+			CPT_LOG_DP_ERR("Request %p : failed with microcode"
+				" error, MC completion code : 0x%x", user_req,
+				ret);
+		}
+		CPT_LOG_DP_DEBUG("MC status %.8x\n",
+			   *((volatile uint32_t *)user_req->alternate_caddr));
+		CPT_LOG_DP_DEBUG("HW status %.8x\n",
+			   *((volatile uint32_t *)user_req->completion_addr));
+	} else if ((cptres->s8x.compcode == CPT_8X_COMP_E_SWERR) ||
+		   (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT)) {
+		ret = (uint8_t)*user_req->alternate_caddr;
+		if (!ret)
+			ret = ERR_BAD_ALT_CCODE;
+		CPT_LOG_DP_DEBUG("Request %p : failed with %s : err code :%x",
+			   user_req,
+			   (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT) ?
+			   "DMA Fault" : "Software error", ret);
+	} else {
+		CPT_LOG_DP_ERR("Request %p : unexpected completion code %d",
+			   user_req, cptres->s8x.compcode);
+		ret = (uint8_t)*user_req->alternate_caddr;
+	}
+
+exit:
+	return ret;
+}
+
+#endif /* _OTX_CRYPTODEV_HW_ACCESS_H_ */
diff --git a/drivers/crypto/octeontx/otx_cryptodev_mbox.c b/drivers/crypto/octeontx/otx_cryptodev_mbox.c
new file mode 100644
index 00000000..a8e51a8e
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev_mbox.c
@@ -0,0 +1,178 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#include <unistd.h>
+
+#include "otx_cryptodev_hw_access.h"
+#include "otx_cryptodev_mbox.h"
+
+void
+otx_cpt_handle_mbox_intr(struct cpt_vf *cptvf)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	/*
+	 * MBOX[0] contains msg
+	 * MBOX[1] contains data
+	 */
+	mbx.msg  = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				CPTX_VFX_PF_MBOXX(0, 0, 0));
+	mbx.data = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				CPTX_VFX_PF_MBOXX(0, 0, 1));
+
+	CPT_LOG_DP_DEBUG("%s: Mailbox msg 0x%lx from PF",
+		    cptvf->dev_name, (unsigned int long)mbx.msg);
+	switch (mbx.msg) {
+	case OTX_CPT_MSG_READY:
+		{
+			otx_cpt_chipid_vfid_t cid;
+
+			cid.u64 = mbx.data;
+			cptvf->pf_acked = true;
+			cptvf->vfid = cid.s.vfid;
+			CPT_LOG_DP_DEBUG("%s: Received VFID %d chip_id %d",
+					 cptvf->dev_name,
+					 cptvf->vfid, cid.s.chip_id);
+		}
+		break;
+	case OTX_CPT_MSG_QBIND_GRP:
+		cptvf->pf_acked = true;
+		cptvf->vftype = mbx.data;
+		CPT_LOG_DP_DEBUG("%s: VF %d type %s group %d",
+				 cptvf->dev_name, cptvf->vfid,
+				 ((mbx.data == SE_TYPE) ? "SE" : "AE"),
+				 cptvf->vfgrp);
+		break;
+	case OTX_CPT_MBOX_MSG_TYPE_ACK:
+		cptvf->pf_acked = true;
+		break;
+	case OTX_CPT_MBOX_MSG_TYPE_NACK:
+		cptvf->pf_nacked = true;
+		break;
+	default:
+		CPT_LOG_DP_DEBUG("%s: Invalid msg from PF, msg 0x%lx",
+				 cptvf->dev_name, (unsigned int long)mbx.msg);
+		break;
+	}
+}
+
+/* Send a mailbox message to PF
+ * @vf: vf from which this message to be sent
+ * @mbx: Message to be sent
+ */
+static void
+otx_cpt_send_msg_to_pf(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
+{
+	/* Writing mbox(1) causes interrupt */
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VFX_PF_MBOXX(0, 0, 0), mbx->msg);
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VFX_PF_MBOXX(0, 0, 1), mbx->data);
+}
+
+static int32_t
+otx_cpt_send_msg_to_pf_timeout(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
+{
+	int timeout = OTX_CPT_MBOX_MSG_TIMEOUT;
+	int sleep_ms = 10;
+
+	cptvf->pf_acked = false;
+	cptvf->pf_nacked = false;
+
+	otx_cpt_send_msg_to_pf(cptvf, mbx);
+
+	/* Wait for previous message to be acked, timeout 2sec */
+	while (!cptvf->pf_acked) {
+		if (cptvf->pf_nacked)
+			return -EINVAL;
+		usleep(sleep_ms * 1000);
+		otx_cpt_poll_misc(cptvf);
+		if (cptvf->pf_acked)
+			break;
+		timeout -= sleep_ms;
+		if (!timeout) {
+			CPT_LOG_ERR("%s: PF didn't ack mbox msg %lx(vfid %u)",
+				    cptvf->dev_name,
+				    (unsigned int long)(mbx->msg & 0xFF),
+				    cptvf->vfid);
+			return -EBUSY;
+		}
+	}
+	return 0;
+}
+
+int
+otx_cpt_check_pf_ready(struct cpt_vf *cptvf)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	mbx.msg = OTX_CPT_MSG_READY;
+	if (otx_cpt_send_msg_to_pf_timeout(cptvf, &mbx)) {
+		CPT_LOG_ERR("%s: PF didn't respond to READY msg",
+			    cptvf->dev_name);
+		return 1;
+	}
+	return 0;
+}
+
+int
+otx_cpt_send_vq_size_msg(struct cpt_vf *cptvf)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	mbx.msg = OTX_CPT_MSG_QLEN;
+
+	mbx.data = cptvf->qsize;
+	if (otx_cpt_send_msg_to_pf_timeout(cptvf, &mbx)) {
+		CPT_LOG_ERR("%s: PF didn't respond to vq_size msg",
+			    cptvf->dev_name);
+		return 1;
+	}
+	return 0;
+}
+
+int
+otx_cpt_send_vf_grp_msg(struct cpt_vf *cptvf, uint32_t group)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	mbx.msg = OTX_CPT_MSG_QBIND_GRP;
+
+	/* Convey group of the VF */
+	mbx.data = group;
+	if (otx_cpt_send_msg_to_pf_timeout(cptvf, &mbx)) {
+		CPT_LOG_ERR("%s: PF didn't respond to vf_type msg",
+			    cptvf->dev_name);
+		return 1;
+	}
+	return 0;
+}
+
+int
+otx_cpt_send_vf_up(struct cpt_vf *cptvf)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	mbx.msg = OTX_CPT_MSG_VF_UP;
+	if (otx_cpt_send_msg_to_pf_timeout(cptvf, &mbx)) {
+		CPT_LOG_ERR("%s: PF didn't respond to UP msg",
+			    cptvf->dev_name);
+		return 1;
+	}
+	return 0;
+}
+
+int
+otx_cpt_send_vf_down(struct cpt_vf *cptvf)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	mbx.msg = OTX_CPT_MSG_VF_DOWN;
+	if (otx_cpt_send_msg_to_pf_timeout(cptvf, &mbx)) {
+		CPT_LOG_ERR("%s: PF didn't respond to DOWN msg",
+			    cptvf->dev_name);
+		return 1;
+	}
+	return 0;
+}
diff --git a/drivers/crypto/octeontx/otx_cryptodev_mbox.h b/drivers/crypto/octeontx/otx_cryptodev_mbox.h
new file mode 100644
index 00000000..b05d1c50
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev_mbox.h
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _OTX_CRYPTODEV_MBOX_H_
+#define _OTX_CRYPTODEV_MBOX_H_
+
+#include <rte_byteorder.h>
+#include <rte_common.h>
+
+#include "cpt_common.h"
+#include "cpt_pmd_logs.h"
+
+#include "otx_cryptodev_hw_access.h"
+
+#define OTX_CPT_MBOX_MSG_TIMEOUT    2000 /* In Milli Seconds */
+
+#define OTX_CPT_MBOX_MSG_TYPE_REQ	0
+#define OTX_CPT_MBOX_MSG_TYPE_ACK	1
+#define OTX_CPT_MBOX_MSG_TYPE_NACK	2
+#define OTX_CPT_MBOX_MSG_TYPE_NOP	3
+
+/* CPT mailbox structure */
+struct cpt_mbox {
+	/** Message type MBOX[0] */
+	uint64_t msg;
+	/** Data         MBOX[1] */
+	uint64_t data;
+};
+
+typedef enum {
+	OTX_CPT_MSG_VF_UP = 1,
+	OTX_CPT_MSG_VF_DOWN,
+	OTX_CPT_MSG_READY,
+	OTX_CPT_MSG_QLEN,
+	OTX_CPT_MSG_QBIND_GRP,
+	OTX_CPT_MSG_VQ_PRIORITY,
+	OTX_CPT_MSG_PF_TYPE,
+} otx_cpt_mbox_opcode_t;
+
+typedef union {
+	uint64_t u64;
+	struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+		uint32_t chip_id;
+		uint8_t vfid;
+		uint8_t reserved[3];
+#else
+		uint8_t reserved[3];
+		uint8_t vfid;
+		uint32_t chip_id;
+#endif
+	} s;
+} otx_cpt_chipid_vfid_t;
+
+/* Poll handler to handle mailbox messages from VFs */
+void
+otx_cpt_handle_mbox_intr(struct cpt_vf *cptvf);
+
+/*
+ * Checks if VF is able to comminicate with PF
+ * and also gets the CPT number this VF is associated to.
+ */
+int
+otx_cpt_check_pf_ready(struct cpt_vf *cptvf);
+
+/*
+ * Communicate VQs size to PF to program CPT(0)_PF_Q(0-15)_CTL of the VF.
+ * Must be ACKed.
+ */
+int
+otx_cpt_send_vq_size_msg(struct cpt_vf *cptvf);
+
+/*
+ * Communicate VF group required to PF and get the VQ binded to that group
+ */
+int
+otx_cpt_send_vf_grp_msg(struct cpt_vf *cptvf, uint32_t group);
+
+/*
+ * Communicate to PF that VF is UP and running
+ */
+int
+otx_cpt_send_vf_up(struct cpt_vf *cptvf);
+
+/*
+ * Communicate to PF that VF is DOWN and running
+ */
+int
+otx_cpt_send_vf_down(struct cpt_vf *cptvf);
+
+#endif /* _OTX_CRYPTODEV_MBOX_H_ */
diff --git a/drivers/crypto/octeontx/otx_cryptodev_ops.c b/drivers/crypto/octeontx/otx_cryptodev_ops.c
new file mode 100644
index 00000000..23f96591
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev_ops.c
@@ -0,0 +1,531 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#include <rte_alarm.h>
+#include <rte_bus_pci.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_malloc.h>
+
+#include "cpt_pmd_logs.h"
+#include "cpt_pmd_ops_helper.h"
+#include "cpt_ucode.h"
+#include "cpt_request_mgr.h"
+
+#include "otx_cryptodev.h"
+#include "otx_cryptodev_capabilities.h"
+#include "otx_cryptodev_hw_access.h"
+#include "otx_cryptodev_ops.h"
+
+static int otx_cryptodev_probe_count;
+static rte_spinlock_t otx_probe_count_lock = RTE_SPINLOCK_INITIALIZER;
+
+static struct rte_mempool *otx_cpt_meta_pool;
+static int otx_cpt_op_mlen;
+static int otx_cpt_op_sb_mlen;
+
+/* Forward declarations */
+
+static int
+otx_cpt_que_pair_release(struct rte_cryptodev *dev, uint16_t que_pair_id);
+
+/*
+ * Initializes global variables used by fast-path code
+ *
+ * @return
+ *   - 0 on success, errcode on error
+ */
+static int
+init_global_resources(void)
+{
+	/* Get meta len for scatter gather mode */
+	otx_cpt_op_mlen = cpt_pmd_ops_helper_get_mlen_sg_mode();
+
+	/* Extra 4B saved for future considerations */
+	otx_cpt_op_mlen += 4 * sizeof(uint64_t);
+
+	otx_cpt_meta_pool = rte_mempool_create("cpt_metabuf-pool", 4096 * 16,
+					       otx_cpt_op_mlen, 512, 0,
+					       NULL, NULL, NULL, NULL,
+					       SOCKET_ID_ANY, 0);
+	if (!otx_cpt_meta_pool) {
+		CPT_LOG_ERR("cpt metabuf pool not created");
+		return -ENOMEM;
+	}
+
+	/* Get meta len for direct mode */
+	otx_cpt_op_sb_mlen = cpt_pmd_ops_helper_get_mlen_direct_mode();
+
+	/* Extra 4B saved for future considerations */
+	otx_cpt_op_sb_mlen += 4 * sizeof(uint64_t);
+
+	return 0;
+}
+
+void
+cleanup_global_resources(void)
+{
+	/* Take lock */
+	rte_spinlock_lock(&otx_probe_count_lock);
+
+	/* Decrement the cryptodev count */
+	otx_cryptodev_probe_count--;
+
+	/* Free buffers */
+	if (otx_cpt_meta_pool && otx_cryptodev_probe_count == 0)
+		rte_mempool_free(otx_cpt_meta_pool);
+
+	/* Free lock */
+	rte_spinlock_unlock(&otx_probe_count_lock);
+}
+
+/* Alarm routines */
+
+static void
+otx_cpt_alarm_cb(void *arg)
+{
+	struct cpt_vf *cptvf = arg;
+	otx_cpt_poll_misc(cptvf);
+	rte_eal_alarm_set(CPT_INTR_POLL_INTERVAL_MS * 1000,
+			  otx_cpt_alarm_cb, cptvf);
+}
+
+static int
+otx_cpt_periodic_alarm_start(void *arg)
+{
+	return rte_eal_alarm_set(CPT_INTR_POLL_INTERVAL_MS * 1000,
+				 otx_cpt_alarm_cb, arg);
+}
+
+static int
+otx_cpt_periodic_alarm_stop(void *arg)
+{
+	return rte_eal_alarm_cancel(otx_cpt_alarm_cb, arg);
+}
+
+/* PMD ops */
+
+static int
+otx_cpt_dev_config(struct rte_cryptodev *dev __rte_unused,
+		   struct rte_cryptodev_config *config __rte_unused)
+{
+	CPT_PMD_INIT_FUNC_TRACE();
+	return 0;
+}
+
+static int
+otx_cpt_dev_start(struct rte_cryptodev *c_dev)
+{
+	void *cptvf = c_dev->data->dev_private;
+
+	CPT_PMD_INIT_FUNC_TRACE();
+
+	return otx_cpt_start_device(cptvf);
+}
+
+static void
+otx_cpt_dev_stop(struct rte_cryptodev *c_dev)
+{
+	void *cptvf = c_dev->data->dev_private;
+
+	CPT_PMD_INIT_FUNC_TRACE();
+
+	otx_cpt_stop_device(cptvf);
+}
+
+static int
+otx_cpt_dev_close(struct rte_cryptodev *c_dev)
+{
+	void *cptvf = c_dev->data->dev_private;
+	int i, ret;
+
+	CPT_PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < c_dev->data->nb_queue_pairs; i++) {
+		ret = otx_cpt_que_pair_release(c_dev, i);
+		if (ret)
+			return ret;
+	}
+
+	otx_cpt_periodic_alarm_stop(cptvf);
+	otx_cpt_deinit_device(cptvf);
+
+	return 0;
+}
+
+static void
+otx_cpt_dev_info_get(struct rte_cryptodev *dev, struct rte_cryptodev_info *info)
+{
+	CPT_PMD_INIT_FUNC_TRACE();
+	if (info != NULL) {
+		info->max_nb_queue_pairs = CPT_NUM_QS_PER_VF;
+		info->feature_flags = dev->feature_flags;
+		info->capabilities = otx_get_capabilities();
+		info->sym.max_nb_sessions = 0;
+		info->driver_id = otx_cryptodev_driver_id;
+		info->min_mbuf_headroom_req = OTX_CPT_MIN_HEADROOM_REQ;
+		info->min_mbuf_tailroom_req = OTX_CPT_MIN_TAILROOM_REQ;
+	}
+}
+
+static void
+otx_cpt_stats_get(struct rte_cryptodev *dev __rte_unused,
+		  struct rte_cryptodev_stats *stats __rte_unused)
+{
+	CPT_PMD_INIT_FUNC_TRACE();
+}
+
+static void
+otx_cpt_stats_reset(struct rte_cryptodev *dev __rte_unused)
+{
+	CPT_PMD_INIT_FUNC_TRACE();
+}
+
+static int
+otx_cpt_que_pair_setup(struct rte_cryptodev *dev,
+		       uint16_t que_pair_id,
+		       const struct rte_cryptodev_qp_conf *qp_conf,
+		       int socket_id __rte_unused,
+		       struct rte_mempool *session_pool __rte_unused)
+{
+	void *cptvf = dev->data->dev_private;
+	struct cpt_instance *instance = NULL;
+	struct rte_pci_device *pci_dev;
+	int ret = -1;
+
+	CPT_PMD_INIT_FUNC_TRACE();
+
+	if (dev->data->queue_pairs[que_pair_id] != NULL) {
+		ret = otx_cpt_que_pair_release(dev, que_pair_id);
+		if (ret)
+			return ret;
+	}
+
+	if (qp_conf->nb_descriptors > DEFAULT_CMD_QLEN) {
+		CPT_LOG_INFO("Number of descriptors too big %d, using default "
+			     "queue length of %d", qp_conf->nb_descriptors,
+			     DEFAULT_CMD_QLEN);
+	}
+
+	pci_dev = RTE_DEV_TO_PCI(dev->device);
+
+	if (pci_dev->mem_resource[0].addr == NULL) {
+		CPT_LOG_ERR("PCI mem address null");
+		return -EIO;
+	}
+
+	ret = otx_cpt_get_resource(cptvf, 0, &instance);
+	if (ret != 0) {
+		CPT_LOG_ERR("Error getting instance handle from device %s : "
+			    "ret = %d", dev->data->name, ret);
+		return ret;
+	}
+
+	instance->queue_id = que_pair_id;
+	dev->data->queue_pairs[que_pair_id] = instance;
+
+	return 0;
+}
+
+static int
+otx_cpt_que_pair_release(struct rte_cryptodev *dev, uint16_t que_pair_id)
+{
+	struct cpt_instance *instance = dev->data->queue_pairs[que_pair_id];
+	int ret;
+
+	CPT_PMD_INIT_FUNC_TRACE();
+
+	ret = otx_cpt_put_resource(instance);
+	if (ret != 0) {
+		CPT_LOG_ERR("Error putting instance handle of device %s : "
+			    "ret = %d", dev->data->name, ret);
+		return ret;
+	}
+
+	dev->data->queue_pairs[que_pair_id] = NULL;
+
+	return 0;
+}
+
+static unsigned int
+otx_cpt_get_session_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return cpt_get_session_size();
+}
+
+static void
+otx_cpt_session_init(void *sym_sess, uint8_t driver_id)
+{
+	struct rte_cryptodev_sym_session *sess = sym_sess;
+	struct cpt_sess_misc *cpt_sess =
+	 (struct cpt_sess_misc *) get_sym_session_private_data(sess, driver_id);
+
+	CPT_PMD_INIT_FUNC_TRACE();
+	cpt_sess->ctx_dma_addr = rte_mempool_virt2iova(cpt_sess) +
+			sizeof(struct cpt_sess_misc);
+}
+
+static int
+otx_cpt_session_cfg(struct rte_cryptodev *dev,
+		    struct rte_crypto_sym_xform *xform,
+		    struct rte_cryptodev_sym_session *sess,
+		    struct rte_mempool *mempool)
+{
+	struct rte_crypto_sym_xform *chain;
+	void *sess_private_data = NULL;
+
+	CPT_PMD_INIT_FUNC_TRACE();
+
+	if (cpt_is_algo_supported(xform))
+		goto err;
+
+	if (unlikely(sess == NULL)) {
+		CPT_LOG_ERR("invalid session struct");
+		return -EINVAL;
+	}
+
+	if (rte_mempool_get(mempool, &sess_private_data)) {
+		CPT_LOG_ERR("Could not allocate sess_private_data");
+		return -ENOMEM;
+	}
+
+	chain = xform;
+	while (chain) {
+		switch (chain->type) {
+		case RTE_CRYPTO_SYM_XFORM_AEAD:
+			if (fill_sess_aead(chain, sess_private_data))
+				goto err;
+			break;
+		case RTE_CRYPTO_SYM_XFORM_CIPHER:
+			if (fill_sess_cipher(chain, sess_private_data))
+				goto err;
+			break;
+		case RTE_CRYPTO_SYM_XFORM_AUTH:
+			if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
+				if (fill_sess_gmac(chain, sess_private_data))
+					goto err;
+			} else {
+				if (fill_sess_auth(chain, sess_private_data))
+					goto err;
+			}
+			break;
+		default:
+			CPT_LOG_ERR("Invalid crypto xform type");
+			break;
+		}
+		chain = chain->next;
+	}
+	set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
+	otx_cpt_session_init(sess, dev->driver_id);
+	return 0;
+
+err:
+	if (sess_private_data)
+		rte_mempool_put(mempool, sess_private_data);
+	return -EPERM;
+}
+
+static void
+otx_cpt_session_clear(struct rte_cryptodev *dev,
+		  struct rte_cryptodev_sym_session *sess)
+{
+	void *sess_priv = get_sym_session_private_data(sess, dev->driver_id);
+
+	CPT_PMD_INIT_FUNC_TRACE();
+	if (sess_priv) {
+		memset(sess_priv, 0, otx_cpt_get_session_size(dev));
+		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+		set_sym_session_private_data(sess, dev->driver_id, NULL);
+		rte_mempool_put(sess_mp, sess_priv);
+	}
+}
+
+static uint16_t
+otx_cpt_pkt_enqueue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+	struct cpt_instance *instance = (struct cpt_instance *)qptr;
+	uint16_t count = 0;
+	int ret;
+	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
+	struct pending_queue *pqueue = &cptvf->pqueue;
+
+	count = DEFAULT_CMD_QLEN - pqueue->pending_count;
+	if (nb_ops > count)
+		nb_ops = count;
+
+	count = 0;
+	while (likely(count < nb_ops)) {
+		ret = cpt_pmd_crypto_operation(instance, ops[count], pqueue,
+						otx_cryptodev_driver_id);
+		if (unlikely(ret))
+			break;
+		count++;
+	}
+	otx_cpt_ring_dbell(instance, count);
+	return count;
+}
+
+static uint16_t
+otx_cpt_pkt_dequeue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+	struct cpt_instance *instance = (struct cpt_instance *)qptr;
+	struct cpt_vf *cptvf = (struct cpt_vf *)instance;
+	struct pending_queue *pqueue = &cptvf->pqueue;
+	uint16_t nb_completed, i = 0;
+	uint8_t compcode[nb_ops];
+
+	nb_completed = cpt_dequeue_burst(instance, nb_ops,
+					 (void **)ops, compcode, pqueue);
+	while (likely(i < nb_completed)) {
+		struct rte_crypto_op *cop;
+		void *metabuf;
+		uintptr_t *rsp;
+		uint8_t status;
+
+		rsp = (void *)ops[i];
+		status = compcode[i];
+		if (likely((i + 1) < nb_completed))
+			rte_prefetch0(ops[i+1]);
+		metabuf = (void *)rsp[0];
+		cop = (void *)rsp[1];
+
+		ops[i] = cop;
+
+		if (likely(status == 0)) {
+			if (likely(!rsp[2]))
+				cop->status =
+					RTE_CRYPTO_OP_STATUS_SUCCESS;
+			else
+				compl_auth_verify(cop, (uint8_t *)rsp[2],
+						  rsp[3]);
+		} else if (status == ERR_GC_ICV_MISCOMPARE) {
+			/*auth data mismatch */
+			cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+		} else {
+			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		}
+		free_op_meta(metabuf, cptvf->meta_info.cptvf_meta_pool);
+		i++;
+	}
+	return nb_completed;
+}
+
+static struct rte_cryptodev_ops cptvf_ops = {
+	/* Device related operations */
+	.dev_configure = otx_cpt_dev_config,
+	.dev_start = otx_cpt_dev_start,
+	.dev_stop = otx_cpt_dev_stop,
+	.dev_close = otx_cpt_dev_close,
+	.dev_infos_get = otx_cpt_dev_info_get,
+
+	.stats_get = otx_cpt_stats_get,
+	.stats_reset = otx_cpt_stats_reset,
+	.queue_pair_setup = otx_cpt_que_pair_setup,
+	.queue_pair_release = otx_cpt_que_pair_release,
+	.queue_pair_count = NULL,
+
+	/* Crypto related operations */
+	.sym_session_get_size = otx_cpt_get_session_size,
+	.sym_session_configure = otx_cpt_session_cfg,
+	.sym_session_clear = otx_cpt_session_clear
+};
+
+static void
+otx_cpt_common_vars_init(struct cpt_vf *cptvf)
+{
+	cptvf->meta_info.cptvf_meta_pool = otx_cpt_meta_pool;
+	cptvf->meta_info.cptvf_op_mlen = otx_cpt_op_mlen;
+	cptvf->meta_info.cptvf_op_sb_mlen = otx_cpt_op_sb_mlen;
+}
+
+int
+otx_cpt_dev_create(struct rte_cryptodev *c_dev)
+{
+	struct rte_pci_device *pdev = RTE_DEV_TO_PCI(c_dev->device);
+	struct cpt_vf *cptvf = NULL;
+	void *reg_base;
+	char dev_name[32];
+	int ret;
+
+	if (pdev->mem_resource[0].phys_addr == 0ULL)
+		return -EIO;
+
+	/* for secondary processes, we don't initialise any further as primary
+	 * has already done this work.
+	 */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	cptvf = rte_zmalloc_socket("otx_cryptodev_private_mem",
+			sizeof(struct cpt_vf), RTE_CACHE_LINE_SIZE,
+			rte_socket_id());
+
+	if (cptvf == NULL) {
+		CPT_LOG_ERR("Cannot allocate memory for device private data");
+		return -ENOMEM;
+	}
+
+	snprintf(dev_name, 32, "%02x:%02x.%x",
+			pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
+
+	reg_base = pdev->mem_resource[0].addr;
+	if (!reg_base) {
+		CPT_LOG_ERR("Failed to map BAR0 of %s", dev_name);
+		ret = -ENODEV;
+		goto fail;
+	}
+
+	ret = otx_cpt_hw_init(cptvf, pdev, reg_base, dev_name);
+	if (ret) {
+		CPT_LOG_ERR("Failed to init cptvf %s", dev_name);
+		ret = -EIO;
+		goto fail;
+	}
+
+	/* Start off timer for mailbox interrupts */
+	otx_cpt_periodic_alarm_start(cptvf);
+
+	rte_spinlock_lock(&otx_probe_count_lock);
+	if (!otx_cryptodev_probe_count) {
+		ret = init_global_resources();
+		if (ret) {
+			rte_spinlock_unlock(&otx_probe_count_lock);
+			goto init_fail;
+		}
+	}
+	otx_cryptodev_probe_count++;
+	rte_spinlock_unlock(&otx_probe_count_lock);
+
+	/* Initialize data path variables used by common code */
+	otx_cpt_common_vars_init(cptvf);
+
+	c_dev->dev_ops = &cptvf_ops;
+
+	c_dev->enqueue_burst = otx_cpt_pkt_enqueue;
+	c_dev->dequeue_burst = otx_cpt_pkt_dequeue;
+
+	c_dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+			RTE_CRYPTODEV_FF_HW_ACCELERATED |
+			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
+			RTE_CRYPTODEV_FF_IN_PLACE_SGL |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT;
+
+	/* Save dev private data */
+	c_dev->data->dev_private = cptvf;
+
+	return 0;
+
+init_fail:
+	otx_cpt_periodic_alarm_stop(cptvf);
+	otx_cpt_deinit_device(cptvf);
+
+fail:
+	if (cptvf) {
+		/* Free private data allocated */
+		rte_free(cptvf);
+	}
+
+	return ret;
+}
diff --git a/drivers/crypto/octeontx/otx_cryptodev_ops.h b/drivers/crypto/octeontx/otx_cryptodev_ops.h
new file mode 100644
index 00000000..b3efecf0
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev_ops.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _OTX_CRYPTODEV_OPS_H_
+#define _OTX_CRYPTODEV_OPS_H_
+
+#define OTX_CPT_MIN_HEADROOM_REQ	(24)
+#define OTX_CPT_MIN_TAILROOM_REQ	(8)
+#define CPT_NUM_QS_PER_VF		(1)
+
+void
+cleanup_global_resources(void);
+
+int
+otx_cpt_dev_create(struct rte_cryptodev *c_dev);
+
+#endif /* _OTX_CRYPTODEV_OPS_H_ */
diff --git a/drivers/crypto/octeontx/rte_pmd_octeontx_crypto_version.map b/drivers/crypto/octeontx/rte_pmd_octeontx_crypto_version.map
new file mode 100644
index 00000000..521e51f4
--- /dev/null
+++ b/drivers/crypto/octeontx/rte_pmd_octeontx_crypto_version.map
@@ -0,0 +1,4 @@
+DPDK_18.11 {
+
+	local: *;
+};
diff --git a/drivers/crypto/openssl/compat.h b/drivers/crypto/openssl/compat.h
index 45f9a33d..eecb7d36 100644
--- a/drivers/crypto/openssl/compat.h
+++ b/drivers/crypto/openssl/compat.h
@@ -7,101 +7,190 @@
 
 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
 
-#define set_rsa_params(rsa, p, q, ret) \
-	do {rsa->p = p; rsa->q = q; ret = 0; } while (0)
-
-#define set_rsa_crt_params(rsa, dmp1, dmq1, iqmp, ret) \
-	do { \
-		rsa->dmp1 = dmp1; \
-		rsa->dmq1 = dmq1; \
-		rsa->iqmp = iqmp; \
-		ret = 0; \
-	} while (0)
-
-#define set_rsa_keys(rsa, n, e, d, ret) \
-	do { \
-		rsa->n = n; rsa->e = e; rsa->d = d; ret = 0; \
-	} while (0)
-
-#define set_dh_params(dh, p, g, ret) \
-	do { \
-		dh->p = p; \
-		dh->q = NULL; \
-		dh->g = g; \
-		ret = 0; \
-	} while (0)
-
-#define set_dh_priv_key(dh, priv_key, ret) \
-	do { dh->priv_key = priv_key; ret = 0; } while (0)
-
-#define set_dsa_params(dsa, p, q, g, ret) \
-	do { dsa->p = p; dsa->q = q; dsa->g = g; ret = 0; } while (0)
-
-#define get_dh_pub_key(dh, pub_key) \
-	(pub_key = dh->pub_key)
-
-#define get_dh_priv_key(dh, priv_key) \
-	(priv_key = dh->priv_key)
-
-#define set_dsa_sign(sign, r, s) \
-	do { sign->r = r; sign->s = s; } while (0)
-
-#define get_dsa_sign(sign, r, s) \
-	do { r = sign->r; s = sign->s; } while (0)
-
-#define set_dsa_keys(dsa, pub, priv, ret) \
-	do { dsa->pub_key = pub; dsa->priv_key = priv; ret = 0; } while (0)
-
-#define set_dsa_pub_key(dsa, pub_key) \
-	(dsa->pub_key = pub_key)
-
-#define get_dsa_priv_key(dsa, priv_key) \
-	(priv_key = dsa->priv_key)
+static __rte_always_inline int
+set_rsa_params(RSA *rsa, BIGNUM *p, BIGNUM *q)
+{
+	rsa->p = p;
+	rsa->q = q;
+	return 0;
+}
+
+static __rte_always_inline int
+set_rsa_crt_params(RSA *rsa, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp)
+{
+	rsa->dmp1 = dmp1;
+	rsa->dmq1 = dmq1;
+	rsa->iqmp = iqmp;
+	return 0;
+}
+
+static __rte_always_inline int
+set_rsa_keys(RSA *rsa, BIGNUM *n, BIGNUM *e, BIGNUM *d)
+{
+	rsa->n = n;
+	rsa->e = e;
+	rsa->d = d;
+	return 0;
+}
+
+static __rte_always_inline int
+set_dh_params(DH *dh, BIGNUM *p, BIGNUM *g)
+{
+	dh->p = p;
+	dh->q = NULL;
+	dh->g = g;
+	return 0;
+}
+
+static __rte_always_inline int
+set_dh_priv_key(DH *dh, BIGNUM *priv_key)
+{
+	dh->priv_key = priv_key;
+	return 0;
+}
+
+static __rte_always_inline int
+set_dsa_params(DSA *dsa, BIGNUM *p, BIGNUM *q, BIGNUM *g)
+{
+	dsa->p = p;
+	dsa->q = q;
+	dsa->g = g;
+	return 0;
+}
+
+static __rte_always_inline void
+get_dh_pub_key(DH *dh, const BIGNUM **pub_key)
+{
+	*pub_key = dh->pub_key;
+}
+
+static __rte_always_inline void
+get_dh_priv_key(DH *dh, const BIGNUM **priv_key)
+{
+	*priv_key = dh->priv_key;
+}
+
+static __rte_always_inline void
+set_dsa_sign(DSA_SIG *sign, BIGNUM *r, BIGNUM *s)
+{
+	sign->r = r;
+	sign->s = s;
+}
+
+static __rte_always_inline void
+get_dsa_sign(DSA_SIG *sign, const BIGNUM **r, const BIGNUM **s)
+{
+	*r = sign->r;
+	*s = sign->s;
+}
+
+static __rte_always_inline int
+set_dsa_keys(DSA *dsa, BIGNUM *pub, BIGNUM *priv)
+{
+	dsa->pub_key = pub;
+	dsa->priv_key = priv;
+	return 0;
+}
+
+static __rte_always_inline void
+set_dsa_pub_key(DSA *dsa, BIGNUM *pub)
+{
+	dsa->pub_key = pub;
+}
+
+static __rte_always_inline void
+get_dsa_priv_key(DSA *dsa, BIGNUM **priv_key)
+{
+	*priv_key = dsa->priv_key;
+}
 
 #else
 
-#define set_rsa_params(rsa, p, q, ret) \
-	(ret = !RSA_set0_factors(rsa, p, q))
+static __rte_always_inline int
+set_rsa_params(RSA *rsa, BIGNUM *p, BIGNUM *q)
+{
+	return !(RSA_set0_factors(rsa, p, q));
+}
 
-#define set_rsa_crt_params(rsa, dmp1, dmq1, iqmp, ret) \
-	(ret = !RSA_set0_crt_params(rsa, dmp1, dmq1, iqmp))
+static __rte_always_inline int
+set_rsa_crt_params(RSA *rsa, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp)
+{
+	return !(RSA_set0_crt_params(rsa, dmp1, dmq1, iqmp));
+}
 
 /* n, e must be non-null, d can be NULL */
-#define set_rsa_keys(rsa, n, e, d, ret) \
-	(ret = !RSA_set0_key(rsa, n, e, d))
-
-#define set_dh_params(dh, p, g, ret) \
-	(ret = !DH_set0_pqg(dh, p, NULL, g))
-
-#define set_dh_priv_key(dh, priv_key, ret) \
-	(ret = !DH_set0_key(dh, NULL, priv_key))
-
-#define get_dh_pub_key(dh, pub_key) \
-	(DH_get0_key(dh_key, &pub_key, NULL))
-
-#define get_dh_priv_key(dh, priv_key) \
-	(DH_get0_key(dh_key, NULL, &priv_key))
-
-#define set_dsa_params(dsa, p, q, g, ret) \
-	(ret = !DSA_set0_pqg(dsa, p, q, g))
-
-#define set_dsa_priv_key(dsa, priv_key) \
-	(DSA_set0_key(dsa, NULL, priv_key))
-
-#define set_dsa_sign(sign, r, s) \
-	(DSA_SIG_set0(sign, r, s))
-
-#define get_dsa_sign(sign, r, s) \
-	(DSA_SIG_get0(sign, &r, &s))
-
-#define set_dsa_keys(dsa, pub, priv, ret) \
-	(ret = !DSA_set0_key(dsa, pub, priv))
-
-#define set_dsa_pub_key(dsa, pub_key) \
-	(DSA_set0_key(dsa, pub_key, NULL))
 
-#define get_dsa_priv_key(dsa, priv_key) \
-	(DSA_get0_key(dsa, NULL, &priv_key))
+static __rte_always_inline  int
+set_rsa_keys(RSA *rsa, BIGNUM *n, BIGNUM *e, BIGNUM *d)
+{
+	return !(RSA_set0_key(rsa, n, e, d));
+}
+
+static __rte_always_inline int
+set_dh_params(DH *dh, BIGNUM *p, BIGNUM *g)
+{
+	return !(DH_set0_pqg(dh, p, NULL, g));
+}
+
+static __rte_always_inline int
+set_dh_priv_key(DH *dh, BIGNUM *priv_key)
+{
+	return !(DH_set0_key(dh, NULL, priv_key));
+}
+
+static __rte_always_inline void
+get_dh_pub_key(DH *dh_key, const BIGNUM **pub_key)
+{
+	DH_get0_key(dh_key, pub_key, NULL);
+}
+
+static __rte_always_inline void
+get_dh_priv_key(DH *dh_key, const BIGNUM **priv_key)
+{
+	DH_get0_key(dh_key, NULL, priv_key);
+}
+
+static __rte_always_inline int
+set_dsa_params(DSA *dsa, BIGNUM *p, BIGNUM *q, BIGNUM *g)
+{
+	return !(DSA_set0_pqg(dsa, p, q, g));
+}
+
+static __rte_always_inline void
+set_dsa_priv_key(DSA *dsa, BIGNUM *priv_key)
+{
+	DSA_set0_key(dsa, NULL, priv_key);
+}
+
+static __rte_always_inline void
+set_dsa_sign(DSA_SIG *sign, BIGNUM *r, BIGNUM *s)
+{
+	DSA_SIG_set0(sign, r, s);
+}
+
+static __rte_always_inline void
+get_dsa_sign(DSA_SIG *sign, const BIGNUM **r, const BIGNUM **s)
+{
+	DSA_SIG_get0(sign, r, s);
+}
+
+static __rte_always_inline int
+set_dsa_keys(DSA *dsa, BIGNUM *pub, BIGNUM *priv)
+{
+	return !(DSA_set0_key(dsa, pub, priv));
+}
+
+static __rte_always_inline void
+set_dsa_pub_key(DSA *dsa, BIGNUM *pub_key)
+{
+	DSA_set0_key(dsa, pub_key, NULL);
+}
+
+static __rte_always_inline void
+get_dsa_priv_key(DSA *dsa, const BIGNUM **priv_key)
+{
+	DSA_get0_key(dsa, NULL, priv_key);
+}
 
 #endif /* version < 10100000 */
 
diff --git a/drivers/crypto/openssl/rte_openssl_pmd.c b/drivers/crypto/openssl/rte_openssl_pmd.c
index 7d263aba..003116dc 100644
--- a/drivers/crypto/openssl/rte_openssl_pmd.c
+++ b/drivers/crypto/openssl/rte_openssl_pmd.c
@@ -1509,15 +1509,7 @@ process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
 
 	srclen = op->sym->auth.data.length;
 
-	if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
-		dst = qp->temp_digest;
-	else {
-		dst = op->sym->auth.digest.data;
-		if (dst == NULL)
-			dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
-					op->sym->auth.data.offset +
-					op->sym->auth.data.length);
-	}
+	dst = qp->temp_digest;
 
 	switch (sess->auth.mode) {
 	case OPENSSL_AUTH_AS_AUTH:
@@ -1540,6 +1532,15 @@ process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
 				sess->auth.digest_length) != 0) {
 			op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 		}
+	} else {
+		uint8_t *auth_dst;
+
+		auth_dst = op->sym->auth.digest.data;
+		if (auth_dst == NULL)
+			auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
+					op->sym->auth.data.offset +
+					op->sym->auth.data.length);
+		memcpy(auth_dst, dst, sess->auth.digest_length);
 	}
 
 	if (status != 0)
@@ -1564,7 +1565,7 @@ process_openssl_dsa_sign_op(struct rte_crypto_op *cop,
 		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 	} else {
 		const BIGNUM *r = NULL, *s = NULL;
-		get_dsa_sign(sign, r, s);
+		get_dsa_sign(sign, &r, &s);
 
 		op->r.length = BN_bn2bin(r, op->r.data);
 		op->s.length = BN_bn2bin(s, op->s.data);
@@ -1666,7 +1667,7 @@ process_openssl_dh_op(struct rte_crypto_op *cop,
 			cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 			return -1;
 		}
-		set_dh_priv_key(dh_key, priv_key, ret);
+		ret = set_dh_priv_key(dh_key, priv_key);
 		if (ret) {
 			OPENSSL_LOG(ERR, "Failed to set private key\n");
 			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
@@ -1715,7 +1716,7 @@ process_openssl_dh_op(struct rte_crypto_op *cop,
 			cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 			return -1;
 		}
-		set_dh_priv_key(dh_key, priv_key, ret);
+		ret = set_dh_priv_key(dh_key, priv_key);
 		if (ret) {
 			OPENSSL_LOG(ERR, "Failed to set private key\n");
 			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
@@ -1743,7 +1744,7 @@ process_openssl_dh_op(struct rte_crypto_op *cop,
 				__func__, __LINE__);
 
 		/* get the generated keys */
-		get_dh_pub_key(dh_key, pub_key);
+		get_dh_pub_key(dh_key, &pub_key);
 
 		/* output public key */
 		op->pub_key.length = BN_bn2bin(pub_key,
@@ -1758,7 +1759,7 @@ process_openssl_dh_op(struct rte_crypto_op *cop,
 				__func__, __LINE__);
 
 		/* get the generated keys */
-		get_dh_priv_key(dh_key, priv_key);
+		get_dh_priv_key(dh_key, &priv_key);
 
 		/* provide generated private key back to user */
 		op->priv_key.length = BN_bn2bin(priv_key,
diff --git a/drivers/crypto/openssl/rte_openssl_pmd_ops.c b/drivers/crypto/openssl/rte_openssl_pmd_ops.c
index de228439..c2b029ec 100644
--- a/drivers/crypto/openssl/rte_openssl_pmd_ops.c
+++ b/drivers/crypto/openssl/rte_openssl_pmd_ops.c
@@ -26,9 +26,9 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = {
 					.increment = 1
 				},
 				.digest_size = {
-					.min = 16,
+					.min = 1,
 					.max = 16,
-					.increment = 0
+					.increment = 1
 				},
 				.iv_size = { 0 }
 			}, }
@@ -68,9 +68,9 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = {
 					.increment = 1
 				},
 				.digest_size = {
-					.min = 20,
+					.min = 1,
 					.max = 20,
-					.increment = 0
+					.increment = 1
 				},
 				.iv_size = { 0 }
 			}, }
@@ -110,9 +110,9 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = {
 					.increment = 1
 				},
 				.digest_size = {
-					.min = 28,
+					.min = 1,
 					.max = 28,
-					.increment = 0
+					.increment = 1
 				},
 				.iv_size = { 0 }
 			}, }
@@ -131,9 +131,9 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = {
 					.increment = 0
 				},
 				.digest_size = {
-					.min = 28,
+					.min = 1,
 					.max = 28,
-					.increment = 0
+					.increment = 1
 				},
 				.iv_size = { 0 }
 			}, }
@@ -152,9 +152,9 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = {
 					.increment = 1
 				},
 				.digest_size = {
-					.min = 32,
+					.min = 1,
 					.max = 32,
-					.increment = 0
+					.increment = 1
 				},
 				.iv_size = { 0 }
 			}, }
@@ -194,9 +194,9 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = {
 					.increment = 1
 				},
 				.digest_size = {
-					.min = 48,
+					.min = 1,
 					.max = 48,
-					.increment = 0
+					.increment = 1
 				},
 				.iv_size = { 0 }
 			}, }
@@ -236,9 +236,9 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = {
 					.increment = 1
 				},
 				.digest_size = {
-					.min = 64,
+					.min = 1,
 					.max = 64,
-					.increment = 0
+					.increment = 1
 				},
 				.iv_size = { 0 }
 			}, }
@@ -875,14 +875,14 @@ static int openssl_set_asym_session_parameters(
 				RSA_free(rsa);
 				goto err_rsa;
 			}
-			set_rsa_params(rsa, p, q, ret);
+			ret = set_rsa_params(rsa, p, q);
 			if (ret) {
 				OPENSSL_LOG(ERR,
 					"failed to set rsa params\n");
 				RSA_free(rsa);
 				goto err_rsa;
 			}
-			set_rsa_crt_params(rsa, dmp1, dmq1, iqmp, ret);
+			ret = set_rsa_crt_params(rsa, dmp1, dmq1, iqmp);
 			if (ret) {
 				OPENSSL_LOG(ERR,
 					"failed to set crt params\n");
@@ -896,7 +896,7 @@ static int openssl_set_asym_session_parameters(
 			}
 		}
 
-		set_rsa_keys(rsa, n, e, d, ret);
+		ret = set_rsa_keys(rsa, n, e, d);
 		if (ret) {
 			OPENSSL_LOG(ERR, "Failed to load rsa keys\n");
 			RSA_free(rsa);
@@ -1005,7 +1005,7 @@ err_rsa:
 				"failed to allocate resources\n");
 			goto err_dh;
 		}
-		set_dh_params(dh, p, g, ret);
+		ret = set_dh_params(dh, p, g);
 		if (ret) {
 			DH_free(dh);
 			goto err_dh;
@@ -1087,7 +1087,7 @@ err_dh:
 			goto err_dsa;
 		}
 
-		set_dsa_params(dsa, p, q, g, ret);
+		ret = set_dsa_params(dsa, p, q, g);
 		if (ret) {
 			DSA_free(dsa);
 			OPENSSL_LOG(ERR, "Failed to dsa params\n");
@@ -1101,7 +1101,7 @@ err_dh:
 		 * both versions
 		 */
 		/* just set dummy public for very 1st call */
-		set_dsa_keys(dsa, pub_key, priv_key, ret);
+		ret = set_dsa_keys(dsa, pub_key, priv_key);
 		if (ret) {
 			DSA_free(dsa);
 			OPENSSL_LOG(ERR, "Failed to set keys\n");
diff --git a/drivers/crypto/qat/qat_sym_capabilities.h b/drivers/crypto/qat/qat_sym_capabilities.h
index eea08bc7..7cba87d6 100644
--- a/drivers/crypto/qat/qat_sym_capabilities.h
+++ b/drivers/crypto/qat/qat_sym_capabilities.h
@@ -154,6 +154,26 @@
 			}, }						\
 		}, }							\
 	},								\
+	{	/* AES CMAC */						\
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,			\
+		{.sym = {						\
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,	\
+			{.auth = {					\
+				.algo = RTE_CRYPTO_AUTH_AES_CMAC,	\
+				.block_size = 16,			\
+				.key_size = {				\
+					.min = 16,			\
+					.max = 16,			\
+					.increment = 0			\
+				},					\
+				.digest_size = {			\
+					.min = 12,			\
+					.max = 16,			\
+					.increment = 4			\
+				}					\
+			}, }						\
+		}, }							\
+	},								\
 	{	/* AES CCM */						\
 		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,			\
 		{.sym = {						\
diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c
index 96f442e8..c3f70040 100644
--- a/drivers/crypto/qat/qat_sym_pmd.c
+++ b/drivers/crypto/qat/qat_sym_pmd.c
@@ -290,6 +290,7 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev)
 		internals->qat_dev_capabilities = qat_gen1_sym_capabilities;
 		break;
 	case QAT_GEN2:
+	case QAT_GEN3:
 		internals->qat_dev_capabilities = qat_gen2_sym_capabilities;
 		break;
 	default:
diff --git a/drivers/crypto/qat/qat_sym_pmd.h b/drivers/crypto/qat/qat_sym_pmd.h
index d3432854..5563d5be 100644
--- a/drivers/crypto/qat/qat_sym_pmd.h
+++ b/drivers/crypto/qat/qat_sym_pmd.h
@@ -12,7 +12,7 @@
 #include "qat_sym_capabilities.h"
 #include "qat_device.h"
 
-/**< Intel(R) QAT Symmetric Crypto PMD device name */
+/** Intel(R) QAT Symmetric Crypto PMD driver name */
 #define CRYPTODEV_NAME_QAT_SYM_PMD	crypto_qat
 
 extern uint8_t cryptodev_qat_driver_id;
diff --git a/drivers/crypto/qat/qat_sym_session.c b/drivers/crypto/qat/qat_sym_session.c
index 1d58220a..8196e233 100644
--- a/drivers/crypto/qat/qat_sym_session.c
+++ b/drivers/crypto/qat/qat_sym_session.c
@@ -498,6 +498,7 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev,
 	struct qat_sym_dev_private *internals = dev->data->dev_private;
 	uint8_t *key_data = auth_xform->key.data;
 	uint8_t key_length = auth_xform->key.length;
+	session->aes_cmac = 0;
 
 	switch (auth_xform->algo) {
 	case RTE_CRYPTO_AUTH_SHA1_HMAC:
@@ -518,6 +519,10 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev,
 	case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
 		break;
+	case RTE_CRYPTO_AUTH_AES_CMAC:
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
+		session->aes_cmac = 1;
+		break;
 	case RTE_CRYPTO_AUTH_AES_GMAC:
 		if (qat_sym_validate_aes_key(auth_xform->key.length,
 				&session->qat_cipher_alg) != 0) {
@@ -555,7 +560,6 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev,
 	case RTE_CRYPTO_AUTH_SHA224:
 	case RTE_CRYPTO_AUTH_SHA384:
 	case RTE_CRYPTO_AUTH_MD5:
-	case RTE_CRYPTO_AUTH_AES_CMAC:
 	case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 		QAT_LOG(ERR, "Crypto: Unsupported hash alg %u",
 				auth_xform->algo);
@@ -817,6 +821,8 @@ static int qat_hash_get_digest_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 		return ICP_QAT_HW_SHA512_STATE1_SZ;
 	case ICP_QAT_HW_AUTH_ALGO_MD5:
 		return ICP_QAT_HW_MD5_STATE1_SZ;
+	case ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC:
+		return ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ;
 	case ICP_QAT_HW_AUTH_ALGO_DELIMITER:
 		/* return maximum digest size in this case */
 		return ICP_QAT_HW_SHA512_STATE1_SZ;
@@ -843,6 +849,8 @@ static int qat_hash_get_block_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 		return SHA512_CBLOCK;
 	case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
 		return 16;
+	case ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC:
+		return ICP_QAT_HW_AES_BLK_SZ;
 	case ICP_QAT_HW_AUTH_ALGO_MD5:
 		return MD5_CBLOCK;
 	case ICP_QAT_HW_AUTH_ALGO_DELIMITER:
@@ -991,11 +999,28 @@ static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg,
 #define HMAC_OPAD_VALUE	0x5c
 #define HASH_XCBC_PRECOMP_KEY_NUM 3
 
+static const uint8_t AES_CMAC_SEED[ICP_QAT_HW_AES_128_KEY_SZ];
+
+static void aes_cmac_key_derive(uint8_t *base, uint8_t *derived)
+{
+	int i;
+
+	derived[0] = base[0] << 1;
+	for (i = 1; i < ICP_QAT_HW_AES_BLK_SZ ; i++) {
+		derived[i] = base[i] << 1;
+		derived[i - 1] |= base[i] >> 7;
+	}
+
+	if (base[0] & 0x80)
+		derived[ICP_QAT_HW_AES_BLK_SZ - 1] ^= QAT_AES_CMAC_CONST_RB;
+}
+
 static int qat_sym_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 				const uint8_t *auth_key,
 				uint16_t auth_keylen,
 				uint8_t *p_state_buf,
-				uint16_t *p_state_len)
+				uint16_t *p_state_len,
+				uint8_t aes_cmac)
 {
 	int block_size;
 	uint8_t ipad[qat_hash_get_block_size(ICP_QAT_HW_AUTH_ALGO_DELIMITER)];
@@ -1003,47 +1028,91 @@ static int qat_sym_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 	int i;
 
 	if (hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC) {
-		static uint8_t qat_aes_xcbc_key_seed[
-					ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ] = {
-			0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-			0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-			0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
-			0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
-			0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
-			0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
-		};
 
-		uint8_t *in = NULL;
-		uint8_t *out = p_state_buf;
-		int x;
-		AES_KEY enc_key;
+		/* CMAC */
+		if (aes_cmac) {
+			AES_KEY enc_key;
+			uint8_t *in = NULL;
+			uint8_t k0[ICP_QAT_HW_AES_128_KEY_SZ];
+			uint8_t *k1, *k2;
 
-		in = rte_zmalloc("working mem for key",
-				ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ, 16);
-		if (in == NULL) {
-			QAT_LOG(ERR, "Failed to alloc memory");
-			return -ENOMEM;
-		}
+			auth_keylen = ICP_QAT_HW_AES_128_KEY_SZ;
+
+			in = rte_zmalloc("AES CMAC K1",
+					 ICP_QAT_HW_AES_128_KEY_SZ, 16);
+
+			if (in == NULL) {
+				QAT_LOG(ERR, "Failed to alloc memory");
+				return -ENOMEM;
+			}
+
+			rte_memcpy(in, AES_CMAC_SEED,
+				   ICP_QAT_HW_AES_128_KEY_SZ);
+			rte_memcpy(p_state_buf, auth_key, auth_keylen);
 
-		rte_memcpy(in, qat_aes_xcbc_key_seed,
-				ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ);
-		for (x = 0; x < HASH_XCBC_PRECOMP_KEY_NUM; x++) {
 			if (AES_set_encrypt_key(auth_key, auth_keylen << 3,
 				&enc_key) != 0) {
-				rte_free(in -
-					(x * ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ));
-				memset(out -
-					(x * ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ),
-					0, ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ);
+				rte_free(in);
 				return -EFAULT;
 			}
-			AES_encrypt(in, out, &enc_key);
-			in += ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ;
-			out += ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ;
+
+			AES_encrypt(in, k0, &enc_key);
+
+			k1 = p_state_buf + ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ;
+			k2 = k1 + ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ;
+
+			aes_cmac_key_derive(k0, k1);
+			aes_cmac_key_derive(k1, k2);
+
+			memset(k0, 0, ICP_QAT_HW_AES_128_KEY_SZ);
+			*p_state_len = ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ;
+			rte_free(in);
+			return 0;
+		} else {
+			static uint8_t qat_aes_xcbc_key_seed[
+					ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ] = {
+				0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+				0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+				0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+				0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+				0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
+				0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
+			};
+
+			uint8_t *in = NULL;
+			uint8_t *out = p_state_buf;
+			int x;
+			AES_KEY enc_key;
+
+			in = rte_zmalloc("working mem for key",
+					ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ, 16);
+			if (in == NULL) {
+				QAT_LOG(ERR, "Failed to alloc memory");
+				return -ENOMEM;
+			}
+
+			rte_memcpy(in, qat_aes_xcbc_key_seed,
+					ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ);
+			for (x = 0; x < HASH_XCBC_PRECOMP_KEY_NUM; x++) {
+				if (AES_set_encrypt_key(auth_key,
+							auth_keylen << 3,
+							&enc_key) != 0) {
+					rte_free(in -
+					  (x * ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ));
+					memset(out -
+					   (x * ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ),
+					  0, ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ);
+					return -EFAULT;
+				}
+				AES_encrypt(in, out, &enc_key);
+				in += ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ;
+				out += ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ;
+			}
+			*p_state_len = ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ;
+			rte_free(in - x*ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ);
+			return 0;
 		}
-		*p_state_len = ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ;
-		rte_free(in - x*ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ);
-		return 0;
+
 	} else if ((hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) ||
 		(hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64)) {
 		uint8_t *in = NULL;
@@ -1417,7 +1486,9 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
 
 	if (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2
 		|| cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9
-		|| cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3)
+		|| cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3
+		|| cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC
+			)
 		hash->auth_counter.counter = 0;
 	else
 		hash->auth_counter.counter = rte_bswap32(
@@ -1430,40 +1501,45 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
 	 */
 	switch (cdesc->qat_hash_alg) {
 	case ICP_QAT_HW_AUTH_ALGO_SHA1:
-		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA1,
-			authkey, authkeylen, cdesc->cd_cur_ptr,	&state1_size)) {
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA1, authkey,
+			authkeylen, cdesc->cd_cur_ptr, &state1_size,
+			cdesc->aes_cmac)) {
 			QAT_LOG(ERR, "(SHA)precompute failed");
 			return -EFAULT;
 		}
 		state2_size = RTE_ALIGN_CEIL(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_SHA224:
-		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA224,
-			authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) {
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA224, authkey,
+			authkeylen, cdesc->cd_cur_ptr, &state1_size,
+			cdesc->aes_cmac)) {
 			QAT_LOG(ERR, "(SHA)precompute failed");
 			return -EFAULT;
 		}
 		state2_size = ICP_QAT_HW_SHA224_STATE2_SZ;
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_SHA256:
-		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA256,
-			authkey, authkeylen, cdesc->cd_cur_ptr,	&state1_size)) {
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA256, authkey,
+			authkeylen, cdesc->cd_cur_ptr,	&state1_size,
+			cdesc->aes_cmac)) {
 			QAT_LOG(ERR, "(SHA)precompute failed");
 			return -EFAULT;
 		}
 		state2_size = ICP_QAT_HW_SHA256_STATE2_SZ;
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_SHA384:
-		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA384,
-			authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) {
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA384, authkey,
+			authkeylen, cdesc->cd_cur_ptr, &state1_size,
+			cdesc->aes_cmac)) {
 			QAT_LOG(ERR, "(SHA)precompute failed");
 			return -EFAULT;
 		}
 		state2_size = ICP_QAT_HW_SHA384_STATE2_SZ;
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_SHA512:
-		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA512,
-			authkey, authkeylen, cdesc->cd_cur_ptr,	&state1_size)) {
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA512, authkey,
+			authkeylen, cdesc->cd_cur_ptr,	&state1_size,
+			cdesc->aes_cmac)) {
 			QAT_LOG(ERR, "(SHA)precompute failed");
 			return -EFAULT;
 		}
@@ -1471,10 +1547,16 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC:
 		state1_size = ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ;
+
+		if (cdesc->aes_cmac)
+			memset(cdesc->cd_cur_ptr, 0, state1_size);
 		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC,
 			authkey, authkeylen, cdesc->cd_cur_ptr + state1_size,
-			&state2_size)) {
-			QAT_LOG(ERR, "(XCBC)precompute failed");
+			&state2_size, cdesc->aes_cmac)) {
+			cdesc->aes_cmac ? QAT_LOG(ERR,
+						  "(CMAC)precompute failed")
+					: QAT_LOG(ERR,
+						  "(XCBC)precompute failed");
 			return -EFAULT;
 		}
 		break;
@@ -1482,9 +1564,9 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
 	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
 		qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM;
 		state1_size = ICP_QAT_HW_GALOIS_128_STATE1_SZ;
-		if (qat_sym_do_precomputes(cdesc->qat_hash_alg,
-			authkey, authkeylen, cdesc->cd_cur_ptr + state1_size,
-			&state2_size)) {
+		if (qat_sym_do_precomputes(cdesc->qat_hash_alg, authkey,
+			authkeylen, cdesc->cd_cur_ptr + state1_size,
+			&state2_size, cdesc->aes_cmac)) {
 			QAT_LOG(ERR, "(GCM)precompute failed");
 			return -EFAULT;
 		}
@@ -1542,9 +1624,9 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
 
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_MD5:
-		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_MD5,
-			authkey, authkeylen, cdesc->cd_cur_ptr,
-			&state1_size)) {
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_MD5, authkey,
+			authkeylen, cdesc->cd_cur_ptr, &state1_size,
+			cdesc->aes_cmac)) {
 			QAT_LOG(ERR, "(MD5)precompute failed");
 			return -EFAULT;
 		}
diff --git a/drivers/crypto/qat/qat_sym_session.h b/drivers/crypto/qat/qat_sym_session.h
index e8f51e5b..43e25ceb 100644
--- a/drivers/crypto/qat/qat_sym_session.h
+++ b/drivers/crypto/qat/qat_sym_session.h
@@ -36,6 +36,8 @@
 					ICP_QAT_HW_CIPHER_KEY_CONVERT, \
 					ICP_QAT_HW_CIPHER_DECRYPT)
 
+#define QAT_AES_CMAC_CONST_RB 0x87
+
 enum qat_sym_proto_flag {
 	QAT_CRYPTO_PROTO_FLAG_NONE = 0,
 	QAT_CRYPTO_PROTO_FLAG_CCM = 1,
@@ -75,6 +77,7 @@ struct qat_sym_session {
 	uint16_t digest_length;
 	rte_spinlock_t lock;	/* protects this struct */
 	enum qat_device_gen min_qat_dev_gen;
+	uint8_t aes_cmac;
 };
 
 int
diff --git a/drivers/crypto/scheduler/meson.build b/drivers/crypto/scheduler/meson.build
new file mode 100644
index 00000000..c5ba2d68
--- /dev/null
+++ b/drivers/crypto/scheduler/meson.build
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
+
+deps += ['bus_vdev', 'reorder']
+name = 'crypto_scheduler'
+sources = files(
+	'rte_cryptodev_scheduler.c',
+	'scheduler_failover.c',
+	'scheduler_multicore.c',
+	'scheduler_pkt_size_distr.c',
+	'scheduler_pmd.c',
+	'scheduler_pmd_ops.c',
+	'scheduler_roundrobin.c',
+)
+
+headers = files(
+	'rte_cryptodev_scheduler.h',
+	'rte_cryptodev_scheduler_operations.h',
+)
diff --git a/drivers/crypto/scheduler/rte_cryptodev_scheduler.c b/drivers/crypto/scheduler/rte_cryptodev_scheduler.c
index 6e4919c4..a2142860 100644
--- a/drivers/crypto/scheduler/rte_cryptodev_scheduler.c
+++ b/drivers/crypto/scheduler/rte_cryptodev_scheduler.c
@@ -174,7 +174,7 @@ rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id)
 		return -ENOTSUP;
 	}
 
-	if (dev->driver_id != cryptodev_driver_id) {
+	if (dev->driver_id != cryptodev_scheduler_driver_id) {
 		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
@@ -233,7 +233,7 @@ rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id)
 		return -ENOTSUP;
 	}
 
-	if (dev->driver_id != cryptodev_driver_id) {
+	if (dev->driver_id != cryptodev_scheduler_driver_id) {
 		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
@@ -290,7 +290,7 @@ rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id,
 		return -ENOTSUP;
 	}
 
-	if (dev->driver_id != cryptodev_driver_id) {
+	if (dev->driver_id != cryptodev_scheduler_driver_id) {
 		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
@@ -308,28 +308,28 @@ rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id,
 	switch (mode) {
 	case CDEV_SCHED_MODE_ROUNDROBIN:
 		if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
-				roundrobin_scheduler) < 0) {
+				crypto_scheduler_roundrobin) < 0) {
 			CR_SCHED_LOG(ERR, "Failed to load scheduler");
 			return -1;
 		}
 		break;
 	case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
 		if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
-				pkt_size_based_distr_scheduler) < 0) {
+				crypto_scheduler_pkt_size_based_distr) < 0) {
 			CR_SCHED_LOG(ERR, "Failed to load scheduler");
 			return -1;
 		}
 		break;
 	case CDEV_SCHED_MODE_FAILOVER:
 		if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
-				failover_scheduler) < 0) {
+				crypto_scheduler_failover) < 0) {
 			CR_SCHED_LOG(ERR, "Failed to load scheduler");
 			return -1;
 		}
 		break;
 	case CDEV_SCHED_MODE_MULTICORE:
 		if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
-				multicore_scheduler) < 0) {
+				crypto_scheduler_multicore) < 0) {
 			CR_SCHED_LOG(ERR, "Failed to load scheduler");
 			return -1;
 		}
@@ -353,7 +353,7 @@ rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id)
 		return -ENOTSUP;
 	}
 
-	if (dev->driver_id != cryptodev_driver_id) {
+	if (dev->driver_id != cryptodev_scheduler_driver_id) {
 		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
@@ -375,7 +375,7 @@ rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id,
 		return -ENOTSUP;
 	}
 
-	if (dev->driver_id != cryptodev_driver_id) {
+	if (dev->driver_id != cryptodev_scheduler_driver_id) {
 		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
@@ -403,7 +403,7 @@ rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id)
 		return -ENOTSUP;
 	}
 
-	if (dev->driver_id != cryptodev_driver_id) {
+	if (dev->driver_id != cryptodev_scheduler_driver_id) {
 		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
@@ -425,7 +425,7 @@ rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id,
 		return -ENOTSUP;
 	}
 
-	if (dev->driver_id != cryptodev_driver_id) {
+	if (dev->driver_id != cryptodev_scheduler_driver_id) {
 		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
@@ -498,7 +498,7 @@ rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves)
 		return -ENOTSUP;
 	}
 
-	if (dev->driver_id != cryptodev_driver_id) {
+	if (dev->driver_id != cryptodev_scheduler_driver_id) {
 		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
@@ -566,7 +566,7 @@ rte_cryptodev_scheduler_option_get(uint8_t scheduler_id,
 		return -EINVAL;
 	}
 
-	if (dev->driver_id != cryptodev_driver_id) {
+	if (dev->driver_id != cryptodev_scheduler_driver_id) {
 		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
diff --git a/drivers/crypto/scheduler/rte_cryptodev_scheduler.h b/drivers/crypto/scheduler/rte_cryptodev_scheduler.h
index 3faea409..9a72a90a 100644
--- a/drivers/crypto/scheduler/rte_cryptodev_scheduler.h
+++ b/drivers/crypto/scheduler/rte_cryptodev_scheduler.h
@@ -270,13 +270,13 @@ struct rte_cryptodev_scheduler {
 };
 
 /** Round-robin mode scheduler */
-extern struct rte_cryptodev_scheduler *roundrobin_scheduler;
+extern struct rte_cryptodev_scheduler *crypto_scheduler_roundrobin;
 /** Packet-size based distribution mode scheduler */
-extern struct rte_cryptodev_scheduler *pkt_size_based_distr_scheduler;
+extern struct rte_cryptodev_scheduler *crypto_scheduler_pkt_size_based_distr;
 /** Fail-over mode scheduler */
-extern struct rte_cryptodev_scheduler *failover_scheduler;
+extern struct rte_cryptodev_scheduler *crypto_scheduler_failover;
 /** multi-core mode scheduler */
-extern struct rte_cryptodev_scheduler *multicore_scheduler;
+extern struct rte_cryptodev_scheduler *crypto_scheduler_multicore;
 
 #ifdef __cplusplus
 }
diff --git a/drivers/crypto/scheduler/scheduler_failover.c b/drivers/crypto/scheduler/scheduler_failover.c
index ddfb5b81..3a023b8a 100644
--- a/drivers/crypto/scheduler/scheduler_failover.c
+++ b/drivers/crypto/scheduler/scheduler_failover.c
@@ -197,7 +197,7 @@ scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev)
 	return 0;
 }
 
-struct rte_cryptodev_scheduler_ops scheduler_fo_ops = {
+static struct rte_cryptodev_scheduler_ops scheduler_fo_ops = {
 	slave_attach,
 	slave_detach,
 	scheduler_start,
@@ -208,7 +208,7 @@ struct rte_cryptodev_scheduler_ops scheduler_fo_ops = {
 	NULL	/*option_get */
 };
 
-struct rte_cryptodev_scheduler fo_scheduler = {
+static struct rte_cryptodev_scheduler fo_scheduler = {
 		.name = "failover-scheduler",
 		.description = "scheduler which enqueues to the primary slave, "
 				"and only then enqueues to the secondary slave "
@@ -217,4 +217,4 @@ struct rte_cryptodev_scheduler fo_scheduler = {
 		.ops = &scheduler_fo_ops
 };
 
-struct rte_cryptodev_scheduler *failover_scheduler = &fo_scheduler;
+struct rte_cryptodev_scheduler *crypto_scheduler_failover = &fo_scheduler;
diff --git a/drivers/crypto/scheduler/scheduler_multicore.c b/drivers/crypto/scheduler/scheduler_multicore.c
index d410e69d..7808e9a3 100644
--- a/drivers/crypto/scheduler/scheduler_multicore.c
+++ b/drivers/crypto/scheduler/scheduler_multicore.c
@@ -392,7 +392,7 @@ exit:
 	return -1;
 }
 
-struct rte_cryptodev_scheduler_ops scheduler_mc_ops = {
+static struct rte_cryptodev_scheduler_ops scheduler_mc_ops = {
 	slave_attach,
 	slave_detach,
 	scheduler_start,
@@ -403,11 +403,11 @@ struct rte_cryptodev_scheduler_ops scheduler_mc_ops = {
 	NULL	/* option_get */
 };
 
-struct rte_cryptodev_scheduler mc_scheduler = {
+static struct rte_cryptodev_scheduler mc_scheduler = {
 		.name = "multicore-scheduler",
 		.description = "scheduler which will run burst across multiple cpu cores",
 		.mode = CDEV_SCHED_MODE_MULTICORE,
 		.ops = &scheduler_mc_ops
 };
 
-struct rte_cryptodev_scheduler *multicore_scheduler = &mc_scheduler;
+struct rte_cryptodev_scheduler *crypto_scheduler_multicore = &mc_scheduler;
diff --git a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c
index 74129b66..45c8dceb 100644
--- a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c
+++ b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c
@@ -398,7 +398,7 @@ scheduler_option_get(struct rte_cryptodev *dev, uint32_t option_type,
 	return 0;
 }
 
-struct rte_cryptodev_scheduler_ops scheduler_ps_ops = {
+static struct rte_cryptodev_scheduler_ops scheduler_ps_ops = {
 	slave_attach,
 	slave_detach,
 	scheduler_start,
@@ -409,7 +409,7 @@ struct rte_cryptodev_scheduler_ops scheduler_ps_ops = {
 	scheduler_option_get
 };
 
-struct rte_cryptodev_scheduler psd_scheduler = {
+static struct rte_cryptodev_scheduler psd_scheduler = {
 		.name = "packet-size-based-scheduler",
 		.description = "scheduler which will distribute crypto op "
 				"burst based on the packet size",
@@ -417,4 +417,4 @@ struct rte_cryptodev_scheduler psd_scheduler = {
 		.ops = &scheduler_ps_ops
 };
 
-struct rte_cryptodev_scheduler *pkt_size_based_distr_scheduler = &psd_scheduler;
+struct rte_cryptodev_scheduler *crypto_scheduler_pkt_size_based_distr = &psd_scheduler;
diff --git a/drivers/crypto/scheduler/scheduler_pmd.c b/drivers/crypto/scheduler/scheduler_pmd.c
index a9221a94..20198ccb 100644
--- a/drivers/crypto/scheduler/scheduler_pmd.c
+++ b/drivers/crypto/scheduler/scheduler_pmd.c
@@ -14,7 +14,7 @@
 #include "rte_cryptodev_scheduler.h"
 #include "scheduler_pmd_private.h"
 
-uint8_t cryptodev_driver_id;
+uint8_t cryptodev_scheduler_driver_id;
 
 struct scheduler_init_params {
 	struct rte_cryptodev_pmd_init_params def_p;
@@ -38,7 +38,7 @@ struct scheduler_init_params {
 #define RTE_CRYPTODEV_VDEV_COREMASK		("coremask")
 #define RTE_CRYPTODEV_VDEV_CORELIST		("corelist")
 
-const char *scheduler_valid_params[] = {
+static const char * const scheduler_valid_params[] = {
 	RTE_CRYPTODEV_VDEV_NAME,
 	RTE_CRYPTODEV_VDEV_SLAVE,
 	RTE_CRYPTODEV_VDEV_MODE,
@@ -91,7 +91,7 @@ cryptodev_scheduler_create(const char *name,
 		return -EFAULT;
 	}
 
-	dev->driver_id = cryptodev_driver_id;
+	dev->driver_id = cryptodev_scheduler_driver_id;
 	dev->dev_ops = rte_crypto_scheduler_pmd_ops;
 
 	sched_ctx = dev->data->dev_private;
@@ -569,4 +569,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
 	"slave=<name>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
 		cryptodev_scheduler_pmd_drv.driver,
-		cryptodev_driver_id);
+		cryptodev_scheduler_driver_id);
diff --git a/drivers/crypto/scheduler/scheduler_pmd_ops.c b/drivers/crypto/scheduler/scheduler_pmd_ops.c
index 778071ca..939105aa 100644
--- a/drivers/crypto/scheduler/scheduler_pmd_ops.c
+++ b/drivers/crypto/scheduler/scheduler_pmd_ops.c
@@ -522,7 +522,7 @@ scheduler_pmd_sym_session_clear(struct rte_cryptodev *dev,
 	}
 }
 
-struct rte_cryptodev_ops scheduler_pmd_ops = {
+static struct rte_cryptodev_ops scheduler_pmd_ops = {
 		.dev_configure		= scheduler_pmd_config,
 		.dev_start		= scheduler_pmd_start,
 		.dev_stop		= scheduler_pmd_stop,
diff --git a/drivers/crypto/scheduler/scheduler_pmd_private.h b/drivers/crypto/scheduler/scheduler_pmd_private.h
index d5e602a2..3ed480c1 100644
--- a/drivers/crypto/scheduler/scheduler_pmd_private.h
+++ b/drivers/crypto/scheduler/scheduler_pmd_private.h
@@ -63,7 +63,7 @@ struct scheduler_qp_ctx {
 } __rte_cache_aligned;
 
 
-extern uint8_t cryptodev_driver_id;
+extern uint8_t cryptodev_scheduler_driver_id;
 
 static __rte_always_inline uint16_t
 get_max_enqueue_order_count(struct rte_ring *order_ring, uint16_t nb_ops)
diff --git a/drivers/crypto/scheduler/scheduler_roundrobin.c b/drivers/crypto/scheduler/scheduler_roundrobin.c
index c7082a64..9b891d97 100644
--- a/drivers/crypto/scheduler/scheduler_roundrobin.c
+++ b/drivers/crypto/scheduler/scheduler_roundrobin.c
@@ -190,7 +190,7 @@ scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev)
 	return 0;
 }
 
-struct rte_cryptodev_scheduler_ops scheduler_rr_ops = {
+static struct rte_cryptodev_scheduler_ops scheduler_rr_ops = {
 	slave_attach,
 	slave_detach,
 	scheduler_start,
@@ -201,7 +201,7 @@ struct rte_cryptodev_scheduler_ops scheduler_rr_ops = {
 	NULL	/* option_get */
 };
 
-struct rte_cryptodev_scheduler scheduler = {
+static struct rte_cryptodev_scheduler scheduler = {
 		.name = "roundrobin-scheduler",
 		.description = "scheduler which will round robin burst across "
 				"slave crypto devices",
@@ -209,4 +209,4 @@ struct rte_cryptodev_scheduler scheduler = {
 		.ops = &scheduler_rr_ops
 };
 
-struct rte_cryptodev_scheduler *roundrobin_scheduler = &scheduler;
+struct rte_cryptodev_scheduler *crypto_scheduler_roundrobin = &scheduler;
diff --git a/drivers/crypto/zuc/meson.build b/drivers/crypto/zuc/meson.build
new file mode 100644
index 00000000..b8ca7107
--- /dev/null
+++ b/drivers/crypto/zuc/meson.build
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+lib = cc.find_library('libsso_zuc', required: false)
+if not lib.found()
+	build = false
+else
+	ext_deps += lib
+endif
+
+sources = files('rte_zuc_pmd.c', 'rte_zuc_pmd_ops.c')
+deps += ['bus_vdev']
diff --git a/drivers/event/Makefile b/drivers/event/Makefile
index f301d8dc..03ad1b6c 100644
--- a/drivers/event/Makefile
+++ b/drivers/event/Makefile
@@ -6,6 +6,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_EVENTDEV) += skeleton
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_SW_EVENTDEV) += sw
+DIRS-$(CONFIG_RTE_LIBRTE_PMD_DSW_EVENTDEV) += dsw
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += octeontx
 ifeq ($(CONFIG_RTE_LIBRTE_DPAA_BUS),y)
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA_EVENTDEV) += dpaa
diff --git a/drivers/event/dpaa/Makefile b/drivers/event/dpaa/Makefile
index ddd85522..6f93e7f4 100644
--- a/drivers/event/dpaa/Makefile
+++ b/drivers/event/dpaa/Makefile
@@ -34,5 +34,6 @@ LDLIBS += -lrte_mempool_dpaa
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_eventdev -lrte_pmd_dpaa -lrte_bus_vdev
+LDLIBS += -lrte_common_dpaax
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/event/dpaa/dpaa_eventdev.c b/drivers/event/dpaa/dpaa_eventdev.c
index 5443ef56..1e247e4f 100644
--- a/drivers/event/dpaa/dpaa_eventdev.c
+++ b/drivers/event/dpaa/dpaa_eventdev.c
@@ -30,6 +30,7 @@
 #include <rte_dpaa_bus.h>
 #include <rte_dpaa_logs.h>
 #include <rte_cycles.h>
+#include <rte_kvargs.h>
 
 #include <dpaa_ethdev.h>
 #include "dpaa_eventdev.h"
@@ -43,19 +44,31 @@
  * 1 Eventdev can have N Eventqueue
  */
 
+#define DISABLE_INTR_MODE "disable_intr"
+
 static int
 dpaa_event_dequeue_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
 				 uint64_t *timeout_ticks)
 {
-	uint64_t cycles_per_second;
-
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(dev);
 
+	uint64_t cycles_per_second;
+
 	cycles_per_second = rte_get_timer_hz();
-	*timeout_ticks = ns * (cycles_per_second / NS_PER_S);
+	*timeout_ticks = (ns * cycles_per_second) / NS_PER_S;
+
+	return 0;
+}
+
+static int
+dpaa_event_dequeue_timeout_ticks_intr(struct rte_eventdev *dev, uint64_t ns,
+				 uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
 
+	*timeout_ticks = ns/1000;
 	return 0;
 }
 
@@ -100,6 +113,56 @@ dpaa_event_enqueue(void *port, const struct rte_event *ev)
 	return dpaa_event_enqueue_burst(port, ev, 1);
 }
 
+static void drain_4_bytes(int fd, fd_set *fdset)
+{
+	if (FD_ISSET(fd, fdset)) {
+		/* drain 4 bytes */
+		uint32_t junk;
+		ssize_t sjunk = read(qman_thread_fd(), &junk, sizeof(junk));
+		if (sjunk != sizeof(junk))
+			DPAA_EVENTDEV_ERR("UIO irq read error");
+	}
+}
+
+static inline int
+dpaa_event_dequeue_wait(uint64_t timeout_ticks)
+{
+	int fd_qman, nfds;
+	int ret;
+	fd_set readset;
+
+	/* Go into (and back out of) IRQ mode for each select,
+	 * it simplifies exit-path considerations and other
+	 * potential nastiness.
+	 */
+	struct timeval tv = {
+		.tv_sec = timeout_ticks / 1000000,
+		.tv_usec = timeout_ticks % 1000000
+	};
+
+	fd_qman = qman_thread_fd();
+	nfds = fd_qman + 1;
+	FD_ZERO(&readset);
+	FD_SET(fd_qman, &readset);
+
+	qman_irqsource_add(QM_PIRQ_DQRI);
+
+	ret = select(nfds, &readset, NULL, NULL, &tv);
+	if (ret < 0)
+		return ret;
+	/* Calling irqsource_remove() prior to thread_irq()
+	 * means thread_irq() will not process whatever caused
+	 * the interrupts, however it does ensure that, once
+	 * thread_irq() re-enables interrupts, they won't fire
+	 * again immediately.
+	 */
+	qman_irqsource_remove(~0);
+	drain_4_bytes(fd_qman, &readset);
+	qman_thread_irq();
+
+	return ret;
+}
+
 static uint16_t
 dpaa_event_dequeue_burst(void *port, struct rte_event ev[],
 			 uint16_t nb_events, uint64_t timeout_ticks)
@@ -107,8 +170,8 @@ dpaa_event_dequeue_burst(void *port, struct rte_event ev[],
 	int ret;
 	u16 ch_id;
 	void *buffers[8];
-	u32 num_frames, i;
-	uint64_t wait_time, cur_ticks, start_ticks;
+	u32 num_frames, i, irq = 0;
+	uint64_t cur_ticks = 0, wait_time_ticks = 0;
 	struct dpaa_port *portal = (struct dpaa_port *)port;
 	struct rte_mbuf *mbuf;
 
@@ -147,20 +210,21 @@ dpaa_event_dequeue_burst(void *port, struct rte_event ev[],
 	}
 	DPAA_PER_LCORE_DQRR_HELD = 0;
 
-	if (portal->timeout == DPAA_EVENT_PORT_DEQUEUE_TIMEOUT_INVALID)
-		wait_time = timeout_ticks;
+	if (timeout_ticks)
+		wait_time_ticks = timeout_ticks;
 	else
-		wait_time = portal->timeout;
+		wait_time_ticks = portal->timeout_us;
 
-	/* Lets dequeue the frames */
-	start_ticks = rte_get_timer_cycles();
-	wait_time += start_ticks;
+	wait_time_ticks += rte_get_timer_cycles();
 	do {
+		/* Lets dequeue the frames */
 		num_frames = qman_portal_dequeue(ev, nb_events, buffers);
-		if (num_frames != 0)
+		if (irq)
+			irq = 0;
+		if (num_frames)
 			break;
 		cur_ticks = rte_get_timer_cycles();
-	} while (cur_ticks < wait_time);
+	} while (cur_ticks < wait_time_ticks);
 
 	return num_frames;
 }
@@ -171,11 +235,91 @@ dpaa_event_dequeue(void *port, struct rte_event *ev, uint64_t timeout_ticks)
 	return dpaa_event_dequeue_burst(port, ev, 1, timeout_ticks);
 }
 
+static uint16_t
+dpaa_event_dequeue_burst_intr(void *port, struct rte_event ev[],
+			      uint16_t nb_events, uint64_t timeout_ticks)
+{
+	int ret;
+	u16 ch_id;
+	void *buffers[8];
+	u32 num_frames, i, irq = 0;
+	uint64_t cur_ticks = 0, wait_time_ticks = 0;
+	struct dpaa_port *portal = (struct dpaa_port *)port;
+	struct rte_mbuf *mbuf;
+
+	if (unlikely(!RTE_PER_LCORE(dpaa_io))) {
+		/* Affine current thread context to a qman portal */
+		ret = rte_dpaa_portal_init((void *)0);
+		if (ret) {
+			DPAA_EVENTDEV_ERR("Unable to initialize portal");
+			return ret;
+		}
+	}
+
+	if (unlikely(!portal->is_port_linked)) {
+		/*
+		 * Affine event queue for current thread context
+		 * to a qman portal.
+		 */
+		for (i = 0; i < portal->num_linked_evq; i++) {
+			ch_id = portal->evq_info[i].ch_id;
+			dpaa_eventq_portal_add(ch_id);
+		}
+		portal->is_port_linked = true;
+	}
+
+	/* Check if there are atomic contexts to be released */
+	i = 0;
+	while (DPAA_PER_LCORE_DQRR_SIZE) {
+		if (DPAA_PER_LCORE_DQRR_HELD & (1 << i)) {
+			qman_dca_index(i, 0);
+			mbuf = DPAA_PER_LCORE_DQRR_MBUF(i);
+			mbuf->seqn = DPAA_INVALID_MBUF_SEQN;
+			DPAA_PER_LCORE_DQRR_HELD &= ~(1 << i);
+			DPAA_PER_LCORE_DQRR_SIZE--;
+		}
+		i++;
+	}
+	DPAA_PER_LCORE_DQRR_HELD = 0;
+
+	if (timeout_ticks)
+		wait_time_ticks = timeout_ticks;
+	else
+		wait_time_ticks = portal->timeout_us;
+
+	do {
+		/* Lets dequeue the frames */
+		num_frames = qman_portal_dequeue(ev, nb_events, buffers);
+		if (irq)
+			irq = 0;
+		if (num_frames)
+			break;
+		if (wait_time_ticks) { /* wait for time */
+			if (dpaa_event_dequeue_wait(wait_time_ticks) > 0) {
+				irq = 1;
+				continue;
+			}
+			break; /* no event after waiting */
+		}
+		cur_ticks = rte_get_timer_cycles();
+	} while (cur_ticks < wait_time_ticks);
+
+	return num_frames;
+}
+
+static uint16_t
+dpaa_event_dequeue_intr(void *port,
+			struct rte_event *ev,
+			uint64_t timeout_ticks)
+{
+	return dpaa_event_dequeue_burst_intr(port, ev, 1, timeout_ticks);
+}
+
 static void
 dpaa_event_dev_info_get(struct rte_eventdev *dev,
 			struct rte_event_dev_info *dev_info)
 {
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(dev);
 	dev_info->driver_name = "event_dpaa";
@@ -184,7 +328,7 @@ dpaa_event_dev_info_get(struct rte_eventdev *dev,
 	dev_info->max_dequeue_timeout_ns =
 		DPAA_EVENT_MAX_DEQUEUE_TIMEOUT;
 	dev_info->dequeue_timeout_ns =
-		DPAA_EVENT_MIN_DEQUEUE_TIMEOUT;
+		DPAA_EVENT_PORT_DEQUEUE_TIMEOUT_NS;
 	dev_info->max_event_queues =
 		DPAA_EVENT_MAX_QUEUES;
 	dev_info->max_event_queue_flows =
@@ -220,8 +364,7 @@ dpaa_event_dev_configure(const struct rte_eventdev *dev)
 	int ret, i;
 	uint32_t *ch_id;
 
-	EVENTDEV_DRV_FUNC_TRACE();
-
+	EVENTDEV_INIT_FUNC_TRACE();
 	priv->dequeue_timeout_ns = conf->dequeue_timeout_ns;
 	priv->nb_events_limit = conf->nb_events_limit;
 	priv->nb_event_queues = conf->nb_event_queues;
@@ -231,26 +374,18 @@ dpaa_event_dev_configure(const struct rte_eventdev *dev)
 	priv->nb_event_port_enqueue_depth = conf->nb_event_port_enqueue_depth;
 	priv->event_dev_cfg = conf->event_dev_cfg;
 
-	/* Check dequeue timeout method is per dequeue or global */
-	if (priv->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) {
-		/*
-		 * Use timeout value as given in dequeue operation.
-		 * So invalidating this timetout value.
-		 */
-		priv->dequeue_timeout_ns = 0;
-	}
-
 	ch_id = rte_malloc("dpaa-channels",
 			  sizeof(uint32_t) * priv->nb_event_queues,
 			  RTE_CACHE_LINE_SIZE);
 	if (ch_id == NULL) {
-		EVENTDEV_DRV_ERR("Fail to allocate memory for dpaa channels\n");
+		DPAA_EVENTDEV_ERR("Fail to allocate memory for dpaa channels\n");
 		return -ENOMEM;
 	}
 	/* Create requested event queues within the given event device */
 	ret = qman_alloc_pool_range(ch_id, priv->nb_event_queues, 1, 0);
 	if (ret < 0) {
-		EVENTDEV_DRV_ERR("Failed to create internal channel\n");
+		DPAA_EVENTDEV_ERR("qman_alloc_pool_range %u, err =%d\n",
+				 priv->nb_event_queues, ret);
 		rte_free(ch_id);
 		return ret;
 	}
@@ -260,30 +395,41 @@ dpaa_event_dev_configure(const struct rte_eventdev *dev)
 	/* Lets prepare event ports */
 	memset(&priv->ports[0], 0,
 	      sizeof(struct dpaa_port) * priv->nb_event_ports);
+
+	/* Check dequeue timeout method is per dequeue or global */
 	if (priv->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) {
-		for (i = 0; i < priv->nb_event_ports; i++) {
-			priv->ports[i].timeout =
-				DPAA_EVENT_PORT_DEQUEUE_TIMEOUT_INVALID;
-		}
-	} else if (priv->dequeue_timeout_ns == 0) {
-		for (i = 0; i < priv->nb_event_ports; i++) {
-			dpaa_event_dequeue_timeout_ticks(NULL,
-				DPAA_EVENT_PORT_DEQUEUE_TIMEOUT_NS,
-				&priv->ports[i].timeout);
-		}
+		/*
+		 * Use timeout value as given in dequeue operation.
+		 * So invalidating this timeout value.
+		 */
+		priv->dequeue_timeout_ns = 0;
+
+	} else if (conf->dequeue_timeout_ns == 0) {
+		priv->dequeue_timeout_ns = DPAA_EVENT_PORT_DEQUEUE_TIMEOUT_NS;
 	} else {
-		for (i = 0; i < priv->nb_event_ports; i++) {
-			dpaa_event_dequeue_timeout_ticks(NULL,
-				priv->dequeue_timeout_ns,
-				&priv->ports[i].timeout);
+		priv->dequeue_timeout_ns = conf->dequeue_timeout_ns;
+	}
+
+	for (i = 0; i < priv->nb_event_ports; i++) {
+		if (priv->intr_mode) {
+			priv->ports[i].timeout_us =
+				priv->dequeue_timeout_ns/1000;
+		} else {
+			uint64_t cycles_per_second;
+
+			cycles_per_second = rte_get_timer_hz();
+			priv->ports[i].timeout_us =
+				(priv->dequeue_timeout_ns * cycles_per_second)
+					/ NS_PER_S;
 		}
 	}
+
 	/*
 	 * TODO: Currently portals are affined with threads. Maximum threads
 	 * can be created equals to number of lcore.
 	 */
 	rte_free(ch_id);
-	EVENTDEV_DRV_LOG("Configured eventdev devid=%d", dev->data->dev_id);
+	DPAA_EVENTDEV_INFO("Configured eventdev devid=%d", dev->data->dev_id);
 
 	return 0;
 }
@@ -291,7 +437,7 @@ dpaa_event_dev_configure(const struct rte_eventdev *dev)
 static int
 dpaa_event_dev_start(struct rte_eventdev *dev)
 {
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 	RTE_SET_USED(dev);
 
 	return 0;
@@ -300,14 +446,14 @@ dpaa_event_dev_start(struct rte_eventdev *dev)
 static void
 dpaa_event_dev_stop(struct rte_eventdev *dev)
 {
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 	RTE_SET_USED(dev);
 }
 
 static int
 dpaa_event_dev_close(struct rte_eventdev *dev)
 {
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 	RTE_SET_USED(dev);
 
 	return 0;
@@ -317,7 +463,7 @@ static void
 dpaa_event_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
 			  struct rte_event_queue_conf *queue_conf)
 {
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(dev);
 	RTE_SET_USED(queue_id);
@@ -334,14 +480,14 @@ dpaa_event_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
 	struct dpaa_eventdev *priv = dev->data->dev_private;
 	struct dpaa_eventq *evq_info = &priv->evq_info[queue_id];
 
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	switch (queue_conf->schedule_type) {
 	case RTE_SCHED_TYPE_PARALLEL:
 	case RTE_SCHED_TYPE_ATOMIC:
 		break;
 	case RTE_SCHED_TYPE_ORDERED:
-		EVENTDEV_DRV_ERR("Schedule type is not supported.");
+		DPAA_EVENTDEV_ERR("Schedule type is not supported.");
 		return -1;
 	}
 	evq_info->event_queue_cfg = queue_conf->event_queue_cfg;
@@ -353,7 +499,7 @@ dpaa_event_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
 static void
 dpaa_event_queue_release(struct rte_eventdev *dev, uint8_t queue_id)
 {
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(dev);
 	RTE_SET_USED(queue_id);
@@ -363,7 +509,7 @@ static void
 dpaa_event_port_default_conf_get(struct rte_eventdev *dev, uint8_t port_id,
 				 struct rte_event_port_conf *port_conf)
 {
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(dev);
 	RTE_SET_USED(port_id);
@@ -379,7 +525,7 @@ dpaa_event_port_setup(struct rte_eventdev *dev, uint8_t port_id,
 {
 	struct dpaa_eventdev *eventdev = dev->data->dev_private;
 
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(port_conf);
 	dev->data->ports[port_id] = &eventdev->ports[port_id];
@@ -390,7 +536,7 @@ dpaa_event_port_setup(struct rte_eventdev *dev, uint8_t port_id,
 static void
 dpaa_event_port_release(void *port)
 {
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(port);
 }
@@ -454,7 +600,8 @@ dpaa_event_port_unlink(struct rte_eventdev *dev, void *port,
 		event_queue->event_port = NULL;
 	}
 
-	event_port->num_linked_evq = event_port->num_linked_evq - i;
+	if (event_port->num_linked_evq)
+		event_port->num_linked_evq = event_port->num_linked_evq - i;
 
 	return (int)i;
 }
@@ -466,7 +613,7 @@ dpaa_event_eth_rx_adapter_caps_get(const struct rte_eventdev *dev,
 {
 	const char *ethdev_driver = eth_dev->device->driver->name;
 
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(dev);
 
@@ -491,14 +638,14 @@ dpaa_event_eth_rx_adapter_queue_add(
 	struct dpaa_if *dpaa_intf = eth_dev->data->dev_private;
 	int ret, i;
 
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	if (rx_queue_id == -1) {
 		for (i = 0; i < dpaa_intf->nb_rx_queues; i++) {
 			ret = dpaa_eth_eventq_attach(eth_dev, i, ch_id,
 						     queue_conf);
 			if (ret) {
-				EVENTDEV_DRV_ERR(
+				DPAA_EVENTDEV_ERR(
 					"Event Queue attach failed:%d\n", ret);
 				goto detach_configured_queues;
 			}
@@ -508,7 +655,7 @@ dpaa_event_eth_rx_adapter_queue_add(
 
 	ret = dpaa_eth_eventq_attach(eth_dev, rx_queue_id, ch_id, queue_conf);
 	if (ret)
-		EVENTDEV_DRV_ERR("dpaa_eth_eventq_attach failed:%d\n", ret);
+		DPAA_EVENTDEV_ERR("dpaa_eth_eventq_attach failed:%d\n", ret);
 	return ret;
 
 detach_configured_queues:
@@ -527,14 +674,14 @@ dpaa_event_eth_rx_adapter_queue_del(const struct rte_eventdev *dev,
 	int ret, i;
 	struct dpaa_if *dpaa_intf = eth_dev->data->dev_private;
 
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(dev);
 	if (rx_queue_id == -1) {
 		for (i = 0; i < dpaa_intf->nb_rx_queues; i++) {
 			ret = dpaa_eth_eventq_detach(eth_dev, i);
 			if (ret)
-				EVENTDEV_DRV_ERR(
+				DPAA_EVENTDEV_ERR(
 					"Event Queue detach failed:%d\n", ret);
 		}
 
@@ -543,7 +690,7 @@ dpaa_event_eth_rx_adapter_queue_del(const struct rte_eventdev *dev,
 
 	ret = dpaa_eth_eventq_detach(eth_dev, rx_queue_id);
 	if (ret)
-		EVENTDEV_DRV_ERR("dpaa_eth_eventq_detach failed:%d\n", ret);
+		DPAA_EVENTDEV_ERR("dpaa_eth_eventq_detach failed:%d\n", ret);
 	return ret;
 }
 
@@ -551,7 +698,7 @@ static int
 dpaa_event_eth_rx_adapter_start(const struct rte_eventdev *dev,
 				const struct rte_eth_dev *eth_dev)
 {
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(dev);
 	RTE_SET_USED(eth_dev);
@@ -563,7 +710,7 @@ static int
 dpaa_event_eth_rx_adapter_stop(const struct rte_eventdev *dev,
 			       const struct rte_eth_dev *eth_dev)
 {
-	EVENTDEV_DRV_FUNC_TRACE();
+	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(dev);
 	RTE_SET_USED(eth_dev);
@@ -593,8 +740,44 @@ static struct rte_eventdev_ops dpaa_eventdev_ops = {
 	.eth_rx_adapter_stop = dpaa_event_eth_rx_adapter_stop,
 };
 
+static int flag_check_handler(__rte_unused const char *key,
+		const char *value, __rte_unused void *opaque)
+{
+	if (strcmp(value, "1"))
+		return -1;
+
+	return 0;
+}
+
+static int
+dpaa_event_check_flags(const char *params)
+{
+	struct rte_kvargs *kvlist;
+
+	if (params == NULL || params[0] == '\0')
+		return 0;
+
+	kvlist = rte_kvargs_parse(params, NULL);
+	if (kvlist == NULL)
+		return 0;
+
+	if (!rte_kvargs_count(kvlist, DISABLE_INTR_MODE)) {
+		rte_kvargs_free(kvlist);
+		return 0;
+	}
+	/* INTR MODE is disabled when there's key-value pair: disable_intr = 1*/
+	if (rte_kvargs_process(kvlist, DISABLE_INTR_MODE,
+				flag_check_handler, NULL) < 0) {
+		rte_kvargs_free(kvlist);
+		return 0;
+	}
+	rte_kvargs_free(kvlist);
+
+	return 1;
+}
+
 static int
-dpaa_event_dev_create(const char *name)
+dpaa_event_dev_create(const char *name, const char *params)
 {
 	struct rte_eventdev *eventdev;
 	struct dpaa_eventdev *priv;
@@ -603,21 +786,30 @@ dpaa_event_dev_create(const char *name)
 					   sizeof(struct dpaa_eventdev),
 					   rte_socket_id());
 	if (eventdev == NULL) {
-		EVENTDEV_DRV_ERR("Failed to create eventdev vdev %s", name);
+		DPAA_EVENTDEV_ERR("Failed to create eventdev vdev %s", name);
 		goto fail;
 	}
+	priv = eventdev->data->dev_private;
 
 	eventdev->dev_ops       = &dpaa_eventdev_ops;
 	eventdev->enqueue       = dpaa_event_enqueue;
 	eventdev->enqueue_burst = dpaa_event_enqueue_burst;
-	eventdev->dequeue       = dpaa_event_dequeue;
-	eventdev->dequeue_burst = dpaa_event_dequeue_burst;
+
+	if (dpaa_event_check_flags(params)) {
+		eventdev->dequeue	= dpaa_event_dequeue;
+		eventdev->dequeue_burst = dpaa_event_dequeue_burst;
+	} else {
+		priv->intr_mode = 1;
+		eventdev->dev_ops->timeout_ticks =
+				dpaa_event_dequeue_timeout_ticks_intr;
+		eventdev->dequeue	= dpaa_event_dequeue_intr;
+		eventdev->dequeue_burst = dpaa_event_dequeue_burst_intr;
+	}
 
 	/* For secondary processes, the primary has done all the work */
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
 
-	priv = eventdev->data->dev_private;
 	priv->max_event_queues = DPAA_EVENT_MAX_QUEUES;
 
 	return 0;
@@ -629,11 +821,14 @@ static int
 dpaa_event_dev_probe(struct rte_vdev_device *vdev)
 {
 	const char *name;
+	const char *params;
 
 	name = rte_vdev_device_name(vdev);
-	EVENTDEV_DRV_LOG("Initializing %s", name);
+	DPAA_EVENTDEV_INFO("Initializing %s", name);
+
+	params = rte_vdev_device_args(vdev);
 
-	return dpaa_event_dev_create(name);
+	return dpaa_event_dev_create(name, params);
 }
 
 static int
@@ -642,7 +837,7 @@ dpaa_event_dev_remove(struct rte_vdev_device *vdev)
 	const char *name;
 
 	name = rte_vdev_device_name(vdev);
-	EVENTDEV_DRV_LOG("Closing %s", name);
+	DPAA_EVENTDEV_INFO("Closing %s", name);
 
 	return rte_event_pmd_vdev_uninit(name);
 }
@@ -653,3 +848,5 @@ static struct rte_vdev_driver vdev_eventdev_dpaa_pmd = {
 };
 
 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_DPAA_PMD, vdev_eventdev_dpaa_pmd);
+RTE_PMD_REGISTER_PARAM_STRING(EVENTDEV_NAME_DPAA_PMD,
+		DISABLE_INTR_MODE "=<int>");
diff --git a/drivers/event/dpaa/dpaa_eventdev.h b/drivers/event/dpaa/dpaa_eventdev.h
index 583e46ca..8134e6ba 100644
--- a/drivers/event/dpaa/dpaa_eventdev.h
+++ b/drivers/event/dpaa/dpaa_eventdev.h
@@ -12,15 +12,8 @@
 
 #define EVENTDEV_NAME_DPAA_PMD		event_dpaa1
 
-#define EVENTDEV_DRV_LOG(fmt, args...)	\
-		DPAA_EVENTDEV_INFO(fmt, ## args)
-#define EVENTDEV_DRV_FUNC_TRACE()	\
-		DPAA_EVENTDEV_DEBUG("%s() Called:\n", __func__)
-#define EVENTDEV_DRV_ERR(fmt, args...)	\
-		DPAA_EVENTDEV_ERR("%s(): " fmt "\n", __func__, ## args)
-
-#define DPAA_EVENT_MAX_PORTS			8
-#define DPAA_EVENT_MAX_QUEUES			16
+#define DPAA_EVENT_MAX_PORTS			4
+#define DPAA_EVENT_MAX_QUEUES			8
 #define DPAA_EVENT_MIN_DEQUEUE_TIMEOUT	1
 #define DPAA_EVENT_MAX_DEQUEUE_TIMEOUT	(UINT32_MAX - 1)
 #define DPAA_EVENT_MAX_QUEUE_FLOWS		2048
@@ -28,7 +21,7 @@
 #define DPAA_EVENT_MAX_EVENT_PRIORITY_LEVELS	0
 #define DPAA_EVENT_MAX_EVENT_PORT		RTE_MIN(RTE_MAX_LCORE, INT8_MAX)
 #define DPAA_EVENT_MAX_PORT_DEQUEUE_DEPTH	8
-#define DPAA_EVENT_PORT_DEQUEUE_TIMEOUT_NS	100UL
+#define DPAA_EVENT_PORT_DEQUEUE_TIMEOUT_NS	100000UL
 #define DPAA_EVENT_PORT_DEQUEUE_TIMEOUT_INVALID	((uint64_t)-1)
 #define DPAA_EVENT_MAX_PORT_ENQUEUE_DEPTH	1
 #define DPAA_EVENT_MAX_NUM_EVENTS		(INT32_MAX - 1)
@@ -61,7 +54,7 @@ struct dpaa_port {
 	struct dpaa_eventq evq_info[DPAA_EVENT_MAX_QUEUES];
 	uint8_t num_linked_evq;
 	uint8_t is_port_linked;
-	uint64_t timeout;
+	uint64_t timeout_us;
 };
 
 struct dpaa_eventdev {
@@ -72,7 +65,7 @@ struct dpaa_eventdev {
 	uint8_t max_event_queues;
 	uint8_t nb_event_queues;
 	uint8_t nb_event_ports;
-	uint8_t resvd;
+	uint8_t intr_mode;
 	uint32_t nb_event_queue_flows;
 	uint32_t nb_event_port_dequeue_depth;
 	uint32_t nb_event_port_enqueue_depth;
diff --git a/drivers/event/dpaa2/Makefile b/drivers/event/dpaa2/Makefile
index 5e1a6320..e0134cc4 100644
--- a/drivers/event/dpaa2/Makefile
+++ b/drivers/event/dpaa2/Makefile
@@ -21,13 +21,19 @@ CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
 LDLIBS += -lrte_eal -lrte_eventdev
 LDLIBS += -lrte_bus_fslmc -lrte_mempool_dpaa2 -lrte_pmd_dpaa2
 LDLIBS += -lrte_bus_vdev
+LDLIBS += -lrte_common_dpaax
 CFLAGS += -I$(RTE_SDK)/drivers/net/dpaa2
 CFLAGS += -I$(RTE_SDK)/drivers/net/dpaa2/mc
 
+ifeq ($(CONFIG_RTE_LIBRTE_SECURITY),y)
+LDLIBS += -lrte_pmd_dpaa2_sec
+CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec
+endif
+
 # versioning export map
 EXPORT_MAP := rte_pmd_dpaa2_event_version.map
 
-LIBABIVER := 1
+LIBABIVER := 2
 
 # depends on fslmc bus which uses experimental API
 CFLAGS += -DALLOW_EXPERIMENTAL_API
diff --git a/drivers/event/dpaa2/dpaa2_eventdev.c b/drivers/event/dpaa2/dpaa2_eventdev.c
index ea1e5cc6..8d168b02 100644
--- a/drivers/event/dpaa2/dpaa2_eventdev.c
+++ b/drivers/event/dpaa2/dpaa2_eventdev.c
@@ -27,6 +27,7 @@
 #include <rte_pci.h>
 #include <rte_bus_vdev.h>
 #include <rte_ethdev_driver.h>
+#include <rte_cryptodev.h>
 #include <rte_event_eth_rx_adapter.h>
 
 #include <fslmc_vfio.h>
@@ -34,6 +35,9 @@
 #include <dpaa2_hw_mempool.h>
 #include <dpaa2_hw_dpio.h>
 #include <dpaa2_ethdev.h>
+#ifdef RTE_LIBRTE_SECURITY
+#include <dpaa2_sec_event.h>
+#endif
 #include "dpaa2_eventdev.h"
 #include "dpaa2_eventdev_logs.h"
 #include <portal/dpaa2_hw_pvt.h>
@@ -54,34 +58,63 @@ static uint16_t
 dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[],
 			     uint16_t nb_events)
 {
-	struct rte_eventdev *ev_dev =
-			((struct dpaa2_io_portal_t *)port)->eventdev;
-	struct dpaa2_eventdev *priv = ev_dev->data->dev_private;
+
+	struct dpaa2_port *dpaa2_portal = port;
+	struct dpaa2_dpio_dev *dpio_dev;
 	uint32_t queue_id = ev[0].queue_id;
-	struct evq_info_t *evq_info = &priv->evq_info[queue_id];
+	struct dpaa2_eventq *evq_info;
 	uint32_t fqid;
 	struct qbman_swp *swp;
 	struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
 	uint32_t loop, frames_to_send;
 	struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS];
 	uint16_t num_tx = 0;
-	int ret;
-
-	RTE_SET_USED(port);
+	int i, n, ret;
+	uint8_t channel_index;
 
 	if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
+		/* Affine current thread context to a qman portal */
 		ret = dpaa2_affine_qbman_swp();
-		if (ret) {
+		if (ret < 0) {
 			DPAA2_EVENTDEV_ERR("Failure in affining portal");
 			return 0;
 		}
 	}
-
+	/* todo - dpaa2_portal shall have dpio_dev - no per thread variable */
+	dpio_dev = DPAA2_PER_LCORE_DPIO;
 	swp = DPAA2_PER_LCORE_PORTAL;
 
+	if (likely(dpaa2_portal->is_port_linked))
+		goto skip_linking;
+
+	/* Create mapping between portal and channel to receive packets */
+	for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) {
+		evq_info = &dpaa2_portal->evq_info[i];
+		if (!evq_info->event_port)
+			continue;
+
+		ret = dpio_add_static_dequeue_channel(dpio_dev->dpio,
+						      CMD_PRI_LOW,
+						      dpio_dev->token,
+						      evq_info->dpcon->dpcon_id,
+						      &channel_index);
+		if (ret < 0) {
+			DPAA2_EVENTDEV_ERR(
+				"Static dequeue config failed: err(%d)", ret);
+			goto err;
+		}
+
+		qbman_swp_push_set(swp, channel_index, 1);
+		evq_info->dpcon->channel_index = channel_index;
+	}
+	dpaa2_portal->is_port_linked = true;
+
+skip_linking:
+	evq_info = &dpaa2_portal->evq_info[queue_id];
+
 	while (nb_events) {
-		frames_to_send = (nb_events >> 3) ?
-			MAX_TX_RING_SLOTS : nb_events;
+		frames_to_send = (nb_events > dpaa2_eqcr_size) ?
+			dpaa2_eqcr_size : nb_events;
 
 		for (loop = 0; loop < frames_to_send; loop++) {
 			const struct rte_event *event = &ev[num_tx + loop];
@@ -99,14 +132,14 @@ dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[],
 			qbman_eq_desc_set_no_orp(&eqdesc[loop], 0);
 			qbman_eq_desc_set_response(&eqdesc[loop], 0, 0);
 
-			if (event->mbuf->seqn) {
+			if (event->sched_type == RTE_SCHED_TYPE_ATOMIC
+				&& event->mbuf->seqn) {
 				uint8_t dqrr_index = event->mbuf->seqn - 1;
 
 				qbman_eq_desc_set_dca(&eqdesc[loop], 1,
 						      dqrr_index, 0);
 				DPAA2_PER_LCORE_DQRR_SIZE--;
-				DPAA2_PER_LCORE_DQRR_HELD &=
-					~(1 << dqrr_index);
+				DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
 			}
 
 			memset(&fd_arr[loop], 0, sizeof(struct qbman_fd));
@@ -116,7 +149,7 @@ dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[],
 			 * to avoid copy
 			 */
 			struct rte_event *ev_temp = rte_malloc(NULL,
-				sizeof(struct rte_event), 0);
+						sizeof(struct rte_event), 0);
 
 			if (!ev_temp) {
 				if (!loop)
@@ -143,6 +176,18 @@ send_partial:
 	}
 
 	return num_tx;
+err:
+	for (n = 0; n < i; n++) {
+		evq_info = &dpaa2_portal->evq_info[n];
+		if (!evq_info->event_port)
+			continue;
+		qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0);
+		dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0,
+						dpio_dev->token,
+						evq_info->dpcon->dpcon_id);
+	}
+	return 0;
+
 }
 
 static uint16_t
@@ -197,6 +242,7 @@ static void dpaa2_eventdev_process_atomic(struct qbman_swp *swp,
 	ev->mbuf->seqn = dqrr_index + 1;
 	DPAA2_PER_LCORE_DQRR_SIZE++;
 	DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
+	DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf;
 }
 
 static uint16_t
@@ -204,22 +250,53 @@ dpaa2_eventdev_dequeue_burst(void *port, struct rte_event ev[],
 			     uint16_t nb_events, uint64_t timeout_ticks)
 {
 	const struct qbman_result *dq;
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+	struct dpaa2_port *dpaa2_portal = port;
+	struct dpaa2_eventq *evq_info;
 	struct qbman_swp *swp;
 	const struct qbman_fd *fd;
 	struct dpaa2_queue *rxq;
-	int num_pkts = 0, ret, i = 0;
-
-	RTE_SET_USED(port);
+	int num_pkts = 0, ret, i = 0, n;
+	uint8_t channel_index;
 
 	if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
+		/* Affine current thread context to a qman portal */
 		ret = dpaa2_affine_qbman_swp();
-		if (ret) {
+		if (ret < 0) {
 			DPAA2_EVENTDEV_ERR("Failure in affining portal");
 			return 0;
 		}
 	}
+
+	dpio_dev = DPAA2_PER_LCORE_DPIO;
 	swp = DPAA2_PER_LCORE_PORTAL;
 
+	if (likely(dpaa2_portal->is_port_linked))
+		goto skip_linking;
+
+	/* Create mapping between portal and channel to receive packets */
+	for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) {
+		evq_info = &dpaa2_portal->evq_info[i];
+		if (!evq_info->event_port)
+			continue;
+
+		ret = dpio_add_static_dequeue_channel(dpio_dev->dpio,
+						      CMD_PRI_LOW,
+						      dpio_dev->token,
+						      evq_info->dpcon->dpcon_id,
+						      &channel_index);
+		if (ret < 0) {
+			DPAA2_EVENTDEV_ERR(
+				"Static dequeue config failed: err(%d)", ret);
+			goto err;
+		}
+
+		qbman_swp_push_set(swp, channel_index, 1);
+		evq_info->dpcon->channel_index = channel_index;
+	}
+	dpaa2_portal->is_port_linked = true;
+
+skip_linking:
 	/* Check if there are atomic contexts to be released */
 	while (DPAA2_PER_LCORE_DQRR_SIZE) {
 		if (DPAA2_PER_LCORE_DQRR_HELD & (1 << i)) {
@@ -258,6 +335,18 @@ dpaa2_eventdev_dequeue_burst(void *port, struct rte_event ev[],
 	} while (num_pkts < nb_events);
 
 	return num_pkts;
+err:
+	for (n = 0; n < i; n++) {
+		evq_info = &dpaa2_portal->evq_info[n];
+		if (!evq_info->event_port)
+			continue;
+
+		qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0);
+		dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0,
+							dpio_dev->token,
+						evq_info->dpcon->dpcon_id);
+	}
+	return 0;
 }
 
 static uint16_t
@@ -283,7 +372,7 @@ dpaa2_eventdev_info_get(struct rte_eventdev *dev,
 	dev_info->max_dequeue_timeout_ns =
 		DPAA2_EVENT_MAX_DEQUEUE_TIMEOUT;
 	dev_info->dequeue_timeout_ns =
-		DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT;
+		DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS;
 	dev_info->max_event_queues = priv->max_event_queues;
 	dev_info->max_event_queue_flows =
 		DPAA2_EVENT_MAX_QUEUE_FLOWS;
@@ -292,6 +381,9 @@ dpaa2_eventdev_info_get(struct rte_eventdev *dev,
 	dev_info->max_event_priority_levels =
 		DPAA2_EVENT_MAX_EVENT_PRIORITY_LEVELS;
 	dev_info->max_event_ports = rte_fslmc_get_device_count(DPAA2_IO);
+	/* we only support dpio upto number of cores*/
+	if (dev_info->max_event_ports > rte_lcore_count())
+		dev_info->max_event_ports = rte_lcore_count();
 	dev_info->max_event_port_dequeue_depth =
 		DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH;
 	dev_info->max_event_port_enqueue_depth =
@@ -313,7 +405,6 @@ dpaa2_eventdev_configure(const struct rte_eventdev *dev)
 
 	EVENTDEV_INIT_FUNC_TRACE();
 
-	priv->dequeue_timeout_ns = conf->dequeue_timeout_ns;
 	priv->nb_event_queues = conf->nb_event_queues;
 	priv->nb_event_ports = conf->nb_event_ports;
 	priv->nb_event_queue_flows = conf->nb_event_queue_flows;
@@ -321,6 +412,20 @@ dpaa2_eventdev_configure(const struct rte_eventdev *dev)
 	priv->nb_event_port_enqueue_depth = conf->nb_event_port_enqueue_depth;
 	priv->event_dev_cfg = conf->event_dev_cfg;
 
+	/* Check dequeue timeout method is per dequeue or global */
+	if (priv->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) {
+		/*
+		 * Use timeout value as given in dequeue operation.
+		 * So invalidating this timeout value.
+		 */
+		priv->dequeue_timeout_ns = 0;
+
+	} else if (conf->dequeue_timeout_ns == 0) {
+		priv->dequeue_timeout_ns = DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS;
+	} else {
+		priv->dequeue_timeout_ns = conf->dequeue_timeout_ns;
+	}
+
 	DPAA2_EVENTDEV_DEBUG("Configured eventdev devid=%d",
 			     dev->data->dev_id);
 	return 0;
@@ -370,31 +475,39 @@ dpaa2_eventdev_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
 	queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
 }
 
-static void
-dpaa2_eventdev_queue_release(struct rte_eventdev *dev, uint8_t queue_id)
-{
-	EVENTDEV_INIT_FUNC_TRACE();
-
-	RTE_SET_USED(dev);
-	RTE_SET_USED(queue_id);
-}
-
 static int
 dpaa2_eventdev_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
 			   const struct rte_event_queue_conf *queue_conf)
 {
 	struct dpaa2_eventdev *priv = dev->data->dev_private;
-	struct evq_info_t *evq_info =
-		&priv->evq_info[queue_id];
+	struct dpaa2_eventq *evq_info = &priv->evq_info[queue_id];
 
 	EVENTDEV_INIT_FUNC_TRACE();
 
+	switch (queue_conf->schedule_type) {
+	case RTE_SCHED_TYPE_PARALLEL:
+	case RTE_SCHED_TYPE_ATOMIC:
+		break;
+	case RTE_SCHED_TYPE_ORDERED:
+		DPAA2_EVENTDEV_ERR("Schedule type is not supported.");
+		return -1;
+	}
 	evq_info->event_queue_cfg = queue_conf->event_queue_cfg;
+	evq_info->event_queue_id = queue_id;
 
 	return 0;
 }
 
 static void
+dpaa2_eventdev_queue_release(struct rte_eventdev *dev, uint8_t queue_id)
+{
+	EVENTDEV_INIT_FUNC_TRACE();
+
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+}
+
+static void
 dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 			     struct rte_event_port_conf *port_conf)
 {
@@ -402,7 +515,6 @@ dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 
 	RTE_SET_USED(dev);
 	RTE_SET_USED(port_id);
-	RTE_SET_USED(port_conf);
 
 	port_conf->new_event_threshold =
 		DPAA2_EVENT_MAX_NUM_EVENTS;
@@ -413,56 +525,44 @@ dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 	port_conf->disable_implicit_release = 0;
 }
 
-static void
-dpaa2_eventdev_port_release(void *port)
-{
-	EVENTDEV_INIT_FUNC_TRACE();
-
-	RTE_SET_USED(port);
-}
-
 static int
 dpaa2_eventdev_port_setup(struct rte_eventdev *dev, uint8_t port_id,
 			  const struct rte_event_port_conf *port_conf)
 {
+	char event_port_name[32];
+	struct dpaa2_port *portal;
+
 	EVENTDEV_INIT_FUNC_TRACE();
 
 	RTE_SET_USED(port_conf);
 
-	if (!dpaa2_io_portal[port_id].dpio_dev) {
-		dpaa2_io_portal[port_id].dpio_dev =
-				dpaa2_get_qbman_swp(port_id);
-		rte_atomic16_inc(&dpaa2_io_portal[port_id].dpio_dev->ref_count);
-		if (!dpaa2_io_portal[port_id].dpio_dev)
-			return -1;
+	sprintf(event_port_name, "event-port-%d", port_id);
+	portal = rte_malloc(event_port_name, sizeof(struct dpaa2_port), 0);
+	if (!portal) {
+		DPAA2_EVENTDEV_ERR("Memory allocation failure");
+		return -ENOMEM;
 	}
 
-	dpaa2_io_portal[port_id].eventdev = dev;
-	dev->data->ports[port_id] = &dpaa2_io_portal[port_id];
+	memset(portal, 0, sizeof(struct dpaa2_port));
+	dev->data->ports[port_id] = portal;
 	return 0;
 }
 
-static int
-dpaa2_eventdev_port_unlink(struct rte_eventdev *dev, void *port,
-			   uint8_t queues[], uint16_t nb_unlinks)
+static void
+dpaa2_eventdev_port_release(void *port)
 {
-	struct dpaa2_eventdev *priv = dev->data->dev_private;
-	struct dpaa2_io_portal_t *dpaa2_portal = port;
-	struct evq_info_t *evq_info;
-	int i;
+	struct dpaa2_port *portal = port;
 
 	EVENTDEV_INIT_FUNC_TRACE();
 
-	for (i = 0; i < nb_unlinks; i++) {
-		evq_info = &priv->evq_info[queues[i]];
-		qbman_swp_push_set(dpaa2_portal->dpio_dev->sw_portal,
-				   evq_info->dpcon->channel_index, 0);
-		dpio_remove_static_dequeue_channel(dpaa2_portal->dpio_dev->dpio,
-					0, dpaa2_portal->dpio_dev->token,
-			evq_info->dpcon->dpcon_id);
-	}
+	/* TODO: Cleanup is required when ports are in linked state. */
+	if (portal->is_port_linked)
+		DPAA2_EVENTDEV_WARN("Event port must be unlinked before release");
 
-	return (int)nb_unlinks;
+	if (portal)
+		rte_free(portal);
+
+	portal = NULL;
 }
 
 static int
@@ -471,51 +571,71 @@ dpaa2_eventdev_port_link(struct rte_eventdev *dev, void *port,
 			uint16_t nb_links)
 {
 	struct dpaa2_eventdev *priv = dev->data->dev_private;
-	struct dpaa2_io_portal_t *dpaa2_portal = port;
-	struct evq_info_t *evq_info;
-	uint8_t channel_index;
-	int ret, i, n;
+	struct dpaa2_port *dpaa2_portal = port;
+	struct dpaa2_eventq *evq_info;
+	uint16_t i;
 
 	EVENTDEV_INIT_FUNC_TRACE();
 
+	RTE_SET_USED(priorities);
+
 	for (i = 0; i < nb_links; i++) {
 		evq_info = &priv->evq_info[queues[i]];
+		memcpy(&dpaa2_portal->evq_info[queues[i]], evq_info,
+			   sizeof(struct dpaa2_eventq));
+		dpaa2_portal->evq_info[queues[i]].event_port = port;
+		dpaa2_portal->num_linked_evq++;
+	}
 
-		ret = dpio_add_static_dequeue_channel(
-			dpaa2_portal->dpio_dev->dpio,
-			CMD_PRI_LOW, dpaa2_portal->dpio_dev->token,
-			evq_info->dpcon->dpcon_id, &channel_index);
-		if (ret < 0) {
-			DPAA2_EVENTDEV_ERR(
-				"Static dequeue config failed: err(%d)", ret);
-			goto err;
-		}
+	return (int)nb_links;
+}
 
-		qbman_swp_push_set(dpaa2_portal->dpio_dev->sw_portal,
-				   channel_index, 1);
-		evq_info->dpcon->channel_index = channel_index;
-	}
+static int
+dpaa2_eventdev_port_unlink(struct rte_eventdev *dev, void *port,
+			   uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dpaa2_port *dpaa2_portal = port;
+	int i;
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+	struct dpaa2_eventq *evq_info;
+	struct qbman_swp *swp;
 
-	RTE_SET_USED(priorities);
+	EVENTDEV_INIT_FUNC_TRACE();
 
-	return (int)nb_links;
-err:
-	for (n = 0; n < i; n++) {
-		evq_info = &priv->evq_info[queues[n]];
-		qbman_swp_push_set(dpaa2_portal->dpio_dev->sw_portal,
-				   evq_info->dpcon->channel_index, 0);
-		dpio_remove_static_dequeue_channel(dpaa2_portal->dpio_dev->dpio,
-					0, dpaa2_portal->dpio_dev->token,
-			evq_info->dpcon->dpcon_id);
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queues);
+
+	for (i = 0; i < nb_unlinks; i++) {
+		evq_info = &dpaa2_portal->evq_info[queues[i]];
+
+		if (DPAA2_PER_LCORE_DPIO && evq_info->dpcon) {
+			/* todo dpaa2_portal shall have dpio_dev-no per lcore*/
+			dpio_dev = DPAA2_PER_LCORE_DPIO;
+			swp = DPAA2_PER_LCORE_PORTAL;
+
+			qbman_swp_push_set(swp,
+					evq_info->dpcon->channel_index, 0);
+			dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0,
+						dpio_dev->token,
+						evq_info->dpcon->dpcon_id);
+		}
+		memset(evq_info, 0, sizeof(struct dpaa2_eventq));
+		if (dpaa2_portal->num_linked_evq)
+			dpaa2_portal->num_linked_evq--;
 	}
-	return ret;
+
+	if (!dpaa2_portal->num_linked_evq)
+		dpaa2_portal->is_port_linked = false;
+
+	return (int)nb_unlinks;
 }
 
+
 static int
 dpaa2_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
 			     uint64_t *timeout_ticks)
 {
-	uint32_t scale = 1;
+	uint32_t scale = 1000*1000;
 
 	EVENTDEV_INIT_FUNC_TRACE();
 
@@ -677,6 +797,151 @@ dpaa2_eventdev_eth_stop(const struct rte_eventdev *dev,
 	return 0;
 }
 
+#ifdef RTE_LIBRTE_SECURITY
+static int
+dpaa2_eventdev_crypto_caps_get(const struct rte_eventdev *dev,
+			    const struct rte_cryptodev *cdev,
+			    uint32_t *caps)
+{
+	const char *name = cdev->data->name;
+
+	EVENTDEV_INIT_FUNC_TRACE();
+
+	RTE_SET_USED(dev);
+
+	if (!strncmp(name, "dpsec-", 6))
+		*caps = RTE_EVENT_CRYPTO_ADAPTER_DPAA2_CAP;
+	else
+		return -1;
+
+	return 0;
+}
+
+static int
+dpaa2_eventdev_crypto_queue_add_all(const struct rte_eventdev *dev,
+		const struct rte_cryptodev *cryptodev,
+		const struct rte_event *ev)
+{
+	struct dpaa2_eventdev *priv = dev->data->dev_private;
+	uint8_t ev_qid = ev->queue_id;
+	uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id;
+	int i, ret;
+
+	EVENTDEV_INIT_FUNC_TRACE();
+
+	for (i = 0; i < cryptodev->data->nb_queue_pairs; i++) {
+		ret = dpaa2_sec_eventq_attach(cryptodev, i,
+				dpcon_id, ev);
+		if (ret) {
+			DPAA2_EVENTDEV_ERR("dpaa2_sec_eventq_attach failed: ret %d\n",
+				    ret);
+			goto fail;
+		}
+	}
+	return 0;
+fail:
+	for (i = (i - 1); i >= 0 ; i--)
+		dpaa2_sec_eventq_detach(cryptodev, i);
+
+	return ret;
+}
+
+static int
+dpaa2_eventdev_crypto_queue_add(const struct rte_eventdev *dev,
+		const struct rte_cryptodev *cryptodev,
+		int32_t rx_queue_id,
+		const struct rte_event *ev)
+{
+	struct dpaa2_eventdev *priv = dev->data->dev_private;
+	uint8_t ev_qid = ev->queue_id;
+	uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id;
+	int ret;
+
+	EVENTDEV_INIT_FUNC_TRACE();
+
+	if (rx_queue_id == -1)
+		return dpaa2_eventdev_crypto_queue_add_all(dev,
+				cryptodev, ev);
+
+	ret = dpaa2_sec_eventq_attach(cryptodev, rx_queue_id,
+			dpcon_id, ev);
+	if (ret) {
+		DPAA2_EVENTDEV_ERR(
+			"dpaa2_sec_eventq_attach failed: ret: %d\n", ret);
+		return ret;
+	}
+	return 0;
+}
+
+static int
+dpaa2_eventdev_crypto_queue_del_all(const struct rte_eventdev *dev,
+			     const struct rte_cryptodev *cdev)
+{
+	int i, ret;
+
+	EVENTDEV_INIT_FUNC_TRACE();
+
+	RTE_SET_USED(dev);
+
+	for (i = 0; i < cdev->data->nb_queue_pairs; i++) {
+		ret = dpaa2_sec_eventq_detach(cdev, i);
+		if (ret) {
+			DPAA2_EVENTDEV_ERR(
+				"dpaa2_sec_eventq_detach failed:ret %d\n", ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
+dpaa2_eventdev_crypto_queue_del(const struct rte_eventdev *dev,
+			     const struct rte_cryptodev *cryptodev,
+			     int32_t rx_queue_id)
+{
+	int ret;
+
+	EVENTDEV_INIT_FUNC_TRACE();
+
+	if (rx_queue_id == -1)
+		return dpaa2_eventdev_crypto_queue_del_all(dev, cryptodev);
+
+	ret = dpaa2_sec_eventq_detach(cryptodev, rx_queue_id);
+	if (ret) {
+		DPAA2_EVENTDEV_ERR(
+			"dpaa2_sec_eventq_detach failed: ret: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int
+dpaa2_eventdev_crypto_start(const struct rte_eventdev *dev,
+			    const struct rte_cryptodev *cryptodev)
+{
+	EVENTDEV_INIT_FUNC_TRACE();
+
+	RTE_SET_USED(dev);
+	RTE_SET_USED(cryptodev);
+
+	return 0;
+}
+
+static int
+dpaa2_eventdev_crypto_stop(const struct rte_eventdev *dev,
+			   const struct rte_cryptodev *cryptodev)
+{
+	EVENTDEV_INIT_FUNC_TRACE();
+
+	RTE_SET_USED(dev);
+	RTE_SET_USED(cryptodev);
+
+	return 0;
+}
+#endif
+
 static struct rte_eventdev_ops dpaa2_eventdev_ops = {
 	.dev_infos_get    = dpaa2_eventdev_info_get,
 	.dev_configure    = dpaa2_eventdev_configure,
@@ -698,6 +963,13 @@ static struct rte_eventdev_ops dpaa2_eventdev_ops = {
 	.eth_rx_adapter_queue_del = dpaa2_eventdev_eth_queue_del,
 	.eth_rx_adapter_start = dpaa2_eventdev_eth_start,
 	.eth_rx_adapter_stop = dpaa2_eventdev_eth_stop,
+#ifdef RTE_LIBRTE_SECURITY
+	.crypto_adapter_caps_get	= dpaa2_eventdev_crypto_caps_get,
+	.crypto_adapter_queue_pair_add	= dpaa2_eventdev_crypto_queue_add,
+	.crypto_adapter_queue_pair_del	= dpaa2_eventdev_crypto_queue_del,
+	.crypto_adapter_start		= dpaa2_eventdev_crypto_start,
+	.crypto_adapter_stop		= dpaa2_eventdev_crypto_stop,
+#endif
 };
 
 static int
@@ -789,6 +1061,8 @@ dpaa2_eventdev_create(const char *name)
 		priv->max_event_queues++;
 	} while (dpcon_dev && dpci_dev);
 
+	RTE_LOG(INFO, PMD, "%s eventdev created\n", name);
+
 	return 0;
 fail:
 	return -EFAULT;
diff --git a/drivers/event/dpaa2/dpaa2_eventdev.h b/drivers/event/dpaa2/dpaa2_eventdev.h
index 229f66af..c847b3ea 100644
--- a/drivers/event/dpaa2/dpaa2_eventdev.h
+++ b/drivers/event/dpaa2/dpaa2_eventdev.h
@@ -21,6 +21,7 @@
 #define DPAA2_EVENT_MAX_QUEUES			16
 #define DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT		1
 #define DPAA2_EVENT_MAX_DEQUEUE_TIMEOUT		(UINT32_MAX - 1)
+#define DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS	100UL
 #define DPAA2_EVENT_MAX_QUEUE_FLOWS		2048
 #define DPAA2_EVENT_MAX_QUEUE_PRIORITY_LEVELS	8
 #define DPAA2_EVENT_MAX_EVENT_PRIORITY_LEVELS	0
@@ -41,6 +42,15 @@ enum {
 		(RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT | \
 		RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ | \
 		RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID)
+
+/**< Crypto Rx adapter cap to return If the packet transfers from
+ * the cryptodev to eventdev with DPAA2 devices.
+ */
+#define RTE_EVENT_CRYPTO_ADAPTER_DPAA2_CAP \
+		(RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW | \
+		RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND | \
+		RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA)
+
 /**< Ethernet Rx adapter cap to return If the packet transfers from
  * the ethdev to eventdev with DPAA2 devices.
  */
@@ -56,17 +66,27 @@ struct dpaa2_dpcon_dev {
 	uint8_t channel_index;
 };
 
-struct evq_info_t {
+struct dpaa2_eventq {
 	/* DPcon device */
 	struct dpaa2_dpcon_dev *dpcon;
 	/* Attached DPCI device */
 	struct dpaa2_dpci_dev *dpci;
+	/* Mapped event port */
+	struct dpaa2_io_portal_t *event_port;
 	/* Configuration provided by the user */
 	uint32_t event_queue_cfg;
+	uint32_t event_queue_id;
+};
+
+struct dpaa2_port {
+	struct dpaa2_eventq evq_info[DPAA2_EVENT_MAX_QUEUES];
+	uint8_t num_linked_evq;
+	uint8_t is_port_linked;
+	uint64_t timeout_us;
 };
 
 struct dpaa2_eventdev {
-	struct evq_info_t evq_info[DPAA2_EVENT_MAX_QUEUES];
+	struct dpaa2_eventq evq_info[DPAA2_EVENT_MAX_QUEUES];
 	uint32_t dequeue_timeout_ns;
 	uint8_t max_event_queues;
 	uint8_t nb_event_queues;
diff --git a/drivers/event/dpaa2/meson.build b/drivers/event/dpaa2/meson.build
index de7a4615..a0db6fc2 100644
--- a/drivers/event/dpaa2/meson.build
+++ b/drivers/event/dpaa2/meson.build
@@ -1,11 +1,14 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright 2018 NXP
 
+version = 2
+
 if host_machine.system() != 'linux'
 	build = false
 endif
-deps += ['bus_vdev', 'pmd_dpaa2']
+deps += ['bus_vdev', 'pmd_dpaa2', 'pmd_dpaa2_sec']
 sources = files('dpaa2_hw_dpcon.c',
 		'dpaa2_eventdev.c')
 
 allow_experimental_apis = true
+includes += include_directories('../../crypto/dpaa2_sec/')
diff --git a/drivers/event/dsw/Makefile b/drivers/event/dsw/Makefile
new file mode 100644
index 00000000..490ed0b9
--- /dev/null
+++ b/drivers/event/dsw/Makefile
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Ericsson AB
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+LIB = librte_pmd_dsw_event.a
+
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+ifneq ($(CONFIG_RTE_TOOLCHAIN_ICC),y)
+CFLAGS += -Wno-format-nonliteral
+endif
+
+LDLIBS += -lrte_eal
+LDLIBS += -lrte_mbuf
+LDLIBS += -lrte_mempool
+LDLIBS += -lrte_ring
+LDLIBS += -lrte_eventdev
+LDLIBS += -lrte_bus_vdev
+
+LIBABIVER := 1
+
+EXPORT_MAP := rte_pmd_dsw_event_version.map
+
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_DSW_EVENTDEV) += \
+	dsw_evdev.c dsw_event.c dsw_xstats.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/event/dsw/dsw_evdev.c b/drivers/event/dsw/dsw_evdev.c
new file mode 100644
index 00000000..33ba1364
--- /dev/null
+++ b/drivers/event/dsw/dsw_evdev.c
@@ -0,0 +1,435 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Ericsson AB
+ */
+
+#include <stdbool.h>
+
+#include <rte_cycles.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_vdev.h>
+#include <rte_random.h>
+
+#include "dsw_evdev.h"
+
+#define EVENTDEV_NAME_DSW_PMD event_dsw
+
+static int
+dsw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
+	       const struct rte_event_port_conf *conf)
+{
+	struct dsw_evdev *dsw = dsw_pmd_priv(dev);
+	struct dsw_port *port;
+	struct rte_event_ring *in_ring;
+	struct rte_ring *ctl_in_ring;
+	char ring_name[RTE_RING_NAMESIZE];
+
+	port = &dsw->ports[port_id];
+
+	*port = (struct dsw_port) {
+		.id = port_id,
+		.dsw = dsw,
+		.dequeue_depth = conf->dequeue_depth,
+		.enqueue_depth = conf->enqueue_depth,
+		.new_event_threshold = conf->new_event_threshold
+	};
+
+	snprintf(ring_name, sizeof(ring_name), "dsw%d_p%u", dev->data->dev_id,
+		 port_id);
+
+	in_ring = rte_event_ring_create(ring_name, DSW_IN_RING_SIZE,
+					dev->data->socket_id,
+					RING_F_SC_DEQ|RING_F_EXACT_SZ);
+
+	if (in_ring == NULL)
+		return -ENOMEM;
+
+	snprintf(ring_name, sizeof(ring_name), "dswctl%d_p%u",
+		 dev->data->dev_id, port_id);
+
+	ctl_in_ring = rte_ring_create(ring_name, DSW_CTL_IN_RING_SIZE,
+				      dev->data->socket_id,
+				      RING_F_SC_DEQ|RING_F_EXACT_SZ);
+
+	if (ctl_in_ring == NULL) {
+		rte_event_ring_free(in_ring);
+		return -ENOMEM;
+	}
+
+	port->in_ring = in_ring;
+	port->ctl_in_ring = ctl_in_ring;
+
+	rte_atomic16_init(&port->load);
+
+	port->load_update_interval =
+		(DSW_LOAD_UPDATE_INTERVAL * rte_get_timer_hz()) / US_PER_S;
+
+	port->migration_interval =
+		(DSW_MIGRATION_INTERVAL * rte_get_timer_hz()) / US_PER_S;
+
+	dev->data->ports[port_id] = port;
+
+	return 0;
+}
+
+static void
+dsw_port_def_conf(struct rte_eventdev *dev __rte_unused,
+		  uint8_t port_id __rte_unused,
+		  struct rte_event_port_conf *port_conf)
+{
+	*port_conf = (struct rte_event_port_conf) {
+		.new_event_threshold = 1024,
+		.dequeue_depth = DSW_MAX_PORT_DEQUEUE_DEPTH / 4,
+		.enqueue_depth = DSW_MAX_PORT_ENQUEUE_DEPTH / 4
+	};
+}
+
+static void
+dsw_port_release(void *p)
+{
+	struct dsw_port *port = p;
+
+	rte_event_ring_free(port->in_ring);
+	rte_ring_free(port->ctl_in_ring);
+}
+
+static int
+dsw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
+		const struct rte_event_queue_conf *conf)
+{
+	struct dsw_evdev *dsw = dsw_pmd_priv(dev);
+	struct dsw_queue *queue = &dsw->queues[queue_id];
+
+	if (RTE_EVENT_QUEUE_CFG_ALL_TYPES & conf->event_queue_cfg)
+		return -ENOTSUP;
+
+	if (conf->schedule_type == RTE_SCHED_TYPE_ORDERED)
+		return -ENOTSUP;
+
+	/* SINGLE_LINK is better off treated as TYPE_ATOMIC, since it
+	 * avoid the "fake" TYPE_PARALLEL flow_id assignment. Since
+	 * the queue will only have a single serving port, no
+	 * migration will ever happen, so the extra TYPE_ATOMIC
+	 * migration overhead is avoided.
+	 */
+	if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg)
+		queue->schedule_type = RTE_SCHED_TYPE_ATOMIC;
+	else /* atomic or parallel */
+		queue->schedule_type = conf->schedule_type;
+
+	queue->num_serving_ports = 0;
+
+	return 0;
+}
+
+static void
+dsw_queue_def_conf(struct rte_eventdev *dev __rte_unused,
+		   uint8_t queue_id __rte_unused,
+		   struct rte_event_queue_conf *queue_conf)
+{
+	*queue_conf = (struct rte_event_queue_conf) {
+		.nb_atomic_flows = 4096,
+		.schedule_type = RTE_SCHED_TYPE_ATOMIC,
+		.priority = RTE_EVENT_DEV_PRIORITY_NORMAL
+	};
+}
+
+static void
+dsw_queue_release(struct rte_eventdev *dev __rte_unused,
+		  uint8_t queue_id __rte_unused)
+{
+}
+
+static void
+queue_add_port(struct dsw_queue *queue, uint16_t port_id)
+{
+	queue->serving_ports[queue->num_serving_ports] = port_id;
+	queue->num_serving_ports++;
+}
+
+static bool
+queue_remove_port(struct dsw_queue *queue, uint16_t port_id)
+{
+	uint16_t i;
+
+	for (i = 0; i < queue->num_serving_ports; i++)
+		if (queue->serving_ports[i] == port_id) {
+			uint16_t last_idx = queue->num_serving_ports - 1;
+			if (i != last_idx)
+				queue->serving_ports[i] =
+					queue->serving_ports[last_idx];
+			queue->num_serving_ports--;
+			return true;
+		}
+	return false;
+}
+
+static int
+dsw_port_link_unlink(struct rte_eventdev *dev, void *port,
+		     const uint8_t queues[], uint16_t num, bool link)
+{
+	struct dsw_evdev *dsw = dsw_pmd_priv(dev);
+	struct dsw_port *p = port;
+	uint16_t i;
+	uint16_t count = 0;
+
+	for (i = 0; i < num; i++) {
+		uint8_t qid = queues[i];
+		struct dsw_queue *q = &dsw->queues[qid];
+		if (link) {
+			queue_add_port(q, p->id);
+			count++;
+		} else {
+			bool removed = queue_remove_port(q, p->id);
+			if (removed)
+				count++;
+		}
+	}
+
+	return count;
+}
+
+static int
+dsw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[],
+	      const uint8_t priorities[] __rte_unused, uint16_t num)
+{
+	return dsw_port_link_unlink(dev, port, queues, num, true);
+}
+
+static int
+dsw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
+		uint16_t num)
+{
+	return dsw_port_link_unlink(dev, port, queues, num, false);
+}
+
+static void
+dsw_info_get(struct rte_eventdev *dev __rte_unused,
+	     struct rte_event_dev_info *info)
+{
+	*info = (struct rte_event_dev_info) {
+		.driver_name = DSW_PMD_NAME,
+		.max_event_queues = DSW_MAX_QUEUES,
+		.max_event_queue_flows = DSW_MAX_FLOWS,
+		.max_event_queue_priority_levels = 1,
+		.max_event_priority_levels = 1,
+		.max_event_ports = DSW_MAX_PORTS,
+		.max_event_port_dequeue_depth = DSW_MAX_PORT_DEQUEUE_DEPTH,
+		.max_event_port_enqueue_depth = DSW_MAX_PORT_ENQUEUE_DEPTH,
+		.max_num_events = DSW_MAX_EVENTS,
+		.event_dev_cap = RTE_EVENT_DEV_CAP_BURST_MODE|
+		RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED
+	};
+}
+
+static int
+dsw_configure(const struct rte_eventdev *dev)
+{
+	struct dsw_evdev *dsw = dsw_pmd_priv(dev);
+	const struct rte_event_dev_config *conf = &dev->data->dev_conf;
+	int32_t min_max_in_flight;
+
+	dsw->num_ports = conf->nb_event_ports;
+	dsw->num_queues = conf->nb_event_queues;
+
+	/* Avoid a situation where consumer ports are holding all the
+	 * credits, without making use of them.
+	 */
+	min_max_in_flight = conf->nb_event_ports * DSW_PORT_MAX_CREDITS;
+
+	dsw->max_inflight = RTE_MAX(conf->nb_events_limit, min_max_in_flight);
+
+	return 0;
+}
+
+
+static void
+initial_flow_to_port_assignment(struct dsw_evdev *dsw)
+{
+	uint8_t queue_id;
+	for (queue_id = 0; queue_id < dsw->num_queues; queue_id++) {
+		struct dsw_queue *queue = &dsw->queues[queue_id];
+		uint16_t flow_hash;
+		for (flow_hash = 0; flow_hash < DSW_MAX_FLOWS; flow_hash++) {
+			uint8_t port_idx =
+				rte_rand() % queue->num_serving_ports;
+			uint8_t port_id =
+				queue->serving_ports[port_idx];
+			dsw->queues[queue_id].flow_to_port_map[flow_hash] =
+				port_id;
+		}
+	}
+}
+
+static int
+dsw_start(struct rte_eventdev *dev)
+{
+	struct dsw_evdev *dsw = dsw_pmd_priv(dev);
+	uint16_t i;
+	uint64_t now;
+
+	rte_atomic32_init(&dsw->credits_on_loan);
+
+	initial_flow_to_port_assignment(dsw);
+
+	now = rte_get_timer_cycles();
+	for (i = 0; i < dsw->num_ports; i++) {
+		dsw->ports[i].measurement_start = now;
+		dsw->ports[i].busy_start = now;
+	}
+
+	return 0;
+}
+
+static void
+dsw_port_drain_buf(uint8_t dev_id, struct rte_event *buf, uint16_t buf_len,
+		   eventdev_stop_flush_t flush, void *flush_arg)
+{
+	uint16_t i;
+
+	for (i = 0; i < buf_len; i++)
+		flush(dev_id, buf[i], flush_arg);
+}
+
+static void
+dsw_port_drain_paused(uint8_t dev_id, struct dsw_port *port,
+		      eventdev_stop_flush_t flush, void *flush_arg)
+{
+	dsw_port_drain_buf(dev_id, port->paused_events, port->paused_events_len,
+			   flush, flush_arg);
+}
+
+static void
+dsw_port_drain_out(uint8_t dev_id, struct dsw_evdev *dsw, struct dsw_port *port,
+		   eventdev_stop_flush_t flush, void *flush_arg)
+{
+	uint16_t dport_id;
+
+	for (dport_id = 0; dport_id < dsw->num_ports; dport_id++)
+		if (dport_id != port->id)
+			dsw_port_drain_buf(dev_id, port->out_buffer[dport_id],
+					   port->out_buffer_len[dport_id],
+					   flush, flush_arg);
+}
+
+static void
+dsw_port_drain_in_ring(uint8_t dev_id, struct dsw_port *port,
+		       eventdev_stop_flush_t flush, void *flush_arg)
+{
+	struct rte_event ev;
+
+	while (rte_event_ring_dequeue_burst(port->in_ring, &ev, 1, NULL))
+		flush(dev_id, ev, flush_arg);
+}
+
+static void
+dsw_drain(uint8_t dev_id, struct dsw_evdev *dsw,
+	  eventdev_stop_flush_t flush, void *flush_arg)
+{
+	uint16_t port_id;
+
+	if (flush == NULL)
+		return;
+
+	for (port_id = 0; port_id < dsw->num_ports; port_id++) {
+		struct dsw_port *port = &dsw->ports[port_id];
+
+		dsw_port_drain_out(dev_id, dsw, port, flush, flush_arg);
+		dsw_port_drain_paused(dev_id, port, flush, flush_arg);
+		dsw_port_drain_in_ring(dev_id, port, flush, flush_arg);
+	}
+}
+
+static void
+dsw_stop(struct rte_eventdev *dev)
+{
+	struct dsw_evdev *dsw = dsw_pmd_priv(dev);
+	uint8_t dev_id;
+	eventdev_stop_flush_t flush;
+	void *flush_arg;
+
+	dev_id = dev->data->dev_id;
+	flush = dev->dev_ops->dev_stop_flush;
+	flush_arg = dev->data->dev_stop_flush_arg;
+
+	dsw_drain(dev_id, dsw, flush, flush_arg);
+}
+
+static int
+dsw_close(struct rte_eventdev *dev)
+{
+	struct dsw_evdev *dsw = dsw_pmd_priv(dev);
+
+	dsw->num_ports = 0;
+	dsw->num_queues = 0;
+
+	return 0;
+}
+
+static struct rte_eventdev_ops dsw_evdev_ops = {
+	.port_setup = dsw_port_setup,
+	.port_def_conf = dsw_port_def_conf,
+	.port_release = dsw_port_release,
+	.queue_setup = dsw_queue_setup,
+	.queue_def_conf = dsw_queue_def_conf,
+	.queue_release = dsw_queue_release,
+	.port_link = dsw_port_link,
+	.port_unlink = dsw_port_unlink,
+	.dev_infos_get = dsw_info_get,
+	.dev_configure = dsw_configure,
+	.dev_start = dsw_start,
+	.dev_stop = dsw_stop,
+	.dev_close = dsw_close,
+	.xstats_get = dsw_xstats_get,
+	.xstats_get_names = dsw_xstats_get_names,
+	.xstats_get_by_name = dsw_xstats_get_by_name
+};
+
+static int
+dsw_probe(struct rte_vdev_device *vdev)
+{
+	const char *name;
+	struct rte_eventdev *dev;
+	struct dsw_evdev *dsw;
+
+	name = rte_vdev_device_name(vdev);
+
+	dev = rte_event_pmd_vdev_init(name, sizeof(struct dsw_evdev),
+				      rte_socket_id());
+	if (dev == NULL)
+		return -EFAULT;
+
+	dev->dev_ops = &dsw_evdev_ops;
+	dev->enqueue = dsw_event_enqueue;
+	dev->enqueue_burst = dsw_event_enqueue_burst;
+	dev->enqueue_new_burst = dsw_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dsw_event_enqueue_forward_burst;
+	dev->dequeue = dsw_event_dequeue;
+	dev->dequeue_burst = dsw_event_dequeue_burst;
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	dsw = dev->data->dev_private;
+	dsw->data = dev->data;
+
+	return 0;
+}
+
+static int
+dsw_remove(struct rte_vdev_device *vdev)
+{
+	const char *name;
+
+	name = rte_vdev_device_name(vdev);
+	if (name == NULL)
+		return -EINVAL;
+
+	return rte_event_pmd_vdev_uninit(name);
+}
+
+static struct rte_vdev_driver evdev_dsw_pmd_drv = {
+	.probe = dsw_probe,
+	.remove = dsw_remove
+};
+
+RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_DSW_PMD, evdev_dsw_pmd_drv);
diff --git a/drivers/event/dsw/dsw_evdev.h b/drivers/event/dsw/dsw_evdev.h
new file mode 100644
index 00000000..dc28ab12
--- /dev/null
+++ b/drivers/event/dsw/dsw_evdev.h
@@ -0,0 +1,279 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Ericsson AB
+ */
+
+#ifndef _DSW_EVDEV_H_
+#define _DSW_EVDEV_H_
+
+#include <rte_event_ring.h>
+#include <rte_eventdev.h>
+
+#define DSW_PMD_NAME RTE_STR(event_dsw)
+
+/* Code changes are required to allow more ports. */
+#define DSW_MAX_PORTS (64)
+#define DSW_MAX_PORT_DEQUEUE_DEPTH (128)
+#define DSW_MAX_PORT_ENQUEUE_DEPTH (128)
+#define DSW_MAX_PORT_OUT_BUFFER (32)
+
+#define DSW_MAX_QUEUES (16)
+
+#define DSW_MAX_EVENTS (16384)
+
+/* Code changes are required to allow more flows than 32k. */
+#define DSW_MAX_FLOWS_BITS (15)
+#define DSW_MAX_FLOWS (1<<(DSW_MAX_FLOWS_BITS))
+#define DSW_MAX_FLOWS_MASK (DSW_MAX_FLOWS-1)
+
+/* Eventdev RTE_SCHED_TYPE_PARALLEL doesn't have a concept of flows,
+ * but the 'dsw' scheduler (more or less) randomly assign flow id to
+ * events on parallel queues, to be able to reuse some of the
+ * migration mechanism and scheduling logic from
+ * RTE_SCHED_TYPE_ATOMIC. By moving one of the parallel "flows" from a
+ * particular port, the likely-hood of events being scheduled to this
+ * port is reduced, and thus a kind of statistical load balancing is
+ * achieved.
+ */
+#define DSW_PARALLEL_FLOWS (1024)
+
+/* 'Background tasks' are polling the control rings for *
+ *  migration-related messages, or flush the output buffer (so
+ *  buffered events doesn't linger too long). Shouldn't be too low,
+ *  since the system won't benefit from the 'batching' effects from
+ *  the output buffer, and shouldn't be too high, since it will make
+ *  buffered events linger too long in case the port goes idle.
+ */
+#define DSW_MAX_PORT_OPS_PER_BG_TASK (128)
+
+/* Avoid making small 'loans' from the central in-flight event credit
+ * pool, to improve efficiency.
+ */
+#define DSW_MIN_CREDIT_LOAN (64)
+#define DSW_PORT_MAX_CREDITS (2*DSW_MIN_CREDIT_LOAN)
+#define DSW_PORT_MIN_CREDITS (DSW_MIN_CREDIT_LOAN)
+
+/* The rings are dimensioned so that all in-flight events can reside
+ * on any one of the port rings, to avoid the trouble of having to
+ * care about the case where there's no room on the destination port's
+ * input ring.
+ */
+#define DSW_IN_RING_SIZE (DSW_MAX_EVENTS)
+
+#define DSW_MAX_LOAD (INT16_MAX)
+#define DSW_LOAD_FROM_PERCENT(x) ((int16_t)(((x)*DSW_MAX_LOAD)/100))
+#define DSW_LOAD_TO_PERCENT(x) ((100*x)/DSW_MAX_LOAD)
+
+/* The thought behind keeping the load update interval shorter than
+ * the migration interval is that the load from newly migrated flows
+ * should 'show up' on the load measurement before new migrations are
+ * considered. This is to avoid having too many flows, from too many
+ * source ports, to be migrated too quickly to a lightly loaded port -
+ * in particular since this might cause the system to oscillate.
+ */
+#define DSW_LOAD_UPDATE_INTERVAL (DSW_MIGRATION_INTERVAL/4)
+#define DSW_OLD_LOAD_WEIGHT (1)
+
+/* The minimum time (in us) between two flow migrations. What puts an
+ * upper limit on the actual migration rate is primarily the pace in
+ * which the ports send and receive control messages, which in turn is
+ * largely a function of how much cycles are spent the processing of
+ * an event burst.
+ */
+#define DSW_MIGRATION_INTERVAL (1000)
+#define DSW_MIN_SOURCE_LOAD_FOR_MIGRATION (DSW_LOAD_FROM_PERCENT(70))
+#define DSW_MAX_TARGET_LOAD_FOR_MIGRATION (DSW_LOAD_FROM_PERCENT(95))
+
+#define DSW_MAX_EVENTS_RECORDED (128)
+
+/* Only one outstanding migration per port is allowed */
+#define DSW_MAX_PAUSED_FLOWS (DSW_MAX_PORTS)
+
+/* Enough room for paus request/confirm and unpaus request/confirm for
+ * all possible senders.
+ */
+#define DSW_CTL_IN_RING_SIZE ((DSW_MAX_PORTS-1)*4)
+
+/* With DSW_SORT_DEQUEUED enabled, the scheduler will, at the point of
+ * dequeue(), arrange events so that events with the same flow id on
+ * the same queue forms a back-to-back "burst", and also so that such
+ * bursts of different flow ids, but on the same queue, also come
+ * consecutively. All this in an attempt to improve data and
+ * instruction cache usage for the application, at the cost of a
+ * scheduler overhead increase.
+ */
+
+/* #define DSW_SORT_DEQUEUED */
+
+struct dsw_queue_flow {
+	uint8_t queue_id;
+	uint16_t flow_hash;
+};
+
+enum dsw_migration_state {
+	DSW_MIGRATION_STATE_IDLE,
+	DSW_MIGRATION_STATE_PAUSING,
+	DSW_MIGRATION_STATE_FORWARDING,
+	DSW_MIGRATION_STATE_UNPAUSING
+};
+
+struct dsw_port {
+	uint16_t id;
+
+	/* Keeping a pointer here to avoid container_of() calls, which
+	 * are expensive since they are very frequent and will result
+	 * in an integer multiplication (since the port id is an index
+	 * into the dsw_evdev port array).
+	 */
+	struct dsw_evdev *dsw;
+
+	uint16_t dequeue_depth;
+	uint16_t enqueue_depth;
+
+	int32_t inflight_credits;
+
+	int32_t new_event_threshold;
+
+	uint16_t pending_releases;
+
+	uint16_t next_parallel_flow_id;
+
+	uint16_t ops_since_bg_task;
+
+	/* most recent 'background' processing */
+	uint64_t last_bg;
+
+	/* For port load measurement. */
+	uint64_t next_load_update;
+	uint64_t load_update_interval;
+	uint64_t measurement_start;
+	uint64_t busy_start;
+	uint64_t busy_cycles;
+	uint64_t total_busy_cycles;
+
+	/* For the ctl interface and flow migration mechanism. */
+	uint64_t next_migration;
+	uint64_t migration_interval;
+	enum dsw_migration_state migration_state;
+
+	uint64_t migration_start;
+	uint64_t migrations;
+	uint64_t migration_latency;
+
+	uint8_t migration_target_port_id;
+	struct dsw_queue_flow migration_target_qf;
+	uint8_t cfm_cnt;
+
+	uint16_t paused_flows_len;
+	struct dsw_queue_flow paused_flows[DSW_MAX_PAUSED_FLOWS];
+
+	/* In a very contrived worst case all inflight events can be
+	 * laying around paused here.
+	 */
+	uint16_t paused_events_len;
+	struct rte_event paused_events[DSW_MAX_EVENTS];
+
+	uint16_t seen_events_len;
+	uint16_t seen_events_idx;
+	struct dsw_queue_flow seen_events[DSW_MAX_EVENTS_RECORDED];
+
+	uint64_t new_enqueued;
+	uint64_t forward_enqueued;
+	uint64_t release_enqueued;
+	uint64_t queue_enqueued[DSW_MAX_QUEUES];
+
+	uint64_t dequeued;
+	uint64_t queue_dequeued[DSW_MAX_QUEUES];
+
+	uint16_t out_buffer_len[DSW_MAX_PORTS];
+	struct rte_event out_buffer[DSW_MAX_PORTS][DSW_MAX_PORT_OUT_BUFFER];
+
+	uint16_t in_buffer_len;
+	uint16_t in_buffer_start;
+	/* This buffer may contain events that were read up from the
+	 * in_ring during the flow migration process.
+	 */
+	struct rte_event in_buffer[DSW_MAX_EVENTS];
+
+	struct rte_event_ring *in_ring __rte_cache_aligned;
+
+	struct rte_ring *ctl_in_ring __rte_cache_aligned;
+
+	/* Estimate of current port load. */
+	rte_atomic16_t load __rte_cache_aligned;
+} __rte_cache_aligned;
+
+struct dsw_queue {
+	uint8_t schedule_type;
+	uint8_t serving_ports[DSW_MAX_PORTS];
+	uint16_t num_serving_ports;
+
+	uint8_t flow_to_port_map[DSW_MAX_FLOWS] __rte_cache_aligned;
+};
+
+struct dsw_evdev {
+	struct rte_eventdev_data *data;
+
+	struct dsw_port ports[DSW_MAX_PORTS];
+	uint16_t num_ports;
+	struct dsw_queue queues[DSW_MAX_QUEUES];
+	uint8_t num_queues;
+	int32_t max_inflight;
+
+	rte_atomic32_t credits_on_loan __rte_cache_aligned;
+};
+
+#define DSW_CTL_PAUS_REQ (0)
+#define DSW_CTL_UNPAUS_REQ (1)
+#define DSW_CTL_CFM (2)
+
+/* sizeof(struct dsw_ctl_msg) must be equal or less than
+ * sizeof(void *), to fit on the control ring.
+ */
+struct dsw_ctl_msg {
+	uint8_t type:2;
+	uint8_t originating_port_id:6;
+	uint8_t queue_id;
+	uint16_t flow_hash;
+} __rte_packed;
+
+uint16_t dsw_event_enqueue(void *port, const struct rte_event *event);
+uint16_t dsw_event_enqueue_burst(void *port,
+				 const struct rte_event events[],
+				 uint16_t events_len);
+uint16_t dsw_event_enqueue_new_burst(void *port,
+				     const struct rte_event events[],
+				     uint16_t events_len);
+uint16_t dsw_event_enqueue_forward_burst(void *port,
+					 const struct rte_event events[],
+					 uint16_t events_len);
+
+uint16_t dsw_event_dequeue(void *port, struct rte_event *ev, uint64_t wait);
+uint16_t dsw_event_dequeue_burst(void *port, struct rte_event *events,
+				 uint16_t num, uint64_t wait);
+
+int dsw_xstats_get_names(const struct rte_eventdev *dev,
+			 enum rte_event_dev_xstats_mode mode,
+			 uint8_t queue_port_id,
+			 struct rte_event_dev_xstats_name *xstats_names,
+			 unsigned int *ids, unsigned int size);
+int dsw_xstats_get(const struct rte_eventdev *dev,
+		   enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		   const unsigned int ids[], uint64_t values[], unsigned int n);
+uint64_t dsw_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id);
+
+static inline struct dsw_evdev *
+dsw_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+#define DSW_LOG_DP(level, fmt, args...)					\
+	RTE_LOG_DP(level, EVENTDEV, "[%s] %s() line %u: " fmt,		\
+		   DSW_PMD_NAME,					\
+		   __func__, __LINE__, ## args)
+
+#define DSW_LOG_DP_PORT(level, port_id, fmt, args...)		\
+	DSW_LOG_DP(level, "<Port %d> " fmt, port_id, ## args)
+
+#endif
diff --git a/drivers/event/dsw/dsw_event.c b/drivers/event/dsw/dsw_event.c
new file mode 100644
index 00000000..61a66fab
--- /dev/null
+++ b/drivers/event/dsw/dsw_event.c
@@ -0,0 +1,1253 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Ericsson AB
+ */
+
+#include "dsw_evdev.h"
+
+#ifdef DSW_SORT_DEQUEUED
+#include "dsw_sort.h"
+#endif
+
+#include <stdbool.h>
+#include <string.h>
+
+#include <rte_atomic.h>
+#include <rte_cycles.h>
+#include <rte_memcpy.h>
+#include <rte_random.h>
+
+static bool
+dsw_port_acquire_credits(struct dsw_evdev *dsw, struct dsw_port *port,
+			 int32_t credits)
+{
+	int32_t inflight_credits = port->inflight_credits;
+	int32_t missing_credits = credits - inflight_credits;
+	int32_t total_on_loan;
+	int32_t available;
+	int32_t acquired_credits;
+	int32_t new_total_on_loan;
+
+	if (likely(missing_credits <= 0)) {
+		port->inflight_credits -= credits;
+		return true;
+	}
+
+	total_on_loan = rte_atomic32_read(&dsw->credits_on_loan);
+	available = dsw->max_inflight - total_on_loan;
+	acquired_credits = RTE_MAX(missing_credits, DSW_PORT_MIN_CREDITS);
+
+	if (available < acquired_credits)
+		return false;
+
+	/* This is a race, no locks are involved, and thus some other
+	 * thread can allocate tokens in between the check and the
+	 * allocation.
+	 */
+	new_total_on_loan = rte_atomic32_add_return(&dsw->credits_on_loan,
+						    acquired_credits);
+
+	if (unlikely(new_total_on_loan > dsw->max_inflight)) {
+		/* Some other port took the last credits */
+		rte_atomic32_sub(&dsw->credits_on_loan, acquired_credits);
+		return false;
+	}
+
+	DSW_LOG_DP_PORT(DEBUG, port->id, "Acquired %d tokens from pool.\n",
+			acquired_credits);
+
+	port->inflight_credits += acquired_credits;
+	port->inflight_credits -= credits;
+
+	return true;
+}
+
+static void
+dsw_port_return_credits(struct dsw_evdev *dsw, struct dsw_port *port,
+			int32_t credits)
+{
+	port->inflight_credits += credits;
+
+	if (unlikely(port->inflight_credits > DSW_PORT_MAX_CREDITS)) {
+		int32_t leave_credits = DSW_PORT_MIN_CREDITS;
+		int32_t return_credits =
+			port->inflight_credits - leave_credits;
+
+		port->inflight_credits = leave_credits;
+
+		rte_atomic32_sub(&dsw->credits_on_loan, return_credits);
+
+		DSW_LOG_DP_PORT(DEBUG, port->id,
+				"Returned %d tokens to pool.\n",
+				return_credits);
+	}
+}
+
+static void
+dsw_port_enqueue_stats(struct dsw_port *port, uint16_t num_new,
+		       uint16_t num_forward, uint16_t num_release)
+{
+	port->new_enqueued += num_new;
+	port->forward_enqueued += num_forward;
+	port->release_enqueued += num_release;
+}
+
+static void
+dsw_port_queue_enqueue_stats(struct dsw_port *source_port, uint8_t queue_id)
+{
+	source_port->queue_enqueued[queue_id]++;
+}
+
+static void
+dsw_port_dequeue_stats(struct dsw_port *port, uint16_t num)
+{
+	port->dequeued += num;
+}
+
+static void
+dsw_port_queue_dequeued_stats(struct dsw_port *source_port, uint8_t queue_id)
+{
+	source_port->queue_dequeued[queue_id]++;
+}
+
+static void
+dsw_port_load_record(struct dsw_port *port, unsigned int dequeued)
+{
+	if (dequeued > 0 && port->busy_start == 0)
+		/* work period begins */
+		port->busy_start = rte_get_timer_cycles();
+	else if (dequeued == 0 && port->busy_start > 0) {
+		/* work period ends */
+		uint64_t work_period =
+			rte_get_timer_cycles() - port->busy_start;
+		port->busy_cycles += work_period;
+		port->busy_start = 0;
+	}
+}
+
+static int16_t
+dsw_port_load_close_period(struct dsw_port *port, uint64_t now)
+{
+	uint64_t passed = now - port->measurement_start;
+	uint64_t busy_cycles = port->busy_cycles;
+
+	if (port->busy_start > 0) {
+		busy_cycles += (now - port->busy_start);
+		port->busy_start = now;
+	}
+
+	int16_t load = (DSW_MAX_LOAD * busy_cycles) / passed;
+
+	port->measurement_start = now;
+	port->busy_cycles = 0;
+
+	port->total_busy_cycles += busy_cycles;
+
+	return load;
+}
+
+static void
+dsw_port_load_update(struct dsw_port *port, uint64_t now)
+{
+	int16_t old_load;
+	int16_t period_load;
+	int16_t new_load;
+
+	old_load = rte_atomic16_read(&port->load);
+
+	period_load = dsw_port_load_close_period(port, now);
+
+	new_load = (period_load + old_load*DSW_OLD_LOAD_WEIGHT) /
+		(DSW_OLD_LOAD_WEIGHT+1);
+
+	rte_atomic16_set(&port->load, new_load);
+}
+
+static void
+dsw_port_consider_load_update(struct dsw_port *port, uint64_t now)
+{
+	if (now < port->next_load_update)
+		return;
+
+	port->next_load_update = now + port->load_update_interval;
+
+	dsw_port_load_update(port, now);
+}
+
+static void
+dsw_port_ctl_enqueue(struct dsw_port *port, struct dsw_ctl_msg *msg)
+{
+	void *raw_msg;
+
+	memcpy(&raw_msg, msg, sizeof(*msg));
+
+	/* there's always room on the ring */
+	while (rte_ring_enqueue(port->ctl_in_ring, raw_msg) != 0)
+		rte_pause();
+}
+
+static int
+dsw_port_ctl_dequeue(struct dsw_port *port, struct dsw_ctl_msg *msg)
+{
+	void *raw_msg;
+	int rc;
+
+	rc = rte_ring_dequeue(port->ctl_in_ring, &raw_msg);
+
+	if (rc == 0)
+		memcpy(msg, &raw_msg, sizeof(*msg));
+
+	return rc;
+}
+
+static void
+dsw_port_ctl_broadcast(struct dsw_evdev *dsw, struct dsw_port *source_port,
+		       uint8_t type, uint8_t queue_id, uint16_t flow_hash)
+{
+	uint16_t port_id;
+	struct dsw_ctl_msg msg = {
+		.type = type,
+		.originating_port_id = source_port->id,
+		.queue_id = queue_id,
+		.flow_hash = flow_hash
+	};
+
+	for (port_id = 0; port_id < dsw->num_ports; port_id++)
+		if (port_id != source_port->id)
+			dsw_port_ctl_enqueue(&dsw->ports[port_id], &msg);
+}
+
+static bool
+dsw_port_is_flow_paused(struct dsw_port *port, uint8_t queue_id,
+			uint16_t flow_hash)
+{
+	uint16_t i;
+
+	for (i = 0; i < port->paused_flows_len; i++) {
+		struct dsw_queue_flow *qf = &port->paused_flows[i];
+		if (qf->queue_id == queue_id &&
+		    qf->flow_hash == flow_hash)
+			return true;
+	}
+	return false;
+}
+
+static void
+dsw_port_add_paused_flow(struct dsw_port *port, uint8_t queue_id,
+			 uint16_t paused_flow_hash)
+{
+	port->paused_flows[port->paused_flows_len] = (struct dsw_queue_flow) {
+		.queue_id = queue_id,
+		.flow_hash = paused_flow_hash
+	};
+	port->paused_flows_len++;
+}
+
+static void
+dsw_port_remove_paused_flow(struct dsw_port *port, uint8_t queue_id,
+			    uint16_t paused_flow_hash)
+{
+	uint16_t i;
+
+	for (i = 0; i < port->paused_flows_len; i++) {
+		struct dsw_queue_flow *qf = &port->paused_flows[i];
+
+		if (qf->queue_id == queue_id &&
+		    qf->flow_hash == paused_flow_hash) {
+			uint16_t last_idx = port->paused_flows_len-1;
+			if (i != last_idx)
+				port->paused_flows[i] =
+					port->paused_flows[last_idx];
+			port->paused_flows_len--;
+			break;
+		}
+	}
+}
+
+static void
+dsw_port_flush_out_buffers(struct dsw_evdev *dsw, struct dsw_port *source_port);
+
+static void
+dsw_port_handle_pause_flow(struct dsw_evdev *dsw, struct dsw_port *port,
+			   uint8_t originating_port_id, uint8_t queue_id,
+			   uint16_t paused_flow_hash)
+{
+	struct dsw_ctl_msg cfm = {
+		.type = DSW_CTL_CFM,
+		.originating_port_id = port->id,
+		.queue_id = queue_id,
+		.flow_hash = paused_flow_hash
+	};
+
+	DSW_LOG_DP_PORT(DEBUG, port->id, "Pausing queue_id %d flow_hash %d.\n",
+			queue_id, paused_flow_hash);
+
+	/* There might be already-scheduled events belonging to the
+	 * paused flow in the output buffers.
+	 */
+	dsw_port_flush_out_buffers(dsw, port);
+
+	dsw_port_add_paused_flow(port, queue_id, paused_flow_hash);
+
+	/* Make sure any stores to the original port's in_ring is seen
+	 * before the ctl message.
+	 */
+	rte_smp_wmb();
+
+	dsw_port_ctl_enqueue(&dsw->ports[originating_port_id], &cfm);
+}
+
+static void
+dsw_find_lowest_load_port(uint8_t *port_ids, uint16_t num_port_ids,
+			  uint8_t exclude_port_id, int16_t *port_loads,
+			  uint8_t *target_port_id, int16_t *target_load)
+{
+	int16_t candidate_port_id = -1;
+	int16_t candidate_load = DSW_MAX_LOAD;
+	uint16_t i;
+
+	for (i = 0; i < num_port_ids; i++) {
+		uint8_t port_id = port_ids[i];
+		if (port_id != exclude_port_id) {
+			int16_t load = port_loads[port_id];
+			if (candidate_port_id == -1 ||
+			    load < candidate_load) {
+				candidate_port_id = port_id;
+				candidate_load = load;
+			}
+		}
+	}
+	*target_port_id = candidate_port_id;
+	*target_load = candidate_load;
+}
+
+struct dsw_queue_flow_burst {
+	struct dsw_queue_flow queue_flow;
+	uint16_t count;
+};
+
+static inline int
+dsw_cmp_burst(const void *v_burst_a, const void *v_burst_b)
+{
+	const struct dsw_queue_flow_burst *burst_a = v_burst_a;
+	const struct dsw_queue_flow_burst *burst_b = v_burst_b;
+
+	int a_count = burst_a->count;
+	int b_count = burst_b->count;
+
+	return a_count - b_count;
+}
+
+#define DSW_QF_TO_INT(_qf)					\
+	((int)((((_qf)->queue_id)<<16)|((_qf)->flow_hash)))
+
+static inline int
+dsw_cmp_qf(const void *v_qf_a, const void *v_qf_b)
+{
+	const struct dsw_queue_flow *qf_a = v_qf_a;
+	const struct dsw_queue_flow *qf_b = v_qf_b;
+
+	return DSW_QF_TO_INT(qf_a) - DSW_QF_TO_INT(qf_b);
+}
+
+static uint16_t
+dsw_sort_qfs_to_bursts(struct dsw_queue_flow *qfs, uint16_t qfs_len,
+		       struct dsw_queue_flow_burst *bursts)
+{
+	uint16_t i;
+	struct dsw_queue_flow_burst *current_burst = NULL;
+	uint16_t num_bursts = 0;
+
+	/* We don't need the stable property, and the list is likely
+	 * large enough for qsort() to outperform dsw_stable_sort(),
+	 * so we use qsort() here.
+	 */
+	qsort(qfs, qfs_len, sizeof(qfs[0]), dsw_cmp_qf);
+
+	/* arrange the (now-consecutive) events into bursts */
+	for (i = 0; i < qfs_len; i++) {
+		if (i == 0 ||
+		    dsw_cmp_qf(&qfs[i], &current_burst->queue_flow) != 0) {
+			current_burst = &bursts[num_bursts];
+			current_burst->queue_flow = qfs[i];
+			current_burst->count = 0;
+			num_bursts++;
+		}
+		current_burst->count++;
+	}
+
+	qsort(bursts, num_bursts, sizeof(bursts[0]), dsw_cmp_burst);
+
+	return num_bursts;
+}
+
+static bool
+dsw_retrieve_port_loads(struct dsw_evdev *dsw, int16_t *port_loads,
+			int16_t load_limit)
+{
+	bool below_limit = false;
+	uint16_t i;
+
+	for (i = 0; i < dsw->num_ports; i++) {
+		int16_t load = rte_atomic16_read(&dsw->ports[i].load);
+		if (load < load_limit)
+			below_limit = true;
+		port_loads[i] = load;
+	}
+	return below_limit;
+}
+
+static bool
+dsw_select_migration_target(struct dsw_evdev *dsw,
+			    struct dsw_port *source_port,
+			    struct dsw_queue_flow_burst *bursts,
+			    uint16_t num_bursts, int16_t *port_loads,
+			    int16_t max_load, struct dsw_queue_flow *target_qf,
+			    uint8_t *target_port_id)
+{
+	uint16_t source_load = port_loads[source_port->id];
+	uint16_t i;
+
+	for (i = 0; i < num_bursts; i++) {
+		struct dsw_queue_flow *qf = &bursts[i].queue_flow;
+
+		if (dsw_port_is_flow_paused(source_port, qf->queue_id,
+					    qf->flow_hash))
+			continue;
+
+		struct dsw_queue *queue = &dsw->queues[qf->queue_id];
+		int16_t target_load;
+
+		dsw_find_lowest_load_port(queue->serving_ports,
+					  queue->num_serving_ports,
+					  source_port->id, port_loads,
+					  target_port_id, &target_load);
+
+		if (target_load < source_load &&
+		    target_load < max_load) {
+			*target_qf = *qf;
+			return true;
+		}
+	}
+
+	DSW_LOG_DP_PORT(DEBUG, source_port->id, "For the %d flows considered, "
+			"no target port found with load less than %d.\n",
+			num_bursts, DSW_LOAD_TO_PERCENT(max_load));
+
+	return false;
+}
+
+static uint8_t
+dsw_schedule(struct dsw_evdev *dsw, uint8_t queue_id, uint16_t flow_hash)
+{
+	struct dsw_queue *queue = &dsw->queues[queue_id];
+	uint8_t port_id;
+
+	if (queue->num_serving_ports > 1)
+		port_id = queue->flow_to_port_map[flow_hash];
+	else
+		/* A single-link queue, or atomic/ordered/parallel but
+		 * with just a single serving port.
+		 */
+		port_id = queue->serving_ports[0];
+
+	DSW_LOG_DP(DEBUG, "Event with queue_id %d flow_hash %d is scheduled "
+		   "to port %d.\n", queue_id, flow_hash, port_id);
+
+	return port_id;
+}
+
+static void
+dsw_port_transmit_buffered(struct dsw_evdev *dsw, struct dsw_port *source_port,
+			   uint8_t dest_port_id)
+{
+	struct dsw_port *dest_port = &(dsw->ports[dest_port_id]);
+	uint16_t *buffer_len = &source_port->out_buffer_len[dest_port_id];
+	struct rte_event *buffer = source_port->out_buffer[dest_port_id];
+	uint16_t enqueued = 0;
+
+	if (*buffer_len == 0)
+		return;
+
+	/* The rings are dimensioned to fit all in-flight events (even
+	 * on a single ring), so looping will work.
+	 */
+	do {
+		enqueued +=
+			rte_event_ring_enqueue_burst(dest_port->in_ring,
+						     buffer+enqueued,
+						     *buffer_len-enqueued,
+						     NULL);
+	} while (unlikely(enqueued != *buffer_len));
+
+	(*buffer_len) = 0;
+}
+
+static uint16_t
+dsw_port_get_parallel_flow_id(struct dsw_port *port)
+{
+	uint16_t flow_id = port->next_parallel_flow_id;
+
+	port->next_parallel_flow_id =
+		(port->next_parallel_flow_id + 1) % DSW_PARALLEL_FLOWS;
+
+	return flow_id;
+}
+
+static void
+dsw_port_buffer_paused(struct dsw_port *port,
+		       const struct rte_event *paused_event)
+{
+	port->paused_events[port->paused_events_len] = *paused_event;
+	port->paused_events_len++;
+}
+
+static void
+dsw_port_buffer_non_paused(struct dsw_evdev *dsw, struct dsw_port *source_port,
+			   uint8_t dest_port_id, const struct rte_event *event)
+{
+	struct rte_event *buffer = source_port->out_buffer[dest_port_id];
+	uint16_t *buffer_len = &source_port->out_buffer_len[dest_port_id];
+
+	if (*buffer_len == DSW_MAX_PORT_OUT_BUFFER)
+		dsw_port_transmit_buffered(dsw, source_port, dest_port_id);
+
+	buffer[*buffer_len] = *event;
+
+	(*buffer_len)++;
+}
+
+#define DSW_FLOW_ID_BITS (24)
+static uint16_t
+dsw_flow_id_hash(uint32_t flow_id)
+{
+	uint16_t hash = 0;
+	uint16_t offset = 0;
+
+	do {
+		hash ^= ((flow_id >> offset) & DSW_MAX_FLOWS_MASK);
+		offset += DSW_MAX_FLOWS_BITS;
+	} while (offset < DSW_FLOW_ID_BITS);
+
+	return hash;
+}
+
+static void
+dsw_port_buffer_parallel(struct dsw_evdev *dsw, struct dsw_port *source_port,
+			 struct rte_event event)
+{
+	uint8_t dest_port_id;
+
+	event.flow_id = dsw_port_get_parallel_flow_id(source_port);
+
+	dest_port_id = dsw_schedule(dsw, event.queue_id,
+				    dsw_flow_id_hash(event.flow_id));
+
+	dsw_port_buffer_non_paused(dsw, source_port, dest_port_id, &event);
+}
+
+static void
+dsw_port_buffer_event(struct dsw_evdev *dsw, struct dsw_port *source_port,
+		      const struct rte_event *event)
+{
+	uint16_t flow_hash;
+	uint8_t dest_port_id;
+
+	if (unlikely(dsw->queues[event->queue_id].schedule_type ==
+		     RTE_SCHED_TYPE_PARALLEL)) {
+		dsw_port_buffer_parallel(dsw, source_port, *event);
+		return;
+	}
+
+	flow_hash = dsw_flow_id_hash(event->flow_id);
+
+	if (unlikely(dsw_port_is_flow_paused(source_port, event->queue_id,
+					     flow_hash))) {
+		dsw_port_buffer_paused(source_port, event);
+		return;
+	}
+
+	dest_port_id = dsw_schedule(dsw, event->queue_id, flow_hash);
+
+	dsw_port_buffer_non_paused(dsw, source_port, dest_port_id, event);
+}
+
+static void
+dsw_port_flush_paused_events(struct dsw_evdev *dsw,
+			     struct dsw_port *source_port,
+			     uint8_t queue_id, uint16_t paused_flow_hash)
+{
+	uint16_t paused_events_len = source_port->paused_events_len;
+	struct rte_event paused_events[paused_events_len];
+	uint8_t dest_port_id;
+	uint16_t i;
+
+	if (paused_events_len == 0)
+		return;
+
+	if (dsw_port_is_flow_paused(source_port, queue_id, paused_flow_hash))
+		return;
+
+	rte_memcpy(paused_events, source_port->paused_events,
+		   paused_events_len * sizeof(struct rte_event));
+
+	source_port->paused_events_len = 0;
+
+	dest_port_id = dsw_schedule(dsw, queue_id, paused_flow_hash);
+
+	for (i = 0; i < paused_events_len; i++) {
+		struct rte_event *event = &paused_events[i];
+		uint16_t flow_hash;
+
+		flow_hash = dsw_flow_id_hash(event->flow_id);
+
+		if (event->queue_id == queue_id &&
+		    flow_hash == paused_flow_hash)
+			dsw_port_buffer_non_paused(dsw, source_port,
+						   dest_port_id, event);
+		else
+			dsw_port_buffer_paused(source_port, event);
+	}
+}
+
+static void
+dsw_port_migration_stats(struct dsw_port *port)
+{
+	uint64_t migration_latency;
+
+	migration_latency = (rte_get_timer_cycles() - port->migration_start);
+	port->migration_latency += migration_latency;
+	port->migrations++;
+}
+
+static void
+dsw_port_end_migration(struct dsw_evdev *dsw, struct dsw_port *port)
+{
+	uint8_t queue_id = port->migration_target_qf.queue_id;
+	uint16_t flow_hash = port->migration_target_qf.flow_hash;
+
+	port->migration_state = DSW_MIGRATION_STATE_IDLE;
+	port->seen_events_len = 0;
+
+	dsw_port_migration_stats(port);
+
+	if (dsw->queues[queue_id].schedule_type != RTE_SCHED_TYPE_PARALLEL) {
+		dsw_port_remove_paused_flow(port, queue_id, flow_hash);
+		dsw_port_flush_paused_events(dsw, port, queue_id, flow_hash);
+	}
+
+	DSW_LOG_DP_PORT(DEBUG, port->id, "Migration completed for queue_id "
+			"%d flow_hash %d.\n", queue_id, flow_hash);
+}
+
+static void
+dsw_port_consider_migration(struct dsw_evdev *dsw,
+			    struct dsw_port *source_port,
+			    uint64_t now)
+{
+	bool any_port_below_limit;
+	struct dsw_queue_flow *seen_events = source_port->seen_events;
+	uint16_t seen_events_len = source_port->seen_events_len;
+	struct dsw_queue_flow_burst bursts[DSW_MAX_EVENTS_RECORDED];
+	uint16_t num_bursts;
+	int16_t source_port_load;
+	int16_t port_loads[dsw->num_ports];
+
+	if (now < source_port->next_migration)
+		return;
+
+	if (dsw->num_ports == 1)
+		return;
+
+	DSW_LOG_DP_PORT(DEBUG, source_port->id, "Considering migration.\n");
+
+	/* Randomize interval to avoid having all threads considering
+	 * migration at the same in point in time, which might lead to
+	 * all choosing the same target port.
+	 */
+	source_port->next_migration = now +
+		source_port->migration_interval / 2 +
+		rte_rand() % source_port->migration_interval;
+
+	if (source_port->migration_state != DSW_MIGRATION_STATE_IDLE) {
+		DSW_LOG_DP_PORT(DEBUG, source_port->id,
+				"Migration already in progress.\n");
+		return;
+	}
+
+	/* For simplicity, avoid migration in the unlikely case there
+	 * is still events to consume in the in_buffer (from the last
+	 * migration).
+	 */
+	if (source_port->in_buffer_len > 0) {
+		DSW_LOG_DP_PORT(DEBUG, source_port->id, "There are still "
+				"events in the input buffer.\n");
+		return;
+	}
+
+	source_port_load = rte_atomic16_read(&source_port->load);
+	if (source_port_load < DSW_MIN_SOURCE_LOAD_FOR_MIGRATION) {
+		DSW_LOG_DP_PORT(DEBUG, source_port->id,
+				"Load %d is below threshold level %d.\n",
+				DSW_LOAD_TO_PERCENT(source_port_load),
+		       DSW_LOAD_TO_PERCENT(DSW_MIN_SOURCE_LOAD_FOR_MIGRATION));
+		return;
+	}
+
+	/* Avoid starting any expensive operations (sorting etc), in
+	 * case of a scenario with all ports above the load limit.
+	 */
+	any_port_below_limit =
+		dsw_retrieve_port_loads(dsw, port_loads,
+					DSW_MAX_TARGET_LOAD_FOR_MIGRATION);
+	if (!any_port_below_limit) {
+		DSW_LOG_DP_PORT(DEBUG, source_port->id,
+				"Candidate target ports are all too highly "
+				"loaded.\n");
+		return;
+	}
+
+	/* Sort flows into 'bursts' to allow attempting to migrating
+	 * small (but still active) flows first - this it to avoid
+	 * having large flows moving around the worker cores too much
+	 * (to avoid cache misses, among other things). Of course, the
+	 * number of recorded events (queue+flow ids) are limited, and
+	 * provides only a snapshot, so only so many conclusions can
+	 * be drawn from this data.
+	 */
+	num_bursts = dsw_sort_qfs_to_bursts(seen_events, seen_events_len,
+					    bursts);
+	/* For non-big-little systems, there's no point in moving the
+	 * only (known) flow.
+	 */
+	if (num_bursts < 2) {
+		DSW_LOG_DP_PORT(DEBUG, source_port->id, "Only a single flow "
+				"queue_id %d flow_hash %d has been seen.\n",
+				bursts[0].queue_flow.queue_id,
+				bursts[0].queue_flow.flow_hash);
+		return;
+	}
+
+	/* The strategy is to first try to find a flow to move to a
+	 * port with low load (below the migration-attempt
+	 * threshold). If that fails, we try to find a port which is
+	 * below the max threshold, and also less loaded than this
+	 * port is.
+	 */
+	if (!dsw_select_migration_target(dsw, source_port, bursts, num_bursts,
+					 port_loads,
+					 DSW_MIN_SOURCE_LOAD_FOR_MIGRATION,
+					 &source_port->migration_target_qf,
+					 &source_port->migration_target_port_id)
+	    &&
+	    !dsw_select_migration_target(dsw, source_port, bursts, num_bursts,
+					 port_loads,
+					 DSW_MAX_TARGET_LOAD_FOR_MIGRATION,
+					 &source_port->migration_target_qf,
+				       &source_port->migration_target_port_id))
+		return;
+
+	DSW_LOG_DP_PORT(DEBUG, source_port->id, "Migrating queue_id %d "
+			"flow_hash %d from port %d to port %d.\n",
+			source_port->migration_target_qf.queue_id,
+			source_port->migration_target_qf.flow_hash,
+			source_port->id, source_port->migration_target_port_id);
+
+	/* We have a winner. */
+
+	source_port->migration_state = DSW_MIGRATION_STATE_PAUSING;
+	source_port->migration_start = rte_get_timer_cycles();
+
+	/* No need to go through the whole pause procedure for
+	 * parallel queues, since atomic/ordered semantics need not to
+	 * be maintained.
+	 */
+
+	if (dsw->queues[source_port->migration_target_qf.queue_id].schedule_type
+	    == RTE_SCHED_TYPE_PARALLEL) {
+		uint8_t queue_id = source_port->migration_target_qf.queue_id;
+		uint16_t flow_hash = source_port->migration_target_qf.flow_hash;
+		uint8_t dest_port_id = source_port->migration_target_port_id;
+
+		/* Single byte-sized stores are always atomic. */
+		dsw->queues[queue_id].flow_to_port_map[flow_hash] =
+			dest_port_id;
+		rte_smp_wmb();
+
+		dsw_port_end_migration(dsw, source_port);
+
+		return;
+	}
+
+	/* There might be 'loopback' events already scheduled in the
+	 * output buffers.
+	 */
+	dsw_port_flush_out_buffers(dsw, source_port);
+
+	dsw_port_add_paused_flow(source_port,
+				 source_port->migration_target_qf.queue_id,
+				 source_port->migration_target_qf.flow_hash);
+
+	dsw_port_ctl_broadcast(dsw, source_port, DSW_CTL_PAUS_REQ,
+			       source_port->migration_target_qf.queue_id,
+			       source_port->migration_target_qf.flow_hash);
+	source_port->cfm_cnt = 0;
+}
+
+static void
+dsw_port_flush_paused_events(struct dsw_evdev *dsw,
+			     struct dsw_port *source_port,
+			     uint8_t queue_id, uint16_t paused_flow_hash);
+
+static void
+dsw_port_handle_unpause_flow(struct dsw_evdev *dsw, struct dsw_port *port,
+			     uint8_t originating_port_id, uint8_t queue_id,
+			     uint16_t paused_flow_hash)
+{
+	struct dsw_ctl_msg cfm = {
+		.type = DSW_CTL_CFM,
+		.originating_port_id = port->id,
+		.queue_id = queue_id,
+		.flow_hash = paused_flow_hash
+	};
+
+	DSW_LOG_DP_PORT(DEBUG, port->id, "Un-pausing queue_id %d flow_hash %d.\n",
+			queue_id, paused_flow_hash);
+
+	dsw_port_remove_paused_flow(port, queue_id, paused_flow_hash);
+
+	rte_smp_rmb();
+
+	dsw_port_ctl_enqueue(&dsw->ports[originating_port_id], &cfm);
+
+	dsw_port_flush_paused_events(dsw, port, queue_id, paused_flow_hash);
+}
+
+#define FORWARD_BURST_SIZE (32)
+
+static void
+dsw_port_forward_migrated_flow(struct dsw_port *source_port,
+			       struct rte_event_ring *dest_ring,
+			       uint8_t queue_id,
+			       uint16_t flow_hash)
+{
+	uint16_t events_left;
+
+	/* Control ring message should been seen before the ring count
+	 * is read on the port's in_ring.
+	 */
+	rte_smp_rmb();
+
+	events_left = rte_event_ring_count(source_port->in_ring);
+
+	while (events_left > 0) {
+		uint16_t in_burst_size =
+			RTE_MIN(FORWARD_BURST_SIZE, events_left);
+		struct rte_event in_burst[in_burst_size];
+		uint16_t in_len;
+		uint16_t i;
+
+		in_len = rte_event_ring_dequeue_burst(source_port->in_ring,
+						      in_burst,
+						      in_burst_size, NULL);
+		/* No need to care about bursting forwarded events (to
+		 * the destination port's in_ring), since migration
+		 * doesn't happen very often, and also the majority of
+		 * the dequeued events will likely *not* be forwarded.
+		 */
+		for (i = 0; i < in_len; i++) {
+			struct rte_event *e = &in_burst[i];
+			if (e->queue_id == queue_id &&
+			    dsw_flow_id_hash(e->flow_id) == flow_hash) {
+				while (rte_event_ring_enqueue_burst(dest_ring,
+								    e, 1,
+								    NULL) != 1)
+					rte_pause();
+			} else {
+				uint16_t last_idx = source_port->in_buffer_len;
+				source_port->in_buffer[last_idx] = *e;
+				source_port->in_buffer_len++;
+			}
+		}
+
+		events_left -= in_len;
+	}
+}
+
+static void
+dsw_port_move_migrating_flow(struct dsw_evdev *dsw,
+			     struct dsw_port *source_port)
+{
+	uint8_t queue_id = source_port->migration_target_qf.queue_id;
+	uint16_t flow_hash = source_port->migration_target_qf.flow_hash;
+	uint8_t dest_port_id = source_port->migration_target_port_id;
+	struct dsw_port *dest_port = &dsw->ports[dest_port_id];
+
+	dsw_port_flush_out_buffers(dsw, source_port);
+
+	rte_smp_wmb();
+
+	dsw->queues[queue_id].flow_to_port_map[flow_hash] =
+		dest_port_id;
+
+	dsw_port_forward_migrated_flow(source_port, dest_port->in_ring,
+				       queue_id, flow_hash);
+
+	/* Flow table update and migration destination port's enqueues
+	 * must be seen before the control message.
+	 */
+	rte_smp_wmb();
+
+	dsw_port_ctl_broadcast(dsw, source_port, DSW_CTL_UNPAUS_REQ, queue_id,
+			       flow_hash);
+	source_port->cfm_cnt = 0;
+	source_port->migration_state = DSW_MIGRATION_STATE_UNPAUSING;
+}
+
+static void
+dsw_port_handle_confirm(struct dsw_evdev *dsw, struct dsw_port *port)
+{
+	port->cfm_cnt++;
+
+	if (port->cfm_cnt == (dsw->num_ports-1)) {
+		switch (port->migration_state) {
+		case DSW_MIGRATION_STATE_PAUSING:
+			DSW_LOG_DP_PORT(DEBUG, port->id, "Going into forwarding "
+					"migration state.\n");
+			port->migration_state = DSW_MIGRATION_STATE_FORWARDING;
+			break;
+		case DSW_MIGRATION_STATE_UNPAUSING:
+			dsw_port_end_migration(dsw, port);
+			break;
+		default:
+			RTE_ASSERT(0);
+			break;
+		}
+	}
+}
+
+static void
+dsw_port_ctl_process(struct dsw_evdev *dsw, struct dsw_port *port)
+{
+	struct dsw_ctl_msg msg;
+
+	/* So any table loads happens before the ring dequeue, in the
+	 * case of a 'paus' message.
+	 */
+	rte_smp_rmb();
+
+	if (dsw_port_ctl_dequeue(port, &msg) == 0) {
+		switch (msg.type) {
+		case DSW_CTL_PAUS_REQ:
+			dsw_port_handle_pause_flow(dsw, port,
+						   msg.originating_port_id,
+						   msg.queue_id, msg.flow_hash);
+			break;
+		case DSW_CTL_UNPAUS_REQ:
+			dsw_port_handle_unpause_flow(dsw, port,
+						     msg.originating_port_id,
+						     msg.queue_id,
+						     msg.flow_hash);
+			break;
+		case DSW_CTL_CFM:
+			dsw_port_handle_confirm(dsw, port);
+			break;
+		}
+	}
+}
+
+static void
+dsw_port_note_op(struct dsw_port *port, uint16_t num_events)
+{
+	/* To pull the control ring reasonbly often on busy ports,
+	 * each dequeued/enqueued event is considered an 'op' too.
+	 */
+	port->ops_since_bg_task += (num_events+1);
+}
+
+static void
+dsw_port_bg_process(struct dsw_evdev *dsw, struct dsw_port *port)
+{
+	if (unlikely(port->migration_state == DSW_MIGRATION_STATE_FORWARDING &&
+		     port->pending_releases == 0))
+		dsw_port_move_migrating_flow(dsw, port);
+
+	/* Polling the control ring is relatively inexpensive, and
+	 * polling it often helps bringing down migration latency, so
+	 * do this for every iteration.
+	 */
+	dsw_port_ctl_process(dsw, port);
+
+	/* To avoid considering migration and flushing output buffers
+	 * on every dequeue/enqueue call, the scheduler only performs
+	 * such 'background' tasks every nth
+	 * (i.e. DSW_MAX_PORT_OPS_PER_BG_TASK) operation.
+	 */
+	if (unlikely(port->ops_since_bg_task >= DSW_MAX_PORT_OPS_PER_BG_TASK)) {
+		uint64_t now;
+
+		now = rte_get_timer_cycles();
+
+		port->last_bg = now;
+
+		/* Logic to avoid having events linger in the output
+		 * buffer too long.
+		 */
+		dsw_port_flush_out_buffers(dsw, port);
+
+		dsw_port_consider_load_update(port, now);
+
+		dsw_port_consider_migration(dsw, port, now);
+
+		port->ops_since_bg_task = 0;
+	}
+}
+
+static void
+dsw_port_flush_out_buffers(struct dsw_evdev *dsw, struct dsw_port *source_port)
+{
+	uint16_t dest_port_id;
+
+	for (dest_port_id = 0; dest_port_id < dsw->num_ports; dest_port_id++)
+		dsw_port_transmit_buffered(dsw, source_port, dest_port_id);
+}
+
+uint16_t
+dsw_event_enqueue(void *port, const struct rte_event *ev)
+{
+	return dsw_event_enqueue_burst(port, ev, unlikely(ev == NULL) ? 0 : 1);
+}
+
+static __rte_always_inline uint16_t
+dsw_event_enqueue_burst_generic(void *port, const struct rte_event events[],
+				uint16_t events_len, bool op_types_known,
+				uint16_t num_new, uint16_t num_release,
+				uint16_t num_non_release)
+{
+	struct dsw_port *source_port = port;
+	struct dsw_evdev *dsw = source_port->dsw;
+	bool enough_credits;
+	uint16_t i;
+
+	DSW_LOG_DP_PORT(DEBUG, source_port->id, "Attempting to enqueue %d "
+			"events to port %d.\n", events_len, source_port->id);
+
+	dsw_port_bg_process(dsw, source_port);
+
+	/* XXX: For performance (=ring efficiency) reasons, the
+	 * scheduler relies on internal non-ring buffers instead of
+	 * immediately sending the event to the destination ring. For
+	 * a producer that doesn't intend to produce or consume any
+	 * more events, the scheduler provides a way to flush the
+	 * buffer, by means of doing an enqueue of zero events. In
+	 * addition, a port cannot be left "unattended" (e.g. unused)
+	 * for long periods of time, since that would stall
+	 * migration. Eventdev API extensions to provide a cleaner way
+	 * to archieve both of these functions should be
+	 * considered.
+	 */
+	if (unlikely(events_len == 0)) {
+		dsw_port_note_op(source_port, DSW_MAX_PORT_OPS_PER_BG_TASK);
+		return 0;
+	}
+
+	if (unlikely(events_len > source_port->enqueue_depth))
+		events_len = source_port->enqueue_depth;
+
+	dsw_port_note_op(source_port, events_len);
+
+	if (!op_types_known)
+		for (i = 0; i < events_len; i++) {
+			switch (events[i].op) {
+			case RTE_EVENT_OP_RELEASE:
+				num_release++;
+				break;
+			case RTE_EVENT_OP_NEW:
+				num_new++;
+				/* Falls through. */
+			default:
+				num_non_release++;
+				break;
+			}
+		}
+
+	/* Technically, we could allow the non-new events up to the
+	 * first new event in the array into the system, but for
+	 * simplicity reasons, we deny the whole burst if the port is
+	 * above the water mark.
+	 */
+	if (unlikely(num_new > 0 && rte_atomic32_read(&dsw->credits_on_loan) >
+		     source_port->new_event_threshold))
+		return 0;
+
+	enough_credits = dsw_port_acquire_credits(dsw, source_port,
+						  num_non_release);
+	if (unlikely(!enough_credits))
+		return 0;
+
+	source_port->pending_releases -= num_release;
+
+	dsw_port_enqueue_stats(source_port, num_new,
+			       num_non_release-num_new, num_release);
+
+	for (i = 0; i < events_len; i++) {
+		const struct rte_event *event = &events[i];
+
+		if (likely(num_release == 0 ||
+			   event->op != RTE_EVENT_OP_RELEASE))
+			dsw_port_buffer_event(dsw, source_port, event);
+		dsw_port_queue_enqueue_stats(source_port, event->queue_id);
+	}
+
+	DSW_LOG_DP_PORT(DEBUG, source_port->id, "%d non-release events "
+			"accepted.\n", num_non_release);
+
+	return num_non_release;
+}
+
+uint16_t
+dsw_event_enqueue_burst(void *port, const struct rte_event events[],
+			uint16_t events_len)
+{
+	return dsw_event_enqueue_burst_generic(port, events, events_len, false,
+					       0, 0, 0);
+}
+
+uint16_t
+dsw_event_enqueue_new_burst(void *port, const struct rte_event events[],
+			    uint16_t events_len)
+{
+	return dsw_event_enqueue_burst_generic(port, events, events_len, true,
+					       events_len, 0, events_len);
+}
+
+uint16_t
+dsw_event_enqueue_forward_burst(void *port, const struct rte_event events[],
+				uint16_t events_len)
+{
+	return dsw_event_enqueue_burst_generic(port, events, events_len, true,
+					       0, 0, events_len);
+}
+
+uint16_t
+dsw_event_dequeue(void *port, struct rte_event *events, uint64_t wait)
+{
+	return dsw_event_dequeue_burst(port, events, 1, wait);
+}
+
+static void
+dsw_port_record_seen_events(struct dsw_port *port, struct rte_event *events,
+			    uint16_t num)
+{
+	uint16_t i;
+
+	dsw_port_dequeue_stats(port, num);
+
+	for (i = 0; i < num; i++) {
+		uint16_t l_idx = port->seen_events_idx;
+		struct dsw_queue_flow *qf = &port->seen_events[l_idx];
+		struct rte_event *event = &events[i];
+		qf->queue_id = event->queue_id;
+		qf->flow_hash = dsw_flow_id_hash(event->flow_id);
+
+		port->seen_events_idx = (l_idx+1) % DSW_MAX_EVENTS_RECORDED;
+
+		dsw_port_queue_dequeued_stats(port, event->queue_id);
+	}
+
+	if (unlikely(port->seen_events_len != DSW_MAX_EVENTS_RECORDED))
+		port->seen_events_len =
+			RTE_MIN(port->seen_events_len + num,
+				DSW_MAX_EVENTS_RECORDED);
+}
+
+#ifdef DSW_SORT_DEQUEUED
+
+#define DSW_EVENT_TO_INT(_event)				\
+	((int)((((_event)->queue_id)<<16)|((_event)->flow_id)))
+
+static inline int
+dsw_cmp_event(const void *v_event_a, const void *v_event_b)
+{
+	const struct rte_event *event_a = v_event_a;
+	const struct rte_event *event_b = v_event_b;
+
+	return DSW_EVENT_TO_INT(event_a) - DSW_EVENT_TO_INT(event_b);
+}
+#endif
+
+static uint16_t
+dsw_port_dequeue_burst(struct dsw_port *port, struct rte_event *events,
+		       uint16_t num)
+{
+	struct dsw_port *source_port = port;
+	struct dsw_evdev *dsw = source_port->dsw;
+
+	dsw_port_ctl_process(dsw, source_port);
+
+	if (unlikely(port->in_buffer_len > 0)) {
+		uint16_t dequeued = RTE_MIN(num, port->in_buffer_len);
+
+		rte_memcpy(events, &port->in_buffer[port->in_buffer_start],
+			   dequeued * sizeof(struct rte_event));
+
+		port->in_buffer_start += dequeued;
+		port->in_buffer_len -= dequeued;
+
+		if (port->in_buffer_len == 0)
+			port->in_buffer_start = 0;
+
+		return dequeued;
+	}
+
+	return rte_event_ring_dequeue_burst(port->in_ring, events, num, NULL);
+}
+
+uint16_t
+dsw_event_dequeue_burst(void *port, struct rte_event *events, uint16_t num,
+			uint64_t wait __rte_unused)
+{
+	struct dsw_port *source_port = port;
+	struct dsw_evdev *dsw = source_port->dsw;
+	uint16_t dequeued;
+
+	source_port->pending_releases = 0;
+
+	dsw_port_bg_process(dsw, source_port);
+
+	if (unlikely(num > source_port->dequeue_depth))
+		num = source_port->dequeue_depth;
+
+	dequeued = dsw_port_dequeue_burst(source_port, events, num);
+
+	source_port->pending_releases = dequeued;
+
+	dsw_port_load_record(source_port, dequeued);
+
+	dsw_port_note_op(source_port, dequeued);
+
+	if (dequeued > 0) {
+		DSW_LOG_DP_PORT(DEBUG, source_port->id, "Dequeued %d events.\n",
+				dequeued);
+
+		dsw_port_return_credits(dsw, source_port, dequeued);
+
+		/* One potential optimization one might think of is to
+		 * add a migration state (prior to 'pausing'), and
+		 * only record seen events when the port is in this
+		 * state (and transit to 'pausing' when enough events
+		 * have been gathered). However, that schema doesn't
+		 * seem to improve performance.
+		 */
+		dsw_port_record_seen_events(port, events, dequeued);
+	}
+	/* XXX: Assuming the port can't produce any more work,
+	 *	consider flushing the output buffer, on dequeued ==
+	 *	0.
+	 */
+
+#ifdef DSW_SORT_DEQUEUED
+	dsw_stable_sort(events, dequeued, sizeof(events[0]), dsw_cmp_event);
+#endif
+
+	return dequeued;
+}
diff --git a/drivers/event/dsw/dsw_sort.h b/drivers/event/dsw/dsw_sort.h
new file mode 100644
index 00000000..609767fd
--- /dev/null
+++ b/drivers/event/dsw/dsw_sort.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Ericsson AB
+ */
+
+#ifndef _DSW_SORT_
+#define _DSW_SORT_
+
+#include <string.h>
+
+#include <rte_common.h>
+
+#define DSW_ARY_ELEM_PTR(_ary, _idx, _elem_size)	\
+	RTE_PTR_ADD(_ary, (_idx) * (_elem_size))
+
+#define DSW_ARY_ELEM_SWAP(_ary, _a_idx, _b_idx, _elem_size)		\
+	do {								\
+		char tmp[_elem_size];					\
+		void *_a_ptr = DSW_ARY_ELEM_PTR(_ary, _a_idx, _elem_size); \
+		void *_b_ptr = DSW_ARY_ELEM_PTR(_ary, _b_idx, _elem_size); \
+		memcpy(tmp, _a_ptr, _elem_size);			\
+		memcpy(_a_ptr, _b_ptr, _elem_size);			\
+		memcpy(_b_ptr, tmp, _elem_size);			\
+	} while (0)
+
+static inline void
+dsw_insertion_sort(void *ary, uint16_t len, uint16_t elem_size,
+		   int (*cmp_fn)(const void *, const void *))
+{
+	uint16_t i;
+
+	for (i = 1; i < len; i++) {
+		uint16_t j;
+		for (j = i; j > 0 &&
+			     cmp_fn(DSW_ARY_ELEM_PTR(ary, j-1, elem_size),
+				    DSW_ARY_ELEM_PTR(ary, j, elem_size)) > 0;
+		     j--)
+			DSW_ARY_ELEM_SWAP(ary, j, j-1, elem_size);
+	}
+}
+
+static inline void
+dsw_stable_sort(void *ary, uint16_t len, uint16_t elem_size,
+		int (*cmp_fn)(const void *, const void *))
+{
+	dsw_insertion_sort(ary, len, elem_size, cmp_fn);
+}
+
+#endif
diff --git a/drivers/event/dsw/dsw_xstats.c b/drivers/event/dsw/dsw_xstats.c
new file mode 100644
index 00000000..bf2eec52
--- /dev/null
+++ b/drivers/event/dsw/dsw_xstats.c
@@ -0,0 +1,288 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Ericsson AB
+ */
+
+#include "dsw_evdev.h"
+
+#include <stdbool.h>
+#include <string.h>
+
+#include <rte_debug.h>
+
+/* The high bits in the xstats id is used to store an additional
+ * parameter (beyond the queue or port id already in the xstats
+ * interface).
+ */
+#define DSW_XSTATS_ID_PARAM_BITS (8)
+#define DSW_XSTATS_ID_STAT_BITS					\
+	(sizeof(unsigned int)*CHAR_BIT - DSW_XSTATS_ID_PARAM_BITS)
+#define DSW_XSTATS_ID_STAT_MASK ((1 << DSW_XSTATS_ID_STAT_BITS) - 1)
+
+#define DSW_XSTATS_ID_GET_PARAM(id)		\
+	((id)>>DSW_XSTATS_ID_STAT_BITS)
+
+#define DSW_XSTATS_ID_GET_STAT(id)		\
+	((id) & DSW_XSTATS_ID_STAT_MASK)
+
+#define DSW_XSTATS_ID_CREATE(id, param_value)			\
+	(((param_value) << DSW_XSTATS_ID_STAT_BITS) | id)
+
+typedef
+uint64_t (*dsw_xstats_dev_get_value_fn)(struct dsw_evdev *dsw);
+
+struct dsw_xstat_dev {
+	const char *name;
+	dsw_xstats_dev_get_value_fn get_value_fn;
+};
+
+typedef
+uint64_t (*dsw_xstats_port_get_value_fn)(struct dsw_evdev *dsw,
+					 uint8_t port_id, uint8_t queue_id);
+
+struct dsw_xstats_port {
+	const char *name_fmt;
+	dsw_xstats_port_get_value_fn get_value_fn;
+	bool per_queue;
+};
+
+static uint64_t
+dsw_xstats_dev_credits_on_loan(struct dsw_evdev *dsw)
+{
+	return rte_atomic32_read(&dsw->credits_on_loan);
+}
+
+static struct dsw_xstat_dev dsw_dev_xstats[] = {
+	{ "dev_credits_on_loan", dsw_xstats_dev_credits_on_loan }
+};
+
+#define DSW_GEN_PORT_ACCESS_FN(_variable)				\
+	static uint64_t							\
+	dsw_xstats_port_get_ ## _variable(struct dsw_evdev *dsw,	\
+					  uint8_t port_id,		\
+					  uint8_t queue_id __rte_unused) \
+	{								\
+		return dsw->ports[port_id]._variable;			\
+	}
+
+DSW_GEN_PORT_ACCESS_FN(new_enqueued)
+DSW_GEN_PORT_ACCESS_FN(forward_enqueued)
+DSW_GEN_PORT_ACCESS_FN(release_enqueued)
+
+static uint64_t
+dsw_xstats_port_get_queue_enqueued(struct dsw_evdev *dsw, uint8_t port_id,
+				   uint8_t queue_id)
+{
+	return dsw->ports[port_id].queue_enqueued[queue_id];
+}
+
+DSW_GEN_PORT_ACCESS_FN(dequeued)
+
+static uint64_t
+dsw_xstats_port_get_queue_dequeued(struct dsw_evdev *dsw, uint8_t port_id,
+				   uint8_t queue_id)
+{
+	return dsw->ports[port_id].queue_dequeued[queue_id];
+}
+
+DSW_GEN_PORT_ACCESS_FN(migrations)
+
+static uint64_t
+dsw_xstats_port_get_migration_latency(struct dsw_evdev *dsw, uint8_t port_id,
+				      uint8_t queue_id __rte_unused)
+{
+	uint64_t total_latency = dsw->ports[port_id].migration_latency;
+	uint64_t num_migrations = dsw->ports[port_id].migrations;
+
+	return num_migrations > 0 ? total_latency / num_migrations : 0;
+}
+
+static uint64_t
+dsw_xstats_port_get_event_proc_latency(struct dsw_evdev *dsw, uint8_t port_id,
+				       uint8_t queue_id __rte_unused)
+{
+	uint64_t total_busy_cycles =
+		dsw->ports[port_id].total_busy_cycles;
+	uint64_t dequeued =
+		dsw->ports[port_id].dequeued;
+
+	return dequeued > 0 ? total_busy_cycles / dequeued : 0;
+}
+
+DSW_GEN_PORT_ACCESS_FN(inflight_credits)
+
+static uint64_t
+dsw_xstats_port_get_load(struct dsw_evdev *dsw, uint8_t port_id,
+			 uint8_t queue_id __rte_unused)
+{
+	int16_t load;
+
+	load = rte_atomic16_read(&dsw->ports[port_id].load);
+
+	return DSW_LOAD_TO_PERCENT(load);
+}
+
+DSW_GEN_PORT_ACCESS_FN(last_bg)
+
+static struct dsw_xstats_port dsw_port_xstats[] = {
+	{ "port_%u_new_enqueued", dsw_xstats_port_get_new_enqueued,
+	  false },
+	{ "port_%u_forward_enqueued", dsw_xstats_port_get_forward_enqueued,
+	  false },
+	{ "port_%u_release_enqueued", dsw_xstats_port_get_release_enqueued,
+	  false },
+	{ "port_%u_queue_%u_enqueued", dsw_xstats_port_get_queue_enqueued,
+	  true },
+	{ "port_%u_dequeued", dsw_xstats_port_get_dequeued,
+	  false },
+	{ "port_%u_queue_%u_dequeued", dsw_xstats_port_get_queue_dequeued,
+	  true },
+	{ "port_%u_migrations", dsw_xstats_port_get_migrations,
+	  false },
+	{ "port_%u_migration_latency", dsw_xstats_port_get_migration_latency,
+	  false },
+	{ "port_%u_event_proc_latency", dsw_xstats_port_get_event_proc_latency,
+	  false },
+	{ "port_%u_inflight_credits", dsw_xstats_port_get_inflight_credits,
+	  false },
+	{ "port_%u_load", dsw_xstats_port_get_load,
+	  false },
+	{ "port_%u_last_bg", dsw_xstats_port_get_last_bg,
+	  false }
+};
+
+static int
+dsw_xstats_dev_get_names(struct rte_event_dev_xstats_name *xstats_names,
+			 unsigned int *ids, unsigned int size)
+{
+	unsigned int i;
+
+	for (i = 0; i < RTE_DIM(dsw_dev_xstats) && i < size; i++) {
+		ids[i] = i;
+		strcpy(xstats_names[i].name, dsw_dev_xstats[i].name);
+	}
+
+	return i;
+}
+
+static int
+dsw_xstats_port_get_names(struct dsw_evdev *dsw, uint8_t port_id,
+			  struct rte_event_dev_xstats_name *xstats_names,
+			  unsigned int *ids, unsigned int size)
+{
+	uint8_t queue_id = 0;
+	unsigned int id_idx;
+	unsigned int stat_idx;
+
+	for (id_idx = 0, stat_idx = 0;
+	     id_idx < size && stat_idx < RTE_DIM(dsw_port_xstats);
+	     id_idx++) {
+		struct dsw_xstats_port *xstat = &dsw_port_xstats[stat_idx];
+
+		if (xstat->per_queue) {
+			ids[id_idx] = DSW_XSTATS_ID_CREATE(stat_idx, queue_id);
+			snprintf(xstats_names[id_idx].name,
+				 RTE_EVENT_DEV_XSTATS_NAME_SIZE,
+				 dsw_port_xstats[stat_idx].name_fmt, port_id,
+				 queue_id);
+			queue_id++;
+		} else {
+			ids[id_idx] = stat_idx;
+			snprintf(xstats_names[id_idx].name,
+				 RTE_EVENT_DEV_XSTATS_NAME_SIZE,
+				 dsw_port_xstats[stat_idx].name_fmt, port_id);
+		}
+
+		if (!(xstat->per_queue && queue_id < dsw->num_queues)) {
+			stat_idx++;
+			queue_id = 0;
+		}
+	}
+	return id_idx;
+}
+
+int
+dsw_xstats_get_names(const struct rte_eventdev *dev,
+		     enum rte_event_dev_xstats_mode mode,
+		     uint8_t queue_port_id,
+		     struct rte_event_dev_xstats_name *xstats_names,
+		     unsigned int *ids, unsigned int size)
+{
+	struct dsw_evdev *dsw = dsw_pmd_priv(dev);
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		return dsw_xstats_dev_get_names(xstats_names, ids, size);
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		return dsw_xstats_port_get_names(dsw, queue_port_id,
+						 xstats_names, ids, size);
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		return 0;
+	default:
+		RTE_ASSERT(false);
+		return -1;
+	}
+}
+
+static int
+dsw_xstats_dev_get(const struct rte_eventdev *dev,
+		   const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dsw_evdev *dsw = dsw_pmd_priv(dev);
+	unsigned int i;
+
+	for (i = 0; i < n; i++) {
+		unsigned int id = ids[i];
+		struct dsw_xstat_dev *xstat = &dsw_dev_xstats[id];
+		values[i] = xstat->get_value_fn(dsw);
+	}
+	return n;
+}
+
+static int
+dsw_xstats_port_get(const struct rte_eventdev *dev, uint8_t port_id,
+		    const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dsw_evdev *dsw = dsw_pmd_priv(dev);
+	unsigned int i;
+
+	for (i = 0; i < n; i++) {
+		unsigned int id = ids[i];
+		unsigned int stat_idx = DSW_XSTATS_ID_GET_STAT(id);
+		struct dsw_xstats_port *xstat = &dsw_port_xstats[stat_idx];
+		uint8_t queue_id = 0;
+
+		if (xstat->per_queue)
+			queue_id = DSW_XSTATS_ID_GET_PARAM(id);
+
+		values[i] = xstat->get_value_fn(dsw, port_id, queue_id);
+	}
+	return n;
+}
+
+int
+dsw_xstats_get(const struct rte_eventdev *dev,
+	       enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+	       const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		return dsw_xstats_dev_get(dev, ids, values, n);
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		return dsw_xstats_port_get(dev, queue_port_id, ids, values, n);
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		return 0;
+	default:
+		RTE_ASSERT(false);
+		return -1;
+	}
+	return 0;
+}
+
+uint64_t dsw_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(id);
+	return 0;
+}
diff --git a/drivers/event/dsw/meson.build b/drivers/event/dsw/meson.build
new file mode 100644
index 00000000..a6b7bfa5
--- /dev/null
+++ b/drivers/event/dsw/meson.build
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Ericsson AB
+
+allow_experimental_apis = true
+deps += ['bus_vdev']
+sources = files('dsw_evdev.c', 'dsw_event.c', 'dsw_xstats.c')
diff --git a/drivers/event/dsw/rte_pmd_dsw_event_version.map b/drivers/event/dsw/rte_pmd_dsw_event_version.map
new file mode 100644
index 00000000..24bd5cdb
--- /dev/null
+++ b/drivers/event/dsw/rte_pmd_dsw_event_version.map
@@ -0,0 +1,3 @@
+DPDK_18.11 {
+	local: *;
+};
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index e9511993..836ecbb7 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['dpaa', 'dpaa2', 'octeontx', 'skeleton', 'sw']
+drivers = ['dpaa', 'dpaa2', 'octeontx', 'opdl', 'skeleton', 'sw', 'dsw']
 std_deps = ['eventdev', 'kvargs']
 config_flag_fmt = 'RTE_LIBRTE_@0@_EVENTDEV_PMD'
 driver_name_fmt = 'rte_pmd_@0@_event'
diff --git a/drivers/event/octeontx/Makefile b/drivers/event/octeontx/Makefile
index 90ad2217..2e07890b 100644
--- a/drivers/event/octeontx/Makefile
+++ b/drivers/event/octeontx/Makefile
@@ -17,7 +17,7 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 
 LDLIBS += -lrte_eal -lrte_eventdev -lrte_common_octeontx -lrte_pmd_octeontx
 LDLIBS += -lrte_bus_pci -lrte_mempool -lrte_mbuf -lrte_kvargs
-LDLIBS += -lrte_bus_vdev
+LDLIBS += -lrte_bus_vdev -lrte_ethdev
 
 EXPORT_MAP := rte_pmd_octeontx_event_version.map
 
diff --git a/drivers/event/octeontx/ssovf_evdev.c b/drivers/event/octeontx/ssovf_evdev.c
index 16a3a04b..a273d4c9 100644
--- a/drivers/event/octeontx/ssovf_evdev.c
+++ b/drivers/event/octeontx/ssovf_evdev.c
@@ -146,6 +146,7 @@ ssovf_fastpath_fns_set(struct rte_eventdev *dev)
 	dev->enqueue_forward_burst = ssows_enq_fwd_burst;
 	dev->dequeue       = ssows_deq;
 	dev->dequeue_burst = ssows_deq_burst;
+	dev->txa_enqueue = sso_event_tx_adapter_enqueue;
 
 	if (edev->is_timeout_deq) {
 		dev->dequeue       = ssows_deq_timeout;
@@ -454,7 +455,6 @@ ssovf_eth_rx_adapter_queue_del(const struct rte_eventdev *dev,
 	const struct octeontx_nic *nic = eth_dev->data->dev_private;
 	pki_del_qos_t pki_qos;
 	RTE_SET_USED(dev);
-	RTE_SET_USED(rx_queue_id);
 
 	ret = strncmp(eth_dev->data->name, "eth_octeontx", 12);
 	if (ret)
@@ -466,7 +466,7 @@ ssovf_eth_rx_adapter_queue_del(const struct rte_eventdev *dev,
 	ret = octeontx_pki_port_delete_qos(nic->port_id, &pki_qos);
 	if (ret < 0)
 		ssovf_log_err("Failed to delete QOS port=%d, q=%d",
-				nic->port_id, queue_conf->ev.queue_id);
+				nic->port_id, rx_queue_id);
 	return ret;
 }
 
@@ -491,6 +491,77 @@ ssovf_eth_rx_adapter_stop(const struct rte_eventdev *dev,
 	return 0;
 }
 
+static int
+ssovf_eth_tx_adapter_caps_get(const struct rte_eventdev *dev,
+		const struct rte_eth_dev *eth_dev, uint32_t *caps)
+{
+	int ret;
+	RTE_SET_USED(dev);
+
+	ret = strncmp(eth_dev->data->name, "eth_octeontx", 12);
+	if (ret)
+		*caps = 0;
+	else
+		*caps = RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT;
+
+	return 0;
+}
+
+static int
+ssovf_eth_tx_adapter_create(uint8_t id, const struct rte_eventdev *dev)
+{
+	RTE_SET_USED(id);
+	RTE_SET_USED(dev);
+	return 0;
+}
+
+static int
+ssovf_eth_tx_adapter_free(uint8_t id, const struct rte_eventdev *dev)
+{
+	RTE_SET_USED(id);
+	RTE_SET_USED(dev);
+	return 0;
+}
+
+static int
+ssovf_eth_tx_adapter_queue_add(uint8_t id, const struct rte_eventdev *dev,
+		const struct rte_eth_dev *eth_dev, int32_t tx_queue_id)
+{
+	RTE_SET_USED(id);
+	RTE_SET_USED(dev);
+	RTE_SET_USED(eth_dev);
+	RTE_SET_USED(tx_queue_id);
+	return 0;
+}
+
+static int
+ssovf_eth_tx_adapter_queue_del(uint8_t id, const struct rte_eventdev *dev,
+		const struct rte_eth_dev *eth_dev, int32_t tx_queue_id)
+{
+	RTE_SET_USED(id);
+	RTE_SET_USED(dev);
+	RTE_SET_USED(eth_dev);
+	RTE_SET_USED(tx_queue_id);
+	return 0;
+}
+
+static int
+ssovf_eth_tx_adapter_start(uint8_t id, const struct rte_eventdev *dev)
+{
+	RTE_SET_USED(id);
+	RTE_SET_USED(dev);
+	return 0;
+}
+
+static int
+ssovf_eth_tx_adapter_stop(uint8_t id, const struct rte_eventdev *dev)
+{
+	RTE_SET_USED(id);
+	RTE_SET_USED(dev);
+	return 0;
+}
+
+
 static void
 ssovf_dump(struct rte_eventdev *dev, FILE *f)
 {
@@ -619,6 +690,14 @@ static struct rte_eventdev_ops ssovf_ops = {
 	.eth_rx_adapter_start = ssovf_eth_rx_adapter_start,
 	.eth_rx_adapter_stop = ssovf_eth_rx_adapter_stop,
 
+	.eth_tx_adapter_caps_get = ssovf_eth_tx_adapter_caps_get,
+	.eth_tx_adapter_create = ssovf_eth_tx_adapter_create,
+	.eth_tx_adapter_free = ssovf_eth_tx_adapter_free,
+	.eth_tx_adapter_queue_add = ssovf_eth_tx_adapter_queue_add,
+	.eth_tx_adapter_queue_del = ssovf_eth_tx_adapter_queue_del,
+	.eth_tx_adapter_start = ssovf_eth_tx_adapter_start,
+	.eth_tx_adapter_stop = ssovf_eth_tx_adapter_stop,
+
 	.timer_adapter_caps_get = ssovf_timvf_caps_get,
 
 	.dev_selftest = test_eventdev_octeontx,
diff --git a/drivers/event/octeontx/ssovf_evdev.h b/drivers/event/octeontx/ssovf_evdev.h
index 18293e96..0e622152 100644
--- a/drivers/event/octeontx/ssovf_evdev.h
+++ b/drivers/event/octeontx/ssovf_evdev.h
@@ -5,6 +5,7 @@
 #ifndef __SSOVF_EVDEV_H__
 #define __SSOVF_EVDEV_H__
 
+#include <rte_event_eth_tx_adapter.h>
 #include <rte_eventdev_pmd_vdev.h>
 #include <rte_io.h>
 
@@ -83,7 +84,7 @@
 #define SSOVF_SELFTEST_ARG               ("selftest")
 
 /*
- * In Cavium OcteonTX SoC, all accesses to the device registers are
+ * In Cavium OCTEON TX SoC, all accesses to the device registers are
  * implictly strongly ordered. So, The relaxed version of IO operation is
  * safe to use with out any IO memory barriers.
  */
@@ -179,6 +180,8 @@ typedef void (*ssows_handle_event_t)(void *arg, struct rte_event ev);
 void ssows_flush_events(struct ssows *ws, uint8_t queue_id,
 		ssows_handle_event_t fn, void *arg);
 void ssows_reset(struct ssows *ws);
+uint16_t sso_event_tx_adapter_enqueue(void *port,
+		struct rte_event ev[], uint16_t nb_events);
 int ssovf_info(struct ssovf_info *info);
 void *ssovf_bar(enum ssovf_type, uint8_t id, uint8_t bar);
 int test_eventdev_octeontx(void);
diff --git a/drivers/event/octeontx/ssovf_worker.c b/drivers/event/octeontx/ssovf_worker.c
index fffa9024..d940b5dd 100644
--- a/drivers/event/octeontx/ssovf_worker.c
+++ b/drivers/event/octeontx/ssovf_worker.c
@@ -261,3 +261,47 @@ ssows_reset(struct ssows *ws)
 			ssows_swtag_untag(ws);
 	}
 }
+
+uint16_t
+sso_event_tx_adapter_enqueue(void *port,
+		struct rte_event ev[], uint16_t nb_events)
+{
+	uint16_t port_id;
+	uint16_t queue_id;
+	struct rte_mbuf *m;
+	struct rte_eth_dev *ethdev;
+	struct ssows *ws = port;
+	struct octeontx_txq *txq;
+	octeontx_dq_t *dq;
+
+	RTE_SET_USED(nb_events);
+	switch (ev->sched_type) {
+	case SSO_SYNC_ORDERED:
+		ssows_swtag_norm(ws, ev->event, SSO_SYNC_ATOMIC);
+		rte_cio_wmb();
+		ssows_swtag_wait(ws);
+		break;
+	case SSO_SYNC_UNTAGGED:
+		ssows_swtag_full(ws, ev->u64, ev->event, SSO_SYNC_ATOMIC,
+				ev->queue_id);
+		rte_cio_wmb();
+		ssows_swtag_wait(ws);
+		break;
+	case SSO_SYNC_ATOMIC:
+		rte_cio_wmb();
+		break;
+	}
+
+	m = ev[0].mbuf;
+	port_id = m->port;
+	queue_id = rte_event_eth_tx_adapter_txq_get(m);
+	ethdev = &rte_eth_devices[port_id];
+	txq = ethdev->data->tx_queues[queue_id];
+	dq = &txq->dq;
+
+	if (__octeontx_xmit_pkts(dq->lmtline_va, dq->ioreg_va, dq->fc_status_va,
+				m) < 0)
+		return 0;
+
+	return 1;
+}
diff --git a/drivers/event/octeontx/ssovf_worker.h b/drivers/event/octeontx/ssovf_worker.h
index 7c7306b5..d1d3a52a 100644
--- a/drivers/event/octeontx/ssovf_worker.h
+++ b/drivers/event/octeontx/ssovf_worker.h
@@ -42,6 +42,7 @@ ssovf_octeontx_wqe_to_pkt(uint64_t work, uint16_t port_info)
 	mbuf->ol_flags = 0;
 	mbuf->port = rte_octeontx_pchan_map[port_info >> 4][port_info & 0xF];
 	rte_mbuf_refcnt_set(mbuf, 1);
+
 	return mbuf;
 }
 
diff --git a/drivers/event/opdl/Makefile b/drivers/event/opdl/Makefile
index cea8118d..bf50a60a 100644
--- a/drivers/event/opdl/Makefile
+++ b/drivers/event/opdl/Makefile
@@ -24,7 +24,7 @@ LDLIBS += -lrte_bus_vdev -lrte_mbuf -lrte_mempool
 LIBABIVER := 1
 
 # versioning export map
-EXPORT_MAP := rte_pmd_evdev_opdl_version.map
+EXPORT_MAP := rte_pmd_opdl_event_version.map
 
 # library source files
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_OPDL_EVENTDEV) += opdl_ring.c
diff --git a/drivers/event/opdl/meson.build b/drivers/event/opdl/meson.build
new file mode 100644
index 00000000..cc6029c6
--- /dev/null
+++ b/drivers/event/opdl/meson.build
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
+
+sources = files(
+	'opdl_evdev.c',
+	'opdl_evdev_init.c',
+	'opdl_evdev_xstats.c',
+	'opdl_ring.c',
+	'opdl_test.c',
+)
+deps += ['bus_vdev']
diff --git a/drivers/event/opdl/rte_pmd_evdev_opdl_version.map b/drivers/event/opdl/rte_pmd_opdl_event_version.map
index 58b94270..58b94270 100644
--- a/drivers/event/opdl/rte_pmd_evdev_opdl_version.map
+++ b/drivers/event/opdl/rte_pmd_opdl_event_version.map
diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
index a6bb9138..1175d6cd 100644
--- a/drivers/event/sw/sw_evdev.c
+++ b/drivers/event/sw/sw_evdev.c
@@ -113,10 +113,22 @@ sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
 			}
 		}
 	}
+
+	p->unlinks_in_progress += unlinked;
+	rte_smp_mb();
+
 	return unlinked;
 }
 
 static int
+sw_port_unlinks_in_progress(struct rte_eventdev *dev, void *port)
+{
+	RTE_SET_USED(dev);
+	struct sw_port *p = port;
+	return p->unlinks_in_progress;
+}
+
+static int
 sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
 		const struct rte_event_port_conf *conf)
 {
@@ -925,6 +937,7 @@ sw_probe(struct rte_vdev_device *vdev)
 			.port_release = sw_port_release,
 			.port_link = sw_port_link,
 			.port_unlink = sw_port_unlink,
+			.port_unlinks_in_progress = sw_port_unlinks_in_progress,
 
 			.eth_rx_adapter_caps_get = sw_eth_rx_adapter_caps_get,
 
diff --git a/drivers/event/sw/sw_evdev.h b/drivers/event/sw/sw_evdev.h
index d90b96d4..7c77b249 100644
--- a/drivers/event/sw/sw_evdev.h
+++ b/drivers/event/sw/sw_evdev.h
@@ -148,6 +148,14 @@ struct sw_port {
 	/* A numeric ID for the port */
 	uint8_t id;
 
+	/* An atomic counter for when the port has been unlinked, and the
+	 * scheduler has not yet acked this unlink - hence there may still be
+	 * events in the buffers going to the port. When the unlinks in
+	 * progress is read by the scheduler, no more events will be pushed to
+	 * the port - hence the scheduler core can just assign zero.
+	 */
+	uint8_t unlinks_in_progress;
+
 	int16_t is_directed; /** Takes from a single directed QID */
 	/**
 	 * For loadbalanced we can optimise pulling packets from
diff --git a/drivers/event/sw/sw_evdev_scheduler.c b/drivers/event/sw/sw_evdev_scheduler.c
index e3a41e02..cff747da 100644
--- a/drivers/event/sw/sw_evdev_scheduler.c
+++ b/drivers/event/sw/sw_evdev_scheduler.c
@@ -51,9 +51,11 @@ sw_schedule_atomic_to_cq(struct sw_evdev *sw, struct sw_qid * const qid,
 		int cq = fid->cq;
 
 		if (cq < 0) {
-			uint32_t cq_idx = qid->cq_next_tx++;
-			if (qid->cq_next_tx == qid->cq_num_mapped_cqs)
+			uint32_t cq_idx;
+			if (qid->cq_next_tx >= qid->cq_num_mapped_cqs)
 				qid->cq_next_tx = 0;
+			cq_idx = qid->cq_next_tx++;
+
 			cq = qid->cq_map[cq_idx];
 
 			/* find least used */
@@ -140,9 +142,10 @@ sw_schedule_parallel_to_cq(struct sw_evdev *sw, struct sw_qid * const qid,
 		do {
 			if (++cq_check_count > qid->cq_num_mapped_cqs)
 				goto exit;
-			cq = qid->cq_map[cq_idx];
-			if (++cq_idx == qid->cq_num_mapped_cqs)
+			if (cq_idx >= qid->cq_num_mapped_cqs)
 				cq_idx = 0;
+			cq = qid->cq_map[cq_idx++];
+
 		} while (rte_event_ring_free_count(
 				sw->ports[cq].cq_worker_ring) == 0 ||
 				sw->ports[cq].inflights == SW_PORT_HIST_LIST);
@@ -220,7 +223,7 @@ sw_schedule_qid_to_cq(struct sw_evdev *sw)
 		int iq_num = PKT_MASK_TO_IQ(qid->iq_pkt_mask);
 
 		/* zero mapped CQs indicates directed */
-		if (iq_num >= SW_IQS_MAX)
+		if (iq_num >= SW_IQS_MAX || qid->cq_num_mapped_cqs == 0)
 			continue;
 
 		uint32_t pkts_done = 0;
@@ -517,13 +520,18 @@ sw_event_schedule(struct rte_eventdev *dev)
 		/* Pull from rx_ring for ports */
 		do {
 			in_pkts = 0;
-			for (i = 0; i < sw->port_count; i++)
+			for (i = 0; i < sw->port_count; i++) {
+				/* ack the unlinks in progress as done */
+				if (sw->ports[i].unlinks_in_progress)
+					sw->ports[i].unlinks_in_progress = 0;
+
 				if (sw->ports[i].is_directed)
 					in_pkts += sw_schedule_pull_port_dir(sw, i);
 				else if (sw->ports[i].num_ordered_qids > 0)
 					in_pkts += sw_schedule_pull_port_lb(sw, i);
 				else
 					in_pkts += sw_schedule_pull_port_no_reorder(sw, i);
+			}
 
 			/* QID scan for re-ordered */
 			in_pkts += sw_schedule_reorder(sw, 0,
diff --git a/drivers/event/sw/sw_evdev_selftest.c b/drivers/event/sw/sw_evdev_selftest.c
index c40912db..d00d5de6 100644
--- a/drivers/event/sw/sw_evdev_selftest.c
+++ b/drivers/event/sw/sw_evdev_selftest.c
@@ -1904,6 +1904,77 @@ qid_priorities(struct test *t)
 }
 
 static int
+unlink_in_progress(struct test *t)
+{
+	/* Test unlinking API, in particular that when an unlink request has
+	 * not yet been seen by the scheduler thread, that the
+	 * unlink_in_progress() function returns the number of unlinks.
+	 */
+	unsigned int i;
+	/* Create instance with 1 ports, and 3 qids */
+	if (init(t, 3, 1) < 0 ||
+			create_ports(t, 1) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < 3; i++) {
+		/* Create QID */
+		const struct rte_event_queue_conf conf = {
+			.schedule_type = RTE_SCHED_TYPE_ATOMIC,
+			/* increase priority (0 == highest), as we go */
+			.priority = RTE_EVENT_DEV_PRIORITY_NORMAL - i,
+			.nb_atomic_flows = 1024,
+			.nb_atomic_order_sequences = 1024,
+		};
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+		t->qid[i] = i;
+	}
+	t->nb_qids = i;
+	/* map all QIDs to port */
+	rte_event_port_link(evdev, t->port[0], NULL, NULL, 0);
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		return -1;
+	}
+
+	/* unlink all ports to have outstanding unlink requests */
+	int ret = rte_event_port_unlink(evdev, t->port[0], NULL, 0);
+	if (ret < 0) {
+		printf("%d: Failed to unlink queues\n", __LINE__);
+		return -1;
+	}
+
+	/* get active unlinks here, expect 3 */
+	int unlinks_in_progress =
+		rte_event_port_unlinks_in_progress(evdev, t->port[0]);
+	if (unlinks_in_progress != 3) {
+		printf("%d: Expected num unlinks in progress == 3, got %d\n",
+				__LINE__, unlinks_in_progress);
+		return -1;
+	}
+
+	/* run scheduler service on this thread to ack the unlinks */
+	rte_service_run_iter_on_app_lcore(t->service_id, 1);
+
+	/* active unlinks expected as 0 as scheduler thread has acked */
+	unlinks_in_progress =
+		rte_event_port_unlinks_in_progress(evdev, t->port[0]);
+	if (unlinks_in_progress != 0) {
+		printf("%d: Expected num unlinks in progress == 0, got %d\n",
+				__LINE__, unlinks_in_progress);
+	}
+
+	cleanup(t);
+	return 0;
+}
+
+static int
 load_balancing(struct test *t)
 {
 	const int rx_enq = 0;
@@ -3260,6 +3331,12 @@ test_sw_eventdev(void)
 		printf("ERROR - QID Priority test FAILED.\n");
 		goto test_fail;
 	}
+	printf("*** Running Unlink-in-progress test...\n");
+	ret = unlink_in_progress(t);
+	if (ret != 0) {
+		printf("ERROR - Unlink in progress test FAILED.\n");
+		goto test_fail;
+	}
 	printf("*** Running Ordered Reconfigure test...\n");
 	ret = ordered_reconfigure(t);
 	if (ret != 0) {
diff --git a/drivers/mempool/dpaa/Makefile b/drivers/mempool/dpaa/Makefile
index da8da1e9..ead5029f 100644
--- a/drivers/mempool/dpaa/Makefile
+++ b/drivers/mempool/dpaa/Makefile
@@ -10,7 +10,6 @@ LIB = librte_mempool_dpaa.a
 
 CFLAGS := -I$(SRCDIR) $(CFLAGS)
 CFLAGS += -O3 $(WERROR_FLAGS)
-CFLAGS += -D _GNU_SOURCE
 CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa
 CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa/include/
 CFLAGS += -I$(RTE_SDK)/drivers/mempool/dpaa
@@ -31,5 +30,6 @@ SRCS-$(CONFIG_RTE_LIBRTE_DPAA_MEMPOOL) += dpaa_mempool.c
 
 LDLIBS += -lrte_bus_dpaa
 LDLIBS += -lrte_eal -lrte_mempool -lrte_ring
+LDLIBS += -lrte_common_dpaax
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/mempool/dpaa/dpaa_mempool.c b/drivers/mempool/dpaa/dpaa_mempool.c
index 10c536bf..021b366f 100644
--- a/drivers/mempool/dpaa/dpaa_mempool.c
+++ b/drivers/mempool/dpaa/dpaa_mempool.c
@@ -26,6 +26,7 @@
 #include <rte_ring.h>
 
 #include <dpaa_mempool.h>
+#include <dpaax_iova_table.h>
 
 /* List of all the memseg information locally maintained in dpaa driver. This
  * is to optimize the PA_to_VA searches until a better mechanism (algo) is
@@ -122,7 +123,7 @@ dpaa_buf_free(struct dpaa_bp_info *bp_info, uint64_t addr)
 	struct bm_buffer buf;
 	int ret;
 
-	DPAA_MEMPOOL_DEBUG("Free 0x%" PRIx64 " to bpid: %d",
+	DPAA_MEMPOOL_DPDEBUG("Free 0x%" PRIx64 " to bpid: %d",
 			   addr, bp_info->bpid);
 
 	bm_buffer_set64(&buf, addr);
@@ -285,6 +286,9 @@ dpaa_populate(struct rte_mempool *mp, unsigned int max_objs,
 		return 0;
 	}
 
+	/* Update the PA-VA Table */
+	dpaax_iova_table_update(paddr, vaddr, len);
+
 	bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp);
 	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
 
@@ -324,7 +328,7 @@ dpaa_populate(struct rte_mempool *mp, unsigned int max_objs,
 					       obj_cb, obj_cb_arg);
 }
 
-struct rte_mempool_ops dpaa_mpool_ops = {
+static const struct rte_mempool_ops dpaa_mpool_ops = {
 	.name = DPAA_MEMPOOL_OPS_NAME,
 	.alloc = dpaa_mbuf_create_pool,
 	.free = dpaa_mbuf_free_pool,
diff --git a/drivers/mempool/dpaa/dpaa_mempool.h b/drivers/mempool/dpaa/dpaa_mempool.h
index 092f326c..533e1c6e 100644
--- a/drivers/mempool/dpaa/dpaa_mempool.h
+++ b/drivers/mempool/dpaa/dpaa_mempool.h
@@ -43,10 +43,8 @@ struct dpaa_bp_info {
 };
 
 static inline void *
-DPAA_MEMPOOL_PTOV(struct dpaa_bp_info *bp_info, uint64_t addr)
+DPAA_MEMPOOL_PTOV(struct dpaa_bp_info *bp_info __rte_unused, uint64_t addr)
 {
-	if (bp_info->ptov_off)
-		return ((void *) (size_t)(addr + bp_info->ptov_off));
 	return rte_dpaa_mem_ptov(addr);
 }
 
diff --git a/drivers/mempool/dpaa2/Makefile b/drivers/mempool/dpaa2/Makefile
index 9e4c87d7..96c0f2b6 100644
--- a/drivers/mempool/dpaa2/Makefile
+++ b/drivers/mempool/dpaa2/Makefile
@@ -19,7 +19,7 @@ CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
 EXPORT_MAP := rte_mempool_dpaa2_version.map
 
 # Lbrary version
-LIBABIVER := 1
+LIBABIVER := 2
 
 # depends on fslmc bus which uses experimental API
 CFLAGS += -DALLOW_EXPERIMENTAL_API
@@ -30,6 +30,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_DPAA2_MEMPOOL) += dpaa2_hw_mempool.c
 
 LDLIBS += -lrte_bus_fslmc
 LDLIBS += -lrte_eal -lrte_mempool -lrte_ring
+LDLIBS += -lrte_common_dpaax
 
 SYMLINK-$(CONFIG_RTE_LIBRTE_DPAA2_MEMPOOL)-include := rte_dpaa2_mempool.h
 
diff --git a/drivers/mempool/dpaa2/dpaa2_hw_mempool.c b/drivers/mempool/dpaa2/dpaa2_hw_mempool.c
index 7d0435f5..790cded8 100644
--- a/drivers/mempool/dpaa2/dpaa2_hw_mempool.c
+++ b/drivers/mempool/dpaa2/dpaa2_hw_mempool.c
@@ -30,16 +30,11 @@
 #include "dpaa2_hw_mempool.h"
 #include "dpaa2_hw_mempool_logs.h"
 
+#include <dpaax_iova_table.h>
+
 struct dpaa2_bp_info rte_dpaa2_bpid_info[MAX_BPID];
 static struct dpaa2_bp_list *h_bp_list;
 
-/* List of all the memseg information locally maintained in dpaa2 driver. This
- * is to optimize the PA_to_VA searches until a better mechanism (algo) is
- * available.
- */
-struct dpaa2_memseg_list rte_dpaa2_memsegs
-	= TAILQ_HEAD_INITIALIZER(rte_dpaa2_memsegs);
-
 /* Dynamic logging identified for mempool */
 int dpaa2_logtype_mempool;
 
@@ -400,37 +395,14 @@ dpaa2_populate(struct rte_mempool *mp, unsigned int max_objs,
 	      void *vaddr, rte_iova_t paddr, size_t len,
 	      rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
 {
-	struct dpaa2_memseg *ms;
-
-	/* For each memory chunk pinned to the Mempool, a linked list of the
-	 * contained memsegs is created for searching when PA to VA
-	 * conversion is required.
-	 */
-	ms = rte_zmalloc(NULL, sizeof(struct dpaa2_memseg), 0);
-	if (!ms) {
-		DPAA2_MEMPOOL_ERR("Unable to allocate internal memory.");
-		DPAA2_MEMPOOL_WARN("Fast Physical to Virtual Addr translation would not be available.");
-		/* If the element is not added, it would only lead to failure
-		 * in searching for the element and the logic would Fallback
-		 * to traditional DPDK memseg traversal code. So, this is not
-		 * a blocking error - but, error would be printed on screen.
-		 */
-		return 0;
-	}
-
-	ms->vaddr = vaddr;
-	ms->iova = paddr;
-	ms->len = len;
-	/* Head insertions are generally faster than tail insertions as the
-	 * buffers pinned are picked from rear end.
-	 */
-	TAILQ_INSERT_HEAD(&rte_dpaa2_memsegs, ms, next);
+	/* Insert entry into the PA->VA Table */
+	dpaax_iova_table_update(paddr, vaddr, len);
 
 	return rte_mempool_op_populate_default(mp, max_objs, vaddr, paddr, len,
 					       obj_cb, obj_cb_arg);
 }
 
-struct rte_mempool_ops dpaa2_mpool_ops = {
+static const struct rte_mempool_ops dpaa2_mpool_ops = {
 	.name = DPAA2_MEMPOOL_OPS_NAME,
 	.alloc = rte_hw_mbuf_create_pool,
 	.free = rte_hw_mbuf_free_pool,
diff --git a/drivers/mempool/dpaa2/meson.build b/drivers/mempool/dpaa2/meson.build
index 90bab606..6b6ead61 100644
--- a/drivers/mempool/dpaa2/meson.build
+++ b/drivers/mempool/dpaa2/meson.build
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright 2018 NXP
 
+version = 2
+
 if host_machine.system() != 'linux'
         build = false
 endif
diff --git a/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map b/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map
index b9d996a6..b45e7a9a 100644
--- a/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map
+++ b/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map
@@ -3,7 +3,6 @@ DPDK_17.05 {
 
 	rte_dpaa2_bpid_info;
 	rte_dpaa2_mbuf_alloc_bulk;
-	rte_dpaa2_memsegs;
 
 	local: *;
 };
diff --git a/drivers/mempool/octeontx/octeontx_fpavf.h b/drivers/mempool/octeontx/octeontx_fpavf.h
index b00be137..e27c4377 100644
--- a/drivers/mempool/octeontx/octeontx_fpavf.h
+++ b/drivers/mempool/octeontx/octeontx_fpavf.h
@@ -50,7 +50,7 @@
 #define OCTEONTX_FPAVF_BUF_OFFSET	128
 
 /*
- * In Cavium OcteonTX SoC, all accesses to the device registers are
+ * In Cavium OCTEON TX SoC, all accesses to the device registers are
  * implicitly strongly ordered. So, the relaxed version of IO operation is
  * safe to use with out any IO memory barriers.
  */
diff --git a/drivers/meson.build b/drivers/meson.build
index f94e2fe6..c3c66bbc 100644
--- a/drivers/meson.build
+++ b/drivers/meson.build
@@ -9,12 +9,17 @@ driver_classes = ['common',
 	       'crypto',  # depends on common, bus and mempool (net in future).
 	       'compress', # depends on common, bus, mempool.
 	       'event',   # depends on common, bus, mempool and net.
+	       'baseband', # depends on common and bus.
 	       'raw']     # depends on common, bus, mempool, net and event.
 
 default_cflags = machine_args
 if cc.has_argument('-Wno-format-truncation')
 	default_cflags += '-Wno-format-truncation'
 endif
+
+# specify -D_GNU_SOURCE unconditionally
+default_cflags += '-D_GNU_SOURCE'
+
 foreach class:driver_classes
 	drivers = []
 	std_deps = []
@@ -24,6 +29,7 @@ foreach class:driver_classes
 	                     # version file for linking
 
 	subdir(class)
+	class_drivers = []
 
 	foreach drv:drivers
 		drv_path = join_paths(class, drv)
@@ -51,6 +57,8 @@ foreach class:driver_classes
 		subdir(drv_path)
 
 		if build
+			class_drivers += name
+
 			dpdk_conf.set(config_flag_fmt.format(name.to_upper()),1)
 			lib_name = driver_name_fmt.format(name)
 
@@ -94,10 +102,8 @@ foreach class:driver_classes
 				lib_version = '@0@.1'.format(version)
 				so_version = '@0@'.format(version)
 			else
-				pver = meson.project_version().split('.')
-				lib_version = '@0@.@1@'.format(pver.get(0),
-						pver.get(1))
-				so_version = lib_version
+				lib_version = major_version
+				so_version = major_version
 			endif
 
 			# now build the static driver
@@ -141,4 +147,9 @@ foreach class:driver_classes
 			set_variable('static_@0@'.format(lib_name), static_dep)
 		endif # build
 	endforeach
+
+	if meson.version().version_compare('>=0.47')
+		# prior to 0.47, set_variable can't take array params
+		set_variable(class + '_drivers', class_drivers)
+	endif
 endforeach
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index 664398de..c0386feb 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -10,6 +10,7 @@ endif
 
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET) += af_packet
 DIRS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark
+DIRS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += atlantic
 DIRS-$(CONFIG_RTE_LIBRTE_AVF_PMD) += avf
 DIRS-$(CONFIG_RTE_LIBRTE_AVP_PMD) += avp
 DIRS-$(CONFIG_RTE_LIBRTE_AXGBE_PMD) += axgbe
@@ -24,6 +25,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_DPAA2_PMD) += dpaa2
 endif
 DIRS-$(CONFIG_RTE_LIBRTE_E1000_PMD) += e1000
 DIRS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena
+DIRS-$(CONFIG_RTE_LIBRTE_ENETC_PMD) += enetc
 DIRS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe
 DIRS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += fm10k
@@ -32,6 +34,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe
 DIRS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += liquidio
 DIRS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4
 DIRS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5
+DIRS-$(CONFIG_RTE_LIBRTE_MVNETA_PMD) += mvneta
 DIRS-$(CONFIG_RTE_LIBRTE_MVPP2_PMD) += mvpp2
 DIRS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += netvsc
 DIRS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += nfp
diff --git a/drivers/net/af_packet/rte_eth_af_packet.c b/drivers/net/af_packet/rte_eth_af_packet.c
index eb3cce3a..95a98c6b 100644
--- a/drivers/net/af_packet/rte_eth_af_packet.c
+++ b/drivers/net/af_packet/rte_eth_af_packet.c
@@ -305,7 +305,6 @@ eth_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	dev_info->max_rx_queues = (uint16_t)internals->nb_queues;
 	dev_info->max_tx_queues = (uint16_t)internals->nb_queues;
 	dev_info->min_rx_bufsize = 0;
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -927,8 +926,7 @@ rte_pmd_af_packet_probe(struct rte_vdev_device *dev)
 
 	PMD_LOG(INFO, "Initializing pmd_af_packet for %s", name);
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
-	    strlen(rte_vdev_device_args(dev)) == 0) {
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 		eth_dev = rte_eth_dev_attach_secondary(name);
 		if (!eth_dev) {
 			PMD_LOG(ERR, "Failed to probe %s", name);
@@ -988,6 +986,12 @@ rte_pmd_af_packet_remove(struct rte_vdev_device *dev)
 	if (eth_dev == NULL)
 		return -1;
 
+	/* mac_addrs must not be freed alone because part of dev_private */
+	eth_dev->data->mac_addrs = NULL;
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return rte_eth_dev_release_port(eth_dev);
+
 	internals = eth_dev->data->dev_private;
 	for (q = 0; q < internals->nb_queues; q++) {
 		rte_free(internals->rx_queue[q].rd);
@@ -995,8 +999,6 @@ rte_pmd_af_packet_remove(struct rte_vdev_device *dev)
 	}
 	free(internals->if_name);
 
-	rte_free(eth_dev->data->dev_private);
-
 	rte_eth_dev_release_port(eth_dev);
 
 	return 0;
diff --git a/drivers/net/ark/ark_ddm.c b/drivers/net/ark/ark_ddm.c
index eea388a1..57026f8d 100644
--- a/drivers/net/ark/ark_ddm.c
+++ b/drivers/net/ark/ark_ddm.c
@@ -11,14 +11,22 @@
 int
 ark_ddm_verify(struct ark_ddm_t *ddm)
 {
+	uint32_t hw_const;
 	if (sizeof(struct ark_ddm_t) != ARK_DDM_EXPECTED_SIZE) {
 		PMD_DRV_LOG(ERR, "ARK: DDM structure looks incorrect %d vs %zd\n",
 			    ARK_DDM_EXPECTED_SIZE, sizeof(struct ark_ddm_t));
 		return -1;
 	}
 
-	if (ddm->cfg.const0 != ARK_DDM_CONST) {
-		PMD_DRV_LOG(ERR, "ARK: DDM module not found as expected 0x%08x\n",
+	hw_const = ddm->cfg.const0;
+	if (hw_const == ARK_DDM_CONST1) {
+		PMD_DRV_LOG(ERR,
+			    "ARK: DDM module is version 1, "
+			    "PMD expects version 2\n");
+		return -1;
+	} else if (hw_const != ARK_DDM_CONST2) {
+		PMD_DRV_LOG(ERR,
+			    "ARK: DDM module not found as expected 0x%08x\n",
 			    ddm->cfg.const0);
 		return -1;
 	}
diff --git a/drivers/net/ark/ark_ddm.h b/drivers/net/ark/ark_ddm.h
index b37d1e09..5456b4b5 100644
--- a/drivers/net/ark/ark_ddm.h
+++ b/drivers/net/ark/ark_ddm.h
@@ -19,7 +19,7 @@
 /* struct defining Tx meta data --  fixed in FPGA -- 16 bytes */
 struct ark_tx_meta {
 	uint64_t physaddr;
-	uint32_t delta_ns;
+	uint32_t user1;
 	uint16_t data_len;		/* of this MBUF */
 #define   ARK_DDM_EOP   0x01
 #define   ARK_DDM_SOP   0x02
@@ -34,7 +34,10 @@ struct ark_tx_meta {
  * structs will never be instantiated in ram memory
  */
 #define ARK_DDM_CFG 0x0000
-#define ARK_DDM_CONST 0xfacecafe
+/* Set unique HW ID for hardware version */
+#define ARK_DDM_CONST2 (0x324d4444)
+#define ARK_DDM_CONST1 (0xfacecafe)
+
 struct ark_ddm_cfg_t {
 	uint32_t r0;
 	volatile uint32_t tlp_stats_clear;
diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
index 552ca01a..4f52e2bd 100644
--- a/drivers/net/ark/ark_ethdev.c
+++ b/drivers/net/ark/ark_ethdev.c
@@ -313,6 +313,9 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 
 	/* We are a single function multi-port device. */
 	ret = ark_config_device(dev);
+	if (ret)
+		return -1;
+
 	dev->dev_ops = &ark_eth_dev_ops;
 
 	dev->data->mac_addrs = rte_zmalloc("ark", ETHER_ADDR_LEN, 0);
@@ -506,7 +509,6 @@ eth_ark_dev_uninit(struct rte_eth_dev *dev)
 	dev->dev_ops = NULL;
 	dev->rx_pkt_burst = NULL;
 	dev->tx_pkt_burst = NULL;
-	rte_free(dev->data->mac_addrs);
 	return 0;
 }
 
diff --git a/drivers/net/ark/ark_ethdev_rx.c b/drivers/net/ark/ark_ethdev_rx.c
index 16f0d11e..300029d6 100644
--- a/drivers/net/ark/ark_ethdev_rx.c
+++ b/drivers/net/ark/ark_ethdev_rx.c
@@ -25,6 +25,9 @@ static uint32_t eth_ark_rx_jumbo(struct ark_rx_queue *queue,
 				 struct rte_mbuf *mbuf0,
 				 uint32_t cons_index);
 static inline int eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue);
+static int eth_ark_rx_seed_recovery(struct ark_rx_queue *queue,
+				    uint32_t *pnb,
+				    struct rte_mbuf **mbufs);
 
 /* ************************************************************************* */
 struct ark_rx_queue {
@@ -50,7 +53,7 @@ struct ark_rx_queue {
 	/* The queue Index is used within the dpdk device structures */
 	uint16_t queue_index;
 
-	uint32_t pad1;
+	uint32_t last_cons;
 
 	/* separate cache line */
 	/* second cache line - fields only used in slow path */
@@ -102,7 +105,10 @@ eth_ark_rx_update_cons_index(struct ark_rx_queue *queue, uint32_t cons_index)
 {
 	queue->cons_index = cons_index;
 	eth_ark_rx_seed_mbufs(queue);
-	ark_mpu_set_producer(queue->mpu, queue->seed_index);
+	if (((cons_index - queue->last_cons) >= 64U)) {
+		queue->last_cons = cons_index;
+		ark_mpu_set_producer(queue->mpu, queue->seed_index);
+	}
 }
 
 /* ************************************************************************* */
@@ -196,20 +202,25 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	/* populate mbuf reserve */
 	status = eth_ark_rx_seed_mbufs(queue);
 
+	if (queue->seed_index != nb_desc) {
+		PMD_DRV_LOG(ERR, "ARK: Failed to allocate %u mbufs for RX queue %d\n",
+			    nb_desc, qidx);
+		status = -1;
+	}
 	/* MPU Setup */
 	if (status == 0)
 		status = eth_ark_rx_hw_setup(dev, queue, qidx, queue_idx);
 
 	if (unlikely(status != 0)) {
-		struct rte_mbuf *mbuf;
+		struct rte_mbuf **mbuf;
 
 		PMD_DRV_LOG(ERR, "Failed to initialize RX queue %d %s\n",
 			    qidx,
 			    __func__);
 		/* Free the mbufs allocated */
-		for (i = 0, mbuf = queue->reserve_q[0];
-		     i < nb_desc; ++i, mbuf++) {
-			rte_pktmbuf_free(mbuf);
+		for (i = 0, mbuf = queue->reserve_q;
+		     i < queue->seed_index; ++i, mbuf++) {
+			rte_pktmbuf_free(*mbuf);
 		}
 		rte_free(queue->reserve_q);
 		rte_free(queue->paddress_q);
@@ -446,8 +457,13 @@ eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue)
 	struct rte_mbuf **mbufs = &queue->reserve_q[seed_m];
 	int status = rte_pktmbuf_alloc_bulk(queue->mb_pool, mbufs, nb);
 
-	if (unlikely(status != 0))
-		return -1;
+	if (unlikely(status != 0)) {
+		/* Try to recover from lack of mbufs in pool */
+		status = eth_ark_rx_seed_recovery(queue, &nb, mbufs);
+		if (unlikely(status != 0)) {
+			return -1;
+		}
+	}
 
 	if (ARK_RX_DEBUG) {		/* DEBUG */
 		while (count != nb) {
@@ -495,6 +511,29 @@ eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue)
 	return 0;
 }
 
+int
+eth_ark_rx_seed_recovery(struct ark_rx_queue *queue,
+			 uint32_t *pnb,
+			 struct rte_mbuf **mbufs)
+{
+	int status = -1;
+
+	/* Ignore small allocation failures */
+	if (*pnb <= 64)
+		return -1;
+
+	*pnb = 64U;
+	status = rte_pktmbuf_alloc_bulk(queue->mb_pool, mbufs, *pnb);
+	if (status != 0) {
+		PMD_DRV_LOG(ERR,
+			    "ARK: Could not allocate %u mbufs from pool for RX queue %u;"
+			    " %u free buffers remaining in queue\n",
+			    *pnb, queue->queue_index,
+			    queue->seed_index - queue->cons_index);
+	}
+	return status;
+}
+
 void
 eth_ark_rx_dump_queue(struct rte_eth_dev *dev, uint16_t queue_id,
 		      const char *msg)
diff --git a/drivers/net/ark/ark_ethdev_tx.c b/drivers/net/ark/ark_ethdev_tx.c
index 57188c24..94da5f95 100644
--- a/drivers/net/ark/ark_ethdev_tx.c
+++ b/drivers/net/ark/ark_ethdev_tx.c
@@ -65,7 +65,7 @@ eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta,
 			  uint8_t flags)
 {
 	meta->physaddr = rte_mbuf_data_iova(mbuf);
-	meta->delta_ns = 0;
+	meta->user1 = (uint32_t)mbuf->udata64;
 	meta->data_len = rte_pktmbuf_data_len(mbuf);
 	meta->flags = flags;
 }
diff --git a/drivers/net/atlantic/Makefile b/drivers/net/atlantic/Makefile
new file mode 100644
index 00000000..62dcdbff
--- /dev/null
+++ b/drivers/net/atlantic/Makefile
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Aquantia Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_pmd_atlantic.a
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+
+EXPORT_MAP := rte_pmd_atlantic_version.map
+
+LIBABIVER := 1
+
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_ethdev -lrte_net
+LDLIBS += -lrte_bus_pci
+
+VPATH += $(SRCDIR)/hw_atl
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += atl_rxtx.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += atl_ethdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += atl_hw_regs.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_utils.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_llh.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_utils_fw2x.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_b0.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/atlantic/atl_common.h b/drivers/net/atlantic/atl_common.h
new file mode 100644
index 00000000..b3a0aa5c
--- /dev/null
+++ b/drivers/net/atlantic/atl_common.h
@@ -0,0 +1,96 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Aquantia Corporation
+ */
+
+#ifndef AQ_COMMON_H
+#define AQ_COMMON_H
+
+#define ATL_PMD_DRIVER_VERSION "0.4.1"
+
+#define PCI_VENDOR_ID_AQUANTIA  0x1D6A
+
+#define AQ_DEVICE_ID_0001	0x0001
+#define AQ_DEVICE_ID_D100	0xD100
+#define AQ_DEVICE_ID_D107	0xD107
+#define AQ_DEVICE_ID_D108	0xD108
+#define AQ_DEVICE_ID_D109	0xD109
+
+#define AQ_DEVICE_ID_AQC100	0x00B1
+#define AQ_DEVICE_ID_AQC107	0x07B1
+#define AQ_DEVICE_ID_AQC108	0x08B1
+#define AQ_DEVICE_ID_AQC109	0x09B1
+#define AQ_DEVICE_ID_AQC111	0x11B1
+#define AQ_DEVICE_ID_AQC112	0x12B1
+
+#define AQ_DEVICE_ID_AQC100S	0x80B1
+#define AQ_DEVICE_ID_AQC107S	0x87B1
+#define AQ_DEVICE_ID_AQC108S	0x88B1
+#define AQ_DEVICE_ID_AQC109S	0x89B1
+#define AQ_DEVICE_ID_AQC111S	0x91B1
+#define AQ_DEVICE_ID_AQC112S	0x92B1
+
+#define AQ_DEVICE_ID_AQC111E	0x51B1
+#define AQ_DEVICE_ID_AQC112E	0x52B1
+
+#define HW_ATL_NIC_NAME "aQuantia AQtion 10Gbit Network Adapter"
+
+#define AQ_HWREV_ANY	0
+#define AQ_HWREV_1	1
+#define AQ_HWREV_2	2
+
+#define AQ_NIC_RATE_10G		BIT(0)
+#define AQ_NIC_RATE_5G		BIT(1)
+#define AQ_NIC_RATE_5G5R	BIT(2)
+#define AQ_NIC_RATE_2G5		BIT(3)
+#define AQ_NIC_RATE_1G		BIT(4)
+#define AQ_NIC_RATE_100M	BIT(5)
+
+#define AQ_NIC_RATE_EEE_10G	BIT(6)
+#define AQ_NIC_RATE_EEE_5G	BIT(7)
+#define AQ_NIC_RATE_EEE_2G5	BIT(8)
+#define AQ_NIC_RATE_EEE_1G	BIT(9)
+
+
+#define ATL_MAX_RING_DESC	(8 * 1024 - 8)
+#define ATL_MIN_RING_DESC	32
+#define ATL_RXD_ALIGN		8
+#define ATL_TXD_ALIGN		8
+#define ATL_TX_MAX_SEG		16
+
+#define ATL_MAX_INTR_QUEUE_NUM  15
+
+#define ATL_MISC_VEC_ID 10
+#define ATL_RX_VEC_START 0
+
+#define AQ_NIC_WOL_ENABLED           BIT(0)
+
+
+#define AQ_NIC_FC_OFF    0U
+#define AQ_NIC_FC_TX     1U
+#define AQ_NIC_FC_RX     2U
+#define AQ_NIC_FC_FULL   3U
+#define AQ_NIC_FC_AUTO   4U
+
+
+#define AQ_CFG_TX_FRAME_MAX  (16U * 1024U)
+#define AQ_CFG_RX_FRAME_MAX  (2U * 1024U)
+
+#define AQ_HW_MULTICAST_ADDRESS_MAX     32
+#define AQ_HW_MAX_SEGS_SIZE    40
+
+#define AQ_HW_MAX_RX_QUEUES    8
+#define AQ_HW_MAX_TX_QUEUES    8
+#define AQ_HW_MIN_RX_RING_SIZE 512
+#define AQ_HW_MAX_RX_RING_SIZE 8192
+#define AQ_HW_MIN_TX_RING_SIZE 512
+#define AQ_HW_MAX_TX_RING_SIZE 8192
+
+#define ATL_DEFAULT_RX_FREE_THRESH 64
+#define ATL_DEFAULT_TX_FREE_THRESH 64
+
+#define ATL_IRQ_CAUSE_LINK 0x8
+
+#define AQ_HW_LED_BLINK    0x2U
+#define AQ_HW_LED_DEFAULT  0x0U
+
+#endif /* AQ_COMMON_H */
diff --git a/drivers/net/atlantic/atl_ethdev.c b/drivers/net/atlantic/atl_ethdev.c
new file mode 100644
index 00000000..5bc04f55
--- /dev/null
+++ b/drivers/net/atlantic/atl_ethdev.c
@@ -0,0 +1,1539 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Aquantia Corporation
+ */
+
+#include <rte_ethdev_pci.h>
+
+#include "atl_ethdev.h"
+#include "atl_common.h"
+#include "atl_hw_regs.h"
+#include "atl_logs.h"
+#include "hw_atl/hw_atl_llh.h"
+#include "hw_atl/hw_atl_b0.h"
+#include "hw_atl/hw_atl_b0_internal.h"
+
+static int eth_atl_dev_init(struct rte_eth_dev *eth_dev);
+static int eth_atl_dev_uninit(struct rte_eth_dev *eth_dev);
+
+static int  atl_dev_configure(struct rte_eth_dev *dev);
+static int  atl_dev_start(struct rte_eth_dev *dev);
+static void atl_dev_stop(struct rte_eth_dev *dev);
+static int  atl_dev_set_link_up(struct rte_eth_dev *dev);
+static int  atl_dev_set_link_down(struct rte_eth_dev *dev);
+static void atl_dev_close(struct rte_eth_dev *dev);
+static int  atl_dev_reset(struct rte_eth_dev *dev);
+static void atl_dev_promiscuous_enable(struct rte_eth_dev *dev);
+static void atl_dev_promiscuous_disable(struct rte_eth_dev *dev);
+static void atl_dev_allmulticast_enable(struct rte_eth_dev *dev);
+static void atl_dev_allmulticast_disable(struct rte_eth_dev *dev);
+static int  atl_dev_link_update(struct rte_eth_dev *dev, int wait);
+
+static int atl_dev_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
+				    struct rte_eth_xstat_name *xstats_names,
+				    unsigned int size);
+
+static int atl_dev_stats_get(struct rte_eth_dev *dev,
+				struct rte_eth_stats *stats);
+
+static int atl_dev_xstats_get(struct rte_eth_dev *dev,
+			      struct rte_eth_xstat *stats, unsigned int n);
+
+static void atl_dev_stats_reset(struct rte_eth_dev *dev);
+
+static int atl_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
+			      size_t fw_size);
+
+static void atl_dev_info_get(struct rte_eth_dev *dev,
+			       struct rte_eth_dev_info *dev_info);
+
+static const uint32_t *atl_dev_supported_ptypes_get(struct rte_eth_dev *dev);
+
+static int atl_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
+
+/* VLAN stuff */
+static int atl_vlan_filter_set(struct rte_eth_dev *dev,
+		uint16_t vlan_id, int on);
+
+static int atl_vlan_offload_set(struct rte_eth_dev *dev, int mask);
+
+static void atl_vlan_strip_queue_set(struct rte_eth_dev *dev,
+				     uint16_t queue_id, int on);
+
+static int atl_vlan_tpid_set(struct rte_eth_dev *dev,
+			     enum rte_vlan_type vlan_type, uint16_t tpid);
+
+/* EEPROM */
+static int atl_dev_get_eeprom_length(struct rte_eth_dev *dev);
+static int atl_dev_get_eeprom(struct rte_eth_dev *dev,
+			      struct rte_dev_eeprom_info *eeprom);
+static int atl_dev_set_eeprom(struct rte_eth_dev *dev,
+			      struct rte_dev_eeprom_info *eeprom);
+
+/* Regs */
+static int atl_dev_get_regs(struct rte_eth_dev *dev,
+			    struct rte_dev_reg_info *regs);
+
+/* Flow control */
+static int atl_flow_ctrl_get(struct rte_eth_dev *dev,
+			       struct rte_eth_fc_conf *fc_conf);
+static int atl_flow_ctrl_set(struct rte_eth_dev *dev,
+			       struct rte_eth_fc_conf *fc_conf);
+
+static void atl_dev_link_status_print(struct rte_eth_dev *dev);
+
+/* Interrupts */
+static int atl_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
+static int atl_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on);
+static int atl_dev_interrupt_get_status(struct rte_eth_dev *dev);
+static int atl_dev_interrupt_action(struct rte_eth_dev *dev,
+				    struct rte_intr_handle *handle);
+static void atl_dev_interrupt_handler(void *param);
+
+
+static int atl_add_mac_addr(struct rte_eth_dev *dev,
+			    struct ether_addr *mac_addr,
+			    uint32_t index, uint32_t pool);
+static void atl_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
+static int atl_set_default_mac_addr(struct rte_eth_dev *dev,
+					   struct ether_addr *mac_addr);
+
+static int atl_dev_set_mc_addr_list(struct rte_eth_dev *dev,
+				    struct ether_addr *mc_addr_set,
+				    uint32_t nb_mc_addr);
+
+/* RSS */
+static int atl_reta_update(struct rte_eth_dev *dev,
+			     struct rte_eth_rss_reta_entry64 *reta_conf,
+			     uint16_t reta_size);
+static int atl_reta_query(struct rte_eth_dev *dev,
+			    struct rte_eth_rss_reta_entry64 *reta_conf,
+			    uint16_t reta_size);
+static int atl_rss_hash_update(struct rte_eth_dev *dev,
+				 struct rte_eth_rss_conf *rss_conf);
+static int atl_rss_hash_conf_get(struct rte_eth_dev *dev,
+				   struct rte_eth_rss_conf *rss_conf);
+
+
+static int eth_atl_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+	struct rte_pci_device *pci_dev);
+static int eth_atl_pci_remove(struct rte_pci_device *pci_dev);
+
+static void atl_dev_info_get(struct rte_eth_dev *dev,
+				struct rte_eth_dev_info *dev_info);
+
+int atl_logtype_init;
+int atl_logtype_driver;
+
+/*
+ * The set of PCI devices this driver supports
+ */
+static const struct rte_pci_id pci_id_atl_map[] = {
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_0001) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_D100) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_D107) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_D108) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_D109) },
+
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC100) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC107) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC108) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC109) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC111) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC112) },
+
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC100S) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC107S) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC108S) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC109S) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC111S) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC112S) },
+
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC111E) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC112E) },
+	{ .vendor_id = 0, /* sentinel */ },
+};
+
+static struct rte_pci_driver rte_atl_pmd = {
+	.id_table = pci_id_atl_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+		     RTE_PCI_DRV_IOVA_AS_VA,
+	.probe = eth_atl_pci_probe,
+	.remove = eth_atl_pci_remove,
+};
+
+#define ATL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_STRIP \
+			| DEV_RX_OFFLOAD_IPV4_CKSUM \
+			| DEV_RX_OFFLOAD_UDP_CKSUM \
+			| DEV_RX_OFFLOAD_TCP_CKSUM \
+			| DEV_RX_OFFLOAD_JUMBO_FRAME)
+
+#define ATL_TX_OFFLOADS (DEV_TX_OFFLOAD_VLAN_INSERT \
+			| DEV_TX_OFFLOAD_IPV4_CKSUM \
+			| DEV_TX_OFFLOAD_UDP_CKSUM \
+			| DEV_TX_OFFLOAD_TCP_CKSUM \
+			| DEV_TX_OFFLOAD_TCP_TSO \
+			| DEV_TX_OFFLOAD_MULTI_SEGS)
+
+static const struct rte_eth_desc_lim rx_desc_lim = {
+	.nb_max = ATL_MAX_RING_DESC,
+	.nb_min = ATL_MIN_RING_DESC,
+	.nb_align = ATL_RXD_ALIGN,
+};
+
+static const struct rte_eth_desc_lim tx_desc_lim = {
+	.nb_max = ATL_MAX_RING_DESC,
+	.nb_min = ATL_MIN_RING_DESC,
+	.nb_align = ATL_TXD_ALIGN,
+	.nb_seg_max = ATL_TX_MAX_SEG,
+	.nb_mtu_seg_max = ATL_TX_MAX_SEG,
+};
+
+#define ATL_XSTATS_FIELD(name) { \
+	#name, \
+	offsetof(struct aq_stats_s, name) \
+}
+
+struct atl_xstats_tbl_s {
+	const char *name;
+	unsigned int offset;
+};
+
+static struct atl_xstats_tbl_s atl_xstats_tbl[] = {
+	ATL_XSTATS_FIELD(uprc),
+	ATL_XSTATS_FIELD(mprc),
+	ATL_XSTATS_FIELD(bprc),
+	ATL_XSTATS_FIELD(erpt),
+	ATL_XSTATS_FIELD(uptc),
+	ATL_XSTATS_FIELD(mptc),
+	ATL_XSTATS_FIELD(bptc),
+	ATL_XSTATS_FIELD(erpr),
+	ATL_XSTATS_FIELD(ubrc),
+	ATL_XSTATS_FIELD(ubtc),
+	ATL_XSTATS_FIELD(mbrc),
+	ATL_XSTATS_FIELD(mbtc),
+	ATL_XSTATS_FIELD(bbrc),
+	ATL_XSTATS_FIELD(bbtc),
+};
+
+static const struct eth_dev_ops atl_eth_dev_ops = {
+	.dev_configure	      = atl_dev_configure,
+	.dev_start	      = atl_dev_start,
+	.dev_stop	      = atl_dev_stop,
+	.dev_set_link_up      = atl_dev_set_link_up,
+	.dev_set_link_down    = atl_dev_set_link_down,
+	.dev_close	      = atl_dev_close,
+	.dev_reset	      = atl_dev_reset,
+
+	/* PROMISC */
+	.promiscuous_enable   = atl_dev_promiscuous_enable,
+	.promiscuous_disable  = atl_dev_promiscuous_disable,
+	.allmulticast_enable  = atl_dev_allmulticast_enable,
+	.allmulticast_disable = atl_dev_allmulticast_disable,
+
+	/* Link */
+	.link_update	      = atl_dev_link_update,
+
+	.get_reg              = atl_dev_get_regs,
+
+	/* Stats */
+	.stats_get	      = atl_dev_stats_get,
+	.xstats_get	      = atl_dev_xstats_get,
+	.xstats_get_names     = atl_dev_xstats_get_names,
+	.stats_reset	      = atl_dev_stats_reset,
+	.xstats_reset	      = atl_dev_stats_reset,
+
+	.fw_version_get       = atl_fw_version_get,
+	.dev_infos_get	      = atl_dev_info_get,
+	.dev_supported_ptypes_get = atl_dev_supported_ptypes_get,
+
+	.mtu_set              = atl_dev_mtu_set,
+
+	/* VLAN */
+	.vlan_filter_set      = atl_vlan_filter_set,
+	.vlan_offload_set     = atl_vlan_offload_set,
+	.vlan_tpid_set        = atl_vlan_tpid_set,
+	.vlan_strip_queue_set = atl_vlan_strip_queue_set,
+
+	/* Queue Control */
+	.rx_queue_start	      = atl_rx_queue_start,
+	.rx_queue_stop	      = atl_rx_queue_stop,
+	.rx_queue_setup       = atl_rx_queue_setup,
+	.rx_queue_release     = atl_rx_queue_release,
+
+	.tx_queue_start	      = atl_tx_queue_start,
+	.tx_queue_stop	      = atl_tx_queue_stop,
+	.tx_queue_setup       = atl_tx_queue_setup,
+	.tx_queue_release     = atl_tx_queue_release,
+
+	.rx_queue_intr_enable = atl_dev_rx_queue_intr_enable,
+	.rx_queue_intr_disable = atl_dev_rx_queue_intr_disable,
+
+	.rx_queue_count       = atl_rx_queue_count,
+	.rx_descriptor_status = atl_dev_rx_descriptor_status,
+	.tx_descriptor_status = atl_dev_tx_descriptor_status,
+
+	/* EEPROM */
+	.get_eeprom_length    = atl_dev_get_eeprom_length,
+	.get_eeprom           = atl_dev_get_eeprom,
+	.set_eeprom           = atl_dev_set_eeprom,
+
+	/* Flow Control */
+	.flow_ctrl_get	      = atl_flow_ctrl_get,
+	.flow_ctrl_set	      = atl_flow_ctrl_set,
+
+	/* MAC */
+	.mac_addr_add	      = atl_add_mac_addr,
+	.mac_addr_remove      = atl_remove_mac_addr,
+	.mac_addr_set	      = atl_set_default_mac_addr,
+	.set_mc_addr_list     = atl_dev_set_mc_addr_list,
+	.rxq_info_get	      = atl_rxq_info_get,
+	.txq_info_get	      = atl_txq_info_get,
+
+	.reta_update          = atl_reta_update,
+	.reta_query           = atl_reta_query,
+	.rss_hash_update      = atl_rss_hash_update,
+	.rss_hash_conf_get    = atl_rss_hash_conf_get,
+};
+
+static inline int32_t
+atl_reset_hw(struct aq_hw_s *hw)
+{
+	return hw_atl_b0_hw_reset(hw);
+}
+
+static inline void
+atl_enable_intr(struct rte_eth_dev *dev)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	hw_atl_itr_irq_msk_setlsw_set(hw, 0xffffffff);
+}
+
+static void
+atl_disable_intr(struct aq_hw_s *hw)
+{
+	PMD_INIT_FUNC_TRACE();
+	hw_atl_itr_irq_msk_clearlsw_set(hw, 0xffffffff);
+}
+
+static int
+eth_atl_dev_init(struct rte_eth_dev *eth_dev)
+{
+	struct atl_adapter *adapter =
+		(struct atl_adapter *)eth_dev->data->dev_private;
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
+	int err = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	eth_dev->dev_ops = &atl_eth_dev_ops;
+	eth_dev->rx_pkt_burst = &atl_recv_pkts;
+	eth_dev->tx_pkt_burst = &atl_xmit_pkts;
+	eth_dev->tx_pkt_prepare = &atl_prep_pkts;
+
+	/* For secondary processes, the primary process has done all the work */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	/* Vendor and Device ID need to be set before init of shared code */
+	hw->device_id = pci_dev->id.device_id;
+	hw->vendor_id = pci_dev->id.vendor_id;
+	hw->mmio = (void *)pci_dev->mem_resource[0].addr;
+
+	/* Hardware configuration - hardcode */
+	adapter->hw_cfg.is_lro = false;
+	adapter->hw_cfg.wol = false;
+	adapter->hw_cfg.is_rss = false;
+	adapter->hw_cfg.num_rss_queues = HW_ATL_B0_RSS_MAX;
+
+	adapter->hw_cfg.link_speed_msk = AQ_NIC_RATE_10G |
+			  AQ_NIC_RATE_5G |
+			  AQ_NIC_RATE_2G5 |
+			  AQ_NIC_RATE_1G |
+			  AQ_NIC_RATE_100M;
+
+	adapter->hw_cfg.flow_control = (AQ_NIC_FC_RX | AQ_NIC_FC_TX);
+	adapter->hw_cfg.aq_rss.indirection_table_size =
+		HW_ATL_B0_RSS_REDIRECTION_MAX;
+
+	hw->aq_nic_cfg = &adapter->hw_cfg;
+
+	/* disable interrupt */
+	atl_disable_intr(hw);
+
+	/* Allocate memory for storing MAC addresses */
+	eth_dev->data->mac_addrs = rte_zmalloc("atlantic", ETHER_ADDR_LEN, 0);
+	if (eth_dev->data->mac_addrs == NULL) {
+		PMD_INIT_LOG(ERR, "MAC Malloc failed");
+		return -ENOMEM;
+	}
+
+	err = hw_atl_utils_initfw(hw, &hw->aq_fw_ops);
+	if (err)
+		return err;
+
+	/* Copy the permanent MAC address */
+	if (hw->aq_fw_ops->get_mac_permanent(hw,
+			eth_dev->data->mac_addrs->addr_bytes) != 0)
+		return -EINVAL;
+
+	/* Reset the hw statistics */
+	atl_dev_stats_reset(eth_dev);
+
+	rte_intr_callback_register(intr_handle,
+				   atl_dev_interrupt_handler, eth_dev);
+
+	/* enable uio/vfio intr/eventfd mapping */
+	rte_intr_enable(intr_handle);
+
+	/* enable support intr */
+	atl_enable_intr(eth_dev);
+
+	return err;
+}
+
+static int
+eth_atl_dev_uninit(struct rte_eth_dev *eth_dev)
+{
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
+	struct aq_hw_s *hw;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return -EPERM;
+
+	hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
+
+	if (hw->adapter_stopped == 0)
+		atl_dev_close(eth_dev);
+
+	eth_dev->dev_ops = NULL;
+	eth_dev->rx_pkt_burst = NULL;
+	eth_dev->tx_pkt_burst = NULL;
+
+	/* disable uio intr before callback unregister */
+	rte_intr_disable(intr_handle);
+	rte_intr_callback_unregister(intr_handle,
+				     atl_dev_interrupt_handler, eth_dev);
+
+	rte_free(eth_dev->data->mac_addrs);
+	eth_dev->data->mac_addrs = NULL;
+
+	return 0;
+}
+
+static int
+eth_atl_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+	struct rte_pci_device *pci_dev)
+{
+	return rte_eth_dev_pci_generic_probe(pci_dev,
+		sizeof(struct atl_adapter), eth_atl_dev_init);
+}
+
+static int
+eth_atl_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_eth_dev_pci_generic_remove(pci_dev, eth_atl_dev_uninit);
+}
+
+static int
+atl_dev_configure(struct rte_eth_dev *dev)
+{
+	struct atl_interrupt *intr =
+		ATL_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* set flag to update link status after init */
+	intr->flags |= ATL_FLAG_NEED_LINK_UPDATE;
+
+	return 0;
+}
+
+/*
+ * Configure device link speed and setup link.
+ * It returns 0 on success.
+ */
+static int
+atl_dev_start(struct rte_eth_dev *dev)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
+	uint32_t intr_vector = 0;
+	uint32_t *link_speeds;
+	uint32_t speed = 0;
+	int status;
+	int err;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* set adapter started */
+	hw->adapter_stopped = 0;
+
+	if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) {
+		PMD_INIT_LOG(ERR,
+		"Invalid link_speeds for port %u, fix speed not supported",
+				dev->data->port_id);
+		return -EINVAL;
+	}
+
+	/* disable uio/vfio intr/eventfd mapping */
+	rte_intr_disable(intr_handle);
+
+	/* reinitialize adapter
+	 * this calls reset and start
+	 */
+	status = atl_reset_hw(hw);
+	if (status != 0)
+		return -EIO;
+
+	err = hw_atl_b0_hw_init(hw, dev->data->mac_addrs->addr_bytes);
+
+	hw_atl_b0_hw_start(hw);
+	/* check and configure queue intr-vector mapping */
+	if ((rte_intr_cap_multiple(intr_handle) ||
+	    !RTE_ETH_DEV_SRIOV(dev).active) &&
+	    dev->data->dev_conf.intr_conf.rxq != 0) {
+		intr_vector = dev->data->nb_rx_queues;
+		if (intr_vector > ATL_MAX_INTR_QUEUE_NUM) {
+			PMD_INIT_LOG(ERR, "At most %d intr queues supported",
+					ATL_MAX_INTR_QUEUE_NUM);
+			return -ENOTSUP;
+		}
+		if (rte_intr_efd_enable(intr_handle, intr_vector)) {
+			PMD_INIT_LOG(ERR, "rte_intr_efd_enable failed");
+			return -1;
+		}
+	}
+
+	if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
+		intr_handle->intr_vec = rte_zmalloc("intr_vec",
+				    dev->data->nb_rx_queues * sizeof(int), 0);
+		if (intr_handle->intr_vec == NULL) {
+			PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
+				     " intr_vec", dev->data->nb_rx_queues);
+			return -ENOMEM;
+		}
+	}
+
+	/* initialize transmission unit */
+	atl_tx_init(dev);
+
+	/* This can fail when allocating mbufs for descriptor rings */
+	err = atl_rx_init(dev);
+	if (err) {
+		PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
+		goto error;
+	}
+
+	PMD_INIT_LOG(DEBUG, "FW version: %u.%u.%u",
+		hw->fw_ver_actual >> 24,
+		(hw->fw_ver_actual >> 16) & 0xFF,
+		hw->fw_ver_actual & 0xFFFF);
+	PMD_INIT_LOG(DEBUG, "Driver version: %s", ATL_PMD_DRIVER_VERSION);
+
+	err = atl_start_queues(dev);
+	if (err < 0) {
+		PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
+		goto error;
+	}
+
+	err = hw->aq_fw_ops->update_link_status(hw);
+
+	if (err)
+		goto error;
+
+	dev->data->dev_link.link_status = hw->aq_link_status.mbps != 0;
+
+	link_speeds = &dev->data->dev_conf.link_speeds;
+
+	speed = 0x0;
+
+	if (*link_speeds == ETH_LINK_SPEED_AUTONEG) {
+		speed = hw->aq_nic_cfg->link_speed_msk;
+	} else {
+		if (*link_speeds & ETH_LINK_SPEED_10G)
+			speed |= AQ_NIC_RATE_10G;
+		if (*link_speeds & ETH_LINK_SPEED_5G)
+			speed |= AQ_NIC_RATE_5G;
+		if (*link_speeds & ETH_LINK_SPEED_1G)
+			speed |= AQ_NIC_RATE_1G;
+		if (*link_speeds & ETH_LINK_SPEED_2_5G)
+			speed |=  AQ_NIC_RATE_2G5;
+		if (*link_speeds & ETH_LINK_SPEED_100M)
+			speed |= AQ_NIC_RATE_100M;
+	}
+
+	err = hw->aq_fw_ops->set_link_speed(hw, speed);
+	if (err)
+		goto error;
+
+	if (rte_intr_allow_others(intr_handle)) {
+		/* check if lsc interrupt is enabled */
+		if (dev->data->dev_conf.intr_conf.lsc != 0)
+			atl_dev_lsc_interrupt_setup(dev, true);
+		else
+			atl_dev_lsc_interrupt_setup(dev, false);
+	} else {
+		rte_intr_callback_unregister(intr_handle,
+					     atl_dev_interrupt_handler, dev);
+		if (dev->data->dev_conf.intr_conf.lsc != 0)
+			PMD_INIT_LOG(INFO, "lsc won't enable because of"
+				     " no intr multiplex");
+	}
+
+	/* check if rxq interrupt is enabled */
+	if (dev->data->dev_conf.intr_conf.rxq != 0 &&
+	    rte_intr_dp_is_en(intr_handle))
+		atl_dev_rxq_interrupt_setup(dev);
+
+	/* enable uio/vfio intr/eventfd mapping */
+	rte_intr_enable(intr_handle);
+
+	/* resume enabled intr since hw reset */
+	atl_enable_intr(dev);
+
+	return 0;
+
+error:
+	atl_stop_queues(dev);
+	return -EIO;
+}
+
+/*
+ * Stop device: disable rx and tx functions to allow for reconfiguring.
+ */
+static void
+atl_dev_stop(struct rte_eth_dev *dev)
+{
+	struct rte_eth_link link;
+	struct aq_hw_s *hw =
+		ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* disable interrupts */
+	atl_disable_intr(hw);
+
+	/* reset the NIC */
+	atl_reset_hw(hw);
+	hw->adapter_stopped = 1;
+
+	atl_stop_queues(dev);
+
+	/* Clear stored conf */
+	dev->data->scattered_rx = 0;
+	dev->data->lro = 0;
+
+	/* Clear recorded link status */
+	memset(&link, 0, sizeof(link));
+	rte_eth_linkstatus_set(dev, &link);
+
+	if (!rte_intr_allow_others(intr_handle))
+		/* resume to the default handler */
+		rte_intr_callback_register(intr_handle,
+					   atl_dev_interrupt_handler,
+					   (void *)dev);
+
+	/* Clean datapath event and queue/vec mapping */
+	rte_intr_efd_disable(intr_handle);
+	if (intr_handle->intr_vec != NULL) {
+		rte_free(intr_handle->intr_vec);
+		intr_handle->intr_vec = NULL;
+	}
+}
+
+/*
+ * Set device link up: enable tx.
+ */
+static int
+atl_dev_set_link_up(struct rte_eth_dev *dev)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	return hw->aq_fw_ops->set_link_speed(hw,
+			hw->aq_nic_cfg->link_speed_msk);
+}
+
+/*
+ * Set device link down: disable tx.
+ */
+static int
+atl_dev_set_link_down(struct rte_eth_dev *dev)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	return hw->aq_fw_ops->set_link_speed(hw, 0);
+}
+
+/*
+ * Reset and stop device.
+ */
+static void
+atl_dev_close(struct rte_eth_dev *dev)
+{
+	PMD_INIT_FUNC_TRACE();
+
+	atl_dev_stop(dev);
+
+	atl_free_queues(dev);
+}
+
+static int
+atl_dev_reset(struct rte_eth_dev *dev)
+{
+	int ret;
+
+	ret = eth_atl_dev_uninit(dev);
+	if (ret)
+		return ret;
+
+	ret = eth_atl_dev_init(dev);
+
+	return ret;
+}
+
+
+static int
+atl_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
+{
+	struct atl_adapter *adapter = ATL_DEV_TO_ADAPTER(dev);
+	struct aq_hw_s *hw = &adapter->hw;
+	struct atl_sw_stats *swstats = &adapter->sw_stats;
+	unsigned int i;
+
+	hw->aq_fw_ops->update_stats(hw);
+
+	/* Fill out the rte_eth_stats statistics structure */
+	stats->ipackets = hw->curr_stats.dma_pkt_rc;
+	stats->ibytes = hw->curr_stats.dma_oct_rc;
+	stats->imissed = hw->curr_stats.dpc;
+	stats->ierrors = hw->curr_stats.erpt;
+
+	stats->opackets = hw->curr_stats.dma_pkt_tc;
+	stats->obytes = hw->curr_stats.dma_oct_tc;
+	stats->oerrors = 0;
+
+	stats->rx_nombuf = swstats->rx_nombuf;
+
+	for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
+		stats->q_ipackets[i] = swstats->q_ipackets[i];
+		stats->q_opackets[i] = swstats->q_opackets[i];
+		stats->q_ibytes[i] = swstats->q_ibytes[i];
+		stats->q_obytes[i] = swstats->q_obytes[i];
+		stats->q_errors[i] = swstats->q_errors[i];
+	}
+	return 0;
+}
+
+static void
+atl_dev_stats_reset(struct rte_eth_dev *dev)
+{
+	struct atl_adapter *adapter = ATL_DEV_TO_ADAPTER(dev);
+	struct aq_hw_s *hw = &adapter->hw;
+
+	hw->aq_fw_ops->update_stats(hw);
+
+	/* Reset software totals */
+	memset(&hw->curr_stats, 0, sizeof(hw->curr_stats));
+
+	memset(&adapter->sw_stats, 0, sizeof(adapter->sw_stats));
+}
+
+static int
+atl_dev_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
+			 struct rte_eth_xstat_name *xstats_names,
+			 unsigned int size)
+{
+	unsigned int i;
+
+	if (!xstats_names)
+		return RTE_DIM(atl_xstats_tbl);
+
+	for (i = 0; i < size && i < RTE_DIM(atl_xstats_tbl); i++)
+		snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
+			atl_xstats_tbl[i].name);
+
+	return size;
+}
+
+static int
+atl_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
+		   unsigned int n)
+{
+	struct atl_adapter *adapter = ATL_DEV_TO_ADAPTER(dev);
+	struct aq_hw_s *hw = &adapter->hw;
+	unsigned int i;
+
+	if (!stats)
+		return 0;
+
+	for (i = 0; i < n && i < RTE_DIM(atl_xstats_tbl); i++) {
+		stats[i].id = i;
+		stats[i].value = *(u64 *)((uint8_t *)&hw->curr_stats +
+					atl_xstats_tbl[i].offset);
+	}
+
+	return n;
+}
+
+static int
+atl_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t fw_ver = 0;
+	unsigned int ret = 0;
+
+	ret = hw_atl_utils_get_fw_version(hw, &fw_ver);
+	if (ret)
+		return -EIO;
+
+	ret = snprintf(fw_version, fw_size, "%u.%u.%u", fw_ver >> 24,
+		       (fw_ver >> 16) & 0xFFU, fw_ver & 0xFFFFU);
+
+	ret += 1; /* add string null-terminator */
+
+	if (fw_size < ret)
+		return ret;
+
+	return 0;
+}
+
+static void
+atl_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
+{
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+
+	dev_info->max_rx_queues = AQ_HW_MAX_RX_QUEUES;
+	dev_info->max_tx_queues = AQ_HW_MAX_TX_QUEUES;
+
+	dev_info->min_rx_bufsize = 1024;
+	dev_info->max_rx_pktlen = HW_ATL_B0_MTU_JUMBO;
+	dev_info->max_mac_addrs = HW_ATL_B0_MAC_MAX;
+	dev_info->max_vfs = pci_dev->max_vfs;
+
+	dev_info->max_hash_mac_addrs = 0;
+	dev_info->max_vmdq_pools = 0;
+	dev_info->vmdq_queue_num = 0;
+
+	dev_info->rx_offload_capa = ATL_RX_OFFLOADS;
+
+	dev_info->tx_offload_capa = ATL_TX_OFFLOADS;
+
+
+	dev_info->default_rxconf = (struct rte_eth_rxconf) {
+		.rx_free_thresh = ATL_DEFAULT_RX_FREE_THRESH,
+	};
+
+	dev_info->default_txconf = (struct rte_eth_txconf) {
+		.tx_free_thresh = ATL_DEFAULT_TX_FREE_THRESH,
+	};
+
+	dev_info->rx_desc_lim = rx_desc_lim;
+	dev_info->tx_desc_lim = tx_desc_lim;
+
+	dev_info->hash_key_size = HW_ATL_B0_RSS_HASHKEY_BITS / 8;
+	dev_info->reta_size = HW_ATL_B0_RSS_REDIRECTION_MAX;
+	dev_info->flow_type_rss_offloads = ATL_RSS_OFFLOAD_ALL;
+
+	dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
+	dev_info->speed_capa |= ETH_LINK_SPEED_100M;
+	dev_info->speed_capa |= ETH_LINK_SPEED_2_5G;
+	dev_info->speed_capa |= ETH_LINK_SPEED_5G;
+}
+
+static const uint32_t *
+atl_dev_supported_ptypes_get(struct rte_eth_dev *dev)
+{
+	static const uint32_t ptypes[] = {
+		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_L2_ETHER_ARP,
+		RTE_PTYPE_L2_ETHER_VLAN,
+		RTE_PTYPE_L3_IPV4,
+		RTE_PTYPE_L3_IPV6,
+		RTE_PTYPE_L4_TCP,
+		RTE_PTYPE_L4_UDP,
+		RTE_PTYPE_L4_SCTP,
+		RTE_PTYPE_L4_ICMP,
+		RTE_PTYPE_UNKNOWN
+	};
+
+	if (dev->rx_pkt_burst == atl_recv_pkts)
+		return ptypes;
+
+	return NULL;
+}
+
+/* return 0 means link status changed, -1 means not changed */
+static int
+atl_dev_link_update(struct rte_eth_dev *dev, int wait __rte_unused)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct atl_interrupt *intr =
+		ATL_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
+	struct rte_eth_link link, old;
+	int err = 0;
+
+	link.link_status = ETH_LINK_DOWN;
+	link.link_speed = 0;
+	link.link_duplex = ETH_LINK_FULL_DUPLEX;
+	link.link_autoneg = hw->is_autoneg ? ETH_LINK_AUTONEG : ETH_LINK_FIXED;
+	memset(&old, 0, sizeof(old));
+
+	/* load old link status */
+	rte_eth_linkstatus_get(dev, &old);
+
+	/* read current link status */
+	err = hw->aq_fw_ops->update_link_status(hw);
+
+	if (err)
+		return 0;
+
+	if (hw->aq_link_status.mbps == 0) {
+		/* write default (down) link status */
+		rte_eth_linkstatus_set(dev, &link);
+		if (link.link_status == old.link_status)
+			return -1;
+		return 0;
+	}
+
+	intr->flags &= ~ATL_FLAG_NEED_LINK_CONFIG;
+
+	link.link_status = ETH_LINK_UP;
+	link.link_duplex = ETH_LINK_FULL_DUPLEX;
+	link.link_speed = hw->aq_link_status.mbps;
+
+	rte_eth_linkstatus_set(dev, &link);
+
+	if (link.link_status == old.link_status)
+		return -1;
+
+	return 0;
+}
+
+static void
+atl_dev_promiscuous_enable(struct rte_eth_dev *dev)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	hw_atl_rpfl2promiscuous_mode_en_set(hw, true);
+}
+
+static void
+atl_dev_promiscuous_disable(struct rte_eth_dev *dev)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	hw_atl_rpfl2promiscuous_mode_en_set(hw, false);
+}
+
+static void
+atl_dev_allmulticast_enable(struct rte_eth_dev *dev)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	hw_atl_rpfl2_accept_all_mc_packets_set(hw, true);
+}
+
+static void
+atl_dev_allmulticast_disable(struct rte_eth_dev *dev)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	if (dev->data->promiscuous == 1)
+		return; /* must remain in all_multicast mode */
+
+	hw_atl_rpfl2_accept_all_mc_packets_set(hw, false);
+}
+
+/**
+ * It clears the interrupt causes and enables the interrupt.
+ * It will be called once only during nic initialized.
+ *
+ * @param dev
+ *  Pointer to struct rte_eth_dev.
+ * @param on
+ *  Enable or Disable.
+ *
+ * @return
+ *  - On success, zero.
+ *  - On failure, a negative value.
+ */
+
+static int
+atl_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on __rte_unused)
+{
+	atl_dev_link_status_print(dev);
+	return 0;
+}
+
+static int
+atl_dev_rxq_interrupt_setup(struct rte_eth_dev *dev __rte_unused)
+{
+	return 0;
+}
+
+
+static int
+atl_dev_interrupt_get_status(struct rte_eth_dev *dev)
+{
+	struct atl_interrupt *intr =
+		ATL_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	u64 cause = 0;
+
+	hw_atl_b0_hw_irq_read(hw, &cause);
+
+	atl_disable_intr(hw);
+	intr->flags = cause & BIT(ATL_IRQ_CAUSE_LINK) ?
+			ATL_FLAG_NEED_LINK_UPDATE : 0;
+
+	return 0;
+}
+
+/**
+ * It gets and then prints the link status.
+ *
+ * @param dev
+ *  Pointer to struct rte_eth_dev.
+ *
+ * @return
+ *  - On success, zero.
+ *  - On failure, a negative value.
+ */
+static void
+atl_dev_link_status_print(struct rte_eth_dev *dev)
+{
+	struct rte_eth_link link;
+
+	memset(&link, 0, sizeof(link));
+	rte_eth_linkstatus_get(dev, &link);
+	if (link.link_status) {
+		PMD_DRV_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
+					(int)(dev->data->port_id),
+					(unsigned int)link.link_speed,
+			link.link_duplex == ETH_LINK_FULL_DUPLEX ?
+					"full-duplex" : "half-duplex");
+	} else {
+		PMD_DRV_LOG(INFO, " Port %d: Link Down",
+				(int)(dev->data->port_id));
+	}
+
+
+#ifdef DEBUG
+{
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+
+	PMD_DRV_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
+				pci_dev->addr.domain,
+				pci_dev->addr.bus,
+				pci_dev->addr.devid,
+				pci_dev->addr.function);
+}
+#endif
+
+	PMD_DRV_LOG(INFO, "Link speed:%d", link.link_speed);
+}
+
+/*
+ * It executes link_update after knowing an interrupt occurred.
+ *
+ * @param dev
+ *  Pointer to struct rte_eth_dev.
+ *
+ * @return
+ *  - On success, zero.
+ *  - On failure, a negative value.
+ */
+static int
+atl_dev_interrupt_action(struct rte_eth_dev *dev,
+			   struct rte_intr_handle *intr_handle)
+{
+	struct atl_interrupt *intr =
+		ATL_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
+
+	if (intr->flags & ATL_FLAG_NEED_LINK_UPDATE) {
+		atl_dev_link_update(dev, 0);
+		intr->flags &= ~ATL_FLAG_NEED_LINK_UPDATE;
+		atl_dev_link_status_print(dev);
+		_rte_eth_dev_callback_process(dev,
+			RTE_ETH_EVENT_INTR_LSC, NULL);
+	}
+
+	atl_enable_intr(dev);
+	rte_intr_enable(intr_handle);
+
+	return 0;
+}
+
+/**
+ * Interrupt handler triggered by NIC  for handling
+ * specific interrupt.
+ *
+ * @param handle
+ *  Pointer to interrupt handle.
+ * @param param
+ *  The address of parameter (struct rte_eth_dev *) regsitered before.
+ *
+ * @return
+ *  void
+ */
+static void
+atl_dev_interrupt_handler(void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+
+	atl_dev_interrupt_get_status(dev);
+	atl_dev_interrupt_action(dev, dev->intr_handle);
+}
+
+#define SFP_EEPROM_SIZE 0xff
+
+static int
+atl_dev_get_eeprom_length(struct rte_eth_dev *dev __rte_unused)
+{
+	return SFP_EEPROM_SIZE;
+}
+
+static int
+atl_dev_get_eeprom(struct rte_eth_dev *dev, struct rte_dev_eeprom_info *eeprom)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	if (hw->aq_fw_ops->get_eeprom == NULL)
+		return -ENOTSUP;
+
+	if (eeprom->length != SFP_EEPROM_SIZE || eeprom->data == NULL)
+		return -EINVAL;
+
+	return hw->aq_fw_ops->get_eeprom(hw, eeprom->data, eeprom->length);
+}
+
+static int
+atl_dev_set_eeprom(struct rte_eth_dev *dev, struct rte_dev_eeprom_info *eeprom)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	if (hw->aq_fw_ops->set_eeprom == NULL)
+		return -ENOTSUP;
+
+	if (eeprom->length != SFP_EEPROM_SIZE || eeprom->data == NULL)
+		return -EINVAL;
+
+	return hw->aq_fw_ops->set_eeprom(hw, eeprom->data, eeprom->length);
+}
+
+static int
+atl_dev_get_regs(struct rte_eth_dev *dev, struct rte_dev_reg_info *regs)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	u32 mif_id;
+	int err;
+
+	if (regs->data == NULL) {
+		regs->length = hw_atl_utils_hw_get_reg_length();
+		regs->width = sizeof(u32);
+		return 0;
+	}
+
+	/* Only full register dump is supported */
+	if (regs->length && regs->length != hw_atl_utils_hw_get_reg_length())
+		return -ENOTSUP;
+
+	err = hw_atl_utils_hw_get_regs(hw, regs->data);
+
+	/* Device version */
+	mif_id = hw_atl_reg_glb_mif_id_get(hw);
+	regs->version = mif_id & 0xFFU;
+
+	return err;
+}
+
+static int
+atl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	if (hw->aq_nic_cfg->flow_control == AQ_NIC_FC_OFF)
+		fc_conf->mode = RTE_FC_NONE;
+	else if (hw->aq_nic_cfg->flow_control & (AQ_NIC_FC_RX | AQ_NIC_FC_TX))
+		fc_conf->mode = RTE_FC_FULL;
+	else if (hw->aq_nic_cfg->flow_control & AQ_NIC_FC_RX)
+		fc_conf->mode = RTE_FC_RX_PAUSE;
+	else if (hw->aq_nic_cfg->flow_control & AQ_NIC_FC_RX)
+		fc_conf->mode = RTE_FC_TX_PAUSE;
+
+	return 0;
+}
+
+static int
+atl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t old_flow_control = hw->aq_nic_cfg->flow_control;
+
+
+	if (hw->aq_fw_ops->set_flow_control == NULL)
+		return -ENOTSUP;
+
+	if (fc_conf->mode == RTE_FC_NONE)
+		hw->aq_nic_cfg->flow_control = AQ_NIC_FC_OFF;
+	else if (fc_conf->mode == RTE_FC_RX_PAUSE)
+		hw->aq_nic_cfg->flow_control = AQ_NIC_FC_RX;
+	else if (fc_conf->mode == RTE_FC_TX_PAUSE)
+		hw->aq_nic_cfg->flow_control = AQ_NIC_FC_TX;
+	else if (fc_conf->mode == RTE_FC_FULL)
+		hw->aq_nic_cfg->flow_control = (AQ_NIC_FC_RX | AQ_NIC_FC_TX);
+
+	if (old_flow_control != hw->aq_nic_cfg->flow_control)
+		return hw->aq_fw_ops->set_flow_control(hw);
+
+	return 0;
+}
+
+static int
+atl_update_mac_addr(struct rte_eth_dev *dev, uint32_t index,
+		    u8 *mac_addr, bool enable)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	unsigned int h = 0U;
+	unsigned int l = 0U;
+	int err;
+
+	if (mac_addr) {
+		h = (mac_addr[0] << 8) | (mac_addr[1]);
+		l = (mac_addr[2] << 24) | (mac_addr[3] << 16) |
+			(mac_addr[4] << 8) | mac_addr[5];
+	}
+
+	hw_atl_rpfl2_uc_flr_en_set(hw, 0U, index);
+	hw_atl_rpfl2unicast_dest_addresslsw_set(hw, l, index);
+	hw_atl_rpfl2unicast_dest_addressmsw_set(hw, h, index);
+
+	if (enable)
+		hw_atl_rpfl2_uc_flr_en_set(hw, 1U, index);
+
+	err = aq_hw_err_from_flags(hw);
+
+	return err;
+}
+
+static int
+atl_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
+			uint32_t index __rte_unused, uint32_t pool __rte_unused)
+{
+	if (is_zero_ether_addr(mac_addr)) {
+		PMD_DRV_LOG(ERR, "Invalid Ethernet Address");
+		return -EINVAL;
+	}
+
+	return atl_update_mac_addr(dev, index, (u8 *)mac_addr, true);
+}
+
+static void
+atl_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
+{
+	atl_update_mac_addr(dev, index, NULL, false);
+}
+
+static int
+atl_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
+{
+	atl_remove_mac_addr(dev, 0);
+	atl_add_mac_addr(dev, addr, 0, 0);
+	return 0;
+}
+
+static int
+atl_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
+{
+	struct rte_eth_dev_info dev_info;
+	uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
+
+	atl_dev_info_get(dev, &dev_info);
+
+	if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen))
+		return -EINVAL;
+
+	/* update max frame size */
+	dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
+
+	return 0;
+}
+
+static int
+atl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
+{
+	struct aq_hw_cfg_s *cfg =
+		ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	int err = 0;
+	int i = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < HW_ATL_B0_MAX_VLAN_IDS; i++) {
+		if (cfg->vlan_filter[i] == vlan_id) {
+			if (!on) {
+				/* Disable VLAN filter. */
+				hw_atl_rpf_vlan_flr_en_set(hw, 0U, i);
+
+				/* Clear VLAN filter entry */
+				cfg->vlan_filter[i] = 0;
+			}
+			break;
+		}
+	}
+
+	/* VLAN_ID was not found. So, nothing to delete. */
+	if (i == HW_ATL_B0_MAX_VLAN_IDS && !on)
+		goto exit;
+
+	/* VLAN_ID already exist, or already removed above. Nothing to do. */
+	if (i != HW_ATL_B0_MAX_VLAN_IDS)
+		goto exit;
+
+	/* Try to found free VLAN filter to add new VLAN_ID */
+	for (i = 0; i < HW_ATL_B0_MAX_VLAN_IDS; i++) {
+		if (cfg->vlan_filter[i] == 0)
+			break;
+	}
+
+	if (i == HW_ATL_B0_MAX_VLAN_IDS) {
+		/* We have no free VLAN filter to add new VLAN_ID*/
+		err = -ENOMEM;
+		goto exit;
+	}
+
+	cfg->vlan_filter[i] = vlan_id;
+	hw_atl_rpf_vlan_flr_act_set(hw, 1U, i);
+	hw_atl_rpf_vlan_id_flr_set(hw, vlan_id, i);
+	hw_atl_rpf_vlan_flr_en_set(hw, 1U, i);
+
+exit:
+	/* Enable VLAN promisc mode if vlan_filter empty  */
+	for (i = 0; i < HW_ATL_B0_MAX_VLAN_IDS; i++) {
+		if (cfg->vlan_filter[i] != 0)
+			break;
+	}
+
+	hw_atl_rpf_vlan_prom_mode_en_set(hw, i == HW_ATL_B0_MAX_VLAN_IDS);
+
+	return err;
+}
+
+static int
+atl_enable_vlan_filter(struct rte_eth_dev *dev, int en)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct aq_hw_cfg_s *cfg =
+		ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < HW_ATL_B0_MAX_VLAN_IDS; i++) {
+		if (cfg->vlan_filter[i])
+			hw_atl_rpf_vlan_flr_en_set(hw, en, i);
+	}
+	return 0;
+}
+
+static int
+atl_vlan_offload_set(struct rte_eth_dev *dev, int mask)
+{
+	struct aq_hw_cfg_s *cfg =
+		ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	int ret = 0;
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	ret = atl_enable_vlan_filter(dev, mask & ETH_VLAN_FILTER_MASK);
+
+	cfg->vlan_strip = !!(mask & ETH_VLAN_STRIP_MASK);
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++)
+		hw_atl_rpo_rx_desc_vlan_stripping_set(hw, cfg->vlan_strip, i);
+
+	if (mask & ETH_VLAN_EXTEND_MASK)
+		ret = -ENOTSUP;
+
+	return ret;
+}
+
+static int
+atl_vlan_tpid_set(struct rte_eth_dev *dev, enum rte_vlan_type vlan_type,
+		  uint16_t tpid)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	int err = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	switch (vlan_type) {
+	case ETH_VLAN_TYPE_INNER:
+		hw_atl_rpf_vlan_inner_etht_set(hw, tpid);
+		break;
+	case ETH_VLAN_TYPE_OUTER:
+		hw_atl_rpf_vlan_outer_etht_set(hw, tpid);
+		break;
+	default:
+		PMD_DRV_LOG(ERR, "Unsupported VLAN type");
+		err = -ENOTSUP;
+	}
+
+	return err;
+}
+
+static void
+atl_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue_id, int on)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (queue_id > dev->data->nb_rx_queues) {
+		PMD_DRV_LOG(ERR, "Invalid queue id");
+		return;
+	}
+
+	hw_atl_rpo_rx_desc_vlan_stripping_set(hw, on, queue_id);
+}
+
+static int
+atl_dev_set_mc_addr_list(struct rte_eth_dev *dev,
+			  struct ether_addr *mc_addr_set,
+			  uint32_t nb_mc_addr)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	u32 i;
+
+	if (nb_mc_addr > AQ_HW_MULTICAST_ADDRESS_MAX - HW_ATL_B0_MAC_MIN)
+		return -EINVAL;
+
+	/* Update whole uc filters table */
+	for (i = 0; i < AQ_HW_MULTICAST_ADDRESS_MAX - HW_ATL_B0_MAC_MIN; i++) {
+		u8 *mac_addr = NULL;
+		u32 l = 0, h = 0;
+
+		if (i < nb_mc_addr) {
+			mac_addr = mc_addr_set[i].addr_bytes;
+			l = (mac_addr[2] << 24) | (mac_addr[3] << 16) |
+				(mac_addr[4] << 8) | mac_addr[5];
+			h = (mac_addr[0] << 8) | mac_addr[1];
+		}
+
+		hw_atl_rpfl2_uc_flr_en_set(hw, 0U, HW_ATL_B0_MAC_MIN + i);
+		hw_atl_rpfl2unicast_dest_addresslsw_set(hw, l,
+							HW_ATL_B0_MAC_MIN + i);
+		hw_atl_rpfl2unicast_dest_addressmsw_set(hw, h,
+							HW_ATL_B0_MAC_MIN + i);
+		hw_atl_rpfl2_uc_flr_en_set(hw, !!mac_addr,
+					   HW_ATL_B0_MAC_MIN + i);
+	}
+
+	return 0;
+}
+
+static int
+atl_reta_update(struct rte_eth_dev *dev,
+		   struct rte_eth_rss_reta_entry64 *reta_conf,
+		   uint16_t reta_size)
+{
+	int i;
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct aq_hw_cfg_s *cf = ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
+
+	for (i = 0; i < reta_size && i < cf->aq_rss.indirection_table_size; i++)
+		cf->aq_rss.indirection_table[i] = min(reta_conf->reta[i],
+					dev->data->nb_rx_queues - 1);
+
+	hw_atl_b0_hw_rss_set(hw, &cf->aq_rss);
+	return 0;
+}
+
+static int
+atl_reta_query(struct rte_eth_dev *dev,
+		    struct rte_eth_rss_reta_entry64 *reta_conf,
+		    uint16_t reta_size)
+{
+	int i;
+	struct aq_hw_cfg_s *cf = ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
+
+	for (i = 0; i < reta_size && i < cf->aq_rss.indirection_table_size; i++)
+		reta_conf->reta[i] = cf->aq_rss.indirection_table[i];
+	reta_conf->mask = ~0U;
+	return 0;
+}
+
+static int
+atl_rss_hash_update(struct rte_eth_dev *dev,
+				 struct rte_eth_rss_conf *rss_conf)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct aq_hw_cfg_s *cfg =
+		ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
+	static u8 def_rss_key[40] = {
+		0x1e, 0xad, 0x71, 0x87, 0x65, 0xfc, 0x26, 0x7d,
+		0x0d, 0x45, 0x67, 0x74, 0xcd, 0x06, 0x1a, 0x18,
+		0xb6, 0xc1, 0xf0, 0xc7, 0xbb, 0x18, 0xbe, 0xf8,
+		0x19, 0x13, 0x4b, 0xa9, 0xd0, 0x3e, 0xfe, 0x70,
+		0x25, 0x03, 0xab, 0x50, 0x6a, 0x8b, 0x82, 0x0c
+	};
+
+	cfg->is_rss = !!rss_conf->rss_hf;
+	if (rss_conf->rss_key) {
+		memcpy(cfg->aq_rss.hash_secret_key, rss_conf->rss_key,
+		       rss_conf->rss_key_len);
+		cfg->aq_rss.hash_secret_key_size = rss_conf->rss_key_len;
+	} else {
+		memcpy(cfg->aq_rss.hash_secret_key, def_rss_key,
+		       sizeof(def_rss_key));
+		cfg->aq_rss.hash_secret_key_size = sizeof(def_rss_key);
+	}
+
+	hw_atl_b0_hw_rss_set(hw, &cfg->aq_rss);
+	hw_atl_b0_hw_rss_hash_set(hw, &cfg->aq_rss);
+	return 0;
+}
+
+static int
+atl_rss_hash_conf_get(struct rte_eth_dev *dev,
+				 struct rte_eth_rss_conf *rss_conf)
+{
+	struct aq_hw_cfg_s *cfg =
+		ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
+
+	rss_conf->rss_hf = cfg->is_rss ? ATL_RSS_OFFLOAD_ALL : 0;
+	if (rss_conf->rss_key) {
+		rss_conf->rss_key_len = cfg->aq_rss.hash_secret_key_size;
+		memcpy(rss_conf->rss_key, cfg->aq_rss.hash_secret_key,
+		       rss_conf->rss_key_len);
+	}
+
+	return 0;
+}
+
+RTE_PMD_REGISTER_PCI(net_atlantic, rte_atl_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(net_atlantic, pci_id_atl_map);
+RTE_PMD_REGISTER_KMOD_DEP(net_atlantic, "* igb_uio | uio_pci_generic");
+
+RTE_INIT(atl_init_log)
+{
+	atl_logtype_init = rte_log_register("pmd.net.atlantic.init");
+	if (atl_logtype_init >= 0)
+		rte_log_set_level(atl_logtype_init, RTE_LOG_NOTICE);
+	atl_logtype_driver = rte_log_register("pmd.net.atlantic.driver");
+	if (atl_logtype_driver >= 0)
+		rte_log_set_level(atl_logtype_driver, RTE_LOG_NOTICE);
+}
+
diff --git a/drivers/net/atlantic/atl_ethdev.h b/drivers/net/atlantic/atl_ethdev.h
new file mode 100644
index 00000000..1e29999b
--- /dev/null
+++ b/drivers/net/atlantic/atl_ethdev.h
@@ -0,0 +1,107 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Aquantia Corporation
+ */
+
+#ifndef _ATLANTIC_ETHDEV_H_
+#define _ATLANTIC_ETHDEV_H_
+#include <rte_errno.h>
+#include "rte_ethdev.h"
+
+#include "atl_types.h"
+#include "hw_atl/hw_atl_utils.h"
+
+#define ATL_RSS_OFFLOAD_ALL ( \
+	ETH_RSS_IPV4 | \
+	ETH_RSS_NONFRAG_IPV4_TCP | \
+	ETH_RSS_NONFRAG_IPV4_UDP | \
+	ETH_RSS_IPV6 | \
+	ETH_RSS_NONFRAG_IPV6_TCP | \
+	ETH_RSS_NONFRAG_IPV6_UDP | \
+	ETH_RSS_IPV6_EX | \
+	ETH_RSS_IPV6_TCP_EX | \
+	ETH_RSS_IPV6_UDP_EX)
+
+#define ATL_DEV_PRIVATE_TO_HW(adapter) \
+	(&((struct atl_adapter *)adapter)->hw)
+
+#define ATL_DEV_TO_ADAPTER(dev) \
+	((struct atl_adapter *)(dev)->data->dev_private)
+
+#define ATL_DEV_PRIVATE_TO_INTR(adapter) \
+	(&((struct atl_adapter *)adapter)->intr)
+
+#define ATL_DEV_PRIVATE_TO_CFG(adapter) \
+	(&((struct atl_adapter *)adapter)->hw_cfg)
+
+#define ATL_FLAG_NEED_LINK_UPDATE (uint32_t)(1 << 0)
+#define ATL_FLAG_NEED_LINK_CONFIG (uint32_t)(4 << 0)
+
+struct atl_interrupt {
+	uint32_t flags;
+	uint32_t mask;
+};
+
+/*
+ * Structure to store private data for each driver instance (for each port).
+ */
+struct atl_adapter {
+	struct aq_hw_s             hw;
+	struct aq_hw_cfg_s         hw_cfg;
+	struct atl_sw_stats        sw_stats;
+	struct atl_interrupt       intr;
+};
+
+/*
+ * RX/TX function prototypes
+ */
+void atl_rx_queue_release(void *rxq);
+void atl_tx_queue_release(void *txq);
+
+int atl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+		uint16_t nb_rx_desc, unsigned int socket_id,
+		const struct rte_eth_rxconf *rx_conf,
+		struct rte_mempool *mb_pool);
+
+int atl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
+		uint16_t nb_tx_desc, unsigned int socket_id,
+		const struct rte_eth_txconf *tx_conf);
+
+uint32_t atl_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+
+int atl_dev_rx_descriptor_status(void *rx_queue, uint16_t offset);
+int atl_dev_tx_descriptor_status(void *tx_queue, uint16_t offset);
+
+int atl_dev_rx_queue_intr_enable(struct rte_eth_dev *eth_dev,
+				 uint16_t queue_id);
+int atl_dev_rx_queue_intr_disable(struct rte_eth_dev *eth_dev,
+				  uint16_t queue_id);
+
+int atl_rx_init(struct rte_eth_dev *dev);
+int atl_tx_init(struct rte_eth_dev *dev);
+
+int atl_start_queues(struct rte_eth_dev *dev);
+int atl_stop_queues(struct rte_eth_dev *dev);
+void atl_free_queues(struct rte_eth_dev *dev);
+
+int atl_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int atl_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+
+int atl_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id);
+int atl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id);
+
+void atl_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
+	struct rte_eth_rxq_info *qinfo);
+
+void atl_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
+	struct rte_eth_txq_info *qinfo);
+
+uint16_t atl_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts);
+
+uint16_t atl_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+		uint16_t nb_pkts);
+
+uint16_t atl_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+		uint16_t nb_pkts);
+
+#endif /* _ATLANTIC_ETHDEV_H_ */
diff --git a/drivers/net/atlantic/atl_hw_regs.c b/drivers/net/atlantic/atl_hw_regs.c
new file mode 100644
index 00000000..bd42c834
--- /dev/null
+++ b/drivers/net/atlantic/atl_hw_regs.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+/* Copyright (C) 2014-2017 aQuantia Corporation. */
+
+/* File aq_hw_utils.c: Definitions of helper functions used across
+ * hardware layer.
+ */
+
+#include "atl_hw_regs.h"
+
+#include <rte_io.h>
+#include <rte_byteorder.h>
+
+void aq_hw_write_reg_bit(struct aq_hw_s *aq_hw, u32 addr, u32 msk,
+			 u32 shift, u32 val)
+{
+	if (msk ^ ~0) {
+		u32 reg_old, reg_new;
+
+		reg_old = aq_hw_read_reg(aq_hw, addr);
+		reg_new = (reg_old & (~msk)) | (val << shift);
+
+		if (reg_old != reg_new)
+			aq_hw_write_reg(aq_hw, addr, reg_new);
+	} else {
+		aq_hw_write_reg(aq_hw, addr, val);
+	}
+}
+
+u32 aq_hw_read_reg_bit(struct aq_hw_s *aq_hw, u32 addr, u32 msk, u32 shift)
+{
+	return ((aq_hw_read_reg(aq_hw, addr) & msk) >> shift);
+}
+
+u32 aq_hw_read_reg(struct aq_hw_s *hw, u32 reg)
+{
+	return rte_le_to_cpu_32(rte_read32((u8 *)hw->mmio + reg));
+}
+
+void aq_hw_write_reg(struct aq_hw_s *hw, u32 reg, u32 value)
+{
+	rte_write32((rte_cpu_to_le_32(value)), (u8 *)hw->mmio + reg);
+}
+
+int aq_hw_err_from_flags(struct aq_hw_s *hw)
+{
+	int err = 0;
+
+	if (aq_hw_read_reg(hw, 0x10U) == ~0U)
+		return -ENXIO;
+
+	return err;
+}
diff --git a/drivers/net/atlantic/atl_hw_regs.h b/drivers/net/atlantic/atl_hw_regs.h
new file mode 100644
index 00000000..a2d6ca80
--- /dev/null
+++ b/drivers/net/atlantic/atl_hw_regs.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) */
+/* Copyright (C) 2014-2017 aQuantia Corporation. */
+
+/* File aq_hw_utils.h: Declaration of helper functions used across hardware
+ * layer.
+ */
+
+#ifndef AQ_HW_UTILS_H
+#define AQ_HW_UTILS_H
+
+#include <rte_common.h>
+#include <rte_io.h>
+#include <rte_byteorder.h>
+#include <rte_random.h>
+#include <rte_cycles.h>
+#include "atl_common.h"
+#include "atl_types.h"
+
+
+#ifndef HIDWORD
+#define LODWORD(_qw)    ((u32)(_qw))
+#define HIDWORD(_qw)    ((u32)(((_qw) >> 32) & 0xffffffff))
+#endif
+
+#define AQ_HW_SLEEP(_US_) rte_delay_ms(_US_)
+
+#define mdelay rte_delay_ms
+#define udelay rte_delay_us
+#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
+#define BIT(x)	(1UL << (x))
+
+#define AQ_HW_WAIT_FOR(_B_, _US_, _N_) \
+do { \
+	unsigned int AQ_HW_WAIT_FOR_i; \
+	for (AQ_HW_WAIT_FOR_i = _N_; (!(_B_)) && (AQ_HW_WAIT_FOR_i);\
+	--AQ_HW_WAIT_FOR_i) {\
+		udelay(_US_); \
+	} \
+	if (!AQ_HW_WAIT_FOR_i) {\
+		err = -ETIMEDOUT; \
+	} \
+} while (0)
+
+#define ATL_WRITE_FLUSH(aq_hw) { (void)aq_hw_read_reg(aq_hw, 0x10); }
+
+void aq_hw_write_reg_bit(struct aq_hw_s *aq_hw, u32 addr, u32 msk,
+			 u32 shift, u32 val);
+u32 aq_hw_read_reg_bit(struct aq_hw_s *aq_hw, u32 addr, u32 msk, u32 shift);
+u32 aq_hw_read_reg(struct aq_hw_s *hw, u32 reg);
+void aq_hw_write_reg(struct aq_hw_s *hw, u32 reg, u32 value);
+int aq_hw_err_from_flags(struct aq_hw_s *hw);
+
+#endif /* AQ_HW_UTILS_H */
diff --git a/drivers/net/atlantic/atl_logs.h b/drivers/net/atlantic/atl_logs.h
new file mode 100644
index 00000000..e3dba334
--- /dev/null
+++ b/drivers/net/atlantic/atl_logs.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Aquantia Corporation
+ */
+#ifndef ATL_LOGS_H
+#define ATL_LOGS_H
+
+#include <rte_log.h>
+
+extern int atl_logtype_init;
+
+#define PMD_INIT_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, atl_logtype_init, \
+		"%s(): " fmt "\n", __func__, ##args)
+
+#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
+
+#define PMD_RX_LOG(level, fmt, args...) \
+	RTE_LOG_DP(level, PMD, "%s(): " fmt "\n", __func__, ## args)
+
+#define PMD_TX_LOG(level, fmt, args...) \
+	RTE_LOG_DP(level, PMD, "%s(): " fmt "\n", __func__, ## args)
+
+extern int atl_logtype_driver;
+#define PMD_DRV_LOG_RAW(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, atl_logtype_driver, "%s(): " fmt, \
+		__func__, ## args)
+
+#define PMD_DRV_LOG(level, fmt, args...) \
+	PMD_DRV_LOG_RAW(level, fmt "\n", ## args)
+
+#endif
diff --git a/drivers/net/atlantic/atl_rxtx.c b/drivers/net/atlantic/atl_rxtx.c
new file mode 100644
index 00000000..fd909476
--- /dev/null
+++ b/drivers/net/atlantic/atl_rxtx.c
@@ -0,0 +1,1357 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Aquantia Corporation
+ */
+
+#include <rte_malloc.h>
+#include <rte_ethdev_driver.h>
+#include <rte_net.h>
+
+#include "atl_ethdev.h"
+#include "atl_hw_regs.h"
+
+#include "atl_logs.h"
+#include "hw_atl/hw_atl_llh.h"
+#include "hw_atl/hw_atl_b0.h"
+#include "hw_atl/hw_atl_b0_internal.h"
+
+#define ATL_TX_CKSUM_OFFLOAD_MASK (			 \
+	PKT_TX_IP_CKSUM |				 \
+	PKT_TX_L4_MASK |				 \
+	PKT_TX_TCP_SEG)
+
+#define ATL_TX_OFFLOAD_MASK (				 \
+	PKT_TX_VLAN |					 \
+	PKT_TX_IP_CKSUM |				 \
+	PKT_TX_L4_MASK |				 \
+	PKT_TX_TCP_SEG)
+
+#define ATL_TX_OFFLOAD_NOTSUP_MASK \
+	(PKT_TX_OFFLOAD_MASK ^ ATL_TX_OFFLOAD_MASK)
+
+/**
+ * Structure associated with each descriptor of the RX ring of a RX queue.
+ */
+struct atl_rx_entry {
+	struct rte_mbuf *mbuf;
+};
+
+/**
+ * Structure associated with each descriptor of the TX ring of a TX queue.
+ */
+struct atl_tx_entry {
+	struct rte_mbuf *mbuf;
+	uint16_t next_id;
+	uint16_t last_id;
+};
+
+/**
+ * Structure associated with each RX queue.
+ */
+struct atl_rx_queue {
+	struct rte_mempool	*mb_pool;
+	struct hw_atl_rxd_s	*hw_ring;
+	uint64_t		hw_ring_phys_addr;
+	struct atl_rx_entry	*sw_ring;
+	uint16_t		nb_rx_desc;
+	uint16_t		rx_tail;
+	uint16_t		nb_rx_hold;
+	uint16_t		rx_free_thresh;
+	uint16_t		queue_id;
+	uint16_t		port_id;
+	uint16_t		buff_size;
+	bool			l3_csum_enabled;
+	bool			l4_csum_enabled;
+};
+
+/**
+ * Structure associated with each TX queue.
+ */
+struct atl_tx_queue {
+	struct hw_atl_txd_s	*hw_ring;
+	uint64_t		hw_ring_phys_addr;
+	struct atl_tx_entry	*sw_ring;
+	uint16_t		nb_tx_desc;
+	uint16_t		tx_tail;
+	uint16_t		tx_head;
+	uint16_t		queue_id;
+	uint16_t		port_id;
+	uint16_t		tx_free_thresh;
+	uint16_t		tx_free;
+};
+
+static inline void
+atl_reset_rx_queue(struct atl_rx_queue *rxq)
+{
+	struct hw_atl_rxd_s *rxd = NULL;
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < rxq->nb_rx_desc; i++) {
+		rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[i];
+		rxd->buf_addr = 0;
+		rxd->hdr_addr = 0;
+	}
+
+	rxq->rx_tail = 0;
+}
+
+int
+atl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+		   uint16_t nb_rx_desc, unsigned int socket_id,
+		   const struct rte_eth_rxconf *rx_conf,
+		   struct rte_mempool *mb_pool)
+{
+	struct atl_rx_queue *rxq;
+	const struct rte_memzone *mz;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* make sure a valid number of descriptors have been requested */
+	if (nb_rx_desc < AQ_HW_MIN_RX_RING_SIZE ||
+			nb_rx_desc > AQ_HW_MAX_RX_RING_SIZE) {
+		PMD_INIT_LOG(ERR, "Number of Rx descriptors must be "
+		"less than or equal to %d, "
+		"greater than or equal to %d", AQ_HW_MAX_RX_RING_SIZE,
+		AQ_HW_MIN_RX_RING_SIZE);
+		return -EINVAL;
+	}
+
+	/*
+	 * if this queue existed already, free the associated memory. The
+	 * queue cannot be reused in case we need to allocate memory on
+	 * different socket than was previously used.
+	 */
+	if (dev->data->rx_queues[rx_queue_id] != NULL) {
+		atl_rx_queue_release(dev->data->rx_queues[rx_queue_id]);
+		dev->data->rx_queues[rx_queue_id] = NULL;
+	}
+
+	/* allocate memory for the queue structure */
+	rxq = rte_zmalloc_socket("atlantic Rx queue", sizeof(*rxq),
+				 RTE_CACHE_LINE_SIZE, socket_id);
+	if (rxq == NULL) {
+		PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
+		return -ENOMEM;
+	}
+
+	/* setup queue */
+	rxq->mb_pool = mb_pool;
+	rxq->nb_rx_desc = nb_rx_desc;
+	rxq->port_id = dev->data->port_id;
+	rxq->queue_id = rx_queue_id;
+	rxq->rx_free_thresh = rx_conf->rx_free_thresh;
+
+	rxq->l3_csum_enabled = dev->data->dev_conf.rxmode.offloads &
+		DEV_RX_OFFLOAD_IPV4_CKSUM;
+	rxq->l4_csum_enabled = dev->data->dev_conf.rxmode.offloads &
+		(DEV_RX_OFFLOAD_UDP_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM);
+	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
+		PMD_DRV_LOG(ERR, "PMD does not support KEEP_CRC offload");
+
+	/* allocate memory for the software ring */
+	rxq->sw_ring = rte_zmalloc_socket("atlantic sw rx ring",
+				nb_rx_desc * sizeof(struct atl_rx_entry),
+				RTE_CACHE_LINE_SIZE, socket_id);
+	if (rxq->sw_ring == NULL) {
+		PMD_INIT_LOG(ERR,
+			"Port %d: Cannot allocate software ring for queue %d",
+			rxq->port_id, rxq->queue_id);
+		rte_free(rxq);
+		return -ENOMEM;
+	}
+
+	/*
+	 * allocate memory for the hardware descriptor ring. A memzone large
+	 * enough to hold the maximum ring size is requested to allow for
+	 * resizing in later calls to the queue setup function.
+	 */
+	mz = rte_eth_dma_zone_reserve(dev, "rx hw_ring", rx_queue_id,
+				      HW_ATL_B0_MAX_RXD *
+					sizeof(struct hw_atl_rxd_s),
+				      128, socket_id);
+	if (mz == NULL) {
+		PMD_INIT_LOG(ERR,
+			"Port %d: Cannot allocate hardware ring for queue %d",
+			rxq->port_id, rxq->queue_id);
+		rte_free(rxq->sw_ring);
+		rte_free(rxq);
+		return -ENOMEM;
+	}
+	rxq->hw_ring = mz->addr;
+	rxq->hw_ring_phys_addr = mz->iova;
+
+	atl_reset_rx_queue(rxq);
+
+	dev->data->rx_queues[rx_queue_id] = rxq;
+	return 0;
+}
+
+static inline void
+atl_reset_tx_queue(struct atl_tx_queue *txq)
+{
+	struct atl_tx_entry *tx_entry;
+	union hw_atl_txc_s *txc;
+	uint16_t i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (!txq) {
+		PMD_DRV_LOG(ERR, "Pointer to txq is NULL");
+		return;
+	}
+
+	tx_entry = txq->sw_ring;
+
+	for (i = 0; i < txq->nb_tx_desc; i++) {
+		txc = (union hw_atl_txc_s *)&txq->hw_ring[i];
+		txc->flags1 = 0;
+		txc->flags2 = 2;
+	}
+
+	for (i = 0; i < txq->nb_tx_desc; i++) {
+		txq->hw_ring[i].dd = 1;
+		tx_entry[i].mbuf = NULL;
+	}
+
+	txq->tx_tail = 0;
+	txq->tx_head = 0;
+	txq->tx_free = txq->nb_tx_desc - 1;
+}
+
+int
+atl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
+		   uint16_t nb_tx_desc, unsigned int socket_id,
+		   const struct rte_eth_txconf *tx_conf)
+{
+	struct atl_tx_queue *txq;
+	const struct rte_memzone *mz;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* make sure a valid number of descriptors have been requested */
+	if (nb_tx_desc < AQ_HW_MIN_TX_RING_SIZE ||
+		nb_tx_desc > AQ_HW_MAX_TX_RING_SIZE) {
+		PMD_INIT_LOG(ERR, "Number of Tx descriptors must be "
+			"less than or equal to %d, "
+			"greater than or equal to %d", AQ_HW_MAX_TX_RING_SIZE,
+			AQ_HW_MIN_TX_RING_SIZE);
+		return -EINVAL;
+	}
+
+	/*
+	 * if this queue existed already, free the associated memory. The
+	 * queue cannot be reused in case we need to allocate memory on
+	 * different socket than was previously used.
+	 */
+	if (dev->data->tx_queues[tx_queue_id] != NULL) {
+		atl_tx_queue_release(dev->data->tx_queues[tx_queue_id]);
+		dev->data->tx_queues[tx_queue_id] = NULL;
+	}
+
+	/* allocate memory for the queue structure */
+	txq = rte_zmalloc_socket("atlantic Tx queue", sizeof(*txq),
+				 RTE_CACHE_LINE_SIZE, socket_id);
+	if (txq == NULL) {
+		PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
+		return -ENOMEM;
+	}
+
+	/* setup queue */
+	txq->nb_tx_desc = nb_tx_desc;
+	txq->port_id = dev->data->port_id;
+	txq->queue_id = tx_queue_id;
+	txq->tx_free_thresh = tx_conf->tx_free_thresh;
+
+
+	/* allocate memory for the software ring */
+	txq->sw_ring = rte_zmalloc_socket("atlantic sw tx ring",
+				nb_tx_desc * sizeof(struct atl_tx_entry),
+				RTE_CACHE_LINE_SIZE, socket_id);
+	if (txq->sw_ring == NULL) {
+		PMD_INIT_LOG(ERR,
+			"Port %d: Cannot allocate software ring for queue %d",
+			txq->port_id, txq->queue_id);
+		rte_free(txq);
+		return -ENOMEM;
+	}
+
+	/*
+	 * allocate memory for the hardware descriptor ring. A memzone large
+	 * enough to hold the maximum ring size is requested to allow for
+	 * resizing in later calls to the queue setup function.
+	 */
+	mz = rte_eth_dma_zone_reserve(dev, "tx hw_ring", tx_queue_id,
+				HW_ATL_B0_MAX_TXD * sizeof(struct hw_atl_txd_s),
+				128, socket_id);
+	if (mz == NULL) {
+		PMD_INIT_LOG(ERR,
+			"Port %d: Cannot allocate hardware ring for queue %d",
+			txq->port_id, txq->queue_id);
+		rte_free(txq->sw_ring);
+		rte_free(txq);
+		return -ENOMEM;
+	}
+	txq->hw_ring = mz->addr;
+	txq->hw_ring_phys_addr = mz->iova;
+
+	atl_reset_tx_queue(txq);
+
+	dev->data->tx_queues[tx_queue_id] = txq;
+	return 0;
+}
+
+int
+atl_tx_init(struct rte_eth_dev *eth_dev)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
+	struct atl_tx_queue *txq;
+	uint64_t base_addr = 0;
+	int i = 0;
+	int err = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
+		txq = eth_dev->data->tx_queues[i];
+		base_addr = txq->hw_ring_phys_addr;
+
+		err = hw_atl_b0_hw_ring_tx_init(hw, base_addr,
+						txq->queue_id,
+						txq->nb_tx_desc, 0,
+						txq->port_id);
+
+		if (err) {
+			PMD_INIT_LOG(ERR,
+				"Port %d: Cannot init TX queue %d",
+				txq->port_id, txq->queue_id);
+			break;
+		}
+	}
+
+	return err;
+}
+
+int
+atl_rx_init(struct rte_eth_dev *eth_dev)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
+	struct aq_rss_parameters *rss_params = &hw->aq_nic_cfg->aq_rss;
+	struct atl_rx_queue *rxq;
+	uint64_t base_addr = 0;
+	int i = 0;
+	int err = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
+		rxq = eth_dev->data->rx_queues[i];
+		base_addr = rxq->hw_ring_phys_addr;
+
+		/* Take requested pool mbuf size and adapt
+		 * descriptor buffer to best fit
+		 */
+		int buff_size = rte_pktmbuf_data_room_size(rxq->mb_pool) -
+				RTE_PKTMBUF_HEADROOM;
+
+		buff_size = RTE_ALIGN_FLOOR(buff_size, 1024);
+		if (buff_size > HW_ATL_B0_RXD_BUF_SIZE_MAX) {
+			PMD_INIT_LOG(WARNING,
+				"Port %d queue %d: mem pool buff size is too big\n",
+				rxq->port_id, rxq->queue_id);
+			buff_size = HW_ATL_B0_RXD_BUF_SIZE_MAX;
+		}
+		if (buff_size < 1024) {
+			PMD_INIT_LOG(ERR,
+				"Port %d queue %d: mem pool buff size is too small\n",
+				rxq->port_id, rxq->queue_id);
+			return -EINVAL;
+		}
+		rxq->buff_size = buff_size;
+
+		err = hw_atl_b0_hw_ring_rx_init(hw, base_addr, rxq->queue_id,
+						rxq->nb_rx_desc, buff_size, 0,
+						rxq->port_id);
+
+		if (err) {
+			PMD_INIT_LOG(ERR, "Port %d: Cannot init RX queue %d",
+				     rxq->port_id, rxq->queue_id);
+			break;
+		}
+	}
+
+	for (i = rss_params->indirection_table_size; i--;)
+		rss_params->indirection_table[i] = i &
+			(eth_dev->data->nb_rx_queues - 1);
+	hw_atl_b0_hw_rss_set(hw, rss_params);
+	return err;
+}
+
+static int
+atl_alloc_rx_queue_mbufs(struct atl_rx_queue *rxq)
+{
+	struct atl_rx_entry *rx_entry = rxq->sw_ring;
+	struct hw_atl_rxd_s *rxd;
+	uint64_t dma_addr = 0;
+	uint32_t i = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* fill Rx ring */
+	for (i = 0; i < rxq->nb_rx_desc; i++) {
+		struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
+
+		if (mbuf == NULL) {
+			PMD_INIT_LOG(ERR,
+				"Port %d: mbuf alloc failed for rx queue %d",
+				rxq->port_id, rxq->queue_id);
+			return -ENOMEM;
+		}
+
+		mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+		mbuf->port = rxq->port_id;
+
+		dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
+		rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[i];
+		rxd->buf_addr = dma_addr;
+		rxd->hdr_addr = 0;
+		rx_entry[i].mbuf = mbuf;
+	}
+
+	return 0;
+}
+
+static void
+atl_rx_queue_release_mbufs(struct atl_rx_queue *rxq)
+{
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rxq->sw_ring != NULL) {
+		for (i = 0; i < rxq->nb_rx_desc; i++) {
+			if (rxq->sw_ring[i].mbuf != NULL) {
+				rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+				rxq->sw_ring[i].mbuf = NULL;
+			}
+		}
+	}
+}
+
+int
+atl_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct atl_rx_queue *rxq = NULL;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rx_queue_id < dev->data->nb_rx_queues) {
+		rxq = dev->data->rx_queues[rx_queue_id];
+
+		if (atl_alloc_rx_queue_mbufs(rxq) != 0) {
+			PMD_INIT_LOG(ERR,
+				"Port %d: Allocate mbufs for queue %d failed",
+				rxq->port_id, rxq->queue_id);
+			return -1;
+		}
+
+		hw_atl_b0_hw_ring_rx_start(hw, rx_queue_id);
+
+		rte_wmb();
+		hw_atl_reg_rx_dma_desc_tail_ptr_set(hw, rxq->nb_rx_desc - 1,
+						    rx_queue_id);
+		dev->data->rx_queue_state[rx_queue_id] =
+			RTE_ETH_QUEUE_STATE_STARTED;
+	} else {
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+atl_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct atl_rx_queue *rxq = NULL;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rx_queue_id < dev->data->nb_rx_queues) {
+		rxq = dev->data->rx_queues[rx_queue_id];
+
+		hw_atl_b0_hw_ring_rx_stop(hw, rx_queue_id);
+
+		atl_rx_queue_release_mbufs(rxq);
+		atl_reset_rx_queue(rxq);
+
+		dev->data->rx_queue_state[rx_queue_id] =
+			RTE_ETH_QUEUE_STATE_STOPPED;
+	} else {
+		return -1;
+	}
+
+	return 0;
+}
+
+void
+atl_rx_queue_release(void *rx_queue)
+{
+	PMD_INIT_FUNC_TRACE();
+
+	if (rx_queue != NULL) {
+		struct atl_rx_queue *rxq = (struct atl_rx_queue *)rx_queue;
+
+		atl_rx_queue_release_mbufs(rxq);
+		rte_free(rxq->sw_ring);
+		rte_free(rxq);
+	}
+}
+
+static void
+atl_tx_queue_release_mbufs(struct atl_tx_queue *txq)
+{
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (txq->sw_ring != NULL) {
+		for (i = 0; i < txq->nb_tx_desc; i++) {
+			if (txq->sw_ring[i].mbuf != NULL) {
+				rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
+				txq->sw_ring[i].mbuf = NULL;
+			}
+		}
+	}
+}
+
+int
+atl_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (tx_queue_id < dev->data->nb_tx_queues) {
+		hw_atl_b0_hw_ring_tx_start(hw, tx_queue_id);
+
+		rte_wmb();
+		hw_atl_b0_hw_tx_ring_tail_update(hw, 0, tx_queue_id);
+		dev->data->tx_queue_state[tx_queue_id] =
+			RTE_ETH_QUEUE_STATE_STARTED;
+	} else {
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+atl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct atl_tx_queue *txq;
+
+	PMD_INIT_FUNC_TRACE();
+
+	txq = dev->data->tx_queues[tx_queue_id];
+
+	hw_atl_b0_hw_ring_tx_stop(hw, tx_queue_id);
+
+	atl_tx_queue_release_mbufs(txq);
+	atl_reset_tx_queue(txq);
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+
+	return 0;
+}
+
+void
+atl_tx_queue_release(void *tx_queue)
+{
+	PMD_INIT_FUNC_TRACE();
+
+	if (tx_queue != NULL) {
+		struct atl_tx_queue *txq = (struct atl_tx_queue *)tx_queue;
+
+		atl_tx_queue_release_mbufs(txq);
+		rte_free(txq->sw_ring);
+		rte_free(txq);
+	}
+}
+
+void
+atl_free_queues(struct rte_eth_dev *dev)
+{
+	unsigned int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		atl_rx_queue_release(dev->data->rx_queues[i]);
+		dev->data->rx_queues[i] = 0;
+	}
+	dev->data->nb_rx_queues = 0;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		atl_tx_queue_release(dev->data->tx_queues[i]);
+		dev->data->tx_queues[i] = 0;
+	}
+	dev->data->nb_tx_queues = 0;
+}
+
+int
+atl_start_queues(struct rte_eth_dev *dev)
+{
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		if (atl_tx_queue_start(dev, i) != 0) {
+			PMD_DRV_LOG(ERR,
+				"Port %d: Start Tx queue %d failed",
+				dev->data->port_id, i);
+			return -1;
+		}
+	}
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		if (atl_rx_queue_start(dev, i) != 0) {
+			PMD_DRV_LOG(ERR,
+				"Port %d: Start Rx queue %d failed",
+				dev->data->port_id, i);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+atl_stop_queues(struct rte_eth_dev *dev)
+{
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		if (atl_tx_queue_stop(dev, i) != 0) {
+			PMD_DRV_LOG(ERR,
+				"Port %d: Stop Tx queue %d failed",
+				dev->data->port_id, i);
+			return -1;
+		}
+	}
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		if (atl_rx_queue_stop(dev, i) != 0) {
+			PMD_DRV_LOG(ERR,
+				"Port %d: Stop Rx queue %d failed",
+				dev->data->port_id, i);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+void
+atl_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
+		 struct rte_eth_rxq_info *qinfo)
+{
+	struct atl_rx_queue *rxq;
+
+	PMD_INIT_FUNC_TRACE();
+
+	rxq = dev->data->rx_queues[queue_id];
+
+	qinfo->mp = rxq->mb_pool;
+	qinfo->scattered_rx = dev->data->scattered_rx;
+	qinfo->nb_desc = rxq->nb_rx_desc;
+}
+
+void
+atl_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
+		 struct rte_eth_txq_info *qinfo)
+{
+	struct atl_tx_queue *txq;
+
+	PMD_INIT_FUNC_TRACE();
+
+	txq = dev->data->tx_queues[queue_id];
+
+	qinfo->nb_desc = txq->nb_tx_desc;
+}
+
+/* Return Rx queue avail count */
+
+uint32_t
+atl_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct atl_rx_queue *rxq;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rx_queue_id >= dev->data->nb_rx_queues) {
+		PMD_DRV_LOG(ERR, "Invalid RX queue id=%d", rx_queue_id);
+		return 0;
+	}
+
+	rxq = dev->data->rx_queues[rx_queue_id];
+
+	if (rxq == NULL)
+		return 0;
+
+	return rxq->nb_rx_desc - rxq->nb_rx_hold;
+}
+
+int
+atl_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
+{
+	struct atl_rx_queue *rxq = rx_queue;
+	struct hw_atl_rxd_wb_s *rxd;
+	uint32_t idx;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (unlikely(offset >= rxq->nb_rx_desc))
+		return -EINVAL;
+
+	if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold)
+		return RTE_ETH_RX_DESC_UNAVAIL;
+
+	idx = rxq->rx_tail + offset;
+
+	if (idx >= rxq->nb_rx_desc)
+		idx -= rxq->nb_rx_desc;
+
+	rxd = (struct hw_atl_rxd_wb_s *)&rxq->hw_ring[idx];
+
+	if (rxd->dd)
+		return RTE_ETH_RX_DESC_DONE;
+
+	return RTE_ETH_RX_DESC_AVAIL;
+}
+
+int
+atl_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
+{
+	struct atl_tx_queue *txq = tx_queue;
+	struct hw_atl_txd_s *txd;
+	uint32_t idx;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (unlikely(offset >= txq->nb_tx_desc))
+		return -EINVAL;
+
+	idx = txq->tx_tail + offset;
+
+	if (idx >= txq->nb_tx_desc)
+		idx -= txq->nb_tx_desc;
+
+	txd = &txq->hw_ring[idx];
+
+	if (txd->dd)
+		return RTE_ETH_TX_DESC_DONE;
+
+	return RTE_ETH_TX_DESC_FULL;
+}
+
+static int
+atl_rx_enable_intr(struct rte_eth_dev *dev, uint16_t queue_id, bool enable)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct atl_rx_queue *rxq;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (queue_id >= dev->data->nb_rx_queues) {
+		PMD_DRV_LOG(ERR, "Invalid RX queue id=%d", queue_id);
+		return -EINVAL;
+	}
+
+	rxq = dev->data->rx_queues[queue_id];
+
+	if (rxq == NULL)
+		return 0;
+
+	/* Mapping interrupt vector */
+	hw_atl_itr_irq_map_en_rx_set(hw, enable, queue_id);
+
+	return 0;
+}
+
+int
+atl_dev_rx_queue_intr_enable(struct rte_eth_dev *eth_dev, uint16_t queue_id)
+{
+	return atl_rx_enable_intr(eth_dev, queue_id, true);
+}
+
+int
+atl_dev_rx_queue_intr_disable(struct rte_eth_dev *eth_dev, uint16_t queue_id)
+{
+	return atl_rx_enable_intr(eth_dev, queue_id, false);
+}
+
+uint16_t
+atl_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
+	      uint16_t nb_pkts)
+{
+	int i, ret;
+	uint64_t ol_flags;
+	struct rte_mbuf *m;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < nb_pkts; i++) {
+		m = tx_pkts[i];
+		ol_flags = m->ol_flags;
+
+		if (m->nb_segs > AQ_HW_MAX_SEGS_SIZE) {
+			rte_errno = -EINVAL;
+			return i;
+		}
+
+		if (ol_flags & ATL_TX_OFFLOAD_NOTSUP_MASK) {
+			rte_errno = -ENOTSUP;
+			return i;
+		}
+
+#ifdef RTE_LIBRTE_ETHDEV_DEBUG
+		ret = rte_validate_tx_offload(m);
+		if (ret != 0) {
+			rte_errno = ret;
+			return i;
+		}
+#endif
+		ret = rte_net_intel_cksum_prepare(m);
+		if (ret != 0) {
+			rte_errno = ret;
+			return i;
+		}
+	}
+
+	return i;
+}
+
+static uint64_t
+atl_desc_to_offload_flags(struct atl_rx_queue *rxq,
+			  struct hw_atl_rxd_wb_s *rxd_wb)
+{
+	uint64_t mbuf_flags = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* IPv4 ? */
+	if (rxq->l3_csum_enabled && ((rxd_wb->pkt_type & 0x3) == 0)) {
+		/* IPv4 csum error ? */
+		if (rxd_wb->rx_stat & BIT(1))
+			mbuf_flags |= PKT_RX_IP_CKSUM_BAD;
+		else
+			mbuf_flags |= PKT_RX_IP_CKSUM_GOOD;
+	} else {
+		mbuf_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
+	}
+
+	/* CSUM calculated ? */
+	if (rxq->l4_csum_enabled && (rxd_wb->rx_stat & BIT(3))) {
+		if (rxd_wb->rx_stat & BIT(2))
+			mbuf_flags |= PKT_RX_L4_CKSUM_BAD;
+		else
+			mbuf_flags |= PKT_RX_L4_CKSUM_GOOD;
+	} else {
+		mbuf_flags |= PKT_RX_L4_CKSUM_UNKNOWN;
+	}
+
+	return mbuf_flags;
+}
+
+static uint32_t
+atl_desc_to_pkt_type(struct hw_atl_rxd_wb_s *rxd_wb)
+{
+	uint32_t type = RTE_PTYPE_UNKNOWN;
+	uint16_t l2_l3_type = rxd_wb->pkt_type & 0x3;
+	uint16_t l4_type = (rxd_wb->pkt_type & 0x1C) >> 2;
+
+	switch (l2_l3_type) {
+	case 0:
+		type = RTE_PTYPE_L3_IPV4;
+		break;
+	case 1:
+		type = RTE_PTYPE_L3_IPV6;
+		break;
+	case 2:
+		type = RTE_PTYPE_L2_ETHER;
+		break;
+	case 3:
+		type = RTE_PTYPE_L2_ETHER_ARP;
+		break;
+	}
+
+	switch (l4_type) {
+	case 0:
+		type |= RTE_PTYPE_L4_TCP;
+		break;
+	case 1:
+		type |= RTE_PTYPE_L4_UDP;
+		break;
+	case 2:
+		type |= RTE_PTYPE_L4_SCTP;
+		break;
+	case 3:
+		type |= RTE_PTYPE_L4_ICMP;
+		break;
+	}
+
+	if (rxd_wb->pkt_type & BIT(5))
+		type |= RTE_PTYPE_L2_ETHER_VLAN;
+
+	return type;
+}
+
+uint16_t
+atl_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	struct atl_rx_queue *rxq = (struct atl_rx_queue *)rx_queue;
+	struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
+	struct atl_adapter *adapter =
+		ATL_DEV_TO_ADAPTER(&rte_eth_devices[rxq->port_id]);
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(adapter);
+	struct aq_hw_cfg_s *cfg =
+		ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
+	struct atl_rx_entry *sw_ring = rxq->sw_ring;
+
+	struct rte_mbuf *new_mbuf;
+	struct rte_mbuf *rx_mbuf, *rx_mbuf_prev, *rx_mbuf_first;
+	struct atl_rx_entry *rx_entry;
+	uint16_t nb_rx = 0;
+	uint16_t nb_hold = 0;
+	struct hw_atl_rxd_wb_s rxd_wb;
+	struct hw_atl_rxd_s *rxd = NULL;
+	uint16_t tail = rxq->rx_tail;
+	uint64_t dma_addr;
+	uint16_t pkt_len = 0;
+
+	while (nb_rx < nb_pkts) {
+		uint16_t eop_tail = tail;
+
+		rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[tail];
+		rxd_wb = *(struct hw_atl_rxd_wb_s *)rxd;
+
+		if (!rxd_wb.dd) { /* RxD is not done */
+			break;
+		}
+
+		PMD_RX_LOG(ERR, "port_id=%u queue_id=%u tail=%u "
+			   "eop=0x%x pkt_len=%u hash=0x%x hash_type=0x%x",
+			   (unsigned int)rxq->port_id,
+			   (unsigned int)rxq->queue_id,
+			   (unsigned int)tail, (unsigned int)rxd_wb.eop,
+			   (unsigned int)rte_le_to_cpu_16(rxd_wb.pkt_len),
+			rxd_wb.rss_hash, rxd_wb.rss_type);
+
+		/* RxD is not done */
+		if (!rxd_wb.eop) {
+			while (true) {
+				struct hw_atl_rxd_wb_s *eop_rxwbd;
+
+				eop_tail = (eop_tail + 1) % rxq->nb_rx_desc;
+				eop_rxwbd = (struct hw_atl_rxd_wb_s *)
+					&rxq->hw_ring[eop_tail];
+				if (!eop_rxwbd->dd) {
+					/* no EOP received yet */
+					eop_tail = tail;
+					break;
+				}
+				if (eop_rxwbd->dd && eop_rxwbd->eop)
+					break;
+			}
+			/* No EOP in ring */
+			if (eop_tail == tail)
+				break;
+		}
+		rx_mbuf_prev = NULL;
+		rx_mbuf_first = NULL;
+
+		/* Run through packet segments */
+		while (true) {
+			new_mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
+			if (new_mbuf == NULL) {
+				PMD_RX_LOG(ERR,
+				   "RX mbuf alloc failed port_id=%u "
+				   "queue_id=%u", (unsigned int)rxq->port_id,
+				   (unsigned int)rxq->queue_id);
+				dev->data->rx_mbuf_alloc_failed++;
+				adapter->sw_stats.rx_nombuf++;
+				goto err_stop;
+			}
+
+			nb_hold++;
+			rx_entry = &sw_ring[tail];
+
+			rx_mbuf = rx_entry->mbuf;
+			rx_entry->mbuf = new_mbuf;
+			dma_addr = rte_cpu_to_le_64(
+				rte_mbuf_data_iova_default(new_mbuf));
+
+			/* setup RX descriptor */
+			rxd->hdr_addr = 0;
+			rxd->buf_addr = dma_addr;
+
+			/*
+			 * Initialize the returned mbuf.
+			 * 1) setup generic mbuf fields:
+			 *	  - number of segments,
+			 *	  - next segment,
+			 *	  - packet length,
+			 *	  - RX port identifier.
+			 * 2) integrate hardware offload data, if any:
+			 *	<  - RSS flag & hash,
+			 *	  - IP checksum flag,
+			 *	  - VLAN TCI, if any,
+			 *	  - error flags.
+			 */
+			pkt_len = (uint16_t)rte_le_to_cpu_16(rxd_wb.pkt_len);
+			rx_mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+			rte_prefetch1((char *)rx_mbuf->buf_addr +
+				rx_mbuf->data_off);
+			rx_mbuf->nb_segs = 0;
+			rx_mbuf->next = NULL;
+			rx_mbuf->pkt_len = pkt_len;
+			rx_mbuf->data_len = pkt_len;
+			if (rxd_wb.eop) {
+				u16 remainder_len = pkt_len % rxq->buff_size;
+				if (!remainder_len)
+					remainder_len = rxq->buff_size;
+				rx_mbuf->data_len = remainder_len;
+			} else {
+				rx_mbuf->data_len = pkt_len > rxq->buff_size ?
+						rxq->buff_size : pkt_len;
+			}
+			rx_mbuf->port = rxq->port_id;
+
+			rx_mbuf->hash.rss = rxd_wb.rss_hash;
+
+			rx_mbuf->vlan_tci = rxd_wb.vlan;
+
+			rx_mbuf->ol_flags =
+				atl_desc_to_offload_flags(rxq, &rxd_wb);
+
+			rx_mbuf->packet_type = atl_desc_to_pkt_type(&rxd_wb);
+
+			if (rx_mbuf->packet_type & RTE_PTYPE_L2_ETHER_VLAN) {
+				rx_mbuf->ol_flags |= PKT_RX_VLAN;
+				rx_mbuf->vlan_tci = rxd_wb.vlan;
+
+				if (cfg->vlan_strip)
+					rx_mbuf->ol_flags |=
+						PKT_RX_VLAN_STRIPPED;
+			}
+
+			if (!rx_mbuf_first)
+				rx_mbuf_first = rx_mbuf;
+			rx_mbuf_first->nb_segs++;
+
+			if (rx_mbuf_prev)
+				rx_mbuf_prev->next = rx_mbuf;
+			rx_mbuf_prev = rx_mbuf;
+
+			tail = (tail + 1) % rxq->nb_rx_desc;
+			/* Prefetch next mbufs */
+			rte_prefetch0(sw_ring[tail].mbuf);
+			if ((tail & 0x3) == 0) {
+				rte_prefetch0(&sw_ring[tail]);
+				rte_prefetch0(&sw_ring[tail]);
+			}
+
+			/* filled mbuf_first */
+			if (rxd_wb.eop)
+				break;
+			rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[tail];
+			rxd_wb = *(struct hw_atl_rxd_wb_s *)rxd;
+		};
+
+		/*
+		 * Store the mbuf address into the next entry of the array
+		 * of returned packets.
+		 */
+		rx_pkts[nb_rx++] = rx_mbuf_first;
+		adapter->sw_stats.q_ipackets[rxq->queue_id]++;
+		adapter->sw_stats.q_ibytes[rxq->queue_id] +=
+			rx_mbuf_first->pkt_len;
+
+		PMD_RX_LOG(ERR, "add mbuf segs=%d pkt_len=%d",
+			rx_mbuf_first->nb_segs,
+			rx_mbuf_first->pkt_len);
+	}
+
+err_stop:
+
+	rxq->rx_tail = tail;
+
+	/*
+	 * If the number of free RX descriptors is greater than the RX free
+	 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
+	 * register.
+	 * Update the RDT with the value of the last processed RX descriptor
+	 * minus 1, to guarantee that the RDT register is never equal to the
+	 * RDH register, which creates a "full" ring situtation from the
+	 * hardware point of view...
+	 */
+	nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
+	if (nb_hold > rxq->rx_free_thresh) {
+		PMD_RX_LOG(ERR, "port_id=%u queue_id=%u rx_tail=%u "
+			"nb_hold=%u nb_rx=%u",
+			(unsigned int)rxq->port_id, (unsigned int)rxq->queue_id,
+			(unsigned int)tail, (unsigned int)nb_hold,
+			(unsigned int)nb_rx);
+		tail = (uint16_t)((tail == 0) ?
+			(rxq->nb_rx_desc - 1) : (tail - 1));
+
+		hw_atl_reg_rx_dma_desc_tail_ptr_set(hw, tail, rxq->queue_id);
+
+		nb_hold = 0;
+	}
+
+	rxq->nb_rx_hold = nb_hold;
+
+	return nb_rx;
+}
+
+static void
+atl_xmit_cleanup(struct atl_tx_queue *txq)
+{
+	struct atl_tx_entry *sw_ring;
+	struct hw_atl_txd_s *txd;
+	int to_clean = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (txq != NULL) {
+		sw_ring = txq->sw_ring;
+		int head = txq->tx_head;
+		int cnt;
+		int i;
+
+		for (i = 0, cnt = head; ; i++) {
+			txd = &txq->hw_ring[cnt];
+
+			if (txd->dd)
+				to_clean++;
+
+			cnt = (cnt + 1) % txq->nb_tx_desc;
+			if (cnt == txq->tx_tail)
+				break;
+		}
+
+		if (to_clean == 0)
+			return;
+
+		while (to_clean) {
+			txd = &txq->hw_ring[head];
+
+			struct atl_tx_entry *rx_entry = &sw_ring[head];
+
+			if (rx_entry->mbuf) {
+				rte_pktmbuf_free_seg(rx_entry->mbuf);
+				rx_entry->mbuf = NULL;
+			}
+
+			if (txd->dd)
+				to_clean--;
+
+			txd->buf_addr = 0;
+			txd->flags = 0;
+
+			head = (head + 1) % txq->nb_tx_desc;
+			txq->tx_free++;
+		}
+
+		txq->tx_head = head;
+	}
+}
+
+static int
+atl_tso_setup(struct rte_mbuf *tx_pkt, union hw_atl_txc_s *txc)
+{
+	uint32_t tx_cmd = 0;
+	uint64_t ol_flags = tx_pkt->ol_flags;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (ol_flags & PKT_TX_TCP_SEG) {
+		PMD_DRV_LOG(DEBUG, "xmit TSO pkt");
+
+		tx_cmd |= tx_desc_cmd_lso | tx_desc_cmd_l4cs;
+
+		txc->cmd = 0x4;
+
+		if (ol_flags & PKT_TX_IPV6)
+			txc->cmd |= 0x2;
+
+		txc->l2_len = tx_pkt->l2_len;
+		txc->l3_len = tx_pkt->l3_len;
+		txc->l4_len = tx_pkt->l4_len;
+
+		txc->mss_len = tx_pkt->tso_segsz;
+	}
+
+	if (ol_flags & PKT_TX_VLAN) {
+		tx_cmd |= tx_desc_cmd_vlan;
+		txc->vlan_tag = tx_pkt->vlan_tci;
+	}
+
+	if (tx_cmd) {
+		txc->type = tx_desc_type_ctx;
+		txc->idx = 0;
+	}
+
+	return tx_cmd;
+}
+
+static inline void
+atl_setup_csum_offload(struct rte_mbuf *mbuf, struct hw_atl_txd_s *txd,
+		       uint32_t tx_cmd)
+{
+	txd->cmd |= tx_desc_cmd_fcs;
+	txd->cmd |= (mbuf->ol_flags & PKT_TX_IP_CKSUM) ? tx_desc_cmd_ipv4 : 0;
+	/* L4 csum requested */
+	txd->cmd |= (mbuf->ol_flags & PKT_TX_L4_MASK) ? tx_desc_cmd_l4cs : 0;
+	txd->cmd |= tx_cmd;
+}
+
+static inline void
+atl_xmit_pkt(struct aq_hw_s *hw, struct atl_tx_queue *txq,
+	     struct rte_mbuf *tx_pkt)
+{
+	struct atl_adapter *adapter =
+		ATL_DEV_TO_ADAPTER(&rte_eth_devices[txq->port_id]);
+	uint32_t pay_len = 0;
+	int tail = 0;
+	struct atl_tx_entry *tx_entry;
+	uint64_t buf_dma_addr;
+	struct rte_mbuf *m_seg;
+	union hw_atl_txc_s *txc = NULL;
+	struct hw_atl_txd_s *txd = NULL;
+	u32 tx_cmd = 0U;
+	int desc_count = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	tail = txq->tx_tail;
+
+	txc = (union hw_atl_txc_s *)&txq->hw_ring[tail];
+
+	txc->flags1 = 0U;
+	txc->flags2 = 0U;
+
+	tx_cmd = atl_tso_setup(tx_pkt, txc);
+
+	if (tx_cmd) {
+		/* We've consumed the first desc, adjust counters */
+		tail = (tail + 1) % txq->nb_tx_desc;
+		txq->tx_tail = tail;
+		txq->tx_free -= 1;
+
+		txd = &txq->hw_ring[tail];
+		txd->flags = 0U;
+	} else {
+		txd = (struct hw_atl_txd_s *)txc;
+	}
+
+	txd->ct_en = !!tx_cmd;
+
+	txd->type = tx_desc_type_desc;
+
+	atl_setup_csum_offload(tx_pkt, txd, tx_cmd);
+
+	if (tx_cmd)
+		txd->ct_idx = 0;
+
+	pay_len = tx_pkt->pkt_len;
+
+	txd->pay_len = pay_len;
+
+	for (m_seg = tx_pkt; m_seg; m_seg = m_seg->next) {
+		if (desc_count > 0) {
+			txd = &txq->hw_ring[tail];
+			txd->flags = 0U;
+		}
+
+		buf_dma_addr = rte_mbuf_data_iova(m_seg);
+		txd->buf_addr = rte_cpu_to_le_64(buf_dma_addr);
+
+		txd->type = tx_desc_type_desc;
+		txd->len = m_seg->data_len;
+		txd->pay_len = pay_len;
+
+		/* Store mbuf for freeing later */
+		tx_entry = &txq->sw_ring[tail];
+
+		if (tx_entry->mbuf)
+			rte_pktmbuf_free_seg(tx_entry->mbuf);
+		tx_entry->mbuf = m_seg;
+
+		tail = (tail + 1) % txq->nb_tx_desc;
+
+		desc_count++;
+	}
+
+	// Last descriptor requires EOP and WB
+	txd->eop = 1U;
+	txd->cmd |= tx_desc_cmd_wb;
+
+	hw_atl_b0_hw_tx_ring_tail_update(hw, tail, txq->queue_id);
+
+	txq->tx_tail = tail;
+
+	txq->tx_free -= desc_count;
+
+	adapter->sw_stats.q_opackets[txq->queue_id]++;
+	adapter->sw_stats.q_obytes[txq->queue_id] += pay_len;
+}
+
+uint16_t
+atl_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+	struct rte_eth_dev *dev = NULL;
+	struct aq_hw_s *hw = NULL;
+	struct atl_tx_queue *txq = tx_queue;
+	struct rte_mbuf *tx_pkt;
+	uint16_t nb_tx;
+
+	dev = &rte_eth_devices[txq->port_id];
+	hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+	PMD_TX_LOG(DEBUG,
+		"port %d txq %d pkts: %d tx_free=%d tx_tail=%d tx_head=%d",
+		txq->port_id, txq->queue_id, nb_pkts, txq->tx_free,
+		txq->tx_tail, txq->tx_head);
+
+	for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
+		tx_pkt = *tx_pkts++;
+
+		/* Clean Tx queue if needed */
+		if (txq->tx_free < txq->tx_free_thresh)
+			atl_xmit_cleanup(txq);
+
+		/* Check if we have enough free descriptors */
+		if (txq->tx_free < tx_pkt->nb_segs)
+			break;
+
+		/* check mbuf is valid */
+		if ((tx_pkt->nb_segs == 0) ||
+			((tx_pkt->nb_segs > 1) && (tx_pkt->next == NULL)))
+			break;
+
+		/* Send the packet */
+		atl_xmit_pkt(hw, txq, tx_pkt);
+	}
+
+	PMD_TX_LOG(DEBUG, "atl_xmit_pkts %d transmitted", nb_tx);
+
+	return nb_tx;
+}
+
diff --git a/drivers/net/atlantic/atl_types.h b/drivers/net/atlantic/atl_types.h
new file mode 100644
index 00000000..3d90f6ca
--- /dev/null
+++ b/drivers/net/atlantic/atl_types.h
@@ -0,0 +1,186 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Aquantia Corporation
+ */
+#ifndef ATL_TYPES_H
+#define ATL_TYPES_H
+
+#include <stdint.h>
+#include <stddef.h>
+#include <inttypes.h>
+#include <string.h>
+#include <stdbool.h>
+#include <netinet/in.h>
+
+typedef uint8_t		u8;
+typedef int8_t		s8;
+typedef uint16_t	u16;
+typedef int16_t		s16;
+typedef uint32_t	u32;
+typedef int32_t		s32;
+typedef uint64_t	u64;
+
+#define min(a, b)	RTE_MIN(a, b)
+#define max(a, b)	RTE_MAX(a, b)
+
+#include "hw_atl/hw_atl_b0_internal.h"
+#include "hw_atl/hw_atl_utils.h"
+
+struct aq_hw_link_status_s {
+	unsigned int mbps;
+};
+
+struct aq_stats_s {
+	u64 uprc;
+	u64 mprc;
+	u64 bprc;
+	u64 erpt;
+	u64 uptc;
+	u64 mptc;
+	u64 bptc;
+	u64 erpr;
+	u64 mbtc;
+	u64 bbtc;
+	u64 mbrc;
+	u64 bbrc;
+	u64 ubrc;
+	u64 ubtc;
+	u64 dpc;
+	u64 dma_pkt_rc;
+	u64 dma_pkt_tc;
+	u64 dma_oct_rc;
+	u64 dma_oct_tc;
+};
+
+struct aq_rss_parameters {
+	u16 base_cpu_number;
+	u16 indirection_table_size;
+	u16 hash_secret_key_size;
+	u32 hash_secret_key[HW_ATL_B0_RSS_HASHKEY_BITS / 8];
+	u8 indirection_table[HW_ATL_B0_RSS_REDIRECTION_MAX];
+};
+
+struct aq_hw_cfg_s {
+	bool is_lro;
+	bool is_rss;
+	unsigned int num_rss_queues;
+	int wol;
+
+	int link_speed_msk;
+	int irq_type;
+	int irq_mask;
+	unsigned int vecs;
+
+	bool vlan_strip;
+	uint32_t vlan_filter[HW_ATL_B0_MAX_VLAN_IDS];
+	uint32_t flow_control;
+
+	struct aq_rss_parameters aq_rss;
+};
+
+struct aq_hw_s {
+	u16 device_id;
+	u16 vendor_id;
+	bool adapter_stopped;
+
+	u8 rbl_enabled:1;
+	struct aq_hw_cfg_s *aq_nic_cfg;
+	const struct aq_fw_ops *aq_fw_ops;
+	void *mmio;
+
+	struct aq_hw_link_status_s aq_link_status;
+	bool is_autoneg;
+
+	struct hw_aq_atl_utils_mbox mbox;
+	struct hw_atl_stats_s last_stats;
+	struct aq_stats_s curr_stats;
+
+	u64 speed;
+	unsigned int chip_features;
+	u32 fw_ver_actual;
+	u32 mbox_addr;
+	u32 rpc_addr;
+	u32 rpc_tid;
+	struct hw_aq_atl_utils_fw_rpc rpc;
+};
+
+struct aq_fw_ops {
+	int (*init)(struct aq_hw_s *self);
+
+	int (*deinit)(struct aq_hw_s *self);
+
+	int (*reset)(struct aq_hw_s *self);
+
+	int (*get_mac_permanent)(struct aq_hw_s *self, u8 *mac);
+
+	int (*set_link_speed)(struct aq_hw_s *self, u32 speed);
+
+	int (*set_state)(struct aq_hw_s *self,
+			enum hal_atl_utils_fw_state_e state);
+
+	int (*update_link_status)(struct aq_hw_s *self);
+
+	int (*update_stats)(struct aq_hw_s *self);
+
+	int (*set_power)(struct aq_hw_s *self, unsigned int power_state,
+			u8 *mac);
+
+	int (*get_temp)(struct aq_hw_s *self, int *temp);
+
+	int (*get_cable_len)(struct aq_hw_s *self, int *cable_len);
+
+	int (*set_eee_rate)(struct aq_hw_s *self, u32 speed);
+
+	int (*get_eee_rate)(struct aq_hw_s *self, u32 *rate,
+			u32 *supported_rates);
+
+	int (*set_flow_control)(struct aq_hw_s *self);
+
+	int (*led_control)(struct aq_hw_s *self, u32 mode);
+
+	int (*get_eeprom)(struct aq_hw_s *self, u32 *data, u32 len);
+
+	int (*set_eeprom)(struct aq_hw_s *self, u32 *data, u32 len);
+};
+
+struct atl_sw_stats {
+	u64 crcerrs;
+	u64 errbc;
+	u64 mspdc;
+	u64 mpctotal;
+	u64 mpc[8];
+	u64 mlfc;
+	u64 mrfc;
+	u64 rlec;
+	u64 lxontxc;
+	u64 lxonrxc;
+	u64 lxofftxc;
+	u64 lxoffrxc;
+	u64 pxontxc[8];
+	u64 pxonrxc[8];
+	u64 pxofftxc[8];
+	u64 pxoffrxc[8];
+	u64 gprc;
+	u64 bprc;
+	u64 mprc;
+	u64 gptc;
+	u64 gorc;
+	u64 gotc;
+	u64 tor;
+	u64 tpr;
+	u64 tpt;
+	u64 mptc;
+	u64 bptc;
+	u64 xec;
+	u64 fccrc;
+	u64 ldpcec;
+	u64 pcrc8ec;
+
+	u64 rx_nombuf;
+	u64 q_ipackets[RTE_ETHDEV_QUEUE_STAT_CNTRS];
+	u64 q_opackets[RTE_ETHDEV_QUEUE_STAT_CNTRS];
+	u64 q_ibytes[RTE_ETHDEV_QUEUE_STAT_CNTRS];
+	u64 q_obytes[RTE_ETHDEV_QUEUE_STAT_CNTRS];
+	u64 q_errors[RTE_ETHDEV_QUEUE_STAT_CNTRS];
+};
+
+#endif
diff --git a/drivers/net/atlantic/hw_atl/hw_atl_b0.c b/drivers/net/atlantic/hw_atl/hw_atl_b0.c
new file mode 100644
index 00000000..9400e0ed
--- /dev/null
+++ b/drivers/net/atlantic/hw_atl/hw_atl_b0.c
@@ -0,0 +1,510 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+/* Copyright (C) 2014-2017 aQuantia Corporation. */
+
+/* File hw_atl_b0.c: Definition of Atlantic hardware specific functions. */
+
+#include "../atl_types.h"
+#include "hw_atl_b0.h"
+
+#include "../atl_hw_regs.h"
+#include "hw_atl_utils.h"
+#include "hw_atl_llh.h"
+#include "hw_atl_b0_internal.h"
+#include "hw_atl_llh_internal.h"
+#include "../atl_logs.h"
+
+int hw_atl_b0_hw_reset(struct aq_hw_s *self)
+{
+	int err = 0;
+
+	err = hw_atl_utils_soft_reset(self);
+	if (err)
+		return err;
+
+	self->aq_fw_ops->set_state(self, MPI_RESET);
+
+	return err;
+}
+
+static int hw_atl_b0_hw_qos_set(struct aq_hw_s *self)
+{
+	u32 tc = 0U;
+	u32 buff_size = 0U;
+	unsigned int i_priority = 0U;
+	bool is_rx_flow_control = false;
+
+	/* TPS Descriptor rate init */
+	hw_atl_tps_tx_pkt_shed_desc_rate_curr_time_res_set(self, 0x0U);
+	hw_atl_tps_tx_pkt_shed_desc_rate_lim_set(self, 0xA);
+
+	/* TPS VM init */
+	hw_atl_tps_tx_pkt_shed_desc_vm_arb_mode_set(self, 0U);
+
+	/* TPS TC credits init */
+	hw_atl_tps_tx_pkt_shed_desc_tc_arb_mode_set(self, 0U);
+	hw_atl_tps_tx_pkt_shed_data_arb_mode_set(self, 0U);
+
+	hw_atl_tps_tx_pkt_shed_tc_data_max_credit_set(self, 0xFFF, 0U);
+	hw_atl_tps_tx_pkt_shed_tc_data_weight_set(self, 0x64, 0U);
+	hw_atl_tps_tx_pkt_shed_desc_tc_max_credit_set(self, 0x50, 0U);
+	hw_atl_tps_tx_pkt_shed_desc_tc_weight_set(self, 0x1E, 0U);
+
+	/* Tx buf size */
+	buff_size = HW_ATL_B0_TXBUF_MAX;
+
+	hw_atl_tpb_tx_pkt_buff_size_per_tc_set(self, buff_size, tc);
+	hw_atl_tpb_tx_buff_hi_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024 / 32U) * 66U) /
+						   100U, tc);
+	hw_atl_tpb_tx_buff_lo_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024 / 32U) * 50U) /
+						   100U, tc);
+
+	/* QoS Rx buf size per TC */
+	tc = 0;
+	is_rx_flow_control = 0;
+	buff_size = HW_ATL_B0_RXBUF_MAX;
+
+	hw_atl_rpb_rx_pkt_buff_size_per_tc_set(self, buff_size, tc);
+	hw_atl_rpb_rx_buff_hi_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024U / 32U) * 66U) /
+						   100U, tc);
+	hw_atl_rpb_rx_buff_lo_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024U / 32U) * 50U) /
+						   100U, tc);
+	hw_atl_rpb_rx_xoff_en_per_tc_set(self,
+					 is_rx_flow_control ? 1U : 0U,
+					 tc);
+
+	/* QoS 802.1p priority -> TC mapping */
+	for (i_priority = 8U; i_priority--;)
+		hw_atl_rpf_rpb_user_priority_tc_map_set(self, i_priority, 0U);
+
+	return aq_hw_err_from_flags(self);
+}
+
+/* calc hash only in IPv4 header, regardless of presence of TCP */
+#define pif_rpf_rss_ipv4_hdr_only_i     (1 << 4)
+/* calc hash only if TCP header and IPv4 */
+#define pif_rpf_rss_ipv4_tcp_hdr_only_i (1 << 3)
+/* calc hash only in IPv6 header, regardless of presence of TCP */
+#define pif_rpf_rss_ipv6_hdr_only_i     (1 << 2)
+/* calc hash only if TCP header and IPv4 */
+#define pif_rpf_rss_ipv6_tcp_hdr_only_i (1 << 1)
+/* bug 5124 - rss hashing types - FIXME */
+#define pif_rpf_rss_dont_use_udp_i      (1 << 0)
+
+static int hw_atl_b0_hw_rss_hash_type_set(struct aq_hw_s *self)
+{
+	/* misc */
+	unsigned int control_reg_val =
+		IS_CHIP_FEATURE(RPF2) ? 0x000F0000U : 0x00000000U;
+
+	/* RSS hash type set for IP/TCP */
+	control_reg_val |= pif_rpf_rss_ipv4_hdr_only_i;//0x1EU;
+
+	aq_hw_write_reg(self, 0x5040U, control_reg_val);
+
+	return aq_hw_err_from_flags(self);
+}
+
+int hw_atl_b0_hw_rss_hash_set(struct aq_hw_s *self,
+				     struct aq_rss_parameters *rss_params)
+{
+	struct aq_hw_cfg_s *cfg = self->aq_nic_cfg;
+	int err = 0;
+	unsigned int i = 0U;
+	unsigned int addr = 0U;
+
+	for (i = 10, addr = 0U; i--; ++addr) {
+		u32 key_data = cfg->is_rss ?
+			htonl(rss_params->hash_secret_key[i]) : 0U;
+		hw_atl_rpf_rss_key_wr_data_set(self, key_data);
+		hw_atl_rpf_rss_key_addr_set(self, addr);
+		hw_atl_rpf_rss_key_wr_en_set(self, 1U);
+		AQ_HW_WAIT_FOR(hw_atl_rpf_rss_key_wr_en_get(self) == 0,
+			       1000U, 10U);
+		if (err < 0)
+			goto err_exit;
+	}
+
+	/* RSS Ring selection */
+	hw_atl_reg_rx_flr_rss_control1set(self,
+				cfg->is_rss ? 0xB3333333U : 0x00000000U);
+	hw_atl_b0_hw_rss_hash_type_set(self);
+
+	err = aq_hw_err_from_flags(self);
+
+err_exit:
+	return err;
+}
+
+
+int hw_atl_b0_hw_rss_set(struct aq_hw_s *self,
+			struct aq_rss_parameters *rss_params)
+{
+	u8 *indirection_table = rss_params->indirection_table;
+	u32 num_rss_queues = max(1U, self->aq_nic_cfg->num_rss_queues);
+	u32 i = 0;
+	u32 addr = 0;
+	u32 val = 0;
+	u32 shift = 0;
+	int err = 0;
+
+	for (i = 0; i < HW_ATL_B0_RSS_REDIRECTION_MAX; i++) {
+		val |= (u32)(indirection_table[i] % num_rss_queues) << shift;
+		shift += 3;
+
+		if (shift < 16)
+			continue;
+
+		hw_atl_rpf_rss_redir_tbl_wr_data_set(self, val & 0xffff);
+		hw_atl_rpf_rss_redir_tbl_addr_set(self, addr);
+
+		hw_atl_rpf_rss_redir_wr_en_set(self, 1U);
+		AQ_HW_WAIT_FOR(hw_atl_rpf_rss_redir_wr_en_get(self) == 0,
+			1000U, 10U);
+
+		if (err < 0)
+			goto err_exit;
+
+		shift -= 16;
+		val >>= 16;
+		addr++;
+	}
+
+err_exit:
+	return err;
+}
+
+static int hw_atl_b0_hw_offload_set(struct aq_hw_s *self)
+				    /*struct aq_nic_cfg_s *aq_nic_cfg)*/
+{
+	unsigned int i;
+
+	/* TX checksums offloads*/
+	hw_atl_tpo_ipv4header_crc_offload_en_set(self, 1);
+	hw_atl_tpo_tcp_udp_crc_offload_en_set(self, 1);
+
+	/* RX checksums offloads*/
+	hw_atl_rpo_ipv4header_crc_offload_en_set(self, 1);
+	hw_atl_rpo_tcp_udp_crc_offload_en_set(self, 1);
+
+	/* LSO offloads*/
+	hw_atl_tdm_large_send_offload_en_set(self, 0xFFFFFFFFU);
+
+/* LRO offloads */
+	{
+		unsigned int val = (8U < HW_ATL_B0_LRO_RXD_MAX) ? 0x3U :
+			((4U < HW_ATL_B0_LRO_RXD_MAX) ? 0x2U :
+			((2U < HW_ATL_B0_LRO_RXD_MAX) ? 0x1U : 0x0));
+
+		for (i = 0; i < HW_ATL_B0_RINGS_MAX; i++)
+			hw_atl_rpo_lro_max_num_of_descriptors_set(self, val, i);
+
+		hw_atl_rpo_lro_time_base_divider_set(self, 0x61AU);
+		hw_atl_rpo_lro_inactive_interval_set(self, 0);
+		hw_atl_rpo_lro_max_coalescing_interval_set(self, 2);
+
+		hw_atl_rpo_lro_qsessions_lim_set(self, 1U);
+
+		hw_atl_rpo_lro_total_desc_lim_set(self, 2U);
+
+		hw_atl_rpo_lro_patch_optimization_en_set(self, 0U);
+
+		hw_atl_rpo_lro_min_pay_of_first_pkt_set(self, 10U);
+
+		hw_atl_rpo_lro_pkt_lim_set(self, 1U);
+
+		hw_atl_rpo_lro_en_set(self,
+				self->aq_nic_cfg->is_lro ? 0xFFFFFFFFU : 0U);
+	}
+	return aq_hw_err_from_flags(self);
+}
+
+static
+int hw_atl_b0_hw_init_tx_path(struct aq_hw_s *self)
+{
+	/* Tx TC/RSS number config */
+	hw_atl_rpb_tps_tx_tc_mode_set(self, 1U);
+
+	hw_atl_thm_lso_tcp_flag_of_first_pkt_set(self, 0x0FF6U);
+	hw_atl_thm_lso_tcp_flag_of_middle_pkt_set(self, 0x0FF6U);
+	hw_atl_thm_lso_tcp_flag_of_last_pkt_set(self, 0x0F7FU);
+
+	/* Tx interrupts */
+	hw_atl_tdm_tx_desc_wr_wb_irq_en_set(self, 0U);
+
+	/* misc */
+	aq_hw_write_reg(self, 0x00007040U, IS_CHIP_FEATURE(TPO2) ?
+			0x00010000U : 0x00000000U);
+	hw_atl_tdm_tx_dca_en_set(self, 0U);
+	hw_atl_tdm_tx_dca_mode_set(self, 0U);
+
+	hw_atl_tpb_tx_path_scp_ins_en_set(self, 1U);
+
+	return aq_hw_err_from_flags(self);
+}
+
+static
+int hw_atl_b0_hw_init_rx_path(struct aq_hw_s *self)
+{
+	struct aq_hw_cfg_s *cfg = self->aq_nic_cfg;
+	int i;
+
+	/* Rx TC/RSS number config */
+	hw_atl_rpb_rpf_rx_traf_class_mode_set(self, 1U); /* 1: 4TC/8Queues */
+
+	/* Rx flow control */
+	hw_atl_rpb_rx_flow_ctl_mode_set(self, 1U);
+
+	/* RSS Ring selection */
+	hw_atl_reg_rx_flr_rss_control1set(self, cfg->is_rss ?
+					0xB3333333U : 0x00000000U);
+
+	/* Multicast filters */
+	for (i = HW_ATL_B0_MAC_MAX; i--;) {
+		hw_atl_rpfl2_uc_flr_en_set(self, (i == 0U) ? 1U : 0U, i);
+		hw_atl_rpfl2unicast_flr_act_set(self, 1U, i);
+	}
+
+	hw_atl_reg_rx_flr_mcst_flr_msk_set(self, 0x00000000U);
+	hw_atl_reg_rx_flr_mcst_flr_set(self, 0x00010FFFU, 0U);
+
+	/* Vlan filters */
+	hw_atl_rpf_vlan_outer_etht_set(self, 0x88A8U);
+	hw_atl_rpf_vlan_inner_etht_set(self, 0x8100U);
+
+	/* VLAN proimisc bu defauld */
+	hw_atl_rpf_vlan_prom_mode_en_set(self, 1);
+
+	/* Rx Interrupts */
+	hw_atl_rdm_rx_desc_wr_wb_irq_en_set(self, 0U);
+
+	hw_atl_b0_hw_rss_hash_type_set(self);
+
+	hw_atl_rpfl2broadcast_flr_act_set(self, 1U);
+	hw_atl_rpfl2broadcast_count_threshold_set(self, 0xFFFFU & (~0U / 256U));
+
+	hw_atl_rdm_rx_dca_en_set(self, 0U);
+	hw_atl_rdm_rx_dca_mode_set(self, 0U);
+
+	return aq_hw_err_from_flags(self);
+}
+
+static int hw_atl_b0_hw_mac_addr_set(struct aq_hw_s *self, u8 *mac_addr)
+{
+	int err = 0;
+	unsigned int h = 0U;
+	unsigned int l = 0U;
+
+	if (!mac_addr) {
+		err = -EINVAL;
+		goto err_exit;
+	}
+	h = (mac_addr[0] << 8) | (mac_addr[1]);
+	l = (mac_addr[2] << 24) | (mac_addr[3] << 16) |
+		(mac_addr[4] << 8) | mac_addr[5];
+
+	hw_atl_rpfl2_uc_flr_en_set(self, 0U, HW_ATL_B0_MAC);
+	hw_atl_rpfl2unicast_dest_addresslsw_set(self, l, HW_ATL_B0_MAC);
+	hw_atl_rpfl2unicast_dest_addressmsw_set(self, h, HW_ATL_B0_MAC);
+	hw_atl_rpfl2_uc_flr_en_set(self, 1U, HW_ATL_B0_MAC);
+
+	err = aq_hw_err_from_flags(self);
+
+err_exit:
+	return err;
+}
+
+int hw_atl_b0_hw_init(struct aq_hw_s *self, u8 *mac_addr)
+{
+	static u32 aq_hw_atl_igcr_table_[4][2] = {
+		{ 0x20000080U, 0x20000080U }, /* AQ_IRQ_INVALID */
+		{ 0x20000080U, 0x20000080U }, /* AQ_IRQ_LEGACY */
+		{ 0x20000021U, 0x20000025U }, /* AQ_IRQ_MSI */
+		{ 0x200000A2U, 0x200000A6U }  /* AQ_IRQ_MSIX */
+	};
+
+	int err = 0;
+	u32 val;
+
+	struct aq_hw_cfg_s *aq_nic_cfg = self->aq_nic_cfg;
+
+	hw_atl_b0_hw_init_tx_path(self);
+	hw_atl_b0_hw_init_rx_path(self);
+
+	hw_atl_b0_hw_mac_addr_set(self, mac_addr);
+
+	self->aq_fw_ops->set_link_speed(self, aq_nic_cfg->link_speed_msk);
+	self->aq_fw_ops->set_state(self, MPI_INIT);
+
+	hw_atl_b0_hw_qos_set(self);
+	hw_atl_b0_hw_rss_set(self, &aq_nic_cfg->aq_rss);
+	hw_atl_b0_hw_rss_hash_set(self, &aq_nic_cfg->aq_rss);
+
+	/* Force limit MRRS on RDM/TDM to 2K */
+	val = aq_hw_read_reg(self, HW_ATL_PCI_REG_CONTROL6_ADR);
+	aq_hw_write_reg(self, HW_ATL_PCI_REG_CONTROL6_ADR,
+			(val & ~0x707) | 0x404);
+
+	/* TX DMA total request limit. B0 hardware is not capable to
+	 * handle more than (8K-MRRS) incoming DMA data.
+	 * Value 24 in 256byte units
+	 */
+	aq_hw_write_reg(self, HW_ATL_TX_DMA_TOTAL_REQ_LIMIT_ADR, 24);
+
+	/* Reset link status and read out initial hardware counters */
+	self->aq_link_status.mbps = 0;
+	self->aq_fw_ops->update_stats(self);
+
+	err = aq_hw_err_from_flags(self);
+	if (err < 0)
+		goto err_exit;
+
+	/* Interrupts */
+	hw_atl_reg_irq_glb_ctl_set(self,
+				   aq_hw_atl_igcr_table_[aq_nic_cfg->irq_type]
+					 [(aq_nic_cfg->vecs > 1U) ?
+					 1 : 0]);
+
+	hw_atl_itr_irq_auto_masklsw_set(self, 0xffffffff);
+
+	/* Interrupts */
+	hw_atl_reg_gen_irq_map_set(self, 0, 0);
+	hw_atl_reg_gen_irq_map_set(self, 0x80 | ATL_IRQ_CAUSE_LINK, 3);
+
+	hw_atl_b0_hw_offload_set(self);
+
+err_exit:
+	return err;
+}
+
+int hw_atl_b0_hw_ring_tx_start(struct aq_hw_s *self, int index)
+{
+	hw_atl_tdm_tx_desc_en_set(self, 1, index);
+	return aq_hw_err_from_flags(self);
+}
+
+int hw_atl_b0_hw_ring_rx_start(struct aq_hw_s *self, int index)
+{
+	hw_atl_rdm_rx_desc_en_set(self, 1, index);
+	return aq_hw_err_from_flags(self);
+}
+
+int hw_atl_b0_hw_start(struct aq_hw_s *self)
+{
+	hw_atl_tpb_tx_buff_en_set(self, 1);
+	hw_atl_rpb_rx_buff_en_set(self, 1);
+	return aq_hw_err_from_flags(self);
+}
+
+int hw_atl_b0_hw_tx_ring_tail_update(struct aq_hw_s *self, int tail, int index)
+{
+	hw_atl_reg_tx_dma_desc_tail_ptr_set(self, tail, index);
+	return 0;
+}
+
+int hw_atl_b0_hw_ring_rx_init(struct aq_hw_s *self, uint64_t base_addr,
+		int index, int size, int buff_size, int cpu, int vec)
+{
+	u32 dma_desc_addr_lsw = (u32)base_addr;
+	u32 dma_desc_addr_msw = (u32)(base_addr >> 32);
+
+	hw_atl_rdm_rx_desc_en_set(self, false, index);
+
+	hw_atl_rdm_rx_desc_head_splitting_set(self, 0U, index);
+
+	hw_atl_reg_rx_dma_desc_base_addresslswset(self, dma_desc_addr_lsw,
+						  index);
+
+	hw_atl_reg_rx_dma_desc_base_addressmswset(self, dma_desc_addr_msw,
+						  index);
+
+	hw_atl_rdm_rx_desc_len_set(self, size / 8U, index);
+
+	hw_atl_rdm_rx_desc_data_buff_size_set(self, buff_size / 1024U, index);
+
+	hw_atl_rdm_rx_desc_head_buff_size_set(self, 0U, index);
+	hw_atl_rdm_rx_desc_head_splitting_set(self, 0U, index);
+	hw_atl_rpo_rx_desc_vlan_stripping_set(self, 0U, index);
+
+	/* Rx ring set mode */
+
+	/* Mapping interrupt vector */
+	hw_atl_itr_irq_map_rx_set(self, vec, index);
+	hw_atl_itr_irq_map_en_rx_set(self, true, index);
+
+	hw_atl_rdm_cpu_id_set(self, cpu, index);
+	hw_atl_rdm_rx_desc_dca_en_set(self, 0U, index);
+	hw_atl_rdm_rx_head_dca_en_set(self, 0U, index);
+	hw_atl_rdm_rx_pld_dca_en_set(self, 0U, index);
+
+	return aq_hw_err_from_flags(self);
+}
+
+int hw_atl_b0_hw_ring_tx_init(struct aq_hw_s *self, uint64_t base_addr,
+			      int index, int size, int cpu, int vec)
+{
+	u32 dma_desc_lsw_addr = (u32)base_addr;
+	u32 dma_desc_msw_addr = (u32)(base_addr >> 32);
+
+	hw_atl_reg_tx_dma_desc_base_addresslswset(self, dma_desc_lsw_addr,
+						  index);
+
+	hw_atl_reg_tx_dma_desc_base_addressmswset(self, dma_desc_msw_addr,
+						  index);
+
+	hw_atl_tdm_tx_desc_len_set(self, size / 8U, index);
+
+	hw_atl_b0_hw_tx_ring_tail_update(self, 0, index);
+
+	/* Set Tx threshold */
+	hw_atl_tdm_tx_desc_wr_wb_threshold_set(self, 0U, index);
+
+	/* Mapping interrupt vector */
+	hw_atl_itr_irq_map_tx_set(self, vec, index);
+	hw_atl_itr_irq_map_en_tx_set(self, true, index);
+
+	hw_atl_tdm_cpu_id_set(self, cpu, index);
+	hw_atl_tdm_tx_desc_dca_en_set(self, 0U, index);
+
+	return aq_hw_err_from_flags(self);
+}
+
+int hw_atl_b0_hw_irq_enable(struct aq_hw_s *self, u64 mask)
+{
+	hw_atl_itr_irq_msk_setlsw_set(self, LODWORD(mask));
+	return aq_hw_err_from_flags(self);
+}
+
+int hw_atl_b0_hw_irq_disable(struct aq_hw_s *self, u64 mask)
+{
+	hw_atl_itr_irq_msk_clearlsw_set(self, LODWORD(mask));
+	hw_atl_itr_irq_status_clearlsw_set(self, LODWORD(mask));
+
+	return aq_hw_err_from_flags(self);
+}
+
+int hw_atl_b0_hw_irq_read(struct aq_hw_s *self, u64 *mask)
+{
+	*mask = hw_atl_itr_irq_statuslsw_get(self);
+	return aq_hw_err_from_flags(self);
+}
+
+int hw_atl_b0_hw_ring_tx_stop(struct aq_hw_s *self, int index)
+{
+	hw_atl_tdm_tx_desc_en_set(self, 0U, index);
+	return aq_hw_err_from_flags(self);
+}
+
+int hw_atl_b0_hw_ring_rx_stop(struct aq_hw_s *self, int index)
+{
+	hw_atl_rdm_rx_desc_en_set(self, 0U, index);
+	return aq_hw_err_from_flags(self);
+}
+
diff --git a/drivers/net/atlantic/hw_atl/hw_atl_b0.h b/drivers/net/atlantic/hw_atl/hw_atl_b0.h
new file mode 100644
index 00000000..06feb56c
--- /dev/null
+++ b/drivers/net/atlantic/hw_atl/hw_atl_b0.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) */
+/* Copyright (C) 2014-2017 aQuantia Corporation. */
+
+/* File hw_atl_b0.h: Declaration of abstract interface for Atlantic hardware
+ * specific functions.
+ */
+
+#ifndef HW_ATL_B0_H
+#define HW_ATL_B0_H
+
+int hw_atl_b0_hw_reset(struct aq_hw_s *self);
+int hw_atl_b0_hw_init(struct aq_hw_s *self, u8 *mac_addr);
+
+int hw_atl_b0_hw_ring_tx_init(struct aq_hw_s *self, uint64_t base_addr,
+		int index, int size, int cpu, int vec);
+int hw_atl_b0_hw_ring_rx_init(struct aq_hw_s *self, uint64_t base_addr,
+		int index, int size, int buff_size, int cpu, int vec);
+
+int hw_atl_b0_hw_start(struct aq_hw_s *self);
+
+int hw_atl_b0_hw_ring_rx_start(struct aq_hw_s *self, int index);
+int hw_atl_b0_hw_ring_tx_start(struct aq_hw_s *self, int index);
+
+
+int hw_atl_b0_hw_ring_tx_stop(struct aq_hw_s *self, int index);
+int hw_atl_b0_hw_ring_rx_stop(struct aq_hw_s *self, int index);
+
+
+int hw_atl_b0_hw_tx_ring_tail_update(struct aq_hw_s *self, int tail, int index);
+
+int hw_atl_b0_hw_rss_hash_set(struct aq_hw_s *self,
+				     struct aq_rss_parameters *rss_params);
+int hw_atl_b0_hw_rss_set(struct aq_hw_s *self,
+				struct aq_rss_parameters *rss_params);
+
+int hw_atl_b0_hw_irq_enable(struct aq_hw_s *self, u64 mask);
+int hw_atl_b0_hw_irq_disable(struct aq_hw_s *self, u64 mask);
+int hw_atl_b0_hw_irq_read(struct aq_hw_s *self, u64 *mask);
+
+#endif /* HW_ATL_B0_H */
diff --git a/drivers/net/atlantic/hw_atl/hw_atl_b0_internal.h b/drivers/net/atlantic/hw_atl/hw_atl_b0_internal.h
new file mode 100644
index 00000000..48152ead
--- /dev/null
+++ b/drivers/net/atlantic/hw_atl/hw_atl_b0_internal.h
@@ -0,0 +1,145 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) */
+/* Copyright (C) 2014-2017 aQuantia Corporation. */
+
+/* File hw_atl_b0_internal.h: Definition of Atlantic B0 chip specific
+ * constants.
+ */
+
+#ifndef HW_ATL_B0_INTERNAL_H
+#define HW_ATL_B0_INTERNAL_H
+
+
+#define HW_ATL_B0_MTU_JUMBO  16352U
+#define HW_ATL_B0_MTU        1514U
+
+#define HW_ATL_B0_TX_RINGS 4U
+#define HW_ATL_B0_RX_RINGS 4U
+
+#define HW_ATL_B0_RINGS_MAX 32U
+#define HW_ATL_B0_TXD_SIZE       (16U)
+#define HW_ATL_B0_RXD_SIZE       (16U)
+
+#define HW_ATL_B0_MAC      0U
+#define HW_ATL_B0_MAC_MIN  1U
+#define HW_ATL_B0_MAC_MAX  33U
+
+/* Maximum supported VLAN filters */
+#define HW_ATL_B0_MAX_VLAN_IDS 16
+
+/* UCAST/MCAST filters */
+#define HW_ATL_B0_UCAST_FILTERS_MAX 38
+#define HW_ATL_B0_MCAST_FILTERS_MAX 8
+
+/* interrupts */
+#define HW_ATL_B0_ERR_INT 8U
+#define HW_ATL_B0_INT_MASK  (0xFFFFFFFFU)
+
+#define HW_ATL_B0_TXD_CTL2_LEN        (0xFFFFC000)
+#define HW_ATL_B0_TXD_CTL2_CTX_EN     (0x00002000)
+#define HW_ATL_B0_TXD_CTL2_CTX_IDX    (0x00001000)
+
+#define HW_ATL_B0_TXD_CTL_DESC_TYPE_TXD   (0x00000001)
+#define HW_ATL_B0_TXD_CTL_DESC_TYPE_TXC   (0x00000002)
+#define HW_ATL_B0_TXD_CTL_BLEN        (0x000FFFF0)
+#define HW_ATL_B0_TXD_CTL_DD          (0x00100000)
+#define HW_ATL_B0_TXD_CTL_EOP         (0x00200000)
+
+#define HW_ATL_B0_TXD_CTL_CMD_X       (0x3FC00000)
+
+#define HW_ATL_B0_TXD_CTL_CMD_VLAN    BIT(22)
+#define HW_ATL_B0_TXD_CTL_CMD_FCS     BIT(23)
+#define HW_ATL_B0_TXD_CTL_CMD_IPCSO   BIT(24)
+#define HW_ATL_B0_TXD_CTL_CMD_TUCSO   BIT(25)
+#define HW_ATL_B0_TXD_CTL_CMD_LSO     BIT(26)
+#define HW_ATL_B0_TXD_CTL_CMD_WB      BIT(27)
+#define HW_ATL_B0_TXD_CTL_CMD_VXLAN   BIT(28)
+
+#define HW_ATL_B0_TXD_CTL_CMD_IPV6    BIT(21)
+#define HW_ATL_B0_TXD_CTL_CMD_TCP     BIT(22)
+
+#define HW_ATL_B0_MPI_CONTROL_ADR       0x0368U
+#define HW_ATL_B0_MPI_STATE_ADR         0x036CU
+
+#define HW_ATL_B0_MPI_SPEED_MSK         0xFFFFU
+#define HW_ATL_B0_MPI_SPEED_SHIFT       16U
+
+#define HW_ATL_B0_TXBUF_MAX  160U
+#define HW_ATL_B0_RXBUF_MAX  320U
+
+#define HW_ATL_B0_RXD_BUF_SIZE_MAX  (16 * 1024)
+
+#define HW_ATL_B0_RSS_REDIRECTION_MAX 64U
+#define HW_ATL_B0_RSS_REDIRECTION_BITS 3U
+#define HW_ATL_B0_RSS_HASHKEY_BITS 320U
+
+#define HW_ATL_B0_TCRSS_4_8  1
+#define HW_ATL_B0_TC_MAX 1U
+#define HW_ATL_B0_RSS_MAX 8U
+
+#define HW_ATL_B0_LRO_RXD_MAX 2U
+#define HW_ATL_B0_RS_SLIP_ENABLED  0U
+
+/* (256k -1(max pay_len) - 54(header)) */
+#define HAL_ATL_B0_LSO_MAX_SEGMENT_SIZE 262089U
+
+/* (256k -1(max pay_len) - 74(header)) */
+#define HAL_ATL_B0_LSO_IPV6_MAX_SEGMENT_SIZE 262069U
+
+#define HW_ATL_B0_CHIP_REVISION_B0      0xA0U
+#define HW_ATL_B0_CHIP_REVISION_UNKNOWN 0xFFU
+
+#define HW_ATL_B0_FW_SEMA_RAM           0x2U
+
+#define HW_ATL_B0_TXC_LEN_TUNLEN    (0x0000FF00)
+#define HW_ATL_B0_TXC_LEN_OUTLEN    (0xFFFF0000)
+
+#define HW_ATL_B0_TXC_CTL_DESC_TYPE (0x00000007)
+#define HW_ATL_B0_TXC_CTL_CTX_ID    (0x00000008)
+#define HW_ATL_B0_TXC_CTL_VLAN      (0x000FFFF0)
+#define HW_ATL_B0_TXC_CTL_CMD       (0x00F00000)
+#define HW_ATL_B0_TXC_CTL_L2LEN     (0x7F000000)
+
+#define HW_ATL_B0_TXC_CTL_L3LEN     (0x80000000)	/* L3LEN lsb */
+#define HW_ATL_B0_TXC_LEN2_L3LEN    (0x000000FF)	/* L3LE upper bits */
+#define HW_ATL_B0_TXC_LEN2_L4LEN    (0x0000FF00)
+#define HW_ATL_B0_TXC_LEN2_MSSLEN   (0xFFFF0000)
+
+#define HW_ATL_B0_RXD_DD    (0x1)
+#define HW_ATL_B0_RXD_NCEA0 (0x1)
+
+#define HW_ATL_B0_RXD_WB_STAT_RSSTYPE (0x0000000F)
+#define HW_ATL_B0_RXD_WB_STAT_PKTTYPE (0x00000FF0)
+#define HW_ATL_B0_RXD_WB_STAT_RXCTRL  (0x00180000)
+#define HW_ATL_B0_RXD_WB_STAT_SPLHDR  (0x00200000)
+#define HW_ATL_B0_RXD_WB_STAT_HDRLEN  (0xFFC00000)
+
+#define HW_ATL_B0_RXD_WB_STAT2_DD      (0x0001)
+#define HW_ATL_B0_RXD_WB_STAT2_EOP     (0x0002)
+#define HW_ATL_B0_RXD_WB_STAT2_RXSTAT  (0x003C)
+#define HW_ATL_B0_RXD_WB_STAT2_MACERR  (0x0004)
+#define HW_ATL_B0_RXD_WB_STAT2_IP4ERR  (0x0008)
+#define HW_ATL_B0_RXD_WB_STAT2_TCPUPDERR  (0x0010)
+#define HW_ATL_B0_RXD_WB_STAT2_RXESTAT (0x0FC0)
+#define HW_ATL_B0_RXD_WB_STAT2_RSCCNT  (0xF000)
+
+#define L2_FILTER_ACTION_DISCARD (0x0)
+#define L2_FILTER_ACTION_HOST    (0x1)
+
+#define HW_ATL_B0_UCP_0X370_REG  (0x370)
+
+#define HW_ATL_B0_FLUSH() AQ_HW_READ_REG(self, 0x10)
+
+#define HW_ATL_INTR_MODER_MAX  0x1FF
+#define HW_ATL_INTR_MODER_MIN  0xFF
+
+#define HW_ATL_B0_MIN_RXD \
+	(ALIGN(AQ_CFG_SKB_FRAGS_MAX + 1U, AQ_HW_RXD_MULTIPLE))
+#define HW_ATL_B0_MIN_TXD \
+	(ALIGN(AQ_CFG_SKB_FRAGS_MAX + 1U, AQ_HW_TXD_MULTIPLE))
+
+#define HW_ATL_B0_MAX_RXD 8184U
+#define HW_ATL_B0_MAX_TXD 8184U
+
+/* HW layer capabilities */
+
+#endif /* HW_ATL_B0_INTERNAL_H */
diff --git a/drivers/net/atlantic/hw_atl/hw_atl_llh.c b/drivers/net/atlantic/hw_atl/hw_atl_llh.c
new file mode 100644
index 00000000..2dc5be2f
--- /dev/null
+++ b/drivers/net/atlantic/hw_atl/hw_atl_llh.c
@@ -0,0 +1,1490 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+/* Copyright (C) 2014-2017 aQuantia Corporation. */
+
+/* File hw_atl_llh.c: Definitions of bitfield and register access functions for
+ * Atlantic registers.
+ */
+
+#include "hw_atl_llh.h"
+
+#include "../atl_hw_regs.h"
+#include "hw_atl_llh_internal.h"
+
+/* global */
+void hw_atl_reg_glb_cpu_sem_set(struct aq_hw_s *aq_hw, u32 glb_cpu_sem,
+				u32 semaphore)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_GLB_CPU_SEM_ADR(semaphore), glb_cpu_sem);
+}
+
+u32 hw_atl_reg_glb_cpu_sem_get(struct aq_hw_s *aq_hw, u32 semaphore)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_GLB_CPU_SEM_ADR(semaphore));
+}
+
+void hw_atl_glb_glb_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 glb_reg_res_dis)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_GLB_REG_RES_DIS_ADR,
+			    HW_ATL_GLB_REG_RES_DIS_MSK,
+			    HW_ATL_GLB_REG_RES_DIS_SHIFT,
+			    glb_reg_res_dis);
+}
+
+void hw_atl_glb_soft_res_set(struct aq_hw_s *aq_hw, u32 soft_res)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_GLB_SOFT_RES_ADR,
+			    HW_ATL_GLB_SOFT_RES_MSK,
+			    HW_ATL_GLB_SOFT_RES_SHIFT, soft_res);
+}
+
+u32 hw_atl_glb_soft_res_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_GLB_SOFT_RES_ADR,
+				  HW_ATL_GLB_SOFT_RES_MSK,
+				  HW_ATL_GLB_SOFT_RES_SHIFT);
+}
+
+u32 hw_atl_reg_glb_mif_id_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_GLB_MIF_ID_ADR);
+}
+
+/* stats */
+u32 hw_atl_rpb_rx_dma_drop_pkt_cnt_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_RPB_RX_DMA_DROP_PKT_CNT_ADR);
+}
+
+u32 hw_atl_stats_rx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_RX_DMA_GOOD_OCTET_COUNTERLSW);
+}
+
+u32 hw_atl_stats_rx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_RX_DMA_GOOD_PKT_COUNTERLSW);
+}
+
+u32 hw_atl_stats_tx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_TX_DMA_GOOD_OCTET_COUNTERLSW);
+}
+
+u32 hw_atl_stats_tx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_TX_DMA_GOOD_PKT_COUNTERLSW);
+}
+
+u32 hw_atl_stats_rx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_RX_DMA_GOOD_OCTET_COUNTERMSW);
+}
+
+u32 hw_atl_stats_rx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_RX_DMA_GOOD_PKT_COUNTERMSW);
+}
+
+u32 hw_atl_stats_tx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_TX_DMA_GOOD_OCTET_COUNTERMSW);
+}
+
+u32 hw_atl_stats_tx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_TX_DMA_GOOD_PKT_COUNTERMSW);
+}
+
+/* interrupt */
+void hw_atl_itr_irq_auto_masklsw_set(struct aq_hw_s *aq_hw,
+				     u32 irq_auto_masklsw)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_ITR_IAMRLSW_ADR, irq_auto_masklsw);
+}
+
+void hw_atl_itr_irq_map_en_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_rx,
+				  u32 rx)
+{
+/* register address for bitfield imr_rx{r}_en */
+	static const u32 itr_imr_rxren_adr[32] = {
+			0x00002100U, 0x00002100U, 0x00002104U, 0x00002104U,
+			0x00002108U, 0x00002108U, 0x0000210CU, 0x0000210CU,
+			0x00002110U, 0x00002110U, 0x00002114U, 0x00002114U,
+			0x00002118U, 0x00002118U, 0x0000211CU, 0x0000211CU,
+			0x00002120U, 0x00002120U, 0x00002124U, 0x00002124U,
+			0x00002128U, 0x00002128U, 0x0000212CU, 0x0000212CU,
+			0x00002130U, 0x00002130U, 0x00002134U, 0x00002134U,
+			0x00002138U, 0x00002138U, 0x0000213CU, 0x0000213CU
+		};
+
+/* bitmask for bitfield imr_rx{r}_en */
+	static const u32 itr_imr_rxren_msk[32] = {
+			0x00008000U, 0x00000080U, 0x00008000U, 0x00000080U,
+			0x00008000U, 0x00000080U, 0x00008000U, 0x00000080U,
+			0x00008000U, 0x00000080U, 0x00008000U, 0x00000080U,
+			0x00008000U, 0x00000080U, 0x00008000U, 0x00000080U,
+			0x00008000U, 0x00000080U, 0x00008000U, 0x00000080U,
+			0x00008000U, 0x00000080U, 0x00008000U, 0x00000080U,
+			0x00008000U, 0x00000080U, 0x00008000U, 0x00000080U,
+			0x00008000U, 0x00000080U, 0x00008000U, 0x00000080U
+		};
+
+/* lower bit position of bitfield imr_rx{r}_en */
+	static const u32 itr_imr_rxren_shift[32] = {
+			15U, 7U, 15U, 7U, 15U, 7U, 15U, 7U,
+			15U, 7U, 15U, 7U, 15U, 7U, 15U, 7U,
+			15U, 7U, 15U, 7U, 15U, 7U, 15U, 7U,
+			15U, 7U, 15U, 7U, 15U, 7U, 15U, 7U
+		};
+
+	aq_hw_write_reg_bit(aq_hw, itr_imr_rxren_adr[rx],
+			    itr_imr_rxren_msk[rx],
+			    itr_imr_rxren_shift[rx],
+			    irq_map_en_rx);
+}
+
+void hw_atl_itr_irq_map_en_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_tx,
+				  u32 tx)
+{
+/* register address for bitfield imr_tx{t}_en */
+	static const u32 itr_imr_txten_adr[32] = {
+			0x00002100U, 0x00002100U, 0x00002104U, 0x00002104U,
+			0x00002108U, 0x00002108U, 0x0000210CU, 0x0000210CU,
+			0x00002110U, 0x00002110U, 0x00002114U, 0x00002114U,
+			0x00002118U, 0x00002118U, 0x0000211CU, 0x0000211CU,
+			0x00002120U, 0x00002120U, 0x00002124U, 0x00002124U,
+			0x00002128U, 0x00002128U, 0x0000212CU, 0x0000212CU,
+			0x00002130U, 0x00002130U, 0x00002134U, 0x00002134U,
+			0x00002138U, 0x00002138U, 0x0000213CU, 0x0000213CU
+		};
+
+/* bitmask for bitfield imr_tx{t}_en */
+	static const u32 itr_imr_txten_msk[32] = {
+			0x80000000U, 0x00800000U, 0x80000000U, 0x00800000U,
+			0x80000000U, 0x00800000U, 0x80000000U, 0x00800000U,
+			0x80000000U, 0x00800000U, 0x80000000U, 0x00800000U,
+			0x80000000U, 0x00800000U, 0x80000000U, 0x00800000U,
+			0x80000000U, 0x00800000U, 0x80000000U, 0x00800000U,
+			0x80000000U, 0x00800000U, 0x80000000U, 0x00800000U,
+			0x80000000U, 0x00800000U, 0x80000000U, 0x00800000U,
+			0x80000000U, 0x00800000U, 0x80000000U, 0x00800000U
+		};
+
+/* lower bit position of bitfield imr_tx{t}_en */
+	static const u32 itr_imr_txten_shift[32] = {
+			31U, 23U, 31U, 23U, 31U, 23U, 31U, 23U,
+			31U, 23U, 31U, 23U, 31U, 23U, 31U, 23U,
+			31U, 23U, 31U, 23U, 31U, 23U, 31U, 23U,
+			31U, 23U, 31U, 23U, 31U, 23U, 31U, 23U
+		};
+
+	aq_hw_write_reg_bit(aq_hw, itr_imr_txten_adr[tx],
+			    itr_imr_txten_msk[tx],
+			    itr_imr_txten_shift[tx],
+			    irq_map_en_tx);
+}
+
+void hw_atl_itr_irq_map_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_rx, u32 rx)
+{
+/* register address for bitfield imr_rx{r}[4:0] */
+	static const u32 itr_imr_rxr_adr[32] = {
+			0x00002100U, 0x00002100U, 0x00002104U, 0x00002104U,
+			0x00002108U, 0x00002108U, 0x0000210CU, 0x0000210CU,
+			0x00002110U, 0x00002110U, 0x00002114U, 0x00002114U,
+			0x00002118U, 0x00002118U, 0x0000211CU, 0x0000211CU,
+			0x00002120U, 0x00002120U, 0x00002124U, 0x00002124U,
+			0x00002128U, 0x00002128U, 0x0000212CU, 0x0000212CU,
+			0x00002130U, 0x00002130U, 0x00002134U, 0x00002134U,
+			0x00002138U, 0x00002138U, 0x0000213CU, 0x0000213CU
+		};
+
+/* bitmask for bitfield imr_rx{r}[4:0] */
+	static const u32 itr_imr_rxr_msk[32] = {
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU
+		};
+
+/* lower bit position of bitfield imr_rx{r}[4:0] */
+	static const u32 itr_imr_rxr_shift[32] = {
+			8U, 0U, 8U, 0U, 8U, 0U, 8U, 0U,
+			8U, 0U, 8U, 0U, 8U, 0U, 8U, 0U,
+			8U, 0U, 8U, 0U, 8U, 0U, 8U, 0U,
+			8U, 0U, 8U, 0U, 8U, 0U, 8U, 0U
+		};
+
+	aq_hw_write_reg_bit(aq_hw, itr_imr_rxr_adr[rx],
+			    itr_imr_rxr_msk[rx],
+			    itr_imr_rxr_shift[rx],
+			    irq_map_rx);
+}
+
+void hw_atl_itr_irq_map_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_tx, u32 tx)
+{
+/* register address for bitfield imr_tx{t}[4:0] */
+	static const u32 itr_imr_txt_adr[32] = {
+			0x00002100U, 0x00002100U, 0x00002104U, 0x00002104U,
+			0x00002108U, 0x00002108U, 0x0000210CU, 0x0000210CU,
+			0x00002110U, 0x00002110U, 0x00002114U, 0x00002114U,
+			0x00002118U, 0x00002118U, 0x0000211CU, 0x0000211CU,
+			0x00002120U, 0x00002120U, 0x00002124U, 0x00002124U,
+			0x00002128U, 0x00002128U, 0x0000212CU, 0x0000212CU,
+			0x00002130U, 0x00002130U, 0x00002134U, 0x00002134U,
+			0x00002138U, 0x00002138U, 0x0000213CU, 0x0000213CU
+		};
+
+/* bitmask for bitfield imr_tx{t}[4:0] */
+	static const u32 itr_imr_txt_msk[32] = {
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U
+		};
+
+/* lower bit position of bitfield imr_tx{t}[4:0] */
+	static const u32 itr_imr_txt_shift[32] = {
+			24U, 16U, 24U, 16U, 24U, 16U, 24U, 16U,
+			24U, 16U, 24U, 16U, 24U, 16U, 24U, 16U,
+			24U, 16U, 24U, 16U, 24U, 16U, 24U, 16U,
+			24U, 16U, 24U, 16U, 24U, 16U, 24U, 16U
+		};
+
+	aq_hw_write_reg_bit(aq_hw, itr_imr_txt_adr[tx],
+			    itr_imr_txt_msk[tx],
+			    itr_imr_txt_shift[tx],
+			    irq_map_tx);
+}
+
+void hw_atl_itr_irq_msk_clearlsw_set(struct aq_hw_s *aq_hw,
+				     u32 irq_msk_clearlsw)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_ITR_IMCRLSW_ADR, irq_msk_clearlsw);
+}
+
+void hw_atl_itr_irq_msk_setlsw_set(struct aq_hw_s *aq_hw, u32 irq_msk_setlsw)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_ITR_IMSRLSW_ADR, irq_msk_setlsw);
+}
+
+void hw_atl_itr_irq_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 irq_reg_res_dis)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_ITR_REG_RES_DSBL_ADR,
+			    HW_ATL_ITR_REG_RES_DSBL_MSK,
+			    HW_ATL_ITR_REG_RES_DSBL_SHIFT, irq_reg_res_dis);
+}
+
+void hw_atl_itr_irq_status_clearlsw_set(struct aq_hw_s *aq_hw,
+					u32 irq_status_clearlsw)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_ITR_ISCRLSW_ADR, irq_status_clearlsw);
+}
+
+u32 hw_atl_itr_irq_statuslsw_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_ITR_ISRLSW_ADR);
+}
+
+u32 hw_atl_itr_res_irq_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_ITR_RES_ADR, HW_ATL_ITR_RES_MSK,
+				  HW_ATL_ITR_RES_SHIFT);
+}
+
+void hw_atl_itr_res_irq_set(struct aq_hw_s *aq_hw, u32 res_irq)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_ITR_RES_ADR, HW_ATL_ITR_RES_MSK,
+			    HW_ATL_ITR_RES_SHIFT, res_irq);
+}
+
+/* rdm */
+void hw_atl_rdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCADCPUID_ADR(dca),
+			    HW_ATL_RDM_DCADCPUID_MSK,
+			    HW_ATL_RDM_DCADCPUID_SHIFT, cpuid);
+}
+
+void hw_atl_rdm_rx_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_dca_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCA_EN_ADR, HW_ATL_RDM_DCA_EN_MSK,
+			    HW_ATL_RDM_DCA_EN_SHIFT, rx_dca_en);
+}
+
+void hw_atl_rdm_rx_dca_mode_set(struct aq_hw_s *aq_hw, u32 rx_dca_mode)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCA_MODE_ADR,
+			    HW_ATL_RDM_DCA_MODE_MSK,
+			    HW_ATL_RDM_DCA_MODE_SHIFT, rx_dca_mode);
+}
+
+void hw_atl_rdm_rx_desc_data_buff_size_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_data_buff_size,
+					   u32 descriptor)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDDATA_SIZE_ADR(descriptor),
+			    HW_ATL_RDM_DESCDDATA_SIZE_MSK,
+			    HW_ATL_RDM_DESCDDATA_SIZE_SHIFT,
+			    rx_desc_data_buff_size);
+}
+
+void hw_atl_rdm_rx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_dca_en,
+				   u32 dca)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCADDESC_EN_ADR(dca),
+			    HW_ATL_RDM_DCADDESC_EN_MSK,
+			    HW_ATL_RDM_DCADDESC_EN_SHIFT,
+			    rx_desc_dca_en);
+}
+
+void hw_atl_rdm_rx_desc_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_en,
+			       u32 descriptor)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDEN_ADR(descriptor),
+			    HW_ATL_RDM_DESCDEN_MSK,
+			    HW_ATL_RDM_DESCDEN_SHIFT,
+			    rx_desc_en);
+}
+
+void hw_atl_rdm_rx_desc_head_buff_size_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_head_buff_size,
+					   u32 descriptor)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDHDR_SIZE_ADR(descriptor),
+			    HW_ATL_RDM_DESCDHDR_SIZE_MSK,
+			    HW_ATL_RDM_DESCDHDR_SIZE_SHIFT,
+			    rx_desc_head_buff_size);
+}
+
+void hw_atl_rdm_rx_desc_head_splitting_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_head_splitting,
+					   u32 descriptor)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDHDR_SPLIT_ADR(descriptor),
+			    HW_ATL_RDM_DESCDHDR_SPLIT_MSK,
+			    HW_ATL_RDM_DESCDHDR_SPLIT_SHIFT,
+			    rx_desc_head_splitting);
+}
+
+u32 hw_atl_rdm_rx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor)
+{
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_RDM_DESCDHD_ADR(descriptor),
+				  HW_ATL_RDM_DESCDHD_MSK,
+				  HW_ATL_RDM_DESCDHD_SHIFT);
+}
+
+void hw_atl_rdm_rx_desc_len_set(struct aq_hw_s *aq_hw, u32 rx_desc_len,
+				u32 descriptor)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDLEN_ADR(descriptor),
+			    HW_ATL_RDM_DESCDLEN_MSK, HW_ATL_RDM_DESCDLEN_SHIFT,
+			    rx_desc_len);
+}
+
+void hw_atl_rdm_rx_desc_res_set(struct aq_hw_s *aq_hw, u32 rx_desc_res,
+				u32 descriptor)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDRESET_ADR(descriptor),
+			    HW_ATL_RDM_DESCDRESET_MSK,
+			    HW_ATL_RDM_DESCDRESET_SHIFT,
+			    rx_desc_res);
+}
+
+void hw_atl_rdm_rx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
+					 u32 rx_desc_wr_wb_irq_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_INT_DESC_WRB_EN_ADR,
+			    HW_ATL_RDM_INT_DESC_WRB_EN_MSK,
+			    HW_ATL_RDM_INT_DESC_WRB_EN_SHIFT,
+			    rx_desc_wr_wb_irq_en);
+}
+
+void hw_atl_rdm_rx_head_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_head_dca_en,
+				   u32 dca)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCADHDR_EN_ADR(dca),
+			    HW_ATL_RDM_DCADHDR_EN_MSK,
+			    HW_ATL_RDM_DCADHDR_EN_SHIFT,
+			    rx_head_dca_en);
+}
+
+void hw_atl_rdm_rx_pld_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_pld_dca_en,
+				  u32 dca)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCADPAY_EN_ADR(dca),
+			    HW_ATL_RDM_DCADPAY_EN_MSK,
+			    HW_ATL_RDM_DCADPAY_EN_SHIFT,
+			    rx_pld_dca_en);
+}
+
+void hw_atl_rdm_rdm_intr_moder_en_set(struct aq_hw_s *aq_hw,
+				      u32 rdm_intr_moder_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_INT_RIM_EN_ADR,
+			    HW_ATL_RDM_INT_RIM_EN_MSK,
+			    HW_ATL_RDM_INT_RIM_EN_SHIFT,
+			    rdm_intr_moder_en);
+}
+
+/* reg */
+void hw_atl_reg_gen_irq_map_set(struct aq_hw_s *aq_hw, u32 gen_intr_map,
+				u32 regidx)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_GEN_INTR_MAP_ADR(regidx), gen_intr_map);
+}
+
+u32 hw_atl_reg_gen_irq_status_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_GEN_INTR_STAT_ADR);
+}
+
+void hw_atl_reg_irq_glb_ctl_set(struct aq_hw_s *aq_hw, u32 intr_glb_ctl)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_INTR_GLB_CTL_ADR, intr_glb_ctl);
+}
+
+void hw_atl_reg_irq_thr_set(struct aq_hw_s *aq_hw, u32 intr_thr, u32 throttle)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_INTR_THR_ADR(throttle), intr_thr);
+}
+
+void hw_atl_reg_rx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
+					       u32 rx_dma_desc_base_addrlsw,
+					       u32 descriptor)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_DMA_DESC_BASE_ADDRLSW_ADR(descriptor),
+			rx_dma_desc_base_addrlsw);
+}
+
+void hw_atl_reg_rx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
+					       u32 rx_dma_desc_base_addrmsw,
+					       u32 descriptor)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_DMA_DESC_BASE_ADDRMSW_ADR(descriptor),
+			rx_dma_desc_base_addrmsw);
+}
+
+u32 hw_atl_reg_rx_dma_desc_status_get(struct aq_hw_s *aq_hw, u32 descriptor)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_RX_DMA_DESC_STAT_ADR(descriptor));
+}
+
+void hw_atl_reg_rx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
+					 u32 rx_dma_desc_tail_ptr,
+					 u32 descriptor)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_DMA_DESC_TAIL_PTR_ADR(descriptor),
+			rx_dma_desc_tail_ptr);
+}
+
+void hw_atl_reg_rx_flr_mcst_flr_msk_set(struct aq_hw_s *aq_hw,
+					u32 rx_flr_mcst_flr_msk)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_FLR_MCST_FLR_MSK_ADR,
+			rx_flr_mcst_flr_msk);
+}
+
+void hw_atl_reg_rx_flr_mcst_flr_set(struct aq_hw_s *aq_hw, u32 rx_flr_mcst_flr,
+				    u32 filter)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_FLR_MCST_FLR_ADR(filter),
+			rx_flr_mcst_flr);
+}
+
+void hw_atl_reg_rx_flr_rss_control1set(struct aq_hw_s *aq_hw,
+				       u32 rx_flr_rss_control1)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_FLR_RSS_CONTROL1_ADR,
+			rx_flr_rss_control1);
+}
+
+void hw_atl_reg_rx_flr_control2_set(struct aq_hw_s *aq_hw,
+				    u32 rx_filter_control2)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_FLR_CONTROL2_ADR, rx_filter_control2);
+}
+
+void hw_atl_reg_rx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
+				       u32 rx_intr_moderation_ctl,
+				       u32 queue)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_INTR_MODERATION_CTL_ADR(queue),
+			rx_intr_moderation_ctl);
+}
+
+void hw_atl_reg_tx_dma_debug_ctl_set(struct aq_hw_s *aq_hw,
+				     u32 tx_dma_debug_ctl)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_TX_DMA_DEBUG_CTL_ADR, tx_dma_debug_ctl);
+}
+
+void hw_atl_reg_tx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
+					       u32 tx_dma_desc_base_addrlsw,
+					       u32 descriptor)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_TX_DMA_DESC_BASE_ADDRLSW_ADR(descriptor),
+			tx_dma_desc_base_addrlsw);
+}
+
+void hw_atl_reg_tx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
+					       u32 tx_dma_desc_base_addrmsw,
+					       u32 descriptor)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_TX_DMA_DESC_BASE_ADDRMSW_ADR(descriptor),
+			tx_dma_desc_base_addrmsw);
+}
+
+void hw_atl_reg_tx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
+					 u32 tx_dma_desc_tail_ptr,
+					 u32 descriptor)
+{
+	rte_wmb();
+
+	aq_hw_write_reg(aq_hw, HW_ATL_TX_DMA_DESC_TAIL_PTR_ADR(descriptor),
+			tx_dma_desc_tail_ptr);
+}
+
+void hw_atl_reg_tx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
+				       u32 tx_intr_moderation_ctl,
+				       u32 queue)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_TX_INTR_MODERATION_CTL_ADR(queue),
+			tx_intr_moderation_ctl);
+}
+
+/* RPB: rx packet buffer */
+void hw_atl_rpb_dma_sys_lbk_set(struct aq_hw_s *aq_hw, u32 dma_sys_lbk)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_DMA_SYS_LBK_ADR,
+			    HW_ATL_RPB_DMA_SYS_LBK_MSK,
+			    HW_ATL_RPB_DMA_SYS_LBK_SHIFT, dma_sys_lbk);
+}
+
+void hw_atl_rpb_rpf_rx_traf_class_mode_set(struct aq_hw_s *aq_hw,
+					   u32 rx_traf_class_mode)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RPF_RX_TC_MODE_ADR,
+			    HW_ATL_RPB_RPF_RX_TC_MODE_MSK,
+			    HW_ATL_RPB_RPF_RX_TC_MODE_SHIFT,
+			    rx_traf_class_mode);
+}
+
+u32 hw_atl_rpb_rpf_rx_traf_class_mode_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_RPB_RPF_RX_TC_MODE_ADR,
+			HW_ATL_RPB_RPF_RX_TC_MODE_MSK,
+			HW_ATL_RPB_RPF_RX_TC_MODE_SHIFT);
+}
+
+void hw_atl_rpb_rx_buff_en_set(struct aq_hw_s *aq_hw, u32 rx_buff_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RX_BUF_EN_ADR,
+			    HW_ATL_RPB_RX_BUF_EN_MSK,
+			    HW_ATL_RPB_RX_BUF_EN_SHIFT, rx_buff_en);
+}
+
+void hw_atl_rpb_rx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 rx_buff_hi_threshold_per_tc,
+						u32 buffer)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RXBHI_THRESH_ADR(buffer),
+			    HW_ATL_RPB_RXBHI_THRESH_MSK,
+			    HW_ATL_RPB_RXBHI_THRESH_SHIFT,
+			    rx_buff_hi_threshold_per_tc);
+}
+
+void hw_atl_rpb_rx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 rx_buff_lo_threshold_per_tc,
+						u32 buffer)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RXBLO_THRESH_ADR(buffer),
+			    HW_ATL_RPB_RXBLO_THRESH_MSK,
+			    HW_ATL_RPB_RXBLO_THRESH_SHIFT,
+			    rx_buff_lo_threshold_per_tc);
+}
+
+void hw_atl_rpb_rx_flow_ctl_mode_set(struct aq_hw_s *aq_hw,
+				     u32 rx_flow_ctl_mode)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RX_FC_MODE_ADR,
+			    HW_ATL_RPB_RX_FC_MODE_MSK,
+			    HW_ATL_RPB_RX_FC_MODE_SHIFT, rx_flow_ctl_mode);
+}
+
+void hw_atl_rpb_rx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
+					    u32 rx_pkt_buff_size_per_tc,
+					    u32 buffer)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RXBBUF_SIZE_ADR(buffer),
+			    HW_ATL_RPB_RXBBUF_SIZE_MSK,
+			    HW_ATL_RPB_RXBBUF_SIZE_SHIFT,
+			    rx_pkt_buff_size_per_tc);
+}
+
+void hw_atl_rpb_rx_xoff_en_per_tc_set(struct aq_hw_s *aq_hw,
+				      u32 rx_xoff_en_per_tc,
+				      u32 buffer)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RXBXOFF_EN_ADR(buffer),
+			    HW_ATL_RPB_RXBXOFF_EN_MSK,
+			    HW_ATL_RPB_RXBXOFF_EN_SHIFT,
+			    rx_xoff_en_per_tc);
+}
+
+/* rpf */
+
+void hw_atl_rpfl2broadcast_count_threshold_set(struct aq_hw_s *aq_hw,
+					       u32 l2broadcast_count_threshold)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2BC_THRESH_ADR,
+			    HW_ATL_RPFL2BC_THRESH_MSK,
+			    HW_ATL_RPFL2BC_THRESH_SHIFT,
+			    l2broadcast_count_threshold);
+}
+
+void hw_atl_rpfl2broadcast_en_set(struct aq_hw_s *aq_hw, u32 l2broadcast_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2BC_EN_ADR, HW_ATL_RPFL2BC_EN_MSK,
+			    HW_ATL_RPFL2BC_EN_SHIFT, l2broadcast_en);
+}
+
+void hw_atl_rpfl2broadcast_flr_act_set(struct aq_hw_s *aq_hw,
+				       u32 l2broadcast_flr_act)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2BC_ACT_ADR,
+			    HW_ATL_RPFL2BC_ACT_MSK,
+			    HW_ATL_RPFL2BC_ACT_SHIFT, l2broadcast_flr_act);
+}
+
+void hw_atl_rpfl2multicast_flr_en_set(struct aq_hw_s *aq_hw,
+				      u32 l2multicast_flr_en,
+				      u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2MC_ENF_ADR(filter),
+			    HW_ATL_RPFL2MC_ENF_MSK,
+			    HW_ATL_RPFL2MC_ENF_SHIFT, l2multicast_flr_en);
+}
+
+void hw_atl_rpfl2promiscuous_mode_en_set(struct aq_hw_s *aq_hw,
+					 u32 l2promiscuous_mode_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2PROMIS_MODE_ADR,
+			    HW_ATL_RPFL2PROMIS_MODE_MSK,
+			    HW_ATL_RPFL2PROMIS_MODE_SHIFT,
+			    l2promiscuous_mode_en);
+}
+
+void hw_atl_rpfl2unicast_flr_act_set(struct aq_hw_s *aq_hw,
+				     u32 l2unicast_flr_act,
+				     u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2UC_ACTF_ADR(filter),
+			    HW_ATL_RPFL2UC_ACTF_MSK, HW_ATL_RPFL2UC_ACTF_SHIFT,
+			    l2unicast_flr_act);
+}
+
+void hw_atl_rpfl2_uc_flr_en_set(struct aq_hw_s *aq_hw, u32 l2unicast_flr_en,
+				u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2UC_ENF_ADR(filter),
+			    HW_ATL_RPFL2UC_ENF_MSK,
+			    HW_ATL_RPFL2UC_ENF_SHIFT, l2unicast_flr_en);
+}
+
+void hw_atl_rpfl2unicast_dest_addresslsw_set(struct aq_hw_s *aq_hw,
+					     u32 l2unicast_dest_addresslsw,
+					     u32 filter)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RPFL2UC_DAFLSW_ADR(filter),
+			l2unicast_dest_addresslsw);
+}
+
+void hw_atl_rpfl2unicast_dest_addressmsw_set(struct aq_hw_s *aq_hw,
+					     u32 l2unicast_dest_addressmsw,
+					     u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2UC_DAFMSW_ADR(filter),
+			    HW_ATL_RPFL2UC_DAFMSW_MSK,
+			    HW_ATL_RPFL2UC_DAFMSW_SHIFT,
+			    l2unicast_dest_addressmsw);
+}
+
+void hw_atl_rpfl2_accept_all_mc_packets_set(struct aq_hw_s *aq_hw,
+					    u32 l2_accept_all_mc_packets)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2MC_ACCEPT_ALL_ADR,
+			    HW_ATL_RPFL2MC_ACCEPT_ALL_MSK,
+			    HW_ATL_RPFL2MC_ACCEPT_ALL_SHIFT,
+			    l2_accept_all_mc_packets);
+}
+
+void hw_atl_rpf_rpb_user_priority_tc_map_set(struct aq_hw_s *aq_hw,
+					     u32 user_priority_tc_map, u32 tc)
+{
+/* register address for bitfield rx_tc_up{t}[2:0] */
+	static const u32 rpf_rpb_rx_tc_upt_adr[8] = {
+			0x000054c4U, 0x000054C4U, 0x000054C4U, 0x000054C4U,
+			0x000054c4U, 0x000054C4U, 0x000054C4U, 0x000054C4U
+		};
+
+/* bitmask for bitfield rx_tc_up{t}[2:0] */
+	static const u32 rpf_rpb_rx_tc_upt_msk[8] = {
+			0x00000007U, 0x00000070U, 0x00000700U, 0x00007000U,
+			0x00070000U, 0x00700000U, 0x07000000U, 0x70000000U
+		};
+
+/* lower bit position of bitfield rx_tc_up{t}[2:0] */
+	static const u32 rpf_rpb_rx_tc_upt_shft[8] = {
+			0U, 4U, 8U, 12U, 16U, 20U, 24U, 28U
+		};
+
+	aq_hw_write_reg_bit(aq_hw, rpf_rpb_rx_tc_upt_adr[tc],
+			    rpf_rpb_rx_tc_upt_msk[tc],
+			    rpf_rpb_rx_tc_upt_shft[tc],
+			    user_priority_tc_map);
+}
+
+void hw_atl_rpf_rss_key_addr_set(struct aq_hw_s *aq_hw, u32 rss_key_addr)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_RSS_KEY_ADDR_ADR,
+			    HW_ATL_RPF_RSS_KEY_ADDR_MSK,
+			    HW_ATL_RPF_RSS_KEY_ADDR_SHIFT,
+			    rss_key_addr);
+}
+
+void hw_atl_rpf_rss_key_wr_data_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_data)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RPF_RSS_KEY_WR_DATA_ADR,
+			rss_key_wr_data);
+}
+
+u32 hw_atl_rpf_rss_key_wr_en_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_RPF_RSS_KEY_WR_ENI_ADR,
+				  HW_ATL_RPF_RSS_KEY_WR_ENI_MSK,
+				  HW_ATL_RPF_RSS_KEY_WR_ENI_SHIFT);
+}
+
+void hw_atl_rpf_rss_key_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_RSS_KEY_WR_ENI_ADR,
+			    HW_ATL_RPF_RSS_KEY_WR_ENI_MSK,
+			    HW_ATL_RPF_RSS_KEY_WR_ENI_SHIFT,
+			    rss_key_wr_en);
+}
+
+void hw_atl_rpf_rss_redir_tbl_addr_set(struct aq_hw_s *aq_hw,
+				       u32 rss_redir_tbl_addr)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_RSS_REDIR_ADDR_ADR,
+			    HW_ATL_RPF_RSS_REDIR_ADDR_MSK,
+			    HW_ATL_RPF_RSS_REDIR_ADDR_SHIFT,
+			    rss_redir_tbl_addr);
+}
+
+void hw_atl_rpf_rss_redir_tbl_wr_data_set(struct aq_hw_s *aq_hw,
+					  u32 rss_redir_tbl_wr_data)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_RSS_REDIR_WR_DATA_ADR,
+			    HW_ATL_RPF_RSS_REDIR_WR_DATA_MSK,
+			    HW_ATL_RPF_RSS_REDIR_WR_DATA_SHIFT,
+			    rss_redir_tbl_wr_data);
+}
+
+u32 hw_atl_rpf_rss_redir_wr_en_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_RPF_RSS_REDIR_WR_ENI_ADR,
+				  HW_ATL_RPF_RSS_REDIR_WR_ENI_MSK,
+				  HW_ATL_RPF_RSS_REDIR_WR_ENI_SHIFT);
+}
+
+void hw_atl_rpf_rss_redir_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_redir_wr_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_RSS_REDIR_WR_ENI_ADR,
+			    HW_ATL_RPF_RSS_REDIR_WR_ENI_MSK,
+			    HW_ATL_RPF_RSS_REDIR_WR_ENI_SHIFT, rss_redir_wr_en);
+}
+
+void hw_atl_rpf_tpo_to_rpf_sys_lbk_set(struct aq_hw_s *aq_hw,
+				       u32 tpo_to_rpf_sys_lbk)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_TPO_RPF_SYS_LBK_ADR,
+			    HW_ATL_RPF_TPO_RPF_SYS_LBK_MSK,
+			    HW_ATL_RPF_TPO_RPF_SYS_LBK_SHIFT,
+			    tpo_to_rpf_sys_lbk);
+}
+
+void hw_atl_rpf_vlan_inner_etht_set(struct aq_hw_s *aq_hw, u32 vlan_inner_etht)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_INNER_TPID_ADR,
+			    HW_ATL_RPF_VL_INNER_TPID_MSK,
+			    HW_ATL_RPF_VL_INNER_TPID_SHIFT,
+			    vlan_inner_etht);
+}
+
+void hw_atl_rpf_vlan_outer_etht_set(struct aq_hw_s *aq_hw, u32 vlan_outer_etht)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_OUTER_TPID_ADR,
+			    HW_ATL_RPF_VL_OUTER_TPID_MSK,
+			    HW_ATL_RPF_VL_OUTER_TPID_SHIFT,
+			    vlan_outer_etht);
+}
+
+void hw_atl_rpf_vlan_prom_mode_en_set(struct aq_hw_s *aq_hw,
+				      u32 vlan_prom_mode_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_PROMIS_MODE_ADR,
+			    HW_ATL_RPF_VL_PROMIS_MODE_MSK,
+			    HW_ATL_RPF_VL_PROMIS_MODE_SHIFT,
+			    vlan_prom_mode_en);
+}
+
+void hw_atl_rpf_vlan_accept_untagged_packets_set(struct aq_hw_s *aq_hw,
+						 u32 vlan_acc_untagged_packets)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_ADR,
+			    HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_MSK,
+			    HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_SHIFT,
+			    vlan_acc_untagged_packets);
+}
+
+void hw_atl_rpf_vlan_untagged_act_set(struct aq_hw_s *aq_hw,
+				      u32 vlan_untagged_act)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_UNTAGGED_ACT_ADR,
+			    HW_ATL_RPF_VL_UNTAGGED_ACT_MSK,
+			    HW_ATL_RPF_VL_UNTAGGED_ACT_SHIFT,
+			    vlan_untagged_act);
+}
+
+void hw_atl_rpf_vlan_flr_en_set(struct aq_hw_s *aq_hw, u32 vlan_flr_en,
+				u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_EN_F_ADR(filter),
+			    HW_ATL_RPF_VL_EN_F_MSK,
+			    HW_ATL_RPF_VL_EN_F_SHIFT,
+			    vlan_flr_en);
+}
+
+void hw_atl_rpf_vlan_flr_act_set(struct aq_hw_s *aq_hw, u32 vlan_flr_act,
+				 u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_ACT_F_ADR(filter),
+			    HW_ATL_RPF_VL_ACT_F_MSK,
+			    HW_ATL_RPF_VL_ACT_F_SHIFT,
+			    vlan_flr_act);
+}
+
+void hw_atl_rpf_vlan_id_flr_set(struct aq_hw_s *aq_hw, u32 vlan_id_flr,
+				u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_ID_F_ADR(filter),
+			    HW_ATL_RPF_VL_ID_F_MSK,
+			    HW_ATL_RPF_VL_ID_F_SHIFT,
+			    vlan_id_flr);
+}
+
+void hw_atl_rpf_etht_flr_en_set(struct aq_hw_s *aq_hw, u32 etht_flr_en,
+				u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_ENF_ADR(filter),
+			    HW_ATL_RPF_ET_ENF_MSK,
+			    HW_ATL_RPF_ET_ENF_SHIFT, etht_flr_en);
+}
+
+void hw_atl_rpf_etht_user_priority_en_set(struct aq_hw_s *aq_hw,
+					  u32 etht_user_priority_en, u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_UPFEN_ADR(filter),
+			    HW_ATL_RPF_ET_UPFEN_MSK, HW_ATL_RPF_ET_UPFEN_SHIFT,
+			    etht_user_priority_en);
+}
+
+void hw_atl_rpf_etht_rx_queue_en_set(struct aq_hw_s *aq_hw,
+				     u32 etht_rx_queue_en,
+				     u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_RXQFEN_ADR(filter),
+			    HW_ATL_RPF_ET_RXQFEN_MSK,
+			    HW_ATL_RPF_ET_RXQFEN_SHIFT,
+			    etht_rx_queue_en);
+}
+
+void hw_atl_rpf_etht_user_priority_set(struct aq_hw_s *aq_hw,
+				       u32 etht_user_priority,
+				       u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_UPF_ADR(filter),
+			    HW_ATL_RPF_ET_UPF_MSK,
+			    HW_ATL_RPF_ET_UPF_SHIFT, etht_user_priority);
+}
+
+void hw_atl_rpf_etht_rx_queue_set(struct aq_hw_s *aq_hw, u32 etht_rx_queue,
+				  u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_RXQF_ADR(filter),
+			    HW_ATL_RPF_ET_RXQF_MSK,
+			    HW_ATL_RPF_ET_RXQF_SHIFT, etht_rx_queue);
+}
+
+void hw_atl_rpf_etht_mgt_queue_set(struct aq_hw_s *aq_hw, u32 etht_mgt_queue,
+				   u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_MNG_RXQF_ADR(filter),
+			    HW_ATL_RPF_ET_MNG_RXQF_MSK,
+			    HW_ATL_RPF_ET_MNG_RXQF_SHIFT,
+			    etht_mgt_queue);
+}
+
+void hw_atl_rpf_etht_flr_act_set(struct aq_hw_s *aq_hw, u32 etht_flr_act,
+				 u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_ACTF_ADR(filter),
+			    HW_ATL_RPF_ET_ACTF_MSK,
+			    HW_ATL_RPF_ET_ACTF_SHIFT, etht_flr_act);
+}
+
+void hw_atl_rpf_etht_flr_set(struct aq_hw_s *aq_hw, u32 etht_flr, u32 filter)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_VALF_ADR(filter),
+			    HW_ATL_RPF_ET_VALF_MSK,
+			    HW_ATL_RPF_ET_VALF_SHIFT, etht_flr);
+}
+
+/* RPO: rx packet offload */
+void hw_atl_rpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					      u32 ipv4header_crc_offload_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_IPV4CHK_EN_ADR,
+			    HW_ATL_RPO_IPV4CHK_EN_MSK,
+			    HW_ATL_RPO_IPV4CHK_EN_SHIFT,
+			    ipv4header_crc_offload_en);
+}
+
+void hw_atl_rpo_rx_desc_vlan_stripping_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_vlan_stripping,
+					   u32 descriptor)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_DESCDVL_STRIP_ADR(descriptor),
+			    HW_ATL_RPO_DESCDVL_STRIP_MSK,
+			    HW_ATL_RPO_DESCDVL_STRIP_SHIFT,
+			    rx_desc_vlan_stripping);
+}
+
+void hw_atl_rpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					   u32 tcp_udp_crc_offload_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPOL4CHK_EN_ADR,
+			    HW_ATL_RPOL4CHK_EN_MSK,
+			    HW_ATL_RPOL4CHK_EN_SHIFT, tcp_udp_crc_offload_en);
+}
+
+void hw_atl_rpo_lro_en_set(struct aq_hw_s *aq_hw, u32 lro_en)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RPO_LRO_EN_ADR, lro_en);
+}
+
+void hw_atl_rpo_lro_patch_optimization_en_set(struct aq_hw_s *aq_hw,
+					      u32 lro_patch_optimization_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_PTOPT_EN_ADR,
+			    HW_ATL_RPO_LRO_PTOPT_EN_MSK,
+			    HW_ATL_RPO_LRO_PTOPT_EN_SHIFT,
+			    lro_patch_optimization_en);
+}
+
+void hw_atl_rpo_lro_qsessions_lim_set(struct aq_hw_s *aq_hw,
+				      u32 lro_qsessions_lim)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_QSES_LMT_ADR,
+			    HW_ATL_RPO_LRO_QSES_LMT_MSK,
+			    HW_ATL_RPO_LRO_QSES_LMT_SHIFT,
+			    lro_qsessions_lim);
+}
+
+void hw_atl_rpo_lro_total_desc_lim_set(struct aq_hw_s *aq_hw,
+				       u32 lro_total_desc_lim)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_TOT_DSC_LMT_ADR,
+			    HW_ATL_RPO_LRO_TOT_DSC_LMT_MSK,
+			    HW_ATL_RPO_LRO_TOT_DSC_LMT_SHIFT,
+			    lro_total_desc_lim);
+}
+
+void hw_atl_rpo_lro_min_pay_of_first_pkt_set(struct aq_hw_s *aq_hw,
+					     u32 lro_min_pld_of_first_pkt)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_PKT_MIN_ADR,
+			    HW_ATL_RPO_LRO_PKT_MIN_MSK,
+			    HW_ATL_RPO_LRO_PKT_MIN_SHIFT,
+			    lro_min_pld_of_first_pkt);
+}
+
+void hw_atl_rpo_lro_pkt_lim_set(struct aq_hw_s *aq_hw, u32 lro_pkt_lim)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_RPO_LRO_RSC_MAX_ADR, lro_pkt_lim);
+}
+
+void hw_atl_rpo_lro_max_num_of_descriptors_set(struct aq_hw_s *aq_hw,
+					u32 lro_max_number_of_descriptors,
+					u32 lro)
+{
+/* Register address for bitfield lro{L}_des_max[1:0] */
+	static const u32 rpo_lro_ldes_max_adr[32] = {
+			0x000055A0U, 0x000055A0U, 0x000055A0U, 0x000055A0U,
+			0x000055A0U, 0x000055A0U, 0x000055A0U, 0x000055A0U,
+			0x000055A4U, 0x000055A4U, 0x000055A4U, 0x000055A4U,
+			0x000055A4U, 0x000055A4U, 0x000055A4U, 0x000055A4U,
+			0x000055A8U, 0x000055A8U, 0x000055A8U, 0x000055A8U,
+			0x000055A8U, 0x000055A8U, 0x000055A8U, 0x000055A8U,
+			0x000055ACU, 0x000055ACU, 0x000055ACU, 0x000055ACU,
+			0x000055ACU, 0x000055ACU, 0x000055ACU, 0x000055ACU
+		};
+
+/* Bitmask for bitfield lro{L}_des_max[1:0] */
+	static const u32 rpo_lro_ldes_max_msk[32] = {
+			0x00000003U, 0x00000030U, 0x00000300U, 0x00003000U,
+			0x00030000U, 0x00300000U, 0x03000000U, 0x30000000U,
+			0x00000003U, 0x00000030U, 0x00000300U, 0x00003000U,
+			0x00030000U, 0x00300000U, 0x03000000U, 0x30000000U,
+			0x00000003U, 0x00000030U, 0x00000300U, 0x00003000U,
+			0x00030000U, 0x00300000U, 0x03000000U, 0x30000000U,
+			0x00000003U, 0x00000030U, 0x00000300U, 0x00003000U,
+			0x00030000U, 0x00300000U, 0x03000000U, 0x30000000U
+		};
+
+/* Lower bit position of bitfield lro{L}_des_max[1:0] */
+	static const u32 rpo_lro_ldes_max_shift[32] = {
+			0U, 4U, 8U, 12U, 16U, 20U, 24U, 28U,
+			0U, 4U, 8U, 12U, 16U, 20U, 24U, 28U,
+			0U, 4U, 8U, 12U, 16U, 20U, 24U, 28U,
+			0U, 4U, 8U, 12U, 16U, 20U, 24U, 28U
+		};
+
+	aq_hw_write_reg_bit(aq_hw, rpo_lro_ldes_max_adr[lro],
+			    rpo_lro_ldes_max_msk[lro],
+			    rpo_lro_ldes_max_shift[lro],
+			    lro_max_number_of_descriptors);
+}
+
+void hw_atl_rpo_lro_time_base_divider_set(struct aq_hw_s *aq_hw,
+					  u32 lro_time_base_divider)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_TB_DIV_ADR,
+			    HW_ATL_RPO_LRO_TB_DIV_MSK,
+			    HW_ATL_RPO_LRO_TB_DIV_SHIFT,
+			    lro_time_base_divider);
+}
+
+void hw_atl_rpo_lro_inactive_interval_set(struct aq_hw_s *aq_hw,
+					  u32 lro_inactive_interval)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_INA_IVAL_ADR,
+			    HW_ATL_RPO_LRO_INA_IVAL_MSK,
+			    HW_ATL_RPO_LRO_INA_IVAL_SHIFT,
+			    lro_inactive_interval);
+}
+
+void hw_atl_rpo_lro_max_coalescing_interval_set(struct aq_hw_s *aq_hw,
+						u32 lro_max_coal_interval)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_MAX_IVAL_ADR,
+			    HW_ATL_RPO_LRO_MAX_IVAL_MSK,
+			    HW_ATL_RPO_LRO_MAX_IVAL_SHIFT,
+			    lro_max_coal_interval);
+}
+
+/* rx */
+void hw_atl_rx_rx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 rx_reg_res_dis)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RX_REG_RES_DSBL_ADR,
+			    HW_ATL_RX_REG_RES_DSBL_MSK,
+			    HW_ATL_RX_REG_RES_DSBL_SHIFT,
+			    rx_reg_res_dis);
+}
+
+/* tdm */
+void hw_atl_tdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DCADCPUID_ADR(dca),
+			    HW_ATL_TDM_DCADCPUID_MSK,
+			    HW_ATL_TDM_DCADCPUID_SHIFT, cpuid);
+}
+
+void hw_atl_tdm_large_send_offload_en_set(struct aq_hw_s *aq_hw,
+					  u32 large_send_offload_en)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_TDM_LSO_EN_ADR, large_send_offload_en);
+}
+
+void hw_atl_tdm_tx_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_dca_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DCA_EN_ADR, HW_ATL_TDM_DCA_EN_MSK,
+			    HW_ATL_TDM_DCA_EN_SHIFT, tx_dca_en);
+}
+
+void hw_atl_tdm_tx_dca_mode_set(struct aq_hw_s *aq_hw, u32 tx_dca_mode)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DCA_MODE_ADR,
+			    HW_ATL_TDM_DCA_MODE_MSK,
+			    HW_ATL_TDM_DCA_MODE_SHIFT, tx_dca_mode);
+}
+
+void hw_atl_tdm_tx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_dca_en,
+				   u32 dca)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DCADDESC_EN_ADR(dca),
+			    HW_ATL_TDM_DCADDESC_EN_MSK,
+			    HW_ATL_TDM_DCADDESC_EN_SHIFT,
+			    tx_desc_dca_en);
+}
+
+void hw_atl_tdm_tx_desc_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_en,
+			       u32 descriptor)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DESCDEN_ADR(descriptor),
+			    HW_ATL_TDM_DESCDEN_MSK,
+			    HW_ATL_TDM_DESCDEN_SHIFT,
+			    tx_desc_en);
+}
+
+u32 hw_atl_tdm_tx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor)
+{
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_TDM_DESCDHD_ADR(descriptor),
+				  HW_ATL_TDM_DESCDHD_MSK,
+				  HW_ATL_TDM_DESCDHD_SHIFT);
+}
+
+void hw_atl_tdm_tx_desc_len_set(struct aq_hw_s *aq_hw, u32 tx_desc_len,
+				u32 descriptor)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DESCDLEN_ADR(descriptor),
+			    HW_ATL_TDM_DESCDLEN_MSK,
+			    HW_ATL_TDM_DESCDLEN_SHIFT,
+			    tx_desc_len);
+}
+
+void hw_atl_tdm_tx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
+					 u32 tx_desc_wr_wb_irq_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_INT_DESC_WRB_EN_ADR,
+			    HW_ATL_TDM_INT_DESC_WRB_EN_MSK,
+			    HW_ATL_TDM_INT_DESC_WRB_EN_SHIFT,
+			    tx_desc_wr_wb_irq_en);
+}
+
+void hw_atl_tdm_tx_desc_wr_wb_threshold_set(struct aq_hw_s *aq_hw,
+					    u32 tx_desc_wr_wb_threshold,
+					    u32 descriptor)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DESCDWRB_THRESH_ADR(descriptor),
+			    HW_ATL_TDM_DESCDWRB_THRESH_MSK,
+			    HW_ATL_TDM_DESCDWRB_THRESH_SHIFT,
+			    tx_desc_wr_wb_threshold);
+}
+
+void hw_atl_tdm_tdm_intr_moder_en_set(struct aq_hw_s *aq_hw,
+				      u32 tdm_irq_moderation_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_INT_MOD_EN_ADR,
+			    HW_ATL_TDM_INT_MOD_EN_MSK,
+			    HW_ATL_TDM_INT_MOD_EN_SHIFT,
+			    tdm_irq_moderation_en);
+}
+
+/* thm */
+void hw_atl_thm_lso_tcp_flag_of_first_pkt_set(struct aq_hw_s *aq_hw,
+					      u32 lso_tcp_flag_of_first_pkt)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_THM_LSO_TCP_FLAG_FIRST_ADR,
+			    HW_ATL_THM_LSO_TCP_FLAG_FIRST_MSK,
+			    HW_ATL_THM_LSO_TCP_FLAG_FIRST_SHIFT,
+			    lso_tcp_flag_of_first_pkt);
+}
+
+void hw_atl_thm_lso_tcp_flag_of_last_pkt_set(struct aq_hw_s *aq_hw,
+					     u32 lso_tcp_flag_of_last_pkt)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_THM_LSO_TCP_FLAG_LAST_ADR,
+			    HW_ATL_THM_LSO_TCP_FLAG_LAST_MSK,
+			    HW_ATL_THM_LSO_TCP_FLAG_LAST_SHIFT,
+			    lso_tcp_flag_of_last_pkt);
+}
+
+void hw_atl_thm_lso_tcp_flag_of_middle_pkt_set(struct aq_hw_s *aq_hw,
+					       u32 lso_tcp_flag_of_middle_pkt)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_THM_LSO_TCP_FLAG_MID_ADR,
+			    HW_ATL_THM_LSO_TCP_FLAG_MID_MSK,
+			    HW_ATL_THM_LSO_TCP_FLAG_MID_SHIFT,
+			    lso_tcp_flag_of_middle_pkt);
+}
+
+/* TPB: tx packet buffer */
+void hw_atl_tpb_tx_buff_en_set(struct aq_hw_s *aq_hw, u32 tx_buff_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TX_BUF_EN_ADR,
+			    HW_ATL_TPB_TX_BUF_EN_MSK,
+			    HW_ATL_TPB_TX_BUF_EN_SHIFT, tx_buff_en);
+}
+
+u32 hw_atl_rpb_tps_tx_tc_mode_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_TPB_TX_TC_MODE_ADDR,
+			HW_ATL_TPB_TX_TC_MODE_MSK,
+			HW_ATL_TPB_TX_TC_MODE_SHIFT);
+}
+
+void hw_atl_rpb_tps_tx_tc_mode_set(struct aq_hw_s *aq_hw,
+				   u32 tx_traf_class_mode)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TX_TC_MODE_ADDR,
+			HW_ATL_TPB_TX_TC_MODE_MSK,
+			HW_ATL_TPB_TX_TC_MODE_SHIFT,
+			tx_traf_class_mode);
+}
+
+void hw_atl_tpb_tx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 tx_buff_hi_threshold_per_tc,
+					 u32 buffer)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TXBHI_THRESH_ADR(buffer),
+			    HW_ATL_TPB_TXBHI_THRESH_MSK,
+			    HW_ATL_TPB_TXBHI_THRESH_SHIFT,
+			    tx_buff_hi_threshold_per_tc);
+}
+
+void hw_atl_tpb_tx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 tx_buff_lo_threshold_per_tc,
+						u32 buffer)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TXBLO_THRESH_ADR(buffer),
+			    HW_ATL_TPB_TXBLO_THRESH_MSK,
+			    HW_ATL_TPB_TXBLO_THRESH_SHIFT,
+			    tx_buff_lo_threshold_per_tc);
+}
+
+void hw_atl_tpb_tx_dma_sys_lbk_en_set(struct aq_hw_s *aq_hw,
+				      u32 tx_dma_sys_lbk_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_DMA_SYS_LBK_ADR,
+			    HW_ATL_TPB_DMA_SYS_LBK_MSK,
+			    HW_ATL_TPB_DMA_SYS_LBK_SHIFT,
+			    tx_dma_sys_lbk_en);
+}
+
+void hw_atl_tpb_tx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
+					    u32 tx_pkt_buff_size_per_tc,
+					    u32 buffer)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TXBBUF_SIZE_ADR(buffer),
+			    HW_ATL_TPB_TXBBUF_SIZE_MSK,
+			    HW_ATL_TPB_TXBBUF_SIZE_SHIFT,
+			    tx_pkt_buff_size_per_tc);
+}
+
+void hw_atl_tpb_tx_path_scp_ins_en_set(struct aq_hw_s *aq_hw,
+				       u32 tx_path_scp_ins_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TX_SCP_INS_EN_ADR,
+			    HW_ATL_TPB_TX_SCP_INS_EN_MSK,
+			    HW_ATL_TPB_TX_SCP_INS_EN_SHIFT,
+			    tx_path_scp_ins_en);
+}
+
+/* TPO: tx packet offload */
+void hw_atl_tpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					      u32 ipv4header_crc_offload_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPO_IPV4CHK_EN_ADR,
+			    HW_ATL_TPO_IPV4CHK_EN_MSK,
+			    HW_ATL_TPO_IPV4CHK_EN_SHIFT,
+			    ipv4header_crc_offload_en);
+}
+
+void hw_atl_tpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					   u32 tcp_udp_crc_offload_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPOL4CHK_EN_ADR,
+			    HW_ATL_TPOL4CHK_EN_MSK,
+			    HW_ATL_TPOL4CHK_EN_SHIFT,
+			    tcp_udp_crc_offload_en);
+}
+
+void hw_atl_tpo_tx_pkt_sys_lbk_en_set(struct aq_hw_s *aq_hw,
+				      u32 tx_pkt_sys_lbk_en)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPO_PKT_SYS_LBK_ADR,
+			    HW_ATL_TPO_PKT_SYS_LBK_MSK,
+			    HW_ATL_TPO_PKT_SYS_LBK_SHIFT,
+			    tx_pkt_sys_lbk_en);
+}
+
+/* TPS: tx packet scheduler */
+void hw_atl_tps_tx_pkt_shed_data_arb_mode_set(struct aq_hw_s *aq_hw,
+					      u32 tx_pkt_shed_data_arb_mode)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DATA_TC_ARB_MODE_ADR,
+			    HW_ATL_TPS_DATA_TC_ARB_MODE_MSK,
+			    HW_ATL_TPS_DATA_TC_ARB_MODE_SHIFT,
+			    tx_pkt_shed_data_arb_mode);
+}
+
+void hw_atl_tps_tx_pkt_shed_desc_rate_curr_time_res_set(struct aq_hw_s *aq_hw,
+							u32 curr_time_res)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_RATE_TA_RST_ADR,
+			    HW_ATL_TPS_DESC_RATE_TA_RST_MSK,
+			    HW_ATL_TPS_DESC_RATE_TA_RST_SHIFT,
+			    curr_time_res);
+}
+
+void hw_atl_tps_tx_pkt_shed_desc_rate_lim_set(struct aq_hw_s *aq_hw,
+					      u32 tx_pkt_shed_desc_rate_lim)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_RATE_LIM_ADR,
+			    HW_ATL_TPS_DESC_RATE_LIM_MSK,
+			    HW_ATL_TPS_DESC_RATE_LIM_SHIFT,
+			    tx_pkt_shed_desc_rate_lim);
+}
+
+void hw_atl_tps_tx_pkt_shed_desc_tc_arb_mode_set(struct aq_hw_s *aq_hw,
+						 u32 arb_mode)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_TC_ARB_MODE_ADR,
+			    HW_ATL_TPS_DESC_TC_ARB_MODE_MSK,
+			    HW_ATL_TPS_DESC_TC_ARB_MODE_SHIFT,
+			    arb_mode);
+}
+
+void hw_atl_tps_tx_pkt_shed_desc_tc_max_credit_set(struct aq_hw_s *aq_hw,
+						   u32 max_credit,
+						   u32 tc)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_TCTCREDIT_MAX_ADR(tc),
+			    HW_ATL_TPS_DESC_TCTCREDIT_MAX_MSK,
+			    HW_ATL_TPS_DESC_TCTCREDIT_MAX_SHIFT,
+			    max_credit);
+}
+
+void hw_atl_tps_tx_pkt_shed_desc_tc_weight_set(struct aq_hw_s *aq_hw,
+					       u32 tx_pkt_shed_desc_tc_weight,
+					       u32 tc)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_TCTWEIGHT_ADR(tc),
+			    HW_ATL_TPS_DESC_TCTWEIGHT_MSK,
+			    HW_ATL_TPS_DESC_TCTWEIGHT_SHIFT,
+			    tx_pkt_shed_desc_tc_weight);
+}
+
+void hw_atl_tps_tx_pkt_shed_desc_vm_arb_mode_set(struct aq_hw_s *aq_hw,
+						 u32 arb_mode)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_VM_ARB_MODE_ADR,
+			    HW_ATL_TPS_DESC_VM_ARB_MODE_MSK,
+			    HW_ATL_TPS_DESC_VM_ARB_MODE_SHIFT,
+			    arb_mode);
+}
+
+void hw_atl_tps_tx_pkt_shed_tc_data_max_credit_set(struct aq_hw_s *aq_hw,
+						   u32 max_credit,
+						   u32 tc)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DATA_TCTCREDIT_MAX_ADR(tc),
+			    HW_ATL_TPS_DATA_TCTCREDIT_MAX_MSK,
+			    HW_ATL_TPS_DATA_TCTCREDIT_MAX_SHIFT,
+			    max_credit);
+}
+
+void hw_atl_tps_tx_pkt_shed_tc_data_weight_set(struct aq_hw_s *aq_hw,
+					       u32 tx_pkt_shed_tc_data_weight,
+					       u32 tc)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DATA_TCTWEIGHT_ADR(tc),
+			    HW_ATL_TPS_DATA_TCTWEIGHT_MSK,
+			    HW_ATL_TPS_DATA_TCTWEIGHT_SHIFT,
+			    tx_pkt_shed_tc_data_weight);
+}
+
+/* tx */
+void hw_atl_tx_tx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 tx_reg_res_dis)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TX_REG_RES_DSBL_ADR,
+			    HW_ATL_TX_REG_RES_DSBL_MSK,
+			    HW_ATL_TX_REG_RES_DSBL_SHIFT, tx_reg_res_dis);
+}
+
+/* msm */
+u32 hw_atl_msm_reg_access_status_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_MSM_REG_ACCESS_BUSY_ADR,
+				  HW_ATL_MSM_REG_ACCESS_BUSY_MSK,
+				  HW_ATL_MSM_REG_ACCESS_BUSY_SHIFT);
+}
+
+void hw_atl_msm_reg_addr_for_indirect_addr_set(struct aq_hw_s *aq_hw,
+					       u32 reg_addr_for_indirect_addr)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_MSM_REG_ADDR_ADR,
+			    HW_ATL_MSM_REG_ADDR_MSK,
+			    HW_ATL_MSM_REG_ADDR_SHIFT,
+			    reg_addr_for_indirect_addr);
+}
+
+void hw_atl_msm_reg_rd_strobe_set(struct aq_hw_s *aq_hw, u32 reg_rd_strobe)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_MSM_REG_RD_STROBE_ADR,
+			    HW_ATL_MSM_REG_RD_STROBE_MSK,
+			    HW_ATL_MSM_REG_RD_STROBE_SHIFT,
+			    reg_rd_strobe);
+}
+
+u32 hw_atl_msm_reg_rd_data_get(struct aq_hw_s *aq_hw)
+{
+	return aq_hw_read_reg(aq_hw, HW_ATL_MSM_REG_RD_DATA_ADR);
+}
+
+void hw_atl_msm_reg_wr_data_set(struct aq_hw_s *aq_hw, u32 reg_wr_data)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_MSM_REG_WR_DATA_ADR, reg_wr_data);
+}
+
+void hw_atl_msm_reg_wr_strobe_set(struct aq_hw_s *aq_hw, u32 reg_wr_strobe)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_MSM_REG_WR_STROBE_ADR,
+			    HW_ATL_MSM_REG_WR_STROBE_MSK,
+			    HW_ATL_MSM_REG_WR_STROBE_SHIFT,
+			    reg_wr_strobe);
+}
+
+/* pci */
+void hw_atl_pci_pci_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 pci_reg_res_dis)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_PCI_REG_RES_DSBL_ADR,
+			    HW_ATL_PCI_REG_RES_DSBL_MSK,
+			    HW_ATL_PCI_REG_RES_DSBL_SHIFT,
+			    pci_reg_res_dis);
+}
+
+void hw_atl_reg_glb_cpu_scratch_scp_set(struct aq_hw_s *aq_hw,
+					u32 glb_cpu_scratch_scp,
+					u32 scratch_scp)
+{
+	aq_hw_write_reg(aq_hw, HW_ATL_GLB_CPU_SCRATCH_SCP_ADR(scratch_scp),
+			glb_cpu_scratch_scp);
+}
+
+void hw_atl_mcp_up_force_intr_set(struct aq_hw_s *aq_hw, u32 up_force_intr)
+{
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_MCP_UP_FORCE_INTERRUPT_ADR,
+			HW_ATL_MCP_UP_FORCE_INTERRUPT_MSK,
+			HW_ATL_MCP_UP_FORCE_INTERRUPT_SHIFT, up_force_intr);
+}
diff --git a/drivers/net/atlantic/hw_atl/hw_atl_llh.h b/drivers/net/atlantic/hw_atl/hw_atl_llh.h
new file mode 100644
index 00000000..e30083ce
--- /dev/null
+++ b/drivers/net/atlantic/hw_atl/hw_atl_llh.h
@@ -0,0 +1,714 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) */
+/* Copyright (C) 2014-2017 aQuantia Corporation. */
+
+/* File hw_atl_llh.h: Declarations of bitfield and register access functions for
+ * Atlantic registers.
+ */
+
+#ifndef HW_ATL_LLH_H
+#define HW_ATL_LLH_H
+
+#include "../atl_types.h"
+
+struct aq_hw_s;
+
+/* global */
+
+/* set global microprocessor semaphore */
+void hw_atl_reg_glb_cpu_sem_set(struct aq_hw_s *aq_hw,	u32 glb_cpu_sem,
+				u32 semaphore);
+
+/* get global microprocessor semaphore */
+u32 hw_atl_reg_glb_cpu_sem_get(struct aq_hw_s *aq_hw, u32 semaphore);
+
+/* set global register reset disable */
+void hw_atl_glb_glb_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 glb_reg_res_dis);
+
+/* set soft reset */
+void hw_atl_glb_soft_res_set(struct aq_hw_s *aq_hw, u32 soft_res);
+
+/* get soft reset */
+u32 hw_atl_glb_soft_res_get(struct aq_hw_s *aq_hw);
+
+/* stats */
+
+u32 hw_atl_rpb_rx_dma_drop_pkt_cnt_get(struct aq_hw_s *aq_hw);
+
+/* get rx dma good octet counter lsw */
+u32 hw_atl_stats_rx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw);
+
+/* get rx dma good packet counter lsw */
+u32 hw_atl_stats_rx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw);
+
+/* get tx dma good octet counter lsw */
+u32 hw_atl_stats_tx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw);
+
+/* get tx dma good packet counter lsw */
+u32 hw_atl_stats_tx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw);
+
+/* get rx dma good octet counter msw */
+u32 hw_atl_stats_rx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw);
+
+/* get rx dma good packet counter msw */
+u32 hw_atl_stats_rx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw);
+
+/* get tx dma good octet counter msw */
+u32 hw_atl_stats_tx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw);
+
+/* get tx dma good packet counter msw */
+u32 hw_atl_stats_tx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw);
+
+/* get msm rx errors counter register */
+u32 hw_atl_reg_mac_msm_rx_errs_cnt_get(struct aq_hw_s *aq_hw);
+
+/* get msm rx unicast frames counter register */
+u32 hw_atl_reg_mac_msm_rx_ucst_frm_cnt_get(struct aq_hw_s *aq_hw);
+
+/* get msm rx multicast frames counter register */
+u32 hw_atl_reg_mac_msm_rx_mcst_frm_cnt_get(struct aq_hw_s *aq_hw);
+
+/* get msm rx broadcast frames counter register */
+u32 hw_atl_reg_mac_msm_rx_bcst_frm_cnt_get(struct aq_hw_s *aq_hw);
+
+/* get msm rx broadcast octets counter register 1 */
+u32 hw_atl_reg_mac_msm_rx_bcst_octets_counter1get(struct aq_hw_s *aq_hw);
+
+/* get msm rx unicast octets counter register 0 */
+u32 hw_atl_reg_mac_msm_rx_ucst_octets_counter0get(struct aq_hw_s *aq_hw);
+
+/* get rx dma statistics counter 7 */
+u32 hw_atl_reg_rx_dma_stat_counter7get(struct aq_hw_s *aq_hw);
+
+/* get msm tx errors counter register */
+u32 hw_atl_reg_mac_msm_tx_errs_cnt_get(struct aq_hw_s *aq_hw);
+
+/* get msm tx unicast frames counter register */
+u32 hw_atl_reg_mac_msm_tx_ucst_frm_cnt_get(struct aq_hw_s *aq_hw);
+
+/* get msm tx multicast frames counter register */
+u32 hw_atl_reg_mac_msm_tx_mcst_frm_cnt_get(struct aq_hw_s *aq_hw);
+
+/* get msm tx broadcast frames counter register */
+u32 hw_atl_reg_mac_msm_tx_bcst_frm_cnt_get(struct aq_hw_s *aq_hw);
+
+/* get msm tx multicast octets counter register 1 */
+u32 hw_atl_reg_mac_msm_tx_mcst_octets_counter1get(struct aq_hw_s *aq_hw);
+
+/* get msm tx broadcast octets counter register 1 */
+u32 hw_atl_reg_mac_msm_tx_bcst_octets_counter1get(struct aq_hw_s *aq_hw);
+
+/* get msm tx unicast octets counter register 0 */
+u32 hw_atl_reg_mac_msm_tx_ucst_octets_counter0get(struct aq_hw_s *aq_hw);
+
+/* get global mif identification */
+u32 hw_atl_reg_glb_mif_id_get(struct aq_hw_s *aq_hw);
+
+/* interrupt */
+
+/* set interrupt auto mask lsw */
+void hw_atl_itr_irq_auto_masklsw_set(struct aq_hw_s *aq_hw,
+				     u32 irq_auto_masklsw);
+
+/* set interrupt mapping enable rx */
+void hw_atl_itr_irq_map_en_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_rx,
+				  u32 rx);
+
+/* set interrupt mapping enable tx */
+void hw_atl_itr_irq_map_en_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_tx,
+				  u32 tx);
+
+/* set interrupt mapping rx */
+void hw_atl_itr_irq_map_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_rx, u32 rx);
+
+/* set interrupt mapping tx */
+void hw_atl_itr_irq_map_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_tx, u32 tx);
+
+/* set interrupt mask clear lsw */
+void hw_atl_itr_irq_msk_clearlsw_set(struct aq_hw_s *aq_hw,
+				     u32 irq_msk_clearlsw);
+
+/* set interrupt mask set lsw */
+void hw_atl_itr_irq_msk_setlsw_set(struct aq_hw_s *aq_hw, u32 irq_msk_setlsw);
+
+/* set interrupt register reset disable */
+void hw_atl_itr_irq_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 irq_reg_res_dis);
+
+/* set interrupt status clear lsw */
+void hw_atl_itr_irq_status_clearlsw_set(struct aq_hw_s *aq_hw,
+					u32 irq_status_clearlsw);
+
+/* get interrupt status lsw */
+u32 hw_atl_itr_irq_statuslsw_get(struct aq_hw_s *aq_hw);
+
+/* get reset interrupt */
+u32 hw_atl_itr_res_irq_get(struct aq_hw_s *aq_hw);
+
+/* set reset interrupt */
+void hw_atl_itr_res_irq_set(struct aq_hw_s *aq_hw, u32 res_irq);
+
+/* rdm */
+
+/* set cpu id */
+void hw_atl_rdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca);
+
+/* set rx dca enable */
+void hw_atl_rdm_rx_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_dca_en);
+
+/* set rx dca mode */
+void hw_atl_rdm_rx_dca_mode_set(struct aq_hw_s *aq_hw, u32 rx_dca_mode);
+
+/* set rx descriptor data buffer size */
+void hw_atl_rdm_rx_desc_data_buff_size_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_data_buff_size,
+				    u32 descriptor);
+
+/* set rx descriptor dca enable */
+void hw_atl_rdm_rx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_dca_en,
+				   u32 dca);
+
+/* set rx descriptor enable */
+void hw_atl_rdm_rx_desc_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_en,
+			       u32 descriptor);
+
+/* set rx descriptor header splitting */
+void hw_atl_rdm_rx_desc_head_splitting_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_head_splitting,
+				    u32 descriptor);
+
+/* get rx descriptor head pointer */
+u32 hw_atl_rdm_rx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor);
+
+/* set rx descriptor length */
+void hw_atl_rdm_rx_desc_len_set(struct aq_hw_s *aq_hw, u32 rx_desc_len,
+				u32 descriptor);
+
+/* set rx descriptor write-back interrupt enable */
+void hw_atl_rdm_rx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
+					 u32 rx_desc_wr_wb_irq_en);
+
+/* set rx header dca enable */
+void hw_atl_rdm_rx_head_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_head_dca_en,
+				   u32 dca);
+
+/* set rx payload dca enable */
+void hw_atl_rdm_rx_pld_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_pld_dca_en,
+				  u32 dca);
+
+/* set rx descriptor header buffer size */
+void hw_atl_rdm_rx_desc_head_buff_size_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_head_buff_size,
+					   u32 descriptor);
+
+/* set rx descriptor reset */
+void hw_atl_rdm_rx_desc_res_set(struct aq_hw_s *aq_hw, u32 rx_desc_res,
+				u32 descriptor);
+
+/* Set RDM Interrupt Moderation Enable */
+void hw_atl_rdm_rdm_intr_moder_en_set(struct aq_hw_s *aq_hw,
+				      u32 rdm_intr_moder_en);
+
+/* reg */
+
+/* set general interrupt mapping register */
+void hw_atl_reg_gen_irq_map_set(struct aq_hw_s *aq_hw, u32 gen_intr_map,
+				u32 regidx);
+
+/* get general interrupt status register */
+u32 hw_atl_reg_gen_irq_status_get(struct aq_hw_s *aq_hw);
+
+/* set interrupt global control register */
+void hw_atl_reg_irq_glb_ctl_set(struct aq_hw_s *aq_hw, u32 intr_glb_ctl);
+
+/* set interrupt throttle register */
+void hw_atl_reg_irq_thr_set(struct aq_hw_s *aq_hw, u32 intr_thr, u32 throttle);
+
+/* set rx dma descriptor base address lsw */
+void hw_atl_reg_rx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
+					       u32 rx_dma_desc_base_addrlsw,
+					u32 descriptor);
+
+/* set rx dma descriptor base address msw */
+void hw_atl_reg_rx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
+					       u32 rx_dma_desc_base_addrmsw,
+					u32 descriptor);
+
+/* get rx dma descriptor status register */
+u32 hw_atl_reg_rx_dma_desc_status_get(struct aq_hw_s *aq_hw, u32 descriptor);
+
+/* set rx dma descriptor tail pointer register */
+void hw_atl_reg_rx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
+					 u32 rx_dma_desc_tail_ptr,
+				  u32 descriptor);
+
+/* set rx filter multicast filter mask register */
+void hw_atl_reg_rx_flr_mcst_flr_msk_set(struct aq_hw_s *aq_hw,
+					u32 rx_flr_mcst_flr_msk);
+
+/* set rx filter multicast filter register */
+void hw_atl_reg_rx_flr_mcst_flr_set(struct aq_hw_s *aq_hw, u32 rx_flr_mcst_flr,
+				    u32 filter);
+
+/* set rx filter rss control register 1 */
+void hw_atl_reg_rx_flr_rss_control1set(struct aq_hw_s *aq_hw,
+				       u32 rx_flr_rss_control1);
+
+/* Set RX Filter Control Register 2 */
+void hw_atl_reg_rx_flr_control2_set(struct aq_hw_s *aq_hw, u32 rx_flr_control2);
+
+/* Set RX Interrupt Moderation Control Register */
+void hw_atl_reg_rx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
+				       u32 rx_intr_moderation_ctl,
+				u32 queue);
+
+/* set tx dma debug control */
+void hw_atl_reg_tx_dma_debug_ctl_set(struct aq_hw_s *aq_hw,
+				     u32 tx_dma_debug_ctl);
+
+/* set tx dma descriptor base address lsw */
+void hw_atl_reg_tx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
+					       u32 tx_dma_desc_base_addrlsw,
+					u32 descriptor);
+
+/* set tx dma descriptor base address msw */
+void hw_atl_reg_tx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
+					       u32 tx_dma_desc_base_addrmsw,
+					u32 descriptor);
+
+/* set tx dma descriptor tail pointer register */
+void hw_atl_reg_tx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
+					 u32 tx_dma_desc_tail_ptr,
+					 u32 descriptor);
+
+/* Set TX Interrupt Moderation Control Register */
+void hw_atl_reg_tx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
+				       u32 tx_intr_moderation_ctl,
+				       u32 queue);
+
+/* set global microprocessor scratch pad */
+void hw_atl_reg_glb_cpu_scratch_scp_set(struct aq_hw_s *aq_hw,
+					u32 glb_cpu_scratch_scp,
+					u32 scratch_scp);
+
+/* rpb */
+
+/* set dma system loopback */
+void hw_atl_rpb_dma_sys_lbk_set(struct aq_hw_s *aq_hw, u32 dma_sys_lbk);
+
+/* set rx traffic class mode */
+void hw_atl_rpb_rpf_rx_traf_class_mode_set(struct aq_hw_s *aq_hw,
+					   u32 rx_traf_class_mode);
+
+/* get rx traffic class mode */
+u32 hw_atl_rpb_rpf_rx_traf_class_mode_get(struct aq_hw_s *aq_hw);
+
+/* set rx buffer enable */
+void hw_atl_rpb_rx_buff_en_set(struct aq_hw_s *aq_hw, u32 rx_buff_en);
+
+/* set rx buffer high threshold (per tc) */
+void hw_atl_rpb_rx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 rx_buff_hi_threshold_per_tc,
+						u32 buffer);
+
+/* set rx buffer low threshold (per tc) */
+void hw_atl_rpb_rx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 rx_buff_lo_threshold_per_tc,
+					 u32 buffer);
+
+/* set rx flow control mode */
+void hw_atl_rpb_rx_flow_ctl_mode_set(struct aq_hw_s *aq_hw,
+				     u32 rx_flow_ctl_mode);
+
+/* set rx packet buffer size (per tc) */
+void hw_atl_rpb_rx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
+					    u32 rx_pkt_buff_size_per_tc,
+					    u32 buffer);
+
+/* set rx xoff enable (per tc) */
+void hw_atl_rpb_rx_xoff_en_per_tc_set(struct aq_hw_s *aq_hw,
+				      u32 rx_xoff_en_per_tc,
+				      u32 buffer);
+
+/* rpf */
+
+/* set l2 broadcast count threshold */
+void hw_atl_rpfl2broadcast_count_threshold_set(struct aq_hw_s *aq_hw,
+					       u32 l2broadcast_count_threshold);
+
+/* set l2 broadcast enable */
+void hw_atl_rpfl2broadcast_en_set(struct aq_hw_s *aq_hw, u32 l2broadcast_en);
+
+/* set l2 broadcast filter action */
+void hw_atl_rpfl2broadcast_flr_act_set(struct aq_hw_s *aq_hw,
+				       u32 l2broadcast_flr_act);
+
+/* set l2 multicast filter enable */
+void hw_atl_rpfl2multicast_flr_en_set(struct aq_hw_s *aq_hw,
+				      u32 l2multicast_flr_en,
+				      u32 filter);
+
+/* set l2 promiscuous mode enable */
+void hw_atl_rpfl2promiscuous_mode_en_set(struct aq_hw_s *aq_hw,
+					 u32 l2promiscuous_mode_en);
+
+/* set l2 unicast filter action */
+void hw_atl_rpfl2unicast_flr_act_set(struct aq_hw_s *aq_hw,
+				     u32 l2unicast_flr_act,
+				     u32 filter);
+
+/* set l2 unicast filter enable */
+void hw_atl_rpfl2_uc_flr_en_set(struct aq_hw_s *aq_hw, u32 l2unicast_flr_en,
+				u32 filter);
+
+/* set l2 unicast destination address lsw */
+void hw_atl_rpfl2unicast_dest_addresslsw_set(struct aq_hw_s *aq_hw,
+					     u32 l2unicast_dest_addresslsw,
+					     u32 filter);
+
+/* set l2 unicast destination address msw */
+void hw_atl_rpfl2unicast_dest_addressmsw_set(struct aq_hw_s *aq_hw,
+					     u32 l2unicast_dest_addressmsw,
+				      u32 filter);
+
+/* Set L2 Accept all Multicast packets */
+void hw_atl_rpfl2_accept_all_mc_packets_set(struct aq_hw_s *aq_hw,
+					    u32 l2_accept_all_mc_packets);
+
+/* set user-priority tc mapping */
+void hw_atl_rpf_rpb_user_priority_tc_map_set(struct aq_hw_s *aq_hw,
+					     u32 user_priority_tc_map, u32 tc);
+
+/* set rss key address */
+void hw_atl_rpf_rss_key_addr_set(struct aq_hw_s *aq_hw, u32 rss_key_addr);
+
+/* set rss key write data */
+void hw_atl_rpf_rss_key_wr_data_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_data);
+
+/* get rss key write enable */
+u32 hw_atl_rpf_rss_key_wr_en_get(struct aq_hw_s *aq_hw);
+
+/* set rss key write enable */
+void hw_atl_rpf_rss_key_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_en);
+
+/* set rss redirection table address */
+void hw_atl_rpf_rss_redir_tbl_addr_set(struct aq_hw_s *aq_hw,
+				       u32 rss_redir_tbl_addr);
+
+/* set rss redirection table write data */
+void hw_atl_rpf_rss_redir_tbl_wr_data_set(struct aq_hw_s *aq_hw,
+					  u32 rss_redir_tbl_wr_data);
+
+/* get rss redirection write enable */
+u32 hw_atl_rpf_rss_redir_wr_en_get(struct aq_hw_s *aq_hw);
+
+/* set rss redirection write enable */
+void hw_atl_rpf_rss_redir_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_redir_wr_en);
+
+/* set tpo to rpf system loopback */
+void hw_atl_rpf_tpo_to_rpf_sys_lbk_set(struct aq_hw_s *aq_hw,
+				       u32 tpo_to_rpf_sys_lbk);
+
+/* set vlan inner ethertype */
+void hw_atl_rpf_vlan_inner_etht_set(struct aq_hw_s *aq_hw, u32 vlan_inner_etht);
+
+/* set vlan outer ethertype */
+void hw_atl_rpf_vlan_outer_etht_set(struct aq_hw_s *aq_hw, u32 vlan_outer_etht);
+
+/* set vlan promiscuous mode enable */
+void hw_atl_rpf_vlan_prom_mode_en_set(struct aq_hw_s *aq_hw,
+				      u32 vlan_prom_mode_en);
+
+/* Set VLAN untagged action */
+void hw_atl_rpf_vlan_untagged_act_set(struct aq_hw_s *aq_hw,
+				      u32 vlan_untagged_act);
+
+/* Set VLAN accept untagged packets */
+void hw_atl_rpf_vlan_accept_untagged_packets_set(struct aq_hw_s *aq_hw,
+						 u32 vlan_acc_untagged_packets);
+
+/* Set VLAN filter enable */
+void hw_atl_rpf_vlan_flr_en_set(struct aq_hw_s *aq_hw, u32 vlan_flr_en,
+				u32 filter);
+
+/* Set VLAN Filter Action */
+void hw_atl_rpf_vlan_flr_act_set(struct aq_hw_s *aq_hw, u32 vlan_filter_act,
+				 u32 filter);
+
+/* Set VLAN ID Filter */
+void hw_atl_rpf_vlan_id_flr_set(struct aq_hw_s *aq_hw, u32 vlan_id_flr,
+				u32 filter);
+
+/* set ethertype filter enable */
+void hw_atl_rpf_etht_flr_en_set(struct aq_hw_s *aq_hw, u32 etht_flr_en,
+				u32 filter);
+
+/* set  ethertype user-priority enable */
+void hw_atl_rpf_etht_user_priority_en_set(struct aq_hw_s *aq_hw,
+					  u32 etht_user_priority_en,
+					  u32 filter);
+
+/* set  ethertype rx queue enable */
+void hw_atl_rpf_etht_rx_queue_en_set(struct aq_hw_s *aq_hw,
+				     u32 etht_rx_queue_en,
+				     u32 filter);
+
+/* set ethertype rx queue */
+void hw_atl_rpf_etht_rx_queue_set(struct aq_hw_s *aq_hw, u32 etht_rx_queue,
+				  u32 filter);
+
+/* set ethertype user-priority */
+void hw_atl_rpf_etht_user_priority_set(struct aq_hw_s *aq_hw,
+				       u32 etht_user_priority,
+				       u32 filter);
+
+/* set ethertype management queue */
+void hw_atl_rpf_etht_mgt_queue_set(struct aq_hw_s *aq_hw, u32 etht_mgt_queue,
+				   u32 filter);
+
+/* set ethertype filter action */
+void hw_atl_rpf_etht_flr_act_set(struct aq_hw_s *aq_hw, u32 etht_flr_act,
+				 u32 filter);
+
+/* set ethertype filter */
+void hw_atl_rpf_etht_flr_set(struct aq_hw_s *aq_hw, u32 etht_flr, u32 filter);
+
+/* rpo */
+
+/* set ipv4 header checksum offload enable */
+void hw_atl_rpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					      u32 ipv4header_crc_offload_en);
+
+/* set rx descriptor vlan stripping */
+void hw_atl_rpo_rx_desc_vlan_stripping_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_vlan_stripping,
+					   u32 descriptor);
+
+/* set tcp/udp checksum offload enable */
+void hw_atl_rpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					   u32 tcp_udp_crc_offload_en);
+
+/* Set LRO Patch Optimization Enable. */
+void hw_atl_rpo_lro_patch_optimization_en_set(struct aq_hw_s *aq_hw,
+					      u32 lro_patch_optimization_en);
+
+/* Set Large Receive Offload Enable */
+void hw_atl_rpo_lro_en_set(struct aq_hw_s *aq_hw, u32 lro_en);
+
+/* Set LRO Q Sessions Limit */
+void hw_atl_rpo_lro_qsessions_lim_set(struct aq_hw_s *aq_hw,
+				      u32 lro_qsessions_lim);
+
+/* Set LRO Total Descriptor Limit */
+void hw_atl_rpo_lro_total_desc_lim_set(struct aq_hw_s *aq_hw,
+				       u32 lro_total_desc_lim);
+
+/* Set LRO Min Payload of First Packet */
+void hw_atl_rpo_lro_min_pay_of_first_pkt_set(struct aq_hw_s *aq_hw,
+					     u32 lro_min_pld_of_first_pkt);
+
+/* Set LRO Packet Limit */
+void hw_atl_rpo_lro_pkt_lim_set(struct aq_hw_s *aq_hw, u32 lro_packet_lim);
+
+/* Set LRO Max Number of Descriptors */
+void hw_atl_rpo_lro_max_num_of_descriptors_set(struct aq_hw_s *aq_hw,
+					       u32 lro_max_desc_num, u32 lro);
+
+/* Set LRO Time Base Divider */
+void hw_atl_rpo_lro_time_base_divider_set(struct aq_hw_s *aq_hw,
+					  u32 lro_time_base_divider);
+
+/*Set LRO Inactive Interval */
+void hw_atl_rpo_lro_inactive_interval_set(struct aq_hw_s *aq_hw,
+					  u32 lro_inactive_interval);
+
+/*Set LRO Max Coalescing Interval */
+void hw_atl_rpo_lro_max_coalescing_interval_set(struct aq_hw_s *aq_hw,
+						u32 lro_max_coal_interval);
+
+/* rx */
+
+/* set rx register reset disable */
+void hw_atl_rx_rx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 rx_reg_res_dis);
+
+/* tdm */
+
+/* set cpu id */
+void hw_atl_tdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca);
+
+/* set large send offload enable */
+void hw_atl_tdm_large_send_offload_en_set(struct aq_hw_s *aq_hw,
+					  u32 large_send_offload_en);
+
+/* set tx descriptor enable */
+void hw_atl_tdm_tx_desc_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_en,
+			       u32 descriptor);
+
+/* set tx dca enable */
+void hw_atl_tdm_tx_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_dca_en);
+
+/* set tx dca mode */
+void hw_atl_tdm_tx_dca_mode_set(struct aq_hw_s *aq_hw, u32 tx_dca_mode);
+
+/* set tx descriptor dca enable */
+void hw_atl_tdm_tx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_dca_en,
+				   u32 dca);
+
+/* get tx descriptor head pointer */
+u32 hw_atl_tdm_tx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor);
+
+/* set tx descriptor length */
+void hw_atl_tdm_tx_desc_len_set(struct aq_hw_s *aq_hw, u32 tx_desc_len,
+				u32 descriptor);
+
+/* set tx descriptor write-back interrupt enable */
+void hw_atl_tdm_tx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
+					 u32 tx_desc_wr_wb_irq_en);
+
+/* set tx descriptor write-back threshold */
+void hw_atl_tdm_tx_desc_wr_wb_threshold_set(struct aq_hw_s *aq_hw,
+					    u32 tx_desc_wr_wb_threshold,
+				     u32 descriptor);
+
+/* Set TDM Interrupt Moderation Enable */
+void hw_atl_tdm_tdm_intr_moder_en_set(struct aq_hw_s *aq_hw,
+				      u32 tdm_irq_moderation_en);
+/* thm */
+
+/* set lso tcp flag of first packet */
+void hw_atl_thm_lso_tcp_flag_of_first_pkt_set(struct aq_hw_s *aq_hw,
+					      u32 lso_tcp_flag_of_first_pkt);
+
+/* set lso tcp flag of last packet */
+void hw_atl_thm_lso_tcp_flag_of_last_pkt_set(struct aq_hw_s *aq_hw,
+					     u32 lso_tcp_flag_of_last_pkt);
+
+/* set lso tcp flag of middle packet */
+void hw_atl_thm_lso_tcp_flag_of_middle_pkt_set(struct aq_hw_s *aq_hw,
+					       u32 lso_tcp_flag_of_middle_pkt);
+
+/* tpb */
+
+/* set TX Traffic Class Mode */
+void hw_atl_rpb_tps_tx_tc_mode_set(struct aq_hw_s *aq_hw,
+				   u32 tx_traf_class_mode);
+
+/* get TX Traffic Class Mode */
+u32 hw_atl_rpb_tps_tx_tc_mode_get(struct aq_hw_s *aq_hw);
+
+/* set tx buffer enable */
+void hw_atl_tpb_tx_buff_en_set(struct aq_hw_s *aq_hw, u32 tx_buff_en);
+
+/* set tx buffer high threshold (per tc) */
+void hw_atl_tpb_tx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 tx_buff_hi_threshold_per_tc,
+					 u32 buffer);
+
+/* set tx buffer low threshold (per tc) */
+void hw_atl_tpb_tx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 tx_buff_lo_threshold_per_tc,
+					 u32 buffer);
+
+/* set tx dma system loopback enable */
+void hw_atl_tpb_tx_dma_sys_lbk_en_set(struct aq_hw_s *aq_hw,
+				      u32 tx_dma_sys_lbk_en);
+
+/* set tx packet buffer size (per tc) */
+void hw_atl_tpb_tx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
+					    u32 tx_pkt_buff_size_per_tc,
+					    u32 buffer);
+
+/* set tx path pad insert enable */
+void hw_atl_tpb_tx_path_scp_ins_en_set(struct aq_hw_s *aq_hw,
+				       u32 tx_path_scp_ins_en);
+
+/* tpo */
+
+/* set ipv4 header checksum offload enable */
+void hw_atl_tpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					      u32 ipv4header_crc_offload_en);
+
+/* set tcp/udp checksum offload enable */
+void hw_atl_tpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					   u32 tcp_udp_crc_offload_en);
+
+/* set tx pkt system loopback enable */
+void hw_atl_tpo_tx_pkt_sys_lbk_en_set(struct aq_hw_s *aq_hw,
+				      u32 tx_pkt_sys_lbk_en);
+
+/* tps */
+
+/* set tx packet scheduler data arbitration mode */
+void hw_atl_tps_tx_pkt_shed_data_arb_mode_set(struct aq_hw_s *aq_hw,
+					      u32 tx_pkt_shed_data_arb_mode);
+
+/* set tx packet scheduler descriptor rate current time reset */
+void hw_atl_tps_tx_pkt_shed_desc_rate_curr_time_res_set(struct aq_hw_s *aq_hw,
+							u32 curr_time_res);
+
+/* set tx packet scheduler descriptor rate limit */
+void hw_atl_tps_tx_pkt_shed_desc_rate_lim_set(struct aq_hw_s *aq_hw,
+					      u32 tx_pkt_shed_desc_rate_lim);
+
+/* set tx packet scheduler descriptor tc arbitration mode */
+void hw_atl_tps_tx_pkt_shed_desc_tc_arb_mode_set(struct aq_hw_s *aq_hw,
+						 u32 arb_mode);
+
+/* set tx packet scheduler descriptor tc max credit */
+void hw_atl_tps_tx_pkt_shed_desc_tc_max_credit_set(struct aq_hw_s *aq_hw,
+						   u32 max_credit,
+					    u32 tc);
+
+/* set tx packet scheduler descriptor tc weight */
+void hw_atl_tps_tx_pkt_shed_desc_tc_weight_set(struct aq_hw_s *aq_hw,
+					       u32 tx_pkt_shed_desc_tc_weight,
+					u32 tc);
+
+/* set tx packet scheduler descriptor vm arbitration mode */
+void hw_atl_tps_tx_pkt_shed_desc_vm_arb_mode_set(struct aq_hw_s *aq_hw,
+						 u32 arb_mode);
+
+/* set tx packet scheduler tc data max credit */
+void hw_atl_tps_tx_pkt_shed_tc_data_max_credit_set(struct aq_hw_s *aq_hw,
+						   u32 max_credit,
+					    u32 tc);
+
+/* set tx packet scheduler tc data weight */
+void hw_atl_tps_tx_pkt_shed_tc_data_weight_set(struct aq_hw_s *aq_hw,
+					       u32 tx_pkt_shed_tc_data_weight,
+					u32 tc);
+
+/* tx */
+
+/* set tx register reset disable */
+void hw_atl_tx_tx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 tx_reg_res_dis);
+
+/* msm */
+
+/* get register access status */
+u32 hw_atl_msm_reg_access_status_get(struct aq_hw_s *aq_hw);
+
+/* set  register address for indirect address */
+void hw_atl_msm_reg_addr_for_indirect_addr_set(struct aq_hw_s *aq_hw,
+					       u32 reg_addr_for_indirect_addr);
+
+/* set register read strobe */
+void hw_atl_msm_reg_rd_strobe_set(struct aq_hw_s *aq_hw, u32 reg_rd_strobe);
+
+/* get  register read data */
+u32 hw_atl_msm_reg_rd_data_get(struct aq_hw_s *aq_hw);
+
+/* set  register write data */
+void hw_atl_msm_reg_wr_data_set(struct aq_hw_s *aq_hw, u32 reg_wr_data);
+
+/* set register write strobe */
+void hw_atl_msm_reg_wr_strobe_set(struct aq_hw_s *aq_hw, u32 reg_wr_strobe);
+
+/* pci */
+
+/* set pci register reset disable */
+void hw_atl_pci_pci_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 pci_reg_res_dis);
+
+/* set uP Force Interrupt */
+void hw_atl_mcp_up_force_intr_set(struct aq_hw_s *aq_hw, u32 up_force_intr);
+
+
+#endif /* HW_ATL_LLH_H */
diff --git a/drivers/net/atlantic/hw_atl/hw_atl_llh_internal.h b/drivers/net/atlantic/hw_atl/hw_atl_llh_internal.h
new file mode 100644
index 00000000..27b9b9cb
--- /dev/null
+++ b/drivers/net/atlantic/hw_atl/hw_atl_llh_internal.h
@@ -0,0 +1,2407 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) */
+/* Copyright (C) 2014-2017 aQuantia Corporation. */
+
+/* File hw_atl_llh_internal.h: Preprocessor definitions
+ * for Atlantic registers.
+ */
+
+#ifndef HW_ATL_LLH_INTERNAL_H
+#define HW_ATL_LLH_INTERNAL_H
+
+/* global microprocessor semaphore  definitions
+ * base address: 0x000003a0
+ * parameter: semaphore {s} | stride size 0x4 | range [0, 15]
+ */
+#define HW_ATL_GLB_CPU_SEM_ADR(semaphore)  (0x000003a0u + (semaphore) * 0x4)
+/* register address for bitfield rx dma good octet counter lsw [1f:0] */
+#define HW_ATL_STATS_RX_DMA_GOOD_OCTET_COUNTERLSW 0x00006808
+/* register address for bitfield rx dma good packet counter lsw [1f:0] */
+#define HW_ATL_STATS_RX_DMA_GOOD_PKT_COUNTERLSW 0x00006800
+/* register address for bitfield tx dma good octet counter lsw [1f:0] */
+#define HW_ATL_STATS_TX_DMA_GOOD_OCTET_COUNTERLSW 0x00008808
+/* register address for bitfield tx dma good packet counter lsw [1f:0] */
+#define HW_ATL_STATS_TX_DMA_GOOD_PKT_COUNTERLSW 0x00008800
+
+/* register address for bitfield rx dma good octet counter msw [3f:20] */
+#define HW_ATL_STATS_RX_DMA_GOOD_OCTET_COUNTERMSW 0x0000680c
+/* register address for bitfield rx dma good packet counter msw [3f:20] */
+#define HW_ATL_STATS_RX_DMA_GOOD_PKT_COUNTERMSW 0x00006804
+/* register address for bitfield tx dma good octet counter msw [3f:20] */
+#define HW_ATL_STATS_TX_DMA_GOOD_OCTET_COUNTERMSW 0x0000880c
+/* register address for bitfield tx dma good packet counter msw [3f:20] */
+#define HW_ATL_STATS_TX_DMA_GOOD_PKT_COUNTERMSW 0x00008804
+
+/* preprocessor definitions for msm rx errors counter register */
+#define HW_ATL_MAC_MSM_RX_ERRS_CNT_ADR 0x00000120u
+
+/* preprocessor definitions for msm rx unicast frames counter register */
+#define HW_ATL_MAC_MSM_RX_UCST_FRM_CNT_ADR 0x000000e0u
+
+/* preprocessor definitions for msm rx multicast frames counter register */
+#define HW_ATL_MAC_MSM_RX_MCST_FRM_CNT_ADR 0x000000e8u
+
+/* preprocessor definitions for msm rx broadcast frames counter register */
+#define HW_ATL_MAC_MSM_RX_BCST_FRM_CNT_ADR 0x000000f0u
+
+/* preprocessor definitions for msm rx broadcast octets counter register 1 */
+#define HW_ATL_MAC_MSM_RX_BCST_OCTETS_COUNTER1_ADR 0x000001b0u
+
+/* preprocessor definitions for msm rx broadcast octets counter register 2 */
+#define HW_ATL_MAC_MSM_RX_BCST_OCTETS_COUNTER2_ADR 0x000001b4u
+
+/* preprocessor definitions for msm rx unicast octets counter register 0 */
+#define HW_ATL_MAC_MSM_RX_UCST_OCTETS_COUNTER0_ADR 0x000001b8u
+
+/* preprocessor definitions for msm tx unicast frames counter register */
+#define HW_ATL_MAC_MSM_TX_UCST_FRM_CNT_ADR 0x00000108u
+
+/* preprocessor definitions for msm tx multicast frames counter register */
+#define HW_ATL_MAC_MSM_TX_MCST_FRM_CNT_ADR 0x00000110u
+
+/* preprocessor definitions for global mif identification */
+#define HW_ATL_GLB_MIF_ID_ADR 0x0000001cu
+
+/* register address for bitfield iamr_lsw[1f:0] */
+#define HW_ATL_ITR_IAMRLSW_ADR 0x00002090
+/* register address for bitfield rx dma drop packet counter [1f:0] */
+#define HW_ATL_RPB_RX_DMA_DROP_PKT_CNT_ADR 0x00006818
+
+/* register address for bitfield imcr_lsw[1f:0] */
+#define HW_ATL_ITR_IMCRLSW_ADR 0x00002070
+/* register address for bitfield imsr_lsw[1f:0] */
+#define HW_ATL_ITR_IMSRLSW_ADR 0x00002060
+/* register address for bitfield itr_reg_res_dsbl */
+#define HW_ATL_ITR_REG_RES_DSBL_ADR 0x00002300
+/* bitmask for bitfield itr_reg_res_dsbl */
+#define HW_ATL_ITR_REG_RES_DSBL_MSK 0x20000000
+/* lower bit position of bitfield itr_reg_res_dsbl */
+#define HW_ATL_ITR_REG_RES_DSBL_SHIFT 29
+/* register address for bitfield iscr_lsw[1f:0] */
+#define HW_ATL_ITR_ISCRLSW_ADR 0x00002050
+/* register address for bitfield isr_lsw[1f:0] */
+#define HW_ATL_ITR_ISRLSW_ADR 0x00002000
+/* register address for bitfield itr_reset */
+#define HW_ATL_ITR_RES_ADR 0x00002300
+/* bitmask for bitfield itr_reset */
+#define HW_ATL_ITR_RES_MSK 0x80000000
+/* lower bit position of bitfield itr_reset */
+#define HW_ATL_ITR_RES_SHIFT 31
+/* register address for bitfield dca{d}_cpuid[7:0] */
+#define HW_ATL_RDM_DCADCPUID_ADR(dca) (0x00006100 + (dca) * 0x4)
+/* bitmask for bitfield dca{d}_cpuid[7:0] */
+#define HW_ATL_RDM_DCADCPUID_MSK 0x000000ff
+/* lower bit position of bitfield dca{d}_cpuid[7:0] */
+#define HW_ATL_RDM_DCADCPUID_SHIFT 0
+/* register address for bitfield dca_en */
+#define HW_ATL_RDM_DCA_EN_ADR 0x00006180
+
+/* rx dca_en bitfield definitions
+ * preprocessor definitions for the bitfield "dca_en".
+ * port="pif_rdm_dca_en_i"
+ */
+
+/* register address for bitfield dca_en */
+#define HW_ATL_RDM_DCA_EN_ADR 0x00006180
+/* bitmask for bitfield dca_en */
+#define HW_ATL_RDM_DCA_EN_MSK 0x80000000
+/* inverted bitmask for bitfield dca_en */
+#define HW_ATL_RDM_DCA_EN_MSKN 0x7fffffff
+/* lower bit position of bitfield dca_en */
+#define HW_ATL_RDM_DCA_EN_SHIFT 31
+/* width of bitfield dca_en */
+#define HW_ATL_RDM_DCA_EN_WIDTH 1
+/* default value of bitfield dca_en */
+#define HW_ATL_RDM_DCA_EN_DEFAULT 0x1
+
+/* rx dca_mode[3:0] bitfield definitions
+ * preprocessor definitions for the bitfield "dca_mode[3:0]".
+ * port="pif_rdm_dca_mode_i[3:0]"
+ */
+
+/* register address for bitfield dca_mode[3:0] */
+#define HW_ATL_RDM_DCA_MODE_ADR 0x00006180
+/* bitmask for bitfield dca_mode[3:0] */
+#define HW_ATL_RDM_DCA_MODE_MSK 0x0000000f
+/* inverted bitmask for bitfield dca_mode[3:0] */
+#define HW_ATL_RDM_DCA_MODE_MSKN 0xfffffff0
+/* lower bit position of bitfield dca_mode[3:0] */
+#define HW_ATL_RDM_DCA_MODE_SHIFT 0
+/* width of bitfield dca_mode[3:0] */
+#define HW_ATL_RDM_DCA_MODE_WIDTH 4
+/* default value of bitfield dca_mode[3:0] */
+#define HW_ATL_RDM_DCA_MODE_DEFAULT 0x0
+
+/* rx desc{d}_data_size[4:0] bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_data_size[4:0]".
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ * port="pif_rdm_desc0_data_size_i[4:0]"
+ */
+
+/* register address for bitfield desc{d}_data_size[4:0] */
+#define HW_ATL_RDM_DESCDDATA_SIZE_ADR(descriptor) \
+	(0x00005b18 + (descriptor) * 0x20)
+/* bitmask for bitfield desc{d}_data_size[4:0] */
+#define HW_ATL_RDM_DESCDDATA_SIZE_MSK 0x0000001f
+/* inverted bitmask for bitfield desc{d}_data_size[4:0] */
+#define HW_ATL_RDM_DESCDDATA_SIZE_MSKN 0xffffffe0
+/* lower bit position of bitfield desc{d}_data_size[4:0] */
+#define HW_ATL_RDM_DESCDDATA_SIZE_SHIFT 0
+/* width of bitfield desc{d}_data_size[4:0] */
+#define HW_ATL_RDM_DESCDDATA_SIZE_WIDTH 5
+/* default value of bitfield desc{d}_data_size[4:0] */
+#define HW_ATL_RDM_DESCDDATA_SIZE_DEFAULT 0x0
+
+/* rx dca{d}_desc_en bitfield definitions
+ * preprocessor definitions for the bitfield "dca{d}_desc_en".
+ * parameter: dca {d} | stride size 0x4 | range [0, 31]
+ * port="pif_rdm_dca_desc_en_i[0]"
+ */
+
+/* register address for bitfield dca{d}_desc_en */
+#define HW_ATL_RDM_DCADDESC_EN_ADR(dca) (0x00006100 + (dca) * 0x4)
+/* bitmask for bitfield dca{d}_desc_en */
+#define HW_ATL_RDM_DCADDESC_EN_MSK 0x80000000
+/* inverted bitmask for bitfield dca{d}_desc_en */
+#define HW_ATL_RDM_DCADDESC_EN_MSKN 0x7fffffff
+/* lower bit position of bitfield dca{d}_desc_en */
+#define HW_ATL_RDM_DCADDESC_EN_SHIFT 31
+/* width of bitfield dca{d}_desc_en */
+#define HW_ATL_RDM_DCADDESC_EN_WIDTH 1
+/* default value of bitfield dca{d}_desc_en */
+#define HW_ATL_RDM_DCADDESC_EN_DEFAULT 0x0
+
+/* rx desc{d}_en bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_en".
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ * port="pif_rdm_desc_en_i[0]"
+ */
+
+/* register address for bitfield desc{d}_en */
+#define HW_ATL_RDM_DESCDEN_ADR(descriptor) (0x00005b08 + (descriptor) * 0x20)
+/* bitmask for bitfield desc{d}_en */
+#define HW_ATL_RDM_DESCDEN_MSK 0x80000000
+/* inverted bitmask for bitfield desc{d}_en */
+#define HW_ATL_RDM_DESCDEN_MSKN 0x7fffffff
+/* lower bit position of bitfield desc{d}_en */
+#define HW_ATL_RDM_DESCDEN_SHIFT 31
+/* width of bitfield desc{d}_en */
+#define HW_ATL_RDM_DESCDEN_WIDTH 1
+/* default value of bitfield desc{d}_en */
+#define HW_ATL_RDM_DESCDEN_DEFAULT 0x0
+
+/* rx desc{d}_hdr_size[4:0] bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_hdr_size[4:0]".
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ * port="pif_rdm_desc0_hdr_size_i[4:0]"
+ */
+
+/* register address for bitfield desc{d}_hdr_size[4:0] */
+#define HW_ATL_RDM_DESCDHDR_SIZE_ADR(descriptor) \
+	(0x00005b18 + (descriptor) * 0x20)
+/* bitmask for bitfield desc{d}_hdr_size[4:0] */
+#define HW_ATL_RDM_DESCDHDR_SIZE_MSK 0x00001f00
+/* inverted bitmask for bitfield desc{d}_hdr_size[4:0] */
+#define HW_ATL_RDM_DESCDHDR_SIZE_MSKN 0xffffe0ff
+/* lower bit position of bitfield desc{d}_hdr_size[4:0] */
+#define HW_ATL_RDM_DESCDHDR_SIZE_SHIFT 8
+/* width of bitfield desc{d}_hdr_size[4:0] */
+#define HW_ATL_RDM_DESCDHDR_SIZE_WIDTH 5
+/* default value of bitfield desc{d}_hdr_size[4:0] */
+#define HW_ATL_RDM_DESCDHDR_SIZE_DEFAULT 0x0
+
+/* rx desc{d}_hdr_split bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_hdr_split".
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ * port="pif_rdm_desc_hdr_split_i[0]"
+ */
+
+/* register address for bitfield desc{d}_hdr_split */
+#define HW_ATL_RDM_DESCDHDR_SPLIT_ADR(descriptor) \
+	(0x00005b08 + (descriptor) * 0x20)
+/* bitmask for bitfield desc{d}_hdr_split */
+#define HW_ATL_RDM_DESCDHDR_SPLIT_MSK 0x10000000
+/* inverted bitmask for bitfield desc{d}_hdr_split */
+#define HW_ATL_RDM_DESCDHDR_SPLIT_MSKN 0xefffffff
+/* lower bit position of bitfield desc{d}_hdr_split */
+#define HW_ATL_RDM_DESCDHDR_SPLIT_SHIFT 28
+/* width of bitfield desc{d}_hdr_split */
+#define HW_ATL_RDM_DESCDHDR_SPLIT_WIDTH 1
+/* default value of bitfield desc{d}_hdr_split */
+#define HW_ATL_RDM_DESCDHDR_SPLIT_DEFAULT 0x0
+
+/* rx desc{d}_hd[c:0] bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_hd[c:0]".
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ * port="rdm_pif_desc0_hd_o[12:0]"
+ */
+
+/* register address for bitfield desc{d}_hd[c:0] */
+#define HW_ATL_RDM_DESCDHD_ADR(descriptor) (0x00005b0c + (descriptor) * 0x20)
+/* bitmask for bitfield desc{d}_hd[c:0] */
+#define HW_ATL_RDM_DESCDHD_MSK 0x00001fff
+/* inverted bitmask for bitfield desc{d}_hd[c:0] */
+#define HW_ATL_RDM_DESCDHD_MSKN 0xffffe000
+/* lower bit position of bitfield desc{d}_hd[c:0] */
+#define HW_ATL_RDM_DESCDHD_SHIFT 0
+/* width of bitfield desc{d}_hd[c:0] */
+#define HW_ATL_RDM_DESCDHD_WIDTH 13
+
+/* rx desc{d}_len[9:0] bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_len[9:0]".
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ * port="pif_rdm_desc0_len_i[9:0]"
+ */
+
+/* register address for bitfield desc{d}_len[9:0] */
+#define HW_ATL_RDM_DESCDLEN_ADR(descriptor) (0x00005b08 + (descriptor) * 0x20)
+/* bitmask for bitfield desc{d}_len[9:0] */
+#define HW_ATL_RDM_DESCDLEN_MSK 0x00001ff8
+/* inverted bitmask for bitfield desc{d}_len[9:0] */
+#define HW_ATL_RDM_DESCDLEN_MSKN 0xffffe007
+/* lower bit position of bitfield desc{d}_len[9:0] */
+#define HW_ATL_RDM_DESCDLEN_SHIFT 3
+/* width of bitfield desc{d}_len[9:0] */
+#define HW_ATL_RDM_DESCDLEN_WIDTH 10
+/* default value of bitfield desc{d}_len[9:0] */
+#define HW_ATL_RDM_DESCDLEN_DEFAULT 0x0
+
+/* rx desc{d}_reset bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_reset".
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ * port="pif_rdm_q_pf_res_i[0]"
+ */
+
+/* register address for bitfield desc{d}_reset */
+#define HW_ATL_RDM_DESCDRESET_ADR(descriptor) (0x00005b08 + (descriptor) * 0x20)
+/* bitmask for bitfield desc{d}_reset */
+#define HW_ATL_RDM_DESCDRESET_MSK 0x02000000
+/* inverted bitmask for bitfield desc{d}_reset */
+#define HW_ATL_RDM_DESCDRESET_MSKN 0xfdffffff
+/* lower bit position of bitfield desc{d}_reset */
+#define HW_ATL_RDM_DESCDRESET_SHIFT 25
+/* width of bitfield desc{d}_reset */
+#define HW_ATL_RDM_DESCDRESET_WIDTH 1
+/* default value of bitfield desc{d}_reset */
+#define HW_ATL_RDM_DESCDRESET_DEFAULT 0x0
+
+/* rx int_desc_wrb_en bitfield definitions
+ * preprocessor definitions for the bitfield "int_desc_wrb_en".
+ * port="pif_rdm_int_desc_wrb_en_i"
+ */
+
+/* register address for bitfield int_desc_wrb_en */
+#define HW_ATL_RDM_INT_DESC_WRB_EN_ADR 0x00005a30
+/* bitmask for bitfield int_desc_wrb_en */
+#define HW_ATL_RDM_INT_DESC_WRB_EN_MSK 0x00000004
+/* inverted bitmask for bitfield int_desc_wrb_en */
+#define HW_ATL_RDM_INT_DESC_WRB_EN_MSKN 0xfffffffb
+/* lower bit position of bitfield int_desc_wrb_en */
+#define HW_ATL_RDM_INT_DESC_WRB_EN_SHIFT 2
+/* width of bitfield int_desc_wrb_en */
+#define HW_ATL_RDM_INT_DESC_WRB_EN_WIDTH 1
+/* default value of bitfield int_desc_wrb_en */
+#define HW_ATL_RDM_INT_DESC_WRB_EN_DEFAULT 0x0
+
+/* rx dca{d}_hdr_en bitfield definitions
+ * preprocessor definitions for the bitfield "dca{d}_hdr_en".
+ * parameter: dca {d} | stride size 0x4 | range [0, 31]
+ * port="pif_rdm_dca_hdr_en_i[0]"
+ */
+
+/* register address for bitfield dca{d}_hdr_en */
+#define HW_ATL_RDM_DCADHDR_EN_ADR(dca) (0x00006100 + (dca) * 0x4)
+/* bitmask for bitfield dca{d}_hdr_en */
+#define HW_ATL_RDM_DCADHDR_EN_MSK 0x40000000
+/* inverted bitmask for bitfield dca{d}_hdr_en */
+#define HW_ATL_RDM_DCADHDR_EN_MSKN 0xbfffffff
+/* lower bit position of bitfield dca{d}_hdr_en */
+#define HW_ATL_RDM_DCADHDR_EN_SHIFT 30
+/* width of bitfield dca{d}_hdr_en */
+#define HW_ATL_RDM_DCADHDR_EN_WIDTH 1
+/* default value of bitfield dca{d}_hdr_en */
+#define HW_ATL_RDM_DCADHDR_EN_DEFAULT 0x0
+
+/* rx dca{d}_pay_en bitfield definitions
+ * preprocessor definitions for the bitfield "dca{d}_pay_en".
+ * parameter: dca {d} | stride size 0x4 | range [0, 31]
+ * port="pif_rdm_dca_pay_en_i[0]"
+ */
+
+/* register address for bitfield dca{d}_pay_en */
+#define HW_ATL_RDM_DCADPAY_EN_ADR(dca) (0x00006100 + (dca) * 0x4)
+/* bitmask for bitfield dca{d}_pay_en */
+#define HW_ATL_RDM_DCADPAY_EN_MSK 0x20000000
+/* inverted bitmask for bitfield dca{d}_pay_en */
+#define HW_ATL_RDM_DCADPAY_EN_MSKN 0xdfffffff
+/* lower bit position of bitfield dca{d}_pay_en */
+#define HW_ATL_RDM_DCADPAY_EN_SHIFT 29
+/* width of bitfield dca{d}_pay_en */
+#define HW_ATL_RDM_DCADPAY_EN_WIDTH 1
+/* default value of bitfield dca{d}_pay_en */
+#define HW_ATL_RDM_DCADPAY_EN_DEFAULT 0x0
+
+/* RX rdm_int_rim_en Bitfield Definitions
+ * Preprocessor definitions for the bitfield "rdm_int_rim_en".
+ * PORT="pif_rdm_int_rim_en_i"
+ */
+
+/* Register address for bitfield rdm_int_rim_en */
+#define HW_ATL_RDM_INT_RIM_EN_ADR 0x00005A30
+/* Bitmask for bitfield rdm_int_rim_en */
+#define HW_ATL_RDM_INT_RIM_EN_MSK 0x00000008
+/* Inverted bitmask for bitfield rdm_int_rim_en */
+#define HW_ATL_RDM_INT_RIM_EN_MSKN 0xFFFFFFF7
+/* Lower bit position of bitfield rdm_int_rim_en */
+#define HW_ATL_RDM_INT_RIM_EN_SHIFT 3
+/* Width of bitfield rdm_int_rim_en */
+#define HW_ATL_RDM_INT_RIM_EN_WIDTH 1
+/* Default value of bitfield rdm_int_rim_en */
+#define HW_ATL_RDM_INT_RIM_EN_DEFAULT 0x0
+
+/* general interrupt mapping register definitions
+ * preprocessor definitions for general interrupt mapping register
+ * base address: 0x00002180
+ * parameter: regidx {f} | stride size 0x4 | range [0, 3]
+ */
+#define HW_ATL_GEN_INTR_MAP_ADR(regidx) (0x00002180u + (regidx) * 0x4)
+
+/* general interrupt status register definitions
+ * preprocessor definitions for general interrupt status register
+ * address: 0x000021A0
+ */
+
+#define HW_ATL_GEN_INTR_STAT_ADR 0x000021A4U
+
+/* interrupt global control register  definitions
+ * preprocessor definitions for interrupt global control register
+ * address: 0x00002300
+ */
+#define HW_ATL_INTR_GLB_CTL_ADR 0x00002300u
+
+/* interrupt throttle register definitions
+ * preprocessor definitions for interrupt throttle register
+ * base address: 0x00002800
+ * parameter: throttle {t} | stride size 0x4 | range [0, 31]
+ */
+#define HW_ATL_INTR_THR_ADR(throttle) (0x00002800u + (throttle) * 0x4)
+
+/* rx dma descriptor base address lsw definitions
+ * preprocessor definitions for rx dma descriptor base address lsw
+ * base address: 0x00005b00
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ */
+#define HW_ATL_RX_DMA_DESC_BASE_ADDRLSW_ADR(descriptor) \
+(0x00005b00u + (descriptor) * 0x20)
+
+/* rx dma descriptor base address msw definitions
+ * preprocessor definitions for rx dma descriptor base address msw
+ * base address: 0x00005b04
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ */
+#define HW_ATL_RX_DMA_DESC_BASE_ADDRMSW_ADR(descriptor) \
+(0x00005b04u + (descriptor) * 0x20)
+
+/* rx dma descriptor status register definitions
+ * preprocessor definitions for rx dma descriptor status register
+ * base address: 0x00005b14
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ */
+#define HW_ATL_RX_DMA_DESC_STAT_ADR(descriptor) \
+	(0x00005b14u + (descriptor) * 0x20)
+
+/* rx dma descriptor tail pointer register definitions
+ * preprocessor definitions for rx dma descriptor tail pointer register
+ * base address: 0x00005b10
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ */
+#define HW_ATL_RX_DMA_DESC_TAIL_PTR_ADR(descriptor) \
+	(0x00005b10u + (descriptor) * 0x20)
+
+/* rx interrupt moderation control register definitions
+ * Preprocessor definitions for RX Interrupt Moderation Control Register
+ * Base Address: 0x00005A40
+ * Parameter: RIM {R} | stride size 0x4 | range [0, 31]
+ */
+#define HW_ATL_RX_INTR_MODERATION_CTL_ADR(rim) (0x00005A40u + (rim) * 0x4)
+
+/* rx filter multicast filter mask register definitions
+ * preprocessor definitions for rx filter multicast filter mask register
+ * address: 0x00005270
+ */
+#define HW_ATL_RX_FLR_MCST_FLR_MSK_ADR 0x00005270u
+
+/* rx filter multicast filter register definitions
+ * preprocessor definitions for rx filter multicast filter register
+ * base address: 0x00005250
+ * parameter: filter {f} | stride size 0x4 | range [0, 7]
+ */
+#define HW_ATL_RX_FLR_MCST_FLR_ADR(filter) (0x00005250u + (filter) * 0x4)
+
+/* RX Filter RSS Control Register 1 Definitions
+ * Preprocessor definitions for RX Filter RSS Control Register 1
+ * Address: 0x000054C0
+ */
+#define HW_ATL_RX_FLR_RSS_CONTROL1_ADR 0x000054C0u
+
+/* RX Filter Control Register 2 Definitions
+ * Preprocessor definitions for RX Filter Control Register 2
+ * Address: 0x00005104
+ */
+#define HW_ATL_RX_FLR_CONTROL2_ADR 0x00005104u
+
+/* tx tx dma debug control [1f:0] bitfield definitions
+ * preprocessor definitions for the bitfield "tx dma debug control [1f:0]".
+ * port="pif_tdm_debug_cntl_i[31:0]"
+ */
+
+/* register address for bitfield tx dma debug control [1f:0] */
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_ADR 0x00008920
+/* bitmask for bitfield tx dma debug control [1f:0] */
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_MSK 0xffffffff
+/* inverted bitmask for bitfield tx dma debug control [1f:0] */
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_MSKN 0x00000000
+/* lower bit position of bitfield tx dma debug control [1f:0] */
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_SHIFT 0
+/* width of bitfield tx dma debug control [1f:0] */
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_WIDTH 32
+/* default value of bitfield tx dma debug control [1f:0] */
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_DEFAULT 0x0
+
+/* tx dma descriptor base address lsw definitions
+ * preprocessor definitions for tx dma descriptor base address lsw
+ * base address: 0x00007c00
+ * parameter: descriptor {d} | stride size 0x40 | range [0, 31]
+ */
+#define HW_ATL_TX_DMA_DESC_BASE_ADDRLSW_ADR(descriptor) \
+	(0x00007c00u + (descriptor) * 0x40)
+
+/* tx dma descriptor tail pointer register definitions
+ * preprocessor definitions for tx dma descriptor tail pointer register
+ * base address: 0x00007c10
+ *  parameter: descriptor {d} | stride size 0x40 | range [0, 31]
+ */
+#define HW_ATL_TX_DMA_DESC_TAIL_PTR_ADR(descriptor) \
+	(0x00007c10u + (descriptor) * 0x40)
+
+/* rx dma_sys_loopback bitfield definitions
+ * preprocessor definitions for the bitfield "dma_sys_loopback".
+ * port="pif_rpb_dma_sys_lbk_i"
+ */
+
+/* register address for bitfield dma_sys_loopback */
+#define HW_ATL_RPB_DMA_SYS_LBK_ADR 0x00005000
+/* bitmask for bitfield dma_sys_loopback */
+#define HW_ATL_RPB_DMA_SYS_LBK_MSK 0x00000040
+/* inverted bitmask for bitfield dma_sys_loopback */
+#define HW_ATL_RPB_DMA_SYS_LBK_MSKN 0xffffffbf
+/* lower bit position of bitfield dma_sys_loopback */
+#define HW_ATL_RPB_DMA_SYS_LBK_SHIFT 6
+/* width of bitfield dma_sys_loopback */
+#define HW_ATL_RPB_DMA_SYS_LBK_WIDTH 1
+/* default value of bitfield dma_sys_loopback */
+#define HW_ATL_RPB_DMA_SYS_LBK_DEFAULT 0x0
+
+/* rx rx_tc_mode bitfield definitions
+ * preprocessor definitions for the bitfield "rx_tc_mode".
+ * port="pif_rpb_rx_tc_mode_i,pif_rpf_rx_tc_mode_i"
+ */
+
+/* register address for bitfield rx_tc_mode */
+#define HW_ATL_RPB_RPF_RX_TC_MODE_ADR 0x00005700
+/* bitmask for bitfield rx_tc_mode */
+#define HW_ATL_RPB_RPF_RX_TC_MODE_MSK 0x00000100
+/* inverted bitmask for bitfield rx_tc_mode */
+#define HW_ATL_RPB_RPF_RX_TC_MODE_MSKN 0xfffffeff
+/* lower bit position of bitfield rx_tc_mode */
+#define HW_ATL_RPB_RPF_RX_TC_MODE_SHIFT 8
+/* width of bitfield rx_tc_mode */
+#define HW_ATL_RPB_RPF_RX_TC_MODE_WIDTH 1
+/* default value of bitfield rx_tc_mode */
+#define HW_ATL_RPB_RPF_RX_TC_MODE_DEFAULT 0x0
+
+/* rx rx_buf_en bitfield definitions
+ * preprocessor definitions for the bitfield "rx_buf_en".
+ * port="pif_rpb_rx_buf_en_i"
+ */
+
+/* register address for bitfield rx_buf_en */
+#define HW_ATL_RPB_RX_BUF_EN_ADR 0x00005700
+/* bitmask for bitfield rx_buf_en */
+#define HW_ATL_RPB_RX_BUF_EN_MSK 0x00000001
+/* inverted bitmask for bitfield rx_buf_en */
+#define HW_ATL_RPB_RX_BUF_EN_MSKN 0xfffffffe
+/* lower bit position of bitfield rx_buf_en */
+#define HW_ATL_RPB_RX_BUF_EN_SHIFT 0
+/* width of bitfield rx_buf_en */
+#define HW_ATL_RPB_RX_BUF_EN_WIDTH 1
+/* default value of bitfield rx_buf_en */
+#define HW_ATL_RPB_RX_BUF_EN_DEFAULT 0x0
+
+/* rx rx{b}_hi_thresh[d:0] bitfield definitions
+ * preprocessor definitions for the bitfield "rx{b}_hi_thresh[d:0]".
+ * parameter: buffer {b} | stride size 0x10 | range [0, 7]
+ * port="pif_rpb_rx0_hi_thresh_i[13:0]"
+ */
+
+/* register address for bitfield rx{b}_hi_thresh[d:0] */
+#define HW_ATL_RPB_RXBHI_THRESH_ADR(buffer) (0x00005714 + (buffer) * 0x10)
+/* bitmask for bitfield rx{b}_hi_thresh[d:0] */
+#define HW_ATL_RPB_RXBHI_THRESH_MSK 0x3fff0000
+/* inverted bitmask for bitfield rx{b}_hi_thresh[d:0] */
+#define HW_ATL_RPB_RXBHI_THRESH_MSKN 0xc000ffff
+/* lower bit position of bitfield rx{b}_hi_thresh[d:0] */
+#define HW_ATL_RPB_RXBHI_THRESH_SHIFT 16
+/* width of bitfield rx{b}_hi_thresh[d:0] */
+#define HW_ATL_RPB_RXBHI_THRESH_WIDTH 14
+/* default value of bitfield rx{b}_hi_thresh[d:0] */
+#define HW_ATL_RPB_RXBHI_THRESH_DEFAULT 0x0
+
+/* rx rx{b}_lo_thresh[d:0] bitfield definitions
+ * preprocessor definitions for the bitfield "rx{b}_lo_thresh[d:0]".
+ * parameter: buffer {b} | stride size 0x10 | range [0, 7]
+ * port="pif_rpb_rx0_lo_thresh_i[13:0]"
+ */
+
+/* register address for bitfield rx{b}_lo_thresh[d:0] */
+#define HW_ATL_RPB_RXBLO_THRESH_ADR(buffer) (0x00005714 + (buffer) * 0x10)
+/* bitmask for bitfield rx{b}_lo_thresh[d:0] */
+#define HW_ATL_RPB_RXBLO_THRESH_MSK 0x00003fff
+/* inverted bitmask for bitfield rx{b}_lo_thresh[d:0] */
+#define HW_ATL_RPB_RXBLO_THRESH_MSKN 0xffffc000
+/* lower bit position of bitfield rx{b}_lo_thresh[d:0] */
+#define HW_ATL_RPB_RXBLO_THRESH_SHIFT 0
+/* width of bitfield rx{b}_lo_thresh[d:0] */
+#define HW_ATL_RPB_RXBLO_THRESH_WIDTH 14
+/* default value of bitfield rx{b}_lo_thresh[d:0] */
+#define HW_ATL_RPB_RXBLO_THRESH_DEFAULT 0x0
+
+/* rx rx_fc_mode[1:0] bitfield definitions
+ * preprocessor definitions for the bitfield "rx_fc_mode[1:0]".
+ * port="pif_rpb_rx_fc_mode_i[1:0]"
+ */
+
+/* register address for bitfield rx_fc_mode[1:0] */
+#define HW_ATL_RPB_RX_FC_MODE_ADR 0x00005700
+/* bitmask for bitfield rx_fc_mode[1:0] */
+#define HW_ATL_RPB_RX_FC_MODE_MSK 0x00000030
+/* inverted bitmask for bitfield rx_fc_mode[1:0] */
+#define HW_ATL_RPB_RX_FC_MODE_MSKN 0xffffffcf
+/* lower bit position of bitfield rx_fc_mode[1:0] */
+#define HW_ATL_RPB_RX_FC_MODE_SHIFT 4
+/* width of bitfield rx_fc_mode[1:0] */
+#define HW_ATL_RPB_RX_FC_MODE_WIDTH 2
+/* default value of bitfield rx_fc_mode[1:0] */
+#define HW_ATL_RPB_RX_FC_MODE_DEFAULT 0x0
+
+/* rx rx{b}_buf_size[8:0] bitfield definitions
+ * preprocessor definitions for the bitfield "rx{b}_buf_size[8:0]".
+ * parameter: buffer {b} | stride size 0x10 | range [0, 7]
+ * port="pif_rpb_rx0_buf_size_i[8:0]"
+ */
+
+/* register address for bitfield rx{b}_buf_size[8:0] */
+#define HW_ATL_RPB_RXBBUF_SIZE_ADR(buffer) (0x00005710 + (buffer) * 0x10)
+/* bitmask for bitfield rx{b}_buf_size[8:0] */
+#define HW_ATL_RPB_RXBBUF_SIZE_MSK 0x000001ff
+/* inverted bitmask for bitfield rx{b}_buf_size[8:0] */
+#define HW_ATL_RPB_RXBBUF_SIZE_MSKN 0xfffffe00
+/* lower bit position of bitfield rx{b}_buf_size[8:0] */
+#define HW_ATL_RPB_RXBBUF_SIZE_SHIFT 0
+/* width of bitfield rx{b}_buf_size[8:0] */
+#define HW_ATL_RPB_RXBBUF_SIZE_WIDTH 9
+/* default value of bitfield rx{b}_buf_size[8:0] */
+#define HW_ATL_RPB_RXBBUF_SIZE_DEFAULT 0x0
+
+/* rx rx{b}_xoff_en bitfield definitions
+ * preprocessor definitions for the bitfield "rx{b}_xoff_en".
+ * parameter: buffer {b} | stride size 0x10 | range [0, 7]
+ * port="pif_rpb_rx_xoff_en_i[0]"
+ */
+
+/* register address for bitfield rx{b}_xoff_en */
+#define HW_ATL_RPB_RXBXOFF_EN_ADR(buffer) (0x00005714 + (buffer) * 0x10)
+/* bitmask for bitfield rx{b}_xoff_en */
+#define HW_ATL_RPB_RXBXOFF_EN_MSK 0x80000000
+/* inverted bitmask for bitfield rx{b}_xoff_en */
+#define HW_ATL_RPB_RXBXOFF_EN_MSKN 0x7fffffff
+/* lower bit position of bitfield rx{b}_xoff_en */
+#define HW_ATL_RPB_RXBXOFF_EN_SHIFT 31
+/* width of bitfield rx{b}_xoff_en */
+#define HW_ATL_RPB_RXBXOFF_EN_WIDTH 1
+/* default value of bitfield rx{b}_xoff_en */
+#define HW_ATL_RPB_RXBXOFF_EN_DEFAULT 0x0
+
+/* rx l2_bc_thresh[f:0] bitfield definitions
+ * preprocessor definitions for the bitfield "l2_bc_thresh[f:0]".
+ * port="pif_rpf_l2_bc_thresh_i[15:0]"
+ */
+
+/* register address for bitfield l2_bc_thresh[f:0] */
+#define HW_ATL_RPFL2BC_THRESH_ADR 0x00005100
+/* bitmask for bitfield l2_bc_thresh[f:0] */
+#define HW_ATL_RPFL2BC_THRESH_MSK 0xffff0000
+/* inverted bitmask for bitfield l2_bc_thresh[f:0] */
+#define HW_ATL_RPFL2BC_THRESH_MSKN 0x0000ffff
+/* lower bit position of bitfield l2_bc_thresh[f:0] */
+#define HW_ATL_RPFL2BC_THRESH_SHIFT 16
+/* width of bitfield l2_bc_thresh[f:0] */
+#define HW_ATL_RPFL2BC_THRESH_WIDTH 16
+/* default value of bitfield l2_bc_thresh[f:0] */
+#define HW_ATL_RPFL2BC_THRESH_DEFAULT 0x0
+
+/* rx l2_bc_en bitfield definitions
+ * preprocessor definitions for the bitfield "l2_bc_en".
+ * port="pif_rpf_l2_bc_en_i"
+ */
+
+/* register address for bitfield l2_bc_en */
+#define HW_ATL_RPFL2BC_EN_ADR 0x00005100
+/* bitmask for bitfield l2_bc_en */
+#define HW_ATL_RPFL2BC_EN_MSK 0x00000001
+/* inverted bitmask for bitfield l2_bc_en */
+#define HW_ATL_RPFL2BC_EN_MSKN 0xfffffffe
+/* lower bit position of bitfield l2_bc_en */
+#define HW_ATL_RPFL2BC_EN_SHIFT 0
+/* width of bitfield l2_bc_en */
+#define HW_ATL_RPFL2BC_EN_WIDTH 1
+/* default value of bitfield l2_bc_en */
+#define HW_ATL_RPFL2BC_EN_DEFAULT 0x0
+
+/* rx l2_bc_act[2:0] bitfield definitions
+ * preprocessor definitions for the bitfield "l2_bc_act[2:0]".
+ * port="pif_rpf_l2_bc_act_i[2:0]"
+ */
+
+/* register address for bitfield l2_bc_act[2:0] */
+#define HW_ATL_RPFL2BC_ACT_ADR 0x00005100
+/* bitmask for bitfield l2_bc_act[2:0] */
+#define HW_ATL_RPFL2BC_ACT_MSK 0x00007000
+/* inverted bitmask for bitfield l2_bc_act[2:0] */
+#define HW_ATL_RPFL2BC_ACT_MSKN 0xffff8fff
+/* lower bit position of bitfield l2_bc_act[2:0] */
+#define HW_ATL_RPFL2BC_ACT_SHIFT 12
+/* width of bitfield l2_bc_act[2:0] */
+#define HW_ATL_RPFL2BC_ACT_WIDTH 3
+/* default value of bitfield l2_bc_act[2:0] */
+#define HW_ATL_RPFL2BC_ACT_DEFAULT 0x0
+
+/* rx l2_mc_en{f} bitfield definitions
+ * preprocessor definitions for the bitfield "l2_mc_en{f}".
+ * parameter: filter {f} | stride size 0x4 | range [0, 7]
+ * port="pif_rpf_l2_mc_en_i[0]"
+ */
+
+/* register address for bitfield l2_mc_en{f} */
+#define HW_ATL_RPFL2MC_ENF_ADR(filter) (0x00005250 + (filter) * 0x4)
+/* bitmask for bitfield l2_mc_en{f} */
+#define HW_ATL_RPFL2MC_ENF_MSK 0x80000000
+/* inverted bitmask for bitfield l2_mc_en{f} */
+#define HW_ATL_RPFL2MC_ENF_MSKN 0x7fffffff
+/* lower bit position of bitfield l2_mc_en{f} */
+#define HW_ATL_RPFL2MC_ENF_SHIFT 31
+/* width of bitfield l2_mc_en{f} */
+#define HW_ATL_RPFL2MC_ENF_WIDTH 1
+/* default value of bitfield l2_mc_en{f} */
+#define HW_ATL_RPFL2MC_ENF_DEFAULT 0x0
+
+/* rx l2_promis_mode bitfield definitions
+ * preprocessor definitions for the bitfield "l2_promis_mode".
+ * port="pif_rpf_l2_promis_mode_i"
+ */
+
+/* register address for bitfield l2_promis_mode */
+#define HW_ATL_RPFL2PROMIS_MODE_ADR 0x00005100
+/* bitmask for bitfield l2_promis_mode */
+#define HW_ATL_RPFL2PROMIS_MODE_MSK 0x00000008
+/* inverted bitmask for bitfield l2_promis_mode */
+#define HW_ATL_RPFL2PROMIS_MODE_MSKN 0xfffffff7
+/* lower bit position of bitfield l2_promis_mode */
+#define HW_ATL_RPFL2PROMIS_MODE_SHIFT 3
+/* width of bitfield l2_promis_mode */
+#define HW_ATL_RPFL2PROMIS_MODE_WIDTH 1
+/* default value of bitfield l2_promis_mode */
+#define HW_ATL_RPFL2PROMIS_MODE_DEFAULT 0x0
+
+/* rx l2_uc_act{f}[2:0] bitfield definitions
+ * preprocessor definitions for the bitfield "l2_uc_act{f}[2:0]".
+ * parameter: filter {f} | stride size 0x8 | range [0, 37]
+ * port="pif_rpf_l2_uc_act0_i[2:0]"
+ */
+
+/* register address for bitfield l2_uc_act{f}[2:0] */
+#define HW_ATL_RPFL2UC_ACTF_ADR(filter) (0x00005114 + (filter) * 0x8)
+/* bitmask for bitfield l2_uc_act{f}[2:0] */
+#define HW_ATL_RPFL2UC_ACTF_MSK 0x00070000
+/* inverted bitmask for bitfield l2_uc_act{f}[2:0] */
+#define HW_ATL_RPFL2UC_ACTF_MSKN 0xfff8ffff
+/* lower bit position of bitfield l2_uc_act{f}[2:0] */
+#define HW_ATL_RPFL2UC_ACTF_SHIFT 16
+/* width of bitfield l2_uc_act{f}[2:0] */
+#define HW_ATL_RPFL2UC_ACTF_WIDTH 3
+/* default value of bitfield l2_uc_act{f}[2:0] */
+#define HW_ATL_RPFL2UC_ACTF_DEFAULT 0x0
+
+/* rx l2_uc_en{f} bitfield definitions
+ * preprocessor definitions for the bitfield "l2_uc_en{f}".
+ * parameter: filter {f} | stride size 0x8 | range [0, 37]
+ * port="pif_rpf_l2_uc_en_i[0]"
+ */
+
+/* register address for bitfield l2_uc_en{f} */
+#define HW_ATL_RPFL2UC_ENF_ADR(filter) (0x00005114 + (filter) * 0x8)
+/* bitmask for bitfield l2_uc_en{f} */
+#define HW_ATL_RPFL2UC_ENF_MSK 0x80000000
+/* inverted bitmask for bitfield l2_uc_en{f} */
+#define HW_ATL_RPFL2UC_ENF_MSKN 0x7fffffff
+/* lower bit position of bitfield l2_uc_en{f} */
+#define HW_ATL_RPFL2UC_ENF_SHIFT 31
+/* width of bitfield l2_uc_en{f} */
+#define HW_ATL_RPFL2UC_ENF_WIDTH 1
+/* default value of bitfield l2_uc_en{f} */
+#define HW_ATL_RPFL2UC_ENF_DEFAULT 0x0
+
+/* register address for bitfield l2_uc_da{f}_lsw[1f:0] */
+#define HW_ATL_RPFL2UC_DAFLSW_ADR(filter) (0x00005110 + (filter) * 0x8)
+/* register address for bitfield l2_uc_da{f}_msw[f:0] */
+#define HW_ATL_RPFL2UC_DAFMSW_ADR(filter) (0x00005114 + (filter) * 0x8)
+/* bitmask for bitfield l2_uc_da{f}_msw[f:0] */
+#define HW_ATL_RPFL2UC_DAFMSW_MSK 0x0000ffff
+/* lower bit position of bitfield l2_uc_da{f}_msw[f:0] */
+#define HW_ATL_RPFL2UC_DAFMSW_SHIFT 0
+
+/* rx l2_mc_accept_all bitfield definitions
+ * Preprocessor definitions for the bitfield "l2_mc_accept_all".
+ * PORT="pif_rpf_l2_mc_all_accept_i"
+ */
+
+/* Register address for bitfield l2_mc_accept_all */
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_ADR 0x00005270
+/* Bitmask for bitfield l2_mc_accept_all */
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_MSK 0x00004000
+/* Inverted bitmask for bitfield l2_mc_accept_all */
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_MSKN 0xFFFFBFFF
+/* Lower bit position of bitfield l2_mc_accept_all */
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_SHIFT 14
+/* Width of bitfield l2_mc_accept_all */
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_WIDTH 1
+/* Default value of bitfield l2_mc_accept_all */
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_DEFAULT 0x0
+
+/* width of bitfield rx_tc_up{t}[2:0] */
+#define HW_ATL_RPF_RPB_RX_TC_UPT_WIDTH 3
+/* default value of bitfield rx_tc_up{t}[2:0] */
+#define HW_ATL_RPF_RPB_RX_TC_UPT_DEFAULT 0x0
+
+/* rx rss_key_addr[4:0] bitfield definitions
+ * preprocessor definitions for the bitfield "rss_key_addr[4:0]".
+ * port="pif_rpf_rss_key_addr_i[4:0]"
+ */
+
+/* register address for bitfield rss_key_addr[4:0] */
+#define HW_ATL_RPF_RSS_KEY_ADDR_ADR 0x000054d0
+/* bitmask for bitfield rss_key_addr[4:0] */
+#define HW_ATL_RPF_RSS_KEY_ADDR_MSK 0x0000001f
+/* inverted bitmask for bitfield rss_key_addr[4:0] */
+#define HW_ATL_RPF_RSS_KEY_ADDR_MSKN 0xffffffe0
+/* lower bit position of bitfield rss_key_addr[4:0] */
+#define HW_ATL_RPF_RSS_KEY_ADDR_SHIFT 0
+/* width of bitfield rss_key_addr[4:0] */
+#define HW_ATL_RPF_RSS_KEY_ADDR_WIDTH 5
+/* default value of bitfield rss_key_addr[4:0] */
+#define HW_ATL_RPF_RSS_KEY_ADDR_DEFAULT 0x0
+
+/* rx rss_key_wr_data[1f:0] bitfield definitions
+ * preprocessor definitions for the bitfield "rss_key_wr_data[1f:0]".
+ * port="pif_rpf_rss_key_wr_data_i[31:0]"
+ */
+
+/* register address for bitfield rss_key_wr_data[1f:0] */
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_ADR 0x000054d4
+/* bitmask for bitfield rss_key_wr_data[1f:0] */
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_MSK 0xffffffff
+/* inverted bitmask for bitfield rss_key_wr_data[1f:0] */
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_MSKN 0x00000000
+/* lower bit position of bitfield rss_key_wr_data[1f:0] */
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_SHIFT 0
+/* width of bitfield rss_key_wr_data[1f:0] */
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_WIDTH 32
+/* default value of bitfield rss_key_wr_data[1f:0] */
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_DEFAULT 0x0
+
+/* rx rss_key_wr_en_i bitfield definitions
+ * preprocessor definitions for the bitfield "rss_key_wr_en_i".
+ * port="pif_rpf_rss_key_wr_en_i"
+ */
+
+/* register address for bitfield rss_key_wr_en_i */
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_ADR 0x000054d0
+/* bitmask for bitfield rss_key_wr_en_i */
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_MSK 0x00000020
+/* inverted bitmask for bitfield rss_key_wr_en_i */
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_MSKN 0xffffffdf
+/* lower bit position of bitfield rss_key_wr_en_i */
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_SHIFT 5
+/* width of bitfield rss_key_wr_en_i */
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_WIDTH 1
+/* default value of bitfield rss_key_wr_en_i */
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_DEFAULT 0x0
+
+/* rx rss_redir_addr[3:0] bitfield definitions
+ * preprocessor definitions for the bitfield "rss_redir_addr[3:0]".
+ * port="pif_rpf_rss_redir_addr_i[3:0]"
+ */
+
+/* register address for bitfield rss_redir_addr[3:0] */
+#define HW_ATL_RPF_RSS_REDIR_ADDR_ADR 0x000054e0
+/* bitmask for bitfield rss_redir_addr[3:0] */
+#define HW_ATL_RPF_RSS_REDIR_ADDR_MSK 0x0000000f
+/* inverted bitmask for bitfield rss_redir_addr[3:0] */
+#define HW_ATL_RPF_RSS_REDIR_ADDR_MSKN 0xfffffff0
+/* lower bit position of bitfield rss_redir_addr[3:0] */
+#define HW_ATL_RPF_RSS_REDIR_ADDR_SHIFT 0
+/* width of bitfield rss_redir_addr[3:0] */
+#define HW_ATL_RPF_RSS_REDIR_ADDR_WIDTH 4
+/* default value of bitfield rss_redir_addr[3:0] */
+#define HW_ATL_RPF_RSS_REDIR_ADDR_DEFAULT 0x0
+
+/* rx rss_redir_wr_data[f:0] bitfield definitions
+ * preprocessor definitions for the bitfield "rss_redir_wr_data[f:0]".
+ * port="pif_rpf_rss_redir_wr_data_i[15:0]"
+ */
+
+/* register address for bitfield rss_redir_wr_data[f:0] */
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_ADR 0x000054e4
+/* bitmask for bitfield rss_redir_wr_data[f:0] */
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_MSK 0x0000ffff
+/* inverted bitmask for bitfield rss_redir_wr_data[f:0] */
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_MSKN 0xffff0000
+/* lower bit position of bitfield rss_redir_wr_data[f:0] */
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_SHIFT 0
+/* width of bitfield rss_redir_wr_data[f:0] */
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_WIDTH 16
+/* default value of bitfield rss_redir_wr_data[f:0] */
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_DEFAULT 0x0
+
+/* rx rss_redir_wr_en_i bitfield definitions
+ * preprocessor definitions for the bitfield "rss_redir_wr_en_i".
+ * port="pif_rpf_rss_redir_wr_en_i"
+ */
+
+/* register address for bitfield rss_redir_wr_en_i */
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_ADR 0x000054e0
+/* bitmask for bitfield rss_redir_wr_en_i */
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_MSK 0x00000010
+/* inverted bitmask for bitfield rss_redir_wr_en_i */
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_MSKN 0xffffffef
+/* lower bit position of bitfield rss_redir_wr_en_i */
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_SHIFT 4
+/* width of bitfield rss_redir_wr_en_i */
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_WIDTH 1
+/* default value of bitfield rss_redir_wr_en_i */
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_DEFAULT 0x0
+
+/* rx tpo_rpf_sys_loopback bitfield definitions
+ * preprocessor definitions for the bitfield "tpo_rpf_sys_loopback".
+ * port="pif_rpf_tpo_pkt_sys_lbk_i"
+ */
+
+/* register address for bitfield tpo_rpf_sys_loopback */
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_ADR 0x00005000
+/* bitmask for bitfield tpo_rpf_sys_loopback */
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_MSK 0x00000100
+/* inverted bitmask for bitfield tpo_rpf_sys_loopback */
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_MSKN 0xfffffeff
+/* lower bit position of bitfield tpo_rpf_sys_loopback */
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_SHIFT 8
+/* width of bitfield tpo_rpf_sys_loopback */
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_WIDTH 1
+/* default value of bitfield tpo_rpf_sys_loopback */
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_DEFAULT 0x0
+
+/* rx vl_inner_tpid[f:0] bitfield definitions
+ * preprocessor definitions for the bitfield "vl_inner_tpid[f:0]".
+ * port="pif_rpf_vl_inner_tpid_i[15:0]"
+ */
+
+/* register address for bitfield vl_inner_tpid[f:0] */
+#define HW_ATL_RPF_VL_INNER_TPID_ADR 0x00005284
+/* bitmask for bitfield vl_inner_tpid[f:0] */
+#define HW_ATL_RPF_VL_INNER_TPID_MSK 0x0000ffff
+/* inverted bitmask for bitfield vl_inner_tpid[f:0] */
+#define HW_ATL_RPF_VL_INNER_TPID_MSKN 0xffff0000
+/* lower bit position of bitfield vl_inner_tpid[f:0] */
+#define HW_ATL_RPF_VL_INNER_TPID_SHIFT 0
+/* width of bitfield vl_inner_tpid[f:0] */
+#define HW_ATL_RPF_VL_INNER_TPID_WIDTH 16
+/* default value of bitfield vl_inner_tpid[f:0] */
+#define HW_ATL_RPF_VL_INNER_TPID_DEFAULT 0x8100
+
+/* rx vl_outer_tpid[f:0] bitfield definitions
+ * preprocessor definitions for the bitfield "vl_outer_tpid[f:0]".
+ * port="pif_rpf_vl_outer_tpid_i[15:0]"
+ */
+
+/* register address for bitfield vl_outer_tpid[f:0] */
+#define HW_ATL_RPF_VL_OUTER_TPID_ADR 0x00005284
+/* bitmask for bitfield vl_outer_tpid[f:0] */
+#define HW_ATL_RPF_VL_OUTER_TPID_MSK 0xffff0000
+/* inverted bitmask for bitfield vl_outer_tpid[f:0] */
+#define HW_ATL_RPF_VL_OUTER_TPID_MSKN 0x0000ffff
+/* lower bit position of bitfield vl_outer_tpid[f:0] */
+#define HW_ATL_RPF_VL_OUTER_TPID_SHIFT 16
+/* width of bitfield vl_outer_tpid[f:0] */
+#define HW_ATL_RPF_VL_OUTER_TPID_WIDTH 16
+/* default value of bitfield vl_outer_tpid[f:0] */
+#define HW_ATL_RPF_VL_OUTER_TPID_DEFAULT 0x88a8
+
+/* rx vl_promis_mode bitfield definitions
+ * preprocessor definitions for the bitfield "vl_promis_mode".
+ * port="pif_rpf_vl_promis_mode_i"
+ */
+
+/* register address for bitfield vl_promis_mode */
+#define HW_ATL_RPF_VL_PROMIS_MODE_ADR 0x00005280
+/* bitmask for bitfield vl_promis_mode */
+#define HW_ATL_RPF_VL_PROMIS_MODE_MSK 0x00000002
+/* inverted bitmask for bitfield vl_promis_mode */
+#define HW_ATL_RPF_VL_PROMIS_MODE_MSKN 0xfffffffd
+/* lower bit position of bitfield vl_promis_mode */
+#define HW_ATL_RPF_VL_PROMIS_MODE_SHIFT 1
+/* width of bitfield vl_promis_mode */
+#define HW_ATL_RPF_VL_PROMIS_MODE_WIDTH 1
+/* default value of bitfield vl_promis_mode */
+#define HW_ATL_RPF_VL_PROMIS_MODE_DEFAULT 0x0
+
+/* RX vl_accept_untagged_mode Bitfield Definitions
+ * Preprocessor definitions for the bitfield "vl_accept_untagged_mode".
+ * PORT="pif_rpf_vl_accept_untagged_i"
+ */
+
+/* Register address for bitfield vl_accept_untagged_mode */
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_ADR 0x00005280
+/* Bitmask for bitfield vl_accept_untagged_mode */
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_MSK 0x00000004
+/* Inverted bitmask for bitfield vl_accept_untagged_mode */
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_MSKN 0xFFFFFFFB
+/* Lower bit position of bitfield vl_accept_untagged_mode */
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_SHIFT 2
+/* Width of bitfield vl_accept_untagged_mode */
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_WIDTH 1
+/* Default value of bitfield vl_accept_untagged_mode */
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_DEFAULT 0x0
+
+/* rX vl_untagged_act[2:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "vl_untagged_act[2:0]".
+ * PORT="pif_rpf_vl_untagged_act_i[2:0]"
+ */
+
+/* Register address for bitfield vl_untagged_act[2:0] */
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_ADR 0x00005280
+/* Bitmask for bitfield vl_untagged_act[2:0] */
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_MSK 0x00000038
+/* Inverted bitmask for bitfield vl_untagged_act[2:0] */
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_MSKN 0xFFFFFFC7
+/* Lower bit position of bitfield vl_untagged_act[2:0] */
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_SHIFT 3
+/* Width of bitfield vl_untagged_act[2:0] */
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_WIDTH 3
+/* Default value of bitfield vl_untagged_act[2:0] */
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_DEFAULT 0x0
+
+/* RX vl_en{F} Bitfield Definitions
+ * Preprocessor definitions for the bitfield "vl_en{F}".
+ * Parameter: filter {F} | stride size 0x4 | range [0, 15]
+ * PORT="pif_rpf_vl_en_i[0]"
+ */
+
+/* Register address for bitfield vl_en{F} */
+#define HW_ATL_RPF_VL_EN_F_ADR(filter) (0x00005290 + (filter) * 0x4)
+/* Bitmask for bitfield vl_en{F} */
+#define HW_ATL_RPF_VL_EN_F_MSK 0x80000000
+/* Inverted bitmask for bitfield vl_en{F} */
+#define HW_ATL_RPF_VL_EN_F_MSKN 0x7FFFFFFF
+/* Lower bit position of bitfield vl_en{F} */
+#define HW_ATL_RPF_VL_EN_F_SHIFT 31
+/* Width of bitfield vl_en{F} */
+#define HW_ATL_RPF_VL_EN_F_WIDTH 1
+/* Default value of bitfield vl_en{F} */
+#define HW_ATL_RPF_VL_EN_F_DEFAULT 0x0
+
+/* RX vl_act{F}[2:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "vl_act{F}[2:0]".
+ * Parameter: filter {F} | stride size 0x4 | range [0, 15]
+ * PORT="pif_rpf_vl_act0_i[2:0]"
+ */
+
+/* Register address for bitfield vl_act{F}[2:0] */
+#define HW_ATL_RPF_VL_ACT_F_ADR(filter) (0x00005290 + (filter) * 0x4)
+/* Bitmask for bitfield vl_act{F}[2:0] */
+#define HW_ATL_RPF_VL_ACT_F_MSK 0x00070000
+/* Inverted bitmask for bitfield vl_act{F}[2:0] */
+#define HW_ATL_RPF_VL_ACT_F_MSKN 0xFFF8FFFF
+/* Lower bit position of bitfield vl_act{F}[2:0] */
+#define HW_ATL_RPF_VL_ACT_F_SHIFT 16
+/* Width of bitfield vl_act{F}[2:0] */
+#define HW_ATL_RPF_VL_ACT_F_WIDTH 3
+/* Default value of bitfield vl_act{F}[2:0] */
+#define HW_ATL_RPF_VL_ACT_F_DEFAULT 0x0
+
+/* RX vl_id{F}[B:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "vl_id{F}[B:0]".
+ * Parameter: filter {F} | stride size 0x4 | range [0, 15]
+ * PORT="pif_rpf_vl_id0_i[11:0]"
+ */
+
+/* Register address for bitfield vl_id{F}[B:0] */
+#define HW_ATL_RPF_VL_ID_F_ADR(filter) (0x00005290 + (filter) * 0x4)
+/* Bitmask for bitfield vl_id{F}[B:0] */
+#define HW_ATL_RPF_VL_ID_F_MSK 0x00000FFF
+/* Inverted bitmask for bitfield vl_id{F}[B:0] */
+#define HW_ATL_RPF_VL_ID_F_MSKN 0xFFFFF000
+/* Lower bit position of bitfield vl_id{F}[B:0] */
+#define HW_ATL_RPF_VL_ID_F_SHIFT 0
+/* Width of bitfield vl_id{F}[B:0] */
+#define HW_ATL_RPF_VL_ID_F_WIDTH 12
+/* Default value of bitfield vl_id{F}[B:0] */
+#define HW_ATL_RPF_VL_ID_F_DEFAULT 0x0
+
+/* RX et_en{F} Bitfield Definitions
+ * Preprocessor definitions for the bitfield "et_en{F}".
+ * Parameter: filter {F} | stride size 0x4 | range [0, 15]
+ * PORT="pif_rpf_et_en_i[0]"
+ */
+
+/* Register address for bitfield et_en{F} */
+#define HW_ATL_RPF_ET_EN_F_ADR(filter) (0x00005300 + (filter) * 0x4)
+/* Bitmask for bitfield et_en{F} */
+#define HW_ATL_RPF_ET_EN_F_MSK 0x80000000
+/* Inverted bitmask for bitfield et_en{F} */
+#define HW_ATL_RPF_ET_EN_F_MSKN 0x7FFFFFFF
+/* Lower bit position of bitfield et_en{F} */
+#define HW_ATL_RPF_ET_EN_F_SHIFT 31
+/* Width of bitfield et_en{F} */
+#define HW_ATL_RPF_ET_EN_F_WIDTH 1
+/* Default value of bitfield et_en{F} */
+#define HW_ATL_RPF_ET_EN_F_DEFAULT 0x0
+
+/* rx et_en{f} bitfield definitions
+ * preprocessor definitions for the bitfield "et_en{f}".
+ * parameter: filter {f} | stride size 0x4 | range [0, 15]
+ * port="pif_rpf_et_en_i[0]"
+ */
+
+/* register address for bitfield et_en{f} */
+#define HW_ATL_RPF_ET_ENF_ADR(filter) (0x00005300 + (filter) * 0x4)
+/* bitmask for bitfield et_en{f} */
+#define HW_ATL_RPF_ET_ENF_MSK 0x80000000
+/* inverted bitmask for bitfield et_en{f} */
+#define HW_ATL_RPF_ET_ENF_MSKN 0x7fffffff
+/* lower bit position of bitfield et_en{f} */
+#define HW_ATL_RPF_ET_ENF_SHIFT 31
+/* width of bitfield et_en{f} */
+#define HW_ATL_RPF_ET_ENF_WIDTH 1
+/* default value of bitfield et_en{f} */
+#define HW_ATL_RPF_ET_ENF_DEFAULT 0x0
+
+/* rx et_up{f}_en bitfield definitions
+ * preprocessor definitions for the bitfield "et_up{f}_en".
+ * parameter: filter {f} | stride size 0x4 | range [0, 15]
+ * port="pif_rpf_et_up_en_i[0]"
+ */
+
+/* register address for bitfield et_up{f}_en */
+#define HW_ATL_RPF_ET_UPFEN_ADR(filter) (0x00005300 + (filter) * 0x4)
+/* bitmask for bitfield et_up{f}_en */
+#define HW_ATL_RPF_ET_UPFEN_MSK 0x40000000
+/* inverted bitmask for bitfield et_up{f}_en */
+#define HW_ATL_RPF_ET_UPFEN_MSKN 0xbfffffff
+/* lower bit position of bitfield et_up{f}_en */
+#define HW_ATL_RPF_ET_UPFEN_SHIFT 30
+/* width of bitfield et_up{f}_en */
+#define HW_ATL_RPF_ET_UPFEN_WIDTH 1
+/* default value of bitfield et_up{f}_en */
+#define HW_ATL_RPF_ET_UPFEN_DEFAULT 0x0
+
+/* rx et_rxq{f}_en bitfield definitions
+ * preprocessor definitions for the bitfield "et_rxq{f}_en".
+ * parameter: filter {f} | stride size 0x4 | range [0, 15]
+ * port="pif_rpf_et_rxq_en_i[0]"
+ */
+
+/* register address for bitfield et_rxq{f}_en */
+#define HW_ATL_RPF_ET_RXQFEN_ADR(filter) (0x00005300 + (filter) * 0x4)
+/* bitmask for bitfield et_rxq{f}_en */
+#define HW_ATL_RPF_ET_RXQFEN_MSK 0x20000000
+/* inverted bitmask for bitfield et_rxq{f}_en */
+#define HW_ATL_RPF_ET_RXQFEN_MSKN 0xdfffffff
+/* lower bit position of bitfield et_rxq{f}_en */
+#define HW_ATL_RPF_ET_RXQFEN_SHIFT 29
+/* width of bitfield et_rxq{f}_en */
+#define HW_ATL_RPF_ET_RXQFEN_WIDTH 1
+/* default value of bitfield et_rxq{f}_en */
+#define HW_ATL_RPF_ET_RXQFEN_DEFAULT 0x0
+
+/* rx et_up{f}[2:0] bitfield definitions
+ * preprocessor definitions for the bitfield "et_up{f}[2:0]".
+ * parameter: filter {f} | stride size 0x4 | range [0, 15]
+ * port="pif_rpf_et_up0_i[2:0]"
+ */
+
+/* register address for bitfield et_up{f}[2:0] */
+#define HW_ATL_RPF_ET_UPF_ADR(filter) (0x00005300 + (filter) * 0x4)
+/* bitmask for bitfield et_up{f}[2:0] */
+#define HW_ATL_RPF_ET_UPF_MSK 0x1c000000
+/* inverted bitmask for bitfield et_up{f}[2:0] */
+#define HW_ATL_RPF_ET_UPF_MSKN 0xe3ffffff
+/* lower bit position of bitfield et_up{f}[2:0] */
+#define HW_ATL_RPF_ET_UPF_SHIFT 26
+/* width of bitfield et_up{f}[2:0] */
+#define HW_ATL_RPF_ET_UPF_WIDTH 3
+/* default value of bitfield et_up{f}[2:0] */
+#define HW_ATL_RPF_ET_UPF_DEFAULT 0x0
+
+/* rx et_rxq{f}[4:0] bitfield definitions
+ * preprocessor definitions for the bitfield "et_rxq{f}[4:0]".
+ * parameter: filter {f} | stride size 0x4 | range [0, 15]
+ * port="pif_rpf_et_rxq0_i[4:0]"
+ */
+
+/* register address for bitfield et_rxq{f}[4:0] */
+#define HW_ATL_RPF_ET_RXQF_ADR(filter) (0x00005300 + (filter) * 0x4)
+/* bitmask for bitfield et_rxq{f}[4:0] */
+#define HW_ATL_RPF_ET_RXQF_MSK 0x01f00000
+/* inverted bitmask for bitfield et_rxq{f}[4:0] */
+#define HW_ATL_RPF_ET_RXQF_MSKN 0xfe0fffff
+/* lower bit position of bitfield et_rxq{f}[4:0] */
+#define HW_ATL_RPF_ET_RXQF_SHIFT 20
+/* width of bitfield et_rxq{f}[4:0] */
+#define HW_ATL_RPF_ET_RXQF_WIDTH 5
+/* default value of bitfield et_rxq{f}[4:0] */
+#define HW_ATL_RPF_ET_RXQF_DEFAULT 0x0
+
+/* rx et_mng_rxq{f} bitfield definitions
+ * preprocessor definitions for the bitfield "et_mng_rxq{f}".
+ * parameter: filter {f} | stride size 0x4 | range [0, 15]
+ * port="pif_rpf_et_mng_rxq_i[0]"
+ */
+
+/* register address for bitfield et_mng_rxq{f} */
+#define HW_ATL_RPF_ET_MNG_RXQF_ADR(filter) (0x00005300 + (filter) * 0x4)
+/* bitmask for bitfield et_mng_rxq{f} */
+#define HW_ATL_RPF_ET_MNG_RXQF_MSK 0x00080000
+/* inverted bitmask for bitfield et_mng_rxq{f} */
+#define HW_ATL_RPF_ET_MNG_RXQF_MSKN 0xfff7ffff
+/* lower bit position of bitfield et_mng_rxq{f} */
+#define HW_ATL_RPF_ET_MNG_RXQF_SHIFT 19
+/* width of bitfield et_mng_rxq{f} */
+#define HW_ATL_RPF_ET_MNG_RXQF_WIDTH 1
+/* default value of bitfield et_mng_rxq{f} */
+#define HW_ATL_RPF_ET_MNG_RXQF_DEFAULT 0x0
+
+/* rx et_act{f}[2:0] bitfield definitions
+ * preprocessor definitions for the bitfield "et_act{f}[2:0]".
+ * parameter: filter {f} | stride size 0x4 | range [0, 15]
+ * port="pif_rpf_et_act0_i[2:0]"
+ */
+
+/* register address for bitfield et_act{f}[2:0] */
+#define HW_ATL_RPF_ET_ACTF_ADR(filter) (0x00005300 + (filter) * 0x4)
+/* bitmask for bitfield et_act{f}[2:0] */
+#define HW_ATL_RPF_ET_ACTF_MSK 0x00070000
+/* inverted bitmask for bitfield et_act{f}[2:0] */
+#define HW_ATL_RPF_ET_ACTF_MSKN 0xfff8ffff
+/* lower bit position of bitfield et_act{f}[2:0] */
+#define HW_ATL_RPF_ET_ACTF_SHIFT 16
+/* width of bitfield et_act{f}[2:0] */
+#define HW_ATL_RPF_ET_ACTF_WIDTH 3
+/* default value of bitfield et_act{f}[2:0] */
+#define HW_ATL_RPF_ET_ACTF_DEFAULT 0x0
+
+/* rx et_val{f}[f:0] bitfield definitions
+ * preprocessor definitions for the bitfield "et_val{f}[f:0]".
+ * parameter: filter {f} | stride size 0x4 | range [0, 15]
+ * port="pif_rpf_et_val0_i[15:0]"
+ */
+
+/* register address for bitfield et_val{f}[f:0] */
+#define HW_ATL_RPF_ET_VALF_ADR(filter) (0x00005300 + (filter) * 0x4)
+/* bitmask for bitfield et_val{f}[f:0] */
+#define HW_ATL_RPF_ET_VALF_MSK 0x0000ffff
+/* inverted bitmask for bitfield et_val{f}[f:0] */
+#define HW_ATL_RPF_ET_VALF_MSKN 0xffff0000
+/* lower bit position of bitfield et_val{f}[f:0] */
+#define HW_ATL_RPF_ET_VALF_SHIFT 0
+/* width of bitfield et_val{f}[f:0] */
+#define HW_ATL_RPF_ET_VALF_WIDTH 16
+/* default value of bitfield et_val{f}[f:0] */
+#define HW_ATL_RPF_ET_VALF_DEFAULT 0x0
+
+/* rx ipv4_chk_en bitfield definitions
+ * preprocessor definitions for the bitfield "ipv4_chk_en".
+ * port="pif_rpo_ipv4_chk_en_i"
+ */
+
+/* register address for bitfield ipv4_chk_en */
+#define HW_ATL_RPO_IPV4CHK_EN_ADR 0x00005580
+/* bitmask for bitfield ipv4_chk_en */
+#define HW_ATL_RPO_IPV4CHK_EN_MSK 0x00000002
+/* inverted bitmask for bitfield ipv4_chk_en */
+#define HW_ATL_RPO_IPV4CHK_EN_MSKN 0xfffffffd
+/* lower bit position of bitfield ipv4_chk_en */
+#define HW_ATL_RPO_IPV4CHK_EN_SHIFT 1
+/* width of bitfield ipv4_chk_en */
+#define HW_ATL_RPO_IPV4CHK_EN_WIDTH 1
+/* default value of bitfield ipv4_chk_en */
+#define HW_ATL_RPO_IPV4CHK_EN_DEFAULT 0x0
+
+/* rx desc{d}_vl_strip bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_vl_strip".
+ * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
+ * port="pif_rpo_desc_vl_strip_i[0]"
+ */
+
+/* register address for bitfield desc{d}_vl_strip */
+#define HW_ATL_RPO_DESCDVL_STRIP_ADR(descriptor) \
+	(0x00005b08 + (descriptor) * 0x20)
+/* bitmask for bitfield desc{d}_vl_strip */
+#define HW_ATL_RPO_DESCDVL_STRIP_MSK 0x20000000
+/* inverted bitmask for bitfield desc{d}_vl_strip */
+#define HW_ATL_RPO_DESCDVL_STRIP_MSKN 0xdfffffff
+/* lower bit position of bitfield desc{d}_vl_strip */
+#define HW_ATL_RPO_DESCDVL_STRIP_SHIFT 29
+/* width of bitfield desc{d}_vl_strip */
+#define HW_ATL_RPO_DESCDVL_STRIP_WIDTH 1
+/* default value of bitfield desc{d}_vl_strip */
+#define HW_ATL_RPO_DESCDVL_STRIP_DEFAULT 0x0
+
+/* rx l4_chk_en bitfield definitions
+ * preprocessor definitions for the bitfield "l4_chk_en".
+ * port="pif_rpo_l4_chk_en_i"
+ */
+
+/* register address for bitfield l4_chk_en */
+#define HW_ATL_RPOL4CHK_EN_ADR 0x00005580
+/* bitmask for bitfield l4_chk_en */
+#define HW_ATL_RPOL4CHK_EN_MSK 0x00000001
+/* inverted bitmask for bitfield l4_chk_en */
+#define HW_ATL_RPOL4CHK_EN_MSKN 0xfffffffe
+/* lower bit position of bitfield l4_chk_en */
+#define HW_ATL_RPOL4CHK_EN_SHIFT 0
+/* width of bitfield l4_chk_en */
+#define HW_ATL_RPOL4CHK_EN_WIDTH 1
+/* default value of bitfield l4_chk_en */
+#define HW_ATL_RPOL4CHK_EN_DEFAULT 0x0
+
+/* rx reg_res_dsbl bitfield definitions
+ * preprocessor definitions for the bitfield "reg_res_dsbl".
+ * port="pif_rx_reg_res_dsbl_i"
+ */
+
+/* register address for bitfield reg_res_dsbl */
+#define HW_ATL_RX_REG_RES_DSBL_ADR 0x00005000
+/* bitmask for bitfield reg_res_dsbl */
+#define HW_ATL_RX_REG_RES_DSBL_MSK 0x20000000
+/* inverted bitmask for bitfield reg_res_dsbl */
+#define HW_ATL_RX_REG_RES_DSBL_MSKN 0xdfffffff
+/* lower bit position of bitfield reg_res_dsbl */
+#define HW_ATL_RX_REG_RES_DSBL_SHIFT 29
+/* width of bitfield reg_res_dsbl */
+#define HW_ATL_RX_REG_RES_DSBL_WIDTH 1
+/* default value of bitfield reg_res_dsbl */
+#define HW_ATL_RX_REG_RES_DSBL_DEFAULT 0x1
+
+/* tx dca{d}_cpuid[7:0] bitfield definitions
+ * preprocessor definitions for the bitfield "dca{d}_cpuid[7:0]".
+ * parameter: dca {d} | stride size 0x4 | range [0, 31]
+ * port="pif_tdm_dca0_cpuid_i[7:0]"
+ */
+
+/* register address for bitfield dca{d}_cpuid[7:0] */
+#define HW_ATL_TDM_DCADCPUID_ADR(dca) (0x00008400 + (dca) * 0x4)
+/* bitmask for bitfield dca{d}_cpuid[7:0] */
+#define HW_ATL_TDM_DCADCPUID_MSK 0x000000ff
+/* inverted bitmask for bitfield dca{d}_cpuid[7:0] */
+#define HW_ATL_TDM_DCADCPUID_MSKN 0xffffff00
+/* lower bit position of bitfield dca{d}_cpuid[7:0] */
+#define HW_ATL_TDM_DCADCPUID_SHIFT 0
+/* width of bitfield dca{d}_cpuid[7:0] */
+#define HW_ATL_TDM_DCADCPUID_WIDTH 8
+/* default value of bitfield dca{d}_cpuid[7:0] */
+#define HW_ATL_TDM_DCADCPUID_DEFAULT 0x0
+
+/* tx lso_en[1f:0] bitfield definitions
+ * preprocessor definitions for the bitfield "lso_en[1f:0]".
+ * port="pif_tdm_lso_en_i[31:0]"
+ */
+
+/* register address for bitfield lso_en[1f:0] */
+#define HW_ATL_TDM_LSO_EN_ADR 0x00007810
+/* bitmask for bitfield lso_en[1f:0] */
+#define HW_ATL_TDM_LSO_EN_MSK 0xffffffff
+/* inverted bitmask for bitfield lso_en[1f:0] */
+#define HW_ATL_TDM_LSO_EN_MSKN 0x00000000
+/* lower bit position of bitfield lso_en[1f:0] */
+#define HW_ATL_TDM_LSO_EN_SHIFT 0
+/* width of bitfield lso_en[1f:0] */
+#define HW_ATL_TDM_LSO_EN_WIDTH 32
+/* default value of bitfield lso_en[1f:0] */
+#define HW_ATL_TDM_LSO_EN_DEFAULT 0x0
+
+/* tx dca_en bitfield definitions
+ * preprocessor definitions for the bitfield "dca_en".
+ * port="pif_tdm_dca_en_i"
+ */
+
+/* register address for bitfield dca_en */
+#define HW_ATL_TDM_DCA_EN_ADR 0x00008480
+/* bitmask for bitfield dca_en */
+#define HW_ATL_TDM_DCA_EN_MSK 0x80000000
+/* inverted bitmask for bitfield dca_en */
+#define HW_ATL_TDM_DCA_EN_MSKN 0x7fffffff
+/* lower bit position of bitfield dca_en */
+#define HW_ATL_TDM_DCA_EN_SHIFT 31
+/* width of bitfield dca_en */
+#define HW_ATL_TDM_DCA_EN_WIDTH 1
+/* default value of bitfield dca_en */
+#define HW_ATL_TDM_DCA_EN_DEFAULT 0x1
+
+/* tx dca_mode[3:0] bitfield definitions
+ * preprocessor definitions for the bitfield "dca_mode[3:0]".
+ * port="pif_tdm_dca_mode_i[3:0]"
+ */
+
+/* register address for bitfield dca_mode[3:0] */
+#define HW_ATL_TDM_DCA_MODE_ADR 0x00008480
+/* bitmask for bitfield dca_mode[3:0] */
+#define HW_ATL_TDM_DCA_MODE_MSK 0x0000000f
+/* inverted bitmask for bitfield dca_mode[3:0] */
+#define HW_ATL_TDM_DCA_MODE_MSKN 0xfffffff0
+/* lower bit position of bitfield dca_mode[3:0] */
+#define HW_ATL_TDM_DCA_MODE_SHIFT 0
+/* width of bitfield dca_mode[3:0] */
+#define HW_ATL_TDM_DCA_MODE_WIDTH 4
+/* default value of bitfield dca_mode[3:0] */
+#define HW_ATL_TDM_DCA_MODE_DEFAULT 0x0
+
+/* tx dca{d}_desc_en bitfield definitions
+ * preprocessor definitions for the bitfield "dca{d}_desc_en".
+ * parameter: dca {d} | stride size 0x4 | range [0, 31]
+ * port="pif_tdm_dca_desc_en_i[0]"
+ */
+
+/* register address for bitfield dca{d}_desc_en */
+#define HW_ATL_TDM_DCADDESC_EN_ADR(dca) (0x00008400 + (dca) * 0x4)
+/* bitmask for bitfield dca{d}_desc_en */
+#define HW_ATL_TDM_DCADDESC_EN_MSK 0x80000000
+/* inverted bitmask for bitfield dca{d}_desc_en */
+#define HW_ATL_TDM_DCADDESC_EN_MSKN 0x7fffffff
+/* lower bit position of bitfield dca{d}_desc_en */
+#define HW_ATL_TDM_DCADDESC_EN_SHIFT 31
+/* width of bitfield dca{d}_desc_en */
+#define HW_ATL_TDM_DCADDESC_EN_WIDTH 1
+/* default value of bitfield dca{d}_desc_en */
+#define HW_ATL_TDM_DCADDESC_EN_DEFAULT 0x0
+
+/* tx desc{d}_en bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_en".
+ * parameter: descriptor {d} | stride size 0x40 | range [0, 31]
+ * port="pif_tdm_desc_en_i[0]"
+ */
+
+/* register address for bitfield desc{d}_en */
+#define HW_ATL_TDM_DESCDEN_ADR(descriptor) (0x00007c08 + (descriptor) * 0x40)
+/* bitmask for bitfield desc{d}_en */
+#define HW_ATL_TDM_DESCDEN_MSK 0x80000000
+/* inverted bitmask for bitfield desc{d}_en */
+#define HW_ATL_TDM_DESCDEN_MSKN 0x7fffffff
+/* lower bit position of bitfield desc{d}_en */
+#define HW_ATL_TDM_DESCDEN_SHIFT 31
+/* width of bitfield desc{d}_en */
+#define HW_ATL_TDM_DESCDEN_WIDTH 1
+/* default value of bitfield desc{d}_en */
+#define HW_ATL_TDM_DESCDEN_DEFAULT 0x0
+
+/* tx desc{d}_hd[c:0] bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_hd[c:0]".
+ * parameter: descriptor {d} | stride size 0x40 | range [0, 31]
+ * port="tdm_pif_desc0_hd_o[12:0]"
+ */
+
+/* register address for bitfield desc{d}_hd[c:0] */
+#define HW_ATL_TDM_DESCDHD_ADR(descriptor) (0x00007c0c + (descriptor) * 0x40)
+/* bitmask for bitfield desc{d}_hd[c:0] */
+#define HW_ATL_TDM_DESCDHD_MSK 0x00001fff
+/* inverted bitmask for bitfield desc{d}_hd[c:0] */
+#define HW_ATL_TDM_DESCDHD_MSKN 0xffffe000
+/* lower bit position of bitfield desc{d}_hd[c:0] */
+#define HW_ATL_TDM_DESCDHD_SHIFT 0
+/* width of bitfield desc{d}_hd[c:0] */
+#define HW_ATL_TDM_DESCDHD_WIDTH 13
+
+/* tx desc{d}_len[9:0] bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_len[9:0]".
+ * parameter: descriptor {d} | stride size 0x40 | range [0, 31]
+ * port="pif_tdm_desc0_len_i[9:0]"
+ */
+
+/* register address for bitfield desc{d}_len[9:0] */
+#define HW_ATL_TDM_DESCDLEN_ADR(descriptor) (0x00007c08 + (descriptor) * 0x40)
+/* bitmask for bitfield desc{d}_len[9:0] */
+#define HW_ATL_TDM_DESCDLEN_MSK 0x00001ff8
+/* inverted bitmask for bitfield desc{d}_len[9:0] */
+#define HW_ATL_TDM_DESCDLEN_MSKN 0xffffe007
+/* lower bit position of bitfield desc{d}_len[9:0] */
+#define HW_ATL_TDM_DESCDLEN_SHIFT 3
+/* width of bitfield desc{d}_len[9:0] */
+#define HW_ATL_TDM_DESCDLEN_WIDTH 10
+/* default value of bitfield desc{d}_len[9:0] */
+#define HW_ATL_TDM_DESCDLEN_DEFAULT 0x0
+
+/* tx int_desc_wrb_en bitfield definitions
+ * preprocessor definitions for the bitfield "int_desc_wrb_en".
+ * port="pif_tdm_int_desc_wrb_en_i"
+ */
+
+/* register address for bitfield int_desc_wrb_en */
+#define HW_ATL_TDM_INT_DESC_WRB_EN_ADR 0x00007b40
+/* bitmask for bitfield int_desc_wrb_en */
+#define HW_ATL_TDM_INT_DESC_WRB_EN_MSK 0x00000002
+/* inverted bitmask for bitfield int_desc_wrb_en */
+#define HW_ATL_TDM_INT_DESC_WRB_EN_MSKN 0xfffffffd
+/* lower bit position of bitfield int_desc_wrb_en */
+#define HW_ATL_TDM_INT_DESC_WRB_EN_SHIFT 1
+/* width of bitfield int_desc_wrb_en */
+#define HW_ATL_TDM_INT_DESC_WRB_EN_WIDTH 1
+/* default value of bitfield int_desc_wrb_en */
+#define HW_ATL_TDM_INT_DESC_WRB_EN_DEFAULT 0x0
+
+/* tx desc{d}_wrb_thresh[6:0] bitfield definitions
+ * preprocessor definitions for the bitfield "desc{d}_wrb_thresh[6:0]".
+ * parameter: descriptor {d} | stride size 0x40 | range [0, 31]
+ * port="pif_tdm_desc0_wrb_thresh_i[6:0]"
+ */
+
+/* register address for bitfield desc{d}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESCDWRB_THRESH_ADR(descriptor) \
+	(0x00007c18 + (descriptor) * 0x40)
+/* bitmask for bitfield desc{d}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESCDWRB_THRESH_MSK 0x00007f00
+/* inverted bitmask for bitfield desc{d}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESCDWRB_THRESH_MSKN 0xffff80ff
+/* lower bit position of bitfield desc{d}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESCDWRB_THRESH_SHIFT 8
+/* width of bitfield desc{d}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESCDWRB_THRESH_WIDTH 7
+/* default value of bitfield desc{d}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESCDWRB_THRESH_DEFAULT 0x0
+
+/* tx lso_tcp_flag_first[b:0] bitfield definitions
+ * preprocessor definitions for the bitfield "lso_tcp_flag_first[b:0]".
+ * port="pif_thm_lso_tcp_flag_first_i[11:0]"
+ */
+
+/* register address for bitfield lso_tcp_flag_first[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_ADR 0x00007820
+/* bitmask for bitfield lso_tcp_flag_first[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_MSK 0x00000fff
+/* inverted bitmask for bitfield lso_tcp_flag_first[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_MSKN 0xfffff000
+/* lower bit position of bitfield lso_tcp_flag_first[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_SHIFT 0
+/* width of bitfield lso_tcp_flag_first[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_WIDTH 12
+/* default value of bitfield lso_tcp_flag_first[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_DEFAULT 0x0
+
+/* tx lso_tcp_flag_last[b:0] bitfield definitions
+ * preprocessor definitions for the bitfield "lso_tcp_flag_last[b:0]".
+ * port="pif_thm_lso_tcp_flag_last_i[11:0]"
+ */
+
+/* register address for bitfield lso_tcp_flag_last[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_ADR 0x00007824
+/* bitmask for bitfield lso_tcp_flag_last[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_MSK 0x00000fff
+/* inverted bitmask for bitfield lso_tcp_flag_last[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_MSKN 0xfffff000
+/* lower bit position of bitfield lso_tcp_flag_last[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_SHIFT 0
+/* width of bitfield lso_tcp_flag_last[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_WIDTH 12
+/* default value of bitfield lso_tcp_flag_last[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_DEFAULT 0x0
+
+/* tx lso_tcp_flag_mid[b:0] bitfield definitions
+ * preprocessor definitions for the bitfield "lso_tcp_flag_mid[b:0]".
+ * port="pif_thm_lso_tcp_flag_mid_i[11:0]"
+ */
+
+/* Register address for bitfield lro_rsc_max[1F:0] */
+#define HW_ATL_RPO_LRO_RSC_MAX_ADR 0x00005598
+/* Bitmask for bitfield lro_rsc_max[1F:0] */
+#define HW_ATL_RPO_LRO_RSC_MAX_MSK 0xFFFFFFFF
+/* Inverted bitmask for bitfield lro_rsc_max[1F:0] */
+#define HW_ATL_RPO_LRO_RSC_MAX_MSKN 0x00000000
+/* Lower bit position of bitfield lro_rsc_max[1F:0] */
+#define HW_ATL_RPO_LRO_RSC_MAX_SHIFT 0
+/* Width of bitfield lro_rsc_max[1F:0] */
+#define HW_ATL_RPO_LRO_RSC_MAX_WIDTH 32
+/* Default value of bitfield lro_rsc_max[1F:0] */
+#define HW_ATL_RPO_LRO_RSC_MAX_DEFAULT 0x0
+
+/* RX lro_en[1F:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "lro_en[1F:0]".
+ * PORT="pif_rpo_lro_en_i[31:0]"
+ */
+
+/* Register address for bitfield lro_en[1F:0] */
+#define HW_ATL_RPO_LRO_EN_ADR 0x00005590
+/* Bitmask for bitfield lro_en[1F:0] */
+#define HW_ATL_RPO_LRO_EN_MSK 0xFFFFFFFF
+/* Inverted bitmask for bitfield lro_en[1F:0] */
+#define HW_ATL_RPO_LRO_EN_MSKN 0x00000000
+/* Lower bit position of bitfield lro_en[1F:0] */
+#define HW_ATL_RPO_LRO_EN_SHIFT 0
+/* Width of bitfield lro_en[1F:0] */
+#define HW_ATL_RPO_LRO_EN_WIDTH 32
+/* Default value of bitfield lro_en[1F:0] */
+#define HW_ATL_RPO_LRO_EN_DEFAULT 0x0
+
+/* RX lro_ptopt_en Bitfield Definitions
+ * Preprocessor definitions for the bitfield "lro_ptopt_en".
+ * PORT="pif_rpo_lro_ptopt_en_i"
+ */
+
+/* Register address for bitfield lro_ptopt_en */
+#define HW_ATL_RPO_LRO_PTOPT_EN_ADR 0x00005594
+/* Bitmask for bitfield lro_ptopt_en */
+#define HW_ATL_RPO_LRO_PTOPT_EN_MSK 0x00008000
+/* Inverted bitmask for bitfield lro_ptopt_en */
+#define HW_ATL_RPO_LRO_PTOPT_EN_MSKN 0xFFFF7FFF
+/* Lower bit position of bitfield lro_ptopt_en */
+#define HW_ATL_RPO_LRO_PTOPT_EN_SHIFT 15
+/* Width of bitfield lro_ptopt_en */
+#define HW_ATL_RPO_LRO_PTOPT_EN_WIDTH 1
+/* Default value of bitfield lro_ptopt_en */
+#define HW_ATL_RPO_LRO_PTOPT_EN_DEFALT 0x1
+
+/* RX lro_q_ses_lmt Bitfield Definitions
+ * Preprocessor definitions for the bitfield "lro_q_ses_lmt".
+ * PORT="pif_rpo_lro_q_ses_lmt_i[1:0]"
+ */
+
+/* Register address for bitfield lro_q_ses_lmt */
+#define HW_ATL_RPO_LRO_QSES_LMT_ADR 0x00005594
+/* Bitmask for bitfield lro_q_ses_lmt */
+#define HW_ATL_RPO_LRO_QSES_LMT_MSK 0x00003000
+/* Inverted bitmask for bitfield lro_q_ses_lmt */
+#define HW_ATL_RPO_LRO_QSES_LMT_MSKN 0xFFFFCFFF
+/* Lower bit position of bitfield lro_q_ses_lmt */
+#define HW_ATL_RPO_LRO_QSES_LMT_SHIFT 12
+/* Width of bitfield lro_q_ses_lmt */
+#define HW_ATL_RPO_LRO_QSES_LMT_WIDTH 2
+/* Default value of bitfield lro_q_ses_lmt */
+#define HW_ATL_RPO_LRO_QSES_LMT_DEFAULT 0x1
+
+/* RX lro_tot_dsc_lmt[1:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "lro_tot_dsc_lmt[1:0]".
+ * PORT="pif_rpo_lro_tot_dsc_lmt_i[1:0]"
+ */
+
+/* Register address for bitfield lro_tot_dsc_lmt[1:0] */
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_ADR 0x00005594
+/* Bitmask for bitfield lro_tot_dsc_lmt[1:0] */
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_MSK 0x00000060
+/* Inverted bitmask for bitfield lro_tot_dsc_lmt[1:0] */
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_MSKN 0xFFFFFF9F
+/* Lower bit position of bitfield lro_tot_dsc_lmt[1:0] */
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_SHIFT 5
+/* Width of bitfield lro_tot_dsc_lmt[1:0] */
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_WIDTH 2
+/* Default value of bitfield lro_tot_dsc_lmt[1:0] */
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_DEFALT 0x1
+
+/* RX lro_pkt_min[4:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "lro_pkt_min[4:0]".
+ * PORT="pif_rpo_lro_pkt_min_i[4:0]"
+ */
+
+/* Register address for bitfield lro_pkt_min[4:0] */
+#define HW_ATL_RPO_LRO_PKT_MIN_ADR 0x00005594
+/* Bitmask for bitfield lro_pkt_min[4:0] */
+#define HW_ATL_RPO_LRO_PKT_MIN_MSK 0x0000001F
+/* Inverted bitmask for bitfield lro_pkt_min[4:0] */
+#define HW_ATL_RPO_LRO_PKT_MIN_MSKN 0xFFFFFFE0
+/* Lower bit position of bitfield lro_pkt_min[4:0] */
+#define HW_ATL_RPO_LRO_PKT_MIN_SHIFT 0
+/* Width of bitfield lro_pkt_min[4:0] */
+#define HW_ATL_RPO_LRO_PKT_MIN_WIDTH 5
+/* Default value of bitfield lro_pkt_min[4:0] */
+#define HW_ATL_RPO_LRO_PKT_MIN_DEFAULT 0x8
+
+/* Width of bitfield lro{L}_des_max[1:0] */
+#define HW_ATL_RPO_LRO_LDES_MAX_WIDTH 2
+/* Default value of bitfield lro{L}_des_max[1:0] */
+#define HW_ATL_RPO_LRO_LDES_MAX_DEFAULT 0x0
+
+/* RX lro_tb_div[11:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "lro_tb_div[11:0]".
+ * PORT="pif_rpo_lro_tb_div_i[11:0]"
+ */
+
+/* Register address for bitfield lro_tb_div[11:0] */
+#define HW_ATL_RPO_LRO_TB_DIV_ADR 0x00005620
+/* Bitmask for bitfield lro_tb_div[11:0] */
+#define HW_ATL_RPO_LRO_TB_DIV_MSK 0xFFF00000
+/* Inverted bitmask for bitfield lro_tb_div[11:0] */
+#define HW_ATL_RPO_LRO_TB_DIV_MSKN 0x000FFFFF
+/* Lower bit position of bitfield lro_tb_div[11:0] */
+#define HW_ATL_RPO_LRO_TB_DIV_SHIFT 20
+/* Width of bitfield lro_tb_div[11:0] */
+#define HW_ATL_RPO_LRO_TB_DIV_WIDTH 12
+/* Default value of bitfield lro_tb_div[11:0] */
+#define HW_ATL_RPO_LRO_TB_DIV_DEFAULT 0xC35
+
+/* RX lro_ina_ival[9:0] Bitfield Definitions
+ *   Preprocessor definitions for the bitfield "lro_ina_ival[9:0]".
+ *   PORT="pif_rpo_lro_ina_ival_i[9:0]"
+ */
+
+/* Register address for bitfield lro_ina_ival[9:0] */
+#define HW_ATL_RPO_LRO_INA_IVAL_ADR 0x00005620
+/* Bitmask for bitfield lro_ina_ival[9:0] */
+#define HW_ATL_RPO_LRO_INA_IVAL_MSK 0x000FFC00
+/* Inverted bitmask for bitfield lro_ina_ival[9:0] */
+#define HW_ATL_RPO_LRO_INA_IVAL_MSKN 0xFFF003FF
+/* Lower bit position of bitfield lro_ina_ival[9:0] */
+#define HW_ATL_RPO_LRO_INA_IVAL_SHIFT 10
+/* Width of bitfield lro_ina_ival[9:0] */
+#define HW_ATL_RPO_LRO_INA_IVAL_WIDTH 10
+/* Default value of bitfield lro_ina_ival[9:0] */
+#define HW_ATL_RPO_LRO_INA_IVAL_DEFAULT 0xA
+
+/* RX lro_max_ival[9:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "lro_max_ival[9:0]".
+ * PORT="pif_rpo_lro_max_ival_i[9:0]"
+ */
+
+/* Register address for bitfield lro_max_ival[9:0] */
+#define HW_ATL_RPO_LRO_MAX_IVAL_ADR 0x00005620
+/* Bitmask for bitfield lro_max_ival[9:0] */
+#define HW_ATL_RPO_LRO_MAX_IVAL_MSK 0x000003FF
+/* Inverted bitmask for bitfield lro_max_ival[9:0] */
+#define HW_ATL_RPO_LRO_MAX_IVAL_MSKN 0xFFFFFC00
+/* Lower bit position of bitfield lro_max_ival[9:0] */
+#define HW_ATL_RPO_LRO_MAX_IVAL_SHIFT 0
+/* Width of bitfield lro_max_ival[9:0] */
+#define HW_ATL_RPO_LRO_MAX_IVAL_WIDTH 10
+/* Default value of bitfield lro_max_ival[9:0] */
+#define HW_ATL_RPO_LRO_MAX_IVAL_DEFAULT 0x19
+
+/* TX dca{D}_cpuid[7:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "dca{D}_cpuid[7:0]".
+ * Parameter: DCA {D} | stride size 0x4 | range [0, 31]
+ * PORT="pif_tdm_dca0_cpuid_i[7:0]"
+ */
+
+/* Register address for bitfield dca{D}_cpuid[7:0] */
+#define HW_ATL_TDM_DCA_DCPUID_ADR(dca) (0x00008400 + (dca) * 0x4)
+/* Bitmask for bitfield dca{D}_cpuid[7:0] */
+#define HW_ATL_TDM_DCA_DCPUID_MSK 0x000000FF
+/* Inverted bitmask for bitfield dca{D}_cpuid[7:0] */
+#define HW_ATL_TDM_DCA_DCPUID_MSKN 0xFFFFFF00
+/* Lower bit position of bitfield dca{D}_cpuid[7:0] */
+#define HW_ATL_TDM_DCA_DCPUID_SHIFT 0
+/* Width of bitfield dca{D}_cpuid[7:0] */
+#define HW_ATL_TDM_DCA_DCPUID_WIDTH 8
+/* Default value of bitfield dca{D}_cpuid[7:0] */
+#define HW_ATL_TDM_DCA_DCPUID_DEFAULT 0x0
+
+/* TX dca{D}_desc_en Bitfield Definitions
+ * Preprocessor definitions for the bitfield "dca{D}_desc_en".
+ * Parameter: DCA {D} | stride size 0x4 | range [0, 31]
+ * PORT="pif_tdm_dca_desc_en_i[0]"
+ */
+
+/* Register address for bitfield dca{D}_desc_en */
+#define HW_ATL_TDM_DCA_DDESC_EN_ADR(dca) (0x00008400 + (dca) * 0x4)
+/* Bitmask for bitfield dca{D}_desc_en */
+#define HW_ATL_TDM_DCA_DDESC_EN_MSK 0x80000000
+/* Inverted bitmask for bitfield dca{D}_desc_en */
+#define HW_ATL_TDM_DCA_DDESC_EN_MSKN 0x7FFFFFFF
+/* Lower bit position of bitfield dca{D}_desc_en */
+#define HW_ATL_TDM_DCA_DDESC_EN_SHIFT 31
+/* Width of bitfield dca{D}_desc_en */
+#define HW_ATL_TDM_DCA_DDESC_EN_WIDTH 1
+/* Default value of bitfield dca{D}_desc_en */
+#define HW_ATL_TDM_DCA_DDESC_EN_DEFAULT 0x0
+
+/* TX desc{D}_en Bitfield Definitions
+ * Preprocessor definitions for the bitfield "desc{D}_en".
+ * Parameter: descriptor {D} | stride size 0x40 | range [0, 31]
+ * PORT="pif_tdm_desc_en_i[0]"
+ */
+
+/* Register address for bitfield desc{D}_en */
+#define HW_ATL_TDM_DESC_DEN_ADR(descriptor) (0x00007C08 + (descriptor) * 0x40)
+/* Bitmask for bitfield desc{D}_en */
+#define HW_ATL_TDM_DESC_DEN_MSK 0x80000000
+/* Inverted bitmask for bitfield desc{D}_en */
+#define HW_ATL_TDM_DESC_DEN_MSKN 0x7FFFFFFF
+/* Lower bit position of bitfield desc{D}_en */
+#define HW_ATL_TDM_DESC_DEN_SHIFT 31
+/* Width of bitfield desc{D}_en */
+#define HW_ATL_TDM_DESC_DEN_WIDTH 1
+/* Default value of bitfield desc{D}_en */
+#define HW_ATL_TDM_DESC_DEN_DEFAULT 0x0
+
+/* TX desc{D}_hd[C:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "desc{D}_hd[C:0]".
+ * Parameter: descriptor {D} | stride size 0x40 | range [0, 31]
+ * PORT="tdm_pif_desc0_hd_o[12:0]"
+ */
+
+/* Register address for bitfield desc{D}_hd[C:0] */
+#define HW_ATL_TDM_DESC_DHD_ADR(descriptor) (0x00007C0C + (descriptor) * 0x40)
+/* Bitmask for bitfield desc{D}_hd[C:0] */
+#define HW_ATL_TDM_DESC_DHD_MSK 0x00001FFF
+/* Inverted bitmask for bitfield desc{D}_hd[C:0] */
+#define HW_ATL_TDM_DESC_DHD_MSKN 0xFFFFE000
+/* Lower bit position of bitfield desc{D}_hd[C:0] */
+#define HW_ATL_TDM_DESC_DHD_SHIFT 0
+/* Width of bitfield desc{D}_hd[C:0] */
+#define HW_ATL_TDM_DESC_DHD_WIDTH 13
+
+/* TX desc{D}_len[9:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "desc{D}_len[9:0]".
+ * Parameter: descriptor {D} | stride size 0x40 | range [0, 31]
+ * PORT="pif_tdm_desc0_len_i[9:0]"
+ */
+
+/* Register address for bitfield desc{D}_len[9:0] */
+#define HW_ATL_TDM_DESC_DLEN_ADR(descriptor) (0x00007C08 + (descriptor) * 0x40)
+/* Bitmask for bitfield desc{D}_len[9:0] */
+#define HW_ATL_TDM_DESC_DLEN_MSK 0x00001FF8
+/* Inverted bitmask for bitfield desc{D}_len[9:0] */
+#define HW_ATL_TDM_DESC_DLEN_MSKN 0xFFFFE007
+/* Lower bit position of bitfield desc{D}_len[9:0] */
+#define HW_ATL_TDM_DESC_DLEN_SHIFT 3
+/* Width of bitfield desc{D}_len[9:0] */
+#define HW_ATL_TDM_DESC_DLEN_WIDTH 10
+/* Default value of bitfield desc{D}_len[9:0] */
+#define HW_ATL_TDM_DESC_DLEN_DEFAULT 0x0
+
+/* TX desc{D}_wrb_thresh[6:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "desc{D}_wrb_thresh[6:0]".
+ * Parameter: descriptor {D} | stride size 0x40 | range [0, 31]
+ * PORT="pif_tdm_desc0_wrb_thresh_i[6:0]"
+ */
+
+/* Register address for bitfield desc{D}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESC_DWRB_THRESH_ADR(descriptor) \
+	(0x00007C18 + (descriptor) * 0x40)
+/* Bitmask for bitfield desc{D}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESC_DWRB_THRESH_MSK 0x00007F00
+/* Inverted bitmask for bitfield desc{D}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESC_DWRB_THRESH_MSKN 0xFFFF80FF
+/* Lower bit position of bitfield desc{D}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESC_DWRB_THRESH_SHIFT 8
+/* Width of bitfield desc{D}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESC_DWRB_THRESH_WIDTH 7
+/* Default value of bitfield desc{D}_wrb_thresh[6:0] */
+#define HW_ATL_TDM_DESC_DWRB_THRESH_DEFAULT 0x0
+
+/* TX tdm_int_mod_en Bitfield Definitions
+ * Preprocessor definitions for the bitfield "tdm_int_mod_en".
+ * PORT="pif_tdm_int_mod_en_i"
+ */
+
+/* Register address for bitfield tdm_int_mod_en */
+#define HW_ATL_TDM_INT_MOD_EN_ADR 0x00007B40
+/* Bitmask for bitfield tdm_int_mod_en */
+#define HW_ATL_TDM_INT_MOD_EN_MSK 0x00000010
+/* Inverted bitmask for bitfield tdm_int_mod_en */
+#define HW_ATL_TDM_INT_MOD_EN_MSKN 0xFFFFFFEF
+/* Lower bit position of bitfield tdm_int_mod_en */
+#define HW_ATL_TDM_INT_MOD_EN_SHIFT 4
+/* Width of bitfield tdm_int_mod_en */
+#define HW_ATL_TDM_INT_MOD_EN_WIDTH 1
+/* Default value of bitfield tdm_int_mod_en */
+#define HW_ATL_TDM_INT_MOD_EN_DEFAULT 0x0
+
+/* TX lso_tcp_flag_mid[B:0] Bitfield Definitions
+ * Preprocessor definitions for the bitfield "lso_tcp_flag_mid[B:0]".
+ * PORT="pif_thm_lso_tcp_flag_mid_i[11:0]"
+ */
+/* register address for bitfield lso_tcp_flag_mid[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_ADR 0x00007820
+/* bitmask for bitfield lso_tcp_flag_mid[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_MSK 0x0fff0000
+/* inverted bitmask for bitfield lso_tcp_flag_mid[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_MSKN 0xf000ffff
+/* lower bit position of bitfield lso_tcp_flag_mid[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_SHIFT 16
+/* width of bitfield lso_tcp_flag_mid[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_WIDTH 12
+/* default value of bitfield lso_tcp_flag_mid[b:0] */
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_DEFAULT 0x0
+
+/* tx tx_buf_en bitfield definitions
+ * preprocessor definitions for the bitfield "tx_buf_en".
+ * port="pif_tpb_tx_buf_en_i"
+ */
+
+/* register address for bitfield tx_buf_en */
+#define HW_ATL_TPB_TX_BUF_EN_ADR 0x00007900
+/* bitmask for bitfield tx_buf_en */
+#define HW_ATL_TPB_TX_BUF_EN_MSK 0x00000001
+/* inverted bitmask for bitfield tx_buf_en */
+#define HW_ATL_TPB_TX_BUF_EN_MSKN 0xfffffffe
+/* lower bit position of bitfield tx_buf_en */
+#define HW_ATL_TPB_TX_BUF_EN_SHIFT 0
+/* width of bitfield tx_buf_en */
+#define HW_ATL_TPB_TX_BUF_EN_WIDTH 1
+/* default value of bitfield tx_buf_en */
+#define HW_ATL_TPB_TX_BUF_EN_DEFAULT 0x0
+
+/* register address for bitfield tx_tc_mode */
+#define HW_ATL_TPB_TX_TC_MODE_ADDR 0x00007900
+/* bitmask for bitfield tx_tc_mode */
+#define HW_ATL_TPB_TX_TC_MODE_MSK 0x00000100
+/* inverted bitmask for bitfield tx_tc_mode */
+#define HW_ATL_TPB_TX_TC_MODE_MSKN 0xFFFFFEFF
+/* lower bit position of bitfield tx_tc_mode */
+#define HW_ATL_TPB_TX_TC_MODE_SHIFT 8
+/* width of bitfield tx_tc_mode */
+#define HW_ATL_TPB_TX_TC_MODE_WIDTH 1
+/* default value of bitfield tx_tc_mode */
+#define HW_ATL_TPB_TX_TC_MODE_DEFAULT 0x0
+
+/* tx tx{b}_hi_thresh[c:0] bitfield definitions
+ * preprocessor definitions for the bitfield "tx{b}_hi_thresh[c:0]".
+ * parameter: buffer {b} | stride size 0x10 | range [0, 7]
+ * port="pif_tpb_tx0_hi_thresh_i[12:0]"
+ */
+
+/* register address for bitfield tx{b}_hi_thresh[c:0] */
+#define HW_ATL_TPB_TXBHI_THRESH_ADR(buffer) (0x00007914 + (buffer) * 0x10)
+/* bitmask for bitfield tx{b}_hi_thresh[c:0] */
+#define HW_ATL_TPB_TXBHI_THRESH_MSK 0x1fff0000
+/* inverted bitmask for bitfield tx{b}_hi_thresh[c:0] */
+#define HW_ATL_TPB_TXBHI_THRESH_MSKN 0xe000ffff
+/* lower bit position of bitfield tx{b}_hi_thresh[c:0] */
+#define HW_ATL_TPB_TXBHI_THRESH_SHIFT 16
+/* width of bitfield tx{b}_hi_thresh[c:0] */
+#define HW_ATL_TPB_TXBHI_THRESH_WIDTH 13
+/* default value of bitfield tx{b}_hi_thresh[c:0] */
+#define HW_ATL_TPB_TXBHI_THRESH_DEFAULT 0x0
+
+/* tx tx{b}_lo_thresh[c:0] bitfield definitions
+ * preprocessor definitions for the bitfield "tx{b}_lo_thresh[c:0]".
+ * parameter: buffer {b} | stride size 0x10 | range [0, 7]
+ * port="pif_tpb_tx0_lo_thresh_i[12:0]"
+ */
+
+/* register address for bitfield tx{b}_lo_thresh[c:0] */
+#define HW_ATL_TPB_TXBLO_THRESH_ADR(buffer) (0x00007914 + (buffer) * 0x10)
+/* bitmask for bitfield tx{b}_lo_thresh[c:0] */
+#define HW_ATL_TPB_TXBLO_THRESH_MSK 0x00001fff
+/* inverted bitmask for bitfield tx{b}_lo_thresh[c:0] */
+#define HW_ATL_TPB_TXBLO_THRESH_MSKN 0xffffe000
+/* lower bit position of bitfield tx{b}_lo_thresh[c:0] */
+#define HW_ATL_TPB_TXBLO_THRESH_SHIFT 0
+/* width of bitfield tx{b}_lo_thresh[c:0] */
+#define HW_ATL_TPB_TXBLO_THRESH_WIDTH 13
+/* default value of bitfield tx{b}_lo_thresh[c:0] */
+#define HW_ATL_TPB_TXBLO_THRESH_DEFAULT 0x0
+
+/* tx dma_sys_loopback bitfield definitions
+ * preprocessor definitions for the bitfield "dma_sys_loopback".
+ * port="pif_tpb_dma_sys_lbk_i"
+ */
+
+/* register address for bitfield dma_sys_loopback */
+#define HW_ATL_TPB_DMA_SYS_LBK_ADR 0x00007000
+/* bitmask for bitfield dma_sys_loopback */
+#define HW_ATL_TPB_DMA_SYS_LBK_MSK 0x00000040
+/* inverted bitmask for bitfield dma_sys_loopback */
+#define HW_ATL_TPB_DMA_SYS_LBK_MSKN 0xffffffbf
+/* lower bit position of bitfield dma_sys_loopback */
+#define HW_ATL_TPB_DMA_SYS_LBK_SHIFT 6
+/* width of bitfield dma_sys_loopback */
+#define HW_ATL_TPB_DMA_SYS_LBK_WIDTH 1
+/* default value of bitfield dma_sys_loopback */
+#define HW_ATL_TPB_DMA_SYS_LBK_DEFAULT 0x0
+
+/* tx tx{b}_buf_size[7:0] bitfield definitions
+ * preprocessor definitions for the bitfield "tx{b}_buf_size[7:0]".
+ * parameter: buffer {b} | stride size 0x10 | range [0, 7]
+ * port="pif_tpb_tx0_buf_size_i[7:0]"
+ */
+
+/* register address for bitfield tx{b}_buf_size[7:0] */
+#define HW_ATL_TPB_TXBBUF_SIZE_ADR(buffer) (0x00007910 + (buffer) * 0x10)
+/* bitmask for bitfield tx{b}_buf_size[7:0] */
+#define HW_ATL_TPB_TXBBUF_SIZE_MSK 0x000000ff
+/* inverted bitmask for bitfield tx{b}_buf_size[7:0] */
+#define HW_ATL_TPB_TXBBUF_SIZE_MSKN 0xffffff00
+/* lower bit position of bitfield tx{b}_buf_size[7:0] */
+#define HW_ATL_TPB_TXBBUF_SIZE_SHIFT 0
+/* width of bitfield tx{b}_buf_size[7:0] */
+#define HW_ATL_TPB_TXBBUF_SIZE_WIDTH 8
+/* default value of bitfield tx{b}_buf_size[7:0] */
+#define HW_ATL_TPB_TXBBUF_SIZE_DEFAULT 0x0
+
+/* tx tx_scp_ins_en bitfield definitions
+ * preprocessor definitions for the bitfield "tx_scp_ins_en".
+ * port="pif_tpb_scp_ins_en_i"
+ */
+
+/* register address for bitfield tx_scp_ins_en */
+#define HW_ATL_TPB_TX_SCP_INS_EN_ADR 0x00007900
+/* bitmask for bitfield tx_scp_ins_en */
+#define HW_ATL_TPB_TX_SCP_INS_EN_MSK 0x00000004
+/* inverted bitmask for bitfield tx_scp_ins_en */
+#define HW_ATL_TPB_TX_SCP_INS_EN_MSKN 0xfffffffb
+/* lower bit position of bitfield tx_scp_ins_en */
+#define HW_ATL_TPB_TX_SCP_INS_EN_SHIFT 2
+/* width of bitfield tx_scp_ins_en */
+#define HW_ATL_TPB_TX_SCP_INS_EN_WIDTH 1
+/* default value of bitfield tx_scp_ins_en */
+#define HW_ATL_TPB_TX_SCP_INS_EN_DEFAULT 0x0
+
+/* tx ipv4_chk_en bitfield definitions
+ * preprocessor definitions for the bitfield "ipv4_chk_en".
+ * port="pif_tpo_ipv4_chk_en_i"
+ */
+
+/* register address for bitfield ipv4_chk_en */
+#define HW_ATL_TPO_IPV4CHK_EN_ADR 0x00007800
+/* bitmask for bitfield ipv4_chk_en */
+#define HW_ATL_TPO_IPV4CHK_EN_MSK 0x00000002
+/* inverted bitmask for bitfield ipv4_chk_en */
+#define HW_ATL_TPO_IPV4CHK_EN_MSKN 0xfffffffd
+/* lower bit position of bitfield ipv4_chk_en */
+#define HW_ATL_TPO_IPV4CHK_EN_SHIFT 1
+/* width of bitfield ipv4_chk_en */
+#define HW_ATL_TPO_IPV4CHK_EN_WIDTH 1
+/* default value of bitfield ipv4_chk_en */
+#define HW_ATL_TPO_IPV4CHK_EN_DEFAULT 0x0
+
+/* tx l4_chk_en bitfield definitions
+ * preprocessor definitions for the bitfield "l4_chk_en".
+ * port="pif_tpo_l4_chk_en_i"
+ */
+
+/* register address for bitfield l4_chk_en */
+#define HW_ATL_TPOL4CHK_EN_ADR 0x00007800
+/* bitmask for bitfield l4_chk_en */
+#define HW_ATL_TPOL4CHK_EN_MSK 0x00000001
+/* inverted bitmask for bitfield l4_chk_en */
+#define HW_ATL_TPOL4CHK_EN_MSKN 0xfffffffe
+/* lower bit position of bitfield l4_chk_en */
+#define HW_ATL_TPOL4CHK_EN_SHIFT 0
+/* width of bitfield l4_chk_en */
+#define HW_ATL_TPOL4CHK_EN_WIDTH 1
+/* default value of bitfield l4_chk_en */
+#define HW_ATL_TPOL4CHK_EN_DEFAULT 0x0
+
+/* tx pkt_sys_loopback bitfield definitions
+ * preprocessor definitions for the bitfield "pkt_sys_loopback".
+ * port="pif_tpo_pkt_sys_lbk_i"
+ */
+
+/* register address for bitfield pkt_sys_loopback */
+#define HW_ATL_TPO_PKT_SYS_LBK_ADR 0x00007000
+/* bitmask for bitfield pkt_sys_loopback */
+#define HW_ATL_TPO_PKT_SYS_LBK_MSK 0x00000080
+/* inverted bitmask for bitfield pkt_sys_loopback */
+#define HW_ATL_TPO_PKT_SYS_LBK_MSKN 0xffffff7f
+/* lower bit position of bitfield pkt_sys_loopback */
+#define HW_ATL_TPO_PKT_SYS_LBK_SHIFT 7
+/* width of bitfield pkt_sys_loopback */
+#define HW_ATL_TPO_PKT_SYS_LBK_WIDTH 1
+/* default value of bitfield pkt_sys_loopback */
+#define HW_ATL_TPO_PKT_SYS_LBK_DEFAULT 0x0
+
+/* tx data_tc_arb_mode bitfield definitions
+ * preprocessor definitions for the bitfield "data_tc_arb_mode".
+ * port="pif_tps_data_tc_arb_mode_i"
+ */
+
+/* register address for bitfield data_tc_arb_mode */
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_ADR 0x00007100
+/* bitmask for bitfield data_tc_arb_mode */
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_MSK 0x00000001
+/* inverted bitmask for bitfield data_tc_arb_mode */
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_MSKN 0xfffffffe
+/* lower bit position of bitfield data_tc_arb_mode */
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_SHIFT 0
+/* width of bitfield data_tc_arb_mode */
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_WIDTH 1
+/* default value of bitfield data_tc_arb_mode */
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_DEFAULT 0x0
+
+/* tx desc_rate_ta_rst bitfield definitions
+ * preprocessor definitions for the bitfield "desc_rate_ta_rst".
+ * port="pif_tps_desc_rate_ta_rst_i"
+ */
+
+/* register address for bitfield desc_rate_ta_rst */
+#define HW_ATL_TPS_DESC_RATE_TA_RST_ADR 0x00007310
+/* bitmask for bitfield desc_rate_ta_rst */
+#define HW_ATL_TPS_DESC_RATE_TA_RST_MSK 0x80000000
+/* inverted bitmask for bitfield desc_rate_ta_rst */
+#define HW_ATL_TPS_DESC_RATE_TA_RST_MSKN 0x7fffffff
+/* lower bit position of bitfield desc_rate_ta_rst */
+#define HW_ATL_TPS_DESC_RATE_TA_RST_SHIFT 31
+/* width of bitfield desc_rate_ta_rst */
+#define HW_ATL_TPS_DESC_RATE_TA_RST_WIDTH 1
+/* default value of bitfield desc_rate_ta_rst */
+#define HW_ATL_TPS_DESC_RATE_TA_RST_DEFAULT 0x0
+
+/* tx desc_rate_limit[a:0] bitfield definitions
+ * preprocessor definitions for the bitfield "desc_rate_limit[a:0]".
+ * port="pif_tps_desc_rate_lim_i[10:0]"
+ */
+
+/* register address for bitfield desc_rate_limit[a:0] */
+#define HW_ATL_TPS_DESC_RATE_LIM_ADR 0x00007310
+/* bitmask for bitfield desc_rate_limit[a:0] */
+#define HW_ATL_TPS_DESC_RATE_LIM_MSK 0x000007ff
+/* inverted bitmask for bitfield desc_rate_limit[a:0] */
+#define HW_ATL_TPS_DESC_RATE_LIM_MSKN 0xfffff800
+/* lower bit position of bitfield desc_rate_limit[a:0] */
+#define HW_ATL_TPS_DESC_RATE_LIM_SHIFT 0
+/* width of bitfield desc_rate_limit[a:0] */
+#define HW_ATL_TPS_DESC_RATE_LIM_WIDTH 11
+/* default value of bitfield desc_rate_limit[a:0] */
+#define HW_ATL_TPS_DESC_RATE_LIM_DEFAULT 0x0
+
+/* tx desc_tc_arb_mode[1:0] bitfield definitions
+ * preprocessor definitions for the bitfield "desc_tc_arb_mode[1:0]".
+ * port="pif_tps_desc_tc_arb_mode_i[1:0]"
+ */
+
+/* register address for bitfield desc_tc_arb_mode[1:0] */
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_ADR 0x00007200
+/* bitmask for bitfield desc_tc_arb_mode[1:0] */
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_MSK 0x00000003
+/* inverted bitmask for bitfield desc_tc_arb_mode[1:0] */
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_MSKN 0xfffffffc
+/* lower bit position of bitfield desc_tc_arb_mode[1:0] */
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_SHIFT 0
+/* width of bitfield desc_tc_arb_mode[1:0] */
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_WIDTH 2
+/* default value of bitfield desc_tc_arb_mode[1:0] */
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_DEFAULT 0x0
+
+/* tx desc_tc{t}_credit_max[b:0] bitfield definitions
+ * preprocessor definitions for the bitfield "desc_tc{t}_credit_max[b:0]".
+ * parameter: tc {t} | stride size 0x4 | range [0, 7]
+ * port="pif_tps_desc_tc0_credit_max_i[11:0]"
+ */
+
+/* register address for bitfield desc_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_ADR(tc) (0x00007210 + (tc) * 0x4)
+/* bitmask for bitfield desc_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_MSK 0x0fff0000
+/* inverted bitmask for bitfield desc_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_MSKN 0xf000ffff
+/* lower bit position of bitfield desc_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_SHIFT 16
+/* width of bitfield desc_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_WIDTH 12
+/* default value of bitfield desc_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_DEFAULT 0x0
+
+/* tx desc_tc{t}_weight[8:0] bitfield definitions
+ * preprocessor definitions for the bitfield "desc_tc{t}_weight[8:0]".
+ * parameter: tc {t} | stride size 0x4 | range [0, 7]
+ * port="pif_tps_desc_tc0_weight_i[8:0]"
+ */
+
+/* register address for bitfield desc_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DESC_TCTWEIGHT_ADR(tc) (0x00007210 + (tc) * 0x4)
+/* bitmask for bitfield desc_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DESC_TCTWEIGHT_MSK 0x000001ff
+/* inverted bitmask for bitfield desc_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DESC_TCTWEIGHT_MSKN 0xfffffe00
+/* lower bit position of bitfield desc_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DESC_TCTWEIGHT_SHIFT 0
+/* width of bitfield desc_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DESC_TCTWEIGHT_WIDTH 9
+/* default value of bitfield desc_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DESC_TCTWEIGHT_DEFAULT 0x0
+
+/* tx desc_vm_arb_mode bitfield definitions
+ * preprocessor definitions for the bitfield "desc_vm_arb_mode".
+ * port="pif_tps_desc_vm_arb_mode_i"
+ */
+
+/* register address for bitfield desc_vm_arb_mode */
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_ADR 0x00007300
+/* bitmask for bitfield desc_vm_arb_mode */
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_MSK 0x00000001
+/* inverted bitmask for bitfield desc_vm_arb_mode */
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_MSKN 0xfffffffe
+/* lower bit position of bitfield desc_vm_arb_mode */
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_SHIFT 0
+/* width of bitfield desc_vm_arb_mode */
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_WIDTH 1
+/* default value of bitfield desc_vm_arb_mode */
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_DEFAULT 0x0
+
+/* tx data_tc{t}_credit_max[b:0] bitfield definitions
+ * preprocessor definitions for the bitfield "data_tc{t}_credit_max[b:0]".
+ * parameter: tc {t} | stride size 0x4 | range [0, 7]
+ * port="pif_tps_data_tc0_credit_max_i[11:0]"
+ */
+
+/* register address for bitfield data_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_ADR(tc) (0x00007110 + (tc) * 0x4)
+/* bitmask for bitfield data_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_MSK 0x0fff0000
+/* inverted bitmask for bitfield data_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_MSKN 0xf000ffff
+/* lower bit position of bitfield data_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_SHIFT 16
+/* width of bitfield data_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_WIDTH 12
+/* default value of bitfield data_tc{t}_credit_max[b:0] */
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_DEFAULT 0x0
+
+/* tx data_tc{t}_weight[8:0] bitfield definitions
+ * preprocessor definitions for the bitfield "data_tc{t}_weight[8:0]".
+ * parameter: tc {t} | stride size 0x4 | range [0, 7]
+ * port="pif_tps_data_tc0_weight_i[8:0]"
+ */
+
+/* register address for bitfield data_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DATA_TCTWEIGHT_ADR(tc) (0x00007110 + (tc) * 0x4)
+/* bitmask for bitfield data_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DATA_TCTWEIGHT_MSK 0x000001ff
+/* inverted bitmask for bitfield data_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DATA_TCTWEIGHT_MSKN 0xfffffe00
+/* lower bit position of bitfield data_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DATA_TCTWEIGHT_SHIFT 0
+/* width of bitfield data_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DATA_TCTWEIGHT_WIDTH 9
+/* default value of bitfield data_tc{t}_weight[8:0] */
+#define HW_ATL_TPS_DATA_TCTWEIGHT_DEFAULT 0x0
+
+/* tx reg_res_dsbl bitfield definitions
+ * preprocessor definitions for the bitfield "reg_res_dsbl".
+ * port="pif_tx_reg_res_dsbl_i"
+ */
+
+/* register address for bitfield reg_res_dsbl */
+#define HW_ATL_TX_REG_RES_DSBL_ADR 0x00007000
+/* bitmask for bitfield reg_res_dsbl */
+#define HW_ATL_TX_REG_RES_DSBL_MSK 0x20000000
+/* inverted bitmask for bitfield reg_res_dsbl */
+#define HW_ATL_TX_REG_RES_DSBL_MSKN 0xdfffffff
+/* lower bit position of bitfield reg_res_dsbl */
+#define HW_ATL_TX_REG_RES_DSBL_SHIFT 29
+/* width of bitfield reg_res_dsbl */
+#define HW_ATL_TX_REG_RES_DSBL_WIDTH 1
+/* default value of bitfield reg_res_dsbl */
+#define HW_ATL_TX_REG_RES_DSBL_DEFAULT 0x1
+
+/* mac_phy register access busy bitfield definitions
+ * preprocessor definitions for the bitfield "register access busy".
+ * port="msm_pif_reg_busy_o"
+ */
+
+/* register address for bitfield register access busy */
+#define HW_ATL_MSM_REG_ACCESS_BUSY_ADR 0x00004400
+/* bitmask for bitfield register access busy */
+#define HW_ATL_MSM_REG_ACCESS_BUSY_MSK 0x00001000
+/* inverted bitmask for bitfield register access busy */
+#define HW_ATL_MSM_REG_ACCESS_BUSY_MSKN 0xffffefff
+/* lower bit position of bitfield register access busy */
+#define HW_ATL_MSM_REG_ACCESS_BUSY_SHIFT 12
+/* width of bitfield register access busy */
+#define HW_ATL_MSM_REG_ACCESS_BUSY_WIDTH 1
+
+/* mac_phy msm register address[7:0] bitfield definitions
+ * preprocessor definitions for the bitfield "msm register address[7:0]".
+ * port="pif_msm_reg_addr_i[7:0]"
+ */
+
+/* register address for bitfield msm register address[7:0] */
+#define HW_ATL_MSM_REG_ADDR_ADR 0x00004400
+/* bitmask for bitfield msm register address[7:0] */
+#define HW_ATL_MSM_REG_ADDR_MSK 0x000000ff
+/* inverted bitmask for bitfield msm register address[7:0] */
+#define HW_ATL_MSM_REG_ADDR_MSKN 0xffffff00
+/* lower bit position of bitfield msm register address[7:0] */
+#define HW_ATL_MSM_REG_ADDR_SHIFT 0
+/* width of bitfield msm register address[7:0] */
+#define HW_ATL_MSM_REG_ADDR_WIDTH 8
+/* default value of bitfield msm register address[7:0] */
+#define HW_ATL_MSM_REG_ADDR_DEFAULT 0x0
+
+/* mac_phy register read strobe bitfield definitions
+ * preprocessor definitions for the bitfield "register read strobe".
+ * port="pif_msm_reg_rden_i"
+ */
+
+/* register address for bitfield register read strobe */
+#define HW_ATL_MSM_REG_RD_STROBE_ADR 0x00004400
+/* bitmask for bitfield register read strobe */
+#define HW_ATL_MSM_REG_RD_STROBE_MSK 0x00000200
+/* inverted bitmask for bitfield register read strobe */
+#define HW_ATL_MSM_REG_RD_STROBE_MSKN 0xfffffdff
+/* lower bit position of bitfield register read strobe */
+#define HW_ATL_MSM_REG_RD_STROBE_SHIFT 9
+/* width of bitfield register read strobe */
+#define HW_ATL_MSM_REG_RD_STROBE_WIDTH 1
+/* default value of bitfield register read strobe */
+#define HW_ATL_MSM_REG_RD_STROBE_DEFAULT 0x0
+
+/* mac_phy msm register read data[31:0] bitfield definitions
+ * preprocessor definitions for the bitfield "msm register read data[31:0]".
+ * port="msm_pif_reg_rd_data_o[31:0]"
+ */
+
+/* register address for bitfield msm register read data[31:0] */
+#define HW_ATL_MSM_REG_RD_DATA_ADR 0x00004408
+/* bitmask for bitfield msm register read data[31:0] */
+#define HW_ATL_MSM_REG_RD_DATA_MSK 0xffffffff
+/* inverted bitmask for bitfield msm register read data[31:0] */
+#define HW_ATL_MSM_REG_RD_DATA_MSKN 0x00000000
+/* lower bit position of bitfield msm register read data[31:0] */
+#define HW_ATL_MSM_REG_RD_DATA_SHIFT 0
+/* width of bitfield msm register read data[31:0] */
+#define HW_ATL_MSM_REG_RD_DATA_WIDTH 32
+
+/* mac_phy msm register write data[31:0] bitfield definitions
+ * preprocessor definitions for the bitfield "msm register write data[31:0]".
+ * port="pif_msm_reg_wr_data_i[31:0]"
+ */
+
+/* register address for bitfield msm register write data[31:0] */
+#define HW_ATL_MSM_REG_WR_DATA_ADR 0x00004404
+/* bitmask for bitfield msm register write data[31:0] */
+#define HW_ATL_MSM_REG_WR_DATA_MSK 0xffffffff
+/* inverted bitmask for bitfield msm register write data[31:0] */
+#define HW_ATL_MSM_REG_WR_DATA_MSKN 0x00000000
+/* lower bit position of bitfield msm register write data[31:0] */
+#define HW_ATL_MSM_REG_WR_DATA_SHIFT 0
+/* width of bitfield msm register write data[31:0] */
+#define HW_ATL_MSM_REG_WR_DATA_WIDTH 32
+/* default value of bitfield msm register write data[31:0] */
+#define HW_ATL_MSM_REG_WR_DATA_DEFAULT 0x0
+
+/* mac_phy register write strobe bitfield definitions
+ * preprocessor definitions for the bitfield "register write strobe".
+ * port="pif_msm_reg_wren_i"
+ */
+
+/* register address for bitfield register write strobe */
+#define HW_ATL_MSM_REG_WR_STROBE_ADR 0x00004400
+/* bitmask for bitfield register write strobe */
+#define HW_ATL_MSM_REG_WR_STROBE_MSK 0x00000100
+/* inverted bitmask for bitfield register write strobe */
+#define HW_ATL_MSM_REG_WR_STROBE_MSKN 0xfffffeff
+/* lower bit position of bitfield register write strobe */
+#define HW_ATL_MSM_REG_WR_STROBE_SHIFT 8
+/* width of bitfield register write strobe */
+#define HW_ATL_MSM_REG_WR_STROBE_WIDTH 1
+/* default value of bitfield register write strobe */
+#define HW_ATL_MSM_REG_WR_STROBE_DEFAULT 0x0
+
+/* mif soft reset bitfield definitions
+ * preprocessor definitions for the bitfield "soft reset".
+ * port="pif_glb_res_i"
+ */
+
+/* register address for bitfield soft reset */
+#define HW_ATL_GLB_SOFT_RES_ADR 0x00000000
+/* bitmask for bitfield soft reset */
+#define HW_ATL_GLB_SOFT_RES_MSK 0x00008000
+/* inverted bitmask for bitfield soft reset */
+#define HW_ATL_GLB_SOFT_RES_MSKN 0xffff7fff
+/* lower bit position of bitfield soft reset */
+#define HW_ATL_GLB_SOFT_RES_SHIFT 15
+/* width of bitfield soft reset */
+#define HW_ATL_GLB_SOFT_RES_WIDTH 1
+/* default value of bitfield soft reset */
+#define HW_ATL_GLB_SOFT_RES_DEFAULT 0x0
+
+/* mif register reset disable bitfield definitions
+ * preprocessor definitions for the bitfield "register reset disable".
+ * port="pif_glb_reg_res_dsbl_i"
+ */
+
+/* register address for bitfield register reset disable */
+#define HW_ATL_GLB_REG_RES_DIS_ADR 0x00000000
+/* bitmask for bitfield register reset disable */
+#define HW_ATL_GLB_REG_RES_DIS_MSK 0x00004000
+/* inverted bitmask for bitfield register reset disable */
+#define HW_ATL_GLB_REG_RES_DIS_MSKN 0xffffbfff
+/* lower bit position of bitfield register reset disable */
+#define HW_ATL_GLB_REG_RES_DIS_SHIFT 14
+/* width of bitfield register reset disable */
+#define HW_ATL_GLB_REG_RES_DIS_WIDTH 1
+/* default value of bitfield register reset disable */
+#define HW_ATL_GLB_REG_RES_DIS_DEFAULT 0x1
+
+/* tx dma debug control definitions */
+#define HW_ATL_TX_DMA_DEBUG_CTL_ADR 0x00008920u
+
+/* tx dma descriptor base address msw definitions */
+#define HW_ATL_TX_DMA_DESC_BASE_ADDRMSW_ADR(descriptor) \
+			(0x00007c04u + (descriptor) * 0x40)
+
+/* tx dma total request limit */
+#define HW_ATL_TX_DMA_TOTAL_REQ_LIMIT_ADR 0x00007b20u
+
+/* tx interrupt moderation control register definitions
+ * Preprocessor definitions for TX Interrupt Moderation Control Register
+ * Base Address: 0x00008980
+ * Parameter: queue {Q} | stride size 0x4 | range [0, 31]
+ */
+
+#define HW_ATL_TX_INTR_MODERATION_CTL_ADR(queue) (0x00008980u + (queue) * 0x4)
+
+/* pcie reg_res_dsbl bitfield definitions
+ * preprocessor definitions for the bitfield "reg_res_dsbl".
+ * port="pif_pci_reg_res_dsbl_i"
+ */
+
+/* register address for bitfield reg_res_dsbl */
+#define HW_ATL_PCI_REG_RES_DSBL_ADR 0x00001000
+/* bitmask for bitfield reg_res_dsbl */
+#define HW_ATL_PCI_REG_RES_DSBL_MSK 0x20000000
+/* inverted bitmask for bitfield reg_res_dsbl */
+#define HW_ATL_PCI_REG_RES_DSBL_MSKN 0xdfffffff
+/* lower bit position of bitfield reg_res_dsbl */
+#define HW_ATL_PCI_REG_RES_DSBL_SHIFT 29
+/* width of bitfield reg_res_dsbl */
+#define HW_ATL_PCI_REG_RES_DSBL_WIDTH 1
+/* default value of bitfield reg_res_dsbl */
+#define HW_ATL_PCI_REG_RES_DSBL_DEFAULT 0x1
+
+/* PCI core control register */
+#define HW_ATL_PCI_REG_CONTROL6_ADR 0x1014u
+
+/* global microprocessor scratch pad definitions */
+#define HW_ATL_GLB_CPU_SCRATCH_SCP_ADR(scratch_scp) \
+	(0x00000300u + (scratch_scp) * 0x4)
+
+/* register address for bitfield uP Force Interrupt */
+#define HW_ATL_MCP_UP_FORCE_INTERRUPT_ADR 0x00000404
+/* bitmask for bitfield uP Force Interrupt */
+#define HW_ATL_MCP_UP_FORCE_INTERRUPT_MSK 0x00000002
+/* inverted bitmask for bitfield uP Force Interrupt */
+#define HW_ATL_MCP_UP_FORCE_INTERRUPT_MSKN 0xFFFFFFFD
+/* lower bit position of bitfield uP Force Interrupt */
+#define HW_ATL_MCP_UP_FORCE_INTERRUPT_SHIFT 1
+/* width of bitfield uP Force Interrupt */
+#define HW_ATL_MCP_UP_FORCE_INTERRUPT_WIDTH 1
+/* default value of bitfield uP Force Interrupt */
+#define HW_ATL_MCP_UP_FORCE_INTERRUPT_DEFAULT 0x0
+
+#endif /* HW_ATL_LLH_INTERNAL_H */
diff --git a/drivers/net/atlantic/hw_atl/hw_atl_utils.c b/drivers/net/atlantic/hw_atl/hw_atl_utils.c
new file mode 100644
index 00000000..f11093a5
--- /dev/null
+++ b/drivers/net/atlantic/hw_atl/hw_atl_utils.c
@@ -0,0 +1,942 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+/* Copyright (C) 2014-2017 aQuantia Corporation. */
+
+/* File hw_atl_utils.c: Definition of common functions for Atlantic hardware
+ * abstraction layer.
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdarg.h>
+#include <inttypes.h>
+#include <rte_ether.h>
+#include "../atl_hw_regs.h"
+
+#include "hw_atl_llh.h"
+#include "hw_atl_llh_internal.h"
+#include "../atl_logs.h"
+
+#define HW_ATL_UCP_0X370_REG    0x0370U
+
+#define HW_ATL_MIF_CMD          0x0200U
+#define HW_ATL_MIF_ADDR         0x0208U
+#define HW_ATL_MIF_VAL          0x020CU
+
+#define HW_ATL_FW_SM_RAM        0x2U
+#define HW_ATL_MPI_FW_VERSION	0x18
+#define HW_ATL_MPI_CONTROL_ADR  0x0368U
+#define HW_ATL_MPI_STATE_ADR    0x036CU
+
+#define HW_ATL_MPI_STATE_MSK      0x00FFU
+#define HW_ATL_MPI_STATE_SHIFT    0U
+#define HW_ATL_MPI_SPEED_MSK      0x00FF0000U
+#define HW_ATL_MPI_SPEED_SHIFT    16U
+#define HW_ATL_MPI_DIRTY_WAKE_MSK 0x02000000U
+
+#define HW_ATL_MPI_DAISY_CHAIN_STATUS	0x704
+#define HW_ATL_MPI_BOOT_EXIT_CODE	0x388
+
+#define HW_ATL_MAC_PHY_CONTROL	0x4000
+#define HW_ATL_MAC_PHY_MPI_RESET_BIT 0x1D
+
+#define HW_ATL_FW_VER_1X 0x01050006U
+#define HW_ATL_FW_VER_2X 0x02000000U
+#define HW_ATL_FW_VER_3X 0x03000000U
+
+#define FORCE_FLASHLESS 0
+
+static int hw_atl_utils_ver_match(u32 ver_expected, u32 ver_actual);
+static int hw_atl_utils_mpi_set_state(struct aq_hw_s *self,
+				enum hal_atl_utils_fw_state_e state);
+
+
+int hw_atl_utils_initfw(struct aq_hw_s *self, const struct aq_fw_ops **fw_ops)
+{
+	int err = 0;
+
+	err = hw_atl_utils_soft_reset(self);
+	if (err)
+		return err;
+
+	hw_atl_utils_hw_chip_features_init(self,
+					   &self->chip_features);
+
+	hw_atl_utils_get_fw_version(self, &self->fw_ver_actual);
+
+	if (hw_atl_utils_ver_match(HW_ATL_FW_VER_1X,
+				   self->fw_ver_actual) == 0) {
+		*fw_ops = &aq_fw_1x_ops;
+	} else if (hw_atl_utils_ver_match(HW_ATL_FW_VER_2X,
+					self->fw_ver_actual) == 0) {
+		*fw_ops = &aq_fw_2x_ops;
+	} else if (hw_atl_utils_ver_match(HW_ATL_FW_VER_3X,
+					self->fw_ver_actual) == 0) {
+		*fw_ops = &aq_fw_2x_ops;
+	} else {
+		PMD_DRV_LOG(ERR, "Bad FW version detected: %x\n",
+			  self->fw_ver_actual);
+		return -EOPNOTSUPP;
+	}
+	self->aq_fw_ops = *fw_ops;
+	err = self->aq_fw_ops->init(self);
+	return err;
+}
+
+static int hw_atl_utils_soft_reset_flb(struct aq_hw_s *self)
+{
+	u32 gsr, val;
+	int k = 0;
+
+	aq_hw_write_reg(self, 0x404, 0x40e1);
+	AQ_HW_SLEEP(50);
+
+	/* Cleanup SPI */
+	val = aq_hw_read_reg(self, 0x53C);
+	aq_hw_write_reg(self, 0x53C, val | 0x10);
+
+	gsr = aq_hw_read_reg(self, HW_ATL_GLB_SOFT_RES_ADR);
+	aq_hw_write_reg(self, HW_ATL_GLB_SOFT_RES_ADR, (gsr & 0xBFFF) | 0x8000);
+
+	/* Kickstart MAC */
+	aq_hw_write_reg(self, 0x404, 0x80e0);
+	aq_hw_write_reg(self, 0x32a8, 0x0);
+	aq_hw_write_reg(self, 0x520, 0x1);
+
+	/* Reset SPI again because of possible interrupted SPI burst */
+	val = aq_hw_read_reg(self, 0x53C);
+	aq_hw_write_reg(self, 0x53C, val | 0x10);
+	AQ_HW_SLEEP(10);
+	/* Clear SPI reset state */
+	aq_hw_write_reg(self, 0x53C, val & ~0x10);
+
+	aq_hw_write_reg(self, 0x404, 0x180e0);
+
+	for (k = 0; k < 1000; k++) {
+		u32 flb_status = aq_hw_read_reg(self,
+						HW_ATL_MPI_DAISY_CHAIN_STATUS);
+
+		flb_status = flb_status & 0x10;
+		if (flb_status)
+			break;
+		AQ_HW_SLEEP(10);
+	}
+	if (k == 1000) {
+		PMD_DRV_LOG(ERR, "MAC kickstart failed\n");
+		return -EIO;
+	}
+
+	/* FW reset */
+	aq_hw_write_reg(self, 0x404, 0x80e0);
+	AQ_HW_SLEEP(50);
+	aq_hw_write_reg(self, 0x3a0, 0x1);
+
+	/* Kickstart PHY - skipped */
+
+	/* Global software reset*/
+	hw_atl_rx_rx_reg_res_dis_set(self, 0U);
+	hw_atl_tx_tx_reg_res_dis_set(self, 0U);
+	aq_hw_write_reg_bit(self, HW_ATL_MAC_PHY_CONTROL,
+			    BIT(HW_ATL_MAC_PHY_MPI_RESET_BIT),
+			    HW_ATL_MAC_PHY_MPI_RESET_BIT, 0x0);
+	gsr = aq_hw_read_reg(self, HW_ATL_GLB_SOFT_RES_ADR);
+	aq_hw_write_reg(self, HW_ATL_GLB_SOFT_RES_ADR, (gsr & 0xBFFF) | 0x8000);
+
+	for (k = 0; k < 1000; k++) {
+		u32 fw_state = aq_hw_read_reg(self, HW_ATL_MPI_FW_VERSION);
+
+		if (fw_state)
+			break;
+		AQ_HW_SLEEP(10);
+	}
+	if (k == 1000) {
+		PMD_DRV_LOG(ERR, "FW kickstart failed\n");
+		return -EIO;
+	}
+	/* Old FW requires fixed delay after init */
+	AQ_HW_SLEEP(15);
+
+	return 0;
+}
+
+static int hw_atl_utils_soft_reset_rbl(struct aq_hw_s *self)
+{
+	u32 gsr, val, rbl_status;
+	int k;
+
+	aq_hw_write_reg(self, 0x404, 0x40e1);
+	aq_hw_write_reg(self, 0x3a0, 0x1);
+	aq_hw_write_reg(self, 0x32a8, 0x0);
+
+	/* Alter RBL status */
+	aq_hw_write_reg(self, 0x388, 0xDEAD);
+
+	/* Cleanup SPI */
+	val = aq_hw_read_reg(self, 0x53C);
+	aq_hw_write_reg(self, 0x53C, val | 0x10);
+
+	/* Global software reset*/
+	hw_atl_rx_rx_reg_res_dis_set(self, 0U);
+	hw_atl_tx_tx_reg_res_dis_set(self, 0U);
+	aq_hw_write_reg_bit(self, HW_ATL_MAC_PHY_CONTROL,
+			    BIT(HW_ATL_MAC_PHY_MPI_RESET_BIT),
+			    HW_ATL_MAC_PHY_MPI_RESET_BIT, 0x0);
+	gsr = aq_hw_read_reg(self, HW_ATL_GLB_SOFT_RES_ADR);
+	aq_hw_write_reg(self, HW_ATL_GLB_SOFT_RES_ADR,
+			(gsr & 0xFFFFBFFF) | 0x8000);
+
+	if (FORCE_FLASHLESS)
+		aq_hw_write_reg(self, 0x534, 0x0);
+
+	aq_hw_write_reg(self, 0x404, 0x40e0);
+
+	/* Wait for RBL boot */
+	for (k = 0; k < 1000; k++) {
+		rbl_status = aq_hw_read_reg(self, 0x388) & 0xFFFF;
+		if (rbl_status && rbl_status != 0xDEAD)
+			break;
+		AQ_HW_SLEEP(10);
+	}
+	if (!rbl_status || rbl_status == 0xDEAD) {
+		PMD_DRV_LOG(ERR, "RBL Restart failed");
+		return -EIO;
+	}
+
+	/* Restore NVR */
+	if (FORCE_FLASHLESS)
+		aq_hw_write_reg(self, 0x534, 0xA0);
+
+	if (rbl_status == 0xF1A7) {
+		PMD_DRV_LOG(ERR, "No FW detected. Dynamic FW load not implemented\n");
+		return -EOPNOTSUPP;
+	}
+
+	for (k = 0; k < 1000; k++) {
+		u32 fw_state = aq_hw_read_reg(self, HW_ATL_MPI_FW_VERSION);
+
+		if (fw_state)
+			break;
+		AQ_HW_SLEEP(10);
+	}
+	if (k == 1000) {
+		PMD_DRV_LOG(ERR, "FW kickstart failed\n");
+		return -EIO;
+	}
+	/* Old FW requires fixed delay after init */
+	AQ_HW_SLEEP(15);
+
+	return 0;
+}
+
+int hw_atl_utils_soft_reset(struct aq_hw_s *self)
+{
+	int err = 0;
+	int k;
+	u32 boot_exit_code = 0;
+
+	for (k = 0; k < 1000; ++k) {
+		u32 flb_status = aq_hw_read_reg(self,
+						HW_ATL_MPI_DAISY_CHAIN_STATUS);
+		boot_exit_code = aq_hw_read_reg(self,
+						HW_ATL_MPI_BOOT_EXIT_CODE);
+		if (flb_status != 0x06000000 || boot_exit_code != 0)
+			break;
+	}
+
+	if (k == 1000) {
+		PMD_DRV_LOG(ERR, "Neither RBL nor FLB firmware started\n");
+		return -EOPNOTSUPP;
+	}
+
+	self->rbl_enabled = (boot_exit_code != 0);
+
+	/* FW 1.x may bootup in an invalid POWER state (WOL feature).
+	 * We should work around this by forcing its state back to DEINIT
+	 */
+	if (!hw_atl_utils_ver_match(HW_ATL_FW_VER_1X,
+				    aq_hw_read_reg(self,
+						   HW_ATL_MPI_FW_VERSION))) {
+		hw_atl_utils_mpi_set_state(self, MPI_DEINIT);
+		AQ_HW_WAIT_FOR((aq_hw_read_reg(self, HW_ATL_MPI_STATE_ADR) &
+			       HW_ATL_MPI_STATE_MSK) == MPI_DEINIT,
+			       10, 1000U);
+	}
+
+	if (self->rbl_enabled)
+		err = hw_atl_utils_soft_reset_rbl(self);
+	else
+		err = hw_atl_utils_soft_reset_flb(self);
+
+	return err;
+}
+
+int hw_atl_utils_fw_downld_dwords(struct aq_hw_s *self, u32 a,
+				  u32 *p, u32 cnt)
+{
+	int err = 0;
+
+	AQ_HW_WAIT_FOR(hw_atl_reg_glb_cpu_sem_get(self,
+						  HW_ATL_FW_SM_RAM) == 1U,
+						  1U, 10000U);
+
+	if (err < 0) {
+		bool is_locked;
+
+		hw_atl_reg_glb_cpu_sem_set(self, 1U, HW_ATL_FW_SM_RAM);
+		is_locked = hw_atl_reg_glb_cpu_sem_get(self, HW_ATL_FW_SM_RAM);
+		if (!is_locked) {
+			err = -ETIMEDOUT;
+			goto err_exit;
+		}
+	}
+
+	aq_hw_write_reg(self, HW_ATL_MIF_ADDR, a);
+
+	for (++cnt; --cnt && !err;) {
+		aq_hw_write_reg(self, HW_ATL_MIF_CMD, 0x00008000U);
+
+		if (IS_CHIP_FEATURE(REVISION_B1))
+			AQ_HW_WAIT_FOR(a != aq_hw_read_reg(self,
+							   HW_ATL_MIF_ADDR),
+				       1, 1000U);
+		else
+			AQ_HW_WAIT_FOR(!(0x100 & aq_hw_read_reg(self,
+							   HW_ATL_MIF_CMD)),
+				       1, 1000U);
+
+		*(p++) = aq_hw_read_reg(self, HW_ATL_MIF_VAL);
+		a += 4;
+	}
+
+	hw_atl_reg_glb_cpu_sem_set(self, 1U, HW_ATL_FW_SM_RAM);
+
+err_exit:
+	return err;
+}
+
+int hw_atl_utils_fw_upload_dwords(struct aq_hw_s *self, u32 a, u32 *p,
+					 u32 cnt)
+{
+	int err = 0;
+	bool is_locked;
+
+	is_locked = hw_atl_reg_glb_cpu_sem_get(self, HW_ATL_FW_SM_RAM);
+	if (!is_locked) {
+		err = -ETIMEDOUT;
+		goto err_exit;
+	}
+	if (IS_CHIP_FEATURE(REVISION_B1)) {
+		u32 offset = 0;
+
+		for (; offset < cnt; ++offset) {
+			aq_hw_write_reg(self, 0x328, p[offset]);
+			aq_hw_write_reg(self, 0x32C,
+				(0x80000000 | (0xFFFF & (offset * 4))));
+			hw_atl_mcp_up_force_intr_set(self, 1);
+			/* 1000 times by 10us = 10ms */
+			AQ_HW_WAIT_FOR((aq_hw_read_reg(self,
+					0x32C) & 0xF0000000) != 0x80000000,
+					10, 1000);
+		}
+	} else {
+		u32 offset = 0;
+
+		aq_hw_write_reg(self, 0x208, a);
+
+		for (; offset < cnt; ++offset) {
+			aq_hw_write_reg(self, 0x20C, p[offset]);
+			aq_hw_write_reg(self, 0x200, 0xC000);
+
+			AQ_HW_WAIT_FOR((aq_hw_read_reg(self, 0x200U)
+					& 0x100) == 0, 10, 1000);
+		}
+	}
+
+	hw_atl_reg_glb_cpu_sem_set(self, 1U, HW_ATL_FW_SM_RAM);
+
+err_exit:
+	return err;
+}
+
+static int hw_atl_utils_ver_match(u32 ver_expected, u32 ver_actual)
+{
+	int err = 0;
+	const u32 dw_major_mask = 0xff000000U;
+	const u32 dw_minor_mask = 0x00ffffffU;
+
+	err = (dw_major_mask & (ver_expected ^ ver_actual)) ? -EOPNOTSUPP : 0;
+	if (err < 0)
+		goto err_exit;
+	err = ((dw_minor_mask & ver_expected) > (dw_minor_mask & ver_actual)) ?
+		-EOPNOTSUPP : 0;
+err_exit:
+	return err;
+}
+
+static int hw_atl_utils_init_ucp(struct aq_hw_s *self)
+{
+	int err = 0;
+
+	if (!aq_hw_read_reg(self, 0x370U)) {
+		unsigned int rnd = (uint32_t)rte_rand();
+		unsigned int ucp_0x370 = 0U;
+
+		ucp_0x370 = 0x02020202U | (0xFEFEFEFEU & rnd);
+		aq_hw_write_reg(self, HW_ATL_UCP_0X370_REG, ucp_0x370);
+	}
+
+	hw_atl_reg_glb_cpu_scratch_scp_set(self, 0x00000000U, 25U);
+
+	/* check 10 times by 1ms */
+	AQ_HW_WAIT_FOR(0U != (self->mbox_addr =
+		       aq_hw_read_reg(self, 0x360U)), 1000U, 10U);
+	AQ_HW_WAIT_FOR(0U != (self->rpc_addr =
+		       aq_hw_read_reg(self, 0x334U)), 1000U, 100U);
+
+	return err;
+}
+
+#define HW_ATL_RPC_CONTROL_ADR 0x0338U
+#define HW_ATL_RPC_STATE_ADR   0x033CU
+
+struct aq_hw_atl_utils_fw_rpc_tid_s {
+	union {
+		u32 val;
+		struct {
+			u16 tid;
+			u16 len;
+		};
+	};
+};
+
+#define hw_atl_utils_fw_rpc_init(_H_) hw_atl_utils_fw_rpc_wait(_H_, NULL)
+
+int hw_atl_utils_fw_rpc_call(struct aq_hw_s *self, unsigned int rpc_size)
+{
+	int err = 0;
+	struct aq_hw_atl_utils_fw_rpc_tid_s sw;
+
+	if (!IS_CHIP_FEATURE(MIPS)) {
+		err = -1;
+		goto err_exit;
+	}
+	err = hw_atl_utils_fw_upload_dwords(self, self->rpc_addr,
+					    (u32 *)(void *)&self->rpc,
+					    (rpc_size + sizeof(u32) -
+					    sizeof(u8)) / sizeof(u32));
+	if (err < 0)
+		goto err_exit;
+
+	sw.tid = 0xFFFFU & (++self->rpc_tid);
+	sw.len = (u16)rpc_size;
+	aq_hw_write_reg(self, HW_ATL_RPC_CONTROL_ADR, sw.val);
+
+err_exit:
+	return err;
+}
+
+int hw_atl_utils_fw_rpc_wait(struct aq_hw_s *self,
+				    struct hw_aq_atl_utils_fw_rpc **rpc)
+{
+	int err = 0;
+	struct aq_hw_atl_utils_fw_rpc_tid_s sw;
+	struct aq_hw_atl_utils_fw_rpc_tid_s fw;
+
+	do {
+		sw.val = aq_hw_read_reg(self, HW_ATL_RPC_CONTROL_ADR);
+
+		self->rpc_tid = sw.tid;
+
+		AQ_HW_WAIT_FOR(sw.tid ==
+				(fw.val =
+				aq_hw_read_reg(self, HW_ATL_RPC_STATE_ADR),
+				fw.tid), 1000U, 100U);
+		if (err < 0)
+			goto err_exit;
+
+		if (fw.len == 0xFFFFU) {
+			err = hw_atl_utils_fw_rpc_call(self, sw.len);
+			if (err < 0)
+				goto err_exit;
+		}
+	} while (sw.tid != fw.tid || 0xFFFFU == fw.len);
+	if (err < 0)
+		goto err_exit;
+
+	if (rpc) {
+		if (fw.len) {
+			err =
+			hw_atl_utils_fw_downld_dwords(self,
+						      self->rpc_addr,
+						      (u32 *)(void *)
+						      &self->rpc,
+						      (fw.len + sizeof(u32) -
+						      sizeof(u8)) /
+						      sizeof(u32));
+			if (err < 0)
+				goto err_exit;
+		}
+
+		*rpc = &self->rpc;
+	}
+
+err_exit:
+	return err;
+}
+
+static int hw_atl_utils_mpi_create(struct aq_hw_s *self)
+{
+	int err = 0;
+
+	err = hw_atl_utils_init_ucp(self);
+	if (err < 0)
+		goto err_exit;
+
+	err = hw_atl_utils_fw_rpc_init(self);
+	if (err < 0)
+		goto err_exit;
+
+err_exit:
+	return err;
+}
+
+int hw_atl_utils_mpi_read_mbox(struct aq_hw_s *self,
+			       struct hw_aq_atl_utils_mbox_header *pmbox)
+{
+	return hw_atl_utils_fw_downld_dwords(self,
+				      self->mbox_addr,
+				      (u32 *)(void *)pmbox,
+				      sizeof(*pmbox) / sizeof(u32));
+}
+
+void hw_atl_utils_mpi_read_stats(struct aq_hw_s *self,
+				 struct hw_aq_atl_utils_mbox *pmbox)
+{
+	int err = 0;
+
+	err = hw_atl_utils_fw_downld_dwords(self,
+					    self->mbox_addr,
+					    (u32 *)(void *)pmbox,
+					    sizeof(*pmbox) / sizeof(u32));
+	if (err < 0)
+		goto err_exit;
+
+	if (IS_CHIP_FEATURE(REVISION_A0)) {
+		unsigned int mtu = 1514;
+		pmbox->stats.ubrc = pmbox->stats.uprc * mtu;
+		pmbox->stats.ubtc = pmbox->stats.uptc * mtu;
+	} else {
+		pmbox->stats.dpc = hw_atl_rpb_rx_dma_drop_pkt_cnt_get(self);
+	}
+
+err_exit:;
+}
+
+static
+int hw_atl_utils_mpi_set_speed(struct aq_hw_s *self, u32 speed)
+{
+	u32 val = aq_hw_read_reg(self, HW_ATL_MPI_CONTROL_ADR);
+
+	val = val & ~HW_ATL_MPI_SPEED_MSK;
+	val |= speed << HW_ATL_MPI_SPEED_SHIFT;
+	aq_hw_write_reg(self, HW_ATL_MPI_CONTROL_ADR, val);
+
+	return 0;
+}
+
+int hw_atl_utils_mpi_set_state(struct aq_hw_s *self,
+				enum hal_atl_utils_fw_state_e state)
+{
+	int err = 0;
+	u32 transaction_id = 0;
+	struct hw_aq_atl_utils_mbox_header mbox;
+	u32 val = aq_hw_read_reg(self, HW_ATL_MPI_CONTROL_ADR);
+
+	if (state == MPI_RESET) {
+		hw_atl_utils_mpi_read_mbox(self, &mbox);
+
+		transaction_id = mbox.transaction_id;
+
+		AQ_HW_WAIT_FOR(transaction_id !=
+				(hw_atl_utils_mpi_read_mbox(self, &mbox),
+				 mbox.transaction_id),
+			       1000U, 100U);
+		if (err < 0)
+			goto err_exit;
+	}
+	/* On interface DEINIT we disable DW (raise bit)
+	 * Otherwise enable DW (clear bit)
+	 */
+	if (state == MPI_DEINIT || state == MPI_POWER)
+		val |= HW_ATL_MPI_DIRTY_WAKE_MSK;
+	else
+		val &= ~HW_ATL_MPI_DIRTY_WAKE_MSK;
+
+	/* Set new state bits */
+	val = val & ~HW_ATL_MPI_STATE_MSK;
+	val |= state & HW_ATL_MPI_STATE_MSK;
+
+	aq_hw_write_reg(self, HW_ATL_MPI_CONTROL_ADR, val);
+err_exit:
+	return err;
+}
+
+int hw_atl_utils_mpi_get_link_status(struct aq_hw_s *self)
+{
+	u32 cp0x036C = aq_hw_read_reg(self, HW_ATL_MPI_STATE_ADR);
+	u32 link_speed_mask = cp0x036C >> HW_ATL_MPI_SPEED_SHIFT;
+	struct aq_hw_link_status_s *link_status = &self->aq_link_status;
+
+	if (!link_speed_mask) {
+		link_status->mbps = 0U;
+	} else {
+		switch (link_speed_mask) {
+		case HAL_ATLANTIC_RATE_10G:
+			link_status->mbps = 10000U;
+			break;
+
+		case HAL_ATLANTIC_RATE_5G:
+		case HAL_ATLANTIC_RATE_5GSR:
+			link_status->mbps = 5000U;
+			break;
+
+		case HAL_ATLANTIC_RATE_2GS:
+			link_status->mbps = 2500U;
+			break;
+
+		case HAL_ATLANTIC_RATE_1G:
+			link_status->mbps = 1000U;
+			break;
+
+		case HAL_ATLANTIC_RATE_100M:
+			link_status->mbps = 100U;
+			break;
+
+		default:
+			return -EBUSY;
+		}
+	}
+
+	return 0;
+}
+
+static int hw_atl_utils_get_mac_permanent(struct aq_hw_s *self,
+				   u8 *mac)
+{
+	int err = 0;
+	u32 h = 0U;
+	u32 l = 0U;
+	u32 mac_addr[2];
+
+	if (!aq_hw_read_reg(self, HW_ATL_UCP_0X370_REG)) {
+		unsigned int rnd = (uint32_t)rte_rand();
+		unsigned int ucp_0x370 = 0;
+
+		//get_random_bytes(&rnd, sizeof(unsigned int));
+
+		ucp_0x370 = 0x02020202 | (0xFEFEFEFE & rnd);
+		aq_hw_write_reg(self, HW_ATL_UCP_0X370_REG, ucp_0x370);
+	}
+
+	err = hw_atl_utils_fw_downld_dwords(self,
+					    aq_hw_read_reg(self, 0x00000374U) +
+					    (40U * 4U),
+					    mac_addr,
+					    ARRAY_SIZE(mac_addr));
+	if (err < 0) {
+		mac_addr[0] = 0U;
+		mac_addr[1] = 0U;
+		err = 0;
+	} else {
+		mac_addr[0] = rte_constant_bswap32(mac_addr[0]);
+		mac_addr[1] = rte_constant_bswap32(mac_addr[1]);
+	}
+
+	ether_addr_copy((struct ether_addr *)mac_addr,
+			(struct ether_addr *)mac);
+
+	if ((mac[0] & 0x01U) || ((mac[0] | mac[1] | mac[2]) == 0x00U)) {
+		/* chip revision */
+		l = 0xE3000000U
+			| (0xFFFFU & aq_hw_read_reg(self, HW_ATL_UCP_0X370_REG))
+			| (0x00 << 16);
+		h = 0x8001300EU;
+
+		mac[5] = (u8)(0xFFU & l);
+		l >>= 8;
+		mac[4] = (u8)(0xFFU & l);
+		l >>= 8;
+		mac[3] = (u8)(0xFFU & l);
+		l >>= 8;
+		mac[2] = (u8)(0xFFU & l);
+		mac[1] = (u8)(0xFFU & h);
+		h >>= 8;
+		mac[0] = (u8)(0xFFU & h);
+	}
+
+	return err;
+}
+
+unsigned int hw_atl_utils_mbps_2_speed_index(unsigned int mbps)
+{
+	unsigned int ret = 0U;
+
+	switch (mbps) {
+	case 100U:
+		ret = 5U;
+		break;
+
+	case 1000U:
+		ret = 4U;
+		break;
+
+	case 2500U:
+		ret = 3U;
+		break;
+
+	case 5000U:
+		ret = 1U;
+		break;
+
+	case 10000U:
+		ret = 0U;
+		break;
+
+	default:
+		break;
+	}
+	return ret;
+}
+
+void hw_atl_utils_hw_chip_features_init(struct aq_hw_s *self, u32 *p)
+{
+	u32 chip_features = 0U;
+	u32 val = hw_atl_reg_glb_mif_id_get(self);
+	u32 mif_rev = val & 0xFFU;
+
+	if ((0xFU & mif_rev) == 1U) {
+		chip_features |= HAL_ATLANTIC_UTILS_CHIP_REVISION_A0 |
+			HAL_ATLANTIC_UTILS_CHIP_MPI_AQ |
+			HAL_ATLANTIC_UTILS_CHIP_MIPS;
+	} else if ((0xFU & mif_rev) == 2U) {
+		chip_features |= HAL_ATLANTIC_UTILS_CHIP_REVISION_B0 |
+			HAL_ATLANTIC_UTILS_CHIP_MPI_AQ |
+			HAL_ATLANTIC_UTILS_CHIP_MIPS |
+			HAL_ATLANTIC_UTILS_CHIP_TPO2 |
+			HAL_ATLANTIC_UTILS_CHIP_RPF2;
+	} else if ((0xFU & mif_rev) == 0xAU) {
+		chip_features |= HAL_ATLANTIC_UTILS_CHIP_REVISION_B1 |
+			HAL_ATLANTIC_UTILS_CHIP_MPI_AQ |
+			HAL_ATLANTIC_UTILS_CHIP_MIPS |
+			HAL_ATLANTIC_UTILS_CHIP_TPO2 |
+			HAL_ATLANTIC_UTILS_CHIP_RPF2;
+	}
+
+	*p = chip_features;
+}
+
+static int hw_atl_fw1x_deinit(struct aq_hw_s *self)
+{
+	hw_atl_utils_mpi_set_speed(self, 0);
+	hw_atl_utils_mpi_set_state(self, MPI_DEINIT);
+	return 0;
+}
+
+int hw_atl_utils_update_stats(struct aq_hw_s *self)
+{
+	struct hw_aq_atl_utils_mbox mbox;
+
+	hw_atl_utils_mpi_read_stats(self, &mbox);
+
+#define AQ_SDELTA(_N_) (self->curr_stats._N_ += \
+			mbox.stats._N_ - self->last_stats._N_)
+
+	if (1) {//self->aq_link_status.mbps) {
+		AQ_SDELTA(uprc);
+		AQ_SDELTA(mprc);
+		AQ_SDELTA(bprc);
+		AQ_SDELTA(erpt);
+
+		AQ_SDELTA(uptc);
+		AQ_SDELTA(mptc);
+		AQ_SDELTA(bptc);
+		AQ_SDELTA(erpr);
+		AQ_SDELTA(ubrc);
+		AQ_SDELTA(ubtc);
+		AQ_SDELTA(mbrc);
+		AQ_SDELTA(mbtc);
+		AQ_SDELTA(bbrc);
+		AQ_SDELTA(bbtc);
+		AQ_SDELTA(dpc);
+	}
+#undef AQ_SDELTA
+	self->curr_stats.dma_pkt_rc =
+	   hw_atl_stats_rx_dma_good_pkt_counterlsw_get(self) +
+	   ((u64)hw_atl_stats_rx_dma_good_pkt_countermsw_get(self) << 32);
+	self->curr_stats.dma_pkt_tc =
+	   hw_atl_stats_tx_dma_good_pkt_counterlsw_get(self) +
+	   ((u64)hw_atl_stats_tx_dma_good_pkt_countermsw_get(self) << 32);
+	self->curr_stats.dma_oct_rc =
+	   hw_atl_stats_rx_dma_good_octet_counterlsw_get(self) +
+	   ((u64)hw_atl_stats_rx_dma_good_octet_countermsw_get(self) << 32);
+	self->curr_stats.dma_oct_tc =
+	   hw_atl_stats_tx_dma_good_octet_counterlsw_get(self) +
+	   ((u64)hw_atl_stats_tx_dma_good_octet_countermsw_get(self) << 32);
+
+	self->curr_stats.dpc = hw_atl_rpb_rx_dma_drop_pkt_cnt_get(self);
+
+	memcpy(&self->last_stats, &mbox.stats, sizeof(mbox.stats));
+
+	return 0;
+}
+
+struct aq_stats_s *hw_atl_utils_get_hw_stats(struct aq_hw_s *self)
+{
+	return &self->curr_stats;
+}
+
+static const u32 hw_atl_utils_hw_mac_regs[] = {
+	0x00005580U, 0x00005590U, 0x000055B0U, 0x000055B4U,
+	0x000055C0U, 0x00005B00U, 0x00005B04U, 0x00005B08U,
+	0x00005B0CU, 0x00005B10U, 0x00005B14U, 0x00005B18U,
+	0x00005B1CU, 0x00005B20U, 0x00005B24U, 0x00005B28U,
+	0x00005B2CU, 0x00005B30U, 0x00005B34U, 0x00005B38U,
+	0x00005B3CU, 0x00005B40U, 0x00005B44U, 0x00005B48U,
+	0x00005B4CU, 0x00005B50U, 0x00005B54U, 0x00005B58U,
+	0x00005B5CU, 0x00005B60U, 0x00005B64U, 0x00005B68U,
+	0x00005B6CU, 0x00005B70U, 0x00005B74U, 0x00005B78U,
+	0x00005B7CU, 0x00007C00U, 0x00007C04U, 0x00007C08U,
+	0x00007C0CU, 0x00007C10U, 0x00007C14U, 0x00007C18U,
+	0x00007C1CU, 0x00007C20U, 0x00007C40U, 0x00007C44U,
+	0x00007C48U, 0x00007C4CU, 0x00007C50U, 0x00007C54U,
+	0x00007C58U, 0x00007C5CU, 0x00007C60U, 0x00007C80U,
+	0x00007C84U, 0x00007C88U, 0x00007C8CU, 0x00007C90U,
+	0x00007C94U, 0x00007C98U, 0x00007C9CU, 0x00007CA0U,
+	0x00007CC0U, 0x00007CC4U, 0x00007CC8U, 0x00007CCCU,
+	0x00007CD0U, 0x00007CD4U, 0x00007CD8U, 0x00007CDCU,
+	0x00007CE0U, 0x00000300U, 0x00000304U, 0x00000308U,
+	0x0000030cU, 0x00000310U, 0x00000314U, 0x00000318U,
+	0x0000031cU, 0x00000360U, 0x00000364U, 0x00000368U,
+	0x0000036cU, 0x00000370U, 0x00000374U, 0x00006900U,
+};
+
+unsigned int hw_atl_utils_hw_get_reg_length(void)
+{
+	return ARRAY_SIZE(hw_atl_utils_hw_mac_regs);
+}
+
+int hw_atl_utils_hw_get_regs(struct aq_hw_s *self,
+			     u32 *regs_buff)
+{
+	unsigned int i = 0U;
+	unsigned int mac_regs_count = hw_atl_utils_hw_get_reg_length();
+
+	for (i = 0; i < mac_regs_count; i++)
+		regs_buff[i] = aq_hw_read_reg(self,
+					      hw_atl_utils_hw_mac_regs[i]);
+	return 0;
+}
+
+int hw_atl_utils_get_fw_version(struct aq_hw_s *self, u32 *fw_version)
+{
+	*fw_version = aq_hw_read_reg(self, 0x18U);
+	return 0;
+}
+
+static int aq_fw1x_set_wol(struct aq_hw_s *self, bool wol_enabled, u8 *mac)
+{
+	struct hw_aq_atl_utils_fw_rpc *prpc = NULL;
+	unsigned int rpc_size = 0U;
+	int err = 0;
+
+	err = hw_atl_utils_fw_rpc_wait(self, &prpc);
+	if (err < 0)
+		goto err_exit;
+
+	memset(prpc, 0, sizeof(*prpc));
+
+	if (wol_enabled) {
+		rpc_size = sizeof(prpc->msg_id) + sizeof(prpc->msg_wol);
+
+		prpc->msg_id = HAL_ATLANTIC_UTILS_FW_MSG_WOL_ADD;
+		prpc->msg_wol.priority = 0x10000000; /* normal priority */
+		prpc->msg_wol.pattern_id = 1U;
+		prpc->msg_wol.wol_packet_type = 2U; /* Magic Packet */
+
+		ether_addr_copy((struct ether_addr *)mac,
+			(struct ether_addr *)&prpc->msg_wol.wol_pattern);
+	} else {
+		rpc_size = sizeof(prpc->msg_id) + sizeof(prpc->msg_del_id);
+
+		prpc->msg_id = HAL_ATLANTIC_UTILS_FW_MSG_WOL_DEL;
+		prpc->msg_wol.pattern_id = 1U;
+	}
+
+	err = hw_atl_utils_fw_rpc_call(self, rpc_size);
+	if (err < 0)
+		goto err_exit;
+err_exit:
+	return err;
+}
+
+static
+int aq_fw1x_set_power(struct aq_hw_s *self,
+		      unsigned int power_state __rte_unused,
+		      u8 *mac)
+{
+	struct hw_aq_atl_utils_fw_rpc *prpc = NULL;
+	unsigned int rpc_size = 0U;
+	int err = 0;
+	if (self->aq_nic_cfg->wol & AQ_NIC_WOL_ENABLED) {
+		err = aq_fw1x_set_wol(self, 1, mac);
+
+		if (err < 0)
+			goto err_exit;
+
+		rpc_size = sizeof(prpc->msg_id) +
+				sizeof(prpc->msg_enable_wakeup);
+
+		err = hw_atl_utils_fw_rpc_wait(self, &prpc);
+
+		if (err < 0)
+			goto err_exit;
+
+		memset(prpc, 0, rpc_size);
+
+		prpc->msg_id = HAL_ATLANTIC_UTILS_FW_MSG_ENABLE_WAKEUP;
+		prpc->msg_enable_wakeup.pattern_mask = 0x00000002;
+
+		err = hw_atl_utils_fw_rpc_call(self, rpc_size);
+		if (err < 0)
+			goto err_exit;
+	}
+
+	hw_atl_utils_mpi_set_speed(self, 0);
+	hw_atl_utils_mpi_set_state(self, MPI_POWER);
+err_exit:
+	return err;
+}
+
+
+
+const struct aq_fw_ops aq_fw_1x_ops = {
+	.init = hw_atl_utils_mpi_create,
+	.deinit = hw_atl_fw1x_deinit,
+	.reset = NULL,
+	.get_mac_permanent = hw_atl_utils_get_mac_permanent,
+	.set_link_speed = hw_atl_utils_mpi_set_speed,
+	.set_state = hw_atl_utils_mpi_set_state,
+	.update_link_status = hw_atl_utils_mpi_get_link_status,
+	.update_stats = hw_atl_utils_update_stats,
+	.set_power = aq_fw1x_set_power,
+	.get_temp = NULL,
+	.get_cable_len = NULL,
+	.set_eee_rate = NULL,
+	.get_eee_rate = NULL,
+	.set_flow_control = NULL,
+	.led_control = NULL,
+	.get_eeprom = NULL,
+	.set_eeprom = NULL,
+};
diff --git a/drivers/net/atlantic/hw_atl/hw_atl_utils.h b/drivers/net/atlantic/hw_atl/hw_atl_utils.h
new file mode 100644
index 00000000..5f3f7084
--- /dev/null
+++ b/drivers/net/atlantic/hw_atl/hw_atl_utils.h
@@ -0,0 +1,510 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) */
+/* Copyright (C) 2014-2017 aQuantia Corporation. */
+
+/* File hw_atl_utils.h: Declaration of common functions for Atlantic hardware
+ * abstraction layer.
+ */
+
+#ifndef HW_ATL_UTILS_H
+#define HW_ATL_UTILS_H
+
+#define HW_ATL_FLUSH() { (void)aq_hw_read_reg(self, 0x10); }
+
+/* Hardware tx descriptor */
+struct hw_atl_txd_s {
+	u64 buf_addr;
+
+	union {
+		struct {
+			u32 type:3;
+			u32:1;
+			u32 len:16;
+			u32 dd:1;
+			u32 eop:1;
+			u32 cmd:8;
+			u32:14;
+			u32 ct_idx:1;
+			u32 ct_en:1;
+			u32 pay_len:18;
+		} __attribute__((__packed__));
+		u64 flags;
+	};
+} __attribute__((__packed__));
+
+/* Hardware tx context descriptor */
+union hw_atl_txc_s {
+	struct {
+		u64 flags1;
+		u64 flags2;
+	};
+
+	struct {
+		u64:40;
+		u32 tun_len:8;
+		u32 out_len:16;
+		u32 type:3;
+		u32 idx:1;
+		u32 vlan_tag:16;
+		u32 cmd:4;
+		u32 l2_len:7;
+		u32 l3_len:9;
+		u32 l4_len:8;
+		u32 mss_len:16;
+	} __attribute__((__packed__));
+} __attribute__((__packed__));
+
+enum aq_tx_desc_type {
+	tx_desc_type_desc = 1,
+	tx_desc_type_ctx = 2,
+};
+
+enum aq_tx_desc_cmd {
+	tx_desc_cmd_vlan = 1,
+	tx_desc_cmd_fcs = 2,
+	tx_desc_cmd_ipv4 = 4,
+	tx_desc_cmd_l4cs = 8,
+	tx_desc_cmd_lso = 0x10,
+	tx_desc_cmd_wb = 0x20,
+};
+
+
+/* Hardware rx descriptor */
+struct hw_atl_rxd_s {
+	u64 buf_addr;
+	u64 hdr_addr;
+} __attribute__((__packed__));
+
+/* Hardware rx descriptor writeback */
+struct hw_atl_rxd_wb_s {
+	u32 rss_type:4;
+	u32 pkt_type:8;
+	u32 type:20;
+	u32 rss_hash;
+	u16 dd:1;
+	u16 eop:1;
+	u16 rx_stat:4;
+	u16 rx_estat:6;
+	u16 rsc_cnt:4;
+	u16 pkt_len;
+	u16 next_desc_ptr;
+	u16 vlan;
+} __attribute__((__packed__));
+
+struct hw_atl_stats_s {
+	u32 uprc;
+	u32 mprc;
+	u32 bprc;
+	u32 erpt;
+	u32 uptc;
+	u32 mptc;
+	u32 bptc;
+	u32 erpr;
+	u32 mbtc;
+	u32 bbtc;
+	u32 mbrc;
+	u32 bbrc;
+	u32 ubrc;
+	u32 ubtc;
+	u32 dpc;
+} __attribute__((__packed__));
+
+union ip_addr {
+	struct {
+		u8 addr[16];
+	} v6;
+	struct {
+		u8 padding[12];
+		u8 addr[4];
+	} v4;
+} __attribute__((__packed__));
+
+struct hw_aq_atl_utils_fw_rpc {
+	u32 msg_id;
+
+	union {
+		struct {
+			u32 pong;
+		} msg_ping;
+
+		struct {
+			u8 mac_addr[6];
+			u32 ip_addr_cnt;
+
+			struct {
+				union ip_addr addr;
+				union ip_addr mask;
+			} ip[1];
+		} msg_arp;
+
+		struct {
+			u32 len;
+			u8 packet[1514U];
+		} msg_inject;
+
+		struct {
+			u32 priority;
+			u32 wol_packet_type;
+			u32 pattern_id;
+			u32 next_wol_pattern_offset;
+			union {
+				struct {
+					u32 flags;
+					u8 ipv4_source_address[4];
+					u8 ipv4_dest_address[4];
+					u16 tcp_source_port_number;
+					u16 tcp_dest_port_number;
+				} ipv4_tcp_syn_parameters;
+
+				struct {
+					u32 flags;
+					u8 ipv6_source_address[16];
+					u8 ipv6_dest_address[16];
+					u16 tcp_source_port_number;
+					u16 tcp_dest_port_number;
+				} ipv6_tcp_syn_parameters;
+
+				struct {
+					u32 flags;
+				} eapol_request_id_message_parameters;
+
+				struct {
+					u32 flags;
+					u32 mask_offset;
+					u32 mask_size;
+					u32 pattern_offset;
+					u32 pattern_size;
+				} wol_bit_map_pattern;
+				struct {
+					u8 mac_addr[6];
+				} wol_magic_packet_pattern;
+
+			} wol_pattern;
+		} msg_wol;
+
+		struct {
+			u16 tc_quanta[8];
+			u16 tc_threshold[8];
+		} msg_msm_pfc_quantas;
+
+		struct {
+			union {
+				u32 pattern_mask;
+				struct {
+					u32 aq_pm_wol_reason_arp_v4_pkt : 1;
+					u32 aq_pm_wol_reason_ipv4_ping_pkt : 1;
+					u32 aq_pm_wol_reason_ipv6_ns_pkt : 1;
+					u32 aq_pm_wol_reason_ipv6_ping_pkt : 1;
+					u32 aq_pm_wol_reason_link_up : 1;
+					u32 aq_pm_wol_reason_link_down : 1;
+					u32 aq_pm_wol_reason_maximum : 1;
+				};
+			};
+			union {
+				u32 offload_mask;
+			};
+		} msg_enable_wakeup;
+
+		struct {
+			u32 priority;
+			u32 protocol_offload_type;
+			u32 protocol_offload_id;
+			u32 next_protocol_offload_offset;
+
+			union {
+				struct {
+					u32 flags;
+					u8 remote_ipv4_addr[4];
+					u8 host_ipv4_addr[4];
+					u8 mac_addr[6];
+				} ipv4_arp_params;
+			};
+		} msg_offload;
+
+		struct {
+			u32 id;
+		} msg_del_id;
+
+	};
+} __attribute__((__packed__));
+
+struct hw_aq_atl_utils_mbox_header {
+	u32 version;
+	u32 transaction_id;
+	u32 error;
+} __attribute__((__packed__));
+
+struct hw_aq_info {
+	u8 reserved[6];
+	u16 phy_fault_code;
+	u16 phy_temperature;
+	u8 cable_len;
+	u8 reserved1;
+	u32 cable_diag_data[4];
+	u8 reserved2[32];
+	u32 caps_lo;
+	u32 caps_hi;
+} __attribute__((__packed__));
+
+struct hw_aq_atl_utils_mbox {
+	struct hw_aq_atl_utils_mbox_header header;
+	struct hw_atl_stats_s stats;
+	struct hw_aq_info info;
+} __attribute__((__packed__));
+
+/* fw2x */
+typedef u16	in_port_t;
+typedef u32	ip4_addr_t;
+typedef int	int32_t;
+typedef short	int16_t;
+typedef u32	fw_offset_t;
+
+struct ip6_addr {
+	u32 addr[4];
+} __attribute__((__packed__));
+
+struct offload_ka_v4 {
+	u32 timeout;
+	in_port_t local_port;
+	in_port_t remote_port;
+	u8 remote_mac_addr[6];
+	u16 win_size;
+	u32 seq_num;
+	u32 ack_num;
+	ip4_addr_t local_ip;
+	ip4_addr_t remote_ip;
+} __attribute__((__packed__));
+
+struct offload_ka_v6 {
+	u32 timeout;
+	in_port_t local_port;
+	in_port_t remote_port;
+	u8 remote_mac_addr[6];
+	u16 win_size;
+	u32 seq_num;
+	u32 ack_num;
+	struct ip6_addr local_ip;
+	struct ip6_addr remote_ip;
+} __attribute__((__packed__));
+
+struct offload_ip_info {
+	u8 v4_local_addr_count;
+	u8 v4_addr_count;
+	u8 v6_local_addr_count;
+	u8 v6_addr_count;
+	fw_offset_t v4_addr;
+	fw_offset_t v4_prefix;
+	fw_offset_t v6_addr;
+	fw_offset_t v6_prefix;
+} __attribute__((__packed__));
+
+struct offload_port_info {
+	u16 udp_port_count;
+	u16 tcp_port_count;
+	fw_offset_t udp_port;
+	fw_offset_t tcp_port;
+} __attribute__((__packed__));
+
+struct offload_ka_info {
+	u16 v4_ka_count;
+	u16 v6_ka_count;
+	u32 retry_count;
+	u32 retry_interval;
+	fw_offset_t v4_ka;
+	fw_offset_t v6_ka;
+} __attribute__((__packed__));
+
+struct offload_rr_info {
+	u32 rr_count;
+	u32 rr_buf_len;
+	fw_offset_t rr_id_x;
+	fw_offset_t rr_buf;
+} __attribute__((__packed__));
+
+struct offload_info {
+	u32 version;		// current version is 0x00000000
+	u32 len;		// The whole structure length
+				// including the variable-size buf
+	u8 mac_addr[6];		// 8 bytes to keep alignment. Only
+				// first 6 meaningful.
+
+	u8 reserved[2];
+
+	struct offload_ip_info ips;
+	struct offload_port_info ports;
+	struct offload_ka_info kas;
+	struct offload_rr_info rrs;
+	u8 buf[0];
+} __attribute__((__packed__));
+
+struct smbus_read_request {
+	u32 offset; /* not used */
+	u32 device_id;
+	u32 address;
+	u32 length;
+} __attribute__((__packed__));
+
+struct smbus_write_request {
+	u32 offset; /* not used */
+	u32 device_id;
+	u32 address;
+	u32 length;
+} __attribute__((__packed__));
+
+#define HAL_ATLANTIC_UTILS_CHIP_MIPS         0x00000001U
+#define HAL_ATLANTIC_UTILS_CHIP_TPO2         0x00000002U
+#define HAL_ATLANTIC_UTILS_CHIP_RPF2         0x00000004U
+#define HAL_ATLANTIC_UTILS_CHIP_MPI_AQ       0x00000010U
+#define HAL_ATLANTIC_UTILS_CHIP_REVISION_A0  0x01000000U
+#define HAL_ATLANTIC_UTILS_CHIP_REVISION_B0  0x02000000U
+#define HAL_ATLANTIC_UTILS_CHIP_REVISION_B1  0x04000000U
+
+
+#define IS_CHIP_FEATURE(_F_) (HAL_ATLANTIC_UTILS_CHIP_##_F_ & \
+	self->chip_features)
+
+enum hal_atl_utils_fw_state_e {
+	MPI_DEINIT = 0,
+	MPI_RESET = 1,
+	MPI_INIT = 2,
+	MPI_POWER = 4,
+};
+
+#define HAL_ATLANTIC_RATE_10G        BIT(0)
+#define HAL_ATLANTIC_RATE_5G         BIT(1)
+#define HAL_ATLANTIC_RATE_5GSR       BIT(2)
+#define HAL_ATLANTIC_RATE_2GS        BIT(3)
+#define HAL_ATLANTIC_RATE_1G         BIT(4)
+#define HAL_ATLANTIC_RATE_100M       BIT(5)
+#define HAL_ATLANTIC_RATE_INVALID    BIT(6)
+
+#define HAL_ATLANTIC_UTILS_FW_MSG_PING     1U
+#define HAL_ATLANTIC_UTILS_FW_MSG_ARP      2U
+#define HAL_ATLANTIC_UTILS_FW_MSG_INJECT   3U
+#define HAL_ATLANTIC_UTILS_FW_MSG_WOL_ADD 4U
+#define HAL_ATLANTIC_UTILS_FW_MSG_WOL_DEL 5U
+#define HAL_ATLANTIC_UTILS_FW_MSG_ENABLE_WAKEUP 6U
+#define HAL_ATLANTIC_UTILS_FW_MSG_MSM_PFC  7U
+#define HAL_ATLANTIC_UTILS_FW_MSG_PROVISIONING 8U
+#define HAL_ATLANTIC_UTILS_FW_MSG_OFFLOAD_ADD  9U
+#define HAL_ATLANTIC_UTILS_FW_MSG_OFFLOAD_DEL  10U
+#define HAL_ATLANTIC_UTILS_FW_MSG_CABLE_DIAG   13U // 0xd
+
+#define SMBUS_READ_REQUEST BIT(13)
+#define SMBUS_WRITE_REQUEST BIT(14)
+#define SMBUS_DEVICE_ID 0x50
+
+enum hw_atl_fw2x_rate {
+	FW2X_RATE_100M    = 0x20,
+	FW2X_RATE_1G      = 0x100,
+	FW2X_RATE_2G5     = 0x200,
+	FW2X_RATE_5G      = 0x400,
+	FW2X_RATE_10G     = 0x800,
+};
+
+enum hw_atl_fw2x_caps_lo {
+	CAPS_LO_10BASET_HD = 0x00,
+	CAPS_LO_10BASET_FD,
+	CAPS_LO_100BASETX_HD,
+	CAPS_LO_100BASET4_HD,
+	CAPS_LO_100BASET2_HD,
+	CAPS_LO_100BASETX_FD,
+	CAPS_LO_100BASET2_FD,
+	CAPS_LO_1000BASET_HD,
+	CAPS_LO_1000BASET_FD,
+	CAPS_LO_2P5GBASET_FD,
+	CAPS_LO_5GBASET_FD,
+	CAPS_LO_10GBASET_FD,
+};
+
+enum hw_atl_fw2x_caps_hi {
+	CAPS_HI_RESERVED1 = 0x00,
+	CAPS_HI_10BASET_EEE,
+	CAPS_HI_RESERVED2,
+	CAPS_HI_PAUSE,
+	CAPS_HI_ASYMMETRIC_PAUSE,
+	CAPS_HI_100BASETX_EEE,
+	CAPS_HI_RESERVED3,
+	CAPS_HI_RESERVED4,
+	CAPS_HI_1000BASET_FD_EEE,
+	CAPS_HI_2P5GBASET_FD_EEE,
+	CAPS_HI_5GBASET_FD_EEE,
+	CAPS_HI_10GBASET_FD_EEE,
+	CAPS_HI_RESERVED5,
+	CAPS_HI_RESERVED6,
+	CAPS_HI_RESERVED7,
+	CAPS_HI_RESERVED8,
+	CAPS_HI_RESERVED9,
+	CAPS_HI_CABLE_DIAG,
+	CAPS_HI_TEMPERATURE,
+	CAPS_HI_DOWNSHIFT,
+	CAPS_HI_PTP_AVB_EN,
+	CAPS_HI_MEDIA_DETECT,
+	CAPS_HI_LINK_DROP,
+	CAPS_HI_SLEEP_PROXY,
+	CAPS_HI_WOL,
+	CAPS_HI_MAC_STOP,
+	CAPS_HI_EXT_LOOPBACK,
+	CAPS_HI_INT_LOOPBACK,
+	CAPS_HI_EFUSE_AGENT,
+	CAPS_HI_WOL_TIMER,
+	CAPS_HI_STATISTICS,
+	CAPS_HI_TRANSACTION_ID,
+};
+
+struct aq_hw_s;
+struct aq_fw_ops;
+struct aq_hw_link_status_s;
+
+int hw_atl_utils_initfw(struct aq_hw_s *self, const struct aq_fw_ops **fw_ops);
+
+int hw_atl_utils_soft_reset(struct aq_hw_s *self);
+
+void hw_atl_utils_hw_chip_features_init(struct aq_hw_s *self, u32 *p);
+
+int hw_atl_utils_mpi_read_mbox(struct aq_hw_s *self,
+			       struct hw_aq_atl_utils_mbox_header *pmbox);
+
+void hw_atl_utils_mpi_read_stats(struct aq_hw_s *self,
+				 struct hw_aq_atl_utils_mbox *pmbox);
+
+void hw_atl_utils_mpi_set(struct aq_hw_s *self,
+			  enum hal_atl_utils_fw_state_e state,
+			  u32 speed);
+
+int hw_atl_utils_mpi_get_link_status(struct aq_hw_s *self);
+
+unsigned int hw_atl_utils_mbps_2_speed_index(unsigned int mbps);
+
+unsigned int hw_atl_utils_hw_get_reg_length(void);
+
+int hw_atl_utils_hw_get_regs(struct aq_hw_s *self,
+			     u32 *regs_buff);
+
+int hw_atl_utils_hw_set_power(struct aq_hw_s *self,
+			      unsigned int power_state);
+
+int hw_atl_utils_hw_deinit(struct aq_hw_s *self);
+
+int hw_atl_utils_get_fw_version(struct aq_hw_s *self, u32 *fw_version);
+
+int hw_atl_utils_update_stats(struct aq_hw_s *self);
+
+struct aq_stats_s *hw_atl_utils_get_hw_stats(struct aq_hw_s *self);
+
+int hw_atl_utils_fw_downld_dwords(struct aq_hw_s *self, u32 a,
+				  u32 *p, u32 cnt);
+
+int hw_atl_utils_fw_upload_dwords(struct aq_hw_s *self, u32 a, u32 *p,
+				u32 cnt);
+
+int hw_atl_utils_fw_set_wol(struct aq_hw_s *self, bool wol_enabled, u8 *mac);
+
+int hw_atl_utils_fw_rpc_call(struct aq_hw_s *self, unsigned int rpc_size);
+
+int hw_atl_utils_fw_rpc_wait(struct aq_hw_s *self,
+		    struct hw_aq_atl_utils_fw_rpc **rpc);
+
+extern const struct aq_fw_ops aq_fw_1x_ops;
+extern const struct aq_fw_ops aq_fw_2x_ops;
+
+#endif /* HW_ATL_UTILS_H */
diff --git a/drivers/net/atlantic/hw_atl/hw_atl_utils_fw2x.c b/drivers/net/atlantic/hw_atl/hw_atl_utils_fw2x.c
new file mode 100644
index 00000000..6841d9bc
--- /dev/null
+++ b/drivers/net/atlantic/hw_atl/hw_atl_utils_fw2x.c
@@ -0,0 +1,618 @@
+// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+/* Copyright (C) 2014-2017 aQuantia Corporation. */
+
+/* File hw_atl_utils_fw2x.c: Definition of firmware 2.x functions for
+ * Atlantic hardware abstraction layer.
+ */
+
+#include <rte_ether.h>
+#include "../atl_hw_regs.h"
+
+#include "../atl_types.h"
+#include "hw_atl_utils.h"
+#include "hw_atl_llh.h"
+
+#define HW_ATL_FW2X_MPI_EFUSE_ADDR	0x364
+#define HW_ATL_FW2X_MPI_MBOX_ADDR	0x360
+#define HW_ATL_FW2X_MPI_RPC_ADDR	0x334
+
+#define HW_ATL_FW2X_MPI_CONTROL_ADDR	0x368
+#define HW_ATL_FW2X_MPI_CONTROL2_ADDR	0x36C
+#define HW_ATL_FW2X_MPI_LED_ADDR	0x31c
+
+#define HW_ATL_FW2X_MPI_STATE_ADDR	0x370
+#define HW_ATL_FW2X_MPI_STATE2_ADDR	0x374
+
+#define HW_ATL_FW2X_CAP_SLEEP_PROXY BIT(CAPS_HI_SLEEP_PROXY)
+#define HW_ATL_FW2X_CAP_WOL BIT(CAPS_HI_WOL)
+
+#define HW_ATL_FW2X_CAP_EEE_1G_MASK   BIT(CAPS_HI_1000BASET_FD_EEE)
+#define HW_ATL_FW2X_CAP_EEE_2G5_MASK  BIT(CAPS_HI_2P5GBASET_FD_EEE)
+#define HW_ATL_FW2X_CAP_EEE_5G_MASK   BIT(CAPS_HI_5GBASET_FD_EEE)
+#define HW_ATL_FW2X_CAP_EEE_10G_MASK  BIT(CAPS_HI_10GBASET_FD_EEE)
+
+#define HAL_ATLANTIC_WOL_FILTERS_COUNT     8
+#define HAL_ATLANTIC_UTILS_FW2X_MSG_WOL    0x0E
+
+#define HW_ATL_FW_FEATURE_EEPROM 0x03010025
+#define HW_ATL_FW_FEATURE_LED 0x03010026
+
+struct fw2x_msg_wol_pattern {
+	u8 mask[16];
+	u32 crc;
+} __attribute__((__packed__));
+
+struct fw2x_msg_wol {
+	u32 msg_id;
+	u8 hw_addr[6];
+	u8 magic_packet_enabled;
+	u8 filter_count;
+	struct fw2x_msg_wol_pattern filter[HAL_ATLANTIC_WOL_FILTERS_COUNT];
+	u8 link_up_enabled;
+	u8 link_down_enabled;
+	u16 reserved;
+	u32 link_up_timeout;
+	u32 link_down_timeout;
+} __attribute__((__packed__));
+
+static int aq_fw2x_set_link_speed(struct aq_hw_s *self, u32 speed);
+static int aq_fw2x_set_state(struct aq_hw_s *self,
+			     enum hal_atl_utils_fw_state_e state);
+
+static int aq_fw2x_init(struct aq_hw_s *self)
+{
+	int err = 0;
+
+	/* check 10 times by 1ms */
+	AQ_HW_WAIT_FOR(0U != (self->mbox_addr =
+		       aq_hw_read_reg(self, HW_ATL_FW2X_MPI_MBOX_ADDR)),
+		       1000U, 10U);
+	AQ_HW_WAIT_FOR(0U != (self->rpc_addr =
+		       aq_hw_read_reg(self, HW_ATL_FW2X_MPI_RPC_ADDR)),
+		       1000U, 100U);
+	return err;
+}
+
+static int aq_fw2x_deinit(struct aq_hw_s *self)
+{
+	int err = aq_fw2x_set_link_speed(self, 0);
+
+	if (!err)
+		err = aq_fw2x_set_state(self, MPI_DEINIT);
+
+	return err;
+}
+
+static enum hw_atl_fw2x_rate link_speed_mask_2fw2x_ratemask(u32 speed)
+{
+	enum hw_atl_fw2x_rate rate = 0;
+
+	if (speed & AQ_NIC_RATE_10G)
+		rate |= FW2X_RATE_10G;
+
+	if (speed & AQ_NIC_RATE_5G)
+		rate |= FW2X_RATE_5G;
+
+	if (speed & AQ_NIC_RATE_5G5R)
+		rate |= FW2X_RATE_5G;
+
+	if (speed & AQ_NIC_RATE_2G5)
+		rate |= FW2X_RATE_2G5;
+
+	if (speed & AQ_NIC_RATE_1G)
+		rate |= FW2X_RATE_1G;
+
+	if (speed & AQ_NIC_RATE_100M)
+		rate |= FW2X_RATE_100M;
+
+	return rate;
+}
+
+static u32 fw2x_to_eee_mask(u32 speed)
+{
+	u32 rate = 0;
+
+	if (speed & HW_ATL_FW2X_CAP_EEE_10G_MASK)
+		rate |= AQ_NIC_RATE_EEE_10G;
+
+	if (speed & HW_ATL_FW2X_CAP_EEE_5G_MASK)
+		rate |= AQ_NIC_RATE_EEE_5G;
+
+	if (speed & HW_ATL_FW2X_CAP_EEE_2G5_MASK)
+		rate |= AQ_NIC_RATE_EEE_2G5;
+
+	if (speed & HW_ATL_FW2X_CAP_EEE_1G_MASK)
+		rate |= AQ_NIC_RATE_EEE_1G;
+
+	return rate;
+}
+
+static int aq_fw2x_set_link_speed(struct aq_hw_s *self, u32 speed)
+{
+	u32 val = link_speed_mask_2fw2x_ratemask(speed);
+
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL_ADDR, val);
+
+	return 0;
+}
+
+static void aq_fw2x_set_mpi_flow_control(struct aq_hw_s *self, u32 *mpi_state)
+{
+	if (self->aq_nic_cfg->flow_control & AQ_NIC_FC_RX)
+		*mpi_state |= BIT(CAPS_HI_PAUSE);
+	else
+		*mpi_state &= ~BIT(CAPS_HI_PAUSE);
+
+	if (self->aq_nic_cfg->flow_control & AQ_NIC_FC_TX)
+		*mpi_state |= BIT(CAPS_HI_ASYMMETRIC_PAUSE);
+	else
+		*mpi_state &= ~BIT(CAPS_HI_ASYMMETRIC_PAUSE);
+}
+
+static int aq_fw2x_set_state(struct aq_hw_s *self,
+			     enum hal_atl_utils_fw_state_e state)
+{
+	u32 mpi_state = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR);
+
+	switch (state) {
+	case MPI_INIT:
+		mpi_state &= ~BIT(CAPS_HI_LINK_DROP);
+		aq_fw2x_set_mpi_flow_control(self, &mpi_state);
+		break;
+	case MPI_DEINIT:
+		mpi_state |= BIT(CAPS_HI_LINK_DROP);
+		break;
+	case MPI_RESET:
+	case MPI_POWER:
+		/* No actions */
+		break;
+	}
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR, mpi_state);
+	return 0;
+}
+
+static int aq_fw2x_update_link_status(struct aq_hw_s *self)
+{
+	u32 mpi_state = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_STATE_ADDR);
+	u32 speed = mpi_state & (FW2X_RATE_100M | FW2X_RATE_1G |
+				FW2X_RATE_2G5 | FW2X_RATE_5G | FW2X_RATE_10G);
+	struct aq_hw_link_status_s *link_status = &self->aq_link_status;
+
+	if (speed) {
+		if (speed & FW2X_RATE_10G)
+			link_status->mbps = 10000;
+		else if (speed & FW2X_RATE_5G)
+			link_status->mbps = 5000;
+		else if (speed & FW2X_RATE_2G5)
+			link_status->mbps = 2500;
+		else if (speed & FW2X_RATE_1G)
+			link_status->mbps = 1000;
+		else if (speed & FW2X_RATE_100M)
+			link_status->mbps = 100;
+		else
+			link_status->mbps = 10000;
+	} else {
+		link_status->mbps = 0;
+	}
+
+	return 0;
+}
+
+static
+int aq_fw2x_get_mac_permanent(struct aq_hw_s *self, u8 *mac)
+{
+	int err = 0;
+	u32 h = 0U;
+	u32 l = 0U;
+	u32 mac_addr[2] = { 0 };
+	u32 efuse_addr = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_EFUSE_ADDR);
+
+	if (efuse_addr != 0) {
+		err = hw_atl_utils_fw_downld_dwords(self,
+						    efuse_addr + (40U * 4U),
+						    mac_addr,
+						    ARRAY_SIZE(mac_addr));
+		if (err)
+			return err;
+		mac_addr[0] = rte_constant_bswap32(mac_addr[0]);
+		mac_addr[1] = rte_constant_bswap32(mac_addr[1]);
+	}
+
+	ether_addr_copy((struct ether_addr *)mac_addr,
+			(struct ether_addr *)mac);
+
+	if ((mac[0] & 0x01U) || ((mac[0] | mac[1] | mac[2]) == 0x00U)) {
+		unsigned int rnd = (uint32_t)rte_rand();
+
+		//get_random_bytes(&rnd, sizeof(unsigned int));
+
+		l = 0xE3000000U
+			| (0xFFFFU & rnd)
+			| (0x00 << 16);
+		h = 0x8001300EU;
+
+		mac[5] = (u8)(0xFFU & l);
+		l >>= 8;
+		mac[4] = (u8)(0xFFU & l);
+		l >>= 8;
+		mac[3] = (u8)(0xFFU & l);
+		l >>= 8;
+		mac[2] = (u8)(0xFFU & l);
+		mac[1] = (u8)(0xFFU & h);
+		h >>= 8;
+		mac[0] = (u8)(0xFFU & h);
+	}
+	return err;
+}
+
+static int aq_fw2x_update_stats(struct aq_hw_s *self)
+{
+	int err = 0;
+	u32 mpi_opts = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR);
+	u32 orig_stats_val = mpi_opts & BIT(CAPS_HI_STATISTICS);
+
+	/* Toggle statistics bit for FW to update */
+	mpi_opts = mpi_opts ^ BIT(CAPS_HI_STATISTICS);
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR, mpi_opts);
+
+	/* Wait FW to report back */
+	AQ_HW_WAIT_FOR(orig_stats_val !=
+		       (aq_hw_read_reg(self, HW_ATL_FW2X_MPI_STATE2_ADDR) &
+				       BIT(CAPS_HI_STATISTICS)),
+		       1U, 10000U);
+	if (err)
+		return err;
+
+	return hw_atl_utils_update_stats(self);
+}
+
+static int aq_fw2x_get_temp(struct aq_hw_s *self, int *temp)
+{
+	int err = 0;
+	u32 mpi_opts = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR);
+	u32 temp_val = mpi_opts & BIT(CAPS_HI_TEMPERATURE);
+	u32 temp_res;
+
+	/* Toggle statistics bit for FW to 0x36C.18 (CAPS_HI_TEMPERATURE) */
+	mpi_opts = mpi_opts ^ BIT(CAPS_HI_TEMPERATURE);
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR, mpi_opts);
+
+	/* Wait FW to report back */
+	AQ_HW_WAIT_FOR(temp_val !=
+			(aq_hw_read_reg(self, HW_ATL_FW2X_MPI_STATE2_ADDR) &
+					BIT(CAPS_HI_TEMPERATURE)), 1U, 10000U);
+	err = hw_atl_utils_fw_downld_dwords(self,
+				self->mbox_addr +
+				offsetof(struct hw_aq_atl_utils_mbox, info) +
+				offsetof(struct hw_aq_info, phy_temperature),
+				&temp_res,
+				sizeof(temp_res) / sizeof(u32));
+
+	if (err)
+		return err;
+
+	*temp = temp_res  * 100 / 256;
+	return 0;
+}
+
+static int aq_fw2x_get_cable_len(struct aq_hw_s *self, int *cable_len)
+{
+	int err = 0;
+	u32 cable_len_res;
+
+	err = hw_atl_utils_fw_downld_dwords(self,
+				self->mbox_addr +
+				offsetof(struct hw_aq_atl_utils_mbox, info) +
+				offsetof(struct hw_aq_info, phy_temperature),
+				&cable_len_res,
+				sizeof(cable_len_res) / sizeof(u32));
+
+	if (err)
+		return err;
+
+	*cable_len = (cable_len_res >> 16) & 0xFF;
+	return 0;
+}
+
+#ifndef ETH_ALEN
+#define ETH_ALEN 6
+#endif
+
+static int aq_fw2x_set_sleep_proxy(struct aq_hw_s *self, u8 *mac)
+{
+	int err = 0;
+	struct hw_aq_atl_utils_fw_rpc *rpc = NULL;
+	struct offload_info *cfg = NULL;
+	unsigned int rpc_size = 0U;
+	u32 mpi_opts;
+
+	rpc_size = sizeof(rpc->msg_id) + sizeof(*cfg);
+
+	err = hw_atl_utils_fw_rpc_wait(self, &rpc);
+	if (err < 0)
+		goto err_exit;
+
+	memset(rpc, 0, rpc_size);
+	cfg = (struct offload_info *)(&rpc->msg_id + 1);
+
+	memcpy(cfg->mac_addr, mac, ETH_ALEN);
+	cfg->len = sizeof(*cfg);
+
+	/* Clear bit 0x36C.23 */
+	mpi_opts = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR);
+	mpi_opts &= ~HW_ATL_FW2X_CAP_SLEEP_PROXY;
+
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR, mpi_opts);
+
+	err = hw_atl_utils_fw_rpc_call(self, rpc_size);
+	if (err < 0)
+		goto err_exit;
+
+	/* Set bit 0x36C.23 */
+	mpi_opts |= HW_ATL_FW2X_CAP_SLEEP_PROXY;
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR, mpi_opts);
+
+	AQ_HW_WAIT_FOR((aq_hw_read_reg(self, HW_ATL_FW2X_MPI_STATE2_ADDR) &
+			HW_ATL_FW2X_CAP_SLEEP_PROXY), 1U, 10000U);
+err_exit:
+	return err;
+}
+
+static int aq_fw2x_set_wol_params(struct aq_hw_s *self, u8 *mac)
+{
+	int err = 0;
+	struct fw2x_msg_wol *msg = NULL;
+	u32 mpi_opts;
+
+	struct hw_aq_atl_utils_fw_rpc *rpc = NULL;
+
+	err = hw_atl_utils_fw_rpc_wait(self, &rpc);
+	if (err < 0)
+		goto err_exit;
+
+	msg = (struct fw2x_msg_wol *)rpc;
+
+	msg->msg_id = HAL_ATLANTIC_UTILS_FW2X_MSG_WOL;
+	msg->magic_packet_enabled = true;
+	memcpy(msg->hw_addr, mac, ETH_ALEN);
+
+	mpi_opts = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR);
+	mpi_opts &= ~(HW_ATL_FW2X_CAP_SLEEP_PROXY | HW_ATL_FW2X_CAP_WOL);
+
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR, mpi_opts);
+
+	err = hw_atl_utils_fw_rpc_call(self, sizeof(*msg));
+	if (err < 0)
+		goto err_exit;
+
+	/* Set bit 0x36C.24 */
+	mpi_opts |= HW_ATL_FW2X_CAP_WOL;
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR, mpi_opts);
+
+	AQ_HW_WAIT_FOR((aq_hw_read_reg(self, HW_ATL_FW2X_MPI_STATE2_ADDR) &
+			HW_ATL_FW2X_CAP_WOL), 1U, 10000U);
+err_exit:
+	return err;
+}
+
+static int aq_fw2x_set_power(struct aq_hw_s *self,
+			     unsigned int power_state __rte_unused,
+			     u8 *mac)
+{
+	int err = 0;
+
+	if (self->aq_nic_cfg->wol & AQ_NIC_WOL_ENABLED) {
+		err = aq_fw2x_set_sleep_proxy(self, mac);
+		if (err < 0)
+			goto err_exit;
+		err = aq_fw2x_set_wol_params(self, mac);
+		if (err < 0)
+			goto err_exit;
+	}
+err_exit:
+	return err;
+}
+
+static int aq_fw2x_set_eee_rate(struct aq_hw_s *self, u32 speed)
+{
+	u32 mpi_opts = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR);
+	mpi_opts &= ~(HW_ATL_FW2X_CAP_EEE_1G_MASK |
+		HW_ATL_FW2X_CAP_EEE_2G5_MASK | HW_ATL_FW2X_CAP_EEE_5G_MASK |
+		HW_ATL_FW2X_CAP_EEE_10G_MASK);
+
+	if (speed & AQ_NIC_RATE_EEE_10G)
+		mpi_opts |= HW_ATL_FW2X_CAP_EEE_10G_MASK;
+
+	if (speed & AQ_NIC_RATE_EEE_5G)
+		mpi_opts |= HW_ATL_FW2X_CAP_EEE_5G_MASK;
+
+	if (speed & AQ_NIC_RATE_EEE_2G5)
+		mpi_opts |= HW_ATL_FW2X_CAP_EEE_2G5_MASK;
+
+	if (speed & AQ_NIC_RATE_EEE_1G)
+		mpi_opts |= HW_ATL_FW2X_CAP_EEE_1G_MASK;
+
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR, mpi_opts);
+
+	return 0;
+}
+
+static int aq_fw2x_get_eee_rate(struct aq_hw_s *self, u32 *rate,
+					u32 *supported_rates)
+{
+	int err = 0;
+	u32 caps_hi;
+	u32 mpi_state;
+
+	err = hw_atl_utils_fw_downld_dwords(self,
+				self->mbox_addr +
+				offsetof(struct hw_aq_atl_utils_mbox, info) +
+				offsetof(struct hw_aq_info, caps_hi),
+				&caps_hi,
+				sizeof(caps_hi) / sizeof(u32));
+
+	if (err)
+		return err;
+
+	*supported_rates = fw2x_to_eee_mask(caps_hi);
+
+	mpi_state = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_STATE2_ADDR);
+	*rate = fw2x_to_eee_mask(mpi_state);
+
+	return err;
+}
+
+
+
+static int aq_fw2x_set_flow_control(struct aq_hw_s *self)
+{
+	u32 mpi_state = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR);
+
+	aq_fw2x_set_mpi_flow_control(self, &mpi_state);
+
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR, mpi_state);
+
+	return 0;
+}
+
+static int aq_fw2x_led_control(struct aq_hw_s *self, u32 mode)
+{
+	if (self->fw_ver_actual < HW_ATL_FW_FEATURE_LED)
+		return -EOPNOTSUPP;
+
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_LED_ADDR, mode);
+	return 0;
+}
+
+static int aq_fw2x_get_eeprom(struct aq_hw_s *self, u32 *data, u32 len)
+{
+	int err = 0;
+	struct smbus_read_request request;
+	u32 mpi_opts;
+	u32 result = 0;
+
+	if (self->fw_ver_actual < HW_ATL_FW_FEATURE_EEPROM)
+		return -EOPNOTSUPP;
+
+	request.device_id = SMBUS_DEVICE_ID;
+	request.address = 0;
+	request.length = len;
+
+	/* Write SMBUS request to cfg memory */
+	err = hw_atl_utils_fw_upload_dwords(self, self->rpc_addr,
+				(u32 *)(void *)&request,
+				RTE_ALIGN(sizeof(request), sizeof(u32)));
+
+	if (err < 0)
+		return err;
+
+	/* Toggle 0x368.SMBUS_READ_REQUEST bit */
+	mpi_opts = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL_ADDR);
+	mpi_opts ^= SMBUS_READ_REQUEST;
+
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL_ADDR, mpi_opts);
+
+	/* Wait until REQUEST_BIT matched in 0x370 */
+
+	AQ_HW_WAIT_FOR((aq_hw_read_reg(self, HW_ATL_FW2X_MPI_STATE_ADDR) &
+		SMBUS_READ_REQUEST) == (mpi_opts & SMBUS_READ_REQUEST),
+		10U, 10000U);
+
+	if (err < 0)
+		return err;
+
+	err = hw_atl_utils_fw_downld_dwords(self, self->rpc_addr + sizeof(u32),
+			&result,
+			RTE_ALIGN(sizeof(result), sizeof(u32)));
+
+	if (err < 0)
+		return err;
+
+	if (result == 0) {
+		err = hw_atl_utils_fw_downld_dwords(self,
+				self->rpc_addr + sizeof(u32) * 2,
+				data,
+				RTE_ALIGN(len, sizeof(u32)));
+
+		if (err < 0)
+			return err;
+	}
+
+	return 0;
+}
+
+
+static int aq_fw2x_set_eeprom(struct aq_hw_s *self, u32 *data, u32 len)
+{
+	struct smbus_write_request request;
+	u32 mpi_opts, result = 0;
+	int err = 0;
+
+	if (self->fw_ver_actual < HW_ATL_FW_FEATURE_EEPROM)
+		return -EOPNOTSUPP;
+
+	request.device_id = SMBUS_DEVICE_ID;
+	request.address = 0;
+	request.length = len;
+
+	/* Write SMBUS request to cfg memory */
+	err = hw_atl_utils_fw_upload_dwords(self, self->rpc_addr,
+				(u32 *)(void *)&request,
+				RTE_ALIGN(sizeof(request), sizeof(u32)));
+
+	if (err < 0)
+		return err;
+
+	/* Write SMBUS data to cfg memory */
+	err = hw_atl_utils_fw_upload_dwords(self,
+				self->rpc_addr + sizeof(request),
+				(u32 *)(void *)data,
+				RTE_ALIGN(len, sizeof(u32)));
+
+	if (err < 0)
+		return err;
+
+	/* Toggle 0x368.SMBUS_WRITE_REQUEST bit */
+	mpi_opts = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL_ADDR);
+	mpi_opts ^= SMBUS_WRITE_REQUEST;
+
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL_ADDR, mpi_opts);
+
+	/* Wait until REQUEST_BIT matched in 0x370 */
+	AQ_HW_WAIT_FOR((aq_hw_read_reg(self, HW_ATL_FW2X_MPI_STATE_ADDR) &
+		SMBUS_WRITE_REQUEST) == (mpi_opts & SMBUS_WRITE_REQUEST),
+		10U, 10000U);
+
+	if (err < 0)
+		return err;
+
+	/* Read status of write operation */
+	err = hw_atl_utils_fw_downld_dwords(self, self->rpc_addr + sizeof(u32),
+				&result,
+				RTE_ALIGN(sizeof(result), sizeof(u32)));
+
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+
+const struct aq_fw_ops aq_fw_2x_ops = {
+	.init = aq_fw2x_init,
+	.deinit = aq_fw2x_deinit,
+	.reset = NULL,
+	.get_mac_permanent = aq_fw2x_get_mac_permanent,
+	.set_link_speed = aq_fw2x_set_link_speed,
+	.set_state = aq_fw2x_set_state,
+	.update_link_status = aq_fw2x_update_link_status,
+	.update_stats = aq_fw2x_update_stats,
+	.set_power = aq_fw2x_set_power,
+	.get_temp = aq_fw2x_get_temp,
+	.get_cable_len = aq_fw2x_get_cable_len,
+	.set_eee_rate = aq_fw2x_set_eee_rate,
+	.get_eee_rate = aq_fw2x_get_eee_rate,
+	.set_flow_control = aq_fw2x_set_flow_control,
+	.led_control = aq_fw2x_led_control,
+	.get_eeprom = aq_fw2x_get_eeprom,
+	.set_eeprom = aq_fw2x_set_eeprom,
+};
diff --git a/drivers/net/atlantic/meson.build b/drivers/net/atlantic/meson.build
new file mode 100644
index 00000000..28fb97ca
--- /dev/null
+++ b/drivers/net/atlantic/meson.build
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Aquantia Corporation
+
+sources = files(
+	'atl_rxtx.c',
+	'atl_ethdev.c',
+	'atl_hw_regs.c',
+	'hw_atl/hw_atl_b0.c',
+	'hw_atl/hw_atl_llh.c',
+	'hw_atl/hw_atl_utils_fw2x.c',
+	'hw_atl/hw_atl_utils.c',
+)
diff --git a/drivers/net/atlantic/rte_pmd_atlantic_version.map b/drivers/net/atlantic/rte_pmd_atlantic_version.map
new file mode 100644
index 00000000..521e51f4
--- /dev/null
+++ b/drivers/net/atlantic/rte_pmd_atlantic_version.map
@@ -0,0 +1,4 @@
+DPDK_18.11 {
+
+	local: *;
+};
diff --git a/drivers/net/avf/Makefile b/drivers/net/avf/Makefile
index 3f815bbc..aec6e4cb 100644
--- a/drivers/net/avf/Makefile
+++ b/drivers/net/avf/Makefile
@@ -8,7 +8,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 #
 LIB = librte_pmd_avf.a
 
-CFLAGS += -O3
+CFLAGS += -O3 $(WERROR_FLAGS) -Wno-strict-aliasing -DALLOW_EXPERIMENTAL_API
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash
 LDLIBS += -lrte_bus_pci
diff --git a/drivers/net/avf/avf_ethdev.c b/drivers/net/avf/avf_ethdev.c
index 3a2baaf2..13eec1b4 100644
--- a/drivers/net/avf/avf_ethdev.c
+++ b/drivers/net/avf/avf_ethdev.c
@@ -154,7 +154,6 @@ static int
 avf_init_rss(struct avf_adapter *adapter)
 {
 	struct avf_info *vf =  AVF_DEV_PRIVATE_TO_VF(adapter);
-	struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(adapter);
 	struct rte_eth_rss_conf *rss_conf;
 	uint8_t i, j, nb_q;
 	int ret;
@@ -259,11 +258,8 @@ avf_init_rxq(struct rte_eth_dev *dev, struct avf_rx_queue *rxq)
 static int
 avf_init_queues(struct rte_eth_dev *dev)
 {
-	struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct avf_rx_queue **rxq =
 		(struct avf_rx_queue **)dev->data->rx_queues;
-	struct avf_tx_queue **txq =
-		(struct avf_tx_queue **)dev->data->tx_queues;
 	int i, ret = AVF_SUCCESS;
 
 	for (i = 0; i < dev->data->nb_rx_queues; i++) {
@@ -415,7 +411,6 @@ avf_dev_start(struct rte_eth_dev *dev)
 		AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 	struct rte_intr_handle *intr_handle = dev->intr_handle;
 
 	PMD_INIT_FUNC_TRACE();
@@ -476,9 +471,7 @@ avf_dev_stop(struct rte_eth_dev *dev)
 	struct avf_adapter *adapter =
 		AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 	struct rte_intr_handle *intr_handle = dev->intr_handle;
-	int ret, i;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -503,8 +496,6 @@ avf_dev_stop(struct rte_eth_dev *dev)
 static void
 avf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
-	struct avf_adapter *adapter =
-		AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 
 	memset(dev_info, 0, sizeof(*dev_info));
@@ -523,8 +514,6 @@ avf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 		DEV_RX_OFFLOAD_UDP_CKSUM |
 		DEV_RX_OFFLOAD_TCP_CKSUM |
 		DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
-		DEV_RX_OFFLOAD_CRC_STRIP |
-		DEV_RX_OFFLOAD_KEEP_CRC |
 		DEV_RX_OFFLOAD_SCATTER |
 		DEV_RX_OFFLOAD_JUMBO_FRAME |
 		DEV_RX_OFFLOAD_VLAN_FILTER;
@@ -569,7 +558,7 @@ avf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 }
 
 static const uint32_t *
-avf_dev_supported_ptypes_get(struct rte_eth_dev *dev)
+avf_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
 {
 	static const uint32_t ptypes[] = {
 		RTE_PTYPE_L2_ETHER,
@@ -915,7 +904,6 @@ avf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 static int
 avf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 {
-	struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	uint32_t frame_size = mtu + AVF_ETH_OVERHEAD;
 	int ret = 0;
 
@@ -1045,8 +1033,6 @@ avf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
 static int
 avf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 {
-	struct avf_adapter *adapter =
-		AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 	struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	uint16_t msix_intr;
@@ -1089,7 +1075,7 @@ avf_check_vf_reset_done(struct avf_hw *hw)
 static int
 avf_init_vf(struct rte_eth_dev *dev)
 {
-	int i, err, bufsz;
+	int err, bufsz;
 	struct avf_adapter *adapter =
 		AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
@@ -1198,7 +1184,6 @@ avf_dev_interrupt_handler(void *param)
 
 	avf_handle_virtchnl_msg(dev);
 
-done:
 	avf_enable_irq0(hw);
 }
 
@@ -1318,9 +1303,6 @@ avf_dev_uninit(struct rte_eth_dev *dev)
 	rte_free(vf->aq_resp);
 	vf->aq_resp = NULL;
 
-	rte_free(dev->data->mac_addrs);
-	dev->data->mac_addrs = NULL;
-
 	if (vf->rss_lut) {
 		rte_free(vf->rss_lut);
 		vf->rss_lut = NULL;
diff --git a/drivers/net/avf/avf_rxtx.c b/drivers/net/avf/avf_rxtx.c
index e03a136f..8c7a9672 100644
--- a/drivers/net/avf/avf_rxtx.c
+++ b/drivers/net/avf/avf_rxtx.c
@@ -247,7 +247,6 @@ alloc_rxq_mbufs(struct avf_rx_queue *rxq)
 static inline void
 release_rxq_mbufs(struct avf_rx_queue *rxq)
 {
-	struct rte_mbuf *mbuf;
 	uint16_t i;
 
 	if (!rxq->sw_ring)
@@ -310,9 +309,8 @@ avf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	struct avf_rx_queue *rxq;
 	const struct rte_memzone *mz;
 	uint32_t ring_size;
-	uint16_t len, i;
+	uint16_t len;
 	uint16_t rx_free_thresh;
-	uint16_t base, bsf, tc_mapping;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -428,13 +426,10 @@ avf_dev_tx_queue_setup(struct rte_eth_dev *dev,
 		       const struct rte_eth_txconf *tx_conf)
 {
 	struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	struct avf_adapter *ad =
-		AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct avf_tx_queue *txq;
 	const struct rte_memzone *mz;
 	uint32_t ring_size;
 	uint16_t tx_rs_thresh, tx_free_thresh;
-	uint16_t i, base, bsf, tc_mapping;
 	uint64_t offloads;
 
 	PMD_INIT_FUNC_TRACE();
@@ -515,8 +510,11 @@ avf_dev_tx_queue_setup(struct rte_eth_dev *dev,
 	txq->ops = &def_txq_ops;
 
 #ifdef RTE_LIBRTE_AVF_INC_VECTOR
-	if (check_tx_vec_allow(txq) == FALSE)
+	if (check_tx_vec_allow(txq) == FALSE) {
+		struct avf_adapter *ad =
+			AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 		ad->tx_vec_allowed = false;
+	}
 #endif
 
 	return 0;
@@ -1268,7 +1266,6 @@ static inline uint16_t
 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 {
 	struct avf_rx_queue *rxq = (struct avf_rx_queue *)rx_queue;
-	struct rte_eth_dev *dev;
 	uint16_t nb_rx = 0;
 
 	if (!nb_pkts)
@@ -1584,10 +1581,6 @@ avf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 
 		if (nb_ctx) {
 			/* Setup TX context descriptor if required */
-			volatile struct avf_tx_context_desc *ctx_txd =
-				(volatile struct avf_tx_context_desc *)
-					&txr[tx_id];
-			uint16_t cd_l2tag2 = 0;
 			uint64_t cd_type_cmd_tso_mss =
 				AVF_TX_DESC_DTYPE_CONTEXT;
 
@@ -1603,7 +1596,7 @@ avf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 				cd_type_cmd_tso_mss |=
 					avf_set_tso_ctx(tx_pkt, tx_offload);
 
-			AVF_DUMP_TX_DESC(txq, ctx_txd, tx_id);
+			AVF_DUMP_TX_DESC(txq, &txr[tx_id], tx_id);
 			txe->last_id = tx_last;
 			tx_id = txe->next_id;
 			txe = txn;
@@ -1925,7 +1918,7 @@ avf_dev_tx_desc_status(void *tx_queue, uint16_t offset)
 	return RTE_ETH_TX_DESC_FULL;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 avf_recv_pkts_vec(__rte_unused void *rx_queue,
 		  __rte_unused struct rte_mbuf **rx_pkts,
 		  __rte_unused uint16_t nb_pkts)
@@ -1933,7 +1926,7 @@ avf_recv_pkts_vec(__rte_unused void *rx_queue,
 	return 0;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 avf_recv_scattered_pkts_vec(__rte_unused void *rx_queue,
 			    __rte_unused struct rte_mbuf **rx_pkts,
 			    __rte_unused uint16_t nb_pkts)
@@ -1941,7 +1934,7 @@ avf_recv_scattered_pkts_vec(__rte_unused void *rx_queue,
 	return 0;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 avf_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 			 __rte_unused struct rte_mbuf **tx_pkts,
 			 __rte_unused uint16_t nb_pkts)
@@ -1949,13 +1942,13 @@ avf_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 	return 0;
 }
 
-int __attribute__((weak))
+__rte_weak int
 avf_rxq_vec_setup(__rte_unused struct avf_rx_queue *rxq)
 {
 	return -1;
 }
 
-int __attribute__((weak))
+__rte_weak int
 avf_txq_vec_setup(__rte_unused struct avf_tx_queue *txq)
 {
 	return -1;
diff --git a/drivers/net/avf/avf_rxtx.h b/drivers/net/avf/avf_rxtx.h
index 297d0776..898d2f38 100644
--- a/drivers/net/avf/avf_rxtx.h
+++ b/drivers/net/avf/avf_rxtx.h
@@ -201,17 +201,17 @@ int avf_txq_vec_setup(struct avf_tx_queue *txq);
 
 static inline
 void avf_dump_rx_descriptor(struct avf_rx_queue *rxq,
-			    const void *desc,
+			    const volatile void *desc,
 			    uint16_t rx_id)
 {
 #ifdef RTE_LIBRTE_AVF_16BYTE_RX_DESC
-	const union avf_16byte_rx_desc *rx_desc = desc;
+	const volatile union avf_16byte_rx_desc *rx_desc = desc;
 
 	printf("Queue %d Rx_desc %d: QW0: 0x%016"PRIx64" QW1: 0x%016"PRIx64"\n",
 	       rxq->queue_id, rx_id, rx_desc->read.pkt_addr,
 	       rx_desc->read.hdr_addr);
 #else
-	const union avf_32byte_rx_desc *rx_desc = desc;
+	const volatile union avf_32byte_rx_desc *rx_desc = desc;
 
 	printf("Queue %d Rx_desc %d: QW0: 0x%016"PRIx64" QW1: 0x%016"PRIx64
 	       " QW2: 0x%016"PRIx64" QW3: 0x%016"PRIx64"\n", rxq->queue_id,
@@ -225,10 +225,10 @@ void avf_dump_rx_descriptor(struct avf_rx_queue *rxq,
  */
 static inline
 void avf_dump_tx_descriptor(const struct avf_tx_queue *txq,
-			    const void *desc, uint16_t tx_id)
+			    const volatile void *desc, uint16_t tx_id)
 {
-	char *name;
-	const struct avf_tx_desc *tx_desc = desc;
+	const char *name;
+	const volatile struct avf_tx_desc *tx_desc = desc;
 	enum avf_tx_desc_dtype_value type;
 
 	type = (enum avf_tx_desc_dtype_value)rte_le_to_cpu_64(
diff --git a/drivers/net/avf/avf_rxtx_vec_sse.c b/drivers/net/avf/avf_rxtx_vec_sse.c
index 8275100f..343a6aac 100644
--- a/drivers/net/avf/avf_rxtx_vec_sse.c
+++ b/drivers/net/avf/avf_rxtx_vec_sse.c
@@ -621,7 +621,7 @@ avf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 	return nb_pkts;
 }
 
-void __attribute__((cold))
+static void __attribute__((cold))
 avf_rx_queue_release_mbufs_sse(struct avf_rx_queue *rxq)
 {
 	_avf_rx_queue_release_mbufs_vec(rxq);
diff --git a/drivers/net/avf/avf_vchnl.c b/drivers/net/avf/avf_vchnl.c
index fa71014e..fd90cc2c 100644
--- a/drivers/net/avf/avf_vchnl.c
+++ b/drivers/net/avf/avf_vchnl.c
@@ -69,7 +69,6 @@ avf_execute_vf_cmd(struct avf_adapter *adapter, struct avf_cmd_info *args)
 {
 	struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(adapter);
 	struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
-	struct avf_arq_event_info event_info;
 	enum avf_status_code ret;
 	int err = 0;
 	int i = 0;
@@ -600,7 +599,6 @@ avf_config_irq_map(struct avf_adapter *adapter)
 	struct virtchnl_irq_map_info *map_info;
 	struct virtchnl_vector_map *vecmap;
 	struct avf_cmd_info args;
-	uint32_t vector_id;
 	int len, i, err;
 
 	len = sizeof(struct virtchnl_irq_map_info) +
diff --git a/drivers/net/avf/base/avf_osdep.h b/drivers/net/avf/base/avf_osdep.h
index 9ef45968..442a5acd 100644
--- a/drivers/net/avf/base/avf_osdep.h
+++ b/drivers/net/avf/base/avf_osdep.h
@@ -93,8 +93,8 @@ typedef uint64_t        u64;
 #define avf_memset(a, b, c, d) memset((a), (b), (c))
 #define avf_memcpy(a, b, c, d) rte_memcpy((a), (b), (c))
 
-#define avf_usec_delay(x) rte_delay_us(x)
-#define avf_msec_delay(x) rte_delay_us(1000*(x))
+#define avf_usec_delay(x) rte_delay_us_sleep(x)
+#define avf_msec_delay(x) avf_usec_delay(1000 * (x))
 
 #define AVF_PCI_REG(reg)		rte_read32(reg)
 #define AVF_PCI_REG_ADDR(a, reg) \
diff --git a/drivers/net/avf/base/meson.build b/drivers/net/avf/base/meson.build
new file mode 100644
index 00000000..6f3d7192
--- /dev/null
+++ b/drivers/net/avf/base/meson.build
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
+
+sources = [
+	'avf_adminq.c',
+	'avf_common.c',
+]
+
+error_cflags = ['-Wno-pointer-to-int-cast']
+c_args = cflags
+if allow_experimental_apis
+	c_args += '-DALLOW_EXPERIMENTAL_API'
+endif
+foreach flag: error_cflags
+	if cc.has_argument(flag)
+		c_args += flag
+	endif
+endforeach
+
+base_lib = static_library('avf_base', sources,
+	dependencies: static_rte_eal,
+	c_args: c_args)
+base_objs = base_lib.extract_all_objects()
diff --git a/drivers/net/avf/meson.build b/drivers/net/avf/meson.build
new file mode 100644
index 00000000..2dfda9d4
--- /dev/null
+++ b/drivers/net/avf/meson.build
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
+
+cflags += ['-Wno-strict-aliasing']
+
+allow_experimental_apis = true
+
+subdir('base')
+objs = [base_objs]
+
+sources = files(
+	'avf_ethdev.c',
+	'avf_rxtx.c',
+	'avf_vchnl.c',
+)
+
+if arch_subdir == 'x86'
+	dpdk_conf.set('RTE_LIBRTE_AVF_INC_VECTOR', 1)
+	sources += files('avf_rxtx_vec_sse.c')
+endif
diff --git a/drivers/net/avp/avp_ethdev.c b/drivers/net/avp/avp_ethdev.c
index 761f6c1c..09388d05 100644
--- a/drivers/net/avp/avp_ethdev.c
+++ b/drivers/net/avp/avp_ethdev.c
@@ -1036,11 +1036,6 @@ eth_avp_dev_uninit(struct rte_eth_dev *eth_dev)
 		return ret;
 	}
 
-	if (eth_dev->data->mac_addrs != NULL) {
-		rte_free(eth_dev->data->mac_addrs);
-		eth_dev->data->mac_addrs = NULL;
-	}
-
 	return 0;
 }
 
@@ -2170,7 +2165,6 @@ avp_dev_info_get(struct rte_eth_dev *eth_dev,
 		dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
 		dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT;
 	}
-	dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
diff --git a/drivers/net/avp/meson.build b/drivers/net/avp/meson.build
index 6076c31b..b7ffdfc8 100644
--- a/drivers/net/avp/meson.build
+++ b/drivers/net/avp/meson.build
@@ -1,5 +1,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2018 Intel Corporation
 
+if host_machine.system() != 'linux'
+        build = false
+endif
 sources = files('avp_ethdev.c')
 install_headers('rte_avp_common.h', 'rte_avp_fifo.h')
diff --git a/drivers/net/axgbe/axgbe_ethdev.c b/drivers/net/axgbe/axgbe_ethdev.c
index 9ae9f063..e89c0ec2 100644
--- a/drivers/net/axgbe/axgbe_ethdev.c
+++ b/drivers/net/axgbe/axgbe_ethdev.c
@@ -364,7 +364,6 @@ axgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 		DEV_RX_OFFLOAD_IPV4_CKSUM |
 		DEV_RX_OFFLOAD_UDP_CKSUM  |
 		DEV_RX_OFFLOAD_TCP_CKSUM  |
-		DEV_RX_OFFLOAD_CRC_STRIP  |
 		DEV_RX_OFFLOAD_KEEP_CRC;
 
 	dev_info->tx_offload_capa =
@@ -719,9 +718,6 @@ eth_axgbe_dev_uninit(struct rte_eth_dev *eth_dev)
 		return 0;
 
 	pci_dev = RTE_DEV_TO_PCI(eth_dev->device);
-	/*Free macaddres*/
-	rte_free(eth_dev->data->mac_addrs);
-	eth_dev->data->mac_addrs = NULL;
 	eth_dev->dev_ops = NULL;
 	eth_dev->rx_pkt_burst = NULL;
 	eth_dev->tx_pkt_burst = NULL;
diff --git a/drivers/net/axgbe/axgbe_rxtx.c b/drivers/net/axgbe/axgbe_rxtx.c
index c5fd5f41..b5a29a95 100644
--- a/drivers/net/axgbe/axgbe_rxtx.c
+++ b/drivers/net/axgbe/axgbe_rxtx.c
@@ -74,7 +74,7 @@ int axgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 		(DMA_CH_INC * rxq->queue_id));
 	rxq->dma_tail_reg = (volatile uint32_t *)((uint8_t *)rxq->dma_regs +
 						  DMA_CH_RDTR_LO);
-	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
 		rxq->crc_len = ETHER_CRC_LEN;
 	else
 		rxq->crc_len = 0;
diff --git a/drivers/net/bnx2x/bnx2x.c b/drivers/net/bnx2x/bnx2x.c
index 4904eaf3..27975936 100644
--- a/drivers/net/bnx2x/bnx2x.c
+++ b/drivers/net/bnx2x/bnx2x.c
@@ -112,6 +112,7 @@ static void bnx2x_pf_disable(struct bnx2x_softc *sc);
 static void bnx2x_update_rx_prod(struct bnx2x_softc *sc,
 				 struct bnx2x_fastpath *fp,
 				 uint16_t rx_bd_prod, uint16_t rx_cq_prod);
+static void bnx2x_link_report_locked(struct bnx2x_softc *sc);
 static void bnx2x_link_report(struct bnx2x_softc *sc);
 void bnx2x_link_status_update(struct bnx2x_softc *sc);
 static int bnx2x_alloc_mem(struct bnx2x_softc *sc);
@@ -178,13 +179,14 @@ bnx2x_dma_alloc(struct bnx2x_softc *sc, size_t size, struct bnx2x_dma *dma,
 					SOCKET_ID_ANY,
 					RTE_MEMZONE_IOVA_CONTIG, align);
 	if (z == NULL) {
-		PMD_DRV_LOG(ERR, "DMA alloc failed for %s", msg);
+		PMD_DRV_LOG(ERR, sc, "DMA alloc failed for %s", msg);
 		return -ENOMEM;
 	}
 	dma->paddr = (uint64_t) z->iova;
 	dma->vaddr = z->addr;
 
-	PMD_DRV_LOG(DEBUG, "%s: virt=%p phys=%" PRIx64, msg, dma->vaddr, dma->paddr);
+	PMD_DRV_LOG(DEBUG, sc,
+		    "%s: virt=%p phys=%" PRIx64, msg, dma->vaddr, dma->paddr);
 
 	return 0;
 }
@@ -197,11 +199,12 @@ static int bnx2x_acquire_hw_lock(struct bnx2x_softc *sc, uint32_t resource)
 	uint32_t hw_lock_control_reg;
 	int cnt;
 
-	PMD_INIT_FUNC_TRACE();
+	if (resource)
+		PMD_INIT_FUNC_TRACE(sc);
 
 	/* validate the resource is within range */
 	if (resource > HW_LOCK_MAX_RESOURCE_VALUE) {
-		PMD_DRV_LOG(NOTICE,
+		PMD_DRV_LOG(NOTICE, sc,
 			    "resource 0x%x > HW_LOCK_MAX_RESOURCE_VALUE",
 			    resource);
 		return -1;
@@ -217,7 +220,7 @@ static int bnx2x_acquire_hw_lock(struct bnx2x_softc *sc, uint32_t resource)
 	/* validate the resource is not already taken */
 	lock_status = REG_RD(sc, hw_lock_control_reg);
 	if (lock_status & resource_bit) {
-		PMD_DRV_LOG(NOTICE,
+		PMD_DRV_LOG(NOTICE, sc,
 			    "resource in use (status 0x%x bit 0x%x)",
 			    lock_status, resource_bit);
 		return -1;
@@ -233,7 +236,8 @@ static int bnx2x_acquire_hw_lock(struct bnx2x_softc *sc, uint32_t resource)
 		DELAY(5000);
 	}
 
-	PMD_DRV_LOG(NOTICE, "Resource lock timeout!");
+	PMD_DRV_LOG(NOTICE, sc, "Resource 0x%x resource_bit 0x%x lock timeout!",
+		    resource, resource_bit);
 	return -1;
 }
 
@@ -244,13 +248,14 @@ static int bnx2x_release_hw_lock(struct bnx2x_softc *sc, uint32_t resource)
 	int func = SC_FUNC(sc);
 	uint32_t hw_lock_control_reg;
 
-	PMD_INIT_FUNC_TRACE();
+	if (resource)
+		PMD_INIT_FUNC_TRACE(sc);
 
 	/* validate the resource is within range */
 	if (resource > HW_LOCK_MAX_RESOURCE_VALUE) {
-		PMD_DRV_LOG(NOTICE,
-			    "resource 0x%x > HW_LOCK_MAX_RESOURCE_VALUE",
-			    resource);
+		PMD_DRV_LOG(NOTICE, sc,
+			    "(resource 0x%x > HW_LOCK_MAX_RESOURCE_VALUE)"
+			    " resource_bit 0x%x", resource, resource_bit);
 		return -1;
 	}
 
@@ -264,7 +269,7 @@ static int bnx2x_release_hw_lock(struct bnx2x_softc *sc, uint32_t resource)
 	/* validate the resource is currently taken */
 	lock_status = REG_RD(sc, hw_lock_control_reg);
 	if (!(lock_status & resource_bit)) {
-		PMD_DRV_LOG(NOTICE,
+		PMD_DRV_LOG(NOTICE, sc,
 			    "resource not in use (status 0x%x bit 0x%x)",
 			    lock_status, resource_bit);
 		return -1;
@@ -274,6 +279,18 @@ static int bnx2x_release_hw_lock(struct bnx2x_softc *sc, uint32_t resource)
 	return 0;
 }
 
+static void bnx2x_acquire_phy_lock(struct bnx2x_softc *sc)
+{
+	BNX2X_PHY_LOCK(sc);
+	bnx2x_acquire_hw_lock(sc, HW_LOCK_RESOURCE_MDIO);
+}
+
+static void bnx2x_release_phy_lock(struct bnx2x_softc *sc)
+{
+	bnx2x_release_hw_lock(sc, HW_LOCK_RESOURCE_MDIO);
+	BNX2X_PHY_UNLOCK(sc);
+}
+
 /* copy command into DMAE command memory and set DMAE command Go */
 void bnx2x_post_dmae(struct bnx2x_softc *sc, struct dmae_command *dmae, int idx)
 {
@@ -366,7 +383,7 @@ bnx2x_issue_dmae_with_comp(struct bnx2x_softc *sc, struct dmae_command *dmae)
 		if (!timeout ||
 		    (sc->recovery_state != BNX2X_RECOVERY_DONE &&
 		     sc->recovery_state != BNX2X_RECOVERY_NIC_LOADING)) {
-			PMD_DRV_LOG(INFO, "DMAE timeout!");
+			PMD_DRV_LOG(INFO, sc, "DMAE timeout!");
 			return DMAE_TIMEOUT;
 		}
 
@@ -375,7 +392,7 @@ bnx2x_issue_dmae_with_comp(struct bnx2x_softc *sc, struct dmae_command *dmae)
 	}
 
 	if (*wb_comp & DMAE_PCI_ERR_FLAG) {
-		PMD_DRV_LOG(INFO, "DMAE PCI error!");
+		PMD_DRV_LOG(INFO, sc, "DMAE PCI error!");
 		return DMAE_PCI_ERROR;
 	}
 
@@ -534,7 +551,7 @@ void
 elink_cb_event_log(__rte_unused struct bnx2x_softc *sc,
 		   __rte_unused const elink_log_id_t elink_log_id, ...)
 {
-	PMD_DRV_LOG(DEBUG, "ELINK EVENT LOG (%d)", elink_log_id);
+	PMD_DRV_LOG(DEBUG, sc, "ELINK EVENT LOG (%d)", elink_log_id);
 }
 
 static int bnx2x_set_spio(struct bnx2x_softc *sc, int spio, uint32_t mode)
@@ -543,7 +560,7 @@ static int bnx2x_set_spio(struct bnx2x_softc *sc, int spio, uint32_t mode)
 
 	/* Only 2 SPIOs are configurable */
 	if ((spio != MISC_SPIO_SPIO4) && (spio != MISC_SPIO_SPIO5)) {
-		PMD_DRV_LOG(NOTICE, "Invalid SPIO 0x%x", spio);
+		PMD_DRV_LOG(NOTICE, sc, "Invalid SPIO 0x%x", spio);
 		return -1;
 	}
 
@@ -593,7 +610,7 @@ static int bnx2x_gpio_read(struct bnx2x_softc *sc, int gpio_num, uint8_t port)
 	uint32_t gpio_reg;
 
 	if (gpio_num > MISC_REGISTERS_GPIO_3) {
-		PMD_DRV_LOG(NOTICE, "Invalid GPIO %d", gpio_num);
+		PMD_DRV_LOG(NOTICE, sc, "Invalid GPIO %d", gpio_num);
 		return -1;
 	}
 
@@ -618,7 +635,7 @@ bnx2x_gpio_write(struct bnx2x_softc *sc, int gpio_num, uint32_t mode, uint8_t po
 	uint32_t gpio_reg;
 
 	if (gpio_num > MISC_REGISTERS_GPIO_3) {
-		PMD_DRV_LOG(NOTICE, "Invalid GPIO %d", gpio_num);
+		PMD_DRV_LOG(NOTICE, sc, "Invalid GPIO %d", gpio_num);
 		return -1;
 	}
 
@@ -687,7 +704,8 @@ bnx2x_gpio_mult_write(struct bnx2x_softc *sc, uint8_t pins, uint32_t mode)
 		break;
 
 	default:
-		PMD_DRV_LOG(NOTICE, "Invalid GPIO mode assignment %d", mode);
+		PMD_DRV_LOG(NOTICE, sc,
+			    "Invalid GPIO mode assignment %d", mode);
 		bnx2x_release_hw_lock(sc, HW_LOCK_RESOURCE_GPIO);
 		return -1;
 	}
@@ -713,7 +731,7 @@ bnx2x_gpio_int_write(struct bnx2x_softc *sc, int gpio_num, uint32_t mode,
 	uint32_t gpio_reg;
 
 	if (gpio_num > MISC_REGISTERS_GPIO_3) {
-		PMD_DRV_LOG(NOTICE, "Invalid GPIO %d", gpio_num);
+		PMD_DRV_LOG(NOTICE, sc, "Invalid GPIO %d", gpio_num);
 		return -1;
 	}
 
@@ -790,7 +808,7 @@ elink_cb_fw_command(struct bnx2x_softc *sc, uint32_t command, uint32_t param)
 	SHMEM_WR(sc, func_mb[mb_idx].drv_mb_param, param);
 	SHMEM_WR(sc, func_mb[mb_idx].drv_mb_header, (command | seq));
 
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 		    "wrote command 0x%08x to FW MB param 0x%08x",
 		    (command | seq), param);
 
@@ -805,7 +823,7 @@ elink_cb_fw_command(struct bnx2x_softc *sc, uint32_t command, uint32_t param)
 		rc &= FW_MSG_CODE_MASK;
 	} else {
 		/* Ruh-roh! */
-		PMD_DRV_LOG(NOTICE, "FW failed to respond!");
+		PMD_DRV_LOG(NOTICE, sc, "FW failed to respond!");
 		rc = 0;
 	}
 
@@ -1023,12 +1041,12 @@ bnx2x_sp_post(struct bnx2x_softc *sc, int command, int cid, uint32_t data_hi,
 
 	if (common) {
 		if (!atomic_load_acq_long(&sc->eq_spq_left)) {
-			PMD_DRV_LOG(INFO, "EQ ring is full!");
+			PMD_DRV_LOG(INFO, sc, "EQ ring is full!");
 			return -1;
 		}
 	} else {
 		if (!atomic_load_acq_long(&sc->cq_spq_left)) {
-			PMD_DRV_LOG(INFO, "SPQ ring is full!");
+			PMD_DRV_LOG(INFO, sc, "SPQ ring is full!");
 			return -1;
 		}
 	}
@@ -1061,7 +1079,7 @@ bnx2x_sp_post(struct bnx2x_softc *sc, int command, int cid, uint32_t data_hi,
 		atomic_subtract_acq_long(&sc->cq_spq_left, 1);
 	}
 
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 		    "SPQE[%x] (%x:%x) (cmd, common?) (%d,%d) hw_cid %x"
 		    "data (%x:%x) type(0x%x) left (CQ, EQ) (%lx,%lx)",
 		    sc->spq_prod_idx,
@@ -1132,44 +1150,45 @@ bnx2x_sp_event(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
 	enum ecore_queue_cmd drv_cmd = ECORE_Q_CMD_MAX;
 	struct ecore_queue_sp_obj *q_obj = &BNX2X_SP_OBJ(sc, fp).q_obj;
 
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 		    "fp=%d cid=%d got ramrod #%d state is %x type is %d",
 		    fp->index, cid, command, sc->state,
 		    rr_cqe->ramrod_cqe.ramrod_type);
 
 	switch (command) {
 	case (RAMROD_CMD_ID_ETH_CLIENT_UPDATE):
-		PMD_DRV_LOG(DEBUG, "got UPDATE ramrod. CID %d", cid);
+		PMD_DRV_LOG(DEBUG, sc, "got UPDATE ramrod. CID %d", cid);
 		drv_cmd = ECORE_Q_CMD_UPDATE;
 		break;
 
 	case (RAMROD_CMD_ID_ETH_CLIENT_SETUP):
-		PMD_DRV_LOG(DEBUG, "got MULTI[%d] setup ramrod", cid);
+		PMD_DRV_LOG(DEBUG, sc, "got MULTI[%d] setup ramrod", cid);
 		drv_cmd = ECORE_Q_CMD_SETUP;
 		break;
 
 	case (RAMROD_CMD_ID_ETH_TX_QUEUE_SETUP):
-		PMD_DRV_LOG(DEBUG, "got MULTI[%d] tx-only setup ramrod", cid);
+		PMD_DRV_LOG(DEBUG, sc,
+			    "got MULTI[%d] tx-only setup ramrod", cid);
 		drv_cmd = ECORE_Q_CMD_SETUP_TX_ONLY;
 		break;
 
 	case (RAMROD_CMD_ID_ETH_HALT):
-		PMD_DRV_LOG(DEBUG, "got MULTI[%d] halt ramrod", cid);
+		PMD_DRV_LOG(DEBUG, sc, "got MULTI[%d] halt ramrod", cid);
 		drv_cmd = ECORE_Q_CMD_HALT;
 		break;
 
 	case (RAMROD_CMD_ID_ETH_TERMINATE):
-		PMD_DRV_LOG(DEBUG, "got MULTI[%d] teminate ramrod", cid);
+		PMD_DRV_LOG(DEBUG, sc, "got MULTI[%d] teminate ramrod", cid);
 		drv_cmd = ECORE_Q_CMD_TERMINATE;
 		break;
 
 	case (RAMROD_CMD_ID_ETH_EMPTY):
-		PMD_DRV_LOG(DEBUG, "got MULTI[%d] empty ramrod", cid);
+		PMD_DRV_LOG(DEBUG, sc, "got MULTI[%d] empty ramrod", cid);
 		drv_cmd = ECORE_Q_CMD_EMPTY;
 		break;
 
 	default:
-		PMD_DRV_LOG(DEBUG,
+		PMD_DRV_LOG(DEBUG, sc,
 			    "ERROR: unexpected MC reply (%d)"
 			    "on fp[%d]", command, fp->index);
 		return;
@@ -1191,7 +1210,7 @@ bnx2x_sp_event(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
 
 	atomic_add_acq_long(&sc->cq_spq_left, 1);
 
-	PMD_DRV_LOG(DEBUG, "sc->cq_spq_left 0x%lx",
+	PMD_DRV_LOG(DEBUG, sc, "sc->cq_spq_left 0x%lx",
 		    atomic_load_acq_long(&sc->cq_spq_left));
 }
 
@@ -1387,7 +1406,7 @@ bnx2x_del_all_macs(struct bnx2x_softc *sc, struct ecore_vlan_mac_obj *mac_obj,
 
 	rc = mac_obj->delete_all(sc, mac_obj, &vlan_mac_flags, &ramrod_flags);
 	if (rc < 0)
-		PMD_DRV_LOG(ERR, "Failed to delete MACs (%d)", rc);
+		PMD_DRV_LOG(ERR, sc, "Failed to delete MACs (%d)", rc);
 
 	return rc;
 }
@@ -1538,13 +1557,13 @@ static int bnx2x_nic_load_no_mcp(struct bnx2x_softc *sc)
 	int path = SC_PATH(sc);
 	int port = SC_PORT(sc);
 
-	PMD_DRV_LOG(INFO, "NO MCP - load counts[%d]      %d, %d, %d",
+	PMD_DRV_LOG(INFO, sc, "NO MCP - load counts[%d]      %d, %d, %d",
 		    path, load_count[path][0], load_count[path][1],
 		    load_count[path][2]);
 
 	load_count[path][0]++;
 	load_count[path][1 + port]++;
-	PMD_DRV_LOG(INFO, "NO MCP - new load counts[%d]  %d, %d, %d",
+	PMD_DRV_LOG(INFO, sc, "NO MCP - new load counts[%d]  %d, %d, %d",
 		    path, load_count[path][0], load_count[path][1],
 		    load_count[path][2]);
 	if (load_count[path][0] == 1)
@@ -1561,12 +1580,12 @@ static int bnx2x_nic_unload_no_mcp(struct bnx2x_softc *sc)
 	int port = SC_PORT(sc);
 	int path = SC_PATH(sc);
 
-	PMD_DRV_LOG(INFO, "NO MCP - load counts[%d]      %d, %d, %d",
+	PMD_DRV_LOG(INFO, sc, "NO MCP - load counts[%d]      %d, %d, %d",
 		    path, load_count[path][0], load_count[path][1],
 		    load_count[path][2]);
 	load_count[path][0]--;
 	load_count[path][1 + port]--;
-	PMD_DRV_LOG(INFO, "NO MCP - new load counts[%d]  %d, %d, %d",
+	PMD_DRV_LOG(INFO, sc, "NO MCP - new load counts[%d]  %d, %d, %d",
 		    path, load_count[path][0], load_count[path][1],
 		    load_count[path][2]);
 	if (load_count[path][0] == 0) {
@@ -1646,7 +1665,7 @@ static int bnx2x_func_wait_started(struct bnx2x_softc *sc)
 		 */
 		struct ecore_func_state_params func_params = { NULL };
 
-		PMD_DRV_LOG(NOTICE, "Unexpected function state! "
+		PMD_DRV_LOG(NOTICE, sc, "Unexpected function state! "
 			    "Forcing STARTED-->TX_STOPPED-->STARTED");
 
 		func_params.f_obj = &sc->func_obj;
@@ -1670,7 +1689,7 @@ static int bnx2x_stop_queue(struct bnx2x_softc *sc, int index)
 	struct ecore_queue_state_params q_params = { NULL };
 	int rc;
 
-	PMD_DRV_LOG(DEBUG, "stopping queue %d cid %d", index, fp->index);
+	PMD_DRV_LOG(DEBUG, sc, "stopping queue %d cid %d", index, fp->index);
 
 	q_params.q_obj = &sc->sp_objs[fp->index].q_obj;
 	/* We want to wait for completion in this context */
@@ -1721,7 +1740,7 @@ static uint8_t bnx2x_wait_sp_comp(struct bnx2x_softc *sc, unsigned long mask)
 
 	tmp = atomic_load_acq_long(&sc->sp_state);
 	if (tmp & mask) {
-		PMD_DRV_LOG(INFO, "Filtering completion timed out: "
+		PMD_DRV_LOG(INFO, sc, "Filtering completion timed out: "
 			    "sp_state 0x%lx, mask 0x%lx", tmp, mask);
 		return FALSE;
 	}
@@ -1747,7 +1766,7 @@ static int bnx2x_func_stop(struct bnx2x_softc *sc)
 	 */
 	rc = ecore_func_state_change(sc, &func_params);
 	if (rc) {
-		PMD_DRV_LOG(NOTICE, "FUNC_STOP ramrod failed. "
+		PMD_DRV_LOG(NOTICE, sc, "FUNC_STOP ramrod failed. "
 			    "Running a dry transaction");
 		bnx2x_set_bit(RAMROD_DRV_CLR_ONLY, &func_params.ramrod_flags);
 		return ecore_func_state_change(sc, &func_params);
@@ -1796,14 +1815,16 @@ bnx2x_chip_cleanup(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_li
 	rc = bnx2x_del_all_macs(sc, &sc->sp_objs[0].mac_obj, ECORE_ETH_MAC,
 			      FALSE);
 	if (rc < 0) {
-		PMD_DRV_LOG(NOTICE, "Failed to delete all ETH MACs (%d)", rc);
+		PMD_DRV_LOG(NOTICE, sc,
+			    "Failed to delete all ETH MACs (%d)", rc);
 	}
 
 	/* Clean up UC list  */
 	rc = bnx2x_del_all_macs(sc, &sc->sp_objs[0].mac_obj, ECORE_UC_LIST_MAC,
 			      TRUE);
 	if (rc < 0) {
-		PMD_DRV_LOG(NOTICE, "Failed to delete UC MACs list (%d)", rc);
+		PMD_DRV_LOG(NOTICE, sc,
+			    "Failed to delete UC MACs list (%d)", rc);
 	}
 
 	/* Disable LLH */
@@ -1826,7 +1847,7 @@ bnx2x_chip_cleanup(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_li
 	rparam.mcast_obj = &sc->mcast_obj;
 	rc = ecore_config_mcast(sc, &rparam, ECORE_MCAST_CMD_DEL);
 	if (rc < 0) {
-		PMD_DRV_LOG(NOTICE,
+		PMD_DRV_LOG(NOTICE, sc,
 			    "Failed to send DEL MCAST command (%d)", rc);
 	}
 
@@ -1843,7 +1864,7 @@ bnx2x_chip_cleanup(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_li
 	 */
 	rc = bnx2x_func_wait_started(sc);
 	if (rc) {
-		PMD_DRV_LOG(NOTICE, "bnx2x_func_wait_started failed");
+		PMD_DRV_LOG(NOTICE, sc, "bnx2x_func_wait_started failed");
 	}
 
 	/*
@@ -1861,14 +1882,14 @@ bnx2x_chip_cleanup(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_li
 	 * very wrong has happen.
 	 */
 	if (!bnx2x_wait_sp_comp(sc, ~0x0UL)) {
-		PMD_DRV_LOG(NOTICE, "Common slow path ramrods got stuck!");
+		PMD_DRV_LOG(NOTICE, sc, "Common slow path ramrods got stuck!");
 	}
 
 unload_error:
 
 	rc = bnx2x_func_stop(sc);
 	if (rc) {
-		PMD_DRV_LOG(NOTICE, "Function stop failed!");
+		PMD_DRV_LOG(NOTICE, sc, "Function stop failed!");
 	}
 
 	/* disable HW interrupts */
@@ -1877,7 +1898,7 @@ unload_error:
 	/* Reset the chip */
 	rc = bnx2x_reset_hw(sc, reset_code);
 	if (rc) {
-		PMD_DRV_LOG(NOTICE, "Hardware reset failed");
+		PMD_DRV_LOG(NOTICE, sc, "Hardware reset failed");
 	}
 
 	/* Report UNLOAD_DONE to MCP */
@@ -1888,7 +1909,7 @@ static void bnx2x_disable_close_the_gate(struct bnx2x_softc *sc)
 {
 	uint32_t val;
 
-	PMD_DRV_LOG(DEBUG, "Disabling 'close the gates'");
+	PMD_DRV_LOG(DEBUG, sc, "Disabling 'close the gates'");
 
 	val = REG_RD(sc, MISC_REG_AEU_GENERAL_MASK);
 	val &= ~(MISC_AEU_GENERAL_MASK_REG_AEU_PXP_CLOSE_MASK |
@@ -1919,7 +1940,7 @@ static void bnx2x_squeeze_objects(struct bnx2x_softc *sc)
 	rc = mac_obj->delete_all(sc, &sc->sp_objs->mac_obj, &vlan_mac_flags,
 				 &ramrod_flags);
 	if (rc != 0) {
-		PMD_DRV_LOG(NOTICE, "Failed to clean ETH MACs (%d)", rc);
+		PMD_DRV_LOG(NOTICE, sc, "Failed to clean ETH MACs (%d)", rc);
 	}
 
 	/* Cleanup UC list */
@@ -1927,7 +1948,8 @@ static void bnx2x_squeeze_objects(struct bnx2x_softc *sc)
 	bnx2x_set_bit(ECORE_UC_LIST_MAC, &vlan_mac_flags);
 	rc = mac_obj->delete_all(sc, mac_obj, &vlan_mac_flags, &ramrod_flags);
 	if (rc != 0) {
-		PMD_DRV_LOG(NOTICE, "Failed to clean UC list MACs (%d)", rc);
+		PMD_DRV_LOG(NOTICE, sc,
+			    "Failed to clean UC list MACs (%d)", rc);
 	}
 
 	/* Now clean mcast object... */
@@ -1938,7 +1960,7 @@ static void bnx2x_squeeze_objects(struct bnx2x_softc *sc)
 	/* Add a DEL command... */
 	rc = ecore_config_mcast(sc, &rparam, ECORE_MCAST_CMD_DEL);
 	if (rc < 0) {
-		PMD_DRV_LOG(NOTICE,
+		PMD_DRV_LOG(NOTICE, sc,
 			    "Failed to send DEL MCAST command (%d)", rc);
 	}
 
@@ -1947,7 +1969,7 @@ static void bnx2x_squeeze_objects(struct bnx2x_softc *sc)
 	rc = ecore_config_mcast(sc, &rparam, ECORE_MCAST_CMD_CONT);
 	while (rc != 0) {
 		if (rc < 0) {
-			PMD_DRV_LOG(NOTICE,
+			PMD_DRV_LOG(NOTICE, sc,
 				    "Failed to clean MCAST object (%d)", rc);
 			return;
 		}
@@ -1964,7 +1986,7 @@ bnx2x_nic_unload(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_link
 	uint8_t global = FALSE;
 	uint32_t val;
 
-	PMD_DRV_LOG(DEBUG, "Starting NIC unload...");
+	PMD_DRV_LOG(DEBUG, sc, "Starting NIC unload...");
 
 	/* mark driver as unloaded in shmem2 */
 	if (IS_PF(sc) && SHMEM2_HAS(sc, drv_capabilities_flag)) {
@@ -1988,7 +2010,7 @@ bnx2x_nic_unload(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_link
 		bnx2x_release_leader_lock(sc);
 		mb();
 
-		PMD_DRV_LOG(NOTICE, "Can't unload in closed or error state");
+		PMD_DRV_LOG(NOTICE, sc, "Can't unload in closed or error state");
 		return -1;
 	}
 
@@ -2093,7 +2115,7 @@ bnx2x_nic_unload(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_link
 		bnx2x_disable_close_the_gate(sc);
 	}
 
-	PMD_DRV_LOG(DEBUG, "Ended NIC unload");
+	PMD_DRV_LOG(DEBUG, sc, "Ended NIC unload");
 
 	return 0;
 }
@@ -2241,7 +2263,7 @@ static void bnx2x_ilt_set_info(struct bnx2x_softc *sc)
 	struct ecore_ilt *ilt = sc->ilt;
 	uint16_t line = 0;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	ilt->start_line = FUNC_ILT_BASE(SC_FUNC(sc));
 
@@ -2395,7 +2417,7 @@ static int bnx2x_alloc_mem(struct bnx2x_softc *sc)
 	bnx2x_alloc_ilt_lines_mem(sc);
 
 	if (ecore_ilt_mem_op(sc, ILT_MEMOP_ALLOC)) {
-		PMD_DRV_LOG(NOTICE, "ecore_ilt_mem_op ILT_MEMOP_ALLOC failed");
+		PMD_DRV_LOG(NOTICE, sc, "ecore_ilt_mem_op ILT_MEMOP_ALLOC failed");
 		bnx2x_free_mem(sc);
 		return -1;
 	}
@@ -2598,7 +2620,7 @@ static void bnx2x_set_pf_load(struct bnx2x_softc *sc)
 
 	bnx2x_acquire_hw_lock(sc, HW_LOCK_RESOURCE_RECOVERY_REG);
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	val = REG_RD(sc, BNX2X_RECOVERY_GLOB_REG);
 
@@ -2651,14 +2673,14 @@ static uint8_t bnx2x_clear_pf_load(struct bnx2x_softc *sc)
 /* send load requrest to mcp and analyze response */
 static int bnx2x_nic_load_request(struct bnx2x_softc *sc, uint32_t * load_code)
 {
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	/* init fw_seq */
 	sc->fw_seq =
 	    (SHMEM_RD(sc, func_mb[SC_FW_MB_IDX(sc)].drv_mb_header) &
 	     DRV_MSG_SEQ_NUMBER_MASK);
 
-	PMD_DRV_LOG(DEBUG, "initial fw_seq 0x%04x", sc->fw_seq);
+	PMD_DRV_LOG(DEBUG, sc, "initial fw_seq 0x%04x", sc->fw_seq);
 
 #ifdef BNX2X_PULSE
 	/* get the current FW pulse sequence */
@@ -2677,13 +2699,13 @@ static int bnx2x_nic_load_request(struct bnx2x_softc *sc, uint32_t * load_code)
 
 	/* if the MCP fails to respond we must abort */
 	if (!(*load_code)) {
-		PMD_DRV_LOG(NOTICE, "MCP response failure!");
+		PMD_DRV_LOG(NOTICE, sc, "MCP response failure!");
 		return -1;
 	}
 
 	/* if MCP refused then must abort */
 	if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
-		PMD_DRV_LOG(NOTICE, "MCP refused load request");
+		PMD_DRV_LOG(NOTICE, sc, "MCP refused load request");
 		return -1;
 	}
 
@@ -2710,12 +2732,12 @@ static int bnx2x_nic_load_analyze_req(struct bnx2x_softc *sc, uint32_t load_code
 
 		/* read loaded FW from chip */
 		loaded_fw = REG_RD(sc, XSEM_REG_PRAM);
-		PMD_DRV_LOG(DEBUG, "loaded FW 0x%08x / my FW 0x%08x",
+		PMD_DRV_LOG(DEBUG, sc, "loaded FW 0x%08x / my FW 0x%08x",
 			    loaded_fw, my_fw);
 
 		/* abort nic load if version mismatch */
 		if (my_fw != loaded_fw) {
-			PMD_DRV_LOG(NOTICE,
+			PMD_DRV_LOG(NOTICE, sc,
 				    "FW 0x%08x already loaded (mine is 0x%08x)",
 				    loaded_fw, my_fw);
 			return -1;
@@ -2730,7 +2752,7 @@ static void bnx2x_nic_load_pmf(struct bnx2x_softc *sc, uint32_t load_code)
 {
 	uint32_t ncsi_oem_data_addr;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
 	    (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
@@ -2745,7 +2767,7 @@ static void bnx2x_nic_load_pmf(struct bnx2x_softc *sc, uint32_t load_code)
 		sc->port.pmf = 0;
 	}
 
-	PMD_DRV_LOG(DEBUG, "pmf %d", sc->port.pmf);
+	PMD_DRV_LOG(DEBUG, sc, "pmf %d", sc->port.pmf);
 
 	if (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) {
 		if (SHMEM2_HAS(sc, ncsi_oem_data_addr)) {
@@ -2788,10 +2810,10 @@ static void bnx2x_read_mf_cfg(struct bnx2x_softc *sc)
 
 	if (sc->devinfo.mf_info.mf_config[SC_VN(sc)] &
 	    FUNC_MF_CFG_FUNC_DISABLED) {
-		PMD_DRV_LOG(DEBUG, "mf_cfg function disabled");
+		PMD_DRV_LOG(DEBUG, sc, "mf_cfg function disabled");
 		sc->flags |= BNX2X_MF_FUNC_DIS;
 	} else {
-		PMD_DRV_LOG(DEBUG, "mf_cfg function enabled");
+		PMD_DRV_LOG(DEBUG, sc, "mf_cfg function enabled");
 		sc->flags &= ~BNX2X_MF_FUNC_DIS;
 	}
 }
@@ -2812,7 +2834,7 @@ static int bnx2x_acquire_alr(struct bnx2x_softc *sc)
 	}
 
 	if (!(val & (1L << 31))) {
-		PMD_DRV_LOG(NOTICE, "Cannot acquire MCP access lock register");
+		PMD_DRV_LOG(NOTICE, sc, "Cannot acquire MCP access lock register");
 		return -1;
 	}
 
@@ -2840,7 +2862,7 @@ static void bnx2x_fan_failure(struct bnx2x_softc *sc)
 		 ext_phy_config);
 
 	/* log the failure */
-	PMD_DRV_LOG(INFO,
+	PMD_DRV_LOG(INFO, sc,
 		    "Fan Failure has caused the driver to shutdown "
 		    "the card to prevent permanent damage. "
 		    "Please contact OEM Support for assistance");
@@ -2897,7 +2919,7 @@ static void bnx2x_link_attn(struct bnx2x_softc *sc)
 		}
 	}
 
-	bnx2x_link_report(sc);
+	bnx2x_link_report_locked(sc);
 
 	if (IS_MF(sc)) {
 		bnx2x_link_sync_notify(sc);
@@ -2918,7 +2940,7 @@ static void bnx2x_attn_int_asserted(struct bnx2x_softc *sc, uint32_t asserted)
 	uint32_t cnt;
 
 	if (sc->attn_state & asserted) {
-		PMD_DRV_LOG(ERR, "IGU ERROR attn=0x%08x", asserted);
+		PMD_DRV_LOG(ERR, sc, "IGU ERROR attn=0x%08x", asserted);
 	}
 
 	bnx2x_acquire_hw_lock(sc, HW_LOCK_RESOURCE_PORT0_ATT_MASK + port);
@@ -2936,6 +2958,7 @@ static void bnx2x_attn_int_asserted(struct bnx2x_softc *sc, uint32_t asserted)
 	if (asserted & ATTN_HARD_WIRED_MASK) {
 		if (asserted & ATTN_NIG_FOR_FUNC) {
 
+			bnx2x_acquire_phy_lock(sc);
 			/* save nig interrupt mask */
 			nig_mask = REG_RD(sc, nig_int_mask_addr);
 
@@ -2950,45 +2973,45 @@ static void bnx2x_attn_int_asserted(struct bnx2x_softc *sc, uint32_t asserted)
 		}
 
 		if (asserted & ATTN_SW_TIMER_4_FUNC) {
-			PMD_DRV_LOG(DEBUG, "ATTN_SW_TIMER_4_FUNC!");
+			PMD_DRV_LOG(DEBUG, sc, "ATTN_SW_TIMER_4_FUNC!");
 		}
 
 		if (asserted & GPIO_2_FUNC) {
-			PMD_DRV_LOG(DEBUG, "GPIO_2_FUNC!");
+			PMD_DRV_LOG(DEBUG, sc, "GPIO_2_FUNC!");
 		}
 
 		if (asserted & GPIO_3_FUNC) {
-			PMD_DRV_LOG(DEBUG, "GPIO_3_FUNC!");
+			PMD_DRV_LOG(DEBUG, sc, "GPIO_3_FUNC!");
 		}
 
 		if (asserted & GPIO_4_FUNC) {
-			PMD_DRV_LOG(DEBUG, "GPIO_4_FUNC!");
+			PMD_DRV_LOG(DEBUG, sc, "GPIO_4_FUNC!");
 		}
 
 		if (port == 0) {
 			if (asserted & ATTN_GENERAL_ATTN_1) {
-				PMD_DRV_LOG(DEBUG, "ATTN_GENERAL_ATTN_1!");
+				PMD_DRV_LOG(DEBUG, sc, "ATTN_GENERAL_ATTN_1!");
 				REG_WR(sc, MISC_REG_AEU_GENERAL_ATTN_1, 0x0);
 			}
 			if (asserted & ATTN_GENERAL_ATTN_2) {
-				PMD_DRV_LOG(DEBUG, "ATTN_GENERAL_ATTN_2!");
+				PMD_DRV_LOG(DEBUG, sc, "ATTN_GENERAL_ATTN_2!");
 				REG_WR(sc, MISC_REG_AEU_GENERAL_ATTN_2, 0x0);
 			}
 			if (asserted & ATTN_GENERAL_ATTN_3) {
-				PMD_DRV_LOG(DEBUG, "ATTN_GENERAL_ATTN_3!");
+				PMD_DRV_LOG(DEBUG, sc, "ATTN_GENERAL_ATTN_3!");
 				REG_WR(sc, MISC_REG_AEU_GENERAL_ATTN_3, 0x0);
 			}
 		} else {
 			if (asserted & ATTN_GENERAL_ATTN_4) {
-				PMD_DRV_LOG(DEBUG, "ATTN_GENERAL_ATTN_4!");
+				PMD_DRV_LOG(DEBUG, sc, "ATTN_GENERAL_ATTN_4!");
 				REG_WR(sc, MISC_REG_AEU_GENERAL_ATTN_4, 0x0);
 			}
 			if (asserted & ATTN_GENERAL_ATTN_5) {
-				PMD_DRV_LOG(DEBUG, "ATTN_GENERAL_ATTN_5!");
+				PMD_DRV_LOG(DEBUG, sc, "ATTN_GENERAL_ATTN_5!");
 				REG_WR(sc, MISC_REG_AEU_GENERAL_ATTN_5, 0x0);
 			}
 			if (asserted & ATTN_GENERAL_ATTN_6) {
-				PMD_DRV_LOG(DEBUG, "ATTN_GENERAL_ATTN_6!");
+				PMD_DRV_LOG(DEBUG, sc, "ATTN_GENERAL_ATTN_6!");
 				REG_WR(sc, MISC_REG_AEU_GENERAL_ATTN_6, 0x0);
 			}
 		}
@@ -3002,7 +3025,7 @@ static void bnx2x_attn_int_asserted(struct bnx2x_softc *sc, uint32_t asserted)
 		reg_addr = (BAR_IGU_INTMEM + IGU_CMD_ATTN_BIT_SET_UPPER * 8);
 	}
 
-	PMD_DRV_LOG(DEBUG, "about to mask 0x%08x at %s addr 0x%08x",
+	PMD_DRV_LOG(DEBUG, sc, "about to mask 0x%08x at %s addr 0x%08x",
 		    asserted,
 		    (sc->devinfo.int_block == INT_BLOCK_HC) ? "HC" : "IGU",
 		    reg_addr);
@@ -3024,7 +3047,7 @@ static void bnx2x_attn_int_asserted(struct bnx2x_softc *sc, uint32_t asserted)
 				 && (++cnt < MAX_IGU_ATTN_ACK_TO));
 
 			if (!igu_acked) {
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Failed to verify IGU ack on time");
 			}
 
@@ -3033,6 +3056,7 @@ static void bnx2x_attn_int_asserted(struct bnx2x_softc *sc, uint32_t asserted)
 
 		REG_WR(sc, nig_int_mask_addr, nig_mask);
 
+		bnx2x_release_phy_lock(sc);
 	}
 }
 
@@ -3040,7 +3064,7 @@ static void
 bnx2x_print_next_block(__rte_unused struct bnx2x_softc *sc, __rte_unused int idx,
 		     __rte_unused const char *blk)
 {
-	PMD_DRV_LOG(INFO, "%s%s", idx ? ", " : "", blk);
+	PMD_DRV_LOG(INFO, sc, "%s%s", idx ? ", " : "", blk);
 }
 
 static int
@@ -3348,7 +3372,7 @@ bnx2x_parity_attn(struct bnx2x_softc *sc, uint8_t * global, uint8_t print,
 	    (sig[2] & HW_PRTY_ASSERT_SET_2) ||
 	    (sig[3] & HW_PRTY_ASSERT_SET_3) ||
 	    (sig[4] & HW_PRTY_ASSERT_SET_4)) {
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "Parity error: HW block parity attention:"
 			    "[0]:0x%08x [1]:0x%08x [2]:0x%08x [3]:0x%08x [4]:0x%08x",
 			    (uint32_t) (sig[0] & HW_PRTY_ASSERT_SET_0),
@@ -3358,7 +3382,7 @@ bnx2x_parity_attn(struct bnx2x_softc *sc, uint8_t * global, uint8_t print,
 			    (uint32_t) (sig[4] & HW_PRTY_ASSERT_SET_4));
 
 		if (print)
-			PMD_DRV_LOG(INFO, "Parity errors detected in blocks: ");
+			PMD_DRV_LOG(INFO, sc, "Parity errors detected in blocks: ");
 
 		par_num =
 		    bnx2x_check_blocks_with_parity0(sc, sig[0] &
@@ -3382,7 +3406,7 @@ bnx2x_parity_attn(struct bnx2x_softc *sc, uint8_t * global, uint8_t print,
 						  par_num, print);
 
 		if (print)
-			PMD_DRV_LOG(INFO, "");
+			PMD_DRV_LOG(INFO, sc, "");
 
 		return TRUE;
 	}
@@ -3414,64 +3438,64 @@ static void bnx2x_attn_int_deasserted4(struct bnx2x_softc *sc, uint32_t attn)
 
 	if (attn & AEU_INPUTS_ATTN_BITS_PGLUE_HW_INTERRUPT) {
 		val = REG_RD(sc, PGLUE_B_REG_PGLUE_B_INT_STS_CLR);
-		PMD_DRV_LOG(INFO, "ERROR: PGLUE hw attention 0x%08x", val);
+		PMD_DRV_LOG(INFO, sc, "ERROR: PGLUE hw attention 0x%08x", val);
 		if (val & PGLUE_B_PGLUE_B_INT_STS_REG_ADDRESS_ERROR)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: PGLUE_B_PGLUE_B_INT_STS_REG_ADDRESS_ERROR");
 		if (val & PGLUE_B_PGLUE_B_INT_STS_REG_INCORRECT_RCV_BEHAVIOR)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: PGLUE_B_PGLUE_B_INT_STS_REG_INCORRECT_RCV_BEHAVIOR");
 		if (val & PGLUE_B_PGLUE_B_INT_STS_REG_WAS_ERROR_ATTN)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: PGLUE_B_PGLUE_B_INT_STS_REG_WAS_ERROR_ATTN");
 		if (val & PGLUE_B_PGLUE_B_INT_STS_REG_VF_LENGTH_VIOLATION_ATTN)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: PGLUE_B_PGLUE_B_INT_STS_REG_VF_LENGTH_VIOLATION_ATTN");
 		if (val &
 		    PGLUE_B_PGLUE_B_INT_STS_REG_VF_GRC_SPACE_VIOLATION_ATTN)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: PGLUE_B_PGLUE_B_INT_STS_REG_VF_GRC_SPACE_VIOLATION_ATTN");
 		if (val &
 		    PGLUE_B_PGLUE_B_INT_STS_REG_VF_MSIX_BAR_VIOLATION_ATTN)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: PGLUE_B_PGLUE_B_INT_STS_REG_VF_MSIX_BAR_VIOLATION_ATTN");
 		if (val & PGLUE_B_PGLUE_B_INT_STS_REG_TCPL_ERROR_ATTN)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: PGLUE_B_PGLUE_B_INT_STS_REG_TCPL_ERROR_ATTN");
 		if (val & PGLUE_B_PGLUE_B_INT_STS_REG_TCPL_IN_TWO_RCBS_ATTN)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: PGLUE_B_PGLUE_B_INT_STS_REG_TCPL_IN_TWO_RCBS_ATTN");
 		if (val & PGLUE_B_PGLUE_B_INT_STS_REG_CSSNOOP_FIFO_OVERFLOW)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: PGLUE_B_PGLUE_B_INT_STS_REG_CSSNOOP_FIFO_OVERFLOW");
 	}
 
 	if (attn & AEU_INPUTS_ATTN_BITS_ATC_HW_INTERRUPT) {
 		val = REG_RD(sc, ATC_REG_ATC_INT_STS_CLR);
-		PMD_DRV_LOG(INFO, "ERROR: ATC hw attention 0x%08x", val);
+		PMD_DRV_LOG(INFO, sc, "ERROR: ATC hw attention 0x%08x", val);
 		if (val & ATC_ATC_INT_STS_REG_ADDRESS_ERROR)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: ATC_ATC_INT_STS_REG_ADDRESS_ERROR");
 		if (val & ATC_ATC_INT_STS_REG_ATC_TCPL_TO_NOT_PEND)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: ATC_ATC_INT_STS_REG_ATC_TCPL_TO_NOT_PEND");
 		if (val & ATC_ATC_INT_STS_REG_ATC_GPA_MULTIPLE_HITS)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: ATC_ATC_INT_STS_REG_ATC_GPA_MULTIPLE_HITS");
 		if (val & ATC_ATC_INT_STS_REG_ATC_RCPL_TO_EMPTY_CNT)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: ATC_ATC_INT_STS_REG_ATC_RCPL_TO_EMPTY_CNT");
 		if (val & ATC_ATC_INT_STS_REG_ATC_TCPL_ERROR)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: ATC_ATC_INT_STS_REG_ATC_TCPL_ERROR");
 		if (val & ATC_ATC_INT_STS_REG_ATC_IREQ_LESS_THAN_STU)
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "ERROR: ATC_ATC_INT_STS_REG_ATC_IREQ_LESS_THAN_STU");
 	}
 
 	if (attn & (AEU_INPUTS_ATTN_BITS_PGLUE_PARITY_ERROR |
 		    AEU_INPUTS_ATTN_BITS_ATC_PARITY_ERROR)) {
-		PMD_DRV_LOG(INFO,
+		PMD_DRV_LOG(INFO, sc,
 			    "ERROR: FATAL parity attention set4 0x%08x",
 			    (uint32_t) (attn &
 					(AEU_INPUTS_ATTN_BITS_PGLUE_PARITY_ERROR
@@ -3594,11 +3618,11 @@ static void bnx2x_dcc_event(struct bnx2x_softc *sc, uint32_t dcc_event)
  */
 		if (sc->devinfo.
 		    mf_info.mf_config[SC_VN(sc)] & FUNC_MF_CFG_FUNC_DISABLED) {
-			PMD_DRV_LOG(DEBUG, "mf_cfg function disabled");
+			PMD_DRV_LOG(DEBUG, sc, "mf_cfg function disabled");
 			sc->flags |= BNX2X_MF_FUNC_DIS;
 			bnx2x_e1h_disable(sc);
 		} else {
-			PMD_DRV_LOG(DEBUG, "mf_cfg function enabled");
+			PMD_DRV_LOG(DEBUG, sc, "mf_cfg function enabled");
 			sc->flags &= ~BNX2X_MF_FUNC_DIS;
 			bnx2x_e1h_enable(sc);
 		}
@@ -3653,7 +3677,7 @@ static int bnx2x_mc_assert(struct bnx2x_softc *sc)
 	last_idx =
 	    REG_RD8(sc, BAR_XSTRORM_INTMEM + XSTORM_ASSERT_LIST_INDEX_OFFSET);
 	if (last_idx)
-		PMD_DRV_LOG(ERR, "XSTORM_ASSERT_LIST_INDEX 0x%x", last_idx);
+		PMD_DRV_LOG(ERR, sc, "XSTORM_ASSERT_LIST_INDEX 0x%x", last_idx);
 
 	/* print the asserts */
 	for (i = 0; i < STORM_ASSERT_ARRAY_SIZE; i++) {
@@ -3675,7 +3699,7 @@ static int bnx2x_mc_assert(struct bnx2x_softc *sc)
 			   12);
 
 		if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
-			PMD_DRV_LOG(ERR,
+			PMD_DRV_LOG(ERR, sc,
 				    "XSTORM_ASSERT_INDEX 0x%x = 0x%08x 0x%08x 0x%08x 0x%08x",
 				    i, row3, row2, row1, row0);
 			rc++;
@@ -3688,7 +3712,7 @@ static int bnx2x_mc_assert(struct bnx2x_softc *sc)
 	last_idx =
 	    REG_RD8(sc, BAR_TSTRORM_INTMEM + TSTORM_ASSERT_LIST_INDEX_OFFSET);
 	if (last_idx) {
-		PMD_DRV_LOG(ERR, "TSTORM_ASSERT_LIST_INDEX 0x%x", last_idx);
+		PMD_DRV_LOG(ERR, sc, "TSTORM_ASSERT_LIST_INDEX 0x%x", last_idx);
 	}
 
 	/* print the asserts */
@@ -3711,7 +3735,7 @@ static int bnx2x_mc_assert(struct bnx2x_softc *sc)
 			   12);
 
 		if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
-			PMD_DRV_LOG(ERR,
+			PMD_DRV_LOG(ERR, sc,
 				    "TSTORM_ASSERT_INDEX 0x%x = 0x%08x 0x%08x 0x%08x 0x%08x",
 				    i, row3, row2, row1, row0);
 			rc++;
@@ -3724,7 +3748,7 @@ static int bnx2x_mc_assert(struct bnx2x_softc *sc)
 	last_idx =
 	    REG_RD8(sc, BAR_CSTRORM_INTMEM + CSTORM_ASSERT_LIST_INDEX_OFFSET);
 	if (last_idx) {
-		PMD_DRV_LOG(ERR, "CSTORM_ASSERT_LIST_INDEX 0x%x", last_idx);
+		PMD_DRV_LOG(ERR, sc, "CSTORM_ASSERT_LIST_INDEX 0x%x", last_idx);
 	}
 
 	/* print the asserts */
@@ -3747,7 +3771,7 @@ static int bnx2x_mc_assert(struct bnx2x_softc *sc)
 			   12);
 
 		if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
-			PMD_DRV_LOG(ERR,
+			PMD_DRV_LOG(ERR, sc,
 				    "CSTORM_ASSERT_INDEX 0x%x = 0x%08x 0x%08x 0x%08x 0x%08x",
 				    i, row3, row2, row1, row0);
 			rc++;
@@ -3760,7 +3784,7 @@ static int bnx2x_mc_assert(struct bnx2x_softc *sc)
 	last_idx =
 	    REG_RD8(sc, BAR_USTRORM_INTMEM + USTORM_ASSERT_LIST_INDEX_OFFSET);
 	if (last_idx) {
-		PMD_DRV_LOG(ERR, "USTORM_ASSERT_LIST_INDEX 0x%x", last_idx);
+		PMD_DRV_LOG(ERR, sc, "USTORM_ASSERT_LIST_INDEX 0x%x", last_idx);
 	}
 
 	/* print the asserts */
@@ -3783,7 +3807,7 @@ static int bnx2x_mc_assert(struct bnx2x_softc *sc)
 			   12);
 
 		if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
-			PMD_DRV_LOG(ERR,
+			PMD_DRV_LOG(ERR, sc,
 				    "USTORM_ASSERT_INDEX 0x%x = 0x%08x 0x%08x 0x%08x 0x%08x",
 				    i, row3, row2, row1, row0);
 			rc++;
@@ -3832,8 +3856,10 @@ static void bnx2x_attn_int_deasserted3(struct bnx2x_softc *sc, uint32_t attn)
 			if (sc->link_vars.periodic_flags &
 			    ELINK_PERIODIC_FLAGS_LINK_EVENT) {
 				/* sync with link */
+				bnx2x_acquire_phy_lock(sc);
 				sc->link_vars.periodic_flags &=
 				    ~ELINK_PERIODIC_FLAGS_LINK_EVENT;
+				bnx2x_release_phy_lock(sc);
 				if (IS_MF(sc)) {
 					bnx2x_link_sync_notify(sc);
 				}
@@ -3848,7 +3874,7 @@ static void bnx2x_attn_int_deasserted3(struct bnx2x_softc *sc, uint32_t attn)
 
 		} else if (attn & BNX2X_MC_ASSERT_BITS) {
 
-			PMD_DRV_LOG(ERR, "MC assert!");
+			PMD_DRV_LOG(ERR, sc, "MC assert!");
 			bnx2x_mc_assert(sc);
 			REG_WR(sc, MISC_REG_AEU_GENERAL_ATTN_10, 0);
 			REG_WR(sc, MISC_REG_AEU_GENERAL_ATTN_9, 0);
@@ -3858,24 +3884,24 @@ static void bnx2x_attn_int_deasserted3(struct bnx2x_softc *sc, uint32_t attn)
 
 		} else if (attn & BNX2X_MCP_ASSERT) {
 
-			PMD_DRV_LOG(ERR, "MCP assert!");
+			PMD_DRV_LOG(ERR, sc, "MCP assert!");
 			REG_WR(sc, MISC_REG_AEU_GENERAL_ATTN_11, 0);
 
 		} else {
-			PMD_DRV_LOG(ERR,
+			PMD_DRV_LOG(ERR, sc,
 				    "Unknown HW assert! (attn 0x%08x)", attn);
 		}
 	}
 
 	if (attn & EVEREST_LATCHED_ATTN_IN_USE_MASK) {
-		PMD_DRV_LOG(ERR, "LATCHED attention 0x%08x (masked)", attn);
+		PMD_DRV_LOG(ERR, sc, "LATCHED attention 0x%08x (masked)", attn);
 		if (attn & BNX2X_GRC_TIMEOUT) {
 			val = REG_RD(sc, MISC_REG_GRC_TIMEOUT_ATTN);
-			PMD_DRV_LOG(ERR, "GRC time-out 0x%08x", val);
+			PMD_DRV_LOG(ERR, sc, "GRC time-out 0x%08x", val);
 		}
 		if (attn & BNX2X_GRC_RSV) {
 			val = REG_RD(sc, MISC_REG_GRC_RSV_ATTN);
-			PMD_DRV_LOG(ERR, "GRC reserved 0x%08x", val);
+			PMD_DRV_LOG(ERR, sc, "GRC reserved 0x%08x", val);
 		}
 		REG_WR(sc, MISC_REG_AEU_CLR_LATCH_SIGNAL, 0x7ff);
 	}
@@ -3890,24 +3916,24 @@ static void bnx2x_attn_int_deasserted2(struct bnx2x_softc *sc, uint32_t attn)
 
 	if (attn & AEU_INPUTS_ATTN_BITS_CFC_HW_INTERRUPT) {
 		val = REG_RD(sc, CFC_REG_CFC_INT_STS_CLR);
-		PMD_DRV_LOG(ERR, "CFC hw attention 0x%08x", val);
+		PMD_DRV_LOG(ERR, sc, "CFC hw attention 0x%08x", val);
 /* CFC error attention */
 		if (val & 0x2) {
-			PMD_DRV_LOG(ERR, "FATAL error from CFC");
+			PMD_DRV_LOG(ERR, sc, "FATAL error from CFC");
 		}
 	}
 
 	if (attn & AEU_INPUTS_ATTN_BITS_PXP_HW_INTERRUPT) {
 		val = REG_RD(sc, PXP_REG_PXP_INT_STS_CLR_0);
-		PMD_DRV_LOG(ERR, "PXP hw attention-0 0x%08x", val);
+		PMD_DRV_LOG(ERR, sc, "PXP hw attention-0 0x%08x", val);
 /* RQ_USDMDP_FIFO_OVERFLOW */
 		if (val & 0x18000) {
-			PMD_DRV_LOG(ERR, "FATAL error from PXP");
+			PMD_DRV_LOG(ERR, sc, "FATAL error from PXP");
 		}
 
 		if (!CHIP_IS_E1x(sc)) {
 			val = REG_RD(sc, PXP_REG_PXP_INT_STS_CLR_1);
-			PMD_DRV_LOG(ERR, "PXP hw attention-1 0x%08x", val);
+			PMD_DRV_LOG(ERR, sc, "PXP hw attention-1 0x%08x", val);
 		}
 	}
 #define PXP2_EOP_ERROR_BIT  PXP2_PXP2_INT_STS_CLR_0_REG_WR_PGLUE_EOP_ERROR
@@ -3935,7 +3961,7 @@ static void bnx2x_attn_int_deasserted2(struct bnx2x_softc *sc, uint32_t attn)
 				val0 = REG_RD(sc, PXP2_REG_PXP2_INT_STS_CLR_0);
 
 			/* print the register, since no one can restore it */
-			PMD_DRV_LOG(ERR,
+			PMD_DRV_LOG(ERR, sc,
 				    "PXP2_REG_PXP2_INT_STS_CLR_0 0x%08x", val0);
 
 			/*
@@ -3943,7 +3969,7 @@ static void bnx2x_attn_int_deasserted2(struct bnx2x_softc *sc, uint32_t attn)
 			 * then notify
 			 */
 			if (val0 & PXP2_EOP_ERROR_BIT) {
-				PMD_DRV_LOG(ERR, "PXP2_WR_PGLUE_EOP_ERROR");
+				PMD_DRV_LOG(ERR, sc, "PXP2_WR_PGLUE_EOP_ERROR");
 
 				/*
 				 * if only PXP2_PXP2_INT_STS_0_REG_WR_PGLUE_EOP_ERROR is
@@ -3964,7 +3990,7 @@ static void bnx2x_attn_int_deasserted2(struct bnx2x_softc *sc, uint32_t attn)
 		val &= ~(attn & HW_INTERRUT_ASSERT_SET_2);
 		REG_WR(sc, reg_offset, val);
 
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "FATAL HW block attention set2 0x%x",
 			    (uint32_t) (attn & HW_INTERRUT_ASSERT_SET_2));
 		rte_panic("HW block attention set2");
@@ -3979,10 +4005,10 @@ static void bnx2x_attn_int_deasserted1(struct bnx2x_softc *sc, uint32_t attn)
 
 	if (attn & AEU_INPUTS_ATTN_BITS_DOORBELLQ_HW_INTERRUPT) {
 		val = REG_RD(sc, DORQ_REG_DORQ_INT_STS_CLR);
-		PMD_DRV_LOG(ERR, "DB hw attention 0x%08x", val);
+		PMD_DRV_LOG(ERR, sc, "DB hw attention 0x%08x", val);
 /* DORQ discard attention */
 		if (val & 0x2) {
-			PMD_DRV_LOG(ERR, "FATAL error from DORQ");
+			PMD_DRV_LOG(ERR, sc, "FATAL error from DORQ");
 		}
 	}
 
@@ -3994,7 +4020,7 @@ static void bnx2x_attn_int_deasserted1(struct bnx2x_softc *sc, uint32_t attn)
 		val &= ~(attn & HW_INTERRUT_ASSERT_SET_1);
 		REG_WR(sc, reg_offset, val);
 
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "FATAL HW block attention set1 0x%08x",
 			    (uint32_t) (attn & HW_INTERRUT_ASSERT_SET_1));
 		rte_panic("HW block attention set1");
@@ -4015,7 +4041,7 @@ static void bnx2x_attn_int_deasserted0(struct bnx2x_softc *sc, uint32_t attn)
 		val &= ~AEU_INPUTS_ATTN_BITS_SPIO5;
 		REG_WR(sc, reg_offset, val);
 
-		PMD_DRV_LOG(WARNING, "SPIO5 hw attention");
+		PMD_DRV_LOG(WARNING, sc, "SPIO5 hw attention");
 
 /* Fan failure attention */
 		elink_hw_reset_phy(&sc->link_params);
@@ -4023,7 +4049,9 @@ static void bnx2x_attn_int_deasserted0(struct bnx2x_softc *sc, uint32_t attn)
 	}
 
 	if ((attn & sc->link_vars.aeu_int_mask) && sc->port.pmf) {
+		bnx2x_acquire_phy_lock(sc);
 		elink_handle_module_detect_int(&sc->link_params);
+		bnx2x_release_phy_lock(sc);
 	}
 
 	if (attn & HW_INTERRUT_ASSERT_SET_0) {
@@ -4105,14 +4133,14 @@ static void bnx2x_attn_int_deasserted(struct bnx2x_softc *sc, uint32_t deasserte
 	}
 
 	val = ~deasserted;
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 		    "about to mask 0x%08x at %s addr 0x%08x", val,
 		    (sc->devinfo.int_block == INT_BLOCK_HC) ? "HC" : "IGU",
 		    reg_addr);
 	REG_WR(sc, reg_addr, val);
 
 	if (~sc->attn_state & deasserted) {
-		PMD_DRV_LOG(ERR, "IGU error");
+		PMD_DRV_LOG(ERR, sc, "IGU error");
 	}
 
 	reg_addr = port ? MISC_REG_AEU_MASK_ATTN_FUNC_1 :
@@ -4142,12 +4170,12 @@ static void bnx2x_attn_int(struct bnx2x_softc *sc)
 	uint32_t asserted = attn_bits & ~attn_ack & ~attn_state;
 	uint32_t deasserted = ~attn_bits & attn_ack & attn_state;
 
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 		    "attn_bits 0x%08x attn_ack 0x%08x asserted 0x%08x deasserted 0x%08x",
 		    attn_bits, attn_ack, asserted, deasserted);
 
 	if (~(attn_bits ^ attn_ack) & (attn_bits ^ attn_state)) {
-		PMD_DRV_LOG(ERR, "BAD attention state");
+		PMD_DRV_LOG(ERR, sc, "BAD attention state");
 	}
 
 	/* handle bits that were raised */
@@ -4204,7 +4232,7 @@ static void bnx2x_handle_mcast_eqe(struct bnx2x_softc *sc)
 	if (sc->mcast_obj.check_pending(&sc->mcast_obj)) {
 		rc = ecore_config_mcast(sc, &rparam, ECORE_MCAST_CMD_CONT);
 		if (rc < 0) {
-			PMD_DRV_LOG(INFO,
+			PMD_DRV_LOG(INFO, sc,
 				    "Failed to send pending mcast commands (%d)",
 				    rc);
 		}
@@ -4224,17 +4252,17 @@ bnx2x_handle_classification_eqe(struct bnx2x_softc *sc, union event_ring_elem *e
 
 	switch (le32toh(elem->message.data.eth_event.echo) >> BNX2X_SWCID_SHIFT) {
 	case ECORE_FILTER_MAC_PENDING:
-		PMD_DRV_LOG(DEBUG, "Got SETUP_MAC completions");
+		PMD_DRV_LOG(DEBUG, sc, "Got SETUP_MAC completions");
 		vlan_mac_obj = &sc->sp_objs[cid].mac_obj;
 		break;
 
 	case ECORE_FILTER_MCAST_PENDING:
-		PMD_DRV_LOG(DEBUG, "Got SETUP_MCAST completions");
+		PMD_DRV_LOG(DEBUG, sc, "Got SETUP_MCAST completions");
 		bnx2x_handle_mcast_eqe(sc);
 		return;
 
 	default:
-		PMD_DRV_LOG(NOTICE, "Unsupported classification command: %d",
+		PMD_DRV_LOG(NOTICE, sc, "Unsupported classification command: %d",
 			    elem->message.data.eth_event.echo);
 		return;
 	}
@@ -4242,9 +4270,10 @@ bnx2x_handle_classification_eqe(struct bnx2x_softc *sc, union event_ring_elem *e
 	rc = vlan_mac_obj->complete(sc, vlan_mac_obj, elem, &ramrod_flags);
 
 	if (rc < 0) {
-		PMD_DRV_LOG(NOTICE, "Failed to schedule new commands (%d)", rc);
+		PMD_DRV_LOG(NOTICE, sc,
+			    "Failed to schedule new commands (%d)", rc);
 	} else if (rc > 0) {
-		PMD_DRV_LOG(DEBUG, "Scheduled next pending commands...");
+		PMD_DRV_LOG(DEBUG, sc, "Scheduled next pending commands...");
 	}
 }
 
@@ -4308,7 +4337,7 @@ static void bnx2x_eq_int(struct bnx2x_softc *sc)
 /* handle eq element */
 		switch (opcode) {
 		case EVENT_RING_OPCODE_STAT_QUERY:
-			PMD_DEBUG_PERIODIC_LOG(DEBUG, "got statistics completion event %d",
+			PMD_DEBUG_PERIODIC_LOG(DEBUG, sc, "got statistics completion event %d",
 				    sc->stats_comp++);
 			/* nothing to do with stats comp */
 			goto next_spqe;
@@ -4316,7 +4345,7 @@ static void bnx2x_eq_int(struct bnx2x_softc *sc)
 		case EVENT_RING_OPCODE_CFC_DEL:
 			/* handle according to cid range */
 			/* we may want to verify here that the sc state is HALTING */
-			PMD_DRV_LOG(DEBUG, "got delete ramrod for MULTI[%d]",
+			PMD_DRV_LOG(DEBUG, sc, "got delete ramrod for MULTI[%d]",
 				    cid);
 			q_obj = bnx2x_cid_to_q_obj(sc, cid);
 			if (q_obj->complete_cmd(sc, q_obj, ECORE_Q_CMD_CFC_DEL)) {
@@ -4325,14 +4354,14 @@ static void bnx2x_eq_int(struct bnx2x_softc *sc)
 			goto next_spqe;
 
 		case EVENT_RING_OPCODE_STOP_TRAFFIC:
-			PMD_DRV_LOG(DEBUG, "got STOP TRAFFIC");
+			PMD_DRV_LOG(DEBUG, sc, "got STOP TRAFFIC");
 			if (f_obj->complete_cmd(sc, f_obj, ECORE_F_CMD_TX_STOP)) {
 				break;
 			}
 			goto next_spqe;
 
 		case EVENT_RING_OPCODE_START_TRAFFIC:
-			PMD_DRV_LOG(DEBUG, "got START TRAFFIC");
+			PMD_DRV_LOG(DEBUG, sc, "got START TRAFFIC");
 			if (f_obj->complete_cmd
 			    (sc, f_obj, ECORE_F_CMD_TX_START)) {
 				break;
@@ -4342,7 +4371,7 @@ static void bnx2x_eq_int(struct bnx2x_softc *sc)
 		case EVENT_RING_OPCODE_FUNCTION_UPDATE:
 			echo = elem->message.data.function_update_event.echo;
 			if (echo == SWITCH_UPDATE) {
-				PMD_DRV_LOG(DEBUG,
+				PMD_DRV_LOG(DEBUG, sc,
 					    "got FUNC_SWITCH_UPDATE ramrod");
 				if (f_obj->complete_cmd(sc, f_obj,
 							ECORE_F_CMD_SWITCH_UPDATE))
@@ -4350,7 +4379,7 @@ static void bnx2x_eq_int(struct bnx2x_softc *sc)
 					break;
 				}
 			} else {
-				PMD_DRV_LOG(DEBUG,
+				PMD_DRV_LOG(DEBUG, sc,
 					    "AFEX: ramrod completed FUNCTION_UPDATE");
 				f_obj->complete_cmd(sc, f_obj,
 						    ECORE_F_CMD_AFEX_UPDATE);
@@ -4366,14 +4395,14 @@ static void bnx2x_eq_int(struct bnx2x_softc *sc)
 			goto next_spqe;
 
 		case EVENT_RING_OPCODE_FUNCTION_START:
-			PMD_DRV_LOG(DEBUG, "got FUNC_START ramrod");
+			PMD_DRV_LOG(DEBUG, sc, "got FUNC_START ramrod");
 			if (f_obj->complete_cmd(sc, f_obj, ECORE_F_CMD_START)) {
 				break;
 			}
 			goto next_spqe;
 
 		case EVENT_RING_OPCODE_FUNCTION_STOP:
-			PMD_DRV_LOG(DEBUG, "got FUNC_STOP ramrod");
+			PMD_DRV_LOG(DEBUG, sc, "got FUNC_STOP ramrod");
 			if (f_obj->complete_cmd(sc, f_obj, ECORE_F_CMD_STOP)) {
 				break;
 			}
@@ -4385,7 +4414,7 @@ static void bnx2x_eq_int(struct bnx2x_softc *sc)
 		case (EVENT_RING_OPCODE_RSS_UPDATE_RULES | BNX2X_STATE_OPENING_WAITING_PORT):
 			cid =
 			    elem->message.data.eth_event.echo & BNX2X_SWCID_MASK;
-			PMD_DRV_LOG(DEBUG, "got RSS_UPDATE ramrod. CID %d",
+			PMD_DRV_LOG(DEBUG, sc, "got RSS_UPDATE ramrod. CID %d",
 				    cid);
 			rss_raw->clear_pending(rss_raw);
 			break;
@@ -4396,7 +4425,7 @@ static void bnx2x_eq_int(struct bnx2x_softc *sc)
 		case (EVENT_RING_OPCODE_CLASSIFICATION_RULES | BNX2X_STATE_OPEN):
 		case (EVENT_RING_OPCODE_CLASSIFICATION_RULES | BNX2X_STATE_DIAG):
 		case (EVENT_RING_OPCODE_CLASSIFICATION_RULES | BNX2X_STATE_CLOSING_WAITING_HALT):
-			PMD_DRV_LOG(DEBUG,
+			PMD_DRV_LOG(DEBUG, sc,
 				    "got (un)set mac ramrod");
 			bnx2x_handle_classification_eqe(sc, elem);
 			break;
@@ -4404,7 +4433,7 @@ static void bnx2x_eq_int(struct bnx2x_softc *sc)
 		case (EVENT_RING_OPCODE_MULTICAST_RULES | BNX2X_STATE_OPEN):
 		case (EVENT_RING_OPCODE_MULTICAST_RULES | BNX2X_STATE_DIAG):
 		case (EVENT_RING_OPCODE_MULTICAST_RULES | BNX2X_STATE_CLOSING_WAITING_HALT):
-			PMD_DRV_LOG(DEBUG,
+			PMD_DRV_LOG(DEBUG, sc,
 				    "got mcast ramrod");
 			bnx2x_handle_mcast_eqe(sc);
 			break;
@@ -4412,14 +4441,14 @@ static void bnx2x_eq_int(struct bnx2x_softc *sc)
 		case (EVENT_RING_OPCODE_FILTERS_RULES | BNX2X_STATE_OPEN):
 		case (EVENT_RING_OPCODE_FILTERS_RULES | BNX2X_STATE_DIAG):
 		case (EVENT_RING_OPCODE_FILTERS_RULES | BNX2X_STATE_CLOSING_WAITING_HALT):
-			PMD_DRV_LOG(DEBUG,
+			PMD_DRV_LOG(DEBUG, sc,
 				    "got rx_mode ramrod");
 			bnx2x_handle_rx_mode_eqe(sc);
 			break;
 
 		default:
 			/* unknown event log error and continue */
-			PMD_DRV_LOG(INFO, "Unknown EQ event %d, sc->state 0x%x",
+			PMD_DRV_LOG(INFO, sc, "Unknown EQ event %d, sc->state 0x%x",
 				    elem->message.opcode, sc->state);
 		}
 
@@ -4445,12 +4474,16 @@ static int bnx2x_handle_sp_tq(struct bnx2x_softc *sc)
 	uint16_t status;
 	int rc = 0;
 
+	PMD_DRV_LOG(DEBUG, sc, "---> SP TASK <---");
+
 	/* what work needs to be performed? */
 	status = bnx2x_update_dsb_idx(sc);
 
+	PMD_DRV_LOG(DEBUG, sc, "dsb status 0x%04x", status);
+
 	/* HW attentions */
 	if (status & BNX2X_DEF_SB_ATT_IDX) {
-		PMD_DRV_LOG(DEBUG, "---> ATTN INTR <---");
+		PMD_DRV_LOG(DEBUG, sc, "---> ATTN INTR <---");
 		bnx2x_attn_int(sc);
 		status &= ~BNX2X_DEF_SB_ATT_IDX;
 		rc = 1;
@@ -4459,7 +4492,7 @@ static int bnx2x_handle_sp_tq(struct bnx2x_softc *sc)
 	/* SP events: STAT_QUERY and others */
 	if (status & BNX2X_DEF_SB_IDX) {
 /* handle EQ completions */
-		PMD_DEBUG_PERIODIC_LOG(DEBUG, "---> EQ INTR <---");
+		PMD_DRV_LOG(DEBUG, sc, "---> EQ INTR <---");
 		bnx2x_eq_int(sc);
 		bnx2x_ack_sb(sc, sc->igu_dsb_id, USTORM_ID,
 			   le16toh(sc->def_idx), IGU_INT_NOP, 1);
@@ -4468,7 +4501,7 @@ static int bnx2x_handle_sp_tq(struct bnx2x_softc *sc)
 
 	/* if status is non zero then something went wrong */
 	if (unlikely(status)) {
-		PMD_DRV_LOG(INFO,
+		PMD_DRV_LOG(INFO, sc,
 			    "Got an unknown SP interrupt! (0x%04x)", status);
 	}
 
@@ -4484,7 +4517,8 @@ static void bnx2x_handle_fp_tq(struct bnx2x_fastpath *fp, int scan_fp)
 	struct bnx2x_softc *sc = fp->sc;
 	uint8_t more_rx = FALSE;
 
-	PMD_DRV_LOG(DEBUG, "---> FP TASK QUEUE (%d) <--", fp->index);
+	PMD_DEBUG_PERIODIC_LOG(DEBUG, sc,
+			       "---> FP TASK QUEUE (%d) <--", fp->index);
 
 	/* update the fastpath index */
 	bnx2x_update_fp_sb_idx(fp);
@@ -4534,25 +4568,31 @@ int bnx2x_intr_legacy(struct bnx2x_softc *sc, int scan_fp)
 		return 0;
 	}
 
-	PMD_DEBUG_PERIODIC_LOG(DEBUG, "Interrupt status 0x%04x", status);
+	PMD_DEBUG_PERIODIC_LOG(DEBUG, sc, "Interrupt status 0x%04x", status);
 	//bnx2x_dump_status_block(sc);
 
 	FOR_EACH_ETH_QUEUE(sc, i) {
 		fp = &sc->fp[i];
 		mask = (0x2 << (fp->index + CNIC_SUPPORT(sc)));
 		if (status & mask) {
+		/* acknowledge and disable further fastpath interrupts */
+			bnx2x_ack_sb(sc, fp->igu_sb_id, USTORM_ID,
+				     0, IGU_INT_DISABLE, 0);
 			bnx2x_handle_fp_tq(fp, scan_fp);
 			status &= ~mask;
 		}
 	}
 
 	if (unlikely(status & 0x1)) {
+		/* acknowledge and disable further slowpath interrupts */
+		bnx2x_ack_sb(sc, sc->igu_dsb_id, USTORM_ID,
+			     0, IGU_INT_DISABLE, 0);
 		rc = bnx2x_handle_sp_tq(sc);
 		status &= ~0x1;
 	}
 
 	if (unlikely(status)) {
-		PMD_DRV_LOG(WARNING,
+		PMD_DRV_LOG(WARNING, sc,
 			    "Unexpected fastpath status (0x%08x)!", status);
 	}
 
@@ -4588,7 +4628,7 @@ static void bnx2x_init_func_obj(struct bnx2x_softc *sc)
 {
 	sc->dmae_ready = 0;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	ecore_init_func_obj(sc,
 			    &sc->func_obj,
@@ -4604,7 +4644,7 @@ static int bnx2x_init_hw(struct bnx2x_softc *sc, uint32_t load_code)
 	struct ecore_func_state_params func_params = { NULL };
 	int rc;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	/* prepare the parameters for function state transitions */
 	bnx2x_set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
@@ -5193,7 +5233,7 @@ static void bnx2x_init_internal(struct bnx2x_softc *sc, uint32_t load_code)
 		break;
 
 	default:
-		PMD_DRV_LOG(NOTICE, "Unknown load_code (0x%x) from MCP",
+		PMD_DRV_LOG(NOTICE, sc, "Unknown load_code (0x%x) from MCP",
 			    load_code);
 		break;
 	}
@@ -5284,7 +5324,7 @@ bnx2x_extract_max_cfg(__rte_unused struct bnx2x_softc *sc, uint32_t mf_cfg)
 			    FUNC_MF_CFG_MAX_BW_SHIFT);
 
 	if (!max_cfg) {
-		PMD_DRV_LOG(DEBUG,
+		PMD_DRV_LOG(DEBUG, sc,
 			    "Max BW configured to 0 - using 100 instead");
 		max_cfg = 100;
 	}
@@ -5548,7 +5588,7 @@ static void bnx2x_igu_int_enable(struct bnx2x_softc *sc)
 
 	val |= IGU_PF_CONF_FUNC_EN;
 
-	PMD_DRV_LOG(DEBUG, "write 0x%x to IGU mode %s",
+	PMD_DRV_LOG(DEBUG, sc, "write 0x%x to IGU mode %s",
 		    val, ((msix) ? "MSI-X" : ((msi) ? "MSI" : "INTx")));
 
 	REG_WR(sc, IGU_REG_PF_CONFIGURATION, val);
@@ -5596,7 +5636,7 @@ static void bnx2x_hc_int_disable(struct bnx2x_softc *sc)
 
 	REG_WR(sc, addr, val);
 	if (REG_RD(sc, addr) != val) {
-		PMD_DRV_LOG(ERR, "proper val not read from HC IGU!");
+		PMD_DRV_LOG(ERR, sc, "proper val not read from HC IGU!");
 	}
 }
 
@@ -5607,14 +5647,14 @@ static void bnx2x_igu_int_disable(struct bnx2x_softc *sc)
 	val &= ~(IGU_PF_CONF_MSI_MSIX_EN |
 		 IGU_PF_CONF_INT_LINE_EN | IGU_PF_CONF_ATTN_BIT_EN);
 
-	PMD_DRV_LOG(DEBUG, "write %x to IGU", val);
+	PMD_DRV_LOG(DEBUG, sc, "write %x to IGU", val);
 
 	/* flush all outstanding writes */
 	mb();
 
 	REG_WR(sc, IGU_REG_PF_CONFIGURATION, val);
 	if (REG_RD(sc, IGU_REG_PF_CONFIGURATION) != val) {
-		PMD_DRV_LOG(ERR, "proper val not read from IGU!");
+		PMD_DRV_LOG(ERR, sc, "proper val not read from IGU!");
 	}
 }
 
@@ -5631,7 +5671,7 @@ static void bnx2x_nic_init(struct bnx2x_softc *sc, int load_code)
 {
 	int i;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	for (i = 0; i < sc->num_queues; i++) {
 		bnx2x_init_eth_fp(sc, i);
@@ -5761,7 +5801,7 @@ static int bnx2x_set_power_state(struct bnx2x_softc *sc, uint8_t state)
 
 	/* If there is no power capability, silently succeed */
 	if (!(sc->devinfo.pcie_cap_flags & BNX2X_PM_CAPABLE_FLAG)) {
-		PMD_DRV_LOG(WARNING, "No power capability");
+		PMD_DRV_LOG(WARNING, sc, "No power capability");
 		return 0;
 	}
 
@@ -5806,7 +5846,7 @@ static int bnx2x_set_power_state(struct bnx2x_softc *sc, uint8_t state)
 		break;
 
 	default:
-		PMD_DRV_LOG(NOTICE, "Can't support PCI power state = %d",
+		PMD_DRV_LOG(NOTICE, sc, "Can't support PCI power state = %d",
 			    state);
 		return -1;
 	}
@@ -5824,7 +5864,7 @@ static uint8_t bnx2x_trylock_hw_lock(struct bnx2x_softc *sc, uint32_t resource)
 
 	/* Validating that the resource is within range */
 	if (resource > HW_LOCK_MAX_RESOURCE_VALUE) {
-		PMD_DRV_LOG(INFO,
+		PMD_DRV_LOG(INFO, sc,
 			    "resource(0x%x) > HW_LOCK_MAX_RESOURCE_VALUE(0x%x)",
 			    resource, HW_LOCK_MAX_RESOURCE_VALUE);
 		return FALSE;
@@ -5844,7 +5884,7 @@ static uint8_t bnx2x_trylock_hw_lock(struct bnx2x_softc *sc, uint32_t resource)
 		return TRUE;
 	}
 
-	PMD_DRV_LOG(NOTICE, "Failed to get a resource lock 0x%x", resource);
+	PMD_DRV_LOG(NOTICE, sc, "Failed to get a resource lock 0x%x", resource);
 
 	return FALSE;
 }
@@ -5937,7 +5977,7 @@ static int bnx2x_er_poll_igu_vq(struct bnx2x_softc *sc)
 	} while (cnt-- > 0);
 
 	if (cnt <= 0) {
-		PMD_DRV_LOG(NOTICE, "Still pending IGU requests bits=0x%08x!",
+		PMD_DRV_LOG(NOTICE, sc, "Still pending IGU requests bits=0x%08x!",
 			    pend_bits);
 		return -1;
 	}
@@ -6018,7 +6058,7 @@ static int bnx2x_init_shmem(struct bnx2x_softc *sc)
 
 	} while (cnt++ < (MCP_TIMEOUT / MCP_ONE_TIMEOUT));
 
-	PMD_DRV_LOG(NOTICE, "BAD MCP validity signature");
+	PMD_DRV_LOG(NOTICE, sc, "BAD MCP validity signature");
 
 	return -1;
 }
@@ -6173,7 +6213,7 @@ static int bnx2x_process_kill(struct bnx2x_softc *sc, uint8_t global)
 	} while (cnt-- > 0);
 
 	if (cnt <= 0) {
-		PMD_DRV_LOG(NOTICE,
+		PMD_DRV_LOG(NOTICE, sc,
 			    "ERROR: Tetris buffer didn't get empty or there "
 			    "are still outstanding read requests after 1s! "
 			    "sr_cnt=0x%08x, blk_cnt=0x%08x, port_is_idle_0=0x%08x, "
@@ -6246,14 +6286,14 @@ static int bnx2x_leader_reset(struct bnx2x_softc *sc)
 		load_code = bnx2x_fw_command(sc, DRV_MSG_CODE_LOAD_REQ,
 					   DRV_MSG_CODE_LOAD_REQ_WITH_LFA);
 		if (!load_code) {
-			PMD_DRV_LOG(NOTICE, "MCP response failure, aborting");
+			PMD_DRV_LOG(NOTICE, sc, "MCP response failure, aborting");
 			rc = -1;
 			goto exit_leader_reset;
 		}
 
 		if ((load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) &&
 		    (load_code != FW_MSG_CODE_DRV_LOAD_COMMON)) {
-			PMD_DRV_LOG(NOTICE,
+			PMD_DRV_LOG(NOTICE, sc,
 				    "MCP unexpected response, aborting");
 			rc = -1;
 			goto exit_leader_reset2;
@@ -6261,7 +6301,7 @@ static int bnx2x_leader_reset(struct bnx2x_softc *sc)
 
 		load_code = bnx2x_fw_command(sc, DRV_MSG_CODE_LOAD_DONE, 0);
 		if (!load_code) {
-			PMD_DRV_LOG(NOTICE, "MCP response failure, aborting");
+			PMD_DRV_LOG(NOTICE, sc, "MCP response failure, aborting");
 			rc = -1;
 			goto exit_leader_reset2;
 		}
@@ -6269,7 +6309,7 @@ static int bnx2x_leader_reset(struct bnx2x_softc *sc)
 
 	/* try to recover after the failure */
 	if (bnx2x_process_kill(sc, global)) {
-		PMD_DRV_LOG(NOTICE, "Something bad occurred on engine %d!",
+		PMD_DRV_LOG(NOTICE, sc, "Something bad occurred on engine %d!",
 			    SC_PATH(sc));
 		rc = -1;
 		goto exit_leader_reset2;
@@ -6428,12 +6468,12 @@ bnx2x_pf_rx_q_prep(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
 	/* validate rings have enough entries to cross high thresholds */
 	if (sc->dropless_fc &&
 	    pause->bd_th_hi + FW_PREFETCH_CNT > sc->rx_ring_size) {
-		PMD_DRV_LOG(WARNING, "rx bd ring threshold limit");
+		PMD_DRV_LOG(WARNING, sc, "rx bd ring threshold limit");
 	}
 
 	if (sc->dropless_fc &&
 	    pause->rcq_th_hi + FW_PREFETCH_CNT > USABLE_RCQ_ENTRIES(rxq)) {
-		PMD_DRV_LOG(WARNING, "rcq ring threshold limit");
+		PMD_DRV_LOG(WARNING, sc, "rcq ring threshold limit");
 	}
 
 	pause->pri_map = 1;
@@ -6504,7 +6544,7 @@ bnx2x_setup_queue(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp, uint8_t lea
 	struct ecore_queue_setup_params *setup_params = &q_params.params.setup;
 	int rc;
 
-	PMD_DRV_LOG(DEBUG, "setting up queue %d", fp->index);
+	PMD_DRV_LOG(DEBUG, sc, "setting up queue %d", fp->index);
 
 	bnx2x_ack_sb(sc, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_ENABLE, 0);
 
@@ -6522,11 +6562,11 @@ bnx2x_setup_queue(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp, uint8_t lea
 	/* Change the state to INIT */
 	rc = ecore_queue_state_change(sc, &q_params);
 	if (rc) {
-		PMD_DRV_LOG(NOTICE, "Queue(%d) INIT failed", fp->index);
+		PMD_DRV_LOG(NOTICE, sc, "Queue(%d) INIT failed", fp->index);
 		return rc;
 	}
 
-	PMD_DRV_LOG(DEBUG, "init complete");
+	PMD_DRV_LOG(DEBUG, sc, "init complete");
 
 	/* now move the Queue to the SETUP state */
 	memset(setup_params, 0, sizeof(*setup_params));
@@ -6550,7 +6590,7 @@ bnx2x_setup_queue(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp, uint8_t lea
 	/* change the state to SETUP */
 	rc = ecore_queue_state_change(sc, &q_params);
 	if (rc) {
-		PMD_DRV_LOG(NOTICE, "Queue(%d) SETUP failed", fp->index);
+		PMD_DRV_LOG(NOTICE, sc, "Queue(%d) SETUP failed", fp->index);
 		return rc;
 	}
 
@@ -6678,11 +6718,11 @@ bnx2x_set_mac_one(struct bnx2x_softc *sc, uint8_t * mac,
 	rc = ecore_config_vlan_mac(sc, &ramrod_param);
 
 	if (rc == ECORE_EXISTS) {
-		PMD_DRV_LOG(INFO, "Failed to schedule ADD operations (EEXIST)");
+		PMD_DRV_LOG(INFO, sc, "Failed to schedule ADD operations (EEXIST)");
 /* do not treat adding same MAC as error */
 		rc = 0;
 	} else if (rc < 0) {
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "%s MAC failed (%d)", (set ? "Set" : "Delete"), rc);
 	}
 
@@ -6693,7 +6733,7 @@ static int bnx2x_set_eth_mac(struct bnx2x_softc *sc, uint8_t set)
 {
 	unsigned long ramrod_flags = 0;
 
-	PMD_DRV_LOG(DEBUG, "Adding Ethernet MAC");
+	PMD_DRV_LOG(DEBUG, sc, "Adding Ethernet MAC");
 
 	bnx2x_set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
 
@@ -6849,7 +6889,7 @@ bnx2x_fill_report_data(struct bnx2x_softc *sc, struct bnx2x_link_report_data *da
 }
 
 /* report link status to OS, should be called under phy_lock */
-static void bnx2x_link_report(struct bnx2x_softc *sc)
+static void bnx2x_link_report_locked(struct bnx2x_softc *sc)
 {
 	struct bnx2x_link_report_data cur_data;
 
@@ -6870,14 +6910,19 @@ static void bnx2x_link_report(struct bnx2x_softc *sc)
 		return;
 	}
 
+	PMD_DRV_LOG(INFO, sc, "Change in link status : cur_data = %lx, last_reported_link = %lx\n",
+		    cur_data.link_report_flags,
+		    sc->last_reported_link.link_report_flags);
+
 	sc->link_cnt++;
 
+	PMD_DRV_LOG(INFO, sc, "link status change count = %x\n", sc->link_cnt);
 	/* report new link params and remember the state for the next time */
 	rte_memcpy(&sc->last_reported_link, &cur_data, sizeof(cur_data));
 
 	if (bnx2x_test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 			 &cur_data.link_report_flags)) {
-		PMD_DRV_LOG(INFO, "NIC Link is Down");
+		PMD_DRV_LOG(INFO, sc, "NIC Link is Down");
 	} else {
 		__rte_unused const char *duplex;
 		__rte_unused const char *flow;
@@ -6917,12 +6962,20 @@ static void bnx2x_link_report(struct bnx2x_softc *sc)
 			flow = "none";
 		}
 
-		PMD_DRV_LOG(INFO,
+		PMD_DRV_LOG(INFO, sc,
 			    "NIC Link is Up, %d Mbps %s duplex, Flow control: %s",
 			    cur_data.line_speed, duplex, flow);
 	}
 }
 
+static void
+bnx2x_link_report(struct bnx2x_softc *sc)
+{
+	bnx2x_acquire_phy_lock(sc);
+	bnx2x_link_report_locked(sc);
+	bnx2x_release_phy_lock(sc);
+}
+
 void bnx2x_link_status_update(struct bnx2x_softc *sc)
 {
 	if (sc->state != BNX2X_STATE_OPEN) {
@@ -7001,6 +7054,8 @@ static int bnx2x_initial_phy_init(struct bnx2x_softc *sc, int load_mode)
 
 	bnx2x_set_requested_fc(sc);
 
+	bnx2x_acquire_phy_lock(sc);
+
 	if (load_mode == LOAD_DIAG) {
 		lp->loopback_mode = ELINK_LOOPBACK_XGXS;
 /* Prefer doing PHY loopback at 10G speed, if possible */
@@ -7020,6 +7075,8 @@ static int bnx2x_initial_phy_init(struct bnx2x_softc *sc, int load_mode)
 
 	rc = elink_phy_init(&sc->link_params, &sc->link_vars);
 
+	bnx2x_release_phy_lock(sc);
+
 	bnx2x_calc_fc_adv(sc);
 
 	if (sc->link_vars.link_up) {
@@ -7058,7 +7115,7 @@ void bnx2x_periodic_callout(struct bnx2x_softc *sc)
 {
 	if ((sc->state != BNX2X_STATE_OPEN) ||
 	    (atomic_load_acq_long(&sc->periodic_flags) == PERIODIC_STOP)) {
-		PMD_DRV_LOG(WARNING, "periodic callout exit (state=0x%x)",
+		PMD_DRV_LOG(INFO, sc, "periodic callout exit (state=0x%x)",
 			    sc->state);
 		return;
 	}
@@ -7070,7 +7127,9 @@ void bnx2x_periodic_callout(struct bnx2x_softc *sc)
  */
 		mb();
 		if (sc->port.pmf) {
+			bnx2x_acquire_phy_lock(sc);
 			elink_period_func(&sc->link_params, &sc->link_vars);
+			bnx2x_release_phy_lock(sc);
 		}
 	}
 #ifdef BNX2X_PULSE
@@ -7095,7 +7154,7 @@ void bnx2x_periodic_callout(struct bnx2x_softc *sc)
 		if ((drv_pulse != mcp_pulse) &&
 		    (drv_pulse != ((mcp_pulse + 1) & MCP_PULSE_SEQ_MASK))) {
 			/* someone lost a heartbeat... */
-			PMD_DRV_LOG(ERR,
+			PMD_DRV_LOG(ERR, sc,
 				    "drv_pulse (0x%x) != mcp_pulse (0x%x)",
 				    drv_pulse, mcp_pulse);
 		}
@@ -7111,7 +7170,7 @@ int bnx2x_nic_load(struct bnx2x_softc *sc)
 	uint32_t load_code = 0;
 	int i, rc = 0;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	sc->state = BNX2X_STATE_OPENING_WAITING_LOAD;
 
@@ -7165,7 +7224,7 @@ int bnx2x_nic_load(struct bnx2x_softc *sc)
 				goto bnx2x_nic_load_error2;
 			}
 		} else {
-			PMD_DRV_LOG(INFO, "Device has no MCP!");
+			PMD_DRV_LOG(INFO, sc, "Device has no MCP!");
 			load_code = bnx2x_nic_load_no_mcp(sc);
 		}
 
@@ -7177,7 +7236,7 @@ int bnx2x_nic_load(struct bnx2x_softc *sc)
 
 /* Initialize HW */
 		if (bnx2x_init_hw(sc, load_code) != 0) {
-			PMD_DRV_LOG(NOTICE, "HW init failed");
+			PMD_DRV_LOG(NOTICE, sc, "HW init failed");
 			bnx2x_fw_command(sc, DRV_MSG_CODE_LOAD_DONE, 0);
 			sc->state = BNX2X_STATE_CLOSED;
 			rc = -ENXIO;
@@ -7197,7 +7256,7 @@ int bnx2x_nic_load(struct bnx2x_softc *sc)
 		sc->state = BNX2X_STATE_OPENING_WAITING_PORT;
 		rc = bnx2x_func_start(sc);
 		if (rc) {
-			PMD_DRV_LOG(NOTICE, "Function start failed!");
+			PMD_DRV_LOG(NOTICE, sc, "Function start failed!");
 			bnx2x_fw_command(sc, DRV_MSG_CODE_LOAD_DONE, 0);
 			sc->state = BNX2X_STATE_ERROR;
 			goto bnx2x_nic_load_error3;
@@ -7208,7 +7267,7 @@ int bnx2x_nic_load(struct bnx2x_softc *sc)
 			load_code =
 			    bnx2x_fw_command(sc, DRV_MSG_CODE_LOAD_DONE, 0);
 			if (!load_code) {
-				PMD_DRV_LOG(NOTICE,
+				PMD_DRV_LOG(NOTICE, sc,
 					    "MCP response failure, aborting");
 				sc->state = BNX2X_STATE_ERROR;
 				rc = -ENXIO;
@@ -7219,7 +7278,7 @@ int bnx2x_nic_load(struct bnx2x_softc *sc)
 
 	rc = bnx2x_setup_leading(sc);
 	if (rc) {
-		PMD_DRV_LOG(NOTICE, "Setup leading failed!");
+		PMD_DRV_LOG(NOTICE, sc, "Setup leading failed!");
 		sc->state = BNX2X_STATE_ERROR;
 		goto bnx2x_nic_load_error3;
 	}
@@ -7231,7 +7290,7 @@ int bnx2x_nic_load(struct bnx2x_softc *sc)
 			rc = bnx2x_vf_setup_queue(sc, &sc->fp[i], FALSE);
 
 		if (rc) {
-			PMD_DRV_LOG(NOTICE, "Queue(%d) setup failed", i);
+			PMD_DRV_LOG(NOTICE, sc, "Queue(%d) setup failed", i);
 			sc->state = BNX2X_STATE_ERROR;
 			goto bnx2x_nic_load_error3;
 		}
@@ -7239,7 +7298,7 @@ int bnx2x_nic_load(struct bnx2x_softc *sc)
 
 	rc = bnx2x_init_rss_pf(sc);
 	if (rc) {
-		PMD_DRV_LOG(NOTICE, "PF RSS init failed");
+		PMD_DRV_LOG(NOTICE, sc, "PF RSS init failed");
 		sc->state = BNX2X_STATE_ERROR;
 		goto bnx2x_nic_load_error3;
 	}
@@ -7255,7 +7314,7 @@ int bnx2x_nic_load(struct bnx2x_softc *sc)
 	}
 
 	if (rc) {
-		PMD_DRV_LOG(NOTICE, "Setting Ethernet MAC failed");
+		PMD_DRV_LOG(NOTICE, sc, "Setting Ethernet MAC failed");
 		sc->state = BNX2X_STATE_ERROR;
 		goto bnx2x_nic_load_error3;
 	}
@@ -7307,13 +7366,13 @@ int bnx2x_nic_load(struct bnx2x_softc *sc)
 
 	/* wait for all pending SP commands to complete */
 	if (IS_PF(sc) && !bnx2x_wait_sp_comp(sc, ~0x0UL)) {
-		PMD_DRV_LOG(NOTICE, "Timeout waiting for all SPs to complete!");
+		PMD_DRV_LOG(NOTICE, sc, "Timeout waiting for all SPs to complete!");
 		bnx2x_periodic_stop(sc);
 		bnx2x_nic_unload(sc, UNLOAD_CLOSE, FALSE);
 		return -ENXIO;
 	}
 
-	PMD_DRV_LOG(DEBUG, "NIC successfully loaded");
+	PMD_DRV_LOG(DEBUG, sc, "NIC successfully loaded");
 
 	return 0;
 
@@ -7362,7 +7421,7 @@ int bnx2x_init(struct bnx2x_softc *sc)
 
 	/* Check if the driver is still running and bail out if it is. */
 	if (sc->state != BNX2X_STATE_CLOSED) {
-		PMD_DRV_LOG(DEBUG, "Init called while driver is running!");
+		PMD_DRV_LOG(DEBUG, sc, "Init called while driver is running!");
 		rc = 0;
 		goto bnx2x_init_done;
 	}
@@ -7400,7 +7459,7 @@ int bnx2x_init(struct bnx2x_softc *sc)
 				     && (!global ||!other_load_status))
 				    && bnx2x_trylock_leader_lock(sc)
 				    && !bnx2x_leader_reset(sc)) {
-					PMD_DRV_LOG(INFO,
+					PMD_DRV_LOG(INFO, sc,
 						    "Recovered during init");
 					break;
 				}
@@ -7410,7 +7469,7 @@ int bnx2x_init(struct bnx2x_softc *sc)
 
 				sc->recovery_state = BNX2X_RECOVERY_FAILED;
 
-				PMD_DRV_LOG(NOTICE,
+				PMD_DRV_LOG(NOTICE, sc,
 					    "Recovery flow hasn't properly "
 					    "completed yet, try again later. "
 					    "If you still see this message after a "
@@ -7429,7 +7488,7 @@ int bnx2x_init(struct bnx2x_softc *sc)
 bnx2x_init_done:
 
 	if (rc) {
-		PMD_DRV_LOG(NOTICE, "Initialization failed, "
+		PMD_DRV_LOG(NOTICE, sc, "Initialization failed, "
 			    "stack notified driver is NOT running!");
 	}
 
@@ -7461,7 +7520,7 @@ static void bnx2x_get_function_num(struct bnx2x_softc *sc)
 		sc->pfunc_abs = (sc->pfunc_rel | sc->path_id);
 	}
 
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 		    "Relative function %d, Absolute function %d, Path %d",
 		    sc->pfunc_rel, sc->pfunc_abs, sc->path_id);
 }
@@ -7498,14 +7557,14 @@ static uint32_t bnx2x_pcie_capability_read(struct bnx2x_softc *sc, int reg)
 	/* ensure PCIe capability is enabled */
 	caps = pci_find_cap(sc, PCIY_EXPRESS, BNX2X_PCI_CAP);
 	if (NULL != caps) {
-		PMD_DRV_LOG(DEBUG, "Found PCIe capability: "
+		PMD_DRV_LOG(DEBUG, sc, "Found PCIe capability: "
 			    "id=0x%04X type=0x%04X addr=0x%08X",
 			    caps->id, caps->type, caps->addr);
 		pci_read(sc, (caps->addr + reg), &ret, 2);
 		return ret;
 	}
 
-	PMD_DRV_LOG(WARNING, "PCIe capability NOT FOUND!!!");
+	PMD_DRV_LOG(WARNING, sc, "PCIe capability NOT FOUND!!!");
 
 	return 0;
 }
@@ -7523,7 +7582,7 @@ static uint8_t bnx2x_is_pcie_pending(struct bnx2x_softc *sc)
 */
 static void bnx2x_probe_pci_caps(struct bnx2x_softc *sc)
 {
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	struct bnx2x_pci_cap *caps;
 	uint16_t link_status;
@@ -7532,7 +7591,7 @@ static void bnx2x_probe_pci_caps(struct bnx2x_softc *sc)
 	/* check if PCI Power Management is enabled */
 	caps = pci_find_cap(sc, PCIY_PMG, BNX2X_PCI_CAP);
 	if (NULL != caps) {
-		PMD_DRV_LOG(DEBUG, "Found PM capability: "
+		PMD_DRV_LOG(DEBUG, sc, "Found PM capability: "
 			    "id=0x%04X type=0x%04X addr=0x%08X",
 			    caps->id, caps->type, caps->addr);
 
@@ -7546,7 +7605,7 @@ static void bnx2x_probe_pci_caps(struct bnx2x_softc *sc)
 	sc->devinfo.pcie_link_width =
 	    ((link_status & PCIM_LINK_STA_WIDTH) >> 4);
 
-	PMD_DRV_LOG(DEBUG, "PCIe link speed=%d width=%d",
+	PMD_DRV_LOG(DEBUG, sc, "PCIe link speed=%d width=%d",
 		    sc->devinfo.pcie_link_speed, sc->devinfo.pcie_link_width);
 
 	sc->devinfo.pcie_cap_flags |= BNX2X_PCIE_CAPABLE_FLAG;
@@ -7554,7 +7613,7 @@ static void bnx2x_probe_pci_caps(struct bnx2x_softc *sc)
 	/* check if MSI capability is enabled */
 	caps = pci_find_cap(sc, PCIY_MSI, BNX2X_PCI_CAP);
 	if (NULL != caps) {
-		PMD_DRV_LOG(DEBUG, "Found MSI capability at 0x%04x", reg);
+		PMD_DRV_LOG(DEBUG, sc, "Found MSI capability at 0x%04x", reg);
 
 		sc->devinfo.pcie_cap_flags |= BNX2X_MSI_CAPABLE_FLAG;
 		sc->devinfo.pcie_msi_cap_reg = caps->addr;
@@ -7563,7 +7622,7 @@ static void bnx2x_probe_pci_caps(struct bnx2x_softc *sc)
 	/* check if MSI-X capability is enabled */
 	caps = pci_find_cap(sc, PCIY_MSIX, BNX2X_PCI_CAP);
 	if (NULL != caps) {
-		PMD_DRV_LOG(DEBUG, "Found MSI-X capability at 0x%04x", reg);
+		PMD_DRV_LOG(DEBUG, sc, "Found MSI-X capability at 0x%04x", reg);
 
 		sc->devinfo.pcie_cap_flags |= BNX2X_MSIX_CAPABLE_FLAG;
 		sc->devinfo.pcie_msix_cap_reg = caps->addr;
@@ -7583,7 +7642,7 @@ static int bnx2x_get_shmem_mf_cfg_info_sd(struct bnx2x_softc *sc)
 	mf_info->multi_vnics_mode = 1;
 
 	if (!VALID_OVLAN(mf_info->ext_id)) {
-		PMD_DRV_LOG(NOTICE, "Invalid VLAN (%d)", mf_info->ext_id);
+		PMD_DRV_LOG(NOTICE, sc, "Invalid VLAN (%d)", mf_info->ext_id);
 		return 1;
 	}
 
@@ -7707,14 +7766,14 @@ static int bnx2x_check_valid_mf_cfg(struct bnx2x_softc *sc)
 	/* various MF mode sanity checks... */
 
 	if (mf_info->mf_config[SC_VN(sc)] & FUNC_MF_CFG_FUNC_HIDE) {
-		PMD_DRV_LOG(NOTICE,
+		PMD_DRV_LOG(NOTICE, sc,
 			    "Enumerated function %d is marked as hidden",
 			    SC_PORT(sc));
 		return 1;
 	}
 
 	if ((mf_info->vnics_per_port > 1) && !mf_info->multi_vnics_mode) {
-		PMD_DRV_LOG(NOTICE, "vnics_per_port=%d multi_vnics_mode=%d",
+		PMD_DRV_LOG(NOTICE, sc, "vnics_per_port=%d multi_vnics_mode=%d",
 			    mf_info->vnics_per_port, mf_info->multi_vnics_mode);
 		return 1;
 	}
@@ -7722,13 +7781,13 @@ static int bnx2x_check_valid_mf_cfg(struct bnx2x_softc *sc)
 	if (mf_info->mf_mode == MULTI_FUNCTION_SD) {
 /* vnic id > 0 must have valid ovlan in switch-dependent mode */
 		if ((SC_VN(sc) > 0) && !VALID_OVLAN(OVLAN(sc))) {
-			PMD_DRV_LOG(NOTICE, "mf_mode=SD vnic_id=%d ovlan=%d",
+			PMD_DRV_LOG(NOTICE, sc, "mf_mode=SD vnic_id=%d ovlan=%d",
 				    SC_VN(sc), OVLAN(sc));
 			return 1;
 		}
 
 		if (!VALID_OVLAN(OVLAN(sc)) && mf_info->multi_vnics_mode) {
-			PMD_DRV_LOG(NOTICE,
+			PMD_DRV_LOG(NOTICE, sc,
 				    "mf_mode=SD multi_vnics_mode=%d ovlan=%d",
 				    mf_info->multi_vnics_mode, OVLAN(sc));
 			return 1;
@@ -7747,7 +7806,7 @@ static int bnx2x_check_valid_mf_cfg(struct bnx2x_softc *sc)
 			      && !VALID_OVLAN(ovlan1))
 			     || ((!mf_info->multi_vnics_mode)
 				 && VALID_OVLAN(ovlan1)))) {
-				PMD_DRV_LOG(NOTICE,
+				PMD_DRV_LOG(NOTICE, sc,
 					    "mf_mode=SD function %d MF config "
 					    "mismatch, multi_vnics_mode=%d ovlan=%d",
 					    i, mf_info->multi_vnics_mode,
@@ -7771,7 +7830,7 @@ static int bnx2x_check_valid_mf_cfg(struct bnx2x_softc *sc)
 				    && !(mf_cfg2 & FUNC_MF_CFG_FUNC_HIDE)
 				    && VALID_OVLAN(ovlan2)
 				    && (ovlan1 == ovlan2)) {
-					PMD_DRV_LOG(NOTICE,
+					PMD_DRV_LOG(NOTICE, sc,
 						    "mf_mode=SD functions %d and %d "
 						    "have the same ovlan (%d)",
 						    i, j, ovlan1);
@@ -7801,7 +7860,7 @@ static int bnx2x_get_mf_cfg_info(struct bnx2x_softc *sc)
 	}
 
 	if (sc->devinfo.mf_cfg_base == SHMEM_MF_CFG_ADDR_NONE) {
-		PMD_DRV_LOG(NOTICE, "Invalid mf_cfg_base!");
+		PMD_DRV_LOG(NOTICE, sc, "Invalid mf_cfg_base!");
 		return 1;
 	}
 
@@ -7819,7 +7878,7 @@ static int bnx2x_get_mf_cfg_info(struct bnx2x_softc *sc)
 		if (mac_upper != FUNC_MF_CFG_UPPERMAC_DEFAULT) {
 			mf_info->mf_mode = MULTI_FUNCTION_SI;
 		} else {
-			PMD_DRV_LOG(NOTICE,
+			PMD_DRV_LOG(NOTICE, sc,
 				    "Invalid config for Switch Independent mode");
 		}
 
@@ -7835,7 +7894,7 @@ static int bnx2x_get_mf_cfg_info(struct bnx2x_softc *sc)
 		    FUNC_MF_CFG_E1HOV_TAG_DEFAULT) {
 			mf_info->mf_mode = MULTI_FUNCTION_SD;
 		} else {
-			PMD_DRV_LOG(NOTICE,
+			PMD_DRV_LOG(NOTICE, sc,
 				    "Invalid config for Switch Dependent mode");
 		}
 
@@ -7859,14 +7918,14 @@ static int bnx2x_get_mf_cfg_info(struct bnx2x_softc *sc)
 		    (mac_upper != FUNC_MF_CFG_UPPERMAC_DEFAULT)) {
 			mf_info->mf_mode = MULTI_FUNCTION_AFEX;
 		} else {
-			PMD_DRV_LOG(NOTICE, "Invalid config for AFEX mode");
+			PMD_DRV_LOG(NOTICE, sc, "Invalid config for AFEX mode");
 		}
 
 		break;
 
 	default:
 
-		PMD_DRV_LOG(NOTICE, "Unknown MF mode (0x%08x)",
+		PMD_DRV_LOG(NOTICE, sc, "Unknown MF mode (0x%08x)",
 			    (val & SHARED_FEAT_CFG_FORCE_SF_MODE_MASK));
 
 		return 1;
@@ -7898,7 +7957,7 @@ static int bnx2x_get_mf_cfg_info(struct bnx2x_softc *sc)
 	if (mf_info->mf_mode == SINGLE_FUNCTION) {
 /* invalid MF config */
 		if (SC_VN(sc) >= 1) {
-			PMD_DRV_LOG(NOTICE, "VNIC ID >= 1 in SF mode");
+			PMD_DRV_LOG(NOTICE, sc, "VNIC ID >= 1 in SF mode");
 			return 1;
 		}
 
@@ -7927,7 +7986,7 @@ static int bnx2x_get_mf_cfg_info(struct bnx2x_softc *sc)
 
 	default:
 
-		PMD_DRV_LOG(NOTICE, "Get MF config failed (mf_mode=0x%08x)",
+		PMD_DRV_LOG(NOTICE, sc, "Get MF config failed (mf_mode=0x%08x)",
 			    mf_info->mf_mode);
 		return 1;
 	}
@@ -7955,7 +8014,7 @@ static int bnx2x_get_shmem_info(struct bnx2x_softc *sc)
 	int port;
 	uint32_t mac_hi, mac_lo, val;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	port = SC_PORT(sc);
 	mac_hi = mac_lo = 0;
@@ -8029,7 +8088,7 @@ static int bnx2x_get_shmem_info(struct bnx2x_softc *sc)
 
 	if ((mac_lo == 0) && (mac_hi == 0)) {
 		*sc->mac_addr_str = 0;
-		PMD_DRV_LOG(NOTICE, "No Ethernet address programmed!");
+		PMD_DRV_LOG(NOTICE, sc, "No Ethernet address programmed!");
 	} else {
 		sc->link_params.mac_addr[0] = (uint8_t) (mac_hi >> 8);
 		sc->link_params.mac_addr[1] = (uint8_t) (mac_hi);
@@ -8045,7 +8104,8 @@ static int bnx2x_get_shmem_info(struct bnx2x_softc *sc)
 			 sc->link_params.mac_addr[3],
 			 sc->link_params.mac_addr[4],
 			 sc->link_params.mac_addr[5]);
-		PMD_DRV_LOG(DEBUG, "Ethernet address: %s", sc->mac_addr_str);
+		PMD_DRV_LOG(DEBUG, sc,
+			    "Ethernet address: %s", sc->mac_addr_str);
 	}
 
 	return 0;
@@ -8060,24 +8120,24 @@ static void bnx2x_media_detect(struct bnx2x_softc *sc)
 	case ELINK_ETH_PHY_XFP_FIBER:
 	case ELINK_ETH_PHY_KR:
 	case ELINK_ETH_PHY_CX4:
-		PMD_DRV_LOG(INFO, "Found 10GBase-CX4 media.");
+		PMD_DRV_LOG(INFO, sc, "Found 10GBase-CX4 media.");
 		sc->media = IFM_10G_CX4;
 		break;
 	case ELINK_ETH_PHY_DA_TWINAX:
-		PMD_DRV_LOG(INFO, "Found 10Gb Twinax media.");
+		PMD_DRV_LOG(INFO, sc, "Found 10Gb Twinax media.");
 		sc->media = IFM_10G_TWINAX;
 		break;
 	case ELINK_ETH_PHY_BASE_T:
-		PMD_DRV_LOG(INFO, "Found 10GBase-T media.");
+		PMD_DRV_LOG(INFO, sc, "Found 10GBase-T media.");
 		sc->media = IFM_10G_T;
 		break;
 	case ELINK_ETH_PHY_NOT_PRESENT:
-		PMD_DRV_LOG(INFO, "Media not present.");
+		PMD_DRV_LOG(INFO, sc, "Media not present.");
 		sc->media = 0;
 		break;
 	case ELINK_ETH_PHY_UNSPECIFIED:
 	default:
-		PMD_DRV_LOG(INFO, "Unknown media!");
+		PMD_DRV_LOG(INFO, sc, "Unknown media!");
 		sc->media = 0;
 		break;
 	}
@@ -8140,7 +8200,7 @@ static int bnx2x_get_igu_cam_info(struct bnx2x_softc *sc)
 	sc->igu_sb_cnt = min(sc->igu_sb_cnt, igu_sb_cnt);
 
 	if (igu_sb_cnt == 0) {
-		PMD_DRV_LOG(ERR, "CAM configuration error");
+		PMD_DRV_LOG(ERR, sc, "CAM configuration error");
 		return -1;
 	}
 
@@ -8177,7 +8237,7 @@ static int bnx2x_get_device_info(struct bnx2x_softc *sc)
 		sc->devinfo.chip_id |= 0x1;
 	}
 
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 		    "chip_id=0x%08x (num=0x%04x rev=0x%01x metal=0x%02x bond=0x%01x)",
 		    sc->devinfo.chip_id,
 		    ((sc->devinfo.chip_id >> 16) & 0xffff),
@@ -8188,7 +8248,7 @@ static int bnx2x_get_device_info(struct bnx2x_softc *sc)
 	val = (REG_RD(sc, 0x2874) & 0x55);
 	if ((sc->devinfo.chip_id & 0x1) || (CHIP_IS_E1H(sc) && (val == 0x55))) {
 		sc->flags |= BNX2X_ONE_PORT_FLAG;
-		PMD_DRV_LOG(DEBUG, "single port device");
+		PMD_DRV_LOG(DEBUG, sc, "single port device");
 	}
 
 	/* set the doorbell size */
@@ -8212,7 +8272,7 @@ static int bnx2x_get_device_info(struct bnx2x_softc *sc)
 		sc->devinfo.chip_port_mode =
 		    (val) ? CHIP_4_PORT_MODE : CHIP_2_PORT_MODE;
 
-		PMD_DRV_LOG(DEBUG, "Port mode = %s", (val) ? "4" : "2");
+		PMD_DRV_LOG(DEBUG, sc, "Port mode = %s", (val) ? "4" : "2");
 	}
 
 	/* get the function and path info for the device */
@@ -8227,7 +8287,7 @@ static int bnx2x_get_device_info(struct bnx2x_softc *sc)
 
 	if (!sc->devinfo.shmem_base) {
 /* this should ONLY prevent upcoming shmem reads */
-		PMD_DRV_LOG(INFO, "MCP not active");
+		PMD_DRV_LOG(INFO, sc, "MCP not active");
 		sc->flags |= BNX2X_NO_MCP_FLAG;
 		return 0;
 	}
@@ -8236,7 +8296,7 @@ static int bnx2x_get_device_info(struct bnx2x_softc *sc)
 	val = SHMEM_RD(sc, validity_map[SC_PORT(sc)]);
 	if ((val & (SHR_MEM_VALIDITY_DEV_INFO | SHR_MEM_VALIDITY_MB)) !=
 	    (SHR_MEM_VALIDITY_DEV_INFO | SHR_MEM_VALIDITY_MB)) {
-		PMD_DRV_LOG(NOTICE, "Invalid SHMEM validity signature: 0x%08x",
+		PMD_DRV_LOG(NOTICE, sc, "Invalid SHMEM validity signature: 0x%08x",
 			    val);
 		return 0;
 	}
@@ -8249,7 +8309,7 @@ static int bnx2x_get_device_info(struct bnx2x_softc *sc)
 		 ((sc->devinfo.bc_ver >> 24) & 0xff),
 		 ((sc->devinfo.bc_ver >> 16) & 0xff),
 		 ((sc->devinfo.bc_ver >> 8) & 0xff));
-	PMD_DRV_LOG(INFO, "Bootcode version: %s", sc->devinfo.bc_ver_str);
+	PMD_DRV_LOG(INFO, sc, "Bootcode version: %s", sc->devinfo.bc_ver_str);
 
 	/* get the bootcode shmem address */
 	sc->devinfo.mf_cfg_base = bnx2x_get_shmem_mf_cfg_base(sc);
@@ -8304,7 +8364,7 @@ static int bnx2x_get_device_info(struct bnx2x_softc *sc)
 			}
 
 			if (REG_RD(sc, IGU_REG_RESET_MEMORIES)) {
-				PMD_DRV_LOG(NOTICE,
+				PMD_DRV_LOG(NOTICE, sc,
 					    "FORCING IGU Normal Mode failed!!!");
 				bnx2x_release_hw_lock(sc, HW_LOCK_RESOURCE_RESET);
 				return -1;
@@ -8312,10 +8372,10 @@ static int bnx2x_get_device_info(struct bnx2x_softc *sc)
 		}
 
 		if (val & IGU_BLOCK_CONFIGURATION_REG_BACKWARD_COMP_EN) {
-			PMD_DRV_LOG(DEBUG, "IGU Backward Compatible Mode");
+			PMD_DRV_LOG(DEBUG, sc, "IGU Backward Compatible Mode");
 			sc->devinfo.int_block |= INT_BLOCK_MODE_BW_COMP;
 		} else {
-			PMD_DRV_LOG(DEBUG, "IGU Normal Mode");
+			PMD_DRV_LOG(DEBUG, sc, "IGU Normal Mode");
 		}
 
 		rc = bnx2x_get_igu_cam_info(sc);
@@ -8389,7 +8449,7 @@ bnx2x_link_settings_supported(struct bnx2x_softc *sc, uint32_t switch_cfg)
 	}
 
 	if (!(sc->port.supported[0] || sc->port.supported[1])) {
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "Invalid phy config in NVRAM (PHY1=0x%08x PHY2=0x%08x)",
 			    SHMEM_RD(sc,
 				     dev_info.port_hw_config
@@ -8415,7 +8475,7 @@ bnx2x_link_settings_supported(struct bnx2x_softc *sc, uint32_t switch_cfg)
 				   NIG_REG_XGXS0_CTRL_PHY_ADDR + port * 0x18);
 			break;
 		default:
-			PMD_DRV_LOG(ERR,
+			PMD_DRV_LOG(ERR, sc,
 				    "Invalid switch config in"
 				    "link_config=0x%08x",
 				    sc->port.link_config[0]);
@@ -8423,7 +8483,7 @@ bnx2x_link_settings_supported(struct bnx2x_softc *sc, uint32_t switch_cfg)
 		}
 	}
 
-	PMD_DRV_LOG(INFO, "PHY addr 0x%08x", sc->port.phy_addr);
+	PMD_DRV_LOG(INFO, sc, "PHY addr 0x%08x", sc->port.phy_addr);
 
 	/* mask what we support according to speed_cap_mask per configuration */
 	for (idx = 0; idx < cfg_size; idx++) {
@@ -8476,7 +8536,7 @@ bnx2x_link_settings_supported(struct bnx2x_softc *sc, uint32_t switch_cfg)
 		}
 	}
 
-	PMD_DRV_LOG(INFO, "PHY supported 0=0x%08x 1=0x%08x",
+	PMD_DRV_LOG(INFO, sc, "PHY supported 0=0x%08x 1=0x%08x",
 		    sc->port.supported[0], sc->port.supported[1]);
 }
 
@@ -8535,7 +8595,7 @@ static void bnx2x_link_settings_requested(struct bnx2x_softc *sc)
 				sc->port.advertising[idx] |=
 				    (ADVERTISED_10baseT_Full | ADVERTISED_TP);
 			} else {
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Invalid NVRAM config link_config=0x%08x "
 					    "speed_cap_mask=0x%08x",
 					    link_config,
@@ -8555,7 +8615,7 @@ static void bnx2x_link_settings_requested(struct bnx2x_softc *sc)
 				sc->port.advertising[idx] |=
 				    (ADVERTISED_10baseT_Half | ADVERTISED_TP);
 			} else {
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Invalid NVRAM config link_config=0x%08x "
 					    "speed_cap_mask=0x%08x",
 					    link_config,
@@ -8574,7 +8634,7 @@ static void bnx2x_link_settings_requested(struct bnx2x_softc *sc)
 				sc->port.advertising[idx] |=
 				    (ADVERTISED_100baseT_Full | ADVERTISED_TP);
 			} else {
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Invalid NVRAM config link_config=0x%08x "
 					    "speed_cap_mask=0x%08x",
 					    link_config,
@@ -8594,7 +8654,7 @@ static void bnx2x_link_settings_requested(struct bnx2x_softc *sc)
 				sc->port.advertising[idx] |=
 				    (ADVERTISED_100baseT_Half | ADVERTISED_TP);
 			} else {
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Invalid NVRAM config link_config=0x%08x "
 					    "speed_cap_mask=0x%08x",
 					    link_config,
@@ -8612,7 +8672,7 @@ static void bnx2x_link_settings_requested(struct bnx2x_softc *sc)
 				sc->port.advertising[idx] |=
 				    (ADVERTISED_1000baseT_Full | ADVERTISED_TP);
 			} else {
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Invalid NVRAM config link_config=0x%08x "
 					    "speed_cap_mask=0x%08x",
 					    link_config,
@@ -8630,7 +8690,7 @@ static void bnx2x_link_settings_requested(struct bnx2x_softc *sc)
 				sc->port.advertising[idx] |=
 				    (ADVERTISED_2500baseX_Full | ADVERTISED_TP);
 			} else {
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Invalid NVRAM config link_config=0x%08x "
 					    "speed_cap_mask=0x%08x",
 					    link_config,
@@ -8649,7 +8709,7 @@ static void bnx2x_link_settings_requested(struct bnx2x_softc *sc)
 				    (ADVERTISED_10000baseT_Full |
 				     ADVERTISED_FIBRE);
 			} else {
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Invalid NVRAM config link_config=0x%08x "
 					    "speed_cap_mask=0x%08x",
 					    link_config,
@@ -8664,7 +8724,7 @@ static void bnx2x_link_settings_requested(struct bnx2x_softc *sc)
 			break;
 
 		default:
-			PMD_DRV_LOG(ERR,
+			PMD_DRV_LOG(ERR, sc,
 				    "Invalid NVRAM config link_config=0x%08x "
 				    "speed_cap_mask=0x%08x", link_config,
 				    sc->link_params.speed_cap_mask[idx]);
@@ -8695,7 +8755,7 @@ static void bnx2x_get_phy_info(struct bnx2x_softc *sc)
 	uint8_t port = SC_PORT(sc);
 	uint32_t eee_mode;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	/* shmem data already read in bnx2x_get_shmem_info() */
 
@@ -8855,7 +8915,7 @@ int bnx2x_alloc_hsi_mem(struct bnx2x_softc *sc)
 		snprintf(buf, sizeof(buf), "fp_%d_sb", i);
 		if (bnx2x_dma_alloc(sc, sizeof(union bnx2x_host_hc_status_block),
 				  &fp->sb_dma, buf, RTE_CACHE_LINE_SIZE) != 0) {
-			PMD_DRV_LOG(NOTICE, "Failed to alloc %s", buf);
+			PMD_DRV_LOG(NOTICE, sc, "Failed to alloc %s", buf);
 			return -1;
 		} else {
 			if (CHIP_IS_E2E3(sc)) {
@@ -8945,7 +9005,7 @@ static int bnx2x_prev_mcp_done(struct bnx2x_softc *sc)
 	uint32_t rc = bnx2x_fw_command(sc, DRV_MSG_CODE_UNLOAD_DONE,
 				     DRV_MSG_CODE_UNLOAD_SKIP_LINK_RESET);
 	if (!rc) {
-		PMD_DRV_LOG(NOTICE, "MCP response failure, aborting");
+		PMD_DRV_LOG(NOTICE, sc, "MCP response failure, aborting");
 		return -1;
 	}
 
@@ -8977,12 +9037,12 @@ static uint8_t bnx2x_prev_is_path_marked(struct bnx2x_softc *sc)
 	tmp = bnx2x_prev_path_get_entry(sc);
 	if (tmp) {
 		if (tmp->aer) {
-			PMD_DRV_LOG(DEBUG,
+			PMD_DRV_LOG(DEBUG, sc,
 				    "Path %d/%d/%d was marked by AER",
 				    sc->pcie_bus, sc->pcie_device, SC_PATH(sc));
 		} else {
 			rc = TRUE;
-			PMD_DRV_LOG(DEBUG,
+			PMD_DRV_LOG(DEBUG, sc,
 				    "Path %d/%d/%d was already cleaned from previous drivers",
 				    sc->pcie_bus, sc->pcie_device, SC_PATH(sc));
 		}
@@ -9003,11 +9063,11 @@ static int bnx2x_prev_mark_path(struct bnx2x_softc *sc, uint8_t after_undi)
 	tmp = bnx2x_prev_path_get_entry(sc);
 	if (tmp) {
 		if (!tmp->aer) {
-			PMD_DRV_LOG(DEBUG,
+			PMD_DRV_LOG(DEBUG, sc,
 				    "Re-marking AER in path %d/%d/%d",
 				    sc->pcie_bus, sc->pcie_device, SC_PATH(sc));
 		} else {
-			PMD_DRV_LOG(DEBUG,
+			PMD_DRV_LOG(DEBUG, sc,
 				    "Removing AER indication from path %d/%d/%d",
 				    sc->pcie_bus, sc->pcie_device, SC_PATH(sc));
 			tmp->aer = 0;
@@ -9023,7 +9083,7 @@ static int bnx2x_prev_mark_path(struct bnx2x_softc *sc, uint8_t after_undi)
 	tmp = rte_malloc("", sizeof(struct bnx2x_prev_list_node),
 			 RTE_CACHE_LINE_SIZE);
 	if (!tmp) {
-		PMD_DRV_LOG(NOTICE, "Failed to allocate 'bnx2x_prev_list_node'");
+		PMD_DRV_LOG(NOTICE, sc, "Failed to allocate 'bnx2x_prev_list_node'");
 		return -1;
 	}
 
@@ -9048,13 +9108,13 @@ static int bnx2x_do_flr(struct bnx2x_softc *sc)
 
 	/* only E2 and onwards support FLR */
 	if (CHIP_IS_E1x(sc)) {
-		PMD_DRV_LOG(WARNING, "FLR not supported in E1H");
+		PMD_DRV_LOG(WARNING, sc, "FLR not supported in E1H");
 		return -1;
 	}
 
 	/* only bootcode REQ_BC_VER_4_INITIATE_FLR and onwards support flr */
 	if (sc->devinfo.bc_ver < REQ_BC_VER_4_INITIATE_FLR) {
-		PMD_DRV_LOG(WARNING,
+		PMD_DRV_LOG(WARNING, sc,
 			    "FLR not supported by BC_VER: 0x%08x",
 			    sc->devinfo.bc_ver);
 		return -1;
@@ -9071,7 +9131,7 @@ static int bnx2x_do_flr(struct bnx2x_softc *sc)
 		}
 	}
 
-	PMD_DRV_LOG(NOTICE, "PCIE transaction is not cleared, "
+	PMD_DRV_LOG(NOTICE, sc, "PCIE transaction is not cleared, "
 		    "proceeding with reset anyway");
 
 clear:
@@ -9219,7 +9279,7 @@ static int bnx2x_prev_unload_common(struct bnx2x_softc *sc)
 		if (reset_reg & MISC_REGISTERS_RESET_REG_1_RST_DORQ) {
 			tmp_reg = REG_RD(sc, DORQ_REG_NORM_CID_OFST);
 			if (tmp_reg == 0x7) {
-				PMD_DRV_LOG(DEBUG, "UNDI previously loaded");
+				PMD_DRV_LOG(DEBUG, sc, "UNDI previously loaded");
 				prev_undi = TRUE;
 				/* clear the UNDI indication */
 				REG_WR(sc, DORQ_REG_NORM_CID_OFST, 0);
@@ -9238,7 +9298,7 @@ static int bnx2x_prev_unload_common(struct bnx2x_softc *sc)
 				break;
 			}
 
-			PMD_DRV_LOG(DEBUG, "BRB still has 0x%08x", tmp_reg);
+			PMD_DRV_LOG(DEBUG, sc, "BRB still has 0x%08x", tmp_reg);
 
 			/* reset timer as long as BRB actually gets emptied */
 			if (prev_brb > tmp_reg) {
@@ -9256,7 +9316,7 @@ static int bnx2x_prev_unload_common(struct bnx2x_softc *sc)
 		}
 
 		if (!timer_count) {
-			PMD_DRV_LOG(NOTICE, "Failed to empty BRB");
+			PMD_DRV_LOG(NOTICE, sc, "Failed to empty BRB");
 		}
 	}
 
@@ -9311,7 +9371,7 @@ static int bnx2x_prev_unload_uncommon(struct bnx2x_softc *sc)
 		return 0;
 	}
 
-	PMD_DRV_LOG(INFO, "Could not FLR");
+	PMD_DRV_LOG(INFO, sc, "Could not FLR");
 
 	/* Close the MCP request, return failure */
 	rc = bnx2x_prev_mcp_done(sc);
@@ -9358,7 +9418,7 @@ static int bnx2x_prev_unload(struct bnx2x_softc *sc)
 		/* Lock MCP using an unload request */
 		fw = bnx2x_fw_command(sc, DRV_MSG_CODE_UNLOAD_REQ_WOL_DIS, 0);
 		if (!fw) {
-			PMD_DRV_LOG(NOTICE, "MCP response failure, aborting");
+			PMD_DRV_LOG(NOTICE, sc, "MCP response failure, aborting");
 			rc = -1;
 			break;
 		}
@@ -9378,7 +9438,7 @@ static int bnx2x_prev_unload(struct bnx2x_softc *sc)
 	} while (--time_counter);
 
 	if (!time_counter || rc) {
-		PMD_DRV_LOG(NOTICE, "Failed to unload previous driver!");
+		PMD_DRV_LOG(NOTICE, sc, "Failed to unload previous driver!");
 		rc = -1;
 	}
 
@@ -9395,7 +9455,7 @@ bnx2x_dcbx_set_state(struct bnx2x_softc *sc, uint8_t dcb_on, uint32_t dcbx_enabl
 		sc->dcb_state = FALSE;
 		sc->dcbx_enabled = BNX2X_DCBX_ENABLED_INVALID;
 	}
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 		    "DCB state [%s:%s]",
 		    dcb_on ? "ON" : "OFF",
 		    (dcbx_enabled == BNX2X_DCBX_ENABLED_OFF) ? "user-mode" :
@@ -9428,7 +9488,7 @@ static void bnx2x_init_multi_cos(struct bnx2x_softc *sc)
 		if (cos < sc->max_cos) {
 			sc->prio_to_cos[pri] = cos;
 		} else {
-			PMD_DRV_LOG(WARNING,
+			PMD_DRV_LOG(WARNING, sc,
 				    "Invalid COS %d for priority %d "
 				    "(max COS is %d), setting to 0", cos, pri,
 				    (sc->max_cos - 1));
@@ -9449,7 +9509,7 @@ static int bnx2x_pci_get_caps(struct bnx2x_softc *sc)
 	cap = sc->pci_caps = rte_zmalloc("caps", sizeof(struct bnx2x_pci_cap),
 					 RTE_CACHE_LINE_SIZE);
 	if (!cap) {
-		PMD_DRV_LOG(NOTICE, "Failed to allocate memory");
+		PMD_DRV_LOG(NOTICE, sc, "Failed to allocate memory");
 		return -ENOMEM;
 	}
 
@@ -9460,7 +9520,7 @@ static int bnx2x_pci_get_caps(struct bnx2x_softc *sc)
 	pci_read(sc, PCIR_STATUS, &status, 2);
 	if (!(status & PCIM_STATUS_CAPPRESENT)) {
 #endif
-		PMD_DRV_LOG(NOTICE, "PCIe capability reading failed");
+		PMD_DRV_LOG(NOTICE, sc, "PCIe capability reading failed");
 		return -1;
 	}
 
@@ -9480,7 +9540,7 @@ static int bnx2x_pci_get_caps(struct bnx2x_softc *sc)
 					sizeof(struct bnx2x_pci_cap),
 					RTE_CACHE_LINE_SIZE);
 		if (!cap->next) {
-			PMD_DRV_LOG(NOTICE, "Failed to allocate memory");
+			PMD_DRV_LOG(NOTICE, sc, "Failed to allocate memory");
 			return -ENOMEM;
 		}
 		cap = cap->next;
@@ -9516,25 +9576,25 @@ void bnx2x_load_firmware(struct bnx2x_softc *sc)
 		? FW_NAME_57711 : FW_NAME_57810;
 	f = open(fwname, O_RDONLY);
 	if (f < 0) {
-		PMD_DRV_LOG(NOTICE, "Can't open firmware file");
+		PMD_DRV_LOG(NOTICE, sc, "Can't open firmware file");
 		return;
 	}
 
 	if (fstat(f, &st) < 0) {
-		PMD_DRV_LOG(NOTICE, "Can't stat firmware file");
+		PMD_DRV_LOG(NOTICE, sc, "Can't stat firmware file");
 		close(f);
 		return;
 	}
 
 	sc->firmware = rte_zmalloc("bnx2x_fw", st.st_size, RTE_CACHE_LINE_SIZE);
 	if (!sc->firmware) {
-		PMD_DRV_LOG(NOTICE, "Can't allocate memory for firmware");
+		PMD_DRV_LOG(NOTICE, sc, "Can't allocate memory for firmware");
 		close(f);
 		return;
 	}
 
 	if (read(f, sc->firmware, st.st_size) != st.st_size) {
-		PMD_DRV_LOG(NOTICE, "Can't read firmware data");
+		PMD_DRV_LOG(NOTICE, sc, "Can't read firmware data");
 		close(f);
 		return;
 	}
@@ -9542,10 +9602,11 @@ void bnx2x_load_firmware(struct bnx2x_softc *sc)
 
 	sc->fw_len = st.st_size;
 	if (sc->fw_len < FW_HEADER_LEN) {
-		PMD_DRV_LOG(NOTICE, "Invalid fw size: %" PRIu64, sc->fw_len);
+		PMD_DRV_LOG(NOTICE, sc,
+			    "Invalid fw size: %" PRIu64, sc->fw_len);
 		return;
 	}
-	PMD_DRV_LOG(DEBUG, "fw_len = %" PRIu64, sc->fw_len);
+	PMD_DRV_LOG(DEBUG, sc, "fw_len = %" PRIu64, sc->fw_len);
 }
 
 static void
@@ -9612,11 +9673,11 @@ int bnx2x_attach(struct bnx2x_softc *sc)
 {
 	int rc;
 
-	PMD_DRV_LOG(DEBUG, "Starting attach...");
+	PMD_DRV_LOG(DEBUG, sc, "Starting attach...");
 
 	rc = bnx2x_pci_get_caps(sc);
 	if (rc) {
-		PMD_DRV_LOG(NOTICE, "PCIe caps reading was failed");
+		PMD_DRV_LOG(NOTICE, sc, "PCIe caps reading was failed");
 		return rc;
 	}
 
@@ -9655,7 +9716,7 @@ int bnx2x_attach(struct bnx2x_softc *sc)
 
 		/* get device info and set params */
 		if (bnx2x_get_device_info(sc) != 0) {
-			PMD_DRV_LOG(NOTICE, "getting device info");
+			PMD_DRV_LOG(NOTICE, sc, "getting device info");
 			return -ENXIO;
 		}
 
@@ -9754,7 +9815,7 @@ bnx2x_igu_clear_sb_gen(struct bnx2x_softc *sc, uint8_t func, uint8_t idu_sb_id,
 
 	mb();
 
-	PMD_DRV_LOG(DEBUG, "write 0x%08x to IGU(via GRC) addr 0x%x",
+	PMD_DRV_LOG(DEBUG, sc, "write 0x%08x to IGU(via GRC) addr 0x%x",
 		    ctl, igu_addr_ctl);
 	REG_WR(sc, igu_addr_ctl, ctl);
 
@@ -9766,7 +9827,7 @@ bnx2x_igu_clear_sb_gen(struct bnx2x_softc *sc, uint8_t func, uint8_t idu_sb_id,
 	}
 
 	if (!(REG_RD(sc, igu_addr_ack) & sb_bit)) {
-		PMD_DRV_LOG(DEBUG,
+		PMD_DRV_LOG(DEBUG, sc,
 			    "Unable to finish IGU cleanup: "
 			    "idu_sb_id %d offset %d bit %d (cnt %d)",
 			    idu_sb_id, idu_sb_id / 32, idu_sb_id % 32, cnt);
@@ -9786,7 +9847,7 @@ static void bnx2x_reset_common(struct bnx2x_softc *sc)
 {
 	uint32_t val = 0x1400;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	/* reset_common */
 	REG_WR(sc, (GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_CLEAR),
@@ -9820,8 +9881,10 @@ static void bnx2x_common_init_phy(struct bnx2x_softc *sc)
 		shmem2_base[1] = SHMEM2_RD(sc, other_shmem2_base_addr);
 	}
 
+	bnx2x_acquire_phy_lock(sc);
 	elink_common_init_phy(sc, shmem_base, shmem2_base,
 			      sc->devinfo.chip_id, 0);
+	bnx2x_release_phy_lock(sc);
 }
 
 static void bnx2x_pf_disable(struct bnx2x_softc *sc)
@@ -9995,7 +10058,8 @@ static int bnx2x_init_hw_common(struct bnx2x_softc *sc)
 	uint8_t abs_func_id;
 	uint32_t val;
 
-	PMD_DRV_LOG(DEBUG, "starting common init for func %d", SC_ABS_FUNC(sc));
+	PMD_DRV_LOG(DEBUG, sc,
+		    "starting common init for func %d", SC_ABS_FUNC(sc));
 
 	/*
 	 * take the RESET lock to protect undi_unload flow from accessing
@@ -10078,12 +10142,12 @@ static int bnx2x_init_hw_common(struct bnx2x_softc *sc)
 
 	val = REG_RD(sc, PXP2_REG_RQ_CFG_DONE);
 	if (val != 1) {
-		PMD_DRV_LOG(NOTICE, "PXP2 CFG failed");
+		PMD_DRV_LOG(NOTICE, sc, "PXP2 CFG failed");
 		return -1;
 	}
 	val = REG_RD(sc, PXP2_REG_RD_INIT_DONE);
 	if (val != 1) {
-		PMD_DRV_LOG(NOTICE, "PXP2 RD_INIT failed");
+		PMD_DRV_LOG(NOTICE, sc, "PXP2 RD_INIT failed");
 		return -1;
 	}
 
@@ -10205,7 +10269,7 @@ static int bnx2x_init_hw_common(struct bnx2x_softc *sc)
 		} while (factor-- && (val != 1));
 
 		if (val != 1) {
-			PMD_DRV_LOG(NOTICE, "ATC_INIT failed");
+			PMD_DRV_LOG(NOTICE, sc, "ATC_INIT failed");
 			return -1;
 		}
 	}
@@ -10343,7 +10407,7 @@ static int bnx2x_init_hw_common(struct bnx2x_softc *sc)
 
 	if (sizeof(union cdu_context) != 1024) {
 /* we currently assume that a context is 1024 bytes */
-		PMD_DRV_LOG(NOTICE,
+		PMD_DRV_LOG(NOTICE, sc,
 			    "please adjust the size of cdu_context(%ld)",
 			    (long)sizeof(union cdu_context));
 	}
@@ -10405,17 +10469,17 @@ static int bnx2x_init_hw_common(struct bnx2x_softc *sc)
 	/* finish CFC init */
 	val = reg_poll(sc, CFC_REG_LL_INIT_DONE, 1, 100, 10);
 	if (val != 1) {
-		PMD_DRV_LOG(NOTICE, "CFC LL_INIT failed");
+		PMD_DRV_LOG(NOTICE, sc, "CFC LL_INIT failed");
 		return -1;
 	}
 	val = reg_poll(sc, CFC_REG_AC_INIT_DONE, 1, 100, 10);
 	if (val != 1) {
-		PMD_DRV_LOG(NOTICE, "CFC AC_INIT failed");
+		PMD_DRV_LOG(NOTICE, sc, "CFC AC_INIT failed");
 		return -1;
 	}
 	val = reg_poll(sc, CFC_REG_CAM_INIT_DONE, 1, 100, 10);
 	if (val != 1) {
-		PMD_DRV_LOG(NOTICE, "CFC CAM_INIT failed");
+		PMD_DRV_LOG(NOTICE, sc, "CFC CAM_INIT failed");
 		return -1;
 	}
 	REG_WR(sc, CFC_REG_DEBUG0, 0);
@@ -10468,7 +10532,7 @@ static int bnx2x_init_hw_port(struct bnx2x_softc *sc)
 	uint32_t low, high;
 	uint32_t val;
 
-	PMD_DRV_LOG(DEBUG, "starting port init for port %d", port);
+	PMD_DRV_LOG(DEBUG, sc, "starting port init for port %d", port);
 
 	REG_WR(sc, NIG_REG_MASK_INTERRUPT_PORT0 + port * 4, 0);
 
@@ -10695,7 +10759,7 @@ bnx2x_flr_clnup_poll_hw_counter(struct bnx2x_softc *sc, uint32_t reg,
 	uint32_t val = bnx2x_flr_clnup_reg_poll(sc, reg, 0, poll_cnt);
 
 	if (val != 0) {
-		PMD_DRV_LOG(NOTICE, "%s usage count=%d", msg, val);
+		PMD_DRV_LOG(NOTICE, sc, "%s usage count=%d", msg, val);
 		return -1;
 	}
 
@@ -10787,7 +10851,7 @@ bnx2x_send_final_clnup(struct bnx2x_softc *sc, uint8_t clnup_func,
 	int ret = 0;
 
 	if (REG_RD(sc, comp_addr)) {
-		PMD_DRV_LOG(NOTICE,
+		PMD_DRV_LOG(NOTICE, sc,
 			    "Cleanup complete was not 0 before sending");
 		return -1;
 	}
@@ -10800,8 +10864,8 @@ bnx2x_send_final_clnup(struct bnx2x_softc *sc, uint8_t clnup_func,
 	REG_WR(sc, XSDM_REG_OPERATION_GEN, op_gen_command);
 
 	if (bnx2x_flr_clnup_reg_poll(sc, comp_addr, 1, poll_cnt) != 1) {
-		PMD_DRV_LOG(NOTICE, "FW final cleanup did not succeed");
-		PMD_DRV_LOG(DEBUG, "At timeout completion address contained %x",
+		PMD_DRV_LOG(NOTICE, sc, "FW final cleanup did not succeed");
+		PMD_DRV_LOG(DEBUG, sc, "At timeout completion address contained %x",
 			    (REG_RD(sc, comp_addr)));
 		rte_panic("FLR cleanup failed");
 		return -1;
@@ -10917,28 +10981,30 @@ static void bnx2x_hw_enable_status(struct bnx2x_softc *sc)
 	__rte_unused uint32_t val;
 
 	val = REG_RD(sc, CFC_REG_WEAK_ENABLE_PF);
-	PMD_DRV_LOG(DEBUG, "CFC_REG_WEAK_ENABLE_PF is 0x%x", val);
+	PMD_DRV_LOG(DEBUG, sc, "CFC_REG_WEAK_ENABLE_PF is 0x%x", val);
 
 	val = REG_RD(sc, PBF_REG_DISABLE_PF);
-	PMD_DRV_LOG(DEBUG, "PBF_REG_DISABLE_PF is 0x%x", val);
+	PMD_DRV_LOG(DEBUG, sc, "PBF_REG_DISABLE_PF is 0x%x", val);
 
 	val = REG_RD(sc, IGU_REG_PCI_PF_MSI_EN);
-	PMD_DRV_LOG(DEBUG, "IGU_REG_PCI_PF_MSI_EN is 0x%x", val);
+	PMD_DRV_LOG(DEBUG, sc, "IGU_REG_PCI_PF_MSI_EN is 0x%x", val);
 
 	val = REG_RD(sc, IGU_REG_PCI_PF_MSIX_EN);
-	PMD_DRV_LOG(DEBUG, "IGU_REG_PCI_PF_MSIX_EN is 0x%x", val);
+	PMD_DRV_LOG(DEBUG, sc, "IGU_REG_PCI_PF_MSIX_EN is 0x%x", val);
 
 	val = REG_RD(sc, IGU_REG_PCI_PF_MSIX_FUNC_MASK);
-	PMD_DRV_LOG(DEBUG, "IGU_REG_PCI_PF_MSIX_FUNC_MASK is 0x%x", val);
+	PMD_DRV_LOG(DEBUG, sc, "IGU_REG_PCI_PF_MSIX_FUNC_MASK is 0x%x", val);
 
 	val = REG_RD(sc, PGLUE_B_REG_SHADOW_BME_PF_7_0_CLR);
-	PMD_DRV_LOG(DEBUG, "PGLUE_B_REG_SHADOW_BME_PF_7_0_CLR is 0x%x", val);
+	PMD_DRV_LOG(DEBUG, sc,
+		    "PGLUE_B_REG_SHADOW_BME_PF_7_0_CLR is 0x%x", val);
 
 	val = REG_RD(sc, PGLUE_B_REG_FLR_REQUEST_PF_7_0_CLR);
-	PMD_DRV_LOG(DEBUG, "PGLUE_B_REG_FLR_REQUEST_PF_7_0_CLR is 0x%x", val);
+	PMD_DRV_LOG(DEBUG, sc,
+		    "PGLUE_B_REG_FLR_REQUEST_PF_7_0_CLR is 0x%x", val);
 
 	val = REG_RD(sc, PGLUE_B_REG_INTERNAL_PFID_ENABLE_MASTER);
-	PMD_DRV_LOG(DEBUG, "PGLUE_B_REG_INTERNAL_PFID_ENABLE_MASTER is 0x%x",
+	PMD_DRV_LOG(DEBUG, sc, "PGLUE_B_REG_INTERNAL_PFID_ENABLE_MASTER is 0x%x",
 		    val);
 }
 
@@ -10982,7 +11048,7 @@ static int bnx2x_pf_flr_clnup(struct bnx2x_softc *sc)
 
 	/* Verify no pending pci transactions */
 	if (bnx2x_is_pcie_pending(sc)) {
-		PMD_DRV_LOG(NOTICE, "PCIE Transactions still pending");
+		PMD_DRV_LOG(NOTICE, sc, "PCIE Transactions still pending");
 	}
 
 	/* Debug */
@@ -11009,13 +11075,13 @@ static int bnx2x_init_hw_func(struct bnx2x_softc *sc)
 	int main_mem_width, rc;
 	uint32_t i;
 
-	PMD_DRV_LOG(DEBUG, "starting func init for func %d", func);
+	PMD_DRV_LOG(DEBUG, sc, "starting func init for func %d", func);
 
 	/* FLR cleanup */
 	if (!CHIP_IS_E1x(sc)) {
 		rc = bnx2x_pf_flr_clnup(sc);
 		if (rc) {
-			PMD_DRV_LOG(NOTICE, "FLR cleanup failed!");
+			PMD_DRV_LOG(NOTICE, sc, "FLR cleanup failed!");
 			return rc;
 		}
 	}
@@ -11262,7 +11328,7 @@ static int bnx2x_init_hw_func(struct bnx2x_softc *sc)
 
 		val = REG_RD(sc, main_mem_prty_clr);
 		if (val) {
-			PMD_DRV_LOG(DEBUG,
+			PMD_DRV_LOG(DEBUG, sc,
 				    "Parity errors in HC block during function init (0x%x)!",
 				    val);
 		}
@@ -11297,10 +11363,12 @@ static int bnx2x_init_hw_func(struct bnx2x_softc *sc)
 static void bnx2x_link_reset(struct bnx2x_softc *sc)
 {
 	if (!BNX2X_NOMCP(sc)) {
+		bnx2x_acquire_phy_lock(sc);
 		elink_lfa_reset(&sc->link_params, &sc->link_vars);
+		bnx2x_release_phy_lock(sc);
 	} else {
 		if (!CHIP_REV_IS_SLOW(sc)) {
-			PMD_DRV_LOG(WARNING,
+			PMD_DRV_LOG(WARNING, sc,
 				    "Bootcode is missing - cannot reset link");
 		}
 	}
@@ -11330,7 +11398,7 @@ static void bnx2x_reset_port(struct bnx2x_softc *sc)
 	/* Check for BRB port occupancy */
 	val = REG_RD(sc, BRB1_REG_PORT_NUM_OCC_BLOCKS_0 + port * 4);
 	if (val) {
-		PMD_DRV_LOG(DEBUG,
+		PMD_DRV_LOG(DEBUG, sc,
 			    "BRB1 is not empty, %d blocks are occupied", val);
 	}
 }
@@ -11524,10 +11592,10 @@ static int ecore_gunzip(struct bnx2x_softc *sc, const uint8_t * zbuf, int len)
 	int ret;
 	int data_begin = cut_gzip_prefix(zbuf, len);
 
-	PMD_DRV_LOG(DEBUG, "ecore_gunzip %d", len);
+	PMD_DRV_LOG(DEBUG, sc, "ecore_gunzip %d", len);
 
 	if (data_begin <= 0) {
-		PMD_DRV_LOG(NOTICE, "bad gzip prefix");
+		PMD_DRV_LOG(NOTICE, sc, "bad gzip prefix");
 		return -1;
 	}
 
@@ -11539,19 +11607,19 @@ static int ecore_gunzip(struct bnx2x_softc *sc, const uint8_t * zbuf, int len)
 
 	ret = inflateInit2(&zlib_stream, -MAX_WBITS);
 	if (ret != Z_OK) {
-		PMD_DRV_LOG(NOTICE, "zlib inflateInit2 error");
+		PMD_DRV_LOG(NOTICE, sc, "zlib inflateInit2 error");
 		return ret;
 	}
 
 	ret = inflate(&zlib_stream, Z_FINISH);
 	if ((ret != Z_STREAM_END) && (ret != Z_OK)) {
-		PMD_DRV_LOG(NOTICE, "zlib inflate error: %d %s", ret,
+		PMD_DRV_LOG(NOTICE, sc, "zlib inflate error: %d %s", ret,
 			    zlib_stream.msg);
 	}
 
 	sc->gz_outlen = zlib_stream.total_out;
 	if (sc->gz_outlen & 0x3) {
-		PMD_DRV_LOG(NOTICE, "firmware is not aligned. gz_outlen == %d",
+		PMD_DRV_LOG(NOTICE, sc, "firmware is not aligned. gz_outlen == %d",
 			    sc->gz_outlen);
 	}
 	sc->gz_outlen >>= 2;
@@ -11670,7 +11738,7 @@ void bnx2x_print_adapter_info(struct bnx2x_softc *sc)
 	int i = 0;
 	__rte_unused uint32_t ext_phy_type;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 	if (sc->link_vars.phy_flags & PHY_XGXS_FLAG)
 		ext_phy_type = ELINK_XGXS_EXT_PHY_TYPE(REG_RD(sc,
 							      sc->
@@ -11689,97 +11757,102 @@ void bnx2x_print_adapter_info(struct bnx2x_softc *sc)
 									 dev_info.port_hw_config
 									 [0].external_phy_config)));
 
-	PMD_INIT_LOG(DEBUG, "\n\n===================================\n");
+	PMD_DRV_LOG(INFO, sc, "\n\n===================================\n");
 	/* Hardware chip info. */
-	PMD_INIT_LOG(DEBUG, "%12s : %#08x", "ASIC", sc->devinfo.chip_id);
-	PMD_INIT_LOG(DEBUG, "%12s : %c%d", "Rev", (CHIP_REV(sc) >> 12) + 'A',
+	PMD_DRV_LOG(INFO, sc, "%12s : %#08x", "ASIC", sc->devinfo.chip_id);
+	PMD_DRV_LOG(INFO, sc, "%12s : %c%d", "Rev", (CHIP_REV(sc) >> 12) + 'A',
 		     (CHIP_METAL(sc) >> 4));
 
 	/* Bus info. */
-	PMD_INIT_LOG(DEBUG, "%12s : %d, ", "Bus PCIe", sc->devinfo.pcie_link_width);
+	PMD_DRV_LOG(INFO, sc,
+		    "%12s : %d, ", "Bus PCIe", sc->devinfo.pcie_link_width);
 	switch (sc->devinfo.pcie_link_speed) {
 	case 1:
-		PMD_INIT_LOG(DEBUG, "%23s", "2.5 Gbps");
+		PMD_DRV_LOG(INFO, sc, "%23s", "2.5 Gbps");
 		break;
 	case 2:
-		PMD_INIT_LOG(DEBUG, "%21s", "5 Gbps");
+		PMD_DRV_LOG(INFO, sc, "%21s", "5 Gbps");
 		break;
 	case 4:
-		PMD_INIT_LOG(DEBUG, "%21s", "8 Gbps");
+		PMD_DRV_LOG(INFO, sc, "%21s", "8 Gbps");
 		break;
 	default:
-		PMD_INIT_LOG(DEBUG, "%33s", "Unknown link speed");
+		PMD_DRV_LOG(INFO, sc, "%33s", "Unknown link speed");
 	}
 
 	/* Device features. */
-	PMD_INIT_LOG(DEBUG, "%12s : ", "Flags");
+	PMD_DRV_LOG(INFO, sc, "%12s : ", "Flags");
 
 	/* Miscellaneous flags. */
 	if (sc->devinfo.pcie_cap_flags & BNX2X_MSI_CAPABLE_FLAG) {
-		PMD_INIT_LOG(DEBUG, "%18s", "MSI");
+		PMD_DRV_LOG(INFO, sc, "%18s", "MSI");
 		i++;
 	}
 
 	if (sc->devinfo.pcie_cap_flags & BNX2X_MSIX_CAPABLE_FLAG) {
 		if (i > 0)
-			PMD_INIT_LOG(DEBUG, "|");
-		PMD_INIT_LOG(DEBUG, "%20s", "MSI-X");
+			PMD_DRV_LOG(INFO, sc, "|");
+		PMD_DRV_LOG(INFO, sc, "%20s", "MSI-X");
 		i++;
 	}
 
 	if (IS_PF(sc)) {
-		PMD_INIT_LOG(DEBUG, "%12s : ", "Queues");
+		PMD_DRV_LOG(INFO, sc, "%12s : ", "Queues");
 		switch (sc->sp->rss_rdata.rss_mode) {
 		case ETH_RSS_MODE_DISABLED:
-			PMD_INIT_LOG(DEBUG, "%19s", "None");
+			PMD_DRV_LOG(INFO, sc, "%19s", "None");
 			break;
 		case ETH_RSS_MODE_REGULAR:
-			PMD_INIT_LOG(DEBUG, "%18s : %d", "RSS", sc->num_queues);
+			PMD_DRV_LOG(INFO, sc,
+				    "%18s : %d", "RSS", sc->num_queues);
 			break;
 		default:
-			PMD_INIT_LOG(DEBUG, "%22s", "Unknown");
+			PMD_DRV_LOG(INFO, sc, "%22s", "Unknown");
 			break;
 		}
 	}
 
 	/* RTE and Driver versions */
-	PMD_INIT_LOG(DEBUG, "%12s : %s", "DPDK",
-		     rte_version());
-	PMD_INIT_LOG(DEBUG, "%12s : %s", "Driver",
-		     bnx2x_pmd_version());
+	PMD_DRV_LOG(INFO, sc, "%12s : %s", "DPDK",
+			rte_version());
+	PMD_DRV_LOG(INFO, sc, "%12s : %s", "Driver",
+			bnx2x_pmd_version());
 
 	/* Firmware versions and device features. */
-	PMD_INIT_LOG(DEBUG, "%12s : %d.%d.%d",
+	PMD_DRV_LOG(INFO, sc, "%12s : %d.%d.%d",
 		     "Firmware",
 		     BNX2X_5710_FW_MAJOR_VERSION,
 		     BNX2X_5710_FW_MINOR_VERSION,
 		     BNX2X_5710_FW_REVISION_VERSION);
-	PMD_INIT_LOG(DEBUG, "%12s : %s",
+	PMD_DRV_LOG(INFO, sc, "%12s : %s",
 		     "Bootcode", sc->devinfo.bc_ver_str);
 
-	PMD_INIT_LOG(DEBUG, "\n\n===================================\n");
-	PMD_INIT_LOG(DEBUG, "%12s : %u", "Bnx2x Func", sc->pcie_func);
-	PMD_INIT_LOG(DEBUG, "%12s : %s", "Bnx2x Flags", get_bnx2x_flags(sc->flags));
-	PMD_INIT_LOG(DEBUG, "%12s : %s", "DMAE Is",
+	PMD_DRV_LOG(INFO, sc, "\n\n===================================\n");
+	PMD_DRV_LOG(INFO, sc, "%12s : %u", "Bnx2x Func", sc->pcie_func);
+	PMD_DRV_LOG(INFO, sc,
+		    "%12s : %s", "Bnx2x Flags", get_bnx2x_flags(sc->flags));
+	PMD_DRV_LOG(INFO, sc, "%12s : %s", "DMAE Is",
 		     (sc->dmae_ready ? "Ready" : "Not Ready"));
-	PMD_INIT_LOG(DEBUG, "%12s : %s", "OVLAN", (OVLAN(sc) ? "YES" : "NO"));
-	PMD_INIT_LOG(DEBUG, "%12s : %s", "MF", (IS_MF(sc) ? "YES" : "NO"));
-	PMD_INIT_LOG(DEBUG, "%12s : %u", "MTU", sc->mtu);
-	PMD_INIT_LOG(DEBUG, "%12s : %s", "PHY Type", get_ext_phy_type(ext_phy_type));
-	PMD_INIT_LOG(DEBUG, "%12s : %x:%x:%x:%x:%x:%x", "MAC Addr",
+	PMD_DRV_LOG(INFO, sc, "%12s : %s", "OVLAN", (OVLAN(sc) ? "YES" : "NO"));
+	PMD_DRV_LOG(INFO, sc, "%12s : %s", "MF", (IS_MF(sc) ? "YES" : "NO"));
+	PMD_DRV_LOG(INFO, sc, "%12s : %u", "MTU", sc->mtu);
+	PMD_DRV_LOG(INFO, sc,
+		    "%12s : %s", "PHY Type", get_ext_phy_type(ext_phy_type));
+	PMD_DRV_LOG(INFO, sc, "%12s : %x:%x:%x:%x:%x:%x", "MAC Addr",
 			sc->link_params.mac_addr[0],
 			sc->link_params.mac_addr[1],
 			sc->link_params.mac_addr[2],
 			sc->link_params.mac_addr[3],
 			sc->link_params.mac_addr[4],
 			sc->link_params.mac_addr[5]);
-	PMD_INIT_LOG(DEBUG, "%12s : %s", "RX Mode", get_rx_mode(sc->rx_mode));
-	PMD_INIT_LOG(DEBUG, "%12s : %s", "State", get_state(sc->state));
+	PMD_DRV_LOG(INFO, sc, "%12s : %s", "RX Mode", get_rx_mode(sc->rx_mode));
+	PMD_DRV_LOG(INFO, sc, "%12s : %s", "State", get_state(sc->state));
 	if (sc->recovery_state)
-		PMD_INIT_LOG(DEBUG, "%12s : %s", "Recovery",
+		PMD_DRV_LOG(INFO, sc, "%12s : %s", "Recovery",
 			     get_recovery_state(sc->recovery_state));
-	PMD_INIT_LOG(DEBUG, "%12s : CQ = %lx,  EQ = %lx", "SPQ Left",
+	PMD_DRV_LOG(INFO, sc, "%12s : CQ = %lx,  EQ = %lx", "SPQ Left",
 		     sc->cq_spq_left, sc->eq_spq_left);
-	PMD_INIT_LOG(DEBUG, "%12s : %x", "Switch", sc->link_params.switch_cfg);
-	PMD_INIT_LOG(DEBUG, "\n\n===================================\n");
+	PMD_DRV_LOG(INFO, sc,
+		    "%12s : %x", "Switch", sc->link_params.switch_cfg);
+	PMD_DRV_LOG(INFO, sc, "\n\n===================================\n");
 }
diff --git a/drivers/net/bnx2x/bnx2x.h b/drivers/net/bnx2x/bnx2x.h
index 0f6024fb..74780725 100644
--- a/drivers/net/bnx2x/bnx2x.h
+++ b/drivers/net/bnx2x/bnx2x.h
@@ -725,6 +725,13 @@ struct bnx2x_port {
 
     uint32_t    phy_addr;
 
+	/* Used to synchronize phy accesses. */
+	rte_spinlock_t	phy_mtx;
+	char		phy_mtx_name[32];
+
+#define BNX2X_PHY_LOCK(sc)          rte_spinlock_lock(&sc->port.phy_mtx)
+#define BNX2X_PHY_UNLOCK(sc)        rte_spinlock_unlock(&sc->port.phy_mtx)
+
     /*
      * MCP scratchpad address for port specific statistics.
      * The device is responsible for writing statistcss
@@ -803,6 +810,10 @@ struct bnx2x_mf_info {
 
 /* Device information data structure. */
 struct bnx2x_devinfo {
+#if 1
+#define NAME_SIZE 128
+	char name[NAME_SIZE];
+#endif
 	/* PCIe info */
 	uint16_t vendor_id;
 	uint16_t device_id;
@@ -820,6 +831,7 @@ struct bnx2x_devinfo {
 #define CHIP_ID(sc)           ((sc)->devinfo.chip_id & 0xffff0000)
 #define CHIP_NUM(sc)          ((sc)->devinfo.chip_id >> 16)
 /* device ids */
+#define CHIP_NUM_57710        0x164e
 #define CHIP_NUM_57711        0x164f
 #define CHIP_NUM_57711E       0x1650
 #define CHIP_NUM_57712        0x1662
@@ -861,6 +873,8 @@ struct bnx2x_devinfo {
 #define CHIP_METAL(sc)      ((sc->devinfo.chip_id) & 0x00000ff0)
 #define CHIP_BOND_ID(sc)    ((sc->devinfo.chip_id) & 0x0000000f)
 
+#define CHIP_IS_E1(sc)      (CHIP_NUM(sc) == CHIP_NUM_57710)
+#define CHIP_IS_57710(sc)   (CHIP_NUM(sc) == CHIP_NUM_57710)
 #define CHIP_IS_57711(sc)   (CHIP_NUM(sc) == CHIP_NUM_57711)
 #define CHIP_IS_57711E(sc)  (CHIP_NUM(sc) == CHIP_NUM_57711E)
 #define CHIP_IS_E1H(sc)     ((CHIP_IS_57711(sc)) || \
@@ -1418,7 +1432,7 @@ struct bnx2x_func_init_params {
 static inline void
 bnx2x_reg_write8(struct bnx2x_softc *sc, size_t offset, uint8_t val)
 {
-	PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%02x",
+	PMD_DEBUG_PERIODIC_LOG(DEBUG, sc, "offset=0x%08lx val=0x%02x",
 			       (unsigned long)offset, val);
 	rte_write8(val, ((uint8_t *)sc->bar[BAR0].base_addr + offset));
 }
@@ -1428,10 +1442,10 @@ bnx2x_reg_write16(struct bnx2x_softc *sc, size_t offset, uint16_t val)
 {
 #ifdef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC
 	if ((offset % 2) != 0)
-		PMD_DRV_LOG(NOTICE, "Unaligned 16-bit write to 0x%08lx",
+		PMD_DRV_LOG(NOTICE, sc, "Unaligned 16-bit write to 0x%08lx",
 			    (unsigned long)offset);
 #endif
-	PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%04x",
+	PMD_DEBUG_PERIODIC_LOG(DEBUG, sc, "offset=0x%08lx val=0x%04x",
 			       (unsigned long)offset, val);
 	rte_write16(val, ((uint8_t *)sc->bar[BAR0].base_addr + offset));
 
@@ -1442,11 +1456,11 @@ bnx2x_reg_write32(struct bnx2x_softc *sc, size_t offset, uint32_t val)
 {
 #ifdef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC
 	if ((offset % 4) != 0)
-		PMD_DRV_LOG(NOTICE, "Unaligned 32-bit write to 0x%08lx",
+		PMD_DRV_LOG(NOTICE, sc, "Unaligned 32-bit write to 0x%08lx",
 			    (unsigned long)offset);
 #endif
 
-	PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%08x",
+	PMD_DEBUG_PERIODIC_LOG(DEBUG, sc, "offset=0x%08lx val=0x%08x",
 			       (unsigned long)offset, val);
 	rte_write32(val, ((uint8_t *)sc->bar[BAR0].base_addr + offset));
 }
@@ -1457,7 +1471,7 @@ bnx2x_reg_read8(struct bnx2x_softc *sc, size_t offset)
 	uint8_t val;
 
 	val = rte_read8((uint8_t *)sc->bar[BAR0].base_addr + offset);
-	PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%02x",
+	PMD_DEBUG_PERIODIC_LOG(DEBUG, sc, "offset=0x%08lx val=0x%02x",
 			       (unsigned long)offset, val);
 
 	return val;
@@ -1470,12 +1484,12 @@ bnx2x_reg_read16(struct bnx2x_softc *sc, size_t offset)
 
 #ifdef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC
 	if ((offset % 2) != 0)
-		PMD_DRV_LOG(NOTICE, "Unaligned 16-bit read from 0x%08lx",
+		PMD_DRV_LOG(NOTICE, sc, "Unaligned 16-bit read from 0x%08lx",
 			    (unsigned long)offset);
 #endif
 
 	val = rte_read16(((uint8_t *)sc->bar[BAR0].base_addr + offset));
-	PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%08x",
+	PMD_DEBUG_PERIODIC_LOG(DEBUG, sc, "offset=0x%08lx val=0x%08x",
 			       (unsigned long)offset, val);
 
 	return val;
@@ -1488,12 +1502,12 @@ bnx2x_reg_read32(struct bnx2x_softc *sc, size_t offset)
 
 #ifdef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC
 	if ((offset % 4) != 0)
-		PMD_DRV_LOG(NOTICE, "Unaligned 32-bit read from 0x%08lx",
+		PMD_DRV_LOG(NOTICE, sc, "Unaligned 32-bit read from 0x%08lx",
 			    (unsigned long)offset);
 #endif
 
 	val = rte_read32(((uint8_t *)sc->bar[BAR0].base_addr + offset));
-	PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%08x",
+	PMD_DEBUG_PERIODIC_LOG(DEBUG, sc, "offset=0x%08lx val=0x%08x",
 			       (unsigned long)offset, val);
 
 	return val;
@@ -1970,7 +1984,7 @@ bnx2x_set_rx_mode(struct bnx2x_softc *sc)
 			bnx2x_vf_set_rx_mode(sc);
 		}
 	} else {
-		PMD_DRV_LOG(NOTICE, "Card is not ready to change mode");
+		PMD_DRV_LOG(NOTICE, sc, "Card is not ready to change mode");
 	}
 }
 
@@ -1978,7 +1992,7 @@ static inline int pci_read(struct bnx2x_softc *sc, size_t addr,
 			   void *val, uint8_t size)
 {
 	if (rte_pci_read_config(sc->pci_dev, val, size, addr) <= 0) {
-		PMD_DRV_LOG(ERR, "Can't read from PCI config space");
+		PMD_DRV_LOG(ERR, sc, "Can't read from PCI config space");
 		return ENXIO;
 	}
 
@@ -1991,7 +2005,7 @@ static inline int pci_write_word(struct bnx2x_softc *sc, size_t addr, off_t val)
 
 	if (rte_pci_write_config(sc->pci_dev, &val16,
 				     sizeof(val16), addr) <= 0) {
-		PMD_DRV_LOG(ERR, "Can't write to PCI config space");
+		PMD_DRV_LOG(ERR, sc, "Can't write to PCI config space");
 		return ENXIO;
 	}
 
@@ -2003,7 +2017,7 @@ static inline int pci_write_long(struct bnx2x_softc *sc, size_t addr, off_t val)
 	uint32_t val32 = val;
 	if (rte_pci_write_config(sc->pci_dev, &val32,
 				     sizeof(val32), addr) <= 0) {
-		PMD_DRV_LOG(ERR, "Can't write to PCI config space");
+		PMD_DRV_LOG(ERR, sc, "Can't write to PCI config space");
 		return ENXIO;
 	}
 
diff --git a/drivers/net/bnx2x/bnx2x_ethdev.c b/drivers/net/bnx2x/bnx2x_ethdev.c
index 575271a8..0057843b 100644
--- a/drivers/net/bnx2x/bnx2x_ethdev.c
+++ b/drivers/net/bnx2x/bnx2x_ethdev.c
@@ -85,7 +85,7 @@ bnx2x_link_update(struct rte_eth_dev *dev)
 	struct bnx2x_softc *sc = dev->data->dev_private;
 	struct rte_eth_link link;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	bnx2x_link_status_update(sc);
 	memset(&link, 0, sizeof(link));
@@ -129,9 +129,11 @@ bnx2x_interrupt_handler(void *param)
 	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 	struct bnx2x_softc *sc = dev->data->dev_private;
 
-	PMD_DEBUG_PERIODIC_LOG(INFO, "Interrupt handled");
+	PMD_DEBUG_PERIODIC_LOG(INFO, sc, "Interrupt handled");
 
+	atomic_store_rel_long(&sc->periodic_flags, PERIODIC_STOP);
 	bnx2x_interrupt_action(dev);
+	atomic_store_rel_long(&sc->periodic_flags, PERIODIC_GO);
 	rte_intr_enable(&sc->pci_dev->intr_handle);
 }
 
@@ -147,8 +149,8 @@ static void bnx2x_periodic_start(void *param)
 		ret = rte_eal_alarm_set(BNX2X_SP_TIMER_PERIOD,
 					bnx2x_periodic_start, (void *)dev);
 		if (ret) {
-			PMD_DRV_LOG(ERR, "Unable to start periodic"
-					 " timer rc %d", ret);
+			PMD_DRV_LOG(ERR, sc, "Unable to start periodic"
+					     " timer rc %d", ret);
 			assert(false && "Unable to start periodic timer");
 		}
 	}
@@ -176,34 +178,34 @@ bnx2x_dev_configure(struct rte_eth_dev *dev)
 
 	int mp_ncpus = sysconf(_SC_NPROCESSORS_CONF);
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	if (rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 		sc->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 
 	if (dev->data->nb_tx_queues > dev->data->nb_rx_queues) {
-		PMD_DRV_LOG(ERR, "The number of TX queues is greater than number of RX queues");
+		PMD_DRV_LOG(ERR, sc, "The number of TX queues is greater than number of RX queues");
 		return -EINVAL;
 	}
 
 	sc->num_queues = MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
 	if (sc->num_queues > mp_ncpus) {
-		PMD_DRV_LOG(ERR, "The number of queues is more than number of CPUs");
+		PMD_DRV_LOG(ERR, sc, "The number of queues is more than number of CPUs");
 		return -EINVAL;
 	}
 
-	PMD_DRV_LOG(DEBUG, "num_queues=%d, mtu=%d",
+	PMD_DRV_LOG(DEBUG, sc, "num_queues=%d, mtu=%d",
 		       sc->num_queues, sc->mtu);
 
 	/* allocate ilt */
 	if (bnx2x_alloc_ilt_mem(sc) != 0) {
-		PMD_DRV_LOG(ERR, "bnx2x_alloc_ilt_mem was failed");
+		PMD_DRV_LOG(ERR, sc, "bnx2x_alloc_ilt_mem was failed");
 		return -ENXIO;
 	}
 
 	/* allocate the host hardware/software hsi structures */
 	if (bnx2x_alloc_hsi_mem(sc) != 0) {
-		PMD_DRV_LOG(ERR, "bnx2x_alloc_hsi_mem was failed");
+		PMD_DRV_LOG(ERR, sc, "bnx2x_alloc_hsi_mem was failed");
 		bnx2x_free_ilt_mem(sc);
 		return -ENXIO;
 	}
@@ -217,7 +219,7 @@ bnx2x_dev_start(struct rte_eth_dev *dev)
 	struct bnx2x_softc *sc = dev->data->dev_private;
 	int ret = 0;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	/* start the periodic callout */
 	if (sc->periodic_flags & PERIODIC_STOP)
@@ -225,7 +227,7 @@ bnx2x_dev_start(struct rte_eth_dev *dev)
 
 	ret = bnx2x_init(sc);
 	if (ret) {
-		PMD_DRV_LOG(DEBUG, "bnx2x_init failed (%d)", ret);
+		PMD_DRV_LOG(DEBUG, sc, "bnx2x_init failed (%d)", ret);
 		return -1;
 	}
 
@@ -234,12 +236,12 @@ bnx2x_dev_start(struct rte_eth_dev *dev)
 				bnx2x_interrupt_handler, (void *)dev);
 
 		if (rte_intr_enable(&sc->pci_dev->intr_handle))
-			PMD_DRV_LOG(ERR, "rte_intr_enable failed");
+			PMD_DRV_LOG(ERR, sc, "rte_intr_enable failed");
 	}
 
 	ret = bnx2x_dev_rx_init(dev);
 	if (ret != 0) {
-		PMD_DRV_LOG(DEBUG, "bnx2x_dev_rx_init returned error code");
+		PMD_DRV_LOG(DEBUG, sc, "bnx2x_dev_rx_init returned error code");
 		return -3;
 	}
 
@@ -255,7 +257,7 @@ bnx2x_dev_stop(struct rte_eth_dev *dev)
 	struct bnx2x_softc *sc = dev->data->dev_private;
 	int ret = 0;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	if (IS_PF(sc)) {
 		rte_intr_disable(&sc->pci_dev->intr_handle);
@@ -268,7 +270,7 @@ bnx2x_dev_stop(struct rte_eth_dev *dev)
 
 	ret = bnx2x_nic_unload(sc, UNLOAD_NORMAL, FALSE);
 	if (ret) {
-		PMD_DRV_LOG(DEBUG, "bnx2x_nic_unload failed (%d)", ret);
+		PMD_DRV_LOG(DEBUG, sc, "bnx2x_nic_unload failed (%d)", ret);
 		return;
 	}
 
@@ -280,7 +282,7 @@ bnx2x_dev_close(struct rte_eth_dev *dev)
 {
 	struct bnx2x_softc *sc = dev->data->dev_private;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	if (IS_VF(sc))
 		bnx2x_vf_close(sc);
@@ -300,7 +302,7 @@ bnx2x_promisc_enable(struct rte_eth_dev *dev)
 {
 	struct bnx2x_softc *sc = dev->data->dev_private;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 	sc->rx_mode = BNX2X_RX_MODE_PROMISC;
 	if (rte_eth_allmulticast_get(dev->data->port_id) == 1)
 		sc->rx_mode = BNX2X_RX_MODE_ALLMULTI_PROMISC;
@@ -312,7 +314,7 @@ bnx2x_promisc_disable(struct rte_eth_dev *dev)
 {
 	struct bnx2x_softc *sc = dev->data->dev_private;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 	sc->rx_mode = BNX2X_RX_MODE_NORMAL;
 	if (rte_eth_allmulticast_get(dev->data->port_id) == 1)
 		sc->rx_mode = BNX2X_RX_MODE_ALLMULTI;
@@ -324,7 +326,7 @@ bnx2x_dev_allmulticast_enable(struct rte_eth_dev *dev)
 {
 	struct bnx2x_softc *sc = dev->data->dev_private;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 	sc->rx_mode = BNX2X_RX_MODE_ALLMULTI;
 	if (rte_eth_promiscuous_get(dev->data->port_id) == 1)
 		sc->rx_mode = BNX2X_RX_MODE_ALLMULTI_PROMISC;
@@ -336,7 +338,7 @@ bnx2x_dev_allmulticast_disable(struct rte_eth_dev *dev)
 {
 	struct bnx2x_softc *sc = dev->data->dev_private;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 	sc->rx_mode = BNX2X_RX_MODE_NORMAL;
 	if (rte_eth_promiscuous_get(dev->data->port_id) == 1)
 		sc->rx_mode = BNX2X_RX_MODE_PROMISC;
@@ -346,7 +348,9 @@ bnx2x_dev_allmulticast_disable(struct rte_eth_dev *dev)
 static int
 bnx2x_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
 {
-	PMD_INIT_FUNC_TRACE();
+	struct bnx2x_softc *sc = dev->data->dev_private;
+
+	PMD_INIT_FUNC_TRACE(sc);
 
 	return bnx2x_link_update(dev);
 }
@@ -361,7 +365,7 @@ bnx2xvf_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_comple
 
 	bnx2x_check_bull(sc);
 	if (sc->old_bulletin.valid_bitmap & (1 << CHANNEL_DOWN)) {
-		PMD_DRV_LOG(ERR, "PF indicated channel is down."
+		PMD_DRV_LOG(ERR, sc, "PF indicated channel is down."
 				"VF device is no longer operational");
 		dev->data->dev_link.link_status = ETH_LINK_DOWN;
 	}
@@ -377,7 +381,7 @@ bnx2x_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 	uint64_t brb_drops;
 	uint64_t brb_truncates;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	bnx2x_stats_handle(sc, STATS_EVENT_UPDATE);
 
@@ -568,27 +572,35 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf)
 {
 	int ret = 0;
 	struct rte_pci_device *pci_dev;
+	struct rte_pci_addr pci_addr;
 	struct bnx2x_softc *sc;
 
-	PMD_INIT_FUNC_TRACE();
+	/* Extract key data structures */
+	sc = eth_dev->data->dev_private;
+	pci_dev = RTE_DEV_TO_PCI(eth_dev->device);
+	pci_addr = pci_dev->addr;
+
+	snprintf(sc->devinfo.name, NAME_SIZE, PCI_SHORT_PRI_FMT ":dpdk-port-%u",
+		 pci_addr.bus, pci_addr.devid, pci_addr.function,
+		 eth_dev->data->port_id);
+
+	PMD_INIT_FUNC_TRACE(sc);
 
 	eth_dev->dev_ops = is_vf ? &bnx2xvf_eth_dev_ops : &bnx2x_eth_dev_ops;
-	pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 
 	rte_eth_copy_pci_info(eth_dev, pci_dev);
 
-	sc = eth_dev->data->dev_private;
 	sc->pcie_bus    = pci_dev->addr.bus;
 	sc->pcie_device = pci_dev->addr.devid;
 
-	if (is_vf)
-		sc->flags = BNX2X_IS_VF_FLAG;
-
 	sc->devinfo.vendor_id    = pci_dev->id.vendor_id;
 	sc->devinfo.device_id    = pci_dev->id.device_id;
 	sc->devinfo.subvendor_id = pci_dev->id.subsystem_vendor_id;
 	sc->devinfo.subdevice_id = pci_dev->id.subsystem_device_id;
 
+	if (is_vf)
+		sc->flags = BNX2X_IS_VF_FLAG;
+
 	sc->pcie_func = pci_dev->addr.function;
 	sc->bar[BAR0].base_addr = (void *)pci_dev->mem_resource[0].addr;
 	if (is_vf)
@@ -616,7 +628,7 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf)
 	sc->pci_dev = pci_dev;
 	ret = bnx2x_attach(sc);
 	if (ret) {
-		PMD_DRV_LOG(ERR, "bnx2x_attach failed (%d)", ret);
+		PMD_DRV_LOG(ERR, sc, "bnx2x_attach failed (%d)", ret);
 		return ret;
 	}
 
@@ -625,21 +637,21 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf)
 		ret = rte_eal_alarm_set(BNX2X_SP_TIMER_PERIOD,
 					bnx2x_periodic_start, (void *)eth_dev);
 		if (ret) {
-			PMD_DRV_LOG(ERR, "Unable to start periodic"
-					  " timer rc %d", ret);
+			PMD_DRV_LOG(ERR, sc, "Unable to start periodic"
+					     " timer rc %d", ret);
 			return -EINVAL;
 		}
 	}
 
 	eth_dev->data->mac_addrs = (struct ether_addr *)sc->link_params.mac_addr;
 
-	PMD_DRV_LOG(INFO, "pcie_bus=%d, pcie_device=%d",
+	PMD_DRV_LOG(INFO, sc, "pcie_bus=%d, pcie_device=%d",
 			sc->pcie_bus, sc->pcie_device);
-	PMD_DRV_LOG(INFO, "bar0.addr=%p, bar1.addr=%p",
+	PMD_DRV_LOG(INFO, sc, "bar0.addr=%p, bar1.addr=%p",
 			sc->bar[BAR0].base_addr, sc->bar[BAR1].base_addr);
-	PMD_DRV_LOG(INFO, "port=%d, path=%d, vnic=%d, func=%d",
+	PMD_DRV_LOG(INFO, sc, "port=%d, path=%d, vnic=%d, func=%d",
 			PORT_ID(sc), PATH_ID(sc), VNIC_ID(sc), FUNC_ID(sc));
-	PMD_DRV_LOG(INFO, "portID=%d vendorID=0x%x deviceID=0x%x",
+	PMD_DRV_LOG(INFO, sc, "portID=%d vendorID=0x%x deviceID=0x%x",
 			eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id);
 
 	if (IS_VF(sc)) {
@@ -679,14 +691,16 @@ out:
 static int
 eth_bnx2x_dev_init(struct rte_eth_dev *eth_dev)
 {
-	PMD_INIT_FUNC_TRACE();
+	struct bnx2x_softc *sc = eth_dev->data->dev_private;
+	PMD_INIT_FUNC_TRACE(sc);
 	return bnx2x_common_dev_init(eth_dev, 0);
 }
 
 static int
 eth_bnx2xvf_dev_init(struct rte_eth_dev *eth_dev)
 {
-	PMD_INIT_FUNC_TRACE();
+	struct bnx2x_softc *sc = eth_dev->data->dev_private;
+	PMD_INIT_FUNC_TRACE(sc);
 	return bnx2x_common_dev_init(eth_dev, 1);
 }
 
diff --git a/drivers/net/bnx2x/bnx2x_logs.h b/drivers/net/bnx2x/bnx2x_logs.h
index 9e232a9b..753bccdf 100644
--- a/drivers/net/bnx2x/bnx2x_logs.h
+++ b/drivers/net/bnx2x/bnx2x_logs.h
@@ -9,11 +9,11 @@
 #define _PMD_LOGS_H_
 
 extern int bnx2x_logtype_init;
-#define PMD_INIT_LOG(level, fmt, args...) \
-	rte_log(RTE_LOG_ ## level, bnx2x_logtype_init, \
-		"%s(): " fmt "\n", __func__, ##args)
+#define PMD_INIT_LOG(level, sc, fmt, args...) \
+	RTE_LOG(level, PMD, \
+	"[bnx2x_pmd: %s] %s() " fmt "\n", (sc)->devinfo.name, __func__, ##args)
 
-#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
+#define PMD_INIT_FUNC_TRACE(sc) PMD_INIT_LOG(DEBUG, sc, " >>")
 
 #ifdef RTE_LIBRTE_BNX2X_DEBUG_RX
 #define PMD_RX_LOG(level, fmt, args...) \
@@ -37,18 +37,19 @@ extern int bnx2x_logtype_init;
 #endif
 
 extern int bnx2x_logtype_driver;
-#define PMD_DRV_LOG_RAW(level, fmt, args...) \
-	rte_log(RTE_LOG_ ## level, bnx2x_logtype_driver, \
-		"%s(): " fmt, __func__, ## args)
+#define PMD_DRV_LOG_RAW(level, sc, fmt, args...) \
+	RTE_LOG(level, PMD, "[%s:%d(%s)] " fmt,	__func__, __LINE__, \
+		(sc)->devinfo.name ? (sc)->devinfo.name : "", ## args)
 
-#define PMD_DRV_LOG(level, fmt, args...) \
-	PMD_DRV_LOG_RAW(level, fmt "\n", ## args)
+#define PMD_DRV_LOG(level, sc, fmt, args...) \
+	PMD_DRV_LOG_RAW(level, sc, fmt "\n", ## args)
 
 #ifdef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC
-#define PMD_DEBUG_PERIODIC_LOG(level, fmt, args...) \
-	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
+#define PMD_DEBUG_PERIODIC_LOG(level, sc, fmt, args...) \
+	RTE_LOG(level, PMD, "%s(%s): " fmt "\n", __func__, \
+		(sc)->devinfo.name ? (sc)->devinfo.name : "", ## args)
 #else
-#define PMD_DEBUG_PERIODIC_LOG(level, fmt, args...) do { } while(0)
+#define PMD_DEBUG_PERIODIC_LOG(level, sc, fmt, args...) do { } while (0)
 #endif
 
 #endif /* _PMD_LOGS_H_ */
diff --git a/drivers/net/bnx2x/bnx2x_rxtx.c b/drivers/net/bnx2x/bnx2x_rxtx.c
index d9a4127d..ca28aacc 100644
--- a/drivers/net/bnx2x/bnx2x_rxtx.c
+++ b/drivers/net/bnx2x/bnx2x_rxtx.c
@@ -12,19 +12,8 @@ static const struct rte_memzone *
 ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
 		      uint16_t queue_id, uint32_t ring_size, int socket_id)
 {
-	char z_name[RTE_MEMZONE_NAMESIZE];
-	const struct rte_memzone *mz;
-
-	snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
-			dev->device->driver->name, ring_name,
-			dev->data->port_id, queue_id);
-
-	mz = rte_memzone_lookup(z_name);
-	if (mz)
-		return mz;
-
-	return rte_memzone_reserve_aligned(z_name, ring_size, socket_id,
-			RTE_MEMZONE_IOVA_CONTIG, BNX2X_PAGE_SIZE);
+	return rte_eth_dma_zone_reserve(dev, ring_name, queue_id,
+			ring_size, BNX2X_PAGE_SIZE, socket_id);
 }
 
 static void
@@ -76,7 +65,7 @@ bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue),
 				 RTE_CACHE_LINE_SIZE, socket_id);
 	if (NULL == rxq) {
-		PMD_INIT_LOG(ERR, "rte_zmalloc for rxq failed!");
+		PMD_DRV_LOG(ERR, sc, "rte_zmalloc for rxq failed!");
 		return -ENOMEM;
 	}
 	rxq->sc = sc;
@@ -92,7 +81,7 @@ bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	sc->rx_ring_size = USABLE_RX_BD(rxq);
 	rxq->nb_cq_pages = RCQ_BD_PAGES(rxq);
 
-	PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, usable_bd=%lu, "
+	PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, usable_bd=%lu, "
 		       "total_bd=%lu, rx_pages=%u, cq_pages=%u",
 		       queue_idx, nb_desc, (unsigned long)USABLE_RX_BD(rxq),
 		       (unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages,
@@ -275,7 +264,7 @@ bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev,
 	txq->tx_free_thresh = min(txq->tx_free_thresh,
 				  txq->nb_tx_desc - BDS_PER_TX_PKT);
 
-	PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
+	PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
 		     "total_bd=%lu, tx_pages=%u",
 		     queue_idx, nb_desc, txq->tx_free_thresh,
 		     (unsigned long)USABLE_TX_BD(txq),
@@ -301,7 +290,7 @@ bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev,
 		return -ENOMEM;
 	}
 
-	/* PMD_DRV_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
+	/* PMD_DRV_LOG(DEBUG, sc, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
 	   txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); */
 
 	/* Link TX pages */
@@ -310,7 +299,9 @@ bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev,
 		busaddr = txq->tx_ring_phys_addr + BNX2X_PAGE_SIZE * (i % txq->nb_tx_pages);
 		tx_n_bd->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
 		tx_n_bd->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
-		/* PMD_DRV_LOG(DEBUG, "link tx page %lu", (TOTAL_TX_BD_PER_PAGE * i - 1)); */
+		/* PMD_DRV_LOG(DEBUG, sc, "link tx page %lu",
+		 *          (TOTAL_TX_BD_PER_PAGE * i - 1));
+		 */
 	}
 
 	txq->queue_id = queue_idx;
@@ -461,9 +452,10 @@ bnx2x_dev_rx_init(struct rte_eth_dev *dev)
 void
 bnx2x_dev_clear_queues(struct rte_eth_dev *dev)
 {
+	struct bnx2x_softc *sc = dev->data->dev_private;
 	uint8_t i;
 
-	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_FUNC_TRACE(sc);
 
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		struct bnx2x_tx_queue *txq = dev->data->tx_queues[i];
diff --git a/drivers/net/bnx2x/bnx2x_stats.c b/drivers/net/bnx2x/bnx2x_stats.c
index edc86ccc..1cd97259 100644
--- a/drivers/net/bnx2x/bnx2x_stats.c
+++ b/drivers/net/bnx2x/bnx2x_stats.c
@@ -82,7 +82,7 @@ bnx2x_storm_stats_post(struct bnx2x_softc *sc)
 		sc->fw_stats_req->hdr.drv_stats_counter =
 			htole16(sc->stats_counter++);
 
-		PMD_DEBUG_PERIODIC_LOG(DEBUG,
+		PMD_DEBUG_PERIODIC_LOG(DEBUG, sc,
 				"sending statistics ramrod %d",
 				le16toh(sc->fw_stats_req->hdr.drv_stats_counter));
 
@@ -154,7 +154,7 @@ bnx2x_stats_comp(struct bnx2x_softc *sc)
 
 	while (*stats_comp != DMAE_COMP_VAL) {
 		if (!cnt) {
-			PMD_DRV_LOG(ERR, "Timeout waiting for stats finished");
+			PMD_DRV_LOG(ERR, sc, "Timeout waiting for stats finished");
 			break;
 		}
 
@@ -189,7 +189,7 @@ bnx2x_stats_pmf_update(struct bnx2x_softc *sc)
 	}
 	/* sanity */
 	if (!sc->port.pmf || !sc->port.port_stx) {
-		PMD_DRV_LOG(ERR, "BUG!");
+		PMD_DRV_LOG(ERR, sc, "BUG!");
 		return;
 	}
 
@@ -239,7 +239,7 @@ bnx2x_port_stats_init(struct bnx2x_softc *sc)
 
     /* sanity */
     if (!sc->link_vars.link_up || !sc->port.pmf) {
-	PMD_DRV_LOG(ERR, "BUG!");
+	PMD_DRV_LOG(ERR, sc, "BUG!");
 	return;
     }
 
@@ -463,7 +463,7 @@ bnx2x_func_stats_init(struct bnx2x_softc *sc)
 
     /* sanity */
     if (!sc->func_stx) {
-	PMD_DRV_LOG(ERR, "BUG!");
+	PMD_DRV_LOG(ERR, sc, "BUG!");
 	return;
     }
 
@@ -797,12 +797,12 @@ bnx2x_hw_stats_update(struct bnx2x_softc *sc)
 	break;
 
     case ELINK_MAC_TYPE_NONE: /* unreached */
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 	      "stats updated by DMAE but no MAC active");
 	return -1;
 
     default: /* unreached */
-	PMD_DRV_LOG(ERR, "stats update failed, unknown MAC type");
+	PMD_DRV_LOG(ERR, sc, "stats update failed, unknown MAC type");
     }
 
     ADD_EXTEND_64(pstats->brb_drop_hi, pstats->brb_drop_lo,
@@ -837,7 +837,7 @@ bnx2x_hw_stats_update(struct bnx2x_softc *sc)
 	nig_timer_max = SHMEM_RD(sc, port_mb[SC_PORT(sc)].stat_nig_timer);
 	if (nig_timer_max != estats->nig_timer_max) {
 	    estats->nig_timer_max = nig_timer_max;
-	    PMD_DRV_LOG(ERR, "invalid NIG timer max (%u)",
+	    PMD_DRV_LOG(ERR, sc, "invalid NIG timer max (%u)",
 		  estats->nig_timer_max);
 	}
     }
@@ -859,7 +859,7 @@ bnx2x_storm_stats_validate_counters(struct bnx2x_softc *sc)
 
     /* are storm stats valid? */
     if (le16toh(counters->xstats_counter) != cur_stats_counter) {
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 	      "stats not updated by xstorm, "
 	      "counter 0x%x != stats_counter 0x%x",
 	      le16toh(counters->xstats_counter), sc->stats_counter);
@@ -867,7 +867,7 @@ bnx2x_storm_stats_validate_counters(struct bnx2x_softc *sc)
     }
 
     if (le16toh(counters->ustats_counter) != cur_stats_counter) {
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 	      "stats not updated by ustorm, "
 	      "counter 0x%x != stats_counter 0x%x",
 	      le16toh(counters->ustats_counter), sc->stats_counter);
@@ -875,7 +875,7 @@ bnx2x_storm_stats_validate_counters(struct bnx2x_softc *sc)
     }
 
     if (le16toh(counters->cstats_counter) != cur_stats_counter) {
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 	      "stats not updated by cstorm, "
 	      "counter 0x%x != stats_counter 0x%x",
 	      le16toh(counters->cstats_counter), sc->stats_counter);
@@ -883,7 +883,7 @@ bnx2x_storm_stats_validate_counters(struct bnx2x_softc *sc)
     }
 
     if (le16toh(counters->tstats_counter) != cur_stats_counter) {
-	PMD_DRV_LOG(DEBUG,
+	PMD_DRV_LOG(DEBUG, sc,
 	      "stats not updated by tstorm, "
 	      "counter 0x%x != stats_counter 0x%x",
 	      le16toh(counters->tstats_counter), sc->stats_counter);
@@ -929,12 +929,13 @@ bnx2x_storm_stats_update(struct bnx2x_softc *sc)
 
 		uint32_t diff;
 
-		/* PMD_DRV_LOG(DEBUG,
+		/* PMD_DRV_LOG(DEBUG, sc,
 				"queue[%d]: ucast_sent 0x%x bcast_sent 0x%x mcast_sent 0x%x",
 				i, xclient->ucast_pkts_sent, xclient->bcast_pkts_sent,
 				xclient->mcast_pkts_sent);
 
-		PMD_DRV_LOG(DEBUG, "---------------"); */
+		PMD_DRV_LOG(DEBUG, sc, "---------------");
+		 */
 
 		UPDATE_QSTAT(tclient->rcv_bcast_bytes,
 				total_broadcast_bytes_received);
@@ -1288,7 +1289,7 @@ void bnx2x_stats_handle(struct bnx2x_softc *sc, enum bnx2x_stats_event event)
 	bnx2x_stats_stm[state][event].action(sc);
 
 	if (event != STATS_EVENT_UPDATE) {
-		PMD_DRV_LOG(DEBUG,
+		PMD_DRV_LOG(DEBUG, sc,
 				"state %d -> event %d -> state %d",
 				state, event, sc->stats_state);
 	}
@@ -1302,7 +1303,7 @@ bnx2x_port_stats_base_init(struct bnx2x_softc *sc)
 
     /* sanity */
     if (!sc->port.pmf || !sc->port.port_stx) {
-	PMD_DRV_LOG(ERR, "BUG!");
+	PMD_DRV_LOG(ERR, sc, "BUG!");
 	return;
     }
 
@@ -1474,7 +1475,7 @@ bnx2x_stats_init(struct bnx2x_softc *sc)
 		sc->func_stx = 0;
 	}
 
-	PMD_DRV_LOG(DEBUG, "port_stx 0x%x func_stx 0x%x",
+	PMD_DRV_LOG(DEBUG, sc, "port_stx 0x%x func_stx 0x%x",
 			sc->port.port_stx, sc->func_stx);
 
 	/* pmf should retrieve port statistics from SP on a non-init*/
diff --git a/drivers/net/bnx2x/bnx2x_vfpf.c b/drivers/net/bnx2x/bnx2x_vfpf.c
index 50099d46..048bf126 100644
--- a/drivers/net/bnx2x/bnx2x_vfpf.c
+++ b/drivers/net/bnx2x/bnx2x_vfpf.c
@@ -37,12 +37,12 @@ bnx2x_check_bull(struct bnx2x_softc *sc)
 			if (bull->crc == bnx2x_vf_crc(bull))
 				break;
 
-			PMD_DRV_LOG(ERR, "bad crc on bulletin board. contained %x computed %x",
+			PMD_DRV_LOG(ERR, sc, "bad crc on bulletin board. contained %x computed %x",
 					bull->crc, bnx2x_vf_crc(bull));
 			++tries;
 		}
 		if (tries == BNX2X_VF_BULLETIN_TRIES) {
-			PMD_DRV_LOG(ERR, "pf to vf bulletin board crc was wrong %d consecutive times. Aborting",
+			PMD_DRV_LOG(ERR, sc, "pf to vf bulletin board crc was wrong %d consecutive times. Aborting",
 					tries);
 			return FALSE;
 		}
@@ -82,7 +82,7 @@ bnx2x_vf_prep(struct bnx2x_softc *sc, struct vf_first_tlv *first_tlv,
 
 	rte_spinlock_lock(&sc->vf2pf_lock);
 
-	PMD_DRV_LOG(DEBUG, "Preparing %d tlv for sending", type);
+	PMD_DRV_LOG(DEBUG, sc, "Preparing %d tlv for sending", type);
 
 	memset(mbox, 0, sizeof(struct bnx2x_vf_mbx_msg));
 
@@ -97,7 +97,7 @@ static void
 bnx2x_vf_finalize(struct bnx2x_softc *sc,
 		  __rte_unused struct vf_first_tlv *first_tlv)
 {
-	PMD_DRV_LOG(DEBUG, "done sending [%d] tlv over vf pf channel",
+	PMD_DRV_LOG(DEBUG, sc, "done sending [%d] tlv over vf pf channel",
 		    first_tlv->tl.type);
 
 	rte_spinlock_unlock(&sc->vf2pf_lock);
@@ -116,14 +116,14 @@ bnx2x_do_req4pf(struct bnx2x_softc *sc, rte_iova_t phys_addr)
 	uint8_t i;
 
 	if (*status) {
-		PMD_DRV_LOG(ERR, "status should be zero before message"
+		PMD_DRV_LOG(ERR, sc, "status should be zero before message"
 				 " to pf was sent");
 		return -EINVAL;
 	}
 
 	bnx2x_check_bull(sc);
 	if (sc->old_bulletin.valid_bitmap & (1 << CHANNEL_DOWN)) {
-		PMD_DRV_LOG(ERR, "channel is down. Aborting message sending");
+		PMD_DRV_LOG(ERR, sc, "channel is down. Aborting message sending");
 		return -EINVAL;
 	}
 
@@ -143,11 +143,11 @@ bnx2x_do_req4pf(struct bnx2x_softc *sc, rte_iova_t phys_addr)
 	}
 
 	if (!*status) {
-		PMD_DRV_LOG(ERR, "Response from PF timed out");
+		PMD_DRV_LOG(ERR, sc, "Response from PF timed out");
 		return -EAGAIN;
 	}
 
-	PMD_DRV_LOG(DEBUG, "Response from PF was received");
+	PMD_DRV_LOG(DEBUG, sc, "Response from PF was received");
 	return 0;
 }
 
@@ -195,7 +195,7 @@ int bnx2x_loop_obtain_resources(struct bnx2x_softc *sc)
 	int rc;
 
 	do {
-		PMD_DRV_LOG(DEBUG, "trying to get resources");
+		PMD_DRV_LOG(DEBUG, sc, "trying to get resources");
 
 		rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr);
 		if (rc)
@@ -207,11 +207,11 @@ int bnx2x_loop_obtain_resources(struct bnx2x_softc *sc)
 
 		/* check PF to request acceptance */
 		if (sc_resp->status == BNX2X_VF_STATUS_SUCCESS) {
-			PMD_DRV_LOG(DEBUG, "resources obtained successfully");
+			PMD_DRV_LOG(DEBUG, sc, "resources obtained successfully");
 			res_obtained = true;
 		} else if (sc_resp->status == BNX2X_VF_STATUS_NO_RESOURCES &&
 			   tries < BNX2X_VF_OBTAIN_MAX_TRIES) {
-			PMD_DRV_LOG(DEBUG,
+			PMD_DRV_LOG(DEBUG, sc,
 			   "PF cannot allocate requested amount of resources");
 
 			res_query = &sc->vf2pf_mbox->query[0].acquire.res_query;
@@ -227,7 +227,7 @@ int bnx2x_loop_obtain_resources(struct bnx2x_softc *sc)
 
 			memset(&sc->vf2pf_mbox->resp, 0, sizeof(union resp_tlvs));
 		} else {
-			PMD_DRV_LOG(ERR, "Failed to get the requested "
+			PMD_DRV_LOG(ERR, sc, "Failed to get the requested "
 					 "amount of resources: %d.",
 					 sc_resp->status);
 			return -EINVAL;
@@ -296,7 +296,7 @@ int bnx2x_vf_get_resources(struct bnx2x_softc *sc, uint8_t tx_count, uint8_t rx_
 	sc->doorbell_size = sc_resp.db_size;
 	sc->flags |= BNX2X_NO_WOL_FLAG | BNX2X_NO_ISCSI_OOO_FLAG | BNX2X_NO_ISCSI_FLAG | BNX2X_NO_FCOE_FLAG;
 
-	PMD_DRV_LOG(DEBUG, "status block count = %d, base status block = %x",
+	PMD_DRV_LOG(DEBUG, sc, "status block count = %d, base status block = %x",
 		sc->igu_sb_cnt, sc->igu_base_sb);
 	strncpy(sc->fw_ver, sc_resp.fw_ver, sizeof(sc->fw_ver));
 
@@ -333,7 +333,7 @@ bnx2x_vf_close(struct bnx2x_softc *sc)
 
 		rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr);
 		if (rc || reply->status != BNX2X_VF_STATUS_SUCCESS)
-			PMD_DRV_LOG(ERR, "Failed to release VF");
+			PMD_DRV_LOG(ERR, sc, "Failed to release VF");
 
 		bnx2x_vf_finalize(sc, &query->first_tlv);
 	}
@@ -367,12 +367,12 @@ bnx2x_vf_init(struct bnx2x_softc *sc)
 	if (rc)
 		goto out;
 	if (reply->status != BNX2X_VF_STATUS_SUCCESS) {
-		PMD_DRV_LOG(ERR, "Failed to init VF");
+		PMD_DRV_LOG(ERR, sc, "Failed to init VF");
 		rc = -EINVAL;
 		goto out;
 	}
 
-	PMD_DRV_LOG(DEBUG, "VF was initialized");
+	PMD_DRV_LOG(DEBUG, sc, "VF was initialized");
 out:
 	bnx2x_vf_finalize(sc, &query->first_tlv);
 	return rc;
@@ -403,7 +403,7 @@ bnx2x_vf_unload(struct bnx2x_softc *sc)
 
 			rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr);
 			if (rc || reply->status != BNX2X_VF_STATUS_SUCCESS)
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Bad reply for vf_q %d teardown", i);
 
 			bnx2x_vf_finalize(sc, &query_op->first_tlv);
@@ -423,7 +423,7 @@ bnx2x_vf_unload(struct bnx2x_softc *sc)
 
 		rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr);
 		if (rc || reply->status != BNX2X_VF_STATUS_SUCCESS)
-			PMD_DRV_LOG(ERR,
+			PMD_DRV_LOG(ERR, sc,
 				    "Bad reply from PF for close message");
 
 		bnx2x_vf_finalize(sc, &query->first_tlv);
@@ -450,7 +450,7 @@ bnx2x_vf_rx_q_prep(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
 
 	rxq = sc->rx_queues[fp->index];
 	if (!rxq) {
-		PMD_DRV_LOG(ERR, "RX queue %d is NULL", fp->index);
+		PMD_DRV_LOG(ERR, sc, "RX queue %d is NULL", fp->index);
 		return;
 	}
 
@@ -474,7 +474,7 @@ bnx2x_vf_tx_q_prep(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
 
 	txq = sc->tx_queues[fp->index];
 	if (!txq) {
-		PMD_DRV_LOG(ERR, "TX queue %d is NULL", fp->index);
+		PMD_DRV_LOG(ERR, sc, "TX queue %d is NULL", fp->index);
 		return;
 	}
 
@@ -511,7 +511,7 @@ bnx2x_vf_setup_queue(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp, int lead
 	if (rc)
 		goto out;
 	if (reply->status != BNX2X_VF_STATUS_SUCCESS) {
-		PMD_DRV_LOG(ERR, "Failed to setup VF queue[%d]",
+		PMD_DRV_LOG(ERR, sc, "Failed to setup VF queue[%d]",
 				 fp->index);
 		rc = -EINVAL;
 	}
@@ -566,7 +566,7 @@ bnx2x_vf_set_mac(struct bnx2x_softc *sc, int set)
 	}
 
 	if (BNX2X_VF_STATUS_SUCCESS != reply->status) {
-		PMD_DRV_LOG(ERR, "Bad reply from PF for SET MAC message: %d",
+		PMD_DRV_LOG(ERR, sc, "Bad reply from PF for SET MAC message: %d",
 				reply->status);
 		rc = -EINVAL;
 	}
@@ -608,7 +608,7 @@ bnx2x_vf_config_rss(struct bnx2x_softc *sc,
 		goto out;
 
 	if (reply->status != BNX2X_VF_STATUS_SUCCESS) {
-		PMD_DRV_LOG(ERR, "Failed to configure RSS");
+		PMD_DRV_LOG(ERR, sc, "Failed to configure RSS");
 		rc = -EINVAL;
 	}
 out:
@@ -652,7 +652,7 @@ bnx2x_vf_set_rx_mode(struct bnx2x_softc *sc)
 		query->rx_mask |= VFPF_RX_MASK_ACCEPT_BROADCAST;
 		break;
 	default:
-		PMD_DRV_LOG(ERR, "BAD rx mode (%d)", sc->rx_mode);
+		PMD_DRV_LOG(ERR, sc, "BAD rx mode (%d)", sc->rx_mode);
 		rc = -EINVAL;
 		goto out;
 	}
@@ -666,7 +666,7 @@ bnx2x_vf_set_rx_mode(struct bnx2x_softc *sc)
 		goto out;
 
 	if (reply->status != BNX2X_VF_STATUS_SUCCESS) {
-		PMD_DRV_LOG(ERR, "Failed to set RX mode");
+		PMD_DRV_LOG(ERR, sc, "Failed to set RX mode");
 		rc = -EINVAL;
 	}
 
diff --git a/drivers/net/bnx2x/ecore_hsi.h b/drivers/net/bnx2x/ecore_hsi.h
index 57085ebb..1192e5dd 100644
--- a/drivers/net/bnx2x/ecore_hsi.h
+++ b/drivers/net/bnx2x/ecore_hsi.h
@@ -500,6 +500,18 @@ struct port_hw_cfg {		    /* port 0: 0x12c  port 1: 0x2bc */
 	#define PORT_HW_CFG_TX_DRV_BROADCAST_MASK                     0x000F0000
 	#define PORT_HW_CFG_TX_DRV_BROADCAST_SHIFT                    16
 
+	/*  Set non-default values for TXFIR in SFP mode. */
+	#define PORT_HW_CFG_TX_DRV_IFIR_MASK                          0x00F00000
+	#define PORT_HW_CFG_TX_DRV_IFIR_SHIFT                         20
+
+	/*  Set non-default values for IPREDRIVER in SFP mode. */
+	#define PORT_HW_CFG_TX_DRV_IPREDRIVER_MASK                    0x0F000000
+	#define PORT_HW_CFG_TX_DRV_IPREDRIVER_SHIFT                   24
+
+	/*  Set non-default values for POST2 in SFP mode. */
+	#define PORT_HW_CFG_TX_DRV_POST2_MASK                         0xF0000000
+	#define PORT_HW_CFG_TX_DRV_POST2_SHIFT                        28
+
 	uint32_t reserved0[5];				    /* 0x17c */
 
 	uint32_t aeu_int_mask;				    /* 0x190 */
@@ -783,6 +795,7 @@ struct port_hw_cfg {		    /* port 0: 0x12c  port 1: 0x2bc */
 		#define PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8722        0x00000f00
 		#define PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X54616       0x00001000
 		#define PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834       0x00001100
+		#define PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84858     0x00001200
 		#define PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT_WC      0x0000fc00
 		#define PORT_HW_CFG_XGXS_EXT_PHY_TYPE_FAILURE        0x0000fd00
 		#define PORT_HW_CFG_XGXS_EXT_PHY_TYPE_NOT_CONN       0x0000ff00
@@ -2532,7 +2545,12 @@ struct shmem2_region {
 	uint32_t drv_func_info_addr;			/* Offset 0x14C */
 	uint32_t drv_func_info_size;			/* Offset 0x150 */
 	uint32_t link_attr_sync[PORT_MAX];		/* Offset 0x154 */
-	#define LINK_ATTR_SYNC_KR2_ENABLE	(1<<0)
+	#define LINK_ATTR_SYNC_KR2_ENABLE       0x00000001
+	#define LINK_ATTR_84858                 0x00000002
+	#define LINK_SFP_EEPROM_COMP_CODE_MASK  0x0000ff00
+	#define LINK_SFP_EEPROM_COMP_CODE_SHIFT          8
+
+	uint32_t link_change_count[PORT_MAX];		/* Offset 0x160-0x164 */
 };
 
 
diff --git a/drivers/net/bnx2x/ecore_init.h b/drivers/net/bnx2x/ecore_init.h
index f2de07e5..97dfe69b 100644
--- a/drivers/net/bnx2x/ecore_init.h
+++ b/drivers/net/bnx2x/ecore_init.h
@@ -741,7 +741,7 @@ static inline void ecore_disable_blocks_parity(struct bnx2x_softc *sc)
 		if (dis_mask) {
 			REG_WR(sc, ecore_blocks_parity_data[i].mask_addr,
 			       dis_mask);
-			ECORE_MSG("Setting parity mask "
+			ECORE_MSG(sc, "Setting parity mask "
 						 "for %s to\t\t0x%x",
 				    ecore_blocks_parity_data[i].name, dis_mask);
 		}
@@ -776,7 +776,7 @@ static inline void ecore_clear_blocks_parity(struct bnx2x_softc *sc)
 			reg_val = REG_RD(sc, ecore_blocks_parity_data[i].
 					 sts_clr_addr);
 			if (reg_val & reg_mask)
-				ECORE_MSG("Parity errors in %s: 0x%x",
+				ECORE_MSG(sc, "Parity errors in %s: 0x%x",
 					   ecore_blocks_parity_data[i].name,
 					   reg_val & reg_mask);
 		}
@@ -785,7 +785,7 @@ static inline void ecore_clear_blocks_parity(struct bnx2x_softc *sc)
 	/* Check if there were parity attentions in MCP */
 	reg_val = REG_RD(sc, MISC_REG_AEU_AFTER_INVERT_4_MCP);
 	if (reg_val & mcp_aeu_bits)
-		ECORE_MSG("Parity error in MCP: 0x%x",
+		ECORE_MSG(sc, "Parity error in MCP: 0x%x",
 			   reg_val & mcp_aeu_bits);
 
 	/* Clear parity attentions in MCP:
diff --git a/drivers/net/bnx2x/ecore_init_ops.h b/drivers/net/bnx2x/ecore_init_ops.h
index 2b003afb..733ad1aa 100644
--- a/drivers/net/bnx2x/ecore_init_ops.h
+++ b/drivers/net/bnx2x/ecore_init_ops.h
@@ -424,20 +424,20 @@ static void ecore_init_pxp_arb(struct bnx2x_softc *sc, int r_order,
 	uint32_t val, i;
 
 	if (r_order > MAX_RD_ORD) {
-		ECORE_MSG("read order of %d  order adjusted to %d",
+		ECORE_MSG(sc, "read order of %d  order adjusted to %d",
 			   r_order, MAX_RD_ORD);
 		r_order = MAX_RD_ORD;
 	}
 	if (w_order > MAX_WR_ORD) {
-		ECORE_MSG("write order of %d  order adjusted to %d",
+		ECORE_MSG(sc, "write order of %d  order adjusted to %d",
 			   w_order, MAX_WR_ORD);
 		w_order = MAX_WR_ORD;
 	}
 	if (CHIP_REV_IS_FPGA(sc)) {
-		ECORE_MSG("write order adjusted to 1 for FPGA");
+		ECORE_MSG(sc, "write order adjusted to 1 for FPGA");
 		w_order = 0;
 	}
-	ECORE_MSG("read order %d  write order %d", r_order, w_order);
+	ECORE_MSG(sc, "read order %d  write order %d", r_order, w_order);
 
 	for (i = 0; i < NUM_RD_Q-1; i++) {
 		REG_WR(sc, read_arb_addr[i].l, read_arb_data[i][r_order].l);
diff --git a/drivers/net/bnx2x/ecore_reg.h b/drivers/net/bnx2x/ecore_reg.h
index ae8a93bb..d69e857b 100644
--- a/drivers/net/bnx2x/ecore_reg.h
+++ b/drivers/net/bnx2x/ecore_reg.h
@@ -1967,6 +1967,7 @@
 #define HW_LOCK_MAX_RESOURCE_VALUE		 31
 #define HW_LOCK_RESOURCE_DRV_FLAGS		 10
 #define HW_LOCK_RESOURCE_GPIO			 1
+#define HW_LOCK_RESOURCE_MDIO			 0
 #define HW_LOCK_RESOURCE_NVRAM			 12
 #define HW_LOCK_RESOURCE_PORT0_ATT_MASK		 3
 #define HW_LOCK_RESOURCE_RECOVERY_LEADER_0	 8
diff --git a/drivers/net/bnx2x/ecore_sp.c b/drivers/net/bnx2x/ecore_sp.c
index 0c8685c6..ab730abe 100644
--- a/drivers/net/bnx2x/ecore_sp.c
+++ b/drivers/net/bnx2x/ecore_sp.c
@@ -53,14 +53,14 @@ ecore_exe_queue_init(struct bnx2x_softc *sc __rte_unused,
 	o->execute = exec;
 	o->get = get;
 
-	ECORE_MSG("Setup the execution queue with the chunk length of %d",
+	ECORE_MSG(sc, "Setup the execution queue with the chunk length of %d",
 		  exe_len);
 }
 
 static void ecore_exe_queue_free_elem(struct bnx2x_softc *sc __rte_unused,
 				      struct ecore_exeq_elem *elem)
 {
-	ECORE_MSG("Deleting an exe_queue element");
+	ECORE_MSG(sc, "Deleting an exe_queue element");
 	ECORE_FREE(sc, elem, sizeof(*elem));
 }
 
@@ -106,7 +106,7 @@ static int ecore_exe_queue_add(struct bnx2x_softc *sc,
 		/* Check if this request is ok */
 		rc = o->validate(sc, o->owner, elem);
 		if (rc) {
-			ECORE_MSG("Preamble failed: %d", rc);
+			ECORE_MSG(sc, "Preamble failed: %d", rc);
 			goto free_and_exit;
 		}
 	}
@@ -176,8 +176,8 @@ static int ecore_exe_queue_step(struct bnx2x_softc *sc,
 	 */
 	if (!ECORE_LIST_IS_EMPTY(&o->pending_comp)) {
 		if (ECORE_TEST_BIT(RAMROD_DRV_CLR_ONLY, ramrod_flags)) {
-			ECORE_MSG
-			    ("RAMROD_DRV_CLR_ONLY requested: resetting a pending_comp list");
+			ECORE_MSG(sc,
+				  "RAMROD_DRV_CLR_ONLY requested: resetting a pending_comp list");
 			__ecore_exe_queue_reset_pending(sc, o);
 		} else {
 			return ECORE_PENDING;
@@ -240,7 +240,7 @@ static struct ecore_exeq_elem *ecore_exe_queue_alloc_elem(struct
 							  bnx2x_softc *sc
 							  __rte_unused)
 {
-	ECORE_MSG("Allocating a new exe_queue element");
+	ECORE_MSG(sc, "Allocating a new exe_queue element");
 	return ECORE_ZALLOC(sizeof(struct ecore_exeq_elem), GFP_ATOMIC, sc);
 }
 
@@ -290,14 +290,14 @@ static int ecore_state_wait(struct bnx2x_softc *sc, int state,
 	if (CHIP_REV_IS_EMUL(sc))
 		cnt *= 20;
 
-	ECORE_MSG("waiting for state to become %d", state);
+	ECORE_MSG(sc, "waiting for state to become %d", state);
 
 	ECORE_MIGHT_SLEEP();
 	while (cnt--) {
 		bnx2x_intr_legacy(sc, 1);
 		if (!ECORE_TEST_BIT(state, pstate)) {
 #ifdef ECORE_STOP_ON_ERROR
-			ECORE_MSG("exit  (cnt %d)", 5000 - cnt);
+			ECORE_MSG(sc, "exit  (cnt %d)", 5000 - cnt);
 #endif
 			return ECORE_SUCCESS;
 		}
@@ -309,7 +309,7 @@ static int ecore_state_wait(struct bnx2x_softc *sc, int state,
 	}
 
 	/* timeout! */
-	PMD_DRV_LOG(ERR, "timeout waiting for state %d", state);
+	PMD_DRV_LOG(ERR, sc, "timeout waiting for state %d", state);
 #ifdef ECORE_STOP_ON_ERROR
 	ecore_panic();
 #endif
@@ -370,11 +370,11 @@ static int __ecore_vlan_mac_h_write_trylock(struct bnx2x_softc *sc __rte_unused,
 					    struct ecore_vlan_mac_obj *o)
 {
 	if (o->head_reader) {
-		ECORE_MSG("vlan_mac_lock writer - There are readers; Busy");
+		ECORE_MSG(sc, "vlan_mac_lock writer - There are readers; Busy");
 		return ECORE_BUSY;
 	}
 
-	ECORE_MSG("vlan_mac_lock writer - Taken");
+	ECORE_MSG(sc, "vlan_mac_lock writer - Taken");
 	return ECORE_SUCCESS;
 }
 
@@ -394,13 +394,13 @@ static void __ecore_vlan_mac_h_exec_pending(struct bnx2x_softc *sc,
 	int rc;
 	unsigned long ramrod_flags = o->saved_ramrod_flags;
 
-	ECORE_MSG("vlan_mac_lock execute pending command with ramrod flags %lu",
+	ECORE_MSG(sc, "vlan_mac_lock execute pending command with ramrod flags %lu",
 		  ramrod_flags);
 	o->head_exe_request = FALSE;
 	o->saved_ramrod_flags = 0;
 	rc = ecore_exe_queue_step(sc, &o->exe_queue, &ramrod_flags);
 	if (rc != ECORE_SUCCESS) {
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "execution of pending commands failed with rc %d",
 			    rc);
 #ifdef ECORE_STOP_ON_ERROR
@@ -425,7 +425,7 @@ static void __ecore_vlan_mac_h_pend(struct bnx2x_softc *sc __rte_unused,
 {
 	o->head_exe_request = TRUE;
 	o->saved_ramrod_flags = ramrod_flags;
-	ECORE_MSG("Placing pending execution with ramrod flags %lu",
+	ECORE_MSG(sc, "Placing pending execution with ramrod flags %lu",
 		  ramrod_flags);
 }
 
@@ -446,8 +446,8 @@ static void __ecore_vlan_mac_h_write_unlock(struct bnx2x_softc *sc,
 	 * executed. If so, execute again. [Ad infinitum]
 	 */
 	while (o->head_exe_request) {
-		ECORE_MSG
-		    ("vlan_mac_lock - writer release encountered a pending request");
+		ECORE_MSG(sc,
+			  "vlan_mac_lock - writer release encountered a pending request");
 		__ecore_vlan_mac_h_exec_pending(sc, o);
 	}
 }
@@ -483,7 +483,8 @@ static int __ecore_vlan_mac_h_read_lock(struct bnx2x_softc *sc __rte_unused,
 {
 	/* If we got here, we're holding lock --> no WRITER exists */
 	o->head_reader++;
-	ECORE_MSG("vlan_mac_lock - locked reader - number %d", o->head_reader);
+	ECORE_MSG(sc,
+		  "vlan_mac_lock - locked reader - number %d", o->head_reader);
 
 	return ECORE_SUCCESS;
 }
@@ -522,14 +523,14 @@ static void __ecore_vlan_mac_h_read_unlock(struct bnx2x_softc *sc,
 					   struct ecore_vlan_mac_obj *o)
 {
 	if (!o->head_reader) {
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "Need to release vlan mac reader lock, but lock isn't taken");
 #ifdef ECORE_STOP_ON_ERROR
 		ecore_panic();
 #endif
 	} else {
 		o->head_reader--;
-		PMD_DRV_LOG(INFO,
+		PMD_DRV_LOG(INFO, sc,
 			    "vlan_mac_lock - decreased readers to %d",
 			    o->head_reader);
 	}
@@ -538,7 +539,7 @@ static void __ecore_vlan_mac_h_read_unlock(struct bnx2x_softc *sc,
 	 * was last - if so we need to execute the command.
 	 */
 	if (!o->head_reader && o->head_exe_request) {
-		PMD_DRV_LOG(INFO,
+		PMD_DRV_LOG(INFO, sc,
 			    "vlan_mac_lock - reader release encountered a pending request");
 
 		/* Writer release will do the trick */
@@ -581,10 +582,10 @@ static int ecore_get_n_elements(struct bnx2x_softc *sc,
 	uint8_t *next = base;
 	int counter = 0, read_lock;
 
-	ECORE_MSG("get_n_elements - taking vlan_mac_lock (reader)");
+	ECORE_MSG(sc, "get_n_elements - taking vlan_mac_lock (reader)");
 	read_lock = ecore_vlan_mac_h_read_lock(sc, o);
 	if (read_lock != ECORE_SUCCESS)
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "get_n_elements failed to get vlan mac reader lock; Access without lock");
 
 	/* traverse list */
@@ -593,15 +594,15 @@ static int ecore_get_n_elements(struct bnx2x_softc *sc,
 		if (counter < n) {
 			ECORE_MEMCPY(next, &pos->u, size);
 			counter++;
-			ECORE_MSG
-			    ("copied element number %d to address %p element was:",
+			    ECORE_MSG
+			    (sc, "copied element number %d to address %p element was:",
 			     counter, next);
 			next += stride + size;
 		}
 	}
 
 	if (read_lock == ECORE_SUCCESS) {
-		ECORE_MSG("get_n_elements - releasing vlan_mac_lock (reader)");
+		ECORE_MSG(sc, "get_n_elements - releasing vlan_mac_lock (reader)");
 		ecore_vlan_mac_h_read_unlock(sc, o);
 	}
 
@@ -615,7 +616,7 @@ static int ecore_check_mac_add(struct bnx2x_softc *sc __rte_unused,
 {
 	struct ecore_vlan_mac_registry_elem *pos;
 
-	ECORE_MSG("Checking MAC %02x:%02x:%02x:%02x:%02x:%02x for ADD command",
+	ECORE_MSG(sc, "Checking MAC %02x:%02x:%02x:%02x:%02x:%02x for ADD command",
 		  data->mac.mac[0], data->mac.mac[1], data->mac.mac[2],
 		  data->mac.mac[3], data->mac.mac[4], data->mac.mac[5]);
 
@@ -644,7 +645,7 @@ static struct ecore_vlan_mac_registry_elem *ecore_check_mac_del(struct bnx2x_sof
 {
 	struct ecore_vlan_mac_registry_elem *pos;
 
-	ECORE_MSG("Checking MAC %02x:%02x:%02x:%02x:%02x:%02x for DEL command",
+	ECORE_MSG(sc, "Checking MAC %02x:%02x:%02x:%02x:%02x:%02x for DEL command",
 		  data->mac.mac[0], data->mac.mac[1], data->mac.mac[2],
 		  data->mac.mac[3], data->mac.mac[4], data->mac.mac[5]);
 
@@ -722,7 +723,7 @@ static void ecore_set_mac_in_nig(struct bnx2x_softc *sc,
 	if (index > ECORE_LLH_CAM_MAX_PF_LINE)
 		return;
 
-	ECORE_MSG("Going to %s LLH configuration at entry %d",
+	ECORE_MSG(sc, "Going to %s LLH configuration at entry %d",
 		  (add ? "ADD" : "DELETE"), index);
 
 	if (add) {
@@ -838,7 +839,7 @@ static void ecore_set_one_mac_e2(struct bnx2x_softc *sc,
 	ecore_vlan_mac_set_cmd_hdr_e2(o, add, CLASSIFY_RULE_OPCODE_MAC,
 				      &rule_entry->mac.header);
 
-	ECORE_MSG("About to %s MAC %02x:%02x:%02x:%02x:%02x:%02x for Queue %d",
+	ECORE_MSG(sc, "About to %s MAC %02x:%02x:%02x:%02x:%02x:%02x for Queue %d",
 		  (add ? "add" : "delete"), mac[0], mac[1], mac[2], mac[3],
 		  mac[4], mac[5], raw->cl_id);
 
@@ -943,7 +944,7 @@ static void ecore_vlan_mac_set_rdata_e1x(struct bnx2x_softc *sc
 	ecore_vlan_mac_set_cfg_entry_e1x(o, add, opcode, mac, vlan_id,
 					 cfg_entry);
 
-	ECORE_MSG("%s MAC %02x:%02x:%02x:%02x:%02x:%02x CLID %d CAM offset %d",
+	ECORE_MSG(sc, "%s MAC %02x:%02x:%02x:%02x:%02x:%02x CLID %d CAM offset %d",
 		  (add ? "setting" : "clearing"),
 		  mac[0], mac[1], mac[2], mac[3], mac[4], mac[5],
 		  o->raw.cl_id, cam_offset);
@@ -1088,8 +1089,8 @@ static int ecore_validate_vlan_mac_add(struct bnx2x_softc *sc,
 	/* Check the registry */
 	rc = o->check_add(sc, o, &elem->cmd_data.vlan_mac.u);
 	if (rc) {
-		ECORE_MSG
-		    ("ADD command is not allowed considering current registry state.");
+		ECORE_MSG(sc,
+			  "ADD command is not allowed considering current registry state.");
 		return rc;
 	}
 
@@ -1097,7 +1098,7 @@ static int ecore_validate_vlan_mac_add(struct bnx2x_softc *sc,
 	 * MAC/VLAN/VLAN-MAC. Return an error if there is.
 	 */
 	if (exeq->get(exeq, elem)) {
-		ECORE_MSG("There is a pending ADD command already");
+		ECORE_MSG(sc, "There is a pending ADD command already");
 		return ECORE_EXISTS;
 	}
 
@@ -1136,8 +1137,8 @@ static int ecore_validate_vlan_mac_del(struct bnx2x_softc *sc,
 	 */
 	pos = o->check_del(sc, o, &elem->cmd_data.vlan_mac.u);
 	if (!pos) {
-		ECORE_MSG
-		    ("DEL command is not allowed considering current registry state");
+		ECORE_MSG(sc,
+			  "DEL command is not allowed considering current registry state");
 		return ECORE_EXISTS;
 	}
 
@@ -1149,13 +1150,13 @@ static int ecore_validate_vlan_mac_del(struct bnx2x_softc *sc,
 	/* Check for MOVE commands */
 	query_elem.cmd_data.vlan_mac.cmd = ECORE_VLAN_MAC_MOVE;
 	if (exeq->get(exeq, &query_elem)) {
-		PMD_DRV_LOG(ERR, "There is a pending MOVE command already");
+		PMD_DRV_LOG(ERR, sc, "There is a pending MOVE command already");
 		return ECORE_INVAL;
 	}
 
 	/* Check for DEL commands */
 	if (exeq->get(exeq, elem)) {
-		ECORE_MSG("There is a pending DEL command already");
+		ECORE_MSG(sc, "There is a pending DEL command already");
 		return ECORE_EXISTS;
 	}
 
@@ -1163,7 +1164,7 @@ static int ecore_validate_vlan_mac_del(struct bnx2x_softc *sc,
 	if (!(ECORE_TEST_BIT(ECORE_DONT_CONSUME_CAM_CREDIT,
 			     &elem->cmd_data.vlan_mac.vlan_mac_flags) ||
 	      o->put_credit(o))) {
-		PMD_DRV_LOG(ERR, "Failed to return a credit");
+		PMD_DRV_LOG(ERR, sc, "Failed to return a credit");
 		return ECORE_INVAL;
 	}
 
@@ -1196,8 +1197,8 @@ static int ecore_validate_vlan_mac_move(struct bnx2x_softc *sc,
 	 * state.
 	 */
 	if (!src_o->check_move(sc, src_o, dest_o, &elem->cmd_data.vlan_mac.u)) {
-		ECORE_MSG
-		    ("MOVE command is not allowed considering current registry state");
+		ECORE_MSG(sc,
+			  "MOVE command is not allowed considering current registry state");
 		return ECORE_INVAL;
 	}
 
@@ -1210,21 +1211,21 @@ static int ecore_validate_vlan_mac_move(struct bnx2x_softc *sc,
 	/* Check DEL on source */
 	query_elem.cmd_data.vlan_mac.cmd = ECORE_VLAN_MAC_DEL;
 	if (src_exeq->get(src_exeq, &query_elem)) {
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "There is a pending DEL command on the source queue already");
 		return ECORE_INVAL;
 	}
 
 	/* Check MOVE on source */
 	if (src_exeq->get(src_exeq, elem)) {
-		ECORE_MSG("There is a pending MOVE command already");
+		ECORE_MSG(sc, "There is a pending MOVE command already");
 		return ECORE_EXISTS;
 	}
 
 	/* Check ADD on destination */
 	query_elem.cmd_data.vlan_mac.cmd = ECORE_VLAN_MAC_ADD;
 	if (dest_exeq->get(dest_exeq, &query_elem)) {
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "There is a pending ADD command on the destination queue already");
 		return ECORE_INVAL;
 	}
@@ -1329,7 +1330,7 @@ static int __ecore_vlan_mac_execute_step(struct bnx2x_softc *sc,
 
 	ECORE_SPIN_LOCK_BH(&o->exe_queue.lock);
 
-	ECORE_MSG("vlan_mac_execute_step - trying to take writer lock");
+	ECORE_MSG(sc, "vlan_mac_execute_step - trying to take writer lock");
 	rc = __ecore_vlan_mac_h_write_trylock(sc, o);
 
 	if (rc != ECORE_SUCCESS) {
@@ -1426,17 +1427,17 @@ static int ecore_optimize_vlan_mac(struct bnx2x_softc *sc,
 				    &pos->cmd_data.vlan_mac.vlan_mac_flags)) {
 			if ((query.cmd_data.vlan_mac.cmd ==
 			     ECORE_VLAN_MAC_ADD) && !o->put_credit(o)) {
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Failed to return the credit for the optimized ADD command");
 				return ECORE_INVAL;
 			} else if (!o->get_credit(o)) {	/* VLAN_MAC_DEL */
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Failed to recover the credit from the optimized DEL command");
 				return ECORE_INVAL;
 			}
 		}
 
-		ECORE_MSG("Optimizing %s command",
+		ECORE_MSG(sc, "Optimizing %s command",
 			  (elem->cmd_data.vlan_mac.cmd == ECORE_VLAN_MAC_ADD) ?
 			  "ADD" : "DEL");
 
@@ -1486,7 +1487,7 @@ static int ecore_vlan_mac_get_registry_elem(struct bnx2x_softc *sc,
 			return ECORE_INVAL;
 		}
 
-		ECORE_MSG("Got cam offset %d", reg_elem->cam_offset);
+		ECORE_MSG(sc, "Got cam offset %d", reg_elem->cam_offset);
 
 		/* Set a VLAN-MAC data */
 		ECORE_MEMCPY(&reg_elem->u, &elem->cmd_data.vlan_mac.u,
@@ -1695,8 +1696,8 @@ int ecore_config_vlan_mac(struct bnx2x_softc *sc,
 		rc = ECORE_PENDING;
 
 	if (ECORE_TEST_BIT(RAMROD_DRV_CLR_ONLY, ramrod_flags)) {
-		ECORE_MSG
-		    ("RAMROD_DRV_CLR_ONLY requested: clearing a pending bit.");
+		ECORE_MSG(sc,
+			  "RAMROD_DRV_CLR_ONLY requested: clearing a pending bit.");
 		raw->clear_pending(raw);
 	}
 
@@ -1775,7 +1776,7 @@ static int ecore_vlan_mac_del_all(struct bnx2x_softc *sc,
 		    *vlan_mac_flags) {
 			rc = exeq->remove(sc, exeq->owner, exeq_pos);
 			if (rc) {
-				PMD_DRV_LOG(ERR, "Failed to remove command");
+				PMD_DRV_LOG(ERR, sc, "Failed to remove command");
 				ECORE_SPIN_UNLOCK_BH(&exeq->lock);
 				return rc;
 			}
@@ -1800,7 +1801,7 @@ static int ecore_vlan_mac_del_all(struct bnx2x_softc *sc,
 	ECORE_CLEAR_BIT_NA(RAMROD_EXEC, &p.ramrod_flags);
 	ECORE_CLEAR_BIT_NA(RAMROD_CONT, &p.ramrod_flags);
 
-	ECORE_MSG("vlan_mac_del_all -- taking vlan_mac_lock (reader)");
+	ECORE_MSG(sc, "vlan_mac_del_all -- taking vlan_mac_lock (reader)");
 	read_lock = ecore_vlan_mac_h_read_lock(sc, o);
 	if (read_lock != ECORE_SUCCESS)
 		return read_lock;
@@ -1812,7 +1813,7 @@ static int ecore_vlan_mac_del_all(struct bnx2x_softc *sc,
 			ECORE_MEMCPY(&p.user_req.u, &pos->u, sizeof(pos->u));
 			rc = ecore_config_vlan_mac(sc, &p);
 			if (rc < 0) {
-				PMD_DRV_LOG(ERR,
+				PMD_DRV_LOG(ERR, sc,
 					    "Failed to add a new DEL command");
 				ecore_vlan_mac_h_read_unlock(sc, o);
 				return rc;
@@ -1820,7 +1821,7 @@ static int ecore_vlan_mac_del_all(struct bnx2x_softc *sc,
 		}
 	}
 
-	ECORE_MSG("vlan_mac_del_all -- releasing vlan_mac_lock (reader)");
+	ECORE_MSG(sc, "vlan_mac_del_all -- releasing vlan_mac_lock (reader)");
 	ecore_vlan_mac_h_read_unlock(sc, o);
 
 	p.ramrod_flags = *ramrod_flags;
@@ -2007,7 +2008,7 @@ static int ecore_set_rx_mode_e1x(struct bnx2x_softc *sc,
 	    mac_filters->unmatched_unicast | mask :
 	    mac_filters->unmatched_unicast & ~mask;
 
-	ECORE_MSG("drop_ucast 0x%xdrop_mcast 0x%x accp_ucast 0x%x"
+	ECORE_MSG(sc, "drop_ucast 0x%xdrop_mcast 0x%x accp_ucast 0x%x"
 		  "accp_mcast 0x%xaccp_bcast 0x%x",
 		  mac_filters->ucast_drop_all, mac_filters->mcast_drop_all,
 		  mac_filters->ucast_accept_all, mac_filters->mcast_accept_all,
@@ -2153,8 +2154,8 @@ static int ecore_set_rx_mode_e2(struct bnx2x_softc *sc,
 	 */
 	ecore_rx_mode_set_rdata_hdr_e2(p->cid, &data->header, rule_idx);
 
-	ECORE_MSG
-	    ("About to configure %d rules, rx_accept_flags 0x%lx, tx_accept_flags 0x%lx",
+	    ECORE_MSG
+	    (sc, "About to configure %d rules, rx_accept_flags 0x%lx, tx_accept_flags 0x%lx",
 	     data->header.rule_cnt, p->rx_accept_flags, p->tx_accept_flags);
 
 	/* No need for an explicit memory barrier here as long we would
@@ -2207,7 +2208,7 @@ int ecore_config_rx_mode(struct bnx2x_softc *sc,
 				return rc;
 		}
 	} else {
-		ECORE_MSG("ERROR: config_rx_mode is NULL");
+		ECORE_MSG(sc, "ERROR: config_rx_mode is NULL");
 		return -1;
 	}
 
@@ -2288,7 +2289,7 @@ static int ecore_mcast_enqueue_cmd(struct bnx2x_softc *sc __rte_unused,
 	if (!new_cmd)
 		return ECORE_NOMEM;
 
-	ECORE_MSG("About to enqueue a new %d command. macs_list_len=%d",
+	ECORE_MSG(sc, "About to enqueue a new %d command. macs_list_len=%d",
 		  cmd, macs_list_len);
 
 	ECORE_LIST_INIT(&new_cmd->data.macs_head);
@@ -2324,7 +2325,7 @@ static int ecore_mcast_enqueue_cmd(struct bnx2x_softc *sc __rte_unused,
 
 	default:
 		ECORE_FREE(sc, new_cmd, total_sz);
-		PMD_DRV_LOG(ERR, "Unknown command: %d", cmd);
+		PMD_DRV_LOG(ERR, sc, "Unknown command: %d", cmd);
 		return ECORE_INVAL;
 	}
 
@@ -2436,11 +2437,11 @@ static void ecore_mcast_set_one_rule_e2(struct bnx2x_softc *sc __rte_unused,
 		break;
 
 	default:
-		PMD_DRV_LOG(ERR, "Unknown command: %d", cmd);
+		PMD_DRV_LOG(ERR, sc, "Unknown command: %d", cmd);
 		return;
 	}
 
-	ECORE_MSG("%s bin %d",
+	ECORE_MSG(sc, "%s bin %d",
 		  ((rx_tx_add_flag & ETH_MULTICAST_RULES_CMD_IS_ADD) ?
 		   "Setting" : "Clearing"), bin);
 
@@ -2475,7 +2476,7 @@ static int ecore_mcast_handle_restore_cmd_e2(struct bnx2x_softc *sc,
 
 		cnt++;
 
-		ECORE_MSG("About to configure a bin %d", cur_bin);
+		ECORE_MSG(sc, "About to configure a bin %d", cur_bin);
 
 		/* Break if we reached the maximum number
 		 * of rules.
@@ -2507,8 +2508,8 @@ static void ecore_mcast_hdl_pending_add_e2(struct bnx2x_softc *sc,
 
 		cnt++;
 
-		ECORE_MSG
-		    ("About to configure %02x:%02x:%02x:%02x:%02x:%02x mcast MAC",
+		    ECORE_MSG
+		    (sc, "About to configure %02x:%02x:%02x:%02x:%02x:%02x mcast MAC",
 		     pmac_pos->mac[0], pmac_pos->mac[1], pmac_pos->mac[2],
 		     pmac_pos->mac[3], pmac_pos->mac[4], pmac_pos->mac[5]);
 
@@ -2543,7 +2544,7 @@ static void ecore_mcast_hdl_pending_del_e2(struct bnx2x_softc *sc,
 
 		cmd_pos->data.macs_num--;
 
-		ECORE_MSG("Deleting MAC. %d left,cnt is %d",
+		ECORE_MSG(sc, "Deleting MAC. %d left,cnt is %d",
 			  cmd_pos->data.macs_num, cnt);
 
 		/* Break if we reached the maximum
@@ -2602,7 +2603,8 @@ static int ecore_mcast_handle_pending_cmds_e2(struct bnx2x_softc *sc, struct
 			break;
 
 		default:
-			PMD_DRV_LOG(ERR, "Unknown command: %d", cmd_pos->type);
+			PMD_DRV_LOG(ERR, sc,
+				    "Unknown command: %d", cmd_pos->type);
 			return ECORE_INVAL;
 		}
 
@@ -2639,8 +2641,8 @@ static void ecore_mcast_hdl_add(struct bnx2x_softc *sc,
 
 		cnt++;
 
-		ECORE_MSG
-		    ("About to configure %02x:%02x:%02x:%02x:%02x:%02x mcast MAC",
+		    ECORE_MSG
+		    (sc, "About to configure %02x:%02x:%02x:%02x:%02x:%02x mcast MAC",
 		     mlist_pos->mac[0], mlist_pos->mac[1], mlist_pos->mac[2],
 		     mlist_pos->mac[3], mlist_pos->mac[4], mlist_pos->mac[5]);
 	}
@@ -2660,7 +2662,8 @@ static void ecore_mcast_hdl_del(struct bnx2x_softc *sc,
 
 		cnt++;
 
-		ECORE_MSG("Deleting MAC. %d left", p->mcast_list_len - i - 1);
+		ECORE_MSG(sc,
+			  "Deleting MAC. %d left", p->mcast_list_len - i - 1);
 	}
 
 	*line_idx = cnt;
@@ -2686,7 +2689,7 @@ static int ecore_mcast_handle_current_cmd(struct bnx2x_softc *sc, struct
 	struct ecore_mcast_obj *o = p->mcast_obj;
 	int cnt = start_cnt;
 
-	ECORE_MSG("p->mcast_list_len=%d", p->mcast_list_len);
+	ECORE_MSG(sc, "p->mcast_list_len=%d", p->mcast_list_len);
 
 	switch (cmd) {
 	case ECORE_MCAST_CMD_ADD:
@@ -2702,7 +2705,7 @@ static int ecore_mcast_handle_current_cmd(struct bnx2x_softc *sc, struct
 		break;
 
 	default:
-		PMD_DRV_LOG(ERR, "Unknown command: %d", cmd);
+		PMD_DRV_LOG(ERR, sc, "Unknown command: %d", cmd);
 		return ECORE_INVAL;
 	}
 
@@ -2747,7 +2750,7 @@ static int ecore_mcast_validate_e2(__rte_unused struct bnx2x_softc *sc,
 		break;
 
 	default:
-		PMD_DRV_LOG(ERR, "Unknown command: %d", cmd);
+		PMD_DRV_LOG(ERR, sc, "Unknown command: %d", cmd);
 		return ECORE_INVAL;
 	}
 
@@ -2933,8 +2936,8 @@ static void ecore_mcast_hdl_add_e1h(struct bnx2x_softc *sc __rte_unused,
 		bit = ecore_mcast_bin_from_mac(mlist_pos->mac);
 		ECORE_57711_SET_MC_FILTER(mc_filter, bit);
 
-		ECORE_MSG
-		    ("About to configure %02x:%02x:%02x:%02x:%02x:%02x mcast MAC, bin %d",
+		    ECORE_MSG
+		    (sc, "About to configure %02x:%02x:%02x:%02x:%02x:%02x mcast MAC, bin %d",
 		     mlist_pos->mac[0], mlist_pos->mac[1], mlist_pos->mac[2],
 		     mlist_pos->mac[3], mlist_pos->mac[4], mlist_pos->mac[5],
 		     bit);
@@ -2954,7 +2957,7 @@ static void ecore_mcast_hdl_restore_e1h(struct bnx2x_softc *sc
 	for (bit = ecore_mcast_get_next_bin(o, 0);
 	     bit >= 0; bit = ecore_mcast_get_next_bin(o, bit + 1)) {
 		ECORE_57711_SET_MC_FILTER(mc_filter, bit);
-		ECORE_MSG("About to set bin %d", bit);
+		ECORE_MSG(sc, "About to set bin %d", bit);
 	}
 }
 
@@ -2985,7 +2988,7 @@ static int ecore_mcast_setup_e1h(struct bnx2x_softc *sc,
 			break;
 
 		case ECORE_MCAST_CMD_DEL:
-			ECORE_MSG("Invalidating multicast MACs configuration");
+			ECORE_MSG(sc, "Invalidating multicast MACs configuration");
 
 			/* clear the registry */
 			ECORE_MEMSET(o->registry.aprox_match.vec, 0,
@@ -2997,7 +3000,7 @@ static int ecore_mcast_setup_e1h(struct bnx2x_softc *sc,
 			break;
 
 		default:
-			PMD_DRV_LOG(ERR, "Unknown command: %d", cmd);
+			PMD_DRV_LOG(ERR, sc, "Unknown command: %d", cmd);
 			return ECORE_INVAL;
 		}
 
@@ -3048,8 +3051,8 @@ int ecore_config_mcast(struct bnx2x_softc *sc,
 	if ((!p->mcast_list_len) && (!o->check_sched(o)))
 		return ECORE_SUCCESS;
 
-	ECORE_MSG
-	    ("o->total_pending_num=%d p->mcast_list_len=%d o->max_cmd_len=%d",
+	    ECORE_MSG
+	    (sc, "o->total_pending_num=%d p->mcast_list_len=%d o->max_cmd_len=%d",
 	     o->total_pending_num, p->mcast_list_len, o->max_cmd_len);
 
 	/* Enqueue the current command to the pending list if we can't complete
@@ -3478,7 +3481,7 @@ static int ecore_setup_rss(struct bnx2x_softc *sc,
 
 	ECORE_MEMSET(data, 0, sizeof(*data));
 
-	ECORE_MSG("Configuring RSS");
+	ECORE_MSG(sc, "Configuring RSS");
 
 	/* Set an echo field */
 	data->echo = ECORE_CPU_TO_LE32((r->cid & ECORE_SWCID_MASK) |
@@ -3492,7 +3495,7 @@ static int ecore_setup_rss(struct bnx2x_softc *sc,
 
 	data->rss_mode = rss_mode;
 
-	ECORE_MSG("rss_mode=%d", rss_mode);
+	ECORE_MSG(sc, "rss_mode=%d", rss_mode);
 
 	/* RSS capabilities */
 	if (ECORE_TEST_BIT(ECORE_RSS_IPV4, &p->rss_flags))
@@ -3532,7 +3535,7 @@ static int ecore_setup_rss(struct bnx2x_softc *sc,
 	/* RSS engine ID */
 	data->rss_engine_id = o->engine_id;
 
-	ECORE_MSG("rss_engine_id=%d", data->rss_engine_id);
+	ECORE_MSG(sc, "rss_engine_id=%d", data->rss_engine_id);
 
 	/* Indirection table */
 	ECORE_MEMCPY(data->indirection_table, p->ind_table,
@@ -3627,15 +3630,15 @@ int ecore_queue_state_change(struct bnx2x_softc *sc,
 	/* Check that the requested transition is legal */
 	rc = o->check_transition(sc, o, params);
 	if (rc) {
-		PMD_DRV_LOG(ERR, "check transition returned an error. rc %d",
+		PMD_DRV_LOG(ERR, sc, "check transition returned an error. rc %d",
 			    rc);
 		return ECORE_INVAL;
 	}
 
 	/* Set "pending" bit */
-	ECORE_MSG("pending bit was=%lx", o->pending);
+	ECORE_MSG(sc, "pending bit was=%lx", o->pending);
 	pending_bit = o->set_pending(o, params);
-	ECORE_MSG("pending bit now=%lx", o->pending);
+	ECORE_MSG(sc, "pending bit now=%lx", o->pending);
 
 	/* Don't send a command if only driver cleanup was requested */
 	if (ECORE_TEST_BIT(RAMROD_DRV_CLR_ONLY, &params->ramrod_flags))
@@ -3702,7 +3705,7 @@ static int ecore_queue_comp_cmd(struct bnx2x_softc *sc __rte_unused,
 	unsigned long cur_pending = o->pending;
 
 	if (!ECORE_TEST_AND_CLEAR_BIT(cmd, &cur_pending)) {
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "Bad MC reply %d for queue %d in state %d pending 0x%lx, next_state %d",
 			    cmd, o->cids[ECORE_PRIMARY_CID_INDEX], o->state,
 			    cur_pending, o->next_state);
@@ -3713,15 +3716,15 @@ static int ecore_queue_comp_cmd(struct bnx2x_softc *sc __rte_unused,
 		/* >= because tx only must always be smaller than cos since the
 		 * primary connection supports COS 0
 		 */
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "illegal value for next tx_only: %d. max cos was %d",
 			    o->next_tx_only, o->max_cos);
 
-	ECORE_MSG("Completing command %d for queue %d, setting state to %d",
+	ECORE_MSG(sc, "Completing command %d for queue %d, setting state to %d",
 		  cmd, o->cids[ECORE_PRIMARY_CID_INDEX], o->next_state);
 
 	if (o->next_tx_only)	/* print num tx-only if any exist */
-		ECORE_MSG("primary cid %d: num tx-only cons %d",
+		ECORE_MSG(sc, "primary cid %d: num tx-only cons %d",
 			  o->cids[ECORE_PRIMARY_CID_INDEX], o->next_tx_only);
 
 	o->state = o->next_state;
@@ -3782,7 +3785,7 @@ static void ecore_q_fill_init_general_data(struct bnx2x_softc *sc __rte_unused,
 	    ECORE_TEST_BIT(ECORE_Q_FLG_FCOE, flags) ?
 	    LLFC_TRAFFIC_TYPE_FCOE : LLFC_TRAFFIC_TYPE_NW;
 
-	ECORE_MSG("flags: active %d, cos %d, stats en %d",
+	ECORE_MSG(sc, "flags: active %d, cos %d, stats en %d",
 		  gen_data->activate_flg, gen_data->cos,
 		  gen_data->statistics_en_flg);
 }
@@ -3923,7 +3926,7 @@ static void ecore_q_fill_setup_tx_only(struct bnx2x_softc *sc, struct ecore_queu
 	ecore_q_fill_init_tx_data(&cmd_params->params.tx_only.txq_params,
 				  &data->tx, &cmd_params->params.tx_only.flags);
 
-	ECORE_MSG("cid %d, tx bd page lo %x hi %x",
+	ECORE_MSG(sc, "cid %d, tx bd page lo %x hi %x",
 		  cmd_params->q_obj->cids[0],
 		  data->tx.tx_bd_page_base.lo, data->tx.tx_bd_page_base.hi);
 }
@@ -3973,9 +3976,9 @@ static int ecore_q_init(struct bnx2x_softc *sc,
 
 	/* Set CDU context validation values */
 	for (cos = 0; cos < o->max_cos; cos++) {
-		ECORE_MSG("setting context validation. cid %d, cos %d",
+		ECORE_MSG(sc, "setting context validation. cid %d, cos %d",
 			  o->cids[cos], cos);
-		ECORE_MSG("context pointer %p", init->cxts[cos]);
+		ECORE_MSG(sc, "context pointer %p", init->cxts[cos]);
 		ECORE_SET_CTX_VALIDATION(sc, init->cxts[cos], o->cids[cos]);
 	}
 
@@ -4059,15 +4062,15 @@ static int ecore_q_send_setup_tx_only(struct bnx2x_softc *sc, struct ecore_queue
 
 	if (ECORE_TEST_BIT(ECORE_Q_TYPE_FWD, &o->type))
 		ramrod = RAMROD_CMD_ID_ETH_FORWARD_SETUP;
-	ECORE_MSG("sending forward tx-only ramrod");
+	ECORE_MSG(sc, "sending forward tx-only ramrod");
 
 	if (cid_index >= o->max_cos) {
-		PMD_DRV_LOG(ERR, "queue[%d]: cid_index (%d) is out of range",
+		PMD_DRV_LOG(ERR, sc, "queue[%d]: cid_index (%d) is out of range",
 			    o->cl_id, cid_index);
 		return ECORE_INVAL;
 	}
 
-	ECORE_MSG("parameters received: cos: %d sp-id: %d",
+	ECORE_MSG(sc, "parameters received: cos: %d sp-id: %d",
 		  tx_only_params->gen_params.cos,
 		  tx_only_params->gen_params.spcl_id);
 
@@ -4077,8 +4080,8 @@ static int ecore_q_send_setup_tx_only(struct bnx2x_softc *sc, struct ecore_queue
 	/* Fill the ramrod data */
 	ecore_q_fill_setup_tx_only(sc, params, rdata);
 
-	ECORE_MSG
-	    ("sending tx-only ramrod: cid %d, client-id %d, sp-client id %d, cos %d",
+	    ECORE_MSG
+	    (sc, "sending tx-only ramrod: cid %d, client-id %d, sp-client id %d, cos %d",
 	     o->cids[cid_index], rdata->general.client_id,
 	     rdata->general.sp_client_id, rdata->general.cos);
 
@@ -4173,7 +4176,7 @@ static int ecore_q_send_update(struct bnx2x_softc *sc,
 	uint8_t cid_index = update_params->cid_index;
 
 	if (cid_index >= o->max_cos) {
-		PMD_DRV_LOG(ERR, "queue[%d]: cid_index (%d) is out of range",
+		PMD_DRV_LOG(ERR, sc, "queue[%d]: cid_index (%d) is out of range",
 			    o->cl_id, cid_index);
 		return ECORE_INVAL;
 	}
@@ -4267,7 +4270,7 @@ static int ecore_q_send_cfc_del(struct bnx2x_softc *sc,
 	uint8_t cid_idx = params->params.cfc_del.cid_index;
 
 	if (cid_idx >= o->max_cos) {
-		PMD_DRV_LOG(ERR, "queue[%d]: cid_index (%d) is out of range",
+		PMD_DRV_LOG(ERR, sc, "queue[%d]: cid_index (%d) is out of range",
 			    o->cl_id, cid_idx);
 		return ECORE_INVAL;
 	}
@@ -4283,7 +4286,7 @@ static int ecore_q_send_terminate(struct bnx2x_softc *sc, struct ecore_queue_sta
 	uint8_t cid_index = params->params.terminate.cid_index;
 
 	if (cid_index >= o->max_cos) {
-		PMD_DRV_LOG(ERR, "queue[%d]: cid_index (%d) is out of range",
+		PMD_DRV_LOG(ERR, sc, "queue[%d]: cid_index (%d) is out of range",
 			    o->cl_id, cid_index);
 		return ECORE_INVAL;
 	}
@@ -4327,7 +4330,7 @@ static int ecore_queue_send_cmd_cmn(struct bnx2x_softc *sc, struct ecore_queue_s
 	case ECORE_Q_CMD_EMPTY:
 		return ecore_q_send_empty(sc, params);
 	default:
-		PMD_DRV_LOG(ERR, "Unknown command: %d", params->cmd);
+		PMD_DRV_LOG(ERR, sc, "Unknown command: %d", params->cmd);
 		return ECORE_INVAL;
 	}
 }
@@ -4350,7 +4353,7 @@ static int ecore_queue_send_cmd_e1x(struct bnx2x_softc *sc,
 	case ECORE_Q_CMD_EMPTY:
 		return ecore_queue_send_cmd_cmn(sc, params);
 	default:
-		PMD_DRV_LOG(ERR, "Unknown command: %d", params->cmd);
+		PMD_DRV_LOG(ERR, sc, "Unknown command: %d", params->cmd);
 		return ECORE_INVAL;
 	}
 }
@@ -4373,7 +4376,7 @@ static int ecore_queue_send_cmd_e2(struct bnx2x_softc *sc,
 	case ECORE_Q_CMD_EMPTY:
 		return ecore_queue_send_cmd_cmn(sc, params);
 	default:
-		PMD_DRV_LOG(ERR, "Unknown command: %d", params->cmd);
+		PMD_DRV_LOG(ERR, sc, "Unknown command: %d", params->cmd);
 		return ECORE_INVAL;
 	}
 }
@@ -4416,7 +4419,7 @@ static int ecore_queue_chk_transition(struct bnx2x_softc *sc __rte_unused,
 	 * the previous one.
 	 */
 	if (o->pending) {
-		PMD_DRV_LOG(ERR, "Blocking transition since pending was %lx",
+		PMD_DRV_LOG(ERR, sc, "Blocking transition since pending was %lx",
 			    o->pending);
 		return ECORE_BUSY;
 	}
@@ -4543,19 +4546,19 @@ static int ecore_queue_chk_transition(struct bnx2x_softc *sc __rte_unused,
 
 		break;
 	default:
-		PMD_DRV_LOG(ERR, "Illegal state: %d", state);
+		PMD_DRV_LOG(ERR, sc, "Illegal state: %d", state);
 	}
 
 	/* Transition is assured */
 	if (next_state != ECORE_Q_STATE_MAX) {
-		ECORE_MSG("Good state transition: %d(%d)->%d",
+		ECORE_MSG(sc, "Good state transition: %d(%d)->%d",
 			  state, cmd, next_state);
 		o->next_state = next_state;
 		o->next_tx_only = next_tx_only;
 		return ECORE_SUCCESS;
 	}
 
-	ECORE_MSG("Bad state transition request: %d %d", state, cmd);
+	ECORE_MSG(sc, "Bad state transition request: %d %d", state, cmd);
 
 	return ECORE_INVAL;
 }
@@ -4606,18 +4609,18 @@ static int ecore_queue_chk_fwd_transition(struct bnx2x_softc *sc __rte_unused,
 
 		break;
 	default:
-		PMD_DRV_LOG(ERR, "Illegal state: %d", state);
+		PMD_DRV_LOG(ERR, sc, "Illegal state: %d", state);
 	}
 
 	/* Transition is assured */
 	if (next_state != ECORE_Q_STATE_MAX) {
-		ECORE_MSG("Good state transition: %d(%d)->%d",
+		ECORE_MSG(sc, "Good state transition: %d(%d)->%d",
 			  state, cmd, next_state);
 		o->next_state = next_state;
 		return ECORE_SUCCESS;
 	}
 
-	ECORE_MSG("Bad state transition request: %d %d", state, cmd);
+	ECORE_MSG(sc, "Bad state transition request: %d %d", state, cmd);
 	return ECORE_INVAL;
 }
 
@@ -4697,14 +4700,14 @@ ecore_func_state_change_comp(struct bnx2x_softc *sc __rte_unused,
 	unsigned long cur_pending = o->pending;
 
 	if (!ECORE_TEST_AND_CLEAR_BIT(cmd, &cur_pending)) {
-		PMD_DRV_LOG(ERR,
+		PMD_DRV_LOG(ERR, sc,
 			    "Bad MC reply %d for func %d in state %d pending 0x%lx, next_state %d",
 			    cmd, ECORE_FUNC_ID(sc), o->state, cur_pending,
 			    o->next_state);
 		return ECORE_INVAL;
 	}
 
-	ECORE_MSG("Completing command %d for func %d, setting state to %d",
+	ECORE_MSG(sc, "Completing command %d for func %d, setting state to %d",
 		  cmd, ECORE_FUNC_ID(sc), o->next_state);
 
 	o->state = o->next_state;
@@ -4827,18 +4830,19 @@ static int ecore_func_chk_transition(struct bnx2x_softc *sc __rte_unused,
 
 		break;
 	default:
-		PMD_DRV_LOG(ERR, "Unknown state: %d", state);
+		PMD_DRV_LOG(ERR, sc, "Unknown state: %d", state);
 	}
 
 	/* Transition is assured */
 	if (next_state != ECORE_F_STATE_MAX) {
-		ECORE_MSG("Good function state transition: %d(%d)->%d",
+		ECORE_MSG(sc, "Good function state transition: %d(%d)->%d",
 			  state, cmd, next_state);
 		o->next_state = next_state;
 		return ECORE_SUCCESS;
 	}
 
-	ECORE_MSG("Bad function state transition request: %d %d", state, cmd);
+	ECORE_MSG(sc,
+		  "Bad function state transition request: %d %d", state, cmd);
 
 	return ECORE_INVAL;
 }
@@ -4928,13 +4932,13 @@ static int ecore_func_hw_init(struct bnx2x_softc *sc,
 	const struct ecore_func_sp_drv_ops *drv = o->drv;
 	int rc = 0;
 
-	ECORE_MSG("function %d  load_code %x",
+	ECORE_MSG(sc, "function %d  load_code %x",
 		  ECORE_ABS_FUNC_ID(sc), load_code);
 
 	/* Prepare FW */
 	rc = drv->init_fw(sc);
 	if (rc) {
-		PMD_DRV_LOG(ERR, "Error loading firmware");
+		PMD_DRV_LOG(ERR, sc, "Error loading firmware");
 		goto init_err;
 	}
 
@@ -4965,7 +4969,7 @@ static int ecore_func_hw_init(struct bnx2x_softc *sc,
 
 		break;
 	default:
-		PMD_DRV_LOG(ERR, "Unknown load_code (0x%x) from MCP",
+		PMD_DRV_LOG(ERR, sc, "Unknown load_code (0x%x) from MCP",
 			    load_code);
 		rc = ECORE_INVAL;
 	}
@@ -5041,7 +5045,7 @@ static int ecore_func_hw_reset(struct bnx2x_softc *sc,
 	struct ecore_func_sp_obj *o = params->f_obj;
 	const struct ecore_func_sp_drv_ops *drv = o->drv;
 
-	ECORE_MSG("function %d  reset_phase %x", ECORE_ABS_FUNC_ID(sc),
+	ECORE_MSG(sc, "function %d  reset_phase %x", ECORE_ABS_FUNC_ID(sc),
 		  reset_phase);
 
 	switch (reset_phase) {
@@ -5055,7 +5059,7 @@ static int ecore_func_hw_reset(struct bnx2x_softc *sc,
 		ecore_func_reset_func(sc, drv);
 		break;
 	default:
-		PMD_DRV_LOG(ERR, "Unknown reset_phase (0x%x) from MCP",
+		PMD_DRV_LOG(ERR, sc, "Unknown reset_phase (0x%x) from MCP",
 			    reset_phase);
 		break;
 	}
@@ -5146,7 +5150,7 @@ static int ecore_func_send_afex_update(struct bnx2x_softc *sc, struct ecore_func
 	 *  read and we will have to put a full memory barrier there
 	 *  (inside ecore_sp_post()).
 	 */
-	ECORE_MSG("afex: sending func_update vif_id 0x%x dvlan 0x%x prio 0x%x",
+	ECORE_MSG(sc, "afex: sending func_update vif_id 0x%x dvlan 0x%x prio 0x%x",
 		  rdata->vif_id,
 		  rdata->afex_default_vlan, rdata->allowed_priorities);
 
@@ -5184,8 +5188,8 @@ inline int ecore_func_send_afex_viflists(struct bnx2x_softc *sc,
 	 *  (inside ecore_sp_post()).
 	 */
 
-	ECORE_MSG
-	    ("afex: ramrod lists, cmd 0x%x index 0x%x func_bit_map 0x%x func_to_clr 0x%x",
+	    ECORE_MSG
+	    (sc, "afex: ramrod lists, cmd 0x%x index 0x%x func_bit_map 0x%x func_to_clr 0x%x",
 	     rdata->afex_vif_list_command, rdata->vif_list_index,
 	     rdata->func_bit_map, rdata->func_to_clear);
 
@@ -5256,7 +5260,7 @@ static int ecore_func_send_cmd(struct bnx2x_softc *sc,
 	case ECORE_F_CMD_SWITCH_UPDATE:
 		return ecore_func_send_switch_update(sc, params);
 	default:
-		PMD_DRV_LOG(ERR, "Unknown command: %d", params->cmd);
+		PMD_DRV_LOG(ERR, sc, "Unknown command: %d", params->cmd);
 		return ECORE_INVAL;
 	}
 }
@@ -5317,7 +5321,7 @@ int ecore_func_state_change(struct bnx2x_softc *sc,
 		}
 		if (rc == ECORE_BUSY) {
 			ECORE_MUTEX_UNLOCK(&o->one_pending_mutex);
-			PMD_DRV_LOG(ERR,
+			PMD_DRV_LOG(ERR, sc,
 				    "timeout waiting for previous ramrod completion");
 			return rc;
 		}
diff --git a/drivers/net/bnx2x/ecore_sp.h b/drivers/net/bnx2x/ecore_sp.h
index 6b65a496..f295bf5a 100644
--- a/drivers/net/bnx2x/ecore_sp.h
+++ b/drivers/net/bnx2x/ecore_sp.h
@@ -215,8 +215,8 @@ ECORE_CRC32_LE(uint32_t seed, uint8_t *mac, uint32_t len)
     } while (0)
 
 
-#define ECORE_MSG(m, ...) \
-	PMD_DRV_LOG(DEBUG, m, ##__VA_ARGS__)
+#define ECORE_MSG(sc, m, ...) \
+	PMD_DRV_LOG(DEBUG, sc, m, ##__VA_ARGS__)
 
 typedef struct _ecore_list_entry_t
 {
diff --git a/drivers/net/bnx2x/elink.c b/drivers/net/bnx2x/elink.c
index b63fd23e..d5693668 100644
--- a/drivers/net/bnx2x/elink.c
+++ b/drivers/net/bnx2x/elink.c
@@ -18,166 +18,159 @@
 #include "ecore_hsi.h"
 #include "ecore_reg.h"
 
-static elink_status_t elink_link_reset(struct elink_params *params,
-				       struct elink_vars *vars,
-				       uint8_t reset_ext_phy);
-static elink_status_t elink_check_half_open_conn(struct elink_params *params,
-						 struct elink_vars *vars,
-						 uint8_t notify);
-static elink_status_t elink_sfp_module_detection(struct elink_phy *phy,
-						 struct elink_params *params);
 
 #define MDIO_REG_BANK_CL73_IEEEB0			0x0
-#define MDIO_CL73_IEEEB0_CL73_AN_CONTROL		0x0
-#define MDIO_CL73_IEEEB0_CL73_AN_CONTROL_RESTART_AN	0x0200
-#define MDIO_CL73_IEEEB0_CL73_AN_CONTROL_AN_EN		0x1000
-#define MDIO_CL73_IEEEB0_CL73_AN_CONTROL_MAIN_RST	0x8000
+	#define MDIO_CL73_IEEEB0_CL73_AN_CONTROL		0x0
+		#define MDIO_CL73_IEEEB0_CL73_AN_CONTROL_RESTART_AN	0x0200
+		#define MDIO_CL73_IEEEB0_CL73_AN_CONTROL_AN_EN		0x1000
+		#define MDIO_CL73_IEEEB0_CL73_AN_CONTROL_MAIN_RST	0x8000
 
 #define MDIO_REG_BANK_CL73_IEEEB1			0x10
-#define MDIO_CL73_IEEEB1_AN_ADV1			0x00
-#define	MDIO_CL73_IEEEB1_AN_ADV1_PAUSE			0x0400
-#define	MDIO_CL73_IEEEB1_AN_ADV1_ASYMMETRIC 		0x0800
-#define	MDIO_CL73_IEEEB1_AN_ADV1_PAUSE_BOTH		0x0C00
-#define	MDIO_CL73_IEEEB1_AN_ADV1_PAUSE_MASK		0x0C00
-#define MDIO_CL73_IEEEB1_AN_ADV2				0x01
-#define MDIO_CL73_IEEEB1_AN_ADV2_ADVR_1000M		0x0000
-#define MDIO_CL73_IEEEB1_AN_ADV2_ADVR_1000M_KX		0x0020
-#define MDIO_CL73_IEEEB1_AN_ADV2_ADVR_10G_KX4		0x0040
-#define MDIO_CL73_IEEEB1_AN_ADV2_ADVR_10G_KR		0x0080
-#define	MDIO_CL73_IEEEB1_AN_LP_ADV1			0x03
-#define	MDIO_CL73_IEEEB1_AN_LP_ADV1_PAUSE		0x0400
-#define	MDIO_CL73_IEEEB1_AN_LP_ADV1_ASYMMETRIC 		0x0800
-#define	MDIO_CL73_IEEEB1_AN_LP_ADV1_PAUSE_BOTH		0x0C00
-#define	MDIO_CL73_IEEEB1_AN_LP_ADV1_PAUSE_MASK		0x0C00
-#define	MDIO_CL73_IEEEB1_AN_LP_ADV2			0x04
+	#define MDIO_CL73_IEEEB1_AN_ADV1			0x00
+		#define	MDIO_CL73_IEEEB1_AN_ADV1_PAUSE			0x0400
+		#define	MDIO_CL73_IEEEB1_AN_ADV1_ASYMMETRIC		0x0800
+		#define	MDIO_CL73_IEEEB1_AN_ADV1_PAUSE_BOTH		0x0C00
+		#define	MDIO_CL73_IEEEB1_AN_ADV1_PAUSE_MASK		0x0C00
+	#define MDIO_CL73_IEEEB1_AN_ADV2				0x01
+		#define MDIO_CL73_IEEEB1_AN_ADV2_ADVR_1000M		0x0000
+		#define MDIO_CL73_IEEEB1_AN_ADV2_ADVR_1000M_KX		0x0020
+		#define MDIO_CL73_IEEEB1_AN_ADV2_ADVR_10G_KX4		0x0040
+		#define MDIO_CL73_IEEEB1_AN_ADV2_ADVR_10G_KR		0x0080
+	#define	MDIO_CL73_IEEEB1_AN_LP_ADV1			0x03
+		#define	MDIO_CL73_IEEEB1_AN_LP_ADV1_PAUSE		0x0400
+		#define	MDIO_CL73_IEEEB1_AN_LP_ADV1_ASYMMETRIC		0x0800
+		#define	MDIO_CL73_IEEEB1_AN_LP_ADV1_PAUSE_BOTH		0x0C00
+		#define	MDIO_CL73_IEEEB1_AN_LP_ADV1_PAUSE_MASK		0x0C00
+	#define	MDIO_CL73_IEEEB1_AN_LP_ADV2			0x04
 
 #define	MDIO_REG_BANK_RX0				0x80b0
-#define	MDIO_RX0_RX_STATUS				0x10
-#define	MDIO_RX0_RX_STATUS_SIGDET			0x8000
-#define	MDIO_RX0_RX_STATUS_RX_SEQ_DONE			0x1000
-#define	MDIO_RX0_RX_EQ_BOOST				0x1c
-#define	MDIO_RX0_RX_EQ_BOOST_EQUALIZER_CTRL_MASK	0x7
-#define	MDIO_RX0_RX_EQ_BOOST_OFFSET_CTRL		0x10
+	#define	MDIO_RX0_RX_STATUS				0x10
+		#define	MDIO_RX0_RX_STATUS_SIGDET			0x8000
+		#define	MDIO_RX0_RX_STATUS_RX_SEQ_DONE			0x1000
+	#define	MDIO_RX0_RX_EQ_BOOST				0x1c
+		#define	MDIO_RX0_RX_EQ_BOOST_EQUALIZER_CTRL_MASK	0x7
+		#define	MDIO_RX0_RX_EQ_BOOST_OFFSET_CTRL		0x10
 
 #define	MDIO_REG_BANK_RX1				0x80c0
-#define	MDIO_RX1_RX_EQ_BOOST				0x1c
-#define	MDIO_RX1_RX_EQ_BOOST_EQUALIZER_CTRL_MASK	0x7
-#define	MDIO_RX1_RX_EQ_BOOST_OFFSET_CTRL		0x10
+	#define	MDIO_RX1_RX_EQ_BOOST				0x1c
+		#define	MDIO_RX1_RX_EQ_BOOST_EQUALIZER_CTRL_MASK	0x7
+		#define	MDIO_RX1_RX_EQ_BOOST_OFFSET_CTRL		0x10
 
 #define	MDIO_REG_BANK_RX2				0x80d0
-#define	MDIO_RX2_RX_EQ_BOOST				0x1c
-#define	MDIO_RX2_RX_EQ_BOOST_EQUALIZER_CTRL_MASK	0x7
-#define	MDIO_RX2_RX_EQ_BOOST_OFFSET_CTRL		0x10
+	#define	MDIO_RX2_RX_EQ_BOOST				0x1c
+		#define	MDIO_RX2_RX_EQ_BOOST_EQUALIZER_CTRL_MASK	0x7
+		#define	MDIO_RX2_RX_EQ_BOOST_OFFSET_CTRL		0x10
 
 #define	MDIO_REG_BANK_RX3				0x80e0
-#define	MDIO_RX3_RX_EQ_BOOST				0x1c
-#define	MDIO_RX3_RX_EQ_BOOST_EQUALIZER_CTRL_MASK	0x7
-#define	MDIO_RX3_RX_EQ_BOOST_OFFSET_CTRL		0x10
+	#define	MDIO_RX3_RX_EQ_BOOST				0x1c
+		#define	MDIO_RX3_RX_EQ_BOOST_EQUALIZER_CTRL_MASK	0x7
+		#define	MDIO_RX3_RX_EQ_BOOST_OFFSET_CTRL		0x10
 
 #define	MDIO_REG_BANK_RX_ALL				0x80f0
-#define	MDIO_RX_ALL_RX_EQ_BOOST				0x1c
-#define	MDIO_RX_ALL_RX_EQ_BOOST_EQUALIZER_CTRL_MASK	0x7
-#define	MDIO_RX_ALL_RX_EQ_BOOST_OFFSET_CTRL	0x10
+	#define	MDIO_RX_ALL_RX_EQ_BOOST				0x1c
+		#define	MDIO_RX_ALL_RX_EQ_BOOST_EQUALIZER_CTRL_MASK	0x7
+		#define	MDIO_RX_ALL_RX_EQ_BOOST_OFFSET_CTRL	0x10
 
 #define	MDIO_REG_BANK_TX0				0x8060
-#define	MDIO_TX0_TX_DRIVER				0x17
-#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_MASK		0xf000
-#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_SHIFT		12
-#define	MDIO_TX0_TX_DRIVER_IDRIVER_MASK			0x0f00
-#define	MDIO_TX0_TX_DRIVER_IDRIVER_SHIFT		8
-#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_MASK		0x00f0
-#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_SHIFT		4
-#define	MDIO_TX0_TX_DRIVER_IFULLSPD_MASK		0x000e
-#define	MDIO_TX0_TX_DRIVER_IFULLSPD_SHIFT		1
-#define	MDIO_TX0_TX_DRIVER_ICBUF1T			1
+	#define	MDIO_TX0_TX_DRIVER				0x17
+		#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_MASK		0xf000
+		#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_SHIFT		12
+		#define	MDIO_TX0_TX_DRIVER_IDRIVER_MASK			0x0f00
+		#define	MDIO_TX0_TX_DRIVER_IDRIVER_SHIFT		8
+		#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_MASK		0x00f0
+		#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_SHIFT		4
+		#define	MDIO_TX0_TX_DRIVER_IFULLSPD_MASK		0x000e
+		#define	MDIO_TX0_TX_DRIVER_IFULLSPD_SHIFT		1
+		#define	MDIO_TX0_TX_DRIVER_ICBUF1T			1
 
 #define	MDIO_REG_BANK_TX1				0x8070
-#define	MDIO_TX1_TX_DRIVER				0x17
-#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_MASK		0xf000
-#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_SHIFT		12
-#define	MDIO_TX0_TX_DRIVER_IDRIVER_MASK			0x0f00
-#define	MDIO_TX0_TX_DRIVER_IDRIVER_SHIFT		8
-#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_MASK		0x00f0
-#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_SHIFT		4
-#define	MDIO_TX0_TX_DRIVER_IFULLSPD_MASK		0x000e
-#define	MDIO_TX0_TX_DRIVER_IFULLSPD_SHIFT		1
-#define	MDIO_TX0_TX_DRIVER_ICBUF1T			1
+	#define	MDIO_TX1_TX_DRIVER				0x17
+		#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_MASK		0xf000
+		#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_SHIFT		12
+		#define	MDIO_TX0_TX_DRIVER_IDRIVER_MASK			0x0f00
+		#define	MDIO_TX0_TX_DRIVER_IDRIVER_SHIFT		8
+		#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_MASK		0x00f0
+		#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_SHIFT		4
+		#define	MDIO_TX0_TX_DRIVER_IFULLSPD_MASK		0x000e
+		#define	MDIO_TX0_TX_DRIVER_IFULLSPD_SHIFT		1
+		#define	MDIO_TX0_TX_DRIVER_ICBUF1T			1
 
 #define	MDIO_REG_BANK_TX2				0x8080
-#define	MDIO_TX2_TX_DRIVER				0x17
-#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_MASK		0xf000
-#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_SHIFT		12
-#define	MDIO_TX0_TX_DRIVER_IDRIVER_MASK			0x0f00
-#define	MDIO_TX0_TX_DRIVER_IDRIVER_SHIFT		8
-#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_MASK		0x00f0
-#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_SHIFT		4
-#define	MDIO_TX0_TX_DRIVER_IFULLSPD_MASK		0x000e
-#define	MDIO_TX0_TX_DRIVER_IFULLSPD_SHIFT		1
-#define	MDIO_TX0_TX_DRIVER_ICBUF1T			1
+	#define	MDIO_TX2_TX_DRIVER				0x17
+		#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_MASK		0xf000
+		#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_SHIFT		12
+		#define	MDIO_TX0_TX_DRIVER_IDRIVER_MASK			0x0f00
+		#define	MDIO_TX0_TX_DRIVER_IDRIVER_SHIFT		8
+		#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_MASK		0x00f0
+		#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_SHIFT		4
+		#define	MDIO_TX0_TX_DRIVER_IFULLSPD_MASK		0x000e
+		#define	MDIO_TX0_TX_DRIVER_IFULLSPD_SHIFT		1
+		#define	MDIO_TX0_TX_DRIVER_ICBUF1T			1
 
 #define	MDIO_REG_BANK_TX3				0x8090
-#define	MDIO_TX3_TX_DRIVER				0x17
-#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_MASK		0xf000
-#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_SHIFT		12
-#define	MDIO_TX0_TX_DRIVER_IDRIVER_MASK			0x0f00
-#define	MDIO_TX0_TX_DRIVER_IDRIVER_SHIFT		8
-#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_MASK		0x00f0
-#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_SHIFT		4
-#define	MDIO_TX0_TX_DRIVER_IFULLSPD_MASK		0x000e
-#define	MDIO_TX0_TX_DRIVER_IFULLSPD_SHIFT		1
-#define	MDIO_TX0_TX_DRIVER_ICBUF1T			1
+	#define	MDIO_TX3_TX_DRIVER				0x17
+		#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_MASK		0xf000
+		#define	MDIO_TX0_TX_DRIVER_PREEMPHASIS_SHIFT		12
+		#define	MDIO_TX0_TX_DRIVER_IDRIVER_MASK			0x0f00
+		#define	MDIO_TX0_TX_DRIVER_IDRIVER_SHIFT		8
+		#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_MASK		0x00f0
+		#define	MDIO_TX0_TX_DRIVER_IPREDRIVER_SHIFT		4
+		#define	MDIO_TX0_TX_DRIVER_IFULLSPD_MASK		0x000e
+		#define	MDIO_TX0_TX_DRIVER_IFULLSPD_SHIFT		1
+		#define	MDIO_TX0_TX_DRIVER_ICBUF1T			1
 
 #define	MDIO_REG_BANK_XGXS_BLOCK0			0x8000
-#define	MDIO_BLOCK0_XGXS_CONTROL			0x10
+	#define	MDIO_BLOCK0_XGXS_CONTROL			0x10
 
 #define	MDIO_REG_BANK_XGXS_BLOCK1			0x8010
-#define	MDIO_BLOCK1_LANE_CTRL0				0x15
-#define	MDIO_BLOCK1_LANE_CTRL1				0x16
-#define	MDIO_BLOCK1_LANE_CTRL2				0x17
-#define	MDIO_BLOCK1_LANE_PRBS				0x19
+	#define	MDIO_BLOCK1_LANE_CTRL0				0x15
+	#define	MDIO_BLOCK1_LANE_CTRL1				0x16
+	#define	MDIO_BLOCK1_LANE_CTRL2				0x17
+	#define	MDIO_BLOCK1_LANE_PRBS				0x19
 
 #define	MDIO_REG_BANK_XGXS_BLOCK2			0x8100
-#define	MDIO_XGXS_BLOCK2_RX_LN_SWAP			0x10
-#define	MDIO_XGXS_BLOCK2_RX_LN_SWAP_ENABLE		0x8000
-#define	MDIO_XGXS_BLOCK2_RX_LN_SWAP_FORCE_ENABLE	0x4000
-#define	MDIO_XGXS_BLOCK2_TX_LN_SWAP		0x11
-#define	MDIO_XGXS_BLOCK2_TX_LN_SWAP_ENABLE		0x8000
-#define	MDIO_XGXS_BLOCK2_UNICORE_MODE_10G	0x14
-#define	MDIO_XGXS_BLOCK2_UNICORE_MODE_10G_CX4_XGXS	0x0001
-#define	MDIO_XGXS_BLOCK2_UNICORE_MODE_10G_HIGIG_XGXS	0x0010
-#define	MDIO_XGXS_BLOCK2_TEST_MODE_LANE		0x15
+	#define	MDIO_XGXS_BLOCK2_RX_LN_SWAP			0x10
+		#define	MDIO_XGXS_BLOCK2_RX_LN_SWAP_ENABLE		0x8000
+		#define	MDIO_XGXS_BLOCK2_RX_LN_SWAP_FORCE_ENABLE	0x4000
+		#define	MDIO_XGXS_BLOCK2_TX_LN_SWAP		0x11
+		#define	MDIO_XGXS_BLOCK2_TX_LN_SWAP_ENABLE		0x8000
+		#define	MDIO_XGXS_BLOCK2_UNICORE_MODE_10G	0x14
+		#define	MDIO_XGXS_BLOCK2_UNICORE_MODE_10G_CX4_XGXS	0x0001
+		#define	MDIO_XGXS_BLOCK2_UNICORE_MODE_10G_HIGIG_XGXS	0x0010
+		#define	MDIO_XGXS_BLOCK2_TEST_MODE_LANE		0x15
 
 #define	MDIO_REG_BANK_GP_STATUS				0x8120
 #define	MDIO_GP_STATUS_TOP_AN_STATUS1				0x1B
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_CL73_AUTONEG_COMPLETE	0x0001
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_CL37_AUTONEG_COMPLETE	0x0002
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_LINK_STATUS		0x0004
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_DUPLEX_STATUS		0x0008
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_CL73_MR_LP_NP_AN_ABLE	0x0010
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_CL73_LP_NP_BAM_ABLE	0x0020
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_PAUSE_RSOLUTION_TXSIDE	0x0040
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_PAUSE_RSOLUTION_RXSIDE	0x0080
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_MASK		0x3f00
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10M		0x0000
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_100M		0x0100
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_1G		0x0200
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_2_5G		0x0300
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_5G		0x0400
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_6G		0x0500
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_HIG	0x0600
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_CX4	0x0700
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_12G_HIG	0x0800
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_12_5G	0x0900
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_13G		0x0A00
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_15G		0x0B00
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_16G		0x0C00
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_1G_KX	0x0D00
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_KX4	0x0E00
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_KR	0x0F00
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_XFI	0x1B00
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_20G_DXGXS	0x1E00
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_SFI	0x1F00
-#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_20G_KR2	0x3900
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_CL73_AUTONEG_COMPLETE	0x0001
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_CL37_AUTONEG_COMPLETE	0x0002
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_LINK_STATUS		0x0004
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_DUPLEX_STATUS		0x0008
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_CL73_MR_LP_NP_AN_ABLE	0x0010
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_CL73_LP_NP_BAM_ABLE	0x0020
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_PAUSE_RSOLUTION_TXSIDE	0x0040
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_PAUSE_RSOLUTION_RXSIDE	0x0080
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_MASK		0x3f00
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10M		0x0000
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_100M		0x0100
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_1G		0x0200
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_2_5G		0x0300
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_5G		0x0400
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_6G		0x0500
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_HIG	0x0600
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_CX4	0x0700
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_12G_HIG	0x0800
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_12_5G	0x0900
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_13G		0x0A00
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_15G		0x0B00
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_16G		0x0C00
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_1G_KX	0x0D00
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_KX4	0x0E00
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_KR	0x0F00
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_XFI	0x1B00
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_20G_DXGXS	0x1E00
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_10G_SFI	0x1F00
+	#define	MDIO_GP_STATUS_TOP_AN_STATUS1_ACTUAL_SPEED_20G_KR2	0x3900
+
 
 #define	MDIO_REG_BANK_10G_PARALLEL_DETECT		0x8130
 #define	MDIO_10G_PARALLEL_DETECT_PAR_DET_10G_STATUS		0x10
@@ -320,6 +313,7 @@ bit15=link,bit12=duplex,bits11:10=speed,bit14=acknowledge.
 Theotherbitsarereservedandshouldbezero*/
 #define	MDIO_COMBO_IEEE0_AUTO_NEG_LINK_PARTNER_ABILITY1_SGMII_MODE	0x0001
 
+
 #define	MDIO_PMA_DEVAD			0x1
 /*ieee*/
 #define	MDIO_PMA_REG_CTRL		0x0
@@ -328,7 +322,7 @@ Theotherbitsarereservedandshouldbezero*/
 #define MDIO_PMA_REG_TX_DISABLE		0x0009
 #define	MDIO_PMA_REG_RX_SD		0xa
 /*bnx2x*/
-#define	MDIO_PMA_REG_BNX2X_CTRL		0x0096
+#define	MDIO_PMA_REG_BCM_CTRL		0x0096
 #define MDIO_PMA_REG_FEC_CTRL		0x00ab
 #define	MDIO_PMA_LASI_RXCTRL		0x9000
 #define	MDIO_PMA_LASI_TXCTRL		0x9001
@@ -343,8 +337,8 @@ Theotherbitsarereservedandshouldbezero*/
 #define	MDIO_PMA_REG_CMU_PLL_BYPASS	0xca09
 #define	MDIO_PMA_REG_MISC_CTRL		0xca0a
 #define	MDIO_PMA_REG_GEN_CTRL		0xca10
-#define	MDIO_PMA_REG_GEN_CTRL_ROM_RESET_INTERNAL_MP	0x0188
-#define	MDIO_PMA_REG_GEN_CTRL_ROM_MICRO_RESET		0x018a
+	#define	MDIO_PMA_REG_GEN_CTRL_ROM_RESET_INTERNAL_MP	0x0188
+	#define	MDIO_PMA_REG_GEN_CTRL_ROM_MICRO_RESET		0x018a
 #define	MDIO_PMA_REG_M8051_MSGIN_REG	0xca12
 #define	MDIO_PMA_REG_M8051_MSGOUT_REG	0xca13
 #define	MDIO_PMA_REG_ROM_VER1		0xca19
@@ -358,21 +352,21 @@ Theotherbitsarereservedandshouldbezero*/
 #define	MDIO_PMA_REG_MISC_CTRL1		0xca85
 
 #define MDIO_PMA_REG_SFP_TWO_WIRE_CTRL		0x8000
-#define MDIO_PMA_REG_SFP_TWO_WIRE_CTRL_STATUS_MASK 	0x000c
-#define MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_IDLE 		0x0000
-#define MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_COMPLETE 	0x0004
-#define MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_IN_PROGRESS 	0x0008
-#define MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_FAILED 	0x000c
+#define MDIO_PMA_REG_SFP_TWO_WIRE_CTRL_STATUS_MASK	0x000c
+#define MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_IDLE		0x0000
+#define MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_COMPLETE	0x0004
+#define MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_IN_PROGRESS	0x0008
+#define MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_FAILED		0x000c
 #define MDIO_PMA_REG_SFP_TWO_WIRE_BYTE_CNT 	0x8002
 #define MDIO_PMA_REG_SFP_TWO_WIRE_MEM_ADDR 	0x8003
 #define MDIO_PMA_REG_8726_TWO_WIRE_DATA_BUF	0xc820
-#define MDIO_PMA_REG_8726_TWO_WIRE_DATA_MASK 0xff
+	#define MDIO_PMA_REG_8726_TWO_WIRE_DATA_MASK 0xff
 #define MDIO_PMA_REG_8726_TX_CTRL1		0xca01
 #define MDIO_PMA_REG_8726_TX_CTRL2		0xca05
 
 #define MDIO_PMA_REG_8727_TWO_WIRE_SLAVE_ADDR	0x8005
 #define MDIO_PMA_REG_8727_TWO_WIRE_DATA_BUF	0x8007
-#define MDIO_PMA_REG_8727_TWO_WIRE_DATA_MASK 0xff
+	#define MDIO_PMA_REG_8727_TWO_WIRE_DATA_MASK 0xff
 #define MDIO_PMA_REG_8727_MISC_CTRL		0x8309
 #define MDIO_PMA_REG_8727_TX_CTRL1		0xca02
 #define MDIO_PMA_REG_8727_TX_CTRL2		0xca05
@@ -404,6 +398,8 @@ Theotherbitsarereservedandshouldbezero*/
 #define MDIO_PMA_REG_8481_LINK_SIGNAL_LED4_ENABLE_MASK	0x800
 #define MDIO_PMA_REG_8481_LINK_SIGNAL_LED4_ENABLE_SHIFT	11
 
+
+
 #define	MDIO_WIS_DEVAD			0x2
 /*bnx2x*/
 #define	MDIO_WIS_REG_LASI_CNTL		0x9002
@@ -415,13 +411,15 @@ Theotherbitsarereservedandshouldbezero*/
 #define MDIO_PCS_REG_7101_DSP_ACCESS	0xD000
 #define MDIO_PCS_REG_7101_SPI_MUX 	0xD008
 #define MDIO_PCS_REG_7101_SPI_CTRL_ADDR 0xE12A
-#define MDIO_PCS_REG_7101_SPI_RESET_BIT (5)
+	#define MDIO_PCS_REG_7101_SPI_RESET_BIT (5)
 #define MDIO_PCS_REG_7101_SPI_FIFO_ADDR 0xE02A
-#define MDIO_PCS_REG_7101_SPI_FIFO_ADDR_WRITE_ENABLE_CMD (6)
-#define MDIO_PCS_REG_7101_SPI_FIFO_ADDR_BULK_ERASE_CMD   (0xC7)
-#define MDIO_PCS_REG_7101_SPI_FIFO_ADDR_PAGE_PROGRAM_CMD (2)
+	#define MDIO_PCS_REG_7101_SPI_FIFO_ADDR_WRITE_ENABLE_CMD (6)
+	#define MDIO_PCS_REG_7101_SPI_FIFO_ADDR_BULK_ERASE_CMD   (0xC7)
+	#define MDIO_PCS_REG_7101_SPI_FIFO_ADDR_PAGE_PROGRAM_CMD (2)
 #define MDIO_PCS_REG_7101_SPI_BYTES_TO_TRANSFER_ADDR 0xE028
 
+
+
 #define	MDIO_XS_DEVAD			0x4
 #define	MDIO_XS_REG_STATUS		0x0001
 #define MDIO_XS_PLL_SEQUENCER 		0x8000
@@ -439,12 +437,12 @@ Theotherbitsarereservedandshouldbezero*/
 /*ieee*/
 #define	MDIO_AN_REG_CTRL		0x0000
 #define	MDIO_AN_REG_STATUS		0x0001
-#define	MDIO_AN_REG_STATUS_AN_COMPLETE		0x0020
+	#define	MDIO_AN_REG_STATUS_AN_COMPLETE		0x0020
 #define	MDIO_AN_REG_ADV_PAUSE		0x0010
-#define	MDIO_AN_REG_ADV_PAUSE_PAUSE		0x0400
-#define	MDIO_AN_REG_ADV_PAUSE_ASYMMETRIC	0x0800
-#define	MDIO_AN_REG_ADV_PAUSE_BOTH		0x0C00
-#define	MDIO_AN_REG_ADV_PAUSE_MASK		0x0C00
+	#define	MDIO_AN_REG_ADV_PAUSE_PAUSE		0x0400
+	#define	MDIO_AN_REG_ADV_PAUSE_ASYMMETRIC	0x0800
+	#define	MDIO_AN_REG_ADV_PAUSE_BOTH		0x0C00
+	#define	MDIO_AN_REG_ADV_PAUSE_MASK		0x0C00
 #define	MDIO_AN_REG_ADV			0x0011
 #define MDIO_AN_REG_ADV2		0x0012
 #define	MDIO_AN_REG_LP_AUTO_NEG		0x0013
@@ -465,13 +463,16 @@ Theotherbitsarereservedandshouldbezero*/
 
 #define MDIO_AN_REG_8481_10GBASE_T_AN_CTRL	0x0020
 #define MDIO_AN_REG_8481_LEGACY_MII_CTRL	0xffe0
-#define MDIO_AN_REG_8481_MII_CTRL_FORCE_1G	0x40
+	#define MDIO_AN_REG_8481_MII_CTRL_FORCE_1G	0x40
 #define MDIO_AN_REG_8481_LEGACY_MII_STATUS	0xffe1
+#define MDIO_AN_REG_848xx_ID_MSB		0xffe2
+	#define BNX2X84858_PHY_ID					0x600d
+#define MDIO_AN_REG_848xx_ID_LSB		0xffe3
 #define MDIO_AN_REG_8481_LEGACY_AN_ADV		0xffe4
 #define MDIO_AN_REG_8481_LEGACY_AN_EXPANSION	0xffe6
 #define MDIO_AN_REG_8481_1000T_CTRL		0xffe9
 #define MDIO_AN_REG_8481_1G_100T_EXT_CTRL	0xfff0
-#define MIDO_AN_REG_8481_EXT_CTRL_FORCE_LEDS_OFF	0x0008
+	#define MIDO_AN_REG_8481_EXT_CTRL_FORCE_LEDS_OFF	0x0008
 #define MDIO_AN_REG_8481_EXPANSION_REG_RD_RW	0xfff5
 #define MDIO_AN_REG_8481_EXPANSION_REG_ACCESS	0xfff7
 #define MDIO_AN_REG_8481_AUX_CTRL		0xfff8
@@ -480,62 +481,62 @@ Theotherbitsarereservedandshouldbezero*/
 /* BNX2X84823 only */
 #define	MDIO_CTL_DEVAD			0x1e
 #define MDIO_CTL_REG_84823_MEDIA		0x401a
-#define MDIO_CTL_REG_84823_MEDIA_MAC_MASK		0x0018
+	#define MDIO_CTL_REG_84823_MEDIA_MAC_MASK		0x0018
 	/* These pins configure the BNX2X84823 interface to MAC after reset. */
-#define MDIO_CTL_REG_84823_CTRL_MAC_XFI			0x0008
-#define MDIO_CTL_REG_84823_MEDIA_MAC_XAUI_M		0x0010
+		#define MDIO_CTL_REG_84823_CTRL_MAC_XFI			0x0008
+		#define MDIO_CTL_REG_84823_MEDIA_MAC_XAUI_M		0x0010
 	/* These pins configure the BNX2X84823 interface to Line after reset. */
-#define MDIO_CTL_REG_84823_MEDIA_LINE_MASK		0x0060
-#define MDIO_CTL_REG_84823_MEDIA_LINE_XAUI_L		0x0020
-#define MDIO_CTL_REG_84823_MEDIA_LINE_XFI		0x0040
+	#define MDIO_CTL_REG_84823_MEDIA_LINE_MASK		0x0060
+		#define MDIO_CTL_REG_84823_MEDIA_LINE_XAUI_L		0x0020
+		#define MDIO_CTL_REG_84823_MEDIA_LINE_XFI		0x0040
 	/* When this pin is active high during reset, 10GBASE-T core is power
 	 * down, When it is active low the 10GBASE-T is power up
 	 */
-#define MDIO_CTL_REG_84823_MEDIA_COPPER_CORE_DOWN	0x0080
-#define MDIO_CTL_REG_84823_MEDIA_PRIORITY_MASK		0x0100
-#define MDIO_CTL_REG_84823_MEDIA_PRIORITY_COPPER	0x0000
-#define MDIO_CTL_REG_84823_MEDIA_PRIORITY_FIBER		0x0100
-#define MDIO_CTL_REG_84823_MEDIA_FIBER_1G			0x1000
+	#define MDIO_CTL_REG_84823_MEDIA_COPPER_CORE_DOWN	0x0080
+	#define MDIO_CTL_REG_84823_MEDIA_PRIORITY_MASK		0x0100
+		#define MDIO_CTL_REG_84823_MEDIA_PRIORITY_COPPER	0x0000
+		#define MDIO_CTL_REG_84823_MEDIA_PRIORITY_FIBER		0x0100
+	#define MDIO_CTL_REG_84823_MEDIA_FIBER_1G			0x1000
 #define MDIO_CTL_REG_84823_USER_CTRL_REG			0x4005
-#define MDIO_CTL_REG_84823_USER_CTRL_CMS			0x0080
+	#define MDIO_CTL_REG_84823_USER_CTRL_CMS			0x0080
 #define MDIO_PMA_REG_84823_CTL_SLOW_CLK_CNT_HIGH		0xa82b
-#define MDIO_PMA_REG_84823_BLINK_RATE_VAL_15P9HZ	0x2f
+	#define MDIO_PMA_REG_84823_BLINK_RATE_VAL_15P9HZ	0x2f
 #define MDIO_PMA_REG_84823_CTL_LED_CTL_1			0xa8e3
 #define MDIO_PMA_REG_84833_CTL_LED_CTL_1			0xa8ec
-#define MDIO_PMA_REG_84823_LED3_STRETCH_EN			0x0080
+	#define MDIO_PMA_REG_84823_LED3_STRETCH_EN			0x0080
 
 /* BNX2X84833 only */
 #define MDIO_84833_TOP_CFG_FW_REV			0x400f
-#define MDIO_84833_TOP_CFG_FW_EEE		0x10b1
-#define MDIO_84833_TOP_CFG_FW_NO_EEE		0x1f81
+#define MDIO_84833_TOP_CFG_FW_EEE			0x10b1
+#define MDIO_84833_TOP_CFG_FW_NO_EEE			0x1f81
 #define MDIO_84833_TOP_CFG_XGPHY_STRAP1 		0x401a
-#define MDIO_84833_SUPER_ISOLATE 		0x8000
-/* These are mailbox register set used by 84833. */
-#define MDIO_84833_TOP_CFG_SCRATCH_REG0			0x4005
-#define MDIO_84833_TOP_CFG_SCRATCH_REG1 		0x4006
-#define MDIO_84833_TOP_CFG_SCRATCH_REG2			0x4007
-#define MDIO_84833_TOP_CFG_SCRATCH_REG3			0x4008
-#define MDIO_84833_TOP_CFG_SCRATCH_REG4			0x4009
-#define MDIO_84833_TOP_CFG_SCRATCH_REG26		0x4037
-#define MDIO_84833_TOP_CFG_SCRATCH_REG27		0x4038
-#define MDIO_84833_TOP_CFG_SCRATCH_REG28		0x4039
-#define MDIO_84833_TOP_CFG_SCRATCH_REG29		0x403a
-#define MDIO_84833_TOP_CFG_SCRATCH_REG30		0x403b
-#define MDIO_84833_TOP_CFG_SCRATCH_REG31		0x403c
-#define MDIO_84833_CMD_HDLR_COMMAND	MDIO_84833_TOP_CFG_SCRATCH_REG0
-#define MDIO_84833_CMD_HDLR_STATUS	MDIO_84833_TOP_CFG_SCRATCH_REG26
-#define MDIO_84833_CMD_HDLR_DATA1	MDIO_84833_TOP_CFG_SCRATCH_REG27
-#define MDIO_84833_CMD_HDLR_DATA2	MDIO_84833_TOP_CFG_SCRATCH_REG28
-#define MDIO_84833_CMD_HDLR_DATA3	MDIO_84833_TOP_CFG_SCRATCH_REG29
-#define MDIO_84833_CMD_HDLR_DATA4	MDIO_84833_TOP_CFG_SCRATCH_REG30
-#define MDIO_84833_CMD_HDLR_DATA5	MDIO_84833_TOP_CFG_SCRATCH_REG31
-
-/* Mailbox command set used by 84833. */
-#define PHY84833_CMD_SET_PAIR_SWAP			0x8001
-#define PHY84833_CMD_GET_EEE_MODE			0x8008
-#define PHY84833_CMD_SET_EEE_MODE			0x8009
-#define PHY84833_CMD_GET_CURRENT_TEMP			0x8031
-/* Mailbox status set used by 84833. */
+#define MDIO_84833_SUPER_ISOLATE			0x8000
+/* These are mailbox register set used by 84833/84858. */
+#define MDIO_848xx_TOP_CFG_SCRATCH_REG0			0x4005
+#define MDIO_848xx_TOP_CFG_SCRATCH_REG1			0x4006
+#define MDIO_848xx_TOP_CFG_SCRATCH_REG2			0x4007
+#define MDIO_848xx_TOP_CFG_SCRATCH_REG3			0x4008
+#define MDIO_848xx_TOP_CFG_SCRATCH_REG4			0x4009
+#define MDIO_848xx_TOP_CFG_SCRATCH_REG26		0x4037
+#define MDIO_848xx_TOP_CFG_SCRATCH_REG27		0x4038
+#define MDIO_848xx_TOP_CFG_SCRATCH_REG28		0x4039
+#define MDIO_848xx_TOP_CFG_SCRATCH_REG29		0x403a
+#define MDIO_848xx_TOP_CFG_SCRATCH_REG30		0x403b
+#define MDIO_848xx_TOP_CFG_SCRATCH_REG31		0x403c
+#define MDIO_848xx_CMD_HDLR_COMMAND	(MDIO_848xx_TOP_CFG_SCRATCH_REG0)
+#define MDIO_848xx_CMD_HDLR_STATUS	(MDIO_848xx_TOP_CFG_SCRATCH_REG26)
+#define MDIO_848xx_CMD_HDLR_DATA1	(MDIO_848xx_TOP_CFG_SCRATCH_REG27)
+#define MDIO_848xx_CMD_HDLR_DATA2	(MDIO_848xx_TOP_CFG_SCRATCH_REG28)
+#define MDIO_848xx_CMD_HDLR_DATA3	(MDIO_848xx_TOP_CFG_SCRATCH_REG29)
+#define MDIO_848xx_CMD_HDLR_DATA4	(MDIO_848xx_TOP_CFG_SCRATCH_REG30)
+#define MDIO_848xx_CMD_HDLR_DATA5	(MDIO_848xx_TOP_CFG_SCRATCH_REG31)
+
+/* Mailbox command set used by 84833/84858 */
+#define PHY848xx_CMD_SET_PAIR_SWAP			0x8001
+#define PHY848xx_CMD_GET_EEE_MODE			0x8008
+#define PHY848xx_CMD_SET_EEE_MODE			0x8009
+#define PHY848xx_CMD_GET_CURRENT_TEMP			0x8031
+/* Mailbox status set used by 84833 only */
 #define PHY84833_STATUS_CMD_RECEIVED			0x0001
 #define PHY84833_STATUS_CMD_IN_PROGRESS			0x0002
 #define PHY84833_STATUS_CMD_COMPLETE_PASS		0x0004
@@ -545,6 +546,19 @@ Theotherbitsarereservedandshouldbezero*/
 #define PHY84833_STATUS_CMD_NOT_OPEN_FOR_CMDS		0x0040
 #define PHY84833_STATUS_CMD_CLEAR_COMPLETE		0x0080
 #define PHY84833_STATUS_CMD_OPEN_OVERRIDE		0xa5a5
+/* Mailbox Process */
+#define PHY84833_MB_PROCESS1				1
+#define PHY84833_MB_PROCESS2				2
+#define PHY84833_MB_PROCESS3				3
+
+
+/* Mailbox status set used by 84858 only */
+#define PHY84858_STATUS_CMD_RECEIVED			0x0001
+#define PHY84858_STATUS_CMD_IN_PROGRESS			0x0002
+#define PHY84858_STATUS_CMD_COMPLETE_PASS		0x0004
+#define PHY84858_STATUS_CMD_COMPLETE_ERROR		0x0008
+#define PHY84858_STATUS_CMD_SYSTEM_BUSY                 0xbbbb
+
 
 /* Warpcore clause 45 addressing */
 #define MDIO_WC_DEVAD					0x3
@@ -553,8 +567,8 @@ Theotherbitsarereservedandshouldbezero*/
 #define MDIO_WC_REG_AN_IEEE1BLK_AN_ADVERTISEMENT0       0x10
 #define MDIO_WC_REG_AN_IEEE1BLK_AN_ADVERTISEMENT1       0x11
 #define MDIO_WC_REG_AN_IEEE1BLK_AN_ADVERTISEMENT2       0x12
-#define MDIO_WC_REG_AN_IEEE1BLK_AN_ADV2_FEC_ABILITY	0x4000
-#define MDIO_WC_REG_AN_IEEE1BLK_AN_ADV2_FEC_REQ		0x8000
+	#define MDIO_WC_REG_AN_IEEE1BLK_AN_ADV2_FEC_ABILITY	0x4000
+	#define MDIO_WC_REG_AN_IEEE1BLK_AN_ADV2_FEC_REQ		0x8000
 #define MDIO_WC_REG_PCS_STATUS2				0x0021
 #define MDIO_WC_REG_PMD_KR_CONTROL			0x0096
 #define MDIO_WC_REG_XGXSBLK0_XGXSCONTROL                0x8000
@@ -570,6 +584,8 @@ Theotherbitsarereservedandshouldbezero*/
 #define MDIO_WC_REG_TX2_ANA_CTRL0			0x8081
 #define MDIO_WC_REG_TX3_ANA_CTRL0			0x8091
 #define MDIO_WC_REG_TX0_TX_DRIVER			0x8067
+#define MDIO_WC_REG_TX0_TX_DRIVER_IFIR_OFFSET			0x01
+#define MDIO_WC_REG_TX0_TX_DRIVER_IFIR_MASK				0x000e
 #define MDIO_WC_REG_TX0_TX_DRIVER_IPRE_DRIVER_OFFSET		0x04
 #define MDIO_WC_REG_TX0_TX_DRIVER_IPRE_DRIVER_MASK			0x00f0
 #define MDIO_WC_REG_TX0_TX_DRIVER_IDRIVER_OFFSET		0x08
@@ -585,7 +601,9 @@ Theotherbitsarereservedandshouldbezero*/
 #define MDIO_WC_REG_RX1_PCI_CTRL			0x80ca
 #define MDIO_WC_REG_RX2_PCI_CTRL			0x80da
 #define MDIO_WC_REG_RX3_PCI_CTRL			0x80ea
+#define MDIO_WC_REG_RXB_ANA_RX_CONTROL_PCI		0x80fa
 #define MDIO_WC_REG_XGXSBLK2_UNICORE_MODE_10G 		0x8104
+#define MDIO_WC_REG_XGXSBLK2_LANE_RESET			0x810a
 #define MDIO_WC_REG_XGXS_STATUS3			0x8129
 #define MDIO_WC_REG_PAR_DET_10G_STATUS			0x8130
 #define MDIO_WC_REG_PAR_DET_10G_CTRL			0x8131
@@ -599,35 +617,35 @@ Theotherbitsarereservedandshouldbezero*/
 #define MDIO_WC_REG_GP2_STATUS_GP_2_2			0x81d2
 #define MDIO_WC_REG_GP2_STATUS_GP_2_3			0x81d3
 #define MDIO_WC_REG_GP2_STATUS_GP_2_4			0x81d4
-#define MDIO_WC_REG_GP2_STATUS_GP_2_4_CL73_AN_CMPL 0x1000
-#define MDIO_WC_REG_GP2_STATUS_GP_2_4_CL37_AN_CMPL 0x0100
-#define MDIO_WC_REG_GP2_STATUS_GP_2_4_CL37_LP_AN_CAP 0x0010
-#define MDIO_WC_REG_GP2_STATUS_GP_2_4_CL37_AN_CAP 0x1
+	#define MDIO_WC_REG_GP2_STATUS_GP_2_4_CL73_AN_CMPL 0x1000
+	#define MDIO_WC_REG_GP2_STATUS_GP_2_4_CL37_AN_CMPL 0x0100
+	#define MDIO_WC_REG_GP2_STATUS_GP_2_4_CL37_LP_AN_CAP 0x0010
+	#define MDIO_WC_REG_GP2_STATUS_GP_2_4_CL37_AN_CAP 0x1
 #define MDIO_WC_REG_UC_INFO_B0_DEAD_TRAP                0x81EE
 #define MDIO_WC_REG_UC_INFO_B1_VERSION                  0x81F0
 #define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE		0x81F2
-#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_LANE0_OFFSET	0x0
-#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE_DEFAULT        0x0
-#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE_SFP_OPT_LR     0x1
-#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE_SFP_DAC        0x2
-#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE_SFP_XLAUI      0x3
-#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE_LONG_CH_6G     0x4
-#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_LANE1_OFFSET	0x4
-#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_LANE2_OFFSET	0x8
-#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_LANE3_OFFSET	0xc
+	#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_LANE0_OFFSET	0x0
+		#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE_DEFAULT        0x0
+		#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE_SFP_OPT_LR     0x1
+		#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE_SFP_DAC        0x2
+		#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE_SFP_XLAUI      0x3
+		#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE_LONG_CH_6G     0x4
+	#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_LANE1_OFFSET	0x4
+	#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_LANE2_OFFSET	0x8
+	#define MDIO_WC_REG_UC_INFO_B1_FIRMWARE_LANE3_OFFSET	0xc
 #define MDIO_WC_REG_UC_INFO_B1_CRC                      0x81FE
 #define MDIO_WC_REG_DSC1B0_UC_CTRL				0x820e
 #define MDIO_WC_REG_DSC1B0_UC_CTRL_RDY4CMD			(1<<7)
 #define MDIO_WC_REG_DSC_SMC				0x8213
 #define MDIO_WC_REG_DSC2B0_DSC_MISC_CTRL0		0x821e
 #define MDIO_WC_REG_TX_FIR_TAP				0x82e2
-#define MDIO_WC_REG_TX_FIR_TAP_PRE_TAP_OFFSET		0x00
-#define MDIO_WC_REG_TX_FIR_TAP_PRE_TAP_MASK			0x000f
-#define MDIO_WC_REG_TX_FIR_TAP_MAIN_TAP_OFFSET		0x04
-#define MDIO_WC_REG_TX_FIR_TAP_MAIN_TAP_MASK		0x03f0
-#define MDIO_WC_REG_TX_FIR_TAP_POST_TAP_OFFSET		0x0a
-#define MDIO_WC_REG_TX_FIR_TAP_POST_TAP_MASK		0x7c00
-#define MDIO_WC_REG_TX_FIR_TAP_ENABLE		0x8000
+	#define MDIO_WC_REG_TX_FIR_TAP_PRE_TAP_OFFSET		0x00
+	#define MDIO_WC_REG_TX_FIR_TAP_PRE_TAP_MASK			0x000f
+	#define MDIO_WC_REG_TX_FIR_TAP_MAIN_TAP_OFFSET		0x04
+	#define MDIO_WC_REG_TX_FIR_TAP_MAIN_TAP_MASK		0x03f0
+	#define MDIO_WC_REG_TX_FIR_TAP_POST_TAP_OFFSET		0x0a
+	#define MDIO_WC_REG_TX_FIR_TAP_POST_TAP_MASK		0x7c00
+	#define MDIO_WC_REG_TX_FIR_TAP_ENABLE		0x8000
 #define MDIO_WC_REG_CL72_USERB0_CL72_TX_FIR_TAP		0x82e2
 #define MDIO_WC_REG_CL72_USERB0_CL72_MISC1_CONTROL      0x82e3
 #define MDIO_WC_REG_CL72_USERB0_CL72_OS_DEF_CTRL	0x82e6
@@ -689,8 +707,8 @@ Theotherbitsarereservedandshouldbezero*/
 #define MDIO_WC_REG_COMBO_IEEE0_MIIISTAT                0xffe1
 
 #define MDIO_WC0_XGXS_BLK2_LANE_RESET                   0x810A
-#define MDIO_WC0_XGXS_BLK2_LANE_RESET_RX_BITSHIFT 	0
-#define MDIO_WC0_XGXS_BLK2_LANE_RESET_TX_BITSHIFT 	4
+#define MDIO_WC0_XGXS_BLK2_LANE_RESET_RX_BITSHIFT	0
+#define MDIO_WC0_XGXS_BLK2_LANE_RESET_TX_BITSHIFT	4
 
 #define MDIO_WC0_XGXS_BLK6_XGXS_X2_CONTROL2             0x8141
 
@@ -700,33 +718,31 @@ Theotherbitsarereservedandshouldbezero*/
 #define MDIO_REG_GPHY_MII_STATUS			0x1
 #define MDIO_REG_GPHY_PHYID_LSB				0x3
 #define MDIO_REG_GPHY_CL45_ADDR_REG			0xd
-#define MDIO_REG_GPHY_CL45_REG_WRITE		0x4000
-#define MDIO_REG_GPHY_CL45_REG_READ		0xc000
+	#define MDIO_REG_GPHY_CL45_REG_WRITE		0x4000
+	#define MDIO_REG_GPHY_CL45_REG_READ		0xc000
 #define MDIO_REG_GPHY_CL45_DATA_REG			0xe
-#define MDIO_REG_GPHY_EEE_RESOLVED		0x803e
+	#define MDIO_REG_GPHY_EEE_RESOLVED		0x803e
 #define MDIO_REG_GPHY_EXP_ACCESS_GATE			0x15
 #define MDIO_REG_GPHY_EXP_ACCESS			0x17
-#define MDIO_REG_GPHY_EXP_ACCESS_TOP		0xd00
-#define MDIO_REG_GPHY_EXP_TOP_2K_BUF		0x40
+	#define MDIO_REG_GPHY_EXP_ACCESS_TOP		0xd00
+	#define MDIO_REG_GPHY_EXP_TOP_2K_BUF		0x40
 #define MDIO_REG_GPHY_AUX_STATUS			0x19
 #define MDIO_REG_INTR_STATUS				0x1a
 #define MDIO_REG_INTR_MASK				0x1b
-#define MDIO_REG_INTR_MASK_LINK_STATUS			(0x1 << 1)
+	#define MDIO_REG_INTR_MASK_LINK_STATUS			(0x1 << 1)
 #define MDIO_REG_GPHY_SHADOW				0x1c
-#define MDIO_REG_GPHY_SHADOW_LED_SEL1			(0x0d << 10)
-#define MDIO_REG_GPHY_SHADOW_LED_SEL2			(0x0e << 10)
-#define MDIO_REG_GPHY_SHADOW_WR_ENA			(0x1 << 15)
-#define MDIO_REG_GPHY_SHADOW_AUTO_DET_MED		(0x1e << 10)
-#define MDIO_REG_GPHY_SHADOW_INVERT_FIB_SD		(0x1 << 8)
-
-typedef elink_status_t(*read_sfp_module_eeprom_func_p) (struct elink_phy * phy,
-							struct elink_params *
-							params,
-							uint8_t dev_addr,
-							uint16_t addr,
-							uint8_t byte_cnt,
-							uint8_t * o_buf,
-							uint8_t);
+	#define MDIO_REG_GPHY_SHADOW_LED_SEL1			(0x0d << 10)
+	#define MDIO_REG_GPHY_SHADOW_LED_SEL2			(0x0e << 10)
+	#define MDIO_REG_GPHY_SHADOW_WR_ENA			(0x1 << 15)
+	#define MDIO_REG_GPHY_SHADOW_AUTO_DET_MED		(0x1e << 10)
+	#define MDIO_REG_GPHY_SHADOW_INVERT_FIB_SD		(0x1 << 8)
+
+
+typedef elink_status_t (*read_sfp_module_eeprom_func_p)(struct elink_phy *phy,
+					     struct elink_params *params,
+					     uint8_t dev_addr, uint16_t addr,
+					     uint8_t byte_cnt,
+					     uint8_t *o_buf, uint8_t);
 /********************************************************/
 #define ELINK_ETH_HLEN			14
 /* L2 header size + 2*VLANs (8 bytes) + LLC SNAP (8 bytes) */
@@ -850,21 +866,29 @@ typedef elink_status_t(*read_sfp_module_eeprom_func_p) (struct elink_phy * phy,
 			 LINK_STATUS_LINK_PARTNER_ASYMMETRIC_PAUSE)
 
 #define ELINK_SFP_EEPROM_CON_TYPE_ADDR		0x2
-#define ELINK_SFP_EEPROM_CON_TYPE_VAL_LC	0x7
-#define ELINK_SFP_EEPROM_CON_TYPE_VAL_COPPER	0x21
-#define ELINK_SFP_EEPROM_CON_TYPE_VAL_RJ45	0x22
+	#define ELINK_SFP_EEPROM_CON_TYPE_VAL_UNKNOWN	0x0
+	#define ELINK_SFP_EEPROM_CON_TYPE_VAL_LC	0x7
+	#define ELINK_SFP_EEPROM_CON_TYPE_VAL_COPPER	0x21
+	#define ELINK_SFP_EEPROM_CON_TYPE_VAL_RJ45	0x22
+
 
-#define ELINK_SFP_EEPROM_COMP_CODE_ADDR		0x3
-#define ELINK_SFP_EEPROM_COMP_CODE_SR_MASK	(1<<4)
-#define ELINK_SFP_EEPROM_COMP_CODE_LR_MASK	(1<<5)
-#define ELINK_SFP_EEPROM_COMP_CODE_LRM_MASK	(1<<6)
+#define ELINK_SFP_EEPROM_10G_COMP_CODE_ADDR		0x3
+	#define ELINK_SFP_EEPROM_10G_COMP_CODE_SR_MASK	(1 << 4)
+	#define ELINK_SFP_EEPROM_10G_COMP_CODE_LR_MASK	(1 << 5)
+	#define ELINK_SFP_EEPROM_10G_COMP_CODE_LRM_MASK	(1 << 6)
+
+#define ELINK_SFP_EEPROM_1G_COMP_CODE_ADDR		0x6
+	#define ELINK_SFP_EEPROM_1G_COMP_CODE_SX	(1 << 0)
+	#define ELINK_SFP_EEPROM_1G_COMP_CODE_LX	(1 << 1)
+	#define ELINK_SFP_EEPROM_1G_COMP_CODE_CX	(1 << 2)
+	#define ELINK_SFP_EEPROM_1G_COMP_CODE_BASE_T	(1 << 3)
 
 #define ELINK_SFP_EEPROM_FC_TX_TECH_ADDR		0x8
-#define ELINK_SFP_EEPROM_FC_TX_TECH_BITMASK_COPPER_PASSIVE 0x4
-#define ELINK_SFP_EEPROM_FC_TX_TECH_BITMASK_COPPER_ACTIVE  0x8
+	#define ELINK_SFP_EEPROM_FC_TX_TECH_BITMASK_COPPER_PASSIVE 0x4
+	#define ELINK_SFP_EEPROM_FC_TX_TECH_BITMASK_COPPER_ACTIVE  0x8
 
 #define ELINK_SFP_EEPROM_OPTIONS_ADDR			0x40
-#define ELINK_SFP_EEPROM_OPTIONS_LINEAR_RX_OUT_MASK 0x1
+	#define ELINK_SFP_EEPROM_OPTIONS_LINEAR_RX_OUT_MASK 0x1
 #define ELINK_SFP_EEPROM_OPTIONS_SIZE			2
 
 #define ELINK_EDC_MODE_LINEAR				0x0022
@@ -883,6 +907,10 @@ typedef elink_status_t(*read_sfp_module_eeprom_func_p) (struct elink_phy * phy,
 
 #define ELINK_MAX_PACKET_SIZE					(9700)
 #define MAX_KR_LINK_RETRY				4
+#define DEFAULT_TX_DRV_BRDCT		2
+#define DEFAULT_TX_DRV_IFIR		0
+#define DEFAULT_TX_DRV_POST2		3
+#define DEFAULT_TX_DRV_IPRE_DRIVER	6
 
 /**********************************************************/
 /*                     INTERFACE                          */
@@ -900,6 +928,11 @@ typedef elink_status_t(*read_sfp_module_eeprom_func_p) (struct elink_phy * phy,
 		(_bank + (_addr & 0xf)), \
 		_val)
 
+static elink_status_t elink_check_half_open_conn(struct elink_params *params,
+				      struct elink_vars *vars, uint8_t notify);
+static elink_status_t elink_sfp_module_detection(struct elink_phy *phy,
+				      struct elink_params *params);
+
 static uint32_t elink_bits_en(struct bnx2x_softc *sc, uint32_t reg, uint32_t bits)
 {
 	uint32_t val = REG_RD(sc, reg);
@@ -935,16 +968,16 @@ static int elink_check_lfa(struct elink_params *params)
 	struct bnx2x_softc *sc = params->sc;
 
 	additional_config =
-	    REG_RD(sc, params->lfa_base +
-		   offsetof(struct shmem_lfa, additional_config));
+		REG_RD(sc, params->lfa_base +
+			   offsetof(struct shmem_lfa, additional_config));
 
 	/* NOTE: must be first condition checked -
-	 * to verify DCC bit is cleared in any case!
-	 */
+	* to verify DCC bit is cleared in any case!
+	*/
 	if (additional_config & NO_LFA_DUE_TO_DCC_MASK) {
-		PMD_DRV_LOG(DEBUG, "No LFA due to DCC flap after clp exit");
+		ELINK_DEBUG_P0(sc, "No LFA due to DCC flap after clp exit");
 		REG_WR(sc, params->lfa_base +
-		       offsetof(struct shmem_lfa, additional_config),
+			   offsetof(struct shmem_lfa, additional_config),
 		       additional_config & ~NO_LFA_DUE_TO_DCC_MASK);
 		return LFA_DCC_LFA_DISABLED;
 	}
@@ -983,8 +1016,8 @@ static int elink_check_lfa(struct elink_params *params)
 			   offsetof(struct shmem_lfa, req_duplex));
 	req_val = params->req_duplex[0] | (params->req_duplex[1] << 16);
 	if ((saved_val & lfa_mask) != (req_val & lfa_mask)) {
-		PMD_DRV_LOG(INFO, "Duplex mismatch %x vs. %x",
-			    (saved_val & lfa_mask), (req_val & lfa_mask));
+		ELINK_DEBUG_P2(sc, "Duplex mismatch %x vs. %x",
+			       (saved_val & lfa_mask), (req_val & lfa_mask));
 		return LFA_DUPLEX_MISMATCH;
 	}
 	/* Compare Flow Control */
@@ -992,8 +1025,8 @@ static int elink_check_lfa(struct elink_params *params)
 			   offsetof(struct shmem_lfa, req_flow_ctrl));
 	req_val = params->req_flow_ctrl[0] | (params->req_flow_ctrl[1] << 16);
 	if ((saved_val & lfa_mask) != (req_val & lfa_mask)) {
-		PMD_DRV_LOG(DEBUG, "Flow control mismatch %x vs. %x",
-			    (saved_val & lfa_mask), (req_val & lfa_mask));
+		ELINK_DEBUG_P2(sc, "Flow control mismatch %x vs. %x",
+			       (saved_val & lfa_mask), (req_val & lfa_mask));
 		return LFA_FLOW_CTRL_MISMATCH;
 	}
 	/* Compare Link Speed */
@@ -1001,8 +1034,8 @@ static int elink_check_lfa(struct elink_params *params)
 			   offsetof(struct shmem_lfa, req_line_speed));
 	req_val = params->req_line_speed[0] | (params->req_line_speed[1] << 16);
 	if ((saved_val & lfa_mask) != (req_val & lfa_mask)) {
-		PMD_DRV_LOG(DEBUG, "Link speed mismatch %x vs. %x",
-			    (saved_val & lfa_mask), (req_val & lfa_mask));
+		ELINK_DEBUG_P2(sc, "Link speed mismatch %x vs. %x",
+			       (saved_val & lfa_mask), (req_val & lfa_mask));
 		return LFA_LINK_SPEED_MISMATCH;
 	}
 
@@ -1012,21 +1045,21 @@ static int elink_check_lfa(struct elink_params *params)
 						     speed_cap_mask[cfg_idx]));
 
 		if (cur_speed_cap_mask != params->speed_cap_mask[cfg_idx]) {
-			PMD_DRV_LOG(DEBUG, "Speed Cap mismatch %x vs. %x",
-				    cur_speed_cap_mask,
-				    params->speed_cap_mask[cfg_idx]);
+			ELINK_DEBUG_P2(sc, "Speed Cap mismatch %x vs. %x",
+				       cur_speed_cap_mask,
+				       params->speed_cap_mask[cfg_idx]);
 			return LFA_SPEED_CAP_MISMATCH;
 		}
 	}
 
 	cur_req_fc_auto_adv =
-	    REG_RD(sc, params->lfa_base +
-		   offsetof(struct shmem_lfa, additional_config)) &
-	    REQ_FC_AUTO_ADV_MASK;
+		REG_RD(sc, params->lfa_base +
+		       offsetof(struct shmem_lfa, additional_config)) &
+		REQ_FC_AUTO_ADV_MASK;
 
-	if ((uint16_t) cur_req_fc_auto_adv != params->req_fc_auto_adv) {
-		PMD_DRV_LOG(DEBUG, "Flow Ctrl AN mismatch %x vs. %x",
-			    cur_req_fc_auto_adv, params->req_fc_auto_adv);
+	if ((uint16_t)cur_req_fc_auto_adv != params->req_fc_auto_adv) {
+		ELINK_DEBUG_P2(sc, "Flow Ctrl AN mismatch %x vs. %x",
+			       cur_req_fc_auto_adv, params->req_fc_auto_adv);
 		return LFA_FLOW_CTRL_MISMATCH;
 	}
 
@@ -1038,26 +1071,25 @@ static int elink_check_lfa(struct elink_params *params)
 	     (params->eee_mode & ELINK_EEE_MODE_ENABLE_LPI)) ||
 	    ((eee_status & SHMEM_EEE_REQUESTED_BIT) ^
 	     (params->eee_mode & ELINK_EEE_MODE_ADV_LPI))) {
-		PMD_DRV_LOG(DEBUG, "EEE mismatch %x vs. %x", params->eee_mode,
-			    eee_status);
+		ELINK_DEBUG_P2(sc, "EEE mismatch %x vs. %x", params->eee_mode,
+			       eee_status);
 		return LFA_EEE_MISMATCH;
 	}
 
 	/* LFA conditions are met */
 	return 0;
 }
-
 /******************************************************************/
 /*			EPIO/GPIO section			  */
 /******************************************************************/
 static void elink_get_epio(struct bnx2x_softc *sc, uint32_t epio_pin,
-			   uint32_t * en)
+			   uint32_t *en)
 {
 	uint32_t epio_mask, gp_oenable;
 	*en = 0;
 	/* Sanity check */
 	if (epio_pin > 31) {
-		PMD_DRV_LOG(DEBUG, "Invalid EPIO pin %d to get", epio_pin);
+		ELINK_DEBUG_P1(sc, "Invalid EPIO pin %d to get", epio_pin);
 		return;
 	}
 
@@ -1068,17 +1100,16 @@ static void elink_get_epio(struct bnx2x_softc *sc, uint32_t epio_pin,
 
 	*en = (REG_RD(sc, MCP_REG_MCPR_GP_INPUTS) & epio_mask) >> epio_pin;
 }
-
 static void elink_set_epio(struct bnx2x_softc *sc, uint32_t epio_pin, uint32_t en)
 {
 	uint32_t epio_mask, gp_output, gp_oenable;
 
 	/* Sanity check */
 	if (epio_pin > 31) {
-		PMD_DRV_LOG(DEBUG, "Invalid EPIO pin %d to set", epio_pin);
+		ELINK_DEBUG_P1(sc, "Invalid EPIO pin %d to set", epio_pin);
 		return;
 	}
-	PMD_DRV_LOG(DEBUG, "Setting EPIO pin %d to %d", epio_pin, en);
+	ELINK_DEBUG_P2(sc, "Setting EPIO pin %d to %d", epio_pin, en);
 	epio_mask = 1 << epio_pin;
 	/* Set this EPIO to output */
 	gp_output = REG_RD(sc, MCP_REG_MCPR_GP_OUTPUTS);
@@ -1104,12 +1135,12 @@ static void elink_set_cfg_pin(struct bnx2x_softc *sc, uint32_t pin_cfg,
 	} else {
 		uint8_t gpio_num = (pin_cfg - PIN_CFG_GPIO0_P0) & 0x3;
 		uint8_t gpio_port = (pin_cfg - PIN_CFG_GPIO0_P0) >> 2;
-		elink_cb_gpio_write(sc, gpio_num, (uint8_t) val, gpio_port);
+		elink_cb_gpio_write(sc, gpio_num, (uint8_t)val, gpio_port);
 	}
 }
 
 static uint32_t elink_get_cfg_pin(struct bnx2x_softc *sc, uint32_t pin_cfg,
-				  uint32_t * val)
+				  uint32_t *val)
 {
 	if (pin_cfg == PIN_CFG_NA)
 		return ELINK_STATUS_ERROR;
@@ -1121,14 +1152,939 @@ static uint32_t elink_get_cfg_pin(struct bnx2x_softc *sc, uint32_t pin_cfg,
 		*val = elink_cb_gpio_read(sc, gpio_num, gpio_port);
 	}
 	return ELINK_STATUS_OK;
+}
+
+/******************************************************************/
+/*				ETS section			  */
+/******************************************************************/
+static void elink_ets_e2e3a0_disabled(struct elink_params *params)
+{
+	/* ETS disabled configuration*/
+	struct bnx2x_softc *sc = params->sc;
+
+	ELINK_DEBUG_P0(sc, "ETS E2E3 disabled configuration");
+
+	/* mapping between entry  priority to client number (0,1,2 -debug and
+	 * management clients, 3 - COS0 client, 4 - COS client)(HIGHEST)
+	 * 3bits client num.
+	 *   PRI4    |    PRI3    |    PRI2    |    PRI1    |    PRI0
+	 * cos1-100     cos0-011     dbg1-010     dbg0-001     MCP-000
+	 */
+
+	REG_WR(sc, NIG_REG_P0_TX_ARB_PRIORITY_CLIENT, 0x4688);
+	/* Bitmap of 5bits length. Each bit specifies whether the entry behaves
+	 * as strict.  Bits 0,1,2 - debug and management entries, 3 -
+	 * COS0 entry, 4 - COS1 entry.
+	 * COS1 | COS0 | DEBUG1 | DEBUG0 | MGMT
+	 * bit4   bit3	  bit2   bit1	  bit0
+	 * MCP and debug are strict
+	 */
+
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CLIENT_IS_STRICT, 0x7);
+	/* defines which entries (clients) are subjected to WFQ arbitration */
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CLIENT_IS_SUBJECT2WFQ, 0);
+	/* For strict priority entries defines the number of consecutive
+	 * slots for the highest priority.
+	 */
+	REG_WR(sc, NIG_REG_P0_TX_ARB_NUM_STRICT_ARB_SLOTS, 0x100);
+	/* mapping between the CREDIT_WEIGHT registers and actual client
+	 * numbers
+	 */
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CLIENT_CREDIT_MAP, 0);
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_0, 0);
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_1, 0);
+
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_0, 0);
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_1, 0);
+	REG_WR(sc, PBF_REG_HIGH_PRIORITY_COS_NUM, 0);
+	/* ETS mode disable */
+	REG_WR(sc, PBF_REG_ETS_ENABLED, 0);
+	/* If ETS mode is enabled (there is no strict priority) defines a WFQ
+	 * weight for COS0/COS1.
+	 */
+	REG_WR(sc, PBF_REG_COS0_WEIGHT, 0x2710);
+	REG_WR(sc, PBF_REG_COS1_WEIGHT, 0x2710);
+	/* Upper bound that COS0_WEIGHT can reach in the WFQ arbiter */
+	REG_WR(sc, PBF_REG_COS0_UPPER_BOUND, 0x989680);
+	REG_WR(sc, PBF_REG_COS1_UPPER_BOUND, 0x989680);
+	/* Defines the number of consecutive slots for the strict priority */
+	REG_WR(sc, PBF_REG_NUM_STRICT_ARB_SLOTS, 0);
+}
+/******************************************************************************
+ * Description:
+ *	Getting min_w_val will be set according to line speed .
+ *.
+ ******************************************************************************/
+static uint32_t elink_ets_get_min_w_val_nig(const struct elink_vars *vars)
+{
+	uint32_t min_w_val = 0;
+	/* Calculate min_w_val.*/
+	if (vars->link_up) {
+		if (vars->line_speed == ELINK_SPEED_20000)
+			min_w_val = ELINK_ETS_E3B0_NIG_MIN_W_VAL_20GBPS;
+		else
+			min_w_val = ELINK_ETS_E3B0_NIG_MIN_W_VAL_UP_TO_10GBPS;
+	} else {
+		min_w_val = ELINK_ETS_E3B0_NIG_MIN_W_VAL_20GBPS;
+	}
+	/* If the link isn't up (static configuration for example ) The
+	 * link will be according to 20GBPS.
+	 */
+	return min_w_val;
+}
+/******************************************************************************
+ * Description:
+ *	Getting credit upper bound form min_w_val.
+ *.
+ ******************************************************************************/
+static uint32_t elink_ets_get_credit_upper_bound(const uint32_t min_w_val)
+{
+	const uint32_t credit_upper_bound = (uint32_t)
+						ELINK_MAXVAL((150 * min_w_val),
+							ELINK_MAX_PACKET_SIZE);
+	return credit_upper_bound;
+}
+/******************************************************************************
+ * Description:
+ *	Set credit upper bound for NIG.
+ *.
+ ******************************************************************************/
+static void elink_ets_e3b0_set_credit_upper_bound_nig(
+	const struct elink_params *params,
+	const uint32_t min_w_val)
+{
+	struct bnx2x_softc *sc = params->sc;
+	const uint8_t port = params->port;
+	const uint32_t credit_upper_bound =
+	    elink_ets_get_credit_upper_bound(min_w_val);
+
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_UPPER_BOUND_0 :
+		NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_0, credit_upper_bound);
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_UPPER_BOUND_1 :
+		   NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_1, credit_upper_bound);
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_UPPER_BOUND_2 :
+		   NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_2, credit_upper_bound);
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_UPPER_BOUND_3 :
+		   NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_3, credit_upper_bound);
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_UPPER_BOUND_4 :
+		   NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_4, credit_upper_bound);
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_UPPER_BOUND_5 :
+		   NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_5, credit_upper_bound);
+
+	if (!port) {
+		REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_6,
+			credit_upper_bound);
+		REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_7,
+			credit_upper_bound);
+		REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_8,
+			credit_upper_bound);
+	}
+}
+/******************************************************************************
+ * Description:
+ *	Will return the NIG ETS registers to init values.Except
+ *	credit_upper_bound.
+ *	That isn't used in this configuration (No WFQ is enabled) and will be
+ *	configured according to spec
+ *.
+ ******************************************************************************/
+static void elink_ets_e3b0_nig_disabled(const struct elink_params *params,
+					const struct elink_vars *vars)
+{
+	struct bnx2x_softc *sc = params->sc;
+	const uint8_t port = params->port;
+	const uint32_t min_w_val = elink_ets_get_min_w_val_nig(vars);
+	/* Mapping between entry  priority to client number (0,1,2 -debug and
+	 * management clients, 3 - COS0 client, 4 - COS1, ... 8 -
+	 * COS5)(HIGHEST) 4bits client num.TODO_ETS - Should be done by
+	 * reset value or init tool
+	 */
+	if (port) {
+		REG_WR(sc, NIG_REG_P1_TX_ARB_PRIORITY_CLIENT2_LSB, 0x543210);
+		REG_WR(sc, NIG_REG_P1_TX_ARB_PRIORITY_CLIENT2_MSB, 0x0);
+	} else {
+		REG_WR(sc, NIG_REG_P0_TX_ARB_PRIORITY_CLIENT2_LSB, 0x76543210);
+		REG_WR(sc, NIG_REG_P0_TX_ARB_PRIORITY_CLIENT2_MSB, 0x8);
+	}
+	/* For strict priority entries defines the number of consecutive
+	 * slots for the highest priority.
+	 */
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_NUM_STRICT_ARB_SLOTS :
+		   NIG_REG_P1_TX_ARB_NUM_STRICT_ARB_SLOTS, 0x100);
+	/* Mapping between the CREDIT_WEIGHT registers and actual client
+	 * numbers
+	 */
+	if (port) {
+		/*Port 1 has 6 COS*/
+		REG_WR(sc, NIG_REG_P1_TX_ARB_CLIENT_CREDIT_MAP2_LSB, 0x210543);
+		REG_WR(sc, NIG_REG_P1_TX_ARB_CLIENT_CREDIT_MAP2_MSB, 0x0);
+	} else {
+		/*Port 0 has 9 COS*/
+		REG_WR(sc, NIG_REG_P0_TX_ARB_CLIENT_CREDIT_MAP2_LSB,
+		       0x43210876);
+		REG_WR(sc, NIG_REG_P0_TX_ARB_CLIENT_CREDIT_MAP2_MSB, 0x5);
+	}
+
+	/* Bitmap of 5bits length. Each bit specifies whether the entry behaves
+	 * as strict.  Bits 0,1,2 - debug and management entries, 3 -
+	 * COS0 entry, 4 - COS1 entry.
+	 * COS1 | COS0 | DEBUG1 | DEBUG0 | MGMT
+	 * bit4   bit3	  bit2   bit1	  bit0
+	 * MCP and debug are strict
+	 */
+	if (port)
+		REG_WR(sc, NIG_REG_P1_TX_ARB_CLIENT_IS_STRICT, 0x3f);
+	else
+		REG_WR(sc, NIG_REG_P0_TX_ARB_CLIENT_IS_STRICT, 0x1ff);
+	/* defines which entries (clients) are subjected to WFQ arbitration */
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CLIENT_IS_SUBJECT2WFQ :
+		   NIG_REG_P0_TX_ARB_CLIENT_IS_SUBJECT2WFQ, 0);
+
+	/* Please notice the register address are note continuous and a
+	 * for here is note appropriate.In 2 port mode port0 only COS0-5
+	 * can be used. DEBUG1,DEBUG1,MGMT are never used for WFQ* In 4
+	 * port mode port1 only COS0-2 can be used. DEBUG1,DEBUG1,MGMT
+	 * are never used for WFQ
+	 */
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_WEIGHT_0 :
+		   NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_0, 0x0);
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_WEIGHT_1 :
+		   NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_1, 0x0);
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_WEIGHT_2 :
+		   NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_2, 0x0);
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_WEIGHT_3 :
+		   NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_3, 0x0);
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_WEIGHT_4 :
+		   NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_4, 0x0);
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CREDIT_WEIGHT_5 :
+		   NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_5, 0x0);
+	if (!port) {
+		REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_6, 0x0);
+		REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_7, 0x0);
+		REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_8, 0x0);
+	}
+
+	elink_ets_e3b0_set_credit_upper_bound_nig(params, min_w_val);
+}
+/******************************************************************************
+ * Description:
+ *	Set credit upper bound for PBF.
+ *.
+ ******************************************************************************/
+static void elink_ets_e3b0_set_credit_upper_bound_pbf(
+	const struct elink_params *params,
+	const uint32_t min_w_val)
+{
+	struct bnx2x_softc *sc = params->sc;
+	const uint32_t credit_upper_bound =
+	    elink_ets_get_credit_upper_bound(min_w_val);
+	const uint8_t port = params->port;
+	uint32_t base_upper_bound = 0;
+	uint8_t max_cos = 0;
+	uint8_t i = 0;
+	/* In 2 port mode port0 has COS0-5 that can be used for WFQ.In 4
+	 * port mode port1 has COS0-2 that can be used for WFQ.
+	 */
+	if (!port) {
+		base_upper_bound = PBF_REG_COS0_UPPER_BOUND_P0;
+		max_cos = ELINK_DCBX_E3B0_MAX_NUM_COS_PORT0;
+	} else {
+		base_upper_bound = PBF_REG_COS0_UPPER_BOUND_P1;
+		max_cos = ELINK_DCBX_E3B0_MAX_NUM_COS_PORT1;
+	}
+
+	for (i = 0; i < max_cos; i++)
+		REG_WR(sc, base_upper_bound + (i << 2), credit_upper_bound);
+}
+
+/******************************************************************************
+ * Description:
+ *	Will return the PBF ETS registers to init values.Except
+ *	credit_upper_bound.
+ *	That isn't used in this configuration (No WFQ is enabled) and will be
+ *	configured according to spec
+ *.
+ ******************************************************************************/
+static void elink_ets_e3b0_pbf_disabled(const struct elink_params *params)
+{
+	struct bnx2x_softc *sc = params->sc;
+	const uint8_t port = params->port;
+	const uint32_t min_w_val_pbf = ELINK_ETS_E3B0_PBF_MIN_W_VAL;
+	uint8_t i = 0;
+	uint32_t base_weight = 0;
+	uint8_t max_cos = 0;
+
+	/* Mapping between entry  priority to client number 0 - COS0
+	 * client, 2 - COS1, ... 5 - COS5)(HIGHEST) 4bits client num.
+	 * TODO_ETS - Should be done by reset value or init tool
+	 */
+	if (port)
+		/*  0x688 (|011|0 10|00 1|000) */
+		REG_WR(sc, PBF_REG_ETS_ARB_PRIORITY_CLIENT_P1, 0x688);
+	else
+		/*  (10 1|100 |011|0 10|00 1|000) */
+		REG_WR(sc, PBF_REG_ETS_ARB_PRIORITY_CLIENT_P0, 0x2C688);
+
+	/* TODO_ETS - Should be done by reset value or init tool */
+	if (port)
+		/* 0x688 (|011|0 10|00 1|000)*/
+		REG_WR(sc, PBF_REG_ETS_ARB_CLIENT_CREDIT_MAP_P1, 0x688);
+	else
+	/* 0x2C688 (10 1|100 |011|0 10|00 1|000) */
+	REG_WR(sc, PBF_REG_ETS_ARB_CLIENT_CREDIT_MAP_P0, 0x2C688);
+
+	REG_WR(sc, (port) ? PBF_REG_ETS_ARB_NUM_STRICT_ARB_SLOTS_P1 :
+		   PBF_REG_ETS_ARB_NUM_STRICT_ARB_SLOTS_P0, 0x100);
+
+
+	REG_WR(sc, (port) ? PBF_REG_ETS_ARB_CLIENT_IS_STRICT_P1 :
+		   PBF_REG_ETS_ARB_CLIENT_IS_STRICT_P0, 0);
+
+	REG_WR(sc, (port) ? PBF_REG_ETS_ARB_CLIENT_IS_SUBJECT2WFQ_P1 :
+		   PBF_REG_ETS_ARB_CLIENT_IS_SUBJECT2WFQ_P0, 0);
+	/* In 2 port mode port0 has COS0-5 that can be used for WFQ.
+	 * In 4 port mode port1 has COS0-2 that can be used for WFQ.
+	 */
+	if (!port) {
+		base_weight = PBF_REG_COS0_WEIGHT_P0;
+		max_cos = ELINK_DCBX_E3B0_MAX_NUM_COS_PORT0;
+	} else {
+		base_weight = PBF_REG_COS0_WEIGHT_P1;
+		max_cos = ELINK_DCBX_E3B0_MAX_NUM_COS_PORT1;
+	}
+
+	for (i = 0; i < max_cos; i++)
+		REG_WR(sc, base_weight + (0x4 * i), 0);
+
+	elink_ets_e3b0_set_credit_upper_bound_pbf(params, min_w_val_pbf);
+}
+/******************************************************************************
+ * Description:
+ *	E3B0 disable will return basicly the values to init values.
+ *.
+ ******************************************************************************/
+static elink_status_t elink_ets_e3b0_disabled(const struct elink_params *params,
+				   const struct elink_vars *vars)
+{
+	struct bnx2x_softc *sc = params->sc;
+
+	if (!CHIP_IS_E3B0(sc)) {
+		ELINK_DEBUG_P0(sc,
+		   "elink_ets_e3b0_disabled the chip isn't E3B0");
+		return ELINK_STATUS_ERROR;
+	}
+
+	elink_ets_e3b0_nig_disabled(params, vars);
+
+	elink_ets_e3b0_pbf_disabled(params);
+
+	return ELINK_STATUS_OK;
+}
+
+/******************************************************************************
+ * Description:
+ *	Disable will return basicly the values to init values.
+ *
+ ******************************************************************************/
+elink_status_t elink_ets_disabled(struct elink_params *params,
+		      struct elink_vars *vars)
+{
+	struct bnx2x_softc *sc = params->sc;
+	elink_status_t elink_status = ELINK_STATUS_OK;
+
+	if ((CHIP_IS_E2(sc)) || (CHIP_IS_E3A0(sc))) {
+		elink_ets_e2e3a0_disabled(params);
+	} else if (CHIP_IS_E3B0(sc)) {
+		elink_status = elink_ets_e3b0_disabled(params, vars);
+	} else {
+		ELINK_DEBUG_P0(sc, "elink_ets_disabled - chip not supported");
+		return ELINK_STATUS_ERROR;
+	}
+
+	return elink_status;
+}
+
+/******************************************************************************
+ * Description
+ *	Set the COS mappimg to SP and BW until this point all the COS are not
+ *	set as SP or BW.
+ ******************************************************************************/
+static elink_status_t elink_ets_e3b0_cli_map(const struct elink_params *params,
+		  __rte_unused const struct elink_ets_params *ets_params,
+		  const uint8_t cos_sp_bitmap,
+		  const uint8_t cos_bw_bitmap)
+{
+	struct bnx2x_softc *sc = params->sc;
+	const uint8_t port = params->port;
+	const uint8_t nig_cli_sp_bitmap = 0x7 | (cos_sp_bitmap << 3);
+	const uint8_t pbf_cli_sp_bitmap = cos_sp_bitmap;
+	const uint8_t nig_cli_subject2wfq_bitmap = cos_bw_bitmap << 3;
+	const uint8_t pbf_cli_subject2wfq_bitmap = cos_bw_bitmap;
+
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CLIENT_IS_STRICT :
+	       NIG_REG_P0_TX_ARB_CLIENT_IS_STRICT, nig_cli_sp_bitmap);
+
+	REG_WR(sc, (port) ? PBF_REG_ETS_ARB_CLIENT_IS_STRICT_P1 :
+	       PBF_REG_ETS_ARB_CLIENT_IS_STRICT_P0, pbf_cli_sp_bitmap);
+
+	REG_WR(sc, (port) ? NIG_REG_P1_TX_ARB_CLIENT_IS_SUBJECT2WFQ :
+	       NIG_REG_P0_TX_ARB_CLIENT_IS_SUBJECT2WFQ,
+	       nig_cli_subject2wfq_bitmap);
+
+	REG_WR(sc, (port) ? PBF_REG_ETS_ARB_CLIENT_IS_SUBJECT2WFQ_P1 :
+	       PBF_REG_ETS_ARB_CLIENT_IS_SUBJECT2WFQ_P0,
+	       pbf_cli_subject2wfq_bitmap);
+
+	return ELINK_STATUS_OK;
+}
+
+/******************************************************************************
+ * Description:
+ *	This function is needed because NIG ARB_CREDIT_WEIGHT_X are
+ *	not continues and ARB_CREDIT_WEIGHT_0 + offset is suitable.
+ ******************************************************************************/
+static elink_status_t elink_ets_e3b0_set_cos_bw(struct bnx2x_softc *sc,
+				     const uint8_t cos_entry,
+				     const uint32_t min_w_val_nig,
+				     const uint32_t min_w_val_pbf,
+				     const uint16_t total_bw,
+				     const uint8_t bw,
+				     const uint8_t port)
+{
+	uint32_t nig_reg_address_crd_weight = 0;
+	uint32_t pbf_reg_address_crd_weight = 0;
+	/* Calculate and set BW for this COS - use 1 instead of 0 for BW */
+	const uint32_t cos_bw_nig = ((bw ? bw : 1) * min_w_val_nig) / total_bw;
+	const uint32_t cos_bw_pbf = ((bw ? bw : 1) * min_w_val_pbf) / total_bw;
+
+	switch (cos_entry) {
+	case 0:
+	    nig_reg_address_crd_weight =
+		 (port) ? NIG_REG_P1_TX_ARB_CREDIT_WEIGHT_0 :
+		     NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_0;
+	     pbf_reg_address_crd_weight = (port) ?
+		 PBF_REG_COS0_WEIGHT_P1 : PBF_REG_COS0_WEIGHT_P0;
+		break;
+	case 1:
+	     nig_reg_address_crd_weight = (port) ?
+		 NIG_REG_P1_TX_ARB_CREDIT_WEIGHT_1 :
+		 NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_1;
+	     pbf_reg_address_crd_weight = (port) ?
+		 PBF_REG_COS1_WEIGHT_P1 : PBF_REG_COS1_WEIGHT_P0;
+		break;
+	case 2:
+	     nig_reg_address_crd_weight = (port) ?
+		 NIG_REG_P1_TX_ARB_CREDIT_WEIGHT_2 :
+		 NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_2;
+
+		 pbf_reg_address_crd_weight = (port) ?
+		     PBF_REG_COS2_WEIGHT_P1 : PBF_REG_COS2_WEIGHT_P0;
+		break;
+	case 3:
+		if (port)
+			return ELINK_STATUS_ERROR;
+		nig_reg_address_crd_weight =
+			NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_3;
+		pbf_reg_address_crd_weight =
+			PBF_REG_COS3_WEIGHT_P0;
+		break;
+	case 4:
+		if (port)
+		return ELINK_STATUS_ERROR;
+	     nig_reg_address_crd_weight =
+		 NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_4;
+	     pbf_reg_address_crd_weight = PBF_REG_COS4_WEIGHT_P0;
+		break;
+	case 5:
+		if (port)
+		return ELINK_STATUS_ERROR;
+	     nig_reg_address_crd_weight =
+		 NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_5;
+	     pbf_reg_address_crd_weight = PBF_REG_COS5_WEIGHT_P0;
+		break;
+	}
+
+	REG_WR(sc, nig_reg_address_crd_weight, cos_bw_nig);
+
+	REG_WR(sc, pbf_reg_address_crd_weight, cos_bw_pbf);
+
+	return ELINK_STATUS_OK;
+}
+/******************************************************************************
+ * Description:
+ *	Calculate the total BW.A value of 0 isn't legal.
+ *
+ ******************************************************************************/
+static elink_status_t elink_ets_e3b0_get_total_bw(
+	const struct elink_params *params,
+	struct elink_ets_params *ets_params,
+	uint16_t *total_bw)
+{
+	struct bnx2x_softc *sc = params->sc;
+	uint8_t cos_idx = 0;
+	uint8_t is_bw_cos_exist = 0;
+
+	*total_bw = 0;
+	/* Calculate total BW requested */
+	for (cos_idx = 0; cos_idx < ets_params->num_of_cos; cos_idx++) {
+		if (ets_params->cos[cos_idx].state == elink_cos_state_bw) {
+			is_bw_cos_exist = 1;
+			if (!ets_params->cos[cos_idx].params.bw_params.bw) {
+				ELINK_DEBUG_P0(sc, "elink_ets_E3B0_config BW"
+						   " was set to 0");
+				/* This is to prevent a state when ramrods
+				 * can't be sent
+				 */
+				ets_params->cos[cos_idx].params.bw_params.bw
+					 = 1;
+			}
+			*total_bw +=
+				ets_params->cos[cos_idx].params.bw_params.bw;
+		}
+	}
+
+	/* Check total BW is valid */
+	if ((is_bw_cos_exist == 1) && (*total_bw != 100)) {
+		if (*total_bw == 0) {
+			ELINK_DEBUG_P0(sc,
+			   "elink_ets_E3B0_config total BW shouldn't be 0");
+			return ELINK_STATUS_ERROR;
+		}
+		ELINK_DEBUG_P0(sc,
+		   "elink_ets_E3B0_config total BW should be 100");
+		/* We can handle a case whre the BW isn't 100 this can happen
+		 * if the TC are joined.
+		 */
+	}
+	return ELINK_STATUS_OK;
+}
+
+/******************************************************************************
+ * Description:
+ *	Invalidate all the sp_pri_to_cos.
+ *
+ ******************************************************************************/
+static void elink_ets_e3b0_sp_pri_to_cos_init(uint8_t *sp_pri_to_cos)
+{
+	uint8_t pri = 0;
+	for (pri = 0; pri < ELINK_DCBX_MAX_NUM_COS; pri++)
+		sp_pri_to_cos[pri] = DCBX_INVALID_COS;
+}
+/******************************************************************************
+ * Description:
+ *	Calculate and set the SP (ARB_PRIORITY_CLIENT) NIG and PBF registers
+ *	according to sp_pri_to_cos.
+ *
+ ******************************************************************************/
+static elink_status_t elink_ets_e3b0_sp_pri_to_cos_set(
+					    const struct elink_params *params,
+					    uint8_t *sp_pri_to_cos,
+					    const uint8_t pri,
+					    const uint8_t cos_entry)
+{
+	struct bnx2x_softc *sc = params->sc;
+	const uint8_t port = params->port;
+	const uint8_t max_num_of_cos = (port) ?
+		ELINK_DCBX_E3B0_MAX_NUM_COS_PORT1 :
+		ELINK_DCBX_E3B0_MAX_NUM_COS_PORT0;
+
+	if (pri >= max_num_of_cos) {
+		ELINK_DEBUG_P0(sc, "elink_ets_e3b0_sp_pri_to_cos_set invalid "
+		   "parameter Illegal strict priority");
+		return ELINK_STATUS_ERROR;
+	}
+
+	if (sp_pri_to_cos[pri] != DCBX_INVALID_COS) {
+		ELINK_DEBUG_P0(sc, "elink_ets_e3b0_sp_pri_to_cos_set invalid "
+				   "parameter There can't be two COS's with "
+				   "the same strict pri");
+		return ELINK_STATUS_ERROR;
+	}
+
+	sp_pri_to_cos[pri] = cos_entry;
+	return ELINK_STATUS_OK;
+}
+
+/******************************************************************************
+ * Description:
+ *	Returns the correct value according to COS and priority in
+ *	the sp_pri_cli register.
+ *
+ ******************************************************************************/
+static uint64_t elink_e3b0_sp_get_pri_cli_reg(const uint8_t cos,
+					 const uint8_t cos_offset,
+					 const uint8_t pri_set,
+					 const uint8_t pri_offset,
+					 const uint8_t entry_size)
+{
+	uint64_t pri_cli_nig = 0;
+	pri_cli_nig = ((uint64_t)(cos + cos_offset)) << (entry_size *
+						    (pri_set + pri_offset));
+
+	return pri_cli_nig;
+}
+/******************************************************************************
+ * Description:
+ *	Returns the correct value according to COS and priority in the
+ *	sp_pri_cli register for NIG.
+ *
+ ******************************************************************************/
+static uint64_t elink_e3b0_sp_get_pri_cli_reg_nig(const uint8_t cos,
+						  const uint8_t pri_set)
+{
+	/* MCP Dbg0 and dbg1 are always with higher strict pri*/
+	const uint8_t nig_cos_offset = 3;
+	const uint8_t nig_pri_offset = 3;
+
+	return elink_e3b0_sp_get_pri_cli_reg(cos, nig_cos_offset, pri_set,
+		nig_pri_offset, 4);
+}
 
+/******************************************************************************
+ * Description:
+ *	Returns the correct value according to COS and priority in the
+ *	sp_pri_cli register for PBF.
+ *
+ ******************************************************************************/
+static uint64_t elink_e3b0_sp_get_pri_cli_reg_pbf(const uint8_t cos,
+						  const uint8_t pri_set)
+{
+	const uint8_t pbf_cos_offset = 0;
+	const uint8_t pbf_pri_offset = 0;
+
+	return elink_e3b0_sp_get_pri_cli_reg(cos, pbf_cos_offset, pri_set,
+		pbf_pri_offset, 3);
+}
+
+/******************************************************************************
+ * Description:
+ *	Calculate and set the SP (ARB_PRIORITY_CLIENT) NIG and PBF registers
+ *	according to sp_pri_to_cos.(which COS has higher priority)
+ *
+ ******************************************************************************/
+static elink_status_t elink_ets_e3b0_sp_set_pri_cli_reg(
+					     const struct elink_params *params,
+					     uint8_t *sp_pri_to_cos)
+{
+	struct bnx2x_softc *sc = params->sc;
+	uint8_t i = 0;
+	const uint8_t port = params->port;
+	/* MCP Dbg0 and dbg1 are always with higher strict pri*/
+	uint64_t pri_cli_nig = 0x210;
+	uint32_t pri_cli_pbf = 0x0;
+	uint8_t pri_set = 0;
+	uint8_t pri_bitmask = 0;
+	const uint8_t max_num_of_cos = (port) ?
+		ELINK_DCBX_E3B0_MAX_NUM_COS_PORT1 :
+		ELINK_DCBX_E3B0_MAX_NUM_COS_PORT0;
+
+	uint8_t cos_bit_to_set = (1 << max_num_of_cos) - 1;
+
+	/* Set all the strict priority first */
+	for (i = 0; i < max_num_of_cos; i++) {
+		if (sp_pri_to_cos[i] != DCBX_INVALID_COS) {
+			if (sp_pri_to_cos[i] >= ELINK_DCBX_MAX_NUM_COS) {
+				ELINK_DEBUG_P0(sc,
+					   "elink_ets_e3b0_sp_set_pri_cli_reg "
+					   "invalid cos entry");
+				return ELINK_STATUS_ERROR;
+			}
+
+			pri_cli_nig |= elink_e3b0_sp_get_pri_cli_reg_nig(
+			    sp_pri_to_cos[i], pri_set);
+
+			pri_cli_pbf |= elink_e3b0_sp_get_pri_cli_reg_pbf(
+			    sp_pri_to_cos[i], pri_set);
+			pri_bitmask = 1 << sp_pri_to_cos[i];
+			/* COS is used remove it from bitmap.*/
+			if (!(pri_bitmask & cos_bit_to_set)) {
+				ELINK_DEBUG_P0(sc,
+					"elink_ets_e3b0_sp_set_pri_cli_reg "
+					"invalid There can't be two COS's with"
+					" the same strict pri");
+				return ELINK_STATUS_ERROR;
+			}
+			cos_bit_to_set &= ~pri_bitmask;
+			pri_set++;
+		}
+	}
+
+	/* Set all the Non strict priority i= COS*/
+	for (i = 0; i < max_num_of_cos; i++) {
+		pri_bitmask = 1 << i;
+		/* Check if COS was already used for SP */
+		if (pri_bitmask & cos_bit_to_set) {
+			/* COS wasn't used for SP */
+			pri_cli_nig |= elink_e3b0_sp_get_pri_cli_reg_nig(
+			    i, pri_set);
+
+			pri_cli_pbf |= elink_e3b0_sp_get_pri_cli_reg_pbf(
+			    i, pri_set);
+			/* COS is used remove it from bitmap.*/
+			cos_bit_to_set &= ~pri_bitmask;
+			pri_set++;
+		}
+	}
+
+	if (pri_set != max_num_of_cos) {
+		ELINK_DEBUG_P0(sc, "elink_ets_e3b0_sp_set_pri_cli_reg not all "
+				   "entries were set");
+		return ELINK_STATUS_ERROR;
+	}
+
+	if (port) {
+		/* Only 6 usable clients*/
+		REG_WR(sc, NIG_REG_P1_TX_ARB_PRIORITY_CLIENT2_LSB,
+		       (uint32_t)pri_cli_nig);
+
+		REG_WR(sc, PBF_REG_ETS_ARB_PRIORITY_CLIENT_P1, pri_cli_pbf);
+	} else {
+		/* Only 9 usable clients*/
+		const uint32_t pri_cli_nig_lsb = (uint32_t)(pri_cli_nig);
+		const uint32_t pri_cli_nig_msb = (uint32_t)
+						((pri_cli_nig >> 32) & 0xF);
+
+		REG_WR(sc, NIG_REG_P0_TX_ARB_PRIORITY_CLIENT2_LSB,
+		       pri_cli_nig_lsb);
+		REG_WR(sc, NIG_REG_P0_TX_ARB_PRIORITY_CLIENT2_MSB,
+		       pri_cli_nig_msb);
+
+		REG_WR(sc, PBF_REG_ETS_ARB_PRIORITY_CLIENT_P0, pri_cli_pbf);
+	}
+	return ELINK_STATUS_OK;
+}
+
+/******************************************************************************
+ * Description:
+ *	Configure the COS to ETS according to BW and SP settings.
+ ******************************************************************************/
+elink_status_t elink_ets_e3b0_config(const struct elink_params *params,
+			 const struct elink_vars *vars,
+			 struct elink_ets_params *ets_params)
+{
+	struct bnx2x_softc *sc = params->sc;
+	elink_status_t elink_status = ELINK_STATUS_OK;
+	const uint8_t port = params->port;
+	uint16_t total_bw = 0;
+	const uint32_t min_w_val_nig = elink_ets_get_min_w_val_nig(vars);
+	const uint32_t min_w_val_pbf = ELINK_ETS_E3B0_PBF_MIN_W_VAL;
+	uint8_t cos_bw_bitmap = 0;
+	uint8_t cos_sp_bitmap = 0;
+	uint8_t sp_pri_to_cos[ELINK_DCBX_MAX_NUM_COS] = {0};
+	const uint8_t max_num_of_cos = (port) ?
+		ELINK_DCBX_E3B0_MAX_NUM_COS_PORT1 :
+		ELINK_DCBX_E3B0_MAX_NUM_COS_PORT0;
+	uint8_t cos_entry = 0;
+
+	if (!CHIP_IS_E3B0(sc)) {
+		ELINK_DEBUG_P0(sc,
+		   "elink_ets_e3b0_disabled the chip isn't E3B0");
+		return ELINK_STATUS_ERROR;
+	}
+
+	if (ets_params->num_of_cos > max_num_of_cos) {
+		ELINK_DEBUG_P0(sc, "elink_ets_E3B0_config the number of COS "
+				   "isn't supported");
+		return ELINK_STATUS_ERROR;
+	}
+
+	/* Prepare sp strict priority parameters*/
+	elink_ets_e3b0_sp_pri_to_cos_init(sp_pri_to_cos);
+
+	/* Prepare BW parameters*/
+	elink_status = elink_ets_e3b0_get_total_bw(params, ets_params,
+						   &total_bw);
+	if (elink_status != ELINK_STATUS_OK) {
+		ELINK_DEBUG_P0(sc,
+		   "elink_ets_E3B0_config get_total_bw failed");
+		return ELINK_STATUS_ERROR;
+	}
+
+	/* Upper bound is set according to current link speed (min_w_val
+	 * should be the same for upper bound and COS credit val).
+	 */
+	elink_ets_e3b0_set_credit_upper_bound_nig(params, min_w_val_nig);
+	elink_ets_e3b0_set_credit_upper_bound_pbf(params, min_w_val_pbf);
+
+
+	for (cos_entry = 0; cos_entry < ets_params->num_of_cos; cos_entry++) {
+		if (elink_cos_state_bw == ets_params->cos[cos_entry].state) {
+			cos_bw_bitmap |= (1 << cos_entry);
+			/* The function also sets the BW in HW(not the mappin
+			 * yet)
+			 */
+			elink_status = elink_ets_e3b0_set_cos_bw(
+				sc, cos_entry, min_w_val_nig, min_w_val_pbf,
+				total_bw,
+				ets_params->cos[cos_entry].params.bw_params.bw,
+				 port);
+		} else if (elink_cos_state_strict ==
+			ets_params->cos[cos_entry].state){
+			cos_sp_bitmap |= (1 << cos_entry);
+
+			elink_status = elink_ets_e3b0_sp_pri_to_cos_set(
+				params,
+				sp_pri_to_cos,
+				ets_params->cos[cos_entry].params.sp_params.pri,
+				cos_entry);
+
+		} else {
+			ELINK_DEBUG_P0(sc,
+			   "elink_ets_e3b0_config cos state not valid");
+			return ELINK_STATUS_ERROR;
+		}
+		if (elink_status != ELINK_STATUS_OK) {
+			ELINK_DEBUG_P0(sc,
+			   "elink_ets_e3b0_config set cos bw failed");
+			return elink_status;
+		}
+	}
+
+	/* Set SP register (which COS has higher priority) */
+	elink_status = elink_ets_e3b0_sp_set_pri_cli_reg(params,
+							 sp_pri_to_cos);
+
+	if (elink_status != ELINK_STATUS_OK) {
+		ELINK_DEBUG_P0(sc,
+		   "elink_ets_E3B0_config set_pri_cli_reg failed");
+		return elink_status;
+	}
+
+	/* Set client mapping of BW and strict */
+	elink_status = elink_ets_e3b0_cli_map(params, ets_params,
+					      cos_sp_bitmap,
+					      cos_bw_bitmap);
+
+	if (elink_status != ELINK_STATUS_OK) {
+		ELINK_DEBUG_P0(sc, "elink_ets_E3B0_config SP failed");
+		return elink_status;
+	}
+	return ELINK_STATUS_OK;
+}
+static void elink_ets_bw_limit_common(const struct elink_params *params)
+{
+	/* ETS disabled configuration */
+	struct bnx2x_softc *sc = params->sc;
+	ELINK_DEBUG_P0(sc, "ETS enabled BW limit configuration");
+	/* Defines which entries (clients) are subjected to WFQ arbitration
+	 * COS0 0x8
+	 * COS1 0x10
+	 */
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CLIENT_IS_SUBJECT2WFQ, 0x18);
+	/* Mapping between the ARB_CREDIT_WEIGHT registers and actual
+	 * client numbers (WEIGHT_0 does not actually have to represent
+	 * client 0)
+	 *    PRI4    |    PRI3    |    PRI2    |    PRI1    |    PRI0
+	 *  cos1-001     cos0-000     dbg1-100     dbg0-011     MCP-010
+	 */
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CLIENT_CREDIT_MAP, 0x111A);
+
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_0,
+	       ELINK_ETS_BW_LIMIT_CREDIT_UPPER_BOUND);
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_UPPER_BOUND_1,
+	       ELINK_ETS_BW_LIMIT_CREDIT_UPPER_BOUND);
+
+	/* ETS mode enabled*/
+	REG_WR(sc, PBF_REG_ETS_ENABLED, 1);
+
+	/* Defines the number of consecutive slots for the strict priority */
+	REG_WR(sc, PBF_REG_NUM_STRICT_ARB_SLOTS, 0);
+	/* Bitmap of 5bits length. Each bit specifies whether the entry behaves
+	 * as strict.  Bits 0,1,2 - debug and management entries, 3 - COS0
+	 * entry, 4 - COS1 entry.
+	 * COS1 | COS0 | DEBUG21 | DEBUG0 | MGMT
+	 * bit4   bit3	  bit2     bit1	   bit0
+	 * MCP and debug are strict
+	 */
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CLIENT_IS_STRICT, 0x7);
+
+	/* Upper bound that COS0_WEIGHT can reach in the WFQ arbiter.*/
+	REG_WR(sc, PBF_REG_COS0_UPPER_BOUND,
+	       ELINK_ETS_BW_LIMIT_CREDIT_UPPER_BOUND);
+	REG_WR(sc, PBF_REG_COS1_UPPER_BOUND,
+	       ELINK_ETS_BW_LIMIT_CREDIT_UPPER_BOUND);
+}
+
+void elink_ets_bw_limit(const struct elink_params *params,
+			const uint32_t cos0_bw,
+			const uint32_t cos1_bw)
+{
+	/* ETS disabled configuration*/
+	struct bnx2x_softc *sc = params->sc;
+	const uint32_t total_bw = cos0_bw + cos1_bw;
+	uint32_t cos0_credit_weight = 0;
+	uint32_t cos1_credit_weight = 0;
+
+	ELINK_DEBUG_P0(sc, "ETS enabled BW limit configuration");
+
+	if ((!total_bw) ||
+	    (!cos0_bw) ||
+	    (!cos1_bw)) {
+		ELINK_DEBUG_P0(sc, "Total BW can't be zero");
+		return;
+	}
+
+	cos0_credit_weight = (cos0_bw * ELINK_ETS_BW_LIMIT_CREDIT_WEIGHT) /
+		total_bw;
+	cos1_credit_weight = (cos1_bw * ELINK_ETS_BW_LIMIT_CREDIT_WEIGHT) /
+		total_bw;
+
+	elink_ets_bw_limit_common(params);
+
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_0, cos0_credit_weight);
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CREDIT_WEIGHT_1, cos1_credit_weight);
+
+	REG_WR(sc, PBF_REG_COS0_WEIGHT, cos0_credit_weight);
+	REG_WR(sc, PBF_REG_COS1_WEIGHT, cos1_credit_weight);
+}
+
+elink_status_t elink_ets_strict(const struct elink_params *params,
+				const uint8_t strict_cos)
+{
+	/* ETS disabled configuration*/
+	struct bnx2x_softc *sc = params->sc;
+	uint32_t val	= 0;
+
+	ELINK_DEBUG_P0(sc, "ETS enabled strict configuration");
+	/* Bitmap of 5bits length. Each bit specifies whether the entry behaves
+	 * as strict.  Bits 0,1,2 - debug and management entries,
+	 * 3 - COS0 entry, 4 - COS1 entry.
+	 *  COS1 | COS0 | DEBUG21 | DEBUG0 | MGMT
+	 *  bit4   bit3	  bit2      bit1     bit0
+	 * MCP and debug are strict
+	 */
+	REG_WR(sc, NIG_REG_P0_TX_ARB_CLIENT_IS_STRICT, 0x1F);
+	/* For strict priority entries defines the number of consecutive slots
+	 * for the highest priority.
+	 */
+	REG_WR(sc, NIG_REG_P0_TX_ARB_NUM_STRICT_ARB_SLOTS, 0x100);
+	/* ETS mode disable */
+	REG_WR(sc, PBF_REG_ETS_ENABLED, 0);
+	/* Defines the number of consecutive slots for the strict priority */
+	REG_WR(sc, PBF_REG_NUM_STRICT_ARB_SLOTS, 0x100);
+
+	/* Defines the number of consecutive slots for the strict priority */
+	REG_WR(sc, PBF_REG_HIGH_PRIORITY_COS_NUM, strict_cos);
+
+	/* Mapping between entry  priority to client number (0,1,2 -debug and
+	 * management clients, 3 - COS0 client, 4 - COS client)(HIGHEST)
+	 * 3bits client num.
+	 *   PRI4    |    PRI3    |    PRI2    |    PRI1    |    PRI0
+	 * dbg0-010     dbg1-001     cos1-100     cos0-011     MCP-000
+	 * dbg0-010     dbg1-001     cos0-011     cos1-100     MCP-000
+	 */
+	val = (!strict_cos) ? 0x2318 : 0x22E0;
+	REG_WR(sc, NIG_REG_P0_TX_ARB_PRIORITY_CLIENT, val);
+
+	return ELINK_STATUS_OK;
 }
 
 /******************************************************************/
 /*			PFC section				  */
 /******************************************************************/
 static void elink_update_pfc_xmac(struct elink_params *params,
-				  struct elink_vars *vars)
+				  struct elink_vars *vars,
+				  __rte_unused uint8_t is_lb)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint32_t xmac_base;
@@ -1143,7 +2099,8 @@ static void elink_update_pfc_xmac(struct elink_params *params,
 	pfc1_val = 0x2;
 
 	/* No PFC support */
-	if (!(params->feature_config_flags & ELINK_FEATURE_CONFIG_PFC_ENABLED)) {
+	if (!(params->feature_config_flags &
+	      ELINK_FEATURE_CONFIG_PFC_ENABLED)) {
 
 		/* RX flow control - Process pause frame in receive direction
 		 */
@@ -1153,12 +2110,12 @@ static void elink_update_pfc_xmac(struct elink_params *params,
 		/* TX flow control - Send pause packet when buffer is full */
 		if (vars->flow_ctrl & ELINK_FLOW_CTRL_TX)
 			pause_val |= XMAC_PAUSE_CTRL_REG_TX_PAUSE_EN;
-	} else {		/* PFC support */
+	} else {/* PFC support */
 		pfc1_val |= XMAC_PFC_CTRL_HI_REG_PFC_REFRESH_EN |
-		    XMAC_PFC_CTRL_HI_REG_PFC_STATS_EN |
-		    XMAC_PFC_CTRL_HI_REG_RX_PFC_EN |
-		    XMAC_PFC_CTRL_HI_REG_TX_PFC_EN |
-		    XMAC_PFC_CTRL_HI_REG_FORCE_PFC_XON;
+			XMAC_PFC_CTRL_HI_REG_PFC_STATS_EN |
+			XMAC_PFC_CTRL_HI_REG_RX_PFC_EN |
+			XMAC_PFC_CTRL_HI_REG_TX_PFC_EN |
+			XMAC_PFC_CTRL_HI_REG_FORCE_PFC_XON;
 		/* Write pause and PFC registers */
 		REG_WR(sc, xmac_base + XMAC_REG_PAUSE_CTRL, pause_val);
 		REG_WR(sc, xmac_base + XMAC_REG_PFC_CTRL, pfc0_val);
@@ -1172,21 +2129,76 @@ static void elink_update_pfc_xmac(struct elink_params *params,
 	REG_WR(sc, xmac_base + XMAC_REG_PFC_CTRL, pfc0_val);
 	REG_WR(sc, xmac_base + XMAC_REG_PFC_CTRL_HI, pfc1_val);
 
+
 	/* Set MAC address for source TX Pause/PFC frames */
 	REG_WR(sc, xmac_base + XMAC_REG_CTRL_SA_LO,
 	       ((params->mac_addr[2] << 24) |
 		(params->mac_addr[3] << 16) |
-		(params->mac_addr[4] << 8) | (params->mac_addr[5])));
+		(params->mac_addr[4] << 8) |
+		(params->mac_addr[5])));
 	REG_WR(sc, xmac_base + XMAC_REG_CTRL_SA_HI,
-	       ((params->mac_addr[0] << 8) | (params->mac_addr[1])));
+	       ((params->mac_addr[0] << 8) |
+		(params->mac_addr[1])));
 
 	DELAY(30);
 }
 
+static void elink_emac_get_pfc_stat(struct elink_params *params,
+				    uint32_t pfc_frames_sent[2],
+				    uint32_t pfc_frames_received[2])
+{
+	/* Read pfc statistic */
+	struct bnx2x_softc *sc = params->sc;
+	uint32_t emac_base = params->port ? GRCBASE_EMAC1 : GRCBASE_EMAC0;
+	uint32_t val_xon = 0;
+	uint32_t val_xoff = 0;
+
+	ELINK_DEBUG_P0(sc, "pfc statistic read from EMAC");
+
+	/* PFC received frames */
+	val_xoff = REG_RD(sc, emac_base +
+				EMAC_REG_RX_PFC_STATS_XOFF_RCVD);
+	val_xoff &= EMAC_REG_RX_PFC_STATS_XOFF_RCVD_COUNT;
+	val_xon = REG_RD(sc, emac_base + EMAC_REG_RX_PFC_STATS_XON_RCVD);
+	val_xon &= EMAC_REG_RX_PFC_STATS_XON_RCVD_COUNT;
+
+	pfc_frames_received[0] = val_xon + val_xoff;
+
+	/* PFC received sent */
+	val_xoff = REG_RD(sc, emac_base +
+				EMAC_REG_RX_PFC_STATS_XOFF_SENT);
+	val_xoff &= EMAC_REG_RX_PFC_STATS_XOFF_SENT_COUNT;
+	val_xon = REG_RD(sc, emac_base + EMAC_REG_RX_PFC_STATS_XON_SENT);
+	val_xon &= EMAC_REG_RX_PFC_STATS_XON_SENT_COUNT;
+
+	pfc_frames_sent[0] = val_xon + val_xoff;
+}
+
+/* Read pfc statistic*/
+void elink_pfc_statistic(struct elink_params *params, struct elink_vars *vars,
+			 uint32_t pfc_frames_sent[2],
+			 uint32_t pfc_frames_received[2])
+{
+	/* Read pfc statistic */
+	struct bnx2x_softc *sc = params->sc;
+
+	ELINK_DEBUG_P0(sc, "pfc statistic");
+
+	if (!vars->link_up)
+		return;
+
+	if (vars->mac_type == ELINK_MAC_TYPE_EMAC) {
+		ELINK_DEBUG_P0(sc, "About to read PFC stats from EMAC");
+		elink_emac_get_pfc_stat(params, pfc_frames_sent,
+					pfc_frames_received);
+	}
+}
 /******************************************************************/
 /*			MAC/PBF section				  */
 /******************************************************************/
-static void elink_set_mdio_clk(struct bnx2x_softc *sc, uint32_t emac_base)
+static void elink_set_mdio_clk(struct bnx2x_softc *sc,
+			       __rte_unused uint32_t chip_id,
+			       uint32_t emac_base)
 {
 	uint32_t new_mode, cur_mode;
 	uint32_t clc_cnt;
@@ -1205,26 +2217,16 @@ static void elink_set_mdio_clk(struct bnx2x_softc *sc, uint32_t emac_base)
 		return;
 
 	new_mode = cur_mode &
-	    ~(EMAC_MDIO_MODE_AUTO_POLL | EMAC_MDIO_MODE_CLOCK_CNT);
+		~(EMAC_MDIO_MODE_AUTO_POLL | EMAC_MDIO_MODE_CLOCK_CNT);
 	new_mode |= clc_cnt;
 	new_mode |= (EMAC_MDIO_MODE_CLAUSE_45);
 
-	PMD_DRV_LOG(DEBUG, "Changing emac_mode from 0x%x to 0x%x",
-		    cur_mode, new_mode);
+	ELINK_DEBUG_P2(sc, "Changing emac_mode from 0x%x to 0x%x",
+	   cur_mode, new_mode);
 	REG_WR(sc, emac_base + EMAC_REG_EMAC_MDIO_MODE, new_mode);
 	DELAY(40);
 }
 
-static void elink_set_mdio_emac_per_phy(struct bnx2x_softc *sc,
-					struct elink_params *params)
-{
-	uint8_t phy_index;
-	/* Set mdio clock per phy */
-	for (phy_index = ELINK_INT_PHY; phy_index < params->num_phys;
-	     phy_index++)
-		elink_set_mdio_clk(sc, params->phy[phy_index].mdio_ctrl);
-}
-
 static uint8_t elink_is_4_port_mode(struct bnx2x_softc *sc)
 {
 	uint32_t port4mode_ovwr_val;
@@ -1232,13 +2234,26 @@ static uint8_t elink_is_4_port_mode(struct bnx2x_softc *sc)
 	port4mode_ovwr_val = REG_RD(sc, MISC_REG_PORT4MODE_EN_OVWR);
 	if (port4mode_ovwr_val & (1 << 0)) {
 		/* Return 4-port mode override value */
-		return (port4mode_ovwr_val & (1 << 1)) == (1 << 1);
+		return ((port4mode_ovwr_val & (1 << 1)) == (1 << 1));
 	}
 	/* Return 4-port mode from input pin */
-	return (uint8_t) REG_RD(sc, MISC_REG_PORT4MODE_EN);
+	return (uint8_t)REG_RD(sc, MISC_REG_PORT4MODE_EN);
+}
+
+static void elink_set_mdio_emac_per_phy(struct bnx2x_softc *sc,
+					struct elink_params *params)
+{
+	uint8_t phy_index;
+
+	/* Set mdio clock per phy */
+	for (phy_index = ELINK_INT_PHY; phy_index < params->num_phys;
+	      phy_index++)
+		elink_set_mdio_clk(sc, params->chip_id,
+				   params->phy[phy_index].mdio_ctrl);
 }
 
-static void elink_emac_init(struct elink_params *params)
+static void elink_emac_init(struct elink_params *params,
+			    __rte_unused struct elink_vars *vars)
 {
 	/* reset and unreset the emac core */
 	struct bnx2x_softc *sc = params->sc;
@@ -1262,9 +2277,9 @@ static void elink_emac_init(struct elink_params *params)
 	timeout = 200;
 	do {
 		val = REG_RD(sc, emac_base + EMAC_REG_EMAC_MODE);
-		PMD_DRV_LOG(DEBUG, "EMAC reset reg is %u", val);
+		ELINK_DEBUG_P1(sc, "EMAC reset reg is %u", val);
 		if (!timeout) {
-			PMD_DRV_LOG(DEBUG, "EMAC timeout!");
+			ELINK_DEBUG_P0(sc, "EMAC timeout!");
 			return;
 		}
 		timeout--;
@@ -1272,17 +2287,20 @@ static void elink_emac_init(struct elink_params *params)
 
 	elink_set_mdio_emac_per_phy(sc, params);
 	/* Set mac address */
-	val = ((params->mac_addr[0] << 8) | params->mac_addr[1]);
+	val = ((params->mac_addr[0] << 8) |
+		params->mac_addr[1]);
 	elink_cb_reg_write(sc, emac_base + EMAC_REG_EMAC_MAC_MATCH, val);
 
 	val = ((params->mac_addr[2] << 24) |
 	       (params->mac_addr[3] << 16) |
-	       (params->mac_addr[4] << 8) | params->mac_addr[5]);
+	       (params->mac_addr[4] << 8) |
+		params->mac_addr[5]);
 	elink_cb_reg_write(sc, emac_base + EMAC_REG_EMAC_MAC_MATCH + 4, val);
 }
 
 static void elink_set_xumac_nig(struct elink_params *params,
-				uint16_t tx_pause_en, uint8_t enable)
+				uint16_t tx_pause_en,
+				uint8_t enable)
 {
 	struct bnx2x_softc *sc = params->sc;
 
@@ -1300,7 +2318,7 @@ static void elink_set_umac_rxtx(struct elink_params *params, uint8_t en)
 	uint32_t val;
 	struct bnx2x_softc *sc = params->sc;
 	if (!(REG_RD(sc, MISC_REG_RESET_REG_2) &
-	      (MISC_REGISTERS_RESET_REG_2_UMAC0 << params->port)))
+		   (MISC_REGISTERS_RESET_REG_2_UMAC0 << params->port)))
 		return;
 	val = REG_RD(sc, umac_base + UMAC_REG_COMMAND_CONFIG);
 	if (en)
@@ -1314,7 +2332,7 @@ static void elink_set_umac_rxtx(struct elink_params *params, uint8_t en)
 }
 
 static void elink_umac_enable(struct elink_params *params,
-			      struct elink_vars *vars, uint8_t lb)
+			    struct elink_vars *vars, uint8_t lb)
 {
 	uint32_t val;
 	uint32_t umac_base = params->port ? GRCBASE_UMAC1 : GRCBASE_UMAC0;
@@ -1327,15 +2345,15 @@ static void elink_umac_enable(struct elink_params *params,
 	REG_WR(sc, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_SET,
 	       (MISC_REGISTERS_RESET_REG_2_UMAC0 << params->port));
 
-	PMD_DRV_LOG(DEBUG, "enabling UMAC");
+	ELINK_DEBUG_P0(sc, "enabling UMAC");
 
 	/* This register opens the gate for the UMAC despite its name */
 	REG_WR(sc, NIG_REG_EGRESS_EMAC0_PORT + params->port * 4, 1);
 
 	val = UMAC_COMMAND_CONFIG_REG_PROMIS_EN |
-	    UMAC_COMMAND_CONFIG_REG_PAD_EN |
-	    UMAC_COMMAND_CONFIG_REG_SW_RESET |
-	    UMAC_COMMAND_CONFIG_REG_NO_LGTH_CHECK;
+		UMAC_COMMAND_CONFIG_REG_PAD_EN |
+		UMAC_COMMAND_CONFIG_REG_SW_RESET |
+		UMAC_COMMAND_CONFIG_REG_NO_LGTH_CHECK;
 	switch (vars->line_speed) {
 	case ELINK_SPEED_10:
 		val |= (0 << 2);
@@ -1350,8 +2368,8 @@ static void elink_umac_enable(struct elink_params *params,
 		val |= (3 << 2);
 		break;
 	default:
-		PMD_DRV_LOG(DEBUG, "Invalid speed for UMAC %d",
-			    vars->line_speed);
+		ELINK_DEBUG_P1(sc, "Invalid speed for UMAC %d",
+			       vars->line_speed);
 		break;
 	}
 	if (!(vars->flow_ctrl & ELINK_FLOW_CTRL_TX))
@@ -1368,7 +2386,7 @@ static void elink_umac_enable(struct elink_params *params,
 
 	/* Configure UMAC for EEE */
 	if (vars->eee_status & SHMEM_EEE_ADV_STATUS_MASK) {
-		PMD_DRV_LOG(DEBUG, "configured UMAC for EEE");
+		ELINK_DEBUG_P0(sc, "configured UMAC for EEE");
 		REG_WR(sc, umac_base + UMAC_REG_UMAC_EEE_CTRL,
 		       UMAC_UMAC_EEE_CTRL_REG_EEE_EN);
 		REG_WR(sc, umac_base + UMAC_REG_EEE_WAKE_TIMER, 0x11);
@@ -1380,13 +2398,16 @@ static void elink_umac_enable(struct elink_params *params,
 	REG_WR(sc, umac_base + UMAC_REG_MAC_ADDR0,
 	       ((params->mac_addr[2] << 24) |
 		(params->mac_addr[3] << 16) |
-		(params->mac_addr[4] << 8) | (params->mac_addr[5])));
+		(params->mac_addr[4] << 8) |
+		(params->mac_addr[5])));
 	REG_WR(sc, umac_base + UMAC_REG_MAC_ADDR1,
-	       ((params->mac_addr[0] << 8) | (params->mac_addr[1])));
+	       ((params->mac_addr[0] << 8) |
+		(params->mac_addr[1])));
 
 	/* Enable RX and TX */
 	val &= ~UMAC_COMMAND_CONFIG_REG_PAD_EN;
-	val |= UMAC_COMMAND_CONFIG_REG_TX_ENA | UMAC_COMMAND_CONFIG_REG_RX_ENA;
+	val |= UMAC_COMMAND_CONFIG_REG_TX_ENA |
+		UMAC_COMMAND_CONFIG_REG_RX_ENA;
 	REG_WR(sc, umac_base + UMAC_REG_COMMAND_CONFIG, val);
 	DELAY(50);
 
@@ -1426,7 +2447,8 @@ static void elink_xmac_init(struct elink_params *params, uint32_t max_speed)
 	    is_port4mode &&
 	    (REG_RD(sc, MISC_REG_RESET_REG_2) &
 	     MISC_REGISTERS_RESET_REG_2_XMAC)) {
-		PMD_DRV_LOG(DEBUG, "XMAC already out of reset in 4-port mode");
+		ELINK_DEBUG_P0(sc,
+		   "XMAC already out of reset in 4-port mode");
 		return;
 	}
 
@@ -1438,7 +2460,7 @@ static void elink_xmac_init(struct elink_params *params, uint32_t max_speed)
 	REG_WR(sc, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_SET,
 	       MISC_REGISTERS_RESET_REG_2_XMAC);
 	if (is_port4mode) {
-		PMD_DRV_LOG(DEBUG, "Init XMAC to 2 ports x 10G per path");
+		ELINK_DEBUG_P0(sc, "Init XMAC to 2 ports x 10G per path");
 
 		/* Set the number of ports on the system side to up to 2 */
 		REG_WR(sc, MISC_REG_XMAC_CORE_PORT_MODE, 1);
@@ -1449,13 +2471,13 @@ static void elink_xmac_init(struct elink_params *params, uint32_t max_speed)
 		/* Set the number of ports on the system side to 1 */
 		REG_WR(sc, MISC_REG_XMAC_CORE_PORT_MODE, 0);
 		if (max_speed == ELINK_SPEED_10000) {
-			PMD_DRV_LOG(DEBUG,
-				    "Init XMAC to 10G x 1 port per path");
+			ELINK_DEBUG_P0(sc,
+			   "Init XMAC to 10G x 1 port per path");
 			/* Set the number of ports on the Warp Core to 10G */
 			REG_WR(sc, MISC_REG_XMAC_PHY_PORT_MODE, 3);
 		} else {
-			PMD_DRV_LOG(DEBUG,
-				    "Init XMAC to 20G x 2 ports per path");
+			ELINK_DEBUG_P0(sc,
+			   "Init XMAC to 20G x 2 ports per path");
 			/* Set the number of ports on the Warp Core to 20G */
 			REG_WR(sc, MISC_REG_XMAC_PHY_PORT_MODE, 1);
 		}
@@ -1477,7 +2499,8 @@ static void elink_set_xmac_rxtx(struct elink_params *params, uint8_t en)
 	uint32_t pfc_ctrl, xmac_base = (port) ? GRCBASE_XMAC1 : GRCBASE_XMAC0;
 	uint32_t val;
 
-	if (REG_RD(sc, MISC_REG_RESET_REG_2) & MISC_REGISTERS_RESET_REG_2_XMAC) {
+	if (REG_RD(sc, MISC_REG_RESET_REG_2) &
+	    MISC_REGISTERS_RESET_REG_2_XMAC) {
 		/* Send an indication to change the state in the NIG back to XON
 		 * Clearing this bit enables the next set of this bit to get
 		 * rising edge
@@ -1487,7 +2510,7 @@ static void elink_set_xmac_rxtx(struct elink_params *params, uint8_t en)
 		       (pfc_ctrl & ~(1 << 1)));
 		REG_WR(sc, xmac_base + XMAC_REG_PFC_CTRL_HI,
 		       (pfc_ctrl | (1 << 1)));
-		PMD_DRV_LOG(DEBUG, "Disable XMAC on port %x", port);
+		ELINK_DEBUG_P1(sc, "Disable XMAC on port %x", port);
 		val = REG_RD(sc, xmac_base + XMAC_REG_CTRL);
 		if (en)
 			val |= (XMAC_CTRL_REG_TX_EN | XMAC_CTRL_REG_RX_EN);
@@ -1498,11 +2521,11 @@ static void elink_set_xmac_rxtx(struct elink_params *params, uint8_t en)
 }
 
 static elink_status_t elink_xmac_enable(struct elink_params *params,
-					struct elink_vars *vars, uint8_t lb)
+			     struct elink_vars *vars, uint8_t lb)
 {
 	uint32_t val, xmac_base;
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "enabling XMAC");
+	ELINK_DEBUG_P0(sc, "enabling XMAC");
 
 	xmac_base = (params->port) ? GRCBASE_XMAC1 : GRCBASE_XMAC0;
 
@@ -1536,10 +2559,10 @@ static elink_status_t elink_xmac_enable(struct elink_params *params,
 	REG_WR(sc, xmac_base + XMAC_REG_TX_CTRL, 0xC800);
 
 	/* update PFC */
-	elink_update_pfc_xmac(params, vars);
+	elink_update_pfc_xmac(params, vars, 0);
 
 	if (vars->eee_status & SHMEM_EEE_ADV_STATUS_MASK) {
-		PMD_DRV_LOG(DEBUG, "Setting XMAC for EEE");
+		ELINK_DEBUG_P0(sc, "Setting XMAC for EEE");
 		REG_WR(sc, xmac_base + XMAC_REG_EEE_TIMERS_HI, 0x1380008);
 		REG_WR(sc, xmac_base + XMAC_REG_EEE_CTRL, 0x1);
 	} else {
@@ -1568,14 +2591,14 @@ static elink_status_t elink_xmac_enable(struct elink_params *params,
 }
 
 static elink_status_t elink_emac_enable(struct elink_params *params,
-					struct elink_vars *vars, uint8_t lb)
+			     struct elink_vars *vars, uint8_t lb)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t port = params->port;
 	uint32_t emac_base = port ? GRCBASE_EMAC1 : GRCBASE_EMAC0;
 	uint32_t val;
 
-	PMD_DRV_LOG(DEBUG, "enabling EMAC");
+	ELINK_DEBUG_P0(sc, "enabling EMAC");
 
 	/* Disable BMAC */
 	REG_WR(sc, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_CLEAR,
@@ -1584,19 +2607,39 @@ static elink_status_t elink_emac_enable(struct elink_params *params,
 	/* enable emac and not bmac */
 	REG_WR(sc, NIG_REG_EGRESS_EMAC0_PORT + port * 4, 1);
 
+#ifdef ELINK_INCLUDE_EMUL
+	/* for paladium */
+	if (CHIP_REV_IS_EMUL(sc)) {
+		/* Use lane 1 (of lanes 0-3) */
+		REG_WR(sc, NIG_REG_XGXS_LANE_SEL_P0 + port * 4, 1);
+		REG_WR(sc, NIG_REG_XGXS_SERDES0_MODE_SEL + port * 4, 1);
+	}
+	/* for fpga */
+	else
+#endif
+#ifdef ELINK_INCLUDE_FPGA
+	if (CHIP_REV_IS_FPGA(sc)) {
+		/* Use lane 1 (of lanes 0-3) */
+		ELINK_DEBUG_P0(sc, "elink_emac_enable: Setting FPGA");
+
+		REG_WR(sc, NIG_REG_XGXS_LANE_SEL_P0 + port * 4, 1);
+		REG_WR(sc, NIG_REG_XGXS_SERDES0_MODE_SEL + port * 4, 0);
+	} else
+#endif
+	/* ASIC */
 	if (vars->phy_flags & PHY_XGXS_FLAG) {
 		uint32_t ser_lane = ((params->lane_config &
-				      PORT_HW_CFG_LANE_SWAP_CFG_MASTER_MASK) >>
-				     PORT_HW_CFG_LANE_SWAP_CFG_MASTER_SHIFT);
+				 PORT_HW_CFG_LANE_SWAP_CFG_MASTER_MASK) >>
+				PORT_HW_CFG_LANE_SWAP_CFG_MASTER_SHIFT);
 
-		PMD_DRV_LOG(DEBUG, "XGXS");
+		ELINK_DEBUG_P0(sc, "XGXS");
 		/* select the master lanes (out of 0-3) */
 		REG_WR(sc, NIG_REG_XGXS_LANE_SEL_P0 + port * 4, ser_lane);
 		/* select XGXS */
 		REG_WR(sc, NIG_REG_XGXS_SERDES0_MODE_SEL + port * 4, 1);
 
-	} else {		/* SerDes */
-		PMD_DRV_LOG(DEBUG, "SerDes");
+	} else { /* SerDes */
+		ELINK_DEBUG_P0(sc, "SerDes");
 		/* select SerDes */
 		REG_WR(sc, NIG_REG_XGXS_SERDES0_MODE_SEL + port * 4, 0);
 	}
@@ -1606,28 +2649,39 @@ static elink_status_t elink_emac_enable(struct elink_params *params,
 	elink_bits_en(sc, emac_base + EMAC_REG_EMAC_TX_MODE,
 		      EMAC_TX_MODE_RESET);
 
-	/* pause enable/disable */
-	elink_bits_dis(sc, emac_base + EMAC_REG_EMAC_RX_MODE,
-		       EMAC_RX_MODE_FLOW_EN);
-
-	elink_bits_dis(sc, emac_base + EMAC_REG_EMAC_TX_MODE,
-		       (EMAC_TX_MODE_EXT_PAUSE_EN |
-			EMAC_TX_MODE_FLOW_EN));
-	if (!(params->feature_config_flags &
-	      ELINK_FEATURE_CONFIG_PFC_ENABLED)) {
-		if (vars->flow_ctrl & ELINK_FLOW_CTRL_RX)
-			elink_bits_en(sc, emac_base +
-				      EMAC_REG_EMAC_RX_MODE,
-				      EMAC_RX_MODE_FLOW_EN);
+#if defined(ELINK_INCLUDE_EMUL) || defined(ELINK_INCLUDE_FPGA)
+	if (CHIP_REV_IS_SLOW(sc)) {
+		/* config GMII mode */
+		val = REG_RD(sc, emac_base + EMAC_REG_EMAC_MODE);
+		elink_cb_reg_write(sc, emac_base + EMAC_REG_EMAC_MODE,
+				   (val | EMAC_MODE_PORT_GMII));
+	} else { /* ASIC */
+#endif
+		/* pause enable/disable */
+		elink_bits_dis(sc, emac_base + EMAC_REG_EMAC_RX_MODE,
+			       EMAC_RX_MODE_FLOW_EN);
 
-		if (vars->flow_ctrl & ELINK_FLOW_CTRL_TX)
-			elink_bits_en(sc, emac_base +
-				      EMAC_REG_EMAC_TX_MODE,
-				      (EMAC_TX_MODE_EXT_PAUSE_EN |
-				       EMAC_TX_MODE_FLOW_EN));
-	} else
-		elink_bits_en(sc, emac_base + EMAC_REG_EMAC_TX_MODE,
-			      EMAC_TX_MODE_FLOW_EN);
+		elink_bits_dis(sc,  emac_base + EMAC_REG_EMAC_TX_MODE,
+			       (EMAC_TX_MODE_EXT_PAUSE_EN |
+				EMAC_TX_MODE_FLOW_EN));
+		if (!(params->feature_config_flags &
+		      ELINK_FEATURE_CONFIG_PFC_ENABLED)) {
+			if (vars->flow_ctrl & ELINK_FLOW_CTRL_RX)
+				elink_bits_en(sc, emac_base +
+					      EMAC_REG_EMAC_RX_MODE,
+					      EMAC_RX_MODE_FLOW_EN);
+
+			if (vars->flow_ctrl & ELINK_FLOW_CTRL_TX)
+				elink_bits_en(sc, emac_base +
+					      EMAC_REG_EMAC_TX_MODE,
+					      (EMAC_TX_MODE_EXT_PAUSE_EN |
+					       EMAC_TX_MODE_FLOW_EN));
+		} else
+			elink_bits_en(sc, emac_base + EMAC_REG_EMAC_TX_MODE,
+				      EMAC_TX_MODE_FLOW_EN);
+#if defined(ELINK_INCLUDE_EMUL) || defined(ELINK_INCLUDE_FPGA)
+	}
+#endif
 
 	/* KEEP_VLAN_TAG, promiscuous */
 	val = REG_RD(sc, emac_base + EMAC_REG_EMAC_RX_MODE);
@@ -1642,18 +2696,18 @@ static elink_status_t elink_emac_enable(struct elink_params *params,
 	 */
 	elink_cb_reg_write(sc, emac_base + EMAC_REG_RX_PFC_MODE, 0);
 	if (params->feature_config_flags & ELINK_FEATURE_CONFIG_PFC_ENABLED) {
-		PMD_DRV_LOG(DEBUG, "PFC is enabled");
+		ELINK_DEBUG_P0(sc, "PFC is enabled");
 		/* Enable PFC again */
 		elink_cb_reg_write(sc, emac_base + EMAC_REG_RX_PFC_MODE,
-				   EMAC_REG_RX_PFC_MODE_RX_EN |
-				   EMAC_REG_RX_PFC_MODE_TX_EN |
-				   EMAC_REG_RX_PFC_MODE_PRIORITIES);
+			EMAC_REG_RX_PFC_MODE_RX_EN |
+			EMAC_REG_RX_PFC_MODE_TX_EN |
+			EMAC_REG_RX_PFC_MODE_PRIORITIES);
 
 		elink_cb_reg_write(sc, emac_base + EMAC_REG_RX_PFC_PARAM,
-				   ((0x0101 <<
-				     EMAC_REG_RX_PFC_PARAM_OPCODE_BITSHIFT) |
-				    (0x00ff <<
-				     EMAC_REG_RX_PFC_PARAM_PRIORITY_EN_BITSHIFT)));
+			((0x0101 <<
+			  EMAC_REG_RX_PFC_PARAM_OPCODE_BITSHIFT) |
+			 (0x00ff <<
+			  EMAC_REG_RX_PFC_PARAM_PRIORITY_EN_BITSHIFT)));
 		val |= EMAC_RX_MODE_KEEP_MAC_CONTROL;
 	}
 	elink_cb_reg_write(sc, emac_base + EMAC_REG_EMAC_RX_MODE, val);
@@ -1671,9 +2725,8 @@ static elink_status_t elink_emac_enable(struct elink_params *params,
 
 	/* Enable emac for jumbo packets */
 	elink_cb_reg_write(sc, emac_base + EMAC_REG_EMAC_RX_MTU_SIZE,
-			   (EMAC_RX_MTU_SIZE_JUMBO_ENA |
-			    (ELINK_ETH_MAX_JUMBO_PACKET_SIZE +
-			     ELINK_ETH_OVREHEAD)));
+		(EMAC_RX_MTU_SIZE_JUMBO_ENA |
+		 (ELINK_ETH_MAX_JUMBO_PACKET_SIZE + ELINK_ETH_OVREHEAD)));
 
 	/* Strip CRC */
 	REG_WR(sc, NIG_REG_NIG_INGRESS_EMAC0_NO_CRC + port * 4, 0x1);
@@ -1687,13 +2740,23 @@ static elink_status_t elink_emac_enable(struct elink_params *params,
 	REG_WR(sc, NIG_REG_EMAC0_IN_EN + port * 4, 0x1);
 	val = 0;
 	if ((params->feature_config_flags &
-	     ELINK_FEATURE_CONFIG_PFC_ENABLED) ||
+	      ELINK_FEATURE_CONFIG_PFC_ENABLED) ||
 	    (vars->flow_ctrl & ELINK_FLOW_CTRL_TX))
 		val = 1;
 
 	REG_WR(sc, NIG_REG_EMAC0_PAUSE_OUT_EN + port * 4, val);
 	REG_WR(sc, NIG_REG_EGRESS_EMAC0_OUT_EN + port * 4, 0x1);
 
+#ifdef ELINK_INCLUDE_EMUL
+	if (CHIP_REV_IS_EMUL(sc)) {
+		/* Take the BigMac out of reset */
+		REG_WR(sc, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_SET,
+		       (MISC_REGISTERS_RESET_REG_2_RST_BMAC0 << port));
+
+		/* Enable access for bmac registers */
+		REG_WR(sc, NIG_REG_BMAC0_REGS_OUT_EN + port * 4, 0x1);
+	} else
+#endif
 	REG_WR(sc, NIG_REG_BMAC0_REGS_OUT_EN + port * 4, 0x0);
 
 	vars->mac_type = ELINK_MAC_TYPE_EMAC;
@@ -1705,13 +2768,13 @@ static void elink_update_pfc_bmac1(struct elink_params *params,
 {
 	uint32_t wb_data[2];
 	struct bnx2x_softc *sc = params->sc;
-	uint32_t bmac_addr = params->port ? NIG_REG_INGRESS_BMAC1_MEM :
-	    NIG_REG_INGRESS_BMAC0_MEM;
+	uint32_t bmac_addr =  params->port ? NIG_REG_INGRESS_BMAC1_MEM :
+		NIG_REG_INGRESS_BMAC0_MEM;
 
 	uint32_t val = 0x14;
 	if ((!(params->feature_config_flags &
-	       ELINK_FEATURE_CONFIG_PFC_ENABLED)) &&
-	    (vars->flow_ctrl & ELINK_FLOW_CTRL_RX))
+	      ELINK_FEATURE_CONFIG_PFC_ENABLED)) &&
+		(vars->flow_ctrl & ELINK_FLOW_CTRL_RX))
 		/* Enable BigMAC to react on received Pause packets */
 		val |= (1 << 5);
 	wb_data[0] = val;
@@ -1722,7 +2785,7 @@ static void elink_update_pfc_bmac1(struct elink_params *params,
 	val = 0xc0;
 	if (!(params->feature_config_flags &
 	      ELINK_FEATURE_CONFIG_PFC_ENABLED) &&
-	    (vars->flow_ctrl & ELINK_FLOW_CTRL_TX))
+		(vars->flow_ctrl & ELINK_FLOW_CTRL_TX))
 		val |= 0x800000;
 	wb_data[0] = val;
 	wb_data[1] = 0;
@@ -1730,7 +2793,8 @@ static void elink_update_pfc_bmac1(struct elink_params *params,
 }
 
 static void elink_update_pfc_bmac2(struct elink_params *params,
-				   struct elink_vars *vars, uint8_t is_lb)
+				   struct elink_vars *vars,
+				   uint8_t is_lb)
 {
 	/* Set rx control: Strip CRC and enable BigMAC to relay
 	 * control packets to the system as well
@@ -1738,12 +2802,12 @@ static void elink_update_pfc_bmac2(struct elink_params *params,
 	uint32_t wb_data[2];
 	struct bnx2x_softc *sc = params->sc;
 	uint32_t bmac_addr = params->port ? NIG_REG_INGRESS_BMAC1_MEM :
-	    NIG_REG_INGRESS_BMAC0_MEM;
+		NIG_REG_INGRESS_BMAC0_MEM;
 	uint32_t val = 0x14;
 
 	if ((!(params->feature_config_flags &
-	       ELINK_FEATURE_CONFIG_PFC_ENABLED)) &&
-	    (vars->flow_ctrl & ELINK_FLOW_CTRL_RX))
+	      ELINK_FEATURE_CONFIG_PFC_ENABLED)) &&
+		(vars->flow_ctrl & ELINK_FLOW_CTRL_RX))
 		/* Enable BigMAC to react on received Pause packets */
 		val |= (1 << 5);
 	wb_data[0] = val;
@@ -1754,7 +2818,7 @@ static void elink_update_pfc_bmac2(struct elink_params *params,
 	/* Tx control */
 	val = 0xc0;
 	if (!(params->feature_config_flags &
-	      ELINK_FEATURE_CONFIG_PFC_ENABLED) &&
+				ELINK_FEATURE_CONFIG_PFC_ENABLED) &&
 	    (vars->flow_ctrl & ELINK_FLOW_CTRL_TX))
 		val |= 0x800000;
 	wb_data[0] = val;
@@ -1762,21 +2826,21 @@ static void elink_update_pfc_bmac2(struct elink_params *params,
 	REG_WR_DMAE(sc, bmac_addr + BIGMAC2_REGISTER_TX_CONTROL, wb_data, 2);
 
 	if (params->feature_config_flags & ELINK_FEATURE_CONFIG_PFC_ENABLED) {
-		PMD_DRV_LOG(DEBUG, "PFC is enabled");
+		ELINK_DEBUG_P0(sc, "PFC is enabled");
 		/* Enable PFC RX & TX & STATS and set 8 COS  */
 		wb_data[0] = 0x0;
-		wb_data[0] |= (1 << 0);	/* RX */
-		wb_data[0] |= (1 << 1);	/* TX */
-		wb_data[0] |= (1 << 2);	/* Force initial Xon */
-		wb_data[0] |= (1 << 3);	/* 8 cos */
-		wb_data[0] |= (1 << 5);	/* STATS */
+		wb_data[0] |= (1 << 0);  /* RX */
+		wb_data[0] |= (1 << 1);  /* TX */
+		wb_data[0] |= (1 << 2);  /* Force initial Xon */
+		wb_data[0] |= (1 << 3);  /* 8 cos */
+		wb_data[0] |= (1 << 5);  /* STATS */
 		wb_data[1] = 0;
 		REG_WR_DMAE(sc, bmac_addr + BIGMAC2_REGISTER_PFC_CONTROL,
 			    wb_data, 2);
 		/* Clear the force Xon */
 		wb_data[0] &= ~(1 << 2);
 	} else {
-		PMD_DRV_LOG(DEBUG, "PFC is disabled");
+		ELINK_DEBUG_P0(sc, "PFC is disabled");
 		/* Disable PFC RX & TX & STATS and set 8 COS */
 		wb_data[0] = 0x8;
 		wb_data[1] = 0;
@@ -1791,7 +2855,7 @@ static void elink_update_pfc_bmac2(struct elink_params *params,
 	 */
 	val = 0x8000;
 	if (params->feature_config_flags & ELINK_FEATURE_CONFIG_PFC_ENABLED)
-		val |= (1 << 16);	/* enable automatic re-send */
+		val |= (1 << 16); /* enable automatic re-send */
 
 	wb_data[0] = val;
 	wb_data[1] = 0;
@@ -1799,10 +2863,10 @@ static void elink_update_pfc_bmac2(struct elink_params *params,
 		    wb_data, 2);
 
 	/* mac control */
-	val = 0x3;		/* Enable RX and TX */
+	val = 0x3; /* Enable RX and TX */
 	if (is_lb) {
-		val |= 0x4;	/* Local loopback */
-		PMD_DRV_LOG(DEBUG, "enable bmac loopback");
+		val |= 0x4; /* Local loopback */
+		ELINK_DEBUG_P0(sc, "enable bmac loopback");
 	}
 	/* When PFC enabled, Pass pause frames towards the NIG. */
 	if (params->feature_config_flags & ELINK_FEATURE_CONFIG_PFC_ENABLED)
@@ -1814,47 +2878,46 @@ static void elink_update_pfc_bmac2(struct elink_params *params,
 }
 
 /******************************************************************************
-* Description:
-*  This function is needed because NIG ARB_CREDIT_WEIGHT_X are
-*  not continues and ARB_CREDIT_WEIGHT_0 + offset is suitable.
-******************************************************************************/
+ * Description:
+ *  This function is needed because NIG ARB_CREDIT_WEIGHT_X are
+ *  not continues and ARB_CREDIT_WEIGHT_0 + offset is suitable.
+ ******************************************************************************/
 static elink_status_t elink_pfc_nig_rx_priority_mask(struct bnx2x_softc *sc,
-						     uint8_t cos_entry,
-						     uint32_t priority_mask,
-						     uint8_t port)
+					   uint8_t cos_entry,
+					   uint32_t priority_mask, uint8_t port)
 {
 	uint32_t nig_reg_rx_priority_mask_add = 0;
 
 	switch (cos_entry) {
 	case 0:
-		nig_reg_rx_priority_mask_add = (port) ?
-		    NIG_REG_P1_RX_COS0_PRIORITY_MASK :
-		    NIG_REG_P0_RX_COS0_PRIORITY_MASK;
+	     nig_reg_rx_priority_mask_add = (port) ?
+		 NIG_REG_P1_RX_COS0_PRIORITY_MASK :
+		 NIG_REG_P0_RX_COS0_PRIORITY_MASK;
 		break;
 	case 1:
-		nig_reg_rx_priority_mask_add = (port) ?
-		    NIG_REG_P1_RX_COS1_PRIORITY_MASK :
-		    NIG_REG_P0_RX_COS1_PRIORITY_MASK;
+	    nig_reg_rx_priority_mask_add = (port) ?
+		NIG_REG_P1_RX_COS1_PRIORITY_MASK :
+		NIG_REG_P0_RX_COS1_PRIORITY_MASK;
 		break;
 	case 2:
-		nig_reg_rx_priority_mask_add = (port) ?
-		    NIG_REG_P1_RX_COS2_PRIORITY_MASK :
-		    NIG_REG_P0_RX_COS2_PRIORITY_MASK;
+	    nig_reg_rx_priority_mask_add = (port) ?
+		NIG_REG_P1_RX_COS2_PRIORITY_MASK :
+		NIG_REG_P0_RX_COS2_PRIORITY_MASK;
 		break;
 	case 3:
 		if (port)
-			return ELINK_STATUS_ERROR;
-		nig_reg_rx_priority_mask_add = NIG_REG_P0_RX_COS3_PRIORITY_MASK;
+		return ELINK_STATUS_ERROR;
+	    nig_reg_rx_priority_mask_add = NIG_REG_P0_RX_COS3_PRIORITY_MASK;
 		break;
 	case 4:
 		if (port)
-			return ELINK_STATUS_ERROR;
-		nig_reg_rx_priority_mask_add = NIG_REG_P0_RX_COS4_PRIORITY_MASK;
+		return ELINK_STATUS_ERROR;
+	    nig_reg_rx_priority_mask_add = NIG_REG_P0_RX_COS4_PRIORITY_MASK;
 		break;
 	case 5:
 		if (port)
-			return ELINK_STATUS_ERROR;
-		nig_reg_rx_priority_mask_add = NIG_REG_P0_RX_COS5_PRIORITY_MASK;
+		return ELINK_STATUS_ERROR;
+	    nig_reg_rx_priority_mask_add = NIG_REG_P0_RX_COS5_PRIORITY_MASK;
 		break;
 	}
 
@@ -1862,7 +2925,6 @@ static elink_status_t elink_pfc_nig_rx_priority_mask(struct bnx2x_softc *sc,
 
 	return ELINK_STATUS_OK;
 }
-
 static void elink_update_mng(struct elink_params *params, uint32_t link_status)
 {
 	struct bnx2x_softc *sc = params->sc;
@@ -1872,31 +2934,20 @@ static void elink_update_mng(struct elink_params *params, uint32_t link_status)
 			port_mb[params->port].link_status), link_status);
 }
 
-static void elink_update_link_attr(struct elink_params *params,
-				   uint32_t link_attr)
-{
-	struct bnx2x_softc *sc = params->sc;
-
-	if (SHMEM2_HAS(sc, link_attr_sync))
-		REG_WR(sc, params->shmem2_base +
-		       offsetof(struct shmem2_region,
-				link_attr_sync[params->port]), link_attr);
-}
-
 static void elink_update_pfc_nig(struct elink_params *params,
-				 struct elink_nig_brb_pfc_port_params
-				 *nig_params)
+		__rte_unused struct elink_vars *vars,
+		struct elink_nig_brb_pfc_port_params *nig_params)
 {
-	uint32_t xcm_mask = 0, ppp_enable = 0, pause_enable = 0, llfc_out_en =
-	    0;
+	uint32_t xcm_mask = 0, ppp_enable = 0, pause_enable = 0;
+	uint32_t llfc_out_en = 0;
 	uint32_t llfc_enable = 0, xcm_out_en = 0, hwpfc_enable = 0;
 	uint32_t pkt_priority_to_cos = 0;
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t port = params->port;
 
 	int set_pfc = params->feature_config_flags &
-	    ELINK_FEATURE_CONFIG_PFC_ENABLED;
-	PMD_DRV_LOG(DEBUG, "updating pfc nig parameters");
+		ELINK_FEATURE_CONFIG_PFC_ENABLED;
+	ELINK_DEBUG_P0(sc, "updating pfc nig parameters");
 
 	/* When NIG_LLH0_XCM_MASK_REG_LLHX_XCM_MASK_BCN bit is set
 	 * MAC control frames (that are not pause packets)
@@ -1916,19 +2967,19 @@ static void elink_update_pfc_nig(struct elink_params *params,
 		else
 			ppp_enable = 1;
 		xcm_mask &= ~(port ? NIG_LLH1_XCM_MASK_REG_LLH1_XCM_MASK_BCN :
-			      NIG_LLH0_XCM_MASK_REG_LLH0_XCM_MASK_BCN);
+				     NIG_LLH0_XCM_MASK_REG_LLH0_XCM_MASK_BCN);
 		xcm_out_en = 0;
 		hwpfc_enable = 1;
-	} else {
+	} else  {
 		if (nig_params) {
 			llfc_out_en = nig_params->llfc_out_en;
 			llfc_enable = nig_params->llfc_enable;
 			pause_enable = nig_params->pause_enable;
-		} else		/* Default non PFC mode - PAUSE */
+		} else  /* Default non PFC mode - PAUSE */
 			pause_enable = 1;
 
 		xcm_mask |= (port ? NIG_LLH1_XCM_MASK_REG_LLH1_XCM_MASK_BCN :
-			     NIG_LLH0_XCM_MASK_REG_LLH0_XCM_MASK_BCN);
+			NIG_LLH0_XCM_MASK_REG_LLH0_XCM_MASK_BCN);
 		xcm_out_en = 1;
 	}
 
@@ -1965,9 +3016,7 @@ static void elink_update_pfc_nig(struct elink_params *params,
 
 		for (i = 0; i < nig_params->num_of_rx_cos_priority_mask; i++)
 			elink_pfc_nig_rx_priority_mask(sc, i,
-						       nig_params->
-						       rx_cos_priority_mask[i],
-						       port);
+		nig_params->rx_cos_priority_mask[i], port);
 
 		REG_WR(sc, port ? NIG_REG_LLFC_HIGH_PRIORITY_CLASSES_1 :
 		       NIG_REG_LLFC_HIGH_PRIORITY_CLASSES_0,
@@ -1978,13 +3027,13 @@ static void elink_update_pfc_nig(struct elink_params *params,
 		       nig_params->llfc_low_priority_classes);
 	}
 	REG_WR(sc, port ? NIG_REG_P1_PKT_PRIORITY_TO_COS :
-	       NIG_REG_P0_PKT_PRIORITY_TO_COS, pkt_priority_to_cos);
+	       NIG_REG_P0_PKT_PRIORITY_TO_COS,
+	       pkt_priority_to_cos);
 }
 
 elink_status_t elink_update_pfc(struct elink_params *params,
-				struct elink_vars *vars,
-				struct elink_nig_brb_pfc_port_params
-				*pfc_params)
+		      struct elink_vars *vars,
+		      struct elink_nig_brb_pfc_port_params *pfc_params)
 {
 	/* The PFC and pause are orthogonal to one another, meaning when
 	 * PFC is enabled, the pause are disabled, and when PFC is
@@ -1992,7 +3041,6 @@ elink_status_t elink_update_pfc(struct elink_params *params,
 	 */
 	uint32_t val;
 	struct bnx2x_softc *sc = params->sc;
-	elink_status_t elink_status = ELINK_STATUS_OK;
 	uint8_t bmac_loopback = (params->loopback_mode == ELINK_LOOPBACK_BMAC);
 
 	if (params->feature_config_flags & ELINK_FEATURE_CONFIG_PFC_ENABLED)
@@ -2003,24 +3051,24 @@ elink_status_t elink_update_pfc(struct elink_params *params,
 	elink_update_mng(params, vars->link_status);
 
 	/* Update NIG params */
-	elink_update_pfc_nig(params, pfc_params);
+	elink_update_pfc_nig(params, vars, pfc_params);
 
 	if (!vars->link_up)
-		return elink_status;
+		return ELINK_STATUS_OK;
 
-	PMD_DRV_LOG(DEBUG, "About to update PFC in BMAC");
+	ELINK_DEBUG_P0(sc, "About to update PFC in BMAC");
 
 	if (CHIP_IS_E3(sc)) {
 		if (vars->mac_type == ELINK_MAC_TYPE_XMAC)
-			elink_update_pfc_xmac(params, vars);
+			elink_update_pfc_xmac(params, vars, 0);
 	} else {
 		val = REG_RD(sc, MISC_REG_RESET_REG_2);
 		if ((val &
 		     (MISC_REGISTERS_RESET_REG_2_RST_BMAC0 << params->port))
 		    == 0) {
-			PMD_DRV_LOG(DEBUG, "About to update PFC in EMAC");
+			ELINK_DEBUG_P0(sc, "About to update PFC in EMAC");
 			elink_emac_enable(params, vars, 0);
-			return elink_status;
+			return ELINK_STATUS_OK;
 		}
 		if (CHIP_IS_E2(sc))
 			elink_update_pfc_bmac2(params, vars, bmac_loopback);
@@ -2034,20 +3082,21 @@ elink_status_t elink_update_pfc(struct elink_params *params,
 			val = 1;
 		REG_WR(sc, NIG_REG_BMAC0_PAUSE_OUT_EN + params->port * 4, val);
 	}
-	return elink_status;
+	return ELINK_STATUS_OK;
 }
 
 static elink_status_t elink_bmac1_enable(struct elink_params *params,
-					 struct elink_vars *vars, uint8_t is_lb)
+			      struct elink_vars *vars,
+			      uint8_t is_lb)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t port = params->port;
 	uint32_t bmac_addr = port ? NIG_REG_INGRESS_BMAC1_MEM :
-	    NIG_REG_INGRESS_BMAC0_MEM;
+			       NIG_REG_INGRESS_BMAC0_MEM;
 	uint32_t wb_data[2];
 	uint32_t val;
 
-	PMD_DRV_LOG(DEBUG, "Enabling BigMAC1");
+	ELINK_DEBUG_P0(sc, "Enabling BigMAC1");
 
 	/* XGXS control */
 	wb_data[0] = 0x3c;
@@ -2057,16 +3106,18 @@ static elink_status_t elink_bmac1_enable(struct elink_params *params,
 
 	/* TX MAC SA */
 	wb_data[0] = ((params->mac_addr[2] << 24) |
-		      (params->mac_addr[3] << 16) |
-		      (params->mac_addr[4] << 8) | params->mac_addr[5]);
-	wb_data[1] = ((params->mac_addr[0] << 8) | params->mac_addr[1]);
+		       (params->mac_addr[3] << 16) |
+		       (params->mac_addr[4] << 8) |
+			params->mac_addr[5]);
+	wb_data[1] = ((params->mac_addr[0] << 8) |
+			params->mac_addr[1]);
 	REG_WR_DMAE(sc, bmac_addr + BIGMAC_REGISTER_TX_SOURCE_ADDR, wb_data, 2);
 
 	/* MAC control */
 	val = 0x3;
 	if (is_lb) {
 		val |= 0x4;
-		PMD_DRV_LOG(DEBUG, "enable bmac loopback");
+		ELINK_DEBUG_P0(sc,  "enable bmac loopback");
 	}
 	wb_data[0] = val;
 	wb_data[1] = 0;
@@ -2094,20 +3145,30 @@ static elink_status_t elink_bmac1_enable(struct elink_params *params,
 	wb_data[1] = 0;
 	REG_WR_DMAE(sc, bmac_addr + BIGMAC_REGISTER_RX_LLFC_MSG_FLDS,
 		    wb_data, 2);
+#ifdef ELINK_INCLUDE_EMUL
+	/* Fix for emulation */
+	if (CHIP_REV_IS_EMUL(sc)) {
+		wb_data[0] = 0xf000;
+		wb_data[1] = 0;
+		REG_WR_DMAE(sc,	bmac_addr + BIGMAC_REGISTER_TX_PAUSE_THRESHOLD,
+			    wb_data, 2);
+	}
+#endif
 
 	return ELINK_STATUS_OK;
 }
 
 static elink_status_t elink_bmac2_enable(struct elink_params *params,
-					 struct elink_vars *vars, uint8_t is_lb)
+			      struct elink_vars *vars,
+			      uint8_t is_lb)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t port = params->port;
 	uint32_t bmac_addr = port ? NIG_REG_INGRESS_BMAC1_MEM :
-	    NIG_REG_INGRESS_BMAC0_MEM;
+			       NIG_REG_INGRESS_BMAC0_MEM;
 	uint32_t wb_data[2];
 
-	PMD_DRV_LOG(DEBUG, "Enabling BigMAC2");
+	ELINK_DEBUG_P0(sc, "Enabling BigMAC2");
 
 	wb_data[0] = 0;
 	wb_data[1] = 0;
@@ -2124,9 +3185,11 @@ static elink_status_t elink_bmac2_enable(struct elink_params *params,
 
 	/* TX MAC SA */
 	wb_data[0] = ((params->mac_addr[2] << 24) |
-		      (params->mac_addr[3] << 16) |
-		      (params->mac_addr[4] << 8) | params->mac_addr[5]);
-	wb_data[1] = ((params->mac_addr[0] << 8) | params->mac_addr[1]);
+		       (params->mac_addr[3] << 16) |
+		       (params->mac_addr[4] << 8) |
+			params->mac_addr[5]);
+	wb_data[1] = ((params->mac_addr[0] << 8) |
+			params->mac_addr[1]);
 	REG_WR_DMAE(sc, bmac_addr + BIGMAC2_REGISTER_TX_SOURCE_ADDR,
 		    wb_data, 2);
 
@@ -2161,8 +3224,8 @@ static elink_status_t elink_bmac2_enable(struct elink_params *params,
 }
 
 static elink_status_t elink_bmac_enable(struct elink_params *params,
-					struct elink_vars *vars,
-					uint8_t is_lb, uint8_t reset_bmac)
+			     struct elink_vars *vars,
+			     uint8_t is_lb, uint8_t reset_bmac)
 {
 	elink_status_t rc = ELINK_STATUS_OK;
 	uint8_t port = params->port;
@@ -2181,7 +3244,7 @@ static elink_status_t elink_bmac_enable(struct elink_params *params,
 	/* Enable access for bmac registers */
 	REG_WR(sc, NIG_REG_BMAC0_REGS_OUT_EN + port * 4, 0x1);
 
-	/* Enable BMAC according to BMAC type */
+	/* Enable BMAC according to BMAC type*/
 	if (CHIP_IS_E2(sc))
 		rc = elink_bmac2_enable(params, vars, is_lb);
 	else
@@ -2191,7 +3254,7 @@ static elink_status_t elink_bmac_enable(struct elink_params *params,
 	REG_WR(sc, NIG_REG_EGRESS_EMAC0_PORT + port * 4, 0x0);
 	val = 0;
 	if ((params->feature_config_flags &
-	     ELINK_FEATURE_CONFIG_PFC_ENABLED) ||
+	      ELINK_FEATURE_CONFIG_PFC_ENABLED) ||
 	    (vars->flow_ctrl & ELINK_FLOW_CTRL_TX))
 		val = 1;
 	REG_WR(sc, NIG_REG_BMAC0_PAUSE_OUT_EN + port * 4, val);
@@ -2205,13 +3268,15 @@ static elink_status_t elink_bmac_enable(struct elink_params *params,
 	return rc;
 }
 
-static void elink_set_bmac_rx(struct bnx2x_softc *sc, uint8_t port, uint8_t en)
+static void elink_set_bmac_rx(struct bnx2x_softc *sc,
+			      __rte_unused uint32_t chip_id,
+			      uint8_t port, uint8_t en)
 {
 	uint32_t bmac_addr = port ? NIG_REG_INGRESS_BMAC1_MEM :
-	    NIG_REG_INGRESS_BMAC0_MEM;
+			NIG_REG_INGRESS_BMAC0_MEM;
 	uint32_t wb_data[2];
-	uint32_t nig_bmac_enable =
-	    REG_RD(sc, NIG_REG_BMAC0_REGS_OUT_EN + port * 4);
+	uint32_t nig_bmac_enable = REG_RD(sc, NIG_REG_BMAC0_REGS_OUT_EN +
+					  port * 4);
 
 	if (CHIP_IS_E2(sc))
 		bmac_addr += BIGMAC2_REGISTER_BMAC_CONTROL;
@@ -2219,7 +3284,8 @@ static void elink_set_bmac_rx(struct bnx2x_softc *sc, uint8_t port, uint8_t en)
 		bmac_addr += BIGMAC_REGISTER_BMAC_CONTROL;
 	/* Only if the bmac is out of reset */
 	if (REG_RD(sc, MISC_REG_RESET_REG_2) &
-	    (MISC_REGISTERS_RESET_REG_2_RST_BMAC0 << port) && nig_bmac_enable) {
+			(MISC_REGISTERS_RESET_REG_2_RST_BMAC0 << port) &&
+	    nig_bmac_enable) {
 		/* Clear Rx Enable bit in BMAC_CONTROL register */
 		REG_RD_DMAE(sc, bmac_addr, wb_data, 2);
 		if (en)
@@ -2232,7 +3298,8 @@ static void elink_set_bmac_rx(struct bnx2x_softc *sc, uint8_t port, uint8_t en)
 }
 
 static elink_status_t elink_pbf_update(struct elink_params *params,
-				       uint32_t flow_ctrl, uint32_t line_speed)
+			    uint32_t flow_ctrl,
+			    uint32_t line_speed)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t port = params->port;
@@ -2245,7 +3312,7 @@ static elink_status_t elink_pbf_update(struct elink_params *params,
 	/* Wait for init credit */
 	init_crd = REG_RD(sc, PBF_REG_P0_INIT_CRD + port * 4);
 	crd = REG_RD(sc, PBF_REG_P0_CREDIT + port * 8);
-	PMD_DRV_LOG(DEBUG, "init_crd 0x%x  crd 0x%x", init_crd, crd);
+	ELINK_DEBUG_P2(sc, "init_crd 0x%x  crd 0x%x", init_crd, crd);
 
 	while ((init_crd != crd) && count) {
 		DELAY(1000 * 5);
@@ -2254,24 +3321,25 @@ static elink_status_t elink_pbf_update(struct elink_params *params,
 	}
 	crd = REG_RD(sc, PBF_REG_P0_CREDIT + port * 8);
 	if (init_crd != crd) {
-		PMD_DRV_LOG(DEBUG, "BUG! init_crd 0x%x != crd 0x%x",
-			    init_crd, crd);
+		ELINK_DEBUG_P2(sc, "BUG! init_crd 0x%x != crd 0x%x",
+			  init_crd, crd);
 		return ELINK_STATUS_ERROR;
 	}
 
 	if (flow_ctrl & ELINK_FLOW_CTRL_RX ||
 	    line_speed == ELINK_SPEED_10 ||
 	    line_speed == ELINK_SPEED_100 ||
-	    line_speed == ELINK_SPEED_1000 || line_speed == ELINK_SPEED_2500) {
+	    line_speed == ELINK_SPEED_1000 ||
+	    line_speed == ELINK_SPEED_2500) {
 		REG_WR(sc, PBF_REG_P0_PAUSE_ENABLE + port * 4, 1);
 		/* Update threshold */
 		REG_WR(sc, PBF_REG_P0_ARB_THRSH + port * 4, 0);
 		/* Update init credit */
-		init_crd = 778;	/* (800-18-4) */
+		init_crd = 778;		/* (800-18-4) */
 
 	} else {
 		uint32_t thresh = (ELINK_ETH_MAX_JUMBO_PACKET_SIZE +
-				   ELINK_ETH_OVREHEAD) / 16;
+			      ELINK_ETH_OVREHEAD) / 16;
 		REG_WR(sc, PBF_REG_P0_PAUSE_ENABLE + port * 4, 0);
 		/* Update threshold */
 		REG_WR(sc, PBF_REG_P0_ARB_THRSH + port * 4, thresh);
@@ -2281,14 +3349,14 @@ static elink_status_t elink_pbf_update(struct elink_params *params,
 			init_crd = thresh + 553 - 22;
 			break;
 		default:
-			PMD_DRV_LOG(DEBUG, "Invalid line_speed 0x%x",
-				    line_speed);
+			ELINK_DEBUG_P1(sc, "Invalid line_speed 0x%x",
+				  line_speed);
 			return ELINK_STATUS_ERROR;
 		}
 	}
 	REG_WR(sc, PBF_REG_P0_INIT_CRD + port * 4, init_crd);
-	PMD_DRV_LOG(DEBUG, "PBF updated to speed %d credit %d",
-		    line_speed, init_crd);
+	ELINK_DEBUG_P2(sc, "PBF updated to speed %d credit %d",
+		 line_speed, init_crd);
 
 	/* Probe the credit changes */
 	REG_WR(sc, PBF_REG_INIT_P0 + port * 4, 0x1);
@@ -2316,7 +3384,7 @@ static elink_status_t elink_pbf_update(struct elink_params *params,
  * the emac_base for the CL45 read/writes operations
  */
 static uint32_t elink_get_emac_base(struct bnx2x_softc *sc,
-				    uint32_t mdc_mdio_access, uint8_t port)
+			       uint32_t mdc_mdio_access, uint8_t port)
 {
 	uint32_t emac_base = 0;
 	switch (mdc_mdio_access) {
@@ -2364,7 +3432,8 @@ static elink_status_t elink_cl22_write(struct bnx2x_softc *sc,
 
 	/* Address */
 	tmp = ((phy->addr << 21) | (reg << 16) | val |
-	       EMAC_MDIO_COMM_COMMAND_WRITE_22 | EMAC_MDIO_COMM_START_BUSY);
+	       EMAC_MDIO_COMM_COMMAND_WRITE_22 |
+	       EMAC_MDIO_COMM_START_BUSY);
 	REG_WR(sc, phy->mdio_ctrl + EMAC_REG_EMAC_MDIO_COMM, tmp);
 
 	for (i = 0; i < 50; i++) {
@@ -2377,7 +3446,7 @@ static elink_status_t elink_cl22_write(struct bnx2x_softc *sc,
 		}
 	}
 	if (tmp & EMAC_MDIO_COMM_START_BUSY) {
-		PMD_DRV_LOG(DEBUG, "write phy register failed");
+		ELINK_DEBUG_P0(sc, "write phy register failed");
 		rc = ELINK_STATUS_TIMEOUT;
 	}
 	REG_WR(sc, phy->mdio_ctrl + EMAC_REG_EMAC_MDIO_MODE, mode);
@@ -2386,7 +3455,7 @@ static elink_status_t elink_cl22_write(struct bnx2x_softc *sc,
 
 static elink_status_t elink_cl22_read(struct bnx2x_softc *sc,
 				      struct elink_phy *phy,
-				      uint16_t reg, uint16_t * ret_val)
+				      uint16_t reg, uint16_t *ret_val)
 {
 	uint32_t val, mode;
 	uint16_t i;
@@ -2399,7 +3468,8 @@ static elink_status_t elink_cl22_read(struct bnx2x_softc *sc,
 
 	/* Address */
 	val = ((phy->addr << 21) | (reg << 16) |
-	       EMAC_MDIO_COMM_COMMAND_READ_22 | EMAC_MDIO_COMM_START_BUSY);
+	       EMAC_MDIO_COMM_COMMAND_READ_22 |
+	       EMAC_MDIO_COMM_START_BUSY);
 	REG_WR(sc, phy->mdio_ctrl + EMAC_REG_EMAC_MDIO_COMM, val);
 
 	for (i = 0; i < 50; i++) {
@@ -2407,13 +3477,13 @@ static elink_status_t elink_cl22_read(struct bnx2x_softc *sc,
 
 		val = REG_RD(sc, phy->mdio_ctrl + EMAC_REG_EMAC_MDIO_COMM);
 		if (!(val & EMAC_MDIO_COMM_START_BUSY)) {
-			*ret_val = (uint16_t) (val & EMAC_MDIO_COMM_DATA);
+			*ret_val = (uint16_t)(val & EMAC_MDIO_COMM_DATA);
 			DELAY(5);
 			break;
 		}
 	}
 	if (val & EMAC_MDIO_COMM_START_BUSY) {
-		PMD_DRV_LOG(DEBUG, "read phy register failed");
+		ELINK_DEBUG_P0(sc, "read phy register failed");
 
 		*ret_val = 0;
 		rc = ELINK_STATUS_TIMEOUT;
@@ -2426,14 +3496,17 @@ static elink_status_t elink_cl22_read(struct bnx2x_softc *sc,
 /*			CL45 access functions			  */
 /******************************************************************/
 static elink_status_t elink_cl45_read(struct bnx2x_softc *sc,
-				      struct elink_phy *phy, uint8_t devad,
-				      uint16_t reg, uint16_t * ret_val)
+			   struct elink_phy *phy,
+			   uint8_t devad, uint16_t reg, uint16_t *ret_val)
 {
 	uint32_t val;
 	uint16_t i;
 	elink_status_t rc = ELINK_STATUS_OK;
+	uint32_t chip_id;
 	if (phy->flags & ELINK_FLAGS_MDC_MDIO_WA_G) {
-		elink_set_mdio_clk(sc, phy->mdio_ctrl);
+		chip_id = (REG_RD(sc, MISC_REG_CHIP_NUM) << 16) |
+			  ((REG_RD(sc, MISC_REG_CHIP_REV) & 0xf) << 12);
+		elink_set_mdio_clk(sc, chip_id, phy->mdio_ctrl);
 	}
 
 	if (phy->flags & ELINK_FLAGS_MDC_MDIO_WA_B0)
@@ -2441,7 +3514,8 @@ static elink_status_t elink_cl45_read(struct bnx2x_softc *sc,
 			      EMAC_MDIO_STATUS_10MB);
 	/* Address */
 	val = ((phy->addr << 21) | (devad << 16) | reg |
-	       EMAC_MDIO_COMM_COMMAND_ADDRESS | EMAC_MDIO_COMM_START_BUSY);
+	       EMAC_MDIO_COMM_COMMAND_ADDRESS |
+	       EMAC_MDIO_COMM_START_BUSY);
 	REG_WR(sc, phy->mdio_ctrl + EMAC_REG_EMAC_MDIO_COMM, val);
 
 	for (i = 0; i < 50; i++) {
@@ -2454,8 +3528,9 @@ static elink_status_t elink_cl45_read(struct bnx2x_softc *sc,
 		}
 	}
 	if (val & EMAC_MDIO_COMM_START_BUSY) {
-		PMD_DRV_LOG(DEBUG, "read phy register failed");
-		elink_cb_event_log(sc, ELINK_LOG_ID_MDIO_ACCESS_TIMEOUT);	// "MDC/MDIO access timeout"
+		ELINK_DEBUG_P0(sc, "read phy register failed");
+		elink_cb_event_log(sc, ELINK_LOG_ID_MDIO_ACCESS_TIMEOUT);
+		/* "MDC/MDIO access timeout" */
 
 		*ret_val = 0;
 		rc = ELINK_STATUS_TIMEOUT;
@@ -2472,14 +3547,16 @@ static elink_status_t elink_cl45_read(struct bnx2x_softc *sc,
 			val = REG_RD(sc, phy->mdio_ctrl +
 				     EMAC_REG_EMAC_MDIO_COMM);
 			if (!(val & EMAC_MDIO_COMM_START_BUSY)) {
-				*ret_val =
-				    (uint16_t) (val & EMAC_MDIO_COMM_DATA);
+				*ret_val = (uint16_t)
+						(val & EMAC_MDIO_COMM_DATA);
 				break;
 			}
 		}
 		if (val & EMAC_MDIO_COMM_START_BUSY) {
-			PMD_DRV_LOG(DEBUG, "read phy register failed");
-			elink_cb_event_log(sc, ELINK_LOG_ID_MDIO_ACCESS_TIMEOUT);	// "MDC/MDIO access timeout"
+			ELINK_DEBUG_P0(sc, "read phy register failed");
+			elink_cb_event_log(sc,
+					   ELINK_LOG_ID_MDIO_ACCESS_TIMEOUT);
+			/* "MDC/MDIO access timeout" */
 
 			*ret_val = 0;
 			rc = ELINK_STATUS_TIMEOUT;
@@ -2501,14 +3578,17 @@ static elink_status_t elink_cl45_read(struct bnx2x_softc *sc,
 }
 
 static elink_status_t elink_cl45_write(struct bnx2x_softc *sc,
-				       struct elink_phy *phy, uint8_t devad,
-				       uint16_t reg, uint16_t val)
+			    struct elink_phy *phy,
+			    uint8_t devad, uint16_t reg, uint16_t val)
 {
 	uint32_t tmp;
 	uint8_t i;
 	elink_status_t rc = ELINK_STATUS_OK;
+	uint32_t chip_id;
 	if (phy->flags & ELINK_FLAGS_MDC_MDIO_WA_G) {
-		elink_set_mdio_clk(sc, phy->mdio_ctrl);
+		chip_id = (REG_RD(sc, MISC_REG_CHIP_NUM) << 16) |
+			  ((REG_RD(sc, MISC_REG_CHIP_REV) & 0xf) << 12);
+		elink_set_mdio_clk(sc, chip_id, phy->mdio_ctrl);
 	}
 
 	if (phy->flags & ELINK_FLAGS_MDC_MDIO_WA_B0)
@@ -2517,7 +3597,8 @@ static elink_status_t elink_cl45_write(struct bnx2x_softc *sc,
 
 	/* Address */
 	tmp = ((phy->addr << 21) | (devad << 16) | reg |
-	       EMAC_MDIO_COMM_COMMAND_ADDRESS | EMAC_MDIO_COMM_START_BUSY);
+	       EMAC_MDIO_COMM_COMMAND_ADDRESS |
+	       EMAC_MDIO_COMM_START_BUSY);
 	REG_WR(sc, phy->mdio_ctrl + EMAC_REG_EMAC_MDIO_COMM, tmp);
 
 	for (i = 0; i < 50; i++) {
@@ -2530,8 +3611,9 @@ static elink_status_t elink_cl45_write(struct bnx2x_softc *sc,
 		}
 	}
 	if (tmp & EMAC_MDIO_COMM_START_BUSY) {
-		PMD_DRV_LOG(DEBUG, "write phy register failed");
-		elink_cb_event_log(sc, ELINK_LOG_ID_MDIO_ACCESS_TIMEOUT);	// "MDC/MDIO access timeout"
+		ELINK_DEBUG_P0(sc, "write phy register failed");
+		elink_cb_event_log(sc, ELINK_LOG_ID_MDIO_ACCESS_TIMEOUT);
+		/* "MDC/MDIO access timeout" */
 
 		rc = ELINK_STATUS_TIMEOUT;
 	} else {
@@ -2552,8 +3634,10 @@ static elink_status_t elink_cl45_write(struct bnx2x_softc *sc,
 			}
 		}
 		if (tmp & EMAC_MDIO_COMM_START_BUSY) {
-			PMD_DRV_LOG(DEBUG, "write phy register failed");
-			elink_cb_event_log(sc, ELINK_LOG_ID_MDIO_ACCESS_TIMEOUT);	// "MDC/MDIO access timeout"
+			ELINK_DEBUG_P0(sc, "write phy register failed");
+			elink_cb_event_log(sc,
+					   ELINK_LOG_ID_MDIO_ACCESS_TIMEOUT);
+			/* "MDC/MDIO access timeout" */
 
 			rc = ELINK_STATUS_TIMEOUT;
 		}
@@ -2580,14 +3664,14 @@ static uint8_t elink_eee_has_cap(struct elink_params *params)
 	struct bnx2x_softc *sc = params->sc;
 
 	if (REG_RD(sc, params->shmem2_base) <=
-	    offsetof(struct shmem2_region, eee_status[params->port]))
-		 return 0;
+		   offsetof(struct shmem2_region, eee_status[params->port]))
+		return 0;
 
 	return 1;
 }
 
 static elink_status_t elink_eee_nvram_to_time(uint32_t nvram_mode,
-					      uint32_t * idle_timer)
+					      uint32_t *idle_timer)
 {
 	switch (nvram_mode) {
 	case PORT_FEAT_CFG_EEE_POWER_MODE_BALANCED:
@@ -2608,7 +3692,7 @@ static elink_status_t elink_eee_nvram_to_time(uint32_t nvram_mode,
 }
 
 static elink_status_t elink_eee_time_to_nvram(uint32_t idle_timer,
-					      uint32_t * nvram_mode)
+					      uint32_t *nvram_mode)
 {
 	switch (idle_timer) {
 	case ELINK_EEE_MODE_NVRAM_BALANCED_TIME:
@@ -2635,7 +3719,7 @@ static uint32_t elink_eee_calc_timer(struct elink_params *params)
 
 	if (params->eee_mode & ELINK_EEE_MODE_OVERRIDE_NVRAM) {
 		if (params->eee_mode & ELINK_EEE_MODE_OUTPUT_TIME) {
-			/* time value in eee_mode --> used directly */
+			/* time value in eee_mode --> used directly*/
 			eee_idle = params->eee_mode & ELINK_EEE_MODE_TIMER_MASK;
 		} else {
 			/* hsi value in eee_mode --> time */
@@ -2645,12 +3729,11 @@ static uint32_t elink_eee_calc_timer(struct elink_params *params)
 				return 0;
 		}
 	} else {
-		/* hsi values in nvram --> time */
+		/* hsi values in nvram --> time*/
 		eee_mode = ((REG_RD(sc, params->shmem_base +
-				    offsetof(struct shmem_region,
-					     dev_info.port_feature_config
-					     [params->
-					      port].eee_power_mode)) &
+				    offsetof(struct shmem_region, dev_info.
+				    port_feature_config[params->port].
+				    eee_power_mode)) &
 			     PORT_FEAT_CFG_EEE_POWER_MODE_MASK) >>
 			    PORT_FEAT_CFG_EEE_POWER_MODE_SHIFT);
 
@@ -2662,7 +3745,7 @@ static uint32_t elink_eee_calc_timer(struct elink_params *params)
 }
 
 static elink_status_t elink_eee_set_timers(struct elink_params *params,
-					   struct elink_vars *vars)
+				   struct elink_vars *vars)
 {
 	uint32_t eee_idle = 0, eee_mode;
 	struct bnx2x_softc *sc = params->sc;
@@ -2675,7 +3758,7 @@ static elink_status_t elink_eee_set_timers(struct elink_params *params,
 	} else if ((params->eee_mode & ELINK_EEE_MODE_ENABLE_LPI) &&
 		   (params->eee_mode & ELINK_EEE_MODE_OVERRIDE_NVRAM) &&
 		   (params->eee_mode & ELINK_EEE_MODE_OUTPUT_TIME)) {
-		PMD_DRV_LOG(DEBUG, "Error: Tx LPI is enabled with timer 0");
+		ELINK_DEBUG_P0(sc, "Error: Tx LPI is enabled with timer 0");
 		return ELINK_STATUS_ERROR;
 	}
 
@@ -2684,7 +3767,7 @@ static elink_status_t elink_eee_set_timers(struct elink_params *params,
 		/* eee_idle in 1u --> eee_status in 16u */
 		eee_idle >>= 4;
 		vars->eee_status |= (eee_idle & SHMEM_EEE_TIMER_MASK) |
-		    SHMEM_EEE_TIME_OUTPUT_BIT;
+				    SHMEM_EEE_TIME_OUTPUT_BIT;
 	} else {
 		if (elink_eee_time_to_nvram(eee_idle, &eee_mode))
 			return ELINK_STATUS_ERROR;
@@ -2695,8 +3778,7 @@ static elink_status_t elink_eee_set_timers(struct elink_params *params,
 }
 
 static elink_status_t elink_eee_initial_config(struct elink_params *params,
-					       struct elink_vars *vars,
-					       uint8_t mode)
+				     struct elink_vars *vars, uint8_t mode)
 {
 	vars->eee_status |= ((uint32_t) mode) << SHMEM_EEE_SUPPORTED_SHIFT;
 
@@ -2715,8 +3797,8 @@ static elink_status_t elink_eee_initial_config(struct elink_params *params,
 }
 
 static elink_status_t elink_eee_disable(struct elink_phy *phy,
-					struct elink_params *params,
-					struct elink_vars *vars)
+				struct elink_params *params,
+				struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 
@@ -2731,9 +3813,8 @@ static elink_status_t elink_eee_disable(struct elink_phy *phy,
 }
 
 static elink_status_t elink_eee_advertise(struct elink_phy *phy,
-					  struct elink_params *params,
-					  struct elink_vars *vars,
-					  uint8_t modes)
+				  struct elink_params *params,
+				  struct elink_vars *vars, uint8_t modes)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t val = 0;
@@ -2742,11 +3823,11 @@ static elink_status_t elink_eee_advertise(struct elink_phy *phy,
 	REG_WR(sc, MISC_REG_CPMU_LP_MASK_EXT_P0 + (params->port << 2), 0xfc20);
 
 	if (modes & SHMEM_EEE_10G_ADV) {
-		PMD_DRV_LOG(DEBUG, "Advertise 10GBase-T EEE");
+		ELINK_DEBUG_P0(sc, "Advertise 10GBase-T EEE");
 		val |= 0x8;
 	}
 	if (modes & SHMEM_EEE_1G_ADV) {
-		PMD_DRV_LOG(DEBUG, "Advertise 1GBase-T EEE");
+		ELINK_DEBUG_P0(sc, "Advertise 1GBase-T EEE");
 		val |= 0x4;
 	}
 
@@ -2770,8 +3851,8 @@ static void elink_update_mng_eee(struct elink_params *params,
 }
 
 static void elink_eee_an_resolve(struct elink_phy *phy,
-				 struct elink_params *params,
-				 struct elink_vars *vars)
+				  struct elink_params *params,
+				  struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t adv = 0, lp = 0;
@@ -2786,7 +3867,7 @@ static void elink_eee_an_resolve(struct elink_phy *phy,
 		if (adv & 0x2) {
 			if (vars->line_speed == ELINK_SPEED_100)
 				neg = 1;
-			PMD_DRV_LOG(DEBUG, "EEE negotiated - 100M");
+			ELINK_DEBUG_P0(sc, "EEE negotiated - 100M");
 		}
 	}
 	if (lp & 0x14) {
@@ -2794,7 +3875,7 @@ static void elink_eee_an_resolve(struct elink_phy *phy,
 		if (adv & 0x14) {
 			if (vars->line_speed == ELINK_SPEED_1000)
 				neg = 1;
-			PMD_DRV_LOG(DEBUG, "EEE negotiated - 1G");
+			ELINK_DEBUG_P0(sc, "EEE negotiated - 1G");
 		}
 	}
 	if (lp & 0x68) {
@@ -2802,7 +3883,7 @@ static void elink_eee_an_resolve(struct elink_phy *phy,
 		if (adv & 0x68) {
 			if (vars->line_speed == ELINK_SPEED_10000)
 				neg = 1;
-			PMD_DRV_LOG(DEBUG, "EEE negotiated - 10G");
+			ELINK_DEBUG_P0(sc, "EEE negotiated - 10G");
 		}
 	}
 
@@ -2810,7 +3891,7 @@ static void elink_eee_an_resolve(struct elink_phy *phy,
 	vars->eee_status |= (lp_adv << SHMEM_EEE_LP_ADV_STATUS_SHIFT);
 
 	if (neg) {
-		PMD_DRV_LOG(DEBUG, "EEE is active");
+		ELINK_DEBUG_P0(sc, "EEE is active");
 		vars->eee_status |= SHMEM_EEE_ACTIVE_BIT;
 	}
 }
@@ -2831,37 +3912,34 @@ static void elink_bsc_module_sel(struct elink_params *params)
 				    dev_info.shared_hw_config.board));
 	i2c_pins[I2C_BSC0] = board_cfg & SHARED_HW_CFG_E3_I2C_MUX0_MASK;
 	i2c_pins[I2C_BSC1] = (board_cfg & SHARED_HW_CFG_E3_I2C_MUX1_MASK) >>
-	    SHARED_HW_CFG_E3_I2C_MUX1_SHIFT;
+			SHARED_HW_CFG_E3_I2C_MUX1_SHIFT;
 
 	/* Read I2C output value */
 	sfp_ctrl = REG_RD(sc, params->shmem_base +
 			  offsetof(struct shmem_region,
-				   dev_info.port_hw_config[port].
-				   e3_cmn_pin_cfg));
+				 dev_info.port_hw_config[port].e3_cmn_pin_cfg));
 	i2c_val[I2C_BSC0] = (sfp_ctrl & PORT_HW_CFG_E3_I2C_MUX0_MASK) > 0;
 	i2c_val[I2C_BSC1] = (sfp_ctrl & PORT_HW_CFG_E3_I2C_MUX1_MASK) > 0;
-	PMD_DRV_LOG(DEBUG, "Setting BSC switch");
+	ELINK_DEBUG_P0(sc, "Setting BSC switch");
 	for (idx = 0; idx < I2C_SWITCH_WIDTH; idx++)
 		elink_set_cfg_pin(sc, i2c_pins[idx], i2c_val[idx]);
 }
 
-static elink_status_t elink_bsc_read(struct elink_params *params,
-				     struct bnx2x_softc *sc,
-				     uint8_t sl_devid,
-				     uint16_t sl_addr,
-				     uint8_t lc_addr,
-				     uint8_t xfer_cnt, uint32_t * data_array)
+static elink_status_t elink_bsc_read(struct bnx2x_softc *sc,
+			  uint8_t sl_devid,
+			  uint16_t sl_addr,
+			  uint8_t lc_addr,
+			  uint8_t xfer_cnt,
+			  uint32_t *data_array)
 {
 	uint32_t val, i;
 	elink_status_t rc = ELINK_STATUS_OK;
 
 	if (xfer_cnt > 16) {
-		PMD_DRV_LOG(DEBUG, "invalid xfer_cnt %d. Max is 16 bytes",
-			    xfer_cnt);
+		ELINK_DEBUG_P1(sc, "invalid xfer_cnt %d. Max is 16 bytes",
+					xfer_cnt);
 		return ELINK_STATUS_ERROR;
 	}
-	if (params)
-		elink_bsc_module_sel(params);
 
 	xfer_cnt = 16 - lc_addr;
 
@@ -2874,11 +3952,11 @@ static elink_status_t elink_bsc_read(struct elink_params *params,
 	val = (sl_devid << 16) | sl_addr;
 	REG_WR(sc, MCP_REG_MCPR_IMC_SLAVE_CONTROL, val);
 
-	/* Start xfer with 0 byte to update the address pointer ??? */
+	/* Start xfer with 0 byte to update the address pointer ???*/
 	val = (MCPR_IMC_COMMAND_ENABLE) |
-	    (MCPR_IMC_COMMAND_WRITE_OP <<
-	     MCPR_IMC_COMMAND_OPERATION_BITSHIFT) |
-	    (lc_addr << MCPR_IMC_COMMAND_TRANSFER_ADDRESS_BITSHIFT) | (0);
+	      (MCPR_IMC_COMMAND_WRITE_OP <<
+		MCPR_IMC_COMMAND_OPERATION_BITSHIFT) |
+		(lc_addr << MCPR_IMC_COMMAND_TRANSFER_ADDRESS_BITSHIFT) | (0);
 	REG_WR(sc, MCP_REG_MCPR_IMC_COMMAND, val);
 
 	/* Poll for completion */
@@ -2888,8 +3966,8 @@ static elink_status_t elink_bsc_read(struct elink_params *params,
 		DELAY(10);
 		val = REG_RD(sc, MCP_REG_MCPR_IMC_COMMAND);
 		if (i++ > 1000) {
-			PMD_DRV_LOG(DEBUG, "wr 0 byte timed out after %d try",
-				    i);
+			ELINK_DEBUG_P1(sc, "wr 0 byte timed out after %d try",
+								i);
 			rc = ELINK_STATUS_TIMEOUT;
 			break;
 		}
@@ -2899,10 +3977,10 @@ static elink_status_t elink_bsc_read(struct elink_params *params,
 
 	/* Start xfer with read op */
 	val = (MCPR_IMC_COMMAND_ENABLE) |
-	    (MCPR_IMC_COMMAND_READ_OP <<
-	     MCPR_IMC_COMMAND_OPERATION_BITSHIFT) |
-	    (lc_addr << MCPR_IMC_COMMAND_TRANSFER_ADDRESS_BITSHIFT) |
-	    (xfer_cnt);
+		(MCPR_IMC_COMMAND_READ_OP <<
+		MCPR_IMC_COMMAND_OPERATION_BITSHIFT) |
+		(lc_addr << MCPR_IMC_COMMAND_TRANSFER_ADDRESS_BITSHIFT) |
+		  (xfer_cnt);
 	REG_WR(sc, MCP_REG_MCPR_IMC_COMMAND, val);
 
 	/* Poll for completion */
@@ -2912,7 +3990,7 @@ static elink_status_t elink_bsc_read(struct elink_params *params,
 		DELAY(10);
 		val = REG_RD(sc, MCP_REG_MCPR_IMC_COMMAND);
 		if (i++ > 1000) {
-			PMD_DRV_LOG(DEBUG, "rd op timed out after %d try", i);
+			ELINK_DEBUG_P1(sc, "rd op timed out after %d try", i);
 			rc = ELINK_STATUS_TIMEOUT;
 			break;
 		}
@@ -2924,17 +4002,18 @@ static elink_status_t elink_bsc_read(struct elink_params *params,
 		data_array[i] = REG_RD(sc, (MCP_REG_MCPR_IMC_DATAREG0 + i * 4));
 #ifdef __BIG_ENDIAN
 		data_array[i] = ((data_array[i] & 0x000000ff) << 24) |
-		    ((data_array[i] & 0x0000ff00) << 8) |
-		    ((data_array[i] & 0x00ff0000) >> 8) |
-		    ((data_array[i] & 0xff000000) >> 24);
+				((data_array[i] & 0x0000ff00) << 8) |
+				((data_array[i] & 0x00ff0000) >> 8) |
+				((data_array[i] & 0xff000000) >> 24);
 #endif
 	}
 	return rc;
 }
 
 static void elink_cl45_read_or_write(struct bnx2x_softc *sc,
-				     struct elink_phy *phy, uint8_t devad,
-				     uint16_t reg, uint16_t or_val)
+				     struct elink_phy *phy,
+				     uint8_t devad, uint16_t reg,
+				     uint16_t or_val)
 {
 	uint16_t val;
 	elink_cl45_read(sc, phy, devad, reg, &val);
@@ -2951,7 +4030,42 @@ static void elink_cl45_read_and_write(struct bnx2x_softc *sc,
 	elink_cl45_write(sc, phy, devad, reg, val & and_val);
 }
 
-static uint8_t elink_get_warpcore_lane(struct elink_params *params)
+elink_status_t elink_phy_read(struct elink_params *params, uint8_t phy_addr,
+		   uint8_t devad, uint16_t reg, uint16_t *ret_val)
+{
+	uint8_t phy_index;
+	/* Probe for the phy according to the given phy_addr, and execute
+	 * the read request on it
+	 */
+	for (phy_index = 0; phy_index < params->num_phys; phy_index++) {
+		if (params->phy[phy_index].addr == phy_addr) {
+			return elink_cl45_read(params->sc,
+					       &params->phy[phy_index], devad,
+					       reg, ret_val);
+		}
+	}
+	return ELINK_STATUS_ERROR;
+}
+
+elink_status_t elink_phy_write(struct elink_params *params, uint8_t phy_addr,
+		    uint8_t devad, uint16_t reg, uint16_t val)
+{
+	uint8_t phy_index;
+	/* Probe for the phy according to the given phy_addr, and execute
+	 * the write request on it
+	 */
+	for (phy_index = 0; phy_index < params->num_phys; phy_index++) {
+		if (params->phy[phy_index].addr == phy_addr) {
+			return elink_cl45_write(params->sc,
+						&params->phy[phy_index], devad,
+						reg, val);
+		}
+	}
+	return ELINK_STATUS_ERROR;
+}
+
+static uint8_t elink_get_warpcore_lane(__rte_unused struct elink_phy *phy,
+				  struct elink_params *params)
 {
 	uint8_t lane = 0;
 	struct bnx2x_softc *sc = params->sc;
@@ -2985,14 +4099,16 @@ static uint8_t elink_get_warpcore_lane(struct elink_params *params)
 			port = port ^ 1;
 
 		lane = (port << 1) + path;
-	} else {		/* Two port mode - no port swap */
+	} else { /* Two port mode - no port swap */
 
 		/* Figure out path swap value */
-		path_swap_ovr = REG_RD(sc, MISC_REG_TWO_PORT_PATH_SWAP_OVWR);
+		path_swap_ovr =
+			REG_RD(sc, MISC_REG_TWO_PORT_PATH_SWAP_OVWR);
 		if (path_swap_ovr & 0x1) {
 			path_swap = (path_swap_ovr & 0x2);
 		} else {
-			path_swap = REG_RD(sc, MISC_REG_TWO_PORT_PATH_SWAP);
+			path_swap =
+				REG_RD(sc, MISC_REG_TWO_PORT_PATH_SWAP);
 		}
 		if (path_swap)
 			path = path ^ 1;
@@ -3002,6 +4118,7 @@ static uint8_t elink_get_warpcore_lane(struct elink_params *params)
 	return lane;
 }
 
+
 static void elink_set_aer_mmd(struct elink_params *params,
 			      struct elink_phy *phy)
 {
@@ -3010,13 +4127,13 @@ static void elink_set_aer_mmd(struct elink_params *params,
 	struct bnx2x_softc *sc = params->sc;
 	ser_lane = ((params->lane_config &
 		     PORT_HW_CFG_LANE_SWAP_CFG_MASTER_MASK) >>
-		    PORT_HW_CFG_LANE_SWAP_CFG_MASTER_SHIFT);
+		     PORT_HW_CFG_LANE_SWAP_CFG_MASTER_SHIFT);
 
 	offset = (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT) ?
-	    (phy->addr + ser_lane) : 0;
+		(phy->addr + ser_lane) : 0;
 
 	if (USES_WARPCORE(sc)) {
-		aer_val = elink_get_warpcore_lane(params);
+		aer_val = elink_get_warpcore_lane(phy, params);
 		/* In Dual-lane mode, two lanes are joined together,
 		 * so in order to configure them, the AER broadcast method is
 		 * used here.
@@ -3049,7 +4166,7 @@ static void elink_set_serdes_access(struct bnx2x_softc *sc, uint8_t port)
 	DELAY(500);
 	REG_WR(sc, emac_base + EMAC_REG_EMAC_MDIO_COMM, 0x245d000f);
 	DELAY(500);
-	/* Set Clause 45 */
+	 /* Set Clause 45 */
 	REG_WR(sc, NIG_REG_SERDES0_CTRL_MD_ST + port * 0x10, 0);
 }
 
@@ -3057,7 +4174,7 @@ static void elink_serdes_deassert(struct bnx2x_softc *sc, uint8_t port)
 {
 	uint32_t val;
 
-	PMD_DRV_LOG(DEBUG, "elink_serdes_deassert");
+	ELINK_DEBUG_P0(sc, "elink_serdes_deassert");
 
 	val = ELINK_SERDES_RESET_BITS << (port * 16);
 
@@ -3092,7 +4209,7 @@ static void elink_xgxs_deassert(struct elink_params *params)
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t port;
 	uint32_t val;
-	PMD_DRV_LOG(DEBUG, "elink_xgxs_deassert");
+	ELINK_DEBUG_P0(sc, "elink_xgxs_deassert");
 	port = params->port;
 
 	val = ELINK_XGXS_RESET_BITS << (port * 16);
@@ -3107,8 +4224,9 @@ static void elink_xgxs_deassert(struct elink_params *params)
 
 static void elink_calc_ieee_aneg_adv(struct elink_phy *phy,
 				     struct elink_params *params,
-				     uint16_t * ieee_fc)
+				     uint16_t *ieee_fc)
 {
+	struct bnx2x_softc *sc = params->sc;
 	*ieee_fc = MDIO_COMBO_IEEE0_AUTO_NEG_ADV_FULL_DUPLEX;
 	/* Resolve pause mode and advertisement Please refer to Table
 	 * 28B-3 of the 802.3ab-1999 spec
@@ -3118,12 +4236,12 @@ static void elink_calc_ieee_aneg_adv(struct elink_phy *phy,
 	case ELINK_FLOW_CTRL_AUTO:
 		switch (params->req_fc_auto_adv) {
 		case ELINK_FLOW_CTRL_BOTH:
+		case ELINK_FLOW_CTRL_RX:
 			*ieee_fc |= MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH;
 			break;
-		case ELINK_FLOW_CTRL_RX:
 		case ELINK_FLOW_CTRL_TX:
 			*ieee_fc |=
-			    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC;
+				MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC;
 			break;
 		default:
 			break;
@@ -3143,16 +4261,18 @@ static void elink_calc_ieee_aneg_adv(struct elink_phy *phy,
 		*ieee_fc |= MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_NONE;
 		break;
 	}
-	PMD_DRV_LOG(DEBUG, "ieee_fc = 0x%x", *ieee_fc);
+	ELINK_DEBUG_P1(sc, "ieee_fc = 0x%x", *ieee_fc);
 }
 
-static void set_phy_vars(struct elink_params *params, struct elink_vars *vars)
+static void set_phy_vars(struct elink_params *params,
+			 struct elink_vars *vars)
 {
+	struct bnx2x_softc *sc = params->sc;
 	uint8_t actual_phy_idx, phy_index, link_cfg_idx;
 	uint8_t phy_config_swapped = params->multi_phy_config &
-	    PORT_HW_CFG_PHY_SWAPPED_ENABLED;
+			PORT_HW_CFG_PHY_SWAPPED_ENABLED;
 	for (phy_index = ELINK_INT_PHY; phy_index < params->num_phys;
-	     phy_index++) {
+	      phy_index++) {
 		link_cfg_idx = ELINK_LINK_CONFIG_IDX(phy_index);
 		actual_phy_idx = phy_index;
 		if (phy_config_swapped) {
@@ -3162,26 +4282,26 @@ static void set_phy_vars(struct elink_params *params, struct elink_vars *vars)
 				actual_phy_idx = ELINK_EXT_PHY1;
 		}
 		params->phy[actual_phy_idx].req_flow_ctrl =
-		    params->req_flow_ctrl[link_cfg_idx];
+			params->req_flow_ctrl[link_cfg_idx];
 
 		params->phy[actual_phy_idx].req_line_speed =
-		    params->req_line_speed[link_cfg_idx];
+			params->req_line_speed[link_cfg_idx];
 
 		params->phy[actual_phy_idx].speed_cap_mask =
-		    params->speed_cap_mask[link_cfg_idx];
+			params->speed_cap_mask[link_cfg_idx];
 
 		params->phy[actual_phy_idx].req_duplex =
-		    params->req_duplex[link_cfg_idx];
+			params->req_duplex[link_cfg_idx];
 
 		if (params->req_line_speed[link_cfg_idx] ==
 		    ELINK_SPEED_AUTO_NEG)
 			vars->link_status |= LINK_STATUS_AUTO_NEGOTIATE_ENABLED;
 
-		PMD_DRV_LOG(DEBUG, "req_flow_ctrl %x, req_line_speed %x,"
-			    " speed_cap_mask %x",
-			    params->phy[actual_phy_idx].req_flow_ctrl,
-			    params->phy[actual_phy_idx].req_line_speed,
-			    params->phy[actual_phy_idx].speed_cap_mask);
+		ELINK_DEBUG_P3(sc, "req_flow_ctrl %x, req_line_speed %x,"
+			   " speed_cap_mask %x",
+			   params->phy[actual_phy_idx].req_flow_ctrl,
+			   params->phy[actual_phy_idx].req_line_speed,
+			   params->phy[actual_phy_idx].speed_cap_mask);
 	}
 }
 
@@ -3199,38 +4319,57 @@ static void elink_ext_phy_set_pause(struct elink_params *params,
 	/* Please refer to Table 28B-3 of 802.3ab-1999 spec. */
 	elink_calc_ieee_aneg_adv(phy, params, &vars->ieee_fc);
 	if ((vars->ieee_fc &
-	     MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC) ==
+	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC) ==
 	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC) {
 		val |= MDIO_AN_REG_ADV_PAUSE_ASYMMETRIC;
 	}
 	if ((vars->ieee_fc &
-	     MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH) ==
+	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH) ==
 	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH) {
 		val |= MDIO_AN_REG_ADV_PAUSE_PAUSE;
 	}
-	PMD_DRV_LOG(DEBUG, "Ext phy AN advertize 0x%x", val);
+	ELINK_DEBUG_P1(sc, "Ext phy AN advertize 0x%x", val);
 	elink_cl45_write(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_ADV_PAUSE, val);
 }
 
-static void elink_pause_resolve(struct elink_vars *vars, uint32_t pause_result)
-{				/*  LD      LP   */
-	switch (pause_result) {	/* ASYM P ASYM P */
-	case 0xb:		/*   1  0   1  1 */
+static void elink_pause_resolve(__rte_unused struct elink_phy *phy,
+				struct elink_params *params,
+				struct elink_vars *vars,
+				uint32_t pause_result)
+{
+	struct bnx2x_softc *sc = params->sc;
+						/*  LD	    LP	 */
+	switch (pause_result) {			/* ASYM P ASYM P */
+	case 0xb:				/*   1  0   1  1 */
+		ELINK_DEBUG_P0(sc, "Flow Control: TX only");
 		vars->flow_ctrl = ELINK_FLOW_CTRL_TX;
 		break;
 
-	case 0xe:		/*   1  1   1  0 */
+	case 0xe:				/*   1  1   1  0 */
+		ELINK_DEBUG_P0(sc, "Flow Control: RX only");
 		vars->flow_ctrl = ELINK_FLOW_CTRL_RX;
 		break;
 
-	case 0x5:		/*   0  1   0  1 */
-	case 0x7:		/*   0  1   1  1 */
-	case 0xd:		/*   1  1   0  1 */
-	case 0xf:		/*   1  1   1  1 */
+	case 0x5:				/*   0  1   0  1 */
+	case 0x7:				/*   0  1   1  1 */
+	case 0xd:				/*   1  1   0  1 */
+	case 0xf:				/*   1  1   1  1 */
+		/* If the user selected to advertise RX ONLY,
+		 * although we advertised both, need to enable
+		 * RX only.
+		 */
+
+		if (params->req_fc_auto_adv == ELINK_FLOW_CTRL_BOTH) {
+			ELINK_DEBUG_P0(sc, "Flow Control: RX & TX");
 		vars->flow_ctrl = ELINK_FLOW_CTRL_BOTH;
+		} else {
+			ELINK_DEBUG_P0(sc, "Flow Control: RX only");
+			vars->flow_ctrl = ELINK_FLOW_CTRL_RX;
+		}
 		break;
-
 	default:
+		ELINK_DEBUG_P0(sc, "Flow Control: None");
+		vars->flow_ctrl = ELINK_FLOW_CTRL_NONE;
 		break;
 	}
 	if (pause_result & (1 << 0))
@@ -3244,22 +4383,23 @@ static void elink_ext_phy_update_adv_fc(struct elink_phy *phy,
 					struct elink_params *params,
 					struct elink_vars *vars)
 {
-	uint16_t ld_pause;	/* local */
-	uint16_t lp_pause;	/* link partner */
+	uint16_t ld_pause;		/* local */
+	uint16_t lp_pause;		/* link partner */
 	uint16_t pause_result;
 	struct bnx2x_softc *sc = params->sc;
-	if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X54618SE) {
+	if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY2_TYPE_BNX2X54618SE) {
 		elink_cl22_read(sc, phy, 0x4, &ld_pause);
 		elink_cl22_read(sc, phy, 0x5, &lp_pause);
-	} else if (CHIP_IS_E3(sc) && ELINK_SINGLE_MEDIA_DIRECT(params)) {
-		uint8_t lane = elink_get_warpcore_lane(params);
+	} else if (CHIP_IS_E3(sc) &&
+		ELINK_SINGLE_MEDIA_DIRECT(params)) {
+		uint8_t lane = elink_get_warpcore_lane(phy, params);
 		uint16_t gp_status, gp_mask;
 		elink_cl45_read(sc, phy,
 				MDIO_AN_DEVAD, MDIO_WC_REG_GP2_STATUS_GP_2_4,
 				&gp_status);
 		gp_mask = (MDIO_WC_REG_GP2_STATUS_GP_2_4_CL73_AN_CMPL |
 			   MDIO_WC_REG_GP2_STATUS_GP_2_4_CL37_LP_AN_CAP) <<
-		    lane;
+			lane;
 		if ((gp_status & gp_mask) == gp_mask) {
 			elink_cl45_read(sc, phy, MDIO_AN_DEVAD,
 					MDIO_AN_REG_ADV_PAUSE, &ld_pause);
@@ -3285,16 +4425,18 @@ static void elink_ext_phy_update_adv_fc(struct elink_phy *phy,
 				MDIO_AN_DEVAD,
 				MDIO_AN_REG_LP_AUTO_NEG, &lp_pause);
 	}
-	pause_result = (ld_pause & MDIO_AN_REG_ADV_PAUSE_MASK) >> 8;
-	pause_result |= (lp_pause & MDIO_AN_REG_ADV_PAUSE_MASK) >> 10;
-	PMD_DRV_LOG(DEBUG, "Ext PHY pause result 0x%x", pause_result);
-	elink_pause_resolve(vars, pause_result);
+	pause_result = (ld_pause &
+			MDIO_AN_REG_ADV_PAUSE_MASK) >> 8;
+	pause_result |= (lp_pause &
+			 MDIO_AN_REG_ADV_PAUSE_MASK) >> 10;
+	ELINK_DEBUG_P1(sc, "Ext PHY pause result 0x%x", pause_result);
+	elink_pause_resolve(phy, params, vars, pause_result);
 
 }
 
 static uint8_t elink_ext_phy_resolve_fc(struct elink_phy *phy,
-					struct elink_params *params,
-					struct elink_vars *vars)
+				   struct elink_params *params,
+				   struct elink_vars *vars)
 {
 	uint8_t ret = 0;
 	vars->flow_ctrl = ELINK_FLOW_CTRL_NONE;
@@ -3312,7 +4454,6 @@ static uint8_t elink_ext_phy_resolve_fc(struct elink_phy *phy,
 	}
 	return ret;
 }
-
 /******************************************************************/
 /*			Warpcore section			  */
 /******************************************************************/
@@ -3321,19 +4462,31 @@ static uint8_t elink_ext_phy_resolve_fc(struct elink_phy *phy,
  * init configuration, and set/clear SGMII flag. Internal
  * phy init is done purely in phy_init stage.
  */
-#define WC_TX_DRIVER(post2, idriver, ipre) \
+#define WC_TX_DRIVER(post2, idriver, ipre, ifir) \
 	((post2 << MDIO_WC_REG_TX0_TX_DRIVER_POST2_COEFF_OFFSET) | \
 	 (idriver << MDIO_WC_REG_TX0_TX_DRIVER_IDRIVER_OFFSET) | \
-	 (ipre << MDIO_WC_REG_TX0_TX_DRIVER_IPRE_DRIVER_OFFSET))
+	 (ipre << MDIO_WC_REG_TX0_TX_DRIVER_IPRE_DRIVER_OFFSET) | \
+	 (ifir << MDIO_WC_REG_TX0_TX_DRIVER_IFIR_OFFSET))
 
 #define WC_TX_FIR(post, main, pre) \
 	((post << MDIO_WC_REG_TX_FIR_TAP_POST_TAP_OFFSET) | \
 	 (main << MDIO_WC_REG_TX_FIR_TAP_MAIN_TAP_OFFSET) | \
 	 (pre << MDIO_WC_REG_TX_FIR_TAP_PRE_TAP_OFFSET))
 
+static void elink_update_link_attr(struct elink_params *params,
+				   uint32_t link_attr)
+{
+	struct bnx2x_softc *sc = params->sc;
+
+	if (SHMEM2_HAS(sc, link_attr_sync))
+		REG_WR(sc, params->shmem2_base +
+		       offsetof(struct shmem2_region,
+				link_attr_sync[params->port]), link_attr);
+}
+
 static void elink_warpcore_enable_AN_KR2(struct elink_phy *phy,
 					 struct elink_params *params,
-					 struct elink_vars *vars)
+					 __rte_unused struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t i;
@@ -3356,7 +4509,7 @@ static void elink_warpcore_enable_AN_KR2(struct elink_phy *phy,
 		{MDIO_WC_DEVAD, MDIO_WC_REG_ETA_CL73_LD_BAM_CODE, 0x0157},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_ETA_CL73_LD_UD_CODE, 0x0620}
 	};
-	PMD_DRV_LOG(DEBUG, "Enabling 20G-KR2");
+	ELINK_DEBUG_P0(sc, "Enabling 20G-KR2");
 
 	elink_cl45_read_or_write(sc, phy, MDIO_WC_DEVAD,
 				 MDIO_WC_REG_CL49_USERB0_CTRL, (3 << 6));
@@ -3366,15 +4519,16 @@ static void elink_warpcore_enable_AN_KR2(struct elink_phy *phy,
 				 reg_set[i].val);
 
 	/* Start KR2 work-around timer which handles BNX2X8073 link-parner */
-	vars->link_attr_sync |= LINK_ATTR_SYNC_KR2_ENABLE;
-	elink_update_link_attr(params, vars->link_attr_sync);
+	params->link_attr_sync |= LINK_ATTR_SYNC_KR2_ENABLE;
+	elink_update_link_attr(params, params->link_attr_sync);
 }
 
 static void elink_disable_kr2(struct elink_params *params,
-			      struct elink_vars *vars, struct elink_phy *phy)
+			      struct elink_vars *vars,
+			      struct elink_phy *phy)
 {
 	struct bnx2x_softc *sc = params->sc;
-	uint32_t i;
+	int i;
 	static struct elink_reg_set reg_set[] = {
 		/* Step 1 - Program the TX/RX alignment markers */
 		{MDIO_WC_DEVAD, MDIO_WC_REG_CL82_USERB1_TX_CTRL5, 0x7690},
@@ -3393,13 +4547,13 @@ static void elink_disable_kr2(struct elink_params *params,
 		{MDIO_WC_DEVAD, MDIO_WC_REG_ETA_CL73_LD_BAM_CODE, 0x0002},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_ETA_CL73_LD_UD_CODE, 0x0000}
 	};
-	PMD_DRV_LOG(DEBUG, "Disabling 20G-KR2");
+	ELINK_DEBUG_P0(sc, "Disabling 20G-KR2");
 
-	for (i = 0; i < ARRAY_SIZE(reg_set); i++)
+	for (i = 0; i < (int)ARRAY_SIZE(reg_set); i++)
 		elink_cl45_write(sc, phy, reg_set[i].devad, reg_set[i].reg,
 				 reg_set[i].val);
-	vars->link_attr_sync &= ~LINK_ATTR_SYNC_KR2_ENABLE;
-	elink_update_link_attr(params, vars->link_attr_sync);
+	params->link_attr_sync &= ~LINK_ATTR_SYNC_KR2_ENABLE;
+	elink_update_link_attr(params, params->link_attr_sync);
 
 	vars->check_kr2_recovery_cnt = ELINK_CHECK_KR2_RECOVERY_CNT;
 }
@@ -3409,7 +4563,7 @@ static void elink_warpcore_set_lpi_passthrough(struct elink_phy *phy,
 {
 	struct bnx2x_softc *sc = params->sc;
 
-	PMD_DRV_LOG(DEBUG, "Configure WC for LPI pass through");
+	ELINK_DEBUG_P0(sc, "Configure WC for LPI pass through");
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
 			 MDIO_WC_REG_EEE_COMBO_CONTROL0, 0x7c);
 	elink_cl45_read_or_write(sc, phy, MDIO_WC_DEVAD,
@@ -3421,7 +4575,7 @@ static void elink_warpcore_restart_AN_KR(struct elink_phy *phy,
 {
 	/* Restart autoneg on the leading lane only */
 	struct bnx2x_softc *sc = params->sc;
-	uint16_t lane = elink_get_warpcore_lane(params);
+	uint16_t lane = elink_get_warpcore_lane(phy, params);
 	CL22_WR_OVER_CL45(sc, phy, MDIO_REG_BANK_AER_BLOCK,
 			  MDIO_AER_BLOCK_AER_REG, lane);
 	elink_cl45_write(sc, phy, MDIO_AN_DEVAD,
@@ -3433,9 +4587,9 @@ static void elink_warpcore_restart_AN_KR(struct elink_phy *phy,
 
 static void elink_warpcore_enable_AN_KR(struct elink_phy *phy,
 					struct elink_params *params,
-					struct elink_vars *vars)
-{
-	uint16_t lane, i, cl72_ctrl, an_adv = 0;
+					struct elink_vars *vars) {
+	uint16_t lane, i, cl72_ctrl, an_adv = 0, val;
+	uint32_t wc_lane_config;
 	struct bnx2x_softc *sc = params->sc;
 	static struct elink_reg_set reg_set[] = {
 		{MDIO_WC_DEVAD, MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X2, 0x7},
@@ -3447,7 +4601,7 @@ static void elink_warpcore_enable_AN_KR(struct elink_phy *phy,
 		{MDIO_PMA_DEVAD, MDIO_WC_REG_PMD_KR_CONTROL, 0x2},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_CL72_USERB0_CL72_TX_FIR_TAP, 0},
 	};
-	PMD_DRV_LOG(DEBUG, "Enable Auto Negotiation for KR");
+	ELINK_DEBUG_P0(sc,  "Enable Auto Negotiation for KR");
 	/* Set to default registers that may be overridden by 10G force */
 	for (i = 0; i < ARRAY_SIZE(reg_set); i++)
 		elink_cl45_write(sc, phy, reg_set[i].devad, reg_set[i].reg,
@@ -3469,11 +4623,11 @@ static void elink_warpcore_enable_AN_KR(struct elink_phy *phy,
 
 		/* Enable CL37 1G Parallel Detect */
 		elink_cl45_read_or_write(sc, phy, MDIO_WC_DEVAD, addr, 0x1);
-		PMD_DRV_LOG(DEBUG, "Advertize 1G");
+		ELINK_DEBUG_P0(sc, "Advertize 1G");
 	}
 	if (((vars->line_speed == ELINK_SPEED_AUTO_NEG) &&
 	     (phy->speed_cap_mask & PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)) ||
-	    (vars->line_speed == ELINK_SPEED_10000)) {
+	    (vars->line_speed ==  ELINK_SPEED_10000)) {
 		/* Check adding advertisement for 10G KR */
 		an_adv |= (1 << 7);
 		/* Enable 10G Parallel Detect */
@@ -3483,23 +4637,25 @@ static void elink_warpcore_enable_AN_KR(struct elink_phy *phy,
 		elink_cl45_write(sc, phy, MDIO_AN_DEVAD,
 				 MDIO_WC_REG_PAR_DET_10G_CTRL, 1);
 		elink_set_aer_mmd(params, phy);
-		PMD_DRV_LOG(DEBUG, "Advertize 10G");
+		ELINK_DEBUG_P0(sc, "Advertize 10G");
 	}
 
 	/* Set Transmit PMD settings */
-	lane = elink_get_warpcore_lane(params);
+	lane = elink_get_warpcore_lane(phy, params);
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
 			 MDIO_WC_REG_TX0_TX_DRIVER + 0x10 * lane,
-			 WC_TX_DRIVER(0x02, 0x06, 0x09));
+			 WC_TX_DRIVER(0x02, 0x06, 0x09, 0));
 	/* Configure the next lane if dual mode */
 	if (phy->flags & ELINK_FLAGS_WC_DUAL_MODE)
 		elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
 				 MDIO_WC_REG_TX0_TX_DRIVER + 0x10 * (lane + 1),
-				 WC_TX_DRIVER(0x02, 0x06, 0x09));
+				 WC_TX_DRIVER(0x02, 0x06, 0x09, 0));
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
-			 MDIO_WC_REG_CL72_USERB0_CL72_OS_DEF_CTRL, 0x03f0);
+			 MDIO_WC_REG_CL72_USERB0_CL72_OS_DEF_CTRL,
+			 0x03f0);
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
-			 MDIO_WC_REG_CL72_USERB0_CL72_2P5_DEF_CTRL, 0x03f0);
+			 MDIO_WC_REG_CL72_USERB0_CL72_2P5_DEF_CTRL,
+			 0x03f0);
 
 	/* Advertised speeds */
 	elink_cl45_write(sc, phy, MDIO_AN_DEVAD,
@@ -3513,14 +4669,13 @@ static void elink_warpcore_enable_AN_KR(struct elink_phy *phy,
 
 	/* Enable CL37 BAM */
 	if (REG_RD(sc, params->shmem_base +
-		   offsetof(struct shmem_region,
-			    dev_info.port_hw_config[params->port].
-			    default_cfg)) &
+		   offsetof(struct shmem_region, dev_info.
+			    port_hw_config[params->port].default_cfg)) &
 	    PORT_HW_CFG_ENABLE_BAM_ON_KR_ENABLED) {
 		elink_cl45_read_or_write(sc, phy, MDIO_WC_DEVAD,
 					 MDIO_WC_REG_DIGITAL6_MP5_NEXTPAGECTRL,
 					 1);
-		PMD_DRV_LOG(DEBUG, "Enable CL37 BAM on KR");
+		ELINK_DEBUG_P0(sc, "Enable CL37 BAM on KR");
 	}
 
 	/* Advertise pause */
@@ -3531,7 +4686,7 @@ static void elink_warpcore_enable_AN_KR(struct elink_phy *phy,
 
 	/* Over 1G - AN local device user page 1 */
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
-			 MDIO_WC_REG_DIGITAL3_UP1, 0x1f);
+			MDIO_WC_REG_DIGITAL3_UP1, 0x1f);
 
 	if (((phy->req_line_speed == ELINK_SPEED_AUTO_NEG) &&
 	     (phy->speed_cap_mask & PORT_HW_CFG_SPEED_CAPABILITY_D0_20G)) ||
@@ -3542,7 +4697,8 @@ static void elink_warpcore_enable_AN_KR(struct elink_phy *phy,
 
 		elink_cl45_read_or_write(sc, phy, MDIO_WC_DEVAD,
 					 MDIO_WC_REG_RX1_PCI_CTRL +
-					 (0x10 * lane), (1 << 11));
+					 (0x10 * lane),
+					 (1 << 11));
 
 		elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
 				 MDIO_WC_REG_XGXS_X2_CONTROL3, 0x7);
@@ -3550,6 +4706,31 @@ static void elink_warpcore_enable_AN_KR(struct elink_phy *phy,
 
 		elink_warpcore_enable_AN_KR2(phy, params, vars);
 	} else {
+		/* Enable Auto-Detect to support 1G over CL37 as well */
+		elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
+				 MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X1, 0x10);
+		wc_lane_config = REG_RD(sc, params->shmem_base +
+					offsetof(struct shmem_region, dev_info.
+					shared_hw_config.wc_lane_config));
+		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
+				MDIO_WC_REG_RX0_PCI_CTRL + (lane << 4), &val);
+		/* Force cl48 sync_status LOW to avoid getting stuck in CL73
+		 * parallel-detect loop when CL73 and CL37 are enabled.
+		 */
+		val |= 1 << 11;
+
+		/* Restore Polarity settings in case it was run over by
+		 * previous link owner
+		 */
+		if (wc_lane_config &
+		    (SHARED_HW_CFG_RX_LANE0_POL_FLIP_ENABLED << lane))
+			val |= 3 << 2;
+		else
+			val &= ~(3 << 2);
+		elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
+				 MDIO_WC_REG_RX0_PCI_CTRL + (lane << 4),
+				 val);
+
 		elink_disable_kr2(params, vars, phy);
 	}
 
@@ -3558,7 +4739,8 @@ static void elink_warpcore_enable_AN_KR(struct elink_phy *phy,
 }
 
 static void elink_warpcore_set_10G_KR(struct elink_phy *phy,
-				      struct elink_params *params)
+				      struct elink_params *params,
+				      __rte_unused struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t val16, i, lane;
@@ -3566,7 +4748,7 @@ static void elink_warpcore_set_10G_KR(struct elink_phy *phy,
 		/* Disable Autoneg */
 		{MDIO_WC_DEVAD, MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X2, 0x7},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_CL72_USERB0_CL72_MISC1_CONTROL,
-		 0x3f00},
+			0x3f00},
 		{MDIO_AN_DEVAD, MDIO_WC_REG_AN_IEEE1BLK_AN_ADVERTISEMENT1, 0},
 		{MDIO_AN_DEVAD, MDIO_WC_REG_IEEE0BLK_MIICNTL, 0x0},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_DIGITAL3_UP1, 0x1},
@@ -3579,7 +4761,7 @@ static void elink_warpcore_set_10G_KR(struct elink_phy *phy,
 		elink_cl45_write(sc, phy, reg_set[i].devad, reg_set[i].reg,
 				 reg_set[i].val);
 
-	lane = elink_get_warpcore_lane(params);
+	lane = elink_get_warpcore_lane(phy, params);
 	/* Global registers */
 	CL22_WR_OVER_CL45(sc, phy, MDIO_REG_BANK_AER_BLOCK,
 			  MDIO_AER_BLOCK_AER_REG, 0);
@@ -3609,7 +4791,8 @@ static void elink_warpcore_set_10G_KR(struct elink_phy *phy,
 			 MDIO_WC_REG_SERDESDIGITAL_MISC2, 0x30);
 
 	/* Turn TX scramble payload only the 64/66 scrambler */
-	elink_cl45_write(sc, phy, MDIO_WC_DEVAD, MDIO_WC_REG_TX66_CONTROL, 0x9);
+	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
+			 MDIO_WC_REG_TX66_CONTROL, 0x9);
 
 	/* Turn RX scramble payload only the 64/66 scrambler */
 	elink_cl45_read_or_write(sc, phy, MDIO_WC_DEVAD,
@@ -3630,7 +4813,7 @@ static void elink_warpcore_set_10G_XFI(struct elink_phy *phy,
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t misc1_val, tap_val, tx_driver_val, lane, val;
 	uint32_t cfg_tap_val, tx_drv_brdct, tx_equal;
-
+	uint32_t ifir_val, ipost2_val, ipre_driver_val;
 	/* Hold rxSeqStart */
 	elink_cl45_read_or_write(sc, phy, MDIO_WC_DEVAD,
 				 MDIO_WC_REG_DSC2B0_DSC_MISC_CTRL0, 0x8000);
@@ -3675,38 +4858,59 @@ static void elink_warpcore_set_10G_XFI(struct elink_phy *phy,
 	if (is_xfi) {
 		misc1_val |= 0x5;
 		tap_val = WC_TX_FIR(0x08, 0x37, 0x00);
-		tx_driver_val = WC_TX_DRIVER(0x00, 0x02, 0x03);
+		tx_driver_val = WC_TX_DRIVER(0x00, 0x02, 0x03, 0);
 	} else {
 		cfg_tap_val = REG_RD(sc, params->shmem_base +
-				     offsetof(struct shmem_region,
-					      dev_info.port_hw_config[params->
-								      port].sfi_tap_values));
+				     offsetof(struct shmem_region, dev_info.
+					      port_hw_config[params->port].
+					      sfi_tap_values));
 
 		tx_equal = cfg_tap_val & PORT_HW_CFG_TX_EQUALIZATION_MASK;
 
-		tx_drv_brdct = (cfg_tap_val &
-				PORT_HW_CFG_TX_DRV_BROADCAST_MASK) >>
-		    PORT_HW_CFG_TX_DRV_BROADCAST_SHIFT;
-
 		misc1_val |= 0x9;
 
 		/* TAP values are controlled by nvram, if value there isn't 0 */
 		if (tx_equal)
-			tap_val = (uint16_t) tx_equal;
+			tap_val = (uint16_t)tx_equal;
 		else
 			tap_val = WC_TX_FIR(0x0f, 0x2b, 0x02);
 
-		if (tx_drv_brdct)
-			tx_driver_val =
-			    WC_TX_DRIVER(0x03, (uint16_t) tx_drv_brdct, 0x06);
-		else
-			tx_driver_val = WC_TX_DRIVER(0x03, 0x02, 0x06);
+		ifir_val = DEFAULT_TX_DRV_IFIR;
+		ipost2_val = DEFAULT_TX_DRV_POST2;
+		ipre_driver_val = DEFAULT_TX_DRV_IPRE_DRIVER;
+		tx_drv_brdct = DEFAULT_TX_DRV_BRDCT;
+
+		/* If any of the IFIR/IPRE_DRIVER/POST@ is set, apply all
+		 * configuration.
+		 */
+		if (cfg_tap_val & (PORT_HW_CFG_TX_DRV_IFIR_MASK |
+				   PORT_HW_CFG_TX_DRV_IPREDRIVER_MASK |
+				   PORT_HW_CFG_TX_DRV_POST2_MASK)) {
+			ifir_val = (cfg_tap_val &
+				    PORT_HW_CFG_TX_DRV_IFIR_MASK) >>
+				PORT_HW_CFG_TX_DRV_IFIR_SHIFT;
+			ipre_driver_val = (cfg_tap_val &
+					   PORT_HW_CFG_TX_DRV_IPREDRIVER_MASK)
+			>> PORT_HW_CFG_TX_DRV_IPREDRIVER_SHIFT;
+			ipost2_val = (cfg_tap_val &
+				      PORT_HW_CFG_TX_DRV_POST2_MASK) >>
+				PORT_HW_CFG_TX_DRV_POST2_SHIFT;
+		}
+
+		if (cfg_tap_val & PORT_HW_CFG_TX_DRV_BROADCAST_MASK) {
+			tx_drv_brdct = (cfg_tap_val &
+					PORT_HW_CFG_TX_DRV_BROADCAST_MASK) >>
+				PORT_HW_CFG_TX_DRV_BROADCAST_SHIFT;
+		}
+
+		tx_driver_val = WC_TX_DRIVER(ipost2_val, tx_drv_brdct,
+					     ipre_driver_val, ifir_val);
 	}
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
 			 MDIO_WC_REG_SERDESDIGITAL_MISC1, misc1_val);
 
 	/* Set Transmit PMD settings */
-	lane = elink_get_warpcore_lane(params);
+	lane = elink_get_warpcore_lane(phy, params);
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
 			 MDIO_WC_REG_TX_FIR_TAP,
 			 tap_val | MDIO_WC_REG_TX_FIR_TAP_ENABLE);
@@ -3754,7 +4958,8 @@ static void elink_warpcore_set_20G_force_KR2(struct elink_phy *phy,
 
 	elink_cl45_read_and_write(sc, phy, MDIO_PMA_DEVAD,
 				  MDIO_WC_REG_PMD_KR_CONTROL, ~(1 << 1));
-	elink_cl45_write(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, 0);
+	elink_cl45_write(sc, phy, MDIO_AN_DEVAD,
+			 MDIO_AN_REG_CTRL, 0);
 	/* Turn off CL73 */
 	elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 			MDIO_WC_REG_CL73_USERB0_CTRL, &val);
@@ -3790,7 +4995,8 @@ static void elink_warpcore_set_20G_force_KR2(struct elink_phy *phy,
 }
 
 static void elink_warpcore_set_20G_DXGXS(struct bnx2x_softc *sc,
-					 struct elink_phy *phy, uint16_t lane)
+					 struct elink_phy *phy,
+					 uint16_t lane)
 {
 	/* Rx0 anaRxControl1G */
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
@@ -3800,13 +5006,17 @@ static void elink_warpcore_set_20G_DXGXS(struct bnx2x_softc *sc,
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
 			 MDIO_WC_REG_RX2_ANARXCONTROL1G, 0x90);
 
-	elink_cl45_write(sc, phy, MDIO_WC_DEVAD, MDIO_WC_REG_RX66_SCW0, 0xE070);
+	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
+			 MDIO_WC_REG_RX66_SCW0, 0xE070);
 
-	elink_cl45_write(sc, phy, MDIO_WC_DEVAD, MDIO_WC_REG_RX66_SCW1, 0xC0D0);
+	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
+			 MDIO_WC_REG_RX66_SCW1, 0xC0D0);
 
-	elink_cl45_write(sc, phy, MDIO_WC_DEVAD, MDIO_WC_REG_RX66_SCW2, 0xA0B0);
+	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
+			 MDIO_WC_REG_RX66_SCW2, 0xA0B0);
 
-	elink_cl45_write(sc, phy, MDIO_WC_DEVAD, MDIO_WC_REG_RX66_SCW3, 0x8090);
+	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
+			 MDIO_WC_REG_RX66_SCW3, 0x8090);
 
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
 			 MDIO_WC_REG_RX66_SCW0_MASK, 0xF0F0);
@@ -3835,7 +5045,7 @@ static void elink_warpcore_set_20G_DXGXS(struct bnx2x_softc *sc,
 			  MDIO_WC_REG_TX_FIR_TAP_ENABLE));
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
 			 MDIO_WC_REG_TX0_TX_DRIVER + 0x10 * lane,
-			 WC_TX_DRIVER(0x02, 0x02, 0x02));
+			 WC_TX_DRIVER(0x02, 0x02, 0x02, 0));
 }
 
 static void elink_warpcore_set_sgmii_speed(struct elink_phy *phy,
@@ -3857,7 +5067,7 @@ static void elink_warpcore_set_sgmii_speed(struct elink_phy *phy,
 		elink_cl45_read_or_write(sc, phy, MDIO_WC_DEVAD,
 					 MDIO_WC_REG_COMBO_IEEE0_MIICTRL,
 					 0x1000);
-		PMD_DRV_LOG(DEBUG, "set SGMII AUTONEG");
+		ELINK_DEBUG_P0(sc, "set SGMII AUTONEG");
 	} else {
 		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 				MDIO_WC_REG_COMBO_IEEE0_MIICTRL, &val16);
@@ -3872,9 +5082,8 @@ static void elink_warpcore_set_sgmii_speed(struct elink_phy *phy,
 			val16 |= 0x0040;
 			break;
 		default:
-			PMD_DRV_LOG(DEBUG,
-				    "Speed not supported: 0x%x",
-				    phy->req_line_speed);
+			ELINK_DEBUG_P1(sc,
+			   "Speed not supported: 0x%x", phy->req_line_speed);
 			return;
 		}
 
@@ -3882,13 +5091,13 @@ static void elink_warpcore_set_sgmii_speed(struct elink_phy *phy,
 			val16 |= 0x0100;
 
 		elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
-				 MDIO_WC_REG_COMBO_IEEE0_MIICTRL, val16);
+				MDIO_WC_REG_COMBO_IEEE0_MIICTRL, val16);
 
-		PMD_DRV_LOG(DEBUG, "set SGMII force speed %d",
-			    phy->req_line_speed);
+		ELINK_DEBUG_P1(sc, "set SGMII force speed %d",
+			       phy->req_line_speed);
 		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 				MDIO_WC_REG_COMBO_IEEE0_MIICTRL, &val16);
-		PMD_DRV_LOG(DEBUG, "  (readback) %x", val16);
+		ELINK_DEBUG_P1(sc, "  (readback) %x", val16);
 	}
 
 	/* SGMII Slave mode and disable signal detect */
@@ -3900,28 +5109,31 @@ static void elink_warpcore_set_sgmii_speed(struct elink_phy *phy,
 		digctrl_kx1 &= 0xff4a;
 
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
-			 MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X1, digctrl_kx1);
+			MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X1,
+			digctrl_kx1);
 
 	/* Turn off parallel detect */
 	elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 			MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X2, &digctrl_kx2);
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
-			 MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X2,
-			 (digctrl_kx2 & ~(1 << 2)));
+			MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X2,
+			(digctrl_kx2 & ~(1 << 2)));
 
 	/* Re-enable parallel detect */
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
-			 MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X2,
-			 (digctrl_kx2 | (1 << 2)));
+			MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X2,
+			(digctrl_kx2 | (1 << 2)));
 
 	/* Enable autodet */
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
-			 MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X1,
-			 (digctrl_kx1 | 0x10));
+			MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X1,
+			(digctrl_kx1 | 0x10));
 }
 
+
 static void elink_warpcore_reset_lane(struct bnx2x_softc *sc,
-				      struct elink_phy *phy, uint8_t reset)
+				      struct elink_phy *phy,
+				      uint8_t reset)
 {
 	uint16_t val;
 	/* Take lane out of reset after configuration is finished */
@@ -3934,7 +5146,7 @@ static void elink_warpcore_reset_lane(struct bnx2x_softc *sc,
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
 			 MDIO_WC_REG_DIGITAL5_MISC6, val);
 	elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
-			MDIO_WC_REG_DIGITAL5_MISC6, &val);
+			 MDIO_WC_REG_DIGITAL5_MISC6, &val);
 }
 
 /* Clear SFI/XFI link settings registers */
@@ -3950,11 +5162,11 @@ static void elink_warpcore_clear_regs(struct elink_phy *phy,
 		{MDIO_WC_DEVAD, MDIO_WC_REG_FX100_CTRL3, 0x0800},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_DIGITAL4_MISC3, 0x8008},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X1,
-		 0x0195},
+			0x0195},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X2,
-		 0x0007},
+			0x0007},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_SERDESDIGITAL_CONTROL1000X3,
-		 0x0002},
+			0x0002},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_SERDESDIGITAL_MISC1, 0x6000},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_TX_FIR_TAP, 0x0000},
 		{MDIO_WC_DEVAD, MDIO_WC_REG_IEEE0BLK_MIICNTL, 0x2040},
@@ -3968,40 +5180,41 @@ static void elink_warpcore_clear_regs(struct elink_phy *phy,
 		elink_cl45_write(sc, phy, wc_regs[i].devad, wc_regs[i].reg,
 				 wc_regs[i].val);
 
-	lane = elink_get_warpcore_lane(params);
+	lane = elink_get_warpcore_lane(phy, params);
 	elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
 			 MDIO_WC_REG_TX0_TX_DRIVER + 0x10 * lane, 0x0990);
 
 }
 
 static elink_status_t elink_get_mod_abs_int_cfg(struct bnx2x_softc *sc,
+						__rte_unused uint32_t chip_id,
 						uint32_t shmem_base,
 						uint8_t port,
-						uint8_t * gpio_num,
-						uint8_t * gpio_port)
+						uint8_t *gpio_num,
+						uint8_t *gpio_port)
 {
 	uint32_t cfg_pin;
 	*gpio_num = 0;
 	*gpio_port = 0;
 	if (CHIP_IS_E3(sc)) {
 		cfg_pin = (REG_RD(sc, shmem_base +
-				  offsetof(struct shmem_region,
-					   dev_info.port_hw_config[port].
-					   e3_sfp_ctrl)) &
-			   PORT_HW_CFG_E3_MOD_ABS_MASK) >>
-		    PORT_HW_CFG_E3_MOD_ABS_SHIFT;
-
-		/* Should not happen. This function called upon interrupt
-		 * triggered by GPIO ( since EPIO can only generate interrupts
-		 * to MCP).
+				offsetof(struct shmem_region,
+				dev_info.port_hw_config[port].e3_sfp_ctrl)) &
+				PORT_HW_CFG_E3_MOD_ABS_MASK) >>
+				PORT_HW_CFG_E3_MOD_ABS_SHIFT;
+
+		/*
+		 * This should not happen since this function is called
+		 * from interrupt triggered by GPIO (since EPIO can only
+		 * generate interrupts to MCP).
 		 * So if this function was called and none of the GPIOs was set,
-		 * it means the shit hit the fan.
+		 * it means something disastrous has already happened.
 		 */
 		if ((cfg_pin < PIN_CFG_GPIO0_P0) ||
 		    (cfg_pin > PIN_CFG_GPIO3_P1)) {
-			PMD_DRV_LOG(DEBUG,
-				    "No cfg pin %x for module detect indication",
-				    cfg_pin);
+			ELINK_DEBUG_P1(sc,
+			   "No cfg pin %x for module detect indication",
+			   cfg_pin);
 			return ELINK_STATUS_ERROR;
 		}
 
@@ -4015,12 +5228,13 @@ static elink_status_t elink_get_mod_abs_int_cfg(struct bnx2x_softc *sc,
 	return ELINK_STATUS_OK;
 }
 
-static int elink_is_sfp_module_plugged(struct elink_params *params)
+static int elink_is_sfp_module_plugged(__rte_unused struct elink_phy *phy,
+				       struct elink_params *params)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t gpio_num, gpio_port;
 	uint32_t gpio_val;
-	if (elink_get_mod_abs_int_cfg(sc,
+	if (elink_get_mod_abs_int_cfg(sc, params->chip_id,
 				      params->shmem_base, params->port,
 				      &gpio_num, &gpio_port) != ELINK_STATUS_OK)
 		return 0;
@@ -4032,17 +5246,16 @@ static int elink_is_sfp_module_plugged(struct elink_params *params)
 	else
 		return 0;
 }
-
 static int elink_warpcore_get_sigdet(struct elink_phy *phy,
 				     struct elink_params *params)
 {
 	uint16_t gp2_status_reg0, lane;
 	struct bnx2x_softc *sc = params->sc;
 
-	lane = elink_get_warpcore_lane(params);
+	lane = elink_get_warpcore_lane(phy, params);
 
 	elink_cl45_read(sc, phy, MDIO_WC_DEVAD, MDIO_WC_REG_GP2_STATUS_GP_2_0,
-			&gp2_status_reg0);
+				 &gp2_status_reg0);
 
 	return (gp2_status_reg0 >> (8 + lane)) & 0x1;
 }
@@ -4061,38 +5274,35 @@ static void elink_warpcore_config_runtime(struct elink_phy *phy,
 		return;
 
 	if (vars->rx_tx_asic_rst) {
-		uint16_t lane = elink_get_warpcore_lane(params);
+		uint16_t lane = elink_get_warpcore_lane(phy, params);
 		serdes_net_if = (REG_RD(sc, params->shmem_base +
-					offsetof(struct shmem_region,
-						 dev_info.port_hw_config
-						 [params->port].
-						 default_cfg)) &
-				 PORT_HW_CFG_NET_SERDES_IF_MASK);
+				offsetof(struct shmem_region, dev_info.
+				port_hw_config[params->port].default_cfg)) &
+				PORT_HW_CFG_NET_SERDES_IF_MASK);
 
 		switch (serdes_net_if) {
 		case PORT_HW_CFG_NET_SERDES_IF_KR:
 			/* Do we get link yet? */
 			elink_cl45_read(sc, phy, MDIO_WC_DEVAD, 0x81d1,
 					&gp_status1);
-			lnkup = (gp_status1 >> (8 + lane)) & 0x1;	/* 1G */
-			/*10G KR */
+			lnkup = (gp_status1 >> (8 + lane)) & 0x1;/* 1G */
+				/*10G KR*/
 			lnkup_kr = (gp_status1 >> (12 + lane)) & 0x1;
 
 			if (lnkup_kr || lnkup) {
 				vars->rx_tx_asic_rst = 0;
 			} else {
-				/* Reset the lane to see if link comes up. */
+				/* Reset the lane to see if link comes up.*/
 				elink_warpcore_reset_lane(sc, phy, 1);
 				elink_warpcore_reset_lane(sc, phy, 0);
 
 				/* Restart Autoneg */
 				elink_cl45_write(sc, phy, MDIO_AN_DEVAD,
-						 MDIO_WC_REG_IEEE0BLK_MIICNTL,
-						 0x1200);
+					MDIO_WC_REG_IEEE0BLK_MIICNTL, 0x1200);
 
 				vars->rx_tx_asic_rst--;
-				PMD_DRV_LOG(DEBUG, "0x%x retry left",
-					    vars->rx_tx_asic_rst);
+				ELINK_DEBUG_P1(sc, "0x%x retry left",
+				vars->rx_tx_asic_rst);
 			}
 			break;
 
@@ -4100,29 +5310,29 @@ static void elink_warpcore_config_runtime(struct elink_phy *phy,
 			break;
 		}
 
-	}
-	/*params->rx_tx_asic_rst */
+	} /*params->rx_tx_asic_rst*/
 }
 
 static void elink_warpcore_config_sfi(struct elink_phy *phy,
 				      struct elink_params *params)
 {
-	uint16_t lane = elink_get_warpcore_lane(params);
-
+	uint16_t lane = elink_get_warpcore_lane(phy, params);
+	struct bnx2x_softc *sc = params->sc;
 	elink_warpcore_clear_regs(phy, params, lane);
 	if ((params->req_line_speed[ELINK_LINK_CONFIG_IDX(ELINK_INT_PHY)] ==
 	     ELINK_SPEED_10000) &&
 	    (phy->media_type != ELINK_ETH_PHY_SFP_1G_FIBER)) {
-		PMD_DRV_LOG(DEBUG, "Setting 10G SFI");
+		ELINK_DEBUG_P0(sc, "Setting 10G SFI");
 		elink_warpcore_set_10G_XFI(phy, params, 0);
 	} else {
-		PMD_DRV_LOG(DEBUG, "Setting 1G Fiber");
+		ELINK_DEBUG_P0(sc, "Setting 1G Fiber");
 		elink_warpcore_set_sgmii_speed(phy, params, 1, 0);
 	}
 }
 
 static void elink_sfp_e3_set_transmitter(struct elink_params *params,
-					 struct elink_phy *phy, uint8_t tx_en)
+					 struct elink_phy *phy,
+					 uint8_t tx_en)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint32_t cfg_pin;
@@ -4131,9 +5341,9 @@ static void elink_sfp_e3_set_transmitter(struct elink_params *params,
 	cfg_pin = REG_RD(sc, params->shmem_base +
 			 offsetof(struct shmem_region,
 				  dev_info.port_hw_config[port].e3_sfp_ctrl)) &
-	    PORT_HW_CFG_E3_TX_LASER_MASK;
+		PORT_HW_CFG_E3_TX_LASER_MASK;
 	/* Set the !tx_en since this pin is DISABLE_TX_LASER */
-	PMD_DRV_LOG(DEBUG, "Setting WC TX to %d", tx_en);
+	ELINK_DEBUG_P1(sc, "Setting WC TX to %d", tx_en);
 
 	/* For 20G, the expected pin to be used is 3 pins after the current */
 	elink_set_cfg_pin(sc, cfg_pin, tx_en ^ 1);
@@ -4142,21 +5352,20 @@ static void elink_sfp_e3_set_transmitter(struct elink_params *params,
 }
 
 static uint8_t elink_warpcore_config_init(struct elink_phy *phy,
-					  struct elink_params *params,
-					  struct elink_vars *vars)
+				       struct elink_params *params,
+				       struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint32_t serdes_net_if;
 	uint8_t fiber_mode;
-	uint16_t lane = elink_get_warpcore_lane(params);
+	uint16_t lane = elink_get_warpcore_lane(phy, params);
 	serdes_net_if = (REG_RD(sc, params->shmem_base +
-				offsetof(struct shmem_region,
-					 dev_info.port_hw_config[params->port].
-					 default_cfg)) &
+			 offsetof(struct shmem_region, dev_info.
+				  port_hw_config[params->port].default_cfg)) &
 			 PORT_HW_CFG_NET_SERDES_IF_MASK);
-	PMD_DRV_LOG(DEBUG,
-		    "Begin Warpcore init, link_speed %d, "
-		    "serdes_net_if = 0x%x", vars->line_speed, serdes_net_if);
+	ELINK_DEBUG_P2(sc, "Begin Warpcore init, link_speed %d, "
+			   "serdes_net_if = 0x%x",
+		       vars->line_speed, serdes_net_if);
 	elink_set_aer_mmd(params, phy);
 	elink_warpcore_reset_lane(sc, phy, 1);
 	vars->phy_flags |= PHY_XGXS_FLAG;
@@ -4165,7 +5374,7 @@ static uint8_t elink_warpcore_config_init(struct elink_phy *phy,
 	     ((phy->req_line_speed == ELINK_SPEED_100) ||
 	      (phy->req_line_speed == ELINK_SPEED_10)))) {
 		vars->phy_flags |= PHY_SGMII_FLAG;
-		PMD_DRV_LOG(DEBUG, "Setting SGMII mode");
+		ELINK_DEBUG_P0(sc, "Setting SGMII mode");
 		elink_warpcore_clear_regs(phy, params, lane);
 		elink_warpcore_set_sgmii_speed(phy, params, 0, 1);
 	} else {
@@ -4175,27 +5384,28 @@ static uint8_t elink_warpcore_config_init(struct elink_phy *phy,
 			if (params->loopback_mode != ELINK_LOOPBACK_EXT)
 				elink_warpcore_enable_AN_KR(phy, params, vars);
 			else {
-				PMD_DRV_LOG(DEBUG, "Setting KR 10G-Force");
-				elink_warpcore_set_10G_KR(phy, params);
+				ELINK_DEBUG_P0(sc, "Setting KR 10G-Force");
+				elink_warpcore_set_10G_KR(phy, params, vars);
 			}
 			break;
 
 		case PORT_HW_CFG_NET_SERDES_IF_XFI:
 			elink_warpcore_clear_regs(phy, params, lane);
 			if (vars->line_speed == ELINK_SPEED_10000) {
-				PMD_DRV_LOG(DEBUG, "Setting 10G XFI");
+				ELINK_DEBUG_P0(sc, "Setting 10G XFI");
 				elink_warpcore_set_10G_XFI(phy, params, 1);
 			} else {
 				if (ELINK_SINGLE_MEDIA_DIRECT(params)) {
-					PMD_DRV_LOG(DEBUG, "1G Fiber");
+					ELINK_DEBUG_P0(sc, "1G Fiber");
 					fiber_mode = 1;
 				} else {
-					PMD_DRV_LOG(DEBUG, "10/100/1G SGMII");
+					ELINK_DEBUG_P0(sc, "10/100/1G SGMII");
 					fiber_mode = 0;
 				}
 				elink_warpcore_set_sgmii_speed(phy,
-							       params,
-							       fiber_mode, 0);
+								params,
+								fiber_mode,
+								0);
 			}
 
 			break;
@@ -4207,7 +5417,7 @@ static uint8_t elink_warpcore_config_init(struct elink_phy *phy,
 			 */
 			if ((params->loopback_mode == ELINK_LOOPBACK_NONE) ||
 			    (params->loopback_mode == ELINK_LOOPBACK_EXT)) {
-				if (elink_is_sfp_module_plugged(params))
+				if (elink_is_sfp_module_plugged(phy, params))
 					elink_sfp_module_detection(phy, params);
 				else
 					elink_sfp_e3_set_transmitter(params,
@@ -4219,10 +5429,10 @@ static uint8_t elink_warpcore_config_init(struct elink_phy *phy,
 
 		case PORT_HW_CFG_NET_SERDES_IF_DXGXS:
 			if (vars->line_speed != ELINK_SPEED_20000) {
-				PMD_DRV_LOG(DEBUG, "Speed not supported yet");
+				ELINK_DEBUG_P0(sc, "Speed not supported yet");
 				return 0;
 			}
-			PMD_DRV_LOG(DEBUG, "Setting 20G DXGXS");
+			ELINK_DEBUG_P0(sc, "Setting 20G DXGXS");
 			elink_warpcore_set_20G_DXGXS(sc, phy, lane);
 			/* Issue Module detection */
 
@@ -4232,21 +5442,21 @@ static uint8_t elink_warpcore_config_init(struct elink_phy *phy,
 			if (!params->loopback_mode) {
 				elink_warpcore_enable_AN_KR(phy, params, vars);
 			} else {
-				PMD_DRV_LOG(DEBUG, "Setting KR 20G-Force");
+				ELINK_DEBUG_P0(sc, "Setting KR 20G-Force");
 				elink_warpcore_set_20G_force_KR2(phy, params);
 			}
 			break;
 		default:
-			PMD_DRV_LOG(DEBUG,
-				    "Unsupported Serdes Net Interface 0x%x",
-				    serdes_net_if);
+			ELINK_DEBUG_P1(sc,
+			   "Unsupported Serdes Net Interface 0x%x",
+			   serdes_net_if);
 			return 0;
 		}
 	}
 
 	/* Take lane out of reset after configuration is finished */
 	elink_warpcore_reset_lane(sc, phy, 0);
-	PMD_DRV_LOG(DEBUG, "Exit config init");
+	ELINK_DEBUG_P0(sc, "Exit config init");
 
 	return 0;
 }
@@ -4275,11 +5485,12 @@ static void elink_warpcore_link_reset(struct elink_phy *phy,
 			  MDIO_AER_BLOCK_AER_REG, 0);
 	/* Enable 1G MDIO (1-copy) */
 	elink_cl45_read_and_write(sc, phy, MDIO_WC_DEVAD,
-				  MDIO_WC_REG_XGXSBLK0_XGXSCONTROL, ~0x10);
+				  MDIO_WC_REG_XGXSBLK0_XGXSCONTROL,
+				  ~0x10);
 
 	elink_cl45_read_and_write(sc, phy, MDIO_WC_DEVAD,
 				  MDIO_WC_REG_XGXSBLK1_LANECTRL2, 0xff00);
-	lane = elink_get_warpcore_lane(params);
+	lane = elink_get_warpcore_lane(phy, params);
 	/* Disable CL36 PCS Tx */
 	elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 			MDIO_WC_REG_XGXSBLK1_LANECTRL0, &val16);
@@ -4311,8 +5522,8 @@ static void elink_set_warpcore_loopback(struct elink_phy *phy,
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t val16;
 	uint32_t lane;
-	PMD_DRV_LOG(DEBUG, "Setting Warpcore loopback type %x, speed %d",
-		    params->loopback_mode, phy->req_line_speed);
+	ELINK_DEBUG_P2(sc, "Setting Warpcore loopback type %x, speed %d",
+		       params->loopback_mode, phy->req_line_speed);
 
 	if (phy->req_line_speed < ELINK_SPEED_10000 ||
 	    phy->supported & ELINK_SUPPORTED_20000baseKR2_Full) {
@@ -4326,14 +5537,15 @@ static void elink_set_warpcore_loopback(struct elink_phy *phy,
 					 MDIO_WC_REG_XGXSBLK0_XGXSCONTROL,
 					 0x10);
 		/* Set 1G loopback based on lane (1-copy) */
-		lane = elink_get_warpcore_lane(params);
+		lane = elink_get_warpcore_lane(phy, params);
 		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 				MDIO_WC_REG_XGXSBLK1_LANECTRL2, &val16);
 		val16 |= (1 << lane);
 		if (phy->flags & ELINK_FLAGS_WC_DUAL_MODE)
 			val16 |= (2 << lane);
 		elink_cl45_write(sc, phy, MDIO_WC_DEVAD,
-				 MDIO_WC_REG_XGXSBLK1_LANECTRL2, val16);
+				 MDIO_WC_REG_XGXSBLK1_LANECTRL2,
+				 val16);
 
 		/* Switch back to 4-copy registers */
 		elink_set_aer_mmd(params, phy);
@@ -4347,8 +5559,10 @@ static void elink_set_warpcore_loopback(struct elink_phy *phy,
 	}
 }
 
+
+
 static void elink_sync_link(struct elink_params *params,
-			    struct elink_vars *vars)
+			     struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t link_10g_plus;
@@ -4356,21 +5570,23 @@ static void elink_sync_link(struct elink_params *params,
 		vars->phy_flags |= PHY_PHYSICAL_LINK_FLAG;
 	vars->link_up = (vars->link_status & LINK_STATUS_LINK_UP);
 	if (vars->link_up) {
-		PMD_DRV_LOG(DEBUG, "phy link up");
+		ELINK_DEBUG_P0(sc, "phy link up");
+		ELINK_DEBUG_P1(sc, "link status = %x", vars->link_status);
 
 		vars->phy_link_up = 1;
 		vars->duplex = DUPLEX_FULL;
-		switch (vars->link_status & LINK_STATUS_SPEED_AND_DUPLEX_MASK) {
+		switch (vars->link_status &
+			LINK_STATUS_SPEED_AND_DUPLEX_MASK) {
 		case ELINK_LINK_10THD:
 			vars->duplex = DUPLEX_HALF;
-			/* Fall through */
+			/* Fall thru */
 		case ELINK_LINK_10TFD:
 			vars->line_speed = ELINK_SPEED_10;
 			break;
 
 		case ELINK_LINK_100TXHD:
 			vars->duplex = DUPLEX_HALF;
-			/* Fall through */
+			/* Fall thru */
 		case ELINK_LINK_100T4:
 		case ELINK_LINK_100TXFD:
 			vars->line_speed = ELINK_SPEED_100;
@@ -4378,14 +5594,14 @@ static void elink_sync_link(struct elink_params *params,
 
 		case ELINK_LINK_1000THD:
 			vars->duplex = DUPLEX_HALF;
-			/* Fall through */
+			/* Fall thru */
 		case ELINK_LINK_1000TFD:
 			vars->line_speed = ELINK_SPEED_1000;
 			break;
 
 		case ELINK_LINK_2500THD:
 			vars->duplex = DUPLEX_HALF;
-			/* Fall through */
+			/* Fall thru */
 		case ELINK_LINK_2500TFD:
 			vars->line_speed = ELINK_SPEED_2500;
 			break;
@@ -4417,7 +5633,8 @@ static void elink_sync_link(struct elink_params *params,
 			vars->phy_flags &= ~PHY_SGMII_FLAG;
 		}
 		if (vars->line_speed &&
-		    USES_WARPCORE(sc) && (vars->line_speed == ELINK_SPEED_1000))
+		    USES_WARPCORE(sc) &&
+		    (vars->line_speed == ELINK_SPEED_1000))
 			vars->phy_flags |= PHY_SGMII_FLAG;
 		/* Anything 10 and over uses the bmac */
 		link_10g_plus = (vars->line_speed >= ELINK_SPEED_10000);
@@ -4433,8 +5650,8 @@ static void elink_sync_link(struct elink_params *params,
 			else
 				vars->mac_type = ELINK_MAC_TYPE_EMAC;
 		}
-	} else {		/* Link down */
-		PMD_DRV_LOG(DEBUG, "phy link down");
+	} else { /* Link down */
+		ELINK_DEBUG_P0(sc, "phy link down");
 
 		vars->phy_link_up = 0;
 
@@ -4478,44 +5695,44 @@ void elink_link_status_update(struct elink_params *params,
 	elink_sync_link(params, vars);
 	/* Sync media type */
 	sync_offset = params->shmem_base +
-	    offsetof(struct shmem_region,
-		     dev_info.port_hw_config[port].media_type);
+			offsetof(struct shmem_region,
+				 dev_info.port_hw_config[port].media_type);
 	media_types = REG_RD(sc, sync_offset);
 
 	params->phy[ELINK_INT_PHY].media_type =
-	    (media_types & PORT_HW_CFG_MEDIA_TYPE_PHY0_MASK) >>
-	    PORT_HW_CFG_MEDIA_TYPE_PHY0_SHIFT;
+		(media_types & PORT_HW_CFG_MEDIA_TYPE_PHY0_MASK) >>
+		PORT_HW_CFG_MEDIA_TYPE_PHY0_SHIFT;
 	params->phy[ELINK_EXT_PHY1].media_type =
-	    (media_types & PORT_HW_CFG_MEDIA_TYPE_PHY1_MASK) >>
-	    PORT_HW_CFG_MEDIA_TYPE_PHY1_SHIFT;
+		(media_types & PORT_HW_CFG_MEDIA_TYPE_PHY1_MASK) >>
+		PORT_HW_CFG_MEDIA_TYPE_PHY1_SHIFT;
 	params->phy[ELINK_EXT_PHY2].media_type =
-	    (media_types & PORT_HW_CFG_MEDIA_TYPE_PHY2_MASK) >>
-	    PORT_HW_CFG_MEDIA_TYPE_PHY2_SHIFT;
-	PMD_DRV_LOG(DEBUG, "media_types = 0x%x", media_types);
+		(media_types & PORT_HW_CFG_MEDIA_TYPE_PHY2_MASK) >>
+		PORT_HW_CFG_MEDIA_TYPE_PHY2_SHIFT;
+	ELINK_DEBUG_P1(sc, "media_types = 0x%x", media_types);
 
 	/* Sync AEU offset */
 	sync_offset = params->shmem_base +
-	    offsetof(struct shmem_region,
-		     dev_info.port_hw_config[port].aeu_int_mask);
+			offsetof(struct shmem_region,
+				 dev_info.port_hw_config[port].aeu_int_mask);
 
 	vars->aeu_int_mask = REG_RD(sc, sync_offset);
 
 	/* Sync PFC status */
 	if (vars->link_status & LINK_STATUS_PFC_ENABLED)
 		params->feature_config_flags |=
-		    ELINK_FEATURE_CONFIG_PFC_ENABLED;
+					ELINK_FEATURE_CONFIG_PFC_ENABLED;
 	else
 		params->feature_config_flags &=
-		    ~ELINK_FEATURE_CONFIG_PFC_ENABLED;
+					~ELINK_FEATURE_CONFIG_PFC_ENABLED;
 
 	if (SHMEM2_HAS(sc, link_attr_sync))
-		vars->link_attr_sync = SHMEM2_RD(sc,
+		params->link_attr_sync = SHMEM2_RD(sc,
 						 link_attr_sync[params->port]);
 
-	PMD_DRV_LOG(DEBUG, "link_status 0x%x  phy_link_up %x int_mask 0x%x",
-		    vars->link_status, vars->phy_link_up, vars->aeu_int_mask);
-	PMD_DRV_LOG(DEBUG, "line_speed %x  duplex %x  flow_ctrl 0x%x",
-		    vars->line_speed, vars->duplex, vars->flow_ctrl);
+	ELINK_DEBUG_P3(sc, "link_status 0x%x  phy_link_up %x int_mask 0x%x",
+		 vars->link_status, vars->phy_link_up, vars->aeu_int_mask);
+	ELINK_DEBUG_P3(sc, "line_speed %x  duplex %x  flow_ctrl 0x%x",
+		 vars->line_speed, vars->duplex, vars->flow_ctrl);
 }
 
 static void elink_set_master_ln(struct elink_params *params,
@@ -4530,7 +5747,8 @@ static void elink_set_master_ln(struct elink_params *params,
 	/* Set the master_ln for AN */
 	CL22_RD_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_XGXS_BLOCK2,
-			  MDIO_XGXS_BLOCK2_TEST_MODE_LANE, &new_master_ln);
+			  MDIO_XGXS_BLOCK2_TEST_MODE_LANE,
+			  &new_master_ln);
 
 	CL22_WR_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_XGXS_BLOCK2,
@@ -4539,8 +5757,8 @@ static void elink_set_master_ln(struct elink_params *params,
 }
 
 static elink_status_t elink_reset_unicore(struct elink_params *params,
-					  struct elink_phy *phy,
-					  uint8_t set_serdes)
+			       struct elink_phy *phy,
+			       uint8_t set_serdes)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t mii_control;
@@ -4553,7 +5771,8 @@ static elink_status_t elink_reset_unicore(struct elink_params *params,
 	CL22_WR_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_COMBO_IEEE0,
 			  MDIO_COMBO_IEEE0_MII_CONTROL,
-			  (mii_control | MDIO_COMBO_IEEO_MII_CONTROL_RESET));
+			  (mii_control |
+			   MDIO_COMBO_IEEO_MII_CONTROL_RESET));
 	if (set_serdes)
 		elink_set_serdes_access(sc, params->port);
 
@@ -4564,7 +5783,8 @@ static elink_status_t elink_reset_unicore(struct elink_params *params,
 		/* The reset erased the previous bank value */
 		CL22_RD_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_COMBO_IEEE0,
-				  MDIO_COMBO_IEEE0_MII_CONTROL, &mii_control);
+				  MDIO_COMBO_IEEE0_MII_CONTROL,
+				  &mii_control);
 
 		if (!(mii_control & MDIO_COMBO_IEEO_MII_CONTROL_RESET)) {
 			DELAY(5);
@@ -4572,10 +5792,12 @@ static elink_status_t elink_reset_unicore(struct elink_params *params,
 		}
 	}
 
-	elink_cb_event_log(sc, ELINK_LOG_ID_PHY_UNINITIALIZED, params->port);	// "Warning: PHY was not initialized,"
-	// " Port %d",
+	elink_cb_event_log(sc, ELINK_LOG_ID_PHY_UNINITIALIZED, params->port);
+			     /* "Warning: PHY was not initialized,"
+			      * " Port %d",
+			      */
 
-	PMD_DRV_LOG(DEBUG, "BUG! XGXS is still in reset!");
+	ELINK_DEBUG_P0(sc, "BUG! XGXS is still in reset!");
 	return ELINK_STATUS_ERROR;
 
 }
@@ -4629,31 +5851,35 @@ static void elink_set_parallel_detection(struct elink_phy *phy,
 	uint16_t control2;
 	CL22_RD_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_SERDES_DIGITAL,
-			  MDIO_SERDES_DIGITAL_A_1000X_CONTROL2, &control2);
+			  MDIO_SERDES_DIGITAL_A_1000X_CONTROL2,
+			  &control2);
 	if (phy->speed_cap_mask & PORT_HW_CFG_SPEED_CAPABILITY_D0_1G)
 		control2 |= MDIO_SERDES_DIGITAL_A_1000X_CONTROL2_PRL_DT_EN;
 	else
 		control2 &= ~MDIO_SERDES_DIGITAL_A_1000X_CONTROL2_PRL_DT_EN;
-	PMD_DRV_LOG(DEBUG, "phy->speed_cap_mask = 0x%x, control2 = 0x%x",
-		    phy->speed_cap_mask, control2);
+	ELINK_DEBUG_P2(sc, "phy->speed_cap_mask = 0x%x, control2 = 0x%x",
+		phy->speed_cap_mask, control2);
 	CL22_WR_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_SERDES_DIGITAL,
-			  MDIO_SERDES_DIGITAL_A_1000X_CONTROL2, control2);
+			  MDIO_SERDES_DIGITAL_A_1000X_CONTROL2,
+			  control2);
 
 	if ((phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT) &&
-	    (phy->speed_cap_mask & PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)) {
-		PMD_DRV_LOG(DEBUG, "XGXS");
+	     (phy->speed_cap_mask &
+		    PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)) {
+		ELINK_DEBUG_P0(sc, "XGXS");
 
 		CL22_WR_OVER_CL45(sc, phy,
-				  MDIO_REG_BANK_10G_PARALLEL_DETECT,
-				  MDIO_10G_PARALLEL_DETECT_PAR_DET_10G_LINK,
-				  MDIO_10G_PARALLEL_DETECT_PAR_DET_10G_LINK_CNT);
+				 MDIO_REG_BANK_10G_PARALLEL_DETECT,
+				 MDIO_10G_PARALLEL_DETECT_PAR_DET_10G_LINK,
+				 MDIO_10G_PARALLEL_DETECT_PAR_DET_10G_LINK_CNT);
 
 		CL22_RD_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_10G_PARALLEL_DETECT,
 				  MDIO_10G_PARALLEL_DETECT_PAR_DET_10G_CONTROL,
 				  &control2);
 
+
 		control2 |=
 		    MDIO_10G_PARALLEL_DETECT_PAR_DET_10G_CONTROL_PARDET10G_EN;
 
@@ -4673,7 +5899,8 @@ static void elink_set_parallel_detection(struct elink_phy *phy,
 
 static void elink_set_autoneg(struct elink_phy *phy,
 			      struct elink_params *params,
-			      struct elink_vars *vars, uint8_t enable_cl73)
+			      struct elink_vars *vars,
+			      uint8_t enable_cl73)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t reg_val;
@@ -4686,7 +5913,7 @@ static void elink_set_autoneg(struct elink_phy *phy,
 	/* CL37 Autoneg Enabled */
 	if (vars->line_speed == ELINK_SPEED_AUTO_NEG)
 		reg_val |= MDIO_COMBO_IEEO_MII_CONTROL_AN_EN;
-	else			/* CL37 Autoneg Disabled */
+	else /* CL37 Autoneg Disabled */
 		reg_val &= ~(MDIO_COMBO_IEEO_MII_CONTROL_AN_EN |
 			     MDIO_COMBO_IEEO_MII_CONTROL_RESTART_AN);
 
@@ -4700,7 +5927,7 @@ static void elink_set_autoneg(struct elink_phy *phy,
 			  MDIO_REG_BANK_SERDES_DIGITAL,
 			  MDIO_SERDES_DIGITAL_A_1000X_CONTROL1, &reg_val);
 	reg_val &= ~(MDIO_SERDES_DIGITAL_A_1000X_CONTROL1_SIGNAL_DETECT_EN |
-		     MDIO_SERDES_DIGITAL_A_1000X_CONTROL1_INVERT_SIGNAL_DETECT);
+		    MDIO_SERDES_DIGITAL_A_1000X_CONTROL1_INVERT_SIGNAL_DETECT);
 	reg_val |= MDIO_SERDES_DIGITAL_A_1000X_CONTROL1_FIBER_MODE;
 	if (vars->line_speed == ELINK_SPEED_AUTO_NEG)
 		reg_val |= MDIO_SERDES_DIGITAL_A_1000X_CONTROL1_AUTODET;
@@ -4714,7 +5941,8 @@ static void elink_set_autoneg(struct elink_phy *phy,
 	/* Enable TetonII and BAM autoneg */
 	CL22_RD_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_BAM_NEXT_PAGE,
-			  MDIO_BAM_NEXT_PAGE_MP5_NEXT_PAGE_CTRL, &reg_val);
+			  MDIO_BAM_NEXT_PAGE_MP5_NEXT_PAGE_CTRL,
+			  &reg_val);
 	if (vars->line_speed == ELINK_SPEED_AUTO_NEG) {
 		/* Enable BAM aneg Mode and TetonII aneg Mode */
 		reg_val |= (MDIO_BAM_NEXT_PAGE_MP5_NEXT_PAGE_CTRL_BAM_MODE |
@@ -4726,40 +5954,45 @@ static void elink_set_autoneg(struct elink_phy *phy,
 	}
 	CL22_WR_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_BAM_NEXT_PAGE,
-			  MDIO_BAM_NEXT_PAGE_MP5_NEXT_PAGE_CTRL, reg_val);
+			  MDIO_BAM_NEXT_PAGE_MP5_NEXT_PAGE_CTRL,
+			  reg_val);
 
 	if (enable_cl73) {
 		/* Enable Cl73 FSM status bits */
 		CL22_WR_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_CL73_USERB0,
-				  MDIO_CL73_USERB0_CL73_UCTRL, 0xe);
+				  MDIO_CL73_USERB0_CL73_UCTRL,
+				  0xe);
 
-		/* Enable BAM Station Manager */
+		/* Enable BAM Station Manager*/
 		CL22_WR_OVER_CL45(sc, phy,
-				  MDIO_REG_BANK_CL73_USERB0,
-				  MDIO_CL73_USERB0_CL73_BAM_CTRL1,
-				  MDIO_CL73_USERB0_CL73_BAM_CTRL1_BAM_EN |
-				  MDIO_CL73_USERB0_CL73_BAM_CTRL1_BAM_STATION_MNGR_EN
-				  |
-				  MDIO_CL73_USERB0_CL73_BAM_CTRL1_BAM_NP_AFTER_BP_EN);
+			MDIO_REG_BANK_CL73_USERB0,
+			MDIO_CL73_USERB0_CL73_BAM_CTRL1,
+			MDIO_CL73_USERB0_CL73_BAM_CTRL1_BAM_EN |
+			MDIO_CL73_USERB0_CL73_BAM_CTRL1_BAM_STATION_MNGR_EN |
+			MDIO_CL73_USERB0_CL73_BAM_CTRL1_BAM_NP_AFTER_BP_EN);
 
 		/* Advertise CL73 link speeds */
 		CL22_RD_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_CL73_IEEEB1,
-				  MDIO_CL73_IEEEB1_AN_ADV2, &reg_val);
-		if (phy->speed_cap_mask & PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)
+				  MDIO_CL73_IEEEB1_AN_ADV2,
+				  &reg_val);
+		if (phy->speed_cap_mask &
+		    PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)
 			reg_val |= MDIO_CL73_IEEEB1_AN_ADV2_ADVR_10G_KX4;
-		if (phy->speed_cap_mask & PORT_HW_CFG_SPEED_CAPABILITY_D0_1G)
+		if (phy->speed_cap_mask &
+		    PORT_HW_CFG_SPEED_CAPABILITY_D0_1G)
 			reg_val |= MDIO_CL73_IEEEB1_AN_ADV2_ADVR_1000M_KX;
 
 		CL22_WR_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_CL73_IEEEB1,
-				  MDIO_CL73_IEEEB1_AN_ADV2, reg_val);
+				  MDIO_CL73_IEEEB1_AN_ADV2,
+				  reg_val);
 
 		/* CL73 Autoneg Enabled */
 		reg_val = MDIO_CL73_IEEEB0_CL73_AN_CONTROL_AN_EN;
 
-	} else			/* CL73 Autoneg Disabled */
+	} else /* CL73 Autoneg Disabled */
 		reg_val = 0;
 
 	CL22_WR_OVER_CL45(sc, phy,
@@ -4775,7 +6008,7 @@ static void elink_program_serdes(struct elink_phy *phy,
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t reg_val;
 
-	/* Program duplex, disable autoneg and sgmii */
+	/* Program duplex, disable autoneg and sgmii*/
 	CL22_RD_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_COMBO_IEEE0,
 			  MDIO_COMBO_IEEE0_MII_CONTROL, &reg_val);
@@ -4795,7 +6028,7 @@ static void elink_program_serdes(struct elink_phy *phy,
 			  MDIO_REG_BANK_SERDES_DIGITAL,
 			  MDIO_SERDES_DIGITAL_MISC1, &reg_val);
 	/* Clearing the speed value before setting the right speed */
-	PMD_DRV_LOG(DEBUG, "MDIO_REG_BANK_SERDES_DIGITAL = 0x%x", reg_val);
+	ELINK_DEBUG_P1(sc, "MDIO_REG_BANK_SERDES_DIGITAL = 0x%x", reg_val);
 
 	reg_val &= ~(MDIO_SERDES_DIGITAL_MISC1_FORCE_SPEED_MASK |
 		     MDIO_SERDES_DIGITAL_MISC1_FORCE_SPEED_SEL);
@@ -4808,7 +6041,7 @@ static void elink_program_serdes(struct elink_phy *phy,
 			    MDIO_SERDES_DIGITAL_MISC1_FORCE_SPEED_SEL);
 		if (vars->line_speed == ELINK_SPEED_10000)
 			reg_val |=
-			    MDIO_SERDES_DIGITAL_MISC1_FORCE_SPEED_10G_CX4;
+				MDIO_SERDES_DIGITAL_MISC1_FORCE_SPEED_10G_CX4;
 	}
 
 	CL22_WR_OVER_CL45(sc, phy,
@@ -4829,10 +6062,12 @@ static void elink_set_brcm_cl37_advertisement(struct elink_phy *phy,
 	if (phy->speed_cap_mask & PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)
 		val |= MDIO_OVER_1G_UP1_10G;
 	CL22_WR_OVER_CL45(sc, phy,
-			  MDIO_REG_BANK_OVER_1G, MDIO_OVER_1G_UP1, val);
+			  MDIO_REG_BANK_OVER_1G,
+			  MDIO_OVER_1G_UP1, val);
 
 	CL22_WR_OVER_CL45(sc, phy,
-			  MDIO_REG_BANK_OVER_1G, MDIO_OVER_1G_UP3, 0x400);
+			  MDIO_REG_BANK_OVER_1G,
+			  MDIO_OVER_1G_UP3, 0x400);
 }
 
 static void elink_set_ieee_aneg_advertisement(struct elink_phy *phy,
@@ -4863,7 +6098,7 @@ static void elink_restart_autoneg(struct elink_phy *phy,
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t mii_control;
 
-	PMD_DRV_LOG(DEBUG, "elink_restart_autoneg");
+	ELINK_DEBUG_P0(sc, "elink_restart_autoneg");
 	/* Enable and restart BAM/CL37 aneg */
 
 	if (enable_cl73) {
@@ -4876,16 +6111,17 @@ static void elink_restart_autoneg(struct elink_phy *phy,
 				  MDIO_REG_BANK_CL73_IEEEB0,
 				  MDIO_CL73_IEEEB0_CL73_AN_CONTROL,
 				  (mii_control |
-				   MDIO_CL73_IEEEB0_CL73_AN_CONTROL_AN_EN |
-				   MDIO_CL73_IEEEB0_CL73_AN_CONTROL_RESTART_AN));
+				  MDIO_CL73_IEEEB0_CL73_AN_CONTROL_AN_EN |
+				  MDIO_CL73_IEEEB0_CL73_AN_CONTROL_RESTART_AN));
 	} else {
 
 		CL22_RD_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_COMBO_IEEE0,
-				  MDIO_COMBO_IEEE0_MII_CONTROL, &mii_control);
-		PMD_DRV_LOG(DEBUG,
-			    "elink_restart_autoneg mii_control before = 0x%x",
-			    mii_control);
+				  MDIO_COMBO_IEEE0_MII_CONTROL,
+				  &mii_control);
+		ELINK_DEBUG_P1(sc,
+			 "elink_restart_autoneg mii_control before = 0x%x",
+			 mii_control);
 		CL22_WR_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_COMBO_IEEE0,
 				  MDIO_COMBO_IEEE0_MII_CONTROL,
@@ -4906,7 +6142,8 @@ static void elink_initialize_sgmii_process(struct elink_phy *phy,
 
 	CL22_RD_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_SERDES_DIGITAL,
-			  MDIO_SERDES_DIGITAL_A_1000X_CONTROL1, &control1);
+			  MDIO_SERDES_DIGITAL_A_1000X_CONTROL1,
+			  &control1);
 	control1 |= MDIO_SERDES_DIGITAL_A_1000X_CONTROL1_INVERT_SIGNAL_DETECT;
 	/* Set sgmii mode (and not fiber) */
 	control1 &= ~(MDIO_SERDES_DIGITAL_A_1000X_CONTROL1_FIBER_MODE |
@@ -4914,7 +6151,8 @@ static void elink_initialize_sgmii_process(struct elink_phy *phy,
 		      MDIO_SERDES_DIGITAL_A_1000X_CONTROL1_MSTR_MODE);
 	CL22_WR_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_SERDES_DIGITAL,
-			  MDIO_SERDES_DIGITAL_A_1000X_CONTROL1, control1);
+			  MDIO_SERDES_DIGITAL_A_1000X_CONTROL1,
+			  control1);
 
 	/* If forced speed */
 	if (!(vars->line_speed == ELINK_SPEED_AUTO_NEG)) {
@@ -4923,7 +6161,8 @@ static void elink_initialize_sgmii_process(struct elink_phy *phy,
 
 		CL22_RD_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_COMBO_IEEE0,
-				  MDIO_COMBO_IEEE0_MII_CONTROL, &mii_control);
+				  MDIO_COMBO_IEEE0_MII_CONTROL,
+				  &mii_control);
 		mii_control &= ~(MDIO_COMBO_IEEO_MII_CONTROL_AN_EN |
 				 MDIO_COMBO_IEEO_MII_CONTROL_MAN_SGMII_SP_MASK |
 				 MDIO_COMBO_IEEO_MII_CONTROL_FULL_DUPLEX);
@@ -4931,30 +6170,32 @@ static void elink_initialize_sgmii_process(struct elink_phy *phy,
 		switch (vars->line_speed) {
 		case ELINK_SPEED_100:
 			mii_control |=
-			    MDIO_COMBO_IEEO_MII_CONTROL_MAN_SGMII_SP_100;
+				MDIO_COMBO_IEEO_MII_CONTROL_MAN_SGMII_SP_100;
 			break;
 		case ELINK_SPEED_1000:
 			mii_control |=
-			    MDIO_COMBO_IEEO_MII_CONTROL_MAN_SGMII_SP_1000;
+				MDIO_COMBO_IEEO_MII_CONTROL_MAN_SGMII_SP_1000;
 			break;
 		case ELINK_SPEED_10:
 			/* There is nothing to set for 10M */
 			break;
 		default:
 			/* Invalid speed for SGMII */
-			PMD_DRV_LOG(DEBUG, "Invalid line_speed 0x%x",
-				    vars->line_speed);
+			ELINK_DEBUG_P1(sc, "Invalid line_speed 0x%x",
+				  vars->line_speed);
 			break;
 		}
 
 		/* Setting the full duplex */
 		if (phy->req_duplex == DUPLEX_FULL)
-			mii_control |= MDIO_COMBO_IEEO_MII_CONTROL_FULL_DUPLEX;
+			mii_control |=
+				MDIO_COMBO_IEEO_MII_CONTROL_FULL_DUPLEX;
 		CL22_WR_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_COMBO_IEEE0,
-				  MDIO_COMBO_IEEE0_MII_CONTROL, mii_control);
+				  MDIO_COMBO_IEEE0_MII_CONTROL,
+				  mii_control);
 
-	} else {		/* AN mode */
+	} else { /* AN mode */
 		/* Enable and restart AN */
 		elink_restart_autoneg(phy, params, 0);
 	}
@@ -4963,8 +6204,7 @@ static void elink_initialize_sgmii_process(struct elink_phy *phy,
 /* Link management
  */
 static elink_status_t elink_direct_parallel_detect_used(struct elink_phy *phy,
-							struct elink_params
-							*params)
+					     struct elink_params *params)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t pd_10g, status2_1000x;
@@ -4972,34 +6212,38 @@ static elink_status_t elink_direct_parallel_detect_used(struct elink_phy *phy,
 		return ELINK_STATUS_OK;
 	CL22_RD_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_SERDES_DIGITAL,
-			  MDIO_SERDES_DIGITAL_A_1000X_STATUS2, &status2_1000x);
+			  MDIO_SERDES_DIGITAL_A_1000X_STATUS2,
+			  &status2_1000x);
 	CL22_RD_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_SERDES_DIGITAL,
-			  MDIO_SERDES_DIGITAL_A_1000X_STATUS2, &status2_1000x);
+			  MDIO_SERDES_DIGITAL_A_1000X_STATUS2,
+			  &status2_1000x);
 	if (status2_1000x & MDIO_SERDES_DIGITAL_A_1000X_STATUS2_AN_DISABLED) {
-		PMD_DRV_LOG(DEBUG, "1G parallel detect link on port %d",
-			    params->port);
-		return ELINK_STATUS_ERROR;
+		ELINK_DEBUG_P1(sc, "1G parallel detect link on port %d",
+			 params->port);
+		return 1;
 	}
 
 	CL22_RD_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_10G_PARALLEL_DETECT,
-			  MDIO_10G_PARALLEL_DETECT_PAR_DET_10G_STATUS, &pd_10g);
+			  MDIO_10G_PARALLEL_DETECT_PAR_DET_10G_STATUS,
+			  &pd_10g);
 
 	if (pd_10g & MDIO_10G_PARALLEL_DETECT_PAR_DET_10G_STATUS_PD_LINK) {
-		PMD_DRV_LOG(DEBUG, "10G parallel detect link on port %d",
-			    params->port);
-		return ELINK_STATUS_ERROR;
+		ELINK_DEBUG_P1(sc, "10G parallel detect link on port %d",
+			 params->port);
+		return 1;
 	}
 	return ELINK_STATUS_OK;
 }
 
 static void elink_update_adv_fc(struct elink_phy *phy,
 				struct elink_params *params,
-				struct elink_vars *vars, uint32_t gp_status)
+				struct elink_vars *vars,
+				uint32_t gp_status)
 {
-	uint16_t ld_pause;	/* local driver */
-	uint16_t lp_pause;	/* link partner */
+	uint16_t ld_pause;   /* local driver */
+	uint16_t lp_pause;   /* link partner */
 	uint16_t pause_result;
 	struct bnx2x_softc *sc = params->sc;
 	if ((gp_status &
@@ -5010,37 +6254,42 @@ static void elink_update_adv_fc(struct elink_phy *phy,
 
 		CL22_RD_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_CL73_IEEEB1,
-				  MDIO_CL73_IEEEB1_AN_ADV1, &ld_pause);
+				  MDIO_CL73_IEEEB1_AN_ADV1,
+				  &ld_pause);
 		CL22_RD_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_CL73_IEEEB1,
-				  MDIO_CL73_IEEEB1_AN_LP_ADV1, &lp_pause);
+				  MDIO_CL73_IEEEB1_AN_LP_ADV1,
+				  &lp_pause);
 		pause_result = (ld_pause &
 				MDIO_CL73_IEEEB1_AN_ADV1_PAUSE_MASK) >> 8;
 		pause_result |= (lp_pause &
 				 MDIO_CL73_IEEEB1_AN_LP_ADV1_PAUSE_MASK) >> 10;
-		PMD_DRV_LOG(DEBUG, "pause_result CL73 0x%x", pause_result);
+		ELINK_DEBUG_P1(sc, "pause_result CL73 0x%x", pause_result);
 	} else {
 		CL22_RD_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_COMBO_IEEE0,
-				  MDIO_COMBO_IEEE0_AUTO_NEG_ADV, &ld_pause);
+				  MDIO_COMBO_IEEE0_AUTO_NEG_ADV,
+				  &ld_pause);
 		CL22_RD_OVER_CL45(sc, phy,
-				  MDIO_REG_BANK_COMBO_IEEE0,
-				  MDIO_COMBO_IEEE0_AUTO_NEG_LINK_PARTNER_ABILITY1,
-				  &lp_pause);
+			MDIO_REG_BANK_COMBO_IEEE0,
+			MDIO_COMBO_IEEE0_AUTO_NEG_LINK_PARTNER_ABILITY1,
+			&lp_pause);
 		pause_result = (ld_pause &
 				MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_MASK) >> 5;
 		pause_result |= (lp_pause &
 				 MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_MASK) >> 7;
-		PMD_DRV_LOG(DEBUG, "pause_result CL37 0x%x", pause_result);
+		ELINK_DEBUG_P1(sc, "pause_result CL37 0x%x", pause_result);
 	}
-	elink_pause_resolve(vars, pause_result);
+	elink_pause_resolve(phy, params, vars, pause_result);
 
 }
 
 static void elink_flow_ctrl_resolve(struct elink_phy *phy,
 				    struct elink_params *params,
-				    struct elink_vars *vars, uint32_t gp_status)
+				    struct elink_vars *vars,
+				    uint32_t gp_status)
 {
+	struct bnx2x_softc *sc = params->sc;
 	vars->flow_ctrl = ELINK_FLOW_CTRL_NONE;
 
 	/* Resolve from gp_status in case of AN complete and not sgmii */
@@ -5060,7 +6309,7 @@ static void elink_flow_ctrl_resolve(struct elink_phy *phy,
 		}
 		elink_update_adv_fc(phy, params, vars, gp_status);
 	}
-	PMD_DRV_LOG(DEBUG, "flow_ctrl 0x%x", vars->flow_ctrl);
+	ELINK_DEBUG_P1(sc, "flow_ctrl 0x%x", vars->flow_ctrl);
 }
 
 static void elink_check_fallback_to_cl37(struct elink_phy *phy,
@@ -5068,14 +6317,16 @@ static void elink_check_fallback_to_cl37(struct elink_phy *phy,
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t rx_status, ustat_val, cl37_fsm_received;
-	PMD_DRV_LOG(DEBUG, "elink_check_fallback_to_cl37");
+	ELINK_DEBUG_P0(sc, "elink_check_fallback_to_cl37");
 	/* Step 1: Make sure signal is detected */
 	CL22_RD_OVER_CL45(sc, phy,
-			  MDIO_REG_BANK_RX0, MDIO_RX0_RX_STATUS, &rx_status);
+			  MDIO_REG_BANK_RX0,
+			  MDIO_RX0_RX_STATUS,
+			  &rx_status);
 	if ((rx_status & MDIO_RX0_RX_STATUS_SIGDET) !=
 	    (MDIO_RX0_RX_STATUS_SIGDET)) {
-		PMD_DRV_LOG(DEBUG, "Signal is not detected. Restoring CL73."
-			    "rx_status(0x80b0) = 0x%x", rx_status);
+		ELINK_DEBUG_P1(sc, "Signal is not detected. Restoring CL73."
+			     "rx_status(0x80b0) = 0x%x", rx_status);
 		CL22_WR_OVER_CL45(sc, phy,
 				  MDIO_REG_BANK_CL73_IEEEB0,
 				  MDIO_CL73_IEEEB0_CL73_AN_CONTROL,
@@ -5085,14 +6336,15 @@ static void elink_check_fallback_to_cl37(struct elink_phy *phy,
 	/* Step 2: Check CL73 state machine */
 	CL22_RD_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_CL73_USERB0,
-			  MDIO_CL73_USERB0_CL73_USTAT1, &ustat_val);
+			  MDIO_CL73_USERB0_CL73_USTAT1,
+			  &ustat_val);
 	if ((ustat_val &
 	     (MDIO_CL73_USERB0_CL73_USTAT1_LINK_STATUS_CHECK |
 	      MDIO_CL73_USERB0_CL73_USTAT1_AN_GOOD_CHECK_BAM37)) !=
 	    (MDIO_CL73_USERB0_CL73_USTAT1_LINK_STATUS_CHECK |
-	     MDIO_CL73_USERB0_CL73_USTAT1_AN_GOOD_CHECK_BAM37)) {
-		PMD_DRV_LOG(DEBUG, "CL73 state-machine is not stable. "
-			    "ustat_val(0x8371) = 0x%x", ustat_val);
+	      MDIO_CL73_USERB0_CL73_USTAT1_AN_GOOD_CHECK_BAM37)) {
+		ELINK_DEBUG_P1(sc, "CL73 state-machine is not stable. "
+			     "ustat_val(0x8371) = 0x%x", ustat_val);
 		return;
 	}
 	/* Step 3: Check CL37 Message Pages received to indicate LP
@@ -5100,14 +6352,16 @@ static void elink_check_fallback_to_cl37(struct elink_phy *phy,
 	 */
 	CL22_RD_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_REMOTE_PHY,
-			  MDIO_REMOTE_PHY_MISC_RX_STATUS, &cl37_fsm_received);
+			  MDIO_REMOTE_PHY_MISC_RX_STATUS,
+			  &cl37_fsm_received);
 	if ((cl37_fsm_received &
 	     (MDIO_REMOTE_PHY_MISC_RX_STATUS_CL37_FSM_RECEIVED_OVER1G_MSG |
-	      MDIO_REMOTE_PHY_MISC_RX_STATUS_CL37_FSM_RECEIVED_BRCM_OUI_MSG)) !=
+	     MDIO_REMOTE_PHY_MISC_RX_STATUS_CL37_FSM_RECEIVED_BRCM_OUI_MSG)) !=
 	    (MDIO_REMOTE_PHY_MISC_RX_STATUS_CL37_FSM_RECEIVED_OVER1G_MSG |
-	     MDIO_REMOTE_PHY_MISC_RX_STATUS_CL37_FSM_RECEIVED_BRCM_OUI_MSG)) {
-		PMD_DRV_LOG(DEBUG, "No CL37 FSM were received. "
-			    "misc_rx_status(0x8330) = 0x%x", cl37_fsm_received);
+	      MDIO_REMOTE_PHY_MISC_RX_STATUS_CL37_FSM_RECEIVED_BRCM_OUI_MSG)) {
+		ELINK_DEBUG_P1(sc, "No CL37 FSM were received. "
+			     "misc_rx_status(0x8330) = 0x%x",
+			 cl37_fsm_received);
 		return;
 	}
 	/* The combined cl37/cl73 fsm state information indicating that
@@ -5119,34 +6373,38 @@ static void elink_check_fallback_to_cl37(struct elink_phy *phy,
 	/* Disable CL73 */
 	CL22_WR_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_CL73_IEEEB0,
-			  MDIO_CL73_IEEEB0_CL73_AN_CONTROL, 0);
+			  MDIO_CL73_IEEEB0_CL73_AN_CONTROL,
+			  0);
 	/* Restart CL37 autoneg */
 	elink_restart_autoneg(phy, params, 0);
-	PMD_DRV_LOG(DEBUG, "Disabling CL73, and restarting CL37 autoneg");
+	ELINK_DEBUG_P0(sc, "Disabling CL73, and restarting CL37 autoneg");
 }
 
 static void elink_xgxs_an_resolve(struct elink_phy *phy,
 				  struct elink_params *params,
-				  struct elink_vars *vars, uint32_t gp_status)
+				  struct elink_vars *vars,
+				  uint32_t gp_status)
 {
 	if (gp_status & ELINK_MDIO_AN_CL73_OR_37_COMPLETE)
-		vars->link_status |= LINK_STATUS_AUTO_NEGOTIATE_COMPLETE;
+		vars->link_status |=
+			LINK_STATUS_AUTO_NEGOTIATE_COMPLETE;
 
 	if (elink_direct_parallel_detect_used(phy, params))
-		vars->link_status |= LINK_STATUS_PARALLEL_DETECTION_USED;
+		vars->link_status |=
+			LINK_STATUS_PARALLEL_DETECTION_USED;
 }
-
 static elink_status_t elink_get_link_speed_duplex(struct elink_phy *phy,
-						  struct elink_params *params __rte_unused,
-						  struct elink_vars *vars,
-						  uint16_t is_link_up,
-						  uint16_t speed_mask,
-						  uint16_t is_duplex)
+				     struct elink_params *params,
+				      struct elink_vars *vars,
+				      uint16_t is_link_up,
+				      uint16_t speed_mask,
+				      uint16_t is_duplex)
 {
+	struct bnx2x_softc *sc = params->sc;
 	if (phy->req_line_speed == ELINK_SPEED_AUTO_NEG)
 		vars->link_status |= LINK_STATUS_AUTO_NEGOTIATE_ENABLED;
 	if (is_link_up) {
-		PMD_DRV_LOG(DEBUG, "phy link up");
+		ELINK_DEBUG_P0(sc, "phy link up");
 
 		vars->phy_link_up = 1;
 		vars->link_status |= LINK_STATUS_LINK_UP;
@@ -5187,9 +6445,9 @@ static elink_status_t elink_get_link_speed_duplex(struct elink_phy *phy,
 
 		case ELINK_GP_STATUS_5G:
 		case ELINK_GP_STATUS_6G:
-			PMD_DRV_LOG(DEBUG,
-				    "link speed unsupported  gp_status 0x%x",
-				    speed_mask);
+			ELINK_DEBUG_P1(sc,
+				 "link speed unsupported  gp_status 0x%x",
+				  speed_mask);
 			return ELINK_STATUS_ERROR;
 
 		case ELINK_GP_STATUS_10G_KX4:
@@ -5207,13 +6465,13 @@ static elink_status_t elink_get_link_speed_duplex(struct elink_phy *phy,
 			vars->link_status |= ELINK_LINK_20GTFD;
 			break;
 		default:
-			PMD_DRV_LOG(DEBUG,
-				    "link speed unsupported gp_status 0x%x",
-				    speed_mask);
+			ELINK_DEBUG_P1(sc,
+				  "link speed unsupported gp_status 0x%x",
+				  speed_mask);
 			return ELINK_STATUS_ERROR;
 		}
-	} else {		/* link_down */
-		PMD_DRV_LOG(DEBUG, "phy link down");
+	} else { /* link_down */
+		ELINK_DEBUG_P0(sc, "phy link down");
 
 		vars->phy_link_up = 0;
 
@@ -5221,14 +6479,16 @@ static elink_status_t elink_get_link_speed_duplex(struct elink_phy *phy,
 		vars->flow_ctrl = ELINK_FLOW_CTRL_NONE;
 		vars->mac_type = ELINK_MAC_TYPE_NONE;
 	}
-	PMD_DRV_LOG(DEBUG, " phy_link_up %x line_speed %d",
+	ELINK_DEBUG_P2(sc, " in elink_get_link_speed_duplex vars->link_status = %x, vars->duplex = %x",
+			vars->link_status, vars->duplex);
+	ELINK_DEBUG_P2(sc, " phy_link_up %x line_speed %d",
 		    vars->phy_link_up, vars->line_speed);
 	return ELINK_STATUS_OK;
 }
 
 static uint8_t elink_link_settings_status(struct elink_phy *phy,
-					  struct elink_params *params,
-					  struct elink_vars *vars)
+				      struct elink_params *params,
+				      struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 
@@ -5238,14 +6498,23 @@ static uint8_t elink_link_settings_status(struct elink_phy *phy,
 	/* Read gp_status */
 	CL22_RD_OVER_CL45(sc, phy,
 			  MDIO_REG_BANK_GP_STATUS,
-			  MDIO_GP_STATUS_TOP_AN_STATUS1, &gp_status);
-	if (gp_status & MDIO_GP_STATUS_TOP_AN_STATUS1_DUPLEX_STATUS)
+			  MDIO_GP_STATUS_TOP_AN_STATUS1,
+			  &gp_status);
+	if (gp_status & MDIO_GP_STATUS_TOP_AN_STATUS1_DUPLEX_STATUS) {
 		duplex = DUPLEX_FULL;
+		ELINK_DEBUG_P1(sc, "duplex status read from phy is = %x",
+				duplex);
+	} else {
+		ELINK_DEBUG_P1(sc, "phy status does not allow interface to be FULL_DUPLEX : %x",
+			gp_status);
+	}
+
+
 	if (gp_status & MDIO_GP_STATUS_TOP_AN_STATUS1_LINK_STATUS)
 		link_up = 1;
 	speed_mask = gp_status & ELINK_GP_STATUS_SPEED_MASK;
-	PMD_DRV_LOG(DEBUG, "gp_status 0x%x, is_link_up %d, speed_mask 0x%x",
-		    gp_status, link_up, speed_mask);
+	ELINK_DEBUG_P3(sc, "gp_status 0x%x, is_link_up %d, speed_mask 0x%x",
+		       gp_status, link_up, speed_mask);
 	rc = elink_get_link_speed_duplex(phy, params, vars, link_up, speed_mask,
 					 duplex);
 	if (rc == ELINK_STATUS_ERROR)
@@ -5259,7 +6528,7 @@ static uint8_t elink_link_settings_status(struct elink_phy *phy,
 				elink_xgxs_an_resolve(phy, params, vars,
 						      gp_status);
 		}
-	} else {		/* Link_down */
+	} else { /* Link_down */
 		if ((phy->req_line_speed == ELINK_SPEED_AUTO_NEG) &&
 		    ELINK_SINGLE_MEDIA_DIRECT(params)) {
 			/* Check signal is detected */
@@ -5267,7 +6536,7 @@ static uint8_t elink_link_settings_status(struct elink_phy *phy,
 		}
 	}
 
-	/* Read LP advertised speeds */
+	/* Read LP advertised speeds*/
 	if (ELINK_SINGLE_MEDIA_DIRECT(params) &&
 	    (vars->link_status & LINK_STATUS_AUTO_NEGOTIATE_COMPLETE)) {
 		uint16_t val;
@@ -5277,61 +6546,69 @@ static uint8_t elink_link_settings_status(struct elink_phy *phy,
 
 		if (val & MDIO_CL73_IEEEB1_AN_ADV2_ADVR_1000M_KX)
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE;
 		if (val & (MDIO_CL73_IEEEB1_AN_ADV2_ADVR_10G_KX4 |
 			   MDIO_CL73_IEEEB1_AN_ADV2_ADVR_10G_KR))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
 
 		CL22_RD_OVER_CL45(sc, phy, MDIO_REG_BANK_OVER_1G,
 				  MDIO_OVER_1G_LP_UP1, &val);
 
 		if (val & MDIO_OVER_1G_UP1_2_5G)
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_2500XFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_2500XFD_CAPABLE;
 		if (val & (MDIO_OVER_1G_UP1_10G | MDIO_OVER_1G_UP1_10GH))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
 	}
 
-	PMD_DRV_LOG(DEBUG, "duplex %x  flow_ctrl 0x%x link_status 0x%x",
-		    vars->duplex, vars->flow_ctrl, vars->link_status);
+	ELINK_DEBUG_P3(sc, "duplex %x  flow_ctrl 0x%x link_status 0x%x",
+		   vars->duplex, vars->flow_ctrl, vars->link_status);
 	return rc;
 }
 
 static uint8_t elink_warpcore_read_status(struct elink_phy *phy,
-					  struct elink_params *params,
-					  struct elink_vars *vars)
+				     struct elink_params *params,
+				     struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t lane;
 	uint16_t gp_status1, gp_speed, link_up, duplex = DUPLEX_FULL;
 	elink_status_t rc = ELINK_STATUS_OK;
-	lane = elink_get_warpcore_lane(params);
+	lane = elink_get_warpcore_lane(phy, params);
 	/* Read gp_status */
-	if ((params->loopback_mode) && (phy->flags & ELINK_FLAGS_WC_DUAL_MODE)) {
+	if ((params->loopback_mode) &&
+	    (phy->flags & ELINK_FLAGS_WC_DUAL_MODE)) {
 		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 				MDIO_WC_REG_DIGITAL5_LINK_STATUS, &link_up);
 		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 				MDIO_WC_REG_DIGITAL5_LINK_STATUS, &link_up);
 		link_up &= 0x1;
+		ELINK_DEBUG_P1(sc, "params->loopback_mode link_up read = %x",
+				link_up);
 	} else if ((phy->req_line_speed > ELINK_SPEED_10000) &&
-		   (phy->supported & ELINK_SUPPORTED_20000baseMLD2_Full)) {
+		(phy->supported & ELINK_SUPPORTED_20000baseMLD2_Full)) {
 		uint16_t temp_link_up;
-		elink_cl45_read(sc, phy, MDIO_WC_DEVAD, 1, &temp_link_up);
-		elink_cl45_read(sc, phy, MDIO_WC_DEVAD, 1, &link_up);
-		PMD_DRV_LOG(DEBUG, "PCS RX link status = 0x%x-->0x%x",
-			    temp_link_up, link_up);
+		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
+				1, &temp_link_up);
+		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
+				1, &link_up);
+		ELINK_DEBUG_P2(sc, "PCS RX link status = 0x%x-->0x%x",
+			       temp_link_up, link_up);
 		link_up &= (1 << 2);
 		if (link_up)
 			elink_ext_phy_resolve_fc(phy, params, vars);
 	} else {
 		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
-				MDIO_WC_REG_GP2_STATUS_GP_2_1, &gp_status1);
-		PMD_DRV_LOG(DEBUG, "0x81d1 = 0x%x", gp_status1);
+				MDIO_WC_REG_GP2_STATUS_GP_2_1,
+				&gp_status1);
+		ELINK_DEBUG_P1(sc, "0x81d1 = 0x%x", gp_status1);
 		/* Check for either KR, 1G, or AN up. */
 		link_up = ((gp_status1 >> 8) |
-			   (gp_status1 >> 12) | (gp_status1)) & (1 << lane);
+			   (gp_status1 >> 12) |
+			   (gp_status1)) &
+			(1 << lane);
 		if (phy->supported & ELINK_SUPPORTED_20000baseKR2_Full) {
 			uint16_t an_link;
 			elink_cl45_read(sc, phy, MDIO_AN_DEVAD,
@@ -5339,6 +6616,8 @@ static uint8_t elink_warpcore_read_status(struct elink_phy *phy,
 			elink_cl45_read(sc, phy, MDIO_AN_DEVAD,
 					MDIO_AN_REG_STATUS, &an_link);
 			link_up |= (an_link & (1 << 2));
+			ELINK_DEBUG_P2(sc, "an_link = %x, link_up = %x",
+					an_link, link_up);
 		}
 		if (link_up && ELINK_SINGLE_MEDIA_DIRECT(params)) {
 			uint16_t pd, gp_status4;
@@ -5349,7 +6628,7 @@ static uint8_t elink_warpcore_read_status(struct elink_phy *phy,
 						&gp_status4);
 				if (gp_status4 & ((1 << 12) << lane))
 					vars->link_status |=
-					    LINK_STATUS_AUTO_NEGOTIATE_COMPLETE;
+					LINK_STATUS_AUTO_NEGOTIATE_COMPLETE;
 
 				/* Check parallel detect used */
 				elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
@@ -5357,13 +6636,19 @@ static uint8_t elink_warpcore_read_status(struct elink_phy *phy,
 						&pd);
 				if (pd & (1 << 15))
 					vars->link_status |=
-					    LINK_STATUS_PARALLEL_DETECTION_USED;
+					LINK_STATUS_PARALLEL_DETECTION_USED;
+				ELINK_DEBUG_P2(sc, "pd = %x, link_status = %x",
+						pd, vars->link_status);
 			}
 			elink_ext_phy_resolve_fc(phy, params, vars);
 			vars->duplex = duplex;
+			ELINK_DEBUG_P3(sc, " ELINK_SINGLE_MEDIA_DIRECT duplex %x  flow_ctrl 0x%x link_status 0x%x",
+					vars->duplex, vars->flow_ctrl,
+					vars->link_status);
 		}
 	}
-
+	ELINK_DEBUG_P3(sc, "duplex %x  flow_ctrl 0x%x link_status 0x%x",
+			vars->duplex, vars->flow_ctrl, vars->link_status);
 	if ((vars->link_status & LINK_STATUS_AUTO_NEGOTIATE_COMPLETE) &&
 	    ELINK_SINGLE_MEDIA_DIRECT(params)) {
 		uint16_t val;
@@ -5373,24 +6658,28 @@ static uint8_t elink_warpcore_read_status(struct elink_phy *phy,
 
 		if (val & MDIO_CL73_IEEEB1_AN_ADV2_ADVR_1000M_KX)
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE;
 		if (val & (MDIO_CL73_IEEEB1_AN_ADV2_ADVR_10G_KX4 |
 			   MDIO_CL73_IEEEB1_AN_ADV2_ADVR_10G_KR))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
-
+				LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
+		ELINK_DEBUG_P2(sc, "val = %x, link_status = %x",
+				val, vars->link_status);
 		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 				MDIO_WC_REG_DIGITAL3_LP_UP1, &val);
 
 		if (val & MDIO_OVER_1G_UP1_2_5G)
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_2500XFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_2500XFD_CAPABLE;
 		if (val & (MDIO_OVER_1G_UP1_10G | MDIO_OVER_1G_UP1_10GH))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
+		ELINK_DEBUG_P2(sc, "val = %x, link_status = %x",
+				val, vars->link_status);
 
 	}
 
+
 	if (lane < 2) {
 		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 				MDIO_WC_REG_GP2_STATUS_GP_2_2, &gp_speed);
@@ -5398,12 +6687,12 @@ static uint8_t elink_warpcore_read_status(struct elink_phy *phy,
 		elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 				MDIO_WC_REG_GP2_STATUS_GP_2_3, &gp_speed);
 	}
-	PMD_DRV_LOG(DEBUG, "lane %d gp_speed 0x%x", lane, gp_speed);
+	ELINK_DEBUG_P2(sc, "lane %d gp_speed 0x%x", lane, gp_speed);
 
 	if ((lane & 1) == 0)
 		gp_speed <<= 8;
 	gp_speed &= 0x3f00;
-	link_up = ! !link_up;
+	link_up = !!link_up;
 
 	/* Reset the TX FIFO to fix SGMII issue */
 	rc = elink_get_link_speed_duplex(phy, params, vars, link_up, gp_speed,
@@ -5414,11 +6703,10 @@ static uint8_t elink_warpcore_read_status(struct elink_phy *phy,
 	    (!(phy->flags & ELINK_FLAGS_WC_DUAL_MODE)))
 		vars->rx_tx_asic_rst = MAX_KR_LINK_RETRY;
 
-	PMD_DRV_LOG(DEBUG, "duplex %x  flow_ctrl 0x%x link_status 0x%x",
-		    vars->duplex, vars->flow_ctrl, vars->link_status);
+	ELINK_DEBUG_P3(sc, "duplex %x  flow_ctrl 0x%x link_status 0x%x",
+		   vars->duplex, vars->flow_ctrl, vars->link_status);
 	return rc;
 }
-
 static void elink_set_gmii_tx_driver(struct elink_params *params)
 {
 	struct bnx2x_softc *sc = params->sc;
@@ -5429,7 +6717,8 @@ static void elink_set_gmii_tx_driver(struct elink_params *params)
 
 	/* Read precomp */
 	CL22_RD_OVER_CL45(sc, phy,
-			  MDIO_REG_BANK_OVER_1G, MDIO_OVER_1G_LP_UP2, &lp_up2);
+			  MDIO_REG_BANK_OVER_1G,
+			  MDIO_OVER_1G_LP_UP2, &lp_up2);
 
 	/* Bits [10:7] at lp_up2, positioned at [15:12] */
 	lp_up2 = (((lp_up2 & MDIO_OVER_1G_LP_UP2_PREEMPHASIS_MASK) >>
@@ -5440,32 +6729,36 @@ static void elink_set_gmii_tx_driver(struct elink_params *params)
 		return;
 
 	for (bank = MDIO_REG_BANK_TX0; bank <= MDIO_REG_BANK_TX3;
-	     bank += (MDIO_REG_BANK_TX1 - MDIO_REG_BANK_TX0)) {
+	      bank += (MDIO_REG_BANK_TX1 - MDIO_REG_BANK_TX0)) {
 		CL22_RD_OVER_CL45(sc, phy,
-				  bank, MDIO_TX0_TX_DRIVER, &tx_driver);
+				  bank,
+				  MDIO_TX0_TX_DRIVER, &tx_driver);
 
 		/* Replace tx_driver bits [15:12] */
-		if (lp_up2 != (tx_driver & MDIO_TX0_TX_DRIVER_PREEMPHASIS_MASK)) {
+		if (lp_up2 !=
+		    (tx_driver & MDIO_TX0_TX_DRIVER_PREEMPHASIS_MASK)) {
 			tx_driver &= ~MDIO_TX0_TX_DRIVER_PREEMPHASIS_MASK;
 			tx_driver |= lp_up2;
 			CL22_WR_OVER_CL45(sc, phy,
-					  bank, MDIO_TX0_TX_DRIVER, tx_driver);
+					  bank,
+					  MDIO_TX0_TX_DRIVER, tx_driver);
 		}
 	}
 }
 
 static elink_status_t elink_emac_program(struct elink_params *params,
-					 struct elink_vars *vars)
+			      struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t port = params->port;
 	uint16_t mode = 0;
 
-	PMD_DRV_LOG(DEBUG, "setting link speed & duplex");
+	ELINK_DEBUG_P0(sc, "setting link speed & duplex");
 	elink_bits_dis(sc, GRCBASE_EMAC0 + port * 0x400 +
 		       EMAC_REG_EMAC_MODE,
 		       (EMAC_MODE_25G_MODE |
-			EMAC_MODE_PORT_MII_10M | EMAC_MODE_HALF_DUPLEX));
+			EMAC_MODE_PORT_MII_10M |
+			EMAC_MODE_HALF_DUPLEX));
 	switch (vars->line_speed) {
 	case ELINK_SPEED_10:
 		mode |= EMAC_MODE_PORT_MII_10M;
@@ -5485,14 +6778,16 @@ static elink_status_t elink_emac_program(struct elink_params *params,
 
 	default:
 		/* 10G not valid for EMAC */
-		PMD_DRV_LOG(DEBUG, "Invalid line_speed 0x%x", vars->line_speed);
+		ELINK_DEBUG_P1(sc, "Invalid line_speed 0x%x",
+			   vars->line_speed);
 		return ELINK_STATUS_ERROR;
 	}
 
 	if (vars->duplex == DUPLEX_HALF)
 		mode |= EMAC_MODE_HALF_DUPLEX;
 	elink_bits_en(sc,
-		      GRCBASE_EMAC0 + port * 0x400 + EMAC_REG_EMAC_MODE, mode);
+		      GRCBASE_EMAC0 + port * 0x400 + EMAC_REG_EMAC_MODE,
+		      mode);
 
 	elink_set_led(params, vars, ELINK_LED_MODE_OPER, vars->line_speed);
 	return ELINK_STATUS_OK;
@@ -5509,24 +6804,26 @@ static void elink_set_preemphasis(struct elink_phy *phy,
 	     bank += (MDIO_REG_BANK_RX1 - MDIO_REG_BANK_RX0), i++) {
 		CL22_WR_OVER_CL45(sc, phy,
 				  bank,
-				  MDIO_RX0_RX_EQ_BOOST, phy->rx_preemphasis[i]);
+				  MDIO_RX0_RX_EQ_BOOST,
+				  phy->rx_preemphasis[i]);
 	}
 
 	for (bank = MDIO_REG_BANK_TX0, i = 0; bank <= MDIO_REG_BANK_TX3;
 	     bank += (MDIO_REG_BANK_TX1 - MDIO_REG_BANK_TX0), i++) {
 		CL22_WR_OVER_CL45(sc, phy,
 				  bank,
-				  MDIO_TX0_TX_DRIVER, phy->tx_preemphasis[i]);
+				  MDIO_TX0_TX_DRIVER,
+				  phy->tx_preemphasis[i]);
 	}
 }
 
 static uint8_t elink_xgxs_config_init(struct elink_phy *phy,
-				      struct elink_params *params,
-				      struct elink_vars *vars)
+				   struct elink_params *params,
+				   struct elink_vars *vars)
 {
+	struct bnx2x_softc *sc = params->sc;
 	uint8_t enable_cl73 = (ELINK_SINGLE_MEDIA_DIRECT(params) ||
-			       (params->loopback_mode == ELINK_LOOPBACK_XGXS));
-
+			  (params->loopback_mode == ELINK_LOOPBACK_XGXS));
 	if (!(vars->phy_flags & PHY_SGMII_FLAG)) {
 		if (ELINK_SINGLE_MEDIA_DIRECT(params) &&
 		    (params->feature_config_flags &
@@ -5537,7 +6834,7 @@ static uint8_t elink_xgxs_config_init(struct elink_phy *phy,
 		if (vars->line_speed != ELINK_SPEED_AUTO_NEG ||
 		    (ELINK_SINGLE_MEDIA_DIRECT(params) &&
 		     params->loopback_mode == ELINK_LOOPBACK_EXT)) {
-			PMD_DRV_LOG(DEBUG, "not SGMII, no AN");
+			ELINK_DEBUG_P0(sc, "not SGMII, no AN");
 
 			/* Disable autoneg */
 			elink_set_autoneg(phy, params, vars, 0);
@@ -5545,8 +6842,8 @@ static uint8_t elink_xgxs_config_init(struct elink_phy *phy,
 			/* Program speed and duplex */
 			elink_program_serdes(phy, params, vars);
 
-		} else {	/* AN_mode */
-			PMD_DRV_LOG(DEBUG, "not SGMII, AN");
+		} else { /* AN_mode */
+			ELINK_DEBUG_P0(sc, "not SGMII, AN");
 
 			/* AN enabled */
 			elink_set_brcm_cl37_advertisement(phy, params);
@@ -5562,8 +6859,8 @@ static uint8_t elink_xgxs_config_init(struct elink_phy *phy,
 			elink_restart_autoneg(phy, params, enable_cl73);
 		}
 
-	} else {		/* SGMII mode */
-		PMD_DRV_LOG(DEBUG, "SGMII");
+	} else { /* SGMII mode */
+		ELINK_DEBUG_P0(sc, "SGMII");
 
 		elink_initialize_sgmii_process(phy, params, vars);
 	}
@@ -5572,8 +6869,8 @@ static uint8_t elink_xgxs_config_init(struct elink_phy *phy,
 }
 
 static elink_status_t elink_prepare_xgxs(struct elink_phy *phy,
-					 struct elink_params *params,
-					 struct elink_vars *vars)
+			  struct elink_params *params,
+			  struct elink_vars *vars)
 {
 	elink_status_t rc;
 	vars->phy_flags |= PHY_XGXS_FLAG;
@@ -5611,28 +6908,32 @@ static elink_status_t elink_prepare_xgxs(struct elink_phy *phy,
 }
 
 static uint16_t elink_wait_reset_complete(struct bnx2x_softc *sc,
-					  struct elink_phy *phy,
-					  struct elink_params *params)
+				     struct elink_phy *phy,
+				     struct elink_params *params)
 {
 	uint16_t cnt, ctrl;
 	/* Wait for soft reset to get cleared up to 1 sec */
 	for (cnt = 0; cnt < 1000; cnt++) {
-		if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X54618SE)
-			elink_cl22_read(sc, phy, MDIO_PMA_REG_CTRL, &ctrl);
+		if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY2_TYPE_BNX2X54618SE)
+			elink_cl22_read(sc, phy,
+				MDIO_PMA_REG_CTRL, &ctrl);
 		else
 			elink_cl45_read(sc, phy,
-					MDIO_PMA_DEVAD,
-					MDIO_PMA_REG_CTRL, &ctrl);
+				MDIO_PMA_DEVAD,
+				MDIO_PMA_REG_CTRL, &ctrl);
 		if (!(ctrl & (1 << 15)))
 			break;
 		DELAY(1000 * 1);
 	}
 
 	if (cnt == 1000)
-		elink_cb_event_log(sc, ELINK_LOG_ID_PHY_UNINITIALIZED, params->port);	// "Warning: PHY was not initialized,"
-	// " Port %d",
+		elink_cb_event_log(sc, ELINK_LOG_ID_PHY_UNINITIALIZED,
+				   params->port);
+				     /* "Warning: PHY was not initialized,"
+				      * " Port %d",
+				      */
 
-	PMD_DRV_LOG(DEBUG, "control reg 0x%x (after %d ms)", ctrl, cnt);
+	ELINK_DEBUG_P2(sc, "control reg 0x%x (after %d ms)", ctrl, cnt);
 	return cnt;
 }
 
@@ -5650,37 +6951,38 @@ static void elink_link_int_enable(struct elink_params *params)
 	} else if (params->switch_cfg == ELINK_SWITCH_CFG_10G) {
 		mask = (ELINK_NIG_MASK_XGXS0_LINK10G |
 			ELINK_NIG_MASK_XGXS0_LINK_STATUS);
-		PMD_DRV_LOG(DEBUG, "enabled XGXS interrupt");
+		ELINK_DEBUG_P0(sc, "enabled XGXS interrupt");
 		if (!(ELINK_SINGLE_MEDIA_DIRECT(params)) &&
-		    params->phy[ELINK_INT_PHY].type !=
-		    PORT_HW_CFG_XGXS_EXT_PHY_TYPE_FAILURE) {
+			params->phy[ELINK_INT_PHY].type !=
+				PORT_HW_CFG_XGXS_EXT_PHY_TYPE_FAILURE) {
 			mask |= ELINK_NIG_MASK_MI_INT;
-			PMD_DRV_LOG(DEBUG, "enabled external phy int");
+			ELINK_DEBUG_P0(sc, "enabled external phy int");
 		}
 
-	} else {		/* SerDes */
+	} else { /* SerDes */
 		mask = ELINK_NIG_MASK_SERDES0_LINK_STATUS;
-		PMD_DRV_LOG(DEBUG, "enabled SerDes interrupt");
+		ELINK_DEBUG_P0(sc, "enabled SerDes interrupt");
 		if (!(ELINK_SINGLE_MEDIA_DIRECT(params)) &&
-		    params->phy[ELINK_INT_PHY].type !=
-		    PORT_HW_CFG_SERDES_EXT_PHY_TYPE_NOT_CONN) {
+			params->phy[ELINK_INT_PHY].type !=
+				PORT_HW_CFG_SERDES_EXT_PHY_TYPE_NOT_CONN) {
 			mask |= ELINK_NIG_MASK_MI_INT;
-			PMD_DRV_LOG(DEBUG, "enabled external phy int");
+			ELINK_DEBUG_P0(sc, "enabled external phy int");
 		}
 	}
-	elink_bits_en(sc, NIG_REG_MASK_INTERRUPT_PORT0 + port * 4, mask);
+	elink_bits_en(sc,
+		      NIG_REG_MASK_INTERRUPT_PORT0 + port * 4,
+		      mask);
 
-	PMD_DRV_LOG(DEBUG, "port %x, is_xgxs %x, int_status 0x%x", port,
-		    (params->switch_cfg == ELINK_SWITCH_CFG_10G),
-		    REG_RD(sc, NIG_REG_STATUS_INTERRUPT_PORT0 + port * 4));
-	PMD_DRV_LOG(DEBUG, " int_mask 0x%x, MI_INT %x, SERDES_LINK %x",
-		    REG_RD(sc, NIG_REG_MASK_INTERRUPT_PORT0 + port * 4),
-		    REG_RD(sc, NIG_REG_EMAC0_STATUS_MISC_MI_INT + port * 0x18),
-		    REG_RD(sc,
-			   NIG_REG_SERDES0_STATUS_LINK_STATUS + port * 0x3c));
-	PMD_DRV_LOG(DEBUG, " 10G %x, XGXS_LINK %x",
-		    REG_RD(sc, NIG_REG_XGXS0_STATUS_LINK10G + port * 0x68),
-		    REG_RD(sc, NIG_REG_XGXS0_STATUS_LINK_STATUS + port * 0x68));
+	ELINK_DEBUG_P3(sc, "port %x, is_xgxs %x, int_status 0x%x", port,
+		 (params->switch_cfg == ELINK_SWITCH_CFG_10G),
+		 REG_RD(sc, NIG_REG_STATUS_INTERRUPT_PORT0 + port * 4));
+	ELINK_DEBUG_P3(sc, " int_mask 0x%x, MI_INT %x, SERDES_LINK %x",
+		 REG_RD(sc, NIG_REG_MASK_INTERRUPT_PORT0 + port * 4),
+		 REG_RD(sc, NIG_REG_EMAC0_STATUS_MISC_MI_INT + port * 0x18),
+		 REG_RD(sc, NIG_REG_SERDES0_STATUS_LINK_STATUS + port * 0x3c));
+	ELINK_DEBUG_P2(sc, " 10G %x, XGXS_LINK %x",
+	   REG_RD(sc, NIG_REG_XGXS0_STATUS_LINK10G + port * 0x68),
+	   REG_RD(sc, NIG_REG_XGXS0_STATUS_LINK_STATUS + port * 0x68));
 }
 
 static void elink_rearm_latch_signal(struct bnx2x_softc *sc, uint8_t port,
@@ -5693,17 +6995,20 @@ static void elink_rearm_latch_signal(struct bnx2x_softc *sc, uint8_t port,
 	 * so in this case we need to write the status to clear the XOR
 	 */
 	/* Read Latched signals */
-	latch_status = REG_RD(sc, NIG_REG_LATCH_STATUS_0 + port * 8);
-	PMD_DRV_LOG(DEBUG, "latch_status = 0x%x", latch_status);
-	/* Handle only those with latched-signal=up. */
+	latch_status = REG_RD(sc,
+				    NIG_REG_LATCH_STATUS_0 + port * 8);
+	ELINK_DEBUG_P1(sc, "latch_status = 0x%x", latch_status);
+	/* Handle only those with latched-signal=up.*/
 	if (exp_mi_int)
 		elink_bits_en(sc,
 			      NIG_REG_STATUS_INTERRUPT_PORT0
-			      + port * 4, ELINK_NIG_STATUS_EMAC0_MI_INT);
+			      + port * 4,
+			      ELINK_NIG_STATUS_EMAC0_MI_INT);
 	else
 		elink_bits_dis(sc,
 			       NIG_REG_STATUS_INTERRUPT_PORT0
-			       + port * 4, ELINK_NIG_STATUS_EMAC0_MI_INT);
+			       + port * 4,
+			       ELINK_NIG_STATUS_EMAC0_MI_INT);
 
 	if (latch_status & 1) {
 
@@ -5738,23 +7043,24 @@ static void elink_link_int_ack(struct elink_params *params,
 				 * the relevant lane in the status register
 				 */
 				uint32_t ser_lane =
-				    ((params->lane_config &
-				      PORT_HW_CFG_LANE_SWAP_CFG_MASTER_MASK) >>
-				     PORT_HW_CFG_LANE_SWAP_CFG_MASTER_SHIFT);
+					((params->lane_config &
+				    PORT_HW_CFG_LANE_SWAP_CFG_MASTER_MASK) >>
+				    PORT_HW_CFG_LANE_SWAP_CFG_MASTER_SHIFT);
 				mask = ((1 << ser_lane) <<
-					ELINK_NIG_STATUS_XGXS0_LINK_STATUS_SIZE);
+				       ELINK_NIG_STATUS_XGXS0_LINK_STATUS_SIZE);
 			} else
 				mask = ELINK_NIG_STATUS_SERDES0_LINK_STATUS;
 		}
-		PMD_DRV_LOG(DEBUG, "Ack link up interrupt with mask 0x%x",
-			    mask);
+		ELINK_DEBUG_P1(sc, "Ack link up interrupt with mask 0x%x",
+			       mask);
 		elink_bits_en(sc,
-			      NIG_REG_STATUS_INTERRUPT_PORT0 + port * 4, mask);
+			      NIG_REG_STATUS_INTERRUPT_PORT0 + port * 4,
+			      mask);
 	}
 }
 
-static uint8_t elink_format_ver(uint32_t num, uint8_t * str,
-				uint16_t * len)
+static elink_status_t elink_format_ver(uint32_t num, uint8_t *str,
+				       uint16_t *len)
 {
 	uint8_t *str_ptr = str;
 	uint32_t mask = 0xf0000000;
@@ -5792,14 +7098,57 @@ static uint8_t elink_format_ver(uint32_t num, uint8_t * str,
 	return ELINK_STATUS_OK;
 }
 
-static uint8_t elink_null_format_ver(__rte_unused uint32_t spirom_ver,
-				     uint8_t * str, uint16_t * len)
+
+static elink_status_t elink_null_format_ver(__rte_unused uint32_t spirom_ver,
+				 uint8_t *str,
+				 uint16_t *len)
 {
 	str[0] = '\0';
 	(*len)--;
 	return ELINK_STATUS_OK;
 }
 
+elink_status_t elink_get_ext_phy_fw_version(struct elink_params *params,
+				 uint8_t *version,
+				 uint16_t len)
+{
+	struct bnx2x_softc *sc;
+	uint32_t spirom_ver = 0;
+	elink_status_t status = ELINK_STATUS_OK;
+	uint8_t *ver_p = version;
+	uint16_t remain_len = len;
+	if (version == NULL || params == NULL)
+		return ELINK_STATUS_ERROR;
+	sc = params->sc;
+
+	/* Extract first external phy*/
+	version[0] = '\0';
+	spirom_ver = REG_RD(sc, params->phy[ELINK_EXT_PHY1].ver_addr);
+
+	if (params->phy[ELINK_EXT_PHY1].format_fw_ver) {
+		status |= params->phy[ELINK_EXT_PHY1].format_fw_ver(spirom_ver,
+							      ver_p,
+							      &remain_len);
+		ver_p += (len - remain_len);
+	}
+	if ((params->num_phys == ELINK_MAX_PHYS) &&
+	    (params->phy[ELINK_EXT_PHY2].ver_addr != 0)) {
+		spirom_ver = REG_RD(sc, params->phy[ELINK_EXT_PHY2].ver_addr);
+		if (params->phy[ELINK_EXT_PHY2].format_fw_ver) {
+			*ver_p = '/';
+			ver_p++;
+			remain_len--;
+			status |= params->phy[ELINK_EXT_PHY2].format_fw_ver(
+				spirom_ver,
+				ver_p,
+				&remain_len);
+			ver_p = version + (len - remain_len);
+		}
+	}
+	*ver_p = '\0';
+	return status;
+}
+
 static void elink_set_xgxs_loopback(struct elink_phy *phy,
 				    struct elink_params *params)
 {
@@ -5809,7 +7158,7 @@ static void elink_set_xgxs_loopback(struct elink_phy *phy,
 	if (phy->req_line_speed != ELINK_SPEED_1000) {
 		uint32_t md_devad = 0;
 
-		PMD_DRV_LOG(DEBUG, "XGXS 10G loopback enable");
+		ELINK_DEBUG_P0(sc, "XGXS 10G loopback enable");
 
 		if (!CHIP_IS_E3(sc)) {
 			/* Change the uni_phy_addr in the nig */
@@ -5823,7 +7172,8 @@ static void elink_set_xgxs_loopback(struct elink_phy *phy,
 		elink_cl45_write(sc, phy,
 				 5,
 				 (MDIO_REG_BANK_AER_BLOCK +
-				  (MDIO_AER_BLOCK_AER_REG & 0xf)), 0x2800);
+				  (MDIO_AER_BLOCK_AER_REG & 0xf)),
+				 0x2800);
 
 		elink_cl45_write(sc, phy,
 				 5,
@@ -5841,22 +7191,21 @@ static void elink_set_xgxs_loopback(struct elink_phy *phy,
 		}
 	} else {
 		uint16_t mii_ctrl;
-		PMD_DRV_LOG(DEBUG, "XGXS 1G loopback enable");
+		ELINK_DEBUG_P0(sc, "XGXS 1G loopback enable");
 		elink_cl45_read(sc, phy, 5,
 				(MDIO_REG_BANK_COMBO_IEEE0 +
-				 (MDIO_COMBO_IEEE0_MII_CONTROL & 0xf)),
+				(MDIO_COMBO_IEEE0_MII_CONTROL & 0xf)),
 				&mii_ctrl);
 		elink_cl45_write(sc, phy, 5,
 				 (MDIO_REG_BANK_COMBO_IEEE0 +
-				  (MDIO_COMBO_IEEE0_MII_CONTROL & 0xf)),
+				 (MDIO_COMBO_IEEE0_MII_CONTROL & 0xf)),
 				 mii_ctrl |
 				 MDIO_COMBO_IEEO_MII_CONTROL_LOOPBACK);
 	}
 }
 
 elink_status_t elink_set_led(struct elink_params *params,
-			     struct elink_vars *vars, uint8_t mode,
-			     uint32_t speed)
+		  struct elink_vars *vars, uint8_t mode, uint32_t speed)
 {
 	uint8_t port = params->port;
 	uint16_t hw_led_mode = params->hw_led_mode;
@@ -5865,15 +7214,21 @@ elink_status_t elink_set_led(struct elink_params *params,
 	uint32_t tmp;
 	uint32_t emac_base = port ? GRCBASE_EMAC1 : GRCBASE_EMAC0;
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "elink_set_led: port %x, mode %d", port, mode);
-	PMD_DRV_LOG(DEBUG, "speed 0x%x, hw_led_mode 0x%x", speed, hw_led_mode);
+	ELINK_DEBUG_P2(sc, "elink_set_led: port %x, mode %d", port, mode);
+	ELINK_DEBUG_P2(sc, "speed 0x%x, hw_led_mode 0x%x",
+		 speed, hw_led_mode);
 	/* In case */
 	for (phy_idx = ELINK_EXT_PHY1; phy_idx < ELINK_MAX_PHYS; phy_idx++) {
 		if (params->phy[phy_idx].set_link_led) {
-			params->phy[phy_idx].set_link_led(&params->phy[phy_idx],
-							  params, mode);
+			params->phy[phy_idx].set_link_led(
+				&params->phy[phy_idx], params, mode);
 		}
 	}
+#ifdef ELINK_INCLUDE_EMUL
+	if (params->feature_config_flags &
+	    ELINK_FEATURE_CONFIG_EMUL_DISABLE_EMAC)
+		return rc;
+#endif
 
 	switch (mode) {
 	case ELINK_LED_MODE_FRONT_PANEL_OFF:
@@ -5884,10 +7239,10 @@ elink_status_t elink_set_led(struct elink_params *params,
 
 		tmp = elink_cb_reg_read(sc, emac_base + EMAC_REG_EMAC_LED);
 		if (params->phy[ELINK_EXT_PHY1].type ==
-		    PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X54618SE)
+			PORT_HW_CFG_XGXS_EXT_PHY2_TYPE_BNX2X54618SE)
 			tmp &= ~(EMAC_LED_1000MB_OVERRIDE |
-				 EMAC_LED_100MB_OVERRIDE |
-				 EMAC_LED_10MB_OVERRIDE);
+				EMAC_LED_100MB_OVERRIDE |
+				EMAC_LED_10MB_OVERRIDE);
 		else
 			tmp |= EMAC_LED_OVERRIDE;
 
@@ -5900,25 +7255,22 @@ elink_status_t elink_set_led(struct elink_params *params,
 		 */
 		if (!vars->link_up)
 			break;
-		/* fall-through */
+		/* fallthrough */
 	case ELINK_LED_MODE_ON:
 		if (((params->phy[ELINK_EXT_PHY1].type ==
-		      PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8727) ||
-		     (params->phy[ELINK_EXT_PHY1].type ==
-		      PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8722)) &&
+			  PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8727) ||
+			 (params->phy[ELINK_EXT_PHY1].type ==
+			  PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8722)) &&
 		    CHIP_IS_E2(sc) && params->num_phys == 2) {
-			/* This is a work-around for E2+8727 Configurations */
+			/* This is a work-around for E2 + 8727 Configurations */
 			if (mode == ELINK_LED_MODE_ON ||
-			    speed == ELINK_SPEED_10000) {
+				speed == ELINK_SPEED_10000){
 				REG_WR(sc, NIG_REG_LED_MODE_P0 + port * 4, 0);
 				REG_WR(sc, NIG_REG_LED_10G_P0 + port * 4, 1);
 
-				tmp =
-				    elink_cb_reg_read(sc,
-						      emac_base +
-						      EMAC_REG_EMAC_LED);
-				elink_cb_reg_write(sc,
-						   emac_base +
+				tmp = elink_cb_reg_read(sc, emac_base +
+							EMAC_REG_EMAC_LED);
+				elink_cb_reg_write(sc, emac_base +
 						   EMAC_REG_EMAC_LED,
 						   (tmp | EMAC_LED_OVERRIDE));
 				/* Return here without enabling traffic
@@ -5934,22 +7286,23 @@ elink_status_t elink_set_led(struct elink_params *params,
 			 * is up in CL73
 			 */
 			if ((!CHIP_IS_E3(sc)) ||
-			    (CHIP_IS_E3(sc) && mode == ELINK_LED_MODE_ON))
+			    (CHIP_IS_E3(sc) &&
+			     mode == ELINK_LED_MODE_ON))
 				REG_WR(sc, NIG_REG_LED_10G_P0 + port * 4, 1);
 
 			if (CHIP_IS_E1x(sc) ||
-			    CHIP_IS_E2(sc) || (mode == ELINK_LED_MODE_ON))
+			    CHIP_IS_E2(sc) ||
+			    (mode == ELINK_LED_MODE_ON))
 				REG_WR(sc, NIG_REG_LED_MODE_P0 + port * 4, 0);
 			else
 				REG_WR(sc, NIG_REG_LED_MODE_P0 + port * 4,
 				       hw_led_mode);
 		} else if ((params->phy[ELINK_EXT_PHY1].type ==
-			    PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X54618SE) &&
+			    PORT_HW_CFG_XGXS_EXT_PHY2_TYPE_BNX2X54618SE) &&
 			   (mode == ELINK_LED_MODE_ON)) {
 			REG_WR(sc, NIG_REG_LED_MODE_P0 + port * 4, 0);
-			tmp =
-			    elink_cb_reg_read(sc,
-					      emac_base + EMAC_REG_EMAC_LED);
+			tmp = elink_cb_reg_read(sc, emac_base +
+						EMAC_REG_EMAC_LED);
 			elink_cb_reg_write(sc, emac_base + EMAC_REG_EMAC_LED,
 					   tmp | EMAC_LED_OVERRIDE |
 					   EMAC_LED_1000MB_OVERRIDE);
@@ -5959,11 +7312,10 @@ elink_status_t elink_set_led(struct elink_params *params,
 			break;
 		} else {
 			uint32_t nig_led_mode = ((params->hw_led_mode <<
-						  SHARED_HW_CFG_LED_MODE_SHIFT)
-						 ==
-						 SHARED_HW_CFG_LED_EXTPHY2)
-			    ? (SHARED_HW_CFG_LED_PHY1 >>
-			       SHARED_HW_CFG_LED_MODE_SHIFT) : hw_led_mode;
+					     SHARED_HW_CFG_LED_MODE_SHIFT) ==
+					    SHARED_HW_CFG_LED_EXTPHY2) ?
+				(SHARED_HW_CFG_LED_PHY1 >>
+				 SHARED_HW_CFG_LED_MODE_SHIFT) : hw_led_mode;
 			REG_WR(sc, NIG_REG_LED_MODE_P0 + port * 4,
 			       nig_led_mode);
 		}
@@ -5977,26 +7329,133 @@ elink_status_t elink_set_led(struct elink_params *params,
 		else
 			REG_WR(sc, NIG_REG_LED_CONTROL_BLINK_RATE_P0 + port * 4,
 			       LED_BLINK_RATE_VAL_E1X_E2);
-		REG_WR(sc, NIG_REG_LED_CONTROL_BLINK_RATE_ENA_P0 + port * 4, 1);
+		REG_WR(sc, NIG_REG_LED_CONTROL_BLINK_RATE_ENA_P0 +
+		       port * 4, 1);
 		tmp = elink_cb_reg_read(sc, emac_base + EMAC_REG_EMAC_LED);
 		elink_cb_reg_write(sc, emac_base + EMAC_REG_EMAC_LED,
-				   (tmp & (~EMAC_LED_OVERRIDE)));
+			(tmp & (~EMAC_LED_OVERRIDE)));
 
+		if (CHIP_IS_E1(sc) &&
+		    ((speed == ELINK_SPEED_2500) ||
+		     (speed == ELINK_SPEED_1000) ||
+		     (speed == ELINK_SPEED_100) ||
+		     (speed == ELINK_SPEED_10))) {
+			/* For speeds less than 10G LED scheme is different */
+			REG_WR(sc, NIG_REG_LED_CONTROL_OVERRIDE_TRAFFIC_P0
+			       + port * 4, 1);
+			REG_WR(sc, NIG_REG_LED_CONTROL_TRAFFIC_P0 +
+			       port * 4, 0);
+			REG_WR(sc, NIG_REG_LED_CONTROL_BLINK_TRAFFIC_P0 +
+			       port * 4, 1);
+		}
 		break;
 
 	default:
 		rc = ELINK_STATUS_ERROR;
-		PMD_DRV_LOG(DEBUG, "elink_set_led: Invalid led mode %d", mode);
+		ELINK_DEBUG_P1(sc, "elink_set_led: Invalid led mode %d",
+			 mode);
 		break;
 	}
 	return rc;
 
 }
 
+/* This function comes to reflect the actual link state read DIRECTLY from the
+ * HW
+ */
+elink_status_t elink_test_link(struct elink_params *params,
+			       __rte_unused struct elink_vars *vars,
+		    uint8_t is_serdes)
+{
+	struct bnx2x_softc *sc = params->sc;
+	uint16_t gp_status = 0, phy_index = 0;
+	uint8_t ext_phy_link_up = 0, serdes_phy_type;
+	struct elink_vars temp_vars;
+	struct elink_phy *int_phy = &params->phy[ELINK_INT_PHY];
+#ifdef ELINK_INCLUDE_FPGA
+	if (CHIP_REV_IS_FPGA(sc))
+		return ELINK_STATUS_OK;
+#endif
+#ifdef ELINK_INCLUDE_EMUL
+	if (CHIP_REV_IS_EMUL(sc))
+		return ELINK_STATUS_OK;
+#endif
+
+	if (CHIP_IS_E3(sc)) {
+		uint16_t link_up;
+		if (params->req_line_speed[ELINK_LINK_CONFIG_IDX(ELINK_INT_PHY)]
+		    > ELINK_SPEED_10000) {
+			/* Check 20G link */
+			elink_cl45_read(sc, int_phy, MDIO_WC_DEVAD,
+					1, &link_up);
+			elink_cl45_read(sc, int_phy, MDIO_WC_DEVAD,
+					1, &link_up);
+			link_up &= (1 << 2);
+		} else {
+			/* Check 10G link and below*/
+			uint8_t lane = elink_get_warpcore_lane(int_phy, params);
+			elink_cl45_read(sc, int_phy, MDIO_WC_DEVAD,
+					MDIO_WC_REG_GP2_STATUS_GP_2_1,
+					&gp_status);
+			gp_status = ((gp_status >> 8) & 0xf) |
+				((gp_status >> 12) & 0xf);
+			link_up = gp_status & (1 << lane);
+		}
+		if (!link_up)
+			return ELINK_STATUS_NO_LINK;
+	} else {
+		CL22_RD_OVER_CL45(sc, int_phy,
+			  MDIO_REG_BANK_GP_STATUS,
+			  MDIO_GP_STATUS_TOP_AN_STATUS1,
+			  &gp_status);
+	/* Link is up only if both local phy and external phy are up */
+	if (!(gp_status & MDIO_GP_STATUS_TOP_AN_STATUS1_LINK_STATUS))
+		return ELINK_STATUS_NO_LINK;
+	}
+	/* In XGXS loopback mode, do not check external PHY */
+	if (params->loopback_mode == ELINK_LOOPBACK_XGXS)
+		return ELINK_STATUS_OK;
+
+	switch (params->num_phys) {
+	case 1:
+		/* No external PHY */
+		return ELINK_STATUS_OK;
+	case 2:
+		ext_phy_link_up = params->phy[ELINK_EXT_PHY1].read_status(
+			&params->phy[ELINK_EXT_PHY1],
+			params, &temp_vars);
+		break;
+	case 3: /* Dual Media */
+		for (phy_index = ELINK_EXT_PHY1; phy_index < params->num_phys;
+		      phy_index++) {
+			serdes_phy_type = ((params->phy[phy_index].media_type ==
+					    ELINK_ETH_PHY_SFPP_10G_FIBER) ||
+					   (params->phy[phy_index].media_type ==
+					    ELINK_ETH_PHY_SFP_1G_FIBER) ||
+					   (params->phy[phy_index].media_type ==
+					    ELINK_ETH_PHY_XFP_FIBER) ||
+					   (params->phy[phy_index].media_type ==
+					    ELINK_ETH_PHY_DA_TWINAX));
+
+			if (is_serdes != serdes_phy_type)
+				continue;
+			if (params->phy[phy_index].read_status) {
+				ext_phy_link_up |=
+					params->phy[phy_index].read_status(
+						&params->phy[phy_index],
+						params, &temp_vars);
+			}
+		}
+		break;
+	}
+	if (ext_phy_link_up)
+		return ELINK_STATUS_OK;
+	return ELINK_STATUS_NO_LINK;
+}
+
 static elink_status_t elink_link_initialize(struct elink_params *params,
-					    struct elink_vars *vars)
+				 struct elink_vars *vars)
 {
-	elink_status_t rc = ELINK_STATUS_OK;
 	uint8_t phy_index, non_ext_phy;
 	struct bnx2x_softc *sc = params->sc;
 	/* In case of external phy existence, the line speed would be the
@@ -6021,11 +7480,12 @@ static elink_status_t elink_link_initialize(struct elink_params *params,
 	    (params->loopback_mode == ELINK_LOOPBACK_EXT_PHY)) {
 		struct elink_phy *phy = &params->phy[ELINK_INT_PHY];
 		if (vars->line_speed == ELINK_SPEED_AUTO_NEG &&
-		    (CHIP_IS_E1x(sc) || CHIP_IS_E2(sc)))
+		    (CHIP_IS_E1x(sc) ||
+		     CHIP_IS_E2(sc)))
 			elink_set_parallel_detection(phy, params);
 		if (params->phy[ELINK_INT_PHY].config_init)
-			params->phy[ELINK_INT_PHY].config_init(phy,
-							       params, vars);
+			params->phy[ELINK_INT_PHY].config_init(phy, params,
+							       vars);
 	}
 
 	/* Re-read this value in case it was changed inside config_init due to
@@ -6033,14 +7493,14 @@ static elink_status_t elink_link_initialize(struct elink_params *params,
 	 */
 	vars->line_speed = params->phy[ELINK_INT_PHY].req_line_speed;
 
-	/* Init external phy */
+	/* Init external phy*/
 	if (non_ext_phy) {
 		if (params->phy[ELINK_INT_PHY].supported &
 		    ELINK_SUPPORTED_FIBRE)
 			vars->link_status |= LINK_STATUS_SERDES_LINK;
 	} else {
 		for (phy_index = ELINK_EXT_PHY1; phy_index < params->num_phys;
-		     phy_index++) {
+		      phy_index++) {
 			/* No need to initialize second phy in case of first
 			 * phy only selection. In case of second phy, we do
 			 * need to initialize the first phy, since they are
@@ -6053,13 +7513,13 @@ static elink_status_t elink_link_initialize(struct elink_params *params,
 			if (phy_index == ELINK_EXT_PHY2 &&
 			    (elink_phy_selection(params) ==
 			     PORT_HW_CFG_PHY_SELECTION_FIRST_PHY)) {
-				PMD_DRV_LOG(DEBUG,
-					    "Not initializing second phy");
+				ELINK_DEBUG_P0(sc,
+				   "Not initializing second phy");
 				continue;
 			}
-			params->phy[phy_index].config_init(&params->
-							   phy[phy_index],
-							   params, vars);
+			params->phy[phy_index].config_init(
+				&params->phy[phy_index],
+				params, vars);
 		}
 	}
 	/* Reset the interrupt indication after phy was initialized */
@@ -6069,7 +7529,7 @@ static elink_status_t elink_link_initialize(struct elink_params *params,
 			ELINK_NIG_STATUS_XGXS0_LINK_STATUS |
 			ELINK_NIG_STATUS_SERDES0_LINK_STATUS |
 			ELINK_NIG_MASK_MI_INT));
-	return rc;
+	return ELINK_STATUS_OK;
 }
 
 static void elink_int_link_reset(__rte_unused struct elink_phy *phy,
@@ -6091,19 +7551,21 @@ static void elink_common_ext_link_reset(__rte_unused struct elink_phy *phy,
 	else
 		gpio_port = params->port;
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_1,
-			    MISC_REGISTERS_GPIO_OUTPUT_LOW, gpio_port);
+		       MISC_REGISTERS_GPIO_OUTPUT_LOW,
+		       gpio_port);
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_2,
-			    MISC_REGISTERS_GPIO_OUTPUT_LOW, gpio_port);
-	PMD_DRV_LOG(DEBUG, "reset external PHY");
+		       MISC_REGISTERS_GPIO_OUTPUT_LOW,
+		       gpio_port);
+	ELINK_DEBUG_P0(sc, "reset external PHY");
 }
 
 static elink_status_t elink_update_link_down(struct elink_params *params,
-					     struct elink_vars *vars)
+				  struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t port = params->port;
 
-	PMD_DRV_LOG(DEBUG, "Port %x: Link is down", port);
+	ELINK_DEBUG_P1(sc, "Port %x: Link is down", port);
 	elink_set_led(params, vars, ELINK_LED_MODE_OFF, 0);
 	vars->phy_flags &= ~PHY_PHYSICAL_LINK_FLAG;
 	/* Indicate no mac active */
@@ -6123,8 +7585,9 @@ static elink_status_t elink_update_link_down(struct elink_params *params,
 
 	DELAY(1000 * 10);
 	/* Reset BigMac/Xmac */
-	if (CHIP_IS_E1x(sc) || CHIP_IS_E2(sc))
-		elink_set_bmac_rx(sc, params->port, 0);
+	if (CHIP_IS_E1x(sc) ||
+	    CHIP_IS_E2(sc))
+		elink_set_bmac_rx(sc, params->chip_id, params->port, 0);
 
 	if (CHIP_IS_E3(sc)) {
 		/* Prevent LPI Generation by chip */
@@ -6144,8 +7607,8 @@ static elink_status_t elink_update_link_down(struct elink_params *params,
 }
 
 static elink_status_t elink_update_link_up(struct elink_params *params,
-					   struct elink_vars *vars,
-					   uint8_t link_10g)
+				struct elink_vars *vars,
+				uint8_t link_10g)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t phy_idx, port = params->port;
@@ -6156,15 +7619,17 @@ static elink_status_t elink_update_link_up(struct elink_params *params,
 	vars->phy_flags |= PHY_PHYSICAL_LINK_FLAG;
 
 	if (vars->flow_ctrl & ELINK_FLOW_CTRL_TX)
-		vars->link_status |= LINK_STATUS_TX_FLOW_CONTROL_ENABLED;
+		vars->link_status |=
+			LINK_STATUS_TX_FLOW_CONTROL_ENABLED;
 
 	if (vars->flow_ctrl & ELINK_FLOW_CTRL_RX)
-		vars->link_status |= LINK_STATUS_RX_FLOW_CONTROL_ENABLED;
+		vars->link_status |=
+			LINK_STATUS_RX_FLOW_CONTROL_ENABLED;
 	if (USES_WARPCORE(sc)) {
 		if (link_10g) {
 			if (elink_xmac_enable(params, vars, 0) ==
 			    ELINK_STATUS_NO_LINK) {
-				PMD_DRV_LOG(DEBUG, "Found errors on XMAC");
+				ELINK_DEBUG_P0(sc, "Found errors on XMAC");
 				vars->link_up = 0;
 				vars->phy_flags |= PHY_HALF_OPEN_CONN_FLAG;
 				vars->link_status &= ~LINK_STATUS_LINK_UP;
@@ -6176,7 +7641,7 @@ static elink_status_t elink_update_link_up(struct elink_params *params,
 
 		if ((vars->eee_status & SHMEM_EEE_ACTIVE_BIT) &&
 		    (vars->eee_status & SHMEM_EEE_LPI_REQUESTED_BIT)) {
-			PMD_DRV_LOG(DEBUG, "Enabling LPI assertion");
+			ELINK_DEBUG_P0(sc, "Enabling LPI assertion");
 			REG_WR(sc, MISC_REG_CPMU_LP_FW_ENABLE_P0 +
 			       (params->port << 2), 1);
 			REG_WR(sc, MISC_REG_CPMU_LP_DR_ENABLE, 1);
@@ -6184,11 +7649,12 @@ static elink_status_t elink_update_link_up(struct elink_params *params,
 			       (params->port << 2), 0xfc20);
 		}
 	}
-	if ((CHIP_IS_E1x(sc) || CHIP_IS_E2(sc))) {
+	if ((CHIP_IS_E1x(sc) ||
+	     CHIP_IS_E2(sc))) {
 		if (link_10g) {
 			if (elink_bmac_enable(params, vars, 0, 1) ==
 			    ELINK_STATUS_NO_LINK) {
-				PMD_DRV_LOG(DEBUG, "Found errors on BMAC");
+				ELINK_DEBUG_P0(sc, "Found errors on BMAC");
 				vars->link_up = 0;
 				vars->phy_flags |= PHY_HALF_OPEN_CONN_FLAG;
 				vars->link_status &= ~LINK_STATUS_LINK_UP;
@@ -6231,6 +7697,24 @@ static elink_status_t elink_update_link_up(struct elink_params *params,
 	return rc;
 }
 
+static void elink_chng_link_count(struct elink_params *params, uint8_t clear)
+{
+	struct bnx2x_softc *sc = params->sc;
+	uint32_t addr, val;
+
+	/* Verify the link_change_count is supported by the MFW */
+	if (!(SHMEM2_HAS(sc, link_change_count)))
+		return;
+
+	addr = params->shmem2_base +
+		offsetof(struct shmem2_region, link_change_count[params->port]);
+	if (clear)
+		val = 0;
+	else
+		val = REG_RD(sc, addr) + 1;
+	REG_WR(sc, addr, val);
+}
+
 /* The elink_link_update function should be called upon link
  * interrupt.
  * Link is considered up as follows:
@@ -6243,24 +7727,24 @@ static elink_status_t elink_update_link_up(struct elink_params *params,
  *   external phy needs to be up, and at least one of the 2
  *   external phy link must be up.
  */
-elink_status_t elink_link_update(struct elink_params * params,
-				 struct elink_vars * vars)
+elink_status_t elink_link_update(struct elink_params *params,
+				 struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	struct elink_vars phy_vars[ELINK_MAX_PHYS];
 	uint8_t port = params->port;
 	uint8_t link_10g_plus, phy_index;
+	uint32_t prev_link_status = vars->link_status;
 	uint8_t ext_phy_link_up = 0, cur_link_up;
 	elink_status_t rc = ELINK_STATUS_OK;
-	__rte_unused uint8_t is_mi_int = 0;
 	uint16_t ext_phy_line_speed = 0, prev_line_speed = vars->line_speed;
 	uint8_t active_external_phy = ELINK_INT_PHY;
 	vars->phy_flags &= ~PHY_HALF_OPEN_CONN_FLAG;
 	vars->link_status &= ~ELINK_LINK_UPDATE_MASK;
 	for (phy_index = ELINK_INT_PHY; phy_index < params->num_phys;
-	     phy_index++) {
+	      phy_index++) {
 		phy_vars[phy_index].flow_ctrl = 0;
-		phy_vars[phy_index].link_status = ETH_LINK_DOWN;
+		phy_vars[phy_index].link_status = 0;
 		phy_vars[phy_index].line_speed = 0;
 		phy_vars[phy_index].duplex = DUPLEX_FULL;
 		phy_vars[phy_index].phy_link_up = 0;
@@ -6273,21 +7757,18 @@ elink_status_t elink_link_update(struct elink_params * params,
 	if (USES_WARPCORE(sc))
 		elink_set_aer_mmd(params, &params->phy[ELINK_INT_PHY]);
 
-	PMD_DRV_LOG(DEBUG, "port %x, XGXS?%x, int_status 0x%x",
-		    port, (vars->phy_flags & PHY_XGXS_FLAG),
-		    REG_RD(sc, NIG_REG_STATUS_INTERRUPT_PORT0 + port * 4));
+	ELINK_DEBUG_P3(sc, "port %x, XGXS?%x, int_status 0x%x",
+		 port, (vars->phy_flags & PHY_XGXS_FLAG),
+		 REG_RD(sc, NIG_REG_STATUS_INTERRUPT_PORT0 + port * 4));
 
-	is_mi_int = (uint8_t) (REG_RD(sc, NIG_REG_EMAC0_STATUS_MISC_MI_INT +
-				      port * 0x18) > 0);
-	PMD_DRV_LOG(DEBUG, "int_mask 0x%x MI_INT %x, SERDES_LINK %x",
-		    REG_RD(sc, NIG_REG_MASK_INTERRUPT_PORT0 + port * 4),
-		    is_mi_int,
-		    REG_RD(sc,
-			   NIG_REG_SERDES0_STATUS_LINK_STATUS + port * 0x3c));
+	ELINK_DEBUG_P3(sc, "int_mask 0x%x MI_INT %x, SERDES_LINK %x",
+		 REG_RD(sc, NIG_REG_MASK_INTERRUPT_PORT0 + port * 4),
+		 REG_RD(sc, NIG_REG_EMAC0_STATUS_MISC_MI_INT + port * 0x18) > 0,
+		 REG_RD(sc, NIG_REG_SERDES0_STATUS_LINK_STATUS + port * 0x3c));
 
-	PMD_DRV_LOG(DEBUG, " 10G %x, XGXS_LINK %x",
-		    REG_RD(sc, NIG_REG_XGXS0_STATUS_LINK10G + port * 0x68),
-		    REG_RD(sc, NIG_REG_XGXS0_STATUS_LINK_STATUS + port * 0x68));
+	ELINK_DEBUG_P2(sc, " 10G %x, XGXS_LINK %x",
+	  REG_RD(sc, NIG_REG_XGXS0_STATUS_LINK10G + port * 0x68),
+	  REG_RD(sc, NIG_REG_XGXS0_STATUS_LINK_STATUS + port * 0x68));
 
 	/* Disable emac */
 	if (!CHIP_IS_E3(sc))
@@ -6301,7 +7782,7 @@ elink_status_t elink_link_update(struct elink_params * params,
 	 * speed/duplex result
 	 */
 	for (phy_index = ELINK_EXT_PHY1; phy_index < params->num_phys;
-	     phy_index++) {
+	      phy_index++) {
 		struct elink_phy *phy = &params->phy[phy_index];
 		if (!phy->read_status)
 			continue;
@@ -6309,11 +7790,11 @@ elink_status_t elink_link_update(struct elink_params * params,
 		cur_link_up = phy->read_status(phy, params,
 					       &phy_vars[phy_index]);
 		if (cur_link_up) {
-			PMD_DRV_LOG(DEBUG, "phy in index %d link is up",
-				    phy_index);
+			ELINK_DEBUG_P1(sc, "phy in index %d link is up",
+				   phy_index);
 		} else {
-			PMD_DRV_LOG(DEBUG, "phy in index %d link is down",
-				    phy_index);
+			ELINK_DEBUG_P1(sc, "phy in index %d link is down",
+				   phy_index);
 			continue;
 		}
 
@@ -6324,30 +7805,30 @@ elink_status_t elink_link_update(struct elink_params * params,
 			switch (elink_phy_selection(params)) {
 			case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
 			case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
-				/* In this option, the first PHY makes sure to pass the
-				 * traffic through itself only.
-				 * Its not clear how to reset the link on the second phy
-				 */
+			/* In this option, the first PHY makes sure to pass the
+			 * traffic through itself only.
+			 * Its not clear how to reset the link on the second phy
+			 */
 				active_external_phy = ELINK_EXT_PHY1;
 				break;
 			case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
-				/* In this option, the first PHY makes sure to pass the
-				 * traffic through the second PHY.
-				 */
+			/* In this option, the first PHY makes sure to pass the
+			 * traffic through the second PHY.
+			 */
 				active_external_phy = ELINK_EXT_PHY2;
 				break;
 			default:
-				/* Link indication on both PHYs with the following cases
-				 * is invalid:
-				 * - FIRST_PHY means that second phy wasn't initialized,
-				 * hence its link is expected to be down
-				 * - SECOND_PHY means that first phy should not be able
-				 * to link up by itself (using configuration)
-				 * - DEFAULT should be overridden during initialization
-				 */
-				PMD_DRV_LOG(DEBUG, "Invalid link indication"
-					    "mpc=0x%x. DISABLING LINK !!!",
-					    params->multi_phy_config);
+			/* Link indication on both PHYs with the following cases
+			 * is invalid:
+			 * - FIRST_PHY means that second phy wasn't initialized,
+			 * hence its link is expected to be down
+			 * - SECOND_PHY means that first phy should not be able
+			 * to link up by itself (using configuration)
+			 * - DEFAULT should be overridden during initialiazation
+			 */
+				ELINK_DEBUG_P1(sc, "Invalid link indication"
+					       " mpc=0x%x. DISABLING LINK !!!",
+					   params->multi_phy_config);
 				ext_phy_link_up = 0;
 				break;
 			}
@@ -6361,9 +7842,9 @@ elink_status_t elink_link_update(struct elink_params * params,
 	 * external phy
 	 */
 	if (params->phy[ELINK_INT_PHY].read_status)
-		params->phy[ELINK_INT_PHY].read_status(&params->
-						       phy[ELINK_INT_PHY],
-						       params, vars);
+		params->phy[ELINK_INT_PHY].read_status(
+			&params->phy[ELINK_INT_PHY],
+			params, vars);
 	/* The INT_PHY flow control reside in the vars. This include the
 	 * case where the speed or flow control are not set to AUTO.
 	 * Otherwise, the active external phy flow control result is set
@@ -6383,11 +7864,11 @@ elink_status_t elink_link_update(struct elink_params * params,
 		 */
 		if (active_external_phy == ELINK_EXT_PHY1) {
 			if (params->phy[ELINK_EXT_PHY2].phy_specific_func) {
-				PMD_DRV_LOG(DEBUG, "Disabling TX on EXT_PHY2");
-				params->phy[ELINK_EXT_PHY2].
-				    phy_specific_func(&params->
-						      phy[ELINK_EXT_PHY2],
-						      params, ELINK_DISABLE_TX);
+				ELINK_DEBUG_P0(sc,
+				   "Disabling TX on EXT_PHY2");
+				params->phy[ELINK_EXT_PHY2].phy_specific_func(
+					&params->phy[ELINK_EXT_PHY2],
+					params, ELINK_DISABLE_TX);
 			}
 		}
 
@@ -6401,12 +7882,27 @@ elink_status_t elink_link_update(struct elink_params * params,
 
 		vars->eee_status = phy_vars[active_external_phy].eee_status;
 
-		PMD_DRV_LOG(DEBUG, "Active external phy selected: %x",
-			    active_external_phy);
-	}
+		ELINK_DEBUG_P1(sc, "Active external phy selected: %x",
+			   active_external_phy);
+	}
+
+	ELINK_DEBUG_P3(sc, "vars : phy_flags = %x, mac_type = %x, phy_link_up = %x",
+		       vars->phy_flags, vars->mac_type, vars->phy_link_up);
+	ELINK_DEBUG_P3(sc, "vars : link_up = %x, line_speed = %x, duplex = %x",
+		       vars->link_up, vars->line_speed, vars->duplex);
+	ELINK_DEBUG_P3(sc, "vars : flow_ctrl = %x, ieee_fc = %x, link_status = %x",
+		       vars->flow_ctrl, vars->ieee_fc, vars->link_status);
+	ELINK_DEBUG_P3(sc, "vars : eee_status = %x, fault_detected = %x, check_kr2_recovery_cnt = %x",
+		       vars->eee_status, vars->fault_detected,
+		       vars->check_kr2_recovery_cnt);
+	ELINK_DEBUG_P3(sc, "vars : periodic_flags = %x, aeu_int_mask = %x, rx_tx_asic_rst = %x",
+		       vars->periodic_flags, vars->aeu_int_mask,
+		       vars->rx_tx_asic_rst);
+	ELINK_DEBUG_P2(sc, "vars : turn_to_run_wc_rt = %x, rsrv2 = %x",
+		       vars->turn_to_run_wc_rt, vars->rsrv2);
 
 	for (phy_index = ELINK_EXT_PHY1; phy_index < params->num_phys;
-	     phy_index++) {
+	      phy_index++) {
 		if (params->phy[phy_index].flags &
 		    ELINK_FLAGS_REARM_LATCH_SIGNAL) {
 			elink_rearm_latch_signal(sc, port,
@@ -6415,9 +7911,9 @@ elink_status_t elink_link_update(struct elink_params * params,
 			break;
 		}
 	}
-	PMD_DRV_LOG(DEBUG, "vars->flow_ctrl = 0x%x, vars->link_status = 0x%x,"
-		    " ext_phy_line_speed = %d", vars->flow_ctrl,
-		    vars->link_status, ext_phy_line_speed);
+	ELINK_DEBUG_P3(sc, "vars->flow_ctrl = 0x%x, vars->link_status = 0x%x,"
+		   " ext_phy_line_speed = %d", vars->flow_ctrl,
+		   vars->link_status, ext_phy_line_speed);
 	/* Upon link speed change set the NIG into drain mode. Comes to
 	 * deals with possible FIFO glitch due to clk change when speed
 	 * is decreased without link down indicator
@@ -6426,15 +7922,15 @@ elink_status_t elink_link_update(struct elink_params * params,
 	if (vars->phy_link_up) {
 		if (!(ELINK_SINGLE_MEDIA_DIRECT(params)) && ext_phy_link_up &&
 		    (ext_phy_line_speed != vars->line_speed)) {
-			PMD_DRV_LOG(DEBUG, "Internal link speed %d is"
-				    " different than the external"
-				    " link speed %d", vars->line_speed,
-				    ext_phy_line_speed);
+			ELINK_DEBUG_P2(sc, "Internal link speed %d is"
+				   " different than the external"
+				   " link speed %d", vars->line_speed,
+				   ext_phy_line_speed);
 			vars->phy_link_up = 0;
+			ELINK_DEBUG_P0(sc, "phy_link_up set to 0");
 		} else if (prev_line_speed != vars->line_speed) {
-			REG_WR(sc,
-			       NIG_REG_EGRESS_DRAIN0_MODE + params->port * 4,
-			       0);
+			REG_WR(sc, NIG_REG_EGRESS_DRAIN0_MODE +
+			       params->port * 4, 0);
 			DELAY(1000 * 1);
 		}
 	}
@@ -6452,11 +7948,11 @@ elink_status_t elink_link_update(struct elink_params * params,
 	 * initialize it
 	 */
 	if (!(ELINK_SINGLE_MEDIA_DIRECT(params))) {
-		PMD_DRV_LOG(DEBUG, "ext_phy_link_up = %d, int_link_up = %d,"
-			    " init_preceding = %d", ext_phy_link_up,
-			    vars->phy_link_up,
-			    params->phy[ELINK_EXT_PHY1].flags &
-			    ELINK_FLAGS_INIT_XGXS_FIRST);
+		ELINK_DEBUG_P3(sc, "ext_phy_link_up = %d, int_link_up = %d,"
+			   " init_preceding = %d", ext_phy_link_up,
+			   vars->phy_link_up,
+			   params->phy[ELINK_EXT_PHY1].flags &
+			   ELINK_FLAGS_INIT_XGXS_FIRST);
 		if (!(params->phy[ELINK_EXT_PHY1].flags &
 		      ELINK_FLAGS_INIT_XGXS_FIRST)
 		    && ext_phy_link_up && !vars->phy_link_up) {
@@ -6467,11 +7963,9 @@ elink_status_t elink_link_update(struct elink_params * params,
 				vars->phy_flags &= ~PHY_SGMII_FLAG;
 
 			if (params->phy[ELINK_INT_PHY].config_init)
-				params->phy[ELINK_INT_PHY].config_init(&params->
-								       phy
-								       [ELINK_INT_PHY],
-								       params,
-								       vars);
+				params->phy[ELINK_INT_PHY].config_init(
+					&params->phy[ELINK_INT_PHY], params,
+						vars);
 		}
 	}
 	/* Link is up only if both local phy and external phy (in case of
@@ -6482,6 +7976,11 @@ elink_status_t elink_link_update(struct elink_params * params,
 			  ELINK_SINGLE_MEDIA_DIRECT(params)) &&
 			 (phy_vars[active_external_phy].fault_detected == 0));
 
+	if (vars->link_up)
+		ELINK_DEBUG_P0(sc, "local phy and external phy are up");
+	else
+		ELINK_DEBUG_P0(sc, "either local phy or external phy or both are down");
+
 	/* Update the PFC configuration in case it was changed */
 	if (params->feature_config_flags & ELINK_FEATURE_CONFIG_PFC_ENABLED)
 		vars->link_status |= LINK_STATUS_PFC_ENABLED;
@@ -6493,9 +7992,12 @@ elink_status_t elink_link_update(struct elink_params * params,
 	else
 		rc = elink_update_link_down(params, vars);
 
+	if ((prev_link_status ^ vars->link_status) & LINK_STATUS_LINK_UP)
+		elink_chng_link_count(params, 0);
+
 	/* Update MCP link status was changed */
-	if (params->
-	    feature_config_flags & ELINK_FEATURE_CONFIG_BC_SUPPORTS_AFEX)
+	if (params->feature_config_flags &
+	    ELINK_FEATURE_CONFIG_BC_SUPPORTS_AFEX)
 		elink_cb_fw_command(sc, DRV_MSG_CODE_LINK_STATUS_CHANGED, 0);
 
 	return rc;
@@ -6504,28 +8006,28 @@ elink_status_t elink_link_update(struct elink_params * params,
 /*****************************************************************************/
 /*			    External Phy section			     */
 /*****************************************************************************/
-static void elink_ext_phy_hw_reset(struct bnx2x_softc *sc, uint8_t port)
+void elink_ext_phy_hw_reset(struct bnx2x_softc *sc, uint8_t port)
 {
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_1,
-			    MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
+		       MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
 	DELAY(1000 * 1);
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_1,
-			    MISC_REGISTERS_GPIO_OUTPUT_HIGH, port);
+		       MISC_REGISTERS_GPIO_OUTPUT_HIGH, port);
 }
 
-static void elink_save_spirom_version(struct bnx2x_softc *sc,
-				      __rte_unused uint8_t port,
+static void elink_save_spirom_version(struct bnx2x_softc *sc, uint8_t port,
 				      uint32_t spirom_ver, uint32_t ver_addr)
 {
-	PMD_DRV_LOG(DEBUG, "FW version 0x%x:0x%x for port %d",
-		    (uint16_t) (spirom_ver >> 16), (uint16_t) spirom_ver, port);
+	ELINK_DEBUG_P3(sc, "FW version 0x%x:0x%x for port %d",
+		 (uint16_t)(spirom_ver >> 16), (uint16_t)spirom_ver, port);
 
 	if (ver_addr)
 		REG_WR(sc, ver_addr, spirom_ver);
 }
 
 static void elink_save_bnx2x_spirom_ver(struct bnx2x_softc *sc,
-				      struct elink_phy *phy, uint8_t port)
+				      struct elink_phy *phy,
+				      uint8_t port)
 {
 	uint16_t fw_ver1, fw_ver2;
 
@@ -6533,18 +8035,21 @@ static void elink_save_bnx2x_spirom_ver(struct bnx2x_softc *sc,
 			MDIO_PMA_REG_ROM_VER1, &fw_ver1);
 	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD,
 			MDIO_PMA_REG_ROM_VER2, &fw_ver2);
-	elink_save_spirom_version(sc, port,
-				  (uint32_t) (fw_ver1 << 16 | fw_ver2),
+	elink_save_spirom_version(sc, port, (uint32_t)(fw_ver1 << 16 | fw_ver2),
 				  phy->ver_addr);
 }
 
 static void elink_ext_phy_10G_an_resolve(struct bnx2x_softc *sc,
-					 struct elink_phy *phy,
-					 struct elink_vars *vars)
+				       struct elink_phy *phy,
+				       struct elink_vars *vars)
 {
 	uint16_t val;
-	elink_cl45_read(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_STATUS, &val);
-	elink_cl45_read(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_STATUS, &val);
+	elink_cl45_read(sc, phy,
+			MDIO_AN_DEVAD,
+			MDIO_AN_REG_STATUS, &val);
+	elink_cl45_read(sc, phy,
+			MDIO_AN_DEVAD,
+			MDIO_AN_REG_STATUS, &val);
 	if (val & (1 << 5))
 		vars->link_status |= LINK_STATUS_AUTO_NEGOTIATE_COMPLETE;
 	if ((val & (1 << 0)) == 0)
@@ -6568,8 +8073,8 @@ static void elink_8073_resolve_fc(struct elink_phy *phy,
 	if (elink_ext_phy_resolve_fc(phy, params, vars) &&
 	    (vars->flow_ctrl == ELINK_FLOW_CTRL_NONE)) {
 		uint16_t pause_result;
-		uint16_t ld_pause;	/* local */
-		uint16_t lp_pause;	/* link partner */
+		uint16_t ld_pause;		/* local */
+		uint16_t lp_pause;		/* link partner */
 		elink_cl45_read(sc, phy,
 				MDIO_AN_DEVAD,
 				MDIO_AN_REG_CL37_FC_LD, &ld_pause);
@@ -6582,31 +8087,35 @@ static void elink_8073_resolve_fc(struct elink_phy *phy,
 		pause_result |= (lp_pause &
 				 MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH) >> 7;
 
-		elink_pause_resolve(vars, pause_result);
-		PMD_DRV_LOG(DEBUG, "Ext PHY CL37 pause result 0x%x",
-			    pause_result);
+		elink_pause_resolve(phy, params, vars, pause_result);
+		ELINK_DEBUG_P1(sc, "Ext PHY CL37 pause result 0x%x",
+			   pause_result);
 	}
 }
-
 static elink_status_t elink_8073_8727_external_rom_boot(struct bnx2x_softc *sc,
-							struct elink_phy *phy,
-							uint8_t port)
+					     struct elink_phy *phy,
+					     uint8_t port)
 {
 	uint32_t count = 0;
-	uint16_t fw_ver1 = 0, fw_msgout;
+	uint16_t fw_ver1, fw_msgout;
 	elink_status_t rc = ELINK_STATUS_OK;
 
 	/* Boot port from external ROM  */
 	/* EDC grst */
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_GEN_CTRL, 0x0001);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_GEN_CTRL,
+			 0x0001);
 
 	/* Ucode reboot and rst */
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_GEN_CTRL, 0x008c);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_GEN_CTRL,
+			 0x008c);
 
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_MISC_CTRL1, 0x0001);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_MISC_CTRL1, 0x0001);
 
 	/* Reset internal microprocessor */
 	elink_cl45_write(sc, phy,
@@ -6627,10 +8136,10 @@ static elink_status_t elink_8073_8727_external_rom_boot(struct bnx2x_softc *sc,
 	do {
 		count++;
 		if (count > 300) {
-			PMD_DRV_LOG(DEBUG,
-				    "elink_8073_8727_external_rom_boot port %x:"
-				    "Download failed. fw version = 0x%x",
-				    port, fw_ver1);
+			ELINK_DEBUG_P2(sc,
+				 "elink_8073_8727_external_rom_boot port %x:"
+				 "Download failed. fw version = 0x%x",
+				 port, fw_ver1);
 			rc = ELINK_STATUS_ERROR;
 			break;
 		}
@@ -6644,17 +8153,19 @@ static elink_status_t elink_8073_8727_external_rom_boot(struct bnx2x_softc *sc,
 
 		DELAY(1000 * 1);
 	} while (fw_ver1 == 0 || fw_ver1 == 0x4321 ||
-		 ((fw_msgout & 0xff) != 0x03 && (phy->type ==
-						 PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8073)));
+			((fw_msgout & 0xff) != 0x03 && (phy->type ==
+			PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8073)));
 
 	/* Clear ser_boot_ctl bit */
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_MISC_CTRL1, 0x0000);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_MISC_CTRL1, 0x0000);
 	elink_save_bnx2x_spirom_ver(sc, phy, port);
 
-	PMD_DRV_LOG(DEBUG,
-		    "elink_8073_8727_external_rom_boot port %x:"
-		    "Download complete. fw version = 0x%x", port, fw_ver1);
+	ELINK_DEBUG_P2(sc,
+		 "elink_8073_8727_external_rom_boot port %x:"
+		 "Download complete. fw version = 0x%x",
+		 port, fw_ver1);
 
 	return rc;
 }
@@ -6668,22 +8179,25 @@ static elink_status_t elink_8073_is_snr_needed(struct bnx2x_softc *sc,
 	/* This is only required for 8073A1, version 102 only */
 	uint16_t val;
 
-	/* Read 8073 HW revision */
+	/* Read 8073 HW revision*/
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, MDIO_PMA_REG_8073_CHIP_REV, &val);
+			MDIO_PMA_DEVAD,
+			MDIO_PMA_REG_8073_CHIP_REV, &val);
 
 	if (val != 1) {
 		/* No need to workaround in 8073 A1 */
 		return ELINK_STATUS_OK;
 	}
 
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_ROM_VER2, &val);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD,
+			MDIO_PMA_REG_ROM_VER2, &val);
 
 	/* SNR should be applied only for version 0x102 */
 	if (val != 0x102)
 		return ELINK_STATUS_OK;
 
-	return ELINK_STATUS_ERROR;
+	return 1;
 }
 
 static elink_status_t elink_8073_xaui_wa(struct bnx2x_softc *sc,
@@ -6692,7 +8206,8 @@ static elink_status_t elink_8073_xaui_wa(struct bnx2x_softc *sc,
 	uint16_t val, cnt, cnt1;
 
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, MDIO_PMA_REG_8073_CHIP_REV, &val);
+			MDIO_PMA_DEVAD,
+			MDIO_PMA_REG_8073_CHIP_REV, &val);
 
 	if (val > 0) {
 		/* No need to workaround in 8073 A1 */
@@ -6707,16 +8222,17 @@ static elink_status_t elink_8073_xaui_wa(struct bnx2x_softc *sc,
 	for (cnt = 0; cnt < 1000; cnt++) {
 		elink_cl45_read(sc, phy,
 				MDIO_PMA_DEVAD,
-				MDIO_PMA_REG_8073_SPEED_LINK_STATUS, &val);
-		/* If bit [14] = 0 or bit [13] = 0, continue on with
-		 * system initialization (XAUI work-around not required, as
-		 * these bits indicate 2.5G or 1G link up).
-		 */
+				MDIO_PMA_REG_8073_SPEED_LINK_STATUS,
+				&val);
+		  /* If bit [14] = 0 or bit [13] = 0, continue on with
+		   * system initialization (XAUI work-around not required, as
+		   * these bits indicate 2.5G or 1G link up).
+		   */
 		if (!(val & (1 << 14)) || !(val & (1 << 13))) {
-			PMD_DRV_LOG(DEBUG, "XAUI work-around not required");
+			ELINK_DEBUG_P0(sc, "XAUI work-around not required");
 			return ELINK_STATUS_OK;
 		} else if (!(val & (1 << 15))) {
-			PMD_DRV_LOG(DEBUG, "bit 15 went off");
+			ELINK_DEBUG_P0(sc, "bit 15 went off");
 			/* If bit 15 is 0, then poll Dev1, Reg $C841 until it's
 			 * MSB (bit15) goes to 1 (indicating that the XAUI
 			 * workaround has completed), then continue on with
@@ -6724,12 +8240,11 @@ static elink_status_t elink_8073_xaui_wa(struct bnx2x_softc *sc,
 			 */
 			for (cnt1 = 0; cnt1 < 1000; cnt1++) {
 				elink_cl45_read(sc, phy,
-						MDIO_PMA_DEVAD,
-						MDIO_PMA_REG_8073_XAUI_WA,
-						&val);
+					MDIO_PMA_DEVAD,
+					MDIO_PMA_REG_8073_XAUI_WA, &val);
 				if (val & (1 << 15)) {
-					PMD_DRV_LOG(DEBUG,
-						    "XAUI workaround has completed");
+					ELINK_DEBUG_P0(sc,
+					  "XAUI workaround has completed");
 					return ELINK_STATUS_OK;
 				}
 				DELAY(1000 * 3);
@@ -6738,19 +8253,21 @@ static elink_status_t elink_8073_xaui_wa(struct bnx2x_softc *sc,
 		}
 		DELAY(1000 * 3);
 	}
-	PMD_DRV_LOG(DEBUG, "Warning: XAUI work-around timeout !!!");
+	ELINK_DEBUG_P0(sc, "Warning: XAUI work-around timeout !!!");
 	return ELINK_STATUS_ERROR;
 }
 
 static void elink_807x_force_10G(struct bnx2x_softc *sc, struct elink_phy *phy)
 {
 	/* Force KR or KX */
-	elink_cl45_write(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 0x2040);
+	elink_cl45_write(sc, phy,
+			 MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 0x2040);
 	elink_cl45_write(sc, phy,
 			 MDIO_PMA_DEVAD, MDIO_PMA_REG_10G_CTRL2, 0x000b);
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_BNX2X_CTRL, 0x0000);
-	elink_cl45_write(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, 0x0000);
+			 MDIO_PMA_DEVAD, MDIO_PMA_REG_BCM_CTRL, 0x0000);
+	elink_cl45_write(sc, phy,
+			 MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, 0x0000);
 }
 
 static void elink_8073_set_pause_cl37(struct elink_params *params,
@@ -6766,21 +8283,22 @@ static void elink_8073_set_pause_cl37(struct elink_params *params,
 	/* Please refer to Table 28B-3 of 802.3ab-1999 spec. */
 	elink_calc_ieee_aneg_adv(phy, params, &vars->ieee_fc);
 	if ((vars->ieee_fc &
-	     MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_SYMMETRIC) ==
+	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_SYMMETRIC) ==
 	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_SYMMETRIC) {
-		cl37_val |= MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_SYMMETRIC;
+		cl37_val |=  MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_SYMMETRIC;
 	}
 	if ((vars->ieee_fc &
-	     MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC) ==
+	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC) ==
 	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC) {
-		cl37_val |= MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC;
+		cl37_val |=  MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC;
 	}
 	if ((vars->ieee_fc &
-	     MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH) ==
+	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH) ==
 	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH) {
 		cl37_val |= MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH;
 	}
-	PMD_DRV_LOG(DEBUG, "Ext phy AN advertize cl37 0x%x", cl37_val);
+	ELINK_DEBUG_P1(sc,
+		 "Ext phy AN advertize cl37 0x%x", cl37_val);
 
 	elink_cl45_write(sc, phy,
 			 MDIO_AN_DEVAD, MDIO_AN_REG_CL37_FC_LD, cl37_val);
@@ -6798,31 +8316,31 @@ static void elink_8073_specific_func(struct elink_phy *phy,
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXCTRL,
 				 (1 << 2));
-		elink_cl45_write(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL,
-				 0x0004);
+		elink_cl45_write(sc, phy,
+				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL,  0x0004);
 		break;
 	}
 }
 
 static uint8_t elink_8073_config_init(struct elink_phy *phy,
-				      struct elink_params *params,
-				      struct elink_vars *vars)
+				  struct elink_params *params,
+				  struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t val = 0, tmp1;
 	uint8_t gpio_port;
-	PMD_DRV_LOG(DEBUG, "Init 8073");
+	ELINK_DEBUG_P0(sc, "Init 8073");
 
 	if (CHIP_IS_E2(sc))
 		gpio_port = SC_PATH(sc);
 	else
 		gpio_port = params->port;
-	/* Restore normal power mode */
+	/* Restore normal power mode*/
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_2,
-			    MISC_REGISTERS_GPIO_OUTPUT_HIGH, gpio_port);
+		       MISC_REGISTERS_GPIO_OUTPUT_HIGH, gpio_port);
 
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_1,
-			    MISC_REGISTERS_GPIO_OUTPUT_HIGH, gpio_port);
+		       MISC_REGISTERS_GPIO_OUTPUT_HIGH, gpio_port);
 
 	elink_8073_specific_func(phy, params, ELINK_PHY_INIT);
 	elink_8073_set_pause_cl37(params, phy, vars);
@@ -6830,14 +8348,15 @@ static uint8_t elink_8073_config_init(struct elink_phy *phy,
 	elink_cl45_read(sc, phy,
 			MDIO_PMA_DEVAD, MDIO_PMA_REG_M8051_MSGOUT_REG, &tmp1);
 
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXSTAT, &tmp1);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXSTAT, &tmp1);
 
-	PMD_DRV_LOG(DEBUG, "Before rom RX_ALARM(port1): 0x%x", tmp1);
+	ELINK_DEBUG_P1(sc, "Before rom RX_ALARM(port1): 0x%x", tmp1);
 
 	/* Swap polarity if required - Must be done only in non-1G mode */
 	if (params->lane_config & PORT_HW_CFG_SWAP_PHY_POLARITY_ENABLED) {
 		/* Configure the 8073 to swap _P and _N of the KR lines */
-		PMD_DRV_LOG(DEBUG, "Swapping polarity for the 8073");
+		ELINK_DEBUG_P0(sc, "Swapping polarity for the 8073");
 		/* 10G Rx/Tx and 1G Tx signal polarity swap */
 		elink_cl45_read(sc, phy,
 				MDIO_PMA_DEVAD,
@@ -6848,31 +8367,33 @@ static uint8_t elink_8073_config_init(struct elink_phy *phy,
 				 (val | (3 << 9)));
 	}
 
+
 	/* Enable CL37 BAM */
 	if (REG_RD(sc, params->shmem_base +
-		   offsetof(struct shmem_region,
-			    dev_info.port_hw_config[params->port].
-			    default_cfg)) &
+			 offsetof(struct shmem_region, dev_info.
+				  port_hw_config[params->port].default_cfg)) &
 	    PORT_HW_CFG_ENABLE_BAM_ON_KR_ENABLED) {
 
 		elink_cl45_read(sc, phy,
-				MDIO_AN_DEVAD, MDIO_AN_REG_8073_BAM, &val);
+				MDIO_AN_DEVAD,
+				MDIO_AN_REG_8073_BAM, &val);
 		elink_cl45_write(sc, phy,
-				 MDIO_AN_DEVAD, MDIO_AN_REG_8073_BAM, val | 1);
-		PMD_DRV_LOG(DEBUG, "Enable CL37 BAM on KR");
+				 MDIO_AN_DEVAD,
+				 MDIO_AN_REG_8073_BAM, val | 1);
+		ELINK_DEBUG_P0(sc, "Enable CL37 BAM on KR");
 	}
 	if (params->loopback_mode == ELINK_LOOPBACK_EXT) {
 		elink_807x_force_10G(sc, phy);
-		PMD_DRV_LOG(DEBUG, "Forced speed 10G on 807X");
+		ELINK_DEBUG_P0(sc, "Forced speed 10G on 807X");
 		return ELINK_STATUS_OK;
 	} else {
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_REG_BNX2X_CTRL, 0x0002);
+				 MDIO_PMA_DEVAD, MDIO_PMA_REG_BCM_CTRL, 0x0002);
 	}
 	if (phy->req_line_speed != ELINK_SPEED_AUTO_NEG) {
 		if (phy->req_line_speed == ELINK_SPEED_10000) {
 			val = (1 << 7);
-		} else if (phy->req_line_speed == ELINK_SPEED_2500) {
+		} else if (phy->req_line_speed ==  ELINK_SPEED_2500) {
 			val = (1 << 5);
 			/* Note that 2.5G works only when used with 1G
 			 * advertisement
@@ -6881,15 +8402,16 @@ static uint8_t elink_8073_config_init(struct elink_phy *phy,
 			val = (1 << 5);
 	} else {
 		val = 0;
-		if (phy->speed_cap_mask & PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)
+		if (phy->speed_cap_mask &
+			PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)
 			val |= (1 << 7);
 
 		/* Note that 2.5G works only when used with 1G advertisement */
 		if (phy->speed_cap_mask &
-		    (PORT_HW_CFG_SPEED_CAPABILITY_D0_1G |
-		     PORT_HW_CFG_SPEED_CAPABILITY_D0_2_5G))
+			(PORT_HW_CFG_SPEED_CAPABILITY_D0_1G |
+			 PORT_HW_CFG_SPEED_CAPABILITY_D0_2_5G))
 			val |= (1 << 5);
-		PMD_DRV_LOG(DEBUG, "807x autoneg val = 0x%x", val);
+		ELINK_DEBUG_P1(sc, "807x autoneg val = 0x%x", val);
 	}
 
 	elink_cl45_write(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_ADV, val);
@@ -6903,13 +8425,13 @@ static uint8_t elink_8073_config_init(struct elink_phy *phy,
 		elink_cl45_read(sc, phy,
 				MDIO_PMA_DEVAD, MDIO_PMA_REG_8073_CHIP_REV,
 				&phy_ver);
-		PMD_DRV_LOG(DEBUG, "Add 2.5G");
+		ELINK_DEBUG_P0(sc, "Add 2.5G");
 		if (phy_ver > 0)
 			tmp1 |= 1;
 		else
 			tmp1 &= 0xfffe;
 	} else {
-		PMD_DRV_LOG(DEBUG, "Disable 2.5G");
+		ELINK_DEBUG_P0(sc, "Disable 2.5G");
 		tmp1 &= 0xfffe;
 	}
 
@@ -6943,14 +8465,14 @@ static uint8_t elink_8073_config_init(struct elink_phy *phy,
 	/* Restart autoneg */
 	DELAY(1000 * 500);
 	elink_cl45_write(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, 0x1200);
-	PMD_DRV_LOG(DEBUG, "807x Autoneg Restart: Advertise 1G=%x, 10G=%x",
-		    ((val & (1 << 5)) > 0), ((val & (1 << 7)) > 0));
+	ELINK_DEBUG_P2(sc, "807x Autoneg Restart: Advertise 1G=%x, 10G=%x",
+		   ((val & (1 << 5)) > 0), ((val & (1 << 7)) > 0));
 	return ELINK_STATUS_OK;
 }
 
 static uint8_t elink_8073_read_status(struct elink_phy *phy,
-				      struct elink_params *params,
-				      struct elink_vars *vars)
+				 struct elink_params *params,
+				 struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t link_up = 0;
@@ -6958,33 +8480,41 @@ static uint8_t elink_8073_read_status(struct elink_phy *phy,
 	uint16_t link_status = 0;
 	uint16_t an1000_status = 0;
 
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val1);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val1);
 
-	PMD_DRV_LOG(DEBUG, "8703 LASI status 0x%x", val1);
+	ELINK_DEBUG_P1(sc, "8703 LASI status 0x%x", val1);
 
 	/* Clear the interrupt LASI status register */
-	elink_cl45_read(sc, phy, MDIO_PCS_DEVAD, MDIO_PCS_REG_STATUS, &val2);
-	elink_cl45_read(sc, phy, MDIO_PCS_DEVAD, MDIO_PCS_REG_STATUS, &val1);
-	PMD_DRV_LOG(DEBUG, "807x PCS status 0x%x->0x%x", val2, val1);
+	elink_cl45_read(sc, phy,
+			MDIO_PCS_DEVAD, MDIO_PCS_REG_STATUS, &val2);
+	elink_cl45_read(sc, phy,
+			MDIO_PCS_DEVAD, MDIO_PCS_REG_STATUS, &val1);
+	ELINK_DEBUG_P2(sc, "807x PCS status 0x%x->0x%x", val2, val1);
 	/* Clear MSG-OUT */
 	elink_cl45_read(sc, phy,
 			MDIO_PMA_DEVAD, MDIO_PMA_REG_M8051_MSGOUT_REG, &val1);
 
 	/* Check the LASI */
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXSTAT, &val2);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXSTAT, &val2);
 
-	PMD_DRV_LOG(DEBUG, "KR 0x9003 0x%x", val2);
+	ELINK_DEBUG_P1(sc, "KR 0x9003 0x%x", val2);
 
 	/* Check the link status */
-	elink_cl45_read(sc, phy, MDIO_PCS_DEVAD, MDIO_PCS_REG_STATUS, &val2);
-	PMD_DRV_LOG(DEBUG, "KR PCS status 0x%x", val2);
+	elink_cl45_read(sc, phy,
+			MDIO_PCS_DEVAD, MDIO_PCS_REG_STATUS, &val2);
+	ELINK_DEBUG_P1(sc, "KR PCS status 0x%x", val2);
 
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val2);
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val1);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val2);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val1);
 	link_up = ((val1 & 4) == 4);
-	PMD_DRV_LOG(DEBUG, "PMA_REG_STATUS=0x%x", val1);
+	ELINK_DEBUG_P1(sc, "PMA_REG_STATUS=0x%x", val1);
 
-	if (link_up && ((phy->req_line_speed != ELINK_SPEED_10000))) {
+	if (link_up &&
+	     ((phy->req_line_speed != ELINK_SPEED_10000))) {
 		if (elink_8073_xaui_wa(sc, phy) != 0)
 			return 0;
 	}
@@ -6994,10 +8524,12 @@ static uint8_t elink_8073_read_status(struct elink_phy *phy,
 			MDIO_AN_DEVAD, MDIO_AN_REG_LINK_STATUS, &an1000_status);
 
 	/* Check the link status on 1.1.2 */
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val2);
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val1);
-	PMD_DRV_LOG(DEBUG, "KR PMA status 0x%x->0x%x,"
-		    "an_link_status=0x%x", val2, val1, an1000_status);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val2);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val1);
+	ELINK_DEBUG_P3(sc, "KR PMA status 0x%x->0x%x,"
+		   "an_link_status=0x%x", val2, val1, an1000_status);
 
 	link_up = (((val1 & 4) == 4) || (an1000_status & (1 << 1)));
 	if (link_up && elink_8073_is_snr_needed(sc, phy)) {
@@ -7022,27 +8554,28 @@ static uint8_t elink_8073_read_status(struct elink_phy *phy,
 	if ((link_status & (1 << 2)) && (!(link_status & (1 << 15)))) {
 		link_up = 1;
 		vars->line_speed = ELINK_SPEED_10000;
-		PMD_DRV_LOG(DEBUG, "port %x: External link up in 10G",
-			    params->port);
+		ELINK_DEBUG_P1(sc, "port %x: External link up in 10G",
+			   params->port);
 	} else if ((link_status & (1 << 1)) && (!(link_status & (1 << 14)))) {
 		link_up = 1;
 		vars->line_speed = ELINK_SPEED_2500;
-		PMD_DRV_LOG(DEBUG, "port %x: External link up in 2.5G",
-			    params->port);
+		ELINK_DEBUG_P1(sc, "port %x: External link up in 2.5G",
+			   params->port);
 	} else if ((link_status & (1 << 0)) && (!(link_status & (1 << 13)))) {
 		link_up = 1;
 		vars->line_speed = ELINK_SPEED_1000;
-		PMD_DRV_LOG(DEBUG, "port %x: External link up in 1G",
-			    params->port);
+		ELINK_DEBUG_P1(sc, "port %x: External link up in 1G",
+			   params->port);
 	} else {
 		link_up = 0;
-		PMD_DRV_LOG(DEBUG, "port %x: External link is down",
-			    params->port);
+		ELINK_DEBUG_P1(sc, "port %x: External link is down",
+			   params->port);
 	}
 
 	if (link_up) {
 		/* Swap polarity if required */
-		if (params->lane_config & PORT_HW_CFG_SWAP_PHY_POLARITY_ENABLED) {
+		if (params->lane_config &
+		    PORT_HW_CFG_SWAP_PHY_POLARITY_ENABLED) {
 			/* Configure the 8073 to swap P and N of the KR lines */
 			elink_cl45_read(sc, phy,
 					MDIO_XS_DEVAD,
@@ -7051,15 +8584,16 @@ static uint8_t elink_8073_read_status(struct elink_phy *phy,
 			 * when it`s in 10G mode.
 			 */
 			if (vars->line_speed == ELINK_SPEED_1000) {
-				PMD_DRV_LOG(DEBUG, "Swapping 1G polarity for"
-					    "the 8073");
+				ELINK_DEBUG_P0(sc, "Swapping 1G polarity for"
+					       " the 8073");
 				val1 |= (1 << 3);
 			} else
 				val1 &= ~(1 << 3);
 
 			elink_cl45_write(sc, phy,
 					 MDIO_XS_DEVAD,
-					 MDIO_XS_REG_8073_RX_CTRL_PCIE, val1);
+					 MDIO_XS_REG_8073_RX_CTRL_PCIE,
+					 val1);
 		}
 		elink_ext_phy_10G_an_resolve(sc, phy, vars);
 		elink_8073_resolve_fc(phy, params, vars);
@@ -7072,10 +8606,10 @@ static uint8_t elink_8073_read_status(struct elink_phy *phy,
 
 		if (val1 & (1 << 5))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE;
 		if (val1 & (1 << 7))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
 	}
 
 	return link_up;
@@ -7090,25 +8624,25 @@ static void elink_8073_link_reset(__rte_unused struct elink_phy *phy,
 		gpio_port = SC_PATH(sc);
 	else
 		gpio_port = params->port;
-	PMD_DRV_LOG(DEBUG, "Setting 8073 port %d into low power mode",
-		    gpio_port);
+	ELINK_DEBUG_P1(sc, "Setting 8073 port %d into low power mode",
+	   gpio_port);
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_2,
-			    MISC_REGISTERS_GPIO_OUTPUT_LOW, gpio_port);
+		       MISC_REGISTERS_GPIO_OUTPUT_LOW,
+		       gpio_port);
 }
 
 /******************************************************************/
 /*			BNX2X8705 PHY SECTION			  */
 /******************************************************************/
 static uint8_t elink_8705_config_init(struct elink_phy *phy,
-				      struct elink_params *params,
-				      __rte_unused struct elink_vars
-					     *vars)
+				  struct elink_params *params,
+				  __rte_unused struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "init 8705");
-	/* Restore normal power mode */
+	ELINK_DEBUG_P0(sc, "init 8705");
+	/* Restore normal power mode*/
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_2,
-			    MISC_REGISTERS_GPIO_OUTPUT_HIGH, params->port);
+		       MISC_REGISTERS_GPIO_OUTPUT_HIGH, params->port);
 	/* HW reset */
 	elink_ext_phy_hw_reset(sc, params->port);
 	elink_cl45_write(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 0xa040);
@@ -7120,36 +8654,40 @@ static uint8_t elink_8705_config_init(struct elink_phy *phy,
 			 MDIO_PMA_DEVAD, MDIO_PMA_REG_PHY_IDENTIFIER, 0x7fbf);
 	elink_cl45_write(sc, phy,
 			 MDIO_PMA_DEVAD, MDIO_PMA_REG_CMU_PLL_BYPASS, 0x0100);
-	elink_cl45_write(sc, phy, MDIO_WIS_DEVAD, MDIO_WIS_REG_LASI_CNTL, 0x1);
+	elink_cl45_write(sc, phy,
+			 MDIO_WIS_DEVAD, MDIO_WIS_REG_LASI_CNTL, 0x1);
 	/* BNX2X8705 doesn't have microcode, hence the 0 */
 	elink_save_spirom_version(sc, params->port, params->shmem_base, 0);
 	return ELINK_STATUS_OK;
 }
 
 static uint8_t elink_8705_read_status(struct elink_phy *phy,
-				      struct elink_params *params,
-				      struct elink_vars *vars)
+				 struct elink_params *params,
+				 struct elink_vars *vars)
 {
 	uint8_t link_up = 0;
 	uint16_t val1, rx_sd;
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "read status 8705");
+	ELINK_DEBUG_P0(sc, "read status 8705");
 	elink_cl45_read(sc, phy,
-			MDIO_WIS_DEVAD, MDIO_WIS_REG_LASI_STATUS, &val1);
-	PMD_DRV_LOG(DEBUG, "8705 LASI status 0x%x", val1);
+		      MDIO_WIS_DEVAD, MDIO_WIS_REG_LASI_STATUS, &val1);
+	ELINK_DEBUG_P1(sc, "8705 LASI status 0x%x", val1);
 
 	elink_cl45_read(sc, phy,
-			MDIO_WIS_DEVAD, MDIO_WIS_REG_LASI_STATUS, &val1);
-	PMD_DRV_LOG(DEBUG, "8705 LASI status 0x%x", val1);
+		      MDIO_WIS_DEVAD, MDIO_WIS_REG_LASI_STATUS, &val1);
+	ELINK_DEBUG_P1(sc, "8705 LASI status 0x%x", val1);
 
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_RX_SD, &rx_sd);
+	elink_cl45_read(sc, phy,
+		      MDIO_PMA_DEVAD, MDIO_PMA_REG_RX_SD, &rx_sd);
 
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, 0xc809, &val1);
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, 0xc809, &val1);
+	elink_cl45_read(sc, phy,
+		      MDIO_PMA_DEVAD, 0xc809, &val1);
+	elink_cl45_read(sc, phy,
+		      MDIO_PMA_DEVAD, 0xc809, &val1);
 
-	PMD_DRV_LOG(DEBUG, "8705 1.c809 val=0x%x", val1);
-	link_up = ((rx_sd & 0x1) && (val1 & (1 << 9))
-		   && ((val1 & (1 << 8)) == 0));
+	ELINK_DEBUG_P1(sc, "8705 1.c809 val=0x%x", val1);
+	link_up = ((rx_sd & 0x1) && (val1 & (1 << 9)) &&
+		   ((val1 & (1 << 8)) == 0));
 	if (link_up) {
 		vars->line_speed = ELINK_SPEED_10000;
 		elink_ext_phy_resolve_fc(phy, params, vars);
@@ -7170,17 +8708,17 @@ static void elink_set_disable_pmd_transmit(struct elink_params *params,
 	 */
 	if (pmd_dis) {
 		if (params->feature_config_flags &
-		    ELINK_FEATURE_CONFIG_BC_SUPPORTS_SFP_TX_DISABLED) {
-			PMD_DRV_LOG(DEBUG, "Disabling PMD transmitter");
+		     ELINK_FEATURE_CONFIG_BC_SUPPORTS_SFP_TX_DISABLED) {
+			ELINK_DEBUG_P0(sc, "Disabling PMD transmitter");
 		} else {
-			PMD_DRV_LOG(DEBUG, "NOT disabling PMD transmitter");
+			ELINK_DEBUG_P0(sc, "NOT disabling PMD transmitter");
 			return;
 		}
-	} else {
-		PMD_DRV_LOG(DEBUG, "Enabling PMD transmitter");
-	}
+	} else
+		ELINK_DEBUG_P0(sc, "Enabling PMD transmitter");
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_TX_DISABLE, pmd_dis);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_TX_DISABLE, pmd_dis);
 }
 
 static uint8_t elink_get_gpio_port(struct elink_params *params)
@@ -7188,37 +8726,38 @@ static uint8_t elink_get_gpio_port(struct elink_params *params)
 	uint8_t gpio_port;
 	uint32_t swap_val, swap_override;
 	struct bnx2x_softc *sc = params->sc;
-	if (CHIP_IS_E2(sc)) {
+	if (CHIP_IS_E2(sc))
 		gpio_port = SC_PATH(sc);
-	} else {
+	else
 		gpio_port = params->port;
-	}
 	swap_val = REG_RD(sc, NIG_REG_PORT_SWAP);
 	swap_override = REG_RD(sc, NIG_REG_STRAP_OVERRIDE);
 	return gpio_port ^ (swap_val && swap_override);
 }
 
 static void elink_sfp_e1e2_set_transmitter(struct elink_params *params,
-					   struct elink_phy *phy, uint8_t tx_en)
+					   struct elink_phy *phy,
+					   uint8_t tx_en)
 {
 	uint16_t val;
 	uint8_t port = params->port;
 	struct bnx2x_softc *sc = params->sc;
 	uint32_t tx_en_mode;
 
-	/* Disable/Enable transmitter ( TX laser of the SFP+ module.) */
+	/* Disable/Enable transmitter ( TX laser of the SFP+ module.)*/
 	tx_en_mode = REG_RD(sc, params->shmem_base +
 			    offsetof(struct shmem_region,
 				     dev_info.port_hw_config[port].sfp_ctrl)) &
-	    PORT_HW_CFG_TX_LASER_MASK;
-	PMD_DRV_LOG(DEBUG, "Setting transmitter tx_en=%x for port %x "
-		    "mode = %x", tx_en, port, tx_en_mode);
+		PORT_HW_CFG_TX_LASER_MASK;
+	ELINK_DEBUG_P3(sc, "Setting transmitter tx_en=%x for port %x "
+			   "mode = %x", tx_en, port, tx_en_mode);
 	switch (tx_en_mode) {
 	case PORT_HW_CFG_TX_LASER_MDIO:
 
 		elink_cl45_read(sc, phy,
 				MDIO_PMA_DEVAD,
-				MDIO_PMA_REG_PHY_IDENTIFIER, &val);
+				MDIO_PMA_REG_PHY_IDENTIFIER,
+				&val);
 
 		if (tx_en)
 			val &= ~(1 << 15);
@@ -7227,36 +8766,38 @@ static void elink_sfp_e1e2_set_transmitter(struct elink_params *params,
 
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD,
-				 MDIO_PMA_REG_PHY_IDENTIFIER, val);
-		break;
+				 MDIO_PMA_REG_PHY_IDENTIFIER,
+				 val);
+	break;
 	case PORT_HW_CFG_TX_LASER_GPIO0:
 	case PORT_HW_CFG_TX_LASER_GPIO1:
 	case PORT_HW_CFG_TX_LASER_GPIO2:
 	case PORT_HW_CFG_TX_LASER_GPIO3:
-		{
-			uint16_t gpio_pin;
-			uint8_t gpio_port, gpio_mode;
-			if (tx_en)
-				gpio_mode = MISC_REGISTERS_GPIO_OUTPUT_HIGH;
-			else
-				gpio_mode = MISC_REGISTERS_GPIO_OUTPUT_LOW;
+	{
+		uint16_t gpio_pin;
+		uint8_t gpio_port, gpio_mode;
+		if (tx_en)
+			gpio_mode = MISC_REGISTERS_GPIO_OUTPUT_HIGH;
+		else
+			gpio_mode = MISC_REGISTERS_GPIO_OUTPUT_LOW;
 
-			gpio_pin = tx_en_mode - PORT_HW_CFG_TX_LASER_GPIO0;
-			gpio_port = elink_get_gpio_port(params);
-			elink_cb_gpio_write(sc, gpio_pin, gpio_mode, gpio_port);
-			break;
-		}
+		gpio_pin = tx_en_mode - PORT_HW_CFG_TX_LASER_GPIO0;
+		gpio_port = elink_get_gpio_port(params);
+		elink_cb_gpio_write(sc, gpio_pin, gpio_mode, gpio_port);
+		break;
+	}
 	default:
-		PMD_DRV_LOG(DEBUG, "Invalid TX_LASER_MDIO 0x%x", tx_en_mode);
+		ELINK_DEBUG_P1(sc, "Invalid TX_LASER_MDIO 0x%x", tx_en_mode);
 		break;
 	}
 }
 
 static void elink_sfp_set_transmitter(struct elink_params *params,
-				      struct elink_phy *phy, uint8_t tx_en)
+				      struct elink_phy *phy,
+				      uint8_t tx_en)
 {
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "Setting SFP+ transmitter to %d", tx_en);
+	ELINK_DEBUG_P1(sc, "Setting SFP+ transmitter to %d", tx_en);
 	if (CHIP_IS_E3(sc))
 		elink_sfp_e3_set_transmitter(params, phy, tx_en);
 	else
@@ -7264,20 +8805,17 @@ static void elink_sfp_set_transmitter(struct elink_params *params,
 }
 
 static elink_status_t elink_8726_read_sfp_module_eeprom(struct elink_phy *phy,
-							struct elink_params
-							*params,
-							uint8_t dev_addr,
-							uint16_t addr,
-							uint8_t byte_cnt,
-							uint8_t * o_buf,
-							__rte_unused uint8_t
-							is_init)
+			     struct elink_params *params,
+			     uint8_t dev_addr, uint16_t addr,
+			     uint8_t byte_cnt,
+			     uint8_t *o_buf, __rte_unused uint8_t is_init)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t val = 0;
 	uint16_t i;
 	if (byte_cnt > ELINK_SFP_EEPROM_PAGE_SIZE) {
-		PMD_DRV_LOG(DEBUG, "Reading from eeprom is limited to 0xf");
+		ELINK_DEBUG_P0(sc,
+		   "Reading from eeprom is limited to 0xf");
 		return ELINK_STATUS_ERROR;
 	}
 	/* Set the read command byte count */
@@ -7307,10 +8845,10 @@ static elink_status_t elink_8726_read_sfp_module_eeprom(struct elink_phy *phy,
 	}
 
 	if ((val & MDIO_PMA_REG_SFP_TWO_WIRE_CTRL_STATUS_MASK) !=
-	    MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_COMPLETE) {
-		PMD_DRV_LOG(DEBUG,
-			    "Got bad status 0x%x when reading from SFP+ EEPROM",
-			    (val & MDIO_PMA_REG_SFP_TWO_WIRE_CTRL_STATUS_MASK));
+		    MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_COMPLETE) {
+		ELINK_DEBUG_P1(sc,
+			 "Got bad status 0x%x when reading from SFP+ EEPROM",
+			 (val & MDIO_PMA_REG_SFP_TWO_WIRE_CTRL_STATUS_MASK));
 		return ELINK_STATUS_ERROR;
 	}
 
@@ -7319,8 +8857,8 @@ static elink_status_t elink_8726_read_sfp_module_eeprom(struct elink_phy *phy,
 		elink_cl45_read(sc, phy,
 				MDIO_PMA_DEVAD,
 				MDIO_PMA_REG_8726_TWO_WIRE_DATA_BUF + i, &val);
-		o_buf[i] =
-		    (uint8_t) (val & MDIO_PMA_REG_8726_TWO_WIRE_DATA_MASK);
+		o_buf[i] = (uint8_t)
+				(val & MDIO_PMA_REG_8726_TWO_WIRE_DATA_MASK);
 	}
 
 	for (i = 0; i < 100; i++) {
@@ -7343,29 +8881,27 @@ static void elink_warpcore_power_module(struct elink_params *params,
 
 	pin_cfg = (REG_RD(sc, params->shmem_base +
 			  offsetof(struct shmem_region,
-				   dev_info.port_hw_config[params->port].
-				   e3_sfp_ctrl)) & PORT_HW_CFG_E3_PWR_DIS_MASK)
-	    >> PORT_HW_CFG_E3_PWR_DIS_SHIFT;
+			dev_info.port_hw_config[params->port].e3_sfp_ctrl)) &
+			PORT_HW_CFG_E3_PWR_DIS_MASK) >>
+			PORT_HW_CFG_E3_PWR_DIS_SHIFT;
 
 	if (pin_cfg == PIN_CFG_NA)
 		return;
-	PMD_DRV_LOG(DEBUG, "Setting SFP+ module power to %d using pin cfg %d",
-		    power, pin_cfg);
+	ELINK_DEBUG_P2(sc, "Setting SFP+ module power to %d using pin cfg %d",
+		       power, pin_cfg);
 	/* Low ==> corresponding SFP+ module is powered
 	 * high ==> the SFP+ module is powered down
 	 */
 	elink_set_cfg_pin(sc, pin_cfg, power ^ 1);
 }
-
-static elink_status_t elink_warpcore_read_sfp_module_eeprom(__rte_unused struct
-							    elink_phy *phy,
-							    struct elink_params
-							    *params,
-							    uint8_t dev_addr,
-							    uint16_t addr,
-							    uint8_t byte_cnt,
-							    uint8_t * o_buf,
-							    uint8_t is_init)
+static elink_status_t elink_warpcore_read_sfp_module_eeprom(
+					 __rte_unused struct elink_phy *phy,
+					 struct elink_params *params,
+					 uint8_t dev_addr,
+					 uint16_t addr,
+					 uint8_t byte_cnt,
+					 uint8_t *o_buf,
+					 uint8_t is_init)
 {
 	elink_status_t rc = ELINK_STATUS_OK;
 	uint8_t i, j = 0, cnt = 0;
@@ -7374,8 +8910,8 @@ static elink_status_t elink_warpcore_read_sfp_module_eeprom(__rte_unused struct
 	struct bnx2x_softc *sc = params->sc;
 
 	if (byte_cnt > ELINK_SFP_EEPROM_PAGE_SIZE) {
-		PMD_DRV_LOG(DEBUG,
-			    "Reading from eeprom is limited to 16 bytes");
+		ELINK_DEBUG_P0(sc,
+		   "Reading from eeprom is limited to 16 bytes");
 		return ELINK_STATUS_ERROR;
 	}
 
@@ -7388,13 +8924,15 @@ static elink_status_t elink_warpcore_read_sfp_module_eeprom(__rte_unused struct
 			DELAY(1000 * 1);
 			elink_warpcore_power_module(params, 1);
 		}
-		rc = elink_bsc_read(params, sc, dev_addr, addr32, 0, byte_cnt,
+
+		elink_bsc_module_sel(params);
+		rc = elink_bsc_read(sc, dev_addr, addr32, 0, byte_cnt,
 				    data_array);
 	} while ((rc != ELINK_STATUS_OK) && (++cnt < I2C_WA_RETRY_CNT));
 
 	if (rc == ELINK_STATUS_OK) {
 		for (i = (addr - addr32); i < byte_cnt + (addr - addr32); i++) {
-			o_buf[j] = *((uint8_t *) data_array + i);
+			o_buf[j] = *((uint8_t *)data_array + i);
 			j++;
 		}
 	}
@@ -7403,20 +8941,18 @@ static elink_status_t elink_warpcore_read_sfp_module_eeprom(__rte_unused struct
 }
 
 static elink_status_t elink_8727_read_sfp_module_eeprom(struct elink_phy *phy,
-							struct elink_params
-							*params,
-							uint8_t dev_addr,
-							uint16_t addr,
-							uint8_t byte_cnt,
-							uint8_t * o_buf,
-							__rte_unused uint8_t
-							is_init)
+					     struct elink_params *params,
+					     uint8_t dev_addr, uint16_t addr,
+					     uint8_t byte_cnt,
+					     uint8_t *o_buf,
+					     __rte_unused uint8_t is_init)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t val, i;
 
 	if (byte_cnt > ELINK_SFP_EEPROM_PAGE_SIZE) {
-		PMD_DRV_LOG(DEBUG, "Reading from eeprom is limited to 0xf");
+		ELINK_DEBUG_P0(sc,
+		   "Reading from eeprom is limited to 0xf");
 		return ELINK_STATUS_ERROR;
 	}
 
@@ -7431,7 +8967,9 @@ static elink_status_t elink_8727_read_sfp_module_eeprom(struct elink_phy *phy,
 
 	/* Need to read from 1.8000 to clear it */
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, MDIO_PMA_REG_SFP_TWO_WIRE_CTRL, &val);
+			MDIO_PMA_DEVAD,
+			MDIO_PMA_REG_SFP_TWO_WIRE_CTRL,
+			&val);
 
 	/* Set the read command byte count */
 	elink_cl45_write(sc, phy,
@@ -7442,16 +8980,19 @@ static elink_status_t elink_8727_read_sfp_module_eeprom(struct elink_phy *phy,
 	/* Set the read command address */
 	elink_cl45_write(sc, phy,
 			 MDIO_PMA_DEVAD,
-			 MDIO_PMA_REG_SFP_TWO_WIRE_MEM_ADDR, addr);
+			 MDIO_PMA_REG_SFP_TWO_WIRE_MEM_ADDR,
+			 addr);
 	/* Set the destination address */
 	elink_cl45_write(sc, phy,
 			 MDIO_PMA_DEVAD,
-			 0x8004, MDIO_PMA_REG_8727_TWO_WIRE_DATA_BUF);
+			 0x8004,
+			 MDIO_PMA_REG_8727_TWO_WIRE_DATA_BUF);
 
 	/* Activate read command */
 	elink_cl45_write(sc, phy,
 			 MDIO_PMA_DEVAD,
-			 MDIO_PMA_REG_SFP_TWO_WIRE_CTRL, 0x8002);
+			 MDIO_PMA_REG_SFP_TWO_WIRE_CTRL,
+			 0x8002);
 	/* Wait appropriate time for two-wire command to finish before
 	 * polling the status register
 	 */
@@ -7469,10 +9010,10 @@ static elink_status_t elink_8727_read_sfp_module_eeprom(struct elink_phy *phy,
 	}
 
 	if ((val & MDIO_PMA_REG_SFP_TWO_WIRE_CTRL_STATUS_MASK) !=
-	    MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_COMPLETE) {
-		PMD_DRV_LOG(DEBUG,
-			    "Got bad status 0x%x when reading from SFP+ EEPROM",
-			    (val & MDIO_PMA_REG_SFP_TWO_WIRE_CTRL_STATUS_MASK));
+		    MDIO_PMA_REG_SFP_TWO_WIRE_STATUS_COMPLETE) {
+		ELINK_DEBUG_P1(sc,
+			 "Got bad status 0x%x when reading from SFP+ EEPROM",
+			 (val & MDIO_PMA_REG_SFP_TWO_WIRE_CTRL_STATUS_MASK));
 		return ELINK_STATUS_TIMEOUT;
 	}
 
@@ -7481,8 +9022,8 @@ static elink_status_t elink_8727_read_sfp_module_eeprom(struct elink_phy *phy,
 		elink_cl45_read(sc, phy,
 				MDIO_PMA_DEVAD,
 				MDIO_PMA_REG_8727_TWO_WIRE_DATA_BUF + i, &val);
-		o_buf[i] =
-		    (uint8_t) (val & MDIO_PMA_REG_8727_TWO_WIRE_DATA_MASK);
+		o_buf[i] = (uint8_t)
+				(val & MDIO_PMA_REG_8727_TWO_WIRE_DATA_MASK);
 	}
 
 	for (i = 0; i < 100; i++) {
@@ -7497,21 +9038,18 @@ static elink_status_t elink_8727_read_sfp_module_eeprom(struct elink_phy *phy,
 
 	return ELINK_STATUS_ERROR;
 }
-
-static elink_status_t elink_read_sfp_module_eeprom(struct elink_phy *phy,
-						   struct elink_params *params,
-						   uint8_t dev_addr,
-						   uint16_t addr,
-						   uint16_t byte_cnt,
-						   uint8_t * o_buf)
+elink_status_t elink_read_sfp_module_eeprom(struct elink_phy *phy,
+				 struct elink_params *params, uint8_t dev_addr,
+				 uint16_t addr, uint16_t byte_cnt,
+				 uint8_t *o_buf)
 {
-	elink_status_t rc = ELINK_STATUS_OK;
+	elink_status_t rc = 0;
+	struct bnx2x_softc *sc = params->sc;
 	uint8_t xfer_size;
 	uint8_t *user_data = o_buf;
 	read_sfp_module_eeprom_func_p read_func;
-
 	if ((dev_addr != 0xa0) && (dev_addr != 0xa2)) {
-		PMD_DRV_LOG(DEBUG, "invalid dev_addr 0x%x", dev_addr);
+		ELINK_DEBUG_P1(sc, "invalid dev_addr 0x%x", dev_addr);
 		return ELINK_STATUS_ERROR;
 	}
 
@@ -7532,7 +9070,7 @@ static elink_status_t elink_read_sfp_module_eeprom(struct elink_phy *phy,
 
 	while (!rc && (byte_cnt > 0)) {
 		xfer_size = (byte_cnt > ELINK_SFP_EEPROM_PAGE_SIZE) ?
-		    ELINK_SFP_EEPROM_PAGE_SIZE : byte_cnt;
+			ELINK_SFP_EEPROM_PAGE_SIZE : byte_cnt;
 		rc = read_func(phy, params, dev_addr, addr, xfer_size,
 			       user_data, 0);
 		byte_cnt -= xfer_size;
@@ -7543,91 +9081,105 @@ static elink_status_t elink_read_sfp_module_eeprom(struct elink_phy *phy,
 }
 
 static elink_status_t elink_get_edc_mode(struct elink_phy *phy,
-					 struct elink_params *params,
-					 uint16_t * edc_mode)
+			      struct elink_params *params,
+			      uint16_t *edc_mode)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint32_t sync_offset = 0, phy_idx, media_types;
-	uint8_t gport, val[2], check_limiting_mode = 0;
+	uint8_t val[ELINK_SFP_EEPROM_FC_TX_TECH_ADDR + 1];
+	uint8_t check_limiting_mode = 0;
 	*edc_mode = ELINK_EDC_MODE_LIMITING;
 	phy->media_type = ELINK_ETH_PHY_UNSPECIFIED;
 	/* First check for copper cable */
 	if (elink_read_sfp_module_eeprom(phy,
 					 params,
 					 ELINK_I2C_DEV_ADDR_A0,
-					 ELINK_SFP_EEPROM_CON_TYPE_ADDR,
-					 2, (uint8_t *) val) != 0) {
-		PMD_DRV_LOG(DEBUG, "Failed to read from SFP+ module EEPROM");
+					 0,
+					 ELINK_SFP_EEPROM_FC_TX_TECH_ADDR + 1,
+					 (uint8_t *)val) != 0) {
+		ELINK_DEBUG_P0(sc, "Failed to read from SFP+ module EEPROM");
 		return ELINK_STATUS_ERROR;
 	}
-
-	switch (val[0]) {
+	params->link_attr_sync &= ~LINK_SFP_EEPROM_COMP_CODE_MASK;
+	params->link_attr_sync |= val[ELINK_SFP_EEPROM_10G_COMP_CODE_ADDR] <<
+		LINK_SFP_EEPROM_COMP_CODE_SHIFT;
+	elink_update_link_attr(params, params->link_attr_sync);
+	switch (val[ELINK_SFP_EEPROM_CON_TYPE_ADDR]) {
 	case ELINK_SFP_EEPROM_CON_TYPE_VAL_COPPER:
-		{
-			uint8_t copper_module_type;
-			phy->media_type = ELINK_ETH_PHY_DA_TWINAX;
-			/* Check if its active cable (includes SFP+ module)
-			 * of passive cable
+	{
+		uint8_t copper_module_type;
+		phy->media_type = ELINK_ETH_PHY_DA_TWINAX;
+		/* Check if its active cable (includes SFP+ module)
+		 * of passive cable
+		 */
+		copper_module_type = val[ELINK_SFP_EEPROM_FC_TX_TECH_ADDR];
+		if (copper_module_type &
+		    ELINK_SFP_EEPROM_FC_TX_TECH_BITMASK_COPPER_ACTIVE) {
+			ELINK_DEBUG_P0(sc, "Active Copper cable detected");
+			if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT)
+				*edc_mode = ELINK_EDC_MODE_ACTIVE_DAC;
+			else
+				check_limiting_mode = 1;
+		} else {
+			*edc_mode = ELINK_EDC_MODE_PASSIVE_DAC;
+			/* Even in case PASSIVE_DAC indication is not set,
+			 * treat it as a passive DAC cable, since some cables
+			 * don't have this indication.
 			 */
-			if (elink_read_sfp_module_eeprom(phy,
-							 params,
-							 ELINK_I2C_DEV_ADDR_A0,
-							 ELINK_SFP_EEPROM_FC_TX_TECH_ADDR,
-							 1,
-							 &copper_module_type) !=
-			    0) {
-				PMD_DRV_LOG(DEBUG,
-					    "Failed to read copper-cable-type"
-					    " from SFP+ EEPROM");
-				return ELINK_STATUS_ERROR;
-			}
-
 			if (copper_module_type &
-			    ELINK_SFP_EEPROM_FC_TX_TECH_BITMASK_COPPER_ACTIVE) {
-				PMD_DRV_LOG(DEBUG,
-					    "Active Copper cable detected");
-				if (phy->type ==
-				    PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT)
-					*edc_mode = ELINK_EDC_MODE_ACTIVE_DAC;
-				else
-					check_limiting_mode = 1;
-			} else if (copper_module_type &
-				   ELINK_SFP_EEPROM_FC_TX_TECH_BITMASK_COPPER_PASSIVE)
-			{
-				PMD_DRV_LOG(DEBUG,
-					    "Passive Copper cable detected");
-				*edc_mode = ELINK_EDC_MODE_PASSIVE_DAC;
+			   ELINK_SFP_EEPROM_FC_TX_TECH_BITMASK_COPPER_PASSIVE) {
+				ELINK_DEBUG_P0(sc,
+					       "Passive Copper cable detected");
 			} else {
-				PMD_DRV_LOG(DEBUG,
-					    "Unknown copper-cable-type 0x%x !!!",
-					    copper_module_type);
-				return ELINK_STATUS_ERROR;
+				ELINK_DEBUG_P0(sc,
+					       "Unknown copper-cable-type");
 			}
-			break;
 		}
+		break;
+	}
+	case ELINK_SFP_EEPROM_CON_TYPE_VAL_UNKNOWN:
 	case ELINK_SFP_EEPROM_CON_TYPE_VAL_LC:
 	case ELINK_SFP_EEPROM_CON_TYPE_VAL_RJ45:
 		check_limiting_mode = 1;
-		if ((val[1] & (ELINK_SFP_EEPROM_COMP_CODE_SR_MASK |
-			       ELINK_SFP_EEPROM_COMP_CODE_LR_MASK |
-			       ELINK_SFP_EEPROM_COMP_CODE_LRM_MASK)) == 0) {
-			PMD_DRV_LOG(DEBUG, "1G SFP module detected");
-			gport = params->port;
+		/* Module is considered as 1G in case it's NOT compliant with
+		 * any 10G ethernet protocol, but is 1G Ethernet compliant.
+		 */
+		if (((val[ELINK_SFP_EEPROM_10G_COMP_CODE_ADDR] &
+		      (ELINK_SFP_EEPROM_10G_COMP_CODE_SR_MASK |
+		       ELINK_SFP_EEPROM_10G_COMP_CODE_LR_MASK |
+		       ELINK_SFP_EEPROM_10G_COMP_CODE_LRM_MASK)) == 0) &&
+		    (val[ELINK_SFP_EEPROM_1G_COMP_CODE_ADDR] != 0)) {
+			ELINK_DEBUG_P0(sc, "1G SFP module detected");
 			phy->media_type = ELINK_ETH_PHY_SFP_1G_FIBER;
 			if (phy->req_line_speed != ELINK_SPEED_1000) {
+				uint8_t gport = params->port;
 				phy->req_line_speed = ELINK_SPEED_1000;
 				if (!CHIP_IS_E1x(sc)) {
 					gport = SC_PATH(sc) +
-					    (params->port << 1);
+					(params->port << 1);
 				}
-				elink_cb_event_log(sc, ELINK_LOG_ID_NON_10G_MODULE, gport);	//"Warning: Link speed was forced to 1000Mbps."
-				// " Current SFP module in port %d is not"
-				// " compliant with 10G Ethernet",
+				elink_cb_event_log(sc,
+						   ELINK_LOG_ID_NON_10G_MODULE,
+						   gport);
+				 /*"Warning: Link speed was forced to 1000Mbps."
+				  *" Current SFP module in port %d is not"
+				  *" compliant with 10G Ethernet",
+				  */
+			}
 
+			if (val[ELINK_SFP_EEPROM_1G_COMP_CODE_ADDR] &
+			    ELINK_SFP_EEPROM_1G_COMP_CODE_BASE_T) {
+				/* Some 1G-baseT modules will not link up,
+				 * unless TX_EN is toggled with long delay in
+				 * between.
+				 */
+				elink_sfp_set_transmitter(params, phy, 0);
+				DELAY(1000 * 40);
+				elink_sfp_set_transmitter(params, phy, 1);
 			}
 		} else {
 			int idx, cfg_idx = 0;
-			PMD_DRV_LOG(DEBUG, "10G Optic module detected");
+			ELINK_DEBUG_P0(sc, "10G Optic module detected");
 			for (idx = ELINK_INT_PHY; idx < ELINK_MAX_PHYS; idx++) {
 				if (params->phy[idx].type == phy->type) {
 					cfg_idx = ELINK_LINK_CONFIG_IDX(idx);
@@ -7639,24 +9191,22 @@ static elink_status_t elink_get_edc_mode(struct elink_phy *phy,
 		}
 		break;
 	default:
-		PMD_DRV_LOG(DEBUG, "Unable to determine module type 0x%x !!!",
-			    val[0]);
+		ELINK_DEBUG_P1(sc, "Unable to determine module type 0x%x !!!",
+			 val[ELINK_SFP_EEPROM_CON_TYPE_ADDR]);
 		return ELINK_STATUS_ERROR;
 	}
 	sync_offset = params->shmem_base +
-	    offsetof(struct shmem_region,
-		     dev_info.port_hw_config[params->port].media_type);
+		offsetof(struct shmem_region,
+			 dev_info.port_hw_config[params->port].media_type);
 	media_types = REG_RD(sc, sync_offset);
 	/* Update media type for non-PMF sync */
 	for (phy_idx = ELINK_INT_PHY; phy_idx < ELINK_MAX_PHYS; phy_idx++) {
 		if (&(params->phy[phy_idx]) == phy) {
 			media_types &= ~(PORT_HW_CFG_MEDIA_TYPE_PHY0_MASK <<
-					 (PORT_HW_CFG_MEDIA_TYPE_PHY1_SHIFT *
-					  phy_idx));
-			media_types |=
-			    ((phy->
-			      media_type & PORT_HW_CFG_MEDIA_TYPE_PHY0_MASK) <<
-			     (PORT_HW_CFG_MEDIA_TYPE_PHY1_SHIFT * phy_idx));
+				(PORT_HW_CFG_MEDIA_TYPE_PHY1_SHIFT * phy_idx));
+			media_types |= ((phy->media_type &
+					PORT_HW_CFG_MEDIA_TYPE_PHY0_MASK) <<
+				(PORT_HW_CFG_MEDIA_TYPE_PHY1_SHIFT * phy_idx));
 			break;
 		}
 	}
@@ -7669,8 +9219,8 @@ static elink_status_t elink_get_edc_mode(struct elink_phy *phy,
 						 ELINK_SFP_EEPROM_OPTIONS_ADDR,
 						 ELINK_SFP_EEPROM_OPTIONS_SIZE,
 						 options) != 0) {
-			PMD_DRV_LOG(DEBUG,
-				    "Failed to read Option field from module EEPROM");
+			ELINK_DEBUG_P0(sc,
+			   "Failed to read Option field from module EEPROM");
 			return ELINK_STATUS_ERROR;
 		}
 		if ((options[0] & ELINK_SFP_EEPROM_OPTIONS_LINEAR_RX_OUT_MASK))
@@ -7678,15 +9228,14 @@ static elink_status_t elink_get_edc_mode(struct elink_phy *phy,
 		else
 			*edc_mode = ELINK_EDC_MODE_LIMITING;
 	}
-	PMD_DRV_LOG(DEBUG, "EDC mode is set to 0x%x", *edc_mode);
+	ELINK_DEBUG_P1(sc, "EDC mode is set to 0x%x", *edc_mode);
 	return ELINK_STATUS_OK;
 }
-
 /* This function read the relevant field from the module (SFP+), and verify it
  * is compliant with this board
  */
 static elink_status_t elink_verify_sfp_module(struct elink_phy *phy,
-					      struct elink_params *params)
+				   struct elink_params *params)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint32_t val, cmd;
@@ -7695,12 +9244,11 @@ static elink_status_t elink_verify_sfp_module(struct elink_phy *phy,
 	char vendor_pn[ELINK_SFP_EEPROM_PART_NO_SIZE + 1];
 	phy->flags &= ~ELINK_FLAGS_SFP_NOT_APPROVED;
 	val = REG_RD(sc, params->shmem_base +
-		     offsetof(struct shmem_region,
-			      dev_info.port_feature_config[params->port].
-			      config));
+			 offsetof(struct shmem_region, dev_info.
+				  port_feature_config[params->port].config));
 	if ((val & PORT_FEAT_CFG_OPT_MDL_ENFRCMNT_MASK) ==
 	    PORT_FEAT_CFG_OPT_MDL_ENFRCMNT_NO_ENFORCEMENT) {
-		PMD_DRV_LOG(DEBUG, "NOT enforcing module verification");
+		ELINK_DEBUG_P0(sc, "NOT enforcing module verification");
 		return ELINK_STATUS_OK;
 	}
 
@@ -7710,23 +9258,24 @@ static elink_status_t elink_verify_sfp_module(struct elink_phy *phy,
 		cmd = DRV_MSG_CODE_VRFY_SPECIFIC_PHY_OPT_MDL;
 	} else if (params->feature_config_flags &
 		   ELINK_FEATURE_CONFIG_BC_SUPPORTS_OPT_MDL_VRFY) {
-		/* Use first phy request only in case of non-dual media */
+		/* Use first phy request only in case of non-dual media*/
 		if (ELINK_DUAL_MEDIA(params)) {
-			PMD_DRV_LOG(DEBUG,
-				    "FW does not support OPT MDL verification");
+			ELINK_DEBUG_P0(sc,
+			   "FW does not support OPT MDL verification");
 			return ELINK_STATUS_ERROR;
 		}
 		cmd = DRV_MSG_CODE_VRFY_FIRST_PHY_OPT_MDL;
 	} else {
 		/* No support in OPT MDL detection */
-		PMD_DRV_LOG(DEBUG, "FW does not support OPT MDL verification");
+		ELINK_DEBUG_P0(sc,
+		   "FW does not support OPT MDL verification");
 		return ELINK_STATUS_ERROR;
 	}
 
 	fw_cmd_param = ELINK_FW_PARAM_SET(phy->addr, phy->type, phy->mdio_ctrl);
 	fw_resp = elink_cb_fw_command(sc, cmd, fw_cmd_param);
 	if (fw_resp == FW_MSG_CODE_VRFY_OPT_MDL_SUCCESS) {
-		PMD_DRV_LOG(DEBUG, "Approved module");
+		ELINK_DEBUG_P0(sc, "Approved module");
 		return ELINK_STATUS_OK;
 	}
 
@@ -7736,7 +9285,7 @@ static elink_status_t elink_verify_sfp_module(struct elink_phy *phy,
 					 ELINK_I2C_DEV_ADDR_A0,
 					 ELINK_SFP_EEPROM_VENDOR_NAME_ADDR,
 					 ELINK_SFP_EEPROM_VENDOR_NAME_SIZE,
-					 (uint8_t *) vendor_name))
+					 (uint8_t *)vendor_name))
 		vendor_name[0] = '\0';
 	else
 		vendor_name[ELINK_SFP_EEPROM_VENDOR_NAME_SIZE] = '\0';
@@ -7745,13 +9294,16 @@ static elink_status_t elink_verify_sfp_module(struct elink_phy *phy,
 					 ELINK_I2C_DEV_ADDR_A0,
 					 ELINK_SFP_EEPROM_PART_NO_ADDR,
 					 ELINK_SFP_EEPROM_PART_NO_SIZE,
-					 (uint8_t *) vendor_pn))
+					 (uint8_t *)vendor_pn))
 		vendor_pn[0] = '\0';
 	else
 		vendor_pn[ELINK_SFP_EEPROM_PART_NO_SIZE] = '\0';
 
-	elink_cb_event_log(sc, ELINK_LOG_ID_UNQUAL_IO_MODULE, params->port, vendor_name, vendor_pn);	// "Warning: Unqualified SFP+ module detected,"
-	// " Port %d from %s part number %s",
+	elink_cb_event_log(sc, ELINK_LOG_ID_UNQUAL_IO_MODULE, params->port,
+			   vendor_name, vendor_pn);
+			     /* "Warning: Unqualified SFP+ module detected,"
+			      * " Port %d from %s part number %s",
+			      */
 
 	if ((val & PORT_FEAT_CFG_OPT_MDL_ENFRCMNT_MASK) !=
 	    PORT_FEAT_CFG_OPT_MDL_ENFRCMNT_WARNING_MSG)
@@ -7759,13 +9311,14 @@ static elink_status_t elink_verify_sfp_module(struct elink_phy *phy,
 	return ELINK_STATUS_ERROR;
 }
 
-static elink_status_t elink_wait_for_sfp_module_initialized(struct elink_phy
-							    *phy,
-							    struct elink_params
-							    *params)
+static elink_status_t elink_wait_for_sfp_module_initialized(
+						 struct elink_phy *phy,
+						 struct elink_params *params)
+
 {
 	uint8_t val;
 	elink_status_t rc;
+	struct bnx2x_softc *sc = params->sc;
 	uint16_t timeout;
 	/* Initialization time after hot-plug may take up to 300ms for
 	 * some phys type ( e.g. JDSU )
@@ -7773,18 +9326,17 @@ static elink_status_t elink_wait_for_sfp_module_initialized(struct elink_phy
 
 	for (timeout = 0; timeout < 60; timeout++) {
 		if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT)
-			rc = elink_warpcore_read_sfp_module_eeprom(phy, params,
-								   ELINK_I2C_DEV_ADDR_A0,
-								   1, 1, &val,
-								   1);
+			rc = elink_warpcore_read_sfp_module_eeprom(
+				phy, params, ELINK_I2C_DEV_ADDR_A0, 1, 1, &val,
+				1);
 		else
 			rc = elink_read_sfp_module_eeprom(phy, params,
 							  ELINK_I2C_DEV_ADDR_A0,
 							  1, 1, &val);
 		if (rc == 0) {
-			PMD_DRV_LOG(DEBUG,
-				    "SFP+ module initialization took %d ms",
-				    timeout * 5);
+			ELINK_DEBUG_P1(sc,
+			   "SFP+ module initialization took %d ms",
+			   timeout * 5);
 			return ELINK_STATUS_OK;
 		}
 		DELAY(1000 * 5);
@@ -7795,8 +9347,8 @@ static elink_status_t elink_wait_for_sfp_module_initialized(struct elink_phy
 }
 
 static void elink_8727_power_module(struct bnx2x_softc *sc,
-				    struct elink_phy *phy, uint8_t is_power_up)
-{
+				    struct elink_phy *phy,
+				    uint8_t is_power_up) {
 	/* Make sure GPIOs are not using for LED mode */
 	uint16_t val;
 	/* In the GPIO register, bit 4 is use to determine if the GPIOs are
@@ -7821,29 +9373,33 @@ static void elink_8727_power_module(struct bnx2x_softc *sc,
 		val = (1 << 1);
 
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_GPIO_CTRL, val);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_8727_GPIO_CTRL,
+			 val);
 }
 
 static elink_status_t elink_8726_set_limiting_mode(struct bnx2x_softc *sc,
-						   struct elink_phy *phy,
-						   uint16_t edc_mode)
+					struct elink_phy *phy,
+					uint16_t edc_mode)
 {
 	uint16_t cur_limiting_mode;
 
 	elink_cl45_read(sc, phy,
 			MDIO_PMA_DEVAD,
-			MDIO_PMA_REG_ROM_VER2, &cur_limiting_mode);
-	PMD_DRV_LOG(DEBUG, "Current Limiting mode is 0x%x", cur_limiting_mode);
+			MDIO_PMA_REG_ROM_VER2,
+			&cur_limiting_mode);
+	ELINK_DEBUG_P1(sc, "Current Limiting mode is 0x%x",
+		 cur_limiting_mode);
 
 	if (edc_mode == ELINK_EDC_MODE_LIMITING) {
-		PMD_DRV_LOG(DEBUG, "Setting LIMITING MODE");
+		ELINK_DEBUG_P0(sc, "Setting LIMITING MODE");
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD,
 				 MDIO_PMA_REG_ROM_VER2,
 				 ELINK_EDC_MODE_LIMITING);
-	} else {		/* LRM mode ( default ) */
+	} else { /* LRM mode ( default )*/
 
-		PMD_DRV_LOG(DEBUG, "Setting LRM MODE");
+		ELINK_DEBUG_P0(sc, "Setting LRM MODE");
 
 		/* Changing to LRM mode takes quite few seconds. So do it only
 		 * if current mode is limiting (default is LRM)
@@ -7852,27 +9408,35 @@ static elink_status_t elink_8726_set_limiting_mode(struct bnx2x_softc *sc,
 			return ELINK_STATUS_OK;
 
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_REG_LRM_MODE, 0);
+				 MDIO_PMA_DEVAD,
+				 MDIO_PMA_REG_LRM_MODE,
+				 0);
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_REG_ROM_VER2, 0x128);
+				 MDIO_PMA_DEVAD,
+				 MDIO_PMA_REG_ROM_VER2,
+				 0x128);
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD,
-				 MDIO_PMA_REG_MISC_CTRL0, 0x4008);
+				 MDIO_PMA_REG_MISC_CTRL0,
+				 0x4008);
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_REG_LRM_MODE, 0xaaaa);
+				 MDIO_PMA_DEVAD,
+				 MDIO_PMA_REG_LRM_MODE,
+				 0xaaaa);
 	}
 	return ELINK_STATUS_OK;
 }
 
 static elink_status_t elink_8727_set_limiting_mode(struct bnx2x_softc *sc,
-						   struct elink_phy *phy,
-						   uint16_t edc_mode)
+					struct elink_phy *phy,
+					uint16_t edc_mode)
 {
 	uint16_t phy_identifier;
 	uint16_t rom_ver2_val;
 	elink_cl45_read(sc, phy,
 			MDIO_PMA_DEVAD,
-			MDIO_PMA_REG_PHY_IDENTIFIER, &phy_identifier);
+			MDIO_PMA_REG_PHY_IDENTIFIER,
+			&phy_identifier);
 
 	elink_cl45_write(sc, phy,
 			 MDIO_PMA_DEVAD,
@@ -7880,7 +9444,9 @@ static elink_status_t elink_8727_set_limiting_mode(struct bnx2x_softc *sc,
 			 (phy_identifier & ~(1 << 9)));
 
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, MDIO_PMA_REG_ROM_VER2, &rom_ver2_val);
+			MDIO_PMA_DEVAD,
+			MDIO_PMA_REG_ROM_VER2,
+			&rom_ver2_val);
 	/* Keep the MSB 8-bits, and set the LSB 8-bits with the edc_mode */
 	elink_cl45_write(sc, phy,
 			 MDIO_PMA_DEVAD,
@@ -7914,12 +9480,14 @@ static void elink_8727_specific_func(struct elink_phy *phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXCTRL,
 				 (1 << 2) | (1 << 5));
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_TXCTRL, 0);
+				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_TXCTRL,
+				 0);
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL, 0x0006);
 		/* Make MOD_ABS give interrupt on change */
 		elink_cl45_read(sc, phy, MDIO_PMA_DEVAD,
-				MDIO_PMA_REG_8727_PCS_OPT_CTRL, &val);
+				MDIO_PMA_REG_8727_PCS_OPT_CTRL,
+				&val);
 		val |= (1 << 12);
 		if (phy->flags & ELINK_FLAGS_NOC)
 			val |= (3 << 5);
@@ -7927,29 +9495,27 @@ static void elink_8727_specific_func(struct elink_phy *phy,
 		 * status which reflect SFP+ module over-current
 		 */
 		if (!(phy->flags & ELINK_FLAGS_NOC))
-			val &= 0xff8f;	/* Reset bits 4-6 */
+			val &= 0xff8f; /* Reset bits 4-6 */
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_PCS_OPT_CTRL,
 				 val);
 		break;
 	default:
-		PMD_DRV_LOG(DEBUG, "Function 0x%x not supported by 8727",
-			    action);
+		ELINK_DEBUG_P1(sc, "Function 0x%x not supported by 8727",
+		   action);
 		return;
 	}
 }
 
 static void elink_set_e1e2_module_fault_led(struct elink_params *params,
-					    uint8_t gpio_mode)
+					   uint8_t gpio_mode)
 {
 	struct bnx2x_softc *sc = params->sc;
 
 	uint32_t fault_led_gpio = REG_RD(sc, params->shmem_base +
-					 offsetof(struct shmem_region,
-						  dev_info.
-						  port_hw_config[params->port].
-						  sfp_ctrl)) &
-	    PORT_HW_CFG_FAULT_MODULE_LED_MASK;
+			    offsetof(struct shmem_region,
+			dev_info.port_hw_config[params->port].sfp_ctrl)) &
+		PORT_HW_CFG_FAULT_MODULE_LED_MASK;
 	switch (fault_led_gpio) {
 	case PORT_HW_CFG_FAULT_MODULE_LED_DISABLED:
 		return;
@@ -7957,19 +9523,19 @@ static void elink_set_e1e2_module_fault_led(struct elink_params *params,
 	case PORT_HW_CFG_FAULT_MODULE_LED_GPIO1:
 	case PORT_HW_CFG_FAULT_MODULE_LED_GPIO2:
 	case PORT_HW_CFG_FAULT_MODULE_LED_GPIO3:
-		{
-			uint8_t gpio_port = elink_get_gpio_port(params);
-			uint16_t gpio_pin = fault_led_gpio -
-			    PORT_HW_CFG_FAULT_MODULE_LED_GPIO0;
-			PMD_DRV_LOG(DEBUG, "Set fault module-detected led "
-				    "pin %x port %x mode %x",
-				    gpio_pin, gpio_port, gpio_mode);
-			elink_cb_gpio_write(sc, gpio_pin, gpio_mode, gpio_port);
-		}
-		break;
+	{
+		uint8_t gpio_port = elink_get_gpio_port(params);
+		uint16_t gpio_pin = fault_led_gpio -
+			PORT_HW_CFG_FAULT_MODULE_LED_GPIO0;
+		ELINK_DEBUG_P3(sc, "Set fault module-detected led "
+				   "pin %x port %x mode %x",
+			       gpio_pin, gpio_port, gpio_mode);
+		elink_cb_gpio_write(sc, gpio_pin, gpio_mode, gpio_port);
+	}
+	break;
 	default:
-		PMD_DRV_LOG(DEBUG, "Error: Invalid fault led mode 0x%x",
-			    fault_led_gpio);
+		ELINK_DEBUG_P1(sc, "Error: Invalid fault led mode 0x%x",
+			       fault_led_gpio);
 	}
 }
 
@@ -7980,12 +9546,12 @@ static void elink_set_e3_module_fault_led(struct elink_params *params,
 	uint8_t port = params->port;
 	struct bnx2x_softc *sc = params->sc;
 	pin_cfg = (REG_RD(sc, params->shmem_base +
-			  offsetof(struct shmem_region,
-				   dev_info.port_hw_config[port].e3_sfp_ctrl)) &
-		   PORT_HW_CFG_E3_FAULT_MDL_LED_MASK) >>
-	    PORT_HW_CFG_E3_FAULT_MDL_LED_SHIFT;
-	PMD_DRV_LOG(DEBUG, "Setting Fault LED to %d using pin cfg %d",
-		    gpio_mode, pin_cfg);
+			 offsetof(struct shmem_region,
+				  dev_info.port_hw_config[port].e3_sfp_ctrl)) &
+		PORT_HW_CFG_E3_FAULT_MDL_LED_MASK) >>
+		PORT_HW_CFG_E3_FAULT_MDL_LED_SHIFT;
+	ELINK_DEBUG_P2(sc, "Setting Fault LED to %d using pin cfg %d",
+		       gpio_mode, pin_cfg);
 	elink_set_cfg_pin(sc, pin_cfg, gpio_mode);
 }
 
@@ -7993,7 +9559,7 @@ static void elink_set_sfp_module_fault_led(struct elink_params *params,
 					   uint8_t gpio_mode)
 {
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "Setting SFP+ module fault LED to %d", gpio_mode);
+	ELINK_DEBUG_P1(sc, "Setting SFP+ module fault LED to %d", gpio_mode);
 	if (CHIP_IS_E3(sc)) {
 		/* Low ==> if SFP+ module is supported otherwise
 		 * High ==> if SFP+ module is not on the approved vendor list
@@ -8018,9 +9584,11 @@ static void elink_warpcore_hw_reset(__rte_unused struct elink_phy *phy,
 }
 
 static void elink_power_sfp_module(struct elink_params *params,
-				   struct elink_phy *phy, uint8_t power)
+				   struct elink_phy *phy,
+				   uint8_t power)
 {
-	PMD_DRV_LOG(DEBUG, "Setting SFP+ power to %x", power);
+	struct bnx2x_softc *sc = params->sc;
+	ELINK_DEBUG_P1(sc, "Setting SFP+ power to %x", power);
 
 	switch (phy->type) {
 	case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8727:
@@ -8034,7 +9602,6 @@ static void elink_power_sfp_module(struct elink_params *params,
 		break;
 	}
 }
-
 static void elink_warpcore_set_limiting_mode(struct elink_params *params,
 					     struct elink_phy *phy,
 					     uint16_t edc_mode)
@@ -8043,7 +9610,7 @@ static void elink_warpcore_set_limiting_mode(struct elink_params *params,
 	uint16_t mode = MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE_DEFAULT;
 	struct bnx2x_softc *sc = params->sc;
 
-	uint8_t lane = elink_get_warpcore_lane(params);
+	uint8_t lane = elink_get_warpcore_lane(phy, params);
 	/* This is a global register which controls all lanes */
 	elink_cl45_read(sc, phy, MDIO_WC_DEVAD,
 			MDIO_WC_REG_UC_INFO_B1_FIRMWARE_MODE, &val);
@@ -8076,7 +9643,8 @@ static void elink_warpcore_set_limiting_mode(struct elink_params *params,
 }
 
 static void elink_set_limiting_mode(struct elink_params *params,
-				    struct elink_phy *phy, uint16_t edc_mode)
+				    struct elink_phy *phy,
+				    uint16_t edc_mode)
 {
 	switch (phy->type) {
 	case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8726:
@@ -8092,30 +9660,28 @@ static void elink_set_limiting_mode(struct elink_params *params,
 	}
 }
 
-static elink_status_t elink_sfp_module_detection(struct elink_phy *phy,
-						 struct elink_params *params)
+elink_status_t elink_sfp_module_detection(struct elink_phy *phy,
+			       struct elink_params *params)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t edc_mode;
 	elink_status_t rc = ELINK_STATUS_OK;
 
 	uint32_t val = REG_RD(sc, params->shmem_base +
-			      offsetof(struct shmem_region,
-				       dev_info.port_feature_config[params->
-								    port].
-				       config));
+			     offsetof(struct shmem_region, dev_info.
+				     port_feature_config[params->port].config));
 	/* Enabled transmitter by default */
 	elink_sfp_set_transmitter(params, phy, 1);
-	PMD_DRV_LOG(DEBUG, "SFP+ module plugged in/out detected on port %d",
-		    params->port);
+	ELINK_DEBUG_P1(sc, "SFP+ module plugged in/out detected on port %d",
+		 params->port);
 	/* Power up module */
 	elink_power_sfp_module(params, phy, 1);
 	if (elink_get_edc_mode(phy, params, &edc_mode) != 0) {
-		PMD_DRV_LOG(DEBUG, "Failed to get valid module type");
+		ELINK_DEBUG_P0(sc, "Failed to get valid module type");
 		return ELINK_STATUS_ERROR;
 	} else if (elink_verify_sfp_module(phy, params) != 0) {
 		/* Check SFP+ module compatibility */
-		PMD_DRV_LOG(DEBUG, "Module verification failed!!");
+		ELINK_DEBUG_P0(sc, "Module verification failed!!");
 		rc = ELINK_STATUS_ERROR;
 		/* Turn on fault module-detected led */
 		elink_set_sfp_module_fault_led(params,
@@ -8123,8 +9689,8 @@ static elink_status_t elink_sfp_module_detection(struct elink_phy *phy,
 
 		/* Check if need to power down the SFP+ module */
 		if ((val & PORT_FEAT_CFG_OPT_MDL_ENFRCMNT_MASK) ==
-		    PORT_FEAT_CFG_OPT_MDL_ENFRCMNT_POWER_DOWN) {
-			PMD_DRV_LOG(DEBUG, "Shutdown SFP+ module!!");
+		     PORT_FEAT_CFG_OPT_MDL_ENFRCMNT_POWER_DOWN) {
+			ELINK_DEBUG_P0(sc, "Shutdown SFP+ module!!");
 			elink_power_sfp_module(params, phy, 0);
 			return rc;
 		}
@@ -8157,22 +9723,22 @@ void elink_handle_module_detect_int(struct elink_params *params)
 	uint8_t gpio_num, gpio_port;
 	if (CHIP_IS_E3(sc)) {
 		phy = &params->phy[ELINK_INT_PHY];
-		/* Always enable TX laser,will be disabled in case of fault */
+		/* Always enable TX laser, will be disabled in case of fault */
 		elink_sfp_set_transmitter(params, phy, 1);
 	} else {
 		phy = &params->phy[ELINK_EXT_PHY1];
 	}
-	if (elink_get_mod_abs_int_cfg(sc, params->shmem_base,
+	if (elink_get_mod_abs_int_cfg(sc, params->chip_id, params->shmem_base,
 				      params->port, &gpio_num, &gpio_port) ==
 	    ELINK_STATUS_ERROR) {
-		PMD_DRV_LOG(DEBUG, "Failed to get MOD_ABS interrupt config");
+		ELINK_DEBUG_P0(sc, "Failed to get MOD_ABS interrupt config");
 		return;
 	}
 
 	/* Set valid module led off */
 	elink_set_sfp_module_fault_led(params, MISC_REGISTERS_GPIO_HIGH);
 
-	/* Get current gpio val reflecting module plugged in / out */
+	/* Get current gpio val reflecting module plugged in / out*/
 	gpio_val = elink_cb_gpio_read(sc, gpio_num, gpio_port);
 
 	/* Call the handling function in case module is detected */
@@ -8182,8 +9748,8 @@ void elink_handle_module_detect_int(struct elink_params *params)
 
 		elink_power_sfp_module(params, phy, 1);
 		elink_cb_gpio_int_write(sc, gpio_num,
-					MISC_REGISTERS_GPIO_INT_OUTPUT_CLR,
-					gpio_port);
+				   MISC_REGISTERS_GPIO_INT_OUTPUT_CLR,
+				   gpio_port);
 		if (elink_wait_for_sfp_module_initialized(phy, params) == 0) {
 			elink_sfp_module_detection(phy, params);
 			if (CHIP_IS_E3(sc)) {
@@ -8205,12 +9771,12 @@ void elink_handle_module_detect_int(struct elink_params *params)
 				}
 			}
 		} else {
-			PMD_DRV_LOG(DEBUG, "SFP+ module is not initialized");
+			ELINK_DEBUG_P0(sc, "SFP+ module is not initialized");
 		}
 	} else {
 		elink_cb_gpio_int_write(sc, gpio_num,
-					MISC_REGISTERS_GPIO_INT_OUTPUT_SET,
-					gpio_port);
+				   MISC_REGISTERS_GPIO_INT_OUTPUT_SET,
+				   gpio_port);
 		/* Module was plugged out.
 		 * Disable transmit for this module
 		 */
@@ -8228,9 +9794,11 @@ static void elink_sfp_mask_fault(struct bnx2x_softc *sc,
 {
 	uint16_t alarm_status, val;
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, alarm_status_offset, &alarm_status);
+			MDIO_PMA_DEVAD, alarm_status_offset,
+			&alarm_status);
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, alarm_status_offset, &alarm_status);
+			MDIO_PMA_DEVAD, alarm_status_offset,
+			&alarm_status);
 	/* Mask or enable the fault event. */
 	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, alarm_ctrl_offset, &val);
 	if (alarm_status & (1 << 0))
@@ -8239,37 +9807,42 @@ static void elink_sfp_mask_fault(struct bnx2x_softc *sc,
 		val |= (1 << 0);
 	elink_cl45_write(sc, phy, MDIO_PMA_DEVAD, alarm_ctrl_offset, val);
 }
-
 /******************************************************************/
 /*		common BNX2X8706/BNX2X8726 PHY SECTION		  */
 /******************************************************************/
 static uint8_t elink_8706_8726_read_status(struct elink_phy *phy,
-					   struct elink_params *params,
-					   struct elink_vars *vars)
+				      struct elink_params *params,
+				      struct elink_vars *vars)
 {
 	uint8_t link_up = 0;
 	uint16_t val1, val2, rx_sd, pcs_status;
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "XGXS 8706/8726");
-	/* Clear RX Alarm */
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXSTAT, &val2);
+	ELINK_DEBUG_P0(sc, "XGXS 8706/8726");
+	/* Clear RX Alarm*/
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXSTAT, &val2);
 
 	elink_sfp_mask_fault(sc, phy, MDIO_PMA_LASI_TXSTAT,
 			     MDIO_PMA_LASI_TXCTRL);
 
-	/* Clear LASI indication */
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val1);
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val2);
-	PMD_DRV_LOG(DEBUG, "8706/8726 LASI status 0x%x--> 0x%x", val1, val2);
+	/* Clear LASI indication*/
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val1);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val2);
+	ELINK_DEBUG_P2(sc, "8706/8726 LASI status 0x%x--> 0x%x", val1, val2);
 
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_RX_SD, &rx_sd);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_RX_SD, &rx_sd);
 	elink_cl45_read(sc, phy,
 			MDIO_PCS_DEVAD, MDIO_PCS_REG_STATUS, &pcs_status);
-	elink_cl45_read(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_LINK_STATUS, &val2);
-	elink_cl45_read(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_LINK_STATUS, &val2);
+	elink_cl45_read(sc, phy,
+			MDIO_AN_DEVAD, MDIO_AN_REG_LINK_STATUS, &val2);
+	elink_cl45_read(sc, phy,
+			MDIO_AN_DEVAD, MDIO_AN_REG_LINK_STATUS, &val2);
 
-	PMD_DRV_LOG(DEBUG, "8706/8726 rx_sd 0x%x pcs_status 0x%x 1Gbps"
-		    " link_status 0x%x", rx_sd, pcs_status, val2);
+	ELINK_DEBUG_P3(sc, "8706/8726 rx_sd 0x%x pcs_status 0x%x 1Gbps"
+			" link_status 0x%x", rx_sd, pcs_status, val2);
 	/* Link is up if both bit 0 of pmd_rx_sd and bit 0 of pcs_status
 	 * are set, or if the autoneg bit 1 is set
 	 */
@@ -8286,9 +9859,9 @@ static uint8_t elink_8706_8726_read_status(struct elink_phy *phy,
 	/* Capture 10G link fault. Read twice to clear stale value. */
 	if (vars->line_speed == ELINK_SPEED_10000) {
 		elink_cl45_read(sc, phy, MDIO_PMA_DEVAD,
-				MDIO_PMA_LASI_TXSTAT, &val1);
+			    MDIO_PMA_LASI_TXSTAT, &val1);
 		elink_cl45_read(sc, phy, MDIO_PMA_DEVAD,
-				MDIO_PMA_LASI_TXSTAT, &val1);
+			    MDIO_PMA_LASI_TXSTAT, &val1);
 		if (val1 & (1 << 0))
 			vars->fault_detected = 1;
 	}
@@ -8300,15 +9873,15 @@ static uint8_t elink_8706_8726_read_status(struct elink_phy *phy,
 /*			BNX2X8706 PHY SECTION			  */
 /******************************************************************/
 static uint8_t elink_8706_config_init(struct elink_phy *phy,
-				      struct elink_params *params,
-				      __rte_unused struct elink_vars *vars)
+				 struct elink_params *params,
+				 __rte_unused struct elink_vars *vars)
 {
 	uint32_t tx_en_mode;
 	uint16_t cnt, val, tmp1;
 	struct bnx2x_softc *sc = params->sc;
 
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_2,
-			    MISC_REGISTERS_GPIO_OUTPUT_HIGH, params->port);
+		       MISC_REGISTERS_GPIO_OUTPUT_HIGH, params->port);
 	/* HW reset */
 	elink_ext_phy_hw_reset(sc, params->port);
 	elink_cl45_write(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 0xa040);
@@ -8322,34 +9895,35 @@ static uint8_t elink_8706_config_init(struct elink_phy *phy,
 			break;
 		DELAY(1000 * 10);
 	}
-	PMD_DRV_LOG(DEBUG, "XGXS 8706 is initialized after %d ms", cnt);
+	ELINK_DEBUG_P1(sc, "XGXS 8706 is initialized after %d ms", cnt);
 	if ((params->feature_config_flags &
 	     ELINK_FEATURE_CONFIG_OVERRIDE_PREEMPHASIS_ENABLED)) {
 		uint8_t i;
 		uint16_t reg;
 		for (i = 0; i < 4; i++) {
 			reg = MDIO_XS_8706_REG_BANK_RX0 +
-			    i * (MDIO_XS_8706_REG_BANK_RX1 -
-				 MDIO_XS_8706_REG_BANK_RX0);
+				i * (MDIO_XS_8706_REG_BANK_RX1 -
+				     MDIO_XS_8706_REG_BANK_RX0);
 			elink_cl45_read(sc, phy, MDIO_XS_DEVAD, reg, &val);
 			/* Clear first 3 bits of the control */
 			val &= ~0x7;
 			/* Set control bits according to configuration */
 			val |= (phy->rx_preemphasis[i] & 0x7);
-			PMD_DRV_LOG(DEBUG, "Setting RX Equalizer to BNX2X8706"
-				    " reg 0x%x <-- val 0x%x", reg, val);
+			ELINK_DEBUG_P2(sc, "Setting RX Equalizer to BNX2X8706"
+				   " reg 0x%x <-- val 0x%x", reg, val);
 			elink_cl45_write(sc, phy, MDIO_XS_DEVAD, reg, val);
 		}
 	}
 	/* Force speed */
 	if (phy->req_line_speed == ELINK_SPEED_10000) {
-		PMD_DRV_LOG(DEBUG, "XGXS 8706 force 10Gbps");
+		ELINK_DEBUG_P0(sc, "XGXS 8706 force 10Gbps");
 
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD,
 				 MDIO_PMA_REG_DIGITAL_CTRL, 0x400);
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_TXCTRL, 0);
+				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_TXCTRL,
+				 0);
 		/* Arm LASI for link and Tx fault. */
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL, 3);
@@ -8357,7 +9931,7 @@ static uint8_t elink_8706_config_init(struct elink_phy *phy,
 		/* Force 1Gbps using autoneg with 1G advertisement */
 
 		/* Allow CL37 through CL73 */
-		PMD_DRV_LOG(DEBUG, "XGXS 8706 AutoNeg");
+		ELINK_DEBUG_P0(sc, "XGXS 8706 AutoNeg");
 		elink_cl45_write(sc, phy,
 				 MDIO_AN_DEVAD, MDIO_AN_REG_CL37_CL73, 0x040c);
 
@@ -8375,9 +9949,11 @@ static uint8_t elink_8706_config_init(struct elink_phy *phy,
 		elink_cl45_write(sc, phy,
 				 MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, 0x1200);
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXCTRL, 0x0400);
+				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXCTRL,
+				 0x0400);
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL, 0x0004);
+				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL,
+				 0x0004);
 	}
 	elink_save_bnx2x_spirom_ver(sc, phy, params->port);
 
@@ -8387,27 +9963,24 @@ static uint8_t elink_8706_config_init(struct elink_phy *phy,
 
 	tx_en_mode = REG_RD(sc, params->shmem_base +
 			    offsetof(struct shmem_region,
-				     dev_info.port_hw_config[params->port].
-				     sfp_ctrl))
-	& PORT_HW_CFG_TX_LASER_MASK;
+				dev_info.port_hw_config[params->port].sfp_ctrl))
+			& PORT_HW_CFG_TX_LASER_MASK;
 
 	if (tx_en_mode == PORT_HW_CFG_TX_LASER_GPIO0) {
-		PMD_DRV_LOG(DEBUG, "Enabling TXONOFF_PWRDN_DIS");
+		ELINK_DEBUG_P0(sc, "Enabling TXONOFF_PWRDN_DIS");
 		elink_cl45_read(sc, phy,
-				MDIO_PMA_DEVAD, MDIO_PMA_REG_DIGITAL_CTRL,
-				&tmp1);
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_DIGITAL_CTRL, &tmp1);
 		tmp1 |= 0x1;
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_REG_DIGITAL_CTRL,
-				 tmp1);
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_DIGITAL_CTRL, tmp1);
 	}
 
 	return ELINK_STATUS_OK;
 }
 
 static uint8_t elink_8706_read_status(struct elink_phy *phy,
-				      struct elink_params *params,
-				      struct elink_vars *vars)
+				  struct elink_params *params,
+				  struct elink_vars *vars)
 {
 	return elink_8706_8726_read_status(phy, params, vars);
 }
@@ -8419,7 +9992,7 @@ static void elink_8726_config_loopback(struct elink_phy *phy,
 				       struct elink_params *params)
 {
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "PMA/PMD ext_phy_loopback: 8726");
+	ELINK_DEBUG_P0(sc, "PMA/PMD ext_phy_loopback: 8726");
 	elink_cl45_write(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 0x0001);
 }
 
@@ -8441,7 +10014,8 @@ static void elink_8726_external_rom_boot(struct elink_phy *phy,
 			 MDIO_PMA_REG_GEN_CTRL_ROM_MICRO_RESET);
 
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_MISC_CTRL1, 0x0001);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_MISC_CTRL1, 0x0001);
 
 	elink_cl45_write(sc, phy,
 			 MDIO_PMA_DEVAD,
@@ -8453,15 +10027,16 @@ static void elink_8726_external_rom_boot(struct elink_phy *phy,
 
 	/* Disable serial boot control, tristates pins SS_N, SCK, MOSI, MISO */
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_MISC_CTRL1, 0x0000);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_MISC_CTRL1, 0x0000);
 
 	DELAY(1000 * 200);
 	elink_save_bnx2x_spirom_ver(sc, phy, params->port);
 }
 
 static uint8_t elink_8726_read_status(struct elink_phy *phy,
-				      struct elink_params *params,
-				      struct elink_vars *vars)
+				 struct elink_params *params,
+				 struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t val1;
@@ -8471,7 +10046,7 @@ static uint8_t elink_8726_read_status(struct elink_phy *phy,
 				MDIO_PMA_DEVAD, MDIO_PMA_REG_PHY_IDENTIFIER,
 				&val1);
 		if (val1 & (1 << 15)) {
-			PMD_DRV_LOG(DEBUG, "Tx is disabled");
+			ELINK_DEBUG_P0(sc, "Tx is disabled");
 			link_up = 0;
 			vars->line_speed = 0;
 		}
@@ -8479,12 +10054,13 @@ static uint8_t elink_8726_read_status(struct elink_phy *phy,
 	return link_up;
 }
 
+
 static uint8_t elink_8726_config_init(struct elink_phy *phy,
-				      struct elink_params *params,
-				      struct elink_vars *vars)
+				  struct elink_params *params,
+				  struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "Initializing BNX2X8726");
+	ELINK_DEBUG_P0(sc, "Initializing BNX2X8726");
 
 	elink_cl45_write(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 1 << 15);
 	elink_wait_reset_complete(sc, phy, params);
@@ -8499,7 +10075,7 @@ static uint8_t elink_8726_config_init(struct elink_phy *phy,
 	elink_sfp_module_detection(phy, params);
 
 	if (phy->req_line_speed == ELINK_SPEED_1000) {
-		PMD_DRV_LOG(DEBUG, "Setting 1G force");
+		ELINK_DEBUG_P0(sc, "Setting 1G force");
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 0x40);
 		elink_cl45_write(sc, phy,
@@ -8507,17 +10083,19 @@ static uint8_t elink_8726_config_init(struct elink_phy *phy,
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL, 0x5);
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXCTRL, 0x400);
+				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXCTRL,
+				 0x400);
 	} else if ((phy->req_line_speed == ELINK_SPEED_AUTO_NEG) &&
 		   (phy->speed_cap_mask &
-		    PORT_HW_CFG_SPEED_CAPABILITY_D0_1G) &&
+		      PORT_HW_CFG_SPEED_CAPABILITY_D0_1G) &&
 		   ((phy->speed_cap_mask &
-		     PORT_HW_CFG_SPEED_CAPABILITY_D0_10G) !=
+		      PORT_HW_CFG_SPEED_CAPABILITY_D0_10G) !=
 		    PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)) {
-		PMD_DRV_LOG(DEBUG, "Setting 1G clause37");
+		ELINK_DEBUG_P0(sc, "Setting 1G clause37");
 		/* Set Flow control */
 		elink_ext_phy_set_pause(params, phy, vars);
-		elink_cl45_write(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_ADV, 0x20);
+		elink_cl45_write(sc, phy,
+				 MDIO_AN_DEVAD, MDIO_AN_REG_ADV, 0x20);
 		elink_cl45_write(sc, phy,
 				 MDIO_AN_DEVAD, MDIO_AN_REG_CL37_CL73, 0x040c);
 		elink_cl45_write(sc, phy,
@@ -8525,16 +10103,17 @@ static uint8_t elink_8726_config_init(struct elink_phy *phy,
 		elink_cl45_write(sc, phy,
 				 MDIO_AN_DEVAD, MDIO_AN_REG_CL37_AN, 0x1000);
 		elink_cl45_write(sc, phy,
-				 MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, 0x1200);
+				MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, 0x1200);
 		/* Enable RX-ALARM control to receive interrupt for 1G speed
 		 * change
 		 */
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL, 0x4);
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXCTRL, 0x400);
+				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXCTRL,
+				 0x400);
 
-	} else {		/* Default 10G. Set only LASI control */
+	} else { /* Default 10G. Set only LASI control */
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL, 1);
 	}
@@ -8542,9 +10121,10 @@ static uint8_t elink_8726_config_init(struct elink_phy *phy,
 	/* Set TX PreEmphasis if needed */
 	if ((params->feature_config_flags &
 	     ELINK_FEATURE_CONFIG_OVERRIDE_PREEMPHASIS_ENABLED)) {
-		PMD_DRV_LOG(DEBUG,
-			    "Setting TX_CTRL1 0x%x, TX_CTRL2 0x%x",
-			    phy->tx_preemphasis[0], phy->tx_preemphasis[1]);
+		ELINK_DEBUG_P2(sc,
+		   "Setting TX_CTRL1 0x%x, TX_CTRL2 0x%x",
+			 phy->tx_preemphasis[0],
+			 phy->tx_preemphasis[1]);
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD,
 				 MDIO_PMA_REG_8726_TX_CTRL1,
@@ -8564,10 +10144,11 @@ static void elink_8726_link_reset(struct elink_phy *phy,
 				  struct elink_params *params)
 {
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "elink_8726_link_reset port %d", params->port);
+	ELINK_DEBUG_P1(sc, "elink_8726_link_reset port %d", params->port);
 	/* Set serial boot control for external load */
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_GEN_CTRL, 0x0001);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_GEN_CTRL, 0x0001);
 }
 
 /******************************************************************/
@@ -8600,22 +10181,28 @@ static void elink_8727_set_link_led(struct elink_phy *phy,
 		break;
 	}
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_PCS_OPT_CTRL, &val);
+			MDIO_PMA_DEVAD,
+			MDIO_PMA_REG_8727_PCS_OPT_CTRL,
+			&val);
 	val &= 0xff8f;
 	val |= led_mode_bitmask;
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_PCS_OPT_CTRL, val);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_8727_PCS_OPT_CTRL,
+			 val);
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_GPIO_CTRL, &val);
+			MDIO_PMA_DEVAD,
+			MDIO_PMA_REG_8727_GPIO_CTRL,
+			&val);
 	val &= 0xffe0;
 	val |= gpio_pins_bitmask;
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_GPIO_CTRL, val);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_8727_GPIO_CTRL,
+			 val);
 }
-
 static void elink_8727_hw_reset(__rte_unused struct elink_phy *phy,
-				struct elink_params *params)
-{
+				struct elink_params *params) {
 	uint32_t swap_val, swap_override;
 	uint8_t port;
 	/* The PHY reset is controlled by GPIO 1. Fake the port number
@@ -8626,7 +10213,7 @@ static void elink_8727_hw_reset(__rte_unused struct elink_phy *phy,
 	swap_override = REG_RD(sc, NIG_REG_STRAP_OVERRIDE);
 	port = (swap_val && swap_override) ^ 1;
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_1,
-			    MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
+		       MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
 }
 
 static void elink_8727_config_speed(struct elink_phy *phy,
@@ -8637,14 +10224,14 @@ static void elink_8727_config_speed(struct elink_phy *phy,
 	/* Set option 1G speed */
 	if ((phy->req_line_speed == ELINK_SPEED_1000) ||
 	    (phy->media_type == ELINK_ETH_PHY_SFP_1G_FIBER)) {
-		PMD_DRV_LOG(DEBUG, "Setting 1G force");
+		ELINK_DEBUG_P0(sc, "Setting 1G force");
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 0x40);
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_REG_10G_CTRL2, 0xD);
 		elink_cl45_read(sc, phy,
 				MDIO_PMA_DEVAD, MDIO_PMA_REG_10G_CTRL2, &tmp1);
-		PMD_DRV_LOG(DEBUG, "1.7 = 0x%x", tmp1);
+		ELINK_DEBUG_P1(sc, "1.7 = 0x%x", tmp1);
 		/* Power down the XAUI until link is up in case of dual-media
 		 * and 1G
 		 */
@@ -8661,10 +10248,10 @@ static void elink_8727_config_speed(struct elink_phy *phy,
 		   ((phy->speed_cap_mask &
 		     PORT_HW_CFG_SPEED_CAPABILITY_D0_1G)) &&
 		   ((phy->speed_cap_mask &
-		     PORT_HW_CFG_SPEED_CAPABILITY_D0_10G) !=
-		    PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)) {
+		      PORT_HW_CFG_SPEED_CAPABILITY_D0_10G) !=
+		   PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)) {
 
-		PMD_DRV_LOG(DEBUG, "Setting 1G clause37");
+		ELINK_DEBUG_P0(sc, "Setting 1G clause37");
 		elink_cl45_write(sc, phy,
 				 MDIO_AN_DEVAD, MDIO_AN_REG_8727_MISC_CTRL, 0);
 		elink_cl45_write(sc, phy,
@@ -8687,9 +10274,8 @@ static void elink_8727_config_speed(struct elink_phy *phy,
 }
 
 static uint8_t elink_8727_config_init(struct elink_phy *phy,
-				      struct elink_params *params,
-				      __rte_unused struct elink_vars
-					     *vars)
+				  struct elink_params *params,
+				  __rte_unused struct elink_vars *vars)
 {
 	uint32_t tx_en_mode;
 	uint16_t tmp1, mod_abs, tmp2;
@@ -8698,7 +10284,7 @@ static uint8_t elink_8727_config_init(struct elink_phy *phy,
 
 	elink_wait_reset_complete(sc, phy, params);
 
-	PMD_DRV_LOG(DEBUG, "Initializing BNX2X8727");
+	ELINK_DEBUG_P0(sc, "Initializing BNX2X8727");
 
 	elink_8727_specific_func(phy, params, ELINK_PHY_INIT);
 	/* Initially configure MOD_ABS to interrupt when module is
@@ -8724,15 +10310,18 @@ static uint8_t elink_8727_config_init(struct elink_phy *phy,
 	elink_cl45_read(sc, phy,
 			MDIO_PMA_DEVAD, MDIO_PMA_REG_M8051_MSGOUT_REG, &tmp1);
 
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXSTAT, &tmp1);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXSTAT, &tmp1);
 
 	elink_8727_config_speed(phy, params);
 
+
 	/* Set TX PreEmphasis if needed */
 	if ((params->feature_config_flags &
 	     ELINK_FEATURE_CONFIG_OVERRIDE_PREEMPHASIS_ENABLED)) {
-		PMD_DRV_LOG(DEBUG, "Setting TX_CTRL1 0x%x, TX_CTRL2 0x%x",
-			    phy->tx_preemphasis[0], phy->tx_preemphasis[1]);
+		ELINK_DEBUG_P2(sc, "Setting TX_CTRL1 0x%x, TX_CTRL2 0x%x",
+			   phy->tx_preemphasis[0],
+			   phy->tx_preemphasis[1]);
 		elink_cl45_write(sc, phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_TX_CTRL1,
 				 phy->tx_preemphasis[0]);
@@ -8747,25 +10336,24 @@ static uint8_t elink_8727_config_init(struct elink_phy *phy,
 	 */
 	tx_en_mode = REG_RD(sc, params->shmem_base +
 			    offsetof(struct shmem_region,
-				     dev_info.port_hw_config[params->port].
-				     sfp_ctrl))
-	& PORT_HW_CFG_TX_LASER_MASK;
+				dev_info.port_hw_config[params->port].sfp_ctrl))
+			& PORT_HW_CFG_TX_LASER_MASK;
 
 	if (tx_en_mode == PORT_HW_CFG_TX_LASER_GPIO0) {
 
-		PMD_DRV_LOG(DEBUG, "Enabling TXONOFF_PWRDN_DIS");
+		ELINK_DEBUG_P0(sc, "Enabling TXONOFF_PWRDN_DIS");
 		elink_cl45_read(sc, phy,
-				MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_OPT_CFG_REG,
-				&tmp2);
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_OPT_CFG_REG, &tmp2);
 		tmp2 |= 0x1000;
 		tmp2 &= 0xFFEF;
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_OPT_CFG_REG,
-				 tmp2);
-		elink_cl45_read(sc, phy, MDIO_PMA_DEVAD,
-				MDIO_PMA_REG_PHY_IDENTIFIER, &tmp2);
-		elink_cl45_write(sc, phy, MDIO_PMA_DEVAD,
-				 MDIO_PMA_REG_PHY_IDENTIFIER, (tmp2 & 0x7fff));
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_OPT_CFG_REG, tmp2);
+		elink_cl45_read(sc, phy,
+				MDIO_PMA_DEVAD, MDIO_PMA_REG_PHY_IDENTIFIER,
+				&tmp2);
+		elink_cl45_write(sc, phy,
+				 MDIO_PMA_DEVAD, MDIO_PMA_REG_PHY_IDENTIFIER,
+				 (tmp2 & 0x7fff));
 	}
 
 	return ELINK_STATUS_OK;
@@ -8777,15 +10365,17 @@ static void elink_8727_handle_mod_abs(struct elink_phy *phy,
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t mod_abs, rx_alarm_status;
 	uint32_t val = REG_RD(sc, params->shmem_base +
-			      offsetof(struct shmem_region,
-				       dev_info.port_feature_config[params->
-								    port].config));
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_PHY_IDENTIFIER,
-			&mod_abs);
+			     offsetof(struct shmem_region, dev_info.
+				      port_feature_config[params->port].
+				      config));
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD,
+			MDIO_PMA_REG_PHY_IDENTIFIER, &mod_abs);
 	if (mod_abs & (1 << 8)) {
 
 		/* Module is absent */
-		PMD_DRV_LOG(DEBUG, "MOD_ABS indication show module is absent");
+		ELINK_DEBUG_P0(sc,
+		   "MOD_ABS indication show module is absent");
 		phy->media_type = ELINK_ETH_PHY_NOT_PRESENT;
 		/* 1. Set mod_abs to detect next module
 		 *    presence event
@@ -8810,7 +10400,8 @@ static void elink_8727_handle_mod_abs(struct elink_phy *phy,
 
 	} else {
 		/* Module is present */
-		PMD_DRV_LOG(DEBUG, "MOD_ABS indication show module is present");
+		ELINK_DEBUG_P0(sc,
+		   "MOD_ABS indication show module is present");
 		/* First disable transmitter, and if the module is ok, the
 		 * module_detection will enable it
 		 * 1. Set mod_abs to detect next module absent event ( bit 8)
@@ -8834,51 +10425,56 @@ static void elink_8727_handle_mod_abs(struct elink_phy *phy,
 				MDIO_PMA_DEVAD,
 				MDIO_PMA_LASI_RXSTAT, &rx_alarm_status);
 
+
 		if ((val & PORT_FEAT_CFG_OPT_MDL_ENFRCMNT_MASK) ==
 		    PORT_FEAT_CFG_OPT_MDL_ENFRCMNT_DISABLE_TX_LASER)
 			elink_sfp_set_transmitter(params, phy, 0);
 
-		if (elink_wait_for_sfp_module_initialized(phy, params) == 0) {
+		if (elink_wait_for_sfp_module_initialized(phy, params) == 0)
 			elink_sfp_module_detection(phy, params);
-		} else {
-			PMD_DRV_LOG(DEBUG, "SFP+ module is not initialized");
-		}
+		else
+			ELINK_DEBUG_P0(sc, "SFP+ module is not initialized");
 
 		/* Reconfigure link speed based on module type limitations */
 		elink_8727_config_speed(phy, params);
 	}
 
-	PMD_DRV_LOG(DEBUG, "8727 RX_ALARM_STATUS 0x%x", rx_alarm_status);
+	ELINK_DEBUG_P1(sc, "8727 RX_ALARM_STATUS 0x%x",
+		   rx_alarm_status);
 	/* No need to check link status in case of module plugged in/out */
 }
 
 static uint8_t elink_8727_read_status(struct elink_phy *phy,
-				      struct elink_params *params,
-				      struct elink_vars *vars)
+				 struct elink_params *params,
+				 struct elink_vars *vars)
+
 {
 	struct bnx2x_softc *sc = params->sc;
-	uint8_t link_up = 0, oc_port = params->port;
+	uint8_t link_up = 0;
 	uint16_t link_status = 0;
 	uint16_t rx_alarm_status, lasi_ctrl, val1;
 
 	/* If PHY is not initialized, do not check link status */
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL, &lasi_ctrl);
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL,
+			&lasi_ctrl);
 	if (!lasi_ctrl)
 		return 0;
 
 	/* Check the LASI on Rx */
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXSTAT, &rx_alarm_status);
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_RXSTAT,
+			&rx_alarm_status);
 	vars->line_speed = 0;
-	PMD_DRV_LOG(DEBUG, "8727 RX_ALARM_STATUS  0x%x", rx_alarm_status);
+	ELINK_DEBUG_P1(sc, "8727 RX_ALARM_STATUS  0x%x", rx_alarm_status);
 
 	elink_sfp_mask_fault(sc, phy, MDIO_PMA_LASI_TXSTAT,
 			     MDIO_PMA_LASI_TXCTRL);
 
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val1);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val1);
 
-	PMD_DRV_LOG(DEBUG, "8727 LASI status 0x%x", val1);
+	ELINK_DEBUG_P1(sc, "8727 LASI status 0x%x", val1);
 
 	/* Clear MSG-OUT */
 	elink_cl45_read(sc, phy,
@@ -8888,24 +10484,28 @@ static uint8_t elink_8727_read_status(struct elink_phy *phy,
 	 * for over current
 	 */
 	if (!(phy->flags & ELINK_FLAGS_NOC) && !(rx_alarm_status & (1 << 5))) {
-		/* Check over-current using 8727 GPIO0 input */
+		/* Check over-current using 8727 GPIO0 input*/
 		elink_cl45_read(sc, phy,
 				MDIO_PMA_DEVAD, MDIO_PMA_REG_8727_GPIO_CTRL,
 				&val1);
 
 		if ((val1 & (1 << 8)) == 0) {
+			uint8_t oc_port = params->port;
 			if (!CHIP_IS_E1x(sc))
 				oc_port = SC_PATH(sc) + (params->port << 1);
-			PMD_DRV_LOG(DEBUG,
-				    "8727 Power fault has been detected on port %d",
-				    oc_port);
-			elink_cb_event_log(sc, ELINK_LOG_ID_OVER_CURRENT, oc_port);	//"Error: Power fault on Port %d has "
-			//  "been detected and the power to "
-			//  "that SFP+ module has been removed "
-			//  "to prevent failure of the card. "
-			//  "Please remove the SFP+ module and "
-			//  "restart the system to clear this "
-			//  "error.",
+			ELINK_DEBUG_P1(sc,
+			   "8727 Power fault has been detected on port %d",
+			   oc_port);
+			elink_cb_event_log(sc, ELINK_LOG_ID_OVER_CURRENT,
+					   oc_port);
+					/* "Error: Power fault on Port %d has "
+					 *  "been detected and the power to "
+					 *  "that SFP+ module has been removed "
+					 *  "to prevent failure of the card. "
+					 *  "Please remove the SFP+ module and "
+					 *  "restart the system to clear this "
+					 *  "error.",
+					 */
 			/* Disable all RX_ALARMs except for mod_abs */
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
@@ -8921,14 +10521,13 @@ static uint8_t elink_8727_read_status(struct elink_phy *phy,
 					 MDIO_PMA_REG_PHY_IDENTIFIER, val1);
 			/* Clear RX alarm */
 			elink_cl45_read(sc, phy,
-					MDIO_PMA_DEVAD,
-					MDIO_PMA_LASI_RXSTAT, &rx_alarm_status);
+				MDIO_PMA_DEVAD,
+				MDIO_PMA_LASI_RXSTAT, &rx_alarm_status);
 			elink_8727_power_module(params->sc, phy, 0);
 			return 0;
 		}
-	}
+	} /* Over current check */
 
-	/* Over current check */
 	/* When module absent bit is set, check module */
 	if (rx_alarm_status & (1 << 5)) {
 		elink_8727_handle_mod_abs(phy, params);
@@ -8939,10 +10538,10 @@ static uint8_t elink_8727_read_status(struct elink_phy *phy,
 	}
 
 	if (!(phy->flags & ELINK_FLAGS_SFP_NOT_APPROVED)) {
-		PMD_DRV_LOG(DEBUG, "Enabling 8727 TX laser");
+		ELINK_DEBUG_P0(sc, "Enabling 8727 TX laser");
 		elink_sfp_set_transmitter(params, phy, 1);
 	} else {
-		PMD_DRV_LOG(DEBUG, "Tx is disabled");
+		ELINK_DEBUG_P0(sc, "Tx is disabled");
 		return 0;
 	}
 
@@ -8956,26 +10555,26 @@ static uint8_t elink_8727_read_status(struct elink_phy *phy,
 	if ((link_status & (1 << 2)) && (!(link_status & (1 << 15)))) {
 		link_up = 1;
 		vars->line_speed = ELINK_SPEED_10000;
-		PMD_DRV_LOG(DEBUG, "port %x: External link up in 10G",
-			    params->port);
+		ELINK_DEBUG_P1(sc, "port %x: External link up in 10G",
+			   params->port);
 	} else if ((link_status & (1 << 0)) && (!(link_status & (1 << 13)))) {
 		link_up = 1;
 		vars->line_speed = ELINK_SPEED_1000;
-		PMD_DRV_LOG(DEBUG, "port %x: External link up in 1G",
-			    params->port);
+		ELINK_DEBUG_P1(sc, "port %x: External link up in 1G",
+			   params->port);
 	} else {
 		link_up = 0;
-		PMD_DRV_LOG(DEBUG, "port %x: External link is down",
-			    params->port);
+		ELINK_DEBUG_P1(sc, "port %x: External link is down",
+			   params->port);
 	}
 
 	/* Capture 10G link fault. */
 	if (vars->line_speed == ELINK_SPEED_10000) {
 		elink_cl45_read(sc, phy, MDIO_PMA_DEVAD,
-				MDIO_PMA_LASI_TXSTAT, &val1);
+			    MDIO_PMA_LASI_TXSTAT, &val1);
 
 		elink_cl45_read(sc, phy, MDIO_PMA_DEVAD,
-				MDIO_PMA_LASI_TXSTAT, &val1);
+			    MDIO_PMA_LASI_TXSTAT, &val1);
 
 		if (val1 & (1 << 0)) {
 			vars->fault_detected = 1;
@@ -8985,7 +10584,7 @@ static uint8_t elink_8727_read_status(struct elink_phy *phy,
 	if (link_up) {
 		elink_ext_phy_resolve_fc(phy, params, vars);
 		vars->duplex = DUPLEX_FULL;
-		PMD_DRV_LOG(DEBUG, "duplex = 0x%x", vars->duplex);
+		ELINK_DEBUG_P1(sc, "duplex = 0x%x", vars->duplex);
 	}
 
 	if ((ELINK_DUAL_MEDIA(params)) &&
@@ -9025,8 +10624,16 @@ static void elink_8727_link_reset(struct elink_phy *phy,
 /******************************************************************/
 /*		BNX2X8481/BNX2X84823/BNX2X84833 PHY SECTION	          */
 /******************************************************************/
+static int elink_is_8483x_8485x(struct elink_phy *phy)
+{
+	return ((phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833) ||
+		(phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834) ||
+		(phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84858));
+}
+
 static void elink_save_848xx_spirom_version(struct elink_phy *phy,
-					    struct bnx2x_softc *sc, uint8_t port)
+					    struct bnx2x_softc *sc,
+					    uint8_t port)
 {
 	uint16_t val, fw_ver2, cnt, i;
 	static struct elink_reg_set reg_set[] = {
@@ -9038,11 +10645,10 @@ static void elink_save_848xx_spirom_version(struct elink_phy *phy,
 	};
 	uint16_t fw_ver1;
 
-	if ((phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833) ||
-	    (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834)) {
+	if (elink_is_8483x_8485x(phy)) {
 		elink_cl45_read(sc, phy, MDIO_CTL_DEVAD, 0x400f, &fw_ver1);
 		elink_save_spirom_version(sc, port, fw_ver1 & 0xfff,
-					  phy->ver_addr);
+				phy->ver_addr);
 	} else {
 		/* For 32-bit registers in 848xx, access via MDIO2ARM i/f. */
 		/* (1) set reg 0xc200_0014(SPI_BRIDGE_CTRL_2) to 0x03000000 */
@@ -9057,12 +10663,14 @@ static void elink_save_848xx_spirom_version(struct elink_phy *phy,
 			DELAY(5);
 		}
 		if (cnt == 100) {
-			PMD_DRV_LOG(DEBUG, "Unable to read 848xx "
-				    "phy fw version(1)");
-			elink_save_spirom_version(sc, port, 0, phy->ver_addr);
+			ELINK_DEBUG_P0(sc, "Unable to read 848xx "
+					"phy fw version(1)");
+			elink_save_spirom_version(sc, port, 0,
+						  phy->ver_addr);
 			return;
 		}
 
+
 		/* 2) read register 0xc200_0000 (SPI_FW_STATUS) */
 		elink_cl45_write(sc, phy, MDIO_PMA_DEVAD, 0xA819, 0x0000);
 		elink_cl45_write(sc, phy, MDIO_PMA_DEVAD, 0xA81A, 0xc200);
@@ -9074,9 +10682,10 @@ static void elink_save_848xx_spirom_version(struct elink_phy *phy,
 			DELAY(5);
 		}
 		if (cnt == 100) {
-			PMD_DRV_LOG(DEBUG, "Unable to read 848xx phy fw "
-				    "version(2)");
-			elink_save_spirom_version(sc, port, 0, phy->ver_addr);
+			ELINK_DEBUG_P0(sc, "Unable to read 848xx phy fw "
+					"version(2)");
+			elink_save_spirom_version(sc, port, 0,
+						  phy->ver_addr);
 			return;
 		}
 
@@ -9090,8 +10699,8 @@ static void elink_save_848xx_spirom_version(struct elink_phy *phy,
 	}
 
 }
-
-static void elink_848xx_set_led(struct bnx2x_softc *sc, struct elink_phy *phy)
+static void elink_848xx_set_led(struct bnx2x_softc *sc,
+				struct elink_phy *phy)
 {
 	uint16_t val, offset, i;
 	static struct elink_reg_set reg_set[] = {
@@ -9100,29 +10709,30 @@ static void elink_848xx_set_led(struct bnx2x_softc *sc, struct elink_phy *phy)
 		{MDIO_PMA_DEVAD, MDIO_PMA_REG_8481_LED3_MASK, 0x0006},
 		{MDIO_PMA_DEVAD, MDIO_PMA_REG_8481_LED3_BLINK, 0x0000},
 		{MDIO_PMA_DEVAD, MDIO_PMA_REG_84823_CTL_SLOW_CLK_CNT_HIGH,
-		 MDIO_PMA_REG_84823_BLINK_RATE_VAL_15P9HZ},
+			MDIO_PMA_REG_84823_BLINK_RATE_VAL_15P9HZ},
 		{MDIO_AN_DEVAD, 0xFFFB, 0xFFFD}
 	};
 	/* PHYC_CTL_LED_CTL */
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, MDIO_PMA_REG_8481_LINK_SIGNAL, &val);
+			MDIO_PMA_DEVAD,
+			MDIO_PMA_REG_8481_LINK_SIGNAL, &val);
 	val &= 0xFE00;
 	val |= 0x0092;
 
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_8481_LINK_SIGNAL, val);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_8481_LINK_SIGNAL, val);
 
 	for (i = 0; i < ARRAY_SIZE(reg_set); i++)
 		elink_cl45_write(sc, phy, reg_set[i].devad, reg_set[i].reg,
 				 reg_set[i].val);
 
-	if ((phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833) ||
-	    (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834))
+	if (elink_is_8483x_8485x(phy))
 		offset = MDIO_PMA_REG_84833_CTL_LED_CTL_1;
 	else
 		offset = MDIO_PMA_REG_84823_CTL_LED_CTL_1;
 
-	/* stretch_en for LED3 */
+	/* stretch_en for LED3*/
 	elink_cl45_read_or_write(sc, phy,
 				 MDIO_PMA_DEVAD, offset,
 				 MDIO_PMA_REG_84823_LED3_STRETCH_EN);
@@ -9135,8 +10745,7 @@ static void elink_848xx_specific_func(struct elink_phy *phy,
 	struct bnx2x_softc *sc = params->sc;
 	switch (action) {
 	case ELINK_PHY_INIT:
-		if ((phy->type != PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833) &&
-		    (phy->type != PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834)) {
+		if (!elink_is_8483x_8485x(phy)) {
 			/* Save spirom version */
 			elink_save_848xx_spirom_version(phy, sc, params->port);
 		}
@@ -9153,14 +10762,15 @@ static void elink_848xx_specific_func(struct elink_phy *phy,
 }
 
 static elink_status_t elink_848xx_cmn_config_init(struct elink_phy *phy,
-						  struct elink_params *params,
-						  struct elink_vars *vars)
+				       struct elink_params *params,
+				       struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t autoneg_val, an_1000_val, an_10_100_val;
 
 	elink_848xx_specific_func(phy, params, ELINK_PHY_INIT);
-	elink_cl45_write(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 0x0000);
+	elink_cl45_write(sc, phy,
+			 MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 0x0000);
 
 	/* set 1000 speed advertisement */
 	elink_cl45_read(sc, phy,
@@ -9170,24 +10780,25 @@ static elink_status_t elink_848xx_cmn_config_init(struct elink_phy *phy,
 	elink_ext_phy_set_pause(params, phy, vars);
 	elink_cl45_read(sc, phy,
 			MDIO_AN_DEVAD,
-			MDIO_AN_REG_8481_LEGACY_AN_ADV, &an_10_100_val);
+			MDIO_AN_REG_8481_LEGACY_AN_ADV,
+			&an_10_100_val);
 	elink_cl45_read(sc, phy,
 			MDIO_AN_DEVAD, MDIO_AN_REG_8481_LEGACY_MII_CTRL,
 			&autoneg_val);
 	/* Disable forced speed */
-	autoneg_val &=
-	    ~((1 << 6) | (1 << 8) | (1 << 9) | (1 << 12) | (1 << 13));
+	autoneg_val &= ~((1 << 6) | (1 << 8) | (1 << 9) | (1 << 12) |
+			 (1 << 13));
 	an_10_100_val &= ~((1 << 5) | (1 << 6) | (1 << 7) | (1 << 8));
 
 	if (((phy->req_line_speed == ELINK_SPEED_AUTO_NEG) &&
 	     (phy->speed_cap_mask &
-	      PORT_HW_CFG_SPEED_CAPABILITY_D0_1G)) ||
+	     PORT_HW_CFG_SPEED_CAPABILITY_D0_1G)) ||
 	    (phy->req_line_speed == ELINK_SPEED_1000)) {
 		an_1000_val |= (1 << 8);
 		autoneg_val |= (1 << 9 | 1 << 12);
 		if (phy->req_duplex == DUPLEX_FULL)
 			an_1000_val |= (1 << 9);
-		PMD_DRV_LOG(DEBUG, "Advertising 1G");
+		ELINK_DEBUG_P0(sc, "Advertising 1G");
 	} else
 		an_1000_val &= ~((1 << 8) | (1 << 9));
 
@@ -9203,7 +10814,7 @@ static elink_status_t elink_848xx_cmn_config_init(struct elink_phy *phy,
 			 */
 			autoneg_val |= (1 << 9 | 1 << 12);
 			an_10_100_val |= (1 << 8);
-			PMD_DRV_LOG(DEBUG, "Advertising 100M-FD");
+			ELINK_DEBUG_P0(sc, "Advertising 100M-FD");
 		}
 
 		if (phy->speed_cap_mask &
@@ -9212,7 +10823,7 @@ static elink_status_t elink_848xx_cmn_config_init(struct elink_phy *phy,
 			 */
 			autoneg_val |= (1 << 9 | 1 << 12);
 			an_10_100_val |= (1 << 7);
-			PMD_DRV_LOG(DEBUG, "Advertising 100M-HD");
+			ELINK_DEBUG_P0(sc, "Advertising 100M-HD");
 		}
 
 		if ((phy->speed_cap_mask &
@@ -9220,7 +10831,7 @@ static elink_status_t elink_848xx_cmn_config_init(struct elink_phy *phy,
 		    (phy->supported & ELINK_SUPPORTED_10baseT_Full)) {
 			an_10_100_val |= (1 << 6);
 			autoneg_val |= (1 << 9 | 1 << 12);
-			PMD_DRV_LOG(DEBUG, "Advertising 10M-FD");
+			ELINK_DEBUG_P0(sc, "Advertising 10M-FD");
 		}
 
 		if ((phy->speed_cap_mask &
@@ -9228,14 +10839,15 @@ static elink_status_t elink_848xx_cmn_config_init(struct elink_phy *phy,
 		    (phy->supported & ELINK_SUPPORTED_10baseT_Half)) {
 			an_10_100_val |= (1 << 5);
 			autoneg_val |= (1 << 9 | 1 << 12);
-			PMD_DRV_LOG(DEBUG, "Advertising 10M-HD");
+			ELINK_DEBUG_P0(sc, "Advertising 10M-HD");
 		}
 	}
 
 	/* Only 10/100 are allowed to work in FORCE mode */
 	if ((phy->req_line_speed == ELINK_SPEED_100) &&
 	    (phy->supported &
-	     (ELINK_SUPPORTED_100baseT_Half | ELINK_SUPPORTED_100baseT_Full))) {
+	     (ELINK_SUPPORTED_100baseT_Half |
+	      ELINK_SUPPORTED_100baseT_Full))) {
 		autoneg_val |= (1 << 13);
 		/* Enabled AUTO-MDIX when autoneg is disabled */
 		elink_cl45_write(sc, phy,
@@ -9243,16 +10855,17 @@ static elink_status_t elink_848xx_cmn_config_init(struct elink_phy *phy,
 				 (1 << 15 | 1 << 9 | 7 << 0));
 		/* The PHY needs this set even for forced link. */
 		an_10_100_val |= (1 << 8) | (1 << 7);
-		PMD_DRV_LOG(DEBUG, "Setting 100M force");
+		ELINK_DEBUG_P0(sc, "Setting 100M force");
 	}
 	if ((phy->req_line_speed == ELINK_SPEED_10) &&
 	    (phy->supported &
-	     (ELINK_SUPPORTED_10baseT_Half | ELINK_SUPPORTED_10baseT_Full))) {
+	     (ELINK_SUPPORTED_10baseT_Half |
+	      ELINK_SUPPORTED_10baseT_Full))) {
 		/* Enabled AUTO-MDIX when autoneg is disabled */
 		elink_cl45_write(sc, phy,
 				 MDIO_AN_DEVAD, MDIO_AN_REG_8481_AUX_CTRL,
 				 (1 << 15 | 1 << 9 | 7 << 0));
-		PMD_DRV_LOG(DEBUG, "Setting 10M force");
+		ELINK_DEBUG_P0(sc, "Setting 10M force");
 	}
 
 	elink_cl45_write(sc, phy,
@@ -9265,42 +10878,44 @@ static elink_status_t elink_848xx_cmn_config_init(struct elink_phy *phy,
 	/* Always write this if this is not 84833/4.
 	 * For 84833/4, write it only when it's a forced speed.
 	 */
-	if (((phy->type != PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833) &&
-	     (phy->type != PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834)) ||
+	if (!elink_is_8483x_8485x(phy) ||
 	    ((autoneg_val & (1 << 12)) == 0))
 		elink_cl45_write(sc, phy,
-				 MDIO_AN_DEVAD,
-				 MDIO_AN_REG_8481_LEGACY_MII_CTRL, autoneg_val);
+			 MDIO_AN_DEVAD,
+			 MDIO_AN_REG_8481_LEGACY_MII_CTRL, autoneg_val);
 
 	if (((phy->req_line_speed == ELINK_SPEED_AUTO_NEG) &&
 	     (phy->speed_cap_mask &
 	      PORT_HW_CFG_SPEED_CAPABILITY_D0_10G)) ||
 	    (phy->req_line_speed == ELINK_SPEED_10000)) {
-		PMD_DRV_LOG(DEBUG, "Advertising 10G");
-		/* Restart autoneg for 10G */
+		ELINK_DEBUG_P0(sc, "Advertising 10G");
+		/* Restart autoneg for 10G*/
 
-		elink_cl45_read_or_write(sc, phy,
-					 MDIO_AN_DEVAD,
-					 MDIO_AN_REG_8481_10GBASE_T_AN_CTRL,
-					 0x1000);
+		elink_cl45_read_or_write(
+			sc, phy,
+			MDIO_AN_DEVAD,
+			MDIO_AN_REG_8481_10GBASE_T_AN_CTRL,
+			0x1000);
 		elink_cl45_write(sc, phy,
-				 MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, 0x3200);
+				 MDIO_AN_DEVAD, MDIO_AN_REG_CTRL,
+				 0x3200);
 	} else
 		elink_cl45_write(sc, phy,
 				 MDIO_AN_DEVAD,
-				 MDIO_AN_REG_8481_10GBASE_T_AN_CTRL, 1);
+				 MDIO_AN_REG_8481_10GBASE_T_AN_CTRL,
+				 1);
 
 	return ELINK_STATUS_OK;
 }
 
 static uint8_t elink_8481_config_init(struct elink_phy *phy,
-					     struct elink_params *params,
-					     struct elink_vars *vars)
+				  struct elink_params *params,
+				  struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
-	/* Restore normal power mode */
+	/* Restore normal power mode*/
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_2,
-			    MISC_REGISTERS_GPIO_OUTPUT_HIGH, params->port);
+		       MISC_REGISTERS_GPIO_OUTPUT_HIGH, params->port);
 
 	/* HW reset */
 	elink_ext_phy_hw_reset(sc, params->port);
@@ -9310,101 +10925,219 @@ static uint8_t elink_8481_config_init(struct elink_phy *phy,
 	return elink_848xx_cmn_config_init(phy, params, vars);
 }
 
-#define PHY84833_CMDHDLR_WAIT 300
-#define PHY84833_CMDHDLR_MAX_ARGS 5
-static elink_status_t elink_84833_cmd_hdlr(struct elink_phy *phy,
+#define PHY848xx_CMDHDLR_WAIT 300
+#define PHY848xx_CMDHDLR_MAX_ARGS 5
+
+static elink_status_t elink_84858_cmd_hdlr(struct elink_phy *phy,
 					   struct elink_params *params,
-					   uint16_t fw_cmd, uint16_t cmd_args[],
-					   int argc)
+					   uint16_t fw_cmd,
+					   uint16_t cmd_args[], int argc)
 {
 	int idx;
 	uint16_t val;
 	struct bnx2x_softc *sc = params->sc;
+
+	/* Step 1: Poll the STATUS register to see whether the previous command
+	 * is in progress or the system is busy (CMD_IN_PROGRESS or
+	 * SYSTEM_BUSY). If previous command is in progress or system is busy,
+	 * check again until the previous command finishes execution and the
+	 * system is available for taking command
+	 */
+
+	for (idx = 0; idx < PHY848xx_CMDHDLR_WAIT; idx++) {
+		elink_cl45_read(sc, phy, MDIO_CTL_DEVAD,
+				MDIO_848xx_CMD_HDLR_STATUS, &val);
+		if ((val != PHY84858_STATUS_CMD_IN_PROGRESS) &&
+		    (val != PHY84858_STATUS_CMD_SYSTEM_BUSY))
+			break;
+		DELAY(1000 * 1);
+	}
+	if (idx >= PHY848xx_CMDHDLR_WAIT) {
+		ELINK_DEBUG_P0(sc, "FW cmd: FW not ready.");
+		return ELINK_STATUS_ERROR;
+	}
+
+	/* Step2: If any parameters are required for the function, write them
+	 * to the required DATA registers
+	 */
+
+	for (idx = 0; idx < argc; idx++) {
+		elink_cl45_write(sc, phy, MDIO_CTL_DEVAD,
+				 MDIO_848xx_CMD_HDLR_DATA1 + idx,
+				 cmd_args[idx]);
+	}
+
+	/* Step3: When the firmware is ready for commands, write the 'Command
+	 * code' to the CMD register
+	 */
+	elink_cl45_write(sc, phy, MDIO_CTL_DEVAD,
+			 MDIO_848xx_CMD_HDLR_COMMAND, fw_cmd);
+
+	/* Step4: Once the command has been written, poll the STATUS register
+	 * to check whether the command has completed (CMD_COMPLETED_PASS/
+	 * CMD_FOR_CMDS or CMD_COMPLETED_ERROR).
+	 */
+
+	for (idx = 0; idx < PHY848xx_CMDHDLR_WAIT; idx++) {
+		elink_cl45_read(sc, phy, MDIO_CTL_DEVAD,
+				MDIO_848xx_CMD_HDLR_STATUS, &val);
+		if ((val == PHY84858_STATUS_CMD_COMPLETE_PASS) ||
+		    (val == PHY84858_STATUS_CMD_COMPLETE_ERROR))
+			break;
+		DELAY(1000 * 1);
+	}
+	if ((idx >= PHY848xx_CMDHDLR_WAIT) ||
+	    (val == PHY84858_STATUS_CMD_COMPLETE_ERROR)) {
+		ELINK_DEBUG_P0(sc, "FW cmd failed.");
+		return ELINK_STATUS_ERROR;
+	}
+	/* Step5: Once the command has completed, read the specficied DATA
+	 * registers for any saved results for the command, if applicable
+	 */
+
+	/* Gather returning data */
+	for (idx = 0; idx < argc; idx++) {
+		elink_cl45_read(sc, phy, MDIO_CTL_DEVAD,
+				MDIO_848xx_CMD_HDLR_DATA1 + idx,
+				&cmd_args[idx]);
+	}
+
+	return ELINK_STATUS_OK;
+}
+
+static elink_status_t elink_84833_cmd_hdlr(struct elink_phy *phy,
+				struct elink_params *params, uint16_t fw_cmd,
+				uint16_t cmd_args[], int argc, int process)
+{
+	int idx;
+	uint16_t val;
+	struct bnx2x_softc *sc = params->sc;
+	elink_status_t rc = ELINK_STATUS_OK;
+
+	if (process == PHY84833_MB_PROCESS2) {
 	/* Write CMD_OPEN_OVERRIDE to STATUS reg */
 	elink_cl45_write(sc, phy, MDIO_CTL_DEVAD,
-			 MDIO_84833_CMD_HDLR_STATUS,
-			 PHY84833_STATUS_CMD_OPEN_OVERRIDE);
-	for (idx = 0; idx < PHY84833_CMDHDLR_WAIT; idx++) {
+				 MDIO_848xx_CMD_HDLR_STATUS,
+			PHY84833_STATUS_CMD_OPEN_OVERRIDE);
+	}
+
+	for (idx = 0; idx < PHY848xx_CMDHDLR_WAIT; idx++) {
 		elink_cl45_read(sc, phy, MDIO_CTL_DEVAD,
-				MDIO_84833_CMD_HDLR_STATUS, &val);
+			       MDIO_848xx_CMD_HDLR_STATUS, &val);
 		if (val == PHY84833_STATUS_CMD_OPEN_FOR_CMDS)
 			break;
 		DELAY(1000 * 1);
 	}
-	if (idx >= PHY84833_CMDHDLR_WAIT) {
-		PMD_DRV_LOG(DEBUG, "FW cmd: FW not ready.");
+	if (idx >= PHY848xx_CMDHDLR_WAIT) {
+		ELINK_DEBUG_P0(sc, "FW cmd: FW not ready.");
+		/* if the status is CMD_COMPLETE_PASS or CMD_COMPLETE_ERROR
+		 * clear the status to CMD_CLEAR_COMPLETE
+		 */
+		if (val == PHY84833_STATUS_CMD_COMPLETE_PASS ||
+		    val == PHY84833_STATUS_CMD_COMPLETE_ERROR) {
+			elink_cl45_write(sc, phy, MDIO_CTL_DEVAD,
+					 MDIO_848xx_CMD_HDLR_STATUS,
+					 PHY84833_STATUS_CMD_CLEAR_COMPLETE);
+		}
 		return ELINK_STATUS_ERROR;
 	}
-
-	/* Prepare argument(s) and issue command */
+	if (process == PHY84833_MB_PROCESS1 ||
+	    process == PHY84833_MB_PROCESS2) {
+		/* Prepare argument(s) */
 	for (idx = 0; idx < argc; idx++) {
 		elink_cl45_write(sc, phy, MDIO_CTL_DEVAD,
-				 MDIO_84833_CMD_HDLR_DATA1 + idx,
-				 cmd_args[idx]);
+					 MDIO_848xx_CMD_HDLR_DATA1 + idx,
+				cmd_args[idx]);
+	}
 	}
+
+	/* Issue command */
 	elink_cl45_write(sc, phy, MDIO_CTL_DEVAD,
-			 MDIO_84833_CMD_HDLR_COMMAND, fw_cmd);
-	for (idx = 0; idx < PHY84833_CMDHDLR_WAIT; idx++) {
+			MDIO_848xx_CMD_HDLR_COMMAND, fw_cmd);
+	for (idx = 0; idx < PHY848xx_CMDHDLR_WAIT; idx++) {
 		elink_cl45_read(sc, phy, MDIO_CTL_DEVAD,
-				MDIO_84833_CMD_HDLR_STATUS, &val);
+			       MDIO_848xx_CMD_HDLR_STATUS, &val);
 		if ((val == PHY84833_STATUS_CMD_COMPLETE_PASS) ||
-		    (val == PHY84833_STATUS_CMD_COMPLETE_ERROR))
+			(val == PHY84833_STATUS_CMD_COMPLETE_ERROR))
 			break;
 		DELAY(1000 * 1);
 	}
-	if ((idx >= PHY84833_CMDHDLR_WAIT) ||
-	    (val == PHY84833_STATUS_CMD_COMPLETE_ERROR)) {
-		PMD_DRV_LOG(DEBUG, "FW cmd failed.");
-		return ELINK_STATUS_ERROR;
+	if ((idx >= PHY848xx_CMDHDLR_WAIT) ||
+		(val == PHY84833_STATUS_CMD_COMPLETE_ERROR)) {
+		ELINK_DEBUG_P0(sc, "FW cmd failed.");
+		rc = ELINK_STATUS_ERROR;
 	}
+	if (process == PHY84833_MB_PROCESS3 && rc == ELINK_STATUS_OK) {
 	/* Gather returning data */
 	for (idx = 0; idx < argc; idx++) {
 		elink_cl45_read(sc, phy, MDIO_CTL_DEVAD,
-				MDIO_84833_CMD_HDLR_DATA1 + idx,
+					MDIO_848xx_CMD_HDLR_DATA1 + idx,
 				&cmd_args[idx]);
 	}
+	}
+	if (val == PHY84833_STATUS_CMD_COMPLETE_ERROR ||
+	    val == PHY84833_STATUS_CMD_COMPLETE_PASS) {
 	elink_cl45_write(sc, phy, MDIO_CTL_DEVAD,
-			 MDIO_84833_CMD_HDLR_STATUS,
-			 PHY84833_STATUS_CMD_CLEAR_COMPLETE);
-	return ELINK_STATUS_OK;
+				 MDIO_848xx_CMD_HDLR_STATUS,
+			PHY84833_STATUS_CMD_CLEAR_COMPLETE);
+	}
+	return rc;
 }
 
-static elink_status_t elink_84833_pair_swap_cfg(struct elink_phy *phy,
-						struct elink_params *params,
-						__rte_unused struct elink_vars
-						*vars)
+static elink_status_t elink_848xx_cmd_hdlr(struct elink_phy *phy,
+					   struct elink_params *params,
+					   uint16_t fw_cmd,
+					   uint16_t cmd_args[], int argc,
+					   int process)
+{
+	struct bnx2x_softc *sc = params->sc;
+
+	if ((phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84858) ||
+	    (REG_RD(sc, params->shmem2_base +
+		    offsetof(struct shmem2_region,
+			     link_attr_sync[params->port])) &
+			     LINK_ATTR_84858)) {
+		return elink_84858_cmd_hdlr(phy, params, fw_cmd, cmd_args,
+					    argc);
+	} else {
+		return elink_84833_cmd_hdlr(phy, params, fw_cmd, cmd_args,
+					    argc, process);
+	}
+}
+
+static elink_status_t elink_848xx_pair_swap_cfg(struct elink_phy *phy,
+				   struct elink_params *params,
+				   __rte_unused struct elink_vars *vars)
 {
 	uint32_t pair_swap;
-	uint16_t data[PHY84833_CMDHDLR_MAX_ARGS];
+	uint16_t data[PHY848xx_CMDHDLR_MAX_ARGS];
 	elink_status_t status;
 	struct bnx2x_softc *sc = params->sc;
 
 	/* Check for configuration. */
 	pair_swap = REG_RD(sc, params->shmem_base +
 			   offsetof(struct shmem_region,
-				    dev_info.port_hw_config[params->port].
-				    xgbt_phy_cfg)) &
-	    PORT_HW_CFG_RJ45_PAIR_SWAP_MASK;
+			dev_info.port_hw_config[params->port].xgbt_phy_cfg)) &
+		PORT_HW_CFG_RJ45_PAIR_SWAP_MASK;
 
 	if (pair_swap == 0)
 		return ELINK_STATUS_OK;
 
 	/* Only the second argument is used for this command */
-	data[1] = (uint16_t) pair_swap;
+	data[1] = (uint16_t)pair_swap;
 
-	status = elink_84833_cmd_hdlr(phy, params,
-				      PHY84833_CMD_SET_PAIR_SWAP, data,
-				      PHY84833_CMDHDLR_MAX_ARGS);
-	if (status == ELINK_STATUS_OK) {
-		PMD_DRV_LOG(DEBUG, "Pairswap OK, val=0x%x", data[1]);
-	}
+	status = elink_848xx_cmd_hdlr(phy, params,
+				      PHY848xx_CMD_SET_PAIR_SWAP, data,
+				      2, PHY84833_MB_PROCESS2);
+	if (status == ELINK_STATUS_OK)
+		ELINK_DEBUG_P1(sc, "Pairswap OK, val=0x%x", data[1]);
 
 	return status;
 }
 
 static uint8_t elink_84833_get_reset_gpios(struct bnx2x_softc *sc,
-					   uint32_t shmem_base_path[],
-					   __rte_unused uint32_t chip_id)
+				      uint32_t shmem_base_path[],
+				      __rte_unused uint32_t chip_id)
 {
 	uint32_t reset_pin[2];
 	uint32_t idx;
@@ -9414,54 +11147,50 @@ static uint8_t elink_84833_get_reset_gpios(struct bnx2x_softc *sc,
 		for (idx = 0; idx < 2; idx++) {
 			/* Map config param to register bit. */
 			reset_pin[idx] = REG_RD(sc, shmem_base_path[idx] +
-						offsetof(struct shmem_region,
-							 dev_info.
-							 port_hw_config[0].
-							 e3_cmn_pin_cfg));
-			reset_pin[idx] =
-			    (reset_pin[idx] & PORT_HW_CFG_E3_PHY_RESET_MASK) >>
-			    PORT_HW_CFG_E3_PHY_RESET_SHIFT;
+				offsetof(struct shmem_region,
+				dev_info.port_hw_config[0].e3_cmn_pin_cfg));
+			reset_pin[idx] = (reset_pin[idx] &
+				PORT_HW_CFG_E3_PHY_RESET_MASK) >>
+				PORT_HW_CFG_E3_PHY_RESET_SHIFT;
 			reset_pin[idx] -= PIN_CFG_GPIO0_P0;
 			reset_pin[idx] = (1 << reset_pin[idx]);
 		}
-		reset_gpios = (uint8_t) (reset_pin[0] | reset_pin[1]);
+		reset_gpios = (uint8_t)(reset_pin[0] | reset_pin[1]);
 	} else {
 		/* E2, look from diff place of shmem. */
 		for (idx = 0; idx < 2; idx++) {
 			reset_pin[idx] = REG_RD(sc, shmem_base_path[idx] +
-						offsetof(struct shmem_region,
-							 dev_info.
-							 port_hw_config[0].
-							 default_cfg));
+				offsetof(struct shmem_region,
+				dev_info.port_hw_config[0].default_cfg));
 			reset_pin[idx] &= PORT_HW_CFG_EXT_PHY_GPIO_RST_MASK;
 			reset_pin[idx] -= PORT_HW_CFG_EXT_PHY_GPIO_RST_GPIO0_P0;
 			reset_pin[idx] >>= PORT_HW_CFG_EXT_PHY_GPIO_RST_SHIFT;
 			reset_pin[idx] = (1 << reset_pin[idx]);
 		}
-		reset_gpios = (uint8_t) (reset_pin[0] | reset_pin[1]);
+		reset_gpios = (uint8_t)(reset_pin[0] | reset_pin[1]);
 	}
 
 	return reset_gpios;
 }
 
 static void elink_84833_hw_reset_phy(struct elink_phy *phy,
-					struct elink_params *params)
+				struct elink_params *params)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t reset_gpios;
 	uint32_t other_shmem_base_addr = REG_RD(sc, params->shmem2_base +
-						offsetof(struct shmem2_region,
-							 other_shmem_base_addr));
+				offsetof(struct shmem2_region,
+				other_shmem_base_addr));
 
 	uint32_t shmem_base_path[2];
 
 	/* Work around for 84833 LED failure inside RESET status */
 	elink_cl45_write(sc, phy, MDIO_AN_DEVAD,
-			 MDIO_AN_REG_8481_LEGACY_MII_CTRL,
-			 MDIO_AN_REG_8481_MII_CTRL_FORCE_1G);
+		MDIO_AN_REG_8481_LEGACY_MII_CTRL,
+		MDIO_AN_REG_8481_MII_CTRL_FORCE_1G);
 	elink_cl45_write(sc, phy, MDIO_AN_DEVAD,
-			 MDIO_AN_REG_8481_1G_100T_EXT_CTRL,
-			 MIDO_AN_REG_8481_EXT_CTRL_FORCE_LEDS_OFF);
+		MDIO_AN_REG_8481_1G_100T_EXT_CTRL,
+		MIDO_AN_REG_8481_EXT_CTRL_FORCE_LEDS_OFF);
 
 	shmem_base_path[0] = params->shmem_base;
 	shmem_base_path[1] = other_shmem_base_addr;
@@ -9472,23 +11201,25 @@ static void elink_84833_hw_reset_phy(struct elink_phy *phy,
 	elink_cb_gpio_mult_write(sc, reset_gpios,
 				 MISC_REGISTERS_GPIO_OUTPUT_LOW);
 	DELAY(10);
-	PMD_DRV_LOG(DEBUG, "84833 hw reset on pin values 0x%x", reset_gpios);
+	ELINK_DEBUG_P1(sc, "84833 hw reset on pin values 0x%x",
+		reset_gpios);
 }
 
 static elink_status_t elink_8483x_disable_eee(struct elink_phy *phy,
-					      struct elink_params *params,
-					      struct elink_vars *vars)
+				   struct elink_params *params,
+				   struct elink_vars *vars)
 {
 	elink_status_t rc;
+	struct bnx2x_softc *sc = params->sc;
 	uint16_t cmd_args = 0;
 
-	PMD_DRV_LOG(DEBUG, "Don't Advertise 10GBase-T EEE");
+	ELINK_DEBUG_P0(sc, "Don't Advertise 10GBase-T EEE");
 
 	/* Prevent Phy from working in EEE and advertising it */
-	rc = elink_84833_cmd_hdlr(phy, params,
-				  PHY84833_CMD_SET_EEE_MODE, &cmd_args, 1);
+	rc = elink_848xx_cmd_hdlr(phy, params, PHY848xx_CMD_SET_EEE_MODE,
+				  &cmd_args, 1, PHY84833_MB_PROCESS1);
 	if (rc != ELINK_STATUS_OK) {
-		PMD_DRV_LOG(DEBUG, "EEE disable failed.");
+		ELINK_DEBUG_P0(sc, "EEE disable failed.");
 		return rc;
 	}
 
@@ -9496,16 +11227,17 @@ static elink_status_t elink_8483x_disable_eee(struct elink_phy *phy,
 }
 
 static elink_status_t elink_8483x_enable_eee(struct elink_phy *phy,
-					     struct elink_params *params,
-					     struct elink_vars *vars)
+				   struct elink_params *params,
+				   struct elink_vars *vars)
 {
 	elink_status_t rc;
+	struct bnx2x_softc *sc = params->sc;
 	uint16_t cmd_args = 1;
 
-	rc = elink_84833_cmd_hdlr(phy, params,
-				  PHY84833_CMD_SET_EEE_MODE, &cmd_args, 1);
+	rc = elink_848xx_cmd_hdlr(phy, params, PHY848xx_CMD_SET_EEE_MODE,
+				  &cmd_args, 1, PHY84833_MB_PROCESS1);
 	if (rc != ELINK_STATUS_OK) {
-		PMD_DRV_LOG(DEBUG, "EEE enable failed.");
+		ELINK_DEBUG_P0(sc, "EEE enable failed.");
 		return rc;
 	}
 
@@ -9514,14 +11246,14 @@ static elink_status_t elink_8483x_enable_eee(struct elink_phy *phy,
 
 #define PHY84833_CONSTANT_LATENCY 1193
 static uint8_t elink_848x3_config_init(struct elink_phy *phy,
-				       struct elink_params *params,
-				       struct elink_vars *vars)
+				   struct elink_params *params,
+				   struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t port, initialize = 1;
 	uint16_t val;
 	uint32_t actual_phy_selection;
-	uint16_t cmd_args[PHY84833_CMDHDLR_MAX_ARGS];
+	uint16_t cmd_args[PHY848xx_CMDHDLR_MAX_ARGS];
 	elink_status_t rc = ELINK_STATUS_OK;
 
 	DELAY(1000 * 1);
@@ -9533,19 +11265,20 @@ static uint8_t elink_848x3_config_init(struct elink_phy *phy,
 
 	if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84823) {
 		elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_3,
-				    MISC_REGISTERS_GPIO_OUTPUT_HIGH, port);
+			       MISC_REGISTERS_GPIO_OUTPUT_HIGH,
+			       port);
 	} else {
 		/* MDIO reset */
 		elink_cl45_write(sc, phy,
-				 MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 0x8000);
+				MDIO_PMA_DEVAD,
+				MDIO_PMA_REG_CTRL, 0x8000);
 	}
 
 	elink_wait_reset_complete(sc, phy, params);
 
 	/* Wait for GPHY to come out of reset */
 	DELAY(1000 * 50);
-	if ((phy->type != PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833) &&
-	    (phy->type != PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834)) {
+	if (!elink_is_8483x_8485x(phy)) {
 		/* BNX2X84823 requires that XGXS links up first @ 10G for normal
 		 * behavior.
 		 */
@@ -9556,7 +11289,19 @@ static uint8_t elink_848x3_config_init(struct elink_phy *phy,
 		elink_program_serdes(&params->phy[ELINK_INT_PHY], params, vars);
 		vars->line_speed = temp;
 	}
+	/* Check if this is actually BNX2X84858 */
+	if (phy->type != PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84858) {
+		uint16_t hw_rev;
+
+		elink_cl45_read(sc, phy, MDIO_AN_DEVAD,
+				MDIO_AN_REG_848xx_ID_MSB, &hw_rev);
+		if (hw_rev == BNX2X84858_PHY_ID) {
+			params->link_attr_sync |= LINK_ATTR_84858;
+			elink_update_link_attr(params, params->link_attr_sync);
+		}
+	}
 
+	/* Set dual-media configuration according to configuration */
 	elink_cl45_read(sc, phy, MDIO_CTL_DEVAD,
 			MDIO_CTL_REG_84823_MEDIA, &val);
 	val &= ~(MDIO_CTL_REG_84823_MEDIA_MAC_MASK |
@@ -9598,39 +11343,33 @@ static uint8_t elink_848x3_config_init(struct elink_phy *phy,
 
 	elink_cl45_write(sc, phy, MDIO_CTL_DEVAD,
 			 MDIO_CTL_REG_84823_MEDIA, val);
-	PMD_DRV_LOG(DEBUG, "Multi_phy config = 0x%x, Media control = 0x%x",
-		    params->multi_phy_config, val);
+	ELINK_DEBUG_P2(sc, "Multi_phy config = 0x%x, Media control = 0x%x",
+		   params->multi_phy_config, val);
 
-	if ((phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833) ||
-	    (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834)) {
-		elink_84833_pair_swap_cfg(phy, params, vars);
+	if (elink_is_8483x_8485x(phy)) {
+		elink_848xx_pair_swap_cfg(phy, params, vars);
 
 		/* Keep AutogrEEEn disabled. */
 		cmd_args[0] = 0x0;
 		cmd_args[1] = 0x0;
 		cmd_args[2] = PHY84833_CONSTANT_LATENCY + 1;
 		cmd_args[3] = PHY84833_CONSTANT_LATENCY;
-		rc = elink_84833_cmd_hdlr(phy, params,
-					  PHY84833_CMD_SET_EEE_MODE, cmd_args,
-					  PHY84833_CMDHDLR_MAX_ARGS);
-		if (rc != ELINK_STATUS_OK) {
-			PMD_DRV_LOG(DEBUG, "Cfg AutogrEEEn failed.");
-		}
+		rc = elink_848xx_cmd_hdlr(phy, params,
+					  PHY848xx_CMD_SET_EEE_MODE, cmd_args,
+					  4, PHY84833_MB_PROCESS1);
+		if (rc != ELINK_STATUS_OK)
+			ELINK_DEBUG_P0(sc, "Cfg AutogrEEEn failed.");
 	}
-	if (initialize) {
+	if (initialize)
 		rc = elink_848xx_cmn_config_init(phy, params, vars);
-	} else {
+	else
 		elink_save_848xx_spirom_version(phy, sc, params->port);
-	}
 	/* 84833 PHY has a better feature and doesn't need to support this. */
 	if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84823) {
 		uint32_t cms_enable = REG_RD(sc, params->shmem_base +
-					     offsetof(struct shmem_region,
-						      dev_info.
-						      port_hw_config[params->
-								     port].
-						      default_cfg)) &
-		    PORT_HW_CFG_ENABLE_CMS_MASK;
+			offsetof(struct shmem_region,
+			dev_info.port_hw_config[params->port].default_cfg)) &
+			PORT_HW_CFG_ENABLE_CMS_MASK;
 
 		elink_cl45_read(sc, phy, MDIO_CTL_DEVAD,
 				MDIO_CTL_REG_84823_USER_CTRL_REG, &val);
@@ -9651,7 +11390,7 @@ static uint8_t elink_848x3_config_init(struct elink_phy *phy,
 	    elink_eee_has_cap(params)) {
 		rc = elink_eee_initial_config(params, vars, SHMEM_EEE_10G_ADV);
 		if (rc != ELINK_STATUS_OK) {
-			PMD_DRV_LOG(DEBUG, "Failed to configure EEE timers");
+			ELINK_DEBUG_P0(sc, "Failed to configure EEE timers");
 			elink_8483x_disable_eee(phy, params, vars);
 			return rc;
 		}
@@ -9664,39 +11403,40 @@ static uint8_t elink_848x3_config_init(struct elink_phy *phy,
 		else
 			rc = elink_8483x_disable_eee(phy, params, vars);
 		if (rc != ELINK_STATUS_OK) {
-			PMD_DRV_LOG(DEBUG, "Failed to set EEE advertisement");
+			ELINK_DEBUG_P0(sc, "Failed to set EEE advertisement");
 			return rc;
 		}
 	} else {
 		vars->eee_status &= ~SHMEM_EEE_SUPPORTED_MASK;
 	}
 
-	if ((phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833) ||
-	    (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834)) {
+	if (elink_is_8483x_8485x(phy)) {
 		/* Bring PHY out of super isolate mode as the final step. */
 		elink_cl45_read_and_write(sc, phy,
 					  MDIO_CTL_DEVAD,
 					  MDIO_84833_TOP_CFG_XGPHY_STRAP1,
-					  (uint16_t) ~
-					  MDIO_84833_SUPER_ISOLATE);
+					  (uint16_t)~MDIO_84833_SUPER_ISOLATE);
 	}
 	return rc;
 }
 
 static uint8_t elink_848xx_read_status(struct elink_phy *phy,
-				       struct elink_params *params,
-				       struct elink_vars *vars)
+				  struct elink_params *params,
+				  struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t val, val1, val2;
 	uint8_t link_up = 0;
 
+
 	/* Check 10G-BaseT link status */
 	/* Check PMD signal ok */
-	elink_cl45_read(sc, phy, MDIO_AN_DEVAD, 0xFFFA, &val1);
 	elink_cl45_read(sc, phy,
-			MDIO_PMA_DEVAD, MDIO_PMA_REG_8481_PMD_SIGNAL, &val2);
-	PMD_DRV_LOG(DEBUG, "BNX2X848xx: PMD_SIGNAL 1.a811 = 0x%x", val2);
+			MDIO_AN_DEVAD, 0xFFFA, &val1);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_8481_PMD_SIGNAL,
+			&val2);
+	ELINK_DEBUG_P1(sc, "BNX2X848xx: PMD_SIGNAL 1.a811 = 0x%x", val2);
 
 	/* Check link 10G */
 	if (val2 & (1 << 11)) {
@@ -9704,8 +11444,8 @@ static uint8_t elink_848xx_read_status(struct elink_phy *phy,
 		vars->duplex = DUPLEX_FULL;
 		link_up = 1;
 		elink_ext_phy_10G_an_resolve(sc, phy, vars);
-	} else {		/* Check Legacy speed link */
-		uint16_t legacy_status, legacy_speed, mii_ctrl;
+	} else { /* Check Legacy speed link */
+		uint16_t legacy_status, legacy_speed;
 
 		/* Enable expansion register 0x42 (Operation mode status) */
 		elink_cl45_write(sc, phy,
@@ -9718,7 +11458,8 @@ static uint8_t elink_848xx_read_status(struct elink_phy *phy,
 				MDIO_AN_REG_8481_EXPANSION_REG_RD_RW,
 				&legacy_status);
 
-		PMD_DRV_LOG(DEBUG, "Legacy speed status = 0x%x", legacy_status);
+		ELINK_DEBUG_P1(sc, "Legacy speed status = 0x%x",
+		   legacy_status);
 		link_up = ((legacy_status & (1 << 11)) == (1 << 11));
 		legacy_speed = (legacy_status & (3 << 9));
 		if (legacy_speed == (0 << 9))
@@ -9727,13 +11468,15 @@ static uint8_t elink_848xx_read_status(struct elink_phy *phy,
 			vars->line_speed = ELINK_SPEED_100;
 		else if (legacy_speed == (2 << 9))
 			vars->line_speed = ELINK_SPEED_1000;
-		else {		/* Should not happen: Treat as link down */
+		else { /* Should not happen: Treat as link down */
 			vars->line_speed = 0;
 			link_up = 0;
 		}
 
 		if (params->feature_config_flags &
-		    ELINK_FEATURE_CONFIG_IEEE_PHY_TEST) {
+			ELINK_FEATURE_CONFIG_IEEE_PHY_TEST) {
+			uint16_t mii_ctrl;
+
 			elink_cl45_read(sc, phy,
 					MDIO_AN_DEVAD,
 					MDIO_AN_REG_8481_LEGACY_MII_CTRL,
@@ -9748,10 +11491,10 @@ static uint8_t elink_848xx_read_status(struct elink_phy *phy,
 			else
 				vars->duplex = DUPLEX_HALF;
 
-			PMD_DRV_LOG(DEBUG,
-				    "Link is up in %dMbps, is_duplex_full= %d",
-				    vars->line_speed,
-				    (vars->duplex == DUPLEX_FULL));
+			ELINK_DEBUG_P2(sc,
+			   "Link is up in %dMbps, is_duplex_full= %d",
+			   vars->line_speed,
+			   (vars->duplex == DUPLEX_FULL));
 			/* Check legacy speed AN resolution */
 			elink_cl45_read(sc, phy,
 					MDIO_AN_DEVAD,
@@ -9759,19 +11502,19 @@ static uint8_t elink_848xx_read_status(struct elink_phy *phy,
 					&val);
 			if (val & (1 << 5))
 				vars->link_status |=
-				    LINK_STATUS_AUTO_NEGOTIATE_COMPLETE;
+					LINK_STATUS_AUTO_NEGOTIATE_COMPLETE;
 			elink_cl45_read(sc, phy,
 					MDIO_AN_DEVAD,
 					MDIO_AN_REG_8481_LEGACY_AN_EXPANSION,
 					&val);
 			if ((val & (1 << 0)) == 0)
 				vars->link_status |=
-				    LINK_STATUS_PARALLEL_DETECTION_USED;
+					LINK_STATUS_PARALLEL_DETECTION_USED;
 		}
 	}
 	if (link_up) {
-		PMD_DRV_LOG(DEBUG, "BNX2X848x3: link speed is %d",
-			    vars->line_speed);
+		ELINK_DEBUG_P1(sc, "BNX2X848x3: link speed is %d",
+			   vars->line_speed);
 		elink_ext_phy_resolve_fc(phy, params, vars);
 
 		/* Read LP advertised speeds */
@@ -9779,48 +11522,47 @@ static uint8_t elink_848xx_read_status(struct elink_phy *phy,
 				MDIO_AN_REG_CL37_FC_LP, &val);
 		if (val & (1 << 5))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_10THD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_10THD_CAPABLE;
 		if (val & (1 << 6))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_10TFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_10TFD_CAPABLE;
 		if (val & (1 << 7))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_100TXHD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_100TXHD_CAPABLE;
 		if (val & (1 << 8))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_100TXFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_100TXFD_CAPABLE;
 		if (val & (1 << 9))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_100T4_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_100T4_CAPABLE;
 
 		elink_cl45_read(sc, phy, MDIO_AN_DEVAD,
 				MDIO_AN_REG_1000T_STATUS, &val);
 
 		if (val & (1 << 10))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_1000THD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_1000THD_CAPABLE;
 		if (val & (1 << 11))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE;
 
 		elink_cl45_read(sc, phy, MDIO_AN_DEVAD,
 				MDIO_AN_REG_MASTER_STATUS, &val);
 
 		if (val & (1 << 11))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
 
 		/* Determine if EEE was negotiated */
-		if ((phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833) ||
-		    (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834))
+		if (elink_is_8483x_8485x(phy))
 			elink_eee_an_resolve(phy, params, vars);
 	}
 
 	return link_up;
 }
 
-static uint8_t elink_848xx_format_ver(uint32_t raw_ver, uint8_t * str,
-					     uint16_t * len)
+static elink_status_t elink_848xx_format_ver(uint32_t raw_ver, uint8_t *str,
+					     uint16_t *len)
 {
 	elink_status_t status = ELINK_STATUS_OK;
 	uint32_t spirom_ver;
@@ -9833,17 +11575,18 @@ static void elink_8481_hw_reset(__rte_unused struct elink_phy *phy,
 				struct elink_params *params)
 {
 	elink_cb_gpio_write(params->sc, MISC_REGISTERS_GPIO_1,
-			    MISC_REGISTERS_GPIO_OUTPUT_LOW, 0);
+		       MISC_REGISTERS_GPIO_OUTPUT_LOW, 0);
 	elink_cb_gpio_write(params->sc, MISC_REGISTERS_GPIO_1,
-			    MISC_REGISTERS_GPIO_OUTPUT_LOW, 1);
+		       MISC_REGISTERS_GPIO_OUTPUT_LOW, 1);
 }
 
 static void elink_8481_link_reset(struct elink_phy *phy,
-				  struct elink_params *params)
+					struct elink_params *params)
 {
 	elink_cl45_write(params->sc, phy,
 			 MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, 0x0000);
-	elink_cl45_write(params->sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 1);
+	elink_cl45_write(params->sc, phy,
+			 MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 1);
 }
 
 static void elink_848x3_link_reset(struct elink_phy *phy,
@@ -9860,7 +11603,8 @@ static void elink_848x3_link_reset(struct elink_phy *phy,
 
 	if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84823) {
 		elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_3,
-				    MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
+			       MISC_REGISTERS_GPIO_OUTPUT_LOW,
+			       port);
 	} else {
 		elink_cl45_read(sc, phy,
 				MDIO_CTL_DEVAD,
@@ -9877,47 +11621,52 @@ static void elink_848xx_set_link_led(struct elink_phy *phy,
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t val;
-	__rte_unused uint8_t port;
+	uint8_t port;
 
 	if (!(CHIP_IS_E1x(sc)))
 		port = SC_PATH(sc);
 	else
 		port = params->port;
-
 	switch (mode) {
 	case ELINK_LED_MODE_OFF:
 
-		PMD_DRV_LOG(DEBUG, "Port 0x%x: LED MODE OFF", port);
+		ELINK_DEBUG_P1(sc, "Port 0x%x: LED MODE OFF", port);
 
 		if ((params->hw_led_mode << SHARED_HW_CFG_LED_MODE_SHIFT) ==
 		    SHARED_HW_CFG_LED_EXTPHY1) {
 
 			/* Set LED masks */
 			elink_cl45_write(sc, phy,
-					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED1_MASK, 0x0);
+					MDIO_PMA_DEVAD,
+					MDIO_PMA_REG_8481_LED1_MASK,
+					0x0);
 
 			elink_cl45_write(sc, phy,
-					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED2_MASK, 0x0);
+					MDIO_PMA_DEVAD,
+					MDIO_PMA_REG_8481_LED2_MASK,
+					0x0);
 
 			elink_cl45_write(sc, phy,
-					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED3_MASK, 0x0);
+					MDIO_PMA_DEVAD,
+					MDIO_PMA_REG_8481_LED3_MASK,
+					0x0);
 
 			elink_cl45_write(sc, phy,
-					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED5_MASK, 0x0);
+					MDIO_PMA_DEVAD,
+					MDIO_PMA_REG_8481_LED5_MASK,
+					0x0);
 
 		} else {
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED1_MASK, 0x0);
+					 MDIO_PMA_REG_8481_LED1_MASK,
+					 0x0);
 		}
 		break;
 	case ELINK_LED_MODE_FRONT_PANEL_OFF:
 
-		PMD_DRV_LOG(DEBUG, "Port 0x%x: LED MODE FRONT PANEL OFF", port);
+		ELINK_DEBUG_P1(sc, "Port 0x%x: LED MODE FRONT PANEL OFF",
+		   port);
 
 		if ((params->hw_led_mode << SHARED_HW_CFG_LED_MODE_SHIFT) ==
 		    SHARED_HW_CFG_LED_EXTPHY1) {
@@ -9925,25 +11674,31 @@ static void elink_848xx_set_link_led(struct elink_phy *phy,
 			/* Set LED masks */
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED1_MASK, 0x0);
+					 MDIO_PMA_REG_8481_LED1_MASK,
+					 0x0);
 
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED2_MASK, 0x0);
+					 MDIO_PMA_REG_8481_LED2_MASK,
+					 0x0);
 
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED3_MASK, 0x0);
+					 MDIO_PMA_REG_8481_LED3_MASK,
+					 0x0);
 
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED5_MASK, 0x20);
+					 MDIO_PMA_REG_8481_LED5_MASK,
+					 0x20);
 
 		} else {
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED1_MASK, 0x0);
-			if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834) {
+					 MDIO_PMA_REG_8481_LED1_MASK,
+					 0x0);
+			if (phy->type ==
+			    PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834) {
 				/* Disable MI_INT interrupt before setting LED4
 				 * source to constant off.
 				 */
@@ -9951,12 +11706,13 @@ static void elink_848xx_set_link_led(struct elink_phy *phy,
 					   params->port * 4) &
 				    ELINK_NIG_MASK_MI_INT) {
 					params->link_flags |=
-					    ELINK_LINK_FLAGS_INT_DISABLED;
+					ELINK_LINK_FLAGS_INT_DISABLED;
 
-					elink_bits_dis(sc,
-						       NIG_REG_MASK_INTERRUPT_PORT0
-						       + params->port * 4,
-						       ELINK_NIG_MASK_MI_INT);
+					elink_bits_dis(
+						sc,
+						NIG_REG_MASK_INTERRUPT_PORT0 +
+						params->port * 4,
+						ELINK_NIG_MASK_MI_INT);
 				}
 				elink_cl45_write(sc, phy,
 						 MDIO_PMA_DEVAD,
@@ -9967,42 +11723,50 @@ static void elink_848xx_set_link_led(struct elink_phy *phy,
 		break;
 	case ELINK_LED_MODE_ON:
 
-		PMD_DRV_LOG(DEBUG, "Port 0x%x: LED MODE ON", port);
+		ELINK_DEBUG_P1(sc, "Port 0x%x: LED MODE ON", port);
 
 		if ((params->hw_led_mode << SHARED_HW_CFG_LED_MODE_SHIFT) ==
 		    SHARED_HW_CFG_LED_EXTPHY1) {
 			/* Set control reg */
 			elink_cl45_read(sc, phy,
 					MDIO_PMA_DEVAD,
-					MDIO_PMA_REG_8481_LINK_SIGNAL, &val);
+					MDIO_PMA_REG_8481_LINK_SIGNAL,
+					&val);
 			val &= 0x8000;
 			val |= 0x2492;
 
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LINK_SIGNAL, val);
+					 MDIO_PMA_REG_8481_LINK_SIGNAL,
+					 val);
 
 			/* Set LED masks */
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED1_MASK, 0x0);
+					 MDIO_PMA_REG_8481_LED1_MASK,
+					 0x0);
 
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED2_MASK, 0x20);
+					 MDIO_PMA_REG_8481_LED2_MASK,
+					 0x20);
 
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED3_MASK, 0x20);
+					 MDIO_PMA_REG_8481_LED3_MASK,
+					 0x20);
 
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED5_MASK, 0x0);
+					 MDIO_PMA_REG_8481_LED5_MASK,
+					 0x0);
 		} else {
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED1_MASK, 0x20);
-			if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834) {
+					 MDIO_PMA_REG_8481_LED1_MASK,
+					 0x20);
+			if (phy->type ==
+			    PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834) {
 				/* Disable MI_INT interrupt before setting LED4
 				 * source to constant on.
 				 */
@@ -10010,12 +11774,13 @@ static void elink_848xx_set_link_led(struct elink_phy *phy,
 					   params->port * 4) &
 				    ELINK_NIG_MASK_MI_INT) {
 					params->link_flags |=
-					    ELINK_LINK_FLAGS_INT_DISABLED;
+					ELINK_LINK_FLAGS_INT_DISABLED;
 
-					elink_bits_dis(sc,
-						       NIG_REG_MASK_INTERRUPT_PORT0
-						       + params->port * 4,
-						       ELINK_NIG_MASK_MI_INT);
+					elink_bits_dis(
+						sc,
+						NIG_REG_MASK_INTERRUPT_PORT0 +
+						params->port * 4,
+						ELINK_NIG_MASK_MI_INT);
 				}
 				elink_cl45_write(sc, phy,
 						 MDIO_PMA_DEVAD,
@@ -10027,7 +11792,7 @@ static void elink_848xx_set_link_led(struct elink_phy *phy,
 
 	case ELINK_LED_MODE_OPER:
 
-		PMD_DRV_LOG(DEBUG, "Port 0x%x: LED MODE OPER", port);
+		ELINK_DEBUG_P1(sc, "Port 0x%x: LED MODE OPER", port);
 
 		if ((params->hw_led_mode << SHARED_HW_CFG_LED_MODE_SHIFT) ==
 		    SHARED_HW_CFG_LED_EXTPHY1) {
@@ -10035,14 +11800,13 @@ static void elink_848xx_set_link_led(struct elink_phy *phy,
 			/* Set control reg */
 			elink_cl45_read(sc, phy,
 					MDIO_PMA_DEVAD,
-					MDIO_PMA_REG_8481_LINK_SIGNAL, &val);
+					MDIO_PMA_REG_8481_LINK_SIGNAL,
+					&val);
 
 			if (!((val &
 			       MDIO_PMA_REG_8481_LINK_SIGNAL_LED4_ENABLE_MASK)
-			      >>
-			      MDIO_PMA_REG_8481_LINK_SIGNAL_LED4_ENABLE_SHIFT))
-			{
-				PMD_DRV_LOG(DEBUG, "Setting LINK_SIGNAL");
+			  >> MDIO_PMA_REG_8481_LINK_SIGNAL_LED4_ENABLE_SHIFT)) {
+				ELINK_DEBUG_P0(sc, "Setting LINK_SIGNAL");
 				elink_cl45_write(sc, phy,
 						 MDIO_PMA_DEVAD,
 						 MDIO_PMA_REG_8481_LINK_SIGNAL,
@@ -10052,19 +11816,23 @@ static void elink_848xx_set_link_led(struct elink_phy *phy,
 			/* Set LED masks */
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED1_MASK, 0x10);
+					 MDIO_PMA_REG_8481_LED1_MASK,
+					 0x10);
 
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED2_MASK, 0x80);
+					 MDIO_PMA_REG_8481_LED2_MASK,
+					 0x80);
 
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED3_MASK, 0x98);
+					 MDIO_PMA_REG_8481_LED3_MASK,
+					 0x98);
 
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED5_MASK, 0x40);
+					 MDIO_PMA_REG_8481_LED5_MASK,
+					 0x40);
 
 		} else {
 			/* EXTPHY2 LED mode indicate that the 100M/1G/10G LED
@@ -10077,18 +11845,22 @@ static void elink_848xx_set_link_led(struct elink_phy *phy,
 
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LED1_MASK, val);
+					 MDIO_PMA_REG_8481_LED1_MASK,
+					 val);
 
 			/* Tell LED3 to blink on source */
 			elink_cl45_read(sc, phy,
 					MDIO_PMA_DEVAD,
-					MDIO_PMA_REG_8481_LINK_SIGNAL, &val);
+					MDIO_PMA_REG_8481_LINK_SIGNAL,
+					&val);
 			val &= ~(7 << 6);
-			val |= (1 << 6);	/* A83B[8:6]= 1 */
+			val |= (1 << 6); /* A83B[8:6]= 1 */
 			elink_cl45_write(sc, phy,
 					 MDIO_PMA_DEVAD,
-					 MDIO_PMA_REG_8481_LINK_SIGNAL, val);
-			if (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834) {
+					 MDIO_PMA_REG_8481_LINK_SIGNAL,
+					 val);
+			if (phy->type ==
+			    PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834) {
 				/* Restore LED4 source to external link,
 				 * and re-enable interrupts.
 				 */
@@ -10100,14 +11872,14 @@ static void elink_848xx_set_link_led(struct elink_phy *phy,
 				    ELINK_LINK_FLAGS_INT_DISABLED) {
 					elink_link_int_enable(params);
 					params->link_flags &=
-					    ~ELINK_LINK_FLAGS_INT_DISABLED;
+						~ELINK_LINK_FLAGS_INT_DISABLED;
 				}
 			}
 		}
 		break;
 	}
 
-	/* This is a workaround for E3+84833 until autoneg
+	/* This is a workaround for E3 + 84833 until autoneg
 	 * restart is fixed in f/w
 	 */
 	if (CHIP_IS_E3(sc)) {
@@ -10132,7 +11904,9 @@ static void elink_54618se_specific_func(struct elink_phy *phy,
 		elink_cl22_write(sc, phy,
 				 MDIO_REG_GPHY_SHADOW,
 				 MDIO_REG_GPHY_SHADOW_LED_SEL2);
-		elink_cl22_read(sc, phy, MDIO_REG_GPHY_SHADOW, &temp);
+		elink_cl22_read(sc, phy,
+				MDIO_REG_GPHY_SHADOW,
+				&temp);
 		temp &= ~(0xf << 4);
 		temp |= (0x6 << 4);
 		elink_cl22_write(sc, phy,
@@ -10147,15 +11921,15 @@ static void elink_54618se_specific_func(struct elink_phy *phy,
 }
 
 static uint8_t elink_54618se_config_init(struct elink_phy *phy,
-					 struct elink_params *params,
-					 struct elink_vars *vars)
+					       struct elink_params *params,
+					       struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t port;
 	uint16_t autoneg_val, an_1000_val, an_10_100_val, fc_val, temp;
 	uint32_t cfg_pin;
 
-	PMD_DRV_LOG(DEBUG, "54618SE cfg init");
+	ELINK_DEBUG_P0(sc, "54618SE cfg init");
 	DELAY(1000 * 1);
 
 	/* This works with E3 only, no need to check the chip
@@ -10164,11 +11938,10 @@ static uint8_t elink_54618se_config_init(struct elink_phy *phy,
 	port = params->port;
 
 	cfg_pin = (REG_RD(sc, params->shmem_base +
-			  offsetof(struct shmem_region,
-				   dev_info.port_hw_config[port].
-				   e3_cmn_pin_cfg)) &
-		   PORT_HW_CFG_E3_PHY_RESET_MASK) >>
-	    PORT_HW_CFG_E3_PHY_RESET_SHIFT;
+			offsetof(struct shmem_region,
+			dev_info.port_hw_config[port].e3_cmn_pin_cfg)) &
+			PORT_HW_CFG_E3_PHY_RESET_MASK) >>
+			PORT_HW_CFG_E3_PHY_RESET_SHIFT;
 
 	/* Drive pin high to bring the GPHY out of reset. */
 	elink_set_cfg_pin(sc, cfg_pin, 1);
@@ -10177,63 +11950,76 @@ static uint8_t elink_54618se_config_init(struct elink_phy *phy,
 	DELAY(1000 * 50);
 
 	/* reset phy */
-	elink_cl22_write(sc, phy, MDIO_PMA_REG_CTRL, 0x8000);
+	elink_cl22_write(sc, phy,
+			 MDIO_PMA_REG_CTRL, 0x8000);
 	elink_wait_reset_complete(sc, phy, params);
 
 	/* Wait for GPHY to reset */
 	DELAY(1000 * 50);
 
+
 	elink_54618se_specific_func(phy, params, ELINK_PHY_INIT);
 	/* Flip the signal detect polarity (set 0x1c.0x1e[8]). */
 	elink_cl22_write(sc, phy,
-			 MDIO_REG_GPHY_SHADOW,
-			 MDIO_REG_GPHY_SHADOW_AUTO_DET_MED);
-	elink_cl22_read(sc, phy, MDIO_REG_GPHY_SHADOW, &temp);
+			MDIO_REG_GPHY_SHADOW,
+			MDIO_REG_GPHY_SHADOW_AUTO_DET_MED);
+	elink_cl22_read(sc, phy,
+			MDIO_REG_GPHY_SHADOW,
+			&temp);
 	temp |= MDIO_REG_GPHY_SHADOW_INVERT_FIB_SD;
 	elink_cl22_write(sc, phy,
-			 MDIO_REG_GPHY_SHADOW,
-			 MDIO_REG_GPHY_SHADOW_WR_ENA | temp);
+			MDIO_REG_GPHY_SHADOW,
+			MDIO_REG_GPHY_SHADOW_WR_ENA | temp);
 
 	/* Set up fc */
 	/* Please refer to Table 28B-3 of 802.3ab-1999 spec. */
 	elink_calc_ieee_aneg_adv(phy, params, &vars->ieee_fc);
 	fc_val = 0;
 	if ((vars->ieee_fc & MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC) ==
-	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC)
+			MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC)
 		fc_val |= MDIO_AN_REG_ADV_PAUSE_ASYMMETRIC;
 
 	if ((vars->ieee_fc & MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH) ==
-	    MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH)
+			MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH)
 		fc_val |= MDIO_AN_REG_ADV_PAUSE_PAUSE;
 
 	/* Read all advertisement */
-	elink_cl22_read(sc, phy, 0x09, &an_1000_val);
+	elink_cl22_read(sc, phy,
+			0x09,
+			&an_1000_val);
 
-	elink_cl22_read(sc, phy, 0x04, &an_10_100_val);
+	elink_cl22_read(sc, phy,
+			0x04,
+			&an_10_100_val);
 
-	elink_cl22_read(sc, phy, MDIO_PMA_REG_CTRL, &autoneg_val);
+	elink_cl22_read(sc, phy,
+			MDIO_PMA_REG_CTRL,
+			&autoneg_val);
 
 	/* Disable forced speed */
-	autoneg_val &=
-	    ~((1 << 6) | (1 << 8) | (1 << 9) | (1 << 12) | (1 << 13));
-	an_10_100_val &=
-	    ~((1 << 5) | (1 << 6) | (1 << 7) | (1 << 8) | (1 << 10) |
-	      (1 << 11));
+	autoneg_val &= ~((1 << 6) | (1 << 8) | (1 << 9) | (1 << 12) |
+			 (1 << 13));
+	an_10_100_val &= ~((1 << 5) | (1 << 6) | (1 << 7) | (1 << 8) |
+			   (1 << 10) | (1 << 11));
 
 	if (((phy->req_line_speed == ELINK_SPEED_AUTO_NEG) &&
-	     (phy->speed_cap_mask &
-	      PORT_HW_CFG_SPEED_CAPABILITY_D0_1G)) ||
-	    (phy->req_line_speed == ELINK_SPEED_1000)) {
+			(phy->speed_cap_mask &
+			PORT_HW_CFG_SPEED_CAPABILITY_D0_1G)) ||
+			(phy->req_line_speed == ELINK_SPEED_1000)) {
 		an_1000_val |= (1 << 8);
 		autoneg_val |= (1 << 9 | 1 << 12);
 		if (phy->req_duplex == DUPLEX_FULL)
 			an_1000_val |= (1 << 9);
-		PMD_DRV_LOG(DEBUG, "Advertising 1G");
+		ELINK_DEBUG_P0(sc, "Advertising 1G");
 	} else
 		an_1000_val &= ~((1 << 8) | (1 << 9));
 
-	elink_cl22_write(sc, phy, 0x09, an_1000_val);
-	elink_cl22_read(sc, phy, 0x09, &an_1000_val);
+	elink_cl22_write(sc, phy,
+			0x09,
+			an_1000_val);
+	elink_cl22_read(sc, phy,
+			0x09,
+			&an_1000_val);
 
 	/* Advertise 10/100 link speed */
 	if (phy->req_line_speed == ELINK_SPEED_AUTO_NEG) {
@@ -10241,25 +12027,25 @@ static uint8_t elink_54618se_config_init(struct elink_phy *phy,
 		    PORT_HW_CFG_SPEED_CAPABILITY_D0_10M_HALF) {
 			an_10_100_val |= (1 << 5);
 			autoneg_val |= (1 << 9 | 1 << 12);
-			PMD_DRV_LOG(DEBUG, "Advertising 10M-HD");
+			ELINK_DEBUG_P0(sc, "Advertising 10M-HD");
 		}
 		if (phy->speed_cap_mask &
-		    PORT_HW_CFG_SPEED_CAPABILITY_D0_10M_HALF) {
+		    PORT_HW_CFG_SPEED_CAPABILITY_D0_10M_FULL) {
 			an_10_100_val |= (1 << 6);
 			autoneg_val |= (1 << 9 | 1 << 12);
-			PMD_DRV_LOG(DEBUG, "Advertising 10M-FD");
+			ELINK_DEBUG_P0(sc, "Advertising 10M-FD");
 		}
 		if (phy->speed_cap_mask &
 		    PORT_HW_CFG_SPEED_CAPABILITY_D0_100M_HALF) {
 			an_10_100_val |= (1 << 7);
 			autoneg_val |= (1 << 9 | 1 << 12);
-			PMD_DRV_LOG(DEBUG, "Advertising 100M-HD");
+			ELINK_DEBUG_P0(sc, "Advertising 100M-HD");
 		}
 		if (phy->speed_cap_mask &
 		    PORT_HW_CFG_SPEED_CAPABILITY_D0_100M_FULL) {
 			an_10_100_val |= (1 << 8);
 			autoneg_val |= (1 << 9 | 1 << 12);
-			PMD_DRV_LOG(DEBUG, "Advertising 100M-FD");
+			ELINK_DEBUG_P0(sc, "Advertising 100M-FD");
 		}
 	}
 
@@ -10267,13 +12053,17 @@ static uint8_t elink_54618se_config_init(struct elink_phy *phy,
 	if (phy->req_line_speed == ELINK_SPEED_100) {
 		autoneg_val |= (1 << 13);
 		/* Enabled AUTO-MDIX when autoneg is disabled */
-		elink_cl22_write(sc, phy, 0x18, (1 << 15 | 1 << 9 | 7 << 0));
-		PMD_DRV_LOG(DEBUG, "Setting 100M force");
+		elink_cl22_write(sc, phy,
+				0x18,
+				(1 << 15 | 1 << 9 | 7 << 0));
+		ELINK_DEBUG_P0(sc, "Setting 100M force");
 	}
 	if (phy->req_line_speed == ELINK_SPEED_10) {
 		/* Enabled AUTO-MDIX when autoneg is disabled */
-		elink_cl22_write(sc, phy, 0x18, (1 << 15 | 1 << 9 | 7 << 0));
-		PMD_DRV_LOG(DEBUG, "Setting 10M force");
+		elink_cl22_write(sc, phy,
+				0x18,
+				(1 << 15 | 1 << 9 | 7 << 0));
+		ELINK_DEBUG_P0(sc, "Setting 10M force");
 	}
 
 	if ((phy->flags & ELINK_FLAGS_EEE) && elink_eee_has_cap(params)) {
@@ -10288,7 +12078,7 @@ static uint8_t elink_54618se_config_init(struct elink_phy *phy,
 
 		rc = elink_eee_initial_config(params, vars, SHMEM_EEE_1G_ADV);
 		if (rc != ELINK_STATUS_OK) {
-			PMD_DRV_LOG(DEBUG, "Failed to configure EEE timers");
+			ELINK_DEBUG_P0(sc, "Failed to configure EEE timers");
 			elink_eee_disable(phy, params, vars);
 		} else if ((params->eee_mode & ELINK_EEE_MODE_ADV_LPI) &&
 			   (phy->req_duplex == DUPLEX_FULL) &&
@@ -10302,38 +12092,42 @@ static uint8_t elink_54618se_config_init(struct elink_phy *phy,
 			elink_eee_advertise(phy, params, vars,
 					    SHMEM_EEE_1G_ADV);
 		} else {
-			PMD_DRV_LOG(DEBUG, "Don't Advertise 1GBase-T EEE");
+			ELINK_DEBUG_P0(sc, "Don't Advertise 1GBase-T EEE");
 			elink_eee_disable(phy, params, vars);
 		}
 	} else {
-		vars->eee_status &= ~SHMEM_EEE_1G_ADV <<
-		    SHMEM_EEE_SUPPORTED_SHIFT;
+		vars->eee_status &= ((uint32_t)(~SHMEM_EEE_1G_ADV) <<
+				    SHMEM_EEE_SUPPORTED_SHIFT);
 
 		if (phy->flags & ELINK_FLAGS_EEE) {
 			/* Handle legacy auto-grEEEn */
 			if (params->feature_config_flags &
 			    ELINK_FEATURE_CONFIG_AUTOGREEEN_ENABLED) {
 				temp = 6;
-				PMD_DRV_LOG(DEBUG, "Enabling Auto-GrEEEn");
+				ELINK_DEBUG_P0(sc, "Enabling Auto-GrEEEn");
 			} else {
 				temp = 0;
-				PMD_DRV_LOG(DEBUG, "Don't Adv. EEE");
+				ELINK_DEBUG_P0(sc, "Don't Adv. EEE");
 			}
 			elink_cl45_write(sc, phy, MDIO_AN_DEVAD,
 					 MDIO_AN_REG_EEE_ADV, temp);
 		}
 	}
 
-	elink_cl22_write(sc, phy, 0x04, an_10_100_val | fc_val);
+	elink_cl22_write(sc, phy,
+			0x04,
+			an_10_100_val | fc_val);
 
 	if (phy->req_duplex == DUPLEX_FULL)
 		autoneg_val |= (1 << 8);
 
-	elink_cl22_write(sc, phy, MDIO_PMA_REG_CTRL, autoneg_val);
+	elink_cl22_write(sc, phy,
+			MDIO_PMA_REG_CTRL, autoneg_val);
 
 	return ELINK_STATUS_OK;
 }
 
+
 static void elink_5461x_set_link_led(struct elink_phy *phy,
 				     struct elink_params *params, uint8_t mode)
 {
@@ -10341,11 +12135,14 @@ static void elink_5461x_set_link_led(struct elink_phy *phy,
 	uint16_t temp;
 
 	elink_cl22_write(sc, phy,
-			 MDIO_REG_GPHY_SHADOW, MDIO_REG_GPHY_SHADOW_LED_SEL1);
-	elink_cl22_read(sc, phy, MDIO_REG_GPHY_SHADOW, &temp);
+		MDIO_REG_GPHY_SHADOW,
+		MDIO_REG_GPHY_SHADOW_LED_SEL1);
+	elink_cl22_read(sc, phy,
+		MDIO_REG_GPHY_SHADOW,
+		&temp);
 	temp &= 0xff00;
 
-	PMD_DRV_LOG(DEBUG, "54618x set link led (mode=%x)", mode);
+	ELINK_DEBUG_P1(sc, "54618x set link led (mode=%x)", mode);
 	switch (mode) {
 	case ELINK_LED_MODE_FRONT_PANEL_OFF:
 	case ELINK_LED_MODE_OFF:
@@ -10361,11 +12158,12 @@ static void elink_5461x_set_link_led(struct elink_phy *phy,
 		break;
 	}
 	elink_cl22_write(sc, phy,
-			 MDIO_REG_GPHY_SHADOW,
-			 MDIO_REG_GPHY_SHADOW_WR_ENA | temp);
+		MDIO_REG_GPHY_SHADOW,
+		MDIO_REG_GPHY_SHADOW_WR_ENA | temp);
 	return;
 }
 
+
 static void elink_54618se_link_reset(struct elink_phy *phy,
 				     struct elink_params *params)
 {
@@ -10382,19 +12180,18 @@ static void elink_54618se_link_reset(struct elink_phy *phy,
 	 */
 	port = params->port;
 	cfg_pin = (REG_RD(sc, params->shmem_base +
-			  offsetof(struct shmem_region,
-				   dev_info.port_hw_config[port].
-				   e3_cmn_pin_cfg)) &
-		   PORT_HW_CFG_E3_PHY_RESET_MASK) >>
-	    PORT_HW_CFG_E3_PHY_RESET_SHIFT;
+			offsetof(struct shmem_region,
+			dev_info.port_hw_config[port].e3_cmn_pin_cfg)) &
+			PORT_HW_CFG_E3_PHY_RESET_MASK) >>
+			PORT_HW_CFG_E3_PHY_RESET_SHIFT;
 
 	/* Drive pin low to put GPHY in reset. */
 	elink_set_cfg_pin(sc, cfg_pin, 0);
 }
 
 static uint8_t elink_54618se_read_status(struct elink_phy *phy,
-					 struct elink_params *params,
-					 struct elink_vars *vars)
+				    struct elink_params *params,
+				    struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t val;
@@ -10402,11 +12199,15 @@ static uint8_t elink_54618se_read_status(struct elink_phy *phy,
 	uint16_t legacy_status, legacy_speed;
 
 	/* Get speed operation status */
-	elink_cl22_read(sc, phy, MDIO_REG_GPHY_AUX_STATUS, &legacy_status);
-	PMD_DRV_LOG(DEBUG, "54618SE read_status: 0x%x", legacy_status);
+	elink_cl22_read(sc, phy,
+			MDIO_REG_GPHY_AUX_STATUS,
+			&legacy_status);
+	ELINK_DEBUG_P1(sc, "54618SE read_status: 0x%x", legacy_status);
 
 	/* Read status to clear the PHY interrupt. */
-	elink_cl22_read(sc, phy, MDIO_REG_INTR_STATUS, &val);
+	elink_cl22_read(sc, phy,
+			MDIO_REG_INTR_STATUS,
+			&val);
 
 	link_up = ((legacy_status & (1 << 2)) == (1 << 2));
 
@@ -10432,25 +12233,30 @@ static uint8_t elink_54618se_read_status(struct elink_phy *phy,
 		} else if (legacy_speed == (1 << 8)) {
 			vars->line_speed = ELINK_SPEED_10;
 			vars->duplex = DUPLEX_HALF;
-		} else		/* Should not happen */
+		} else /* Should not happen */
 			vars->line_speed = 0;
 
-		PMD_DRV_LOG(DEBUG,
-			    "Link is up in %dMbps, is_duplex_full= %d",
-			    vars->line_speed, (vars->duplex == DUPLEX_FULL));
+		ELINK_DEBUG_P2(sc,
+		   "Link is up in %dMbps, is_duplex_full= %d",
+		   vars->line_speed,
+		   (vars->duplex == DUPLEX_FULL));
 
 		/* Check legacy speed AN resolution */
-		elink_cl22_read(sc, phy, 0x01, &val);
+		elink_cl22_read(sc, phy,
+				0x01,
+				&val);
 		if (val & (1 << 5))
 			vars->link_status |=
-			    LINK_STATUS_AUTO_NEGOTIATE_COMPLETE;
-		elink_cl22_read(sc, phy, 0x06, &val);
+				LINK_STATUS_AUTO_NEGOTIATE_COMPLETE;
+		elink_cl22_read(sc, phy,
+				0x06,
+				&val);
 		if ((val & (1 << 0)) == 0)
 			vars->link_status |=
-			    LINK_STATUS_PARALLEL_DETECTION_USED;
+				LINK_STATUS_PARALLEL_DETECTION_USED;
 
-		PMD_DRV_LOG(DEBUG, "BNX2X54618SE: link speed is %d",
-			    vars->line_speed);
+		ELINK_DEBUG_P1(sc, "BNX2X4618SE: link speed is %d",
+			   vars->line_speed);
 
 		elink_ext_phy_resolve_fc(phy, params, vars);
 
@@ -10460,27 +12266,27 @@ static uint8_t elink_54618se_read_status(struct elink_phy *phy,
 
 			if (val & (1 << 5))
 				vars->link_status |=
-				    LINK_STATUS_LINK_PARTNER_10THD_CAPABLE;
+				  LINK_STATUS_LINK_PARTNER_10THD_CAPABLE;
 			if (val & (1 << 6))
 				vars->link_status |=
-				    LINK_STATUS_LINK_PARTNER_10TFD_CAPABLE;
+				  LINK_STATUS_LINK_PARTNER_10TFD_CAPABLE;
 			if (val & (1 << 7))
 				vars->link_status |=
-				    LINK_STATUS_LINK_PARTNER_100TXHD_CAPABLE;
+				  LINK_STATUS_LINK_PARTNER_100TXHD_CAPABLE;
 			if (val & (1 << 8))
 				vars->link_status |=
-				    LINK_STATUS_LINK_PARTNER_100TXFD_CAPABLE;
+				  LINK_STATUS_LINK_PARTNER_100TXFD_CAPABLE;
 			if (val & (1 << 9))
 				vars->link_status |=
-				    LINK_STATUS_LINK_PARTNER_100T4_CAPABLE;
+				  LINK_STATUS_LINK_PARTNER_100T4_CAPABLE;
 
 			elink_cl22_read(sc, phy, 0xa, &val);
 			if (val & (1 << 10))
 				vars->link_status |=
-				    LINK_STATUS_LINK_PARTNER_1000THD_CAPABLE;
+				  LINK_STATUS_LINK_PARTNER_1000THD_CAPABLE;
 			if (val & (1 << 11))
 				vars->link_status |=
-				    LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE;
+				  LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE;
 
 			if ((phy->flags & ELINK_FLAGS_EEE) &&
 			    elink_eee_has_cap(params))
@@ -10497,7 +12303,7 @@ static void elink_54618se_config_loopback(struct elink_phy *phy,
 	uint16_t val;
 	uint32_t umac_base = params->port ? GRCBASE_UMAC1 : GRCBASE_UMAC0;
 
-	PMD_DRV_LOG(DEBUG, "2PMA/PMD ext_phy_loopback: 54618se");
+	ELINK_DEBUG_P0(sc, "2PMA/PMD ext_phy_loopback: 54618se");
 
 	/* Enable master/slave manual mmode and set to master */
 	/* mii write 9 [bits set 11 12] */
@@ -10543,30 +12349,33 @@ static void elink_7101_config_loopback(struct elink_phy *phy,
 }
 
 static uint8_t elink_7101_config_init(struct elink_phy *phy,
-				      struct elink_params *params,
-				      struct elink_vars *vars)
+				  struct elink_params *params,
+				  struct elink_vars *vars)
 {
 	uint16_t fw_ver1, fw_ver2, val;
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "Setting the SFX7101 LASI indication");
+	ELINK_DEBUG_P0(sc, "Setting the SFX7101 LASI indication");
 
-	/* Restore normal power mode */
+	/* Restore normal power mode*/
 	elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_2,
-			    MISC_REGISTERS_GPIO_OUTPUT_HIGH, params->port);
+		       MISC_REGISTERS_GPIO_OUTPUT_HIGH, params->port);
 	/* HW reset */
 	elink_ext_phy_hw_reset(sc, params->port);
 	elink_wait_reset_complete(sc, phy, params);
 
-	elink_cl45_write(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL, 0x1);
-	PMD_DRV_LOG(DEBUG, "Setting the SFX7101 LED to blink on traffic");
+	elink_cl45_write(sc, phy,
+			 MDIO_PMA_DEVAD, MDIO_PMA_LASI_CTRL, 0x1);
+	ELINK_DEBUG_P0(sc, "Setting the SFX7101 LED to blink on traffic");
 	elink_cl45_write(sc, phy,
 			 MDIO_PMA_DEVAD, MDIO_PMA_REG_7107_LED_CNTL, (1 << 3));
 
 	elink_ext_phy_set_pause(params, phy, vars);
 	/* Restart autoneg */
-	elink_cl45_read(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, &val);
+	elink_cl45_read(sc, phy,
+			MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, &val);
 	val |= 0x200;
-	elink_cl45_write(sc, phy, MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, val);
+	elink_cl45_write(sc, phy,
+			 MDIO_AN_DEVAD, MDIO_AN_REG_CTRL, val);
 
 	/* Save spirom version */
 	elink_cl45_read(sc, phy,
@@ -10575,24 +12384,30 @@ static uint8_t elink_7101_config_init(struct elink_phy *phy,
 	elink_cl45_read(sc, phy,
 			MDIO_PMA_DEVAD, MDIO_PMA_REG_7101_VER2, &fw_ver2);
 	elink_save_spirom_version(sc, params->port,
-				  (uint32_t) (fw_ver1 << 16 | fw_ver2),
+				  (uint32_t)(fw_ver1 << 16 | fw_ver2),
 				  phy->ver_addr);
 	return ELINK_STATUS_OK;
 }
 
 static uint8_t elink_7101_read_status(struct elink_phy *phy,
-				      struct elink_params *params,
-				      struct elink_vars *vars)
+				 struct elink_params *params,
+				 struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t link_up;
 	uint16_t val1, val2;
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val2);
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val1);
-	PMD_DRV_LOG(DEBUG, "10G-base-T LASI status 0x%x->0x%x", val2, val1);
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val2);
-	elink_cl45_read(sc, phy, MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val1);
-	PMD_DRV_LOG(DEBUG, "10G-base-T PMA status 0x%x->0x%x", val2, val1);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val2);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_LASI_STAT, &val1);
+	ELINK_DEBUG_P2(sc, "10G-base-T LASI status 0x%x->0x%x",
+		   val2, val1);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val2);
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD, MDIO_PMA_REG_STATUS, &val1);
+	ELINK_DEBUG_P2(sc, "10G-base-T PMA status 0x%x->0x%x",
+		   val2, val1);
 	link_up = ((val1 & 4) == 4);
 	/* If link is up print the AN outcome of the SFX7101 PHY */
 	if (link_up) {
@@ -10601,21 +12416,21 @@ static uint8_t elink_7101_read_status(struct elink_phy *phy,
 				&val2);
 		vars->line_speed = ELINK_SPEED_10000;
 		vars->duplex = DUPLEX_FULL;
-		PMD_DRV_LOG(DEBUG, "SFX7101 AN status 0x%x->Master=%x",
-			    val2, (val2 & (1 << 14)));
+		ELINK_DEBUG_P2(sc, "SFX7101 AN status 0x%x->Master=%x",
+			   val2, (val2 & (1 << 14)));
 		elink_ext_phy_10G_an_resolve(sc, phy, vars);
 		elink_ext_phy_resolve_fc(phy, params, vars);
 
 		/* Read LP advertised speeds */
 		if (val2 & (1 << 11))
 			vars->link_status |=
-			    LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
+				LINK_STATUS_LINK_PARTNER_10GXFD_CAPABLE;
 	}
 	return link_up;
 }
 
-static uint8_t elink_7101_format_ver(uint32_t spirom_ver, uint8_t * str,
-				     uint16_t * len)
+static elink_status_t elink_7101_format_ver(uint32_t spirom_ver, uint8_t *str,
+					    uint16_t *len)
 {
 	if (*len < 5)
 		return ELINK_STATUS_ERROR;
@@ -10628,15 +12443,39 @@ static uint8_t elink_7101_format_ver(uint32_t spirom_ver, uint8_t * str,
 	return ELINK_STATUS_OK;
 }
 
-static void elink_7101_hw_reset(__rte_unused struct elink_phy *phy,
-				struct elink_params *params)
+void elink_sfx7101_sp_sw_reset(struct bnx2x_softc *sc, struct elink_phy *phy)
 {
+	uint16_t val, cnt;
+
+	elink_cl45_read(sc, phy,
+			MDIO_PMA_DEVAD,
+			MDIO_PMA_REG_7101_RESET, &val);
+
+	for (cnt = 0; cnt < 10; cnt++) {
+		DELAY(1000 * 50);
+		/* Writes a self-clearing reset */
+		elink_cl45_write(sc, phy,
+				 MDIO_PMA_DEVAD,
+				 MDIO_PMA_REG_7101_RESET,
+				 (val | (1 << 15)));
+		/* Wait for clear */
+		elink_cl45_read(sc, phy,
+				MDIO_PMA_DEVAD,
+				MDIO_PMA_REG_7101_RESET, &val);
+
+		if ((val & (1 << 15)) == 0)
+			break;
+	}
+}
+
+static void elink_7101_hw_reset(__rte_unused struct elink_phy *phy,
+				struct elink_params *params) {
 	/* Low power mode is controlled by GPIO 2 */
 	elink_cb_gpio_write(params->sc, MISC_REGISTERS_GPIO_2,
-			    MISC_REGISTERS_GPIO_OUTPUT_LOW, params->port);
+		       MISC_REGISTERS_GPIO_OUTPUT_LOW, params->port);
 	/* The PHY reset is controlled by GPIO 1 */
 	elink_cb_gpio_write(params->sc, MISC_REGISTERS_GPIO_1,
-			    MISC_REGISTERS_GPIO_OUTPUT_LOW, params->port);
+		       MISC_REGISTERS_GPIO_OUTPUT_LOW, params->port);
 }
 
 static void elink_7101_set_link_led(struct elink_phy *phy,
@@ -10657,7 +12496,9 @@ static void elink_7101_set_link_led(struct elink_phy *phy,
 		break;
 	}
 	elink_cl45_write(sc, phy,
-			 MDIO_PMA_DEVAD, MDIO_PMA_REG_7107_LINK_LED_CNTL, val);
+			 MDIO_PMA_DEVAD,
+			 MDIO_PMA_REG_7107_LINK_LED_CNTL,
+			 val);
 }
 
 /******************************************************************/
@@ -10665,482 +12506,532 @@ static void elink_7101_set_link_led(struct elink_phy *phy,
 /******************************************************************/
 
 static const struct elink_phy phy_null = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_NOT_CONN,
-	.addr = 0,
-	.def_md_devad = 0,
-	.flags = ELINK_FLAGS_INIT_XGXS_FIRST,
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = 0,
-	.media_type = ELINK_ETH_PHY_NOT_PRESENT,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = NULL,
-	.read_status = NULL,
-	.link_reset = NULL,
-	.config_loopback = NULL,
-	.format_fw_ver = NULL,
-	.hw_reset = NULL,
-	.set_link_led = NULL,
-	.phy_specific_func = NULL
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_NOT_CONN,
+	.addr		= 0,
+	.def_md_devad	= 0,
+	.flags		= ELINK_FLAGS_INIT_XGXS_FIRST,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= 0,
+	.media_type	= ELINK_ETH_PHY_NOT_PRESENT,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)NULL,
+	.read_status	= (read_status_t)NULL,
+	.link_reset	= (link_reset_t)NULL,
+	.config_loopback = (config_loopback_t)NULL,
+	.format_fw_ver	= (format_fw_ver_t)NULL,
+	.hw_reset	= (hw_reset_t)NULL,
+	.set_link_led	= (set_link_led_t)NULL,
+	.phy_specific_func = (phy_specific_func_t)NULL
 };
 
 static const struct elink_phy phy_serdes = {
-	.type = PORT_HW_CFG_SERDES_EXT_PHY_TYPE_DIRECT,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = 0,
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10baseT_Half |
-		      ELINK_SUPPORTED_10baseT_Full |
-		      ELINK_SUPPORTED_100baseT_Half |
-		      ELINK_SUPPORTED_100baseT_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_2500baseX_Full |
-		      ELINK_SUPPORTED_TP |
-		      ELINK_SUPPORTED_Autoneg |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_BASE_T,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_xgxs_config_init,
-	.read_status = elink_link_settings_status,
-	.link_reset = elink_int_link_reset,
-	.config_loopback = NULL,
-	.format_fw_ver = NULL,
-	.hw_reset = NULL,
-	.set_link_led = NULL,
-	.phy_specific_func = NULL
+	.type		= PORT_HW_CFG_SERDES_EXT_PHY_TYPE_DIRECT,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= 0,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10baseT_Half |
+			   ELINK_SUPPORTED_10baseT_Full |
+			   ELINK_SUPPORTED_100baseT_Half |
+			   ELINK_SUPPORTED_100baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_2500baseX_Full |
+			   ELINK_SUPPORTED_TP |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_BASE_T,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_xgxs_config_init,
+	.read_status	= (read_status_t)elink_link_settings_status,
+	.link_reset	= (link_reset_t)elink_int_link_reset,
+	.config_loopback = (config_loopback_t)NULL,
+	.format_fw_ver	= (format_fw_ver_t)NULL,
+	.hw_reset	= (hw_reset_t)NULL,
+	.set_link_led	= (set_link_led_t)NULL,
+	.phy_specific_func = (phy_specific_func_t)NULL
 };
 
 static const struct elink_phy phy_xgxs = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = 0,
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10baseT_Half |
-		      ELINK_SUPPORTED_10baseT_Full |
-		      ELINK_SUPPORTED_100baseT_Half |
-		      ELINK_SUPPORTED_100baseT_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_2500baseX_Full |
-		      ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_FIBRE |
-		      ELINK_SUPPORTED_Autoneg |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_CX4,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_xgxs_config_init,
-	.read_status = elink_link_settings_status,
-	.link_reset = elink_int_link_reset,
-	.config_loopback = elink_set_xgxs_loopback,
-	.format_fw_ver = NULL,
-	.hw_reset = NULL,
-	.set_link_led = NULL,
-	.phy_specific_func = elink_xgxs_specific_func
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= 0,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10baseT_Half |
+			   ELINK_SUPPORTED_10baseT_Full |
+			   ELINK_SUPPORTED_100baseT_Half |
+			   ELINK_SUPPORTED_100baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_2500baseX_Full |
+			   ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_FIBRE |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_CX4,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_xgxs_config_init,
+	.read_status	= (read_status_t)elink_link_settings_status,
+	.link_reset	= (link_reset_t)elink_int_link_reset,
+	.config_loopback = (config_loopback_t)elink_set_xgxs_loopback,
+	.format_fw_ver	= (format_fw_ver_t)NULL,
+	.hw_reset	= (hw_reset_t)NULL,
+	.set_link_led	= (set_link_led_t)NULL,
+	.phy_specific_func = (phy_specific_func_t)elink_xgxs_specific_func
 };
-
 static const struct elink_phy phy_warpcore = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = ELINK_FLAGS_TX_ERROR_CHECK,
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10baseT_Half |
-		      ELINK_SUPPORTED_10baseT_Full |
-		      ELINK_SUPPORTED_100baseT_Half |
-		      ELINK_SUPPORTED_100baseT_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_20000baseKR2_Full |
-		      ELINK_SUPPORTED_20000baseMLD2_Full |
-		      ELINK_SUPPORTED_FIBRE |
-		      ELINK_SUPPORTED_Autoneg |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_UNSPECIFIED,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	/* req_duplex = */ 0,
-	/* rsrv = */ 0,
-	.config_init = elink_warpcore_config_init,
-	.read_status = elink_warpcore_read_status,
-	.link_reset = elink_warpcore_link_reset,
-	.config_loopback = elink_set_warpcore_loopback,
-	.format_fw_ver = NULL,
-	.hw_reset = elink_warpcore_hw_reset,
-	.set_link_led = NULL,
-	.phy_specific_func = NULL
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_DIRECT,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= ELINK_FLAGS_TX_ERROR_CHECK,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10baseT_Half |
+			   ELINK_SUPPORTED_10baseT_Full |
+			   ELINK_SUPPORTED_100baseT_Half |
+			   ELINK_SUPPORTED_100baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_1000baseKX_Full |
+			   ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_10000baseKR_Full |
+			   ELINK_SUPPORTED_20000baseKR2_Full |
+			   ELINK_SUPPORTED_20000baseMLD2_Full |
+			   ELINK_SUPPORTED_FIBRE |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_UNSPECIFIED,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	/* req_duplex = */0,
+	/* rsrv = */0,
+	.config_init	= (config_init_t)elink_warpcore_config_init,
+	.read_status	= (read_status_t)elink_warpcore_read_status,
+	.link_reset	= (link_reset_t)elink_warpcore_link_reset,
+	.config_loopback = (config_loopback_t)elink_set_warpcore_loopback,
+	.format_fw_ver	= (format_fw_ver_t)NULL,
+	.hw_reset	= (hw_reset_t)elink_warpcore_hw_reset,
+	.set_link_led	= (set_link_led_t)NULL,
+	.phy_specific_func = (phy_specific_func_t)NULL
 };
 
+
 static const struct elink_phy phy_7101 = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_SFX7101,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = ELINK_FLAGS_FAN_FAILURE_DET_REQ,
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_TP |
-		      ELINK_SUPPORTED_Autoneg |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_BASE_T,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_7101_config_init,
-	.read_status = elink_7101_read_status,
-	.link_reset = elink_common_ext_link_reset,
-	.config_loopback = elink_7101_config_loopback,
-	.format_fw_ver = elink_7101_format_ver,
-	.hw_reset = elink_7101_hw_reset,
-	.set_link_led = elink_7101_set_link_led,
-	.phy_specific_func = NULL
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_SFX7101,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= ELINK_FLAGS_FAN_FAILURE_DET_REQ,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_TP |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_BASE_T,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_7101_config_init,
+	.read_status	= (read_status_t)elink_7101_read_status,
+	.link_reset	= (link_reset_t)elink_common_ext_link_reset,
+	.config_loopback = (config_loopback_t)elink_7101_config_loopback,
+	.format_fw_ver	= (format_fw_ver_t)elink_7101_format_ver,
+	.hw_reset	= (hw_reset_t)elink_7101_hw_reset,
+	.set_link_led	= (set_link_led_t)elink_7101_set_link_led,
+	.phy_specific_func = (phy_specific_func_t)NULL
 };
-
 static const struct elink_phy phy_8073 = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8073,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = 0,
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_2500baseX_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_FIBRE |
-		      ELINK_SUPPORTED_Autoneg |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_KR,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_8073_config_init,
-	.read_status = elink_8073_read_status,
-	.link_reset = elink_8073_link_reset,
-	.config_loopback = NULL,
-	.format_fw_ver = elink_format_ver,
-	.hw_reset = NULL,
-	.set_link_led = NULL,
-	.phy_specific_func = elink_8073_specific_func
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8073,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= 0,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_2500baseX_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_FIBRE |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_KR,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_8073_config_init,
+	.read_status	= (read_status_t)elink_8073_read_status,
+	.link_reset	= (link_reset_t)elink_8073_link_reset,
+	.config_loopback = (config_loopback_t)NULL,
+	.format_fw_ver	= (format_fw_ver_t)elink_format_ver,
+	.hw_reset	= (hw_reset_t)NULL,
+	.set_link_led	= (set_link_led_t)NULL,
+	.phy_specific_func = (phy_specific_func_t)elink_8073_specific_func
 };
-
 static const struct elink_phy phy_8705 = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8705,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = ELINK_FLAGS_INIT_XGXS_FIRST,
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_FIBRE |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_XFP_FIBER,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_8705_config_init,
-	.read_status = elink_8705_read_status,
-	.link_reset = elink_common_ext_link_reset,
-	.config_loopback = NULL,
-	.format_fw_ver = elink_null_format_ver,
-	.hw_reset = NULL,
-	.set_link_led = NULL,
-	.phy_specific_func = NULL
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8705,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= ELINK_FLAGS_INIT_XGXS_FIRST,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_FIBRE |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_XFP_FIBER,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_8705_config_init,
+	.read_status	= (read_status_t)elink_8705_read_status,
+	.link_reset	= (link_reset_t)elink_common_ext_link_reset,
+	.config_loopback = (config_loopback_t)NULL,
+	.format_fw_ver	= (format_fw_ver_t)elink_null_format_ver,
+	.hw_reset	= (hw_reset_t)NULL,
+	.set_link_led	= (set_link_led_t)NULL,
+	.phy_specific_func = (phy_specific_func_t)NULL
 };
-
 static const struct elink_phy phy_8706 = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8706,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = ELINK_FLAGS_INIT_XGXS_FIRST,
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_FIBRE |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_SFPP_10G_FIBER,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_8706_config_init,
-	.read_status = elink_8706_read_status,
-	.link_reset = elink_common_ext_link_reset,
-	.config_loopback = NULL,
-	.format_fw_ver = elink_format_ver,
-	.hw_reset = NULL,
-	.set_link_led = NULL,
-	.phy_specific_func = NULL
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8706,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= ELINK_FLAGS_INIT_XGXS_FIRST,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_FIBRE |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_SFPP_10G_FIBER,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_8706_config_init,
+	.read_status	= (read_status_t)elink_8706_read_status,
+	.link_reset	= (link_reset_t)elink_common_ext_link_reset,
+	.config_loopback = (config_loopback_t)NULL,
+	.format_fw_ver	= (format_fw_ver_t)elink_format_ver,
+	.hw_reset	= (hw_reset_t)NULL,
+	.set_link_led	= (set_link_led_t)NULL,
+	.phy_specific_func = (phy_specific_func_t)NULL
 };
 
 static const struct elink_phy phy_8726 = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8726,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = (ELINK_FLAGS_INIT_XGXS_FIRST | ELINK_FLAGS_TX_ERROR_CHECK),
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_Autoneg |
-		      ELINK_SUPPORTED_FIBRE |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_NOT_PRESENT,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_8726_config_init,
-	.read_status = elink_8726_read_status,
-	.link_reset = elink_8726_link_reset,
-	.config_loopback = elink_8726_config_loopback,
-	.format_fw_ver = elink_format_ver,
-	.hw_reset = NULL,
-	.set_link_led = NULL,
-	.phy_specific_func = NULL
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8726,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= (ELINK_FLAGS_INIT_XGXS_FIRST |
+			   ELINK_FLAGS_TX_ERROR_CHECK),
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_FIBRE |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_NOT_PRESENT,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_8726_config_init,
+	.read_status	= (read_status_t)elink_8726_read_status,
+	.link_reset	= (link_reset_t)elink_8726_link_reset,
+	.config_loopback = (config_loopback_t)elink_8726_config_loopback,
+	.format_fw_ver	= (format_fw_ver_t)elink_format_ver,
+	.hw_reset	= (hw_reset_t)NULL,
+	.set_link_led	= (set_link_led_t)NULL,
+	.phy_specific_func = (phy_specific_func_t)NULL
 };
 
 static const struct elink_phy phy_8727 = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8727,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = (ELINK_FLAGS_FAN_FAILURE_DET_REQ | ELINK_FLAGS_TX_ERROR_CHECK),
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_FIBRE |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_NOT_PRESENT,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_8727_config_init,
-	.read_status = elink_8727_read_status,
-	.link_reset = elink_8727_link_reset,
-	.config_loopback = NULL,
-	.format_fw_ver = elink_format_ver,
-	.hw_reset = elink_8727_hw_reset,
-	.set_link_led = elink_8727_set_link_led,
-	.phy_specific_func = elink_8727_specific_func
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8727,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= (ELINK_FLAGS_FAN_FAILURE_DET_REQ |
+			   ELINK_FLAGS_TX_ERROR_CHECK),
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_FIBRE |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_NOT_PRESENT,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_8727_config_init,
+	.read_status	= (read_status_t)elink_8727_read_status,
+	.link_reset	= (link_reset_t)elink_8727_link_reset,
+	.config_loopback = (config_loopback_t)NULL,
+	.format_fw_ver	= (format_fw_ver_t)elink_format_ver,
+	.hw_reset	= (hw_reset_t)elink_8727_hw_reset,
+	.set_link_led	= (set_link_led_t)elink_8727_set_link_led,
+	.phy_specific_func = (phy_specific_func_t)elink_8727_specific_func
 };
-
 static const struct elink_phy phy_8481 = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8481,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = ELINK_FLAGS_FAN_FAILURE_DET_REQ |
-	    ELINK_FLAGS_REARM_LATCH_SIGNAL,
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10baseT_Half |
-		      ELINK_SUPPORTED_10baseT_Full |
-		      ELINK_SUPPORTED_100baseT_Half |
-		      ELINK_SUPPORTED_100baseT_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_TP |
-		      ELINK_SUPPORTED_Autoneg |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_BASE_T,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_8481_config_init,
-	.read_status = elink_848xx_read_status,
-	.link_reset = elink_8481_link_reset,
-	.config_loopback = NULL,
-	.format_fw_ver = elink_848xx_format_ver,
-	.hw_reset = elink_8481_hw_reset,
-	.set_link_led = elink_848xx_set_link_led,
-	.phy_specific_func = NULL
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8481,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= ELINK_FLAGS_FAN_FAILURE_DET_REQ |
+			  ELINK_FLAGS_REARM_LATCH_SIGNAL,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10baseT_Half |
+			   ELINK_SUPPORTED_10baseT_Full |
+			   ELINK_SUPPORTED_100baseT_Half |
+			   ELINK_SUPPORTED_100baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_TP |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_BASE_T,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_8481_config_init,
+	.read_status	= (read_status_t)elink_848xx_read_status,
+	.link_reset	= (link_reset_t)elink_8481_link_reset,
+	.config_loopback = (config_loopback_t)NULL,
+	.format_fw_ver	= (format_fw_ver_t)elink_848xx_format_ver,
+	.hw_reset	= (hw_reset_t)elink_8481_hw_reset,
+	.set_link_led	= (set_link_led_t)elink_848xx_set_link_led,
+	.phy_specific_func = (phy_specific_func_t)NULL
 };
 
 static const struct elink_phy phy_84823 = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84823,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = (ELINK_FLAGS_FAN_FAILURE_DET_REQ |
-		  ELINK_FLAGS_REARM_LATCH_SIGNAL | ELINK_FLAGS_TX_ERROR_CHECK),
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10baseT_Half |
-		      ELINK_SUPPORTED_10baseT_Full |
-		      ELINK_SUPPORTED_100baseT_Half |
-		      ELINK_SUPPORTED_100baseT_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_TP |
-		      ELINK_SUPPORTED_Autoneg |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_BASE_T,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_848x3_config_init,
-	.read_status = elink_848xx_read_status,
-	.link_reset = elink_848x3_link_reset,
-	.config_loopback = NULL,
-	.format_fw_ver = elink_848xx_format_ver,
-	.hw_reset = NULL,
-	.set_link_led = elink_848xx_set_link_led,
-	.phy_specific_func = elink_848xx_specific_func
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84823,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= (ELINK_FLAGS_FAN_FAILURE_DET_REQ |
+			   ELINK_FLAGS_REARM_LATCH_SIGNAL |
+			   ELINK_FLAGS_TX_ERROR_CHECK),
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10baseT_Half |
+			   ELINK_SUPPORTED_10baseT_Full |
+			   ELINK_SUPPORTED_100baseT_Half |
+			   ELINK_SUPPORTED_100baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_TP |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_BASE_T,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_848x3_config_init,
+	.read_status	= (read_status_t)elink_848xx_read_status,
+	.link_reset	= (link_reset_t)elink_848x3_link_reset,
+	.config_loopback = (config_loopback_t)NULL,
+	.format_fw_ver	= (format_fw_ver_t)elink_848xx_format_ver,
+	.hw_reset	= (hw_reset_t)NULL,
+	.set_link_led	= (set_link_led_t)elink_848xx_set_link_led,
+	.phy_specific_func = (phy_specific_func_t)elink_848xx_specific_func
 };
 
 static const struct elink_phy phy_84833 = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = (ELINK_FLAGS_FAN_FAILURE_DET_REQ |
-		  ELINK_FLAGS_REARM_LATCH_SIGNAL |
-		  ELINK_FLAGS_TX_ERROR_CHECK | ELINK_FLAGS_TEMPERATURE),
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_100baseT_Half |
-		      ELINK_SUPPORTED_100baseT_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_TP |
-		      ELINK_SUPPORTED_Autoneg |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_BASE_T,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_848x3_config_init,
-	.read_status = elink_848xx_read_status,
-	.link_reset = elink_848x3_link_reset,
-	.config_loopback = NULL,
-	.format_fw_ver = elink_848xx_format_ver,
-	.hw_reset = elink_84833_hw_reset_phy,
-	.set_link_led = elink_848xx_set_link_led,
-	.phy_specific_func = elink_848xx_specific_func
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= (ELINK_FLAGS_FAN_FAILURE_DET_REQ |
+			   ELINK_FLAGS_REARM_LATCH_SIGNAL |
+			   ELINK_FLAGS_TX_ERROR_CHECK |
+			   ELINK_FLAGS_TEMPERATURE),
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_100baseT_Half |
+			   ELINK_SUPPORTED_100baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_TP |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_BASE_T,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_848x3_config_init,
+	.read_status	= (read_status_t)elink_848xx_read_status,
+	.link_reset	= (link_reset_t)elink_848x3_link_reset,
+	.config_loopback = (config_loopback_t)NULL,
+	.format_fw_ver	= (format_fw_ver_t)elink_848xx_format_ver,
+	.hw_reset	= (hw_reset_t)elink_84833_hw_reset_phy,
+	.set_link_led	= (set_link_led_t)elink_848xx_set_link_led,
+	.phy_specific_func = (phy_specific_func_t)elink_848xx_specific_func
 };
 
 static const struct elink_phy phy_84834 = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = ELINK_FLAGS_FAN_FAILURE_DET_REQ |
-	    ELINK_FLAGS_REARM_LATCH_SIGNAL,
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_100baseT_Half |
-		      ELINK_SUPPORTED_100baseT_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_10000baseT_Full |
-		      ELINK_SUPPORTED_TP |
-		      ELINK_SUPPORTED_Autoneg |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_BASE_T,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	.req_duplex = 0,
-	.rsrv = 0,
-	.config_init = elink_848x3_config_init,
-	.read_status = elink_848xx_read_status,
-	.link_reset = elink_848x3_link_reset,
-	.config_loopback = NULL,
-	.format_fw_ver = elink_848xx_format_ver,
-	.hw_reset = elink_84833_hw_reset_phy,
-	.set_link_led = elink_848xx_set_link_led,
-	.phy_specific_func = elink_848xx_specific_func
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= ELINK_FLAGS_FAN_FAILURE_DET_REQ |
+			    ELINK_FLAGS_REARM_LATCH_SIGNAL,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_100baseT_Half |
+			   ELINK_SUPPORTED_100baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_TP |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_BASE_T,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_848x3_config_init,
+	.read_status	= (read_status_t)elink_848xx_read_status,
+	.link_reset	= (link_reset_t)elink_848x3_link_reset,
+	.config_loopback = (config_loopback_t)NULL,
+	.format_fw_ver	= (format_fw_ver_t)elink_848xx_format_ver,
+	.hw_reset	= (hw_reset_t)elink_84833_hw_reset_phy,
+	.set_link_led	= (set_link_led_t)elink_848xx_set_link_led,
+	.phy_specific_func = (phy_specific_func_t)elink_848xx_specific_func
 };
 
-static const struct elink_phy phy_54618se = {
-	.type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X54618SE,
-	.addr = 0xff,
-	.def_md_devad = 0,
-	.flags = ELINK_FLAGS_INIT_XGXS_FIRST,
-	.rx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.tx_preemphasis = {0xffff, 0xffff, 0xffff, 0xffff},
-	.mdio_ctrl = 0,
-	.supported = (ELINK_SUPPORTED_10baseT_Half |
-		      ELINK_SUPPORTED_10baseT_Full |
-		      ELINK_SUPPORTED_100baseT_Half |
-		      ELINK_SUPPORTED_100baseT_Full |
-		      ELINK_SUPPORTED_1000baseT_Full |
-		      ELINK_SUPPORTED_TP |
-		      ELINK_SUPPORTED_Autoneg |
-		      ELINK_SUPPORTED_Pause | ELINK_SUPPORTED_Asym_Pause),
-	.media_type = ELINK_ETH_PHY_BASE_T,
-	.ver_addr = 0,
-	.req_flow_ctrl = 0,
-	.req_line_speed = 0,
-	.speed_cap_mask = 0,
-	/* req_duplex = */ 0,
-	/* rsrv = */ 0,
-	.config_init = elink_54618se_config_init,
-	.read_status = elink_54618se_read_status,
-	.link_reset = elink_54618se_link_reset,
-	.config_loopback = elink_54618se_config_loopback,
-	.format_fw_ver = NULL,
-	.hw_reset = NULL,
-	.set_link_led = elink_5461x_set_link_led,
-	.phy_specific_func = elink_54618se_specific_func
+static const struct elink_phy phy_84858 = {
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84858,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= ELINK_FLAGS_FAN_FAILURE_DET_REQ |
+			    ELINK_FLAGS_REARM_LATCH_SIGNAL,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_100baseT_Half |
+			   ELINK_SUPPORTED_100baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_10000baseT_Full |
+			   ELINK_SUPPORTED_TP |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_BASE_T,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	.req_duplex	= 0,
+	.rsrv		= 0,
+	.config_init	= (config_init_t)elink_848x3_config_init,
+	.read_status	= (read_status_t)elink_848xx_read_status,
+	.link_reset	= (link_reset_t)elink_848x3_link_reset,
+	.config_loopback = (config_loopback_t)NULL,
+	.format_fw_ver	= (format_fw_ver_t)elink_848xx_format_ver,
+	.hw_reset	= (hw_reset_t)elink_84833_hw_reset_phy,
+	.set_link_led	= (set_link_led_t)elink_848xx_set_link_led,
+	.phy_specific_func = (phy_specific_func_t)elink_848xx_specific_func
 };
 
+
+static const struct elink_phy phy_54618se = {
+	.type		= PORT_HW_CFG_XGXS_EXT_PHY2_TYPE_BNX2X54618SE,
+	.addr		= 0xff,
+	.def_md_devad	= 0,
+	.flags		= ELINK_FLAGS_INIT_XGXS_FIRST,
+	.rx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.tx_preemphasis	= {0xffff, 0xffff, 0xffff, 0xffff},
+	.mdio_ctrl	= 0,
+	.supported	= (ELINK_SUPPORTED_10baseT_Half |
+			   ELINK_SUPPORTED_10baseT_Full |
+			   ELINK_SUPPORTED_100baseT_Half |
+			   ELINK_SUPPORTED_100baseT_Full |
+			   ELINK_SUPPORTED_1000baseT_Full |
+			   ELINK_SUPPORTED_TP |
+			   ELINK_SUPPORTED_Autoneg |
+			   ELINK_SUPPORTED_Pause |
+			   ELINK_SUPPORTED_Asym_Pause),
+	.media_type	= ELINK_ETH_PHY_BASE_T,
+	.ver_addr	= 0,
+	.req_flow_ctrl	= 0,
+	.req_line_speed	= 0,
+	.speed_cap_mask	= 0,
+	/* req_duplex = */0,
+	/* rsrv = */0,
+	.config_init	= (config_init_t)elink_54618se_config_init,
+	.read_status	= (read_status_t)elink_54618se_read_status,
+	.link_reset	= (link_reset_t)elink_54618se_link_reset,
+	.config_loopback = (config_loopback_t)elink_54618se_config_loopback,
+	.format_fw_ver	= (format_fw_ver_t)NULL,
+	.hw_reset	= (hw_reset_t)NULL,
+	.set_link_led	= (set_link_led_t)elink_5461x_set_link_led,
+	.phy_specific_func = (phy_specific_func_t)elink_54618se_specific_func
+};
 /*****************************************************************/
 /*                                                               */
 /* Populate the phy according. Main function: elink_populate_phy   */
@@ -11148,9 +13039,9 @@ static const struct elink_phy phy_54618se = {
 /*****************************************************************/
 
 static void elink_populate_preemphasis(struct bnx2x_softc *sc,
-				       uint32_t shmem_base,
-				       struct elink_phy *phy, uint8_t port,
-				       uint8_t phy_index)
+				     uint32_t shmem_base,
+				     struct elink_phy *phy, uint8_t port,
+				     uint8_t phy_index)
 {
 	/* Get the 4 lanes xgxs config rx and tx */
 	uint32_t rx = 0, tx = 0, i;
@@ -11162,23 +13053,19 @@ static void elink_populate_preemphasis(struct bnx2x_softc *sc,
 		if (phy_index == ELINK_INT_PHY || phy_index == ELINK_EXT_PHY1) {
 			rx = REG_RD(sc, shmem_base +
 				    offsetof(struct shmem_region,
-					     dev_info.port_hw_config[port].
-					     xgxs_config_rx[i << 1]));
+			dev_info.port_hw_config[port].xgxs_config_rx[i << 1]));
 
 			tx = REG_RD(sc, shmem_base +
 				    offsetof(struct shmem_region,
-					     dev_info.port_hw_config[port].
-					     xgxs_config_tx[i << 1]));
+			dev_info.port_hw_config[port].xgxs_config_tx[i << 1]));
 		} else {
 			rx = REG_RD(sc, shmem_base +
 				    offsetof(struct shmem_region,
-					     dev_info.port_hw_config[port].
-					     xgxs_config2_rx[i << 1]));
+			dev_info.port_hw_config[port].xgxs_config2_rx[i << 1]));
 
 			tx = REG_RD(sc, shmem_base +
 				    offsetof(struct shmem_region,
-					     dev_info.port_hw_config[port].
-					     xgxs_config2_rx[i << 1]));
+			dev_info.port_hw_config[port].xgxs_config2_rx[i << 1]));
 		}
 
 		phy->rx_preemphasis[i << 1] = ((rx >> 16) & 0xffff);
@@ -11186,65 +13073,62 @@ static void elink_populate_preemphasis(struct bnx2x_softc *sc,
 
 		phy->tx_preemphasis[i << 1] = ((tx >> 16) & 0xffff);
 		phy->tx_preemphasis[(i << 1) + 1] = (tx & 0xffff);
+		ELINK_DEBUG_P2(sc, "phy->rx_preemphasis = %x, phy->tx_preemphasis = %x",
+			phy->rx_preemphasis[i << 1],
+			phy->tx_preemphasis[i << 1]);
 	}
 }
 
 static uint32_t elink_get_ext_phy_config(struct bnx2x_softc *sc,
-					 uint32_t shmem_base, uint8_t phy_index,
-					 uint8_t port)
+				    uint32_t shmem_base,
+				    uint8_t phy_index, uint8_t port)
 {
 	uint32_t ext_phy_config = 0;
 	switch (phy_index) {
 	case ELINK_EXT_PHY1:
 		ext_phy_config = REG_RD(sc, shmem_base +
-					offsetof(struct shmem_region,
-						 dev_info.port_hw_config[port].
-						 external_phy_config));
+					      offsetof(struct shmem_region,
+			dev_info.port_hw_config[port].external_phy_config));
 		break;
 	case ELINK_EXT_PHY2:
 		ext_phy_config = REG_RD(sc, shmem_base +
-					offsetof(struct shmem_region,
-						 dev_info.port_hw_config[port].
-						 external_phy_config2));
+					      offsetof(struct shmem_region,
+			dev_info.port_hw_config[port].external_phy_config2));
 		break;
 	default:
-		PMD_DRV_LOG(DEBUG, "Invalid phy_index %d", phy_index);
+		ELINK_DEBUG_P1(sc, "Invalid phy_index %d", phy_index);
 		return ELINK_STATUS_ERROR;
 	}
 
 	return ext_phy_config;
 }
-
 static elink_status_t elink_populate_int_phy(struct bnx2x_softc *sc,
-					     uint32_t shmem_base, uint8_t port,
-					     struct elink_phy *phy)
+				  uint32_t shmem_base, uint8_t port,
+				  struct elink_phy *phy)
 {
 	uint32_t phy_addr;
-	__rte_unused uint32_t chip_id;
+	uint32_t chip_id;
 	uint32_t switch_cfg = (REG_RD(sc, shmem_base +
-				      offsetof(struct shmem_region,
-					       dev_info.
-					       port_feature_config[port].
-					       link_config)) &
-			       PORT_FEATURE_CONNECTED_SWITCH_MASK);
-	chip_id =
-	    (REG_RD(sc, MISC_REG_CHIP_NUM) << 16) |
-	    ((REG_RD(sc, MISC_REG_CHIP_REV) & 0xf) << 12);
-
-	PMD_DRV_LOG(DEBUG, ":chip_id = 0x%x", chip_id);
+				       offsetof(struct shmem_region,
+			dev_info.port_feature_config[port].link_config)) &
+			  PORT_FEATURE_CONNECTED_SWITCH_MASK);
+	chip_id = (REG_RD(sc, MISC_REG_CHIP_NUM) << 16) |
+		((REG_RD(sc, MISC_REG_CHIP_REV) & 0xf) << 12);
+
+	ELINK_DEBUG_P1(sc, ":chip_id = 0x%x", chip_id);
 	if (USES_WARPCORE(sc)) {
 		uint32_t serdes_net_if;
-		phy_addr = REG_RD(sc, MISC_REG_WC0_CTRL_PHY_ADDR);
+		phy_addr = REG_RD(sc,
+				  MISC_REG_WC0_CTRL_PHY_ADDR);
 		*phy = phy_warpcore;
 		if (REG_RD(sc, MISC_REG_PORT4MODE_EN_OVWR) == 0x3)
 			phy->flags |= ELINK_FLAGS_4_PORT_MODE;
 		else
 			phy->flags &= ~ELINK_FLAGS_4_PORT_MODE;
-		/* Check Dual mode */
+			/* Check Dual mode */
 		serdes_net_if = (REG_RD(sc, shmem_base +
-					offsetof(struct shmem_region,
-						 dev_info.port_hw_config[port].
-						 default_cfg)) &
+					offsetof(struct shmem_region, dev_info.
+					port_hw_config[port].default_cfg)) &
 				 PORT_HW_CFG_NET_SERDES_IF_MASK);
 		/* Set the appropriate supported and flags indications per
 		 * interface type of the chip
@@ -11280,8 +13164,8 @@ static elink_status_t elink_populate_int_phy(struct bnx2x_softc *sc,
 			break;
 		case PORT_HW_CFG_NET_SERDES_IF_KR:
 			phy->media_type = ELINK_ETH_PHY_KR;
-			phy->supported &= (ELINK_SUPPORTED_1000baseT_Full |
-					   ELINK_SUPPORTED_10000baseT_Full |
+			phy->supported &= (ELINK_SUPPORTED_1000baseKX_Full |
+					   ELINK_SUPPORTED_10000baseKR_Full |
 					   ELINK_SUPPORTED_FIBRE |
 					   ELINK_SUPPORTED_Autoneg |
 					   ELINK_SUPPORTED_Pause |
@@ -11299,8 +13183,8 @@ static elink_status_t elink_populate_int_phy(struct bnx2x_softc *sc,
 			phy->media_type = ELINK_ETH_PHY_KR;
 			phy->flags |= ELINK_FLAGS_WC_DUAL_MODE;
 			phy->supported &= (ELINK_SUPPORTED_20000baseKR2_Full |
-					   ELINK_SUPPORTED_10000baseT_Full |
-					   ELINK_SUPPORTED_1000baseT_Full |
+					   ELINK_SUPPORTED_10000baseKR_Full |
+					   ELINK_SUPPORTED_1000baseKX_Full |
 					   ELINK_SUPPORTED_Autoneg |
 					   ELINK_SUPPORTED_FIBRE |
 					   ELINK_SUPPORTED_Pause |
@@ -11308,8 +13192,8 @@ static elink_status_t elink_populate_int_phy(struct bnx2x_softc *sc,
 			phy->flags &= ~ELINK_FLAGS_TX_ERROR_CHECK;
 			break;
 		default:
-			PMD_DRV_LOG(DEBUG, "Unknown WC interface type 0x%x",
-				    serdes_net_if);
+			ELINK_DEBUG_P1(sc, "Unknown WC interface type 0x%x",
+				       serdes_net_if);
 			break;
 		}
 
@@ -11321,6 +13205,8 @@ static elink_status_t elink_populate_int_phy(struct bnx2x_softc *sc,
 			phy->flags |= ELINK_FLAGS_MDC_MDIO_WA;
 		else
 			phy->flags |= ELINK_FLAGS_MDC_MDIO_WA_B0;
+		ELINK_DEBUG_P3(sc, "media_type = %x, flags = %x, supported = %x",
+				phy->media_type, phy->flags, phy->supported);
 	} else {
 		switch (switch_cfg) {
 		case ELINK_SWITCH_CFG_1G:
@@ -11336,32 +13222,32 @@ static elink_status_t elink_populate_int_phy(struct bnx2x_softc *sc,
 			*phy = phy_xgxs;
 			break;
 		default:
-			PMD_DRV_LOG(DEBUG, "Invalid switch_cfg");
+			ELINK_DEBUG_P0(sc, "Invalid switch_cfg");
 			return ELINK_STATUS_ERROR;
 		}
 	}
-	phy->addr = (uint8_t) phy_addr;
+	phy->addr = (uint8_t)phy_addr;
 	phy->mdio_ctrl = elink_get_emac_base(sc,
-					     SHARED_HW_CFG_MDC_MDIO_ACCESS1_BOTH,
-					     port);
+					    SHARED_HW_CFG_MDC_MDIO_ACCESS1_BOTH,
+					    port);
 	if (CHIP_IS_E2(sc))
 		phy->def_md_devad = ELINK_E2_DEFAULT_PHY_DEV_ADDR;
 	else
 		phy->def_md_devad = ELINK_DEFAULT_PHY_DEV_ADDR;
 
-	PMD_DRV_LOG(DEBUG, "Internal phy port=%d, addr=0x%x, mdio_ctl=0x%x",
-		    port, phy->addr, phy->mdio_ctrl);
+	ELINK_DEBUG_P3(sc, "Internal phy port=%d, addr=0x%x, mdio_ctl=0x%x",
+		   port, phy->addr, phy->mdio_ctrl);
 
 	elink_populate_preemphasis(sc, shmem_base, phy, port, ELINK_INT_PHY);
 	return ELINK_STATUS_OK;
 }
 
 static elink_status_t elink_populate_ext_phy(struct bnx2x_softc *sc,
-					     uint8_t phy_index,
-					     uint32_t shmem_base,
-					     uint32_t shmem2_base,
-					     uint8_t port,
-					     struct elink_phy *phy)
+				  uint8_t phy_index,
+				  uint32_t shmem_base,
+				  uint32_t shmem2_base,
+				  uint8_t port,
+				  struct elink_phy *phy)
 {
 	uint32_t ext_phy_config, phy_type, config2;
 	uint32_t mdc_mdio_access = SHARED_HW_CFG_MDC_MDIO_ACCESS1_BOTH;
@@ -11407,10 +13293,13 @@ static elink_status_t elink_populate_ext_phy(struct bnx2x_softc *sc,
 	case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834:
 		*phy = phy_84834;
 		break;
+	case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84858:
+		*phy = phy_84858;
+		break;
 	case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X54616:
-	case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X54618SE:
+	case PORT_HW_CFG_XGXS_EXT_PHY2_TYPE_BNX2X54618SE:
 		*phy = phy_54618se;
-		if (phy_type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X54618SE)
+		if (phy_type == PORT_HW_CFG_XGXS_EXT_PHY2_TYPE_BNX2X54618SE)
 			phy->flags |= ELINK_FLAGS_EEE;
 		break;
 	case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_SFX7101:
@@ -11436,21 +13325,20 @@ static elink_status_t elink_populate_ext_phy(struct bnx2x_softc *sc,
 	 * the address
 	 */
 	config2 = REG_RD(sc, shmem_base + offsetof(struct shmem_region,
-						   dev_info.shared_hw_config.
-						   config2));
+					dev_info.shared_hw_config.config2));
 	if (phy_index == ELINK_EXT_PHY1) {
 		phy->ver_addr = shmem_base + offsetof(struct shmem_region,
-						      port_mb[port].
-						      ext_phy_fw_version);
+				port_mb[port].ext_phy_fw_version);
 
 		/* Check specific mdc mdio settings */
 		if (config2 & SHARED_HW_CFG_MDC_MDIO_ACCESS1_MASK)
 			mdc_mdio_access = config2 &
-			    SHARED_HW_CFG_MDC_MDIO_ACCESS1_MASK;
+			SHARED_HW_CFG_MDC_MDIO_ACCESS1_MASK;
 	} else {
 		uint32_t size = REG_RD(sc, shmem2_base);
 
-		if (size > offsetof(struct shmem2_region, ext_phy_fw_version2)) {
+		if (size >
+		    offsetof(struct shmem2_region, ext_phy_fw_version2)) {
 			phy->ver_addr = shmem2_base +
 			    offsetof(struct shmem2_region,
 				     ext_phy_fw_version2[port]);
@@ -11458,35 +13346,34 @@ static elink_status_t elink_populate_ext_phy(struct bnx2x_softc *sc,
 		/* Check specific mdc mdio settings */
 		if (config2 & SHARED_HW_CFG_MDC_MDIO_ACCESS2_MASK)
 			mdc_mdio_access = (config2 &
-					   SHARED_HW_CFG_MDC_MDIO_ACCESS2_MASK)
-			    >> (SHARED_HW_CFG_MDC_MDIO_ACCESS2_SHIFT -
-				SHARED_HW_CFG_MDC_MDIO_ACCESS1_SHIFT);
+			SHARED_HW_CFG_MDC_MDIO_ACCESS2_MASK) >>
+			(SHARED_HW_CFG_MDC_MDIO_ACCESS2_SHIFT -
+			 SHARED_HW_CFG_MDC_MDIO_ACCESS1_SHIFT);
 	}
 	phy->mdio_ctrl = elink_get_emac_base(sc, mdc_mdio_access, port);
 
-	if (((phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833) ||
-	     (phy->type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834)) &&
-	    (phy->ver_addr)) {
+	if (elink_is_8483x_8485x(phy) && (phy->ver_addr)) {
 		/* Remove 100Mb link supported for BNX2X84833/4 when phy fw
 		 * version lower than or equal to 1.39
 		 */
 		uint32_t raw_ver = REG_RD(sc, phy->ver_addr);
-		if (((raw_ver & 0x7F) <= 39) && (((raw_ver & 0xF80) >> 7) <= 1))
+		if (((raw_ver & 0x7F) <= 39) &&
+		    (((raw_ver & 0xF80) >> 7) <= 1))
 			phy->supported &= ~(ELINK_SUPPORTED_100baseT_Half |
 					    ELINK_SUPPORTED_100baseT_Full);
 	}
 
-	PMD_DRV_LOG(DEBUG, "phy_type 0x%x port %d found in index %d",
-		    phy_type, port, phy_index);
-	PMD_DRV_LOG(DEBUG, "             addr=0x%x, mdio_ctl=0x%x",
-		    phy->addr, phy->mdio_ctrl);
+	ELINK_DEBUG_P3(sc, "phy_type 0x%x port %d found in index %d",
+		   phy_type, port, phy_index);
+	ELINK_DEBUG_P2(sc, "             addr=0x%x, mdio_ctl=0x%x",
+		   phy->addr, phy->mdio_ctrl);
 	return ELINK_STATUS_OK;
 }
 
 static elink_status_t elink_populate_phy(struct bnx2x_softc *sc,
-					 uint8_t phy_index, uint32_t shmem_base,
-					 uint32_t shmem2_base, uint8_t port,
-					 struct elink_phy *phy)
+			      uint8_t phy_index, uint32_t shmem_base,
+			      uint32_t shmem2_base, uint8_t port,
+			      struct elink_phy *phy)
 {
 	elink_status_t status = ELINK_STATUS_OK;
 	phy->type = PORT_HW_CFG_XGXS_EXT_PHY_TYPE_NOT_CONN;
@@ -11498,50 +13385,44 @@ static elink_status_t elink_populate_phy(struct bnx2x_softc *sc,
 }
 
 static void elink_phy_def_cfg(struct elink_params *params,
-			      struct elink_phy *phy, uint8_t phy_index)
+			      struct elink_phy *phy,
+			      uint8_t phy_index)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint32_t link_config;
 	/* Populate the default phy configuration for MF mode */
 	if (phy_index == ELINK_EXT_PHY2) {
 		link_config = REG_RD(sc, params->shmem_base +
-				     offsetof(struct shmem_region,
-					      dev_info.port_feature_config
-					      [params->port].link_config2));
-		phy->speed_cap_mask =
-		    REG_RD(sc,
-			   params->shmem_base + offsetof(struct shmem_region,
-							 dev_info.port_hw_config
-							 [params->port].
-							 speed_capability_mask2));
+				     offsetof(struct shmem_region, dev_info.
+			port_feature_config[params->port].link_config2));
+		phy->speed_cap_mask = REG_RD(sc, params->shmem_base +
+					     offsetof(struct shmem_region,
+						      dev_info.
+			port_hw_config[params->port].speed_capability_mask2));
 	} else {
 		link_config = REG_RD(sc, params->shmem_base +
-				     offsetof(struct shmem_region,
-					      dev_info.port_feature_config
-					      [params->port].link_config));
-		phy->speed_cap_mask =
-		    REG_RD(sc,
-			   params->shmem_base + offsetof(struct shmem_region,
-							 dev_info.port_hw_config
-							 [params->port].
-							 speed_capability_mask));
+				     offsetof(struct shmem_region, dev_info.
+				port_feature_config[params->port].link_config));
+		phy->speed_cap_mask = REG_RD(sc, params->shmem_base +
+					     offsetof(struct shmem_region,
+						      dev_info.
+			port_hw_config[params->port].speed_capability_mask));
 	}
-
-	PMD_DRV_LOG(DEBUG,
-		    "Default config phy idx %x cfg 0x%x speed_cap_mask 0x%x",
-		    phy_index, link_config, phy->speed_cap_mask);
+	ELINK_DEBUG_P3(sc,
+	   "Default config phy idx %x cfg 0x%x speed_cap_mask 0x%x",
+	   phy_index, link_config, phy->speed_cap_mask);
 
 	phy->req_duplex = DUPLEX_FULL;
-	switch (link_config & PORT_FEATURE_LINK_SPEED_MASK) {
+	switch (link_config  & PORT_FEATURE_LINK_SPEED_MASK) {
 	case PORT_FEATURE_LINK_SPEED_10M_HALF:
 		phy->req_duplex = DUPLEX_HALF;
-		/* fall-through */
+		/* fallthrough */
 	case PORT_FEATURE_LINK_SPEED_10M_FULL:
 		phy->req_line_speed = ELINK_SPEED_10;
 		break;
 	case PORT_FEATURE_LINK_SPEED_100M_HALF:
 		phy->req_duplex = DUPLEX_HALF;
-		/* fall-through */
+		/* fallthrough */
 	case PORT_FEATURE_LINK_SPEED_100M_FULL:
 		phy->req_line_speed = ELINK_SPEED_100;
 		break;
@@ -11559,7 +13440,10 @@ static void elink_phy_def_cfg(struct elink_params *params,
 		break;
 	}
 
-	switch (link_config & PORT_FEATURE_FLOW_CONTROL_MASK) {
+	ELINK_DEBUG_P2(sc, "Default config phy idx %x, req_duplex config %x",
+			phy_index, phy->req_duplex);
+
+	switch (link_config  & PORT_FEATURE_FLOW_CONTROL_MASK) {
 	case PORT_FEATURE_FLOW_CONTROL_AUTO:
 		phy->req_flow_ctrl = ELINK_FLOW_CTRL_AUTO;
 		break;
@@ -11576,6 +13460,9 @@ static void elink_phy_def_cfg(struct elink_params *params,
 		phy->req_flow_ctrl = ELINK_FLOW_CTRL_NONE;
 		break;
 	}
+	ELINK_DEBUG_P3(sc, "Requested Duplex = %x, line_speed = %x, flow_ctrl = %x",
+		       phy->req_duplex, phy->req_line_speed,
+		       phy->req_flow_ctrl);
 }
 
 uint32_t elink_phy_selection(struct elink_params *params)
@@ -11584,25 +13471,24 @@ uint32_t elink_phy_selection(struct elink_params *params)
 	uint32_t return_cfg = PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT;
 
 	phy_config_swapped = params->multi_phy_config &
-	    PORT_HW_CFG_PHY_SWAPPED_ENABLED;
+		PORT_HW_CFG_PHY_SWAPPED_ENABLED;
 
-	prio_cfg = params->multi_phy_config & PORT_HW_CFG_PHY_SELECTION_MASK;
+	prio_cfg = params->multi_phy_config &
+			PORT_HW_CFG_PHY_SELECTION_MASK;
 
 	if (phy_config_swapped) {
 		switch (prio_cfg) {
 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
-			return_cfg =
-			    PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY;
+		     return_cfg = PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY;
 			break;
 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
-			return_cfg =
-			    PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY;
+		     return_cfg = PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY;
 			break;
 		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
-			return_cfg = PORT_HW_CFG_PHY_SELECTION_FIRST_PHY;
+		     return_cfg = PORT_HW_CFG_PHY_SELECTION_FIRST_PHY;
 			break;
 		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
-			return_cfg = PORT_HW_CFG_PHY_SELECTION_SECOND_PHY;
+		     return_cfg = PORT_HW_CFG_PHY_SELECTION_SECOND_PHY;
 			break;
 		}
 	} else
@@ -11611,19 +13497,23 @@ uint32_t elink_phy_selection(struct elink_params *params)
 	return return_cfg;
 }
 
-elink_status_t elink_phy_probe(struct elink_params * params)
+elink_status_t elink_phy_probe(struct elink_params *params)
 {
 	uint8_t phy_index, actual_phy_idx;
 	uint32_t phy_config_swapped, sync_offset, media_types;
 	struct bnx2x_softc *sc = params->sc;
 	struct elink_phy *phy;
 	params->num_phys = 0;
-	PMD_DRV_LOG(DEBUG, "Begin phy probe");
-
+	ELINK_DEBUG_P0(sc, "Begin phy probe");
+#ifdef ELINK_INCLUDE_EMUL
+	if (CHIP_REV_IS_EMUL(sc))
+		return ELINK_STATUS_OK;
+#endif
 	phy_config_swapped = params->multi_phy_config &
-	    PORT_HW_CFG_PHY_SWAPPED_ENABLED;
+		PORT_HW_CFG_PHY_SWAPPED_ENABLED;
 
-	for (phy_index = ELINK_INT_PHY; phy_index < ELINK_MAX_PHYS; phy_index++) {
+	for (phy_index = ELINK_INT_PHY; phy_index < ELINK_MAX_PHYS;
+	      phy_index++) {
 		actual_phy_idx = phy_index;
 		if (phy_config_swapped) {
 			if (phy_index == ELINK_EXT_PHY1)
@@ -11631,18 +13521,19 @@ elink_status_t elink_phy_probe(struct elink_params * params)
 			else if (phy_index == ELINK_EXT_PHY2)
 				actual_phy_idx = ELINK_EXT_PHY1;
 		}
-		PMD_DRV_LOG(DEBUG, "phy_config_swapped %x, phy_index %x,"
-			    " actual_phy_idx %x", phy_config_swapped,
-			    phy_index, actual_phy_idx);
+		ELINK_DEBUG_P3(sc, "phy_config_swapped %x, phy_index %x,"
+			       " actual_phy_idx %x", phy_config_swapped,
+			   phy_index, actual_phy_idx);
 		phy = &params->phy[actual_phy_idx];
 		if (elink_populate_phy(sc, phy_index, params->shmem_base,
 				       params->shmem2_base, params->port,
 				       phy) != ELINK_STATUS_OK) {
 			params->num_phys = 0;
-			PMD_DRV_LOG(DEBUG, "phy probe failed in phy index %d",
-				    phy_index);
+			ELINK_DEBUG_P1(sc, "phy probe failed in phy index %d",
+				   phy_index);
 			for (phy_index = ELINK_INT_PHY;
-			     phy_index < ELINK_MAX_PHYS; phy_index++)
+			      phy_index < ELINK_MAX_PHYS;
+			      phy_index++)
 				*phy = phy_null;
 			return ELINK_STATUS_ERROR;
 		}
@@ -11658,8 +13549,8 @@ elink_status_t elink_phy_probe(struct elink_params * params)
 			phy->flags |= ELINK_FLAGS_MDC_MDIO_WA_G;
 
 		sync_offset = params->shmem_base +
-		    offsetof(struct shmem_region,
-			     dev_info.port_hw_config[params->port].media_type);
+			offsetof(struct shmem_region,
+			dev_info.port_hw_config[params->port].media_type);
 		media_types = REG_RD(sc, sync_offset);
 
 		/* Update media type for non-PMF sync only for the first time
@@ -11670,9 +13561,9 @@ elink_status_t elink_phy_probe(struct elink_params * params)
 				    (PORT_HW_CFG_MEDIA_TYPE_PHY1_SHIFT *
 				     actual_phy_idx))) == 0) {
 			media_types |= ((phy->media_type &
-					 PORT_HW_CFG_MEDIA_TYPE_PHY0_MASK) <<
-					(PORT_HW_CFG_MEDIA_TYPE_PHY1_SHIFT *
-					 actual_phy_idx));
+					PORT_HW_CFG_MEDIA_TYPE_PHY0_MASK) <<
+				(PORT_HW_CFG_MEDIA_TYPE_PHY1_SHIFT *
+				 actual_phy_idx));
 		}
 		REG_WR(sc, sync_offset, media_types);
 
@@ -11680,47 +13571,231 @@ elink_status_t elink_phy_probe(struct elink_params * params)
 		params->num_phys++;
 	}
 
-	PMD_DRV_LOG(DEBUG, "End phy probe. #phys found %x", params->num_phys);
+	ELINK_DEBUG_P1(sc, "End phy probe. #phys found %x", params->num_phys);
 	return ELINK_STATUS_OK;
 }
 
-static void elink_init_bmac_loopback(struct elink_params *params,
-				     struct elink_vars *vars)
+#ifdef ELINK_INCLUDE_EMUL
+static elink_status_t elink_init_e3_emul_mac(struct elink_params *params,
+					     struct elink_vars *vars)
+{
+	struct bnx2x_softc *sc = params->sc;
+	vars->line_speed = params->req_line_speed[0];
+	/* In case link speed is auto, set speed the highest as possible */
+	if (params->req_line_speed[0] == ELINK_SPEED_AUTO_NEG) {
+		if (params->feature_config_flags &
+		    ELINK_FEATURE_CONFIG_EMUL_DISABLE_XMAC)
+			vars->line_speed = ELINK_SPEED_2500;
+		else if (elink_is_4_port_mode(sc))
+			vars->line_speed = ELINK_SPEED_10000;
+		else
+			vars->line_speed = ELINK_SPEED_20000;
+	}
+	if (vars->line_speed < ELINK_SPEED_10000) {
+		if ((params->feature_config_flags &
+		     ELINK_FEATURE_CONFIG_EMUL_DISABLE_UMAC)) {
+			ELINK_DEBUG_P1(sc, "Invalid line speed %d while UMAC is"
+				   " disabled!", params->req_line_speed[0]);
+			return ELINK_STATUS_ERROR;
+		}
+		switch (vars->line_speed) {
+		case ELINK_SPEED_10:
+			vars->link_status = ELINK_LINK_10TFD;
+			break;
+		case ELINK_SPEED_100:
+			vars->link_status = ELINK_LINK_100TXFD;
+			break;
+		case ELINK_SPEED_1000:
+			vars->link_status = ELINK_LINK_1000TFD;
+			break;
+		case ELINK_SPEED_2500:
+			vars->link_status = ELINK_LINK_2500TFD;
+			break;
+		default:
+			ELINK_DEBUG_P1(sc, "Invalid line speed %d for UMAC",
+				   vars->line_speed);
+			return ELINK_STATUS_ERROR;
+		}
+		vars->link_status |= LINK_STATUS_LINK_UP;
+
+		if (params->loopback_mode == ELINK_LOOPBACK_UMAC)
+			elink_umac_enable(params, vars, 1);
+		else
+			elink_umac_enable(params, vars, 0);
+	} else {
+		/* Link speed >= 10000 requires XMAC enabled */
+		if (params->feature_config_flags &
+		    ELINK_FEATURE_CONFIG_EMUL_DISABLE_XMAC) {
+			ELINK_DEBUG_P1(sc, "Invalid line speed %d while XMAC is"
+				   " disabled!", params->req_line_speed[0]);
+		return ELINK_STATUS_ERROR;
+	}
+		/* Check link speed */
+		switch (vars->line_speed) {
+		case ELINK_SPEED_10000:
+			vars->link_status = ELINK_LINK_10GTFD;
+			break;
+		case ELINK_SPEED_20000:
+			vars->link_status = ELINK_LINK_20GTFD;
+			break;
+		default:
+			ELINK_DEBUG_P1(sc, "Invalid line speed %d for XMAC",
+				   vars->line_speed);
+			return ELINK_STATUS_ERROR;
+		}
+		vars->link_status |= LINK_STATUS_LINK_UP;
+		if (params->loopback_mode == ELINK_LOOPBACK_XMAC)
+			elink_xmac_enable(params, vars, 1);
+		else
+			elink_xmac_enable(params, vars, 0);
+	}
+		return ELINK_STATUS_OK;
+}
+
+static elink_status_t elink_init_emul(struct elink_params *params,
+			    struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
+	if (CHIP_IS_E3(sc)) {
+		if (elink_init_e3_emul_mac(params, vars) !=
+		    ELINK_STATUS_OK)
+			return ELINK_STATUS_ERROR;
+	} else {
+		if (params->feature_config_flags &
+		    ELINK_FEATURE_CONFIG_EMUL_DISABLE_BMAC) {
+			vars->line_speed = ELINK_SPEED_1000;
+			vars->link_status = (LINK_STATUS_LINK_UP |
+					     ELINK_LINK_1000XFD);
+			if (params->loopback_mode ==
+			    ELINK_LOOPBACK_EMAC)
+				elink_emac_enable(params, vars, 1);
+			else
+				elink_emac_enable(params, vars, 0);
+		} else {
+			vars->line_speed = ELINK_SPEED_10000;
+			vars->link_status = (LINK_STATUS_LINK_UP |
+					     ELINK_LINK_10GTFD);
+			if (params->loopback_mode ==
+			    ELINK_LOOPBACK_BMAC)
+				elink_bmac_enable(params, vars, 1, 1);
+			else
+				elink_bmac_enable(params, vars, 0, 1);
+		}
+	}
 	vars->link_up = 1;
-	vars->line_speed = ELINK_SPEED_10000;
 	vars->duplex = DUPLEX_FULL;
 	vars->flow_ctrl = ELINK_FLOW_CTRL_NONE;
-	vars->mac_type = ELINK_MAC_TYPE_BMAC;
 
-	vars->phy_flags = PHY_XGXS_FLAG;
-
-	elink_xgxs_deassert(params);
+		if (CHIP_IS_E1x(sc))
+			elink_pbf_update(params, vars->flow_ctrl,
+					 vars->line_speed);
+		/* Disable drain */
+		REG_WR(sc, NIG_REG_EGRESS_DRAIN0_MODE + params->port * 4, 0);
 
-	/* Set bmac loopback */
-	elink_bmac_enable(params, vars, 1, 1);
+		/* update shared memory */
+		elink_update_mng(params, vars->link_status);
+	return ELINK_STATUS_OK;
+}
+#endif
+#ifdef ELINK_INCLUDE_FPGA
+static elink_status_t elink_init_fpga(struct elink_params *params,
+			    struct elink_vars *vars)
+{
+	/* Enable on E1.5 FPGA */
+	struct bnx2x_softc *sc = params->sc;
+	vars->duplex = DUPLEX_FULL;
+	vars->flow_ctrl = ELINK_FLOW_CTRL_NONE;
+	if (!(CHIP_IS_E1(sc))) {
+		vars->flow_ctrl = (ELINK_FLOW_CTRL_TX |
+				   ELINK_FLOW_CTRL_RX);
+		vars->link_status |= (LINK_STATUS_TX_FLOW_CONTROL_ENABLED |
+				      LINK_STATUS_RX_FLOW_CONTROL_ENABLED);
+	}
+	if (CHIP_IS_E3(sc)) {
+		vars->line_speed = params->req_line_speed[0];
+		switch (vars->line_speed) {
+		case ELINK_SPEED_AUTO_NEG:
+			vars->line_speed = ELINK_SPEED_2500;
+		case ELINK_SPEED_2500:
+			vars->link_status = ELINK_LINK_2500TFD;
+			break;
+		case ELINK_SPEED_1000:
+			vars->link_status = ELINK_LINK_1000XFD;
+			break;
+		case ELINK_SPEED_100:
+			vars->link_status = ELINK_LINK_100TXFD;
+			break;
+		case ELINK_SPEED_10:
+			vars->link_status = ELINK_LINK_10TFD;
+			break;
+		default:
+			ELINK_DEBUG_P1(sc, "Invalid link speed %d",
+				   params->req_line_speed[0]);
+			return ELINK_STATUS_ERROR;
+		}
+		vars->link_status |= LINK_STATUS_LINK_UP;
+		if (params->loopback_mode == ELINK_LOOPBACK_UMAC)
+			elink_umac_enable(params, vars, 1);
+		else
+			elink_umac_enable(params, vars, 0);
+	} else {
+		vars->line_speed = ELINK_SPEED_10000;
+		vars->link_status = (LINK_STATUS_LINK_UP | ELINK_LINK_10GTFD);
+		if (params->loopback_mode == ELINK_LOOPBACK_EMAC)
+			elink_emac_enable(params, vars, 1);
+		else
+			elink_emac_enable(params, vars, 0);
+	}
+	vars->link_up = 1;
 
+	if (CHIP_IS_E1x(sc))
+		elink_pbf_update(params, vars->flow_ctrl,
+				 vars->line_speed);
+	/* Disable drain */
 	REG_WR(sc, NIG_REG_EGRESS_DRAIN0_MODE + params->port * 4, 0);
+
+	/* Update shared memory */
+	elink_update_mng(params, vars->link_status);
+		return ELINK_STATUS_OK;
+}
+#endif
+static void elink_init_bmac_loopback(struct elink_params *params,
+				     struct elink_vars *vars)
+{
+	struct bnx2x_softc *sc = params->sc;
+		vars->link_up = 1;
+		vars->line_speed = ELINK_SPEED_10000;
+		vars->duplex = DUPLEX_FULL;
+		vars->flow_ctrl = ELINK_FLOW_CTRL_NONE;
+		vars->mac_type = ELINK_MAC_TYPE_BMAC;
+
+		vars->phy_flags = PHY_XGXS_FLAG;
+
+		elink_xgxs_deassert(params);
+
+		/* Set bmac loopback */
+		elink_bmac_enable(params, vars, 1, 1);
+
+		REG_WR(sc, NIG_REG_EGRESS_DRAIN0_MODE + params->port * 4, 0);
 }
 
 static void elink_init_emac_loopback(struct elink_params *params,
 				     struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
-	vars->link_up = 1;
-	vars->line_speed = ELINK_SPEED_1000;
-	vars->duplex = DUPLEX_FULL;
-	vars->flow_ctrl = ELINK_FLOW_CTRL_NONE;
-	vars->mac_type = ELINK_MAC_TYPE_EMAC;
+		vars->link_up = 1;
+		vars->line_speed = ELINK_SPEED_1000;
+		vars->duplex = DUPLEX_FULL;
+		vars->flow_ctrl = ELINK_FLOW_CTRL_NONE;
+		vars->mac_type = ELINK_MAC_TYPE_EMAC;
 
-	vars->phy_flags = PHY_XGXS_FLAG;
+		vars->phy_flags = PHY_XGXS_FLAG;
 
-	elink_xgxs_deassert(params);
-	/* Set bmac loopback */
-	elink_emac_enable(params, vars, 1);
-	elink_emac_program(params, vars);
-	REG_WR(sc, NIG_REG_EGRESS_DRAIN0_MODE + params->port * 4, 0);
+		elink_xgxs_deassert(params);
+		/* Set bmac loopback */
+		elink_emac_enable(params, vars, 1);
+		elink_emac_program(params, vars);
+		REG_WR(sc, NIG_REG_EGRESS_DRAIN0_MODE + params->port * 4, 0);
 }
 
 static void elink_init_xmac_loopback(struct elink_params *params,
@@ -11741,8 +13816,9 @@ static void elink_init_xmac_loopback(struct elink_params *params,
 	 */
 	elink_set_aer_mmd(params, &params->phy[0]);
 	elink_warpcore_reset_lane(sc, &params->phy[0], 0);
-	params->phy[ELINK_INT_PHY].config_loopback(&params->phy[ELINK_INT_PHY],
-						   params);
+	params->phy[ELINK_INT_PHY].config_loopback(
+			&params->phy[ELINK_INT_PHY],
+			params);
 
 	elink_xmac_enable(params, vars, 1);
 	REG_WR(sc, NIG_REG_EGRESS_DRAIN0_MODE + params->port * 4, 0);
@@ -11804,12 +13880,11 @@ static void elink_init_xgxs_loopback(struct elink_params *params,
 		/* Set external phy loopback */
 		uint8_t phy_index;
 		for (phy_index = ELINK_EXT_PHY1;
-		     phy_index < params->num_phys; phy_index++)
+		      phy_index < params->num_phys; phy_index++)
 			if (params->phy[phy_index].config_loopback)
-				params->phy[phy_index].config_loopback(&params->
-								       phy
-								       [phy_index],
-								       params);
+				params->phy[phy_index].config_loopback(
+					&params->phy[phy_index],
+					params);
 	}
 	REG_WR(sc, NIG_REG_EGRESS_DRAIN0_MODE + params->port * 4, 0);
 
@@ -11826,12 +13901,14 @@ void elink_set_rx_filter(struct elink_params *params, uint8_t en)
 		val |= en * 0x20;
 	REG_WR(sc, NIG_REG_LLH0_BRB1_DRV_MASK + params->port * 4, val);
 
-	REG_WR(sc, NIG_REG_LLH0_BRB1_DRV_MASK_MF + params->port * 4, en * 0x3);
+	if (!CHIP_IS_E1(sc)) {
+		REG_WR(sc, NIG_REG_LLH0_BRB1_DRV_MASK_MF + params->port * 4,
+		       en * 0x3);
+	}
 
 	REG_WR(sc, (params->port ? NIG_REG_LLH1_BRB1_NOT_MCP :
 		    NIG_REG_LLH0_BRB1_NOT_MCP), en);
 }
-
 static elink_status_t elink_avoid_link_flap(struct elink_params *params,
 					    struct elink_vars *vars)
 {
@@ -11839,6 +13916,7 @@ static elink_status_t elink_avoid_link_flap(struct elink_params *params,
 	uint32_t dont_clear_stat, lfa_sts;
 	struct bnx2x_softc *sc = params->sc;
 
+	elink_set_mdio_emac_per_phy(sc, params);
 	/* Sync the link parameters */
 	elink_link_status_update(params, vars);
 
@@ -11850,7 +13928,7 @@ static elink_status_t elink_avoid_link_flap(struct elink_params *params,
 	for (phy_idx = ELINK_INT_PHY; phy_idx < params->num_phys; phy_idx++) {
 		struct elink_phy *phy = &params->phy[phy_idx];
 		if (phy->phy_specific_func) {
-			PMD_DRV_LOG(DEBUG, "Calling PHY specific func");
+			ELINK_DEBUG_P0(sc, "Calling PHY specific func");
 			phy->phy_specific_func(phy, params, ELINK_PHY_INIT);
 		}
 		if ((phy->media_type == ELINK_ETH_PHY_SFPP_10G_FIBER) ||
@@ -11859,7 +13937,8 @@ static elink_status_t elink_avoid_link_flap(struct elink_params *params,
 			elink_verify_sfp_module(phy, params);
 	}
 	lfa_sts = REG_RD(sc, params->lfa_base +
-			 offsetof(struct shmem_lfa, lfa_sts));
+			 offsetof(struct shmem_lfa,
+				  lfa_sts));
 
 	dont_clear_stat = lfa_sts & SHMEM_LFA_DONT_CLEAR_STAT;
 
@@ -11970,12 +14049,12 @@ elink_status_t elink_phy_init(struct elink_params *params,
 {
 	int lfa_status;
 	struct bnx2x_softc *sc = params->sc;
-	PMD_DRV_LOG(DEBUG, "Phy Initialization started");
-	PMD_DRV_LOG(DEBUG, "(1) req_speed %d, req_flowctrl %d",
-		    params->req_line_speed[0], params->req_flow_ctrl[0]);
-	PMD_DRV_LOG(DEBUG, "(2) req_speed %d, req_flowctrl %d",
-		    params->req_line_speed[1], params->req_flow_ctrl[1]);
-	PMD_DRV_LOG(DEBUG, "req_adv_flow_ctrl 0x%x", params->req_fc_auto_adv);
+	ELINK_DEBUG_P0(sc, "Phy Initialization started");
+	ELINK_DEBUG_P2(sc, "(1) req_speed %d, req_flowctrl %d",
+		   params->req_line_speed[0], params->req_flow_ctrl[0]);
+	ELINK_DEBUG_P2(sc, "(2) req_speed %d, req_flowctrl %d",
+		   params->req_line_speed[1], params->req_flow_ctrl[1]);
+	ELINK_DEBUG_P1(sc, "req_adv_flow_ctrl 0x%x", params->req_fc_auto_adv);
 	vars->link_status = 0;
 	vars->phy_link_up = 0;
 	vars->link_up = 0;
@@ -11988,15 +14067,33 @@ elink_status_t elink_phy_init(struct elink_params *params,
 	params->link_flags = ELINK_PHY_INITIALIZED;
 	/* Driver opens NIG-BRB filters */
 	elink_set_rx_filter(params, 1);
+	elink_chng_link_count(params, 1);
 	/* Check if link flap can be avoided */
 	lfa_status = elink_check_lfa(params);
 
+	ELINK_DEBUG_P3(sc, " params : port = %x, loopback_mode = %x req_duplex = %x",
+		       params->port, params->loopback_mode,
+		       params->req_duplex[0]);
+	ELINK_DEBUG_P3(sc, " params : switch_cfg = %x, lane_config = %x req_duplex[1] = %x",
+		       params->switch_cfg, params->lane_config,
+		       params->req_duplex[1]);
+	ELINK_DEBUG_P3(sc, " params : chip_id = %x, feature_config_flags = %x, num_phys = %x",
+		       params->chip_id, params->feature_config_flags,
+		       params->num_phys);
+	ELINK_DEBUG_P3(sc, " params : rsrv = %x, eee_mode = %x, hw_led_mode = %x",
+		       params->rsrv, params->eee_mode, params->hw_led_mode);
+	ELINK_DEBUG_P3(sc, " params : multi_phy = %x, req_fc_auto_adv = %x, link_flags = %x",
+		       params->multi_phy_config, params->req_fc_auto_adv,
+		       params->link_flags);
+	ELINK_DEBUG_P2(sc, " params : lfa_base = %x, link_attr = %x",
+		       params->lfa_base, params->link_attr_sync);
 	if (lfa_status == 0) {
-		PMD_DRV_LOG(DEBUG, "Link Flap Avoidance in progress");
+		ELINK_DEBUG_P0(sc, "Link Flap Avoidance in progress");
 		return elink_avoid_link_flap(params, vars);
 	}
 
-	PMD_DRV_LOG(DEBUG, "Cannot avoid link flap lfa_sta=0x%x", lfa_status);
+	ELINK_DEBUG_P1(sc, "Cannot avoid link flap lfa_sta=0x%x",
+		       lfa_status);
 	elink_cannot_avoid_link_flap(params, vars, lfa_status);
 
 	/* Disable attentions */
@@ -12005,20 +14102,34 @@ elink_status_t elink_phy_init(struct elink_params *params,
 			ELINK_NIG_MASK_XGXS0_LINK10G |
 			ELINK_NIG_MASK_SERDES0_LINK_STATUS |
 			ELINK_NIG_MASK_MI_INT));
+#ifdef ELINK_INCLUDE_EMUL
+	if (!(params->feature_config_flags &
+	      ELINK_FEATURE_CONFIG_EMUL_DISABLE_EMAC))
+#endif
 
-	elink_emac_init(params);
+	elink_emac_init(params, vars);
 
 	if (params->feature_config_flags & ELINK_FEATURE_CONFIG_PFC_ENABLED)
 		vars->link_status |= LINK_STATUS_PFC_ENABLED;
 
-	if ((params->num_phys == 0) && !CHIP_REV_IS_SLOW(sc)) {
-		PMD_DRV_LOG(DEBUG, "No phy found for initialization !!");
+	if ((params->num_phys == 0) &&
+	    !CHIP_REV_IS_SLOW(sc)) {
+		ELINK_DEBUG_P0(sc, "No phy found for initialization !!");
 		return ELINK_STATUS_ERROR;
 	}
 	set_phy_vars(params, vars);
 
-	PMD_DRV_LOG(DEBUG, "Num of phys on board: %d", params->num_phys);
-
+	ELINK_DEBUG_P1(sc, "Num of phys on board: %d", params->num_phys);
+#ifdef ELINK_INCLUDE_FPGA
+	if (CHIP_REV_IS_FPGA(sc)) {
+		return elink_init_fpga(params, vars);
+	} else
+#endif
+#ifdef ELINK_INCLUDE_EMUL
+	if (CHIP_REV_IS_EMUL(sc)) {
+		return elink_init_emul(params, vars);
+	} else
+#endif
 	switch (params->loopback_mode) {
 	case ELINK_LOOPBACK_BMAC:
 		elink_init_bmac_loopback(params, vars);
@@ -12054,15 +14165,16 @@ elink_status_t elink_phy_init(struct elink_params *params,
 	return ELINK_STATUS_OK;
 }
 
-static elink_status_t elink_link_reset(struct elink_params *params,
-				       struct elink_vars *vars,
-				       uint8_t reset_ext_phy)
+elink_status_t elink_link_reset(struct elink_params *params,
+		     struct elink_vars *vars,
+		     uint8_t reset_ext_phy)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint8_t phy_index, port = params->port, clear_latch_ind = 0;
-	PMD_DRV_LOG(DEBUG, "Resetting the link of port %d", port);
+	ELINK_DEBUG_P1(sc, "Resetting the link of port %d", port);
 	/* Disable attentions */
 	vars->link_status = 0;
+	elink_chng_link_count(params, 1);
 	elink_update_mng(params, vars->link_status);
 	vars->eee_status &= ~(SHMEM_EEE_LP_ADV_STATUS_MASK |
 			      SHMEM_EEE_ACTIVE_BIT);
@@ -12081,12 +14193,24 @@ static elink_status_t elink_link_reset(struct elink_params *params,
 		REG_WR(sc, NIG_REG_BMAC0_OUT_EN + port * 4, 0);
 		REG_WR(sc, NIG_REG_EGRESS_EMAC0_OUT_EN + port * 4, 0);
 	}
-	if (!CHIP_IS_E3(sc))
-		elink_set_bmac_rx(sc, port, 0);
-	if (CHIP_IS_E3(sc) && !CHIP_REV_IS_FPGA(sc)) {
-		elink_set_xmac_rxtx(params, 0);
-		elink_set_umac_rxtx(params, 0);
-	}
+
+#ifdef ELINK_INCLUDE_EMUL
+	/* Stop BigMac rx */
+	if (!(params->feature_config_flags &
+	      ELINK_FEATURE_CONFIG_EMUL_DISABLE_BMAC))
+#endif
+		if (!CHIP_IS_E3(sc))
+			elink_set_bmac_rx(sc, params->chip_id, port, 0);
+#ifdef ELINK_INCLUDE_EMUL
+	/* Stop XMAC/UMAC rx */
+	if (!(params->feature_config_flags &
+	      ELINK_FEATURE_CONFIG_EMUL_DISABLE_XMAC))
+#endif
+		if (CHIP_IS_E3(sc) &&
+		!CHIP_REV_IS_FPGA(sc)) {
+			elink_set_xmac_rxtx(params, 0);
+			elink_set_umac_rxtx(params, 0);
+		}
 	/* Disable emac */
 	if (!CHIP_IS_E3(sc))
 		REG_WR(sc, NIG_REG_NIG_EMAC0_EN + port * 4, 0);
@@ -12095,20 +14219,19 @@ static elink_status_t elink_link_reset(struct elink_params *params,
 	/* The PHY reset is controlled by GPIO 1
 	 * Hold it as vars low
 	 */
-	/* Clear link led */
+	 /* Clear link led */
 	elink_set_mdio_emac_per_phy(sc, params);
 	elink_set_led(params, vars, ELINK_LED_MODE_OFF, 0);
 
 	if (reset_ext_phy && (!CHIP_REV_IS_SLOW(sc))) {
 		for (phy_index = ELINK_EXT_PHY1; phy_index < params->num_phys;
-		     phy_index++) {
+		      phy_index++) {
 			if (params->phy[phy_index].link_reset) {
 				elink_set_aer_mmd(params,
 						  &params->phy[phy_index]);
-				params->phy[phy_index].link_reset(&params->
-								  phy
-								  [phy_index],
-								  params);
+				params->phy[phy_index].link_reset(
+					&params->phy[phy_index],
+					params);
 			}
 			if (params->phy[phy_index].flags &
 			    ELINK_FLAGS_REARM_LATCH_SIGNAL)
@@ -12122,11 +14245,12 @@ static elink_status_t elink_link_reset(struct elink_params *params,
 		elink_bits_dis(sc, NIG_REG_LATCH_BC_0 + port * 4,
 			       1 << ELINK_NIG_LATCH_BC_ENABLE_MI_INT);
 	}
+#if defined(ELINK_INCLUDE_EMUL) || defined(ELINK_INCLUDE_FPGA)
+	if (!CHIP_REV_IS_SLOW(sc))
+#endif
 	if (params->phy[ELINK_INT_PHY].link_reset)
-		params->phy[ELINK_INT_PHY].link_reset(&params->
-						      phy
-						      [ELINK_INT_PHY],
-						      params);
+		params->phy[ELINK_INT_PHY].link_reset(
+			&params->phy[ELINK_INT_PHY], params);
 
 	/* Disable nig ingress interface */
 	if (!CHIP_IS_E3(sc)) {
@@ -12136,8 +14260,8 @@ static elink_status_t elink_link_reset(struct elink_params *params,
 		REG_WR(sc, NIG_REG_BMAC0_IN_EN + port * 4, 0);
 		REG_WR(sc, NIG_REG_EMAC0_IN_EN + port * 4, 0);
 	} else {
-		uint32_t xmac_base =
-		    (params->port) ? GRCBASE_XMAC1 : GRCBASE_XMAC0;
+		uint32_t xmac_base = (params->port) ? GRCBASE_XMAC1 :
+						      GRCBASE_XMAC0;
 		elink_set_xumac_nig(params, 0, 0);
 		if (REG_RD(sc, MISC_REG_RESET_REG_2) &
 		    MISC_REGISTERS_RESET_REG_2_XMAC)
@@ -12148,9 +14272,8 @@ static elink_status_t elink_link_reset(struct elink_params *params,
 	vars->phy_flags = 0;
 	return ELINK_STATUS_OK;
 }
-
-elink_status_t elink_lfa_reset(struct elink_params * params,
-			       struct elink_vars * vars)
+elink_status_t elink_lfa_reset(struct elink_params *params,
+			       struct elink_vars *vars)
 {
 	struct bnx2x_softc *sc = params->sc;
 	vars->link_up = 0;
@@ -12169,13 +14292,13 @@ elink_status_t elink_lfa_reset(struct elink_params * params,
 	 * are passed.
 	 */
 	if (!CHIP_IS_E3(sc))
-		elink_set_bmac_rx(sc, params->port, 0);
+		elink_set_bmac_rx(sc, params->chip_id, params->port, 0);
 
 	if (CHIP_IS_E3(sc)) {
 		elink_set_xmac_rxtx(params, 0);
 		elink_set_umac_rxtx(params, 0);
 	}
-	/* Wait 10ms for the pipe to clean up */
+	/* Wait 10ms for the pipe to clean up*/
 	DELAY(1000 * 10);
 
 	/* Clean the NIG-BRB using the network filters in a way that will
@@ -12190,7 +14313,7 @@ elink_status_t elink_lfa_reset(struct elink_params * params,
 	 * minimum management protocol down time.
 	 */
 	if (!CHIP_IS_E3(sc))
-		elink_set_bmac_rx(sc, params->port, 1);
+		elink_set_bmac_rx(sc, params->chip_id, params->port, 1);
 
 	if (CHIP_IS_E3(sc)) {
 		elink_set_xmac_rxtx(params, 1);
@@ -12205,10 +14328,10 @@ elink_status_t elink_lfa_reset(struct elink_params * params,
 /*				Common function				    */
 /****************************************************************************/
 static elink_status_t elink_8073_common_init_phy(struct bnx2x_softc *sc,
-						 uint32_t shmem_base_path[],
-						 uint32_t shmem2_base_path[],
-						 uint8_t phy_index,
-						 __rte_unused uint32_t chip_id)
+				      uint32_t shmem_base_path[],
+				      uint32_t shmem2_base_path[],
+				      uint8_t phy_index,
+				      __rte_unused uint32_t chip_id)
 {
 	struct elink_phy phy[PORT_MAX];
 	struct elink_phy *phy_blk[PORT_MAX];
@@ -12216,8 +14339,8 @@ static elink_status_t elink_8073_common_init_phy(struct bnx2x_softc *sc,
 	int8_t port = 0;
 	int8_t port_of_path = 0;
 	uint32_t swap_val, swap_override;
-	swap_val = REG_RD(sc, NIG_REG_PORT_SWAP);
-	swap_override = REG_RD(sc, NIG_REG_STRAP_OVERRIDE);
+	swap_val = REG_RD(sc,  NIG_REG_PORT_SWAP);
+	swap_override = REG_RD(sc,  NIG_REG_STRAP_OVERRIDE);
 	port ^= (swap_val && swap_override);
 	elink_ext_phy_hw_reset(sc, port);
 	/* PART1 - Reset both phys */
@@ -12238,7 +14361,7 @@ static elink_status_t elink_8073_common_init_phy(struct bnx2x_softc *sc,
 		if (elink_populate_phy(sc, phy_index, shmem_base, shmem2_base,
 				       port_of_path, &phy[port]) !=
 		    ELINK_STATUS_OK) {
-			PMD_DRV_LOG(DEBUG, "populate_phy failed");
+			ELINK_DEBUG_P0(sc, "populate_phy failed");
 			return ELINK_STATUS_ERROR;
 		}
 		/* Disable attentions */
@@ -12253,11 +14376,14 @@ static elink_status_t elink_8073_common_init_phy(struct bnx2x_softc *sc,
 		 * to write to access its registers
 		 */
 		elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_2,
-				    MISC_REGISTERS_GPIO_OUTPUT_HIGH, port);
+			       MISC_REGISTERS_GPIO_OUTPUT_HIGH,
+			       port);
 
 		/* Reset the phy */
 		elink_cl45_write(sc, &phy[port],
-				 MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 1 << 15);
+				 MDIO_PMA_DEVAD,
+				 MDIO_PMA_REG_CTRL,
+				 1 << 15);
 	}
 
 	/* Add delay of 150ms after reset */
@@ -12278,8 +14404,8 @@ static elink_status_t elink_8073_common_init_phy(struct bnx2x_softc *sc,
 		else
 			port_of_path = 0;
 
-		PMD_DRV_LOG(DEBUG, "Loading spirom for phy address 0x%x",
-			    phy_blk[port]->addr);
+		ELINK_DEBUG_P1(sc, "Loading spirom for phy address 0x%x",
+			   phy_blk[port]->addr);
 		if (elink_8073_8727_external_rom_boot(sc, phy_blk[port],
 						      port_of_path))
 			return ELINK_STATUS_ERROR;
@@ -12292,7 +14418,8 @@ static elink_status_t elink_8073_common_init_phy(struct bnx2x_softc *sc,
 		/* Phase1 of TX_POWER_DOWN reset */
 		elink_cl45_write(sc, phy_blk[port],
 				 MDIO_PMA_DEVAD,
-				 MDIO_PMA_REG_TX_POWER_DOWN, (val | 1 << 10));
+				 MDIO_PMA_REG_TX_POWER_DOWN,
+				 (val | 1 << 10));
 	}
 
 	/* Toggle Transmitter: Power down and then up with 600ms delay
@@ -12309,9 +14436,9 @@ static elink_status_t elink_8073_common_init_phy(struct bnx2x_softc *sc,
 				MDIO_PMA_REG_TX_POWER_DOWN, &val);
 
 		elink_cl45_write(sc, phy_blk[port],
-				 MDIO_PMA_DEVAD,
-				 MDIO_PMA_REG_TX_POWER_DOWN,
-				 (val & (~(1 << 10))));
+				MDIO_PMA_DEVAD,
+				MDIO_PMA_REG_TX_POWER_DOWN,
+				(val & (~(1 << 10))));
 		DELAY(1000 * 15);
 
 		/* Read modify write the SPI-ROM version select register */
@@ -12324,16 +14451,15 @@ static elink_status_t elink_8073_common_init_phy(struct bnx2x_softc *sc,
 
 		/* set GPIO2 back to LOW */
 		elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_2,
-				    MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
+			       MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
 	}
 	return ELINK_STATUS_OK;
 }
-
 static elink_status_t elink_8726_common_init_phy(struct bnx2x_softc *sc,
-						 uint32_t shmem_base_path[],
-						 uint32_t shmem2_base_path[],
-						 uint8_t phy_index,
-						 __rte_unused uint32_t chip_id)
+				      uint32_t shmem_base_path[],
+				      uint32_t shmem2_base_path[],
+				      uint8_t phy_index,
+				      __rte_unused uint32_t chip_id)
 {
 	uint32_t val;
 	int8_t port;
@@ -12342,8 +14468,8 @@ static elink_status_t elink_8726_common_init_phy(struct bnx2x_softc *sc,
 	/* Enable the module detection interrupt */
 	val = REG_RD(sc, MISC_REG_GPIO_EVENT_EN);
 	val |= ((1 << MISC_REGISTERS_GPIO_3) |
-		(1 <<
-		 (MISC_REGISTERS_GPIO_3 + MISC_REGISTERS_GPIO_PORT_SHIFT)));
+		(1 << (MISC_REGISTERS_GPIO_3 +
+		 MISC_REGISTERS_GPIO_PORT_SHIFT)));
 	REG_WR(sc, MISC_REG_GPIO_EVENT_EN, val);
 
 	elink_ext_phy_hw_reset(sc, 0);
@@ -12361,33 +14487,33 @@ static elink_status_t elink_8726_common_init_phy(struct bnx2x_softc *sc,
 		}
 		/* Extract the ext phy address for the port */
 		if (elink_populate_phy(sc, phy_index, shmem_base, shmem2_base,
-				       port, &phy) != ELINK_STATUS_OK) {
-			PMD_DRV_LOG(DEBUG, "populate phy failed");
+				       port, &phy) !=
+		    ELINK_STATUS_OK) {
+			ELINK_DEBUG_P0(sc, "populate phy failed");
 			return ELINK_STATUS_ERROR;
 		}
 
-		/* Reset phy */
+		/* Reset phy*/
 		elink_cl45_write(sc, &phy,
 				 MDIO_PMA_DEVAD, MDIO_PMA_REG_GEN_CTRL, 0x0001);
 
+
 		/* Set fault module detected LED on */
 		elink_cb_gpio_write(sc, MISC_REGISTERS_GPIO_0,
-				    MISC_REGISTERS_GPIO_HIGH, port);
+			       MISC_REGISTERS_GPIO_HIGH,
+			       port);
 	}
 
 	return ELINK_STATUS_OK;
 }
-
 static void elink_get_ext_phy_reset_gpio(struct bnx2x_softc *sc,
-					 uint32_t shmem_base, uint8_t * io_gpio,
-					 uint8_t * io_port)
+					 uint32_t shmem_base,
+					 uint8_t *io_gpio, uint8_t *io_port)
 {
 
 	uint32_t phy_gpio_reset = REG_RD(sc, shmem_base +
-					 offsetof(struct shmem_region,
-						  dev_info.
-						  port_hw_config[PORT_0].
-						  default_cfg));
+					  offsetof(struct shmem_region,
+				dev_info.port_hw_config[PORT_0].default_cfg));
 	switch (phy_gpio_reset) {
 	case PORT_HW_CFG_EXT_PHY_GPIO_RST_GPIO0_P0:
 		*io_gpio = 0;
@@ -12428,10 +14554,10 @@ static void elink_get_ext_phy_reset_gpio(struct bnx2x_softc *sc,
 }
 
 static elink_status_t elink_8727_common_init_phy(struct bnx2x_softc *sc,
-						 uint32_t shmem_base_path[],
-						 uint32_t shmem2_base_path[],
-						 uint8_t phy_index,
-						 __rte_unused uint32_t chip_id)
+				      uint32_t shmem_base_path[],
+				      uint32_t shmem2_base_path[],
+				      uint8_t phy_index,
+				      __rte_unused uint32_t chip_id)
 {
 	int8_t port, reset_gpio;
 	uint32_t swap_val, swap_override;
@@ -12448,18 +14574,17 @@ static elink_status_t elink_8727_common_init_phy(struct bnx2x_softc *sc,
 	 * Default is GPIO1, PORT1
 	 */
 	elink_get_ext_phy_reset_gpio(sc, shmem_base_path[0],
-				     (uint8_t *) & reset_gpio,
-				     (uint8_t *) & port);
+				     (uint8_t *)&reset_gpio, (uint8_t *)&port);
 
 	/* Calculate the port based on port swap */
 	port ^= (swap_val && swap_override);
 
-	/* Initiate PHY reset */
+	/* Initiate PHY reset*/
 	elink_cb_gpio_write(sc, reset_gpio, MISC_REGISTERS_GPIO_OUTPUT_LOW,
-			    port);
+		       port);
 	DELAY(1000 * 1);
 	elink_cb_gpio_write(sc, reset_gpio, MISC_REGISTERS_GPIO_OUTPUT_HIGH,
-			    port);
+		       port);
 
 	DELAY(1000 * 5);
 
@@ -12481,8 +14606,8 @@ static elink_status_t elink_8727_common_init_phy(struct bnx2x_softc *sc,
 		/* Extract the ext phy address for the port */
 		if (elink_populate_phy(sc, phy_index, shmem_base, shmem2_base,
 				       port_of_path, &phy[port]) !=
-		    ELINK_STATUS_OK) {
-			PMD_DRV_LOG(DEBUG, "populate phy failed");
+				       ELINK_STATUS_OK) {
+			ELINK_DEBUG_P0(sc, "populate phy failed");
 			return ELINK_STATUS_ERROR;
 		}
 		/* disable attentions */
@@ -12493,6 +14618,7 @@ static elink_status_t elink_8727_common_init_phy(struct bnx2x_softc *sc,
 				ELINK_NIG_MASK_SERDES0_LINK_STATUS |
 				ELINK_NIG_MASK_MI_INT));
 
+
 		/* Reset the phy */
 		elink_cl45_write(sc, &phy[port],
 				 MDIO_PMA_DEVAD, MDIO_PMA_REG_CTRL, 1 << 15);
@@ -12513,25 +14639,25 @@ static elink_status_t elink_8727_common_init_phy(struct bnx2x_softc *sc,
 			port_of_path = port;
 		else
 			port_of_path = 0;
-		PMD_DRV_LOG(DEBUG, "Loading spirom for phy address 0x%x",
-			    phy_blk[port]->addr);
+		ELINK_DEBUG_P1(sc, "Loading spirom for phy address 0x%x",
+			   phy_blk[port]->addr);
 		if (elink_8073_8727_external_rom_boot(sc, phy_blk[port],
 						      port_of_path))
 			return ELINK_STATUS_ERROR;
 		/* Disable PHY transmitter output */
 		elink_cl45_write(sc, phy_blk[port],
-				 MDIO_PMA_DEVAD, MDIO_PMA_REG_TX_DISABLE, 1);
+				 MDIO_PMA_DEVAD,
+				 MDIO_PMA_REG_TX_DISABLE, 1);
 
 	}
 	return ELINK_STATUS_OK;
 }
 
 static elink_status_t elink_84833_common_init_phy(struct bnx2x_softc *sc,
-						  uint32_t shmem_base_path[],
-						  __rte_unused uint32_t
-						  shmem2_base_path[],
-						  __rte_unused uint8_t
-						  phy_index, uint32_t chip_id)
+				uint32_t shmem_base_path[],
+				__rte_unused uint32_t shmem2_base_path[],
+				__rte_unused uint8_t phy_index,
+				uint32_t chip_id)
 {
 	uint8_t reset_gpios;
 	reset_gpios = elink_84833_get_reset_gpios(sc, shmem_base_path, chip_id);
@@ -12540,16 +14666,15 @@ static elink_status_t elink_84833_common_init_phy(struct bnx2x_softc *sc,
 	DELAY(10);
 	elink_cb_gpio_mult_write(sc, reset_gpios,
 				 MISC_REGISTERS_GPIO_OUTPUT_HIGH);
-	PMD_DRV_LOG(DEBUG, "84833 reset pulse on pin values 0x%x", reset_gpios);
+	ELINK_DEBUG_P1(sc, "84833 reset pulse on pin values 0x%x",
+		reset_gpios);
 	return ELINK_STATUS_OK;
 }
-
 static elink_status_t elink_ext_phy_common_init(struct bnx2x_softc *sc,
-						uint32_t shmem_base_path[],
-						uint32_t shmem2_base_path[],
-						uint8_t phy_index,
-						uint32_t ext_phy_type,
-						uint32_t chip_id)
+				     uint32_t shmem_base_path[],
+				     uint32_t shmem2_base_path[],
+				     uint8_t phy_index,
+				     uint32_t ext_phy_type, uint32_t chip_id)
 {
 	elink_status_t rc = ELINK_STATUS_OK;
 
@@ -12577,44 +14702,50 @@ static elink_status_t elink_ext_phy_common_init(struct bnx2x_softc *sc,
 		break;
 	case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84833:
 	case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84834:
+	case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X84858:
 		/* GPIO3's are linked, and so both need to be toggled
 		 * to obtain required 2us pulse.
 		 */
 		rc = elink_84833_common_init_phy(sc, shmem_base_path,
-						 shmem2_base_path,
-						 phy_index, chip_id);
+						shmem2_base_path,
+						phy_index, chip_id);
 		break;
 	case PORT_HW_CFG_XGXS_EXT_PHY_TYPE_FAILURE:
 		rc = ELINK_STATUS_ERROR;
 		break;
 	default:
-		PMD_DRV_LOG(DEBUG,
-			    "ext_phy 0x%x common init not required",
-			    ext_phy_type);
+		ELINK_DEBUG_P1(sc,
+			   "ext_phy 0x%x common init not required",
+			   ext_phy_type);
 		break;
 	}
 
 	if (rc != ELINK_STATUS_OK)
-		elink_cb_event_log(sc, ELINK_LOG_ID_PHY_UNINITIALIZED, 0);	// "Warning: PHY was not initialized,"
-	// " Port %d",
+		elink_cb_event_log(sc, ELINK_LOG_ID_PHY_UNINITIALIZED, 0);
+				     /* "Warning: PHY was not initialized,"
+				      * " Port %d",
+				      */
 
 	return rc;
 }
 
-elink_status_t elink_common_init_phy(struct bnx2x_softc * sc,
-				     uint32_t shmem_base_path[],
-				     uint32_t shmem2_base_path[],
-				     uint32_t chip_id,
-				     __rte_unused uint8_t one_port_enabled)
+elink_status_t elink_common_init_phy(struct bnx2x_softc *sc,
+			  uint32_t shmem_base_path[],
+			  uint32_t shmem2_base_path[], uint32_t chip_id,
+			  __rte_unused uint8_t one_port_enabled)
 {
 	elink_status_t rc = ELINK_STATUS_OK;
 	uint32_t phy_ver, val;
 	uint8_t phy_index = 0;
 	uint32_t ext_phy_type, ext_phy_config;
+#if defined(ELINK_INCLUDE_EMUL) || defined(ELINK_INCLUDE_FPGA)
+	if (CHIP_REV_IS_EMUL(sc) || CHIP_REV_IS_FPGA(sc))
+		return ELINK_STATUS_OK;
+#endif
 
-	elink_set_mdio_clk(sc, GRCBASE_EMAC0);
-	elink_set_mdio_clk(sc, GRCBASE_EMAC1);
-	PMD_DRV_LOG(DEBUG, "Begin common phy init");
+	elink_set_mdio_clk(sc, chip_id, GRCBASE_EMAC0);
+	elink_set_mdio_clk(sc, chip_id, GRCBASE_EMAC1);
+	ELINK_DEBUG_P0(sc, "Begin common phy init");
 	if (CHIP_IS_E3(sc)) {
 		/* Enable EPIO */
 		val = REG_RD(sc, MISC_REG_GEN_PURP_HWG);
@@ -12625,14 +14756,14 @@ elink_status_t elink_common_init_phy(struct bnx2x_softc * sc,
 			 offsetof(struct shmem_region,
 				  port_mb[PORT_0].ext_phy_fw_version));
 	if (phy_ver) {
-		PMD_DRV_LOG(DEBUG, "Not doing common init; phy ver is 0x%x",
-			    phy_ver);
+		ELINK_DEBUG_P1(sc, "Not doing common init; phy ver is 0x%x",
+			       phy_ver);
 		return ELINK_STATUS_OK;
 	}
 
 	/* Read the ext_phy_type for arbitrary port(0) */
 	for (phy_index = ELINK_EXT_PHY1; phy_index < ELINK_MAX_PHYS;
-	     phy_index++) {
+	      phy_index++) {
 		ext_phy_config = elink_get_ext_phy_config(sc,
 							  shmem_base_path[0],
 							  phy_index, 0);
@@ -12655,10 +14786,9 @@ static void elink_check_over_curr(struct elink_params *params,
 
 	cfg_pin = (REG_RD(sc, params->shmem_base +
 			  offsetof(struct shmem_region,
-				   dev_info.port_hw_config[port].
-				   e3_cmn_pin_cfg1)) &
+			       dev_info.port_hw_config[port].e3_cmn_pin_cfg1)) &
 		   PORT_HW_CFG_E3_OVER_CURRENT_MASK) >>
-	    PORT_HW_CFG_E3_OVER_CURRENT_SHIFT;
+		PORT_HW_CFG_E3_OVER_CURRENT_SHIFT;
 
 	/* Ignore check if no external input PIN available */
 	if (elink_get_cfg_pin(sc, cfg_pin, &pin_val) != ELINK_STATUS_OK)
@@ -12666,13 +14796,16 @@ static void elink_check_over_curr(struct elink_params *params,
 
 	if (!pin_val) {
 		if ((vars->phy_flags & PHY_OVER_CURRENT_FLAG) == 0) {
-			elink_cb_event_log(sc, ELINK_LOG_ID_OVER_CURRENT, params->port);	//"Error:  Power fault on Port %d has"
-			//  " been detected and the power to "
-			//  "that SFP+ module has been removed"
-			//  " to prevent failure of the card."
-			//  " Please remove the SFP+ module and"
-			//  " restart the system to clear this"
-			//  " error.",
+			elink_cb_event_log(sc, ELINK_LOG_ID_OVER_CURRENT,
+					   params->port);
+					/* "Error:  Power fault on Port %d has"
+					 *  " been detected and the power to "
+					 *  "that SFP+ module has been removed"
+					 *  " to prevent failure of the card."
+					 *  " Please remove the SFP+ module and"
+					 *  " restart the system to clear this"
+					 *  " error.",
+					 */
 			vars->phy_flags |= PHY_OVER_CURRENT_FLAG;
 			elink_warpcore_power_module(params, 0);
 		}
@@ -12682,9 +14815,9 @@ static void elink_check_over_curr(struct elink_params *params,
 
 /* Returns 0 if no change occurred since last check; 1 otherwise. */
 static uint8_t elink_analyze_link_error(struct elink_params *params,
-					struct elink_vars *vars,
-					uint32_t status, uint32_t phy_flag,
-					uint32_t link_flag, uint8_t notify)
+				    struct elink_vars *vars, uint32_t status,
+				    uint32_t phy_flag, uint32_t link_flag,
+				    uint8_t notify)
 {
 	struct bnx2x_softc *sc = params->sc;
 	/* Compare new value with previous value */
@@ -12697,16 +14830,20 @@ static uint8_t elink_analyze_link_error(struct elink_params *params,
 	/* If values differ */
 	switch (phy_flag) {
 	case PHY_HALF_OPEN_CONN_FLAG:
-		PMD_DRV_LOG(DEBUG, "Analyze Remote Fault");
+		ELINK_DEBUG_P0(sc, "Analyze Remote Fault");
 		break;
 	case PHY_SFP_TX_FAULT_FLAG:
-		PMD_DRV_LOG(DEBUG, "Analyze TX Fault");
+		ELINK_DEBUG_P0(sc, "Analyze TX Fault");
 		break;
 	default:
-		PMD_DRV_LOG(DEBUG, "Analyze UNKNOWN");
+		ELINK_DEBUG_P0(sc, "Analyze UNKNOWN");
 	}
-	PMD_DRV_LOG(DEBUG, "Link changed:[%x %x]->%x", vars->link_up,
-		    old_status, status);
+	ELINK_DEBUG_P3(sc, "Link changed:[%x %x]->%x", vars->link_up,
+	   old_status, status);
+
+	/* Do not touch the link in case physical link down */
+	if ((vars->phy_flags & PHY_PHYSICAL_LINK_FLAG) == 0)
+		return 1;
 
 	/* a. Update shmem->link_status accordingly
 	 * b. Update elink_vars->link_up
@@ -12749,17 +14886,18 @@ static uint8_t elink_analyze_link_error(struct elink_params *params,
 }
 
 /******************************************************************************
-* Description:
-*	This function checks for half opened connection change indication.
-*	When such change occurs, it calls the elink_analyze_link_error
-*	to check if Remote Fault is set or cleared. Reception of remote fault
-*	status message in the MAC indicates that the peer's MAC has detected
-*	a fault, for example, due to break in the TX side of fiber.
-*
-******************************************************************************/
-static elink_status_t elink_check_half_open_conn(struct elink_params *params,
-						 struct elink_vars *vars,
-						 uint8_t notify)
+ * Description:
+ *	This function checks for half opened connection change indication.
+ *	When such change occurs, it calls the elink_analyze_link_error
+ *	to check if Remote Fault is set or cleared. Reception of remote fault
+ *	status message in the MAC indicates that the peer's MAC has detected
+ *	a fault, for example, due to break in the TX side of fiber.
+ *
+ ******************************************************************************/
+static
+elink_status_t elink_check_half_open_conn(struct elink_params *params,
+				struct elink_vars *vars,
+				uint8_t notify)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint32_t lss_status = 0;
@@ -12771,7 +14909,7 @@ static elink_status_t elink_check_half_open_conn(struct elink_params *params,
 
 	if (CHIP_IS_E3(sc) &&
 	    (REG_RD(sc, MISC_REG_RESET_REG_2) &
-	     (MISC_REGISTERS_RESET_REG_2_XMAC))) {
+	      (MISC_REGISTERS_RESET_REG_2_XMAC))) {
 		/* Check E3 XMAC */
 		/* Note that link speed cannot be queried here, since it may be
 		 * zero while link is down. In case UMAC is active, LSS will
@@ -12796,7 +14934,7 @@ static elink_status_t elink_check_half_open_conn(struct elink_params *params,
 		uint32_t lss_status_reg;
 		uint32_t wb_data[2];
 		mac_base = params->port ? NIG_REG_INGRESS_BMAC1_MEM :
-		    NIG_REG_INGRESS_BMAC0_MEM;
+			NIG_REG_INGRESS_BMAC0_MEM;
 		/*  Read BIGMAC_REGISTER_RX_LSS_STATUS */
 		if (CHIP_IS_E2(sc))
 			lss_status_reg = BIGMAC2_REGISTER_RX_LSS_STAT;
@@ -12812,7 +14950,6 @@ static elink_status_t elink_check_half_open_conn(struct elink_params *params,
 	}
 	return ELINK_STATUS_OK;
 }
-
 static void elink_sfp_tx_fault_detection(struct elink_phy *phy,
 					 struct elink_params *params,
 					 struct elink_vars *vars)
@@ -12823,15 +14960,12 @@ static void elink_sfp_tx_fault_detection(struct elink_phy *phy,
 
 	/* Get The SFP+ TX_Fault controlling pin ([eg]pio) */
 	cfg_pin = (REG_RD(sc, params->shmem_base + offsetof(struct shmem_region,
-							    dev_info.
-							    port_hw_config
-							    [port].
-							    e3_cmn_pin_cfg)) &
+			  dev_info.port_hw_config[port].e3_cmn_pin_cfg)) &
 		   PORT_HW_CFG_E3_TX_FAULT_MASK) >>
-	    PORT_HW_CFG_E3_TX_FAULT_SHIFT;
+		  PORT_HW_CFG_E3_TX_FAULT_SHIFT;
 
 	if (elink_get_cfg_pin(sc, cfg_pin, &value)) {
-		PMD_DRV_LOG(DEBUG, "Failed to read pin 0x%02x", cfg_pin);
+		ELINK_DEBUG_P1(sc, "Failed to read pin 0x%02x", cfg_pin);
 		return;
 	}
 
@@ -12853,24 +14987,25 @@ static void elink_sfp_tx_fault_detection(struct elink_phy *phy,
 
 		/* If module is unapproved, led should be on regardless */
 		if (!(phy->flags & ELINK_FLAGS_SFP_NOT_APPROVED)) {
-			PMD_DRV_LOG(DEBUG, "Change TX_Fault LED: ->%x",
-				    led_mode);
+			ELINK_DEBUG_P1(sc, "Change TX_Fault LED: ->%x",
+			   led_mode);
 			elink_set_e3_module_fault_led(params, led_mode);
 		}
 	}
 }
-
 static void elink_kr2_recovery(struct elink_params *params,
-			       struct elink_vars *vars, struct elink_phy *phy)
+			       struct elink_vars *vars,
+			       struct elink_phy *phy)
 {
-	PMD_DRV_LOG(DEBUG, "KR2 recovery");
-
+	struct bnx2x_softc *sc = params->sc;
+	ELINK_DEBUG_P0(sc, "KR2 recovery");
 	elink_warpcore_enable_AN_KR2(phy, params, vars);
 	elink_warpcore_restart_AN_KR(phy, params);
 }
 
 static void elink_check_kr2_wa(struct elink_params *params,
-			       struct elink_vars *vars, struct elink_phy *phy)
+			       struct elink_vars *vars,
+			       struct elink_phy *phy)
 {
 	struct bnx2x_softc *sc = params->sc;
 	uint16_t base_page, next_page, not_kr2_device, lane;
@@ -12888,14 +15023,14 @@ static void elink_check_kr2_wa(struct elink_params *params,
 
 	sigdet = elink_warpcore_get_sigdet(phy, params);
 	if (!sigdet) {
-		if (!(vars->link_attr_sync & LINK_ATTR_SYNC_KR2_ENABLE)) {
+		if (!(params->link_attr_sync & LINK_ATTR_SYNC_KR2_ENABLE)) {
 			elink_kr2_recovery(params, vars, phy);
-			PMD_DRV_LOG(DEBUG, "No sigdet");
+			ELINK_DEBUG_P0(sc, "No sigdet");
 		}
 		return;
 	}
 
-	lane = elink_get_warpcore_lane(params);
+	lane = elink_get_warpcore_lane(phy, params);
 	CL22_WR_OVER_CL45(sc, phy, MDIO_REG_BANK_AER_BLOCK,
 			  MDIO_AER_BLOCK_AER_REG, lane);
 	elink_cl45_read(sc, phy, MDIO_AN_DEVAD,
@@ -12906,9 +15041,9 @@ static void elink_check_kr2_wa(struct elink_params *params,
 
 	/* CL73 has not begun yet */
 	if (base_page == 0) {
-		if (!(vars->link_attr_sync & LINK_ATTR_SYNC_KR2_ENABLE)) {
+		if (!(params->link_attr_sync & LINK_ATTR_SYNC_KR2_ENABLE)) {
 			elink_kr2_recovery(params, vars, phy);
-			PMD_DRV_LOG(DEBUG, "No BP");
+			ELINK_DEBUG_P0(sc, "No BP");
 		}
 		return;
 	}
@@ -12922,10 +15057,10 @@ static void elink_check_kr2_wa(struct elink_params *params,
 			    ((next_page & 0xe0) == 0x20))));
 
 	/* In case KR2 is already disabled, check if we need to re-enable it */
-	if (!(vars->link_attr_sync & LINK_ATTR_SYNC_KR2_ENABLE)) {
+	if (!(params->link_attr_sync & LINK_ATTR_SYNC_KR2_ENABLE)) {
 		if (!not_kr2_device) {
-			PMD_DRV_LOG(DEBUG, "BP=0x%x, NP=0x%x", base_page,
-				    next_page);
+			ELINK_DEBUG_P2(sc, "BP=0x%x, NP=0x%x", base_page,
+			   next_page);
 			elink_kr2_recovery(params, vars, phy);
 		}
 		return;
@@ -12933,7 +15068,7 @@ static void elink_check_kr2_wa(struct elink_params *params,
 	/* KR2 is enabled, but not KR2 device */
 	if (not_kr2_device) {
 		/* Disable KR2 on both lanes */
-		PMD_DRV_LOG(DEBUG, "BP=0x%x, NP=0x%x", base_page, next_page);
+		ELINK_DEBUG_P2(sc, "BP=0x%x, NP=0x%x", base_page, next_page);
 		elink_disable_kr2(params, vars, phy);
 		/* Restart AN on leading lane */
 		elink_warpcore_restart_AN_KR(phy, params);
@@ -12949,9 +15084,8 @@ void elink_period_func(struct elink_params *params, struct elink_vars *vars)
 		if (params->phy[phy_idx].flags & ELINK_FLAGS_TX_ERROR_CHECK) {
 			elink_set_aer_mmd(params, &params->phy[phy_idx]);
 			if (elink_check_half_open_conn(params, vars, 1) !=
-			    ELINK_STATUS_OK) {
-				PMD_DRV_LOG(DEBUG, "Fault detection failed");
-			}
+			    ELINK_STATUS_OK)
+				ELINK_DEBUG_P0(sc, "Fault detection failed");
 			break;
 		}
 	}
@@ -12959,22 +15093,24 @@ void elink_period_func(struct elink_params *params, struct elink_vars *vars)
 	if (CHIP_IS_E3(sc)) {
 		struct elink_phy *phy = &params->phy[ELINK_INT_PHY];
 		elink_set_aer_mmd(params, phy);
-		if ((phy->supported & ELINK_SUPPORTED_20000baseKR2_Full) &&
-		    (phy->speed_cap_mask & PORT_HW_CFG_SPEED_CAPABILITY_D0_20G))
+		if (((phy->req_line_speed == ELINK_SPEED_AUTO_NEG) &&
+		     (phy->speed_cap_mask &
+		      PORT_HW_CFG_SPEED_CAPABILITY_D0_20G)) ||
+		    (phy->req_line_speed == ELINK_SPEED_20000))
 			elink_check_kr2_wa(params, vars, phy);
 		elink_check_over_curr(params, vars);
 		if (vars->rx_tx_asic_rst)
 			elink_warpcore_config_runtime(phy, params, vars);
 
 		if ((REG_RD(sc, params->shmem_base +
-			    offsetof(struct shmem_region,
-				     dev_info.port_hw_config[params->port].
-				     default_cfg))
-		     & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
+			    offsetof(struct shmem_region, dev_info.
+				port_hw_config[params->port].default_cfg))
+		    & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
 		    PORT_HW_CFG_NET_SERDES_IF_SFI) {
-			if (elink_is_sfp_module_plugged(params)) {
+			if (elink_is_sfp_module_plugged(phy, params)) {
 				elink_sfp_tx_fault_detection(phy, params, vars);
-			} else if (vars->link_status & LINK_STATUS_SFP_TX_FAULT) {
+			} else if (vars->link_status &
+				LINK_STATUS_SFP_TX_FAULT) {
 				/* Clean trail, interrupt corrects the leds */
 				vars->link_status &= ~LINK_STATUS_SFP_TX_FAULT;
 				vars->phy_flags &= ~PHY_SFP_TX_FAULT_FLAG;
@@ -12986,17 +15122,18 @@ void elink_period_func(struct elink_params *params, struct elink_vars *vars)
 }
 
 uint8_t elink_fan_failure_det_req(struct bnx2x_softc *sc,
-				  uint32_t shmem_base,
-				  uint32_t shmem2_base, uint8_t port)
+			     uint32_t shmem_base,
+			     uint32_t shmem2_base,
+			     uint8_t port)
 {
 	uint8_t phy_index, fan_failure_det_req = 0;
 	struct elink_phy phy;
 	for (phy_index = ELINK_EXT_PHY1; phy_index < ELINK_MAX_PHYS;
-	     phy_index++) {
+	      phy_index++) {
 		if (elink_populate_phy(sc, phy_index, shmem_base, shmem2_base,
 				       port, &phy)
 		    != ELINK_STATUS_OK) {
-			PMD_DRV_LOG(DEBUG, "populate phy failed");
+			ELINK_DEBUG_P0(sc, "populate phy failed");
 			return 0;
 		}
 		fan_failure_det_req |= (phy.flags &
@@ -13016,24 +15153,27 @@ void elink_hw_reset_phy(struct elink_params *params)
 			ELINK_NIG_MASK_SERDES0_LINK_STATUS |
 			ELINK_NIG_MASK_MI_INT));
 
-	for (phy_index = ELINK_INT_PHY; phy_index < ELINK_MAX_PHYS; phy_index++) {
+	for (phy_index = ELINK_INT_PHY; phy_index < ELINK_MAX_PHYS;
+	      phy_index++) {
 		if (params->phy[phy_index].hw_reset) {
-			params->phy[phy_index].hw_reset(&params->phy[phy_index],
-							params);
+			params->phy[phy_index].hw_reset(
+				&params->phy[phy_index],
+				params);
 			params->phy[phy_index] = phy_null;
 		}
 	}
 }
 
 void elink_init_mod_abs_int(struct bnx2x_softc *sc, struct elink_vars *vars,
-			    __rte_unused uint32_t chip_id, uint32_t shmem_base,
-			    uint32_t shmem2_base, uint8_t port)
+			    uint32_t chip_id, uint32_t shmem_base,
+			    uint32_t shmem2_base,
+			    uint8_t port)
 {
 	uint8_t gpio_num = 0xff, gpio_port = 0xff, phy_index;
 	uint32_t val;
 	uint32_t offset, aeu_mask, swap_val, swap_override, sync_offset;
 	if (CHIP_IS_E3(sc)) {
-		if (elink_get_mod_abs_int_cfg(sc,
+		if (elink_get_mod_abs_int_cfg(sc, chip_id,
 					      shmem_base,
 					      port,
 					      &gpio_num,
@@ -13042,11 +15182,11 @@ void elink_init_mod_abs_int(struct bnx2x_softc *sc, struct elink_vars *vars,
 	} else {
 		struct elink_phy phy;
 		for (phy_index = ELINK_EXT_PHY1; phy_index < ELINK_MAX_PHYS;
-		     phy_index++) {
+		      phy_index++) {
 			if (elink_populate_phy(sc, phy_index, shmem_base,
 					       shmem2_base, port, &phy)
 			    != ELINK_STATUS_OK) {
-				PMD_DRV_LOG(DEBUG, "populate phy failed");
+				ELINK_DEBUG_P0(sc, "populate phy failed");
 				return;
 			}
 			if (phy.type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8726) {
@@ -13069,15 +15209,15 @@ void elink_init_mod_abs_int(struct bnx2x_softc *sc, struct elink_vars *vars,
 	gpio_port ^= (swap_val && swap_override);
 
 	vars->aeu_int_mask = AEU_INPUTS_ATTN_BITS_GPIO0_FUNCTION_0 <<
-	    (gpio_num + (gpio_port << 2));
+		(gpio_num + (gpio_port << 2));
 
 	sync_offset = shmem_base +
-	    offsetof(struct shmem_region,
-		     dev_info.port_hw_config[port].aeu_int_mask);
+		offsetof(struct shmem_region,
+			 dev_info.port_hw_config[port].aeu_int_mask);
 	REG_WR(sc, sync_offset, vars->aeu_int_mask);
 
-	PMD_DRV_LOG(DEBUG, "Setting MOD_ABS (GPIO%d_P%d) AEU to 0x%x",
-		    gpio_num, gpio_port, vars->aeu_int_mask);
+	ELINK_DEBUG_P3(sc, "Setting MOD_ABS (GPIO%d_P%d) AEU to 0x%x",
+		       gpio_num, gpio_port, vars->aeu_int_mask);
 
 	if (port == 0)
 		offset = MISC_REG_AEU_ENABLE1_FUNC_0_OUT_0;
diff --git a/drivers/net/bnx2x/elink.h b/drivers/net/bnx2x/elink.h
index 40000c24..c8b08bc3 100644
--- a/drivers/net/bnx2x/elink.h
+++ b/drivers/net/bnx2x/elink.h
@@ -14,7 +14,7 @@
 #ifndef ELINK_H
 #define ELINK_H
 
-#define ELINK_DEBUG
+#include "bnx2x_logs.h"
 
 
 
@@ -29,6 +29,11 @@ struct bnx2x_softc;
 
 extern uint32_t elink_cb_reg_read(struct bnx2x_softc *sc, uint32_t reg_addr);
 extern void elink_cb_reg_write(struct bnx2x_softc *sc, uint32_t reg_addr, uint32_t val);
+/* wb_write - pointer to 2 32 bits vars to be passed to the DMAE*/
+extern void elink_cb_reg_wb_write(struct bnx2x_softc *sc, uint32_t offset,
+				uint32_t *wb_write, uint16_t len);
+extern void elink_cb_reg_wb_read(struct bnx2x_softc *sc, uint32_t offset,
+			       uint32_t *wb_write, uint16_t len);
 
 /* mode - 0( LOW ) /1(HIGH)*/
 extern uint8_t elink_cb_gpio_write(struct bnx2x_softc *sc,
@@ -45,6 +50,9 @@ extern uint8_t elink_cb_gpio_int_write(struct bnx2x_softc *sc,
 
 extern uint32_t elink_cb_fw_command(struct bnx2x_softc *sc, uint32_t command, uint32_t param);
 
+/* Delay */
+extern void elink_cb_udelay(struct bnx2x_softc *sc, uint32_t microsecond);
+
 /* This function is called every 1024 bytes downloading of phy firmware.
 Driver can use it to print to screen indication for download progress */
 extern void elink_cb_download_progress(struct bnx2x_softc *sc, uint32_t cur, uint32_t total);
@@ -69,6 +77,8 @@ typedef enum elink_status {
 extern void elink_cb_event_log(struct bnx2x_softc *sc, const elink_log_id_t log_id, ...);
 extern void elink_cb_load_warpcore_microcode(void);
 
+extern uint8_t elink_cb_path_id(struct bnx2x_softc *sc);
+
 extern void elink_cb_notify_link_changed(struct bnx2x_softc *sc);
 
 #define ELINK_EVENT_LOG_LEVEL_ERROR 	1
@@ -78,6 +88,32 @@ extern void elink_cb_notify_link_changed(struct bnx2x_softc *sc);
 
 #define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0]))
 /* Debug prints */
+#ifdef ELINK_DEBUG
+
+extern void elink_cb_dbg(struct bnx2x_softc *sc,  const char *fmt);
+extern void elink_cb_dbg1(struct bnx2x_softc *sc,  const char *fmt,
+			  uint32_t arg1);
+extern void elink_cb_dbg2(struct bnx2x_softc *sc,  const char *fmt,
+			  uint32_t arg1, uint32_t arg2);
+extern void elink_cb_dbg3(struct bnx2x_softc *sc,  const char *fmt,
+			  uint32_t arg1, uint32_t arg2,
+			  uint32_t arg3);
+
+#define ELINK_DEBUG_P0(sc, fmt)			elink_cb_dbg(sc, fmt)
+#define ELINK_DEBUG_P1(sc, fmt, arg1)		elink_cb_dbg1(sc, fmt, arg1)
+#define ELINK_DEBUG_P2(sc, fmt, arg1, arg2)	\
+	elink_cb_dbg2(sc, fmt, arg1, arg2)
+#define ELINK_DEBUG_P3(sc, fmt, arg1, arg2, arg3) \
+	elink_cb_dbg3(sc, fmt, arg1, arg2, arg3)
+#else
+#define ELINK_DEBUG_P0(sc, fmt)                   PMD_DRV_LOG(DEBUG, sc, fmt)
+#define ELINK_DEBUG_P1(sc, fmt, arg1)             \
+	PMD_DRV_LOG(DEBUG, sc, fmt, arg1)
+#define ELINK_DEBUG_P2(sc, fmt, arg1, arg2)       \
+	PMD_DRV_LOG(DEBUG, sc, fmt, arg1, arg2)
+#define ELINK_DEBUG_P3(sc, fmt, arg1, arg2, arg3) \
+	PMD_DRV_LOG(DEBUG, sc, fmt, arg1, arg2, arg3)
+#endif
 
 /***********************************************************/
 /*                         Defines                         */
@@ -126,9 +162,12 @@ extern void elink_cb_notify_link_changed(struct bnx2x_softc *sc);
 #define ELINK_SFP_EEPROM_DATE_SIZE			6
 #define ELINK_SFP_EEPROM_DIAG_TYPE_ADDR			0x5c
 #define ELINK_SFP_EEPROM_DIAG_TYPE_SIZE			1
-#define ELINK_SFP_EEPROM_DIAG_ADDR_CHANGE_REQ		(1<<2)
+#define ELINK_SFP_EEPROM_DIAG_ADDR_CHANGE_REQ		(1 << 2)
 #define ELINK_SFP_EEPROM_SFF_8472_COMP_ADDR		0x5e
 #define ELINK_SFP_EEPROM_SFF_8472_COMP_SIZE		1
+#define ELINK_SFP_EEPROM_VENDOR_SPECIFIC_ADDR	0x60
+#define ELINK_SFP_EEPROM_VENDOR_SPECIFIC_SIZE	16
+
 
 #define ELINK_SFP_EEPROM_A2_CHECKSUM_RANGE		0x5e
 #define ELINK_SFP_EEPROM_A2_CC_DMI_ADDR			0x5f
@@ -199,7 +238,7 @@ typedef void (*link_reset_t)(struct elink_phy *phy,
 			     struct elink_params *params);
 typedef void (*config_loopback_t)(struct elink_phy *phy,
 				  struct elink_params *params);
-typedef uint8_t (*format_fw_ver_t)(uint32_t raw, uint8_t *str, uint16_t *len);
+typedef elink_status_t (*format_fw_ver_t)(uint32_t raw, uint8_t *str, uint16_t *len);
 typedef void (*hw_reset_t)(struct elink_phy *phy, struct elink_params *params);
 typedef void (*set_link_led_t)(struct elink_phy *phy,
 			       struct elink_params *params, uint8_t mode);
@@ -219,23 +258,23 @@ struct elink_phy {
 	uint8_t def_md_devad;
 	uint16_t flags;
 	/* No Over-Current detection */
-#define ELINK_FLAGS_NOC			(1<<1)
+#define ELINK_FLAGS_NOC			(1 << 1)
 	/* Fan failure detection required */
-#define ELINK_FLAGS_FAN_FAILURE_DET_REQ	(1<<2)
+#define ELINK_FLAGS_FAN_FAILURE_DET_REQ	(1 << 2)
 	/* Initialize first the XGXS and only then the phy itself */
-#define ELINK_FLAGS_INIT_XGXS_FIRST		(1<<3)
-#define ELINK_FLAGS_WC_DUAL_MODE		(1<<4)
-#define ELINK_FLAGS_4_PORT_MODE		(1<<5)
-#define ELINK_FLAGS_REARM_LATCH_SIGNAL		(1<<6)
-#define ELINK_FLAGS_SFP_NOT_APPROVED		(1<<7)
-#define ELINK_FLAGS_MDC_MDIO_WA		(1<<8)
-#define ELINK_FLAGS_DUMMY_READ			(1<<9)
-#define ELINK_FLAGS_MDC_MDIO_WA_B0		(1<<10)
-#define ELINK_FLAGS_SFP_MODULE_PLUGGED_IN_WC	(1<<11)
-#define ELINK_FLAGS_TX_ERROR_CHECK		(1<<12)
-#define ELINK_FLAGS_EEE			(1<<13)
-#define ELINK_FLAGS_TEMPERATURE		(1<<14)
-#define ELINK_FLAGS_MDC_MDIO_WA_G		(1<<15)
+#define ELINK_FLAGS_INIT_XGXS_FIRST		(1 << 3)
+#define ELINK_FLAGS_WC_DUAL_MODE		(1 << 4)
+#define ELINK_FLAGS_4_PORT_MODE		(1 << 5)
+#define ELINK_FLAGS_REARM_LATCH_SIGNAL		(1 << 6)
+#define ELINK_FLAGS_SFP_NOT_APPROVED		(1 << 7)
+#define ELINK_FLAGS_MDC_MDIO_WA		(1 << 8)
+#define ELINK_FLAGS_DUMMY_READ			(1 << 9)
+#define ELINK_FLAGS_MDC_MDIO_WA_B0		(1 << 10)
+#define ELINK_FLAGS_SFP_MODULE_PLUGGED_IN_WC	(1 << 11)
+#define ELINK_FLAGS_TX_ERROR_CHECK		(1 << 12)
+#define ELINK_FLAGS_EEE			(1 << 13)
+#define ELINK_FLAGS_TEMPERATURE		(1 << 14)
+#define ELINK_FLAGS_MDC_MDIO_WA_G		(1 << 15)
 
 	/* preemphasis values for the rx side */
 	uint16_t rx_preemphasis[4];
@@ -247,20 +286,22 @@ struct elink_phy {
 	uint32_t mdio_ctrl;
 
 	uint32_t supported;
-#define ELINK_SUPPORTED_10baseT_Half		(1<<0)
-#define ELINK_SUPPORTED_10baseT_Full		(1<<1)
-#define ELINK_SUPPORTED_100baseT_Half		(1<<2)
-#define ELINK_SUPPORTED_100baseT_Full 		(1<<3)
-#define ELINK_SUPPORTED_1000baseT_Full 	(1<<4)
-#define ELINK_SUPPORTED_2500baseX_Full 	(1<<5)
-#define ELINK_SUPPORTED_10000baseT_Full 	(1<<6)
-#define ELINK_SUPPORTED_TP 			(1<<7)
-#define ELINK_SUPPORTED_FIBRE 			(1<<8)
-#define ELINK_SUPPORTED_Autoneg 		(1<<9)
-#define ELINK_SUPPORTED_Pause 			(1<<10)
-#define ELINK_SUPPORTED_Asym_Pause		(1<<11)
-#define ELINK_SUPPORTED_20000baseMLD2_Full	(1<<21)
-#define ELINK_SUPPORTED_20000baseKR2_Full	(1<<22)
+#define ELINK_SUPPORTED_10baseT_Half		(1 << 0)
+#define ELINK_SUPPORTED_10baseT_Full		(1 << 1)
+#define ELINK_SUPPORTED_100baseT_Half		(1 << 2)
+#define ELINK_SUPPORTED_100baseT_Full		(1 << 3)
+#define ELINK_SUPPORTED_1000baseT_Full		(1 << 4)
+#define ELINK_SUPPORTED_2500baseX_Full		(1 << 5)
+#define ELINK_SUPPORTED_10000baseT_Full		(1 << 6)
+#define ELINK_SUPPORTED_TP			(1 << 7)
+#define ELINK_SUPPORTED_FIBRE			(1 << 8)
+#define ELINK_SUPPORTED_Autoneg			(1 << 9)
+#define ELINK_SUPPORTED_Pause			(1 << 10)
+#define ELINK_SUPPORTED_Asym_Pause		(1 << 11)
+#define ELINK_SUPPORTED_1000baseKX_Full		(1 << 17)
+#define ELINK_SUPPORTED_10000baseKR_Full	(1 << 19)
+#define ELINK_SUPPORTED_20000baseMLD2_Full	(1 << 21)
+#define ELINK_SUPPORTED_20000baseKR2_Full	(1 << 22)
 
 	uint32_t media_type;
 #define	ELINK_ETH_PHY_UNSPECIFIED	0x0
@@ -353,17 +394,22 @@ struct elink_params {
 
 	/* features */
 	uint32_t feature_config_flags;
-#define ELINK_FEATURE_CONFIG_OVERRIDE_PREEMPHASIS_ENABLED	(1<<0)
-#define ELINK_FEATURE_CONFIG_PFC_ENABLED			(1<<1)
-#define ELINK_FEATURE_CONFIG_BC_SUPPORTS_OPT_MDL_VRFY		(1<<2)
-#define ELINK_FEATURE_CONFIG_BC_SUPPORTS_DUAL_PHY_OPT_MDL_VRFY	(1<<3)
-#define ELINK_FEATURE_CONFIG_BC_SUPPORTS_AFEX			(1<<8)
-#define ELINK_FEATURE_CONFIG_AUTOGREEEN_ENABLED		(1<<9)
-#define ELINK_FEATURE_CONFIG_BC_SUPPORTS_SFP_TX_DISABLED	(1<<10)
-#define ELINK_FEATURE_CONFIG_DISABLE_REMOTE_FAULT_DET		(1<<11)
-#define ELINK_FEATURE_CONFIG_IEEE_PHY_TEST			(1<<12)
-#define ELINK_FEATURE_CONFIG_MT_SUPPORT			(1<<13)
-#define ELINK_FEATURE_CONFIG_BOOT_FROM_SAN			(1<<14)
+#define ELINK_FEATURE_CONFIG_OVERRIDE_PREEMPHASIS_ENABLED	(1 << 0)
+#define ELINK_FEATURE_CONFIG_PFC_ENABLED			(1 << 1)
+#define ELINK_FEATURE_CONFIG_BC_SUPPORTS_OPT_MDL_VRFY		(1 << 2)
+#define ELINK_FEATURE_CONFIG_BC_SUPPORTS_DUAL_PHY_OPT_MDL_VRFY	(1 << 3)
+#define ELINK_FEATURE_CONFIG_EMUL_DISABLE_EMAC			(1 << 4)
+#define ELINK_FEATURE_CONFIG_EMUL_DISABLE_BMAC			(1 << 5)
+#define ELINK_FEATURE_CONFIG_EMUL_DISABLE_UMAC			(1 << 6)
+#define ELINK_FEATURE_CONFIG_EMUL_DISABLE_XMAC			(1 << 7)
+#define ELINK_FEATURE_CONFIG_BC_SUPPORTS_AFEX			(1 << 8)
+#define ELINK_FEATURE_CONFIG_AUTOGREEEN_ENABLED		(1 << 9)
+#define ELINK_FEATURE_CONFIG_BC_SUPPORTS_SFP_TX_DISABLED	(1 << 10)
+#define ELINK_FEATURE_CONFIG_DISABLE_REMOTE_FAULT_DET		(1 << 11)
+#define ELINK_FEATURE_CONFIG_IEEE_PHY_TEST			(1 << 12)
+#define ELINK_FEATURE_CONFIG_MT_SUPPORT			(1 << 13)
+#define ELINK_FEATURE_CONFIG_BOOT_FROM_SAN			(1 << 14)
+#define ELINK_FEATURE_CONFIG_DISABLE_PD				(1 << 15)
 
 	/* Will be populated during common init */
 	struct elink_phy phy[ELINK_MAX_PHYS];
@@ -391,10 +437,10 @@ struct elink_params {
 #define ELINK_EEE_MODE_NVRAM_LATENCY_TIME		(0x6000)
 #define ELINK_EEE_MODE_NVRAM_MASK		(0x3)
 #define ELINK_EEE_MODE_TIMER_MASK		(0xfffff)
-#define ELINK_EEE_MODE_OUTPUT_TIME		(1<<28)
-#define ELINK_EEE_MODE_OVERRIDE_NVRAM		(1<<29)
-#define ELINK_EEE_MODE_ENABLE_LPI		(1<<30)
-#define ELINK_EEE_MODE_ADV_LPI			(1<<31)
+#define ELINK_EEE_MODE_OUTPUT_TIME		(1 << 28)
+#define ELINK_EEE_MODE_OVERRIDE_NVRAM		(1 << 29)
+#define ELINK_EEE_MODE_ENABLE_LPI		(1 << 30)
+#define ELINK_EEE_MODE_ADV_LPI			(1 << 31)
 
 	uint16_t hw_led_mode; /* part of the hw_config read from the shmem */
 	uint32_t multi_phy_config;
@@ -404,20 +450,23 @@ struct elink_params {
 	uint16_t req_fc_auto_adv; /* Should be set to TX / BOTH when
 				req_flow_ctrl is set to AUTO */
 	uint16_t link_flags;
-#define ELINK_LINK_FLAGS_INT_DISABLED		(1<<0)
-#define ELINK_PHY_INITIALIZED		(1<<1)
+#define ELINK_LINK_FLAGS_INT_DISABLED		(1 << 0)
+#define ELINK_PHY_INITIALIZED		(1 << 1)
 	uint32_t lfa_base;
+
+	/* The same definitions as the shmem2 parameter */
+	uint32_t link_attr_sync;
 };
 
 /* Output parameters */
 struct elink_vars {
 	uint8_t phy_flags;
-#define PHY_XGXS_FLAG			(1<<0)
-#define PHY_SGMII_FLAG			(1<<1)
-#define PHY_PHYSICAL_LINK_FLAG		(1<<2)
-#define PHY_HALF_OPEN_CONN_FLAG		(1<<3)
-#define PHY_OVER_CURRENT_FLAG		(1<<4)
-#define PHY_SFP_TX_FAULT_FLAG		(1<<5)
+#define PHY_XGXS_FLAG			(1 << 0)
+#define PHY_SGMII_FLAG			(1 << 1)
+#define PHY_PHYSICAL_LINK_FLAG		(1 << 2)
+#define PHY_HALF_OPEN_CONN_FLAG		(1 << 3)
+#define PHY_OVER_CURRENT_FLAG		(1 << 4)
+#define PHY_SFP_TX_FAULT_FLAG		(1 << 5)
 
 	uint8_t mac_type;
 #define ELINK_MAC_TYPE_NONE		0
@@ -448,8 +497,7 @@ struct elink_vars {
 	uint8_t rx_tx_asic_rst;
 	uint8_t turn_to_run_wc_rt;
 	uint16_t rsrv2;
-	/* The same definitions as the shmem2 parameter */
-	uint32_t link_attr_sync;
+
 };
 
 /***********************************************************/
@@ -460,14 +508,32 @@ elink_status_t elink_phy_init(struct elink_params *params, struct elink_vars *va
 /* Reset the link. Should be called when driver or interface goes down
    Before calling phy firmware upgrade, the reset_ext_phy should be set
    to 0 */
+elink_status_t elink_link_reset(struct elink_params *params,
+		     struct elink_vars *vars,
+		     uint8_t reset_ext_phy);
 elink_status_t elink_lfa_reset(struct elink_params *params, struct elink_vars *vars);
 /* elink_link_update should be called upon link interrupt */
 elink_status_t elink_link_update(struct elink_params *params, struct elink_vars *vars);
 
+/* use the following phy functions to read/write from external_phy
+ * In order to use it to read/write internal phy registers, use
+ * ELINK_DEFAULT_PHY_DEV_ADDR as devad, and (_bank + (_addr & 0xf)) as
+ * the register
+ */
+elink_status_t elink_phy_read(struct elink_params *params, uint8_t phy_addr,
+		   uint8_t devad, uint16_t reg, uint16_t *ret_val);
+
+elink_status_t elink_phy_write(struct elink_params *params, uint8_t phy_addr,
+		    uint8_t devad, uint16_t reg, uint16_t val);
+
 /* Reads the link_status from the shmem,
    and update the link vars accordingly */
 void elink_link_status_update(struct elink_params *input,
 			    struct elink_vars *output);
+/* returns string representing the fw_version of the external phy */
+elink_status_t elink_get_ext_phy_fw_version(struct elink_params *params,
+				 uint8_t *version,
+				 uint16_t len);
 
 /* Set/Unset the led
    Basically, the CLC takes care of the led for the link, but in case one needs
@@ -481,12 +547,34 @@ elink_status_t elink_set_led(struct elink_params *params,
 #define ELINK_LED_MODE_FRONT_PANEL_OFF	3
 
 /* elink_handle_module_detect_int should be called upon module detection
-   interrupt */
+ * interrupt
+ */
 void elink_handle_module_detect_int(struct elink_params *params);
 
+/* Get the actual link status. In case it returns ELINK_STATUS_OK, link is up,
+ * otherwise link is down
+ */
+elink_status_t elink_test_link(struct elink_params *params,
+		    struct elink_vars *vars,
+		    uint8_t is_serdes);
+
+
 /* One-time initialization for external phy after power up */
 elink_status_t elink_common_init_phy(struct bnx2x_softc *sc, uint32_t shmem_base_path[],
-			  uint32_t shmem2_base_path[], uint32_t chip_id, uint8_t one_port_enabled);
+			  uint32_t shmem2_base_path[], uint32_t chip_id,
+			  uint8_t one_port_enabled);
+
+/* Reset the external PHY using GPIO */
+void elink_ext_phy_hw_reset(struct bnx2x_softc *sc, uint8_t port);
+
+/* Reset the external of SFX7101 */
+void elink_sfx7101_sp_sw_reset(struct bnx2x_softc *sc, struct elink_phy *phy);
+
+/* Read "byte_cnt" bytes from address "addr" from the SFP+ EEPROM */
+elink_status_t elink_read_sfp_module_eeprom(struct elink_phy *phy,
+				 struct elink_params *params, uint8_t dev_addr,
+				 uint16_t addr, uint16_t byte_cnt,
+				 uint8_t *o_buf);
 
 void elink_hw_reset_phy(struct elink_params *params);
 
@@ -569,12 +657,42 @@ elink_status_t elink_update_pfc(struct elink_params *params,
 		      struct elink_vars *vars,
 		      struct elink_nig_brb_pfc_port_params *pfc_params);
 
+
+/* Used to configure the ETS to disable */
+elink_status_t elink_ets_disabled(struct elink_params *params,
+		       struct elink_vars *vars);
+
+/* Used to configure the ETS to BW limited */
+void elink_ets_bw_limit(const struct elink_params *params,
+			const uint32_t cos0_bw,
+			const uint32_t cos1_bw);
+
+/* Used to configure the ETS to strict */
+elink_status_t elink_ets_strict(const struct elink_params *params,
+				const uint8_t strict_cos);
+
+
+/*  Configure the COS to ETS according to BW and SP settings.*/
+elink_status_t elink_ets_e3b0_config(const struct elink_params *params,
+			 const struct elink_vars *vars,
+			 struct elink_ets_params *ets_params);
+/* Read pfc statistic*/
+void elink_pfc_statistic(struct elink_params *params, struct elink_vars *vars,
+			 uint32_t pfc_frames_sent[2],
+			 uint32_t pfc_frames_received[2]);
 void elink_init_mod_abs_int(struct bnx2x_softc *sc, struct elink_vars *vars,
 			    uint32_t chip_id, uint32_t shmem_base, uint32_t shmem2_base,
 			    uint8_t port);
+/* elink_status_t elink_sfp_module_detection(struct elink_phy *phy,
+ *			       struct elink_params *params);
+ */
 
 void elink_period_func(struct elink_params *params, struct elink_vars *vars);
 
+/*elink_status_t elink_check_half_open_conn(struct elink_params *params,
+ *			            struct elink_vars *vars, uint8_t notify);
+ */
+
 void elink_enable_pmd_tx(struct elink_params *params);
 
 
diff --git a/drivers/net/bnxt/bnxt.h b/drivers/net/bnxt/bnxt.h
index db5c4eb0..f75b0ad3 100644
--- a/drivers/net/bnxt/bnxt.h
+++ b/drivers/net/bnxt/bnxt.h
@@ -20,7 +20,7 @@
 
 #include "bnxt_cpr.h"
 
-#define BNXT_MAX_MTU		9500
+#define BNXT_MAX_MTU		9574
 #define VLAN_TAG_SIZE		4
 #define BNXT_VF_RSV_NUM_RSS_CTX	1
 #define BNXT_VF_RSV_NUM_L2_CTX	4
@@ -33,6 +33,13 @@
 #define BNXT_MAX_RX_RING_DESC	8192
 #define BNXT_DB_SIZE		0x80
 
+/* Chimp Communication Channel */
+#define GRCPF_REG_CHIMP_CHANNEL_OFFSET		0x0
+#define GRCPF_REG_CHIMP_COMM_TRIGGER		0x100
+/* Kong Communication Channel */
+#define GRCPF_REG_KONG_CHANNEL_OFFSET		0xA00
+#define GRCPF_REG_KONG_COMM_TRIGGER		0xB00
+
 #define BNXT_INT_LAT_TMR_MIN			75
 #define BNXT_INT_LAT_TMR_MAX			150
 #define BNXT_NUM_CMPL_AGGR_INT			36
@@ -250,6 +257,11 @@ struct bnxt {
 #define BNXT_FLAG_UPDATE_HASH	(1 << 5)
 #define BNXT_FLAG_PTP_SUPPORTED	(1 << 6)
 #define BNXT_FLAG_MULTI_HOST    (1 << 7)
+#define BNXT_FLAG_EXT_RX_PORT_STATS	(1 << 8)
+#define BNXT_FLAG_EXT_TX_PORT_STATS	(1 << 9)
+#define BNXT_FLAG_KONG_MB_EN	(1 << 10)
+#define BNXT_FLAG_TRUSTED_VF_EN	(1 << 11)
+#define BNXT_FLAG_DFLT_VNIC_SET	(1 << 12)
 #define BNXT_FLAG_NEW_RM	(1 << 30)
 #define BNXT_FLAG_INIT_DONE	(1 << 31)
 #define BNXT_PF(bp)		(!((bp)->flags & BNXT_FLAG_VF))
@@ -257,6 +269,9 @@ struct bnxt {
 #define BNXT_NPAR(bp)		((bp)->port_partition_type)
 #define BNXT_MH(bp)             ((bp)->flags & BNXT_FLAG_MULTI_HOST)
 #define BNXT_SINGLE_PF(bp)      (BNXT_PF(bp) && !BNXT_NPAR(bp) && !BNXT_MH(bp))
+#define BNXT_USE_CHIMP_MB	0 //For non-CFA commands, everything uses Chimp.
+#define BNXT_USE_KONG(bp)	((bp)->flags & BNXT_FLAG_KONG_MB_EN)
+#define BNXT_VF_IS_TRUSTED(bp)	((bp)->flags & BNXT_FLAG_TRUSTED_VF_EN)
 
 	unsigned int		rx_nr_rings;
 	unsigned int		rx_cp_nr_rings;
@@ -264,6 +279,9 @@ struct bnxt {
 	const void		*rx_mem_zone;
 	struct rx_port_stats    *hw_rx_port_stats;
 	rte_iova_t		hw_rx_port_stats_map;
+	struct rx_port_stats_ext    *hw_rx_port_stats_ext;
+	rte_iova_t		hw_rx_port_stats_ext_map;
+	uint16_t		fw_rx_port_stats_ext_size;
 
 	unsigned int		tx_nr_rings;
 	unsigned int		tx_cp_nr_rings;
@@ -271,6 +289,9 @@ struct bnxt {
 	const void		*tx_mem_zone;
 	struct tx_port_stats    *hw_tx_port_stats;
 	rte_iova_t		hw_tx_port_stats_map;
+	struct tx_port_stats_ext    *hw_tx_port_stats_ext;
+	rte_iova_t		hw_tx_port_stats_ext_map;
+	uint16_t		fw_tx_port_stats_ext_size;
 
 	/* Default completion ring */
 	struct bnxt_cp_ring_info	*def_cp_ring;
@@ -285,16 +306,13 @@ struct bnxt {
 	struct bnxt_filter_info	*filter_info;
 	STAILQ_HEAD(, bnxt_filter_info)	free_filter_list;
 
-	/* VNIC pointer for flow filter (VMDq) pools */
-#define MAX_FF_POOLS	256
-	STAILQ_HEAD(, bnxt_vnic_info)	ff_pool[MAX_FF_POOLS];
-
 	struct bnxt_irq         *irq_tbl;
 
 #define MAX_NUM_MAC_ADDR	32
 	uint8_t			mac_addr[ETHER_ADDR_LEN];
 
 	uint16_t			hwrm_cmd_seq;
+	uint16_t			kong_cmd_seq;
 	void				*hwrm_cmd_resp_addr;
 	rte_iova_t			hwrm_cmd_resp_dma_addr;
 	void				*hwrm_short_cmd_req_addr;
diff --git a/drivers/net/bnxt/bnxt_cpr.c b/drivers/net/bnxt/bnxt_cpr.c
index ff20b6fd..0fd6e51e 100644
--- a/drivers/net/bnxt/bnxt_cpr.c
+++ b/drivers/net/bnxt/bnxt_cpr.c
@@ -35,6 +35,7 @@ void bnxt_handle_async_event(struct bnxt *bp,
 		break;
 	case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
 		PMD_DRV_LOG(INFO, "Async event: VF config changed\n");
+		bnxt_hwrm_func_qcfg(bp);
 		break;
 	case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED:
 		PMD_DRV_LOG(INFO, "Port conn async event\n");
diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c
index cc7e4391..801c6ffa 100644
--- a/drivers/net/bnxt/bnxt_ethdev.c
+++ b/drivers/net/bnxt/bnxt_ethdev.c
@@ -149,7 +149,6 @@ static const struct rte_pci_id bnxt_pci_id_map[] = {
 				     DEV_RX_OFFLOAD_TCP_CKSUM | \
 				     DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | \
 				     DEV_RX_OFFLOAD_JUMBO_FRAME | \
-				     DEV_RX_OFFLOAD_CRC_STRIP | \
 				     DEV_RX_OFFLOAD_KEEP_CRC | \
 				     DEV_RX_OFFLOAD_TCP_LRO)
 
@@ -203,7 +202,9 @@ static int bnxt_init_chip(struct bnxt *bp)
 	struct bnxt_rx_queue *rxq;
 	struct rte_eth_link new;
 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(bp->eth_dev);
+	struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
+	uint64_t rx_offloads = dev_conf->rxmode.offloads;
 	uint32_t intr_vector = 0;
 	uint32_t queue_id, base = BNXT_MISC_VEC_ID;
 	uint32_t vec = BNXT_MISC_VEC_ID;
@@ -263,6 +264,9 @@ static int bnxt_init_chip(struct bnxt *bp)
 		}
 		memset(vnic->fw_grp_ids, -1, size);
 
+		PMD_DRV_LOG(DEBUG, "vnic[%d] = %p vnic->fw_grp_ids = %p\n",
+			    i, vnic, vnic->fw_grp_ids);
+
 		rc = bnxt_hwrm_vnic_alloc(bp, vnic);
 		if (rc) {
 			PMD_DRV_LOG(ERR, "HWRM vnic %d alloc failure rc: %x\n",
@@ -281,6 +285,16 @@ static int bnxt_init_chip(struct bnxt *bp)
 			}
 		}
 
+		/*
+		 * Firmware sets pf pair in default vnic cfg. If the VLAN strip
+		 * setting is not available at this time, it will not be
+		 * configured correctly in the CFA.
+		 */
+		if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
+			vnic->vlan_strip = true;
+		else
+			vnic->vlan_strip = false;
+
 		rc = bnxt_hwrm_vnic_cfg(bp, vnic);
 		if (rc) {
 			PMD_DRV_LOG(ERR, "HWRM vnic %d cfg failure rc: %x\n",
@@ -299,6 +313,10 @@ static int bnxt_init_chip(struct bnxt *bp)
 		for (j = 0; j < bp->rx_nr_rings; j++) {
 			rxq = bp->eth_dev->data->rx_queues[j];
 
+			PMD_DRV_LOG(DEBUG,
+				    "rxq[%d]->vnic=%p vnic->fw_grp_ids=%p\n",
+				    j, rxq->vnic, rxq->vnic->fw_grp_ids);
+
 			if (rxq->rx_deferred_start)
 				rxq->vnic->fw_grp_ids[j] = INVALID_HW_RING_ID;
 		}
@@ -445,7 +463,7 @@ static void bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev,
 	/* Fast path specifics */
 	dev_info->min_rx_bufsize = 1;
 	dev_info->max_rx_pktlen = BNXT_MAX_MTU + ETHER_HDR_LEN + ETHER_CRC_LEN
-				  + VLAN_TAG_SIZE;
+				  + VLAN_TAG_SIZE * 2;
 
 	dev_info->rx_offload_capa = BNXT_DEV_RX_OFFLOAD_SUPPORT;
 	if (bp->flags & BNXT_FLAG_PTP_SUPPORTED)
@@ -694,7 +712,6 @@ static void bnxt_dev_close_op(struct rte_eth_dev *eth_dev)
 	if (bp->dev_stopped == 0)
 		bnxt_dev_stop_op(eth_dev);
 
-	bnxt_free_mem(bp);
 	if (eth_dev->data->mac_addrs != NULL) {
 		rte_free(eth_dev->data->mac_addrs);
 		eth_dev->data->mac_addrs = NULL;
@@ -714,34 +731,30 @@ static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev,
 	uint64_t pool_mask = eth_dev->data->mac_pool_sel[index];
 	struct bnxt_vnic_info *vnic;
 	struct bnxt_filter_info *filter, *temp_filter;
-	uint32_t pool = RTE_MIN(MAX_FF_POOLS, ETH_64_POOLS);
 	uint32_t i;
 
 	/*
 	 * Loop through all VNICs from the specified filter flow pools to
 	 * remove the corresponding MAC addr filter
 	 */
-	for (i = 0; i < pool; i++) {
+	for (i = 0; i < bp->nr_vnics; i++) {
 		if (!(pool_mask & (1ULL << i)))
 			continue;
 
-		STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) {
-			filter = STAILQ_FIRST(&vnic->filter);
-			while (filter) {
-				temp_filter = STAILQ_NEXT(filter, next);
-				if (filter->mac_index == index) {
-					STAILQ_REMOVE(&vnic->filter, filter,
-						      bnxt_filter_info, next);
-					bnxt_hwrm_clear_l2_filter(bp, filter);
-					filter->mac_index = INVALID_MAC_INDEX;
-					memset(&filter->l2_addr, 0,
-					       ETHER_ADDR_LEN);
-					STAILQ_INSERT_TAIL(
-							&bp->free_filter_list,
-							filter, next);
-				}
-				filter = temp_filter;
+		vnic = &bp->vnic_info[i];
+		filter = STAILQ_FIRST(&vnic->filter);
+		while (filter) {
+			temp_filter = STAILQ_NEXT(filter, next);
+			if (filter->mac_index == index) {
+				STAILQ_REMOVE(&vnic->filter, filter,
+						bnxt_filter_info, next);
+				bnxt_hwrm_clear_l2_filter(bp, filter);
+				filter->mac_index = INVALID_MAC_INDEX;
+				memset(&filter->l2_addr, 0, ETHER_ADDR_LEN);
+				STAILQ_INSERT_TAIL(&bp->free_filter_list,
+						   filter, next);
 			}
+			filter = temp_filter;
 		}
 	}
 }
@@ -751,10 +764,10 @@ static int bnxt_mac_addr_add_op(struct rte_eth_dev *eth_dev,
 				uint32_t index, uint32_t pool)
 {
 	struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private;
-	struct bnxt_vnic_info *vnic = STAILQ_FIRST(&bp->ff_pool[pool]);
+	struct bnxt_vnic_info *vnic = &bp->vnic_info[pool];
 	struct bnxt_filter_info *filter;
 
-	if (BNXT_VF(bp)) {
+	if (BNXT_VF(bp) & !BNXT_VF_IS_TRUSTED(bp)) {
 		PMD_DRV_LOG(ERR, "Cannot add MAC address to a VF interface\n");
 		return -ENOTSUP;
 	}
@@ -898,12 +911,10 @@ static int bnxt_reta_update_op(struct rte_eth_dev *eth_dev,
 		return -EINVAL;
 	}
 	/* Update the RSS VNIC(s) */
-	for (i = 0; i < MAX_FF_POOLS; i++) {
-		STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) {
-			memcpy(vnic->rss_table, reta_conf, reta_size);
-
-			bnxt_hwrm_vnic_rss_cfg(bp, vnic);
-		}
+	for (i = 0; i < bp->max_vnics; i++) {
+		vnic = &bp->vnic_info[i];
+		memcpy(vnic->rss_table, reta_conf, reta_size);
+		bnxt_hwrm_vnic_rss_cfg(bp, vnic);
 	}
 	return 0;
 }
@@ -947,7 +958,7 @@ static int bnxt_rss_hash_update_op(struct rte_eth_dev *eth_dev,
 	struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
 	struct bnxt_vnic_info *vnic;
 	uint16_t hash_type = 0;
-	int i;
+	unsigned int i;
 
 	/*
 	 * If RSS enablement were different than dev_configure,
@@ -978,21 +989,20 @@ static int bnxt_rss_hash_update_op(struct rte_eth_dev *eth_dev,
 		hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6;
 
 	/* Update the RSS VNIC(s) */
-	for (i = 0; i < MAX_FF_POOLS; i++) {
-		STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) {
-			vnic->hash_type = hash_type;
-
-			/*
-			 * Use the supplied key if the key length is
-			 * acceptable and the rss_key is not NULL
-			 */
-			if (rss_conf->rss_key &&
-			    rss_conf->rss_key_len <= HW_HASH_KEY_SIZE)
-				memcpy(vnic->rss_hash_key, rss_conf->rss_key,
-				       rss_conf->rss_key_len);
-
-			bnxt_hwrm_vnic_rss_cfg(bp, vnic);
-		}
+	for (i = 0; i < bp->nr_vnics; i++) {
+		vnic = &bp->vnic_info[i];
+		vnic->hash_type = hash_type;
+
+		/*
+		 * Use the supplied key if the key length is
+		 * acceptable and the rss_key is not NULL
+		 */
+		if (rss_conf->rss_key &&
+		    rss_conf->rss_key_len <= HW_HASH_KEY_SIZE)
+			memcpy(vnic->rss_hash_key, rss_conf->rss_key,
+			       rss_conf->rss_key_len);
+
+		bnxt_hwrm_vnic_rss_cfg(bp, vnic);
 	}
 	return 0;
 }
@@ -1269,53 +1279,51 @@ static int bnxt_del_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
 		 * else
 		 *      VLAN filter doesn't exist, just skip and continue
 		 */
-		STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) {
-			filter = STAILQ_FIRST(&vnic->filter);
-			while (filter) {
-				temp_filter = STAILQ_NEXT(filter, next);
-
-				if (filter->enables & chk &&
-				    filter->l2_ovlan == vlan_id) {
-					/* Must delete the filter */
-					STAILQ_REMOVE(&vnic->filter, filter,
-						      bnxt_filter_info, next);
-					bnxt_hwrm_clear_l2_filter(bp, filter);
-					STAILQ_INSERT_TAIL(
-							&bp->free_filter_list,
-							filter, next);
-
-					/*
-					 * Need to examine to see if the MAC
-					 * filter already existed or not before
-					 * allocating a new one
-					 */
-
-					new_filter = bnxt_alloc_filter(bp);
-					if (!new_filter) {
-						PMD_DRV_LOG(ERR,
+		vnic = &bp->vnic_info[i];
+		filter = STAILQ_FIRST(&vnic->filter);
+		while (filter) {
+			temp_filter = STAILQ_NEXT(filter, next);
+
+			if (filter->enables & chk &&
+			    filter->l2_ovlan == vlan_id) {
+				/* Must delete the filter */
+				STAILQ_REMOVE(&vnic->filter, filter,
+					      bnxt_filter_info, next);
+				bnxt_hwrm_clear_l2_filter(bp, filter);
+				STAILQ_INSERT_TAIL(&bp->free_filter_list,
+						   filter, next);
+
+				/*
+				 * Need to examine to see if the MAC
+				 * filter already existed or not before
+				 * allocating a new one
+				 */
+
+				new_filter = bnxt_alloc_filter(bp);
+				if (!new_filter) {
+					PMD_DRV_LOG(ERR,
 							"MAC/VLAN filter alloc failed\n");
-						rc = -ENOMEM;
-						goto exit;
-					}
-					STAILQ_INSERT_TAIL(&vnic->filter,
-							   new_filter, next);
-					/* Inherit MAC from previous filter */
-					new_filter->mac_index =
-							filter->mac_index;
-					memcpy(new_filter->l2_addr,
-					       filter->l2_addr, ETHER_ADDR_LEN);
-					/* MAC only filter */
-					rc = bnxt_hwrm_set_l2_filter(bp,
-							vnic->fw_vnic_id,
-							new_filter);
-					if (rc)
-						goto exit;
-					PMD_DRV_LOG(INFO,
-						"Del Vlan filter for %d\n",
-						vlan_id);
+					rc = -ENOMEM;
+					goto exit;
 				}
-				filter = temp_filter;
+				STAILQ_INSERT_TAIL(&vnic->filter,
+						new_filter, next);
+				/* Inherit MAC from previous filter */
+				new_filter->mac_index =
+					filter->mac_index;
+				memcpy(new_filter->l2_addr, filter->l2_addr,
+				       ETHER_ADDR_LEN);
+				/* MAC only filter */
+				rc = bnxt_hwrm_set_l2_filter(bp,
+							     vnic->fw_vnic_id,
+							     new_filter);
+				if (rc)
+					goto exit;
+				PMD_DRV_LOG(INFO,
+					    "Del Vlan filter for %d\n",
+					    vlan_id);
 			}
+			filter = temp_filter;
 		}
 	}
 exit:
@@ -1345,51 +1353,48 @@ static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
 		 *   Remove the old MAC only filter
 		 *    Add a new MAC+VLAN filter
 		 */
-		STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) {
-			filter = STAILQ_FIRST(&vnic->filter);
-			while (filter) {
-				temp_filter = STAILQ_NEXT(filter, next);
-
-				if (filter->enables & chk) {
-					if (filter->l2_ovlan == vlan_id)
-						goto cont;
-				} else {
-					/* Must delete the MAC filter */
-					STAILQ_REMOVE(&vnic->filter, filter,
-						      bnxt_filter_info, next);
-					bnxt_hwrm_clear_l2_filter(bp, filter);
-					filter->l2_ovlan = 0;
-					STAILQ_INSERT_TAIL(
-							&bp->free_filter_list,
-							filter, next);
-				}
-				new_filter = bnxt_alloc_filter(bp);
-				if (!new_filter) {
-					PMD_DRV_LOG(ERR,
+		vnic = &bp->vnic_info[i];
+		filter = STAILQ_FIRST(&vnic->filter);
+		while (filter) {
+			temp_filter = STAILQ_NEXT(filter, next);
+
+			if (filter->enables & chk) {
+				if (filter->l2_ivlan == vlan_id)
+					goto cont;
+			} else {
+				/* Must delete the MAC filter */
+				STAILQ_REMOVE(&vnic->filter, filter,
+						bnxt_filter_info, next);
+				bnxt_hwrm_clear_l2_filter(bp, filter);
+				filter->l2_ovlan = 0;
+				STAILQ_INSERT_TAIL(&bp->free_filter_list,
+						   filter, next);
+			}
+			new_filter = bnxt_alloc_filter(bp);
+			if (!new_filter) {
+				PMD_DRV_LOG(ERR,
 						"MAC/VLAN filter alloc failed\n");
-					rc = -ENOMEM;
-					goto exit;
-				}
-				STAILQ_INSERT_TAIL(&vnic->filter, new_filter,
-						   next);
-				/* Inherit MAC from the previous filter */
-				new_filter->mac_index = filter->mac_index;
-				memcpy(new_filter->l2_addr, filter->l2_addr,
-				       ETHER_ADDR_LEN);
-				/* MAC + VLAN ID filter */
-				new_filter->l2_ivlan = vlan_id;
-				new_filter->l2_ivlan_mask = 0xF000;
-				new_filter->enables |= en;
-				rc = bnxt_hwrm_set_l2_filter(bp,
-							     vnic->fw_vnic_id,
-							     new_filter);
-				if (rc)
-					goto exit;
-				PMD_DRV_LOG(INFO,
-					"Added Vlan filter for %d\n", vlan_id);
-cont:
-				filter = temp_filter;
+				rc = -ENOMEM;
+				goto exit;
 			}
+			STAILQ_INSERT_TAIL(&vnic->filter, new_filter, next);
+			/* Inherit MAC from the previous filter */
+			new_filter->mac_index = filter->mac_index;
+			memcpy(new_filter->l2_addr, filter->l2_addr,
+			       ETHER_ADDR_LEN);
+			/* MAC + VLAN ID filter */
+			new_filter->l2_ivlan = vlan_id;
+			new_filter->l2_ivlan_mask = 0xF000;
+			new_filter->enables |= en;
+			rc = bnxt_hwrm_set_l2_filter(bp,
+					vnic->fw_vnic_id,
+					new_filter);
+			if (rc)
+				goto exit;
+			PMD_DRV_LOG(INFO,
+				    "Added Vlan filter for %d\n", vlan_id);
+cont:
+			filter = temp_filter;
 		}
 	}
 exit:
@@ -1397,7 +1402,7 @@ exit:
 }
 
 static int bnxt_vlan_filter_set_op(struct rte_eth_dev *eth_dev,
-				   uint16_t vlan_id, int on)
+		uint16_t vlan_id, int on)
 {
 	struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private;
 
@@ -1454,7 +1459,7 @@ bnxt_set_default_mac_addr_op(struct rte_eth_dev *dev, struct ether_addr *addr)
 	struct bnxt_filter_info *filter;
 	int rc;
 
-	if (BNXT_VF(bp))
+	if (BNXT_VF(bp) && !BNXT_VF_IS_TRUSTED(bp))
 		return -EPERM;
 
 	memcpy(bp->mac_addr, addr, sizeof(bp->mac_addr));
@@ -1571,21 +1576,17 @@ static int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu)
 {
 	struct bnxt *bp = eth_dev->data->dev_private;
 	struct rte_eth_dev_info dev_info;
-	uint32_t max_dev_mtu;
 	uint32_t rc = 0;
 	uint32_t i;
 
 	bnxt_dev_info_get_op(eth_dev, &dev_info);
-	max_dev_mtu = dev_info.max_rx_pktlen -
-		      ETHER_HDR_LEN - ETHER_CRC_LEN - VLAN_TAG_SIZE * 2;
 
-	if (new_mtu < ETHER_MIN_MTU || new_mtu > max_dev_mtu) {
+	if (new_mtu < ETHER_MIN_MTU || new_mtu > BNXT_MAX_MTU) {
 		PMD_DRV_LOG(ERR, "MTU requested must be within (%d, %d)\n",
-			ETHER_MIN_MTU, max_dev_mtu);
+			ETHER_MIN_MTU, BNXT_MAX_MTU);
 		return -EINVAL;
 	}
 
-
 	if (new_mtu > ETHER_MTU) {
 		bp->flags |= BNXT_FLAG_JUMBO;
 		bp->eth_dev->data->dev_conf.rxmode.offloads |=
@@ -1805,8 +1806,8 @@ bnxt_match_and_validate_ether_filter(struct bnxt *bp,
 		goto exit;
 	}
 
-	vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
-	vnic = STAILQ_FIRST(&bp->ff_pool[efilter->queue]);
+	vnic0 = &bp->vnic_info[0];
+	vnic = &bp->vnic_info[efilter->queue];
 	if (vnic == NULL) {
 		PMD_DRV_LOG(ERR, "Invalid queue %d\n", efilter->queue);
 		*ret = -EINVAL;
@@ -1864,8 +1865,8 @@ bnxt_ethertype_filter(struct rte_eth_dev *dev,
 		return -EINVAL;
 	}
 
-	vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
-	vnic = STAILQ_FIRST(&bp->ff_pool[efilter->queue]);
+	vnic0 = &bp->vnic_info[0];
+	vnic = &bp->vnic_info[efilter->queue];
 
 	switch (filter_op) {
 	case RTE_ETH_FILTER_ADD:
@@ -2081,8 +2082,8 @@ bnxt_cfg_ntuple_filter(struct bnxt *bp,
 	if (ret < 0)
 		goto free_filter;
 
-	vnic = STAILQ_FIRST(&bp->ff_pool[nfilter->queue]);
-	vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
+	vnic = &bp->vnic_info[nfilter->queue];
+	vnic0 = &bp->vnic_info[0];
 	filter1 = STAILQ_FIRST(&vnic0->filter);
 	if (filter1 == NULL) {
 		ret = -1;
@@ -2375,8 +2376,8 @@ bnxt_parse_fdir_filter(struct bnxt *bp,
 		return -EINVAL;
 	}
 
-	vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
-	vnic = STAILQ_FIRST(&bp->ff_pool[fdir->action.rx_queue]);
+	vnic0 = &bp->vnic_info[0];
+	vnic = &bp->vnic_info[fdir->action.rx_queue];
 	if (vnic == NULL) {
 		PMD_DRV_LOG(ERR, "Invalid queue %d\n", fdir->action.rx_queue);
 		return -EINVAL;
@@ -2497,9 +2498,9 @@ bnxt_fdir_filter(struct rte_eth_dev *dev,
 		filter->filter_type = HWRM_CFA_NTUPLE_FILTER;
 
 		if (fdir->action.behavior == RTE_ETH_FDIR_REJECT)
-			vnic = STAILQ_FIRST(&bp->ff_pool[0]);
+			vnic = &bp->vnic_info[0];
 		else
-			vnic = STAILQ_FIRST(&bp->ff_pool[fdir->action.rx_queue]);
+			vnic = &bp->vnic_info[fdir->action.rx_queue];
 
 		match = bnxt_match_fdir(bp, filter, &mvnic);
 		if (match != NULL && filter_op == RTE_ETH_FILTER_ADD) {
@@ -3226,7 +3227,9 @@ skip_init:
 		mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
 		mz = rte_memzone_lookup(mz_name);
 		total_alloc_len = RTE_CACHE_LINE_ROUNDUP(
-				sizeof(struct rx_port_stats) + 512);
+					sizeof(struct rx_port_stats) +
+					sizeof(struct rx_port_stats_ext) +
+					512);
 		if (!mz) {
 			mz = rte_memzone_reserve(mz_name, total_alloc_len,
 					SOCKET_ID_ANY,
@@ -3262,7 +3265,9 @@ skip_init:
 		mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
 		mz = rte_memzone_lookup(mz_name);
 		total_alloc_len = RTE_CACHE_LINE_ROUNDUP(
-				sizeof(struct tx_port_stats) + 512);
+					sizeof(struct tx_port_stats) +
+					sizeof(struct tx_port_stats_ext) +
+					512);
 		if (!mz) {
 			mz = rte_memzone_reserve(mz_name,
 					total_alloc_len,
@@ -3293,8 +3298,30 @@ skip_init:
 		bp->hw_tx_port_stats_map = mz_phys_addr;
 
 		bp->flags |= BNXT_FLAG_PORT_STATS;
+
+		/* Display extended statistics if FW supports it */
+		if (bp->hwrm_spec_code < HWRM_SPEC_CODE_1_8_4 ||
+		    bp->hwrm_spec_code == HWRM_SPEC_CODE_1_9_0)
+			goto skip_ext_stats;
+
+		bp->hw_rx_port_stats_ext = (void *)
+			(bp->hw_rx_port_stats + sizeof(struct rx_port_stats));
+		bp->hw_rx_port_stats_ext_map = bp->hw_rx_port_stats_map +
+			sizeof(struct rx_port_stats);
+		bp->flags |= BNXT_FLAG_EXT_RX_PORT_STATS;
+
+
+		if (bp->hwrm_spec_code < HWRM_SPEC_CODE_1_9_2) {
+			bp->hw_tx_port_stats_ext = (void *)
+			(bp->hw_tx_port_stats + sizeof(struct tx_port_stats));
+			bp->hw_tx_port_stats_ext_map =
+				bp->hw_tx_port_stats_map +
+				sizeof(struct tx_port_stats);
+			bp->flags |= BNXT_FLAG_EXT_TX_PORT_STATS;
+		}
 	}
 
+skip_ext_stats:
 	rc = bnxt_alloc_hwrm_resources(bp);
 	if (rc) {
 		PMD_DRV_LOG(ERR,
@@ -3474,10 +3501,6 @@ bnxt_dev_uninit(struct rte_eth_dev *eth_dev)
 	bnxt_disable_int(bp);
 	bnxt_free_int(bp);
 	bnxt_free_mem(bp);
-	if (eth_dev->data->mac_addrs != NULL) {
-		rte_free(eth_dev->data->mac_addrs);
-		eth_dev->data->mac_addrs = NULL;
-	}
 	if (bp->grp_info != NULL) {
 		rte_free(bp->grp_info);
 		bp->grp_info = NULL;
@@ -3515,7 +3538,11 @@ static int bnxt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 
 static int bnxt_pci_remove(struct rte_pci_device *pci_dev)
 {
-	return rte_eth_dev_pci_generic_remove(pci_dev, bnxt_dev_uninit);
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+		return rte_eth_dev_pci_generic_remove(pci_dev,
+				bnxt_dev_uninit);
+	else
+		return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
 }
 
 static struct rte_pci_driver bnxt_rte_pmd = {
@@ -3542,7 +3569,7 @@ bool is_bnxt_supported(struct rte_eth_dev *dev)
 
 RTE_INIT(bnxt_init_log)
 {
-	bnxt_logtype_driver = rte_log_register("pmd.bnxt.driver");
+	bnxt_logtype_driver = rte_log_register("pmd.net.bnxt.driver");
 	if (bnxt_logtype_driver >= 0)
 		rte_log_set_level(bnxt_logtype_driver, RTE_LOG_INFO);
 }
diff --git a/drivers/net/bnxt/bnxt_filter.c b/drivers/net/bnxt/bnxt_filter.c
index 1038941e..f43fe0db 100644
--- a/drivers/net/bnxt/bnxt_filter.c
+++ b/drivers/net/bnxt/bnxt_filter.c
@@ -80,21 +80,21 @@ void bnxt_free_all_filters(struct bnxt *bp)
 {
 	struct bnxt_vnic_info *vnic;
 	struct bnxt_filter_info *filter, *temp_filter;
-	int i;
-
-	for (i = 0; i < MAX_FF_POOLS; i++) {
-		STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) {
-			filter = STAILQ_FIRST(&vnic->filter);
-			while (filter) {
-				temp_filter = STAILQ_NEXT(filter, next);
-				STAILQ_REMOVE(&vnic->filter, filter,
-					      bnxt_filter_info, next);
-				STAILQ_INSERT_TAIL(&bp->free_filter_list,
-						   filter, next);
-				filter = temp_filter;
-			}
-			STAILQ_INIT(&vnic->filter);
+	unsigned int i;
+
+//	for (i = 0; i < MAX_FF_POOLS; i++) {
+	for (i = 0; i < bp->nr_vnics; i++) {
+		vnic = &bp->vnic_info[i];
+		filter = STAILQ_FIRST(&vnic->filter);
+		while (filter) {
+			temp_filter = STAILQ_NEXT(filter, next);
+			STAILQ_REMOVE(&vnic->filter, filter,
+					bnxt_filter_info, next);
+			STAILQ_INSERT_TAIL(&bp->free_filter_list,
+					filter, next);
+			filter = temp_filter;
 		}
+		STAILQ_INIT(&vnic->filter);
 	}
 
 	for (i = 0; i < bp->pf.max_vfs; i++) {
diff --git a/drivers/net/bnxt/bnxt_flow.c b/drivers/net/bnxt/bnxt_flow.c
index ac765674..1afe6740 100644
--- a/drivers/net/bnxt/bnxt_flow.c
+++ b/drivers/net/bnxt/bnxt_flow.c
@@ -678,7 +678,7 @@ bnxt_get_l2_filter(struct bnxt *bp, struct bnxt_filter_info *nf,
 	struct bnxt_vnic_info *vnic0;
 	int rc;
 
-	vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
+	vnic0 = &bp->vnic_info[0];
 	f0 = STAILQ_FIRST(&vnic0->filter);
 
 	/* This flow has same DST MAC as the port/l2 filter. */
@@ -763,8 +763,8 @@ bnxt_validate_and_parse_flow(struct rte_eth_dev *dev,
 		}
 		PMD_DRV_LOG(DEBUG, "Queue index %d\n", act_q->index);
 
-		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
-		vnic = STAILQ_FIRST(&bp->ff_pool[act_q->index]);
+		vnic0 = &bp->vnic_info[0];
+		vnic =  &bp->vnic_info[act_q->index];
 		if (vnic == NULL) {
 			rte_flow_error_set(error,
 					   EINVAL,
@@ -786,7 +786,7 @@ bnxt_validate_and_parse_flow(struct rte_eth_dev *dev,
 		PMD_DRV_LOG(DEBUG, "VNIC found\n");
 		break;
 	case RTE_FLOW_ACTION_TYPE_DROP:
-		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
+		vnic0 = &bp->vnic_info[0];
 		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
 		if (filter1 == NULL) {
 			rc = -ENOSPC;
@@ -802,7 +802,7 @@ bnxt_validate_and_parse_flow(struct rte_eth_dev *dev,
 				HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP;
 		break;
 	case RTE_FLOW_ACTION_TYPE_COUNT:
-		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
+		vnic0 = &bp->vnic_info[0];
 		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
 		if (filter1 == NULL) {
 			rc = -ENOSPC;
@@ -854,7 +854,7 @@ bnxt_validate_and_parse_flow(struct rte_eth_dev *dev,
 		filter->mirror_vnic_id = dflt_vnic;
 		filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_MIRROR_VNIC_ID;
 
-		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
+		vnic0 = &bp->vnic_info[0];
 		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
 		if (filter1 == NULL) {
 			rc = -ENOSPC;
diff --git a/drivers/net/bnxt/bnxt_hwrm.c b/drivers/net/bnxt/bnxt_hwrm.c
index c682488a..99997605 100644
--- a/drivers/net/bnxt/bnxt_hwrm.c
+++ b/drivers/net/bnxt/bnxt_hwrm.c
@@ -26,7 +26,7 @@
 
 #include <rte_io.h>
 
-#define HWRM_CMD_TIMEOUT		10000
+#define HWRM_CMD_TIMEOUT		6000000
 #define HWRM_SPEC_CODE_1_8_3		0x10803
 #define HWRM_VERSION_1_9_1		0x10901
 
@@ -70,7 +70,7 @@ static int page_roundup(size_t size)
  */
 
 static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg,
-					uint32_t msg_len)
+				  uint32_t msg_len, bool use_kong_mb)
 {
 	unsigned int i;
 	struct input *req = msg;
@@ -80,6 +80,10 @@ static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg,
 	uint8_t *valid;
 	uint16_t max_req_len = bp->max_req_len;
 	struct hwrm_short_input short_input = { 0 };
+	uint16_t bar_offset = use_kong_mb ?
+		GRCPF_REG_KONG_CHANNEL_OFFSET : GRCPF_REG_CHIMP_CHANNEL_OFFSET;
+	uint16_t mb_trigger_offset = use_kong_mb ?
+		GRCPF_REG_KONG_COMM_TRIGGER : GRCPF_REG_CHIMP_COMM_TRIGGER;
 
 	if (bp->flags & BNXT_FLAG_SHORT_CMD) {
 		void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
@@ -105,19 +109,19 @@ static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg,
 
 	/* Write request msg to hwrm channel */
 	for (i = 0; i < msg_len; i += 4) {
-		bar = (uint8_t *)bp->bar0 + i;
+		bar = (uint8_t *)bp->bar0 + bar_offset + i;
 		rte_write32(*data, bar);
 		data++;
 	}
 
 	/* Zero the rest of the request space */
 	for (; i < max_req_len; i += 4) {
-		bar = (uint8_t *)bp->bar0 + i;
+		bar = (uint8_t *)bp->bar0 + bar_offset + i;
 		rte_write32(0, bar);
 	}
 
 	/* Ring channel doorbell */
-	bar = (uint8_t *)bp->bar0 + 0x100;
+	bar = (uint8_t *)bp->bar0 + mb_trigger_offset;
 	rte_write32(1, bar);
 
 	/* Poll for the valid bit */
@@ -131,7 +135,7 @@ static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg,
 			if (*valid == HWRM_RESP_VALID_KEY)
 				break;
 		}
-		rte_delay_us(600);
+		rte_delay_us(1);
 	}
 
 	if (i >= HWRM_CMD_TIMEOUT) {
@@ -156,12 +160,13 @@ err_ret:
  *
  * HWRM_UNLOCK() must be called after all response processing is completed.
  */
-#define HWRM_PREP(req, type) do { \
+#define HWRM_PREP(req, type, kong) do { \
 	rte_spinlock_lock(&bp->hwrm_lock); \
 	memset(bp->hwrm_cmd_resp_addr, 0, bp->max_resp_len); \
 	req.req_type = rte_cpu_to_le_16(HWRM_##type); \
 	req.cmpl_ring = rte_cpu_to_le_16(-1); \
-	req.seq_id = rte_cpu_to_le_16(bp->hwrm_cmd_seq++); \
+	req.seq_id = kong ? rte_cpu_to_le_16(bp->kong_cmd_seq++) :\
+		rte_cpu_to_le_16(bp->hwrm_cmd_seq++); \
 	req.target_id = rte_cpu_to_le_16(0xffff); \
 	req.resp_addr = rte_cpu_to_le_64(bp->hwrm_cmd_resp_dma_addr); \
 } while (0)
@@ -220,11 +225,11 @@ int bnxt_hwrm_cfa_l2_clear_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	struct hwrm_cfa_l2_set_rx_mask_input req = {.req_type = 0 };
 	struct hwrm_cfa_l2_set_rx_mask_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, CFA_L2_SET_RX_MASK);
+	HWRM_PREP(req, CFA_L2_SET_RX_MASK, BNXT_USE_CHIMP_MB);
 	req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
 	req.mask = 0;
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -245,7 +250,7 @@ int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp,
 	if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
 		return rc;
 
-	HWRM_PREP(req, CFA_L2_SET_RX_MASK);
+	HWRM_PREP(req, CFA_L2_SET_RX_MASK, BNXT_USE_CHIMP_MB);
 	req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
 
 	/* FIXME add multicast flag, when multicast adding options is supported
@@ -275,7 +280,7 @@ int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp,
 	}
 	req.mask = rte_cpu_to_le_32(mask);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -307,14 +312,14 @@ int bnxt_hwrm_cfa_vlan_antispoof_cfg(struct bnxt *bp, uint16_t fid,
 				return 0;
 		}
 	}
-	HWRM_PREP(req, CFA_VLAN_ANTISPOOF_CFG);
+	HWRM_PREP(req, CFA_VLAN_ANTISPOOF_CFG, BNXT_USE_CHIMP_MB);
 	req.fid = rte_cpu_to_le_16(fid);
 
 	req.vlan_tag_mask_tbl_addr =
 		rte_cpu_to_le_64(rte_mem_virt2iova(vlan_table));
 	req.num_vlan_entries = rte_cpu_to_le_32((uint32_t)vlan_count);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -332,11 +337,11 @@ int bnxt_hwrm_clear_l2_filter(struct bnxt *bp,
 	if (filter->fw_l2_filter_id == UINT64_MAX)
 		return 0;
 
-	HWRM_PREP(req, CFA_L2_FILTER_FREE);
+	HWRM_PREP(req, CFA_L2_FILTER_FREE, BNXT_USE_CHIMP_MB);
 
 	req.l2_filter_id = rte_cpu_to_le_64(filter->fw_l2_filter_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -375,9 +380,11 @@ int bnxt_hwrm_set_l2_filter(struct bnxt *bp,
 	if (filter->fw_l2_filter_id != UINT64_MAX)
 		bnxt_hwrm_clear_l2_filter(bp, filter);
 
-	HWRM_PREP(req, CFA_L2_FILTER_ALLOC);
+	HWRM_PREP(req, CFA_L2_FILTER_ALLOC, BNXT_USE_CHIMP_MB);
 
 	req.flags = rte_cpu_to_le_32(filter->flags);
+	req.flags |=
+	rte_cpu_to_le_32(HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST);
 
 	enables = filter->enables |
 	      HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_DST_ID;
@@ -410,7 +417,7 @@ int bnxt_hwrm_set_l2_filter(struct bnxt *bp,
 
 	req.enables = rte_cpu_to_le_32(enables);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -430,7 +437,7 @@ int bnxt_hwrm_ptp_cfg(struct bnxt *bp)
 	if (!ptp)
 		return 0;
 
-	HWRM_PREP(req, PORT_MAC_CFG);
+	HWRM_PREP(req, PORT_MAC_CFG, BNXT_USE_CHIMP_MB);
 
 	if (ptp->rx_filter)
 		flags |= HWRM_PORT_MAC_CFG_INPUT_FLAGS_PTP_RX_TS_CAPTURE_ENABLE;
@@ -447,7 +454,7 @@ int bnxt_hwrm_ptp_cfg(struct bnxt *bp)
 		(HWRM_PORT_MAC_CFG_INPUT_ENABLES_RX_TS_CAPTURE_PTP_MSG_TYPE);
 	req.rx_ts_capture_ptp_msg_type = rte_cpu_to_le_16(ptp->rxctl);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 	HWRM_UNLOCK();
 
 	return rc;
@@ -464,11 +471,11 @@ static int bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
 	if (ptp)
 		return 0;
 
-	HWRM_PREP(req, PORT_MAC_PTP_QCFG);
+	HWRM_PREP(req, PORT_MAC_PTP_QCFG, BNXT_USE_CHIMP_MB);
 
 	req.port_id = rte_cpu_to_le_16(bp->pf.port_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -513,11 +520,11 @@ static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
 	uint32_t flags;
 	int i;
 
-	HWRM_PREP(req, FUNC_QCAPS);
+	HWRM_PREP(req, FUNC_QCAPS, BNXT_USE_CHIMP_MB);
 
 	req.fid = rte_cpu_to_le_16(0xffff);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -615,11 +622,11 @@ int bnxt_hwrm_func_reset(struct bnxt *bp)
 	struct hwrm_func_reset_input req = {.req_type = 0 };
 	struct hwrm_func_reset_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_RESET);
+	HWRM_PREP(req, FUNC_RESET, BNXT_USE_CHIMP_MB);
 
 	req.enables = rte_cpu_to_le_32(0);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -636,7 +643,7 @@ int bnxt_hwrm_func_driver_register(struct bnxt *bp)
 	if (bp->flags & BNXT_FLAG_REGISTERED)
 		return 0;
 
-	HWRM_PREP(req, FUNC_DRV_RGTR);
+	HWRM_PREP(req, FUNC_DRV_RGTR, BNXT_USE_CHIMP_MB);
 	req.enables = rte_cpu_to_le_32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VER |
 			HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD);
 	req.ver_maj = RTE_VER_YEAR;
@@ -668,7 +675,7 @@ int bnxt_hwrm_func_driver_register(struct bnxt *bp)
 		rte_cpu_to_le_32(ASYNC_CMPL_EVENT_ID_PF_DRVR_UNLOAD |
 				 ASYNC_CMPL_EVENT_ID_VF_CFG_CHANGE);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -694,7 +701,7 @@ int bnxt_hwrm_func_reserve_vf_resc(struct bnxt *bp, bool test)
 	struct hwrm_func_vf_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 	struct hwrm_func_vf_cfg_input req = {0};
 
-	HWRM_PREP(req, FUNC_VF_CFG);
+	HWRM_PREP(req, FUNC_VF_CFG, BNXT_USE_CHIMP_MB);
 
 	req.enables = rte_cpu_to_le_32
 			(HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_RX_RINGS  |
@@ -733,7 +740,7 @@ int bnxt_hwrm_func_reserve_vf_resc(struct bnxt *bp, bool test)
 
 	req.flags = rte_cpu_to_le_32(flags);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	if (test)
 		HWRM_CHECK_RESULT_SILENT();
@@ -750,10 +757,10 @@ int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp)
 	struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
 	struct hwrm_func_resource_qcaps_input req = {0};
 
-	HWRM_PREP(req, FUNC_RESOURCE_QCAPS);
+	HWRM_PREP(req, FUNC_RESOURCE_QCAPS, BNXT_USE_CHIMP_MB);
 	req.fid = rte_cpu_to_le_16(0xffff);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -782,20 +789,19 @@ int bnxt_hwrm_ver_get(struct bnxt *bp)
 	int rc = 0;
 	struct hwrm_ver_get_input req = {.req_type = 0 };
 	struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
-	uint32_t my_version;
 	uint32_t fw_version;
 	uint16_t max_resp_len;
 	char type[RTE_MEMZONE_NAMESIZE];
 	uint32_t dev_caps_cfg;
 
 	bp->max_req_len = HWRM_MAX_REQ_LEN;
-	HWRM_PREP(req, VER_GET);
+	HWRM_PREP(req, VER_GET, BNXT_USE_CHIMP_MB);
 
 	req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
 	req.hwrm_intf_min = HWRM_VERSION_MINOR;
 	req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -810,10 +816,6 @@ int bnxt_hwrm_ver_get(struct bnxt *bp)
 	PMD_DRV_LOG(INFO, "Driver HWRM version: %d.%d.%d\n",
 		HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, HWRM_VERSION_UPDATE);
 
-	my_version = HWRM_VERSION_MAJOR << 16;
-	my_version |= HWRM_VERSION_MINOR << 8;
-	my_version |= HWRM_VERSION_UPDATE;
-
 	fw_version = resp->hwrm_intf_maj_8b << 16;
 	fw_version |= resp->hwrm_intf_min_8b << 8;
 	fw_version |= resp->hwrm_intf_upd_8b;
@@ -825,21 +827,6 @@ int bnxt_hwrm_ver_get(struct bnxt *bp)
 		goto error;
 	}
 
-	if (my_version != fw_version) {
-		PMD_DRV_LOG(INFO, "BNXT Driver/HWRM API mismatch.\n");
-		if (my_version < fw_version) {
-			PMD_DRV_LOG(INFO,
-				"Firmware API version is newer than driver.\n");
-			PMD_DRV_LOG(INFO,
-				"The driver may be missing features.\n");
-		} else {
-			PMD_DRV_LOG(INFO,
-				"Firmware API version is older than driver.\n");
-			PMD_DRV_LOG(INFO,
-				"Not all driver features may be functional.\n");
-		}
-	}
-
 	if (bp->max_req_len > resp->max_req_win_len) {
 		PMD_DRV_LOG(ERR, "Unsupported request length\n");
 		rc = -EINVAL;
@@ -899,6 +886,14 @@ int bnxt_hwrm_ver_get(struct bnxt *bp)
 
 		bp->flags |= BNXT_FLAG_SHORT_CMD;
 	}
+	if (dev_caps_cfg &
+	    HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) {
+		bp->flags |= BNXT_FLAG_KONG_MB_EN;
+		PMD_DRV_LOG(DEBUG, "Kong mailbox channel enabled\n");
+	}
+	if (dev_caps_cfg &
+	    HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
+		PMD_DRV_LOG(DEBUG, "FW supports Trusted VFs\n");
 
 error:
 	HWRM_UNLOCK();
@@ -914,10 +909,10 @@ int bnxt_hwrm_func_driver_unregister(struct bnxt *bp, uint32_t flags)
 	if (!(bp->flags & BNXT_FLAG_REGISTERED))
 		return 0;
 
-	HWRM_PREP(req, FUNC_DRV_UNRGTR);
+	HWRM_PREP(req, FUNC_DRV_UNRGTR, BNXT_USE_CHIMP_MB);
 	req.flags = flags;
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -934,7 +929,7 @@ static int bnxt_hwrm_port_phy_cfg(struct bnxt *bp, struct bnxt_link_info *conf)
 	struct hwrm_port_phy_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 	uint32_t enables = 0;
 
-	HWRM_PREP(req, PORT_PHY_CFG);
+	HWRM_PREP(req, PORT_PHY_CFG, BNXT_USE_CHIMP_MB);
 
 	if (conf->link_up) {
 		/* Setting Fixed Speed. But AutoNeg is ON, So disable it */
@@ -983,7 +978,7 @@ static int bnxt_hwrm_port_phy_cfg(struct bnxt *bp, struct bnxt_link_info *conf)
 		PMD_DRV_LOG(INFO, "Force Link Down\n");
 	}
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -998,9 +993,9 @@ static int bnxt_hwrm_port_phy_qcfg(struct bnxt *bp,
 	struct hwrm_port_phy_qcfg_input req = {0};
 	struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, PORT_PHY_QCFG);
+	HWRM_PREP(req, PORT_PHY_QCFG, BNXT_USE_CHIMP_MB);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -1046,14 +1041,14 @@ int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
 	struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
 	int i;
 
-	HWRM_PREP(req, QUEUE_QPORTCFG);
+	HWRM_PREP(req, QUEUE_QPORTCFG, BNXT_USE_CHIMP_MB);
 
 	req.flags = HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_TX;
 	/* HWRM Version >= 1.9.1 */
 	if (bp->hwrm_spec_code >= HWRM_VERSION_1_9_1)
 		req.drv_qmap_cap =
 			HWRM_QUEUE_QPORTCFG_INPUT_DRV_QMAP_CAP_ENABLED;
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -1099,7 +1094,7 @@ int bnxt_hwrm_ring_alloc(struct bnxt *bp,
 	struct hwrm_ring_alloc_input req = {.req_type = 0 };
 	struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, RING_ALLOC);
+	HWRM_PREP(req, RING_ALLOC, BNXT_USE_CHIMP_MB);
 
 	req.page_tbl_addr = rte_cpu_to_le_64(ring->bd_dma);
 	req.fbo = rte_cpu_to_le_32(0);
@@ -1135,7 +1130,7 @@ int bnxt_hwrm_ring_alloc(struct bnxt *bp,
 	}
 	req.enables = rte_cpu_to_le_32(enables);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	if (rc || resp->error_code) {
 		if (rc == 0 && resp->error_code)
@@ -1175,12 +1170,12 @@ int bnxt_hwrm_ring_free(struct bnxt *bp,
 	struct hwrm_ring_free_input req = {.req_type = 0 };
 	struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, RING_FREE);
+	HWRM_PREP(req, RING_FREE, BNXT_USE_CHIMP_MB);
 
 	req.ring_type = ring_type;
 	req.ring_id = rte_cpu_to_le_16(ring->fw_ring_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	if (rc || resp->error_code) {
 		if (rc == 0 && resp->error_code)
@@ -1215,14 +1210,14 @@ int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp, unsigned int idx)
 	struct hwrm_ring_grp_alloc_input req = {.req_type = 0 };
 	struct hwrm_ring_grp_alloc_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, RING_GRP_ALLOC);
+	HWRM_PREP(req, RING_GRP_ALLOC, BNXT_USE_CHIMP_MB);
 
 	req.cr = rte_cpu_to_le_16(bp->grp_info[idx].cp_fw_ring_id);
 	req.rr = rte_cpu_to_le_16(bp->grp_info[idx].rx_fw_ring_id);
 	req.ar = rte_cpu_to_le_16(bp->grp_info[idx].ag_fw_ring_id);
 	req.sc = rte_cpu_to_le_16(bp->grp_info[idx].fw_stats_ctx);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -1240,11 +1235,11 @@ int bnxt_hwrm_ring_grp_free(struct bnxt *bp, unsigned int idx)
 	struct hwrm_ring_grp_free_input req = {.req_type = 0 };
 	struct hwrm_ring_grp_free_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, RING_GRP_FREE);
+	HWRM_PREP(req, RING_GRP_FREE, BNXT_USE_CHIMP_MB);
 
 	req.ring_group_id = rte_cpu_to_le_16(bp->grp_info[idx].fw_grp_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -1262,11 +1257,11 @@ int bnxt_hwrm_stat_clear(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
 	if (cpr->hw_stats_ctx_id == (uint32_t)HWRM_NA_SIGNATURE)
 		return rc;
 
-	HWRM_PREP(req, STAT_CTX_CLR_STATS);
+	HWRM_PREP(req, STAT_CTX_CLR_STATS, BNXT_USE_CHIMP_MB);
 
 	req.stat_ctx_id = rte_cpu_to_le_16(cpr->hw_stats_ctx_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -1281,14 +1276,14 @@ int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 	struct hwrm_stat_ctx_alloc_input req = {.req_type = 0 };
 	struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, STAT_CTX_ALLOC);
+	HWRM_PREP(req, STAT_CTX_ALLOC, BNXT_USE_CHIMP_MB);
 
 	req.update_period_ms = rte_cpu_to_le_32(0);
 
 	req.stats_dma_addr =
 	    rte_cpu_to_le_64(cpr->hw_stats_map);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -1306,11 +1301,11 @@ int bnxt_hwrm_stat_ctx_free(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 	struct hwrm_stat_ctx_free_input req = {.req_type = 0 };
 	struct hwrm_stat_ctx_free_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, STAT_CTX_FREE);
+	HWRM_PREP(req, STAT_CTX_FREE, BNXT_USE_CHIMP_MB);
 
 	req.stat_ctx_id = rte_cpu_to_le_16(cpr->hw_stats_ctx_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -1336,12 +1331,12 @@ int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	vnic->lb_rule = (uint16_t)HWRM_NA_SIGNATURE;
 	vnic->mru = bp->eth_dev->data->mtu + ETHER_HDR_LEN +
 				ETHER_CRC_LEN + VLAN_TAG_SIZE;
-	HWRM_PREP(req, VNIC_ALLOC);
+	HWRM_PREP(req, VNIC_ALLOC, BNXT_USE_CHIMP_MB);
 
 	if (vnic->func_default)
 		req.flags =
 			rte_cpu_to_le_32(HWRM_VNIC_ALLOC_INPUT_FLAGS_DEFAULT);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -1359,11 +1354,11 @@ static int bnxt_hwrm_vnic_plcmodes_qcfg(struct bnxt *bp,
 	struct hwrm_vnic_plcmodes_qcfg_input req = {.req_type = 0 };
 	struct hwrm_vnic_plcmodes_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, VNIC_PLCMODES_QCFG);
+	HWRM_PREP(req, VNIC_PLCMODES_QCFG, BNXT_USE_CHIMP_MB);
 
 	req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -1387,7 +1382,7 @@ static int bnxt_hwrm_vnic_plcmodes_cfg(struct bnxt *bp,
 	struct hwrm_vnic_plcmodes_cfg_input req = {.req_type = 0 };
 	struct hwrm_vnic_plcmodes_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, VNIC_PLCMODES_CFG);
+	HWRM_PREP(req, VNIC_PLCMODES_CFG, BNXT_USE_CHIMP_MB);
 
 	req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
 	req.flags = rte_cpu_to_le_32(pmode->flags);
@@ -1400,7 +1395,7 @@ static int bnxt_hwrm_vnic_plcmodes_cfg(struct bnxt *bp,
 	    HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_JUMBO_THRESH_VALID
 	);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -1425,7 +1420,7 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	if (rc)
 		return rc;
 
-	HWRM_PREP(req, VNIC_CFG);
+	HWRM_PREP(req, VNIC_CFG, BNXT_USE_CHIMP_MB);
 
 	/* Only RSS support for now TBD: COS & LB */
 	req.enables =
@@ -1445,9 +1440,12 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	req.cos_rule = rte_cpu_to_le_16(vnic->cos_rule);
 	req.lb_rule = rte_cpu_to_le_16(vnic->lb_rule);
 	req.mru = rte_cpu_to_le_16(vnic->mru);
-	if (vnic->func_default)
+	/* Configure default VNIC only once. */
+	if (vnic->func_default && !(bp->flags & BNXT_FLAG_DFLT_VNIC_SET)) {
 		req.flags |=
 		    rte_cpu_to_le_32(HWRM_VNIC_CFG_INPUT_FLAGS_DEFAULT);
+		bp->flags |= BNXT_FLAG_DFLT_VNIC_SET;
+	}
 	if (vnic->vlan_strip)
 		req.flags |=
 		    rte_cpu_to_le_32(HWRM_VNIC_CFG_INPUT_FLAGS_VLAN_STRIP_MODE);
@@ -1464,7 +1462,7 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 		req.flags |= rte_cpu_to_le_32(
 			HWRM_VNIC_QCFG_OUTPUT_FLAGS_RSS_DFLT_CR_MODE);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -1485,14 +1483,14 @@ int bnxt_hwrm_vnic_qcfg(struct bnxt *bp, struct bnxt_vnic_info *vnic,
 		PMD_DRV_LOG(DEBUG, "VNIC QCFG ID %d\n", vnic->fw_vnic_id);
 		return rc;
 	}
-	HWRM_PREP(req, VNIC_QCFG);
+	HWRM_PREP(req, VNIC_QCFG, BNXT_USE_CHIMP_MB);
 
 	req.enables =
 		rte_cpu_to_le_32(HWRM_VNIC_QCFG_INPUT_ENABLES_VF_ID_VALID);
 	req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
 	req.vf_id = rte_cpu_to_le_16(fw_vf_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -1526,9 +1524,9 @@ int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
 						bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_ALLOC);
+	HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_ALLOC, BNXT_USE_CHIMP_MB);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -1550,11 +1548,11 @@ int bnxt_hwrm_vnic_ctx_free(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 		PMD_DRV_LOG(DEBUG, "VNIC RSS Rule %x\n", vnic->rss_rule);
 		return rc;
 	}
-	HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_FREE);
+	HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_FREE, BNXT_USE_CHIMP_MB);
 
 	req.rss_cos_lb_ctx_id = rte_cpu_to_le_16(vnic->rss_rule);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -1575,16 +1573,20 @@ int bnxt_hwrm_vnic_free(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 		return rc;
 	}
 
-	HWRM_PREP(req, VNIC_FREE);
+	HWRM_PREP(req, VNIC_FREE, BNXT_USE_CHIMP_MB);
 
 	req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
 
 	vnic->fw_vnic_id = INVALID_HW_RING_ID;
+	/* Configure default VNIC again if necessary. */
+	if (vnic->func_default && (bp->flags & BNXT_FLAG_DFLT_VNIC_SET))
+		bp->flags &= ~BNXT_FLAG_DFLT_VNIC_SET;
+
 	return rc;
 }
 
@@ -1595,7 +1597,7 @@ int bnxt_hwrm_vnic_rss_cfg(struct bnxt *bp,
 	struct hwrm_vnic_rss_cfg_input req = {.req_type = 0 };
 	struct hwrm_vnic_rss_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, VNIC_RSS_CFG);
+	HWRM_PREP(req, VNIC_RSS_CFG, BNXT_USE_CHIMP_MB);
 
 	req.hash_type = rte_cpu_to_le_32(vnic->hash_type);
 	req.hash_mode_flags = vnic->hash_mode;
@@ -1606,7 +1608,7 @@ int bnxt_hwrm_vnic_rss_cfg(struct bnxt *bp,
 	    rte_cpu_to_le_64(vnic->rss_hash_key_dma_addr);
 	req.rss_ctx_idx = rte_cpu_to_le_16(vnic->rss_rule);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -1627,7 +1629,7 @@ int bnxt_hwrm_vnic_plcmode_cfg(struct bnxt *bp,
 		return rc;
 	}
 
-	HWRM_PREP(req, VNIC_PLCMODES_CFG);
+	HWRM_PREP(req, VNIC_PLCMODES_CFG, BNXT_USE_CHIMP_MB);
 
 	req.flags = rte_cpu_to_le_32(
 			HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_JUMBO_PLACEMENT);
@@ -1641,7 +1643,7 @@ int bnxt_hwrm_vnic_plcmode_cfg(struct bnxt *bp,
 	req.jumbo_thresh = rte_cpu_to_le_16(size);
 	req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -1656,7 +1658,7 @@ int bnxt_hwrm_vnic_tpa_cfg(struct bnxt *bp,
 	struct hwrm_vnic_tpa_cfg_input req = {.req_type = 0 };
 	struct hwrm_vnic_tpa_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, VNIC_TPA_CFG);
+	HWRM_PREP(req, VNIC_TPA_CFG, BNXT_USE_CHIMP_MB);
 
 	if (enable) {
 		req.enables = rte_cpu_to_le_32(
@@ -1677,7 +1679,7 @@ int bnxt_hwrm_vnic_tpa_cfg(struct bnxt *bp,
 	}
 	req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -1697,9 +1699,9 @@ int bnxt_hwrm_func_vf_mac(struct bnxt *bp, uint16_t vf, const uint8_t *mac_addr)
 	memcpy(req.dflt_mac_addr, mac_addr, sizeof(req.dflt_mac_addr));
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 
-	HWRM_PREP(req, FUNC_CFG);
+	HWRM_PREP(req, FUNC_CFG, BNXT_USE_CHIMP_MB);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
 
@@ -1715,11 +1717,11 @@ int bnxt_hwrm_func_qstats_tx_drop(struct bnxt *bp, uint16_t fid,
 	struct hwrm_func_qstats_input req = {.req_type = 0};
 	struct hwrm_func_qstats_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_QSTATS);
+	HWRM_PREP(req, FUNC_QSTATS, BNXT_USE_CHIMP_MB);
 
 	req.fid = rte_cpu_to_le_16(fid);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -1738,11 +1740,11 @@ int bnxt_hwrm_func_qstats(struct bnxt *bp, uint16_t fid,
 	struct hwrm_func_qstats_input req = {.req_type = 0};
 	struct hwrm_func_qstats_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_QSTATS);
+	HWRM_PREP(req, FUNC_QSTATS, BNXT_USE_CHIMP_MB);
 
 	req.fid = rte_cpu_to_le_16(fid);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -1775,11 +1777,11 @@ int bnxt_hwrm_func_clr_stats(struct bnxt *bp, uint16_t fid)
 	struct hwrm_func_clr_stats_input req = {.req_type = 0};
 	struct hwrm_func_clr_stats_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_CLR_STATS);
+	HWRM_PREP(req, FUNC_CLR_STATS, BNXT_USE_CHIMP_MB);
 
 	req.fid = rte_cpu_to_le_16(fid);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -2435,10 +2437,10 @@ int bnxt_hwrm_func_qcfg(struct bnxt *bp)
 	uint16_t flags;
 	int rc = 0;
 
-	HWRM_PREP(req, FUNC_QCFG);
+	HWRM_PREP(req, FUNC_QCFG, BNXT_USE_CHIMP_MB);
 	req.fid = rte_cpu_to_le_16(0xffff);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -2448,6 +2450,11 @@ int bnxt_hwrm_func_qcfg(struct bnxt *bp)
 	if (BNXT_PF(bp) && (flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_MULTI_HOST))
 		bp->flags |= BNXT_FLAG_MULTI_HOST;
 
+	if (BNXT_VF(bp) && (flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_TRUSTED_VF)) {
+		bp->flags |= BNXT_FLAG_TRUSTED_VF_EN;
+		PMD_DRV_LOG(INFO, "Trusted VF cap enabled\n");
+	}
+
 	switch (resp->port_partition_type) {
 	case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0:
 	case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_5:
@@ -2522,9 +2529,9 @@ static int bnxt_hwrm_pf_func_cfg(struct bnxt *bp, int tx_rings)
 	req.num_hw_ring_grps = rte_cpu_to_le_16(bp->max_ring_grps);
 	req.fid = rte_cpu_to_le_16(0xffff);
 
-	HWRM_PREP(req, FUNC_CFG);
+	HWRM_PREP(req, FUNC_CFG, BNXT_USE_CHIMP_MB);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -2595,9 +2602,9 @@ static void reserve_resources_from_vf(struct bnxt *bp,
 	int rc;
 
 	/* Get the actual allocated values now */
-	HWRM_PREP(req, FUNC_QCAPS);
+	HWRM_PREP(req, FUNC_QCAPS, BNXT_USE_CHIMP_MB);
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	if (rc) {
 		PMD_DRV_LOG(ERR, "hwrm_func_qcaps failed rc:%d\n", rc);
@@ -2631,9 +2638,9 @@ int bnxt_hwrm_func_qcfg_current_vf_vlan(struct bnxt *bp, int vf)
 	int rc;
 
 	/* Check for zero MAC address */
-	HWRM_PREP(req, FUNC_QCFG);
+	HWRM_PREP(req, FUNC_QCFG, BNXT_USE_CHIMP_MB);
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 	if (rc) {
 		PMD_DRV_LOG(ERR, "hwrm_func_qcfg failed rc:%d\n", rc);
 		return -1;
@@ -2656,9 +2663,9 @@ static int update_pf_resource_max(struct bnxt *bp)
 	int rc;
 
 	/* And copy the allocated numbers into the pf struct */
-	HWRM_PREP(req, FUNC_QCFG);
+	HWRM_PREP(req, FUNC_QCFG, BNXT_USE_CHIMP_MB);
 	req.fid = rte_cpu_to_le_16(0xffff);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 	HWRM_CHECK_RESULT();
 
 	/* Only TX ring value reflects actual allocation? TODO */
@@ -2758,10 +2765,13 @@ int bnxt_hwrm_allocate_vfs(struct bnxt *bp, int num_vfs)
 	for (i = 0; i < num_vfs; i++) {
 		add_random_mac_if_needed(bp, &req, i);
 
-		HWRM_PREP(req, FUNC_CFG);
+		HWRM_PREP(req, FUNC_CFG, BNXT_USE_CHIMP_MB);
 		req.flags = rte_cpu_to_le_32(bp->pf.vf_info[i].func_cfg_flags);
 		req.fid = rte_cpu_to_le_16(bp->pf.vf_info[i].fid);
-		rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+		rc = bnxt_hwrm_send_message(bp,
+					    &req,
+					    sizeof(req),
+					    BNXT_USE_CHIMP_MB);
 
 		/* Clear enable flag for next pass */
 		req.enables &= ~rte_cpu_to_le_32(
@@ -2811,13 +2821,13 @@ int bnxt_hwrm_pf_evb_mode(struct bnxt *bp)
 	struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 	int rc;
 
-	HWRM_PREP(req, FUNC_CFG);
+	HWRM_PREP(req, FUNC_CFG, BNXT_USE_CHIMP_MB);
 
 	req.fid = rte_cpu_to_le_16(0xffff);
 	req.enables = rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_EVB_MODE);
 	req.evb_mode = bp->pf.evb_mode;
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
 
@@ -2831,10 +2841,10 @@ int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, uint16_t port,
 	struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
 	int rc = 0;
 
-	HWRM_PREP(req, TUNNEL_DST_PORT_ALLOC);
+	HWRM_PREP(req, TUNNEL_DST_PORT_ALLOC, BNXT_USE_CHIMP_MB);
 	req.tunnel_type = tunnel_type;
 	req.tunnel_dst_port_val = port;
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 	HWRM_CHECK_RESULT();
 
 	switch (tunnel_type) {
@@ -2862,11 +2872,11 @@ int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, uint16_t port,
 	struct hwrm_tunnel_dst_port_free_output *resp = bp->hwrm_cmd_resp_addr;
 	int rc = 0;
 
-	HWRM_PREP(req, TUNNEL_DST_PORT_FREE);
+	HWRM_PREP(req, TUNNEL_DST_PORT_FREE, BNXT_USE_CHIMP_MB);
 
 	req.tunnel_type = tunnel_type;
 	req.tunnel_dst_port_id = rte_cpu_to_be_16(port);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -2881,11 +2891,11 @@ int bnxt_hwrm_func_cfg_vf_set_flags(struct bnxt *bp, uint16_t vf,
 	struct hwrm_func_cfg_input req = {0};
 	int rc;
 
-	HWRM_PREP(req, FUNC_CFG);
+	HWRM_PREP(req, FUNC_CFG, BNXT_USE_CHIMP_MB);
 
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 	req.flags = rte_cpu_to_le_32(flags);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -2911,7 +2921,7 @@ int bnxt_hwrm_func_buf_rgtr(struct bnxt *bp)
 	struct hwrm_func_buf_rgtr_input req = {.req_type = 0 };
 	struct hwrm_func_buf_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_BUF_RGTR);
+	HWRM_PREP(req, FUNC_BUF_RGTR, BNXT_USE_CHIMP_MB);
 
 	req.req_buf_num_pages = rte_cpu_to_le_16(1);
 	req.req_buf_page_size = rte_cpu_to_le_16(
@@ -2925,7 +2935,7 @@ int bnxt_hwrm_func_buf_rgtr(struct bnxt *bp)
 		return -ENOMEM;
 	}
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -2939,9 +2949,9 @@ int bnxt_hwrm_func_buf_unrgtr(struct bnxt *bp)
 	struct hwrm_func_buf_unrgtr_input req = {.req_type = 0 };
 	struct hwrm_func_buf_unrgtr_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, FUNC_BUF_UNRGTR);
+	HWRM_PREP(req, FUNC_BUF_UNRGTR, BNXT_USE_CHIMP_MB);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -2955,7 +2965,7 @@ int bnxt_hwrm_func_cfg_def_cp(struct bnxt *bp)
 	struct hwrm_func_cfg_input req = {0};
 	int rc;
 
-	HWRM_PREP(req, FUNC_CFG);
+	HWRM_PREP(req, FUNC_CFG, BNXT_USE_CHIMP_MB);
 
 	req.fid = rte_cpu_to_le_16(0xffff);
 	req.flags = rte_cpu_to_le_32(bp->pf.func_cfg_flags);
@@ -2963,7 +2973,7 @@ int bnxt_hwrm_func_cfg_def_cp(struct bnxt *bp)
 			HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR);
 	req.async_event_cr = rte_cpu_to_le_16(
 			bp->def_cp_ring->cp_ring_struct->fw_ring_id);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -2977,13 +2987,13 @@ int bnxt_hwrm_vf_func_cfg_def_cp(struct bnxt *bp)
 	struct hwrm_func_vf_cfg_input req = {0};
 	int rc;
 
-	HWRM_PREP(req, FUNC_VF_CFG);
+	HWRM_PREP(req, FUNC_VF_CFG, BNXT_USE_CHIMP_MB);
 
 	req.enables = rte_cpu_to_le_32(
-			HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR);
+			HWRM_FUNC_VF_CFG_INPUT_ENABLES_ASYNC_EVENT_CR);
 	req.async_event_cr = rte_cpu_to_le_16(
 			bp->def_cp_ring->cp_ring_struct->fw_ring_id);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -2999,7 +3009,7 @@ int bnxt_hwrm_set_default_vlan(struct bnxt *bp, int vf, uint8_t is_vf)
 	uint32_t func_cfg_flags;
 	int rc = 0;
 
-	HWRM_PREP(req, FUNC_CFG);
+	HWRM_PREP(req, FUNC_CFG, BNXT_USE_CHIMP_MB);
 
 	if (is_vf) {
 		dflt_vlan = bp->pf.vf_info[vf].dflt_vlan;
@@ -3016,7 +3026,7 @@ int bnxt_hwrm_set_default_vlan(struct bnxt *bp, int vf, uint8_t is_vf)
 	req.enables |= rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_VLAN);
 	req.dflt_vlan = rte_cpu_to_le_16(dflt_vlan);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3031,13 +3041,13 @@ int bnxt_hwrm_func_bw_cfg(struct bnxt *bp, uint16_t vf,
 	struct hwrm_func_cfg_input req = {0};
 	int rc;
 
-	HWRM_PREP(req, FUNC_CFG);
+	HWRM_PREP(req, FUNC_CFG, BNXT_USE_CHIMP_MB);
 
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 	req.enables |= rte_cpu_to_le_32(enables);
 	req.flags = rte_cpu_to_le_32(bp->pf.vf_info[vf].func_cfg_flags);
 	req.max_bw = rte_cpu_to_le_32(max_bw);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3051,14 +3061,14 @@ int bnxt_hwrm_set_vf_vlan(struct bnxt *bp, int vf)
 	struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 	int rc = 0;
 
-	HWRM_PREP(req, FUNC_CFG);
+	HWRM_PREP(req, FUNC_CFG, BNXT_USE_CHIMP_MB);
 
 	req.flags = rte_cpu_to_le_32(bp->pf.vf_info[vf].func_cfg_flags);
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 	req.enables |= rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_VLAN);
 	req.dflt_vlan = rte_cpu_to_le_16(bp->pf.vf_info[vf].dflt_vlan);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3088,12 +3098,12 @@ int bnxt_hwrm_reject_fwd_resp(struct bnxt *bp, uint16_t target_id,
 	if (ec_size > sizeof(req.encap_request))
 		return -1;
 
-	HWRM_PREP(req, REJECT_FWD_RESP);
+	HWRM_PREP(req, REJECT_FWD_RESP, BNXT_USE_CHIMP_MB);
 
 	req.encap_resp_target_id = rte_cpu_to_le_16(target_id);
 	memcpy(req.encap_request, encaped, ec_size);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3108,10 +3118,10 @@ int bnxt_hwrm_func_qcfg_vf_default_mac(struct bnxt *bp, uint16_t vf,
 	struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
 	int rc;
 
-	HWRM_PREP(req, FUNC_QCFG);
+	HWRM_PREP(req, FUNC_QCFG, BNXT_USE_CHIMP_MB);
 
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -3132,12 +3142,12 @@ int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, uint16_t target_id,
 	if (ec_size > sizeof(req.encap_request))
 		return -1;
 
-	HWRM_PREP(req, EXEC_FWD_RESP);
+	HWRM_PREP(req, EXEC_FWD_RESP, BNXT_USE_CHIMP_MB);
 
 	req.encap_resp_target_id = rte_cpu_to_le_16(target_id);
 	memcpy(req.encap_request, encaped, ec_size);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3152,11 +3162,11 @@ int bnxt_hwrm_ctx_qstats(struct bnxt *bp, uint32_t cid, int idx,
 	struct hwrm_stat_ctx_query_input req = {.req_type = 0};
 	struct hwrm_stat_ctx_query_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, STAT_CTX_QUERY);
+	HWRM_PREP(req, STAT_CTX_QUERY, BNXT_USE_CHIMP_MB);
 
 	req.stat_ctx_id = rte_cpu_to_le_32(cid);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -3192,12 +3202,12 @@ int bnxt_hwrm_port_qstats(struct bnxt *bp)
 	struct bnxt_pf_info *pf = &bp->pf;
 	int rc;
 
-	HWRM_PREP(req, PORT_QSTATS);
+	HWRM_PREP(req, PORT_QSTATS, BNXT_USE_CHIMP_MB);
 
 	req.port_id = rte_cpu_to_le_16(pf->port_id);
 	req.tx_stat_host_addr = rte_cpu_to_le_64(bp->hw_tx_port_stats_map);
 	req.rx_stat_host_addr = rte_cpu_to_le_64(bp->hw_rx_port_stats_map);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3217,10 +3227,10 @@ int bnxt_hwrm_port_clr_stats(struct bnxt *bp)
 	    BNXT_NPAR(bp) || BNXT_MH(bp) || BNXT_TOTAL_VFS(bp))
 		return 0;
 
-	HWRM_PREP(req, PORT_CLR_STATS);
+	HWRM_PREP(req, PORT_CLR_STATS, BNXT_USE_CHIMP_MB);
 
 	req.port_id = rte_cpu_to_le_16(pf->port_id);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3237,9 +3247,9 @@ int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
 	if (BNXT_VF(bp))
 		return 0;
 
-	HWRM_PREP(req, PORT_LED_QCAPS);
+	HWRM_PREP(req, PORT_LED_QCAPS, BNXT_USE_CHIMP_MB);
 	req.port_id = bp->pf.port_id;
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -3279,7 +3289,7 @@ int bnxt_hwrm_port_led_cfg(struct bnxt *bp, bool led_on)
 	if (!bp->num_leds || BNXT_VF(bp))
 		return -EOPNOTSUPP;
 
-	HWRM_PREP(req, PORT_LED_CFG);
+	HWRM_PREP(req, PORT_LED_CFG, BNXT_USE_CHIMP_MB);
 
 	if (led_on) {
 		led_state = HWRM_PORT_LED_CFG_INPUT_LED0_STATE_BLINKALT;
@@ -3297,7 +3307,7 @@ int bnxt_hwrm_port_led_cfg(struct bnxt *bp, bool led_on)
 		led_cfg->led_group_id = bp->leds[i].led_group_id;
 	}
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3312,9 +3322,9 @@ int bnxt_hwrm_nvm_get_dir_info(struct bnxt *bp, uint32_t *entries,
 	struct hwrm_nvm_get_dir_info_input req = {0};
 	struct hwrm_nvm_get_dir_info_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, NVM_GET_DIR_INFO);
+	HWRM_PREP(req, NVM_GET_DIR_INFO, BNXT_USE_CHIMP_MB);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3357,9 +3367,9 @@ int bnxt_get_nvram_directory(struct bnxt *bp, uint32_t len, uint8_t *data)
 			"unable to map response address to physical memory\n");
 		return -ENOMEM;
 	}
-	HWRM_PREP(req, NVM_GET_DIR_ENTRIES);
+	HWRM_PREP(req, NVM_GET_DIR_ENTRIES, BNXT_USE_CHIMP_MB);
 	req.host_dest_addr = rte_cpu_to_le_64(dma_handle);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	if (rc == 0)
 		memcpy(data, buf, len > buflen ? buflen : len);
@@ -3392,12 +3402,12 @@ int bnxt_hwrm_get_nvram_item(struct bnxt *bp, uint32_t index,
 			"unable to map response address to physical memory\n");
 		return -ENOMEM;
 	}
-	HWRM_PREP(req, NVM_READ);
+	HWRM_PREP(req, NVM_READ, BNXT_USE_CHIMP_MB);
 	req.host_dest_addr = rte_cpu_to_le_64(dma_handle);
 	req.dir_idx = rte_cpu_to_le_16(index);
 	req.offset = rte_cpu_to_le_32(offset);
 	req.len = rte_cpu_to_le_32(length);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 	if (rc == 0)
 		memcpy(data, buf, length);
 
@@ -3414,9 +3424,9 @@ int bnxt_hwrm_erase_nvram_directory(struct bnxt *bp, uint8_t index)
 	struct hwrm_nvm_erase_dir_entry_input req = {0};
 	struct hwrm_nvm_erase_dir_entry_output *resp = bp->hwrm_cmd_resp_addr;
 
-	HWRM_PREP(req, NVM_ERASE_DIR_ENTRY);
+	HWRM_PREP(req, NVM_ERASE_DIR_ENTRY, BNXT_USE_CHIMP_MB);
 	req.dir_idx = rte_cpu_to_le_16(index);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
 
@@ -3448,7 +3458,7 @@ int bnxt_hwrm_flash_nvram(struct bnxt *bp, uint16_t dir_type,
 	}
 	memcpy(buf, data, data_len);
 
-	HWRM_PREP(req, NVM_WRITE);
+	HWRM_PREP(req, NVM_WRITE, BNXT_USE_CHIMP_MB);
 
 	req.dir_type = rte_cpu_to_le_16(dir_type);
 	req.dir_ordinal = rte_cpu_to_le_16(dir_ordinal);
@@ -3457,7 +3467,7 @@ int bnxt_hwrm_flash_nvram(struct bnxt *bp, uint16_t dir_type,
 	req.dir_data_length = rte_cpu_to_le_32(data_len);
 	req.host_src_addr = rte_cpu_to_le_64(dma_handle);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	rte_free(buf);
 	HWRM_CHECK_RESULT();
@@ -3499,7 +3509,7 @@ static int bnxt_hwrm_func_vf_vnic_query(struct bnxt *bp, uint16_t vf,
 	int rc;
 
 	/* First query all VNIC ids */
-	HWRM_PREP(req, FUNC_VF_VNIC_IDS_QUERY);
+	HWRM_PREP(req, FUNC_VF_VNIC_IDS_QUERY, BNXT_USE_CHIMP_MB);
 
 	req.vf_id = rte_cpu_to_le_16(bp->pf.first_vf_id + vf);
 	req.max_vnic_id_cnt = rte_cpu_to_le_32(bp->pf.total_vnics);
@@ -3511,7 +3521,7 @@ static int bnxt_hwrm_func_vf_vnic_query(struct bnxt *bp, uint16_t vf,
 		"unable to map VNIC ID table address to physical memory\n");
 		return -ENOMEM;
 	}
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 	if (rc) {
 		HWRM_UNLOCK();
 		PMD_DRV_LOG(ERR, "hwrm_func_vf_vnic_query failed rc:%d\n", rc);
@@ -3591,7 +3601,7 @@ int bnxt_hwrm_func_cfg_vf_set_vlan_anti_spoof(struct bnxt *bp, uint16_t vf,
 	struct hwrm_func_cfg_input req = {0};
 	int rc;
 
-	HWRM_PREP(req, FUNC_CFG);
+	HWRM_PREP(req, FUNC_CFG, BNXT_USE_CHIMP_MB);
 
 	req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid);
 	req.enables |= rte_cpu_to_le_32(
@@ -3599,7 +3609,7 @@ int bnxt_hwrm_func_cfg_vf_set_vlan_anti_spoof(struct bnxt *bp, uint16_t vf,
 	req.vlan_antispoof_mode = on ?
 		HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_VALIDATE_VLAN :
 		HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_NOCHECK;
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3668,7 +3678,7 @@ int bnxt_hwrm_set_em_filter(struct bnxt *bp,
 	if (filter->fw_em_filter_id != UINT64_MAX)
 		bnxt_hwrm_clear_em_filter(bp, filter);
 
-	HWRM_PREP(req, CFA_EM_FLOW_ALLOC);
+	HWRM_PREP(req, CFA_EM_FLOW_ALLOC, BNXT_USE_KONG(bp));
 
 	req.flags = rte_cpu_to_le_32(filter->flags);
 
@@ -3721,7 +3731,7 @@ int bnxt_hwrm_set_em_filter(struct bnxt *bp,
 
 	req.enables = rte_cpu_to_le_32(enables);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_KONG(bp));
 
 	HWRM_CHECK_RESULT();
 
@@ -3741,11 +3751,11 @@ int bnxt_hwrm_clear_em_filter(struct bnxt *bp, struct bnxt_filter_info *filter)
 		return 0;
 
 	PMD_DRV_LOG(ERR, "Clear EM filter\n");
-	HWRM_PREP(req, CFA_EM_FLOW_FREE);
+	HWRM_PREP(req, CFA_EM_FLOW_FREE, BNXT_USE_KONG(bp));
 
 	req.em_filter_id = rte_cpu_to_le_64(filter->fw_em_filter_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_KONG(bp));
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3769,7 +3779,7 @@ int bnxt_hwrm_set_ntuple_filter(struct bnxt *bp,
 	if (filter->fw_ntuple_filter_id != UINT64_MAX)
 		bnxt_hwrm_clear_ntuple_filter(bp, filter);
 
-	HWRM_PREP(req, CFA_NTUPLE_FILTER_ALLOC);
+	HWRM_PREP(req, CFA_NTUPLE_FILTER_ALLOC, BNXT_USE_CHIMP_MB);
 
 	req.flags = rte_cpu_to_le_32(filter->flags);
 
@@ -3832,7 +3842,7 @@ int bnxt_hwrm_set_ntuple_filter(struct bnxt *bp,
 
 	req.enables = rte_cpu_to_le_32(enables);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 
@@ -3853,11 +3863,11 @@ int bnxt_hwrm_clear_ntuple_filter(struct bnxt *bp,
 	if (filter->fw_ntuple_filter_id == UINT64_MAX)
 		return 0;
 
-	HWRM_PREP(req, CFA_NTUPLE_FILTER_FREE);
+	HWRM_PREP(req, CFA_NTUPLE_FILTER_FREE, BNXT_USE_CHIMP_MB);
 
 	req.ntuple_filter_id = rte_cpu_to_le_64(filter->fw_ntuple_filter_id);
 
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
@@ -3937,11 +3947,55 @@ int bnxt_hwrm_set_ring_coal(struct bnxt *bp,
 	if (!bnxt_stratus_device(bp))
 		return 0;
 
-	HWRM_PREP(req, RING_CMPL_RING_CFG_AGGINT_PARAMS);
+	HWRM_PREP(req, RING_CMPL_RING_CFG_AGGINT_PARAMS, BNXT_USE_CHIMP_MB);
 	bnxt_hwrm_set_coal_params(coal, &req);
 	req.ring_id = rte_cpu_to_le_16(ring_id);
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
 	return 0;
 }
+
+int bnxt_hwrm_ext_port_qstats(struct bnxt *bp)
+{
+	struct hwrm_port_qstats_ext_input req = {0};
+	struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
+	struct bnxt_pf_info *pf = &bp->pf;
+	int rc;
+
+	if (!(bp->flags & BNXT_FLAG_EXT_RX_PORT_STATS ||
+	      bp->flags & BNXT_FLAG_EXT_TX_PORT_STATS))
+		return 0;
+
+	HWRM_PREP(req, PORT_QSTATS_EXT, BNXT_USE_CHIMP_MB);
+
+	req.port_id = rte_cpu_to_le_16(pf->port_id);
+	if (bp->flags & BNXT_FLAG_EXT_TX_PORT_STATS) {
+		req.tx_stat_host_addr =
+			rte_cpu_to_le_64(bp->hw_tx_port_stats_map);
+		req.tx_stat_size =
+			rte_cpu_to_le_16(sizeof(struct tx_port_stats_ext));
+	}
+	if (bp->flags & BNXT_FLAG_EXT_RX_PORT_STATS) {
+		req.rx_stat_host_addr =
+			rte_cpu_to_le_64(bp->hw_rx_port_stats_map);
+		req.rx_stat_size =
+			rte_cpu_to_le_16(sizeof(struct rx_port_stats_ext));
+	}
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
+
+	if (rc) {
+		bp->fw_rx_port_stats_ext_size = 0;
+		bp->fw_tx_port_stats_ext_size = 0;
+	} else {
+		bp->fw_rx_port_stats_ext_size =
+			rte_le_to_cpu_16(resp->rx_stat_size);
+		bp->fw_tx_port_stats_ext_size =
+			rte_le_to_cpu_16(resp->tx_stat_size);
+	}
+
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
+
+	return rc;
+}
diff --git a/drivers/net/bnxt/bnxt_hwrm.h b/drivers/net/bnxt/bnxt_hwrm.h
index 379aac6e..ec9b3e00 100644
--- a/drivers/net/bnxt/bnxt_hwrm.h
+++ b/drivers/net/bnxt/bnxt_hwrm.h
@@ -32,6 +32,10 @@ struct bnxt_cp_ring_info;
 #define HWRM_FUNC_RESOURCE_QCAPS_OUTPUT_VF_RESV_STRATEGY_MINIMAL_STATIC \
 	HWRM_FUNC_RESOURCE_QCAPS_OUTPUT_VF_RESERVATION_STRATEGY_MINIMAL_STATIC
 
+#define HWRM_SPEC_CODE_1_8_4		0x10804
+#define HWRM_SPEC_CODE_1_9_0		0x10900
+#define HWRM_SPEC_CODE_1_9_2		0x10902
+
 int bnxt_hwrm_cfa_l2_clear_rx_mask(struct bnxt *bp,
 				   struct bnxt_vnic_info *vnic);
 int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic,
@@ -174,4 +178,5 @@ int bnxt_vnic_rss_configure(struct bnxt *bp,
 int bnxt_hwrm_set_ring_coal(struct bnxt *bp,
 			struct bnxt_coal *coal, uint16_t ring_id);
 int bnxt_hwrm_check_vf_rings(struct bnxt *bp);
+int bnxt_hwrm_ext_port_qstats(struct bnxt *bp);
 #endif
diff --git a/drivers/net/bnxt/bnxt_rxq.c b/drivers/net/bnxt/bnxt_rxq.c
index 832fc9ec..5345d393 100644
--- a/drivers/net/bnxt/bnxt_rxq.c
+++ b/drivers/net/bnxt/bnxt_rxq.c
@@ -43,21 +43,19 @@ int bnxt_mq_rx_configure(struct bnxt *bp)
 
 	/* Single queue mode */
 	if (bp->rx_cp_nr_rings < 2) {
-		vnic = bnxt_alloc_vnic(bp);
+		vnic = &bp->vnic_info[0];
 		if (!vnic) {
 			PMD_DRV_LOG(ERR, "VNIC alloc failed\n");
 			rc = -ENOMEM;
 			goto err_out;
 		}
 		vnic->flags |= BNXT_VNIC_INFO_BCAST;
-		STAILQ_INSERT_TAIL(&bp->ff_pool[0], vnic, next);
 		bp->nr_vnics++;
 
 		rxq = bp->eth_dev->data->rx_queues[0];
 		rxq->vnic = vnic;
 
 		vnic->func_default = true;
-		vnic->ff_pool_idx = 0;
 		vnic->start_grp_id = 0;
 		vnic->end_grp_id = vnic->start_grp_id;
 		filter = bnxt_alloc_filter(bp);
@@ -85,6 +83,9 @@ int bnxt_mq_rx_configure(struct bnxt *bp)
 					    RTE_MIN(bp->max_l2_ctx,
 					    RTE_MIN(bp->max_rsscos_ctx,
 						    ETH_64_POOLS)));
+			PMD_DRV_LOG(DEBUG,
+				    "pools = %u max_pools = %u\n",
+				    pools, max_pools);
 			if (pools > max_pools)
 				pools = max_pools;
 			break;
@@ -98,25 +99,27 @@ int bnxt_mq_rx_configure(struct bnxt *bp)
 			goto err_out;
 		}
 	}
-
 	nb_q_per_grp = bp->rx_cp_nr_rings / pools;
+	PMD_DRV_LOG(ERR, "pools = %u nb_q_per_grp = %u\n", pools, nb_q_per_grp);
 	start_grp_id = 0;
 	end_grp_id = nb_q_per_grp;
 
 	for (i = 0; i < pools; i++) {
-		vnic = bnxt_alloc_vnic(bp);
+		vnic = &bp->vnic_info[i];
 		if (!vnic) {
 			PMD_DRV_LOG(ERR, "VNIC alloc failed\n");
 			rc = -ENOMEM;
 			goto err_out;
 		}
 		vnic->flags |= BNXT_VNIC_INFO_BCAST;
-		STAILQ_INSERT_TAIL(&bp->ff_pool[i], vnic, next);
 		bp->nr_vnics++;
 
 		for (j = 0; j < nb_q_per_grp; j++, ring_idx++) {
 			rxq = bp->eth_dev->data->rx_queues[ring_idx];
 			rxq->vnic = vnic;
+			PMD_DRV_LOG(DEBUG,
+				    "rxq[%d] = %p vnic[%d] = %p\n",
+				    ring_idx, rxq, i, vnic);
 		}
 		if (i == 0) {
 			if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB) {
@@ -125,7 +128,6 @@ int bnxt_mq_rx_configure(struct bnxt *bp)
 			}
 			vnic->func_default = true;
 		}
-		vnic->ff_pool_idx = i;
 		vnic->start_grp_id = start_grp_id;
 		vnic->end_grp_id = end_grp_id;
 
@@ -176,7 +178,7 @@ out:
 			hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6;
 
 		for (i = 0; i < bp->nr_vnics; i++) {
-			STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) {
+			vnic = &bp->vnic_info[i];
 			vnic->hash_type = hash_type;
 
 			/*
@@ -187,7 +189,6 @@ out:
 			    rss->rss_key_len <= HW_HASH_KEY_SIZE)
 				memcpy(vnic->rss_hash_key,
 				       rss->rss_key, rss->rss_key_len);
-			}
 		}
 	}
 
@@ -331,8 +332,10 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev,
 
 	rxq->queue_id = queue_idx;
 	rxq->port_id = eth_dev->data->port_id;
-	rxq->crc_len = rte_eth_dev_must_keep_crc(rx_offloads) ?
-		ETHER_CRC_LEN : 0;
+	if (rx_offloads & DEV_RX_OFFLOAD_KEEP_CRC)
+		rxq->crc_len = ETHER_CRC_LEN;
+	else
+		rxq->crc_len = 0;
 
 	eth_dev->data->rx_queues[queue_idx] = rxq;
 	/* Allocate RX ring hardware descriptors */
diff --git a/drivers/net/bnxt/bnxt_stats.c b/drivers/net/bnxt/bnxt_stats.c
index a5d3c866..c16bf99d 100644
--- a/drivers/net/bnxt/bnxt_stats.c
+++ b/drivers/net/bnxt/bnxt_stats.c
@@ -26,8 +26,8 @@ static const struct bnxt_xstats_name_off bnxt_rx_stats_strings[] = {
 				rx_256b_511b_frames)},
 	{"rx_512b_1023b_frames", offsetof(struct rx_port_stats,
 				rx_512b_1023b_frames)},
-	{"rx_1024b_1518_frames", offsetof(struct rx_port_stats,
-				rx_1024b_1518_frames)},
+	{"rx_1024b_1518b_frames", offsetof(struct rx_port_stats,
+				rx_1024b_1518b_frames)},
 	{"rx_good_vlan_frames", offsetof(struct rx_port_stats,
 				rx_good_vlan_frames)},
 	{"rx_1519b_2047b_frames", offsetof(struct rx_port_stats,
@@ -93,12 +93,12 @@ static const struct bnxt_xstats_name_off bnxt_tx_stats_strings[] = {
 				tx_256b_511b_frames)},
 	{"tx_512b_1023b_frames", offsetof(struct tx_port_stats,
 				tx_512b_1023b_frames)},
-	{"tx_1024b_1518_frames", offsetof(struct tx_port_stats,
-				tx_1024b_1518_frames)},
+	{"tx_1024b_1518b_frames", offsetof(struct tx_port_stats,
+				tx_1024b_1518b_frames)},
 	{"tx_good_vlan_frames", offsetof(struct tx_port_stats,
 				tx_good_vlan_frames)},
-	{"tx_1519b_2047_frames", offsetof(struct tx_port_stats,
-				tx_1519b_2047_frames)},
+	{"tx_1519b_2047b_frames", offsetof(struct tx_port_stats,
+				tx_1519b_2047b_frames)},
 	{"tx_2048b_4095b_frames", offsetof(struct tx_port_stats,
 				tx_2048b_4095b_frames)},
 	{"tx_4096b_9216b_frames", offsetof(struct tx_port_stats,
@@ -180,6 +180,150 @@ static const struct bnxt_xstats_name_off bnxt_func_stats_strings[] = {
 				rx_agg_aborts)},
 };
 
+static const struct bnxt_xstats_name_off bnxt_rx_ext_stats_strings[] = {
+	{"link_down_events", offsetof(struct rx_port_stats_ext,
+				link_down_events)},
+	{"continuous_pause_events", offsetof(struct rx_port_stats_ext,
+				continuous_pause_events)},
+	{"resume_pause_events", offsetof(struct rx_port_stats_ext,
+				resume_pause_events)},
+	{"continuous_roce_pause_events", offsetof(struct rx_port_stats_ext,
+				continuous_roce_pause_events)},
+	{"resume_roce_pause_events", offsetof(struct rx_port_stats_ext,
+				resume_roce_pause_events)},
+	{"rx_bytes_cos0", offsetof(struct rx_port_stats_ext,
+				rx_bytes_cos0)},
+	{"rx_bytes_cos1", offsetof(struct rx_port_stats_ext,
+				rx_bytes_cos1)},
+	{"rx_bytes_cos2", offsetof(struct rx_port_stats_ext,
+				rx_bytes_cos2)},
+	{"rx_bytes_cos3", offsetof(struct rx_port_stats_ext,
+				rx_bytes_cos3)},
+	{"rx_bytes_cos4", offsetof(struct rx_port_stats_ext,
+				rx_bytes_cos4)},
+	{"rx_bytes_cos5", offsetof(struct rx_port_stats_ext,
+				rx_bytes_cos5)},
+	{"rx_bytes_cos6", offsetof(struct rx_port_stats_ext,
+				rx_bytes_cos6)},
+	{"rx_bytes_cos7", offsetof(struct rx_port_stats_ext,
+				rx_bytes_cos7)},
+	{"rx_packets_cos0", offsetof(struct rx_port_stats_ext,
+				rx_packets_cos0)},
+	{"rx_packets_cos1", offsetof(struct rx_port_stats_ext,
+				rx_packets_cos1)},
+	{"rx_packets_cos2", offsetof(struct rx_port_stats_ext,
+				rx_packets_cos2)},
+	{"rx_packets_cos3", offsetof(struct rx_port_stats_ext,
+				rx_packets_cos3)},
+	{"rx_packets_cos4", offsetof(struct rx_port_stats_ext,
+				rx_packets_cos4)},
+	{"rx_packets_cos5", offsetof(struct rx_port_stats_ext,
+				rx_packets_cos5)},
+	{"rx_packets_cos6", offsetof(struct rx_port_stats_ext,
+				rx_packets_cos6)},
+	{"rx_packets_cos7", offsetof(struct rx_port_stats_ext,
+				rx_packets_cos7)},
+	{"pfc_pri0_rx_duration_us", offsetof(struct rx_port_stats_ext,
+				pfc_pri0_rx_duration_us)},
+	{"pfc_pri0_rx_transitions", offsetof(struct rx_port_stats_ext,
+				pfc_pri0_rx_transitions)},
+	{"pfc_pri1_rx_duration_us", offsetof(struct rx_port_stats_ext,
+				pfc_pri1_rx_duration_us)},
+	{"pfc_pri1_rx_transitions", offsetof(struct rx_port_stats_ext,
+				pfc_pri1_rx_transitions)},
+	{"pfc_pri2_rx_duration_us", offsetof(struct rx_port_stats_ext,
+				pfc_pri2_rx_duration_us)},
+	{"pfc_pri2_rx_transitions", offsetof(struct rx_port_stats_ext,
+				pfc_pri2_rx_transitions)},
+	{"pfc_pri3_rx_duration_us", offsetof(struct rx_port_stats_ext,
+				pfc_pri3_rx_duration_us)},
+	{"pfc_pri3_rx_transitions", offsetof(struct rx_port_stats_ext,
+				pfc_pri3_rx_transitions)},
+	{"pfc_pri4_rx_duration_us", offsetof(struct rx_port_stats_ext,
+				pfc_pri4_rx_duration_us)},
+	{"pfc_pri4_rx_transitions", offsetof(struct rx_port_stats_ext,
+				pfc_pri4_rx_transitions)},
+	{"pfc_pri5_rx_duration_us", offsetof(struct rx_port_stats_ext,
+				pfc_pri5_rx_duration_us)},
+	{"pfc_pri5_rx_transitions", offsetof(struct rx_port_stats_ext,
+				pfc_pri5_rx_transitions)},
+	{"pfc_pri6_rx_duration_us", offsetof(struct rx_port_stats_ext,
+				pfc_pri6_rx_duration_us)},
+	{"pfc_pri6_rx_transitions", offsetof(struct rx_port_stats_ext,
+				pfc_pri6_rx_transitions)},
+	{"pfc_pri7_rx_duration_us", offsetof(struct rx_port_stats_ext,
+				pfc_pri7_rx_duration_us)},
+	{"pfc_pri7_rx_transitions", offsetof(struct rx_port_stats_ext,
+				pfc_pri7_rx_transitions)},
+};
+
+static const struct bnxt_xstats_name_off bnxt_tx_ext_stats_strings[] = {
+	{"tx_bytes_cos0", offsetof(struct tx_port_stats_ext,
+				tx_bytes_cos0)},
+	{"tx_bytes_cos1", offsetof(struct tx_port_stats_ext,
+				tx_bytes_cos1)},
+	{"tx_bytes_cos2", offsetof(struct tx_port_stats_ext,
+				tx_bytes_cos2)},
+	{"tx_bytes_cos3", offsetof(struct tx_port_stats_ext,
+				tx_bytes_cos3)},
+	{"tx_bytes_cos4", offsetof(struct tx_port_stats_ext,
+				tx_bytes_cos4)},
+	{"tx_bytes_cos5", offsetof(struct tx_port_stats_ext,
+				tx_bytes_cos5)},
+	{"tx_bytes_cos6", offsetof(struct tx_port_stats_ext,
+				tx_bytes_cos6)},
+	{"tx_bytes_cos7", offsetof(struct tx_port_stats_ext,
+				tx_bytes_cos7)},
+	{"tx_packets_cos0", offsetof(struct tx_port_stats_ext,
+				tx_packets_cos0)},
+	{"tx_packets_cos1", offsetof(struct tx_port_stats_ext,
+				tx_packets_cos1)},
+	{"tx_packets_cos2", offsetof(struct tx_port_stats_ext,
+				tx_packets_cos2)},
+	{"tx_packets_cos3", offsetof(struct tx_port_stats_ext,
+				tx_packets_cos3)},
+	{"tx_packets_cos4", offsetof(struct tx_port_stats_ext,
+				tx_packets_cos4)},
+	{"tx_packets_cos5", offsetof(struct tx_port_stats_ext,
+				tx_packets_cos5)},
+	{"tx_packets_cos6", offsetof(struct tx_port_stats_ext,
+				tx_packets_cos6)},
+	{"tx_packets_cos7", offsetof(struct tx_port_stats_ext,
+				tx_packets_cos7)},
+	{"pfc_pri0_tx_duration_us", offsetof(struct tx_port_stats_ext,
+				pfc_pri0_tx_duration_us)},
+	{"pfc_pri0_tx_transitions", offsetof(struct tx_port_stats_ext,
+				pfc_pri0_tx_transitions)},
+	{"pfc_pri1_tx_duration_us", offsetof(struct tx_port_stats_ext,
+				pfc_pri1_tx_duration_us)},
+	{"pfc_pri1_tx_transitions", offsetof(struct tx_port_stats_ext,
+				pfc_pri1_tx_transitions)},
+	{"pfc_pri2_tx_duration_us", offsetof(struct tx_port_stats_ext,
+				pfc_pri2_tx_duration_us)},
+	{"pfc_pri2_tx_transitions", offsetof(struct tx_port_stats_ext,
+				pfc_pri2_tx_transitions)},
+	{"pfc_pri3_tx_duration_us", offsetof(struct tx_port_stats_ext,
+				pfc_pri3_tx_duration_us)},
+	{"pfc_pri3_tx_transitions", offsetof(struct tx_port_stats_ext,
+				pfc_pri3_tx_transitions)},
+	{"pfc_pri4_tx_duration_us", offsetof(struct tx_port_stats_ext,
+				pfc_pri4_tx_duration_us)},
+	{"pfc_pri4_tx_transitions", offsetof(struct tx_port_stats_ext,
+				pfc_pri4_tx_transitions)},
+	{"pfc_pri5_tx_duration_us", offsetof(struct tx_port_stats_ext,
+				pfc_pri5_tx_duration_us)},
+	{"pfc_pri5_tx_transitions", offsetof(struct tx_port_stats_ext,
+				pfc_pri5_tx_transitions)},
+	{"pfc_pri6_tx_duration_us", offsetof(struct tx_port_stats_ext,
+				pfc_pri6_tx_duration_us)},
+	{"pfc_pri6_tx_transitions", offsetof(struct tx_port_stats_ext,
+				pfc_pri6_tx_transitions)},
+	{"pfc_pri7_tx_duration_us", offsetof(struct tx_port_stats_ext,
+				pfc_pri7_tx_duration_us)},
+	{"pfc_pri7_tx_transitions", offsetof(struct tx_port_stats_ext,
+				pfc_pri7_tx_transitions)},
+};
+
 /*
  * Statistics functions
  */
@@ -265,12 +409,22 @@ int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev,
 
 	unsigned int count, i;
 	uint64_t tx_drop_pkts;
+	unsigned int rx_port_stats_ext_cnt;
+	unsigned int tx_port_stats_ext_cnt;
+	unsigned int stat_size = sizeof(uint64_t);
+	unsigned int stat_count;
 
 	bnxt_hwrm_port_qstats(bp);
 	bnxt_hwrm_func_qstats_tx_drop(bp, 0xffff, &tx_drop_pkts);
+	bnxt_hwrm_ext_port_qstats(bp);
+	rx_port_stats_ext_cnt = bp->fw_rx_port_stats_ext_size / stat_size;
+	tx_port_stats_ext_cnt = bp->fw_tx_port_stats_ext_size / stat_size;
 
 	count = RTE_DIM(bnxt_rx_stats_strings) +
-		RTE_DIM(bnxt_tx_stats_strings) + 1; /* For tx_drop_pkts */
+		RTE_DIM(bnxt_tx_stats_strings) + 1/* For tx_drop_pkts */ +
+		RTE_DIM(bnxt_rx_ext_stats_strings) +
+		RTE_DIM(bnxt_tx_ext_stats_strings);
+	stat_count = count;
 
 	if (n < count)
 		return count;
@@ -299,7 +453,27 @@ int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev,
 	xstats[count].value = rte_le_to_cpu_64(tx_drop_pkts);
 	count++;
 
-	return count;
+	for (i = 0; i < tx_port_stats_ext_cnt; i++) {
+		uint64_t *tx_stats_ext = (uint64_t *)bp->hw_tx_port_stats_ext;
+
+		xstats[count].value = rte_le_to_cpu_64
+					(*(uint64_t *)((char *)tx_stats_ext +
+					 bnxt_tx_ext_stats_strings[i].offset));
+
+		count++;
+	}
+
+	for (i = 0; i < rx_port_stats_ext_cnt; i++) {
+		uint64_t *rx_stats_ext = (uint64_t *)bp->hw_rx_port_stats_ext;
+
+		xstats[count].value = rte_le_to_cpu_64
+					(*(uint64_t *)((char *)rx_stats_ext +
+					 bnxt_rx_ext_stats_strings[i].offset));
+
+		count++;
+	}
+
+	return stat_count;
 }
 
 int bnxt_dev_xstats_get_names_op(__rte_unused struct rte_eth_dev *eth_dev,
@@ -308,7 +482,9 @@ int bnxt_dev_xstats_get_names_op(__rte_unused struct rte_eth_dev *eth_dev,
 {
 	/* Account for the Tx drop pkts aka the Anti spoof counter */
 	const unsigned int stat_cnt = RTE_DIM(bnxt_rx_stats_strings) +
-				RTE_DIM(bnxt_tx_stats_strings) + 1;
+				RTE_DIM(bnxt_tx_stats_strings) + 1 +
+				RTE_DIM(bnxt_rx_ext_stats_strings) +
+				RTE_DIM(bnxt_tx_ext_stats_strings);
 	unsigned int i, count;
 
 	if (xstats_names != NULL) {
@@ -335,6 +511,25 @@ int bnxt_dev_xstats_get_names_op(__rte_unused struct rte_eth_dev *eth_dev,
 				"%s",
 				bnxt_func_stats_strings[4].name);
 		count++;
+
+		for (i = 0; i < RTE_DIM(bnxt_rx_ext_stats_strings); i++) {
+			snprintf(xstats_names[count].name,
+				 sizeof(xstats_names[count].name),
+				 "%s",
+				 bnxt_rx_ext_stats_strings[i].name);
+
+			count++;
+		}
+
+		for (i = 0; i < RTE_DIM(bnxt_tx_ext_stats_strings); i++) {
+			snprintf(xstats_names[count].name,
+				 sizeof(xstats_names[count].name),
+				 "%s",
+				 bnxt_tx_ext_stats_strings[i].name);
+
+			count++;
+		}
+
 	}
 	return stat_cnt;
 }
@@ -359,7 +554,9 @@ int bnxt_dev_xstats_get_by_id_op(struct rte_eth_dev *dev, const uint64_t *ids,
 {
 	/* Account for the Tx drop pkts aka the Anti spoof counter */
 	const unsigned int stat_cnt = RTE_DIM(bnxt_rx_stats_strings) +
-				RTE_DIM(bnxt_tx_stats_strings) + 1;
+				RTE_DIM(bnxt_tx_stats_strings) + 1 +
+				RTE_DIM(bnxt_rx_ext_stats_strings) +
+				RTE_DIM(bnxt_tx_ext_stats_strings);
 	struct rte_eth_xstat xstats[stat_cnt];
 	uint64_t values_copy[stat_cnt];
 	uint16_t i;
@@ -384,7 +581,9 @@ int bnxt_dev_xstats_get_names_by_id_op(struct rte_eth_dev *dev,
 {
 	/* Account for the Tx drop pkts aka the Anti spoof counter */
 	const unsigned int stat_cnt = RTE_DIM(bnxt_rx_stats_strings) +
-				RTE_DIM(bnxt_tx_stats_strings) + 1;
+				RTE_DIM(bnxt_tx_stats_strings) + 1 +
+				RTE_DIM(bnxt_rx_ext_stats_strings) +
+				RTE_DIM(bnxt_tx_ext_stats_strings);
 	struct rte_eth_xstat_name xstats_names_copy[stat_cnt];
 	uint16_t i;
 
diff --git a/drivers/net/bnxt/bnxt_txr.c b/drivers/net/bnxt/bnxt_txr.c
index 67bb35e0..39be7bdf 100644
--- a/drivers/net/bnxt/bnxt_txr.c
+++ b/drivers/net/bnxt/bnxt_txr.c
@@ -120,7 +120,7 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt,
 {
 	struct bnxt_tx_ring_info *txr = txq->tx_ring;
 	struct tx_bd_long *txbd;
-	struct tx_bd_long_hi *txbd1;
+	struct tx_bd_long_hi *txbd1 = NULL;
 	uint32_t vlan_tag_flags, cfa_action;
 	bool long_bd = false;
 	uint16_t last_prod = 0;
@@ -295,7 +295,8 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt,
 	}
 
 	txbd->flags_type |= TX_BD_LONG_FLAGS_PACKET_END;
-	txbd1->lflags = rte_cpu_to_le_32(txbd1->lflags);
+	if (txbd1)
+		txbd1->lflags = rte_cpu_to_le_32(txbd1->lflags);
 
 	txr->tx_prod = RING_NEXT(txr->tx_ring_struct, txr->tx_prod);
 
diff --git a/drivers/net/bnxt/bnxt_vnic.c b/drivers/net/bnxt/bnxt_vnic.c
index c0577cd7..aebfb1f1 100644
--- a/drivers/net/bnxt/bnxt_vnic.c
+++ b/drivers/net/bnxt/bnxt_vnic.c
@@ -57,29 +57,6 @@ void bnxt_init_vnics(struct bnxt *bp)
 		STAILQ_INIT(&vnic->flow_list);
 		STAILQ_INSERT_TAIL(&bp->free_vnic_list, vnic, next);
 	}
-	for (i = 0; i < MAX_FF_POOLS; i++)
-		STAILQ_INIT(&bp->ff_pool[i]);
-}
-
-int bnxt_free_vnic(struct bnxt *bp, struct bnxt_vnic_info *vnic,
-			  int pool)
-{
-	struct bnxt_vnic_info *temp;
-
-	temp = STAILQ_FIRST(&bp->ff_pool[pool]);
-	while (temp) {
-		if (temp == vnic) {
-			STAILQ_REMOVE(&bp->ff_pool[pool], vnic,
-				      bnxt_vnic_info, next);
-			vnic->fw_vnic_id = (uint16_t)HWRM_NA_SIGNATURE;
-			STAILQ_INSERT_TAIL(&bp->free_vnic_list, vnic,
-					   next);
-			return 0;
-		}
-		temp = STAILQ_NEXT(temp, next);
-	}
-	PMD_DRV_LOG(ERR, "VNIC %p is not found in pool[%d]\n", vnic, pool);
-	return -EINVAL;
 }
 
 struct bnxt_vnic_info *bnxt_alloc_vnic(struct bnxt *bp)
@@ -98,26 +75,22 @@ struct bnxt_vnic_info *bnxt_alloc_vnic(struct bnxt *bp)
 
 void bnxt_free_all_vnics(struct bnxt *bp)
 {
-	struct bnxt_vnic_info *temp, *next;
-	int i;
+	struct bnxt_vnic_info *temp;
+	unsigned int i;
 
-	for (i = 0; i < MAX_FF_POOLS; i++) {
-		temp = STAILQ_FIRST(&bp->ff_pool[i]);
-		while (temp) {
-			next = STAILQ_NEXT(temp, next);
-			STAILQ_REMOVE(&bp->ff_pool[i], temp, bnxt_vnic_info,
-				      next);
-			STAILQ_INSERT_TAIL(&bp->free_vnic_list, temp, next);
-			temp = next;
-		}
+	for (i = 0; i < bp->nr_vnics; i++) {
+		temp = &bp->vnic_info[i];
+		STAILQ_INSERT_TAIL(&bp->free_vnic_list, temp, next);
 	}
 }
 
 void bnxt_free_vnic_attributes(struct bnxt *bp)
 {
 	struct bnxt_vnic_info *vnic;
+	unsigned int i;
 
-	STAILQ_FOREACH(vnic, &bp->free_vnic_list, next) {
+	for (i = 0; i < bp->max_vnics; i++) {
+		vnic = &bp->vnic_info[i];
 		if (vnic->rss_table) {
 			/* 'Unreserve' the rss_table */
 			/* N/A */
diff --git a/drivers/net/bnxt/hsi_struct_def_dpdk.h b/drivers/net/bnxt/hsi_struct_def_dpdk.h
index f5c7b422..e8005793 100644
--- a/drivers/net/bnxt/hsi_struct_def_dpdk.h
+++ b/drivers/net/bnxt/hsi_struct_def_dpdk.h
@@ -67,6 +67,10 @@ struct hwrm_resp_hdr {
 #define TLV_TYPE_HWRM_RESPONSE                   UINT32_C(0x2)
 /* RoCE slow path command */
 #define TLV_TYPE_ROCE_SP_COMMAND                 UINT32_C(0x3)
+/* RoCE slow path command to query CC Gen1 support. */
+#define TLV_TYPE_QUERY_ROCE_CC_GEN1              UINT32_C(0xcommand 0x0005)
+/* RoCE slow path command to modify CC Gen1 support. */
+#define TLV_TYPE_MODIFY_ROCE_CC_GEN1             UINT32_C(0xcommand 0x0005)
 /* Engine CKV - The device's serial number. */
 #define TLV_TYPE_ENGINE_CKV_DEVICE_SERIAL_NUMBER UINT32_C(0x8001)
 /* Engine CKV - Per-function random nonce data. */
@@ -256,6 +260,7 @@ struct cmd_nums {
 	 */
 	uint16_t	req_type;
 	#define HWRM_VER_GET                              UINT32_C(0x0)
+	#define HWRM_FUNC_DRV_IF_CHANGE                   UINT32_C(0xd)
 	#define HWRM_FUNC_BUF_UNRGTR                      UINT32_C(0xe)
 	#define HWRM_FUNC_VF_CFG                          UINT32_C(0xf)
 	/* Reserved for future use. */
@@ -328,6 +333,7 @@ struct cmd_nums {
 	#define HWRM_RING_FREE                            UINT32_C(0x51)
 	#define HWRM_RING_CMPL_RING_QAGGINT_PARAMS        UINT32_C(0x52)
 	#define HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS     UINT32_C(0x53)
+	#define HWRM_RING_AGGINT_QCAPS                    UINT32_C(0x54)
 	#define HWRM_RING_RESET                           UINT32_C(0x5e)
 	#define HWRM_RING_GRP_ALLOC                       UINT32_C(0x60)
 	#define HWRM_RING_GRP_FREE                        UINT32_C(0x61)
@@ -367,6 +373,8 @@ struct cmd_nums {
 	#define HWRM_PORT_QSTATS_EXT                      UINT32_C(0xb4)
 	#define HWRM_FW_RESET                             UINT32_C(0xc0)
 	#define HWRM_FW_QSTATUS                           UINT32_C(0xc1)
+	#define HWRM_FW_HEALTH_CHECK                      UINT32_C(0xc2)
+	#define HWRM_FW_SYNC                              UINT32_C(0xc3)
 	/* Experimental */
 	#define HWRM_FW_SET_TIME                          UINT32_C(0xc8)
 	/* Experimental */
@@ -433,6 +441,7 @@ struct cmd_nums {
 	/* Experimental */
 	#define HWRM_FW_IPC_MSG                           UINT32_C(0x110)
 	#define HWRM_CFA_REDIRECT_TUNNEL_TYPE_INFO        UINT32_C(0x111)
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE       UINT32_C(0x112)
 	/* Engine CKV - Ping the device and SRT firmware to get the public key. */
 	#define HWRM_ENGINE_CKV_HELLO                     UINT32_C(0x12d)
 	/* Engine CKV - Get the current allocation status of keys provisioned in the key vault. */
@@ -515,6 +524,10 @@ struct cmd_nums {
 	#define HWRM_FUNC_BACKING_STORE_CFG               UINT32_C(0x193)
 	/* Experimental */
 	#define HWRM_FUNC_BACKING_STORE_QCFG              UINT32_C(0x194)
+	/* Configures the BW of any VF */
+	#define HWRM_FUNC_VF_BW_CFG                       UINT32_C(0x195)
+	/* Queries the BW of any VF */
+	#define HWRM_FUNC_VF_BW_QCFG                      UINT32_C(0x196)
 	/* Experimental */
 	#define HWRM_SELFTEST_QLIST                       UINT32_C(0x200)
 	/* Experimental */
@@ -544,8 +557,12 @@ struct cmd_nums {
 	#define HWRM_DBG_COREDUMP_INITIATE                UINT32_C(0xff18)
 	/* Experimental */
 	#define HWRM_DBG_COREDUMP_RETRIEVE                UINT32_C(0xff19)
+	/* Experimental */
+	#define HWRM_DBG_FW_CLI                           UINT32_C(0xff1a)
 	/*  */
 	#define HWRM_DBG_I2C_CMD                          UINT32_C(0xff1b)
+	/*  */
+	#define HWRM_DBG_RING_INFO_GET                    UINT32_C(0xff1c)
 	/* Experimental */
 	#define HWRM_NVM_FACTORY_DEFAULTS                 UINT32_C(0xffee)
 	#define HWRM_NVM_VALIDATE_OPTION                  UINT32_C(0xffef)
@@ -616,6 +633,11 @@ struct ret_codes {
 	 */
 	#define HWRM_ERR_CODE_NO_BUFFER              UINT32_C(0x8)
 	/*
+	 * This error code is only reported by firmware when some
+	 * sub-option of a supported HWRM command is unsupported.
+	 */
+	#define HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR UINT32_C(0x9)
+	/*
 	 * Generic HWRM execution error that represents an
 	 * internal error.
 	 */
@@ -686,8 +708,8 @@ struct hwrm_err_output {
 #define HWRM_VERSION_MINOR 9
 #define HWRM_VERSION_UPDATE 2
 /* non-zero means beta version */
-#define HWRM_VERSION_RSVD 9
-#define HWRM_VERSION_STR "1.9.2.9"
+#define HWRM_VERSION_RSVD 53
+#define HWRM_VERSION_STR "1.9.2.53"
 
 /****************
  * hwrm_ver_get *
@@ -902,6 +924,42 @@ struct hwrm_ver_get_output {
 	#define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_REQUIRED \
 		UINT32_C(0x8)
 	/*
+	 * If set to 1, then the KONG host mailbox channel is supported.
+	 * If set to 0, then the KONG host mailbox channel is not supported.
+	 * By default, this flag should be 0 for older version of core firmware.
+	 */
+	#define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED \
+		UINT32_C(0x10)
+	/*
+	 * If set to 1, then the 64bit flow handle is supported in addition to the
+	 * legacy 16bit flow handle. If set to 0, then the 64bit flow handle is not
+	 * supported. By default, this flag should be 0 for older version of core firmware.
+	 */
+	#define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED \
+		UINT32_C(0x20)
+	/*
+	 * If set to 1, then filter type can be provided in filter_alloc or filter_cfg
+	 * filter types like L2 for l2 traffic and ROCE for roce & l2 traffic.
+	 * If set to 0, then filter types not supported.
+	 * By default, this flag should be 0 for older version of core firmware.
+	 */
+	#define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_L2_FILTER_TYPES_ROCE_OR_L2_SUPPORTED \
+		UINT32_C(0x40)
+	/*
+	 * If set to 1, firmware is capable to support virtio vSwitch offload model.
+	 * If set to 0, firmware can't supported virtio vSwitch offload model.
+	 * By default, this flag should be 0 for older version of core firmware.
+	 */
+	#define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_VIRTIO_VSWITCH_OFFLOAD_SUPPORTED \
+		UINT32_C(0x80)
+	/*
+	 * If set to 1, firmware is capable to support trusted VF.
+	 * If set to 0, firmware is not capable to support trusted VF.
+	 * By default, this flag should be 0 for older version of core firmware.
+	 */
+	#define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED \
+		UINT32_C(0x100)
+	/*
 	 * This field represents the major version of RoCE firmware.
 	 * A change in major version represents a major release.
 	 */
@@ -1154,39 +1212,45 @@ struct hwrm_ver_get_output {
 struct bd_base {
 	uint8_t	type;
 	/* This value identifies the type of buffer descriptor. */
-	#define BD_BASE_TYPE_MASK       UINT32_C(0x3f)
-	#define BD_BASE_TYPE_SFT        0
+	#define BD_BASE_TYPE_MASK             UINT32_C(0x3f)
+	#define BD_BASE_TYPE_SFT              0
 	/*
 	 * Indicates that this BD is 16B long and is used for
 	 * normal L2 packet transmission.
 	 */
-	#define BD_BASE_TYPE_TX_BD_SHORT  UINT32_C(0x0)
+	#define BD_BASE_TYPE_TX_BD_SHORT        UINT32_C(0x0)
 	/*
 	 * Indicates that this BD is 1BB long and is an empty
 	 * TX BD.  Not valid for use by the driver.
 	 */
-	#define BD_BASE_TYPE_TX_BD_EMPTY  UINT32_C(0x1)
+	#define BD_BASE_TYPE_TX_BD_EMPTY        UINT32_C(0x1)
 	/*
 	 * Indicates that this BD is 16B long and is an RX Producer
 	 * (ie. empty) buffer descriptor.
 	 */
-	#define BD_BASE_TYPE_RX_PROD_PKT  UINT32_C(0x4)
+	#define BD_BASE_TYPE_RX_PROD_PKT        UINT32_C(0x4)
 	/*
 	 * Indicates that this BD is 16B long and is an RX
 	 * Producer Buffer BD.
 	 */
-	#define BD_BASE_TYPE_RX_PROD_BFR  UINT32_C(0x5)
+	#define BD_BASE_TYPE_RX_PROD_BFR        UINT32_C(0x5)
 	/*
 	 * Indicates that this BD is 16B long and is an
 	 * RX Producer Assembly Buffer Descriptor.
 	 */
-	#define BD_BASE_TYPE_RX_PROD_AGG  UINT32_C(0x6)
+	#define BD_BASE_TYPE_RX_PROD_AGG        UINT32_C(0x6)
 	/*
 	 * Indicates that this BD is 32B long and is used for
 	 * normal L2 packet transmission.
 	 */
-	#define BD_BASE_TYPE_TX_BD_LONG   UINT32_C(0x10)
-	#define BD_BASE_TYPE_LAST        BD_BASE_TYPE_TX_BD_LONG
+	#define BD_BASE_TYPE_TX_BD_LONG         UINT32_C(0x10)
+	/*
+	 * Indicates that this BD is 32B long and is used for
+	 * L2 packet transmission for small packets that require
+	 * low latency.
+	 */
+	#define BD_BASE_TYPE_TX_BD_LONG_INLINE  UINT32_C(0x11)
+	#define BD_BASE_TYPE_LAST              BD_BASE_TYPE_TX_BD_LONG_INLINE
 	uint8_t	unused_1[7];
 } __attribute__((packed));
 
@@ -1406,6 +1470,7 @@ struct tx_bd_long {
 	uint64_t	address;
 } __attribute__((packed));
 
+/* Last 16 bytes of tx_bd_long. */
 /* tx_bd_long_hi (size:128b/16B) */
 struct tx_bd_long_hi {
 	/*
@@ -1595,6 +1660,219 @@ struct tx_bd_long_hi {
 		TX_BD_LONG_CFA_META_KEY_VLAN_TAG
 } __attribute__((packed));
 
+/*
+ * This structure is used to inform the NIC of packet data that needs to be
+ * transmitted with additional processing that requires extra data such as
+ * VLAN insertion plus attached inline data. This BD type may be used to
+ * improve latency for small packets needing the additional extended features
+ * supported by long BDs.
+ */
+/* tx_bd_long_inline (size:256b/32B) */
+struct tx_bd_long_inline {
+	uint16_t	flags_type;
+	/* This value identifies the type of buffer descriptor. */
+	#define TX_BD_LONG_INLINE_TYPE_MASK             UINT32_C(0x3f)
+	#define TX_BD_LONG_INLINE_TYPE_SFT              0
+	/*
+	 * This type of BD is 32B long and is used for inline L2 packet
+	 * transmission.
+	 */
+	#define TX_BD_LONG_INLINE_TYPE_TX_BD_LONG_INLINE  UINT32_C(0x11)
+	#define TX_BD_LONG_INLINE_TYPE_LAST \
+		TX_BD_LONG_INLINE_TYPE_TX_BD_LONG_INLINE
+	/*
+	 * All bits in this field may be set on the first BD of a packet.
+	 * Only the packet_end bit may be set in non-first BDs.
+	 */
+	#define TX_BD_LONG_INLINE_FLAGS_MASK            UINT32_C(0xffc0)
+	#define TX_BD_LONG_INLINE_FLAGS_SFT             6
+	/*
+	 * If set to 1, the packet ends with the data in the buffer
+	 * pointed to by this descriptor.  This flag must be
+	 * valid on every BD.
+	 */
+	#define TX_BD_LONG_INLINE_FLAGS_PACKET_END       UINT32_C(0x40)
+	/*
+	 * If set to 1, the device will not generate a completion for
+	 * this transmit packet unless there is an error in its processing.
+	 * If this bit is set to 0, then the packet will be completed
+	 * normally.
+	 *
+	 * This bit may be set only on the first BD of a packet.
+	 */
+	#define TX_BD_LONG_INLINE_FLAGS_NO_CMPL          UINT32_C(0x80)
+	/*
+	 * This value indicates how many 16B BD locations are consumed
+	 * in the ring by this packet, including the BD and inline
+	 * data.
+	 */
+	#define TX_BD_LONG_INLINE_FLAGS_BD_CNT_MASK      UINT32_C(0x1f00)
+	#define TX_BD_LONG_INLINE_FLAGS_BD_CNT_SFT       8
+	/* This field is deprecated. */
+	#define TX_BD_LONG_INLINE_FLAGS_LHINT_MASK       UINT32_C(0x6000)
+	#define TX_BD_LONG_INLINE_FLAGS_LHINT_SFT        13
+	/*
+	 * If set to 1, the device immediately updates the Send Consumer
+	 * Index after the buffer associated with this descriptor has
+	 * been transferred via DMA to NIC memory from host memory. An
+	 * interrupt may or may not be generated according to the state
+	 * of the interrupt avoidance mechanisms. If this bit
+	 * is set to 0, then the Consumer Index is only updated as soon
+	 * as one of the host interrupt coalescing conditions has been met.
+	 *
+	 * This bit must be valid on the first BD of a packet.
+	 */
+	#define TX_BD_LONG_INLINE_FLAGS_COAL_NOW         UINT32_C(0x8000)
+	/*
+	 * This is the length of the inline data, not including BD length, in
+	 * bytes.
+	 * The maximum value is 480.
+	 *
+	 * This field must be valid on all BDs of a packet.
+	 */
+	uint16_t	len;
+	/*
+	 * The opaque data field is passed through to the completion and can be
+	 * used for any data that the driver wants to associate with the transmit
+	 * BD.
+	 *
+	 * This field must be valid on the first BD of a packet.
+	 */
+	uint32_t	opaque;
+	uint64_t	unused1;
+	/*
+	 * All bits in this field must be valid on the first BD of a packet.
+	 * Their value on other BDs of the packet is ignored.
+	 */
+	uint16_t	lflags;
+	/*
+	 * If set to 1, the controller replaces the TCP/UPD checksum
+	 * fields of normal TCP/UPD checksum, or the inner TCP/UDP
+	 * checksum field of the encapsulated TCP/UDP packets with the
+	 * hardware calculated TCP/UDP checksum for the packet associated
+	 * with this descriptor. The flag is ignored if the LSO flag is set.
+	 */
+	#define TX_BD_LONG_INLINE_LFLAGS_TCP_UDP_CHKSUM     UINT32_C(0x1)
+	/*
+	 * If set to 1, the controller replaces the IP checksum of the
+	 * normal packets, or the inner IP checksum of the encapsulated
+	 * packets with the hardware calculated IP checksum for the
+	 * packet associated with this descriptor.
+	 */
+	#define TX_BD_LONG_INLINE_LFLAGS_IP_CHKSUM          UINT32_C(0x2)
+	/*
+	 * If set to 1, the controller will not append an Ethernet CRC
+	 * to the end of the frame.
+	 *
+	 * Packet must be 64B or longer when this flag is set. It is not
+	 * useful to use this bit with any form of TX offload such as
+	 * CSO or LSO. The intent is that the packet from the host already
+	 * has a valid Ethernet CRC on the packet.
+	 */
+	#define TX_BD_LONG_INLINE_LFLAGS_NOCRC              UINT32_C(0x4)
+	/*
+	 * If set to 1, the device will record the time at which the packet
+	 * was actually transmitted at the TX MAC.
+	 */
+	#define TX_BD_LONG_INLINE_LFLAGS_STAMP              UINT32_C(0x8)
+	/*
+	 * If set to 1, the controller replaces the tunnel IP checksum
+	 * field with hardware calculated IP checksum for the IP header
+	 * of the packet associated with this descriptor. The hardware
+	 * updates an outer UDP checksum if it is non-zero.
+	 */
+	#define TX_BD_LONG_INLINE_LFLAGS_T_IP_CHKSUM        UINT32_C(0x10)
+	/*
+	 * This bit must be 0 for BDs of this type. LSO is not supported with
+	 * inline BDs.
+	 */
+	#define TX_BD_LONG_INLINE_LFLAGS_LSO                UINT32_C(0x20)
+	/* Since LSO is not supported with inline BDs, this bit is not used. */
+	#define TX_BD_LONG_INLINE_LFLAGS_IPID_FMT           UINT32_C(0x40)
+	/* Since LSO is not supported with inline BDs, this bit is not used. */
+	#define TX_BD_LONG_INLINE_LFLAGS_T_IPID             UINT32_C(0x80)
+	/*
+	 * If set to '1', then the RoCE ICRC will be appended to the
+	 * packet.  Packet must be a valid RoCE format packet.
+	 */
+	#define TX_BD_LONG_INLINE_LFLAGS_ROCE_CRC           UINT32_C(0x100)
+	/*
+	 * If set to '1', then the FCoE CRC will be appended to the
+	 * packet.  Packet must be a valid FCoE format packet.
+	 */
+	#define TX_BD_LONG_INLINE_LFLAGS_FCOE_CRC           UINT32_C(0x200)
+	uint16_t	unused2;
+	uint32_t	unused3;
+	uint16_t	unused4;
+	/*
+	 * This value selects a CFA action to perform on the packet.
+	 * Set this value to zero if no CFA action is desired.
+	 *
+	 * This value must be valid on the first BD of a packet.
+	 */
+	uint16_t	cfa_action;
+	/*
+	 * This value is action meta-data that defines CFA edit operations
+	 * that are done in addition to any action editing.
+	 */
+	uint32_t	cfa_meta;
+	/* When key = 1, this is the VLAN tag VID value. */
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_VID_MASK     UINT32_C(0xfff)
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_VID_SFT      0
+	/* When key = 1, this is the VLAN tag DE value. */
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_DE           UINT32_C(0x1000)
+	/* When key = 1, this is the VLAN tag PRI value. */
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_PRI_MASK     UINT32_C(0xe000)
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_PRI_SFT      13
+	/* When key = 1, this is the VLAN tag TPID select value. */
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_TPID_MASK    UINT32_C(0x70000)
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_TPID_SFT     16
+	/* 0x88a8 */
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_TPID_TPID88A8 \
+		(UINT32_C(0x0) << 16)
+	/* 0x8100 */
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_TPID_TPID8100 \
+		(UINT32_C(0x1) << 16)
+	/* 0x9100 */
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_TPID_TPID9100 \
+		(UINT32_C(0x2) << 16)
+	/* 0x9200 */
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_TPID_TPID9200 \
+		(UINT32_C(0x3) << 16)
+	/* 0x9300 */
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_TPID_TPID9300 \
+		(UINT32_C(0x4) << 16)
+	/* Value programmed in CFA VLANTPID register. */
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_TPID_TPIDCFG \
+		(UINT32_C(0x5) << 16)
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_TPID_LAST \
+		TX_BD_LONG_INLINE_CFA_META_VLAN_TPID_TPIDCFG
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_RESERVED_MASK \
+		UINT32_C(0xff80000)
+	#define TX_BD_LONG_INLINE_CFA_META_VLAN_RESERVED_SFT 19
+	/*
+	 * This field identifies the type of edit to be performed
+	 * on the packet.
+	 *
+	 * This value must be valid on the first BD of a packet.
+	 */
+	#define TX_BD_LONG_INLINE_CFA_META_KEY_MASK \
+		UINT32_C(0xf0000000)
+	#define TX_BD_LONG_INLINE_CFA_META_KEY_SFT           28
+	/* No editing */
+	#define TX_BD_LONG_INLINE_CFA_META_KEY_NONE \
+		(UINT32_C(0x0) << 28)
+	/*
+	 * - meta[17:16] - TPID select value (0 = 0x8100).
+	 * - meta[15:12] - PRI/DE value.
+	 * - meta[11:0] - VID value.
+	 */
+	#define TX_BD_LONG_INLINE_CFA_META_KEY_VLAN_TAG \
+		(UINT32_C(0x1) << 28)
+	#define TX_BD_LONG_INLINE_CFA_META_KEY_LAST \
+		TX_BD_LONG_INLINE_CFA_META_KEY_VLAN_TAG
+} __attribute__((packed));
+
 /* tx_bd_empty (size:128b/16B) */
 struct tx_bd_empty {
 	/* This value identifies the type of buffer descriptor. */
@@ -2121,6 +2399,7 @@ struct rx_pkt_cmpl {
 	uint32_t	rss_hash;
 } __attribute__((packed));
 
+/* Last 16 bytes of rx_pkt_cmpl. */
 /* rx_pkt_cmpl_hi (size:128b/16B) */
 struct rx_pkt_cmpl_hi {
 	uint32_t	flags2;
@@ -2566,6 +2845,7 @@ struct rx_tpa_start_cmpl {
 	uint32_t	rss_hash;
 } __attribute__((packed));
 
+/* Last 16 bytes of rx_tpq_start_cmpl. */
 /* rx_tpa_start_cmpl_hi (size:128b/16B) */
 struct rx_tpa_start_cmpl_hi {
 	uint32_t	flags2;
@@ -2830,6 +3110,7 @@ struct rx_tpa_end_cmpl {
 	uint32_t	tsdelta;
 } __attribute__((packed));
 
+/* Last 16 bytes of rx_tpa_end_cmpl. */
 /* rx_tpa_end_cmpl_hi (size:128b/16B) */
 struct rx_tpa_end_cmpl_hi {
 	/*
@@ -3153,6 +3434,9 @@ struct hwrm_async_event_cmpl {
 	/* Port PHY configuration change */
 	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE \
 		UINT32_C(0x7)
+	/* Reset notification to clients */
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY \
+		UINT32_C(0x8)
 	/* Function driver unloaded */
 	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD \
 		UINT32_C(0x10)
@@ -3790,6 +4074,96 @@ struct hwrm_async_event_cmpl_port_phy_cfg_change {
 		UINT32_C(0x40000)
 } __attribute__((packed));
 
+/* hwrm_async_event_cmpl_reset_notify (size:128b/16B) */
+struct hwrm_async_event_cmpl_reset_notify {
+	uint16_t	type;
+	/*
+	 * This field indicates the exact type of the completion.
+	 * By convention, the LSB identifies the length of the
+	 * record in 16B units.  Even values indicate 16B
+	 * records.  Odd values indicate 32B
+	 * records.
+	 */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_TYPE_MASK \
+		UINT32_C(0x3f)
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_TYPE_SFT             0
+	/* HWRM Asynchronous Event Information */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_TYPE_HWRM_ASYNC_EVENT \
+		UINT32_C(0x2e)
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_TYPE_LAST \
+		HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_TYPE_HWRM_ASYNC_EVENT
+	/* Identifiers of events. */
+	uint16_t	event_id;
+	/* Notify clients of imminent reset. */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_ID_RESET_NOTIFY \
+		UINT32_C(0x8)
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_ID_LAST \
+		HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_ID_RESET_NOTIFY
+	/* Event specific data */
+	uint32_t	event_data2;
+	uint8_t	opaque_v;
+	/*
+	 * This value is written by the NIC such that it will be different
+	 * for each pass through the completion queue.   The even passes
+	 * will write 1.  The odd passes will write 0.
+	 */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_V          UINT32_C(0x1)
+	/* opaque is 7 b */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_OPAQUE_MASK UINT32_C(0xfe)
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_OPAQUE_SFT 1
+	/* 8-lsb timestamp from POR (100-msec resolution) */
+	uint8_t	timestamp_lo;
+	/* 16-lsb timestamp from POR (100-msec resolution) */
+	uint16_t	timestamp_hi;
+	/* Event specific data */
+	uint32_t	event_data1;
+	/* Indicates driver action requested */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_MASK \
+		UINT32_C(0xff)
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_SFT \
+		0
+	/*
+	 * If set to 1, it indicates that the l2 client should
+	 * stop sending in band traffic to Nitro.
+	 * if set to 0, there is no change in L2 client behavior.
+	 */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_DRIVER_STOP_TX_QUEUE \
+		UINT32_C(0x1)
+	/*
+	 * If set to 1, it indicates that the L2 client should
+	 * bring down the interface.
+	 * If set to 0, then there is no change in L2 client behavior.
+	 */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_DRIVER_IFDOWN \
+		UINT32_C(0x2)
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_LAST \
+		HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DRIVER_ACTION_DRIVER_IFDOWN
+	/* Indicates reason for reset. */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_MASK \
+		UINT32_C(0xff00)
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_SFT \
+		8
+	/* A management client has requested reset. */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_MANAGEMENT_RESET_REQUEST \
+		(UINT32_C(0x1) << 8)
+	/* A fatal firmware exception has occurred. */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_EXCEPTION_FATAL \
+		(UINT32_C(0x2) << 8)
+	/* A non-fatal firmware exception has occurred. */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_EXCEPTION_NON_FATAL \
+		(UINT32_C(0x3) << 8)
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_LAST \
+		HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_EXCEPTION_NON_FATAL
+	/*
+	 * Minimum time before driver should attempt access - units 100ms ticks.
+	 * Range 0-65535
+	 */
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DELAY_IN_100MS_TICKS_MASK \
+		UINT32_C(0xffff0000)
+	#define HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_DELAY_IN_100MS_TICKS_SFT \
+		16
+} __attribute__((packed));
+
 /* hwrm_async_event_cmpl_func_drvr_unload (size:128b/16B) */
 struct hwrm_async_event_cmpl_func_drvr_unload {
 	uint16_t	type;
@@ -4285,6 +4659,13 @@ struct hwrm_async_event_cmpl_vf_cfg_change {
 	 */
 	#define HWRM_ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_DFLT_VLAN_CHANGE \
 		UINT32_C(0x8)
+	/*
+	 * If this bit is set to 1, then the value of trusted VF enable
+	 * was changed on this VF.
+	 * If set to 0, then this bit should be ignored.
+	 */
+	#define HWRM_ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_TRUSTED_VF_CFG_CHANGE \
+		UINT32_C(0x10)
 } __attribute__((packed));
 
 /* hwrm_async_event_cmpl_llfc_pfc_change (size:128b/16B) */
@@ -5306,6 +5687,20 @@ struct hwrm_func_qcaps_output {
 	#define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ADMIN_PF_SUPPORTED \
 		UINT32_C(0x40000)
 	/*
+	 * If the query is for a VF, then this flag shall be ignored.
+	 * If this query is for a PF and this flag is set to 1, then
+	 * the PF will know that the firmware has the capability to track
+	 * the virtual link status.
+	 */
+	#define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_LINK_ADMIN_STATUS_SUPPORTED \
+		UINT32_C(0x80000)
+	/*
+	 * If 1, then this function supports the push mode that uses
+	 * write combine buffers and the long inline tx buffer descriptor.
+	 */
+	#define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WCB_PUSH_MODE \
+		UINT32_C(0x100000)
+	/*
 	 * This value is current MAC address configured for this
 	 * function. A value of 00-00-00-00-00-00 indicates no
 	 * MAC address is currently configured.
@@ -5548,6 +5943,15 @@ struct hwrm_func_qcfg_output {
 	#define HWRM_FUNC_QCFG_OUTPUT_FLAGS_MULTI_HOST \
 		UINT32_C(0x20)
 	/*
+	 * If the function that is being queried is a PF, then the HWRM shall
+	 * set this field to 0 and the HWRM client shall ignore this field.
+	 * If the function that is being queried is a VF, then the HWRM shall
+	 * set this field to 1 if the queried VF is trusted, otherwise the HWRM
+	 * shall set this field to 0.
+	 */
+	#define HWRM_FUNC_QCFG_OUTPUT_FLAGS_TRUSTED_VF \
+		UINT32_C(0x40)
+	/*
 	 * This value is current MAC address configured for this
 	 * function. A value of 00-00-00-00-00-00 indicates no
 	 * MAC address is currently configured.
@@ -5755,7 +6159,7 @@ struct hwrm_func_qcfg_output {
 	 */
 	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_CACHE_LINESIZE_MASK \
 		UINT32_C(0x3)
-	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_CACHE_LINESIZE_SFT     0
+	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_CACHE_LINESIZE_SFT          0
 	/* Cache Line Size 64 bytes */
 	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_CACHE_LINESIZE_SIZE_64 \
 		UINT32_C(0x0)
@@ -5764,10 +6168,25 @@ struct hwrm_func_qcfg_output {
 		UINT32_C(0x1)
 	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_CACHE_LINESIZE_LAST \
 		HWRM_FUNC_QCFG_OUTPUT_OPTIONS_CACHE_LINESIZE_SIZE_128
+	/* This value is the virtual link admin state setting. */
+	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_LINK_ADMIN_STATE_MASK \
+		UINT32_C(0xc)
+	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_LINK_ADMIN_STATE_SFT        2
+	/* Admin link state is in forced down mode. */
+	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_LINK_ADMIN_STATE_FORCED_DOWN \
+		(UINT32_C(0x0) << 2)
+	/* Admin link state is in forced up mode. */
+	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_LINK_ADMIN_STATE_FORCED_UP \
+		(UINT32_C(0x1) << 2)
+	/* Admin link state is in auto mode  - follows the physical link state. */
+	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_LINK_ADMIN_STATE_AUTO \
+		(UINT32_C(0x2) << 2)
+	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_LINK_ADMIN_STATE_LAST \
+		HWRM_FUNC_QCFG_OUTPUT_OPTIONS_LINK_ADMIN_STATE_AUTO
 	/* Reserved for future. */
 	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_RSVD_MASK \
-		UINT32_C(0xfc)
-	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_RSVD_SFT               2
+		UINT32_C(0xf0)
+	#define HWRM_FUNC_QCFG_OUTPUT_OPTIONS_RSVD_SFT                    4
 	/*
 	 * The number of VFs that are allocated to the function.
 	 * This is valid only on the PF with SR-IOV enabled.
@@ -5814,217 +6233,6 @@ struct hwrm_func_qcfg_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/***********************
- * hwrm_func_vlan_qcfg *
- ***********************/
-
-
-/* hwrm_func_vlan_qcfg_input (size:192b/24B) */
-struct hwrm_func_vlan_qcfg_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/*
-	 * Function ID of the function that is being
-	 * configured.
-	 * If set to 0xFF... (All Fs), then the configuration is
-	 * for the requesting function.
-	 */
-	uint16_t	fid;
-	uint8_t	unused_0[6];
-} __attribute__((packed));
-
-/* hwrm_func_vlan_qcfg_output (size:320b/40B) */
-struct hwrm_func_vlan_qcfg_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-	/* S-TAG VLAN identifier configured for the function. */
-	uint16_t	stag_vid;
-	/* S-TAG PCP value configured for the function. */
-	uint8_t	stag_pcp;
-	uint8_t	unused_1;
-	/*
-	 * S-TAG TPID value configured for the function. This field is specified in
-	 * network byte order.
-	 */
-	uint16_t	stag_tpid;
-	/* C-TAG VLAN identifier configured for the function. */
-	uint16_t	ctag_vid;
-	/* C-TAG PCP value configured for the function. */
-	uint8_t	ctag_pcp;
-	uint8_t	unused_2;
-	/*
-	 * C-TAG TPID value configured for the function. This field is specified in
-	 * network byte order.
-	 */
-	uint16_t	ctag_tpid;
-	/* Future use. */
-	uint32_t	rsvd2;
-	/* Future use. */
-	uint32_t	rsvd3;
-	uint32_t	unused_3;
-} __attribute__((packed));
-
-/**********************
- * hwrm_func_vlan_cfg *
- **********************/
-
-
-/* hwrm_func_vlan_cfg_input (size:384b/48B) */
-struct hwrm_func_vlan_cfg_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/*
-	 * Function ID of the function that is being
-	 * configured.
-	 * If set to 0xFF... (All Fs), then the configuration is
-	 * for the requesting function.
-	 */
-	uint16_t	fid;
-	uint8_t	unused_0[2];
-	uint32_t	enables;
-	/*
-	 * This bit must be '1' for the stag_vid field to be
-	 * configured.
-	 */
-	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_VID      UINT32_C(0x1)
-	/*
-	 * This bit must be '1' for the ctag_vid field to be
-	 * configured.
-	 */
-	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_VID      UINT32_C(0x2)
-	/*
-	 * This bit must be '1' for the stag_pcp field to be
-	 * configured.
-	 */
-	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_PCP      UINT32_C(0x4)
-	/*
-	 * This bit must be '1' for the ctag_pcp field to be
-	 * configured.
-	 */
-	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_PCP      UINT32_C(0x8)
-	/*
-	 * This bit must be '1' for the stag_tpid field to be
-	 * configured.
-	 */
-	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_TPID     UINT32_C(0x10)
-	/*
-	 * This bit must be '1' for the ctag_tpid field to be
-	 * configured.
-	 */
-	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_TPID     UINT32_C(0x20)
-	/* S-TAG VLAN identifier configured for the function. */
-	uint16_t	stag_vid;
-	/* S-TAG PCP value configured for the function. */
-	uint8_t	stag_pcp;
-	uint8_t	unused_1;
-	/*
-	 * S-TAG TPID value configured for the function. This field is specified in
-	 * network byte order.
-	 */
-	uint16_t	stag_tpid;
-	/* C-TAG VLAN identifier configured for the function. */
-	uint16_t	ctag_vid;
-	/* C-TAG PCP value configured for the function. */
-	uint8_t	ctag_pcp;
-	uint8_t	unused_2;
-	/*
-	 * C-TAG TPID value configured for the function. This field is specified in
-	 * network byte order.
-	 */
-	uint16_t	ctag_tpid;
-	/* Future use. */
-	uint32_t	rsvd1;
-	/* Future use. */
-	uint32_t	rsvd2;
-	uint8_t	unused_3[4];
-} __attribute__((packed));
-
-/* hwrm_func_vlan_cfg_output (size:128b/16B) */
-struct hwrm_func_vlan_cfg_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
 /*****************
  * hwrm_func_cfg *
  *****************/
@@ -6205,6 +6413,17 @@ struct hwrm_func_cfg_input {
 	 */
 	#define HWRM_FUNC_CFG_INPUT_FLAGS_L2_CTX_ASSETS_TEST \
 		UINT32_C(0x100000)
+	/*
+	 * This configuration change can be initiated by a PF driver. This
+	 * configuration request shall be targeted to a VF. From local host
+	 * resident HWRM clients, only the parent PF driver shall be allowed
+	 * to initiate this change on one of its children VFs. If this bit is
+	 * set to 1, then the VF that is being configured is requested to be
+	 * trusted. If this bit is set to 0, then the VF that is being configured
+	 * is requested to be not trusted.
+	 */
+	#define HWRM_FUNC_CFG_INPUT_FLAGS_TRUSTED_VF_ENABLE \
+		UINT32_C(0x200000)
 	uint32_t	enables;
 	/*
 	 * This bit must be '1' for the mtu field to be
@@ -6339,6 +6558,12 @@ struct hwrm_func_cfg_input {
 	#define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_MSIX \
 		UINT32_C(0x200000)
 	/*
+	 * This bit must be '1' for the link admin state field to be
+	 * configured.
+	 */
+	#define HWRM_FUNC_CFG_INPUT_ENABLES_ADMIN_LINK_STATE \
+		UINT32_C(0x400000)
+	/*
 	 * The maximum transmission unit of the function.
 	 * The HWRM should make sure that the mtu of
 	 * the function does not exceed the mtu of the physical
@@ -6569,7 +6794,7 @@ struct hwrm_func_cfg_input {
 	 */
 	#define HWRM_FUNC_CFG_INPUT_OPTIONS_CACHE_LINESIZE_MASK \
 		UINT32_C(0x3)
-	#define HWRM_FUNC_CFG_INPUT_OPTIONS_CACHE_LINESIZE_SFT     0
+	#define HWRM_FUNC_CFG_INPUT_OPTIONS_CACHE_LINESIZE_SFT          0
 	/* Cache Line Size 64 bytes */
 	#define HWRM_FUNC_CFG_INPUT_OPTIONS_CACHE_LINESIZE_SIZE_64 \
 		UINT32_C(0x0)
@@ -6578,10 +6803,25 @@ struct hwrm_func_cfg_input {
 		UINT32_C(0x1)
 	#define HWRM_FUNC_CFG_INPUT_OPTIONS_CACHE_LINESIZE_LAST \
 		HWRM_FUNC_CFG_INPUT_OPTIONS_CACHE_LINESIZE_SIZE_128
+	/* This value is the virtual link admin state setting. */
+	#define HWRM_FUNC_CFG_INPUT_OPTIONS_LINK_ADMIN_STATE_MASK \
+		UINT32_C(0xc)
+	#define HWRM_FUNC_CFG_INPUT_OPTIONS_LINK_ADMIN_STATE_SFT        2
+	/* Admin state is forced down. */
+	#define HWRM_FUNC_CFG_INPUT_OPTIONS_LINK_ADMIN_STATE_FORCED_DOWN \
+		(UINT32_C(0x0) << 2)
+	/* Admin state is forced up. */
+	#define HWRM_FUNC_CFG_INPUT_OPTIONS_LINK_ADMIN_STATE_FORCED_UP \
+		(UINT32_C(0x1) << 2)
+	/* Admin state is in auto mode - is to follow the physical link state. */
+	#define HWRM_FUNC_CFG_INPUT_OPTIONS_LINK_ADMIN_STATE_AUTO \
+		(UINT32_C(0x2) << 2)
+	#define HWRM_FUNC_CFG_INPUT_OPTIONS_LINK_ADMIN_STATE_LAST \
+		HWRM_FUNC_CFG_INPUT_OPTIONS_LINK_ADMIN_STATE_AUTO
 	/* Reserved for future. */
 	#define HWRM_FUNC_CFG_INPUT_OPTIONS_RSVD_MASK \
-		UINT32_C(0xfc)
-	#define HWRM_FUNC_CFG_INPUT_OPTIONS_RSVD_SFT               2
+		UINT32_C(0xf0)
+	#define HWRM_FUNC_CFG_INPUT_OPTIONS_RSVD_SFT                    4
 	/*
 	 * The number of multicast filters that should
 	 * be reserved for this function on the RX side.
@@ -6862,79 +7102,6 @@ struct hwrm_func_vf_resc_free_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/*******************************
- * hwrm_func_vf_vnic_ids_query *
- *******************************/
-
-
-/* hwrm_func_vf_vnic_ids_query_input (size:256b/32B) */
-struct hwrm_func_vf_vnic_ids_query_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/*
-	 * This value is used to identify a Virtual Function (VF).
-	 * The scope of VF ID is local within a PF.
-	 */
-	uint16_t	vf_id;
-	uint8_t	unused_0[2];
-	/* Max number of vnic ids in vnic id table */
-	uint32_t	max_vnic_id_cnt;
-	/* This is the address for VF VNIC ID table */
-	uint64_t	vnic_id_tbl_addr;
-} __attribute__((packed));
-
-/* hwrm_func_vf_vnic_ids_query_output (size:128b/16B) */
-struct hwrm_func_vf_vnic_ids_query_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	/*
-	 * Actual number of vnic ids
-	 *
-	 * Each VNIC ID is written as a 32-bit number.
-	 */
-	uint32_t	vnic_id_cnt;
-	uint8_t	unused_0[3];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
 /**********************
  * hwrm_func_drv_rgtr *
  **********************/
@@ -6978,7 +7145,8 @@ struct hwrm_func_drv_rgtr_input {
 	 * If a VF driver sets this flag, it should be ignored
 	 * by the HWRM.
 	 */
-	#define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FWD_ALL_MODE       UINT32_C(0x1)
+	#define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FWD_ALL_MODE \
+		UINT32_C(0x1)
 	/*
 	 * When this bit is '1', the function is requesting none of
 	 * the requests from its children VF drivers to be
@@ -6987,7 +7155,8 @@ struct hwrm_func_drv_rgtr_input {
 	 * If a VF driver sets this flag, it should be ignored
 	 * by the HWRM.
 	 */
-	#define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FWD_NONE_MODE      UINT32_C(0x2)
+	#define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FWD_NONE_MODE \
+		UINT32_C(0x2)
 	/*
 	 * When this bit is '1', then ver_maj_8b, ver_min_8b, ver_upd_8b
 	 * fields shall be ignored and ver_maj, ver_min, ver_upd
@@ -6996,7 +7165,22 @@ struct hwrm_func_drv_rgtr_input {
 	 * fields shall be used for the driver version information and
 	 * ver_maj, ver_min, ver_upd and ver_patch shall be ignored.
 	 */
-	#define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_16BIT_VER_MODE     UINT32_C(0x4)
+	#define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_16BIT_VER_MODE \
+		UINT32_C(0x4)
+	/*
+	 * When this bit is '1', the function is indicating support of
+	 * 64bit flow handle.  The firmware that only supports 64bit flow
+	 * handle should check this bit before allowing processing of
+	 * HWRM_CFA_FLOW_XXX commands from the requesting function as firmware
+	 * with 64bit flow handle support can only be compatible with drivers
+	 * that support 64bit flow handle. The legacy drivers that don't support
+	 * 64bit flow handle won't be able to use HWRM_CFA_FLOW_XXX commands when
+	 * running with new firmware that only supports 64bit flow handle. The new
+	 * firmware support 64bit flow handle returns HWRM_ERR_CODE_CMD_NOT_SUPPORTED
+	 * status to the legacy driver when encounters these commands.
+	 */
+	#define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FLOW_HANDLE_64BIT_MODE \
+		UINT32_C(0x8)
 	uint32_t	enables;
 	/*
 	 * This bit must be '1' for the os_type field to be
@@ -7117,7 +7301,14 @@ struct hwrm_func_drv_rgtr_output {
 	uint16_t	seq_id;
 	/* The length of the response data in number of bytes. */
 	uint16_t	resp_len;
-	uint8_t	unused_0[7];
+	uint32_t	flags;
+	/*
+	 * When this bit is '1', it indicates that the
+	 * HWRM_FUNC_DRV_IF_CHANGE call is supported.
+	 */
+	#define HWRM_FUNC_DRV_RGTR_OUTPUT_FLAGS_IF_CHANGE_SUPPORTED \
+		UINT32_C(0x1)
+	uint8_t	unused_0[3];
 	/*
 	 * This field is used in Output records to indicate that the output
 	 * is completely written to RAM.  This field should be read as '1'
@@ -7441,7 +7632,7 @@ struct hwrm_func_drv_qver_input {
 	uint8_t	unused_0[2];
 } __attribute__((packed));
 
-/* hwrm_func_drv_qver_output (size:192b/24B) */
+/* hwrm_func_drv_qver_output (size:256b/32B) */
 struct hwrm_func_drv_qver_output {
 	/* The specific error status for the command. */
 	uint16_t	error_code;
@@ -7483,15 +7674,7 @@ struct hwrm_func_drv_qver_output {
 	uint8_t	ver_min_8b;
 	/* This is the 8bit update version of the driver. */
 	uint8_t	ver_upd_8b;
-	uint8_t	unused_0[2];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
+	uint8_t	unused_0[3];
 	/* This is the 16bit major version of the driver. */
 	uint16_t	ver_maj;
 	/* This is the 16bit minor version of the driver. */
@@ -7500,6 +7683,15 @@ struct hwrm_func_drv_qver_output {
 	uint16_t	ver_upd;
 	/* This is the 16bit patch version of the driver. */
 	uint16_t	ver_patch;
+	uint8_t	unused_1[7];
+	/*
+	 * This field is used in Output records to indicate that the output
+	 * is completely written to RAM.  This field should be read as '1'
+	 * to indicate that the output has been completely written.
+	 * When writing a command completion or response to an internal processor,
+	 * the order of writes has to be such that this field is written last.
+	 */
+	uint8_t	valid;
 } __attribute__((packed));
 
 /****************************
@@ -7612,117 +7804,15 @@ struct hwrm_func_resource_qcaps_output {
 	 * The number of TX rings assigned to the function cannot exceed this value.
 	 */
 	uint16_t	max_tx_scheduler_inputs;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/*****************************
- * hwrm_func_vf_resource_cfg *
- *****************************/
-
-
-/* hwrm_func_vf_resource_cfg_input (size:448b/56B) */
-struct hwrm_func_vf_resource_cfg_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
+	uint16_t	flags;
 	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
+	 * When this bit is '1', it indicates that VF_RESOURCE_CFG supports
+	 * feature to reserve all minimum resources when minimum >= 1, otherwise
+	 * returns an error.
 	 */
-	uint64_t	resp_addr;
-	/* VF ID that is being configured by PF */
-	uint16_t	vf_id;
-	/* Maximum guaranteed number of MSI-X vectors for the function */
-	uint16_t	max_msix;
-	/* Minimum guaranteed number of RSS/COS contexts */
-	uint16_t	min_rsscos_ctx;
-	/* Maximum non-guaranteed number of RSS/COS contexts */
-	uint16_t	max_rsscos_ctx;
-	/* Minimum guaranteed number of completion rings */
-	uint16_t	min_cmpl_rings;
-	/* Maximum non-guaranteed number of completion rings */
-	uint16_t	max_cmpl_rings;
-	/* Minimum guaranteed number of transmit rings */
-	uint16_t	min_tx_rings;
-	/* Maximum non-guaranteed number of transmit rings */
-	uint16_t	max_tx_rings;
-	/* Minimum guaranteed number of receive rings */
-	uint16_t	min_rx_rings;
-	/* Maximum non-guaranteed number of receive rings */
-	uint16_t	max_rx_rings;
-	/* Minimum guaranteed number of L2 contexts */
-	uint16_t	min_l2_ctxs;
-	/* Maximum non-guaranteed number of L2 contexts */
-	uint16_t	max_l2_ctxs;
-	/* Minimum guaranteed number of VNICs */
-	uint16_t	min_vnics;
-	/* Maximum non-guaranteed number of VNICs */
-	uint16_t	max_vnics;
-	/* Minimum guaranteed number of statistic contexts */
-	uint16_t	min_stat_ctx;
-	/* Maximum non-guaranteed number of statistic contexts */
-	uint16_t	max_stat_ctx;
-	/* Minimum guaranteed number of ring groups */
-	uint16_t	min_hw_ring_grps;
-	/* Maximum non-guaranteed number of ring groups */
-	uint16_t	max_hw_ring_grps;
-	uint8_t	unused_0[4];
-} __attribute__((packed));
-
-/* hwrm_func_vf_resource_cfg_output (size:256b/32B) */
-struct hwrm_func_vf_resource_cfg_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	/* Reserved number of RSS/COS contexts */
-	uint16_t	reserved_rsscos_ctx;
-	/* Reserved number of completion rings */
-	uint16_t	reserved_cmpl_rings;
-	/* Reserved number of transmit rings */
-	uint16_t	reserved_tx_rings;
-	/* Reserved number of receive rings */
-	uint16_t	reserved_rx_rings;
-	/* Reserved number of L2 contexts */
-	uint16_t	reserved_l2_ctxs;
-	/* Reserved number of VNICs */
-	uint16_t	reserved_vnics;
-	/* Reserved number of statistic contexts */
-	uint16_t	reserved_stat_ctx;
-	/* Reserved number of ring groups */
-	uint16_t	reserved_hw_ring_grps;
-	uint8_t	unused_0[7];
+	#define HWRM_FUNC_RESOURCE_QCAPS_OUTPUT_FLAGS_MIN_GUARANTEED \
+		UINT32_C(0x1)
+	uint8_t	unused_0[5];
 	/*
 	 * This field is used in Output records to indicate that the output
 	 * is completely written to RAM.  This field should be read as '1'
@@ -7769,7 +7859,7 @@ struct hwrm_func_backing_store_qcaps_input {
 	uint64_t	resp_addr;
 } __attribute__((packed));
 
-/* hwrm_func_backing_store_qcaps_output (size:512b/64B) */
+/* hwrm_func_backing_store_qcaps_output (size:576b/72B) */
 struct hwrm_func_backing_store_qcaps_output {
 	/* The specific error status for the command. */
 	uint16_t	error_code;
@@ -7813,19 +7903,51 @@ struct hwrm_func_backing_store_qcaps_output {
 	uint32_t	stat_max_entries;
 	/* Number of bytes that must be allocated for each context entry. */
 	uint16_t	stat_entry_size;
-	/* Maximum number of TQM context entries supported per ring. */
-	uint16_t	tqm_max_entries_per_ring;
 	/* Number of bytes that must be allocated for each context entry. */
 	uint16_t	tqm_entry_size;
-	/* Number of bytes that must be allocated for each context entry. */
-	uint16_t	mrav_entry_size;
+	/* Minimum number of TQM context entries required per ring. */
+	uint32_t	tqm_min_entries_per_ring;
+	/*
+	 * Maximum number of TQM context entries supported per ring. This is
+	 * actually a recommended TQM queue size based on worst case usage of
+	 * the TQM queue.
+	 *
+	 * TQM fastpath rings should be sized large enough to accommodate the
+	 * maximum number of QPs (either L2 or RoCE, or both if shared)
+	 * that can be enqueued to the TQM ring.
+	 *
+	 * TQM slowpath rings should be sized as follows:
+	 *
+	 * num_entries = num_vnics + num_l2_tx_rings + num_roce_qps + tqm_min_size
+	 *
+	 * Where:
+	 *   num_vnics is the number of VNICs allocated in the VNIC backing store
+	 *   num_l2_tx_rings is the number of L2 rings in the QP backing store
+	 *   num_roce_qps is the number of RoCE QPs in the QP backing store
+	 *   tqm_min_size is tqm_min_entries_per_ring reported by
+	 *     HWRM_FUNC_BACKING_STORE_QCAPS
+	 *
+	 * Note that TQM ring sizes cannot be extended while the system is
+	 * operational. If a PF driver needs to extend a TQM ring, it needs
+	 * to reset the function (e.g. HWRM_FUNC_RESET) and then reallocate
+	 * the backing store.
+	 */
+	uint32_t	tqm_max_entries_per_ring;
 	/* Maximum number of MR/AV context entries supported for this function. */
 	uint32_t	mrav_max_entries;
-	/* Maximum number of Timer context entries supported for this function. */
-	uint32_t	tim_max_entries;
+	/* Number of bytes that must be allocated for each context entry. */
+	uint16_t	mrav_entry_size;
 	/* Number of bytes that must be allocated for each context entry. */
 	uint16_t	tim_entry_size;
-	uint8_t	unused_0;
+	/* Maximum number of Timer context entries supported for this function. */
+	uint32_t	tim_max_entries;
+	uint8_t	unused_0[2];
+	/*
+	 * The number of entries specified for any TQM ring must be a
+	 * multiple of this value to prevent any resource allocation
+	 * limitations.
+	 */
+	uint8_t	tqm_entries_multiple;
 	/*
 	 * This field is used in Output records to indicate that the output
 	 * is completely written to RAM.  This field should be read as '1'
@@ -8672,23 +8794,129 @@ struct hwrm_func_backing_store_cfg_input {
 	uint32_t	cq_num_entries;
 	/* Number of Stats. */
 	uint32_t	stat_num_entries;
-	/* Number of TQM slowpath entries. */
+	/*
+	 * Number of TQM slowpath entries.
+	 *
+	 * TQM slowpath rings should be sized as follows:
+	 *
+	 * num_entries = num_vnics + num_l2_tx_rings + num_roce_qps + tqm_min_size
+	 *
+	 * Where:
+	 *   num_vnics is the number of VNICs allocated in the VNIC backing store
+	 *   num_l2_tx_rings is the number of L2 rings in the QP backing store
+	 *   num_roce_qps is the number of RoCE QPs in the QP backing store
+	 *   tqm_min_size is tqm_min_entries_per_ring reported by
+	 *     HWRM_FUNC_BACKING_STORE_QCAPS
+	 *
+	 * Note that TQM ring sizes cannot be extended while the system is
+	 * operational. If a PF driver needs to extend a TQM ring, it needs
+	 * to reset the function (e.g. HWRM_FUNC_RESET) and then reallocate
+	 * the backing store.
+	 */
 	uint32_t	tqm_sp_num_entries;
-	/* Number of TQM ring 0 entries. */
+	/*
+	 * Number of TQM ring 0 entries.
+	 *
+	 * TQM fastpath rings should be sized large enough to accommodate the
+	 * maximum number of QPs (either L2 or RoCE, or both if shared)
+	 * that can be enqueued to the TQM ring.
+	 *
+	 * Note that TQM ring sizes cannot be extended while the system is
+	 * operational. If a PF driver needs to extend a TQM ring, it needs
+	 * to reset the function (e.g. HWRM_FUNC_RESET) and then reallocate
+	 * the backing store.
+	 */
 	uint32_t	tqm_ring0_num_entries;
-	/* Number of TQM ring 1 entries. */
+	/*
+	 * Number of TQM ring 1 entries.
+	 *
+	 * TQM fastpath rings should be sized large enough to accommodate the
+	 * maximum number of QPs (either L2 or RoCE, or both if shared)
+	 * that can be enqueued to the TQM ring.
+	 *
+	 * Note that TQM ring sizes cannot be extended while the system is
+	 * operational. If a PF driver needs to extend a TQM ring, it needs
+	 * to reset the function (e.g. HWRM_FUNC_RESET) and then reallocate
+	 * the backing store.
+	 */
 	uint32_t	tqm_ring1_num_entries;
-	/* Number of TQM ring 2 entries. */
+	/*
+	 * Number of TQM ring 2 entries.
+	 *
+	 * TQM fastpath rings should be sized large enough to accommodate the
+	 * maximum number of QPs (either L2 or RoCE, or both if shared)
+	 * that can be enqueued to the TQM ring.
+	 *
+	 * Note that TQM ring sizes cannot be extended while the system is
+	 * operational. If a PF driver needs to extend a TQM ring, it needs
+	 * to reset the function (e.g. HWRM_FUNC_RESET) and then reallocate
+	 * the backing store.
+	 */
 	uint32_t	tqm_ring2_num_entries;
-	/* Number of TQM ring 3 entries. */
+	/*
+	 * Number of TQM ring 3 entries.
+	 *
+	 * TQM fastpath rings should be sized large enough to accommodate the
+	 * maximum number of QPs (either L2 or RoCE, or both if shared)
+	 * that can be enqueued to the TQM ring.
+	 *
+	 * Note that TQM ring sizes cannot be extended while the system is
+	 * operational. If a PF driver needs to extend a TQM ring, it needs
+	 * to reset the function (e.g. HWRM_FUNC_RESET) and then reallocate
+	 * the backing store.
+	 */
 	uint32_t	tqm_ring3_num_entries;
-	/* Number of TQM ring 4 entries. */
+	/*
+	 * Number of TQM ring 4 entries.
+	 *
+	 * TQM fastpath rings should be sized large enough to accommodate the
+	 * maximum number of QPs (either L2 or RoCE, or both if shared)
+	 * that can be enqueued to the TQM ring.
+	 *
+	 * Note that TQM ring sizes cannot be extended while the system is
+	 * operational. If a PF driver needs to extend a TQM ring, it needs
+	 * to reset the function (e.g. HWRM_FUNC_RESET) and then reallocate
+	 * the backing store.
+	 */
 	uint32_t	tqm_ring4_num_entries;
-	/* Number of TQM ring 5 entries. */
+	/*
+	 * Number of TQM ring 5 entries.
+	 *
+	 * TQM fastpath rings should be sized large enough to accommodate the
+	 * maximum number of QPs (either L2 or RoCE, or both if shared)
+	 * that can be enqueued to the TQM ring.
+	 *
+	 * Note that TQM ring sizes cannot be extended while the system is
+	 * operational. If a PF driver needs to extend a TQM ring, it needs
+	 * to reset the function (e.g. HWRM_FUNC_RESET) and then reallocate
+	 * the backing store.
+	 */
 	uint32_t	tqm_ring5_num_entries;
-	/* Number of TQM ring 6 entries. */
+	/*
+	 * Number of TQM ring 6 entries.
+	 *
+	 * TQM fastpath rings should be sized large enough to accommodate the
+	 * maximum number of QPs (either L2 or RoCE, or both if shared)
+	 * that can be enqueued to the TQM ring.
+	 *
+	 * Note that TQM ring sizes cannot be extended while the system is
+	 * operational. If a PF driver needs to extend a TQM ring, it needs
+	 * to reset the function (e.g. HWRM_FUNC_RESET) and then reallocate
+	 * the backing store.
+	 */
 	uint32_t	tqm_ring6_num_entries;
-	/* Number of TQM ring 7 entries. */
+	/*
+	 * Number of TQM ring 7 entries.
+	 *
+	 * TQM fastpath rings should be sized large enough to accommodate the
+	 * maximum number of QPs (either L2 or RoCE, or both if shared)
+	 * that can be enqueued to the TQM ring.
+	 *
+	 * Note that TQM ring sizes cannot be extended while the system is
+	 * operational. If a PF driver needs to extend a TQM ring, it needs
+	 * to reset the function (e.g. HWRM_FUNC_RESET) and then reallocate
+	 * the backing store.
+	 */
 	uint32_t	tqm_ring7_num_entries;
 	/* Number of MR/AV entries. */
 	uint32_t	mrav_num_entries;
@@ -9638,6 +9866,633 @@ struct hwrm_func_backing_store_qcfg_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
+/***********************
+ * hwrm_func_vlan_qcfg *
+ ***********************/
+
+
+/* hwrm_func_vlan_qcfg_input (size:192b/24B) */
+struct hwrm_func_vlan_qcfg_input {
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/*
+	 * The completion ring to send the completion event on. This should
+	 * be the NQ ID returned from the `nq_alloc` HWRM command.
+	 */
+	uint16_t	cmpl_ring;
+	/*
+	 * The sequence ID is used by the driver for tracking multiple
+	 * commands. This ID is treated as opaque data by the firmware and
+	 * the value is returned in the `hwrm_resp_hdr` upon completion.
+	 */
+	uint16_t	seq_id;
+	/*
+	 * The target ID of the command:
+	 * * 0x0-0xFFF8 - The function ID
+	 * * 0xFFF8-0xFFFE - Reserved for internal processors
+	 * * 0xFFFF - HWRM
+	 */
+	uint16_t	target_id;
+	/*
+	 * A physical address pointer pointing to a host buffer that the
+	 * command's response data will be written. This can be either a host
+	 * physical address (HPA) or a guest physical address (GPA) and must
+	 * point to a physically contiguous block of memory.
+	 */
+	uint64_t	resp_addr;
+	/*
+	 * Function ID of the function that is being
+	 * configured.
+	 * If set to 0xFF... (All Fs), then the configuration is
+	 * for the requesting function.
+	 */
+	uint16_t	fid;
+	uint8_t	unused_0[6];
+} __attribute__((packed));
+
+/* hwrm_func_vlan_qcfg_output (size:320b/40B) */
+struct hwrm_func_vlan_qcfg_output {
+	/* The specific error status for the command. */
+	uint16_t	error_code;
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/* The sequence ID from the original command. */
+	uint16_t	seq_id;
+	/* The length of the response data in number of bytes. */
+	uint16_t	resp_len;
+	uint64_t	unused_0;
+	/* S-TAG VLAN identifier configured for the function. */
+	uint16_t	stag_vid;
+	/* S-TAG PCP value configured for the function. */
+	uint8_t	stag_pcp;
+	uint8_t	unused_1;
+	/*
+	 * S-TAG TPID value configured for the function. This field is specified in
+	 * network byte order.
+	 */
+	uint16_t	stag_tpid;
+	/* C-TAG VLAN identifier configured for the function. */
+	uint16_t	ctag_vid;
+	/* C-TAG PCP value configured for the function. */
+	uint8_t	ctag_pcp;
+	uint8_t	unused_2;
+	/*
+	 * C-TAG TPID value configured for the function. This field is specified in
+	 * network byte order.
+	 */
+	uint16_t	ctag_tpid;
+	/* Future use. */
+	uint32_t	rsvd2;
+	/* Future use. */
+	uint32_t	rsvd3;
+	uint8_t	unused_3[3];
+	/*
+	 * This field is used in Output records to indicate that the output
+	 * is completely written to RAM.  This field should be read as '1'
+	 * to indicate that the output has been completely written.
+	 * When writing a command completion or response to an internal processor,
+	 * the order of writes has to be such that this field is written last.
+	 */
+	uint8_t	valid;
+} __attribute__((packed));
+
+/**********************
+ * hwrm_func_vlan_cfg *
+ **********************/
+
+
+/* hwrm_func_vlan_cfg_input (size:384b/48B) */
+struct hwrm_func_vlan_cfg_input {
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/*
+	 * The completion ring to send the completion event on. This should
+	 * be the NQ ID returned from the `nq_alloc` HWRM command.
+	 */
+	uint16_t	cmpl_ring;
+	/*
+	 * The sequence ID is used by the driver for tracking multiple
+	 * commands. This ID is treated as opaque data by the firmware and
+	 * the value is returned in the `hwrm_resp_hdr` upon completion.
+	 */
+	uint16_t	seq_id;
+	/*
+	 * The target ID of the command:
+	 * * 0x0-0xFFF8 - The function ID
+	 * * 0xFFF8-0xFFFE - Reserved for internal processors
+	 * * 0xFFFF - HWRM
+	 */
+	uint16_t	target_id;
+	/*
+	 * A physical address pointer pointing to a host buffer that the
+	 * command's response data will be written. This can be either a host
+	 * physical address (HPA) or a guest physical address (GPA) and must
+	 * point to a physically contiguous block of memory.
+	 */
+	uint64_t	resp_addr;
+	/*
+	 * Function ID of the function that is being
+	 * configured.
+	 * If set to 0xFF... (All Fs), then the configuration is
+	 * for the requesting function.
+	 */
+	uint16_t	fid;
+	uint8_t	unused_0[2];
+	uint32_t	enables;
+	/*
+	 * This bit must be '1' for the stag_vid field to be
+	 * configured.
+	 */
+	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_VID      UINT32_C(0x1)
+	/*
+	 * This bit must be '1' for the ctag_vid field to be
+	 * configured.
+	 */
+	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_VID      UINT32_C(0x2)
+	/*
+	 * This bit must be '1' for the stag_pcp field to be
+	 * configured.
+	 */
+	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_PCP      UINT32_C(0x4)
+	/*
+	 * This bit must be '1' for the ctag_pcp field to be
+	 * configured.
+	 */
+	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_PCP      UINT32_C(0x8)
+	/*
+	 * This bit must be '1' for the stag_tpid field to be
+	 * configured.
+	 */
+	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_TPID     UINT32_C(0x10)
+	/*
+	 * This bit must be '1' for the ctag_tpid field to be
+	 * configured.
+	 */
+	#define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_TPID     UINT32_C(0x20)
+	/* S-TAG VLAN identifier configured for the function. */
+	uint16_t	stag_vid;
+	/* S-TAG PCP value configured for the function. */
+	uint8_t	stag_pcp;
+	uint8_t	unused_1;
+	/*
+	 * S-TAG TPID value configured for the function. This field is specified in
+	 * network byte order.
+	 */
+	uint16_t	stag_tpid;
+	/* C-TAG VLAN identifier configured for the function. */
+	uint16_t	ctag_vid;
+	/* C-TAG PCP value configured for the function. */
+	uint8_t	ctag_pcp;
+	uint8_t	unused_2;
+	/*
+	 * C-TAG TPID value configured for the function. This field is specified in
+	 * network byte order.
+	 */
+	uint16_t	ctag_tpid;
+	/* Future use. */
+	uint32_t	rsvd1;
+	/* Future use. */
+	uint32_t	rsvd2;
+	uint8_t	unused_3[4];
+} __attribute__((packed));
+
+/* hwrm_func_vlan_cfg_output (size:128b/16B) */
+struct hwrm_func_vlan_cfg_output {
+	/* The specific error status for the command. */
+	uint16_t	error_code;
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/* The sequence ID from the original command. */
+	uint16_t	seq_id;
+	/* The length of the response data in number of bytes. */
+	uint16_t	resp_len;
+	uint8_t	unused_0[7];
+	/*
+	 * This field is used in Output records to indicate that the output
+	 * is completely written to RAM.  This field should be read as '1'
+	 * to indicate that the output has been completely written.
+	 * When writing a command completion or response to an internal processor,
+	 * the order of writes has to be such that this field is written last.
+	 */
+	uint8_t	valid;
+} __attribute__((packed));
+
+/*******************************
+ * hwrm_func_vf_vnic_ids_query *
+ *******************************/
+
+
+/* hwrm_func_vf_vnic_ids_query_input (size:256b/32B) */
+struct hwrm_func_vf_vnic_ids_query_input {
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/*
+	 * The completion ring to send the completion event on. This should
+	 * be the NQ ID returned from the `nq_alloc` HWRM command.
+	 */
+	uint16_t	cmpl_ring;
+	/*
+	 * The sequence ID is used by the driver for tracking multiple
+	 * commands. This ID is treated as opaque data by the firmware and
+	 * the value is returned in the `hwrm_resp_hdr` upon completion.
+	 */
+	uint16_t	seq_id;
+	/*
+	 * The target ID of the command:
+	 * * 0x0-0xFFF8 - The function ID
+	 * * 0xFFF8-0xFFFE - Reserved for internal processors
+	 * * 0xFFFF - HWRM
+	 */
+	uint16_t	target_id;
+	/*
+	 * A physical address pointer pointing to a host buffer that the
+	 * command's response data will be written. This can be either a host
+	 * physical address (HPA) or a guest physical address (GPA) and must
+	 * point to a physically contiguous block of memory.
+	 */
+	uint64_t	resp_addr;
+	/*
+	 * This value is used to identify a Virtual Function (VF).
+	 * The scope of VF ID is local within a PF.
+	 */
+	uint16_t	vf_id;
+	uint8_t	unused_0[2];
+	/* Max number of vnic ids in vnic id table */
+	uint32_t	max_vnic_id_cnt;
+	/* This is the address for VF VNIC ID table */
+	uint64_t	vnic_id_tbl_addr;
+} __attribute__((packed));
+
+/* hwrm_func_vf_vnic_ids_query_output (size:128b/16B) */
+struct hwrm_func_vf_vnic_ids_query_output {
+	/* The specific error status for the command. */
+	uint16_t	error_code;
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/* The sequence ID from the original command. */
+	uint16_t	seq_id;
+	/* The length of the response data in number of bytes. */
+	uint16_t	resp_len;
+	/*
+	 * Actual number of vnic ids
+	 *
+	 * Each VNIC ID is written as a 32-bit number.
+	 */
+	uint32_t	vnic_id_cnt;
+	uint8_t	unused_0[3];
+	/*
+	 * This field is used in Output records to indicate that the output
+	 * is completely written to RAM.  This field should be read as '1'
+	 * to indicate that the output has been completely written.
+	 * When writing a command completion or response to an internal processor,
+	 * the order of writes has to be such that this field is written last.
+	 */
+	uint8_t	valid;
+} __attribute__((packed));
+
+/***********************
+ * hwrm_func_vf_bw_cfg *
+ ***********************/
+
+
+/* hwrm_func_vf_bw_cfg_input (size:960b/120B) */
+struct hwrm_func_vf_bw_cfg_input {
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/*
+	 * The completion ring to send the completion event on. This should
+	 * be the NQ ID returned from the `nq_alloc` HWRM command.
+	 */
+	uint16_t	cmpl_ring;
+	/*
+	 * The sequence ID is used by the driver for tracking multiple
+	 * commands. This ID is treated as opaque data by the firmware and
+	 * the value is returned in the `hwrm_resp_hdr` upon completion.
+	 */
+	uint16_t	seq_id;
+	/*
+	 * The target ID of the command:
+	 * * 0x0-0xFFF8 - The function ID
+	 * * 0xFFF8-0xFFFE - Reserved for internal processors
+	 * * 0xFFFF - HWRM
+	 */
+	uint16_t	target_id;
+	/*
+	 * A physical address pointer pointing to a host buffer that the
+	 * command's response data will be written. This can be either a host
+	 * physical address (HPA) or a guest physical address (GPA) and must
+	 * point to a physically contiguous block of memory.
+	 */
+	uint64_t	resp_addr;
+	/*
+	 * The number of VF functions that are being configured.
+	 * The cmd space allows up to 50 VFs' BW to be configured with one cmd.
+	 */
+	uint16_t	num_vfs;
+	uint16_t	unused[3];
+	/* These 16-bit fields contain the VF fid and the rate scale percentage. */
+	uint16_t	vfn[48];
+	/* The physical VF id the adjustment will be made to. */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_VFID_MASK     UINT32_C(0xfff)
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_VFID_SFT      0
+	/*
+	 * This field configures the rate scale percentage of the VF as specified
+	 * by the physical VF id.
+	 */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_MASK     UINT32_C(0xf000)
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_SFT      12
+	/* 0% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_0 \
+		(UINT32_C(0x0) << 12)
+	/* 6.66% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_6_66 \
+		(UINT32_C(0x1) << 12)
+	/* 13.33% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_13_33 \
+		(UINT32_C(0x2) << 12)
+	/* 20% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_20 \
+		(UINT32_C(0x3) << 12)
+	/* 26.66% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_26_66 \
+		(UINT32_C(0x4) << 12)
+	/* 33% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_33_33 \
+		(UINT32_C(0x5) << 12)
+	/* 40% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_40 \
+		(UINT32_C(0x6) << 12)
+	/* 46.66% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_46_66 \
+		(UINT32_C(0x7) << 12)
+	/* 53.33% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_53_33 \
+		(UINT32_C(0x8) << 12)
+	/* 60% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_60 \
+		(UINT32_C(0x9) << 12)
+	/* 66.66% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_66_66 \
+		(UINT32_C(0xa) << 12)
+	/* 53.33% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_73_33 \
+		(UINT32_C(0xb) << 12)
+	/* 80% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_80 \
+		(UINT32_C(0xc) << 12)
+	/* 86.66% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_86_66 \
+		(UINT32_C(0xd) << 12)
+	/* 93.33% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_93_33 \
+		(UINT32_C(0xe) << 12)
+	/* 100% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_100 \
+		(UINT32_C(0xf) << 12)
+	#define HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_LAST \
+		HWRM_FUNC_VF_BW_CFG_INPUT_VFN_RATE_PCT_100
+} __attribute__((packed));
+
+/* hwrm_func_vf_bw_cfg_output (size:128b/16B) */
+struct hwrm_func_vf_bw_cfg_output {
+	/* The specific error status for the command. */
+	uint16_t	error_code;
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/* The sequence ID from the original command. */
+	uint16_t	seq_id;
+	/* The length of the response data in number of bytes. */
+	uint16_t	resp_len;
+	uint8_t	unused_0[7];
+	/*
+	 * This field is used in Output records to indicate that the output
+	 * is completely written to RAM.  This field should be read as '1'
+	 * to indicate that the output has been completely written.
+	 * When writing a command completion or response to an internal processor,
+	 * the order of writes has to be such that this field is written last.
+	 */
+	uint8_t	valid;
+} __attribute__((packed));
+
+/************************
+ * hwrm_func_vf_bw_qcfg *
+ ************************/
+
+
+/* hwrm_func_vf_bw_qcfg_input (size:960b/120B) */
+struct hwrm_func_vf_bw_qcfg_input {
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/*
+	 * The completion ring to send the completion event on. This should
+	 * be the NQ ID returned from the `nq_alloc` HWRM command.
+	 */
+	uint16_t	cmpl_ring;
+	/*
+	 * The sequence ID is used by the driver for tracking multiple
+	 * commands. This ID is treated as opaque data by the firmware and
+	 * the value is returned in the `hwrm_resp_hdr` upon completion.
+	 */
+	uint16_t	seq_id;
+	/*
+	 * The target ID of the command:
+	 * * 0x0-0xFFF8 - The function ID
+	 * * 0xFFF8-0xFFFE - Reserved for internal processors
+	 * * 0xFFFF - HWRM
+	 */
+	uint16_t	target_id;
+	/*
+	 * A physical address pointer pointing to a host buffer that the
+	 * command's response data will be written. This can be either a host
+	 * physical address (HPA) or a guest physical address (GPA) and must
+	 * point to a physically contiguous block of memory.
+	 */
+	uint64_t	resp_addr;
+	/*
+	 * The number of VF functions that are being queried.
+	 * The inline response space allows the host to query up to 50 VFs'
+	 * rate scale percentage
+	 */
+	uint16_t	num_vfs;
+	uint16_t	unused[3];
+	/* These 16-bit fields contain the VF fid */
+	uint16_t	vfn[48];
+	/* The physical VF id of interest */
+	#define HWRM_FUNC_VF_BW_QCFG_INPUT_VFN_VFID_MASK UINT32_C(0xfff)
+	#define HWRM_FUNC_VF_BW_QCFG_INPUT_VFN_VFID_SFT 0
+} __attribute__((packed));
+
+/* hwrm_func_vf_bw_qcfg_output (size:960b/120B) */
+struct hwrm_func_vf_bw_qcfg_output {
+	/* The specific error status for the command. */
+	uint16_t	error_code;
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/* The sequence ID from the original command. */
+	uint16_t	seq_id;
+	/* The length of the response data in number of bytes. */
+	uint16_t	resp_len;
+	/*
+	 * The number of VF functions that are being queried.
+	 * The inline response space allows the host to query up to 50 VFs' rate
+	 * scale percentage
+	 */
+	uint16_t	num_vfs;
+	uint16_t	unused[3];
+	/* These 16-bit fields contain the VF fid and the rate scale percentage. */
+	uint16_t	vfn[48];
+	/* The physical VF id the adjustment will be made to. */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_VFID_MASK     UINT32_C(0xfff)
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_VFID_SFT      0
+	/*
+	 * This field configures the rate scale percentage of the VF as specified
+	 * by the physical VF id.
+	 */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_MASK     UINT32_C(0xf000)
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_SFT      12
+	/* 0% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_0 \
+		(UINT32_C(0x0) << 12)
+	/* 6.66% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_6_66 \
+		(UINT32_C(0x1) << 12)
+	/* 13.33% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_13_33 \
+		(UINT32_C(0x2) << 12)
+	/* 20% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_20 \
+		(UINT32_C(0x3) << 12)
+	/* 26.66% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_26_66 \
+		(UINT32_C(0x4) << 12)
+	/* 33% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_33_33 \
+		(UINT32_C(0x5) << 12)
+	/* 40% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_40 \
+		(UINT32_C(0x6) << 12)
+	/* 46.66% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_46_66 \
+		(UINT32_C(0x7) << 12)
+	/* 53.33% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_53_33 \
+		(UINT32_C(0x8) << 12)
+	/* 60% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_60 \
+		(UINT32_C(0x9) << 12)
+	/* 66.66% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_66_66 \
+		(UINT32_C(0xa) << 12)
+	/* 53.33% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_73_33 \
+		(UINT32_C(0xb) << 12)
+	/* 80% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_80 \
+		(UINT32_C(0xc) << 12)
+	/* 86.66% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_86_66 \
+		(UINT32_C(0xd) << 12)
+	/* 93.33% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_93_33 \
+		(UINT32_C(0xe) << 12)
+	/* 100% of the max tx rate */
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_100 \
+		(UINT32_C(0xf) << 12)
+	#define HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_LAST \
+		HWRM_FUNC_VF_BW_QCFG_OUTPUT_VFN_RATE_PCT_100
+	uint8_t	unused_0[7];
+	/*
+	 * This field is used in Output records to indicate that the output
+	 * is completely written to RAM.  This field should be read as '1'
+	 * to indicate that the output has been completely written.
+	 * When writing a command completion or response to an internal processor,
+	 * the order of writes has to be such that this field is written last.
+	 */
+	uint8_t	valid;
+} __attribute__((packed));
+
+/***************************
+ * hwrm_func_drv_if_change *
+ ***************************/
+
+
+/* hwrm_func_drv_if_change_input (size:192b/24B) */
+struct hwrm_func_drv_if_change_input {
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/*
+	 * The completion ring to send the completion event on. This should
+	 * be the NQ ID returned from the `nq_alloc` HWRM command.
+	 */
+	uint16_t	cmpl_ring;
+	/*
+	 * The sequence ID is used by the driver for tracking multiple
+	 * commands. This ID is treated as opaque data by the firmware and
+	 * the value is returned in the `hwrm_resp_hdr` upon completion.
+	 */
+	uint16_t	seq_id;
+	/*
+	 * The target ID of the command:
+	 * * 0x0-0xFFF8 - The function ID
+	 * * 0xFFF8-0xFFFE - Reserved for internal processors
+	 * * 0xFFFF - HWRM
+	 */
+	uint16_t	target_id;
+	/*
+	 * A physical address pointer pointing to a host buffer that the
+	 * command's response data will be written. This can be either a host
+	 * physical address (HPA) or a guest physical address (GPA) and must
+	 * point to a physically contiguous block of memory.
+	 */
+	uint64_t	resp_addr;
+	uint32_t	flags;
+	/*
+	 * When this bit is '1', the function driver is indicating
+	 * that the IF state is changing to UP state.  The call should
+	 * be made at the beginning of the driver's open call before
+	 * resources are allocated.  After making the call, the driver
+	 * should check the response to see if any resources may have
+	 * changed (see the response below).  If the driver fails
+	 * the open call, the driver should make this call again with
+	 * this bit cleared to indicate that the IF state is not UP.
+	 * During the driver's close call when the IF state is changing
+	 * to DOWN, the driver should make this call with the bit cleared
+	 * after all resources have been freed.
+	 */
+	#define HWRM_FUNC_DRV_IF_CHANGE_INPUT_FLAGS_UP     UINT32_C(0x1)
+	uint32_t	unused;
+} __attribute__((packed));
+
+/* hwrm_func_drv_if_change_output (size:128b/16B) */
+struct hwrm_func_drv_if_change_output {
+	/* The specific error status for the command. */
+	uint16_t	error_code;
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/* The sequence ID from the original command. */
+	uint16_t	seq_id;
+	/* The length of the response data in number of bytes. */
+	uint16_t	resp_len;
+	uint32_t	flags;
+	/*
+	 * When this bit is '1', it indicates that the resources reserved
+	 * for this function may have changed.  The driver should check
+	 * resource capabilities and reserve resources again before
+	 * allocating resources.
+	 */
+	#define HWRM_FUNC_DRV_IF_CHANGE_OUTPUT_FLAGS_RESC_CHANGE \
+		UINT32_C(0x1)
+	uint8_t	unused_0[3];
+	/*
+	 * This field is used in Output records to indicate that the output
+	 * is completely written to RAM.  This field should be read as '1'
+	 * to indicate that the output has been completely written.
+	 * When writing a command completion or response to an internal processor,
+	 * the order of writes has to be such that this field is written last.
+	 */
+	uint8_t	valid;
+} __attribute__((packed));
+
 /*********************
  * hwrm_port_phy_cfg *
  *********************/
@@ -11916,6 +12771,362 @@ struct hwrm_port_mac_ptp_qcfg_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
+/* Port Tx Statistics Formats */
+/* tx_port_stats (size:3264b/408B) */
+struct tx_port_stats {
+	/* Total Number of 64 Bytes frames transmitted */
+	uint64_t	tx_64b_frames;
+	/* Total Number of 65-127 Bytes frames transmitted */
+	uint64_t	tx_65b_127b_frames;
+	/* Total Number of 128-255 Bytes frames transmitted */
+	uint64_t	tx_128b_255b_frames;
+	/* Total Number of 256-511 Bytes frames transmitted */
+	uint64_t	tx_256b_511b_frames;
+	/* Total Number of 512-1023 Bytes frames transmitted */
+	uint64_t	tx_512b_1023b_frames;
+	/* Total Number of 1024-1518 Bytes frames transmitted */
+	uint64_t	tx_1024b_1518b_frames;
+	/*
+	 * Total Number of each good VLAN (exludes FCS errors)
+	 * frame transmitted which is 1519 to 1522 bytes in length
+	 * inclusive (excluding framing bits but including FCS bytes).
+	 */
+	uint64_t	tx_good_vlan_frames;
+	/* Total Number of 1519-2047 Bytes frames transmitted */
+	uint64_t	tx_1519b_2047b_frames;
+	/* Total Number of 2048-4095 Bytes frames transmitted */
+	uint64_t	tx_2048b_4095b_frames;
+	/* Total Number of 4096-9216 Bytes frames transmitted */
+	uint64_t	tx_4096b_9216b_frames;
+	/* Total Number of 9217-16383 Bytes frames transmitted */
+	uint64_t	tx_9217b_16383b_frames;
+	/* Total Number of good frames transmitted */
+	uint64_t	tx_good_frames;
+	/* Total Number of frames transmitted */
+	uint64_t	tx_total_frames;
+	/* Total number of unicast frames transmitted */
+	uint64_t	tx_ucast_frames;
+	/* Total number of multicast frames transmitted */
+	uint64_t	tx_mcast_frames;
+	/* Total number of broadcast frames transmitted */
+	uint64_t	tx_bcast_frames;
+	/* Total number of PAUSE control frames transmitted */
+	uint64_t	tx_pause_frames;
+	/*
+	 * Total number of PFC/per-priority PAUSE
+	 * control frames transmitted
+	 */
+	uint64_t	tx_pfc_frames;
+	/* Total number of jabber frames transmitted */
+	uint64_t	tx_jabber_frames;
+	/* Total number of frames transmitted with FCS error */
+	uint64_t	tx_fcs_err_frames;
+	/* Total number of control frames transmitted */
+	uint64_t	tx_control_frames;
+	/* Total number of over-sized frames transmitted */
+	uint64_t	tx_oversz_frames;
+	/* Total number of frames with single deferral */
+	uint64_t	tx_single_dfrl_frames;
+	/* Total number of frames with multiple deferrals */
+	uint64_t	tx_multi_dfrl_frames;
+	/* Total number of frames with single collision */
+	uint64_t	tx_single_coll_frames;
+	/* Total number of frames with multiple collisions */
+	uint64_t	tx_multi_coll_frames;
+	/* Total number of frames with late collisions */
+	uint64_t	tx_late_coll_frames;
+	/* Total number of frames with excessive collisions */
+	uint64_t	tx_excessive_coll_frames;
+	/* Total number of fragmented frames transmitted */
+	uint64_t	tx_frag_frames;
+	/* Total number of transmit errors */
+	uint64_t	tx_err;
+	/* Total number of single VLAN tagged frames transmitted */
+	uint64_t	tx_tagged_frames;
+	/* Total number of double VLAN tagged frames transmitted */
+	uint64_t	tx_dbl_tagged_frames;
+	/* Total number of runt frames transmitted */
+	uint64_t	tx_runt_frames;
+	/* Total number of TX FIFO under runs */
+	uint64_t	tx_fifo_underruns;
+	/*
+	 * Total number of PFC frames with PFC enabled bit for
+	 * Pri 0 transmitted
+	 */
+	uint64_t	tx_pfc_ena_frames_pri0;
+	/*
+	 * Total number of PFC frames with PFC enabled bit for
+	 * Pri 1 transmitted
+	 */
+	uint64_t	tx_pfc_ena_frames_pri1;
+	/*
+	 * Total number of PFC frames with PFC enabled bit for
+	 * Pri 2 transmitted
+	 */
+	uint64_t	tx_pfc_ena_frames_pri2;
+	/*
+	 * Total number of PFC frames with PFC enabled bit for
+	 * Pri 3 transmitted
+	 */
+	uint64_t	tx_pfc_ena_frames_pri3;
+	/*
+	 * Total number of PFC frames with PFC enabled bit for
+	 * Pri 4 transmitted
+	 */
+	uint64_t	tx_pfc_ena_frames_pri4;
+	/*
+	 * Total number of PFC frames with PFC enabled bit for
+	 * Pri 5 transmitted
+	 */
+	uint64_t	tx_pfc_ena_frames_pri5;
+	/*
+	 * Total number of PFC frames with PFC enabled bit for
+	 * Pri 6 transmitted
+	 */
+	uint64_t	tx_pfc_ena_frames_pri6;
+	/*
+	 * Total number of PFC frames with PFC enabled bit for
+	 * Pri 7 transmitted
+	 */
+	uint64_t	tx_pfc_ena_frames_pri7;
+	/* Total number of EEE LPI Events on TX */
+	uint64_t	tx_eee_lpi_events;
+	/* EEE LPI Duration Counter on TX */
+	uint64_t	tx_eee_lpi_duration;
+	/*
+	 * Total number of Link Level Flow Control (LLFC) messages
+	 * transmitted
+	 */
+	uint64_t	tx_llfc_logical_msgs;
+	/* Total number of HCFC messages transmitted */
+	uint64_t	tx_hcfc_msgs;
+	/* Total number of TX collisions */
+	uint64_t	tx_total_collisions;
+	/* Total number of transmitted bytes */
+	uint64_t	tx_bytes;
+	/* Total number of end-to-end HOL frames */
+	uint64_t	tx_xthol_frames;
+	/* Total Tx Drops per Port reported by STATS block */
+	uint64_t	tx_stat_discard;
+	/* Total Tx Error Drops per Port reported by STATS block */
+	uint64_t	tx_stat_error;
+} __attribute__((packed));
+
+/* Port Rx Statistics Formats */
+/* rx_port_stats (size:4224b/528B) */
+struct rx_port_stats {
+	/* Total Number of 64 Bytes frames received */
+	uint64_t	rx_64b_frames;
+	/* Total Number of 65-127 Bytes frames received */
+	uint64_t	rx_65b_127b_frames;
+	/* Total Number of 128-255 Bytes frames received */
+	uint64_t	rx_128b_255b_frames;
+	/* Total Number of 256-511 Bytes frames received */
+	uint64_t	rx_256b_511b_frames;
+	/* Total Number of 512-1023 Bytes frames received */
+	uint64_t	rx_512b_1023b_frames;
+	/* Total Number of 1024-1518 Bytes frames received */
+	uint64_t	rx_1024b_1518b_frames;
+	/*
+	 * Total Number of each good VLAN (exludes FCS errors)
+	 * frame received which is 1519 to 1522 bytes in length
+	 * inclusive (excluding framing bits but including FCS bytes).
+	 */
+	uint64_t	rx_good_vlan_frames;
+	/* Total Number of 1519-2047 Bytes frames received */
+	uint64_t	rx_1519b_2047b_frames;
+	/* Total Number of 2048-4095 Bytes frames received */
+	uint64_t	rx_2048b_4095b_frames;
+	/* Total Number of 4096-9216 Bytes frames received */
+	uint64_t	rx_4096b_9216b_frames;
+	/* Total Number of 9217-16383 Bytes frames received */
+	uint64_t	rx_9217b_16383b_frames;
+	/* Total number of frames received */
+	uint64_t	rx_total_frames;
+	/* Total number of unicast frames received */
+	uint64_t	rx_ucast_frames;
+	/* Total number of multicast frames received */
+	uint64_t	rx_mcast_frames;
+	/* Total number of broadcast frames received */
+	uint64_t	rx_bcast_frames;
+	/* Total number of received frames with FCS error */
+	uint64_t	rx_fcs_err_frames;
+	/* Total number of control frames received */
+	uint64_t	rx_ctrl_frames;
+	/* Total number of PAUSE frames received */
+	uint64_t	rx_pause_frames;
+	/* Total number of PFC frames received */
+	uint64_t	rx_pfc_frames;
+	/*
+	 * Total number of frames received with an unsupported
+	 * opcode
+	 */
+	uint64_t	rx_unsupported_opcode_frames;
+	/*
+	 * Total number of frames received with an unsupported
+	 * DA for pause and PFC
+	 */
+	uint64_t	rx_unsupported_da_pausepfc_frames;
+	/* Total number of frames received with an unsupported SA */
+	uint64_t	rx_wrong_sa_frames;
+	/* Total number of received packets with alignment error */
+	uint64_t	rx_align_err_frames;
+	/* Total number of received frames with out-of-range length */
+	uint64_t	rx_oor_len_frames;
+	/* Total number of received frames with error termination */
+	uint64_t	rx_code_err_frames;
+	/*
+	 * Total number of received frames with a false carrier is
+	 * detected during idle, as defined by RX_ER samples active
+	 * and RXD is 0xE. The event is reported along with the
+	 * statistics generated on the next received frame. Only
+	 * one false carrier condition can be detected and logged
+	 * between frames.
+	 *
+	 * Carrier event, valid for 10M/100M speed modes only.
+	 */
+	uint64_t	rx_false_carrier_frames;
+	/* Total number of over-sized frames received */
+	uint64_t	rx_ovrsz_frames;
+	/* Total number of jabber packets received */
+	uint64_t	rx_jbr_frames;
+	/* Total number of received frames with MTU error */
+	uint64_t	rx_mtu_err_frames;
+	/* Total number of received frames with CRC match */
+	uint64_t	rx_match_crc_frames;
+	/* Total number of frames received promiscuously */
+	uint64_t	rx_promiscuous_frames;
+	/*
+	 * Total number of received frames with one or two VLAN
+	 * tags
+	 */
+	uint64_t	rx_tagged_frames;
+	/* Total number of received frames with two VLAN tags */
+	uint64_t	rx_double_tagged_frames;
+	/* Total number of truncated frames received */
+	uint64_t	rx_trunc_frames;
+	/* Total number of good frames (without errors) received */
+	uint64_t	rx_good_frames;
+	/*
+	 * Total number of received PFC frames with transition from
+	 * XON to XOFF on Pri 0
+	 */
+	uint64_t	rx_pfc_xon2xoff_frames_pri0;
+	/*
+	 * Total number of received PFC frames with transition from
+	 * XON to XOFF on Pri 1
+	 */
+	uint64_t	rx_pfc_xon2xoff_frames_pri1;
+	/*
+	 * Total number of received PFC frames with transition from
+	 * XON to XOFF on Pri 2
+	 */
+	uint64_t	rx_pfc_xon2xoff_frames_pri2;
+	/*
+	 * Total number of received PFC frames with transition from
+	 * XON to XOFF on Pri 3
+	 */
+	uint64_t	rx_pfc_xon2xoff_frames_pri3;
+	/*
+	 * Total number of received PFC frames with transition from
+	 * XON to XOFF on Pri 4
+	 */
+	uint64_t	rx_pfc_xon2xoff_frames_pri4;
+	/*
+	 * Total number of received PFC frames with transition from
+	 * XON to XOFF on Pri 5
+	 */
+	uint64_t	rx_pfc_xon2xoff_frames_pri5;
+	/*
+	 * Total number of received PFC frames with transition from
+	 * XON to XOFF on Pri 6
+	 */
+	uint64_t	rx_pfc_xon2xoff_frames_pri6;
+	/*
+	 * Total number of received PFC frames with transition from
+	 * XON to XOFF on Pri 7
+	 */
+	uint64_t	rx_pfc_xon2xoff_frames_pri7;
+	/*
+	 * Total number of received PFC frames with PFC enabled
+	 * bit for Pri 0
+	 */
+	uint64_t	rx_pfc_ena_frames_pri0;
+	/*
+	 * Total number of received PFC frames with PFC enabled
+	 * bit for Pri 1
+	 */
+	uint64_t	rx_pfc_ena_frames_pri1;
+	/*
+	 * Total number of received PFC frames with PFC enabled
+	 * bit for Pri 2
+	 */
+	uint64_t	rx_pfc_ena_frames_pri2;
+	/*
+	 * Total number of received PFC frames with PFC enabled
+	 * bit for Pri 3
+	 */
+	uint64_t	rx_pfc_ena_frames_pri3;
+	/*
+	 * Total number of received PFC frames with PFC enabled
+	 * bit for Pri 4
+	 */
+	uint64_t	rx_pfc_ena_frames_pri4;
+	/*
+	 * Total number of received PFC frames with PFC enabled
+	 * bit for Pri 5
+	 */
+	uint64_t	rx_pfc_ena_frames_pri5;
+	/*
+	 * Total number of received PFC frames with PFC enabled
+	 * bit for Pri 6
+	 */
+	uint64_t	rx_pfc_ena_frames_pri6;
+	/*
+	 * Total number of received PFC frames with PFC enabled
+	 * bit for Pri 7
+	 */
+	uint64_t	rx_pfc_ena_frames_pri7;
+	/* Total Number of frames received with SCH CRC error */
+	uint64_t	rx_sch_crc_err_frames;
+	/* Total Number of under-sized frames received */
+	uint64_t	rx_undrsz_frames;
+	/* Total Number of fragmented frames received */
+	uint64_t	rx_frag_frames;
+	/* Total number of RX EEE LPI Events */
+	uint64_t	rx_eee_lpi_events;
+	/* EEE LPI Duration Counter on RX */
+	uint64_t	rx_eee_lpi_duration;
+	/*
+	 * Total number of physical type Link Level Flow Control
+	 * (LLFC) messages received
+	 */
+	uint64_t	rx_llfc_physical_msgs;
+	/*
+	 * Total number of logical type Link Level Flow Control
+	 * (LLFC) messages received
+	 */
+	uint64_t	rx_llfc_logical_msgs;
+	/*
+	 * Total number of logical type Link Level Flow Control
+	 * (LLFC) messages received with CRC error
+	 */
+	uint64_t	rx_llfc_msgs_with_crc_err;
+	/* Total number of HCFC messages received */
+	uint64_t	rx_hcfc_msgs;
+	/* Total number of HCFC messages received with CRC error */
+	uint64_t	rx_hcfc_msgs_with_crc_err;
+	/* Total number of received bytes */
+	uint64_t	rx_bytes;
+	/* Total number of bytes received in runt frames */
+	uint64_t	rx_runt_bytes;
+	/* Total number of runt frames received */
+	uint64_t	rx_runt_frames;
+	/* Total Rx Discards per Port reported by STATS block */
+	uint64_t	rx_stat_discard;
+	uint64_t	rx_stat_err;
+} __attribute__((packed));
+
 /********************
  * hwrm_port_qstats *
  ********************/
@@ -11990,6 +13201,154 @@ struct hwrm_port_qstats_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
+/* Port Tx Statistics extended Formats */
+/* tx_port_stats_ext (size:2048b/256B) */
+struct tx_port_stats_ext {
+	/* Total number of tx bytes count on cos queue 0 */
+	uint64_t	tx_bytes_cos0;
+	/* Total number of tx bytes count on cos queue 1 */
+	uint64_t	tx_bytes_cos1;
+	/* Total number of tx bytes count on cos queue 2 */
+	uint64_t	tx_bytes_cos2;
+	/* Total number of tx bytes count on cos queue 3 */
+	uint64_t	tx_bytes_cos3;
+	/* Total number of tx bytes count on cos queue 4 */
+	uint64_t	tx_bytes_cos4;
+	/* Total number of tx bytes count on cos queue 5 */
+	uint64_t	tx_bytes_cos5;
+	/* Total number of tx bytes count on cos queue 6 */
+	uint64_t	tx_bytes_cos6;
+	/* Total number of tx bytes count on cos queue 7 */
+	uint64_t	tx_bytes_cos7;
+	/* Total number of tx packets count on cos queue 0 */
+	uint64_t	tx_packets_cos0;
+	/* Total number of tx packets count on cos queue 1 */
+	uint64_t	tx_packets_cos1;
+	/* Total number of tx packets count on cos queue 2 */
+	uint64_t	tx_packets_cos2;
+	/* Total number of tx packets count on cos queue 3 */
+	uint64_t	tx_packets_cos3;
+	/* Total number of tx packets count on cos queue 4 */
+	uint64_t	tx_packets_cos4;
+	/* Total number of tx packets count on cos queue 5 */
+	uint64_t	tx_packets_cos5;
+	/* Total number of tx packets count on cos queue 6 */
+	uint64_t	tx_packets_cos6;
+	/* Total number of tx packets count on cos queue 7 */
+	uint64_t	tx_packets_cos7;
+	/* time duration between transmitting a XON -> XOFF and a subsequent XOFF -> XON for priority 0 */
+	uint64_t	pfc_pri0_tx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 0 */
+	uint64_t	pfc_pri0_tx_transitions;
+	/* time duration between transmitting a XON -> XOFF and a subsequent XOFF -> XON for priority 1 */
+	uint64_t	pfc_pri1_tx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 1 */
+	uint64_t	pfc_pri1_tx_transitions;
+	/* time duration between transmitting a XON -> XOFF and a subsequent XOFF -> XON for priority 2 */
+	uint64_t	pfc_pri2_tx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 2 */
+	uint64_t	pfc_pri2_tx_transitions;
+	/* time duration between transmitting a XON -> XOFF and a subsequent XOFF -> XON for priority 3 */
+	uint64_t	pfc_pri3_tx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 3 */
+	uint64_t	pfc_pri3_tx_transitions;
+	/* time duration between transmitting a XON -> XOFF and a subsequent XOFF -> XON for priority 4 */
+	uint64_t	pfc_pri4_tx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 4 */
+	uint64_t	pfc_pri4_tx_transitions;
+	/* time duration between transmitting a XON -> XOFF and a subsequent XOFF -> XON for priority 5 */
+	uint64_t	pfc_pri5_tx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 5 */
+	uint64_t	pfc_pri5_tx_transitions;
+	/* time duration between transmitting a XON -> XOFF and a subsequent XOFF -> XON for priority 6 */
+	uint64_t	pfc_pri6_tx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 6 */
+	uint64_t	pfc_pri6_tx_transitions;
+	/* time duration between transmitting a XON -> XOFF and a subsequent XOFF -> XON for priority 7 */
+	uint64_t	pfc_pri7_tx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 7 */
+	uint64_t	pfc_pri7_tx_transitions;
+} __attribute__((packed));
+
+/* Port Rx Statistics extended Formats */
+/* rx_port_stats_ext (size:2368b/296B) */
+struct rx_port_stats_ext {
+	/* Number of times link state changed to down */
+	uint64_t	link_down_events;
+	/* Number of times the idle rings with pause bit are found */
+	uint64_t	continuous_pause_events;
+	/* Number of times the active rings pause bit resumed back */
+	uint64_t	resume_pause_events;
+	/* Number of times, the ROCE cos queue PFC is disabled to avoid pause flood/burst */
+	uint64_t	continuous_roce_pause_events;
+	/* Number of times, the ROCE cos queue PFC is enabled back */
+	uint64_t	resume_roce_pause_events;
+	/* Total number of rx bytes count on cos queue 0 */
+	uint64_t	rx_bytes_cos0;
+	/* Total number of rx bytes count on cos queue 1 */
+	uint64_t	rx_bytes_cos1;
+	/* Total number of rx bytes count on cos queue 2 */
+	uint64_t	rx_bytes_cos2;
+	/* Total number of rx bytes count on cos queue 3 */
+	uint64_t	rx_bytes_cos3;
+	/* Total number of rx bytes count on cos queue 4 */
+	uint64_t	rx_bytes_cos4;
+	/* Total number of rx bytes count on cos queue 5 */
+	uint64_t	rx_bytes_cos5;
+	/* Total number of rx bytes count on cos queue 6 */
+	uint64_t	rx_bytes_cos6;
+	/* Total number of rx bytes count on cos queue 7 */
+	uint64_t	rx_bytes_cos7;
+	/* Total number of rx packets count on cos queue 0 */
+	uint64_t	rx_packets_cos0;
+	/* Total number of rx packets count on cos queue 1 */
+	uint64_t	rx_packets_cos1;
+	/* Total number of rx packets count on cos queue 2 */
+	uint64_t	rx_packets_cos2;
+	/* Total number of rx packets count on cos queue 3 */
+	uint64_t	rx_packets_cos3;
+	/* Total number of rx packets count on cos queue 4 */
+	uint64_t	rx_packets_cos4;
+	/* Total number of rx packets count on cos queue 5 */
+	uint64_t	rx_packets_cos5;
+	/* Total number of rx packets count on cos queue 6 */
+	uint64_t	rx_packets_cos6;
+	/* Total number of rx packets count on cos queue 7 */
+	uint64_t	rx_packets_cos7;
+	/* time duration receiving a XON -> XOFF and a subsequent XOFF -> XON for priority 0 */
+	uint64_t	pfc_pri0_rx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 0 */
+	uint64_t	pfc_pri0_rx_transitions;
+	/* time duration receiving a XON -> XOFF and a subsequent XOFF -> XON for priority 1 */
+	uint64_t	pfc_pri1_rx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 1 */
+	uint64_t	pfc_pri1_rx_transitions;
+	/* time duration receiving a XON -> XOFF and a subsequent XOFF -> XON for priority 2 */
+	uint64_t	pfc_pri2_rx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 2 */
+	uint64_t	pfc_pri2_rx_transitions;
+	/* time duration receiving a XON -> XOFF and a subsequent XOFF -> XON for priority 3 */
+	uint64_t	pfc_pri3_rx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 3 */
+	uint64_t	pfc_pri3_rx_transitions;
+	/* time duration receiving a XON -> XOFF and a subsequent XOFF -> XON for priority 4 */
+	uint64_t	pfc_pri4_rx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 4 */
+	uint64_t	pfc_pri4_rx_transitions;
+	/* time duration receiving a XON -> XOFF and a subsequent XOFF -> XON for priority 5 */
+	uint64_t	pfc_pri5_rx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 5 */
+	uint64_t	pfc_pri5_rx_transitions;
+	/* time duration receiving a XON -> XOFF and a subsequent XOFF -> XON for priority 6 */
+	uint64_t	pfc_pri6_rx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 6 */
+	uint64_t	pfc_pri6_rx_transitions;
+	/* time duration receiving a XON -> XOFF and a subsequent XOFF -> XON for priority 7 */
+	uint64_t	pfc_pri7_rx_duration_us;
+	/* Number of times, a XON -> XOFF and XOFF -> XON transitions occur for priority 7 */
+	uint64_t	pfc_pri7_rx_transitions;
+} __attribute__((packed));
+
 /************************
  * hwrm_port_qstats_ext *
  ************************/
@@ -12063,7 +13422,15 @@ struct hwrm_port_qstats_ext_output {
 	uint16_t	tx_stat_size;
 	/* The size of RX port statistics block in bytes. */
 	uint16_t	rx_stat_size;
-	uint8_t	unused_0[3];
+	/* Total number of active cos queues available. */
+	uint16_t	total_active_cos_queues;
+	uint8_t	flags;
+	/*
+	 * If set to 1, then this field indicates that clear
+	 * roce specific counters is supported.
+	 */
+	#define HWRM_PORT_QSTATS_EXT_OUTPUT_FLAGS_CLEAR_ROCE_COUNTERS_SUPPORTED \
+		UINT32_C(0x1)
 	/*
 	 * This field is used in Output records to indicate that the output
 	 * is completely written to RAM.  This field should be read as '1'
@@ -12187,68 +13554,22 @@ struct hwrm_port_clr_stats_input {
 	uint64_t	resp_addr;
 	/* Port ID of port that is being queried. */
 	uint16_t	port_id;
-	uint8_t	unused_0[6];
-} __attribute__((packed));
-
-/* hwrm_port_clr_stats_output (size:128b/16B) */
-struct hwrm_port_clr_stats_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/****************************
- * hwrm_port_lpbk_clr_stats *
- ****************************/
-
-
-/* hwrm_port_lpbk_clr_stats_input (size:128b/16B) */
-struct hwrm_port_lpbk_clr_stats_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
+	uint8_t	flags;
 	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
+	 * If set to 1, then this field indicates clear the following RoCE
+	 * specific counters.
+	 * RoCE associated TX/RX cos counters
+	 * CNP associated TX/RX cos counters
+	 * RoCE/CNP specific TX/RX flow counters
+	 * Firmware will determine the RoCE/CNP cos queue based on qos profile.
+	 * This flag is honored only when RoCE is enabled on that port.
 	 */
-	uint64_t	resp_addr;
+	#define HWRM_PORT_CLR_STATS_INPUT_FLAGS_ROCE_COUNTERS     UINT32_C(0x1)
+	uint8_t	unused_0[5];
 } __attribute__((packed));
 
-/* hwrm_port_lpbk_clr_stats_output (size:128b/16B) */
-struct hwrm_port_lpbk_clr_stats_output {
+/* hwrm_port_clr_stats_output (size:128b/16B) */
+struct hwrm_port_clr_stats_output {
 	/* The specific error status for the command. */
 	uint16_t	error_code;
 	/* The HWRM command request type. */
@@ -12268,83 +13589,6 @@ struct hwrm_port_lpbk_clr_stats_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/**********************
- * hwrm_port_ts_query *
- **********************/
-
-
-/* hwrm_port_ts_query_input (size:192b/24B) */
-struct hwrm_port_ts_query_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	uint32_t	flags;
-	/*
-	 * Enumeration denoting the RX, TX type of the resource.
-	 * This enumeration is used for resources that are similar for both
-	 * TX and RX paths of the chip.
-	 */
-	#define HWRM_PORT_TS_QUERY_INPUT_FLAGS_PATH     UINT32_C(0x1)
-	/* tx path */
-	#define HWRM_PORT_TS_QUERY_INPUT_FLAGS_PATH_TX    UINT32_C(0x0)
-	/* rx path */
-	#define HWRM_PORT_TS_QUERY_INPUT_FLAGS_PATH_RX    UINT32_C(0x1)
-	#define HWRM_PORT_TS_QUERY_INPUT_FLAGS_PATH_LAST \
-		HWRM_PORT_TS_QUERY_INPUT_FLAGS_PATH_RX
-	/* Port ID of port that is being queried. */
-	uint16_t	port_id;
-	uint8_t	unused_0[2];
-} __attribute__((packed));
-
-/* hwrm_port_ts_query_output (size:192b/24B) */
-struct hwrm_port_ts_query_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	/* Timestamp value of PTP message captured. */
-	uint64_t	ptp_msg_ts;
-	/* Sequence ID of the PTP message captured. */
-	uint16_t	ptp_msg_seqid;
-	uint8_t	unused_0[5];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
 /***********************
  * hwrm_port_phy_qcaps *
  ***********************/
@@ -12593,172 +13837,6 @@ struct hwrm_port_phy_qcaps_output {
 	#define HWRM_PORT_PHY_QCAPS_OUTPUT_VALID_SFT             24
 } __attribute__((packed));
 
-/***************************
- * hwrm_port_phy_i2c_write *
- ***************************/
-
-
-/* hwrm_port_phy_i2c_write_input (size:832b/104B) */
-struct hwrm_port_phy_i2c_write_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	uint32_t	flags;
-	uint32_t	enables;
-	/*
-	 * This bit must be '1' for the page_offset field to be
-	 * configured.
-	 */
-	#define HWRM_PORT_PHY_I2C_WRITE_INPUT_ENABLES_PAGE_OFFSET \
-		UINT32_C(0x1)
-	/* Port ID of port. */
-	uint16_t	port_id;
-	/* 8-bit I2C slave address. */
-	uint8_t	i2c_slave_addr;
-	uint8_t	unused_0;
-	/* The page number that is being accessed over I2C. */
-	uint16_t	page_number;
-	/* Offset within the page that is being accessed over I2C. */
-	uint16_t	page_offset;
-	/*
-	 * Length of data to write, in bytes starting at the offset
-	 * specified above. If the offset is not specified, then
-	 * the data shall be written from the beginning of the page.
-	 */
-	uint8_t	data_length;
-	uint8_t	unused_1[7];
-	/* Up to 64B of data. */
-	uint32_t	data[16];
-} __attribute__((packed));
-
-/* hwrm_port_phy_i2c_write_output (size:128b/16B) */
-struct hwrm_port_phy_i2c_write_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/**************************
- * hwrm_port_phy_i2c_read *
- **************************/
-
-
-/* hwrm_port_phy_i2c_read_input (size:320b/40B) */
-struct hwrm_port_phy_i2c_read_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	uint32_t	flags;
-	uint32_t	enables;
-	/*
-	 * This bit must be '1' for the page_offset field to be
-	 * configured.
-	 */
-	#define HWRM_PORT_PHY_I2C_READ_INPUT_ENABLES_PAGE_OFFSET \
-		UINT32_C(0x1)
-	/* Port ID of port. */
-	uint16_t	port_id;
-	/* 8-bit I2C slave address. */
-	uint8_t	i2c_slave_addr;
-	uint8_t	unused_0;
-	/* The page number that is being accessed over I2C. */
-	uint16_t	page_number;
-	/* Offset within the page that is being accessed over I2C. */
-	uint16_t	page_offset;
-	/*
-	 * Length of data to read, in bytes starting at the offset
-	 * specified above. If the offset is not specified, then
-	 * the data shall be read from the beginning of the page.
-	 */
-	uint8_t	data_length;
-	uint8_t	unused_1[7];
-} __attribute__((packed));
-
-/* hwrm_port_phy_i2c_read_output (size:640b/80B) */
-struct hwrm_port_phy_i2c_read_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	/* Up to 64B of data. */
-	uint32_t	data[16];
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
 /*********************
  * hwrm_port_led_cfg *
  *********************/
@@ -17127,247 +18205,6 @@ struct hwrm_queue_cos2bw_cfg_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/*************************
- * hwrm_queue_dscp_qcaps *
- *************************/
-
-
-/* hwrm_queue_dscp_qcaps_input (size:192b/24B) */
-struct hwrm_queue_dscp_qcaps_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/*
-	 * Port ID of port for which the table is being configured.
-	 * The HWRM needs to check whether this function is allowed
-	 * to configure pri2cos mapping on this port.
-	 */
-	uint8_t	port_id;
-	uint8_t	unused_0[7];
-} __attribute__((packed));
-
-/* hwrm_queue_dscp_qcaps_output (size:128b/16B) */
-struct hwrm_queue_dscp_qcaps_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	/* The number of bits provided by the hardware for the DSCP value. */
-	uint8_t	num_dscp_bits;
-	uint8_t	unused_0;
-	/* Max number of DSCP-MASK-PRI entries supported. */
-	uint16_t	max_entries;
-	uint8_t	unused_1[3];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/****************************
- * hwrm_queue_dscp2pri_qcfg *
- ****************************/
-
-
-/* hwrm_queue_dscp2pri_qcfg_input (size:256b/32B) */
-struct hwrm_queue_dscp2pri_qcfg_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/*
-	 * This is the host address where the 24-bits DSCP-MASK-PRI
-	 * tuple(s) will be copied to.
-	 */
-	uint64_t	dest_data_addr;
-	/*
-	 * Port ID of port for which the table is being configured.
-	 * The HWRM needs to check whether this function is allowed
-	 * to configure pri2cos mapping on this port.
-	 */
-	uint8_t	port_id;
-	uint8_t	unused_0;
-	/* Size of the buffer pointed to by dest_data_addr. */
-	uint16_t	dest_data_buffer_size;
-	uint8_t	unused_1[4];
-} __attribute__((packed));
-
-/* hwrm_queue_dscp2pri_qcfg_output (size:128b/16B) */
-struct hwrm_queue_dscp2pri_qcfg_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	/*
-	 * A count of the number of DSCP-MASK-PRI tuple(s) pointed to
-	 * by the dest_data_addr.
-	 */
-	uint16_t	entry_cnt;
-	/*
-	 * This is the default PRI which un-initialized DSCP values are
-	 * mapped to.
-	 */
-	uint8_t	default_pri;
-	uint8_t	unused_0[4];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/***************************
- * hwrm_queue_dscp2pri_cfg *
- ***************************/
-
-
-/* hwrm_queue_dscp2pri_cfg_input (size:320b/40B) */
-struct hwrm_queue_dscp2pri_cfg_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/*
-	 * This is the host address where the 24-bits DSCP-MASK-PRI tuple
-	 * will be copied from.
-	 */
-	uint64_t	src_data_addr;
-	uint32_t	flags;
-	/* use_hw_default_pri is 1 b */
-	#define HWRM_QUEUE_DSCP2PRI_CFG_INPUT_FLAGS_USE_HW_DEFAULT_PRI \
-		UINT32_C(0x1)
-	uint32_t	enables;
-	/*
-	 * This bit must be '1' for the default_pri field to be
-	 * configured.
-	 */
-	#define HWRM_QUEUE_DSCP2PRI_CFG_INPUT_ENABLES_DEFAULT_PRI \
-		UINT32_C(0x1)
-	/*
-	 * Port ID of port for which the table is being configured.
-	 * The HWRM needs to check whether this function is allowed
-	 * to configure pri2cos mapping on this port.
-	 */
-	uint8_t	port_id;
-	/*
-	 * This is the default PRI which un-initialized DSCP values will be
-	 * mapped to.
-	 */
-	uint8_t	default_pri;
-	/*
-	 * A count of the number of DSCP-MASK-PRI tuple(s) in the data pointed
-	 * to by src_data_addr.
-	 */
-	uint16_t	entry_cnt;
-	uint8_t	unused_0[4];
-} __attribute__((packed));
-
-/* hwrm_queue_dscp2pri_cfg_output (size:128b/16B) */
-struct hwrm_queue_dscp2pri_cfg_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
 /*******************
  * hwrm_vnic_alloc *
  *******************/
@@ -18182,177 +19019,6 @@ struct hwrm_vnic_tpa_cfg_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/**********************
- * hwrm_vnic_tpa_qcfg *
- **********************/
-
-
-/* hwrm_vnic_tpa_qcfg_input (size:192b/24B) */
-struct hwrm_vnic_tpa_qcfg_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/* Logical vnic ID */
-	uint16_t	vnic_id;
-	uint8_t	unused_0[6];
-} __attribute__((packed));
-
-/* hwrm_vnic_tpa_qcfg_output (size:256b/32B) */
-struct hwrm_vnic_tpa_qcfg_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint32_t	flags;
-	/*
-	 * When this bit is '1', the VNIC is configured to
-	 * perform transparent packet aggregation (TPA) of
-	 * non-tunneled TCP packets.
-	 */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_FLAGS_TPA \
-		UINT32_C(0x1)
-	/*
-	 * When this bit is '1', the VNIC is configured to
-	 * perform transparent packet aggregation (TPA) of
-	 * tunneled TCP packets.
-	 */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_FLAGS_ENCAP_TPA \
-		UINT32_C(0x2)
-	/*
-	 * When this bit is '1', the VNIC is configured to
-	 * perform transparent packet aggregation (TPA) according
-	 * to Windows Receive Segment Coalescing (RSC) rules.
-	 */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_FLAGS_RSC_WND_UPDATE \
-		UINT32_C(0x4)
-	/*
-	 * When this bit is '1', the VNIC is configured to
-	 * perform transparent packet aggregation (TPA) according
-	 * to Linux Generic Receive Offload (GRO) rules.
-	 */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_FLAGS_GRO \
-		UINT32_C(0x8)
-	/*
-	 * When this bit is '1', the VNIC is configured to
-	 * perform transparent packet aggregation (TPA) for TCP
-	 * packets with IP ECN set to non-zero.
-	 */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_FLAGS_AGG_WITH_ECN \
-		UINT32_C(0x10)
-	/*
-	 * When this bit is '1', the VNIC is configured to
-	 * perform transparent packet aggregation (TPA) for
-	 * GRE tunneled TCP packets only if all packets have the
-	 * same GRE sequence.
-	 */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_FLAGS_AGG_WITH_SAME_GRE_SEQ \
-		UINT32_C(0x20)
-	/*
-	 * When this bit is '1' and the GRO mode is enabled,
-	 * the VNIC is configured to
-	 * perform transparent packet aggregation (TPA) for
-	 * TCP/IPv4 packets with consecutively increasing IPIDs.
-	 * In other words, the last packet that is being
-	 * aggregated to an already existing aggregation context
-	 * shall have IPID 1 more than the IPID of the last packet
-	 * that was aggregated in that aggregation context.
-	 */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_FLAGS_GRO_IPID_CHECK \
-		UINT32_C(0x40)
-	/*
-	 * When this bit is '1' and the GRO mode is enabled,
-	 * the VNIC is configured to
-	 * perform transparent packet aggregation (TPA) for
-	 * TCP packets with the same TTL (IPv4) or Hop limit (IPv6)
-	 * value.
-	 */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_FLAGS_GRO_TTL_CHECK \
-		UINT32_C(0x80)
-	/*
-	 * This is the maximum number of TCP segments that can
-	 * be aggregated (unit is Log2). Max value is 31.
-	 */
-	uint16_t	max_agg_segs;
-	/* 1 segment */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGG_SEGS_1   UINT32_C(0x0)
-	/* 2 segments */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGG_SEGS_2   UINT32_C(0x1)
-	/* 4 segments */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGG_SEGS_4   UINT32_C(0x2)
-	/* 8 segments */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGG_SEGS_8   UINT32_C(0x3)
-	/* Any segment size larger than this is not valid */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGG_SEGS_MAX UINT32_C(0x1f)
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGG_SEGS_LAST \
-		HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGG_SEGS_MAX
-	/*
-	 * This is the maximum number of aggregations this VNIC is
-	 * allowed (unit is Log2). Max value is 7
-	 */
-	uint16_t	max_aggs;
-	/* 1 aggregation */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGGS_1   UINT32_C(0x0)
-	/* 2 aggregations */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGGS_2   UINT32_C(0x1)
-	/* 4 aggregations */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGGS_4   UINT32_C(0x2)
-	/* 8 aggregations */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGGS_8   UINT32_C(0x3)
-	/* 16 aggregations */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGGS_16  UINT32_C(0x4)
-	/* Any aggregation size larger than this is not valid */
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGGS_MAX UINT32_C(0x7)
-	#define HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGGS_LAST \
-		HWRM_VNIC_TPA_QCFG_OUTPUT_MAX_AGGS_MAX
-	/*
-	 * This is the maximum amount of time allowed for
-	 * an aggregation context to complete after it was initiated.
-	 */
-	uint32_t	max_agg_timer;
-	/*
-	 * This is the minimum amount of payload length required to
-	 * start an aggregation context.
-	 */
-	uint32_t	min_agg_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
 /*********************
  * hwrm_vnic_rss_cfg *
  *********************/
@@ -19060,7 +19726,7 @@ struct hwrm_vnic_rss_cos_lb_ctx_free_output {
  *******************/
 
 
-/* hwrm_ring_alloc_input (size:640b/80B) */
+/* hwrm_ring_alloc_input (size:704b/88B) */
 struct hwrm_ring_alloc_input {
 	/* The HWRM command request type. */
 	uint16_t	req_type;
@@ -19142,7 +19808,18 @@ struct hwrm_ring_alloc_input {
 	#define HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ        UINT32_C(0x5)
 	#define HWRM_RING_ALLOC_INPUT_RING_TYPE_LAST \
 		HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ
-	uint8_t	unused_0[3];
+	uint8_t	unused_0;
+	/* Ring allocation flags. */
+	uint16_t	flags;
+	/*
+	 * For Rx rings, the incoming packet data can be placed at either
+	 * a 0B or 2B offset from the start of the Rx packet buffer. When
+	 * '1', the received packet will be padded with 2B of zeros at the
+	 * front of the packet. Note that this flag is only used for
+	 * Rx rings and is ignored for all other rings included Rx
+	 * Aggregation rings.
+	 */
+	#define HWRM_RING_ALLOC_INPUT_FLAGS_RX_SOP_PAD     UINT32_C(0x1)
 	/*
 	 * This value is a pointer to the page table for the
 	 * Ring.
@@ -19347,6 +20024,13 @@ struct hwrm_ring_alloc_input {
 	#define HWRM_RING_ALLOC_INPUT_INT_MODE_LAST \
 		HWRM_RING_ALLOC_INPUT_INT_MODE_POLL
 	uint8_t	unused_4[3];
+	/*
+	 * The cq_handle is specified when allocating a completion ring. For
+	 * devices that support NQs, this cq_handle will be included in the
+	 * NQE to specify which CQ should be read to retrieve the completion
+	 * record.
+	 */
+	uint64_t	cq_handle;
 } __attribute__((packed));
 
 /* hwrm_ring_alloc_output (size:128b/16B) */
@@ -19454,6 +20138,228 @@ struct hwrm_ring_free_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
+/*******************
+ * hwrm_ring_reset *
+ *******************/
+
+
+/* hwrm_ring_reset_input (size:192b/24B) */
+struct hwrm_ring_reset_input {
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/*
+	 * The completion ring to send the completion event on. This should
+	 * be the NQ ID returned from the `nq_alloc` HWRM command.
+	 */
+	uint16_t	cmpl_ring;
+	/*
+	 * The sequence ID is used by the driver for tracking multiple
+	 * commands. This ID is treated as opaque data by the firmware and
+	 * the value is returned in the `hwrm_resp_hdr` upon completion.
+	 */
+	uint16_t	seq_id;
+	/*
+	 * The target ID of the command:
+	 * * 0x0-0xFFF8 - The function ID
+	 * * 0xFFF8-0xFFFE - Reserved for internal processors
+	 * * 0xFFFF - HWRM
+	 */
+	uint16_t	target_id;
+	/*
+	 * A physical address pointer pointing to a host buffer that the
+	 * command's response data will be written. This can be either a host
+	 * physical address (HPA) or a guest physical address (GPA) and must
+	 * point to a physically contiguous block of memory.
+	 */
+	uint64_t	resp_addr;
+	/* Ring Type. */
+	uint8_t	ring_type;
+	/* L2 Completion Ring (CR) */
+	#define HWRM_RING_RESET_INPUT_RING_TYPE_L2_CMPL   UINT32_C(0x0)
+	/* TX Ring (TR) */
+	#define HWRM_RING_RESET_INPUT_RING_TYPE_TX        UINT32_C(0x1)
+	/* RX Ring (RR) */
+	#define HWRM_RING_RESET_INPUT_RING_TYPE_RX        UINT32_C(0x2)
+	/* RoCE Notification Completion Ring (ROCE_CR) */
+	#define HWRM_RING_RESET_INPUT_RING_TYPE_ROCE_CMPL UINT32_C(0x3)
+	#define HWRM_RING_RESET_INPUT_RING_TYPE_LAST \
+		HWRM_RING_RESET_INPUT_RING_TYPE_ROCE_CMPL
+	uint8_t	unused_0;
+	/* Physical number of the ring. */
+	uint16_t	ring_id;
+	uint8_t	unused_1[4];
+} __attribute__((packed));
+
+/* hwrm_ring_reset_output (size:128b/16B) */
+struct hwrm_ring_reset_output {
+	/* The specific error status for the command. */
+	uint16_t	error_code;
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/* The sequence ID from the original command. */
+	uint16_t	seq_id;
+	/* The length of the response data in number of bytes. */
+	uint16_t	resp_len;
+	uint8_t	unused_0[7];
+	/*
+	 * This field is used in Output records to indicate that the output
+	 * is completely written to RAM.  This field should be read as '1'
+	 * to indicate that the output has been completely written.
+	 * When writing a command completion or response to an internal processor,
+	 * the order of writes has to be such that this field is written last.
+	 */
+	uint8_t	valid;
+} __attribute__((packed));
+
+/**************************
+ * hwrm_ring_aggint_qcaps *
+ **************************/
+
+
+/* hwrm_ring_aggint_qcaps_input (size:128b/16B) */
+struct hwrm_ring_aggint_qcaps_input {
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/*
+	 * The completion ring to send the completion event on. This should
+	 * be the NQ ID returned from the `nq_alloc` HWRM command.
+	 */
+	uint16_t	cmpl_ring;
+	/*
+	 * The sequence ID is used by the driver for tracking multiple
+	 * commands. This ID is treated as opaque data by the firmware and
+	 * the value is returned in the `hwrm_resp_hdr` upon completion.
+	 */
+	uint16_t	seq_id;
+	/*
+	 * The target ID of the command:
+	 * * 0x0-0xFFF8 - The function ID
+	 * * 0xFFF8-0xFFFE - Reserved for internal processors
+	 * * 0xFFFF - HWRM
+	 */
+	uint16_t	target_id;
+	/*
+	 * A physical address pointer pointing to a host buffer that the
+	 * command's response data will be written. This can be either a host
+	 * physical address (HPA) or a guest physical address (GPA) and must
+	 * point to a physically contiguous block of memory.
+	 */
+	uint64_t	resp_addr;
+} __attribute__((packed));
+
+/* hwrm_ring_aggint_qcaps_output (size:384b/48B) */
+struct hwrm_ring_aggint_qcaps_output {
+	/* The specific error status for the command. */
+	uint16_t	error_code;
+	/* The HWRM command request type. */
+	uint16_t	req_type;
+	/* The sequence ID from the original command. */
+	uint16_t	seq_id;
+	/* The length of the response data in number of bytes. */
+	uint16_t	resp_len;
+	uint32_t	cmpl_params;
+	/*
+	 * When this bit is set to '1', int_lat_tmr_min can be configured
+	 * on completion rings.
+	 */
+	#define HWRM_RING_AGGINT_QCAPS_OUTPUT_CMPL_PARAMS_INT_LAT_TMR_MIN \
+		UINT32_C(0x1)
+	/*
+	 * When this bit is set to '1', int_lat_tmr_max can be configured
+	 * on completion rings.
+	 */
+	#define HWRM_RING_AGGINT_QCAPS_OUTPUT_CMPL_PARAMS_INT_LAT_TMR_MAX \
+		UINT32_C(0x2)
+	/*
+	 * When this bit is set to '1', timer_reset can be enabled
+	 * on completion rings.
+	 */
+	#define HWRM_RING_AGGINT_QCAPS_OUTPUT_CMPL_PARAMS_TIMER_RESET \
+		UINT32_C(0x4)
+	/*
+	 * When this bit is set to '1', ring_idle can be enabled
+	 * on completion rings.
+	 */
+	#define HWRM_RING_AGGINT_QCAPS_OUTPUT_CMPL_PARAMS_RING_IDLE \
+		UINT32_C(0x8)
+	/*
+	 * When this bit is set to '1', num_cmpl_dma_aggr can be configured
+	 * on completion rings.
+	 */
+	#define HWRM_RING_AGGINT_QCAPS_OUTPUT_CMPL_PARAMS_NUM_CMPL_DMA_AGGR \
+		UINT32_C(0x10)
+	/*
+	 * When this bit is set to '1', num_cmpl_dma_aggr_during_int can be configured
+	 * on completion rings.
+	 */
+	#define HWRM_RING_AGGINT_QCAPS_OUTPUT_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT \
+		UINT32_C(0x20)
+	/*
+	 * When this bit is set to '1', cmpl_aggr_dma_tmr can be configured
+	 * on completion rings.
+	 */
+	#define HWRM_RING_AGGINT_QCAPS_OUTPUT_CMPL_PARAMS_CMPL_AGGR_DMA_TMR \
+		UINT32_C(0x40)
+	/*
+	 * When this bit is set to '1', cmpl_aggr_dma_tmr_during_int can be configured
+	 * on completion rings.
+	 */
+	#define HWRM_RING_AGGINT_QCAPS_OUTPUT_CMPL_PARAMS_CMPL_AGGR_DMA_TMR_DURING_INT \
+		UINT32_C(0x80)
+	/*
+	 * When this bit is set to '1', num_cmpl_aggr_int can be configured
+	 * on completion rings.
+	 */
+	#define HWRM_RING_AGGINT_QCAPS_OUTPUT_CMPL_PARAMS_NUM_CMPL_AGGR_INT \
+		UINT32_C(0x100)
+	uint32_t	nq_params;
+	/*
+	 * When this bit is set to '1', int_lat_tmr_min can be configured
+	 * on notification queues.
+	 */
+	#define HWRM_RING_AGGINT_QCAPS_OUTPUT_NQ_PARAMS_INT_LAT_TMR_MIN \
+		UINT32_C(0x1)
+	/* Minimum value for num_cmpl_dma_aggr */
+	uint16_t	num_cmpl_dma_aggr_min;
+	/* Maximum value for num_cmpl_dma_aggr */
+	uint16_t	num_cmpl_dma_aggr_max;
+	/* Minimum value for num_cmpl_dma_aggr_during_int */
+	uint16_t	num_cmpl_dma_aggr_during_int_min;
+	/* Maximum value for num_cmpl_dma_aggr_during_int */
+	uint16_t	num_cmpl_dma_aggr_during_int_max;
+	/* Minimum value for cmpl_aggr_dma_tmr */
+	uint16_t	cmpl_aggr_dma_tmr_min;
+	/* Maximum value for cmpl_aggr_dma_tmr */
+	uint16_t	cmpl_aggr_dma_tmr_max;
+	/* Minimum value for cmpl_aggr_dma_tmr_during_int */
+	uint16_t	cmpl_aggr_dma_tmr_during_int_min;
+	/* Maximum value for cmpl_aggr_dma_tmr_during_int */
+	uint16_t	cmpl_aggr_dma_tmr_during_int_max;
+	/* Minimum value for int_lat_tmr_min */
+	uint16_t	int_lat_tmr_min_min;
+	/* Maximum value for int_lat_tmr_min */
+	uint16_t	int_lat_tmr_min_max;
+	/* Minimum value for int_lat_tmr_max */
+	uint16_t	int_lat_tmr_max_min;
+	/* Maximum value for int_lat_tmr_max */
+	uint16_t	int_lat_tmr_max_max;
+	/* Minimum value for num_cmpl_aggr_int */
+	uint16_t	num_cmpl_aggr_int_min;
+	/* Maximum value for num_cmpl_aggr_int */
+	uint16_t	num_cmpl_aggr_int_max;
+	/* The units for timer parameters, in nanoseconds. */
+	uint16_t	timer_units;
+	uint8_t	unused_0[1];
+	/*
+	 * This field is used in Output records to indicate that the output
+	 * is completely written to RAM.  This field should be read as '1'
+	 * to indicate that the output has been completely written.
+	 * When writing a command completion or response to an internal processor,
+	 * the order of writes has to be such that this field is written last.
+	 */
+	uint8_t	valid;
+} __attribute__((packed));
+
 /**************************************
  * hwrm_ring_cmpl_ring_qaggint_params *
  **************************************/
@@ -19716,79 +20622,6 @@ struct hwrm_ring_cmpl_ring_cfg_aggint_params_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/*******************
- * hwrm_ring_reset *
- *******************/
-
-
-/* hwrm_ring_reset_input (size:192b/24B) */
-struct hwrm_ring_reset_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/* Ring Type. */
-	uint8_t	ring_type;
-	/* L2 Completion Ring (CR) */
-	#define HWRM_RING_RESET_INPUT_RING_TYPE_L2_CMPL   UINT32_C(0x0)
-	/* TX Ring (TR) */
-	#define HWRM_RING_RESET_INPUT_RING_TYPE_TX        UINT32_C(0x1)
-	/* RX Ring (RR) */
-	#define HWRM_RING_RESET_INPUT_RING_TYPE_RX        UINT32_C(0x2)
-	/* RoCE Notification Completion Ring (ROCE_CR) */
-	#define HWRM_RING_RESET_INPUT_RING_TYPE_ROCE_CMPL UINT32_C(0x3)
-	#define HWRM_RING_RESET_INPUT_RING_TYPE_LAST \
-		HWRM_RING_RESET_INPUT_RING_TYPE_ROCE_CMPL
-	uint8_t	unused_0;
-	/* Physical number of the ring. */
-	uint16_t	ring_id;
-	uint8_t	unused_1[4];
-} __attribute__((packed));
-
-/* hwrm_ring_reset_output (size:128b/16B) */
-struct hwrm_ring_reset_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
 /***********************
  * hwrm_ring_grp_alloc *
  ***********************/
@@ -20000,6 +20833,24 @@ struct hwrm_cfa_l2_filter_alloc_input {
 	 */
 	#define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST \
 		UINT32_C(0x8)
+	/*
+	 * Enumeration denoting NO_ROCE_L2 to support old drivers.
+	 * New driver L2 for only L2 traffic, ROCE for roce and l2 traffic
+	 */
+	#define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_TRAFFIC_MASK \
+		UINT32_C(0x30)
+	#define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_TRAFFIC_SFT       4
+	/* To support old drivers */
+	#define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_TRAFFIC_NO_ROCE_L2 \
+		(UINT32_C(0x0) << 4)
+	/* Only L2 traffic */
+	#define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_TRAFFIC_L2 \
+		(UINT32_C(0x1) << 4)
+	/* Roce & L2 traffic */
+	#define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_TRAFFIC_ROCE \
+		(UINT32_C(0x2) << 4)
+	#define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_TRAFFIC_LAST \
+		HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_TRAFFIC_ROCE
 	uint32_t	enables;
 	/*
 	 * This bit must be '1' for the l2_addr field to be
@@ -20419,18 +21270,40 @@ struct hwrm_cfa_l2_filter_cfg_input {
 	 * This enumeration is used for resources that are similar for both
 	 * TX and RX paths of the chip.
 	 */
-	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH     UINT32_C(0x1)
+	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH \
+		UINT32_C(0x1)
 	/* tx path */
-	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_TX    UINT32_C(0x0)
+	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_TX \
+		UINT32_C(0x0)
 	/* rx path */
-	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_RX    UINT32_C(0x1)
+	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_RX \
+		UINT32_C(0x1)
 	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_LAST \
 		HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_RX
 	/*
 	 * Setting of this flag indicates drop action. If this flag is not set,
 	 * then it should be considered accept action.
 	 */
-	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_DROP     UINT32_C(0x2)
+	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_DROP \
+		UINT32_C(0x2)
+	/*
+	 * Enumeration denoting NO_ROCE_L2 to support old drivers.
+	 * New driver L2 for only L2 traffic, ROCE for roce and l2 traffic
+	 */
+	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_TRAFFIC_MASK \
+		UINT32_C(0xc)
+	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_TRAFFIC_SFT       2
+	/* To support old drivers */
+	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_TRAFFIC_NO_ROCE_L2 \
+		(UINT32_C(0x0) << 2)
+	/* Only L2 traffic */
+	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_TRAFFIC_L2 \
+		(UINT32_C(0x1) << 2)
+	/* Roce & L2 traffic */
+	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_TRAFFIC_ROCE \
+		(UINT32_C(0x2) << 2)
+	#define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_TRAFFIC_LAST \
+		HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_TRAFFIC_ROCE
 	uint32_t	enables;
 	/*
 	 * This bit must be '1' for the dst_id field to be
@@ -21532,7 +22405,7 @@ struct hwrm_vxlan_ipv6_hdr {
 	uint32_t	dest_ip_addr[4];
 } __attribute__((packed));
 
-/* hwrm_cfa_encap_data_vxlan (size:576b/72B) */
+/* hwrm_cfa_encap_data_vxlan (size:640b/80B) */
 struct hwrm_cfa_encap_data_vxlan {
 	/* Source MAC address. */
 	uint8_t	src_mac_addr[6];
@@ -21568,6 +22441,13 @@ struct hwrm_cfa_encap_data_vxlan {
 	uint16_t	dst_port;
 	/* VXLAN Network Identifier. */
 	uint32_t	vni;
+	/* 3 bytes VXLAN header reserve fields from 1st dword of the VXLAN header. */
+	uint8_t	hdr_rsvd0[3];
+	/* 1 byte VXLAN header reserve field from 2nd dword of the VXLAN header. */
+	uint8_t	hdr_rsvd1;
+	/* VXLAN header flags field. */
+	uint8_t	hdr_flags;
+	uint8_t	unused[3];
 } __attribute__((packed));
 
 /*******************************
@@ -21634,8 +22514,11 @@ struct hwrm_cfa_encap_record_alloc_input {
 	/* Generic Routing Encapsulation (GRE) inside IP datagram payload */
 	#define HWRM_CFA_ENCAP_RECORD_ALLOC_INPUT_ENCAP_TYPE_IPGRE \
 		UINT32_C(0x8)
+	/* IPV4 over virtual eXtensible Local Area Network (IPV4oVXLAN) */
+	#define HWRM_CFA_ENCAP_RECORD_ALLOC_INPUT_ENCAP_TYPE_VXLAN_V4 \
+		UINT32_C(0x9)
 	#define HWRM_CFA_ENCAP_RECORD_ALLOC_INPUT_ENCAP_TYPE_LAST \
-		HWRM_CFA_ENCAP_RECORD_ALLOC_INPUT_ENCAP_TYPE_IPGRE
+		HWRM_CFA_ENCAP_RECORD_ALLOC_INPUT_ENCAP_TYPE_VXLAN_V4
 	uint8_t	unused_0[3];
 	/* This value is encap data used for the given encap type. */
 	uint32_t	encap_data[20];
@@ -22703,922 +23586,6 @@ struct hwrm_cfa_em_flow_free_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/************************
- * hwrm_cfa_em_flow_cfg *
- ************************/
-
-
-/* hwrm_cfa_em_flow_cfg_input (size:384b/48B) */
-struct hwrm_cfa_em_flow_cfg_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	uint32_t	enables;
-	/*
-	 * This bit must be '1' for the new_dst_id field to be
-	 * configured.
-	 */
-	#define HWRM_CFA_EM_FLOW_CFG_INPUT_ENABLES_NEW_DST_ID \
-		UINT32_C(0x1)
-	/*
-	 * This bit must be '1' for the new_mirror_vnic_id field to be
-	 * configured.
-	 */
-	#define HWRM_CFA_EM_FLOW_CFG_INPUT_ENABLES_NEW_MIRROR_VNIC_ID \
-		UINT32_C(0x2)
-	/*
-	 * This bit must be '1' for the new_meter_instance_id field to be
-	 * configured.
-	 */
-	#define HWRM_CFA_EM_FLOW_CFG_INPUT_ENABLES_NEW_METER_INSTANCE_ID \
-		UINT32_C(0x4)
-	uint8_t	unused_0[4];
-	/* This value is an opaque id into CFA data structures. */
-	uint64_t	em_filter_id;
-	/*
-	 * If set, this value shall represent the new
-	 * Logical VNIC ID of the destination VNIC for the RX
-	 * path and network port id of the destination port for
-	 * the TX path.
-	 */
-	uint32_t	new_dst_id;
-	/*
-	 * New Logical VNIC ID of the VNIC where traffic is
-	 * mirrored.
-	 */
-	uint32_t	new_mirror_vnic_id;
-	/*
-	 * New meter to attach to the flow. Specifying the
-	 * invalid instance ID is used to remove any existing
-	 * meter from the flow.
-	 */
-	uint16_t	new_meter_instance_id;
-	/*
-	 * A value of 0xfff is considered invalid and implies the
-	 * instance is not configured.
-	 */
-	#define HWRM_CFA_EM_FLOW_CFG_INPUT_NEW_METER_INSTANCE_ID_INVALID \
-		UINT32_C(0xffff)
-	#define HWRM_CFA_EM_FLOW_CFG_INPUT_NEW_METER_INSTANCE_ID_LAST \
-		HWRM_CFA_EM_FLOW_CFG_INPUT_NEW_METER_INSTANCE_ID_INVALID
-	uint8_t	unused_1[6];
-} __attribute__((packed));
-
-/* hwrm_cfa_em_flow_cfg_output (size:128b/16B) */
-struct hwrm_cfa_em_flow_cfg_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/********************************
- * hwrm_cfa_meter_profile_alloc *
- ********************************/
-
-
-/* hwrm_cfa_meter_profile_alloc_input (size:320b/40B) */
-struct hwrm_cfa_meter_profile_alloc_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	uint8_t	flags;
-	/*
-	 * Enumeration denoting the RX, TX type of the resource.
-	 * This enumeration is used for resources that are similar for both
-	 * TX and RX paths of the chip.
-	 */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_FLAGS_PATH     UINT32_C(0x1)
-	/* tx path */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_FLAGS_PATH_TX \
-		UINT32_C(0x0)
-	/* rx path */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_FLAGS_PATH_RX \
-		UINT32_C(0x1)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_FLAGS_PATH_LAST \
-		HWRM_CFA_METER_PROFILE_ALLOC_INPUT_FLAGS_PATH_RX
-	/* The meter algorithm type. */
-	uint8_t	meter_type;
-	/* RFC 2697 (srTCM) */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_METER_TYPE_RFC2697 \
-		UINT32_C(0x0)
-	/* RFC 2698 (trTCM) */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_METER_TYPE_RFC2698 \
-		UINT32_C(0x1)
-	/* RFC 4115 (trTCM) */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_METER_TYPE_RFC4115 \
-		UINT32_C(0x2)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_METER_TYPE_LAST \
-		HWRM_CFA_METER_PROFILE_ALLOC_INPUT_METER_TYPE_RFC4115
-	/*
-	 * This field is reserved for the future use.
-	 * It shall be set to 0.
-	 */
-	uint16_t	reserved1;
-	/*
-	 * This field is reserved for the future use.
-	 * It shall be set to 0.
-	 */
-	uint32_t	reserved2;
-	/* A meter rate specified in bytes-per-second. */
-	uint32_t	commit_rate;
-	/* The bandwidth value. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_MASK \
-		UINT32_C(0xfffffff)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_SFT \
-		0
-	/* The granularity of the value (bits or bytes). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_SCALE \
-		UINT32_C(0x10000000)
-	/* Value is in bits. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_SCALE_BITS \
-		(UINT32_C(0x0) << 28)
-	/* Value is in bytes. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_SCALE_BYTES \
-		(UINT32_C(0x1) << 28)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_SCALE_LAST \
-		HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_SCALE_BYTES
-	/* bw_value_unit is 3 b */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_MASK \
-		UINT32_C(0xe0000000)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_SFT \
-		29
-	/* Value is in Mb or MB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_MEGA \
-		(UINT32_C(0x0) << 29)
-	/* Value is in Kb or KB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_KILO \
-		(UINT32_C(0x2) << 29)
-	/* Value is in bits or bytes. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_BASE \
-		(UINT32_C(0x4) << 29)
-	/* Value is in Gb or GB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_GIGA \
-		(UINT32_C(0x6) << 29)
-	/* Value is in 1/100th of a percentage of total bandwidth. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_PERCENT1_100 \
-		(UINT32_C(0x1) << 29)
-	/* Invalid unit */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_INVALID \
-		(UINT32_C(0x7) << 29)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_LAST \
-		HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_INVALID
-	/* A meter burst size specified in bytes. */
-	uint32_t	commit_burst;
-	/* The bandwidth value. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_MASK \
-		UINT32_C(0xfffffff)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_SFT \
-		0
-	/* The granularity of the value (bits or bytes). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_SCALE \
-		UINT32_C(0x10000000)
-	/* Value is in bits. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_SCALE_BITS \
-		(UINT32_C(0x0) << 28)
-	/* Value is in bytes. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_SCALE_BYTES \
-		(UINT32_C(0x1) << 28)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_SCALE_LAST \
-		HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_SCALE_BYTES
-	/* bw_value_unit is 3 b */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_MASK \
-		UINT32_C(0xe0000000)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_SFT \
-		29
-	/* Value is in Mb or MB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_MEGA \
-		(UINT32_C(0x0) << 29)
-	/* Value is in Kb or KB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_KILO \
-		(UINT32_C(0x2) << 29)
-	/* Value is in bits or bytes. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_BASE \
-		(UINT32_C(0x4) << 29)
-	/* Value is in Gb or GB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_GIGA \
-		(UINT32_C(0x6) << 29)
-	/* Value is in 1/100th of a percentage of total bandwidth. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_PERCENT1_100 \
-		(UINT32_C(0x1) << 29)
-	/* Invalid unit */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_INVALID \
-		(UINT32_C(0x7) << 29)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_LAST \
-		HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_INVALID
-	/* A meter rate specified in bytes-per-second. */
-	uint32_t	excess_peak_rate;
-	/* The bandwidth value. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_MASK \
-		UINT32_C(0xfffffff)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_SFT \
-		0
-	/* The granularity of the value (bits or bytes). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_SCALE \
-		UINT32_C(0x10000000)
-	/* Value is in bits. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_SCALE_BITS \
-		(UINT32_C(0x0) << 28)
-	/* Value is in bytes. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_SCALE_BYTES \
-		(UINT32_C(0x1) << 28)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_SCALE_LAST \
-		HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_SCALE_BYTES
-	/* bw_value_unit is 3 b */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_MASK \
-		UINT32_C(0xe0000000)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_SFT \
-		29
-	/* Value is in Mb or MB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_MEGA \
-		(UINT32_C(0x0) << 29)
-	/* Value is in Kb or KB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_KILO \
-		(UINT32_C(0x2) << 29)
-	/* Value is in bits or bytes. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_BASE \
-		(UINT32_C(0x4) << 29)
-	/* Value is in Gb or GB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_GIGA \
-		(UINT32_C(0x6) << 29)
-	/* Value is in 1/100th of a percentage of total bandwidth. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_PERCENT1_100 \
-		(UINT32_C(0x1) << 29)
-	/* Invalid unit */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_INVALID \
-		(UINT32_C(0x7) << 29)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_LAST \
-		HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_INVALID
-	/* A meter burst size specified in bytes. */
-	uint32_t	excess_peak_burst;
-	/* The bandwidth value. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_MASK \
-		UINT32_C(0xfffffff)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_SFT \
-		0
-	/* The granularity of the value (bits or bytes). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_SCALE \
-		UINT32_C(0x10000000)
-	/* Value is in bits. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_SCALE_BITS \
-		(UINT32_C(0x0) << 28)
-	/* Value is in bytes. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_SCALE_BYTES \
-		(UINT32_C(0x1) << 28)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_SCALE_LAST \
-		HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_SCALE_BYTES
-	/* bw_value_unit is 3 b */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_MASK \
-		UINT32_C(0xe0000000)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_SFT \
-		29
-	/* Value is in Mb or MB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_MEGA \
-		(UINT32_C(0x0) << 29)
-	/* Value is in Kb or KB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_KILO \
-		(UINT32_C(0x2) << 29)
-	/* Value is in bits or bytes. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_BASE \
-		(UINT32_C(0x4) << 29)
-	/* Value is in Gb or GB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_GIGA \
-		(UINT32_C(0x6) << 29)
-	/* Value is in 1/100th of a percentage of total bandwidth. */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_PERCENT1_100 \
-		(UINT32_C(0x1) << 29)
-	/* Invalid unit */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_INVALID \
-		(UINT32_C(0x7) << 29)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_LAST \
-		HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_INVALID
-} __attribute__((packed));
-
-/* hwrm_cfa_meter_profile_alloc_output (size:128b/16B) */
-struct hwrm_cfa_meter_profile_alloc_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	/* This value identifies a meter profile in CFA. */
-	uint16_t	meter_profile_id;
-	/*
-	 * A value of 0xfff is considered invalid and implies the
-	 * profile is not configured.
-	 */
-	#define HWRM_CFA_METER_PROFILE_ALLOC_OUTPUT_METER_PROFILE_ID_INVALID \
-		UINT32_C(0xffff)
-	#define HWRM_CFA_METER_PROFILE_ALLOC_OUTPUT_METER_PROFILE_ID_LAST \
-		HWRM_CFA_METER_PROFILE_ALLOC_OUTPUT_METER_PROFILE_ID_INVALID
-	uint8_t	unused_0[5];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/*******************************
- * hwrm_cfa_meter_profile_free *
- *******************************/
-
-
-/* hwrm_cfa_meter_profile_free_input (size:192b/24B) */
-struct hwrm_cfa_meter_profile_free_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	uint8_t	flags;
-	/*
-	 * Enumeration denoting the RX, TX type of the resource.
-	 * This enumeration is used for resources that are similar for both
-	 * TX and RX paths of the chip.
-	 */
-	#define HWRM_CFA_METER_PROFILE_FREE_INPUT_FLAGS_PATH     UINT32_C(0x1)
-	/* tx path */
-	#define HWRM_CFA_METER_PROFILE_FREE_INPUT_FLAGS_PATH_TX \
-		UINT32_C(0x0)
-	/* rx path */
-	#define HWRM_CFA_METER_PROFILE_FREE_INPUT_FLAGS_PATH_RX \
-		UINT32_C(0x1)
-	#define HWRM_CFA_METER_PROFILE_FREE_INPUT_FLAGS_PATH_LAST \
-		HWRM_CFA_METER_PROFILE_FREE_INPUT_FLAGS_PATH_RX
-	uint8_t	unused_0;
-	/* This value identifies a meter profile in CFA. */
-	uint16_t	meter_profile_id;
-	/*
-	 * A value of 0xfff is considered invalid and implies the
-	 * profile is not configured.
-	 */
-	#define HWRM_CFA_METER_PROFILE_FREE_INPUT_METER_PROFILE_ID_INVALID \
-		UINT32_C(0xffff)
-	#define HWRM_CFA_METER_PROFILE_FREE_INPUT_METER_PROFILE_ID_LAST \
-		HWRM_CFA_METER_PROFILE_FREE_INPUT_METER_PROFILE_ID_INVALID
-	uint8_t	unused_1[4];
-} __attribute__((packed));
-
-/* hwrm_cfa_meter_profile_free_output (size:128b/16B) */
-struct hwrm_cfa_meter_profile_free_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/******************************
- * hwrm_cfa_meter_profile_cfg *
- ******************************/
-
-
-/* hwrm_cfa_meter_profile_cfg_input (size:320b/40B) */
-struct hwrm_cfa_meter_profile_cfg_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	uint8_t	flags;
-	/*
-	 * Enumeration denoting the RX, TX type of the resource.
-	 * This enumeration is used for resources that are similar for both
-	 * TX and RX paths of the chip.
-	 */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_FLAGS_PATH     UINT32_C(0x1)
-	/* tx path */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_FLAGS_PATH_TX    UINT32_C(0x0)
-	/* rx path */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_FLAGS_PATH_RX    UINT32_C(0x1)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_FLAGS_PATH_LAST \
-		HWRM_CFA_METER_PROFILE_CFG_INPUT_FLAGS_PATH_RX
-	/* The meter algorithm type. */
-	uint8_t	meter_type;
-	/* RFC 2697 (srTCM) */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_TYPE_RFC2697 \
-		UINT32_C(0x0)
-	/* RFC 2698 (trTCM) */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_TYPE_RFC2698 \
-		UINT32_C(0x1)
-	/* RFC 4115 (trTCM) */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_TYPE_RFC4115 \
-		UINT32_C(0x2)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_TYPE_LAST \
-		HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_TYPE_RFC4115
-	/* This value identifies a meter profile in CFA. */
-	uint16_t	meter_profile_id;
-	/*
-	 * A value of 0xfff is considered invalid and implies the
-	 * profile is not configured.
-	 */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_PROFILE_ID_INVALID \
-		UINT32_C(0xffff)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_PROFILE_ID_LAST \
-		HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_PROFILE_ID_INVALID
-	/*
-	 * This field is reserved for the future use.
-	 * It shall be set to 0.
-	 */
-	uint32_t	reserved;
-	/* A meter rate specified in bytes-per-second. */
-	uint32_t	commit_rate;
-	/* The bandwidth value. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_MASK \
-		UINT32_C(0xfffffff)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_SFT \
-		0
-	/* The granularity of the value (bits or bytes). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_SCALE \
-		UINT32_C(0x10000000)
-	/* Value is in bits. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_SCALE_BITS \
-		(UINT32_C(0x0) << 28)
-	/* Value is in bytes. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_SCALE_BYTES \
-		(UINT32_C(0x1) << 28)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_SCALE_LAST \
-		HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_SCALE_BYTES
-	/* bw_value_unit is 3 b */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_MASK \
-		UINT32_C(0xe0000000)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_SFT \
-		29
-	/* Value is in Mb or MB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_MEGA \
-		(UINT32_C(0x0) << 29)
-	/* Value is in Kb or KB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_KILO \
-		(UINT32_C(0x2) << 29)
-	/* Value is in bits or bytes. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_BASE \
-		(UINT32_C(0x4) << 29)
-	/* Value is in Gb or GB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_GIGA \
-		(UINT32_C(0x6) << 29)
-	/* Value is in 1/100th of a percentage of total bandwidth. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_PERCENT1_100 \
-		(UINT32_C(0x1) << 29)
-	/* Invalid unit */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_INVALID \
-		(UINT32_C(0x7) << 29)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_LAST \
-		HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_INVALID
-	/* A meter burst size specified in bytes. */
-	uint32_t	commit_burst;
-	/* The bandwidth value. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_MASK \
-		UINT32_C(0xfffffff)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_SFT \
-		0
-	/* The granularity of the value (bits or bytes). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_SCALE \
-		UINT32_C(0x10000000)
-	/* Value is in bits. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_SCALE_BITS \
-		(UINT32_C(0x0) << 28)
-	/* Value is in bytes. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_SCALE_BYTES \
-		(UINT32_C(0x1) << 28)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_SCALE_LAST \
-		HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_SCALE_BYTES
-	/* bw_value_unit is 3 b */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_MASK \
-		UINT32_C(0xe0000000)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_SFT \
-		29
-	/* Value is in Mb or MB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_MEGA \
-		(UINT32_C(0x0) << 29)
-	/* Value is in Kb or KB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_KILO \
-		(UINT32_C(0x2) << 29)
-	/* Value is in bits or bytes. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_BASE \
-		(UINT32_C(0x4) << 29)
-	/* Value is in Gb or GB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_GIGA \
-		(UINT32_C(0x6) << 29)
-	/* Value is in 1/100th of a percentage of total bandwidth. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_PERCENT1_100 \
-		(UINT32_C(0x1) << 29)
-	/* Invalid unit */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_INVALID \
-		(UINT32_C(0x7) << 29)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_LAST \
-		HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_INVALID
-	/* A meter rate specified in bytes-per-second. */
-	uint32_t	excess_peak_rate;
-	/* The bandwidth value. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_MASK \
-		UINT32_C(0xfffffff)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_SFT \
-		0
-	/* The granularity of the value (bits or bytes). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_SCALE \
-		UINT32_C(0x10000000)
-	/* Value is in bits. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_SCALE_BITS \
-		(UINT32_C(0x0) << 28)
-	/* Value is in bytes. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_SCALE_BYTES \
-		(UINT32_C(0x1) << 28)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_SCALE_LAST \
-		HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_SCALE_BYTES
-	/* bw_value_unit is 3 b */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_MASK \
-		UINT32_C(0xe0000000)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_SFT \
-		29
-	/* Value is in Mb or MB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_MEGA \
-		(UINT32_C(0x0) << 29)
-	/* Value is in Kb or KB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_KILO \
-		(UINT32_C(0x2) << 29)
-	/* Value is in bits or bytes. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_BASE \
-		(UINT32_C(0x4) << 29)
-	/* Value is in Gb or GB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_GIGA \
-		(UINT32_C(0x6) << 29)
-	/* Value is in 1/100th of a percentage of total bandwidth. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_PERCENT1_100 \
-		(UINT32_C(0x1) << 29)
-	/* Invalid unit */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_INVALID \
-		(UINT32_C(0x7) << 29)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_LAST \
-		HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_INVALID
-	/* A meter burst size specified in bytes. */
-	uint32_t	excess_peak_burst;
-	/* The bandwidth value. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_MASK \
-		UINT32_C(0xfffffff)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_SFT \
-		0
-	/* The granularity of the value (bits or bytes). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_SCALE \
-		UINT32_C(0x10000000)
-	/* Value is in bits. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_SCALE_BITS \
-		(UINT32_C(0x0) << 28)
-	/* Value is in bytes. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_SCALE_BYTES \
-		(UINT32_C(0x1) << 28)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_SCALE_LAST \
-		HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_SCALE_BYTES
-	/* bw_value_unit is 3 b */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_MASK \
-		UINT32_C(0xe0000000)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_SFT \
-		29
-	/* Value is in Mb or MB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_MEGA \
-		(UINT32_C(0x0) << 29)
-	/* Value is in Kb or KB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_KILO \
-		(UINT32_C(0x2) << 29)
-	/* Value is in bits or bytes. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_BASE \
-		(UINT32_C(0x4) << 29)
-	/* Value is in Gb or GB (base 10). */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_GIGA \
-		(UINT32_C(0x6) << 29)
-	/* Value is in 1/100th of a percentage of total bandwidth. */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_PERCENT1_100 \
-		(UINT32_C(0x1) << 29)
-	/* Invalid unit */
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_INVALID \
-		(UINT32_C(0x7) << 29)
-	#define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_LAST \
-		HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_INVALID
-} __attribute__((packed));
-
-/* hwrm_cfa_meter_profile_cfg_output (size:128b/16B) */
-struct hwrm_cfa_meter_profile_cfg_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/*********************************
- * hwrm_cfa_meter_instance_alloc *
- *********************************/
-
-
-/* hwrm_cfa_meter_instance_alloc_input (size:192b/24B) */
-struct hwrm_cfa_meter_instance_alloc_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	uint8_t	flags;
-	/*
-	 * Enumeration denoting the RX, TX type of the resource.
-	 * This enumeration is used for resources that are similar for both
-	 * TX and RX paths of the chip.
-	 */
-	#define HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_FLAGS_PATH \
-		UINT32_C(0x1)
-	/* tx path */
-	#define HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_FLAGS_PATH_TX \
-		UINT32_C(0x0)
-	/* rx path */
-	#define HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_FLAGS_PATH_RX \
-		UINT32_C(0x1)
-	#define HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_FLAGS_PATH_LAST \
-		HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_FLAGS_PATH_RX
-	uint8_t	unused_0;
-	/* This value identifies a meter profile in CFA. */
-	uint16_t	meter_profile_id;
-	/*
-	 * A value of 0xfff is considered invalid and implies the
-	 * profile is not configured.
-	 */
-	#define HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_METER_PROFILE_ID_INVALID \
-		UINT32_C(0xffff)
-	#define HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_METER_PROFILE_ID_LAST \
-		HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_METER_PROFILE_ID_INVALID
-	uint8_t	unused_1[4];
-} __attribute__((packed));
-
-/* hwrm_cfa_meter_instance_alloc_output (size:128b/16B) */
-struct hwrm_cfa_meter_instance_alloc_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	/* This value identifies a meter instance in CFA. */
-	uint16_t	meter_instance_id;
-	/*
-	 * A value of 0xfff is considered invalid and implies the
-	 * instance is not configured.
-	 */
-	#define HWRM_CFA_METER_INSTANCE_ALLOC_OUTPUT_METER_INSTANCE_ID_INVALID \
-		UINT32_C(0xffff)
-	#define HWRM_CFA_METER_INSTANCE_ALLOC_OUTPUT_METER_INSTANCE_ID_LAST \
-		HWRM_CFA_METER_INSTANCE_ALLOC_OUTPUT_METER_INSTANCE_ID_INVALID
-	uint8_t	unused_0[5];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/********************************
- * hwrm_cfa_meter_instance_free *
- ********************************/
-
-
-/* hwrm_cfa_meter_instance_free_input (size:192b/24B) */
-struct hwrm_cfa_meter_instance_free_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	uint8_t	flags;
-	/*
-	 * Enumeration denoting the RX, TX type of the resource.
-	 * This enumeration is used for resources that are similar for both
-	 * TX and RX paths of the chip.
-	 */
-	#define HWRM_CFA_METER_INSTANCE_FREE_INPUT_FLAGS_PATH     UINT32_C(0x1)
-	/* tx path */
-	#define HWRM_CFA_METER_INSTANCE_FREE_INPUT_FLAGS_PATH_TX \
-		UINT32_C(0x0)
-	/* rx path */
-	#define HWRM_CFA_METER_INSTANCE_FREE_INPUT_FLAGS_PATH_RX \
-		UINT32_C(0x1)
-	#define HWRM_CFA_METER_INSTANCE_FREE_INPUT_FLAGS_PATH_LAST \
-		HWRM_CFA_METER_INSTANCE_FREE_INPUT_FLAGS_PATH_RX
-	uint8_t	unused_0;
-	/* This value identifies a meter instance in CFA. */
-	uint16_t	meter_instance_id;
-	/*
-	 * A value of 0xfff is considered invalid and implies the
-	 * instance is not configured.
-	 */
-	#define HWRM_CFA_METER_INSTANCE_FREE_INPUT_METER_INSTANCE_ID_INVALID \
-		UINT32_C(0xffff)
-	#define HWRM_CFA_METER_INSTANCE_FREE_INPUT_METER_INSTANCE_ID_LAST \
-		HWRM_CFA_METER_INSTANCE_FREE_INPUT_METER_INSTANCE_ID_INVALID
-	uint8_t	unused_1[4];
-} __attribute__((packed));
-
-/* hwrm_cfa_meter_instance_free_output (size:128b/16B) */
-struct hwrm_cfa_meter_instance_free_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
 /*******************************
  * hwrm_cfa_decap_filter_alloc *
  *******************************/
@@ -24030,10 +23997,12 @@ struct hwrm_cfa_flow_alloc_input {
 	uint64_t	resp_addr;
 	uint16_t	flags;
 	/* tunnel is 1 b */
-	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_TUNNEL       UINT32_C(0x1)
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_TUNNEL \
+		UINT32_C(0x1)
 	/* num_vlan is 2 b */
-	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_MASK UINT32_C(0x6)
-	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_SFT 1
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_MASK \
+		UINT32_C(0x6)
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_SFT           1
 	/* no tags */
 	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_NONE \
 		(UINT32_C(0x0) << 1)
@@ -24046,8 +24015,9 @@ struct hwrm_cfa_flow_alloc_input {
 	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_LAST \
 		HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_TWO
 	/* Enumeration denoting the Flow Type. */
-	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_MASK UINT32_C(0x38)
-	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_SFT 3
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_MASK \
+		UINT32_C(0x38)
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_SFT           3
 	/* L2 flow */
 	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_L2 \
 		(UINT32_C(0x0) << 3)
@@ -24060,6 +24030,29 @@ struct hwrm_cfa_flow_alloc_input {
 	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_LAST \
 		HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_IPV6
 	/*
+	 * when set to 1, indicates TX flow offload for function specified in src_fid and
+	 * the dst_fid should be set to invalid value. To indicate a VM to VM flow, both
+	 * of the path_tx and path_rx flags need to be set. For virtio vSwitch offload
+	 * case, the src_fid and dst_fid is set to the same fid value. For the SRIOV
+	 * vSwitch offload case, the src_fid and dst_fid must be set to the same VF FID
+	 * belong to the children VFs of the same PF to indicate VM to VM flow.
+	 */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_PATH_TX \
+		UINT32_C(0x40)
+	/*
+	 * when set to 1, indicates RX flow offload for function specified in dst_fid and
+	 * the src_fid should be set to invalid value.
+	 */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_PATH_RX \
+		UINT32_C(0x80)
+	/*
+	 * Set to 1 to indicate matching of VXLAN VNI from the custom vxlan header is
+	 * required and the VXLAN VNI value is stored in the first 24 bits of the dmac field.
+	 * This flag is only valid when the flow direction is RX.
+	 */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_MATCH_VXLAN_IP_VNI \
+		UINT32_C(0x100)
+	/*
 	 * Tx Flow: vf fid.
 	 * Rx Flow: pf fid.
 	 */
@@ -24104,6 +24097,14 @@ struct hwrm_cfa_flow_alloc_input {
 	#define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_TTL_DECREMENT \
 		UINT32_C(0x200)
 	/*
+	 * If set to 1 and flow direction is TX, it indicates decap of L2 header
+	 * and encap of tunnel header. If set to 1 and flow direction is RX, it
+	 * indicates decap of tunnel header and encap L2 header. The type of tunnel
+	 * is specified in the tunnel_type field.
+	 */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_TUNNEL_IP \
+		UINT32_C(0x400)
+	/*
 	 * Tx Flow: pf or vf fid.
 	 * Rx Flow: vf fid.
 	 */
@@ -24171,10 +24172,35 @@ struct hwrm_cfa_flow_alloc_input {
 	uint16_t	l2_rewrite_smac[3];
 	/* The value of ip protocol. */
 	uint8_t	ip_proto;
-	uint8_t	unused_0;
+	/* Tunnel Type. */
+	uint8_t	tunnel_type;
+	/* Non-tunnel */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_NONTUNNEL UINT32_C(0x0)
+	/* Virtual eXtensible Local Area Network (VXLAN) */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_VXLAN     UINT32_C(0x1)
+	/* Network Virtualization Generic Routing Encapsulation (NVGRE) */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_NVGRE     UINT32_C(0x2)
+	/* Generic Routing Encapsulation (GRE) inside Ethernet payload */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_L2GRE     UINT32_C(0x3)
+	/* IP in IP */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_IPIP      UINT32_C(0x4)
+	/* Generic Network Virtualization Encapsulation (Geneve) */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_GENEVE    UINT32_C(0x5)
+	/* Multi-Protocol Lable Switching (MPLS) */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_MPLS      UINT32_C(0x6)
+	/* Stateless Transport Tunnel (STT) */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_STT       UINT32_C(0x7)
+	/* Generic Routing Encapsulation (GRE) inside IP datagram payload */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_IPGRE     UINT32_C(0x8)
+	/* IPV4 over virtual eXtensible Local Area Network (IPV4oVXLAN) */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_VXLAN_V4  UINT32_C(0x9)
+	/* Any tunneled traffic */
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_ANYTUNNEL UINT32_C(0xff)
+	#define HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_LAST \
+		HWRM_CFA_FLOW_ALLOC_INPUT_TUNNEL_TYPE_ANYTUNNEL
 } __attribute__((packed));
 
-/* hwrm_cfa_flow_alloc_output (size:128b/16B) */
+/* hwrm_cfa_flow_alloc_output (size:256b/32B) */
 struct hwrm_cfa_flow_alloc_output {
 	/* The specific error status for the command. */
 	uint16_t	error_code;
@@ -24186,7 +24212,18 @@ struct hwrm_cfa_flow_alloc_output {
 	uint16_t	resp_len;
 	/* Flow record index. */
 	uint16_t	flow_handle;
-	uint8_t	unused_0[5];
+	uint8_t	unused_0[2];
+	/*
+	 * This is the ID of the flow associated with this
+	 * filter.
+	 * This value shall be used to match and associate the
+	 * flow identifier returned in completion records.
+	 * A value of 0xFFFFFFFF shall indicate no flow id.
+	 */
+	uint32_t	flow_id;
+	/* This value identifies a set of CFA data structures used for a flow. */
+	uint64_t	ext_flow_handle;
+	uint8_t	unused_1[7];
 	/*
 	 * This field is used in Output records to indicate that the output
 	 * is completely written to RAM.  This field should be read as '1'
@@ -24202,7 +24239,7 @@ struct hwrm_cfa_flow_alloc_output {
  **********************/
 
 
-/* hwrm_cfa_flow_free_input (size:192b/24B) */
+/* hwrm_cfa_flow_free_input (size:256b/32B) */
 struct hwrm_cfa_flow_free_input {
 	/* The HWRM command request type. */
 	uint16_t	req_type;
@@ -24234,6 +24271,8 @@ struct hwrm_cfa_flow_free_input {
 	/* Flow record index. */
 	uint16_t	flow_handle;
 	uint8_t	unused_0[6];
+	/* This value identifies a set of CFA data structures used for a flow. */
+	uint64_t	ext_flow_handle;
 } __attribute__((packed));
 
 /* hwrm_cfa_flow_free_output (size:256b/32B) */
@@ -24261,98 +24300,6 @@ struct hwrm_cfa_flow_free_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/**********************
- * hwrm_cfa_flow_info *
- **********************/
-
-
-/* hwrm_cfa_flow_info_input (size:192b/24B) */
-struct hwrm_cfa_flow_info_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/* Flow record index. */
-	uint16_t	flow_handle;
-	/* Max flow handle */
-	#define HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_MAX_MASK \
-		UINT32_C(0xfff)
-	#define HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_MAX_SFT     0
-	/* CNP flow handle */
-	#define HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT \
-		UINT32_C(0x1000)
-	/* Direction rx = 1 */
-	#define HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX \
-		UINT32_C(0x8000)
-	uint8_t	unused_0[6];
-} __attribute__((packed));
-
-/* hwrm_cfa_flow_info_output (size:448b/56B) */
-struct hwrm_cfa_flow_info_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	/* flags is 8 b */
-	uint8_t	flags;
-	/* profile is 8 b */
-	uint8_t	profile;
-	/* src_fid is 16 b */
-	uint16_t	src_fid;
-	/* dst_fid is 16 b */
-	uint16_t	dst_fid;
-	/* l2_ctxt_id is 16 b */
-	uint16_t	l2_ctxt_id;
-	/* em_info is 64 b */
-	uint64_t	em_info;
-	/* tcam_info is 64 b */
-	uint64_t	tcam_info;
-	/* vfp_tcam_info is 64 b */
-	uint64_t	vfp_tcam_info;
-	/* ar_id is 16 b */
-	uint16_t	ar_id;
-	/* flow_handle is 16 b */
-	uint16_t	flow_handle;
-	/* tunnel_handle is 32 b */
-	uint32_t	tunnel_handle;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
 /***********************
  * hwrm_cfa_flow_flush *
  ***********************/
@@ -24417,7 +24364,7 @@ struct hwrm_cfa_flow_flush_output {
  ***********************/
 
 
-/* hwrm_cfa_flow_stats_input (size:320b/40B) */
+/* hwrm_cfa_flow_stats_input (size:640b/80B) */
 struct hwrm_cfa_flow_stats_input {
 	/* The HWRM command request type. */
 	uint16_t	req_type;
@@ -24469,6 +24416,26 @@ struct hwrm_cfa_flow_stats_input {
 	/* Flow handle. */
 	uint16_t	flow_handle_9;
 	uint8_t	unused_0[2];
+	/* Flow ID of a flow. */
+	uint32_t	flow_id_0;
+	/* Flow ID of a flow. */
+	uint32_t	flow_id_1;
+	/* Flow ID of a flow. */
+	uint32_t	flow_id_2;
+	/* Flow ID of a flow. */
+	uint32_t	flow_id_3;
+	/* Flow ID of a flow. */
+	uint32_t	flow_id_4;
+	/* Flow ID of a flow. */
+	uint32_t	flow_id_5;
+	/* Flow ID of a flow. */
+	uint32_t	flow_id_6;
+	/* Flow ID of a flow. */
+	uint32_t	flow_id_7;
+	/* Flow ID of a flow. */
+	uint32_t	flow_id_8;
+	/* Flow ID of a flow. */
+	uint32_t	flow_id_9;
 } __attribute__((packed));
 
 /* hwrm_cfa_flow_stats_output (size:1408b/176B) */
@@ -24532,421 +24499,6 @@ struct hwrm_cfa_flow_stats_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/**************************
- * hwrm_cfa_vf_pair_alloc *
- **************************/
-
-
-/* hwrm_cfa_vf_pair_alloc_input (size:448b/56B) */
-struct hwrm_cfa_vf_pair_alloc_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/* Logical VF number (range: 0 -> MAX_VFS -1). */
-	uint16_t	vf_a_id;
-	/* Logical VF number (range: 0 -> MAX_VFS -1). */
-	uint16_t	vf_b_id;
-	uint8_t	unused_0[4];
-	/* VF Pair name (32 byte string). */
-	char	pair_name[32];
-} __attribute__((packed));
-
-/* hwrm_cfa_vf_pair_alloc_output (size:128b/16B) */
-struct hwrm_cfa_vf_pair_alloc_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/*************************
- * hwrm_cfa_vf_pair_free *
- *************************/
-
-
-/* hwrm_cfa_vf_pair_free_input (size:384b/48B) */
-struct hwrm_cfa_vf_pair_free_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/* VF Pair name (32 byte string). */
-	char	pair_name[32];
-} __attribute__((packed));
-
-/* hwrm_cfa_vf_pair_free_output (size:128b/16B) */
-struct hwrm_cfa_vf_pair_free_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/*************************
- * hwrm_cfa_vf_pair_info *
- *************************/
-
-
-/* hwrm_cfa_vf_pair_info_input (size:448b/56B) */
-struct hwrm_cfa_vf_pair_info_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	uint32_t	flags;
-	/* If this flag is set, lookup by name else lookup by index. */
-	#define HWRM_CFA_VF_PAIR_INFO_INPUT_FLAGS_LOOKUP_TYPE     UINT32_C(0x1)
-	/* vf pair table index. */
-	uint16_t	vf_pair_index;
-	uint8_t	unused_0[2];
-	/* VF Pair name (32 byte string). */
-	char	vf_pair_name[32];
-} __attribute__((packed));
-
-/* hwrm_cfa_vf_pair_info_output (size:512b/64B) */
-struct hwrm_cfa_vf_pair_info_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	/* vf pair table index. */
-	uint16_t	next_vf_pair_index;
-	/* vf pair member a's vf_fid. */
-	uint16_t	vf_a_fid;
-	/* vf pair member a's Linux logical VF number. */
-	uint16_t	vf_a_index;
-	/* vf pair member b's vf_fid. */
-	uint16_t	vf_b_fid;
-	/* vf pair member a's Linux logical VF number. */
-	uint16_t	vf_b_index;
-	/* vf pair state. */
-	uint8_t	pair_state;
-	/* Pair has been allocated */
-	#define HWRM_CFA_VF_PAIR_INFO_OUTPUT_PAIR_STATE_ALLOCATED UINT32_C(0x1)
-	/* Both pair members are active */
-	#define HWRM_CFA_VF_PAIR_INFO_OUTPUT_PAIR_STATE_ACTIVE    UINT32_C(0x2)
-	#define HWRM_CFA_VF_PAIR_INFO_OUTPUT_PAIR_STATE_LAST \
-		HWRM_CFA_VF_PAIR_INFO_OUTPUT_PAIR_STATE_ACTIVE
-	uint8_t	unused_0[5];
-	/* VF Pair name (32 byte string). */
-	char	pair_name[32];
-	uint8_t	unused_1[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/***********************
- * hwrm_cfa_pair_alloc *
- ***********************/
-
-
-/* hwrm_cfa_pair_alloc_input (size:576b/72B) */
-struct hwrm_cfa_pair_alloc_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/* Pair mode (0-vf2fn, 1-rep2fn, 2-rep2rep, 3-proxy, 4-pfpair, 5-rep2fn_mod). */
-	uint8_t	pair_mode;
-	/* Pair between VF on local host with PF or VF on specified host. */
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_PAIR_MODE_VF2FN         UINT32_C(0x0)
-	/* Pair between REP on local host with PF or VF on specified host. */
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_PAIR_MODE_REP2FN        UINT32_C(0x1)
-	/* Pair between REP on local host with REP on specified host. */
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_PAIR_MODE_REP2REP       UINT32_C(0x2)
-	/* Pair for the proxy interface. */
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_PAIR_MODE_PROXY         UINT32_C(0x3)
-	/* Pair for the PF interface. */
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_PAIR_MODE_PFPAIR        UINT32_C(0x4)
-	/* Modify exiting rep2fn pair and move pair to new PF. */
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_PAIR_MODE_REP2FN_MOD    UINT32_C(0x5)
-	/* Modify exiting rep2fn pairs paired with same PF and move pairs to new PF. */
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_PAIR_MODE_REP2FN_MODALL UINT32_C(0x6)
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_PAIR_MODE_LAST \
-		HWRM_CFA_PAIR_ALLOC_INPUT_PAIR_MODE_REP2FN_MODALL
-	uint8_t	unused_0;
-	/* Logical VF number (range: 0 -> MAX_VFS -1). */
-	uint16_t	vf_a_id;
-	/* Logical Host (0xff-local host). */
-	uint8_t	host_b_id;
-	/* Logical PF (0xff-PF for command channel). */
-	uint8_t	pf_b_id;
-	/* Logical VF number (range: 0 -> MAX_VFS -1). */
-	uint16_t	vf_b_id;
-	/* Loopback port (0xff-internal loopback), valid for mode-3. */
-	uint8_t	port_id;
-	/* Priority used for encap of loopback packets valid for mode-3. */
-	uint8_t	pri;
-	/* New PF for rep2fn modify, valid for mode 5. */
-	uint16_t	new_pf_fid;
-	uint32_t	enables;
-	/*
-	 * This bit must be '1' for the q_ab field to be
-	 * configured.
-	 */
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_ENABLES_Q_AB_VALID      UINT32_C(0x1)
-	/*
-	 * This bit must be '1' for the q_ba field to be
-	 * configured.
-	 */
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_ENABLES_Q_BA_VALID      UINT32_C(0x2)
-	/*
-	 * This bit must be '1' for the fc_ab field to be
-	 * configured.
-	 */
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_ENABLES_FC_AB_VALID     UINT32_C(0x4)
-	/*
-	 * This bit must be '1' for the fc_ba field to be
-	 * configured.
-	 */
-	#define HWRM_CFA_PAIR_ALLOC_INPUT_ENABLES_FC_BA_VALID     UINT32_C(0x8)
-	/* VF Pair name (32 byte string). */
-	char	pair_name[32];
-	/*
-	 * The q_ab value specifies the logical index of the TX/RX CoS
-	 * queue to be assigned for traffic in the A to B direction of
-	 * the interface pair. The default value is 0.
-	 */
-	uint8_t	q_ab;
-	/*
-	 * The q_ba value specifies the logical index of the TX/RX CoS
-	 * queue to be assigned for traffic in the B to A direction of
-	 * the interface pair. The default value is 1.
-	 */
-	uint8_t	q_ba;
-	/*
-	 * Specifies whether RX ring flow control is disabled (0) or enabled
-	 * (1) in the A to B direction. The default value is 0, meaning that
-	 * packets will be dropped when the B-side RX rings are full.
-	 */
-	uint8_t	fc_ab;
-	/*
-	 * Specifies whether RX ring flow control is disabled (0) or enabled
-	 * (1) in the B to A direction. The default value is 1, meaning that
-	 * the RX CoS queue will be flow controlled when the A-side RX rings
-	 * are full.
-	 */
-	uint8_t	fc_ba;
-	uint8_t	unused_1[4];
-} __attribute__((packed));
-
-/* hwrm_cfa_pair_alloc_output (size:192b/24B) */
-struct hwrm_cfa_pair_alloc_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	/* Only valid for modes 1 and 2. */
-	uint16_t	rx_cfa_code_a;
-	/* Only valid for modes 1 and 2. */
-	uint16_t	tx_cfa_action_a;
-	/* Only valid for mode 2. */
-	uint16_t	rx_cfa_code_b;
-	/* Only valid for mode 2. */
-	uint16_t	tx_cfa_action_b;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/**********************
- * hwrm_cfa_pair_free *
- **********************/
-
-
-/* hwrm_cfa_pair_free_input (size:384b/48B) */
-struct hwrm_cfa_pair_free_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/* VF Pair name (32 byte string). */
-	char	pair_name[32];
-} __attribute__((packed));
-
-/* hwrm_cfa_pair_free_output (size:128b/16B) */
-struct hwrm_cfa_pair_free_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
 /**********************
  * hwrm_cfa_pair_info *
  **********************/
@@ -25067,13 +24619,13 @@ struct hwrm_cfa_pair_info_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/**********************
- * hwrm_cfa_vfr_alloc *
- **********************/
+/***************************************
+ * hwrm_cfa_redirect_query_tunnel_type *
+ ***************************************/
 
 
-/* hwrm_cfa_vfr_alloc_input (size:448b/56B) */
-struct hwrm_cfa_vfr_alloc_input {
+/* hwrm_cfa_redirect_query_tunnel_type_input (size:192b/24B) */
+struct hwrm_cfa_redirect_query_tunnel_type_input {
 	/* The HWRM command request type. */
 	uint16_t	req_type;
 	/*
@@ -25101,20 +24653,13 @@ struct hwrm_cfa_vfr_alloc_input {
 	 * point to a physically contiguous block of memory.
 	 */
 	uint64_t	resp_addr;
-	/* Logical VF number (range: 0 -> MAX_VFS -1). */
-	uint16_t	vf_id;
-	/*
-	 * This field is reserved for the future use.
-	 * It shall be set to 0.
-	 */
-	uint16_t	reserved;
-	uint8_t	unused_0[4];
-	/* VF Representor name (32 byte string). */
-	char	vfr_name[32];
+	/* The source function id. */
+	uint16_t	src_fid;
+	uint8_t	unused_0[6];
 } __attribute__((packed));
 
-/* hwrm_cfa_vfr_alloc_output (size:128b/16B) */
-struct hwrm_cfa_vfr_alloc_output {
+/* hwrm_cfa_redirect_query_tunnel_type_output (size:128b/16B) */
+struct hwrm_cfa_redirect_query_tunnel_type_output {
 	/* The specific error status for the command. */
 	uint16_t	error_code;
 	/* The HWRM command request type. */
@@ -25123,10 +24668,44 @@ struct hwrm_cfa_vfr_alloc_output {
 	uint16_t	seq_id;
 	/* The length of the response data in number of bytes. */
 	uint16_t	resp_len;
-	/* Rx CFA code. */
-	uint16_t	rx_cfa_code;
-	/* Tx CFA action. */
-	uint16_t	tx_cfa_action;
+	/* Tunnel Mask. */
+	uint32_t	tunnel_mask;
+	/* Non-tunnel */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_NONTUNNEL \
+		UINT32_C(0x1)
+	/* Virtual eXtensible Local Area Network (VXLAN) */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_VXLAN \
+		UINT32_C(0x2)
+	/* Network Virtualization Generic Routing Encapsulation (NVGRE) */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_NVGRE \
+		UINT32_C(0x4)
+	/* Generic Routing Encapsulation (GRE) inside Ethernet payload */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_L2GRE \
+		UINT32_C(0x8)
+	/* IP in IP */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_IPIP \
+		UINT32_C(0x10)
+	/* Generic Network Virtualization Encapsulation (Geneve) */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_GENEVE \
+		UINT32_C(0x20)
+	/* Multi-Protocol Lable Switching (MPLS) */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_MPLS \
+		UINT32_C(0x40)
+	/* Stateless Transport Tunnel (STT) */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_STT \
+		UINT32_C(0x80)
+	/* Generic Routing Encapsulation (GRE) inside IP datagram payload */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_IPGRE \
+		UINT32_C(0x100)
+	/* IPV4 over virtual eXtensible Local Area Network (IPV4oVXLAN) */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_VXLAN_V4 \
+		UINT32_C(0x200)
+	/* Enhance Generic Routing Encapsulation (GRE version 1) inside IP datagram payload */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_IPGRE_V1 \
+		UINT32_C(0x400)
+	/* Any tunneled traffic */
+	#define HWRM_CFA_REDIRECT_QUERY_TUNNEL_TYPE_OUTPUT_TUNNEL_MASK_ANYTUNNEL \
+		UINT32_C(0x800)
 	uint8_t	unused_0[3];
 	/*
 	 * This field is used in Output records to indicate that the output
@@ -25138,65 +24717,6 @@ struct hwrm_cfa_vfr_alloc_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/*********************
- * hwrm_cfa_vfr_free *
- *********************/
-
-
-/* hwrm_cfa_vfr_free_input (size:384b/48B) */
-struct hwrm_cfa_vfr_free_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/* VF Representor name (32 byte string). */
-	char	vfr_name[32];
-} __attribute__((packed));
-
-/* hwrm_cfa_vfr_free_output (size:128b/16B) */
-struct hwrm_cfa_vfr_free_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	unused_0[7];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
 /******************************
  * hwrm_tunnel_dst_port_query *
  ******************************/
@@ -25460,6 +24980,7 @@ struct hwrm_tunnel_dst_port_free_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
+/* Periodic statistics context DMA to host. */
 /* ctx_hw_stats (size:1280b/160B) */
 struct ctx_hw_stats {
 	/* Number of received unicast packets */
@@ -25882,377 +25403,6 @@ struct hwrm_pcie_qstats_output {
 	uint8_t	valid;
 } __attribute__((packed));
 
-/* Port Tx Statistics Formats */
-/* tx_port_stats (size:3264b/408B) */
-struct tx_port_stats {
-	/* Total Number of 64 Bytes frames transmitted */
-	uint64_t	tx_64b_frames;
-	/* Total Number of 65-127 Bytes frames transmitted */
-	uint64_t	tx_65b_127b_frames;
-	/* Total Number of 128-255 Bytes frames transmitted */
-	uint64_t	tx_128b_255b_frames;
-	/* Total Number of 256-511 Bytes frames transmitted */
-	uint64_t	tx_256b_511b_frames;
-	/* Total Number of 512-1023 Bytes frames transmitted */
-	uint64_t	tx_512b_1023b_frames;
-	/* Total Number of 1024-1518 Bytes frames transmitted */
-	uint64_t	tx_1024b_1518_frames;
-	/*
-	 * Total Number of each good VLAN (exludes FCS errors)
-	 * frame transmitted which is 1519 to 1522 bytes in length
-	 * inclusive (excluding framing bits but including FCS bytes).
-	 */
-	uint64_t	tx_good_vlan_frames;
-	/* Total Number of 1519-2047 Bytes frames transmitted */
-	uint64_t	tx_1519b_2047_frames;
-	/* Total Number of 2048-4095 Bytes frames transmitted */
-	uint64_t	tx_2048b_4095b_frames;
-	/* Total Number of 4096-9216 Bytes frames transmitted */
-	uint64_t	tx_4096b_9216b_frames;
-	/* Total Number of 9217-16383 Bytes frames transmitted */
-	uint64_t	tx_9217b_16383b_frames;
-	/* Total Number of good frames transmitted */
-	uint64_t	tx_good_frames;
-	/* Total Number of frames transmitted */
-	uint64_t	tx_total_frames;
-	/* Total number of unicast frames transmitted */
-	uint64_t	tx_ucast_frames;
-	/* Total number of multicast frames transmitted */
-	uint64_t	tx_mcast_frames;
-	/* Total number of broadcast frames transmitted */
-	uint64_t	tx_bcast_frames;
-	/* Total number of PAUSE control frames transmitted */
-	uint64_t	tx_pause_frames;
-	/*
-	 * Total number of PFC/per-priority PAUSE
-	 * control frames transmitted
-	 */
-	uint64_t	tx_pfc_frames;
-	/* Total number of jabber frames transmitted */
-	uint64_t	tx_jabber_frames;
-	/* Total number of frames transmitted with FCS error */
-	uint64_t	tx_fcs_err_frames;
-	/* Total number of control frames transmitted */
-	uint64_t	tx_control_frames;
-	/* Total number of over-sized frames transmitted */
-	uint64_t	tx_oversz_frames;
-	/* Total number of frames with single deferral */
-	uint64_t	tx_single_dfrl_frames;
-	/* Total number of frames with multiple deferrals */
-	uint64_t	tx_multi_dfrl_frames;
-	/* Total number of frames with single collision */
-	uint64_t	tx_single_coll_frames;
-	/* Total number of frames with multiple collisions */
-	uint64_t	tx_multi_coll_frames;
-	/* Total number of frames with late collisions */
-	uint64_t	tx_late_coll_frames;
-	/* Total number of frames with excessive collisions */
-	uint64_t	tx_excessive_coll_frames;
-	/* Total number of fragmented frames transmitted */
-	uint64_t	tx_frag_frames;
-	/* Total number of transmit errors */
-	uint64_t	tx_err;
-	/* Total number of single VLAN tagged frames transmitted */
-	uint64_t	tx_tagged_frames;
-	/* Total number of double VLAN tagged frames transmitted */
-	uint64_t	tx_dbl_tagged_frames;
-	/* Total number of runt frames transmitted */
-	uint64_t	tx_runt_frames;
-	/* Total number of TX FIFO under runs */
-	uint64_t	tx_fifo_underruns;
-	/*
-	 * Total number of PFC frames with PFC enabled bit for
-	 * Pri 0 transmitted
-	 */
-	uint64_t	tx_pfc_ena_frames_pri0;
-	/*
-	 * Total number of PFC frames with PFC enabled bit for
-	 * Pri 1 transmitted
-	 */
-	uint64_t	tx_pfc_ena_frames_pri1;
-	/*
-	 * Total number of PFC frames with PFC enabled bit for
-	 * Pri 2 transmitted
-	 */
-	uint64_t	tx_pfc_ena_frames_pri2;
-	/*
-	 * Total number of PFC frames with PFC enabled bit for
-	 * Pri 3 transmitted
-	 */
-	uint64_t	tx_pfc_ena_frames_pri3;
-	/*
-	 * Total number of PFC frames with PFC enabled bit for
-	 * Pri 4 transmitted
-	 */
-	uint64_t	tx_pfc_ena_frames_pri4;
-	/*
-	 * Total number of PFC frames with PFC enabled bit for
-	 * Pri 5 transmitted
-	 */
-	uint64_t	tx_pfc_ena_frames_pri5;
-	/*
-	 * Total number of PFC frames with PFC enabled bit for
-	 * Pri 6 transmitted
-	 */
-	uint64_t	tx_pfc_ena_frames_pri6;
-	/*
-	 * Total number of PFC frames with PFC enabled bit for
-	 * Pri 7 transmitted
-	 */
-	uint64_t	tx_pfc_ena_frames_pri7;
-	/* Total number of EEE LPI Events on TX */
-	uint64_t	tx_eee_lpi_events;
-	/* EEE LPI Duration Counter on TX */
-	uint64_t	tx_eee_lpi_duration;
-	/*
-	 * Total number of Link Level Flow Control (LLFC) messages
-	 * transmitted
-	 */
-	uint64_t	tx_llfc_logical_msgs;
-	/* Total number of HCFC messages transmitted */
-	uint64_t	tx_hcfc_msgs;
-	/* Total number of TX collisions */
-	uint64_t	tx_total_collisions;
-	/* Total number of transmitted bytes */
-	uint64_t	tx_bytes;
-	/* Total number of end-to-end HOL frames */
-	uint64_t	tx_xthol_frames;
-	/* Total Tx Drops per Port reported by STATS block */
-	uint64_t	tx_stat_discard;
-	/* Total Tx Error Drops per Port reported by STATS block */
-	uint64_t	tx_stat_error;
-} __attribute__((packed));
-
-/* Port Rx Statistics Formats */
-/* rx_port_stats (size:4224b/528B) */
-struct rx_port_stats {
-	/* Total Number of 64 Bytes frames received */
-	uint64_t	rx_64b_frames;
-	/* Total Number of 65-127 Bytes frames received */
-	uint64_t	rx_65b_127b_frames;
-	/* Total Number of 128-255 Bytes frames received */
-	uint64_t	rx_128b_255b_frames;
-	/* Total Number of 256-511 Bytes frames received */
-	uint64_t	rx_256b_511b_frames;
-	/* Total Number of 512-1023 Bytes frames received */
-	uint64_t	rx_512b_1023b_frames;
-	/* Total Number of 1024-1518 Bytes frames received */
-	uint64_t	rx_1024b_1518_frames;
-	/*
-	 * Total Number of each good VLAN (exludes FCS errors)
-	 * frame received which is 1519 to 1522 bytes in length
-	 * inclusive (excluding framing bits but including FCS bytes).
-	 */
-	uint64_t	rx_good_vlan_frames;
-	/* Total Number of 1519-2047 Bytes frames received */
-	uint64_t	rx_1519b_2047b_frames;
-	/* Total Number of 2048-4095 Bytes frames received */
-	uint64_t	rx_2048b_4095b_frames;
-	/* Total Number of 4096-9216 Bytes frames received */
-	uint64_t	rx_4096b_9216b_frames;
-	/* Total Number of 9217-16383 Bytes frames received */
-	uint64_t	rx_9217b_16383b_frames;
-	/* Total number of frames received */
-	uint64_t	rx_total_frames;
-	/* Total number of unicast frames received */
-	uint64_t	rx_ucast_frames;
-	/* Total number of multicast frames received */
-	uint64_t	rx_mcast_frames;
-	/* Total number of broadcast frames received */
-	uint64_t	rx_bcast_frames;
-	/* Total number of received frames with FCS error */
-	uint64_t	rx_fcs_err_frames;
-	/* Total number of control frames received */
-	uint64_t	rx_ctrl_frames;
-	/* Total number of PAUSE frames received */
-	uint64_t	rx_pause_frames;
-	/* Total number of PFC frames received */
-	uint64_t	rx_pfc_frames;
-	/*
-	 * Total number of frames received with an unsupported
-	 * opcode
-	 */
-	uint64_t	rx_unsupported_opcode_frames;
-	/*
-	 * Total number of frames received with an unsupported
-	 * DA for pause and PFC
-	 */
-	uint64_t	rx_unsupported_da_pausepfc_frames;
-	/* Total number of frames received with an unsupported SA */
-	uint64_t	rx_wrong_sa_frames;
-	/* Total number of received packets with alignment error */
-	uint64_t	rx_align_err_frames;
-	/* Total number of received frames with out-of-range length */
-	uint64_t	rx_oor_len_frames;
-	/* Total number of received frames with error termination */
-	uint64_t	rx_code_err_frames;
-	/*
-	 * Total number of received frames with a false carrier is
-	 * detected during idle, as defined by RX_ER samples active
-	 * and RXD is 0xE. The event is reported along with the
-	 * statistics generated on the next received frame. Only
-	 * one false carrier condition can be detected and logged
-	 * between frames.
-	 *
-	 * Carrier event, valid for 10M/100M speed modes only.
-	 */
-	uint64_t	rx_false_carrier_frames;
-	/* Total number of over-sized frames received */
-	uint64_t	rx_ovrsz_frames;
-	/* Total number of jabber packets received */
-	uint64_t	rx_jbr_frames;
-	/* Total number of received frames with MTU error */
-	uint64_t	rx_mtu_err_frames;
-	/* Total number of received frames with CRC match */
-	uint64_t	rx_match_crc_frames;
-	/* Total number of frames received promiscuously */
-	uint64_t	rx_promiscuous_frames;
-	/*
-	 * Total number of received frames with one or two VLAN
-	 * tags
-	 */
-	uint64_t	rx_tagged_frames;
-	/* Total number of received frames with two VLAN tags */
-	uint64_t	rx_double_tagged_frames;
-	/* Total number of truncated frames received */
-	uint64_t	rx_trunc_frames;
-	/* Total number of good frames (without errors) received */
-	uint64_t	rx_good_frames;
-	/*
-	 * Total number of received PFC frames with transition from
-	 * XON to XOFF on Pri 0
-	 */
-	uint64_t	rx_pfc_xon2xoff_frames_pri0;
-	/*
-	 * Total number of received PFC frames with transition from
-	 * XON to XOFF on Pri 1
-	 */
-	uint64_t	rx_pfc_xon2xoff_frames_pri1;
-	/*
-	 * Total number of received PFC frames with transition from
-	 * XON to XOFF on Pri 2
-	 */
-	uint64_t	rx_pfc_xon2xoff_frames_pri2;
-	/*
-	 * Total number of received PFC frames with transition from
-	 * XON to XOFF on Pri 3
-	 */
-	uint64_t	rx_pfc_xon2xoff_frames_pri3;
-	/*
-	 * Total number of received PFC frames with transition from
-	 * XON to XOFF on Pri 4
-	 */
-	uint64_t	rx_pfc_xon2xoff_frames_pri4;
-	/*
-	 * Total number of received PFC frames with transition from
-	 * XON to XOFF on Pri 5
-	 */
-	uint64_t	rx_pfc_xon2xoff_frames_pri5;
-	/*
-	 * Total number of received PFC frames with transition from
-	 * XON to XOFF on Pri 6
-	 */
-	uint64_t	rx_pfc_xon2xoff_frames_pri6;
-	/*
-	 * Total number of received PFC frames with transition from
-	 * XON to XOFF on Pri 7
-	 */
-	uint64_t	rx_pfc_xon2xoff_frames_pri7;
-	/*
-	 * Total number of received PFC frames with PFC enabled
-	 * bit for Pri 0
-	 */
-	uint64_t	rx_pfc_ena_frames_pri0;
-	/*
-	 * Total number of received PFC frames with PFC enabled
-	 * bit for Pri 1
-	 */
-	uint64_t	rx_pfc_ena_frames_pri1;
-	/*
-	 * Total number of received PFC frames with PFC enabled
-	 * bit for Pri 2
-	 */
-	uint64_t	rx_pfc_ena_frames_pri2;
-	/*
-	 * Total number of received PFC frames with PFC enabled
-	 * bit for Pri 3
-	 */
-	uint64_t	rx_pfc_ena_frames_pri3;
-	/*
-	 * Total number of received PFC frames with PFC enabled
-	 * bit for Pri 4
-	 */
-	uint64_t	rx_pfc_ena_frames_pri4;
-	/*
-	 * Total number of received PFC frames with PFC enabled
-	 * bit for Pri 5
-	 */
-	uint64_t	rx_pfc_ena_frames_pri5;
-	/*
-	 * Total number of received PFC frames with PFC enabled
-	 * bit for Pri 6
-	 */
-	uint64_t	rx_pfc_ena_frames_pri6;
-	/*
-	 * Total number of received PFC frames with PFC enabled
-	 * bit for Pri 7
-	 */
-	uint64_t	rx_pfc_ena_frames_pri7;
-	/* Total Number of frames received with SCH CRC error */
-	uint64_t	rx_sch_crc_err_frames;
-	/* Total Number of under-sized frames received */
-	uint64_t	rx_undrsz_frames;
-	/* Total Number of fragmented frames received */
-	uint64_t	rx_frag_frames;
-	/* Total number of RX EEE LPI Events */
-	uint64_t	rx_eee_lpi_events;
-	/* EEE LPI Duration Counter on RX */
-	uint64_t	rx_eee_lpi_duration;
-	/*
-	 * Total number of physical type Link Level Flow Control
-	 * (LLFC) messages received
-	 */
-	uint64_t	rx_llfc_physical_msgs;
-	/*
-	 * Total number of logical type Link Level Flow Control
-	 * (LLFC) messages received
-	 */
-	uint64_t	rx_llfc_logical_msgs;
-	/*
-	 * Total number of logical type Link Level Flow Control
-	 * (LLFC) messages received with CRC error
-	 */
-	uint64_t	rx_llfc_msgs_with_crc_err;
-	/* Total number of HCFC messages received */
-	uint64_t	rx_hcfc_msgs;
-	/* Total number of HCFC messages received with CRC error */
-	uint64_t	rx_hcfc_msgs_with_crc_err;
-	/* Total number of received bytes */
-	uint64_t	rx_bytes;
-	/* Total number of bytes received in runt frames */
-	uint64_t	rx_runt_bytes;
-	/* Total number of runt frames received */
-	uint64_t	rx_runt_frames;
-	/* Total Rx Discards per Port reported by STATS block */
-	uint64_t	rx_stat_discard;
-	uint64_t	rx_stat_err;
-} __attribute__((packed));
-
-/* Port Rx Statistics extended Formats */
-/* rx_port_stats_ext (size:320b/40B) */
-struct rx_port_stats_ext {
-	/* Number of times link state changed to down */
-	uint64_t	link_down_events;
-	/* Number of times the idle rings with pause bit are found */
-	uint64_t	continuous_pause_events;
-	/* Number of times the active rings pause bit resumed back */
-	uint64_t	resume_pause_events;
-	/* Number of times, the ROCE cos queue PFC is disabled to avoid pause flood/burst */
-	uint64_t	continuous_roce_pause_events;
-	/* Number of times, the ROCE cos queue PFC is enabled back */
-	uint64_t	resume_roce_pause_events;
-} __attribute__((packed));
-
 /* PCIe Statistics Formats */
 /* pcie_ctx_hw_stats (size:768b/96B) */
 struct pcie_ctx_hw_stats {
@@ -28109,103 +27259,4 @@ struct hwrm_nvm_validate_option_cmd_err {
 	uint8_t	unused_0[7];
 } __attribute__((packed));
 
-/*****************************
- * hwrm_nvm_factory_defaults *
- *****************************/
-
-
-/* hwrm_nvm_factory_defaults_input (size:192b/24B) */
-struct hwrm_nvm_factory_defaults_input {
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/*
-	 * The completion ring to send the completion event on. This should
-	 * be the NQ ID returned from the `nq_alloc` HWRM command.
-	 */
-	uint16_t	cmpl_ring;
-	/*
-	 * The sequence ID is used by the driver for tracking multiple
-	 * commands. This ID is treated as opaque data by the firmware and
-	 * the value is returned in the `hwrm_resp_hdr` upon completion.
-	 */
-	uint16_t	seq_id;
-	/*
-	 * The target ID of the command:
-	 * * 0x0-0xFFF8 - The function ID
-	 * * 0xFFF8-0xFFFE - Reserved for internal processors
-	 * * 0xFFFF - HWRM
-	 */
-	uint16_t	target_id;
-	/*
-	 * A physical address pointer pointing to a host buffer that the
-	 * command's response data will be written. This can be either a host
-	 * physical address (HPA) or a guest physical address (GPA) and must
-	 * point to a physically contiguous block of memory.
-	 */
-	uint64_t	resp_addr;
-	/* mode is 8 b */
-	uint8_t	mode;
-	/* If set to 1, it will trigger restoration of factory default settings */
-	#define HWRM_NVM_FACTORY_DEFAULTS_INPUT_MODE_RESTORE UINT32_C(0x0)
-	/* If set to 1, it will trigger creation of factory default settings */
-	#define HWRM_NVM_FACTORY_DEFAULTS_INPUT_MODE_CREATE  UINT32_C(0x1)
-	#define HWRM_NVM_FACTORY_DEFAULTS_INPUT_MODE_LAST \
-		HWRM_NVM_FACTORY_DEFAULTS_INPUT_MODE_CREATE
-	uint8_t	unused_0[7];
-} __attribute__((packed));
-
-/* hwrm_nvm_factory_defaults_output (size:128b/16B) */
-struct hwrm_nvm_factory_defaults_output {
-	/* The specific error status for the command. */
-	uint16_t	error_code;
-	/* The HWRM command request type. */
-	uint16_t	req_type;
-	/* The sequence ID from the original command. */
-	uint16_t	seq_id;
-	/* The length of the response data in number of bytes. */
-	uint16_t	resp_len;
-	uint8_t	result;
-	/* factory defaults created successfully. */
-	#define HWRM_NVM_FACTORY_DEFAULTS_OUTPUT_RESULT_CREATE_OK \
-		UINT32_C(0x0)
-	/* factory defaults restored successfully. */
-	#define HWRM_NVM_FACTORY_DEFAULTS_OUTPUT_RESULT_RESTORE_OK \
-		UINT32_C(0x1)
-	/* factory defaults already created. */
-	#define HWRM_NVM_FACTORY_DEFAULTS_OUTPUT_RESULT_CREATE_ALREADY \
-		UINT32_C(0x2)
-	#define HWRM_NVM_FACTORY_DEFAULTS_OUTPUT_RESULT_LAST \
-		HWRM_NVM_FACTORY_DEFAULTS_OUTPUT_RESULT_CREATE_ALREADY
-	uint8_t	unused_0[6];
-	/*
-	 * This field is used in Output records to indicate that the output
-	 * is completely written to RAM.  This field should be read as '1'
-	 * to indicate that the output has been completely written.
-	 * When writing a command completion or response to an internal processor,
-	 * the order of writes has to be such that this field is written last.
-	 */
-	uint8_t	valid;
-} __attribute__((packed));
-
-/* hwrm_nvm_factory_defaults_cmd_err (size:64b/8B) */
-struct hwrm_nvm_factory_defaults_cmd_err {
-	/*
-	 * command specific error codes that goes to
-	 * the cmd_err field in Common HWRM Error Response.
-	 */
-	uint8_t	code;
-	/* Unknown error */
-	#define HWRM_NVM_FACTORY_DEFAULTS_CMD_ERR_CODE_UNKNOWN \
-		UINT32_C(0x0)
-	/* valid configuration not present to create defaults */
-	#define HWRM_NVM_FACTORY_DEFAULTS_CMD_ERR_CODE_NO_VALID_CFG \
-		UINT32_C(0x1)
-	/* No saved configuration present to restore, restore failed */
-	#define HWRM_NVM_FACTORY_DEFAULTS_CMD_ERR_CODE_NO_SAVED_CFG \
-		UINT32_C(0x2)
-	#define HWRM_NVM_FACTORY_DEFAULTS_CMD_ERR_CODE_LAST \
-		HWRM_NVM_FACTORY_DEFAULTS_CMD_ERR_CODE_NO_SAVED_CFG
-	uint8_t	unused_0[7];
-} __attribute__((packed));
-
 #endif /* _HSI_STRUCT_DEF_DPDK_H_ */
diff --git a/drivers/net/bonding/Makefile b/drivers/net/bonding/Makefile
index acad16a1..1893e3ca 100644
--- a/drivers/net/bonding/Makefile
+++ b/drivers/net/bonding/Makefile
@@ -8,6 +8,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 #
 LIB = librte_pmd_bond.a
 
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
diff --git a/drivers/net/bonding/meson.build b/drivers/net/bonding/meson.build
index 602d2880..00374edb 100644
--- a/drivers/net/bonding/meson.build
+++ b/drivers/net/bonding/meson.build
@@ -3,6 +3,7 @@
 
 name = 'bond' #, james bond :-)
 version = 2
+allow_experimental_apis = true
 sources = files('rte_eth_bond_api.c', 'rte_eth_bond_pmd.c', 'rte_eth_bond_flow.c',
 	'rte_eth_bond_args.c', 'rte_eth_bond_8023ad.c', 'rte_eth_bond_alb.c')
 
diff --git a/drivers/net/bonding/rte_eth_bond_8023ad.c b/drivers/net/bonding/rte_eth_bond_8023ad.c
index f8cea4b6..dd847c6f 100644
--- a/drivers/net/bonding/rte_eth_bond_8023ad.c
+++ b/drivers/net/bonding/rte_eth_bond_8023ad.c
@@ -130,7 +130,7 @@ static const struct ether_addr lacp_mac_addr = {
 	.addr_bytes = { 0x01, 0x80, 0xC2, 0x00, 0x00, 0x02 }
 };
 
-struct port mode_8023ad_ports[RTE_MAX_ETHPORTS];
+struct port bond_mode_8023ad_ports[RTE_MAX_ETHPORTS];
 
 static void
 timer_cancel(uint64_t *timer)
@@ -187,7 +187,7 @@ set_warning_flags(struct port *port, uint16_t flags)
 static void
 show_warnings(uint16_t slave_id)
 {
-	struct port *port = &mode_8023ad_ports[slave_id];
+	struct port *port = &bond_mode_8023ad_ports[slave_id];
 	uint8_t warnings;
 
 	do {
@@ -260,7 +260,7 @@ static void
 rx_machine(struct bond_dev_private *internals, uint16_t slave_id,
 		struct lacpdu *lacp)
 {
-	struct port *agg, *port = &mode_8023ad_ports[slave_id];
+	struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 	uint64_t timeout;
 
 	if (SM_FLAG(port, BEGIN)) {
@@ -319,7 +319,7 @@ rx_machine(struct bond_dev_private *internals, uint16_t slave_id,
 		ACTOR_STATE_CLR(port, DEFAULTED);
 
 		/* If LACP partner params match this port actor params */
-		agg = &mode_8023ad_ports[port->aggregator_port_id];
+		agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 		bool match = port->actor.system_priority ==
 			lacp->partner.port_params.system_priority &&
 			is_same_ether_addr(&agg->actor.system,
@@ -380,7 +380,7 @@ rx_machine(struct bond_dev_private *internals, uint16_t slave_id,
 static void
 periodic_machine(struct bond_dev_private *internals, uint16_t slave_id)
 {
-	struct port *port = &mode_8023ad_ports[slave_id];
+	struct port *port = &bond_mode_8023ad_ports[slave_id];
 	/* Calculate if either site is LACP enabled */
 	uint64_t timeout;
 	uint8_t active = ACTOR_STATE(port, LACP_ACTIVE) ||
@@ -442,7 +442,7 @@ periodic_machine(struct bond_dev_private *internals, uint16_t slave_id)
 static void
 mux_machine(struct bond_dev_private *internals, uint16_t slave_id)
 {
-	struct port *port = &mode_8023ad_ports[slave_id];
+	struct port *port = &bond_mode_8023ad_ports[slave_id];
 
 	/* Save current state for later use */
 	const uint8_t state_mask = STATE_SYNCHRONIZATION | STATE_DISTRIBUTING |
@@ -545,7 +545,7 @@ mux_machine(struct bond_dev_private *internals, uint16_t slave_id)
 static void
 tx_machine(struct bond_dev_private *internals, uint16_t slave_id)
 {
-	struct port *agg, *port = &mode_8023ad_ports[slave_id];
+	struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 
 	struct rte_mbuf *lacp_pkt = NULL;
 	struct lacpdu_header *hdr;
@@ -591,7 +591,7 @@ tx_machine(struct bond_dev_private *internals, uint16_t slave_id)
 	lacpdu->actor.info_length = sizeof(struct lacpdu_actor_partner_params);
 	memcpy(&hdr->lacpdu.actor.port_params, &port->actor,
 			sizeof(port->actor));
-	agg = &mode_8023ad_ports[port->aggregator_port_id];
+	agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 	ether_addr_copy(&agg->actor.system, &hdr->lacpdu.actor.port_params.system);
 	lacpdu->actor.state = port->actor_state;
 
@@ -677,11 +677,11 @@ selection_logic(struct bond_dev_private *internals, uint8_t slave_id)
 
 	slaves = internals->active_slaves;
 	slaves_count = internals->active_slave_count;
-	port = &mode_8023ad_ports[slave_id];
+	port = &bond_mode_8023ad_ports[slave_id];
 
 	/* Search for aggregator suitable for this port */
 	for (i = 0; i < slaves_count; ++i) {
-		agg = &mode_8023ad_ports[slaves[i]];
+		agg = &bond_mode_8023ad_ports[slaves[i]];
 		/* Skip ports that are not aggreagators */
 		if (agg->aggregator_port_id != slaves[i])
 			continue;
@@ -824,7 +824,7 @@ bond_mode_8023ad_periodic_cb(void *arg)
 		} else
 			key = 0;
 
-		port = &mode_8023ad_ports[slave_id];
+		port = &bond_mode_8023ad_ports[slave_id];
 
 		key = rte_cpu_to_be_16(key);
 		if (key != port->actor.key) {
@@ -844,7 +844,7 @@ bond_mode_8023ad_periodic_cb(void *arg)
 
 	for (i = 0; i < internals->active_slave_count; i++) {
 		slave_id = internals->active_slaves[i];
-		port = &mode_8023ad_ports[slave_id];
+		port = &bond_mode_8023ad_ports[slave_id];
 
 		if ((port->actor.key &
 				rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) == 0) {
@@ -907,7 +907,7 @@ bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev,
 {
 	struct bond_dev_private *internals = bond_dev->data->dev_private;
 
-	struct port *port = &mode_8023ad_ports[slave_id];
+	struct port *port = &bond_mode_8023ad_ports[slave_id];
 	struct port_params initial = {
 			.system = { { 0 } },
 			.system_priority = rte_cpu_to_be_16(0xFFFF),
@@ -1008,7 +1008,7 @@ bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *bond_dev __rte_unused,
 	struct port *port = NULL;
 	uint8_t old_partner_state;
 
-	port = &mode_8023ad_ports[slave_id];
+	port = &bond_mode_8023ad_ports[slave_id];
 
 	ACTOR_STATE_CLR(port, AGGREGATION);
 	port->selected = UNSELECTED;
@@ -1045,7 +1045,7 @@ bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev)
 
 	for (i = 0; i < internals->active_slave_count; i++) {
 		slave_id = internals->active_slaves[i];
-		slave = &mode_8023ad_ports[slave_id];
+		slave = &bond_mode_8023ad_ports[slave_id];
 		rte_eth_macaddr_get(slave_id, &slave_addr);
 
 		if (is_same_ether_addr(&slave_addr, &slave->actor.system))
@@ -1058,7 +1058,7 @@ bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev)
 			continue;
 
 		for (j = 0; j < internals->active_slave_count; j++) {
-			agg_slave = &mode_8023ad_ports[internals->active_slaves[j]];
+			agg_slave = &bond_mode_8023ad_ports[internals->active_slaves[j]];
 			if (agg_slave->aggregator_port_id == slave_id)
 				SM_FLAG_SET(agg_slave, NTT);
 		}
@@ -1191,7 +1191,7 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
 				  uint16_t slave_id, struct rte_mbuf *pkt)
 {
 	struct mode8023ad_private *mode4 = &internals->mode4;
-	struct port *port = &mode_8023ad_ports[slave_id];
+	struct port *port = &bond_mode_8023ad_ports[slave_id];
 	struct marker_header *m_hdr;
 	uint64_t marker_timer, old_marker_timer;
 	int retval;
@@ -1395,7 +1395,7 @@ rte_eth_bond_8023ad_slave_info(uint16_t port_id, uint16_t slave_id,
 				internals->active_slave_count)
 		return -EINVAL;
 
-	port = &mode_8023ad_ports[slave_id];
+	port = &bond_mode_8023ad_ports[slave_id];
 	info->selected = port->selected;
 
 	info->actor_state = port->actor_state;
@@ -1447,7 +1447,7 @@ rte_eth_bond_8023ad_ext_collect(uint16_t port_id, uint16_t slave_id,
 	if (res != 0)
 		return res;
 
-	port = &mode_8023ad_ports[slave_id];
+	port = &bond_mode_8023ad_ports[slave_id];
 
 	if (enabled)
 		ACTOR_STATE_SET(port, COLLECTING);
@@ -1468,7 +1468,7 @@ rte_eth_bond_8023ad_ext_distrib(uint16_t port_id, uint16_t slave_id,
 	if (res != 0)
 		return res;
 
-	port = &mode_8023ad_ports[slave_id];
+	port = &bond_mode_8023ad_ports[slave_id];
 
 	if (enabled)
 		ACTOR_STATE_SET(port, DISTRIBUTING);
@@ -1488,7 +1488,7 @@ rte_eth_bond_8023ad_ext_distrib_get(uint16_t port_id, uint16_t slave_id)
 	if (err != 0)
 		return err;
 
-	port = &mode_8023ad_ports[slave_id];
+	port = &bond_mode_8023ad_ports[slave_id];
 	return ACTOR_STATE(port, DISTRIBUTING);
 }
 
@@ -1502,7 +1502,7 @@ rte_eth_bond_8023ad_ext_collect_get(uint16_t port_id, uint16_t slave_id)
 	if (err != 0)
 		return err;
 
-	port = &mode_8023ad_ports[slave_id];
+	port = &bond_mode_8023ad_ports[slave_id];
 	return ACTOR_STATE(port, COLLECTING);
 }
 
@@ -1517,7 +1517,7 @@ rte_eth_bond_8023ad_ext_slowtx(uint16_t port_id, uint16_t slave_id,
 	if (res != 0)
 		return res;
 
-	port = &mode_8023ad_ports[slave_id];
+	port = &bond_mode_8023ad_ports[slave_id];
 
 	if (rte_pktmbuf_pkt_len(lacp_pkt) < sizeof(struct lacpdu_header))
 		return -EINVAL;
@@ -1546,7 +1546,7 @@ bond_mode_8023ad_ext_periodic_cb(void *arg)
 
 	for (i = 0; i < internals->active_slave_count; i++) {
 		slave_id = internals->active_slaves[i];
-		port = &mode_8023ad_ports[slave_id];
+		port = &bond_mode_8023ad_ports[slave_id];
 
 		if (rte_ring_dequeue(port->rx_ring, &pkt) == 0) {
 			struct rte_mbuf *lacp_pkt = pkt;
diff --git a/drivers/net/bonding/rte_eth_bond_8023ad_private.h b/drivers/net/bonding/rte_eth_bond_8023ad_private.h
index 0f490a51..c51426b8 100644
--- a/drivers/net/bonding/rte_eth_bond_8023ad_private.h
+++ b/drivers/net/bonding/rte_eth_bond_8023ad_private.h
@@ -174,7 +174,7 @@ struct mode8023ad_private {
  * The pool of *port* structures. The size of the pool
  * is configured at compile-time in the <rte_eth_bond_8023ad.c> file.
  */
-extern struct port mode_8023ad_ports[];
+extern struct port bond_mode_8023ad_ports[];
 
 /* Forward declaration */
 struct bond_dev_private;
diff --git a/drivers/net/bonding/rte_eth_bond_api.c b/drivers/net/bonding/rte_eth_bond_api.c
index 8bc04cfd..21bcd504 100644
--- a/drivers/net/bonding/rte_eth_bond_api.c
+++ b/drivers/net/bonding/rte_eth_bond_api.c
@@ -245,9 +245,9 @@ slave_rte_flow_prepare(uint16_t slave_id, struct bond_dev_private *internals)
 	}
 	TAILQ_FOREACH(flow, &internals->flow_list, next) {
 		flow->flows[slave_id] = rte_flow_create(slave_port_id,
-							&flow->fd->attr,
-							flow->fd->items,
-							flow->fd->actions,
+							flow->rule.attr,
+							flow->rule.pattern,
+							flow->rule.actions,
 							&ferror);
 		if (flow->flows[slave_id] == NULL) {
 			RTE_BOND_LOG(ERR, "Cannot create flow for slave"
@@ -269,6 +269,173 @@ slave_rte_flow_prepare(uint16_t slave_id, struct bond_dev_private *internals)
 	return 0;
 }
 
+static void
+eth_bond_slave_inherit_dev_info_rx_first(struct bond_dev_private *internals,
+					 const struct rte_eth_dev_info *di)
+{
+	struct rte_eth_rxconf *rxconf_i = &internals->default_rxconf;
+
+	internals->reta_size = di->reta_size;
+
+	/* Inherit Rx offload capabilities from the first slave device */
+	internals->rx_offload_capa = di->rx_offload_capa;
+	internals->rx_queue_offload_capa = di->rx_queue_offload_capa;
+	internals->flow_type_rss_offloads = di->flow_type_rss_offloads;
+
+	/* Inherit maximum Rx packet size from the first slave device */
+	internals->candidate_max_rx_pktlen = di->max_rx_pktlen;
+
+	/* Inherit default Rx queue settings from the first slave device */
+	memcpy(rxconf_i, &di->default_rxconf, sizeof(*rxconf_i));
+
+	/*
+	 * Turn off descriptor prefetch and writeback by default for all
+	 * slave devices. Applications may tweak this setting if need be.
+	 */
+	rxconf_i->rx_thresh.pthresh = 0;
+	rxconf_i->rx_thresh.hthresh = 0;
+	rxconf_i->rx_thresh.wthresh = 0;
+
+	/* Setting this to zero should effectively enable default values */
+	rxconf_i->rx_free_thresh = 0;
+
+	/* Disable deferred start by default for all slave devices */
+	rxconf_i->rx_deferred_start = 0;
+}
+
+static void
+eth_bond_slave_inherit_dev_info_tx_first(struct bond_dev_private *internals,
+					 const struct rte_eth_dev_info *di)
+{
+	struct rte_eth_txconf *txconf_i = &internals->default_txconf;
+
+	/* Inherit Tx offload capabilities from the first slave device */
+	internals->tx_offload_capa = di->tx_offload_capa;
+	internals->tx_queue_offload_capa = di->tx_queue_offload_capa;
+
+	/* Inherit default Tx queue settings from the first slave device */
+	memcpy(txconf_i, &di->default_txconf, sizeof(*txconf_i));
+
+	/*
+	 * Turn off descriptor prefetch and writeback by default for all
+	 * slave devices. Applications may tweak this setting if need be.
+	 */
+	txconf_i->tx_thresh.pthresh = 0;
+	txconf_i->tx_thresh.hthresh = 0;
+	txconf_i->tx_thresh.wthresh = 0;
+
+	/*
+	 * Setting these parameters to zero assumes that default
+	 * values will be configured implicitly by slave devices.
+	 */
+	txconf_i->tx_free_thresh = 0;
+	txconf_i->tx_rs_thresh = 0;
+
+	/* Disable deferred start by default for all slave devices */
+	txconf_i->tx_deferred_start = 0;
+}
+
+static void
+eth_bond_slave_inherit_dev_info_rx_next(struct bond_dev_private *internals,
+					const struct rte_eth_dev_info *di)
+{
+	struct rte_eth_rxconf *rxconf_i = &internals->default_rxconf;
+	const struct rte_eth_rxconf *rxconf = &di->default_rxconf;
+
+	internals->rx_offload_capa &= di->rx_offload_capa;
+	internals->rx_queue_offload_capa &= di->rx_queue_offload_capa;
+	internals->flow_type_rss_offloads &= di->flow_type_rss_offloads;
+
+	/*
+	 * If at least one slave device suggests enabling this
+	 * setting by default, enable it for all slave devices
+	 * since disabling it may not be necessarily supported.
+	 */
+	if (rxconf->rx_drop_en == 1)
+		rxconf_i->rx_drop_en = 1;
+
+	/*
+	 * Adding a new slave device may cause some of previously inherited
+	 * offloads to be withdrawn from the internal rx_queue_offload_capa
+	 * value. Thus, the new internal value of default Rx queue offloads
+	 * has to be masked by rx_queue_offload_capa to make sure that only
+	 * commonly supported offloads are preserved from both the previous
+	 * value and the value being inhereted from the new slave device.
+	 */
+	rxconf_i->offloads = (rxconf_i->offloads | rxconf->offloads) &
+			     internals->rx_queue_offload_capa;
+
+	/*
+	 * RETA size is GCD of all slaves RETA sizes, so, if all sizes will be
+	 * the power of 2, the lower one is GCD
+	 */
+	if (internals->reta_size > di->reta_size)
+		internals->reta_size = di->reta_size;
+
+	if (!internals->max_rx_pktlen &&
+	    di->max_rx_pktlen < internals->candidate_max_rx_pktlen)
+		internals->candidate_max_rx_pktlen = di->max_rx_pktlen;
+}
+
+static void
+eth_bond_slave_inherit_dev_info_tx_next(struct bond_dev_private *internals,
+					const struct rte_eth_dev_info *di)
+{
+	struct rte_eth_txconf *txconf_i = &internals->default_txconf;
+	const struct rte_eth_txconf *txconf = &di->default_txconf;
+
+	internals->tx_offload_capa &= di->tx_offload_capa;
+	internals->tx_queue_offload_capa &= di->tx_queue_offload_capa;
+
+	/*
+	 * Adding a new slave device may cause some of previously inherited
+	 * offloads to be withdrawn from the internal tx_queue_offload_capa
+	 * value. Thus, the new internal value of default Tx queue offloads
+	 * has to be masked by tx_queue_offload_capa to make sure that only
+	 * commonly supported offloads are preserved from both the previous
+	 * value and the value being inhereted from the new slave device.
+	 */
+	txconf_i->offloads = (txconf_i->offloads | txconf->offloads) &
+			     internals->tx_queue_offload_capa;
+}
+
+static void
+eth_bond_slave_inherit_desc_lim_first(struct rte_eth_desc_lim *bond_desc_lim,
+		const struct rte_eth_desc_lim *slave_desc_lim)
+{
+	memcpy(bond_desc_lim, slave_desc_lim, sizeof(*bond_desc_lim));
+}
+
+static int
+eth_bond_slave_inherit_desc_lim_next(struct rte_eth_desc_lim *bond_desc_lim,
+		const struct rte_eth_desc_lim *slave_desc_lim)
+{
+	bond_desc_lim->nb_max = RTE_MIN(bond_desc_lim->nb_max,
+					slave_desc_lim->nb_max);
+	bond_desc_lim->nb_min = RTE_MAX(bond_desc_lim->nb_min,
+					slave_desc_lim->nb_min);
+	bond_desc_lim->nb_align = RTE_MAX(bond_desc_lim->nb_align,
+					  slave_desc_lim->nb_align);
+
+	if (bond_desc_lim->nb_min > bond_desc_lim->nb_max ||
+	    bond_desc_lim->nb_align > bond_desc_lim->nb_max) {
+		RTE_BOND_LOG(ERR, "Failed to inherit descriptor limits");
+		return -EINVAL;
+	}
+
+	/* Treat maximum number of segments equal to 0 as unspecified */
+	if (slave_desc_lim->nb_seg_max != 0 &&
+	    (bond_desc_lim->nb_seg_max == 0 ||
+	     slave_desc_lim->nb_seg_max < bond_desc_lim->nb_seg_max))
+		bond_desc_lim->nb_seg_max = slave_desc_lim->nb_seg_max;
+	if (slave_desc_lim->nb_mtu_seg_max != 0 &&
+	    (bond_desc_lim->nb_mtu_seg_max == 0 ||
+	     slave_desc_lim->nb_mtu_seg_max < bond_desc_lim->nb_mtu_seg_max))
+		bond_desc_lim->nb_mtu_seg_max = slave_desc_lim->nb_mtu_seg_max;
+
+	return 0;
+}
+
 static int
 __eth_bond_slave_add_lock_free(uint16_t bonded_port_id, uint16_t slave_port_id)
 {
@@ -326,34 +493,28 @@ __eth_bond_slave_add_lock_free(uint16_t bonded_port_id, uint16_t slave_port_id)
 		internals->nb_rx_queues = slave_eth_dev->data->nb_rx_queues;
 		internals->nb_tx_queues = slave_eth_dev->data->nb_tx_queues;
 
-		internals->reta_size = dev_info.reta_size;
+		eth_bond_slave_inherit_dev_info_rx_first(internals, &dev_info);
+		eth_bond_slave_inherit_dev_info_tx_first(internals, &dev_info);
 
-		/* Take the first dev's offload capabilities */
-		internals->rx_offload_capa = dev_info.rx_offload_capa;
-		internals->tx_offload_capa = dev_info.tx_offload_capa;
-		internals->rx_queue_offload_capa = dev_info.rx_queue_offload_capa;
-		internals->tx_queue_offload_capa = dev_info.tx_queue_offload_capa;
-		internals->flow_type_rss_offloads = dev_info.flow_type_rss_offloads;
+		eth_bond_slave_inherit_desc_lim_first(&internals->rx_desc_lim,
+						      &dev_info.rx_desc_lim);
+		eth_bond_slave_inherit_desc_lim_first(&internals->tx_desc_lim,
+						      &dev_info.tx_desc_lim);
+	} else {
+		int ret;
 
-		/* Inherit first slave's max rx packet size */
-		internals->candidate_max_rx_pktlen = dev_info.max_rx_pktlen;
+		eth_bond_slave_inherit_dev_info_rx_next(internals, &dev_info);
+		eth_bond_slave_inherit_dev_info_tx_next(internals, &dev_info);
 
-	} else {
-		internals->rx_offload_capa &= dev_info.rx_offload_capa;
-		internals->tx_offload_capa &= dev_info.tx_offload_capa;
-		internals->rx_queue_offload_capa &= dev_info.rx_queue_offload_capa;
-		internals->tx_queue_offload_capa &= dev_info.tx_queue_offload_capa;
-		internals->flow_type_rss_offloads &= dev_info.flow_type_rss_offloads;
-
-		/* RETA size is GCD of all slaves RETA sizes, so, if all sizes will be
-		 * the power of 2, the lower one is GCD
-		 */
-		if (internals->reta_size > dev_info.reta_size)
-			internals->reta_size = dev_info.reta_size;
+		ret = eth_bond_slave_inherit_desc_lim_next(
+				&internals->rx_desc_lim, &dev_info.rx_desc_lim);
+		if (ret != 0)
+			return ret;
 
-		if (!internals->max_rx_pktlen &&
-		    dev_info.max_rx_pktlen < internals->candidate_max_rx_pktlen)
-			internals->candidate_max_rx_pktlen = dev_info.max_rx_pktlen;
+		ret = eth_bond_slave_inherit_desc_lim_next(
+				&internals->tx_desc_lim, &dev_info.tx_desc_lim);
+		if (ret != 0)
+			return ret;
 	}
 
 	bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf &=
diff --git a/drivers/net/bonding/rte_eth_bond_flow.c b/drivers/net/bonding/rte_eth_bond_flow.c
index 31e4bcae..f94d46ca 100644
--- a/drivers/net/bonding/rte_eth_bond_flow.c
+++ b/drivers/net/bonding/rte_eth_bond_flow.c
@@ -2,8 +2,11 @@
  * Copyright 2018 Mellanox Technologies, Ltd
  */
 
+#include <stddef.h>
+#include <string.h>
 #include <sys/queue.h>
 
+#include <rte_errno.h>
 #include <rte_malloc.h>
 #include <rte_tailq.h>
 #include <rte_flow.h>
@@ -16,19 +19,33 @@ bond_flow_alloc(int numa_node, const struct rte_flow_attr *attr,
 		   const struct rte_flow_action *actions)
 {
 	struct rte_flow *flow;
-	size_t fdsz;
+	const struct rte_flow_conv_rule rule = {
+		.attr_ro = attr,
+		.pattern_ro = items,
+		.actions_ro = actions,
+	};
+	struct rte_flow_error error;
+	int ret;
 
-	fdsz = rte_flow_copy(NULL, 0, attr, items, actions);
-	flow = rte_zmalloc_socket(NULL, sizeof(struct rte_flow) + fdsz,
+	ret = rte_flow_conv(RTE_FLOW_CONV_OP_RULE, NULL, 0, &rule, &error);
+	if (ret < 0) {
+		RTE_BOND_LOG(ERR, "Unable to process flow rule (%s): %s",
+			     error.message ? error.message : "unspecified",
+			     strerror(rte_errno));
+		return NULL;
+	}
+	flow = rte_zmalloc_socket(NULL, offsetof(struct rte_flow, rule) + ret,
 				  RTE_CACHE_LINE_SIZE, numa_node);
 	if (unlikely(flow == NULL)) {
 		RTE_BOND_LOG(ERR, "Could not allocate new flow");
 		return NULL;
 	}
-	flow->fd = (void *)((uintptr_t)flow + sizeof(*flow));
-	if (unlikely(rte_flow_copy(flow->fd, fdsz, attr, items, actions) !=
-		     fdsz)) {
-		RTE_BOND_LOG(ERR, "Failed to copy flow description");
+	ret = rte_flow_conv(RTE_FLOW_CONV_OP_RULE, &flow->rule, ret, &rule,
+			    &error);
+	if (ret < 0) {
+		RTE_BOND_LOG(ERR, "Failed to copy flow rule (%s): %s",
+			     error.message ? error.message : "unspecified",
+			     strerror(rte_errno));
 		rte_free(flow);
 		return NULL;
 	}
diff --git a/drivers/net/bonding/rte_eth_bond_pmd.c b/drivers/net/bonding/rte_eth_bond_pmd.c
index 58f7377c..156f31c6 100644
--- a/drivers/net/bonding/rte_eth_bond_pmd.c
+++ b/drivers/net/bonding/rte_eth_bond_pmd.c
@@ -37,7 +37,8 @@ get_vlan_offset(struct ether_hdr *eth_hdr, uint16_t *proto)
 {
 	size_t vlan_offset = 0;
 
-	if (rte_cpu_to_be_16(ETHER_TYPE_VLAN) == *proto) {
+	if (rte_cpu_to_be_16(ETHER_TYPE_VLAN) == *proto ||
+		rte_cpu_to_be_16(ETHER_TYPE_QINQ) == *proto) {
 		struct vlan_hdr *vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1);
 
 		vlan_offset = sizeof(struct vlan_hdr);
@@ -57,28 +58,34 @@ bond_ethdev_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 {
 	struct bond_dev_private *internals;
 
-	uint16_t num_rx_slave = 0;
 	uint16_t num_rx_total = 0;
-
+	uint16_t slave_count;
+	uint16_t active_slave;
 	int i;
 
 	/* Cast to structure, containing bonded device's port id and queue id */
 	struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
-
 	internals = bd_rx_q->dev_private;
+	slave_count = internals->active_slave_count;
+	active_slave = internals->active_slave;
 
+	for (i = 0; i < slave_count && nb_pkts; i++) {
+		uint16_t num_rx_slave;
 
-	for (i = 0; i < internals->active_slave_count && nb_pkts; i++) {
 		/* Offset of pointer to *bufs increases as packets are received
 		 * from other slaves */
-		num_rx_slave = rte_eth_rx_burst(internals->active_slaves[i],
-				bd_rx_q->queue_id, bufs + num_rx_total, nb_pkts);
-		if (num_rx_slave) {
-			num_rx_total += num_rx_slave;
-			nb_pkts -= num_rx_slave;
-		}
+		num_rx_slave =
+			rte_eth_rx_burst(internals->active_slaves[active_slave],
+					 bd_rx_q->queue_id,
+					 bufs + num_rx_total, nb_pkts);
+		num_rx_total += num_rx_slave;
+		nb_pkts -= num_rx_slave;
+		if (++active_slave == slave_count)
+			active_slave = 0;
 	}
 
+	if (++internals->active_slave == slave_count)
+		internals->active_slave = 0;
 	return num_rx_total;
 }
 
@@ -257,25 +264,32 @@ bond_ethdev_rx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs,
 	uint16_t num_rx_total = 0;	/* Total number of received packets */
 	uint16_t slaves[RTE_MAX_ETHPORTS];
 	uint16_t slave_count;
-
-	uint16_t i, idx;
+	uint16_t active_slave;
+	uint16_t i;
 
 	/* Copy slave list to protect against slave up/down changes during tx
 	 * bursting */
 	slave_count = internals->active_slave_count;
+	active_slave = internals->active_slave;
 	memcpy(slaves, internals->active_slaves,
 			sizeof(internals->active_slaves[0]) * slave_count);
 
-	for (i = 0, idx = internals->active_slave;
-			i < slave_count && num_rx_total < nb_pkts; i++, idx++) {
-		idx = idx % slave_count;
+	for (i = 0; i < slave_count && nb_pkts; i++) {
+		uint16_t num_rx_slave;
 
 		/* Read packets from this slave */
-		num_rx_total += rte_eth_rx_burst(slaves[idx], bd_rx_q->queue_id,
-				&bufs[num_rx_total], nb_pkts - num_rx_total);
+		num_rx_slave = rte_eth_rx_burst(slaves[active_slave],
+						bd_rx_q->queue_id,
+						bufs + num_rx_total, nb_pkts);
+		num_rx_total += num_rx_slave;
+		nb_pkts -= num_rx_slave;
+
+		if (++active_slave == slave_count)
+			active_slave = 0;
 	}
 
-	internals->active_slave = idx;
+	if (++internals->active_slave == slave_count)
+		internals->active_slave = 0;
 
 	return num_rx_total;
 }
@@ -300,10 +314,10 @@ bond_ethdev_tx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs,
 	/* Mapping array generated by hash function to map mbufs to slaves */
 	uint16_t bufs_slave_port_idxs[RTE_MAX_ETHPORTS] = { 0 };
 
-	uint16_t slave_tx_count, slave_tx_fail_count[RTE_MAX_ETHPORTS] = { 0 };
+	uint16_t slave_tx_count;
 	uint16_t total_tx_count = 0, total_tx_fail_count = 0;
 
-	uint16_t i, j;
+	uint16_t i;
 
 	if (unlikely(nb_bufs == 0))
 		return 0;
@@ -320,7 +334,7 @@ bond_ethdev_tx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs,
 
 	dist_slave_count = 0;
 	for (i = 0; i < slave_count; i++) {
-		struct port *port = &mode_8023ad_ports[slave_port_ids[i]];
+		struct port *port = &bond_mode_8023ad_ports[slave_port_ids[i]];
 
 		if (ACTOR_STATE(port, DISTRIBUTING))
 			dist_slave_port_ids[dist_slave_count++] =
@@ -358,34 +372,12 @@ bond_ethdev_tx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs,
 
 		/* If tx burst fails move packets to end of bufs */
 		if (unlikely(slave_tx_count < slave_nb_bufs[i])) {
-			slave_tx_fail_count[i] = slave_nb_bufs[i] -
+			int slave_tx_fail_count = slave_nb_bufs[i] -
 					slave_tx_count;
-			total_tx_fail_count += slave_tx_fail_count[i];
-
-			/*
-			 * Shift bufs to beginning of array to allow reordering
-			 * later
-			 */
-			for (j = 0; j < slave_tx_fail_count[i]; j++) {
-				slave_bufs[i][j] =
-					slave_bufs[i][(slave_tx_count - 1) + j];
-			}
-		}
-	}
-
-	/*
-	 * If there are tx burst failures we move packets to end of bufs to
-	 * preserve expected PMD behaviour of all failed transmitted being
-	 * at the end of the input mbuf array
-	 */
-	if (unlikely(total_tx_fail_count > 0)) {
-		int bufs_idx = nb_bufs - total_tx_fail_count - 1;
-
-		for (i = 0; i < slave_count; i++) {
-			if (slave_tx_fail_count[i] > 0) {
-				for (j = 0; j < slave_tx_fail_count[i]; j++)
-					bufs[bufs_idx++] = slave_bufs[i][j];
-			}
+			total_tx_fail_count += slave_tx_fail_count;
+			memcpy(&bufs[nb_bufs - total_tx_fail_count],
+			       &slave_bufs[i][slave_tx_count],
+			       slave_tx_fail_count * sizeof(bufs[0]));
 		}
 	}
 
@@ -400,8 +392,9 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 	/* Cast to structure, containing bonded device's port id and queue id */
 	struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
 	struct bond_dev_private *internals = bd_rx_q->dev_private;
-	struct ether_addr bond_mac;
-
+	struct rte_eth_dev *bonded_eth_dev =
+					&rte_eth_devices[internals->port_id];
+	struct ether_addr *bond_mac = bonded_eth_dev->data->mac_addrs;
 	struct ether_hdr *hdr;
 
 	const uint16_t ether_type_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW);
@@ -414,7 +407,6 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 	uint8_t i, j, k;
 	uint8_t subtype;
 
-	rte_eth_macaddr_get(internals->port_id, &bond_mac);
 	/* Copy slave list to protect against slave up/down changes during tx
 	 * bursting */
 	slave_count = internals->active_slave_count;
@@ -428,7 +420,7 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 	}
 	for (i = 0; i < slave_count && num_rx_total < nb_pkts; i++) {
 		j = num_rx_total;
-		collecting = ACTOR_STATE(&mode_8023ad_ports[slaves[idx]],
+		collecting = ACTOR_STATE(&bond_mode_8023ad_ports[slaves[idx]],
 					 COLLECTING);
 
 		/* Read packets from this slave */
@@ -457,9 +449,11 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 			 * in collecting state or bonding interface is not in promiscuous
 			 * mode and packet address does not match. */
 			if (unlikely(is_lacp_packets(hdr->ether_type, subtype, bufs[j]) ||
-				!collecting || (!promisc &&
-					!is_multicast_ether_addr(&hdr->d_addr) &&
-					!is_same_ether_addr(&bond_mac, &hdr->d_addr)))) {
+				!collecting ||
+				(!promisc &&
+				 !is_multicast_ether_addr(&hdr->d_addr) &&
+				 !is_same_ether_addr(bond_mac,
+						     &hdr->d_addr)))) {
 
 				if (hdr->ether_type == ether_type_slow_be) {
 					bond_mode_8023ad_handle_slow_pkt(
@@ -480,7 +474,9 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 			idx = 0;
 	}
 
-	internals->active_slave = idx;
+	if (++internals->active_slave == slave_count)
+		internals->active_slave = 0;
+
 	return num_rx_total;
 }
 
@@ -715,8 +711,8 @@ bond_ethdev_tx_burst_round_robin(void *queue, struct rte_mbuf **bufs,
 				tx_fail_total += tx_fail_slave;
 
 				memcpy(&bufs[nb_pkts - tx_fail_total],
-						&slave_bufs[i][num_tx_slave],
-						tx_fail_slave * sizeof(bufs[0]));
+				       &slave_bufs[i][num_tx_slave],
+				       tx_fail_slave * sizeof(bufs[0]));
 			}
 			num_tx_total += num_tx_slave;
 		}
@@ -1221,10 +1217,10 @@ bond_ethdev_tx_burst_balance(void *queue, struct rte_mbuf **bufs,
 	/* Mapping array generated by hash function to map mbufs to slaves */
 	uint16_t bufs_slave_port_idxs[nb_bufs];
 
-	uint16_t slave_tx_count, slave_tx_fail_count[RTE_MAX_ETHPORTS] = { 0 };
+	uint16_t slave_tx_count;
 	uint16_t total_tx_count = 0, total_tx_fail_count = 0;
 
-	uint16_t i, j;
+	uint16_t i;
 
 	if (unlikely(nb_bufs == 0))
 		return 0;
@@ -1265,34 +1261,12 @@ bond_ethdev_tx_burst_balance(void *queue, struct rte_mbuf **bufs,
 
 		/* If tx burst fails move packets to end of bufs */
 		if (unlikely(slave_tx_count < slave_nb_bufs[i])) {
-			slave_tx_fail_count[i] = slave_nb_bufs[i] -
+			int slave_tx_fail_count = slave_nb_bufs[i] -
 					slave_tx_count;
-			total_tx_fail_count += slave_tx_fail_count[i];
-
-			/*
-			 * Shift bufs to beginning of array to allow reordering
-			 * later
-			 */
-			for (j = 0; j < slave_tx_fail_count[i]; j++) {
-				slave_bufs[i][j] =
-					slave_bufs[i][(slave_tx_count - 1) + j];
-			}
-		}
-	}
-
-	/*
-	 * If there are tx burst failures we move packets to end of bufs to
-	 * preserve expected PMD behaviour of all failed transmitted being
-	 * at the end of the input mbuf array
-	 */
-	if (unlikely(total_tx_fail_count > 0)) {
-		int bufs_idx = nb_bufs - total_tx_fail_count - 1;
-
-		for (i = 0; i < slave_count; i++) {
-			if (slave_tx_fail_count[i] > 0) {
-				for (j = 0; j < slave_tx_fail_count[i]; j++)
-					bufs[bufs_idx++] = slave_bufs[i][j];
-			}
+			total_tx_fail_count += slave_tx_fail_count;
+			memcpy(&bufs[nb_bufs - total_tx_fail_count],
+			       &slave_bufs[i][slave_tx_count],
+			       slave_tx_fail_count * sizeof(bufs[0]));
 		}
 	}
 
@@ -1319,10 +1293,10 @@ bond_ethdev_tx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 	/* Mapping array generated by hash function to map mbufs to slaves */
 	uint16_t bufs_slave_port_idxs[RTE_MAX_ETHPORTS] = { 0 };
 
-	uint16_t slave_tx_count, slave_tx_fail_count[RTE_MAX_ETHPORTS] = { 0 };
+	uint16_t slave_tx_count;
 	uint16_t total_tx_count = 0, total_tx_fail_count = 0;
 
-	uint16_t i, j;
+	uint16_t i;
 
 	if (unlikely(nb_bufs == 0))
 		return 0;
@@ -1338,7 +1312,7 @@ bond_ethdev_tx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 
 	dist_slave_count = 0;
 	for (i = 0; i < slave_count; i++) {
-		struct port *port = &mode_8023ad_ports[slave_port_ids[i]];
+		struct port *port = &bond_mode_8023ad_ports[slave_port_ids[i]];
 
 		if (ACTOR_STATE(port, DISTRIBUTING))
 			dist_slave_port_ids[dist_slave_count++] =
@@ -1380,46 +1354,20 @@ bond_ethdev_tx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 
 			/* If tx burst fails move packets to end of bufs */
 			if (unlikely(slave_tx_count < slave_nb_bufs[i])) {
-				slave_tx_fail_count[i] = slave_nb_bufs[i] -
+				int slave_tx_fail_count = slave_nb_bufs[i] -
 						slave_tx_count;
-				total_tx_fail_count += slave_tx_fail_count[i];
-
-				/*
-				 * Shift bufs to beginning of array to allow
-				 * reordering later
-				 */
-				for (j = 0; j < slave_tx_fail_count[i]; j++)
-					slave_bufs[i][j] =
-						slave_bufs[i]
-							[(slave_tx_count - 1)
-							+ j];
-			}
-		}
+				total_tx_fail_count += slave_tx_fail_count;
 
-		/*
-		 * If there are tx burst failures we move packets to end of
-		 * bufs to preserve expected PMD behaviour of all failed
-		 * transmitted being at the end of the input mbuf array
-		 */
-		if (unlikely(total_tx_fail_count > 0)) {
-			int bufs_idx = nb_bufs - total_tx_fail_count - 1;
-
-			for (i = 0; i < slave_count; i++) {
-				if (slave_tx_fail_count[i] > 0) {
-					for (j = 0;
-						j < slave_tx_fail_count[i];
-						j++) {
-						bufs[bufs_idx++] =
-							slave_bufs[i][j];
-					}
-				}
+				memcpy(&bufs[nb_bufs - total_tx_fail_count],
+				       &slave_bufs[i][slave_tx_count],
+				       slave_tx_fail_count * sizeof(bufs[0]));
 			}
 		}
 	}
 
 	/* Check for LACP control packets and send if available */
 	for (i = 0; i < slave_count; i++) {
-		struct port *port = &mode_8023ad_ports[slave_port_ids[i]];
+		struct port *port = &bond_mode_8023ad_ports[slave_port_ids[i]];
 		struct rte_mbuf *ctrl_pkt = NULL;
 
 		if (likely(rte_ring_empty(port->tx_ring)))
@@ -1770,7 +1718,7 @@ slave_configure_slow_queue(struct rte_eth_dev *bonded_eth_dev,
 	int errval = 0;
 	struct bond_dev_private *internals = (struct bond_dev_private *)
 		bonded_eth_dev->data->dev_private;
-	struct port *port = &mode_8023ad_ports[slave_eth_dev->data->port_id];
+	struct port *port = &bond_mode_8023ad_ports[slave_eth_dev->data->port_id];
 
 	if (port->slow_pool == NULL) {
 		char mem_name[256];
@@ -1847,12 +1795,11 @@ slave_configure(struct rte_eth_dev *bonded_eth_dev,
 
 	/* If RSS is enabled for bonding, try to enable it for slaves  */
 	if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
-		if (bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len
-				!= 0) {
+		if (internals->rss_key_len != 0) {
 			slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len =
-					bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len;
+					internals->rss_key_len;
 			slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key =
-					bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key;
+					internals->rss_key;
 		} else {
 			slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
 		}
@@ -2210,7 +2157,7 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev)
 
 		/* Discard all messages to/from mode 4 state machines */
 		for (i = 0; i < internals->active_slave_count; i++) {
-			port = &mode_8023ad_ports[internals->active_slaves[i]];
+			port = &bond_mode_8023ad_ports[internals->active_slaves[i]];
 
 			RTE_ASSERT(port->rx_ring != NULL);
 			while (rte_ring_dequeue(port->rx_ring, &pkt) != -ENOENT)
@@ -2229,12 +2176,15 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev)
 			tlb_last_obytets[internals->active_slaves[i]] = 0;
 	}
 
-	internals->link_status_polling_enabled = 0;
-	for (i = 0; i < internals->slave_count; i++)
-		internals->slaves[i].last_link_status = 0;
-
 	eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
 	eth_dev->data->dev_started = 0;
+
+	internals->link_status_polling_enabled = 0;
+	for (i = 0; i < internals->slave_count; i++) {
+		internals->slaves[i].last_link_status = 0;
+		rte_eth_dev_stop(internals->slaves[i].port_id);
+		deactivate_slave(eth_dev, internals->slaves[i].port_id);
+	}
 }
 
 void
@@ -2303,6 +2253,16 @@ bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	dev_info->max_rx_queues = max_nb_rx_queues;
 	dev_info->max_tx_queues = max_nb_tx_queues;
 
+	memcpy(&dev_info->default_rxconf, &internals->default_rxconf,
+	       sizeof(dev_info->default_rxconf));
+	memcpy(&dev_info->default_txconf, &internals->default_txconf,
+	       sizeof(dev_info->default_txconf));
+
+	memcpy(&dev_info->rx_desc_lim, &internals->rx_desc_lim,
+	       sizeof(dev_info->rx_desc_lim));
+	memcpy(&dev_info->tx_desc_lim, &internals->tx_desc_lim,
+	       sizeof(dev_info->tx_desc_lim));
+
 	/**
 	 * If dedicated hw queues enabled for link bonding device in LACP mode
 	 * then we need to reduce the maximum number of data path queues by 1.
@@ -3123,6 +3083,14 @@ bond_alloc(struct rte_vdev_device *dev, uint8_t mode)
 	/* Initially allow to choose any offload type */
 	internals->flow_type_rss_offloads = ETH_RSS_PROTO_MASK;
 
+	memset(&internals->default_rxconf, 0,
+	       sizeof(internals->default_rxconf));
+	memset(&internals->default_txconf, 0,
+	       sizeof(internals->default_txconf));
+
+	memset(&internals->rx_desc_lim, 0, sizeof(internals->rx_desc_lim));
+	memset(&internals->tx_desc_lim, 0, sizeof(internals->tx_desc_lim));
+
 	memset(internals->active_slaves, 0, sizeof(internals->active_slaves));
 	memset(internals->slaves, 0, sizeof(internals->slaves));
 
@@ -3162,10 +3130,9 @@ bond_alloc(struct rte_vdev_device *dev, uint8_t mode)
 
 err:
 	rte_free(internals);
-	if (eth_dev != NULL) {
-		rte_free(eth_dev->data->mac_addrs);
-		rte_eth_dev_release_port(eth_dev);
-	}
+	if (eth_dev != NULL)
+		eth_dev->data->dev_private = NULL;
+	rte_eth_dev_release_port(eth_dev);
 	return -1;
 }
 
@@ -3186,8 +3153,7 @@ bond_probe(struct rte_vdev_device *dev)
 	name = rte_vdev_device_name(dev);
 	RTE_BOND_LOG(INFO, "Initializing pmd_bond for %s", name);
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
-	    strlen(rte_vdev_device_args(dev)) == 0) {
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 		eth_dev = rte_eth_dev_attach_secondary(name);
 		if (!eth_dev) {
 			RTE_BOND_LOG(ERR, "Failed to probe %s", name);
@@ -3302,6 +3268,9 @@ bond_remove(struct rte_vdev_device *dev)
 	if (eth_dev == NULL)
 		return -ENODEV;
 
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return rte_eth_dev_release_port(eth_dev);
+
 	RTE_ASSERT(eth_dev->device == &dev->device);
 
 	internals = eth_dev->data->dev_private;
@@ -3324,8 +3293,6 @@ bond_remove(struct rte_vdev_device *dev)
 	rte_mempool_free(internals->mode6.mempool);
 	rte_bitmap_free(internals->vlan_filter_bmp);
 	rte_free(internals->vlan_filter_bmpmem);
-	rte_free(eth_dev->data->dev_private);
-	rte_free(eth_dev->data->mac_addrs);
 
 	rte_eth_dev_release_port(eth_dev);
 
@@ -3353,16 +3320,30 @@ bond_ethdev_configure(struct rte_eth_dev *dev)
 
 	unsigned i, j;
 
-	/* If RSS is enabled, fill table and key with default values */
+	/*
+	 * If RSS is enabled, fill table with default values and
+	 * set key to the the value specified in port RSS configuration.
+	 * Fall back to default RSS key if the key is not specified
+	 */
 	if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) {
-		dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = internals->rss_key;
-		dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len = 0;
-		memcpy(internals->rss_key, default_rss_key, 40);
+		if (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key != NULL) {
+			internals->rss_key_len =
+				dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len;
+			memcpy(internals->rss_key,
+			       dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key,
+			       internals->rss_key_len);
+		} else {
+			internals->rss_key_len = sizeof(default_rss_key);
+			memcpy(internals->rss_key, default_rss_key,
+			       internals->rss_key_len);
+		}
 
 		for (i = 0; i < RTE_DIM(internals->reta_conf); i++) {
 			internals->reta_conf[i].mask = ~0LL;
 			for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
-				internals->reta_conf[i].reta[j] = j % dev->data->nb_rx_queues;
+				internals->reta_conf[i].reta[j] =
+						(i * RTE_RETA_GROUP_SIZE + j) %
+						dev->data->nb_rx_queues;
 		}
 	}
 
@@ -3618,7 +3599,7 @@ int bond_logtype;
 
 RTE_INIT(bond_init_log)
 {
-	bond_logtype = rte_log_register("pmd.net.bon");
+	bond_logtype = rte_log_register("pmd.net.bond");
 	if (bond_logtype >= 0)
 		rte_log_set_level(bond_logtype, RTE_LOG_NOTICE);
 }
diff --git a/drivers/net/bonding/rte_eth_bond_private.h b/drivers/net/bonding/rte_eth_bond_private.h
index 43e0e448..3ea5d686 100644
--- a/drivers/net/bonding/rte_eth_bond_private.h
+++ b/drivers/net/bonding/rte_eth_bond_private.h
@@ -5,9 +5,11 @@
 #ifndef _RTE_ETH_BOND_PRIVATE_H_
 #define _RTE_ETH_BOND_PRIVATE_H_
 
+#include <stdint.h>
 #include <sys/queue.h>
 
 #include <rte_ethdev_driver.h>
+#include <rte_flow.h>
 #include <rte_spinlock.h>
 #include <rte_bitmap.h>
 #include <rte_flow_driver.h>
@@ -93,7 +95,8 @@ struct rte_flow {
 	/* Slaves flows */
 	struct rte_flow *flows[RTE_MAX_ETHPORTS];
 	/* Flow description for synchronization */
-	struct rte_flow_desc *fd;
+	struct rte_flow_conv_rule rule;
+	uint8_t rule_data[];
 };
 
 typedef void (*burst_xmit_hash_t)(struct rte_mbuf **buf, uint16_t nb_pkts,
@@ -160,6 +163,11 @@ struct bond_dev_private {
 	/** Bit mask of RSS offloads, the bit offset also means flow type */
 	uint64_t flow_type_rss_offloads;
 
+	struct rte_eth_rxconf default_rxconf;	/**< Default RxQ conf. */
+	struct rte_eth_txconf default_txconf;	/**< Default TxQ conf. */
+	struct rte_eth_desc_lim rx_desc_lim;	/**< Rx descriptor limits */
+	struct rte_eth_desc_lim tx_desc_lim;	/**< Tx descriptor limits */
+
 	uint16_t reta_size;
 	struct rte_eth_rss_reta_entry64 reta_conf[ETH_RSS_RETA_SIZE_512 /
 			RTE_RETA_GROUP_SIZE];
diff --git a/drivers/net/cxgbe/Makefile b/drivers/net/cxgbe/Makefile
index 5d66c4b3..68466f13 100644
--- a/drivers/net/cxgbe/Makefile
+++ b/drivers/net/cxgbe/Makefile
@@ -53,6 +53,8 @@ SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_filter.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_flow.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += t4_hw.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += clip_tbl.c
+SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += mps_tcam.c
+SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += l2t.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += t4vf_hw.c
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/cxgbe/base/adapter.h b/drivers/net/cxgbe/base/adapter.h
index e98dd218..47cfc5f5 100644
--- a/drivers/net/cxgbe/base/adapter.h
+++ b/drivers/net/cxgbe/base/adapter.h
@@ -324,7 +324,11 @@ struct adapter {
 
 	unsigned int clipt_start; /* CLIP table start */
 	unsigned int clipt_end;   /* CLIP table end */
+	unsigned int l2t_start;   /* Layer 2 table start */
+	unsigned int l2t_end;     /* Layer 2 table end */
 	struct clip_tbl *clipt;   /* CLIP table */
+	struct l2t_data *l2t;     /* Layer 2 table */
+	struct mpstcam_table *mpstcam;
 
 	struct tid_info tids;     /* Info used to access TID related tables */
 };
diff --git a/drivers/net/cxgbe/base/common.h b/drivers/net/cxgbe/base/common.h
index 157201da..fd200668 100644
--- a/drivers/net/cxgbe/base/common.h
+++ b/drivers/net/cxgbe/base/common.h
@@ -157,6 +157,7 @@ struct tp_params {
 	int port_shift;
 	int protocol_shift;
 	int ethertype_shift;
+	int macmatch_shift;
 
 	u64 hash_filter_mask;
 };
@@ -270,6 +271,7 @@ struct adapter_params {
 
 	bool ulptx_memwrite_dsgl;          /* use of T5 DSGL allowed */
 	u8 fw_caps_support;		  /* 32-bit Port Capabilities */
+	u8 filter2_wr_support;            /* FW support for FILTER2_WR */
 };
 
 /* Firmware Port Capabilities types.
@@ -388,6 +390,12 @@ int t4_free_vi(struct adapter *adap, unsigned int mbox,
 int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		  int mtu, int promisc, int all_multi, int bcast, int vlanex,
 		  bool sleep_ok);
+int t4_free_raw_mac_filt(struct adapter *adap, unsigned int viid,
+			 const u8 *addr, const u8 *mask, unsigned int idx,
+			 u8 lookup_type, u8 port_id, bool sleep_ok);
+int t4_alloc_raw_mac_filt(struct adapter *adap, unsigned int viid,
+			  const u8 *addr, const u8 *mask, unsigned int idx,
+			  u8 lookup_type, u8 port_id, bool sleep_ok);
 int t4_change_mac(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		  int idx, const u8 *addr, bool persist, bool add_smt);
 int t4_enable_vi_params(struct adapter *adap, unsigned int mbox,
diff --git a/drivers/net/cxgbe/base/t4_hw.c b/drivers/net/cxgbe/base/t4_hw.c
index 31762c9c..701e0b1f 100644
--- a/drivers/net/cxgbe/base/t4_hw.c
+++ b/drivers/net/cxgbe/base/t4_hw.c
@@ -4162,6 +4162,112 @@ int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
 }
 
 /**
+ *	t4_alloc_raw_mac_filt - Adds a raw mac entry in mps tcam
+ *	@adap: the adapter
+ *	@viid: the VI id
+ *	@mac: the MAC address
+ *	@mask: the mask
+ *	@idx: index at which to add this entry
+ *	@port_id: the port index
+ *	@lookup_type: MAC address for inner (1) or outer (0) header
+ *	@sleep_ok: call is allowed to sleep
+ *
+ *	Adds the mac entry at the specified index using raw mac interface.
+ *
+ *	Returns a negative error number or the allocated index for this mac.
+ */
+int t4_alloc_raw_mac_filt(struct adapter *adap, unsigned int viid,
+			  const u8 *addr, const u8 *mask, unsigned int idx,
+			  u8 lookup_type, u8 port_id, bool sleep_ok)
+{
+	int ret = 0;
+	struct fw_vi_mac_cmd c;
+	struct fw_vi_mac_raw *p = &c.u.raw;
+	u32 val;
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_VI_MAC_CMD) |
+				   F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
+				   V_FW_VI_MAC_CMD_VIID(viid));
+	val = V_FW_CMD_LEN16(1) |
+	      V_FW_VI_MAC_CMD_ENTRY_TYPE(FW_VI_MAC_TYPE_RAW);
+	c.freemacs_to_len16 = cpu_to_be32(val);
+
+	/* Specify that this is an inner mac address */
+	p->raw_idx_pkd = cpu_to_be32(V_FW_VI_MAC_CMD_RAW_IDX(idx));
+
+	/* Lookup Type. Outer header: 0, Inner header: 1 */
+	p->data0_pkd = cpu_to_be32(V_DATALKPTYPE(lookup_type) |
+				   V_DATAPORTNUM(port_id));
+	/* Lookup mask and port mask */
+	p->data0m_pkd = cpu_to_be64(V_DATALKPTYPE(M_DATALKPTYPE) |
+				    V_DATAPORTNUM(M_DATAPORTNUM));
+
+	/* Copy the address and the mask */
+	memcpy((u8 *)&p->data1[0] + 2, addr, ETHER_ADDR_LEN);
+	memcpy((u8 *)&p->data1m[0] + 2, mask, ETHER_ADDR_LEN);
+
+	ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok);
+	if (ret == 0) {
+		ret = G_FW_VI_MAC_CMD_RAW_IDX(be32_to_cpu(p->raw_idx_pkd));
+		if (ret != (int)idx)
+			ret = -ENOMEM;
+	}
+
+	return ret;
+}
+
+/**
+ *	t4_free_raw_mac_filt - Frees a raw mac entry in mps tcam
+ *	@adap: the adapter
+ *	@viid: the VI id
+ *	@addr: the MAC address
+ *	@mask: the mask
+ *	@idx: index of the entry in mps tcam
+ *	@lookup_type: MAC address for inner (1) or outer (0) header
+ *	@port_id: the port index
+ *	@sleep_ok: call is allowed to sleep
+ *
+ *	Removes the mac entry at the specified index using raw mac interface.
+ *
+ *	Returns a negative error number on failure.
+ */
+int t4_free_raw_mac_filt(struct adapter *adap, unsigned int viid,
+			 const u8 *addr, const u8 *mask, unsigned int idx,
+			 u8 lookup_type, u8 port_id, bool sleep_ok)
+{
+	struct fw_vi_mac_cmd c;
+	struct fw_vi_mac_raw *p = &c.u.raw;
+	u32 raw;
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_VI_MAC_CMD) |
+				   F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
+				   V_FW_CMD_EXEC(0) |
+				   V_FW_VI_MAC_CMD_VIID(viid));
+	raw = V_FW_VI_MAC_CMD_ENTRY_TYPE(FW_VI_MAC_TYPE_RAW);
+	c.freemacs_to_len16 = cpu_to_be32(V_FW_VI_MAC_CMD_FREEMACS(0) |
+					  raw |
+					  V_FW_CMD_LEN16(1));
+
+	p->raw_idx_pkd = cpu_to_be32(V_FW_VI_MAC_CMD_RAW_IDX(idx) |
+				     FW_VI_MAC_ID_BASED_FREE);
+
+	/* Lookup Type. Outer header: 0, Inner header: 1 */
+	p->data0_pkd = cpu_to_be32(V_DATALKPTYPE(lookup_type) |
+				   V_DATAPORTNUM(port_id));
+	/* Lookup mask and port mask */
+	p->data0m_pkd = cpu_to_be64(V_DATALKPTYPE(M_DATALKPTYPE) |
+				    V_DATAPORTNUM(M_DATAPORTNUM));
+
+	/* Copy the address and the mask */
+	memcpy((u8 *)&p->data1[0] + 2, addr, ETHER_ADDR_LEN);
+	memcpy((u8 *)&p->data1m[0] + 2, mask, ETHER_ADDR_LEN);
+
+	return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok);
+}
+
+/**
  * t4_change_mac - modifies the exact-match filter for a MAC address
  * @adap: the adapter
  * @mbox: mailbox to use for the FW command
@@ -5145,6 +5251,8 @@ int t4_init_tp_params(struct adapter *adap)
 							       F_PROTOCOL);
 	adap->params.tp.ethertype_shift = t4_filter_field_shift(adap,
 								F_ETHERTYPE);
+	adap->params.tp.macmatch_shift = t4_filter_field_shift(adap,
+							       F_MACMATCH);
 
 	/*
 	 * If TP_INGRESS_CONFIG.VNID == 0, then TP_VLAN_PRI_MAP.VNIC_ID
diff --git a/drivers/net/cxgbe/base/t4_msg.h b/drivers/net/cxgbe/base/t4_msg.h
index 5d433c91..9e052b0f 100644
--- a/drivers/net/cxgbe/base/t4_msg.h
+++ b/drivers/net/cxgbe/base/t4_msg.h
@@ -11,7 +11,9 @@ enum {
 	CPL_SET_TCB_FIELD     = 0x5,
 	CPL_ABORT_REQ         = 0xA,
 	CPL_ABORT_RPL         = 0xB,
+	CPL_L2T_WRITE_REQ     = 0x12,
 	CPL_TID_RELEASE       = 0x1A,
+	CPL_L2T_WRITE_RPL     = 0x23,
 	CPL_ACT_OPEN_RPL      = 0x25,
 	CPL_ABORT_RPL_RSS     = 0x2D,
 	CPL_SET_TCB_RPL       = 0x3A,
@@ -30,6 +32,7 @@ enum CPL_error {
 
 enum {
 	ULP_MODE_NONE          = 0,
+	ULP_MODE_TCPDDP        = 5,
 };
 
 enum {
@@ -66,6 +69,9 @@ union opcode_tid {
 #define M_TID_TID    0x3fff
 #define G_TID_TID(x) (((x) >> S_TID_TID) & M_TID_TID)
 
+#define S_TID_QID    14
+#define V_TID_QID(x) ((x) << S_TID_QID)
+
 struct rss_header {
 	__u8 opcode;
 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
@@ -133,6 +139,12 @@ struct work_request_hdr {
 #define V_TCAM_BYPASS(x) ((__u64)(x) << S_TCAM_BYPASS)
 #define F_TCAM_BYPASS    V_TCAM_BYPASS(1ULL)
 
+#define S_L2T_IDX    36
+#define V_L2T_IDX(x) ((__u64)(x) << S_L2T_IDX)
+
+#define S_NAGLE    49
+#define V_NAGLE(x) ((__u64)(x) << S_NAGLE)
+
 /* option 2 fields */
 #define S_RSS_QUEUE    0
 #define V_RSS_QUEUE(x) ((x) << S_RSS_QUEUE)
@@ -151,6 +163,9 @@ struct work_request_hdr {
 #define S_CCTRL_ECN    27
 #define V_CCTRL_ECN(x) ((x) << S_CCTRL_ECN)
 
+#define S_SACK_EN    30
+#define V_SACK_EN(x) ((x) << S_SACK_EN)
+
 #define S_T5_OPT_2_VALID    31
 #define V_T5_OPT_2_VALID(x) ((x) << S_T5_OPT_2_VALID)
 #define F_T5_OPT_2_VALID    V_T5_OPT_2_VALID(1U)
@@ -421,6 +436,35 @@ struct cpl_rx_pkt {
 	__be16 err_vec;
 };
 
+struct cpl_l2t_write_req {
+	WR_HDR;
+	union opcode_tid ot;
+	__be16 params;
+	__be16 l2t_idx;
+	__be16 vlan;
+	__u8   dst_mac[6];
+};
+
+/* cpl_l2t_write_req.params fields */
+#define S_L2T_W_PORT    8
+#define V_L2T_W_PORT(x) ((x) << S_L2T_W_PORT)
+
+#define S_L2T_W_LPBK    10
+#define V_L2T_W_LPBK(x) ((x) << S_L2T_W_LPBK)
+
+#define S_L2T_W_ARPMISS         11
+#define V_L2T_W_ARPMISS(x)      ((x) << S_L2T_W_ARPMISS)
+
+#define S_L2T_W_NOREPLY    15
+#define V_L2T_W_NOREPLY(x) ((x) << S_L2T_W_NOREPLY)
+
+struct cpl_l2t_write_rpl {
+	RSS_HDR
+	union opcode_tid ot;
+	__u8 status;
+	__u8 rsvd[3];
+};
+
 /* rx_pkt.l2info fields */
 #define S_RXF_UDP    22
 #define V_RXF_UDP(x) ((x) << S_RXF_UDP)
diff --git a/drivers/net/cxgbe/base/t4_regs.h b/drivers/net/cxgbe/base/t4_regs.h
index 6f872edc..af8c741e 100644
--- a/drivers/net/cxgbe/base/t4_regs.h
+++ b/drivers/net/cxgbe/base/t4_regs.h
@@ -45,6 +45,14 @@
 #define MPS_T5_CLS_SRAM_H(idx) (A_MPS_T5_CLS_SRAM_H + (idx) * 8)
 #define NUM_MPS_T5_CLS_SRAM_H_INSTANCES 512
 
+#define S_DATAPORTNUM    12
+#define M_DATAPORTNUM    0xfU
+#define V_DATAPORTNUM(x) ((x) << S_DATAPORTNUM)
+
+#define S_DATALKPTYPE    10
+#define M_DATALKPTYPE    0x3U
+#define V_DATALKPTYPE(x) ((x) << S_DATALKPTYPE)
+
 /* registers for module SGE */
 #define SGE_BASE_ADDR 0x1000
 
diff --git a/drivers/net/cxgbe/base/t4_tcb.h b/drivers/net/cxgbe/base/t4_tcb.h
index 25435f9f..68cda773 100644
--- a/drivers/net/cxgbe/base/t4_tcb.h
+++ b/drivers/net/cxgbe/base/t4_tcb.h
@@ -6,6 +6,9 @@
 #ifndef _T4_TCB_DEFS_H
 #define _T4_TCB_DEFS_H
 
+/* 95:32 */
+#define W_TCB_T_FLAGS    1
+
 /* 105:96 */
 #define W_TCB_RSS_INFO    3
 #define S_TCB_RSS_INFO    0
@@ -23,4 +26,6 @@
 #define M_TCB_T_RTT_TS_RECENT_AGE    0xffffffffULL
 #define V_TCB_T_RTT_TS_RECENT_AGE(x) ((x) << S_TCB_T_RTT_TS_RECENT_AGE)
 
+#define S_TF_CCTRL_RFR    62
+
 #endif /* _T4_TCB_DEFS_H */
diff --git a/drivers/net/cxgbe/base/t4fw_interface.h b/drivers/net/cxgbe/base/t4fw_interface.h
index e80b58a3..06d3ef3a 100644
--- a/drivers/net/cxgbe/base/t4fw_interface.h
+++ b/drivers/net/cxgbe/base/t4fw_interface.h
@@ -61,6 +61,7 @@ enum fw_wr_opcodes {
 	FW_ETH_TX_PKTS_WR	= 0x09,
 	FW_ETH_TX_PKT_VM_WR	= 0x11,
 	FW_ETH_TX_PKTS_VM_WR	= 0x12,
+	FW_FILTER2_WR		= 0x77,
 	FW_ETH_TX_PKTS2_WR      = 0x78,
 };
 
@@ -165,7 +166,7 @@ enum fw_filter_wr_cookie {
 	FW_FILTER_WR_EINVAL,
 };
 
-struct fw_filter_wr {
+struct fw_filter2_wr {
 	__be32 op_pkd;
 	__be32 len16_pkd;
 	__be64 r3;
@@ -195,6 +196,19 @@ struct fw_filter_wr {
 	__be16 fpm;
 	__be16 r7;
 	__u8   sma[6];
+	__be16 r8;
+	__u8   filter_type_swapmac;
+	__u8   natmode_to_ulp_type;
+	__be16 newlport;
+	__be16 newfport;
+	__u8   newlip[16];
+	__u8   newfip[16];
+	__be32 natseqcheck;
+	__be32 r9;
+	__be64 r10;
+	__be64 r11;
+	__be64 r12;
+	__be64 r13;
 };
 
 #define S_FW_FILTER_WR_TID	12
@@ -300,6 +314,15 @@ struct fw_filter_wr {
 #define S_FW_FILTER_WR_MATCHTYPEM	0
 #define V_FW_FILTER_WR_MATCHTYPEM(x)	((x) << S_FW_FILTER_WR_MATCHTYPEM)
 
+#define S_FW_FILTER2_WR_SWAPMAC		0
+#define V_FW_FILTER2_WR_SWAPMAC(x)	((x) << S_FW_FILTER2_WR_SWAPMAC)
+
+#define S_FW_FILTER2_WR_NATMODE		5
+#define V_FW_FILTER2_WR_NATMODE(x)	((x) << S_FW_FILTER2_WR_NATMODE)
+
+#define S_FW_FILTER2_WR_ULP_TYPE	0
+#define V_FW_FILTER2_WR_ULP_TYPE(x)	((x) << S_FW_FILTER2_WR_ULP_TYPE)
+
 /******************************************************************************
  *  C O M M A N D s
  *********************/
@@ -655,6 +678,7 @@ enum fw_params_param_dev {
 	FW_PARAMS_PARAM_DEV_FWREV	= 0x0B, /* fw version */
 	FW_PARAMS_PARAM_DEV_TPREV	= 0x0C, /* tp version */
 	FW_PARAMS_PARAM_DEV_ULPTX_MEMWRITE_DSGL = 0x17,
+	FW_PARAMS_PARAM_DEV_FILTER2_WR	= 0x1D,
 };
 
 /*
@@ -665,6 +689,8 @@ enum fw_params_param_pfvf {
 	FW_PARAMS_PARAM_PFVF_CLIP_END = 0x04,
 	FW_PARAMS_PARAM_PFVF_FILTER_START = 0x05,
 	FW_PARAMS_PARAM_PFVF_FILTER_END = 0x06,
+	FW_PARAMS_PARAM_PFVF_L2T_START = 0x13,
+	FW_PARAMS_PARAM_PFVF_L2T_END = 0x14,
 	FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP = 0x31,
 	FW_PARAMS_PARAM_PFVF_PORT_CAPS32 = 0x3A
 };
@@ -1280,12 +1306,17 @@ struct fw_vi_cmd {
 /* Special VI_MAC command index ids */
 #define FW_VI_MAC_ADD_MAC		0x3FF
 #define FW_VI_MAC_ADD_PERSIST_MAC	0x3FE
+#define FW_VI_MAC_ID_BASED_FREE         0x3FC
 
 enum fw_vi_mac_smac {
 	FW_VI_MAC_MPS_TCAM_ENTRY,
 	FW_VI_MAC_SMT_AND_MPSTCAM
 };
 
+enum fw_vi_mac_entry_types {
+	FW_VI_MAC_TYPE_RAW = 0x2,
+};
+
 struct fw_vi_mac_cmd {
 	__be32 op_to_viid;
 	__be32 freemacs_to_len16;
@@ -1297,6 +1328,13 @@ struct fw_vi_mac_cmd {
 		struct fw_vi_mac_hash {
 			__be64 hashvec;
 		} hash;
+		struct fw_vi_mac_raw {
+			__be32 raw_idx_pkd;
+			__be32 data0_pkd;
+			__be32 data1[2];
+			__be64 data0m_pkd;
+			__be32 data1m[2];
+		} raw;
 	} u;
 };
 
@@ -1306,6 +1344,12 @@ struct fw_vi_mac_cmd {
 #define G_FW_VI_MAC_CMD_VIID(x)	\
 	(((x) >> S_FW_VI_MAC_CMD_VIID) & M_FW_VI_MAC_CMD_VIID)
 
+#define S_FW_VI_MAC_CMD_FREEMACS	31
+#define V_FW_VI_MAC_CMD_FREEMACS(x)	((x) << S_FW_VI_MAC_CMD_FREEMACS)
+
+#define S_FW_VI_MAC_CMD_ENTRY_TYPE      23
+#define V_FW_VI_MAC_CMD_ENTRY_TYPE(x)   ((x) << S_FW_VI_MAC_CMD_ENTRY_TYPE)
+
 #define S_FW_VI_MAC_CMD_VALID		15
 #define M_FW_VI_MAC_CMD_VALID		0x1
 #define V_FW_VI_MAC_CMD_VALID(x)	((x) << S_FW_VI_MAC_CMD_VALID)
@@ -1325,6 +1369,12 @@ struct fw_vi_mac_cmd {
 #define G_FW_VI_MAC_CMD_IDX(x)	\
 	(((x) >> S_FW_VI_MAC_CMD_IDX) & M_FW_VI_MAC_CMD_IDX)
 
+#define S_FW_VI_MAC_CMD_RAW_IDX         16
+#define M_FW_VI_MAC_CMD_RAW_IDX         0xffff
+#define V_FW_VI_MAC_CMD_RAW_IDX(x)      ((x) << S_FW_VI_MAC_CMD_RAW_IDX)
+#define G_FW_VI_MAC_CMD_RAW_IDX(x)      \
+	(((x) >> S_FW_VI_MAC_CMD_RAW_IDX) & M_FW_VI_MAC_CMD_RAW_IDX)
+
 struct fw_vi_rxmode_cmd {
 	__be32 op_to_viid;
 	__be32 retval_len16;
diff --git a/drivers/net/cxgbe/cxgbe.h b/drivers/net/cxgbe/cxgbe.h
index 5e6f5c98..eb58f880 100644
--- a/drivers/net/cxgbe/cxgbe.h
+++ b/drivers/net/cxgbe/cxgbe.h
@@ -34,6 +34,21 @@
 				    ETH_RSS_IPV6_UDP_EX)
 #define CXGBE_RSS_HF_ALL (ETH_RSS_IP | ETH_RSS_TCP | ETH_RSS_UDP)
 
+/* Tx/Rx Offloads supported */
+#define CXGBE_TX_OFFLOADS (DEV_TX_OFFLOAD_VLAN_INSERT | \
+			   DEV_TX_OFFLOAD_IPV4_CKSUM | \
+			   DEV_TX_OFFLOAD_UDP_CKSUM | \
+			   DEV_TX_OFFLOAD_TCP_CKSUM | \
+			   DEV_TX_OFFLOAD_TCP_TSO)
+
+#define CXGBE_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_STRIP | \
+			   DEV_RX_OFFLOAD_IPV4_CKSUM | \
+			   DEV_RX_OFFLOAD_UDP_CKSUM | \
+			   DEV_RX_OFFLOAD_TCP_CKSUM | \
+			   DEV_RX_OFFLOAD_JUMBO_FRAME | \
+			   DEV_RX_OFFLOAD_SCATTER)
+
+
 #define CXGBE_DEVARG_KEEP_OVLAN "keep_ovlan"
 #define CXGBE_DEVARG_FORCE_LINK_UP "force_link_up"
 
diff --git a/drivers/net/cxgbe/cxgbe_ethdev.c b/drivers/net/cxgbe/cxgbe_ethdev.c
index 4dcad7a2..b2f83ea3 100644
--- a/drivers/net/cxgbe/cxgbe_ethdev.c
+++ b/drivers/net/cxgbe/cxgbe_ethdev.c
@@ -59,19 +59,6 @@
  */
 #include "t4_pci_id_tbl.h"
 
-#define CXGBE_TX_OFFLOADS (DEV_TX_OFFLOAD_VLAN_INSERT |\
-			   DEV_TX_OFFLOAD_IPV4_CKSUM |\
-			   DEV_TX_OFFLOAD_UDP_CKSUM |\
-			   DEV_TX_OFFLOAD_TCP_CKSUM |\
-			   DEV_TX_OFFLOAD_TCP_TSO)
-
-#define CXGBE_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_STRIP |\
-			   DEV_RX_OFFLOAD_CRC_STRIP |\
-			   DEV_RX_OFFLOAD_IPV4_CKSUM |\
-			   DEV_RX_OFFLOAD_JUMBO_FRAME |\
-			   DEV_RX_OFFLOAD_UDP_CKSUM |\
-			   DEV_RX_OFFLOAD_TCP_CKSUM)
-
 uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 			 uint16_t nb_pkts)
 {
@@ -341,6 +328,7 @@ void cxgbe_dev_close(struct rte_eth_dev *eth_dev)
 int cxgbe_dev_start(struct rte_eth_dev *eth_dev)
 {
 	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	struct rte_eth_rxmode *rx_conf = &eth_dev->data->dev_conf.rxmode;
 	struct adapter *adapter = pi->adapter;
 	int err = 0, i;
 
@@ -361,6 +349,11 @@ int cxgbe_dev_start(struct rte_eth_dev *eth_dev)
 			goto out;
 	}
 
+	if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)
+		eth_dev->data->scattered_rx = 1;
+	else
+		eth_dev->data->scattered_rx = 0;
+
 	cxgbe_enable_rx_queues(pi);
 
 	err = setup_rss(pi);
@@ -407,26 +400,16 @@ void cxgbe_dev_stop(struct rte_eth_dev *eth_dev)
 	 *  have been disabled
 	 */
 	t4_sge_eth_clear_queues(pi);
+	eth_dev->data->scattered_rx = 0;
 }
 
 int cxgbe_dev_configure(struct rte_eth_dev *eth_dev)
 {
 	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 	struct adapter *adapter = pi->adapter;
-	uint64_t configured_offloads;
 	int err;
 
 	CXGBE_FUNC_TRACE();
-	configured_offloads = eth_dev->data->dev_conf.rxmode.offloads;
-
-	/* KEEP_CRC offload flag is not supported by PMD
-	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
-	 */
-	if (rte_eth_dev_must_keep_crc(configured_offloads)) {
-		dev_info(adapter, "can't disable hw crc strip\n");
-		eth_dev->data->dev_conf.rxmode.offloads |=
-			DEV_RX_OFFLOAD_CRC_STRIP;
-	}
 
 	if (!(adapter->flags & FW_QUEUE_BOUND)) {
 		err = setup_sge_fwevtq(adapter);
@@ -1075,11 +1058,9 @@ static int cxgbe_get_regs(struct rte_eth_dev *eth_dev,
 int cxgbe_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr)
 {
 	struct port_info *pi = (struct port_info *)(dev->data->dev_private);
-	struct adapter *adapter = pi->adapter;
 	int ret;
 
-	ret = t4_change_mac(adapter, adapter->mbox, pi->viid,
-			    pi->xact_addr_filt, (u8 *)addr, true, true);
+	ret = cxgbe_mpstcam_modify(pi, (int)pi->xact_addr_filt, (u8 *)addr);
 	if (ret < 0) {
 		dev_err(adapter, "failed to set mac addr; err = %d\n",
 			ret);
diff --git a/drivers/net/cxgbe/cxgbe_filter.c b/drivers/net/cxgbe/cxgbe_filter.c
index 7f0d3800..ef1102be 100644
--- a/drivers/net/cxgbe/cxgbe_filter.c
+++ b/drivers/net/cxgbe/cxgbe_filter.c
@@ -8,6 +8,7 @@
 #include "t4_regs.h"
 #include "cxgbe_filter.h"
 #include "clip_tbl.h"
+#include "l2t.h"
 
 /**
  * Initialize Hash Filters
@@ -65,7 +66,8 @@ int validate_filter(struct adapter *adapter, struct ch_filter_specification *fs)
 #define U(_mask, _field) \
 	(!(fconf & (_mask)) && S(_field))
 
-	if (U(F_PORT, iport) || U(F_ETHERTYPE, ethtype) || U(F_PROTOCOL, proto))
+	if (U(F_PORT, iport) || U(F_ETHERTYPE, ethtype) ||
+	    U(F_PROTOCOL, proto) || U(F_MACMATCH, macidx))
 		return -EOPNOTSUPP;
 
 #undef S
@@ -87,6 +89,12 @@ int validate_filter(struct adapter *adapter, struct ch_filter_specification *fs)
 	if (fs->val.iport >= adapter->params.nports)
 		return -ERANGE;
 
+	if (!fs->cap && fs->nat_mode && !adapter->params.filter2_wr_support)
+		return -EOPNOTSUPP;
+
+	if (!fs->cap && fs->swapmac && !adapter->params.filter2_wr_support)
+		return -EOPNOTSUPP;
+
 	return 0;
 }
 
@@ -165,6 +173,16 @@ static void set_tcb_field(struct adapter *adapter, unsigned int ftid,
 }
 
 /**
+ * Set one of the t_flags bits in the TCB.
+ */
+static void set_tcb_tflag(struct adapter *adap, unsigned int ftid,
+			  unsigned int bit_pos, unsigned int val, int no_reply)
+{
+	set_tcb_field(adap, ftid,  W_TCB_T_FLAGS, 1ULL << bit_pos,
+		      (unsigned long long)val << bit_pos, no_reply);
+}
+
+/**
  * Build a CPL_SET_TCB_FIELD message as payload of a ULP_TX_PKT command.
  */
 static inline void mk_set_tcb_field_ulp(struct filter_entry *f,
@@ -257,6 +275,8 @@ static u64 hash_filter_ntuple(const struct filter_entry *f)
 
 	if (tp->ethertype_shift >= 0 && f->fs.mask.ethtype)
 		ntuple |= (u64)(f->fs.val.ethtype) << tp->ethertype_shift;
+	if (tp->macmatch_shift >= 0 && f->fs.mask.macidx)
+		ntuple |= (u64)(f->fs.val.macidx) << tp->macmatch_shift;
 
 	if (ntuple != tp->hash_filter_mask)
 		return 0;
@@ -425,7 +445,10 @@ static void mk_act_open_req6(struct filter_entry *f, struct rte_mbuf *mbuf,
 	req->local_ip_lo = local_lo;
 	req->peer_ip_hi = peer_hi;
 	req->peer_ip_lo = peer_lo;
-	req->opt0 = cpu_to_be64(V_DELACK(f->fs.hitcnts) |
+	req->opt0 = cpu_to_be64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
+					f->fs.newvlan == VLAN_REWRITE) |
+				V_DELACK(f->fs.hitcnts) |
+				V_L2T_IDX(f->l2t ? f->l2t->idx : 0) |
 				V_SMAC_SEL((cxgbe_port_viid(f->dev) & 0x7F)
 					   << 1) |
 				V_TX_CHAN(f->fs.eport) |
@@ -436,6 +459,7 @@ static void mk_act_open_req6(struct filter_entry *f, struct rte_mbuf *mbuf,
 			    V_RSS_QUEUE(f->fs.iq) |
 			    F_T5_OPT_2_VALID |
 			    F_RX_CHANNEL |
+			    V_SACK_EN(f->fs.swapmac) |
 			    V_CONG_CNTRL((f->fs.action == FILTER_DROP) |
 					 (f->fs.dirsteer << 1)) |
 			    V_CCTRL_ECN(f->fs.action == FILTER_SWITCH));
@@ -468,7 +492,10 @@ static void mk_act_open_req(struct filter_entry *f, struct rte_mbuf *mbuf,
 			f->fs.val.lip[2] << 16 | f->fs.val.lip[3] << 24;
 	req->peer_ip = f->fs.val.fip[0] | f->fs.val.fip[1] << 8 |
 			f->fs.val.fip[2] << 16 | f->fs.val.fip[3] << 24;
-	req->opt0 = cpu_to_be64(V_DELACK(f->fs.hitcnts) |
+	req->opt0 = cpu_to_be64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
+					f->fs.newvlan == VLAN_REWRITE) |
+				V_DELACK(f->fs.hitcnts) |
+				V_L2T_IDX(f->l2t ? f->l2t->idx : 0) |
 				V_SMAC_SEL((cxgbe_port_viid(f->dev) & 0x7F)
 					   << 1) |
 				V_TX_CHAN(f->fs.eport) |
@@ -479,6 +506,7 @@ static void mk_act_open_req(struct filter_entry *f, struct rte_mbuf *mbuf,
 			    V_RSS_QUEUE(f->fs.iq) |
 			    F_T5_OPT_2_VALID |
 			    F_RX_CHANNEL |
+			    V_SACK_EN(f->fs.swapmac) |
 			    V_CONG_CNTRL((f->fs.action == FILTER_DROP) |
 					 (f->fs.dirsteer << 1)) |
 			    V_CCTRL_ECN(f->fs.action == FILTER_SWITCH));
@@ -518,6 +546,22 @@ static int cxgbe_set_hash_filter(struct rte_eth_dev *dev,
 	f->dev = dev;
 	f->fs.iq = iq;
 
+	/*
+	 * If the new filter requires loopback Destination MAC and/or VLAN
+	 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
+	 * the filter.
+	 */
+	if (f->fs.newvlan == VLAN_INSERT ||
+	    f->fs.newvlan == VLAN_REWRITE) {
+		/* allocate L2T entry for new filter */
+		f->l2t = cxgbe_l2t_alloc_switching(dev, f->fs.vlan,
+						   f->fs.eport, f->fs.dmac);
+		if (!f->l2t) {
+			ret = -ENOMEM;
+			goto out_err;
+		}
+	}
+
 	atid = cxgbe_alloc_atid(t, f);
 	if (atid < 0)
 		goto out_err;
@@ -591,6 +635,7 @@ void clear_filter(struct filter_entry *f)
 
 /**
  * t4_mk_filtdelwr - create a delete filter WR
+ * @adap: adapter context
  * @ftid: the filter ID
  * @wr: the filter work request to populate
  * @qid: ingress queue to receive the delete notification
@@ -598,10 +643,14 @@ void clear_filter(struct filter_entry *f)
  * Creates a filter work request to delete the supplied filter.  If @qid is
  * negative the delete notification is suppressed.
  */
-static void t4_mk_filtdelwr(unsigned int ftid, struct fw_filter_wr *wr, int qid)
+static void t4_mk_filtdelwr(struct adapter *adap, unsigned int ftid,
+			    struct fw_filter2_wr *wr, int qid)
 {
 	memset(wr, 0, sizeof(*wr));
-	wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER_WR));
+	if (adap->params.filter2_wr_support)
+		wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER2_WR));
+	else
+		wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER_WR));
 	wr->len16_pkd = cpu_to_be32(V_FW_WR_LEN16(sizeof(*wr) / 16));
 	wr->tid_to_iq = cpu_to_be32(V_FW_FILTER_WR_TID(ftid) |
 				    V_FW_FILTER_WR_NOREPLY(qid < 0));
@@ -619,7 +668,7 @@ static int del_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
 	struct adapter *adapter = ethdev2adap(dev);
 	struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
 	struct rte_mbuf *mbuf;
-	struct fw_filter_wr *fwr;
+	struct fw_filter2_wr *fwr;
 	struct sge_ctrl_txq *ctrlq;
 	unsigned int port_id = ethdev2pinfo(dev)->port_id;
 
@@ -631,8 +680,8 @@ static int del_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
 	mbuf->data_len = sizeof(*fwr);
 	mbuf->pkt_len = mbuf->data_len;
 
-	fwr = rte_pktmbuf_mtod(mbuf, struct fw_filter_wr *);
-	t4_mk_filtdelwr(f->tid, fwr, adapter->sge.fw_evtq.abs_id);
+	fwr = rte_pktmbuf_mtod(mbuf, struct fw_filter2_wr *);
+	t4_mk_filtdelwr(adapter, f->tid, fwr, adapter->sge.fw_evtq.abs_id);
 
 	/*
 	 * Mark the filter as "pending" and ship off the Filter Work Request.
@@ -648,11 +697,24 @@ int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
 	struct adapter *adapter = ethdev2adap(dev);
 	struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
 	struct rte_mbuf *mbuf;
-	struct fw_filter_wr *fwr;
+	struct fw_filter2_wr *fwr;
 	struct sge_ctrl_txq *ctrlq;
 	unsigned int port_id = ethdev2pinfo(dev)->port_id;
 	int ret;
 
+	/*
+	 * If the new filter requires loopback Destination MAC and/or VLAN
+	 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
+	 * the filter.
+	 */
+	if (f->fs.newvlan) {
+		/* allocate L2T entry for new filter */
+		f->l2t = cxgbe_l2t_alloc_switching(f->dev, f->fs.vlan,
+						   f->fs.eport, f->fs.dmac);
+		if (!f->l2t)
+			return -ENOMEM;
+	}
+
 	ctrlq = &adapter->sge.ctrlq[port_id];
 	mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool);
 	if (!mbuf) {
@@ -663,13 +725,16 @@ int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
 	mbuf->data_len = sizeof(*fwr);
 	mbuf->pkt_len = mbuf->data_len;
 
-	fwr = rte_pktmbuf_mtod(mbuf, struct fw_filter_wr *);
+	fwr = rte_pktmbuf_mtod(mbuf, struct fw_filter2_wr *);
 	memset(fwr, 0, sizeof(*fwr));
 
 	/*
 	 * Construct the work request to set the filter.
 	 */
-	fwr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER_WR));
+	if (adapter->params.filter2_wr_support)
+		fwr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER2_WR));
+	else
+		fwr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER_WR));
 	fwr->len16_pkd = cpu_to_be32(V_FW_WR_LEN16(sizeof(*fwr) / 16));
 	fwr->tid_to_iq =
 		cpu_to_be32(V_FW_FILTER_WR_TID(f->tid) |
@@ -680,9 +745,16 @@ int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
 		cpu_to_be32(V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
 			    V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
 			    V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
+			    V_FW_FILTER_WR_INSVLAN
+				(f->fs.newvlan == VLAN_INSERT ||
+				 f->fs.newvlan == VLAN_REWRITE) |
+			    V_FW_FILTER_WR_RMVLAN
+				(f->fs.newvlan == VLAN_REMOVE ||
+				 f->fs.newvlan == VLAN_REWRITE) |
 			    V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
 			    V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
-			    V_FW_FILTER_WR_PRIO(f->fs.prio));
+			    V_FW_FILTER_WR_PRIO(f->fs.prio) |
+			    V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
 	fwr->ethtype = cpu_to_be16(f->fs.val.ethtype);
 	fwr->ethtypem = cpu_to_be16(f->fs.mask.ethtype);
 	fwr->smac_sel = 0;
@@ -691,7 +763,9 @@ int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
 			    V_FW_FILTER_WR_RX_RPL_IQ(adapter->sge.fw_evtq.abs_id
 						     ));
 	fwr->maci_to_matchtypem =
-		cpu_to_be32(V_FW_FILTER_WR_PORT(f->fs.val.iport) |
+		cpu_to_be32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
+			    V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
+			    V_FW_FILTER_WR_PORT(f->fs.val.iport) |
 			    V_FW_FILTER_WR_PORTM(f->fs.mask.iport));
 	fwr->ptcl = f->fs.val.proto;
 	fwr->ptclm = f->fs.mask.proto;
@@ -704,6 +778,20 @@ int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
 	fwr->fp = cpu_to_be16(f->fs.val.fport);
 	fwr->fpm = cpu_to_be16(f->fs.mask.fport);
 
+	if (adapter->params.filter2_wr_support) {
+		fwr->filter_type_swapmac =
+			 V_FW_FILTER2_WR_SWAPMAC(f->fs.swapmac);
+		fwr->natmode_to_ulp_type =
+			V_FW_FILTER2_WR_ULP_TYPE(f->fs.nat_mode ?
+						 ULP_MODE_TCPDDP :
+						 ULP_MODE_NONE) |
+			V_FW_FILTER2_WR_NATMODE(f->fs.nat_mode);
+		memcpy(fwr->newlip, f->fs.nat_lip, sizeof(fwr->newlip));
+		memcpy(fwr->newfip, f->fs.nat_fip, sizeof(fwr->newfip));
+		fwr->newlport = cpu_to_be16(f->fs.nat_lport);
+		fwr->newfport = cpu_to_be16(f->fs.nat_fport);
+	}
+
 	/*
 	 * Mark the filter as "pending" and ship off the Filter Work Request.
 	 * When we get the Work Request Reply we'll clear the pending status.
@@ -1046,6 +1134,9 @@ void hash_filter_rpl(struct adapter *adap, const struct cpl_act_open_rpl *rpl)
 				      V_TCB_TIMESTAMP(0ULL) |
 				      V_TCB_T_RTT_TS_RECENT_AGE(0ULL),
 				      1);
+		if (f->fs.newvlan == VLAN_INSERT ||
+		    f->fs.newvlan == VLAN_REWRITE)
+			set_tcb_tflag(adap, tid, S_TF_CCTRL_RFR, 1, 1);
 		break;
 	}
 	default:
diff --git a/drivers/net/cxgbe/cxgbe_filter.h b/drivers/net/cxgbe/cxgbe_filter.h
index af8fa752..b7bcbf56 100644
--- a/drivers/net/cxgbe/cxgbe_filter.h
+++ b/drivers/net/cxgbe/cxgbe_filter.h
@@ -77,6 +77,7 @@ struct ch_filter_tuple {
  * Filter specification
  */
 struct ch_filter_specification {
+	void *private;
 	/* Administrative fields for filter. */
 	uint32_t hitcnts:1;	/* count filter hits in TCB */
 	uint32_t prio:1;	/* filter has priority over active/server */
@@ -99,6 +100,22 @@ struct ch_filter_specification {
 	uint32_t iq:10;		/* ingress queue */
 
 	uint32_t eport:2;	/* egress port to switch packet out */
+	uint32_t swapmac:1;     /* swap SMAC/DMAC for loopback packet */
+	uint32_t newvlan:2;     /* rewrite VLAN Tag */
+	uint8_t dmac[ETHER_ADDR_LEN];   /* new destination MAC address */
+	uint16_t vlan;          /* VLAN Tag to insert */
+
+	/*
+	 * Switch proxy/rewrite fields.  An ingress packet which matches a
+	 * filter with "switch" set will be looped back out as an egress
+	 * packet -- potentially with some header rewriting.
+	 */
+	uint32_t nat_mode:3;	/* specify NAT operation mode */
+
+	uint8_t nat_lip[16];	/* local IP to use after NAT'ing */
+	uint8_t nat_fip[16];	/* foreign IP to use after NAT'ing */
+	uint16_t nat_lport;	/* local port number to use after NAT'ing */
+	uint16_t nat_fport;	/* foreign port number to use after NAT'ing */
 
 	/* Filter rule value/mask pairs. */
 	struct ch_filter_tuple val;
@@ -111,6 +128,23 @@ enum {
 	FILTER_SWITCH
 };
 
+enum {
+	VLAN_REMOVE = 1,
+	VLAN_INSERT,
+	VLAN_REWRITE
+};
+
+enum {
+	NAT_MODE_NONE = 0,	/* No NAT performed */
+	NAT_MODE_DIP,		/* NAT on Dst IP */
+	NAT_MODE_DIP_DP,	/* NAT on Dst IP, Dst Port */
+	NAT_MODE_DIP_DP_SIP,	/* NAT on Dst IP, Dst Port and Src IP */
+	NAT_MODE_DIP_DP_SP,	/* NAT on Dst IP, Dst Port and Src Port */
+	NAT_MODE_SIP_SP,	/* NAT on Src IP and Src Port */
+	NAT_MODE_DIP_SIP_SP,	/* NAT on Dst IP, Src IP and Src Port */
+	NAT_MODE_ALL		/* NAT on entire 4-tuple */
+};
+
 enum filter_type {
 	FILTER_TYPE_IPV4 = 0,
 	FILTER_TYPE_IPV6,
@@ -145,6 +179,7 @@ struct filter_entry {
 	u32 pending:1;              /* filter action is pending FW reply */
 	struct filter_ctx *ctx;     /* caller's completion hook */
 	struct clip_entry *clipt;   /* CLIP Table entry for IPv6 */
+	struct l2t_entry *l2t;      /* Layer Two Table entry for dmac */
 	struct rte_eth_dev *dev;    /* Port's rte eth device */
 	void *private;              /* For use by apps using filter_entry */
 
diff --git a/drivers/net/cxgbe/cxgbe_flow.c b/drivers/net/cxgbe/cxgbe_flow.c
index 01c945f1..54ec7e59 100644
--- a/drivers/net/cxgbe/cxgbe_flow.c
+++ b/drivers/net/cxgbe/cxgbe_flow.c
@@ -95,6 +95,8 @@ cxgbe_fill_filter_region(struct adapter *adap,
 		ntuple_mask |= (u64)fs->mask.ethtype << tp->ethertype_shift;
 	if (tp->port_shift >= 0)
 		ntuple_mask |= (u64)fs->mask.iport << tp->port_shift;
+	if (tp->macmatch_shift >= 0)
+		ntuple_mask |= (u64)fs->mask.macidx << tp->macmatch_shift;
 
 	if (ntuple_mask != hash_filter_mask)
 		return;
@@ -103,6 +105,46 @@ cxgbe_fill_filter_region(struct adapter *adap,
 }
 
 static int
+ch_rte_parsetype_eth(const void *dmask, const struct rte_flow_item *item,
+		     struct ch_filter_specification *fs,
+		     struct rte_flow_error *e)
+{
+	const struct rte_flow_item_eth *spec = item->spec;
+	const struct rte_flow_item_eth *umask = item->mask;
+	const struct rte_flow_item_eth *mask;
+
+	/* If user has not given any mask, then use chelsio supported mask. */
+	mask = umask ? umask : (const struct rte_flow_item_eth *)dmask;
+
+	/* we don't support SRC_MAC filtering*/
+	if (!is_zero_ether_addr(&mask->src))
+		return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "src mac filtering not supported");
+
+	if (!is_zero_ether_addr(&mask->dst)) {
+		const u8 *addr = (const u8 *)&spec->dst.addr_bytes[0];
+		const u8 *m = (const u8 *)&mask->dst.addr_bytes[0];
+		struct rte_flow *flow = (struct rte_flow *)fs->private;
+		struct port_info *pi = (struct port_info *)
+					(flow->dev->data->dev_private);
+		int idx;
+
+		idx = cxgbe_mpstcam_alloc(pi, addr, m);
+		if (idx <= 0)
+			return rte_flow_error_set(e, idx,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  NULL, "unable to allocate mac"
+						  " entry in h/w");
+		CXGBE_FILL_FS(idx, 0x1ff, macidx);
+	}
+
+	CXGBE_FILL_FS(be16_to_cpu(spec->type),
+		      be16_to_cpu(mask->type), ethtype);
+	return 0;
+}
+
+static int
 ch_rte_parsetype_port(const void *dmask, const struct rte_flow_item *item,
 		      struct ch_filter_specification *fs,
 		      struct rte_flow_error *e)
@@ -327,17 +369,199 @@ static int cxgbe_get_fidx(struct rte_flow *flow, unsigned int *fidx)
 }
 
 static int
+cxgbe_get_flow_item_index(const struct rte_flow_item items[], u32 type)
+{
+	const struct rte_flow_item *i;
+	int j, index = -ENOENT;
+
+	for (i = items, j = 0; i->type != RTE_FLOW_ITEM_TYPE_END; i++, j++) {
+		if (i->type == type) {
+			index = j;
+			break;
+		}
+	}
+
+	return index;
+}
+
+static int
+ch_rte_parse_nat(uint8_t nmode, struct ch_filter_specification *fs)
+{
+	/* nmode:
+	 * BIT_0 = [src_ip],   BIT_1 = [dst_ip]
+	 * BIT_2 = [src_port], BIT_3 = [dst_port]
+	 *
+	 * Only below cases are supported as per our spec.
+	 */
+	switch (nmode) {
+	case 0:  /* 0000b */
+		fs->nat_mode = NAT_MODE_NONE;
+		break;
+	case 2:  /* 0010b */
+		fs->nat_mode = NAT_MODE_DIP;
+		break;
+	case 5:  /* 0101b */
+		fs->nat_mode = NAT_MODE_SIP_SP;
+		break;
+	case 7:  /* 0111b */
+		fs->nat_mode = NAT_MODE_DIP_SIP_SP;
+		break;
+	case 10: /* 1010b */
+		fs->nat_mode = NAT_MODE_DIP_DP;
+		break;
+	case 11: /* 1011b */
+		fs->nat_mode = NAT_MODE_DIP_DP_SIP;
+		break;
+	case 14: /* 1110b */
+		fs->nat_mode = NAT_MODE_DIP_DP_SP;
+		break;
+	case 15: /* 1111b */
+		fs->nat_mode = NAT_MODE_ALL;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
 ch_rte_parse_atype_switch(const struct rte_flow_action *a,
+			  const struct rte_flow_item items[],
+			  uint8_t *nmode,
 			  struct ch_filter_specification *fs,
 			  struct rte_flow_error *e)
 {
+	const struct rte_flow_action_of_set_vlan_vid *vlanid;
+	const struct rte_flow_action_of_push_vlan *pushvlan;
+	const struct rte_flow_action_set_ipv4 *ipv4;
+	const struct rte_flow_action_set_ipv6 *ipv6;
+	const struct rte_flow_action_set_tp *tp_port;
 	const struct rte_flow_action_phy_port *port;
+	int item_index;
 
 	switch (a->type) {
+	case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
+		vlanid = (const struct rte_flow_action_of_set_vlan_vid *)
+			  a->conf;
+		fs->newvlan = VLAN_REWRITE;
+		fs->vlan = vlanid->vlan_vid;
+		break;
+	case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
+		pushvlan = (const struct rte_flow_action_of_push_vlan *)
+			    a->conf;
+		if (pushvlan->ethertype != ETHER_TYPE_VLAN)
+			return rte_flow_error_set(e, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION, a,
+						  "only ethertype 0x8100 "
+						  "supported for push vlan.");
+		fs->newvlan = VLAN_INSERT;
+		break;
+	case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
+		fs->newvlan = VLAN_REMOVE;
+		break;
 	case RTE_FLOW_ACTION_TYPE_PHY_PORT:
 		port = (const struct rte_flow_action_phy_port *)a->conf;
 		fs->eport = port->index;
 		break;
+	case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
+		item_index = cxgbe_get_flow_item_index(items,
+						       RTE_FLOW_ITEM_TYPE_IPV4);
+		if (item_index < 0)
+			return rte_flow_error_set(e, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION, a,
+						  "No RTE_FLOW_ITEM_TYPE_IPV4 "
+						  "found.");
+
+		ipv4 = (const struct rte_flow_action_set_ipv4 *)a->conf;
+		memcpy(fs->nat_fip, &ipv4->ipv4_addr, sizeof(ipv4->ipv4_addr));
+		*nmode |= 1 << 0;
+		break;
+	case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
+		item_index = cxgbe_get_flow_item_index(items,
+						       RTE_FLOW_ITEM_TYPE_IPV4);
+		if (item_index < 0)
+			return rte_flow_error_set(e, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION, a,
+						  "No RTE_FLOW_ITEM_TYPE_IPV4 "
+						  "found.");
+
+		ipv4 = (const struct rte_flow_action_set_ipv4 *)a->conf;
+		memcpy(fs->nat_lip, &ipv4->ipv4_addr, sizeof(ipv4->ipv4_addr));
+		*nmode |= 1 << 1;
+		break;
+	case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
+		item_index = cxgbe_get_flow_item_index(items,
+						       RTE_FLOW_ITEM_TYPE_IPV6);
+		if (item_index < 0)
+			return rte_flow_error_set(e, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION, a,
+						  "No RTE_FLOW_ITEM_TYPE_IPV6 "
+						  "found.");
+
+		ipv6 = (const struct rte_flow_action_set_ipv6 *)a->conf;
+		memcpy(fs->nat_fip, ipv6->ipv6_addr, sizeof(ipv6->ipv6_addr));
+		*nmode |= 1 << 0;
+		break;
+	case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
+		item_index = cxgbe_get_flow_item_index(items,
+						       RTE_FLOW_ITEM_TYPE_IPV6);
+		if (item_index < 0)
+			return rte_flow_error_set(e, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION, a,
+						  "No RTE_FLOW_ITEM_TYPE_IPV6 "
+						  "found.");
+
+		ipv6 = (const struct rte_flow_action_set_ipv6 *)a->conf;
+		memcpy(fs->nat_lip, ipv6->ipv6_addr, sizeof(ipv6->ipv6_addr));
+		*nmode |= 1 << 1;
+		break;
+	case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
+		item_index = cxgbe_get_flow_item_index(items,
+						       RTE_FLOW_ITEM_TYPE_TCP);
+		if (item_index < 0) {
+			item_index =
+				cxgbe_get_flow_item_index(items,
+						RTE_FLOW_ITEM_TYPE_UDP);
+			if (item_index < 0)
+				return rte_flow_error_set(e, EINVAL,
+						RTE_FLOW_ERROR_TYPE_ACTION, a,
+						"No RTE_FLOW_ITEM_TYPE_TCP or "
+						"RTE_FLOW_ITEM_TYPE_UDP found");
+		}
+
+		tp_port = (const struct rte_flow_action_set_tp *)a->conf;
+		fs->nat_fport = be16_to_cpu(tp_port->port);
+		*nmode |= 1 << 2;
+		break;
+	case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
+		item_index = cxgbe_get_flow_item_index(items,
+						       RTE_FLOW_ITEM_TYPE_TCP);
+		if (item_index < 0) {
+			item_index =
+				cxgbe_get_flow_item_index(items,
+						RTE_FLOW_ITEM_TYPE_UDP);
+			if (item_index < 0)
+				return rte_flow_error_set(e, EINVAL,
+						RTE_FLOW_ERROR_TYPE_ACTION, a,
+						"No RTE_FLOW_ITEM_TYPE_TCP or "
+						"RTE_FLOW_ITEM_TYPE_UDP found");
+		}
+
+		tp_port = (const struct rte_flow_action_set_tp *)a->conf;
+		fs->nat_lport = be16_to_cpu(tp_port->port);
+		*nmode |= 1 << 3;
+		break;
+	case RTE_FLOW_ACTION_TYPE_MAC_SWAP:
+		item_index = cxgbe_get_flow_item_index(items,
+						       RTE_FLOW_ITEM_TYPE_ETH);
+		if (item_index < 0)
+			return rte_flow_error_set(e, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION, a,
+						  "No RTE_FLOW_ITEM_TYPE_ETH "
+						  "found");
+		fs->swapmac = 1;
+		break;
 	default:
 		/* We are not supposed to come here */
 		return rte_flow_error_set(e, EINVAL,
@@ -350,10 +574,12 @@ ch_rte_parse_atype_switch(const struct rte_flow_action *a,
 
 static int
 cxgbe_rtef_parse_actions(struct rte_flow *flow,
+			 const struct rte_flow_item items[],
 			 const struct rte_flow_action action[],
 			 struct rte_flow_error *e)
 {
 	struct ch_filter_specification *fs = &flow->fs;
+	uint8_t nmode = 0, nat_ipv4 = 0, nat_ipv6 = 0;
 	const struct rte_flow_action_queue *q;
 	const struct rte_flow_action *a;
 	char abit = 0;
@@ -391,7 +617,22 @@ cxgbe_rtef_parse_actions(struct rte_flow *flow,
 		case RTE_FLOW_ACTION_TYPE_COUNT:
 			fs->hitcnts = 1;
 			break;
+		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
+		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
+		case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
 		case RTE_FLOW_ACTION_TYPE_PHY_PORT:
+		case RTE_FLOW_ACTION_TYPE_MAC_SWAP:
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
+			nat_ipv4++;
+			goto action_switch;
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
+			nat_ipv6++;
+			goto action_switch;
+		case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
+action_switch:
 			/* We allow multiple switch actions, but switch is
 			 * not compatible with either queue or drop
 			 */
@@ -399,7 +640,14 @@ cxgbe_rtef_parse_actions(struct rte_flow *flow,
 				return rte_flow_error_set(e, EINVAL,
 						RTE_FLOW_ERROR_TYPE_ACTION, a,
 						"overlapping action specified");
-			ret = ch_rte_parse_atype_switch(a, fs, e);
+			if (nat_ipv4 && nat_ipv6)
+				return rte_flow_error_set(e, EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION, a,
+					"Can't have one address ipv4 and the"
+					" other ipv6");
+
+			ret = ch_rte_parse_atype_switch(a, items, &nmode, fs,
+							e);
 			if (ret)
 				return ret;
 			fs->action = FILTER_SWITCH;
@@ -412,11 +660,24 @@ cxgbe_rtef_parse_actions(struct rte_flow *flow,
 		}
 	}
 
+	if (ch_rte_parse_nat(nmode, fs))
+		return rte_flow_error_set(e, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, a,
+					  "invalid settings for swich action");
 	return 0;
 }
 
-struct chrte_fparse parseitem[] = {
-		[RTE_FLOW_ITEM_TYPE_PHY_PORT] = {
+static struct chrte_fparse parseitem[] = {
+	[RTE_FLOW_ITEM_TYPE_ETH] = {
+		.fptr  = ch_rte_parsetype_eth,
+		.dmask = &(const struct rte_flow_item_eth){
+			.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+			.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+			.type = 0xffff,
+		}
+	},
+
+	[RTE_FLOW_ITEM_TYPE_PHY_PORT] = {
 		.fptr = ch_rte_parsetype_port,
 		.dmask = &(const struct rte_flow_item_phy_port){
 			.index = 0x7,
@@ -454,10 +715,10 @@ cxgbe_rtef_parse_items(struct rte_flow *flow,
 	char repeat[ARRAY_SIZE(parseitem)] = {0};
 
 	for (i = items; i->type != RTE_FLOW_ITEM_TYPE_END; i++) {
-		struct chrte_fparse *idx = &flow->item_parser[i->type];
+		struct chrte_fparse *idx;
 		int ret;
 
-		if (i->type > ARRAY_SIZE(parseitem))
+		if (i->type >= ARRAY_SIZE(parseitem))
 			return rte_flow_error_set(e, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  i, "Item not supported");
@@ -478,6 +739,7 @@ cxgbe_rtef_parse_items(struct rte_flow *flow,
 			if (ret)
 				return ret;
 
+			idx = &flow->item_parser[i->type];
 			if (!idx || !idx->fptr) {
 				return rte_flow_error_set(e, ENOTSUP,
 						RTE_FLOW_ERROR_TYPE_ITEM, i,
@@ -503,7 +765,6 @@ cxgbe_flow_parse(struct rte_flow *flow,
 		 struct rte_flow_error *e)
 {
 	int ret;
-
 	/* parse user request into ch_filter_specification */
 	ret = cxgbe_rtef_parse_attr(flow, attr, e);
 	if (ret)
@@ -511,7 +772,7 @@ cxgbe_flow_parse(struct rte_flow *flow,
 	ret = cxgbe_rtef_parse_items(flow, item, e);
 	if (ret)
 		return ret;
-	return cxgbe_rtef_parse_actions(flow, action, e);
+	return cxgbe_rtef_parse_actions(flow, item, action, e);
 }
 
 static int __cxgbe_flow_create(struct rte_eth_dev *dev, struct rte_flow *flow)
@@ -582,6 +843,7 @@ cxgbe_flow_create(struct rte_eth_dev *dev,
 
 	flow->item_parser = parseitem;
 	flow->dev = dev;
+	flow->fs.private = (void *)flow;
 
 	if (cxgbe_flow_parse(flow, attr, item, action, e)) {
 		t4_os_free(flow);
@@ -636,6 +898,17 @@ static int __cxgbe_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		return ctx.result;
 	}
 
+	fs = &flow->fs;
+	if (fs->mask.macidx) {
+		struct port_info *pi = (struct port_info *)
+					(dev->data->dev_private);
+		int ret;
+
+		ret = cxgbe_mpstcam_remove(pi, fs->val.macidx);
+		if (!ret)
+			return ret;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/net/cxgbe/cxgbe_flow.h b/drivers/net/cxgbe/cxgbe_flow.h
index 0f750474..718bf3d0 100644
--- a/drivers/net/cxgbe/cxgbe_flow.h
+++ b/drivers/net/cxgbe/cxgbe_flow.h
@@ -7,6 +7,7 @@
 
 #include <rte_flow_driver.h>
 #include "cxgbe_filter.h"
+#include "mps_tcam.h"
 #include "cxgbe.h"
 
 #define CXGBE_FLOW_POLL_US  10
diff --git a/drivers/net/cxgbe/cxgbe_main.c b/drivers/net/cxgbe/cxgbe_main.c
index c3938e8d..88dc851f 100644
--- a/drivers/net/cxgbe/cxgbe_main.c
+++ b/drivers/net/cxgbe/cxgbe_main.c
@@ -38,6 +38,8 @@
 #include "t4_msg.h"
 #include "cxgbe.h"
 #include "clip_tbl.h"
+#include "l2t.h"
+#include "mps_tcam.h"
 
 /**
  * Allocate a chunk of memory. The allocated memory is cleared.
@@ -99,6 +101,10 @@ static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 		const struct cpl_act_open_rpl *p = (const void *)rsp;
 
 		hash_filter_rpl(q->adapter, p);
+	} else if (opcode == CPL_L2T_WRITE_RPL) {
+		const struct cpl_l2t_write_rpl *p = (const void *)rsp;
+
+		do_l2t_write_rpl(q->adapter, p);
 	} else {
 		dev_err(adapter, "unexpected CPL %#x on FW event queue\n",
 			opcode);
@@ -1135,13 +1141,17 @@ static int adap_init0(struct adapter *adap)
 	 V_FW_PARAMS_PARAM_Y(0) | \
 	 V_FW_PARAMS_PARAM_Z(0))
 
-	params[0] = FW_PARAM_PFVF(FILTER_START);
-	params[1] = FW_PARAM_PFVF(FILTER_END);
-	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
+	params[0] = FW_PARAM_PFVF(L2T_START);
+	params[1] = FW_PARAM_PFVF(L2T_END);
+	params[2] = FW_PARAM_PFVF(FILTER_START);
+	params[3] = FW_PARAM_PFVF(FILTER_END);
+	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 4, params, val);
 	if (ret < 0)
 		goto bye;
-	adap->tids.ftid_base = val[0];
-	adap->tids.nftids = val[1] - val[0] + 1;
+	adap->l2t_start = val[0];
+	adap->l2t_end = val[1];
+	adap->tids.ftid_base = val[2];
+	adap->tids.nftids = val[3] - val[2] + 1;
 
 	params[0] = FW_PARAM_PFVF(CLIP_START);
 	params[1] = FW_PARAM_PFVF(CLIP_END);
@@ -1170,6 +1180,16 @@ static int adap_init0(struct adapter *adap)
 			goto bye;
 	}
 
+	/* See if FW supports FW_FILTER2 work request */
+	if (is_t4(adap->params.chip)) {
+		adap->params.filter2_wr_support = 0;
+	} else {
+		params[0] = FW_PARAM_DEV(FILTER2_WR);
+		ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+				      1, params, val);
+		adap->params.filter2_wr_support = (ret == 0 && val[0] != 0);
+	}
+
 	/* query tid-related parameters */
 	params[0] = FW_PARAM_DEV(NTID);
 	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1,
@@ -1332,10 +1352,8 @@ int link_start(struct port_info *pi)
 	ret = t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu, -1, -1,
 			    -1, 1, true);
 	if (ret == 0) {
-		ret = t4_change_mac(adapter, adapter->mbox, pi->viid,
-				    pi->xact_addr_filt,
-				    (u8 *)&pi->eth_dev->data->mac_addrs[0],
-				    true, true);
+		ret = cxgbe_mpstcam_modify(pi, (int)pi->xact_addr_filt,
+				(u8 *)&pi->eth_dev->data->mac_addrs[0]);
 		if (ret >= 0) {
 			pi->xact_addr_filt = ret;
 			ret = 0;
@@ -1679,10 +1697,12 @@ void cxgbe_close(struct adapter *adapter)
 	int i;
 
 	if (adapter->flags & FULL_INIT_DONE) {
-		if (is_pf4(adapter))
-			t4_intr_disable(adapter);
 		tid_free(&adapter->tids);
+		t4_cleanup_mpstcam(adapter);
 		t4_cleanup_clip_tbl(adapter);
+		t4_cleanup_l2t(adapter);
+		if (is_pf4(adapter))
+			t4_intr_disable(adapter);
 		t4_sge_tx_monitor_stop(adapter);
 		t4_free_sge_resources(adapter);
 		for_each_port(adapter, i) {
@@ -1690,12 +1710,7 @@ void cxgbe_close(struct adapter *adapter)
 			if (pi->viid != 0)
 				t4_free_vi(adapter, adapter->mbox,
 					   adapter->pf, 0, pi->viid);
-			rte_free(pi->eth_dev->data->mac_addrs);
-			/* Skip first port since it'll be freed by DPDK stack */
-			if (i) {
-				rte_free(pi->eth_dev->data->dev_private);
-				rte_eth_dev_release_port(pi->eth_dev);
-			}
+			rte_eth_dev_release_port(pi->eth_dev);
 		}
 		adapter->flags &= ~FULL_INIT_DONE;
 	}
@@ -1855,12 +1870,23 @@ allocate_mac:
 		dev_warn(adapter, "could not allocate CLIP. Continuing\n");
 	}
 
+	adapter->l2t = t4_init_l2t(adapter->l2t_start, adapter->l2t_end);
+	if (!adapter->l2t) {
+		/* We tolerate a lack of L2T, giving up some functionality */
+		dev_warn(adapter, "could not allocate L2T. Continuing\n");
+	}
+
 	if (tid_init(&adapter->tids) < 0) {
 		/* Disable filtering support */
 		dev_warn(adapter, "could not allocate TID table, "
 			 "filter support disabled. Continuing\n");
 	}
 
+	adapter->mpstcam = t4_init_mpstcam(adapter);
+	if (!adapter->mpstcam)
+		dev_warn(adapter, "could not allocate mps tcam table."
+			 " Continuing\n");
+
 	if (is_hashfilter(adapter)) {
 		if (t4_read_reg(adapter, A_LE_DB_CONFIG) & F_HASHEN) {
 			u32 hash_base, hash_reg;
@@ -1887,14 +1913,7 @@ out_free:
 		if (pi->viid != 0)
 			t4_free_vi(adapter, adapter->mbox, adapter->pf,
 				   0, pi->viid);
-		/* Skip first port since it'll be de-allocated by DPDK */
-		if (i == 0)
-			continue;
-		if (pi->eth_dev) {
-			if (pi->eth_dev->data->dev_private)
-				rte_free(pi->eth_dev->data->dev_private);
-			rte_eth_dev_release_port(pi->eth_dev);
-		}
+		rte_eth_dev_release_port(pi->eth_dev);
 	}
 
 	if (adapter->flags & FW_OK)
diff --git a/drivers/net/cxgbe/cxgbevf_main.c b/drivers/net/cxgbe/cxgbevf_main.c
index 4214d031..6223e125 100644
--- a/drivers/net/cxgbe/cxgbevf_main.c
+++ b/drivers/net/cxgbe/cxgbevf_main.c
@@ -282,14 +282,7 @@ out_free:
 		if (pi->viid != 0)
 			t4_free_vi(adapter, adapter->mbox, adapter->pf,
 				   0, pi->viid);
-		/* Skip first port since it'll be de-allocated by DPDK */
-		if (i == 0)
-			continue;
-		if (pi->eth_dev) {
-			if (pi->eth_dev->data->dev_private)
-				rte_free(pi->eth_dev->data->dev_private);
-			rte_eth_dev_release_port(pi->eth_dev);
-		}
+		rte_eth_dev_release_port(pi->eth_dev);
 	}
 	return -err;
 }
diff --git a/drivers/net/cxgbe/l2t.c b/drivers/net/cxgbe/l2t.c
new file mode 100644
index 00000000..814188fe
--- /dev/null
+++ b/drivers/net/cxgbe/l2t.c
@@ -0,0 +1,227 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+#include "common.h"
+#include "l2t.h"
+
+/**
+ * cxgbe_l2t_release - Release associated L2T entry
+ * @e: L2T entry to release
+ *
+ * Releases ref count and frees up an L2T entry from L2T table
+ */
+void cxgbe_l2t_release(struct l2t_entry *e)
+{
+	if (rte_atomic32_read(&e->refcnt) != 0)
+		rte_atomic32_dec(&e->refcnt);
+}
+
+/**
+ * Process a CPL_L2T_WRITE_RPL. Note that the TID in the reply is really
+ * the L2T index it refers to.
+ */
+void do_l2t_write_rpl(struct adapter *adap, const struct cpl_l2t_write_rpl *rpl)
+{
+	struct l2t_data *d = adap->l2t;
+	unsigned int tid = GET_TID(rpl);
+	unsigned int l2t_idx = tid % L2T_SIZE;
+
+	if (unlikely(rpl->status != CPL_ERR_NONE)) {
+		dev_err(adap,
+			"Unexpected L2T_WRITE_RPL status %u for entry %u\n",
+			rpl->status, l2t_idx);
+		return;
+	}
+
+	if (tid & F_SYNC_WR) {
+		struct l2t_entry *e = &d->l2tab[l2t_idx - d->l2t_start];
+
+		t4_os_lock(&e->lock);
+		if (e->state != L2T_STATE_SWITCHING)
+			e->state = L2T_STATE_VALID;
+		t4_os_unlock(&e->lock);
+	}
+}
+
+/**
+ * Write an L2T entry.  Must be called with the entry locked.
+ * The write may be synchronous or asynchronous.
+ */
+static int write_l2e(struct rte_eth_dev *dev, struct l2t_entry *e, int sync,
+		     bool loopback, bool arpmiss)
+{
+	struct adapter *adap = ethdev2adap(dev);
+	struct l2t_data *d = adap->l2t;
+	struct rte_mbuf *mbuf;
+	struct cpl_l2t_write_req *req;
+	struct sge_ctrl_txq *ctrlq;
+	unsigned int l2t_idx = e->idx + d->l2t_start;
+	unsigned int port_id = ethdev2pinfo(dev)->port_id;
+
+	ctrlq = &adap->sge.ctrlq[port_id];
+	mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool);
+	if (!mbuf)
+		return -ENOMEM;
+
+	mbuf->data_len = sizeof(*req);
+	mbuf->pkt_len = mbuf->data_len;
+
+	req = rte_pktmbuf_mtod(mbuf, struct cpl_l2t_write_req *);
+	INIT_TP_WR(req, 0);
+
+	OPCODE_TID(req) =
+		cpu_to_be32(MK_OPCODE_TID(CPL_L2T_WRITE_REQ,
+					  l2t_idx | V_SYNC_WR(sync) |
+					  V_TID_QID(adap->sge.fw_evtq.abs_id)));
+	req->params = cpu_to_be16(V_L2T_W_PORT(e->lport) |
+				  V_L2T_W_LPBK(loopback) |
+				  V_L2T_W_ARPMISS(arpmiss) |
+				  V_L2T_W_NOREPLY(!sync));
+	req->l2t_idx = cpu_to_be16(l2t_idx);
+	req->vlan = cpu_to_be16(e->vlan);
+	rte_memcpy(req->dst_mac, e->dmac, ETHER_ADDR_LEN);
+
+	if (loopback)
+		memset(req->dst_mac, 0, ETHER_ADDR_LEN);
+
+	t4_mgmt_tx(ctrlq, mbuf);
+
+	if (sync && e->state != L2T_STATE_SWITCHING)
+		e->state = L2T_STATE_SYNC_WRITE;
+
+	return 0;
+}
+
+/**
+ * find_or_alloc_l2e - Find/Allocate a free L2T entry
+ * @d: L2T table
+ * @vlan: VLAN id to compare/add
+ * @port: port id to compare/add
+ * @dmac: Destination MAC address to compare/add
+ * Returns pointer to the L2T entry found/created
+ *
+ * Finds/Allocates an L2T entry to be used by switching rule of a filter.
+ */
+static struct l2t_entry *find_or_alloc_l2e(struct l2t_data *d, u16 vlan,
+					   u8 port, u8 *dmac)
+{
+	struct l2t_entry *end, *e;
+	struct l2t_entry *first_free = NULL;
+
+	for (e = &d->l2tab[0], end = &d->l2tab[d->l2t_size]; e != end; ++e) {
+		if (rte_atomic32_read(&e->refcnt) == 0) {
+			if (!first_free)
+				first_free = e;
+		} else {
+			if (e->state == L2T_STATE_SWITCHING) {
+				if ((!memcmp(e->dmac, dmac, ETHER_ADDR_LEN)) &&
+				    e->vlan == vlan && e->lport == port)
+					goto exists;
+			}
+		}
+	}
+
+	if (first_free) {
+		e = first_free;
+		goto found;
+	}
+
+	return NULL;
+
+found:
+	e->state = L2T_STATE_UNUSED;
+
+exists:
+	return e;
+}
+
+static struct l2t_entry *t4_l2t_alloc_switching(struct rte_eth_dev *dev,
+						u16 vlan, u8 port,
+						u8 *eth_addr)
+{
+	struct adapter *adap = ethdev2adap(dev);
+	struct l2t_data *d = adap->l2t;
+	struct l2t_entry *e;
+	int ret = 0;
+
+	t4_os_write_lock(&d->lock);
+	e = find_or_alloc_l2e(d, vlan, port, eth_addr);
+	if (e) {
+		t4_os_lock(&e->lock);
+		if (!rte_atomic32_read(&e->refcnt)) {
+			e->state = L2T_STATE_SWITCHING;
+			e->vlan = vlan;
+			e->lport = port;
+			rte_memcpy(e->dmac, eth_addr, ETHER_ADDR_LEN);
+			rte_atomic32_set(&e->refcnt, 1);
+			ret = write_l2e(dev, e, 0, !L2T_LPBK, !L2T_ARPMISS);
+			if (ret < 0)
+				dev_debug(adap, "Failed to write L2T entry: %d",
+					  ret);
+		} else {
+			rte_atomic32_inc(&e->refcnt);
+		}
+		t4_os_unlock(&e->lock);
+	}
+	t4_os_write_unlock(&d->lock);
+
+	return ret ? NULL : e;
+}
+
+/**
+ * cxgbe_l2t_alloc_switching - Allocate a L2T entry for switching rule
+ * @dev: rte_eth_dev pointer
+ * @vlan: VLAN Id
+ * @port: Associated port
+ * @dmac: Destination MAC address to add to L2T
+ * Returns pointer to the allocated l2t entry
+ *
+ * Allocates a L2T entry for use by switching rule of a filter
+ */
+struct l2t_entry *cxgbe_l2t_alloc_switching(struct rte_eth_dev *dev, u16 vlan,
+					    u8 port, u8 *dmac)
+{
+	return t4_l2t_alloc_switching(dev, vlan, port, dmac);
+}
+
+/**
+ * Initialize L2 Table
+ */
+struct l2t_data *t4_init_l2t(unsigned int l2t_start, unsigned int l2t_end)
+{
+	unsigned int l2t_size;
+	unsigned int i;
+	struct l2t_data *d;
+
+	if (l2t_start >= l2t_end || l2t_end >= L2T_SIZE)
+		return NULL;
+	l2t_size = l2t_end - l2t_start + 1;
+
+	d = t4_os_alloc(sizeof(*d) + l2t_size * sizeof(struct l2t_entry));
+	if (!d)
+		return NULL;
+
+	d->l2t_start = l2t_start;
+	d->l2t_size = l2t_size;
+
+	t4_os_rwlock_init(&d->lock);
+
+	for (i = 0; i < d->l2t_size; ++i) {
+		d->l2tab[i].idx = i;
+		d->l2tab[i].state = L2T_STATE_UNUSED;
+		t4_os_lock_init(&d->l2tab[i].lock);
+		rte_atomic32_set(&d->l2tab[i].refcnt, 0);
+	}
+
+	return d;
+}
+
+/**
+ * Cleanup L2 Table
+ */
+void t4_cleanup_l2t(struct adapter *adap)
+{
+	if (adap->l2t)
+		t4_os_free(adap->l2t);
+}
diff --git a/drivers/net/cxgbe/l2t.h b/drivers/net/cxgbe/l2t.h
new file mode 100644
index 00000000..22a34e38
--- /dev/null
+++ b/drivers/net/cxgbe/l2t.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+#ifndef _CXGBE_L2T_H_
+#define _CXGBE_L2T_H_
+
+#include "t4_msg.h"
+
+enum {
+	L2T_SIZE = 4096       /* # of L2T entries */
+};
+
+enum {
+	L2T_STATE_VALID,      /* entry is up to date */
+	L2T_STATE_SYNC_WRITE, /* synchronous write of entry underway */
+
+	/* when state is one of the below the entry is not hashed */
+	L2T_STATE_SWITCHING,  /* entry is being used by a switching filter */
+	L2T_STATE_UNUSED      /* entry not in use */
+};
+
+/*
+ * State for the corresponding entry of the HW L2 table.
+ */
+struct l2t_entry {
+	u16 state;                  /* entry state */
+	u16 idx;                    /* entry index within in-memory table */
+	u16 vlan;                   /* VLAN TCI (id: bits 0-11, prio: 13-15 */
+	u8  lport;                  /* destination port */
+	u8  dmac[ETHER_ADDR_LEN];   /* destination MAC address */
+	rte_spinlock_t lock;        /* entry lock */
+	rte_atomic32_t refcnt;      /* entry reference count */
+};
+
+struct l2t_data {
+	unsigned int l2t_start;     /* start index of our piece of the L2T */
+	unsigned int l2t_size;      /* number of entries in l2tab */
+	rte_rwlock_t lock;          /* table rw lock */
+	struct l2t_entry l2tab[0];  /* MUST BE LAST */
+};
+
+#define L2T_LPBK	true
+#define L2T_ARPMISS	true
+
+/* identifies sync vs async L2T_WRITE_REQs */
+#define S_SYNC_WR    12
+#define V_SYNC_WR(x) ((x) << S_SYNC_WR)
+#define F_SYNC_WR    V_SYNC_WR(1)
+
+struct l2t_data *t4_init_l2t(unsigned int l2t_start, unsigned int l2t_end);
+void t4_cleanup_l2t(struct adapter *adap);
+struct l2t_entry *cxgbe_l2t_alloc_switching(struct rte_eth_dev *dev, u16 vlan,
+					    u8 port, u8 *dmac);
+void cxgbe_l2t_release(struct l2t_entry *e);
+void do_l2t_write_rpl(struct adapter *p, const struct cpl_l2t_write_rpl *rpl);
+#endif /* _CXGBE_L2T_H_ */
diff --git a/drivers/net/cxgbe/meson.build b/drivers/net/cxgbe/meson.build
index 7c69a34b..c51af26e 100644
--- a/drivers/net/cxgbe/meson.build
+++ b/drivers/net/cxgbe/meson.build
@@ -9,6 +9,8 @@ sources = files('cxgbe_ethdev.c',
 	'cxgbe_filter.c',
 	'cxgbe_flow.c',
 	'clip_tbl.c',
+	'mps_tcam.c',
+	'l2t.c',
 	'base/t4_hw.c',
 	'base/t4vf_hw.c')
 includes += include_directories('base')
diff --git a/drivers/net/cxgbe/mps_tcam.c b/drivers/net/cxgbe/mps_tcam.c
new file mode 100644
index 00000000..02ec69a9
--- /dev/null
+++ b/drivers/net/cxgbe/mps_tcam.c
@@ -0,0 +1,243 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+
+#include "mps_tcam.h"
+
+static inline bool
+match_entry(struct mps_tcam_entry *entry, const u8 *eth_addr, const u8 *mask)
+{
+	if (!memcmp(eth_addr, entry->eth_addr, ETHER_ADDR_LEN) &&
+	    !memcmp(mask, entry->mask, ETHER_ADDR_LEN))
+		return true;
+	return false;
+}
+
+static int cxgbe_update_free_idx(struct mpstcam_table *t)
+{
+	struct mps_tcam_entry *entry = t->entry;
+	u16 i, next = t->free_idx + 1;
+
+	if (entry[t->free_idx].state == MPS_ENTRY_UNUSED)
+		/* You are already pointing to a free entry !! */
+		return 0;
+
+	/* loop, till we don't rollback to same index where we started */
+	for (i = next; i != t->free_idx; i++) {
+		if (i == t->size)
+			/* rollback and search free entry from start */
+			i = 0;
+
+		if (entry[i].state == MPS_ENTRY_UNUSED) {
+			t->free_idx = i;
+			return 0;
+		}
+	}
+
+	return -1;	/* table is full */
+}
+
+static struct mps_tcam_entry *
+cxgbe_mpstcam_lookup(struct mpstcam_table *t, const u8 *eth_addr,
+		     const u8 *mask)
+{
+	struct mps_tcam_entry *entry = t->entry;
+	int i;
+
+	if (!entry)
+		return NULL;
+
+	for (i = 0; i < t->size; i++) {
+		if (entry[i].state == MPS_ENTRY_UNUSED)
+			continue;	/* entry is not being used */
+		if (match_entry(&entry[i], eth_addr, mask))
+			return &entry[i];
+	}
+
+	return NULL;
+}
+
+int cxgbe_mpstcam_alloc(struct port_info *pi, const u8 *eth_addr,
+			const u8 *mask)
+{
+	struct adapter *adap = pi->adapter;
+	struct mpstcam_table *mpstcam = adap->mpstcam;
+	struct mps_tcam_entry *entry;
+	int ret;
+
+	if (!adap->mpstcam) {
+		dev_err(adap, "mpstcam table is not available\n");
+		return -EOPNOTSUPP;
+	}
+
+	/* If entry already present, return it. */
+	t4_os_write_lock(&mpstcam->lock);
+	entry = cxgbe_mpstcam_lookup(adap->mpstcam, eth_addr, mask);
+	if (entry) {
+		rte_atomic32_add(&entry->refcnt, 1);
+		t4_os_write_unlock(&mpstcam->lock);
+		return entry->idx;
+	}
+
+	if (mpstcam->full) {
+		t4_os_write_unlock(&mpstcam->lock);
+		dev_err(adap, "mps-tcam table is full\n");
+		return -ENOMEM;
+	}
+
+	ret = t4_alloc_raw_mac_filt(adap, pi->viid, eth_addr, mask,
+				    mpstcam->free_idx, 0, pi->port_id, false);
+	if (ret <= 0) {
+		t4_os_write_unlock(&mpstcam->lock);
+		return ret;
+	}
+
+	/* Fill in the new values */
+	entry = &mpstcam->entry[ret];
+	memcpy(entry->eth_addr, eth_addr, ETHER_ADDR_LEN);
+	memcpy(entry->mask, mask, ETHER_ADDR_LEN);
+	rte_atomic32_set(&entry->refcnt, 1);
+	entry->state = MPS_ENTRY_USED;
+
+	if (cxgbe_update_free_idx(mpstcam))
+		mpstcam->full = true;
+
+	t4_os_write_unlock(&mpstcam->lock);
+	return ret;
+}
+
+int cxgbe_mpstcam_modify(struct port_info *pi, int idx, const u8 *addr)
+{
+	struct adapter *adap = pi->adapter;
+	struct mpstcam_table *mpstcam = adap->mpstcam;
+	struct mps_tcam_entry *entry;
+
+	if (!mpstcam)
+		return -EOPNOTSUPP;
+	t4_os_write_lock(&mpstcam->lock);
+	if (idx != -1 && idx >= mpstcam->size) {
+		t4_os_write_unlock(&mpstcam->lock);
+		return -EINVAL;
+	}
+	if (idx >= 0) {
+		entry = &mpstcam->entry[idx];
+		/* user wants to modify an existing entry.
+		 * verify if entry exists
+		 */
+		if (entry->state != MPS_ENTRY_USED) {
+			t4_os_write_unlock(&mpstcam->lock);
+			return -EINVAL;
+		}
+	}
+
+	idx = t4_change_mac(adap, adap->mbox, pi->viid, idx, addr, true, true);
+	if (idx < 0) {
+		t4_os_write_unlock(&mpstcam->lock);
+		return idx;
+	}
+
+	/* idx can now be different from what user provided */
+	entry = &mpstcam->entry[idx];
+	memcpy(entry->eth_addr, addr, ETHER_ADDR_LEN);
+	/* NOTE: we have considered the case that idx returned by t4_change_mac
+	 * will be different from the user provided value only if user
+	 * provided value is -1
+	 */
+	if (entry->state == MPS_ENTRY_UNUSED) {
+		rte_atomic32_set(&entry->refcnt, 1);
+		entry->state = MPS_ENTRY_USED;
+	}
+
+	if (cxgbe_update_free_idx(mpstcam))
+		mpstcam->full = true;
+
+	t4_os_write_unlock(&mpstcam->lock);
+	return idx;
+}
+
+/**
+ * hold appropriate locks while calling this.
+ */
+static inline void reset_mpstcam_entry(struct mps_tcam_entry *entry)
+{
+	memset(entry->eth_addr, 0, ETHER_ADDR_LEN);
+	memset(entry->mask, 0, ETHER_ADDR_LEN);
+	rte_atomic32_clear(&entry->refcnt);
+	entry->state = MPS_ENTRY_UNUSED;
+}
+
+/**
+ * ret < 0: fatal error
+ * ret = 0: entry removed in h/w
+ * ret > 0: updated refcount.
+ */
+int cxgbe_mpstcam_remove(struct port_info *pi, u16 idx)
+{
+	struct adapter *adap = pi->adapter;
+	struct mpstcam_table *t = adap->mpstcam;
+	struct mps_tcam_entry *entry;
+	int ret;
+
+	if (!t)
+		return -EOPNOTSUPP;
+	t4_os_write_lock(&t->lock);
+	entry = &t->entry[idx];
+	if (entry->state == MPS_ENTRY_UNUSED) {
+		t4_os_write_unlock(&t->lock);
+		return -EINVAL;
+	}
+
+	if (rte_atomic32_read(&entry->refcnt) == 1)
+		ret = t4_free_raw_mac_filt(adap, pi->viid, entry->eth_addr,
+					   entry->mask, idx, 1, pi->port_id,
+					   false);
+	else
+		ret = rte_atomic32_sub_return(&entry->refcnt, 1);
+
+	if (ret == 0) {
+		reset_mpstcam_entry(entry);
+		t->full = false;	/* We have atleast 1 free entry */
+		cxgbe_update_free_idx(t);
+	}
+
+	t4_os_write_unlock(&t->lock);
+	return ret;
+}
+
+struct mpstcam_table *t4_init_mpstcam(struct adapter *adap)
+{
+	struct mpstcam_table *t;
+	int i;
+	u16 size = adap->params.arch.mps_tcam_size;
+
+	t =  t4_os_alloc(sizeof(*t) + size * sizeof(struct mps_tcam_entry));
+	if (!t)
+		return NULL;
+
+	t4_os_rwlock_init(&t->lock);
+	t->full = false;
+	t->size = size;
+
+	for (i = 0; i < size; i++) {
+		reset_mpstcam_entry(&t->entry[i]);
+		t->entry[i].mpstcam = t;
+		t->entry[i].idx = i;
+	}
+
+	/* first entry is used by chip. this is overwritten only
+	 * in t4_cleanup_mpstcam()
+	 */
+	t->entry[0].state = MPS_ENTRY_USED;
+	t->free_idx = 1;
+
+	return t;
+}
+
+void t4_cleanup_mpstcam(struct adapter *adap)
+{
+	if (adap->mpstcam) {
+		t4_os_free(adap->mpstcam->entry);
+		t4_os_free(adap->mpstcam);
+	}
+}
diff --git a/drivers/net/cxgbe/mps_tcam.h b/drivers/net/cxgbe/mps_tcam.h
new file mode 100644
index 00000000..c3d6fe0d
--- /dev/null
+++ b/drivers/net/cxgbe/mps_tcam.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+
+#ifndef _CXGBE_MPSTCAM_H_
+#define _CXGBE_MPSTCAM_H_
+
+#include "common.h"
+
+enum {
+	MPS_ENTRY_UNUSED,	/* Keep this first so memset 0 renders
+				 * the correct state. Other states can
+				 * be added in future like MPS_ENTRY_BUSY
+				 * to reduce contention while mboxing
+				 * the request to f/w or to denote attributes
+				 * for a specific entry
+				 */
+	MPS_ENTRY_USED,
+};
+
+struct mps_tcam_entry {
+	u8 state;
+	u16 idx;
+
+	/* add data here which uniquely defines an entry */
+	u8 eth_addr[ETHER_ADDR_LEN];
+	u8 mask[ETHER_ADDR_LEN];
+
+	struct mpstcam_table *mpstcam; /* backptr */
+	rte_atomic32_t refcnt;
+};
+
+struct mpstcam_table {
+	u16 size;
+	rte_rwlock_t lock;
+	u16 free_idx;	/* next free index */
+	bool full;	/* since free index can be present
+			 * anywhere in the table, size and
+			 * free_idx cannot alone determine
+			 * if the table is full
+			 */
+	struct mps_tcam_entry entry[0];
+};
+
+struct mpstcam_table *t4_init_mpstcam(struct adapter *adap);
+void t4_cleanup_mpstcam(struct adapter *adap);
+int cxgbe_mpstcam_alloc(struct port_info *pi, const u8 *mac, const u8 *mask);
+int cxgbe_mpstcam_remove(struct port_info *pi, u16 idx);
+int cxgbe_mpstcam_modify(struct port_info *pi, int idx, const u8 *addr);
+
+#endif /* _CXGBE_MPSTCAM_H_ */
diff --git a/drivers/net/cxgbe/sge.c b/drivers/net/cxgbe/sge.c
index 4ea40d19..f9d2d48a 100644
--- a/drivers/net/cxgbe/sge.c
+++ b/drivers/net/cxgbe/sge.c
@@ -1873,10 +1873,9 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
 	/* Size needs to be multiple of 16, including status entry. */
 	iq->size = cxgbe_roundup(iq->size, 16);
 
-	snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
-		 eth_dev->device->driver->name,
-		 fwevtq ? "fwq_ring" : "rx_ring",
-		 eth_dev->data->port_id, queue_id);
+	snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
+			eth_dev->data->port_id, queue_id,
+			fwevtq ? "fwq_ring" : "rx_ring");
 	snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
 
 	iq->desc = alloc_ring(iq->size, iq->iqe_len, 0, &iq->phys_addr, NULL, 0,
@@ -1938,10 +1937,9 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
 			fl->size = s->fl_starve_thres - 1 + 2 * 8;
 		fl->size = cxgbe_roundup(fl->size, 8);
 
-		snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
-			 eth_dev->device->driver->name,
-			 fwevtq ? "fwq_ring" : "fl_ring",
-			 eth_dev->data->port_id, queue_id);
+		snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
+				eth_dev->data->port_id, queue_id,
+				fwevtq ? "fwq_ring" : "fl_ring");
 		snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
 
 		fl->desc = alloc_ring(fl->size, sizeof(__be64),
@@ -2144,9 +2142,8 @@ int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
 	/* Add status entries */
 	nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
 
-	snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
-		 eth_dev->device->driver->name, "tx_ring",
-		 eth_dev->data->port_id, queue_id);
+	snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
+			eth_dev->data->port_id, queue_id, "tx_ring");
 	snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
 
 	txq->q.desc = alloc_ring(txq->q.size, sizeof(struct tx_desc),
@@ -2223,9 +2220,8 @@ int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
 	/* Add status entries */
 	nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
 
-	snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
-		 eth_dev->device->driver->name, "ctrl_tx_ring",
-		 eth_dev->data->port_id, queue_id);
+	snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
+			eth_dev->data->port_id, queue_id, "ctrl_tx_ring");
 	snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
 
 	txq->q.desc = alloc_ring(txq->q.size, sizeof(struct tx_desc),
diff --git a/drivers/net/dpaa/Makefile b/drivers/net/dpaa/Makefile
index d7a0a50c..1c4f7d91 100644
--- a/drivers/net/dpaa/Makefile
+++ b/drivers/net/dpaa/Makefile
@@ -38,6 +38,7 @@ LDLIBS += -lrte_bus_dpaa
 LDLIBS += -lrte_mempool_dpaa
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
+LDLIBS += -lrte_common_dpaax
 
 # install this header file
 SYMLINK-$(CONFIG_RTE_LIBRTE_DPAA_PMD)-include := rte_pmd_dpaa.h
diff --git a/drivers/net/dpaa/dpaa_ethdev.c b/drivers/net/dpaa/dpaa_ethdev.c
index 7a950ac0..d0572b3d 100644
--- a/drivers/net/dpaa/dpaa_ethdev.c
+++ b/drivers/net/dpaa/dpaa_ethdev.c
@@ -47,16 +47,15 @@
 
 /* Supported Rx offloads */
 static uint64_t dev_rx_offloads_sup =
-		DEV_RX_OFFLOAD_JUMBO_FRAME;
+		DEV_RX_OFFLOAD_JUMBO_FRAME |
+		DEV_RX_OFFLOAD_SCATTER;
 
 /* Rx offloads which cannot be disabled */
 static uint64_t dev_rx_offloads_nodis =
 		DEV_RX_OFFLOAD_IPV4_CKSUM |
 		DEV_RX_OFFLOAD_UDP_CKSUM |
 		DEV_RX_OFFLOAD_TCP_CKSUM |
-		DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
-		DEV_RX_OFFLOAD_CRC_STRIP |
-		DEV_RX_OFFLOAD_SCATTER;
+		DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM;
 
 /* Supported Tx offloads */
 static uint64_t dev_tx_offloads_sup;
@@ -148,11 +147,30 @@ dpaa_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 	struct dpaa_if *dpaa_intf = dev->data->dev_private;
 	uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
 				+ VLAN_TAG_SIZE;
+	uint32_t buffsz = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
 
 	PMD_INIT_FUNC_TRACE();
 
 	if (mtu < ETHER_MIN_MTU || frame_size > DPAA_MAX_RX_PKT_LEN)
 		return -EINVAL;
+	/*
+	 * Refuse mtu that requires the support of scattered packets
+	 * when this feature has not been enabled before.
+	 */
+	if (dev->data->min_rx_buf_size &&
+		!dev->data->scattered_rx && frame_size > buffsz) {
+		DPAA_PMD_ERR("SG not enabled, will not fit in one buffer");
+		return -EINVAL;
+	}
+
+	/* check <seg size> * <max_seg>  >= max_frame */
+	if (dev->data->min_rx_buf_size && dev->data->scattered_rx &&
+		(frame_size > buffsz * DPAA_SGT_MAX_ENTRIES)) {
+		DPAA_PMD_ERR("Too big to fit for Max SG list %d",
+				buffsz * DPAA_SGT_MAX_ENTRIES);
+		return -EINVAL;
+	}
+
 	if (frame_size > ETHER_MAX_LEN)
 		dev->data->dev_conf.rxmode.offloads &=
 						DEV_RX_OFFLOAD_JUMBO_FRAME;
@@ -194,15 +212,32 @@ dpaa_eth_dev_configure(struct rte_eth_dev *dev)
 	}
 
 	if (rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
+		uint32_t max_len;
+
+		DPAA_PMD_DEBUG("enabling jumbo");
+
 		if (dev->data->dev_conf.rxmode.max_rx_pkt_len <=
-		    DPAA_MAX_RX_PKT_LEN) {
-			fman_if_set_maxfrm(dpaa_intf->fif,
-				dev->data->dev_conf.rxmode.max_rx_pkt_len);
-			return 0;
-		} else {
-			return -1;
+		    DPAA_MAX_RX_PKT_LEN)
+			max_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
+		else {
+			DPAA_PMD_INFO("enabling jumbo override conf max len=%d "
+				"supported is %d",
+				dev->data->dev_conf.rxmode.max_rx_pkt_len,
+				DPAA_MAX_RX_PKT_LEN);
+			max_len = DPAA_MAX_RX_PKT_LEN;
 		}
+
+		fman_if_set_maxfrm(dpaa_intf->fif, max_len);
+		dev->data->mtu = max_len
+				- ETHER_HDR_LEN - ETHER_CRC_LEN - VLAN_TAG_SIZE;
 	}
+
+	if (rx_offloads & DEV_RX_OFFLOAD_SCATTER) {
+		DPAA_PMD_DEBUG("enabling scatter mode");
+		fman_if_set_sg(dpaa_intf->fif, 1);
+		dev->data->scattered_rx = 1;
+	}
+
 	return 0;
 }
 
@@ -300,15 +335,21 @@ static void dpaa_eth_dev_info(struct rte_eth_dev *dev,
 
 	dev_info->max_rx_queues = dpaa_intf->nb_rx_queues;
 	dev_info->max_tx_queues = dpaa_intf->nb_tx_queues;
-	dev_info->min_rx_bufsize = DPAA_MIN_RX_BUF_SIZE;
 	dev_info->max_rx_pktlen = DPAA_MAX_RX_PKT_LEN;
 	dev_info->max_mac_addrs = DPAA_MAX_MAC_FILTER;
 	dev_info->max_hash_mac_addrs = 0;
 	dev_info->max_vfs = 0;
 	dev_info->max_vmdq_pools = ETH_16_POOLS;
 	dev_info->flow_type_rss_offloads = DPAA_RSS_OFFLOAD_ALL;
-	dev_info->speed_capa = (ETH_LINK_SPEED_1G |
-				ETH_LINK_SPEED_10G);
+
+	if (dpaa_intf->fif->mac_type == fman_mac_1g)
+		dev_info->speed_capa = ETH_LINK_SPEED_1G;
+	else if (dpaa_intf->fif->mac_type == fman_mac_10g)
+		dev_info->speed_capa = (ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G);
+	else
+		DPAA_PMD_ERR("invalid link_speed: %s, %d",
+			     dpaa_intf->name, dpaa_intf->fif->mac_type);
+
 	dev_info->rx_offload_capa = dev_rx_offloads_sup |
 					dev_rx_offloads_nodis;
 	dev_info->tx_offload_capa = dev_tx_offloads_sup |
@@ -514,6 +555,7 @@ int dpaa_eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	struct qm_mcc_initfq opts = {0};
 	u32 flags = 0;
 	int ret;
+	u32 buffsz = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -527,6 +569,28 @@ int dpaa_eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	DPAA_PMD_INFO("Rx queue setup for queue index: %d fq_id (0x%x)",
 			queue_idx, rxq->fqid);
 
+	/* Max packet can fit in single buffer */
+	if (dev->data->dev_conf.rxmode.max_rx_pkt_len <= buffsz) {
+		;
+	} else if (dev->data->dev_conf.rxmode.offloads &
+			DEV_RX_OFFLOAD_SCATTER) {
+		if (dev->data->dev_conf.rxmode.max_rx_pkt_len >
+			buffsz * DPAA_SGT_MAX_ENTRIES) {
+			DPAA_PMD_ERR("max RxPkt size %d too big to fit "
+				"MaxSGlist %d",
+				dev->data->dev_conf.rxmode.max_rx_pkt_len,
+				buffsz * DPAA_SGT_MAX_ENTRIES);
+			rte_errno = EOVERFLOW;
+			return -rte_errno;
+		}
+	} else {
+		DPAA_PMD_WARN("The requested maximum Rx packet size (%u) is"
+		     " larger than a single mbuf (%u) and scattered"
+		     " mode has not been requested",
+		     dev->data->dev_conf.rxmode.max_rx_pkt_len,
+		     buffsz - RTE_PKTMBUF_HEADROOM);
+	}
+
 	if (!dpaa_intf->bp_info || dpaa_intf->bp_info->mp != mp) {
 		struct fman_if_ic_params icp;
 		uint32_t fd_offset;
@@ -553,10 +617,13 @@ int dpaa_eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 		fman_if_set_bp(dpaa_intf->fif, mp->size,
 			       dpaa_intf->bp_info->bpid, bp_size);
 		dpaa_intf->valid = 1;
-		DPAA_PMD_INFO("if =%s - fd_offset = %d offset = %d",
-			    dpaa_intf->name, fd_offset,
-			fman_if_get_fdoff(dpaa_intf->fif));
+		DPAA_PMD_DEBUG("if:%s fd_offset = %d offset = %d",
+				dpaa_intf->name, fd_offset,
+				fman_if_get_fdoff(dpaa_intf->fif));
 	}
+	DPAA_PMD_DEBUG("if:%s sg_on = %d, max_frm =%d", dpaa_intf->name,
+		fman_if_get_sg_enable(dpaa_intf->fif),
+		dev->data->dev_conf.rxmode.max_rx_pkt_len);
 	/* checking if push mode only, no error check for now */
 	if (dpaa_push_mode_max_queue > dpaa_push_queue_idx) {
 		dpaa_push_queue_idx++;
@@ -594,8 +661,13 @@ int dpaa_eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 				"ret:%d(%s)", rxq->fqid, ret, strerror(ret));
 			return ret;
 		}
-		rxq->cb.dqrr_dpdk_pull_cb = dpaa_rx_cb;
-		rxq->cb.dqrr_prepare = dpaa_rx_cb_prepare;
+		if (dpaa_svr_family == SVR_LS1043A_FAMILY) {
+			rxq->cb.dqrr_dpdk_pull_cb = dpaa_rx_cb_no_prefetch;
+		} else {
+			rxq->cb.dqrr_dpdk_pull_cb = dpaa_rx_cb;
+			rxq->cb.dqrr_prepare = dpaa_rx_cb_prepare;
+		}
+
 		rxq->is_static = true;
 	}
 	dev->data->rx_queues[queue_idx] = rxq;
@@ -630,7 +702,8 @@ dpaa_eth_eventq_attach(const struct rte_eth_dev *dev,
 	struct qm_mcc_initfq opts = {0};
 
 	if (dpaa_push_mode_max_queue)
-		DPAA_PMD_WARN("PUSH mode already enabled for first %d queues.\n"
+		DPAA_PMD_WARN("PUSH mode q and EVENTDEV are not compatible\n"
+			      "PUSH mode already enabled for first %d queues.\n"
 			      "To disable set DPAA_PUSH_QUEUES_NUMBER to 0\n",
 			      dpaa_push_mode_max_queue);
 
@@ -1012,7 +1085,7 @@ static int dpaa_rx_queue_init(struct qman_fq *fq, struct qman_cgr *cgr_rx,
 {
 	struct qm_mcc_initfq opts = {0};
 	int ret;
-	u32 flags = 0;
+	u32 flags = QMAN_FQ_FLAG_NO_ENQUEUE;
 	struct qm_mcc_initcgr cgr_opts = {
 		.we_mask = QM_CGR_WE_CS_THRES |
 				QM_CGR_WE_CSTD_EN |
@@ -1025,15 +1098,18 @@ static int dpaa_rx_queue_init(struct qman_fq *fq, struct qman_cgr *cgr_rx,
 
 	PMD_INIT_FUNC_TRACE();
 
-	ret = qman_reserve_fqid(fqid);
-	if (ret) {
-		DPAA_PMD_ERR("reserve rx fqid 0x%x failed with ret: %d",
-			     fqid, ret);
-		return -EINVAL;
+	if (fqid) {
+		ret = qman_reserve_fqid(fqid);
+		if (ret) {
+			DPAA_PMD_ERR("reserve rx fqid 0x%x failed with ret: %d",
+				     fqid, ret);
+			return -EINVAL;
+		}
+	} else {
+		flags |= QMAN_FQ_FLAG_DYNAMIC_FQID;
 	}
-
 	DPAA_PMD_DEBUG("creating rx fq %p, fqid 0x%x", fq, fqid);
-	ret = qman_create_fq(fqid, QMAN_FQ_FLAG_NO_ENQUEUE, fq);
+	ret = qman_create_fq(fqid, flags, fq);
 	if (ret) {
 		DPAA_PMD_ERR("create rx fqid 0x%x failed with ret: %d",
 			fqid, ret);
@@ -1052,7 +1128,7 @@ static int dpaa_rx_queue_init(struct qman_fq *fq, struct qman_cgr *cgr_rx,
 		if (ret) {
 			DPAA_PMD_WARN(
 				"rx taildrop init fail on rx fqid 0x%x(ret=%d)",
-				fqid, ret);
+				fq->fqid, ret);
 			goto without_cgr;
 		}
 		opts.we_mask |= QM_INITFQ_WE_CGID;
@@ -1060,7 +1136,7 @@ static int dpaa_rx_queue_init(struct qman_fq *fq, struct qman_cgr *cgr_rx,
 		opts.fqd.fq_ctrl |= QM_FQCTRL_CGE;
 	}
 without_cgr:
-	ret = qman_init_fq(fq, flags, &opts);
+	ret = qman_init_fq(fq, 0, &opts);
 	if (ret)
 		DPAA_PMD_ERR("init rx fqid 0x%x failed with ret:%d", fqid, ret);
 	return ret;
@@ -1213,7 +1289,7 @@ dpaa_dev_init(struct rte_eth_dev *eth_dev)
 		if (default_q)
 			fqid = cfg->rx_def;
 		else
-			fqid = DPAA_PCD_FQID_START + dpaa_intf->ifid *
+			fqid = DPAA_PCD_FQID_START + dpaa_intf->fif->mac_idx *
 				DPAA_PCD_FQID_MULTIPLIER + loop;
 
 		if (dpaa_intf->cgr_rx)
@@ -1304,6 +1380,9 @@ dpaa_dev_init(struct rte_eth_dev *eth_dev)
 	fman_if_reset_mcast_filter_table(fman_intf);
 	/* Reset interface statistics */
 	fman_if_stats_reset(fman_intf);
+	/* Disable SG by default */
+	fman_if_set_sg(fman_intf, 0);
+	fman_if_set_maxfrm(fman_intf, ETHER_MAX_LEN + VLAN_TAG_SIZE);
 
 	return 0;
 
@@ -1360,10 +1439,6 @@ dpaa_dev_uninit(struct rte_eth_dev *dev)
 	rte_free(dpaa_intf->tx_queues);
 	dpaa_intf->tx_queues = NULL;
 
-	/* free memory for storing MAC addresses */
-	rte_free(dev->data->mac_addrs);
-	dev->data->mac_addrs = NULL;
-
 	dev->dev_ops = NULL;
 	dev->rx_pkt_burst = NULL;
 	dev->tx_pkt_burst = NULL;
@@ -1372,7 +1447,7 @@ dpaa_dev_uninit(struct rte_eth_dev *dev)
 }
 
 static int
-rte_dpaa_probe(struct rte_dpaa_driver *dpaa_drv,
+rte_dpaa_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused,
 	       struct rte_dpaa_device *dpaa_dev)
 {
 	int diag;
@@ -1456,7 +1531,6 @@ rte_dpaa_probe(struct rte_dpaa_driver *dpaa_drv,
 	}
 
 	eth_dev->device = &dpaa_dev->device;
-	eth_dev->device->driver = &dpaa_drv->driver;
 	dpaa_dev->eth_dev = eth_dev;
 
 	/* Invoke PMD device initialization function */
@@ -1466,9 +1540,6 @@ rte_dpaa_probe(struct rte_dpaa_driver *dpaa_drv,
 		return 0;
 	}
 
-	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
-		rte_free(eth_dev->data->dev_private);
-
 	rte_eth_dev_release_port(eth_dev);
 	return diag;
 }
@@ -1483,9 +1554,6 @@ rte_dpaa_remove(struct rte_dpaa_device *dpaa_dev)
 	eth_dev = dpaa_dev->eth_dev;
 	dpaa_dev_uninit(eth_dev);
 
-	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
-		rte_free(eth_dev->data->dev_private);
-
 	rte_eth_dev_release_port(eth_dev);
 
 	return 0;
diff --git a/drivers/net/dpaa/dpaa_ethdev.h b/drivers/net/dpaa/dpaa_ethdev.h
index c79b9f86..2fc72317 100644
--- a/drivers/net/dpaa/dpaa_ethdev.h
+++ b/drivers/net/dpaa/dpaa_ethdev.h
@@ -39,9 +39,10 @@
 /* Alignment to use for cpu-local structs to avoid coherency problems. */
 #define MAX_CACHELINE			64
 
-#define DPAA_MIN_RX_BUF_SIZE 512
 #define DPAA_MAX_RX_PKT_LEN  10240
 
+#define DPAA_SGT_MAX_ENTRIES 16 /* maximum number of entries in SG Table */
+
 /* RX queue tail drop threshold (CGR Based) in frame count */
 #define CGR_RX_PERFQ_THRESH 256
 
@@ -63,7 +64,7 @@
 #define DPAA_PCD_FQID_START		0x400
 #define DPAA_PCD_FQID_MULTIPLIER	0x100
 #define DPAA_DEFAULT_NUM_PCD_QUEUES	1
-#define DPAA_MAX_NUM_PCD_QUEUES		32
+#define DPAA_MAX_NUM_PCD_QUEUES		4
 
 #define DPAA_IF_TX_PRIORITY		3
 #define DPAA_IF_RX_PRIORITY		0
diff --git a/drivers/net/dpaa/dpaa_rxtx.c b/drivers/net/dpaa/dpaa_rxtx.c
index 168b77e4..c4471c22 100644
--- a/drivers/net/dpaa/dpaa_rxtx.c
+++ b/drivers/net/dpaa/dpaa_rxtx.c
@@ -306,8 +306,6 @@ dpaa_eth_sg_to_mbuf(const struct qm_fd *fd, uint32_t ifid)
 	int i = 0;
 	uint8_t fd_offset = fd->offset;
 
-	DPAA_DP_LOG(DEBUG, "Received an SG frame");
-
 	vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd));
 	if (!vaddr) {
 		DPAA_PMD_ERR("unable to convert physical address");
@@ -349,6 +347,8 @@ dpaa_eth_sg_to_mbuf(const struct qm_fd *fd, uint32_t ifid)
 		}
 		prev_seg = cur_seg;
 	}
+	DPAA_DP_LOG(DEBUG, "Received an SG frame len =%d, num_sg =%d",
+			first_seg->pkt_len, first_seg->nb_segs);
 
 	dpaa_eth_packet_info(first_seg, vaddr);
 	rte_pktmbuf_free_seg(temp);
@@ -367,22 +367,21 @@ dpaa_eth_fd_to_mbuf(const struct qm_fd *fd, uint32_t ifid)
 	uint16_t offset;
 	uint32_t length;
 
-	DPAA_DP_LOG(DEBUG, " FD--->MBUF");
-
 	if (unlikely(format == qm_fd_sg))
 		return dpaa_eth_sg_to_mbuf(fd, ifid);
 
-	ptr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd));
-
-	rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF));
-
 	offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> DPAA_FD_OFFSET_SHIFT;
 	length = fd->opaque & DPAA_FD_LENGTH_MASK;
 
+	DPAA_DP_LOG(DEBUG, " FD--->MBUF off %d len = %d", offset, length);
+
 	/* Ignoring case when format != qm_fd_contig */
 	dpaa_display_frame(fd);
+	ptr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd));
 
 	mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size);
+	/* Prefetch the Parse results and packet data to L1 */
+	rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF));
 
 	mbuf->data_off = offset;
 	mbuf->data_len = length;
@@ -398,8 +397,9 @@ dpaa_eth_fd_to_mbuf(const struct qm_fd *fd, uint32_t ifid)
 	return mbuf;
 }
 
+/* Specific for LS1043 */
 void
-dpaa_rx_cb(struct qman_fq **fq, struct qm_dqrr_entry **dqrr,
+dpaa_rx_cb_no_prefetch(struct qman_fq **fq, struct qm_dqrr_entry **dqrr,
 	   void **bufs, int num_bufs)
 {
 	struct rte_mbuf *mbuf;
@@ -411,17 +411,13 @@ dpaa_rx_cb(struct qman_fq **fq, struct qm_dqrr_entry **dqrr,
 	uint32_t length;
 	uint8_t format;
 
-	if (dpaa_svr_family != SVR_LS1046A_FAMILY) {
-		bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[0]->fd.bpid);
-		ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[0]->fd));
-		rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF));
-		bufs[0] = (struct rte_mbuf *)((char *)ptr -
-				bp_info->meta_data_size);
-	}
+	bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[0]->fd.bpid);
+	ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[0]->fd));
+	rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF));
+	bufs[0] = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size);
 
 	for (i = 0; i < num_bufs; i++) {
-		if (dpaa_svr_family != SVR_LS1046A_FAMILY &&
-		    i < num_bufs - 1) {
+		if (i < num_bufs - 1) {
 			bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[i + 1]->fd.bpid);
 			ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[i + 1]->fd));
 			rte_prefetch0((void *)((uint8_t *)ptr +
@@ -458,6 +454,46 @@ dpaa_rx_cb(struct qman_fq **fq, struct qm_dqrr_entry **dqrr,
 	}
 }
 
+void
+dpaa_rx_cb(struct qman_fq **fq, struct qm_dqrr_entry **dqrr,
+	   void **bufs, int num_bufs)
+{
+	struct rte_mbuf *mbuf;
+	const struct qm_fd *fd;
+	struct dpaa_if *dpaa_intf;
+	uint16_t offset, i;
+	uint32_t length;
+	uint8_t format;
+
+	for (i = 0; i < num_bufs; i++) {
+		fd = &dqrr[i]->fd;
+		dpaa_intf = fq[0]->dpaa_intf;
+
+		format = (fd->opaque & DPAA_FD_FORMAT_MASK) >>
+				DPAA_FD_FORMAT_SHIFT;
+		if (unlikely(format == qm_fd_sg)) {
+			bufs[i] = dpaa_eth_sg_to_mbuf(fd, dpaa_intf->ifid);
+			continue;
+		}
+
+		offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >>
+				DPAA_FD_OFFSET_SHIFT;
+		length = fd->opaque & DPAA_FD_LENGTH_MASK;
+
+		mbuf = bufs[i];
+		mbuf->data_off = offset;
+		mbuf->data_len = length;
+		mbuf->pkt_len = length;
+		mbuf->port = dpaa_intf->ifid;
+
+		mbuf->nb_segs = 1;
+		mbuf->ol_flags = 0;
+		mbuf->next = NULL;
+		rte_mbuf_refcnt_set(mbuf, 1);
+		dpaa_eth_packet_info(mbuf, mbuf->buf_addr);
+	}
+}
+
 void dpaa_rx_cb_prepare(struct qm_dqrr_entry *dq, void **bufs)
 {
 	struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(dq->fd.bpid);
@@ -468,8 +504,7 @@ void dpaa_rx_cb_prepare(struct qm_dqrr_entry *dq, void **bufs)
 	 * So we prefetch the annoation beforehand, so that it is available
 	 * in cache when accessed.
 	 */
-	if (dpaa_svr_family == SVR_LS1046A_FAMILY)
-		rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF));
+	rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF));
 
 	*bufs = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size);
 }
@@ -870,6 +905,19 @@ dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs)
 				DPAA_TX_BURST_SIZE : nb_bufs;
 		for (loop = 0; loop < frames_to_send; loop++) {
 			mbuf = *(bufs++);
+			seqn = mbuf->seqn;
+			if (seqn != DPAA_INVALID_MBUF_SEQN) {
+				index = seqn - 1;
+				if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) {
+					flags[loop] =
+					   ((index & QM_EQCR_DCA_IDXMASK) << 8);
+					flags[loop] |= QMAN_ENQUEUE_FLAG_DCA;
+					DPAA_PER_LCORE_DQRR_SIZE--;
+					DPAA_PER_LCORE_DQRR_HELD &=
+								~(1 << index);
+				}
+			}
+
 			if (likely(RTE_MBUF_DIRECT(mbuf))) {
 				mp = mbuf->pool;
 				bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp);
@@ -916,18 +964,6 @@ dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs)
 					goto send_pkts;
 				}
 			}
-			seqn = mbuf->seqn;
-			if (seqn != DPAA_INVALID_MBUF_SEQN) {
-				index = seqn - 1;
-				if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) {
-					flags[loop] =
-					   ((index & QM_EQCR_DCA_IDXMASK) << 8);
-					flags[loop] |= QMAN_ENQUEUE_FLAG_DCA;
-					DPAA_PER_LCORE_DQRR_SIZE--;
-					DPAA_PER_LCORE_DQRR_HELD &=
-								~(1 << index);
-				}
-			}
 		}
 
 send_pkts:
diff --git a/drivers/net/dpaa/dpaa_rxtx.h b/drivers/net/dpaa/dpaa_rxtx.h
index d3e63516..75b093c1 100644
--- a/drivers/net/dpaa/dpaa_rxtx.h
+++ b/drivers/net/dpaa/dpaa_rxtx.h
@@ -32,8 +32,6 @@
 /* L4 Type field: TCP */
 #define DPAA_L4_PARSE_RESULT_TCP	0x20
 
-#define DPAA_SGT_MAX_ENTRIES 16 /* maximum number of entries in SG Table */
-
 #define DPAA_MAX_DEQUEUE_NUM_FRAMES    63
 	/** <Maximum number of frames to be dequeued in a single rx call*/
 
@@ -272,4 +270,7 @@ void dpaa_rx_cb(struct qman_fq **fq,
 		struct qm_dqrr_entry **dqrr, void **bufs, int num_bufs);
 
 void dpaa_rx_cb_prepare(struct qm_dqrr_entry *dq, void **bufs);
+
+void dpaa_rx_cb_no_prefetch(struct qman_fq **fq,
+		    struct qm_dqrr_entry **dqrr, void **bufs, int num_bufs);
 #endif
diff --git a/drivers/net/dpaa2/Makefile b/drivers/net/dpaa2/Makefile
index 9b0b1433..ca5f7a33 100644
--- a/drivers/net/dpaa2/Makefile
+++ b/drivers/net/dpaa2/Makefile
@@ -25,7 +25,7 @@ CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
 EXPORT_MAP := rte_pmd_dpaa2_version.map
 
 # library version
-LIBABIVER := 1
+LIBABIVER := 2
 
 # depends on fslmc bus which uses experimental API
 CFLAGS += -DALLOW_EXPERIMENTAL_API
@@ -40,5 +40,6 @@ LDLIBS += -lrte_bus_fslmc
 LDLIBS += -lrte_mempool_dpaa2
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
+LDLIBS += -lrte_common_dpaax
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h b/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h
index 779cdf2b..adb730b7 100644
--- a/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h
+++ b/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h
@@ -213,6 +213,46 @@ struct dpaa2_annot_hdr {
 #define DPAA2_L3_IPv6_UDP (L3_IPV6_1_PRESENT | L3_IPV6_1_UNICAST | \
 	L3_PROTO_UDP_PRESENT | L4_UNKNOWN_PROTOCOL)
 
+/**
+ * Macros to get values in word5
+ */
+#define SHIM_OFFSET_1(var)		((uint64_t)(var) & 0xFF00000000000000)
+#define SHIM_OFFSET_2(var)		((uint64_t)(var) & 0x00FF000000000000)
+#define IP_PID_OFFSET(var)		((uint64_t)(var) & 0x0000FF0000000000)
+#define ETH_OFFSET(var)			((uint64_t)(var) & 0x000000FF00000000)
+#define LLC_SNAP_OFFSET(var)		((uint64_t)(var) & 0x00000000FF000000)
+#define VLAN_TCI_OFFSET_1(var)		((uint64_t)(var) & 0x0000000000FF0000)
+#define VLAN_TCI_OFFSET_N(var)		((uint64_t)(var) & 0x000000000000FF00)
+#define LAST_ETYPE_OFFSET(var)		((uint64_t)(var) & 0x00000000000000FF)
+
+/**
+ * Macros to get values in word6
+ */
+#define PPPOE_OFFSET(var)		((uint64_t)(var) & 0xFF00000000000000)
+#define MPLS_OFFSET_1(var)		((uint64_t)(var) & 0x00FF000000000000)
+#define MPLS_OFFSET_N(var)		((uint64_t)(var) & 0x0000FF0000000000)
+#define ARP_OR_IP_OFFSET_1(var)		((uint64_t)(var) & 0x000000FF00000000)
+#define IP_N_OR_MIN_ENCAP_OFFSET(var)	((uint64_t)(var) & 0x00000000FF000000)
+#define GRE_OFFSET(var)			((uint64_t)(var) & 0x0000000000FF0000)
+#define L4_OFFSET(var)			((uint64_t)(var) & 0x000000000000FF00)
+#define GTP_OR_ESP_OR_IPSEC_OFFSET(var)	((uint64_t)(var) & 0x00000000000000FF)
+
+/**
+ * Macros to get values in word7
+ */
+#define IPV6_ROUTING_HDR_OFFSET_1(var)	((uint64_t)(var) & 0xFF00000000000000)
+#define IPV6_ROUTING_HDR_OFFSET_2(var)	((uint64_t)(var) & 0x00FF000000000000)
+#define NEXT_HDR_OFFSET(var)		((uint64_t)(var) & 0x0000FF0000000000)
+#define IPV6_FRAG_OFFSET(var)		((uint64_t)(var) & 0x000000FF00000000)
+#define GROSS_RUNNING_SUM(var)		((uint64_t)(var) & 0x00000000FFFF0000)
+#define RUNNING_SUM(var)		((uint64_t)(var) & 0x000000000000FFFF)
+
+/**
+ * Macros to get values in word8
+ */
+#define PARSE_ERROR_CODE(var)		((uint64_t)(var) & 0xFF00000000000000)
+#define SOFT_PARSING_CONTEXT(var)	((uint64_t)(var) & 0x00FFFFFFFFFFFFFF)
+
 /* Debug frame, otherwise supposed to be discarded */
 #define DPAA2_ETH_FAS_DISC	      0x80000000
 /* MACSEC frame */
diff --git a/drivers/net/dpaa2/dpaa2_ethdev.c b/drivers/net/dpaa2/dpaa2_ethdev.c
index c5047367..fa71807e 100644
--- a/drivers/net/dpaa2/dpaa2_ethdev.c
+++ b/drivers/net/dpaa2/dpaa2_ethdev.c
@@ -39,7 +39,6 @@ static uint64_t dev_rx_offloads_sup =
 
 /* Rx offloads which cannot be disabled */
 static uint64_t dev_rx_offloads_nodis =
-		DEV_RX_OFFLOAD_CRC_STRIP |
 		DEV_RX_OFFLOAD_SCATTER;
 
 /* Supported Tx offloads */
@@ -292,6 +291,35 @@ fail:
 	return -1;
 }
 
+static void
+dpaa2_free_rx_tx_queues(struct rte_eth_dev *dev)
+{
+	struct dpaa2_dev_priv *priv = dev->data->dev_private;
+	struct dpaa2_queue *dpaa2_q;
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* Queue allocation base */
+	if (priv->rx_vq[0]) {
+		/* cleaning up queue storage */
+		for (i = 0; i < priv->nb_rx_queues; i++) {
+			dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[i];
+			if (dpaa2_q->q_storage)
+				rte_free(dpaa2_q->q_storage);
+		}
+		/* cleanup tx queue cscn */
+		for (i = 0; i < priv->nb_tx_queues; i++) {
+			dpaa2_q = (struct dpaa2_queue *)priv->tx_vq[i];
+			if (!dpaa2_q->cscn)
+				rte_free(dpaa2_q->cscn);
+		}
+		/*free memory for all queues (RX+TX) */
+		rte_free(priv->rx_vq[0]);
+		priv->rx_vq[0] = NULL;
+	}
+}
+
 static int
 dpaa2_eth_dev_configure(struct rte_eth_dev *dev)
 {
@@ -406,7 +434,8 @@ dpaa2_eth_dev_configure(struct rte_eth_dev *dev)
 		}
 	}
 
-	dpaa2_vlan_offload_set(dev, ETH_VLAN_FILTER_MASK);
+	if (rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
+		dpaa2_vlan_offload_set(dev, ETH_VLAN_FILTER_MASK);
 
 	/* update the current status */
 	dpaa2_dev_link_update(dev, 0);
@@ -569,7 +598,8 @@ dpaa2_dev_tx_queue_setup(struct rte_eth_dev *dev,
 		 */
 		cong_notif_cfg.threshold_exit = CONG_EXIT_TX_THRESHOLD;
 		cong_notif_cfg.message_ctx = 0;
-		cong_notif_cfg.message_iova = (size_t)dpaa2_q->cscn;
+		cong_notif_cfg.message_iova =
+				(size_t)DPAA2_VADDR_TO_IOVA(dpaa2_q->cscn);
 		cong_notif_cfg.dest_cfg.dest_type = DPNI_DEST_NONE;
 		cong_notif_cfg.notification_mode =
 					 DPNI_CONG_OPT_WRITE_MEM_ON_ENTER |
@@ -867,23 +897,13 @@ dpaa2_dev_stop(struct rte_eth_dev *dev)
 static void
 dpaa2_dev_close(struct rte_eth_dev *dev)
 {
-	struct rte_eth_dev_data *data = dev->data;
 	struct dpaa2_dev_priv *priv = dev->data->dev_private;
 	struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw;
-	int i, ret;
+	int ret;
 	struct rte_eth_link link;
-	struct dpaa2_queue *dpaa2_q;
 
 	PMD_INIT_FUNC_TRACE();
 
-	for (i = 0; i < data->nb_tx_queues; i++) {
-		dpaa2_q = (struct dpaa2_queue *)data->tx_queues[i];
-		if (!dpaa2_q->cscn) {
-			rte_free(dpaa2_q->cscn);
-			dpaa2_q->cscn = NULL;
-		}
-	}
-
 	/* Clean the device first */
 	ret = dpni_reset(dpni, CMD_PRI_LOW, priv->token);
 	if (ret) {
@@ -1117,6 +1137,8 @@ int dpaa2_dev_stats_get(struct rte_eth_dev *dev,
 	int32_t  retcode;
 	uint8_t page0 = 0, page1 = 1, page2 = 2;
 	union dpni_statistics value;
+	int i;
+	struct dpaa2_queue *dpaa2_rxq, *dpaa2_txq;
 
 	memset(&value, 0, sizeof(union dpni_statistics));
 
@@ -1164,6 +1186,21 @@ int dpaa2_dev_stats_get(struct rte_eth_dev *dev,
 	stats->oerrors = value.page_2.egress_discarded_frames;
 	stats->imissed = value.page_2.ingress_nobuffer_discards;
 
+	/* Fill in per queue stats */
+	for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
+		(i < priv->nb_rx_queues || i < priv->nb_tx_queues); ++i) {
+		dpaa2_rxq = (struct dpaa2_queue *)priv->rx_vq[i];
+		dpaa2_txq = (struct dpaa2_queue *)priv->tx_vq[i];
+		if (dpaa2_rxq)
+			stats->q_ipackets[i] = dpaa2_rxq->rx_pkts;
+		if (dpaa2_txq)
+			stats->q_opackets[i] = dpaa2_txq->tx_pkts;
+
+		/* Byte counting is not implemented */
+		stats->q_ibytes[i]   = 0;
+		stats->q_obytes[i]   = 0;
+	}
+
 	return 0;
 
 err:
@@ -1323,6 +1360,8 @@ dpaa2_dev_stats_reset(struct rte_eth_dev *dev)
 	struct dpaa2_dev_priv *priv = dev->data->dev_private;
 	struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw;
 	int32_t  retcode;
+	int i;
+	struct dpaa2_queue *dpaa2_q;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -1335,6 +1374,19 @@ dpaa2_dev_stats_reset(struct rte_eth_dev *dev)
 	if (retcode)
 		goto error;
 
+	/* Reset the per queue stats in dpaa2_queue structure */
+	for (i = 0; i < priv->nb_rx_queues; i++) {
+		dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[i];
+		if (dpaa2_q)
+			dpaa2_q->rx_pkts = 0;
+	}
+
+	for (i = 0; i < priv->nb_tx_queues; i++) {
+		dpaa2_q = (struct dpaa2_queue *)priv->tx_vq[i];
+		if (dpaa2_q)
+			dpaa2_q->tx_pkts = 0;
+	}
+
 	return;
 
 error:
@@ -1360,7 +1412,7 @@ dpaa2_dev_link_update(struct rte_eth_dev *dev,
 
 	ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state);
 	if (ret < 0) {
-		DPAA2_PMD_ERR("error: dpni_get_link_state %d", ret);
+		DPAA2_PMD_DEBUG("error: dpni_get_link_state %d", ret);
 		return -1;
 	}
 
@@ -1422,7 +1474,7 @@ dpaa2_dev_set_link_up(struct rte_eth_dev *dev)
 	}
 	ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state);
 	if (ret < 0) {
-		DPAA2_PMD_ERR("Unable to get link state (%d)", ret);
+		DPAA2_PMD_DEBUG("Unable to get link state (%d)", ret);
 		return -1;
 	}
 
@@ -1785,6 +1837,74 @@ static struct eth_dev_ops dpaa2_ethdev_ops = {
 	.rss_hash_conf_get    = dpaa2_dev_rss_hash_conf_get,
 };
 
+/* Populate the mac address from physically available (u-boot/firmware) and/or
+ * one set by higher layers like MC (restool) etc.
+ * Returns the table of MAC entries (multiple entries)
+ */
+static int
+populate_mac_addr(struct fsl_mc_io *dpni_dev, struct dpaa2_dev_priv *priv,
+		  struct ether_addr *mac_entry)
+{
+	int ret;
+	struct ether_addr phy_mac, prime_mac;
+
+	memset(&phy_mac, 0, sizeof(struct ether_addr));
+	memset(&prime_mac, 0, sizeof(struct ether_addr));
+
+	/* Get the physical device MAC address */
+	ret = dpni_get_port_mac_addr(dpni_dev, CMD_PRI_LOW, priv->token,
+				     phy_mac.addr_bytes);
+	if (ret) {
+		DPAA2_PMD_ERR("DPNI get physical port MAC failed: %d", ret);
+		goto cleanup;
+	}
+
+	ret = dpni_get_primary_mac_addr(dpni_dev, CMD_PRI_LOW, priv->token,
+					prime_mac.addr_bytes);
+	if (ret) {
+		DPAA2_PMD_ERR("DPNI get Prime port MAC failed: %d", ret);
+		goto cleanup;
+	}
+
+	/* Now that both MAC have been obtained, do:
+	 *  if not_empty_mac(phy) && phy != Prime, overwrite prime with Phy
+	 *     and return phy
+	 *  If empty_mac(phy), return prime.
+	 *  if both are empty, create random MAC, set as prime and return
+	 */
+	if (!is_zero_ether_addr(&phy_mac)) {
+		/* If the addresses are not same, overwrite prime */
+		if (!is_same_ether_addr(&phy_mac, &prime_mac)) {
+			ret = dpni_set_primary_mac_addr(dpni_dev, CMD_PRI_LOW,
+							priv->token,
+							phy_mac.addr_bytes);
+			if (ret) {
+				DPAA2_PMD_ERR("Unable to set MAC Address: %d",
+					      ret);
+				goto cleanup;
+			}
+			memcpy(&prime_mac, &phy_mac, sizeof(struct ether_addr));
+		}
+	} else if (is_zero_ether_addr(&prime_mac)) {
+		/* In case phys and prime, both are zero, create random MAC */
+		eth_random_addr(prime_mac.addr_bytes);
+		ret = dpni_set_primary_mac_addr(dpni_dev, CMD_PRI_LOW,
+						priv->token,
+						prime_mac.addr_bytes);
+		if (ret) {
+			DPAA2_PMD_ERR("Unable to set MAC Address: %d", ret);
+			goto cleanup;
+		}
+	}
+
+	/* prime_mac the final MAC address */
+	memcpy(mac_entry, &prime_mac, sizeof(struct ether_addr));
+	return 0;
+
+cleanup:
+	return -1;
+}
+
 static int
 dpaa2_dev_init(struct rte_eth_dev *eth_dev)
 {
@@ -1867,7 +1987,10 @@ dpaa2_dev_init(struct rte_eth_dev *eth_dev)
 		goto init_err;
 	}
 
-	/* Allocate memory for storing MAC addresses */
+	/* Allocate memory for storing MAC addresses.
+	 * Table of mac_filter_entries size is allocated so that RTE ether lib
+	 * can add MAC entries when rte_eth_dev_mac_addr_add is called.
+	 */
 	eth_dev->data->mac_addrs = rte_zmalloc("dpni",
 		ETHER_ADDR_LEN * attr.mac_filter_entries, 0);
 	if (eth_dev->data->mac_addrs == NULL) {
@@ -1878,12 +2001,11 @@ dpaa2_dev_init(struct rte_eth_dev *eth_dev)
 		goto init_err;
 	}
 
-	ret = dpni_get_primary_mac_addr(dpni_dev, CMD_PRI_LOW,
-					priv->token,
-			(uint8_t *)(eth_dev->data->mac_addrs[0].addr_bytes));
+	ret = populate_mac_addr(dpni_dev, priv, &eth_dev->data->mac_addrs[0]);
 	if (ret) {
-		DPAA2_PMD_ERR("DPNI get mac address failed:Err Code = %d",
-			     ret);
+		DPAA2_PMD_ERR("Unable to fetch MAC Address for device");
+		rte_free(eth_dev->data->mac_addrs);
+		eth_dev->data->mac_addrs = NULL;
 		goto init_err;
 	}
 
@@ -1927,8 +2049,7 @@ dpaa2_dev_uninit(struct rte_eth_dev *eth_dev)
 {
 	struct dpaa2_dev_priv *priv = eth_dev->data->dev_private;
 	struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw;
-	int i, ret;
-	struct dpaa2_queue *dpaa2_q;
+	int ret;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -1942,23 +2063,7 @@ dpaa2_dev_uninit(struct rte_eth_dev *eth_dev)
 
 	dpaa2_dev_close(eth_dev);
 
-	if (priv->rx_vq[0]) {
-		/* cleaning up queue storage */
-		for (i = 0; i < priv->nb_rx_queues; i++) {
-			dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[i];
-			if (dpaa2_q->q_storage)
-				rte_free(dpaa2_q->q_storage);
-		}
-		/*free the all queue memory */
-		rte_free(priv->rx_vq[0]);
-		priv->rx_vq[0] = NULL;
-	}
-
-	/* free memory for storing MAC addresses */
-	if (eth_dev->data->mac_addrs) {
-		rte_free(eth_dev->data->mac_addrs);
-		eth_dev->data->mac_addrs = NULL;
-	}
+	dpaa2_free_rx_tx_queues(eth_dev);
 
 	/* Close the device at underlying layer*/
 	ret = dpni_close(dpni, CMD_PRI_LOW, priv->token);
@@ -2008,7 +2113,6 @@ rte_dpaa2_probe(struct rte_dpaa2_driver *dpaa2_drv,
 	}
 
 	eth_dev->device = &dpaa2_dev->device;
-	eth_dev->device->driver = &dpaa2_drv->driver;
 
 	dpaa2_dev->eth_dev = eth_dev;
 	eth_dev->data->rx_mbuf_alloc_failed = 0;
@@ -2023,8 +2127,6 @@ rte_dpaa2_probe(struct rte_dpaa2_driver *dpaa2_drv,
 		return 0;
 	}
 
-	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
-		rte_free(eth_dev->data->dev_private);
 	rte_eth_dev_release_port(eth_dev);
 	return diag;
 }
@@ -2037,8 +2139,6 @@ rte_dpaa2_remove(struct rte_dpaa2_device *dpaa2_dev)
 	eth_dev = dpaa2_dev->eth_dev;
 	dpaa2_dev_uninit(eth_dev);
 
-	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
-		rte_free(eth_dev->data->dev_private);
 	rte_eth_dev_release_port(eth_dev);
 
 	return 0;
diff --git a/drivers/net/dpaa2/dpaa2_rxtx.c b/drivers/net/dpaa2/dpaa2_rxtx.c
index ef109a62..eab943dc 100644
--- a/drivers/net/dpaa2/dpaa2_rxtx.c
+++ b/drivers/net/dpaa2/dpaa2_rxtx.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016 NXP
+ *   Copyright 2016-2018 NXP
  *
  */
 
@@ -25,18 +25,24 @@
 #include "dpaa2_ethdev.h"
 #include "base/dpaa2_hw_dpni_annot.h"
 
+static inline uint32_t __attribute__((hot))
+dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf,
+			struct dpaa2_annot_hdr *annotation);
+
 #define DPAA2_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid)  do { \
 	DPAA2_SET_FD_ADDR(_fd, DPAA2_MBUF_VADDR_TO_IOVA(_mbuf)); \
 	DPAA2_SET_FD_LEN(_fd, _mbuf->data_len); \
 	DPAA2_SET_ONLY_FD_BPID(_fd, _bpid); \
 	DPAA2_SET_FD_OFFSET(_fd, _mbuf->data_off); \
-	DPAA2_SET_FD_ASAL(_fd, DPAA2_ASAL_VAL); \
+	DPAA2_SET_FD_FRC(_fd, 0);		\
+	DPAA2_RESET_FD_CTRL(_fd);		\
+	DPAA2_RESET_FD_FLC(_fd);		\
 } while (0)
 
 static inline void __attribute__((hot))
-dpaa2_dev_rx_parse_frc(struct rte_mbuf *m, uint16_t frc)
+dpaa2_dev_rx_parse_new(struct rte_mbuf *m, const struct qbman_fd *fd)
 {
-	DPAA2_PMD_DP_DEBUG("frc = 0x%x\t", frc);
+	uint16_t frc = DPAA2_GET_FD_FRC_PARSE_SUM(fd);
 
 	m->packet_type = RTE_PTYPE_UNKNOWN;
 	switch (frc) {
@@ -91,29 +97,45 @@ dpaa2_dev_rx_parse_frc(struct rte_mbuf *m, uint16_t frc)
 		m->packet_type = RTE_PTYPE_L2_ETHER |
 			RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_ICMP;
 		break;
-	case DPAA2_PKT_TYPE_VLAN_1:
-	case DPAA2_PKT_TYPE_VLAN_2:
-		m->ol_flags |= PKT_RX_VLAN;
-		break;
-	/* More switch cases can be added */
-	/* TODO: Add handling for checksum error check from FRC */
 	default:
-		m->packet_type = RTE_PTYPE_UNKNOWN;
+		m->packet_type = dpaa2_dev_rx_parse_slow(m,
+		  (void *)((size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
+			 + DPAA2_FD_PTA_SIZE));
 	}
+	m->hash.rss = fd->simple.flc_hi;
+	m->ol_flags |= PKT_RX_RSS_HASH;
 }
 
 static inline uint32_t __attribute__((hot))
-dpaa2_dev_rx_parse_slow(struct dpaa2_annot_hdr *annotation)
+dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf,
+			struct dpaa2_annot_hdr *annotation)
 {
 	uint32_t pkt_type = RTE_PTYPE_UNKNOWN;
+	uint16_t *vlan_tci;
+
+	DPAA2_PMD_DP_DEBUG("(slow parse)annotation(3)=0x%" PRIx64 "\t"
+			"(4)=0x%" PRIx64 "\t",
+			annotation->word3, annotation->word4);
+
+	if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_1_PRESENT)) {
+		vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *,
+			(VLAN_TCI_OFFSET_1(annotation->word5) >> 16));
+		mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci);
+		mbuf->ol_flags |= PKT_RX_VLAN;
+		pkt_type |= RTE_PTYPE_L2_ETHER_VLAN;
+	} else if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_N_PRESENT)) {
+		vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *,
+			(VLAN_TCI_OFFSET_1(annotation->word5) >> 16));
+		mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci);
+		mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_QINQ;
+		pkt_type |= RTE_PTYPE_L2_ETHER_QINQ;
+	}
 
-	DPAA2_PMD_DP_DEBUG("(slow parse) Annotation = 0x%" PRIx64 "\t",
-			   annotation->word4);
 	if (BIT_ISSET_AT_POS(annotation->word3, L2_ARP_PRESENT)) {
-		pkt_type = RTE_PTYPE_L2_ETHER_ARP;
+		pkt_type |= RTE_PTYPE_L2_ETHER_ARP;
 		goto parse_done;
 	} else if (BIT_ISSET_AT_POS(annotation->word3, L2_ETH_MAC_PRESENT)) {
-		pkt_type = RTE_PTYPE_L2_ETHER;
+		pkt_type |= RTE_PTYPE_L2_ETHER;
 	} else {
 		goto parse_done;
 	}
@@ -135,6 +157,11 @@ dpaa2_dev_rx_parse_slow(struct dpaa2_annot_hdr *annotation)
 		goto parse_done;
 	}
 
+	if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE))
+		mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
+	else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE))
+		mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
+
 	if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT |
 	    L3_IP_1_MORE_FRAGMENT |
 	    L3_IP_N_FIRST_FRAGMENT |
@@ -173,16 +200,15 @@ dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr)
 	DPAA2_PMD_DP_DEBUG("(fast parse) Annotation = 0x%" PRIx64 "\t",
 			   annotation->word4);
 
-	/* Check offloads first */
-	if (BIT_ISSET_AT_POS(annotation->word3,
-			     L2_VLAN_1_PRESENT | L2_VLAN_N_PRESENT))
-		mbuf->ol_flags |= PKT_RX_VLAN;
-
 	if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE))
 		mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 	else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE))
 		mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 
+	/* Check detailed parsing requirement */
+	if (annotation->word3 & 0x7FFFFC3FFFF)
+		return dpaa2_dev_rx_parse_slow(mbuf, annotation);
+
 	/* Return some common types from parse processing */
 	switch (annotation->word4) {
 	case DPAA2_L3_IPv4:
@@ -205,7 +231,7 @@ dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr)
 		break;
 	}
 
-	return dpaa2_dev_rx_parse_slow(annotation);
+	return dpaa2_dev_rx_parse_slow(mbuf, annotation);
 }
 
 static inline struct rte_mbuf *__attribute__((hot))
@@ -236,8 +262,7 @@ eth_sg_fd_to_mbuf(const struct qbman_fd *fd)
 	first_seg->nb_segs = 1;
 	first_seg->next = NULL;
 	if (dpaa2_svr_family == SVR_LX2160A)
-		dpaa2_dev_rx_parse_frc(first_seg,
-				DPAA2_GET_FD_FRC_PARSE_SUM(fd));
+		dpaa2_dev_rx_parse_new(first_seg, fd);
 	else
 		first_seg->packet_type = dpaa2_dev_rx_parse(first_seg,
 			(void *)((size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
@@ -293,7 +318,7 @@ eth_fd_to_mbuf(const struct qbman_fd *fd)
 	 */
 
 	if (dpaa2_svr_family == SVR_LX2160A)
-		dpaa2_dev_rx_parse_frc(mbuf, DPAA2_GET_FD_FRC_PARSE_SUM(fd));
+		dpaa2_dev_rx_parse_new(mbuf, fd);
 	else
 		mbuf->packet_type = dpaa2_dev_rx_parse(mbuf,
 			(void *)((size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
@@ -476,8 +501,7 @@ dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		}
 	}
 	swp = DPAA2_PER_LCORE_ETHRX_PORTAL;
-	pull_size = (nb_pkts > DPAA2_DQRR_RING_SIZE) ?
-					       DPAA2_DQRR_RING_SIZE : nb_pkts;
+	pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts;
 	if (unlikely(!q_storage->active_dqs)) {
 		q_storage->toggle = 0;
 		dq_storage = q_storage->dq_storage[q_storage->toggle];
@@ -555,10 +579,12 @@ dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		}
 		fd = qbman_result_DQ_fd(dq_storage);
 
-		next_fd = qbman_result_DQ_fd(dq_storage + 1);
-		/* Prefetch Annotation address for the parse results */
-		rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(next_fd)
-				+ DPAA2_FD_PTA_SIZE + 16));
+		if (dpaa2_svr_family != SVR_LX2160A) {
+			next_fd = qbman_result_DQ_fd(dq_storage + 1);
+			/* Prefetch Annotation address for the parse results */
+			rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(
+				      next_fd) + DPAA2_FD_PTA_SIZE + 16));
+		}
 
 		if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg))
 			bufs[num_rx] = eth_sg_fd_to_mbuf(fd);
@@ -685,7 +711,6 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 	/*Prepare enqueue descriptor*/
 	qbman_eq_desc_clear(&eqdesc);
 	qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
-	qbman_eq_desc_set_response(&eqdesc, 0, 0);
 	qbman_eq_desc_set_qd(&eqdesc, priv->qdid,
 			     dpaa2_q->flow_id, dpaa2_q->tc_index);
 	/*Clear the unused FD fields before sending*/
@@ -699,7 +724,8 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 				goto skip_tx;
 		}
 
-		frames_to_send = (nb_pkts >> 3) ? MAX_TX_RING_SLOTS : nb_pkts;
+		frames_to_send = (nb_pkts > dpaa2_eqcr_size) ?
+			dpaa2_eqcr_size : nb_pkts;
 
 		for (loop = 0; loop < frames_to_send; loop++) {
 			if ((*bufs)->seqn) {
@@ -712,9 +738,6 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 				(*bufs)->seqn = DPAA2_INVALID_MBUF_SEQN;
 			}
 
-			fd_arr[loop].simple.frc = 0;
-			DPAA2_RESET_FD_CTRL((&fd_arr[loop]));
-			DPAA2_SET_FD_FLC((&fd_arr[loop]), (size_t)NULL);
 			if (likely(RTE_MBUF_DIRECT(*bufs))) {
 				mp = (*bufs)->pool;
 				/* Check the basic scenario and set
diff --git a/drivers/net/dpaa2/mc/dpni.c b/drivers/net/dpaa2/mc/dpni.c
index 9f228169..44b5604d 100644
--- a/drivers/net/dpaa2/mc/dpni.c
+++ b/drivers/net/dpaa2/mc/dpni.c
@@ -121,6 +121,7 @@ int dpni_create(struct fsl_mc_io *mc_io,
 	cmd_params->num_queues = cfg->num_queues;
 	cmd_params->num_tcs = cfg->num_tcs;
 	cmd_params->mac_filter_entries = cfg->mac_filter_entries;
+	cmd_params->num_rx_tcs = cfg->num_rx_tcs;
 	cmd_params->vlan_filter_entries =  cfg->vlan_filter_entries;
 	cmd_params->qos_entries = cfg->qos_entries;
 	cmd_params->fs_entries = cpu_to_le16(cfg->fs_entries);
@@ -664,9 +665,14 @@ int dpni_get_buffer_layout(struct fsl_mc_io *mc_io,
 
 	/* retrieve response parameters */
 	rsp_params = (struct dpni_rsp_get_buffer_layout *)cmd.params;
-	layout->pass_timestamp = dpni_get_field(rsp_params->flags, PASS_TS);
-	layout->pass_parser_result = dpni_get_field(rsp_params->flags, PASS_PR);
-	layout->pass_frame_status = dpni_get_field(rsp_params->flags, PASS_FS);
+	layout->pass_timestamp =
+				(int)dpni_get_field(rsp_params->flags, PASS_TS);
+	layout->pass_parser_result =
+				(int)dpni_get_field(rsp_params->flags, PASS_PR);
+	layout->pass_frame_status =
+				(int)dpni_get_field(rsp_params->flags, PASS_FS);
+	layout->pass_sw_opaque =
+			(int)dpni_get_field(rsp_params->flags, PASS_SWO);
 	layout->private_data_size = le16_to_cpu(rsp_params->private_data_size);
 	layout->data_align = le16_to_cpu(rsp_params->data_align);
 	layout->data_head_room = le16_to_cpu(rsp_params->head_room);
@@ -702,10 +708,11 @@ int dpni_set_buffer_layout(struct fsl_mc_io *mc_io,
 					  token);
 	cmd_params = (struct dpni_cmd_set_buffer_layout *)cmd.params;
 	cmd_params->qtype = qtype;
-	cmd_params->options = cpu_to_le16(layout->options);
+	cmd_params->options = cpu_to_le16((uint16_t)layout->options);
 	dpni_set_field(cmd_params->flags, PASS_TS, layout->pass_timestamp);
 	dpni_set_field(cmd_params->flags, PASS_PR, layout->pass_parser_result);
 	dpni_set_field(cmd_params->flags, PASS_FS, layout->pass_frame_status);
+	dpni_set_field(cmd_params->flags, PASS_SWO, layout->pass_sw_opaque);
 	cmd_params->private_data_size = cpu_to_le16(layout->private_data_size);
 	cmd_params->data_align = cpu_to_le16(layout->data_align);
 	cmd_params->head_room = cpu_to_le16(layout->data_head_room);
@@ -893,6 +900,7 @@ int dpni_set_link_cfg(struct fsl_mc_io *mc_io,
 	cmd_params = (struct dpni_cmd_set_link_cfg *)cmd.params;
 	cmd_params->rate = cpu_to_le32(cfg->rate);
 	cmd_params->options = cpu_to_le64(cfg->options);
+	cmd_params->advertising = cpu_to_le64(cfg->advertising);
 
 	/* send command to mc*/
 	return mc_send_command(mc_io, &cmd);
@@ -929,8 +937,11 @@ int dpni_get_link_state(struct fsl_mc_io *mc_io,
 	/* retrieve response parameters */
 	rsp_params = (struct dpni_rsp_get_link_state *)cmd.params;
 	state->up = dpni_get_field(rsp_params->flags, LINK_STATE);
+	state->state_valid = dpni_get_field(rsp_params->flags, STATE_VALID);
 	state->rate = le32_to_cpu(rsp_params->rate);
 	state->options = le64_to_cpu(rsp_params->options);
+	state->supported = le64_to_cpu(rsp_params->supported);
+	state->advertising = le64_to_cpu(rsp_params->advertising);
 
 	return 0;
 }
@@ -1471,6 +1482,9 @@ int dpni_set_rx_tc_dist(struct fsl_mc_io *mc_io,
 	dpni_set_field(cmd_params->keep_hash_key,
 		       KEEP_HASH_KEY,
 		       cfg->fs_cfg.keep_hash_key);
+	dpni_set_field(cmd_params->keep_hash_key,
+		       KEEP_ENTRIES,
+		       cfg->fs_cfg.keep_entries);
 
 	/* send command to mc*/
 	return mc_send_command(mc_io, &cmd);
@@ -1764,8 +1778,8 @@ int dpni_get_queue(struct fsl_mc_io *mc_io,
  * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
  * @token:	Token of DPNI object
  * @page:	Selects the statistics page to retrieve, see
- *		DPNI_GET_STATISTICS output. Pages are numbered 0 to 2.
- * @param:  Custom parameter for some pages used to select
+ *		DPNI_GET_STATISTICS output. Pages are numbered 0 to 3.
+ * @param:	Custom parameter for some pages used to select
  *		a certain statistic source, for example the TC.
  * @stat:	Structure containing the statistics
  *
@@ -1941,3 +1955,111 @@ int dpni_get_taildrop(struct fsl_mc_io *mc_io,
 
 	return 0;
 }
+
+/**
+ * dpni_set_opr() - Set Order Restoration configuration.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPNI object
+ * @tc:		Traffic class, in range 0 to NUM_TCS - 1
+ * @index:	Selects the specific queue out of the set allocated
+ *			for the same TC. Value must be in range 0 to
+ *			NUM_QUEUES - 1
+ * @options:	Configuration mode options
+ *			can be OPR_OPT_CREATE or OPR_OPT_RETIRE
+ * @cfg:	Configuration options for the OPR
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dpni_set_opr(struct fsl_mc_io *mc_io,
+		 uint32_t cmd_flags,
+		 uint16_t token,
+		 uint8_t tc,
+		 uint8_t index,
+		 uint8_t options,
+		 struct opr_cfg *cfg)
+{
+	struct dpni_cmd_set_opr *cmd_params;
+	struct mc_command cmd = { 0 };
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(
+			DPNI_CMDID_SET_OPR,
+			cmd_flags,
+			token);
+	cmd_params = (struct dpni_cmd_set_opr *)cmd.params;
+	cmd_params->tc_id = tc;
+	cmd_params->index = index;
+	cmd_params->options = options;
+	cmd_params->oloe = cfg->oloe;
+	cmd_params->oeane = cfg->oeane;
+	cmd_params->olws = cfg->olws;
+	cmd_params->oa = cfg->oa;
+	cmd_params->oprrws = cfg->oprrws;
+
+	/* send command to mc*/
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dpni_get_opr() - Retrieve Order Restoration config and query.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPNI object
+ * @tc:		Traffic class, in range 0 to NUM_TCS - 1
+ * @index:	Selects the specific queue out of the set allocated
+ *			for the same TC. Value must be in range 0 to
+ *			NUM_QUEUES - 1
+ * @cfg:	Returned OPR configuration
+ * @qry:	Returned OPR query
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dpni_get_opr(struct fsl_mc_io *mc_io,
+		 uint32_t cmd_flags,
+		 uint16_t token,
+		 uint8_t tc,
+		 uint8_t index,
+		 struct opr_cfg *cfg,
+		 struct opr_qry *qry)
+{
+	struct dpni_rsp_get_opr *rsp_params;
+	struct dpni_cmd_get_opr *cmd_params;
+	struct mc_command cmd = { 0 };
+	int err;
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_OPR,
+					  cmd_flags,
+					  token);
+	cmd_params = (struct dpni_cmd_get_opr *)cmd.params;
+	cmd_params->index = index;
+	cmd_params->tc_id = tc;
+
+	/* send command to mc*/
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	/* retrieve response parameters */
+	rsp_params = (struct dpni_rsp_get_opr *)cmd.params;
+	cfg->oloe = rsp_params->oloe;
+	cfg->oeane = rsp_params->oeane;
+	cfg->olws = rsp_params->olws;
+	cfg->oa = rsp_params->oa;
+	cfg->oprrws = rsp_params->oprrws;
+	qry->rip = dpni_get_field(rsp_params->flags, RIP);
+	qry->enable = dpni_get_field(rsp_params->flags, OPR_ENABLE);
+	qry->nesn = le16_to_cpu(rsp_params->nesn);
+	qry->ndsn = le16_to_cpu(rsp_params->ndsn);
+	qry->ea_tseq = le16_to_cpu(rsp_params->ea_tseq);
+	qry->tseq_nlis = dpni_get_field(rsp_params->tseq_nlis, TSEQ_NLIS);
+	qry->ea_hseq = le16_to_cpu(rsp_params->ea_hseq);
+	qry->hseq_nlis = dpni_get_field(rsp_params->hseq_nlis, HSEQ_NLIS);
+	qry->ea_hptr = le16_to_cpu(rsp_params->ea_hptr);
+	qry->ea_tptr = le16_to_cpu(rsp_params->ea_tptr);
+	qry->opr_vid = le16_to_cpu(rsp_params->opr_vid);
+	qry->opr_id = le16_to_cpu(rsp_params->opr_id);
+
+	return 0;
+}
diff --git a/drivers/net/dpaa2/mc/fsl_dpkg.h b/drivers/net/dpaa2/mc/fsl_dpkg.h
index 4de70f30..02fe8d50 100644
--- a/drivers/net/dpaa2/mc/fsl_dpkg.h
+++ b/drivers/net/dpaa2/mc/fsl_dpkg.h
@@ -71,45 +71,41 @@ struct dpkg_mask {
 /**
  * struct dpkg_extract - A structure for defining a single extraction
  * @type: Determines how the union below is interpreted:
- *		DPKG_EXTRACT_FROM_HDR: selects 'from_hdr';
- *		DPKG_EXTRACT_FROM_DATA: selects 'from_data';
- *		DPKG_EXTRACT_FROM_PARSE: selects 'from_parse'
+ *	DPKG_EXTRACT_FROM_HDR: selects 'from_hdr';
+ *	DPKG_EXTRACT_FROM_DATA: selects 'from_data';
+ *	DPKG_EXTRACT_FROM_PARSE: selects 'from_parse'
  * @extract: Selects extraction method
+ * @extract.from_hdr: Used when 'type = DPKG_EXTRACT_FROM_HDR'
+ * @extract.from_data: Used when 'type = DPKG_EXTRACT_FROM_DATA'
+ * @extract.from_parse:  Used when 'type = DPKG_EXTRACT_FROM_PARSE'
+ * @extract.from_hdr.prot: Any of the supported headers
+ * @extract.from_hdr.type: Defines the type of header extraction:
+ *	DPKG_FROM_HDR: use size & offset below;
+ *	DPKG_FROM_FIELD: use field, size and offset below;
+ *	DPKG_FULL_FIELD: use field below
+ * @extract.from_hdr.field: One of the supported fields (NH_FLD_)
+ * @extract.from_hdr.size: Size in bytes
+ * @extract.from_hdr.offset: Byte offset
+ * @extract.from_hdr.hdr_index: Clear for cases not listed below;
+ *	Used for protocols that may have more than a single
+ *	header, 0 indicates an outer header;
+ *	Supported protocols (possible values):
+ *	NET_PROT_VLAN (0, HDR_INDEX_LAST);
+ *	NET_PROT_MPLS (0, 1, HDR_INDEX_LAST);
+ *	NET_PROT_IP(0, HDR_INDEX_LAST);
+ *	NET_PROT_IPv4(0, HDR_INDEX_LAST);
+ *	NET_PROT_IPv6(0, HDR_INDEX_LAST);
+ * @extract.from_data.size: Size in bytes
+ * @extract.from_data.offset: Byte offset
+ * @extract.from_parse.size: Size in bytes
+ * @extract.from_parse.offset: Byte offset
  * @num_of_byte_masks: Defines the number of valid entries in the array below;
  *		This is	also the number of bytes to be used as masks
  * @masks: Masks parameters
  */
 struct dpkg_extract {
 	enum dpkg_extract_type type;
-	/**
-	 * union extract - Selects extraction method
-	 * @from_hdr - Used when 'type = DPKG_EXTRACT_FROM_HDR'
-	 * @from_data - Used when 'type = DPKG_EXTRACT_FROM_DATA'
-	 * @from_parse - Used when 'type = DPKG_EXTRACT_FROM_PARSE'
-	 */
 	union {
-		/**
-		 * struct from_hdr - Used when 'type = DPKG_EXTRACT_FROM_HDR'
-		 * @prot: Any of the supported headers
-		 * @type: Defines the type of header extraction:
-		 *	DPKG_FROM_HDR: use size & offset below;
-		 *	DPKG_FROM_FIELD: use field, size and offset below;
-		 *	DPKG_FULL_FIELD: use field below
-		 * @field: One of the supported fields (NH_FLD_)
-		 *
-		 * @size: Size in bytes
-		 * @offset: Byte offset
-		 * @hdr_index: Clear for cases not listed below;
-		 *	Used for protocols that may have more than a single
-		 *	header, 0 indicates an outer header;
-		 *	Supported protocols (possible values):
-		 *	NET_PROT_VLAN (0, HDR_INDEX_LAST);
-		 *	NET_PROT_MPLS (0, 1, HDR_INDEX_LAST);
-		 *	NET_PROT_IP(0, HDR_INDEX_LAST);
-		 *	NET_PROT_IPv4(0, HDR_INDEX_LAST);
-		 *	NET_PROT_IPv6(0, HDR_INDEX_LAST);
-		 */
-
 		struct {
 			enum net_prot prot;
 			enum dpkg_extract_from_hdr_type type;
@@ -118,23 +114,10 @@ struct dpkg_extract {
 			uint8_t offset;
 			uint8_t hdr_index;
 		} from_hdr;
-		/**
-		 * struct from_data
-		 *	Used when 'type = DPKG_EXTRACT_FROM_DATA'
-		 * @size: Size in bytes
-		 * @offset: Byte offset
-		 */
 		struct {
 			uint8_t size;
 			uint8_t offset;
 		} from_data;
-
-		/**
-		 * struct from_parse
-		 *	Used when 'type = DPKG_EXTRACT_FROM_PARSE'
-		 * @size: Size in bytes
-		 * @offset: Byte offset
-		 */
 		struct {
 			uint8_t size;
 			uint8_t offset;
diff --git a/drivers/net/dpaa2/mc/fsl_dpni.h b/drivers/net/dpaa2/mc/fsl_dpni.h
index f0edcd27..de1bcb5b 100644
--- a/drivers/net/dpaa2/mc/fsl_dpni.h
+++ b/drivers/net/dpaa2/mc/fsl_dpni.h
@@ -8,6 +8,7 @@
 #define __FSL_DPNI_H
 
 #include <fsl_dpkg.h>
+#include <fsl_dpopr.h>
 
 struct fsl_mc_io;
 
@@ -77,6 +78,20 @@ struct fsl_mc_io;
  */
 #define DPNI_OPT_NO_FS				0x000020
 
+/**
+ * Enable the Order Restoration support
+ */
+#define DPNI_OPT_HAS_OPR				0x000040
+
+/**
+ * Order Point Records are shared for the entire TC
+ */
+#define DPNI_OPT_OPR_PER_TC				0x000080
+/**
+ * All Tx traffic classes will use a single sender (ignore num_queueus for tx)
+ */
+#define DPNI_OPT_SINGLE_SENDER			0x000100
+
 int dpni_open(struct fsl_mc_io *mc_io,
 	      uint32_t cmd_flags,
 	      int dpni_id,
@@ -88,71 +103,74 @@ int dpni_close(struct fsl_mc_io *mc_io,
 
 /**
  * struct dpni_cfg - Structure representing DPNI configuration
- * @mac_addr:	Primary MAC address
- * @adv:	Advanced parameters; default is all zeros;
- *		use this structure to change default settings
+ * @options: Any combination of the following options:
+ *		DPNI_OPT_TX_FRM_RELEASE
+ *		DPNI_OPT_NO_MAC_FILTER
+ *		DPNI_OPT_HAS_POLICING
+ *		DPNI_OPT_SHARED_CONGESTION
+ *		DPNI_OPT_HAS_KEY_MASKING
+ *		DPNI_OPT_NO_FS
+ *		DPNI_OPT_SINGLE_SENDER
+ * @fs_entries: Number of entries in the flow steering table.
+ *		This table is used to select the ingress queue for
+ *		ingress traffic, targeting a GPP core or another.
+ *		In addition it can be used to discard traffic that
+ *		matches the set rule. It is either an exact match table
+ *		or a TCAM table, depending on DPNI_OPT_ HAS_KEY_MASKING
+ *		bit in OPTIONS field. This field is ignored if
+ *		DPNI_OPT_NO_FS bit is set in OPTIONS field. Otherwise,
+ *		value 0 defaults to 64. Maximum supported value is 1024.
+ *		Note that the total number of entries is limited on the
+ *		SoC to as low as 512 entries if TCAM is used.
+ * @vlan_filter_entries: Number of entries in the VLAN address filtering
+ *		table. This is an exact match table used to filter
+ *		ingress traffic based on VLAN IDs. Value 0 disables VLAN
+ *		filtering. Maximum supported value is 16.
+ * @mac_filter_entries: Number of entries in the MAC address filtering
+ *		table. This is an exact match table and allows both
+ *		unicast and multicast entries. The primary MAC address
+ *		of the network interface is not part of this table,
+ *		this contains only entries in addition to it. This
+ *		field is ignored if DPNI_OPT_ NO_MAC_FILTER is set in
+ *		OPTIONS field. Otherwise, value 0 defaults to 80.
+ *		Maximum supported value is 80.
+ * @num_queues: Number of Tx and Rx queues used for traffic
+ *		distribution. This is orthogonal to QoS and is only
+ *		used to distribute traffic to multiple GPP cores.
+ *		This configuration affects the number of Tx queues
+ *		(logical FQs, all associated with a single CEETM queue),
+ *		Rx queues and Tx confirmation queues, if applicable.
+ *		Value 0 defaults to one queue. Maximum supported value
+ *		is 8.
+ * @num_tcs: Number of traffic classes (TCs), reserved for the DPNI.
+ *		TCs can have different priority levels for the purpose
+ *		of Tx scheduling (see DPNI_SET_TX_PRIORITIES), different
+ *		BPs (DPNI_ SET_POOLS), policers. There are dedicated QM
+ *		queues for traffic classes (including class queues on
+ *		Tx). Value 0 defaults to one TC. Maximum supported value
+ *		is 16. There are maximum 16 TCs for Tx and 8 TCs for Rx.
+ *		When num_tcs>8 Tx will use this value but Rx will have
+ *		only 8 traffic classes.
+ * @num_rx_tcs: if set to other value than zero represents number
+ *		of TCs used for Rx. Maximum value is 8. If set to zero the
+ *		number of Rx TCs will be initialized with the value provided
+ *		in num_tcs parameter.
+ * @qos_entries: Number of entries in the QoS classification table. This
+ *		table is used to select the TC for ingress traffic. It
+ *		is either an exact match or a TCAM table, depending on
+ *		DPNI_OPT_ HAS_KEY_MASKING bit in OPTIONS field. This
+ *		field is ignored if the DPNI has a single TC. Otherwise,
+ *		a value of 0 defaults to 64. Maximum supported value
+ *		is 64.
  */
 struct dpni_cfg {
-	/**
-	 * @options: Any combination of the following options:
-	 *		DPNI_OPT_TX_FRM_RELEASE
-	 *		DPNI_OPT_NO_MAC_FILTER
-	 *		DPNI_OPT_HAS_POLICING
-	 *		DPNI_OPT_SHARED_CONGESTION
-	 *		DPNI_OPT_HAS_KEY_MASKING
-	 *		DPNI_OPT_NO_FS
-	 * @fs_entries: Number of entries in the flow steering table.
-	 *		This table is used to select the ingress queue for
-	 *		ingress traffic, targeting a GPP core or another.
-	 *		In addition it can be used to discard traffic that
-	 *		matches the set rule. It is either an exact match table
-	 *		or a TCAM table, depending on DPNI_OPT_ HAS_KEY_MASKING
-	 *		bit in OPTIONS field. This field is ignored if
-	 *		DPNI_OPT_NO_FS bit is set in OPTIONS field. Otherwise,
-	 *		value 0 defaults to 64. Maximum supported value is 1024.
-	 *		Note that the total number of entries is limited on the
-	 *		SoC to as low as 512 entries if TCAM is used.
-	 * @vlan_filter_entries: Number of entries in the VLAN address filtering
-	 *		table. This is an exact match table used to filter
-	 *		ingress traffic based on VLAN IDs. Value 0 disables VLAN
-	 *		filtering. Maximum supported value is 16.
-	 * @mac_filter_entries: Number of entries in the MAC address filtering
-	 *		table. This is an exact match table and allows both
-	 *		unicast and multicast entries. The primary MAC address
-	 *		of the network interface is not part of this table,
-	 *		this contains only entries in addition to it. This
-	 *		field is ignored if DPNI_OPT_ NO_MAC_FILTER is set in
-	 *		OPTIONS field. Otherwise, value 0 defaults to 80.
-	 *		Maximum supported value is 80.
-	 * @num_queues: Number of Tx and Rx queues used for traffic
-	 *		distribution. This is orthogonal to QoS and is only
-	 *		used to distribute traffic to multiple GPP cores.
-	 *		This configuration affects the number of Tx queues
-	 *		(logical FQs, all associated with a single CEETM queue),
-	 *		Rx queues and Tx confirmation queues, if applicable.
-	 *		Value 0 defaults to one queue. Maximum supported value
-	 *		is 8.
-	 * @num_tcs: Number of traffic classes (TCs), reserved for the DPNI.
-	 *		TCs can have different priority levels for the purpose
-	 *		of Tx scheduling (see DPNI_SET_TX_SELECTION), different
-	 *		BPs (DPNI_ SET_POOLS), policers. There are dedicated QM
-	 *		queues for traffic classes (including class queues on
-	 *		Tx). Value 0 defaults to one TC. Maximum supported value
-	 *		is 8.
-	 * @qos_entries: Number of entries in the QoS classification table. This
-	 *		table is used to select the TC for ingress traffic. It
-	 *		is either an exact match or a TCAM table, depending on
-	 *		DPNI_OPT_ HAS_KEY_MASKING bit in OPTIONS field. This
-	 *		field is ignored if the DPNI has a single TC. Otherwise,
-	 *		a value of 0 defaults to 64. Maximum supported value
-	 *		is 64.
-	 */
 	uint32_t options;
 	uint16_t fs_entries;
 	uint8_t  vlan_filter_entries;
 	uint8_t  mac_filter_entries;
 	uint8_t  num_queues;
 	uint8_t  num_tcs;
+	uint8_t  num_rx_tcs;
 	uint8_t  qos_entries;
 };
 
@@ -172,17 +190,14 @@ int dpni_destroy(struct fsl_mc_io *mc_io,
  * @num_dpbp:	Number of DPBPs
  * @pools:	Array of buffer pools parameters; The number of valid entries
  *		must match 'num_dpbp' value
+ * @pools.dpbp_id:     DPBP object ID
+ * @pools.priority:    Priority mask that indicates TC's used with this buffer.
+ *		       I set to 0x00 MC will assume value 0xff.
+ * @pools.buffer_size: Buffer size
+ * @pools.backup_pool: Backup pool
  */
 struct dpni_pools_cfg {
 	uint8_t num_dpbp;
-	/**
-	 * struct pools - Buffer pools parameters
-	 * @dpbp_id: DPBP object ID
-	 * @priority: priority mask that indicates TC's used with this buffer.
-	 * I set to 0x00 MC will assume value 0xff.
-	 * @buffer_size: Buffer size
-	 * @backup_pool: Backup pool
-	 */
 	struct {
 		int		dpbp_id;
 		uint8_t		priority_mask;
@@ -296,6 +311,8 @@ int dpni_clear_irq_status(struct fsl_mc_io *mc_io,
  *			variants,
  *			- 0x422 - WRIOP version 1.1.2, used on LS1088 and
  *			variants.
+ *			- 0xC00 - WRIOP version 3.0.0, used on LX2160 and
+ *			variants.
  */
 struct dpni_attr {
 	uint32_t options;
@@ -321,6 +338,13 @@ int dpni_get_attributes(struct fsl_mc_io *mc_io,
  */
 
 /**
+ * Discard error. When set all discarded frames in wriop will be enqueued to
+ * error queue. To be used in dpni_set_errors_behavior() only if error_action
+ * parameter is set to DPNI_ERROR_ACTION_SEND_TO_ERROR_QUEUE.
+ */
+#define DPNI_ERROR_DISC		0x80000000
+
+/**
  * Extract out of frame header error
  */
 #define DPNI_ERROR_EOFHE	0x00020000
@@ -408,6 +432,10 @@ int dpni_set_errors_behavior(struct fsl_mc_io *mc_io,
  * Select to modify the data-tail-room setting
  */
 #define DPNI_BUF_LAYOUT_OPT_DATA_TAIL_ROOM	0x00000040
+/**
+ * Select to modify the sw-opaque value setting
+ */
+#define DPNI_BUF_LAYOUT_OPT_SW_OPAQUE		0x00000080
 
 /**
  * struct dpni_buffer_layout - Structure representing DPNI buffer layout
@@ -427,6 +455,7 @@ struct dpni_buffer_layout {
 	int pass_timestamp;
 	int pass_parser_result;
 	int pass_frame_status;
+	int pass_sw_opaque;
 	uint16_t private_data_size;
 	uint16_t data_align;
 	uint16_t data_head_room;
@@ -501,16 +530,48 @@ int dpni_get_tx_data_offset(struct fsl_mc_io *mc_io,
 
 #define DPNI_STATISTICS_CNT		7
 
+/**
+ * union dpni_statistics - Union describing the DPNI statistics
+ * @page_0: Page_0 statistics structure
+ * @page_0.ingress_all_frames: Ingress frame count
+ * @page_0.ingress_all_bytes: Ingress byte count
+ * @page_0.ingress_multicast_frames: Ingress multicast frame count
+ * @page_0.ingress_multicast_bytes: Ingress multicast byte count
+ * @page_0.ingress_broadcast_frames: Ingress broadcast frame count
+ * @page_0.ingress_broadcast_bytes: Ingress broadcast byte count
+ * @page_1: Page_1 statistics structure
+ * @page_1.egress_all_frames: Egress frame count
+ * @page_1.egress_all_bytes: Egress byte count
+ * @page_1.egress_multicast_frames: Egress multicast frame count
+ * @page_1.egress_multicast_bytes: Egress multicast byte count
+ * @page_1.egress_broadcast_frames: Egress broadcast frame count
+ * @page_1.egress_broadcast_bytes: Egress broadcast byte count
+ * @page_2: Page_2 statistics structure
+ * @page_2.ingress_filtered_frames: Ingress filtered frame count
+ * @page_2.ingress_discarded_frames: Ingress discarded frame count
+ * @page_2.ingress_nobuffer_discards: Ingress discarded frame count due to
+ *	lack of buffers
+ * @page_2.egress_discarded_frames: Egress discarded frame count
+ * @page_2.egress_confirmed_frames: Egress confirmed frame count
+ * @page_3: Page_3 statistics structure with values for the selected TC
+ * @page_3.ceetm_dequeue_bytes: Cumulative count of the number of bytes dequeued
+ * @page_3.ceetm_dequeue_frames: Cumulative count of the number of frames
+ *	dequeued
+ * @page_3.ceetm_reject_bytes: Cumulative count of the number of bytes in all
+ *	frames whose enqueue was rejected
+ * @page_3.ceetm_reject_frames: Cumulative count of all frame enqueues rejected
+ * @page_4: congestion point drops for seleted TC
+ * @page_4.cgr_reject_frames: number of rejected frames due to congestion point
+ * @page_4.cgr_reject_bytes: number of rejected bytes due to congestion point
+ * @page_5: policer statistics per TC
+ * @page_5.policer_cnt_red: NUmber of red colored frames
+ * @page_5.policer_cnt_yellow: number of yellow colored frames
+ * @page_5.policer_cnt_green: number of green colored frames
+ * @page_5.policer_cnt_re_red: number of recolored red frames
+ * @page_5.policer_cnt_re_yellow: number of recolored yellow frames
+ * @raw: raw statistics structure, used to index counters
+ */
 union dpni_statistics {
-	/**
-	 * struct page_0 - Page_0 statistics structure
-	 * @ingress_all_frames: Ingress frame count
-	 * @ingress_all_bytes: Ingress byte count
-	 * @ingress_multicast_frames: Ingress multicast frame count
-	 * @ingress_multicast_bytes: Ingress multicast byte count
-	 * @ingress_broadcast_frames: Ingress broadcast frame count
-	 * @ingress_broadcast_bytes: Ingress broadcast byte count
-	 */
 	struct {
 		uint64_t ingress_all_frames;
 		uint64_t ingress_all_bytes;
@@ -519,15 +580,6 @@ union dpni_statistics {
 		uint64_t ingress_broadcast_frames;
 		uint64_t ingress_broadcast_bytes;
 	} page_0;
-	/**
-	 * struct page_1 - Page_1 statistics structure
-	 * @egress_all_frames: Egress frame count
-	 * @egress_all_bytes: Egress byte count
-	 * @egress_multicast_frames: Egress multicast frame count
-	 * @egress_multicast_bytes: Egress multicast byte count
-	 * @egress_broadcast_frames: Egress broadcast frame count
-	 * @egress_broadcast_bytes: Egress broadcast byte count
-	 */
 	struct {
 		uint64_t egress_all_frames;
 		uint64_t egress_all_bytes;
@@ -536,15 +588,6 @@ union dpni_statistics {
 		uint64_t egress_broadcast_frames;
 		uint64_t egress_broadcast_bytes;
 	} page_1;
-	/**
-	 * struct page_2 - Page_2 statistics structure
-	 * @ingress_filtered_frames: Ingress filtered frame count
-	 * @ingress_discarded_frames: Ingress discarded frame count
-	 * @ingress_nobuffer_discards: Ingress discarded frame count due to
-	 *					lack of buffers
-	 * @egress_discarded_frames: Egress discarded frame count
-	 * @egress_confirmed_frames: Egress confirmed frame count
-	 */
 	struct {
 		uint64_t ingress_filtered_frames;
 		uint64_t ingress_discarded_frames;
@@ -552,26 +595,23 @@ union dpni_statistics {
 		uint64_t egress_discarded_frames;
 		uint64_t egress_confirmed_frames;
 	} page_2;
-	/**
-	 * struct page_3 - Page_3 statistics structure with values for the
-	 *			selected TC
-	 * @ceetm_dequeue_bytes: Cumulative count of the number of bytes
-	 *			dequeued
-	 * @ceetm_dequeue_frames: Cumulative count of the number of frames
-	 *			dequeued
-	 * @ceetm_reject_bytes: Cumulative count of the number of bytes in all
-	 *			frames whose enqueue was rejected
-	 * @ceetm_reject_frames: Cumulative count of all frame enqueues rejected
-	 */
 	struct {
 		uint64_t ceetm_dequeue_bytes;
 		uint64_t ceetm_dequeue_frames;
 		uint64_t ceetm_reject_bytes;
 		uint64_t ceetm_reject_frames;
 	} page_3;
-	/**
-	 * struct raw - raw statistics structure, used to index counters
-	 */
+	struct {
+		uint64_t cgr_reject_frames;
+		uint64_t cgr_reject_bytes;
+	} page_4;
+	struct {
+		uint64_t policer_cnt_red;
+		uint64_t policer_cnt_yellow;
+		uint64_t policer_cnt_green;
+		uint64_t policer_cnt_re_red;
+		uint64_t policer_cnt_re_yellow;
+	} page_5;
 	struct {
 		uint64_t counter[DPNI_STATISTICS_CNT];
 	} raw;
@@ -602,10 +642,12 @@ union dpni_statistics {
  * struct - Structure representing DPNI link configuration
  * @rate: Rate
  * @options: Mask of available options; use 'DPNI_LINK_OPT_<X>' values
+ * @advertising: Speeds that are advertised for autoneg (bitmap)
  */
 struct dpni_link_cfg {
 	uint32_t rate;
 	uint64_t options;
+	uint64_t advertising;
 };
 
 int dpni_set_link_cfg(struct fsl_mc_io *mc_io,
@@ -618,11 +660,17 @@ int dpni_set_link_cfg(struct fsl_mc_io *mc_io,
  * @rate:	Rate
  * @options:	Mask of available options; use 'DPNI_LINK_OPT_<X>' values
  * @up:		Link state; '0' for down, '1' for up
+ * @state_valid: Ignore/Update the state of the link
+ * @supported: Speeds capability of the phy (bitmap)
+ * @advertising: Speeds that are advertised for autoneg (bitmap)
  */
 struct dpni_link_state {
 	uint32_t rate;
 	uint64_t options;
 	int up;
+	int     state_valid;
+	uint64_t supported;
+	uint64_t advertising;
 };
 
 int dpni_get_link_state(struct fsl_mc_io *mc_io,
@@ -750,11 +798,20 @@ enum dpni_fs_miss_action {
  * struct dpni_fs_tbl_cfg - Flow Steering table configuration
  * @miss_action:	Miss action selection
  * @default_flow_id:	Used when 'miss_action = DPNI_FS_MISS_EXPLICIT_FLOWID'
+ * @keep_hash_key: used only when miss_action is set to DPNI_FS_MISS_HASH. When
+ *	set to one unclassified frames will be distributed according to previous
+ *	used hash key. If set to zero hash key will be replaced with the key
+ *	provided for flow steering.
+ * @keep_entries: if set to one command will not delete the entries that already
+ *	exist into FS table. Use this option with caution: if the table
+ *	entries	are not compatible with the distribution key the packets
+ *	will not be classified properly.
  */
 struct dpni_fs_tbl_cfg {
 	enum dpni_fs_miss_action miss_action;
 	uint16_t default_flow_id;
 	char keep_hash_key;
+	uint8_t keep_entries;
 };
 
 /**
@@ -915,34 +972,52 @@ int dpni_get_congestion_notification(struct fsl_mc_io *mc_io,
 
 /**
  * struct dpni_queue - Queue structure
- * @user_context:	User data, presented to the user along with any frames
- *			from this queue. Not relevant for Tx queues.
+ * @destination - Destination structure
+ * @destination.id: ID of the destination, only relevant if DEST_TYPE is > 0.
+ *	Identifies either a DPIO or a DPCON object.
+ *	Not relevant for Tx queues.
+ * @destination.type:	May be one of the following:
+ *	0 - No destination, queue can be manually
+ *		queried, but will not push traffic or
+ *		notifications to a DPIO;
+ *	1 - The destination is a DPIO. When traffic
+ *		becomes available in the queue a FQDAN
+ *		(FQ data available notification) will be
+ *		generated to selected DPIO;
+ *	2 - The destination is a DPCON. The queue is
+ *		associated with a DPCON object for the
+ *		purpose of scheduling between multiple
+ *		queues. The DPCON may be independently
+ *		configured to generate notifications.
+ *		Not relevant for Tx queues.
+ * @destination.hold_active: Hold active, maintains a queue scheduled for longer
+ *	in a DPIO during dequeue to reduce spread of traffic.
+ *	Only relevant if queues are
+ *	not affined to a single DPIO.
+ * @user_context: User data, presented to the user along with any frames
+ *	from this queue. Not relevant for Tx queues.
+ * @flc: FD FLow Context structure
+ * @flc.value: Default FLC value for traffic dequeued from
+ *      this queue.  Please check description of FD
+ *      structure for more information.
+ *      Note that FLC values set using dpni_add_fs_entry,
+ *      if any, take precedence over values per queue.
+ * @flc.stash_control: Boolean, indicates whether the 6 lowest
+ *      - significant bits are used for stash control.
+ *      significant bits are used for stash control.  If set, the 6
+ *      least significant bits in value are interpreted as follows:
+ *      - bits 0-1: indicates the number of 64 byte units of context
+ *      that are stashed.  FLC value is interpreted as a memory address
+ *      in this case, excluding the 6 LS bits.
+ *      - bits 2-3: indicates the number of 64 byte units of frame
+ *      annotation to be stashed.  Annotation is placed at FD[ADDR].
+ *      - bits 4-5: indicates the number of 64 byte units of frame
+ *      data to be stashed.  Frame data is placed at FD[ADDR] +
+ *      FD[OFFSET].
+ *      For more details check the Frame Descriptor section in the
+ *      hardware documentation.
  */
 struct dpni_queue {
-	/**
-	 * struct destination - Destination structure
-	 * @id:	ID of the destination, only relevant if DEST_TYPE is > 0.
-	 *			Identifies either a DPIO or a DPCON object.
-	 *			Not relevant for Tx queues.
-	 * @type:	May be one of the following:
-	 *			0 - No destination, queue can be manually
-	 *				queried, but will not push traffic or
-	 *				notifications to a DPIO;
-	 *			1 - The destination is a DPIO. When traffic
-	 *				becomes available in the queue a FQDAN
-	 *				(FQ data available notification) will be
-	 *				generated to selected DPIO;
-	 *			2 - The destination is a DPCON. The queue is
-	 *				associated with a DPCON object for the
-	 *				purpose of scheduling between multiple
-	 *				queues. The DPCON may be independently
-	 *				configured to generate notifications.
-	 *				Not relevant for Tx queues.
-	 * @hold_active: Hold active, maintains a queue scheduled for longer
-	 *		in a DPIO during dequeue to reduce spread of traffic.
-	 *		Only relevant if queues are
-	 *		not affined to a single DPIO.
-	 */
 	struct {
 		uint16_t id;
 		enum dpni_dest type;
@@ -950,28 +1025,6 @@ struct dpni_queue {
 		uint8_t priority;
 	} destination;
 	uint64_t user_context;
-	/**
-	 * struct flc - FD FLow Context structure
-	 * @value: Default FLC value for traffic dequeued from
-	 *      this queue.  Please check description of FD
-	 *      structure for more information.
-	 *      Note that FLC values set using dpni_add_fs_entry,
-	 *      if any, take precedence over values per queue.
-	 * @stash_control: Boolean, indicates whether the 6 lowest
-	 *      - significant bits are used for stash control.
-	 *      significant bits are used for stash control.  If set, the 6
-	 *      least significant bits in value are interpreted as follows:
-	 *      - bits 0-1: indicates the number of 64 byte units of context
-	 *      that are stashed.  FLC value is interpreted as a memory address
-	 *      in this case, excluding the 6 LS bits.
-	 *      - bits 2-3: indicates the number of 64 byte units of frame
-	 *      annotation to be stashed.  Annotation is placed at FD[ADDR].
-	 *      - bits 4-5: indicates the number of 64 byte units of frame
-	 *      data to be stashed.  Frame data is placed at FD[ADDR] +
-	 *      FD[OFFSET].
-	 *      For more details check the Frame Descriptor section in the
-	 *      hardware documentation.
-	 */
 	struct {
 		uint64_t value;
 		char stash_control;
@@ -1132,4 +1185,21 @@ int dpni_get_taildrop(struct fsl_mc_io *mc_io,
 		      uint8_t tc,
 		      uint8_t q_index,
 		      struct dpni_taildrop *taildrop);
+
+int dpni_set_opr(struct fsl_mc_io *mc_io,
+		 uint32_t cmd_flags,
+		 uint16_t token,
+		 uint8_t tc,
+		 uint8_t index,
+		 uint8_t options,
+		 struct opr_cfg *cfg);
+
+int dpni_get_opr(struct fsl_mc_io *mc_io,
+		 uint32_t cmd_flags,
+		 uint16_t token,
+		 uint8_t tc,
+		 uint8_t index,
+		 struct opr_cfg *cfg,
+		 struct opr_qry *qry);
+
 #endif /* __FSL_DPNI_H */
diff --git a/drivers/net/dpaa2/mc/fsl_dpni_cmd.h b/drivers/net/dpaa2/mc/fsl_dpni_cmd.h
index eb3e9987..3df5bcf1 100644
--- a/drivers/net/dpaa2/mc/fsl_dpni_cmd.h
+++ b/drivers/net/dpaa2/mc/fsl_dpni_cmd.h
@@ -9,19 +9,21 @@
 
 /* DPNI Version */
 #define DPNI_VER_MAJOR				7
-#define DPNI_VER_MINOR				3
+#define DPNI_VER_MINOR				8
 
 #define DPNI_CMD_BASE_VERSION			1
 #define DPNI_CMD_VERSION_2			2
+#define DPNI_CMD_VERSION_3			3
 #define DPNI_CMD_ID_OFFSET			4
 
 #define DPNI_CMD(id)	(((id) << DPNI_CMD_ID_OFFSET) | DPNI_CMD_BASE_VERSION)
 #define DPNI_CMD_V2(id)	(((id) << DPNI_CMD_ID_OFFSET) | DPNI_CMD_VERSION_2)
+#define DPNI_CMD_V3(id)	(((id) << DPNI_CMD_ID_OFFSET) | DPNI_CMD_VERSION_3)
 
 /* Command IDs */
 #define DPNI_CMDID_OPEN				DPNI_CMD(0x801)
 #define DPNI_CMDID_CLOSE			DPNI_CMD(0x800)
-#define DPNI_CMDID_CREATE			DPNI_CMD(0x901)
+#define DPNI_CMDID_CREATE			DPNI_CMD_V2(0x901)
 #define DPNI_CMDID_DESTROY			DPNI_CMD(0x981)
 #define DPNI_CMDID_GET_API_VERSION		DPNI_CMD(0xa01)
 
@@ -44,10 +46,10 @@
 #define DPNI_CMDID_GET_QDID			DPNI_CMD(0x210)
 #define DPNI_CMDID_GET_SP_INFO			DPNI_CMD(0x211)
 #define DPNI_CMDID_GET_TX_DATA_OFFSET		DPNI_CMD(0x212)
-#define DPNI_CMDID_GET_LINK_STATE		DPNI_CMD(0x215)
+#define DPNI_CMDID_GET_LINK_STATE		DPNI_CMD_V2(0x215)
 #define DPNI_CMDID_SET_MAX_FRAME_LENGTH		DPNI_CMD(0x216)
 #define DPNI_CMDID_GET_MAX_FRAME_LENGTH		DPNI_CMD(0x217)
-#define DPNI_CMDID_SET_LINK_CFG			DPNI_CMD(0x21A)
+#define DPNI_CMDID_SET_LINK_CFG			DPNI_CMD_V2(0x21A)
 #define DPNI_CMDID_SET_TX_SHAPING		DPNI_CMD_V2(0x21B)
 
 #define DPNI_CMDID_SET_MCAST_PROMISC		DPNI_CMD(0x220)
@@ -65,7 +67,7 @@
 #define DPNI_CMDID_REMOVE_VLAN_ID		DPNI_CMD(0x232)
 #define DPNI_CMDID_CLR_VLAN_FILTERS		DPNI_CMD(0x233)
 
-#define DPNI_CMDID_SET_RX_TC_DIST		DPNI_CMD_V2(0x235)
+#define DPNI_CMDID_SET_RX_TC_DIST		DPNI_CMD_V3(0x235)
 
 #define DPNI_CMDID_GET_STATISTICS		DPNI_CMD_V2(0x25D)
 #define DPNI_CMDID_RESET_STATISTICS		DPNI_CMD(0x25E)
@@ -76,8 +78,8 @@
 
 #define DPNI_CMDID_GET_PORT_MAC_ADDR		DPNI_CMD(0x263)
 
-#define DPNI_CMDID_GET_BUFFER_LAYOUT		DPNI_CMD(0x264)
-#define DPNI_CMDID_SET_BUFFER_LAYOUT		DPNI_CMD(0x265)
+#define DPNI_CMDID_GET_BUFFER_LAYOUT		DPNI_CMD_V2(0x264)
+#define DPNI_CMDID_SET_BUFFER_LAYOUT		DPNI_CMD_V2(0x265)
 
 #define DPNI_CMDID_SET_CONGESTION_NOTIFICATION	DPNI_CMD(0x267)
 #define DPNI_CMDID_GET_CONGESTION_NOTIFICATION	DPNI_CMD(0x268)
@@ -87,6 +89,8 @@
 #define DPNI_CMDID_SET_OFFLOAD			DPNI_CMD(0x26C)
 #define DPNI_CMDID_SET_TX_CONFIRMATION_MODE	DPNI_CMD(0x266)
 #define DPNI_CMDID_GET_TX_CONFIRMATION_MODE	DPNI_CMD(0x26D)
+#define DPNI_CMDID_SET_OPR			DPNI_CMD(0x26e)
+#define DPNI_CMDID_GET_OPR			DPNI_CMD(0x26f)
 
 /* Macros for accessing command fields smaller than 1byte */
 #define DPNI_MASK(field)	\
@@ -113,6 +117,7 @@ struct dpni_cmd_create {
 	uint8_t qos_entries;
 	uint8_t pad3;
 	uint16_t fs_entries;
+	uint8_t num_rx_tcs;
 };
 
 struct dpni_cmd_destroy {
@@ -228,6 +233,8 @@ struct dpni_cmd_set_errors_behavior {
 #define DPNI_PASS_PR_SIZE		1
 #define DPNI_PASS_FS_SHIFT		2
 #define DPNI_PASS_FS_SIZE		1
+#define DPNI_PASS_SWO_SHIFT		3
+#define DPNI_PASS_SWO_SIZE		1
 
 struct dpni_cmd_get_buffer_layout {
 	uint8_t qtype;
@@ -307,10 +314,13 @@ struct dpni_cmd_set_link_cfg {
 	uint32_t rate;
 	uint32_t pad1;
 	uint64_t options;
+	uint64_t advertising;
 };
 
 #define DPNI_LINK_STATE_SHIFT		0
 #define DPNI_LINK_STATE_SIZE		1
+#define DPNI_STATE_VALID_SHIFT		1
+#define DPNI_STATE_VALID_SIZE		1
 
 struct dpni_rsp_get_link_state {
 	uint32_t pad0;
@@ -320,6 +330,8 @@ struct dpni_rsp_get_link_state {
 	uint32_t rate;
 	uint32_t pad2;
 	uint64_t options;
+	uint64_t supported;
+	uint64_t advertising;
 };
 
 struct dpni_cmd_set_max_frame_length {
@@ -415,6 +427,8 @@ struct dpni_cmd_set_tx_priorities {
 #define DPNI_MISS_ACTION_SIZE		4
 #define DPNI_KEEP_HASH_KEY_SHIFT	7
 #define DPNI_KEEP_HASH_KEY_SIZE		1
+#define DPNI_KEEP_ENTRIES_SHIFT		6
+#define DPNI_KEEP_ENTRIES_SIZE		1
 
 struct dpni_cmd_set_rx_tc_dist {
 	uint16_t dist_size;
@@ -601,5 +615,64 @@ struct dpni_rsp_get_congestion_notification {
 	uint32_t threshold_exit;
 };
 
+struct dpni_cmd_set_opr {
+	uint8_t pad0;
+	uint8_t tc_id;
+	uint8_t index;
+	uint8_t options;
+	uint8_t pad1[7];
+	uint8_t oloe;
+	uint8_t oeane;
+	uint8_t olws;
+	uint8_t oa;
+	uint8_t oprrws;
+};
+
+struct dpni_cmd_get_opr {
+	uint8_t pad;
+	uint8_t tc_id;
+	uint8_t index;
+};
+
+#define DPNI_RIP_SHIFT	0
+#define DPNI_RIP_SIZE		1
+#define DPNI_OPR_ENABLE_SHIFT	1
+#define DPNI_OPR_ENABLE_SIZE	1
+#define DPNI_TSEQ_NLIS_SHIFT	0
+#define DPNI_TSEQ_NLIS_SIZE	1
+#define DPNI_HSEQ_NLIS_SHIFT	0
+#define DPNI_HSEQ_NLIS_SIZE	1
+
+struct dpni_rsp_get_opr {
+	uint64_t pad0;
+	/* from LSB: rip:1 enable:1 */
+	uint8_t flags;
+	uint16_t pad1;
+	uint8_t oloe;
+	uint8_t oeane;
+	uint8_t olws;
+	uint8_t oa;
+	uint8_t oprrws;
+	uint16_t nesn;
+	uint16_t pad8;
+	uint16_t ndsn;
+	uint16_t pad2;
+	uint16_t ea_tseq;
+	/* only the LSB */
+	uint8_t tseq_nlis;
+	uint8_t pad3;
+	uint16_t ea_hseq;
+	/* only the LSB */
+	uint8_t hseq_nlis;
+	uint8_t pad4;
+	uint16_t ea_hptr;
+	uint16_t pad5;
+	uint16_t ea_tptr;
+	uint16_t pad6;
+	uint16_t opr_vid;
+	uint16_t pad7;
+	uint16_t opr_id;
+};
+
 #pragma pack(pop)
 #endif /* _FSL_DPNI_CMD_H */
diff --git a/drivers/net/dpaa2/mc/fsl_net.h b/drivers/net/dpaa2/mc/fsl_net.h
index 964870ba..0dc0131b 100644
--- a/drivers/net/dpaa2/mc/fsl_net.h
+++ b/drivers/net/dpaa2/mc/fsl_net.h
@@ -180,7 +180,7 @@
 #define NH_FLD_SCTP_CHUNK_DATA_STREAM_SQN     (NH_FLD_SCTP_CHUNK_DATA_TYPE << 5)
 #define NH_FLD_SCTP_CHUNK_DATA_PAYLOAD_PID    (NH_FLD_SCTP_CHUNK_DATA_TYPE << 6)
 #define NH_FLD_SCTP_CHUNK_DATA_UNORDERED      (NH_FLD_SCTP_CHUNK_DATA_TYPE << 7)
-#define NH_FLD_SCTP_CHUNK_DATA_BEGGINNING     (NH_FLD_SCTP_CHUNK_DATA_TYPE << 8)
+#define NH_FLD_SCTP_CHUNK_DATA_BEGGINING      (NH_FLD_SCTP_CHUNK_DATA_TYPE << 8)
 #define NH_FLD_SCTP_CHUNK_DATA_END            (NH_FLD_SCTP_CHUNK_DATA_TYPE << 9)
 #define NH_FLD_SCTP_CHUNK_DATA_ALL_FIELDS \
 	((NH_FLD_SCTP_CHUNK_DATA_TYPE << 10) - 1)
diff --git a/drivers/net/dpaa2/meson.build b/drivers/net/dpaa2/meson.build
index 213f0d72..b3459525 100644
--- a/drivers/net/dpaa2/meson.build
+++ b/drivers/net/dpaa2/meson.build
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright 2018 NXP
 
+version = 2
+
 if host_machine.system() != 'linux'
         build = false
 endif
diff --git a/drivers/net/e1000/Makefile b/drivers/net/e1000/Makefile
index 9c87e883..0ed62765 100644
--- a/drivers/net/e1000/Makefile
+++ b/drivers/net/e1000/Makefile
@@ -10,6 +10,7 @@ LIB = librte_pmd_e1000.a
 
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
diff --git a/drivers/net/e1000/base/e1000_82571.c b/drivers/net/e1000/base/e1000_82571.c
index 7c279dbb..397dd948 100644
--- a/drivers/net/e1000/base/e1000_82571.c
+++ b/drivers/net/e1000/base/e1000_82571.c
@@ -1257,6 +1257,11 @@ STATIC s32 e1000_init_hw_82571(struct e1000_hw *hw)
 	 */
 	e1000_clear_hw_cntrs_82571(hw);
 
+	/* MSI-X configure for 82574 */
+	if (mac->type == e1000_82574)
+		E1000_WRITE_REG(hw, E1000_IVAR,
+				(E1000_IVAR_INT_ALLOC_VALID << 16));
+
 	return ret_val;
 }
 
diff --git a/drivers/net/e1000/base/e1000_osdep.h b/drivers/net/e1000/base/e1000_osdep.h
index b8868049..5958ea15 100644
--- a/drivers/net/e1000/base/e1000_osdep.h
+++ b/drivers/net/e1000/base/e1000_osdep.h
@@ -48,7 +48,7 @@
 
 #include "../e1000_logs.h"
 
-#define DELAY(x) rte_delay_us(x)
+#define DELAY(x) rte_delay_us_sleep(x)
 #define usec_delay(x) DELAY(x)
 #define usec_delay_irq(x) DELAY(x)
 #define msec_delay(x) DELAY(1000*(x))
diff --git a/drivers/net/e1000/base/meson.build b/drivers/net/e1000/base/meson.build
index 5e1716de..f26f2429 100644
--- a/drivers/net/e1000/base/meson.build
+++ b/drivers/net/e1000/base/meson.build
@@ -25,6 +25,9 @@ error_cflags = ['-Wno-uninitialized', '-Wno-unused-parameter',
 	'-Wno-unused-variable', '-Wno-misleading-indentation',
 	'-Wno-implicit-fallthrough']
 c_args = cflags
+if allow_experimental_apis
+	c_args += '-DALLOW_EXPERIMENTAL_API'
+endif
 foreach flag: error_cflags
 	if cc.has_argument(flag)
 		c_args += flag
diff --git a/drivers/net/e1000/e1000_ethdev.h b/drivers/net/e1000/e1000_ethdev.h
index 902001f3..94edff08 100644
--- a/drivers/net/e1000/e1000_ethdev.h
+++ b/drivers/net/e1000/e1000_ethdev.h
@@ -236,7 +236,8 @@ struct igb_ethertype_filter {
 struct igb_rte_flow_rss_conf {
 	struct rte_flow_action_rss conf; /**< RSS parameters. */
 	uint8_t key[IGB_HKEY_MAX_INDEX * sizeof(uint32_t)]; /* Hash key. */
-	uint16_t queue[IGB_MAX_RX_QUEUE_NUM]; /**< Queues indices to use. */
+	/* Queues indices to use. */
+	uint16_t queue[IGB_MAX_RX_QUEUE_NUM_82576];
 };
 
 /*
@@ -506,7 +507,8 @@ int eth_igb_syn_filter_set(struct rte_eth_dev *dev,
 int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
 			struct rte_eth_flex_filter *filter,
 			bool add);
-int igb_rss_conf_init(struct igb_rte_flow_rss_conf *out,
+int igb_rss_conf_init(struct rte_eth_dev *dev,
+		      struct igb_rte_flow_rss_conf *out,
 		      const struct rte_flow_action_rss *in);
 int igb_action_rss_same(const struct rte_flow_action_rss *comp,
 			const struct rte_flow_action_rss *with);
diff --git a/drivers/net/e1000/em_ethdev.c b/drivers/net/e1000/em_ethdev.c
index 053e855b..8230824e 100644
--- a/drivers/net/e1000/em_ethdev.c
+++ b/drivers/net/e1000/em_ethdev.c
@@ -329,9 +329,6 @@ eth_em_dev_uninit(struct rte_eth_dev *eth_dev)
 	eth_dev->rx_pkt_burst = NULL;
 	eth_dev->tx_pkt_burst = NULL;
 
-	rte_free(eth_dev->data->mac_addrs);
-	eth_dev->data->mac_addrs = NULL;
-
 	/* disable uio intr before callback unregister */
 	rte_intr_disable(intr_handle);
 	rte_intr_callback_unregister(intr_handle,
@@ -1444,7 +1441,8 @@ eth_em_interrupt_setup(struct rte_eth_dev *dev)
 	/* clear interrupt */
 	E1000_READ_REG(hw, E1000_ICR);
 	regval = E1000_READ_REG(hw, E1000_IMS);
-	E1000_WRITE_REG(hw, E1000_IMS, regval | E1000_ICR_LSC);
+	E1000_WRITE_REG(hw, E1000_IMS,
+			regval | E1000_ICR_LSC | E1000_ICR_OTHER);
 	return 0;
 }
 
@@ -1494,7 +1492,7 @@ em_rxq_intr_enable(struct e1000_hw *hw)
 static void
 em_lsc_intr_disable(struct e1000_hw *hw)
 {
-	E1000_WRITE_REG(hw, E1000_IMC, E1000_IMS_LSC);
+	E1000_WRITE_REG(hw, E1000_IMC, E1000_IMS_LSC | E1000_IMS_OTHER);
 	E1000_WRITE_FLUSH(hw);
 }
 
diff --git a/drivers/net/e1000/em_rxtx.c b/drivers/net/e1000/em_rxtx.c
index 7d2ac4eb..a9cd7651 100644
--- a/drivers/net/e1000/em_rxtx.c
+++ b/drivers/net/e1000/em_rxtx.c
@@ -1160,6 +1160,7 @@ em_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
 
 	RTE_SET_USED(dev);
 	tx_offload_capa =
+		DEV_TX_OFFLOAD_MULTI_SEGS  |
 		DEV_TX_OFFLOAD_VLAN_INSERT |
 		DEV_TX_OFFLOAD_IPV4_CKSUM  |
 		DEV_TX_OFFLOAD_UDP_CKSUM   |
@@ -1363,7 +1364,6 @@ em_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
 		DEV_RX_OFFLOAD_IPV4_CKSUM  |
 		DEV_RX_OFFLOAD_UDP_CKSUM   |
 		DEV_RX_OFFLOAD_TCP_CKSUM   |
-		DEV_RX_OFFLOAD_CRC_STRIP   |
 		DEV_RX_OFFLOAD_KEEP_CRC    |
 		DEV_RX_OFFLOAD_SCATTER;
 	if (max_rx_pktlen > ETHER_MAX_LEN)
@@ -1417,12 +1417,13 @@ eth_em_rx_queue_setup(struct rte_eth_dev *dev,
 	}
 
 	/*
-	 * EM devices don't support drop_en functionality
+	 * EM devices don't support drop_en functionality.
+	 * It's an optimization that does nothing on single-queue devices,
+	 * so just log the issue and carry on.
 	 */
 	if (rx_conf->rx_drop_en) {
-		PMD_INIT_LOG(ERR, "drop_en functionality not supported by "
+		PMD_INIT_LOG(NOTICE, "drop_en functionality not supported by "
 			     "device");
-		return -EINVAL;
 	}
 
 	/* Free memory prior to re-allocation if needed. */
@@ -1459,7 +1460,7 @@ eth_em_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->rx_free_thresh = rx_conf->rx_free_thresh;
 	rxq->queue_id = queue_idx;
 	rxq->port_id = dev->data->port_id;
-	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
 		rxq->crc_len = ETHER_CRC_LEN;
 	else
 		rxq->crc_len = 0;
@@ -1795,7 +1796,7 @@ eth_em_rx_init(struct rte_eth_dev *dev)
 		 * Reset crc_len in case it was changed after queue setup by a
 		 *  call to configure
 		 */
-		if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+		if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
 			rxq->crc_len = ETHER_CRC_LEN;
 		else
 			rxq->crc_len = 0;
@@ -1877,7 +1878,7 @@ eth_em_rx_init(struct rte_eth_dev *dev)
 	}
 
 	/* Setup the Receive Control Register. */
-	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
 		rctl &= ~E1000_RCTL_SECRC; /* Do not Strip Ethernet CRC. */
 	else
 		rctl |= E1000_RCTL_SECRC; /* Strip Ethernet CRC. */
diff --git a/drivers/net/e1000/igb_ethdev.c b/drivers/net/e1000/igb_ethdev.c
index 64dfe683..d9d29d22 100644
--- a/drivers/net/e1000/igb_ethdev.c
+++ b/drivers/net/e1000/igb_ethdev.c
@@ -74,6 +74,7 @@ static void eth_igb_stop(struct rte_eth_dev *dev);
 static int  eth_igb_dev_set_link_up(struct rte_eth_dev *dev);
 static int  eth_igb_dev_set_link_down(struct rte_eth_dev *dev);
 static void eth_igb_close(struct rte_eth_dev *dev);
+static int eth_igb_reset(struct rte_eth_dev *dev);
 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
@@ -351,6 +352,7 @@ static const struct eth_dev_ops eth_igb_ops = {
 	.dev_set_link_up      = eth_igb_dev_set_link_up,
 	.dev_set_link_down    = eth_igb_dev_set_link_down,
 	.dev_close            = eth_igb_close,
+	.dev_reset            = eth_igb_reset,
 	.promiscuous_enable   = eth_igb_promiscuous_enable,
 	.promiscuous_disable  = eth_igb_promiscuous_disable,
 	.allmulticast_enable  = eth_igb_allmulticast_enable,
@@ -915,9 +917,6 @@ eth_igb_dev_uninit(struct rte_eth_dev *eth_dev)
 	/* Reset any pending lock */
 	igb_reset_swfw_lock(hw);
 
-	rte_free(eth_dev->data->mac_addrs);
-	eth_dev->data->mac_addrs = NULL;
-
 	/* uninitialize PF if max_vfs not zero */
 	igb_pf_host_uninit(eth_dev);
 
@@ -1071,9 +1070,6 @@ eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev)
 	eth_dev->rx_pkt_burst = NULL;
 	eth_dev->tx_pkt_burst = NULL;
 
-	rte_free(eth_dev->data->mac_addrs);
-	eth_dev->data->mac_addrs = NULL;
-
 	/* disable uio intr before callback unregister */
 	rte_intr_disable(&pci_dev->intr_handle);
 	rte_intr_callback_unregister(&pci_dev->intr_handle,
@@ -1593,6 +1589,33 @@ eth_igb_close(struct rte_eth_dev *dev)
 	rte_eth_linkstatus_set(dev, &link);
 }
 
+/*
+ * Reset PF device.
+ */
+static int
+eth_igb_reset(struct rte_eth_dev *dev)
+{
+	int ret;
+
+	/* When a DPDK PMD PF begin to reset PF port, it should notify all
+	 * its VF to make them align with it. The detailed notification
+	 * mechanism is PMD specific and is currently not implemented.
+	 * To avoid unexpected behavior in VF, currently reset of PF with
+	 * SR-IOV activation is not supported. It might be supported later.
+	 */
+	if (dev->data->sriov.active)
+		return -ENOTSUP;
+
+	ret = eth_igb_dev_uninit(dev);
+	if (ret)
+		return ret;
+
+	ret = eth_igb_dev_init(dev);
+
+	return ret;
+}
+
+
 static int
 igb_get_rx_buffer_size(struct e1000_hw *hw)
 {
@@ -3197,14 +3220,14 @@ igbvf_dev_configure(struct rte_eth_dev *dev)
 	 * Keep the persistent behavior the same as Host PF
 	 */
 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
-	if (rte_eth_dev_must_keep_crc(conf->rxmode.offloads)) {
+	if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
 		PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
-		conf->rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
+		conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC;
 	}
 #else
-	if (!rte_eth_dev_must_keep_crc(conf->rxmode.offloads)) {
+	if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) {
 		PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
-		conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_CRC_STRIP;
+		conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC;
 	}
 #endif
 
diff --git a/drivers/net/e1000/igb_flow.c b/drivers/net/e1000/igb_flow.c
index 07385291..33683498 100644
--- a/drivers/net/e1000/igb_flow.c
+++ b/drivers/net/e1000/igb_flow.c
@@ -1307,6 +1307,7 @@ igb_parse_rss_filter(struct rte_eth_dev *dev,
 			struct igb_rte_flow_rss_conf *rss_conf,
 			struct rte_flow_error *error)
 {
+	struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	const struct rte_flow_action *act;
 	const struct rte_flow_action_rss *rss;
 	uint16_t n, index;
@@ -1357,11 +1358,14 @@ igb_parse_rss_filter(struct rte_eth_dev *dev,
 		return rte_flow_error_set
 			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, act,
 			 "RSS hash key must be exactly 40 bytes");
-	if (rss->queue_num > RTE_DIM(rss_conf->queue))
+	if (((hw->mac.type == e1000_82576) &&
+	     (rss->queue_num > IGB_MAX_RX_QUEUE_NUM_82576)) ||
+	    ((hw->mac.type != e1000_82576) &&
+	     (rss->queue_num > IGB_MAX_RX_QUEUE_NUM)))
 		return rte_flow_error_set
 			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, act,
 			 "too many queues for RSS context");
-	if (igb_rss_conf_init(rss_conf, rss))
+	if (igb_rss_conf_init(dev, rss_conf, rss))
 		return rte_flow_error_set
 			(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, act,
 			 "RSS context initialization failure");
@@ -1574,7 +1578,7 @@ igb_flow_create(struct rte_eth_dev *dev,
 				PMD_DRV_LOG(ERR, "failed to allocate memory");
 				goto out;
 			}
-			igb_rss_conf_init(&rss_filter_ptr->filter_info,
+			igb_rss_conf_init(dev, &rss_filter_ptr->filter_info,
 					  &rss_conf.conf);
 			TAILQ_INSERT_TAIL(&igb_filter_rss_list,
 				rss_filter_ptr, entries);
diff --git a/drivers/net/e1000/igb_rxtx.c b/drivers/net/e1000/igb_rxtx.c
index b955068a..25ff5f68 100644
--- a/drivers/net/e1000/igb_rxtx.c
+++ b/drivers/net/e1000/igb_rxtx.c
@@ -1452,10 +1452,10 @@ igb_reset_tx_queue(struct igb_tx_queue *txq, struct rte_eth_dev *dev)
 uint64_t
 igb_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
 {
-	uint64_t rx_offload_capa;
+	uint64_t tx_offload_capa;
 
 	RTE_SET_USED(dev);
-	rx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
+	tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
 			  DEV_TX_OFFLOAD_IPV4_CKSUM  |
 			  DEV_TX_OFFLOAD_UDP_CKSUM   |
 			  DEV_TX_OFFLOAD_TCP_CKSUM   |
@@ -1463,17 +1463,17 @@ igb_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
 			  DEV_TX_OFFLOAD_TCP_TSO     |
 			  DEV_TX_OFFLOAD_MULTI_SEGS;
 
-	return rx_offload_capa;
+	return tx_offload_capa;
 }
 
 uint64_t
 igb_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
 {
-	uint64_t rx_queue_offload_capa;
+	uint64_t tx_queue_offload_capa;
 
-	rx_queue_offload_capa = igb_get_tx_port_offloads_capa(dev);
+	tx_queue_offload_capa = igb_get_tx_port_offloads_capa(dev);
 
-	return rx_queue_offload_capa;
+	return tx_queue_offload_capa;
 }
 
 int
@@ -1638,7 +1638,6 @@ igb_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
 			  DEV_RX_OFFLOAD_UDP_CKSUM   |
 			  DEV_RX_OFFLOAD_TCP_CKSUM   |
 			  DEV_RX_OFFLOAD_JUMBO_FRAME |
-			  DEV_RX_OFFLOAD_CRC_STRIP   |
 			  DEV_RX_OFFLOAD_KEEP_CRC    |
 			  DEV_RX_OFFLOAD_SCATTER;
 
@@ -1721,7 +1720,7 @@ eth_igb_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
 		queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
 	rxq->port_id = dev->data->port_id;
-	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
 		rxq->crc_len = ETHER_CRC_LEN;
 	else
 		rxq->crc_len = 0;
@@ -2374,7 +2373,7 @@ eth_igb_rx_init(struct rte_eth_dev *dev)
 		 * Reset crc_len in case it was changed after queue setup by a
 		 *  call to configure
 		 */
-		if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+		if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
 			rxq->crc_len = ETHER_CRC_LEN;
 		else
 			rxq->crc_len = 0;
@@ -2506,7 +2505,7 @@ eth_igb_rx_init(struct rte_eth_dev *dev)
 	E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
 
 	/* Setup the Receive Control Register. */
-	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads)) {
+	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
 		rctl &= ~E1000_RCTL_SECRC; /* Do not Strip Ethernet CRC. */
 
 		/* clear STRCRC bit in all queues */
@@ -2852,11 +2851,17 @@ igb_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 }
 
 int
-igb_rss_conf_init(struct igb_rte_flow_rss_conf *out,
+igb_rss_conf_init(struct rte_eth_dev *dev,
+		  struct igb_rte_flow_rss_conf *out,
 		  const struct rte_flow_action_rss *in)
 {
+	struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
 	if (in->key_len > RTE_DIM(out->key) ||
-	    in->queue_num > RTE_DIM(out->queue))
+	    ((hw->mac.type == e1000_82576) &&
+	     (in->queue_num > IGB_MAX_RX_QUEUE_NUM_82576)) ||
+	    ((hw->mac.type != e1000_82576) &&
+	     (in->queue_num > IGB_MAX_RX_QUEUE_NUM)))
 		return -EINVAL;
 	out->conf = (struct rte_flow_action_rss){
 		.func = in->func,
@@ -2945,7 +2950,7 @@ igb_config_rss_filter(struct rte_eth_dev *dev,
 		rss_conf.rss_key = rss_intel_key; /* Default hash key */
 	igb_hw_rss_hash_set(hw, &rss_conf);
 
-	if (igb_rss_conf_init(&filter_info->rss_info, &conf->conf))
+	if (igb_rss_conf_init(dev, &filter_info->rss_info, &conf->conf))
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/net/e1000/meson.build b/drivers/net/e1000/meson.build
index cf456995..d0901d37 100644
--- a/drivers/net/e1000/meson.build
+++ b/drivers/net/e1000/meson.build
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
+allow_experimental_apis = true
+
 subdir('base')
 objs = [base_objs]
 
diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c
index c255dc6d..0c0ed930 100644
--- a/drivers/net/ena/ena_ethdev.c
+++ b/drivers/net/ena/ena_ethdev.c
@@ -55,7 +55,7 @@
 
 #define DRV_MODULE_VER_MAJOR	1
 #define DRV_MODULE_VER_MINOR	1
-#define DRV_MODULE_VER_SUBMINOR	0
+#define DRV_MODULE_VER_SUBMINOR	1
 
 #define ENA_IO_TXQ_IDX(q)	(2 * (q))
 #define ENA_IO_RXQ_IDX(q)	(2 * (q) + 1)
@@ -239,6 +239,8 @@ static void ena_rx_queue_release_bufs(struct ena_ring *ring);
 static void ena_tx_queue_release_bufs(struct ena_ring *ring);
 static int ena_link_update(struct rte_eth_dev *dev,
 			   int wait_to_complete);
+static int ena_create_io_queue(struct ena_ring *ring);
+static void ena_free_io_queues_all(struct ena_adapter *adapter);
 static int ena_queue_restart(struct ena_ring *ring);
 static int ena_queue_restart_all(struct rte_eth_dev *dev,
 				 enum ena_ring_type ring_type);
@@ -510,7 +512,8 @@ static void ena_close(struct rte_eth_dev *dev)
 	struct ena_adapter *adapter =
 		(struct ena_adapter *)(dev->data->dev_private);
 
-	ena_stop(dev);
+	if (adapter->state == ENA_ADAPTER_STATE_RUNNING)
+		ena_stop(dev);
 	adapter->state = ENA_ADAPTER_STATE_CLOSED;
 
 	ena_rx_queue_release_all(dev);
@@ -746,21 +749,12 @@ static void ena_tx_queue_release_all(struct rte_eth_dev *dev)
 static void ena_rx_queue_release(void *queue)
 {
 	struct ena_ring *ring = (struct ena_ring *)queue;
-	struct ena_adapter *adapter = ring->adapter;
-	int ena_qid;
 
 	ena_assert_msg(ring->configured,
 		       "API violation - releasing not configured queue");
 	ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING,
 		       "API violation");
 
-	/* Destroy HW queue */
-	ena_qid = ENA_IO_RXQ_IDX(ring->id);
-	ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid);
-
-	/* Free all bufs */
-	ena_rx_queue_release_bufs(ring);
-
 	/* Free ring resources */
 	if (ring->rx_buffer_info)
 		rte_free(ring->rx_buffer_info);
@@ -779,18 +773,12 @@ static void ena_rx_queue_release(void *queue)
 static void ena_tx_queue_release(void *queue)
 {
 	struct ena_ring *ring = (struct ena_ring *)queue;
-	struct ena_adapter *adapter = ring->adapter;
-	int ena_qid;
 
 	ena_assert_msg(ring->configured,
 		       "API violation. Releasing not configured queue");
 	ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING,
 		       "API violation");
 
-	/* Destroy HW queue */
-	ena_qid = ENA_IO_TXQ_IDX(ring->id);
-	ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid);
-
 	/* Free all bufs */
 	ena_tx_queue_release_bufs(ring);
 
@@ -1078,10 +1066,86 @@ static void ena_stop(struct rte_eth_dev *dev)
 		(struct ena_adapter *)(dev->data->dev_private);
 
 	rte_timer_stop_sync(&adapter->timer_wd);
+	ena_free_io_queues_all(adapter);
 
 	adapter->state = ENA_ADAPTER_STATE_STOPPED;
 }
 
+static int ena_create_io_queue(struct ena_ring *ring)
+{
+	struct ena_adapter *adapter;
+	struct ena_com_dev *ena_dev;
+	struct ena_com_create_io_ctx ctx =
+		/* policy set to _HOST just to satisfy icc compiler */
+		{ ENA_ADMIN_PLACEMENT_POLICY_HOST,
+		  0, 0, 0, 0, 0 };
+	uint16_t ena_qid;
+	int rc;
+
+	adapter = ring->adapter;
+	ena_dev = &adapter->ena_dev;
+
+	if (ring->type == ENA_RING_TYPE_TX) {
+		ena_qid = ENA_IO_TXQ_IDX(ring->id);
+		ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
+		ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
+		ctx.queue_size = adapter->tx_ring_size;
+	} else {
+		ena_qid = ENA_IO_RXQ_IDX(ring->id);
+		ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
+		ctx.queue_size = adapter->rx_ring_size;
+	}
+	ctx.qid = ena_qid;
+	ctx.msix_vector = -1; /* interrupts not used */
+	ctx.numa_node = ena_cpu_to_node(ring->id);
+
+	rc = ena_com_create_io_queue(ena_dev, &ctx);
+	if (rc) {
+		RTE_LOG(ERR, PMD,
+			"failed to create io queue #%d (qid:%d) rc: %d\n",
+			ring->id, ena_qid, rc);
+		return rc;
+	}
+
+	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
+				     &ring->ena_com_io_sq,
+				     &ring->ena_com_io_cq);
+	if (rc) {
+		RTE_LOG(ERR, PMD,
+			"Failed to get io queue handlers. queue num %d rc: %d\n",
+			ring->id, rc);
+		ena_com_destroy_io_queue(ena_dev, ena_qid);
+		return rc;
+	}
+
+	if (ring->type == ENA_RING_TYPE_TX)
+		ena_com_update_numa_node(ring->ena_com_io_cq, ctx.numa_node);
+
+	return 0;
+}
+
+static void ena_free_io_queues_all(struct ena_adapter *adapter)
+{
+	struct rte_eth_dev *eth_dev = adapter->rte_dev;
+	struct ena_com_dev *ena_dev = &adapter->ena_dev;
+	int i;
+	uint16_t ena_qid;
+	uint16_t nb_rxq = eth_dev->data->nb_rx_queues;
+	uint16_t nb_txq = eth_dev->data->nb_tx_queues;
+
+	for (i = 0; i < nb_txq; ++i) {
+		ena_qid = ENA_IO_TXQ_IDX(i);
+		ena_com_destroy_io_queue(ena_dev, ena_qid);
+	}
+
+	for (i = 0; i < nb_rxq; ++i) {
+		ena_qid = ENA_IO_RXQ_IDX(i);
+		ena_com_destroy_io_queue(ena_dev, ena_qid);
+
+		ena_rx_queue_release_bufs(&adapter->rx_ring[i]);
+	}
+}
+
 static int ena_queue_restart(struct ena_ring *ring)
 {
 	int rc, bufs_num;
@@ -1089,6 +1153,12 @@ static int ena_queue_restart(struct ena_ring *ring)
 	ena_assert_msg(ring->configured == 1,
 		       "Trying to restart unconfigured queue\n");
 
+	rc = ena_create_io_queue(ring);
+	if (rc) {
+		PMD_INIT_LOG(ERR, "Failed to create IO queue!\n");
+		return rc;
+	}
+
 	ring->next_to_clean = 0;
 	ring->next_to_use = 0;
 
@@ -1111,17 +1181,10 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,
 			      __rte_unused unsigned int socket_id,
 			      const struct rte_eth_txconf *tx_conf)
 {
-	struct ena_com_create_io_ctx ctx =
-		/* policy set to _HOST just to satisfy icc compiler */
-		{ ENA_ADMIN_PLACEMENT_POLICY_HOST,
-		  ENA_COM_IO_QUEUE_DIRECTION_TX, 0, 0, 0, 0 };
 	struct ena_ring *txq = NULL;
 	struct ena_adapter *adapter =
 		(struct ena_adapter *)(dev->data->dev_private);
 	unsigned int i;
-	int ena_qid;
-	int rc;
-	struct ena_com_dev *ena_dev = &adapter->ena_dev;
 
 	txq = &adapter->tx_ring[queue_idx];
 
@@ -1146,37 +1209,6 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,
 		return -EINVAL;
 	}
 
-	ena_qid = ENA_IO_TXQ_IDX(queue_idx);
-
-	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
-	ctx.qid = ena_qid;
-	ctx.msix_vector = -1; /* admin interrupts not used */
-	ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
-	ctx.queue_size = adapter->tx_ring_size;
-	ctx.numa_node = ena_cpu_to_node(queue_idx);
-
-	rc = ena_com_create_io_queue(ena_dev, &ctx);
-	if (rc) {
-		RTE_LOG(ERR, PMD,
-			"failed to create io TX queue #%d (qid:%d) rc: %d\n",
-			queue_idx, ena_qid, rc);
-		return rc;
-	}
-	txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid];
-	txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid];
-
-	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
-				     &txq->ena_com_io_sq,
-				     &txq->ena_com_io_cq);
-	if (rc) {
-		RTE_LOG(ERR, PMD,
-			"Failed to get TX queue handlers. TX queue num %d rc: %d\n",
-			queue_idx, rc);
-		goto err_destroy_io_queue;
-	}
-
-	ena_com_update_numa_node(txq->ena_com_io_cq, ctx.numa_node);
-
 	txq->port_id = dev->data->port_id;
 	txq->next_to_clean = 0;
 	txq->next_to_use = 0;
@@ -1188,8 +1220,7 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,
 					  RTE_CACHE_LINE_SIZE);
 	if (!txq->tx_buffer_info) {
 		RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n");
-		rc = -ENOMEM;
-		goto err_destroy_io_queue;
+		return -ENOMEM;
 	}
 
 	txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs",
@@ -1197,8 +1228,8 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,
 					 RTE_CACHE_LINE_SIZE);
 	if (!txq->empty_tx_reqs) {
 		RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n");
-		rc = -ENOMEM;
-		goto err_free;
+		rte_free(txq->tx_buffer_info);
+		return -ENOMEM;
 	}
 
 	for (i = 0; i < txq->ring_size; i++)
@@ -1214,13 +1245,6 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,
 	dev->data->tx_queues[queue_idx] = txq;
 
 	return 0;
-
-err_free:
-	rte_free(txq->tx_buffer_info);
-
-err_destroy_io_queue:
-	ena_com_destroy_io_queue(ena_dev, ena_qid);
-	return rc;
 }
 
 static int ena_rx_queue_setup(struct rte_eth_dev *dev,
@@ -1230,16 +1254,10 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev,
 			      __rte_unused const struct rte_eth_rxconf *rx_conf,
 			      struct rte_mempool *mp)
 {
-	struct ena_com_create_io_ctx ctx =
-		/* policy set to _HOST just to satisfy icc compiler */
-		{ ENA_ADMIN_PLACEMENT_POLICY_HOST,
-		  ENA_COM_IO_QUEUE_DIRECTION_RX, 0, 0, 0, 0 };
 	struct ena_adapter *adapter =
 		(struct ena_adapter *)(dev->data->dev_private);
 	struct ena_ring *rxq = NULL;
-	uint16_t ena_qid = 0;
-	int i, rc = 0;
-	struct ena_com_dev *ena_dev = &adapter->ena_dev;
+	int i;
 
 	rxq = &adapter->rx_ring[queue_idx];
 	if (rxq->configured) {
@@ -1263,36 +1281,6 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev,
 		return -EINVAL;
 	}
 
-	ena_qid = ENA_IO_RXQ_IDX(queue_idx);
-
-	ctx.qid = ena_qid;
-	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
-	ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
-	ctx.msix_vector = -1; /* admin interrupts not used */
-	ctx.queue_size = adapter->rx_ring_size;
-	ctx.numa_node = ena_cpu_to_node(queue_idx);
-
-	rc = ena_com_create_io_queue(ena_dev, &ctx);
-	if (rc) {
-		RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc: %d\n",
-			queue_idx, rc);
-		return rc;
-	}
-
-	rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid];
-	rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid];
-
-	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
-				     &rxq->ena_com_io_sq,
-				     &rxq->ena_com_io_cq);
-	if (rc) {
-		RTE_LOG(ERR, PMD,
-			"Failed to get RX queue handlers. RX queue num %d rc: %d\n",
-			queue_idx, rc);
-		ena_com_destroy_io_queue(ena_dev, ena_qid);
-		return rc;
-	}
-
 	rxq->port_id = dev->data->port_id;
 	rxq->next_to_clean = 0;
 	rxq->next_to_use = 0;
@@ -1304,7 +1292,6 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev,
 					  RTE_CACHE_LINE_SIZE);
 	if (!rxq->rx_buffer_info) {
 		RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n");
-		ena_com_destroy_io_queue(ena_dev, ena_qid);
 		return -ENOMEM;
 	}
 
@@ -1315,7 +1302,6 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev,
 		RTE_LOG(ERR, PMD, "failed to alloc mem for empty rx reqs\n");
 		rte_free(rxq->rx_buffer_info);
 		rxq->rx_buffer_info = NULL;
-		ena_com_destroy_io_queue(ena_dev, ena_qid);
 		return -ENOMEM;
 	}
 
@@ -1326,7 +1312,7 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->configured = 1;
 	dev->data->rx_queues[queue_idx] = rxq;
 
-	return rc;
+	return 0;
 }
 
 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count)
@@ -1703,7 +1689,7 @@ static int eth_ena_dev_uninit(struct rte_eth_dev *eth_dev)
 		(struct ena_adapter *)(eth_dev->data->dev_private);
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
-		return -EPERM;
+		return 0;
 
 	if (adapter->state != ENA_ADAPTER_STATE_CLOSED)
 		ena_close(eth_dev);
@@ -1924,7 +1910,7 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 
 		/* fill mbuf attributes if any */
 		ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx);
-		mbuf_head->hash.rss = (uint32_t)rx_ring->id;
+		mbuf_head->hash.rss = ena_rx_ctx.hash;
 
 		/* pass to DPDK application head mbuf */
 		rx_pkts[recv_idx] = mbuf_head;
diff --git a/drivers/net/enetc/Makefile b/drivers/net/enetc/Makefile
new file mode 100644
index 00000000..9895501d
--- /dev/null
+++ b/drivers/net/enetc/Makefile
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2018 NXP
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_pmd_enetc.a
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+EXPORT_MAP := rte_pmd_enetc_version.map
+LIBABIVER := 1
+
+SRCS-$(CONFIG_RTE_LIBRTE_ENETC_PMD) += enetc_ethdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_ENETC_PMD) += enetc_rxtx.c
+
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool
+LDLIBS += -lrte_ethdev
+LDLIBS += -lrte_bus_pci
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/enetc/base/enetc_hw.h b/drivers/net/enetc/base/enetc_hw.h
new file mode 100644
index 00000000..f36fa11e
--- /dev/null
+++ b/drivers/net/enetc/base/enetc_hw.h
@@ -0,0 +1,226 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#ifndef _ENETC_HW_H_
+#define _ENETC_HW_H_
+#include <rte_io.h>
+
+#define BIT(x)		((uint64_t)1 << ((x)))
+
+/* ENETC device IDs */
+#define ENETC_DEV_ID_VF		0xef00
+#define ENETC_DEV_ID		0xe100
+
+/* ENETC register block BAR */
+#define ENETC_BAR_REGS			0x0
+
+/* SI regs, offset: 0h */
+#define ENETC_SIMR			0x0
+#define ENETC_SIMR_EN			BIT(31)
+
+#define ENETC_SIPMAR0			0x80
+#define ENETC_SIPMAR1			0x84
+
+#define ENETC_SICAPR0			0x900
+#define ENETC_SICAPR1			0x904
+
+#define ENETC_SIMSITRV(n)		(0xB00 + (n) * 0x4)
+#define ENETC_SIMSIRRV(n)		(0xB80 + (n) * 0x4)
+
+#define ENETC_SICCAPR			0x1200
+
+/* enum for BD type */
+enum enetc_bdr_type {TX, RX};
+
+#define ENETC_BDR(type, n, off)		(0x8000 + (type) * 0x100 + (n) * 0x200 \
+							+ (off))
+/* RX BDR reg offsets */
+#define ENETC_RBMR		0x0 /* RX BDR mode register*/
+#define ENETC_RBMR_EN		BIT(31)
+
+#define ENETC_RBSR		0x4  /* Rx BDR status register*/
+#define ENETC_RBBSR		0x8  /* Rx BDR buffer size register*/
+#define ENETC_RBCIR		0xc  /* Rx BDR consumer index register*/
+#define ENETC_RBBAR0		0x10 /* Rx BDR base address register 0 */
+#define ENETC_RBBAR1		0x14 /* Rx BDR base address register 1*/
+#define ENETC_RBPIR		0x18 /* Rx BDR producer index register*/
+#define ENETC_RBLENR		0x20 /* Rx BDR length register*/
+#define ENETC_RBIER		0xa0 /* Rx BDR interrupt enable register*/
+#define ENETC_RBIER_RXTIE	BIT(0)
+#define ENETC_RBIDR		0xa4 /* Rx BDR interrupt detect register*/
+#define ENETC_RBICIR0		0xa8 /* Rx BDR inetrrupt coalescing register 0*/
+#define ENETC_RBICIR0_ICEN	BIT(31)
+
+
+#define ENETC_TBMR	0x0  /* Tx BDR mode register (TBMR) 32 RW */
+#define ENETC_TBSR	0x4  /* x BDR status register (TBSR) 32 RO */
+#define ENETC_TBBAR0	0x10 /* Tx BDR base address register 0 (TBBAR0) 32 RW */
+#define ENETC_TBBAR1	0x14 /* Tx BDR base address register 1 (TBBAR1) 32 RW */
+#define ENETC_TBCIR	0x18 /* Tx BDR consumer index register (TBCIR) 32 RW */
+#define ENETC_TBCISR	0x1C /* Tx BDR consumer index shadow register 32 RW */
+#define ENETC_TBIER	0xA0 /* Tx BDR interrupt enable register 32 RW */
+#define ENETC_TBIDR	0xA4 /* Tx BDR interrupt detect register 32 RO */
+#define ENETC_TBICR0	0xA8 /* Tx BDR interrupt coalescing register 0 32 RW */
+#define ENETC_TBICR1	0xAC /* Tx BDR interrupt coalescing register 1 32 RW */
+#define ENETC_TBLENR	0x20
+
+#define ENETC_TBCISR_IDX_MASK		0xffff
+#define ENETC_TBIER_TXFIE		BIT(1)
+
+#define ENETC_RTBLENR_LEN(n)		((n) & ~0x7)
+#define ENETC_TBMR_EN			BIT(31)
+
+/* Port regs, offset: 1_0000h */
+#define ENETC_PORT_BASE			0x10000
+#define ENETC_PMR			0x00000
+#define ENETC_PMR_EN			(BIT(16) | BIT(17) | BIT(18))
+#define ENETC_PSR			0x00004 /* RO */
+#define ENETC_PSIPMR			0x00018
+#define ENETC_PSIPMR_SET_UP(n)		(0x1 << (n)) /* n = SI index */
+#define ENETC_PSIPMR_SET_MP(n)		(0x1 << ((n) + 8))
+#define ENETC_PSIPMR_SET_VLAN_MP(n)	(0x1 << ((n) + 16))
+#define ENETC_PSIPMAR0(n)		(0x00100 + (n) * 0x20)
+#define ENETC_PSIPMAR1(n)		(0x00104 + (n) * 0x20)
+#define ENETC_PCAPR0			0x00900
+#define ENETC_PCAPR1			0x00904
+
+#define ENETC_PV0CFGR(n)		(0x00920 + (n) * 0x10)
+#define ENETC_PVCFGR_SET_TXBDR(val)	((val) & 0xff)
+#define ENETC_PVCFGR_SET_RXBDR(val)	(((val) & 0xff) << 16)
+
+#define ENETC_PM0_CMD_CFG		0x08008
+#define ENETC_PM0_TX_EN			BIT(0)
+#define ENETC_PM0_RX_EN			BIT(1)
+
+#define ENETC_PM0_MAXFRM		0x08014
+#define ENETC_SET_MAXFRM(val)		((val) << 16)
+
+#define ENETC_PM0_STATUS		0x08304
+#define ENETC_LINK_MODE			0x0000000000080000ULL
+#define ENETC_LINK_STATUS		0x0000000000010000ULL
+#define ENETC_LINK_SPEED_MASK		0x0000000000060000ULL
+#define ENETC_LINK_SPEED_10M		0x0ULL
+#define ENETC_LINK_SPEED_100M		0x0000000000020000ULL
+#define ENETC_LINK_SPEED_1G		0x0000000000040000ULL
+
+/* Global regs, offset: 2_0000h */
+#define ENETC_GLOBAL_BASE		0x20000
+#define ENETC_G_EIPBRR0			0x00bf8
+#define ENETC_G_EIPBRR1			0x00bfc
+
+/* general register accessors */
+#define enetc_rd_reg(reg)	rte_read32((void *)(reg))
+#define enetc_wr_reg(reg, val)	rte_write32((val), (void *)(reg))
+#define enetc_rd(hw, off)	enetc_rd_reg((size_t)(hw)->reg + (off))
+#define enetc_wr(hw, off, val)	enetc_wr_reg((size_t)(hw)->reg + (off), val)
+/* port register accessors - PF only */
+#define enetc_port_rd(hw, off)	enetc_rd_reg((size_t)(hw)->port + (off))
+#define enetc_port_wr(hw, off, val) \
+				enetc_wr_reg((size_t)(hw)->port + (off), val)
+/* global register accessors - PF only */
+#define enetc_global_rd(hw, off) \
+				enetc_rd_reg((size_t)(hw)->global + (off))
+#define enetc_global_wr(hw, off, val) \
+				enetc_wr_reg((size_t)(hw)->global + (off), val)
+/* BDR register accessors, see ENETC_BDR() */
+#define enetc_bdr_rd(hw, t, n, off) \
+				enetc_rd(hw, ENETC_BDR(t, n, off))
+#define enetc_bdr_wr(hw, t, n, off, val) \
+				enetc_wr(hw, ENETC_BDR(t, n, off), val)
+
+#define enetc_txbdr_rd(hw, n, off) enetc_bdr_rd(hw, TX, n, off)
+#define enetc_rxbdr_rd(hw, n, off) enetc_bdr_rd(hw, RX, n, off)
+#define enetc_txbdr_wr(hw, n, off, val) \
+				enetc_bdr_wr(hw, TX, n, off, val)
+#define enetc_rxbdr_wr(hw, n, off, val) \
+				enetc_bdr_wr(hw, RX, n, off, val)
+
+#define ENETC_TX_ADDR(txq, addr) ((void *)((txq)->enetc_txbdr + (addr)))
+
+#define ENETC_TXBD_FLAGS_IE		BIT(13)
+#define ENETC_TXBD_FLAGS_F		BIT(15)
+
+/* ENETC Parsed values (Little Endian) */
+#define ENETC_PKT_TYPE_ETHER            0x0060
+#define ENETC_PKT_TYPE_IPV4             0x0000
+#define ENETC_PKT_TYPE_IPV6             0x0020
+#define ENETC_PKT_TYPE_IPV4_TCP \
+			(0x0010 | ENETC_PKT_TYPE_IPV4)
+#define ENETC_PKT_TYPE_IPV6_TCP \
+			(0x0010 | ENETC_PKT_TYPE_IPV6)
+#define ENETC_PKT_TYPE_IPV4_UDP \
+			(0x0011 | ENETC_PKT_TYPE_IPV4)
+#define ENETC_PKT_TYPE_IPV6_UDP \
+			(0x0011 | ENETC_PKT_TYPE_IPV6)
+#define ENETC_PKT_TYPE_IPV4_SCTP \
+			(0x0013 | ENETC_PKT_TYPE_IPV4)
+#define ENETC_PKT_TYPE_IPV6_SCTP \
+			(0x0013 | ENETC_PKT_TYPE_IPV6)
+#define ENETC_PKT_TYPE_IPV4_ICMP \
+			(0x0003 | ENETC_PKT_TYPE_IPV4)
+#define ENETC_PKT_TYPE_IPV6_ICMP \
+			(0x0003 | ENETC_PKT_TYPE_IPV6)
+
+/* PCI device info */
+struct enetc_hw {
+	void *reg;	/* SI registers, used by all PCI functions */
+	void *port;	/* Port registers, PF only */
+	void *global;	/* IP global registers, PF only */
+};
+
+struct enetc_eth_mac_info {
+	uint8_t addr[ETHER_ADDR_LEN];
+	uint8_t perm_addr[ETHER_ADDR_LEN];
+	uint8_t get_link_status;
+};
+
+struct enetc_eth_hw {
+	struct rte_eth_dev *ndev;
+	struct enetc_hw hw;
+	uint16_t device_id;
+	uint16_t vendor_id;
+	uint8_t revision_id;
+	struct enetc_eth_mac_info mac;
+};
+
+/* Transmit Descriptor */
+struct enetc_tx_desc {
+	uint64_t addr;
+	uint16_t frm_len;
+	uint16_t buf_len;
+	uint32_t flags_errors;
+};
+
+/* TX Buffer Descriptors (BD) */
+struct enetc_tx_bd {
+	uint64_t addr;
+	uint16_t buf_len;
+	uint16_t frm_len;
+	uint16_t err_csum;
+	uint16_t flags;
+};
+
+/* RX buffer descriptor */
+union enetc_rx_bd {
+	struct {
+		uint64_t addr;
+		uint8_t reserved[8];
+	} w;
+	struct {
+		uint16_t inet_csum;
+		uint16_t parse_summary;
+		uint32_t rss_hash;
+		uint16_t buf_len;
+		uint16_t vlan_opt;
+		union {
+			struct {
+				uint16_t flags;
+				uint16_t error;
+			};
+			uint32_t lstatus;
+		};
+	} r;
+};
+
+#endif
diff --git a/drivers/net/enetc/enetc.h b/drivers/net/enetc/enetc.h
new file mode 100644
index 00000000..0e80d1c5
--- /dev/null
+++ b/drivers/net/enetc/enetc.h
@@ -0,0 +1,113 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#ifndef _ENETC_H_
+#define _ENETC_H_
+
+#include <rte_time.h>
+
+#include "base/enetc_hw.h"
+
+#define PCI_VENDOR_ID_FREESCALE 0x1957
+
+/* Max TX rings per ENETC. */
+#define MAX_TX_RINGS	2
+
+/* Max RX rings per ENTEC. */
+#define MAX_RX_RINGS	1
+
+/* Max BD counts per Ring. */
+#define MAX_BD_COUNT   64000
+/* Min BD counts per Ring. */
+#define MIN_BD_COUNT   32
+/* BD ALIGN */
+#define BD_ALIGN       8
+
+/*
+ * upper_32_bits - return bits 32-63 of a number
+ * @n: the number we're accessing
+ *
+ * A basic shift-right of a 64- or 32-bit quantity.  Use this to suppress
+ * the "right shift count >= width of type" warning when that quantity is
+ * 32-bits.
+ */
+#define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16))
+
+/*
+ * lower_32_bits - return bits 0-31 of a number
+ * @n: the number we're accessing
+ */
+#define lower_32_bits(n) ((uint32_t)(n))
+
+#define ENETC_TXBD(BDR, i) (&(((struct enetc_tx_bd *)((BDR).bd_base))[i]))
+#define ENETC_RXBD(BDR, i) (&(((union enetc_rx_bd *)((BDR).bd_base))[i]))
+
+struct enetc_swbd {
+	struct rte_mbuf *buffer_addr;
+};
+
+struct enetc_bdr {
+	struct rte_eth_dev *ndev;
+	struct rte_mempool *mb_pool;   /* mbuf pool to populate RX ring. */
+	void *bd_base;			/* points to Rx or Tx BD ring */
+	union {
+		void *tcir;
+		void *rcir;
+	};
+	uint16_t index;
+	int bd_count; /* # of BDs */
+	int next_to_use;
+	int next_to_clean;
+	struct enetc_swbd *q_swbd;
+	union {
+		void *tcisr; /* Tx */
+		int next_to_alloc; /* Rx */
+	};
+};
+
+/*
+ * Structure to store private data for each driver instance (for each port).
+ */
+struct enetc_eth_adapter {
+	struct rte_eth_dev *ndev;
+	struct enetc_eth_hw hw;
+};
+
+#define ENETC_DEV_PRIVATE(adapter) \
+	((struct enetc_eth_adapter *)adapter)
+
+#define ENETC_DEV_PRIVATE_TO_HW(adapter) \
+	(&((struct enetc_eth_adapter *)adapter)->hw)
+
+#define ENETC_DEV_PRIVATE_TO_STATS(adapter) \
+	(&((struct enetc_eth_adapter *)adapter)->stats)
+
+#define ENETC_DEV_PRIVATE_TO_INTR(adapter) \
+	(&((struct enetc_eth_adapter *)adapter)->intr)
+
+#define ENETC_GET_HW_ADDR(reg, addr) ((void *)(((size_t)reg) + (addr)))
+#define ENETC_REG_READ(addr) (*(uint32_t *)addr)
+#define ENETC_REG_WRITE(addr, val) (*(uint32_t *)addr = val)
+#define ENETC_REG_WRITE_RELAXED(addr, val) (*(uint32_t *)addr = val)
+
+/*
+ * RX/TX ENETC function prototypes
+ */
+uint16_t enetc_xmit_pkts(void *txq, struct rte_mbuf **tx_pkts,
+		uint16_t nb_pkts);
+uint16_t enetc_recv_pkts(void *rxq, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts);
+
+
+int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt);
+
+static inline int
+enetc_bd_unused(struct enetc_bdr *bdr)
+{
+	if (bdr->next_to_clean > bdr->next_to_use)
+		return bdr->next_to_clean - bdr->next_to_use - 1;
+
+	return bdr->bd_count + bdr->next_to_clean - bdr->next_to_use - 1;
+}
+#endif /* _ENETC_H_ */
diff --git a/drivers/net/enetc/enetc_ethdev.c b/drivers/net/enetc/enetc_ethdev.c
new file mode 100644
index 00000000..023fe751
--- /dev/null
+++ b/drivers/net/enetc/enetc_ethdev.c
@@ -0,0 +1,629 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#include <stdbool.h>
+#include <rte_ethdev_pci.h>
+
+#include "enetc_logs.h"
+#include "enetc.h"
+
+int enetc_logtype_pmd;
+
+/* Functions Prototypes */
+static int enetc_dev_configure(struct rte_eth_dev *dev);
+static int enetc_dev_start(struct rte_eth_dev *dev);
+static void enetc_dev_stop(struct rte_eth_dev *dev);
+static void enetc_dev_close(struct rte_eth_dev *dev);
+static void enetc_dev_infos_get(struct rte_eth_dev *dev,
+				struct rte_eth_dev_info *dev_info);
+static int enetc_link_update(struct rte_eth_dev *dev, int wait_to_complete);
+static int enetc_hardware_init(struct enetc_eth_hw *hw);
+static int enetc_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+		uint16_t nb_rx_desc, unsigned int socket_id,
+		const struct rte_eth_rxconf *rx_conf,
+		struct rte_mempool *mb_pool);
+static void enetc_rx_queue_release(void *rxq);
+static int enetc_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
+		uint16_t nb_tx_desc, unsigned int socket_id,
+		const struct rte_eth_txconf *tx_conf);
+static void enetc_tx_queue_release(void *txq);
+static const uint32_t *enetc_supported_ptypes_get(struct rte_eth_dev *dev);
+
+/*
+ * The set of PCI devices this driver supports
+ */
+static const struct rte_pci_id pci_id_enetc_map[] = {
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, ENETC_DEV_ID) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, ENETC_DEV_ID_VF) },
+	{ .vendor_id = 0, /* sentinel */ },
+};
+
+/* Features supported by this driver */
+static const struct eth_dev_ops enetc_ops = {
+	.dev_configure        = enetc_dev_configure,
+	.dev_start            = enetc_dev_start,
+	.dev_stop             = enetc_dev_stop,
+	.dev_close            = enetc_dev_close,
+	.link_update          = enetc_link_update,
+	.dev_infos_get        = enetc_dev_infos_get,
+	.rx_queue_setup       = enetc_rx_queue_setup,
+	.rx_queue_release     = enetc_rx_queue_release,
+	.tx_queue_setup       = enetc_tx_queue_setup,
+	.tx_queue_release     = enetc_tx_queue_release,
+	.dev_supported_ptypes_get = enetc_supported_ptypes_get,
+};
+
+/**
+ * Initialisation of the enetc device
+ *
+ * @param eth_dev
+ *   - Pointer to the structure rte_eth_dev
+ *
+ * @return
+ *   - On success, zero.
+ *   - On failure, negative value.
+ */
+static int
+enetc_dev_init(struct rte_eth_dev *eth_dev)
+{
+	int error = 0;
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+	struct enetc_eth_hw *hw =
+		ENETC_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
+
+	PMD_INIT_FUNC_TRACE();
+	eth_dev->dev_ops = &enetc_ops;
+	eth_dev->rx_pkt_burst = &enetc_recv_pkts;
+	eth_dev->tx_pkt_burst = &enetc_xmit_pkts;
+
+	/* Retrieving and storing the HW base address of device */
+	hw->hw.reg = (void *)pci_dev->mem_resource[0].addr;
+	hw->device_id = pci_dev->id.device_id;
+
+	error = enetc_hardware_init(hw);
+	if (error != 0) {
+		ENETC_PMD_ERR("Hardware initialization failed");
+		return -1;
+	}
+
+	/* Allocate memory for storing MAC addresses */
+	eth_dev->data->mac_addrs = rte_zmalloc("enetc_eth", ETHER_ADDR_LEN, 0);
+	if (!eth_dev->data->mac_addrs) {
+		ENETC_PMD_ERR("Failed to allocate %d bytes needed to "
+			      "store MAC addresses",
+			      ETHER_ADDR_LEN * 1);
+		error = -ENOMEM;
+		return -1;
+	}
+
+	/* Copy the permanent MAC address */
+	ether_addr_copy((struct ether_addr *)hw->mac.addr,
+			&eth_dev->data->mac_addrs[0]);
+
+	ENETC_PMD_DEBUG("port_id %d vendorID=0x%x deviceID=0x%x",
+			eth_dev->data->port_id, pci_dev->id.vendor_id,
+			pci_dev->id.device_id);
+	return 0;
+}
+
+static int
+enetc_dev_uninit(struct rte_eth_dev *eth_dev __rte_unused)
+{
+	PMD_INIT_FUNC_TRACE();
+	return 0;
+}
+
+static int
+enetc_dev_configure(struct rte_eth_dev *dev __rte_unused)
+{
+	PMD_INIT_FUNC_TRACE();
+	return 0;
+}
+
+static int
+enetc_dev_start(struct rte_eth_dev *dev)
+{
+	struct enetc_eth_hw *hw =
+		ENETC_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t val;
+
+	PMD_INIT_FUNC_TRACE();
+	val = ENETC_REG_READ(ENETC_GET_HW_ADDR(hw->hw.port,
+			     ENETC_PM0_CMD_CFG));
+	ENETC_REG_WRITE(ENETC_GET_HW_ADDR(hw->hw.port, ENETC_PM0_CMD_CFG),
+			val | ENETC_PM0_TX_EN | ENETC_PM0_RX_EN);
+
+	/* Enable port */
+	val = ENETC_REG_READ(ENETC_GET_HW_ADDR(hw->hw.port, ENETC_PMR));
+	ENETC_REG_WRITE(ENETC_GET_HW_ADDR(hw->hw.port, ENETC_PMR),
+			val | ENETC_PMR_EN);
+
+	return 0;
+}
+
+static void
+enetc_dev_stop(struct rte_eth_dev *dev)
+{
+	struct enetc_eth_hw *hw =
+		ENETC_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t val;
+
+	PMD_INIT_FUNC_TRACE();
+	/* Disable port */
+	val = ENETC_REG_READ(ENETC_GET_HW_ADDR(hw->hw.port, ENETC_PMR));
+	ENETC_REG_WRITE(ENETC_GET_HW_ADDR(hw->hw.port, ENETC_PMR),
+			val & (~ENETC_PMR_EN));
+
+	val = ENETC_REG_READ(ENETC_GET_HW_ADDR(hw->hw.port,
+			     ENETC_PM0_CMD_CFG));
+	ENETC_REG_WRITE(ENETC_GET_HW_ADDR(hw->hw.port, ENETC_PM0_CMD_CFG),
+			val & (~(ENETC_PM0_TX_EN | ENETC_PM0_RX_EN)));
+}
+
+static void
+enetc_dev_close(struct rte_eth_dev *dev)
+{
+	uint16_t i;
+
+	PMD_INIT_FUNC_TRACE();
+	enetc_dev_stop(dev);
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		enetc_rx_queue_release(dev->data->rx_queues[i]);
+		dev->data->rx_queues[i] = NULL;
+	}
+	dev->data->nb_rx_queues = 0;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		enetc_tx_queue_release(dev->data->tx_queues[i]);
+		dev->data->tx_queues[i] = NULL;
+	}
+	dev->data->nb_tx_queues = 0;
+}
+
+static const uint32_t *
+enetc_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
+{
+	static const uint32_t ptypes[] = {
+		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_L3_IPV4,
+		RTE_PTYPE_L3_IPV6,
+		RTE_PTYPE_L4_TCP,
+		RTE_PTYPE_L4_UDP,
+		RTE_PTYPE_L4_SCTP,
+		RTE_PTYPE_L4_ICMP,
+		RTE_PTYPE_UNKNOWN
+	};
+
+	return ptypes;
+}
+
+/* return 0 means link status changed, -1 means not changed */
+static int
+enetc_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
+{
+	struct enetc_eth_hw *hw =
+		ENETC_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_eth_link link;
+	uint32_t status;
+
+	PMD_INIT_FUNC_TRACE();
+
+	memset(&link, 0, sizeof(link));
+
+	status = ENETC_REG_READ(ENETC_GET_HW_ADDR(hw->hw.port,
+				ENETC_PM0_STATUS));
+
+	if (status & ENETC_LINK_MODE)
+		link.link_duplex = ETH_LINK_FULL_DUPLEX;
+	else
+		link.link_duplex = ETH_LINK_HALF_DUPLEX;
+
+	if (status & ENETC_LINK_STATUS)
+		link.link_status = ETH_LINK_UP;
+	else
+		link.link_status = ETH_LINK_DOWN;
+
+	switch (status & ENETC_LINK_SPEED_MASK) {
+	case ENETC_LINK_SPEED_1G:
+		link.link_speed = ETH_SPEED_NUM_1G;
+		break;
+
+	case ENETC_LINK_SPEED_100M:
+		link.link_speed = ETH_SPEED_NUM_100M;
+		break;
+
+	default:
+	case ENETC_LINK_SPEED_10M:
+		link.link_speed = ETH_SPEED_NUM_10M;
+	}
+
+	return rte_eth_linkstatus_set(dev, &link);
+}
+
+static int
+enetc_hardware_init(struct enetc_eth_hw *hw)
+{
+	uint32_t psipmr = 0;
+
+	PMD_INIT_FUNC_TRACE();
+	/* Calculating and storing the base HW addresses */
+	hw->hw.port = (void *)((size_t)hw->hw.reg + ENETC_PORT_BASE);
+	hw->hw.global = (void *)((size_t)hw->hw.reg + ENETC_GLOBAL_BASE);
+
+	/* Enabling Station Interface */
+	ENETC_REG_WRITE(ENETC_GET_HW_ADDR(hw->hw.reg, ENETC_SIMR),
+					  ENETC_SIMR_EN);
+
+	/* Setting to accept broadcast packets for each inetrface */
+	psipmr |= ENETC_PSIPMR_SET_UP(0) | ENETC_PSIPMR_SET_MP(0) |
+		  ENETC_PSIPMR_SET_VLAN_MP(0);
+	psipmr |= ENETC_PSIPMR_SET_UP(1) | ENETC_PSIPMR_SET_MP(1) |
+		  ENETC_PSIPMR_SET_VLAN_MP(1);
+	psipmr |= ENETC_PSIPMR_SET_UP(2) | ENETC_PSIPMR_SET_MP(2) |
+		  ENETC_PSIPMR_SET_VLAN_MP(2);
+
+	ENETC_REG_WRITE(ENETC_GET_HW_ADDR(hw->hw.port, ENETC_PSIPMR),
+			psipmr);
+
+	/* Enabling broadcast address */
+	ENETC_REG_WRITE(ENETC_GET_HW_ADDR(hw->hw.port, ENETC_PSIPMAR0(0)),
+			0xFFFFFFFF);
+	ENETC_REG_WRITE(ENETC_GET_HW_ADDR(hw->hw.port, ENETC_PSIPMAR1(0)),
+			0xFFFF << 16);
+
+	return 0;
+}
+
+static void
+enetc_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
+		    struct rte_eth_dev_info *dev_info)
+{
+	PMD_INIT_FUNC_TRACE();
+	dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
+		.nb_max = MAX_BD_COUNT,
+		.nb_min = MIN_BD_COUNT,
+		.nb_align = BD_ALIGN,
+	};
+	dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
+		.nb_max = MAX_BD_COUNT,
+		.nb_min = MIN_BD_COUNT,
+		.nb_align = BD_ALIGN,
+	};
+	dev_info->max_rx_queues = MAX_RX_RINGS;
+	dev_info->max_tx_queues = MAX_TX_RINGS;
+	dev_info->max_rx_pktlen = 1500;
+}
+
+static int
+enetc_alloc_txbdr(struct enetc_bdr *txr, uint16_t nb_desc)
+{
+	int size;
+
+	size = nb_desc * sizeof(struct enetc_swbd);
+	txr->q_swbd = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (txr->q_swbd == NULL)
+		return -ENOMEM;
+
+	size = nb_desc * sizeof(struct enetc_tx_bd);
+	txr->bd_base = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (txr->bd_base == NULL) {
+		rte_free(txr->q_swbd);
+		txr->q_swbd = NULL;
+		return -ENOMEM;
+	}
+
+	txr->bd_count = nb_desc;
+	txr->next_to_clean = 0;
+	txr->next_to_use = 0;
+
+	return 0;
+}
+
+static void
+enetc_free_bdr(struct enetc_bdr *rxr)
+{
+	rte_free(rxr->q_swbd);
+	rte_free(rxr->bd_base);
+	rxr->q_swbd = NULL;
+	rxr->bd_base = NULL;
+}
+
+static void
+enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
+{
+	int idx = tx_ring->index;
+	uintptr_t base_addr;
+	uint32_t tbmr;
+
+	base_addr = (uintptr_t)tx_ring->bd_base;
+	enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
+		       lower_32_bits((uint64_t)base_addr));
+	enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
+		       upper_32_bits((uint64_t)base_addr));
+	enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
+		       ENETC_RTBLENR_LEN(tx_ring->bd_count));
+
+	tbmr = ENETC_TBMR_EN;
+	/* enable ring */
+	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
+	enetc_txbdr_wr(hw, idx, ENETC_TBCIR, 0);
+	enetc_txbdr_wr(hw, idx, ENETC_TBCISR, 0);
+	tx_ring->tcir = (void *)((size_t)hw->reg +
+			ENETC_BDR(TX, idx, ENETC_TBCIR));
+	tx_ring->tcisr = (void *)((size_t)hw->reg +
+			 ENETC_BDR(TX, idx, ENETC_TBCISR));
+}
+
+static int
+enetc_alloc_tx_resources(struct rte_eth_dev *dev,
+			 uint16_t queue_idx,
+			 uint16_t nb_desc)
+{
+	int err;
+	struct enetc_bdr *tx_ring;
+	struct rte_eth_dev_data *data = dev->data;
+	struct enetc_eth_adapter *priv =
+			ENETC_DEV_PRIVATE(data->dev_private);
+
+	tx_ring = rte_zmalloc(NULL, sizeof(struct enetc_bdr), 0);
+	if (tx_ring == NULL) {
+		ENETC_PMD_ERR("Failed to allocate TX ring memory");
+		err = -ENOMEM;
+		return -1;
+	}
+
+	err = enetc_alloc_txbdr(tx_ring, nb_desc);
+	if (err)
+		goto fail;
+
+	tx_ring->index = queue_idx;
+	tx_ring->ndev = dev;
+	enetc_setup_txbdr(&priv->hw.hw, tx_ring);
+	data->tx_queues[queue_idx] = tx_ring;
+
+	return 0;
+fail:
+	rte_free(tx_ring);
+
+	return err;
+}
+
+static int
+enetc_tx_queue_setup(struct rte_eth_dev *dev,
+		     uint16_t queue_idx,
+		     uint16_t nb_desc,
+		     unsigned int socket_id __rte_unused,
+		     const struct rte_eth_txconf *tx_conf __rte_unused)
+{
+	int err = 0;
+
+	PMD_INIT_FUNC_TRACE();
+	if (nb_desc > MAX_BD_COUNT)
+		return -1;
+
+	err = enetc_alloc_tx_resources(dev, queue_idx, nb_desc);
+
+	return err;
+}
+
+static void
+enetc_tx_queue_release(void *txq)
+{
+	if (txq == NULL)
+		return;
+
+	struct enetc_bdr *tx_ring = (struct enetc_bdr *)txq;
+	struct enetc_eth_hw *eth_hw =
+		ENETC_DEV_PRIVATE_TO_HW(tx_ring->ndev->data->dev_private);
+	struct enetc_hw *hw;
+	struct enetc_swbd *tx_swbd;
+	int i;
+	uint32_t val;
+
+	/* Disable the ring */
+	hw = &eth_hw->hw;
+	val = enetc_txbdr_rd(hw, tx_ring->index, ENETC_TBMR);
+	val &= (~ENETC_TBMR_EN);
+	enetc_txbdr_wr(hw, tx_ring->index, ENETC_TBMR, val);
+
+	/* clean the ring*/
+	i = tx_ring->next_to_clean;
+	tx_swbd = &tx_ring->q_swbd[i];
+	while (tx_swbd->buffer_addr != NULL) {
+		rte_pktmbuf_free(tx_swbd->buffer_addr);
+		tx_swbd->buffer_addr = NULL;
+		tx_swbd++;
+		i++;
+		if (unlikely(i == tx_ring->bd_count)) {
+			i = 0;
+			tx_swbd = &tx_ring->q_swbd[i];
+		}
+	}
+
+	enetc_free_bdr(tx_ring);
+	rte_free(tx_ring);
+}
+
+static int
+enetc_alloc_rxbdr(struct enetc_bdr *rxr,
+		  uint16_t nb_rx_desc)
+{
+	int size;
+
+	size = nb_rx_desc * sizeof(struct enetc_swbd);
+	rxr->q_swbd = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (rxr->q_swbd == NULL)
+		return -ENOMEM;
+
+	size = nb_rx_desc * sizeof(union enetc_rx_bd);
+	rxr->bd_base = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (rxr->bd_base == NULL) {
+		rte_free(rxr->q_swbd);
+		rxr->q_swbd = NULL;
+		return -ENOMEM;
+	}
+
+	rxr->bd_count = nb_rx_desc;
+	rxr->next_to_clean = 0;
+	rxr->next_to_use = 0;
+	rxr->next_to_alloc = 0;
+
+	return 0;
+}
+
+static void
+enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,
+		  struct rte_mempool *mb_pool)
+{
+	int idx = rx_ring->index;
+	uintptr_t base_addr;
+	uint16_t buf_size;
+
+	base_addr = (uintptr_t)rx_ring->bd_base;
+	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
+		       lower_32_bits((uint64_t)base_addr));
+	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
+		       upper_32_bits((uint64_t)base_addr));
+	enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
+		       ENETC_RTBLENR_LEN(rx_ring->bd_count));
+
+	rx_ring->mb_pool = mb_pool;
+	/* enable ring */
+	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, ENETC_RBMR_EN);
+	enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
+	rx_ring->rcir = (void *)((size_t)hw->reg +
+			ENETC_BDR(RX, idx, ENETC_RBCIR));
+	enetc_refill_rx_ring(rx_ring, (enetc_bd_unused(rx_ring)));
+	buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rx_ring->mb_pool) -
+		   RTE_PKTMBUF_HEADROOM);
+	enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, buf_size);
+}
+
+static int
+enetc_alloc_rx_resources(struct rte_eth_dev *dev,
+			 uint16_t rx_queue_id,
+			 uint16_t nb_rx_desc,
+			 struct rte_mempool *mb_pool)
+{
+	int err;
+	struct enetc_bdr *rx_ring;
+	struct rte_eth_dev_data *data =  dev->data;
+	struct enetc_eth_adapter *adapter =
+			ENETC_DEV_PRIVATE(data->dev_private);
+
+	rx_ring = rte_zmalloc(NULL, sizeof(struct enetc_bdr), 0);
+	if (rx_ring == NULL) {
+		ENETC_PMD_ERR("Failed to allocate RX ring memory");
+		err = -ENOMEM;
+		return err;
+	}
+
+	err = enetc_alloc_rxbdr(rx_ring, nb_rx_desc);
+	if (err)
+		goto fail;
+
+	rx_ring->index = rx_queue_id;
+	rx_ring->ndev = dev;
+	enetc_setup_rxbdr(&adapter->hw.hw, rx_ring, mb_pool);
+	data->rx_queues[rx_queue_id] = rx_ring;
+
+	return 0;
+fail:
+	rte_free(rx_ring);
+
+	return err;
+}
+
+static int
+enetc_rx_queue_setup(struct rte_eth_dev *dev,
+		     uint16_t rx_queue_id,
+		     uint16_t nb_rx_desc,
+		     unsigned int socket_id __rte_unused,
+		     const struct rte_eth_rxconf *rx_conf __rte_unused,
+		     struct rte_mempool *mb_pool)
+{
+	int err = 0;
+
+	PMD_INIT_FUNC_TRACE();
+	if (nb_rx_desc > MAX_BD_COUNT)
+		return -1;
+
+	err = enetc_alloc_rx_resources(dev, rx_queue_id,
+				       nb_rx_desc,
+				       mb_pool);
+
+	return err;
+}
+
+static void
+enetc_rx_queue_release(void *rxq)
+{
+	if (rxq == NULL)
+		return;
+
+	struct enetc_bdr *rx_ring = (struct enetc_bdr *)rxq;
+	struct enetc_eth_hw *eth_hw =
+		ENETC_DEV_PRIVATE_TO_HW(rx_ring->ndev->data->dev_private);
+	struct enetc_swbd *q_swbd;
+	struct enetc_hw *hw;
+	uint32_t val;
+	int i;
+
+	/* Disable the ring */
+	hw = &eth_hw->hw;
+	val = enetc_rxbdr_rd(hw, rx_ring->index, ENETC_RBMR);
+	val &= (~ENETC_RBMR_EN);
+	enetc_rxbdr_wr(hw, rx_ring->index, ENETC_RBMR, val);
+
+	/* Clean the ring */
+	i = rx_ring->next_to_clean;
+	q_swbd = &rx_ring->q_swbd[i];
+	while (i != rx_ring->next_to_use) {
+		rte_pktmbuf_free(q_swbd->buffer_addr);
+		q_swbd->buffer_addr = NULL;
+		q_swbd++;
+		i++;
+		if (unlikely(i == rx_ring->bd_count)) {
+			i = 0;
+			q_swbd = &rx_ring->q_swbd[i];
+		}
+	}
+
+	enetc_free_bdr(rx_ring);
+	rte_free(rx_ring);
+}
+
+static int
+enetc_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+			   struct rte_pci_device *pci_dev)
+{
+	return rte_eth_dev_pci_generic_probe(pci_dev,
+					     sizeof(struct enetc_eth_adapter),
+					     enetc_dev_init);
+}
+
+static int
+enetc_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_eth_dev_pci_generic_remove(pci_dev, enetc_dev_uninit);
+}
+
+static struct rte_pci_driver rte_enetc_pmd = {
+	.id_table = pci_id_enetc_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA,
+	.probe = enetc_pci_probe,
+	.remove = enetc_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(net_enetc, rte_enetc_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(net_enetc, pci_id_enetc_map);
+RTE_PMD_REGISTER_KMOD_DEP(net_enetc, "* vfio-pci");
+
+RTE_INIT(enetc_pmd_init_log)
+{
+	enetc_logtype_pmd = rte_log_register("pmd.net.enetc");
+	if (enetc_logtype_pmd >= 0)
+		rte_log_set_level(enetc_logtype_pmd, RTE_LOG_NOTICE);
+}
diff --git a/drivers/net/enetc/enetc_logs.h b/drivers/net/enetc/enetc_logs.h
new file mode 100644
index 00000000..c8a6c0cf
--- /dev/null
+++ b/drivers/net/enetc/enetc_logs.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#ifndef _ENETC_LOGS_H_
+#define _ENETC_LOGS_H_
+
+extern int enetc_logtype_pmd;
+
+#define ENETC_PMD_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, enetc_logtype_pmd, "enetc_net: " \
+		fmt "\n", ##args)
+
+#define ENETC_PMD_DEBUG(fmt, args...) \
+	rte_log(RTE_LOG_DEBUG, enetc_logtype_pmd, "enetc_net: %s(): "\
+		fmt "\n", __func__, ##args)
+
+#define PMD_INIT_FUNC_TRACE() ENETC_PMD_DEBUG(">>")
+
+#define ENETC_PMD_CRIT(fmt, args...) \
+	ENETC_PMD_LOG(CRIT, fmt, ## args)
+#define ENETC_PMD_INFO(fmt, args...) \
+	ENETC_PMD_LOG(INFO, fmt, ## args)
+#define ENETC_PMD_ERR(fmt, args...) \
+	ENETC_PMD_LOG(ERR, fmt, ## args)
+#define ENETC_PMD_WARN(fmt, args...) \
+	ENETC_PMD_LOG(WARNING, fmt, ## args)
+
+/* DP Logs, toggled out at compile time if level lower than current level */
+#define ENETC_PMD_DP_LOG(level, fmt, args...) \
+	RTE_LOG_DP(level, PMD, fmt, ## args)
+
+#define ENETC_PMD_DP_DEBUG(fmt, args...) \
+	ENETC_PMD_DP_LOG(DEBUG, fmt, ## args)
+#define ENETC_PMD_DP_INFO(fmt, args...) \
+	ENETC_PMD_DP_LOG(INFO, fmt, ## args)
+#define ENETC_PMD_DP_WARN(fmt, args...) \
+	ENETC_PMD_DP_LOG(WARNING, fmt, ## args)
+
+#endif /* _ENETC_LOGS_H_*/
diff --git a/drivers/net/enetc/enetc_rxtx.c b/drivers/net/enetc/enetc_rxtx.c
new file mode 100644
index 00000000..631e2430
--- /dev/null
+++ b/drivers/net/enetc/enetc_rxtx.c
@@ -0,0 +1,239 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 NXP
+ */
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <unistd.h>
+
+#include "rte_ethdev.h"
+#include "rte_malloc.h"
+#include "rte_memzone.h"
+
+#include "base/enetc_hw.h"
+#include "enetc.h"
+#include "enetc_logs.h"
+
+#define ENETC_RXBD_BUNDLE 8 /* Number of BDs to update at once */
+
+static int
+enetc_clean_tx_ring(struct enetc_bdr *tx_ring)
+{
+	int tx_frm_cnt = 0;
+	struct enetc_swbd *tx_swbd;
+	int i;
+
+	i = tx_ring->next_to_clean;
+	tx_swbd = &tx_ring->q_swbd[i];
+	while ((int)(enetc_rd_reg(tx_ring->tcisr) &
+	       ENETC_TBCISR_IDX_MASK) != i) {
+		rte_pktmbuf_free(tx_swbd->buffer_addr);
+		tx_swbd->buffer_addr = NULL;
+		tx_swbd++;
+		i++;
+		if (unlikely(i == tx_ring->bd_count)) {
+			i = 0;
+			tx_swbd = &tx_ring->q_swbd[0];
+		}
+
+		tx_frm_cnt++;
+	}
+
+	tx_ring->next_to_clean = i;
+	return tx_frm_cnt++;
+}
+
+uint16_t
+enetc_xmit_pkts(void *tx_queue,
+		struct rte_mbuf **tx_pkts,
+		uint16_t nb_pkts)
+{
+	struct enetc_swbd *tx_swbd;
+	int i, start;
+	struct enetc_tx_bd *txbd;
+	struct enetc_bdr *tx_ring = (struct enetc_bdr *)tx_queue;
+
+	i = tx_ring->next_to_use;
+	start = 0;
+	while (nb_pkts--) {
+		enetc_clean_tx_ring(tx_ring);
+		tx_ring->q_swbd[i].buffer_addr = tx_pkts[start];
+		txbd = ENETC_TXBD(*tx_ring, i);
+		tx_swbd = &tx_ring->q_swbd[i];
+		txbd->frm_len = tx_pkts[start]->pkt_len;
+		txbd->buf_len = txbd->frm_len;
+		txbd->flags = rte_cpu_to_le_16(ENETC_TXBD_FLAGS_F);
+		txbd->addr = (uint64_t)(uintptr_t)
+		rte_cpu_to_le_64((size_t)tx_swbd->buffer_addr->buf_addr +
+				 tx_swbd->buffer_addr->data_off);
+		i++;
+		start++;
+		if (unlikely(i == tx_ring->bd_count))
+			i = 0;
+	}
+
+	tx_ring->next_to_use = i;
+	enetc_wr_reg(tx_ring->tcir, i);
+	return start;
+}
+
+int
+enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
+{
+	struct enetc_swbd *rx_swbd;
+	union enetc_rx_bd *rxbd;
+	int i, j;
+
+	i = rx_ring->next_to_use;
+	rx_swbd = &rx_ring->q_swbd[i];
+	rxbd = ENETC_RXBD(*rx_ring, i);
+	for (j = 0; j < buff_cnt; j++) {
+		rx_swbd->buffer_addr =
+			rte_cpu_to_le_64(rte_mbuf_raw_alloc(rx_ring->mb_pool));
+		rxbd->w.addr = (uint64_t)(uintptr_t)
+			       rx_swbd->buffer_addr->buf_addr +
+			       rx_swbd->buffer_addr->data_off;
+		/* clear 'R" as well */
+		rxbd->r.lstatus = 0;
+		rx_swbd++;
+		rxbd++;
+		i++;
+		if (unlikely(i == rx_ring->bd_count)) {
+			i = 0;
+			rxbd = ENETC_RXBD(*rx_ring, 0);
+			rx_swbd = &rx_ring->q_swbd[i];
+		}
+	}
+
+	if (likely(j)) {
+		rx_ring->next_to_alloc = i;
+		rx_ring->next_to_use = i;
+		enetc_wr_reg(rx_ring->rcir, i);
+	}
+
+	return j;
+}
+
+
+static inline void __attribute__((hot))
+enetc_dev_rx_parse(struct rte_mbuf *m, uint16_t parse_results)
+{
+	ENETC_PMD_DP_DEBUG("parse summary = 0x%x   ", parse_results);
+
+	m->packet_type = RTE_PTYPE_UNKNOWN;
+	switch (parse_results) {
+	case ENETC_PKT_TYPE_ETHER:
+		m->packet_type = RTE_PTYPE_L2_ETHER;
+		break;
+	case ENETC_PKT_TYPE_IPV4:
+		m->packet_type = RTE_PTYPE_L2_ETHER |
+				 RTE_PTYPE_L3_IPV4;
+		break;
+	case ENETC_PKT_TYPE_IPV6:
+		m->packet_type = RTE_PTYPE_L2_ETHER |
+				 RTE_PTYPE_L3_IPV6;
+		break;
+	case ENETC_PKT_TYPE_IPV4_TCP:
+		m->packet_type = RTE_PTYPE_L2_ETHER |
+				 RTE_PTYPE_L3_IPV4 |
+				 RTE_PTYPE_L4_TCP;
+		break;
+	case ENETC_PKT_TYPE_IPV6_TCP:
+		m->packet_type = RTE_PTYPE_L2_ETHER |
+				 RTE_PTYPE_L3_IPV6 |
+				 RTE_PTYPE_L4_TCP;
+		break;
+	case ENETC_PKT_TYPE_IPV4_UDP:
+		m->packet_type = RTE_PTYPE_L2_ETHER |
+				 RTE_PTYPE_L3_IPV4 |
+				 RTE_PTYPE_L4_UDP;
+		break;
+	case ENETC_PKT_TYPE_IPV6_UDP:
+		m->packet_type = RTE_PTYPE_L2_ETHER |
+				 RTE_PTYPE_L3_IPV6 |
+				 RTE_PTYPE_L4_UDP;
+		break;
+	case ENETC_PKT_TYPE_IPV4_SCTP:
+		m->packet_type = RTE_PTYPE_L2_ETHER |
+				 RTE_PTYPE_L3_IPV4 |
+				 RTE_PTYPE_L4_SCTP;
+		break;
+	case ENETC_PKT_TYPE_IPV6_SCTP:
+		m->packet_type = RTE_PTYPE_L2_ETHER |
+				 RTE_PTYPE_L3_IPV6 |
+				 RTE_PTYPE_L4_SCTP;
+		break;
+	case ENETC_PKT_TYPE_IPV4_ICMP:
+		m->packet_type = RTE_PTYPE_L2_ETHER |
+				 RTE_PTYPE_L3_IPV4 |
+				 RTE_PTYPE_L4_ICMP;
+		break;
+	case ENETC_PKT_TYPE_IPV6_ICMP:
+		m->packet_type = RTE_PTYPE_L2_ETHER |
+				 RTE_PTYPE_L3_IPV6 |
+				 RTE_PTYPE_L4_ICMP;
+		break;
+	/* More switch cases can be added */
+	default:
+		m->packet_type = RTE_PTYPE_UNKNOWN;
+	}
+}
+
+static int
+enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
+		    struct rte_mbuf **rx_pkts,
+		    int work_limit)
+{
+	int rx_frm_cnt = 0;
+	int cleaned_cnt, i;
+	struct enetc_swbd *rx_swbd;
+
+	cleaned_cnt = enetc_bd_unused(rx_ring);
+	/* next descriptor to process */
+	i = rx_ring->next_to_clean;
+	rx_swbd = &rx_ring->q_swbd[i];
+	while (likely(rx_frm_cnt < work_limit)) {
+		union enetc_rx_bd *rxbd;
+		uint32_t bd_status;
+
+		if (cleaned_cnt >= ENETC_RXBD_BUNDLE) {
+			int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);
+
+			cleaned_cnt -= count;
+		}
+
+		rxbd = ENETC_RXBD(*rx_ring, i);
+		bd_status = rte_le_to_cpu_32(rxbd->r.lstatus);
+		if (!bd_status)
+			break;
+
+		rx_swbd->buffer_addr->pkt_len = rxbd->r.buf_len;
+		rx_swbd->buffer_addr->data_len = rxbd->r.buf_len;
+		rx_swbd->buffer_addr->hash.rss = rxbd->r.rss_hash;
+		rx_swbd->buffer_addr->ol_flags = 0;
+		enetc_dev_rx_parse(rx_swbd->buffer_addr,
+				   rxbd->r.parse_summary);
+		rx_pkts[rx_frm_cnt] = rx_swbd->buffer_addr;
+		cleaned_cnt++;
+		rx_swbd++;
+		i++;
+		if (unlikely(i == rx_ring->bd_count)) {
+			i = 0;
+			rx_swbd = &rx_ring->q_swbd[i];
+		}
+
+		rx_ring->next_to_clean = i;
+		rx_frm_cnt++;
+	}
+
+	return rx_frm_cnt;
+}
+
+uint16_t
+enetc_recv_pkts(void *rxq, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts)
+{
+	struct enetc_bdr *rx_ring = (struct enetc_bdr *)rxq;
+
+	return enetc_clean_rx_ring(rx_ring, rx_pkts, nb_pkts);
+}
diff --git a/drivers/net/enetc/meson.build b/drivers/net/enetc/meson.build
new file mode 100644
index 00000000..733156bb
--- /dev/null
+++ b/drivers/net/enetc/meson.build
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2018 NXP
+
+if host_machine.system() != 'linux'
+	build = false
+endif
+
+sources = files('enetc_ethdev.c',
+		'enetc_rxtx.c')
+
+includes += include_directories('base')
diff --git a/drivers/net/enetc/rte_pmd_enetc_version.map b/drivers/net/enetc/rte_pmd_enetc_version.map
new file mode 100644
index 00000000..521e51f4
--- /dev/null
+++ b/drivers/net/enetc/rte_pmd_enetc_version.map
@@ -0,0 +1,4 @@
+DPDK_18.11 {
+
+	local: *;
+};
diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile
index 7c6c29cc..e39e4763 100644
--- a/drivers/net/enic/Makefile
+++ b/drivers/net/enic/Makefile
@@ -39,4 +39,32 @@ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_intr.c
 SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_rq.c
 SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_rss.c
 
+# The current implementation assumes 64-bit pointers
+CC_AVX2_SUPPORT=0
+ifeq ($(CONFIG_RTE_ARCH_X86_64),y)
+# Figure out if the compiler supports avx2. The extra check using
+# -march=core-avx2 is necessary to support users who build for the
+# 'default' machine (corei7 which has no avx2) and run the binary on
+# newer CPUs that have avx2.
+# This part is verbatim from i40e makefile.
+ifeq ($(findstring RTE_MACHINE_CPUFLAG_AVX2,$(CFLAGS)),RTE_MACHINE_CPUFLAG_AVX2)
+	CC_AVX2_SUPPORT=1
+else
+	CC_AVX2_SUPPORT=\
+	$(shell $(CC) -march=core-avx2 -dM -E - </dev/null 2>&1 | \
+	grep -q AVX2 && echo 1)
+	ifeq ($(CC_AVX2_SUPPORT), 1)
+		ifeq ($(CONFIG_RTE_TOOLCHAIN_ICC),y)
+			CFLAGS_enic_rxtx_vec_avx2.o += -march=core-avx2
+		else
+			CFLAGS_enic_rxtx_vec_avx2.o += -mavx2
+		endif
+	endif
+endif
+endif
+
+ifeq ($(CC_AVX2_SUPPORT), 1)
+	SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx_vec_avx2.c
+endif
+
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/enic/base/vnic_dev.c b/drivers/net/enic/base/vnic_dev.c
index 16e8814a..fd303fec 100644
--- a/drivers/net/enic/base/vnic_dev.c
+++ b/drivers/net/enic/base/vnic_dev.c
@@ -57,6 +57,9 @@ struct vnic_dev {
 	void (*free_consistent)(void *priv,
 		size_t size, void *vaddr,
 		dma_addr_t dma_handle);
+	struct vnic_counter_counts *flow_counters;
+	dma_addr_t flow_counters_pa;
+	u8 flow_counters_dma_active;
 };
 
 #define VNIC_MAX_RES_HDR_SIZE \
@@ -64,6 +67,8 @@ struct vnic_dev {
 	sizeof(struct vnic_resource) * RES_TYPE_MAX)
 #define VNIC_RES_STRIDE	128
 
+#define VNIC_MAX_FLOW_COUNTERS 2048
+
 void *vnic_dev_priv(struct vnic_dev *vdev)
 {
 	return vdev->priv;
@@ -611,6 +616,35 @@ int vnic_dev_stats_dump(struct vnic_dev *vdev, struct vnic_stats **stats)
 	return vnic_dev_cmd(vdev, CMD_STATS_DUMP, &a0, &a1, wait);
 }
 
+/*
+ * Configure counter DMA
+ */
+int vnic_dev_counter_dma_cfg(struct vnic_dev *vdev, u32 period,
+			     u32 num_counters)
+{
+	u64 args[3];
+	int wait = 1000;
+	int err;
+
+	if (num_counters > VNIC_MAX_FLOW_COUNTERS)
+		return -ENOMEM;
+	if (period > 0 && (period < VNIC_COUNTER_DMA_MIN_PERIOD ||
+	    num_counters == 0))
+		return -EINVAL;
+
+	args[0] = num_counters;
+	args[1] = vdev->flow_counters_pa;
+	args[2] = period;
+	err =  vnic_dev_cmd_args(vdev, CMD_COUNTER_DMA_CONFIG, args, 3, wait);
+
+	/* record if DMAs need to be stopped on close */
+	if (!err)
+		vdev->flow_counters_dma_active = (num_counters != 0 &&
+						  period != 0);
+
+	return err;
+}
+
 int vnic_dev_close(struct vnic_dev *vdev)
 {
 	u64 a0 = 0, a1 = 0;
@@ -939,6 +973,24 @@ int vnic_dev_alloc_stats_mem(struct vnic_dev *vdev)
 	return vdev->stats == NULL ? -ENOMEM : 0;
 }
 
+/*
+ * Initialize for up to VNIC_MAX_FLOW_COUNTERS
+ */
+int vnic_dev_alloc_counter_mem(struct vnic_dev *vdev)
+{
+	char name[NAME_MAX];
+	static u32 instance;
+
+	snprintf((char *)name, sizeof(name), "vnic_flow_ctrs-%u", instance++);
+	vdev->flow_counters = vdev->alloc_consistent(vdev->priv,
+					     sizeof(struct vnic_counter_counts)
+					     * VNIC_MAX_FLOW_COUNTERS,
+					     &vdev->flow_counters_pa,
+					     (u8 *)name);
+	vdev->flow_counters_dma_active = 0;
+	return vdev->flow_counters == NULL ? -ENOMEM : 0;
+}
+
 void vnic_dev_unregister(struct vnic_dev *vdev)
 {
 	if (vdev) {
@@ -951,6 +1003,16 @@ void vnic_dev_unregister(struct vnic_dev *vdev)
 			vdev->free_consistent(vdev->priv,
 				sizeof(struct vnic_stats),
 				vdev->stats, vdev->stats_pa);
+		if (vdev->flow_counters) {
+			/* turn off counter DMAs before freeing memory */
+			if (vdev->flow_counters_dma_active)
+				vnic_dev_counter_dma_cfg(vdev, 0, 0);
+
+			vdev->free_consistent(vdev->priv,
+				sizeof(struct vnic_counter_counts)
+				* VNIC_MAX_FLOW_COUNTERS,
+				vdev->flow_counters, vdev->flow_counters_pa);
+		}
 		if (vdev->fw_info)
 			vdev->free_consistent(vdev->priv,
 				sizeof(struct vnic_devcmd_fw_info),
@@ -1094,3 +1156,46 @@ int vnic_dev_capable_vxlan(struct vnic_dev *vdev)
 		(a1 & (FEATURE_VXLAN_IPV6 | FEATURE_VXLAN_MULTI_WQ)) ==
 		(FEATURE_VXLAN_IPV6 | FEATURE_VXLAN_MULTI_WQ);
 }
+
+bool vnic_dev_counter_alloc(struct vnic_dev *vdev, uint32_t *idx)
+{
+	u64 a0 = 0;
+	u64 a1 = 0;
+	int wait = 1000;
+
+	if (vnic_dev_cmd(vdev, CMD_COUNTER_ALLOC, &a0, &a1, wait))
+		return false;
+	*idx = (uint32_t)a0;
+	return true;
+}
+
+bool vnic_dev_counter_free(struct vnic_dev *vdev, uint32_t idx)
+{
+	u64 a0 = idx;
+	u64 a1 = 0;
+	int wait = 1000;
+
+	return vnic_dev_cmd(vdev, CMD_COUNTER_FREE, &a0, &a1,
+			    wait) == 0;
+}
+
+bool vnic_dev_counter_query(struct vnic_dev *vdev, uint32_t idx,
+			    bool reset, uint64_t *packets, uint64_t *bytes)
+{
+	u64 a0 = idx;
+	u64 a1 = reset ? 1 : 0;
+	int wait = 1000;
+
+	if (reset) {
+		/* query/reset returns updated counters */
+		if (vnic_dev_cmd(vdev, CMD_COUNTER_QUERY, &a0, &a1, wait))
+			return false;
+		*packets = a0;
+		*bytes = a1;
+	} else {
+		/* Get values DMA'd from the adapter */
+		*packets = vdev->flow_counters[idx].vcc_packets;
+		*bytes = vdev->flow_counters[idx].vcc_bytes;
+	}
+	return true;
+}
diff --git a/drivers/net/enic/base/vnic_dev.h b/drivers/net/enic/base/vnic_dev.h
index 270a47bd..de2645c4 100644
--- a/drivers/net/enic/base/vnic_dev.h
+++ b/drivers/net/enic/base/vnic_dev.h
@@ -118,6 +118,8 @@ int vnic_dev_spec(struct vnic_dev *vdev, unsigned int offset, size_t size,
 	void *value);
 int vnic_dev_stats_clear(struct vnic_dev *vdev);
 int vnic_dev_stats_dump(struct vnic_dev *vdev, struct vnic_stats **stats);
+int vnic_dev_counter_dma_cfg(struct vnic_dev *vdev, u32 period,
+			     u32 num_counters);
 int vnic_dev_hang_notify(struct vnic_dev *vdev);
 int vnic_dev_packet_filter(struct vnic_dev *vdev, int directed, int multicast,
 	int broadcast, int promisc, int allmulti);
@@ -170,6 +172,7 @@ struct vnic_dev *vnic_dev_register(struct vnic_dev *vdev,
 	unsigned int num_bars);
 struct rte_pci_device *vnic_dev_get_pdev(struct vnic_dev *vdev);
 int vnic_dev_alloc_stats_mem(struct vnic_dev *vdev);
+int vnic_dev_alloc_counter_mem(struct vnic_dev *vdev);
 int vnic_dev_cmd_init(struct vnic_dev *vdev, int fallback);
 int vnic_dev_get_size(void);
 int vnic_dev_int13(struct vnic_dev *vdev, u64 arg, u32 op);
@@ -187,4 +190,9 @@ int vnic_dev_overlay_offload_ctrl(struct vnic_dev *vdev,
 int vnic_dev_overlay_offload_cfg(struct vnic_dev *vdev, u8 overlay,
 	u16 vxlan_udp_port_number);
 int vnic_dev_capable_vxlan(struct vnic_dev *vdev);
+bool vnic_dev_counter_alloc(struct vnic_dev *vdev, uint32_t *idx);
+bool vnic_dev_counter_free(struct vnic_dev *vdev, uint32_t idx);
+bool vnic_dev_counter_query(struct vnic_dev *vdev, uint32_t idx,
+			    bool reset, uint64_t *packets, uint64_t *bytes);
+
 #endif /* _VNIC_DEV_H_ */
diff --git a/drivers/net/enic/base/vnic_devcmd.h b/drivers/net/enic/base/vnic_devcmd.h
index a22d8a76..3aad2dbd 100644
--- a/drivers/net/enic/base/vnic_devcmd.h
+++ b/drivers/net/enic/base/vnic_devcmd.h
@@ -598,6 +598,48 @@ enum vnic_devcmd_cmd {
 	 *                       a3 = bitmask of supported actions
 	 */
 	CMD_ADD_ADV_FILTER = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ENET, 77),
+
+	/*
+	 * Allocate a counter for use with CMD_ADD_FILTER
+	 * out:(u32) a0 = counter index
+	 */
+	CMD_COUNTER_ALLOC = _CMDC(_CMD_DIR_READ, _CMD_VTYPE_ENET, 85),
+
+	/*
+	 * Free a counter
+	 * in: (u32) a0 = counter_id
+	 */
+	CMD_COUNTER_FREE = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 86),
+
+	/*
+	 * Read a counter
+	 * in: (u32) a0 = counter_id
+	 *     (u32) a1 = clear counter if non-zero
+	 * out:(u64) a0 = packet count
+	 *     (u64) a1 = byte count
+	 */
+	CMD_COUNTER_QUERY = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ENET, 87),
+
+	/*
+	 * Configure periodic counter DMA.  This will trigger an immediate
+	 * DMA of the counters (unless period == 0), and then schedule a DMA
+	 * of the counters every <period> seconds until disdabled.
+	 * Each new COUNTER_DMA_CONFIG will override all previous commands on
+	 * this vnic.
+	 * Setting a2 (period) = 0 will disable periodic DMAs
+	 * If a0 (num_counters) != 0, an immediate DMA will always be done,
+	 * irrespective of the value in a2.
+	 * in: (u32) a0 = number of counters to DMA
+	 *     (u64) a1 = host target DMA address
+	 *     (u32) a2 = DMA period in milliseconds (0 to disable)
+	 */
+	CMD_COUNTER_DMA_CONFIG = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 88),
+#define VNIC_COUNTER_DMA_MIN_PERIOD 500
+
+	/*
+	 * Clear all counters on a vnic
+	 */
+	CMD_COUNTER_CLEAR_ALL = _CMDC(_CMD_DIR_NONE, _CMD_VTYPE_ENET, 89),
 };
 
 /* Modes for exchanging advanced filter capabilities. The modes supported by
@@ -863,9 +905,11 @@ struct filter_action {
 #define FILTER_ACTION_RQ_STEERING_FLAG	(1 << 0)
 #define FILTER_ACTION_FILTER_ID_FLAG	(1 << 1)
 #define FILTER_ACTION_DROP_FLAG		(1 << 2)
+#define FILTER_ACTION_COUNTER_FLAG      (1 << 3)
 #define FILTER_ACTION_V2_ALL		(FILTER_ACTION_RQ_STEERING_FLAG \
+					 | FILTER_ACTION_FILTER_ID_FLAG \
 					 | FILTER_ACTION_DROP_FLAG \
-					 | FILTER_ACTION_FILTER_ID_FLAG)
+					 | FILTER_ACTION_COUNTER_FLAG)
 
 /* Version 2 of filter action must be a strict extension of struct filter_action
  * where the first fields exactly match in size and meaning.
@@ -875,7 +919,8 @@ struct filter_action_v2 {
 	u32 rq_idx;
 	u32 flags;                     /* use FILTER_ACTION_XXX_FLAG defines */
 	u16 filter_id;
-	u_int8_t reserved[32];         /* for future expansion */
+	u32 counter_index;
+	uint8_t reserved[28];         /* for future expansion */
 } __attribute__((packed));
 
 /* Specifies the filter type. */
@@ -941,9 +986,9 @@ enum {
 };
 
 struct filter_tlv {
-	u_int32_t type;
-	u_int32_t length;
-	u_int32_t val[0];
+	uint32_t type;
+	uint32_t length;
+	uint32_t val[0];
 };
 
 /* Data for CMD_ADD_FILTER is 2 TLV and filter + action structs */
@@ -957,10 +1002,10 @@ struct filter_tlv {
  * drivers should use this instead of "sizeof (struct filter_v2)" when
  * computing length for TLV.
  */
-static inline u_int32_t
+static inline uint32_t
 vnic_filter_size(struct filter_v2 *fp)
 {
-	u_int32_t size;
+	uint32_t size;
 
 	switch (fp->type) {
 	case FILTER_USNIC_ID:
@@ -999,10 +1044,10 @@ enum {
  * drivers should use this instead of "sizeof (struct filter_action_v2)"
  * when computing length for TLV.
  */
-static inline u_int32_t
+static inline uint32_t
 vnic_action_size(struct filter_action_v2 *fap)
 {
-	u_int32_t size;
+	uint32_t size;
 
 	switch (fap->type) {
 	case FILTER_ACTION_RQ_STEERING:
@@ -1122,4 +1167,13 @@ typedef enum {
 	GRPINTR_UPD_VECT,
 } grpintr_subcmd_t;
 
+/*
+ * Structure for counter DMA
+ * (DMAed by CMD_COUNTER_DMA_CONFIG)
+ */
+struct vnic_counter_counts {
+	u64 vcc_packets;
+	u64 vcc_bytes;
+};
+
 #endif /* _VNIC_DEVCMD_H_ */
diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
index 7c27bd51..7bca3cad 100644
--- a/drivers/net/enic/enic.h
+++ b/drivers/net/enic/enic.h
@@ -39,6 +39,9 @@
 #define PAGE_ROUND_UP(x) \
 	((((unsigned long)(x)) + ENIC_PAGE_SIZE-1) & (~(ENIC_PAGE_SIZE-1)))
 
+/* must be >= VNIC_COUNTER_DMA_MIN_PERIOD */
+#define VNIC_FLOW_COUNTER_UPDATE_MSECS 500
+
 #define ENICPMD_VFIO_PATH          "/dev/vfio/vfio"
 /*#define ENIC_DESC_COUNT_MAKE_ODD (x) do{if ((~(x)) & 1) { (x)--; } }while(0)*/
 
@@ -94,6 +97,7 @@ struct rte_flow {
 	LIST_ENTRY(rte_flow) next;
 	u16 enic_filter_id;
 	struct filter_v2 enic_filter;
+	int counter_idx; /* NIC allocated counter index (-1 = invalid) */
 };
 
 /* Per-instance private data structure */
@@ -104,6 +108,11 @@ struct enic {
 	struct vnic_dev_bar bar0;
 	struct vnic_dev *vdev;
 
+	/*
+	 * mbuf_initializer contains 64 bits of mbuf rearm_data, used by
+	 * the avx2 handler at this time.
+	 */
+	uint64_t mbuf_initializer;
 	unsigned int port_id;
 	bool overlay_offload;
 	struct rte_eth_dev *rte_dev;
@@ -126,6 +135,7 @@ struct enic {
 	u8 filter_actions; /* HW supported actions */
 	bool vxlan;
 	bool disable_overlay; /* devargs disable_overlay=1 */
+	uint8_t enable_avx2_rx;  /* devargs enable-avx2-rx=1 */
 	bool nic_cfg_chk;     /* NIC_CFG_CHK available */
 	bool udp_rss_weak;    /* Bodega style UDP RSS */
 	uint8_t ig_vlan_rewrite_mode; /* devargs ig-vlan-rewrite */
@@ -165,6 +175,7 @@ struct enic {
 	rte_spinlock_t mtu_lock;
 
 	LIST_HEAD(enic_flows, rte_flow) flows;
+	int max_flow_counter;
 	rte_spinlock_t flows_lock;
 
 	/* RSS */
@@ -326,6 +337,7 @@ uint16_t enic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 			uint16_t nb_pkts);
 int enic_set_mtu(struct enic *enic, uint16_t new_mtu);
 int enic_link_update(struct enic *enic);
+bool enic_use_vector_rx_handler(struct enic *enic);
 void enic_fdir_info(struct enic *enic);
 void enic_fdir_info_get(struct enic *enic, struct rte_eth_fdir_info *stats);
 void copy_fltr_v1(struct filter_v2 *fltr, struct rte_eth_fdir_input *input,
diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c
index b3d57771..996bb554 100644
--- a/drivers/net/enic/enic_ethdev.c
+++ b/drivers/net/enic/enic_ethdev.c
@@ -37,6 +37,7 @@ static const struct rte_pci_id pci_id_enic_map[] = {
 };
 
 #define ENIC_DEVARG_DISABLE_OVERLAY "disable-overlay"
+#define ENIC_DEVARG_ENABLE_AVX2_RX "enable-avx2-rx"
 #define ENIC_DEVARG_IG_VLAN_REWRITE "ig-vlan-rewrite"
 
 RTE_INIT(enicpmd_init_log)
@@ -521,10 +522,34 @@ static const uint32_t *enicpmd_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 		RTE_PTYPE_L4_NONFRAG,
 		RTE_PTYPE_UNKNOWN
 	};
+	static const uint32_t ptypes_overlay[] = {
+		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_L2_ETHER_VLAN,
+		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+		RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+		RTE_PTYPE_L4_TCP,
+		RTE_PTYPE_L4_UDP,
+		RTE_PTYPE_L4_FRAG,
+		RTE_PTYPE_L4_NONFRAG,
+		RTE_PTYPE_TUNNEL_GRENAT,
+		RTE_PTYPE_INNER_L2_ETHER,
+		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
+		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
+		RTE_PTYPE_INNER_L4_TCP,
+		RTE_PTYPE_INNER_L4_UDP,
+		RTE_PTYPE_INNER_L4_FRAG,
+		RTE_PTYPE_INNER_L4_NONFRAG,
+		RTE_PTYPE_UNKNOWN
+	};
 
-	if (dev->rx_pkt_burst == enic_recv_pkts ||
-	    dev->rx_pkt_burst == enic_noscatter_recv_pkts)
-		return ptypes;
+	if (dev->rx_pkt_burst != enic_dummy_recv_pkts &&
+	    dev->rx_pkt_burst != NULL) {
+		struct enic *enic = pmd_priv(dev);
+		if (enic->overlay_offload)
+			return ptypes_overlay;
+		else
+			return ptypes;
+	}
 	return NULL;
 }
 
@@ -915,22 +940,27 @@ static const struct eth_dev_ops enicpmd_eth_dev_ops = {
 	.udp_tunnel_port_del  = enicpmd_dev_udp_tunnel_port_del,
 };
 
-static int enic_parse_disable_overlay(__rte_unused const char *key,
-				      const char *value,
-				      void *opaque)
+static int enic_parse_zero_one(const char *key,
+			       const char *value,
+			       void *opaque)
 {
 	struct enic *enic;
+	bool b;
 
 	enic = (struct enic *)opaque;
 	if (strcmp(value, "0") == 0) {
-		enic->disable_overlay = false;
+		b = false;
 	} else if (strcmp(value, "1") == 0) {
-		enic->disable_overlay = true;
+		b = true;
 	} else {
-		dev_err(enic, "Invalid value for " ENIC_DEVARG_DISABLE_OVERLAY
-			": expected=0|1 given=%s\n", value);
+		dev_err(enic, "Invalid value for %s"
+			": expected=0|1 given=%s\n", key, value);
 		return -EINVAL;
 	}
+	if (strcmp(key, ENIC_DEVARG_DISABLE_OVERLAY) == 0)
+		enic->disable_overlay = b;
+	if (strcmp(key, ENIC_DEVARG_ENABLE_AVX2_RX) == 0)
+		enic->enable_avx2_rx = b;
 	return 0;
 }
 
@@ -971,6 +1001,7 @@ static int enic_check_devargs(struct rte_eth_dev *dev)
 {
 	static const char *const valid_keys[] = {
 		ENIC_DEVARG_DISABLE_OVERLAY,
+		ENIC_DEVARG_ENABLE_AVX2_RX,
 		ENIC_DEVARG_IG_VLAN_REWRITE,
 		NULL};
 	struct enic *enic = pmd_priv(dev);
@@ -979,6 +1010,7 @@ static int enic_check_devargs(struct rte_eth_dev *dev)
 	ENICPMD_FUNC_TRACE();
 
 	enic->disable_overlay = false;
+	enic->enable_avx2_rx = false;
 	enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PASS_THRU;
 	if (!dev->device->devargs)
 		return 0;
@@ -986,7 +1018,9 @@ static int enic_check_devargs(struct rte_eth_dev *dev)
 	if (!kvlist)
 		return -EINVAL;
 	if (rte_kvargs_process(kvlist, ENIC_DEVARG_DISABLE_OVERLAY,
-			       enic_parse_disable_overlay, enic) < 0 ||
+			       enic_parse_zero_one, enic) < 0 ||
+	    rte_kvargs_process(kvlist, ENIC_DEVARG_ENABLE_AVX2_RX,
+			       enic_parse_zero_one, enic) < 0 ||
 	    rte_kvargs_process(kvlist, ENIC_DEVARG_IG_VLAN_REWRITE,
 			       enic_parse_ig_vlan_rewrite, enic) < 0) {
 		rte_kvargs_free(kvlist);
@@ -996,7 +1030,6 @@ static int enic_check_devargs(struct rte_eth_dev *dev)
 	return 0;
 }
 
-struct enic *enicpmd_list_head = NULL;
 /* Initialize the driver
  * It returns 0 on success.
  */
@@ -1044,7 +1077,8 @@ static int eth_enic_pci_remove(struct rte_pci_device *pci_dev)
 
 static struct rte_pci_driver rte_enic_pmd = {
 	.id_table = pci_id_enic_map,
-	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+		     RTE_PCI_DRV_IOVA_AS_VA,
 	.probe = eth_enic_pci_probe,
 	.remove = eth_enic_pci_remove,
 };
@@ -1054,4 +1088,5 @@ RTE_PMD_REGISTER_PCI_TABLE(net_enic, pci_id_enic_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_enic, "* igb_uio | uio_pci_generic | vfio-pci");
 RTE_PMD_REGISTER_PARAM_STRING(net_enic,
 	ENIC_DEVARG_DISABLE_OVERLAY "=0|1 "
+	ENIC_DEVARG_ENABLE_AVX2_RX "=0|1 "
 	ENIC_DEVARG_IG_VLAN_REWRITE "=trunk|untag|priority|pass");
diff --git a/drivers/net/enic/enic_flow.c b/drivers/net/enic/enic_flow.c
index 0cf04aef..bb9ed037 100644
--- a/drivers/net/enic/enic_flow.c
+++ b/drivers/net/enic/enic_flow.c
@@ -289,6 +289,15 @@ static const enum rte_flow_action_type enic_supported_actions_v2_drop[] = {
 	RTE_FLOW_ACTION_TYPE_END,
 };
 
+static const enum rte_flow_action_type enic_supported_actions_v2_count[] = {
+	RTE_FLOW_ACTION_TYPE_QUEUE,
+	RTE_FLOW_ACTION_TYPE_MARK,
+	RTE_FLOW_ACTION_TYPE_FLAG,
+	RTE_FLOW_ACTION_TYPE_DROP,
+	RTE_FLOW_ACTION_TYPE_COUNT,
+	RTE_FLOW_ACTION_TYPE_END,
+};
+
 /** Action capabilities indexed by NIC version information */
 static const struct enic_action_cap enic_action_cap[] = {
 	[FILTER_ACTION_RQ_STEERING_FLAG] = {
@@ -303,6 +312,10 @@ static const struct enic_action_cap enic_action_cap[] = {
 		.actions = enic_supported_actions_v2_drop,
 		.copy_fn = enic_copy_action_v2,
 	},
+	[FILTER_ACTION_COUNTER_FLAG] = {
+		.actions = enic_supported_actions_v2_count,
+		.copy_fn = enic_copy_action_v2,
+	},
 };
 
 static int
@@ -1068,6 +1081,10 @@ enic_copy_action_v2(const struct rte_flow_action actions[],
 			enic_action->flags |= FILTER_ACTION_DROP_FLAG;
 			break;
 		}
+		case RTE_FLOW_ACTION_TYPE_COUNT: {
+			enic_action->flags |= FILTER_ACTION_COUNTER_FLAG;
+			break;
+		}
 		case RTE_FLOW_ACTION_TYPE_VOID:
 			continue;
 		default:
@@ -1112,7 +1129,9 @@ enic_get_action_cap(struct enic *enic)
 	uint8_t actions;
 
 	actions = enic->filter_actions;
-	if (actions & FILTER_ACTION_DROP_FLAG)
+	if (actions & FILTER_ACTION_COUNTER_FLAG)
+		ea = &enic_action_cap[FILTER_ACTION_COUNTER_FLAG];
+	else if (actions & FILTER_ACTION_DROP_FLAG)
 		ea = &enic_action_cap[FILTER_ACTION_DROP_FLAG];
 	else if (actions & FILTER_ACTION_FILTER_ID_FLAG)
 		ea = &enic_action_cap[FILTER_ACTION_FILTER_ID_FLAG];
@@ -1395,8 +1414,10 @@ enic_flow_add_filter(struct enic *enic, struct filter_v2 *enic_filter,
 		   struct rte_flow_error *error)
 {
 	struct rte_flow *flow;
-	int ret;
-	u16 entry;
+	int err;
+	uint16_t entry;
+	int ctr_idx;
+	int last_max_flow_ctr;
 
 	FLOW_TRACE();
 
@@ -1407,20 +1428,64 @@ enic_flow_add_filter(struct enic *enic, struct filter_v2 *enic_filter,
 		return NULL;
 	}
 
+	flow->counter_idx = -1;
+	last_max_flow_ctr = -1;
+	if (enic_action->flags & FILTER_ACTION_COUNTER_FLAG) {
+		if (!vnic_dev_counter_alloc(enic->vdev, (uint32_t *)&ctr_idx)) {
+			rte_flow_error_set(error, ENOMEM,
+					   RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					   NULL, "cannot allocate counter");
+			goto unwind_flow_alloc;
+		}
+		flow->counter_idx = ctr_idx;
+		enic_action->counter_index = ctr_idx;
+
+		/* If index is the largest, increase the counter DMA size */
+		if (ctr_idx > enic->max_flow_counter) {
+			err = vnic_dev_counter_dma_cfg(enic->vdev,
+						 VNIC_FLOW_COUNTER_UPDATE_MSECS,
+						 ctr_idx + 1);
+			if (err) {
+				rte_flow_error_set(error, -err,
+					   RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					   NULL, "counter DMA config failed");
+				goto unwind_ctr_alloc;
+			}
+			last_max_flow_ctr = enic->max_flow_counter;
+			enic->max_flow_counter = ctr_idx;
+		}
+	}
+
 	/* entry[in] is the queue id, entry[out] is the filter Id for delete */
 	entry = enic_action->rq_idx;
-	ret = vnic_dev_classifier(enic->vdev, CLSF_ADD, &entry, enic_filter,
+	err = vnic_dev_classifier(enic->vdev, CLSF_ADD, &entry, enic_filter,
 				  enic_action);
-	if (!ret) {
-		flow->enic_filter_id = entry;
-		flow->enic_filter = *enic_filter;
-	} else {
-		rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_HANDLE,
+	if (err) {
+		rte_flow_error_set(error, -err, RTE_FLOW_ERROR_TYPE_HANDLE,
 				   NULL, "vnic_dev_classifier error");
-		rte_free(flow);
-		return NULL;
+		goto unwind_ctr_dma_cfg;
 	}
+
+	flow->enic_filter_id = entry;
+	flow->enic_filter = *enic_filter;
+
 	return flow;
+
+/* unwind if there are errors */
+unwind_ctr_dma_cfg:
+	if (last_max_flow_ctr != -1) {
+		/* reduce counter DMA size */
+		vnic_dev_counter_dma_cfg(enic->vdev,
+					 VNIC_FLOW_COUNTER_UPDATE_MSECS,
+					 last_max_flow_ctr + 1);
+		enic->max_flow_counter = last_max_flow_ctr;
+	}
+unwind_ctr_alloc:
+	if (flow->counter_idx != -1)
+		vnic_dev_counter_free(enic->vdev, ctr_idx);
+unwind_flow_alloc:
+	rte_free(flow);
+	return NULL;
 }
 
 /**
@@ -1435,18 +1500,29 @@ enic_flow_add_filter(struct enic *enic, struct filter_v2 *enic_filter,
  * @param error[out]
  */
 static int
-enic_flow_del_filter(struct enic *enic, u16 filter_id,
+enic_flow_del_filter(struct enic *enic, struct rte_flow *flow,
 		   struct rte_flow_error *error)
 {
-	int ret;
+	u16 filter_id;
+	int err;
 
 	FLOW_TRACE();
 
-	ret = vnic_dev_classifier(enic->vdev, CLSF_DEL, &filter_id, NULL, NULL);
-	if (!ret)
-		rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_HANDLE,
+	filter_id = flow->enic_filter_id;
+	err = vnic_dev_classifier(enic->vdev, CLSF_DEL, &filter_id, NULL, NULL);
+	if (err) {
+		rte_flow_error_set(error, -err, RTE_FLOW_ERROR_TYPE_HANDLE,
 				   NULL, "vnic_dev_classifier failed");
-	return ret;
+		return -err;
+	}
+
+	if (flow->counter_idx != -1) {
+		if (!vnic_dev_counter_free(enic->vdev, flow->counter_idx))
+			dev_err(enic, "counter free failed, idx: %d\n",
+				flow->counter_idx);
+		flow->counter_idx = -1;
+	}
+	return 0;
 }
 
 /*
@@ -1529,9 +1605,10 @@ enic_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 	FLOW_TRACE();
 
 	rte_spinlock_lock(&enic->flows_lock);
-	enic_flow_del_filter(enic, flow->enic_filter_id, error);
+	enic_flow_del_filter(enic, flow, error);
 	LIST_REMOVE(flow, next);
 	rte_spinlock_unlock(&enic->flows_lock);
+	rte_free(flow);
 	return 0;
 }
 
@@ -1553,13 +1630,77 @@ enic_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
 
 	while (!LIST_EMPTY(&enic->flows)) {
 		flow = LIST_FIRST(&enic->flows);
-		enic_flow_del_filter(enic, flow->enic_filter_id, error);
+		enic_flow_del_filter(enic, flow, error);
 		LIST_REMOVE(flow, next);
+		rte_free(flow);
 	}
 	rte_spinlock_unlock(&enic->flows_lock);
 	return 0;
 }
 
+static int
+enic_flow_query_count(struct rte_eth_dev *dev,
+		      struct rte_flow *flow, void *data,
+		      struct rte_flow_error *error)
+{
+	struct enic *enic = pmd_priv(dev);
+	struct rte_flow_query_count *query;
+	uint64_t packets, bytes;
+
+	FLOW_TRACE();
+
+	if (flow->counter_idx == -1) {
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "flow does not have counter");
+	}
+	query = (struct rte_flow_query_count *)data;
+	if (!vnic_dev_counter_query(enic->vdev, flow->counter_idx,
+				    !!query->reset, &packets, &bytes)) {
+		return rte_flow_error_set
+			(error, EINVAL,
+			 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			 NULL,
+			 "cannot read counter");
+	}
+	query->hits_set = 1;
+	query->bytes_set = 1;
+	query->hits = packets;
+	query->bytes = bytes;
+	return 0;
+}
+
+static int
+enic_flow_query(struct rte_eth_dev *dev,
+		struct rte_flow *flow,
+		const struct rte_flow_action *actions,
+		void *data,
+		struct rte_flow_error *error)
+{
+	int ret = 0;
+
+	FLOW_TRACE();
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = enic_flow_query_count(dev, flow, data, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
+			return ret;
+	}
+	return 0;
+}
+
 /**
  * Flow callback registration.
  *
@@ -1570,4 +1711,5 @@ const struct rte_flow_ops enic_flow_ops = {
 	.create = enic_flow_create,
 	.destroy = enic_flow_destroy,
 	.flush = enic_flow_flush,
+	.query = enic_flow_query,
 };
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index fd940c58..e81c3f3b 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -514,12 +514,29 @@ static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
 	}
 }
 
+/*
+ * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
+ * used when that file is not compiled.
+ */
+bool __attribute__((weak))
+enic_use_vector_rx_handler(__rte_unused struct enic *enic)
+{
+	return false;
+}
+
 static void pick_rx_handler(struct enic *enic)
 {
 	struct rte_eth_dev *eth_dev;
 
-	/* Use the non-scatter, simplified RX handler if possible. */
+	/*
+	 * Preference order:
+	 * 1. The vectorized handler if possible and requested.
+	 * 2. The non-scatter, simplified handler if scatter Rx is not used.
+	 * 3. The default handler as a fallback.
+	 */
 	eth_dev = enic->rte_dev;
+	if (enic_use_vector_rx_handler(enic))
+		return;
 	if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
 		PMD_INIT_LOG(DEBUG, " use the non-scatter Rx handler");
 		eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
@@ -534,6 +551,25 @@ int enic_enable(struct enic *enic)
 	unsigned int index;
 	int err;
 	struct rte_eth_dev *eth_dev = enic->rte_dev;
+	uint64_t simple_tx_offloads;
+	uintptr_t p;
+
+	if (enic->enable_avx2_rx) {
+		struct rte_mbuf mb_def = { .buf_addr = 0 };
+
+		/*
+		 * mbuf_initializer contains const-after-init fields of
+		 * receive mbufs (i.e. 64 bits of fields from rearm_data).
+		 * It is currently used by the vectorized handler.
+		 */
+		mb_def.nb_segs = 1;
+		mb_def.data_off = RTE_PKTMBUF_HEADROOM;
+		mb_def.port = enic->port_id;
+		rte_mbuf_refcnt_set(&mb_def, 1);
+		rte_compiler_barrier();
+		p = (uintptr_t)&mb_def.rearm_data;
+		enic->mbuf_initializer = *(uint64_t *)p;
+	}
 
 	eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
 	eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
@@ -572,10 +608,17 @@ int enic_enable(struct enic *enic)
 	}
 
 	/*
-	 * Use the simple TX handler if possible. All offloads must be
-	 * disabled.
+	 * Use the simple TX handler if possible. Only checksum offloads
+	 * and vlan insertion are supported.
 	 */
-	if (eth_dev->data->dev_conf.txmode.offloads == 0) {
+	simple_tx_offloads = enic->tx_offload_capa &
+		(DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
+		 DEV_TX_OFFLOAD_VLAN_INSERT |
+		 DEV_TX_OFFLOAD_IPV4_CKSUM |
+		 DEV_TX_OFFLOAD_UDP_CKSUM |
+		 DEV_TX_OFFLOAD_TCP_CKSUM);
+	if ((eth_dev->data->dev_conf.txmode.offloads &
+	     ~simple_tx_offloads) == 0) {
 		PMD_INIT_LOG(DEBUG, " use the simple tx handler");
 		eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
 		for (index = 0; index < enic->wq_count; index++)
@@ -1639,11 +1682,25 @@ static int enic_dev_init(struct enic *enic)
 
 	LIST_INIT(&enic->flows);
 	rte_spinlock_init(&enic->flows_lock);
+	enic->max_flow_counter = -1;
 
 	/* set up link status checking */
 	vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
 
 	enic->overlay_offload = false;
+	if (enic->disable_overlay && enic->vxlan) {
+		/*
+		 * Explicitly disable overlay offload as the setting is
+		 * sticky, and resetting vNIC does not disable it.
+		 */
+		if (vnic_dev_overlay_offload_ctrl(enic->vdev,
+						  OVERLAY_FEATURE_VXLAN,
+						  OVERLAY_OFFLOAD_DISABLE)) {
+			dev_err(enic, "failed to disable overlay offload\n");
+		} else {
+			dev_info(enic, "Overlay offload is disabled\n");
+		}
+	}
 	if (!enic->disable_overlay && enic->vxlan &&
 	    /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
 	    vnic_dev_overlay_offload_ctrl(enic->vdev,
@@ -1653,11 +1710,9 @@ static int enic_dev_init(struct enic *enic)
 			DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 			DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
 			DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
-		/*
-		 * Do not add PKT_TX_OUTER_{IPV4,IPV6} as they are not
-		 * 'offload' flags (i.e. not part of PKT_TX_OFFLOAD_MASK).
-		 */
 		enic->tx_offload_mask |=
+			PKT_TX_OUTER_IPV6 |
+			PKT_TX_OUTER_IPV4 |
 			PKT_TX_OUTER_IP_CKSUM |
 			PKT_TX_TUNNEL_MASK;
 		enic->overlay_offload = true;
@@ -1708,14 +1763,20 @@ int enic_probe(struct enic *enic)
 		enic_free_consistent);
 
 	/*
-	 * Allocate the consistent memory for stats upfront so both primary and
-	 * secondary processes can dump stats.
+	 * Allocate the consistent memory for stats and counters upfront so
+	 * both primary and secondary processes can access them.
 	 */
 	err = vnic_dev_alloc_stats_mem(enic->vdev);
 	if (err) {
 		dev_err(enic, "Failed to allocate cmd memory, aborting\n");
 		goto err_out_unregister;
 	}
+	err = vnic_dev_alloc_counter_mem(enic->vdev);
+	if (err) {
+		dev_err(enic, "Failed to allocate counter memory, aborting\n");
+		goto err_out_unregister;
+	}
+
 	/* Issue device open to get device in known state */
 	err = enic_dev_open(enic);
 	if (err) {
diff --git a/drivers/net/enic/enic_res.c b/drivers/net/enic/enic_res.c
index 8d493ffe..24b2844f 100644
--- a/drivers/net/enic/enic_res.c
+++ b/drivers/net/enic/enic_res.c
@@ -85,7 +85,7 @@ int enic_get_vnic_config(struct enic *enic)
 	vnic_dev_capable_udp_rss_weak(enic->vdev, &enic->nic_cfg_chk,
 				      &enic->udp_rss_weak);
 
-	dev_info(enic, "Flow api filter mode: %s Actions: %s%s%s\n",
+	dev_info(enic, "Flow api filter mode: %s Actions: %s%s%s%s\n",
 		((enic->flow_filter_mode == FILTER_DPDK_1) ? "DPDK" :
 		((enic->flow_filter_mode == FILTER_USNIC_IP) ? "USNIC" :
 		((enic->flow_filter_mode == FILTER_IPV4_5TUPLE) ? "5TUPLE" :
@@ -95,7 +95,9 @@ int enic_get_vnic_config(struct enic *enic)
 		((enic->filter_actions & FILTER_ACTION_FILTER_ID_FLAG) ?
 		 "tag " : ""),
 		((enic->filter_actions & FILTER_ACTION_DROP_FLAG) ?
-		 "drop " : ""));
+		 "drop " : ""),
+		((enic->filter_actions & FILTER_ACTION_COUNTER_FLAG) ?
+		 "count " : ""));
 
 	c->wq_desc_count =
 		min_t(u32, ENIC_MAX_WQ_DESCS,
@@ -195,13 +197,14 @@ int enic_get_vnic_config(struct enic *enic)
 	enic->rx_offload_capa =
 		DEV_RX_OFFLOAD_SCATTER |
 		DEV_RX_OFFLOAD_JUMBO_FRAME |
-		DEV_RX_OFFLOAD_CRC_STRIP |
 		DEV_RX_OFFLOAD_VLAN_STRIP |
 		DEV_RX_OFFLOAD_IPV4_CKSUM |
 		DEV_RX_OFFLOAD_UDP_CKSUM |
 		DEV_RX_OFFLOAD_TCP_CKSUM;
 	enic->tx_offload_mask =
-		PKT_TX_VLAN_PKT |
+		PKT_TX_IPV6 |
+		PKT_TX_IPV4 |
+		PKT_TX_VLAN |
 		PKT_TX_IP_CKSUM |
 		PKT_TX_L4_MASK |
 		PKT_TX_TCP_SEG;
diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c
index 7129e121..5189ee63 100644
--- a/drivers/net/enic/enic_rxtx.c
+++ b/drivers/net/enic/enic_rxtx.c
@@ -11,6 +11,7 @@
 #include "enic_compat.h"
 #include "rq_enet_desc.h"
 #include "enic.h"
+#include "enic_rxtx_common.h"
 #include <rte_ether.h>
 #include <rte_ip.h>
 #include <rte_tcp.h>
@@ -30,266 +31,6 @@
 #define rte_packet_prefetch(p) do {} while (0)
 #endif
 
-static inline uint16_t
-enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
-{
-	return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
-}
-
-static inline uint16_t
-enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
-{
-	return le16_to_cpu(crd->bytes_written_flags) &
-			   ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
-}
-
-static inline uint8_t
-enic_cq_rx_desc_packet_error(uint16_t bwflags)
-{
-	return (bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
-		CQ_ENET_RQ_DESC_FLAGS_TRUNCATED;
-}
-
-static inline uint8_t
-enic_cq_rx_desc_eop(uint16_t ciflags)
-{
-	return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
-		== CQ_ENET_RQ_DESC_FLAGS_EOP;
-}
-
-static inline uint8_t
-enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
-{
-	return (le16_to_cpu(cqrd->q_number_rss_type_flags) &
-		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
-		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC;
-}
-
-static inline uint8_t
-enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
-{
-	return (cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
-		CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK;
-}
-
-static inline uint8_t
-enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
-{
-	return (cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
-		CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK;
-}
-
-static inline uint8_t
-enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
-{
-	return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
-		CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
-}
-
-static inline uint32_t
-enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
-{
-	return le32_to_cpu(cqrd->rss_hash);
-}
-
-static inline uint16_t
-enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
-{
-	return le16_to_cpu(cqrd->vlan);
-}
-
-static inline uint16_t
-enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
-{
-	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-	return le16_to_cpu(cqrd->bytes_written_flags) &
-		CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
-}
-
-
-static inline uint8_t
-enic_cq_rx_check_err(struct cq_desc *cqd)
-{
-	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-	uint16_t bwflags;
-
-	bwflags = enic_cq_rx_desc_bwflags(cqrd);
-	if (unlikely(enic_cq_rx_desc_packet_error(bwflags)))
-		return 1;
-	return 0;
-}
-
-/* Lookup table to translate RX CQ flags to mbuf flags. */
-static inline uint32_t
-enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd, uint8_t tnl)
-{
-	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-	uint8_t cqrd_flags = cqrd->flags;
-	/*
-	 * Odd-numbered entries are for tunnel packets. All packet type info
-	 * applies to the inner packet, and there is no info on the outer
-	 * packet. The outer flags in these entries exist only to avoid
-	 * changing enic_cq_rx_to_pkt_flags(). They are cleared from mbuf
-	 * afterwards.
-	 *
-	 * Also, as there is no tunnel type info (VXLAN, NVGRE, or GENEVE), set
-	 * RTE_PTYPE_TUNNEL_GRENAT..
-	 */
-	static const uint32_t cq_type_table[128] __rte_cache_aligned = {
-		[0x00] = RTE_PTYPE_UNKNOWN,
-		[0x01] = RTE_PTYPE_UNKNOWN |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER,
-		[0x20] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG,
-		[0x21] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_NONFRAG,
-		[0x22] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
-		[0x23] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_UDP,
-		[0x24] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
-		[0x25] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_TCP,
-		[0x60] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
-		[0x61] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_FRAG,
-		[0x62] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
-		[0x63] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_FRAG,
-		[0x64] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
-		[0x65] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_FRAG,
-		[0x10] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG,
-		[0x11] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_NONFRAG,
-		[0x12] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
-		[0x13] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_UDP,
-		[0x14] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
-		[0x15] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_TCP,
-		[0x50] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
-		[0x51] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_FRAG,
-		[0x52] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
-		[0x53] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_FRAG,
-		[0x54] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
-		[0x55] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
-			 RTE_PTYPE_TUNNEL_GRENAT |
-			 RTE_PTYPE_INNER_L2_ETHER |
-			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_FRAG,
-		/* All others reserved */
-	};
-	cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
-		| CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
-		| CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
-	return cq_type_table[cqrd_flags + tnl];
-}
-
-static inline void
-enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
-{
-	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-	uint16_t bwflags, pkt_flags = 0, vlan_tci;
-	bwflags = enic_cq_rx_desc_bwflags(cqrd);
-	vlan_tci = enic_cq_rx_desc_vlan(cqrd);
-
-	/* VLAN STRIPPED flag. The L2 packet type updated here also */
-	if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
-		pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
-		mbuf->packet_type |= RTE_PTYPE_L2_ETHER;
-	} else {
-		if (vlan_tci != 0)
-			mbuf->packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
-		else
-			mbuf->packet_type |= RTE_PTYPE_L2_ETHER;
-	}
-	mbuf->vlan_tci = vlan_tci;
-
-	if ((cqd->type_color & CQ_DESC_TYPE_MASK) == CQ_DESC_TYPE_CLASSIFIER) {
-		struct cq_enet_rq_clsf_desc *clsf_cqd;
-		uint16_t filter_id;
-		clsf_cqd = (struct cq_enet_rq_clsf_desc *)cqd;
-		filter_id = clsf_cqd->filter_id;
-		if (filter_id) {
-			pkt_flags |= PKT_RX_FDIR;
-			if (filter_id != ENIC_MAGIC_FILTER_ID) {
-				mbuf->hash.fdir.hi = clsf_cqd->filter_id;
-				pkt_flags |= PKT_RX_FDIR_ID;
-			}
-		}
-	} else if (enic_cq_rx_desc_rss_type(cqrd)) {
-		/* RSS flag */
-		pkt_flags |= PKT_RX_RSS_HASH;
-		mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
-	}
-
-	/* checksum flags */
-	if (mbuf->packet_type & (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV6)) {
-		if (!enic_cq_rx_desc_csum_not_calc(cqrd)) {
-			uint32_t l4_flags;
-			l4_flags = mbuf->packet_type & RTE_PTYPE_L4_MASK;
-
-			/*
-			 * When overlay offload is enabled, the NIC may
-			 * set ipv4_csum_ok=1 if the inner packet is IPv6..
-			 * So, explicitly check for IPv4 before checking
-			 * ipv4_csum_ok.
-			 */
-			if (mbuf->packet_type & RTE_PTYPE_L3_IPV4) {
-				if (enic_cq_rx_desc_ipv4_csum_ok(cqrd))
-					pkt_flags |= PKT_RX_IP_CKSUM_GOOD;
-				else
-					pkt_flags |= PKT_RX_IP_CKSUM_BAD;
-			}
-
-			if (l4_flags == RTE_PTYPE_L4_UDP ||
-			    l4_flags == RTE_PTYPE_L4_TCP) {
-				if (enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))
-					pkt_flags |= PKT_RX_L4_CKSUM_GOOD;
-				else
-					pkt_flags |= PKT_RX_L4_CKSUM_BAD;
-			}
-		}
-	}
-
-	mbuf->ol_flags = pkt_flags;
-}
-
 /* dummy receive function to replace actual function in
  * order to do safe reconfiguration operations.
  */
@@ -707,7 +448,7 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	wq_desc_avail = vnic_wq_desc_avail(wq);
 	head_idx = wq->head_idx;
 	desc_count = wq->ring.desc_count;
-	ol_flags_mask = PKT_TX_VLAN_PKT | PKT_TX_IP_CKSUM | PKT_TX_L4_MASK;
+	ol_flags_mask = PKT_TX_VLAN | PKT_TX_IP_CKSUM | PKT_TX_L4_MASK;
 	tx_oversized = &enic->soft_stats.tx_oversized;
 
 	nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX);
@@ -735,7 +476,7 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 		mss = 0;
 		vlan_id = tx_pkt->vlan_tci;
-		vlan_tag_insert = !!(ol_flags & PKT_TX_VLAN_PKT);
+		vlan_tag_insert = !!(ol_flags & PKT_TX_VLAN);
 		bus_addr = (dma_addr_t)
 			   (tx_pkt->buf_iova + tx_pkt->data_off);
 
@@ -840,12 +581,33 @@ static void enqueue_simple_pkts(struct rte_mbuf **pkts,
 				struct enic *enic)
 {
 	struct rte_mbuf *p;
+	uint16_t mss;
 
 	while (n) {
 		n--;
 		p = *pkts++;
 		desc->address = p->buf_iova + p->data_off;
 		desc->length = p->pkt_len;
+		/* VLAN insert */
+		desc->vlan_tag = p->vlan_tci;
+		desc->header_length_flags &=
+			((1 << WQ_ENET_FLAGS_EOP_SHIFT) |
+			 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT));
+		if (p->ol_flags & PKT_TX_VLAN) {
+			desc->header_length_flags |=
+				1 << WQ_ENET_FLAGS_VLAN_TAG_INSERT_SHIFT;
+		}
+		/*
+		 * Checksum offload. We use WQ_ENET_OFFLOAD_MODE_CSUM, which
+		 * is 0, so no need to set offload_mode.
+		 */
+		mss = 0;
+		if (p->ol_flags & PKT_TX_IP_CKSUM)
+			mss |= ENIC_CALC_IP_CKSUM << WQ_ENET_MSS_SHIFT;
+		if (p->ol_flags & PKT_TX_L4_MASK)
+			mss |= ENIC_CALC_TCP_UDP_CKSUM << WQ_ENET_MSS_SHIFT;
+		desc->mss_loopback = mss;
+
 		/*
 		 * The app should not send oversized
 		 * packets. tx_pkt_prepare includes a check as
diff --git a/drivers/net/enic/enic_rxtx_common.h b/drivers/net/enic/enic_rxtx_common.h
new file mode 100644
index 00000000..bfbb4909
--- /dev/null
+++ b/drivers/net/enic/enic_rxtx_common.h
@@ -0,0 +1,271 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2008-2018 Cisco Systems, Inc.  All rights reserved.
+ * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
+ */
+
+#ifndef _ENIC_RXTX_COMMON_H_
+#define _ENIC_RXTX_COMMON_H_
+
+static inline uint16_t
+enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
+{
+	return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
+}
+
+static inline uint16_t
+enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
+{
+	return le16_to_cpu(crd->bytes_written_flags) &
+			   ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
+}
+
+static inline uint8_t
+enic_cq_rx_desc_packet_error(uint16_t bwflags)
+{
+	return (bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
+		CQ_ENET_RQ_DESC_FLAGS_TRUNCATED;
+}
+
+static inline uint8_t
+enic_cq_rx_desc_eop(uint16_t ciflags)
+{
+	return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
+		== CQ_ENET_RQ_DESC_FLAGS_EOP;
+}
+
+static inline uint8_t
+enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
+{
+	return (le16_to_cpu(cqrd->q_number_rss_type_flags) &
+		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
+		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC;
+}
+
+static inline uint8_t
+enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
+{
+	return (cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
+		CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK;
+}
+
+static inline uint8_t
+enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
+{
+	return (cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
+		CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK;
+}
+
+static inline uint8_t
+enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
+{
+	return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
+		CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
+}
+
+static inline uint32_t
+enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
+{
+	return le32_to_cpu(cqrd->rss_hash);
+}
+
+static inline uint16_t
+enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
+{
+	return le16_to_cpu(cqrd->vlan);
+}
+
+static inline uint16_t
+enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
+{
+	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+	return le16_to_cpu(cqrd->bytes_written_flags) &
+		CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
+}
+
+
+static inline uint8_t
+enic_cq_rx_check_err(struct cq_desc *cqd)
+{
+	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+	uint16_t bwflags;
+
+	bwflags = enic_cq_rx_desc_bwflags(cqrd);
+	if (unlikely(enic_cq_rx_desc_packet_error(bwflags)))
+		return 1;
+	return 0;
+}
+
+/* Lookup table to translate RX CQ flags to mbuf flags. */
+static uint32_t
+enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd, uint8_t tnl)
+{
+	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+	uint8_t cqrd_flags = cqrd->flags;
+	/*
+	 * Odd-numbered entries are for tunnel packets. All packet type info
+	 * applies to the inner packet, and there is no info on the outer
+	 * packet. The outer flags in these entries exist only to avoid
+	 * changing enic_cq_rx_to_pkt_flags(). They are cleared from mbuf
+	 * afterwards.
+	 *
+	 * Also, as there is no tunnel type info (VXLAN, NVGRE, or GENEVE), set
+	 * RTE_PTYPE_TUNNEL_GRENAT..
+	 */
+	static const uint32_t cq_type_table[128] __rte_cache_aligned = {
+		[0x00] = RTE_PTYPE_UNKNOWN,
+		[0x01] = RTE_PTYPE_UNKNOWN |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER,
+		[0x20] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG,
+		[0x21] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_NONFRAG,
+		[0x22] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
+		[0x23] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_UDP,
+		[0x24] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
+		[0x25] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_TCP,
+		[0x60] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
+		[0x61] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_FRAG,
+		[0x62] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
+		[0x63] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_FRAG,
+		[0x64] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
+		[0x65] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_FRAG,
+		[0x10] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG,
+		[0x11] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_NONFRAG,
+		[0x12] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
+		[0x13] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_UDP,
+		[0x14] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
+		[0x15] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_TCP,
+		[0x50] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
+		[0x51] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_FRAG,
+		[0x52] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
+		[0x53] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_FRAG,
+		[0x54] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
+		[0x55] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_FRAG,
+		/* All others reserved */
+	};
+	cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
+		| CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
+		| CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
+	return cq_type_table[cqrd_flags + tnl];
+}
+
+static void
+enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
+{
+	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+	uint16_t bwflags, pkt_flags = 0, vlan_tci;
+	bwflags = enic_cq_rx_desc_bwflags(cqrd);
+	vlan_tci = enic_cq_rx_desc_vlan(cqrd);
+
+	/* VLAN STRIPPED flag. The L2 packet type updated here also */
+	if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
+		pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
+		mbuf->packet_type |= RTE_PTYPE_L2_ETHER;
+	} else {
+		if (vlan_tci != 0) {
+			pkt_flags |= PKT_RX_VLAN;
+			mbuf->packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
+		} else {
+			mbuf->packet_type |= RTE_PTYPE_L2_ETHER;
+		}
+	}
+	mbuf->vlan_tci = vlan_tci;
+
+	if ((cqd->type_color & CQ_DESC_TYPE_MASK) == CQ_DESC_TYPE_CLASSIFIER) {
+		struct cq_enet_rq_clsf_desc *clsf_cqd;
+		uint16_t filter_id;
+		clsf_cqd = (struct cq_enet_rq_clsf_desc *)cqd;
+		filter_id = clsf_cqd->filter_id;
+		if (filter_id) {
+			pkt_flags |= PKT_RX_FDIR;
+			if (filter_id != ENIC_MAGIC_FILTER_ID) {
+				mbuf->hash.fdir.hi = clsf_cqd->filter_id;
+				pkt_flags |= PKT_RX_FDIR_ID;
+			}
+		}
+	} else if (enic_cq_rx_desc_rss_type(cqrd)) {
+		/* RSS flag */
+		pkt_flags |= PKT_RX_RSS_HASH;
+		mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
+	}
+
+	/* checksum flags */
+	if (mbuf->packet_type & (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV6)) {
+		if (!enic_cq_rx_desc_csum_not_calc(cqrd)) {
+			uint32_t l4_flags;
+			l4_flags = mbuf->packet_type & RTE_PTYPE_L4_MASK;
+
+			/*
+			 * When overlay offload is enabled, the NIC may
+			 * set ipv4_csum_ok=1 if the inner packet is IPv6..
+			 * So, explicitly check for IPv4 before checking
+			 * ipv4_csum_ok.
+			 */
+			if (mbuf->packet_type & RTE_PTYPE_L3_IPV4) {
+				if (enic_cq_rx_desc_ipv4_csum_ok(cqrd))
+					pkt_flags |= PKT_RX_IP_CKSUM_GOOD;
+				else
+					pkt_flags |= PKT_RX_IP_CKSUM_BAD;
+			}
+
+			if (l4_flags == RTE_PTYPE_L4_UDP ||
+			    l4_flags == RTE_PTYPE_L4_TCP) {
+				if (enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))
+					pkt_flags |= PKT_RX_L4_CKSUM_GOOD;
+				else
+					pkt_flags |= PKT_RX_L4_CKSUM_BAD;
+			}
+		}
+	}
+
+	mbuf->ol_flags = pkt_flags;
+}
+
+#endif /* _ENIC_RXTX_COMMON_H_ */
diff --git a/drivers/net/enic/enic_rxtx_vec_avx2.c b/drivers/net/enic/enic_rxtx_vec_avx2.c
new file mode 100644
index 00000000..d2185490
--- /dev/null
+++ b/drivers/net/enic/enic_rxtx_vec_avx2.c
@@ -0,0 +1,831 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2008-2018 Cisco Systems, Inc.  All rights reserved.
+ * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
+ */
+
+#include <rte_mbuf.h>
+#include <rte_ethdev_driver.h>
+
+#include "enic_compat.h"
+#include "rq_enet_desc.h"
+#include "enic.h"
+#include "enic_rxtx_common.h"
+
+#include <x86intrin.h>
+
+static struct rte_mbuf *
+rx_one(struct cq_enet_rq_desc *cqd, struct rte_mbuf *mb, struct enic *enic)
+{
+	bool tnl;
+
+	*(uint64_t *)&mb->rearm_data = enic->mbuf_initializer;
+	mb->data_len = cqd->bytes_written_flags &
+		CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
+	mb->pkt_len = mb->data_len;
+	tnl = enic->overlay_offload && (cqd->completed_index_flags &
+					CQ_ENET_RQ_DESC_FLAGS_FCOE) != 0;
+	mb->packet_type =
+		enic_cq_rx_flags_to_pkt_type((struct cq_desc *)cqd, tnl);
+	enic_cq_rx_to_pkt_flags((struct cq_desc *)cqd, mb);
+	/* Wipe the outer types set by enic_cq_rx_flags_to_pkt_type() */
+	if (tnl) {
+		mb->packet_type &= ~(RTE_PTYPE_L3_MASK |
+				     RTE_PTYPE_L4_MASK);
+	}
+	return mb;
+}
+
+static uint16_t
+enic_noscatter_vec_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+			     uint16_t nb_pkts)
+{
+	struct rte_mbuf **rx, **rxmb;
+	uint16_t cq_idx, nb_rx, max_rx;
+	struct cq_enet_rq_desc *cqd;
+	struct rq_enet_desc *rqd;
+	struct vnic_cq *cq;
+	struct vnic_rq *rq;
+	struct enic *enic;
+	uint8_t color;
+
+	rq = rx_queue;
+	enic = vnic_dev_priv(rq->vdev);
+	cq = &enic->cq[enic_cq_rq(enic, rq->index)];
+	cq_idx = cq->to_clean;
+
+	/*
+	 * Fill up the reserve of free mbufs. Below, we restock the receive
+	 * ring with these mbufs to avoid allocation failures.
+	 */
+	if (rq->num_free_mbufs == 0) {
+		if (rte_mempool_get_bulk(rq->mp, (void **)rq->free_mbufs,
+					 ENIC_RX_BURST_MAX))
+			return 0;
+		rq->num_free_mbufs = ENIC_RX_BURST_MAX;
+	}
+	/* Receive until the end of the ring, at most. */
+	max_rx = RTE_MIN(nb_pkts, rq->num_free_mbufs);
+	max_rx = RTE_MIN(max_rx, cq->ring.desc_count - cq_idx);
+
+	rxmb = rq->mbuf_ring + cq_idx;
+	color = cq->last_color;
+	cqd = (struct cq_enet_rq_desc *)(cq->ring.descs) + cq_idx;
+	rx = rx_pkts;
+	if (max_rx == 0 ||
+	    (cqd->type_color & CQ_DESC_COLOR_MASK_NOSHIFT) == color)
+		return 0;
+
+	/* Step 1: Process one packet to do aligned 256-bit load below */
+	if (cq_idx & 0x1) {
+		if (unlikely(cqd->bytes_written_flags &
+			     CQ_ENET_RQ_DESC_FLAGS_TRUNCATED)) {
+			rte_pktmbuf_free(*rxmb++);
+			rte_atomic64_inc(&enic->soft_stats.rx_packet_errors);
+		} else {
+			*rx++ = rx_one(cqd, *rxmb++, enic);
+		}
+		cqd++;
+		max_rx--;
+	}
+
+	const __m256i mask =
+		_mm256_set_epi8(/* Second descriptor */
+			0xff, /* type_color */
+			(CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT |
+			 CQ_ENET_RQ_DESC_FLAGS_IPV4 |
+			 CQ_ENET_RQ_DESC_FLAGS_IPV6 |
+			 CQ_ENET_RQ_DESC_FLAGS_TCP |
+			 CQ_ENET_RQ_DESC_FLAGS_UDP), /* flags */
+			0, 0, /* checksum_fcoe */
+			0xff, 0xff, /* vlan */
+			0x3f, 0xff, /* bytes_written_flags */
+			0xff, 0xff, 0xff, 0xff, /* rss_hash */
+			0xff, 0xff, /* q_number_rss_type_flags */
+			0, 0, /* completed_index_flags */
+			/* First descriptor */
+			0xff, /* type_color */
+			(CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT |
+			 CQ_ENET_RQ_DESC_FLAGS_IPV4 |
+			 CQ_ENET_RQ_DESC_FLAGS_IPV6 |
+			 CQ_ENET_RQ_DESC_FLAGS_TCP |
+			 CQ_ENET_RQ_DESC_FLAGS_UDP), /* flags */
+			0, 0, /* checksum_fcoe */
+			0xff, 0xff, /* vlan */
+			0x3f, 0xff, /* bytes_written_flags */
+			0xff, 0xff, 0xff, 0xff, /* rss_hash */
+			0xff, 0xff, /* q_number_rss_type_flags */
+			0, 0 /* completed_index_flags */
+			);
+	const __m256i shuffle_mask =
+		_mm256_set_epi8(/* Second descriptor */
+			7, 6, 5, 4,             /* rss = rss_hash */
+			11, 10,                 /* vlan_tci = vlan */
+			9, 8,                   /* data_len = bytes_written */
+			0x80, 0x80, 9, 8,       /* pkt_len = bytes_written */
+			0x80, 0x80, 0x80, 0x80, /* packet_type = 0 */
+			/* First descriptor */
+			7, 6, 5, 4,             /* rss = rss_hash */
+			11, 10,                 /* vlan_tci = vlan */
+			9, 8,                   /* data_len = bytes_written */
+			0x80, 0x80, 9, 8,       /* pkt_len = bytes_written */
+			0x80, 0x80, 0x80, 0x80  /* packet_type = 0 */
+			);
+	/* Used to collect 8 flags from 8 desc into one register */
+	const __m256i flags_shuffle_mask =
+		_mm256_set_epi8(/* Second descriptor */
+			1, 3, 9, 14,
+			1, 3, 9, 14,
+			1, 3, 9, 14,
+			1, 3, 9, 14,
+			/* First descriptor */
+			1, 3, 9, 14,
+			1, 3, 9, 14,
+			1, 3, 9, 14,
+			/*
+			 * Byte 3: upper byte of completed_index_flags
+			 *         bit 5 = fcoe (tunnel)
+			 * Byte 2: upper byte of q_number_rss_type_flags
+			 *         bits 2,3,4,5 = rss type
+			 *         bit 6 = csum_not_calc
+			 * Byte 1: upper byte of bytes_written_flags
+			 *         bit 6 = truncated
+			 *         bit 7 = vlan stripped
+			 * Byte 0: flags
+			 */
+			1, 3, 9, 14
+			);
+	/* Used to collect 8 VLAN IDs from 8 desc into one register */
+	const __m256i vlan_shuffle_mask =
+		_mm256_set_epi8(/* Second descriptor */
+			0x80, 0x80, 11, 10,
+			0x80, 0x80, 11, 10,
+			0x80, 0x80, 11, 10,
+			0x80, 0x80, 11, 10,
+			/* First descriptor */
+			0x80, 0x80, 11, 10,
+			0x80, 0x80, 11, 10,
+			0x80, 0x80, 11, 10,
+			0x80, 0x80, 11, 10);
+	/* PKT_RX_RSS_HASH is 1<<1 so fits in 8-bit integer */
+	const __m256i rss_shuffle =
+		_mm256_set_epi8(/* second 128 bits */
+			PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
+			PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
+			PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
+			PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
+			PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
+			0, /* rss_types = 0 */
+			/* first 128 bits */
+			PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
+			PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
+			PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
+			PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
+			PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
+			0 /* rss_types = 0 */);
+	/*
+	 * VLAN offload flags.
+	 * shuffle index:
+	 * vlan_stripped => bit 0
+	 * vlan_id == 0  => bit 1
+	 */
+	const __m256i vlan_shuffle =
+		_mm256_set_epi32(0, 0, 0, 0,
+			PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, 0,
+			PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, PKT_RX_VLAN);
+	/* Use the same shuffle index as vlan_shuffle */
+	const __m256i vlan_ptype_shuffle =
+		_mm256_set_epi32(0, 0, 0, 0,
+				 RTE_PTYPE_L2_ETHER,
+				 RTE_PTYPE_L2_ETHER,
+				 RTE_PTYPE_L2_ETHER,
+				 RTE_PTYPE_L2_ETHER_VLAN);
+	/*
+	 * CKSUM flags. Shift right so they fit int 8-bit integers.
+	 * shuffle index:
+	 * ipv4_csum_ok    => bit 3
+	 * ip4             => bit 2
+	 * tcp_or_udp      => bit 1
+	 * tcp_udp_csum_ok => bit 0
+	 */
+	const __m256i csum_shuffle =
+		_mm256_set_epi8(/* second 128 bits */
+			/* 1111 ip4+ip4_ok+l4+l4_ok */
+			((PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1),
+			/* 1110 ip4_ok+ip4+l4+!l4_ok */
+			((PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> 1),
+			(PKT_RX_IP_CKSUM_GOOD >> 1), /* 1101 ip4+ip4_ok */
+			(PKT_RX_IP_CKSUM_GOOD >> 1), /* 1100 ip4_ok+ip4 */
+			(PKT_RX_L4_CKSUM_GOOD >> 1), /* 1011 l4+l4_ok */
+			(PKT_RX_L4_CKSUM_BAD >> 1),  /* 1010 l4+!l4_ok */
+			0, /* 1001 */
+			0, /* 1000 */
+			/* 0111 !ip4_ok+ip4+l4+l4_ok */
+			((PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD) >> 1),
+			/* 0110 !ip4_ok+ip4+l4+!l4_ok */
+			((PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD) >> 1),
+			(PKT_RX_IP_CKSUM_BAD >> 1),  /* 0101 !ip4_ok+ip4 */
+			(PKT_RX_IP_CKSUM_BAD >> 1),  /* 0100 !ip4_ok+ip4 */
+			(PKT_RX_L4_CKSUM_GOOD >> 1), /* 0011 l4+l4_ok */
+			(PKT_RX_L4_CKSUM_BAD >> 1),  /* 0010 l4+!l4_ok */
+			0, /* 0001 */
+			0, /* 0000 */
+			/* first 128 bits */
+			((PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1),
+			((PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> 1),
+			(PKT_RX_IP_CKSUM_GOOD >> 1),
+			(PKT_RX_IP_CKSUM_GOOD >> 1),
+			(PKT_RX_L4_CKSUM_GOOD >> 1),
+			(PKT_RX_L4_CKSUM_BAD >> 1),
+			0, 0,
+			((PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD) >> 1),
+			((PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD) >> 1),
+			(PKT_RX_IP_CKSUM_BAD >> 1),
+			(PKT_RX_IP_CKSUM_BAD >> 1),
+			(PKT_RX_L4_CKSUM_GOOD >> 1),
+			(PKT_RX_L4_CKSUM_BAD >> 1),
+			0, 0);
+	/*
+	 * Non-fragment PTYPEs.
+	 * Shuffle 4-bit index:
+	 * ip6 => bit 0
+	 * ip4 => bit 1
+	 * udp => bit 2
+	 * tcp => bit 3
+	 *   bit
+	 * 3 2 1 0
+	 * -------
+	 * 0 0 0 0 unknown
+	 * 0 0 0 1 ip6 | nonfrag
+	 * 0 0 1 0 ip4 | nonfrag
+	 * 0 0 1 1 unknown
+	 * 0 1 0 0 unknown
+	 * 0 1 0 1 ip6 | udp
+	 * 0 1 1 0 ip4 | udp
+	 * 0 1 1 1 unknown
+	 * 1 0 0 0 unknown
+	 * 1 0 0 1 ip6 | tcp
+	 * 1 0 1 0 ip4 | tcp
+	 * 1 0 1 1 unknown
+	 * 1 1 0 0 unknown
+	 * 1 1 0 1 unknown
+	 * 1 1 1 0 unknown
+	 * 1 1 1 1 unknown
+	 *
+	 * PTYPEs do not fit in 8 bits, so shift right 4..
+	 */
+	const __m256i nonfrag_ptype_shuffle =
+		_mm256_set_epi8(/* second 128 bits */
+			RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP) >> 4,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP) >> 4,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_NONFRAG) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_NONFRAG) >> 4,
+			RTE_PTYPE_UNKNOWN,
+			/* first 128 bits */
+			RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP) >> 4,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP) >> 4,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_NONFRAG) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_NONFRAG) >> 4,
+			RTE_PTYPE_UNKNOWN);
+	/* Fragment PTYPEs. Use the same shuffle index as above. */
+	const __m256i frag_ptype_shuffle =
+		_mm256_set_epi8(/* second 128 bits */
+			RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			RTE_PTYPE_UNKNOWN,
+			/* first 128 bits */
+			RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			(RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			(RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_L4_FRAG) >> 4,
+			RTE_PTYPE_UNKNOWN);
+	/*
+	 * Tunnel PTYPEs. Use the same shuffle index as above.
+	 * L4 types are not part of this table. They come from non-tunnel
+	 * types above.
+	 */
+	const __m256i tnl_l3_ptype_shuffle =
+		_mm256_set_epi8(/* second 128 bits */
+			RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_UNKNOWN,
+			/* first 128 bits */
+			RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN >> 16,
+			RTE_PTYPE_UNKNOWN);
+
+	const __m256i mbuf_init = _mm256_set_epi64x(0, enic->mbuf_initializer,
+						    0, enic->mbuf_initializer);
+
+	/*
+	 * --- cq desc fields ---    offset
+	 * completed_index_flags    - 0   use: fcoe
+	 * q_number_rss_type_flags  - 2   use: rss types, csum_not_calc
+	 * rss_hash                 - 4   ==> mbuf.hash.rss
+	 * bytes_written_flags      - 8   ==> mbuf.pkt_len,data_len
+	 *                                use: truncated, vlan_stripped
+	 * vlan                     - 10  ==> mbuf.vlan_tci
+	 * checksum_fcoe            - 12  (unused)
+	 * flags                    - 14  use: all bits
+	 * type_color               - 15  (unused)
+	 *
+	 * --- mbuf fields ---       offset
+	 * rearm_data              ---- 16
+	 * data_off    - 0      (mbuf_init) -+
+	 * refcnt      - 2      (mbuf_init)  |
+	 * nb_segs     - 4      (mbuf_init)  | 16B 128b
+	 * port        - 6      (mbuf_init)  |
+	 * ol_flag     - 8      (from cqd)  -+
+	 * rx_descriptor_fields1   ---- 32
+	 * packet_type - 0      (from cqd)  -+
+	 * pkt_len     - 4      (from cqd)   |
+	 * data_len    - 8      (from cqd)   | 16B 128b
+	 * vlan_tci    - 10     (from cqd)   |
+	 * rss         - 12     (from cqd)  -+
+	 */
+
+	__m256i overlay_enabled =
+		_mm256_set1_epi32((uint32_t)enic->overlay_offload);
+
+	/* Step 2: Process 8 packets per loop using SIMD */
+	while (max_rx > 7 && (((cqd + 7)->type_color &
+			       CQ_DESC_COLOR_MASK_NOSHIFT) != color)) {
+		/* Load 8 16B CQ descriptors */
+		__m256i cqd01 = _mm256_load_si256((void *)cqd);
+		__m256i cqd23 = _mm256_load_si256((void *)(cqd + 2));
+		__m256i cqd45 = _mm256_load_si256((void *)(cqd + 4));
+		__m256i cqd67 = _mm256_load_si256((void *)(cqd + 6));
+		/* Copy 8 mbuf pointers to rx_pkts */
+		_mm256_storeu_si256((void *)rx,
+				    _mm256_loadu_si256((void *)rxmb));
+		_mm256_storeu_si256((void *)(rx + 4),
+				    _mm256_loadu_si256((void *)(rxmb + 4)));
+
+		/*
+		 * Collect 8 flags (each 32 bits) into one register.
+		 * 4 shuffles, 3 blends, 1 permute for 8 desc: 1 inst/desc
+		 */
+		__m256i flags01 =
+			_mm256_shuffle_epi8(cqd01, flags_shuffle_mask);
+		/*
+		 * Shuffle above produces 8 x 32-bit flags for 8 descriptors
+		 * in this order: 0, 0, 0, 0, 1, 1, 1, 1
+		 * The duplicates in each 128-bit lane simplifies blending
+		 * below.
+		 */
+		__m256i flags23 =
+			_mm256_shuffle_epi8(cqd23, flags_shuffle_mask);
+		__m256i flags45 =
+			_mm256_shuffle_epi8(cqd45, flags_shuffle_mask);
+		__m256i flags67 =
+			_mm256_shuffle_epi8(cqd67, flags_shuffle_mask);
+		/* 1st blend produces flags for desc: 0, 2, 0, 0, 1, 3, 1, 1 */
+		__m256i flags0_3 = _mm256_blend_epi32(flags01, flags23, 0x22);
+		/* 2nd blend produces flags for desc: 4, 4, 4, 6, 5, 5, 5, 7 */
+		__m256i flags4_7 = _mm256_blend_epi32(flags45, flags67, 0x88);
+		/* 3rd blend produces flags for desc: 0, 2, 4, 6, 1, 3, 5, 7 */
+		__m256i flags0_7 = _mm256_blend_epi32(flags0_3, flags4_7, 0xcc);
+		/*
+		 * Swap to reorder flags in this order: 1, 3, 5, 7, 0, 2, 4, 6
+		 * This order simplifies blend operations way below that
+		 * produce 'rearm' data for each mbuf.
+		 */
+		flags0_7 = _mm256_permute4x64_epi64(flags0_7,
+			(1 << 6) + (0 << 4) + (3 << 2) + 2);
+
+		/*
+		 * Check truncated bits and bail out early on.
+		 * 6 avx inst, 1 or, 1 if-then-else for 8 desc: 1 inst/desc
+		 */
+		__m256i trunc =
+			_mm256_srli_epi32(_mm256_slli_epi32(flags0_7, 17), 31);
+		trunc = _mm256_add_epi64(trunc, _mm256_permute4x64_epi64(trunc,
+			(1 << 6) + (0 << 4) + (3 << 2) + 2));
+		/* 0:63 contains 1+3+0+2 and 64:127 contains 5+7+4+6 */
+		if (_mm256_extract_epi64(trunc, 0) ||
+		    _mm256_extract_epi64(trunc, 1))
+			break;
+
+		/*
+		 * Compute PKT_RX_RSS_HASH.
+		 * Use 2 shifts and 1 shuffle for 8 desc: 0.375 inst/desc
+		 * RSS types in byte 0, 4, 8, 12, 16, 20, 24, 28
+		 * Everything else is zero.
+		 */
+		__m256i rss_types =
+			_mm256_srli_epi32(_mm256_slli_epi32(flags0_7, 10), 28);
+		/*
+		 * RSS flags (PKT_RX_RSS_HASH) are in
+		 * byte 0, 4, 8, 12, 16, 20, 24, 28
+		 * Everything else is zero.
+		 */
+		__m256i rss_flags = _mm256_shuffle_epi8(rss_shuffle, rss_types);
+
+		/*
+		 * Compute CKSUM flags. First build the index and then
+		 * use it to shuffle csum_shuffle.
+		 * 20 instructions including const loads: 2.5 inst/desc
+		 */
+		/*
+		 * csum_not_calc (bit 22)
+		 * csum_not_calc (0) => 0xffffffff
+		 * csum_not_calc (1) => 0x0
+		 */
+		const __m256i zero4 = _mm256_setzero_si256();
+		const __m256i mask22 = _mm256_set1_epi32(0x400000);
+		__m256i csum_not_calc = _mm256_cmpeq_epi32(zero4,
+			_mm256_and_si256(flags0_7, mask22));
+		/*
+		 * (tcp|udp) && !fragment => bit 1
+		 * tcp = bit 2, udp = bit 1, frag = bit 6
+		 */
+		const __m256i mask1 = _mm256_set1_epi32(0x2);
+		__m256i tcp_udp =
+			_mm256_andnot_si256(_mm256_srli_epi32(flags0_7, 5),
+				_mm256_or_si256(flags0_7,
+					_mm256_srli_epi32(flags0_7, 1)));
+		tcp_udp = _mm256_and_si256(tcp_udp, mask1);
+		/* ipv4 (bit 5) => bit 2 */
+		const __m256i mask2 = _mm256_set1_epi32(0x4);
+		__m256i ipv4 = _mm256_and_si256(mask2,
+			_mm256_srli_epi32(flags0_7, 3));
+		/*
+		 * ipv4_csum_ok (bit 3) => bit 3
+		 * tcp_udp_csum_ok (bit 0) => bit 0
+		 * 0x9
+		 */
+		const __m256i mask0_3 = _mm256_set1_epi32(0x9);
+		__m256i csum_idx = _mm256_and_si256(flags0_7, mask0_3);
+		csum_idx = _mm256_and_si256(csum_not_calc,
+			_mm256_or_si256(_mm256_or_si256(csum_idx, ipv4),
+				tcp_udp));
+		__m256i csum_flags =
+			_mm256_shuffle_epi8(csum_shuffle, csum_idx);
+		/* Shift left to restore CKSUM flags. See csum_shuffle. */
+		csum_flags = _mm256_slli_epi32(csum_flags, 1);
+		/* Combine csum flags and offload flags: 0.125 inst/desc */
+		rss_flags = _mm256_or_si256(rss_flags, csum_flags);
+
+		/*
+		 * Collect 8 VLAN IDs and compute vlan_id != 0 on each.
+		 * 4 shuffles, 3 blends, 1 permute, 1 cmp, 1 sub for 8 desc:
+		 * 1.25 inst/desc
+		 */
+		__m256i vlan01 = _mm256_shuffle_epi8(cqd01, vlan_shuffle_mask);
+		__m256i vlan23 = _mm256_shuffle_epi8(cqd23, vlan_shuffle_mask);
+		__m256i vlan45 = _mm256_shuffle_epi8(cqd45, vlan_shuffle_mask);
+		__m256i vlan67 = _mm256_shuffle_epi8(cqd67, vlan_shuffle_mask);
+		__m256i vlan0_3 = _mm256_blend_epi32(vlan01, vlan23, 0x22);
+		__m256i vlan4_7 = _mm256_blend_epi32(vlan45, vlan67, 0x88);
+		/* desc: 0, 2, 4, 6, 1, 3, 5, 7 */
+		__m256i vlan0_7 = _mm256_blend_epi32(vlan0_3, vlan4_7, 0xcc);
+		/* desc: 1, 3, 5, 7, 0, 2, 4, 6 */
+		vlan0_7 = _mm256_permute4x64_epi64(vlan0_7,
+			(1 << 6) + (0 << 4) + (3 << 2) + 2);
+		/*
+		 * Compare 0 == vlan_id produces 0xffffffff (-1) if
+		 * vlan 0 and 0 if vlan non-0. Then subtracting the
+		 * result from 0 produces 0 - (-1) = 1 for vlan 0, and
+		 * 0 - 0 = 0 for vlan non-0.
+		 */
+		vlan0_7 = _mm256_cmpeq_epi32(zero4, vlan0_7);
+		/* vlan_id != 0 => 0, vlan_id == 0 => 1 */
+		vlan0_7 = _mm256_sub_epi32(zero4, vlan0_7);
+
+		/*
+		 * Compute PKT_RX_VLAN and PKT_RX_VLAN_STRIPPED.
+		 * Use 3 shifts, 1 or,  1 shuffle for 8 desc: 0.625 inst/desc
+		 * VLAN offload flags in byte 0, 4, 8, 12, 16, 20, 24, 28
+		 * Everything else is zero.
+		 */
+		__m256i vlan_idx =
+			_mm256_or_si256(/* vlan_stripped => bit 0 */
+				_mm256_srli_epi32(_mm256_slli_epi32(flags0_7,
+					16), 31),
+				/* (vlan_id == 0) => bit 1 */
+				_mm256_slli_epi32(vlan0_7, 1));
+		/*
+		 * The index captures 4 cases.
+		 * stripped, id = 0   ==> 11b = 3
+		 * stripped, id != 0  ==> 01b = 1
+		 * not strip, id == 0 ==> 10b = 2
+		 * not strip, id != 0 ==> 00b = 0
+		 */
+		__m256i vlan_flags = _mm256_permutevar8x32_epi32(vlan_shuffle,
+			vlan_idx);
+		/* Combine vlan and offload flags: 0.125 inst/desc */
+		rss_flags = _mm256_or_si256(rss_flags, vlan_flags);
+
+		/*
+		 * Compute non-tunnel PTYPEs.
+		 * 17 inst / 8 desc = 2.125 inst/desc
+		 */
+		/* ETHER and ETHER_VLAN */
+		__m256i vlan_ptype =
+			_mm256_permutevar8x32_epi32(vlan_ptype_shuffle,
+				vlan_idx);
+		/* Build the ptype index from flags */
+		tcp_udp = _mm256_slli_epi32(flags0_7, 29);
+		tcp_udp = _mm256_slli_epi32(_mm256_srli_epi32(tcp_udp, 30), 2);
+		__m256i ip4_ip6 =
+			_mm256_srli_epi32(_mm256_slli_epi32(flags0_7, 26), 30);
+		__m256i ptype_idx = _mm256_or_si256(tcp_udp, ip4_ip6);
+		__m256i frag_bit =
+			_mm256_srli_epi32(_mm256_slli_epi32(flags0_7, 25), 31);
+		__m256i nonfrag_ptype =
+			_mm256_shuffle_epi8(nonfrag_ptype_shuffle, ptype_idx);
+		__m256i frag_ptype =
+			_mm256_shuffle_epi8(frag_ptype_shuffle, ptype_idx);
+		/*
+		 * Zero out the unwanted types and combine the remaining bits.
+		 * The effect is same as selecting non-frag or frag types
+		 * depending on the frag bit.
+		 */
+		nonfrag_ptype = _mm256_and_si256(nonfrag_ptype,
+			_mm256_cmpeq_epi32(zero4, frag_bit));
+		frag_ptype = _mm256_and_si256(frag_ptype,
+			_mm256_cmpgt_epi32(frag_bit, zero4));
+		__m256i ptype = _mm256_or_si256(nonfrag_ptype, frag_ptype);
+		ptype = _mm256_slli_epi32(ptype, 4);
+		/*
+		 * Compute tunnel PTYPEs.
+		 * 15 inst / 8 desc = 1.875 inst/desc
+		 */
+		__m256i tnl_l3_ptype =
+			_mm256_shuffle_epi8(tnl_l3_ptype_shuffle, ptype_idx);
+		tnl_l3_ptype = _mm256_slli_epi32(tnl_l3_ptype, 16);
+		/*
+		 * Shift non-tunnel L4 types to make them tunnel types.
+		 * RTE_PTYPE_L4_TCP << 16 == RTE_PTYPE_INNER_L4_TCP
+		 */
+		__m256i tnl_l4_ptype =
+			_mm256_slli_epi32(_mm256_and_si256(ptype,
+				_mm256_set1_epi32(RTE_PTYPE_L4_MASK)), 16);
+		__m256i tnl_ptype =
+			_mm256_or_si256(tnl_l3_ptype, tnl_l4_ptype);
+		tnl_ptype = _mm256_or_si256(tnl_ptype,
+			_mm256_set1_epi32(RTE_PTYPE_TUNNEL_GRENAT |
+				RTE_PTYPE_INNER_L2_ETHER));
+		/*
+		 * Select non-tunnel or tunnel types by zeroing out the
+		 * unwanted ones.
+		 */
+		__m256i tnl_flags = _mm256_and_si256(overlay_enabled,
+			_mm256_srli_epi32(_mm256_slli_epi32(flags0_7, 2), 31));
+		tnl_ptype = _mm256_and_si256(tnl_ptype,
+			_mm256_sub_epi32(zero4, tnl_flags));
+		ptype =	_mm256_and_si256(ptype,
+			_mm256_cmpeq_epi32(zero4, tnl_flags));
+		/*
+		 * Combine types and swap to have ptypes in the same order
+		 * as desc.
+		 * desc: 0 2 4 6 1 3 5 7
+		 * 3 inst / 8 desc = 0.375 inst/desc
+		 */
+		ptype = _mm256_or_si256(ptype, tnl_ptype);
+		ptype = _mm256_or_si256(ptype, vlan_ptype);
+		ptype = _mm256_permute4x64_epi64(ptype,
+			(1 << 6) + (0 << 4) + (3 << 2) + 2);
+
+		/*
+		 * Mask packet length.
+		 * Use 4 ands: 0.5 instructions/desc
+		 */
+		cqd01 = _mm256_and_si256(cqd01, mask);
+		cqd23 = _mm256_and_si256(cqd23, mask);
+		cqd45 = _mm256_and_si256(cqd45, mask);
+		cqd67 = _mm256_and_si256(cqd67, mask);
+		/*
+		 * Shuffle. Two 16B sets of the mbuf fields.
+		 * packet_type, pkt_len, data_len, vlan_tci, rss
+		 */
+		__m256i rearm01 = _mm256_shuffle_epi8(cqd01, shuffle_mask);
+		__m256i rearm23 = _mm256_shuffle_epi8(cqd23, shuffle_mask);
+		__m256i rearm45 = _mm256_shuffle_epi8(cqd45, shuffle_mask);
+		__m256i rearm67 = _mm256_shuffle_epi8(cqd67, shuffle_mask);
+
+		/*
+		 * Blend in ptypes
+		 * 4 blends and 3 shuffles for 8 desc: 0.875 inst/desc
+		 */
+		rearm01 = _mm256_blend_epi32(rearm01, ptype, 0x11);
+		rearm23 = _mm256_blend_epi32(rearm23,
+			_mm256_shuffle_epi32(ptype, 1), 0x11);
+		rearm45 = _mm256_blend_epi32(rearm45,
+			_mm256_shuffle_epi32(ptype, 2), 0x11);
+		rearm67 = _mm256_blend_epi32(rearm67,
+			_mm256_shuffle_epi32(ptype, 3), 0x11);
+
+		/*
+		 * Move rss_flags into ol_flags in mbuf_init.
+		 * Use 1 shift and 1 blend for each desc: 2 inst/desc
+		 */
+		__m256i mbuf_init4_5 = _mm256_blend_epi32(mbuf_init,
+			rss_flags, 0x44);
+		__m256i mbuf_init2_3 = _mm256_blend_epi32(mbuf_init,
+			_mm256_slli_si256(rss_flags, 4), 0x44);
+		__m256i mbuf_init0_1 = _mm256_blend_epi32(mbuf_init,
+			_mm256_slli_si256(rss_flags, 8), 0x44);
+		__m256i mbuf_init6_7 = _mm256_blend_epi32(mbuf_init,
+			_mm256_srli_si256(rss_flags, 4), 0x44);
+
+		/*
+		 * Build rearm, one per desc.
+		 * 8 blends and 4 permutes: 1.5 inst/desc
+		 */
+		__m256i rearm0 = _mm256_blend_epi32(rearm01,
+			mbuf_init0_1, 0xf0);
+		__m256i rearm1 = _mm256_blend_epi32(mbuf_init0_1,
+			rearm01, 0xf0);
+		__m256i rearm2 = _mm256_blend_epi32(rearm23,
+			mbuf_init2_3, 0xf0);
+		__m256i rearm3 = _mm256_blend_epi32(mbuf_init2_3,
+			rearm23, 0xf0);
+		/* Swap upper and lower 64 bits */
+		rearm0 = _mm256_permute4x64_epi64(rearm0,
+			(1 << 6) + (0 << 4) + (3 << 2) + 2);
+		rearm2 = _mm256_permute4x64_epi64(rearm2,
+			(1 << 6) + (0 << 4) + (3 << 2) + 2);
+		/* Second set of 4 descriptors */
+		__m256i rearm4 = _mm256_blend_epi32(rearm45,
+			mbuf_init4_5, 0xf0);
+		__m256i rearm5 = _mm256_blend_epi32(mbuf_init4_5,
+			rearm45, 0xf0);
+		__m256i rearm6 = _mm256_blend_epi32(rearm67,
+			mbuf_init6_7, 0xf0);
+		__m256i rearm7 = _mm256_blend_epi32(mbuf_init6_7,
+			rearm67, 0xf0);
+		rearm4 = _mm256_permute4x64_epi64(rearm4,
+			(1 << 6) + (0 << 4) + (3 << 2) + 2);
+		rearm6 = _mm256_permute4x64_epi64(rearm6,
+			(1 << 6) + (0 << 4) + (3 << 2) + 2);
+
+		/*
+		 * Write out 32B of mbuf fields.
+		 * data_off    - off 0  (mbuf_init)
+		 * refcnt      - 2      (mbuf_init)
+		 * nb_segs     - 4      (mbuf_init)
+		 * port        - 6      (mbuf_init)
+		 * ol_flag     - 8      (from cqd)
+		 * packet_type - 16     (from cqd)
+		 * pkt_len     - 20     (from cqd)
+		 * data_len    - 24     (from cqd)
+		 * vlan_tci    - 26     (from cqd)
+		 * rss         - 28     (from cqd)
+		 */
+		_mm256_storeu_si256((__m256i *)&rxmb[0]->rearm_data, rearm0);
+		_mm256_storeu_si256((__m256i *)&rxmb[1]->rearm_data, rearm1);
+		_mm256_storeu_si256((__m256i *)&rxmb[2]->rearm_data, rearm2);
+		_mm256_storeu_si256((__m256i *)&rxmb[3]->rearm_data, rearm3);
+		_mm256_storeu_si256((__m256i *)&rxmb[4]->rearm_data, rearm4);
+		_mm256_storeu_si256((__m256i *)&rxmb[5]->rearm_data, rearm5);
+		_mm256_storeu_si256((__m256i *)&rxmb[6]->rearm_data, rearm6);
+		_mm256_storeu_si256((__m256i *)&rxmb[7]->rearm_data, rearm7);
+
+		max_rx -= 8;
+		cqd += 8;
+		rx += 8;
+		rxmb += 8;
+	}
+
+	/*
+	 * Step 3: Slow path to handle a small (<8) number of packets and
+	 * occasional truncated packets.
+	 */
+	while (max_rx && ((cqd->type_color &
+			   CQ_DESC_COLOR_MASK_NOSHIFT) != color)) {
+		if (unlikely(cqd->bytes_written_flags &
+			     CQ_ENET_RQ_DESC_FLAGS_TRUNCATED)) {
+			rte_pktmbuf_free(*rxmb++);
+			rte_atomic64_inc(&enic->soft_stats.rx_packet_errors);
+		} else {
+			*rx++ = rx_one(cqd, *rxmb++, enic);
+		}
+		cqd++;
+		max_rx--;
+	}
+
+	/* Number of descriptors visited */
+	nb_rx = cqd - (struct cq_enet_rq_desc *)(cq->ring.descs) - cq_idx;
+	if (nb_rx == 0)
+		return 0;
+	rqd = ((struct rq_enet_desc *)rq->ring.descs) + cq_idx;
+	rxmb = rq->mbuf_ring + cq_idx;
+	cq_idx += nb_rx;
+	rq->rx_nb_hold += nb_rx;
+	if (unlikely(cq_idx == cq->ring.desc_count)) {
+		cq_idx = 0;
+		cq->last_color ^= CQ_DESC_COLOR_MASK_NOSHIFT;
+	}
+	cq->to_clean = cq_idx;
+
+	/* Step 4: Restock RQ with new mbufs */
+	memcpy(rxmb, rq->free_mbufs + ENIC_RX_BURST_MAX - rq->num_free_mbufs,
+	       sizeof(struct rte_mbuf *) * nb_rx);
+	rq->num_free_mbufs -= nb_rx;
+	while (nb_rx) {
+		rqd->address = (*rxmb)->buf_iova + RTE_PKTMBUF_HEADROOM;
+		nb_rx--;
+		rqd++;
+		rxmb++;
+	}
+	if (rq->rx_nb_hold > rq->rx_free_thresh) {
+		rq->posted_index = enic_ring_add(rq->ring.desc_count,
+						 rq->posted_index,
+						 rq->rx_nb_hold);
+		rq->rx_nb_hold = 0;
+		rte_wmb();
+		iowrite32_relaxed(rq->posted_index,
+				  &rq->ctrl->posted_index);
+	}
+
+	return rx - rx_pkts;
+}
+
+bool
+enic_use_vector_rx_handler(struct enic *enic)
+{
+	struct rte_eth_dev *eth_dev;
+	struct rte_fdir_conf *fconf;
+
+	eth_dev = enic->rte_dev;
+	/* User needs to request for the avx2 handler */
+	if (!enic->enable_avx2_rx)
+		return false;
+	/* Do not support scatter Rx */
+	if (!(enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0))
+		return false;
+	/* Do not support fdir/flow */
+	fconf = &eth_dev->data->dev_conf.fdir_conf;
+	if (fconf->mode != RTE_FDIR_MODE_NONE)
+		return false;
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) {
+		PMD_INIT_LOG(DEBUG, " use the non-scatter avx2 Rx handler");
+		eth_dev->rx_pkt_burst = &enic_noscatter_vec_recv_pkts;
+		return true;
+	}
+	return false;
+}
diff --git a/drivers/net/enic/meson.build b/drivers/net/enic/meson.build
index bfd4e237..06448711 100644
--- a/drivers/net/enic/meson.build
+++ b/drivers/net/enic/meson.build
@@ -17,3 +17,19 @@ sources = files(
 	)
 deps += ['hash']
 includes += include_directories('base')
+
+# The current implementation assumes 64-bit pointers
+if dpdk_conf.has('RTE_MACHINE_CPUFLAG_AVX2') and cc.sizeof('void *') == 8
+	sources += files('enic_rxtx_vec_avx2.c')
+# Build the avx2 handler if the compiler supports it, even though 'machine'
+# does not. This is to support users who build for the min supported machine
+# and need to run the binary on newer CPUs too.
+# This part is from i40e meson.build
+elif cc.has_argument('-mavx2') and cc.sizeof('void *') == 8
+	enic_avx2_lib = static_library('enic_avx2_lib',
+			'enic_rxtx_vec_avx2.c',
+			dependencies: [static_rte_ethdev, static_rte_bus_pci],
+			include_directories: includes,
+			c_args: [cflags, '-mavx2'])
+	objs += enic_avx2_lib.extract_objects('enic_rxtx_vec_avx2.c')
+endif
diff --git a/drivers/net/failsafe/failsafe.c b/drivers/net/failsafe/failsafe.c
index 657919f9..06e859e9 100644
--- a/drivers/net/failsafe/failsafe.c
+++ b/drivers/net/failsafe/failsafe.c
@@ -71,7 +71,7 @@ failsafe_hotplug_alarm_install(struct rte_eth_dev *dev)
 		return -EINVAL;
 	if (PRIV(dev)->pending_alarm)
 		return 0;
-	ret = rte_eal_alarm_set(hotplug_poll * 1000,
+	ret = rte_eal_alarm_set(failsafe_hotplug_poll * 1000,
 				fs_hotplug_alarm,
 				dev);
 	if (ret) {
@@ -225,7 +225,7 @@ fs_eth_dev_create(struct rte_vdev_device *vdev)
 		goto unregister_new_callback;
 	}
 	mac = &dev->data->mac_addrs[0];
-	if (mac_from_arg) {
+	if (failsafe_mac_from_arg) {
 		/*
 		 * If MAC address was provided as a parameter,
 		 * apply to all probed slaves.
@@ -280,7 +280,8 @@ free_args:
 free_subs:
 	fs_sub_device_free(dev);
 free_dev:
-	rte_free(PRIV(dev));
+	/* mac_addrs must not be freed alone because part of dev_private */
+	dev->data->mac_addrs = NULL;
 	rte_eth_dev_release_port(dev);
 	return -1;
 }
@@ -304,7 +305,9 @@ fs_rte_eth_free(const char *name)
 	ret = pthread_mutex_destroy(&PRIV(dev)->hotplug_mutex);
 	if (ret)
 		ERROR("Error while destroying hotplug mutex");
-	rte_free(PRIV(dev));
+	rte_free(PRIV(dev)->mcast_addrs);
+	/* mac_addrs must not be freed alone because part of dev_private */
+	dev->data->mac_addrs = NULL;
 	rte_eth_dev_release_port(dev);
 	return ret;
 }
diff --git a/drivers/net/failsafe/failsafe_args.c b/drivers/net/failsafe/failsafe_args.c
index 626883ce..c4b220c4 100644
--- a/drivers/net/failsafe/failsafe_args.c
+++ b/drivers/net/failsafe/failsafe_args.c
@@ -22,10 +22,10 @@
 typedef int (parse_cb)(struct rte_eth_dev *dev, const char *params,
 		uint8_t head);
 
-uint64_t hotplug_poll = FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS;
-int mac_from_arg = 0;
+uint64_t failsafe_hotplug_poll = FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS;
+int failsafe_mac_from_arg;
 
-const char *pmd_failsafe_init_parameters[] = {
+static const char * const pmd_failsafe_init_parameters[] = {
 	PMD_FAILSAFE_HOTPLUG_POLL_KVARG,
 	PMD_FAILSAFE_MAC_KVARG,
 	NULL,
@@ -420,7 +420,7 @@ failsafe_args_parse(struct rte_eth_dev *dev, const char *params)
 		if (arg_count == 1) {
 			ret = rte_kvargs_process(kvlist,
 					PMD_FAILSAFE_HOTPLUG_POLL_KVARG,
-					&fs_get_u64_arg, &hotplug_poll);
+					&fs_get_u64_arg, &failsafe_hotplug_poll);
 			if (ret < 0)
 				goto free_kvlist;
 		}
@@ -435,7 +435,7 @@ failsafe_args_parse(struct rte_eth_dev *dev, const char *params)
 			if (ret < 0)
 				goto free_kvlist;
 
-			mac_from_arg = 1;
+			failsafe_mac_from_arg = 1;
 		}
 	}
 	PRIV(dev)->state = DEV_PARSED;
diff --git a/drivers/net/failsafe/failsafe_eal.c b/drivers/net/failsafe/failsafe_eal.c
index ce1633f1..8a888b1f 100644
--- a/drivers/net/failsafe/failsafe_eal.c
+++ b/drivers/net/failsafe/failsafe_eal.c
@@ -144,8 +144,7 @@ fs_bus_uninit(struct rte_eth_dev *dev)
 	int ret = 0;
 
 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
-		sdev_ret = rte_eal_hotplug_remove(sdev->bus->name,
-							sdev->dev->name);
+		sdev_ret = rte_dev_remove(sdev->dev);
 		if (sdev_ret) {
 			ERROR("Failed to remove requested device %s (err: %d)",
 			      sdev->dev->name, sdev_ret);
diff --git a/drivers/net/failsafe/failsafe_ether.c b/drivers/net/failsafe/failsafe_ether.c
index 5b5cb3b4..17831652 100644
--- a/drivers/net/failsafe/failsafe_ether.c
+++ b/drivers/net/failsafe/failsafe_ether.c
@@ -179,6 +179,23 @@ fs_eth_dev_conf_apply(struct rte_eth_dev *dev,
 			return ret;
 		}
 	}
+	/*
+	 * Propagate multicast MAC addresses to sub-devices,
+	 * if non zero number of addresses is set.
+	 * The condition is required to avoid breakage of failsafe
+	 * for sub-devices which do not support the operation
+	 * if the feature is really not used.
+	 */
+	if (PRIV(dev)->nb_mcast_addr > 0) {
+		DEBUG("Configuring multicast MAC addresses");
+		ret = rte_eth_dev_set_mc_addr_list(PORT_ID(sdev),
+						   PRIV(dev)->mcast_addrs,
+						   PRIV(dev)->nb_mcast_addr);
+		if (ret) {
+			ERROR("Failed to apply multicast MAC addresses");
+			return ret;
+		}
+	}
 	/* VLAN filter */
 	vfc1 = &dev->data->vlan_filter_conf;
 	vfc2 = &edev->data->vlan_filter_conf;
@@ -230,9 +247,9 @@ fs_eth_dev_conf_apply(struct rte_eth_dev *dev,
 			DEBUG("Creating flow #%" PRIu32, i++);
 			flow->flows[SUB_ID(sdev)] =
 				rte_flow_create(PORT_ID(sdev),
-						&flow->fd->attr,
-						flow->fd->items,
-						flow->fd->actions,
+						flow->rule.attr,
+						flow->rule.pattern,
+						flow->rule.actions,
 						&ferror);
 			ret = rte_errno;
 			if (ret)
@@ -265,8 +282,7 @@ fs_dev_remove(struct sub_device *sdev)
 		sdev->state = DEV_PROBED;
 		/* fallthrough */
 	case DEV_PROBED:
-		ret = rte_eal_hotplug_remove(sdev->bus->name,
-					     sdev->dev->name);
+		ret = rte_dev_remove(sdev->dev);
 		if (ret) {
 			ERROR("Bus detach failed for sub_device %u",
 			      SUB_ID(sdev));
@@ -366,6 +382,88 @@ failsafe_dev_remove(struct rte_eth_dev *dev)
 		}
 }
 
+static int
+failsafe_eth_dev_rx_queues_sync(struct rte_eth_dev *dev)
+{
+	struct rxq *rxq;
+	int ret;
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		rxq = dev->data->rx_queues[i];
+
+		if (rxq->info.conf.rx_deferred_start &&
+		    dev->data->rx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STARTED) {
+			/*
+			 * The subdevice Rx queue does not launch on device
+			 * start if deferred start flag is set. It needs to be
+			 * started manually in case an appropriate failsafe Rx
+			 * queue has been started earlier.
+			 */
+			ret = dev->dev_ops->rx_queue_start(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Rx queue %d", i);
+				return ret;
+			}
+		} else if (dev->data->rx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STOPPED) {
+			/*
+			 * The subdevice Rx queue needs to be stopped manually
+			 * in case an appropriate failsafe Rx queue has been
+			 * stopped earlier.
+			 */
+			ret = dev->dev_ops->rx_queue_stop(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Rx queue %d", i);
+				return ret;
+			}
+		}
+	}
+	return 0;
+}
+
+static int
+failsafe_eth_dev_tx_queues_sync(struct rte_eth_dev *dev)
+{
+	struct txq *txq;
+	int ret;
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		txq = dev->data->tx_queues[i];
+
+		if (txq->info.conf.tx_deferred_start &&
+		    dev->data->tx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STARTED) {
+			/*
+			 * The subdevice Tx queue does not launch on device
+			 * start if deferred start flag is set. It needs to be
+			 * started manually in case an appropriate failsafe Tx
+			 * queue has been started earlier.
+			 */
+			ret = dev->dev_ops->tx_queue_start(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Tx queue %d", i);
+				return ret;
+			}
+		} else if (dev->data->tx_queue_state[i] ==
+						RTE_ETH_QUEUE_STATE_STOPPED) {
+			/*
+			 * The subdevice Tx queue needs to be stopped manually
+			 * in case an appropriate failsafe Tx queue has been
+			 * stopped earlier.
+			 */
+			ret = dev->dev_ops->tx_queue_stop(dev, i);
+			if (ret) {
+				ERROR("Could not synchronize Tx queue %d", i);
+				return ret;
+			}
+		}
+	}
+	return 0;
+}
+
 int
 failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 {
@@ -424,6 +522,12 @@ failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 	ret = dev->dev_ops->dev_start(dev);
 	if (ret)
 		goto err_remove;
+	ret = failsafe_eth_dev_rx_queues_sync(dev);
+	if (ret)
+		goto err_remove;
+	ret = failsafe_eth_dev_tx_queues_sync(dev);
+	if (ret)
+		goto err_remove;
 	return 0;
 err_remove:
 	FOREACH_SUBDEV(sdev, i, dev)
@@ -466,7 +570,7 @@ failsafe_eth_rmv_event_callback(uint16_t port_id __rte_unused,
 	/* Switch as soon as possible tx_dev. */
 	fs_switch_dev(sdev->fs_dev, sdev);
 	/* Use safe bursts in any case. */
-	set_burst_fn(sdev->fs_dev, 1);
+	failsafe_set_burst_fn(sdev->fs_dev, 1);
 	/*
 	 * Async removal, the sub-PMD will try to unregister
 	 * the callback at the source of the current thread context.
diff --git a/drivers/net/failsafe/failsafe_flow.c b/drivers/net/failsafe/failsafe_flow.c
index bfe42fce..5e2b5f7c 100644
--- a/drivers/net/failsafe/failsafe_flow.c
+++ b/drivers/net/failsafe/failsafe_flow.c
@@ -3,8 +3,11 @@
  * Copyright 2017 Mellanox Technologies, Ltd
  */
 
+#include <stddef.h>
+#include <string.h>
 #include <sys/queue.h>
 
+#include <rte_errno.h>
 #include <rte_malloc.h>
 #include <rte_tailq.h>
 #include <rte_flow.h>
@@ -18,19 +21,33 @@ fs_flow_allocate(const struct rte_flow_attr *attr,
 		 const struct rte_flow_action *actions)
 {
 	struct rte_flow *flow;
-	size_t fdsz;
+	const struct rte_flow_conv_rule rule = {
+		.attr_ro = attr,
+		.pattern_ro = items,
+		.actions_ro = actions,
+	};
+	struct rte_flow_error error;
+	int ret;
 
-	fdsz = rte_flow_copy(NULL, 0, attr, items, actions);
-	flow = rte_zmalloc(NULL,
-			   sizeof(struct rte_flow) + fdsz,
+	ret = rte_flow_conv(RTE_FLOW_CONV_OP_RULE, NULL, 0, &rule, &error);
+	if (ret < 0) {
+		ERROR("Unable to process flow rule (%s): %s",
+		      error.message ? error.message : "unspecified",
+		      strerror(rte_errno));
+		return NULL;
+	}
+	flow = rte_zmalloc(NULL, offsetof(struct rte_flow, rule) + ret,
 			   RTE_CACHE_LINE_SIZE);
 	if (flow == NULL) {
 		ERROR("Could not allocate new flow");
 		return NULL;
 	}
-	flow->fd = (void *)((uintptr_t)flow + sizeof(*flow));
-	if (rte_flow_copy(flow->fd, fdsz, attr, items, actions) != fdsz) {
-		ERROR("Failed to copy flow description");
+	ret = rte_flow_conv(RTE_FLOW_CONV_OP_RULE, &flow->rule, ret, &rule,
+			    &error);
+	if (ret < 0) {
+		ERROR("Failed to copy flow rule (%s): %s",
+		      error.message ? error.message : "unspecified",
+		      strerror(rte_errno));
 		rte_free(flow);
 		return NULL;
 	}
diff --git a/drivers/net/failsafe/failsafe_intr.c b/drivers/net/failsafe/failsafe_intr.c
index fc6ec37f..1c2cb71c 100644
--- a/drivers/net/failsafe/failsafe_intr.c
+++ b/drivers/net/failsafe/failsafe_intr.c
@@ -372,7 +372,7 @@ void failsafe_rx_intr_uninstall_subdevice(struct sub_device *sdev)
 	for (qid = 0; qid < ETH(sdev)->data->nb_rx_queues; qid++) {
 		if (qid < fsdev->data->nb_rx_queues) {
 			fsrxq = fsdev->data->rx_queues[qid];
-			if (fsrxq->enable_events)
+			if (fsrxq != NULL && fsrxq->enable_events)
 				rte_eth_dev_rx_intr_disable(PORT_ID(sdev),
 							    qid);
 		}
diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c
index 24e91c93..7f8bcd4c 100644
--- a/drivers/net/failsafe/failsafe_ops.c
+++ b/drivers/net/failsafe/failsafe_ops.c
@@ -57,7 +57,6 @@ static struct rte_eth_dev_info default_infos = {
 		DEV_RX_OFFLOAD_VLAN_FILTER |
 		DEV_RX_OFFLOAD_VLAN_EXTEND |
 		DEV_RX_OFFLOAD_JUMBO_FRAME |
-		DEV_RX_OFFLOAD_CRC_STRIP |
 		DEV_RX_OFFLOAD_SCATTER |
 		DEV_RX_OFFLOAD_TIMESTAMP |
 		DEV_RX_OFFLOAD_SECURITY,
@@ -74,7 +73,6 @@ static struct rte_eth_dev_info default_infos = {
 		DEV_RX_OFFLOAD_VLAN_FILTER |
 		DEV_RX_OFFLOAD_VLAN_EXTEND |
 		DEV_RX_OFFLOAD_JUMBO_FRAME |
-		DEV_RX_OFFLOAD_CRC_STRIP |
 		DEV_RX_OFFLOAD_SCATTER |
 		DEV_RX_OFFLOAD_TIMESTAMP |
 		DEV_RX_OFFLOAD_SECURITY,
@@ -88,6 +86,9 @@ static struct rte_eth_dev_info default_infos = {
 			ETH_RSS_IP |
 			ETH_RSS_UDP |
 			ETH_RSS_TCP,
+	.dev_capa =
+		RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP |
+		RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP,
 };
 
 static int
@@ -170,6 +171,27 @@ fs_dev_configure(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static void
+fs_set_queues_state_start(struct rte_eth_dev *dev)
+{
+	struct rxq *rxq;
+	struct txq *txq;
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		rxq = dev->data->rx_queues[i];
+		if (rxq != NULL && !rxq->info.conf.rx_deferred_start)
+			dev->data->rx_queue_state[i] =
+						RTE_ETH_QUEUE_STATE_STARTED;
+	}
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		txq = dev->data->tx_queues[i];
+		if (txq != NULL && !txq->info.conf.tx_deferred_start)
+			dev->data->tx_queue_state[i] =
+						RTE_ETH_QUEUE_STATE_STARTED;
+	}
+}
+
 static int
 fs_dev_start(struct rte_eth_dev *dev)
 {
@@ -204,14 +226,31 @@ fs_dev_start(struct rte_eth_dev *dev)
 		}
 		sdev->state = DEV_STARTED;
 	}
-	if (PRIV(dev)->state < DEV_STARTED)
+	if (PRIV(dev)->state < DEV_STARTED) {
 		PRIV(dev)->state = DEV_STARTED;
+		fs_set_queues_state_start(dev);
+	}
 	fs_switch_dev(dev, NULL);
 	fs_unlock(dev, 0);
 	return 0;
 }
 
 static void
+fs_set_queues_state_stop(struct rte_eth_dev *dev)
+{
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++)
+		if (dev->data->rx_queues[i] != NULL)
+			dev->data->rx_queue_state[i] =
+						RTE_ETH_QUEUE_STATE_STOPPED;
+	for (i = 0; i < dev->data->nb_tx_queues; i++)
+		if (dev->data->tx_queues[i] != NULL)
+			dev->data->tx_queue_state[i] =
+						RTE_ETH_QUEUE_STATE_STOPPED;
+}
+
+static void
 fs_dev_stop(struct rte_eth_dev *dev)
 {
 	struct sub_device *sdev;
@@ -225,6 +264,7 @@ fs_dev_stop(struct rte_eth_dev *dev)
 		sdev->state = DEV_STARTED - 1;
 	}
 	failsafe_rx_intr_uninstall(dev);
+	fs_set_queues_state_stop(dev);
 	fs_unlock(dev, 0);
 }
 
@@ -294,6 +334,112 @@ fs_dev_close(struct rte_eth_dev *dev)
 	fs_unlock(dev, 0);
 }
 
+static int
+fs_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+	int err = 0;
+	bool failure = true;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_rx_queue_stop(port_id, rx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Rx queue stop failed for subdevice %d", i);
+			err = ret;
+		} else {
+			failure = false;
+		}
+	}
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+	fs_unlock(dev, 0);
+	/* Return 0 in case of at least one successful queue stop */
+	return (failure) ? err : 0;
+}
+
+static int
+fs_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_rx_queue_start(port_id, rx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Rx queue start failed for subdevice %d", i);
+			fs_rx_queue_stop(dev, rx_queue_id);
+			fs_unlock(dev, 0);
+			return ret;
+		}
+	}
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+	fs_unlock(dev, 0);
+	return 0;
+}
+
+static int
+fs_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+	int err = 0;
+	bool failure = true;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_tx_queue_stop(port_id, tx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Tx queue stop failed for subdevice %d", i);
+			err = ret;
+		} else {
+			failure = false;
+		}
+	}
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+	fs_unlock(dev, 0);
+	/* Return 0 in case of at least one successful queue stop */
+	return (failure) ? err : 0;
+}
+
+static int
+fs_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+
+	fs_lock(dev, 0);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		uint16_t port_id = ETH(sdev)->data->port_id;
+
+		ret = rte_eth_dev_tx_queue_start(port_id, tx_queue_id);
+		ret = fs_err(sdev, ret);
+		if (ret) {
+			ERROR("Tx queue start failed for subdevice %d", i);
+			fs_tx_queue_stop(dev, tx_queue_id);
+			fs_unlock(dev, 0);
+			return ret;
+		}
+	}
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+	fs_unlock(dev, 0);
+	return 0;
+}
+
 static void
 fs_rx_queue_release(void *queue)
 {
@@ -309,9 +455,13 @@ fs_rx_queue_release(void *queue)
 	fs_lock(dev, 0);
 	if (rxq->event_fd > 0)
 		close(rxq->event_fd);
-	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
-		SUBOPS(sdev, rx_queue_release)
-			(ETH(sdev)->data->rx_queues[rxq->qid]);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		if (ETH(sdev)->data->rx_queues != NULL &&
+		    ETH(sdev)->data->rx_queues[rxq->qid] != NULL) {
+			SUBOPS(sdev, rx_queue_release)
+				(ETH(sdev)->data->rx_queues[rxq->qid]);
+		}
+	}
 	dev->data->rx_queues[rxq->qid] = NULL;
 	rte_free(rxq);
 	fs_unlock(dev, 0);
@@ -341,6 +491,16 @@ fs_rx_queue_setup(struct rte_eth_dev *dev,
 	int ret;
 
 	fs_lock(dev, 0);
+	if (rx_conf->rx_deferred_start) {
+		FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
+			if (SUBOPS(sdev, rx_queue_start) == NULL) {
+				ERROR("Rx queue deferred start is not "
+					"supported for subdevice %d", i);
+				fs_unlock(dev, 0);
+				return -EINVAL;
+			}
+		}
+	}
 	rxq = dev->data->rx_queues[rx_queue_id];
 	if (rxq != NULL) {
 		fs_rx_queue_release(rxq);
@@ -477,9 +637,13 @@ fs_tx_queue_release(void *queue)
 	txq = queue;
 	dev = txq->priv->dev;
 	fs_lock(dev, 0);
-	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
-		SUBOPS(sdev, tx_queue_release)
-			(ETH(sdev)->data->tx_queues[txq->qid]);
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		if (ETH(sdev)->data->tx_queues != NULL &&
+		    ETH(sdev)->data->tx_queues[txq->qid] != NULL) {
+			SUBOPS(sdev, tx_queue_release)
+				(ETH(sdev)->data->tx_queues[txq->qid]);
+		}
+	}
 	dev->data->tx_queues[txq->qid] = NULL;
 	rte_free(txq);
 	fs_unlock(dev, 0);
@@ -498,6 +662,16 @@ fs_tx_queue_setup(struct rte_eth_dev *dev,
 	int ret;
 
 	fs_lock(dev, 0);
+	if (tx_conf->tx_deferred_start) {
+		FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
+			if (SUBOPS(sdev, tx_queue_start) == NULL) {
+				ERROR("Tx queue deferred start is not "
+					"supported for subdevice %d", i);
+				fs_unlock(dev, 0);
+				return -EINVAL;
+			}
+		}
+	}
 	txq = dev->data->tx_queues[tx_queue_id];
 	if (txq != NULL) {
 		fs_tx_queue_release(txq);
@@ -716,6 +890,8 @@ fs_stats_reset(struct rte_eth_dev *dev)
  *      all sub_devices and the default capabilities.
  *      Uses a logical AND of TX capabilities among
  *      the active probed sub_device and the default capabilities.
+ *      Uses a logical AND of device capabilities among
+ *      all sub_devices and the default capabilities.
  *
  */
 static void
@@ -734,10 +910,12 @@ fs_dev_infos_get(struct rte_eth_dev *dev,
 		uint64_t rx_offload_capa;
 		uint64_t rxq_offload_capa;
 		uint64_t rss_hf_offload_capa;
+		uint64_t dev_capa;
 
 		rx_offload_capa = default_infos.rx_offload_capa;
 		rxq_offload_capa = default_infos.rx_queue_offload_capa;
 		rss_hf_offload_capa = default_infos.flow_type_rss_offloads;
+		dev_capa = default_infos.dev_capa;
 		FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 			rte_eth_dev_info_get(PORT_ID(sdev),
 					&PRIV(dev)->infos);
@@ -746,12 +924,14 @@ fs_dev_infos_get(struct rte_eth_dev *dev,
 					PRIV(dev)->infos.rx_queue_offload_capa;
 			rss_hf_offload_capa &=
 					PRIV(dev)->infos.flow_type_rss_offloads;
+			dev_capa &= PRIV(dev)->infos.dev_capa;
 		}
 		sdev = TX_SUBDEV(dev);
 		rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos);
 		PRIV(dev)->infos.rx_offload_capa = rx_offload_capa;
 		PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa;
 		PRIV(dev)->infos.flow_type_rss_offloads = rss_hf_offload_capa;
+		PRIV(dev)->infos.dev_capa = dev_capa;
 		PRIV(dev)->infos.tx_offload_capa &=
 					default_infos.tx_offload_capa;
 		PRIV(dev)->infos.tx_queue_offload_capa &=
@@ -953,6 +1133,55 @@ fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 }
 
 static int
+fs_set_mc_addr_list(struct rte_eth_dev *dev,
+		    struct ether_addr *mc_addr_set, uint32_t nb_mc_addr)
+{
+	struct sub_device *sdev;
+	uint8_t i;
+	int ret;
+	void *mcast_addrs;
+
+	fs_lock(dev, 0);
+
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		ret = rte_eth_dev_set_mc_addr_list(PORT_ID(sdev),
+						   mc_addr_set, nb_mc_addr);
+		if (ret != 0) {
+			ERROR("Operation rte_eth_dev_set_mc_addr_list failed for sub_device %d with error %d",
+			      i, ret);
+			goto rollback;
+		}
+	}
+
+	mcast_addrs = rte_realloc(PRIV(dev)->mcast_addrs,
+		nb_mc_addr * sizeof(PRIV(dev)->mcast_addrs[0]), 0);
+	if (mcast_addrs == NULL && nb_mc_addr > 0) {
+		ret = -ENOMEM;
+		goto rollback;
+	}
+	rte_memcpy(mcast_addrs, mc_addr_set,
+		   nb_mc_addr * sizeof(PRIV(dev)->mcast_addrs[0]));
+	PRIV(dev)->nb_mcast_addr = nb_mc_addr;
+	PRIV(dev)->mcast_addrs = mcast_addrs;
+
+	fs_unlock(dev, 0);
+	return 0;
+
+rollback:
+	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
+		int rc = rte_eth_dev_set_mc_addr_list(PORT_ID(sdev),
+			PRIV(dev)->mcast_addrs,	PRIV(dev)->nb_mcast_addr);
+		if (rc != 0) {
+			ERROR("Multicast MAC address list rollback for sub_device %d failed with error %d",
+			      i, rc);
+		}
+	}
+
+	fs_unlock(dev, 0);
+	return ret;
+}
+
+static int
 fs_rss_hash_update(struct rte_eth_dev *dev,
 			struct rte_eth_rss_conf *rss_conf)
 {
@@ -1025,6 +1254,10 @@ const struct eth_dev_ops failsafe_ops = {
 	.dev_supported_ptypes_get = fs_dev_supported_ptypes_get,
 	.mtu_set = fs_mtu_set,
 	.vlan_filter_set = fs_vlan_filter_set,
+	.rx_queue_start = fs_rx_queue_start,
+	.rx_queue_stop = fs_rx_queue_stop,
+	.tx_queue_start = fs_tx_queue_start,
+	.tx_queue_stop = fs_tx_queue_stop,
 	.rx_queue_setup = fs_rx_queue_setup,
 	.tx_queue_setup = fs_tx_queue_setup,
 	.rx_queue_release = fs_rx_queue_release,
@@ -1036,6 +1269,7 @@ const struct eth_dev_ops failsafe_ops = {
 	.mac_addr_remove = fs_mac_addr_remove,
 	.mac_addr_add = fs_mac_addr_add,
 	.mac_addr_set = fs_mac_addr_set,
+	.set_mc_addr_list = fs_set_mc_addr_list,
 	.rss_hash_update = fs_rss_hash_update,
 	.filter_ctrl = fs_filter_ctrl,
 };
diff --git a/drivers/net/failsafe/failsafe_private.h b/drivers/net/failsafe/failsafe_private.h
index 886af861..7e318968 100644
--- a/drivers/net/failsafe/failsafe_private.h
+++ b/drivers/net/failsafe/failsafe_private.h
@@ -6,6 +6,7 @@
 #ifndef _RTE_ETH_FAILSAFE_PRIVATE_H_
 #define _RTE_ETH_FAILSAFE_PRIVATE_H_
 
+#include <stdint.h>
 #include <sys/queue.h>
 #include <pthread.h>
 
@@ -13,6 +14,7 @@
 #include <rte_dev.h>
 #include <rte_ethdev_driver.h>
 #include <rte_devargs.h>
+#include <rte_flow.h>
 #include <rte_interrupts.h>
 
 #define FAILSAFE_DRIVER_NAME "Fail-safe PMD"
@@ -81,7 +83,8 @@ struct rte_flow {
 	/* sub_flows */
 	struct rte_flow *flows[FAILSAFE_MAX_ETHPORTS];
 	/* flow description for synchronization */
-	struct rte_flow_desc *fd;
+	struct rte_flow_conv_rule rule;
+	uint8_t rule_data[];
 };
 
 enum dev_state {
@@ -143,6 +146,8 @@ struct fs_priv {
 	uint32_t nb_mac_addr;
 	struct ether_addr mac_addrs[FAILSAFE_MAX_ETHADDR];
 	uint32_t mac_addr_pool[FAILSAFE_MAX_ETHADDR];
+	uint32_t nb_mcast_addr;
+	struct ether_addr *mcast_addrs;
 	/* current capabilities */
 	struct rte_eth_dev_info infos;
 	struct rte_eth_dev_owner my_owner; /* Unique owner. */
@@ -188,7 +193,7 @@ int failsafe_hotplug_alarm_cancel(struct rte_eth_dev *dev);
 
 /* RX / TX */
 
-void set_burst_fn(struct rte_eth_dev *dev, int force_safe);
+void failsafe_set_burst_fn(struct rte_eth_dev *dev, int force_safe);
 
 uint16_t failsafe_rx_burst(void *rxq,
 		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
@@ -234,8 +239,8 @@ int failsafe_eth_new_event_callback(uint16_t port_id,
 extern const char pmd_failsafe_driver_name[];
 extern const struct eth_dev_ops failsafe_ops;
 extern const struct rte_flow_ops fs_flow_ops;
-extern uint64_t hotplug_poll;
-extern int mac_from_arg;
+extern uint64_t failsafe_hotplug_poll;
+extern int failsafe_mac_from_arg;
 
 /* HELPERS */
 
@@ -468,7 +473,7 @@ fs_switch_dev(struct rte_eth_dev *dev,
 	} else {
 		return;
 	}
-	set_burst_fn(dev, 0);
+	failsafe_set_burst_fn(dev, 0);
 	rte_wmb();
 }
 
diff --git a/drivers/net/failsafe/failsafe_rxtx.c b/drivers/net/failsafe/failsafe_rxtx.c
index 7bd0f963..034f47b8 100644
--- a/drivers/net/failsafe/failsafe_rxtx.c
+++ b/drivers/net/failsafe/failsafe_rxtx.c
@@ -29,7 +29,7 @@ fs_tx_unsafe(struct sub_device *sdev)
 }
 
 void
-set_burst_fn(struct rte_eth_dev *dev, int force_safe)
+failsafe_set_burst_fn(struct rte_eth_dev *dev, int force_safe)
 {
 	struct sub_device *sdev;
 	uint8_t i;
diff --git a/drivers/net/fm10k/base/meson.build b/drivers/net/fm10k/base/meson.build
index a8fc5fa8..5525cdc8 100644
--- a/drivers/net/fm10k/base/meson.build
+++ b/drivers/net/fm10k/base/meson.build
@@ -15,6 +15,9 @@ error_cflags = ['-Wno-unused-parameter', '-Wno-unused-value',
 	'-Wno-unused-variable', '-Wno-missing-field-initializers'
 ]
 c_args = cflags
+if allow_experimental_apis
+	c_args += '-DALLOW_EXPERIMENTAL_API'
+endif
 foreach flag: error_cflags
 	if cc.has_argument(flag)
 		c_args += flag
diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 541a49b7..c852022d 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -72,7 +72,7 @@ struct fm10k_xstats_name_off {
 	unsigned offset;
 };
 
-struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
+static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
 	{"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
 	{"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
 	{"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
@@ -87,7 +87,7 @@ struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 		sizeof(fm10k_hw_stats_strings[0]))
 
-struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
+static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 	{"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 	{"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 	{"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
@@ -96,7 +96,7 @@ struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 		sizeof(fm10k_hw_stats_rx_q_strings[0]))
 
-struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
+static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 	{"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 	{"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 };
@@ -129,13 +129,13 @@ fm10k_mbx_unlock(struct fm10k_hw *hw)
 }
 
 /* Stubs needed for linkage when vPMD is disabled */
-int __attribute__((weak))
+__rte_weak int
 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 {
 	return -1;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 fm10k_recv_pkts_vec(
 	__rte_unused void *rx_queue,
 	__rte_unused struct rte_mbuf **rx_pkts,
@@ -144,7 +144,7 @@ fm10k_recv_pkts_vec(
 	return 0;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 fm10k_recv_scattered_pkts_vec(
 		__rte_unused void *rx_queue,
 		__rte_unused struct rte_mbuf **rx_pkts,
@@ -153,33 +153,33 @@ fm10k_recv_scattered_pkts_vec(
 	return 0;
 }
 
-int __attribute__((weak))
+__rte_weak int
 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 
 {
 	return -1;
 }
 
-void __attribute__((weak))
+__rte_weak void
 fm10k_rx_queue_release_mbufs_vec(
 		__rte_unused struct fm10k_rx_queue *rxq)
 {
 	return;
 }
 
-void __attribute__((weak))
+__rte_weak void
 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 {
 	return;
 }
 
-int __attribute__((weak))
+__rte_weak int
 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 {
 	return -1;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 			   __rte_unused struct rte_mbuf **tx_pkts,
 			   __rte_unused uint16_t nb_pkts)
@@ -451,12 +451,6 @@ fm10k_dev_configure(struct rte_eth_dev *dev)
 
 	PMD_INIT_FUNC_TRACE();
 
-	/* KEEP_CRC offload flag is not supported by PMD
-	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
-	 */
-	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
-		PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
-
 	/* multipe queue mode checking */
 	ret  = fm10k_check_mq_mode(dev);
 	if (ret != 0) {
@@ -1325,7 +1319,7 @@ fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
 static int
 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 {
-	uint64_t ipackets, opackets, ibytes, obytes;
+	uint64_t ipackets, opackets, ibytes, obytes, imissed;
 	struct fm10k_hw *hw =
 		FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct fm10k_hw_stats *hw_stats =
@@ -1336,22 +1330,25 @@ fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 
 	fm10k_update_hw_stats(hw, hw_stats);
 
-	ipackets = opackets = ibytes = obytes = 0;
+	ipackets = opackets = ibytes = obytes = imissed = 0;
 	for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 		(i < hw->mac.max_queues); ++i) {
 		stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 		stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 		stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 		stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
+		stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
 		ipackets += stats->q_ipackets[i];
 		opackets += stats->q_opackets[i];
 		ibytes   += stats->q_ibytes[i];
 		obytes   += stats->q_obytes[i];
+		imissed  += stats->q_errors[i];
 	}
 	stats->ipackets = ipackets;
 	stats->opackets = opackets;
 	stats->ibytes = ibytes;
 	stats->obytes = obytes;
+	stats->imissed = imissed;
 	return 0;
 }
 
@@ -1796,7 +1793,6 @@ static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
 			   DEV_RX_OFFLOAD_UDP_CKSUM   |
 			   DEV_RX_OFFLOAD_TCP_CKSUM   |
 			   DEV_RX_OFFLOAD_JUMBO_FRAME |
-			   DEV_RX_OFFLOAD_CRC_STRIP   |
 			   DEV_RX_OFFLOAD_HEADER_SPLIT);
 }
 
@@ -1982,6 +1978,7 @@ static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
 	RTE_SET_USED(dev);
 
 	return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
+			  DEV_TX_OFFLOAD_MULTI_SEGS  |
 			  DEV_TX_OFFLOAD_IPV4_CKSUM  |
 			  DEV_TX_OFFLOAD_UDP_CKSUM   |
 			  DEV_TX_OFFLOAD_TCP_CKSUM   |
@@ -3237,14 +3234,6 @@ eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
 			fm10k_dev_interrupt_handler_vf, (void *)dev);
 	}
 
-	/* free mac memory */
-	if (dev->data->mac_addrs) {
-		rte_free(dev->data->mac_addrs);
-		dev->data->mac_addrs = NULL;
-	}
-
-	memset(hw, 0, sizeof(*hw));
-
 	return 0;
 }
 
diff --git a/drivers/net/i40e/base/README b/drivers/net/i40e/base/README
index 247ba11d..84f191fa 100644
--- a/drivers/net/i40e/base/README
+++ b/drivers/net/i40e/base/README
@@ -34,7 +34,7 @@ Intel® I40E driver
 ==================
 
 This directory contains source code of FreeBSD i40e driver of version
-cid-i40e.2018.01.02.tar.gz released by the team which develops
+cid-i40e.2018.09.13.tar.gz released by the team which develops
 basic drivers for any i40e NIC. The directory of base/ contains the
 original source package.
 This driver is valid for the product(s) listed below
diff --git a/drivers/net/i40e/base/i40e_adminq.c b/drivers/net/i40e/base/i40e_adminq.c
index 612be883..38214a37 100644
--- a/drivers/net/i40e/base/i40e_adminq.c
+++ b/drivers/net/i40e/base/i40e_adminq.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "i40e_status.h"
 #include "i40e_type.h"
@@ -126,6 +97,7 @@ enum i40e_status_code i40e_alloc_adminq_arq_ring(struct i40e_hw *hw)
  **/
 void i40e_free_adminq_asq(struct i40e_hw *hw)
 {
+	i40e_free_virt_mem(hw, &hw->aq.asq.cmd_buf);
 	i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf);
 }
 
@@ -433,7 +405,7 @@ enum i40e_status_code i40e_init_asq(struct i40e_hw *hw)
 	/* initialize base registers */
 	ret_code = i40e_config_asq_regs(hw);
 	if (ret_code != I40E_SUCCESS)
-		goto init_adminq_free_rings;
+		goto init_config_regs;
 
 	/* success! */
 	hw->aq.asq.count = hw->aq.num_asq_entries;
@@ -441,6 +413,10 @@ enum i40e_status_code i40e_init_asq(struct i40e_hw *hw)
 
 init_adminq_free_rings:
 	i40e_free_adminq_asq(hw);
+	return ret_code;
+
+init_config_regs:
+	i40e_free_asq_bufs(hw);
 
 init_adminq_exit:
 	return ret_code;
@@ -692,6 +668,12 @@ enum i40e_status_code i40e_init_adminq(struct i40e_hw *hw)
 	    hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
 	    hw->aq.api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710) {
 		hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE;
+		hw->flags |= I40E_HW_FLAG_FW_LLDP_STOPPABLE;
+	}
+	if (hw->mac.type == I40E_MAC_X722 &&
+	    hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
+	    hw->aq.api_min_ver >= I40E_MINOR_VER_FW_LLDP_STOPPABLE_X722) {
+		hw->flags |= I40E_HW_FLAG_FW_LLDP_STOPPABLE;
 	}
 
 	/* Newer versions of firmware require lock when reading the NVM */
@@ -987,6 +969,8 @@ enum i40e_status_code i40e_asq_send_command(struct i40e_hw *hw,
 		cmd_completed = true;
 		if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_OK)
 			status = I40E_SUCCESS;
+		else if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_EBUSY)
+			status = I40E_ERR_NOT_READY;
 		else
 			status = I40E_ERR_ADMIN_QUEUE_ERROR;
 		hw->aq.asq_last_status = (enum i40e_admin_queue_err)retval;
diff --git a/drivers/net/i40e/base/i40e_adminq.h b/drivers/net/i40e/base/i40e_adminq.h
index de4ab3f3..769d8480 100644
--- a/drivers/net/i40e/base/i40e_adminq.h
+++ b/drivers/net/i40e/base/i40e_adminq.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_ADMINQ_H_
 #define _I40E_ADMINQ_H_
diff --git a/drivers/net/i40e/base/i40e_adminq_cmd.h b/drivers/net/i40e/base/i40e_adminq_cmd.h
index 801c0ff1..83062602 100644
--- a/drivers/net/i40e/base/i40e_adminq_cmd.h
+++ b/drivers/net/i40e/base/i40e_adminq_cmd.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_ADMINQ_CMD_H_
 #define _I40E_ADMINQ_CMD_H_
@@ -41,7 +12,7 @@ POSSIBILITY OF SUCH DAMAGE.
  */
 
 #define I40E_FW_API_VERSION_MAJOR	0x0001
-#define I40E_FW_API_VERSION_MINOR_X722	0x0005
+#define I40E_FW_API_VERSION_MINOR_X722	0x0006
 #define I40E_FW_API_VERSION_MINOR_X710	0x0007
 
 #define I40E_FW_MINOR_VERSION(_h) ((_h)->mac.type == I40E_MAC_XL710 ? \
@@ -50,6 +21,8 @@ POSSIBILITY OF SUCH DAMAGE.
 
 /* API version 1.7 implements additional link and PHY-specific APIs  */
 #define I40E_MINOR_VER_GET_LINK_INFO_XL710 0x0007
+/* API version 1.6 for X722 devices adds ability to stop FW LLDP agent */
+#define I40E_MINOR_VER_FW_LLDP_STOPPABLE_X722 0x0006
 
 struct i40e_aq_desc {
 	__le16 flags;
@@ -804,7 +777,35 @@ struct i40e_aqc_set_switch_config {
 	 */
 	__le16	first_tag;
 	__le16	second_tag;
-	u8	reserved[6];
+	/* Next byte is split into following:
+	 * Bit 7    : 0 : No action, 1: Switch to mode defined by bits 6:0
+	 * Bit 6    : 0 : Destination Port, 1: source port
+	 * Bit 5..4 : L4 type
+	 * 0: rsvd
+	 * 1: TCP
+	 * 2: UDP
+	 * 3: Both TCP and UDP
+	 * Bits 3:0 Mode
+	 * 0: default mode
+	 * 1: L4 port only mode
+	 * 2: non-tunneled mode
+	 * 3: tunneled mode
+	 */
+#define I40E_AQ_SET_SWITCH_BIT7_VALID		0x80
+
+#define I40E_AQ_SET_SWITCH_L4_SRC_PORT		0x40
+
+#define I40E_AQ_SET_SWITCH_L4_TYPE_RSVD		0x00
+#define I40E_AQ_SET_SWITCH_L4_TYPE_TCP		0x10
+#define I40E_AQ_SET_SWITCH_L4_TYPE_UDP		0x20
+#define I40E_AQ_SET_SWITCH_L4_TYPE_BOTH		0x30
+
+#define I40E_AQ_SET_SWITCH_MODE_DEFAULT		0x00
+#define I40E_AQ_SET_SWITCH_MODE_L4_PORT		0x01
+#define I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL	0x02
+#define I40E_AQ_SET_SWITCH_MODE_TUNNEL		0x03
+	u8	mode;
+	u8	rsvd5[5];
 };
 
 I40E_CHECK_CMD_LENGTH(i40e_aqc_set_switch_config);
@@ -1359,6 +1360,7 @@ struct i40e_aqc_add_remove_cloud_filters {
 					I40E_AQC_ADD_CLOUD_CMD_SEID_NUM_SHIFT)
 	u8	big_buffer_flag;
 #define I40E_AQC_ADD_REM_CLOUD_CMD_BIG_BUFFER	1
+#define I40E_AQC_ADD_CLOUD_CMD_BB		1
 	u8	reserved2[3];
 	__le32	addr_high;
 	__le32	addr_low;
@@ -1366,7 +1368,7 @@ struct i40e_aqc_add_remove_cloud_filters {
 
 I40E_CHECK_CMD_LENGTH(i40e_aqc_add_remove_cloud_filters);
 
-struct i40e_aqc_add_remove_cloud_filters_element_data {
+struct i40e_aqc_cloud_filters_element_data {
 	u8	outer_mac[6];
 	u8	inner_mac[6];
 	__le16	inner_vlan;
@@ -1378,6 +1380,9 @@ struct i40e_aqc_add_remove_cloud_filters_element_data {
 		struct {
 			u8 data[16];
 		} v6;
+		struct {
+			__le16 data[8];
+		} raw_v6;
 	} ipaddr;
 	__le16	flags;
 #define I40E_AQC_ADD_CLOUD_FILTER_SHIFT			0
@@ -1397,6 +1402,9 @@ struct i40e_aqc_add_remove_cloud_filters_element_data {
 #define I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC	0x000B
 #define I40E_AQC_ADD_CLOUD_FILTER_IIP			0x000C
 /* 0x0010 to 0x0017 is for custom filters */
+#define I40E_AQC_ADD_CLOUD_FILTER_IP_PORT		0x0010 /* Dest IP + L4 Port */
+#define I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT		0x0011 /* Dest MAC + L4 Port */
+#define I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT		0x0012 /* Dest MAC + VLAN + L4 Port */
 
 #define I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE		0x0080
 #define I40E_AQC_ADD_CLOUD_VNK_SHIFT			6
@@ -1436,7 +1444,7 @@ struct i40e_aqc_add_remove_cloud_filters_element_data {
  * DCR288
  */
 struct i40e_aqc_add_rm_cloud_filt_elem_ext {
-	struct i40e_aqc_add_remove_cloud_filters_element_data element;
+	struct i40e_aqc_cloud_filters_element_data element;
 	u16     general_fields[32];
 #define I40E_AQC_ADD_CLOUD_FV_FLU_0X10_WORD0	0
 #define I40E_AQC_ADD_CLOUD_FV_FLU_0X10_WORD1	1
@@ -1471,6 +1479,49 @@ struct i40e_aqc_add_rm_cloud_filt_elem_ext {
 #define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD7	30
 };
 
+I40E_CHECK_STRUCT_LEN(0x40, i40e_aqc_cloud_filters_element_data);
+
+/* i40e_aqc_cloud_filters_element_bb is used when
+ * I40E_AQC_CLOUD_CMD_BB flag is set.
+ */
+struct i40e_aqc_cloud_filters_element_bb {
+	struct i40e_aqc_cloud_filters_element_data element;
+	u16     general_fields[32];
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X10_WORD0	0
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X10_WORD1	1
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X10_WORD2	2
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD0	3
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD1	4
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD2	5
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD0	6
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD1	7
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD2	8
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD0	9
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD1	10
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD2	11
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X14_WORD0	12
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X14_WORD1	13
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X14_WORD2	14
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0	15
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD1	16
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD2	17
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD3	18
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD4	19
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD5	20
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD6	21
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD7	22
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD0	23
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD1	24
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD2	25
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD3	26
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD4	27
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD5	28
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD6	29
+#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD7	30
+};
+
+I40E_CHECK_STRUCT_LEN(0x80, i40e_aqc_cloud_filters_element_bb);
+
 struct i40e_aqc_remove_cloud_filters_completion {
 	__le16 perfect_ovlan_used;
 	__le16 perfect_ovlan_free;
@@ -1491,6 +1542,8 @@ struct i40e_filter_data {
 	u8 input[3];
 };
 
+I40E_CHECK_STRUCT_LEN(4, i40e_filter_data);
+
 struct i40e_aqc_replace_cloud_filters_cmd {
 	u8	valid_flags;
 #define I40E_AQC_REPLACE_L1_FILTER		0x0
@@ -1501,11 +1554,14 @@ struct i40e_aqc_replace_cloud_filters_cmd {
 	u8	old_filter_type;
 	u8	new_filter_type;
 	u8	tr_bit;
-	u8	reserved[4];
+	u8	tr_bit2;
+	u8	reserved[3];
 	__le32 addr_high;
 	__le32 addr_low;
 };
 
+I40E_CHECK_CMD_LENGTH(i40e_aqc_replace_cloud_filters_cmd);
+
 struct i40e_aqc_replace_cloud_filters_cmd_buf {
 	u8	data[32];
 /* Filter type INPUT codes*/
@@ -1530,6 +1586,8 @@ struct i40e_aqc_replace_cloud_filters_cmd_buf {
 	struct i40e_filter_data	filters[8];
 };
 
+I40E_CHECK_STRUCT_LEN(0x40, i40e_aqc_replace_cloud_filters_cmd_buf);
+
 /* Add Mirror Rule (indirect or direct 0x0260)
  * Delete Mirror Rule (indirect or direct 0x0261)
  * note: some rule types (4,5) do not use an external buffer.
@@ -1878,23 +1936,115 @@ enum i40e_aq_phy_type {
 	I40E_PHY_TYPE_25GBASE_LR		= 0x22,
 	I40E_PHY_TYPE_25GBASE_AOC		= 0x23,
 	I40E_PHY_TYPE_25GBASE_ACC		= 0x24,
+#ifdef CARLSVILLE_HW
+	I40E_PHY_TYPE_2_5GBASE_T		= 0x30,
+	I40E_PHY_TYPE_5GBASE_T			= 0x31,
+#endif
 	I40E_PHY_TYPE_MAX,
 	I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP	= 0xFD,
 	I40E_PHY_TYPE_EMPTY			= 0xFE,
 	I40E_PHY_TYPE_DEFAULT			= 0xFF,
 };
 
+#ifdef CARLSVILLE_HW
+#define I40E_PHY_TYPES_BITMASK (BIT_ULL(I40E_PHY_TYPE_SGMII) | \
+				BIT_ULL(I40E_PHY_TYPE_1000BASE_KX) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_KX4) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_KR) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_KR4) | \
+				BIT_ULL(I40E_PHY_TYPE_XAUI) | \
+				BIT_ULL(I40E_PHY_TYPE_XFI) | \
+				BIT_ULL(I40E_PHY_TYPE_SFI) | \
+				BIT_ULL(I40E_PHY_TYPE_XLAUI) | \
+				BIT_ULL(I40E_PHY_TYPE_XLPPI) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_CR4_CU) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_CR1_CU) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_AOC) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_AOC) | \
+				BIT_ULL(I40E_PHY_TYPE_UNRECOGNIZED) | \
+				BIT_ULL(I40E_PHY_TYPE_UNSUPPORTED) | \
+				BIT_ULL(I40E_PHY_TYPE_100BASE_TX) | \
+				BIT_ULL(I40E_PHY_TYPE_1000BASE_T) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_T) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_SR) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_LR) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_SFPP_CU) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_CR1) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_CR4) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_SR4) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_LR4) | \
+				BIT_ULL(I40E_PHY_TYPE_1000BASE_SX) | \
+				BIT_ULL(I40E_PHY_TYPE_1000BASE_LX) | \
+				BIT_ULL(I40E_PHY_TYPE_1000BASE_T_OPTICAL) | \
+				BIT_ULL(I40E_PHY_TYPE_20GBASE_KR2) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_KR) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_CR) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_SR) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_LR) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_AOC) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_ACC) | \
+				BIT_ULL(I40E_PHY_TYPE_2_5GBASE_T) | \
+				BIT_ULL(I40E_PHY_TYPE_5GBASE_T))
+#else
+#define I40E_PHY_TYPES_BITMASK (BIT_ULL(I40E_PHY_TYPE_SGMII) | \
+				BIT_ULL(I40E_PHY_TYPE_1000BASE_KX) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_KX4) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_KR) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_KR4) | \
+				BIT_ULL(I40E_PHY_TYPE_XAUI) | \
+				BIT_ULL(I40E_PHY_TYPE_XFI) | \
+				BIT_ULL(I40E_PHY_TYPE_SFI) | \
+				BIT_ULL(I40E_PHY_TYPE_XLAUI) | \
+				BIT_ULL(I40E_PHY_TYPE_XLPPI) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_CR4_CU) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_CR1_CU) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_AOC) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_AOC) | \
+				BIT_ULL(I40E_PHY_TYPE_UNRECOGNIZED) | \
+				BIT_ULL(I40E_PHY_TYPE_UNSUPPORTED) | \
+				BIT_ULL(I40E_PHY_TYPE_100BASE_TX) | \
+				BIT_ULL(I40E_PHY_TYPE_1000BASE_T) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_T) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_SR) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_LR) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_SFPP_CU) | \
+				BIT_ULL(I40E_PHY_TYPE_10GBASE_CR1) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_CR4) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_SR4) | \
+				BIT_ULL(I40E_PHY_TYPE_40GBASE_LR4) | \
+				BIT_ULL(I40E_PHY_TYPE_1000BASE_SX) | \
+				BIT_ULL(I40E_PHY_TYPE_1000BASE_LX) | \
+				BIT_ULL(I40E_PHY_TYPE_1000BASE_T_OPTICAL) | \
+				BIT_ULL(I40E_PHY_TYPE_20GBASE_KR2) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_KR) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_CR) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_SR) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_LR) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_AOC) | \
+				BIT_ULL(I40E_PHY_TYPE_25GBASE_ACC))
+#endif
+
+#ifdef CARLSVILLE_HW
+#define I40E_LINK_SPEED_2_5GB_SHIFT	0x0
+#endif
 #define I40E_LINK_SPEED_100MB_SHIFT	0x1
 #define I40E_LINK_SPEED_1000MB_SHIFT	0x2
 #define I40E_LINK_SPEED_10GB_SHIFT	0x3
 #define I40E_LINK_SPEED_40GB_SHIFT	0x4
 #define I40E_LINK_SPEED_20GB_SHIFT	0x5
 #define I40E_LINK_SPEED_25GB_SHIFT	0x6
+#ifdef CARLSVILLE_HW
+#define I40E_LINK_SPEED_5GB_SHIFT	0x7
+#endif
 
 enum i40e_aq_link_speed {
 	I40E_LINK_SPEED_UNKNOWN	= 0,
 	I40E_LINK_SPEED_100MB	= (1 << I40E_LINK_SPEED_100MB_SHIFT),
 	I40E_LINK_SPEED_1GB	= (1 << I40E_LINK_SPEED_1000MB_SHIFT),
+#ifdef CARLSVILLE_HW
+	I40E_LINK_SPEED_2_5GB	= (1 << I40E_LINK_SPEED_2_5GB_SHIFT),
+	I40E_LINK_SPEED_5GB	= (1 << I40E_LINK_SPEED_5GB_SHIFT),
+#endif
 	I40E_LINK_SPEED_10GB	= (1 << I40E_LINK_SPEED_10GB_SHIFT),
 	I40E_LINK_SPEED_40GB	= (1 << I40E_LINK_SPEED_40GB_SHIFT),
 	I40E_LINK_SPEED_20GB	= (1 << I40E_LINK_SPEED_20GB_SHIFT),
@@ -1940,6 +2090,10 @@ struct i40e_aq_get_phy_abilities_resp {
 #define I40E_AQ_PHY_TYPE_EXT_25G_LR	0x08
 #define I40E_AQ_PHY_TYPE_EXT_25G_AOC	0x10
 #define I40E_AQ_PHY_TYPE_EXT_25G_ACC	0x20
+#ifdef CARLSVILLE_HW
+#define I40E_AQ_PHY_TYPE_EXT_2_5GBASE_T	0x40
+#define I40E_AQ_PHY_TYPE_EXT_5GBASE_T	0x80
+#endif
 	u8	fec_cfg_curr_mod_ext_info;
 #define I40E_AQ_ENABLE_FEC_KR		0x01
 #define I40E_AQ_ENABLE_FEC_RS		0x02
@@ -2184,7 +2338,9 @@ struct i40e_aqc_phy_register_access {
 #define I40E_AQ_PHY_REG_ACCESS_EXTERNAL	1
 #define I40E_AQ_PHY_REG_ACCESS_EXTERNAL_MODULE	2
 	u8	dev_addres;
-	u8	reserved1[2];
+	u8	cmd_flags;
+#define I40E_AQ_PHY_REG_ACCESS_DONT_CHANGE_QSFP_PAGE	1
+	u8	reserved1;
 	__le32	reg_address;
 	__le32	reg_value;
 	u8	reserved2[4];
@@ -2199,6 +2355,8 @@ I40E_CHECK_CMD_LENGTH(i40e_aqc_phy_register_access);
 struct i40e_aqc_nvm_update {
 	u8	command_flags;
 #define I40E_AQ_NVM_LAST_CMD			0x01
+#define I40E_AQ_NVM_REARRANGE_TO_FLAT		0x20
+#define I40E_AQ_NVM_REARRANGE_TO_STRUCT		0x40
 #define I40E_AQ_NVM_FLASH_ONLY			0x80
 #define I40E_AQ_NVM_PRESERVATION_FLAGS_SHIFT	1
 #define I40E_AQ_NVM_PRESERVATION_FLAGS_MASK	0x03
diff --git a/drivers/net/i40e/base/i40e_alloc.h b/drivers/net/i40e/base/i40e_alloc.h
index 38c2f655..4fc18601 100644
--- a/drivers/net/i40e/base/i40e_alloc.h
+++ b/drivers/net/i40e/base/i40e_alloc.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_ALLOC_H_
 #define _I40E_ALLOC_H_
diff --git a/drivers/net/i40e/base/i40e_common.c b/drivers/net/i40e/base/i40e_common.c
index e0a5be14..8a98afff 100644
--- a/drivers/net/i40e/base/i40e_common.c
+++ b/drivers/net/i40e/base/i40e_common.c
@@ -1,42 +1,12 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "i40e_type.h"
 #include "i40e_adminq.h"
 #include "i40e_prototype.h"
 #include "virtchnl.h"
 
-
 /**
  * i40e_set_mac_type - Sets MAC type
  * @hw: pointer to the HW structure
@@ -65,6 +35,9 @@ STATIC enum i40e_status_code i40e_set_mac_type(struct i40e_hw *hw)
 		case I40E_DEV_ID_QSFP_C:
 		case I40E_DEV_ID_10G_BASE_T:
 		case I40E_DEV_ID_10G_BASE_T4:
+#ifdef CARLSVILLE_HW
+		case I40E_DEV_ID_10G_BASE_T_BC:
+#endif
 		case I40E_DEV_ID_20G_KR2:
 		case I40E_DEV_ID_20G_KR2_A:
 		case I40E_DEV_ID_25G_B:
@@ -1290,6 +1263,10 @@ STATIC enum i40e_media_type i40e_get_media_type(struct i40e_hw *hw)
 		break;
 	case I40E_PHY_TYPE_100BASE_TX:
 	case I40E_PHY_TYPE_1000BASE_T:
+#ifdef CARLSVILLE_HW
+	case I40E_PHY_TYPE_2_5GBASE_T:
+	case I40E_PHY_TYPE_5GBASE_T:
+#endif
 	case I40E_PHY_TYPE_10GBASE_T:
 		media = I40E_MEDIA_TYPE_BASET;
 		break;
@@ -1326,6 +1303,29 @@ STATIC enum i40e_media_type i40e_get_media_type(struct i40e_hw *hw)
 	return media;
 }
 
+/**
+ * i40e_poll_globr - Poll for Global Reset completion
+ * @hw: pointer to the hardware structure
+ * @retry_limit: how many times to retry before failure
+ **/
+STATIC enum i40e_status_code i40e_poll_globr(struct i40e_hw *hw,
+					     u32 retry_limit)
+{
+	u32 cnt, reg = 0;
+
+	for (cnt = 0; cnt < retry_limit; cnt++) {
+		reg = rd32(hw, I40E_GLGEN_RSTAT);
+		if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK))
+			return I40E_SUCCESS;
+		i40e_msec_delay(100);
+	}
+
+	DEBUGOUT("Global reset failed.\n");
+	DEBUGOUT1("I40E_GLGEN_RSTAT = 0x%x\n", reg);
+
+	return I40E_ERR_RESET_FAILED;
+}
+
 #define I40E_PF_RESET_WAIT_COUNT	200
 /**
  * i40e_pf_reset - Reset the PF
@@ -1349,7 +1349,7 @@ enum i40e_status_code i40e_pf_reset(struct i40e_hw *hw)
 			I40E_GLGEN_RSTCTL_GRSTDEL_MASK) >>
 			I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT;
 
-	grst_del = grst_del * 20;
+	grst_del = min(grst_del * 20, 160U);
 
 	for (cnt = 0; cnt < grst_del; cnt++) {
 		reg = rd32(hw, I40E_GLGEN_RSTAT);
@@ -1395,14 +1395,14 @@ enum i40e_status_code i40e_pf_reset(struct i40e_hw *hw)
 			if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK))
 				break;
 			reg2 = rd32(hw, I40E_GLGEN_RSTAT);
-			if (reg2 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) {
-				DEBUGOUT("Core reset upcoming. Skipping PF reset request.\n");
-				DEBUGOUT1("I40E_GLGEN_RSTAT = 0x%x\n", reg2);
-				return I40E_ERR_NOT_READY;
-			}
+			if (reg2 & I40E_GLGEN_RSTAT_DEVSTATE_MASK)
+				break;
 			i40e_msec_delay(1);
 		}
-		if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) {
+		if (reg2 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) {
+			if (i40e_poll_globr(hw, grst_del) != I40E_SUCCESS)
+				return I40E_ERR_RESET_FAILED;
+		} else if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) {
 			DEBUGOUT("PF reset polling failed to complete.\n");
 			return I40E_ERR_RESET_FAILED;
 		}
@@ -1883,6 +1883,10 @@ enum i40e_status_code i40e_aq_set_mac_config(struct i40e_hw *hw,
 	if (crc_en)
 		cmd->params |= I40E_AQ_SET_MAC_CONFIG_CRC_EN;
 
+#define I40E_AQ_SET_MAC_CONFIG_FC_DEFAULT_THRESHOLD	0x7FFF
+	cmd->fc_refresh_threshold =
+		CPU_TO_LE16(I40E_AQ_SET_MAC_CONFIG_FC_DEFAULT_THRESHOLD);
+
 	status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details);
 
 	return status;
@@ -2711,13 +2715,14 @@ enum i40e_status_code i40e_aq_get_switch_config(struct i40e_hw *hw,
  * i40e_aq_set_switch_config
  * @hw: pointer to the hardware structure
  * @flags: bit flag values to set
+ * @mode: cloud filter mode
  * @valid_flags: which bit flags to set
  * @cmd_details: pointer to command details structure or NULL
  *
  * Set switch configuration bits
  **/
 enum i40e_status_code i40e_aq_set_switch_config(struct i40e_hw *hw,
-				u16 flags, u16 valid_flags,
+				u16 flags, u16 valid_flags, u8 mode,
 				struct i40e_asq_cmd_details *cmd_details)
 {
 	struct i40e_aq_desc desc;
@@ -2729,6 +2734,7 @@ enum i40e_status_code i40e_aq_set_switch_config(struct i40e_hw *hw,
 					  i40e_aqc_opc_set_switch_config);
 	scfg->flags = CPU_TO_LE16(flags);
 	scfg->valid_flags = CPU_TO_LE16(valid_flags);
+	scfg->mode = mode;
 	if (hw->flags & I40E_HW_FLAG_802_1AD_CAPABLE) {
 		scfg->switch_tag = CPU_TO_LE16(hw->switch_tag);
 		scfg->first_tag = CPU_TO_LE16(hw->first_tag);
@@ -3708,9 +3714,10 @@ STATIC void i40e_parse_discover_capabilities(struct i40e_hw *hw, void *buff,
 	u32 valid_functions, num_functions;
 	u32 number, logical_id, phys_id;
 	struct i40e_hw_capabilities *p;
+	enum i40e_status_code status;
+	u16 id, ocp_cfg_word0;
 	u8 major_rev;
 	u32 i = 0;
-	u16 id;
 
 	cap = (struct i40e_aqc_list_capabilities_element_resp *) buff;
 
@@ -4002,6 +4009,26 @@ STATIC void i40e_parse_discover_capabilities(struct i40e_hw *hw, void *buff,
 			hw->num_ports++;
 	}
 
+	/* OCP cards case: if a mezz is removed the ethernet port is at
+	 * disabled state in PRTGEN_CNF register. Additional NVM read is
+	 * needed in order to check if we are dealing with OCP card.
+	 * Those cards have 4 PFs at minimum, so using PRTGEN_CNF for counting
+	 * physical ports results in wrong partition id calculation and thus
+	 * not supporting WoL.
+	 */
+	if (hw->mac.type == I40E_MAC_X722) {
+		if (i40e_acquire_nvm(hw, I40E_RESOURCE_READ) == I40E_SUCCESS) {
+			status = i40e_aq_read_nvm(hw, I40E_SR_EMP_MODULE_PTR,
+						  2 * I40E_SR_OCP_CFG_WORD0,
+						  sizeof(ocp_cfg_word0),
+						  &ocp_cfg_word0, true, NULL);
+			if (status == I40E_SUCCESS &&
+			    (ocp_cfg_word0 & I40E_SR_OCP_ENABLED))
+				hw->num_ports = 4;
+			i40e_release_nvm(hw);
+		}
+	}
+
 	valid_functions = p->valid_functions;
 	num_functions = 0;
 	while (valid_functions) {
@@ -4132,6 +4159,43 @@ i40e_aq_update_nvm_exit:
 }
 
 /**
+ * i40e_aq_rearrange_nvm
+ * @hw: pointer to the hw struct
+ * @rearrange_nvm: defines direction of rearrangement
+ * @cmd_details: pointer to command details structure or NULL
+ *
+ * Rearrange NVM structure, available only for transition FW
+ **/
+enum i40e_status_code i40e_aq_rearrange_nvm(struct i40e_hw *hw,
+				u8 rearrange_nvm,
+				struct i40e_asq_cmd_details *cmd_details)
+{
+	struct i40e_aqc_nvm_update *cmd;
+	enum i40e_status_code status;
+	struct i40e_aq_desc desc;
+
+	DEBUGFUNC("i40e_aq_rearrange_nvm");
+
+	cmd = (struct i40e_aqc_nvm_update *)&desc.params.raw;
+
+	i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_nvm_update);
+
+	rearrange_nvm &= (I40E_AQ_NVM_REARRANGE_TO_FLAT |
+			 I40E_AQ_NVM_REARRANGE_TO_STRUCT);
+
+	if (!rearrange_nvm) {
+		status = I40E_ERR_PARAM;
+		goto i40e_aq_rearrange_nvm_exit;
+	}
+
+	cmd->command_flags |= rearrange_nvm;
+	status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details);
+
+i40e_aq_rearrange_nvm_exit:
+	return status;
+}
+
+/**
  * i40e_aq_nvm_progress
  * @hw: pointer to the hw struct
  * @progress: pointer to progress returned from AQ
@@ -4488,6 +4552,9 @@ i40e_aq_set_dcb_parameters(struct i40e_hw *hw, bool dcb_enable,
 		(struct i40e_aqc_set_dcb_parameters *)&desc.params.raw;
 	enum i40e_status_code status;
 
+	if (!(hw->flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
+		return I40E_ERR_DEVICE_NOT_SUPPORTED;
+
 	i40e_fill_default_direct_cmd_desc(&desc,
 					  i40e_aqc_opc_set_dcb_parameters);
 
@@ -5693,10 +5760,10 @@ void i40e_add_filter_to_drop_tx_flow_control_frames(struct i40e_hw *hw,
  * to be shifted 1 byte over from the VxLAN VNI
  **/
 STATIC void i40e_fix_up_geneve_vni(
-	struct i40e_aqc_add_remove_cloud_filters_element_data *filters,
+	struct i40e_aqc_cloud_filters_element_data *filters,
 	u8 filter_count)
 {
-	struct i40e_aqc_add_remove_cloud_filters_element_data *f = filters;
+	struct i40e_aqc_cloud_filters_element_data *f = filters;
 	int i;
 
 	for (i = 0; i < filter_count; i++) {
@@ -5721,13 +5788,13 @@ STATIC void i40e_fix_up_geneve_vni(
  * @filter_count: number of filters contained in the buffer
  *
  * Set the cloud filters for a given VSI.  The contents of the
- * i40e_aqc_add_remove_cloud_filters_element_data are filled
+ * i40e_aqc_cloud_filters_element_data are filled
  * in by the caller of the function.
  *
  **/
 enum i40e_status_code i40e_aq_add_cloud_filters(struct i40e_hw *hw,
 	u16 seid,
-	struct i40e_aqc_add_remove_cloud_filters_element_data *filters,
+	struct i40e_aqc_cloud_filters_element_data *filters,
 	u8 filter_count)
 {
 	struct i40e_aq_desc desc;
@@ -5753,21 +5820,21 @@ enum i40e_status_code i40e_aq_add_cloud_filters(struct i40e_hw *hw,
 }
 
 /**
- * i40e_aq_add_cloud_filters_big_buffer
+ * i40e_aq_add_cloud_filters_bb
  * @hw: pointer to the hardware structure
  * @seid: VSI seid to add cloud filters from
  * @filters: Buffer which contains the filters in big buffer to be added
  * @filter_count: number of filters contained in the buffer
  *
  * Set the cloud filters for a given VSI.  The contents of the
- * i40e_aqc_add_rm_cloud_filt_elem_ext are filled in by the caller of
+ * i40e_aqc_cloud_filters_element_bb are filled in by the caller of the
  * the function.
  *
  **/
-enum i40e_status_code i40e_aq_add_cloud_filters_big_buffer(struct i40e_hw *hw,
-	u16 seid,
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext *filters,
-	u8 filter_count)
+enum i40e_status_code
+i40e_aq_add_cloud_filters_bb(struct i40e_hw *hw, u16 seid,
+			     struct i40e_aqc_cloud_filters_element_bb *filters,
+			     u8 filter_count)
 {
 	struct i40e_aq_desc desc;
 	struct i40e_aqc_add_remove_cloud_filters *cmd =
@@ -5784,9 +5851,8 @@ enum i40e_status_code i40e_aq_add_cloud_filters_big_buffer(struct i40e_hw *hw,
 	desc.flags |= CPU_TO_LE16((u16)(I40E_AQ_FLAG_BUF | I40E_AQ_FLAG_RD));
 	cmd->num_filters = filter_count;
 	cmd->seid = CPU_TO_LE16(seid);
-	cmd->big_buffer_flag = I40E_AQC_ADD_REM_CLOUD_CMD_BIG_BUFFER;
+	cmd->big_buffer_flag = I40E_AQC_ADD_CLOUD_CMD_BB;
 
-	/* adjust Geneve VNI for HW issue */
 	for (i = 0; i < filter_count; i++) {
 		u16 tnl_type;
 		u32 ti;
@@ -5794,6 +5860,11 @@ enum i40e_status_code i40e_aq_add_cloud_filters_big_buffer(struct i40e_hw *hw,
 		tnl_type = (LE16_TO_CPU(filters[i].element.flags) &
 			   I40E_AQC_ADD_CLOUD_TNL_TYPE_MASK) >>
 			   I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT;
+
+		/* Due to hardware eccentricities, the VNI for Geneve is shifted
+		 * one more byte further than normally used for Tenant ID in
+		 * other tunnel types.
+		 */
 		if (tnl_type == I40E_AQC_ADD_CLOUD_TNL_TYPE_GENEVE) {
 			ti = LE32_TO_CPU(filters[i].element.tenant_id);
 			filters[i].element.tenant_id = CPU_TO_LE32(ti << 8);
@@ -5806,21 +5877,21 @@ enum i40e_status_code i40e_aq_add_cloud_filters_big_buffer(struct i40e_hw *hw,
 }
 
 /**
- * i40e_aq_remove_cloud_filters
+ * i40e_aq_rem_cloud_filters
  * @hw: pointer to the hardware structure
  * @seid: VSI seid to remove cloud filters from
  * @filters: Buffer which contains the filters to be removed
  * @filter_count: number of filters contained in the buffer
  *
  * Remove the cloud filters for a given VSI.  The contents of the
- * i40e_aqc_add_remove_cloud_filters_element_data are filled
- * in by the caller of the function.
+ * i40e_aqc_cloud_filters_element_data are filled in by the caller
+ * of the function.
  *
  **/
-enum i40e_status_code i40e_aq_remove_cloud_filters(struct i40e_hw *hw,
-	u16 seid,
-	struct i40e_aqc_add_remove_cloud_filters_element_data *filters,
-	u8 filter_count)
+enum i40e_status_code
+i40e_aq_rem_cloud_filters(struct i40e_hw *hw, u16 seid,
+			  struct i40e_aqc_cloud_filters_element_data *filters,
+			  u8 filter_count)
 {
 	struct i40e_aq_desc desc;
 	struct i40e_aqc_add_remove_cloud_filters *cmd =
@@ -5845,22 +5916,21 @@ enum i40e_status_code i40e_aq_remove_cloud_filters(struct i40e_hw *hw,
 }
 
 /**
- * i40e_aq_remove_cloud_filters_big_buffer
+ * i40e_aq_rem_cloud_filters_bb
  * @hw: pointer to the hardware structure
  * @seid: VSI seid to remove cloud filters from
  * @filters: Buffer which contains the filters in big buffer to be removed
  * @filter_count: number of filters contained in the buffer
  *
- * Remove the cloud filters for a given VSI.  The contents of the
- * i40e_aqc_add_rm_cloud_filt_elem_ext are filled in by the caller of
- * the function.
+ * Remove the big buffer cloud filters for a given VSI.  The contents of the
+ * i40e_aqc_cloud_filters_element_bb are filled in by the caller of the
+ * function.
  *
  **/
-enum i40e_status_code i40e_aq_remove_cloud_filters_big_buffer(
-	struct i40e_hw *hw,
-	u16 seid,
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext *filters,
-	u8 filter_count)
+enum i40e_status_code
+i40e_aq_rem_cloud_filters_bb(struct i40e_hw *hw, u16 seid,
+			     struct i40e_aqc_cloud_filters_element_bb *filters,
+			     u8 filter_count)
 {
 	struct i40e_aq_desc desc;
 	struct i40e_aqc_add_remove_cloud_filters *cmd =
@@ -5877,9 +5947,8 @@ enum i40e_status_code i40e_aq_remove_cloud_filters_big_buffer(
 	desc.flags |= CPU_TO_LE16((u16)(I40E_AQ_FLAG_BUF | I40E_AQ_FLAG_RD));
 	cmd->num_filters = filter_count;
 	cmd->seid = CPU_TO_LE16(seid);
-	cmd->big_buffer_flag = I40E_AQC_ADD_REM_CLOUD_CMD_BIG_BUFFER;
+	cmd->big_buffer_flag = I40E_AQC_ADD_CLOUD_CMD_BB;
 
-	/* adjust Geneve VNI for HW issue */
 	for (i = 0; i < filter_count; i++) {
 		u16 tnl_type;
 		u32 ti;
@@ -5887,6 +5956,11 @@ enum i40e_status_code i40e_aq_remove_cloud_filters_big_buffer(
 		tnl_type = (LE16_TO_CPU(filters[i].element.flags) &
 			   I40E_AQC_ADD_CLOUD_TNL_TYPE_MASK) >>
 			   I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT;
+
+		/* Due to hardware eccentricities, the VNI for Geneve is shifted
+		 * one more byte further than normally used for Tenant ID in
+		 * other tunnel types.
+		 */
 		if (tnl_type == I40E_AQC_ADD_CLOUD_TNL_TYPE_GENEVE) {
 			ti = LE32_TO_CPU(filters[i].element.tenant_id);
 			filters[i].element.tenant_id = CPU_TO_LE32(ti << 8);
@@ -5916,6 +5990,14 @@ i40e_status_code i40e_aq_replace_cloud_filters(struct i40e_hw *hw,
 	enum i40e_status_code status = I40E_SUCCESS;
 	int i = 0;
 
+	/* X722 doesn't support this command */
+	if (hw->mac.type == I40E_MAC_X722)
+		return I40E_ERR_DEVICE_NOT_SUPPORTED;
+
+	/* need FW version greater than 6.00 */
+	if (hw->aq.fw_maj_ver < 6)
+		return I40E_NOT_SUPPORTED;
+
 	i40e_fill_default_direct_cmd_desc(&desc,
 					  i40e_aqc_opc_replace_cloud_filters);
 
@@ -5925,6 +6007,7 @@ i40e_status_code i40e_aq_replace_cloud_filters(struct i40e_hw *hw,
 	cmd->new_filter_type = filters->new_filter_type;
 	cmd->valid_flags = filters->valid_flags;
 	cmd->tr_bit = filters->tr_bit;
+	cmd->tr_bit2 = filters->tr_bit2;
 
 	status = i40e_asq_send_command(hw, &desc, cmd_buf,
 		sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf),  NULL);
@@ -6618,6 +6701,9 @@ enum i40e_status_code i40e_write_phy_register(struct i40e_hw *hw,
 		break;
 	case I40E_DEV_ID_10G_BASE_T:
 	case I40E_DEV_ID_10G_BASE_T4:
+#ifdef CARLSVILLE_HW
+	case I40E_DEV_ID_10G_BASE_T_BC:
+#endif
 	case I40E_DEV_ID_10G_BASE_T_X722:
 	case I40E_DEV_ID_25G_B:
 	case I40E_DEV_ID_25G_SFP28:
@@ -6773,7 +6859,7 @@ static enum i40e_status_code i40e_led_get_reg(struct i40e_hw *hw, u16 led_addr,
 	if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) {
 		status = i40e_aq_get_phy_register(hw,
 						I40E_AQ_PHY_REG_ACCESS_EXTERNAL,
-						I40E_PHY_COM_REG_PAGE,
+						I40E_PHY_COM_REG_PAGE, true,
 						I40E_PHY_LED_PROV_REG_1,
 						reg_val, NULL);
 	} else {
@@ -6801,7 +6887,7 @@ static enum i40e_status_code i40e_led_set_reg(struct i40e_hw *hw, u16 led_addr,
 	if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) {
 		status = i40e_aq_set_phy_register(hw,
 						I40E_AQ_PHY_REG_ACCESS_EXTERNAL,
-						I40E_PHY_COM_REG_PAGE,
+						I40E_PHY_COM_REG_PAGE, true,
 						I40E_PHY_LED_PROV_REG_1,
 						reg_val, NULL);
 	} else {
@@ -6835,7 +6921,7 @@ enum i40e_status_code i40e_led_get_phy(struct i40e_hw *hw, u16 *led_addr,
 	if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) {
 		status = i40e_aq_get_phy_register(hw,
 						I40E_AQ_PHY_REG_ACCESS_EXTERNAL,
-						I40E_PHY_COM_REG_PAGE,
+						I40E_PHY_COM_REG_PAGE, true,
 						I40E_PHY_LED_PROV_REG_1,
 						&reg_val_aq, NULL);
 		if (status == I40E_SUCCESS)
@@ -7036,11 +7122,13 @@ do_retry:
 		wr32(hw, reg_addr, reg_val);
 }
 
+#ifdef PF_DRIVER
 /**
  * i40e_aq_set_phy_register
  * @hw: pointer to the hw struct
  * @phy_select: select which phy should be accessed
  * @dev_addr: PHY device address
+ * @page_change: enable auto page change
  * @reg_addr: PHY register address
  * @reg_val: new register value
  * @cmd_details: pointer to command details structure or NULL
@@ -7048,7 +7136,7 @@ do_retry:
  * Write the external PHY register.
  **/
 enum i40e_status_code i40e_aq_set_phy_register(struct i40e_hw *hw,
-				u8 phy_select, u8 dev_addr,
+				u8 phy_select, u8 dev_addr, bool page_change,
 				u32 reg_addr, u32 reg_val,
 				struct i40e_asq_cmd_details *cmd_details)
 {
@@ -7065,6 +7153,9 @@ enum i40e_status_code i40e_aq_set_phy_register(struct i40e_hw *hw,
 	cmd->reg_address = CPU_TO_LE32(reg_addr);
 	cmd->reg_value = CPU_TO_LE32(reg_val);
 
+	if (!page_change)
+		cmd->cmd_flags = I40E_AQ_PHY_REG_ACCESS_DONT_CHANGE_QSFP_PAGE;
+
 	status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details);
 
 	return status;
@@ -7075,6 +7166,7 @@ enum i40e_status_code i40e_aq_set_phy_register(struct i40e_hw *hw,
  * @hw: pointer to the hw struct
  * @phy_select: select which phy should be accessed
  * @dev_addr: PHY device address
+ * @page_change: enable auto page change
  * @reg_addr: PHY register address
  * @reg_val: read register value
  * @cmd_details: pointer to command details structure or NULL
@@ -7082,7 +7174,7 @@ enum i40e_status_code i40e_aq_set_phy_register(struct i40e_hw *hw,
  * Read the external PHY register.
  **/
 enum i40e_status_code i40e_aq_get_phy_register(struct i40e_hw *hw,
-				u8 phy_select, u8 dev_addr,
+				u8 phy_select, u8 dev_addr, bool page_change,
 				u32 reg_addr, u32 *reg_val,
 				struct i40e_asq_cmd_details *cmd_details)
 {
@@ -7098,6 +7190,9 @@ enum i40e_status_code i40e_aq_get_phy_register(struct i40e_hw *hw,
 	cmd->dev_addres = dev_addr;
 	cmd->reg_address = CPU_TO_LE32(reg_addr);
 
+	if (!page_change)
+		cmd->cmd_flags = I40E_AQ_PHY_REG_ACCESS_DONT_CHANGE_QSFP_PAGE;
+
 	status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details);
 	if (!status)
 		*reg_val = LE32_TO_CPU(cmd->reg_value);
@@ -7105,6 +7200,7 @@ enum i40e_status_code i40e_aq_get_phy_register(struct i40e_hw *hw,
 	return status;
 }
 
+#endif /* PF_DRIVER */
 #ifdef VF_DRIVER
 
 /**
diff --git a/drivers/net/i40e/base/i40e_dcb.c b/drivers/net/i40e/base/i40e_dcb.c
index 7600c922..a26f82b3 100644
--- a/drivers/net/i40e/base/i40e_dcb.c
+++ b/drivers/net/i40e/base/i40e_dcb.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "i40e_adminq.h"
 #include "i40e_prototype.h"
@@ -1291,18 +1262,20 @@ static enum i40e_status_code _i40e_read_lldp_cfg(struct i40e_hw *hw,
 {
 	u32 address, offset = (2 * word_offset);
 	enum i40e_status_code ret;
+	__le16 raw_mem;
 	u16 mem;
 
 	ret = i40e_acquire_nvm(hw, I40E_RESOURCE_READ);
 	if (ret != I40E_SUCCESS)
 		return ret;
 
-	ret = i40e_aq_read_nvm(hw, 0x0, module * 2, sizeof(mem), &mem, true,
-			       NULL);
+	ret = i40e_aq_read_nvm(hw, 0x0, module * 2, sizeof(raw_mem), &raw_mem,
+			       true, NULL);
 	i40e_release_nvm(hw);
 	if (ret != I40E_SUCCESS)
 		return ret;
 
+	mem = LE16_TO_CPU(raw_mem);
 	/* Check if this pointer needs to be read in word size or 4K sector
 	 * units.
 	 */
@@ -1315,12 +1288,13 @@ static enum i40e_status_code _i40e_read_lldp_cfg(struct i40e_hw *hw,
 	if (ret != I40E_SUCCESS)
 		goto err_lldp_cfg;
 
-	ret = i40e_aq_read_nvm(hw, module, offset, sizeof(mem), &mem, true,
-			       NULL);
+	ret = i40e_aq_read_nvm(hw, module, offset, sizeof(raw_mem), &raw_mem,
+			       true, NULL);
 	i40e_release_nvm(hw);
 	if (ret != I40E_SUCCESS)
 		return ret;
 
+	mem = LE16_TO_CPU(raw_mem);
 	offset = mem + word_offset;
 	offset *= 2;
 
diff --git a/drivers/net/i40e/base/i40e_dcb.h b/drivers/net/i40e/base/i40e_dcb.h
index 3b709efd..85b0eed3 100644
--- a/drivers/net/i40e/base/i40e_dcb.h
+++ b/drivers/net/i40e/base/i40e_dcb.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_DCB_H_
 #define _I40E_DCB_H_
diff --git a/drivers/net/i40e/base/i40e_devids.h b/drivers/net/i40e/base/i40e_devids.h
index 66ff1ccf..8b667c2a 100644
--- a/drivers/net/i40e/base/i40e_devids.h
+++ b/drivers/net/i40e/base/i40e_devids.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_DEVIDS_H_
 #define _I40E_DEVIDS_H_
@@ -51,6 +22,9 @@ POSSIBILITY OF SUCH DAMAGE.
 #define I40E_DEV_ID_10G_BASE_T4		0x1589
 #define I40E_DEV_ID_25G_B		0x158A
 #define I40E_DEV_ID_25G_SFP28		0x158B
+#ifdef CARLSVILLE_HW
+#define I40E_DEV_ID_10G_BASE_T_BC	0x15FF
+#endif
 #if defined(INTEGRATED_VF) || defined(VF_DRIVER) || defined(I40E_NDIS_SUPPORT)
 #define I40E_DEV_ID_VF			0x154C
 #define I40E_DEV_ID_VF_HV		0x1571
diff --git a/drivers/net/i40e/base/i40e_diag.c b/drivers/net/i40e/base/i40e_diag.c
index c3c76a0c..3ccbea48 100644
--- a/drivers/net/i40e/base/i40e_diag.c
+++ b/drivers/net/i40e/base/i40e_diag.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "i40e_diag.h"
 #include "i40e_prototype.h"
@@ -84,7 +55,7 @@ static enum i40e_status_code i40e_diag_reg_pattern_test(struct i40e_hw *hw,
 	return I40E_SUCCESS;
 }
 
-struct i40e_diag_reg_test_info i40e_reg_list[] = {
+static struct i40e_diag_reg_test_info i40e_reg_list[] = {
 	/* offset               mask         elements   stride */
 	{I40E_QTX_CTL(0),       0x0000FFBF, 1, I40E_QTX_CTL(1) - I40E_QTX_CTL(0)},
 	{I40E_PFINT_ITR0(0),    0x00000FFF, 3, I40E_PFINT_ITR0(1) - I40E_PFINT_ITR0(0)},
diff --git a/drivers/net/i40e/base/i40e_diag.h b/drivers/net/i40e/base/i40e_diag.h
index 105b1191..4434fc96 100644
--- a/drivers/net/i40e/base/i40e_diag.h
+++ b/drivers/net/i40e/base/i40e_diag.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_DIAG_H_
 #define _I40E_DIAG_H_
@@ -50,8 +21,6 @@ struct i40e_diag_reg_test_info {
 	u32 stride;	/* bytes between each element */
 };
 
-extern struct i40e_diag_reg_test_info i40e_reg_list[];
-
 enum i40e_status_code i40e_diag_set_loopback(struct i40e_hw *hw,
 					     enum i40e_lb_mode mode);
 enum i40e_status_code i40e_diag_fw_alive_test(struct i40e_hw *hw);
diff --git a/drivers/net/i40e/base/i40e_hmc.c b/drivers/net/i40e/base/i40e_hmc.c
index 502407bd..11c9ae20 100644
--- a/drivers/net/i40e/base/i40e_hmc.c
+++ b/drivers/net/i40e/base/i40e_hmc.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "i40e_osdep.h"
 #include "i40e_register.h"
diff --git a/drivers/net/i40e/base/i40e_hmc.h b/drivers/net/i40e/base/i40e_hmc.h
index 343b251f..289264ed 100644
--- a/drivers/net/i40e/base/i40e_hmc.h
+++ b/drivers/net/i40e/base/i40e_hmc.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_HMC_H_
 #define _I40E_HMC_H_
diff --git a/drivers/net/i40e/base/i40e_lan_hmc.c b/drivers/net/i40e/base/i40e_lan_hmc.c
index f03f3813..0afee49b 100644
--- a/drivers/net/i40e/base/i40e_lan_hmc.c
+++ b/drivers/net/i40e/base/i40e_lan_hmc.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "i40e_osdep.h"
 #include "i40e_register.h"
@@ -143,7 +114,7 @@ enum i40e_status_code i40e_init_lan_hmc(struct i40e_hw *hw, u32 txq_num,
 		ret_code = I40E_ERR_INVALID_HMC_OBJ_COUNT;
 		DEBUGOUT3("i40e_init_lan_hmc: Tx context: asks for 0x%x but max allowed is 0x%x, returns error %d\n",
 			  txq_num, obj->max_cnt, ret_code);
-		goto init_lan_hmc_out;
+		goto free_hmc_out;
 	}
 
 	/* aggregate values into the full LAN object for later */
@@ -166,7 +137,7 @@ enum i40e_status_code i40e_init_lan_hmc(struct i40e_hw *hw, u32 txq_num,
 		ret_code = I40E_ERR_INVALID_HMC_OBJ_COUNT;
 		DEBUGOUT3("i40e_init_lan_hmc: Rx context: asks for 0x%x but max allowed is 0x%x, returns error %d\n",
 			  rxq_num, obj->max_cnt, ret_code);
-		goto init_lan_hmc_out;
+		goto free_hmc_out;
 	}
 
 	/* aggregate values into the full LAN object for later */
@@ -189,7 +160,7 @@ enum i40e_status_code i40e_init_lan_hmc(struct i40e_hw *hw, u32 txq_num,
 		ret_code = I40E_ERR_INVALID_HMC_OBJ_COUNT;
 		DEBUGOUT3("i40e_init_lan_hmc: FCoE context: asks for 0x%x but max allowed is 0x%x, returns error %d\n",
 			  fcoe_cntx_num, obj->max_cnt, ret_code);
-		goto init_lan_hmc_out;
+		goto free_hmc_out;
 	}
 
 	/* aggregate values into the full LAN object for later */
@@ -212,7 +183,7 @@ enum i40e_status_code i40e_init_lan_hmc(struct i40e_hw *hw, u32 txq_num,
 		ret_code = I40E_ERR_INVALID_HMC_OBJ_COUNT;
 		DEBUGOUT3("i40e_init_lan_hmc: FCoE filter: asks for 0x%x but max allowed is 0x%x, returns error %d\n",
 			  fcoe_filt_num, obj->max_cnt, ret_code);
-		goto init_lan_hmc_out;
+		goto free_hmc_out;
 	}
 
 	/* aggregate values into the full LAN object for later */
@@ -233,7 +204,7 @@ enum i40e_status_code i40e_init_lan_hmc(struct i40e_hw *hw, u32 txq_num,
 					  (sizeof(struct i40e_hmc_sd_entry) *
 					  hw->hmc.sd_table.sd_cnt));
 		if (ret_code)
-			goto init_lan_hmc_out;
+			goto free_hmc_out;
 		hw->hmc.sd_table.sd_entry =
 			(struct i40e_hmc_sd_entry *)hw->hmc.sd_table.addr.va;
 	}
@@ -242,6 +213,11 @@ enum i40e_status_code i40e_init_lan_hmc(struct i40e_hw *hw, u32 txq_num,
 
 init_lan_hmc_out:
 	return ret_code;
+free_hmc_out:
+	if (hw->hmc.hmc_obj_virt_mem.va)
+		i40e_free_virt_mem(hw, &hw->hmc.hmc_obj_virt_mem);
+
+	return ret_code;
 }
 
 /**
diff --git a/drivers/net/i40e/base/i40e_lan_hmc.h b/drivers/net/i40e/base/i40e_lan_hmc.h
index b2a43104..e531ec49 100644
--- a/drivers/net/i40e/base/i40e_lan_hmc.h
+++ b/drivers/net/i40e/base/i40e_lan_hmc.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_LAN_HMC_H_
 #define _I40E_LAN_HMC_H_
diff --git a/drivers/net/i40e/base/i40e_nvm.c b/drivers/net/i40e/base/i40e_nvm.c
index c77dac02..6c8ca877 100644
--- a/drivers/net/i40e/base/i40e_nvm.c
+++ b/drivers/net/i40e/base/i40e_nvm.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "i40e_prototype.h"
 
diff --git a/drivers/net/i40e/base/i40e_osdep.h b/drivers/net/i40e/base/i40e_osdep.h
index 8e5c593c..8a2d82a8 100644
--- a/drivers/net/i40e/base/i40e_osdep.h
+++ b/drivers/net/i40e/base/i40e_osdep.h
@@ -1,34 +1,6 @@
-/******************************************************************************
-
-  Copyright (c) 2001-2015, Intel Corporation
-  All rights reserved.
-
-  Redistribution and use in source and binary forms, with or without
-  modification, are permitted provided that the following conditions are met:
-
-   1. Redistributions of source code must retain the above copyright notice,
-      this list of conditions and the following disclaimer.
-
-   2. Redistributions in binary form must reproduce the above copyright
-      notice, this list of conditions and the following disclaimer in the
-      documentation and/or other materials provided with the distribution.
-
-   3. Neither the name of the Intel Corporation nor the names of its
-      contributors may be used to endorse or promote products derived from
-      this software without specific prior written permission.
-
-  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-  POSSIBILITY OF SUCH DAMAGE.
-******************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_OSDEP_H_
 #define _I40E_OSDEP_H_
@@ -233,9 +205,9 @@ struct i40e_spinlock {
 #define i40e_memcpy(a, b, c, d) rte_memcpy((a), (b), (c))
 
 #define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
-#define DELAY(x) rte_delay_us(x)
-#define i40e_usec_delay(x) rte_delay_us(x)
-#define i40e_msec_delay(x) rte_delay_us(1000*(x))
+#define DELAY(x) rte_delay_us_sleep(x)
+#define i40e_usec_delay(x) DELAY(x)
+#define i40e_msec_delay(x) DELAY(1000 * (x))
 #define udelay(x) DELAY(x)
 #define msleep(x) DELAY(1000*(x))
 #define usleep_range(min, max) msleep(DIV_ROUND_UP(min, 1000))
diff --git a/drivers/net/i40e/base/i40e_prototype.h b/drivers/net/i40e/base/i40e_prototype.h
index c6ec2d76..0cf006da 100644
--- a/drivers/net/i40e/base/i40e_prototype.h
+++ b/drivers/net/i40e/base/i40e_prototype.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_PROTOTYPE_H_
 #define _I40E_PROTOTYPE_H_
@@ -228,7 +199,7 @@ enum i40e_status_code i40e_aq_get_switch_config(struct i40e_hw *hw,
 				u16 buf_size, u16 *start_seid,
 				struct i40e_asq_cmd_details *cmd_details);
 enum i40e_status_code i40e_aq_set_switch_config(struct i40e_hw *hw,
-				u16 flags, u16 valid_flags,
+				u16 flags, u16 valid_flags, u8 mode,
 				struct i40e_asq_cmd_details *cmd_details);
 enum i40e_status_code i40e_aq_request_resource(struct i40e_hw *hw,
 				enum i40e_aq_resources_ids resource,
@@ -265,6 +236,9 @@ enum i40e_status_code i40e_aq_update_nvm(struct i40e_hw *hw, u8 module_pointer,
 				u32 offset, u16 length, void *data,
 				bool last_command, u8 preservation_flags,
 				struct i40e_asq_cmd_details *cmd_details);
+enum i40e_status_code i40e_aq_rearrange_nvm(struct i40e_hw *hw,
+				u8 rearrange_nvm,
+				struct i40e_asq_cmd_details *cmd_details);
 enum i40e_status_code i40e_aq_nvm_progress(struct i40e_hw *hw, u8 *progress,
 				struct i40e_asq_cmd_details *cmd_details);
 enum i40e_status_code i40e_aq_get_lldp_mib(struct i40e_hw *hw, u8 bridge_type,
@@ -404,24 +378,24 @@ enum i40e_status_code i40e_aq_query_switch_comp_bw_config(struct i40e_hw *hw,
 		struct i40e_asq_cmd_details *cmd_details);
 enum i40e_status_code i40e_aq_resume_port_tx(struct i40e_hw *hw,
 				struct i40e_asq_cmd_details *cmd_details);
+enum i40e_status_code
+i40e_aq_add_cloud_filters_bb(struct i40e_hw *hw, u16 seid,
+			     struct i40e_aqc_cloud_filters_element_bb *filters,
+			     u8 filter_count);
+enum i40e_status_code
+i40e_aq_add_cloud_filters(struct i40e_hw *hw, u16 vsi,
+			  struct i40e_aqc_cloud_filters_element_data *filters,
+			  u8 filter_count);
+enum i40e_status_code
+i40e_aq_rem_cloud_filters(struct i40e_hw *hw, u16 vsi,
+			  struct i40e_aqc_cloud_filters_element_data *filters,
+			  u8 filter_count);
+enum i40e_status_code
+i40e_aq_rem_cloud_filters_bb(struct i40e_hw *hw, u16 seid,
+			     struct i40e_aqc_cloud_filters_element_bb *filters,
+			     u8 filter_count);
 enum i40e_status_code i40e_read_lldp_cfg(struct i40e_hw *hw,
 					struct i40e_lldp_variables *lldp_cfg);
-enum i40e_status_code i40e_aq_add_cloud_filters(struct i40e_hw *hw,
-		u16 vsi,
-		struct i40e_aqc_add_remove_cloud_filters_element_data *filters,
-		u8 filter_count);
-enum i40e_status_code i40e_aq_add_cloud_filters_big_buffer(struct i40e_hw *hw,
-	u16 seid,
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext *filters,
-	u8 filter_count);
-enum i40e_status_code i40e_aq_remove_cloud_filters(struct i40e_hw *hw,
-		u16 vsi,
-		struct i40e_aqc_add_remove_cloud_filters_element_data *filters,
-		u8 filter_count);
-enum i40e_status_code i40e_aq_remove_cloud_filters_big_buffer(
-	struct i40e_hw *hw, u16 seid,
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext *filters,
-	u8 filter_count);
 enum i40e_status_code i40e_aq_replace_cloud_filters(struct i40e_hw *hw,
 		struct i40e_aqc_replace_cloud_filters_cmd *filters,
 		struct i40e_aqc_replace_cloud_filters_cmd_buf *cmd_buf);
@@ -574,11 +548,11 @@ enum i40e_status_code i40e_aq_rx_ctl_write_register(struct i40e_hw *hw,
 				struct i40e_asq_cmd_details *cmd_details);
 void i40e_write_rx_ctl(struct i40e_hw *hw, u32 reg_addr, u32 reg_val);
 enum i40e_status_code i40e_aq_set_phy_register(struct i40e_hw *hw,
-				u8 phy_select, u8 dev_addr,
+				u8 phy_select, u8 dev_addr, bool page_change,
 				u32 reg_addr, u32 reg_val,
 				struct i40e_asq_cmd_details *cmd_details);
 enum i40e_status_code i40e_aq_get_phy_register(struct i40e_hw *hw,
-				u8 phy_select, u8 dev_addr,
+				u8 phy_select, u8 dev_addr, bool page_change,
 				u32 reg_addr, u32 *reg_val,
 				struct i40e_asq_cmd_details *cmd_details);
 
diff --git a/drivers/net/i40e/base/i40e_register.h b/drivers/net/i40e/base/i40e_register.h
index df66e76a..e93ec3f5 100644
--- a/drivers/net/i40e/base/i40e_register.h
+++ b/drivers/net/i40e/base/i40e_register.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_REGISTER_H_
 #define _I40E_REGISTER_H_
diff --git a/drivers/net/i40e/base/i40e_status.h b/drivers/net/i40e/base/i40e_status.h
index 49af2d9f..1dad4f4b 100644
--- a/drivers/net/i40e/base/i40e_status.h
+++ b/drivers/net/i40e/base/i40e_status.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_STATUS_H_
 #define _I40E_STATUS_H_
diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 006a11a8..77562f24 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _I40E_TYPE_H_
 #define _I40E_TYPE_H_
@@ -358,6 +329,14 @@ struct i40e_phy_info {
 					     I40E_PHY_TYPE_OFFSET)
 #define I40E_CAP_PHY_TYPE_25GBASE_ACC BIT_ULL(I40E_PHY_TYPE_25GBASE_ACC + \
 					     I40E_PHY_TYPE_OFFSET)
+#ifdef CARLSVILLE_HW
+/* Offset for 2.5G/5G PHY Types value to bit number conversion */
+#define I40E_PHY_TYPE_OFFSET2 (-10)
+#define I40E_CAP_PHY_TYPE_2_5GBASE_T BIT_ULL(I40E_PHY_TYPE_2_5GBASE_T + \
+					     I40E_PHY_TYPE_OFFSET2)
+#define I40E_CAP_PHY_TYPE_5GBASE_T BIT_ULL(I40E_PHY_TYPE_5GBASE_T + \
+					     I40E_PHY_TYPE_OFFSET2)
+#endif
 #define I40E_HW_CAP_MAX_GPIO			30
 #define I40E_HW_CAP_MDIO_PORT_MODE_MDIO		0
 #define I40E_HW_CAP_MDIO_PORT_MODE_I2C		1
@@ -378,6 +357,16 @@ struct i40e_hw_capabilities {
 #define I40E_NVM_IMAGE_TYPE_CLOUD	0x2
 #define I40E_NVM_IMAGE_TYPE_UDP_CLOUD	0x3
 
+	/* Cloud filter modes:
+	 * Mode1: Filter on L4 port only
+	 * Mode2: Filter for non-tunneled traffic
+	 * Mode3: Filter for tunnel traffic
+	 */
+#define I40E_CLOUD_FILTER_MODE1	0x6
+#define I40E_CLOUD_FILTER_MODE2	0x7
+#define I40E_CLOUD_FILTER_MODE3	0x8
+#define I40E_SWITCH_MODE_MASK	0xF
+
 	u32  management_mode;
 	u32  mng_protocols_over_mctp;
 #define I40E_MNG_PROTOCOL_PLDM		0x2
@@ -731,6 +720,7 @@ struct i40e_hw {
 #define I40E_HW_FLAG_802_1AD_CAPABLE        BIT_ULL(1)
 #define I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE  BIT_ULL(2)
 #define I40E_HW_FLAG_NVM_READ_REQUIRES_LOCK BIT_ULL(3)
+#define I40E_HW_FLAG_FW_LLDP_STOPPABLE	    BIT_ULL(4)
 	u64 flags;
 
 	/* Used in set switch config AQ command */
@@ -1541,7 +1531,9 @@ struct i40e_hw_port_stats {
 #define I40E_SR_CONTROL_WORD_1_MASK	(0x03 << I40E_SR_CONTROL_WORD_1_SHIFT)
 #define I40E_SR_CONTROL_WORD_1_NVM_BANK_VALID	BIT(5)
 #define I40E_SR_NVM_MAP_STRUCTURE_TYPE		BIT(12)
-#define I40E_PTR_TYPE                           BIT(15)
+#define I40E_PTR_TYPE				BIT(15)
+#define I40E_SR_OCP_CFG_WORD0			0x2B
+#define I40E_SR_OCP_ENABLED			BIT(15)
 
 /* Shadow RAM related */
 #define I40E_SR_SECTOR_SIZE_IN_WORDS	0x800
diff --git a/drivers/net/i40e/base/meson.build b/drivers/net/i40e/base/meson.build
index 401a1477..d4c8f872 100644
--- a/drivers/net/i40e/base/meson.build
+++ b/drivers/net/i40e/base/meson.build
@@ -12,10 +12,13 @@ sources = [
 ]
 
 error_cflags = ['-Wno-sign-compare', '-Wno-unused-value',
-		'-Wno-format', '-Wno-unused-but-set-variable',
-		'-Wno-strict-aliasing'
+		'-Wno-format', '-Wno-error=format-security',
+		'-Wno-strict-aliasing', '-Wno-unused-but-set-variable'
 ]
 c_args = cflags
+if allow_experimental_apis
+	c_args += '-DALLOW_EXPERIMENTAL_API'
+endif
 foreach flag: error_cflags
 	if cc.has_argument(flag)
 		c_args += flag
diff --git a/drivers/net/i40e/base/virtchnl.h b/drivers/net/i40e/base/virtchnl.h
index b2d5fe73..88096cb4 100644
--- a/drivers/net/i40e/base/virtchnl.h
+++ b/drivers/net/i40e/base/virtchnl.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2013 - 2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _VIRTCHNL_H_
 #define _VIRTCHNL_H_
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 85a6a867..1c779068 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -44,6 +44,7 @@
 #define ETH_I40E_FLOATING_VEB_LIST_ARG	"floating_veb_list"
 #define ETH_I40E_SUPPORT_MULTI_DRIVER	"support-multi-driver"
 #define ETH_I40E_QUEUE_NUM_PER_VF_ARG	"queue-num-per-vf"
+#define ETH_I40E_USE_LATEST_VEC	"use-latest-supported-vec"
 
 #define I40E_CLEAR_PXE_WAIT_MS     200
 
@@ -292,6 +293,7 @@ static void i40e_stat_update_48(struct i40e_hw *hw,
 			       uint64_t *stat);
 static void i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue);
 static void i40e_dev_interrupt_handler(void *param);
+static void i40e_dev_alarm_handler(void *param);
 static int i40e_res_pool_init(struct i40e_res_pool_info *pool,
 				uint32_t base, uint32_t num);
 static void i40e_res_pool_destroy(struct i40e_res_pool_info *pool);
@@ -389,7 +391,7 @@ static int i40e_sw_ethertype_filter_insert(struct i40e_pf *pf,
 				   struct i40e_ethertype_filter *filter);
 
 static int i40e_tunnel_filter_convert(
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext *cld_filter,
+	struct i40e_aqc_cloud_filters_element_bb *cld_filter,
 	struct i40e_tunnel_filter *tunnel_filter);
 static int i40e_sw_tunnel_filter_insert(struct i40e_pf *pf,
 				struct i40e_tunnel_filter *tunnel_filter);
@@ -408,6 +410,7 @@ static const char *const valid_keys[] = {
 	ETH_I40E_FLOATING_VEB_LIST_ARG,
 	ETH_I40E_SUPPORT_MULTI_DRIVER,
 	ETH_I40E_QUEUE_NUM_PER_VF_ARG,
+	ETH_I40E_USE_LATEST_VEC,
 	NULL};
 
 static const struct rte_pci_id pci_id_i40e_map[] = {
@@ -1202,6 +1205,66 @@ i40e_aq_debug_write_global_register(struct i40e_hw *hw,
 }
 
 static int
+i40e_parse_latest_vec_handler(__rte_unused const char *key,
+				const char *value,
+				void *opaque)
+{
+	struct i40e_adapter *ad;
+	int use_latest_vec;
+
+	ad = (struct i40e_adapter *)opaque;
+
+	use_latest_vec = atoi(value);
+
+	if (use_latest_vec != 0 && use_latest_vec != 1)
+		PMD_DRV_LOG(WARNING, "Value should be 0 or 1, set it as 1!");
+
+	ad->use_latest_vec = (uint8_t)use_latest_vec;
+
+	return 0;
+}
+
+static int
+i40e_use_latest_vec(struct rte_eth_dev *dev)
+{
+	struct i40e_adapter *ad =
+		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct rte_kvargs *kvlist;
+	int kvargs_count;
+
+	ad->use_latest_vec = false;
+
+	if (!dev->device->devargs)
+		return 0;
+
+	kvlist = rte_kvargs_parse(dev->device->devargs->args, valid_keys);
+	if (!kvlist)
+		return -EINVAL;
+
+	kvargs_count = rte_kvargs_count(kvlist, ETH_I40E_USE_LATEST_VEC);
+	if (!kvargs_count) {
+		rte_kvargs_free(kvlist);
+		return 0;
+	}
+
+	if (kvargs_count > 1)
+		PMD_DRV_LOG(WARNING, "More than one argument \"%s\" and only "
+			    "the first invalid or last valid one is used !",
+			    ETH_I40E_USE_LATEST_VEC);
+
+	if (rte_kvargs_process(kvlist, ETH_I40E_USE_LATEST_VEC,
+				i40e_parse_latest_vec_handler, ad) < 0) {
+		rte_kvargs_free(kvlist);
+		return -EINVAL;
+	}
+
+	rte_kvargs_free(kvlist);
+	return 0;
+}
+
+#define I40E_ALARM_INTERVAL 50000 /* us */
+
+static int
 eth_i40e_dev_init(struct rte_eth_dev *dev, void *init_params __rte_unused)
 {
 	struct rte_pci_device *pci_dev;
@@ -1263,13 +1326,12 @@ eth_i40e_dev_init(struct rte_eth_dev *dev, void *init_params __rte_unused)
 
 	/* Check if need to support multi-driver */
 	i40e_support_multi_driver(dev);
+	/* Check if users want the latest supported vec path */
+	i40e_use_latest_vec(dev);
 
 	/* Make sure all is clean before doing PF reset */
 	i40e_clear_hw(hw);
 
-	/* Initialize the hardware */
-	i40e_hw_init(dev);
-
 	/* Reset here to make sure all is clean for each PF */
 	ret = i40e_pf_reset(hw);
 	if (ret) {
@@ -1284,6 +1346,23 @@ eth_i40e_dev_init(struct rte_eth_dev *dev, void *init_params __rte_unused)
 		return ret;
 	}
 
+	/* Initialize the parameters for adminq */
+	i40e_init_adminq_parameter(hw);
+	ret = i40e_init_adminq(hw);
+	if (ret != I40E_SUCCESS) {
+		PMD_INIT_LOG(ERR, "Failed to init adminq: %d", ret);
+		return -EIO;
+	}
+	PMD_INIT_LOG(INFO, "FW %d.%d API %d.%d NVM %02d.%02d.%02d eetrack %04x",
+		     hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
+		     hw->aq.api_maj_ver, hw->aq.api_min_ver,
+		     ((hw->nvm.version >> 12) & 0xf),
+		     ((hw->nvm.version >> 4) & 0xff),
+		     (hw->nvm.version & 0xf), hw->nvm.eetrack);
+
+	/* Initialize the hardware */
+	i40e_hw_init(dev);
+
 	i40e_config_automask(pf);
 
 	i40e_set_default_pctype_table(dev);
@@ -1299,20 +1378,6 @@ eth_i40e_dev_init(struct rte_eth_dev *dev, void *init_params __rte_unused)
 	/* Initialize the input set for filters (hash and fd) to default value */
 	i40e_filter_input_set_init(pf);
 
-	/* Initialize the parameters for adminq */
-	i40e_init_adminq_parameter(hw);
-	ret = i40e_init_adminq(hw);
-	if (ret != I40E_SUCCESS) {
-		PMD_INIT_LOG(ERR, "Failed to init adminq: %d", ret);
-		return -EIO;
-	}
-	PMD_INIT_LOG(INFO, "FW %d.%d API %d.%d NVM %02d.%02d.%02d eetrack %04x",
-		     hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
-		     hw->aq.api_maj_ver, hw->aq.api_min_ver,
-		     ((hw->nvm.version >> 12) & 0xf),
-		     ((hw->nvm.version >> 4) & 0xff),
-		     (hw->nvm.version & 0xf), hw->nvm.eetrack);
-
 	/* initialise the L3_MAP register */
 	if (!pf->support_multi_driver) {
 		ret = i40e_aq_debug_write_global_register(hw,
@@ -1663,9 +1728,6 @@ eth_i40e_dev_uninit(struct rte_eth_dev *dev)
 	/* uninitialize pf host driver */
 	i40e_pf_host_uninit(dev);
 
-	rte_free(dev->data->mac_addrs);
-	dev->data->mac_addrs = NULL;
-
 	/* disable uio intr before callback unregister */
 	rte_intr_disable(intr_handle);
 
@@ -1722,6 +1784,10 @@ i40e_dev_configure(struct rte_eth_dev *dev)
 	ad->tx_simple_allowed = true;
 	ad->tx_vec_allowed = true;
 
+	/* Only legacy filter API needs the following fdir config. So when the
+	 * legacy filter API is deprecated, the following codes should also be
+	 * removed.
+	 */
 	if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT) {
 		ret = i40e_fdir_setup(pf);
 		if (ret != I40E_SUCCESS) {
@@ -1779,7 +1845,11 @@ err_dcb:
 	rte_free(pf->vmdq);
 	pf->vmdq = NULL;
 err:
-	/* need to release fdir resource if exists */
+	/* Need to release fdir resource if exists.
+	 * Only legacy filter API needs the following fdir config. So when the
+	 * legacy filter API is deprecated, the following code should also be
+	 * removed.
+	 */
 	i40e_fdir_teardown(pf);
 	return ret;
 }
@@ -2293,8 +2363,13 @@ i40e_dev_start(struct rte_eth_dev *dev)
 		i40e_dev_link_update(dev, 0);
 	}
 
-	/* enable uio intr after callback register */
-	rte_intr_enable(intr_handle);
+	if (dev->data->dev_conf.intr_conf.rxq == 0) {
+		rte_eal_alarm_set(I40E_ALARM_INTERVAL,
+				  i40e_dev_alarm_handler, dev);
+	} else {
+		/* enable uio intr after callback register */
+		rte_intr_enable(intr_handle);
+	}
 
 	i40e_filter_restore(pf);
 
@@ -2324,6 +2399,12 @@ i40e_dev_stop(struct rte_eth_dev *dev)
 
 	if (hw->adapter_stopped == 1)
 		return;
+
+	if (dev->data->dev_conf.intr_conf.rxq == 0) {
+		rte_eal_alarm_cancel(i40e_dev_alarm_handler, dev);
+		rte_intr_enable(intr_handle);
+	}
+
 	/* Disable all queues */
 	i40e_dev_switch_queues(pf, FALSE);
 
@@ -2406,6 +2487,11 @@ i40e_dev_close(struct rte_eth_dev *dev)
 	i40e_pf_disable_irq0(hw);
 	rte_intr_disable(intr_handle);
 
+	/*
+	 * Only legacy filter API needs the following fdir config. So when the
+	 * legacy filter API is deprecated, the following code should also be
+	 * removed.
+	 */
 	i40e_fdir_teardown(pf);
 
 	/* shutdown and destroy the HMC */
@@ -2498,6 +2584,10 @@ i40e_dev_promiscuous_disable(struct rte_eth_dev *dev)
 	if (status != I40E_SUCCESS)
 		PMD_DRV_LOG(ERR, "Failed to disable unicast promiscuous");
 
+	/* must remain in all_multicast mode */
+	if (dev->data->all_multicast == 1)
+		return;
+
 	status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
 							false, NULL);
 	if (status != I40E_SUCCESS)
@@ -3363,8 +3453,8 @@ i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 		DEV_RX_OFFLOAD_UDP_CKSUM |
 		DEV_RX_OFFLOAD_TCP_CKSUM |
 		DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
-		DEV_RX_OFFLOAD_CRC_STRIP |
 		DEV_RX_OFFLOAD_KEEP_CRC |
+		DEV_RX_OFFLOAD_SCATTER |
 		DEV_RX_OFFLOAD_VLAN_EXTEND |
 		DEV_RX_OFFLOAD_VLAN_FILTER |
 		DEV_RX_OFFLOAD_JUMBO_FRAME;
@@ -3577,7 +3667,7 @@ i40e_vlan_tpid_set(struct rte_eth_dev *dev,
 			if (vlan_type == ETH_VLAN_TYPE_OUTER)
 				hw->second_tag = rte_cpu_to_le_16(tpid);
 		}
-		ret = i40e_aq_set_switch_config(hw, 0, 0, NULL);
+		ret = i40e_aq_set_switch_config(hw, 0, 0, 0, NULL);
 		if (ret != I40E_SUCCESS) {
 			PMD_DRV_LOG(ERR,
 				    "Set switch config failed aq_err: %d",
@@ -5282,7 +5372,7 @@ i40e_enable_pf_lb(struct i40e_pf *pf)
 	int ret;
 
 	/* Use the FW API if FW >= v5.0 */
-	if (hw->aq.fw_maj_ver < 5) {
+	if (hw->aq.fw_maj_ver < 5 && hw->mac.type != I40E_MAC_X722) {
 		PMD_INIT_LOG(ERR, "FW < v5.0, cannot enable loopback");
 		return;
 	}
@@ -5553,7 +5643,7 @@ i40e_vsi_setup(struct i40e_pf *pf,
 		ctxt.flags = I40E_AQ_VSI_TYPE_VF;
 
 		/* Use the VEB configuration if FW >= v5.0 */
-		if (hw->aq.fw_maj_ver >= 5) {
+		if (hw->aq.fw_maj_ver >= 5 || hw->mac.type == I40E_MAC_X722) {
 			/* Configure switch ID */
 			ctxt.info.valid_sections |=
 			rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
@@ -6549,7 +6639,53 @@ i40e_dev_interrupt_handler(void *param)
 done:
 	/* Enable interrupt */
 	i40e_pf_enable_irq0(hw);
-	rte_intr_enable(dev->intr_handle);
+}
+
+static void
+i40e_dev_alarm_handler(void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t icr0;
+
+	/* Disable interrupt */
+	i40e_pf_disable_irq0(hw);
+
+	/* read out interrupt causes */
+	icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
+
+	/* No interrupt event indicated */
+	if (!(icr0 & I40E_PFINT_ICR0_INTEVENT_MASK))
+		goto done;
+	if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
+		PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error");
+	if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
+		PMD_DRV_LOG(ERR, "ICR0: malicious programming detected");
+	if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
+		PMD_DRV_LOG(INFO, "ICR0: global reset requested");
+	if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
+		PMD_DRV_LOG(INFO, "ICR0: PCI exception activated");
+	if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
+		PMD_DRV_LOG(INFO, "ICR0: a change in the storm control state");
+	if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
+		PMD_DRV_LOG(ERR, "ICR0: HMC error");
+	if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
+		PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error");
+
+	if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
+		PMD_DRV_LOG(INFO, "ICR0: VF reset detected");
+		i40e_dev_handle_vfr_event(dev);
+	}
+	if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
+		PMD_DRV_LOG(INFO, "ICR0: adminq event");
+		i40e_dev_handle_aq_msg(dev);
+	}
+
+done:
+	/* Enable interrupt */
+	i40e_pf_enable_irq0(hw);
+	rte_eal_alarm_set(I40E_ALARM_INTERVAL,
+			  i40e_dev_alarm_handler, dev);
 }
 
 int
@@ -7370,7 +7506,7 @@ i40e_dev_get_filter_type(uint16_t filter_type, uint16_t *flag)
 /* Convert tunnel filter structure */
 static int
 i40e_tunnel_filter_convert(
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext *cld_filter,
+	struct i40e_aqc_cloud_filters_element_bb *cld_filter,
 	struct i40e_tunnel_filter *tunnel_filter)
 {
 	ether_addr_copy((struct ether_addr *)&cld_filter->element.outer_mac,
@@ -7468,8 +7604,8 @@ i40e_dev_tunnel_filter_set(struct i40e_pf *pf,
 	int val, ret = 0;
 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 	struct i40e_vsi *vsi = pf->main_vsi;
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext *cld_filter;
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext *pfilter;
+	struct i40e_aqc_cloud_filters_element_bb *cld_filter;
+	struct i40e_aqc_cloud_filters_element_bb *pfilter;
 	struct i40e_tunnel_rule *tunnel_rule = &pf->tunnel;
 	struct i40e_tunnel_filter *tunnel, *node;
 	struct i40e_tunnel_filter check_filter; /* Check if filter exists */
@@ -7577,7 +7713,7 @@ i40e_dev_tunnel_filter_set(struct i40e_pf *pf,
 		if (ret < 0)
 			rte_free(tunnel);
 	} else {
-		ret = i40e_aq_remove_cloud_filters(hw, vsi->seid,
+		ret = i40e_aq_rem_cloud_filters(hw, vsi->seid,
 						   &cld_filter->element, 1);
 		if (ret < 0) {
 			PMD_DRV_LOG(ERR, "Failed to delete a tunnel filter.");
@@ -7910,8 +8046,8 @@ i40e_dev_consistent_tunnel_filter_set(struct i40e_pf *pf,
 	struct i40e_pf_vf *vf = NULL;
 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 	struct i40e_vsi *vsi;
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext *cld_filter;
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext *pfilter;
+	struct i40e_aqc_cloud_filters_element_bb *cld_filter;
+	struct i40e_aqc_cloud_filters_element_bb *pfilter;
 	struct i40e_tunnel_rule *tunnel_rule = &pf->tunnel;
 	struct i40e_tunnel_filter *tunnel, *node;
 	struct i40e_tunnel_filter check_filter; /* Check if filter exists */
@@ -8114,7 +8250,7 @@ i40e_dev_consistent_tunnel_filter_set(struct i40e_pf *pf,
 
 	if (add) {
 		if (big_buffer)
-			ret = i40e_aq_add_cloud_filters_big_buffer(hw,
+			ret = i40e_aq_add_cloud_filters_bb(hw,
 						   vsi->seid, cld_filter, 1);
 		else
 			ret = i40e_aq_add_cloud_filters(hw,
@@ -8137,11 +8273,11 @@ i40e_dev_consistent_tunnel_filter_set(struct i40e_pf *pf,
 			rte_free(tunnel);
 	} else {
 		if (big_buffer)
-			ret = i40e_aq_remove_cloud_filters_big_buffer(
+			ret = i40e_aq_rem_cloud_filters_bb(
 				hw, vsi->seid, cld_filter, 1);
 		else
-			ret = i40e_aq_remove_cloud_filters(hw, vsi->seid,
-						   &cld_filter->element, 1);
+			ret = i40e_aq_rem_cloud_filters(hw, vsi->seid,
+						&cld_filter->element, 1);
 		if (ret < 0) {
 			PMD_DRV_LOG(ERR, "Failed to delete a tunnel filter.");
 			rte_free(cld_filter);
@@ -11249,6 +11385,16 @@ i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb)
 	 * LLDP MIB change event.
 	 */
 	if (sw_dcb == TRUE) {
+		/* When using NVM 6.01 or later, the RX data path does
+		 * not hang if the FW LLDP is stopped.
+		 */
+		if (((hw->nvm.version >> 12) & 0xf) >= 6 &&
+		    ((hw->nvm.version >> 4) & 0xff) >= 1) {
+			ret = i40e_aq_stop_lldp(hw, TRUE, NULL);
+			if (ret != I40E_SUCCESS)
+				PMD_INIT_LOG(DEBUG, "Failed to stop lldp");
+		}
+
 		ret = i40e_init_dcb(hw);
 		/* If lldp agent is stopped, the return value from
 		 * i40e_init_dcb we expect is failure with I40E_AQ_RC_EPERM
@@ -11584,7 +11730,7 @@ static int i40e_get_module_info(struct rte_eth_dev *dev,
 	case I40E_MODULE_TYPE_SFP:
 		status = i40e_aq_get_phy_register(hw,
 				I40E_AQ_PHY_REG_ACCESS_EXTERNAL_MODULE,
-				I40E_I2C_EEPROM_DEV_ADDR,
+				I40E_I2C_EEPROM_DEV_ADDR, 1,
 				I40E_MODULE_SFF_8472_COMP,
 				&sff8472_comp, NULL);
 		if (status)
@@ -11592,7 +11738,7 @@ static int i40e_get_module_info(struct rte_eth_dev *dev,
 
 		status = i40e_aq_get_phy_register(hw,
 				I40E_AQ_PHY_REG_ACCESS_EXTERNAL_MODULE,
-				I40E_I2C_EEPROM_DEV_ADDR,
+				I40E_I2C_EEPROM_DEV_ADDR, 1,
 				I40E_MODULE_SFF_8472_SWAP,
 				&sff8472_swap, NULL);
 		if (status)
@@ -11620,7 +11766,7 @@ static int i40e_get_module_info(struct rte_eth_dev *dev,
 		/* Read from memory page 0. */
 		status = i40e_aq_get_phy_register(hw,
 				I40E_AQ_PHY_REG_ACCESS_EXTERNAL_MODULE,
-				0,
+				0, 1,
 				I40E_MODULE_REVISION_ADDR,
 				&sff8636_rev, NULL);
 		if (status)
@@ -11681,7 +11827,7 @@ static int i40e_get_module_eeprom(struct rte_eth_dev *dev,
 		}
 		status = i40e_aq_get_phy_register(hw,
 				I40E_AQ_PHY_REG_ACCESS_EXTERNAL_MODULE,
-				addr, offset, &value, NULL);
+				addr, offset, 1, &value, NULL);
 		if (status)
 			return -EIO;
 		data[i] = (uint8_t)value;
@@ -11812,7 +11958,7 @@ i40e_tunnel_filter_restore(struct i40e_pf *pf)
 	struct i40e_tunnel_filter_list
 		*tunnel_list = &pf->tunnel.tunnel_list;
 	struct i40e_tunnel_filter *f;
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext cld_filter;
+	struct i40e_aqc_cloud_filters_element_bb cld_filter;
 	bool big_buffer = 0;
 
 	TAILQ_FOREACH(f, tunnel_list, rules) {
@@ -11847,8 +11993,8 @@ i40e_tunnel_filter_restore(struct i40e_pf *pf)
 			big_buffer = 1;
 
 		if (big_buffer)
-			i40e_aq_add_cloud_filters_big_buffer(hw,
-					     vsi->seid, &cld_filter, 1);
+			i40e_aq_add_cloud_filters_bb(hw,
+					vsi->seid, &cld_filter, 1);
 		else
 			i40e_aq_add_cloud_filters(hw, vsi->seid,
 						  &cld_filter.element, 1);
@@ -12527,4 +12673,5 @@ RTE_PMD_REGISTER_PARAM_STRING(net_i40e,
 			      ETH_I40E_FLOATING_VEB_ARG "=1"
 			      ETH_I40E_FLOATING_VEB_LIST_ARG "=<string>"
 			      ETH_I40E_QUEUE_NUM_PER_VF_ARG "=1|2|4|8|16"
-			      ETH_I40E_SUPPORT_MULTI_DRIVER "=1");
+			      ETH_I40E_SUPPORT_MULTI_DRIVER "=1"
+			      ETH_I40E_USE_LATEST_VEC "=0|1");
diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h
index 3fffe5a5..11ecfc30 100644
--- a/drivers/net/i40e/i40e_ethdev.h
+++ b/drivers/net/i40e/i40e_ethdev.h
@@ -1078,6 +1078,9 @@ struct i40e_adapter {
 	uint64_t pctypes_tbl[I40E_FLOW_TYPE_MAX] __rte_cache_min_aligned;
 	uint64_t flow_types_mask;
 	uint64_t pctypes_mask;
+
+	/* For devargs */
+	uint8_t use_latest_vec;
 };
 
 /**
@@ -1393,6 +1396,8 @@ i40e_calc_itr_interval(bool is_pf, bool is_multi_drv)
 	(((phy_type) & I40E_CAP_PHY_TYPE_25GBASE_KR) || \
 	((phy_type) & I40E_CAP_PHY_TYPE_25GBASE_CR) || \
 	((phy_type) & I40E_CAP_PHY_TYPE_25GBASE_SR) || \
-	((phy_type) & I40E_CAP_PHY_TYPE_25GBASE_LR))
+	((phy_type) & I40E_CAP_PHY_TYPE_25GBASE_LR) || \
+	((phy_type) & I40E_CAP_PHY_TYPE_25GBASE_AOC) || \
+	((phy_type) & I40E_CAP_PHY_TYPE_25GBASE_ACC))
 
 #endif /* _I40E_ETHDEV_H_ */
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 001c301b..ae55b9b1 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -1384,10 +1384,8 @@ i40evf_dev_alarm_handler(void *param)
 	icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
 
 	/* No interrupt event indicated */
-	if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
-		PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do");
+	if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK))
 		goto done;
-	}
 
 	if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
 		PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported");
@@ -1485,9 +1483,6 @@ i40evf_dev_uninit(struct rte_eth_dev *eth_dev)
 		return -1;
 	}
 
-	rte_free(eth_dev->data->mac_addrs);
-	eth_dev->data->mac_addrs = NULL;
-
 	return 0;
 }
 
@@ -1522,8 +1517,6 @@ i40evf_dev_configure(struct rte_eth_dev *dev)
 {
 	struct i40e_adapter *ad =
 		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
-	struct rte_eth_conf *conf = &dev->data->dev_conf;
-	struct i40e_vf *vf;
 
 	/* Initialize to TRUE. If any of Rx queues doesn't meet the bulk
 	 * allocation or vector Rx preconditions we will reset it.
@@ -1533,19 +1526,6 @@ i40evf_dev_configure(struct rte_eth_dev *dev)
 	ad->tx_simple_allowed = true;
 	ad->tx_vec_allowed = true;
 
-	/* For non-DPDK PF drivers, VF has no ability to disable HW
-	 * CRC strip, and is implicitly enabled by the PF.
-	 */
-	if (rte_eth_dev_must_keep_crc(conf->rxmode.offloads)) {
-		vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
-		if ((vf->version_major == VIRTCHNL_VERSION_MAJOR) &&
-		    (vf->version_minor <= VIRTCHNL_VERSION_MINOR)) {
-			/* Peer is running non-DPDK PF driver. */
-			PMD_INIT_LOG(ERR, "VF can't disable HW CRC Strip");
-			return -EINVAL;
-		}
-	}
-
 	return i40evf_init_vlan(dev);
 }
 
@@ -2180,8 +2160,6 @@ i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 		DEV_RX_OFFLOAD_UDP_CKSUM |
 		DEV_RX_OFFLOAD_TCP_CKSUM |
 		DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
-		DEV_RX_OFFLOAD_CRC_STRIP |
-		DEV_RX_OFFLOAD_KEEP_CRC |
 		DEV_RX_OFFLOAD_SCATTER |
 		DEV_RX_OFFLOAD_JUMBO_FRAME |
 		DEV_RX_OFFLOAD_VLAN_FILTER;
@@ -2268,7 +2246,6 @@ i40evf_dev_close(struct rte_eth_dev *dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
-	rte_eal_alarm_cancel(i40evf_dev_alarm_handler, dev);
 	i40evf_dev_stop(dev);
 	i40e_dev_free_queues(dev);
 	/*
@@ -2282,6 +2259,7 @@ i40evf_dev_close(struct rte_eth_dev *dev)
 	i40evf_reset_vf(hw);
 	i40e_shutdown_adminq(hw);
 	i40evf_disable_irq0(hw);
+	rte_eal_alarm_cancel(i40evf_dev_alarm_handler, dev);
 }
 
 /*
diff --git a/drivers/net/i40e/i40e_flow.c b/drivers/net/i40e/i40e_flow.c
index c67b264d..3694df25 100644
--- a/drivers/net/i40e/i40e_flow.c
+++ b/drivers/net/i40e/i40e_flow.c
@@ -133,8 +133,8 @@ const struct rte_flow_ops i40e_flow_ops = {
 	.flush = i40e_flow_flush,
 };
 
-union i40e_filter_t cons_filter;
-enum rte_filter_type cons_filter_type = RTE_ETH_FILTER_NONE;
+static union i40e_filter_t cons_filter;
+static enum rte_filter_type cons_filter_type = RTE_ETH_FILTER_NONE;
 
 /* Pattern matched ethertype filter */
 static enum rte_flow_item_type pattern_ethertype[] = {
@@ -3127,6 +3127,7 @@ i40e_flow_parse_fdir_filter(struct rte_eth_dev *dev,
 			    struct rte_flow_error *error,
 			    union i40e_filter_t *filter)
 {
+	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct i40e_fdir_filter_conf *fdir_filter =
 		&filter->fdir_filter;
 	int ret;
@@ -3148,14 +3149,29 @@ i40e_flow_parse_fdir_filter(struct rte_eth_dev *dev,
 
 	if (dev->data->dev_conf.fdir_conf.mode !=
 	    RTE_FDIR_MODE_PERFECT) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "Check the mode in fdir_conf.");
-		return -rte_errno;
+		/* Enable fdir when fdir flow is added at first time. */
+		ret = i40e_fdir_setup(pf);
+		if (ret != I40E_SUCCESS) {
+			rte_flow_error_set(error, ENOTSUP,
+					   RTE_FLOW_ERROR_TYPE_HANDLE,
+					   NULL, "Failed to setup fdir.");
+			return -rte_errno;
+		}
+		ret = i40e_fdir_configure(dev);
+		if (ret < 0) {
+			rte_flow_error_set(error, ENOTSUP,
+					   RTE_FLOW_ERROR_TYPE_HANDLE,
+					   NULL, "Failed to configure fdir.");
+			goto err;
+		}
+
+		dev->data->dev_conf.fdir_conf.mode = RTE_FDIR_MODE_PERFECT;
 	}
 
 	return 0;
+err:
+	i40e_fdir_teardown(pf);
+	return -rte_errno;
 }
 
 /* Parse to get the action info of a tunnel filter
@@ -4708,6 +4724,13 @@ i40e_flow_destroy(struct rte_eth_dev *dev,
 	case RTE_ETH_FILTER_FDIR:
 		ret = i40e_flow_add_del_fdir_filter(dev,
 		       &((struct i40e_fdir_filter *)flow->rule)->fdir, 0);
+
+		/* If the last flow is destroyed, disable fdir. */
+		if (!ret && !TAILQ_EMPTY(&pf->fdir.fdir_list)) {
+			i40e_fdir_teardown(pf);
+			dev->data->dev_conf.fdir_conf.mode =
+				   RTE_FDIR_MODE_NONE;
+		}
 		break;
 	case RTE_ETH_FILTER_HASH:
 		ret = i40e_config_rss_filter_del(dev,
@@ -4773,7 +4796,7 @@ i40e_flow_destroy_tunnel_filter(struct i40e_pf *pf,
 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 	struct i40e_vsi *vsi;
 	struct i40e_pf_vf *vf;
-	struct i40e_aqc_add_rm_cloud_filt_elem_ext cld_filter;
+	struct i40e_aqc_cloud_filters_element_bb cld_filter;
 	struct i40e_tunnel_rule *tunnel_rule = &pf->tunnel;
 	struct i40e_tunnel_filter *node;
 	bool big_buffer = 0;
@@ -4808,11 +4831,11 @@ i40e_flow_destroy_tunnel_filter(struct i40e_pf *pf,
 		big_buffer = 1;
 
 	if (big_buffer)
-		ret = i40e_aq_remove_cloud_filters_big_buffer(hw, vsi->seid,
-							      &cld_filter, 1);
+		ret = i40e_aq_rem_cloud_filters_bb(hw, vsi->seid,
+						&cld_filter, 1);
 	else
-		ret = i40e_aq_remove_cloud_filters(hw, vsi->seid,
-						   &cld_filter.element, 1);
+		ret = i40e_aq_rem_cloud_filters(hw, vsi->seid,
+						&cld_filter.element, 1);
 	if (ret < 0)
 		return -ENOTSUP;
 
@@ -4900,6 +4923,8 @@ i40e_flow_flush_fdir_filter(struct i40e_pf *pf)
 			pf->fdir.inset_flag[pctype] = 0;
 	}
 
+	i40e_fdir_teardown(pf);
+
 	return ret;
 }
 
diff --git a/drivers/net/i40e/i40e_rxtx.c b/drivers/net/i40e/i40e_rxtx.c
index 2a28ee34..8bfa2517 100644
--- a/drivers/net/i40e/i40e_rxtx.c
+++ b/drivers/net/i40e/i40e_rxtx.c
@@ -55,6 +55,10 @@
 		PKT_TX_OUTER_IP_CKSUM)
 
 #define I40E_TX_OFFLOAD_MASK (  \
+		PKT_TX_OUTER_IPV4 |	\
+		PKT_TX_OUTER_IPV6 |	\
+		PKT_TX_IPV4 |		\
+		PKT_TX_IPV6 |		\
 		PKT_TX_IP_CKSUM |       \
 		PKT_TX_L4_MASK |        \
 		PKT_TX_OUTER_IP_CKSUM | \
@@ -83,7 +87,8 @@ i40e_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union i40e_rx_desc *rxdp)
 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
 	if (rte_le_to_cpu_16(rxdp->wb.qword2.ext_status) &
 		(1 << I40E_RX_DESC_EXT_STATUS_L2TAG2P_SHIFT)) {
-		mb->ol_flags |= PKT_RX_QINQ_STRIPPED;
+		mb->ol_flags |= PKT_RX_QINQ_STRIPPED | PKT_RX_QINQ |
+			PKT_RX_VLAN_STRIPPED | PKT_RX_VLAN;
 		mb->vlan_tci_outer = mb->vlan_tci;
 		mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_2);
 		PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u",
@@ -1828,7 +1833,7 @@ i40e_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->queue_id = queue_idx;
 	rxq->reg_idx = reg_idx;
 	rxq->port_id = dev->data->port_id;
-	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
 		rxq->crc_len = ETHER_CRC_LEN;
 	else
 		rxq->crc_len = 0;
@@ -2909,6 +2914,35 @@ i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 	qinfo->conf.offloads = txq->offloads;
 }
 
+static eth_rx_burst_t
+i40e_get_latest_rx_vec(bool scatter)
+{
+#ifdef RTE_ARCH_X86
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
+		return scatter ? i40e_recv_scattered_pkts_vec_avx2 :
+				 i40e_recv_pkts_vec_avx2;
+#endif
+	return scatter ? i40e_recv_scattered_pkts_vec :
+			 i40e_recv_pkts_vec;
+}
+
+static eth_rx_burst_t
+i40e_get_recommend_rx_vec(bool scatter)
+{
+#ifdef RTE_ARCH_X86
+	/*
+	 * since AVX frequency can be different to base frequency, limit
+	 * use of AVX2 version to later plaforms, not all those that could
+	 * theoretically run it.
+	 */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
+		return scatter ? i40e_recv_scattered_pkts_vec_avx2 :
+				 i40e_recv_pkts_vec_avx2;
+#endif
+	return scatter ? i40e_recv_scattered_pkts_vec :
+			 i40e_recv_pkts_vec;
+}
+
 void __attribute__((cold))
 i40e_set_rx_function(struct rte_eth_dev *dev)
 {
@@ -2940,57 +2974,17 @@ i40e_set_rx_function(struct rte_eth_dev *dev)
 		}
 	}
 
-	if (dev->data->scattered_rx) {
-		/* Set the non-LRO scattered callback: there are Vector and
-		 * single allocation versions.
-		 */
-		if (ad->rx_vec_allowed) {
-			PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx "
-					    "callback (port=%d).",
-				     dev->data->port_id);
-
-			dev->rx_pkt_burst = i40e_recv_scattered_pkts_vec;
-#ifdef RTE_ARCH_X86
-			/*
-			 * since AVX frequency can be different to base
-			 * frequency, limit use of AVX2 version to later
-			 * plaforms, not all those that could theoretically
-			 * run it.
-			 */
-			if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
-				dev->rx_pkt_burst =
-					i40e_recv_scattered_pkts_vec_avx2;
-#endif
-		} else {
-			PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
-					   "allocation callback (port=%d).",
-				     dev->data->port_id);
-			dev->rx_pkt_burst = i40e_recv_scattered_pkts;
-		}
-	/* If parameters allow we are going to choose between the following
-	 * callbacks:
-	 *    - Vector
-	 *    - Bulk Allocation
-	 *    - Single buffer allocation (the simplest one)
-	 */
-	} else if (ad->rx_vec_allowed) {
-		PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure RX "
-				    "burst size no less than %d (port=%d).",
-			     RTE_I40E_DESCS_PER_LOOP,
-			     dev->data->port_id);
-
-		dev->rx_pkt_burst = i40e_recv_pkts_vec;
-#ifdef RTE_ARCH_X86
-		/*
-		 * since AVX frequency can be different to base
-		 * frequency, limit use of AVX2 version to later
-		 * plaforms, not all those that could theoretically
-		 * run it.
-		 */
-		if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
-			dev->rx_pkt_burst = i40e_recv_pkts_vec_avx2;
-#endif
-	} else if (ad->rx_bulk_alloc_allowed) {
+	if (ad->rx_vec_allowed) {
+		/* Vec Rx path */
+		PMD_INIT_LOG(DEBUG, "Vector Rx path will be used on port=%d.",
+				dev->data->port_id);
+		if (ad->use_latest_vec)
+			dev->rx_pkt_burst =
+			i40e_get_latest_rx_vec(dev->data->scattered_rx);
+		else
+			dev->rx_pkt_burst =
+			i40e_get_recommend_rx_vec(dev->data->scattered_rx);
+	} else if (!dev->data->scattered_rx && ad->rx_bulk_alloc_allowed) {
 		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
 				    "satisfied. Rx Burst Bulk Alloc function "
 				    "will be used on port=%d.",
@@ -2998,12 +2992,12 @@ i40e_set_rx_function(struct rte_eth_dev *dev)
 
 		dev->rx_pkt_burst = i40e_recv_pkts_bulk_alloc;
 	} else {
-		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
-				    "satisfied, or Scattered Rx is requested "
-				    "(port=%d).",
+		/* Simple Rx Path. */
+		PMD_INIT_LOG(DEBUG, "Simple Rx path will be used on port=%d.",
 			     dev->data->port_id);
-
-		dev->rx_pkt_burst = i40e_recv_pkts;
+		dev->rx_pkt_burst = dev->data->scattered_rx ?
+					i40e_recv_scattered_pkts :
+					i40e_recv_pkts;
 	}
 
 	/* Propagate information about RX function choice through all queues. */
@@ -3049,6 +3043,31 @@ i40e_set_tx_function_flag(struct rte_eth_dev *dev, struct i40e_tx_queue *txq)
 				txq->queue_id);
 }
 
+static eth_tx_burst_t
+i40e_get_latest_tx_vec(void)
+{
+#ifdef RTE_ARCH_X86
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
+		return i40e_xmit_pkts_vec_avx2;
+#endif
+	return i40e_xmit_pkts_vec;
+}
+
+static eth_tx_burst_t
+i40e_get_recommend_tx_vec(void)
+{
+#ifdef RTE_ARCH_X86
+	/*
+	 * since AVX frequency can be different to base frequency, limit
+	 * use of AVX2 version to later plaforms, not all those that could
+	 * theoretically run it.
+	 */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
+		return i40e_xmit_pkts_vec_avx2;
+#endif
+	return i40e_xmit_pkts_vec;
+}
+
 void __attribute__((cold))
 i40e_set_tx_function(struct rte_eth_dev *dev)
 {
@@ -3073,17 +3092,12 @@ i40e_set_tx_function(struct rte_eth_dev *dev)
 	if (ad->tx_simple_allowed) {
 		if (ad->tx_vec_allowed) {
 			PMD_INIT_LOG(DEBUG, "Vector tx finally be used.");
-			dev->tx_pkt_burst = i40e_xmit_pkts_vec;
-#ifdef RTE_ARCH_X86
-			/*
-			 * since AVX frequency can be different to base
-			 * frequency, limit use of AVX2 version to later
-			 * plaforms, not all those that could theoretically
-			 * run it.
-			 */
-			if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
-				dev->tx_pkt_burst = i40e_xmit_pkts_vec_avx2;
-#endif
+			if (ad->use_latest_vec)
+				dev->tx_pkt_burst =
+					i40e_get_latest_tx_vec();
+			else
+				dev->tx_pkt_burst =
+					i40e_get_recommend_tx_vec();
 		} else {
 			PMD_INIT_LOG(DEBUG, "Simple tx finally be used.");
 			dev->tx_pkt_burst = i40e_xmit_pkts_simple;
@@ -3166,13 +3180,13 @@ i40e_set_default_pctype_table(struct rte_eth_dev *dev)
 }
 
 /* Stubs needed for linkage when CONFIG_RTE_I40E_INC_VECTOR is set to 'n' */
-int __attribute__((weak))
+__rte_weak int
 i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
 {
 	return -1;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 i40e_recv_pkts_vec(
 	void __rte_unused *rx_queue,
 	struct rte_mbuf __rte_unused **rx_pkts,
@@ -3181,7 +3195,7 @@ i40e_recv_pkts_vec(
 	return 0;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 i40e_recv_scattered_pkts_vec(
 	void __rte_unused *rx_queue,
 	struct rte_mbuf __rte_unused **rx_pkts,
@@ -3190,7 +3204,7 @@ i40e_recv_scattered_pkts_vec(
 	return 0;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 i40e_recv_pkts_vec_avx2(void __rte_unused *rx_queue,
 			struct rte_mbuf __rte_unused **rx_pkts,
 			uint16_t __rte_unused nb_pkts)
@@ -3198,7 +3212,7 @@ i40e_recv_pkts_vec_avx2(void __rte_unused *rx_queue,
 	return 0;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 i40e_recv_scattered_pkts_vec_avx2(void __rte_unused *rx_queue,
 			struct rte_mbuf __rte_unused **rx_pkts,
 			uint16_t __rte_unused nb_pkts)
@@ -3206,25 +3220,25 @@ i40e_recv_scattered_pkts_vec_avx2(void __rte_unused *rx_queue,
 	return 0;
 }
 
-int __attribute__((weak))
+__rte_weak int
 i40e_rxq_vec_setup(struct i40e_rx_queue __rte_unused *rxq)
 {
 	return -1;
 }
 
-int __attribute__((weak))
+__rte_weak int
 i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq)
 {
 	return -1;
 }
 
-void __attribute__((weak))
+__rte_weak void
 i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue __rte_unused*rxq)
 {
 	return;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 i40e_xmit_fixed_burst_vec(void __rte_unused * tx_queue,
 			  struct rte_mbuf __rte_unused **tx_pkts,
 			  uint16_t __rte_unused nb_pkts)
@@ -3232,7 +3246,7 @@ i40e_xmit_fixed_burst_vec(void __rte_unused * tx_queue,
 	return 0;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 i40e_xmit_pkts_vec_avx2(void __rte_unused * tx_queue,
 			  struct rte_mbuf __rte_unused **tx_pkts,
 			  uint16_t __rte_unused nb_pkts)
diff --git a/drivers/net/i40e/i40e_rxtx_vec_common.h b/drivers/net/i40e/i40e_rxtx_vec_common.h
index 63cb1774..f00f6d64 100644
--- a/drivers/net/i40e/i40e_rxtx_vec_common.h
+++ b/drivers/net/i40e/i40e_rxtx_vec_common.h
@@ -199,9 +199,7 @@ i40e_rx_vec_dev_conf_condition_check_default(struct rte_eth_dev *dev)
 	if (fconf->mode != RTE_FDIR_MODE_NONE)
 		return -1;
 
-	 /* - no csum error report support
-	 * - no header split support
-	 */
+	 /* no header split support */
 	if (rxmode->offloads & DEV_RX_OFFLOAD_HEADER_SPLIT)
 		return -1;
 
diff --git a/drivers/net/i40e/i40e_vf_representor.c b/drivers/net/i40e/i40e_vf_representor.c
index f9f13161..45a15d3a 100644
--- a/drivers/net/i40e/i40e_vf_representor.c
+++ b/drivers/net/i40e/i40e_vf_representor.c
@@ -48,6 +48,7 @@ i40e_vf_representor_dev_infos_get(struct rte_eth_dev *ethdev,
 		DEV_RX_OFFLOAD_UDP_CKSUM |
 		DEV_RX_OFFLOAD_TCP_CKSUM;
 	dev_info->tx_offload_capa =
+		DEV_TX_OFFLOAD_MULTI_SEGS  |
 		DEV_TX_OFFLOAD_VLAN_INSERT |
 		DEV_TX_OFFLOAD_QINQ_INSERT |
 		DEV_TX_OFFLOAD_IPV4_CKSUM |
@@ -419,7 +420,7 @@ i40e_vf_representor_vlan_pvid_set(struct rte_eth_dev *ethdev, uint16_t vlan_id,
 		representor->vf_id, vlan_id);
 }
 
-struct eth_dev_ops i40e_representor_dev_ops = {
+static const struct eth_dev_ops i40e_representor_dev_ops = {
 	.dev_infos_get        = i40e_vf_representor_dev_infos_get,
 
 	.dev_start            = i40e_vf_representor_dev_start,
@@ -486,9 +487,6 @@ i40e_vf_representor_init(struct rte_eth_dev *ethdev, void *init_params)
 	if (representor->vf_id >= pf->vf_num)
 		return -ENODEV;
 
-	/** representor shares the same driver as it's PF device */
-	ethdev->device->driver = representor->adapter->eth_dev->device->driver;
-
 	/* Set representor device ops */
 	ethdev->dev_ops = &i40e_representor_dev_ops;
 
@@ -506,6 +504,7 @@ i40e_vf_representor_init(struct rte_eth_dev *ethdev, void *init_params)
 	}
 
 	ethdev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
+	ethdev->data->representor_id = representor->vf_id;
 
 	/* Setting the number queues allocated to the VF */
 	ethdev->data->nb_rx_queues = vf->vsi->nb_qps;
@@ -525,7 +524,10 @@ i40e_vf_representor_init(struct rte_eth_dev *ethdev, void *init_params)
 }
 
 int
-i40e_vf_representor_uninit(struct rte_eth_dev *ethdev __rte_unused)
+i40e_vf_representor_uninit(struct rte_eth_dev *ethdev)
 {
+	/* mac_addrs must not be freed because part of i40e_pf_vf */
+	ethdev->data->mac_addrs = NULL;
+
 	return 0;
 }
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index bba62b1c..7ce5d02f 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -338,7 +338,7 @@ i40e_vsi_set_tx_loopback(struct i40e_vsi *vsi, uint8_t on)
 	hw = I40E_VSI_TO_HW(vsi);
 
 	/* Use the FW API if FW >= v5.0 */
-	if (hw->aq.fw_maj_ver < 5) {
+	if (hw->aq.fw_maj_ver < 5 && hw->mac.type != I40E_MAC_X722) {
 		PMD_INIT_LOG(ERR, "FW < v5.0, cannot enable loopback");
 		return -ENOTSUP;
 	}
diff --git a/drivers/net/ifc/base/ifcvf.c b/drivers/net/ifc/base/ifcvf.c
index 4b22d9ed..3c0b2dff 100644
--- a/drivers/net/ifc/base/ifcvf.c
+++ b/drivers/net/ifc/base/ifcvf.c
@@ -249,7 +249,7 @@ ifcvf_hw_disable(struct ifcvf_hw *hw)
 		IFCVF_WRITE_REG16(IFCVF_MSI_NO_VECTOR, &cfg->queue_msix_vector);
 		ring_state = *(u32 *)(hw->lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
 				(i / 2) * IFCVF_LM_CFG_SIZE + (i % 2) * 4);
-		hw->vring[i].last_avail_idx = (u16)ring_state;
+		hw->vring[i].last_avail_idx = (u16)(ring_state >> 16);
 		hw->vring[i].last_used_idx = (u16)(ring_state >> 16);
 	}
 }
@@ -279,6 +279,37 @@ ifcvf_stop_hw(struct ifcvf_hw *hw)
 }
 
 void
+ifcvf_enable_logging(struct ifcvf_hw *hw, u64 log_base, u64 log_size)
+{
+	u8 *lm_cfg;
+
+	lm_cfg = hw->lm_cfg;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_LOW) =
+		log_base & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_HIGH) =
+		(log_base >> 32) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_LOW) =
+		(log_base + log_size) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_HIGH) =
+		((log_base + log_size) >> 32) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_ENABLE_VF;
+}
+
+void
+ifcvf_disable_logging(struct ifcvf_hw *hw)
+{
+	u8 *lm_cfg;
+
+	lm_cfg = hw->lm_cfg;
+	*(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_DISABLE;
+}
+
+void
 ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
 {
 	IFCVF_WRITE_REG16(qid, hw->notify_addr[qid]);
diff --git a/drivers/net/ifc/base/ifcvf.h b/drivers/net/ifc/base/ifcvf.h
index badacb61..f026c70a 100644
--- a/drivers/net/ifc/base/ifcvf.h
+++ b/drivers/net/ifc/base/ifcvf.h
@@ -49,6 +49,7 @@
 #define IFCVF_LM_DISABLE		0x0
 #define IFCVF_LM_ENABLE_VF		0x1
 #define IFCVF_LM_ENABLE_PF		0x3
+#define IFCVF_LOG_BASE			0x100000000000
 
 #define IFCVF_32_BIT_MASK		0xffffffff
 
@@ -143,6 +144,12 @@ void
 ifcvf_stop_hw(struct ifcvf_hw *hw);
 
 void
+ifcvf_enable_logging(struct ifcvf_hw *hw, u64 log_base, u64 log_size);
+
+void
+ifcvf_disable_logging(struct ifcvf_hw *hw);
+
+void
 ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid);
 
 u8
diff --git a/drivers/net/ifc/base/ifcvf_osdep.h b/drivers/net/ifc/base/ifcvf_osdep.h
index cf151ef5..6aef25ea 100644
--- a/drivers/net/ifc/base/ifcvf_osdep.h
+++ b/drivers/net/ifc/base/ifcvf_osdep.h
@@ -17,7 +17,7 @@
 #define DEBUGOUT(S, args...)    RTE_LOG(DEBUG, PMD, S, ##args)
 #define STATIC                  static
 
-#define msec_delay	rte_delay_ms
+#define msec_delay(x)	rte_delay_us_sleep(1000 * (x))
 
 #define IFCVF_READ_REG8(reg)		rte_read8(reg)
 #define IFCVF_WRITE_REG8(val, reg)	rte_write8((val), (reg))
diff --git a/drivers/net/ifc/ifcvf_vdpa.c b/drivers/net/ifc/ifcvf_vdpa.c
index 88d81403..97a57f18 100644
--- a/drivers/net/ifc/ifcvf_vdpa.c
+++ b/drivers/net/ifc/ifcvf_vdpa.c
@@ -7,6 +7,7 @@
 #include <fcntl.h>
 #include <sys/ioctl.h>
 #include <sys/epoll.h>
+#include <linux/virtio_net.h>
 
 #include <rte_malloc.h>
 #include <rte_memory.h>
@@ -111,7 +112,6 @@ ifcvf_vfio_setup(struct ifcvf_internal *internal)
 	struct rte_pci_device *dev = internal->pdev;
 	char devname[RTE_DEV_NAME_MAX_LEN] = {0};
 	int iommu_group_num;
-	int ret = 0;
 	int i;
 
 	internal->vfio_dev_fd = -1;
@@ -145,9 +145,8 @@ ifcvf_vfio_setup(struct ifcvf_internal *internal)
 		internal->hw.mem_resource[i].len =
 			internal->pdev->mem_resource[i].len;
 	}
-	ret = ifcvf_init_hw(&internal->hw, internal->pdev);
 
-	return ret;
+	return 0;
 
 err:
 	rte_vfio_container_destroy(internal->vfio_container_fd);
@@ -205,7 +204,7 @@ exit:
 }
 
 static uint64_t
-qva_to_gpa(int vid, uint64_t qva)
+hva_to_gpa(int vid, uint64_t hva)
 {
 	struct rte_vhost_memory *mem = NULL;
 	struct rte_vhost_mem_region *reg;
@@ -218,9 +217,9 @@ qva_to_gpa(int vid, uint64_t qva)
 	for (i = 0; i < mem->nregions; i++) {
 		reg = &mem->regions[i];
 
-		if (qva >= reg->host_user_addr &&
-				qva < reg->host_user_addr + reg->size) {
-			gpa = qva - reg->host_user_addr + reg->guest_phys_addr;
+		if (hva >= reg->host_user_addr &&
+				hva < reg->host_user_addr + reg->size) {
+			gpa = hva - reg->host_user_addr + reg->guest_phys_addr;
 			break;
 		}
 	}
@@ -246,21 +245,21 @@ vdpa_ifcvf_start(struct ifcvf_internal *internal)
 
 	for (i = 0; i < nr_vring; i++) {
 		rte_vhost_get_vhost_vring(vid, i, &vq);
-		gpa = qva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc);
 		if (gpa == 0) {
 			DRV_LOG(ERR, "Fail to get GPA for descriptor ring.");
 			return -1;
 		}
 		hw->vring[i].desc = gpa;
 
-		gpa = qva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail);
 		if (gpa == 0) {
 			DRV_LOG(ERR, "Fail to get GPA for available ring.");
 			return -1;
 		}
 		hw->vring[i].avail = gpa;
 
-		gpa = qva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
+		gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used);
 		if (gpa == 0) {
 			DRV_LOG(ERR, "Fail to get GPA for used ring.");
 			return -1;
@@ -277,11 +276,29 @@ vdpa_ifcvf_start(struct ifcvf_internal *internal)
 }
 
 static void
+ifcvf_used_ring_log(struct ifcvf_hw *hw, uint32_t queue, uint8_t *log_buf)
+{
+	uint32_t i, size;
+	uint64_t pfn;
+
+	pfn = hw->vring[queue].used / PAGE_SIZE;
+	size = hw->vring[queue].size * sizeof(struct vring_used_elem) +
+			sizeof(uint16_t) * 3;
+
+	for (i = 0; i <= size / PAGE_SIZE; i++)
+		__sync_fetch_and_or_8(&log_buf[(pfn + i) / 8],
+				1 << ((pfn + i) % 8));
+}
+
+static void
 vdpa_ifcvf_stop(struct ifcvf_internal *internal)
 {
 	struct ifcvf_hw *hw = &internal->hw;
 	uint32_t i;
 	int vid;
+	uint64_t features;
+	uint64_t log_base, log_size;
+	uint8_t *log_buf;
 
 	vid = internal->vid;
 	ifcvf_stop_hw(hw);
@@ -289,6 +306,21 @@ vdpa_ifcvf_stop(struct ifcvf_internal *internal)
 	for (i = 0; i < hw->nr_vring; i++)
 		rte_vhost_set_vring_base(vid, i, hw->vring[i].last_avail_idx,
 				hw->vring[i].last_used_idx);
+
+	rte_vhost_get_negotiated_features(vid, &features);
+	if (RTE_VHOST_NEED_LOG(features)) {
+		ifcvf_disable_logging(hw);
+		rte_vhost_get_log_base(internal->vid, &log_base, &log_size);
+		rte_vfio_container_dma_unmap(internal->vfio_container_fd,
+				log_base, IFCVF_LOG_BASE, log_size);
+		/*
+		 * IFCVF marks dirty memory pages for only packet buffer,
+		 * SW helps to mark the used ring as dirty after device stops.
+		 */
+		log_buf = (uint8_t *)(uintptr_t)log_base;
+		for (i = 0; i < hw->nr_vring; i++)
+			ifcvf_used_ring_log(hw, i, log_buf);
+	}
 }
 
 #define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
@@ -469,11 +501,11 @@ update_datapath(struct ifcvf_internal *internal)
 		if (ret)
 			goto err;
 
-		ret = setup_notify_relay(internal);
+		ret = vdpa_ifcvf_start(internal);
 		if (ret)
 			goto err;
 
-		ret = vdpa_ifcvf_start(internal);
+		ret = setup_notify_relay(internal);
 		if (ret)
 			goto err;
 
@@ -481,12 +513,12 @@ update_datapath(struct ifcvf_internal *internal)
 	} else if (rte_atomic32_read(&internal->running) &&
 		   (!rte_atomic32_read(&internal->started) ||
 		    !rte_atomic32_read(&internal->dev_attached))) {
-		vdpa_ifcvf_stop(internal);
-
 		ret = unset_notify_relay(internal);
 		if (ret)
 			goto err;
 
+		vdpa_ifcvf_stop(internal);
+
 		ret = vdpa_disable_vfio_intr(internal);
 		if (ret)
 			goto err;
@@ -549,6 +581,35 @@ ifcvf_dev_close(int vid)
 }
 
 static int
+ifcvf_set_features(int vid)
+{
+	uint64_t features;
+	int did;
+	struct internal_list *list;
+	struct ifcvf_internal *internal;
+	uint64_t log_base, log_size;
+
+	did = rte_vhost_get_vdpa_device_id(vid);
+	list = find_internal_resource_by_did(did);
+	if (list == NULL) {
+		DRV_LOG(ERR, "Invalid device id: %d", did);
+		return -1;
+	}
+
+	internal = list->internal;
+	rte_vhost_get_negotiated_features(vid, &features);
+
+	if (RTE_VHOST_NEED_LOG(features)) {
+		rte_vhost_get_log_base(vid, &log_base, &log_size);
+		rte_vfio_container_dma_map(internal->vfio_container_fd,
+				log_base, IFCVF_LOG_BASE, log_size);
+		ifcvf_enable_logging(&internal->hw, IFCVF_LOG_BASE, log_size);
+	}
+
+	return 0;
+}
+
+static int
 ifcvf_get_vfio_group_fd(int vid)
 {
 	int did;
@@ -657,14 +718,14 @@ ifcvf_get_protocol_features(int did __rte_unused, uint64_t *features)
 	return 0;
 }
 
-struct rte_vdpa_dev_ops ifcvf_ops = {
+static struct rte_vdpa_dev_ops ifcvf_ops = {
 	.get_queue_num = ifcvf_get_queue_num,
 	.get_features = ifcvf_get_vdpa_features,
 	.get_protocol_features = ifcvf_get_protocol_features,
 	.dev_conf = ifcvf_dev_config,
 	.dev_close = ifcvf_dev_close,
 	.set_vring_state = NULL,
-	.set_features = NULL,
+	.set_features = ifcvf_set_features,
 	.migration_done = NULL,
 	.get_vfio_group_fd = ifcvf_get_vfio_group_fd,
 	.get_vfio_device_fd = ifcvf_get_vfio_device_fd,
@@ -695,11 +756,18 @@ ifcvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	if (ifcvf_vfio_setup(internal) < 0)
 		return -1;
 
+	if (ifcvf_init_hw(&internal->hw, internal->pdev) < 0)
+		return -1;
+
 	internal->max_queues = IFCVF_MAX_QUEUES;
 	features = ifcvf_get_features(&internal->hw);
 	internal->features = (features &
 		~(1ULL << VIRTIO_F_IOMMU_PLATFORM)) |
-		(1ULL << VHOST_USER_F_PROTOCOL_FEATURES);
+		(1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) |
+		(1ULL << VIRTIO_NET_F_CTRL_VQ) |
+		(1ULL << VIRTIO_NET_F_STATUS) |
+		(1ULL << VHOST_USER_F_PROTOCOL_FEATURES) |
+		(1ULL << VHOST_F_LOG_ALL);
 
 	internal->dev_addr.pci_addr = pci_dev->addr;
 	internal->dev_addr.type = PCI_ADDR;
diff --git a/drivers/net/ixgbe/base/README b/drivers/net/ixgbe/base/README
index 70fdfe7c..431be026 100644
--- a/drivers/net/ixgbe/base/README
+++ b/drivers/net/ixgbe/base/README
@@ -1,7 +1,7 @@
 ..
      BSD LICENSE
    
-     Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
+     Copyright(c) 2010-2018 Intel Corporation. All rights reserved.
      All rights reserved.
    
      Redistribution and use in source and binary forms, with or without
@@ -34,7 +34,7 @@ Intel® IXGBE driver
 ===================
 
 This directory contains source code of FreeBSD ixgbe driver of version
-cid-ixgbe.2018.01.02.tar.gz released by the team which develop
+cid-ixgbe.2018.08.28.tar.gz released by the team which develop
 basic drivers for any ixgbe NIC. The sub-directory of base/
 contains the original source package.
 This driver is valid for the product(s) listed below
diff --git a/drivers/net/ixgbe/base/ixgbe_82598.c b/drivers/net/ixgbe/base/ixgbe_82598.c
index ee7ce2e9..245ff75d 100644
--- a/drivers/net/ixgbe/base/ixgbe_82598.c
+++ b/drivers/net/ixgbe/base/ixgbe_82598.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "ixgbe_type.h"
 #include "ixgbe_82598.h"
diff --git a/drivers/net/ixgbe/base/ixgbe_82598.h b/drivers/net/ixgbe/base/ixgbe_82598.h
index 20aab9fc..8013f495 100644
--- a/drivers/net/ixgbe/base/ixgbe_82598.h
+++ b/drivers/net/ixgbe/base/ixgbe_82598.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_82598_H_
 #define _IXGBE_82598_H_
diff --git a/drivers/net/ixgbe/base/ixgbe_82599.c b/drivers/net/ixgbe/base/ixgbe_82599.c
index 26217212..7de753fd 100644
--- a/drivers/net/ixgbe/base/ixgbe_82599.c
+++ b/drivers/net/ixgbe/base/ixgbe_82599.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "ixgbe_type.h"
 #include "ixgbe_82599.h"
@@ -87,9 +58,6 @@ void ixgbe_init_mac_link_ops_82599(struct ixgbe_hw *hw)
 		mac->ops.setup_mac_link = ixgbe_setup_mac_link_82599;
 		mac->ops.set_rate_select_speed =
 					       ixgbe_set_hard_rate_select_speed;
-		if (ixgbe_get_media_type(hw) == ixgbe_media_type_fiber_fixed)
-			mac->ops.set_rate_select_speed =
-					       ixgbe_set_soft_rate_select_speed;
 	} else {
 		if ((ixgbe_get_media_type(hw) == ixgbe_media_type_backplane) &&
 		     (hw->phy.smart_speed == ixgbe_smart_speed_auto ||
@@ -561,16 +529,9 @@ enum ixgbe_media_type ixgbe_get_media_type_82599(struct ixgbe_hw *hw)
 	case IXGBE_DEV_ID_82599_T3_LOM:
 		media_type = ixgbe_media_type_copper;
 		break;
-	case IXGBE_DEV_ID_82599_LS:
-		media_type = ixgbe_media_type_fiber_lco;
-		break;
 	case IXGBE_DEV_ID_82599_QSFP_SF_QP:
 		media_type = ixgbe_media_type_fiber_qsfp;
 		break;
-	case IXGBE_DEV_ID_82599_BYPASS:
-		media_type = ixgbe_media_type_fiber_fixed;
-		hw->phy.multispeed_fiber = true;
-		break;
 	default:
 		media_type = ixgbe_media_type_unknown;
 		break;
diff --git a/drivers/net/ixgbe/base/ixgbe_82599.h b/drivers/net/ixgbe/base/ixgbe_82599.h
index d555dbce..a32eb1f5 100644
--- a/drivers/net/ixgbe/base/ixgbe_82599.h
+++ b/drivers/net/ixgbe/base/ixgbe_82599.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_82599_H_
 #define _IXGBE_82599_H_
diff --git a/drivers/net/ixgbe/base/ixgbe_api.c b/drivers/net/ixgbe/base/ixgbe_api.c
index e50c1045..873c0799 100644
--- a/drivers/net/ixgbe/base/ixgbe_api.c
+++ b/drivers/net/ixgbe/base/ixgbe_api.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "ixgbe_api.h"
 #include "ixgbe_common.h"
@@ -177,8 +148,6 @@ s32 ixgbe_set_mac_type(struct ixgbe_hw *hw)
 	case IXGBE_DEV_ID_82599_QSFP_SF_QP:
 	case IXGBE_DEV_ID_82599EN_SFP:
 	case IXGBE_DEV_ID_82599_CX4:
-	case IXGBE_DEV_ID_82599_LS:
-	case IXGBE_DEV_ID_82599_BYPASS:
 	case IXGBE_DEV_ID_82599_T3_LOM:
 		hw->mac.type = ixgbe_mac_82599EB;
 		break;
@@ -193,7 +162,6 @@ s32 ixgbe_set_mac_type(struct ixgbe_hw *hw)
 		break;
 	case IXGBE_DEV_ID_X540T:
 	case IXGBE_DEV_ID_X540T1:
-	case IXGBE_DEV_ID_X540_BYPASS:
 		hw->mac.type = ixgbe_mac_X540;
 		hw->mvals = ixgbe_mvals_X540;
 		break;
@@ -1360,6 +1328,18 @@ void ixgbe_restore_mdd_vf(struct ixgbe_hw *hw, u32 vf)
 }
 
 /**
+ *  ixgbe_fw_recovery_mode - Check if in FW NVM recovery mode
+ *  @hw: pointer to hardware structure
+ *
+ **/
+bool ixgbe_fw_recovery_mode(struct ixgbe_hw *hw)
+{
+	if (hw->mac.ops.fw_recovery_mode)
+		return hw->mac.ops.fw_recovery_mode(hw);
+	return false;
+}
+
+/**
  *  ixgbe_enter_lplu - Transition to low power states
  *  @hw: pointer to hardware structure
  *
diff --git a/drivers/net/ixgbe/base/ixgbe_api.h b/drivers/net/ixgbe/base/ixgbe_api.h
index 2f532aa8..ff8f7b26 100644
--- a/drivers/net/ixgbe/base/ixgbe_api.h
+++ b/drivers/net/ixgbe/base/ixgbe_api.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_API_H_
 #define _IXGBE_API_H_
@@ -214,6 +185,7 @@ void ixgbe_disable_mdd(struct ixgbe_hw *hw);
 void ixgbe_enable_mdd(struct ixgbe_hw *hw);
 void ixgbe_mdd_event(struct ixgbe_hw *hw, u32 *vf_bitmap);
 void ixgbe_restore_mdd_vf(struct ixgbe_hw *hw, u32 vf);
+bool ixgbe_fw_recovery_mode(struct ixgbe_hw *hw);
 s32 ixgbe_enter_lplu(struct ixgbe_hw *hw);
 s32 ixgbe_handle_lasi(struct ixgbe_hw *hw);
 void ixgbe_set_rate_select_speed(struct ixgbe_hw *hw, ixgbe_link_speed speed);
diff --git a/drivers/net/ixgbe/base/ixgbe_common.c b/drivers/net/ixgbe/base/ixgbe_common.c
index e7e9256e..21f973e5 100644
--- a/drivers/net/ixgbe/base/ixgbe_common.c
+++ b/drivers/net/ixgbe/base/ixgbe_common.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "ixgbe_common.h"
 #include "ixgbe_phy.h"
@@ -167,7 +138,6 @@ bool ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw)
 	DEBUGFUNC("ixgbe_device_supports_autoneg_fc");
 
 	switch (hw->phy.media_type) {
-	case ixgbe_media_type_fiber_fixed:
 	case ixgbe_media_type_fiber_qsfp:
 	case ixgbe_media_type_fiber:
 		/* flow control autoneg black list */
@@ -201,7 +171,6 @@ bool ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw)
 		case IXGBE_DEV_ID_82599_T3_LOM:
 		case IXGBE_DEV_ID_X540T:
 		case IXGBE_DEV_ID_X540T1:
-		case IXGBE_DEV_ID_X540_BYPASS:
 		case IXGBE_DEV_ID_X550T:
 		case IXGBE_DEV_ID_X550T1:
 		case IXGBE_DEV_ID_X550EM_X_10G_T:
@@ -267,7 +236,6 @@ s32 ixgbe_setup_fc_generic(struct ixgbe_hw *hw)
 			goto out;
 
 		/* fall through - only backplane uses autoc */
-	case ixgbe_media_type_fiber_fixed:
 	case ixgbe_media_type_fiber_qsfp:
 	case ixgbe_media_type_fiber:
 		reg = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
@@ -3127,7 +3095,6 @@ void ixgbe_fc_autoneg(struct ixgbe_hw *hw)
 
 	switch (hw->phy.media_type) {
 	/* Autoneg flow control on fiber adapters */
-	case ixgbe_media_type_fiber_fixed:
 	case ixgbe_media_type_fiber_qsfp:
 	case ixgbe_media_type_fiber:
 		if (speed == IXGBE_LINK_SPEED_1GB_FULL)
@@ -5267,7 +5234,6 @@ s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw,
 
 		/* Set the module link speed */
 		switch (hw->phy.media_type) {
-		case ixgbe_media_type_fiber_fixed:
 		case ixgbe_media_type_fiber:
 			ixgbe_set_rate_select_speed(hw,
 						    IXGBE_LINK_SPEED_10GB_FULL);
@@ -5296,7 +5262,7 @@ s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw,
 		 * Section 73.10.2, we may have to wait up to 500ms if KR is
 		 * attempted.  82599 uses the same timing for 10g SFI.
 		 */
-		for (i = 0; i < 5; i++) {
+		for (i = 0; i < 10; i++) {
 			/* Wait for the link partner to also set speed */
 			msec_delay(100);
 
@@ -5318,7 +5284,6 @@ s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw,
 
 		/* Set the module link speed */
 		switch (hw->phy.media_type) {
-		case ixgbe_media_type_fiber_fixed:
 		case ixgbe_media_type_fiber:
 			ixgbe_set_rate_select_speed(hw,
 						    IXGBE_LINK_SPEED_1GB_FULL);
diff --git a/drivers/net/ixgbe/base/ixgbe_common.h b/drivers/net/ixgbe/base/ixgbe_common.h
index fd35dcc4..3bb24751 100644
--- a/drivers/net/ixgbe/base/ixgbe_common.h
+++ b/drivers/net/ixgbe/base/ixgbe_common.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_COMMON_H_
 #define _IXGBE_COMMON_H_
diff --git a/drivers/net/ixgbe/base/ixgbe_dcb.c b/drivers/net/ixgbe/base/ixgbe_dcb.c
index 2877f22b..a590e0e0 100644
--- a/drivers/net/ixgbe/base/ixgbe_dcb.c
+++ b/drivers/net/ixgbe/base/ixgbe_dcb.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 
 #include "ixgbe_type.h"
diff --git a/drivers/net/ixgbe/base/ixgbe_dcb.h b/drivers/net/ixgbe/base/ixgbe_dcb.h
index 41208049..503d0601 100644
--- a/drivers/net/ixgbe/base/ixgbe_dcb.h
+++ b/drivers/net/ixgbe/base/ixgbe_dcb.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_DCB_H_
 #define _IXGBE_DCB_H_
diff --git a/drivers/net/ixgbe/base/ixgbe_dcb_82598.c b/drivers/net/ixgbe/base/ixgbe_dcb_82598.c
index 3ed8337b..d87cb588 100644
--- a/drivers/net/ixgbe/base/ixgbe_dcb_82598.c
+++ b/drivers/net/ixgbe/base/ixgbe_dcb_82598.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 
 #include "ixgbe_type.h"
diff --git a/drivers/net/ixgbe/base/ixgbe_dcb_82598.h b/drivers/net/ixgbe/base/ixgbe_dcb_82598.h
index eb88b3d3..1a147444 100644
--- a/drivers/net/ixgbe/base/ixgbe_dcb_82598.h
+++ b/drivers/net/ixgbe/base/ixgbe_dcb_82598.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_DCB_82598_H_
 #define _IXGBE_DCB_82598_H_
diff --git a/drivers/net/ixgbe/base/ixgbe_dcb_82599.c b/drivers/net/ixgbe/base/ixgbe_dcb_82599.c
index 8f9e1590..f4f0ff01 100644
--- a/drivers/net/ixgbe/base/ixgbe_dcb_82599.c
+++ b/drivers/net/ixgbe/base/ixgbe_dcb_82599.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 
 #include "ixgbe_type.h"
diff --git a/drivers/net/ixgbe/base/ixgbe_dcb_82599.h b/drivers/net/ixgbe/base/ixgbe_dcb_82599.h
index dc0fb284..085ada27 100644
--- a/drivers/net/ixgbe/base/ixgbe_dcb_82599.h
+++ b/drivers/net/ixgbe/base/ixgbe_dcb_82599.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_DCB_82599_H_
 #define _IXGBE_DCB_82599_H_
diff --git a/drivers/net/ixgbe/base/ixgbe_hv_vf.c b/drivers/net/ixgbe/base/ixgbe_hv_vf.c
index 40dad775..67a124d8 100644
--- a/drivers/net/ixgbe/base/ixgbe_hv_vf.c
+++ b/drivers/net/ixgbe/base/ixgbe_hv_vf.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "ixgbe_vf.h"
 #include "ixgbe_hv_vf.h"
diff --git a/drivers/net/ixgbe/base/ixgbe_hv_vf.h b/drivers/net/ixgbe/base/ixgbe_hv_vf.h
index 9119f29f..9664f3bd 100644
--- a/drivers/net/ixgbe/base/ixgbe_hv_vf.h
+++ b/drivers/net/ixgbe/base/ixgbe_hv_vf.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2016, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_HV_VF_H_
 #define _IXGBE_HV_VF_H_
diff --git a/drivers/net/ixgbe/base/ixgbe_mbx.c b/drivers/net/ixgbe/base/ixgbe_mbx.c
index 2785bbad..cb82942d 100644
--- a/drivers/net/ixgbe/base/ixgbe_mbx.c
+++ b/drivers/net/ixgbe/base/ixgbe_mbx.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "ixgbe_type.h"
 #include "ixgbe_mbx.h"
diff --git a/drivers/net/ixgbe/base/ixgbe_mbx.h b/drivers/net/ixgbe/base/ixgbe_mbx.h
index bde50a51..5d32cbc0 100644
--- a/drivers/net/ixgbe/base/ixgbe_mbx.h
+++ b/drivers/net/ixgbe/base/ixgbe_mbx.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_MBX_H_
 #define _IXGBE_MBX_H_
diff --git a/drivers/net/ixgbe/base/ixgbe_osdep.h b/drivers/net/ixgbe/base/ixgbe_osdep.h
index bb5dfd2a..ea8dc1cb 100644
--- a/drivers/net/ixgbe/base/ixgbe_osdep.h
+++ b/drivers/net/ixgbe/base/ixgbe_osdep.h
@@ -1,36 +1,6 @@
-/******************************************************************************
-
-  Copyright (c) 2001-2015, Intel Corporation
-  All rights reserved.
-  
-  Redistribution and use in source and binary forms, with or without 
-  modification, are permitted provided that the following conditions are met:
-  
-   1. Redistributions of source code must retain the above copyright notice, 
-      this list of conditions and the following disclaimer.
-  
-   2. Redistributions in binary form must reproduce the above copyright 
-      notice, this list of conditions and the following disclaimer in the 
-      documentation and/or other materials provided with the distribution.
-  
-   3. Neither the name of the Intel Corporation nor the names of its 
-      contributors may be used to endorse or promote products derived from 
-      this software without specific prior written permission.
-  
-  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
-  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
-  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
-  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
-  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
-  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
-  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
-  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
-  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-  POSSIBILITY OF SUCH DAMAGE.
-
-******************************************************************************/
-/*$FreeBSD$*/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_OS_H_
 #define _IXGBE_OS_H_
@@ -51,7 +21,7 @@
 
 #define ASSERT(x) if(!(x)) rte_panic("IXGBE: x")
 
-#define DELAY(x) rte_delay_us(x)
+#define DELAY(x) rte_delay_us_sleep(x)
 #define usec_delay(x) DELAY(x)
 #define msec_delay(x) DELAY(1000*(x))
 
diff --git a/drivers/net/ixgbe/base/ixgbe_phy.c b/drivers/net/ixgbe/base/ixgbe_phy.c
index 2df068ee..6cdd8fba 100644
--- a/drivers/net/ixgbe/base/ixgbe_phy.c
+++ b/drivers/net/ixgbe/base/ixgbe_phy.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "ixgbe_api.h"
 #include "ixgbe_common.h"
@@ -2594,7 +2565,6 @@ STATIC bool ixgbe_get_i2c_data(struct ixgbe_hw *hw, u32 *i2cctl)
 {
 	u32 data_oe_bit = IXGBE_I2C_DATA_OE_N_EN_BY_MAC(hw);
 	bool data;
-	UNREFERENCED_1PARAMETER(hw);
 
 	DEBUGFUNC("ixgbe_get_i2c_data");
 
diff --git a/drivers/net/ixgbe/base/ixgbe_phy.h b/drivers/net/ixgbe/base/ixgbe_phy.h
index cf8cadd9..132fa542 100644
--- a/drivers/net/ixgbe/base/ixgbe_phy.h
+++ b/drivers/net/ixgbe/base/ixgbe_phy.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_PHY_H_
 #define _IXGBE_PHY_H_
diff --git a/drivers/net/ixgbe/base/ixgbe_type.h b/drivers/net/ixgbe/base/ixgbe_type.h
index 6e03089e..cee6ba2e 100644
--- a/drivers/net/ixgbe/base/ixgbe_type.h
+++ b/drivers/net/ixgbe/base/ixgbe_type.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_TYPE_H_
 #define _IXGBE_TYPE_H_
@@ -46,8 +17,8 @@ POSSIBILITY OF SUCH DAMAGE.
  *
  * - IXGBE_ERROR_POLLING
  * This category is for errors related to polling/timeout issues and should be
- * used in any case where the timeout occured, or a failure to obtain a lock, or
- * failure to receive data within the time limit.
+ * used in any case where the timeout occurred, or a failure to obtain a lock,
+ * or failure to receive data within the time limit.
  *
  * - IXGBE_ERROR_CAUTION
  * This category should be used for reporting issues that may be the cause of
@@ -122,12 +93,9 @@ POSSIBILITY OF SUCH DAMAGE.
 #define IXGBE_DEV_ID_82599_T3_LOM		0x151C
 #define IXGBE_DEV_ID_82599_VF			0x10ED
 #define IXGBE_DEV_ID_82599_VF_HV		0x152E
-#define IXGBE_DEV_ID_82599_LS			0x154F
-#define IXGBE_DEV_ID_82599_BYPASS		0x155D
 #define IXGBE_DEV_ID_X540T			0x1528
 #define IXGBE_DEV_ID_X540_VF			0x1515
 #define IXGBE_DEV_ID_X540_VF_HV			0x1530
-#define IXGBE_DEV_ID_X540_BYPASS		0x155C
 #define IXGBE_DEV_ID_X540T1			0x1560
 #define IXGBE_DEV_ID_X550T			0x1563
 #define IXGBE_DEV_ID_X550T1			0x15D1
@@ -882,6 +850,10 @@ struct ixgbe_dmac_config {
 #define IXGBE_RTTDQSEL		0x04904
 #define IXGBE_RTTDT1C		0x04908
 #define IXGBE_RTTDT1S		0x0490C
+#define IXGBE_RTTQCNCR		0x08B00
+#define IXGBE_RTTQCNTG		0x04A90
+#define IXGBE_RTTBCNRD		0x0498C
+#define IXGBE_RTTQCNRR		0x0498C
 #define IXGBE_RTTDTECC		0x04990
 #define IXGBE_RTTDTECC_NO_BCN	0x00000100
 
@@ -892,6 +864,7 @@ struct ixgbe_dmac_config {
 #define IXGBE_RTTBCNRC_RF_INT_MASK \
 	(IXGBE_RTTBCNRC_RF_DEC_MASK << IXGBE_RTTBCNRC_RF_INT_SHIFT)
 #define IXGBE_RTTBCNRM	0x04980
+#define IXGBE_RTTQCNRM	0x04980
 
 /* BCN (for DCB) Registers */
 #define IXGBE_RTTBCNRS	0x04988
@@ -1099,6 +1072,9 @@ struct ixgbe_dmac_config {
 #define IXGBE_FWSM_MODE_MASK	0xE
 #define IXGBE_FWSM_TS_ENABLED	0x1
 #define IXGBE_FWSM_FW_MODE_PT	0x4
+#define IXGBE_FWSM_FW_NVM_RECOVERY_MODE (1 << 5)
+#define IXGBE_FWSM_EXT_ERR_IND_MASK 0x01F80000
+#define IXGBE_FWSM_FW_VAL_BIT	(1 << 15)
 
 /* ARC Subsystem registers */
 #define IXGBE_HICR		0x15F00
@@ -3755,9 +3731,7 @@ enum ixgbe_sfp_type {
 enum ixgbe_media_type {
 	ixgbe_media_type_unknown = 0,
 	ixgbe_media_type_fiber,
-	ixgbe_media_type_fiber_fixed,
 	ixgbe_media_type_fiber_qsfp,
-	ixgbe_media_type_fiber_lco,
 	ixgbe_media_type_copper,
 	ixgbe_media_type_backplane,
 	ixgbe_media_type_cx4,
@@ -4050,6 +4024,7 @@ struct ixgbe_mac_operations {
 	void (*enable_mdd)(struct ixgbe_hw *hw);
 	void (*mdd_event)(struct ixgbe_hw *hw, u32 *vf_bitmap);
 	void (*restore_mdd_vf)(struct ixgbe_hw *hw, u32 vf);
+	bool (*fw_recovery_mode)(struct ixgbe_hw *hw);
 };
 
 struct ixgbe_phy_operations {
diff --git a/drivers/net/ixgbe/base/ixgbe_vf.c b/drivers/net/ixgbe/base/ixgbe_vf.c
index 5b25a6b4..aac37822 100644
--- a/drivers/net/ixgbe/base/ixgbe_vf.c
+++ b/drivers/net/ixgbe/base/ixgbe_vf.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 
 #include "ixgbe_api.h"
diff --git a/drivers/net/ixgbe/base/ixgbe_vf.h b/drivers/net/ixgbe/base/ixgbe_vf.h
index 3efffe82..dba643fc 100644
--- a/drivers/net/ixgbe/base/ixgbe_vf.h
+++ b/drivers/net/ixgbe/base/ixgbe_vf.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_VF_H_
 #define _IXGBE_VF_H_
diff --git a/drivers/net/ixgbe/base/ixgbe_x540.c b/drivers/net/ixgbe/base/ixgbe_x540.c
index 716664bb..f00f0eae 100644
--- a/drivers/net/ixgbe/base/ixgbe_x540.c
+++ b/drivers/net/ixgbe/base/ixgbe_x540.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "ixgbe_x540.h"
 #include "ixgbe_type.h"
diff --git a/drivers/net/ixgbe/base/ixgbe_x540.h b/drivers/net/ixgbe/base/ixgbe_x540.h
index 8a19ae2e..231dfe56 100644
--- a/drivers/net/ixgbe/base/ixgbe_x540.h
+++ b/drivers/net/ixgbe/base/ixgbe_x540.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_X540_H_
 #define _IXGBE_X540_H_
diff --git a/drivers/net/ixgbe/base/ixgbe_x550.c b/drivers/net/ixgbe/base/ixgbe_x550.c
index f66f5407..f7b98af5 100644
--- a/drivers/net/ixgbe/base/ixgbe_x550.c
+++ b/drivers/net/ixgbe/base/ixgbe_x550.c
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #include "ixgbe_x550.h"
 #include "ixgbe_x540.h"
@@ -82,6 +53,7 @@ s32 ixgbe_init_ops_X550(struct ixgbe_hw *hw)
 	mac->ops.enable_mdd = ixgbe_enable_mdd_X550;
 	mac->ops.mdd_event = ixgbe_mdd_event_X550;
 	mac->ops.restore_mdd_vf = ixgbe_restore_mdd_vf_X550;
+	mac->ops.fw_recovery_mode = ixgbe_fw_recovery_mode_X550;
 	mac->ops.disable_rx = ixgbe_disable_rx_x550;
 	/* Manageability interface */
 	mac->ops.set_fw_drv_ver = ixgbe_set_fw_drv_ver_x550;
@@ -349,7 +321,7 @@ STATIC s32 ixgbe_identify_phy_x550em(struct ixgbe_hw *hw)
 
 	switch (hw->device_id) {
 	case IXGBE_DEV_ID_X550EM_A_SFP:
-		return ixgbe_identify_module_generic(hw);
+		return ixgbe_identify_sfp_module_X550em(hw);
 	case IXGBE_DEV_ID_X550EM_X_SFP:
 		/* set up for CS4227 usage */
 		ixgbe_setup_mux_ctl(hw);
@@ -357,7 +329,7 @@ STATIC s32 ixgbe_identify_phy_x550em(struct ixgbe_hw *hw)
 		/* Fallthrough */
 
 	case IXGBE_DEV_ID_X550EM_A_SFP_N:
-		return ixgbe_identify_module_generic(hw);
+		return ixgbe_identify_sfp_module_X550em(hw);
 		break;
 	case IXGBE_DEV_ID_X550EM_X_KX4:
 		hw->phy.type = ixgbe_phy_x550em_kx4;
@@ -2808,9 +2780,9 @@ s32 ixgbe_setup_mac_link_sfp_x550a(struct ixgbe_hw *hw,
 				 (IXGBE_CS4227_EDC_MODE_SR << 1));
 
 		if (setup_linear)
-			reg_phy_ext = (IXGBE_CS4227_EDC_MODE_CX1 << 1) | 0x1;
+			reg_phy_ext |= (IXGBE_CS4227_EDC_MODE_CX1 << 1) | 0x1;
 		else
-			reg_phy_ext = (IXGBE_CS4227_EDC_MODE_SR << 1) | 0x1;
+			reg_phy_ext |= (IXGBE_CS4227_EDC_MODE_SR << 1) | 0x1;
 		ret_val = hw->phy.ops.write_reg(hw, reg_slice,
 					 IXGBE_MDIO_ZERO_DEV_TYPE, reg_phy_ext);
 
@@ -4661,3 +4633,18 @@ s32 ixgbe_set_fw_drv_ver_x550(struct ixgbe_hw *hw, u8 maj, u8 min,
 
 	return ret_val;
 }
+
+/**
+ * ixgbe_fw_recovery_mode_X550 - Check FW NVM recovery mode
+ * @hw: pointer t hardware structure
+ *
+ * Returns true if in FW NVM recovery mode.
+ **/
+bool ixgbe_fw_recovery_mode_X550(struct ixgbe_hw *hw)
+{
+	u32 fwsm;
+
+	fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM_BY_MAC(hw));
+
+	return !!(fwsm & IXGBE_FWSM_FW_NVM_RECOVERY_MODE);
+}
diff --git a/drivers/net/ixgbe/base/ixgbe_x550.h b/drivers/net/ixgbe/base/ixgbe_x550.h
index 6d188741..3bd98f24 100644
--- a/drivers/net/ixgbe/base/ixgbe_x550.h
+++ b/drivers/net/ixgbe/base/ixgbe_x550.h
@@ -1,35 +1,6 @@
-/*******************************************************************************
-
-Copyright (c) 2001-2015, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Intel Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2001-2018
+ */
 
 #ifndef _IXGBE_X550_H_
 #define _IXGBE_X550_H_
@@ -121,4 +92,5 @@ s32 ixgbe_reset_phy_t_X550em(struct ixgbe_hw *hw);
 s32 ixgbe_identify_sfp_module_X550em(struct ixgbe_hw *hw);
 s32 ixgbe_led_on_t_X550em(struct ixgbe_hw *hw, u32 led_idx);
 s32 ixgbe_led_off_t_X550em(struct ixgbe_hw *hw, u32 led_idx);
+bool ixgbe_fw_recovery_mode_X550(struct ixgbe_hw *hw);
 #endif /* _IXGBE_X550_H_ */
diff --git a/drivers/net/ixgbe/base/meson.build b/drivers/net/ixgbe/base/meson.build
index 3147e110..21ac64bf 100644
--- a/drivers/net/ixgbe/base/meson.build
+++ b/drivers/net/ixgbe/base/meson.build
@@ -20,6 +20,9 @@ sources = [
 error_cflags = ['-Wno-unused-value',
 		'-Wno-unused-but-set-variable']
 c_args = cflags
+if allow_experimental_apis
+	c_args += '-DALLOW_EXPERIMENTAL_API'
+endif
 foreach flag: error_cflags
 	if cc.has_argument(flag)
 		c_args += flag
diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c b/drivers/net/ixgbe/ixgbe_ethdev.c
index 26b19273..269595b7 100644
--- a/drivers/net/ixgbe/ixgbe_ethdev.c
+++ b/drivers/net/ixgbe/ixgbe_ethdev.c
@@ -217,8 +217,7 @@ static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on);
 static int ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev);
 static int ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
-static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev,
-				      struct rte_intr_handle *handle);
+static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 static void ixgbe_dev_interrupt_handler(void *param);
 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 static int ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
@@ -437,7 +436,6 @@ static const struct rte_pci_id pci_id_ixgbe_map[] = {
 	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP) },
 	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM) },
 	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM) },
-	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_LS) },
 	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T) },
 	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1) },
 	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP) },
@@ -1119,6 +1117,14 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
 		return -EIO;
 	}
 
+	if (hw->mac.ops.fw_recovery_mode && hw->mac.ops.fw_recovery_mode(hw)) {
+		PMD_INIT_LOG(ERR, "\nERROR: "
+			"Firmware recovery mode detected. Limiting functionality.\n"
+			"Refer to the Intel(R) Ethernet Adapters and Devices "
+			"User Guide for details on firmware recovery mode.");
+		return -EIO;
+	}
+
 	/* pick up the PCI bus settings for reporting later */
 	ixgbe_get_bus_info(hw);
 
@@ -1331,12 +1337,6 @@ eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev)
 	/* uninitialize PF if max_vfs not zero */
 	ixgbe_pf_host_uninit(eth_dev);
 
-	rte_free(eth_dev->data->mac_addrs);
-	eth_dev->data->mac_addrs = NULL;
-
-	rte_free(eth_dev->data->hash_mac_addrs);
-	eth_dev->data->hash_mac_addrs = NULL;
-
 	/* remove all the fdir filters & hash */
 	ixgbe_fdir_filter_uninit(eth_dev);
 
@@ -1619,7 +1619,12 @@ eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev)
 	 */
 	if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
 		PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
-		return diag;
+		/*
+		 * This error code will be propagated to the app by
+		 * rte_eth_dev_reset, so use a public error code rather than
+		 * the internal-only IXGBE_ERR_RESET_FAILED
+		 */
+		return -EAGAIN;
 	}
 
 	/* negotiate mailbox API version to use with the PF. */
@@ -1711,9 +1716,6 @@ eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev)
 	/* Disable the interrupts for VF */
 	ixgbevf_intr_disable(eth_dev);
 
-	rte_free(eth_dev->data->mac_addrs);
-	eth_dev->data->mac_addrs = NULL;
-
 	rte_intr_disable(intr_handle);
 	rte_intr_callback_unregister(intr_handle,
 				     ixgbevf_dev_interrupt_handler, eth_dev);
@@ -4282,8 +4284,7 @@ ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
  *  - On failure, a negative value.
  */
 static int
-ixgbe_dev_interrupt_action(struct rte_eth_dev *dev,
-			   struct rte_intr_handle *intr_handle)
+ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
 {
 	struct ixgbe_interrupt *intr =
 		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
@@ -4334,7 +4335,6 @@ ixgbe_dev_interrupt_action(struct rte_eth_dev *dev,
 
 	PMD_DRV_LOG(DEBUG, "enable intr immediately");
 	ixgbe_enable_intr(dev);
-	rte_intr_enable(intr_handle);
 
 	return 0;
 }
@@ -4417,7 +4417,7 @@ ixgbe_dev_interrupt_handler(void *param)
 	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 
 	ixgbe_dev_interrupt_get_status(dev);
-	ixgbe_dev_interrupt_action(dev, dev->intr_handle);
+	ixgbe_dev_interrupt_action(dev);
 }
 
 static int
@@ -5008,14 +5008,14 @@ ixgbevf_dev_configure(struct rte_eth_dev *dev)
 	 * Keep the persistent behavior the same as Host PF
 	 */
 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
-	if (rte_eth_dev_must_keep_crc(conf->rxmode.offloads)) {
+	if (conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
 		PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
-		conf->rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
+		conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC;
 	}
 #else
-	if (!rte_eth_dev_must_keep_crc(conf->rxmode.offloads)) {
+	if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) {
 		PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
-		conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_CRC_STRIP;
+		conf->rxmode.offloads |= DEV_RX_OFFLOAD_KEEP_CRC;
 	}
 #endif
 
diff --git a/drivers/net/ixgbe/ixgbe_flow.c b/drivers/net/ixgbe/ixgbe_flow.c
index 1adf1b80..f0fafebc 100644
--- a/drivers/net/ixgbe/ixgbe_flow.c
+++ b/drivers/net/ixgbe/ixgbe_flow.c
@@ -363,6 +363,17 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr,
 				item, "Not supported by ntuple filter");
 			return -rte_errno;
 		}
+		if ((ipv4_mask->hdr.src_addr != 0 &&
+			ipv4_mask->hdr.src_addr != UINT32_MAX) ||
+			(ipv4_mask->hdr.dst_addr != 0 &&
+			ipv4_mask->hdr.dst_addr != UINT32_MAX) ||
+			(ipv4_mask->hdr.next_proto_id != UINT8_MAX &&
+			ipv4_mask->hdr.next_proto_id != 0)) {
+			rte_flow_error_set(error,
+				EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
+				item, "Not supported by ntuple filter");
+			return -rte_errno;
+		}
 
 		filter->dst_ip_mask = ipv4_mask->hdr.dst_addr;
 		filter->src_ip_mask = ipv4_mask->hdr.src_addr;
@@ -432,6 +443,15 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr,
 				item, "Not supported by ntuple filter");
 			return -rte_errno;
 		}
+		if ((tcp_mask->hdr.src_port != 0 &&
+			tcp_mask->hdr.src_port != UINT16_MAX) ||
+			(tcp_mask->hdr.dst_port != 0 &&
+			tcp_mask->hdr.dst_port != UINT16_MAX)) {
+			rte_flow_error_set(error,
+				EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
+				item, "Not supported by ntuple filter");
+			return -rte_errno;
+		}
 
 		filter->dst_port_mask  = tcp_mask->hdr.dst_port;
 		filter->src_port_mask  = tcp_mask->hdr.src_port;
@@ -467,6 +487,15 @@ cons_parse_ntuple_filter(const struct rte_flow_attr *attr,
 				item, "Not supported by ntuple filter");
 			return -rte_errno;
 		}
+		if ((udp_mask->hdr.src_port != 0 &&
+			udp_mask->hdr.src_port != UINT16_MAX) ||
+			(udp_mask->hdr.dst_port != 0 &&
+			udp_mask->hdr.dst_port != UINT16_MAX)) {
+			rte_flow_error_set(error,
+				EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
+				item, "Not supported by ntuple filter");
+			return -rte_errno;
+		}
 
 		filter->dst_port_mask = udp_mask->hdr.dst_port;
 		filter->src_port_mask = udp_mask->hdr.src_port;
diff --git a/drivers/net/ixgbe/ixgbe_ipsec.c b/drivers/net/ixgbe/ixgbe_ipsec.c
index 08405f1e..5a416885 100644
--- a/drivers/net/ixgbe/ixgbe_ipsec.c
+++ b/drivers/net/ixgbe/ixgbe_ipsec.c
@@ -364,6 +364,7 @@ ixgbe_crypto_create_session(void *device,
 			conf->crypto_xform->aead.algo !=
 					RTE_CRYPTO_AEAD_AES_GCM) {
 		PMD_DRV_LOG(ERR, "Unsupported crypto transformation mode\n");
+		rte_mempool_put(mempool, (void *)ic_session);
 		return -ENOTSUP;
 	}
 	aead_xform = &conf->crypto_xform->aead;
@@ -373,6 +374,7 @@ ixgbe_crypto_create_session(void *device,
 			ic_session->op = IXGBE_OP_AUTHENTICATED_DECRYPTION;
 		} else {
 			PMD_DRV_LOG(ERR, "IPsec decryption not enabled\n");
+			rte_mempool_put(mempool, (void *)ic_session);
 			return -ENOTSUP;
 		}
 	} else {
@@ -380,6 +382,7 @@ ixgbe_crypto_create_session(void *device,
 			ic_session->op = IXGBE_OP_AUTHENTICATED_ENCRYPTION;
 		} else {
 			PMD_DRV_LOG(ERR, "IPsec encryption not enabled\n");
+			rte_mempool_put(mempool, (void *)ic_session);
 			return -ENOTSUP;
 		}
 	}
@@ -395,6 +398,7 @@ ixgbe_crypto_create_session(void *device,
 	if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) {
 		if (ixgbe_crypto_add_sa(ic_session)) {
 			PMD_DRV_LOG(ERR, "Failed to add SA\n");
+			rte_mempool_put(mempool, (void *)ic_session);
 			return -EPERM;
 		}
 	}
@@ -609,7 +613,7 @@ ixgbe_crypto_enable_ipsec(struct rte_eth_dev *dev)
 		PMD_DRV_LOG(ERR, "RSC and IPsec not supported");
 		return -1;
 	}
-	if (rte_eth_dev_must_keep_crc(rx_offloads)) {
+	if (rx_offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
 		PMD_DRV_LOG(ERR, "HW CRC strip needs to be enabled for IPsec");
 		return -1;
 	}
diff --git a/drivers/net/ixgbe/ixgbe_rxtx.c b/drivers/net/ixgbe/ixgbe_rxtx.c
index f82b74a9..2f0262ae 100644
--- a/drivers/net/ixgbe/ixgbe_rxtx.c
+++ b/drivers/net/ixgbe/ixgbe_rxtx.c
@@ -58,6 +58,10 @@
 #endif
 /* Bit Mask to indicate what bits required for building TX context */
 #define IXGBE_TX_OFFLOAD_MASK (			 \
+		PKT_TX_OUTER_IPV6 |		 \
+		PKT_TX_OUTER_IPV4 |		 \
+		PKT_TX_IPV6 |			 \
+		PKT_TX_IPV4 |			 \
 		PKT_TX_VLAN_PKT |		 \
 		PKT_TX_IP_CKSUM |		 \
 		PKT_TX_L4_MASK |		 \
@@ -2057,8 +2061,7 @@ next_desc:
 		 * of the ixgbe PMD.
 		 *
 		 * TODO:
-		 *    - Get rid of "volatile" crap and let the compiler do its
-		 *      job.
+		 *    - Get rid of "volatile" and let the compiler do its job.
 		 *    - Use the proper memory barrier (rte_rmb()) to ensure the
 		 *      memory ordering below.
 		 */
@@ -2848,7 +2851,6 @@ ixgbe_get_rx_port_offloads(struct rte_eth_dev *dev)
 	offloads = DEV_RX_OFFLOAD_IPV4_CKSUM  |
 		   DEV_RX_OFFLOAD_UDP_CKSUM   |
 		   DEV_RX_OFFLOAD_TCP_CKSUM   |
-		   DEV_RX_OFFLOAD_CRC_STRIP   |
 		   DEV_RX_OFFLOAD_KEEP_CRC    |
 		   DEV_RX_OFFLOAD_JUMBO_FRAME |
 		   DEV_RX_OFFLOAD_SCATTER;
@@ -2936,7 +2938,7 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
 		queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
 	rxq->port_id = dev->data->port_id;
-	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
 		rxq->crc_len = ETHER_CRC_LEN;
 	else
 		rxq->crc_len = 0;
@@ -4705,7 +4707,7 @@ ixgbe_set_rsc(struct rte_eth_dev *dev)
 
 	/* RSC global configuration (chapter 4.6.7.2.1 of 82599 Spec) */
 
-	if (rte_eth_dev_must_keep_crc(rx_conf->offloads) &&
+	if ((rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) &&
 	     (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
 		/*
 		 * According to chapter of 4.6.7.2.1 of the Spec Rev.
@@ -4854,7 +4856,7 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
 	 * Configure CRC stripping, if any.
 	 */
 	hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
-	if (rte_eth_dev_must_keep_crc(rx_conf->offloads))
+	if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
 		hlreg0 &= ~IXGBE_HLREG0_RXCRCSTRP;
 	else
 		hlreg0 |= IXGBE_HLREG0_RXCRCSTRP;
@@ -4895,8 +4897,10 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
 		 * Reset crc_len in case it was changed after queue setup by a
 		 * call to configure.
 		 */
-		rxq->crc_len = rte_eth_dev_must_keep_crc(rx_conf->offloads) ?
-				ETHER_CRC_LEN : 0;
+		if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
+			rxq->crc_len = ETHER_CRC_LEN;
+		else
+			rxq->crc_len = 0;
 
 		/* Setup the Base and Length of the Rx Descriptor Rings */
 		bus_addr = rxq->rx_ring_phys_addr;
@@ -4965,7 +4969,7 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
 	if (hw->mac.type == ixgbe_mac_82599EB ||
 	    hw->mac.type == ixgbe_mac_X540) {
 		rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
-		if (rte_eth_dev_must_keep_crc(rx_conf->offloads))
+		if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
 			rdrxctl &= ~IXGBE_RDRXCTL_CRCSTRIP;
 		else
 			rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
@@ -5702,7 +5706,7 @@ ixgbe_config_rss_filter(struct rte_eth_dev *dev,
 	 */
 	if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
 		ixgbe_rss_disable(dev);
-		return -EINVAL;
+		return 0;
 	}
 	if (rss_conf.rss_key == NULL)
 		rss_conf.rss_key = rss_intel_key; /* Default hash key */
@@ -5715,13 +5719,13 @@ ixgbe_config_rss_filter(struct rte_eth_dev *dev,
 }
 
 /* Stubs needed for linkage when CONFIG_RTE_IXGBE_INC_VECTOR is set to 'n' */
-int __attribute__((weak))
+__rte_weak int
 ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
 {
 	return -1;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 ixgbe_recv_pkts_vec(
 	void __rte_unused *rx_queue,
 	struct rte_mbuf __rte_unused **rx_pkts,
@@ -5730,7 +5734,7 @@ ixgbe_recv_pkts_vec(
 	return 0;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 ixgbe_recv_scattered_pkts_vec(
 	void __rte_unused *rx_queue,
 	struct rte_mbuf __rte_unused **rx_pkts,
@@ -5739,7 +5743,7 @@ ixgbe_recv_scattered_pkts_vec(
 	return 0;
 }
 
-int __attribute__((weak))
+__rte_weak int
 ixgbe_rxq_vec_setup(struct ixgbe_rx_queue __rte_unused *rxq)
 {
 	return -1;
diff --git a/drivers/net/ixgbe/ixgbe_vf_representor.c b/drivers/net/ixgbe/ixgbe_vf_representor.c
index db516d99..5d2e3e02 100644
--- a/drivers/net/ixgbe/ixgbe_vf_representor.c
+++ b/drivers/net/ixgbe/ixgbe_vf_representor.c
@@ -65,7 +65,7 @@ ixgbe_vf_representor_dev_infos_get(struct rte_eth_dev *ethdev,
 	dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
 		DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM |
 		DEV_TX_OFFLOAD_TCP_CKSUM | DEV_TX_OFFLOAD_SCTP_CKSUM |
-		DEV_TX_OFFLOAD_TCP_TSO;
+		DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_MULTI_SEGS;
 	/**< Device TX offload capabilities. */
 
 	dev_info->speed_capa =
@@ -135,7 +135,7 @@ ixgbe_vf_representor_vlan_strip_queue_set(struct rte_eth_dev *ethdev,
 		representor->vf_id, on);
 }
 
-struct eth_dev_ops ixgbe_vf_representor_dev_ops = {
+static const struct eth_dev_ops ixgbe_vf_representor_dev_ops = {
 	.dev_infos_get		= ixgbe_vf_representor_dev_infos_get,
 
 	.dev_start		= ixgbe_vf_representor_dev_start,
@@ -192,6 +192,7 @@ ixgbe_vf_representor_init(struct rte_eth_dev *ethdev, void *init_params)
 		return -ENODEV;
 
 	ethdev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
+	ethdev->data->representor_id = representor->vf_id;
 
 	/* Set representor device ops */
 	ethdev->dev_ops = &ixgbe_vf_representor_dev_ops;
@@ -225,7 +226,10 @@ ixgbe_vf_representor_init(struct rte_eth_dev *ethdev, void *init_params)
 }
 
 int
-ixgbe_vf_representor_uninit(struct rte_eth_dev *ethdev __rte_unused)
+ixgbe_vf_representor_uninit(struct rte_eth_dev *ethdev)
 {
+	/* mac_addrs must not be freed because part of ixgbe_vf_info */
+	ethdev->data->mac_addrs = NULL;
+
 	return 0;
 }
diff --git a/drivers/net/ixgbe/meson.build b/drivers/net/ixgbe/meson.build
index 02d5ef5e..544a1414 100644
--- a/drivers/net/ixgbe/meson.build
+++ b/drivers/net/ixgbe/meson.build
@@ -5,10 +5,11 @@ version = 2
 
 cflags += ['-DRTE_LIBRTE_IXGBE_BYPASS']
 
+allow_experimental_apis = true
+
 subdir('base')
 objs = [base_objs]
 
-allow_experimental_apis = true
 sources = files(
 	'ixgbe_82599_bypass.c',
 	'ixgbe_bypass.c',
diff --git a/drivers/net/kni/rte_eth_kni.c b/drivers/net/kni/rte_eth_kni.c
index 085bb845..a1e9970d 100644
--- a/drivers/net/kni/rte_eth_kni.c
+++ b/drivers/net/kni/rte_eth_kni.c
@@ -207,7 +207,6 @@ eth_kni_dev_info(struct rte_eth_dev *dev __rte_unused,
 	dev_info->max_rx_queues = KNI_MAX_QUEUE_PER_PORT;
 	dev_info->max_tx_queues = KNI_MAX_QUEUE_PER_PORT;
 	dev_info->min_rx_bufsize = 0;
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -411,8 +410,7 @@ eth_kni_probe(struct rte_vdev_device *vdev)
 	params = rte_vdev_device_args(vdev);
 	PMD_LOG(INFO, "Initializing eth_kni for %s", name);
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
-	    strlen(params) == 0) {
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 		eth_dev = rte_eth_dev_attach_secondary(name);
 		if (!eth_dev) {
 			PMD_LOG(ERR, "Failed to probe %s", name);
@@ -465,13 +463,17 @@ eth_kni_remove(struct rte_vdev_device *vdev)
 	if (eth_dev == NULL)
 		return -1;
 
+	/* mac_addrs must not be freed alone because part of dev_private */
+	eth_dev->data->mac_addrs = NULL;
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return rte_eth_dev_release_port(eth_dev);
+
 	eth_kni_dev_stop(eth_dev);
 
 	internals = eth_dev->data->dev_private;
 	rte_kni_release(internals->kni);
 
-	rte_free(internals);
-
 	rte_eth_dev_release_port(eth_dev);
 
 	is_kni_initialized--;
diff --git a/drivers/net/liquidio/lio_ethdev.c b/drivers/net/liquidio/lio_ethdev.c
index 93e89007..d13ab06c 100644
--- a/drivers/net/liquidio/lio_ethdev.c
+++ b/drivers/net/liquidio/lio_ethdev.c
@@ -2038,14 +2038,11 @@ lio_eth_dev_uninit(struct rte_eth_dev *eth_dev)
 	PMD_INIT_FUNC_TRACE();
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
-		return -EPERM;
+		return 0;
 
 	/* lio_free_sc_buffer_pool */
 	lio_free_sc_buffer_pool(lio_dev);
 
-	rte_free(eth_dev->data->mac_addrs);
-	eth_dev->data->mac_addrs = NULL;
-
 	eth_dev->dev_ops = NULL;
 	eth_dev->rx_pkt_burst = NULL;
 	eth_dev->tx_pkt_burst = NULL;
diff --git a/drivers/net/meson.build b/drivers/net/meson.build
index 9c28ed4d..980eec23 100644
--- a/drivers/net/meson.build
+++ b/drivers/net/meson.build
@@ -3,6 +3,8 @@
 
 drivers = ['af_packet',
 	'ark',
+	'atlantic',
+	'avf',
 	'avp',
 	'axgbe', 'bonding',
 	'bnx2x',
@@ -11,6 +13,7 @@ drivers = ['af_packet',
 	'dpaa', 'dpaa2',
 	'e1000',
 	'ena',
+	'enetc',
 	'enic',
 	'failsafe',
 	'fm10k', 'i40e',
@@ -18,16 +21,23 @@ drivers = ['af_packet',
 	'ixgbe',
 	'kni',
 	'liquidio',
+	'mlx4',
+	'mlx5',
+	'mvneta',
 	'mvpp2',
 	'netvsc',
 	'nfp',
-	'null', 'octeontx', 'pcap', 'ring',
+	'null', 'octeontx', 'pcap', 'qede', 'ring',
 	'sfc',
 	'softnic',
 	'szedata2',
+	'tap',
 	'thunderx',
+	'vdev_netvsc',
 	'vhost',
-	'virtio']
+	'virtio',
+	'vmxnet3',
+]
 std_deps = ['ethdev', 'kvargs'] # 'ethdev' also pulls in mbuf, net, eal etc
 std_deps += ['bus_pci']         # very many PMDs depend on PCI, so make std
 std_deps += ['bus_vdev']        # same with vdev bus
diff --git a/drivers/net/mlx4/meson.build b/drivers/net/mlx4/meson.build
new file mode 100644
index 00000000..7de571e2
--- /dev/null
+++ b/drivers/net/mlx4/meson.build
@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2018 6WIND S.A.
+# Copyright 2018 Mellanox Technologies, Ltd
+
+pmd_dlopen = get_option('enable_driver_mlx_glue')
+LIB_GLUE_BASE = 'librte_pmd_mlx4_glue.so'
+LIB_GLUE_VERSION = '18.02.0'
+LIB_GLUE = LIB_GLUE_BASE + '.' + LIB_GLUE_VERSION
+if pmd_dlopen
+	dpdk_conf.set('RTE_LIBRTE_MLX4_DLOPEN_DEPS', 1)
+	cflags += [
+		'-DMLX4_GLUE="@0@"'.format(LIB_GLUE),
+		'-DMLX4_GLUE_VERSION="@0@"'.format(LIB_GLUE_VERSION),
+	]
+endif
+libs = [
+	cc.find_library('mnl', required:false),
+	cc.find_library('mlx4', required:false),
+	cc.find_library('ibverbs', required:false),
+]
+build = true
+foreach lib:libs
+	if not lib.found()
+		build = false
+	endif
+endforeach
+# Compile PMD
+if build
+	allow_experimental_apis = true
+	ext_deps += libs
+	sources = files(
+		'mlx4.c',
+		'mlx4_ethdev.c',
+		'mlx4_flow.c',
+		'mlx4_intr.c',
+		'mlx4_mr.c',
+		'mlx4_rxq.c',
+		'mlx4_rxtx.c',
+		'mlx4_txq.c',
+		'mlx4_utils.c',
+	)
+	if not pmd_dlopen
+		sources += files('mlx4_glue.c')
+	endif
+	cflags_options = [
+		'-Wextra',
+		'-std=c11',
+		'-Wno-strict-prototypes',
+		'-D_BSD_SOURCE',
+		'-D_DEFAULT_SOURCE',
+		'-D_XOPEN_SOURCE=600'
+	]
+	foreach option:cflags_options
+		if cc.has_argument(option)
+			cflags += option
+		endif
+	endforeach
+	if get_option('buildtype').contains('debug')
+		cflags += [ '-pedantic', '-UNDEBUG', '-DPEDANTIC' ]
+	else
+		cflags += [ '-DNDEBUG', '-UPEDANTIC' ]
+	endif
+	# To maintain the compatibility with the make build system
+	# mlx4_autoconf.h file is still generated.
+	# input array for meson member search:
+	# [ "MACRO to define if found", "header for the search",
+	#   "symbol to search","struct member to search" ]
+	#
+	has_member_args = [
+		[ 'HAVE_IBV_MLX4_WQE_LSO_SEG', 'infiniband/mlx4dv.h',
+		'struct mlx4_wqe_lso_seg', 'mss_hdr_size' ],
+	]
+	config = configuration_data()
+	foreach arg:has_member_args
+		file_prefix = '#include<' + arg[1] + '>'
+		config.set(arg[0], cc.has_member(arg[2], arg[3],
+			prefix : file_prefix))
+	endforeach
+	configure_file(output : 'mlx4_autoconf.h', configuration : config)
+endif
+# Build Glue Library
+if pmd_dlopen and build
+	dlopen_name = 'mlx4_glue'
+	dlopen_lib_name = driver_name_fmt.format(dlopen_name)
+	dlopen_so_version = LIB_GLUE_VERSION
+	dlopen_sources = files('mlx4_glue.c')
+	dlopen_install_dir = [ eal_pmd_path + '-glue' ]
+	shared_lib = shared_library(
+		dlopen_lib_name,
+		dlopen_sources,
+		include_directories: global_inc,
+		c_args: cflags,
+		dependencies: libs,
+		link_args: [
+		'-Wl,-export-dynamic',
+		'-Wl,-h,@0@'.format(LIB_GLUE),
+		],
+		soversion: dlopen_so_version,
+		install: true,
+		install_dir: dlopen_install_dir,
+	)
+endif
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index defc0d4b..7f07b8dc 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -734,7 +734,6 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		eth_dev->data->mac_addrs = priv->mac;
 		eth_dev->device = &pci_dev->device;
 		rte_eth_copy_pci_info(eth_dev, pci_dev);
-		eth_dev->device->driver = &mlx4_driver.driver;
 		/* Initialize local interrupt handle for current port. */
 		priv->intr_handle = (struct rte_intr_handle){
 			.fd = -1,
@@ -782,12 +781,17 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		continue;
 port_error:
 		rte_free(priv);
+		if (eth_dev != NULL)
+			eth_dev->data->dev_private = NULL;
 		if (pd)
 			claim_zero(mlx4_glue->dealloc_pd(pd));
 		if (ctx)
 			claim_zero(mlx4_glue->close_device(ctx));
-		if (eth_dev)
+		if (eth_dev != NULL) {
+			/* mac_addrs must not be freed because part of dev_private */
+			eth_dev->data->mac_addrs = NULL;
 			rte_eth_dev_release_port(eth_dev);
+		}
 		break;
 	}
 	/*
diff --git a/drivers/net/mlx4/mlx4_mr.c b/drivers/net/mlx4/mlx4_mr.c
index d23d3c61..bee85864 100644
--- a/drivers/net/mlx4/mlx4_mr.c
+++ b/drivers/net/mlx4/mlx4_mr.c
@@ -289,6 +289,23 @@ mr_find_next_chunk(struct mlx4_mr *mr, struct mlx4_mr_cache *entry,
 	uintptr_t end = 0;
 	uint32_t idx = 0;
 
+	/* MR for external memory doesn't have memseg list. */
+	if (mr->msl == NULL) {
+		struct ibv_mr *ibv_mr = mr->ibv_mr;
+
+		assert(mr->ms_bmp_n == 1);
+		assert(mr->ms_n == 1);
+		assert(base_idx == 0);
+		/*
+		 * Can't search it from memseg list but get it directly from
+		 * verbs MR as there's only one chunk.
+		 */
+		entry->start = (uintptr_t)ibv_mr->addr;
+		entry->end = (uintptr_t)ibv_mr->addr + mr->ibv_mr->length;
+		entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
+		/* Returning 1 ends iteration. */
+		return 1;
+	}
 	for (idx = base_idx; idx < mr->ms_bmp_n; ++idx) {
 		if (rte_bitmap_get(mr->ms_bmp, idx)) {
 			const struct rte_memseg_list *msl;
@@ -809,6 +826,7 @@ mlx4_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
 		mr = mr_lookup_dev_list(dev, &entry, start);
 		if (mr == NULL)
 			continue;
+		assert(mr->msl); /* Can't be external memory. */
 		ms = rte_mem_virt2memseg((void *)start, msl);
 		assert(ms != NULL);
 		assert(msl->page_sz == ms->hugepage_sz);
@@ -1055,6 +1073,133 @@ mlx4_mr_flush_local_cache(struct mlx4_mr_ctrl *mr_ctrl)
 	      (void *)mr_ctrl, mr_ctrl->cur_gen);
 }
 
+/**
+ * Called during rte_mempool_mem_iter() by mlx4_mr_update_ext_mp().
+ *
+ * Externally allocated chunk is registered and a MR is created for the chunk.
+ * The MR object is added to the global list. If memseg list of a MR object
+ * (mr->msl) is null, the MR object can be regarded as externally allocated
+ * memory.
+ *
+ * Once external memory is registered, it should be static. If the memory is
+ * freed and the virtual address range has different physical memory mapped
+ * again, it may cause crash on device due to the wrong translation entry. PMD
+ * can't track the free event of the external memory for now.
+ */
+static void
+mlx4_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque,
+			 struct rte_mempool_memhdr *memhdr,
+			 unsigned mem_idx __rte_unused)
+{
+	struct mr_update_mp_data *data = opaque;
+	struct rte_eth_dev *dev = data->dev;
+	struct priv *priv = dev->data->dev_private;
+	struct mlx4_mr_ctrl *mr_ctrl = data->mr_ctrl;
+	struct mlx4_mr *mr = NULL;
+	uintptr_t addr = (uintptr_t)memhdr->addr;
+	size_t len = memhdr->len;
+	struct mlx4_mr_cache entry;
+	uint32_t lkey;
+
+	/* If already registered, it should return. */
+	rte_rwlock_read_lock(&priv->mr.rwlock);
+	lkey = mr_lookup_dev(dev, &entry, addr);
+	rte_rwlock_read_unlock(&priv->mr.rwlock);
+	if (lkey != UINT32_MAX)
+		return;
+	mr = rte_zmalloc_socket(NULL,
+				RTE_ALIGN_CEIL(sizeof(*mr),
+					       RTE_CACHE_LINE_SIZE),
+				RTE_CACHE_LINE_SIZE, mp->socket_id);
+	if (mr == NULL) {
+		WARN("port %u unable to allocate memory for a new MR of"
+		     " mempool (%s).",
+		     dev->data->port_id, mp->name);
+		data->ret = -1;
+		return;
+	}
+	DEBUG("port %u register MR for chunk #%d of mempool (%s)",
+	      dev->data->port_id, mem_idx, mp->name);
+	mr->ibv_mr = mlx4_glue->reg_mr(priv->pd, (void *)addr, len,
+				       IBV_ACCESS_LOCAL_WRITE);
+	if (mr->ibv_mr == NULL) {
+		WARN("port %u fail to create a verbs MR for address (%p)",
+		     dev->data->port_id, (void *)addr);
+		rte_free(mr);
+		data->ret = -1;
+		return;
+	}
+	mr->msl = NULL; /* Mark it is external memory. */
+	mr->ms_bmp = NULL;
+	mr->ms_n = 1;
+	mr->ms_bmp_n = 1;
+	rte_rwlock_write_lock(&priv->mr.rwlock);
+	LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
+	DEBUG("port %u MR CREATED (%p) for external memory %p:\n"
+	      "  [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
+	      " lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
+	      dev->data->port_id, (void *)mr, (void *)addr,
+	      addr, addr + len, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+	      mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
+	/* Insert to the global cache table. */
+	mr_insert_dev_cache(dev, mr);
+	rte_rwlock_write_unlock(&priv->mr.rwlock);
+	/* Insert to the local cache table */
+	mlx4_mr_addr2mr_bh(dev, mr_ctrl, addr);
+}
+
+/**
+ * Register MR for entire memory chunks in a Mempool having externally allocated
+ * memory and fill in local cache.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param mr_ctrl
+ *   Pointer to per-queue MR control structure.
+ * @param mp
+ *   Pointer to registering Mempool.
+ *
+ * @return
+ *   0 on success, -1 on failure.
+ */
+static uint32_t
+mlx4_mr_update_ext_mp(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
+		      struct rte_mempool *mp)
+{
+	struct mr_update_mp_data data = {
+		.dev = dev,
+		.mr_ctrl = mr_ctrl,
+		.ret = 0,
+	};
+
+	rte_mempool_mem_iter(mp, mlx4_mr_update_ext_mp_cb, &data);
+	return data.ret;
+}
+
+/**
+ * Register MR entire memory chunks in a Mempool having externally allocated
+ * memory and search LKey of the address to return.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param addr
+ *   Search key.
+ * @param mp
+ *   Pointer to registering Mempool where addr belongs.
+ *
+ * @return
+ *   LKey for address on success, UINT32_MAX on failure.
+ */
+uint32_t
+mlx4_tx_update_ext_mp(struct txq *txq, uintptr_t addr, struct rte_mempool *mp)
+{
+	struct mlx4_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+	struct priv *priv = txq->priv;
+
+	mlx4_mr_update_ext_mp(priv->dev, mr_ctrl, mp);
+	return mlx4_tx_addr2mr_bh(txq, addr);
+}
+
 /* Called during rte_mempool_mem_iter() by mlx4_mr_update_mp(). */
 static void
 mlx4_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void *opaque,
@@ -1098,6 +1243,10 @@ mlx4_mr_update_mp(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
 	};
 
 	rte_mempool_mem_iter(mp, mlx4_mr_update_mp_cb, &data);
+	if (data.ret < 0 && rte_errno == ENXIO) {
+		/* Mempool may have externally allocated memory. */
+		return mlx4_mr_update_ext_mp(dev, mr_ctrl, mp);
+	}
 	return data.ret;
 }
 
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 9737da2e..6804c634 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -678,7 +678,6 @@ uint64_t
 mlx4_get_rx_queue_offloads(struct priv *priv)
 {
 	uint64_t offloads = DEV_RX_OFFLOAD_SCATTER |
-			    DEV_RX_OFFLOAD_CRC_STRIP |
 			    DEV_RX_OFFLOAD_KEEP_CRC |
 			    DEV_RX_OFFLOAD_JUMBO_FRAME;
 
@@ -780,7 +779,7 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	}
 	/* By default, FCS (CRC) is stripped by hardware. */
 	crc_present = 0;
-	if (rte_eth_dev_must_keep_crc(offloads)) {
+	if (offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
 		if (priv->hw_fcs_strip) {
 			crc_present = 1;
 		} else {
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index ffa8abfc..1be060cd 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -163,6 +163,26 @@ void mlx4_tx_queue_release(void *dpdk_txq);
 void mlx4_mr_flush_local_cache(struct mlx4_mr_ctrl *mr_ctrl);
 uint32_t mlx4_rx_addr2mr_bh(struct rxq *rxq, uintptr_t addr);
 uint32_t mlx4_tx_addr2mr_bh(struct txq *txq, uintptr_t addr);
+uint32_t mlx4_tx_update_ext_mp(struct txq *txq, uintptr_t addr,
+			       struct rte_mempool *mp);
+
+/**
+ * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the
+ * cloned mbuf is allocated is returned instead.
+ *
+ * @param buf
+ *   Pointer to mbuf.
+ *
+ * @return
+ *   Memory pool where data is located for given mbuf.
+ */
+static struct rte_mempool *
+mlx4_mb2mp(struct rte_mbuf *buf)
+{
+	if (unlikely(RTE_MBUF_INDIRECT(buf)))
+		return rte_mbuf_from_indirect(buf)->pool;
+	return buf->pool;
+}
 
 /**
  * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx
@@ -222,6 +242,19 @@ mlx4_tx_addr2mr(struct txq *txq, uintptr_t addr)
 	return mlx4_tx_addr2mr_bh(txq, addr);
 }
 
-#define mlx4_tx_mb2mr(rxq, mb) mlx4_tx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+static __rte_always_inline uint32_t
+mlx4_tx_mb2mr(struct txq *txq, struct rte_mbuf *mb)
+{
+	uintptr_t addr = (uintptr_t)mb->buf_addr;
+	uint32_t lkey = mlx4_tx_addr2mr(txq, addr);
+
+	if (likely(lkey != UINT32_MAX))
+		return lkey;
+	if (rte_errno == ENXIO) {
+		/* Mempool may have externally allocated memory. */
+		lkey = mlx4_tx_update_ext_mp(txq, addr, mlx4_mb2mp(mb));
+	}
+	return lkey;
+}
 
 #endif /* MLX4_RXTX_H_ */
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 2e70dec5..fecb57c1 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -8,7 +8,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 LIB = librte_pmd_mlx5.a
 LIB_GLUE = $(LIB_GLUE_BASE).$(LIB_GLUE_VERSION)
 LIB_GLUE_BASE = librte_pmd_mlx5_glue.so
-LIB_GLUE_VERSION = 18.05.0
+LIB_GLUE_VERSION = 18.11.0
 
 # Sources.
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c
@@ -31,9 +31,11 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_stats.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_dv.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_tcf.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_verbs.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_nl.c
-SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_nl_flow.c
 
 ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
 INSTALL-$(CONFIG_RTE_LIBRTE_MLX5_PMD)-lib += $(LIB_GLUE)
@@ -135,6 +137,11 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
+		HAVE_IBV_FLOW_DV_SUPPORT \
+		infiniband/mlx5dv.h \
+		enum MLX5DV_FLOW_ACTION_TAG \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
 		HAVE_ETHTOOL_LINK_MODE_25G \
 		/usr/include/linux/ethtool.h \
 		enum ETHTOOL_LINK_MODE_25000baseCR_Full_BIT \
@@ -150,11 +157,16 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		enum ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
-		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
+		HAVE_IBV_DEVICE_COUNTERS_SET_V42 \
 		infiniband/verbs.h \
 		type 'struct ibv_counter_set_init_attr' \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_COUNTERS_SET_V45 \
+		infiniband/verbs.h \
+		type 'struct ibv_counters_init_attr' \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
 		HAVE_RDMA_NL_NLDEV \
 		rdma/rdma_netlink.h \
 		enum RDMA_NL_NLDEV \
@@ -200,6 +212,11 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		enum IFLA_PHYS_PORT_NAME \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
+		HAVE_TCA_CHAIN \
+		linux/rtnetlink.h \
+		enum TCA_CHAIN \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
 		HAVE_TCA_FLOWER_ACT \
 		linux/pkt_cls.h \
 		enum TCA_FLOWER_ACT \
@@ -335,11 +352,31 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		enum TCA_FLOWER_KEY_VLAN_ETH_TYPE \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_TCP_FLAGS \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_TCP_FLAGS \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_TCP_FLAGS_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TC_ACT_GOTO_CHAIN \
+		linux/pkt_cls.h \
+		define TC_ACT_GOTO_CHAIN \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
 		HAVE_TC_ACT_VLAN \
 		linux/tc_act/tc_vlan.h \
 		enum TCA_VLAN_PUSH_VLAN_PRIORITY \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
+		HAVE_TC_ACT_PEDIT \
+		linux/tc_act/tc_pedit.h \
+		enum TCA_PEDIT_KEY_EX_HDR_TYPE_UDP \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
 		HAVE_SUPPORTED_40000baseKR4_Full \
 		/usr/include/linux/ethtool.h \
 		define SUPPORTED_40000baseKR4_Full \
diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
new file mode 100644
index 00000000..e8cbe3ee
--- /dev/null
+++ b/drivers/net/mlx5/meson.build
@@ -0,0 +1,244 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2018 6WIND S.A.
+# Copyright 2018 Mellanox Technologies, Ltd
+
+pmd_dlopen = get_option('enable_driver_mlx_glue')
+LIB_GLUE_BASE = 'librte_pmd_mlx5_glue.so'
+LIB_GLUE_VERSION = '18.11.0'
+LIB_GLUE = LIB_GLUE_BASE + '.' + LIB_GLUE_VERSION
+if pmd_dlopen
+	dpdk_conf.set('RTE_LIBRTE_MLX5_DLOPEN_DEPS', 1)
+	cflags += [
+		'-DMLX5_GLUE="@0@"'.format(LIB_GLUE),
+		'-DMLX5_GLUE_VERSION="@0@"'.format(LIB_GLUE_VERSION),
+	]
+endif
+libs = [
+	cc.find_library('mnl', required:false),
+	cc.find_library('mlx5', required:false),
+	cc.find_library('ibverbs', required:false),
+]
+build = true
+foreach lib:libs
+	if not lib.found()
+		build = false
+	endif
+endforeach
+if build
+	allow_experimental_apis = true
+	ext_deps += libs
+	sources = files(
+		'mlx5.c',
+		'mlx5_ethdev.c',
+		'mlx5_flow.c',
+		'mlx5_flow_dv.c',
+		'mlx5_flow_tcf.c',
+		'mlx5_flow_verbs.c',
+		'mlx5_mac.c',
+		'mlx5_mr.c',
+		'mlx5_nl.c',
+		'mlx5_rss.c',
+		'mlx5_rxmode.c',
+		'mlx5_rxq.c',
+		'mlx5_rxtx.c',
+		'mlx5_socket.c',
+		'mlx5_stats.c',
+		'mlx5_trigger.c',
+		'mlx5_txq.c',
+		'mlx5_vlan.c',
+	)
+	if dpdk_conf.has('RTE_ARCH_X86_64') or dpdk_conf.has('RTE_ARCH_ARM64')
+		sources += files('mlx5_rxtx_vec.c')
+	endif
+	if not pmd_dlopen
+		sources += files('mlx5_glue.c')
+	endif
+	cflags_options = [
+		'-Wextra',
+		'-std=c11',
+		'-Wno-strict-prototypes',
+		'-D_BSD_SOURCE',
+		'-D_DEFAULT_SOURCE',
+		'-D_XOPEN_SOURCE=600'
+	]
+	foreach option:cflags_options
+		if cc.has_argument(option)
+			cflags += option
+		endif
+	endforeach
+	if get_option('buildtype').contains('debug')
+		cflags += [ '-pedantic', '-UNDEBUG', '-DPEDANTIC' ]
+	else
+		cflags += [ '-DNDEBUG', '-UPEDANTIC' ]
+	endif
+	# To maintain the compatibility with the make build system
+	# mlx5_autoconf.h file is still generated.
+	# input array for meson member search:
+	# [ "MACRO to define if found", "header for the search",
+	#   "symbol to search", "struct member to search" ]
+	has_member_args = [
+		[ 'HAVE_IBV_MLX5_MOD_SWP', 'infiniband/mlx5dv.h',
+		'struct mlx5dv_sw_parsing_caps', 'sw_parsing_offloads' ],
+		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V42', 'infiniband/verbs.h',
+		'struct ibv_counter_set_init_attr', 'counter_set_id' ],
+		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V45', 'infiniband/verbs.h',
+		'struct ibv_counters_init_attr', 'comp_mask' ],
+	]
+	# input array for meson symbol search:
+	# [ "MACRO to define if found", "header for the search",
+	#   "symbol to search" ]
+	has_sym_args = [
+		[ 'HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT', 'infiniband/mlx5dv.h',
+		'MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX' ],
+		[ 'HAVE_IBV_DEVICE_TUNNEL_SUPPORT', 'infiniband/mlx5dv.h',
+		'MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS' ],
+		[ 'HAVE_IBV_MLX5_MOD_MPW', 'infiniband/mlx5dv.h',
+		'MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED' ],
+		[ 'HAVE_IBV_MLX5_MOD_CQE_128B_COMP', 'infiniband/mlx5dv.h',
+		'MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP' ],
+		[ 'HAVE_IBV_FLOW_DV_SUPPORT', 'infiniband/mlx5dv.h',
+		'MLX5DV_FLOW_ACTION_TAG' ],
+		[ 'HAVE_IBV_DEVICE_MPLS_SUPPORT', 'infiniband/verbs.h',
+		'IBV_FLOW_SPEC_MPLS' ],
+		[ 'HAVE_IBV_WQ_FLAG_RX_END_PADDING', 'infiniband/verbs.h',
+		'IBV_WQ_FLAG_RX_END_PADDING' ],
+		[ 'HAVE_SUPPORTED_40000baseKR4_Full', 'linux/ethtool.h',
+		'SUPPORTED_40000baseKR4_Full' ],
+		[ 'HAVE_SUPPORTED_40000baseCR4_Full', 'linux/ethtool.h',
+		'SUPPORTED_40000baseCR4_Full' ],
+		[ 'HAVE_SUPPORTED_40000baseSR4_Full', 'linux/ethtool.h',
+		'SUPPORTED_40000baseSR4_Full' ],
+		[ 'HAVE_SUPPORTED_40000baseLR4_Full', 'linux/ethtool.h',
+		'SUPPORTED_40000baseLR4_Full' ],
+		[ 'HAVE_SUPPORTED_56000baseKR4_Full', 'linux/ethtool.h',
+		'SUPPORTED_56000baseKR4_Full' ],
+		[ 'HAVE_SUPPORTED_56000baseCR4_Full', 'linux/ethtool.h',
+		'SUPPORTED_56000baseCR4_Full' ],
+		[ 'HAVE_SUPPORTED_56000baseSR4_Full', 'linux/ethtool.h',
+		'SUPPORTED_56000baseSR4_Full' ],
+		[ 'HAVE_SUPPORTED_56000baseLR4_Full', 'linux/ethtool.h',
+		'SUPPORTED_56000baseLR4_Full' ],
+		[ 'HAVE_ETHTOOL_LINK_MODE_25G', 'linux/ethtool.h',
+		'ETHTOOL_LINK_MODE_25000baseCR_Full_BIT' ],
+		[ 'HAVE_ETHTOOL_LINK_MODE_50G', 'linux/ethtool.h',
+		'ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT' ],
+		[ 'HAVE_ETHTOOL_LINK_MODE_100G', 'linux/ethtool.h',
+		'ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT' ],
+		[ 'HAVE_IFLA_PHYS_SWITCH_ID', 'linux/if_link.h',
+		'IFLA_PHYS_SWITCH_ID' ],
+		[ 'HAVE_IFLA_PHYS_PORT_NAME', 'linux/if_link.h',
+		'IFLA_PHYS_PORT_NAME' ],
+		[ 'HAVE_TCA_CHAIN', 'linux/rtnetlink.h',
+		'TCA_CHAIN' ],
+		[ 'HAVE_TCA_FLOWER_ACT', 'linux/pkt_cls.h',
+		'TCA_FLOWER_ACT' ],
+		[ 'HAVE_TCA_FLOWER_FLAGS', 'linux/pkt_cls.h',
+		'TCA_FLOWER_FLAGS' ],
+		[ 'HAVE_TCA_FLOWER_KEY_ETH_TYPE', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_ETH_TYPE' ],
+		[ 'HAVE_TCA_FLOWER_KEY_ETH_DST', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_ETH_DST' ],
+		[ 'HAVE_TCA_FLOWER_KEY_ETH_DST_MASK', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_ETH_DST_MASK' ],
+		[ 'HAVE_TCA_FLOWER_KEY_ETH_SRC', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_ETH_SRC' ],
+		[ 'HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_ETH_SRC_MASK' ],
+		[ 'HAVE_TCA_FLOWER_KEY_IP_PROTO', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_IP_PROTO' ],
+		[ 'HAVE_TCA_FLOWER_KEY_IPV4_SRC', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_IPV4_SRC' ],
+		[ 'HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_IPV4_SRC_MASK' ],
+		[ 'HAVE_TCA_FLOWER_KEY_IPV4_DST', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_IPV4_DST' ],
+		[ 'HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_IPV4_DST_MASK' ],
+		[ 'HAVE_TCA_FLOWER_KEY_IPV6_SRC', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_IPV6_SRC' ],
+		[ 'HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_IPV6_SRC_MASK' ],
+		[ 'HAVE_TCA_FLOWER_KEY_IPV6_DST', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_IPV6_DST' ],
+		[ 'HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_IPV6_DST_MASK' ],
+		[ 'HAVE_TCA_FLOWER_KEY_TCP_SRC', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_TCP_SRC' ],
+		[ 'HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_TCP_SRC_MASK' ],
+		[ 'HAVE_TCA_FLOWER_KEY_TCP_DST', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_TCP_DST' ],
+		[ 'HAVE_TCA_FLOWER_KEY_TCP_DST_MASK', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_TCP_DST_MASK' ],
+		[ 'HAVE_TCA_FLOWER_KEY_UDP_SRC', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_UDP_SRC' ],
+		[ 'HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_UDP_SRC_MASK' ],
+		[ 'HAVE_TCA_FLOWER_KEY_UDP_DST', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_UDP_DST' ],
+		[ 'HAVE_TCA_FLOWER_KEY_UDP_DST_MASK', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_UDP_DST_MASK' ],
+		[ 'HAVE_TCA_FLOWER_KEY_VLAN_ID', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_VLAN_ID' ],
+		[ 'HAVE_TCA_FLOWER_KEY_VLAN_PRIO', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_VLAN_PRIO' ],
+		[ 'HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_VLAN_ETH_TYPE' ],
+		[ 'HAVE_TCA_FLOWER_KEY_TCP_FLAGS', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_TCP_FLAGS' ],
+		[ 'HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK', 'linux/pkt_cls.h',
+		'TCA_FLOWER_KEY_TCP_FLAGS_MASK' ],
+		[ 'HAVE_TC_ACT_GOTO_CHAIN', 'linux/pkt_cls.h',
+		'TC_ACT_GOTO_CHAIN' ],
+		[ 'HAVE_TC_ACT_VLAN', 'linux/tc_act/tc_vlan.h',
+		'TCA_VLAN_PUSH_VLAN_PRIORITY' ],
+		[ 'HAVE_TC_ACT_PEDIT', 'linux/tc_act/tc_pedit.h',
+		'TCA_PEDIT_KEY_EX_HDR_TYPE_UDP' ],
+		[ 'HAVE_RDMA_NL_NLDEV', 'rdma/rdma_netlink.h',
+		'RDMA_NL_NLDEV' ],
+		[ 'HAVE_RDMA_NLDEV_CMD_GET', 'rdma/rdma_netlink.h',
+		'RDMA_NLDEV_CMD_GET' ],
+		[ 'HAVE_RDMA_NLDEV_CMD_PORT_GET', 'rdma/rdma_netlink.h',
+		'RDMA_NLDEV_CMD_PORT_GET' ],
+		[ 'HAVE_RDMA_NLDEV_ATTR_DEV_INDEX', 'rdma/rdma_netlink.h',
+		'RDMA_NLDEV_ATTR_DEV_INDEX' ],
+		[ 'HAVE_RDMA_NLDEV_ATTR_DEV_NAME', 'rdma/rdma_netlink.h',
+		'RDMA_NLDEV_ATTR_DEV_NAME' ],
+		[ 'HAVE_RDMA_NLDEV_ATTR_PORT_INDEX', 'rdma/rdma_netlink.h',
+		'RDMA_NLDEV_ATTR_PORT_INDEX' ],
+		[ 'HAVE_RDMA_NLDEV_ATTR_NDEV_INDEX', 'rdma/rdma_netlink.h',
+		'RDMA_NLDEV_ATTR_NDEV_INDEX' ],
+	]
+	config = configuration_data()
+	foreach arg:has_sym_args
+		config.set(arg[0], cc.has_header_symbol(arg[1], arg[2]))
+	endforeach
+	foreach arg:has_member_args
+		file_prefix = '#include<' + arg[1] + '>'
+		config.set(arg[0], cc.has_member(arg[2], arg[3],
+			prefix : file_prefix))
+	endforeach
+	configure_file(output : 'mlx5_autoconf.h', configuration : config)
+endif
+# Build Glue Library
+if pmd_dlopen and build
+	dlopen_name = 'mlx5_glue'
+	dlopen_lib_name = driver_name_fmt.format(dlopen_name)
+	dlopen_so_version = LIB_GLUE_VERSION
+	dlopen_sources = files('mlx5_glue.c')
+	dlopen_install_dir = [ eal_pmd_path + '-glue' ]
+	shared_lib = shared_library(
+		dlopen_lib_name,
+		dlopen_sources,
+		include_directories: global_inc,
+		c_args: cflags,
+		dependencies: libs,
+		link_args: [
+		'-Wl,-export-dynamic',
+		'-Wl,-h,@0@'.format(LIB_GLUE),
+		],
+		soversion: dlopen_so_version,
+		install: true,
+		install_dir: dlopen_install_dir,
+	)
+endif
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index ec63bc6e..a277b573 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -46,6 +46,7 @@
 #include "mlx5_defs.h"
 #include "mlx5_glue.h"
 #include "mlx5_mr.h"
+#include "mlx5_flow.h"
 
 /* Device parameter to enable RX completion queue compression. */
 #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en"
@@ -89,6 +90,9 @@
 /* Allow L3 VXLAN flow creation. */
 #define MLX5_L3_VXLAN_EN "l3_vxlan_en"
 
+/* Activate DV flow steering. */
+#define MLX5_DV_FLOW_EN "dv_flow_en"
+
 /* Activate Netlink support in VF mode. */
 #define MLX5_VF_NL_EN "vf_nl_en"
 
@@ -282,8 +286,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 		close(priv->nl_socket_route);
 	if (priv->nl_socket_rdma >= 0)
 		close(priv->nl_socket_rdma);
-	if (priv->mnl_socket)
-		mlx5_nl_flow_socket_destroy(priv->mnl_socket);
+	if (priv->tcf_context)
+		mlx5_flow_tcf_context_destroy(priv->tcf_context);
 	ret = mlx5_hrxq_ibv_verify(dev);
 	if (ret)
 		DRV_LOG(WARNING, "port %u some hash Rx queue still remain",
@@ -333,6 +337,17 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	}
 	memset(priv, 0, sizeof(*priv));
 	priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
+	/*
+	 * flag to rte_eth_dev_close() that it should release the port resources
+	 * (calling rte_eth_dev_release_port()) in addition to closing it.
+	 */
+	dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
+	/*
+	 * Reset mac_addrs to NULL such that it is not freed as part of
+	 * rte_eth_dev_release_port(). mac_addrs is part of dev_private so
+	 * it is freed when dev_private is freed.
+	 */
+	dev->data->mac_addrs = NULL;
 }
 
 const struct eth_dev_ops mlx5_dev_ops = {
@@ -477,7 +492,7 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 	} else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) {
 		config->txqs_inline = tmp;
 	} else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) {
-		config->mps = !!tmp ? config->mps : 0;
+		config->mps = !!tmp;
 	} else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) {
 		config->mpw_hdr_dseg = !!tmp;
 	} else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) {
@@ -490,6 +505,8 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->l3_vxlan_en = !!tmp;
 	} else if (strcmp(MLX5_VF_NL_EN, key) == 0) {
 		config->vf_nl_en = !!tmp;
+	} else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) {
+		config->dv_flow_en = !!tmp;
 	} else {
 		DRV_LOG(WARNING, "%s: unknown parameter", key);
 		rte_errno = EINVAL;
@@ -527,6 +544,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_RX_VEC_EN,
 		MLX5_L3_VXLAN_EN,
 		MLX5_VF_NL_EN,
+		MLX5_DV_FLOW_EN,
 		MLX5_REPRESENTOR,
 		NULL,
 	};
@@ -568,11 +586,13 @@ static struct rte_pci_driver mlx5_driver;
 static void *uar_base;
 
 static int
-find_lower_va_bound(const struct rte_memseg_list *msl __rte_unused,
+find_lower_va_bound(const struct rte_memseg_list *msl,
 		const struct rte_memseg *ms, void *arg)
 {
 	void **addr = arg;
 
+	if (msl->external)
+		return 0;
 	if (*addr == NULL)
 		*addr = ms->addr;
 	else
@@ -685,9 +705,10 @@ mlx5_uar_init_secondary(struct rte_eth_dev *dev)
  *
  * @return
  *   A valid Ethernet device object on success, NULL otherwise and rte_errno
- *   is set. The following error is defined:
+ *   is set. The following errors are defined:
  *
  *   EBUSY: device is not supposed to be spawned.
+ *   EEXIST: device is already spawned
  */
 static struct rte_eth_dev *
 mlx5_dev_spawn(struct rte_device *dpdk_dev,
@@ -702,6 +723,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	struct mlx5dv_context dv_attr = { .comp_mask = 0 };
 	struct mlx5_dev_config config = {
 		.vf = !!vf,
+		.mps = MLX5_ARG_UNSET,
 		.tx_vec_en = 1,
 		.rx_vec_en = 1,
 		.mpw_hdr_dseg = 0,
@@ -729,12 +751,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	unsigned int mprq_max_stride_size_n = 0;
 	unsigned int mprq_min_stride_num_n = 0;
 	unsigned int mprq_max_stride_num_n = 0;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
-#endif
 	struct ether_addr mac;
 	char name[RTE_ETH_NAME_MAX_LEN];
 	int own_domain_id = 0;
+	uint16_t port_id;
 	unsigned int i;
 
 	/* Determine if this port representor is supposed to be spawned. */
@@ -757,6 +777,17 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			return NULL;
 		}
 	}
+	/* Build device name. */
+	if (!switch_info->representor)
+		rte_strlcpy(name, dpdk_dev->name, sizeof(name));
+	else
+		snprintf(name, sizeof(name), "%s_representor_%u",
+			 dpdk_dev->name, switch_info->port_name);
+	/* check if the device is already spawned */
+	if (rte_eth_dev_get_port_by_name(name, &port_id) == 0) {
+		rte_errno = EEXIST;
+		return NULL;
+	}
 	/* Prepare shared data between primary and secondary process. */
 	mlx5_prepare_shared_data();
 	errno = 0;
@@ -791,7 +822,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		DRV_LOG(DEBUG, "MPW isn't supported");
 		mps = MLX5_MPW_DISABLED;
 	}
-	config.mps = mps;
 #ifdef HAVE_IBV_MLX5_MOD_SWP
 	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_SWP)
 		swp = dv_attr.sw_parsing_caps.sw_parsing_offloads;
@@ -864,11 +894,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		DEBUG("ibv_query_device_ex() failed");
 		goto error;
 	}
-	if (!switch_info->representor)
-		rte_strlcpy(name, dpdk_dev->name, sizeof(name));
-	else
-		snprintf(name, sizeof(name), "%s_representor_%u",
-			 dpdk_dev->name, switch_info->port_name);
 	DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name);
 	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 		eth_dev = rte_eth_dev_attach_secondary(name);
@@ -1000,12 +1025,15 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	config.hw_csum = !!(attr.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM);
 	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
 		(config.hw_csum ? "" : "not "));
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	config.flow_counter_en = !!attr.max_counter_sets;
-	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
-	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
-		cs_desc.counter_type, cs_desc.num_of_cs,
-		cs_desc.attributes);
+#if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
+	!defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	DRV_LOG(DEBUG, "counters are not supported");
+#endif
+#ifndef HAVE_IBV_FLOW_DV_SUPPORT
+	if (config.dv_flow_en) {
+		DRV_LOG(WARNING, "DV flow is not supported");
+		config.dv_flow_en = 0;
+	}
 #endif
 	config.ind_table_max_size =
 		attr.rss_caps.max_rwq_indirection_table_size;
@@ -1035,13 +1063,15 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		       (1 << IBV_QPT_RAW_PACKET)));
 	if (config.tso)
 		config.tso_max_payload_sz = attr.tso_caps.max_tso;
-	if (config.mps && !mps) {
-		DRV_LOG(ERR,
-			"multi-packet send not supported on this device"
-			" (" MLX5_TXQ_MPW_EN ")");
-		err = ENOTSUP;
-		goto error;
-	}
+	/*
+	 * MPW is disabled by default, while the Enhanced MPW is enabled
+	 * by default.
+	 */
+	if (config.mps == MLX5_ARG_UNSET)
+		config.mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED :
+							  MLX5_MPW_DISABLED;
+	else
+		config.mps = config.mps ? mps : MLX5_MPW_DISABLED;
 	DRV_LOG(INFO, "%sMPS is %s",
 		config.mps == MLX5_MPW_ENHANCED ? "enhanced " : "",
 		config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled");
@@ -1073,13 +1103,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		err = ENOMEM;
 		goto error;
 	}
-	if (priv->representor)
+	if (priv->representor) {
 		eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
+		eth_dev->data->representor_id = priv->representor_id;
+	}
 	eth_dev->data->dev_private = priv;
 	priv->dev_data = eth_dev->data;
 	eth_dev->data->mac_addrs = priv->mac;
 	eth_dev->device = dpdk_dev;
-	eth_dev->device->driver = &mlx5_driver.driver;
 	err = mlx5_uar_init_primary(eth_dev);
 	if (err) {
 		err = rte_errno;
@@ -1128,8 +1159,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
 	if (vf && config.vf_nl_en)
 		mlx5_nl_mac_addr_sync(eth_dev);
-	priv->mnl_socket = mlx5_nl_flow_socket_create();
-	if (!priv->mnl_socket) {
+	priv->tcf_context = mlx5_flow_tcf_context_create();
+	if (!priv->tcf_context) {
 		err = -rte_errno;
 		DRV_LOG(WARNING,
 			"flow rules relying on switch offloads will not be"
@@ -1144,16 +1175,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			error.message =
 				"cannot retrieve network interface index";
 		} else {
-			err = mlx5_nl_flow_init(priv->mnl_socket, ifindex,
-						&error);
+			err = mlx5_flow_tcf_init(priv->tcf_context,
+						 ifindex, &error);
 		}
 		if (err) {
 			DRV_LOG(WARNING,
 				"flow rules relying on switch offloads will"
 				" not be supported: %s: %s",
 				error.message, strerror(rte_errno));
-			mlx5_nl_flow_socket_destroy(priv->mnl_socket);
-			priv->mnl_socket = NULL;
+			mlx5_flow_tcf_context_destroy(priv->tcf_context);
+			priv->tcf_context = NULL;
 		}
 	}
 	TAILQ_INIT(&priv->flows);
@@ -1208,16 +1239,21 @@ error:
 			close(priv->nl_socket_route);
 		if (priv->nl_socket_rdma >= 0)
 			close(priv->nl_socket_rdma);
-		if (priv->mnl_socket)
-			mlx5_nl_flow_socket_destroy(priv->mnl_socket);
+		if (priv->tcf_context)
+			mlx5_flow_tcf_context_destroy(priv->tcf_context);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
 		rte_free(priv);
+		if (eth_dev != NULL)
+			eth_dev->data->dev_private = NULL;
 	}
 	if (pd)
 		claim_zero(mlx5_glue->dealloc_pd(pd));
-	if (eth_dev)
+	if (eth_dev != NULL) {
+		/* mac_addrs must not be freed alone because part of dev_private */
+		eth_dev->data->mac_addrs = NULL;
 		rte_eth_dev_release_port(eth_dev);
+	}
 	if (ctx)
 		claim_zero(mlx5_glue->close_device(ctx));
 	assert(err > 0);
@@ -1404,9 +1440,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 		list[i].eth_dev = mlx5_dev_spawn
 			(&pci_dev->device, list[i].ibv_dev, vf, &list[i].info);
 		if (!list[i].eth_dev) {
-			if (rte_errno != EBUSY)
+			if (rte_errno != EBUSY && rte_errno != EEXIST)
 				break;
-			/* Device is disabled, ignore it. */
+			/* Device is disabled or already spawned. Ignore it. */
 			continue;
 		}
 		restore = list[i].eth_dev->data->dev_flags;
@@ -1437,8 +1473,8 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 			if (!list[i].eth_dev)
 				continue;
 			mlx5_dev_close(list[i].eth_dev);
-			if (rte_eal_process_type() == RTE_PROC_PRIMARY)
-				rte_free(list[i].eth_dev->data->dev_private);
+			/* mac_addrs must not be freed because in dev_private */
+			list[i].eth_dev->data->mac_addrs = NULL;
 			claim_zero(rte_eth_dev_release_port(list[i].eth_dev));
 		}
 		/* Restore original error. */
@@ -1449,6 +1485,32 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	return ret;
 }
 
+/**
+ * DPDK callback to remove a PCI device.
+ *
+ * This function removes all Ethernet devices belong to a given PCI device.
+ *
+ * @param[in] pci_dev
+ *   Pointer to the PCI device.
+ *
+ * @return
+ *   0 on success, the function cannot fail.
+ */
+static int
+mlx5_pci_remove(struct rte_pci_device *pci_dev)
+{
+	uint16_t port_id;
+	struct rte_eth_dev *port;
+
+	for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) {
+		port = &rte_eth_devices[port_id];
+		if (port->state != RTE_ETH_DEV_UNUSED &&
+				port->device == &pci_dev->device)
+			rte_eth_dev_close(port_id);
+	}
+	return 0;
+}
+
 static const struct rte_pci_id mlx5_pci_id_map[] = {
 	{
 		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
@@ -1487,6 +1549,10 @@ static const struct rte_pci_id mlx5_pci_id_map[] = {
 			       PCI_DEVICE_ID_MELLANOX_CONNECTX5BF)
 	},
 	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+			       PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF)
+	},
+	{
 		.vendor_id = 0
 	}
 };
@@ -1497,7 +1563,9 @@ static struct rte_pci_driver mlx5_driver = {
 	},
 	.id_table = mlx5_pci_id_map,
 	.probe = mlx5_pci_probe,
-	.drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV,
+	.remove = mlx5_pci_remove,
+	.drv_flags = (RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
+		      RTE_PCI_DRV_PROBE_AGAIN),
 };
 
 #ifdef RTE_LIBRTE_MLX5_DLOPEN_DEPS
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index a3a34cff..74d87c05 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -51,6 +51,7 @@ enum {
 	PCI_DEVICE_ID_MELLANOX_CONNECTX5EX = 0x1019,
 	PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF = 0x101a,
 	PCI_DEVICE_ID_MELLANOX_CONNECTX5BF = 0xa2d2,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF = 0xa2d3,
 };
 
 /** Switch information returned by mlx5_nl_switch_info(). */
@@ -71,12 +72,23 @@ struct mlx5_shared_data {
 
 extern struct mlx5_shared_data *mlx5_shared_data;
 
+struct mlx5_counter_ctrl {
+	/* Name of the counter. */
+	char dpdk_name[RTE_ETH_XSTATS_NAME_SIZE];
+	/* Name of the counter on the device table. */
+	char ctr_name[RTE_ETH_XSTATS_NAME_SIZE];
+	uint32_t ib:1; /**< Nonzero for IB counters. */
+};
+
 struct mlx5_xstats_ctrl {
 	/* Number of device stats. */
 	uint16_t stats_n;
+	/* Number of device stats identified by PMD. */
+	uint16_t  mlx5_stats_n;
 	/* Index in the device counters table. */
 	uint16_t dev_table_idx[MLX5_MAX_XSTATS];
 	uint64_t base[MLX5_MAX_XSTATS];
+	struct mlx5_counter_ctrl info[MLX5_MAX_XSTATS];
 };
 
 /* Flow list . */
@@ -99,11 +111,9 @@ struct mlx5_dev_config {
 	unsigned int hw_fcs_strip:1; /* FCS stripping is supported. */
 	unsigned int hw_padding:1; /* End alignment padding is supported. */
 	unsigned int vf:1; /* This is a VF. */
-	unsigned int mps:2; /* Multi-packet send supported mode. */
 	unsigned int tunnel_en:1;
 	/* Whether tunnel stateless offloads are supported. */
 	unsigned int mpls_en:1; /* MPLS over GRE/UDP is enabled. */
-	unsigned int flow_counter_en:1; /* Whether flow counter is supported. */
 	unsigned int cqe_comp:1; /* CQE compression is enabled. */
 	unsigned int tso:1; /* Whether TSO is supported. */
 	unsigned int tx_vec_en:1; /* Tx vector is enabled. */
@@ -111,6 +121,7 @@ struct mlx5_dev_config {
 	unsigned int mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */
 	unsigned int l3_vxlan_en:1; /* Enable L3 VXLAN flow creation. */
 	unsigned int vf_nl_en:1; /* Enable Netlink requests in VF mode. */
+	unsigned int dv_flow_en:1; /* Enable DV flow. */
 	unsigned int swp:1; /* Tx generic tunnel checksum and TSO offload. */
 	struct {
 		unsigned int enabled:1; /* Whether MPRQ is enabled. */
@@ -122,6 +133,7 @@ struct mlx5_dev_config {
 		unsigned int min_rxqs_num;
 		/* Rx queue count threshold to enable MPRQ. */
 	} mprq; /* Configurations for Multi-Packet RQ. */
+	int mps; /* Multi-packet send supported mode. */
 	unsigned int flow_prio; /* Number of flow priorities. */
 	unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
 	unsigned int ind_table_max_size; /* Maximum indirection table size. */
@@ -156,13 +168,7 @@ struct mlx5_drop {
 	struct mlx5_rxq_ibv *rxq; /* Verbs Rx queue. */
 };
 
-/** DPDK port to network interface index (ifindex) conversion. */
-struct mlx5_nl_flow_ptoi {
-	uint16_t port_id; /**< DPDK port ID. */
-	unsigned int ifindex; /**< Network interface index. */
-};
-
-struct mnl_socket;
+struct mlx5_flow_tcf_context;
 
 struct priv {
 	LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
@@ -212,6 +218,7 @@ struct priv {
 	LIST_HEAD(txqibv, mlx5_txq_ibv) txqsibv; /* Verbs Tx queues. */
 	/* Verbs Indirection tables. */
 	LIST_HEAD(ind_tables, mlx5_ind_table_ibv) ind_tbls;
+	LIST_HEAD(matchers, mlx5_flow_dv_matcher) matchers;
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	int primary_socket; /* Unix socket for primary process. */
@@ -228,7 +235,7 @@ struct priv {
 	rte_spinlock_t uar_lock[MLX5_UAR_PAGE_NUM_MAX];
 	/* UAR same-page access control required in 32bit implementations. */
 #endif
-	struct mnl_socket *mnl_socket; /* Libmnl socket. */
+	struct mlx5_flow_tcf_context *tcf_context; /* TC flower context. */
 };
 
 #define PORT_ID(priv) ((priv)->dev_data->port_id)
@@ -240,12 +247,9 @@ int mlx5_getenv_int(const char *);
 
 /* mlx5_ethdev.c */
 
-int mlx5_get_master_ifname(const struct rte_eth_dev *dev,
-			   char (*ifname)[IF_NAMESIZE]);
 int mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]);
 unsigned int mlx5_ifindex(const struct rte_eth_dev *dev);
-int mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr,
-	       int master);
+int mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr);
 int mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu);
 int mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep,
 		   unsigned int flags);
@@ -396,23 +400,4 @@ unsigned int mlx5_nl_ifindex(int nl, const char *name);
 int mlx5_nl_switch_info(int nl, unsigned int ifindex,
 			struct mlx5_switch_info *info);
 
-/* mlx5_nl_flow.c */
-
-int mlx5_nl_flow_transpose(void *buf,
-			   size_t size,
-			   const struct mlx5_nl_flow_ptoi *ptoi,
-			   const struct rte_flow_attr *attr,
-			   const struct rte_flow_item *pattern,
-			   const struct rte_flow_action *actions,
-			   struct rte_flow_error *error);
-void mlx5_nl_flow_brand(void *buf, uint32_t handle);
-int mlx5_nl_flow_create(struct mnl_socket *nl, void *buf,
-			struct rte_flow_error *error);
-int mlx5_nl_flow_destroy(struct mnl_socket *nl, void *buf,
-			 struct rte_flow_error *error);
-int mlx5_nl_flow_init(struct mnl_socket *nl, unsigned int ifindex,
-		      struct rte_flow_error *error);
-struct mnl_socket *mlx5_nl_flow_socket_create(void);
-void mlx5_nl_flow_socket_destroy(struct mnl_socket *nl);
-
 #endif /* RTE_PMD_MLX5_H_ */
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 34c5b95e..d178ed6a 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -3,8 +3,6 @@
  * Copyright 2015 Mellanox Technologies, Ltd
  */
 
-#define _GNU_SOURCE
-
 #include <stddef.h>
 #include <assert.h>
 #include <inttypes.h>
@@ -129,7 +127,7 @@ struct ethtool_link_settings {
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-int
+static int
 mlx5_get_master_ifname(const struct rte_eth_dev *dev,
 		       char (*ifname)[IF_NAMESIZE])
 {
@@ -270,16 +268,12 @@ mlx5_ifindex(const struct rte_eth_dev *dev)
  *   Request number to pass to ioctl().
  * @param[out] ifr
  *   Interface request structure output buffer.
- * @param master
- *   When device is a port representor, perform request on master device
- *   instead.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr,
-	   int master)
+mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
 {
 	int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 	int ret = 0;
@@ -288,10 +282,7 @@ mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr,
 		rte_errno = errno;
 		return -rte_errno;
 	}
-	if (master)
-		ret = mlx5_get_master_ifname(dev, &ifr->ifr_name);
-	else
-		ret = mlx5_get_ifname(dev, &ifr->ifr_name);
+	ret = mlx5_get_ifname(dev, &ifr->ifr_name);
 	if (ret)
 		goto error;
 	ret = ioctl(sock, req, ifr);
@@ -321,7 +312,7 @@ int
 mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
 {
 	struct ifreq request;
-	int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request, 0);
+	int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
 
 	if (ret)
 		return ret;
@@ -345,7 +336,7 @@ mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 {
 	struct ifreq request = { .ifr_mtu = mtu, };
 
-	return mlx5_ifreq(dev, SIOCSIFMTU, &request, 0);
+	return mlx5_ifreq(dev, SIOCSIFMTU, &request);
 }
 
 /**
@@ -365,13 +356,13 @@ int
 mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
 {
 	struct ifreq request;
-	int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request, 0);
+	int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
 
 	if (ret)
 		return ret;
 	request.ifr_flags &= keep;
 	request.ifr_flags |= flags & ~keep;
-	return mlx5_ifreq(dev, SIOCSIFFLAGS, &request, 0);
+	return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
 }
 
 /**
@@ -627,17 +618,20 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 	int link_speed = 0;
 	int ret;
 
-	ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr, 1);
+	ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 	if (ret) {
 		DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 			dev->data->port_id, strerror(rte_errno));
 		return ret;
 	}
-	memset(&dev_link, 0, sizeof(dev_link));
-	dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
-				(ifr.ifr_flags & IFF_RUNNING));
-	ifr.ifr_data = (void *)&edata;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	dev_link = (struct rte_eth_link) {
+		.link_status = ((ifr.ifr_flags & IFF_UP) &&
+				(ifr.ifr_flags & IFF_RUNNING)),
+	};
+	ifr = (struct ifreq) {
+		.ifr_data = (void *)&edata,
+	};
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 	if (ret) {
 		DRV_LOG(WARNING,
 			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s",
@@ -666,8 +660,8 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 				ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 	dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 			ETH_LINK_SPEED_FIXED);
-	if ((dev_link.link_speed && !dev_link.link_status) ||
-	    (!dev_link.link_speed && dev_link.link_status)) {
+	if (((dev_link.link_speed && !dev_link.link_status) ||
+	     (!dev_link.link_speed && dev_link.link_status))) {
 		rte_errno = EAGAIN;
 		return -rte_errno;
 	}
@@ -698,17 +692,20 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 	uint64_t sc;
 	int ret;
 
-	ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr, 1);
+	ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 	if (ret) {
 		DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 			dev->data->port_id, strerror(rte_errno));
 		return ret;
 	}
-	memset(&dev_link, 0, sizeof(dev_link));
-	dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
-				(ifr.ifr_flags & IFF_RUNNING));
-	ifr.ifr_data = (void *)&gcmd;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	dev_link = (struct rte_eth_link) {
+		.link_status = ((ifr.ifr_flags & IFF_UP) &&
+				(ifr.ifr_flags & IFF_RUNNING)),
+	};
+	ifr = (struct ifreq) {
+		.ifr_data = (void *)&gcmd,
+	};
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 	if (ret) {
 		DRV_LOG(DEBUG,
 			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
@@ -725,7 +722,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 
 	*ecmd = gcmd;
 	ifr.ifr_data = (void *)ecmd;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 	if (ret) {
 		DRV_LOG(DEBUG,
 			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
@@ -775,8 +772,8 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 				ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 	dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 				  ETH_LINK_SPEED_FIXED);
-	if ((dev_link.link_speed && !dev_link.link_status) ||
-	    (!dev_link.link_speed && dev_link.link_status)) {
+	if (((dev_link.link_speed && !dev_link.link_status) ||
+	     (!dev_link.link_speed && dev_link.link_status))) {
 		rte_errno = EAGAIN;
 		return -rte_errno;
 	}
@@ -888,7 +885,7 @@ mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 	int ret;
 
 	ifr.ifr_data = (void *)&ethpause;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 	if (ret) {
 		DRV_LOG(WARNING,
 			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
@@ -941,7 +938,7 @@ mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 		ethpause.tx_pause = 1;
 	else
 		ethpause.tx_pause = 0;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 0);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 	if (ret) {
 		DRV_LOG(WARNING,
 			"port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
@@ -1306,10 +1303,7 @@ mlx5_dev_to_port_id(const struct rte_device *dev, uint16_t *port_list,
 	RTE_ETH_FOREACH_DEV(id) {
 		struct rte_eth_dev *ldev = &rte_eth_devices[id];
 
-		if (!ldev->device ||
-		    !ldev->device->driver ||
-		    strcmp(ldev->device->driver->name, MLX5_DRIVER_NAME) ||
-		    ldev->device != dev)
+		if (ldev->device != dev)
 			continue;
 		if (n < port_list_n)
 			port_list[n] = id;
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index ca4625b6..280af0ab 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -3,6 +3,7 @@
  * Copyright 2016 Mellanox Technologies, Ltd
  */
 
+#include <netinet/in.h>
 #include <sys/queue.h>
 #include <stdalign.h>
 #include <stdint.h>
@@ -31,74 +32,30 @@
 #include "mlx5_defs.h"
 #include "mlx5_prm.h"
 #include "mlx5_glue.h"
+#include "mlx5_flow.h"
 
 /* Dev ops structure defined in mlx5.c */
 extern const struct eth_dev_ops mlx5_dev_ops;
 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
-/* Pattern outer Layer bits. */
-#define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
-#define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
-#define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
-#define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
-#define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
-#define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
-
-/* Pattern inner Layer bits. */
-#define MLX5_FLOW_LAYER_INNER_L2 (1u << 6)
-#define MLX5_FLOW_LAYER_INNER_L3_IPV4 (1u << 7)
-#define MLX5_FLOW_LAYER_INNER_L3_IPV6 (1u << 8)
-#define MLX5_FLOW_LAYER_INNER_L4_UDP (1u << 9)
-#define MLX5_FLOW_LAYER_INNER_L4_TCP (1u << 10)
-#define MLX5_FLOW_LAYER_INNER_VLAN (1u << 11)
-
-/* Pattern tunnel Layer bits. */
-#define MLX5_FLOW_LAYER_VXLAN (1u << 12)
-#define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
-#define MLX5_FLOW_LAYER_GRE (1u << 14)
-#define MLX5_FLOW_LAYER_MPLS (1u << 15)
-
-/* Outer Masks. */
-#define MLX5_FLOW_LAYER_OUTER_L3 \
-	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
-#define MLX5_FLOW_LAYER_OUTER_L4 \
-	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
-#define MLX5_FLOW_LAYER_OUTER \
-	(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_L3 | \
-	 MLX5_FLOW_LAYER_OUTER_L4)
-
-/* Tunnel Masks. */
-#define MLX5_FLOW_LAYER_TUNNEL \
-	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
-	 MLX5_FLOW_LAYER_GRE | MLX5_FLOW_LAYER_MPLS)
-
-/* Inner Masks. */
-#define MLX5_FLOW_LAYER_INNER_L3 \
-	(MLX5_FLOW_LAYER_INNER_L3_IPV4 | MLX5_FLOW_LAYER_INNER_L3_IPV6)
-#define MLX5_FLOW_LAYER_INNER_L4 \
-	(MLX5_FLOW_LAYER_INNER_L4_UDP | MLX5_FLOW_LAYER_INNER_L4_TCP)
-#define MLX5_FLOW_LAYER_INNER \
-	(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3 | \
-	 MLX5_FLOW_LAYER_INNER_L4)
-
-/* Actions that modify the fate of matching traffic. */
-#define MLX5_FLOW_FATE_DROP (1u << 0)
-#define MLX5_FLOW_FATE_QUEUE (1u << 1)
-#define MLX5_FLOW_FATE_RSS (1u << 2)
-
-/* Modify a packet. */
-#define MLX5_FLOW_MOD_FLAG (1u << 0)
-#define MLX5_FLOW_MOD_MARK (1u << 1)
-#define MLX5_FLOW_MOD_COUNT (1u << 2)
-
-/* possible L3 layers protocols filtering. */
-#define MLX5_IP_PROTOCOL_TCP 6
-#define MLX5_IP_PROTOCOL_UDP 17
-#define MLX5_IP_PROTOCOL_GRE 47
-#define MLX5_IP_PROTOCOL_MPLS 147
-
-/* Priority reserved for default flows. */
-#define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
+/** Device flow drivers. */
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+extern const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops;
+#endif
+extern const struct mlx5_flow_driver_ops mlx5_flow_tcf_drv_ops;
+extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops;
+
+const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops;
+
+const struct mlx5_flow_driver_ops *flow_drv_ops[] = {
+	[MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops,
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	[MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops,
+#endif
+	[MLX5_FLOW_TYPE_TCF] = &mlx5_flow_tcf_drv_ops,
+	[MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops,
+	[MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops
+};
 
 enum mlx5_expansion {
 	MLX5_EXPANSION_ROOT,
@@ -270,53 +227,6 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 	},
 };
 
-/** Handles information leading to a drop fate. */
-struct mlx5_flow_verbs {
-	LIST_ENTRY(mlx5_flow_verbs) next;
-	unsigned int size; /**< Size of the attribute. */
-	struct {
-		struct ibv_flow_attr *attr;
-		/**< Pointer to the Specification buffer. */
-		uint8_t *specs; /**< Pointer to the specifications. */
-	};
-	struct ibv_flow *flow; /**< Verbs flow pointer. */
-	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
-	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
-};
-
-/* Counters information. */
-struct mlx5_flow_counter {
-	LIST_ENTRY(mlx5_flow_counter) next; /**< Pointer to the next counter. */
-	uint32_t shared:1; /**< Share counter ID with other flow rules. */
-	uint32_t ref_cnt:31; /**< Reference counter. */
-	uint32_t id; /**< Counter ID. */
-	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
-	uint64_t hits; /**< Number of packets matched by the rule. */
-	uint64_t bytes; /**< Number of bytes matched by the rule. */
-};
-
-/* Flow structure. */
-struct rte_flow {
-	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
-	struct rte_flow_attr attributes; /**< User flow attribute. */
-	uint32_t l3_protocol_en:1; /**< Protocol filtering requested. */
-	uint32_t layers;
-	/**< Bit-fields of present layers see MLX5_FLOW_LAYER_*. */
-	uint32_t modifier;
-	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
-	uint32_t fate;
-	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
-	uint8_t l3_protocol; /**< valid when l3_protocol_en is set. */
-	LIST_HEAD(verbs, mlx5_flow_verbs) verbs; /**< Verbs flows list. */
-	struct mlx5_flow_verbs *cur_verbs;
-	/**< Current Verbs flow structure being filled. */
-	struct mlx5_flow_counter *counter; /**< Holds Verbs flow counter. */
-	struct rte_flow_action_rss rss;/**< RSS context. */
-	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
-	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
-	void *nl_flow; /**< Netlink flow buffer if relevant. */
-};
-
 static const struct rte_flow_ops mlx5_flow_ops = {
 	.validate = mlx5_flow_validate,
 	.create = mlx5_flow_create,
@@ -352,23 +262,6 @@ struct mlx5_fdir {
 	struct rte_flow_action_queue queue;
 };
 
-/* Verbs specification header. */
-struct ibv_spec_header {
-	enum ibv_flow_spec_type type;
-	uint16_t size;
-};
-
-/*
- * Number of sub priorities.
- * For each kind of pattern matching i.e. L2, L3, L4 to have a correct
- * matching on the NIC (firmware dependent) L4 most have the higher priority
- * followed by L3 and ending with L2.
- */
-#define MLX5_PRIORITY_MAP_L2 2
-#define MLX5_PRIORITY_MAP_L3 1
-#define MLX5_PRIORITY_MAP_L4 0
-#define MLX5_PRIORITY_MAP_MAX 3
-
 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
 	{ 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
@@ -413,7 +306,7 @@ static struct mlx5_flow_tunnel_info tunnels_info[] = {
  * Discover the maximum number of priority available.
  *
  * @param[in] dev
- *   Pointer to Ethernet device.
+ *   Pointer to the Ethernet device structure.
  *
  * @return
  *   number of supported flow priority on success, a negative errno
@@ -478,160 +371,33 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 }
 
 /**
- * Adjust flow priority.
+ * Adjust flow priority based on the highest layer and the request priority.
  *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to an rte flow.
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] priority
+ *   The rule base priority.
+ * @param[in] subpriority
+ *   The priority based on the items.
+ *
+ * @return
+ *   The new priority.
  */
-static void
-mlx5_flow_adjust_priority(struct rte_eth_dev *dev, struct rte_flow *flow)
+uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
+				   uint32_t subpriority)
 {
+	uint32_t res = 0;
 	struct priv *priv = dev->data->dev_private;
-	uint32_t priority = flow->attributes.priority;
-	uint32_t subpriority = flow->cur_verbs->attr->priority;
 
 	switch (priv->config.flow_prio) {
 	case RTE_DIM(priority_map_3):
-		priority = priority_map_3[priority][subpriority];
+		res = priority_map_3[priority][subpriority];
 		break;
 	case RTE_DIM(priority_map_5):
-		priority = priority_map_5[priority][subpriority];
+		res = priority_map_5[priority][subpriority];
 		break;
 	}
-	flow->cur_verbs->attr->priority = priority;
-}
-
-/**
- * Get a flow counter.
- *
- * @param[in] dev
- *   Pointer to Ethernet device.
- * @param[in] shared
- *   Indicate if this counter is shared with other flows.
- * @param[in] id
- *   Counter identifier.
- *
- * @return
- *   A pointer to the counter, NULL otherwise and rte_errno is set.
- */
-static struct mlx5_flow_counter *
-mlx5_flow_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
-{
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_flow_counter *cnt;
-
-	LIST_FOREACH(cnt, &priv->flow_counters, next) {
-		if (!cnt->shared || cnt->shared != shared)
-			continue;
-		if (cnt->id != id)
-			continue;
-		cnt->ref_cnt++;
-		return cnt;
-	}
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-
-	struct mlx5_flow_counter tmpl = {
-		.shared = shared,
-		.id = id,
-		.cs = mlx5_glue->create_counter_set
-			(priv->ctx,
-			 &(struct ibv_counter_set_init_attr){
-				 .counter_set_id = id,
-			 }),
-		.hits = 0,
-		.bytes = 0,
-	};
-
-	if (!tmpl.cs) {
-		rte_errno = errno;
-		return NULL;
-	}
-	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
-	if (!cnt) {
-		rte_errno = ENOMEM;
-		return NULL;
-	}
-	*cnt = tmpl;
-	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
-	return cnt;
-#endif
-	rte_errno = ENOTSUP;
-	return NULL;
-}
-
-/**
- * Release a flow counter.
- *
- * @param[in] counter
- *   Pointer to the counter handler.
- */
-static void
-mlx5_flow_counter_release(struct mlx5_flow_counter *counter)
-{
-	if (--counter->ref_cnt == 0) {
-		claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
-		LIST_REMOVE(counter, next);
-		rte_free(counter);
-	}
-}
-
-/**
- * Verify the @p attributes will be correctly understood by the NIC and store
- * them in the @p flow if everything is correct.
- *
- * @param[in] dev
- *   Pointer to Ethernet device.
- * @param[in] attributes
- *   Pointer to flow attributes
- * @param[in, out] flow
- *   Pointer to the rte_flow structure.
- * @param[out] error
- *   Pointer to error structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_attributes(struct rte_eth_dev *dev,
-		     const struct rte_flow_attr *attributes,
-		     struct rte_flow *flow,
-		     struct rte_flow_error *error)
-{
-	uint32_t priority_max =
-		((struct priv *)dev->data->dev_private)->config.flow_prio - 1;
-
-	if (attributes->group)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
-					  NULL,
-					  "groups is not supported");
-	if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
-	    attributes->priority >= priority_max)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
-					  NULL,
-					  "priority out of range");
-	if (attributes->egress)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
-					  NULL,
-					  "egress is not supported");
-	if (attributes->transfer)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
-					  NULL,
-					  "transfer is not supported");
-	if (!attributes->ingress)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
-					  NULL,
-					  "ingress attribute is mandatory");
-	flow->attributes = *attributes;
-	if (attributes->priority == MLX5_FLOW_PRIO_RSVD)
-		flow->attributes.priority = priority_max;
-	return 0;
+	return  res;
 }
 
 /**
@@ -652,7 +418,7 @@ mlx5_flow_attributes(struct rte_eth_dev *dev,
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
+int
 mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 			  const uint8_t *mask,
 			  const uint8_t *nic_mask,
@@ -671,8 +437,7 @@ mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 						  " bits");
 	if (!item->spec && (item->mask || item->last))
 		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "mask/last without a spec is not"
 					  " supported");
 	if (item->spec && item->last) {
@@ -687,206 +452,635 @@ mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 		}
 		ret = memcmp(spec, last, size);
 		if (ret != 0)
-			return rte_flow_error_set(error, ENOTSUP,
+			return rte_flow_error_set(error, EINVAL,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  item,
-						  "range is not supported");
+						  "range is not valid");
 	}
 	return 0;
 }
 
 /**
- * Add a verbs item specification into @p flow.
+ * Adjust the hash fields according to the @p flow information.
  *
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] src
- *   Create specification.
- * @param[in] size
- *   Size in bytes of the specification to copy.
+ * @param[in] dev_flow.
+ *   Pointer to the mlx5_flow.
+ * @param[in] tunnel
+ *   1 when the hash field is for a tunnel item.
+ * @param[in] layer_types
+ *   ETH_RSS_* types.
+ * @param[in] hash_fields
+ *   Item hash fields.
+ *
+ * @return
+ *   The hash fileds that should be used.
+ */
+uint64_t
+mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow,
+			    int tunnel __rte_unused, uint64_t layer_types,
+			    uint64_t hash_fields)
+{
+	struct rte_flow *flow = dev_flow->flow;
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+	int rss_request_inner = flow->rss.level >= 2;
+
+	/* Check RSS hash level for tunnel. */
+	if (tunnel && rss_request_inner)
+		hash_fields |= IBV_RX_HASH_INNER;
+	else if (tunnel || rss_request_inner)
+		return 0;
+#endif
+	/* Check if requested layer matches RSS hash fields. */
+	if (!(flow->rss.types & layer_types))
+		return 0;
+	return hash_fields;
+}
+
+/**
+ * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
+ * if several tunnel rules are used on this queue, the tunnel ptype will be
+ * cleared.
+ *
+ * @param rxq_ctrl
+ *   Rx queue to update.
  */
 static void
-mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
+flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 {
-	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+	unsigned int i;
+	uint32_t tunnel_ptype = 0;
+
+	/* Look up for the ptype to use. */
+	for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
+		if (!rxq_ctrl->flow_tunnels_n[i])
+			continue;
+		if (!tunnel_ptype) {
+			tunnel_ptype = tunnels_info[i].ptype;
+		} else {
+			tunnel_ptype = 0;
+			break;
+		}
+	}
+	rxq_ctrl->rxq.tunnel = tunnel_ptype;
+}
+
+/**
+ * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the devive
+ * flow.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] dev_flow
+ *   Pointer to device flow structure.
+ */
+static void
+flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rte_flow *flow = dev_flow->flow;
+	const int mark = !!(flow->actions &
+			    (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
+	const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int i;
+
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int idx = (*flow->queue)[i];
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[idx],
+				     struct mlx5_rxq_ctrl, rxq);
 
-	if (verbs->specs) {
-		void *dst;
+		if (mark) {
+			rxq_ctrl->rxq.mark = 1;
+			rxq_ctrl->flow_mark_n++;
+		}
+		if (tunnel) {
+			unsigned int j;
 
-		dst = (void *)(verbs->specs + verbs->size);
-		memcpy(dst, src, size);
-		++verbs->attr->num_of_specs;
+			/* Increase the counter matching the flow. */
+			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
+				if ((tunnels_info[j].tunnel &
+				     dev_flow->layers) ==
+				    tunnels_info[j].tunnel) {
+					rxq_ctrl->flow_tunnels_n[j]++;
+					break;
+				}
+			}
+			flow_rxq_tunnel_ptype_update(rxq_ctrl);
+		}
 	}
-	verbs->size += size;
 }
 
 /**
- * Adjust verbs hash fields according to the @p flow information.
+ * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow
  *
- * @param[in, out] flow.
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] flow
  *   Pointer to flow structure.
- * @param[in] tunnel
- *   1 when the hash field is for a tunnel item.
- * @param[in] layer_types
- *   ETH_RSS_* types.
- * @param[in] hash_fields
- *   Item hash fields.
  */
 static void
-mlx5_flow_verbs_hashfields_adjust(struct rte_flow *flow,
-				  int tunnel __rte_unused,
-				  uint32_t layer_types, uint64_t hash_fields)
+flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct mlx5_flow *dev_flow;
+
+	LIST_FOREACH(dev_flow, &flow->dev_flows, next)
+		flow_drv_rxq_flags_set(dev, dev_flow);
+}
+
+/**
+ * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
+ * device flow if no other flow uses it with the same kind of request.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] dev_flow
+ *   Pointer to the device flow.
+ */
+static void
+flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rte_flow *flow = dev_flow->flow;
+	const int mark = !!(flow->actions &
+			    (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
+	const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int i;
+
+	assert(dev->data->dev_started);
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int idx = (*flow->queue)[i];
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[idx],
+				     struct mlx5_rxq_ctrl, rxq);
+
+		if (mark) {
+			rxq_ctrl->flow_mark_n--;
+			rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
+		}
+		if (tunnel) {
+			unsigned int j;
+
+			/* Decrease the counter matching the flow. */
+			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
+				if ((tunnels_info[j].tunnel &
+				     dev_flow->layers) ==
+				    tunnels_info[j].tunnel) {
+					rxq_ctrl->flow_tunnels_n[j]--;
+					break;
+				}
+			}
+			flow_rxq_tunnel_ptype_update(rxq_ctrl);
+		}
+	}
+}
+
+/**
+ * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
+ * @p flow if no other flow uses it with the same kind of request.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] flow
+ *   Pointer to the flow.
+ */
+static void
+flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct mlx5_flow *dev_flow;
+
+	LIST_FOREACH(dev_flow, &flow->dev_flows, next)
+		flow_drv_rxq_flags_trim(dev, dev_flow);
+}
+
+/**
+ * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+static void
+flow_rxq_flags_clear(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+
+	for (i = 0; i != priv->rxqs_n; ++i) {
+		struct mlx5_rxq_ctrl *rxq_ctrl;
+		unsigned int j;
+
+		if (!(*priv->rxqs)[i])
+			continue;
+		rxq_ctrl = container_of((*priv->rxqs)[i],
+					struct mlx5_rxq_ctrl, rxq);
+		rxq_ctrl->flow_mark_n = 0;
+		rxq_ctrl->rxq.mark = 0;
+		for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
+			rxq_ctrl->flow_tunnels_n[j] = 0;
+		rxq_ctrl->rxq.tunnel = 0;
+	}
+}
+
+/*
+ * Validate the flag action.
+ *
+ * @param[in] action_flags
+ *   Bit-fields that holds the actions detected until now.
+ * @param[in] attr
+ *   Attributes of flow that includes this action.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_flow_validate_action_flag(uint64_t action_flags,
+			       const struct rte_flow_attr *attr,
+			       struct rte_flow_error *error)
+{
+
+	if (action_flags & MLX5_FLOW_ACTION_DROP)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't drop and flag in same flow");
+	if (action_flags & MLX5_FLOW_ACTION_MARK)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't mark and flag in same flow");
+	if (action_flags & MLX5_FLOW_ACTION_FLAG)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't have 2 flag"
+					  " actions in same flow");
+	if (attr->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
+					  "flag action not supported for "
+					  "egress");
+	return 0;
+}
+
+/*
+ * Validate the mark action.
+ *
+ * @param[in] action
+ *   Pointer to the queue action.
+ * @param[in] action_flags
+ *   Bit-fields that holds the actions detected until now.
+ * @param[in] attr
+ *   Attributes of flow that includes this action.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
+			       uint64_t action_flags,
+			       const struct rte_flow_attr *attr,
+			       struct rte_flow_error *error)
+{
+	const struct rte_flow_action_mark *mark = action->conf;
+
+	if (!mark)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "configuration cannot be null");
+	if (mark->id >= MLX5_FLOW_MARK_MAX)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &mark->id,
+					  "mark id must in 0 <= id < "
+					  RTE_STR(MLX5_FLOW_MARK_MAX));
+	if (action_flags & MLX5_FLOW_ACTION_DROP)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't drop and mark in same flow");
+	if (action_flags & MLX5_FLOW_ACTION_FLAG)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't flag and mark in same flow");
+	if (action_flags & MLX5_FLOW_ACTION_MARK)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't have 2 mark actions in same"
+					  " flow");
+	if (attr->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
+					  "mark action not supported for "
+					  "egress");
+	return 0;
+}
+
+/*
+ * Validate the drop action.
+ *
+ * @param[in] action_flags
+ *   Bit-fields that holds the actions detected until now.
+ * @param[in] attr
+ *   Attributes of flow that includes this action.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_ernno is set.
+ */
+int
+mlx5_flow_validate_action_drop(uint64_t action_flags,
+			       const struct rte_flow_attr *attr,
+			       struct rte_flow_error *error)
+{
+	if (action_flags & MLX5_FLOW_ACTION_FLAG)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't drop and flag in same flow");
+	if (action_flags & MLX5_FLOW_ACTION_MARK)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't drop and mark in same flow");
+	if (action_flags & MLX5_FLOW_FATE_ACTIONS)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't have 2 fate actions in"
+					  " same flow");
+	if (attr->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
+					  "drop action not supported for "
+					  "egress");
+	return 0;
+}
+
+/*
+ * Validate the queue action.
+ *
+ * @param[in] action
+ *   Pointer to the queue action.
+ * @param[in] action_flags
+ *   Bit-fields that holds the actions detected until now.
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] attr
+ *   Attributes of flow that includes this action.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_ernno is set.
+ */
+int
+mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
+				uint64_t action_flags,
+				struct rte_eth_dev *dev,
+				const struct rte_flow_attr *attr,
+				struct rte_flow_error *error)
 {
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_queue *queue = action->conf;
+
+	if (action_flags & MLX5_FLOW_FATE_ACTIONS)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't have 2 fate actions in"
+					  " same flow");
+	if (queue->index >= priv->rxqs_n)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue index out of range");
+	if (!(*priv->rxqs)[queue->index])
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue is not configured");
+	if (attr->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
+					  "queue action not supported for "
+					  "egress");
+	return 0;
+}
+
+/*
+ * Validate the rss action.
+ *
+ * @param[in] action
+ *   Pointer to the queue action.
+ * @param[in] action_flags
+ *   Bit-fields that holds the actions detected until now.
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] attr
+ *   Attributes of flow that includes this action.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_ernno is set.
+ */
+int
+mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
+			      uint64_t action_flags,
+			      struct rte_eth_dev *dev,
+			      const struct rte_flow_attr *attr,
+			      struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_rss *rss = action->conf;
+	unsigned int i;
+
+	if (action_flags & MLX5_FLOW_FATE_ACTIONS)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't have 2 fate actions"
+					  " in same flow");
+	if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
+	    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->func,
+					  "RSS hash function not supported");
 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	hash_fields |= (tunnel ? IBV_RX_HASH_INNER : 0);
-	if (flow->rss.level == 2 && !tunnel)
-		hash_fields = 0;
-	else if (flow->rss.level < 2 && tunnel)
-		hash_fields = 0;
+	if (rss->level > 2)
+#else
+	if (rss->level > 1)
 #endif
-	if (!(flow->rss.types & layer_types))
-		hash_fields = 0;
-	flow->cur_verbs->hash_fields |= hash_fields;
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->level,
+					  "tunnel RSS is not supported");
+	if (rss->key_len < MLX5_RSS_HASH_KEY_LEN)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too small");
+	if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too large");
+	if (rss->queue_num > priv->config.ind_table_max_size)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->queue_num,
+					  "number of queues too large");
+	if (rss->types & MLX5_RSS_HF_MASK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->types,
+					  "some RSS protocols are not"
+					  " supported");
+	for (i = 0; i != rss->queue_num; ++i) {
+		if (!(*priv->rxqs)[rss->queue[i]])
+			return rte_flow_error_set
+				(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 &rss->queue[i], "queue is not configured");
+	}
+	if (attr->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
+					  "rss action not supported for "
+					  "egress");
+	return 0;
+}
+
+/*
+ * Validate the count action.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] attr
+ *   Attributes of flow that includes this action.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_ernno is set.
+ */
+int
+mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
+				const struct rte_flow_attr *attr,
+				struct rte_flow_error *error)
+{
+	if (attr->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
+					  "count action not supported for "
+					  "egress");
+	return 0;
 }
 
 /**
- * Convert the @p item into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
+ * Verify the @p attributes will be correctly understood by the NIC and store
+ * them in the @p flow if everything is correct.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] attributes
+ *   Pointer to flow attributes
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
+			      const struct rte_flow_attr *attributes,
+			      struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint32_t priority_max = priv->config.flow_prio - 1;
+
+	if (attributes->group)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+					  NULL, "groups is not supported");
+	if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
+	    attributes->priority >= priority_max)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+					  NULL, "priority out of range");
+	if (attributes->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
+					  "egress is not supported");
+	if (attributes->transfer)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
+					  NULL, "transfer is not supported");
+	if (!attributes->ingress)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+					  NULL,
+					  "ingress attribute is mandatory");
+	return 0;
+}
+
+/**
+ * Validate Ethernet item.
  *
  * @param[in] item
  *   Item specification.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
+ * @param[in] item_flags
+ *   Bit-fields that holds the items detected until now.
  * @param[out] error
  *   Pointer to error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p item has fully been converted,
- *   otherwise another call with this returned memory size should be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
-		   const size_t flow_size, struct rte_flow_error *error)
+int
+mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
+			    uint64_t item_flags,
+			    struct rte_flow_error *error)
 {
-	const struct rte_flow_item_eth *spec = item->spec;
 	const struct rte_flow_item_eth *mask = item->mask;
 	const struct rte_flow_item_eth nic_mask = {
 		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 		.type = RTE_BE16(0xffff),
 	};
-	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
-	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
-	struct ibv_flow_spec_eth eth = {
-		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
-		.size = size,
-	};
 	int ret;
+	int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 
-	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
-			    MLX5_FLOW_LAYER_OUTER_L2))
+	if (item_flags & MLX5_FLOW_LAYER_OUTER_L2)
 		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L2 layers already configured");
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
+					  "3 levels of l2 are not supported");
+	if ((item_flags & MLX5_FLOW_LAYER_INNER_L2) && !tunnel)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
+					  "2 L2 without tunnel are not supported");
 	if (!mask)
 		mask = &rte_flow_item_eth_mask;
 	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
 					(const uint8_t *)&nic_mask,
 					sizeof(struct rte_flow_item_eth),
 					error);
-	if (ret)
-		return ret;
-	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
-		MLX5_FLOW_LAYER_OUTER_L2;
-	if (size > flow_size)
-		return size;
-	if (spec) {
-		unsigned int i;
-
-		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
-		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
-		eth.val.ether_type = spec->type;
-		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
-		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
-		eth.mask.ether_type = mask->type;
-		/* Remove unwanted bits from values. */
-		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
-			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
-			eth.val.src_mac[i] &= eth.mask.src_mac[i];
-		}
-		eth.val.ether_type &= eth.mask.ether_type;
-	}
-	flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
-	mlx5_flow_spec_verbs_add(flow, &eth, size);
-	return size;
-}
-
-/**
- * Update the VLAN tag in the Verbs Ethernet specification.
- *
- * @param[in, out] attr
- *   Pointer to Verbs attributes structure.
- * @param[in] eth
- *   Verbs structure containing the VLAN information to copy.
- */
-static void
-mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
-			   struct ibv_flow_spec_eth *eth)
-{
-	unsigned int i;
-	const enum ibv_flow_spec_type search = eth->type;
-	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
-		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
-
-	for (i = 0; i != attr->num_of_specs; ++i) {
-		if (hdr->type == search) {
-			struct ibv_flow_spec_eth *e =
-				(struct ibv_flow_spec_eth *)hdr;
-
-			e->val.vlan_tag = eth->val.vlan_tag;
-			e->mask.vlan_tag = eth->mask.vlan_tag;
-			e->val.ether_type = eth->val.ether_type;
-			e->mask.ether_type = eth->mask.ether_type;
-			break;
-		}
-		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
-	}
+	return ret;
 }
 
 /**
- * Convert the @p item into @p flow (or by updating the already present
- * Ethernet Verbs) specification after ensuring the NIC will understand and
- * process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
+ * Validate VLAN item.
  *
  * @param[in] item
  *   Item specification.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
+ * @param[in] item_flags
+ *   Bit-fields that holds the items detected until now.
  * @param[out] error
  *   Pointer to error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p item has fully been converted,
- *   otherwise another call with this returned memory size should be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
-		    const size_t flow_size, struct rte_flow_error *error)
+int
+mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
+			     int64_t item_flags,
+			     struct rte_flow_error *error)
 {
 	const struct rte_flow_item_vlan *spec = item->spec;
 	const struct rte_flow_item_vlan *mask = item->mask;
@@ -894,100 +1088,66 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 		.tci = RTE_BE16(0x0fff),
 		.inner_type = RTE_BE16(0xffff),
 	};
-	unsigned int size = sizeof(struct ibv_flow_spec_eth);
-	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
-	struct ibv_flow_spec_eth eth = {
-		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
-		.size = size,
-	};
+	uint16_t vlan_tag = 0;
+	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 	int ret;
 	const uint32_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
 					MLX5_FLOW_LAYER_INNER_L4) :
-		(MLX5_FLOW_LAYER_OUTER_L3 | MLX5_FLOW_LAYER_OUTER_L4);
+				       (MLX5_FLOW_LAYER_OUTER_L3 |
+					MLX5_FLOW_LAYER_OUTER_L4);
 	const uint32_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
-		MLX5_FLOW_LAYER_OUTER_VLAN;
-	const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
-		MLX5_FLOW_LAYER_OUTER_L2;
+					MLX5_FLOW_LAYER_OUTER_VLAN;
 
-	if (flow->layers & vlanm)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (item_flags & vlanm)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "VLAN layer already configured");
-	else if ((flow->layers & l34m) != 0)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	else if ((item_flags & l34m) != 0)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "L2 layer cannot follow L3/L4 layer");
 	if (!mask)
 		mask = &rte_flow_item_vlan_mask;
-	ret = mlx5_flow_item_acceptable
-		(item, (const uint8_t *)mask,
-		 (const uint8_t *)&nic_mask,
-		 sizeof(struct rte_flow_item_vlan), error);
+	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
+					(const uint8_t *)&nic_mask,
+					sizeof(struct rte_flow_item_vlan),
+					error);
 	if (ret)
 		return ret;
 	if (spec) {
-		eth.val.vlan_tag = spec->tci;
-		eth.mask.vlan_tag = mask->tci;
-		eth.val.vlan_tag &= eth.mask.vlan_tag;
-		eth.val.ether_type = spec->inner_type;
-		eth.mask.ether_type = mask->inner_type;
-		eth.val.ether_type &= eth.mask.ether_type;
+		vlan_tag = spec->tci;
+		vlan_tag &= mask->tci;
 	}
 	/*
 	 * From verbs perspective an empty VLAN is equivalent
 	 * to a packet without VLAN layer.
 	 */
-	if (!eth.mask.vlan_tag)
+	if (!vlan_tag)
 		return rte_flow_error_set(error, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
 					  item->spec,
 					  "VLAN cannot be empty");
-	if (!(flow->layers & l2m)) {
-		if (size <= flow_size) {
-			flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
-			mlx5_flow_spec_verbs_add(flow, &eth, size);
-		}
-	} else {
-		if (flow->cur_verbs)
-			mlx5_flow_item_vlan_update(flow->cur_verbs->attr,
-						   &eth);
-		size = 0; /* Only an update is done in eth specification. */
-	}
-	flow->layers |= tunnel ?
-		(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_VLAN) :
-		(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN);
-	return size;
+	return 0;
 }
 
 /**
- * Convert the @p item into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
+ * Validate IPV4 item.
  *
  * @param[in] item
  *   Item specification.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
+ * @param[in] item_flags
+ *   Bit-fields that holds the items detected until now.
  * @param[out] error
  *   Pointer to error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p item has fully been converted,
- *   otherwise another call with this returned memory size should be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
-		    const size_t flow_size, struct rte_flow_error *error)
+int
+mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
+			     int64_t item_flags,
+			     struct rte_flow_error *error)
 {
-	const struct rte_flow_item_ipv4 *spec = item->spec;
 	const struct rte_flow_item_ipv4 *mask = item->mask;
 	const struct rte_flow_item_ipv4 nic_mask = {
 		.hdr = {
@@ -997,97 +1157,48 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 			.next_proto_id = 0xff,
 		},
 	};
-	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
-	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
-	struct ibv_flow_spec_ipv4_ext ipv4 = {
-		.type = IBV_FLOW_SPEC_IPV4_EXT |
-			(tunnel ? IBV_FLOW_SPEC_INNER : 0),
-		.size = size,
-	};
+	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 	int ret;
 
-	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
-			    MLX5_FLOW_LAYER_OUTER_L3))
+	if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+				   MLX5_FLOW_LAYER_OUTER_L3))
 		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "multiple L3 layers not supported");
-	else if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
-				 MLX5_FLOW_LAYER_OUTER_L4))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	else if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+					MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "L3 cannot follow an L4 layer.");
 	if (!mask)
 		mask = &rte_flow_item_ipv4_mask;
-	ret = mlx5_flow_item_acceptable
-		(item, (const uint8_t *)mask,
-		 (const uint8_t *)&nic_mask,
-		 sizeof(struct rte_flow_item_ipv4), error);
+	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
+					(const uint8_t *)&nic_mask,
+					sizeof(struct rte_flow_item_ipv4),
+					error);
 	if (ret < 0)
 		return ret;
-	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
-		MLX5_FLOW_LAYER_OUTER_L3_IPV4;
-	if (spec) {
-		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
-			.src_ip = spec->hdr.src_addr,
-			.dst_ip = spec->hdr.dst_addr,
-			.proto = spec->hdr.next_proto_id,
-			.tos = spec->hdr.type_of_service,
-		};
-		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
-			.src_ip = mask->hdr.src_addr,
-			.dst_ip = mask->hdr.dst_addr,
-			.proto = mask->hdr.next_proto_id,
-			.tos = mask->hdr.type_of_service,
-		};
-		/* Remove unwanted bits from values. */
-		ipv4.val.src_ip &= ipv4.mask.src_ip;
-		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
-		ipv4.val.proto &= ipv4.mask.proto;
-		ipv4.val.tos &= ipv4.mask.tos;
-	}
-	flow->l3_protocol_en = !!ipv4.mask.proto;
-	flow->l3_protocol = ipv4.val.proto;
-	if (size <= flow_size) {
-		mlx5_flow_verbs_hashfields_adjust
-			(flow, tunnel,
-			 (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
-			  ETH_RSS_NONFRAG_IPV4_OTHER),
-			 (IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4));
-		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
-		mlx5_flow_spec_verbs_add(flow, &ipv4, size);
-	}
-	return size;
+	return 0;
 }
 
 /**
- * Convert the @p item into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
+ * Validate IPV6 item.
  *
  * @param[in] item
  *   Item specification.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
+ * @param[in] item_flags
+ *   Bit-fields that holds the items detected until now.
  * @param[out] error
  *   Pointer to error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p item has fully been converted,
- *   otherwise another call with this returned memory size should be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
-		    const size_t flow_size, struct rte_flow_error *error)
+int
+mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
+			     uint64_t item_flags,
+			     struct rte_flow_error *error)
 {
-	const struct rte_flow_item_ipv6 *spec = item->spec;
 	const struct rte_flow_item_ipv6 *mask = item->mask;
 	const struct rte_flow_item_ipv6 nic_mask = {
 		.hdr = {
@@ -1102,25 +1213,18 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 			.hop_limits = 0xff,
 		},
 	};
-	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
-	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
-	struct ibv_flow_spec_ipv6 ipv6 = {
-		.type = IBV_FLOW_SPEC_IPV6 | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
-		.size = size,
-	};
+	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 	int ret;
 
-	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
-			    MLX5_FLOW_LAYER_OUTER_L3))
+	if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+				   MLX5_FLOW_LAYER_OUTER_L3))
 		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "multiple L3 layers not supported");
-	else if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
-				 MLX5_FLOW_LAYER_OUTER_L4))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	else if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+					MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "L3 cannot follow an L4 layer.");
 	/*
 	 * IPv6 is not recognised by the NIC inside a GRE tunnel.
@@ -1128,130 +1232,64 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 	 * accepted.  Issue reproduced with Mellanox OFED 4.3-3.0.2.1 and
 	 * Mellanox OFED 4.4-1.0.0.0.
 	 */
-	if (tunnel && flow->layers & MLX5_FLOW_LAYER_GRE)
+	if (tunnel && item_flags & MLX5_FLOW_LAYER_GRE)
 		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "IPv6 inside a GRE tunnel is"
 					  " not recognised.");
 	if (!mask)
 		mask = &rte_flow_item_ipv6_mask;
-	ret = mlx5_flow_item_acceptable
-		(item, (const uint8_t *)mask,
-		 (const uint8_t *)&nic_mask,
-		 sizeof(struct rte_flow_item_ipv6), error);
+	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
+					(const uint8_t *)&nic_mask,
+					sizeof(struct rte_flow_item_ipv6),
+					error);
 	if (ret < 0)
 		return ret;
-	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
-		MLX5_FLOW_LAYER_OUTER_L3_IPV6;
-	if (spec) {
-		unsigned int i;
-		uint32_t vtc_flow_val;
-		uint32_t vtc_flow_mask;
-
-		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
-		       RTE_DIM(ipv6.val.src_ip));
-		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
-		       RTE_DIM(ipv6.val.dst_ip));
-		memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
-		       RTE_DIM(ipv6.mask.src_ip));
-		memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
-		       RTE_DIM(ipv6.mask.dst_ip));
-		vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
-		vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
-		ipv6.val.flow_label =
-			rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
-					 IPV6_HDR_FL_SHIFT);
-		ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
-					 IPV6_HDR_TC_SHIFT;
-		ipv6.val.next_hdr = spec->hdr.proto;
-		ipv6.val.hop_limit = spec->hdr.hop_limits;
-		ipv6.mask.flow_label =
-			rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
-					 IPV6_HDR_FL_SHIFT);
-		ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
-					  IPV6_HDR_TC_SHIFT;
-		ipv6.mask.next_hdr = mask->hdr.proto;
-		ipv6.mask.hop_limit = mask->hdr.hop_limits;
-		/* Remove unwanted bits from values. */
-		for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
-			ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
-			ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
-		}
-		ipv6.val.flow_label &= ipv6.mask.flow_label;
-		ipv6.val.traffic_class &= ipv6.mask.traffic_class;
-		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
-		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
-	}
-	flow->l3_protocol_en = !!ipv6.mask.next_hdr;
-	flow->l3_protocol = ipv6.val.next_hdr;
-	if (size <= flow_size) {
-		mlx5_flow_verbs_hashfields_adjust
-			(flow, tunnel,
-			 (ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER),
-			 (IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6));
-		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
-		mlx5_flow_spec_verbs_add(flow, &ipv6, size);
-	}
-	return size;
+	return 0;
 }
 
 /**
- * Convert the @p item into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
+ * Validate UDP item.
  *
  * @param[in] item
  *   Item specification.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
+ * @param[in] item_flags
+ *   Bit-fields that holds the items detected until now.
+ * @param[in] target_protocol
+ *   The next protocol in the previous item.
+ * @param[in] flow_mask
+ *   mlx5 flow-specific (TCF, DV, verbs, etc.) supported header fields mask.
  * @param[out] error
  *   Pointer to error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p item has fully been converted,
- *   otherwise another call with this returned memory size should be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
-		   const size_t flow_size, struct rte_flow_error *error)
+int
+mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
+			    uint64_t item_flags,
+			    uint8_t target_protocol,
+			    struct rte_flow_error *error)
 {
-	const struct rte_flow_item_udp *spec = item->spec;
 	const struct rte_flow_item_udp *mask = item->mask;
-	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
-	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
-	struct ibv_flow_spec_tcp_udp udp = {
-		.type = IBV_FLOW_SPEC_UDP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
-		.size = size,
-	};
+	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 	int ret;
 
-	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_UDP)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (target_protocol != 0xff && target_protocol != IPPROTO_UDP)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "protocol filtering not compatible"
 					  " with UDP layer");
-	if (!(flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
-			      MLX5_FLOW_LAYER_OUTER_L3)))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 is mandatory to filter"
-					  " on L4");
-	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
-			    MLX5_FLOW_LAYER_OUTER_L4))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L4 layer is already"
-					  " present");
+	if (!(item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+				     MLX5_FLOW_LAYER_OUTER_L3)))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
+					  "L3 is mandatory to filter on L4");
+	if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				   MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
+					  "L4 layer is already present");
 	if (!mask)
 		mask = &rte_flow_item_udp_mask;
 	ret = mlx5_flow_item_acceptable
@@ -1260,178 +1298,118 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 		 sizeof(struct rte_flow_item_udp), error);
 	if (ret < 0)
 		return ret;
-	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
-		MLX5_FLOW_LAYER_OUTER_L4_UDP;
-	if (spec) {
-		udp.val.dst_port = spec->hdr.dst_port;
-		udp.val.src_port = spec->hdr.src_port;
-		udp.mask.dst_port = mask->hdr.dst_port;
-		udp.mask.src_port = mask->hdr.src_port;
-		/* Remove unwanted bits from values. */
-		udp.val.src_port &= udp.mask.src_port;
-		udp.val.dst_port &= udp.mask.dst_port;
-	}
-	if (size <= flow_size) {
-		mlx5_flow_verbs_hashfields_adjust(flow, tunnel, ETH_RSS_UDP,
-						  (IBV_RX_HASH_SRC_PORT_UDP |
-						   IBV_RX_HASH_DST_PORT_UDP));
-		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
-		mlx5_flow_spec_verbs_add(flow, &udp, size);
-	}
-	return size;
+	return 0;
 }
 
 /**
- * Convert the @p item into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
+ * Validate TCP item.
  *
  * @param[in] item
  *   Item specification.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
+ * @param[in] item_flags
+ *   Bit-fields that holds the items detected until now.
+ * @param[in] target_protocol
+ *   The next protocol in the previous item.
  * @param[out] error
  *   Pointer to error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p item has fully been converted,
- *   otherwise another call with this returned memory size should be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
-		   const size_t flow_size, struct rte_flow_error *error)
+int
+mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
+			    uint64_t item_flags,
+			    uint8_t target_protocol,
+			    const struct rte_flow_item_tcp *flow_mask,
+			    struct rte_flow_error *error)
 {
-	const struct rte_flow_item_tcp *spec = item->spec;
 	const struct rte_flow_item_tcp *mask = item->mask;
-	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
-	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
-	struct ibv_flow_spec_tcp_udp tcp = {
-		.type = IBV_FLOW_SPEC_TCP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
-		.size = size,
-	};
+	const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 	int ret;
 
-	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_TCP)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	assert(flow_mask);
+	if (target_protocol != 0xff && target_protocol != IPPROTO_TCP)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "protocol filtering not compatible"
 					  " with TCP layer");
-	if (!(flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
-			      MLX5_FLOW_LAYER_OUTER_L3)))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (!(item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+				     MLX5_FLOW_LAYER_OUTER_L3)))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "L3 is mandatory to filter on L4");
-	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
-			    MLX5_FLOW_LAYER_OUTER_L4))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (item_flags & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				   MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "L4 layer is already present");
 	if (!mask)
 		mask = &rte_flow_item_tcp_mask;
 	ret = mlx5_flow_item_acceptable
 		(item, (const uint8_t *)mask,
-		 (const uint8_t *)&rte_flow_item_tcp_mask,
+		 (const uint8_t *)flow_mask,
 		 sizeof(struct rte_flow_item_tcp), error);
 	if (ret < 0)
 		return ret;
-	flow->layers |=  tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
-		MLX5_FLOW_LAYER_OUTER_L4_TCP;
-	if (spec) {
-		tcp.val.dst_port = spec->hdr.dst_port;
-		tcp.val.src_port = spec->hdr.src_port;
-		tcp.mask.dst_port = mask->hdr.dst_port;
-		tcp.mask.src_port = mask->hdr.src_port;
-		/* Remove unwanted bits from values. */
-		tcp.val.src_port &= tcp.mask.src_port;
-		tcp.val.dst_port &= tcp.mask.dst_port;
-	}
-	if (size <= flow_size) {
-		mlx5_flow_verbs_hashfields_adjust(flow, tunnel, ETH_RSS_TCP,
-						  (IBV_RX_HASH_SRC_PORT_TCP |
-						   IBV_RX_HASH_DST_PORT_TCP));
-		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
-		mlx5_flow_spec_verbs_add(flow, &tcp, size);
-	}
-	return size;
+	return 0;
 }
 
 /**
- * Convert the @p item into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
+ * Validate VXLAN item.
  *
  * @param[in] item
  *   Item specification.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
+ * @param[in] item_flags
+ *   Bit-fields that holds the items detected until now.
+ * @param[in] target_protocol
+ *   The next protocol in the previous item.
  * @param[out] error
  *   Pointer to error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p item has fully been converted,
- *   otherwise another call with this returned memory size should be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
-		     const size_t flow_size, struct rte_flow_error *error)
+int
+mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
+			      uint64_t item_flags,
+			      struct rte_flow_error *error)
 {
 	const struct rte_flow_item_vxlan *spec = item->spec;
 	const struct rte_flow_item_vxlan *mask = item->mask;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel vxlan = {
-		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
 	int ret;
 	union vni {
 		uint32_t vlan_id;
 		uint8_t vni[4];
 	} id = { .vlan_id = 0, };
+	uint32_t vlan_id = 0;
+
 
-	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+	if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
 		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "a tunnel is already present");
 	/*
 	 * Verify only UDPv4 is present as defined in
 	 * https://tools.ietf.org/html/rfc7348
 	 */
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "no outer UDP layer found");
 	if (!mask)
 		mask = &rte_flow_item_vxlan_mask;
 	ret = mlx5_flow_item_acceptable
 		(item, (const uint8_t *)mask,
 		 (const uint8_t *)&rte_flow_item_vxlan_mask,
-		 sizeof(struct rte_flow_item_vxlan), error);
+		 sizeof(struct rte_flow_item_vxlan),
+		 error);
 	if (ret < 0)
 		return ret;
 	if (spec) {
 		memcpy(&id.vni[1], spec->vni, 3);
-		vxlan.val.tunnel_id = id.vlan_id;
+		vlan_id = id.vlan_id;
 		memcpy(&id.vni[1], mask->vni, 3);
-		vxlan.mask.tunnel_id = id.vlan_id;
-		/* Remove unwanted bits from values. */
-		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+		vlan_id &= id.vlan_id;
 	}
 	/*
 	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if
@@ -1442,109 +1420,88 @@ mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
 	 * match this rule.  To avoid such situation, VNI 0 is
 	 * currently refused.
 	 */
-	if (!vxlan.val.tunnel_id)
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (!vlan_id)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "VXLAN vni cannot be 0");
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER))
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "VXLAN tunnel must be fully defined");
-	if (size <= flow_size) {
-		mlx5_flow_spec_verbs_add(flow, &vxlan, size);
-		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
-	}
-	flow->layers |= MLX5_FLOW_LAYER_VXLAN;
-	return size;
+	return 0;
 }
 
 /**
- * Convert the @p item into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
+ * Validate VXLAN_GPE item.
  *
- * @param dev
- *   Pointer to Ethernet device.
  * @param[in] item
  *   Item specification.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
+ * @param[in] item_flags
+ *   Bit-fields that holds the items detected until now.
+ * @param[in] priv
+ *   Pointer to the private data structure.
+ * @param[in] target_protocol
+ *   The next protocol in the previous item.
  * @param[out] error
  *   Pointer to error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p item has fully been converted,
- *   otherwise another call with this returned memory size should be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
-			 const struct rte_flow_item *item,
-			 struct rte_flow *flow, const size_t flow_size,
-			 struct rte_flow_error *error)
+int
+mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
+				  uint64_t item_flags,
+				  struct rte_eth_dev *dev,
+				  struct rte_flow_error *error)
 {
+	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel vxlan_gpe = {
-		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
 	int ret;
 	union vni {
 		uint32_t vlan_id;
 		uint8_t vni[4];
 	} id = { .vlan_id = 0, };
+	uint32_t vlan_id = 0;
 
-	if (!((struct priv *)dev->data->dev_private)->config.l3_vxlan_en)
+	if (!priv->config.l3_vxlan_en)
 		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "L3 VXLAN is not enabled by device"
 					  " parameter and/or not configured in"
 					  " firmware");
-	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+	if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
 		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "a tunnel is already present");
 	/*
 	 * Verify only UDPv4 is present as defined in
 	 * https://tools.ietf.org/html/rfc7348
 	 */
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "no outer UDP layer found");
 	if (!mask)
 		mask = &rte_flow_item_vxlan_gpe_mask;
 	ret = mlx5_flow_item_acceptable
 		(item, (const uint8_t *)mask,
 		 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
-		 sizeof(struct rte_flow_item_vxlan_gpe), error);
+		 sizeof(struct rte_flow_item_vxlan_gpe),
+		 error);
 	if (ret < 0)
 		return ret;
 	if (spec) {
+		if (spec->protocol)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "VxLAN-GPE protocol"
+						  " not supported");
 		memcpy(&id.vni[1], spec->vni, 3);
-		vxlan_gpe.val.tunnel_id = id.vlan_id;
+		vlan_id = id.vlan_id;
 		memcpy(&id.vni[1], mask->vni, 3);
-		vxlan_gpe.mask.tunnel_id = id.vlan_id;
-		if (spec->protocol)
-			return rte_flow_error_set
-				(error, EINVAL,
-				 RTE_FLOW_ERROR_TYPE_ITEM,
-				 item,
-				 "VxLAN-GPE protocol not supported");
-		/* Remove unwanted bits from values. */
-		vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
+		vlan_id &= id.vlan_id;
 	}
 	/*
 	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
@@ -1554,141 +1511,55 @@ mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
 	 * before will also match this rule.  To avoid such situation, VNI 0
 	 * is currently refused.
 	 */
-	if (!vxlan_gpe.val.tunnel_id)
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (!vlan_id)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "VXLAN-GPE vni cannot be 0");
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER))
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (!(item_flags & MLX5_FLOW_LAYER_OUTER))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "VXLAN-GPE tunnel must be fully"
 					  " defined");
-	if (size <= flow_size) {
-		mlx5_flow_spec_verbs_add(flow, &vxlan_gpe, size);
-		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
-	}
-	flow->layers |= MLX5_FLOW_LAYER_VXLAN_GPE;
-	return size;
-}
-
-/**
- * Update the protocol in Verbs IPv4/IPv6 spec.
- *
- * @param[in, out] attr
- *   Pointer to Verbs attributes structure.
- * @param[in] search
- *   Specification type to search in order to update the IP protocol.
- * @param[in] protocol
- *   Protocol value to set if none is present in the specification.
- */
-static void
-mlx5_flow_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
-				      enum ibv_flow_spec_type search,
-				      uint8_t protocol)
-{
-	unsigned int i;
-	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
-		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
-
-	if (!attr)
-		return;
-	for (i = 0; i != attr->num_of_specs; ++i) {
-		if (hdr->type == search) {
-			union {
-				struct ibv_flow_spec_ipv4_ext *ipv4;
-				struct ibv_flow_spec_ipv6 *ipv6;
-			} ip;
-
-			switch (search) {
-			case IBV_FLOW_SPEC_IPV4_EXT:
-				ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
-				if (!ip.ipv4->val.proto) {
-					ip.ipv4->val.proto = protocol;
-					ip.ipv4->mask.proto = 0xff;
-				}
-				break;
-			case IBV_FLOW_SPEC_IPV6:
-				ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
-				if (!ip.ipv6->val.next_hdr) {
-					ip.ipv6->val.next_hdr = protocol;
-					ip.ipv6->mask.next_hdr = 0xff;
-				}
-				break;
-			default:
-				break;
-			}
-			break;
-		}
-		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
-	}
+	return 0;
 }
 
 /**
- * Convert the @p item into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * It will also update the previous L3 layer with the protocol value matching
- * the GRE.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
+ * Validate GRE item.
  *
- * @param dev
- *   Pointer to Ethernet device.
  * @param[in] item
  *   Item specification.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
+ * @param[in] item_flags
+ *   Bit flags to mark detected items.
+ * @param[in] target_protocol
+ *   The next protocol in the previous item.
  * @param[out] error
  *   Pointer to error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p item has fully been converted,
- *   otherwise another call with this returned memory size should be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_item_gre(const struct rte_flow_item *item,
-		   struct rte_flow *flow, const size_t flow_size,
-		   struct rte_flow_error *error)
+int
+mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
+			    uint64_t item_flags,
+			    uint8_t target_protocol,
+			    struct rte_flow_error *error)
 {
-	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
-	const struct rte_flow_item_gre *spec = item->spec;
+	const struct rte_flow_item_gre *spec __rte_unused = item->spec;
 	const struct rte_flow_item_gre *mask = item->mask;
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	unsigned int size = sizeof(struct ibv_flow_spec_gre);
-	struct ibv_flow_spec_gre tunnel = {
-		.type = IBV_FLOW_SPEC_GRE,
-		.size = size,
-	};
-#else
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel tunnel = {
-		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-#endif
 	int ret;
 
-	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_GRE)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (target_protocol != 0xff && target_protocol != IPPROTO_GRE)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "protocol filtering not compatible"
 					  " with this GRE layer");
-	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+	if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
 		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "a tunnel is already present");
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+	if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3))
 		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "L3 Layer is missing");
 	if (!mask)
 		mask = &rte_flow_item_gre_mask;
@@ -1698,92 +1569,50 @@ mlx5_flow_item_gre(const struct rte_flow_item *item,
 		 sizeof(struct rte_flow_item_gre), error);
 	if (ret < 0)
 		return ret;
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	if (spec) {
-		tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
-		tunnel.val.protocol = spec->protocol;
-		tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
-		tunnel.mask.protocol = mask->protocol;
-		/* Remove unwanted bits from values. */
-		tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
-		tunnel.val.protocol &= tunnel.mask.protocol;
-		tunnel.val.key &= tunnel.mask.key;
-	}
-#else
+#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 	if (spec && (spec->protocol & mask->protocol))
 		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "without MPLS support the"
 					  " specification cannot be used for"
 					  " filtering");
-#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
-	if (size <= flow_size) {
-		if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
-			mlx5_flow_item_gre_ip_protocol_update
-				(verbs->attr, IBV_FLOW_SPEC_IPV4_EXT,
-				 MLX5_IP_PROTOCOL_GRE);
-		else
-			mlx5_flow_item_gre_ip_protocol_update
-				(verbs->attr, IBV_FLOW_SPEC_IPV6,
-				 MLX5_IP_PROTOCOL_GRE);
-		mlx5_flow_spec_verbs_add(flow, &tunnel, size);
-		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
-	}
-	flow->layers |= MLX5_FLOW_LAYER_GRE;
-	return size;
+#endif
+	return 0;
 }
 
 /**
- * Convert the @p item into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
+ * Validate MPLS item.
  *
  * @param[in] item
  *   Item specification.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
+ * @param[in] item_flags
+ *   Bit-fields that holds the items detected until now.
+ * @param[in] target_protocol
+ *   The next protocol in the previous item.
  * @param[out] error
  *   Pointer to error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p item has fully been converted,
- *   otherwise another call with this returned memory size should be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_item_mpls(const struct rte_flow_item *item __rte_unused,
-		    struct rte_flow *flow __rte_unused,
-		    const size_t flow_size __rte_unused,
-		    struct rte_flow_error *error)
+int
+mlx5_flow_validate_item_mpls(const struct rte_flow_item *item __rte_unused,
+			     uint64_t item_flags __rte_unused,
+			     uint8_t target_protocol __rte_unused,
+			     struct rte_flow_error *error)
 {
 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	const struct rte_flow_item_mpls *spec = item->spec;
 	const struct rte_flow_item_mpls *mask = item->mask;
-	unsigned int size = sizeof(struct ibv_flow_spec_mpls);
-	struct ibv_flow_spec_mpls mpls = {
-		.type = IBV_FLOW_SPEC_MPLS,
-		.size = size,
-	};
 	int ret;
 
-	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_MPLS)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+	if (target_protocol != 0xff && target_protocol != IPPROTO_MPLS)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "protocol filtering not compatible"
 					  " with MPLS layer");
-	/* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
-	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL &&
-	    (flow->layers & MLX5_FLOW_LAYER_GRE) != MLX5_FLOW_LAYER_GRE)
+	if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
 		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
 					  "a tunnel is already"
 					  " present");
 	if (!mask)
@@ -1794,1056 +1623,298 @@ mlx5_flow_item_mpls(const struct rte_flow_item *item __rte_unused,
 		 sizeof(struct rte_flow_item_mpls), error);
 	if (ret < 0)
 		return ret;
-	if (spec) {
-		memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
-		memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
-		/* Remove unwanted bits from values.  */
-		mpls.val.label &= mpls.mask.label;
-	}
-	if (size <= flow_size) {
-		mlx5_flow_spec_verbs_add(flow, &mpls, size);
-		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
-	}
-	flow->layers |= MLX5_FLOW_LAYER_MPLS;
-	return size;
-#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
+	return 0;
+#endif
 	return rte_flow_error_set(error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_ITEM,
-				  item,
+				  RTE_FLOW_ERROR_TYPE_ITEM, item,
 				  "MPLS is not supported by Verbs, please"
 				  " update.");
 }
 
-/**
- * Convert the @p pattern into a Verbs specifications after ensuring the NIC
- * will understand and process it correctly.
- * The conversion is performed item per item, each of them is written into
- * the @p flow if its size is lesser or equal to @p flow_size.
- * Validation and memory consumption computation are still performed until the
- * end of @p pattern, unless an error is encountered.
- *
- * @param[in] pattern
- *   Flow pattern.
- * @param[in, out] flow
- *   Pointer to the rte_flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small some
- *   garbage may be present.
- * @param[out] error
- *   Pointer to error structure.
- *
- * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @pattern  has fully been
- *   converted, otherwise another call with this returned memory size should
- *   be done.
- *   On error, a negative errno value is returned and rte_errno is set.
- */
 static int
-mlx5_flow_items(struct rte_eth_dev *dev,
-		const struct rte_flow_item pattern[],
-		struct rte_flow *flow, const size_t flow_size,
-		struct rte_flow_error *error)
+flow_null_validate(struct rte_eth_dev *dev __rte_unused,
+		   const struct rte_flow_attr *attr __rte_unused,
+		   const struct rte_flow_item items[] __rte_unused,
+		   const struct rte_flow_action actions[] __rte_unused,
+		   struct rte_flow_error *error __rte_unused)
 {
-	int remain = flow_size;
-	size_t size = 0;
-
-	for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
-		int ret = 0;
-
-		switch (pattern->type) {
-		case RTE_FLOW_ITEM_TYPE_VOID:
-			break;
-		case RTE_FLOW_ITEM_TYPE_ETH:
-			ret = mlx5_flow_item_eth(pattern, flow, remain, error);
-			break;
-		case RTE_FLOW_ITEM_TYPE_VLAN:
-			ret = mlx5_flow_item_vlan(pattern, flow, remain, error);
-			break;
-		case RTE_FLOW_ITEM_TYPE_IPV4:
-			ret = mlx5_flow_item_ipv4(pattern, flow, remain, error);
-			break;
-		case RTE_FLOW_ITEM_TYPE_IPV6:
-			ret = mlx5_flow_item_ipv6(pattern, flow, remain, error);
-			break;
-		case RTE_FLOW_ITEM_TYPE_UDP:
-			ret = mlx5_flow_item_udp(pattern, flow, remain, error);
-			break;
-		case RTE_FLOW_ITEM_TYPE_TCP:
-			ret = mlx5_flow_item_tcp(pattern, flow, remain, error);
-			break;
-		case RTE_FLOW_ITEM_TYPE_VXLAN:
-			ret = mlx5_flow_item_vxlan(pattern, flow, remain,
-						   error);
-			break;
-		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
-			ret = mlx5_flow_item_vxlan_gpe(dev, pattern, flow,
-						       remain, error);
-			break;
-		case RTE_FLOW_ITEM_TYPE_GRE:
-			ret = mlx5_flow_item_gre(pattern, flow, remain, error);
-			break;
-		case RTE_FLOW_ITEM_TYPE_MPLS:
-			ret = mlx5_flow_item_mpls(pattern, flow, remain, error);
-			break;
-		default:
-			return rte_flow_error_set(error, ENOTSUP,
-						  RTE_FLOW_ERROR_TYPE_ITEM,
-						  pattern,
-						  "item not supported");
-		}
-		if (ret < 0)
-			return ret;
-		if (remain > ret)
-			remain -= ret;
-		else
-			remain = 0;
-		size += ret;
-	}
-	if (!flow->layers) {
-		const struct rte_flow_item item = {
-			.type = RTE_FLOW_ITEM_TYPE_ETH,
-		};
-
-		return mlx5_flow_item_eth(&item, flow, flow_size, error);
-	}
-	return size;
+	rte_errno = ENOTSUP;
+	return -rte_errno;
 }
 
-/**
- * Convert the @p action into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
- *
- * @param[in] action
- *   Action configuration.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
- * @param[out] error
- *   Pointer to error structure.
- *
- * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p action has fully been
- *   converted, otherwise another call with this returned memory size should
- *   be done.
- *   On error, a negative errno value is returned and rte_errno is set.
- */
-static int
-mlx5_flow_action_drop(const struct rte_flow_action *action,
-		      struct rte_flow *flow, const size_t flow_size,
-		      struct rte_flow_error *error)
+static struct mlx5_flow *
+flow_null_prepare(const struct rte_flow_attr *attr __rte_unused,
+		  const struct rte_flow_item items[] __rte_unused,
+		  const struct rte_flow_action actions[] __rte_unused,
+		  uint64_t *item_flags __rte_unused,
+		  uint64_t *action_flags __rte_unused,
+		  struct rte_flow_error *error __rte_unused)
 {
-	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
-	struct ibv_flow_spec_action_drop drop = {
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-			.size = size,
-	};
-
-	if (flow->fate)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  action,
-					  "multiple fate actions are not"
-					  " supported");
-	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  action,
-					  "drop is not compatible with"
-					  " flag/mark action");
-	if (size < flow_size)
-		mlx5_flow_spec_verbs_add(flow, &drop, size);
-	flow->fate |= MLX5_FLOW_FATE_DROP;
-	return size;
+	rte_errno = ENOTSUP;
+	return NULL;
 }
 
-/**
- * Convert the @p action into @p flow after ensuring the NIC will understand
- * and process it correctly.
- *
- * @param[in] dev
- *   Pointer to Ethernet device structure.
- * @param[in] action
- *   Action configuration.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[out] error
- *   Pointer to error structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
 static int
-mlx5_flow_action_queue(struct rte_eth_dev *dev,
-		       const struct rte_flow_action *action,
-		       struct rte_flow *flow,
-		       struct rte_flow_error *error)
+flow_null_translate(struct rte_eth_dev *dev __rte_unused,
+		    struct mlx5_flow *dev_flow __rte_unused,
+		    const struct rte_flow_attr *attr __rte_unused,
+		    const struct rte_flow_item items[] __rte_unused,
+		    const struct rte_flow_action actions[] __rte_unused,
+		    struct rte_flow_error *error __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	const struct rte_flow_action_queue *queue = action->conf;
-
-	if (flow->fate)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  action,
-					  "multiple fate actions are not"
-					  " supported");
-	if (queue->index >= priv->rxqs_n)
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-					  &queue->index,
-					  "queue index out of range");
-	if (!(*priv->rxqs)[queue->index])
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-					  &queue->index,
-					  "queue is not configured");
-	if (flow->queue)
-		(*flow->queue)[0] = queue->index;
-	flow->rss.queue_num = 1;
-	flow->fate |= MLX5_FLOW_FATE_QUEUE;
-	return 0;
+	rte_errno = ENOTSUP;
+	return -rte_errno;
 }
 
-/**
- * Ensure the @p action will be understood and used correctly by the  NIC.
- *
- * @param dev
- *   Pointer to Ethernet device structure.
- * @param action[in]
- *   Pointer to flow actions array.
- * @param flow[in, out]
- *   Pointer to the rte_flow structure.
- * @param error[in, out]
- *   Pointer to error structure.
- *
- * @return
- *   On success @p flow->queue array and @p flow->rss are filled and valid.
- *   On error, a negative errno value is returned and rte_errno is set.
- */
 static int
-mlx5_flow_action_rss(struct rte_eth_dev *dev,
-		     const struct rte_flow_action *action,
-		     struct rte_flow *flow,
-		     struct rte_flow_error *error)
+flow_null_apply(struct rte_eth_dev *dev __rte_unused,
+		struct rte_flow *flow __rte_unused,
+		struct rte_flow_error *error __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	const struct rte_flow_action_rss *rss = action->conf;
-	unsigned int i;
-
-	if (flow->fate)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  action,
-					  "multiple fate actions are not"
-					  " supported");
-	if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
-	    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-					  &rss->func,
-					  "RSS hash function not supported");
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	if (rss->level > 2)
-#else
-	if (rss->level > 1)
-#endif
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-					  &rss->level,
-					  "tunnel RSS is not supported");
-	if (rss->key_len < MLX5_RSS_HASH_KEY_LEN)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-					  &rss->key_len,
-					  "RSS hash key too small");
-	if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-					  &rss->key_len,
-					  "RSS hash key too large");
-	if (!rss->queue_num)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-					  rss,
-					  "no queues were provided for RSS");
-	if (rss->queue_num > priv->config.ind_table_max_size)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-					  &rss->queue_num,
-					  "number of queues too large");
-	if (rss->types & MLX5_RSS_HF_MASK)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-					  &rss->types,
-					  "some RSS protocols are not"
-					  " supported");
-	for (i = 0; i != rss->queue_num; ++i) {
-		if (rss->queue[i] >= priv->rxqs_n)
-			return rte_flow_error_set
-				(error, EINVAL,
-				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-				 rss,
-				 "queue index out of range");
-		if (!(*priv->rxqs)[rss->queue[i]])
-			return rte_flow_error_set
-				(error, EINVAL,
-				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-				 &rss->queue[i],
-				 "queue is not configured");
-	}
-	if (flow->queue)
-		memcpy((*flow->queue), rss->queue,
-		       rss->queue_num * sizeof(uint16_t));
-	flow->rss.queue_num = rss->queue_num;
-	memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
-	flow->rss.types = rss->types;
-	flow->rss.level = rss->level;
-	flow->fate |= MLX5_FLOW_FATE_RSS;
-	return 0;
+	rte_errno = ENOTSUP;
+	return -rte_errno;
 }
 
-/**
- * Convert the @p action into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
- *
- * @param[in] action
- *   Action configuration.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
- * @param[out] error
- *   Pointer to error structure.
- *
- * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p action has fully been
- *   converted, otherwise another call with this returned memory size should
- *   be done.
- *   On error, a negative errno value is returned and rte_errno is set.
- */
-static int
-mlx5_flow_action_flag(const struct rte_flow_action *action,
-		      struct rte_flow *flow, const size_t flow_size,
-		      struct rte_flow_error *error)
-{
-	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
-	struct ibv_flow_spec_action_tag tag = {
-		.type = IBV_FLOW_SPEC_ACTION_TAG,
-		.size = size,
-		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
-	};
-	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
-
-	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  action,
-					  "flag action already present");
-	if (flow->fate & MLX5_FLOW_FATE_DROP)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  action,
-					  "flag is not compatible with drop"
-					  " action");
-	if (flow->modifier & MLX5_FLOW_MOD_MARK)
-		size = 0;
-	else if (size <= flow_size && verbs)
-		mlx5_flow_spec_verbs_add(flow, &tag, size);
-	flow->modifier |= MLX5_FLOW_MOD_FLAG;
-	return size;
+static void
+flow_null_remove(struct rte_eth_dev *dev __rte_unused,
+		 struct rte_flow *flow __rte_unused)
+{
 }
 
-/**
- * Update verbs specification to modify the flag to mark.
- *
- * @param[in, out] verbs
- *   Pointer to the mlx5_flow_verbs structure.
- * @param[in] mark_id
- *   Mark identifier to replace the flag.
- */
 static void
-mlx5_flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
+flow_null_destroy(struct rte_eth_dev *dev __rte_unused,
+		  struct rte_flow *flow __rte_unused)
 {
-	struct ibv_spec_header *hdr;
-	int i;
-
-	if (!verbs)
-		return;
-	/* Update Verbs specification. */
-	hdr = (struct ibv_spec_header *)verbs->specs;
-	if (!hdr)
-		return;
-	for (i = 0; i != verbs->attr->num_of_specs; ++i) {
-		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
-			struct ibv_flow_spec_action_tag *t =
-				(struct ibv_flow_spec_action_tag *)hdr;
-
-			t->tag_id = mlx5_flow_mark_set(mark_id);
-		}
-		hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
-	}
 }
 
-/**
- * Convert the @p action into @p flow (or by updating the already present
- * Flag Verbs specification) after ensuring the NIC will understand and
- * process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
- *
- * @param[in] action
- *   Action configuration.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
- * @param[out] error
- *   Pointer to error structure.
- *
- * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p action has fully been
- *   converted, otherwise another call with this returned memory size should
- *   be done.
- *   On error, a negative errno value is returned and rte_errno is set.
- */
 static int
-mlx5_flow_action_mark(const struct rte_flow_action *action,
-		      struct rte_flow *flow, const size_t flow_size,
-		      struct rte_flow_error *error)
+flow_null_query(struct rte_eth_dev *dev __rte_unused,
+		struct rte_flow *flow __rte_unused,
+		const struct rte_flow_action *actions __rte_unused,
+		void *data __rte_unused,
+		struct rte_flow_error *error __rte_unused)
 {
-	const struct rte_flow_action_mark *mark = action->conf;
-	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
-	struct ibv_flow_spec_action_tag tag = {
-		.type = IBV_FLOW_SPEC_ACTION_TAG,
-		.size = size,
-	};
-	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
-
-	if (!mark)
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  action,
-					  "configuration cannot be null");
-	if (mark->id >= MLX5_FLOW_MARK_MAX)
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-					  &mark->id,
-					  "mark id must in 0 <= id < "
-					  RTE_STR(MLX5_FLOW_MARK_MAX));
-	if (flow->modifier & MLX5_FLOW_MOD_MARK)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  action,
-					  "mark action already present");
-	if (flow->fate & MLX5_FLOW_FATE_DROP)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  action,
-					  "mark is not compatible with drop"
-					  " action");
-	if (flow->modifier & MLX5_FLOW_MOD_FLAG) {
-		mlx5_flow_verbs_mark_update(verbs, mark->id);
-		size = 0;
-	} else if (size <= flow_size) {
-		tag.tag_id = mlx5_flow_mark_set(mark->id);
-		mlx5_flow_spec_verbs_add(flow, &tag, size);
-	}
-	flow->modifier |= MLX5_FLOW_MOD_MARK;
-	return size;
+	rte_errno = ENOTSUP;
+	return -rte_errno;
 }
 
+/* Void driver to protect from null pointer reference. */
+const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = {
+	.validate = flow_null_validate,
+	.prepare = flow_null_prepare,
+	.translate = flow_null_translate,
+	.apply = flow_null_apply,
+	.remove = flow_null_remove,
+	.destroy = flow_null_destroy,
+	.query = flow_null_query,
+};
+
 /**
- * Convert the @p action into a Verbs specification after ensuring the NIC
- * will understand and process it correctly.
- * If the necessary size for the conversion is greater than the @p flow_size,
- * nothing is written in @p flow, the validation is still performed.
- *
- * @param action[in]
- *   Action configuration.
- * @param flow[in, out]
- *   Pointer to flow structure.
- * @param flow_size[in]
- *   Size in bytes of the available space in @p flow, if too small, nothing is
- *   written.
- * @param error[int, out]
- *   Pointer to error structure.
+ * Select flow driver type according to flow attributes and device
+ * configuration.
+ *
+ * @param[in] dev
+ *   Pointer to the dev structure.
+ * @param[in] attr
+ *   Pointer to the flow attributes.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p action has fully been
- *   converted, otherwise another call with this returned memory size should
- *   be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   flow driver type, MLX5_FLOW_TYPE_MAX otherwise.
  */
-static int
-mlx5_flow_action_count(struct rte_eth_dev *dev,
-		       const struct rte_flow_action *action,
-		       struct rte_flow *flow,
-		       const size_t flow_size __rte_unused,
-		       struct rte_flow_error *error)
-{
-	const struct rte_flow_action_count *count = action->conf;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-	struct ibv_flow_spec_counter_action counter = {
-		.type = IBV_FLOW_SPEC_ACTION_COUNT,
-		.size = size,
-	};
-#endif
+static enum mlx5_flow_drv_type
+flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr)
+{
+	struct priv *priv = dev->data->dev_private;
+	enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX;
 
-	if (!flow->counter) {
-		flow->counter = mlx5_flow_counter_new(dev, count->shared,
-						      count->id);
-		if (!flow->counter)
-			return rte_flow_error_set(error, ENOTSUP,
-						  RTE_FLOW_ERROR_TYPE_ACTION,
-						  action,
-						  "cannot get counter"
-						  " context.");
-	}
-	if (!((struct priv *)dev->data->dev_private)->config.flow_counter_en)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  action,
-					  "flow counters are not supported.");
-	flow->modifier |= MLX5_FLOW_MOD_COUNT;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	counter.counter_set_handle = flow->counter->cs->handle;
-	if (size <= flow_size)
-		mlx5_flow_spec_verbs_add(flow, &counter, size);
-	return size;
-#endif
-	return 0;
+	if (attr->transfer)
+		type = MLX5_FLOW_TYPE_TCF;
+	else
+		type = priv->config.dv_flow_en ? MLX5_FLOW_TYPE_DV :
+						 MLX5_FLOW_TYPE_VERBS;
+	return type;
 }
 
+#define flow_get_drv_ops(type) flow_drv_ops[type]
+
 /**
- * Convert the @p action into @p flow after ensuring the NIC will understand
- * and process it correctly.
- * The conversion is performed action per action, each of them is written into
- * the @p flow if its size is lesser or equal to @p flow_size.
- * Validation and memory consumption computation are still performed until the
- * end of @p action, unless an error is encountered.
+ * Flow driver validation API. This abstracts calling driver specific functions.
+ * The type of flow driver is determined according to flow attributes.
  *
  * @param[in] dev
- *   Pointer to Ethernet device structure.
+ *   Pointer to the dev structure.
+ * @param[in] attr
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
  * @param[in] actions
- *   Pointer to flow actions array.
- * @param[in, out] flow
- *   Pointer to the rte_flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small some
- *   garbage may be present.
+ *   Pointer to the list of actions.
  * @param[out] error
- *   Pointer to error structure.
+ *   Pointer to the error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the @p actions has fully been
- *   converted, otherwise another call with this returned memory size should
- *   be done.
- *   On error, a negative errno value is returned and rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_ernno is set.
  */
-static int
-mlx5_flow_actions(struct rte_eth_dev *dev,
+static inline int
+flow_drv_validate(struct rte_eth_dev *dev,
+		  const struct rte_flow_attr *attr,
+		  const struct rte_flow_item items[],
 		  const struct rte_flow_action actions[],
-		  struct rte_flow *flow, const size_t flow_size,
 		  struct rte_flow_error *error)
 {
-	size_t size = 0;
-	int remain = flow_size;
-	int ret = 0;
+	const struct mlx5_flow_driver_ops *fops;
+	enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr);
 
-	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
-		switch (actions->type) {
-		case RTE_FLOW_ACTION_TYPE_VOID:
-			break;
-		case RTE_FLOW_ACTION_TYPE_FLAG:
-			ret = mlx5_flow_action_flag(actions, flow, remain,
-						    error);
-			break;
-		case RTE_FLOW_ACTION_TYPE_MARK:
-			ret = mlx5_flow_action_mark(actions, flow, remain,
-						    error);
-			break;
-		case RTE_FLOW_ACTION_TYPE_DROP:
-			ret = mlx5_flow_action_drop(actions, flow, remain,
-						    error);
-			break;
-		case RTE_FLOW_ACTION_TYPE_QUEUE:
-			ret = mlx5_flow_action_queue(dev, actions, flow, error);
-			break;
-		case RTE_FLOW_ACTION_TYPE_RSS:
-			ret = mlx5_flow_action_rss(dev, actions, flow, error);
-			break;
-		case RTE_FLOW_ACTION_TYPE_COUNT:
-			ret = mlx5_flow_action_count(dev, actions, flow, remain,
-						     error);
-			break;
-		default:
-			return rte_flow_error_set(error, ENOTSUP,
-						  RTE_FLOW_ERROR_TYPE_ACTION,
-						  actions,
-						  "action not supported");
-		}
-		if (ret < 0)
-			return ret;
-		if (remain > ret)
-			remain -= ret;
-		else
-			remain = 0;
-		size += ret;
-	}
-	if (!flow->fate)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "no fate action found");
-	return size;
+	fops = flow_get_drv_ops(type);
+	return fops->validate(dev, attr, items, actions, error);
 }
 
 /**
- * Validate flow rule and fill flow structure accordingly.
+ * Flow driver preparation API. This abstracts calling driver specific
+ * functions. Parent flow (rte_flow) should have driver type (drv_type). It
+ * calculates the size of memory required for device flow, allocates the memory,
+ * initializes the device flow and returns the pointer.
  *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[out] flow
- *   Pointer to flow structure.
- * @param flow_size
- *   Size of allocated space for @p flow.
  * @param[in] attr
- *   Flow rule attributes.
- * @param[in] pattern
- *   Pattern specification (list terminated by the END pattern item).
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
  * @param[in] actions
- *   Associated actions (list terminated by the END action).
+ *   Pointer to the list of actions.
+ * @param[out] item_flags
+ *   Pointer to bit mask of all items detected.
+ * @param[out] action_flags
+ *   Pointer to bit mask of all actions detected.
  * @param[out] error
- *   Perform verbose error reporting if not NULL.
+ *   Pointer to the error structure.
  *
  * @return
- *   A positive value representing the size of the flow object in bytes
- *   regardless of @p flow_size on success, a negative errno value otherwise
- *   and rte_errno is set.
+ *   Pointer to device flow on success, otherwise NULL and rte_ernno is set.
  */
-static int
-mlx5_flow_merge_switch(struct rte_eth_dev *dev,
-		       struct rte_flow *flow,
-		       size_t flow_size,
-		       const struct rte_flow_attr *attr,
-		       const struct rte_flow_item pattern[],
-		       const struct rte_flow_action actions[],
-		       struct rte_flow_error *error)
-{
-	unsigned int n = mlx5_dev_to_port_id(dev->device, NULL, 0);
-	uint16_t port_id[!n + n];
-	struct mlx5_nl_flow_ptoi ptoi[!n + n + 1];
-	size_t off = RTE_ALIGN_CEIL(sizeof(*flow), alignof(max_align_t));
-	unsigned int i;
-	unsigned int own = 0;
-	int ret;
-
-	/* At least one port is needed when no switch domain is present. */
-	if (!n) {
-		n = 1;
-		port_id[0] = dev->data->port_id;
-	} else {
-		n = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, n), n);
-	}
-	for (i = 0; i != n; ++i) {
-		struct rte_eth_dev_info dev_info;
-
-		rte_eth_dev_info_get(port_id[i], &dev_info);
-		if (port_id[i] == dev->data->port_id)
-			own = i;
-		ptoi[i].port_id = port_id[i];
-		ptoi[i].ifindex = dev_info.if_index;
-	}
-	/* Ensure first entry of ptoi[] is the current device. */
-	if (own) {
-		ptoi[n] = ptoi[0];
-		ptoi[0] = ptoi[own];
-		ptoi[own] = ptoi[n];
-	}
-	/* An entry with zero ifindex terminates ptoi[]. */
-	ptoi[n].port_id = 0;
-	ptoi[n].ifindex = 0;
-	if (flow_size < off)
-		flow_size = 0;
-	ret = mlx5_nl_flow_transpose((uint8_t *)flow + off,
-				     flow_size ? flow_size - off : 0,
-				     ptoi, attr, pattern, actions, error);
-	if (ret < 0)
-		return ret;
-	if (flow_size) {
-		*flow = (struct rte_flow){
-			.attributes = *attr,
-			.nl_flow = (uint8_t *)flow + off,
-		};
-		/*
-		 * Generate a reasonably unique handle based on the address
-		 * of the target buffer.
-		 *
-		 * This is straightforward on 32-bit systems where the flow
-		 * pointer can be used directly. Otherwise, its least
-		 * significant part is taken after shifting it by the
-		 * previous power of two of the pointed buffer size.
-		 */
-		if (sizeof(flow) <= 4)
-			mlx5_nl_flow_brand(flow->nl_flow, (uintptr_t)flow);
-		else
-			mlx5_nl_flow_brand
-				(flow->nl_flow,
-				 (uintptr_t)flow >>
-				 rte_log2_u32(rte_align32prevpow2(flow_size)));
-	}
-	return off + ret;
-}
-
-static unsigned int
-mlx5_find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
+static inline struct mlx5_flow *
+flow_drv_prepare(struct rte_flow *flow,
+		 const struct rte_flow_attr *attr,
+		 const struct rte_flow_item items[],
+		 const struct rte_flow_action actions[],
+		 uint64_t *item_flags,
+		 uint64_t *action_flags,
+		 struct rte_flow_error *error)
 {
-	const struct rte_flow_item *item;
-	unsigned int has_vlan = 0;
+	const struct mlx5_flow_driver_ops *fops;
+	enum mlx5_flow_drv_type type = flow->drv_type;
 
-	for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
-		if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
-			has_vlan = 1;
-			break;
-		}
-	}
-	if (has_vlan)
-		return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
-				       MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
-	return rss_level < 2 ? MLX5_EXPANSION_ROOT :
-			       MLX5_EXPANSION_ROOT_OUTER;
+	assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
+	fops = flow_get_drv_ops(type);
+	return fops->prepare(attr, items, actions, item_flags, action_flags,
+			     error);
 }
 
 /**
- * Convert the @p attributes, @p pattern, @p action, into an flow for the NIC
- * after ensuring the NIC will understand and process it correctly.
- * The conversion is only performed item/action per item/action, each of
- * them is written into the @p flow if its size is lesser or equal to @p
- * flow_size.
- * Validation and memory consumption computation are still performed until the
- * end, unless an error is encountered.
+ * Flow driver translation API. This abstracts calling driver specific
+ * functions. Parent flow (rte_flow) should have driver type (drv_type). It
+ * translates a generic flow into a driver flow. flow_drv_prepare() must
+ * precede.
+ *
  *
  * @param[in] dev
- *   Pointer to Ethernet device.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[in] flow_size
- *   Size in bytes of the available space in @p flow, if too small some
- *   garbage may be present.
- * @param[in] attributes
- *   Flow rule attributes.
- * @param[in] pattern
- *   Pattern specification (list terminated by the END pattern item).
+ *   Pointer to the rte dev structure.
+ * @param[in, out] dev_flow
+ *   Pointer to the mlx5 flow.
+ * @param[in] attr
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
  * @param[in] actions
- *   Associated actions (list terminated by the END action).
+ *   Pointer to the list of actions.
  * @param[out] error
- *   Perform verbose error reporting if not NULL.
+ *   Pointer to the error structure.
  *
  * @return
- *   On success the number of bytes consumed/necessary, if the returned value
- *   is lesser or equal to @p flow_size, the flow has fully been converted and
- *   can be applied, otherwise another call with this returned memory size
- *   should be done.
- *   On error, a negative errno value is returned and rte_errno is set.
- */
-static int
-mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
-		const size_t flow_size,
-		const struct rte_flow_attr *attributes,
-		const struct rte_flow_item pattern[],
-		const struct rte_flow_action actions[],
-		struct rte_flow_error *error)
-{
-	struct rte_flow local_flow = { .layers = 0, };
-	size_t size = sizeof(*flow);
-	union {
-		struct rte_flow_expand_rss buf;
-		uint8_t buffer[2048];
-	} expand_buffer;
-	struct rte_flow_expand_rss *buf = &expand_buffer.buf;
-	struct mlx5_flow_verbs *original_verbs = NULL;
-	size_t original_verbs_size = 0;
-	uint32_t original_layers = 0;
-	int expanded_pattern_idx = 0;
-	int ret;
-	uint32_t i;
-
-	if (attributes->transfer)
-		return mlx5_flow_merge_switch(dev, flow, flow_size,
-					      attributes, pattern,
-					      actions, error);
-	if (size > flow_size)
-		flow = &local_flow;
-	ret = mlx5_flow_attributes(dev, attributes, flow, error);
-	if (ret < 0)
-		return ret;
-	ret = mlx5_flow_actions(dev, actions, &local_flow, 0, error);
-	if (ret < 0)
-		return ret;
-	if (local_flow.rss.types) {
-		unsigned int graph_root;
-
-		graph_root = mlx5_find_graph_root(pattern,
-						  local_flow.rss.level);
-		ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
-					  pattern, local_flow.rss.types,
-					  mlx5_support_expansion,
-					  graph_root);
-		assert(ret > 0 &&
-		       (unsigned int)ret < sizeof(expand_buffer.buffer));
-	} else {
-		buf->entries = 1;
-		buf->entry[0].pattern = (void *)(uintptr_t)pattern;
-	}
-	size += RTE_ALIGN_CEIL(local_flow.rss.queue_num * sizeof(uint16_t),
-			       sizeof(void *));
-	if (size <= flow_size)
-		flow->queue = (void *)(flow + 1);
-	LIST_INIT(&flow->verbs);
-	flow->layers = 0;
-	flow->modifier = 0;
-	flow->fate = 0;
-	for (i = 0; i != buf->entries; ++i) {
-		size_t off = size;
-		size_t off2;
-
-		flow->layers = original_layers;
-		size += sizeof(struct ibv_flow_attr) +
-			sizeof(struct mlx5_flow_verbs);
-		off2 = size;
-		if (size < flow_size) {
-			flow->cur_verbs = (void *)((uintptr_t)flow + off);
-			flow->cur_verbs->attr = (void *)(flow->cur_verbs + 1);
-			flow->cur_verbs->specs =
-				(void *)(flow->cur_verbs->attr + 1);
-		}
-		/* First iteration convert the pattern into Verbs. */
-		if (i == 0) {
-			/* Actions don't need to be converted several time. */
-			ret = mlx5_flow_actions(dev, actions, flow,
-						(size < flow_size) ?
-						flow_size - size : 0,
-						error);
-			if (ret < 0)
-				return ret;
-			size += ret;
-		} else {
-			/*
-			 * Next iteration means the pattern has already been
-			 * converted and an expansion is necessary to match
-			 * the user RSS request.  For that only the expanded
-			 * items will be converted, the common part with the
-			 * user pattern are just copied into the next buffer
-			 * zone.
-			 */
-			size += original_verbs_size;
-			if (size < flow_size) {
-				rte_memcpy(flow->cur_verbs->attr,
-					   original_verbs->attr,
-					   original_verbs_size +
-					   sizeof(struct ibv_flow_attr));
-				flow->cur_verbs->size = original_verbs_size;
-			}
-		}
-		ret = mlx5_flow_items
-			(dev,
-			 (const struct rte_flow_item *)
-			 &buf->entry[i].pattern[expanded_pattern_idx],
-			 flow,
-			 (size < flow_size) ? flow_size - size : 0, error);
-		if (ret < 0)
-			return ret;
-		size += ret;
-		if (size <= flow_size) {
-			mlx5_flow_adjust_priority(dev, flow);
-			LIST_INSERT_HEAD(&flow->verbs, flow->cur_verbs, next);
-		}
-		/*
-		 * Keep a pointer of the first verbs conversion and the layers
-		 * it has encountered.
-		 */
-		if (i == 0) {
-			original_verbs = flow->cur_verbs;
-			original_verbs_size = size - off2;
-			original_layers = flow->layers;
-			/*
-			 * move the index of the expanded pattern to the
-			 * first item not addressed yet.
-			 */
-			if (pattern->type == RTE_FLOW_ITEM_TYPE_END) {
-				expanded_pattern_idx++;
-			} else {
-				const struct rte_flow_item *item = pattern;
-
-				for (item = pattern;
-				     item->type != RTE_FLOW_ITEM_TYPE_END;
-				     ++item)
-					expanded_pattern_idx++;
-			}
-		}
-	}
-	/* Restore the origin layers in the flow. */
-	flow->layers = original_layers;
-	return size;
-}
-
-/**
- * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
- * if several tunnel rules are used on this queue, the tunnel ptype will be
- * cleared.
- *
- * @param rxq_ctrl
- *   Rx queue to update.
+ *   0 on success, a negative errno value otherwise and rte_ernno is set.
  */
-static void
-mlx5_flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
+static inline int
+flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
+		   const struct rte_flow_attr *attr,
+		   const struct rte_flow_item items[],
+		   const struct rte_flow_action actions[],
+		   struct rte_flow_error *error)
 {
-	unsigned int i;
-	uint32_t tunnel_ptype = 0;
+	const struct mlx5_flow_driver_ops *fops;
+	enum mlx5_flow_drv_type type = dev_flow->flow->drv_type;
 
-	/* Look up for the ptype to use. */
-	for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
-		if (!rxq_ctrl->flow_tunnels_n[i])
-			continue;
-		if (!tunnel_ptype) {
-			tunnel_ptype = tunnels_info[i].ptype;
-		} else {
-			tunnel_ptype = 0;
-			break;
-		}
-	}
-	rxq_ctrl->rxq.tunnel = tunnel_ptype;
+	assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
+	fops = flow_get_drv_ops(type);
+	return fops->translate(dev, dev_flow, attr, items, actions, error);
 }
 
 /**
- * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the flow.
+ * Flow driver apply API. This abstracts calling driver specific functions.
+ * Parent flow (rte_flow) should have driver type (drv_type). It applies
+ * translated driver flows on to device. flow_drv_translate() must precede.
  *
  * @param[in] dev
- *   Pointer to Ethernet device.
- * @param[in] flow
+ *   Pointer to Ethernet device structure.
+ * @param[in, out] flow
  *   Pointer to flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static void
-mlx5_flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
+static inline int
+flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
+	       struct rte_flow_error *error)
 {
-	struct priv *priv = dev->data->dev_private;
-	const int mark = !!(flow->modifier &
-			    (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK));
-	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
-	unsigned int i;
+	const struct mlx5_flow_driver_ops *fops;
+	enum mlx5_flow_drv_type type = flow->drv_type;
 
-	for (i = 0; i != flow->rss.queue_num; ++i) {
-		int idx = (*flow->queue)[i];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of((*priv->rxqs)[idx],
-				     struct mlx5_rxq_ctrl, rxq);
-
-		if (mark) {
-			rxq_ctrl->rxq.mark = 1;
-			rxq_ctrl->flow_mark_n++;
-		}
-		if (tunnel) {
-			unsigned int j;
-
-			/* Increase the counter matching the flow. */
-			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
-				if ((tunnels_info[j].tunnel & flow->layers) ==
-				    tunnels_info[j].tunnel) {
-					rxq_ctrl->flow_tunnels_n[j]++;
-					break;
-				}
-			}
-			mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
-		}
-	}
+	assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
+	fops = flow_get_drv_ops(type);
+	return fops->apply(dev, flow, error);
 }
 
 /**
- * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
- * @p flow if no other flow uses it with the same kind of request.
+ * Flow driver remove API. This abstracts calling driver specific functions.
+ * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
+ * on device. All the resources of the flow should be freed by calling
+ * flow_dv_destroy().
  *
- * @param dev
+ * @param[in] dev
  *   Pointer to Ethernet device.
- * @param[in] flow
- *   Pointer to the flow.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
  */
-static void
-mlx5_flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
+static inline void
+flow_drv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
-	struct priv *priv = dev->data->dev_private;
-	const int mark = !!(flow->modifier &
-			    (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK));
-	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
-	unsigned int i;
-
-	assert(dev->data->dev_started);
-	for (i = 0; i != flow->rss.queue_num; ++i) {
-		int idx = (*flow->queue)[i];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of((*priv->rxqs)[idx],
-				     struct mlx5_rxq_ctrl, rxq);
-
-		if (mark) {
-			rxq_ctrl->flow_mark_n--;
-			rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
-		}
-		if (tunnel) {
-			unsigned int j;
+	const struct mlx5_flow_driver_ops *fops;
+	enum mlx5_flow_drv_type type = flow->drv_type;
 
-			/* Decrease the counter matching the flow. */
-			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
-				if ((tunnels_info[j].tunnel & flow->layers) ==
-				    tunnels_info[j].tunnel) {
-					rxq_ctrl->flow_tunnels_n[j]--;
-					break;
-				}
-			}
-			mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
-		}
-	}
+	assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
+	fops = flow_get_drv_ops(type);
+	fops->remove(dev, flow);
 }
 
 /**
- * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
+ * Flow driver destroy API. This abstracts calling driver specific functions.
+ * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow
+ * on device and releases resources of the flow.
  *
- * @param dev
+ * @param[in] dev
  *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
  */
-static void
-mlx5_flow_rxq_flags_clear(struct rte_eth_dev *dev)
+static inline void
+flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-
-	for (i = 0; i != priv->rxqs_n; ++i) {
-		struct mlx5_rxq_ctrl *rxq_ctrl;
-		unsigned int j;
+	const struct mlx5_flow_driver_ops *fops;
+	enum mlx5_flow_drv_type type = flow->drv_type;
 
-		if (!(*priv->rxqs)[i])
-			continue;
-		rxq_ctrl = container_of((*priv->rxqs)[i],
-					struct mlx5_rxq_ctrl, rxq);
-		rxq_ctrl->flow_mark_n = 0;
-		rxq_ctrl->rxq.mark = 0;
-		for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
-			rxq_ctrl->flow_tunnels_n[j] = 0;
-		rxq_ctrl->rxq.tunnel = 0;
-	}
+	assert(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
+	fops = flow_get_drv_ops(type);
+	fops->destroy(dev, flow);
 }
 
 /**
@@ -2859,134 +1930,55 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
 		   const struct rte_flow_action actions[],
 		   struct rte_flow_error *error)
 {
-	int ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+	int ret;
 
+	ret = flow_drv_validate(dev, attr, items, actions, error);
 	if (ret < 0)
 		return ret;
 	return 0;
 }
 
 /**
- * Remove the flow.
+ * Get RSS action from the action list.
  *
- * @param[in] dev
- *   Pointer to Ethernet device.
- * @param[in, out] flow
- *   Pointer to flow structure.
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ *
+ * @return
+ *   Pointer to the RSS action if exist, else return NULL.
  */
-static void
-mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
+static const struct rte_flow_action_rss*
+flow_get_rss_action(const struct rte_flow_action actions[])
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_flow_verbs *verbs;
-
-	if (flow->nl_flow && priv->mnl_socket)
-		mlx5_nl_flow_destroy(priv->mnl_socket, flow->nl_flow, NULL);
-	LIST_FOREACH(verbs, &flow->verbs, next) {
-		if (verbs->flow) {
-			claim_zero(mlx5_glue->destroy_flow(verbs->flow));
-			verbs->flow = NULL;
-		}
-		if (verbs->hrxq) {
-			if (flow->fate & MLX5_FLOW_FATE_DROP)
-				mlx5_hrxq_drop_release(dev);
-			else
-				mlx5_hrxq_release(dev, verbs->hrxq);
-			verbs->hrxq = NULL;
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_RSS:
+			return (const struct rte_flow_action_rss *)
+			       actions->conf;
+		default:
+			break;
 		}
 	}
-	if (flow->counter) {
-		mlx5_flow_counter_release(flow->counter);
-		flow->counter = NULL;
-	}
+	return NULL;
 }
 
-/**
- * Apply the flow.
- *
- * @param[in] dev
- *   Pointer to Ethernet device structure.
- * @param[in, out] flow
- *   Pointer to flow structure.
- * @param[out] error
- *   Pointer to error structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
-		struct rte_flow_error *error)
+static unsigned int
+find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_flow_verbs *verbs;
-	int err;
-
-	LIST_FOREACH(verbs, &flow->verbs, next) {
-		if (flow->fate & MLX5_FLOW_FATE_DROP) {
-			verbs->hrxq = mlx5_hrxq_drop_new(dev);
-			if (!verbs->hrxq) {
-				rte_flow_error_set
-					(error, errno,
-					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					 NULL,
-					 "cannot get drop hash queue");
-				goto error;
-			}
-		} else {
-			struct mlx5_hrxq *hrxq;
-
-			hrxq = mlx5_hrxq_get(dev, flow->key,
-					     MLX5_RSS_HASH_KEY_LEN,
-					     verbs->hash_fields,
-					     (*flow->queue),
-					     flow->rss.queue_num);
-			if (!hrxq)
-				hrxq = mlx5_hrxq_new(dev, flow->key,
-						     MLX5_RSS_HASH_KEY_LEN,
-						     verbs->hash_fields,
-						     (*flow->queue),
-						     flow->rss.queue_num,
-						     !!(flow->layers &
-						      MLX5_FLOW_LAYER_TUNNEL));
-			if (!hrxq) {
-				rte_flow_error_set
-					(error, rte_errno,
-					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					 NULL,
-					 "cannot get hash queue");
-				goto error;
-			}
-			verbs->hrxq = hrxq;
-		}
-		verbs->flow =
-			mlx5_glue->create_flow(verbs->hrxq->qp, verbs->attr);
-		if (!verbs->flow) {
-			rte_flow_error_set(error, errno,
-					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					   NULL,
-					   "hardware refuses to create flow");
-			goto error;
-		}
-	}
-	if (flow->nl_flow &&
-	    priv->mnl_socket &&
-	    mlx5_nl_flow_create(priv->mnl_socket, flow->nl_flow, error))
-		goto error;
-	return 0;
-error:
-	err = rte_errno; /* Save rte_errno before cleanup. */
-	LIST_FOREACH(verbs, &flow->verbs, next) {
-		if (verbs->hrxq) {
-			if (flow->fate & MLX5_FLOW_FATE_DROP)
-				mlx5_hrxq_drop_release(dev);
-			else
-				mlx5_hrxq_release(dev, verbs->hrxq);
-			verbs->hrxq = NULL;
+	const struct rte_flow_item *item;
+	unsigned int has_vlan = 0;
+
+	for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
+		if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
+			has_vlan = 1;
+			break;
 		}
 	}
-	rte_errno = err; /* Restore rte_errno. */
-	return -rte_errno;
+	if (has_vlan)
+		return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
+				       MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
+	return rss_level < 2 ? MLX5_EXPANSION_ROOT :
+			       MLX5_EXPANSION_ROOT_OUTER;
 }
 
 /**
@@ -3009,50 +2001,90 @@ error:
  *   A flow on success, NULL otherwise and rte_errno is set.
  */
 static struct rte_flow *
-mlx5_flow_list_create(struct rte_eth_dev *dev,
-		      struct mlx5_flows *list,
-		      const struct rte_flow_attr *attr,
-		      const struct rte_flow_item items[],
-		      const struct rte_flow_action actions[],
-		      struct rte_flow_error *error)
+flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list,
+		 const struct rte_flow_attr *attr,
+		 const struct rte_flow_item items[],
+		 const struct rte_flow_action actions[],
+		 struct rte_flow_error *error)
 {
 	struct rte_flow *flow = NULL;
-	size_t size = 0;
+	struct mlx5_flow *dev_flow;
+	uint64_t action_flags = 0;
+	uint64_t item_flags = 0;
+	const struct rte_flow_action_rss *rss;
+	union {
+		struct rte_flow_expand_rss buf;
+		uint8_t buffer[2048];
+	} expand_buffer;
+	struct rte_flow_expand_rss *buf = &expand_buffer.buf;
 	int ret;
+	uint32_t i;
+	uint32_t flow_size;
 
-	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
+	ret = flow_drv_validate(dev, attr, items, actions, error);
 	if (ret < 0)
 		return NULL;
-	size = ret;
-	flow = rte_calloc(__func__, 1, size, 0);
-	if (!flow) {
-		rte_flow_error_set(error, ENOMEM,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "not enough memory to create flow");
-		return NULL;
+	flow_size = sizeof(struct rte_flow);
+	rss = flow_get_rss_action(actions);
+	if (rss)
+		flow_size += RTE_ALIGN_CEIL(rss->queue_num * sizeof(uint16_t),
+					    sizeof(void *));
+	else
+		flow_size += RTE_ALIGN_CEIL(sizeof(uint16_t), sizeof(void *));
+	flow = rte_calloc(__func__, 1, flow_size, 0);
+	flow->drv_type = flow_get_drv_type(dev, attr);
+	assert(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
+	       flow->drv_type < MLX5_FLOW_TYPE_MAX);
+	flow->queue = (void *)(flow + 1);
+	LIST_INIT(&flow->dev_flows);
+	if (rss && rss->types) {
+		unsigned int graph_root;
+
+		graph_root = find_graph_root(items, rss->level);
+		ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
+					  items, rss->types,
+					  mlx5_support_expansion,
+					  graph_root);
+		assert(ret > 0 &&
+		       (unsigned int)ret < sizeof(expand_buffer.buffer));
+	} else {
+		buf->entries = 1;
+		buf->entry[0].pattern = (void *)(uintptr_t)items;
 	}
-	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
-	if (ret < 0) {
-		rte_free(flow);
-		return NULL;
+	for (i = 0; i < buf->entries; ++i) {
+		dev_flow = flow_drv_prepare(flow, attr, buf->entry[i].pattern,
+					    actions, &item_flags, &action_flags,
+					    error);
+		if (!dev_flow)
+			goto error;
+		dev_flow->flow = flow;
+		dev_flow->layers = item_flags;
+		/* Store actions once as expanded flows have same actions. */
+		if (i == 0)
+			flow->actions = action_flags;
+		assert(flow->actions == action_flags);
+		LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next);
+		ret = flow_drv_translate(dev, dev_flow, attr,
+					 buf->entry[i].pattern,
+					 actions, error);
+		if (ret < 0)
+			goto error;
 	}
-	assert((size_t)ret == size);
 	if (dev->data->dev_started) {
-		ret = mlx5_flow_apply(dev, flow, error);
-		if (ret < 0) {
-			ret = rte_errno; /* Save rte_errno before cleanup. */
-			if (flow) {
-				mlx5_flow_remove(dev, flow);
-				rte_free(flow);
-			}
-			rte_errno = ret; /* Restore rte_errno. */
-			return NULL;
-		}
+		ret = flow_drv_apply(dev, flow, error);
+		if (ret < 0)
+			goto error;
 	}
 	TAILQ_INSERT_TAIL(list, flow, next);
-	mlx5_flow_rxq_flags_set(dev, flow);
+	flow_rxq_flags_set(dev, flow);
 	return flow;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	assert(flow);
+	flow_drv_destroy(dev, flow);
+	rte_free(flow);
+	rte_errno = ret; /* Restore rte_errno. */
+	return NULL;
 }
 
 /**
@@ -3068,9 +2100,9 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	return mlx5_flow_list_create
-		(dev, &((struct priv *)dev->data->dev_private)->flows,
-		 attr, items, actions, error);
+	return flow_list_create(dev,
+				&((struct priv *)dev->data->dev_private)->flows,
+				attr, items, actions, error);
 }
 
 /**
@@ -3084,17 +2116,17 @@ mlx5_flow_create(struct rte_eth_dev *dev,
  *   Flow to destroy.
  */
 static void
-mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
-		       struct rte_flow *flow)
+flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
+		  struct rte_flow *flow)
 {
-	mlx5_flow_remove(dev, flow);
+	flow_drv_destroy(dev, flow);
 	TAILQ_REMOVE(list, flow, next);
 	/*
 	 * Update RX queue flags only if port is started, otherwise it is
 	 * already clean.
 	 */
 	if (dev->data->dev_started)
-		mlx5_flow_rxq_flags_trim(dev, flow);
+		flow_rxq_flags_trim(dev, flow);
 	rte_free(flow);
 }
 
@@ -3113,7 +2145,7 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
 		struct rte_flow *flow;
 
 		flow = TAILQ_FIRST(list);
-		mlx5_flow_list_destroy(dev, list, flow);
+		flow_list_destroy(dev, list, flow);
 	}
 }
 
@@ -3131,8 +2163,8 @@ mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 	struct rte_flow *flow;
 
 	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
-		mlx5_flow_remove(dev, flow);
-	mlx5_flow_rxq_flags_clear(dev);
+		flow_drv_remove(dev, flow);
+	flow_rxq_flags_clear(dev);
 }
 
 /**
@@ -3154,10 +2186,10 @@ mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 	int ret = 0;
 
 	TAILQ_FOREACH(flow, list, next) {
-		ret = mlx5_flow_apply(dev, flow, &error);
+		ret = flow_drv_apply(dev, flow, &error);
 		if (ret < 0)
 			goto error;
-		mlx5_flow_rxq_flags_set(dev, flow);
+		flow_rxq_flags_set(dev, flow);
 	}
 	return 0;
 error:
@@ -3228,7 +2260,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 		},
 		{
 			.type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
-				RTE_FLOW_ITEM_TYPE_END,
+					      RTE_FLOW_ITEM_TYPE_END,
 			.spec = vlan_spec,
 			.last = NULL,
 			.mask = vlan_mask,
@@ -3266,8 +2298,8 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	}
 	for (i = 0; i != priv->reta_idx_n; ++i)
 		queue[i] = (*priv->reta_idx)[i];
-	flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items,
-				     actions, &error);
+	flow = flow_list_create(dev, &priv->ctrl_flows,
+				&attr, items, actions, &error);
 	if (!flow)
 		return -rte_errno;
 	return 0;
@@ -3307,7 +2339,7 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
 {
 	struct priv *priv = dev->data->dev_private;
 
-	mlx5_flow_list_destroy(dev, &priv->flows, flow);
+	flow_list_destroy(dev, &priv->flows, flow);
 	return 0;
 }
 
@@ -3356,92 +2388,45 @@ mlx5_flow_isolate(struct rte_eth_dev *dev,
 }
 
 /**
- * Query flow counter.
- *
- * @param flow
- *   Pointer to the flow.
+ * Query a flow.
  *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ * @see rte_flow_query()
+ * @see rte_flow_ops
  */
 static int
-mlx5_flow_query_count(struct rte_flow *flow __rte_unused,
-		      void *data __rte_unused,
-		      struct rte_flow_error *error)
-{
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	if (flow->modifier & MLX5_FLOW_MOD_COUNT) {
-		struct rte_flow_query_count *qc = data;
-		uint64_t counters[2] = {0, 0};
-		struct ibv_query_counter_set_attr query_cs_attr = {
-			.cs = flow->counter->cs,
-			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
-		};
-		struct ibv_counter_set_data query_out = {
-			.out = counters,
-			.outlen = 2 * sizeof(uint64_t),
-		};
-		int err = mlx5_glue->query_counter_set(&query_cs_attr,
-						       &query_out);
+flow_drv_query(struct rte_eth_dev *dev,
+	       struct rte_flow *flow,
+	       const struct rte_flow_action *actions,
+	       void *data,
+	       struct rte_flow_error *error)
+{
+	const struct mlx5_flow_driver_ops *fops;
+	enum mlx5_flow_drv_type ftype = flow->drv_type;
 
-		if (err)
-			return rte_flow_error_set
-				(error, err,
-				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				 NULL,
-				 "cannot read counter");
-		qc->hits_set = 1;
-		qc->bytes_set = 1;
-		qc->hits = counters[0] - flow->counter->hits;
-		qc->bytes = counters[1] - flow->counter->bytes;
-		if (qc->reset) {
-			flow->counter->hits = counters[0];
-			flow->counter->bytes = counters[1];
-		}
-		return 0;
-	}
-	return rte_flow_error_set(error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				  NULL,
-				  "flow does not have counter");
-#endif
-	return rte_flow_error_set(error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				  NULL,
-				  "counters are not available");
+	assert(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX);
+	fops = flow_get_drv_ops(ftype);
+
+	return fops->query(dev, flow, actions, data, error);
 }
 
 /**
- * Query a flows.
+ * Query a flow.
  *
  * @see rte_flow_query()
  * @see rte_flow_ops
  */
 int
-mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
+mlx5_flow_query(struct rte_eth_dev *dev,
 		struct rte_flow *flow,
 		const struct rte_flow_action *actions,
 		void *data,
 		struct rte_flow_error *error)
 {
-	int ret = 0;
+	int ret;
 
-	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
-		switch (actions->type) {
-		case RTE_FLOW_ACTION_TYPE_VOID:
-			break;
-		case RTE_FLOW_ACTION_TYPE_COUNT:
-			ret = mlx5_flow_query_count(flow, data, error);
-			break;
-		default:
-			return rte_flow_error_set(error, ENOTSUP,
-						  RTE_FLOW_ERROR_TYPE_ACTION,
-						  actions,
-						  "action not supported");
-		}
-		if (ret < 0)
-			return ret;
-	}
+	ret = flow_drv_query(dev, flow, actions, data, error);
+	if (ret < 0)
+		return ret;
 	return 0;
 }
 
@@ -3511,7 +2496,6 @@ mlx5_fdir_filter_convert(struct rte_eth_dev *dev,
 			.dst_addr = input->flow.ip4_flow.dst_ip,
 			.time_to_live = input->flow.ip4_flow.ttl,
 			.type_of_service = input->flow.ip4_flow.tos,
-			.next_proto_id = input->flow.ip4_flow.proto,
 		};
 		attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){
 			.src_addr = mask->ipv4_mask.src_ip,
@@ -3663,9 +2647,8 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
 	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
 	if (ret)
 		return ret;
-	flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
-				     attributes.items, attributes.actions,
-				     &error);
+	flow = flow_list_create(dev, &priv->flows, &attributes.attr,
+				attributes.items, attributes.actions, &error);
 	if (flow) {
 		DRV_LOG(DEBUG, "port %u FDIR created %p", dev->data->port_id,
 			(void *)flow);
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
new file mode 100644
index 00000000..61299d66
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -0,0 +1,375 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_FLOW_H_
+#define RTE_PMD_MLX5_FLOW_H_
+
+#include <netinet/in.h>
+#include <sys/queue.h>
+#include <stdalign.h>
+#include <stdint.h>
+#include <string.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+/* Pattern outer Layer bits. */
+#define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
+#define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
+#define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
+#define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
+
+/* Pattern inner Layer bits. */
+#define MLX5_FLOW_LAYER_INNER_L2 (1u << 6)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV4 (1u << 7)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV6 (1u << 8)
+#define MLX5_FLOW_LAYER_INNER_L4_UDP (1u << 9)
+#define MLX5_FLOW_LAYER_INNER_L4_TCP (1u << 10)
+#define MLX5_FLOW_LAYER_INNER_VLAN (1u << 11)
+
+/* Pattern tunnel Layer bits. */
+#define MLX5_FLOW_LAYER_VXLAN (1u << 12)
+#define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
+#define MLX5_FLOW_LAYER_GRE (1u << 14)
+#define MLX5_FLOW_LAYER_MPLS (1u << 15)
+
+/* General pattern items bits. */
+#define MLX5_FLOW_ITEM_METADATA (1u << 16)
+
+/* Outer Masks. */
+#define MLX5_FLOW_LAYER_OUTER_L3 \
+	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
+#define MLX5_FLOW_LAYER_OUTER_L4 \
+	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
+#define MLX5_FLOW_LAYER_OUTER \
+	(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_L3 | \
+	 MLX5_FLOW_LAYER_OUTER_L4)
+
+/* Tunnel Masks. */
+#define MLX5_FLOW_LAYER_TUNNEL \
+	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
+	 MLX5_FLOW_LAYER_GRE | MLX5_FLOW_LAYER_MPLS)
+
+/* Inner Masks. */
+#define MLX5_FLOW_LAYER_INNER_L3 \
+	(MLX5_FLOW_LAYER_INNER_L3_IPV4 | MLX5_FLOW_LAYER_INNER_L3_IPV6)
+#define MLX5_FLOW_LAYER_INNER_L4 \
+	(MLX5_FLOW_LAYER_INNER_L4_UDP | MLX5_FLOW_LAYER_INNER_L4_TCP)
+#define MLX5_FLOW_LAYER_INNER \
+	(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3 | \
+	 MLX5_FLOW_LAYER_INNER_L4)
+
+/* Actions */
+#define MLX5_FLOW_ACTION_DROP (1u << 0)
+#define MLX5_FLOW_ACTION_QUEUE (1u << 1)
+#define MLX5_FLOW_ACTION_RSS (1u << 2)
+#define MLX5_FLOW_ACTION_FLAG (1u << 3)
+#define MLX5_FLOW_ACTION_MARK (1u << 4)
+#define MLX5_FLOW_ACTION_COUNT (1u << 5)
+#define MLX5_FLOW_ACTION_PORT_ID (1u << 6)
+#define MLX5_FLOW_ACTION_OF_POP_VLAN (1u << 7)
+#define MLX5_FLOW_ACTION_OF_PUSH_VLAN (1u << 8)
+#define MLX5_FLOW_ACTION_OF_SET_VLAN_VID (1u << 9)
+#define MLX5_FLOW_ACTION_OF_SET_VLAN_PCP (1u << 10)
+#define MLX5_FLOW_ACTION_SET_IPV4_SRC (1u << 11)
+#define MLX5_FLOW_ACTION_SET_IPV4_DST (1u << 12)
+#define MLX5_FLOW_ACTION_SET_IPV6_SRC (1u << 13)
+#define MLX5_FLOW_ACTION_SET_IPV6_DST (1u << 14)
+#define MLX5_FLOW_ACTION_SET_TP_SRC (1u << 15)
+#define MLX5_FLOW_ACTION_SET_TP_DST (1u << 16)
+#define MLX5_FLOW_ACTION_JUMP (1u << 17)
+#define MLX5_FLOW_ACTION_SET_TTL (1u << 18)
+#define MLX5_FLOW_ACTION_DEC_TTL (1u << 19)
+#define MLX5_FLOW_ACTION_SET_MAC_SRC (1u << 20)
+#define MLX5_FLOW_ACTION_SET_MAC_DST (1u << 21)
+
+#define MLX5_FLOW_FATE_ACTIONS \
+	(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)
+
+#ifndef IPPROTO_MPLS
+#define IPPROTO_MPLS 137
+#endif
+
+/* UDP port numbers for VxLAN. */
+#define MLX5_UDP_PORT_VXLAN 4789
+#define MLX5_UDP_PORT_VXLAN_GPE 4790
+
+/* Priority reserved for default flows. */
+#define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
+
+/*
+ * Number of sub priorities.
+ * For each kind of pattern matching i.e. L2, L3, L4 to have a correct
+ * matching on the NIC (firmware dependent) L4 most have the higher priority
+ * followed by L3 and ending with L2.
+ */
+#define MLX5_PRIORITY_MAP_L2 2
+#define MLX5_PRIORITY_MAP_L3 1
+#define MLX5_PRIORITY_MAP_L4 0
+#define MLX5_PRIORITY_MAP_MAX 3
+
+/* Valid layer type for IPV4 RSS. */
+#define MLX5_IPV4_LAYER_TYPES \
+	(ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | \
+	 ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV4_UDP | \
+	 ETH_RSS_NONFRAG_IPV4_OTHER)
+
+/* IBV hash source bits  for IPV4. */
+#define MLX5_IPV4_IBV_RX_HASH (IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4)
+
+/* Valid layer type for IPV6 RSS. */
+#define MLX5_IPV6_LAYER_TYPES \
+	(ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_TCP | \
+	 ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_EX  | ETH_RSS_IPV6_TCP_EX | \
+	 ETH_RSS_IPV6_UDP_EX | ETH_RSS_NONFRAG_IPV6_OTHER)
+
+/* IBV hash source bits  for IPV6. */
+#define MLX5_IPV6_IBV_RX_HASH (IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6)
+
+/* Max number of actions per DV flow. */
+#define MLX5_DV_MAX_NUMBER_OF_ACTIONS 8
+
+enum mlx5_flow_drv_type {
+	MLX5_FLOW_TYPE_MIN,
+	MLX5_FLOW_TYPE_DV,
+	MLX5_FLOW_TYPE_TCF,
+	MLX5_FLOW_TYPE_VERBS,
+	MLX5_FLOW_TYPE_MAX,
+};
+
+/* Matcher PRM representation */
+struct mlx5_flow_dv_match_params {
+	size_t size;
+	/**< Size of match value. Do NOT split size and key! */
+	uint32_t buf[MLX5_ST_SZ_DW(fte_match_param)];
+	/**< Matcher value. This value is used as the mask or as a key. */
+};
+
+#define MLX5_DV_MAX_NUMBER_OF_ACTIONS 8
+
+/* Matcher structure. */
+struct mlx5_flow_dv_matcher {
+	LIST_ENTRY(mlx5_flow_dv_matcher) next;
+	/* Pointer to the next element. */
+	rte_atomic32_t refcnt; /**< Reference counter. */
+	void *matcher_object; /**< Pointer to DV matcher */
+	uint16_t crc; /**< CRC of key. */
+	uint16_t priority; /**< Priority of matcher. */
+	uint8_t egress; /**< Egress matcher. */
+	struct mlx5_flow_dv_match_params mask; /**< Matcher mask. */
+};
+
+/* DV flows structure. */
+struct mlx5_flow_dv {
+	uint64_t hash_fields; /**< Fields that participate in the hash. */
+	struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
+	/* Flow DV api: */
+	struct mlx5_flow_dv_matcher *matcher; /**< Cache to matcher. */
+	struct mlx5_flow_dv_match_params value;
+	/**< Holds the value that the packet is compared to. */
+	struct ibv_flow *flow; /**< Installed flow. */
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_flow_action_attr actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS];
+	/**< Action list. */
+#endif
+	int actions_n; /**< number of actions. */
+};
+
+/** Linux TC flower driver for E-Switch flow. */
+struct mlx5_flow_tcf {
+	struct nlmsghdr *nlh;
+	struct tcmsg *tcm;
+};
+
+/* Verbs specification header. */
+struct ibv_spec_header {
+	enum ibv_flow_spec_type type;
+	uint16_t size;
+};
+
+/** Handles information leading to a drop fate. */
+struct mlx5_flow_verbs {
+	LIST_ENTRY(mlx5_flow_verbs) next;
+	unsigned int size; /**< Size of the attribute. */
+	struct {
+		struct ibv_flow_attr *attr;
+		/**< Pointer to the Specification buffer. */
+		uint8_t *specs; /**< Pointer to the specifications. */
+	};
+	struct ibv_flow *flow; /**< Verbs flow pointer. */
+	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
+	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
+};
+
+/** Device flow structure. */
+struct mlx5_flow {
+	LIST_ENTRY(mlx5_flow) next;
+	struct rte_flow *flow; /**< Pointer to the main flow. */
+	uint64_t layers;
+	/**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */
+	union {
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+		struct mlx5_flow_dv dv;
+#endif
+		struct mlx5_flow_tcf tcf;
+		struct mlx5_flow_verbs verbs;
+	};
+};
+
+/* Counters information. */
+struct mlx5_flow_counter {
+	LIST_ENTRY(mlx5_flow_counter) next; /**< Pointer to the next counter. */
+	uint32_t shared:1; /**< Share counter ID with other flow rules. */
+	uint32_t ref_cnt:31; /**< Reference counter. */
+	uint32_t id; /**< Counter ID. */
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
+	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	struct ibv_counters *cs; /**< Holds the counters for the rule. */
+#endif
+	uint64_t hits; /**< Number of packets matched by the rule. */
+	uint64_t bytes; /**< Number of bytes matched by the rule. */
+};
+
+/* Flow structure. */
+struct rte_flow {
+	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
+	enum mlx5_flow_drv_type drv_type; /**< Drvier type. */
+	struct mlx5_flow_counter *counter; /**< Holds flow counter. */
+	struct rte_flow_action_rss rss;/**< RSS context. */
+	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
+	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
+	LIST_HEAD(dev_flows, mlx5_flow) dev_flows;
+	/**< Device flows that are part of the flow. */
+	uint64_t actions;
+	/**< Bit-fields of detected actions, see MLX5_FLOW_ACTION_*. */
+};
+typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev,
+				    const struct rte_flow_attr *attr,
+				    const struct rte_flow_item items[],
+				    const struct rte_flow_action actions[],
+				    struct rte_flow_error *error);
+typedef struct mlx5_flow *(*mlx5_flow_prepare_t)
+	(const struct rte_flow_attr *attr, const struct rte_flow_item items[],
+	 const struct rte_flow_action actions[], uint64_t *item_flags,
+	 uint64_t *action_flags, struct rte_flow_error *error);
+typedef int (*mlx5_flow_translate_t)(struct rte_eth_dev *dev,
+				     struct mlx5_flow *dev_flow,
+				     const struct rte_flow_attr *attr,
+				     const struct rte_flow_item items[],
+				     const struct rte_flow_action actions[],
+				     struct rte_flow_error *error);
+typedef int (*mlx5_flow_apply_t)(struct rte_eth_dev *dev, struct rte_flow *flow,
+				 struct rte_flow_error *error);
+typedef void (*mlx5_flow_remove_t)(struct rte_eth_dev *dev,
+				   struct rte_flow *flow);
+typedef void (*mlx5_flow_destroy_t)(struct rte_eth_dev *dev,
+				    struct rte_flow *flow);
+typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev,
+				 struct rte_flow *flow,
+				 const struct rte_flow_action *actions,
+				 void *data,
+				 struct rte_flow_error *error);
+struct mlx5_flow_driver_ops {
+	mlx5_flow_validate_t validate;
+	mlx5_flow_prepare_t prepare;
+	mlx5_flow_translate_t translate;
+	mlx5_flow_apply_t apply;
+	mlx5_flow_remove_t remove;
+	mlx5_flow_destroy_t destroy;
+	mlx5_flow_query_t query;
+};
+
+/* mlx5_flow.c */
+
+uint64_t mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, int tunnel,
+				     uint64_t layer_types,
+				     uint64_t hash_fields);
+uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
+				   uint32_t subpriority);
+int mlx5_flow_validate_action_count(struct rte_eth_dev *dev,
+				    const struct rte_flow_attr *attr,
+				    struct rte_flow_error *error);
+int mlx5_flow_validate_action_drop(uint64_t action_flags,
+				   const struct rte_flow_attr *attr,
+				   struct rte_flow_error *error);
+int mlx5_flow_validate_action_flag(uint64_t action_flags,
+				   const struct rte_flow_attr *attr,
+				   struct rte_flow_error *error);
+int mlx5_flow_validate_action_mark(const struct rte_flow_action *action,
+				   uint64_t action_flags,
+				   const struct rte_flow_attr *attr,
+				   struct rte_flow_error *error);
+int mlx5_flow_validate_action_queue(const struct rte_flow_action *action,
+				    uint64_t action_flags,
+				    struct rte_eth_dev *dev,
+				    const struct rte_flow_attr *attr,
+				    struct rte_flow_error *error);
+int mlx5_flow_validate_action_rss(const struct rte_flow_action *action,
+				  uint64_t action_flags,
+				  struct rte_eth_dev *dev,
+				  const struct rte_flow_attr *attr,
+				  struct rte_flow_error *error);
+int mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
+				  const struct rte_flow_attr *attributes,
+				  struct rte_flow_error *error);
+int mlx5_flow_item_acceptable(const struct rte_flow_item *item,
+			      const uint8_t *mask,
+			      const uint8_t *nic_mask,
+			      unsigned int size,
+			      struct rte_flow_error *error);
+int mlx5_flow_validate_item_eth(const struct rte_flow_item *item,
+				uint64_t item_flags,
+				struct rte_flow_error *error);
+int mlx5_flow_validate_item_gre(const struct rte_flow_item *item,
+				uint64_t item_flags,
+				uint8_t target_protocol,
+				struct rte_flow_error *error);
+int mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item,
+				 int64_t item_flags,
+				 struct rte_flow_error *error);
+int mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item,
+				 uint64_t item_flags,
+				 struct rte_flow_error *error);
+int mlx5_flow_validate_item_mpls(const struct rte_flow_item *item,
+				 uint64_t item_flags,
+				 uint8_t target_protocol,
+				 struct rte_flow_error *error);
+int mlx5_flow_validate_item_tcp(const struct rte_flow_item *item,
+				uint64_t item_flags,
+				uint8_t target_protocol,
+				const struct rte_flow_item_tcp *flow_mask,
+				struct rte_flow_error *error);
+int mlx5_flow_validate_item_udp(const struct rte_flow_item *item,
+				uint64_t item_flags,
+				uint8_t target_protocol,
+				struct rte_flow_error *error);
+int mlx5_flow_validate_item_vlan(const struct rte_flow_item *item,
+				 int64_t item_flags,
+				 struct rte_flow_error *error);
+int mlx5_flow_validate_item_vxlan(const struct rte_flow_item *item,
+				  uint64_t item_flags,
+				  struct rte_flow_error *error);
+int mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item,
+				      uint64_t item_flags,
+				      struct rte_eth_dev *dev,
+				      struct rte_flow_error *error);
+
+/* mlx5_flow_tcf.c */
+
+int mlx5_flow_tcf_init(struct mlx5_flow_tcf_context *ctx,
+		       unsigned int ifindex, struct rte_flow_error *error);
+struct mlx5_flow_tcf_context *mlx5_flow_tcf_context_create(void);
+void mlx5_flow_tcf_context_destroy(struct mlx5_flow_tcf_context *ctx);
+
+#endif /* RTE_PMD_MLX5_FLOW_H_ */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
new file mode 100644
index 00000000..8f729f44
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -0,0 +1,1492 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <sys/queue.h>
+#include <stdalign.h>
+#include <stdint.h>
+#include <string.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_common.h>
+#include <rte_ether.h>
+#include <rte_eth_ctrl.h>
+#include <rte_ethdev_driver.h>
+#include <rte_flow.h>
+#include <rte_flow_driver.h>
+#include <rte_malloc.h>
+#include <rte_ip.h>
+
+#include "mlx5.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+#include "mlx5_glue.h"
+#include "mlx5_flow.h"
+
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+
+/**
+ * Validate META item.
+ *
+ * @param[in] dev
+ *   Pointer to the rte_eth_dev structure.
+ * @param[in] item
+ *   Item specification.
+ * @param[in] attr
+ *   Attributes of flow that includes this item.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_validate_item_meta(struct rte_eth_dev *dev,
+			   const struct rte_flow_item *item,
+			   const struct rte_flow_attr *attr,
+			   struct rte_flow_error *error)
+{
+	const struct rte_flow_item_meta *spec = item->spec;
+	const struct rte_flow_item_meta *mask = item->mask;
+	const struct rte_flow_item_meta nic_mask = {
+		.data = RTE_BE32(UINT32_MAX)
+	};
+	int ret;
+	uint64_t offloads = dev->data->dev_conf.txmode.offloads;
+
+	if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
+		return rte_flow_error_set(error, EPERM,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  NULL,
+					  "match on metadata offload "
+					  "configuration is off for this port");
+	if (!spec)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					  item->spec,
+					  "data cannot be empty");
+	if (!spec->data)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					  NULL,
+					  "data cannot be zero");
+	if (!mask)
+		mask = &rte_flow_item_meta_mask;
+	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
+					(const uint8_t *)&nic_mask,
+					sizeof(struct rte_flow_item_meta),
+					error);
+	if (ret < 0)
+		return ret;
+	if (attr->ingress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+					  NULL,
+					  "pattern not supported for ingress");
+	return 0;
+}
+
+/**
+ * Verify the @p attributes will be correctly understood by the NIC and store
+ * them in the @p flow if everything is correct.
+ *
+ * @param[in] dev
+ *   Pointer to dev struct.
+ * @param[in] attributes
+ *   Pointer to flow attributes
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_validate_attributes(struct rte_eth_dev *dev,
+			    const struct rte_flow_attr *attributes,
+			    struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint32_t priority_max = priv->config.flow_prio - 1;
+
+	if (attributes->group)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+					  NULL,
+					  "groups is not supported");
+	if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
+	    attributes->priority >= priority_max)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+					  NULL,
+					  "priority out of range");
+	if (attributes->transfer)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
+					  NULL,
+					  "transfer is not supported");
+	if (!(attributes->egress ^ attributes->ingress))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR, NULL,
+					  "must specify exactly one of "
+					  "ingress or egress");
+	return 0;
+}
+
+/**
+ * Internal validation function. For validating both actions and items.
+ *
+ * @param[in] dev
+ *   Pointer to the rte_eth_dev structure.
+ * @param[in] attr
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_ernno is set.
+ */
+static int
+flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
+		 const struct rte_flow_item items[],
+		 const struct rte_flow_action actions[],
+		 struct rte_flow_error *error)
+{
+	int ret;
+	uint64_t action_flags = 0;
+	uint64_t item_flags = 0;
+	int tunnel = 0;
+	uint8_t next_protocol = 0xff;
+	int actions_n = 0;
+
+	if (items == NULL)
+		return -1;
+	ret = flow_dv_validate_attributes(dev, attr, error);
+	if (ret < 0)
+		return ret;
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+		tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
+		switch (items->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			ret = mlx5_flow_validate_item_eth(items, item_flags,
+							  error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+					       MLX5_FLOW_LAYER_OUTER_L2;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			ret = mlx5_flow_validate_item_vlan(items, item_flags,
+							   error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
+					       MLX5_FLOW_LAYER_OUTER_VLAN;
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			ret = mlx5_flow_validate_item_ipv4(items, item_flags,
+							   error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
+					       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+			if (items->mask != NULL &&
+			    ((const struct rte_flow_item_ipv4 *)
+			     items->mask)->hdr.next_proto_id)
+				next_protocol =
+					((const struct rte_flow_item_ipv4 *)
+					 (items->spec))->hdr.next_proto_id;
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			ret = mlx5_flow_validate_item_ipv6(items, item_flags,
+							   error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
+					       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+			if (items->mask != NULL &&
+			    ((const struct rte_flow_item_ipv6 *)
+			     items->mask)->hdr.proto)
+				next_protocol =
+					((const struct rte_flow_item_ipv6 *)
+					 items->spec)->hdr.proto;
+			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			ret = mlx5_flow_validate_item_tcp
+						(items, item_flags,
+						 next_protocol,
+						 &rte_flow_item_tcp_mask,
+						 error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
+					       MLX5_FLOW_LAYER_OUTER_L4_TCP;
+			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			ret = mlx5_flow_validate_item_udp(items, item_flags,
+							  next_protocol,
+							  error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
+					       MLX5_FLOW_LAYER_OUTER_L4_UDP;
+			break;
+		case RTE_FLOW_ITEM_TYPE_GRE:
+		case RTE_FLOW_ITEM_TYPE_NVGRE:
+			ret = mlx5_flow_validate_item_gre(items, item_flags,
+							  next_protocol, error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_GRE;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN:
+			ret = mlx5_flow_validate_item_vxlan(items, item_flags,
+							    error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_VXLAN;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+			ret = mlx5_flow_validate_item_vxlan_gpe(items,
+								item_flags, dev,
+								error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
+			break;
+		case RTE_FLOW_ITEM_TYPE_META:
+			ret = flow_dv_validate_item_meta(dev, items, attr,
+							 error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_ITEM_METADATA;
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  NULL, "item not supported");
+		}
+	}
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions, "too many actions");
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_FLAG:
+			ret = mlx5_flow_validate_action_flag(action_flags,
+							     attr, error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_FLAG;
+			++actions_n;
+			break;
+		case RTE_FLOW_ACTION_TYPE_MARK:
+			ret = mlx5_flow_validate_action_mark(actions,
+							     action_flags,
+							     attr, error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_MARK;
+			++actions_n;
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			ret = mlx5_flow_validate_action_drop(action_flags,
+							     attr, error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_DROP;
+			++actions_n;
+			break;
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			ret = mlx5_flow_validate_action_queue(actions,
+							      action_flags, dev,
+							      attr, error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_QUEUE;
+			++actions_n;
+			break;
+		case RTE_FLOW_ACTION_TYPE_RSS:
+			ret = mlx5_flow_validate_action_rss(actions,
+							    action_flags, dev,
+							    attr, error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_RSS;
+			++actions_n;
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_validate_action_count(dev, attr, error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_COUNT;
+			++actions_n;
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+	}
+	if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, actions,
+					  "no fate action is found");
+	return 0;
+}
+
+/**
+ * Internal preparation function. Allocates the DV flow size,
+ * this size is constant.
+ *
+ * @param[in] attr
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ * @param[out] item_flags
+ *   Pointer to bit mask of all items detected.
+ * @param[out] action_flags
+ *   Pointer to bit mask of all actions detected.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   Pointer to mlx5_flow object on success,
+ *   otherwise NULL and rte_ernno is set.
+ */
+static struct mlx5_flow *
+flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
+		const struct rte_flow_item items[] __rte_unused,
+		const struct rte_flow_action actions[] __rte_unused,
+		uint64_t *item_flags __rte_unused,
+		uint64_t *action_flags __rte_unused,
+		struct rte_flow_error *error)
+{
+	uint32_t size = sizeof(struct mlx5_flow);
+	struct mlx5_flow *flow;
+
+	flow = rte_calloc(__func__, 1, size, 0);
+	if (!flow) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "not enough memory to create flow");
+		return NULL;
+	}
+	flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param);
+	return flow;
+}
+
+/**
+ * Add Ethernet item to matcher and to the value.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_eth(void *matcher, void *key,
+			   const struct rte_flow_item *item, int inner)
+{
+	const struct rte_flow_item_eth *eth_m = item->mask;
+	const struct rte_flow_item_eth *eth_v = item->spec;
+	const struct rte_flow_item_eth nic_mask = {
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.type = RTE_BE16(0xffff),
+	};
+	void *headers_m;
+	void *headers_v;
+	char *l24_v;
+	unsigned int i;
+
+	if (!eth_v)
+		return;
+	if (!eth_m)
+		eth_m = &nic_mask;
+	if (inner) {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 inner_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
+	} else {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 outer_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
+	}
+	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
+	       &eth_m->dst, sizeof(eth_m->dst));
+	/* The value must be in the range of the mask. */
+	l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
+	for (i = 0; i < sizeof(eth_m->dst); ++i)
+		l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
+	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
+	       &eth_m->src, sizeof(eth_m->src));
+	l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
+	/* The value must be in the range of the mask. */
+	for (i = 0; i < sizeof(eth_m->dst); ++i)
+		l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
+		 rte_be_to_cpu_16(eth_m->type));
+	l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
+	*(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
+}
+
+/**
+ * Add VLAN item to matcher and to the value.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_vlan(void *matcher, void *key,
+			    const struct rte_flow_item *item,
+			    int inner)
+{
+	const struct rte_flow_item_vlan *vlan_m = item->mask;
+	const struct rte_flow_item_vlan *vlan_v = item->spec;
+	const struct rte_flow_item_vlan nic_mask = {
+		.tci = RTE_BE16(0x0fff),
+		.inner_type = RTE_BE16(0xffff),
+	};
+	void *headers_m;
+	void *headers_v;
+	uint16_t tci_m;
+	uint16_t tci_v;
+
+	if (!vlan_v)
+		return;
+	if (!vlan_m)
+		vlan_m = &nic_mask;
+	if (inner) {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 inner_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
+	} else {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 outer_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
+	}
+	tci_m = rte_be_to_cpu_16(vlan_m->tci);
+	tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
+}
+
+/**
+ * Add IPV4 item to matcher and to the value.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_ipv4(void *matcher, void *key,
+			    const struct rte_flow_item *item,
+			    int inner)
+{
+	const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
+	const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
+	const struct rte_flow_item_ipv4 nic_mask = {
+		.hdr = {
+			.src_addr = RTE_BE32(0xffffffff),
+			.dst_addr = RTE_BE32(0xffffffff),
+			.type_of_service = 0xff,
+			.next_proto_id = 0xff,
+		},
+	};
+	void *headers_m;
+	void *headers_v;
+	char *l24_m;
+	char *l24_v;
+	uint8_t tos;
+
+	if (inner) {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 inner_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
+	} else {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 outer_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
+	}
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
+	if (!ipv4_v)
+		return;
+	if (!ipv4_m)
+		ipv4_m = &nic_mask;
+	l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
+			     dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
+	l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
+			     dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
+	*(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
+	*(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
+	l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
+			  src_ipv4_src_ipv6.ipv4_layout.ipv4);
+	l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
+			  src_ipv4_src_ipv6.ipv4_layout.ipv4);
+	*(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
+	*(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
+	tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
+		 ipv4_m->hdr.type_of_service);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
+		 ipv4_m->hdr.type_of_service >> 2);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
+		 ipv4_m->hdr.next_proto_id);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
+		 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
+}
+
+/**
+ * Add IPV6 item to matcher and to the value.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_ipv6(void *matcher, void *key,
+			    const struct rte_flow_item *item,
+			    int inner)
+{
+	const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
+	const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
+	const struct rte_flow_item_ipv6 nic_mask = {
+		.hdr = {
+			.src_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.dst_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.vtc_flow = RTE_BE32(0xffffffff),
+			.proto = 0xff,
+			.hop_limits = 0xff,
+		},
+	};
+	void *headers_m;
+	void *headers_v;
+	void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
+	void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
+	char *l24_m;
+	char *l24_v;
+	uint32_t vtc_m;
+	uint32_t vtc_v;
+	int i;
+	int size;
+
+	if (inner) {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 inner_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
+	} else {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 outer_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
+	}
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
+	if (!ipv6_v)
+		return;
+	if (!ipv6_m)
+		ipv6_m = &nic_mask;
+	size = sizeof(ipv6_m->hdr.dst_addr);
+	l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
+			     dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
+	l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
+			     dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
+	memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
+	for (i = 0; i < size; ++i)
+		l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
+	l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
+			     src_ipv4_src_ipv6.ipv6_layout.ipv6);
+	l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
+			     src_ipv4_src_ipv6.ipv6_layout.ipv6);
+	memcpy(l24_m, ipv6_m->hdr.src_addr, size);
+	for (i = 0; i < size; ++i)
+		l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
+	/* TOS. */
+	vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
+	vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
+	/* Label. */
+	if (inner) {
+		MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
+			 vtc_m);
+		MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
+			 vtc_v);
+	} else {
+		MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
+			 vtc_m);
+		MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
+			 vtc_v);
+	}
+	/* Protocol. */
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
+		 ipv6_m->hdr.proto);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
+		 ipv6_v->hdr.proto & ipv6_m->hdr.proto);
+}
+
+/**
+ * Add TCP item to matcher and to the value.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_tcp(void *matcher, void *key,
+			   const struct rte_flow_item *item,
+			   int inner)
+{
+	const struct rte_flow_item_tcp *tcp_m = item->mask;
+	const struct rte_flow_item_tcp *tcp_v = item->spec;
+	void *headers_m;
+	void *headers_v;
+
+	if (inner) {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 inner_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
+	} else {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 outer_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
+	}
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
+	if (!tcp_v)
+		return;
+	if (!tcp_m)
+		tcp_m = &rte_flow_item_tcp_mask;
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
+		 rte_be_to_cpu_16(tcp_m->hdr.src_port));
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
+		 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
+		 rte_be_to_cpu_16(tcp_m->hdr.dst_port));
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
+		 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
+}
+
+/**
+ * Add UDP item to matcher and to the value.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_udp(void *matcher, void *key,
+			   const struct rte_flow_item *item,
+			   int inner)
+{
+	const struct rte_flow_item_udp *udp_m = item->mask;
+	const struct rte_flow_item_udp *udp_v = item->spec;
+	void *headers_m;
+	void *headers_v;
+
+	if (inner) {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 inner_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
+	} else {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 outer_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
+	}
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
+	if (!udp_v)
+		return;
+	if (!udp_m)
+		udp_m = &rte_flow_item_udp_mask;
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
+		 rte_be_to_cpu_16(udp_m->hdr.src_port));
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
+		 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
+		 rte_be_to_cpu_16(udp_m->hdr.dst_port));
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
+		 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
+}
+
+/**
+ * Add GRE item to matcher and to the value.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_gre(void *matcher, void *key,
+			   const struct rte_flow_item *item,
+			   int inner)
+{
+	const struct rte_flow_item_gre *gre_m = item->mask;
+	const struct rte_flow_item_gre *gre_v = item->spec;
+	void *headers_m;
+	void *headers_v;
+	void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
+	void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
+
+	if (inner) {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 inner_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
+	} else {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 outer_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
+	}
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
+	if (!gre_v)
+		return;
+	if (!gre_m)
+		gre_m = &rte_flow_item_gre_mask;
+	MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
+		 rte_be_to_cpu_16(gre_m->protocol));
+	MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
+		 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
+}
+
+/**
+ * Add NVGRE item to matcher and to the value.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_nvgre(void *matcher, void *key,
+			     const struct rte_flow_item *item,
+			     int inner)
+{
+	const struct rte_flow_item_nvgre *nvgre_m = item->mask;
+	const struct rte_flow_item_nvgre *nvgre_v = item->spec;
+	void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
+	void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
+	const char *tni_flow_id_m = (const char *)nvgre_m->tni;
+	const char *tni_flow_id_v = (const char *)nvgre_v->tni;
+	char *gre_key_m;
+	char *gre_key_v;
+	int size;
+	int i;
+
+	flow_dv_translate_item_gre(matcher, key, item, inner);
+	if (!nvgre_v)
+		return;
+	if (!nvgre_m)
+		nvgre_m = &rte_flow_item_nvgre_mask;
+	size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
+	gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
+	gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
+	memcpy(gre_key_m, tni_flow_id_m, size);
+	for (i = 0; i < size; ++i)
+		gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
+}
+
+/**
+ * Add VXLAN item to matcher and to the value.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_vxlan(void *matcher, void *key,
+			     const struct rte_flow_item *item,
+			     int inner)
+{
+	const struct rte_flow_item_vxlan *vxlan_m = item->mask;
+	const struct rte_flow_item_vxlan *vxlan_v = item->spec;
+	void *headers_m;
+	void *headers_v;
+	void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
+	void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
+	char *vni_m;
+	char *vni_v;
+	uint16_t dport;
+	int size;
+	int i;
+
+	if (inner) {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 inner_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
+	} else {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 outer_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
+	}
+	dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
+		MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
+	if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
+		MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
+	}
+	if (!vxlan_v)
+		return;
+	if (!vxlan_m)
+		vxlan_m = &rte_flow_item_vxlan_mask;
+	size = sizeof(vxlan_m->vni);
+	vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
+	vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
+	memcpy(vni_m, vxlan_m->vni, size);
+	for (i = 0; i < size; ++i)
+		vni_v[i] = vni_m[i] & vxlan_v->vni[i];
+}
+
+/**
+ * Add META item to matcher
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_meta(void *matcher, void *key,
+			    const struct rte_flow_item *item)
+{
+	const struct rte_flow_item_meta *meta_m;
+	const struct rte_flow_item_meta *meta_v;
+	void *misc2_m =
+		MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
+	void *misc2_v =
+		MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
+
+	meta_m = (const void *)item->mask;
+	if (!meta_m)
+		meta_m = &rte_flow_item_meta_mask;
+	meta_v = (const void *)item->spec;
+	if (meta_v) {
+		MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
+			 rte_be_to_cpu_32(meta_m->data));
+		MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
+			 rte_be_to_cpu_32(meta_v->data & meta_m->data));
+	}
+}
+
+/**
+ * Update the matcher and the value based the selected item.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in, out] dev_flow
+ *   Pointer to the mlx5_flow.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_create_item(void *matcher, void *key,
+		    const struct rte_flow_item *item,
+		    struct mlx5_flow *dev_flow,
+		    int inner)
+{
+	struct mlx5_flow_dv_matcher *tmatcher = matcher;
+
+	switch (item->type) {
+	case RTE_FLOW_ITEM_TYPE_ETH:
+		flow_dv_translate_item_eth(tmatcher->mask.buf, key, item,
+					   inner);
+		tmatcher->priority = MLX5_PRIORITY_MAP_L2;
+		break;
+	case RTE_FLOW_ITEM_TYPE_VLAN:
+		flow_dv_translate_item_vlan(tmatcher->mask.buf, key, item,
+					    inner);
+		break;
+	case RTE_FLOW_ITEM_TYPE_IPV4:
+		flow_dv_translate_item_ipv4(tmatcher->mask.buf, key, item,
+					    inner);
+		tmatcher->priority = MLX5_PRIORITY_MAP_L3;
+		dev_flow->dv.hash_fields |=
+			mlx5_flow_hashfields_adjust(dev_flow, inner,
+						    MLX5_IPV4_LAYER_TYPES,
+						    MLX5_IPV4_IBV_RX_HASH);
+		break;
+	case RTE_FLOW_ITEM_TYPE_IPV6:
+		flow_dv_translate_item_ipv6(tmatcher->mask.buf, key, item,
+					    inner);
+		tmatcher->priority = MLX5_PRIORITY_MAP_L3;
+		dev_flow->dv.hash_fields |=
+			mlx5_flow_hashfields_adjust(dev_flow, inner,
+						    MLX5_IPV6_LAYER_TYPES,
+						    MLX5_IPV6_IBV_RX_HASH);
+		break;
+	case RTE_FLOW_ITEM_TYPE_TCP:
+		flow_dv_translate_item_tcp(tmatcher->mask.buf, key, item,
+					   inner);
+		tmatcher->priority = MLX5_PRIORITY_MAP_L4;
+		dev_flow->dv.hash_fields |=
+			mlx5_flow_hashfields_adjust(dev_flow, inner,
+						    ETH_RSS_TCP,
+						    (IBV_RX_HASH_SRC_PORT_TCP |
+						     IBV_RX_HASH_DST_PORT_TCP));
+		break;
+	case RTE_FLOW_ITEM_TYPE_UDP:
+		flow_dv_translate_item_udp(tmatcher->mask.buf, key, item,
+					   inner);
+		tmatcher->priority = MLX5_PRIORITY_MAP_L4;
+		dev_flow->verbs.hash_fields |=
+			mlx5_flow_hashfields_adjust(dev_flow, inner,
+						    ETH_RSS_UDP,
+						    (IBV_RX_HASH_SRC_PORT_UDP |
+						     IBV_RX_HASH_DST_PORT_UDP));
+		break;
+	case RTE_FLOW_ITEM_TYPE_GRE:
+		flow_dv_translate_item_gre(tmatcher->mask.buf, key, item,
+					   inner);
+		break;
+	case RTE_FLOW_ITEM_TYPE_NVGRE:
+		flow_dv_translate_item_nvgre(tmatcher->mask.buf, key, item,
+					     inner);
+		break;
+	case RTE_FLOW_ITEM_TYPE_VXLAN:
+	case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+		flow_dv_translate_item_vxlan(tmatcher->mask.buf, key, item,
+					     inner);
+		break;
+	case RTE_FLOW_ITEM_TYPE_META:
+		flow_dv_translate_item_meta(tmatcher->mask.buf, key, item);
+		break;
+	default:
+		break;
+	}
+}
+
+/**
+ * Store the requested actions in an array.
+ *
+ * @param[in] action
+ *   Flow action to translate.
+ * @param[in, out] dev_flow
+ *   Pointer to the mlx5_flow.
+ */
+static void
+flow_dv_create_action(const struct rte_flow_action *action,
+		      struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_action_queue *queue;
+	const struct rte_flow_action_rss *rss;
+	int actions_n = dev_flow->dv.actions_n;
+	struct rte_flow *flow = dev_flow->flow;
+
+	switch (action->type) {
+	case RTE_FLOW_ACTION_TYPE_VOID:
+		break;
+	case RTE_FLOW_ACTION_TYPE_FLAG:
+		dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
+		dev_flow->dv.actions[actions_n].tag_value =
+			mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
+		actions_n++;
+		flow->actions |= MLX5_FLOW_ACTION_FLAG;
+		break;
+	case RTE_FLOW_ACTION_TYPE_MARK:
+		dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
+		dev_flow->dv.actions[actions_n].tag_value =
+			mlx5_flow_mark_set
+			(((const struct rte_flow_action_mark *)
+			  (action->conf))->id);
+		flow->actions |= MLX5_FLOW_ACTION_MARK;
+		actions_n++;
+		break;
+	case RTE_FLOW_ACTION_TYPE_DROP:
+		dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_DROP;
+		flow->actions |= MLX5_FLOW_ACTION_DROP;
+		break;
+	case RTE_FLOW_ACTION_TYPE_QUEUE:
+		queue = action->conf;
+		flow->rss.queue_num = 1;
+		(*flow->queue)[0] = queue->index;
+		flow->actions |= MLX5_FLOW_ACTION_QUEUE;
+		break;
+	case RTE_FLOW_ACTION_TYPE_RSS:
+		rss = action->conf;
+		if (flow->queue)
+			memcpy((*flow->queue), rss->queue,
+			       rss->queue_num * sizeof(uint16_t));
+		flow->rss.queue_num = rss->queue_num;
+		memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
+		flow->rss.types = rss->types;
+		flow->rss.level = rss->level;
+		/* Added to array only in apply since we need the QP */
+		flow->actions |= MLX5_FLOW_ACTION_RSS;
+		break;
+	default:
+		break;
+	}
+	dev_flow->dv.actions_n = actions_n;
+}
+
+static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
+
+#define HEADER_IS_ZERO(match_criteria, headers)				     \
+	!(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
+		 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
+
+/**
+ * Calculate flow matcher enable bitmap.
+ *
+ * @param match_criteria
+ *   Pointer to flow matcher criteria.
+ *
+ * @return
+ *   Bitmap of enabled fields.
+ */
+static uint8_t
+flow_dv_matcher_enable(uint32_t *match_criteria)
+{
+	uint8_t match_criteria_enable;
+
+	match_criteria_enable =
+		(!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
+		MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
+	match_criteria_enable |=
+		(!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
+		MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
+	match_criteria_enable |=
+		(!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
+		MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
+	match_criteria_enable |=
+		(!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
+		MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
+
+	return match_criteria_enable;
+}
+
+/**
+ * Register the flow matcher.
+ *
+ * @param dev[in, out]
+ *   Pointer to rte_eth_dev structure.
+ * @param[in, out] matcher
+ *   Pointer to flow matcher.
+ * @parm[in, out] dev_flow
+ *   Pointer to the dev_flow.
+ * @param[out] error
+ *   pointer to error structure.
+ *
+ * @return
+ *   0 on success otherwise -errno and errno is set.
+ */
+static int
+flow_dv_matcher_register(struct rte_eth_dev *dev,
+			 struct mlx5_flow_dv_matcher *matcher,
+			 struct mlx5_flow *dev_flow,
+			 struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_dv_matcher *cache_matcher;
+	struct mlx5dv_flow_matcher_attr dv_attr = {
+		.type = IBV_FLOW_ATTR_NORMAL,
+		.match_mask = (void *)&matcher->mask,
+	};
+
+	/* Lookup from cache. */
+	LIST_FOREACH(cache_matcher, &priv->matchers, next) {
+		if (matcher->crc == cache_matcher->crc &&
+		    matcher->priority == cache_matcher->priority &&
+		    matcher->egress == cache_matcher->egress &&
+		    !memcmp((const void *)matcher->mask.buf,
+			    (const void *)cache_matcher->mask.buf,
+			    cache_matcher->mask.size)) {
+			DRV_LOG(DEBUG,
+				"priority %hd use %s matcher %p: refcnt %d++",
+				cache_matcher->priority,
+				cache_matcher->egress ? "tx" : "rx",
+				(void *)cache_matcher,
+				rte_atomic32_read(&cache_matcher->refcnt));
+			rte_atomic32_inc(&cache_matcher->refcnt);
+			dev_flow->dv.matcher = cache_matcher;
+			return 0;
+		}
+	}
+	/* Register new matcher. */
+	cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
+	if (!cache_matcher)
+		return rte_flow_error_set(error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "cannot allocate matcher memory");
+	*cache_matcher = *matcher;
+	dv_attr.match_criteria_enable =
+		flow_dv_matcher_enable(cache_matcher->mask.buf);
+	dv_attr.priority = matcher->priority;
+	if (matcher->egress)
+		dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
+	cache_matcher->matcher_object =
+		mlx5_glue->dv_create_flow_matcher(priv->ctx, &dv_attr);
+	if (!cache_matcher->matcher_object)
+		return rte_flow_error_set(error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL, "cannot create matcher");
+	rte_atomic32_inc(&cache_matcher->refcnt);
+	LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next);
+	dev_flow->dv.matcher = cache_matcher;
+	DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
+		cache_matcher->priority,
+		cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
+		rte_atomic32_read(&cache_matcher->refcnt));
+	return 0;
+}
+
+
+/**
+ * Fill the flow with DV spec.
+ *
+ * @param[in] dev
+ *   Pointer to rte_eth_dev structure.
+ * @param[in, out] dev_flow
+ *   Pointer to the sub flow.
+ * @param[in] attr
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_ernno is set.
+ */
+static int
+flow_dv_translate(struct rte_eth_dev *dev,
+		  struct mlx5_flow *dev_flow,
+		  const struct rte_flow_attr *attr,
+		  const struct rte_flow_item items[],
+		  const struct rte_flow_action actions[] __rte_unused,
+		  struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint64_t priority = attr->priority;
+	struct mlx5_flow_dv_matcher matcher = {
+		.mask = {
+			.size = sizeof(matcher.mask.buf),
+		},
+	};
+	void *match_value = dev_flow->dv.value.buf;
+	int tunnel = 0;
+
+	if (priority == MLX5_FLOW_PRIO_RSVD)
+		priority = priv->config.flow_prio - 1;
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+		tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+		flow_dv_create_item(&matcher, match_value, items, dev_flow,
+				    tunnel);
+	}
+	matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
+				     matcher.mask.size);
+	if (priority == MLX5_FLOW_PRIO_RSVD)
+		priority = priv->config.flow_prio - 1;
+	matcher.priority = mlx5_flow_adjust_priority(dev, priority,
+						     matcher.priority);
+	matcher.egress = attr->egress;
+	if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
+		return -rte_errno;
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
+		flow_dv_create_action(actions, dev_flow);
+	return 0;
+}
+
+/**
+ * Apply the flow to the NIC.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
+	      struct rte_flow_error *error)
+{
+	struct mlx5_flow_dv *dv;
+	struct mlx5_flow *dev_flow;
+	int n;
+	int err;
+
+	LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
+		dv = &dev_flow->dv;
+		n = dv->actions_n;
+		if (flow->actions & MLX5_FLOW_ACTION_DROP) {
+			dv->hrxq = mlx5_hrxq_drop_new(dev);
+			if (!dv->hrxq) {
+				rte_flow_error_set
+					(error, errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					 "cannot get drop hash queue");
+				goto error;
+			}
+			dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
+			dv->actions[n].qp = dv->hrxq->qp;
+			n++;
+		} else if (flow->actions &
+			   (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
+			struct mlx5_hrxq *hrxq;
+			hrxq = mlx5_hrxq_get(dev, flow->key,
+					     MLX5_RSS_HASH_KEY_LEN,
+					     dv->hash_fields,
+					     (*flow->queue),
+					     flow->rss.queue_num);
+			if (!hrxq)
+				hrxq = mlx5_hrxq_new
+					(dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
+					 dv->hash_fields, (*flow->queue),
+					 flow->rss.queue_num,
+					 !!(dev_flow->layers &
+					    MLX5_FLOW_LAYER_TUNNEL));
+			if (!hrxq) {
+				rte_flow_error_set
+					(error, rte_errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					 "cannot get hash queue");
+				goto error;
+			}
+			dv->hrxq = hrxq;
+			dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
+			dv->actions[n].qp = hrxq->qp;
+			n++;
+		}
+		dv->flow =
+			mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
+						  (void *)&dv->value, n,
+						  dv->actions);
+		if (!dv->flow) {
+			rte_flow_error_set(error, errno,
+					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					   NULL,
+					   "hardware refuses to create flow");
+			goto error;
+		}
+	}
+	return 0;
+error:
+	err = rte_errno; /* Save rte_errno before cleanup. */
+	LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
+		struct mlx5_flow_dv *dv = &dev_flow->dv;
+		if (dv->hrxq) {
+			if (flow->actions & MLX5_FLOW_ACTION_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, dv->hrxq);
+			dv->hrxq = NULL;
+		}
+	}
+	rte_errno = err; /* Restore rte_errno. */
+	return -rte_errno;
+}
+
+/**
+ * Release the flow matcher.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param flow
+ *   Pointer to mlx5_flow.
+ *
+ * @return
+ *   1 while a reference on it exists, 0 when freed.
+ */
+static int
+flow_dv_matcher_release(struct rte_eth_dev *dev,
+			struct mlx5_flow *flow)
+{
+	struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
+
+	assert(matcher->matcher_object);
+	DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
+		dev->data->port_id, (void *)matcher,
+		rte_atomic32_read(&matcher->refcnt));
+	if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
+		claim_zero(mlx5_glue->dv_destroy_flow_matcher
+			   (matcher->matcher_object));
+		LIST_REMOVE(matcher, next);
+		rte_free(matcher);
+		DRV_LOG(DEBUG, "port %u matcher %p: removed",
+			dev->data->port_id, (void *)matcher);
+		return 0;
+	}
+	return 1;
+}
+
+/**
+ * Remove the flow from the NIC but keeps it in memory.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ */
+static void
+flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct mlx5_flow_dv *dv;
+	struct mlx5_flow *dev_flow;
+
+	if (!flow)
+		return;
+	LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
+		dv = &dev_flow->dv;
+		if (dv->flow) {
+			claim_zero(mlx5_glue->destroy_flow(dv->flow));
+			dv->flow = NULL;
+		}
+		if (dv->hrxq) {
+			if (flow->actions & MLX5_FLOW_ACTION_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, dv->hrxq);
+			dv->hrxq = NULL;
+		}
+	}
+	if (flow->counter)
+		flow->counter = NULL;
+}
+
+/**
+ * Remove the flow from the NIC and the memory.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ */
+static void
+flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct mlx5_flow *dev_flow;
+
+	if (!flow)
+		return;
+	flow_dv_remove(dev, flow);
+	while (!LIST_EMPTY(&flow->dev_flows)) {
+		dev_flow = LIST_FIRST(&flow->dev_flows);
+		LIST_REMOVE(dev_flow, next);
+		if (dev_flow->dv.matcher)
+			flow_dv_matcher_release(dev, dev_flow);
+		rte_free(dev_flow);
+	}
+}
+
+/**
+ * Query a flow.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+static int
+flow_dv_query(struct rte_eth_dev *dev __rte_unused,
+	      struct rte_flow *flow __rte_unused,
+	      const struct rte_flow_action *actions __rte_unused,
+	      void *data __rte_unused,
+	      struct rte_flow_error *error __rte_unused)
+{
+	rte_errno = ENOTSUP;
+	return -rte_errno;
+}
+
+
+const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
+	.validate = flow_dv_validate,
+	.prepare = flow_dv_prepare,
+	.translate = flow_dv_translate,
+	.apply = flow_dv_apply,
+	.remove = flow_dv_remove,
+	.destroy = flow_dv_destroy,
+	.query = flow_dv_query,
+};
+
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
diff --git a/drivers/net/mlx5/mlx5_flow_tcf.c b/drivers/net/mlx5/mlx5_flow_tcf.c
new file mode 100644
index 00000000..719fb106
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_flow_tcf.c
@@ -0,0 +1,2913 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <libmnl/libmnl.h>
+#include <linux/gen_stats.h>
+#include <linux/if_ether.h>
+#include <linux/netlink.h>
+#include <linux/pkt_cls.h>
+#include <linux/pkt_sched.h>
+#include <linux/rtnetlink.h>
+#include <linux/tc_act/tc_gact.h>
+#include <linux/tc_act/tc_mirred.h>
+#include <netinet/in.h>
+#include <stdalign.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/socket.h>
+
+#include <rte_byteorder.h>
+#include <rte_errno.h>
+#include <rte_ether.h>
+#include <rte_flow.h>
+#include <rte_malloc.h>
+#include <rte_common.h>
+
+#include "mlx5.h"
+#include "mlx5_flow.h"
+#include "mlx5_autoconf.h"
+
+#ifdef HAVE_TC_ACT_VLAN
+
+#include <linux/tc_act/tc_vlan.h>
+
+#else /* HAVE_TC_ACT_VLAN */
+
+#define TCA_VLAN_ACT_POP 1
+#define TCA_VLAN_ACT_PUSH 2
+#define TCA_VLAN_ACT_MODIFY 3
+#define TCA_VLAN_PARMS 2
+#define TCA_VLAN_PUSH_VLAN_ID 3
+#define TCA_VLAN_PUSH_VLAN_PROTOCOL 4
+#define TCA_VLAN_PAD 5
+#define TCA_VLAN_PUSH_VLAN_PRIORITY 6
+
+struct tc_vlan {
+	tc_gen;
+	int v_action;
+};
+
+#endif /* HAVE_TC_ACT_VLAN */
+
+#ifdef HAVE_TC_ACT_PEDIT
+
+#include <linux/tc_act/tc_pedit.h>
+
+#else /* HAVE_TC_ACT_VLAN */
+
+enum {
+	TCA_PEDIT_UNSPEC,
+	TCA_PEDIT_TM,
+	TCA_PEDIT_PARMS,
+	TCA_PEDIT_PAD,
+	TCA_PEDIT_PARMS_EX,
+	TCA_PEDIT_KEYS_EX,
+	TCA_PEDIT_KEY_EX,
+	__TCA_PEDIT_MAX
+};
+
+enum {
+	TCA_PEDIT_KEY_EX_HTYPE = 1,
+	TCA_PEDIT_KEY_EX_CMD = 2,
+	__TCA_PEDIT_KEY_EX_MAX
+};
+
+enum pedit_header_type {
+	TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK = 0,
+	TCA_PEDIT_KEY_EX_HDR_TYPE_ETH = 1,
+	TCA_PEDIT_KEY_EX_HDR_TYPE_IP4 = 2,
+	TCA_PEDIT_KEY_EX_HDR_TYPE_IP6 = 3,
+	TCA_PEDIT_KEY_EX_HDR_TYPE_TCP = 4,
+	TCA_PEDIT_KEY_EX_HDR_TYPE_UDP = 5,
+	__PEDIT_HDR_TYPE_MAX,
+};
+
+enum pedit_cmd {
+	TCA_PEDIT_KEY_EX_CMD_SET = 0,
+	TCA_PEDIT_KEY_EX_CMD_ADD = 1,
+	__PEDIT_CMD_MAX,
+};
+
+struct tc_pedit_key {
+	__u32 mask; /* AND */
+	__u32 val; /*XOR */
+	__u32 off; /*offset */
+	__u32 at;
+	__u32 offmask;
+	__u32 shift;
+};
+
+__extension__
+struct tc_pedit_sel {
+	tc_gen;
+	unsigned char nkeys;
+	unsigned char flags;
+	struct tc_pedit_key keys[0];
+};
+
+#endif /* HAVE_TC_ACT_VLAN */
+
+/* Normally found in linux/netlink.h. */
+#ifndef NETLINK_CAP_ACK
+#define NETLINK_CAP_ACK 10
+#endif
+
+/* Normally found in linux/pkt_sched.h. */
+#ifndef TC_H_MIN_INGRESS
+#define TC_H_MIN_INGRESS 0xfff2u
+#endif
+
+/* Normally found in linux/pkt_cls.h. */
+#ifndef TCA_CLS_FLAGS_SKIP_SW
+#define TCA_CLS_FLAGS_SKIP_SW (1 << 1)
+#endif
+#ifndef HAVE_TCA_CHAIN
+#define TCA_CHAIN 11
+#endif
+#ifndef HAVE_TCA_FLOWER_ACT
+#define TCA_FLOWER_ACT 3
+#endif
+#ifndef HAVE_TCA_FLOWER_FLAGS
+#define TCA_FLOWER_FLAGS 22
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_TYPE
+#define TCA_FLOWER_KEY_ETH_TYPE 8
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_DST
+#define TCA_FLOWER_KEY_ETH_DST 4
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_DST_MASK
+#define TCA_FLOWER_KEY_ETH_DST_MASK 5
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC
+#define TCA_FLOWER_KEY_ETH_SRC 6
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK
+#define TCA_FLOWER_KEY_ETH_SRC_MASK 7
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IP_PROTO
+#define TCA_FLOWER_KEY_IP_PROTO 9
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC
+#define TCA_FLOWER_KEY_IPV4_SRC 10
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK
+#define TCA_FLOWER_KEY_IPV4_SRC_MASK 11
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST
+#define TCA_FLOWER_KEY_IPV4_DST 12
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK
+#define TCA_FLOWER_KEY_IPV4_DST_MASK 13
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC
+#define TCA_FLOWER_KEY_IPV6_SRC 14
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK
+#define TCA_FLOWER_KEY_IPV6_SRC_MASK 15
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST
+#define TCA_FLOWER_KEY_IPV6_DST 16
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK
+#define TCA_FLOWER_KEY_IPV6_DST_MASK 17
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC
+#define TCA_FLOWER_KEY_TCP_SRC 18
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK
+#define TCA_FLOWER_KEY_TCP_SRC_MASK 35
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_DST
+#define TCA_FLOWER_KEY_TCP_DST 19
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_DST_MASK
+#define TCA_FLOWER_KEY_TCP_DST_MASK 36
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC
+#define TCA_FLOWER_KEY_UDP_SRC 20
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK
+#define TCA_FLOWER_KEY_UDP_SRC_MASK 37
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_DST
+#define TCA_FLOWER_KEY_UDP_DST 21
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_DST_MASK
+#define TCA_FLOWER_KEY_UDP_DST_MASK 38
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_VLAN_ID
+#define TCA_FLOWER_KEY_VLAN_ID 23
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_VLAN_PRIO
+#define TCA_FLOWER_KEY_VLAN_PRIO 24
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE
+#define TCA_FLOWER_KEY_VLAN_ETH_TYPE 25
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS
+#define TCA_FLOWER_KEY_TCP_FLAGS 71
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_FLAGS_MASK
+#define TCA_FLOWER_KEY_TCP_FLAGS_MASK 72
+#endif
+#ifndef HAVE_TC_ACT_GOTO_CHAIN
+#define TC_ACT_GOTO_CHAIN 0x20000000
+#endif
+
+#ifndef IPV6_ADDR_LEN
+#define IPV6_ADDR_LEN 16
+#endif
+
+#ifndef IPV4_ADDR_LEN
+#define IPV4_ADDR_LEN 4
+#endif
+
+#ifndef TP_PORT_LEN
+#define TP_PORT_LEN 2 /* Transport Port (UDP/TCP) Length */
+#endif
+
+#ifndef TTL_LEN
+#define TTL_LEN 1
+#endif
+
+#ifndef TCA_ACT_MAX_PRIO
+#define TCA_ACT_MAX_PRIO 32
+#endif
+
+/**
+ * Structure for holding netlink context.
+ * Note the size of the message buffer which is MNL_SOCKET_BUFFER_SIZE.
+ * Using this (8KB) buffer size ensures that netlink messages will never be
+ * truncated.
+ */
+struct mlx5_flow_tcf_context {
+	struct mnl_socket *nl; /* NETLINK_ROUTE libmnl socket. */
+	uint32_t seq; /* Message sequence number. */
+	uint32_t buf_size; /* Message buffer size. */
+	uint8_t *buf; /* Message buffer. */
+};
+
+/** Structure used when extracting the values of a flow counters
+ * from a netlink message.
+ */
+struct flow_tcf_stats_basic {
+	bool valid;
+	struct gnet_stats_basic counters;
+};
+
+/** Empty masks for known item types. */
+static const union {
+	struct rte_flow_item_port_id port_id;
+	struct rte_flow_item_eth eth;
+	struct rte_flow_item_vlan vlan;
+	struct rte_flow_item_ipv4 ipv4;
+	struct rte_flow_item_ipv6 ipv6;
+	struct rte_flow_item_tcp tcp;
+	struct rte_flow_item_udp udp;
+} flow_tcf_mask_empty;
+
+/** Supported masks for known item types. */
+static const struct {
+	struct rte_flow_item_port_id port_id;
+	struct rte_flow_item_eth eth;
+	struct rte_flow_item_vlan vlan;
+	struct rte_flow_item_ipv4 ipv4;
+	struct rte_flow_item_ipv6 ipv6;
+	struct rte_flow_item_tcp tcp;
+	struct rte_flow_item_udp udp;
+} flow_tcf_mask_supported = {
+	.port_id = {
+		.id = 0xffffffff,
+	},
+	.eth = {
+		.type = RTE_BE16(0xffff),
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+	},
+	.vlan = {
+		/* PCP and VID only, no DEI. */
+		.tci = RTE_BE16(0xefff),
+		.inner_type = RTE_BE16(0xffff),
+	},
+	.ipv4.hdr = {
+		.next_proto_id = 0xff,
+		.src_addr = RTE_BE32(0xffffffff),
+		.dst_addr = RTE_BE32(0xffffffff),
+	},
+	.ipv6.hdr = {
+		.proto = 0xff,
+		.src_addr =
+			"\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff",
+		.dst_addr =
+			"\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff",
+	},
+	.tcp.hdr = {
+		.src_port = RTE_BE16(0xffff),
+		.dst_port = RTE_BE16(0xffff),
+		.tcp_flags = 0xff,
+	},
+	.udp.hdr = {
+		.src_port = RTE_BE16(0xffff),
+		.dst_port = RTE_BE16(0xffff),
+	},
+};
+
+#define SZ_NLATTR_HDR MNL_ALIGN(sizeof(struct nlattr))
+#define SZ_NLATTR_NEST SZ_NLATTR_HDR
+#define SZ_NLATTR_DATA_OF(len) MNL_ALIGN(SZ_NLATTR_HDR + (len))
+#define SZ_NLATTR_TYPE_OF(typ) SZ_NLATTR_DATA_OF(sizeof(typ))
+#define SZ_NLATTR_STRZ_OF(str) SZ_NLATTR_DATA_OF(strlen(str) + 1)
+
+#define PTOI_TABLE_SZ_MAX(dev) (mlx5_dev_to_port_id((dev)->device, NULL, 0) + 2)
+
+/** DPDK port to network interface index (ifindex) conversion. */
+struct flow_tcf_ptoi {
+	uint16_t port_id; /**< DPDK port ID. */
+	unsigned int ifindex; /**< Network interface index. */
+};
+
+/* Due to a limitation on driver/FW. */
+#define MLX5_TCF_GROUP_ID_MAX 3
+#define MLX5_TCF_GROUP_PRIORITY_MAX 14
+
+#define MLX5_TCF_FATE_ACTIONS \
+	(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_PORT_ID | \
+	 MLX5_FLOW_ACTION_JUMP)
+
+#define MLX5_TCF_VLAN_ACTIONS \
+	(MLX5_FLOW_ACTION_OF_POP_VLAN | MLX5_FLOW_ACTION_OF_PUSH_VLAN | \
+	 MLX5_FLOW_ACTION_OF_SET_VLAN_VID | MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
+
+#define MLX5_TCF_PEDIT_ACTIONS \
+	(MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST | \
+	 MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST | \
+	 MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST | \
+	 MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL | \
+	 MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)
+
+#define MLX5_TCF_CONFIG_ACTIONS \
+	(MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_JUMP | \
+	 MLX5_FLOW_ACTION_OF_PUSH_VLAN | MLX5_FLOW_ACTION_OF_SET_VLAN_VID | \
+	 MLX5_FLOW_ACTION_OF_SET_VLAN_PCP | \
+	 (MLX5_TCF_PEDIT_ACTIONS & ~MLX5_FLOW_ACTION_DEC_TTL))
+
+#define MAX_PEDIT_KEYS 128
+#define SZ_PEDIT_KEY_VAL 4
+
+#define NUM_OF_PEDIT_KEYS(sz) \
+	(((sz) / SZ_PEDIT_KEY_VAL) + (((sz) % SZ_PEDIT_KEY_VAL) ? 1 : 0))
+
+struct pedit_key_ex {
+	enum pedit_header_type htype;
+	enum pedit_cmd cmd;
+};
+
+struct pedit_parser {
+	struct tc_pedit_sel sel;
+	struct tc_pedit_key keys[MAX_PEDIT_KEYS];
+	struct pedit_key_ex keys_ex[MAX_PEDIT_KEYS];
+};
+
+/**
+ * Create space for using the implicitly created TC flow counter.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ *
+ * @return
+ *   A pointer to the counter data structure, NULL otherwise and
+ *   rte_errno is set.
+ */
+static struct mlx5_flow_counter *
+flow_tcf_counter_new(void)
+{
+	struct mlx5_flow_counter *cnt;
+
+	/*
+	 * eswitch counter cannot be shared and its id is unknown.
+	 * currently returning all with id 0.
+	 * in the future maybe better to switch to unique numbers.
+	 */
+	struct mlx5_flow_counter tmpl = {
+		.ref_cnt = 1,
+	};
+	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
+	if (!cnt) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	*cnt = tmpl;
+	/* Implicit counter, do not add to list. */
+	return cnt;
+}
+
+/**
+ * Set pedit key of MAC address
+ *
+ * @param[in] actions
+ *   pointer to action specification
+ * @param[in,out] p_parser
+ *   pointer to pedit_parser
+ */
+static void
+flow_tcf_pedit_key_set_mac(const struct rte_flow_action *actions,
+			   struct pedit_parser *p_parser)
+{
+	int idx = p_parser->sel.nkeys;
+	uint32_t off = actions->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
+					offsetof(struct ether_hdr, s_addr) :
+					offsetof(struct ether_hdr, d_addr);
+	const struct rte_flow_action_set_mac *conf =
+		(const struct rte_flow_action_set_mac *)actions->conf;
+
+	p_parser->keys[idx].off = off;
+	p_parser->keys[idx].mask = ~UINT32_MAX;
+	p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
+	p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
+	memcpy(&p_parser->keys[idx].val,
+		conf->mac_addr, SZ_PEDIT_KEY_VAL);
+	idx++;
+	p_parser->keys[idx].off = off + SZ_PEDIT_KEY_VAL;
+	p_parser->keys[idx].mask = 0xFFFF0000;
+	p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_ETH;
+	p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
+	memcpy(&p_parser->keys[idx].val,
+		conf->mac_addr + SZ_PEDIT_KEY_VAL,
+		ETHER_ADDR_LEN - SZ_PEDIT_KEY_VAL);
+	p_parser->sel.nkeys = (++idx);
+}
+
+/**
+ * Set pedit key of decrease/set ttl
+ *
+ * @param[in] actions
+ *   pointer to action specification
+ * @param[in,out] p_parser
+ *   pointer to pedit_parser
+ * @param[in] item_flags
+ *   flags of all items presented
+ */
+static void
+flow_tcf_pedit_key_set_dec_ttl(const struct rte_flow_action *actions,
+				struct pedit_parser *p_parser,
+				uint64_t item_flags)
+{
+	int idx = p_parser->sel.nkeys;
+
+	p_parser->keys[idx].mask = 0xFFFFFF00;
+	if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) {
+		p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
+		p_parser->keys[idx].off =
+			offsetof(struct ipv4_hdr, time_to_live);
+	}
+	if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6) {
+		p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
+		p_parser->keys[idx].off =
+			offsetof(struct ipv6_hdr, hop_limits);
+	}
+	if (actions->type == RTE_FLOW_ACTION_TYPE_DEC_TTL) {
+		p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_ADD;
+		p_parser->keys[idx].val = 0x000000FF;
+	} else {
+		p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
+		p_parser->keys[idx].val =
+			(__u32)((const struct rte_flow_action_set_ttl *)
+			 actions->conf)->ttl_value;
+	}
+	p_parser->sel.nkeys = (++idx);
+}
+
+/**
+ * Set pedit key of transport (TCP/UDP) port value
+ *
+ * @param[in] actions
+ *   pointer to action specification
+ * @param[in,out] p_parser
+ *   pointer to pedit_parser
+ * @param[in] item_flags
+ *   flags of all items presented
+ */
+static void
+flow_tcf_pedit_key_set_tp_port(const struct rte_flow_action *actions,
+				struct pedit_parser *p_parser,
+				uint64_t item_flags)
+{
+	int idx = p_parser->sel.nkeys;
+
+	if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)
+		p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_UDP;
+	if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP)
+		p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_TCP;
+	p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
+	/* offset of src/dst port is same for TCP and UDP */
+	p_parser->keys[idx].off =
+		actions->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
+		offsetof(struct tcp_hdr, src_port) :
+		offsetof(struct tcp_hdr, dst_port);
+	p_parser->keys[idx].mask = 0xFFFF0000;
+	p_parser->keys[idx].val =
+		(__u32)((const struct rte_flow_action_set_tp *)
+				actions->conf)->port;
+	p_parser->sel.nkeys = (++idx);
+}
+
+/**
+ * Set pedit key of ipv6 address
+ *
+ * @param[in] actions
+ *   pointer to action specification
+ * @param[in,out] p_parser
+ *   pointer to pedit_parser
+ */
+static void
+flow_tcf_pedit_key_set_ipv6_addr(const struct rte_flow_action *actions,
+				 struct pedit_parser *p_parser)
+{
+	int idx = p_parser->sel.nkeys;
+	int keys = NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
+	int off_base =
+		actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
+		offsetof(struct ipv6_hdr, src_addr) :
+		offsetof(struct ipv6_hdr, dst_addr);
+	const struct rte_flow_action_set_ipv6 *conf =
+		(const struct rte_flow_action_set_ipv6 *)actions->conf;
+
+	for (int i = 0; i < keys; i++, idx++) {
+		p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP6;
+		p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
+		p_parser->keys[idx].off = off_base + i * SZ_PEDIT_KEY_VAL;
+		p_parser->keys[idx].mask = ~UINT32_MAX;
+		memcpy(&p_parser->keys[idx].val,
+			conf->ipv6_addr + i *  SZ_PEDIT_KEY_VAL,
+			SZ_PEDIT_KEY_VAL);
+	}
+	p_parser->sel.nkeys += keys;
+}
+
+/**
+ * Set pedit key of ipv4 address
+ *
+ * @param[in] actions
+ *   pointer to action specification
+ * @param[in,out] p_parser
+ *   pointer to pedit_parser
+ */
+static void
+flow_tcf_pedit_key_set_ipv4_addr(const struct rte_flow_action *actions,
+				 struct pedit_parser *p_parser)
+{
+	int idx = p_parser->sel.nkeys;
+
+	p_parser->keys_ex[idx].htype = TCA_PEDIT_KEY_EX_HDR_TYPE_IP4;
+	p_parser->keys_ex[idx].cmd = TCA_PEDIT_KEY_EX_CMD_SET;
+	p_parser->keys[idx].off =
+		actions->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
+		offsetof(struct ipv4_hdr, src_addr) :
+		offsetof(struct ipv4_hdr, dst_addr);
+	p_parser->keys[idx].mask = ~UINT32_MAX;
+	p_parser->keys[idx].val =
+		((const struct rte_flow_action_set_ipv4 *)
+		 actions->conf)->ipv4_addr;
+	p_parser->sel.nkeys = (++idx);
+}
+
+/**
+ * Create the pedit's na attribute in netlink message
+ * on pre-allocate message buffer
+ *
+ * @param[in,out] nl
+ *   pointer to pre-allocated netlink message buffer
+ * @param[in,out] actions
+ *   pointer to pointer of actions specification.
+ * @param[in,out] action_flags
+ *   pointer to actions flags
+ * @param[in] item_flags
+ *   flags of all item presented
+ */
+static void
+flow_tcf_create_pedit_mnl_msg(struct nlmsghdr *nl,
+			      const struct rte_flow_action **actions,
+			      uint64_t item_flags)
+{
+	struct pedit_parser p_parser;
+	struct nlattr *na_act_options;
+	struct nlattr *na_pedit_keys;
+
+	memset(&p_parser, 0, sizeof(p_parser));
+	mnl_attr_put_strz(nl, TCA_ACT_KIND, "pedit");
+	na_act_options = mnl_attr_nest_start(nl, TCA_ACT_OPTIONS);
+	/* all modify header actions should be in one tc-pedit action */
+	for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
+		switch ((*actions)->type) {
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
+			flow_tcf_pedit_key_set_ipv4_addr(*actions, &p_parser);
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
+			flow_tcf_pedit_key_set_ipv6_addr(*actions, &p_parser);
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
+			flow_tcf_pedit_key_set_tp_port(*actions,
+							&p_parser, item_flags);
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_TTL:
+		case RTE_FLOW_ACTION_TYPE_DEC_TTL:
+			flow_tcf_pedit_key_set_dec_ttl(*actions,
+							&p_parser, item_flags);
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
+			flow_tcf_pedit_key_set_mac(*actions, &p_parser);
+			break;
+		default:
+			goto pedit_mnl_msg_done;
+		}
+	}
+pedit_mnl_msg_done:
+	p_parser.sel.action = TC_ACT_PIPE;
+	mnl_attr_put(nl, TCA_PEDIT_PARMS_EX,
+		     sizeof(p_parser.sel) +
+		     p_parser.sel.nkeys * sizeof(struct tc_pedit_key),
+		     &p_parser);
+	na_pedit_keys =
+		mnl_attr_nest_start(nl, TCA_PEDIT_KEYS_EX | NLA_F_NESTED);
+	for (int i = 0; i < p_parser.sel.nkeys; i++) {
+		struct nlattr *na_pedit_key =
+			mnl_attr_nest_start(nl,
+					    TCA_PEDIT_KEY_EX | NLA_F_NESTED);
+		mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_HTYPE,
+				 p_parser.keys_ex[i].htype);
+		mnl_attr_put_u16(nl, TCA_PEDIT_KEY_EX_CMD,
+				 p_parser.keys_ex[i].cmd);
+		mnl_attr_nest_end(nl, na_pedit_key);
+	}
+	mnl_attr_nest_end(nl, na_pedit_keys);
+	mnl_attr_nest_end(nl, na_act_options);
+	(*actions)--;
+}
+
+/**
+ * Calculate max memory size of one TC-pedit actions.
+ * One TC-pedit action can contain set of keys each defining
+ * a rewrite element (rte_flow action)
+ *
+ * @param[in,out] actions
+ *   actions specification.
+ * @param[in,out] action_flags
+ *   actions flags
+ * @param[in,out] size
+ *   accumulated size
+ * @return
+ *   Max memory size of one TC-pedit action
+ */
+static int
+flow_tcf_get_pedit_actions_size(const struct rte_flow_action **actions,
+				uint64_t *action_flags)
+{
+	int pedit_size = 0;
+	int keys = 0;
+	uint64_t flags = 0;
+
+	pedit_size += SZ_NLATTR_NEST + /* na_act_index. */
+		      SZ_NLATTR_STRZ_OF("pedit") +
+		      SZ_NLATTR_NEST; /* TCA_ACT_OPTIONS. */
+	for (; (*actions)->type != RTE_FLOW_ACTION_TYPE_END; (*actions)++) {
+		switch ((*actions)->type) {
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
+			keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
+			flags |= MLX5_FLOW_ACTION_SET_IPV4_SRC;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
+			keys += NUM_OF_PEDIT_KEYS(IPV4_ADDR_LEN);
+			flags |= MLX5_FLOW_ACTION_SET_IPV4_DST;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
+			keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
+			flags |= MLX5_FLOW_ACTION_SET_IPV6_SRC;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
+			keys += NUM_OF_PEDIT_KEYS(IPV6_ADDR_LEN);
+			flags |= MLX5_FLOW_ACTION_SET_IPV6_DST;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
+			/* TCP is as same as UDP */
+			keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
+			flags |= MLX5_FLOW_ACTION_SET_TP_SRC;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
+			/* TCP is as same as UDP */
+			keys += NUM_OF_PEDIT_KEYS(TP_PORT_LEN);
+			flags |= MLX5_FLOW_ACTION_SET_TP_DST;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_TTL:
+			keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
+			flags |= MLX5_FLOW_ACTION_SET_TTL;
+			break;
+		case RTE_FLOW_ACTION_TYPE_DEC_TTL:
+			keys += NUM_OF_PEDIT_KEYS(TTL_LEN);
+			flags |= MLX5_FLOW_ACTION_DEC_TTL;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
+			keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
+			flags |= MLX5_FLOW_ACTION_SET_MAC_SRC;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
+			keys += NUM_OF_PEDIT_KEYS(ETHER_ADDR_LEN);
+			flags |= MLX5_FLOW_ACTION_SET_MAC_DST;
+			break;
+		default:
+			goto get_pedit_action_size_done;
+		}
+	}
+get_pedit_action_size_done:
+	/* TCA_PEDIT_PARAMS_EX */
+	pedit_size +=
+		SZ_NLATTR_DATA_OF(sizeof(struct tc_pedit_sel) +
+				  keys * sizeof(struct tc_pedit_key));
+	pedit_size += SZ_NLATTR_NEST; /* TCA_PEDIT_KEYS */
+	pedit_size += keys *
+		      /* TCA_PEDIT_KEY_EX + HTYPE + CMD */
+		      (SZ_NLATTR_NEST + SZ_NLATTR_DATA_OF(2) +
+		       SZ_NLATTR_DATA_OF(2));
+	(*action_flags) |= flags;
+	(*actions)--;
+	return pedit_size;
+}
+
+/**
+ * Retrieve mask for pattern item.
+ *
+ * This function does basic sanity checks on a pattern item in order to
+ * return the most appropriate mask for it.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in] mask_default
+ *   Default mask for pattern item as specified by the flow API.
+ * @param[in] mask_supported
+ *   Mask fields supported by the implementation.
+ * @param[in] mask_empty
+ *   Empty mask to return when there is no specification.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   Either @p item->mask or one of the mask parameters on success, NULL
+ *   otherwise and rte_errno is set.
+ */
+static const void *
+flow_tcf_item_mask(const struct rte_flow_item *item, const void *mask_default,
+		   const void *mask_supported, const void *mask_empty,
+		   size_t mask_size, struct rte_flow_error *error)
+{
+	const uint8_t *mask;
+	size_t i;
+
+	/* item->last and item->mask cannot exist without item->spec. */
+	if (!item->spec && (item->mask || item->last)) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ITEM, item,
+				   "\"mask\" or \"last\" field provided without"
+				   " a corresponding \"spec\"");
+		return NULL;
+	}
+	/* No spec, no mask, no problem. */
+	if (!item->spec)
+		return mask_empty;
+	mask = item->mask ? item->mask : mask_default;
+	assert(mask);
+	/*
+	 * Single-pass check to make sure that:
+	 * - Mask is supported, no bits are set outside mask_supported.
+	 * - Both item->spec and item->last are included in mask.
+	 */
+	for (i = 0; i != mask_size; ++i) {
+		if (!mask[i])
+			continue;
+		if ((mask[i] | ((const uint8_t *)mask_supported)[i]) !=
+		    ((const uint8_t *)mask_supported)[i]) {
+			rte_flow_error_set(error, ENOTSUP,
+					   RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask,
+					   "unsupported field found"
+					   " in \"mask\"");
+			return NULL;
+		}
+		if (item->last &&
+		    (((const uint8_t *)item->spec)[i] & mask[i]) !=
+		    (((const uint8_t *)item->last)[i] & mask[i])) {
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM_LAST,
+					   item->last,
+					   "range between \"spec\" and \"last\""
+					   " not comprised in \"mask\"");
+			return NULL;
+		}
+	}
+	return mask;
+}
+
+/**
+ * Build a conversion table between port ID and ifindex.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[out] ptoi
+ *   Pointer to ptoi table.
+ * @param[in] len
+ *   Size of ptoi table provided.
+ *
+ * @return
+ *   Size of ptoi table filled.
+ */
+static unsigned int
+flow_tcf_build_ptoi_table(struct rte_eth_dev *dev, struct flow_tcf_ptoi *ptoi,
+			  unsigned int len)
+{
+	unsigned int n = mlx5_dev_to_port_id(dev->device, NULL, 0);
+	uint16_t port_id[n + 1];
+	unsigned int i;
+	unsigned int own = 0;
+
+	/* At least one port is needed when no switch domain is present. */
+	if (!n) {
+		n = 1;
+		port_id[0] = dev->data->port_id;
+	} else {
+		n = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, n), n);
+	}
+	if (n > len)
+		return 0;
+	for (i = 0; i != n; ++i) {
+		struct rte_eth_dev_info dev_info;
+
+		rte_eth_dev_info_get(port_id[i], &dev_info);
+		if (port_id[i] == dev->data->port_id)
+			own = i;
+		ptoi[i].port_id = port_id[i];
+		ptoi[i].ifindex = dev_info.if_index;
+	}
+	/* Ensure first entry of ptoi[] is the current device. */
+	if (own) {
+		ptoi[n] = ptoi[0];
+		ptoi[0] = ptoi[own];
+		ptoi[own] = ptoi[n];
+	}
+	/* An entry with zero ifindex terminates ptoi[]. */
+	ptoi[n].port_id = 0;
+	ptoi[n].ifindex = 0;
+	return n;
+}
+
+/**
+ * Verify the @p attr will be correctly understood by the E-switch.
+ *
+ * @param[in] attr
+ *   Pointer to flow attributes
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_tcf_validate_attributes(const struct rte_flow_attr *attr,
+			     struct rte_flow_error *error)
+{
+	/*
+	 * Supported attributes: groups, some priorities and ingress only.
+	 * group is supported only if kernel supports chain. Don't care about
+	 * transfer as it is the caller's problem.
+	 */
+	if (attr->group > MLX5_TCF_GROUP_ID_MAX)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
+					  "group ID larger than "
+					  RTE_STR(MLX5_TCF_GROUP_ID_MAX)
+					  " isn't supported");
+	else if (attr->group > 0 &&
+		 attr->priority > MLX5_TCF_GROUP_PRIORITY_MAX)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+					  attr,
+					  "lowest priority level is "
+					  RTE_STR(MLX5_TCF_GROUP_PRIORITY_MAX)
+					  " when group is configured");
+	else if (attr->priority > 0xfffe)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+					  attr,
+					  "lowest priority level is 0xfffe");
+	if (!attr->ingress)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+					  attr, "only ingress is supported");
+	if (attr->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+					  attr, "egress is not supported");
+	return 0;
+}
+
+/**
+ * Validate flow for E-Switch.
+ *
+ * @param[in] priv
+ *   Pointer to the priv structure.
+ * @param[in] attr
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_ernno is set.
+ */
+static int
+flow_tcf_validate(struct rte_eth_dev *dev,
+		  const struct rte_flow_attr *attr,
+		  const struct rte_flow_item items[],
+		  const struct rte_flow_action actions[],
+		  struct rte_flow_error *error)
+{
+	union {
+		const struct rte_flow_item_port_id *port_id;
+		const struct rte_flow_item_eth *eth;
+		const struct rte_flow_item_vlan *vlan;
+		const struct rte_flow_item_ipv4 *ipv4;
+		const struct rte_flow_item_ipv6 *ipv6;
+		const struct rte_flow_item_tcp *tcp;
+		const struct rte_flow_item_udp *udp;
+	} spec, mask;
+	union {
+		const struct rte_flow_action_port_id *port_id;
+		const struct rte_flow_action_jump *jump;
+		const struct rte_flow_action_of_push_vlan *of_push_vlan;
+		const struct rte_flow_action_of_set_vlan_vid *
+			of_set_vlan_vid;
+		const struct rte_flow_action_of_set_vlan_pcp *
+			of_set_vlan_pcp;
+		const struct rte_flow_action_set_ipv4 *set_ipv4;
+		const struct rte_flow_action_set_ipv6 *set_ipv6;
+	} conf;
+	uint64_t item_flags = 0;
+	uint64_t action_flags = 0;
+	uint8_t next_protocol = -1;
+	unsigned int tcm_ifindex = 0;
+	uint8_t pedit_validated = 0;
+	struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
+	struct rte_eth_dev *port_id_dev = NULL;
+	bool in_port_id_set;
+	int ret;
+
+	claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
+						PTOI_TABLE_SZ_MAX(dev)));
+	ret = flow_tcf_validate_attributes(attr, error);
+	if (ret < 0)
+		return ret;
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+		unsigned int i;
+
+		switch (items->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_PORT_ID:
+			mask.port_id = flow_tcf_item_mask
+				(items, &rte_flow_item_port_id_mask,
+				 &flow_tcf_mask_supported.port_id,
+				 &flow_tcf_mask_empty.port_id,
+				 sizeof(flow_tcf_mask_supported.port_id),
+				 error);
+			if (!mask.port_id)
+				return -rte_errno;
+			if (mask.port_id == &flow_tcf_mask_empty.port_id) {
+				in_port_id_set = 1;
+				break;
+			}
+			spec.port_id = items->spec;
+			if (mask.port_id->id && mask.port_id->id != 0xffffffff)
+				return rte_flow_error_set
+					(error, ENOTSUP,
+					 RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+					 mask.port_id,
+					 "no support for partial mask on"
+					 " \"id\" field");
+			if (!mask.port_id->id)
+				i = 0;
+			else
+				for (i = 0; ptoi[i].ifindex; ++i)
+					if (ptoi[i].port_id == spec.port_id->id)
+						break;
+			if (!ptoi[i].ifindex)
+				return rte_flow_error_set
+					(error, ENODEV,
+					 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					 spec.port_id,
+					 "missing data to convert port ID to"
+					 " ifindex");
+			if (in_port_id_set && ptoi[i].ifindex != tcm_ifindex)
+				return rte_flow_error_set
+					(error, ENOTSUP,
+					 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					 spec.port_id,
+					 "cannot match traffic for"
+					 " several port IDs through"
+					 " a single flow rule");
+			tcm_ifindex = ptoi[i].ifindex;
+			in_port_id_set = 1;
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			ret = mlx5_flow_validate_item_eth(items, item_flags,
+							  error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_OUTER_L2;
+			/* TODO:
+			 * Redundant check due to different supported mask.
+			 * Same for the rest of items.
+			 */
+			mask.eth = flow_tcf_item_mask
+				(items, &rte_flow_item_eth_mask,
+				 &flow_tcf_mask_supported.eth,
+				 &flow_tcf_mask_empty.eth,
+				 sizeof(flow_tcf_mask_supported.eth),
+				 error);
+			if (!mask.eth)
+				return -rte_errno;
+			if (mask.eth->type && mask.eth->type !=
+			    RTE_BE16(0xffff))
+				return rte_flow_error_set
+					(error, ENOTSUP,
+					 RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+					 mask.eth,
+					 "no support for partial mask on"
+					 " \"type\" field");
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			ret = mlx5_flow_validate_item_vlan(items, item_flags,
+							   error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
+			mask.vlan = flow_tcf_item_mask
+				(items, &rte_flow_item_vlan_mask,
+				 &flow_tcf_mask_supported.vlan,
+				 &flow_tcf_mask_empty.vlan,
+				 sizeof(flow_tcf_mask_supported.vlan),
+				 error);
+			if (!mask.vlan)
+				return -rte_errno;
+			if ((mask.vlan->tci & RTE_BE16(0xe000) &&
+			     (mask.vlan->tci & RTE_BE16(0xe000)) !=
+			      RTE_BE16(0xe000)) ||
+			    (mask.vlan->tci & RTE_BE16(0x0fff) &&
+			     (mask.vlan->tci & RTE_BE16(0x0fff)) !=
+			      RTE_BE16(0x0fff)) ||
+			    (mask.vlan->inner_type &&
+			     mask.vlan->inner_type != RTE_BE16(0xffff)))
+				return rte_flow_error_set
+					(error, ENOTSUP,
+					 RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+					 mask.vlan,
+					 "no support for partial masks on"
+					 " \"tci\" (PCP and VID parts) and"
+					 " \"inner_type\" fields");
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			ret = mlx5_flow_validate_item_ipv4(items, item_flags,
+							   error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+			mask.ipv4 = flow_tcf_item_mask
+				(items, &rte_flow_item_ipv4_mask,
+				 &flow_tcf_mask_supported.ipv4,
+				 &flow_tcf_mask_empty.ipv4,
+				 sizeof(flow_tcf_mask_supported.ipv4),
+				 error);
+			if (!mask.ipv4)
+				return -rte_errno;
+			if (mask.ipv4->hdr.next_proto_id &&
+			    mask.ipv4->hdr.next_proto_id != 0xff)
+				return rte_flow_error_set
+					(error, ENOTSUP,
+					 RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+					 mask.ipv4,
+					 "no support for partial mask on"
+					 " \"hdr.next_proto_id\" field");
+			else if (mask.ipv4->hdr.next_proto_id)
+				next_protocol =
+					((const struct rte_flow_item_ipv4 *)
+					 (items->spec))->hdr.next_proto_id;
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			ret = mlx5_flow_validate_item_ipv6(items, item_flags,
+							   error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+			mask.ipv6 = flow_tcf_item_mask
+				(items, &rte_flow_item_ipv6_mask,
+				 &flow_tcf_mask_supported.ipv6,
+				 &flow_tcf_mask_empty.ipv6,
+				 sizeof(flow_tcf_mask_supported.ipv6),
+				 error);
+			if (!mask.ipv6)
+				return -rte_errno;
+			if (mask.ipv6->hdr.proto &&
+			    mask.ipv6->hdr.proto != 0xff)
+				return rte_flow_error_set
+					(error, ENOTSUP,
+					 RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+					 mask.ipv6,
+					 "no support for partial mask on"
+					 " \"hdr.proto\" field");
+			else if (mask.ipv6->hdr.proto)
+				next_protocol =
+					((const struct rte_flow_item_ipv6 *)
+					 (items->spec))->hdr.proto;
+			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			ret = mlx5_flow_validate_item_udp(items, item_flags,
+							  next_protocol, error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
+			mask.udp = flow_tcf_item_mask
+				(items, &rte_flow_item_udp_mask,
+				 &flow_tcf_mask_supported.udp,
+				 &flow_tcf_mask_empty.udp,
+				 sizeof(flow_tcf_mask_supported.udp),
+				 error);
+			if (!mask.udp)
+				return -rte_errno;
+			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			ret = mlx5_flow_validate_item_tcp
+					     (items, item_flags,
+					      next_protocol,
+					      &flow_tcf_mask_supported.tcp,
+					      error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
+			mask.tcp = flow_tcf_item_mask
+				(items, &rte_flow_item_tcp_mask,
+				 &flow_tcf_mask_supported.tcp,
+				 &flow_tcf_mask_empty.tcp,
+				 sizeof(flow_tcf_mask_supported.tcp),
+				 error);
+			if (!mask.tcp)
+				return -rte_errno;
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  NULL, "item not supported");
+		}
+	}
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		unsigned int i;
+		uint64_t current_action_flag = 0;
+
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_PORT_ID:
+			current_action_flag = MLX5_FLOW_ACTION_PORT_ID;
+			if (!actions->conf)
+				break;
+			conf.port_id = actions->conf;
+			if (conf.port_id->original)
+				i = 0;
+			else
+				for (i = 0; ptoi[i].ifindex; ++i)
+					if (ptoi[i].port_id == conf.port_id->id)
+						break;
+			if (!ptoi[i].ifindex)
+				return rte_flow_error_set
+					(error, ENODEV,
+					 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					 conf.port_id,
+					 "missing data to convert port ID to"
+					 " ifindex");
+			port_id_dev = &rte_eth_devices[conf.port_id->id];
+			break;
+		case RTE_FLOW_ACTION_TYPE_JUMP:
+			current_action_flag = MLX5_FLOW_ACTION_JUMP;
+			if (!actions->conf)
+				break;
+			conf.jump = actions->conf;
+			if (attr->group >= conf.jump->group)
+				return rte_flow_error_set
+					(error, ENOTSUP,
+					 RTE_FLOW_ERROR_TYPE_ACTION,
+					 actions,
+					 "can jump only to a group forward");
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			current_action_flag = MLX5_FLOW_ACTION_DROP;
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			break;
+		case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
+			current_action_flag = MLX5_FLOW_ACTION_OF_POP_VLAN;
+			break;
+		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
+			current_action_flag = MLX5_FLOW_ACTION_OF_PUSH_VLAN;
+			break;
+		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
+			if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
+				return rte_flow_error_set
+					(error, ENOTSUP,
+					 RTE_FLOW_ERROR_TYPE_ACTION, actions,
+					 "vlan modify is not supported,"
+					 " set action must follow push action");
+			current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
+			break;
+		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
+			if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
+				return rte_flow_error_set
+					(error, ENOTSUP,
+					 RTE_FLOW_ERROR_TYPE_ACTION, actions,
+					 "vlan modify is not supported,"
+					 " set action must follow push action");
+			current_action_flag = MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
+			current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_SRC;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
+			current_action_flag = MLX5_FLOW_ACTION_SET_IPV4_DST;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
+			current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_SRC;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
+			current_action_flag = MLX5_FLOW_ACTION_SET_IPV6_DST;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
+			current_action_flag = MLX5_FLOW_ACTION_SET_TP_SRC;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
+			current_action_flag = MLX5_FLOW_ACTION_SET_TP_DST;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_TTL:
+			current_action_flag = MLX5_FLOW_ACTION_SET_TTL;
+			break;
+		case RTE_FLOW_ACTION_TYPE_DEC_TTL:
+			current_action_flag = MLX5_FLOW_ACTION_DEC_TTL;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
+			current_action_flag = MLX5_FLOW_ACTION_SET_MAC_SRC;
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
+			current_action_flag = MLX5_FLOW_ACTION_SET_MAC_DST;
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (current_action_flag & MLX5_TCF_CONFIG_ACTIONS) {
+			if (!actions->conf)
+				return rte_flow_error_set(error, EINVAL,
+						RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+						actions,
+						"action configuration not set");
+		}
+		if ((current_action_flag & MLX5_TCF_PEDIT_ACTIONS) &&
+		    pedit_validated)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "set actions should be "
+						  "listed successively");
+		if ((current_action_flag & ~MLX5_TCF_PEDIT_ACTIONS) &&
+		    (action_flags & MLX5_TCF_PEDIT_ACTIONS))
+			pedit_validated = 1;
+		if ((current_action_flag & MLX5_TCF_FATE_ACTIONS) &&
+		    (action_flags & MLX5_TCF_FATE_ACTIONS))
+			return rte_flow_error_set(error, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "can't have multiple fate"
+						  " actions");
+		action_flags |= current_action_flag;
+	}
+	if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
+	    (action_flags & MLX5_FLOW_ACTION_DROP))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "set action is not compatible with "
+					  "drop action");
+	if ((action_flags & MLX5_TCF_PEDIT_ACTIONS) &&
+	    !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "set action must be followed by "
+					  "port_id action");
+	if (action_flags &
+	   (MLX5_FLOW_ACTION_SET_IPV4_SRC | MLX5_FLOW_ACTION_SET_IPV4_DST)) {
+		if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4))
+			return rte_flow_error_set(error, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "no ipv4 item found in"
+						  " pattern");
+	}
+	if (action_flags &
+	   (MLX5_FLOW_ACTION_SET_IPV6_SRC | MLX5_FLOW_ACTION_SET_IPV6_DST)) {
+		if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6))
+			return rte_flow_error_set(error, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "no ipv6 item found in"
+						  " pattern");
+	}
+	if (action_flags &
+	   (MLX5_FLOW_ACTION_SET_TP_SRC | MLX5_FLOW_ACTION_SET_TP_DST)) {
+		if (!(item_flags &
+		     (MLX5_FLOW_LAYER_OUTER_L4_UDP |
+		      MLX5_FLOW_LAYER_OUTER_L4_TCP)))
+			return rte_flow_error_set(error, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "no TCP/UDP item found in"
+						  " pattern");
+	}
+	/*
+	 * FW syndrome (0xA9C090):
+	 *     set_flow_table_entry: push vlan action fte in fdb can ONLY be
+	 *     forward to the uplink.
+	 */
+	if ((action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
+	    (action_flags & MLX5_FLOW_ACTION_PORT_ID) &&
+	    ((struct priv *)port_id_dev->data->dev_private)->representor)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION, actions,
+					  "vlan push can only be applied"
+					  " when forwarding to uplink port");
+	/*
+	 * FW syndrome (0x294609):
+	 *     set_flow_table_entry: modify/pop/push actions in fdb flow table
+	 *     are supported only while forwarding to vport.
+	 */
+	if ((action_flags & MLX5_TCF_VLAN_ACTIONS) &&
+	    !(action_flags & MLX5_FLOW_ACTION_PORT_ID))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION, actions,
+					  "vlan actions are supported"
+					  " only with port_id action");
+	if (!(action_flags & MLX5_TCF_FATE_ACTIONS))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, actions,
+					  "no fate action is found");
+	if (action_flags &
+	   (MLX5_FLOW_ACTION_SET_TTL | MLX5_FLOW_ACTION_DEC_TTL)) {
+		if (!(item_flags &
+		     (MLX5_FLOW_LAYER_OUTER_L3_IPV4 |
+		      MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
+			return rte_flow_error_set(error, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "no IP found in pattern");
+	}
+	if (action_flags &
+	    (MLX5_FLOW_ACTION_SET_MAC_SRC | MLX5_FLOW_ACTION_SET_MAC_DST)) {
+		if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L2))
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "no ethernet found in"
+						  " pattern");
+	}
+	return 0;
+}
+
+/**
+ * Calculate maximum size of memory for flow items of Linux TC flower and
+ * extract specified items.
+ *
+ * @param[in] items
+ *   Pointer to the list of items.
+ * @param[out] item_flags
+ *   Pointer to the detected items.
+ *
+ * @return
+ *   Maximum size of memory for items.
+ */
+static int
+flow_tcf_get_items_and_size(const struct rte_flow_attr *attr,
+			    const struct rte_flow_item items[],
+			    uint64_t *item_flags)
+{
+	int size = 0;
+	uint64_t flags = 0;
+
+	size += SZ_NLATTR_STRZ_OF("flower") +
+		SZ_NLATTR_NEST + /* TCA_OPTIONS. */
+		SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CLS_FLAGS_SKIP_SW. */
+	if (attr->group > 0)
+		size += SZ_NLATTR_TYPE_OF(uint32_t); /* TCA_CHAIN. */
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+		switch (items->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_PORT_ID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
+				SZ_NLATTR_DATA_OF(ETHER_ADDR_LEN) * 4;
+				/* dst/src MAC addr and mask. */
+			flags |= MLX5_FLOW_LAYER_OUTER_L2;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
+				SZ_NLATTR_TYPE_OF(uint16_t) +
+				/* VLAN Ether type. */
+				SZ_NLATTR_TYPE_OF(uint8_t) + /* VLAN prio. */
+				SZ_NLATTR_TYPE_OF(uint16_t); /* VLAN ID. */
+			flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
+				SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
+				SZ_NLATTR_TYPE_OF(uint32_t) * 4;
+				/* dst/src IP addr and mask. */
+			flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			size += SZ_NLATTR_TYPE_OF(uint16_t) + /* Ether type. */
+				SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
+				SZ_NLATTR_TYPE_OF(IPV6_ADDR_LEN) * 4;
+				/* dst/src IP addr and mask. */
+			flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
+				SZ_NLATTR_TYPE_OF(uint16_t) * 4;
+				/* dst/src port and mask. */
+			flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
+			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			size += SZ_NLATTR_TYPE_OF(uint8_t) + /* IP proto. */
+				SZ_NLATTR_TYPE_OF(uint16_t) * 4;
+				/* dst/src port and mask. */
+			flags |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
+			break;
+		default:
+			DRV_LOG(WARNING,
+				"unsupported item %p type %d,"
+				" items must be validated before flow creation",
+				(const void *)items, items->type);
+			break;
+		}
+	}
+	*item_flags = flags;
+	return size;
+}
+
+/**
+ * Calculate maximum size of memory for flow actions of Linux TC flower and
+ * extract specified actions.
+ *
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ * @param[out] action_flags
+ *   Pointer to the detected actions.
+ *
+ * @return
+ *   Maximum size of memory for actions.
+ */
+static int
+flow_tcf_get_actions_and_size(const struct rte_flow_action actions[],
+			      uint64_t *action_flags)
+{
+	int size = 0;
+	uint64_t flags = 0;
+
+	size += SZ_NLATTR_NEST; /* TCA_FLOWER_ACT. */
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_PORT_ID:
+			size += SZ_NLATTR_NEST + /* na_act_index. */
+				SZ_NLATTR_STRZ_OF("mirred") +
+				SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
+				SZ_NLATTR_TYPE_OF(struct tc_mirred);
+			flags |= MLX5_FLOW_ACTION_PORT_ID;
+			break;
+		case RTE_FLOW_ACTION_TYPE_JUMP:
+			size += SZ_NLATTR_NEST + /* na_act_index. */
+				SZ_NLATTR_STRZ_OF("gact") +
+				SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
+				SZ_NLATTR_TYPE_OF(struct tc_gact);
+			flags |= MLX5_FLOW_ACTION_JUMP;
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			size += SZ_NLATTR_NEST + /* na_act_index. */
+				SZ_NLATTR_STRZ_OF("gact") +
+				SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
+				SZ_NLATTR_TYPE_OF(struct tc_gact);
+			flags |= MLX5_FLOW_ACTION_DROP;
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			break;
+		case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
+			flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
+			goto action_of_vlan;
+		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
+			flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
+			goto action_of_vlan;
+		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
+			flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
+			goto action_of_vlan;
+		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
+			flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
+			goto action_of_vlan;
+action_of_vlan:
+			size += SZ_NLATTR_NEST + /* na_act_index. */
+				SZ_NLATTR_STRZ_OF("vlan") +
+				SZ_NLATTR_NEST + /* TCA_ACT_OPTIONS. */
+				SZ_NLATTR_TYPE_OF(struct tc_vlan) +
+				SZ_NLATTR_TYPE_OF(uint16_t) +
+				/* VLAN protocol. */
+				SZ_NLATTR_TYPE_OF(uint16_t) + /* VLAN ID. */
+				SZ_NLATTR_TYPE_OF(uint8_t); /* VLAN prio. */
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
+		case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
+		case RTE_FLOW_ACTION_TYPE_SET_TTL:
+		case RTE_FLOW_ACTION_TYPE_DEC_TTL:
+		case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
+			size += flow_tcf_get_pedit_actions_size(&actions,
+								&flags);
+			break;
+		default:
+			DRV_LOG(WARNING,
+				"unsupported action %p type %d,"
+				" items must be validated before flow creation",
+				(const void *)actions, actions->type);
+			break;
+		}
+	}
+	*action_flags = flags;
+	return size;
+}
+
+/**
+ * Brand rtnetlink buffer with unique handle.
+ *
+ * This handle should be unique for a given network interface to avoid
+ * collisions.
+ *
+ * @param nlh
+ *   Pointer to Netlink message.
+ * @param handle
+ *   Unique 32-bit handle to use.
+ */
+static void
+flow_tcf_nl_brand(struct nlmsghdr *nlh, uint32_t handle)
+{
+	struct tcmsg *tcm = mnl_nlmsg_get_payload(nlh);
+
+	tcm->tcm_handle = handle;
+	DRV_LOG(DEBUG, "Netlink msg %p is branded with handle %x",
+		(void *)nlh, handle);
+}
+
+/**
+ * Prepare a flow object for Linux TC flower. It calculates the maximum size of
+ * memory required, allocates the memory, initializes Netlink message headers
+ * and set unique TC message handle.
+ *
+ * @param[in] attr
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ * @param[out] item_flags
+ *   Pointer to bit mask of all items detected.
+ * @param[out] action_flags
+ *   Pointer to bit mask of all actions detected.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   Pointer to mlx5_flow object on success,
+ *   otherwise NULL and rte_ernno is set.
+ */
+static struct mlx5_flow *
+flow_tcf_prepare(const struct rte_flow_attr *attr,
+		 const struct rte_flow_item items[],
+		 const struct rte_flow_action actions[],
+		 uint64_t *item_flags, uint64_t *action_flags,
+		 struct rte_flow_error *error)
+{
+	size_t size = sizeof(struct mlx5_flow) +
+		      MNL_ALIGN(sizeof(struct nlmsghdr)) +
+		      MNL_ALIGN(sizeof(struct tcmsg));
+	struct mlx5_flow *dev_flow;
+	struct nlmsghdr *nlh;
+	struct tcmsg *tcm;
+
+	size += flow_tcf_get_items_and_size(attr, items, item_flags);
+	size += flow_tcf_get_actions_and_size(actions, action_flags);
+	dev_flow = rte_zmalloc(__func__, size, MNL_ALIGNTO);
+	if (!dev_flow) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "not enough memory to create E-Switch flow");
+		return NULL;
+	}
+	nlh = mnl_nlmsg_put_header((void *)(dev_flow + 1));
+	tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
+	*dev_flow = (struct mlx5_flow){
+		.tcf = (struct mlx5_flow_tcf){
+			.nlh = nlh,
+			.tcm = tcm,
+		},
+	};
+	/*
+	 * Generate a reasonably unique handle based on the address of the
+	 * target buffer.
+	 *
+	 * This is straightforward on 32-bit systems where the flow pointer can
+	 * be used directly. Otherwise, its least significant part is taken
+	 * after shifting it by the previous power of two of the pointed buffer
+	 * size.
+	 */
+	if (sizeof(dev_flow) <= 4)
+		flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow);
+	else
+		flow_tcf_nl_brand(nlh, (uintptr_t)dev_flow >>
+				       rte_log2_u32(rte_align32prevpow2(size)));
+	return dev_flow;
+}
+
+/**
+ * Make adjustments for supporting count actions.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] dev_flow
+ *   Pointer to mlx5_flow.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 On success else a negative errno value is returned and rte_errno is set.
+ */
+static int
+flow_tcf_translate_action_count(struct rte_eth_dev *dev __rte_unused,
+				  struct mlx5_flow *dev_flow,
+				  struct rte_flow_error *error)
+{
+	struct rte_flow *flow = dev_flow->flow;
+
+	if (!flow->counter) {
+		flow->counter = flow_tcf_counter_new();
+		if (!flow->counter)
+			return rte_flow_error_set(error, rte_errno,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  NULL,
+						  "cannot get counter"
+						  " context.");
+	}
+	return 0;
+}
+
+/**
+ * Translate flow for Linux TC flower and construct Netlink message.
+ *
+ * @param[in] priv
+ *   Pointer to the priv structure.
+ * @param[in, out] flow
+ *   Pointer to the sub flow.
+ * @param[in] attr
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_ernno is set.
+ */
+static int
+flow_tcf_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow,
+		   const struct rte_flow_attr *attr,
+		   const struct rte_flow_item items[],
+		   const struct rte_flow_action actions[],
+		   struct rte_flow_error *error)
+{
+	union {
+		const struct rte_flow_item_port_id *port_id;
+		const struct rte_flow_item_eth *eth;
+		const struct rte_flow_item_vlan *vlan;
+		const struct rte_flow_item_ipv4 *ipv4;
+		const struct rte_flow_item_ipv6 *ipv6;
+		const struct rte_flow_item_tcp *tcp;
+		const struct rte_flow_item_udp *udp;
+	} spec, mask;
+	union {
+		const struct rte_flow_action_port_id *port_id;
+		const struct rte_flow_action_jump *jump;
+		const struct rte_flow_action_of_push_vlan *of_push_vlan;
+		const struct rte_flow_action_of_set_vlan_vid *
+			of_set_vlan_vid;
+		const struct rte_flow_action_of_set_vlan_pcp *
+			of_set_vlan_pcp;
+	} conf;
+	struct flow_tcf_ptoi ptoi[PTOI_TABLE_SZ_MAX(dev)];
+	struct nlmsghdr *nlh = dev_flow->tcf.nlh;
+	struct tcmsg *tcm = dev_flow->tcf.tcm;
+	uint32_t na_act_index_cur;
+	bool eth_type_set = 0;
+	bool vlan_present = 0;
+	bool vlan_eth_type_set = 0;
+	bool ip_proto_set = 0;
+	struct nlattr *na_flower;
+	struct nlattr *na_flower_act;
+	struct nlattr *na_vlan_id = NULL;
+	struct nlattr *na_vlan_priority = NULL;
+	uint64_t item_flags = 0;
+	int ret;
+
+	claim_nonzero(flow_tcf_build_ptoi_table(dev, ptoi,
+						PTOI_TABLE_SZ_MAX(dev)));
+	nlh = dev_flow->tcf.nlh;
+	tcm = dev_flow->tcf.tcm;
+	/* Prepare API must have been called beforehand. */
+	assert(nlh != NULL && tcm != NULL);
+	tcm->tcm_family = AF_UNSPEC;
+	tcm->tcm_ifindex = ptoi[0].ifindex;
+	tcm->tcm_parent = TC_H_MAKE(TC_H_INGRESS, TC_H_MIN_INGRESS);
+	/*
+	 * Priority cannot be zero to prevent the kernel from picking one
+	 * automatically.
+	 */
+	tcm->tcm_info = TC_H_MAKE((attr->priority + 1) << 16,
+				  RTE_BE16(ETH_P_ALL));
+	if (attr->group > 0)
+		mnl_attr_put_u32(nlh, TCA_CHAIN, attr->group);
+	mnl_attr_put_strz(nlh, TCA_KIND, "flower");
+	na_flower = mnl_attr_nest_start(nlh, TCA_OPTIONS);
+	mnl_attr_put_u32(nlh, TCA_FLOWER_FLAGS, TCA_CLS_FLAGS_SKIP_SW);
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+		unsigned int i;
+
+		switch (items->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_PORT_ID:
+			mask.port_id = flow_tcf_item_mask
+				(items, &rte_flow_item_port_id_mask,
+				 &flow_tcf_mask_supported.port_id,
+				 &flow_tcf_mask_empty.port_id,
+				 sizeof(flow_tcf_mask_supported.port_id),
+				 error);
+			assert(mask.port_id);
+			if (mask.port_id == &flow_tcf_mask_empty.port_id)
+				break;
+			spec.port_id = items->spec;
+			if (!mask.port_id->id)
+				i = 0;
+			else
+				for (i = 0; ptoi[i].ifindex; ++i)
+					if (ptoi[i].port_id == spec.port_id->id)
+						break;
+			assert(ptoi[i].ifindex);
+			tcm->tcm_ifindex = ptoi[i].ifindex;
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			item_flags |= MLX5_FLOW_LAYER_OUTER_L2;
+			mask.eth = flow_tcf_item_mask
+				(items, &rte_flow_item_eth_mask,
+				 &flow_tcf_mask_supported.eth,
+				 &flow_tcf_mask_empty.eth,
+				 sizeof(flow_tcf_mask_supported.eth),
+				 error);
+			assert(mask.eth);
+			if (mask.eth == &flow_tcf_mask_empty.eth)
+				break;
+			spec.eth = items->spec;
+			if (mask.eth->type) {
+				mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
+						 spec.eth->type);
+				eth_type_set = 1;
+			}
+			if (!is_zero_ether_addr(&mask.eth->dst)) {
+				mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST,
+					     ETHER_ADDR_LEN,
+					     spec.eth->dst.addr_bytes);
+				mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_DST_MASK,
+					     ETHER_ADDR_LEN,
+					     mask.eth->dst.addr_bytes);
+			}
+			if (!is_zero_ether_addr(&mask.eth->src)) {
+				mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC,
+					     ETHER_ADDR_LEN,
+					     spec.eth->src.addr_bytes);
+				mnl_attr_put(nlh, TCA_FLOWER_KEY_ETH_SRC_MASK,
+					     ETHER_ADDR_LEN,
+					     mask.eth->src.addr_bytes);
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			item_flags |= MLX5_FLOW_LAYER_OUTER_VLAN;
+			mask.vlan = flow_tcf_item_mask
+				(items, &rte_flow_item_vlan_mask,
+				 &flow_tcf_mask_supported.vlan,
+				 &flow_tcf_mask_empty.vlan,
+				 sizeof(flow_tcf_mask_supported.vlan),
+				 error);
+			assert(mask.vlan);
+			if (!eth_type_set)
+				mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_ETH_TYPE,
+						 RTE_BE16(ETH_P_8021Q));
+			eth_type_set = 1;
+			vlan_present = 1;
+			if (mask.vlan == &flow_tcf_mask_empty.vlan)
+				break;
+			spec.vlan = items->spec;
+			if (mask.vlan->inner_type) {
+				mnl_attr_put_u16(nlh,
+						 TCA_FLOWER_KEY_VLAN_ETH_TYPE,
+						 spec.vlan->inner_type);
+				vlan_eth_type_set = 1;
+			}
+			if (mask.vlan->tci & RTE_BE16(0xe000))
+				mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_VLAN_PRIO,
+						(rte_be_to_cpu_16
+						 (spec.vlan->tci) >> 13) & 0x7);
+			if (mask.vlan->tci & RTE_BE16(0x0fff))
+				mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_VLAN_ID,
+						 rte_be_to_cpu_16
+						 (spec.vlan->tci &
+						  RTE_BE16(0x0fff)));
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+			mask.ipv4 = flow_tcf_item_mask
+				(items, &rte_flow_item_ipv4_mask,
+				 &flow_tcf_mask_supported.ipv4,
+				 &flow_tcf_mask_empty.ipv4,
+				 sizeof(flow_tcf_mask_supported.ipv4),
+				 error);
+			assert(mask.ipv4);
+			if (!eth_type_set || !vlan_eth_type_set)
+				mnl_attr_put_u16(nlh,
+						 vlan_present ?
+						 TCA_FLOWER_KEY_VLAN_ETH_TYPE :
+						 TCA_FLOWER_KEY_ETH_TYPE,
+						 RTE_BE16(ETH_P_IP));
+			eth_type_set = 1;
+			vlan_eth_type_set = 1;
+			if (mask.ipv4 == &flow_tcf_mask_empty.ipv4)
+				break;
+			spec.ipv4 = items->spec;
+			if (mask.ipv4->hdr.next_proto_id) {
+				mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
+						spec.ipv4->hdr.next_proto_id);
+				ip_proto_set = 1;
+			}
+			if (mask.ipv4->hdr.src_addr) {
+				mnl_attr_put_u32(nlh, TCA_FLOWER_KEY_IPV4_SRC,
+						 spec.ipv4->hdr.src_addr);
+				mnl_attr_put_u32(nlh,
+						 TCA_FLOWER_KEY_IPV4_SRC_MASK,
+						 mask.ipv4->hdr.src_addr);
+			}
+			if (mask.ipv4->hdr.dst_addr) {
+				mnl_attr_put_u32(nlh, TCA_FLOWER_KEY_IPV4_DST,
+						 spec.ipv4->hdr.dst_addr);
+				mnl_attr_put_u32(nlh,
+						 TCA_FLOWER_KEY_IPV4_DST_MASK,
+						 mask.ipv4->hdr.dst_addr);
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			item_flags |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+			mask.ipv6 = flow_tcf_item_mask
+				(items, &rte_flow_item_ipv6_mask,
+				 &flow_tcf_mask_supported.ipv6,
+				 &flow_tcf_mask_empty.ipv6,
+				 sizeof(flow_tcf_mask_supported.ipv6),
+				 error);
+			assert(mask.ipv6);
+			if (!eth_type_set || !vlan_eth_type_set)
+				mnl_attr_put_u16(nlh,
+						 vlan_present ?
+						 TCA_FLOWER_KEY_VLAN_ETH_TYPE :
+						 TCA_FLOWER_KEY_ETH_TYPE,
+						 RTE_BE16(ETH_P_IPV6));
+			eth_type_set = 1;
+			vlan_eth_type_set = 1;
+			if (mask.ipv6 == &flow_tcf_mask_empty.ipv6)
+				break;
+			spec.ipv6 = items->spec;
+			if (mask.ipv6->hdr.proto) {
+				mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
+						spec.ipv6->hdr.proto);
+				ip_proto_set = 1;
+			}
+			if (!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.src_addr)) {
+				mnl_attr_put(nlh, TCA_FLOWER_KEY_IPV6_SRC,
+					     sizeof(spec.ipv6->hdr.src_addr),
+					     spec.ipv6->hdr.src_addr);
+				mnl_attr_put(nlh, TCA_FLOWER_KEY_IPV6_SRC_MASK,
+					     sizeof(mask.ipv6->hdr.src_addr),
+					     mask.ipv6->hdr.src_addr);
+			}
+			if (!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.dst_addr)) {
+				mnl_attr_put(nlh, TCA_FLOWER_KEY_IPV6_DST,
+					     sizeof(spec.ipv6->hdr.dst_addr),
+					     spec.ipv6->hdr.dst_addr);
+				mnl_attr_put(nlh, TCA_FLOWER_KEY_IPV6_DST_MASK,
+					     sizeof(mask.ipv6->hdr.dst_addr),
+					     mask.ipv6->hdr.dst_addr);
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			item_flags |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
+			mask.udp = flow_tcf_item_mask
+				(items, &rte_flow_item_udp_mask,
+				 &flow_tcf_mask_supported.udp,
+				 &flow_tcf_mask_empty.udp,
+				 sizeof(flow_tcf_mask_supported.udp),
+				 error);
+			assert(mask.udp);
+			if (!ip_proto_set)
+				mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
+						IPPROTO_UDP);
+			if (mask.udp == &flow_tcf_mask_empty.udp)
+				break;
+			spec.udp = items->spec;
+			if (mask.udp->hdr.src_port) {
+				mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_UDP_SRC,
+						 spec.udp->hdr.src_port);
+				mnl_attr_put_u16(nlh,
+						 TCA_FLOWER_KEY_UDP_SRC_MASK,
+						 mask.udp->hdr.src_port);
+			}
+			if (mask.udp->hdr.dst_port) {
+				mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_UDP_DST,
+						 spec.udp->hdr.dst_port);
+				mnl_attr_put_u16(nlh,
+						 TCA_FLOWER_KEY_UDP_DST_MASK,
+						 mask.udp->hdr.dst_port);
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			item_flags |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
+			mask.tcp = flow_tcf_item_mask
+				(items, &rte_flow_item_tcp_mask,
+				 &flow_tcf_mask_supported.tcp,
+				 &flow_tcf_mask_empty.tcp,
+				 sizeof(flow_tcf_mask_supported.tcp),
+				 error);
+			assert(mask.tcp);
+			if (!ip_proto_set)
+				mnl_attr_put_u8(nlh, TCA_FLOWER_KEY_IP_PROTO,
+						IPPROTO_TCP);
+			if (mask.tcp == &flow_tcf_mask_empty.tcp)
+				break;
+			spec.tcp = items->spec;
+			if (mask.tcp->hdr.src_port) {
+				mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_SRC,
+						 spec.tcp->hdr.src_port);
+				mnl_attr_put_u16(nlh,
+						 TCA_FLOWER_KEY_TCP_SRC_MASK,
+						 mask.tcp->hdr.src_port);
+			}
+			if (mask.tcp->hdr.dst_port) {
+				mnl_attr_put_u16(nlh, TCA_FLOWER_KEY_TCP_DST,
+						 spec.tcp->hdr.dst_port);
+				mnl_attr_put_u16(nlh,
+						 TCA_FLOWER_KEY_TCP_DST_MASK,
+						 mask.tcp->hdr.dst_port);
+			}
+			if (mask.tcp->hdr.tcp_flags) {
+				mnl_attr_put_u16
+					(nlh,
+					 TCA_FLOWER_KEY_TCP_FLAGS,
+					 rte_cpu_to_be_16
+						(spec.tcp->hdr.tcp_flags));
+				mnl_attr_put_u16
+					(nlh,
+					 TCA_FLOWER_KEY_TCP_FLAGS_MASK,
+					 rte_cpu_to_be_16
+						(mask.tcp->hdr.tcp_flags));
+			}
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  NULL, "item not supported");
+		}
+	}
+	na_flower_act = mnl_attr_nest_start(nlh, TCA_FLOWER_ACT);
+	na_act_index_cur = 1;
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		struct nlattr *na_act_index;
+		struct nlattr *na_act;
+		unsigned int vlan_act;
+		unsigned int i;
+
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_PORT_ID:
+			conf.port_id = actions->conf;
+			if (conf.port_id->original)
+				i = 0;
+			else
+				for (i = 0; ptoi[i].ifindex; ++i)
+					if (ptoi[i].port_id == conf.port_id->id)
+						break;
+			assert(ptoi[i].ifindex);
+			na_act_index =
+				mnl_attr_nest_start(nlh, na_act_index_cur++);
+			assert(na_act_index);
+			mnl_attr_put_strz(nlh, TCA_ACT_KIND, "mirred");
+			na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
+			assert(na_act);
+			mnl_attr_put(nlh, TCA_MIRRED_PARMS,
+				     sizeof(struct tc_mirred),
+				     &(struct tc_mirred){
+					.action = TC_ACT_STOLEN,
+					.eaction = TCA_EGRESS_REDIR,
+					.ifindex = ptoi[i].ifindex,
+				     });
+			mnl_attr_nest_end(nlh, na_act);
+			mnl_attr_nest_end(nlh, na_act_index);
+			break;
+		case RTE_FLOW_ACTION_TYPE_JUMP:
+			conf.jump = actions->conf;
+			na_act_index =
+				mnl_attr_nest_start(nlh, na_act_index_cur++);
+			assert(na_act_index);
+			mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
+			na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
+			assert(na_act);
+			mnl_attr_put(nlh, TCA_GACT_PARMS,
+				     sizeof(struct tc_gact),
+				     &(struct tc_gact){
+					.action = TC_ACT_GOTO_CHAIN |
+						  conf.jump->group,
+				     });
+			mnl_attr_nest_end(nlh, na_act);
+			mnl_attr_nest_end(nlh, na_act_index);
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			na_act_index =
+				mnl_attr_nest_start(nlh, na_act_index_cur++);
+			assert(na_act_index);
+			mnl_attr_put_strz(nlh, TCA_ACT_KIND, "gact");
+			na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
+			assert(na_act);
+			mnl_attr_put(nlh, TCA_GACT_PARMS,
+				     sizeof(struct tc_gact),
+				     &(struct tc_gact){
+					.action = TC_ACT_SHOT,
+				     });
+			mnl_attr_nest_end(nlh, na_act);
+			mnl_attr_nest_end(nlh, na_act_index);
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			/*
+			 * Driver adds the count action implicitly for
+			 * each rule it creates.
+			 */
+			ret = flow_tcf_translate_action_count(dev,
+							      dev_flow, error);
+			if (ret < 0)
+				return ret;
+			break;
+		case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
+			conf.of_push_vlan = NULL;
+			vlan_act = TCA_VLAN_ACT_POP;
+			goto action_of_vlan;
+		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
+			conf.of_push_vlan = actions->conf;
+			vlan_act = TCA_VLAN_ACT_PUSH;
+			goto action_of_vlan;
+		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
+			conf.of_set_vlan_vid = actions->conf;
+			if (na_vlan_id)
+				goto override_na_vlan_id;
+			vlan_act = TCA_VLAN_ACT_MODIFY;
+			goto action_of_vlan;
+		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
+			conf.of_set_vlan_pcp = actions->conf;
+			if (na_vlan_priority)
+				goto override_na_vlan_priority;
+			vlan_act = TCA_VLAN_ACT_MODIFY;
+			goto action_of_vlan;
+action_of_vlan:
+			na_act_index =
+				mnl_attr_nest_start(nlh, na_act_index_cur++);
+			assert(na_act_index);
+			mnl_attr_put_strz(nlh, TCA_ACT_KIND, "vlan");
+			na_act = mnl_attr_nest_start(nlh, TCA_ACT_OPTIONS);
+			assert(na_act);
+			mnl_attr_put(nlh, TCA_VLAN_PARMS,
+				     sizeof(struct tc_vlan),
+				     &(struct tc_vlan){
+					.action = TC_ACT_PIPE,
+					.v_action = vlan_act,
+				     });
+			if (vlan_act == TCA_VLAN_ACT_POP) {
+				mnl_attr_nest_end(nlh, na_act);
+				mnl_attr_nest_end(nlh, na_act_index);
+				break;
+			}
+			if (vlan_act == TCA_VLAN_ACT_PUSH)
+				mnl_attr_put_u16(nlh,
+						 TCA_VLAN_PUSH_VLAN_PROTOCOL,
+						 conf.of_push_vlan->ethertype);
+			na_vlan_id = mnl_nlmsg_get_payload_tail(nlh);
+			mnl_attr_put_u16(nlh, TCA_VLAN_PAD, 0);
+			na_vlan_priority = mnl_nlmsg_get_payload_tail(nlh);
+			mnl_attr_put_u8(nlh, TCA_VLAN_PAD, 0);
+			mnl_attr_nest_end(nlh, na_act);
+			mnl_attr_nest_end(nlh, na_act_index);
+			if (actions->type ==
+			    RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
+override_na_vlan_id:
+				na_vlan_id->nla_type = TCA_VLAN_PUSH_VLAN_ID;
+				*(uint16_t *)mnl_attr_get_payload(na_vlan_id) =
+					rte_be_to_cpu_16
+					(conf.of_set_vlan_vid->vlan_vid);
+			} else if (actions->type ==
+				   RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
+override_na_vlan_priority:
+				na_vlan_priority->nla_type =
+					TCA_VLAN_PUSH_VLAN_PRIORITY;
+				*(uint8_t *)mnl_attr_get_payload
+					(na_vlan_priority) =
+					conf.of_set_vlan_pcp->vlan_pcp;
+			}
+			break;
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
+		case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
+		case RTE_FLOW_ACTION_TYPE_SET_TTL:
+		case RTE_FLOW_ACTION_TYPE_DEC_TTL:
+		case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
+		case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
+			na_act_index =
+				mnl_attr_nest_start(nlh, na_act_index_cur++);
+			flow_tcf_create_pedit_mnl_msg(nlh,
+						      &actions, item_flags);
+			mnl_attr_nest_end(nlh, na_act_index);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+	}
+	assert(na_flower);
+	assert(na_flower_act);
+	mnl_attr_nest_end(nlh, na_flower_act);
+	mnl_attr_nest_end(nlh, na_flower);
+	return 0;
+}
+
+/**
+ * Send Netlink message with acknowledgment.
+ *
+ * @param ctx
+ *   Flow context to use.
+ * @param nlh
+ *   Message to send. This function always raises the NLM_F_ACK flag before
+ *   sending.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_tcf_nl_ack(struct mlx5_flow_tcf_context *ctx, struct nlmsghdr *nlh)
+{
+	alignas(struct nlmsghdr)
+	uint8_t ans[mnl_nlmsg_size(sizeof(struct nlmsgerr)) +
+		    nlh->nlmsg_len - sizeof(*nlh)];
+	uint32_t seq = ctx->seq++;
+	struct mnl_socket *nl = ctx->nl;
+	int ret;
+
+	nlh->nlmsg_flags |= NLM_F_ACK;
+	nlh->nlmsg_seq = seq;
+	ret = mnl_socket_sendto(nl, nlh, nlh->nlmsg_len);
+	if (ret != -1)
+		ret = mnl_socket_recvfrom(nl, ans, sizeof(ans));
+	if (ret != -1)
+		ret = mnl_cb_run
+			(ans, ret, seq, mnl_socket_get_portid(nl), NULL, NULL);
+	if (ret > 0)
+		return 0;
+	rte_errno = errno;
+	return -rte_errno;
+}
+
+/**
+ * Apply flow to E-Switch by sending Netlink message.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to the sub flow.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_ernno is set.
+ */
+static int
+flow_tcf_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
+	       struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
+	struct mlx5_flow *dev_flow;
+	struct nlmsghdr *nlh;
+
+	dev_flow = LIST_FIRST(&flow->dev_flows);
+	/* E-Switch flow can't be expanded. */
+	assert(!LIST_NEXT(dev_flow, next));
+	nlh = dev_flow->tcf.nlh;
+	nlh->nlmsg_type = RTM_NEWTFILTER;
+	nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
+	if (!flow_tcf_nl_ack(ctx, nlh))
+		return 0;
+	return rte_flow_error_set(error, rte_errno,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				  "netlink: failed to create TC flow rule");
+}
+
+/**
+ * Remove flow from E-Switch by sending Netlink message.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to the sub flow.
+ */
+static void
+flow_tcf_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
+	struct mlx5_flow *dev_flow;
+	struct nlmsghdr *nlh;
+
+	if (!flow)
+		return;
+	if (flow->counter) {
+		if (--flow->counter->ref_cnt == 0) {
+			rte_free(flow->counter);
+			flow->counter = NULL;
+		}
+	}
+	dev_flow = LIST_FIRST(&flow->dev_flows);
+	if (!dev_flow)
+		return;
+	/* E-Switch flow can't be expanded. */
+	assert(!LIST_NEXT(dev_flow, next));
+	nlh = dev_flow->tcf.nlh;
+	nlh->nlmsg_type = RTM_DELTFILTER;
+	nlh->nlmsg_flags = NLM_F_REQUEST;
+	flow_tcf_nl_ack(ctx, nlh);
+}
+
+/**
+ * Remove flow from E-Switch and release resources of the device flow.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to the sub flow.
+ */
+static void
+flow_tcf_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct mlx5_flow *dev_flow;
+
+	if (!flow)
+		return;
+	flow_tcf_remove(dev, flow);
+	dev_flow = LIST_FIRST(&flow->dev_flows);
+	if (!dev_flow)
+		return;
+	/* E-Switch flow can't be expanded. */
+	assert(!LIST_NEXT(dev_flow, next));
+	LIST_REMOVE(dev_flow, next);
+	rte_free(dev_flow);
+}
+
+/**
+ * Helper routine for figuring the space size required for a parse buffer.
+ *
+ * @param array
+ *   array of values to use.
+ * @param idx
+ *   Current location in array.
+ * @param value
+ *   Value to compare with.
+ *
+ * @return
+ *   The maximum between the given value and the array value on index.
+ */
+static uint16_t
+flow_tcf_arr_val_max(uint16_t array[], int idx, uint16_t value)
+{
+	return idx < 0 ? (value) : RTE_MAX((array)[idx], value);
+}
+
+/**
+ * Parse rtnetlink message attributes filling the attribute table with the info
+ * retrieved.
+ *
+ * @param tb
+ *   Attribute table to be filled.
+ * @param[out] max
+ *   Maxinum entry in the attribute table.
+ * @param rte
+ *   The attributes section in the message to be parsed.
+ * @param len
+ *   The length of the attributes section in the message.
+ */
+static void
+flow_tcf_nl_parse_rtattr(struct rtattr *tb[], int max,
+			 struct rtattr *rta, int len)
+{
+	unsigned short type;
+	memset(tb, 0, sizeof(struct rtattr *) * (max + 1));
+	while (RTA_OK(rta, len)) {
+		type = rta->rta_type;
+		if (type <= max && !tb[type])
+			tb[type] = rta;
+		rta = RTA_NEXT(rta, len);
+	}
+}
+
+/**
+ * Extract flow counters from flower action.
+ *
+ * @param rta
+ *   flower action stats properties in the Netlink message received.
+ * @param rta_type
+ *   The backward sequence of rta_types, as written in the attribute table,
+ *   we need to traverse in order to get to the requested object.
+ * @param idx
+ *   Current location in rta_type table.
+ * @param[out] data
+ *   data holding the count statistics of the rte_flow retrieved from
+ *   the message.
+ *
+ * @return
+ *   0 if data was found and retrieved, -1 otherwise.
+ */
+static int
+flow_tcf_nl_action_stats_parse_and_get(struct rtattr *rta,
+				       uint16_t rta_type[], int idx,
+				       struct gnet_stats_basic *data)
+{
+	int tca_stats_max = flow_tcf_arr_val_max(rta_type, idx,
+						 TCA_STATS_BASIC);
+	struct rtattr *tbs[tca_stats_max + 1];
+
+	if (rta == NULL || idx < 0)
+		return -1;
+	flow_tcf_nl_parse_rtattr(tbs, tca_stats_max,
+				 RTA_DATA(rta), RTA_PAYLOAD(rta));
+	switch (rta_type[idx]) {
+	case TCA_STATS_BASIC:
+		if (tbs[TCA_STATS_BASIC]) {
+			memcpy(data, RTA_DATA(tbs[TCA_STATS_BASIC]),
+			       RTE_MIN(RTA_PAYLOAD(tbs[TCA_STATS_BASIC]),
+			       sizeof(*data)));
+			return 0;
+		}
+		break;
+	default:
+		break;
+	}
+	return -1;
+}
+
+/**
+ * Parse flower single action retrieving the requested action attribute,
+ * if found.
+ *
+ * @param arg
+ *   flower action properties in the Netlink message received.
+ * @param rta_type
+ *   The backward sequence of rta_types, as written in the attribute table,
+ *   we need to traverse in order to get to the requested object.
+ * @param idx
+ *   Current location in rta_type table.
+ * @param[out] data
+ *   Count statistics retrieved from the message query.
+ *
+ * @return
+ *   0 if data was found and retrieved, -1 otherwise.
+ */
+static int
+flow_tcf_nl_parse_one_action_and_get(struct rtattr *arg,
+				     uint16_t rta_type[], int idx, void *data)
+{
+	int tca_act_max = flow_tcf_arr_val_max(rta_type, idx, TCA_ACT_STATS);
+	struct rtattr *tb[tca_act_max + 1];
+
+	if (arg == NULL || idx < 0)
+		return -1;
+	flow_tcf_nl_parse_rtattr(tb, tca_act_max,
+				 RTA_DATA(arg), RTA_PAYLOAD(arg));
+	if (tb[TCA_ACT_KIND] == NULL)
+		return -1;
+	switch (rta_type[idx]) {
+	case TCA_ACT_STATS:
+		if (tb[TCA_ACT_STATS])
+			return flow_tcf_nl_action_stats_parse_and_get
+					(tb[TCA_ACT_STATS],
+					 rta_type, --idx,
+					 (struct gnet_stats_basic *)data);
+		break;
+	default:
+		break;
+	}
+	return -1;
+}
+
+/**
+ * Parse flower action section in the message retrieving the requested
+ * attribute from the first action that provides it.
+ *
+ * @param opt
+ *   flower section in the Netlink message received.
+ * @param rta_type
+ *   The backward sequence of rta_types, as written in the attribute table,
+ *   we need to traverse in order to get to the requested object.
+ * @param idx
+ *   Current location in rta_type table.
+ * @param[out] data
+ *   data retrieved from the message query.
+ *
+ * @return
+ *   0 if data was found and retrieved, -1 otherwise.
+ */
+static int
+flow_tcf_nl_action_parse_and_get(struct rtattr *arg,
+				 uint16_t rta_type[], int idx, void *data)
+{
+	struct rtattr *tb[TCA_ACT_MAX_PRIO + 1];
+	int i;
+
+	if (arg == NULL || idx < 0)
+		return -1;
+	flow_tcf_nl_parse_rtattr(tb, TCA_ACT_MAX_PRIO,
+				 RTA_DATA(arg), RTA_PAYLOAD(arg));
+	switch (rta_type[idx]) {
+	/*
+	 * flow counters are stored in the actions defined by the flow
+	 * and not in the flow itself, therefore we need to traverse the
+	 * flower chain of actions in search for them.
+	 *
+	 * Note that the index is not decremented here.
+	 */
+	case TCA_ACT_STATS:
+		for (i = 0; i <= TCA_ACT_MAX_PRIO; i++) {
+			if (tb[i] &&
+			!flow_tcf_nl_parse_one_action_and_get(tb[i],
+							      rta_type,
+							      idx, data))
+				return 0;
+		}
+		break;
+	default:
+		break;
+	}
+	return -1;
+}
+
+/**
+ * Parse flower classifier options in the message, retrieving the requested
+ * attribute if found.
+ *
+ * @param opt
+ *   flower section in the Netlink message received.
+ * @param rta_type
+ *   The backward sequence of rta_types, as written in the attribute table,
+ *   we need to traverse in order to get to the requested object.
+ * @param idx
+ *   Current location in rta_type table.
+ * @param[out] data
+ *   data retrieved from the message query.
+ *
+ * @return
+ *   0 if data was found and retrieved, -1 otherwise.
+ */
+static int
+flow_tcf_nl_opts_parse_and_get(struct rtattr *opt,
+			       uint16_t rta_type[], int idx, void *data)
+{
+	int tca_flower_max = flow_tcf_arr_val_max(rta_type, idx,
+						  TCA_FLOWER_ACT);
+	struct rtattr *tb[tca_flower_max + 1];
+
+	if (!opt || idx < 0)
+		return -1;
+	flow_tcf_nl_parse_rtattr(tb, tca_flower_max,
+				 RTA_DATA(opt), RTA_PAYLOAD(opt));
+	switch (rta_type[idx]) {
+	case TCA_FLOWER_ACT:
+		if (tb[TCA_FLOWER_ACT])
+			return flow_tcf_nl_action_parse_and_get
+							(tb[TCA_FLOWER_ACT],
+							 rta_type, --idx, data);
+		break;
+	default:
+		break;
+	}
+	return -1;
+}
+
+/**
+ * Parse Netlink reply on filter query, retrieving the flow counters.
+ *
+ * @param nlh
+ *   Message received from Netlink.
+ * @param rta_type
+ *   The backward sequence of rta_types, as written in the attribute table,
+ *   we need to traverse in order to get to the requested object.
+ * @param idx
+ *   Current location in rta_type table.
+ * @param[out] data
+ *   data retrieved from the message query.
+ *
+ * @return
+ *   0 if data was found and retrieved, -1 otherwise.
+ */
+static int
+flow_tcf_nl_filter_parse_and_get(struct nlmsghdr *cnlh,
+				 uint16_t rta_type[], int idx, void *data)
+{
+	struct nlmsghdr *nlh = cnlh;
+	struct tcmsg *t = NLMSG_DATA(nlh);
+	int len = nlh->nlmsg_len;
+	int tca_max = flow_tcf_arr_val_max(rta_type, idx, TCA_OPTIONS);
+	struct rtattr *tb[tca_max + 1];
+
+	if (idx < 0)
+		return -1;
+	if (nlh->nlmsg_type != RTM_NEWTFILTER &&
+	    nlh->nlmsg_type != RTM_GETTFILTER &&
+	    nlh->nlmsg_type != RTM_DELTFILTER)
+		return -1;
+	len -= NLMSG_LENGTH(sizeof(*t));
+	if (len < 0)
+		return -1;
+	flow_tcf_nl_parse_rtattr(tb, tca_max, TCA_RTA(t), len);
+	/* Not a TC flower flow - bail out */
+	if (!tb[TCA_KIND] ||
+	    strcmp(RTA_DATA(tb[TCA_KIND]), "flower"))
+		return -1;
+	switch (rta_type[idx]) {
+	case TCA_OPTIONS:
+		if (tb[TCA_OPTIONS])
+			return flow_tcf_nl_opts_parse_and_get(tb[TCA_OPTIONS],
+							      rta_type,
+							      --idx, data);
+		break;
+	default:
+		break;
+	}
+	return -1;
+}
+
+/**
+ * A callback to parse Netlink reply on TC flower query.
+ *
+ * @param nlh
+ *   Message received from Netlink.
+ * @param[out] data
+ *   Pointer to data area to be filled by the parsing routine.
+ *   assumed to be a pinter to struct flow_tcf_stats_basic.
+ *
+ * @return
+ *   MNL_CB_OK value.
+ */
+static int
+flow_tcf_nl_message_get_stats_basic(const struct nlmsghdr *nlh, void *data)
+{
+	/*
+	 * The backward sequence of rta_types to pass in order to get
+	 *  to the counters.
+	 */
+	uint16_t rta_type[] = { TCA_STATS_BASIC, TCA_ACT_STATS,
+				TCA_FLOWER_ACT, TCA_OPTIONS };
+	struct flow_tcf_stats_basic *sb_data = data;
+	union {
+		const struct nlmsghdr *c;
+		struct nlmsghdr *nc;
+	} tnlh = { .c = nlh };
+
+	if (!flow_tcf_nl_filter_parse_and_get(tnlh.nc, rta_type,
+					      RTE_DIM(rta_type) - 1,
+					      (void *)&sb_data->counters))
+		sb_data->valid = true;
+	return MNL_CB_OK;
+}
+
+/**
+ * Query a TC flower rule for its statistics via netlink.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] flow
+ *   Pointer to the sub flow.
+ * @param[out] data
+ *   data retrieved by the query.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_tcf_query_count(struct rte_eth_dev *dev,
+			  struct rte_flow *flow,
+			  void *data,
+			  struct rte_flow_error *error)
+{
+	struct flow_tcf_stats_basic sb_data = { 0 };
+	struct rte_flow_query_count *qc = data;
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_tcf_context *ctx = priv->tcf_context;
+	struct mnl_socket *nl = ctx->nl;
+	struct mlx5_flow *dev_flow;
+	struct nlmsghdr *nlh;
+	uint32_t seq = priv->tcf_context->seq++;
+	ssize_t ret;
+	assert(qc);
+
+	dev_flow = LIST_FIRST(&flow->dev_flows);
+	/* E-Switch flow can't be expanded. */
+	assert(!LIST_NEXT(dev_flow, next));
+	if (!dev_flow->flow->counter)
+		goto notsup_exit;
+	nlh = dev_flow->tcf.nlh;
+	nlh->nlmsg_type = RTM_GETTFILTER;
+	nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_ECHO;
+	nlh->nlmsg_seq = seq;
+	if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) == -1)
+		goto error_exit;
+	do {
+		ret = mnl_socket_recvfrom(nl, ctx->buf, ctx->buf_size);
+		if (ret <= 0)
+			break;
+		ret = mnl_cb_run(ctx->buf, ret, seq,
+				 mnl_socket_get_portid(nl),
+				 flow_tcf_nl_message_get_stats_basic,
+				 (void *)&sb_data);
+	} while (ret > 0);
+	/* Return the delta from last reset. */
+	if (sb_data.valid) {
+		/* Return the delta from last reset. */
+		qc->hits_set = 1;
+		qc->bytes_set = 1;
+		qc->hits = sb_data.counters.packets - flow->counter->hits;
+		qc->bytes = sb_data.counters.bytes - flow->counter->bytes;
+		if (qc->reset) {
+			flow->counter->hits = sb_data.counters.packets;
+			flow->counter->bytes = sb_data.counters.bytes;
+		}
+		return 0;
+	}
+	return rte_flow_error_set(error, EINVAL,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "flow does not have counter");
+error_exit:
+	return rte_flow_error_set
+			(error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			 NULL, "netlink: failed to read flow rule counters");
+notsup_exit:
+	return rte_flow_error_set
+			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			 NULL, "counters are not available.");
+}
+
+/**
+ * Query a flow.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+static int
+flow_tcf_query(struct rte_eth_dev *dev,
+	       struct rte_flow *flow,
+	       const struct rte_flow_action *actions,
+	       void *data,
+	       struct rte_flow_error *error)
+{
+	int ret = -EINVAL;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = flow_tcf_query_count(dev, flow, data, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+	}
+	return ret;
+}
+
+const struct mlx5_flow_driver_ops mlx5_flow_tcf_drv_ops = {
+	.validate = flow_tcf_validate,
+	.prepare = flow_tcf_prepare,
+	.translate = flow_tcf_translate,
+	.apply = flow_tcf_apply,
+	.remove = flow_tcf_remove,
+	.destroy = flow_tcf_destroy,
+	.query = flow_tcf_query,
+};
+
+/**
+ * Create and configure a libmnl socket for Netlink flow rules.
+ *
+ * @return
+ *   A valid libmnl socket object pointer on success, NULL otherwise and
+ *   rte_errno is set.
+ */
+static struct mnl_socket *
+flow_tcf_mnl_socket_create(void)
+{
+	struct mnl_socket *nl = mnl_socket_open(NETLINK_ROUTE);
+
+	if (nl) {
+		mnl_socket_setsockopt(nl, NETLINK_CAP_ACK, &(int){ 1 },
+				      sizeof(int));
+		if (!mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID))
+			return nl;
+	}
+	rte_errno = errno;
+	if (nl)
+		mnl_socket_close(nl);
+	return NULL;
+}
+
+/**
+ * Destroy a libmnl socket.
+ *
+ * @param nl
+ *   Libmnl socket of the @p NETLINK_ROUTE kind.
+ */
+static void
+flow_tcf_mnl_socket_destroy(struct mnl_socket *nl)
+{
+	if (nl)
+		mnl_socket_close(nl);
+}
+
+/**
+ * Initialize ingress qdisc of a given network interface.
+ *
+ * @param ctx
+ *   Pointer to tc-flower context to use.
+ * @param ifindex
+ *   Index of network interface to initialize.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_flow_tcf_init(struct mlx5_flow_tcf_context *ctx,
+		   unsigned int ifindex, struct rte_flow_error *error)
+{
+	struct nlmsghdr *nlh;
+	struct tcmsg *tcm;
+	alignas(struct nlmsghdr)
+	uint8_t buf[mnl_nlmsg_size(sizeof(*tcm) + 128)];
+
+	/* Destroy existing ingress qdisc and everything attached to it. */
+	nlh = mnl_nlmsg_put_header(buf);
+	nlh->nlmsg_type = RTM_DELQDISC;
+	nlh->nlmsg_flags = NLM_F_REQUEST;
+	tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
+	tcm->tcm_family = AF_UNSPEC;
+	tcm->tcm_ifindex = ifindex;
+	tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
+	tcm->tcm_parent = TC_H_INGRESS;
+	/* Ignore errors when qdisc is already absent. */
+	if (flow_tcf_nl_ack(ctx, nlh) &&
+	    rte_errno != EINVAL && rte_errno != ENOENT)
+		return rte_flow_error_set(error, rte_errno,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "netlink: failed to remove ingress"
+					  " qdisc");
+	/* Create fresh ingress qdisc. */
+	nlh = mnl_nlmsg_put_header(buf);
+	nlh->nlmsg_type = RTM_NEWQDISC;
+	nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
+	tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
+	tcm->tcm_family = AF_UNSPEC;
+	tcm->tcm_ifindex = ifindex;
+	tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
+	tcm->tcm_parent = TC_H_INGRESS;
+	mnl_attr_put_strz_check(nlh, sizeof(buf), TCA_KIND, "ingress");
+	if (flow_tcf_nl_ack(ctx, nlh))
+		return rte_flow_error_set(error, rte_errno,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "netlink: failed to create ingress"
+					  " qdisc");
+	return 0;
+}
+
+/**
+ * Create libmnl context for Netlink flow rules.
+ *
+ * @return
+ *   A valid libmnl socket object pointer on success, NULL otherwise and
+ *   rte_errno is set.
+ */
+struct mlx5_flow_tcf_context *
+mlx5_flow_tcf_context_create(void)
+{
+	struct mlx5_flow_tcf_context *ctx = rte_zmalloc(__func__,
+							sizeof(*ctx),
+							sizeof(uint32_t));
+	if (!ctx)
+		goto error;
+	ctx->nl = flow_tcf_mnl_socket_create();
+	if (!ctx->nl)
+		goto error;
+	ctx->buf_size = MNL_SOCKET_BUFFER_SIZE;
+	ctx->buf = rte_zmalloc(__func__,
+			       ctx->buf_size, sizeof(uint32_t));
+	if (!ctx->buf)
+		goto error;
+	ctx->seq = random();
+	return ctx;
+error:
+	mlx5_flow_tcf_context_destroy(ctx);
+	return NULL;
+}
+
+/**
+ * Destroy a libmnl context.
+ *
+ * @param ctx
+ *   Libmnl socket of the @p NETLINK_ROUTE kind.
+ */
+void
+mlx5_flow_tcf_context_destroy(struct mlx5_flow_tcf_context *ctx)
+{
+	if (!ctx)
+		return;
+	flow_tcf_mnl_socket_destroy(ctx->nl);
+	rte_free(ctx->buf);
+	rte_free(ctx);
+}
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
new file mode 100644
index 00000000..81bc39f9
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -0,0 +1,1825 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <netinet/in.h>
+#include <sys/queue.h>
+#include <stdalign.h>
+#include <stdint.h>
+#include <string.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_common.h>
+#include <rte_ether.h>
+#include <rte_eth_ctrl.h>
+#include <rte_ethdev_driver.h>
+#include <rte_flow.h>
+#include <rte_flow_driver.h>
+#include <rte_malloc.h>
+#include <rte_ip.h>
+
+#include "mlx5.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+#include "mlx5_glue.h"
+#include "mlx5_flow.h"
+
+/**
+ * Create Verbs flow counter with Verbs library.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in, out] counter
+ *   mlx5 flow counter object, contains the counter id,
+ *   handle of created Verbs flow counter is returned
+ *   in cs field (if counters are supported).
+ *
+ * @return
+ *   0 On success else a negative errno value is returned
+ *   and rte_errno is set.
+ */
+static int
+flow_verbs_counter_create(struct rte_eth_dev *dev,
+			  struct mlx5_flow_counter *counter)
+{
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_counter_set_init_attr init = {
+			 .counter_set_id = counter->id};
+
+	counter->cs = mlx5_glue->create_counter_set(priv->ctx, &init);
+	if (!counter->cs) {
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	return 0;
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_counters_init_attr init = {0};
+	struct ibv_counter_attach_attr attach = {0};
+	int ret;
+
+	counter->cs = mlx5_glue->create_counters(priv->ctx, &init);
+	if (!counter->cs) {
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	attach.counter_desc = IBV_COUNTER_PACKETS;
+	attach.index = 0;
+	ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
+	if (!ret) {
+		attach.counter_desc = IBV_COUNTER_BYTES;
+		attach.index = 1;
+		ret = mlx5_glue->attach_counters
+					(counter->cs, &attach, NULL);
+	}
+	if (ret) {
+		claim_zero(mlx5_glue->destroy_counters(counter->cs));
+		counter->cs = NULL;
+		rte_errno = ret;
+		return -ret;
+	}
+	return 0;
+#else
+	(void)dev;
+	(void)counter;
+	rte_errno = ENOTSUP;
+	return -ENOTSUP;
+#endif
+}
+
+/**
+ * Get a flow counter.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] shared
+ *   Indicate if this counter is shared with other flows.
+ * @param[in] id
+ *   Counter identifier.
+ *
+ * @return
+ *   A pointer to the counter, NULL otherwise and rte_errno is set.
+ */
+static struct mlx5_flow_counter *
+flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_counter *cnt;
+	int ret;
+
+	LIST_FOREACH(cnt, &priv->flow_counters, next) {
+		if (!cnt->shared || cnt->shared != shared)
+			continue;
+		if (cnt->id != id)
+			continue;
+		cnt->ref_cnt++;
+		return cnt;
+	}
+	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
+	if (!cnt) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	cnt->id = id;
+	cnt->shared = shared;
+	cnt->ref_cnt = 1;
+	cnt->hits = 0;
+	cnt->bytes = 0;
+	/* Create counter with Verbs. */
+	ret = flow_verbs_counter_create(dev, cnt);
+	if (!ret) {
+		LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
+		return cnt;
+	}
+	/* Some error occurred in Verbs library. */
+	rte_free(cnt);
+	rte_errno = -ret;
+	return NULL;
+}
+
+/**
+ * Release a flow counter.
+ *
+ * @param[in] counter
+ *   Pointer to the counter handler.
+ */
+static void
+flow_verbs_counter_release(struct mlx5_flow_counter *counter)
+{
+	if (--counter->ref_cnt == 0) {
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
+		claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+		claim_zero(mlx5_glue->destroy_counters(counter->cs));
+#endif
+		LIST_REMOVE(counter, next);
+		rte_free(counter);
+	}
+}
+
+/**
+ * Query a flow counter via Verbs library call.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+static int
+flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
+			 struct rte_flow *flow, void *data,
+			 struct rte_flow_error *error)
+{
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
+		struct rte_flow_query_count *qc = data;
+		uint64_t counters[2] = {0, 0};
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
+		struct ibv_query_counter_set_attr query_cs_attr = {
+			.cs = flow->counter->cs,
+			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
+		};
+		struct ibv_counter_set_data query_out = {
+			.out = counters,
+			.outlen = 2 * sizeof(uint64_t),
+		};
+		int err = mlx5_glue->query_counter_set(&query_cs_attr,
+						       &query_out);
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+		int err = mlx5_glue->query_counters
+			       (flow->counter->cs, counters,
+				RTE_DIM(counters),
+				IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
+#endif
+		if (err)
+			return rte_flow_error_set
+				(error, err,
+				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				 NULL,
+				 "cannot read counter");
+		qc->hits_set = 1;
+		qc->bytes_set = 1;
+		qc->hits = counters[0] - flow->counter->hits;
+		qc->bytes = counters[1] - flow->counter->bytes;
+		if (qc->reset) {
+			flow->counter->hits = counters[0];
+			flow->counter->bytes = counters[1];
+		}
+		return 0;
+	}
+	return rte_flow_error_set(error, EINVAL,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "flow does not have counter");
+#else
+	(void)flow;
+	(void)data;
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "counters are not available");
+#endif
+}
+
+/**
+ * Add a verbs item specification into @p flow.
+ *
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] src
+ *   Create specification.
+ * @param[in] size
+ *   Size in bytes of the specification to copy.
+ */
+static void
+flow_verbs_spec_add(struct mlx5_flow *flow, void *src, unsigned int size)
+{
+	struct mlx5_flow_verbs *verbs = &flow->verbs;
+
+	if (verbs->specs) {
+		void *dst;
+
+		dst = (void *)(verbs->specs + verbs->size);
+		memcpy(dst, src, size);
+		++verbs->attr->num_of_specs;
+	}
+	verbs->size += size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested item
+ * into the flow.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in] item_flags
+ *   Bit field with all detected items.
+ * @param[in, out] dev_flow
+ *   Pointer to dev_flow structure.
+ */
+static void
+flow_verbs_translate_item_eth(const struct rte_flow_item *item,
+			      uint64_t *item_flags,
+			      struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_item_eth *spec = item->spec;
+	const struct rte_flow_item_eth *mask = item->mask;
+	const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
+	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+
+	if (!mask)
+		mask = &rte_flow_item_eth_mask;
+	if (spec) {
+		unsigned int i;
+
+		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.val.ether_type = spec->type;
+		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.mask.ether_type = mask->type;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
+			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
+			eth.val.src_mac[i] &= eth.mask.src_mac[i];
+		}
+		eth.val.ether_type &= eth.mask.ether_type;
+		dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow_verbs_spec_add(dev_flow, &eth, size);
+	*item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+				MLX5_FLOW_LAYER_OUTER_L2;
+}
+
+/**
+ * Update the VLAN tag in the Verbs Ethernet specification.
+ * This function assumes that the input is valid and there is space to add
+ * the requested item.
+ *
+ * @param[in, out] attr
+ *   Pointer to Verbs attributes structure.
+ * @param[in] eth
+ *   Verbs structure containing the VLAN information to copy.
+ */
+static void
+flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
+			    struct ibv_flow_spec_eth *eth)
+{
+	unsigned int i;
+	const enum ibv_flow_spec_type search = eth->type;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			struct ibv_flow_spec_eth *e =
+				(struct ibv_flow_spec_eth *)hdr;
+
+			e->val.vlan_tag = eth->val.vlan_tag;
+			e->mask.vlan_tag = eth->mask.vlan_tag;
+			e->val.ether_type = eth->val.ether_type;
+			e->mask.ether_type = eth->mask.ether_type;
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p item into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested item
+ * into the flow.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] item_flags
+ *   Bit mask that holds all detected items.
+ * @param[in, out] dev_flow
+ *   Pointer to dev_flow structure.
+ */
+static void
+flow_verbs_translate_item_vlan(const struct rte_flow_item *item,
+			       uint64_t *item_flags,
+			       struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_item_vlan *spec = item->spec;
+	const struct rte_flow_item_vlan *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+	const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+				      MLX5_FLOW_LAYER_OUTER_L2;
+
+	if (!mask)
+		mask = &rte_flow_item_vlan_mask;
+	if (spec) {
+		eth.val.vlan_tag = spec->tci;
+		eth.mask.vlan_tag = mask->tci;
+		eth.val.vlan_tag &= eth.mask.vlan_tag;
+		eth.val.ether_type = spec->inner_type;
+		eth.mask.ether_type = mask->inner_type;
+		eth.val.ether_type &= eth.mask.ether_type;
+	}
+	if (!(*item_flags & l2m)) {
+		dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
+		flow_verbs_spec_add(dev_flow, &eth, size);
+	} else {
+		flow_verbs_item_vlan_update(dev_flow->verbs.attr, &eth);
+		size = 0; /* Only an update is done in eth specification. */
+	}
+	*item_flags |= tunnel ?
+		       (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_VLAN) :
+		       (MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN);
+}
+
+/**
+ * Convert the @p item into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested item
+ * into the flow.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] item_flags
+ *   Bit mask that marks all detected items.
+ * @param[in, out] dev_flow
+ *   Pointer to sepacific flow structure.
+ */
+static void
+flow_verbs_translate_item_ipv4(const struct rte_flow_item *item,
+			       uint64_t *item_flags,
+			       struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_item_ipv4 *spec = item->spec;
+	const struct rte_flow_item_ipv4 *mask = item->mask;
+	const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
+	struct ibv_flow_spec_ipv4_ext ipv4 = {
+		.type = IBV_FLOW_SPEC_IPV4_EXT |
+			(tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+
+	if (!mask)
+		mask = &rte_flow_item_ipv4_mask;
+	*item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
+				MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+	if (spec) {
+		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = spec->hdr.src_addr,
+			.dst_ip = spec->hdr.dst_addr,
+			.proto = spec->hdr.next_proto_id,
+			.tos = spec->hdr.type_of_service,
+		};
+		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = mask->hdr.src_addr,
+			.dst_ip = mask->hdr.dst_addr,
+			.proto = mask->hdr.next_proto_id,
+			.tos = mask->hdr.type_of_service,
+		};
+		/* Remove unwanted bits from values. */
+		ipv4.val.src_ip &= ipv4.mask.src_ip;
+		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
+		ipv4.val.proto &= ipv4.mask.proto;
+		ipv4.val.tos &= ipv4.mask.tos;
+	}
+	dev_flow->verbs.hash_fields |=
+		mlx5_flow_hashfields_adjust(dev_flow, tunnel,
+					    MLX5_IPV4_LAYER_TYPES,
+					    MLX5_IPV4_IBV_RX_HASH);
+	dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L3;
+	flow_verbs_spec_add(dev_flow, &ipv4, size);
+}
+
+/**
+ * Convert the @p item into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested item
+ * into the flow.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] item_flags
+ *   Bit mask that marks all detected items.
+ * @param[in, out] dev_flow
+ *   Pointer to sepacific flow structure.
+ */
+static void
+flow_verbs_translate_item_ipv6(const struct rte_flow_item *item,
+			       uint64_t *item_flags,
+			       struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_item_ipv6 *spec = item->spec;
+	const struct rte_flow_item_ipv6 *mask = item->mask;
+	const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
+	struct ibv_flow_spec_ipv6 ipv6 = {
+		.type = IBV_FLOW_SPEC_IPV6 | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+
+	if (!mask)
+		mask = &rte_flow_item_ipv6_mask;
+	 *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
+				 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+	if (spec) {
+		unsigned int i;
+		uint32_t vtc_flow_val;
+		uint32_t vtc_flow_mask;
+
+		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
+		       RTE_DIM(ipv6.val.src_ip));
+		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
+		       RTE_DIM(ipv6.val.dst_ip));
+		memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
+		       RTE_DIM(ipv6.mask.src_ip));
+		memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
+		       RTE_DIM(ipv6.mask.dst_ip));
+		vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
+		vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
+		ipv6.val.flow_label =
+			rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
+					 IPV6_HDR_TC_SHIFT;
+		ipv6.val.next_hdr = spec->hdr.proto;
+		ipv6.val.hop_limit = spec->hdr.hop_limits;
+		ipv6.mask.flow_label =
+			rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
+					  IPV6_HDR_TC_SHIFT;
+		ipv6.mask.next_hdr = mask->hdr.proto;
+		ipv6.mask.hop_limit = mask->hdr.hop_limits;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
+			ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
+			ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
+		}
+		ipv6.val.flow_label &= ipv6.mask.flow_label;
+		ipv6.val.traffic_class &= ipv6.mask.traffic_class;
+		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
+		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
+	}
+	dev_flow->verbs.hash_fields |=
+		mlx5_flow_hashfields_adjust(dev_flow, tunnel,
+					    MLX5_IPV6_LAYER_TYPES,
+					    MLX5_IPV6_IBV_RX_HASH);
+	dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L3;
+	flow_verbs_spec_add(dev_flow, &ipv6, size);
+}
+
+/**
+ * Convert the @p item into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested item
+ * into the flow.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] item_flags
+ *   Bit mask that marks all detected items.
+ * @param[in, out] dev_flow
+ *   Pointer to sepacific flow structure.
+ */
+static void
+flow_verbs_translate_item_udp(const struct rte_flow_item *item,
+			      uint64_t *item_flags,
+			      struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_item_udp *spec = item->spec;
+	const struct rte_flow_item_udp *mask = item->mask;
+	const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp udp = {
+		.type = IBV_FLOW_SPEC_UDP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+
+	if (!mask)
+		mask = &rte_flow_item_udp_mask;
+	*item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
+				MLX5_FLOW_LAYER_OUTER_L4_UDP;
+	if (spec) {
+		udp.val.dst_port = spec->hdr.dst_port;
+		udp.val.src_port = spec->hdr.src_port;
+		udp.mask.dst_port = mask->hdr.dst_port;
+		udp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		udp.val.src_port &= udp.mask.src_port;
+		udp.val.dst_port &= udp.mask.dst_port;
+	}
+	dev_flow->verbs.hash_fields |=
+		mlx5_flow_hashfields_adjust(dev_flow, tunnel, ETH_RSS_UDP,
+					    (IBV_RX_HASH_SRC_PORT_UDP |
+					     IBV_RX_HASH_DST_PORT_UDP));
+	dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L4;
+	flow_verbs_spec_add(dev_flow, &udp, size);
+}
+
+/**
+ * Convert the @p item into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested item
+ * into the flow.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] item_flags
+ *   Bit mask that marks all detected items.
+ * @param[in, out] dev_flow
+ *   Pointer to sepacific flow structure.
+ */
+static void
+flow_verbs_translate_item_tcp(const struct rte_flow_item *item,
+			      uint64_t *item_flags,
+			      struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_item_tcp *spec = item->spec;
+	const struct rte_flow_item_tcp *mask = item->mask;
+	const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp tcp = {
+		.type = IBV_FLOW_SPEC_TCP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+
+	if (!mask)
+		mask = &rte_flow_item_tcp_mask;
+	*item_flags |=  tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
+				 MLX5_FLOW_LAYER_OUTER_L4_TCP;
+	if (spec) {
+		tcp.val.dst_port = spec->hdr.dst_port;
+		tcp.val.src_port = spec->hdr.src_port;
+		tcp.mask.dst_port = mask->hdr.dst_port;
+		tcp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		tcp.val.src_port &= tcp.mask.src_port;
+		tcp.val.dst_port &= tcp.mask.dst_port;
+	}
+	dev_flow->verbs.hash_fields |=
+		mlx5_flow_hashfields_adjust(dev_flow, tunnel, ETH_RSS_TCP,
+					    (IBV_RX_HASH_SRC_PORT_TCP |
+					     IBV_RX_HASH_DST_PORT_TCP));
+	dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L4;
+	flow_verbs_spec_add(dev_flow, &tcp, size);
+}
+
+/**
+ * Convert the @p item into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested item
+ * into the flow.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] item_flags
+ *   Bit mask that marks all detected items.
+ * @param[in, out] dev_flow
+ *   Pointer to sepacific flow structure.
+ */
+static void
+flow_verbs_translate_item_vxlan(const struct rte_flow_item *item,
+				uint64_t *item_flags,
+				struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_item_vxlan *spec = item->spec;
+	const struct rte_flow_item_vxlan *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel vxlan = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+	union vni {
+		uint32_t vlan_id;
+		uint8_t vni[4];
+	} id = { .vlan_id = 0, };
+
+	if (!mask)
+		mask = &rte_flow_item_vxlan_mask;
+	if (spec) {
+		memcpy(&id.vni[1], spec->vni, 3);
+		vxlan.val.tunnel_id = id.vlan_id;
+		memcpy(&id.vni[1], mask->vni, 3);
+		vxlan.mask.tunnel_id = id.vlan_id;
+		/* Remove unwanted bits from values. */
+		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+	}
+	flow_verbs_spec_add(dev_flow, &vxlan, size);
+	dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
+	*item_flags |= MLX5_FLOW_LAYER_VXLAN;
+}
+
+/**
+ * Convert the @p item into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested item
+ * into the flow.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] item_flags
+ *   Bit mask that marks all detected items.
+ * @param[in, out] dev_flow
+ *   Pointer to sepacific flow structure.
+ */
+static void
+flow_verbs_translate_item_vxlan_gpe(const struct rte_flow_item *item,
+				    uint64_t *item_flags,
+				    struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
+	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel vxlan_gpe = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+	union vni {
+		uint32_t vlan_id;
+		uint8_t vni[4];
+	} id = { .vlan_id = 0, };
+
+	if (!mask)
+		mask = &rte_flow_item_vxlan_gpe_mask;
+	if (spec) {
+		memcpy(&id.vni[1], spec->vni, 3);
+		vxlan_gpe.val.tunnel_id = id.vlan_id;
+		memcpy(&id.vni[1], mask->vni, 3);
+		vxlan_gpe.mask.tunnel_id = id.vlan_id;
+		/* Remove unwanted bits from values. */
+		vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
+	}
+	flow_verbs_spec_add(dev_flow, &vxlan_gpe, size);
+	dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
+	*item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
+}
+
+/**
+ * Update the protocol in Verbs IPv4/IPv6 spec.
+ *
+ * @param[in, out] attr
+ *   Pointer to Verbs attributes structure.
+ * @param[in] search
+ *   Specification type to search in order to update the IP protocol.
+ * @param[in] protocol
+ *   Protocol value to set if none is present in the specification.
+ */
+static void
+flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
+				       enum ibv_flow_spec_type search,
+				       uint8_t protocol)
+{
+	unsigned int i;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	if (!attr)
+		return;
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			union {
+				struct ibv_flow_spec_ipv4_ext *ipv4;
+				struct ibv_flow_spec_ipv6 *ipv6;
+			} ip;
+
+			switch (search) {
+			case IBV_FLOW_SPEC_IPV4_EXT:
+				ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
+				if (!ip.ipv4->val.proto) {
+					ip.ipv4->val.proto = protocol;
+					ip.ipv4->mask.proto = 0xff;
+				}
+				break;
+			case IBV_FLOW_SPEC_IPV6:
+				ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
+				if (!ip.ipv6->val.next_hdr) {
+					ip.ipv6->val.next_hdr = protocol;
+					ip.ipv6->mask.next_hdr = 0xff;
+				}
+				break;
+			default:
+				break;
+			}
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p item into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested item
+ * into the flow.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] item_flags
+ *   Bit mask that marks all detected items.
+ * @param[in, out] dev_flow
+ *   Pointer to sepacific flow structure.
+ */
+static void
+flow_verbs_translate_item_gre(const struct rte_flow_item *item __rte_unused,
+			      uint64_t *item_flags,
+			      struct mlx5_flow *dev_flow)
+{
+	struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
+#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel tunnel = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+#else
+	const struct rte_flow_item_gre *spec = item->spec;
+	const struct rte_flow_item_gre *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_gre);
+	struct ibv_flow_spec_gre tunnel = {
+		.type = IBV_FLOW_SPEC_GRE,
+		.size = size,
+	};
+
+	if (!mask)
+		mask = &rte_flow_item_gre_mask;
+	if (spec) {
+		tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
+		tunnel.val.protocol = spec->protocol;
+		tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
+		tunnel.mask.protocol = mask->protocol;
+		/* Remove unwanted bits from values. */
+		tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
+		tunnel.val.protocol &= tunnel.mask.protocol;
+		tunnel.val.key &= tunnel.mask.key;
+	}
+#endif
+	if (*item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
+		flow_verbs_item_gre_ip_protocol_update(verbs->attr,
+						       IBV_FLOW_SPEC_IPV4_EXT,
+						       IPPROTO_GRE);
+	else
+		flow_verbs_item_gre_ip_protocol_update(verbs->attr,
+						       IBV_FLOW_SPEC_IPV6,
+						       IPPROTO_GRE);
+	flow_verbs_spec_add(dev_flow, &tunnel, size);
+	verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	*item_flags |= MLX5_FLOW_LAYER_GRE;
+}
+
+/**
+ * Convert the @p action into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested action
+ * into the flow. This function also return the action that was added.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] item_flags
+ *   Bit mask that marks all detected items.
+ * @param[in, out] dev_flow
+ *   Pointer to sepacific flow structure.
+ */
+static void
+flow_verbs_translate_item_mpls(const struct rte_flow_item *item __rte_unused,
+			       uint64_t *action_flags __rte_unused,
+			       struct mlx5_flow *dev_flow __rte_unused)
+{
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	const struct rte_flow_item_mpls *spec = item->spec;
+	const struct rte_flow_item_mpls *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_mpls);
+	struct ibv_flow_spec_mpls mpls = {
+		.type = IBV_FLOW_SPEC_MPLS,
+		.size = size,
+	};
+
+	if (!mask)
+		mask = &rte_flow_item_mpls_mask;
+	if (spec) {
+		memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
+		memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
+		/* Remove unwanted bits from values.  */
+		mpls.val.label &= mpls.mask.label;
+	}
+	flow_verbs_spec_add(dev_flow, &mpls, size);
+	dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
+	*action_flags |= MLX5_FLOW_LAYER_MPLS;
+#endif
+}
+
+/**
+ * Convert the @p action into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested action
+ * into the flow. This function also return the action that was added.
+ *
+ * @param[in, out] action_flags
+ *   Pointer to the detected actions.
+ * @param[in] dev_flow
+ *   Pointer to mlx5_flow.
+ */
+static void
+flow_verbs_translate_action_drop(uint64_t *action_flags,
+				 struct mlx5_flow *dev_flow)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
+	struct ibv_flow_spec_action_drop drop = {
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+			.size = size,
+	};
+
+	flow_verbs_spec_add(dev_flow, &drop, size);
+	*action_flags |= MLX5_FLOW_ACTION_DROP;
+}
+
+/**
+ * Convert the @p action into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested action
+ * into the flow. This function also return the action that was added.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] action_flags
+ *   Pointer to the detected actions.
+ * @param[in] dev_flow
+ *   Pointer to mlx5_flow.
+ */
+static void
+flow_verbs_translate_action_queue(const struct rte_flow_action *action,
+				  uint64_t *action_flags,
+				  struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_action_queue *queue = action->conf;
+	struct rte_flow *flow = dev_flow->flow;
+
+	if (flow->queue)
+		(*flow->queue)[0] = queue->index;
+	flow->rss.queue_num = 1;
+	*action_flags |= MLX5_FLOW_ACTION_QUEUE;
+}
+
+/**
+ * Convert the @p action into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested action
+ * into the flow. This function also return the action that was added.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] action_flags
+ *   Pointer to the detected actions.
+ * @param[in] dev_flow
+ *   Pointer to mlx5_flow.
+ */
+static void
+flow_verbs_translate_action_rss(const struct rte_flow_action *action,
+				uint64_t *action_flags,
+				struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_action_rss *rss = action->conf;
+	struct rte_flow *flow = dev_flow->flow;
+
+	if (flow->queue)
+		memcpy((*flow->queue), rss->queue,
+		       rss->queue_num * sizeof(uint16_t));
+	flow->rss.queue_num = rss->queue_num;
+	memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
+	flow->rss.types = rss->types;
+	flow->rss.level = rss->level;
+	*action_flags |= MLX5_FLOW_ACTION_RSS;
+}
+
+/**
+ * Convert the @p action into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested action
+ * into the flow. This function also return the action that was added.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] action_flags
+ *   Pointer to the detected actions.
+ * @param[in] dev_flow
+ *   Pointer to mlx5_flow.
+ */
+static void
+flow_verbs_translate_action_flag
+			(const struct rte_flow_action *action __rte_unused,
+			 uint64_t *action_flags,
+			 struct mlx5_flow *dev_flow)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
+	};
+	*action_flags |= MLX5_FLOW_ACTION_MARK;
+	flow_verbs_spec_add(dev_flow, &tag, size);
+}
+
+/**
+ * Update verbs specification to modify the flag to mark.
+ *
+ * @param[in, out] verbs
+ *   Pointer to the mlx5_flow_verbs structure.
+ * @param[in] mark_id
+ *   Mark identifier to replace the flag.
+ */
+static void
+flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
+{
+	struct ibv_spec_header *hdr;
+	int i;
+
+	if (!verbs)
+		return;
+	/* Update Verbs specification. */
+	hdr = (struct ibv_spec_header *)verbs->specs;
+	if (!hdr)
+		return;
+	for (i = 0; i != verbs->attr->num_of_specs; ++i) {
+		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
+			struct ibv_flow_spec_action_tag *t =
+				(struct ibv_flow_spec_action_tag *)hdr;
+
+			t->tag_id = mlx5_flow_mark_set(mark_id);
+		}
+		hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p action into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested action
+ * into the flow. This function also return the action that was added.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] action_flags
+ *   Pointer to the detected actions.
+ * @param[in] dev_flow
+ *   Pointer to mlx5_flow.
+ */
+static void
+flow_verbs_translate_action_mark(const struct rte_flow_action *action,
+				 uint64_t *action_flags,
+				 struct mlx5_flow *dev_flow)
+{
+	const struct rte_flow_action_mark *mark = action->conf;
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+	};
+	struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
+
+	if (*action_flags & MLX5_FLOW_ACTION_FLAG) {
+		flow_verbs_mark_update(verbs, mark->id);
+		size = 0;
+	} else {
+		tag.tag_id = mlx5_flow_mark_set(mark->id);
+		flow_verbs_spec_add(dev_flow, &tag, size);
+	}
+	*action_flags |= MLX5_FLOW_ACTION_MARK;
+}
+
+/**
+ * Convert the @p action into a Verbs specification. This function assumes that
+ * the input is valid and that there is space to insert the requested action
+ * into the flow. This function also return the action that was added.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] action_flags
+ *   Pointer to the detected actions.
+ * @param[in] dev_flow
+ *   Pointer to mlx5_flow.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 On success else a negative errno value is returned and rte_errno is set.
+ */
+static int
+flow_verbs_translate_action_count(struct rte_eth_dev *dev,
+				  const struct rte_flow_action *action,
+				  uint64_t *action_flags,
+				  struct mlx5_flow *dev_flow,
+				  struct rte_flow_error *error)
+{
+	const struct rte_flow_action_count *count = action->conf;
+	struct rte_flow *flow = dev_flow->flow;
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
+	struct ibv_flow_spec_counter_action counter = {
+		.type = IBV_FLOW_SPEC_ACTION_COUNT,
+		.size = size,
+	};
+#endif
+
+	if (!flow->counter) {
+		flow->counter = flow_verbs_counter_new(dev, count->shared,
+						       count->id);
+		if (!flow->counter)
+			return rte_flow_error_set(error, rte_errno,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  action,
+						  "cannot get counter"
+						  " context.");
+	}
+	*action_flags |= MLX5_FLOW_ACTION_COUNT;
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
+	counter.counter_set_handle = flow->counter->cs->handle;
+	flow_verbs_spec_add(dev_flow, &counter, size);
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	counter.counters = flow->counter->cs;
+	flow_verbs_spec_add(dev_flow, &counter, size);
+#endif
+	return 0;
+}
+
+/**
+ * Internal validation function. For validating both actions and items.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] attr
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_verbs_validate(struct rte_eth_dev *dev,
+		    const struct rte_flow_attr *attr,
+		    const struct rte_flow_item items[],
+		    const struct rte_flow_action actions[],
+		    struct rte_flow_error *error)
+{
+	int ret;
+	uint64_t action_flags = 0;
+	uint64_t item_flags = 0;
+	int tunnel = 0;
+	uint8_t next_protocol = 0xff;
+
+	if (items == NULL)
+		return -1;
+	ret = mlx5_flow_validate_attributes(dev, attr, error);
+	if (ret < 0)
+		return ret;
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+		int ret = 0;
+
+		tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
+		switch (items->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			ret = mlx5_flow_validate_item_eth(items, item_flags,
+							  error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+					       MLX5_FLOW_LAYER_OUTER_L2;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			ret = mlx5_flow_validate_item_vlan(items, item_flags,
+							   error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
+					       MLX5_FLOW_LAYER_OUTER_VLAN;
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			ret = mlx5_flow_validate_item_ipv4(items, item_flags,
+							   error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
+					       MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+			if (items->mask != NULL &&
+			    ((const struct rte_flow_item_ipv4 *)
+			     items->mask)->hdr.next_proto_id)
+				next_protocol =
+					((const struct rte_flow_item_ipv4 *)
+					 (items->spec))->hdr.next_proto_id;
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			ret = mlx5_flow_validate_item_ipv6(items, item_flags,
+							   error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
+					       MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+			if (items->mask != NULL &&
+			    ((const struct rte_flow_item_ipv6 *)
+			     items->mask)->hdr.proto)
+				next_protocol =
+					((const struct rte_flow_item_ipv6 *)
+					 items->spec)->hdr.proto;
+			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			ret = mlx5_flow_validate_item_udp(items, item_flags,
+							  next_protocol,
+							  error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
+					       MLX5_FLOW_LAYER_OUTER_L4_UDP;
+			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			ret = mlx5_flow_validate_item_tcp
+						(items, item_flags,
+						 next_protocol,
+						 &rte_flow_item_tcp_mask,
+						 error);
+			if (ret < 0)
+				return ret;
+			item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
+					       MLX5_FLOW_LAYER_OUTER_L4_TCP;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN:
+			ret = mlx5_flow_validate_item_vxlan(items, item_flags,
+							    error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_VXLAN;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+			ret = mlx5_flow_validate_item_vxlan_gpe(items,
+								item_flags,
+								dev, error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
+			break;
+		case RTE_FLOW_ITEM_TYPE_GRE:
+			ret = mlx5_flow_validate_item_gre(items, item_flags,
+							  next_protocol, error);
+			if (ret < 0)
+				return ret;
+			item_flags |= MLX5_FLOW_LAYER_GRE;
+			break;
+		case RTE_FLOW_ITEM_TYPE_MPLS:
+			ret = mlx5_flow_validate_item_mpls(items, item_flags,
+							   next_protocol,
+							   error);
+			if (ret < 0)
+				return ret;
+			if (next_protocol != 0xff &&
+			    next_protocol != IPPROTO_MPLS)
+				return rte_flow_error_set
+					(error, EINVAL,
+					 RTE_FLOW_ERROR_TYPE_ITEM, items,
+					 "protocol filtering not compatible"
+					 " with MPLS layer");
+			item_flags |= MLX5_FLOW_LAYER_MPLS;
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  NULL, "item not supported");
+		}
+	}
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_FLAG:
+			ret = mlx5_flow_validate_action_flag(action_flags,
+							     attr,
+							     error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_FLAG;
+			break;
+		case RTE_FLOW_ACTION_TYPE_MARK:
+			ret = mlx5_flow_validate_action_mark(actions,
+							     action_flags,
+							     attr,
+							     error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_MARK;
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			ret = mlx5_flow_validate_action_drop(action_flags,
+							     attr,
+							     error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_DROP;
+			break;
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			ret = mlx5_flow_validate_action_queue(actions,
+							      action_flags, dev,
+							      attr,
+							      error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_QUEUE;
+			break;
+		case RTE_FLOW_ACTION_TYPE_RSS:
+			ret = mlx5_flow_validate_action_rss(actions,
+							    action_flags, dev,
+							    attr,
+							    error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_RSS;
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_validate_action_count(dev, attr, error);
+			if (ret < 0)
+				return ret;
+			action_flags |= MLX5_FLOW_ACTION_COUNT;
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+	}
+	if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, actions,
+					  "no fate action is found");
+	return 0;
+}
+
+/**
+ * Calculate the required bytes that are needed for the action part of the verbs
+ * flow, in addtion returns bit-fields with all the detected action, in order to
+ * avoid another interation over the actions.
+ *
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ * @param[out] action_flags
+ *   Pointer to the detected actions.
+ *
+ * @return
+ *   The size of the memory needed for all actions.
+ */
+static int
+flow_verbs_get_actions_and_size(const struct rte_flow_action actions[],
+				uint64_t *action_flags)
+{
+	int size = 0;
+	uint64_t detected_actions = 0;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_FLAG:
+			size += sizeof(struct ibv_flow_spec_action_tag);
+			detected_actions |= MLX5_FLOW_ACTION_FLAG;
+			break;
+		case RTE_FLOW_ACTION_TYPE_MARK:
+			size += sizeof(struct ibv_flow_spec_action_tag);
+			detected_actions |= MLX5_FLOW_ACTION_MARK;
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			size += sizeof(struct ibv_flow_spec_action_drop);
+			detected_actions |= MLX5_FLOW_ACTION_DROP;
+			break;
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			detected_actions |= MLX5_FLOW_ACTION_QUEUE;
+			break;
+		case RTE_FLOW_ACTION_TYPE_RSS:
+			detected_actions |= MLX5_FLOW_ACTION_RSS;
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+			size += sizeof(struct ibv_flow_spec_counter_action);
+#endif
+			detected_actions |= MLX5_FLOW_ACTION_COUNT;
+			break;
+		default:
+			break;
+		}
+	}
+	*action_flags = detected_actions;
+	return size;
+}
+
+/**
+ * Calculate the required bytes that are needed for the item part of the verbs
+ * flow, in addtion returns bit-fields with all the detected action, in order to
+ * avoid another interation over the actions.
+ *
+ * @param[in] actions
+ *   Pointer to the list of items.
+ * @param[in, out] item_flags
+ *   Pointer to the detected items.
+ *
+ * @return
+ *   The size of the memory needed for all items.
+ */
+static int
+flow_verbs_get_items_and_size(const struct rte_flow_item items[],
+			      uint64_t *item_flags)
+{
+	int size = 0;
+	uint64_t detected_items = 0;
+
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+		int tunnel = !!(detected_items & MLX5_FLOW_LAYER_TUNNEL);
+
+		switch (items->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			size += sizeof(struct ibv_flow_spec_eth);
+			detected_items |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+						   MLX5_FLOW_LAYER_OUTER_L2;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			size += sizeof(struct ibv_flow_spec_eth);
+			detected_items |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
+						   MLX5_FLOW_LAYER_OUTER_VLAN;
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			size += sizeof(struct ibv_flow_spec_ipv4_ext);
+			detected_items |= tunnel ?
+					  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
+					  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			size += sizeof(struct ibv_flow_spec_ipv6);
+			detected_items |= tunnel ?
+					  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
+					  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			size += sizeof(struct ibv_flow_spec_tcp_udp);
+			detected_items |= tunnel ?
+					  MLX5_FLOW_LAYER_INNER_L4_UDP :
+					  MLX5_FLOW_LAYER_OUTER_L4_UDP;
+			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			size += sizeof(struct ibv_flow_spec_tcp_udp);
+			detected_items |= tunnel ?
+					  MLX5_FLOW_LAYER_INNER_L4_TCP :
+					  MLX5_FLOW_LAYER_OUTER_L4_TCP;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN:
+			size += sizeof(struct ibv_flow_spec_tunnel);
+			detected_items |= MLX5_FLOW_LAYER_VXLAN;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+			size += sizeof(struct ibv_flow_spec_tunnel);
+			detected_items |= MLX5_FLOW_LAYER_VXLAN_GPE;
+			break;
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+		case RTE_FLOW_ITEM_TYPE_GRE:
+			size += sizeof(struct ibv_flow_spec_gre);
+			detected_items |= MLX5_FLOW_LAYER_GRE;
+			break;
+		case RTE_FLOW_ITEM_TYPE_MPLS:
+			size += sizeof(struct ibv_flow_spec_mpls);
+			detected_items |= MLX5_FLOW_LAYER_MPLS;
+			break;
+#else
+		case RTE_FLOW_ITEM_TYPE_GRE:
+			size += sizeof(struct ibv_flow_spec_tunnel);
+			detected_items |= MLX5_FLOW_LAYER_TUNNEL;
+			break;
+#endif
+		default:
+			break;
+		}
+	}
+	*item_flags = detected_items;
+	return size;
+}
+
+/**
+ * Internal preparation function. Allocate mlx5_flow with the required size.
+ * The required size is calculate based on the actions and items. This function
+ * also returns the detected actions and items for later use.
+ *
+ * @param[in] attr
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ * @param[out] item_flags
+ *   Pointer to bit mask of all items detected.
+ * @param[out] action_flags
+ *   Pointer to bit mask of all actions detected.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
+ *   is set.
+ */
+static struct mlx5_flow *
+flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused,
+		   const struct rte_flow_item items[],
+		   const struct rte_flow_action actions[],
+		   uint64_t *item_flags,
+		   uint64_t *action_flags,
+		   struct rte_flow_error *error)
+{
+	uint32_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr);
+	struct mlx5_flow *flow;
+
+	size += flow_verbs_get_actions_and_size(actions, action_flags);
+	size += flow_verbs_get_items_and_size(items, item_flags);
+	flow = rte_calloc(__func__, 1, size, 0);
+	if (!flow) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "not enough memory to create flow");
+		return NULL;
+	}
+	flow->verbs.attr = (void *)(flow + 1);
+	flow->verbs.specs =
+		(uint8_t *)(flow + 1) + sizeof(struct ibv_flow_attr);
+	return flow;
+}
+
+/**
+ * Fill the flow with verb spec.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] dev_flow
+ *   Pointer to the mlx5 flow.
+ * @param[in] attr
+ *   Pointer to the flow attributes.
+ * @param[in] items
+ *   Pointer to the list of items.
+ * @param[in] actions
+ *   Pointer to the list of actions.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   0 on success, else a negative errno value otherwise and rte_ernno is set.
+ */
+static int
+flow_verbs_translate(struct rte_eth_dev *dev,
+		     struct mlx5_flow *dev_flow,
+		     const struct rte_flow_attr *attr,
+		     const struct rte_flow_item items[],
+		     const struct rte_flow_action actions[],
+		     struct rte_flow_error *error)
+{
+	uint64_t action_flags = 0;
+	uint64_t item_flags = 0;
+	uint64_t priority = attr->priority;
+	struct priv *priv = dev->data->dev_private;
+
+	if (priority == MLX5_FLOW_PRIO_RSVD)
+		priority = priv->config.flow_prio - 1;
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		int ret;
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_FLAG:
+			flow_verbs_translate_action_flag(actions,
+							 &action_flags,
+							 dev_flow);
+			break;
+		case RTE_FLOW_ACTION_TYPE_MARK:
+			flow_verbs_translate_action_mark(actions,
+							 &action_flags,
+							 dev_flow);
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			flow_verbs_translate_action_drop(&action_flags,
+							 dev_flow);
+			break;
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			flow_verbs_translate_action_queue(actions,
+							  &action_flags,
+							  dev_flow);
+			break;
+		case RTE_FLOW_ACTION_TYPE_RSS:
+			flow_verbs_translate_action_rss(actions,
+							&action_flags,
+							dev_flow);
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = flow_verbs_translate_action_count(dev,
+								actions,
+								&action_flags,
+								dev_flow,
+								error);
+			if (ret < 0)
+				return ret;
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+	}
+	/* Device flow should have action flags by flow_drv_prepare(). */
+	assert(dev_flow->flow->actions == action_flags);
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+		switch (items->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			flow_verbs_translate_item_eth(items, &item_flags,
+						      dev_flow);
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			flow_verbs_translate_item_vlan(items, &item_flags,
+						       dev_flow);
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			flow_verbs_translate_item_ipv4(items, &item_flags,
+						       dev_flow);
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			flow_verbs_translate_item_ipv6(items, &item_flags,
+						       dev_flow);
+			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			flow_verbs_translate_item_udp(items, &item_flags,
+						      dev_flow);
+			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			flow_verbs_translate_item_tcp(items, &item_flags,
+						      dev_flow);
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN:
+			flow_verbs_translate_item_vxlan(items, &item_flags,
+							dev_flow);
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+			flow_verbs_translate_item_vxlan_gpe(items, &item_flags,
+							    dev_flow);
+			break;
+		case RTE_FLOW_ITEM_TYPE_GRE:
+			flow_verbs_translate_item_gre(items, &item_flags,
+						      dev_flow);
+			break;
+		case RTE_FLOW_ITEM_TYPE_MPLS:
+			flow_verbs_translate_item_mpls(items, &item_flags,
+						       dev_flow);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  NULL,
+						  "item not supported");
+		}
+	}
+	dev_flow->verbs.attr->priority =
+		mlx5_flow_adjust_priority(dev, priority,
+					  dev_flow->verbs.attr->priority);
+	return 0;
+}
+
+/**
+ * Remove the flow from the NIC but keeps it in memory.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ */
+static void
+flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct mlx5_flow_verbs *verbs;
+	struct mlx5_flow *dev_flow;
+
+	if (!flow)
+		return;
+	LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
+		verbs = &dev_flow->verbs;
+		if (verbs->flow) {
+			claim_zero(mlx5_glue->destroy_flow(verbs->flow));
+			verbs->flow = NULL;
+		}
+		if (verbs->hrxq) {
+			if (flow->actions & MLX5_FLOW_ACTION_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
+		}
+	}
+	if (flow->counter) {
+		flow_verbs_counter_release(flow->counter);
+		flow->counter = NULL;
+	}
+}
+
+/**
+ * Remove the flow from the NIC and the memory.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ */
+static void
+flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct mlx5_flow *dev_flow;
+
+	if (!flow)
+		return;
+	flow_verbs_remove(dev, flow);
+	while (!LIST_EMPTY(&flow->dev_flows)) {
+		dev_flow = LIST_FIRST(&flow->dev_flows);
+		LIST_REMOVE(dev_flow, next);
+		rte_free(dev_flow);
+	}
+}
+
+/**
+ * Apply the flow to the NIC.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
+		 struct rte_flow_error *error)
+{
+	struct mlx5_flow_verbs *verbs;
+	struct mlx5_flow *dev_flow;
+	int err;
+
+	LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
+		verbs = &dev_flow->verbs;
+		if (flow->actions & MLX5_FLOW_ACTION_DROP) {
+			verbs->hrxq = mlx5_hrxq_drop_new(dev);
+			if (!verbs->hrxq) {
+				rte_flow_error_set
+					(error, errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					 "cannot get drop hash queue");
+				goto error;
+			}
+		} else {
+			struct mlx5_hrxq *hrxq;
+
+			hrxq = mlx5_hrxq_get(dev, flow->key,
+					     MLX5_RSS_HASH_KEY_LEN,
+					     verbs->hash_fields,
+					     (*flow->queue),
+					     flow->rss.queue_num);
+			if (!hrxq)
+				hrxq = mlx5_hrxq_new(dev, flow->key,
+						     MLX5_RSS_HASH_KEY_LEN,
+						     verbs->hash_fields,
+						     (*flow->queue),
+						     flow->rss.queue_num,
+						     !!(dev_flow->layers &
+						      MLX5_FLOW_LAYER_TUNNEL));
+			if (!hrxq) {
+				rte_flow_error_set
+					(error, rte_errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					 "cannot get hash queue");
+				goto error;
+			}
+			verbs->hrxq = hrxq;
+		}
+		verbs->flow = mlx5_glue->create_flow(verbs->hrxq->qp,
+						     verbs->attr);
+		if (!verbs->flow) {
+			rte_flow_error_set(error, errno,
+					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					   NULL,
+					   "hardware refuses to create flow");
+			goto error;
+		}
+	}
+	return 0;
+error:
+	err = rte_errno; /* Save rte_errno before cleanup. */
+	LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
+		verbs = &dev_flow->verbs;
+		if (verbs->hrxq) {
+			if (flow->actions & MLX5_FLOW_ACTION_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
+		}
+	}
+	rte_errno = err; /* Restore rte_errno. */
+	return -rte_errno;
+}
+
+/**
+ * Query a flow.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+static int
+flow_verbs_query(struct rte_eth_dev *dev,
+		 struct rte_flow *flow,
+		 const struct rte_flow_action *actions,
+		 void *data,
+		 struct rte_flow_error *error)
+{
+	int ret = -EINVAL;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = flow_verbs_counter_query(dev, flow, data, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+	}
+	return ret;
+}
+
+const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
+	.validate = flow_verbs_validate,
+	.prepare = flow_verbs_prepare,
+	.translate = flow_verbs_translate,
+	.apply = flow_verbs_apply,
+	.remove = flow_verbs_remove,
+	.destroy = flow_verbs_destroy,
+	.query = flow_verbs_query,
+};
diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
index 84f9492a..1afb114f 100644
--- a/drivers/net/mlx5/mlx5_glue.c
+++ b/drivers/net/mlx5/mlx5_glue.c
@@ -215,7 +215,7 @@ static struct ibv_counter_set *
 mlx5_glue_create_counter_set(struct ibv_context *context,
 			     struct ibv_counter_set_init_attr *init_attr)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)context;
 	(void)init_attr;
 	return NULL;
@@ -227,7 +227,7 @@ mlx5_glue_create_counter_set(struct ibv_context *context,
 static int
 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)cs;
 	return ENOTSUP;
 #else
@@ -240,7 +240,7 @@ mlx5_glue_describe_counter_set(struct ibv_context *context,
 			       uint16_t counter_set_id,
 			       struct ibv_counter_set_description *cs_desc)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)context;
 	(void)counter_set_id;
 	(void)cs_desc;
@@ -254,7 +254,7 @@ static int
 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
 			    struct ibv_counter_set_data *cs_data)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)query_attr;
 	(void)cs_data;
 	return ENOTSUP;
@@ -263,6 +263,62 @@ mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
 #endif
 }
 
+static struct ibv_counters *
+mlx5_glue_create_counters(struct ibv_context *context,
+			  struct ibv_counters_init_attr *init_attr)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)context;
+	(void)init_attr;
+	return NULL;
+#else
+	return ibv_create_counters(context, init_attr);
+#endif
+}
+
+static int
+mlx5_glue_destroy_counters(struct ibv_counters *counters)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)counters;
+	return ENOTSUP;
+#else
+	return ibv_destroy_counters(counters);
+#endif
+}
+
+static int
+mlx5_glue_attach_counters(struct ibv_counters *counters,
+			  struct ibv_counter_attach_attr *attr,
+			  struct ibv_flow *flow)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)counters;
+	(void)attr;
+	(void)flow;
+	return ENOTSUP;
+#else
+	return ibv_attach_counters_point_flow(counters, attr, flow);
+#endif
+}
+
+static int
+mlx5_glue_query_counters(struct ibv_counters *counters,
+			 uint64_t *counters_value,
+			 uint32_t ncounters,
+			 uint32_t flags)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)counters;
+	(void)counters_value;
+	(void)ncounters;
+	(void)flags;
+	return ENOTSUP;
+#else
+	return ibv_read_counters(counters, counters_value, ncounters, flags);
+#endif
+}
+
 static void
 mlx5_glue_ack_async_event(struct ibv_async_event *event)
 {
@@ -346,6 +402,48 @@ mlx5_glue_dv_create_qp(struct ibv_context *context,
 #endif
 }
 
+static struct mlx5dv_flow_matcher *
+mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
+				 struct mlx5dv_flow_matcher_attr *matcher_attr)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	return mlx5dv_create_flow_matcher(context, matcher_attr);
+#else
+	(void)context;
+	(void)matcher_attr;
+	return NULL;
+#endif
+}
+
+static struct ibv_flow *
+mlx5_glue_dv_create_flow(struct mlx5dv_flow_matcher *matcher,
+			 struct mlx5dv_flow_match_parameters *match_value,
+			 size_t num_actions,
+			 struct mlx5dv_flow_action_attr *actions_attr)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	return mlx5dv_create_flow(matcher, match_value,
+				  num_actions, actions_attr);
+#else
+	(void)matcher;
+	(void)match_value;
+	(void)num_actions;
+	(void)actions_attr;
+	return NULL;
+#endif
+}
+
+static int
+mlx5_glue_dv_destroy_flow_matcher(struct mlx5dv_flow_matcher *matcher)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	return mlx5dv_destroy_flow_matcher(matcher);
+#else
+	(void)matcher;
+	return 0;
+#endif
+}
+
 alignas(RTE_CACHE_LINE_SIZE)
 const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
 	.version = MLX5_GLUE_VERSION,
@@ -382,6 +480,10 @@ const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
 	.destroy_counter_set = mlx5_glue_destroy_counter_set,
 	.describe_counter_set = mlx5_glue_describe_counter_set,
 	.query_counter_set = mlx5_glue_query_counter_set,
+	.create_counters = mlx5_glue_create_counters,
+	.destroy_counters = mlx5_glue_destroy_counters,
+	.attach_counters = mlx5_glue_attach_counters,
+	.query_counters = mlx5_glue_query_counters,
 	.ack_async_event = mlx5_glue_ack_async_event,
 	.get_async_event = mlx5_glue_get_async_event,
 	.port_state_str = mlx5_glue_port_state_str,
@@ -392,4 +494,7 @@ const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
 	.dv_set_context_attr = mlx5_glue_dv_set_context_attr,
 	.dv_init_obj = mlx5_glue_dv_init_obj,
 	.dv_create_qp = mlx5_glue_dv_create_qp,
+	.dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher,
+	.dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher,
+	.dv_create_flow = mlx5_glue_dv_create_flow,
 };
diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
index e584d367..44bfefed 100644
--- a/drivers/net/mlx5/mlx5_glue.h
+++ b/drivers/net/mlx5/mlx5_glue.h
@@ -23,7 +23,7 @@
 #define MLX5_GLUE_VERSION ""
 #endif
 
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 struct ibv_counter_set;
 struct ibv_counter_set_data;
 struct ibv_counter_set_description;
@@ -31,6 +31,12 @@ struct ibv_counter_set_init_attr;
 struct ibv_query_counter_set_attr;
 #endif
 
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+struct ibv_counters;
+struct ibv_counters_init_attr;
+struct ibv_counter_attach_attr;
+#endif
+
 #ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 struct mlx5dv_qp_init_attr;
 #endif
@@ -39,6 +45,13 @@ struct mlx5dv_qp_init_attr;
 struct mlx5dv_wq_init_attr;
 #endif
 
+#ifndef HAVE_IBV_FLOW_DV_SUPPORT
+struct mlx5dv_flow_matcher;
+struct mlx5dv_flow_matcher_attr;
+struct mlx5dv_flow_action_attr;
+struct mlx5dv_flow_match_parameters;
+#endif
+
 /* LIB_GLUE_VERSION must be updated every time this structure is modified. */
 struct mlx5_glue {
 	const char *version;
@@ -99,6 +112,17 @@ struct mlx5_glue {
 		 struct ibv_counter_set_description *cs_desc);
 	int (*query_counter_set)(struct ibv_query_counter_set_attr *query_attr,
 				 struct ibv_counter_set_data *cs_data);
+	struct ibv_counters *(*create_counters)
+		(struct ibv_context *context,
+		 struct ibv_counters_init_attr *init_attr);
+	int (*destroy_counters)(struct ibv_counters *counters);
+	int (*attach_counters)(struct ibv_counters *counters,
+			       struct ibv_counter_attach_attr *attr,
+			       struct ibv_flow *flow);
+	int (*query_counters)(struct ibv_counters *counters,
+			      uint64_t *counters_value,
+			      uint32_t ncounters,
+			      uint32_t flags);
 	void (*ack_async_event)(struct ibv_async_event *event);
 	int (*get_async_event)(struct ibv_context *context,
 			       struct ibv_async_event *event);
@@ -122,6 +146,14 @@ struct mlx5_glue {
 		(struct ibv_context *context,
 		 struct ibv_qp_init_attr_ex *qp_init_attr_ex,
 		 struct mlx5dv_qp_init_attr *dv_qp_init_attr);
+	struct mlx5dv_flow_matcher *(*dv_create_flow_matcher)
+		(struct ibv_context *context,
+		 struct mlx5dv_flow_matcher_attr *matcher_attr);
+	int (*dv_destroy_flow_matcher)(struct mlx5dv_flow_matcher *matcher);
+	struct ibv_flow *(*dv_create_flow)(struct mlx5dv_flow_matcher *matcher,
+			  struct mlx5dv_flow_match_parameters *match_value,
+			  size_t num_actions,
+			  struct mlx5dv_flow_action_attr *actions_attr);
 };
 
 const struct mlx5_glue *mlx5_glue;
diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c
index 12ee37f5..672a4761 100644
--- a/drivers/net/mlx5/mlx5_mac.c
+++ b/drivers/net/mlx5/mlx5_mac.c
@@ -49,7 +49,7 @@ mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN])
 	struct ifreq request;
 	int ret;
 
-	ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request, 0);
+	ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request);
 	if (ret)
 		return ret;
 	memcpy(mac, request.ifr_hwaddr.sa_data, ETHER_ADDR_LEN);
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 1d1bcb5f..f4b15d3f 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -277,6 +277,23 @@ mr_find_next_chunk(struct mlx5_mr *mr, struct mlx5_mr_cache *entry,
 	uintptr_t end = 0;
 	uint32_t idx = 0;
 
+	/* MR for external memory doesn't have memseg list. */
+	if (mr->msl == NULL) {
+		struct ibv_mr *ibv_mr = mr->ibv_mr;
+
+		assert(mr->ms_bmp_n == 1);
+		assert(mr->ms_n == 1);
+		assert(base_idx == 0);
+		/*
+		 * Can't search it from memseg list but get it directly from
+		 * verbs MR as there's only one chunk.
+		 */
+		entry->start = (uintptr_t)ibv_mr->addr;
+		entry->end = (uintptr_t)ibv_mr->addr + mr->ibv_mr->length;
+		entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
+		/* Returning 1 ends iteration. */
+		return 1;
+	}
 	for (idx = base_idx; idx < mr->ms_bmp_n; ++idx) {
 		if (rte_bitmap_get(mr->ms_bmp, idx)) {
 			const struct rte_memseg_list *msl;
@@ -811,6 +828,7 @@ mlx5_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
 		mr = mr_lookup_dev_list(dev, &entry, start);
 		if (mr == NULL)
 			continue;
+		assert(mr->msl); /* Can't be external memory. */
 		ms = rte_mem_virt2memseg((void *)start, msl);
 		assert(ms != NULL);
 		assert(msl->page_sz == ms->hugepage_sz);
@@ -1061,6 +1079,139 @@ mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)
 		(void *)mr_ctrl, mr_ctrl->cur_gen);
 }
 
+/**
+ * Called during rte_mempool_mem_iter() by mlx5_mr_update_ext_mp().
+ *
+ * Externally allocated chunk is registered and a MR is created for the chunk.
+ * The MR object is added to the global list. If memseg list of a MR object
+ * (mr->msl) is null, the MR object can be regarded as externally allocated
+ * memory.
+ *
+ * Once external memory is registered, it should be static. If the memory is
+ * freed and the virtual address range has different physical memory mapped
+ * again, it may cause crash on device due to the wrong translation entry. PMD
+ * can't track the free event of the external memory for now.
+ */
+static void
+mlx5_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque,
+			 struct rte_mempool_memhdr *memhdr,
+			 unsigned mem_idx __rte_unused)
+{
+	struct mr_update_mp_data *data = opaque;
+	struct rte_eth_dev *dev = data->dev;
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_mr_ctrl *mr_ctrl = data->mr_ctrl;
+	struct mlx5_mr *mr = NULL;
+	uintptr_t addr = (uintptr_t)memhdr->addr;
+	size_t len = memhdr->len;
+	struct mlx5_mr_cache entry;
+	uint32_t lkey;
+
+	/* If already registered, it should return. */
+	rte_rwlock_read_lock(&priv->mr.rwlock);
+	lkey = mr_lookup_dev(dev, &entry, addr);
+	rte_rwlock_read_unlock(&priv->mr.rwlock);
+	if (lkey != UINT32_MAX)
+		return;
+	mr = rte_zmalloc_socket(NULL,
+				RTE_ALIGN_CEIL(sizeof(*mr),
+					       RTE_CACHE_LINE_SIZE),
+				RTE_CACHE_LINE_SIZE, mp->socket_id);
+	if (mr == NULL) {
+		DRV_LOG(WARNING,
+			"port %u unable to allocate memory for a new MR of"
+			" mempool (%s).",
+			dev->data->port_id, mp->name);
+		data->ret = -1;
+		return;
+	}
+	DRV_LOG(DEBUG, "port %u register MR for chunk #%d of mempool (%s)",
+		dev->data->port_id, mem_idx, mp->name);
+	mr->ibv_mr = mlx5_glue->reg_mr(priv->pd, (void *)addr, len,
+				       IBV_ACCESS_LOCAL_WRITE);
+	if (mr->ibv_mr == NULL) {
+		DRV_LOG(WARNING,
+			"port %u fail to create a verbs MR for address (%p)",
+			dev->data->port_id, (void *)addr);
+		rte_free(mr);
+		data->ret = -1;
+		return;
+	}
+	mr->msl = NULL; /* Mark it is external memory. */
+	mr->ms_bmp = NULL;
+	mr->ms_n = 1;
+	mr->ms_bmp_n = 1;
+	rte_rwlock_write_lock(&priv->mr.rwlock);
+	LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
+	DRV_LOG(DEBUG,
+		"port %u MR CREATED (%p) for external memory %p:\n"
+		"  [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
+		" lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
+		dev->data->port_id, (void *)mr, (void *)addr,
+		addr, addr + len, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+		mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
+	/* Insert to the global cache table. */
+	mr_insert_dev_cache(dev, mr);
+	rte_rwlock_write_unlock(&priv->mr.rwlock);
+	/* Insert to the local cache table */
+	mlx5_mr_addr2mr_bh(dev, mr_ctrl, addr);
+}
+
+/**
+ * Register MR for entire memory chunks in a Mempool having externally allocated
+ * memory and fill in local cache.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param mr_ctrl
+ *   Pointer to per-queue MR control structure.
+ * @param mp
+ *   Pointer to registering Mempool.
+ *
+ * @return
+ *   0 on success, -1 on failure.
+ */
+static uint32_t
+mlx5_mr_update_ext_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+		      struct rte_mempool *mp)
+{
+	struct mr_update_mp_data data = {
+		.dev = dev,
+		.mr_ctrl = mr_ctrl,
+		.ret = 0,
+	};
+
+	rte_mempool_mem_iter(mp, mlx5_mr_update_ext_mp_cb, &data);
+	return data.ret;
+}
+
+/**
+ * Register MR entire memory chunks in a Mempool having externally allocated
+ * memory and search LKey of the address to return.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param addr
+ *   Search key.
+ * @param mp
+ *   Pointer to registering Mempool where addr belongs.
+ *
+ * @return
+ *   LKey for address on success, UINT32_MAX on failure.
+ */
+uint32_t
+mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
+		      struct rte_mempool *mp)
+{
+	struct mlx5_txq_ctrl *txq_ctrl =
+		container_of(txq, struct mlx5_txq_ctrl, txq);
+	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+	struct priv *priv = txq_ctrl->priv;
+
+	mlx5_mr_update_ext_mp(ETH_DEV(priv), mr_ctrl, mp);
+	return mlx5_tx_addr2mr_bh(txq, addr);
+}
+
 /* Called during rte_mempool_mem_iter() by mlx5_mr_update_mp(). */
 static void
 mlx5_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void *opaque,
@@ -1104,6 +1255,10 @@ mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
 	};
 
 	rte_mempool_mem_iter(mp, mlx5_mr_update_mp_cb, &data);
+	if (data.ret < 0 && rte_errno == ENXIO) {
+		/* Mempool may have externally allocated memory. */
+		return mlx5_mr_update_ext_mp(dev, mr_ctrl, mp);
+	}
 	return data.ret;
 }
 
diff --git a/drivers/net/mlx5/mlx5_nl_flow.c b/drivers/net/mlx5/mlx5_nl_flow.c
deleted file mode 100644
index a1c8c340..00000000
--- a/drivers/net/mlx5/mlx5_nl_flow.c
+++ /dev/null
@@ -1,1248 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2018 6WIND S.A.
- * Copyright 2018 Mellanox Technologies, Ltd
- */
-
-#include <assert.h>
-#include <errno.h>
-#include <libmnl/libmnl.h>
-#include <linux/if_ether.h>
-#include <linux/netlink.h>
-#include <linux/pkt_cls.h>
-#include <linux/pkt_sched.h>
-#include <linux/rtnetlink.h>
-#include <linux/tc_act/tc_gact.h>
-#include <linux/tc_act/tc_mirred.h>
-#include <netinet/in.h>
-#include <stdalign.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <sys/socket.h>
-
-#include <rte_byteorder.h>
-#include <rte_errno.h>
-#include <rte_ether.h>
-#include <rte_flow.h>
-
-#include "mlx5.h"
-#include "mlx5_autoconf.h"
-
-#ifdef HAVE_TC_ACT_VLAN
-
-#include <linux/tc_act/tc_vlan.h>
-
-#else /* HAVE_TC_ACT_VLAN */
-
-#define TCA_VLAN_ACT_POP 1
-#define TCA_VLAN_ACT_PUSH 2
-#define TCA_VLAN_ACT_MODIFY 3
-#define TCA_VLAN_PARMS 2
-#define TCA_VLAN_PUSH_VLAN_ID 3
-#define TCA_VLAN_PUSH_VLAN_PROTOCOL 4
-#define TCA_VLAN_PAD 5
-#define TCA_VLAN_PUSH_VLAN_PRIORITY 6
-
-struct tc_vlan {
-	tc_gen;
-	int v_action;
-};
-
-#endif /* HAVE_TC_ACT_VLAN */
-
-/* Normally found in linux/netlink.h. */
-#ifndef NETLINK_CAP_ACK
-#define NETLINK_CAP_ACK 10
-#endif
-
-/* Normally found in linux/pkt_sched.h. */
-#ifndef TC_H_MIN_INGRESS
-#define TC_H_MIN_INGRESS 0xfff2u
-#endif
-
-/* Normally found in linux/pkt_cls.h. */
-#ifndef TCA_CLS_FLAGS_SKIP_SW
-#define TCA_CLS_FLAGS_SKIP_SW (1 << 1)
-#endif
-#ifndef HAVE_TCA_FLOWER_ACT
-#define TCA_FLOWER_ACT 3
-#endif
-#ifndef HAVE_TCA_FLOWER_FLAGS
-#define TCA_FLOWER_FLAGS 22
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_ETH_TYPE
-#define TCA_FLOWER_KEY_ETH_TYPE 8
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_ETH_DST
-#define TCA_FLOWER_KEY_ETH_DST 4
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_ETH_DST_MASK
-#define TCA_FLOWER_KEY_ETH_DST_MASK 5
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC
-#define TCA_FLOWER_KEY_ETH_SRC 6
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK
-#define TCA_FLOWER_KEY_ETH_SRC_MASK 7
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_IP_PROTO
-#define TCA_FLOWER_KEY_IP_PROTO 9
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC
-#define TCA_FLOWER_KEY_IPV4_SRC 10
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK
-#define TCA_FLOWER_KEY_IPV4_SRC_MASK 11
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST
-#define TCA_FLOWER_KEY_IPV4_DST 12
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK
-#define TCA_FLOWER_KEY_IPV4_DST_MASK 13
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC
-#define TCA_FLOWER_KEY_IPV6_SRC 14
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK
-#define TCA_FLOWER_KEY_IPV6_SRC_MASK 15
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST
-#define TCA_FLOWER_KEY_IPV6_DST 16
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK
-#define TCA_FLOWER_KEY_IPV6_DST_MASK 17
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC
-#define TCA_FLOWER_KEY_TCP_SRC 18
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK
-#define TCA_FLOWER_KEY_TCP_SRC_MASK 35
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_TCP_DST
-#define TCA_FLOWER_KEY_TCP_DST 19
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_TCP_DST_MASK
-#define TCA_FLOWER_KEY_TCP_DST_MASK 36
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC
-#define TCA_FLOWER_KEY_UDP_SRC 20
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK
-#define TCA_FLOWER_KEY_UDP_SRC_MASK 37
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_UDP_DST
-#define TCA_FLOWER_KEY_UDP_DST 21
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_UDP_DST_MASK
-#define TCA_FLOWER_KEY_UDP_DST_MASK 38
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_VLAN_ID
-#define TCA_FLOWER_KEY_VLAN_ID 23
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_VLAN_PRIO
-#define TCA_FLOWER_KEY_VLAN_PRIO 24
-#endif
-#ifndef HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE
-#define TCA_FLOWER_KEY_VLAN_ETH_TYPE 25
-#endif
-
-/** Parser state definitions for mlx5_nl_flow_trans[]. */
-enum mlx5_nl_flow_trans {
-	INVALID,
-	BACK,
-	ATTR,
-	PATTERN,
-	ITEM_VOID,
-	ITEM_PORT_ID,
-	ITEM_ETH,
-	ITEM_VLAN,
-	ITEM_IPV4,
-	ITEM_IPV6,
-	ITEM_TCP,
-	ITEM_UDP,
-	ACTIONS,
-	ACTION_VOID,
-	ACTION_PORT_ID,
-	ACTION_DROP,
-	ACTION_OF_POP_VLAN,
-	ACTION_OF_PUSH_VLAN,
-	ACTION_OF_SET_VLAN_VID,
-	ACTION_OF_SET_VLAN_PCP,
-	END,
-};
-
-#define TRANS(...) (const enum mlx5_nl_flow_trans []){ __VA_ARGS__, INVALID, }
-
-#define PATTERN_COMMON \
-	ITEM_VOID, ITEM_PORT_ID, ACTIONS
-#define ACTIONS_COMMON \
-	ACTION_VOID, ACTION_OF_POP_VLAN, ACTION_OF_PUSH_VLAN, \
-	ACTION_OF_SET_VLAN_VID, ACTION_OF_SET_VLAN_PCP
-#define ACTIONS_FATE \
-	ACTION_PORT_ID, ACTION_DROP
-
-/** Parser state transitions used by mlx5_nl_flow_transpose(). */
-static const enum mlx5_nl_flow_trans *const mlx5_nl_flow_trans[] = {
-	[INVALID] = NULL,
-	[BACK] = NULL,
-	[ATTR] = TRANS(PATTERN),
-	[PATTERN] = TRANS(ITEM_ETH, PATTERN_COMMON),
-	[ITEM_VOID] = TRANS(BACK),
-	[ITEM_PORT_ID] = TRANS(BACK),
-	[ITEM_ETH] = TRANS(ITEM_IPV4, ITEM_IPV6, ITEM_VLAN, PATTERN_COMMON),
-	[ITEM_VLAN] = TRANS(ITEM_IPV4, ITEM_IPV6, PATTERN_COMMON),
-	[ITEM_IPV4] = TRANS(ITEM_TCP, ITEM_UDP, PATTERN_COMMON),
-	[ITEM_IPV6] = TRANS(ITEM_TCP, ITEM_UDP, PATTERN_COMMON),
-	[ITEM_TCP] = TRANS(PATTERN_COMMON),
-	[ITEM_UDP] = TRANS(PATTERN_COMMON),
-	[ACTIONS] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
-	[ACTION_VOID] = TRANS(BACK),
-	[ACTION_PORT_ID] = TRANS(ACTION_VOID, END),
-	[ACTION_DROP] = TRANS(ACTION_VOID, END),
-	[ACTION_OF_POP_VLAN] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
-	[ACTION_OF_PUSH_VLAN] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
-	[ACTION_OF_SET_VLAN_VID] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
-	[ACTION_OF_SET_VLAN_PCP] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
-	[END] = NULL,
-};
-
-/** Empty masks for known item types. */
-static const union {
-	struct rte_flow_item_port_id port_id;
-	struct rte_flow_item_eth eth;
-	struct rte_flow_item_vlan vlan;
-	struct rte_flow_item_ipv4 ipv4;
-	struct rte_flow_item_ipv6 ipv6;
-	struct rte_flow_item_tcp tcp;
-	struct rte_flow_item_udp udp;
-} mlx5_nl_flow_mask_empty;
-
-/** Supported masks for known item types. */
-static const struct {
-	struct rte_flow_item_port_id port_id;
-	struct rte_flow_item_eth eth;
-	struct rte_flow_item_vlan vlan;
-	struct rte_flow_item_ipv4 ipv4;
-	struct rte_flow_item_ipv6 ipv6;
-	struct rte_flow_item_tcp tcp;
-	struct rte_flow_item_udp udp;
-} mlx5_nl_flow_mask_supported = {
-	.port_id = {
-		.id = 0xffffffff,
-	},
-	.eth = {
-		.type = RTE_BE16(0xffff),
-		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-	},
-	.vlan = {
-		/* PCP and VID only, no DEI. */
-		.tci = RTE_BE16(0xefff),
-		.inner_type = RTE_BE16(0xffff),
-	},
-	.ipv4.hdr = {
-		.next_proto_id = 0xff,
-		.src_addr = RTE_BE32(0xffffffff),
-		.dst_addr = RTE_BE32(0xffffffff),
-	},
-	.ipv6.hdr = {
-		.proto = 0xff,
-		.src_addr =
-			"\xff\xff\xff\xff\xff\xff\xff\xff"
-			"\xff\xff\xff\xff\xff\xff\xff\xff",
-		.dst_addr =
-			"\xff\xff\xff\xff\xff\xff\xff\xff"
-			"\xff\xff\xff\xff\xff\xff\xff\xff",
-	},
-	.tcp.hdr = {
-		.src_port = RTE_BE16(0xffff),
-		.dst_port = RTE_BE16(0xffff),
-	},
-	.udp.hdr = {
-		.src_port = RTE_BE16(0xffff),
-		.dst_port = RTE_BE16(0xffff),
-	},
-};
-
-/**
- * Retrieve mask for pattern item.
- *
- * This function does basic sanity checks on a pattern item in order to
- * return the most appropriate mask for it.
- *
- * @param[in] item
- *   Item specification.
- * @param[in] mask_default
- *   Default mask for pattern item as specified by the flow API.
- * @param[in] mask_supported
- *   Mask fields supported by the implementation.
- * @param[in] mask_empty
- *   Empty mask to return when there is no specification.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   Either @p item->mask or one of the mask parameters on success, NULL
- *   otherwise and rte_errno is set.
- */
-static const void *
-mlx5_nl_flow_item_mask(const struct rte_flow_item *item,
-		       const void *mask_default,
-		       const void *mask_supported,
-		       const void *mask_empty,
-		       size_t mask_size,
-		       struct rte_flow_error *error)
-{
-	const uint8_t *mask;
-	size_t i;
-
-	/* item->last and item->mask cannot exist without item->spec. */
-	if (!item->spec && (item->mask || item->last)) {
-		rte_flow_error_set
-			(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
-			 "\"mask\" or \"last\" field provided without a"
-			 " corresponding \"spec\"");
-		return NULL;
-	}
-	/* No spec, no mask, no problem. */
-	if (!item->spec)
-		return mask_empty;
-	mask = item->mask ? item->mask : mask_default;
-	assert(mask);
-	/*
-	 * Single-pass check to make sure that:
-	 * - Mask is supported, no bits are set outside mask_supported.
-	 * - Both item->spec and item->last are included in mask.
-	 */
-	for (i = 0; i != mask_size; ++i) {
-		if (!mask[i])
-			continue;
-		if ((mask[i] | ((const uint8_t *)mask_supported)[i]) !=
-		    ((const uint8_t *)mask_supported)[i]) {
-			rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
-				 mask, "unsupported field found in \"mask\"");
-			return NULL;
-		}
-		if (item->last &&
-		    (((const uint8_t *)item->spec)[i] & mask[i]) !=
-		    (((const uint8_t *)item->last)[i] & mask[i])) {
-			rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_LAST,
-				 item->last,
-				 "range between \"spec\" and \"last\" not"
-				 " comprised in \"mask\"");
-			return NULL;
-		}
-	}
-	return mask;
-}
-
-/**
- * Transpose flow rule description to rtnetlink message.
- *
- * This function transposes a flow rule description to a traffic control
- * (TC) filter creation message ready to be sent over Netlink.
- *
- * Target interface is specified as the first entry of the @p ptoi table.
- * Subsequent entries enable this function to resolve other DPDK port IDs
- * found in the flow rule.
- *
- * @param[out] buf
- *   Output message buffer. May be NULL when @p size is 0.
- * @param size
- *   Size of @p buf. Message may be truncated if not large enough.
- * @param[in] ptoi
- *   DPDK port ID to network interface index translation table. This table
- *   is terminated by an entry with a zero ifindex value.
- * @param[in] attr
- *   Flow rule attributes.
- * @param[in] pattern
- *   Pattern specification.
- * @param[in] actions
- *   Associated actions.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   A positive value representing the exact size of the message in bytes
- *   regardless of the @p size parameter on success, a negative errno value
- *   otherwise and rte_errno is set.
- */
-int
-mlx5_nl_flow_transpose(void *buf,
-		       size_t size,
-		       const struct mlx5_nl_flow_ptoi *ptoi,
-		       const struct rte_flow_attr *attr,
-		       const struct rte_flow_item *pattern,
-		       const struct rte_flow_action *actions,
-		       struct rte_flow_error *error)
-{
-	alignas(struct nlmsghdr)
-	uint8_t buf_tmp[mnl_nlmsg_size(sizeof(struct tcmsg) + 1024)];
-	const struct rte_flow_item *item;
-	const struct rte_flow_action *action;
-	unsigned int n;
-	uint32_t act_index_cur;
-	bool in_port_id_set;
-	bool eth_type_set;
-	bool vlan_present;
-	bool vlan_eth_type_set;
-	bool ip_proto_set;
-	struct nlattr *na_flower;
-	struct nlattr *na_flower_act;
-	struct nlattr *na_vlan_id;
-	struct nlattr *na_vlan_priority;
-	const enum mlx5_nl_flow_trans *trans;
-	const enum mlx5_nl_flow_trans *back;
-
-	if (!size)
-		goto error_nobufs;
-init:
-	item = pattern;
-	action = actions;
-	n = 0;
-	act_index_cur = 0;
-	in_port_id_set = false;
-	eth_type_set = false;
-	vlan_present = false;
-	vlan_eth_type_set = false;
-	ip_proto_set = false;
-	na_flower = NULL;
-	na_flower_act = NULL;
-	na_vlan_id = NULL;
-	na_vlan_priority = NULL;
-	trans = TRANS(ATTR);
-	back = trans;
-trans:
-	switch (trans[n++]) {
-		union {
-			const struct rte_flow_item_port_id *port_id;
-			const struct rte_flow_item_eth *eth;
-			const struct rte_flow_item_vlan *vlan;
-			const struct rte_flow_item_ipv4 *ipv4;
-			const struct rte_flow_item_ipv6 *ipv6;
-			const struct rte_flow_item_tcp *tcp;
-			const struct rte_flow_item_udp *udp;
-		} spec, mask;
-		union {
-			const struct rte_flow_action_port_id *port_id;
-			const struct rte_flow_action_of_push_vlan *of_push_vlan;
-			const struct rte_flow_action_of_set_vlan_vid *
-				of_set_vlan_vid;
-			const struct rte_flow_action_of_set_vlan_pcp *
-				of_set_vlan_pcp;
-		} conf;
-		struct nlmsghdr *nlh;
-		struct tcmsg *tcm;
-		struct nlattr *act_index;
-		struct nlattr *act;
-		unsigned int i;
-
-	case INVALID:
-		if (item->type)
-			return rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
-				 item, "unsupported pattern item combination");
-		else if (action->type)
-			return rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
-				 action, "unsupported action combination");
-		return rte_flow_error_set
-			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-			 "flow rule lacks some kind of fate action");
-	case BACK:
-		trans = back;
-		n = 0;
-		goto trans;
-	case ATTR:
-		/*
-		 * Supported attributes: no groups, some priorities and
-		 * ingress only. Don't care about transfer as it is the
-		 * caller's problem.
-		 */
-		if (attr->group)
-			return rte_flow_error_set
-				(error, ENOTSUP,
-				 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
-				 attr, "groups are not supported");
-		if (attr->priority > 0xfffe)
-			return rte_flow_error_set
-				(error, ENOTSUP,
-				 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
-				 attr, "lowest priority level is 0xfffe");
-		if (!attr->ingress)
-			return rte_flow_error_set
-				(error, ENOTSUP,
-				 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
-				 attr, "only ingress is supported");
-		if (attr->egress)
-			return rte_flow_error_set
-				(error, ENOTSUP,
-				 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
-				 attr, "egress is not supported");
-		if (size < mnl_nlmsg_size(sizeof(*tcm)))
-			goto error_nobufs;
-		nlh = mnl_nlmsg_put_header(buf);
-		nlh->nlmsg_type = 0;
-		nlh->nlmsg_flags = 0;
-		nlh->nlmsg_seq = 0;
-		tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
-		tcm->tcm_family = AF_UNSPEC;
-		tcm->tcm_ifindex = ptoi[0].ifindex;
-		/*
-		 * Let kernel pick a handle by default. A predictable handle
-		 * can be set by the caller on the resulting buffer through
-		 * mlx5_nl_flow_brand().
-		 */
-		tcm->tcm_handle = 0;
-		tcm->tcm_parent = TC_H_MAKE(TC_H_INGRESS, TC_H_MIN_INGRESS);
-		/*
-		 * Priority cannot be zero to prevent the kernel from
-		 * picking one automatically.
-		 */
-		tcm->tcm_info = TC_H_MAKE((attr->priority + 1) << 16,
-					  RTE_BE16(ETH_P_ALL));
-		break;
-	case PATTERN:
-		if (!mnl_attr_put_strz_check(buf, size, TCA_KIND, "flower"))
-			goto error_nobufs;
-		na_flower = mnl_attr_nest_start_check(buf, size, TCA_OPTIONS);
-		if (!na_flower)
-			goto error_nobufs;
-		if (!mnl_attr_put_u32_check(buf, size, TCA_FLOWER_FLAGS,
-					    TCA_CLS_FLAGS_SKIP_SW))
-			goto error_nobufs;
-		break;
-	case ITEM_VOID:
-		if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
-			goto trans;
-		++item;
-		break;
-	case ITEM_PORT_ID:
-		if (item->type != RTE_FLOW_ITEM_TYPE_PORT_ID)
-			goto trans;
-		mask.port_id = mlx5_nl_flow_item_mask
-			(item, &rte_flow_item_port_id_mask,
-			 &mlx5_nl_flow_mask_supported.port_id,
-			 &mlx5_nl_flow_mask_empty.port_id,
-			 sizeof(mlx5_nl_flow_mask_supported.port_id), error);
-		if (!mask.port_id)
-			return -rte_errno;
-		if (mask.port_id == &mlx5_nl_flow_mask_empty.port_id) {
-			in_port_id_set = 1;
-			++item;
-			break;
-		}
-		spec.port_id = item->spec;
-		if (mask.port_id->id && mask.port_id->id != 0xffffffff)
-			return rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
-				 mask.port_id,
-				 "no support for partial mask on"
-				 " \"id\" field");
-		if (!mask.port_id->id)
-			i = 0;
-		else
-			for (i = 0; ptoi[i].ifindex; ++i)
-				if (ptoi[i].port_id == spec.port_id->id)
-					break;
-		if (!ptoi[i].ifindex)
-			return rte_flow_error_set
-				(error, ENODEV, RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
-				 spec.port_id,
-				 "missing data to convert port ID to ifindex");
-		tcm = mnl_nlmsg_get_payload(buf);
-		if (in_port_id_set &&
-		    ptoi[i].ifindex != (unsigned int)tcm->tcm_ifindex)
-			return rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
-				 spec.port_id,
-				 "cannot match traffic for several port IDs"
-				 " through a single flow rule");
-		tcm->tcm_ifindex = ptoi[i].ifindex;
-		in_port_id_set = 1;
-		++item;
-		break;
-	case ITEM_ETH:
-		if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
-			goto trans;
-		mask.eth = mlx5_nl_flow_item_mask
-			(item, &rte_flow_item_eth_mask,
-			 &mlx5_nl_flow_mask_supported.eth,
-			 &mlx5_nl_flow_mask_empty.eth,
-			 sizeof(mlx5_nl_flow_mask_supported.eth), error);
-		if (!mask.eth)
-			return -rte_errno;
-		if (mask.eth == &mlx5_nl_flow_mask_empty.eth) {
-			++item;
-			break;
-		}
-		spec.eth = item->spec;
-		if (mask.eth->type && mask.eth->type != RTE_BE16(0xffff))
-			return rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
-				 mask.eth,
-				 "no support for partial mask on"
-				 " \"type\" field");
-		if (mask.eth->type) {
-			if (!mnl_attr_put_u16_check(buf, size,
-						    TCA_FLOWER_KEY_ETH_TYPE,
-						    spec.eth->type))
-				goto error_nobufs;
-			eth_type_set = 1;
-		}
-		if ((!is_zero_ether_addr(&mask.eth->dst) &&
-		     (!mnl_attr_put_check(buf, size,
-					  TCA_FLOWER_KEY_ETH_DST,
-					  ETHER_ADDR_LEN,
-					  spec.eth->dst.addr_bytes) ||
-		      !mnl_attr_put_check(buf, size,
-					  TCA_FLOWER_KEY_ETH_DST_MASK,
-					  ETHER_ADDR_LEN,
-					  mask.eth->dst.addr_bytes))) ||
-		    (!is_zero_ether_addr(&mask.eth->src) &&
-		     (!mnl_attr_put_check(buf, size,
-					  TCA_FLOWER_KEY_ETH_SRC,
-					  ETHER_ADDR_LEN,
-					  spec.eth->src.addr_bytes) ||
-		      !mnl_attr_put_check(buf, size,
-					  TCA_FLOWER_KEY_ETH_SRC_MASK,
-					  ETHER_ADDR_LEN,
-					  mask.eth->src.addr_bytes))))
-			goto error_nobufs;
-		++item;
-		break;
-	case ITEM_VLAN:
-		if (item->type != RTE_FLOW_ITEM_TYPE_VLAN)
-			goto trans;
-		mask.vlan = mlx5_nl_flow_item_mask
-			(item, &rte_flow_item_vlan_mask,
-			 &mlx5_nl_flow_mask_supported.vlan,
-			 &mlx5_nl_flow_mask_empty.vlan,
-			 sizeof(mlx5_nl_flow_mask_supported.vlan), error);
-		if (!mask.vlan)
-			return -rte_errno;
-		if (!eth_type_set &&
-		    !mnl_attr_put_u16_check(buf, size,
-					    TCA_FLOWER_KEY_ETH_TYPE,
-					    RTE_BE16(ETH_P_8021Q)))
-			goto error_nobufs;
-		eth_type_set = 1;
-		vlan_present = 1;
-		if (mask.vlan == &mlx5_nl_flow_mask_empty.vlan) {
-			++item;
-			break;
-		}
-		spec.vlan = item->spec;
-		if ((mask.vlan->tci & RTE_BE16(0xe000) &&
-		     (mask.vlan->tci & RTE_BE16(0xe000)) != RTE_BE16(0xe000)) ||
-		    (mask.vlan->tci & RTE_BE16(0x0fff) &&
-		     (mask.vlan->tci & RTE_BE16(0x0fff)) != RTE_BE16(0x0fff)) ||
-		    (mask.vlan->inner_type &&
-		     mask.vlan->inner_type != RTE_BE16(0xffff)))
-			return rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
-				 mask.vlan,
-				 "no support for partial masks on"
-				 " \"tci\" (PCP and VID parts) and"
-				 " \"inner_type\" fields");
-		if (mask.vlan->inner_type) {
-			if (!mnl_attr_put_u16_check
-			    (buf, size, TCA_FLOWER_KEY_VLAN_ETH_TYPE,
-			     spec.vlan->inner_type))
-				goto error_nobufs;
-			vlan_eth_type_set = 1;
-		}
-		if ((mask.vlan->tci & RTE_BE16(0xe000) &&
-		     !mnl_attr_put_u8_check
-		     (buf, size, TCA_FLOWER_KEY_VLAN_PRIO,
-		      (rte_be_to_cpu_16(spec.vlan->tci) >> 13) & 0x7)) ||
-		    (mask.vlan->tci & RTE_BE16(0x0fff) &&
-		     !mnl_attr_put_u16_check
-		     (buf, size, TCA_FLOWER_KEY_VLAN_ID,
-		      rte_be_to_cpu_16(spec.vlan->tci & RTE_BE16(0x0fff)))))
-			goto error_nobufs;
-		++item;
-		break;
-	case ITEM_IPV4:
-		if (item->type != RTE_FLOW_ITEM_TYPE_IPV4)
-			goto trans;
-		mask.ipv4 = mlx5_nl_flow_item_mask
-			(item, &rte_flow_item_ipv4_mask,
-			 &mlx5_nl_flow_mask_supported.ipv4,
-			 &mlx5_nl_flow_mask_empty.ipv4,
-			 sizeof(mlx5_nl_flow_mask_supported.ipv4), error);
-		if (!mask.ipv4)
-			return -rte_errno;
-		if ((!eth_type_set || !vlan_eth_type_set) &&
-		    !mnl_attr_put_u16_check(buf, size,
-					    vlan_present ?
-					    TCA_FLOWER_KEY_VLAN_ETH_TYPE :
-					    TCA_FLOWER_KEY_ETH_TYPE,
-					    RTE_BE16(ETH_P_IP)))
-			goto error_nobufs;
-		eth_type_set = 1;
-		vlan_eth_type_set = 1;
-		if (mask.ipv4 == &mlx5_nl_flow_mask_empty.ipv4) {
-			++item;
-			break;
-		}
-		spec.ipv4 = item->spec;
-		if (mask.ipv4->hdr.next_proto_id &&
-		    mask.ipv4->hdr.next_proto_id != 0xff)
-			return rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
-				 mask.ipv4,
-				 "no support for partial mask on"
-				 " \"hdr.next_proto_id\" field");
-		if (mask.ipv4->hdr.next_proto_id) {
-			if (!mnl_attr_put_u8_check
-			    (buf, size, TCA_FLOWER_KEY_IP_PROTO,
-			     spec.ipv4->hdr.next_proto_id))
-				goto error_nobufs;
-			ip_proto_set = 1;
-		}
-		if ((mask.ipv4->hdr.src_addr &&
-		     (!mnl_attr_put_u32_check(buf, size,
-					      TCA_FLOWER_KEY_IPV4_SRC,
-					      spec.ipv4->hdr.src_addr) ||
-		      !mnl_attr_put_u32_check(buf, size,
-					      TCA_FLOWER_KEY_IPV4_SRC_MASK,
-					      mask.ipv4->hdr.src_addr))) ||
-		    (mask.ipv4->hdr.dst_addr &&
-		     (!mnl_attr_put_u32_check(buf, size,
-					      TCA_FLOWER_KEY_IPV4_DST,
-					      spec.ipv4->hdr.dst_addr) ||
-		      !mnl_attr_put_u32_check(buf, size,
-					      TCA_FLOWER_KEY_IPV4_DST_MASK,
-					      mask.ipv4->hdr.dst_addr))))
-			goto error_nobufs;
-		++item;
-		break;
-	case ITEM_IPV6:
-		if (item->type != RTE_FLOW_ITEM_TYPE_IPV6)
-			goto trans;
-		mask.ipv6 = mlx5_nl_flow_item_mask
-			(item, &rte_flow_item_ipv6_mask,
-			 &mlx5_nl_flow_mask_supported.ipv6,
-			 &mlx5_nl_flow_mask_empty.ipv6,
-			 sizeof(mlx5_nl_flow_mask_supported.ipv6), error);
-		if (!mask.ipv6)
-			return -rte_errno;
-		if ((!eth_type_set || !vlan_eth_type_set) &&
-		    !mnl_attr_put_u16_check(buf, size,
-					    vlan_present ?
-					    TCA_FLOWER_KEY_VLAN_ETH_TYPE :
-					    TCA_FLOWER_KEY_ETH_TYPE,
-					    RTE_BE16(ETH_P_IPV6)))
-			goto error_nobufs;
-		eth_type_set = 1;
-		vlan_eth_type_set = 1;
-		if (mask.ipv6 == &mlx5_nl_flow_mask_empty.ipv6) {
-			++item;
-			break;
-		}
-		spec.ipv6 = item->spec;
-		if (mask.ipv6->hdr.proto && mask.ipv6->hdr.proto != 0xff)
-			return rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
-				 mask.ipv6,
-				 "no support for partial mask on"
-				 " \"hdr.proto\" field");
-		if (mask.ipv6->hdr.proto) {
-			if (!mnl_attr_put_u8_check
-			    (buf, size, TCA_FLOWER_KEY_IP_PROTO,
-			     spec.ipv6->hdr.proto))
-				goto error_nobufs;
-			ip_proto_set = 1;
-		}
-		if ((!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.src_addr) &&
-		     (!mnl_attr_put_check(buf, size,
-					  TCA_FLOWER_KEY_IPV6_SRC,
-					  sizeof(spec.ipv6->hdr.src_addr),
-					  spec.ipv6->hdr.src_addr) ||
-		      !mnl_attr_put_check(buf, size,
-					  TCA_FLOWER_KEY_IPV6_SRC_MASK,
-					  sizeof(mask.ipv6->hdr.src_addr),
-					  mask.ipv6->hdr.src_addr))) ||
-		    (!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.dst_addr) &&
-		     (!mnl_attr_put_check(buf, size,
-					  TCA_FLOWER_KEY_IPV6_DST,
-					  sizeof(spec.ipv6->hdr.dst_addr),
-					  spec.ipv6->hdr.dst_addr) ||
-		      !mnl_attr_put_check(buf, size,
-					  TCA_FLOWER_KEY_IPV6_DST_MASK,
-					  sizeof(mask.ipv6->hdr.dst_addr),
-					  mask.ipv6->hdr.dst_addr))))
-			goto error_nobufs;
-		++item;
-		break;
-	case ITEM_TCP:
-		if (item->type != RTE_FLOW_ITEM_TYPE_TCP)
-			goto trans;
-		mask.tcp = mlx5_nl_flow_item_mask
-			(item, &rte_flow_item_tcp_mask,
-			 &mlx5_nl_flow_mask_supported.tcp,
-			 &mlx5_nl_flow_mask_empty.tcp,
-			 sizeof(mlx5_nl_flow_mask_supported.tcp), error);
-		if (!mask.tcp)
-			return -rte_errno;
-		if (!ip_proto_set &&
-		    !mnl_attr_put_u8_check(buf, size,
-					   TCA_FLOWER_KEY_IP_PROTO,
-					   IPPROTO_TCP))
-			goto error_nobufs;
-		if (mask.tcp == &mlx5_nl_flow_mask_empty.tcp) {
-			++item;
-			break;
-		}
-		spec.tcp = item->spec;
-		if ((mask.tcp->hdr.src_port &&
-		     mask.tcp->hdr.src_port != RTE_BE16(0xffff)) ||
-		    (mask.tcp->hdr.dst_port &&
-		     mask.tcp->hdr.dst_port != RTE_BE16(0xffff)))
-			return rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
-				 mask.tcp,
-				 "no support for partial masks on"
-				 " \"hdr.src_port\" and \"hdr.dst_port\""
-				 " fields");
-		if ((mask.tcp->hdr.src_port &&
-		     (!mnl_attr_put_u16_check(buf, size,
-					      TCA_FLOWER_KEY_TCP_SRC,
-					      spec.tcp->hdr.src_port) ||
-		      !mnl_attr_put_u16_check(buf, size,
-					      TCA_FLOWER_KEY_TCP_SRC_MASK,
-					      mask.tcp->hdr.src_port))) ||
-		    (mask.tcp->hdr.dst_port &&
-		     (!mnl_attr_put_u16_check(buf, size,
-					      TCA_FLOWER_KEY_TCP_DST,
-					      spec.tcp->hdr.dst_port) ||
-		      !mnl_attr_put_u16_check(buf, size,
-					      TCA_FLOWER_KEY_TCP_DST_MASK,
-					      mask.tcp->hdr.dst_port))))
-			goto error_nobufs;
-		++item;
-		break;
-	case ITEM_UDP:
-		if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
-			goto trans;
-		mask.udp = mlx5_nl_flow_item_mask
-			(item, &rte_flow_item_udp_mask,
-			 &mlx5_nl_flow_mask_supported.udp,
-			 &mlx5_nl_flow_mask_empty.udp,
-			 sizeof(mlx5_nl_flow_mask_supported.udp), error);
-		if (!mask.udp)
-			return -rte_errno;
-		if (!ip_proto_set &&
-		    !mnl_attr_put_u8_check(buf, size,
-					   TCA_FLOWER_KEY_IP_PROTO,
-					   IPPROTO_UDP))
-			goto error_nobufs;
-		if (mask.udp == &mlx5_nl_flow_mask_empty.udp) {
-			++item;
-			break;
-		}
-		spec.udp = item->spec;
-		if ((mask.udp->hdr.src_port &&
-		     mask.udp->hdr.src_port != RTE_BE16(0xffff)) ||
-		    (mask.udp->hdr.dst_port &&
-		     mask.udp->hdr.dst_port != RTE_BE16(0xffff)))
-			return rte_flow_error_set
-				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
-				 mask.udp,
-				 "no support for partial masks on"
-				 " \"hdr.src_port\" and \"hdr.dst_port\""
-				 " fields");
-		if ((mask.udp->hdr.src_port &&
-		     (!mnl_attr_put_u16_check(buf, size,
-					      TCA_FLOWER_KEY_UDP_SRC,
-					      spec.udp->hdr.src_port) ||
-		      !mnl_attr_put_u16_check(buf, size,
-					      TCA_FLOWER_KEY_UDP_SRC_MASK,
-					      mask.udp->hdr.src_port))) ||
-		    (mask.udp->hdr.dst_port &&
-		     (!mnl_attr_put_u16_check(buf, size,
-					      TCA_FLOWER_KEY_UDP_DST,
-					      spec.udp->hdr.dst_port) ||
-		      !mnl_attr_put_u16_check(buf, size,
-					      TCA_FLOWER_KEY_UDP_DST_MASK,
-					      mask.udp->hdr.dst_port))))
-			goto error_nobufs;
-		++item;
-		break;
-	case ACTIONS:
-		if (item->type != RTE_FLOW_ITEM_TYPE_END)
-			goto trans;
-		assert(na_flower);
-		assert(!na_flower_act);
-		na_flower_act =
-			mnl_attr_nest_start_check(buf, size, TCA_FLOWER_ACT);
-		if (!na_flower_act)
-			goto error_nobufs;
-		act_index_cur = 1;
-		break;
-	case ACTION_VOID:
-		if (action->type != RTE_FLOW_ACTION_TYPE_VOID)
-			goto trans;
-		++action;
-		break;
-	case ACTION_PORT_ID:
-		if (action->type != RTE_FLOW_ACTION_TYPE_PORT_ID)
-			goto trans;
-		conf.port_id = action->conf;
-		if (conf.port_id->original)
-			i = 0;
-		else
-			for (i = 0; ptoi[i].ifindex; ++i)
-				if (ptoi[i].port_id == conf.port_id->id)
-					break;
-		if (!ptoi[i].ifindex)
-			return rte_flow_error_set
-				(error, ENODEV, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-				 conf.port_id,
-				 "missing data to convert port ID to ifindex");
-		act_index =
-			mnl_attr_nest_start_check(buf, size, act_index_cur++);
-		if (!act_index ||
-		    !mnl_attr_put_strz_check(buf, size, TCA_ACT_KIND, "mirred"))
-			goto error_nobufs;
-		act = mnl_attr_nest_start_check(buf, size, TCA_ACT_OPTIONS);
-		if (!act)
-			goto error_nobufs;
-		if (!mnl_attr_put_check(buf, size, TCA_MIRRED_PARMS,
-					sizeof(struct tc_mirred),
-					&(struct tc_mirred){
-						.action = TC_ACT_STOLEN,
-						.eaction = TCA_EGRESS_REDIR,
-						.ifindex = ptoi[i].ifindex,
-					}))
-			goto error_nobufs;
-		mnl_attr_nest_end(buf, act);
-		mnl_attr_nest_end(buf, act_index);
-		++action;
-		break;
-	case ACTION_DROP:
-		if (action->type != RTE_FLOW_ACTION_TYPE_DROP)
-			goto trans;
-		act_index =
-			mnl_attr_nest_start_check(buf, size, act_index_cur++);
-		if (!act_index ||
-		    !mnl_attr_put_strz_check(buf, size, TCA_ACT_KIND, "gact"))
-			goto error_nobufs;
-		act = mnl_attr_nest_start_check(buf, size, TCA_ACT_OPTIONS);
-		if (!act)
-			goto error_nobufs;
-		if (!mnl_attr_put_check(buf, size, TCA_GACT_PARMS,
-					sizeof(struct tc_gact),
-					&(struct tc_gact){
-						.action = TC_ACT_SHOT,
-					}))
-			goto error_nobufs;
-		mnl_attr_nest_end(buf, act);
-		mnl_attr_nest_end(buf, act_index);
-		++action;
-		break;
-	case ACTION_OF_POP_VLAN:
-		if (action->type != RTE_FLOW_ACTION_TYPE_OF_POP_VLAN)
-			goto trans;
-		conf.of_push_vlan = NULL;
-		i = TCA_VLAN_ACT_POP;
-		goto action_of_vlan;
-	case ACTION_OF_PUSH_VLAN:
-		if (action->type != RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN)
-			goto trans;
-		conf.of_push_vlan = action->conf;
-		i = TCA_VLAN_ACT_PUSH;
-		goto action_of_vlan;
-	case ACTION_OF_SET_VLAN_VID:
-		if (action->type != RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
-			goto trans;
-		conf.of_set_vlan_vid = action->conf;
-		if (na_vlan_id)
-			goto override_na_vlan_id;
-		i = TCA_VLAN_ACT_MODIFY;
-		goto action_of_vlan;
-	case ACTION_OF_SET_VLAN_PCP:
-		if (action->type != RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)
-			goto trans;
-		conf.of_set_vlan_pcp = action->conf;
-		if (na_vlan_priority)
-			goto override_na_vlan_priority;
-		i = TCA_VLAN_ACT_MODIFY;
-		goto action_of_vlan;
-action_of_vlan:
-		act_index =
-			mnl_attr_nest_start_check(buf, size, act_index_cur++);
-		if (!act_index ||
-		    !mnl_attr_put_strz_check(buf, size, TCA_ACT_KIND, "vlan"))
-			goto error_nobufs;
-		act = mnl_attr_nest_start_check(buf, size, TCA_ACT_OPTIONS);
-		if (!act)
-			goto error_nobufs;
-		if (!mnl_attr_put_check(buf, size, TCA_VLAN_PARMS,
-					sizeof(struct tc_vlan),
-					&(struct tc_vlan){
-						.action = TC_ACT_PIPE,
-						.v_action = i,
-					}))
-			goto error_nobufs;
-		if (i == TCA_VLAN_ACT_POP) {
-			mnl_attr_nest_end(buf, act);
-			mnl_attr_nest_end(buf, act_index);
-			++action;
-			break;
-		}
-		if (i == TCA_VLAN_ACT_PUSH &&
-		    !mnl_attr_put_u16_check(buf, size,
-					    TCA_VLAN_PUSH_VLAN_PROTOCOL,
-					    conf.of_push_vlan->ethertype))
-			goto error_nobufs;
-		na_vlan_id = mnl_nlmsg_get_payload_tail(buf);
-		if (!mnl_attr_put_u16_check(buf, size, TCA_VLAN_PAD, 0))
-			goto error_nobufs;
-		na_vlan_priority = mnl_nlmsg_get_payload_tail(buf);
-		if (!mnl_attr_put_u8_check(buf, size, TCA_VLAN_PAD, 0))
-			goto error_nobufs;
-		mnl_attr_nest_end(buf, act);
-		mnl_attr_nest_end(buf, act_index);
-		if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
-override_na_vlan_id:
-			na_vlan_id->nla_type = TCA_VLAN_PUSH_VLAN_ID;
-			*(uint16_t *)mnl_attr_get_payload(na_vlan_id) =
-				rte_be_to_cpu_16
-				(conf.of_set_vlan_vid->vlan_vid);
-		} else if (action->type ==
-			   RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
-override_na_vlan_priority:
-			na_vlan_priority->nla_type =
-				TCA_VLAN_PUSH_VLAN_PRIORITY;
-			*(uint8_t *)mnl_attr_get_payload(na_vlan_priority) =
-				conf.of_set_vlan_pcp->vlan_pcp;
-		}
-		++action;
-		break;
-	case END:
-		if (item->type != RTE_FLOW_ITEM_TYPE_END ||
-		    action->type != RTE_FLOW_ACTION_TYPE_END)
-			goto trans;
-		if (na_flower_act)
-			mnl_attr_nest_end(buf, na_flower_act);
-		if (na_flower)
-			mnl_attr_nest_end(buf, na_flower);
-		nlh = buf;
-		return nlh->nlmsg_len;
-	}
-	back = trans;
-	trans = mlx5_nl_flow_trans[trans[n - 1]];
-	n = 0;
-	goto trans;
-error_nobufs:
-	if (buf != buf_tmp) {
-		buf = buf_tmp;
-		size = sizeof(buf_tmp);
-		goto init;
-	}
-	return rte_flow_error_set
-		(error, ENOBUFS, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-		 "generated TC message is too large");
-}
-
-/**
- * Brand rtnetlink buffer with unique handle.
- *
- * This handle should be unique for a given network interface to avoid
- * collisions.
- *
- * @param buf
- *   Flow rule buffer previously initialized by mlx5_nl_flow_transpose().
- * @param handle
- *   Unique 32-bit handle to use.
- */
-void
-mlx5_nl_flow_brand(void *buf, uint32_t handle)
-{
-	struct tcmsg *tcm = mnl_nlmsg_get_payload(buf);
-
-	tcm->tcm_handle = handle;
-}
-
-/**
- * Send Netlink message with acknowledgment.
- *
- * @param nl
- *   Libmnl socket to use.
- * @param nlh
- *   Message to send. This function always raises the NLM_F_ACK flag before
- *   sending.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_nl_flow_nl_ack(struct mnl_socket *nl, struct nlmsghdr *nlh)
-{
-	alignas(struct nlmsghdr)
-	uint8_t ans[mnl_nlmsg_size(sizeof(struct nlmsgerr)) +
-		    nlh->nlmsg_len - sizeof(*nlh)];
-	uint32_t seq = random();
-	int ret;
-
-	nlh->nlmsg_flags |= NLM_F_ACK;
-	nlh->nlmsg_seq = seq;
-	ret = mnl_socket_sendto(nl, nlh, nlh->nlmsg_len);
-	if (ret != -1)
-		ret = mnl_socket_recvfrom(nl, ans, sizeof(ans));
-	if (ret != -1)
-		ret = mnl_cb_run
-			(ans, ret, seq, mnl_socket_get_portid(nl), NULL, NULL);
-	if (!ret)
-		return 0;
-	rte_errno = errno;
-	return -rte_errno;
-}
-
-/**
- * Create a Netlink flow rule.
- *
- * @param nl
- *   Libmnl socket to use.
- * @param buf
- *   Flow rule buffer previously initialized by mlx5_nl_flow_transpose().
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-int
-mlx5_nl_flow_create(struct mnl_socket *nl, void *buf,
-		    struct rte_flow_error *error)
-{
-	struct nlmsghdr *nlh = buf;
-
-	nlh->nlmsg_type = RTM_NEWTFILTER;
-	nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
-	if (!mlx5_nl_flow_nl_ack(nl, nlh))
-		return 0;
-	return rte_flow_error_set
-		(error, rte_errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-		 "netlink: failed to create TC flow rule");
-}
-
-/**
- * Destroy a Netlink flow rule.
- *
- * @param nl
- *   Libmnl socket to use.
- * @param buf
- *   Flow rule buffer previously initialized by mlx5_nl_flow_transpose().
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-int
-mlx5_nl_flow_destroy(struct mnl_socket *nl, void *buf,
-		     struct rte_flow_error *error)
-{
-	struct nlmsghdr *nlh = buf;
-
-	nlh->nlmsg_type = RTM_DELTFILTER;
-	nlh->nlmsg_flags = NLM_F_REQUEST;
-	if (!mlx5_nl_flow_nl_ack(nl, nlh))
-		return 0;
-	return rte_flow_error_set
-		(error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-		 "netlink: failed to destroy TC flow rule");
-}
-
-/**
- * Initialize ingress qdisc of a given network interface.
- *
- * @param nl
- *   Libmnl socket of the @p NETLINK_ROUTE kind.
- * @param ifindex
- *   Index of network interface to initialize.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-int
-mlx5_nl_flow_init(struct mnl_socket *nl, unsigned int ifindex,
-		  struct rte_flow_error *error)
-{
-	struct nlmsghdr *nlh;
-	struct tcmsg *tcm;
-	alignas(struct nlmsghdr)
-	uint8_t buf[mnl_nlmsg_size(sizeof(*tcm) + 128)];
-
-	/* Destroy existing ingress qdisc and everything attached to it. */
-	nlh = mnl_nlmsg_put_header(buf);
-	nlh->nlmsg_type = RTM_DELQDISC;
-	nlh->nlmsg_flags = NLM_F_REQUEST;
-	tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
-	tcm->tcm_family = AF_UNSPEC;
-	tcm->tcm_ifindex = ifindex;
-	tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
-	tcm->tcm_parent = TC_H_INGRESS;
-	/* Ignore errors when qdisc is already absent. */
-	if (mlx5_nl_flow_nl_ack(nl, nlh) &&
-	    rte_errno != EINVAL && rte_errno != ENOENT)
-		return rte_flow_error_set
-			(error, rte_errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			 NULL, "netlink: failed to remove ingress qdisc");
-	/* Create fresh ingress qdisc. */
-	nlh = mnl_nlmsg_put_header(buf);
-	nlh->nlmsg_type = RTM_NEWQDISC;
-	nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
-	tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
-	tcm->tcm_family = AF_UNSPEC;
-	tcm->tcm_ifindex = ifindex;
-	tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
-	tcm->tcm_parent = TC_H_INGRESS;
-	mnl_attr_put_strz_check(nlh, sizeof(buf), TCA_KIND, "ingress");
-	if (mlx5_nl_flow_nl_ack(nl, nlh))
-		return rte_flow_error_set
-			(error, rte_errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			 NULL, "netlink: failed to create ingress qdisc");
-	return 0;
-}
-
-/**
- * Create and configure a libmnl socket for Netlink flow rules.
- *
- * @return
- *   A valid libmnl socket object pointer on success, NULL otherwise and
- *   rte_errno is set.
- */
-struct mnl_socket *
-mlx5_nl_flow_socket_create(void)
-{
-	struct mnl_socket *nl = mnl_socket_open(NETLINK_ROUTE);
-
-	if (nl) {
-		mnl_socket_setsockopt(nl, NETLINK_CAP_ACK, &(int){ 1 },
-				      sizeof(int));
-		if (!mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID))
-			return nl;
-	}
-	rte_errno = errno;
-	if (nl)
-		mnl_socket_close(nl);
-	return NULL;
-}
-
-/**
- * Destroy a libmnl socket.
- */
-void
-mlx5_nl_flow_socket_destroy(struct mnl_socket *nl)
-{
-	mnl_socket_close(nl);
-}
diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h
index 0870d32f..29742b13 100644
--- a/drivers/net/mlx5/mlx5_prm.h
+++ b/drivers/net/mlx5/mlx5_prm.h
@@ -159,7 +159,7 @@ struct mlx5_wqe_eth_seg_small {
 	uint8_t	cs_flags;
 	uint8_t	rsvd1;
 	uint16_t mss;
-	uint32_t rsvd2;
+	uint32_t flow_table_metadata;
 	uint16_t inline_hdr_sz;
 	uint8_t inline_hdr[2];
 } __rte_aligned(MLX5_WQE_DWORD_SIZE);
@@ -280,6 +280,226 @@ struct mlx5_cqe {
 /* CQE format value. */
 #define MLX5_COMPRESSED 0x3
 
+/* The field of packet to be modified. */
+enum mlx5_modificaiton_field {
+	MLX5_MODI_OUT_SMAC_47_16 = 1,
+	MLX5_MODI_OUT_SMAC_15_0,
+	MLX5_MODI_OUT_ETHERTYPE,
+	MLX5_MODI_OUT_DMAC_47_16,
+	MLX5_MODI_OUT_DMAC_15_0,
+	MLX5_MODI_OUT_IP_DSCP,
+	MLX5_MODI_OUT_TCP_FLAGS,
+	MLX5_MODI_OUT_TCP_SPORT,
+	MLX5_MODI_OUT_TCP_DPORT,
+	MLX5_MODI_OUT_IPV4_TTL,
+	MLX5_MODI_OUT_UDP_SPORT,
+	MLX5_MODI_OUT_UDP_DPORT,
+	MLX5_MODI_OUT_SIPV6_127_96,
+	MLX5_MODI_OUT_SIPV6_95_64,
+	MLX5_MODI_OUT_SIPV6_63_32,
+	MLX5_MODI_OUT_SIPV6_31_0,
+	MLX5_MODI_OUT_DIPV6_127_96,
+	MLX5_MODI_OUT_DIPV6_95_64,
+	MLX5_MODI_OUT_DIPV6_63_32,
+	MLX5_MODI_OUT_DIPV6_31_0,
+	MLX5_MODI_OUT_SIPV4,
+	MLX5_MODI_OUT_DIPV4,
+	MLX5_MODI_IN_SMAC_47_16 = 0x31,
+	MLX5_MODI_IN_SMAC_15_0,
+	MLX5_MODI_IN_ETHERTYPE,
+	MLX5_MODI_IN_DMAC_47_16,
+	MLX5_MODI_IN_DMAC_15_0,
+	MLX5_MODI_IN_IP_DSCP,
+	MLX5_MODI_IN_TCP_FLAGS,
+	MLX5_MODI_IN_TCP_SPORT,
+	MLX5_MODI_IN_TCP_DPORT,
+	MLX5_MODI_IN_IPV4_TTL,
+	MLX5_MODI_IN_UDP_SPORT,
+	MLX5_MODI_IN_UDP_DPORT,
+	MLX5_MODI_IN_SIPV6_127_96,
+	MLX5_MODI_IN_SIPV6_95_64,
+	MLX5_MODI_IN_SIPV6_63_32,
+	MLX5_MODI_IN_SIPV6_31_0,
+	MLX5_MODI_IN_DIPV6_127_96,
+	MLX5_MODI_IN_DIPV6_95_64,
+	MLX5_MODI_IN_DIPV6_63_32,
+	MLX5_MODI_IN_DIPV6_31_0,
+	MLX5_MODI_IN_SIPV4,
+	MLX5_MODI_IN_DIPV4,
+	MLX5_MODI_OUT_IPV6_HOPLIMIT,
+	MLX5_MODI_IN_IPV6_HOPLIMIT,
+	MLX5_MODI_META_DATA_REG_A,
+	MLX5_MODI_META_DATA_REG_B = 0x50,
+};
+
+/* Modification sub command. */
+struct mlx5_modification_cmd {
+	union {
+		uint32_t data0;
+		struct {
+			unsigned int bits:5;
+			unsigned int rsvd0:3;
+			unsigned int src_offset:5; /* Start bit offset. */
+			unsigned int rsvd1:3;
+			unsigned int src_field:12;
+			unsigned int type:4;
+		};
+	};
+	union {
+		uint32_t data1;
+		uint8_t data[4];
+		struct {
+			unsigned int rsvd2:8;
+			unsigned int dst_offset:8;
+			unsigned int dst_field:12;
+			unsigned int rsvd3:4;
+		};
+	};
+};
+
+typedef uint32_t u32;
+typedef uint16_t u16;
+typedef uint8_t u8;
+
+#define __mlx5_nullp(typ) ((struct mlx5_ifc_##typ##_bits *)0)
+#define __mlx5_bit_sz(typ, fld) sizeof(__mlx5_nullp(typ)->fld)
+#define __mlx5_bit_off(typ, fld) ((unsigned int)(unsigned long) \
+				  (&(__mlx5_nullp(typ)->fld)))
+#define __mlx5_dw_bit_off(typ, fld) (32 - __mlx5_bit_sz(typ, fld) - \
+				    (__mlx5_bit_off(typ, fld) & 0x1f))
+#define __mlx5_dw_off(typ, fld) (__mlx5_bit_off(typ, fld) / 32)
+#define __mlx5_dw_mask(typ, fld) (__mlx5_mask(typ, fld) << \
+				  __mlx5_dw_bit_off(typ, fld))
+#define __mlx5_mask(typ, fld) ((u32)((1ull << __mlx5_bit_sz(typ, fld)) - 1))
+#define __mlx5_16_off(typ, fld) (__mlx5_bit_off(typ, fld) / 16)
+#define __mlx5_16_bit_off(typ, fld) (16 - __mlx5_bit_sz(typ, fld) - \
+				    (__mlx5_bit_off(typ, fld) & 0xf))
+#define __mlx5_mask16(typ, fld) ((u16)((1ull << __mlx5_bit_sz(typ, fld)) - 1))
+#define MLX5_ST_SZ_DW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 32)
+#define MLX5_ST_SZ_DB(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 8)
+#define MLX5_BYTE_OFF(typ, fld) (__mlx5_bit_off(typ, fld) / 8)
+#define MLX5_ADDR_OF(typ, p, fld) ((char *)(p) + MLX5_BYTE_OFF(typ, fld))
+
+/* insert a value to a struct */
+#define MLX5_SET(typ, p, fld, v) \
+	do { \
+		u32 _v = v; \
+		*((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \
+		rte_cpu_to_be_32((rte_be_to_cpu_32(*((u32 *)(p) + \
+				  __mlx5_dw_off(typ, fld))) & \
+				  (~__mlx5_dw_mask(typ, fld))) | \
+				 (((_v) & __mlx5_mask(typ, fld)) << \
+				   __mlx5_dw_bit_off(typ, fld))); \
+	} while (0)
+#define MLX5_GET16(typ, p, fld) \
+	((rte_be_to_cpu_16(*((__be16 *)(p) + \
+	  __mlx5_16_off(typ, fld))) >> __mlx5_16_bit_off(typ, fld)) & \
+	 __mlx5_mask16(typ, fld))
+#define MLX5_FLD_SZ_BYTES(typ, fld) (__mlx5_bit_sz(typ, fld) / 8)
+
+struct mlx5_ifc_fte_match_set_misc_bits {
+	u8 reserved_at_0[0x8];
+	u8 source_sqn[0x18];
+	u8 reserved_at_20[0x10];
+	u8 source_port[0x10];
+	u8 outer_second_prio[0x3];
+	u8 outer_second_cfi[0x1];
+	u8 outer_second_vid[0xc];
+	u8 inner_second_prio[0x3];
+	u8 inner_second_cfi[0x1];
+	u8 inner_second_vid[0xc];
+	u8 outer_second_cvlan_tag[0x1];
+	u8 inner_second_cvlan_tag[0x1];
+	u8 outer_second_svlan_tag[0x1];
+	u8 inner_second_svlan_tag[0x1];
+	u8 reserved_at_64[0xc];
+	u8 gre_protocol[0x10];
+	u8 gre_key_h[0x18];
+	u8 gre_key_l[0x8];
+	u8 vxlan_vni[0x18];
+	u8 reserved_at_b8[0x8];
+	u8 reserved_at_c0[0x20];
+	u8 reserved_at_e0[0xc];
+	u8 outer_ipv6_flow_label[0x14];
+	u8 reserved_at_100[0xc];
+	u8 inner_ipv6_flow_label[0x14];
+	u8 reserved_at_120[0xe0];
+};
+
+struct mlx5_ifc_ipv4_layout_bits {
+	u8 reserved_at_0[0x60];
+	u8 ipv4[0x20];
+};
+
+struct mlx5_ifc_ipv6_layout_bits {
+	u8 ipv6[16][0x8];
+};
+
+union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits {
+	struct mlx5_ifc_ipv6_layout_bits ipv6_layout;
+	struct mlx5_ifc_ipv4_layout_bits ipv4_layout;
+	u8 reserved_at_0[0x80];
+};
+
+struct mlx5_ifc_fte_match_set_lyr_2_4_bits {
+	u8 smac_47_16[0x20];
+	u8 smac_15_0[0x10];
+	u8 ethertype[0x10];
+	u8 dmac_47_16[0x20];
+	u8 dmac_15_0[0x10];
+	u8 first_prio[0x3];
+	u8 first_cfi[0x1];
+	u8 first_vid[0xc];
+	u8 ip_protocol[0x8];
+	u8 ip_dscp[0x6];
+	u8 ip_ecn[0x2];
+	u8 cvlan_tag[0x1];
+	u8 svlan_tag[0x1];
+	u8 frag[0x1];
+	u8 ip_version[0x4];
+	u8 tcp_flags[0x9];
+	u8 tcp_sport[0x10];
+	u8 tcp_dport[0x10];
+	u8 reserved_at_c0[0x20];
+	u8 udp_sport[0x10];
+	u8 udp_dport[0x10];
+	union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits src_ipv4_src_ipv6;
+	union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits dst_ipv4_dst_ipv6;
+};
+
+struct mlx5_ifc_fte_match_mpls_bits {
+	u8 mpls_label[0x14];
+	u8 mpls_exp[0x3];
+	u8 mpls_s_bos[0x1];
+	u8 mpls_ttl[0x8];
+};
+
+struct mlx5_ifc_fte_match_set_misc2_bits {
+	struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls;
+	struct mlx5_ifc_fte_match_mpls_bits inner_first_mpls;
+	struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls_over_gre;
+	struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls_over_udp;
+	u8 reserved_at_80[0x100];
+	u8 metadata_reg_a[0x20];
+	u8 reserved_at_1a0[0x60];
+};
+
+/* Flow matcher. */
+struct mlx5_ifc_fte_match_param_bits {
+	struct mlx5_ifc_fte_match_set_lyr_2_4_bits outer_headers;
+	struct mlx5_ifc_fte_match_set_misc_bits misc_parameters;
+	struct mlx5_ifc_fte_match_set_lyr_2_4_bits inner_headers;
+	struct mlx5_ifc_fte_match_set_misc2_bits misc_parameters_2;
+	u8 reserved_at_800[0x800];
+};
+
+enum {
+	MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT,
+	MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT,
+	MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT,
+	MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT
+};
+
 /* CQE format mask. */
 #define MLX5E_CQE_FORMAT_MASK 0xc
 
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 1f7bfd44..ed993ea6 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -388,7 +388,6 @@ mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev)
 			     DEV_RX_OFFLOAD_TIMESTAMP |
 			     DEV_RX_OFFLOAD_JUMBO_FRAME);
 
-	offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
 	if (config->hw_fcs_strip)
 		offloads |= DEV_RX_OFFLOAD_KEEP_CRC;
 
@@ -1438,7 +1437,7 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	tmpl->rxq.vlan_strip = !!(offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
 	/* By default, FCS (CRC) is stripped by hardware. */
 	tmpl->rxq.crc_present = 0;
-	if (rte_eth_dev_must_keep_crc(offloads)) {
+	if (offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
 		if (config->hw_fcs_strip) {
 			tmpl->rxq.crc_present = 1;
 		} else {
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 2d14f8a6..24a054d5 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -523,6 +523,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		uint8_t tso = txq->tso_en && (buf->ol_flags & PKT_TX_TCP_SEG);
 		uint32_t swp_offsets = 0;
 		uint8_t swp_types = 0;
+		rte_be32_t metadata;
 		uint16_t tso_segsz = 0;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		uint32_t total_length = 0;
@@ -566,6 +567,9 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		cs_flags = txq_ol_cksum_to_cs(buf);
 		txq_mbuf_to_swp(txq, buf, (uint8_t *)&swp_offsets, &swp_types);
 		raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE;
+		/* Copy metadata from mbuf if valid */
+		metadata = buf->ol_flags & PKT_TX_METADATA ? buf->tx_metadata :
+							     0;
 		/* Replace the Ethernet type by the VLAN if necessary. */
 		if (buf->ol_flags & PKT_TX_VLAN_PKT) {
 			uint32_t vlan = rte_cpu_to_be_32(0x81000000 |
@@ -781,7 +785,7 @@ next_pkt:
 				swp_offsets,
 				cs_flags | (swp_types << 8) |
 				(rte_cpu_to_be_16(tso_segsz) << 16),
-				0,
+				metadata,
 				(ehdr << 16) | rte_cpu_to_be_16(tso_header_sz),
 			};
 		} else {
@@ -795,7 +799,7 @@ next_pkt:
 			wqe->eseg = (rte_v128u32_t){
 				swp_offsets,
 				cs_flags | (swp_types << 8),
-				0,
+				metadata,
 				(ehdr << 16) | rte_cpu_to_be_16(pkt_inline_sz),
 			};
 		}
@@ -861,7 +865,7 @@ mlx5_mpw_new(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw, uint32_t length)
 	mpw->wqe->eseg.inline_hdr_sz = 0;
 	mpw->wqe->eseg.rsvd0 = 0;
 	mpw->wqe->eseg.rsvd1 = 0;
-	mpw->wqe->eseg.rsvd2 = 0;
+	mpw->wqe->eseg.flow_table_metadata = 0;
 	mpw->wqe->ctrl[0] = rte_cpu_to_be_32((MLX5_OPC_MOD_MPW << 24) |
 					     (txq->wqe_ci << 8) |
 					     MLX5_OPCODE_TSO);
@@ -948,6 +952,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		uint32_t length;
 		unsigned int segs_n = buf->nb_segs;
 		uint32_t cs_flags;
+		rte_be32_t metadata;
 
 		/*
 		 * Make sure there is enough room to store this packet and
@@ -964,6 +969,9 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		max_elts -= segs_n;
 		--pkts_n;
 		cs_flags = txq_ol_cksum_to_cs(buf);
+		/* Copy metadata from mbuf if valid */
+		metadata = buf->ol_flags & PKT_TX_METADATA ? buf->tx_metadata :
+							     0;
 		/* Retrieve packet information. */
 		length = PKT_LEN(buf);
 		assert(length);
@@ -971,6 +979,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		if ((mpw.state == MLX5_MPW_STATE_OPENED) &&
 		    ((mpw.len != length) ||
 		     (segs_n != 1) ||
+		     (mpw.wqe->eseg.flow_table_metadata != metadata) ||
 		     (mpw.wqe->eseg.cs_flags != cs_flags)))
 			mlx5_mpw_close(txq, &mpw);
 		if (mpw.state == MLX5_MPW_STATE_CLOSED) {
@@ -984,6 +993,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			max_wqe -= 2;
 			mlx5_mpw_new(txq, &mpw, length);
 			mpw.wqe->eseg.cs_flags = cs_flags;
+			mpw.wqe->eseg.flow_table_metadata = metadata;
 		}
 		/* Multi-segment packets must be alone in their MPW. */
 		assert((segs_n == 1) || (mpw.pkts_n == 0));
@@ -1082,7 +1092,7 @@ mlx5_mpw_inline_new(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw,
 	mpw->wqe->eseg.cs_flags = 0;
 	mpw->wqe->eseg.rsvd0 = 0;
 	mpw->wqe->eseg.rsvd1 = 0;
-	mpw->wqe->eseg.rsvd2 = 0;
+	mpw->wqe->eseg.flow_table_metadata = 0;
 	inl = (struct mlx5_wqe_inl_small *)
 		(((uintptr_t)mpw->wqe) + 2 * MLX5_WQE_DWORD_SIZE);
 	mpw->data.raw = (uint8_t *)&inl->raw;
@@ -1172,6 +1182,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 		uint32_t length;
 		unsigned int segs_n = buf->nb_segs;
 		uint8_t cs_flags;
+		rte_be32_t metadata;
 
 		/*
 		 * Make sure there is enough room to store this packet and
@@ -1193,18 +1204,23 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 		 */
 		max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 		cs_flags = txq_ol_cksum_to_cs(buf);
+		/* Copy metadata from mbuf if valid */
+		metadata = buf->ol_flags & PKT_TX_METADATA ? buf->tx_metadata :
+							     0;
 		/* Retrieve packet information. */
 		length = PKT_LEN(buf);
 		/* Start new session if packet differs. */
 		if (mpw.state == MLX5_MPW_STATE_OPENED) {
 			if ((mpw.len != length) ||
 			    (segs_n != 1) ||
+			    (mpw.wqe->eseg.flow_table_metadata != metadata) ||
 			    (mpw.wqe->eseg.cs_flags != cs_flags))
 				mlx5_mpw_close(txq, &mpw);
 		} else if (mpw.state == MLX5_MPW_INL_STATE_OPENED) {
 			if ((mpw.len != length) ||
 			    (segs_n != 1) ||
 			    (length > inline_room) ||
+			    (mpw.wqe->eseg.flow_table_metadata != metadata) ||
 			    (mpw.wqe->eseg.cs_flags != cs_flags)) {
 				mlx5_mpw_inline_close(txq, &mpw);
 				inline_room =
@@ -1224,12 +1240,14 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 				max_wqe -= 2;
 				mlx5_mpw_new(txq, &mpw, length);
 				mpw.wqe->eseg.cs_flags = cs_flags;
+				mpw.wqe->eseg.flow_table_metadata = metadata;
 			} else {
 				if (unlikely(max_wqe < wqe_inl_n))
 					break;
 				max_wqe -= wqe_inl_n;
 				mlx5_mpw_inline_new(txq, &mpw, length);
 				mpw.wqe->eseg.cs_flags = cs_flags;
+				mpw.wqe->eseg.flow_table_metadata = metadata;
 			}
 		}
 		/* Multi-segment packets must be alone in their MPW. */
@@ -1461,6 +1479,7 @@ txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 		unsigned int do_inline = 0; /* Whether inline is possible. */
 		uint32_t length;
 		uint8_t cs_flags;
+		rte_be32_t metadata;
 
 		/* Multi-segmented packet is handled in slow-path outside. */
 		assert(NB_SEGS(buf) == 1);
@@ -1468,6 +1487,9 @@ txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 		if (max_elts - j == 0)
 			break;
 		cs_flags = txq_ol_cksum_to_cs(buf);
+		/* Copy metadata from mbuf if valid */
+		metadata = buf->ol_flags & PKT_TX_METADATA ? buf->tx_metadata :
+							     0;
 		/* Retrieve packet information. */
 		length = PKT_LEN(buf);
 		/* Start new session if:
@@ -1482,6 +1504,7 @@ txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 			    (length <= txq->inline_max_packet_sz &&
 			     inl_pad + sizeof(inl_hdr) + length >
 			     mpw_room) ||
+			     (mpw.wqe->eseg.flow_table_metadata != metadata) ||
 			    (mpw.wqe->eseg.cs_flags != cs_flags))
 				max_wqe -= mlx5_empw_close(txq, &mpw);
 		}
@@ -1505,6 +1528,7 @@ txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 				    sizeof(inl_hdr) + length <= mpw_room &&
 				    !txq->mpw_hdr_dseg;
 			mpw.wqe->eseg.cs_flags = cs_flags;
+			mpw.wqe->eseg.flow_table_metadata = metadata;
 		} else {
 			/* Evaluate whether the next packet can be inlined.
 			 * Inlininig is possible when:
@@ -2097,7 +2121,7 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	const unsigned int wq_mask = (1 << rxq->elts_n) - 1;
 	volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
 	unsigned int i = 0;
-	uint16_t rq_ci = rxq->rq_ci;
+	uint32_t rq_ci = rxq->rq_ci;
 	uint16_t consumed_strd = rxq->consumed_strd;
 	struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
 
@@ -2324,7 +2348,7 @@ removed_rx_burst(void *dpdk_txq __rte_unused,
  * (e.g.  mlx5_rxtx_vec_sse.c for x86).
  */
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 mlx5_tx_burst_raw_vec(void *dpdk_txq __rte_unused,
 		      struct rte_mbuf **pkts __rte_unused,
 		      uint16_t pkts_n __rte_unused)
@@ -2332,7 +2356,7 @@ mlx5_tx_burst_raw_vec(void *dpdk_txq __rte_unused,
 	return 0;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 mlx5_tx_burst_vec(void *dpdk_txq __rte_unused,
 		  struct rte_mbuf **pkts __rte_unused,
 		  uint16_t pkts_n __rte_unused)
@@ -2340,7 +2364,7 @@ mlx5_tx_burst_vec(void *dpdk_txq __rte_unused,
 	return 0;
 }
 
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 mlx5_rx_burst_vec(void *dpdk_txq __rte_unused,
 		  struct rte_mbuf **pkts __rte_unused,
 		  uint16_t pkts_n __rte_unused)
@@ -2348,25 +2372,25 @@ mlx5_rx_burst_vec(void *dpdk_txq __rte_unused,
 	return 0;
 }
 
-int __attribute__((weak))
+__rte_weak int
 mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev __rte_unused)
 {
 	return -ENOTSUP;
 }
 
-int __attribute__((weak))
+__rte_weak int
 mlx5_check_vec_tx_support(struct rte_eth_dev *dev __rte_unused)
 {
 	return -ENOTSUP;
 }
 
-int __attribute__((weak))
+__rte_weak int
 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused)
 {
 	return -ENOTSUP;
 }
 
-int __attribute__((weak))
+__rte_weak int
 mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused)
 {
 	return -ENOTSUP;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 48ed2b20..1db468c3 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -97,10 +97,10 @@ struct mlx5_rxq_data {
 	volatile uint32_t *rq_db;
 	volatile uint32_t *cq_db;
 	uint16_t port_id;
-	uint16_t rq_ci;
+	uint32_t rq_ci;
 	uint16_t consumed_strd; /* Number of consumed strides in WQE. */
-	uint16_t rq_pi;
-	uint16_t cq_ci;
+	uint32_t rq_pi;
+	uint32_t cq_ci;
 	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
 	uint16_t mprq_max_memcpy_len; /* Maximum size of packet to memcpy. */
 	volatile void *wqes;
@@ -363,6 +363,8 @@ uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
 void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
 uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr);
 uint32_t mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr);
+uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
+			       struct rte_mempool *mp);
 
 /**
  * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
@@ -607,6 +609,24 @@ mlx5_tx_complete(struct mlx5_txq_data *txq)
 }
 
 /**
+ * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the
+ * cloned mbuf is allocated is returned instead.
+ *
+ * @param buf
+ *   Pointer to mbuf.
+ *
+ * @return
+ *   Memory pool where data is located for given mbuf.
+ */
+static struct rte_mempool *
+mlx5_mb2mp(struct rte_mbuf *buf)
+{
+	if (unlikely(RTE_MBUF_INDIRECT(buf)))
+		return rte_mbuf_from_indirect(buf)->pool;
+	return buf->pool;
+}
+
+/**
  * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx
  * as mempool is pre-configured and static.
  *
@@ -664,7 +684,20 @@ mlx5_tx_addr2mr(struct mlx5_txq_data *txq, uintptr_t addr)
 	return mlx5_tx_addr2mr_bh(txq, addr);
 }
 
-#define mlx5_tx_mb2mr(rxq, mb) mlx5_tx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+static __rte_always_inline uint32_t
+mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
+{
+	uintptr_t addr = (uintptr_t)mb->buf_addr;
+	uint32_t lkey = mlx5_tx_addr2mr(txq, addr);
+
+	if (likely(lkey != UINT32_MAX))
+		return lkey;
+	if (rte_errno == ENXIO) {
+		/* Mempool may have externally allocated memory. */
+		lkey = mlx5_tx_update_ext_mp(txq, addr, mlx5_mb2mp(mb));
+	}
+	return lkey;
+}
 
 /**
  * Ring TX queue doorbell and flush the update if requested.
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c
index 0a4aed8f..1453f4ff 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.c
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.c
@@ -40,7 +40,8 @@
 #endif
 
 /**
- * Count the number of packets having same ol_flags and calculate cs_flags.
+ * Count the number of packets having same ol_flags and same metadata (if
+ * PKT_TX_METADATA is set in ol_flags), and calculate cs_flags.
  *
  * @param pkts
  *   Pointer to array of packets.
@@ -48,26 +49,45 @@
  *   Number of packets.
  * @param cs_flags
  *   Pointer of flags to be returned.
+ * @param metadata
+ *   Pointer of metadata to be returned.
+ * @param txq_offloads
+ *   Offloads enabled on Tx queue
  *
  * @return
- *   Number of packets having same ol_flags.
+ *   Number of packets having same ol_flags and metadata, if relevant.
  */
 static inline unsigned int
-txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags)
+txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags,
+		 rte_be32_t *metadata, const uint64_t txq_offloads)
 {
 	unsigned int pos;
-	const uint64_t ol_mask =
+	const uint64_t cksum_ol_mask =
 		PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM |
 		PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE |
 		PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM;
+	rte_be32_t p0_metadata, pn_metadata;
 
 	if (!pkts_n)
 		return 0;
-	/* Count the number of packets having same ol_flags. */
-	for (pos = 1; pos < pkts_n; ++pos)
-		if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)
+	p0_metadata = pkts[0]->ol_flags & PKT_TX_METADATA ?
+			pkts[0]->tx_metadata : 0;
+	/* Count the number of packets having same offload parameters. */
+	for (pos = 1; pos < pkts_n; ++pos) {
+		/* Check if packet has same checksum flags. */
+		if ((txq_offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP) &&
+		    ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & cksum_ol_mask))
 			break;
+		/* Check if packet has same metadata. */
+		if (txq_offloads & DEV_TX_OFFLOAD_MATCH_METADATA) {
+			pn_metadata = pkts[pos]->ol_flags & PKT_TX_METADATA ?
+					pkts[pos]->tx_metadata : 0;
+			if (pn_metadata != p0_metadata)
+				break;
+		}
+	}
 	*cs_flags = txq_ol_cksum_to_cs(pkts[0]);
+	*metadata = p0_metadata;
 	return pos;
 }
 
@@ -96,7 +116,7 @@ mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts,
 		uint16_t ret;
 
 		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
-		ret = txq_burst_v(txq, &pkts[nb_tx], n, 0);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, 0, 0);
 		nb_tx += ret;
 		if (!ret)
 			break;
@@ -127,6 +147,7 @@ mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		uint8_t cs_flags = 0;
 		uint16_t n;
 		uint16_t ret;
+		rte_be32_t metadata = 0;
 
 		/* Transmit multi-seg packets in the head of pkts list. */
 		if ((txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) &&
@@ -137,9 +158,12 @@ mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
 		if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
 			n = txq_count_contig_single_seg(&pkts[nb_tx], n);
-		if (txq->offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP)
-			n = txq_calc_offload(&pkts[nb_tx], n, &cs_flags);
-		ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
+		if (txq->offloads & (MLX5_VEC_TX_CKSUM_OFFLOAD_CAP |
+				     DEV_TX_OFFLOAD_MATCH_METADATA))
+			n = txq_calc_offload(&pkts[nb_tx], n,
+					     &cs_flags, &metadata,
+					     txq->offloads);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags, metadata);
 		nb_tx += ret;
 		if (!ret)
 			break;
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h
index fb884f92..fda7004e 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.h
@@ -22,6 +22,7 @@
 /* HW offload capabilities of vectorized Tx. */
 #define MLX5_VEC_TX_OFFLOAD_CAP \
 	(MLX5_VEC_TX_CKSUM_OFFLOAD_CAP | \
+	 DEV_TX_OFFLOAD_MATCH_METADATA | \
 	 DEV_TX_OFFLOAD_MULTI_SEGS)
 
 /*
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
index b37b7381..0b729f18 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
@@ -201,13 +201,15 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
  *   Number of packets to be sent (<= MLX5_VPMD_TX_MAX_BURST).
  * @param cs_flags
  *   Checksum offload flags to be written in the descriptor.
+ * @param metadata
+ *   Metadata value to be written in the descriptor.
  *
  * @return
  *   Number of packets successfully transmitted (<= pkts_n).
  */
 static inline uint16_t
 txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
-	    uint8_t cs_flags)
+	    uint8_t cs_flags, rte_be32_t metadata)
 {
 	struct rte_mbuf **elts;
 	uint16_t elts_head = txq->elts_head;
@@ -293,11 +295,8 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
 	ctrl = vqtbl1q_u8(ctrl, ctrl_shuf_m);
 	vst1q_u8((void *)t_wqe, ctrl);
 	/* Fill ESEG in the header. */
-	vst1q_u8((void *)(t_wqe + 1),
-		 ((uint8x16_t) { 0, 0, 0, 0,
-				 cs_flags, 0, 0, 0,
-				 0, 0, 0, 0,
-				 0, 0, 0, 0 }));
+	vst1q_u32((void *)(t_wqe + 1),
+		 ((uint32x4_t) { 0, cs_flags, metadata, 0 }));
 #ifdef MLX5_PMD_SOFT_COUNTERS
 	txq->stats.opackets += pkts_n;
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
index 54b3783c..e0f95f92 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
@@ -202,13 +202,15 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
  *   Number of packets to be sent (<= MLX5_VPMD_TX_MAX_BURST).
  * @param cs_flags
  *   Checksum offload flags to be written in the descriptor.
+ * @param metadata
+ *   Metadata value to be written in the descriptor.
  *
  * @return
  *   Number of packets successfully transmitted (<= pkts_n).
  */
 static inline uint16_t
 txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
-	    uint8_t cs_flags)
+	    uint8_t cs_flags, rte_be32_t metadata)
 {
 	struct rte_mbuf **elts;
 	uint16_t elts_head = txq->elts_head;
@@ -292,11 +294,7 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
 	ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
 	_mm_store_si128(t_wqe, ctrl);
 	/* Fill ESEG in the header. */
-	_mm_store_si128(t_wqe + 1,
-			_mm_set_epi8(0, 0, 0, 0,
-				     0, 0, 0, 0,
-				     0, 0, 0, cs_flags,
-				     0, 0, 0, 0));
+	_mm_store_si128(t_wqe + 1, _mm_set_epi32(0, metadata, cs_flags, 0));
 #ifdef MLX5_PMD_SOFT_COUNTERS
 	txq->stats.opackets += pkts_n;
 #endif
diff --git a/drivers/net/mlx5/mlx5_socket.c b/drivers/net/mlx5/mlx5_socket.c
index a3a52291..00106171 100644
--- a/drivers/net/mlx5/mlx5_socket.c
+++ b/drivers/net/mlx5/mlx5_socket.c
@@ -3,8 +3,6 @@
  * Copyright 2016 Mellanox Technologies, Ltd
  */
 
-#define _GNU_SOURCE
-
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c
index 91f3d474..a14d1e49 100644
--- a/drivers/net/mlx5/mlx5_stats.c
+++ b/drivers/net/mlx5/mlx5_stats.c
@@ -17,14 +17,6 @@
 #include "mlx5_rxtx.h"
 #include "mlx5_defs.h"
 
-struct mlx5_counter_ctrl {
-	/* Name of the counter. */
-	char dpdk_name[RTE_ETH_XSTATS_NAME_SIZE];
-	/* Name of the counter on the device table. */
-	char ctr_name[RTE_ETH_XSTATS_NAME_SIZE];
-	uint32_t ib:1; /**< Nonzero for IB counters. */
-};
-
 static const struct mlx5_counter_ctrl mlx5_counters_init[] = {
 	{
 		.dpdk_name = "rx_port_unicast_bytes",
@@ -115,6 +107,23 @@ static const struct mlx5_counter_ctrl mlx5_counters_init[] = {
 		.dpdk_name = "rx_bytes_phy",
 		.ctr_name = "rx_bytes_phy",
 	},
+	/* Representor only */
+	{
+		.dpdk_name = "rx_packets",
+		.ctr_name = "vport_rx_packets",
+	},
+	{
+		.dpdk_name = "rx_bytes",
+		.ctr_name = "vport_rx_bytes",
+	},
+	{
+		.dpdk_name = "tx_packets",
+		.ctr_name = "vport_tx_packets",
+	},
+	{
+		.dpdk_name = "tx_bytes",
+		.ctr_name = "vport_tx_bytes",
+	},
 };
 
 static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
@@ -146,19 +155,19 @@ mlx5_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
 	et_stats->cmd = ETHTOOL_GSTATS;
 	et_stats->n_stats = xstats_ctrl->stats_n;
 	ifr.ifr_data = (caddr_t)et_stats;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 	if (ret) {
 		DRV_LOG(WARNING,
 			"port %u unable to read statistic values from device",
 			dev->data->port_id);
 		return ret;
 	}
-	for (i = 0; i != xstats_n; ++i) {
-		if (mlx5_counters_init[i].ib) {
+	for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) {
+		if (xstats_ctrl->info[i].ib) {
 			FILE *file;
 			MKSTR(path, "%s/ports/1/hw_counters/%s",
 			      priv->ibdev_path,
-			      mlx5_counters_init[i].ctr_name);
+			      xstats_ctrl->info[i].ctr_name);
 
 			file = fopen(path, "rb");
 			if (file) {
@@ -194,7 +203,7 @@ mlx5_ethtool_get_stats_n(struct rte_eth_dev *dev) {
 
 	drvinfo.cmd = ETHTOOL_GDRVINFO;
 	ifr.ifr_data = (caddr_t)&drvinfo;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 	if (ret) {
 		DRV_LOG(WARNING, "port %u unable to query number of statistics",
 			dev->data->port_id);
@@ -222,6 +231,8 @@ mlx5_xstats_init(struct rte_eth_dev *dev)
 	unsigned int str_sz;
 	int ret;
 
+	/* So that it won't aggregate for each init. */
+	xstats_ctrl->mlx5_stats_n = 0;
 	ret = mlx5_ethtool_get_stats_n(dev);
 	if (ret < 0) {
 		DRV_LOG(WARNING, "port %u no extended statistics available",
@@ -229,7 +240,6 @@ mlx5_xstats_init(struct rte_eth_dev *dev)
 		return;
 	}
 	dev_stats_n = ret;
-	xstats_ctrl->stats_n = dev_stats_n;
 	/* Allocate memory to grab stat names and values. */
 	str_sz = dev_stats_n * ETH_GSTRING_LEN;
 	strings = (struct ethtool_gstrings *)
@@ -244,14 +254,12 @@ mlx5_xstats_init(struct rte_eth_dev *dev)
 	strings->string_set = ETH_SS_STATS;
 	strings->len = dev_stats_n;
 	ifr.ifr_data = (caddr_t)strings;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 	if (ret) {
 		DRV_LOG(WARNING, "port %u unable to get statistic names",
 			dev->data->port_id);
 		goto free;
 	}
-	for (j = 0; j != xstats_n; ++j)
-		xstats_ctrl->dev_table_idx[j] = dev_stats_n;
 	for (i = 0; i != dev_stats_n; ++i) {
 		const char *curr_string = (const char *)
 			&strings->data[i * ETH_GSTRING_LEN];
@@ -259,24 +267,25 @@ mlx5_xstats_init(struct rte_eth_dev *dev)
 		for (j = 0; j != xstats_n; ++j) {
 			if (!strcmp(mlx5_counters_init[j].ctr_name,
 				    curr_string)) {
-				xstats_ctrl->dev_table_idx[j] = i;
+				unsigned int idx = xstats_ctrl->mlx5_stats_n++;
+
+				xstats_ctrl->dev_table_idx[idx] = i;
+				xstats_ctrl->info[idx] = mlx5_counters_init[j];
 				break;
 			}
 		}
 	}
-	for (j = 0; j != xstats_n; ++j) {
-		if (mlx5_counters_init[j].ib)
-			continue;
-		if (xstats_ctrl->dev_table_idx[j] >= dev_stats_n) {
-			DRV_LOG(WARNING,
-				"port %u counter \"%s\" is not recognized",
-				dev->data->port_id,
-				mlx5_counters_init[j].dpdk_name);
-			goto free;
+	/* Add IB counters. */
+	for (i = 0; i != xstats_n; ++i) {
+		if (mlx5_counters_init[i].ib) {
+			unsigned int idx = xstats_ctrl->mlx5_stats_n++;
+
+			xstats_ctrl->info[idx] = mlx5_counters_init[i];
 		}
 	}
+	assert(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS);
+	xstats_ctrl->stats_n = dev_stats_n;
 	/* Copy to base at first time. */
-	assert(xstats_n <= MLX5_MAX_XSTATS);
 	ret = mlx5_read_dev_counters(dev, xstats_ctrl->base);
 	if (ret)
 		DRV_LOG(ERR, "port %u cannot read device counters: %s",
@@ -306,9 +315,10 @@ mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
 	struct priv *priv = dev->data->dev_private;
 	unsigned int i;
 	uint64_t counters[n];
+	struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
+	uint16_t mlx5_stats_n = xstats_ctrl->mlx5_stats_n;
 
-	if (n >= xstats_n && stats) {
-		struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
+	if (n >= mlx5_stats_n && stats) {
 		int stats_n;
 		int ret;
 
@@ -320,12 +330,12 @@ mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
 		ret = mlx5_read_dev_counters(dev, counters);
 		if (ret)
 			return ret;
-		for (i = 0; i != xstats_n; ++i) {
+		for (i = 0; i != mlx5_stats_n; ++i) {
 			stats[i].id = i;
 			stats[i].value = (counters[i] - xstats_ctrl->base[i]);
 		}
 	}
-	return xstats_n;
+	return mlx5_stats_n;
 }
 
 /**
@@ -441,7 +451,7 @@ mlx5_xstats_reset(struct rte_eth_dev *dev)
 	struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
 	int stats_n;
 	unsigned int i;
-	unsigned int n = xstats_n;
+	unsigned int n = xstats_ctrl->mlx5_stats_n;
 	uint64_t counters[n];
 	int ret;
 
@@ -481,14 +491,17 @@ mlx5_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
 		      struct rte_eth_xstat_name *xstats_names, unsigned int n)
 {
 	unsigned int i;
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
+	unsigned int mlx5_xstats_n = xstats_ctrl->mlx5_stats_n;
 
-	if (n >= xstats_n && xstats_names) {
-		for (i = 0; i != xstats_n; ++i) {
+	if (n >= mlx5_xstats_n && xstats_names) {
+		for (i = 0; i != mlx5_xstats_n; ++i) {
 			strncpy(xstats_names[i].name,
-				mlx5_counters_init[i].dpdk_name,
+				xstats_ctrl->info[i].dpdk_name,
 				RTE_ETH_XSTATS_NAME_SIZE);
 			xstats_names[i].name[RTE_ETH_XSTATS_NAME_SIZE - 1] = 0;
 		}
 	}
-	return xstats_n;
+	return mlx5_xstats_n;
 }
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index f9bc4739..b01bd675 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -120,7 +120,6 @@ mlx5_get_tx_port_offloads(struct rte_eth_dev *dev)
 			offloads |= (DEV_TX_OFFLOAD_IP_TNL_TSO |
 				     DEV_TX_OFFLOAD_UDP_TNL_TSO);
 	}
-
 	if (config->tunnel_en) {
 		if (config->hw_csum)
 			offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
@@ -128,6 +127,10 @@ mlx5_get_tx_port_offloads(struct rte_eth_dev *dev)
 			offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
 				     DEV_TX_OFFLOAD_GRE_TNL_TSO);
 	}
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	if (config->dv_flow_en)
+		offloads |= DEV_TX_OFFLOAD_MATCH_METADATA;
+#endif
 	return offloads;
 }
 
diff --git a/drivers/net/mvneta/Makefile b/drivers/net/mvneta/Makefile
new file mode 100644
index 00000000..05a0487c
--- /dev/null
+++ b/drivers/net/mvneta/Makefile
@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Marvell International Ltd.
+# Copyright(c) 2018 Semihalf.
+# All rights reserved.
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(MAKECMDGOALS),config)
+ifeq ($(LIBMUSDK_PATH),)
+$(error "Please define LIBMUSDK_PATH environment variable")
+endif
+endif
+endif
+
+# library name
+LIB = librte_pmd_mvneta.a
+
+# library version
+LIBABIVER := 1
+
+# versioning export map
+EXPORT_MAP := rte_pmd_mvneta_version.map
+
+# external library dependencies
+CFLAGS += -I$(RTE_SDK)/drivers/common/mvep
+CFLAGS += -I$(LIBMUSDK_PATH)/include
+CFLAGS += -DMVCONF_TYPES_PUBLIC
+CFLAGS += -DMVCONF_DMA_PHYS_ADDR_T_PUBLIC
+CFLAGS += -DMVCONF_DMA_PHYS_ADDR_T_SIZE=64
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -O3
+LDLIBS += -L$(LIBMUSDK_PATH)/lib
+LDLIBS += -lmusdk
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_cfgfile
+LDLIBS += -lrte_bus_vdev -lrte_common_mvep
+
+# library source files
+SRCS-$(CONFIG_RTE_LIBRTE_MVNETA_PMD) += mvneta_ethdev.c mvneta_rxtx.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/mvneta/meson.build b/drivers/net/mvneta/meson.build
new file mode 100644
index 00000000..c0b1bce0
--- /dev/null
+++ b/drivers/net/mvneta/meson.build
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Marvell International Ltd.
+# Copyright(c) 2018 Semihalf.
+# All rights reserved.
+
+path = get_option('lib_musdk_dir')
+lib_dir = path + '/lib'
+inc_dir = path + '/include'
+
+lib = cc.find_library('libmusdk', dirs : [lib_dir], required: false)
+if not lib.found()
+	build = false
+else
+	ext_deps += lib
+	includes += include_directories(inc_dir)
+	cflags += [
+	  '-DMVCONF_TYPES_PUBLIC',
+	  '-DMVCONF_DMA_PHYS_ADDR_T_PUBLIC',
+	  '-DMVCONF_DMA_PHYS_ADDR_T_SIZE=64'
+	]
+endif
+
+sources = files(
+	'mvneta_ethdev.c',
+	'mvneta_rxtx.c'
+)
+
+deps += ['cfgfile', 'common_mvep']
diff --git a/drivers/net/mvneta/mvneta_ethdev.c b/drivers/net/mvneta/mvneta_ethdev.c
new file mode 100644
index 00000000..2d766645
--- /dev/null
+++ b/drivers/net/mvneta/mvneta_ethdev.c
@@ -0,0 +1,987 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Marvell International Ltd.
+ * Copyright(c) 2018 Semihalf.
+ * All rights reserved.
+ */
+
+#include <rte_ethdev_driver.h>
+#include <rte_kvargs.h>
+#include <rte_bus_vdev.h>
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <linux/ethtool.h>
+#include <linux/sockios.h>
+#include <net/if.h>
+#include <net/if_arp.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <rte_mvep_common.h>
+
+#include "mvneta_rxtx.h"
+
+
+#define MVNETA_IFACE_NAME_ARG "iface"
+
+#define MVNETA_RX_OFFLOADS (DEV_RX_OFFLOAD_JUMBO_FRAME | \
+			  DEV_RX_OFFLOAD_CHECKSUM)
+
+/** Port Tx offloads capabilities */
+#define MVNETA_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
+			  DEV_TX_OFFLOAD_UDP_CKSUM | \
+			  DEV_TX_OFFLOAD_TCP_CKSUM | \
+			  DEV_TX_OFFLOAD_MULTI_SEGS)
+
+#define MVNETA_PKT_SIZE_MAX (16382 - MV_MH_SIZE) /* 9700B */
+#define MVNETA_DEFAULT_MTU 1500
+
+#define MVNETA_MAC_ADDRS_MAX 256 /*16 UC, 256 IP, 256 MC/BC */
+/** Maximum length of a match string */
+#define MVNETA_MATCH_LEN 16
+
+int mvneta_logtype;
+
+static const char * const valid_args[] = {
+	MVNETA_IFACE_NAME_ARG,
+	NULL
+};
+
+struct mvneta_ifnames {
+	const char *names[NETA_NUM_ETH_PPIO];
+	int idx;
+};
+
+static int mvneta_dev_num;
+
+/**
+ * Deinitialize packet processor.
+ */
+static void
+mvneta_neta_deinit(void)
+{
+	neta_deinit();
+}
+
+/**
+ * Initialize packet processor.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static int
+mvneta_neta_init(void)
+{
+	return neta_init();
+}
+
+/**
+ * Callback used by rte_kvargs_process() during argument parsing.
+ *
+ * @param key
+ *   Pointer to the parsed key (unused).
+ * @param value
+ *   Pointer to the parsed value.
+ * @param extra_args
+ *   Pointer to the extra arguments which contains address of the
+ *   table of pointers to parsed interface names.
+ *
+ * @return
+ *   Always 0.
+ */
+static int
+mvneta_ifnames_get(const char *key __rte_unused, const char *value,
+		 void *extra_args)
+{
+	struct mvneta_ifnames *ifnames = extra_args;
+
+	ifnames->names[ifnames->idx++] = value;
+
+	return 0;
+}
+
+/**
+ * Ethernet device configuration.
+ *
+ * Prepare the driver for a given number of TX and RX queues and
+ * configure RSS if supported.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static int
+mvneta_dev_configure(struct rte_eth_dev *dev)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	struct neta_ppio_params *ppio_params;
+
+	if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE) {
+		MVNETA_LOG(INFO, "Unsupported RSS and rx multi queue mode %d",
+			dev->data->dev_conf.rxmode.mq_mode);
+		if (dev->data->nb_rx_queues > 1)
+			return -EINVAL;
+	}
+
+	if (dev->data->dev_conf.rxmode.split_hdr_size) {
+		MVNETA_LOG(INFO, "Split headers not supported");
+		return -EINVAL;
+	}
+
+	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
+		dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
+				 MRVL_NETA_ETH_HDRS_LEN;
+
+	if (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
+		priv->multiseg = 1;
+
+	ppio_params = &priv->ppio_params;
+	ppio_params->outqs_params.num_outqs = dev->data->nb_tx_queues;
+	/* Default: 1 TC, no QoS supported. */
+	ppio_params->inqs_params.num_tcs = 1;
+	ppio_params->inqs_params.tcs_params[0].pkt_offset = MRVL_NETA_PKT_OFFS;
+	priv->ppio_id = dev->data->port_id;
+
+	return 0;
+}
+
+/**
+ * DPDK callback to get information about the device.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure (unused).
+ * @param info
+ *   Info structure output buffer.
+ */
+static void
+mvneta_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
+		   struct rte_eth_dev_info *info)
+{
+	info->speed_capa = ETH_LINK_SPEED_10M |
+			   ETH_LINK_SPEED_100M |
+			   ETH_LINK_SPEED_1G |
+			   ETH_LINK_SPEED_2_5G;
+
+	info->max_rx_queues = MRVL_NETA_RXQ_MAX;
+	info->max_tx_queues = MRVL_NETA_TXQ_MAX;
+	info->max_mac_addrs = MVNETA_MAC_ADDRS_MAX;
+
+	info->rx_desc_lim.nb_max = MRVL_NETA_RXD_MAX;
+	info->rx_desc_lim.nb_min = MRVL_NETA_RXD_MIN;
+	info->rx_desc_lim.nb_align = MRVL_NETA_RXD_ALIGN;
+
+	info->tx_desc_lim.nb_max = MRVL_NETA_TXD_MAX;
+	info->tx_desc_lim.nb_min = MRVL_NETA_TXD_MIN;
+	info->tx_desc_lim.nb_align = MRVL_NETA_TXD_ALIGN;
+
+	info->rx_offload_capa = MVNETA_RX_OFFLOADS;
+	info->rx_queue_offload_capa = MVNETA_RX_OFFLOADS;
+
+	info->tx_offload_capa =  MVNETA_TX_OFFLOADS;
+	info->tx_queue_offload_capa =  MVNETA_TX_OFFLOADS;
+
+	/* By default packets are dropped if no descriptors are available */
+	info->default_rxconf.rx_drop_en = 1;
+	/* Deferred tx queue start is not supported */
+	info->default_txconf.tx_deferred_start = 0;
+	info->default_txconf.offloads = 0;
+
+	info->max_rx_pktlen = MVNETA_PKT_SIZE_MAX;
+}
+
+/**
+ * Return supported packet types.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure (unused).
+ *
+ * @return
+ *   Const pointer to the table with supported packet types.
+ */
+static const uint32_t *
+mvneta_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
+{
+	static const uint32_t ptypes[] = {
+		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_L2_ETHER_VLAN,
+		RTE_PTYPE_L3_IPV4,
+		RTE_PTYPE_L3_IPV6,
+		RTE_PTYPE_L4_TCP,
+		RTE_PTYPE_L4_UDP
+	};
+
+	return ptypes;
+}
+
+/**
+ * DPDK callback to change the MTU.
+ *
+ * Setting the MTU affects hardware MRU (packets larger than the MRU
+ * will be dropped).
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param mtu
+ *   New MTU.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static int
+mvneta_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	uint16_t mbuf_data_size = 0; /* SW buffer size */
+	uint16_t mru;
+	int ret;
+
+	mru = MRVL_NETA_MTU_TO_MRU(mtu);
+	/*
+	 * min_rx_buf_size is equal to mbuf data size
+	 * if pmd didn't set it differently
+	 */
+	mbuf_data_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
+	/* Prevent PMD from:
+	 * - setting mru greater than the mbuf size resulting in
+	 * hw and sw buffer size mismatch
+	 * - setting mtu that requires the support of scattered packets
+	 * when this feature has not been enabled/supported so far.
+	 */
+	if (!dev->data->scattered_rx &&
+	    (mru + MRVL_NETA_PKT_OFFS > mbuf_data_size)) {
+		mru = mbuf_data_size - MRVL_NETA_PKT_OFFS;
+		mtu = MRVL_NETA_MRU_TO_MTU(mru);
+		MVNETA_LOG(WARNING, "MTU too big, max MTU possible limitted by"
+			" current mbuf size: %u. Set MTU to %u, MRU to %u",
+			mbuf_data_size, mtu, mru);
+	}
+
+	if (mtu < ETHER_MIN_MTU || mru > MVNETA_PKT_SIZE_MAX) {
+		MVNETA_LOG(ERR, "Invalid MTU [%u] or MRU [%u]", mtu, mru);
+		return -EINVAL;
+	}
+
+	dev->data->mtu = mtu;
+	dev->data->dev_conf.rxmode.max_rx_pkt_len = mru - MV_MH_SIZE;
+
+	if (!priv->ppio)
+		/* It is OK. New MTU will be set later on mvneta_dev_start */
+		return 0;
+
+	ret = neta_ppio_set_mru(priv->ppio, mru);
+	if (ret) {
+		MVNETA_LOG(ERR, "Failed to change MRU");
+		return ret;
+	}
+
+	ret = neta_ppio_set_mtu(priv->ppio, mtu);
+	if (ret) {
+		MVNETA_LOG(ERR, "Failed to change MTU");
+		return ret;
+	}
+	MVNETA_LOG(INFO, "MTU changed to %u, MRU = %u", mtu, mru);
+
+	return 0;
+}
+
+/**
+ * DPDK callback to bring the link up.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static int
+mvneta_dev_set_link_up(struct rte_eth_dev *dev)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+
+	if (!priv->ppio)
+		return 0;
+
+	return neta_ppio_enable(priv->ppio);
+}
+
+/**
+ * DPDK callback to bring the link down.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static int
+mvneta_dev_set_link_down(struct rte_eth_dev *dev)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+
+	if (!priv->ppio)
+		return 0;
+
+	return neta_ppio_disable(priv->ppio);
+}
+
+/**
+ * DPDK callback to start the device.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ *
+ * @return
+ *   0 on success, negative errno value on failure.
+ */
+static int
+mvneta_dev_start(struct rte_eth_dev *dev)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	char match[MVNETA_MATCH_LEN];
+	int ret = 0, i;
+
+	if (priv->ppio)
+		return mvneta_dev_set_link_up(dev);
+
+	snprintf(match, sizeof(match), "%s", dev->data->name);
+	priv->ppio_params.match = match;
+	priv->ppio_params.inqs_params.mtu = dev->data->mtu;
+
+	ret = neta_ppio_init(&priv->ppio_params, &priv->ppio);
+	if (ret) {
+		MVNETA_LOG(ERR, "Failed to init ppio");
+		return ret;
+	}
+	priv->ppio_id = priv->ppio->port_id;
+
+	/*
+	 * In case there are some some stale uc/mc mac addresses flush them
+	 * here. It cannot be done during mvneta_dev_close() as port information
+	 * is already gone at that point (due to neta_ppio_deinit() in
+	 * mvneta_dev_stop()).
+	 */
+	if (!priv->uc_mc_flushed) {
+		ret = neta_ppio_flush_mac_addrs(priv->ppio, 0, 1);
+		if (ret) {
+			MVNETA_LOG(ERR,
+				"Failed to flush uc/mc filter list");
+			goto out;
+		}
+		priv->uc_mc_flushed = 1;
+	}
+
+	ret = mvneta_alloc_rx_bufs(dev);
+	if (ret)
+		goto out;
+
+	ret = mvneta_mtu_set(dev, dev->data->mtu);
+	if (ret) {
+		MVNETA_LOG(ERR, "Failed to set MTU %d", dev->data->mtu);
+		goto out;
+	}
+
+	ret = mvneta_dev_set_link_up(dev);
+	if (ret) {
+		MVNETA_LOG(ERR, "Failed to set link up");
+		goto out;
+	}
+
+	/* start tx queues */
+	for (i = 0; i < dev->data->nb_tx_queues; i++)
+		dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
+
+	mvneta_set_tx_function(dev);
+
+	return 0;
+
+out:
+	MVNETA_LOG(ERR, "Failed to start device");
+	neta_ppio_deinit(priv->ppio);
+	return ret;
+}
+
+/**
+ * DPDK callback to stop the device.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+static void
+mvneta_dev_stop(struct rte_eth_dev *dev)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+
+	if (!priv->ppio)
+		return;
+
+	mvneta_dev_set_link_down(dev);
+	mvneta_flush_queues(dev);
+	neta_ppio_deinit(priv->ppio);
+
+	priv->ppio = NULL;
+}
+
+/**
+ * DPDK callback to close the device.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+static void
+mvneta_dev_close(struct rte_eth_dev *dev)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	int i;
+
+	if (priv->ppio)
+		mvneta_dev_stop(dev);
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		mvneta_rx_queue_release(dev->data->rx_queues[i]);
+		dev->data->rx_queues[i] = NULL;
+	}
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		mvneta_tx_queue_release(dev->data->tx_queues[i]);
+		dev->data->tx_queues[i] = NULL;
+	}
+}
+
+/**
+ * DPDK callback to retrieve physical link information.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param wait_to_complete
+ *   Wait for request completion (ignored).
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static int
+mvneta_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
+{
+	/*
+	 * TODO
+	 * once MUSDK provides necessary API use it here
+	 */
+	struct mvneta_priv *priv = dev->data->dev_private;
+	struct ethtool_cmd edata;
+	struct ifreq req;
+	int ret, fd, link_up;
+
+	if (!priv->ppio)
+		return -EPERM;
+
+	edata.cmd = ETHTOOL_GSET;
+
+	strcpy(req.ifr_name, dev->data->name);
+	req.ifr_data = (void *)&edata;
+
+	fd = socket(AF_INET, SOCK_DGRAM, 0);
+	if (fd == -1)
+		return -EFAULT;
+	ret = ioctl(fd, SIOCETHTOOL, &req);
+	if (ret == -1) {
+		close(fd);
+		return -EFAULT;
+	}
+
+	close(fd);
+
+	switch (ethtool_cmd_speed(&edata)) {
+	case SPEED_10:
+		dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
+		break;
+	case SPEED_100:
+		dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
+		break;
+	case SPEED_1000:
+		dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
+		break;
+	case SPEED_2500:
+		dev->data->dev_link.link_speed = ETH_SPEED_NUM_2_5G;
+		break;
+	default:
+		dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
+	}
+
+	dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
+							 ETH_LINK_HALF_DUPLEX;
+	dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
+							   ETH_LINK_FIXED;
+
+	neta_ppio_get_link_state(priv->ppio, &link_up);
+	dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
+
+	return 0;
+}
+
+/**
+ * DPDK callback to enable promiscuous mode.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+static void
+mvneta_promiscuous_enable(struct rte_eth_dev *dev)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	int ret, en;
+
+	if (!priv->ppio)
+		return;
+
+	neta_ppio_get_promisc(priv->ppio, &en);
+	if (en) {
+		MVNETA_LOG(INFO, "Promiscuous already enabled");
+		return;
+	}
+
+	ret = neta_ppio_set_promisc(priv->ppio, 1);
+	if (ret)
+		MVNETA_LOG(ERR, "Failed to enable promiscuous mode");
+}
+
+/**
+ * DPDK callback to disable allmulticast mode.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+static void
+mvneta_promiscuous_disable(struct rte_eth_dev *dev)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	int ret, en;
+
+	if (!priv->ppio)
+		return;
+
+	neta_ppio_get_promisc(priv->ppio, &en);
+	if (!en) {
+		MVNETA_LOG(INFO, "Promiscuous already disabled");
+		return;
+	}
+
+	ret = neta_ppio_set_promisc(priv->ppio, 0);
+	if (ret)
+		MVNETA_LOG(ERR, "Failed to disable promiscuous mode");
+}
+
+/**
+ * DPDK callback to remove a MAC address.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param index
+ *   MAC address index.
+ */
+static void
+mvneta_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	char buf[ETHER_ADDR_FMT_SIZE];
+	int ret;
+
+	if (!priv->ppio)
+		return;
+
+	ret = neta_ppio_remove_mac_addr(priv->ppio,
+				       dev->data->mac_addrs[index].addr_bytes);
+	if (ret) {
+		ether_format_addr(buf, sizeof(buf),
+				  &dev->data->mac_addrs[index]);
+		MVNETA_LOG(ERR, "Failed to remove mac %s", buf);
+	}
+}
+
+/**
+ * DPDK callback to add a MAC address.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param mac_addr
+ *   MAC address to register.
+ * @param index
+ *   MAC address index.
+ * @param vmdq
+ *   VMDq pool index to associate address with (unused).
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static int
+mvneta_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
+		  uint32_t index, uint32_t vmdq __rte_unused)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	char buf[ETHER_ADDR_FMT_SIZE];
+	int ret;
+
+	if (index == 0)
+		/* For setting index 0, mrvl_mac_addr_set() should be used.*/
+		return -1;
+
+	if (!priv->ppio)
+		return 0;
+
+	ret = neta_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
+	if (ret) {
+		ether_format_addr(buf, sizeof(buf), mac_addr);
+		MVNETA_LOG(ERR, "Failed to add mac %s", buf);
+		return -1;
+	}
+
+	return 0;
+}
+
+/**
+ * DPDK callback to set the primary MAC address.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param mac_addr
+ *   MAC address to register.
+ */
+static int
+mvneta_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	int ret;
+
+	if (!priv->ppio)
+		return -EINVAL;
+
+	ret = neta_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
+	if (ret) {
+		char buf[ETHER_ADDR_FMT_SIZE];
+		ether_format_addr(buf, sizeof(buf), mac_addr);
+		MVNETA_LOG(ERR, "Failed to set mac to %s", buf);
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to get device statistics.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param stats
+ *   Stats structure output buffer.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static int
+mvneta_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	struct neta_ppio_statistics ppio_stats;
+	unsigned int ret;
+
+	if (!priv->ppio)
+		return -EPERM;
+
+	ret = neta_ppio_get_statistics(priv->ppio, &ppio_stats);
+	if (unlikely(ret)) {
+		MVNETA_LOG(ERR, "Failed to update port statistics");
+		return ret;
+	}
+
+	stats->ipackets += ppio_stats.rx_packets +
+			ppio_stats.rx_broadcast_packets +
+			ppio_stats.rx_multicast_packets -
+			priv->prev_stats.ipackets;
+	stats->opackets += ppio_stats.tx_packets +
+			ppio_stats.tx_broadcast_packets +
+			ppio_stats.tx_multicast_packets -
+			priv->prev_stats.opackets;
+	stats->ibytes += ppio_stats.rx_bytes - priv->prev_stats.ibytes;
+	stats->obytes += ppio_stats.tx_bytes - priv->prev_stats.obytes;
+	stats->imissed += ppio_stats.rx_discard +
+			  ppio_stats.rx_overrun -
+			  priv->prev_stats.imissed;
+
+	stats->ierrors = ppio_stats.rx_packets_err +
+			ppio_stats.rx_errors +
+			ppio_stats.rx_crc_error -
+			priv->prev_stats.ierrors;
+	stats->oerrors = ppio_stats.tx_errors - priv->prev_stats.oerrors;
+
+	return 0;
+}
+
+/**
+ * DPDK callback to clear device statistics.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+static void
+mvneta_stats_reset(struct rte_eth_dev *dev)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	unsigned int ret;
+
+	if (!priv->ppio)
+		return;
+
+	ret = mvneta_stats_get(dev, &priv->prev_stats);
+	if (unlikely(ret))
+		RTE_LOG(ERR, PMD, "Failed to reset port statistics");
+}
+
+
+static const struct eth_dev_ops mvneta_ops = {
+	.dev_configure = mvneta_dev_configure,
+	.dev_start = mvneta_dev_start,
+	.dev_stop = mvneta_dev_stop,
+	.dev_set_link_up = mvneta_dev_set_link_up,
+	.dev_set_link_down = mvneta_dev_set_link_down,
+	.dev_close = mvneta_dev_close,
+	.link_update = mvneta_link_update,
+	.promiscuous_enable = mvneta_promiscuous_enable,
+	.promiscuous_disable = mvneta_promiscuous_disable,
+	.mac_addr_remove = mvneta_mac_addr_remove,
+	.mac_addr_add = mvneta_mac_addr_add,
+	.mac_addr_set = mvneta_mac_addr_set,
+	.mtu_set = mvneta_mtu_set,
+	.stats_get = mvneta_stats_get,
+	.stats_reset = mvneta_stats_reset,
+	.dev_infos_get = mvneta_dev_infos_get,
+	.dev_supported_ptypes_get = mvneta_dev_supported_ptypes_get,
+	.rxq_info_get = mvneta_rxq_info_get,
+	.txq_info_get = mvneta_txq_info_get,
+	.rx_queue_setup = mvneta_rx_queue_setup,
+	.rx_queue_release = mvneta_rx_queue_release,
+	.tx_queue_setup = mvneta_tx_queue_setup,
+	.tx_queue_release = mvneta_tx_queue_release,
+};
+
+/**
+ * Create device representing Ethernet port.
+ *
+ * @param name
+ *   Pointer to the port's name.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static int
+mvneta_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
+{
+	int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
+	struct rte_eth_dev *eth_dev;
+	struct mvneta_priv *priv;
+	struct ifreq req;
+
+	eth_dev = rte_eth_dev_allocate(name);
+	if (!eth_dev)
+		return -ENOMEM;
+
+	priv = rte_zmalloc_socket(name, sizeof(*priv), 0, rte_socket_id());
+	if (!priv) {
+		ret = -ENOMEM;
+		goto out_free;
+	}
+	eth_dev->data->dev_private = priv;
+
+	eth_dev->data->mac_addrs =
+		rte_zmalloc("mac_addrs",
+			    ETHER_ADDR_LEN * MVNETA_MAC_ADDRS_MAX, 0);
+	if (!eth_dev->data->mac_addrs) {
+		MVNETA_LOG(ERR, "Failed to allocate space for eth addrs");
+		ret = -ENOMEM;
+		goto out_free;
+	}
+
+	memset(&req, 0, sizeof(req));
+	strcpy(req.ifr_name, name);
+	ret = ioctl(fd, SIOCGIFHWADDR, &req);
+	if (ret)
+		goto out_free;
+
+	memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
+	       req.ifr_addr.sa_data, ETHER_ADDR_LEN);
+
+	eth_dev->data->kdrv = RTE_KDRV_NONE;
+	eth_dev->device = &vdev->device;
+	eth_dev->rx_pkt_burst = mvneta_rx_pkt_burst;
+	mvneta_set_tx_function(eth_dev);
+	eth_dev->dev_ops = &mvneta_ops;
+
+	rte_eth_dev_probing_finish(eth_dev);
+	return 0;
+out_free:
+	rte_eth_dev_release_port(eth_dev);
+
+	return ret;
+}
+
+/**
+ * Cleanup previously created device representing Ethernet port.
+ *
+ * @param eth_dev
+ *   Pointer to the corresponding rte_eth_dev structure.
+ */
+static void
+mvneta_eth_dev_destroy(struct rte_eth_dev *eth_dev)
+{
+	rte_eth_dev_release_port(eth_dev);
+}
+
+/**
+ * Cleanup previously created device representing Ethernet port.
+ *
+ * @param name
+ *   Pointer to the port name.
+ */
+static void
+mvneta_eth_dev_destroy_name(const char *name)
+{
+	struct rte_eth_dev *eth_dev;
+
+	eth_dev = rte_eth_dev_allocated(name);
+	if (!eth_dev)
+		return;
+
+	mvneta_eth_dev_destroy(eth_dev);
+}
+
+/**
+ * DPDK callback to register the virtual device.
+ *
+ * @param vdev
+ *   Pointer to the virtual device.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static int
+rte_pmd_mvneta_probe(struct rte_vdev_device *vdev)
+{
+	struct rte_kvargs *kvlist;
+	struct mvneta_ifnames ifnames;
+	int ret = -EINVAL;
+	uint32_t i, ifnum;
+	const char *params;
+
+	params = rte_vdev_device_args(vdev);
+	if (!params)
+		return -EINVAL;
+
+	kvlist = rte_kvargs_parse(params, valid_args);
+	if (!kvlist)
+		return -EINVAL;
+
+	ifnum = rte_kvargs_count(kvlist, MVNETA_IFACE_NAME_ARG);
+	if (ifnum > RTE_DIM(ifnames.names))
+		goto out_free_kvlist;
+
+	ifnames.idx = 0;
+	rte_kvargs_process(kvlist, MVNETA_IFACE_NAME_ARG,
+			   mvneta_ifnames_get, &ifnames);
+
+	/*
+	 * The below system initialization should be done only once,
+	 * on the first provided configuration file
+	 */
+	if (mvneta_dev_num)
+		goto init_devices;
+
+	MVNETA_LOG(INFO, "Perform MUSDK initializations");
+
+	ret = rte_mvep_init(MVEP_MOD_T_NETA, kvlist);
+	if (ret)
+		goto out_free_kvlist;
+
+	ret = mvneta_neta_init();
+	if (ret) {
+		MVNETA_LOG(ERR, "Failed to init NETA!");
+		rte_mvep_deinit(MVEP_MOD_T_NETA);
+		goto out_free_kvlist;
+	}
+
+init_devices:
+	for (i = 0; i < ifnum; i++) {
+		MVNETA_LOG(INFO, "Creating %s", ifnames.names[i]);
+		ret = mvneta_eth_dev_create(vdev, ifnames.names[i]);
+		if (ret)
+			goto out_cleanup;
+	}
+	mvneta_dev_num += ifnum;
+
+	rte_kvargs_free(kvlist);
+
+	return 0;
+out_cleanup:
+	for (; i > 0; i--)
+		mvneta_eth_dev_destroy_name(ifnames.names[i]);
+
+	if (mvneta_dev_num == 0) {
+		mvneta_neta_deinit();
+		rte_mvep_deinit(MVEP_MOD_T_NETA);
+	}
+out_free_kvlist:
+	rte_kvargs_free(kvlist);
+
+	return ret;
+}
+
+/**
+ * DPDK callback to remove virtual device.
+ *
+ * @param vdev
+ *   Pointer to the removed virtual device.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static int
+rte_pmd_mvneta_remove(struct rte_vdev_device *vdev)
+{
+	int i;
+	const char *name;
+
+	name = rte_vdev_device_name(vdev);
+	if (!name)
+		return -EINVAL;
+
+	MVNETA_LOG(INFO, "Removing %s", name);
+
+	RTE_ETH_FOREACH_DEV(i) {
+		if (rte_eth_devices[i].device != &vdev->device)
+			continue;
+
+		mvneta_eth_dev_destroy(&rte_eth_devices[i]);
+		mvneta_dev_num--;
+	}
+
+	if (mvneta_dev_num == 0) {
+		MVNETA_LOG(INFO, "Perform MUSDK deinit");
+		mvneta_neta_deinit();
+		rte_mvep_deinit(MVEP_MOD_T_NETA);
+	}
+
+	return 0;
+}
+
+static struct rte_vdev_driver pmd_mvneta_drv = {
+	.probe = rte_pmd_mvneta_probe,
+	.remove = rte_pmd_mvneta_remove,
+};
+
+RTE_PMD_REGISTER_VDEV(net_mvneta, pmd_mvneta_drv);
+RTE_PMD_REGISTER_PARAM_STRING(net_mvneta, "iface=<ifc>");
+
+RTE_INIT(mvneta_init_log)
+{
+	mvneta_logtype = rte_log_register("pmd.net.mvneta");
+	if (mvneta_logtype >= 0)
+		rte_log_set_level(mvneta_logtype, RTE_LOG_NOTICE);
+}
diff --git a/drivers/net/mvneta/mvneta_ethdev.h b/drivers/net/mvneta/mvneta_ethdev.h
new file mode 100644
index 00000000..101b0a81
--- /dev/null
+++ b/drivers/net/mvneta/mvneta_ethdev.h
@@ -0,0 +1,79 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Marvell International Ltd.
+ * Copyright(c) 2018 Semihalf.
+ * All rights reserved.
+ */
+
+#ifndef _MVNETA_ETHDEV_H_
+#define _MVNETA_ETHDEV_H_
+
+#include <rte_ethdev.h>
+#include <rte_malloc.h>
+#include <rte_log.h>
+
+/*
+ * container_of is defined by both DPDK and MUSDK,
+ * we'll declare only one version.
+ *
+ * Note that it is not used in this PMD anyway.
+ */
+#ifdef container_of
+#undef container_of
+#endif
+
+#include <drivers/mv_neta.h>
+#include <drivers/mv_neta_ppio.h>
+
+/** Packet offset inside RX buffer. */
+#define MRVL_NETA_PKT_OFFS 64
+
+/** Maximum number of rx/tx queues per port */
+#define MRVL_NETA_RXQ_MAX 8
+#define MRVL_NETA_TXQ_MAX 8
+
+/** Minimum/maximum number of descriptors in tx queue */
+#define MRVL_NETA_TXD_MIN 16
+#define MRVL_NETA_TXD_MAX 2048
+
+/** Tx queue descriptors alignment in B */
+#define MRVL_NETA_TXD_ALIGN 32
+
+/** Minimum/maximum number of descriptors in rx queue */
+#define MRVL_NETA_RXD_MIN 16
+#define MRVL_NETA_RXD_MAX 2048
+
+/** Rx queue descriptors alignment in B */
+#define MRVL_NETA_RXD_ALIGN 32
+
+#define MRVL_NETA_VLAN_TAG_LEN		4
+#define MRVL_NETA_ETH_HDRS_LEN		(ETHER_HDR_LEN + ETHER_CRC_LEN + \
+					MRVL_NETA_VLAN_TAG_LEN)
+
+#define MRVL_NETA_HDRS_LEN		(MV_MH_SIZE + MRVL_NETA_ETH_HDRS_LEN)
+#define MRVL_NETA_MTU_TO_MRU(mtu)	((mtu) + MRVL_NETA_HDRS_LEN)
+#define MRVL_NETA_MRU_TO_MTU(mru)	((mru) - MRVL_NETA_HDRS_LEN)
+
+
+struct mvneta_priv {
+	/* Hot fields, used in fast path. */
+	struct neta_ppio	*ppio;    /**< Port handler pointer */
+
+	uint8_t pp_id;
+	uint8_t ppio_id;	/* ppio port id */
+	uint8_t uc_mc_flushed;
+	uint8_t multiseg;
+
+	struct neta_ppio_params ppio_params;
+
+	uint64_t rate_max;
+	struct rte_eth_stats prev_stats;
+};
+
+/** Current log type. */
+extern int mvneta_logtype;
+
+#define MVNETA_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, mvneta_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+#endif /* _MVNETA_ETHDEV_H_ */
diff --git a/drivers/net/mvneta/mvneta_rxtx.c b/drivers/net/mvneta/mvneta_rxtx.c
new file mode 100644
index 00000000..62caa684
--- /dev/null
+++ b/drivers/net/mvneta/mvneta_rxtx.c
@@ -0,0 +1,1030 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Marvell International Ltd.
+ * Copyright(c) 2018 Semihalf.
+ * All rights reserved.
+ */
+
+#include "mvneta_rxtx.h"
+
+#define MVNETA_PKT_EFFEC_OFFS (MRVL_NETA_PKT_OFFS + MV_MH_SIZE)
+
+#define MRVL_NETA_DEFAULT_TC 0
+
+/** Maximum number of descriptors in shadow queue. Must be power of 2 */
+#define MRVL_NETA_TX_SHADOWQ_SIZE MRVL_NETA_TXD_MAX
+
+/** Shadow queue size mask (since shadow queue size is power of 2) */
+#define MRVL_NETA_TX_SHADOWQ_MASK (MRVL_NETA_TX_SHADOWQ_SIZE - 1)
+
+/** Minimum number of sent buffers to release from shadow queue to BM */
+#define MRVL_NETA_BUF_RELEASE_BURST_SIZE_MIN	16
+
+/** Maximum number of sent buffers to release from shadow queue to BM */
+#define MRVL_NETA_BUF_RELEASE_BURST_SIZE_MAX	64
+
+#define MVNETA_COOKIE_ADDR_INVALID ~0ULL
+#define MVNETA_COOKIE_HIGH_ADDR_SHIFT	(sizeof(neta_cookie_t) * 8)
+#define MVNETA_COOKIE_HIGH_ADDR_MASK	(~0ULL << MVNETA_COOKIE_HIGH_ADDR_SHIFT)
+
+#define MVNETA_SET_COOKIE_HIGH_ADDR(addr) {				\
+	if (unlikely(cookie_addr_high == MVNETA_COOKIE_ADDR_INVALID))	\
+		cookie_addr_high =					\
+			(uint64_t)(addr) & MVNETA_COOKIE_HIGH_ADDR_MASK;\
+}
+
+#define MVNETA_CHECK_COOKIE_HIGH_ADDR(addr)		\
+	((likely(cookie_addr_high ==			\
+	((uint64_t)(addr) & MVNETA_COOKIE_HIGH_ADDR_MASK))) ? 1 : 0)
+
+struct mvneta_rxq {
+	struct mvneta_priv *priv;
+	struct rte_mempool *mp;
+	int queue_id;
+	int port_id;
+	int size;
+	int cksum_enabled;
+	uint64_t bytes_recv;
+	uint64_t drop_mac;
+	uint64_t pkts_processed;
+};
+
+/*
+ * To use buffer harvesting based on loopback port shadow queue structure
+ * was introduced for buffers information bookkeeping.
+ */
+struct mvneta_shadow_txq {
+	int head;           /* write index - used when sending buffers */
+	int tail;           /* read index - used when releasing buffers */
+	u16 size;           /* queue occupied size */
+	struct neta_buff_inf ent[MRVL_NETA_TX_SHADOWQ_SIZE]; /* q entries */
+};
+
+struct mvneta_txq {
+	struct mvneta_priv *priv;
+	int queue_id;
+	int port_id;
+	uint64_t bytes_sent;
+	struct mvneta_shadow_txq shadow_txq;
+	int tx_deferred_start;
+};
+
+static uint64_t cookie_addr_high = MVNETA_COOKIE_ADDR_INVALID;
+static uint16_t rx_desc_free_thresh = MRVL_NETA_BUF_RELEASE_BURST_SIZE_MIN;
+
+static inline int
+mvneta_buffs_refill(struct mvneta_priv *priv, struct mvneta_rxq *rxq, u16 *num)
+{
+	struct rte_mbuf *mbufs[MRVL_NETA_BUF_RELEASE_BURST_SIZE_MAX];
+	struct neta_buff_inf entries[MRVL_NETA_BUF_RELEASE_BURST_SIZE_MAX];
+	int i, ret;
+	uint16_t nb_desc = *num;
+
+	ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, nb_desc);
+	if (ret) {
+		MVNETA_LOG(ERR, "Failed to allocate %u mbufs.", nb_desc);
+		*num = 0;
+		return -1;
+	}
+
+	MVNETA_SET_COOKIE_HIGH_ADDR(mbufs[0]);
+
+	for (i = 0; i < nb_desc; i++) {
+		if (unlikely(!MVNETA_CHECK_COOKIE_HIGH_ADDR(mbufs[i]))) {
+			MVNETA_LOG(ERR,
+				"mbuf virt high addr 0x%lx out of range 0x%lx",
+				(uint64_t)mbufs[i] >> 32,
+				cookie_addr_high >> 32);
+			*num = 0;
+			goto out;
+		}
+		entries[i].addr = rte_mbuf_data_iova_default(mbufs[i]);
+		entries[i].cookie = (neta_cookie_t)(uint64_t)mbufs[i];
+	}
+	neta_ppio_inq_put_buffs(priv->ppio, rxq->queue_id, entries, num);
+
+out:
+	for (i = *num; i < nb_desc; i++)
+		rte_pktmbuf_free(mbufs[i]);
+
+	return 0;
+}
+
+/**
+ * Allocate buffers from mempool
+ * and store addresses in rx descriptors.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static inline int
+mvneta_buffs_alloc(struct mvneta_priv *priv, struct mvneta_rxq *rxq, int *num)
+{
+	uint16_t nb_desc, nb_desc_burst, sent = 0;
+	int ret = 0;
+
+	nb_desc = *num;
+
+	do {
+		nb_desc_burst =
+			(nb_desc < MRVL_NETA_BUF_RELEASE_BURST_SIZE_MAX) ?
+			nb_desc : MRVL_NETA_BUF_RELEASE_BURST_SIZE_MAX;
+
+		ret = mvneta_buffs_refill(priv, rxq, &nb_desc_burst);
+		if (unlikely(ret || !nb_desc_burst))
+			break;
+
+		sent += nb_desc_burst;
+		nb_desc -= nb_desc_burst;
+
+	} while (nb_desc);
+
+	*num = sent;
+
+	return ret;
+}
+
+static inline void
+mvneta_fill_shadowq(struct mvneta_shadow_txq *sq, struct rte_mbuf *buf)
+{
+	sq->ent[sq->head].cookie = (uint64_t)buf;
+	sq->ent[sq->head].addr = buf ?
+		rte_mbuf_data_iova_default(buf) : 0;
+
+	sq->head = (sq->head + 1) & MRVL_NETA_TX_SHADOWQ_MASK;
+	sq->size++;
+}
+
+static inline void
+mvneta_fill_desc(struct neta_ppio_desc *desc, struct rte_mbuf *buf)
+{
+	neta_ppio_outq_desc_reset(desc);
+	neta_ppio_outq_desc_set_phys_addr(desc, rte_pktmbuf_iova(buf));
+	neta_ppio_outq_desc_set_pkt_offset(desc, 0);
+	neta_ppio_outq_desc_set_pkt_len(desc, rte_pktmbuf_data_len(buf));
+}
+
+/**
+ * Release already sent buffers to mempool.
+ *
+ * @param ppio
+ *   Pointer to the port structure.
+ * @param sq
+ *   Pointer to the shadow queue.
+ * @param qid
+ *   Queue id number.
+ * @param force
+ *   Force releasing packets.
+ */
+static inline void
+mvneta_sent_buffers_free(struct neta_ppio *ppio,
+			 struct mvneta_shadow_txq *sq, int qid)
+{
+	struct neta_buff_inf *entry;
+	uint16_t nb_done = 0;
+	int i;
+	int tail = sq->tail;
+
+	neta_ppio_get_num_outq_done(ppio, qid, &nb_done);
+
+	if (nb_done > sq->size) {
+		MVNETA_LOG(ERR, "nb_done: %d, sq->size %d",
+			   nb_done, sq->size);
+		return;
+	}
+
+	for (i = 0; i < nb_done; i++) {
+		entry = &sq->ent[tail];
+
+		if (unlikely(!entry->addr)) {
+			MVNETA_LOG(DEBUG,
+				"Shadow memory @%d: cookie(%lx), pa(%lx)!",
+				tail, (u64)entry->cookie,
+				(u64)entry->addr);
+			tail = (tail + 1) & MRVL_NETA_TX_SHADOWQ_MASK;
+			continue;
+		}
+
+		struct rte_mbuf *mbuf;
+
+		mbuf = (struct rte_mbuf *)
+			   (cookie_addr_high | entry->cookie);
+		rte_pktmbuf_free(mbuf);
+		tail = (tail + 1) & MRVL_NETA_TX_SHADOWQ_MASK;
+	}
+
+	sq->tail = tail;
+	sq->size -= nb_done;
+}
+
+/**
+ * Return packet type information and l3/l4 offsets.
+ *
+ * @param desc
+ *   Pointer to the received packet descriptor.
+ * @param l3_offset
+ *   l3 packet offset.
+ * @param l4_offset
+ *   l4 packet offset.
+ *
+ * @return
+ *   Packet type information.
+ */
+static inline uint64_t
+mvneta_desc_to_packet_type_and_offset(struct neta_ppio_desc *desc,
+				    uint8_t *l3_offset, uint8_t *l4_offset)
+{
+	enum neta_inq_l3_type l3_type;
+	enum neta_inq_l4_type l4_type;
+	uint64_t packet_type;
+
+	neta_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
+	neta_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
+
+	packet_type = RTE_PTYPE_L2_ETHER;
+
+	if (NETA_RXD_GET_VLAN_INFO(desc))
+		packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
+
+	switch (l3_type) {
+	case NETA_INQ_L3_TYPE_IPV4_BAD:
+	case NETA_INQ_L3_TYPE_IPV4_OK:
+		packet_type |= RTE_PTYPE_L3_IPV4;
+		break;
+	case NETA_INQ_L3_TYPE_IPV6:
+		packet_type |= RTE_PTYPE_L3_IPV6;
+		break;
+	default:
+		packet_type |= RTE_PTYPE_UNKNOWN;
+		MVNETA_LOG(DEBUG, "Failed to recognize l3 packet type");
+		break;
+	}
+
+	switch (l4_type) {
+	case NETA_INQ_L4_TYPE_TCP:
+		packet_type |= RTE_PTYPE_L4_TCP;
+		break;
+	case NETA_INQ_L4_TYPE_UDP:
+		packet_type |= RTE_PTYPE_L4_UDP;
+		break;
+	default:
+		packet_type |= RTE_PTYPE_UNKNOWN;
+		MVNETA_LOG(DEBUG, "Failed to recognize l4 packet type");
+		break;
+	}
+
+	return packet_type;
+}
+
+/**
+ * Prepare offload information.
+ *
+ * @param ol_flags
+ *   Offload flags.
+ * @param packet_type
+ *   Packet type bitfield.
+ * @param l3_type
+ *   Pointer to the neta_ouq_l3_type structure.
+ * @param l4_type
+ *   Pointer to the neta_outq_l4_type structure.
+ * @param gen_l3_cksum
+ *   Will be set to 1 in case l3 checksum is computed.
+ * @param l4_cksum
+ *   Will be set to 1 in case l4 checksum is computed.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+static inline int
+mvneta_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
+			enum neta_outq_l3_type *l3_type,
+			enum neta_outq_l4_type *l4_type,
+			int *gen_l3_cksum,
+			int *gen_l4_cksum)
+{
+	/*
+	 * Based on ol_flags prepare information
+	 * for neta_ppio_outq_desc_set_proto_info() which setups descriptor
+	 * for offloading.
+	 */
+	if (ol_flags & PKT_TX_IPV4) {
+		*l3_type = NETA_OUTQ_L3_TYPE_IPV4;
+		*gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
+	} else if (ol_flags & PKT_TX_IPV6) {
+		*l3_type = NETA_OUTQ_L3_TYPE_IPV6;
+		/* no checksum for ipv6 header */
+		*gen_l3_cksum = 0;
+	} else {
+		/* if something different then stop processing */
+		return -1;
+	}
+
+	ol_flags &= PKT_TX_L4_MASK;
+	if ((packet_type & RTE_PTYPE_L4_TCP) &&
+	    ol_flags == PKT_TX_TCP_CKSUM) {
+		*l4_type = NETA_OUTQ_L4_TYPE_TCP;
+		*gen_l4_cksum = 1;
+	} else if ((packet_type & RTE_PTYPE_L4_UDP) &&
+		   ol_flags == PKT_TX_UDP_CKSUM) {
+		*l4_type = NETA_OUTQ_L4_TYPE_UDP;
+		*gen_l4_cksum = 1;
+	} else {
+		*l4_type = NETA_OUTQ_L4_TYPE_OTHER;
+		/* no checksum for other type */
+		*gen_l4_cksum = 0;
+	}
+
+	return 0;
+}
+
+/**
+ * Get offload information from the received packet descriptor.
+ *
+ * @param desc
+ *   Pointer to the received packet descriptor.
+ *
+ * @return
+ *   Mbuf offload flags.
+ */
+static inline uint64_t
+mvneta_desc_to_ol_flags(struct neta_ppio_desc *desc)
+{
+	uint64_t flags;
+	enum neta_inq_desc_status status;
+
+	status = neta_ppio_inq_desc_get_l3_pkt_error(desc);
+	if (unlikely(status != NETA_DESC_ERR_OK))
+		flags = PKT_RX_IP_CKSUM_BAD;
+	else
+		flags = PKT_RX_IP_CKSUM_GOOD;
+
+	status = neta_ppio_inq_desc_get_l4_pkt_error(desc);
+	if (unlikely(status != NETA_DESC_ERR_OK))
+		flags |= PKT_RX_L4_CKSUM_BAD;
+	else
+		flags |= PKT_RX_L4_CKSUM_GOOD;
+
+	return flags;
+}
+
+/**
+ * DPDK callback for transmit.
+ *
+ * @param txq
+ *   Generic pointer transmit queue.
+ * @param tx_pkts
+ *   Packets to transmit.
+ * @param nb_pkts
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted.
+ */
+static uint16_t
+mvneta_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+	struct mvneta_txq *q = txq;
+	struct mvneta_shadow_txq *sq;
+	struct neta_ppio_desc descs[nb_pkts];
+
+	int i, ret, bytes_sent = 0;
+	uint16_t num, sq_free_size;
+	uint64_t addr;
+
+	sq = &q->shadow_txq;
+	if (unlikely(!nb_pkts || !q->priv->ppio))
+		return 0;
+
+	if (sq->size)
+		mvneta_sent_buffers_free(q->priv->ppio,
+					 sq, q->queue_id);
+
+	sq_free_size = MRVL_NETA_TX_SHADOWQ_SIZE - sq->size - 1;
+	if (unlikely(nb_pkts > sq_free_size)) {
+		MVNETA_LOG(DEBUG,
+			"No room in shadow queue for %d packets! %d packets will be sent.",
+			nb_pkts, sq_free_size);
+		nb_pkts = sq_free_size;
+	}
+
+
+	for (i = 0; i < nb_pkts; i++) {
+		struct rte_mbuf *mbuf = tx_pkts[i];
+		int gen_l3_cksum, gen_l4_cksum;
+		enum neta_outq_l3_type l3_type;
+		enum neta_outq_l4_type l4_type;
+
+		/* Fill first mbuf info in shadow queue */
+		mvneta_fill_shadowq(sq, mbuf);
+		mvneta_fill_desc(&descs[i], mbuf);
+
+		bytes_sent += rte_pktmbuf_pkt_len(mbuf);
+
+		ret = mvneta_prepare_proto_info(mbuf->ol_flags,
+						mbuf->packet_type,
+						&l3_type, &l4_type,
+						&gen_l3_cksum,
+						&gen_l4_cksum);
+		if (unlikely(ret))
+			continue;
+
+		neta_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
+						   mbuf->l2_len,
+						   mbuf->l2_len + mbuf->l3_len,
+						   gen_l3_cksum, gen_l4_cksum);
+	}
+	num = nb_pkts;
+	neta_ppio_send(q->priv->ppio, q->queue_id, descs, &nb_pkts);
+
+
+	/* number of packets that were not sent */
+	if (unlikely(num > nb_pkts)) {
+		for (i = nb_pkts; i < num; i++) {
+			sq->head = (MRVL_NETA_TX_SHADOWQ_SIZE + sq->head - 1) &
+				MRVL_NETA_TX_SHADOWQ_MASK;
+			addr = cookie_addr_high | sq->ent[sq->head].cookie;
+			bytes_sent -=
+				rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
+		}
+		sq->size -= num - nb_pkts;
+	}
+
+	q->bytes_sent += bytes_sent;
+
+	return nb_pkts;
+}
+
+/** DPDK callback for S/G transmit.
+ *
+ * @param txq
+ *   Generic pointer transmit queue.
+ * @param tx_pkts
+ *   Packets to transmit.
+ * @param nb_pkts
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted.
+ */
+static uint16_t
+mvneta_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+	struct mvneta_txq *q = txq;
+	struct mvneta_shadow_txq *sq;
+	struct neta_ppio_desc descs[nb_pkts * NETA_PPIO_DESC_NUM_FRAGS];
+	struct neta_ppio_sg_pkts pkts;
+	uint8_t frags[nb_pkts];
+	int i, j, ret, bytes_sent = 0;
+	int tail, tail_first;
+	uint16_t num, sq_free_size;
+	uint16_t nb_segs, total_descs = 0;
+	uint64_t addr;
+
+	sq = &q->shadow_txq;
+	pkts.frags = frags;
+	pkts.num = 0;
+
+	if (unlikely(!q->priv->ppio))
+		return 0;
+
+	if (sq->size)
+		mvneta_sent_buffers_free(q->priv->ppio,
+					 sq, q->queue_id);
+	/* Save shadow queue free size */
+	sq_free_size = MRVL_NETA_TX_SHADOWQ_SIZE - sq->size - 1;
+
+	tail = 0;
+	for (i = 0; i < nb_pkts; i++) {
+		struct rte_mbuf *mbuf = tx_pkts[i];
+		struct rte_mbuf *seg = NULL;
+		int gen_l3_cksum, gen_l4_cksum;
+		enum neta_outq_l3_type l3_type;
+		enum neta_outq_l4_type l4_type;
+
+		nb_segs = mbuf->nb_segs;
+		total_descs += nb_segs;
+
+		/*
+		 * Check if total_descs does not exceed
+		 * shadow queue free size
+		 */
+		if (unlikely(total_descs > sq_free_size)) {
+			total_descs -= nb_segs;
+			MVNETA_LOG(DEBUG,
+				"No room in shadow queue for %d packets! "
+				"%d packets will be sent.",
+				nb_pkts, i);
+			break;
+		}
+
+
+		/* Check if nb_segs does not exceed the max nb of desc per
+		 * fragmented packet
+		 */
+		if (unlikely(nb_segs > NETA_PPIO_DESC_NUM_FRAGS)) {
+			total_descs -= nb_segs;
+			MVNETA_LOG(ERR,
+				"Too many segments. Packet won't be sent.");
+			break;
+		}
+
+		pkts.frags[pkts.num] = nb_segs;
+		pkts.num++;
+		tail_first = tail;
+
+		seg = mbuf;
+		for (j = 0; j < nb_segs - 1; j++) {
+			/* For the subsequent segments, set shadow queue
+			 * buffer to NULL
+			 */
+			mvneta_fill_shadowq(sq, NULL);
+			mvneta_fill_desc(&descs[tail], seg);
+
+			tail++;
+			seg = seg->next;
+		}
+		/* Put first mbuf info in last shadow queue entry */
+		mvneta_fill_shadowq(sq, mbuf);
+		/* Update descriptor with last segment */
+		mvneta_fill_desc(&descs[tail++], seg);
+
+		bytes_sent += rte_pktmbuf_pkt_len(mbuf);
+
+		ret = mvneta_prepare_proto_info(mbuf->ol_flags,
+						mbuf->packet_type,
+						&l3_type, &l4_type,
+						&gen_l3_cksum,
+						&gen_l4_cksum);
+		if (unlikely(ret))
+			continue;
+
+		neta_ppio_outq_desc_set_proto_info(&descs[tail_first],
+						   l3_type, l4_type,
+						   mbuf->l2_len,
+						   mbuf->l2_len + mbuf->l3_len,
+						   gen_l3_cksum, gen_l4_cksum);
+	}
+	num = total_descs;
+	neta_ppio_send_sg(q->priv->ppio, q->queue_id, descs, &total_descs,
+			  &pkts);
+
+	/* number of packets that were not sent */
+	if (unlikely(num > total_descs)) {
+		for (i = total_descs; i < num; i++) {
+			sq->head = (MRVL_NETA_TX_SHADOWQ_SIZE +
+					sq->head - 1) &
+					MRVL_NETA_TX_SHADOWQ_MASK;
+			addr = sq->ent[sq->head].cookie;
+			if (addr) {
+				struct rte_mbuf *mbuf;
+
+				mbuf = (struct rte_mbuf *)
+						(cookie_addr_high | addr);
+				bytes_sent -= rte_pktmbuf_pkt_len(mbuf);
+			}
+		}
+		sq->size -= num - total_descs;
+		nb_pkts = pkts.num;
+	}
+
+	q->bytes_sent += bytes_sent;
+
+	return nb_pkts;
+}
+
+/**
+ * Set tx burst function according to offload flag
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+void
+mvneta_set_tx_function(struct rte_eth_dev *dev)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+
+	/* Use a simple Tx queue (no offloads, no multi segs) if possible */
+	if (priv->multiseg) {
+		MVNETA_LOG(INFO, "Using multi-segment tx callback");
+		dev->tx_pkt_burst = mvneta_tx_sg_pkt_burst;
+	} else {
+		MVNETA_LOG(INFO, "Using single-segment tx callback");
+		dev->tx_pkt_burst = mvneta_tx_pkt_burst;
+	}
+}
+
+/**
+ * DPDK callback for receive.
+ *
+ * @param rxq
+ *   Generic pointer to the receive queue.
+ * @param rx_pkts
+ *   Array to store received packets.
+ * @param nb_pkts
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received.
+ */
+uint16_t
+mvneta_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	struct mvneta_rxq *q = rxq;
+	struct neta_ppio_desc descs[nb_pkts];
+	int i, ret, rx_done = 0, rx_dropped = 0;
+
+	if (unlikely(!q || !q->priv->ppio))
+		return 0;
+
+	ret = neta_ppio_recv(q->priv->ppio, q->queue_id,
+			descs, &nb_pkts);
+
+	if (unlikely(ret < 0)) {
+		MVNETA_LOG(ERR, "Failed to receive packets");
+		return 0;
+	}
+
+	for (i = 0; i < nb_pkts; i++) {
+		struct rte_mbuf *mbuf;
+		uint8_t l3_offset, l4_offset;
+		enum neta_inq_desc_status status;
+		uint64_t addr;
+
+		addr = cookie_addr_high |
+			neta_ppio_inq_desc_get_cookie(&descs[i]);
+		mbuf = (struct rte_mbuf *)addr;
+
+		rte_pktmbuf_reset(mbuf);
+
+		/* drop packet in case of mac, overrun or resource error */
+		status = neta_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
+		if (unlikely(status != NETA_DESC_ERR_OK)) {
+			/* Release the mbuf to the mempool since
+			 * it won't be transferred to tx path
+			 */
+			rte_pktmbuf_free(mbuf);
+			q->drop_mac++;
+			rx_dropped++;
+			continue;
+		}
+
+		mbuf->data_off += MVNETA_PKT_EFFEC_OFFS;
+		mbuf->pkt_len = neta_ppio_inq_desc_get_pkt_len(&descs[i]);
+		mbuf->data_len = mbuf->pkt_len;
+		mbuf->port = q->port_id;
+		mbuf->packet_type =
+			mvneta_desc_to_packet_type_and_offset(&descs[i],
+								&l3_offset,
+								&l4_offset);
+		mbuf->l2_len = l3_offset;
+		mbuf->l3_len = l4_offset - l3_offset;
+
+		if (likely(q->cksum_enabled))
+			mbuf->ol_flags = mvneta_desc_to_ol_flags(&descs[i]);
+
+		rx_pkts[rx_done++] = mbuf;
+		q->bytes_recv += mbuf->pkt_len;
+	}
+	q->pkts_processed += rx_done + rx_dropped;
+
+	if (q->pkts_processed > rx_desc_free_thresh) {
+		int buf_to_refill = rx_desc_free_thresh;
+
+		ret = mvneta_buffs_alloc(q->priv, q, &buf_to_refill);
+		if (ret)
+			MVNETA_LOG(ERR, "Refill failed");
+		q->pkts_processed -= buf_to_refill;
+	}
+
+	return rx_done;
+}
+
+/**
+ * DPDK callback to configure the receive queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param idx
+ *   RX queue index.
+ * @param desc
+ *   Number of descriptors to configure in queue.
+ * @param socket
+ *   NUMA socket on which memory must be allocated.
+ * @param conf
+ *   Thresholds parameters (unused_).
+ * @param mp
+ *   Memory pool for buffer allocations.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+int
+mvneta_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+		      unsigned int socket,
+		      const struct rte_eth_rxconf *conf __rte_unused,
+		      struct rte_mempool *mp)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	struct mvneta_rxq *rxq;
+	uint32_t frame_size, buf_size = rte_pktmbuf_data_room_size(mp);
+	uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
+
+	frame_size = buf_size - RTE_PKTMBUF_HEADROOM - MVNETA_PKT_EFFEC_OFFS;
+
+	if (frame_size < max_rx_pkt_len) {
+		MVNETA_LOG(ERR,
+			"Mbuf size must be increased to %u bytes to hold up "
+			"to %u bytes of data.",
+			buf_size + max_rx_pkt_len - frame_size,
+			max_rx_pkt_len);
+		dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
+		MVNETA_LOG(INFO, "Setting max rx pkt len to %u",
+			dev->data->dev_conf.rxmode.max_rx_pkt_len);
+	}
+
+	if (dev->data->rx_queues[idx]) {
+		rte_free(dev->data->rx_queues[idx]);
+		dev->data->rx_queues[idx] = NULL;
+	}
+
+	rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
+	if (!rxq)
+		return -ENOMEM;
+
+	rxq->priv = priv;
+	rxq->mp = mp;
+	rxq->cksum_enabled = dev->data->dev_conf.rxmode.offloads &
+			     DEV_RX_OFFLOAD_IPV4_CKSUM;
+	rxq->queue_id = idx;
+	rxq->port_id = dev->data->port_id;
+	rxq->size = desc;
+	rx_desc_free_thresh = RTE_MIN(rx_desc_free_thresh, (desc / 2));
+	priv->ppio_params.inqs_params.tcs_params[MRVL_NETA_DEFAULT_TC].size =
+		desc;
+
+	dev->data->rx_queues[idx] = rxq;
+
+	return 0;
+}
+
+/**
+ * DPDK callback to configure the transmit queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param idx
+ *   Transmit queue index.
+ * @param desc
+ *   Number of descriptors to configure in the queue.
+ * @param socket
+ *   NUMA socket on which memory must be allocated.
+ * @param conf
+ *   Tx queue configuration parameters.
+ *
+ * @return
+ *   0 on success, negative error value otherwise.
+ */
+int
+mvneta_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+		      unsigned int socket, const struct rte_eth_txconf *conf)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	struct mvneta_txq *txq;
+
+	if (dev->data->tx_queues[idx]) {
+		rte_free(dev->data->tx_queues[idx]);
+		dev->data->tx_queues[idx] = NULL;
+	}
+
+	txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
+	if (!txq)
+		return -ENOMEM;
+
+	txq->priv = priv;
+	txq->queue_id = idx;
+	txq->port_id = dev->data->port_id;
+	txq->tx_deferred_start = conf->tx_deferred_start;
+	dev->data->tx_queues[idx] = txq;
+
+	priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
+	priv->ppio_params.outqs_params.outqs_params[idx].weight = 1;
+
+	return 0;
+}
+
+/**
+ * DPDK callback to release the transmit queue.
+ *
+ * @param txq
+ *   Generic transmit queue pointer.
+ */
+void
+mvneta_tx_queue_release(void *txq)
+{
+	struct mvneta_txq *q = txq;
+
+	if (!q)
+		return;
+
+	rte_free(q);
+}
+
+/**
+ * Return mbufs to mempool.
+ *
+ * @param rxq
+ *    Pointer to rx queue structure
+ * @param desc
+ *    Array of rx descriptors
+ */
+static void
+mvneta_recv_buffs_free(struct neta_ppio_desc *desc, uint16_t num)
+{
+	uint64_t addr;
+	uint8_t i;
+
+	for (i = 0; i < num; i++) {
+		if (desc) {
+			addr = cookie_addr_high |
+					neta_ppio_inq_desc_get_cookie(desc);
+			if (addr)
+				rte_pktmbuf_free((struct rte_mbuf *)addr);
+			desc++;
+		}
+	}
+}
+
+int
+mvneta_alloc_rx_bufs(struct rte_eth_dev *dev)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+	int ret = 0, i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		struct mvneta_rxq *rxq = dev->data->rx_queues[i];
+		int num = rxq->size;
+
+		ret = mvneta_buffs_alloc(priv, rxq, &num);
+		if (ret || num != rxq->size) {
+			rte_free(rxq);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Flush single receive queue.
+ *
+ * @param rxq
+ *   Pointer to rx queue structure.
+ * @param descs
+ *   Array of rx descriptors
+ */
+static void
+mvneta_rx_queue_flush(struct mvneta_rxq *rxq)
+{
+	struct neta_ppio_desc *descs;
+	struct neta_buff_inf *bufs;
+	uint16_t num;
+	int ret, i;
+
+	descs = rte_malloc("rxdesc", MRVL_NETA_RXD_MAX * sizeof(*descs), 0);
+	bufs = rte_malloc("buffs", MRVL_NETA_RXD_MAX * sizeof(*bufs), 0);
+
+	do {
+		num = MRVL_NETA_RXD_MAX;
+		ret = neta_ppio_recv(rxq->priv->ppio,
+				     rxq->queue_id,
+				     descs, &num);
+		mvneta_recv_buffs_free(descs, num);
+	} while (ret == 0 && num);
+
+	rxq->pkts_processed = 0;
+
+	num = MRVL_NETA_RXD_MAX;
+
+	neta_ppio_inq_get_all_buffs(rxq->priv->ppio, rxq->queue_id, bufs, &num);
+	MVNETA_LOG(INFO, "freeing %u unused bufs.", num);
+
+	for (i = 0; i < num; i++) {
+		uint64_t addr;
+		if (bufs[i].cookie) {
+			addr = cookie_addr_high | bufs[i].cookie;
+			rte_pktmbuf_free((struct rte_mbuf *)addr);
+		}
+	}
+
+	rte_free(descs);
+	rte_free(bufs);
+}
+
+/**
+ * Flush single transmit queue.
+ *
+ * @param txq
+ *     Pointer to tx queue structure
+ */
+static void
+mvneta_tx_queue_flush(struct mvneta_txq *txq)
+{
+	struct mvneta_shadow_txq *sq = &txq->shadow_txq;
+
+	if (sq->size)
+		mvneta_sent_buffers_free(txq->priv->ppio, sq,
+					 txq->queue_id);
+
+	/* free the rest of them */
+	while (sq->tail != sq->head) {
+		uint64_t addr = cookie_addr_high |
+			sq->ent[sq->tail].cookie;
+		rte_pktmbuf_free((struct rte_mbuf *)addr);
+		sq->tail = (sq->tail + 1) & MRVL_NETA_TX_SHADOWQ_MASK;
+	}
+	memset(sq, 0, sizeof(*sq));
+}
+
+void
+mvneta_flush_queues(struct rte_eth_dev *dev)
+{
+	int i;
+
+	MVNETA_LOG(INFO, "Flushing rx queues");
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		struct mvneta_rxq *rxq = dev->data->rx_queues[i];
+
+		mvneta_rx_queue_flush(rxq);
+	}
+
+	MVNETA_LOG(INFO, "Flushing tx queues");
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		struct mvneta_txq *txq = dev->data->tx_queues[i];
+
+		mvneta_tx_queue_flush(txq);
+	}
+}
+
+/**
+ * DPDK callback to release the receive queue.
+ *
+ * @param rxq
+ *   Generic receive queue pointer.
+ */
+void
+mvneta_rx_queue_release(void *rxq)
+{
+	struct mvneta_rxq *q = rxq;
+
+	if (!q)
+		return;
+
+	/* If dev_stop was called already, mbufs are already
+	 * returned to mempool and ppio is deinitialized.
+	 * Skip this step.
+	 */
+
+	if (q->priv->ppio)
+		mvneta_rx_queue_flush(q);
+
+	rte_free(rxq);
+}
+
+/**
+ * DPDK callback to get information about specific receive queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param rx_queue_id
+ *   Receive queue index.
+ * @param qinfo
+ *   Receive queue information structure.
+ */
+void
+mvneta_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+		    struct rte_eth_rxq_info *qinfo)
+{
+	struct mvneta_rxq *q = dev->data->rx_queues[rx_queue_id];
+
+	qinfo->mp = q->mp;
+	qinfo->nb_desc = q->size;
+}
+
+/**
+ * DPDK callback to get information about specific transmit queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param tx_queue_id
+ *   Transmit queue index.
+ * @param qinfo
+ *   Transmit queue information structure.
+ */
+void
+mvneta_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
+		    struct rte_eth_txq_info *qinfo)
+{
+	struct mvneta_priv *priv = dev->data->dev_private;
+
+	qinfo->nb_desc =
+		priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
+}
diff --git a/drivers/net/mvneta/mvneta_rxtx.h b/drivers/net/mvneta/mvneta_rxtx.h
new file mode 100644
index 00000000..cc291901
--- /dev/null
+++ b/drivers/net/mvneta/mvneta_rxtx.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Marvell International Ltd.
+ * Copyright(c) 2018 Semihalf.
+ * All rights reserved.
+ */
+
+#ifndef _MVNETA_RXTX_H_
+#define _MVNETA_RXTX_H_
+
+#include "mvneta_ethdev.h"
+
+int mvneta_alloc_rx_bufs(struct rte_eth_dev *dev);
+
+void mvneta_flush_queues(struct rte_eth_dev *dev);
+
+void mvneta_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+			 struct rte_eth_rxq_info *qinfo);
+void mvneta_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
+			 struct rte_eth_txq_info *qinfo);
+
+void mvneta_set_tx_function(struct rte_eth_dev *dev);
+
+uint16_t
+mvneta_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+
+int
+mvneta_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+		      unsigned int socket,
+		      const struct rte_eth_rxconf *conf __rte_unused,
+		      struct rte_mempool *mp);
+int
+mvneta_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+		      unsigned int socket, const struct rte_eth_txconf *conf);
+
+void mvneta_rx_queue_release(void *rxq);
+void mvneta_tx_queue_release(void *txq);
+
+#endif /* _MVNETA_RXTX_H_ */
diff --git a/drivers/net/mvneta/rte_pmd_mvneta_version.map b/drivers/net/mvneta/rte_pmd_mvneta_version.map
new file mode 100644
index 00000000..24bd5cdb
--- /dev/null
+++ b/drivers/net/mvneta/rte_pmd_mvneta_version.map
@@ -0,0 +1,3 @@
+DPDK_18.11 {
+	local: *;
+};
diff --git a/drivers/net/mvpp2/Makefile b/drivers/net/mvpp2/Makefile
index 492aef97..661d2cda 100644
--- a/drivers/net/mvpp2/Makefile
+++ b/drivers/net/mvpp2/Makefile
@@ -23,6 +23,7 @@ LIBABIVER := 1
 EXPORT_MAP := rte_pmd_mvpp2_version.map
 
 # external library dependencies
+CFLAGS += -I$(RTE_SDK)/drivers/common/mvep
 CFLAGS += -I$(LIBMUSDK_PATH)/include
 CFLAGS += -DMVCONF_TYPES_PUBLIC
 CFLAGS += -DMVCONF_DMA_PHYS_ADDR_T_PUBLIC
@@ -32,11 +33,13 @@ LDLIBS += -L$(LIBMUSDK_PATH)/lib
 LDLIBS += -lmusdk
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_cfgfile
-LDLIBS += -lrte_bus_vdev
+LDLIBS += -lrte_bus_vdev -lrte_common_mvep
 
 # library source files
 SRCS-$(CONFIG_RTE_LIBRTE_MVPP2_PMD) += mrvl_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_MVPP2_PMD) += mrvl_qos.c
 SRCS-$(CONFIG_RTE_LIBRTE_MVPP2_PMD) += mrvl_flow.c
+SRCS-$(CONFIG_RTE_LIBRTE_MVPP2_PMD) += mrvl_mtr.c
+SRCS-$(CONFIG_RTE_LIBRTE_MVPP2_PMD) += mrvl_tm.c
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/mvpp2/meson.build b/drivers/net/mvpp2/meson.build
index e1398895..70ef2d64 100644
--- a/drivers/net/mvpp2/meson.build
+++ b/drivers/net/mvpp2/meson.build
@@ -19,7 +19,9 @@ endif
 sources = files(
 	'mrvl_ethdev.c',
 	'mrvl_flow.c',
-	'mrvl_qos.c'
+	'mrvl_qos.c',
+	'mrvl_mtr.c',
+	'mrvl_tm.c'
 )
 
-deps += ['cfgfile']
+deps += ['cfgfile', 'common_mvep']
diff --git a/drivers/net/mvpp2/mrvl_ethdev.c b/drivers/net/mvpp2/mrvl_ethdev.c
index a2d0576e..ab4c14e5 100644
--- a/drivers/net/mvpp2/mrvl_ethdev.c
+++ b/drivers/net/mvpp2/mrvl_ethdev.c
@@ -10,15 +10,6 @@
 #include <rte_malloc.h>
 #include <rte_bus_vdev.h>
 
-/* Unluckily, container_of is defined by both DPDK and MUSDK,
- * we'll declare only one version.
- *
- * Note that it is not used in this PMD anyway.
- */
-#ifdef container_of
-#undef container_of
-#endif
-
 #include <fcntl.h>
 #include <linux/ethtool.h>
 #include <linux/sockios.h>
@@ -29,8 +20,12 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 
+#include <rte_mvep_common.h>
 #include "mrvl_ethdev.h"
 #include "mrvl_qos.h"
+#include "mrvl_flow.h"
+#include "mrvl_mtr.h"
+#include "mrvl_tm.h"
 
 /* bitmask with reserved hifs */
 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
@@ -59,23 +54,18 @@
 #define MRVL_ARP_LENGTH 28
 
 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
-
-#define MRVL_COOKIE_HIGH_ADDR_SHIFT	(sizeof(pp2_cookie_t) * 8)
-#define MRVL_COOKIE_HIGH_ADDR_MASK	(~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
-
-/* Memory size (in bytes) for MUSDK dma buffers */
-#define MRVL_MUSDK_DMA_MEMSIZE 41943040
+#define MRVL_COOKIE_HIGH_ADDR_MASK 0xffffff0000000000
 
 /** Port Rx offload capabilities */
 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
 			  DEV_RX_OFFLOAD_JUMBO_FRAME | \
-			  DEV_RX_OFFLOAD_CRC_STRIP | \
 			  DEV_RX_OFFLOAD_CHECKSUM)
 
 /** Port Tx offloads capabilities */
 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
 			  DEV_TX_OFFLOAD_UDP_CKSUM | \
-			  DEV_TX_OFFLOAD_TCP_CKSUM)
+			  DEV_TX_OFFLOAD_TCP_CKSUM | \
+			  DEV_TX_OFFLOAD_MULTI_SEGS)
 
 static const char * const valid_args[] = {
 	MRVL_IFACE_NAME_ARG,
@@ -86,13 +76,12 @@ static const char * const valid_args[] = {
 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 static struct pp2_hif *hifs[RTE_MAX_LCORE];
 static int used_bpools[PP2_NUM_PKT_PROC] = {
-	MRVL_MUSDK_BPOOLS_RESERVED,
-	MRVL_MUSDK_BPOOLS_RESERVED
+	[0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
 };
 
-struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
-int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
-uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
+static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
+static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
+static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 
 int mrvl_logtype;
 
@@ -116,7 +105,9 @@ struct mrvl_shadow_txq {
 	int head;           /* write index - used when sending buffers */
 	int tail;           /* read index - used when releasing buffers */
 	u16 size;           /* queue occupied size */
-	u16 num_to_release; /* number of buffers sent, that can be released */
+	u16 num_to_release; /* number of descriptors sent, that can be
+			     * released
+			     */
 	struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 };
 
@@ -148,6 +139,12 @@ static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 			struct pp2_hif *hif, unsigned int core_id,
 			struct mrvl_shadow_txq *sq, int qid, int force);
 
+static uint16_t mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
+				  uint16_t nb_pkts);
+static uint16_t mrvl_tx_sg_pkt_burst(void *txq,	struct rte_mbuf **tx_pkts,
+				     uint16_t nb_pkts);
+
+
 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 	#name, offsetof(struct pp2_ppio_statistics, name),	\
 	sizeof(((struct pp2_ppio_statistics *)0)->name)		\
@@ -174,6 +171,31 @@ static struct {
 	MRVL_XSTATS_TBL_ENTRY(tx_errors)
 };
 
+static inline void
+mrvl_fill_shadowq(struct mrvl_shadow_txq *sq, struct rte_mbuf *buf)
+{
+	sq->ent[sq->head].buff.cookie = (uint64_t)buf;
+	sq->ent[sq->head].buff.addr = buf ?
+		rte_mbuf_data_iova_default(buf) : 0;
+
+	sq->ent[sq->head].bpool =
+		(unlikely(!buf || buf->port >= RTE_MAX_ETHPORTS ||
+		 buf->refcnt > 1)) ? NULL :
+		 mrvl_port_to_bpool_lookup[buf->port];
+
+	sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
+	sq->size++;
+}
+
+static inline void
+mrvl_fill_desc(struct pp2_ppio_desc *desc, struct rte_mbuf *buf)
+{
+	pp2_ppio_outq_desc_reset(desc);
+	pp2_ppio_outq_desc_set_phys_addr(desc, rte_pktmbuf_iova(buf));
+	pp2_ppio_outq_desc_set_pkt_offset(desc, 0);
+	pp2_ppio_outq_desc_set_pkt_len(desc, rte_pktmbuf_data_len(buf));
+}
+
 static inline int
 mrvl_get_bpool_size(int pp2_id, int pool_id)
 {
@@ -253,6 +275,27 @@ out:
 }
 
 /**
+ * Set tx burst function according to offload flag
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+static void
+mrvl_set_tx_function(struct rte_eth_dev *dev)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+
+	/* Use a simple Tx queue (no offloads, no multi segs) if possible */
+	if (priv->multiseg) {
+		RTE_LOG(INFO, PMD, "Using multi-segment tx callback\n");
+		dev->tx_pkt_burst = mrvl_tx_sg_pkt_burst;
+	} else {
+		RTE_LOG(INFO, PMD, "Using single-segment tx callback\n");
+		dev->tx_pkt_burst = mrvl_tx_pkt_burst;
+	}
+}
+
+/**
  * Configure rss based on dpdk rss configuration.
  *
  * @param priv
@@ -307,6 +350,11 @@ mrvl_dev_configure(struct rte_eth_dev *dev)
 	struct mrvl_priv *priv = dev->data->dev_private;
 	int ret;
 
+	if (priv->ppio) {
+		MRVL_LOG(INFO, "Device reconfiguration is not supported");
+		return -EINVAL;
+	}
+
 	if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 	    dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 		MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
@@ -314,14 +362,6 @@ mrvl_dev_configure(struct rte_eth_dev *dev)
 		return -EINVAL;
 	}
 
-	/* KEEP_CRC offload flag is not supported by PMD
-	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
-	 */
-	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads)) {
-		MRVL_LOG(INFO, "L2 CRC stripping is always enabled in hw");
-		dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
-	}
-
 	if (dev->data->dev_conf.rxmode.split_hdr_size) {
 		MRVL_LOG(INFO, "Split headers not supported");
 		return -EINVAL;
@@ -329,7 +369,10 @@ mrvl_dev_configure(struct rte_eth_dev *dev)
 
 	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 		dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
-				 ETHER_HDR_LEN - ETHER_CRC_LEN;
+				 MRVL_PP2_ETH_HDRS_LEN;
+
+	if (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
+		priv->multiseg = 1;
 
 	ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 				  dev->data->nb_rx_queues);
@@ -345,6 +388,10 @@ mrvl_dev_configure(struct rte_eth_dev *dev)
 	priv->ppio_params.maintain_stats = 1;
 	priv->nb_rx_queues = dev->data->nb_rx_queues;
 
+	ret = mrvl_tm_init(dev);
+	if (ret < 0)
+		return ret;
+
 	if (dev->data->nb_rx_queues == 1 &&
 	    dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 		MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
@@ -375,21 +422,55 @@ static int
 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 {
 	struct mrvl_priv *priv = dev->data->dev_private;
-	/* extra MV_MH_SIZE bytes are required for Marvell tag */
-	uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
+	uint16_t mru;
+	uint16_t mbuf_data_size = 0; /* SW buffer size */
 	int ret;
 
-	if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
+	mru = MRVL_PP2_MTU_TO_MRU(mtu);
+	/*
+	 * min_rx_buf_size is equal to mbuf data size
+	 * if pmd didn't set it differently
+	 */
+	mbuf_data_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
+	/* Prevent PMD from:
+	 * - setting mru greater than the mbuf size resulting in
+	 * hw and sw buffer size mismatch
+	 * - setting mtu that requires the support of scattered packets
+	 * when this feature has not been enabled/supported so far
+	 * (TODO check scattered_rx flag here once scattered RX is supported).
+	 */
+	if (mru + MRVL_PKT_OFFS > mbuf_data_size) {
+		mru = mbuf_data_size - MRVL_PKT_OFFS;
+		mtu = MRVL_PP2_MRU_TO_MTU(mru);
+		MRVL_LOG(WARNING, "MTU too big, max MTU possible limitted "
+			"by current mbuf size: %u. Set MTU to %u, MRU to %u",
+			mbuf_data_size, mtu, mru);
+	}
+
+	if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX) {
+		MRVL_LOG(ERR, "Invalid MTU [%u] or MRU [%u]", mtu, mru);
 		return -EINVAL;
+	}
+
+	dev->data->mtu = mtu;
+	dev->data->dev_conf.rxmode.max_rx_pkt_len = mru - MV_MH_SIZE;
 
 	if (!priv->ppio)
 		return 0;
 
 	ret = pp2_ppio_set_mru(priv->ppio, mru);
-	if (ret)
+	if (ret) {
+		MRVL_LOG(ERR, "Failed to change MRU");
 		return ret;
+	}
 
-	return pp2_ppio_set_mtu(priv->ppio, mtu);
+	ret = pp2_ppio_set_mtu(priv->ppio, mtu);
+	if (ret) {
+		MRVL_LOG(ERR, "Failed to change MTU");
+		return ret;
+	}
+
+	return 0;
 }
 
 /**
@@ -528,6 +609,9 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 	char match[MRVL_MATCH_LEN];
 	int ret = 0, i, def_init_size;
 
+	if (priv->ppio)
+		return mrvl_dev_set_link_up(dev);
+
 	snprintf(match, sizeof(match), "ppio-%d:%d",
 		 priv->pp_id, priv->ppio_id);
 	priv->ppio_params.match = match;
@@ -597,9 +681,13 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 		}
 		priv->vlan_flushed = 1;
 	}
+	ret = mrvl_mtu_set(dev, dev->data->mtu);
+	if (ret)
+		MRVL_LOG(ERR, "Failed to set MTU to %d", dev->data->mtu);
 
 	/* For default QoS config, don't start classifier. */
-	if (mrvl_qos_cfg) {
+	if (mrvl_qos_cfg  &&
+	    mrvl_qos_cfg->port[dev->data->port_id].use_global_defaults == 0) {
 		ret = mrvl_start_qos_mapping(priv);
 		if (ret) {
 			MRVL_LOG(ERR, "Failed to setup QoS mapping");
@@ -631,6 +719,10 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 			goto out;
 	}
 
+	mrvl_flow_init(dev);
+	mrvl_mtr_init(dev);
+	mrvl_set_tx_function(dev);
+
 	return 0;
 out:
 	MRVL_LOG(ERR, "Failed to start device");
@@ -752,28 +844,7 @@ mrvl_flush_bpool(struct rte_eth_dev *dev)
 static void
 mrvl_dev_stop(struct rte_eth_dev *dev)
 {
-	struct mrvl_priv *priv = dev->data->dev_private;
-
 	mrvl_dev_set_link_down(dev);
-	mrvl_flush_rx_queues(dev);
-	mrvl_flush_tx_shadow_queues(dev);
-	if (priv->cls_tbl) {
-		pp2_cls_tbl_deinit(priv->cls_tbl);
-		priv->cls_tbl = NULL;
-	}
-	if (priv->qos_tbl) {
-		pp2_cls_qos_tbl_deinit(priv->qos_tbl);
-		priv->qos_tbl = NULL;
-	}
-	if (priv->ppio)
-		pp2_ppio_deinit(priv->ppio);
-	priv->ppio = NULL;
-
-	/* policer must be released after ppio deinitialization */
-	if (priv->policer) {
-		pp2_cls_plcr_deinit(priv->policer);
-		priv->policer = NULL;
-	}
 }
 
 /**
@@ -788,6 +859,11 @@ mrvl_dev_close(struct rte_eth_dev *dev)
 	struct mrvl_priv *priv = dev->data->dev_private;
 	size_t i;
 
+	mrvl_flush_rx_queues(dev);
+	mrvl_flush_tx_shadow_queues(dev);
+	mrvl_flow_deinit(dev);
+	mrvl_mtr_deinit(dev);
+
 	for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 		struct pp2_ppio_tc_params *tc_params =
 			&priv->ppio_params.inqs_params.tcs_params[i];
@@ -798,7 +874,29 @@ mrvl_dev_close(struct rte_eth_dev *dev)
 		}
 	}
 
+	if (priv->cls_tbl) {
+		pp2_cls_tbl_deinit(priv->cls_tbl);
+		priv->cls_tbl = NULL;
+	}
+
+	if (priv->qos_tbl) {
+		pp2_cls_qos_tbl_deinit(priv->qos_tbl);
+		priv->qos_tbl = NULL;
+	}
+
 	mrvl_flush_bpool(dev);
+	mrvl_tm_deinit(dev);
+
+	if (priv->ppio) {
+		pp2_ppio_deinit(priv->ppio);
+		priv->ppio = NULL;
+	}
+
+	/* policer must be released after ppio deinitialization */
+	if (priv->default_policer) {
+		pp2_cls_plcr_deinit(priv->default_policer);
+		priv->default_policer = NULL;
+	}
 }
 
 /**
@@ -1337,7 +1435,6 @@ mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
 
 	/* By default packets are dropped if no descriptors are available */
 	info->default_rxconf.rx_drop_en = 1;
-	info->default_rxconf.offloads = DEV_RX_OFFLOAD_CRC_STRIP;
 
 	info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
 }
@@ -1356,6 +1453,8 @@ mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
 {
 	static const uint32_t ptypes[] = {
 		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_L2_ETHER_VLAN,
+		RTE_PTYPE_L2_ETHER_QINQ,
 		RTE_PTYPE_L3_IPV4,
 		RTE_PTYPE_L3_IPV4_EXT,
 		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
@@ -1492,7 +1591,7 @@ mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
 
 		entries[i].buff.addr =
 			rte_mbuf_data_iova_default(mbufs[i]);
-		entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
+		entries[i].buff.cookie = (uint64_t)mbufs[i];
 		entries[i].bpool = bpool;
 	}
 
@@ -1537,8 +1636,8 @@ mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 {
 	struct mrvl_priv *priv = dev->data->dev_private;
 	struct mrvl_rxq *rxq;
-	uint32_t min_size,
-		 max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
+	uint32_t frame_size, buf_size = rte_pktmbuf_data_room_size(mp);
+	uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 	int ret, tc, inq;
 	uint64_t offloads;
 
@@ -1553,15 +1652,16 @@ mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		return -EFAULT;
 	}
 
-	min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
-		   MRVL_PKT_EFFEC_OFFS;
-	if (min_size < max_rx_pkt_len) {
-		MRVL_LOG(ERR,
-			"Mbuf size must be increased to %u bytes to hold up to %u bytes of data.",
-			max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
-			MRVL_PKT_EFFEC_OFFS,
+	frame_size = buf_size - RTE_PKTMBUF_HEADROOM - MRVL_PKT_EFFEC_OFFS;
+	if (frame_size < max_rx_pkt_len) {
+		MRVL_LOG(WARNING,
+			"Mbuf size must be increased to %u bytes to hold up "
+			"to %u bytes of data.",
+			buf_size + max_rx_pkt_len - frame_size,
 			max_rx_pkt_len);
-		return -EINVAL;
+		dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
+		MRVL_LOG(INFO, "Setting max rx pkt len to %u",
+			dev->data->dev_conf.rxmode.max_rx_pkt_len);
 	}
 
 	if (dev->data->rx_queues[idx]) {
@@ -1867,6 +1967,44 @@ mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
 	}
 }
 
+/**
+ * DPDK callback to get rte_mtr callbacks.
+ *
+ * @param dev
+ *   Pointer to the device structure.
+ * @param ops
+ *   Pointer to pass the mtr ops.
+ *
+ * @return
+ *   Always 0.
+ */
+static int
+mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
+{
+	*(const void **)ops = &mrvl_mtr_ops;
+
+	return 0;
+}
+
+/**
+ * DPDK callback to get rte_tm callbacks.
+ *
+ * @param dev
+ *   Pointer to the device structure.
+ * @param ops
+ *   Pointer to pass the tm ops.
+ *
+ * @return
+ *   Always 0.
+ */
+static int
+mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
+{
+	*(const void **)ops = &mrvl_tm_ops;
+
+	return 0;
+}
+
 static const struct eth_dev_ops mrvl_ops = {
 	.dev_configure = mrvl_dev_configure,
 	.dev_start = mrvl_dev_start,
@@ -1904,6 +2042,8 @@ static const struct eth_dev_ops mrvl_ops = {
 	.rss_hash_update = mrvl_rss_hash_update,
 	.rss_hash_conf_get = mrvl_rss_hash_conf_get,
 	.filter_ctrl = mrvl_eth_filter_ctrl,
+	.mtr_ops_get = mrvl_mtr_ops_get,
+	.tm_ops_get = mrvl_tm_ops_get,
 };
 
 /**
@@ -1925,13 +2065,27 @@ mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
 {
 	enum pp2_inq_l3_type l3_type;
 	enum pp2_inq_l4_type l4_type;
+	enum pp2_inq_vlan_tag vlan_tag;
 	uint64_t packet_type;
 
 	pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
 	pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
+	pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
 
 	packet_type = RTE_PTYPE_L2_ETHER;
 
+	switch (vlan_tag) {
+	case PP2_INQ_VLAN_TAG_SINGLE:
+		packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
+		break;
+	case PP2_INQ_VLAN_TAG_DOUBLE:
+	case PP2_INQ_VLAN_TAG_TRIPLE:
+		packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
+		break;
+	default:
+		break;
+	}
+
 	switch (l3_type) {
 	case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
 		packet_type |= RTE_PTYPE_L3_IPV4;
@@ -2073,7 +2227,7 @@ mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		if (unlikely(status != PP2_DESC_ERR_OK)) {
 			struct pp2_buff_inf binf = {
 				.addr = rte_mbuf_data_iova_default(mbuf),
-				.cookie = (pp2_cookie_t)(uint64_t)mbuf,
+				.cookie = (uint64_t)mbuf,
 			};
 
 			pp2_bpool_put_buff(hif, bpool, &binf);
@@ -2334,22 +2488,8 @@ mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			rte_mbuf_prefetch_part2(pref_pkt_hdr);
 		}
 
-		sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
-		sq->ent[sq->head].buff.addr =
-			rte_mbuf_data_iova_default(mbuf);
-		sq->ent[sq->head].bpool =
-			(unlikely(mbuf->port >= RTE_MAX_ETHPORTS ||
-			 mbuf->refcnt > 1)) ? NULL :
-			 mrvl_port_to_bpool_lookup[mbuf->port];
-		sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
-		sq->size++;
-
-		pp2_ppio_outq_desc_reset(&descs[i]);
-		pp2_ppio_outq_desc_set_phys_addr(&descs[i],
-						 rte_pktmbuf_iova(mbuf));
-		pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
-		pp2_ppio_outq_desc_set_pkt_len(&descs[i],
-					       rte_pktmbuf_pkt_len(mbuf));
+		mrvl_fill_shadowq(sq, mbuf);
+		mrvl_fill_desc(&descs[i], mbuf);
 
 		bytes_sent += rte_pktmbuf_pkt_len(mbuf);
 		/*
@@ -2387,6 +2527,152 @@ mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	return nb_pkts;
 }
 
+/** DPDK callback for S/G transmit.
+ *
+ * @param txq
+ *   Generic pointer transmit queue.
+ * @param tx_pkts
+ *   Packets to transmit.
+ * @param nb_pkts
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted.
+ */
+static uint16_t
+mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
+		     uint16_t nb_pkts)
+{
+	struct mrvl_txq *q = txq;
+	struct mrvl_shadow_txq *sq;
+	struct pp2_hif *hif;
+	struct pp2_ppio_desc descs[nb_pkts * PP2_PPIO_DESC_NUM_FRAGS];
+	struct pp2_ppio_sg_pkts pkts;
+	uint8_t frags[nb_pkts];
+	unsigned int core_id = rte_lcore_id();
+	int i, j, ret, bytes_sent = 0;
+	int tail, tail_first;
+	uint16_t num, sq_free_size;
+	uint16_t nb_segs, total_descs = 0;
+	uint64_t addr;
+
+	hif = mrvl_get_hif(q->priv, core_id);
+	sq = &q->shadow_txqs[core_id];
+	pkts.frags = frags;
+	pkts.num = 0;
+
+	if (unlikely(!q->priv->ppio || !hif))
+		return 0;
+
+	if (sq->size)
+		mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
+				       sq, q->queue_id, 0);
+
+	/* Save shadow queue free size */
+	sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
+
+	tail = 0;
+	for (i = 0; i < nb_pkts; i++) {
+		struct rte_mbuf *mbuf = tx_pkts[i];
+		struct rte_mbuf *seg = NULL;
+		int gen_l3_cksum, gen_l4_cksum;
+		enum pp2_outq_l3_type l3_type;
+		enum pp2_outq_l4_type l4_type;
+
+		nb_segs = mbuf->nb_segs;
+		tail_first = tail;
+		total_descs += nb_segs;
+
+		/*
+		 * Check if total_descs does not exceed
+		 * shadow queue free size
+		 */
+		if (unlikely(total_descs > sq_free_size)) {
+			total_descs -= nb_segs;
+			RTE_LOG(DEBUG, PMD,
+				"No room in shadow queue for %d packets! "
+				"%d packets will be sent.\n",
+				nb_pkts, i);
+			break;
+		}
+
+		/* Check if nb_segs does not exceed the max nb of desc per
+		 * fragmented packet
+		 */
+		if (nb_segs > PP2_PPIO_DESC_NUM_FRAGS) {
+			total_descs -= nb_segs;
+			RTE_LOG(ERR, PMD,
+				"Too many segments. Packet won't be sent.\n");
+			break;
+		}
+
+		if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
+			struct rte_mbuf *pref_pkt_hdr;
+
+			pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
+			rte_mbuf_prefetch_part1(pref_pkt_hdr);
+			rte_mbuf_prefetch_part2(pref_pkt_hdr);
+		}
+
+		pkts.frags[pkts.num] = nb_segs;
+		pkts.num++;
+
+		seg = mbuf;
+		for (j = 0; j < nb_segs - 1; j++) {
+			/* For the subsequent segments, set shadow queue
+			 * buffer to NULL
+			 */
+			mrvl_fill_shadowq(sq, NULL);
+			mrvl_fill_desc(&descs[tail], seg);
+
+			tail++;
+			seg = seg->next;
+		}
+		/* Put first mbuf info in last shadow queue entry */
+		mrvl_fill_shadowq(sq, mbuf);
+		/* Update descriptor with last segment */
+		mrvl_fill_desc(&descs[tail++], seg);
+
+		bytes_sent += rte_pktmbuf_pkt_len(mbuf);
+		/* In case unsupported ol_flags were passed
+		 * do not update descriptor offload information
+		 */
+		ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
+					      &l3_type, &l4_type, &gen_l3_cksum,
+					      &gen_l4_cksum);
+		if (unlikely(ret))
+			continue;
+
+		pp2_ppio_outq_desc_set_proto_info(&descs[tail_first], l3_type,
+						  l4_type, mbuf->l2_len,
+						  mbuf->l2_len + mbuf->l3_len,
+						  gen_l3_cksum, gen_l4_cksum);
+	}
+
+	num = total_descs;
+	pp2_ppio_send_sg(q->priv->ppio, hif, q->queue_id, descs,
+			 &total_descs, &pkts);
+	/* number of packets that were not sent */
+	if (unlikely(num > total_descs)) {
+		for (i = total_descs; i < num; i++) {
+			sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
+				MRVL_PP2_TX_SHADOWQ_MASK;
+
+			addr = sq->ent[sq->head].buff.cookie;
+			if (addr)
+				bytes_sent -=
+					rte_pktmbuf_pkt_len((struct rte_mbuf *)
+						(cookie_addr_high | addr));
+		}
+		sq->size -= num - total_descs;
+		nb_pkts = pkts.num;
+	}
+
+	q->bytes_sent += bytes_sent;
+
+	return nb_pkts;
+}
+
 /**
  * Initialize packet processor.
  *
@@ -2494,8 +2780,9 @@ mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
 	priv = mrvl_priv_create(name);
 	if (!priv) {
 		ret = -ENOMEM;
-		goto out_free_dev;
+		goto out_free;
 	}
+	eth_dev->data->dev_private = priv;
 
 	eth_dev->data->mac_addrs =
 		rte_zmalloc("mac_addrs",
@@ -2503,33 +2790,28 @@ mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
 	if (!eth_dev->data->mac_addrs) {
 		MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
 		ret = -ENOMEM;
-		goto out_free_priv;
+		goto out_free;
 	}
 
 	memset(&req, 0, sizeof(req));
 	strcpy(req.ifr_name, name);
 	ret = ioctl(fd, SIOCGIFHWADDR, &req);
 	if (ret)
-		goto out_free_mac;
+		goto out_free;
 
 	memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
 	       req.ifr_addr.sa_data, ETHER_ADDR_LEN);
 
-	eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
-	eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
 	eth_dev->data->kdrv = RTE_KDRV_NONE;
-	eth_dev->data->dev_private = priv;
 	eth_dev->device = &vdev->device;
+	eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
+	mrvl_set_tx_function(eth_dev);
 	eth_dev->dev_ops = &mrvl_ops;
 
 	rte_eth_dev_probing_finish(eth_dev);
 	return 0;
-out_free_mac:
-	rte_free(eth_dev->data->mac_addrs);
-out_free_dev:
+out_free:
 	rte_eth_dev_release_port(eth_dev);
-out_free_priv:
-	rte_free(priv);
 
 	return ret;
 }
@@ -2553,8 +2835,6 @@ mrvl_eth_dev_destroy(const char *name)
 	priv = eth_dev->data->dev_private;
 	pp2_bpool_deinit(priv->bpool);
 	used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
-	rte_free(priv);
-	rte_free(eth_dev->data->mac_addrs);
 	rte_eth_dev_release_port(eth_dev);
 }
 
@@ -2654,23 +2934,16 @@ rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
 		goto init_devices;
 
 	MRVL_LOG(INFO, "Perform MUSDK initializations");
-	/*
-	 * ret == -EEXIST is correct, it means DMA
-	 * has been already initialized (by another PMD).
-	 */
-	ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE);
-	if (ret < 0) {
-		if (ret != -EEXIST)
-			goto out_free_kvlist;
-		else
-			MRVL_LOG(INFO,
-				"DMA memory has been already initialized by a different driver.");
-	}
+
+	ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
+	if (ret)
+		goto out_free_kvlist;
 
 	ret = mrvl_init_pp2();
 	if (ret) {
 		MRVL_LOG(ERR, "Failed to init PP!");
-		goto out_deinit_dma;
+		rte_mvep_deinit(MVEP_MOD_T_PP2);
+		goto out_free_kvlist;
 	}
 
 	memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
@@ -2695,11 +2968,10 @@ out_cleanup:
 	for (; i > 0; i--)
 		mrvl_eth_dev_destroy(ifnames.names[i]);
 
-	if (mrvl_dev_num == 0)
+	if (mrvl_dev_num == 0) {
 		mrvl_deinit_pp2();
-out_deinit_dma:
-	if (mrvl_dev_num == 0)
-		mv_sys_dma_mem_destroy();
+		rte_mvep_deinit(MVEP_MOD_T_PP2);
+	}
 out_free_kvlist:
 	rte_kvargs_free(kvlist);
 
@@ -2739,7 +3011,7 @@ rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
 		MRVL_LOG(INFO, "Perform MUSDK deinit");
 		mrvl_deinit_hifs();
 		mrvl_deinit_pp2();
-		mv_sys_dma_mem_destroy();
+		rte_mvep_deinit(MVEP_MOD_T_PP2);
 	}
 
 	return 0;
diff --git a/drivers/net/mvpp2/mrvl_ethdev.h b/drivers/net/mvpp2/mrvl_ethdev.h
index 3726f788..0120b9e8 100644
--- a/drivers/net/mvpp2/mrvl_ethdev.h
+++ b/drivers/net/mvpp2/mrvl_ethdev.h
@@ -9,6 +9,18 @@
 
 #include <rte_spinlock.h>
 #include <rte_flow_driver.h>
+#include <rte_mtr_driver.h>
+#include <rte_tm_driver.h>
+
+/*
+ * container_of is defined by both DPDK and MUSDK,
+ * we'll declare only one version.
+ *
+ * Note that it is not used in this PMD anyway.
+ */
+#ifdef container_of
+#undef container_of
+#endif
 
 #include <env/mv_autogen_comp_flags.h>
 #include <drivers/mv_pp2.h>
@@ -16,6 +28,7 @@
 #include <drivers/mv_pp2_cls.h>
 #include <drivers/mv_pp2_hif.h>
 #include <drivers/mv_pp2_ppio.h>
+#include "env/mv_common.h" /* for BIT() */
 
 /** Maximum number of rx queues per port */
 #define MRVL_PP2_RXQ_MAX 32
@@ -59,6 +72,99 @@
 /** Minimum number of sent buffers to release from shadow queue to BM */
 #define MRVL_PP2_BUF_RELEASE_BURST_SIZE	64
 
+#define MRVL_PP2_VLAN_TAG_LEN		4
+#define MRVL_PP2_ETH_HDRS_LEN		(ETHER_HDR_LEN + ETHER_CRC_LEN + \
+					(2 * MRVL_PP2_VLAN_TAG_LEN))
+#define MRVL_PP2_HDRS_LEN		(MV_MH_SIZE + MRVL_PP2_ETH_HDRS_LEN)
+#define MRVL_PP2_MTU_TO_MRU(mtu)	((mtu) + MRVL_PP2_HDRS_LEN)
+#define MRVL_PP2_MRU_TO_MTU(mru)	((mru) - MRVL_PP2_HDRS_LEN)
+
+/** Maximum length of a match string */
+#define MRVL_MATCH_LEN 16
+
+/** Parsed fields in processed rte_flow_item. */
+enum mrvl_parsed_fields {
+	/* eth flags */
+	F_DMAC =         BIT(0),
+	F_SMAC =         BIT(1),
+	F_TYPE =         BIT(2),
+	/* vlan flags */
+	F_VLAN_PRI =     BIT(3),
+	F_VLAN_ID =      BIT(4),
+	F_VLAN_TCI =     BIT(5), /* not supported by MUSDK yet */
+	/* ip4 flags */
+	F_IP4_TOS =      BIT(6),
+	F_IP4_SIP =      BIT(7),
+	F_IP4_DIP =      BIT(8),
+	F_IP4_PROTO =    BIT(9),
+	/* ip6 flags */
+	F_IP6_TC =       BIT(10), /* not supported by MUSDK yet */
+	F_IP6_SIP =      BIT(11),
+	F_IP6_DIP =      BIT(12),
+	F_IP6_FLOW =     BIT(13),
+	F_IP6_NEXT_HDR = BIT(14),
+	/* tcp flags */
+	F_TCP_SPORT =    BIT(15),
+	F_TCP_DPORT =    BIT(16),
+	/* udp flags */
+	F_UDP_SPORT =    BIT(17),
+	F_UDP_DPORT =    BIT(18),
+};
+
+/** PMD-specific definition of a flow rule handle. */
+struct mrvl_mtr;
+struct rte_flow {
+	LIST_ENTRY(rte_flow) next;
+	struct mrvl_mtr *mtr;
+
+	enum mrvl_parsed_fields pattern;
+
+	struct pp2_cls_tbl_rule rule;
+	struct pp2_cls_cos_desc cos;
+	struct pp2_cls_tbl_action action;
+};
+
+struct mrvl_mtr_profile {
+	LIST_ENTRY(mrvl_mtr_profile) next;
+	uint32_t profile_id;
+	int refcnt;
+	struct rte_mtr_meter_profile profile;
+};
+
+struct mrvl_mtr {
+	LIST_ENTRY(mrvl_mtr) next;
+	uint32_t mtr_id;
+	int refcnt;
+	int shared;
+	int enabled;
+	int plcr_bit;
+	struct mrvl_mtr_profile *profile;
+	struct pp2_cls_plcr *plcr;
+};
+
+struct mrvl_tm_shaper_profile {
+	LIST_ENTRY(mrvl_tm_shaper_profile) next;
+	uint32_t id;
+	int refcnt;
+	struct rte_tm_shaper_params params;
+};
+
+enum {
+	MRVL_NODE_PORT,
+	MRVL_NODE_QUEUE,
+};
+
+struct mrvl_tm_node {
+	LIST_ENTRY(mrvl_tm_node) next;
+	uint32_t id;
+	uint32_t type;
+	int refcnt;
+	struct mrvl_tm_node *parent;
+	struct mrvl_tm_shaper_profile *profile;
+	uint8_t weight;
+	uint64_t stats_mask;
+};
+
 struct mrvl_priv {
 	/* Hot fields, used in fast path. */
 	struct pp2_bpool *bpool;  /**< BPool pointer */
@@ -82,6 +188,7 @@ struct mrvl_priv {
 	uint8_t uc_mc_flushed;
 	uint8_t vlan_flushed;
 	uint8_t isolated;
+	uint8_t multiseg;
 
 	struct pp2_ppio_params ppio_params;
 	struct pp2_cls_qos_tbl_params qos_tbl_params;
@@ -93,12 +200,26 @@ struct mrvl_priv {
 	uint32_t cls_tbl_pattern;
 	LIST_HEAD(mrvl_flows, rte_flow) flows;
 
-	struct pp2_cls_plcr *policer;
+	struct pp2_cls_plcr *default_policer;
+
+	LIST_HEAD(profiles, mrvl_mtr_profile) profiles;
+	LIST_HEAD(mtrs, mrvl_mtr) mtrs;
+	uint32_t used_plcrs;
+
+	LIST_HEAD(shaper_profiles, mrvl_tm_shaper_profile) shaper_profiles;
+	LIST_HEAD(nodes, mrvl_tm_node) nodes;
+	uint64_t rate_max;
 };
 
 /** Flow operations forward declaration. */
 extern const struct rte_flow_ops mrvl_flow_ops;
 
+/** Meter operations forward declaration. */
+extern const struct rte_mtr_ops mrvl_mtr_ops;
+
+/** Traffic manager operations forward declaration. */
+extern const struct rte_tm_ops mrvl_tm_ops;
+
 /** Current log type. */
 extern int mrvl_logtype;
 
diff --git a/drivers/net/mvpp2/mrvl_flow.c b/drivers/net/mvpp2/mrvl_flow.c
index ecc34192..ffd1dab9 100644
--- a/drivers/net/mvpp2/mrvl_flow.c
+++ b/drivers/net/mvpp2/mrvl_flow.c
@@ -11,13 +11,8 @@
 
 #include <arpa/inet.h>
 
-#ifdef container_of
-#undef container_of
-#endif
-
-#include "mrvl_ethdev.h"
+#include "mrvl_flow.h"
 #include "mrvl_qos.h"
-#include "env/mv_common.h" /* for BIT() */
 
 /** Number of rules in the classifier table. */
 #define MRVL_CLS_MAX_NUM_RULES 20
@@ -25,46 +20,6 @@
 /** Size of the classifier key and mask strings. */
 #define MRVL_CLS_STR_SIZE_MAX 40
 
-/** Parsed fields in processed rte_flow_item. */
-enum mrvl_parsed_fields {
-	/* eth flags */
-	F_DMAC =         BIT(0),
-	F_SMAC =         BIT(1),
-	F_TYPE =         BIT(2),
-	/* vlan flags */
-	F_VLAN_ID =      BIT(3),
-	F_VLAN_PRI =     BIT(4),
-	F_VLAN_TCI =     BIT(5), /* not supported by MUSDK yet */
-	/* ip4 flags */
-	F_IP4_TOS =      BIT(6),
-	F_IP4_SIP =      BIT(7),
-	F_IP4_DIP =      BIT(8),
-	F_IP4_PROTO =    BIT(9),
-	/* ip6 flags */
-	F_IP6_TC =       BIT(10), /* not supported by MUSDK yet */
-	F_IP6_SIP =      BIT(11),
-	F_IP6_DIP =      BIT(12),
-	F_IP6_FLOW =     BIT(13),
-	F_IP6_NEXT_HDR = BIT(14),
-	/* tcp flags */
-	F_TCP_SPORT =    BIT(15),
-	F_TCP_DPORT =    BIT(16),
-	/* udp flags */
-	F_UDP_SPORT =    BIT(17),
-	F_UDP_DPORT =    BIT(18),
-};
-
-/** PMD-specific definition of a flow rule handle. */
-struct rte_flow {
-	LIST_ENTRY(rte_flow) next;
-
-	enum mrvl_parsed_fields pattern;
-
-	struct pp2_cls_tbl_rule rule;
-	struct pp2_cls_cos_desc cos;
-	struct pp2_cls_tbl_action action;
-};
-
 static const enum rte_flow_item_type pattern_eth[] = {
 	RTE_FLOW_ITEM_TYPE_ETH,
 	RTE_FLOW_ITEM_TYPE_END
@@ -394,7 +349,8 @@ mrvl_parse_init(const struct rte_flow_item *item,
  *
  * @param spec Pointer to the specific flow item.
  * @param mask Pointer to the specific flow item's mask.
- * @param mask Pointer to the flow.
+ * @param parse_dst Parse either destination or source mac address.
+ * @param flow Pointer to the flow.
  * @return 0 in case of success, negative error value otherwise.
  */
 static int
@@ -613,6 +569,7 @@ mrvl_parse_ip4_dscp(const struct rte_flow_item_ipv4 *spec,
  *
  * @param spec Pointer to the specific flow item.
  * @param mask Pointer to the specific flow item's mask.
+ * @param parse_dst Parse either destination or source ip address.
  * @param flow Pointer to the flow.
  * @return 0 in case of success, negative error value otherwise.
  */
@@ -726,6 +683,7 @@ mrvl_parse_ip4_proto(const struct rte_flow_item_ipv4 *spec,
  *
  * @param spec Pointer to the specific flow item.
  * @param mask Pointer to the specific flow item's mask.
+ * @param parse_dst Parse either destination or source ipv6 address.
  * @param flow Pointer to the flow.
  * @return 0 in case of success, negative error value otherwise.
  */
@@ -874,6 +832,7 @@ mrvl_parse_ip6_next_hdr(const struct rte_flow_item_ipv6 *spec,
  *
  * @param spec Pointer to the specific flow item.
  * @param mask Pointer to the specific flow item's mask.
+ * @param parse_dst Parse either destination or source port.
  * @param flow Pointer to the flow.
  * @return 0 in case of success, negative error value otherwise.
  */
@@ -949,6 +908,7 @@ mrvl_parse_tcp_dport(const struct rte_flow_item_tcp *spec,
  *
  * @param spec Pointer to the specific flow item.
  * @param mask Pointer to the specific flow item's mask.
+ * @param parse_dst Parse either destination or source port.
  * @param flow Pointer to the flow.
  * @return 0 in case of success, negative error value otherwise.
  */
@@ -1022,7 +982,6 @@ mrvl_parse_udp_dport(const struct rte_flow_item_udp *spec,
  * @param item Pointer to the flow item.
  * @param flow Pointer to the flow.
  * @param error Pointer to the flow error.
- * @param fields Pointer to the parsed parsed fields enum.
  * @returns 0 on success, negative value otherwise.
  */
 static int
@@ -1073,7 +1032,6 @@ out:
  * @param item Pointer to the flow item.
  * @param flow Pointer to the flow.
  * @param error Pointer to the flow error.
- * @param fields Pointer to the parsed parsed fields enum.
  * @returns 0 on success, negative value otherwise.
  */
 static int
@@ -1139,7 +1097,6 @@ out:
  * @param item Pointer to the flow item.
  * @param flow Pointer to the flow.
  * @param error Pointer to the flow error.
- * @param fields Pointer to the parsed parsed fields enum.
  * @returns 0 on success, negative value otherwise.
  */
 static int
@@ -1205,7 +1162,6 @@ out:
  * @param item Pointer to the flow item.
  * @param flow Pointer to the flow.
  * @param error Pointer to the flow error.
- * @param fields Pointer to the parsed parsed fields enum.
  * @returns 0 on success, negative value otherwise.
  */
 static int
@@ -1276,7 +1232,6 @@ out:
  * @param item Pointer to the flow item.
  * @param flow Pointer to the flow.
  * @param error Pointer to the flow error.
- * @param fields Pointer to the parsed parsed fields enum.
  * @returns 0 on success, negative value otherwise.
  */
 static int
@@ -1332,7 +1287,6 @@ out:
  * @param item Pointer to the flow item.
  * @param flow Pointer to the flow.
  * @param error Pointer to the flow error.
- * @param fields Pointer to the parsed parsed fields enum.
  * @returns 0 on success, negative value otherwise.
  */
 static int
@@ -1981,6 +1935,7 @@ mrvl_parse_pattern_ip6_tcp(const struct rte_flow_item pattern[],
  * @param pattern Pointer to the flow pattern table.
  * @param flow Pointer to the flow.
  * @param error Pointer to the flow error.
+ * @param ip6 1 to parse ip6 item, 0 to parse ip4 item.
  * @returns 0 in case of success, negative value otherwise.
  */
 static int
@@ -2300,19 +2255,59 @@ mrvl_flow_parse_actions(struct mrvl_priv *priv,
 			flow->action.type = PP2_CLS_TBL_ACT_DONE;
 			flow->action.cos = &flow->cos;
 			specified++;
+		} else if (action->type == RTE_FLOW_ACTION_TYPE_METER) {
+			const struct rte_flow_action_meter *meter;
+			struct mrvl_mtr *mtr;
+
+			meter = action->conf;
+			if (!meter)
+				return -rte_flow_error_set(error, EINVAL,
+						RTE_FLOW_ERROR_TYPE_ACTION,
+						NULL, "Invalid meter\n");
+
+			LIST_FOREACH(mtr, &priv->mtrs, next)
+				if (mtr->mtr_id == meter->mtr_id)
+					break;
+
+			if (!mtr)
+				return -rte_flow_error_set(error, EINVAL,
+						RTE_FLOW_ERROR_TYPE_ACTION,
+						NULL,
+						"Meter id does not exist\n");
+
+			if (!mtr->shared && mtr->refcnt)
+				return -rte_flow_error_set(error, EPERM,
+						RTE_FLOW_ERROR_TYPE_ACTION,
+						NULL,
+						"Meter cannot be shared\n");
+
+			/*
+			 * In case cos has already been set
+			 * do not modify it.
+			 */
+			if (!flow->cos.ppio) {
+				flow->cos.ppio = priv->ppio;
+				flow->cos.tc = 0;
+			}
+
+			flow->action.type = PP2_CLS_TBL_ACT_DONE;
+			flow->action.cos = &flow->cos;
+			flow->action.plcr = mtr->enabled ? mtr->plcr : NULL;
+			flow->mtr = mtr;
+			mtr->refcnt++;
+			specified++;
 		} else {
 			rte_flow_error_set(error, ENOTSUP,
 					   RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					   "Action not supported");
 			return -rte_errno;
 		}
-
 	}
 
 	if (!specified) {
 		rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL, "Action not specified");
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "Action not specified");
 		return -rte_errno;
 	}
 
@@ -2350,6 +2345,12 @@ mrvl_flow_parse(struct mrvl_priv *priv, const struct rte_flow_attr *attr,
 	return mrvl_flow_parse_actions(priv, actions, flow, error);
 }
 
+/**
+ * Get engine type for the given flow.
+ *
+ * @param field Pointer to the flow.
+ * @returns The type of the engine.
+ */
 static inline enum pp2_cls_tbl_type
 mrvl_engine_type(const struct rte_flow *flow)
 {
@@ -2369,6 +2370,13 @@ mrvl_engine_type(const struct rte_flow *flow)
 	return PP2_CLS_TBL_MASKABLE;
 }
 
+/**
+ * Create classifier table.
+ *
+ * @param dev Pointer to the device.
+ * @param flow Pointer to the very first flow.
+ * @returns 0 in case of success, negative value otherwise.
+ */
 static int
 mrvl_create_cls_table(struct rte_eth_dev *dev, struct rte_flow *first_flow)
 {
@@ -2429,7 +2437,8 @@ mrvl_create_cls_table(struct rte_eth_dev *dev, struct rte_flow *first_flow)
 
 	if (first_flow->pattern & F_IP4_TOS) {
 		key->proto_field[key->num_fields].proto = MV_NET_PROTO_IP4;
-		key->proto_field[key->num_fields].field.ipv4 = MV_NET_IP4_F_TOS;
+		key->proto_field[key->num_fields].field.ipv4 =
+							MV_NET_IP4_F_DSCP;
 		key->key_size += 1;
 		key->num_fields += 1;
 	}
@@ -2649,13 +2658,18 @@ mrvl_flow_remove(struct mrvl_priv *priv, struct rte_flow *flow,
 
 	mrvl_free_all_key_mask(&flow->rule);
 
+	if (flow->mtr) {
+		flow->mtr->refcnt--;
+		flow->mtr = NULL;
+	}
+
 	return 0;
 }
 
 /**
  * DPDK flow destroy callback called when flow is to be removed.
  *
- * @param priv Pointer to the port's private data.
+ * @param dev Pointer to the device.
  * @param flow Pointer to the flow.
  * @param error Pointer to the flow error.
  * @returns 0 in case of success, negative value otherwise.
@@ -2777,3 +2791,34 @@ const struct rte_flow_ops mrvl_flow_ops = {
 	.flush = mrvl_flow_flush,
 	.isolate = mrvl_flow_isolate
 };
+
+/**
+ * Initialize flow resources.
+ *
+ * @param dev Pointer to the device.
+ */
+void
+mrvl_flow_init(struct rte_eth_dev *dev)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+
+	LIST_INIT(&priv->flows);
+}
+
+/**
+ * Cleanup flow resources.
+ *
+ * @param dev Pointer to the device.
+ */
+void
+mrvl_flow_deinit(struct rte_eth_dev *dev)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+
+	mrvl_flow_flush(dev, NULL);
+
+	if (priv->cls_tbl) {
+		pp2_cls_tbl_deinit(priv->cls_tbl);
+		priv->cls_tbl = NULL;
+	}
+}
diff --git a/drivers/net/mvpp2/mrvl_flow.h b/drivers/net/mvpp2/mrvl_flow.h
new file mode 100644
index 00000000..f63747c1
--- /dev/null
+++ b/drivers/net/mvpp2/mrvl_flow.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Marvell International Ltd.
+ * Copyright(c) 2018 Semihalf.
+ * All rights reserved.
+ */
+
+#ifndef _MRVL_FLOW_H_
+#define _MRVL_FLOW_H_
+
+#include "mrvl_ethdev.h"
+
+void mrvl_flow_init(struct rte_eth_dev *dev);
+void mrvl_flow_deinit(struct rte_eth_dev *dev);
+
+#endif /* _MRVL_FLOW_H_ */
diff --git a/drivers/net/mvpp2/mrvl_mtr.c b/drivers/net/mvpp2/mrvl_mtr.c
new file mode 100644
index 00000000..9cd53bed
--- /dev/null
+++ b/drivers/net/mvpp2/mrvl_mtr.c
@@ -0,0 +1,512 @@
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Marvell International Ltd.
+ * Copyright(c) 2018 Semihalf.
+ * All rights reserved.
+ */
+
+#include <rte_log.h>
+#include <rte_malloc.h>
+
+#include "mrvl_mtr.h"
+
+/** Maximum meter rate */
+#define MRVL_SRTCM_RFC2697_CIR_MAX 1023000
+
+/** Invalid plcr bit */
+#define MRVL_PLCR_BIT_INVALID -1
+
+/**
+ * Return meter object capabilities.
+ *
+ * @param dev Pointer to the device (unused).
+ * @param cap Pointer to the meter object capabilities.
+ * @param error Pointer to the error (unused).
+ * @returns 0 always.
+ */
+static int
+mrvl_capabilities_get(struct rte_eth_dev *dev __rte_unused,
+			  struct rte_mtr_capabilities *cap,
+			  struct rte_mtr_error *error __rte_unused)
+{
+	struct rte_mtr_capabilities capa = {
+		.n_max = PP2_CLS_PLCR_NUM,
+		.n_shared_max = PP2_CLS_PLCR_NUM,
+		.shared_n_flows_per_mtr_max = -1,
+		.meter_srtcm_rfc2697_n_max = PP2_CLS_PLCR_NUM,
+		.meter_rate_max = MRVL_SRTCM_RFC2697_CIR_MAX,
+	};
+
+	memcpy(cap, &capa, sizeof(capa));
+
+	return 0;
+}
+
+/**
+ * Get profile using it's id.
+ *
+ * @param priv Pointer to the port's private data.
+ * @param meter_profile_id Profile id used by the meter.
+ * @returns Pointer to the profile if exists, NULL otherwise.
+ */
+static struct mrvl_mtr_profile *
+mrvl_mtr_profile_from_id(struct mrvl_priv *priv, uint32_t meter_profile_id)
+{
+	struct mrvl_mtr_profile *profile = NULL;
+
+	LIST_FOREACH(profile, &priv->profiles, next)
+		if (profile->profile_id == meter_profile_id)
+			break;
+
+	return profile;
+}
+
+/**
+ * Add profile to the list of profiles.
+ *
+ * @param dev Pointer to the device.
+ * @param meter_profile_id Id of the new profile.
+ * @param profile Pointer to the profile configuration.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_meter_profile_add(struct rte_eth_dev *dev, uint32_t meter_profile_id,
+		       struct rte_mtr_meter_profile *profile,
+		       struct rte_mtr_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_mtr_profile *prof;
+
+	if (!profile)
+		return -rte_mtr_error_set(error, EINVAL,
+					  RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					  NULL, NULL);
+
+	if (profile->alg != RTE_MTR_SRTCM_RFC2697)
+		return -rte_mtr_error_set(error, EINVAL,
+					  RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "Only srTCM RFC 2697 is supported\n");
+
+	prof = mrvl_mtr_profile_from_id(priv, meter_profile_id);
+	if (prof)
+		return -rte_mtr_error_set(error, EEXIST,
+					  RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+					  NULL, "Profile id already exists\n");
+
+	prof = rte_zmalloc_socket(NULL, sizeof(*prof), 0, rte_socket_id());
+	if (!prof)
+		return -rte_mtr_error_set(error, ENOMEM,
+					  RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					  NULL, NULL);
+
+	prof->profile_id = meter_profile_id;
+	memcpy(&prof->profile, profile, sizeof(*profile));
+
+	LIST_INSERT_HEAD(&priv->profiles, prof, next);
+
+	return 0;
+}
+
+/**
+ * Remove profile from the list of profiles.
+ *
+ * @param dev Pointer to the device.
+ * @param meter_profile_id Id of the profile to remove.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_meter_profile_delete(struct rte_eth_dev *dev,
+			      uint32_t meter_profile_id,
+			      struct rte_mtr_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_mtr_profile *profile;
+
+	profile = mrvl_mtr_profile_from_id(priv, meter_profile_id);
+	if (!profile)
+		return -rte_mtr_error_set(error, ENODEV,
+					  RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+					  NULL, "Profile id does not exist\n");
+
+	if (profile->refcnt)
+		return -rte_mtr_error_set(error, EPERM,
+					  RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+					  NULL, "Profile is used\n");
+
+	LIST_REMOVE(profile, next);
+	rte_free(profile);
+
+	return 0;
+}
+
+/**
+ * Get meter using it's id.
+ *
+ * @param priv Pointer to port's private data.
+ * @param mtr_id Id of the meter.
+ * @returns Pointer to the meter if exists, NULL otherwise.
+ */
+static struct mrvl_mtr *
+mrvl_mtr_from_id(struct mrvl_priv *priv, uint32_t mtr_id)
+{
+	struct mrvl_mtr *mtr = NULL;
+
+	LIST_FOREACH(mtr, &priv->mtrs, next)
+		if (mtr->mtr_id == mtr_id)
+			break;
+
+	return mtr;
+}
+
+/**
+ * Reserve a policer bit in a bitmap.
+ *
+ * @param plcrs Pointer to the policers bitmap.
+ * @returns Reserved bit number on success, negative value otherwise.
+ */
+static int
+mrvl_reserve_plcr(uint32_t *plcrs)
+{
+	uint32_t i, num;
+
+	num = PP2_CLS_PLCR_NUM;
+	if (num > sizeof(uint32_t) * 8) {
+		num = sizeof(uint32_t) * 8;
+		MRVL_LOG(WARNING, "Plcrs number was limited to 32.");
+	}
+
+	for (i = 0; i < num; i++) {
+		uint32_t bit = BIT(i);
+
+		if (!(*plcrs & bit)) {
+			*plcrs |= bit;
+
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+/**
+ * Enable meter object.
+ *
+ * @param dev Pointer to the device.
+ * @param mtr_id Id of the meter.
+ * @param error Pointer to the error.
+ * @returns 0 in success, negative value otherwise.
+ */
+static int
+mrvl_meter_enable(struct rte_eth_dev *dev, uint32_t mtr_id,
+		  struct rte_mtr_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_mtr *mtr = mrvl_mtr_from_id(priv, mtr_id);
+	struct pp2_cls_plcr_params params;
+	char match[MRVL_MATCH_LEN];
+	struct rte_flow *flow;
+	int ret;
+
+	if (!priv->ppio)
+		return -rte_mtr_error_set(error, EPERM,
+					  RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					  NULL, "Port is uninitialized\n");
+
+	if (!mtr)
+		return -rte_mtr_error_set(error, ENODEV,
+					  RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
+					  "Meter id does not exist\n");
+
+	if (mtr->plcr)
+		goto skip;
+
+	mtr->plcr_bit = mrvl_reserve_plcr(&priv->used_plcrs);
+	if (mtr->plcr_bit < 0)
+		return -rte_mtr_error_set(error, ENOSPC,
+					  RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "Failed to reserve plcr entry\n");
+
+	memset(&params, 0, sizeof(params));
+	snprintf(match, sizeof(match), "policer-%d:%d", priv->pp_id,
+		 mtr->plcr_bit);
+	params.match = match;
+	params.token_unit = PP2_CLS_PLCR_BYTES_TOKEN_UNIT;
+	params.color_mode = PP2_CLS_PLCR_COLOR_BLIND_MODE;
+	params.cir = mtr->profile->profile.srtcm_rfc2697.cir;
+	params.cbs = mtr->profile->profile.srtcm_rfc2697.cbs;
+	params.ebs = mtr->profile->profile.srtcm_rfc2697.ebs;
+
+	ret = pp2_cls_plcr_init(&params, &mtr->plcr);
+	if (ret) {
+		priv->used_plcrs &= ~BIT(mtr->plcr_bit);
+		mtr->plcr_bit = MRVL_PLCR_BIT_INVALID;
+
+		return -rte_mtr_error_set(error, -ret,
+					  RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					  NULL, "Failed to setup policer\n");
+	}
+
+	mtr->enabled = 1;
+skip:
+	/* iterate over flows that have this mtr attached */
+	LIST_FOREACH(flow, &priv->flows, next) {
+		if (flow->mtr != mtr)
+			continue;
+
+		flow->action.plcr = mtr->plcr;
+
+		ret = pp2_cls_tbl_modify_rule(priv->cls_tbl, &flow->rule,
+					      &flow->action);
+		if (ret)
+			return -rte_mtr_error_set(error, -ret,
+					  RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					  NULL, "Failed to update cls rule\n");
+	}
+
+	return 0;
+}
+
+/**
+ * Disable meter object.
+ *
+ * @param dev Pointer to the device.
+ * @param mtr Id of the meter.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_meter_disable(struct rte_eth_dev *dev, uint32_t mtr_id,
+		       struct rte_mtr_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_mtr *mtr = mrvl_mtr_from_id(priv, mtr_id);
+	struct rte_flow *flow;
+	int ret;
+
+	if (!mtr)
+		return -rte_mtr_error_set(error, ENODEV,
+					  RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
+					  "Meter id does not exist\n");
+
+	LIST_FOREACH(flow, &priv->flows, next) {
+		if (flow->mtr != mtr)
+			continue;
+
+		flow->action.plcr = NULL;
+
+		ret = pp2_cls_tbl_modify_rule(priv->cls_tbl, &flow->rule,
+					      &flow->action);
+		if (ret)
+			return -rte_mtr_error_set(error, -ret,
+					RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					NULL, "Failed to disable meter\n");
+	}
+
+	mtr->enabled = 0;
+
+	return 0;
+}
+
+/**
+ * Create new meter.
+ *
+ * @param dev Pointer to the device.
+ * @param mtr_id Id of the meter.
+ * @param params Pointer to the meter parameters.
+ * @param shared Flags indicating whether meter is shared.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_create(struct rte_eth_dev *dev, uint32_t mtr_id,
+	    struct rte_mtr_params *params, int shared,
+	    struct rte_mtr_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_mtr_profile *profile;
+	struct mrvl_mtr *mtr;
+
+	mtr = mrvl_mtr_from_id(priv, mtr_id);
+	if (mtr)
+		return -rte_mtr_error_set(error, EEXIST,
+					  RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
+					  "Meter id already exists\n");
+
+	mtr = rte_zmalloc_socket(NULL, sizeof(*mtr), 0, rte_socket_id());
+	if (!mtr)
+		return -rte_mtr_error_set(error, ENOMEM,
+					  RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					  NULL, NULL);
+
+	profile = mrvl_mtr_profile_from_id(priv, params->meter_profile_id);
+	if (!profile)
+		return -rte_mtr_error_set(error, EINVAL,
+					  RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+					  NULL, "Profile id does not exist\n");
+
+	mtr->shared = shared;
+	mtr->mtr_id = mtr_id;
+	mtr->plcr_bit = MRVL_PLCR_BIT_INVALID;
+	mtr->profile = profile;
+	profile->refcnt++;
+	LIST_INSERT_HEAD(&priv->mtrs, mtr, next);
+
+	if (params->meter_enable)
+		return mrvl_meter_enable(dev, mtr_id, error);
+
+	return 0;
+}
+
+/**
+ * Destroy meter object.
+ *
+ * @param dev Pointer to the device.
+ * @param mtr_id Id of the meter object.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_destroy(struct rte_eth_dev *dev, uint32_t mtr_id,
+		 struct rte_mtr_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_mtr *mtr;
+
+	if (!priv->ppio)
+		return -rte_mtr_error_set(error, EPERM,
+					  RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					  NULL, "Port is uninitialized\n");
+
+	mtr = mrvl_mtr_from_id(priv, mtr_id);
+	if (!mtr)
+		return -rte_mtr_error_set(error, EEXIST,
+					  RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
+					  "Meter id does not exist\n");
+
+	if (mtr->refcnt)
+		return -rte_mtr_error_set(error, EPERM,
+					  RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
+					  "Meter is used\n");
+
+	LIST_REMOVE(mtr, next);
+	mtr->profile->refcnt--;
+
+	if (mtr->plcr_bit != MRVL_PLCR_BIT_INVALID)
+		priv->used_plcrs &= ~BIT(mtr->plcr_bit);
+
+	if (mtr->plcr)
+		pp2_cls_plcr_deinit(mtr->plcr);
+
+	rte_free(mtr);
+
+	return 0;
+}
+
+/**
+ * Update profile used by the meter.
+ *
+ * @param dev Pointer to the device.
+ * @param mtr_id Id of the meter object.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_meter_profile_update(struct rte_eth_dev *dev, uint32_t mtr_id,
+			  uint32_t meter_profile_id,
+			  struct rte_mtr_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_mtr_profile *profile;
+	struct mrvl_mtr *mtr;
+	int ret, enabled;
+
+	if (!priv->ppio)
+		return -rte_mtr_error_set(error, EPERM,
+					  RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					  NULL, "Port is uninitialized\n");
+
+	mtr = mrvl_mtr_from_id(priv, mtr_id);
+	if (!mtr)
+		return -rte_mtr_error_set(error, EEXIST,
+					  RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
+					  "Meter id does not exist\n");
+
+	profile = mrvl_mtr_profile_from_id(priv, meter_profile_id);
+	if (!profile)
+		return -rte_mtr_error_set(error, EINVAL,
+					  RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+					  NULL, "Profile id does not exist\n");
+
+	ret = mrvl_meter_disable(dev, mtr_id, error);
+	if (ret)
+		return -rte_mtr_error_set(error, EPERM,
+					  RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
+					  NULL);
+
+	if (mtr->plcr) {
+		enabled = 1;
+		pp2_cls_plcr_deinit(mtr->plcr);
+		mtr->plcr = NULL;
+	}
+
+	mtr->profile->refcnt--;
+	mtr->profile = profile;
+	profile->refcnt++;
+
+	if (enabled)
+		return mrvl_meter_enable(dev, mtr_id, error);
+
+	return 0;
+}
+
+const struct rte_mtr_ops mrvl_mtr_ops = {
+	.capabilities_get = mrvl_capabilities_get,
+	.meter_profile_add = mrvl_meter_profile_add,
+	.meter_profile_delete = mrvl_meter_profile_delete,
+	.create = mrvl_create,
+	.destroy = mrvl_destroy,
+	.meter_enable = mrvl_meter_enable,
+	.meter_disable = mrvl_meter_disable,
+	.meter_profile_update = mrvl_meter_profile_update,
+};
+
+/**
+ * Initialize metering resources.
+ *
+ * @param dev Pointer to the device.
+ */
+void
+mrvl_mtr_init(struct rte_eth_dev *dev)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+
+	LIST_INIT(&priv->profiles);
+	LIST_INIT(&priv->mtrs);
+}
+
+/**
+ * Cleanup metering resources.
+ *
+ * @param dev Pointer to the device.
+ */
+void
+mrvl_mtr_deinit(struct rte_eth_dev *dev)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_mtr_profile *profile, *tmp_profile;
+	struct mrvl_mtr *mtr, *tmp_mtr;
+
+	for (mtr = LIST_FIRST(&priv->mtrs);
+	     mtr && (tmp_mtr = LIST_NEXT(mtr, next), 1);
+	     mtr = tmp_mtr)
+		mrvl_destroy(dev, mtr->mtr_id, NULL);
+
+	for (profile = LIST_FIRST(&priv->profiles);
+	     profile && (tmp_profile = LIST_NEXT(profile, next), 1);
+	     profile = tmp_profile)
+		mrvl_meter_profile_delete(dev, profile->profile_id, NULL);
+}
diff --git a/drivers/net/mvpp2/mrvl_mtr.h b/drivers/net/mvpp2/mrvl_mtr.h
new file mode 100644
index 00000000..302a20fb
--- /dev/null
+++ b/drivers/net/mvpp2/mrvl_mtr.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Marvell International Ltd.
+ * Copyright(c) 2018 Semihalf.
+ * All rights reserved.
+ */
+
+#ifndef _MRVL_MTR_H_
+#define _MRVL_MTR_H_
+
+#include "mrvl_ethdev.h"
+
+void mrvl_mtr_init(struct rte_eth_dev *dev);
+void mrvl_mtr_deinit(struct rte_eth_dev *dev);
+
+#endif /* _MRVL_MTR_H_ */
diff --git a/drivers/net/mvpp2/mrvl_qos.c b/drivers/net/mvpp2/mrvl_qos.c
index 71856c1a..7fd97030 100644
--- a/drivers/net/mvpp2/mrvl_qos.c
+++ b/drivers/net/mvpp2/mrvl_qos.c
@@ -15,15 +15,6 @@
 #include <rte_malloc.h>
 #include <rte_string_fns.h>
 
-/* Unluckily, container_of is defined by both DPDK and MUSDK,
- * we'll declare only one version.
- *
- * Note that it is not used in this PMD anyway.
- */
-#ifdef container_of
-#undef container_of
-#endif
-
 #include "mrvl_qos.h"
 
 /* Parsing tokens. Defined conveniently, so that any correction is easy. */
@@ -51,7 +42,8 @@
 #define MRVL_TOK_WRR_WEIGHT "wrr_weight"
 
 /* policer specific configuration tokens */
-#define MRVL_TOK_PLCR_ENABLE "policer_enable"
+#define MRVL_TOK_PLCR "policer"
+#define MRVL_TOK_PLCR_DEFAULT "default_policer"
 #define MRVL_TOK_PLCR_UNIT "token_unit"
 #define MRVL_TOK_PLCR_UNIT_BYTES "bytes"
 #define MRVL_TOK_PLCR_UNIT_PACKETS "packets"
@@ -332,6 +324,7 @@ parse_tc_cfg(struct rte_cfgfile *file, int port, int tc,
 	if (rte_cfgfile_num_sections(file, sec_name, strlen(sec_name)) <= 0)
 		return 0;
 
+	cfg->port[port].use_global_defaults = 0;
 	entry = rte_cfgfile_get_entry(file, sec_name, MRVL_TOK_RXQ);
 	if (entry) {
 		n = get_entry_values(entry,
@@ -377,6 +370,9 @@ parse_tc_cfg(struct rte_cfgfile *file, int port, int tc,
 		cfg->port[port].tc[tc].dscps = n;
 	}
 
+	if (!cfg->port[port].setup_policer)
+		return 0;
+
 	entry = rte_cfgfile_get_entry(file, sec_name,
 			MRVL_TOK_PLCR_DEFAULT_COLOR);
 	if (entry) {
@@ -399,6 +395,85 @@ parse_tc_cfg(struct rte_cfgfile *file, int port, int tc,
 }
 
 /**
+ * Parse default port policer.
+ *
+ * @param file Config file handle.
+ * @param sec_name Section name with policer configuration
+ * @param port Port number.
+ * @param cfg[out] Parsing results.
+ * @returns 0 in case of success, negative value otherwise.
+ */
+static int
+parse_policer(struct rte_cfgfile *file, int port, const char *sec_name,
+		struct mrvl_qos_cfg *cfg)
+{
+	const char *entry;
+	uint32_t val;
+
+	/* Read policer token unit */
+	entry = rte_cfgfile_get_entry(file, sec_name, MRVL_TOK_PLCR_UNIT);
+	if (entry) {
+		if (!strncmp(entry, MRVL_TOK_PLCR_UNIT_BYTES,
+					sizeof(MRVL_TOK_PLCR_UNIT_BYTES))) {
+			cfg->port[port].policer_params.token_unit =
+				PP2_CLS_PLCR_BYTES_TOKEN_UNIT;
+		} else if (!strncmp(entry, MRVL_TOK_PLCR_UNIT_PACKETS,
+					sizeof(MRVL_TOK_PLCR_UNIT_PACKETS))) {
+			cfg->port[port].policer_params.token_unit =
+				PP2_CLS_PLCR_PACKETS_TOKEN_UNIT;
+		} else {
+			MRVL_LOG(ERR, "Unknown token: %s", entry);
+			return -1;
+		}
+	}
+
+	/* Read policer color mode */
+	entry = rte_cfgfile_get_entry(file, sec_name, MRVL_TOK_PLCR_COLOR);
+	if (entry) {
+		if (!strncmp(entry, MRVL_TOK_PLCR_COLOR_BLIND,
+					sizeof(MRVL_TOK_PLCR_COLOR_BLIND))) {
+			cfg->port[port].policer_params.color_mode =
+				PP2_CLS_PLCR_COLOR_BLIND_MODE;
+		} else if (!strncmp(entry, MRVL_TOK_PLCR_COLOR_AWARE,
+					sizeof(MRVL_TOK_PLCR_COLOR_AWARE))) {
+			cfg->port[port].policer_params.color_mode =
+				PP2_CLS_PLCR_COLOR_AWARE_MODE;
+		} else {
+			MRVL_LOG(ERR, "Error in parsing: %s", entry);
+			return -1;
+		}
+	}
+
+	/* Read policer cir */
+	entry = rte_cfgfile_get_entry(file, sec_name, MRVL_TOK_PLCR_CIR);
+	if (entry) {
+		if (get_val_securely(entry, &val) < 0)
+			return -1;
+		cfg->port[port].policer_params.cir = val;
+	}
+
+	/* Read policer cbs */
+	entry = rte_cfgfile_get_entry(file, sec_name, MRVL_TOK_PLCR_CBS);
+	if (entry) {
+		if (get_val_securely(entry, &val) < 0)
+			return -1;
+		cfg->port[port].policer_params.cbs = val;
+	}
+
+	/* Read policer ebs */
+	entry = rte_cfgfile_get_entry(file, sec_name, MRVL_TOK_PLCR_EBS);
+	if (entry) {
+		if (get_val_securely(entry, &val) < 0)
+			return -1;
+		cfg->port[port].policer_params.ebs = val;
+	}
+
+	cfg->port[port].setup_policer = 1;
+
+	return 0;
+}
+
+/**
  * Parse QoS configuration - rte_kvargs_process handler.
  *
  * Opens configuration file and parses its content.
@@ -444,110 +519,15 @@ mrvl_get_qoscfg(const char *key __rte_unused, const char *path,
 		snprintf(sec_name, sizeof(sec_name), "%s %d %s",
 			MRVL_TOK_PORT, n, MRVL_TOK_DEFAULT);
 
+		/* Use global defaults, unless an override occurs */
+		(*cfg)->port[n].use_global_defaults = 1;
+
 		/* Skip ports non-existing in configuration. */
 		if (rte_cfgfile_num_sections(file, sec_name,
 				strlen(sec_name)) <= 0) {
-			(*cfg)->port[n].use_global_defaults = 1;
-			(*cfg)->port[n].mapping_priority =
-				PP2_CLS_QOS_TBL_VLAN_IP_PRI;
 			continue;
 		}
 
-		entry = rte_cfgfile_get_entry(file, sec_name,
-				MRVL_TOK_DEFAULT_TC);
-		if (entry) {
-			if (get_val_securely(entry, &val) < 0 ||
-				val > USHRT_MAX)
-				return -1;
-			(*cfg)->port[n].default_tc = (uint8_t)val;
-		} else {
-			MRVL_LOG(ERR,
-				"Default Traffic Class required in custom configuration!");
-			return -1;
-		}
-
-		entry = rte_cfgfile_get_entry(file, sec_name,
-				MRVL_TOK_PLCR_ENABLE);
-		if (entry) {
-			if (get_val_securely(entry, &val) < 0)
-				return -1;
-			(*cfg)->port[n].policer_enable = val;
-		}
-
-		if ((*cfg)->port[n].policer_enable) {
-			enum pp2_cls_plcr_token_unit unit;
-
-			/* Read policer token unit */
-			entry = rte_cfgfile_get_entry(file, sec_name,
-					MRVL_TOK_PLCR_UNIT);
-			if (entry) {
-				if (!strncmp(entry, MRVL_TOK_PLCR_UNIT_BYTES,
-					sizeof(MRVL_TOK_PLCR_UNIT_BYTES))) {
-					unit = PP2_CLS_PLCR_BYTES_TOKEN_UNIT;
-				} else if (!strncmp(entry,
-						MRVL_TOK_PLCR_UNIT_PACKETS,
-					sizeof(MRVL_TOK_PLCR_UNIT_PACKETS))) {
-					unit = PP2_CLS_PLCR_PACKETS_TOKEN_UNIT;
-				} else {
-					MRVL_LOG(ERR, "Unknown token: %s",
-						entry);
-					return -1;
-				}
-				(*cfg)->port[n].policer_params.token_unit =
-					unit;
-			}
-
-			/* Read policer color mode */
-			entry = rte_cfgfile_get_entry(file, sec_name,
-					MRVL_TOK_PLCR_COLOR);
-			if (entry) {
-				enum pp2_cls_plcr_color_mode mode;
-
-				if (!strncmp(entry, MRVL_TOK_PLCR_COLOR_BLIND,
-					sizeof(MRVL_TOK_PLCR_COLOR_BLIND))) {
-					mode = PP2_CLS_PLCR_COLOR_BLIND_MODE;
-				} else if (!strncmp(entry,
-						MRVL_TOK_PLCR_COLOR_AWARE,
-					sizeof(MRVL_TOK_PLCR_COLOR_AWARE))) {
-					mode = PP2_CLS_PLCR_COLOR_AWARE_MODE;
-				} else {
-					MRVL_LOG(ERR,
-						"Error in parsing: %s",
-						entry);
-					return -1;
-				}
-				(*cfg)->port[n].policer_params.color_mode =
-					mode;
-			}
-
-			/* Read policer cir */
-			entry = rte_cfgfile_get_entry(file, sec_name,
-					MRVL_TOK_PLCR_CIR);
-			if (entry) {
-				if (get_val_securely(entry, &val) < 0)
-					return -1;
-				(*cfg)->port[n].policer_params.cir = val;
-			}
-
-			/* Read policer cbs */
-			entry = rte_cfgfile_get_entry(file, sec_name,
-					MRVL_TOK_PLCR_CBS);
-			if (entry) {
-				if (get_val_securely(entry, &val) < 0)
-					return -1;
-				(*cfg)->port[n].policer_params.cbs = val;
-			}
-
-			/* Read policer ebs */
-			entry = rte_cfgfile_get_entry(file, sec_name,
-					MRVL_TOK_PLCR_EBS);
-			if (entry) {
-				if (get_val_securely(entry, &val) < 0)
-					return -1;
-				(*cfg)->port[n].policer_params.ebs = val;
-			}
-		}
-
 		/*
 		 * Read per-port rate limiting. Setting that will
 		 * disable per-queue rate limiting.
@@ -581,6 +561,7 @@ mrvl_get_qoscfg(const char *key __rte_unused, const char *path,
 		entry = rte_cfgfile_get_entry(file, sec_name,
 				MRVL_TOK_MAPPING_PRIORITY);
 		if (entry) {
+			(*cfg)->port[n].use_global_defaults = 0;
 			if (!strncmp(entry, MRVL_TOK_VLAN_IP,
 				sizeof(MRVL_TOK_VLAN_IP)))
 				(*cfg)->port[n].mapping_priority =
@@ -606,6 +587,21 @@ mrvl_get_qoscfg(const char *key __rte_unused, const char *path,
 				PP2_CLS_QOS_TBL_VLAN_IP_PRI;
 		}
 
+		/* Parse policer configuration (if any) */
+		entry = rte_cfgfile_get_entry(file, sec_name,
+				MRVL_TOK_PLCR_DEFAULT);
+		if (entry) {
+			(*cfg)->port[n].use_global_defaults = 0;
+			if (get_val_securely(entry, &val) < 0)
+				return -1;
+
+			snprintf(sec_name, sizeof(sec_name), "%s %d",
+					MRVL_TOK_PLCR, val);
+			ret = parse_policer(file, n, sec_name, *cfg);
+			if (ret)
+				return -1;
+		}
+
 		for (i = 0; i < MRVL_PP2_RXQ_MAX; ++i) {
 			ret = get_outq_cfg(file, n, i, *cfg);
 			if (ret < 0)
@@ -621,6 +617,21 @@ mrvl_get_qoscfg(const char *key __rte_unused, const char *path,
 					"Error %d parsing port %d tc %d!\n",
 					ret, n, i);
 		}
+
+		entry = rte_cfgfile_get_entry(file, sec_name,
+					      MRVL_TOK_DEFAULT_TC);
+		if (entry) {
+			if (get_val_securely(entry, &val) < 0 ||
+			    val > USHRT_MAX)
+				return -1;
+			(*cfg)->port[n].default_tc = (uint8_t)val;
+		} else {
+			if ((*cfg)->port[n].use_global_defaults == 0) {
+				MRVL_LOG(ERR,
+					 "Default Traffic Class required in custom configuration!");
+				return -1;
+			}
+		}
 	}
 
 	return 0;
@@ -643,7 +654,7 @@ setup_tc(struct pp2_ppio_tc_params *param, uint8_t inqs,
 	struct pp2_ppio_inq_params *inq_params;
 
 	param->pkt_offset = MRVL_PKT_OFFS;
-	param->pools[0] = bpool;
+	param->pools[0][0] = bpool;
 	param->default_color = color;
 
 	inq_params = rte_zmalloc_socket("inq_params",
@@ -668,6 +679,7 @@ setup_tc(struct pp2_ppio_tc_params *param, uint8_t inqs,
  *
  * @param priv Port's private data.
  * @param params Pointer to the policer's configuration.
+ * @param plcr_id Policer id.
  * @returns 0 in case of success, negative values otherwise.
  */
 static int
@@ -676,17 +688,23 @@ setup_policer(struct mrvl_priv *priv, struct pp2_cls_plcr_params *params)
 	char match[16];
 	int ret;
 
-	snprintf(match, sizeof(match), "policer-%d:%d\n",
-			priv->pp_id, priv->ppio_id);
+	/*
+	 * At this point no other policers are used which means
+	 * any policer can be picked up and used as a default one.
+	 *
+	 * Lets use 0th then.
+	 */
+	sprintf(match, "policer-%d:%d\n", priv->pp_id, 0);
 	params->match = match;
 
-	ret = pp2_cls_plcr_init(params, &priv->policer);
+	ret = pp2_cls_plcr_init(params, &priv->default_policer);
 	if (ret) {
 		MRVL_LOG(ERR, "Failed to setup %s", match);
 		return -1;
 	}
 
-	priv->ppio_params.inqs_params.plcr = priv->policer;
+	priv->ppio_params.inqs_params.plcr = priv->default_policer;
+	priv->used_plcrs = BIT(0);
 
 	return 0;
 }
@@ -818,7 +836,7 @@ mrvl_configure_rxqs(struct mrvl_priv *priv, uint16_t portid,
 
 	priv->ppio_params.inqs_params.num_tcs = i;
 
-	if (port_cfg->policer_enable)
+	if (port_cfg->setup_policer)
 		return setup_policer(priv, &port_cfg->policer_params);
 
 	return 0;
diff --git a/drivers/net/mvpp2/mrvl_qos.h b/drivers/net/mvpp2/mrvl_qos.h
index fa9ddecb..f03e7731 100644
--- a/drivers/net/mvpp2/mrvl_qos.h
+++ b/drivers/net/mvpp2/mrvl_qos.h
@@ -43,7 +43,7 @@ struct mrvl_qos_cfg {
 		uint8_t default_tc;
 		uint8_t use_global_defaults;
 		struct pp2_cls_plcr_params policer_params;
-		uint8_t policer_enable;
+		uint8_t setup_policer;
 	} port[RTE_MAX_ETHPORTS];
 };
 
diff --git a/drivers/net/mvpp2/mrvl_tm.c b/drivers/net/mvpp2/mrvl_tm.c
new file mode 100644
index 00000000..3de89970
--- /dev/null
+++ b/drivers/net/mvpp2/mrvl_tm.c
@@ -0,0 +1,1009 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Marvell International Ltd.
+ * Copyright(c) 2018 Semihalf.
+ * All rights reserved.
+ */
+
+#include <rte_malloc.h>
+
+#include <linux/ethtool.h>
+#include <linux/sockios.h>
+#include <net/if.h>
+#include <sys/ioctl.h>
+
+#include "mrvl_tm.h"
+
+/** Minimum rate value in Bytes/s */
+#define MRVL_RATE_MIN (PP2_PPIO_MIN_CIR * 1000 / 8)
+
+/** Minimum burst size in Bytes */
+#define MRVL_BURST_MIN (PP2_PPIO_MIN_CBS * 1000)
+
+/** Maximum burst size in Bytes */
+#define MRVL_BURST_MAX 256000000
+
+/** Maximum WRR weight */
+#define MRVL_WEIGHT_MAX 255
+
+/**
+ * Get maximum port rate in Bytes/s.
+ *
+ * @param dev Pointer to the device.
+ * @param rate Pointer to the rate.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_get_max_rate(struct rte_eth_dev *dev, uint64_t *rate)
+{
+	struct ethtool_cmd edata;
+	struct ifreq req;
+	int ret, fd;
+
+	memset(&edata, 0, sizeof(edata));
+	memset(&req, 0, sizeof(req));
+	edata.cmd = ETHTOOL_GSET;
+	strcpy(req.ifr_name, dev->data->name);
+	req.ifr_data = (void *)&edata;
+
+	fd = socket(AF_INET, SOCK_DGRAM, 0);
+	if (fd == -1)
+		return -1;
+
+	ret = ioctl(fd, SIOCETHTOOL, &req);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+
+	close(fd);
+
+	*rate = ethtool_cmd_speed(&edata) * 1000 * 1000 / 8;
+
+	return 0;
+}
+
+/**
+ * Initialize traffic manager related data.
+ *
+ * @param dev Pointer to the device.
+ * @returns 0 on success, failure otherwise.
+ */
+int
+mrvl_tm_init(struct rte_eth_dev *dev)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+
+	LIST_INIT(&priv->shaper_profiles);
+	LIST_INIT(&priv->nodes);
+
+	if (priv->rate_max)
+		return 0;
+
+	return mrvl_get_max_rate(dev, &priv->rate_max);
+}
+
+/**
+ * Cleanup traffic manager related data.
+ *
+ * @param dev Pointer to the device.
+ */
+void mrvl_tm_deinit(struct rte_eth_dev *dev)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_shaper_profile *profile =
+		LIST_FIRST(&priv->shaper_profiles);
+	struct mrvl_tm_node *node = LIST_FIRST(&priv->nodes);
+
+	while (profile) {
+		struct mrvl_tm_shaper_profile *next = LIST_NEXT(profile, next);
+
+		LIST_REMOVE(profile, next);
+		rte_free(profile);
+		profile = next;
+	}
+
+	while (node) {
+		struct mrvl_tm_node *next = LIST_NEXT(node, next);
+
+		LIST_REMOVE(node, next);
+		rte_free(node);
+		node = next;
+	}
+}
+
+/**
+ * Get node using its id.
+ *
+ * @param priv Pointer to the port's private data.
+ * @param node_id Id used by this node.
+ * @returns Pointer to the node if exists, NULL otherwise.
+ */
+static struct mrvl_tm_node *
+mrvl_node_from_id(struct mrvl_priv *priv, uint32_t node_id)
+{
+	struct mrvl_tm_node *node;
+
+	LIST_FOREACH(node, &priv->nodes, next)
+		if (node->id == node_id)
+			return node;
+
+	return NULL;
+}
+
+/**
+ * Check whether node is leaf or root.
+ *
+ * @param dev Pointer to the device.
+ * @param node_id Id used by this node.
+ * @param is_leaf Pointer to flag indicating whether node is a leaf.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_node_type_get(struct rte_eth_dev *dev, uint32_t node_id, int *is_leaf,
+		   struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_node *node;
+
+	if (!is_leaf)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, NULL);
+
+	node = mrvl_node_from_id(priv, node_id);
+	if (!node)
+		return -rte_tm_error_set(error, ENODEV,
+					 RTE_TM_ERROR_TYPE_NODE_ID,
+					 NULL, "Node id does not exist\n");
+
+	*is_leaf = node->type == MRVL_NODE_QUEUE ? 1 : 0;
+
+	return 0;
+}
+
+/**
+ * Get traffic manager capabilities.
+ *
+ * @param dev Pointer to the device (unused).
+ * @param cap Pointer to the capabilities.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_capabilities_get(struct rte_eth_dev *dev,
+		      struct rte_tm_capabilities *cap,
+		      struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+
+	if (!cap)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, "Capabilities are missing\n");
+
+	memset(cap, 0, sizeof(*cap));
+
+	cap->n_nodes_max = 1 + dev->data->nb_tx_queues; /* port + txqs number */
+	cap->n_levels_max = 2; /* port level + txqs level */
+	cap->non_leaf_nodes_identical = 1;
+	cap->leaf_nodes_identical = 1;
+
+	cap->shaper_n_max = cap->n_nodes_max;
+	cap->shaper_private_n_max = cap->shaper_n_max;
+	cap->shaper_private_rate_min = MRVL_RATE_MIN;
+	cap->shaper_private_rate_max = priv->rate_max;
+
+	cap->sched_n_children_max = dev->data->nb_tx_queues;
+	cap->sched_sp_n_priorities_max = dev->data->nb_tx_queues;
+	cap->sched_wfq_n_children_per_group_max = dev->data->nb_tx_queues;
+	cap->sched_wfq_n_groups_max = 1;
+	cap->sched_wfq_weight_max = MRVL_WEIGHT_MAX;
+
+	cap->dynamic_update_mask = RTE_TM_UPDATE_NODE_SUSPEND_RESUME |
+				   RTE_TM_UPDATE_NODE_STATS;
+	cap->stats_mask = RTE_TM_STATS_N_PKTS | RTE_TM_STATS_N_BYTES;
+
+	return 0;
+}
+
+/**
+ * Get traffic manager hierarchy level capabilities.
+ *
+ * @param dev Pointer to the device.
+ * @param level_id Id of the level.
+ * @param cap Pointer to the level capabilities.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_level_capabilities_get(struct rte_eth_dev *dev,
+			    uint32_t level_id,
+			    struct rte_tm_level_capabilities *cap,
+			    struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+
+	if (!cap)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, NULL);
+
+	memset(cap, 0, sizeof(*cap));
+
+	if (level_id != MRVL_NODE_PORT && level_id != MRVL_NODE_QUEUE)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_LEVEL_ID,
+					 NULL, "Wrong level id\n");
+
+	if (level_id == MRVL_NODE_PORT) {
+		cap->n_nodes_max = 1;
+		cap->n_nodes_nonleaf_max = 1;
+		cap->non_leaf_nodes_identical = 1;
+
+		cap->nonleaf.shaper_private_supported = 1;
+		cap->nonleaf.shaper_private_rate_min = MRVL_RATE_MIN;
+		cap->nonleaf.shaper_private_rate_max = priv->rate_max;
+
+		cap->nonleaf.sched_n_children_max = dev->data->nb_tx_queues;
+		cap->nonleaf.sched_sp_n_priorities_max = 1;
+		cap->nonleaf.sched_wfq_n_children_per_group_max =
+			dev->data->nb_tx_queues;
+		cap->nonleaf.sched_wfq_n_groups_max = 1;
+		cap->nonleaf.sched_wfq_weight_max = MRVL_WEIGHT_MAX;
+		cap->nonleaf.stats_mask = RTE_TM_STATS_N_PKTS |
+					  RTE_TM_STATS_N_BYTES;
+	} else { /* level_id == MRVL_NODE_QUEUE */
+		cap->n_nodes_max = dev->data->nb_tx_queues;
+		cap->n_nodes_leaf_max = dev->data->nb_tx_queues;
+		cap->leaf_nodes_identical = 1;
+
+		cap->leaf.shaper_private_supported = 1;
+		cap->leaf.shaper_private_rate_min = MRVL_RATE_MIN;
+		cap->leaf.shaper_private_rate_max = priv->rate_max;
+		cap->leaf.stats_mask = RTE_TM_STATS_N_PKTS;
+	}
+
+	return 0;
+}
+
+/**
+ * Get node capabilities.
+ *
+ * @param dev Pointer to the device.
+ * @param node_id Id of the node.
+ * @param cap Pointer to the capabilities.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_node_capabilities_get(struct rte_eth_dev *dev, uint32_t node_id,
+			   struct rte_tm_node_capabilities *cap,
+			   struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_node *node;
+
+	if (!cap)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, NULL);
+
+	memset(cap, 0, sizeof(*cap));
+
+	node = mrvl_node_from_id(priv, node_id);
+	if (!node)
+		return -rte_tm_error_set(error, ENODEV,
+					 RTE_TM_ERROR_TYPE_NODE_ID,
+					 NULL, "Node id does not exist\n");
+
+	cap->shaper_private_supported = 1;
+	cap->shaper_private_rate_min = MRVL_RATE_MIN;
+	cap->shaper_private_rate_max = priv->rate_max;
+
+	if (node->type == MRVL_NODE_PORT) {
+		cap->nonleaf.sched_n_children_max = dev->data->nb_tx_queues;
+		cap->nonleaf.sched_sp_n_priorities_max = 1;
+		cap->nonleaf.sched_wfq_n_children_per_group_max =
+			dev->data->nb_tx_queues;
+		cap->nonleaf.sched_wfq_n_groups_max = 1;
+		cap->nonleaf.sched_wfq_weight_max = MRVL_WEIGHT_MAX;
+		cap->stats_mask = RTE_TM_STATS_N_PKTS | RTE_TM_STATS_N_BYTES;
+	} else {
+		cap->stats_mask = RTE_TM_STATS_N_PKTS;
+	}
+
+	return 0;
+}
+
+/**
+ * Get shaper profile using its id.
+ *
+ * @param priv Pointer to the port's private data.
+ * @param shaper_profile_id Id used by the shaper.
+ * @returns Pointer to the shaper profile if exists, NULL otherwise.
+ */
+static struct mrvl_tm_shaper_profile *
+mrvl_shaper_profile_from_id(struct mrvl_priv *priv, uint32_t shaper_profile_id)
+{
+	struct mrvl_tm_shaper_profile *profile;
+
+	LIST_FOREACH(profile, &priv->shaper_profiles, next)
+		if (profile->id == shaper_profile_id)
+			return profile;
+
+	return NULL;
+}
+
+/**
+ * Add a new shaper profile.
+ *
+ * @param dev Pointer to the device.
+ * @param shaper_profile_id Id of the new profile.
+ * @param params Pointer to the shaper profile parameters.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_shaper_profile_add(struct rte_eth_dev *dev, uint32_t shaper_profile_id,
+			struct rte_tm_shaper_params *params,
+			struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_shaper_profile *profile;
+
+	if (!params)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, NULL);
+
+	if (params->committed.rate)
+		return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE,
+				NULL, "Committed rate not supported\n");
+
+	if (params->committed.size)
+		return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE,
+				NULL, "Committed bucket size not supported\n");
+
+	if (params->peak.rate < MRVL_RATE_MIN ||
+	    params->peak.rate > priv->rate_max)
+		return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_RATE,
+				NULL, "Peak rate is out of range\n");
+
+	if (params->peak.size < MRVL_BURST_MIN ||
+	    params->peak.size > MRVL_BURST_MAX)
+		return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE,
+				NULL, "Peak size is out of range\n");
+
+	if (shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID,
+					 NULL, "Wrong shaper profile id\n");
+
+	profile = mrvl_shaper_profile_from_id(priv, shaper_profile_id);
+	if (profile)
+		return -rte_tm_error_set(error, EEXIST,
+					 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID,
+					 NULL, "Profile id already exists\n");
+
+	profile = rte_zmalloc_socket(NULL, sizeof(*profile), 0,
+				     rte_socket_id());
+	if (!profile)
+		return -rte_tm_error_set(error, ENOMEM,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, NULL);
+
+	profile->id = shaper_profile_id;
+	rte_memcpy(&profile->params, params, sizeof(profile->params));
+
+	LIST_INSERT_HEAD(&priv->shaper_profiles, profile, next);
+
+	return 0;
+}
+
+/**
+ * Remove a shaper profile.
+ *
+ * @param dev Pointer to the device.
+ * @param shaper_profile_id Id of the shaper profile.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_shaper_profile_delete(struct rte_eth_dev *dev, uint32_t shaper_profile_id,
+			   struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_shaper_profile *profile;
+
+	profile = mrvl_shaper_profile_from_id(priv, shaper_profile_id);
+	if (!profile)
+		return -rte_tm_error_set(error, ENODEV,
+					 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID,
+					 NULL, "Profile id does not exist\n");
+
+	if (profile->refcnt)
+		return -rte_tm_error_set(error, EPERM,
+					 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID,
+					 NULL, "Profile is used\n");
+
+	LIST_REMOVE(profile, next);
+	rte_free(profile);
+
+	return 0;
+}
+
+/**
+ * Check node parameters.
+ *
+ * @param dev Pointer to the device.
+ * @param node_id Id used by the node.
+ * @param priority Priority value.
+ * @param weight Weight value.
+ * @param level_id Id of the level.
+ * @param params Pointer to the node parameters.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_node_check_params(struct rte_eth_dev *dev, uint32_t node_id,
+		       uint32_t priority, uint32_t weight, uint32_t level_id,
+		       struct rte_tm_node_params *params,
+		       struct rte_tm_error *error)
+{
+	if (node_id == RTE_TM_NODE_ID_NULL)
+		return -rte_tm_error_set(error, EINVAL, RTE_TM_NODE_ID_NULL,
+					 NULL, "Node id is invalid\n");
+
+	if (priority)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_NODE_PRIORITY,
+					 NULL, "Priority should be 0\n");
+
+	if (weight > MRVL_WEIGHT_MAX)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_NODE_WEIGHT,
+					 NULL, "Weight is out of range\n");
+
+	if (level_id != MRVL_NODE_PORT && level_id != MRVL_NODE_QUEUE)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_LEVEL_ID,
+					 NULL, "Wrong level id\n");
+
+	if (!params)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, NULL);
+
+	if (params->shared_shaper_id)
+		return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_SHAPER_ID,
+				NULL, "Shared shaper is not supported\n");
+
+	if (params->n_shared_shapers)
+		return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS,
+				NULL, "Shared shaper is not supported\n");
+
+	/* verify port (root node) settings */
+	if (node_id >= dev->data->nb_tx_queues) {
+		if (params->nonleaf.wfq_weight_mode)
+			return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE,
+				NULL, "WFQ is not supported\n");
+
+		if (params->nonleaf.n_sp_priorities != 1)
+			return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES,
+				NULL, "SP is not supported\n");
+
+		if (params->stats_mask & ~(RTE_TM_STATS_N_PKTS |
+					   RTE_TM_STATS_N_BYTES))
+			return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS,
+				NULL,
+				"Requested port stats are not supported\n");
+
+		return 0;
+	}
+
+	/* verify txq (leaf node) settings */
+	if (params->leaf.cman)
+		return -rte_tm_error_set(error, EINVAL,
+					 RTE_TM_ERROR_TYPE_NODE_PARAMS_CMAN,
+					 NULL,
+					 "Congestion mngmt is not supported\n");
+
+	if (params->leaf.wred.wred_profile_id)
+		return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_NODE_PARAMS_WRED_PROFILE_ID,
+				NULL, "WRED is not supported\n");
+
+	if (params->leaf.wred.shared_wred_context_id)
+		return -rte_tm_error_set(error, EINVAL,
+			RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_WRED_CONTEXT_ID,
+			NULL, "WRED is not supported\n");
+
+	if (params->leaf.wred.n_shared_wred_contexts)
+		return -rte_tm_error_set(error, EINVAL,
+			RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_WRED_CONTEXTS,
+			NULL, "WRED is not supported\n");
+
+	if (params->stats_mask & ~RTE_TM_STATS_N_PKTS)
+		return -rte_tm_error_set(error, EINVAL,
+			RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS,
+			NULL,
+			"Requested txq stats are not supported\n");
+
+	return 0;
+}
+
+/**
+ * Add a new node.
+ *
+ * @param dev Pointer to the device.
+ * @param node_id Id of the node.
+ * @param parent_node_id Id of the parent node.
+ * @param priority Priority value.
+ * @param weight Weight value.
+ * @param level_id Id of the level.
+ * @param params Pointer to the node parameters.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_node_add(struct rte_eth_dev *dev, uint32_t node_id,
+	      uint32_t parent_node_id, uint32_t priority, uint32_t weight,
+	      uint32_t level_id, struct rte_tm_node_params *params,
+	      struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_shaper_profile *profile = NULL;
+	struct mrvl_tm_node *node, *parent = NULL;
+	int ret;
+
+	if (priv->ppio)
+		return -rte_tm_error_set(error, EPERM,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, "Port is already started\n");
+
+	ret = mrvl_node_check_params(dev, node_id, priority, weight, level_id,
+				     params, error);
+	if (ret)
+		return ret;
+
+	if (params->shaper_profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE) {
+		profile = mrvl_shaper_profile_from_id(priv,
+						 params->shaper_profile_id);
+		if (!profile)
+			return -rte_tm_error_set(error, ENODEV,
+					RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID,
+					NULL, "Shaper id does not exist\n");
+	}
+
+	if (parent_node_id == RTE_TM_NODE_ID_NULL) {
+		LIST_FOREACH(node, &priv->nodes, next) {
+			if (node->type != MRVL_NODE_PORT)
+				continue;
+
+			return -rte_tm_error_set(error, EINVAL,
+						 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+						 NULL, "Root node exists\n");
+		}
+	} else {
+		parent = mrvl_node_from_id(priv, parent_node_id);
+		if (!parent)
+			return -rte_tm_error_set(error, EINVAL,
+					RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID,
+					NULL, "Node id does not exist\n");
+	}
+
+	node = mrvl_node_from_id(priv, node_id);
+	if (node)
+		return -rte_tm_error_set(error, ENODEV,
+					 RTE_TM_ERROR_TYPE_NODE_ID,
+					 NULL, "Node id already exists\n");
+
+	node = rte_zmalloc_socket(NULL, sizeof(*node), 0, rte_socket_id());
+	if (!node)
+		return -rte_tm_error_set(error, ENOMEM,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, NULL);
+
+	node->id = node_id;
+	node->type = parent_node_id == RTE_TM_NODE_ID_NULL ? MRVL_NODE_PORT :
+							     MRVL_NODE_QUEUE;
+
+	if (parent) {
+		node->parent = parent;
+		parent->refcnt++;
+	}
+
+	if (profile) {
+		node->profile = profile;
+		profile->refcnt++;
+	}
+
+	node->weight = weight;
+	node->stats_mask = params->stats_mask;
+
+	LIST_INSERT_HEAD(&priv->nodes, node, next);
+
+	return 0;
+}
+
+/**
+ * Delete a node.
+ *
+ * @param dev Pointer to the device.
+ * @param node_id Id of the node.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_node_delete(struct rte_eth_dev *dev, uint32_t node_id,
+		 struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_node *node;
+
+	if (priv->ppio) {
+		return -rte_tm_error_set(error, EPERM,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, "Port is already started\n");
+	}
+
+	node = mrvl_node_from_id(priv, node_id);
+	if (!node)
+		return -rte_tm_error_set(error, ENODEV,
+					 RTE_TM_ERROR_TYPE_NODE_ID,
+					 NULL, "Node id does not exist\n");
+
+	if (node->refcnt)
+		return -rte_tm_error_set(error, EPERM,
+					 RTE_TM_ERROR_TYPE_NODE_ID,
+					 NULL, "Node id is used\n");
+
+	if (node->parent)
+		node->parent->refcnt--;
+
+	if (node->profile)
+		node->profile->refcnt--;
+
+	LIST_REMOVE(node, next);
+	rte_free(node);
+
+	return 0;
+}
+
+/**
+ * Helper for suspending specific tx queue.
+ *
+ * @param dev Pointer to the device.
+ * @param node_id Id used by this node.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int mrvl_node_suspend_one(struct rte_eth_dev *dev, uint32_t node_id,
+				 struct rte_tm_error *error)
+{
+	int ret = dev->dev_ops->tx_queue_stop(dev, node_id);
+	if (ret)
+		return -rte_tm_error_set(error, ret,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, "Failed to suspend a txq\n");
+
+	return 0;
+}
+
+/**
+ * Suspend a node.
+ *
+ * @param dev Pointer to the device.
+ * @param node_id Id of the node.
+ * @param error Pointer to the error.
+ * returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_node_suspend(struct rte_eth_dev *dev, uint32_t node_id,
+		  struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_node *node, *tmp;
+	int ret;
+
+	node = mrvl_node_from_id(priv, node_id);
+	if (!node)
+		return -rte_tm_error_set(error, ENODEV,
+					 RTE_TM_ERROR_TYPE_NODE_ID,
+					 NULL, "Node id does not exist\n");
+
+	if (!node->parent) {
+		LIST_FOREACH(tmp, &priv->nodes, next) {
+			if (!tmp->parent)
+				continue;
+
+			if (node != tmp->parent)
+				continue;
+
+			ret = mrvl_node_suspend_one(dev, tmp->id, error);
+			if (ret)
+				return ret;
+		}
+
+		return 0;
+	}
+
+	return mrvl_node_suspend_one(dev, node_id, error);
+}
+
+/**
+ * Resume a node.
+ *
+ * @param dev Pointer to the device.
+ * @param node_id Id of the node.
+ * @param error Pointer to the error.
+ * returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_node_resume(struct rte_eth_dev *dev, uint32_t node_id,
+		 struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_node *node;
+	int ret;
+
+	node = mrvl_node_from_id(priv, node_id);
+	if (!node)
+		return -rte_tm_error_set(error, ENODEV,
+					 RTE_TM_ERROR_TYPE_NODE_ID,
+					 NULL, "Node id does not exist\n");
+
+
+	if (!node->parent)
+		return -rte_tm_error_set(error, EPERM,
+					 RTE_TM_ERROR_TYPE_NODE_ID,
+					 NULL, "Cannot suspend a port\n");
+
+	ret = dev->dev_ops->tx_queue_start(dev, node_id);
+	if (ret)
+		return -rte_tm_error_set(error, ret,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, "Failed to resume a txq\n");
+	return 0;
+}
+
+/**
+ * Apply traffic manager hierarchy.
+ *
+ * @param dev Pointer to the device.
+ * @param clear_on_fail Flag indicating whether to do cleanup on the failure.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_hierarchy_commit(struct rte_eth_dev *dev, int clear_on_fail,
+		      struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_node *node;
+	int ret;
+
+	if (priv->ppio) {
+		ret = -rte_tm_error_set(error, EPERM,
+					RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					NULL, "Port is already started\n");
+		goto out;
+	}
+
+	LIST_FOREACH(node, &priv->nodes, next) {
+		struct pp2_ppio_outq_params *p;
+
+		if (node->type == MRVL_NODE_PORT) {
+			if (!node->profile)
+				continue;
+
+			priv->ppio_params.rate_limit_enable = 1;
+			priv->ppio_params.rate_limit_params.cir =
+				node->profile->params.peak.rate * 8 / 1000;
+			priv->ppio_params.rate_limit_params.cbs =
+				node->profile->params.peak.size / 1000;
+
+			MRVL_LOG(INFO,
+				"Port rate limit overrides txqs rate limit");
+
+			continue;
+		}
+
+		if (node->id >= dev->data->nb_tx_queues) {
+			ret = -rte_tm_error_set(error, EINVAL,
+					RTE_TM_ERROR_TYPE_NODE_ID, NULL,
+					"Not enough txqs are configured\n");
+			goto out;
+		}
+
+		p = &priv->ppio_params.outqs_params.outqs_params[node->id];
+
+		if (node->weight) {
+			p->sched_mode = PP2_PPIO_SCHED_M_WRR;
+			p->weight = node->weight;
+		} else {
+			p->sched_mode = PP2_PPIO_SCHED_M_SP;
+			p->weight = 0;
+		}
+
+		if (node->profile) {
+			p->rate_limit_enable = 1;
+			/* convert Bytes/s to kilo bits/s */
+			p->rate_limit_params.cir =
+				node->profile->params.peak.rate * 8 / 1000;
+			/* convert bits to kilo bits */
+			p->rate_limit_params.cbs =
+				node->profile->params.peak.size / 1000;
+		} else {
+			p->rate_limit_enable = 0;
+			p->rate_limit_params.cir = 0;
+			p->rate_limit_params.cbs = 0;
+		}
+	}
+
+	/* reset to defaults in case applied tm hierarchy is empty */
+	if (LIST_EMPTY(&priv->nodes)) {
+		int i;
+
+		for (i = 0; i < priv->ppio_params.outqs_params.num_outqs; i++) {
+			struct pp2_ppio_outq_params *p =
+				&priv->ppio_params.outqs_params.outqs_params[i];
+
+			p->sched_mode = PP2_PPIO_SCHED_M_WRR;
+			p->weight = 0;
+			p->rate_limit_enable = 0;
+			p->rate_limit_params.cir = 0;
+			p->rate_limit_params.cbs = 0;
+		}
+	}
+
+	return 0;
+out:
+	if (clear_on_fail) {
+		mrvl_tm_deinit(dev);
+		mrvl_tm_init(dev);
+	}
+
+	return ret;
+}
+
+/**
+ * Read statistics counters for current node.
+ *
+ * @param dev Pointer to the device.
+ * @param node_id Id of the node.
+ * @param stats Pointer to the statistics counters.
+ * @param stats_mask Pointer to mask of enabled statistics counters
+ *                   that are retrieved.
+ * @param clear Flag indicating whether to clear statistics.
+ *              Non-zero value clears statistics.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_node_stats_read(struct rte_eth_dev *dev, uint32_t node_id,
+		     struct rte_tm_node_stats *stats, uint64_t *stats_mask,
+		     int clear, struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_node *node;
+	int ret;
+
+	if (!priv->ppio) {
+		return -rte_tm_error_set(error, EPERM,
+					 RTE_TM_ERROR_TYPE_UNSPECIFIED,
+					 NULL, "Port is not started\n");
+	}
+
+	node = mrvl_node_from_id(priv, node_id);
+	if (!node)
+		return -rte_tm_error_set(error, ENODEV,
+					 RTE_TM_ERROR_TYPE_NODE_ID,
+					 NULL, "Node id does not exist\n");
+
+	if (stats_mask)
+		*stats_mask = node->stats_mask;
+
+	if (!stats)
+		return 0;
+
+	memset(stats, 0, sizeof(*stats));
+
+	if (!node->parent) {
+		struct pp2_ppio_statistics s;
+
+		memset(&s, 0, sizeof(s));
+		ret = pp2_ppio_get_statistics(priv->ppio, &s, clear);
+		if (ret)
+			return -rte_tm_error_set(error, -ret,
+					RTE_TM_ERROR_TYPE_UNSPECIFIED, NULL,
+					"Failed to read port statistics\n");
+
+		if (node->stats_mask & RTE_TM_STATS_N_PKTS)
+			stats->n_pkts = s.tx_packets;
+
+		if (node->stats_mask & RTE_TM_STATS_N_BYTES)
+			stats->n_bytes = s.tx_bytes;
+	} else {
+		struct pp2_ppio_outq_statistics s;
+
+		memset(&s, 0, sizeof(s));
+		ret = pp2_ppio_outq_get_statistics(priv->ppio, node_id, &s,
+						   clear);
+		if (ret)
+			return -rte_tm_error_set(error, -ret,
+					RTE_TM_ERROR_TYPE_UNSPECIFIED, NULL,
+					"Failed to read txq statistics\n");
+
+		if (node->stats_mask & RTE_TM_STATS_N_PKTS)
+			stats->n_pkts = s.deq_desc;
+	}
+
+	return 0;
+}
+
+/**
+ * Update node statistics.
+ *
+ * @param dev Pointer to the device.
+ * @param node_id Id of the node.
+ * @param stats_mask Bitmask of statistics counters to be enabled.
+ * @param error Pointer to the error.
+ * @returns 0 on success, negative value otherwise.
+ */
+static int
+mrvl_node_stats_update(struct rte_eth_dev *dev, uint32_t node_id,
+		       uint64_t stats_mask, struct rte_tm_error *error)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+	struct mrvl_tm_node *node;
+
+	node = mrvl_node_from_id(priv, node_id);
+	if (!node)
+		return -rte_tm_error_set(error, ENODEV,
+					 RTE_TM_ERROR_TYPE_NODE_ID,
+					 NULL, "Node id does not exist\n");
+
+	if (!node->parent) {
+		if (stats_mask & ~(RTE_TM_STATS_N_PKTS | RTE_TM_STATS_N_BYTES))
+			return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS,
+				NULL,
+				"Requested port stats are not supported\n");
+	} else {
+		if (stats_mask & ~RTE_TM_STATS_N_PKTS)
+			return -rte_tm_error_set(error, EINVAL,
+				RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS,
+				NULL,
+				"Requested txq stats are not supported\n");
+	}
+
+	node->stats_mask = stats_mask;
+
+	return 0;
+}
+
+const struct rte_tm_ops mrvl_tm_ops = {
+	.node_type_get = mrvl_node_type_get,
+	.capabilities_get = mrvl_capabilities_get,
+	.level_capabilities_get = mrvl_level_capabilities_get,
+	.node_capabilities_get = mrvl_node_capabilities_get,
+	.shaper_profile_add = mrvl_shaper_profile_add,
+	.shaper_profile_delete = mrvl_shaper_profile_delete,
+	.node_add = mrvl_node_add,
+	.node_delete = mrvl_node_delete,
+	.node_suspend = mrvl_node_suspend,
+	.node_resume = mrvl_node_resume,
+	.hierarchy_commit = mrvl_hierarchy_commit,
+	.node_stats_update = mrvl_node_stats_update,
+	.node_stats_read = mrvl_node_stats_read,
+};
diff --git a/drivers/net/mvpp2/mrvl_tm.h b/drivers/net/mvpp2/mrvl_tm.h
new file mode 100644
index 00000000..9d81ede2
--- /dev/null
+++ b/drivers/net/mvpp2/mrvl_tm.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Marvell International Ltd.
+ * Copyright(c) 2018 Semihalf.
+ * All rights reserved.
+ */
+
+#ifndef _MRVL_TM_H_
+#define _MRVL_TM_H_
+
+#include "mrvl_ethdev.h"
+
+int mrvl_tm_init(struct rte_eth_dev *dev);
+void mrvl_tm_deinit(struct rte_eth_dev *dev);
+
+#endif /* _MRVL_TM_H_ */
diff --git a/drivers/net/netvsc/Makefile b/drivers/net/netvsc/Makefile
index 3c713af3..71482591 100644
--- a/drivers/net/netvsc/Makefile
+++ b/drivers/net/netvsc/Makefile
@@ -15,6 +15,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += hn_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += hn_rxtx.c
 SRCS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += hn_rndis.c
 SRCS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += hn_nvs.c
+SRCS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += hn_vf.c
 
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c
index 78b842ba..aa38ee7a 100644
--- a/drivers/net/netvsc/hn_ethdev.c
+++ b/drivers/net/netvsc/hn_ethdev.c
@@ -14,7 +14,9 @@
 #include <rte_memcpy.h>
 #include <rte_string_fns.h>
 #include <rte_memzone.h>
+#include <rte_devargs.h>
 #include <rte_malloc.h>
+#include <rte_kvargs.h>
 #include <rte_atomic.h>
 #include <rte_branch_prediction.h>
 #include <rte_ether.h>
@@ -40,8 +42,7 @@
 			    DEV_TX_OFFLOAD_VLAN_INSERT)
 
 #define HN_RX_OFFLOAD_CAPS (DEV_RX_OFFLOAD_CHECKSUM | \
-			    DEV_RX_OFFLOAD_VLAN_STRIP | \
-			    DEV_RX_OFFLOAD_CRC_STRIP)
+			    DEV_RX_OFFLOAD_VLAN_STRIP)
 
 int hn_logtype_init;
 int hn_logtype_driver;
@@ -55,7 +56,7 @@ static const struct hn_xstats_name_off hn_stat_strings[] = {
 	{ "good_packets",           offsetof(struct hn_stats, packets) },
 	{ "good_bytes",             offsetof(struct hn_stats, bytes) },
 	{ "errors",                 offsetof(struct hn_stats, errors) },
-	{ "allocation_failed",      offsetof(struct hn_stats, nomemory) },
+	{ "ring full",              offsetof(struct hn_stats, ring_full) },
 	{ "multicast_packets",      offsetof(struct hn_stats, multicast) },
 	{ "broadcast_packets",      offsetof(struct hn_stats, broadcast) },
 	{ "undersize_packets",      offsetof(struct hn_stats, size_bins[0]) },
@@ -105,6 +106,10 @@ eth_dev_vmbus_allocate(struct rte_vmbus_device *dev, size_t private_data_size)
 	}
 
 	eth_dev->device = &dev->device;
+
+	/* interrupt is simulated */
+	dev->intr_handle.type = RTE_INTR_HANDLE_EXT;
+	eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
 	eth_dev->intr_handle = &dev->intr_handle;
 
 	return eth_dev;
@@ -113,22 +118,66 @@ eth_dev_vmbus_allocate(struct rte_vmbus_device *dev, size_t private_data_size)
 static void
 eth_dev_vmbus_release(struct rte_eth_dev *eth_dev)
 {
+	/* mac_addrs must not be freed alone because part of dev_private */
+	eth_dev->data->mac_addrs = NULL;
 	/* free ether device */
 	rte_eth_dev_release_port(eth_dev);
 
-	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
-		rte_free(eth_dev->data->dev_private);
+	eth_dev->device = NULL;
+	eth_dev->intr_handle = NULL;
+}
 
-	eth_dev->data->dev_private = NULL;
+/* handle "latency=X" from devargs */
+static int hn_set_latency(const char *key, const char *value, void *opaque)
+{
+	struct hn_data *hv = opaque;
+	char *endp = NULL;
+	unsigned long lat;
 
-	/*
-	 * Secondary process will check the name to attach.
-	 * Clear this field to avoid attaching a released ports.
-	 */
-	eth_dev->data->name[0] = '\0';
+	errno = 0;
+	lat = strtoul(value, &endp, 0);
 
-	eth_dev->device = NULL;
-	eth_dev->intr_handle = NULL;
+	if (*value == '\0' || *endp != '\0') {
+		PMD_DRV_LOG(ERR, "invalid parameter %s=%s", key, value);
+		return -EINVAL;
+	}
+
+	PMD_DRV_LOG(DEBUG, "set latency %lu usec", lat);
+
+	hv->latency = lat * 1000;	/* usec to nsec */
+	return 0;
+}
+
+/* Parse device arguments */
+static int hn_parse_args(const struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_devargs *devargs = dev->device->devargs;
+	static const char * const valid_keys[] = {
+		"latency",
+		NULL
+	};
+	struct rte_kvargs *kvlist;
+	int ret;
+
+	if (!devargs)
+		return 0;
+
+	PMD_INIT_LOG(DEBUG, "device args %s %s",
+		     devargs->name, devargs->args);
+
+	kvlist = rte_kvargs_parse(devargs->args, valid_keys);
+	if (!kvlist) {
+		PMD_DRV_LOG(NOTICE, "invalid parameters");
+		return -EINVAL;
+	}
+
+	ret = rte_kvargs_process(kvlist, "latency", hn_set_latency, hv);
+	if (ret)
+		PMD_DRV_LOG(ERR, "Unable to process latency arg\n");
+
+	rte_kvargs_free(kvlist);
+	return ret;
 }
 
 /* Update link status.
@@ -136,9 +185,9 @@ eth_dev_vmbus_release(struct rte_eth_dev *eth_dev)
  *   means block this call until link is up.
  *   which is not worth supporting.
  */
-static int
+int
 hn_dev_link_update(struct rte_eth_dev *dev,
-		   __rte_unused int wait_to_complete)
+		   int wait_to_complete)
 {
 	struct hn_data *hv = dev->data->dev_private;
 	struct rte_eth_link link, old;
@@ -152,6 +201,8 @@ hn_dev_link_update(struct rte_eth_dev *dev,
 
 	hn_rndis_get_linkspeed(hv);
 
+	hn_vf_link_update(dev, wait_to_complete);
+
 	link = (struct rte_eth_link) {
 		.link_duplex = ETH_LINK_FULL_DUPLEX,
 		.link_autoneg = ETH_LINK_SPEED_FIXED,
@@ -190,6 +241,7 @@ static void hn_dev_info_get(struct rte_eth_dev *dev,
 	dev_info->max_tx_queues = hv->max_queues;
 
 	hn_rndis_get_offload(hv, dev_info);
+	hn_vf_info_get(hv, dev_info);
 }
 
 static void
@@ -198,6 +250,7 @@ hn_dev_promiscuous_enable(struct rte_eth_dev *dev)
 	struct hn_data *hv = dev->data->dev_private;
 
 	hn_rndis_set_rxfilter(hv, NDIS_PACKET_TYPE_PROMISCUOUS);
+	hn_vf_promiscuous_enable(dev);
 }
 
 static void
@@ -210,6 +263,7 @@ hn_dev_promiscuous_disable(struct rte_eth_dev *dev)
 	if (dev->data->all_multicast)
 		filter |= NDIS_PACKET_TYPE_ALL_MULTICAST;
 	hn_rndis_set_rxfilter(hv, filter);
+	hn_vf_promiscuous_disable(dev);
 }
 
 static void
@@ -220,6 +274,7 @@ hn_dev_allmulticast_enable(struct rte_eth_dev *dev)
 	hn_rndis_set_rxfilter(hv, NDIS_PACKET_TYPE_DIRECTED |
 			      NDIS_PACKET_TYPE_ALL_MULTICAST |
 			NDIS_PACKET_TYPE_BROADCAST);
+	hn_vf_allmulticast_enable(dev);
 }
 
 static void
@@ -229,6 +284,16 @@ hn_dev_allmulticast_disable(struct rte_eth_dev *dev)
 
 	hn_rndis_set_rxfilter(hv, NDIS_PACKET_TYPE_DIRECTED |
 			     NDIS_PACKET_TYPE_BROADCAST);
+	hn_vf_allmulticast_disable(dev);
+}
+
+static int
+hn_dev_mc_addr_list(struct rte_eth_dev *dev,
+		     struct ether_addr *mc_addr_set,
+		     uint32_t nb_mc_addr)
+{
+	/* No filtering on the synthetic path, but can do it on VF */
+	return hn_vf_mc_addr_list(dev, mc_addr_set, nb_mc_addr);
 }
 
 /* Setup shared rx/tx queue data */
@@ -264,6 +329,8 @@ static int hn_subchan_configure(struct hn_data *hv,
 			return err;
 		}
 
+		rte_vmbus_set_latency(hv->vmbus, new_sc, hv->latency);
+
 		retry = 0;
 		chn_index = rte_vmbus_sub_channel_index(new_sc);
 		if (chn_index == 0 || chn_index > hv->max_queues) {
@@ -338,7 +405,7 @@ static int hn_dev_configure(struct rte_eth_dev *dev)
 		}
 	}
 
-	return 0;
+	return hn_vf_configure(dev, dev_conf);
 }
 
 static int hn_dev_stats_get(struct rte_eth_dev *dev,
@@ -346,6 +413,8 @@ static int hn_dev_stats_get(struct rte_eth_dev *dev,
 {
 	unsigned int i;
 
+	hn_vf_stats_get(dev, stats);
+
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		const struct hn_tx_queue *txq = dev->data->tx_queues[i];
 
@@ -354,7 +423,7 @@ static int hn_dev_stats_get(struct rte_eth_dev *dev,
 
 		stats->opackets += txq->stats.packets;
 		stats->obytes += txq->stats.bytes;
-		stats->oerrors += txq->stats.errors + txq->stats.nomemory;
+		stats->oerrors += txq->stats.errors;
 
 		if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
 			stats->q_opackets[i] = txq->stats.packets;
@@ -371,7 +440,7 @@ static int hn_dev_stats_get(struct rte_eth_dev *dev,
 		stats->ipackets += rxq->stats.packets;
 		stats->ibytes += rxq->stats.bytes;
 		stats->ierrors += rxq->stats.errors;
-		stats->imissed += rxq->ring_full;
+		stats->imissed += rxq->stats.ring_full;
 
 		if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
 			stats->q_ipackets[i] = rxq->stats.packets;
@@ -405,22 +474,41 @@ hn_dev_stats_reset(struct rte_eth_dev *dev)
 			continue;
 
 		memset(&rxq->stats, 0, sizeof(struct hn_stats));
-		rxq->ring_full = 0;
 	}
 }
 
+static void
+hn_dev_xstats_reset(struct rte_eth_dev *dev)
+{
+	hn_dev_stats_reset(dev);
+	hn_vf_xstats_reset(dev);
+}
+
+static int
+hn_dev_xstats_count(struct rte_eth_dev *dev)
+{
+	int ret, count;
+
+	count = dev->data->nb_tx_queues * RTE_DIM(hn_stat_strings);
+	count += dev->data->nb_rx_queues * RTE_DIM(hn_stat_strings);
+
+	ret = hn_vf_xstats_get_names(dev, NULL, 0);
+	if (ret < 0)
+		return ret;
+
+	return count + ret;
+}
+
 static int
 hn_dev_xstats_get_names(struct rte_eth_dev *dev,
 			struct rte_eth_xstat_name *xstats_names,
-			__rte_unused unsigned int limit)
+			unsigned int limit)
 {
 	unsigned int i, t, count = 0;
-
-	PMD_INIT_FUNC_TRACE();
+	int ret;
 
 	if (!xstats_names)
-		return dev->data->nb_tx_queues * RTE_DIM(hn_stat_strings)
-			+ dev->data->nb_rx_queues * RTE_DIM(hn_stat_strings);
+		return hn_dev_xstats_count(dev);
 
 	/* Note: limit checked in rte_eth_xstats_names() */
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
@@ -429,6 +517,9 @@ hn_dev_xstats_get_names(struct rte_eth_dev *dev,
 		if (!txq)
 			continue;
 
+		if (count >= limit)
+			break;
+
 		for (t = 0; t < RTE_DIM(hn_stat_strings); t++)
 			snprintf(xstats_names[count++].name,
 				 RTE_ETH_XSTATS_NAME_SIZE,
@@ -441,6 +532,9 @@ hn_dev_xstats_get_names(struct rte_eth_dev *dev,
 		if (!rxq)
 			continue;
 
+		if (count >= limit)
+			break;
+
 		for (t = 0; t < RTE_DIM(hn_stat_strings); t++)
 			snprintf(xstats_names[count++].name,
 				 RTE_ETH_XSTATS_NAME_SIZE,
@@ -448,7 +542,12 @@ hn_dev_xstats_get_names(struct rte_eth_dev *dev,
 				 hn_stat_strings[t].name);
 	}
 
-	return count;
+	ret = hn_vf_xstats_get_names(dev, xstats_names + count,
+				     limit - count);
+	if (ret < 0)
+		return ret;
+
+	return count + ret;
 }
 
 static int
@@ -457,11 +556,9 @@ hn_dev_xstats_get(struct rte_eth_dev *dev,
 		  unsigned int n)
 {
 	unsigned int i, t, count = 0;
-
-	const unsigned int nstats =
-		dev->data->nb_tx_queues * RTE_DIM(hn_stat_strings)
-		+ dev->data->nb_rx_queues * RTE_DIM(hn_stat_strings);
+	const unsigned int nstats = hn_dev_xstats_count(dev);
 	const char *stats;
+	int ret;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -492,26 +589,33 @@ hn_dev_xstats_get(struct rte_eth_dev *dev,
 				(stats + hn_stat_strings[t].offset);
 	}
 
-	return count;
+	ret = hn_vf_xstats_get(dev, xstats + count, n - count);
+	if (ret < 0)
+		return ret;
+
+	return count + ret;
 }
 
 static int
 hn_dev_start(struct rte_eth_dev *dev)
 {
 	struct hn_data *hv = dev->data->dev_private;
+	int error;
 
 	PMD_INIT_FUNC_TRACE();
 
-	/* check if lsc interrupt feature is enabled */
-	if (dev->data->dev_conf.intr_conf.lsc) {
-		PMD_DRV_LOG(ERR, "link status not supported yet");
-		return -ENOTSUP;
-	}
+	error = hn_rndis_set_rxfilter(hv,
+				      NDIS_PACKET_TYPE_BROADCAST |
+				      NDIS_PACKET_TYPE_ALL_MULTICAST |
+				      NDIS_PACKET_TYPE_DIRECTED);
+	if (error)
+		return error;
+
+	error = hn_vf_start(dev);
+	if (error)
+		hn_rndis_set_rxfilter(hv, 0);
 
-	return hn_rndis_set_rxfilter(hv,
-				     NDIS_PACKET_TYPE_BROADCAST |
-				     NDIS_PACKET_TYPE_ALL_MULTICAST |
-				     NDIS_PACKET_TYPE_DIRECTED);
+	return error;
 }
 
 static void
@@ -522,12 +626,15 @@ hn_dev_stop(struct rte_eth_dev *dev)
 	PMD_INIT_FUNC_TRACE();
 
 	hn_rndis_set_rxfilter(hv, 0);
+	hn_vf_stop(dev);
 }
 
 static void
 hn_dev_close(struct rte_eth_dev *dev __rte_unused)
 {
 	PMD_INIT_LOG(DEBUG, "close");
+
+	hn_vf_close(dev);
 }
 
 static const struct eth_dev_ops hn_eth_dev_ops = {
@@ -536,22 +643,23 @@ static const struct eth_dev_ops hn_eth_dev_ops = {
 	.dev_stop		= hn_dev_stop,
 	.dev_close		= hn_dev_close,
 	.dev_infos_get		= hn_dev_info_get,
-	.txq_info_get		= hn_dev_tx_queue_info,
-	.rxq_info_get		= hn_dev_rx_queue_info,
+	.dev_supported_ptypes_get = hn_vf_supported_ptypes,
 	.promiscuous_enable     = hn_dev_promiscuous_enable,
 	.promiscuous_disable    = hn_dev_promiscuous_disable,
 	.allmulticast_enable    = hn_dev_allmulticast_enable,
 	.allmulticast_disable   = hn_dev_allmulticast_disable,
+	.set_mc_addr_list	= hn_dev_mc_addr_list,
 	.tx_queue_setup		= hn_dev_tx_queue_setup,
 	.tx_queue_release	= hn_dev_tx_queue_release,
+	.tx_done_cleanup        = hn_dev_tx_done_cleanup,
 	.rx_queue_setup		= hn_dev_rx_queue_setup,
 	.rx_queue_release	= hn_dev_rx_queue_release,
 	.link_update		= hn_dev_link_update,
 	.stats_get		= hn_dev_stats_get,
+	.stats_reset            = hn_dev_stats_reset,
 	.xstats_get		= hn_dev_xstats_get,
 	.xstats_get_names	= hn_dev_xstats_get_names,
-	.stats_reset            = hn_dev_stats_reset,
-	.xstats_reset		= hn_dev_stats_reset,
+	.xstats_reset		= hn_dev_xstats_reset,
 };
 
 /*
@@ -623,12 +731,27 @@ eth_hn_dev_init(struct rte_eth_dev *eth_dev)
 	hv->rxbuf_res = &vmbus->resource[HV_RECV_BUF_MAP];
 	hv->chim_res  = &vmbus->resource[HV_SEND_BUF_MAP];
 	hv->port_id = eth_dev->data->port_id;
+	hv->latency = HN_CHAN_LATENCY_NS;
+
+	err = hn_parse_args(eth_dev);
+	if (err)
+		return err;
+
+	strlcpy(hv->owner.name, eth_dev->device->name,
+		RTE_ETH_MAX_OWNER_NAME_LEN);
+	err = rte_eth_dev_owner_new(&hv->owner.id);
+	if (err) {
+		PMD_INIT_LOG(ERR, "Can not get owner id");
+		return err;
+	}
 
 	/* Initialize primary channel input for control operations */
 	err = rte_vmbus_chan_open(vmbus, &hv->channels[0]);
 	if (err)
 		return err;
 
+	rte_vmbus_set_latency(hv->vmbus, hv->channels[0], hv->latency);
+
 	hv->primary = hn_rx_queue_alloc(hv, 0,
 					eth_dev->device->numa_node);
 
@@ -657,6 +780,15 @@ eth_hn_dev_init(struct rte_eth_dev *eth_dev)
 
 	hv->max_queues = RTE_MIN(rxr_cnt, (unsigned int)max_chan);
 
+	/* If VF was reported but not added, do it now */
+	if (hv->vf_present && !hv->vf_dev) {
+		PMD_INIT_LOG(DEBUG, "Adding VF device");
+
+		err = hn_vf_add(eth_dev, hv);
+		if (err)
+			goto failed;
+	}
+
 	return 0;
 
 failed:
@@ -686,8 +818,7 @@ eth_hn_dev_uninit(struct rte_eth_dev *eth_dev)
 	hn_detach(hv);
 	rte_vmbus_chan_close(hv->primary->chan);
 	rte_free(hv->primary);
-
-	eth_dev->data->mac_addrs = NULL;
+	rte_eth_dev_owner_delete(hv->owner.id);
 
 	return 0;
 }
diff --git a/drivers/net/netvsc/hn_nvs.c b/drivers/net/netvsc/hn_nvs.c
index 77d3b839..9690c5f8 100644
--- a/drivers/net/netvsc/hn_nvs.c
+++ b/drivers/net/netvsc/hn_nvs.c
@@ -279,14 +279,13 @@ hn_nvs_conn_chim(struct hn_data *hv)
 			       NVS_TYPE_CHIM_CONNRESP);
 	if (error) {
 		PMD_DRV_LOG(ERR, "exec nvs chim conn failed");
-		goto cleanup;
+		return error;
 	}
 
 	if (resp.status != NVS_STATUS_OK) {
 		PMD_DRV_LOG(ERR, "nvs chim conn failed: %x",
 			    resp.status);
-		error = -EIO;
-		goto cleanup;
+		return -EIO;
 	}
 
 	sectsz = resp.sectsz;
@@ -295,7 +294,8 @@ hn_nvs_conn_chim(struct hn_data *hv)
 		PMD_DRV_LOG(NOTICE,
 			    "invalid chimney sending buffer section size: %u",
 			    sectsz);
-		return 0;
+		error = -EINVAL;
+		goto cleanup;
 	}
 
 	hv->chim_szmax = sectsz;
@@ -304,11 +304,6 @@ hn_nvs_conn_chim(struct hn_data *hv)
 	PMD_DRV_LOG(INFO, "send buffer %lu section size:%u, count:%u",
 		    len, hv->chim_szmax, hv->chim_cnt);
 
-	if (len % hv->chim_szmax != 0) {
-		PMD_DRV_LOG(NOTICE,
-			    "chimney sending sections are not properly aligned");
-	}
-
 	/* Done! */
 	return 0;
 
@@ -537,10 +532,19 @@ void
 hn_nvs_set_datapath(struct hn_data *hv, uint32_t path)
 {
 	struct hn_nvs_datapath dp;
+	int error;
+
+	PMD_DRV_LOG(DEBUG, "set datapath %s",
+		    path ? "VF" : "Synthetic");
 
 	memset(&dp, 0, sizeof(dp));
 	dp.type = NVS_TYPE_SET_DATAPATH;
 	dp.active_path = path;
 
-	hn_nvs_req_send(hv, &dp, sizeof(dp));
+	error = hn_nvs_req_send(hv, &dp, sizeof(dp));
+	if (error) {
+		PMD_DRV_LOG(ERR,
+			    "send set datapath failed: %d",
+			    error);
+	}
 }
diff --git a/drivers/net/netvsc/hn_nvs.h b/drivers/net/netvsc/hn_nvs.h
index 984a9c11..2563fd8d 100644
--- a/drivers/net/netvsc/hn_nvs.h
+++ b/drivers/net/netvsc/hn_nvs.h
@@ -105,6 +105,12 @@ struct hn_nvs_ndis_init {
 	uint8_t		rsvd[28];
 } __rte_packed;
 
+struct hn_nvs_vf_association {
+	uint32_t	type;	/* NVS_TYPE_VFASSOC_NOTE */
+	uint32_t	allocated;
+	uint32_t	serial;
+} __rte_packed;
+
 #define NVS_DATAPATH_SYNTHETIC	0
 #define NVS_DATAPATH_VF		1
 
@@ -207,6 +213,9 @@ void	hn_nvs_detach(struct hn_data *hv);
 void	hn_nvs_ack_rxbuf(struct vmbus_channel *chan, uint64_t tid);
 int	hn_nvs_alloc_subchans(struct hn_data *hv, uint32_t *nsubch);
 void	hn_nvs_set_datapath(struct hn_data *hv, uint32_t path);
+void	hn_nvs_handle_vfassoc(struct rte_eth_dev *dev,
+			      const struct vmbus_chanpkt_hdr *hdr,
+			      const void *data);
 
 static inline int
 hn_nvs_send(struct vmbus_channel *chan, uint16_t flags,
diff --git a/drivers/net/netvsc/hn_rndis.c b/drivers/net/netvsc/hn_rndis.c
index bde33969..0134ecb6 100644
--- a/drivers/net/netvsc/hn_rndis.c
+++ b/drivers/net/netvsc/hn_rndis.c
@@ -11,6 +11,7 @@
 #include <errno.h>
 #include <unistd.h>
 
+#include <rte_ethdev_driver.h>
 #include <rte_ethdev.h>
 #include <rte_string_fns.h>
 #include <rte_memzone.h>
@@ -281,7 +282,7 @@ static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan,
 				  &nvs_rndis, sizeof(nvs_rndis), 0U, NULL);
 }
 
-void hn_rndis_link_status(struct hn_data *hv __rte_unused, const void *msg)
+void hn_rndis_link_status(struct rte_eth_dev *dev, const void *msg)
 {
 	const struct rndis_status_msg *indicate = msg;
 
@@ -290,15 +291,19 @@ void hn_rndis_link_status(struct hn_data *hv __rte_unused, const void *msg)
 	PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status);
 
 	switch (indicate->status) {
-	case RNDIS_STATUS_LINK_SPEED_CHANGE:
 	case RNDIS_STATUS_NETWORK_CHANGE:
 	case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG:
 		/* ignore not in DPDK API */
 		break;
 
+	case RNDIS_STATUS_LINK_SPEED_CHANGE:
 	case RNDIS_STATUS_MEDIA_CONNECT:
 	case RNDIS_STATUS_MEDIA_DISCONNECT:
-		/* TODO handle as LSC interrupt  */
+		if (dev->data->dev_conf.intr_conf.lsc &&
+		    hn_dev_link_update(dev, 0) == 0)
+			_rte_eth_dev_callback_process(dev,
+						      RTE_ETH_EVENT_INTR_LSC,
+						      NULL);
 		break;
 	default:
 		PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x",
@@ -382,7 +387,7 @@ static int hn_rndis_exec1(struct hn_data *hv,
 	if (comp) {
 		/* Poll primary channel until response received */
 		while (hv->rndis_pending == rid)
-			hn_process_events(hv, 0);
+			hn_process_events(hv, 0, 1);
 
 		memcpy(comp, hv->rndis_resp, comp_len);
 	}
@@ -892,8 +897,7 @@ int hn_rndis_get_offload(struct hn_data *hv,
 	    == HN_NDIS_LSOV2_CAP_IP6)
 		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
 
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
-				    DEV_RX_OFFLOAD_CRC_STRIP;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
 
 	if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
 		dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM;
@@ -909,6 +913,37 @@ int hn_rndis_get_offload(struct hn_data *hv,
 	return 0;
 }
 
+uint32_t
+hn_rndis_get_ptypes(struct hn_data *hv)
+{
+	struct ndis_offload hwcaps;
+	uint32_t ptypes;
+	int error;
+
+	memset(&hwcaps, 0, sizeof(hwcaps));
+
+	error = hn_rndis_query_hwcaps(hv, &hwcaps);
+	if (error) {
+		PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
+		return RTE_PTYPE_L2_ETHER;
+	}
+
+	ptypes = RTE_PTYPE_L2_ETHER;
+
+	if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
+		ptypes |= RTE_PTYPE_L3_IPV4;
+
+	if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) ||
+	    (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6))
+		ptypes |= RTE_PTYPE_L4_TCP;
+
+	if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) ||
+	    (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6))
+		ptypes |= RTE_PTYPE_L4_UDP;
+
+	return ptypes;
+}
+
 int
 hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter)
 {
diff --git a/drivers/net/netvsc/hn_rndis.h b/drivers/net/netvsc/hn_rndis.h
index 89e2e6ba..319b497a 100644
--- a/drivers/net/netvsc/hn_rndis.h
+++ b/drivers/net/netvsc/hn_rndis.h
@@ -6,7 +6,7 @@ struct hn_data;
 
 void hn_rndis_receive_response(struct hn_data *hv,
 			      const void *data, uint32_t len);
-void	hn_rndis_link_status(struct hn_data *hv, const void *data);
+void	hn_rndis_link_status(struct rte_eth_dev *dev, const void *msg);
 int	hn_rndis_attach(struct hn_data *hv);
 void	hn_rndis_detach(struct hn_data *hv);
 int	hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr);
@@ -24,6 +24,7 @@ int	hn_rndis_query_rsscaps(struct hn_data *hv,
 			       unsigned int *rxr_cnt0);
 int	hn_rndis_conf_rss(struct hn_data *hv,
 			  const struct rte_eth_rss_conf *rss_conf);
+uint32_t hn_rndis_get_ptypes(struct hn_data *hv);
 
 #ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP
 void hn_rndis_dump(const void *buf);
diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
index 02ef27e3..f4a36641 100644
--- a/drivers/net/netvsc/hn_rxtx.c
+++ b/drivers/net/netvsc/hn_rxtx.c
@@ -10,6 +10,7 @@
 #include <errno.h>
 #include <unistd.h>
 #include <strings.h>
+#include <malloc.h>
 
 #include <rte_ethdev.h>
 #include <rte_memcpy.h>
@@ -216,6 +217,7 @@ hn_dev_tx_queue_setup(struct rte_eth_dev *dev,
 	struct hn_data *hv = dev->data->dev_private;
 	struct hn_tx_queue *txq;
 	uint32_t tx_free_thresh;
+	int err;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -245,8 +247,14 @@ hn_dev_tx_queue_setup(struct rte_eth_dev *dev,
 
 	hn_reset_txagg(txq);
 
-	dev->data->tx_queues[queue_idx] = txq;
+	err = hn_vf_tx_queue_setup(dev, queue_idx, nb_desc,
+				     socket_id, tx_conf);
+	if (err) {
+		rte_free(txq);
+		return err;
+	}
 
+	dev->data->tx_queues[queue_idx] = txq;
 	return 0;
 }
 
@@ -269,17 +277,6 @@ hn_dev_tx_queue_release(void *arg)
 	rte_free(txq);
 }
 
-void
-hn_dev_tx_queue_info(struct rte_eth_dev *dev, uint16_t queue_idx,
-		     struct rte_eth_txq_info *qinfo)
-{
-	struct hn_data *hv = dev->data->dev_private;
-	struct hn_tx_queue *txq = dev->data->rx_queues[queue_idx];
-
-	qinfo->conf.tx_free_thresh = txq->free_thresh;
-	qinfo->nb_desc = hv->tx_pool->size;
-}
-
 static void
 hn_nvs_send_completed(struct rte_eth_dev *dev, uint16_t queue_id,
 		      unsigned long xactid, const struct hn_nvs_rndis_ack *ack)
@@ -533,7 +530,7 @@ static void hn_rxpkt(struct hn_rx_queue *rxq, struct hn_rx_bufinfo *rxb,
 	hn_update_packet_stats(&rxq->stats, m);
 
 	if (unlikely(rte_ring_sp_enqueue(rxq->rx_ring, m) != 0)) {
-		++rxq->ring_full;
+		++rxq->stats.ring_full;
 		rte_pktmbuf_free(m);
 	}
 }
@@ -600,7 +597,7 @@ error:
 }
 
 static void
-hn_rndis_receive(const struct rte_eth_dev *dev, struct hn_rx_queue *rxq,
+hn_rndis_receive(struct rte_eth_dev *dev, struct hn_rx_queue *rxq,
 		 struct hn_rx_bufinfo *rxb, void *buf, uint32_t len)
 {
 	const struct rndis_msghdr *hdr = buf;
@@ -612,7 +609,7 @@ hn_rndis_receive(const struct rte_eth_dev *dev, struct hn_rx_queue *rxq,
 		break;
 
 	case RNDIS_INDICATE_STATUS_MSG:
-		hn_rndis_link_status(rxq->hv, buf);
+		hn_rndis_link_status(dev, buf);
 		break;
 
 	case RNDIS_INITIALIZE_CMPLT:
@@ -712,22 +709,59 @@ hn_nvs_handle_rxbuf(struct rte_eth_dev *dev,
 	hn_rx_buf_release(rxb);
 }
 
+/*
+ * Called when NVS inband events are received.
+ * Send up a two part message with port_id and the NVS message
+ * to the pipe to the netvsc-vf-event control thread.
+ */
+static void hn_nvs_handle_notify(struct rte_eth_dev *dev,
+				 const struct vmbus_chanpkt_hdr *pkt,
+				 const void *data)
+{
+	const struct hn_nvs_hdr *hdr = data;
+
+	switch (hdr->type) {
+	case NVS_TYPE_TXTBL_NOTE:
+		/* Transmit indirection table has locking problems
+		 * in DPDK and therefore not implemented
+		 */
+		PMD_DRV_LOG(DEBUG, "host notify of transmit indirection table");
+		break;
+
+	case NVS_TYPE_VFASSOC_NOTE:
+		hn_nvs_handle_vfassoc(dev, pkt, data);
+		break;
+
+	default:
+		PMD_DRV_LOG(INFO,
+			    "got notify, nvs type %u", hdr->type);
+	}
+}
+
 struct hn_rx_queue *hn_rx_queue_alloc(struct hn_data *hv,
 				      uint16_t queue_id,
 				      unsigned int socket_id)
 {
 	struct hn_rx_queue *rxq;
 
-	rxq = rte_zmalloc_socket("HN_RXQ",
-				 sizeof(*rxq) + HN_RXQ_EVENT_DEFAULT,
+	rxq = rte_zmalloc_socket("HN_RXQ", sizeof(*rxq),
 				 RTE_CACHE_LINE_SIZE, socket_id);
-	if (rxq) {
-		rxq->hv = hv;
-		rxq->chan = hv->channels[queue_id];
-		rte_spinlock_init(&rxq->ring_lock);
-		rxq->port_id = hv->port_id;
-		rxq->queue_id = queue_id;
+	if (!rxq)
+		return NULL;
+
+	rxq->hv = hv;
+	rxq->chan = hv->channels[queue_id];
+	rte_spinlock_init(&rxq->ring_lock);
+	rxq->port_id = hv->port_id;
+	rxq->queue_id = queue_id;
+	rxq->event_sz = HN_RXQ_EVENT_DEFAULT;
+	rxq->event_buf = rte_malloc_socket("HN_EVENTS", HN_RXQ_EVENT_DEFAULT,
+					   RTE_CACHE_LINE_SIZE, socket_id);
+	if (!rxq->event_buf) {
+		rte_free(rxq);
+		return NULL;
 	}
+
 	return rxq;
 }
 
@@ -735,13 +769,14 @@ int
 hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
 		      uint16_t queue_idx, uint16_t nb_desc,
 		      unsigned int socket_id,
-		      const struct rte_eth_rxconf *rx_conf __rte_unused,
+		      const struct rte_eth_rxconf *rx_conf,
 		      struct rte_mempool *mp)
 {
 	struct hn_data *hv = dev->data->dev_private;
 	char ring_name[RTE_RING_NAMESIZE];
 	struct hn_rx_queue *rxq;
 	unsigned int count;
+	int error = -ENOMEM;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -771,6 +806,11 @@ hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	if (!rxq->rx_ring)
 		goto fail;
 
+	error = hn_vf_rx_queue_setup(dev, queue_idx, nb_desc,
+				     socket_id, rx_conf, mp);
+	if (error)
+		goto fail;
+
 	dev->data->rx_queues[queue_idx] = rxq;
 	return 0;
 
@@ -778,7 +818,7 @@ fail:
 	rte_ring_free(rxq->rx_ring);
 	rte_free(rxq->event_buf);
 	rte_free(rxq);
-	return -ENOMEM;
+	return error;
 }
 
 void
@@ -795,77 +835,79 @@ hn_dev_rx_queue_release(void *arg)
 	rxq->rx_ring = NULL;
 	rxq->mb_pool = NULL;
 
+	hn_vf_rx_queue_release(rxq->hv, rxq->queue_id);
+
+	/* Keep primary queue to allow for control operations */
 	if (rxq != rxq->hv->primary) {
 		rte_free(rxq->event_buf);
 		rte_free(rxq);
 	}
 }
 
-void
-hn_dev_rx_queue_info(struct rte_eth_dev *dev, uint16_t queue_idx,
-		     struct rte_eth_rxq_info *qinfo)
-{
-	struct hn_rx_queue *rxq = dev->data->rx_queues[queue_idx];
-
-	qinfo->mp = rxq->mb_pool;
-	qinfo->scattered_rx = 1;
-	qinfo->nb_desc = rte_ring_get_capacity(rxq->rx_ring);
-}
-
-static void
-hn_nvs_handle_notify(const struct vmbus_chanpkt_hdr *pkthdr,
-		     const void *data)
+int
+hn_dev_tx_done_cleanup(void *arg, uint32_t free_cnt)
 {
-	const struct hn_nvs_hdr *hdr = data;
-
-	if (unlikely(vmbus_chanpkt_datalen(pkthdr) < sizeof(*hdr))) {
-		PMD_DRV_LOG(ERR, "invalid nvs notify");
-		return;
-	}
+	struct hn_tx_queue *txq = arg;
 
-	PMD_DRV_LOG(INFO,
-		    "got notify, nvs type %u", hdr->type);
+	return hn_process_events(txq->hv, txq->queue_id, free_cnt);
 }
 
 /*
  * Process pending events on the channel.
  * Called from both Rx queue poll and Tx cleanup
  */
-void hn_process_events(struct hn_data *hv, uint16_t queue_id)
+uint32_t hn_process_events(struct hn_data *hv, uint16_t queue_id,
+			   uint32_t tx_limit)
 {
 	struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id];
 	struct hn_rx_queue *rxq;
 	uint32_t bytes_read = 0;
+	uint32_t tx_done = 0;
 	int ret = 0;
 
 	rxq = queue_id == 0 ? hv->primary : dev->data->rx_queues[queue_id];
 
 	/* If no pending data then nothing to do */
 	if (rte_vmbus_chan_rx_empty(rxq->chan))
-		return;
+		return 0;
 
 	/*
 	 * Since channel is shared between Rx and TX queue need to have a lock
 	 * since DPDK does not force same CPU to be used for Rx/Tx.
 	 */
 	if (unlikely(!rte_spinlock_trylock(&rxq->ring_lock)))
-		return;
+		return 0;
 
 	for (;;) {
 		const struct vmbus_chanpkt_hdr *pkt;
-		uint32_t len = HN_RXQ_EVENT_DEFAULT;
+		uint32_t len = rxq->event_sz;
 		const void *data;
 
+retry:
 		ret = rte_vmbus_chan_recv_raw(rxq->chan, rxq->event_buf, &len);
 		if (ret == -EAGAIN)
 			break;	/* ring is empty */
 
-		else if (ret == -ENOBUFS)
-			rte_exit(EXIT_FAILURE, "event buffer not big enough (%u < %u)",
-				 HN_RXQ_EVENT_DEFAULT, len);
-		else if (ret <= 0)
+		if (unlikely(ret == -ENOBUFS)) {
+			/* event buffer not large enough to read ring */
+
+			PMD_DRV_LOG(DEBUG,
+				    "event buffer expansion (need %u)", len);
+			rxq->event_sz = len + len / 4;
+			rxq->event_buf = rte_realloc(rxq->event_buf, rxq->event_sz,
+						     RTE_CACHE_LINE_SIZE);
+			if (rxq->event_buf)
+				goto retry;
+			/* out of memory, no more events now */
+			rxq->event_sz = 0;
+			break;
+		}
+
+		if (unlikely(ret <= 0)) {
+			/* This indicates a failure to communicate (or worse) */
 			rte_exit(EXIT_FAILURE,
 				 "vmbus ring buffer error: %d", ret);
+		}
 
 		bytes_read += ret;
 		pkt = (const struct vmbus_chanpkt_hdr *)rxq->event_buf;
@@ -873,6 +915,7 @@ void hn_process_events(struct hn_data *hv, uint16_t queue_id)
 
 		switch (pkt->type) {
 		case VMBUS_CHANPKT_TYPE_COMP:
+			++tx_done;
 			hn_nvs_handle_comp(dev, queue_id, pkt, data);
 			break;
 
@@ -881,7 +924,7 @@ void hn_process_events(struct hn_data *hv, uint16_t queue_id)
 			break;
 
 		case VMBUS_CHANPKT_TYPE_INBAND:
-			hn_nvs_handle_notify(pkt, data);
+			hn_nvs_handle_notify(dev, pkt, data);
 			break;
 
 		default:
@@ -889,6 +932,9 @@ void hn_process_events(struct hn_data *hv, uint16_t queue_id)
 			break;
 		}
 
+		if (tx_limit && tx_done >= tx_limit)
+			break;
+
 		if (rxq->rx_ring && rte_ring_full(rxq->rx_ring))
 			break;
 	}
@@ -897,6 +943,8 @@ void hn_process_events(struct hn_data *hv, uint16_t queue_id)
 		rte_vmbus_chan_signal_read(rxq->chan, bytes_read);
 
 	rte_spinlock_unlock(&rxq->ring_lock);
+
+	return tx_done;
 }
 
 static void hn_append_to_chim(struct hn_tx_queue *txq,
@@ -967,7 +1015,7 @@ static struct hn_txdesc *hn_new_txd(struct hn_data *hv,
 	struct hn_txdesc *txd;
 
 	if (rte_mempool_get(hv->tx_pool, (void **)&txd)) {
-		++txq->stats.nomemory;
+		++txq->stats.ring_full;
 		PMD_TX_LOG(DEBUG, "tx pool exhausted!");
 		return NULL;
 	}
@@ -1235,7 +1283,9 @@ uint16_t
 hn_xmit_pkts(void *ptxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 {
 	struct hn_tx_queue *txq = ptxq;
+	uint16_t queue_id = txq->queue_id;
 	struct hn_data *hv = txq->hv;
+	struct rte_eth_dev *vf_dev;
 	bool need_sig = false;
 	uint16_t nb_tx;
 	int ret;
@@ -1243,8 +1293,17 @@ hn_xmit_pkts(void *ptxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	if (unlikely(hv->closed))
 		return 0;
 
+	/* Transmit over VF if present and up */
+	vf_dev = hv->vf_dev;
+	rte_compiler_barrier();
+	if (vf_dev && vf_dev->data->dev_started) {
+		void *sub_q = vf_dev->data->tx_queues[queue_id];
+
+		return (*vf_dev->tx_pkt_burst)(sub_q, tx_pkts, nb_pkts);
+	}
+
 	if (rte_mempool_avail_count(hv->tx_pool) <= txq->free_thresh)
-		hn_process_events(hv, txq->queue_id);
+		hn_process_events(hv, txq->queue_id, 0);
 
 	for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
 		struct rte_mbuf *m = tx_pkts[nb_tx];
@@ -1264,7 +1323,7 @@ hn_xmit_pkts(void *ptxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			if (unlikely(!pkt))
 				break;
 
-			hn_encap(pkt, txq->queue_id, m);
+			hn_encap(pkt, queue_id, m);
 			hn_append_to_chim(txq, pkt, m);
 
 			rte_pktmbuf_free(m);
@@ -1291,7 +1350,7 @@ hn_xmit_pkts(void *ptxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			txd->data_size += m->pkt_len;
 			++txd->packets;
 
-			hn_encap(pkt, txq->queue_id, m);
+			hn_encap(pkt, queue_id, m);
 
 			ret = hn_xmit_sg(txq, txd, m, &need_sig);
 			if (unlikely(ret != 0)) {
@@ -1320,15 +1379,36 @@ hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 {
 	struct hn_rx_queue *rxq = prxq;
 	struct hn_data *hv = rxq->hv;
+	struct rte_eth_dev *vf_dev;
+	uint16_t nb_rcv;
 
 	if (unlikely(hv->closed))
 		return 0;
 
-	/* If ring is empty then process more */
-	if (rte_ring_count(rxq->rx_ring) < nb_pkts)
-		hn_process_events(hv, rxq->queue_id);
+	vf_dev = hv->vf_dev;
+	rte_compiler_barrier();
+
+	if (vf_dev && vf_dev->data->dev_started) {
+		/* Normally, with SR-IOV the ring buffer will be empty */
+		hn_process_events(hv, rxq->queue_id, 0);
+
+		/* Get mbufs some bufs off of staging ring */
+		nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
+						   (void **)rx_pkts,
+						   nb_pkts / 2, NULL);
+		/* And rest off of VF */
+		nb_rcv += rte_eth_rx_burst(vf_dev->data->port_id,
+					   rxq->queue_id,
+					   rx_pkts + nb_rcv, nb_pkts - nb_rcv);
+	} else {
+		/* If receive ring is not full then get more */
+		if (rte_ring_count(rxq->rx_ring) < nb_pkts)
+			hn_process_events(hv, rxq->queue_id, 0);
+
+		nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
+						   (void **)rx_pkts,
+						   nb_pkts, NULL);
+	}
 
-	/* Get mbufs off staging ring */
-	return rte_ring_sc_dequeue_burst(rxq->rx_ring, (void **)rx_pkts,
-					 nb_pkts, NULL);
+	return nb_rcv;
 }
diff --git a/drivers/net/netvsc/hn_var.h b/drivers/net/netvsc/hn_var.h
index f7ff8585..e1072c7c 100644
--- a/drivers/net/netvsc/hn_var.h
+++ b/drivers/net/netvsc/hn_var.h
@@ -20,6 +20,9 @@
 /* Retry interval */
 #define HN_CHAN_INTERVAL_US	100
 
+/* Host monitor interval */
+#define HN_CHAN_LATENCY_NS	50000
+
 /* Buffers need to be aligned */
 #ifndef PAGE_SIZE
 #define PAGE_SIZE 4096
@@ -36,7 +39,7 @@ struct hn_stats {
 	uint64_t	packets;
 	uint64_t	bytes;
 	uint64_t	errors;
-	uint64_t	nomemory;
+	uint64_t	ring_full;
 	uint64_t	multicast;
 	uint64_t	broadcast;
 	/* Size bins in array as RFC 2819, undersized [0], 64 [1], etc */
@@ -75,9 +78,8 @@ struct hn_rx_queue {
 	uint16_t port_id;
 	uint16_t queue_id;
 	struct hn_stats stats;
-	uint64_t ring_full;
 
-	uint8_t	event_buf[];
+	void *event_buf;
 };
 
 
@@ -92,8 +94,11 @@ struct hn_rx_bufinfo {
 struct hn_data {
 	struct rte_vmbus_device *vmbus;
 	struct hn_rx_queue *primary;
+	struct rte_eth_dev *vf_dev;		/* Subordinate device */
+	rte_spinlock_t  vf_lock;
 	uint16_t	port_id;
 	bool		closed;
+	bool		vf_present;
 	uint32_t	link_status;
 	uint32_t	link_speed;
 
@@ -110,6 +115,7 @@ struct hn_data {
 	uint32_t	chim_szmax;		/* Max size per buffer */
 	uint32_t	chim_cnt;		/* Max packets per buffer */
 
+	uint32_t	latency;
 	uint32_t	nvs_ver;
 	uint32_t	ndis_ver;
 	uint32_t	rndis_agg_size;
@@ -121,6 +127,10 @@ struct hn_data {
 	uint8_t		rndis_resp[256];
 
 	struct ether_addr mac_addr;
+
+	struct rte_eth_dev_owner owner;
+	struct rte_intr_handle vf_intr;
+
 	struct vmbus_channel *channels[HN_MAX_CHANNELS];
 };
 
@@ -130,7 +140,8 @@ hn_primary_chan(const struct hn_data *hv)
 	return hv->channels[0];
 }
 
-void hn_process_events(struct hn_data *hv, uint16_t queue_id);
+uint32_t hn_process_events(struct hn_data *hv, uint16_t queue_id,
+		       uint32_t tx_limit);
 
 uint16_t hn_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		      uint16_t nb_pkts);
@@ -138,12 +149,14 @@ uint16_t hn_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		      uint16_t nb_pkts);
 
 int	hn_tx_pool_init(struct rte_eth_dev *dev);
+int	hn_dev_link_update(struct rte_eth_dev *dev, int wait);
 int	hn_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 			      uint16_t nb_desc, unsigned int socket_id,
 			      const struct rte_eth_txconf *tx_conf);
 void	hn_dev_tx_queue_release(void *arg);
 void	hn_dev_tx_queue_info(struct rte_eth_dev *dev, uint16_t queue_idx,
 			     struct rte_eth_txq_info *qinfo);
+int	hn_dev_tx_done_cleanup(void *arg, uint32_t free_cnt);
 
 struct hn_rx_queue *hn_rx_queue_alloc(struct hn_data *hv,
 				      uint16_t queue_id,
@@ -154,5 +167,46 @@ int	hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
 			      const struct rte_eth_rxconf *rx_conf,
 			      struct rte_mempool *mp);
 void	hn_dev_rx_queue_release(void *arg);
-void	hn_dev_rx_queue_info(struct rte_eth_dev *dev, uint16_t queue_idx,
-			     struct rte_eth_rxq_info *qinfo);
+
+void	hn_vf_info_get(struct hn_data *hv,
+		       struct rte_eth_dev_info *info);
+int	hn_vf_add(struct rte_eth_dev *dev, struct hn_data *hv);
+int	hn_vf_configure(struct rte_eth_dev *dev,
+			const struct rte_eth_conf *dev_conf);
+const uint32_t *hn_vf_supported_ptypes(struct rte_eth_dev *dev);
+int	hn_vf_start(struct rte_eth_dev *dev);
+void	hn_vf_reset(struct rte_eth_dev *dev);
+void	hn_vf_stop(struct rte_eth_dev *dev);
+void	hn_vf_close(struct rte_eth_dev *dev);
+
+void	hn_vf_allmulticast_enable(struct rte_eth_dev *dev);
+void	hn_vf_allmulticast_disable(struct rte_eth_dev *dev);
+void	hn_vf_promiscuous_enable(struct rte_eth_dev *dev);
+void	hn_vf_promiscuous_disable(struct rte_eth_dev *dev);
+int	hn_vf_mc_addr_list(struct rte_eth_dev *dev,
+			   struct ether_addr *mc_addr_set,
+			   uint32_t nb_mc_addr);
+
+int	hn_vf_link_update(struct rte_eth_dev *dev,
+			  int wait_to_complete);
+int	hn_vf_tx_queue_setup(struct rte_eth_dev *dev,
+			     uint16_t queue_idx, uint16_t nb_desc,
+			     unsigned int socket_id,
+			     const struct rte_eth_txconf *tx_conf);
+void	hn_vf_tx_queue_release(struct hn_data *hv, uint16_t queue_id);
+int	hn_vf_rx_queue_setup(struct rte_eth_dev *dev,
+			     uint16_t queue_idx, uint16_t nb_desc,
+			     unsigned int socket_id,
+			     const struct rte_eth_rxconf *rx_conf,
+			     struct rte_mempool *mp);
+void	hn_vf_rx_queue_release(struct hn_data *hv, uint16_t queue_id);
+
+int	hn_vf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats);
+void	hn_vf_stats_reset(struct rte_eth_dev *dev);
+int	hn_vf_xstats_get_names(struct rte_eth_dev *dev,
+			       struct rte_eth_xstat_name *xstats_names,
+			       unsigned int size);
+int	hn_vf_xstats_get(struct rte_eth_dev *dev,
+			 struct rte_eth_xstat *xstats,
+			 unsigned int n);
+void	hn_vf_xstats_reset(struct rte_eth_dev *dev);
diff --git a/drivers/net/netvsc/hn_vf.c b/drivers/net/netvsc/hn_vf.c
new file mode 100644
index 00000000..7a84ad8c
--- /dev/null
+++ b/drivers/net/netvsc/hn_vf.c
@@ -0,0 +1,549 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018 Microsoft Corp.
+ * All rights reserved.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <sys/types.h>
+#include <sys/fcntl.h>
+#include <sys/uio.h>
+
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_ethdev_driver.h>
+#include <rte_lcore.h>
+#include <rte_memory.h>
+#include <rte_bus_vmbus.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_log.h>
+#include <rte_string_fns.h>
+
+#include "hn_logs.h"
+#include "hn_var.h"
+#include "hn_nvs.h"
+
+/* Search for VF with matching MAC address, return port id */
+static int hn_vf_match(const struct rte_eth_dev *dev)
+{
+	const struct ether_addr *mac = dev->data->mac_addrs;
+	char buf[32];
+	int i;
+
+	ether_format_addr(buf, sizeof(buf), mac);
+	RTE_ETH_FOREACH_DEV(i) {
+		const struct rte_eth_dev *vf_dev = &rte_eth_devices[i];
+		const struct ether_addr *vf_mac = vf_dev->data->mac_addrs;
+
+		if (vf_dev == dev)
+			continue;
+
+		ether_format_addr(buf, sizeof(buf), vf_mac);
+		if (is_same_ether_addr(mac, vf_mac))
+			return i;
+	}
+	return -ENOENT;
+}
+
+/*
+ * Attach new PCI VF device and return the port_id
+ */
+static int hn_vf_attach(struct hn_data *hv, uint16_t port_id,
+			struct rte_eth_dev **vf_dev)
+{
+	struct rte_eth_dev_owner owner = { .id = RTE_ETH_DEV_NO_OWNER };
+	int ret;
+
+	ret = rte_eth_dev_owner_get(port_id, &owner);
+	if (ret < 0) {
+		PMD_DRV_LOG(ERR, "Can not find owner for port %d", port_id);
+		return ret;
+	}
+
+	if (owner.id != RTE_ETH_DEV_NO_OWNER) {
+		PMD_DRV_LOG(ERR, "Port %u already owned by other device %s",
+			    port_id, owner.name);
+		return -EBUSY;
+	}
+
+	ret = rte_eth_dev_owner_set(port_id, &hv->owner);
+	if (ret < 0) {
+		PMD_DRV_LOG(ERR, "Can set owner for port %d", port_id);
+		return ret;
+	}
+
+	PMD_DRV_LOG(DEBUG, "Attach VF device %u", port_id);
+	rte_smp_wmb();
+	*vf_dev = &rte_eth_devices[port_id];
+	return 0;
+}
+
+/* Add new VF device to synthetic device */
+int hn_vf_add(struct rte_eth_dev *dev, struct hn_data *hv)
+{
+	int port, err;
+
+	port = hn_vf_match(dev);
+	if (port < 0) {
+		PMD_DRV_LOG(NOTICE, "No matching MAC found");
+		return port;
+	}
+
+	rte_spinlock_lock(&hv->vf_lock);
+	if (hv->vf_dev) {
+		PMD_DRV_LOG(ERR, "VF already attached");
+		err = -EBUSY;
+	} else {
+		err = hn_vf_attach(hv, port, &hv->vf_dev);
+	}
+
+	if (err == 0) {
+		dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
+		hv->vf_intr = (struct rte_intr_handle) {
+			.fd = -1,
+			.type = RTE_INTR_HANDLE_EXT,
+		};
+		dev->intr_handle = &hv->vf_intr;
+		hn_nvs_set_datapath(hv, NVS_DATAPATH_VF);
+	}
+	rte_spinlock_unlock(&hv->vf_lock);
+
+	return err;
+}
+
+/* Remove new VF device */
+static void hn_vf_remove(struct hn_data *hv)
+{
+	struct rte_eth_dev *vf_dev;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (!vf_dev) {
+		PMD_DRV_LOG(ERR, "VF path not active");
+		rte_spinlock_unlock(&hv->vf_lock);
+		return;
+	}
+
+	/* Stop incoming packets from arriving on VF */
+	hn_nvs_set_datapath(hv, NVS_DATAPATH_SYNTHETIC);
+	hv->vf_dev = NULL;
+
+	/* Give back ownership */
+	rte_eth_dev_owner_unset(vf_dev->data->port_id, hv->owner.id);
+	rte_spinlock_unlock(&hv->vf_lock);
+}
+
+/* Handle VF association message from host */
+void
+hn_nvs_handle_vfassoc(struct rte_eth_dev *dev,
+		      const struct vmbus_chanpkt_hdr *hdr,
+		      const void *data)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	const struct hn_nvs_vf_association *vf_assoc = data;
+
+	if (unlikely(vmbus_chanpkt_datalen(hdr) < sizeof(*vf_assoc))) {
+		PMD_DRV_LOG(ERR, "invalid vf association NVS");
+		return;
+	}
+
+	PMD_DRV_LOG(DEBUG, "VF serial %u %s port %u",
+		    vf_assoc->serial,
+		    vf_assoc->allocated ? "add to" : "remove from",
+		    dev->data->port_id);
+
+	hv->vf_present = vf_assoc->allocated;
+
+	if (dev->state != RTE_ETH_DEV_ATTACHED)
+		return;
+
+	if (vf_assoc->allocated)
+		hn_vf_add(dev, hv);
+	else
+		hn_vf_remove(hv);
+}
+
+/*
+ * Merge the info from the VF and synthetic path.
+ * use the default config of the VF
+ * and the minimum number of queues and buffer sizes.
+ */
+static void hn_vf_info_merge(struct rte_eth_dev *vf_dev,
+			     struct rte_eth_dev_info *info)
+{
+	struct rte_eth_dev_info vf_info;
+
+	rte_eth_dev_info_get(vf_dev->data->port_id, &vf_info);
+
+	info->speed_capa = vf_info.speed_capa;
+	info->default_rxportconf = vf_info.default_rxportconf;
+	info->default_txportconf = vf_info.default_txportconf;
+
+	info->max_rx_queues = RTE_MIN(vf_info.max_rx_queues,
+				      info->max_rx_queues);
+	info->rx_offload_capa &= vf_info.rx_offload_capa;
+	info->rx_queue_offload_capa &= vf_info.rx_queue_offload_capa;
+	info->flow_type_rss_offloads &= vf_info.flow_type_rss_offloads;
+
+	info->max_tx_queues = RTE_MIN(vf_info.max_tx_queues,
+				      info->max_tx_queues);
+	info->tx_offload_capa &= vf_info.tx_offload_capa;
+	info->tx_queue_offload_capa &= vf_info.tx_queue_offload_capa;
+
+	info->min_rx_bufsize = RTE_MAX(vf_info.min_rx_bufsize,
+				       info->min_rx_bufsize);
+	info->max_rx_pktlen  = RTE_MAX(vf_info.max_rx_pktlen,
+				       info->max_rx_pktlen);
+}
+
+void hn_vf_info_get(struct hn_data *hv, struct rte_eth_dev_info *info)
+{
+	struct rte_eth_dev *vf_dev;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev)
+		hn_vf_info_merge(vf_dev, info);
+	rte_spinlock_unlock(&hv->vf_lock);
+}
+
+int hn_vf_link_update(struct rte_eth_dev *dev,
+		      int wait_to_complete)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+	int ret = 0;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev && vf_dev->dev_ops->link_update)
+		ret = (*vf_dev->dev_ops->link_update)(dev, wait_to_complete);
+	rte_spinlock_unlock(&hv->vf_lock);
+
+	return ret;
+}
+
+/* called when VF has link state interrupts enabled */
+static int hn_vf_lsc_event(uint16_t port_id __rte_unused,
+			   enum rte_eth_event_type event,
+			   void *cb_arg, void *out __rte_unused)
+{
+	struct rte_eth_dev *dev = cb_arg;
+
+	if (event != RTE_ETH_EVENT_INTR_LSC)
+		return 0;
+
+	/* if link state has changed pass on */
+	if (hn_dev_link_update(dev, 0) == 0)
+		return 0; /* no change */
+
+	return _rte_eth_dev_callback_process(dev,
+					     RTE_ETH_EVENT_INTR_LSC,
+					     NULL);
+}
+
+static int _hn_vf_configure(struct rte_eth_dev *dev,
+			    struct rte_eth_dev *vf_dev,
+			    const struct rte_eth_conf *dev_conf)
+{
+	struct rte_eth_conf vf_conf = *dev_conf;
+	uint16_t vf_port = vf_dev->data->port_id;
+	int ret;
+
+	if (dev_conf->intr_conf.lsc &&
+	    (vf_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
+		PMD_DRV_LOG(DEBUG, "enabling LSC for VF %u",
+			    vf_port);
+		vf_conf.intr_conf.lsc = 1;
+	} else {
+		PMD_DRV_LOG(DEBUG, "disabling LSC for VF %u",
+			    vf_port);
+		vf_conf.intr_conf.lsc = 0;
+	}
+
+	ret = rte_eth_dev_configure(vf_port,
+				    dev->data->nb_rx_queues,
+				    dev->data->nb_tx_queues,
+				    &vf_conf);
+	if (ret) {
+		PMD_DRV_LOG(ERR,
+			    "VF configuration failed: %d", ret);
+	} else if (vf_conf.intr_conf.lsc) {
+		ret = rte_eth_dev_callback_register(vf_port,
+						    RTE_ETH_DEV_INTR_LSC,
+						    hn_vf_lsc_event, dev);
+		if (ret)
+			PMD_DRV_LOG(ERR,
+				    "Failed to register LSC callback for VF %u",
+				    vf_port);
+	}
+	return ret;
+}
+
+/*
+ * Configure VF if present.
+ * Force VF to have same number of queues as synthetic device
+ */
+int hn_vf_configure(struct rte_eth_dev *dev,
+		    const struct rte_eth_conf *dev_conf)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+	int ret = 0;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev)
+		ret = _hn_vf_configure(dev, vf_dev, dev_conf);
+	rte_spinlock_unlock(&hv->vf_lock);
+	return ret;
+}
+
+const uint32_t *hn_vf_supported_ptypes(struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+	const uint32_t *ptypes = NULL;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev && vf_dev->dev_ops->dev_supported_ptypes_get)
+		ptypes = (*vf_dev->dev_ops->dev_supported_ptypes_get)(vf_dev);
+	rte_spinlock_unlock(&hv->vf_lock);
+
+	return ptypes;
+}
+
+int hn_vf_start(struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+	int ret = 0;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev)
+		ret = rte_eth_dev_start(vf_dev->data->port_id);
+	rte_spinlock_unlock(&hv->vf_lock);
+	return ret;
+}
+
+void hn_vf_stop(struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev)
+		rte_eth_dev_stop(vf_dev->data->port_id);
+	rte_spinlock_unlock(&hv->vf_lock);
+}
+
+/* If VF is present, then cascade configuration down */
+#define VF_ETHDEV_FUNC(dev, func)				\
+	{							\
+		struct hn_data *hv = (dev)->data->dev_private;	\
+		struct rte_eth_dev *vf_dev;			\
+		rte_spinlock_lock(&hv->vf_lock);		\
+		vf_dev = hv->vf_dev;				\
+		if (vf_dev)					\
+			func(vf_dev->data->port_id);		\
+		rte_spinlock_unlock(&hv->vf_lock);		\
+	}
+
+void hn_vf_reset(struct rte_eth_dev *dev)
+{
+	VF_ETHDEV_FUNC(dev, rte_eth_dev_reset);
+}
+
+void hn_vf_close(struct rte_eth_dev *dev)
+{
+	VF_ETHDEV_FUNC(dev, rte_eth_dev_close);
+}
+
+void hn_vf_stats_reset(struct rte_eth_dev *dev)
+{
+	VF_ETHDEV_FUNC(dev, rte_eth_stats_reset);
+}
+
+void hn_vf_allmulticast_enable(struct rte_eth_dev *dev)
+{
+	VF_ETHDEV_FUNC(dev, rte_eth_allmulticast_enable);
+}
+
+void hn_vf_allmulticast_disable(struct rte_eth_dev *dev)
+{
+	VF_ETHDEV_FUNC(dev, rte_eth_allmulticast_disable);
+}
+
+void hn_vf_promiscuous_enable(struct rte_eth_dev *dev)
+{
+	VF_ETHDEV_FUNC(dev, rte_eth_promiscuous_enable);
+}
+
+void hn_vf_promiscuous_disable(struct rte_eth_dev *dev)
+{
+	VF_ETHDEV_FUNC(dev, rte_eth_promiscuous_disable);
+}
+
+int hn_vf_mc_addr_list(struct rte_eth_dev *dev,
+			struct ether_addr *mc_addr_set,
+			uint32_t nb_mc_addr)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+	int ret = 0;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev)
+		ret = rte_eth_dev_set_mc_addr_list(vf_dev->data->port_id,
+						   mc_addr_set, nb_mc_addr);
+	rte_spinlock_unlock(&hv->vf_lock);
+	return ret;
+}
+
+int hn_vf_tx_queue_setup(struct rte_eth_dev *dev,
+			 uint16_t queue_idx, uint16_t nb_desc,
+			 unsigned int socket_id,
+			 const struct rte_eth_txconf *tx_conf)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+	int ret = 0;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev)
+		ret = rte_eth_tx_queue_setup(vf_dev->data->port_id,
+					     queue_idx, nb_desc,
+					     socket_id, tx_conf);
+	rte_spinlock_unlock(&hv->vf_lock);
+	return ret;
+}
+
+void hn_vf_tx_queue_release(struct hn_data *hv, uint16_t queue_id)
+{
+	struct rte_eth_dev *vf_dev;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev && vf_dev->dev_ops->tx_queue_release) {
+		void *subq = vf_dev->data->tx_queues[queue_id];
+
+		(*vf_dev->dev_ops->tx_queue_release)(subq);
+	}
+
+	rte_spinlock_unlock(&hv->vf_lock);
+}
+
+int hn_vf_rx_queue_setup(struct rte_eth_dev *dev,
+			 uint16_t queue_idx, uint16_t nb_desc,
+			 unsigned int socket_id,
+			 const struct rte_eth_rxconf *rx_conf,
+			 struct rte_mempool *mp)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+	int ret = 0;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev)
+		ret = rte_eth_rx_queue_setup(vf_dev->data->port_id,
+					     queue_idx, nb_desc,
+					     socket_id, rx_conf, mp);
+	rte_spinlock_unlock(&hv->vf_lock);
+	return ret;
+}
+
+void hn_vf_rx_queue_release(struct hn_data *hv, uint16_t queue_id)
+{
+	struct rte_eth_dev *vf_dev;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev && vf_dev->dev_ops->rx_queue_release) {
+		void *subq = vf_dev->data->rx_queues[queue_id];
+
+		(*vf_dev->dev_ops->rx_queue_release)(subq);
+	}
+	rte_spinlock_unlock(&hv->vf_lock);
+}
+
+int hn_vf_stats_get(struct rte_eth_dev *dev,
+		    struct rte_eth_stats *stats)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+	int ret = 0;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev)
+		ret = rte_eth_stats_get(vf_dev->data->port_id, stats);
+	rte_spinlock_unlock(&hv->vf_lock);
+	return ret;
+}
+
+int hn_vf_xstats_get_names(struct rte_eth_dev *dev,
+			   struct rte_eth_xstat_name *names,
+			   unsigned int n)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+	int i, count = 0;
+	char tmp[RTE_ETH_XSTATS_NAME_SIZE];
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev && vf_dev->dev_ops->xstats_get_names)
+		count = vf_dev->dev_ops->xstats_get_names(vf_dev, names, n);
+	rte_spinlock_unlock(&hv->vf_lock);
+
+	/* add vf_ prefix to xstat names */
+	if (names) {
+		for (i = 0; i < count; i++) {
+			snprintf(tmp, sizeof(tmp), "vf_%s", names[i].name);
+			strlcpy(names[i].name, tmp, sizeof(names[i].name));
+		}
+	}
+
+	return count;
+}
+
+int hn_vf_xstats_get(struct rte_eth_dev *dev,
+		     struct rte_eth_xstat *xstats,
+		     unsigned int n)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+	int count = 0;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev && vf_dev->dev_ops->xstats_get)
+		count = vf_dev->dev_ops->xstats_get(vf_dev, xstats, n);
+	rte_spinlock_unlock(&hv->vf_lock);
+
+	return count;
+}
+
+void hn_vf_xstats_reset(struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_dev *vf_dev;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_dev = hv->vf_dev;
+	if (vf_dev && vf_dev->dev_ops->xstats_reset)
+		vf_dev->dev_ops->xstats_reset(vf_dev);
+	rte_spinlock_unlock(&hv->vf_lock);
+}
diff --git a/drivers/net/netvsc/meson.build b/drivers/net/netvsc/meson.build
index a717cdd4..c8426971 100644
--- a/drivers/net/netvsc/meson.build
+++ b/drivers/net/netvsc/meson.build
@@ -3,7 +3,7 @@
 
 build = dpdk_conf.has('RTE_LIBRTE_VMBUS_BUS')
 version = 2
-sources = files('hn_ethdev.c', 'hn_rxtx.c', 'hn_rndis.c', 'hn_nvs.c')
+sources = files('hn_ethdev.c', 'hn_rxtx.c', 'hn_rndis.c', 'hn_nvs.c', 'hn_vf.c')
 
 deps += ['bus_vmbus' ]
 
diff --git a/drivers/net/nfp/Makefile b/drivers/net/nfp/Makefile
index ab4e0a7d..d3fa5699 100644
--- a/drivers/net/nfp/Makefile
+++ b/drivers/net/nfp/Makefile
@@ -10,6 +10,7 @@ LIB = librte_pmd_nfp.a
 
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 
 LDLIBS += -lm
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
diff --git a/drivers/net/nfp/meson.build b/drivers/net/nfp/meson.build
index 3ba37e27..ba6a22e8 100644
--- a/drivers/net/nfp/meson.build
+++ b/drivers/net/nfp/meson.build
@@ -1,6 +1,9 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2018 Intel Corporation
 
+if host_machine.system() != 'linux'
+        build = false
+endif
 sources = files('nfpcore/nfp_cpp_pcie_ops.c',
 	'nfpcore/nfp_nsp.c',
 	'nfpcore/nfp_cppcore.c',
@@ -14,3 +17,5 @@ sources = files('nfpcore/nfp_cpp_pcie_ops.c',
 	'nfpcore/nfp_nsp_eth.c',
 	'nfpcore/nfp_hwinfo.c',
 	'nfp_net.c')
+
+allow_experimental_apis = true
diff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c
index 6e5e305f..bab1f68e 100644
--- a/drivers/net/nfp/nfp_net.c
+++ b/drivers/net/nfp/nfp_net.c
@@ -411,12 +411,6 @@ nfp_net_configure(struct rte_eth_dev *dev)
 		return -EINVAL;
 	}
 
-	/* KEEP_CRC offload flag is not supported by PMD
-	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
-	 */
-	if (rte_eth_dev_must_keep_crc(rxmode->offloads))
-		PMD_INIT_LOG(INFO, "HW does strip CRC. No configurable!");
-
 	return 0;
 }
 
@@ -1168,8 +1162,7 @@ nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 					     DEV_RX_OFFLOAD_UDP_CKSUM |
 					     DEV_RX_OFFLOAD_TCP_CKSUM;
 
-	dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME |
-				     DEV_RX_OFFLOAD_KEEP_CRC;
+	dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME;
 
 	if (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)
 		dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT;
@@ -1205,8 +1198,10 @@ nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 		.tx_rs_thresh = DEFAULT_TX_RSBIT_THRESH,
 	};
 
-	dev_info->flow_type_rss_offloads = ETH_RSS_NONFRAG_IPV4_TCP |
+	dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
+					   ETH_RSS_NONFRAG_IPV4_TCP |
 					   ETH_RSS_NONFRAG_IPV4_UDP |
+					   ETH_RSS_IPV6 |
 					   ETH_RSS_NONFRAG_IPV6_TCP |
 					   ETH_RSS_NONFRAG_IPV6_UDP;
 
@@ -1786,21 +1781,20 @@ nfp_net_rx_cksum(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd,
 		return;
 
 	/* If IPv4 and IP checksum error, fail */
-	if ((rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM) &&
-	    !(rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM_OK))
+	if (unlikely((rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM) &&
+	    !(rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM_OK)))
 		mb->ol_flags |= PKT_RX_IP_CKSUM_BAD;
+	else
+		mb->ol_flags |= PKT_RX_IP_CKSUM_GOOD;
 
 	/* If neither UDP nor TCP return */
 	if (!(rxd->rxd.flags & PCIE_DESC_RX_TCP_CSUM) &&
 	    !(rxd->rxd.flags & PCIE_DESC_RX_UDP_CSUM))
 		return;
 
-	if ((rxd->rxd.flags & PCIE_DESC_RX_TCP_CSUM) &&
-	    !(rxd->rxd.flags & PCIE_DESC_RX_TCP_CSUM_OK))
-		mb->ol_flags |= PKT_RX_L4_CKSUM_BAD;
-
-	if ((rxd->rxd.flags & PCIE_DESC_RX_UDP_CSUM) &&
-	    !(rxd->rxd.flags & PCIE_DESC_RX_UDP_CSUM_OK))
+	if (likely(rxd->rxd.flags & PCIE_DESC_RX_L4_CSUM_OK))
+		mb->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
+	else
 		mb->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 }
 
@@ -1884,6 +1878,18 @@ nfp_net_set_hash(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd,
 	case NFP_NET_RSS_IPV6_EX:
 		mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;
 		break;
+	case NFP_NET_RSS_IPV4_TCP:
+		mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;
+		break;
+	case NFP_NET_RSS_IPV6_TCP:
+		mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;
+		break;
+	case NFP_NET_RSS_IPV4_UDP:
+		mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;
+		break;
+	case NFP_NET_RSS_IPV6_UDP:
+		mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;
+		break;
 	default:
 		mbuf->packet_type |= RTE_PTYPE_INNER_L4_MASK;
 	}
@@ -2465,14 +2471,22 @@ nfp_net_rss_hash_write(struct rte_eth_dev *dev,
 	rss_hf = rss_conf->rss_hf;
 
 	if (rss_hf & ETH_RSS_IPV4)
-		cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4 |
-				NFP_NET_CFG_RSS_IPV4_TCP |
-				NFP_NET_CFG_RSS_IPV4_UDP;
+		cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4;
+
+	if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
+		cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_TCP;
+
+	if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
+		cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_UDP;
 
 	if (rss_hf & ETH_RSS_IPV6)
-		cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6 |
-				NFP_NET_CFG_RSS_IPV6_TCP |
-				NFP_NET_CFG_RSS_IPV6_UDP;
+		cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6;
+
+	if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
+		cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_TCP;
+
+	if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
+		cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_UDP;
 
 	cfg_rss_ctrl |= NFP_NET_CFG_RSS_MASK;
 	cfg_rss_ctrl |= NFP_NET_CFG_RSS_TOEPLITZ;
@@ -2688,6 +2702,14 @@ nfp_net_init(struct rte_eth_dev *eth_dev)
 
 	pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 
+	/* NFP can not handle DMA addresses requiring more than 40 bits */
+	if (rte_eal_check_dma_mask(40)) {
+		RTE_LOG(ERR, PMD, "device %s can not be used:",
+				   pci_dev->device.name);
+		RTE_LOG(ERR, PMD, "\trestricted dma mask to 40 bits!\n");
+		return -ENODEV;
+	};
+
 	if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) ||
 	    (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) {
 		port = get_pf_port_number(eth_dev->data->name);
@@ -2886,6 +2908,9 @@ nfp_net_init(struct rte_eth_dev *eth_dev)
 	ether_addr_copy((struct ether_addr *)hw->mac_addr,
 			&eth_dev->data->mac_addrs[0]);
 
+	if (!(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
+		eth_dev->data->dev_flags |= RTE_ETH_DEV_NOLIVE_MAC_ADDR;
+
 	PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x "
 		     "mac=%02x:%02x:%02x:%02x:%02x:%02x",
 		     eth_dev->data->port_id, pci_dev->id.vendor_id,
@@ -3265,14 +3290,16 @@ static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev)
 
 static struct rte_pci_driver rte_nfp_net_pf_pmd = {
 	.id_table = pci_id_nfp_pf_net_map,
-	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+		     RTE_PCI_DRV_IOVA_AS_VA,
 	.probe = nfp_pf_pci_probe,
 	.remove = eth_nfp_pci_remove,
 };
 
 static struct rte_pci_driver rte_nfp_net_vf_pmd = {
 	.id_table = pci_id_nfp_vf_net_map,
-	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+		     RTE_PCI_DRV_IOVA_AS_VA,
 	.probe = eth_nfp_pci_probe,
 	.remove = eth_nfp_pci_remove,
 };
diff --git a/drivers/net/nfp/nfp_net_pmd.h b/drivers/net/nfp/nfp_net_pmd.h
index c1b044ee..b01036df 100644
--- a/drivers/net/nfp/nfp_net_pmd.h
+++ b/drivers/net/nfp/nfp_net_pmd.h
@@ -293,6 +293,8 @@ struct nfp_net_txq {
 #define PCIE_DESC_RX_UDP_CSUM_OK        (1 <<  1)
 #define PCIE_DESC_RX_VLAN               (1 <<  0)
 
+#define PCIE_DESC_RX_L4_CSUM_OK         (PCIE_DESC_RX_TCP_CSUM_OK | \
+					 PCIE_DESC_RX_UDP_CSUM_OK)
 struct nfp_net_rx_desc {
 	union {
 		/* Freelist descriptor */
diff --git a/drivers/net/null/rte_eth_null.c b/drivers/net/null/rte_eth_null.c
index 244f8654..159c1c1f 100644
--- a/drivers/net/null/rte_eth_null.c
+++ b/drivers/net/null/rte_eth_null.c
@@ -305,7 +305,6 @@ eth_dev_info(struct rte_eth_dev *dev,
 	dev_info->min_rx_bufsize = 0;
 	dev_info->reta_size = internals->reta_size;
 	dev_info->flow_type_rss_offloads = internals->flow_type_rss_offloads;
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -615,8 +614,7 @@ rte_pmd_null_probe(struct rte_vdev_device *dev)
 	params = rte_vdev_device_args(dev);
 	PMD_LOG(INFO, "Initializing pmd_null for %s", name);
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
-	    strlen(params) == 0) {
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 		eth_dev = rte_eth_dev_attach_secondary(name);
 		if (!eth_dev) {
 			PMD_LOG(ERR, "Failed to probe %s", name);
@@ -681,7 +679,9 @@ rte_pmd_null_remove(struct rte_vdev_device *dev)
 	if (eth_dev == NULL)
 		return -1;
 
-	rte_free(eth_dev->data->dev_private);
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+		/* mac_addrs must not be freed alone because part of dev_private */
+		eth_dev->data->mac_addrs = NULL;
 
 	rte_eth_dev_release_port(eth_dev);
 
diff --git a/drivers/net/octeontx/base/meson.build b/drivers/net/octeontx/base/meson.build
index 09f657ab..a06a2c89 100644
--- a/drivers/net/octeontx/base/meson.build
+++ b/drivers/net/octeontx/base/meson.build
@@ -13,8 +13,12 @@ foreach d: depends
 	static_objs += [get_variable('static_rte_' + d)]
 endforeach
 
+c_args = cflags
+if allow_experimental_apis
+	c_args += '-DALLOW_EXPERIMENTAL_API'
+endif
 base_lib = static_library('octeontx_base', sources,
-	c_args: cflags,
+	c_args: c_args,
 	dependencies: static_objs,
 )
 
diff --git a/drivers/net/octeontx/base/octeontx_io.h b/drivers/net/octeontx/base/octeontx_io.h
index d51ded23..04b9ce19 100644
--- a/drivers/net/octeontx/base/octeontx_io.h
+++ b/drivers/net/octeontx/base/octeontx_io.h
@@ -10,7 +10,7 @@
 
 #include <rte_io.h>
 
-/* In Cavium OcteonTX SoC, all accesses to the device registers are
+/* In Cavium OCTEON TX SoC, all accesses to the device registers are
  * implicitly strongly ordered. So, The relaxed version of IO operation is
  * safe to use with out any IO memory barriers.
  */
diff --git a/drivers/net/octeontx/octeontx_ethdev.c b/drivers/net/octeontx/octeontx_ethdev.c
index 0f3d5d67..06814862 100644
--- a/drivers/net/octeontx/octeontx_ethdev.c
+++ b/drivers/net/octeontx/octeontx_ethdev.c
@@ -281,14 +281,6 @@ octeontx_dev_configure(struct rte_eth_dev *dev)
 		return -EINVAL;
 	}
 
-	/* KEEP_CRC offload flag is not supported by PMD
-	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
-	 */
-	if (rte_eth_dev_must_keep_crc(rxmode->offloads)) {
-		PMD_INIT_LOG(NOTICE, "can't disable hw crc strip");
-		rxmode->offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
-	}
-
 	if (!(txmode->offloads & DEV_TX_OFFLOAD_MT_LOCKFREE)) {
 		PMD_INIT_LOG(NOTICE, "cant disable lockfree tx");
 		txmode->offloads |= DEV_TX_OFFLOAD_MT_LOCKFREE;
@@ -1023,12 +1015,22 @@ octeontx_create(struct rte_vdev_device *dev, int port, uint8_t evdev,
 		return 0;
 	}
 
+	/* Reserve an ethdev entry */
+	eth_dev = rte_eth_dev_allocate(octtx_name);
+	if (eth_dev == NULL) {
+		octeontx_log_err("failed to allocate rte_eth_dev");
+		res = -ENOMEM;
+		goto err;
+	}
+	data = eth_dev->data;
+
 	nic = rte_zmalloc_socket(octtx_name, sizeof(*nic), 0, socket_id);
 	if (nic == NULL) {
 		octeontx_log_err("failed to allocate nic structure");
 		res = -ENOMEM;
 		goto err;
 	}
+	data->dev_private = nic;
 
 	nic->port_id = port;
 	nic->evdev = evdev;
@@ -1045,21 +1047,11 @@ octeontx_create(struct rte_vdev_device *dev, int port, uint8_t evdev,
 		goto err;
 	}
 
-	/* Reserve an ethdev entry */
-	eth_dev = rte_eth_dev_allocate(octtx_name);
-	if (eth_dev == NULL) {
-		octeontx_log_err("failed to allocate rte_eth_dev");
-		res = -ENOMEM;
-		goto err;
-	}
-
 	eth_dev->device = &dev->device;
 	eth_dev->intr_handle = NULL;
 	eth_dev->data->kdrv = RTE_KDRV_NONE;
 	eth_dev->data->numa_node = dev->device.numa_node;
 
-	data = eth_dev->data;
-	data->dev_private = nic;
 	data->port_id = eth_dev->data->port_id;
 
 	nic->ev_queues = 1;
@@ -1111,12 +1103,7 @@ err:
 	if (nic)
 		octeontx_port_close(nic);
 
-	if (eth_dev != NULL) {
-		rte_free(eth_dev->data->mac_addrs);
-		rte_free(data);
-		rte_free(nic);
-		rte_eth_dev_release_port(eth_dev);
-	}
+	rte_eth_dev_release_port(eth_dev);
 
 	return res;
 }
@@ -1141,16 +1128,22 @@ octeontx_remove(struct rte_vdev_device *dev)
 		if (eth_dev == NULL)
 			return -ENODEV;
 
+		if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+			rte_eth_dev_release_port(eth_dev);
+			continue;
+		}
+
 		nic = octeontx_pmd_priv(eth_dev);
 		rte_event_dev_stop(nic->evdev);
 		PMD_INIT_LOG(INFO, "Closing octeontx device %s", octtx_name);
 
-		rte_free(eth_dev->data->mac_addrs);
-		rte_free(eth_dev->data->dev_private);
 		rte_eth_dev_release_port(eth_dev);
 		rte_event_dev_close(nic->evdev);
 	}
 
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
 	/* Free FC resource */
 	octeontx_pko_fc_free();
 
diff --git a/drivers/net/octeontx/octeontx_ethdev.h b/drivers/net/octeontx/octeontx_ethdev.h
index 14f16969..920f6f89 100644
--- a/drivers/net/octeontx/octeontx_ethdev.h
+++ b/drivers/net/octeontx/octeontx_ethdev.h
@@ -28,8 +28,7 @@
 #define OCTEONTX_MAX_BGX_PORTS			4
 #define OCTEONTX_MAX_LMAC_PER_BGX		4
 
-#define OCTEONTX_RX_OFFLOADS			(DEV_RX_OFFLOAD_CRC_STRIP \
-						| DEV_RX_OFFLOAD_CHECKSUM)
+#define OCTEONTX_RX_OFFLOADS			DEV_RX_OFFLOAD_CHECKSUM
 #define OCTEONTX_TX_OFFLOADS			DEV_TX_OFFLOAD_MT_LOCKFREE
 
 static inline struct octeontx_nic *
diff --git a/drivers/net/octeontx/octeontx_rxtx.c b/drivers/net/octeontx/octeontx_rxtx.c
index a9149b4e..1e201f32 100644
--- a/drivers/net/octeontx/octeontx_rxtx.c
+++ b/drivers/net/octeontx/octeontx_rxtx.c
@@ -19,40 +19,6 @@
 #include "octeontx_rxtx.h"
 #include "octeontx_logs.h"
 
-
-static __rte_always_inline uint16_t __hot
-__octeontx_xmit_pkts(void *lmtline_va, void *ioreg_va, int64_t *fc_status_va,
-			struct rte_mbuf *tx_pkt)
-{
-	uint64_t cmd_buf[4];
-	uint16_t gaura_id;
-
-	if (unlikely(*((volatile int64_t *)fc_status_va) < 0))
-		return -ENOSPC;
-
-	/* Get the gaura Id */
-	gaura_id = octeontx_fpa_bufpool_gaura((uintptr_t)tx_pkt->pool->pool_id);
-
-	/* Setup PKO_SEND_HDR_S */
-	cmd_buf[0] = tx_pkt->data_len & 0xffff;
-	cmd_buf[1] = 0x0;
-
-	/* Set don't free bit if reference count > 1 */
-	if (rte_mbuf_refcnt_read(tx_pkt) > 1)
-		cmd_buf[0] |= (1ULL << 58); /* SET DF */
-
-	/* Setup PKO_SEND_GATHER_S */
-	cmd_buf[(1 << 1) | 1] = rte_mbuf_data_iova(tx_pkt);
-	cmd_buf[(1 << 1) | 0] = PKO_SEND_GATHER_SUBDC |
-				PKO_SEND_GATHER_LDTYPE(0x1ull) |
-				PKO_SEND_GATHER_GAUAR((long)gaura_id) |
-				tx_pkt->data_len;
-
-	octeontx_reg_lmtst(lmtline_va, ioreg_va, cmd_buf, PKO_CMD_SZ);
-
-	return 0;
-}
-
 uint16_t __hot
 octeontx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 {
@@ -63,6 +29,7 @@ octeontx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 
 	count = 0;
 
+	rte_cio_wmb();
 	while (count < nb_pkts) {
 		res = __octeontx_xmit_pkts(dq->lmtline_va, dq->ioreg_va,
 					   dq->fc_status_va,
diff --git a/drivers/net/octeontx/octeontx_rxtx.h b/drivers/net/octeontx/octeontx_rxtx.h
index fe3e5ccd..d0d73b30 100644
--- a/drivers/net/octeontx/octeontx_rxtx.h
+++ b/drivers/net/octeontx/octeontx_rxtx.h
@@ -100,6 +100,39 @@ ptype_table[PTYPE_SIZE][PTYPE_SIZE][PTYPE_SIZE] = {
 
 };
 
+static __rte_always_inline int
+__octeontx_xmit_pkts(void *lmtline_va, void *ioreg_va, int64_t *fc_status_va,
+			struct rte_mbuf *tx_pkt)
+{
+	uint64_t cmd_buf[4] __rte_cache_aligned;
+	uint16_t gaura_id;
+
+	if (unlikely(*((volatile int64_t *)fc_status_va) < 0))
+		return -ENOSPC;
+
+	/* Get the gaura Id */
+	gaura_id = octeontx_fpa_bufpool_gpool((uintptr_t)tx_pkt->pool->pool_id);
+
+	/* Setup PKO_SEND_HDR_S */
+	cmd_buf[0] = tx_pkt->data_len & 0xffff;
+	cmd_buf[1] = 0x0;
+
+	/* Set don't free bit if reference count > 1 */
+	if (rte_mbuf_refcnt_read(tx_pkt) > 1)
+		cmd_buf[0] |= (1ULL << 58); /* SET DF */
+
+	/* Setup PKO_SEND_GATHER_S */
+	cmd_buf[(1 << 1) | 1] = rte_mbuf_data_iova(tx_pkt);
+	cmd_buf[(1 << 1) | 0] = PKO_SEND_GATHER_SUBDC |
+				PKO_SEND_GATHER_LDTYPE(0x1ull) |
+				PKO_SEND_GATHER_GAUAR((long)gaura_id) |
+				tx_pkt->data_len;
+
+	octeontx_reg_lmtst(lmtline_va, ioreg_va, cmd_buf, PKO_CMD_SZ);
+
+	return 0;
+}
+
 uint16_t
 octeontx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
 
diff --git a/drivers/net/pcap/rte_eth_pcap.c b/drivers/net/pcap/rte_eth_pcap.c
index e8810a17..7bbe72e2 100644
--- a/drivers/net/pcap/rte_eth_pcap.c
+++ b/drivers/net/pcap/rte_eth_pcap.c
@@ -7,6 +7,14 @@
 #include <time.h>
 
 #include <net/if.h>
+#include <sys/socket.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+
+#if defined(RTE_EXEC_ENV_BSDAPP)
+#include <sys/sysctl.h>
+#include <net/if_dl.h>
+#endif
 
 #include <pcap.h>
 
@@ -17,6 +25,7 @@
 #include <rte_malloc.h>
 #include <rte_mbuf.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #define RTE_ETH_PCAP_SNAPSHOT_LEN 65535
 #define RTE_ETH_PCAP_SNAPLEN ETHER_MAX_JUMBO_FRAME_LEN
@@ -29,6 +38,7 @@
 #define ETH_PCAP_RX_IFACE_IN_ARG "rx_iface_in"
 #define ETH_PCAP_TX_IFACE_ARG "tx_iface"
 #define ETH_PCAP_IFACE_ARG    "iface"
+#define ETH_PCAP_PHY_MAC_ARG  "phy_mac"
 
 #define ETH_PCAP_ARG_MAXLEN	64
 
@@ -39,6 +49,7 @@ static unsigned char tx_pcap_data[RTE_ETH_PCAP_SNAPLEN];
 static struct timeval start_time;
 static uint64_t start_cycles;
 static uint64_t hz;
+static uint8_t iface_idx;
 
 struct queue_stat {
 	volatile unsigned long pkts;
@@ -66,8 +77,10 @@ struct pcap_tx_queue {
 struct pmd_internals {
 	struct pcap_rx_queue rx_queue[RTE_PMD_PCAP_MAX_QUEUES];
 	struct pcap_tx_queue tx_queue[RTE_PMD_PCAP_MAX_QUEUES];
+	struct ether_addr eth_addr;
 	int if_index;
 	int single_iface;
+	int phy_mac;
 };
 
 struct pmd_devargs {
@@ -78,6 +91,7 @@ struct pmd_devargs {
 		const char *name;
 		const char *type;
 	} queue[RTE_PMD_PCAP_MAX_QUEUES];
+	int phy_mac;
 };
 
 static const char *valid_arguments[] = {
@@ -87,13 +101,10 @@ static const char *valid_arguments[] = {
 	ETH_PCAP_RX_IFACE_IN_ARG,
 	ETH_PCAP_TX_IFACE_ARG,
 	ETH_PCAP_IFACE_ARG,
+	ETH_PCAP_PHY_MAC_ARG,
 	NULL
 };
 
-static struct ether_addr eth_addr = {
-	.addr_bytes = { 0, 0, 0, 0x1, 0x2, 0x3 }
-};
-
 static struct rte_eth_link pmd_link = {
 		.link_speed = ETH_SPEED_NUM_10G,
 		.link_duplex = ETH_LINK_FULL_DUPLEX,
@@ -553,7 +564,6 @@ eth_dev_info(struct rte_eth_dev *dev,
 	dev_info->max_rx_queues = dev->data->nb_rx_queues;
 	dev_info->max_tx_queues = dev->data->nb_tx_queues;
 	dev_info->min_rx_bufsize = 0;
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -862,6 +872,20 @@ open_tx_iface(const char *key, const char *value, void *extra_args)
 	return open_iface(key, value, extra_args);
 }
 
+static int
+select_phy_mac(const char *key __rte_unused, const char *value,
+		void *extra_args)
+{
+	if (extra_args) {
+		const int phy_mac = atoi(value);
+		int *enable_phy_mac = extra_args;
+
+		if (phy_mac)
+			*enable_phy_mac = 1;
+	}
+	return 0;
+}
+
 static struct rte_vdev_driver pmd_pcap_drv;
 
 static int
@@ -889,11 +913,20 @@ pmd_init_internals(struct rte_vdev_device *vdev,
 	 * - and point eth_dev structure to new eth_dev_data structure
 	 */
 	*internals = (*eth_dev)->data->dev_private;
+	/*
+	 * Interface MAC = 02:70:63:61:70:<iface_idx>
+	 * derived from: 'locally administered':'p':'c':'a':'p':'iface_idx'
+	 * where the middle 4 characters are converted to hex.
+	 */
+	(*internals)->eth_addr = (struct ether_addr) {
+		.addr_bytes = { 0x02, 0x70, 0x63, 0x61, 0x70, iface_idx++ }
+	};
+	(*internals)->phy_mac = 0;
 	data = (*eth_dev)->data;
 	data->nb_rx_queues = (uint16_t)nb_rx_queues;
 	data->nb_tx_queues = (uint16_t)nb_tx_queues;
 	data->dev_link = pmd_link;
-	data->mac_addrs = &eth_addr;
+	data->mac_addrs = &(*internals)->eth_addr;
 
 	/*
 	 * NOTE: we'll replace the data element, of originally allocated
@@ -905,14 +938,95 @@ pmd_init_internals(struct rte_vdev_device *vdev,
 }
 
 static int
+eth_pcap_update_mac(const char *if_name, struct rte_eth_dev *eth_dev,
+		const unsigned int numa_node)
+{
+#if defined(RTE_EXEC_ENV_LINUXAPP)
+	void *mac_addrs;
+	struct ifreq ifr;
+	int if_fd = socket(AF_INET, SOCK_DGRAM, 0);
+
+	if (if_fd == -1)
+		return -1;
+
+	rte_strscpy(ifr.ifr_name, if_name, sizeof(ifr.ifr_name));
+	if (ioctl(if_fd, SIOCGIFHWADDR, &ifr)) {
+		close(if_fd);
+		return -1;
+	}
+
+	mac_addrs = rte_zmalloc_socket(NULL, ETHER_ADDR_LEN, 0, numa_node);
+	if (!mac_addrs) {
+		close(if_fd);
+		return -1;
+	}
+
+	PMD_LOG(INFO, "Setting phy MAC for %s", if_name);
+	eth_dev->data->mac_addrs = mac_addrs;
+	rte_memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
+			ifr.ifr_hwaddr.sa_data, ETHER_ADDR_LEN);
+
+	close(if_fd);
+
+	return 0;
+
+#elif defined(RTE_EXEC_ENV_BSDAPP)
+	void *mac_addrs;
+	struct if_msghdr *ifm;
+	struct sockaddr_dl *sdl;
+	int mib[6];
+	size_t len = 0;
+	char *buf;
+
+	mib[0] = CTL_NET;
+	mib[1] = AF_ROUTE;
+	mib[2] = 0;
+	mib[3] = AF_LINK;
+	mib[4] = NET_RT_IFLIST;
+	mib[5] = if_nametoindex(if_name);
+
+	if (sysctl(mib, 6, NULL, &len, NULL, 0) < 0)
+		return -1;
+
+	if (len == 0)
+		return -1;
+
+	buf = rte_malloc(NULL, len, 0);
+	if (!buf)
+		return -1;
+
+	if (sysctl(mib, 6, buf, &len, NULL, 0) < 0) {
+		rte_free(buf);
+		return -1;
+	}
+	ifm = (struct if_msghdr *)buf;
+	sdl = (struct sockaddr_dl *)(ifm + 1);
+
+	mac_addrs = rte_zmalloc_socket(NULL, ETHER_ADDR_LEN, 0, numa_node);
+	if (!mac_addrs) {
+		rte_free(buf);
+		return -1;
+	}
+
+	PMD_LOG(INFO, "Setting phy MAC for %s", if_name);
+	eth_dev->data->mac_addrs = mac_addrs;
+	rte_memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
+			LLADDR(sdl), ETHER_ADDR_LEN);
+
+	rte_free(buf);
+
+	return 0;
+#else
+	return -1;
+#endif
+}
+
+static int
 eth_from_pcaps_common(struct rte_vdev_device *vdev,
 		struct pmd_devargs *rx_queues, const unsigned int nb_rx_queues,
 		struct pmd_devargs *tx_queues, const unsigned int nb_tx_queues,
-		struct rte_kvargs *kvlist, struct pmd_internals **internals,
-		struct rte_eth_dev **eth_dev)
+		struct pmd_internals **internals, struct rte_eth_dev **eth_dev)
 {
-	struct rte_kvargs_pair *pair = NULL;
-	unsigned int k_idx;
 	unsigned int i;
 
 	/* do some parameter checking */
@@ -944,17 +1058,6 @@ eth_from_pcaps_common(struct rte_vdev_device *vdev,
 		snprintf(tx->type, sizeof(tx->type), "%s", queue->type);
 	}
 
-	for (k_idx = 0; k_idx < kvlist->count; k_idx++) {
-		pair = &kvlist->pairs[k_idx];
-		if (strstr(pair->key, ETH_PCAP_IFACE_ARG) != NULL)
-			break;
-	}
-
-	if (pair == NULL)
-		(*internals)->if_index = 0;
-	else
-		(*internals)->if_index = if_nametoindex(pair->value);
-
 	return 0;
 }
 
@@ -962,15 +1065,14 @@ static int
 eth_from_pcaps(struct rte_vdev_device *vdev,
 		struct pmd_devargs *rx_queues, const unsigned int nb_rx_queues,
 		struct pmd_devargs *tx_queues, const unsigned int nb_tx_queues,
-		struct rte_kvargs *kvlist, int single_iface,
-		unsigned int using_dumpers)
+		int single_iface, unsigned int using_dumpers)
 {
 	struct pmd_internals *internals = NULL;
 	struct rte_eth_dev *eth_dev = NULL;
 	int ret;
 
 	ret = eth_from_pcaps_common(vdev, rx_queues, nb_rx_queues,
-		tx_queues, nb_tx_queues, kvlist, &internals, &eth_dev);
+		tx_queues, nb_tx_queues, &internals, &eth_dev);
 
 	if (ret < 0)
 		return ret;
@@ -978,6 +1080,18 @@ eth_from_pcaps(struct rte_vdev_device *vdev,
 	/* store weather we are using a single interface for rx/tx or not */
 	internals->single_iface = single_iface;
 
+	if (single_iface) {
+		internals->if_index = if_nametoindex(rx_queues->queue[0].name);
+
+		/* phy_mac arg is applied only only if "iface" devarg is provided */
+		if (rx_queues->phy_mac) {
+			int ret = eth_pcap_update_mac(rx_queues->queue[0].name,
+					eth_dev, vdev->device.numa_node);
+			if (ret == 0)
+				internals->phy_mac = 1;
+		}
+	}
+
 	eth_dev->rx_pkt_burst = eth_pcap_rx;
 
 	if (using_dumpers)
@@ -1008,8 +1122,7 @@ pmd_pcap_probe(struct rte_vdev_device *dev)
 	start_cycles = rte_get_timer_cycles();
 	hz = rte_get_timer_hz();
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
-	    strlen(rte_vdev_device_args(dev)) == 0) {
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 		eth_dev = rte_eth_dev_attach_secondary(name);
 		if (!eth_dev) {
 			PMD_LOG(ERR, "Failed to probe %s", name);
@@ -1034,12 +1147,18 @@ pmd_pcap_probe(struct rte_vdev_device *dev)
 
 		ret = rte_kvargs_process(kvlist, ETH_PCAP_IFACE_ARG,
 				&open_rx_tx_iface, &pcaps);
-
 		if (ret < 0)
 			goto free_kvlist;
 
 		dumpers.queue[0] = pcaps.queue[0];
 
+		ret = rte_kvargs_process(kvlist, ETH_PCAP_PHY_MAC_ARG,
+				&select_phy_mac, &pcaps.phy_mac);
+		if (ret < 0)
+			goto free_kvlist;
+
+		dumpers.phy_mac = pcaps.phy_mac;
+
 		single_iface = 1;
 		pcaps.num_of_queue = 1;
 		dumpers.num_of_queue = 1;
@@ -1084,7 +1203,7 @@ pmd_pcap_probe(struct rte_vdev_device *dev)
 
 create_eth:
 	ret = eth_from_pcaps(dev, &pcaps, pcaps.num_of_queue, &dumpers,
-		dumpers.num_of_queue, kvlist, single_iface, is_tx_pcap);
+		dumpers.num_of_queue, single_iface, is_tx_pcap);
 
 free_kvlist:
 	rte_kvargs_free(kvlist);
@@ -1095,6 +1214,7 @@ free_kvlist:
 static int
 pmd_pcap_remove(struct rte_vdev_device *dev)
 {
+	struct pmd_internals *internals = NULL;
 	struct rte_eth_dev *eth_dev = NULL;
 
 	PMD_LOG(INFO, "Closing pcap ethdev on numa socket %d",
@@ -1108,7 +1228,12 @@ pmd_pcap_remove(struct rte_vdev_device *dev)
 	if (eth_dev == NULL)
 		return -1;
 
-	rte_free(eth_dev->data->dev_private);
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		internals = eth_dev->data->dev_private;
+		if (internals != NULL && internals->phy_mac == 0)
+			/* not dynamically allocated, must not be freed */
+			eth_dev->data->mac_addrs = NULL;
+	}
 
 	rte_eth_dev_release_port(eth_dev);
 
@@ -1128,7 +1253,8 @@ RTE_PMD_REGISTER_PARAM_STRING(net_pcap,
 	ETH_PCAP_RX_IFACE_ARG "=<ifc> "
 	ETH_PCAP_RX_IFACE_IN_ARG "=<ifc> "
 	ETH_PCAP_TX_IFACE_ARG "=<ifc> "
-	ETH_PCAP_IFACE_ARG "=<ifc>");
+	ETH_PCAP_IFACE_ARG "=<ifc> "
+	ETH_PCAP_PHY_MAC_ARG "=<int>");
 
 RTE_INIT(eth_pcap_init_log)
 {
diff --git a/drivers/net/qede/Makefile b/drivers/net/qede/Makefile
index 488ca1d9..2ecbd8d2 100644
--- a/drivers/net/qede/Makefile
+++ b/drivers/net/qede/Makefile
@@ -105,6 +105,6 @@ SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_vf.c
 SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_main.c
 SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_rxtx.c
-SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_fdir.c
+SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_filter.c
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/qede/base/bcm_osal.c b/drivers/net/qede/base/bcm_osal.c
index d5d6f8e2..693328f1 100644
--- a/drivers/net/qede/base/bcm_osal.c
+++ b/drivers/net/qede/base/bcm_osal.c
@@ -17,7 +17,7 @@
 /* Array of memzone pointers */
 static const struct rte_memzone *ecore_mz_mapping[RTE_MAX_MEMZONE];
 /* Counter to track current memzone allocated */
-uint16_t ecore_mz_count;
+static uint16_t ecore_mz_count;
 
 unsigned long qede_log2_align(unsigned long n)
 {
diff --git a/drivers/net/qede/base/bcm_osal.h b/drivers/net/qede/base/bcm_osal.h
index 630867fa..1abf44fa 100644
--- a/drivers/net/qede/base/bcm_osal.h
+++ b/drivers/net/qede/base/bcm_osal.h
@@ -447,10 +447,13 @@ u32 qede_crc32(u32 crc, u8 *ptr, u32 length);
 #define OSAL_CRC8(table, pdata, nbytes, crc) 0
 #define OSAL_MFW_TLV_REQ(p_hwfn) nothing
 #define OSAL_MFW_FILL_TLV_DATA(type, buf, data) (0)
+#define OSAL_HW_INFO_CHANGE(p_hwfn, change) nothing
 #define OSAL_MFW_CMD_PREEMPT(p_hwfn) nothing
 #define OSAL_PF_VALIDATE_MODIFY_TUNN_CONFIG(p_hwfn, mask, b_update, tunn) 0
 
 #define OSAL_DIV_S64(a, b)	((a) / (b))
 #define OSAL_LLDP_RX_TLVS(p_hwfn, tlv_buf, tlv_size) nothing
+#define OSAL_DBG_ALLOC_USER_DATA(p_hwfn, user_data_ptr) (0)
+#define OSAL_DB_REC_OCCURRED(p_hwfn) nothing
 
 #endif /* __BCM_OSAL_H */
diff --git a/drivers/net/qede/base/common_hsi.h b/drivers/net/qede/base/common_hsi.h
index ca8e59db..2aaf298f 100644
--- a/drivers/net/qede/base/common_hsi.h
+++ b/drivers/net/qede/base/common_hsi.h
@@ -95,8 +95,8 @@
 
 
 #define FW_MAJOR_VERSION        8
-#define FW_MINOR_VERSION        33
-#define FW_REVISION_VERSION     12
+#define FW_MINOR_VERSION        37
+#define FW_REVISION_VERSION     7
 #define FW_ENGINEERING_VERSION  0
 
 /***********************/
@@ -1033,13 +1033,14 @@ struct db_rdma_dpm_params {
 #define DB_RDMA_DPM_PARAMS_WQE_SIZE_SHIFT           16
 #define DB_RDMA_DPM_PARAMS_RESERVED0_MASK           0x1
 #define DB_RDMA_DPM_PARAMS_RESERVED0_SHIFT          27
-/* RoCE completion flag */
-#define DB_RDMA_DPM_PARAMS_COMPLETION_FLG_MASK      0x1
-#define DB_RDMA_DPM_PARAMS_COMPLETION_FLG_SHIFT     28
+/* RoCE ack request (will be set 1) */
+#define DB_RDMA_DPM_PARAMS_ACK_REQUEST_MASK         0x1
+#define DB_RDMA_DPM_PARAMS_ACK_REQUEST_SHIFT        28
 #define DB_RDMA_DPM_PARAMS_S_FLG_MASK               0x1 /* RoCE S flag */
 #define DB_RDMA_DPM_PARAMS_S_FLG_SHIFT              29
-#define DB_RDMA_DPM_PARAMS_RESERVED1_MASK           0x1
-#define DB_RDMA_DPM_PARAMS_RESERVED1_SHIFT          30
+/* RoCE completion flag for FW use */
+#define DB_RDMA_DPM_PARAMS_COMPLETION_FLG_MASK      0x1
+#define DB_RDMA_DPM_PARAMS_COMPLETION_FLG_SHIFT     30
 /* Connection type is iWARP */
 #define DB_RDMA_DPM_PARAMS_CONN_TYPE_IS_IWARP_MASK  0x1
 #define DB_RDMA_DPM_PARAMS_CONN_TYPE_IS_IWARP_SHIFT 31
diff --git a/drivers/net/qede/base/ecore.h b/drivers/net/qede/base/ecore.h
index 5d79fdf0..524a1dd4 100644
--- a/drivers/net/qede/base/ecore.h
+++ b/drivers/net/qede/base/ecore.h
@@ -19,6 +19,7 @@
 #include <zlib.h>
 #endif
 
+#include "ecore_status.h"
 #include "ecore_hsi_common.h"
 #include "ecore_hsi_debug_tools.h"
 #include "ecore_hsi_init_func.h"
@@ -27,8 +28,8 @@
 #include "mcp_public.h"
 
 #define ECORE_MAJOR_VERSION		8
-#define ECORE_MINOR_VERSION		30
-#define ECORE_REVISION_VERSION		8
+#define ECORE_MINOR_VERSION		37
+#define ECORE_REVISION_VERSION		20
 #define ECORE_ENGINEERING_VERSION	0
 
 #define ECORE_VERSION							\
@@ -207,6 +208,7 @@ struct ecore_l2_info;
 struct ecore_igu_info;
 struct ecore_mcp_info;
 struct ecore_dcbx_info;
+struct ecore_llh_info;
 
 struct ecore_rt_data {
 	u32	*init_val;
@@ -543,6 +545,9 @@ enum ecore_mf_mode_bit {
 
 	/* Use stag for steering */
 	ECORE_MF_8021AD_TAGGING,
+
+	/* Allow FIP discovery fallback */
+	ECORE_MF_FIP_SPECIAL,
 };
 
 enum ecore_ufp_mode {
@@ -660,6 +665,7 @@ struct ecore_hwfn {
 #endif
 
 	struct dbg_tools_data		dbg_info;
+	void				*dbg_user_info;
 
 	struct z_stream_s		*stream;
 
@@ -739,6 +745,7 @@ struct ecore_dev {
 #endif
 #define ECORE_IS_AH(dev)	((dev)->type == ECORE_DEV_TYPE_AH)
 #define ECORE_IS_K2(dev)	ECORE_IS_AH(dev)
+#define ECORE_IS_E4(dev)	(ECORE_IS_BB(dev) || ECORE_IS_AH(dev))
 
 	u16 vendor_id;
 	u16 device_id;
@@ -833,8 +840,26 @@ struct ecore_dev {
 	/* HW functions */
 	u8				num_hwfns;
 	struct ecore_hwfn		hwfns[MAX_HWFNS_PER_DEVICE];
+#define ECORE_LEADING_HWFN(dev)		(&dev->hwfns[0])
 #define ECORE_IS_CMT(dev)		((dev)->num_hwfns > 1)
 
+	/* Engine affinity */
+	u8				l2_affin_hint;
+	u8				fir_affin;
+	u8				iwarp_affin;
+	/* Macro for getting the engine-affinitized hwfn for FCoE/iSCSI/RoCE */
+#define ECORE_FIR_AFFIN_HWFN(dev)	(&dev->hwfns[dev->fir_affin])
+	/* Macro for getting the engine-affinitized hwfn for iWARP */
+#define ECORE_IWARP_AFFIN_HWFN(dev)	(&dev->hwfns[dev->iwarp_affin])
+	/* Generic macro for getting the engine-affinitized hwfn */
+#define ECORE_AFFIN_HWFN(dev) \
+	(ECORE_IS_IWARP_PERSONALITY(ECORE_LEADING_HWFN(dev)) ? \
+	 ECORE_IWARP_AFFIN_HWFN(dev) : \
+	 ECORE_FIR_AFFIN_HWFN(dev))
+	/* Macro for getting the index (0/1) of the engine-affinitized hwfn */
+#define ECORE_AFFIN_HWFN_IDX(dev) \
+	(IS_LEAD_HWFN(ECORE_AFFIN_HWFN(dev)) ? 0 : 1)
+
 	/* SRIOV */
 	struct ecore_hw_sriov_info	*p_iov_info;
 #define IS_ECORE_SRIOV(p_dev)		(!!(p_dev)->p_iov_info)
@@ -869,6 +894,12 @@ struct ecore_dev {
 #ifndef ASIC_ONLY
 	bool				b_is_emul_full;
 #endif
+	/* LLH info */
+	u8				ppfid_bitmap;
+	struct ecore_llh_info		*p_llh_info;
+
+	/* Indicates whether this PF serves a storage target */
+	bool				b_is_target;
 
 #ifdef CONFIG_ECORE_BINARY_FW /* @DPDK */
 	void				*firmware;
@@ -958,6 +989,8 @@ void ecore_db_recovery_dp(struct ecore_hwfn *p_hwfn);
 void ecore_db_recovery_execute(struct ecore_hwfn *p_hwfn,
 			       enum ecore_db_rec_exec);
 
+bool ecore_edpm_enabled(struct ecore_hwfn *p_hwfn);
+
 /* amount of resources used in qm init */
 u8 ecore_init_qm_get_num_tcs(struct ecore_hwfn *p_hwfn);
 u16 ecore_init_qm_get_num_vfs(struct ecore_hwfn *p_hwfn);
@@ -965,6 +998,29 @@ u16 ecore_init_qm_get_num_pf_rls(struct ecore_hwfn *p_hwfn);
 u16 ecore_init_qm_get_num_vports(struct ecore_hwfn *p_hwfn);
 u16 ecore_init_qm_get_num_pqs(struct ecore_hwfn *p_hwfn);
 
-#define ECORE_LEADING_HWFN(dev)	(&dev->hwfns[0])
+#define MFW_PORT(_p_hwfn)	((_p_hwfn)->abs_pf_id % \
+				 ecore_device_num_ports((_p_hwfn)->p_dev))
+
+/* The PFID<->PPFID calculation is based on the relative index of a PF on its
+ * port. In BB there is a bug in the LLH in which the PPFID is actually engine
+ * based, and thus it equals the PFID.
+ */
+#define ECORE_PFID_BY_PPFID(_p_hwfn, abs_ppfid) \
+	(ECORE_IS_BB((_p_hwfn)->p_dev) ? \
+	 (abs_ppfid) : \
+	 (abs_ppfid) * (_p_hwfn)->p_dev->num_ports_in_engine + \
+	 MFW_PORT(_p_hwfn))
+#define ECORE_PPFID_BY_PFID(_p_hwfn) \
+	(ECORE_IS_BB((_p_hwfn)->p_dev) ? \
+	 (_p_hwfn)->rel_pf_id : \
+	 (_p_hwfn)->rel_pf_id / (_p_hwfn)->p_dev->num_ports_in_engine)
+
+enum _ecore_status_t ecore_all_ppfids_wr(struct ecore_hwfn *p_hwfn,
+					 struct ecore_ptt *p_ptt, u32 addr,
+					 u32 val);
+
+/* Utility functions for dumping the content of the NIG LLH filters */
+enum _ecore_status_t ecore_llh_dump_ppfid(struct ecore_dev *p_dev, u8 ppfid);
+enum _ecore_status_t ecore_llh_dump_all(struct ecore_dev *p_dev);
 
 #endif /* __ECORE_H */
diff --git a/drivers/net/qede/base/ecore_cxt.c b/drivers/net/qede/base/ecore_cxt.c
index bf36ce58..5c3370e1 100644
--- a/drivers/net/qede/base/ecore_cxt.c
+++ b/drivers/net/qede/base/ecore_cxt.c
@@ -1133,6 +1133,9 @@ enum _ecore_status_t ecore_cxt_mngr_alloc(struct ecore_hwfn *p_hwfn)
 		return ECORE_NOMEM;
 	}
 
+	/* Set the cxt mangr pointer prior to further allocations */
+	p_hwfn->p_cxt_mngr = p_mngr;
+
 	/* Initialize ILT client registers */
 	clients = p_mngr->clients;
 	clients[ILT_CLI_CDUC].first.reg = ILT_CFG_REG(CDUC, FIRST_ILT);
@@ -1174,13 +1177,13 @@ enum _ecore_status_t ecore_cxt_mngr_alloc(struct ecore_hwfn *p_hwfn)
 
 	/* Initialize the dynamic ILT allocation mutex */
 #ifdef CONFIG_ECORE_LOCK_ALLOC
-	OSAL_MUTEX_ALLOC(p_hwfn, &p_mngr->mutex);
+	if (OSAL_MUTEX_ALLOC(p_hwfn, &p_mngr->mutex)) {
+		DP_NOTICE(p_hwfn, false, "Failed to alloc p_mngr->mutex\n");
+		return ECORE_NOMEM;
+	}
 #endif
 	OSAL_MUTEX_INIT(&p_mngr->mutex);
 
-	/* Set the cxt mangr pointer priori to further allocations */
-	p_hwfn->p_cxt_mngr = p_mngr;
-
 	return ECORE_SUCCESS;
 }
 
@@ -2111,7 +2114,7 @@ ecore_cxt_dynamic_ilt_alloc(struct ecore_hwfn *p_hwfn,
 
 	ecore_dmae_host2grc(p_hwfn, p_ptt, (u64)(osal_uintptr_t)&ilt_hw_entry,
 			    reg_offset, sizeof(ilt_hw_entry) / sizeof(u32),
-			    0 /* no flags */);
+			    OSAL_NULL /* default parameters */);
 
 	if (elem_type == ECORE_ELEM_CXT) {
 		u32 last_cid_allocated = (1 + (iid / elems_per_p)) *
@@ -2218,7 +2221,7 @@ ecore_cxt_free_ilt_range(struct ecore_hwfn *p_hwfn,
 				    (u64)(osal_uintptr_t)&ilt_hw_entry,
 				    reg_offset,
 				    sizeof(ilt_hw_entry) / sizeof(u32),
-				    0 /* no flags */);
+				    OSAL_NULL /* default parameters */);
 	}
 
 	ecore_ptt_release(p_hwfn, p_ptt);
diff --git a/drivers/net/qede/base/ecore_dcbx.c b/drivers/net/qede/base/ecore_dcbx.c
index 96678745..cbc69cde 100644
--- a/drivers/net/qede/base/ecore_dcbx.c
+++ b/drivers/net/qede/base/ecore_dcbx.c
@@ -129,7 +129,7 @@ u8 ecore_dcbx_get_dscp_value(struct ecore_hwfn *p_hwfn, u8 pri)
 
 static void
 ecore_dcbx_set_params(struct ecore_dcbx_results *p_data,
-		      struct ecore_hwfn *p_hwfn,
+		      struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 		      bool enable, u8 prio, u8 tc,
 		      enum dcbx_protocol_type type,
 		      enum ecore_pci_personality personality)
@@ -154,12 +154,19 @@ ecore_dcbx_set_params(struct ecore_dcbx_results *p_data,
 	/* QM reconf data */
 	if (p_hwfn->hw_info.personality == personality)
 		p_hwfn->hw_info.offload_tc = tc;
+
+	/* Configure dcbx vlan priority in doorbell block for roce EDPM */
+	if (OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, &p_hwfn->p_dev->mf_bits) &&
+	    (type == DCBX_PROTOCOL_ROCE)) {
+		ecore_wr(p_hwfn, p_ptt, DORQ_REG_TAG1_OVRD_MODE, 1);
+		ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_PCP_BB_K2, prio << 1);
+	}
 }
 
 /* Update app protocol data and hw_info fields with the TLV info */
 static void
 ecore_dcbx_update_app_info(struct ecore_dcbx_results *p_data,
-			   struct ecore_hwfn *p_hwfn,
+			   struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 			   bool enable, u8 prio, u8 tc,
 			   enum dcbx_protocol_type type)
 {
@@ -175,7 +182,7 @@ ecore_dcbx_update_app_info(struct ecore_dcbx_results *p_data,
 
 		personality = ecore_dcbx_app_update[i].personality;
 
-		ecore_dcbx_set_params(p_data, p_hwfn, enable,
+		ecore_dcbx_set_params(p_data, p_hwfn, p_ptt, enable,
 				      prio, tc, type, personality);
 	}
 }
@@ -231,7 +238,7 @@ ecore_dcbx_get_app_protocol_type(struct ecore_hwfn *p_hwfn,
  * reconfiguring QM. Get protocol specific data for PF update ramrod command.
  */
 static enum _ecore_status_t
-ecore_dcbx_process_tlv(struct ecore_hwfn *p_hwfn,
+ecore_dcbx_process_tlv(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 		       struct ecore_dcbx_results *p_data,
 		       struct dcbx_app_priority_entry *p_tbl, u32 pri_tc_tbl,
 		       int count, u8 dcbx_version)
@@ -280,8 +287,8 @@ ecore_dcbx_process_tlv(struct ecore_hwfn *p_hwfn,
 				enable = true;
 			}
 
-			ecore_dcbx_update_app_info(p_data, p_hwfn, enable,
-						   priority, tc, type);
+			ecore_dcbx_update_app_info(p_data, p_hwfn, p_ptt,
+						   enable, priority, tc, type);
 		}
 	}
 
@@ -302,8 +309,8 @@ ecore_dcbx_process_tlv(struct ecore_hwfn *p_hwfn,
 		if (p_data->arr[type].update)
 			continue;
 
-		enable = (type == DCBX_PROTOCOL_ETH) ? false : !!dcbx_version;
-		ecore_dcbx_update_app_info(p_data, p_hwfn, enable,
+		/* if no app tlv was present, don't override in FW */
+		ecore_dcbx_update_app_info(p_data, p_hwfn, p_ptt, false,
 					   priority, tc, type);
 	}
 
@@ -314,11 +321,11 @@ ecore_dcbx_process_tlv(struct ecore_hwfn *p_hwfn,
  * reconfiguring QM. Get protocol specific data for PF update ramrod command.
  */
 static enum _ecore_status_t
-ecore_dcbx_process_mib_info(struct ecore_hwfn *p_hwfn)
+ecore_dcbx_process_mib_info(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
 {
 	struct dcbx_app_priority_feature *p_app;
 	struct dcbx_app_priority_entry *p_tbl;
-	struct ecore_dcbx_results data = { 0 };
+	struct ecore_dcbx_results data;
 	struct dcbx_ets_feature *p_ets;
 	struct ecore_hw_info *p_info;
 	u32 pri_tc_tbl, flags;
@@ -338,7 +345,8 @@ ecore_dcbx_process_mib_info(struct ecore_hwfn *p_hwfn)
 	p_info = &p_hwfn->hw_info;
 	num_entries = GET_MFW_FIELD(p_app->flags, DCBX_APP_NUM_ENTRIES);
 
-	rc = ecore_dcbx_process_tlv(p_hwfn, &data, p_tbl, pri_tc_tbl,
+	OSAL_MEMSET(&data, 0, sizeof(struct ecore_dcbx_results));
+	rc = ecore_dcbx_process_tlv(p_hwfn, p_ptt, &data, p_tbl, pri_tc_tbl,
 				    num_entries, dcbx_version);
 	if (rc != ECORE_SUCCESS)
 		return rc;
@@ -879,7 +887,7 @@ ecore_dcbx_mib_update_event(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 	if (type == ECORE_DCBX_OPERATIONAL_MIB) {
 		ecore_dcbx_get_dscp_params(p_hwfn, &p_hwfn->p_dcbx_info->get);
 
-		rc = ecore_dcbx_process_mib_info(p_hwfn);
+		rc = ecore_dcbx_process_mib_info(p_hwfn, p_ptt);
 		if (!rc) {
 			/* reconfigure tcs of QM queues according
 			 * to negotiation results
@@ -893,12 +901,19 @@ ecore_dcbx_mib_update_event(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 
 	ecore_dcbx_get_params(p_hwfn, &p_hwfn->p_dcbx_info->get, type);
 
-	/* Update the DSCP to TC mapping bit if required */
+	/* Update the DSCP to TC mapping enable bit if required */
 	if ((type == ECORE_DCBX_OPERATIONAL_MIB) &&
 	    p_hwfn->p_dcbx_info->dscp_nig_update) {
 		u8 val = !!p_hwfn->p_dcbx_info->get.dscp.enabled;
+		u32 addr = NIG_REG_DSCP_TO_TC_MAP_ENABLE;
+
+		rc = ecore_all_ppfids_wr(p_hwfn, p_ptt, addr, val);
+		if (rc != ECORE_SUCCESS) {
+			DP_NOTICE(p_hwfn, false,
+				  "Failed to update the DSCP to TC mapping enable bit\n");
+			return rc;
+		}
 
-		ecore_wr(p_hwfn, p_ptt, NIG_REG_DSCP_TO_TC_MAP_ENABLE, val);
 		p_hwfn->p_dcbx_info->dscp_nig_update = false;
 	}
 
@@ -1533,3 +1548,59 @@ ecore_lldp_set_system_tlvs(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 
 	return rc;
 }
+
+enum _ecore_status_t
+ecore_dcbx_get_dscp_priority(struct ecore_hwfn *p_hwfn,
+			     u8 dscp_index, u8 *p_dscp_pri)
+{
+	struct ecore_dcbx_get *p_dcbx_info;
+	enum _ecore_status_t rc;
+
+	if (dscp_index >= ECORE_DCBX_DSCP_SIZE) {
+		DP_ERR(p_hwfn, "Invalid dscp index %d\n", dscp_index);
+		return ECORE_INVAL;
+	}
+
+	p_dcbx_info = OSAL_ALLOC(p_hwfn->p_dev, GFP_KERNEL,
+				 sizeof(*p_dcbx_info));
+	if (!p_dcbx_info)
+		return ECORE_NOMEM;
+
+	OSAL_MEMSET(p_dcbx_info, 0, sizeof(*p_dcbx_info));
+	rc = ecore_dcbx_query_params(p_hwfn, p_dcbx_info,
+				     ECORE_DCBX_OPERATIONAL_MIB);
+	if (rc) {
+		OSAL_FREE(p_hwfn->p_dev, p_dcbx_info);
+		return rc;
+	}
+
+	*p_dscp_pri = p_dcbx_info->dscp.dscp_pri_map[dscp_index];
+	OSAL_FREE(p_hwfn->p_dev, p_dcbx_info);
+
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t
+ecore_dcbx_set_dscp_priority(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			     u8 dscp_index, u8 pri_val)
+{
+	struct ecore_dcbx_set dcbx_set;
+	enum _ecore_status_t rc;
+
+	if (dscp_index >= ECORE_DCBX_DSCP_SIZE ||
+	    pri_val >= ECORE_MAX_PFC_PRIORITIES) {
+		DP_ERR(p_hwfn, "Invalid dscp params: index = %d pri = %d\n",
+		       dscp_index, pri_val);
+		return ECORE_INVAL;
+	}
+
+	OSAL_MEMSET(&dcbx_set, 0, sizeof(dcbx_set));
+	rc = ecore_dcbx_get_config_params(p_hwfn, &dcbx_set);
+	if (rc)
+		return rc;
+
+	dcbx_set.override_flags = ECORE_DCBX_OVERRIDE_DSCP_CFG;
+	dcbx_set.dscp.dscp_pri_map[dscp_index] = pri_val;
+
+	return ecore_dcbx_config_params(p_hwfn, p_ptt, &dcbx_set, 1);
+}
diff --git a/drivers/net/qede/base/ecore_dcbx_api.h b/drivers/net/qede/base/ecore_dcbx_api.h
index eaf8e082..6fad2ecc 100644
--- a/drivers/net/qede/base/ecore_dcbx_api.h
+++ b/drivers/net/qede/base/ecore_dcbx_api.h
@@ -228,6 +228,16 @@ enum _ecore_status_t
 ecore_lldp_set_system_tlvs(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 			   struct ecore_lldp_sys_tlvs *p_params);
 
+/* Returns priority value for a given dscp index */
+enum _ecore_status_t
+ecore_dcbx_get_dscp_priority(struct ecore_hwfn *p_hwfn,
+			     u8 dscp_index, u8 *p_dscp_pri);
+
+/* Sets priority value for a given dscp index */
+enum _ecore_status_t
+ecore_dcbx_set_dscp_priority(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			     u8 dscp_index, u8 pri_val);
+
 static const struct ecore_dcbx_app_metadata ecore_dcbx_app_update[] = {
 	{DCBX_PROTOCOL_ISCSI, "ISCSI", ECORE_PCI_ISCSI},
 	{DCBX_PROTOCOL_FCOE, "FCOE", ECORE_PCI_FCOE},
diff --git a/drivers/net/qede/base/ecore_dev.c b/drivers/net/qede/base/ecore_dev.c
index 31f1f3ee..cf454b19 100644
--- a/drivers/net/qede/base/ecore_dev.c
+++ b/drivers/net/qede/base/ecore_dev.c
@@ -352,6 +352,1189 @@ void ecore_db_recovery_execute(struct ecore_hwfn *p_hwfn,
 }
 /******************** Doorbell Recovery end ****************/
 
+/********************************** NIG LLH ***********************************/
+
+enum ecore_llh_filter_type {
+	ECORE_LLH_FILTER_TYPE_MAC,
+	ECORE_LLH_FILTER_TYPE_PROTOCOL,
+};
+
+struct ecore_llh_mac_filter {
+	u8 addr[ETH_ALEN];
+};
+
+struct ecore_llh_protocol_filter {
+	enum ecore_llh_prot_filter_type_t type;
+	u16 source_port_or_eth_type;
+	u16 dest_port;
+};
+
+union ecore_llh_filter {
+	struct ecore_llh_mac_filter mac;
+	struct ecore_llh_protocol_filter protocol;
+};
+
+struct ecore_llh_filter_info {
+	bool b_enabled;
+	u32 ref_cnt;
+	enum ecore_llh_filter_type type;
+	union ecore_llh_filter filter;
+};
+
+struct ecore_llh_info {
+	/* Number of LLH filters banks */
+	u8 num_ppfid;
+
+#define MAX_NUM_PPFID	8
+	u8 ppfid_array[MAX_NUM_PPFID];
+
+	/* Array of filters arrays:
+	 * "num_ppfid" elements of filters banks, where each is an array of
+	 * "NIG_REG_LLH_FUNC_FILTER_EN_SIZE" filters.
+	 */
+	struct ecore_llh_filter_info **pp_filters;
+};
+
+static void ecore_llh_free(struct ecore_dev *p_dev)
+{
+	struct ecore_llh_info *p_llh_info = p_dev->p_llh_info;
+	u32 i;
+
+	if (p_llh_info != OSAL_NULL) {
+		if (p_llh_info->pp_filters != OSAL_NULL) {
+			for (i = 0; i < p_llh_info->num_ppfid; i++)
+				OSAL_FREE(p_dev, p_llh_info->pp_filters[i]);
+		}
+
+		OSAL_FREE(p_dev, p_llh_info->pp_filters);
+	}
+
+	OSAL_FREE(p_dev, p_llh_info);
+	p_dev->p_llh_info = OSAL_NULL;
+}
+
+static enum _ecore_status_t ecore_llh_alloc(struct ecore_dev *p_dev)
+{
+	struct ecore_llh_info *p_llh_info;
+	u32 size;
+	u8 i;
+
+	p_llh_info = OSAL_ZALLOC(p_dev, GFP_KERNEL, sizeof(*p_llh_info));
+	if (!p_llh_info)
+		return ECORE_NOMEM;
+	p_dev->p_llh_info = p_llh_info;
+
+	for (i = 0; i < MAX_NUM_PPFID; i++) {
+		if (!(p_dev->ppfid_bitmap & (0x1 << i)))
+			continue;
+
+		p_llh_info->ppfid_array[p_llh_info->num_ppfid] = i;
+		DP_VERBOSE(p_dev, ECORE_MSG_SP, "ppfid_array[%d] = %hhd\n",
+			   p_llh_info->num_ppfid, i);
+		p_llh_info->num_ppfid++;
+	}
+
+	size = p_llh_info->num_ppfid * sizeof(*p_llh_info->pp_filters);
+	p_llh_info->pp_filters = OSAL_ZALLOC(p_dev, GFP_KERNEL, size);
+	if (!p_llh_info->pp_filters)
+		return ECORE_NOMEM;
+
+	size = NIG_REG_LLH_FUNC_FILTER_EN_SIZE *
+	       sizeof(**p_llh_info->pp_filters);
+	for (i = 0; i < p_llh_info->num_ppfid; i++) {
+		p_llh_info->pp_filters[i] = OSAL_ZALLOC(p_dev, GFP_KERNEL,
+							size);
+		if (!p_llh_info->pp_filters[i])
+			return ECORE_NOMEM;
+	}
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t ecore_llh_shadow_sanity(struct ecore_dev *p_dev,
+						    u8 ppfid, u8 filter_idx,
+						    const char *action)
+{
+	struct ecore_llh_info *p_llh_info = p_dev->p_llh_info;
+
+	if (ppfid >= p_llh_info->num_ppfid) {
+		DP_NOTICE(p_dev, false,
+			  "LLH shadow [%s]: using ppfid %d while only %d ppfids are available\n",
+			  action, ppfid, p_llh_info->num_ppfid);
+		return ECORE_INVAL;
+	}
+
+	if (filter_idx >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) {
+		DP_NOTICE(p_dev, false,
+			  "LLH shadow [%s]: using filter_idx %d while only %d filters are available\n",
+			  action, filter_idx, NIG_REG_LLH_FUNC_FILTER_EN_SIZE);
+		return ECORE_INVAL;
+	}
+
+	return ECORE_SUCCESS;
+}
+
+#define ECORE_LLH_INVALID_FILTER_IDX	0xff
+
+static enum _ecore_status_t
+ecore_llh_shadow_search_filter(struct ecore_dev *p_dev, u8 ppfid,
+			       union ecore_llh_filter *p_filter,
+			       u8 *p_filter_idx)
+{
+	struct ecore_llh_info *p_llh_info = p_dev->p_llh_info;
+	struct ecore_llh_filter_info *p_filters;
+	enum _ecore_status_t rc;
+	u8 i;
+
+	rc = ecore_llh_shadow_sanity(p_dev, ppfid, 0, "search");
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	*p_filter_idx = ECORE_LLH_INVALID_FILTER_IDX;
+
+	p_filters = p_llh_info->pp_filters[ppfid];
+	for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) {
+		if (!OSAL_MEMCMP(p_filter, &p_filters[i].filter,
+				 sizeof(*p_filter))) {
+			*p_filter_idx = i;
+			break;
+		}
+	}
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+ecore_llh_shadow_get_free_idx(struct ecore_dev *p_dev, u8 ppfid,
+			      u8 *p_filter_idx)
+{
+	struct ecore_llh_info *p_llh_info = p_dev->p_llh_info;
+	struct ecore_llh_filter_info *p_filters;
+	enum _ecore_status_t rc;
+	u8 i;
+
+	rc = ecore_llh_shadow_sanity(p_dev, ppfid, 0, "get_free_idx");
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	*p_filter_idx = ECORE_LLH_INVALID_FILTER_IDX;
+
+	p_filters = p_llh_info->pp_filters[ppfid];
+	for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) {
+		if (!p_filters[i].b_enabled) {
+			*p_filter_idx = i;
+			break;
+		}
+	}
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+__ecore_llh_shadow_add_filter(struct ecore_dev *p_dev, u8 ppfid, u8 filter_idx,
+			      enum ecore_llh_filter_type type,
+			      union ecore_llh_filter *p_filter, u32 *p_ref_cnt)
+{
+	struct ecore_llh_info *p_llh_info = p_dev->p_llh_info;
+	struct ecore_llh_filter_info *p_filters;
+	enum _ecore_status_t rc;
+
+	rc = ecore_llh_shadow_sanity(p_dev, ppfid, filter_idx, "add");
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_filters = p_llh_info->pp_filters[ppfid];
+	if (!p_filters[filter_idx].ref_cnt) {
+		p_filters[filter_idx].b_enabled = true;
+		p_filters[filter_idx].type = type;
+		OSAL_MEMCPY(&p_filters[filter_idx].filter, p_filter,
+			    sizeof(p_filters[filter_idx].filter));
+	}
+
+	*p_ref_cnt = ++p_filters[filter_idx].ref_cnt;
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+ecore_llh_shadow_add_filter(struct ecore_dev *p_dev, u8 ppfid,
+			    enum ecore_llh_filter_type type,
+			    union ecore_llh_filter *p_filter,
+			    u8 *p_filter_idx, u32 *p_ref_cnt)
+{
+	enum _ecore_status_t rc;
+
+	/* Check if the same filter already exist */
+	rc = ecore_llh_shadow_search_filter(p_dev, ppfid, p_filter,
+					    p_filter_idx);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	/* Find a new entry in case of a new filter */
+	if (*p_filter_idx == ECORE_LLH_INVALID_FILTER_IDX) {
+		rc = ecore_llh_shadow_get_free_idx(p_dev, ppfid, p_filter_idx);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+	}
+
+	/* No free entry was found */
+	if (*p_filter_idx == ECORE_LLH_INVALID_FILTER_IDX) {
+		DP_NOTICE(p_dev, false,
+			  "Failed to find an empty LLH filter to utilize [ppfid %d]\n",
+			  ppfid);
+		return ECORE_NORESOURCES;
+	}
+
+	return __ecore_llh_shadow_add_filter(p_dev, ppfid, *p_filter_idx, type,
+					     p_filter, p_ref_cnt);
+}
+
+static enum _ecore_status_t
+__ecore_llh_shadow_remove_filter(struct ecore_dev *p_dev, u8 ppfid,
+				 u8 filter_idx, u32 *p_ref_cnt)
+{
+	struct ecore_llh_info *p_llh_info = p_dev->p_llh_info;
+	struct ecore_llh_filter_info *p_filters;
+	enum _ecore_status_t rc;
+
+	rc = ecore_llh_shadow_sanity(p_dev, ppfid, filter_idx, "remove");
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_filters = p_llh_info->pp_filters[ppfid];
+	if (!p_filters[filter_idx].ref_cnt) {
+		DP_NOTICE(p_dev, false,
+			  "LLH shadow: trying to remove a filter with ref_cnt=0\n");
+		return ECORE_INVAL;
+	}
+
+	*p_ref_cnt = --p_filters[filter_idx].ref_cnt;
+	if (!p_filters[filter_idx].ref_cnt)
+		OSAL_MEM_ZERO(&p_filters[filter_idx],
+			      sizeof(p_filters[filter_idx]));
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+ecore_llh_shadow_remove_filter(struct ecore_dev *p_dev, u8 ppfid,
+			       union ecore_llh_filter *p_filter,
+			       u8 *p_filter_idx, u32 *p_ref_cnt)
+{
+	enum _ecore_status_t rc;
+
+	rc = ecore_llh_shadow_search_filter(p_dev, ppfid, p_filter,
+					    p_filter_idx);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	/* No matching filter was found */
+	if (*p_filter_idx == ECORE_LLH_INVALID_FILTER_IDX) {
+		DP_NOTICE(p_dev, false,
+			  "Failed to find a filter in the LLH shadow\n");
+		return ECORE_INVAL;
+	}
+
+	return __ecore_llh_shadow_remove_filter(p_dev, ppfid, *p_filter_idx,
+						p_ref_cnt);
+}
+
+static enum _ecore_status_t
+ecore_llh_shadow_remove_all_filters(struct ecore_dev *p_dev, u8 ppfid)
+{
+	struct ecore_llh_info *p_llh_info = p_dev->p_llh_info;
+	struct ecore_llh_filter_info *p_filters;
+	enum _ecore_status_t rc;
+
+	rc = ecore_llh_shadow_sanity(p_dev, ppfid, 0, "remove_all");
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	p_filters = p_llh_info->pp_filters[ppfid];
+	OSAL_MEM_ZERO(p_filters,
+		      NIG_REG_LLH_FUNC_FILTER_EN_SIZE * sizeof(*p_filters));
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t ecore_abs_ppfid(struct ecore_dev *p_dev,
+					    u8 rel_ppfid, u8 *p_abs_ppfid)
+{
+	struct ecore_llh_info *p_llh_info = p_dev->p_llh_info;
+	u8 ppfids = p_llh_info->num_ppfid - 1;
+
+	if (rel_ppfid >= p_llh_info->num_ppfid) {
+		DP_NOTICE(p_dev, false,
+			  "rel_ppfid %d is not valid, available indices are 0..%hhd\n",
+			  rel_ppfid, ppfids);
+		return ECORE_INVAL;
+	}
+
+	*p_abs_ppfid = p_llh_info->ppfid_array[rel_ppfid];
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+__ecore_llh_set_engine_affin(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
+{
+	struct ecore_dev *p_dev = p_hwfn->p_dev;
+	enum ecore_eng eng;
+	u8 ppfid;
+	enum _ecore_status_t rc;
+
+	rc = ecore_mcp_get_engine_config(p_hwfn, p_ptt);
+	if (rc != ECORE_SUCCESS && rc != ECORE_NOTIMPL) {
+		DP_NOTICE(p_hwfn, false,
+			  "Failed to get the engine affinity configuration\n");
+		return rc;
+	}
+
+	/* RoCE PF is bound to a single engine */
+	if (ECORE_IS_ROCE_PERSONALITY(p_hwfn)) {
+		eng = p_dev->fir_affin ? ECORE_ENG1 : ECORE_ENG0;
+		rc = ecore_llh_set_roce_affinity(p_dev, eng);
+		if (rc != ECORE_SUCCESS) {
+			DP_NOTICE(p_dev, false,
+				  "Failed to set the RoCE engine affinity\n");
+			return rc;
+		}
+
+		DP_VERBOSE(p_dev, ECORE_MSG_SP,
+			   "LLH: Set the engine affinity of RoCE packets as %d\n",
+			   eng);
+	}
+
+	/* Storage PF is bound to a single engine while L2 PF uses both */
+	if (ECORE_IS_FCOE_PERSONALITY(p_hwfn) ||
+	    ECORE_IS_ISCSI_PERSONALITY(p_hwfn))
+		eng = p_dev->fir_affin ? ECORE_ENG1 : ECORE_ENG0;
+	else /* L2_PERSONALITY */
+		eng = ECORE_BOTH_ENG;
+
+	for (ppfid = 0; ppfid < p_dev->p_llh_info->num_ppfid; ppfid++) {
+		rc = ecore_llh_set_ppfid_affinity(p_dev, ppfid, eng);
+		if (rc != ECORE_SUCCESS) {
+			DP_NOTICE(p_dev, false,
+				  "Failed to set the engine affinity of ppfid %d\n",
+				  ppfid);
+			return rc;
+		}
+	}
+
+	DP_VERBOSE(p_dev, ECORE_MSG_SP,
+		   "LLH: Set the engine affinity of non-RoCE packets as %d\n",
+		   eng);
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+ecore_llh_set_engine_affin(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			   bool avoid_eng_affin)
+{
+	struct ecore_dev *p_dev = p_hwfn->p_dev;
+	enum _ecore_status_t rc;
+
+	/* Backwards compatible mode:
+	 * - RoCE packets     - Use engine 0.
+	 * - Non-RoCE packets - Use connection based classification for L2 PFs,
+	 *                      and engine 0 otherwise.
+	 */
+	if (avoid_eng_affin) {
+		enum ecore_eng eng;
+		u8 ppfid;
+
+		if (ECORE_IS_ROCE_PERSONALITY(p_hwfn)) {
+			eng = ECORE_ENG0;
+			rc = ecore_llh_set_roce_affinity(p_dev, eng);
+			if (rc != ECORE_SUCCESS) {
+				DP_NOTICE(p_dev, false,
+					  "Failed to set the RoCE engine affinity\n");
+				return rc;
+			}
+
+			DP_VERBOSE(p_dev, ECORE_MSG_SP,
+				   "LLH [backwards compatible mode]: Set the engine affinity of RoCE packets as %d\n",
+				   eng);
+		}
+
+		eng = (ECORE_IS_FCOE_PERSONALITY(p_hwfn) ||
+		       ECORE_IS_ISCSI_PERSONALITY(p_hwfn)) ? ECORE_ENG0
+							   : ECORE_BOTH_ENG;
+		for (ppfid = 0; ppfid < p_dev->p_llh_info->num_ppfid; ppfid++) {
+			rc = ecore_llh_set_ppfid_affinity(p_dev, ppfid, eng);
+			if (rc != ECORE_SUCCESS) {
+				DP_NOTICE(p_dev, false,
+					  "Failed to set the engine affinity of ppfid %d\n",
+					  ppfid);
+				return rc;
+			}
+		}
+
+		DP_VERBOSE(p_dev, ECORE_MSG_SP,
+			   "LLH [backwards compatible mode]: Set the engine affinity of non-RoCE packets as %d\n",
+			   eng);
+
+		return ECORE_SUCCESS;
+	}
+
+	return __ecore_llh_set_engine_affin(p_hwfn, p_ptt);
+}
+
+static enum _ecore_status_t ecore_llh_hw_init_pf(struct ecore_hwfn *p_hwfn,
+						 struct ecore_ptt *p_ptt,
+						 bool avoid_eng_affin)
+{
+	struct ecore_dev *p_dev = p_hwfn->p_dev;
+	u8 ppfid, abs_ppfid;
+	enum _ecore_status_t rc;
+
+	for (ppfid = 0; ppfid < p_dev->p_llh_info->num_ppfid; ppfid++) {
+		u32 addr;
+
+		rc = ecore_abs_ppfid(p_dev, ppfid, &abs_ppfid);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		addr = NIG_REG_LLH_PPFID2PFID_TBL_0 + abs_ppfid * 0x4;
+		ecore_wr(p_hwfn, p_ptt, addr, p_hwfn->rel_pf_id);
+	}
+
+	if (OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS, &p_dev->mf_bits) &&
+	    !ECORE_IS_FCOE_PERSONALITY(p_hwfn)) {
+		rc = ecore_llh_add_mac_filter(p_dev, 0,
+					      p_hwfn->hw_info.hw_mac_addr);
+		if (rc != ECORE_SUCCESS)
+			DP_NOTICE(p_dev, false,
+				  "Failed to add an LLH filter with the primary MAC\n");
+	}
+
+	if (ECORE_IS_CMT(p_dev)) {
+		rc = ecore_llh_set_engine_affin(p_hwfn, p_ptt, avoid_eng_affin);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+	}
+
+	return ECORE_SUCCESS;
+}
+
+u8 ecore_llh_get_num_ppfid(struct ecore_dev *p_dev)
+{
+	return p_dev->p_llh_info->num_ppfid;
+}
+
+enum ecore_eng ecore_llh_get_l2_affinity_hint(struct ecore_dev *p_dev)
+{
+	return p_dev->l2_affin_hint ? ECORE_ENG1 : ECORE_ENG0;
+}
+
+/* TBD - should be removed when these definitions are available in reg_addr.h */
+#define NIG_REG_PPF_TO_ENGINE_SEL_ROCE_MASK		0x3
+#define NIG_REG_PPF_TO_ENGINE_SEL_ROCE_SHIFT		0
+#define NIG_REG_PPF_TO_ENGINE_SEL_NON_ROCE_MASK		0x3
+#define NIG_REG_PPF_TO_ENGINE_SEL_NON_ROCE_SHIFT	2
+
+enum _ecore_status_t ecore_llh_set_ppfid_affinity(struct ecore_dev *p_dev,
+						  u8 ppfid, enum ecore_eng eng)
+{
+	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
+	struct ecore_ptt *p_ptt = ecore_ptt_acquire(p_hwfn);
+	u32 addr, val, eng_sel;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+	u8 abs_ppfid;
+
+	if (p_ptt == OSAL_NULL)
+		return ECORE_AGAIN;
+
+	if (!ECORE_IS_CMT(p_dev))
+		goto out;
+
+	rc = ecore_abs_ppfid(p_dev, ppfid, &abs_ppfid);
+	if (rc != ECORE_SUCCESS)
+		goto out;
+
+	switch (eng) {
+	case ECORE_ENG0:
+		eng_sel = 0;
+		break;
+	case ECORE_ENG1:
+		eng_sel = 1;
+		break;
+	case ECORE_BOTH_ENG:
+		eng_sel = 2;
+		break;
+	default:
+		DP_NOTICE(p_dev, false,
+			  "Invalid affinity value for ppfid [%d]\n", eng);
+		rc = ECORE_INVAL;
+		goto out;
+	}
+
+	addr = NIG_REG_PPF_TO_ENGINE_SEL + abs_ppfid * 0x4;
+	val = ecore_rd(p_hwfn, p_ptt, addr);
+	SET_FIELD(val, NIG_REG_PPF_TO_ENGINE_SEL_NON_ROCE, eng_sel);
+	ecore_wr(p_hwfn, p_ptt, addr, val);
+
+	/* The iWARP affinity is set as the affinity of ppfid 0 */
+	if (!ppfid && ECORE_IS_IWARP_PERSONALITY(p_hwfn))
+		p_dev->iwarp_affin = (eng == ECORE_ENG1) ? 1 : 0;
+out:
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	return rc;
+}
+
+enum _ecore_status_t ecore_llh_set_roce_affinity(struct ecore_dev *p_dev,
+						 enum ecore_eng eng)
+{
+	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
+	struct ecore_ptt *p_ptt = ecore_ptt_acquire(p_hwfn);
+	u32 addr, val, eng_sel;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+	u8 ppfid, abs_ppfid;
+
+	if (p_ptt == OSAL_NULL)
+		return ECORE_AGAIN;
+
+	if (!ECORE_IS_CMT(p_dev))
+		goto out;
+
+	switch (eng) {
+	case ECORE_ENG0:
+		eng_sel = 0;
+		break;
+	case ECORE_ENG1:
+		eng_sel = 1;
+		break;
+	case ECORE_BOTH_ENG:
+		eng_sel = 2;
+		ecore_wr(p_hwfn, p_ptt, NIG_REG_LLH_ENG_CLS_ROCE_QP_SEL,
+			 0xf /* QP bit 15 */);
+		break;
+	default:
+		DP_NOTICE(p_dev, false,
+			  "Invalid affinity value for RoCE [%d]\n", eng);
+		rc = ECORE_INVAL;
+		goto out;
+	}
+
+	for (ppfid = 0; ppfid < p_dev->p_llh_info->num_ppfid; ppfid++) {
+		rc = ecore_abs_ppfid(p_dev, ppfid, &abs_ppfid);
+		if (rc != ECORE_SUCCESS)
+			goto out;
+
+		addr = NIG_REG_PPF_TO_ENGINE_SEL + abs_ppfid * 0x4;
+		val = ecore_rd(p_hwfn, p_ptt, addr);
+		SET_FIELD(val, NIG_REG_PPF_TO_ENGINE_SEL_ROCE, eng_sel);
+		ecore_wr(p_hwfn, p_ptt, addr, val);
+	}
+out:
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	return rc;
+}
+
+struct ecore_llh_filter_e4_details {
+	u64 value;
+	u32 mode;
+	u32 protocol_type;
+	u32 hdr_sel;
+	u32 enable;
+};
+
+static enum _ecore_status_t
+ecore_llh_access_filter_e4(struct ecore_hwfn *p_hwfn,
+			   struct ecore_ptt *p_ptt, u8 abs_ppfid, u8 filter_idx,
+			   struct ecore_llh_filter_e4_details *p_details,
+			   bool b_write_access)
+{
+	u8 pfid = ECORE_PFID_BY_PPFID(p_hwfn, abs_ppfid);
+	struct ecore_dmae_params params;
+	enum _ecore_status_t rc;
+	u32 addr;
+
+	/* The NIG/LLH registers that are accessed in this function have only 16
+	 * rows which are exposed to a PF. I.e. only the 16 filters of its
+	 * default ppfid
+	 * Accessing filters of other ppfids requires pretending to other PFs,
+	 * and thus the usage of the ecore_ppfid_rd/wr() functions.
+	 */
+
+	/* Filter enable - should be done first when removing a filter */
+	if (b_write_access && !p_details->enable) {
+		addr = NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + filter_idx * 0x4;
+		ecore_ppfid_wr(p_hwfn, p_ptt, abs_ppfid, addr,
+			       p_details->enable);
+	}
+
+	/* Filter value */
+	addr = NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + 2 * filter_idx * 0x4;
+	OSAL_MEMSET(&params, 0, sizeof(params));
+
+	if (b_write_access) {
+		params.flags = ECORE_DMAE_FLAG_PF_DST;
+		params.dst_pfid = pfid;
+		rc = ecore_dmae_host2grc(p_hwfn, p_ptt,
+					 (u64)(osal_uintptr_t)&p_details->value,
+					 addr, 2 /* size_in_dwords */, &params);
+	} else {
+		params.flags = ECORE_DMAE_FLAG_PF_SRC |
+			       ECORE_DMAE_FLAG_COMPLETION_DST;
+		params.src_pfid = pfid;
+		rc = ecore_dmae_grc2host(p_hwfn, p_ptt, addr,
+					 (u64)(osal_uintptr_t)&p_details->value,
+					 2 /* size_in_dwords */, &params);
+	}
+
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	/* Filter mode */
+	addr = NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 + filter_idx * 0x4;
+	if (b_write_access)
+		ecore_ppfid_wr(p_hwfn, p_ptt, abs_ppfid, addr, p_details->mode);
+	else
+		p_details->mode = ecore_ppfid_rd(p_hwfn, p_ptt, abs_ppfid,
+						 addr);
+
+	/* Filter protocol type */
+	addr = NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 + filter_idx * 0x4;
+	if (b_write_access)
+		ecore_ppfid_wr(p_hwfn, p_ptt, abs_ppfid, addr,
+			       p_details->protocol_type);
+	else
+		p_details->protocol_type = ecore_ppfid_rd(p_hwfn, p_ptt,
+							  abs_ppfid, addr);
+
+	/* Filter header select */
+	addr = NIG_REG_LLH_FUNC_FILTER_HDR_SEL_BB_K2 + filter_idx * 0x4;
+	if (b_write_access)
+		ecore_ppfid_wr(p_hwfn, p_ptt, abs_ppfid, addr,
+			       p_details->hdr_sel);
+	else
+		p_details->hdr_sel = ecore_ppfid_rd(p_hwfn, p_ptt, abs_ppfid,
+						    addr);
+
+	/* Filter enable - should be done last when adding a filter */
+	if (!b_write_access || p_details->enable) {
+		addr = NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + filter_idx * 0x4;
+		if (b_write_access)
+			ecore_ppfid_wr(p_hwfn, p_ptt, abs_ppfid, addr,
+				       p_details->enable);
+		else
+			p_details->enable = ecore_ppfid_rd(p_hwfn, p_ptt,
+							   abs_ppfid, addr);
+	}
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+ecore_llh_add_filter_e4(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			u8 abs_ppfid, u8 filter_idx, u8 filter_prot_type,
+			u32 high, u32 low)
+{
+	struct ecore_llh_filter_e4_details filter_details;
+
+	filter_details.enable = 1;
+	filter_details.value = ((u64)high << 32) | low;
+	filter_details.hdr_sel =
+		OSAL_TEST_BIT(ECORE_MF_OVLAN_CLSS, &p_hwfn->p_dev->mf_bits) ?
+		1 : /* inner/encapsulated header */
+		0;  /* outer/tunnel header */
+	filter_details.protocol_type = filter_prot_type;
+	filter_details.mode = filter_prot_type ?
+			      1 : /* protocol-based classification */
+			      0;  /* MAC-address based classification */
+
+	return ecore_llh_access_filter_e4(p_hwfn, p_ptt, abs_ppfid, filter_idx,
+					  &filter_details,
+					  true /* write access */);
+}
+
+static enum _ecore_status_t
+ecore_llh_remove_filter_e4(struct ecore_hwfn *p_hwfn,
+			   struct ecore_ptt *p_ptt, u8 abs_ppfid, u8 filter_idx)
+{
+	struct ecore_llh_filter_e4_details filter_details;
+
+	OSAL_MEMSET(&filter_details, 0, sizeof(filter_details));
+
+	return ecore_llh_access_filter_e4(p_hwfn, p_ptt, abs_ppfid, filter_idx,
+					  &filter_details,
+					  true /* write access */);
+}
+
+static enum _ecore_status_t
+ecore_llh_add_filter(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+		     u8 abs_ppfid, u8 filter_idx, u8 filter_prot_type, u32 high,
+		     u32 low)
+{
+	return ecore_llh_add_filter_e4(p_hwfn, p_ptt, abs_ppfid,
+				       filter_idx, filter_prot_type,
+				       high, low);
+}
+
+static enum _ecore_status_t
+ecore_llh_remove_filter(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			u8 abs_ppfid, u8 filter_idx)
+{
+	return ecore_llh_remove_filter_e4(p_hwfn, p_ptt, abs_ppfid,
+					  filter_idx);
+}
+
+enum _ecore_status_t ecore_llh_add_mac_filter(struct ecore_dev *p_dev, u8 ppfid,
+					      u8 mac_addr[ETH_ALEN])
+{
+	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
+	struct ecore_ptt *p_ptt = ecore_ptt_acquire(p_hwfn);
+	union ecore_llh_filter filter;
+	u8 filter_idx, abs_ppfid;
+	u32 high, low, ref_cnt;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	if (p_ptt == OSAL_NULL)
+		return ECORE_AGAIN;
+
+	if (!OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS, &p_dev->mf_bits))
+		goto out;
+
+	OSAL_MEM_ZERO(&filter, sizeof(filter));
+	OSAL_MEMCPY(filter.mac.addr, mac_addr, ETH_ALEN);
+	rc = ecore_llh_shadow_add_filter(p_dev, ppfid,
+					 ECORE_LLH_FILTER_TYPE_MAC,
+					 &filter, &filter_idx, &ref_cnt);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	rc = ecore_abs_ppfid(p_dev, ppfid, &abs_ppfid);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	/* Configure the LLH only in case of a new the filter */
+	if (ref_cnt == 1) {
+		high = mac_addr[1] | (mac_addr[0] << 8);
+		low = mac_addr[5] | (mac_addr[4] << 8) | (mac_addr[3] << 16) |
+		      (mac_addr[2] << 24);
+		rc = ecore_llh_add_filter(p_hwfn, p_ptt, abs_ppfid, filter_idx,
+					  0, high, low);
+		if (rc != ECORE_SUCCESS)
+			goto err;
+	}
+
+	DP_VERBOSE(p_dev, ECORE_MSG_SP,
+		   "LLH: Added MAC filter [%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx] to ppfid %hhd [abs %hhd] at idx %hhd [ref_cnt %d]\n",
+		   mac_addr[0], mac_addr[1], mac_addr[2], mac_addr[3],
+		   mac_addr[4], mac_addr[5], ppfid, abs_ppfid, filter_idx,
+		   ref_cnt);
+
+	goto out;
+
+err:
+	DP_NOTICE(p_dev, false,
+		  "LLH: Failed to add MAC filter [%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx] to ppfid %hhd\n",
+		  mac_addr[0], mac_addr[1], mac_addr[2], mac_addr[3],
+		  mac_addr[4], mac_addr[5], ppfid);
+out:
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	return rc;
+}
+
+static enum _ecore_status_t
+ecore_llh_protocol_filter_stringify(struct ecore_dev *p_dev,
+				    enum ecore_llh_prot_filter_type_t type,
+				    u16 source_port_or_eth_type, u16 dest_port,
+				    char *str, osal_size_t str_len)
+{
+	switch (type) {
+	case ECORE_LLH_FILTER_ETHERTYPE:
+		OSAL_SNPRINTF(str, str_len, "Ethertype 0x%04x",
+			      source_port_or_eth_type);
+		break;
+	case ECORE_LLH_FILTER_TCP_SRC_PORT:
+		OSAL_SNPRINTF(str, str_len, "TCP src port 0x%04x",
+			      source_port_or_eth_type);
+		break;
+	case ECORE_LLH_FILTER_UDP_SRC_PORT:
+		OSAL_SNPRINTF(str, str_len, "UDP src port 0x%04x",
+			      source_port_or_eth_type);
+		break;
+	case ECORE_LLH_FILTER_TCP_DEST_PORT:
+		OSAL_SNPRINTF(str, str_len, "TCP dst port 0x%04x", dest_port);
+		break;
+	case ECORE_LLH_FILTER_UDP_DEST_PORT:
+		OSAL_SNPRINTF(str, str_len, "UDP dst port 0x%04x", dest_port);
+		break;
+	case ECORE_LLH_FILTER_TCP_SRC_AND_DEST_PORT:
+		OSAL_SNPRINTF(str, str_len, "TCP src/dst ports 0x%04x/0x%04x",
+			      source_port_or_eth_type, dest_port);
+		break;
+	case ECORE_LLH_FILTER_UDP_SRC_AND_DEST_PORT:
+		OSAL_SNPRINTF(str, str_len, "UDP src/dst ports 0x%04x/0x%04x",
+			      source_port_or_eth_type, dest_port);
+		break;
+	default:
+		DP_NOTICE(p_dev, true,
+			  "Non valid LLH protocol filter type %d\n", type);
+		return ECORE_INVAL;
+	}
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+ecore_llh_protocol_filter_to_hilo(struct ecore_dev *p_dev,
+				  enum ecore_llh_prot_filter_type_t type,
+				  u16 source_port_or_eth_type, u16 dest_port,
+				  u32 *p_high, u32 *p_low)
+{
+	*p_high = 0;
+	*p_low = 0;
+
+	switch (type) {
+	case ECORE_LLH_FILTER_ETHERTYPE:
+		*p_high = source_port_or_eth_type;
+		break;
+	case ECORE_LLH_FILTER_TCP_SRC_PORT:
+	case ECORE_LLH_FILTER_UDP_SRC_PORT:
+		*p_low = source_port_or_eth_type << 16;
+		break;
+	case ECORE_LLH_FILTER_TCP_DEST_PORT:
+	case ECORE_LLH_FILTER_UDP_DEST_PORT:
+		*p_low = dest_port;
+		break;
+	case ECORE_LLH_FILTER_TCP_SRC_AND_DEST_PORT:
+	case ECORE_LLH_FILTER_UDP_SRC_AND_DEST_PORT:
+		*p_low = (source_port_or_eth_type << 16) | dest_port;
+		break;
+	default:
+		DP_NOTICE(p_dev, true,
+			  "Non valid LLH protocol filter type %d\n", type);
+		return ECORE_INVAL;
+	}
+
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t
+ecore_llh_add_protocol_filter(struct ecore_dev *p_dev, u8 ppfid,
+			      enum ecore_llh_prot_filter_type_t type,
+			      u16 source_port_or_eth_type, u16 dest_port)
+{
+	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
+	struct ecore_ptt *p_ptt = ecore_ptt_acquire(p_hwfn);
+	u8 filter_idx, abs_ppfid, type_bitmap;
+	char str[32];
+	union ecore_llh_filter filter;
+	u32 high, low, ref_cnt;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	if (p_ptt == OSAL_NULL)
+		return ECORE_AGAIN;
+
+	if (!OSAL_TEST_BIT(ECORE_MF_LLH_PROTO_CLSS, &p_dev->mf_bits))
+		goto out;
+
+	rc = ecore_llh_protocol_filter_stringify(p_dev, type,
+						 source_port_or_eth_type,
+						 dest_port, str, sizeof(str));
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	OSAL_MEM_ZERO(&filter, sizeof(filter));
+	filter.protocol.type = type;
+	filter.protocol.source_port_or_eth_type = source_port_or_eth_type;
+	filter.protocol.dest_port = dest_port;
+	rc = ecore_llh_shadow_add_filter(p_dev, ppfid,
+					 ECORE_LLH_FILTER_TYPE_PROTOCOL,
+					 &filter, &filter_idx, &ref_cnt);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	rc = ecore_abs_ppfid(p_dev, ppfid, &abs_ppfid);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	/* Configure the LLH only in case of a new the filter */
+	if (ref_cnt == 1) {
+		rc = ecore_llh_protocol_filter_to_hilo(p_dev, type,
+						       source_port_or_eth_type,
+						       dest_port, &high, &low);
+		if (rc != ECORE_SUCCESS)
+			goto err;
+
+		type_bitmap = 0x1 << type;
+		rc = ecore_llh_add_filter(p_hwfn, p_ptt, abs_ppfid, filter_idx,
+					  type_bitmap, high, low);
+		if (rc != ECORE_SUCCESS)
+			goto err;
+	}
+
+	DP_VERBOSE(p_dev, ECORE_MSG_SP,
+		   "LLH: Added protocol filter [%s] to ppfid %hhd [abs %hhd] at idx %hhd [ref_cnt %d]\n",
+		   str, ppfid, abs_ppfid, filter_idx, ref_cnt);
+
+	goto out;
+
+err:
+	DP_NOTICE(p_hwfn, false,
+		  "LLH: Failed to add protocol filter [%s] to ppfid %hhd\n",
+		  str, ppfid);
+out:
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	return rc;
+}
+
+void ecore_llh_remove_mac_filter(struct ecore_dev *p_dev, u8 ppfid,
+				 u8 mac_addr[ETH_ALEN])
+{
+	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
+	struct ecore_ptt *p_ptt = ecore_ptt_acquire(p_hwfn);
+	union ecore_llh_filter filter;
+	u8 filter_idx, abs_ppfid;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+	u32 ref_cnt;
+
+	if (p_ptt == OSAL_NULL)
+		return;
+
+	if (!OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS, &p_dev->mf_bits))
+		goto out;
+
+	OSAL_MEM_ZERO(&filter, sizeof(filter));
+	OSAL_MEMCPY(filter.mac.addr, mac_addr, ETH_ALEN);
+	rc = ecore_llh_shadow_remove_filter(p_dev, ppfid, &filter, &filter_idx,
+					    &ref_cnt);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	rc = ecore_abs_ppfid(p_dev, ppfid, &abs_ppfid);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	/* Remove from the LLH in case the filter is not in use */
+	if (!ref_cnt) {
+		rc = ecore_llh_remove_filter(p_hwfn, p_ptt, abs_ppfid,
+					     filter_idx);
+		if (rc != ECORE_SUCCESS)
+			goto err;
+	}
+
+	DP_VERBOSE(p_dev, ECORE_MSG_SP,
+		   "LLH: Removed MAC filter [%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx] from ppfid %hhd [abs %hhd] at idx %hhd [ref_cnt %d]\n",
+		   mac_addr[0], mac_addr[1], mac_addr[2], mac_addr[3],
+		   mac_addr[4], mac_addr[5], ppfid, abs_ppfid, filter_idx,
+		   ref_cnt);
+
+	goto out;
+
+err:
+	DP_NOTICE(p_dev, false,
+		  "LLH: Failed to remove MAC filter [%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx] from ppfid %hhd\n",
+		  mac_addr[0], mac_addr[1], mac_addr[2], mac_addr[3],
+		  mac_addr[4], mac_addr[5], ppfid);
+out:
+	ecore_ptt_release(p_hwfn, p_ptt);
+}
+
+void ecore_llh_remove_protocol_filter(struct ecore_dev *p_dev, u8 ppfid,
+				      enum ecore_llh_prot_filter_type_t type,
+				      u16 source_port_or_eth_type,
+				      u16 dest_port)
+{
+	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
+	struct ecore_ptt *p_ptt = ecore_ptt_acquire(p_hwfn);
+	u8 filter_idx, abs_ppfid;
+	char str[32];
+	union ecore_llh_filter filter;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+	u32 ref_cnt;
+
+	if (p_ptt == OSAL_NULL)
+		return;
+
+	if (!OSAL_TEST_BIT(ECORE_MF_LLH_PROTO_CLSS, &p_dev->mf_bits))
+		goto out;
+
+	rc = ecore_llh_protocol_filter_stringify(p_dev, type,
+						 source_port_or_eth_type,
+						 dest_port, str, sizeof(str));
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	OSAL_MEM_ZERO(&filter, sizeof(filter));
+	filter.protocol.type = type;
+	filter.protocol.source_port_or_eth_type = source_port_or_eth_type;
+	filter.protocol.dest_port = dest_port;
+	rc = ecore_llh_shadow_remove_filter(p_dev, ppfid, &filter, &filter_idx,
+					    &ref_cnt);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	rc = ecore_abs_ppfid(p_dev, ppfid, &abs_ppfid);
+	if (rc != ECORE_SUCCESS)
+		goto err;
+
+	/* Remove from the LLH in case the filter is not in use */
+	if (!ref_cnt) {
+		rc = ecore_llh_remove_filter(p_hwfn, p_ptt, abs_ppfid,
+					     filter_idx);
+		if (rc != ECORE_SUCCESS)
+			goto err;
+	}
+
+	DP_VERBOSE(p_dev, ECORE_MSG_SP,
+		   "LLH: Removed protocol filter [%s] from ppfid %hhd [abs %hhd] at idx %hhd [ref_cnt %d]\n",
+		   str, ppfid, abs_ppfid, filter_idx, ref_cnt);
+
+	goto out;
+
+err:
+	DP_NOTICE(p_dev, false,
+		  "LLH: Failed to remove protocol filter [%s] from ppfid %hhd\n",
+		  str, ppfid);
+out:
+	ecore_ptt_release(p_hwfn, p_ptt);
+}
+
+void ecore_llh_clear_ppfid_filters(struct ecore_dev *p_dev, u8 ppfid)
+{
+	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
+	struct ecore_ptt *p_ptt = ecore_ptt_acquire(p_hwfn);
+	u8 filter_idx, abs_ppfid;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	if (p_ptt == OSAL_NULL)
+		return;
+
+	if (!OSAL_TEST_BIT(ECORE_MF_LLH_PROTO_CLSS, &p_dev->mf_bits) &&
+	    !OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS, &p_dev->mf_bits))
+		goto out;
+
+	rc = ecore_abs_ppfid(p_dev, ppfid, &abs_ppfid);
+	if (rc != ECORE_SUCCESS)
+		goto out;
+
+	rc = ecore_llh_shadow_remove_all_filters(p_dev, ppfid);
+	if (rc != ECORE_SUCCESS)
+		goto out;
+
+	for (filter_idx = 0; filter_idx < NIG_REG_LLH_FUNC_FILTER_EN_SIZE;
+	     filter_idx++) {
+		rc = ecore_llh_remove_filter_e4(p_hwfn, p_ptt,
+						abs_ppfid, filter_idx);
+		if (rc != ECORE_SUCCESS)
+			goto out;
+	}
+out:
+	ecore_ptt_release(p_hwfn, p_ptt);
+}
+
+void ecore_llh_clear_all_filters(struct ecore_dev *p_dev)
+{
+	u8 ppfid;
+
+	if (!OSAL_TEST_BIT(ECORE_MF_LLH_PROTO_CLSS, &p_dev->mf_bits) &&
+	    !OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS, &p_dev->mf_bits))
+		return;
+
+	for (ppfid = 0; ppfid < p_dev->p_llh_info->num_ppfid; ppfid++)
+		ecore_llh_clear_ppfid_filters(p_dev, ppfid);
+}
+
+enum _ecore_status_t ecore_all_ppfids_wr(struct ecore_hwfn *p_hwfn,
+					 struct ecore_ptt *p_ptt, u32 addr,
+					 u32 val)
+{
+	struct ecore_dev *p_dev = p_hwfn->p_dev;
+	u8 ppfid, abs_ppfid;
+	enum _ecore_status_t rc;
+
+	for (ppfid = 0; ppfid < p_dev->p_llh_info->num_ppfid; ppfid++) {
+		rc = ecore_abs_ppfid(p_dev, ppfid, &abs_ppfid);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		ecore_ppfid_wr(p_hwfn, p_ptt, abs_ppfid, addr, val);
+	}
+
+	return ECORE_SUCCESS;
+}
+
+static enum _ecore_status_t
+ecore_llh_dump_ppfid_e4(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			u8 ppfid)
+{
+	struct ecore_llh_filter_e4_details filter_details;
+	u8 abs_ppfid, filter_idx;
+	u32 addr;
+	enum _ecore_status_t rc;
+
+	rc = ecore_abs_ppfid(p_hwfn->p_dev, ppfid, &abs_ppfid);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	addr = NIG_REG_PPF_TO_ENGINE_SEL + abs_ppfid * 0x4;
+	DP_NOTICE(p_hwfn, false,
+		  "[rel_pf_id %hhd, ppfid={rel %hhd, abs %hhd}, engine_sel 0x%x]\n",
+		  p_hwfn->rel_pf_id, ppfid, abs_ppfid,
+		  ecore_rd(p_hwfn, p_ptt, addr));
+
+	for (filter_idx = 0; filter_idx < NIG_REG_LLH_FUNC_FILTER_EN_SIZE;
+	     filter_idx++) {
+		OSAL_MEMSET(&filter_details, 0, sizeof(filter_details));
+		rc =  ecore_llh_access_filter_e4(p_hwfn, p_ptt, abs_ppfid,
+						 filter_idx, &filter_details,
+						 false /* read access */);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+
+		DP_NOTICE(p_hwfn, false,
+			  "filter %2hhd: enable %d, value 0x%016lx, mode %d, protocol_type 0x%x, hdr_sel 0x%x\n",
+			  filter_idx, filter_details.enable,
+			  (unsigned long)filter_details.value,
+			  filter_details.mode,
+			  filter_details.protocol_type, filter_details.hdr_sel);
+	}
+
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t ecore_llh_dump_ppfid(struct ecore_dev *p_dev, u8 ppfid)
+{
+	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
+	struct ecore_ptt *p_ptt = ecore_ptt_acquire(p_hwfn);
+	enum _ecore_status_t rc;
+
+	if (p_ptt == OSAL_NULL)
+		return ECORE_AGAIN;
+
+	rc = ecore_llh_dump_ppfid_e4(p_hwfn, p_ptt, ppfid);
+
+	ecore_ptt_release(p_hwfn, p_ptt);
+
+	return rc;
+}
+
+enum _ecore_status_t ecore_llh_dump_all(struct ecore_dev *p_dev)
+{
+	u8 ppfid;
+	enum _ecore_status_t rc;
+
+	for (ppfid = 0; ppfid < p_dev->p_llh_info->num_ppfid; ppfid++) {
+		rc = ecore_llh_dump_ppfid(p_dev, ppfid);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+	}
+
+	return ECORE_SUCCESS;
+}
+
+/******************************* NIG LLH - End ********************************/
+
 /* Configurable */
 #define ECORE_MIN_DPIS		(4)	/* The minimal num of DPIs required to
 					 * load the driver. The number was
@@ -456,6 +1639,12 @@ static void ecore_qm_info_free(struct ecore_hwfn *p_hwfn)
 	OSAL_FREE(p_hwfn->p_dev, qm_info->wfq_data);
 }
 
+static void ecore_dbg_user_data_free(struct ecore_hwfn *p_hwfn)
+{
+	OSAL_FREE(p_hwfn->p_dev, p_hwfn->dbg_user_info);
+	p_hwfn->dbg_user_info = OSAL_NULL;
+}
+
 void ecore_resc_free(struct ecore_dev *p_dev)
 {
 	int i;
@@ -470,6 +1659,8 @@ void ecore_resc_free(struct ecore_dev *p_dev)
 
 	OSAL_FREE(p_dev, p_dev->reset_stats);
 
+	ecore_llh_free(p_dev);
+
 	for_each_hwfn(p_dev, i) {
 		struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i];
 
@@ -483,6 +1674,7 @@ void ecore_resc_free(struct ecore_dev *p_dev)
 		ecore_l2_free(p_hwfn);
 		ecore_dmae_info_free(p_hwfn);
 		ecore_dcbx_info_free(p_hwfn);
+		ecore_dbg_user_data_free(p_hwfn);
 		/* @@@TBD Flush work-queue ? */
 
 		/* destroy doorbell recovery mechanism */
@@ -562,12 +1754,11 @@ u16 ecore_init_qm_get_num_pf_rls(struct ecore_hwfn *p_hwfn)
 {
 	u16 num_pf_rls, num_vfs = ecore_init_qm_get_num_vfs(p_hwfn);
 
-	/* @DPDK */
 	/* num RLs can't exceed resource amount of rls or vports or the
 	 * dcqcn qps
 	 */
 	num_pf_rls = (u16)OSAL_MIN_T(u32, RESC_NUM(p_hwfn, ECORE_RL),
-				     (u16)RESC_NUM(p_hwfn, ECORE_VPORT));
+				     RESC_NUM(p_hwfn, ECORE_VPORT));
 
 	/* make sure after we reserve the default and VF rls we'll have
 	 * something left
@@ -828,7 +2019,7 @@ u16 ecore_get_cm_pq_idx_mcos(struct ecore_hwfn *p_hwfn, u8 tc)
 	if (tc > max_tc)
 		DP_ERR(p_hwfn, "tc %d must be smaller than %d\n", tc, max_tc);
 
-	return ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_MCOS) + tc;
+	return ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_MCOS) + (tc % max_tc);
 }
 
 u16 ecore_get_cm_pq_idx_vf(struct ecore_hwfn *p_hwfn, u16 vf)
@@ -838,17 +2029,17 @@ u16 ecore_get_cm_pq_idx_vf(struct ecore_hwfn *p_hwfn, u16 vf)
 	if (vf > max_vf)
 		DP_ERR(p_hwfn, "vf %d must be smaller than %d\n", vf, max_vf);
 
-	return ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_VFS) + vf;
+	return ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_VFS) + (vf % max_vf);
 }
 
 u16 ecore_get_cm_pq_idx_rl(struct ecore_hwfn *p_hwfn, u16 rl)
 {
 	u16 max_rl = ecore_init_qm_get_num_pf_rls(p_hwfn);
 
-	if (rl > max_rl)
-		DP_ERR(p_hwfn, "rl %d must be smaller than %d\n", rl, max_rl);
-
-	return ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_RLS) + rl;
+	/* for rate limiters, it is okay to use the modulo behavior - no
+	 * DP_ERR
+	 */
+	return ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_RLS) + (rl % max_rl);
 }
 
 u16 ecore_get_qm_vport_idx_rl(struct ecore_hwfn *p_hwfn, u16 rl)
@@ -1334,6 +2525,20 @@ enum _ecore_status_t ecore_resc_alloc(struct ecore_dev *p_dev)
 				  "Failed to allocate memory for dcbx structure\n");
 			goto alloc_err;
 		}
+
+		rc = OSAL_DBG_ALLOC_USER_DATA(p_hwfn, &p_hwfn->dbg_user_info);
+		if (rc) {
+			DP_NOTICE(p_hwfn, false,
+				  "Failed to allocate dbg user info structure\n");
+			goto alloc_err;
+		}
+	} /* hwfn loop */
+
+	rc = ecore_llh_alloc(p_dev);
+	if (rc != ECORE_SUCCESS) {
+		DP_NOTICE(p_dev, true,
+			  "Failed to allocate memory for the llh_info structure\n");
+		goto alloc_err;
 	}
 
 	p_dev->reset_stats = OSAL_ZALLOC(p_dev, GFP_KERNEL,
@@ -1476,8 +2681,7 @@ static enum _ecore_status_t ecore_calc_hw_mode(struct ecore_hwfn *p_hwfn)
 		return ECORE_INVAL;
 	}
 
-	if (OSAL_TEST_BIT(ECORE_MF_OVLAN_CLSS,
-			  &p_hwfn->p_dev->mf_bits))
+	if (OSAL_TEST_BIT(ECORE_MF_OVLAN_CLSS, &p_hwfn->p_dev->mf_bits))
 		hw_mode |= 1 << MODE_MF_SD;
 	else
 		hw_mode |= 1 << MODE_MF_SI;
@@ -1960,6 +3164,14 @@ enum ECORE_ROCE_EDPM_MODE {
 	ECORE_ROCE_EDPM_MODE_DISABLE = 2,
 };
 
+bool ecore_edpm_enabled(struct ecore_hwfn *p_hwfn)
+{
+	if (p_hwfn->dcbx_no_edpm || p_hwfn->db_bar_no_edpm)
+		return false;
+
+	return true;
+}
+
 static enum _ecore_status_t
 ecore_hw_init_pf_doorbell_bar(struct ecore_hwfn *p_hwfn,
 			      struct ecore_ptt *p_ptt)
@@ -2047,7 +3259,7 @@ ecore_hw_init_pf_doorbell_bar(struct ecore_hwfn *p_hwfn,
 	DP_INFO(p_hwfn,
 		" dpi_size=%d, dpi_count=%d, roce_edpm=%s\n",
 		p_hwfn->dpi_size, p_hwfn->dpi_count,
-		((p_hwfn->dcbx_no_edpm) || (p_hwfn->db_bar_no_edpm)) ?
+		(!ecore_edpm_enabled(p_hwfn)) ?
 		"disabled" : "enabled");
 
 	/* Check return codes from above calls */
@@ -2073,17 +3285,7 @@ static enum _ecore_status_t ecore_hw_init_port(struct ecore_hwfn *p_hwfn,
 					       struct ecore_ptt *p_ptt,
 					       int hw_mode)
 {
-	u32 ppf_to_eng_sel[NIG_REG_PPF_TO_ENGINE_SEL_RT_SIZE];
-	u32 val;
 	enum _ecore_status_t rc	= ECORE_SUCCESS;
-	u8 i;
-
-	/* In CMT for non-RoCE packets - use connection based classification */
-	val = ECORE_IS_CMT(p_hwfn->p_dev) ? 0x8 : 0x0;
-	for (i = 0; i < NIG_REG_PPF_TO_ENGINE_SEL_RT_SIZE; i++)
-		ppf_to_eng_sel[i] = val;
-	STORE_RT_REG_AGG(p_hwfn, NIG_REG_PPF_TO_ENGINE_SEL_RT_OFFSET,
-			 ppf_to_eng_sel);
 
 	/* In CMT the gate should be cleared by the 2nd hwfn */
 	if (!ECORE_IS_CMT(p_hwfn->p_dev) || !IS_LEAD_HWFN(p_hwfn))
@@ -2135,12 +3337,8 @@ static enum _ecore_status_t ecore_hw_init_port(struct ecore_hwfn *p_hwfn,
 }
 
 static enum _ecore_status_t
-ecore_hw_init_pf(struct ecore_hwfn *p_hwfn,
-		 struct ecore_ptt *p_ptt,
-		 struct ecore_tunnel_info *p_tunn,
-		 int hw_mode,
-		 bool b_hw_start,
-		 enum ecore_int_mode int_mode, bool allow_npar_tx_switch)
+ecore_hw_init_pf(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+		 int hw_mode, struct ecore_hw_init_params *p_params)
 {
 	u8 rel_pf_id = p_hwfn->rel_pf_id;
 	u32 prs_reg;
@@ -2228,17 +3426,18 @@ ecore_hw_init_pf(struct ecore_hwfn *p_hwfn,
 	*/
 
 	rc = ecore_hw_init_pf_doorbell_bar(p_hwfn, p_ptt);
-	if (rc)
+	if (rc != ECORE_SUCCESS)
 		return rc;
-	if (b_hw_start) {
+
+	if (p_params->b_hw_start) {
 		/* enable interrupts */
-		rc = ecore_int_igu_enable(p_hwfn, p_ptt, int_mode);
+		rc = ecore_int_igu_enable(p_hwfn, p_ptt, p_params->int_mode);
 		if (rc != ECORE_SUCCESS)
 			return rc;
 
 		/* send function start command */
-		rc = ecore_sp_pf_start(p_hwfn, p_ptt, p_tunn,
-				       allow_npar_tx_switch);
+		rc = ecore_sp_pf_start(p_hwfn, p_ptt, p_params->p_tunn,
+				       p_params->allow_npar_tx_switch);
 		if (rc) {
 			DP_NOTICE(p_hwfn, true,
 				  "Function start ramrod failed\n");
@@ -2410,6 +3609,7 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev,
 	bool b_default_mtu = true;
 	struct ecore_hwfn *p_hwfn;
 	enum _ecore_status_t rc = ECORE_SUCCESS;
+	u16 ether_type;
 	int i;
 
 	if ((p_params->int_mode == ECORE_INT_MODE_MSI) && ECORE_IS_CMT(p_dev)) {
@@ -2442,6 +3642,25 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev,
 		if (rc != ECORE_SUCCESS)
 			return rc;
 
+		if (IS_PF(p_dev) && (OSAL_TEST_BIT(ECORE_MF_8021Q_TAGGING,
+						   &p_dev->mf_bits) ||
+				     OSAL_TEST_BIT(ECORE_MF_8021AD_TAGGING,
+						   &p_dev->mf_bits))) {
+			if (OSAL_TEST_BIT(ECORE_MF_8021Q_TAGGING,
+					  &p_dev->mf_bits))
+				ether_type = ETHER_TYPE_VLAN;
+			else
+				ether_type = ETHER_TYPE_QINQ;
+			STORE_RT_REG(p_hwfn, PRS_REG_TAG_ETHERTYPE_0_RT_OFFSET,
+				     ether_type);
+			STORE_RT_REG(p_hwfn, NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET,
+				     ether_type);
+			STORE_RT_REG(p_hwfn, PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET,
+				     ether_type);
+			STORE_RT_REG(p_hwfn, DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET,
+				     ether_type);
+		}
+
 		ecore_set_spq_block_timeout(p_hwfn, p_params->spq_timeout_ms);
 
 		rc = ecore_fill_load_req_params(p_hwfn, &load_req_params,
@@ -2542,11 +3761,8 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev,
 			/* Fall into */
 		case FW_MSG_CODE_DRV_LOAD_FUNCTION:
 			rc = ecore_hw_init_pf(p_hwfn, p_hwfn->p_main_ptt,
-					      p_params->p_tunn,
 					      p_hwfn->hw_info.hw_mode,
-					      p_params->b_hw_start,
-					      p_params->int_mode,
-					      p_params->allow_npar_tx_switch);
+					      p_params);
 			break;
 		default:
 			DP_NOTICE(p_hwfn, false,
@@ -2591,6 +3807,34 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev,
 	}
 
 	if (IS_PF(p_dev)) {
+		/* Get pre-negotiated values for stag, bandwidth etc. */
+		p_hwfn = ECORE_LEADING_HWFN(p_dev);
+		DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ,
+			   "Sending GET_OEM_UPDATES command to trigger stag/bandwidth attention handling\n");
+		rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt,
+				   DRV_MSG_CODE_GET_OEM_UPDATES,
+				   1 << DRV_MB_PARAM_DUMMY_OEM_UPDATES_OFFSET,
+				   &resp, &param);
+		if (rc != ECORE_SUCCESS)
+			DP_NOTICE(p_hwfn, false,
+				  "Failed to send GET_OEM_UPDATES attention request\n");
+	}
+
+	if (IS_PF(p_dev)) {
+		/* Get pre-negotiated values for stag, bandwidth etc. */
+		p_hwfn = ECORE_LEADING_HWFN(p_dev);
+		DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ,
+			   "Sending GET_OEM_UPDATES command to trigger stag/bandwidth attention handling\n");
+		rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt,
+				   DRV_MSG_CODE_GET_OEM_UPDATES,
+				   1 << DRV_MB_PARAM_DUMMY_OEM_UPDATES_OFFSET,
+				   &resp, &param);
+		if (rc != ECORE_SUCCESS)
+			DP_NOTICE(p_hwfn, false,
+				  "Failed to send GET_OEM_UPDATES attention request\n");
+	}
+
+	if (IS_PF(p_dev)) {
 		p_hwfn = ECORE_LEADING_HWFN(p_dev);
 		drv_mb_param = STORM_FW_VERSION;
 		rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt,
@@ -2599,17 +3843,23 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev,
 		if (rc != ECORE_SUCCESS)
 			DP_INFO(p_hwfn, "Failed to update firmware version\n");
 
-		if (!b_default_mtu)
+		if (!b_default_mtu) {
 			rc = ecore_mcp_ov_update_mtu(p_hwfn, p_hwfn->p_main_ptt,
 						      p_hwfn->hw_info.mtu);
-		if (rc != ECORE_SUCCESS)
-			DP_INFO(p_hwfn, "Failed to update default mtu\n");
+			if (rc != ECORE_SUCCESS)
+				DP_INFO(p_hwfn, "Failed to update default mtu\n");
+		}
 
 		rc = ecore_mcp_ov_update_driver_state(p_hwfn,
 						      p_hwfn->p_main_ptt,
 						ECORE_OV_DRIVER_STATE_DISABLED);
 		if (rc != ECORE_SUCCESS)
 			DP_INFO(p_hwfn, "Failed to update driver state\n");
+
+		rc = ecore_mcp_ov_update_eswitch(p_hwfn, p_hwfn->p_main_ptt,
+						 ECORE_OV_ESWITCH_NONE);
+		if (rc != ECORE_SUCCESS)
+			DP_INFO(p_hwfn, "Failed to update eswitch mode\n");
 	}
 
 	return rc;
@@ -2742,6 +3992,12 @@ enum _ecore_status_t ecore_hw_stop(struct ecore_dev *p_dev)
 			rc2 = ECORE_UNKNOWN_ERROR;
 		}
 
+		OSAL_DPC_SYNC(p_hwfn);
+
+		/* After this point we don't expect the FW to send us async
+		 * events
+		 */
+
 		/* perform debug action after PF stop was sent */
 		OSAL_AFTER_PF_STOP((void *)p_dev, p_hwfn->my_id);
 
@@ -2778,6 +4034,12 @@ enum _ecore_status_t ecore_hw_stop(struct ecore_dev *p_dev)
 		/* Need to wait 1ms to guarantee SBs are cleared */
 		OSAL_MSLEEP(1);
 
+		if (IS_LEAD_HWFN(p_hwfn) &&
+		    OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS, &p_dev->mf_bits) &&
+		    !ECORE_IS_FCOE_PERSONALITY(p_hwfn))
+			ecore_llh_remove_mac_filter(p_dev, 0,
+						   p_hwfn->hw_info.hw_mac_addr);
+
 		if (!p_dev->recov_in_prog) {
 			ecore_verify_reg_val(p_hwfn, p_ptt,
 					     QM_REG_USG_CNT_PF_TX, 0);
@@ -2987,15 +4249,30 @@ static void ecore_hw_set_feat(struct ecore_hwfn *p_hwfn)
 				   FEAT_NUM(p_hwfn, ECORE_VF_L2_QUE));
 	}
 
-	if (ECORE_IS_FCOE_PERSONALITY(p_hwfn))
-		feat_num[ECORE_FCOE_CQ] =
-			OSAL_MIN_T(u32, sb_cnt.cnt, RESC_NUM(p_hwfn,
-							     ECORE_CMDQS_CQS));
+	if (ECORE_IS_FCOE_PERSONALITY(p_hwfn) ||
+	    ECORE_IS_ISCSI_PERSONALITY(p_hwfn)) {
+		u32 *p_storage_feat = ECORE_IS_FCOE_PERSONALITY(p_hwfn) ?
+				      &feat_num[ECORE_FCOE_CQ] :
+				      &feat_num[ECORE_ISCSI_CQ];
+		u32 limit = sb_cnt.cnt;
 
-	if (ECORE_IS_ISCSI_PERSONALITY(p_hwfn))
-		feat_num[ECORE_ISCSI_CQ] =
-			OSAL_MIN_T(u32, sb_cnt.cnt, RESC_NUM(p_hwfn,
-							     ECORE_CMDQS_CQS));
+		/* The number of queues should not exceed the number of FP SBs.
+		 * In storage target, the queues are divided into pairs of a CQ
+		 * and a CmdQ, and each pair uses a single SB. The limit in
+		 * this case should allow a max ratio of 2:1 instead of 1:1.
+		 */
+		if (p_hwfn->p_dev->b_is_target)
+			limit *= 2;
+		*p_storage_feat = OSAL_MIN_T(u32, limit,
+					     RESC_NUM(p_hwfn, ECORE_CMDQS_CQS));
+
+		/* @DPDK */
+		/* The size of "cq_cmdq_sb_num_arr" in the fcoe/iscsi init
+		 * ramrod is limited to "NUM_OF_GLOBAL_QUEUES / 2".
+		 */
+		*p_storage_feat = OSAL_MIN_T(u32, *p_storage_feat,
+					     (NUM_OF_GLOBAL_QUEUES / 2));
+	}
 
 	DP_VERBOSE(p_hwfn, ECORE_MSG_PROBE,
 		   "#PF_L2_QUEUE=%d VF_L2_QUEUES=%d #ROCE_CNQ=%d #FCOE_CQ=%d #ISCSI_CQ=%d #SB=%d\n",
@@ -3276,6 +4553,59 @@ static enum _ecore_status_t ecore_hw_set_resc_info(struct ecore_hwfn *p_hwfn,
 	return ECORE_SUCCESS;
 }
 
+#define ECORE_NONUSED_PPFID_MASK_BB_4P_LO_PORTS	0xaa
+#define ECORE_NONUSED_PPFID_MASK_BB_4P_HI_PORTS	0x55
+#define ECORE_NONUSED_PPFID_MASK_AH_4P		0xf0
+
+static enum _ecore_status_t ecore_hw_get_ppfid_bitmap(struct ecore_hwfn *p_hwfn,
+						      struct ecore_ptt *p_ptt)
+{
+	u8 native_ppfid_idx = ECORE_PPFID_BY_PFID(p_hwfn), new_bitmap;
+	struct ecore_dev *p_dev = p_hwfn->p_dev;
+	enum _ecore_status_t rc;
+
+	rc = ecore_mcp_get_ppfid_bitmap(p_hwfn, p_ptt);
+	if (rc != ECORE_SUCCESS && rc != ECORE_NOTIMPL)
+		return rc;
+	else if (rc == ECORE_NOTIMPL)
+		p_dev->ppfid_bitmap = 0x1 << native_ppfid_idx;
+
+	/* 4-ports mode has limitations that should be enforced:
+	 * - BB: the MFW can access only PPFIDs which their corresponding PFIDs
+	 *       belong to this certain port.
+	 * - AH/E5: only 4 PPFIDs per port are available.
+	 */
+	if (ecore_device_num_ports(p_dev) == 4) {
+		u8 mask;
+
+		if (ECORE_IS_BB(p_dev))
+			mask = MFW_PORT(p_hwfn) > 1 ?
+			       ECORE_NONUSED_PPFID_MASK_BB_4P_HI_PORTS :
+			       ECORE_NONUSED_PPFID_MASK_BB_4P_LO_PORTS;
+		else
+			mask = ECORE_NONUSED_PPFID_MASK_AH_4P;
+
+		if (p_dev->ppfid_bitmap & mask) {
+			new_bitmap = p_dev->ppfid_bitmap & ~mask;
+			DP_INFO(p_hwfn,
+				"Fix the PPFID bitmap for 4-ports mode: 0x%hhx -> 0x%hhx\n",
+				p_dev->ppfid_bitmap, new_bitmap);
+			p_dev->ppfid_bitmap = new_bitmap;
+		}
+	}
+
+	/* The native PPFID is expected to be part of the allocated bitmap */
+	if (!(p_dev->ppfid_bitmap & (0x1 << native_ppfid_idx))) {
+		new_bitmap = 0x1 << native_ppfid_idx;
+		DP_INFO(p_hwfn,
+			"Fix the PPFID bitmap to inculde the native PPFID: %hhd -> 0x%hhx\n",
+			p_dev->ppfid_bitmap, new_bitmap);
+		p_dev->ppfid_bitmap = new_bitmap;
+	}
+
+	return ECORE_SUCCESS;
+}
+
 static enum _ecore_status_t ecore_hw_get_resc(struct ecore_hwfn *p_hwfn,
 					      struct ecore_ptt *p_ptt,
 					      bool drv_resc_alloc)
@@ -3350,6 +4680,13 @@ static enum _ecore_status_t ecore_hw_get_resc(struct ecore_hwfn *p_hwfn,
 				"Failed to release the resource lock for the resource allocation commands\n");
 	}
 
+	/* PPFID bitmap */
+	if (IS_LEAD_HWFN(p_hwfn)) {
+		rc = ecore_hw_get_ppfid_bitmap(p_hwfn, p_ptt);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+	}
+
 #ifndef ASIC_ONLY
 	if (CHIP_REV_IS_SLOW(p_hwfn->p_dev)) {
 		/* Reduced build contains less PQs */
@@ -3621,7 +4958,8 @@ ecore_hw_get_nvm_info(struct ecore_hwfn *p_hwfn,
 	case NVM_CFG1_GLOB_MF_MODE_BD:
 		p_hwfn->p_dev->mf_bits = 1 << ECORE_MF_OVLAN_CLSS |
 					 1 << ECORE_MF_LLH_PROTO_CLSS |
-					 1 << ECORE_MF_8021AD_TAGGING;
+					 1 << ECORE_MF_8021AD_TAGGING |
+					 1 << ECORE_MF_FIP_SPECIAL;
 		break;
 	case NVM_CFG1_GLOB_MF_MODE_NPAR1_0:
 		p_hwfn->p_dev->mf_bits = 1 << ECORE_MF_LLH_MAC_CLSS |
@@ -4139,9 +5477,8 @@ void ecore_prepare_hibernate(struct ecore_dev *p_dev)
 #endif
 
 static enum _ecore_status_t
-ecore_hw_prepare_single(struct ecore_hwfn *p_hwfn,
-			void OSAL_IOMEM * p_regview,
-			void OSAL_IOMEM * p_doorbells,
+ecore_hw_prepare_single(struct ecore_hwfn *p_hwfn, void OSAL_IOMEM *p_regview,
+			void OSAL_IOMEM *p_doorbells, u64 db_phys_addr,
 			struct ecore_hw_prepare_params *p_params)
 {
 	struct ecore_mdump_retain_data mdump_retain;
@@ -4152,6 +5489,7 @@ ecore_hw_prepare_single(struct ecore_hwfn *p_hwfn,
 	/* Split PCI bars evenly between hwfns */
 	p_hwfn->regview = p_regview;
 	p_hwfn->doorbells = p_doorbells;
+	p_hwfn->db_phys_addr = db_phys_addr;
 
 	if (IS_VF(p_dev))
 		return ecore_vf_hw_prepare(p_hwfn);
@@ -4217,6 +5555,13 @@ ecore_hw_prepare_single(struct ecore_hwfn *p_hwfn,
 		rc = ecore_mcp_initiate_pf_flr(p_hwfn, p_hwfn->p_main_ptt);
 		if (rc != ECORE_SUCCESS)
 			DP_NOTICE(p_hwfn, false, "Failed to initiate PF FLR\n");
+
+		/* Workaround for MFW issue where PF FLR does not cleanup
+		 * IGU block
+		 */
+		if (!(p_hwfn->mcp_info->capabilities &
+		      FW_MB_PARAM_FEATURE_SUPPORT_IGU_CLEANUP))
+			ecore_pf_flr_igu_cleanup(p_hwfn);
 	}
 
 	/* Check if mdump logs/data are present and update the epoch value */
@@ -4287,6 +5632,7 @@ enum _ecore_status_t ecore_hw_prepare(struct ecore_dev *p_dev,
 	p_dev->chk_reg_fifo = p_params->chk_reg_fifo;
 	p_dev->allow_mdump = p_params->allow_mdump;
 	p_hwfn->b_en_pacing = p_params->b_en_pacing;
+	p_dev->b_is_target = p_params->b_is_target;
 
 	if (p_params->b_relaxed_probe)
 		p_params->p_relaxed_res = ECORE_HW_PREPARE_SUCCESS;
@@ -4296,9 +5642,9 @@ enum _ecore_status_t ecore_hw_prepare(struct ecore_dev *p_dev,
 		ecore_init_iro_array(p_dev);
 
 	/* Initialize the first hwfn - will learn number of hwfns */
-	rc = ecore_hw_prepare_single(p_hwfn,
-				     p_dev->regview,
-				     p_dev->doorbells, p_params);
+	rc = ecore_hw_prepare_single(p_hwfn, p_dev->regview,
+				     p_dev->doorbells, p_dev->db_phys_addr,
+				     p_params);
 	if (rc != ECORE_SUCCESS)
 		return rc;
 
@@ -4308,24 +5654,26 @@ enum _ecore_status_t ecore_hw_prepare(struct ecore_dev *p_dev,
 	if (ECORE_IS_CMT(p_dev)) {
 		void OSAL_IOMEM *p_regview, *p_doorbell;
 		u8 OSAL_IOMEM *addr;
+		u64 db_phys_addr;
+		u32 offset;
 
 		/* adjust bar offset for second engine */
-		addr = (u8 OSAL_IOMEM *)p_dev->regview +
-					ecore_hw_bar_size(p_hwfn,
-							  p_hwfn->p_main_ptt,
-							  BAR_ID_0) / 2;
+		offset = ecore_hw_bar_size(p_hwfn, p_hwfn->p_main_ptt,
+					   BAR_ID_0) / 2;
+		addr = (u8 OSAL_IOMEM *)p_dev->regview + offset;
 		p_regview = (void OSAL_IOMEM *)addr;
 
-		addr = (u8 OSAL_IOMEM *)p_dev->doorbells +
-					ecore_hw_bar_size(p_hwfn,
-							  p_hwfn->p_main_ptt,
-							  BAR_ID_1) / 2;
+		offset = ecore_hw_bar_size(p_hwfn, p_hwfn->p_main_ptt,
+					   BAR_ID_1) / 2;
+		addr = (u8 OSAL_IOMEM *)p_dev->doorbells + offset;
 		p_doorbell = (void OSAL_IOMEM *)addr;
+		db_phys_addr = p_dev->db_phys_addr + offset;
 
 		p_dev->hwfns[1].b_en_pacing = p_params->b_en_pacing;
 		/* prepare second hw function */
 		rc = ecore_hw_prepare_single(&p_dev->hwfns[1], p_regview,
-					     p_doorbell, p_params);
+					     p_doorbell, db_phys_addr,
+					     p_params);
 
 		/* in case of error, need to free the previously
 		 * initiliazed hwfn 0.
@@ -4722,419 +6070,6 @@ enum _ecore_status_t ecore_fw_rss_eng(struct ecore_hwfn *p_hwfn,
 	return ECORE_SUCCESS;
 }
 
-static enum _ecore_status_t
-ecore_llh_add_mac_filter_bb_ah(struct ecore_hwfn *p_hwfn,
-			       struct ecore_ptt *p_ptt, u32 high, u32 low,
-			       u32 *p_entry_num)
-{
-	u32 en;
-	int i;
-
-	/* Find a free entry and utilize it */
-	for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) {
-		en = ecore_rd(p_hwfn, p_ptt,
-			      NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 +
-			      i * sizeof(u32));
-		if (en)
-			continue;
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			 2 * i * sizeof(u32), low);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			 (2 * i + 1) * sizeof(u32), high);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 +
-			 i * sizeof(u32), 0);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 +
-			 i * sizeof(u32), 0);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 +
-			 i * sizeof(u32), 1);
-		break;
-	}
-
-	if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE)
-		return ECORE_NORESOURCES;
-
-	*p_entry_num = i;
-
-	return ECORE_SUCCESS;
-}
-
-enum _ecore_status_t ecore_llh_add_mac_filter(struct ecore_hwfn *p_hwfn,
-					  struct ecore_ptt *p_ptt, u8 *p_filter)
-{
-	u32 high, low, entry_num;
-	enum _ecore_status_t rc = ECORE_SUCCESS;
-
-	if (!OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS,
-			   &p_hwfn->p_dev->mf_bits))
-		return ECORE_SUCCESS;
-
-	high = p_filter[1] | (p_filter[0] << 8);
-	low = p_filter[5] | (p_filter[4] << 8) |
-	      (p_filter[3] << 16) | (p_filter[2] << 24);
-
-	if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev))
-		rc = ecore_llh_add_mac_filter_bb_ah(p_hwfn, p_ptt, high, low,
-						    &entry_num);
-	if (rc != ECORE_SUCCESS) {
-		DP_NOTICE(p_hwfn, false,
-			  "Failed to find an empty LLH filter to utilize\n");
-		return rc;
-	}
-
-	DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
-		   "MAC: %02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx is added at %d\n",
-		   p_filter[0], p_filter[1], p_filter[2], p_filter[3],
-		   p_filter[4], p_filter[5], entry_num);
-
-	return rc;
-}
-
-static enum _ecore_status_t
-ecore_llh_remove_mac_filter_bb_ah(struct ecore_hwfn *p_hwfn,
-				  struct ecore_ptt *p_ptt, u32 high, u32 low,
-				  u32 *p_entry_num)
-{
-	int i;
-
-	/* Find the entry and clean it */
-	for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) {
-		if (ecore_rd(p_hwfn, p_ptt,
-			     NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			     2 * i * sizeof(u32)) != low)
-			continue;
-		if (ecore_rd(p_hwfn, p_ptt,
-			     NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			     (2 * i + 1) * sizeof(u32)) != high)
-			continue;
-
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + i * sizeof(u32), 0);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			 2 * i * sizeof(u32), 0);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			 (2 * i + 1) * sizeof(u32), 0);
-		break;
-	}
-
-	if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE)
-		return ECORE_INVAL;
-
-	*p_entry_num = i;
-
-	return ECORE_SUCCESS;
-}
-
-void ecore_llh_remove_mac_filter(struct ecore_hwfn *p_hwfn,
-			     struct ecore_ptt *p_ptt, u8 *p_filter)
-{
-	u32 high, low, entry_num;
-	enum _ecore_status_t rc = ECORE_SUCCESS;
-
-	if (!OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS,
-			   &p_hwfn->p_dev->mf_bits))
-		return;
-
-	high = p_filter[1] | (p_filter[0] << 8);
-	low = p_filter[5] | (p_filter[4] << 8) |
-	      (p_filter[3] << 16) | (p_filter[2] << 24);
-
-	if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev))
-		rc = ecore_llh_remove_mac_filter_bb_ah(p_hwfn, p_ptt, high,
-						       low, &entry_num);
-	if (rc != ECORE_SUCCESS) {
-		DP_NOTICE(p_hwfn, false,
-			  "Tried to remove a non-configured filter\n");
-		return;
-	}
-
-
-	DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
-		   "MAC: %02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx was removed from %d\n",
-		   p_filter[0], p_filter[1], p_filter[2], p_filter[3],
-		   p_filter[4], p_filter[5], entry_num);
-}
-
-static enum _ecore_status_t
-ecore_llh_add_protocol_filter_bb_ah(struct ecore_hwfn *p_hwfn,
-				    struct ecore_ptt *p_ptt,
-				    enum ecore_llh_port_filter_type_t type,
-				    u32 high, u32 low, u32 *p_entry_num)
-{
-	u32 en;
-	int i;
-
-	/* Find a free entry and utilize it */
-	for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) {
-		en = ecore_rd(p_hwfn, p_ptt,
-			      NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 +
-			      i * sizeof(u32));
-		if (en)
-			continue;
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			 2 * i * sizeof(u32), low);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			 (2 * i + 1) * sizeof(u32), high);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 +
-			 i * sizeof(u32), 1);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 +
-			 i * sizeof(u32), 1 << type);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + i * sizeof(u32), 1);
-		break;
-	}
-
-	if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE)
-		return ECORE_NORESOURCES;
-
-	*p_entry_num = i;
-
-	return ECORE_SUCCESS;
-}
-
-enum _ecore_status_t
-ecore_llh_add_protocol_filter(struct ecore_hwfn *p_hwfn,
-			      struct ecore_ptt *p_ptt,
-			      u16 source_port_or_eth_type,
-			      u16 dest_port,
-			      enum ecore_llh_port_filter_type_t type)
-{
-	u32 high, low, entry_num;
-	enum _ecore_status_t rc = ECORE_SUCCESS;
-
-	if (!OSAL_TEST_BIT(ECORE_MF_LLH_PROTO_CLSS,
-			   &p_hwfn->p_dev->mf_bits))
-		return rc;
-
-	high = 0;
-	low = 0;
-
-	switch (type) {
-	case ECORE_LLH_FILTER_ETHERTYPE:
-		high = source_port_or_eth_type;
-		break;
-	case ECORE_LLH_FILTER_TCP_SRC_PORT:
-	case ECORE_LLH_FILTER_UDP_SRC_PORT:
-		low = source_port_or_eth_type << 16;
-		break;
-	case ECORE_LLH_FILTER_TCP_DEST_PORT:
-	case ECORE_LLH_FILTER_UDP_DEST_PORT:
-		low = dest_port;
-		break;
-	case ECORE_LLH_FILTER_TCP_SRC_AND_DEST_PORT:
-	case ECORE_LLH_FILTER_UDP_SRC_AND_DEST_PORT:
-		low = (source_port_or_eth_type << 16) | dest_port;
-		break;
-	default:
-		DP_NOTICE(p_hwfn, true,
-			  "Non valid LLH protocol filter type %d\n", type);
-		return ECORE_INVAL;
-	}
-
-	if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev))
-		rc = ecore_llh_add_protocol_filter_bb_ah(p_hwfn, p_ptt, type,
-							 high, low, &entry_num);
-	if (rc != ECORE_SUCCESS) {
-		DP_NOTICE(p_hwfn, false,
-			  "Failed to find an empty LLH filter to utilize\n");
-		return rc;
-	}
-	switch (type) {
-	case ECORE_LLH_FILTER_ETHERTYPE:
-		DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
-			   "ETH type %x is added at %d\n",
-			   source_port_or_eth_type, entry_num);
-		break;
-	case ECORE_LLH_FILTER_TCP_SRC_PORT:
-		DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
-			   "TCP src port %x is added at %d\n",
-			   source_port_or_eth_type, entry_num);
-		break;
-	case ECORE_LLH_FILTER_UDP_SRC_PORT:
-		DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
-			   "UDP src port %x is added at %d\n",
-			   source_port_or_eth_type, entry_num);
-		break;
-	case ECORE_LLH_FILTER_TCP_DEST_PORT:
-		DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
-			   "TCP dst port %x is added at %d\n", dest_port,
-			   entry_num);
-		break;
-	case ECORE_LLH_FILTER_UDP_DEST_PORT:
-		DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
-			   "UDP dst port %x is added at %d\n", dest_port,
-			   entry_num);
-		break;
-	case ECORE_LLH_FILTER_TCP_SRC_AND_DEST_PORT:
-		DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
-			   "TCP src/dst ports %x/%x are added at %d\n",
-			   source_port_or_eth_type, dest_port, entry_num);
-		break;
-	case ECORE_LLH_FILTER_UDP_SRC_AND_DEST_PORT:
-		DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
-			   "UDP src/dst ports %x/%x are added at %d\n",
-			   source_port_or_eth_type, dest_port, entry_num);
-		break;
-	}
-
-	return rc;
-}
-
-static enum _ecore_status_t
-ecore_llh_remove_protocol_filter_bb_ah(struct ecore_hwfn *p_hwfn,
-				       struct ecore_ptt *p_ptt,
-				       enum ecore_llh_port_filter_type_t type,
-				       u32 high, u32 low, u32 *p_entry_num)
-{
-	int i;
-
-	/* Find the entry and clean it */
-	for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) {
-		if (!ecore_rd(p_hwfn, p_ptt,
-			      NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 +
-			      i * sizeof(u32)))
-			continue;
-		if (!ecore_rd(p_hwfn, p_ptt,
-			      NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 +
-			      i * sizeof(u32)))
-			continue;
-		if (!(ecore_rd(p_hwfn, p_ptt,
-			       NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 +
-			       i * sizeof(u32)) & (1 << type)))
-			continue;
-		if (ecore_rd(p_hwfn, p_ptt,
-			     NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			     2 * i * sizeof(u32)) != low)
-			continue;
-		if (ecore_rd(p_hwfn, p_ptt,
-			     NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			     (2 * i + 1) * sizeof(u32)) != high)
-			continue;
-
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + i * sizeof(u32), 0);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 +
-			 i * sizeof(u32), 0);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 +
-			 i * sizeof(u32), 0);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			 2 * i * sizeof(u32), 0);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			 (2 * i + 1) * sizeof(u32), 0);
-		break;
-	}
-
-	if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE)
-		return ECORE_INVAL;
-
-	*p_entry_num = i;
-
-	return ECORE_SUCCESS;
-}
-
-void
-ecore_llh_remove_protocol_filter(struct ecore_hwfn *p_hwfn,
-				 struct ecore_ptt *p_ptt,
-				 u16 source_port_or_eth_type,
-				 u16 dest_port,
-				 enum ecore_llh_port_filter_type_t type)
-{
-	u32 high, low, entry_num;
-	enum _ecore_status_t rc = ECORE_SUCCESS;
-
-	if (!OSAL_TEST_BIT(ECORE_MF_LLH_PROTO_CLSS,
-			   &p_hwfn->p_dev->mf_bits))
-		return;
-
-	high = 0;
-	low = 0;
-
-	switch (type) {
-	case ECORE_LLH_FILTER_ETHERTYPE:
-		high = source_port_or_eth_type;
-		break;
-	case ECORE_LLH_FILTER_TCP_SRC_PORT:
-	case ECORE_LLH_FILTER_UDP_SRC_PORT:
-		low = source_port_or_eth_type << 16;
-		break;
-	case ECORE_LLH_FILTER_TCP_DEST_PORT:
-	case ECORE_LLH_FILTER_UDP_DEST_PORT:
-		low = dest_port;
-		break;
-	case ECORE_LLH_FILTER_TCP_SRC_AND_DEST_PORT:
-	case ECORE_LLH_FILTER_UDP_SRC_AND_DEST_PORT:
-		low = (source_port_or_eth_type << 16) | dest_port;
-		break;
-	default:
-		DP_NOTICE(p_hwfn, true,
-			  "Non valid LLH protocol filter type %d\n", type);
-		return;
-	}
-
-	if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev))
-		rc = ecore_llh_remove_protocol_filter_bb_ah(p_hwfn, p_ptt, type,
-							    high, low,
-							    &entry_num);
-	if (rc != ECORE_SUCCESS) {
-		DP_NOTICE(p_hwfn, false,
-			  "Tried to remove a non-configured filter [type %d, source_port_or_eth_type 0x%x, dest_port 0x%x]\n",
-			  type, source_port_or_eth_type, dest_port);
-		return;
-	}
-
-	DP_VERBOSE(p_hwfn, ECORE_MSG_HW,
-		   "Protocol filter [type %d, source_port_or_eth_type 0x%x, dest_port 0x%x] was removed from %d\n",
-		   type, source_port_or_eth_type, dest_port, entry_num);
-}
-
-static void ecore_llh_clear_all_filters_bb_ah(struct ecore_hwfn *p_hwfn,
-					      struct ecore_ptt *p_ptt)
-{
-	int i;
-
-	if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn)))
-		return;
-
-	for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) {
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_EN_BB_K2  +
-			 i * sizeof(u32), 0);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			 2 * i * sizeof(u32), 0);
-		ecore_wr(p_hwfn, p_ptt,
-			 NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 +
-			 (2 * i + 1) * sizeof(u32), 0);
-	}
-}
-
-void ecore_llh_clear_all_filters(struct ecore_hwfn *p_hwfn,
-			     struct ecore_ptt *p_ptt)
-{
-	if (!OSAL_TEST_BIT(ECORE_MF_LLH_PROTO_CLSS,
-			   &p_hwfn->p_dev->mf_bits) &&
-	    !OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS,
-			   &p_hwfn->p_dev->mf_bits))
-		return;
-
-	if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev))
-		ecore_llh_clear_all_filters_bb_ah(p_hwfn, p_ptt);
-}
-
 enum _ecore_status_t
 ecore_llh_set_function_as_default(struct ecore_hwfn *p_hwfn,
 				  struct ecore_ptt *p_ptt)
@@ -5713,3 +6648,8 @@ void ecore_set_fw_mac_addr(__le16 *fw_msb,
 	((u8 *)fw_lsb)[0] = mac[5];
 	((u8 *)fw_lsb)[1] = mac[4];
 }
+
+bool ecore_is_mf_fip_special(struct ecore_dev *p_dev)
+{
+	return !!OSAL_TEST_BIT(ECORE_MF_FIP_SPECIAL, &p_dev->mf_bits);
+}
diff --git a/drivers/net/qede/base/ecore_dev_api.h b/drivers/net/qede/base/ecore_dev_api.h
index 02bacc22..73080632 100644
--- a/drivers/net/qede/base/ecore_dev_api.h
+++ b/drivers/net/qede/base/ecore_dev_api.h
@@ -114,6 +114,9 @@ struct ecore_hw_init_params {
 	/* Driver load parameters */
 	struct ecore_drv_load_params *p_drv_load_params;
 
+	/* Avoid engine affinity for RoCE/storage in case of CMT mode */
+	bool avoid_eng_affin;
+
 	/* SPQ block timeout in msec */
 	u32 spq_timeout_ms;
 };
@@ -271,6 +274,9 @@ struct ecore_hw_prepare_params {
 
 	/* Enable/disable request by ecore client for pacing */
 	bool b_en_pacing;
+
+	/* Indicates whether this PF serves a storage target */
+	bool b_is_target;
 };
 
 /**
@@ -425,11 +431,17 @@ enum ecore_dmae_address_type_t {
 #define ECORE_DMAE_FLAG_VF_SRC		0x00000002
 #define ECORE_DMAE_FLAG_VF_DST		0x00000004
 #define ECORE_DMAE_FLAG_COMPLETION_DST	0x00000008
+#define ECORE_DMAE_FLAG_PORT		0x00000010
+#define ECORE_DMAE_FLAG_PF_SRC		0x00000020
+#define ECORE_DMAE_FLAG_PF_DST		0x00000040
 
 struct ecore_dmae_params {
 	u32 flags; /* consists of ECORE_DMAE_FLAG_* values */
 	u8 src_vfid;
 	u8 dst_vfid;
+	u8 port_id;
+	u8 src_pfid;
+	u8 dst_pfid;
 };
 
 /**
@@ -441,7 +453,9 @@ struct ecore_dmae_params {
  * @param source_addr
  * @param grc_addr (dmae_data_offset)
  * @param size_in_dwords
- * @param flags (one of the flags defined above)
+ * @param p_params (default parameters will be used in case of OSAL_NULL)
+ *
+ * @return enum _ecore_status_t
  */
 enum _ecore_status_t
 ecore_dmae_host2grc(struct ecore_hwfn *p_hwfn,
@@ -449,7 +463,7 @@ ecore_dmae_host2grc(struct ecore_hwfn *p_hwfn,
 		    u64 source_addr,
 		    u32 grc_addr,
 		    u32 size_in_dwords,
-		    u32 flags);
+		    struct ecore_dmae_params *p_params);
 
 /**
  * @brief ecore_dmae_grc2host - Read data from dmae data offset
@@ -459,7 +473,9 @@ ecore_dmae_host2grc(struct ecore_hwfn *p_hwfn,
  * @param grc_addr (dmae_data_offset)
  * @param dest_addr
  * @param size_in_dwords
- * @param flags - one of the flags defined above
+ * @param p_params (default parameters will be used in case of OSAL_NULL)
+ *
+ * @return enum _ecore_status_t
  */
 enum _ecore_status_t
 ecore_dmae_grc2host(struct ecore_hwfn *p_hwfn,
@@ -467,7 +483,7 @@ ecore_dmae_grc2host(struct ecore_hwfn *p_hwfn,
 		    u32 grc_addr,
 		    dma_addr_t dest_addr,
 		    u32 size_in_dwords,
-		    u32 flags);
+		    struct ecore_dmae_params *p_params);
 
 /**
  * @brief ecore_dmae_host2host - copy data from to source address
@@ -478,7 +494,9 @@ ecore_dmae_grc2host(struct ecore_hwfn *p_hwfn,
  * @param source_addr
  * @param dest_addr
  * @param size_in_dwords
- * @param params
+ * @param p_params (default parameters will be used in case of OSAL_NULL)
+ *
+ * @return enum _ecore_status_t
  */
 enum _ecore_status_t
 ecore_dmae_host2host(struct ecore_hwfn *p_hwfn,
@@ -559,28 +577,79 @@ enum _ecore_status_t ecore_fw_rss_eng(struct ecore_hwfn *p_hwfn,
 				      u8 *dst_id);
 
 /**
- * @brief ecore_llh_add_mac_filter - configures a MAC filter in llh
+ * @brief ecore_llh_get_num_ppfid - Return the allocated number of LLH filter
+ *	banks that are allocated to the PF.
  *
- * @param p_hwfn
- * @param p_ptt
- * @param p_filter - MAC to add
+ * @param p_dev
+ *
+ * @return u8 - Number of LLH filter banks
  */
-enum _ecore_status_t ecore_llh_add_mac_filter(struct ecore_hwfn *p_hwfn,
-					  struct ecore_ptt *p_ptt,
-					  u8 *p_filter);
+u8 ecore_llh_get_num_ppfid(struct ecore_dev *p_dev);
+
+enum ecore_eng {
+	ECORE_ENG0,
+	ECORE_ENG1,
+	ECORE_BOTH_ENG,
+};
 
 /**
- * @brief ecore_llh_remove_mac_filter - removes a MAC filtre from llh
+ * @brief ecore_llh_get_l2_affinity_hint - Return the hint for the L2 affinity
  *
- * @param p_hwfn
- * @param p_ptt
- * @param p_filter - MAC to remove
+ * @param p_dev
+ *
+ * @return enum ecore_eng - L2 affintiy hint
+ */
+enum ecore_eng ecore_llh_get_l2_affinity_hint(struct ecore_dev *p_dev);
+
+/**
+ * @brief ecore_llh_set_ppfid_affinity - Set the engine affinity for the given
+ *	LLH filter bank.
+ *
+ * @param p_dev
+ * @param ppfid - relative within the allocated ppfids ('0' is the default one).
+ * @param eng
+ *
+ * @return enum _ecore_status_t
+ */
+enum _ecore_status_t ecore_llh_set_ppfid_affinity(struct ecore_dev *p_dev,
+						  u8 ppfid, enum ecore_eng eng);
+
+/**
+ * @brief ecore_llh_set_roce_affinity - Set the RoCE engine affinity
+ *
+ * @param p_dev
+ * @param eng
+ *
+ * @return enum _ecore_status_t
+ */
+enum _ecore_status_t ecore_llh_set_roce_affinity(struct ecore_dev *p_dev,
+						 enum ecore_eng eng);
+
+/**
+ * @brief ecore_llh_add_mac_filter - Add a LLH MAC filter into the given filter
+ *	bank.
+ *
+ * @param p_dev
+ * @param ppfid - relative within the allocated ppfids ('0' is the default one).
+ * @param mac_addr - MAC to add
+ *
+ * @return enum _ecore_status_t
+ */
+enum _ecore_status_t ecore_llh_add_mac_filter(struct ecore_dev *p_dev, u8 ppfid,
+					      u8 mac_addr[ETH_ALEN]);
+
+/**
+ * @brief ecore_llh_remove_mac_filter - Remove a LLH MAC filter from the given
+ *	filter bank.
+ *
+ * @param p_dev
+ * @param ppfid - relative within the allocated ppfids ('0' is the default one).
+ * @param mac_addr - MAC to remove
  */
-void ecore_llh_remove_mac_filter(struct ecore_hwfn *p_hwfn,
-			     struct ecore_ptt *p_ptt,
-			     u8 *p_filter);
+void ecore_llh_remove_mac_filter(struct ecore_dev *p_dev, u8 ppfid,
+				 u8 mac_addr[ETH_ALEN]);
 
-enum ecore_llh_port_filter_type_t {
+enum ecore_llh_prot_filter_type_t {
 	ECORE_LLH_FILTER_ETHERTYPE,
 	ECORE_LLH_FILTER_TCP_SRC_PORT,
 	ECORE_LLH_FILTER_TCP_DEST_PORT,
@@ -591,45 +660,52 @@ enum ecore_llh_port_filter_type_t {
 };
 
 /**
- * @brief ecore_llh_add_protocol_filter - configures a protocol filter in llh
+ * @brief ecore_llh_add_protocol_filter - Add a LLH protocol filter into the
+ *	given filter bank.
  *
- * @param p_hwfn
- * @param p_ptt
+ * @param p_dev
+ * @param ppfid - relative within the allocated ppfids ('0' is the default one).
+ * @param type - type of filters and comparing
  * @param source_port_or_eth_type - source port or ethertype to add
  * @param dest_port - destination port to add
- * @param type - type of filters and comparing
+ *
+ * @return enum _ecore_status_t
  */
 enum _ecore_status_t
-ecore_llh_add_protocol_filter(struct ecore_hwfn *p_hwfn,
-			      struct ecore_ptt *p_ptt,
-			      u16 source_port_or_eth_type,
-			      u16 dest_port,
-			      enum ecore_llh_port_filter_type_t type);
+ecore_llh_add_protocol_filter(struct ecore_dev *p_dev, u8 ppfid,
+			      enum ecore_llh_prot_filter_type_t type,
+			      u16 source_port_or_eth_type, u16 dest_port);
 
 /**
- * @brief ecore_llh_remove_protocol_filter - remove a protocol filter in llh
+ * @brief ecore_llh_remove_protocol_filter - Remove a LLH protocol filter from
+ *	the given filter bank.
  *
- * @param p_hwfn
- * @param p_ptt
+ * @param p_dev
+ * @param ppfid - relative within the allocated ppfids ('0' is the default one).
+ * @param type - type of filters and comparing
  * @param source_port_or_eth_type - source port or ethertype to add
  * @param dest_port - destination port to add
- * @param type - type of filters and comparing
  */
-void
-ecore_llh_remove_protocol_filter(struct ecore_hwfn *p_hwfn,
-				 struct ecore_ptt *p_ptt,
-				 u16 source_port_or_eth_type,
-				 u16 dest_port,
-				 enum ecore_llh_port_filter_type_t type);
+void ecore_llh_remove_protocol_filter(struct ecore_dev *p_dev, u8 ppfid,
+				      enum ecore_llh_prot_filter_type_t type,
+				      u16 source_port_or_eth_type,
+				      u16 dest_port);
 
 /**
- * @brief ecore_llh_clear_all_filters - removes all MAC filters from llh
+ * @brief ecore_llh_clear_ppfid_filters - Remove all LLH filters from the given
+ *	filter bank.
  *
- * @param p_hwfn
- * @param p_ptt
+ * @param p_dev
+ * @param ppfid - relative within the allocated ppfids ('0' is the default one).
  */
-void ecore_llh_clear_all_filters(struct ecore_hwfn *p_hwfn,
-			     struct ecore_ptt *p_ptt);
+void ecore_llh_clear_ppfid_filters(struct ecore_dev *p_dev, u8 ppfid);
+
+/**
+ * @brief ecore_llh_clear_all_filters - Remove all LLH filters
+ *
+ * @param p_dev
+ */
+void ecore_llh_clear_all_filters(struct ecore_dev *p_dev);
 
 /**
  * @brief ecore_llh_set_function_as_default - set function as default per port
@@ -701,4 +777,13 @@ ecore_set_queue_coalesce(struct ecore_hwfn *p_hwfn, u16 rx_coal,
 enum _ecore_status_t ecore_pglueb_set_pfid_enable(struct ecore_hwfn *p_hwfn,
 						  struct ecore_ptt *p_ptt,
 						  bool b_enable);
+
+/**
+ * @brief Whether FIP discovery fallback special mode is enabled or not.
+ *
+ * @param cdev
+ *
+ * @return true if device is in FIP special mode, false otherwise.
+ */
+bool ecore_is_mf_fip_special(struct ecore_dev *p_dev);
 #endif
diff --git a/drivers/net/qede/base/ecore_hsi_common.h b/drivers/net/qede/base/ecore_hsi_common.h
index 2d761b97..6d4a4dd7 100644
--- a/drivers/net/qede/base/ecore_hsi_common.h
+++ b/drivers/net/qede/base/ecore_hsi_common.h
@@ -922,7 +922,11 @@ struct core_rx_start_ramrod_data {
 	struct core_rx_action_on_error action_on_error;
 /* set when in GSI offload mode on ROCE connection */
 	u8 gsi_offload_flag;
-	u8 reserved[6];
+/* If set, the inner vlan (802.1q tag) priority that is written to cqe will be
+ * zero out, used for TenantDcb
+ */
+	u8 wipe_inner_vlan_pri_en;
+	u8 reserved[5];
 };
 
 
@@ -1044,7 +1048,11 @@ struct core_tx_start_ramrod_data {
 	__le16 qm_pq_id /* QM PQ ID */;
 /* set when in GSI offload mode on ROCE connection */
 	u8 gsi_offload_flag;
-	u8 resrved[3];
+/* vport id of the current connection, used to access non_rdma_in_to_in_pri_map
+ * which is per vport
+ */
+	u8 vport_id;
+	u8 resrved[2];
 };
 
 
@@ -1171,6 +1179,25 @@ struct eth_rx_rate_limit {
 };
 
 
+/* Update RSS indirection table entry command. One outstanding command supported
+ * per PF.
+ */
+struct eth_tstorm_rss_update_data {
+/* Valid flag. Driver must set this flag, FW clear valid flag when ready for new
+ * RSS update command.
+ */
+	u8 valid;
+/* Global VPORT ID. If RSS is disable for VPORT, RSS update command will be
+ * ignored.
+ */
+	u8 vport_id;
+	u8 ind_table_index /* RSS indirect table index that will be updated. */;
+	u8 reserved;
+	__le16 ind_table_value /* RSS indirect table new value. */;
+	__le16 reserved1 /* reserved. */;
+};
+
+
 struct eth_ustorm_per_pf_stat {
 /* number of total ucast bytes received on loopback port without errors */
 	struct regpair rcv_lb_ucast_bytes;
@@ -1463,6 +1490,10 @@ struct pf_start_tunnel_config {
  * FW will use a default port
  */
 	u8 set_geneve_udp_port_flg;
+/* Set no-innet-L2 VXLAN tunnel UDP destination port to
+ * no_inner_l2_vxlan_udp_port. If not set - FW will use a default port
+ */
+	u8 set_no_inner_l2_vxlan_udp_port_flg;
 	u8 tunnel_clss_vxlan /* Rx classification scheme for VXLAN tunnel. */;
 /* Rx classification scheme for l2 GENEVE tunnel. */
 	u8 tunnel_clss_l2geneve;
@@ -1470,11 +1501,15 @@ struct pf_start_tunnel_config {
 	u8 tunnel_clss_ipgeneve;
 	u8 tunnel_clss_l2gre /* Rx classification scheme for l2 GRE tunnel. */;
 	u8 tunnel_clss_ipgre /* Rx classification scheme for ip GRE tunnel. */;
-	u8 reserved;
 /* VXLAN tunnel UDP destination port. Valid if set_vxlan_udp_port_flg=1 */
 	__le16 vxlan_udp_port;
 /* GENEVE tunnel UDP destination port. Valid if set_geneve_udp_port_flg=1 */
 	__le16 geneve_udp_port;
+/* no-innet-L2 VXLAN  tunnel UDP destination port. Valid if
+ * set_no_inner_l2_vxlan_udp_port_flg=1
+ */
+	__le16 no_inner_l2_vxlan_udp_port;
+	__le16 reserved[3];
 };
 
 /*
@@ -1547,6 +1582,8 @@ struct pf_update_tunnel_config {
 	u8 set_vxlan_udp_port_flg;
 /* Update GENEVE tunnel UDP destination port. */
 	u8 set_geneve_udp_port_flg;
+/* Update no-innet-L2 VXLAN  tunnel UDP destination port. */
+	u8 set_no_inner_l2_vxlan_udp_port_flg;
 	u8 tunnel_clss_vxlan /* Classification scheme for VXLAN tunnel. */;
 /* Classification scheme for l2 GENEVE tunnel. */
 	u8 tunnel_clss_l2geneve;
@@ -1554,9 +1591,12 @@ struct pf_update_tunnel_config {
 	u8 tunnel_clss_ipgeneve;
 	u8 tunnel_clss_l2gre /* Classification scheme for l2 GRE tunnel. */;
 	u8 tunnel_clss_ipgre /* Classification scheme for ip GRE tunnel. */;
+	u8 reserved;
 	__le16 vxlan_udp_port /* VXLAN tunnel UDP destination port. */;
 	__le16 geneve_udp_port /* GENEVE tunnel UDP destination port. */;
-	__le16 reserved;
+/* no-innet-L2 VXLAN  tunnel UDP destination port. */
+	__le16 no_inner_l2_vxlan_udp_port;
+	__le16 reserved1[3];
 };
 
 /*
@@ -1686,6 +1726,13 @@ struct rl_update_ramrod_data {
 /* ID of last RL, that will be updated. If clear, single RL will updated. */
 	u8 rl_id_last;
 	u8 rl_dc_qcn_flg /* If set, RL will used for DCQCN. */;
+/* If set, alpha will be reset to 1 when the state machine is idle. */
+	u8 dcqcn_reset_alpha_on_idle;
+/* Byte counter threshold to change rate increase stage. */
+	u8 rl_bc_stage_th;
+/* Timer threshold to change rate increase stage. */
+	u8 rl_timer_stage_th;
+	u8 reserved1;
 	__le32 rl_bc_rate /* Byte Counter Limit. */;
 	__le16 rl_max_rate /* Maximum rate in 1.6 Mbps resolution. */;
 	__le16 rl_r_ai /* Active increase rate. */;
@@ -1694,7 +1741,7 @@ struct rl_update_ramrod_data {
 	__le32 dcqcn_k_us /* DCQCN Alpha update interval. */;
 	__le32 dcqcn_timeuot_us /* DCQCN timeout. */;
 	__le32 qcn_timeuot_us /* QCN timeout. */;
-	__le32 reserved[2];
+	__le32 reserved2;
 };
 
 
diff --git a/drivers/net/qede/base/ecore_hsi_debug_tools.h b/drivers/net/qede/base/ecore_hsi_debug_tools.h
index bf548722..085af0a3 100644
--- a/drivers/net/qede/base/ecore_hsi_debug_tools.h
+++ b/drivers/net/qede/base/ecore_hsi_debug_tools.h
@@ -1091,6 +1091,15 @@ struct idle_chk_data {
 };
 
 /*
+ * Pretend parameters
+ */
+struct pretend_params {
+	u8 split_type /* Pretend split type (from enum init_split_types) */;
+	u8 reserved;
+	u16 split_id /* Preted split ID (within the pretend split type) */;
+};
+
+/*
  * Debug Tools data (per HW function)
  */
 struct dbg_tools_data {
@@ -1102,11 +1111,17 @@ struct dbg_tools_data {
 	u8 block_in_reset[88];
 	u8 chip_id /* Chip ID (from enum chip_ids) */;
 	u8 platform_id /* Platform ID */;
+	u8 num_ports /* Number of ports in the chip */;
+	u8 num_pfs_per_port /* Number of PFs in each port */;
+	u8 num_vfs /* Number of VFs in the chip */;
 	u8 initialized /* Indicates if the data was initialized */;
 	u8 use_dmae /* Indicates if DMAE should be used */;
+	u8 reserved;
+	struct pretend_params pretend /* Current pretend parameters */;
 /* Numbers of registers that were read since last log */
 	u32 num_regs_read;
 };
 
 
+
 #endif /* __ECORE_HSI_DEBUG_TOOLS__ */
diff --git a/drivers/net/qede/base/ecore_hsi_eth.h b/drivers/net/qede/base/ecore_hsi_eth.h
index 6b512305..158ca673 100644
--- a/drivers/net/qede/base/ecore_hsi_eth.h
+++ b/drivers/net/qede/base/ecore_hsi_eth.h
@@ -832,6 +832,26 @@ enum eth_filter_type {
 
 
 /*
+ * inner to inner vlan priority translation configurations
+ */
+struct eth_in_to_in_pri_map_cfg {
+/* If set, non_rdma_in_to_in_pri_map or rdma_in_to_in_pri_map will be used for
+ * inner to inner priority mapping depending on protocol type
+ */
+	u8 inner_vlan_pri_remap_en;
+	u8 reserved[7];
+/* Map for inner to inner vlan priority translation for Non RDMA protocols, used
+ * for TenantDcb. Set inner_vlan_pri_remap_en, when init the map.
+ */
+	u8 non_rdma_in_to_in_pri_map[8];
+/* Map for inner to inner vlan priority translation for RDMA protocols, used for
+ * TenantDcb. Set inner_vlan_pri_remap_en, when init the map.
+ */
+	u8 rdma_in_to_in_pri_map[8];
+};
+
+
+/*
  * eth IPv4 Fragment Type
  */
 enum eth_ipv4_frag_type {
@@ -1030,8 +1050,11 @@ struct eth_vport_rx_mode {
 /* accept all broadcast packets (subject to vlan) */
 #define ETH_VPORT_RX_MODE_BCAST_ACCEPT_ALL_MASK        0x1
 #define ETH_VPORT_RX_MODE_BCAST_ACCEPT_ALL_SHIFT       5
-#define ETH_VPORT_RX_MODE_RESERVED1_MASK               0x3FF
-#define ETH_VPORT_RX_MODE_RESERVED1_SHIFT              6
+/* accept any VNI in tunnel VNI classification. Used for default queue. */
+#define ETH_VPORT_RX_MODE_ACCEPT_ANY_VNI_MASK          0x1
+#define ETH_VPORT_RX_MODE_ACCEPT_ANY_VNI_SHIFT         6
+#define ETH_VPORT_RX_MODE_RESERVED1_MASK               0x1FF
+#define ETH_VPORT_RX_MODE_RESERVED1_SHIFT              7
 };
 
 
@@ -1357,6 +1380,20 @@ struct tx_queue_update_ramrod_data {
 };
 
 
+/*
+ * Inner to Inner VLAN priority map update mode
+ */
+enum update_in_to_in_pri_map_mode_enum {
+/* Inner to Inner VLAN priority map update Disabled */
+	ETH_IN_TO_IN_PRI_MAP_UPDATE_DISABLED,
+/* Update Inner to Inner VLAN priority map for non RDMA protocols */
+	ETH_IN_TO_IN_PRI_MAP_UPDATE_NON_RDMA_TBL,
+/* Update Inner to Inner VLAN priority map for RDMA protocols */
+	ETH_IN_TO_IN_PRI_MAP_UPDATE_RDMA_TBL,
+	MAX_UPDATE_IN_TO_IN_PRI_MAP_MODE_ENUM
+};
+
+
 
 /*
  * Ramrod data for vport update ramrod
@@ -1405,7 +1442,12 @@ struct vport_start_ramrod_data {
 	u8 ctl_frame_mac_check_en;
 /* If set, control frames will be filtered according to ethtype check. */
 	u8 ctl_frame_ethtype_check_en;
-	u8 reserved[1];
+/* If set, the inner vlan (802.1q tag) priority that is written to cqe will be
+ * zero out, used for TenantDcb
+ */
+	u8 wipe_inner_vlan_pri_en;
+/* inner to inner vlan priority translation configurations */
+	struct eth_in_to_in_pri_map_cfg in_to_in_vlan_pri_map_cfg;
 };
 
 
@@ -1473,7 +1515,14 @@ struct vport_update_ramrod_data_cmn {
 	u8 ctl_frame_mac_check_en;
 /* If set, control frames will be filtered according to ethtype check. */
 	u8 ctl_frame_ethtype_check_en;
-	u8 reserved[15];
+/* Indicates to update RDMA or NON-RDMA vlan remapping priority table according
+ * to update_in_to_in_pri_map_mode_enum, used for TenantDcb (use enum
+ * update_in_to_in_pri_map_mode_enum)
+ */
+	u8 update_in_to_in_pri_map_mode;
+/* Map for inner to inner vlan priority translation, used for TenantDcb. */
+	u8 in_to_in_pri_map[8];
+	u8 reserved[6];
 };
 
 struct vport_update_ramrod_mcast {
diff --git a/drivers/net/qede/base/ecore_hw.c b/drivers/net/qede/base/ecore_hw.c
index 51bba27e..72cd7e9c 100644
--- a/drivers/net/qede/base/ecore_hw.c
+++ b/drivers/net/qede/base/ecore_hw.c
@@ -407,6 +407,30 @@ void ecore_port_unpretend(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
 			*(u32 *)&p_ptt->pxp.pretend);
 }
 
+void ecore_port_fid_pretend(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			    u8 port_id, u16 fid)
+{
+	u16 control = 0;
+
+	SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id);
+	SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1);
+	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
+
+	SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1);
+	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1);
+
+	if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID))
+		fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID);
+
+	p_ptt->pxp.pretend.control = OSAL_CPU_TO_LE16(control);
+	p_ptt->pxp.pretend.fid.concrete_fid.fid = OSAL_CPU_TO_LE16(fid);
+
+	REG_WR(p_hwfn,
+	       ecore_ptt_config_addr(p_ptt) +
+	       OFFSETOF(struct pxp_ptt_entry, pretend),
+	       *(u32 *)&p_ptt->pxp.pretend);
+}
+
 u32 ecore_vfid_to_concrete(struct ecore_hwfn *p_hwfn, u8 vfid)
 {
 	u32 concrete_fid = 0;
@@ -426,14 +450,17 @@ u32 ecore_vfid_to_concrete(struct ecore_hwfn *p_hwfn, u8 vfid)
  * If this changes, this needs to be revisted.
  */
 
-/* Ecore DMAE
- * =============
- */
+/* DMAE */
+
+#define ECORE_DMAE_FLAGS_IS_SET(params, flag)	\
+	((params) != OSAL_NULL && ((params)->flags & ECORE_DMAE_FLAG_##flag))
+
 static void ecore_dmae_opcode(struct ecore_hwfn *p_hwfn,
 			      const u8 is_src_type_grc,
 			      const u8 is_dst_type_grc,
 			      struct ecore_dmae_params *p_params)
 {
+	u8 src_pfid, dst_pfid, port_id;
 	u16 opcode_b = 0;
 	u32 opcode = 0;
 
@@ -443,16 +470,20 @@ static void ecore_dmae_opcode(struct ecore_hwfn *p_hwfn,
 	 */
 	opcode |= (is_src_type_grc ? DMAE_CMD_SRC_MASK_GRC
 		   : DMAE_CMD_SRC_MASK_PCIE) << DMAE_CMD_SRC_SHIFT;
-	opcode |= (p_hwfn->rel_pf_id & DMAE_CMD_SRC_PF_ID_MASK) <<
-	    DMAE_CMD_SRC_PF_ID_SHIFT;
+	src_pfid = ECORE_DMAE_FLAGS_IS_SET(p_params, PF_SRC) ?
+		   p_params->src_pfid : p_hwfn->rel_pf_id;
+	opcode |= (src_pfid & DMAE_CMD_SRC_PF_ID_MASK) <<
+		  DMAE_CMD_SRC_PF_ID_SHIFT;
 
 	/* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */
 	opcode |= (is_dst_type_grc ? DMAE_CMD_DST_MASK_GRC
 		   : DMAE_CMD_DST_MASK_PCIE) << DMAE_CMD_DST_SHIFT;
-	opcode |= (p_hwfn->rel_pf_id & DMAE_CMD_DST_PF_ID_MASK) <<
-	    DMAE_CMD_DST_PF_ID_SHIFT;
+	dst_pfid = ECORE_DMAE_FLAGS_IS_SET(p_params, PF_DST) ?
+		   p_params->dst_pfid : p_hwfn->rel_pf_id;
+	opcode |= (dst_pfid & DMAE_CMD_DST_PF_ID_MASK) <<
+		  DMAE_CMD_DST_PF_ID_SHIFT;
 
-	/* DMAE_E4_TODO need to check which value to specifiy here. */
+	/* DMAE_E4_TODO need to check which value to specify here. */
 	/* opcode |= (!b_complete_to_host)<< DMAE_CMD_C_DST_SHIFT; */
 
 	/* Whether to write a completion word to the completion destination:
@@ -462,7 +493,7 @@ static void ecore_dmae_opcode(struct ecore_hwfn *p_hwfn,
 	opcode |= DMAE_CMD_COMP_WORD_EN_MASK << DMAE_CMD_COMP_WORD_EN_SHIFT;
 	opcode |= DMAE_CMD_SRC_ADDR_RESET_MASK << DMAE_CMD_SRC_ADDR_RESET_SHIFT;
 
-	if (p_params->flags & ECORE_DMAE_FLAG_COMPLETION_DST)
+	if (ECORE_DMAE_FLAGS_IS_SET(p_params, COMPLETION_DST))
 		opcode |= 1 << DMAE_CMD_COMP_FUNC_SHIFT;
 
 	/* swapping mode 3 - big endian there should be a define ifdefed in
@@ -470,7 +501,9 @@ static void ecore_dmae_opcode(struct ecore_hwfn *p_hwfn,
 	 */
 	opcode |= DMAE_CMD_ENDIANITY << DMAE_CMD_ENDIANITY_MODE_SHIFT;
 
-	opcode |= p_hwfn->port_id << DMAE_CMD_PORT_ID_SHIFT;
+	port_id = (ECORE_DMAE_FLAGS_IS_SET(p_params, PORT)) ?
+		  p_params->port_id : p_hwfn->port_id;
+	opcode |= port_id << DMAE_CMD_PORT_ID_SHIFT;
 
 	/* reset source address in next go */
 	opcode |= DMAE_CMD_SRC_ADDR_RESET_MASK << DMAE_CMD_SRC_ADDR_RESET_SHIFT;
@@ -479,14 +512,14 @@ static void ecore_dmae_opcode(struct ecore_hwfn *p_hwfn,
 	opcode |= DMAE_CMD_DST_ADDR_RESET_MASK << DMAE_CMD_DST_ADDR_RESET_SHIFT;
 
 	/* SRC/DST VFID: all 1's - pf, otherwise VF id */
-	if (p_params->flags & ECORE_DMAE_FLAG_VF_SRC) {
+	if (ECORE_DMAE_FLAGS_IS_SET(p_params, VF_SRC)) {
 		opcode |= (1 << DMAE_CMD_SRC_VF_ID_VALID_SHIFT);
 		opcode_b |= (p_params->src_vfid << DMAE_CMD_SRC_VF_ID_SHIFT);
 	} else {
 		opcode_b |= (DMAE_CMD_SRC_VF_ID_MASK <<
 			     DMAE_CMD_SRC_VF_ID_SHIFT);
 	}
-	if (p_params->flags & ECORE_DMAE_FLAG_VF_DST) {
+	if (ECORE_DMAE_FLAGS_IS_SET(p_params, VF_DST)) {
 		opcode |= 1 << DMAE_CMD_DST_VF_ID_VALID_SHIFT;
 		opcode_b |= p_params->dst_vfid << DMAE_CMD_DST_VF_ID_SHIFT;
 	} else {
@@ -831,7 +864,7 @@ ecore_dmae_execute_command(struct ecore_hwfn *p_hwfn,
 	for (i = 0; i <= cnt_split; i++) {
 		offset = length_limit * i;
 
-		if (!(p_params->flags & ECORE_DMAE_FLAG_RW_REPL_SRC)) {
+		if (!ECORE_DMAE_FLAGS_IS_SET(p_params, RW_REPL_SRC)) {
 			if (src_type == ECORE_DMAE_ADDRESS_GRC)
 				src_addr_split = src_addr + offset;
 			else
@@ -872,51 +905,45 @@ ecore_dmae_execute_command(struct ecore_hwfn *p_hwfn,
 	return ecore_status;
 }
 
-enum _ecore_status_t
-ecore_dmae_host2grc(struct ecore_hwfn *p_hwfn,
-		    struct ecore_ptt *p_ptt,
-		    u64 source_addr,
-		    u32 grc_addr, u32 size_in_dwords, u32 flags)
+enum _ecore_status_t ecore_dmae_host2grc(struct ecore_hwfn *p_hwfn,
+					 struct ecore_ptt *p_ptt,
+					 u64 source_addr,
+					 u32 grc_addr,
+					 u32 size_in_dwords,
+					 struct ecore_dmae_params *p_params)
 {
 	u32 grc_addr_in_dw = grc_addr / sizeof(u32);
-	struct ecore_dmae_params params;
 	enum _ecore_status_t rc;
 
-	OSAL_MEMSET(&params, 0, sizeof(struct ecore_dmae_params));
-	params.flags = flags;
-
 	OSAL_SPIN_LOCK(&p_hwfn->dmae_info.lock);
 
 	rc = ecore_dmae_execute_command(p_hwfn, p_ptt, source_addr,
 					grc_addr_in_dw,
 					ECORE_DMAE_ADDRESS_HOST_VIRT,
 					ECORE_DMAE_ADDRESS_GRC,
-					size_in_dwords, &params);
+					size_in_dwords, p_params);
 
 	OSAL_SPIN_UNLOCK(&p_hwfn->dmae_info.lock);
 
 	return rc;
 }
 
-enum _ecore_status_t
-ecore_dmae_grc2host(struct ecore_hwfn *p_hwfn,
-		    struct ecore_ptt *p_ptt,
-		    u32 grc_addr,
-		    dma_addr_t dest_addr, u32 size_in_dwords, u32 flags)
+enum _ecore_status_t ecore_dmae_grc2host(struct ecore_hwfn *p_hwfn,
+					 struct ecore_ptt *p_ptt,
+					 u32 grc_addr,
+					 dma_addr_t dest_addr,
+					 u32 size_in_dwords,
+					 struct ecore_dmae_params *p_params)
 {
 	u32 grc_addr_in_dw = grc_addr / sizeof(u32);
-	struct ecore_dmae_params params;
 	enum _ecore_status_t rc;
 
-	OSAL_MEMSET(&params, 0, sizeof(struct ecore_dmae_params));
-	params.flags = flags;
-
 	OSAL_SPIN_LOCK(&p_hwfn->dmae_info.lock);
 
 	rc = ecore_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw,
 					dest_addr, ECORE_DMAE_ADDRESS_GRC,
 					ECORE_DMAE_ADDRESS_HOST_VIRT,
-					size_in_dwords, &params);
+					size_in_dwords, p_params);
 
 	OSAL_SPIN_UNLOCK(&p_hwfn->dmae_info.lock);
 
@@ -965,7 +992,6 @@ enum _ecore_status_t ecore_dmae_sanity(struct ecore_hwfn *p_hwfn,
 				       const char *phase)
 {
 	u32 size = OSAL_PAGE_SIZE / 2, val;
-	struct ecore_dmae_params params;
 	enum _ecore_status_t rc = ECORE_SUCCESS;
 	dma_addr_t p_phys;
 	void *p_virt;
@@ -997,9 +1023,9 @@ enum _ecore_status_t ecore_dmae_sanity(struct ecore_hwfn *p_hwfn,
 		   (unsigned long)(p_phys + size),
 		   (u8 *)p_virt + size, size);
 
-	OSAL_MEMSET(&params, 0, sizeof(params));
 	rc = ecore_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size,
-				  size / 4 /* size_in_dwords */, &params);
+				  size / 4 /* size_in_dwords */,
+				  OSAL_NULL /* default parameters */);
 	if (rc != ECORE_SUCCESS) {
 		DP_NOTICE(p_hwfn, false,
 			  "DMAE sanity [%s]: ecore_dmae_host2host() failed. rc = %d.\n",
@@ -1030,3 +1056,32 @@ out:
 	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_virt, p_phys, 2 * size);
 	return rc;
 }
+
+void ecore_ppfid_wr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+		    u8 abs_ppfid, u32 hw_addr, u32 val)
+{
+	u8 pfid = ECORE_PFID_BY_PPFID(p_hwfn, abs_ppfid);
+
+	ecore_fid_pretend(p_hwfn, p_ptt,
+			  pfid << PXP_PRETEND_CONCRETE_FID_PFID_SHIFT);
+	ecore_wr(p_hwfn, p_ptt, hw_addr, val);
+	ecore_fid_pretend(p_hwfn, p_ptt,
+			  p_hwfn->rel_pf_id <<
+			  PXP_PRETEND_CONCRETE_FID_PFID_SHIFT);
+}
+
+u32 ecore_ppfid_rd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+		   u8 abs_ppfid, u32 hw_addr)
+{
+	u8 pfid = ECORE_PFID_BY_PPFID(p_hwfn, abs_ppfid);
+	u32 val;
+
+	ecore_fid_pretend(p_hwfn, p_ptt,
+			  pfid << PXP_PRETEND_CONCRETE_FID_PFID_SHIFT);
+	val = ecore_rd(p_hwfn, p_ptt, hw_addr);
+	ecore_fid_pretend(p_hwfn, p_ptt,
+			  p_hwfn->rel_pf_id <<
+			  PXP_PRETEND_CONCRETE_FID_PFID_SHIFT);
+
+	return val;
+}
diff --git a/drivers/net/qede/base/ecore_hw.h b/drivers/net/qede/base/ecore_hw.h
index 394207eb..0b5b40c4 100644
--- a/drivers/net/qede/base/ecore_hw.h
+++ b/drivers/net/qede/base/ecore_hw.h
@@ -134,8 +134,8 @@ struct ecore_ptt *ecore_get_reserved_ptt(struct ecore_hwfn	*p_hwfn,
  *
  * @param p_hwfn
  * @param p_ptt
- * @param val
  * @param hw_addr
+ * @param val
  */
 void ecore_wr(struct ecore_hwfn	*p_hwfn,
 	      struct ecore_ptt	*p_ptt,
@@ -147,7 +147,6 @@ void ecore_wr(struct ecore_hwfn	*p_hwfn,
  *
  * @param p_hwfn
  * @param p_ptt
- * @param val
  * @param hw_addr
  */
 u32 ecore_rd(struct ecore_hwfn	*p_hwfn,
@@ -223,6 +222,18 @@ void ecore_port_unpretend(struct ecore_hwfn	*p_hwfn,
 			  struct ecore_ptt	*p_ptt);
 
 /**
+ * @brief ecore_port_fid_pretend - pretend to another port and another function
+ *        when accessing the ptt window
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ * @param port_id - the port to pretend to
+ * @param fid - fid field of pxp_pretend structure. Can contain either pf / vf.
+ */
+void ecore_port_fid_pretend(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			    u8 port_id, u16 fid);
+
+/**
  * @brief ecore_vfid_to_concrete - build a concrete FID for a
  *        given VF ID
  *
@@ -257,4 +268,29 @@ enum _ecore_status_t ecore_dmae_sanity(struct ecore_hwfn *p_hwfn,
 				       struct ecore_ptt *p_ptt,
 				       const char *phase);
 
+/**
+ * @brief ecore_ppfid_wr - Write value to BAR using the given ptt while
+ *	pretending to a PF to which the given PPFID pertains.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ * @param abs_ppfid
+ * @param hw_addr
+ * @param val
+ */
+void ecore_ppfid_wr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+		    u8 abs_ppfid, u32 hw_addr, u32 val);
+
+/**
+ * @brief ecore_ppfid_rd - Read value from BAR using the given ptt while
+ *	 pretending to a PF to which the given PPFID pertains.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ * @param abs_ppfid
+ * @param hw_addr
+ */
+u32 ecore_ppfid_rd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+		   u8 abs_ppfid, u32 hw_addr);
+
 #endif /* __ECORE_HW_H__ */
diff --git a/drivers/net/qede/base/ecore_init_fw_funcs.c b/drivers/net/qede/base/ecore_init_fw_funcs.c
index b8496cb2..cfc1156e 100644
--- a/drivers/net/qede/base/ecore_init_fw_funcs.c
+++ b/drivers/net/qede/base/ecore_init_fw_funcs.c
@@ -1665,7 +1665,7 @@ void ecore_gft_config(struct ecore_hwfn *p_hwfn,
 			       bool ipv6,
 			       enum gft_profile_type profile_type)
 {
-	u32 reg_val, cam_line, ram_line_lo, ram_line_hi;
+	u32 reg_val, cam_line, ram_line_lo, ram_line_hi, search_non_ip_as_gft;
 
 	if (!ipv6 && !ipv4)
 		DP_NOTICE(p_hwfn, true, "gft_config: must accept at least on of - ipv4 or ipv6'\n");
@@ -1729,6 +1729,9 @@ void ecore_gft_config(struct ecore_hwfn *p_hwfn,
 	ram_line_lo = 0;
 	ram_line_hi = 0;
 
+	/* Search no IP as GFT */
+	search_non_ip_as_gft = 0;
+
 	/* Tunnel type */
 	SET_FIELD(ram_line_lo, GFT_RAM_LINE_TUNNEL_DST_PORT, 1);
 	SET_FIELD(ram_line_lo, GFT_RAM_LINE_TUNNEL_OVER_IP_PROTOCOL, 1);
@@ -1752,8 +1755,13 @@ void ecore_gft_config(struct ecore_hwfn *p_hwfn,
 		SET_FIELD(ram_line_lo, GFT_RAM_LINE_ETHERTYPE, 1);
 	} else if (profile_type == GFT_PROFILE_TYPE_TUNNEL_TYPE) {
 		SET_FIELD(ram_line_lo, GFT_RAM_LINE_TUNNEL_ETHERTYPE, 1);
+
+		/* Allow tunneled traffic without inner IP */
+		search_non_ip_as_gft = 1;
 	}
 
+	ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_NON_IP_AS_GFT,
+		 search_non_ip_as_gft);
 	ecore_wr(p_hwfn, p_ptt,
 		 PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE * pf_id,
 		 ram_line_lo);
@@ -1996,52 +2004,49 @@ void ecore_enable_context_validation(struct ecore_hwfn *p_hwfn,
 	ecore_wr(p_hwfn, p_ptt, CDU_REG_TCFC_CTX_VALID0, ctx_validation);
 }
 
-#define RSS_IND_TABLE_BASE_ADDR       4112
-#define RSS_IND_TABLE_VPORT_SIZE      16
-#define RSS_IND_TABLE_ENTRY_PER_LINE  8
 
-/* Update RSS indirection table entry. */
-void ecore_update_eth_rss_ind_table_entry(struct ecore_hwfn *p_hwfn,
-					  struct ecore_ptt *p_ptt,
-					  u8 rss_id,
-					  u8 ind_table_index,
-					  u16 ind_table_value)
+/*******************************************************************************
+ * File name : rdma_init.c
+ * Author    : Michael Shteinbok
+ *******************************************************************************
+ *******************************************************************************
+ * Description:
+ * RDMA HSI functions
+ *
+ *******************************************************************************
+ * Notes: This is the input to the auto generated file drv_init_fw_funcs.c
+ *
+ *******************************************************************************
+ */
+static u32 ecore_get_rdma_assert_ram_addr(struct ecore_hwfn *p_hwfn,
+					  u8 storm_id)
 {
-	u32 cnt, rss_addr;
-	u32 *reg_val;
-	u16 rss_ind_entry[RSS_IND_TABLE_ENTRY_PER_LINE];
-	u16 rss_ind_mask[RSS_IND_TABLE_ENTRY_PER_LINE];
-
-	/* get entry address */
-	rss_addr =  RSS_IND_TABLE_BASE_ADDR +
-		    RSS_IND_TABLE_VPORT_SIZE * rss_id +
-		    ind_table_index / RSS_IND_TABLE_ENTRY_PER_LINE;
-
-	/* prepare update command */
-	ind_table_index %= RSS_IND_TABLE_ENTRY_PER_LINE;
-
-	for (cnt = 0; cnt < RSS_IND_TABLE_ENTRY_PER_LINE; cnt++) {
-		if (cnt == ind_table_index) {
-			rss_ind_entry[cnt] = ind_table_value;
-			rss_ind_mask[cnt]  = 0xFFFF;
-		} else {
-			rss_ind_entry[cnt] = 0;
-			rss_ind_mask[cnt]  = 0;
-		}
+	switch (storm_id) {
+	case 0: return TSEM_REG_FAST_MEMORY + SEM_FAST_REG_INT_RAM +
+		       TSTORM_RDMA_ASSERT_LEVEL_OFFSET(p_hwfn->rel_pf_id);
+	case 1: return MSEM_REG_FAST_MEMORY + SEM_FAST_REG_INT_RAM +
+		       MSTORM_RDMA_ASSERT_LEVEL_OFFSET(p_hwfn->rel_pf_id);
+	case 2: return USEM_REG_FAST_MEMORY + SEM_FAST_REG_INT_RAM +
+		       USTORM_RDMA_ASSERT_LEVEL_OFFSET(p_hwfn->rel_pf_id);
+	case 3: return XSEM_REG_FAST_MEMORY + SEM_FAST_REG_INT_RAM +
+		       XSTORM_RDMA_ASSERT_LEVEL_OFFSET(p_hwfn->rel_pf_id);
+	case 4: return YSEM_REG_FAST_MEMORY + SEM_FAST_REG_INT_RAM +
+		       YSTORM_RDMA_ASSERT_LEVEL_OFFSET(p_hwfn->rel_pf_id);
+	case 5: return PSEM_REG_FAST_MEMORY + SEM_FAST_REG_INT_RAM +
+		       PSTORM_RDMA_ASSERT_LEVEL_OFFSET(p_hwfn->rel_pf_id);
+
+	default: return 0;
 	}
+}
 
-	/* Update entry in HW*/
-	ecore_wr(p_hwfn, p_ptt, RSS_REG_RSS_RAM_ADDR, rss_addr);
-
-	reg_val = (u32 *)rss_ind_mask;
-	ecore_wr(p_hwfn, p_ptt, RSS_REG_RSS_RAM_MASK, reg_val[0]);
-	ecore_wr(p_hwfn, p_ptt, RSS_REG_RSS_RAM_MASK + 4, reg_val[1]);
-	ecore_wr(p_hwfn, p_ptt, RSS_REG_RSS_RAM_MASK + 8, reg_val[2]);
-	ecore_wr(p_hwfn, p_ptt, RSS_REG_RSS_RAM_MASK + 12, reg_val[3]);
+void ecore_set_rdma_error_level(struct ecore_hwfn *p_hwfn,
+				struct ecore_ptt *p_ptt,
+				u8 assert_level[NUM_STORMS])
+{
+	u8 storm_id;
+	for (storm_id = 0; storm_id < NUM_STORMS; storm_id++) {
+		u32 ram_addr = ecore_get_rdma_assert_ram_addr(p_hwfn, storm_id);
 
-	reg_val = (u32 *)rss_ind_entry;
-	ecore_wr(p_hwfn, p_ptt, RSS_REG_RSS_RAM_DATA, reg_val[0]);
-	ecore_wr(p_hwfn, p_ptt, RSS_REG_RSS_RAM_DATA + 4, reg_val[1]);
-	ecore_wr(p_hwfn, p_ptt, RSS_REG_RSS_RAM_DATA + 8, reg_val[2]);
-	ecore_wr(p_hwfn, p_ptt, RSS_REG_RSS_RAM_DATA + 12, reg_val[3]);
+		ecore_wr(p_hwfn, p_ptt, ram_addr, assert_level[storm_id]);
+	}
 }
diff --git a/drivers/net/qede/base/ecore_init_fw_funcs.h b/drivers/net/qede/base/ecore_init_fw_funcs.h
index 1024bb26..3503a90c 100644
--- a/drivers/net/qede/base/ecore_init_fw_funcs.h
+++ b/drivers/net/qede/base/ecore_init_fw_funcs.h
@@ -472,21 +472,35 @@ void ecore_memset_task_ctx(void *p_ctx_mem,
 			   u32 ctx_size,
 			   u8 ctx_type);
 
-/**
- * @brief ecore_update_eth_rss_ind_table_entry - Update RSS indirection table
- * entry.
- * The function must run in exclusive mode to prevent wrong RSS configuration.
+
+/*******************************************************************************
+ * File name : rdma_init.h
+ * Author    : Michael Shteinbok
+ *******************************************************************************
+ *******************************************************************************
+ * Description:
+ * RDMA HSI functions header
+ *
+ *******************************************************************************
+ * Notes: This is the input to the auto generated file drv_init_fw_funcs.h
  *
- * @param p_hwfn    - HW device data
- * @param p_ptt  - ptt window used for writing the registers.
- * @param rss_id - RSS engine ID.
- * @param ind_table_index -  RSS indirect table index.
- * @param ind_table_value -  RSS indirect table new value.
+ *******************************************************************************
  */
-void ecore_update_eth_rss_ind_table_entry(struct ecore_hwfn *p_hwfn,
-					  struct ecore_ptt *p_ptt,
-					  u8 rss_id,
-					  u8 ind_table_index,
-					  u16 ind_table_value);
+#define NUM_STORMS 6
+
+
+
+/**
+ * @brief ecore_set_rdma_error_level - Sets the RDMA assert level.
+ *                                     If the severity of the error will be
+ *				       above the level, the FW will assert.
+ * @param p_hwfn -		   HW device data
+ * @param p_ptt -		   ptt window used for writing the registers
+ * @param assert_level - An array of assert levels for each storm.
+ */
+void ecore_set_rdma_error_level(struct ecore_hwfn *p_hwfn,
+				struct ecore_ptt *p_ptt,
+				u8 assert_level[NUM_STORMS]);
+
 
 #endif
diff --git a/drivers/net/qede/base/ecore_init_ops.c b/drivers/net/qede/base/ecore_init_ops.c
index b7636f36..044308bf 100644
--- a/drivers/net/qede/base/ecore_init_ops.c
+++ b/drivers/net/qede/base/ecore_init_ops.c
@@ -101,7 +101,8 @@ static enum _ecore_status_t ecore_init_rt(struct ecore_hwfn *p_hwfn,
 
 		rc = ecore_dmae_host2grc(p_hwfn, p_ptt,
 					 (osal_uintptr_t)(p_init_val + i),
-					 addr + (i << 2), segment, 0);
+					 addr + (i << 2), segment,
+					 OSAL_NULL /* default parameters */);
 		if (rc != ECORE_SUCCESS)
 			return rc;
 
@@ -165,8 +166,9 @@ static enum _ecore_status_t ecore_init_array_dmae(struct ecore_hwfn *p_hwfn,
 	} else {
 		rc = ecore_dmae_host2grc(p_hwfn, p_ptt,
 					 (osal_uintptr_t)(p_buf +
-							   dmae_data_offset),
-					 addr, size, 0);
+							  dmae_data_offset),
+					 addr, size,
+					 OSAL_NULL /* default parameters */);
 	}
 
 	return rc;
@@ -177,13 +179,15 @@ static enum _ecore_status_t ecore_init_fill_dmae(struct ecore_hwfn *p_hwfn,
 						 u32 addr, u32 fill_count)
 {
 	static u32 zero_buffer[DMAE_MAX_RW_SIZE];
+	struct ecore_dmae_params params;
 
 	OSAL_MEMSET(zero_buffer, 0, sizeof(u32) * DMAE_MAX_RW_SIZE);
 
+	OSAL_MEMSET(&params, 0, sizeof(params));
+	params.flags = ECORE_DMAE_FLAG_RW_REPL_SRC;
 	return ecore_dmae_host2grc(p_hwfn, p_ptt,
 				   (osal_uintptr_t)&zero_buffer[0],
-				   addr, fill_count,
-				   ECORE_DMAE_FLAG_RW_REPL_SRC);
+				   addr, fill_count, &params);
 }
 
 static void ecore_init_fill(struct ecore_hwfn *p_hwfn,
@@ -416,11 +420,11 @@ static u8 ecore_init_cmd_mode_match(struct ecore_hwfn *p_hwfn,
 				    u16 *p_offset, int modes)
 {
 	struct ecore_dev *p_dev = p_hwfn->p_dev;
-	const u8 *modes_tree_buf;
 	u8 arg1, arg2, tree_val;
+	const u8 *modes_tree;
 
-	modes_tree_buf = p_dev->fw_data->modes_tree_buf;
-	tree_val = modes_tree_buf[(*p_offset)++];
+	modes_tree = p_dev->fw_data->modes_tree_buf;
+	tree_val = modes_tree[(*p_offset)++];
 	switch (tree_val) {
 	case INIT_MODE_OP_NOT:
 		return ecore_init_cmd_mode_match(p_hwfn, p_offset, modes) ^ 1;
@@ -470,12 +474,12 @@ enum _ecore_status_t ecore_init_run(struct ecore_hwfn *p_hwfn,
 {
 	struct ecore_dev *p_dev = p_hwfn->p_dev;
 	u32 cmd_num, num_init_ops;
-	union init_op *init_ops;
+	union init_op *init;
 	bool b_dmae = false;
 	enum _ecore_status_t rc = ECORE_SUCCESS;
 
 	num_init_ops = p_dev->fw_data->init_ops_size;
-	init_ops = p_dev->fw_data->init_ops;
+	init = p_dev->fw_data->init_ops;
 
 #ifdef CONFIG_ECORE_ZIPPED_FW
 	p_hwfn->unzip_buf = OSAL_ZALLOC(p_hwfn->p_dev, GFP_ATOMIC,
@@ -487,7 +491,7 @@ enum _ecore_status_t ecore_init_run(struct ecore_hwfn *p_hwfn,
 #endif
 
 	for (cmd_num = 0; cmd_num < num_init_ops; cmd_num++) {
-		union init_op *cmd = &init_ops[cmd_num];
+		union init_op *cmd = &init[cmd_num];
 		u32 data = OSAL_LE32_TO_CPU(cmd->raw.op_data);
 
 		switch (GET_FIELD(data, INIT_CALLBACK_OP_OP)) {
diff --git a/drivers/net/qede/base/ecore_int.c b/drivers/net/qede/base/ecore_int.c
index 4c271d35..7368d55f 100644
--- a/drivers/net/qede/base/ecore_int.c
+++ b/drivers/net/qede/base/ecore_int.c
@@ -428,14 +428,13 @@ ecore_general_attention_35(struct ecore_hwfn *p_hwfn)
 #define ECORE_DORQ_ATTENTION_SIZE_MASK		(0x7f)
 #define ECORE_DORQ_ATTENTION_SIZE_SHIFT		(16)
 
-#define ECORE_DB_REC_COUNT			10
+#define ECORE_DB_REC_COUNT			1000
 #define ECORE_DB_REC_INTERVAL			100
 
-/* assumes sticky overflow indication was set for this PF */
-static enum _ecore_status_t ecore_db_rec_attn(struct ecore_hwfn *p_hwfn,
-					      struct ecore_ptt *p_ptt)
+static enum _ecore_status_t ecore_db_rec_flush_queue(struct ecore_hwfn *p_hwfn,
+						     struct ecore_ptt *p_ptt)
 {
-	u8 count = ECORE_DB_REC_COUNT;
+	u32 count = ECORE_DB_REC_COUNT;
 	u32 usage = 1;
 
 	/* wait for usage to zero or count to run out. This is necessary since
@@ -461,6 +460,28 @@ static enum _ecore_status_t ecore_db_rec_attn(struct ecore_hwfn *p_hwfn,
 		return ECORE_TIMEOUT;
 	}
 
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t ecore_db_rec_handler(struct ecore_hwfn *p_hwfn,
+					  struct ecore_ptt *p_ptt)
+{
+	u32 overflow;
+	enum _ecore_status_t rc;
+
+	overflow = ecore_rd(p_hwfn, p_ptt, DORQ_REG_PF_OVFL_STICKY);
+	DP_NOTICE(p_hwfn, false, "PF Overflow sticky 0x%x\n", overflow);
+	if (!overflow) {
+		ecore_db_recovery_execute(p_hwfn, DB_REC_ONCE);
+		return ECORE_SUCCESS;
+	}
+
+	if (ecore_edpm_enabled(p_hwfn)) {
+		rc = ecore_db_rec_flush_queue(p_hwfn, p_ptt);
+		if (rc != ECORE_SUCCESS)
+			return rc;
+	}
+
 	/* flush any pedning (e)dpm as they may never arrive */
 	ecore_wr(p_hwfn, p_ptt, DORQ_REG_DPM_FORCE_ABORT, 0x1);
 
@@ -477,8 +498,7 @@ static enum _ecore_status_t ecore_db_rec_attn(struct ecore_hwfn *p_hwfn,
 
 static enum _ecore_status_t ecore_dorq_attn_cb(struct ecore_hwfn *p_hwfn)
 {
-	u32 int_sts, first_drop_reason, details, address, overflow,
-		all_drops_reason;
+	u32 int_sts, first_drop_reason, details, address, all_drops_reason;
 	struct ecore_ptt *p_ptt = p_hwfn->p_dpc_ptt;
 	enum _ecore_status_t rc;
 
@@ -504,8 +524,6 @@ static enum _ecore_status_t ecore_dorq_attn_cb(struct ecore_hwfn *p_hwfn)
 				   DORQ_REG_DB_DROP_DETAILS);
 		address = ecore_rd(p_hwfn, p_ptt,
 				   DORQ_REG_DB_DROP_DETAILS_ADDRESS);
-		overflow = ecore_rd(p_hwfn, p_ptt,
-				    DORQ_REG_PF_OVFL_STICKY);
 		all_drops_reason = ecore_rd(p_hwfn, p_ptt,
 					    DORQ_REG_DB_DROP_DETAILS_REASON);
 
@@ -516,19 +534,16 @@ static enum _ecore_status_t ecore_dorq_attn_cb(struct ecore_hwfn *p_hwfn)
 			  "FID\t\t0x%04x\t\t(Opaque FID)\n"
 			  "Size\t\t0x%04x\t\t(in bytes)\n"
 			  "1st drop reason\t0x%08x\t(details on first drop since last handling)\n"
-			  "Sticky reasons\t0x%08x\t(all drop reasons since last handling)\n"
-			  "Overflow\t0x%x\t\t(a per PF indication)\n",
+			  "Sticky reasons\t0x%08x\t(all drop reasons since last handling)\n",
 			  address,
 			  GET_FIELD(details, ECORE_DORQ_ATTENTION_OPAQUE),
 			  GET_FIELD(details, ECORE_DORQ_ATTENTION_SIZE) * 4,
-			  first_drop_reason, all_drops_reason, overflow);
+			  first_drop_reason, all_drops_reason);
 
-		/* if this PF caused overflow, initiate recovery */
-		if (overflow) {
-			rc = ecore_db_rec_attn(p_hwfn, p_ptt);
-			if (rc != ECORE_SUCCESS)
-				return rc;
-		}
+		rc = ecore_db_rec_handler(p_hwfn, p_ptt);
+		OSAL_DB_REC_OCCURRED(p_hwfn);
+		if (rc != ECORE_SUCCESS)
+			return rc;
 
 		/* clear the doorbell drop details and prepare for next drop */
 		ecore_wr(p_hwfn, p_ptt, DORQ_REG_DB_DROP_DETAILS_REL, 0);
@@ -1209,8 +1224,9 @@ static enum _ecore_status_t ecore_int_attentions(struct ecore_hwfn *p_hwfn)
 static void ecore_sb_ack_attn(struct ecore_hwfn *p_hwfn,
 			      void OSAL_IOMEM *igu_addr, u32 ack_cons)
 {
-	struct igu_prod_cons_update igu_ack = { 0 };
+	struct igu_prod_cons_update igu_ack;
 
+	OSAL_MEMSET(&igu_ack, 0, sizeof(struct igu_prod_cons_update));
 	igu_ack.sb_id_and_flags =
 	    ((ack_cons << IGU_PROD_CONS_UPDATE_SB_INDEX_SHIFT) |
 	     (1 << IGU_PROD_CONS_UPDATE_UPDATE_FLAG_SHIFT) |
@@ -1546,11 +1562,13 @@ void ecore_int_cau_conf_sb(struct ecore_hwfn *p_hwfn,
 		ecore_dmae_host2grc(p_hwfn, p_ptt,
 				    (u64)(osal_uintptr_t)&phys_addr,
 				    CAU_REG_SB_ADDR_MEMORY +
-				    igu_sb_id * sizeof(u64), 2, 0);
+				    igu_sb_id * sizeof(u64), 2,
+				    OSAL_NULL /* default parameters */);
 		ecore_dmae_host2grc(p_hwfn, p_ptt,
 				    (u64)(osal_uintptr_t)&sb_entry,
 				    CAU_REG_SB_VAR_MEMORY +
-				    igu_sb_id * sizeof(u64), 2, 0);
+				    igu_sb_id * sizeof(u64), 2,
+				    OSAL_NULL /* default parameters */);
 	} else {
 		/* Initialize Status Block Address */
 		STORE_RT_REG_AGG(p_hwfn,
@@ -2631,7 +2649,8 @@ enum _ecore_status_t ecore_int_set_timer_res(struct ecore_hwfn *p_hwfn,
 
 	rc = ecore_dmae_grc2host(p_hwfn, p_ptt, CAU_REG_SB_VAR_MEMORY +
 				 sb_id * sizeof(u64),
-				 (u64)(osal_uintptr_t)&sb_entry, 2, 0);
+				 (u64)(osal_uintptr_t)&sb_entry, 2,
+				 OSAL_NULL /* default parameters */);
 	if (rc != ECORE_SUCCESS) {
 		DP_ERR(p_hwfn, "dmae_grc2host failed %d\n", rc);
 		return rc;
@@ -2644,8 +2663,8 @@ enum _ecore_status_t ecore_int_set_timer_res(struct ecore_hwfn *p_hwfn,
 
 	rc = ecore_dmae_host2grc(p_hwfn, p_ptt,
 				 (u64)(osal_uintptr_t)&sb_entry,
-				 CAU_REG_SB_VAR_MEMORY +
-				 sb_id * sizeof(u64), 2, 0);
+				 CAU_REG_SB_VAR_MEMORY + sb_id * sizeof(u64), 2,
+				 OSAL_NULL /* default parameters */);
 	if (rc != ECORE_SUCCESS) {
 		DP_ERR(p_hwfn, "dmae_host2grc failed %d\n", rc);
 		return rc;
@@ -2681,3 +2700,35 @@ enum _ecore_status_t ecore_int_get_sb_dbg(struct ecore_hwfn *p_hwfn,
 
 	return ECORE_SUCCESS;
 }
+
+void ecore_pf_flr_igu_cleanup(struct ecore_hwfn *p_hwfn)
+{
+	struct ecore_ptt *p_ptt = p_hwfn->p_main_ptt;
+	struct ecore_ptt *p_dpc_ptt = ecore_get_reserved_ptt(p_hwfn,
+							     RESERVED_PTT_DPC);
+	int i;
+
+	/* Do not reorder the following cleanup sequence */
+	/* Ack all attentions */
+	ecore_wr(p_hwfn, p_ptt, IGU_REG_ATTENTION_ACK_BITS, 0xfff);
+
+	/* Clear driver attention */
+	ecore_wr(p_hwfn,  p_dpc_ptt,
+		((p_hwfn->rel_pf_id << 3) + MISC_REG_AEU_GENERAL_ATTN_0), 0);
+
+	/* Clear per-PF IGU registers to restore them as if the IGU
+	 * was reset for this PF
+	 */
+	ecore_wr(p_hwfn, p_ptt, IGU_REG_LEADING_EDGE_LATCH, 0);
+	ecore_wr(p_hwfn, p_ptt, IGU_REG_TRAILING_EDGE_LATCH, 0);
+	ecore_wr(p_hwfn, p_ptt, IGU_REG_PF_CONFIGURATION, 0);
+
+	/* Execute IGU clean up*/
+	ecore_wr(p_hwfn, p_ptt, IGU_REG_PF_FUNCTIONAL_CLEANUP, 1);
+
+	/* Clear Stats */
+	ecore_wr(p_hwfn, p_ptt, IGU_REG_STATISTIC_NUM_OF_INTA_ASSERTED, 0);
+
+	for (i = 0; i < IGU_REG_PBA_STS_PF_SIZE; i++)
+		ecore_wr(p_hwfn, p_ptt, IGU_REG_PBA_STS_PF + i * 4, 0);
+}
diff --git a/drivers/net/qede/base/ecore_int.h b/drivers/net/qede/base/ecore_int.h
index 041240d7..ff2310cf 100644
--- a/drivers/net/qede/base/ecore_int.h
+++ b/drivers/net/qede/base/ecore_int.h
@@ -256,5 +256,6 @@ enum _ecore_status_t ecore_int_set_timer_res(struct ecore_hwfn *p_hwfn,
 enum _ecore_status_t ecore_pglueb_rbc_attn_handler(struct ecore_hwfn *p_hwfn,
 						   struct ecore_ptt *p_ptt,
 						   bool is_hw_init);
+void ecore_pf_flr_igu_cleanup(struct ecore_hwfn *p_hwfn);
 
 #endif /* __ECORE_INT_H__ */
diff --git a/drivers/net/qede/base/ecore_int_api.h b/drivers/net/qede/base/ecore_int_api.h
index aeaf469e..42538a46 100644
--- a/drivers/net/qede/base/ecore_int_api.h
+++ b/drivers/net/qede/base/ecore_int_api.h
@@ -92,8 +92,9 @@ static OSAL_INLINE u16 ecore_sb_update_sb_idx(struct ecore_sb_info *sb_info)
 static OSAL_INLINE void ecore_sb_ack(struct ecore_sb_info *sb_info,
 				     enum igu_int_cmd int_cmd, u8 upd_flg)
 {
-	struct igu_prod_cons_update igu_ack = { 0 };
+	struct igu_prod_cons_update igu_ack;
 
+	OSAL_MEMSET(&igu_ack, 0, sizeof(struct igu_prod_cons_update));
 	igu_ack.sb_id_and_flags =
 	    ((sb_info->sb_ack << IGU_PROD_CONS_UPDATE_SB_INDEX_SHIFT) |
 	     (upd_flg << IGU_PROD_CONS_UPDATE_UPDATE_FLAG_SHIFT) |
@@ -343,4 +344,15 @@ enum _ecore_status_t ecore_int_get_sb_dbg(struct ecore_hwfn *p_hwfn,
 enum _ecore_status_t
 ecore_int_igu_relocate_sb(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 			  u16 sb_id, bool b_to_vf);
+
+/**
+ * @brief - Doorbell Recovery handler.
+ *          Run DB_REAL_DEAL doorbell recovery in case of PF overflow
+ *          (and flush DORQ if needed), otherwise run DB_REC_ONCE.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ */
+enum _ecore_status_t ecore_db_rec_handler(struct ecore_hwfn *p_hwfn,
+					  struct ecore_ptt *p_ptt);
 #endif
diff --git a/drivers/net/qede/base/ecore_iov_api.h b/drivers/net/qede/base/ecore_iov_api.h
index 29001d71..55de7086 100644
--- a/drivers/net/qede/base/ecore_iov_api.h
+++ b/drivers/net/qede/base/ecore_iov_api.h
@@ -84,6 +84,13 @@ struct ecore_public_vf_info {
 	 */
 	u8 forced_mac[ETH_ALEN];
 	u16 forced_vlan;
+
+	/* Trusted VFs can configure promiscuous mode and
+	 * set MAC address inspite PF has set forced MAC.
+	 * Also store shadow promisc configuration if needed.
+	 */
+	bool is_trusted_configured;
+	bool is_trusted_request;
 };
 
 struct ecore_iov_vf_init_params {
@@ -695,6 +702,16 @@ bool ecore_iov_is_vf_started(struct ecore_hwfn *p_hwfn,
  */
 int ecore_iov_get_vf_min_rate(struct ecore_hwfn *p_hwfn, int vfid);
 
+/**
+ * @brief - Configure min rate for VF's vport.
+ * @param p_dev
+ * @param vfid
+ * @param - rate in Mbps
+ *
+ * @return
+ */
+enum _ecore_status_t ecore_iov_configure_min_tx_rate(struct ecore_dev *p_dev,
+						     int vfid, u32 rate);
 #endif
 
 /**
diff --git a/drivers/net/qede/base/ecore_iro.h b/drivers/net/qede/base/ecore_iro.h
index 05693029..12d45c1c 100644
--- a/drivers/net/qede/base/ecore_iro.h
+++ b/drivers/net/qede/base/ecore_iro.h
@@ -113,91 +113,129 @@
 /* Tstorm Eth limit Rx rate */
 #define ETH_RX_RATE_LIMIT_OFFSET(pf_id) (IRO[29].base + ((pf_id) * IRO[29].m1))
 #define ETH_RX_RATE_LIMIT_SIZE (IRO[29].size)
+/* RSS indirection table entry update command per PF offset in TSTORM PF BAR0.
+ * Use eth_tstorm_rss_update_data for update.
+ */
+#define TSTORM_ETH_RSS_UPDATE_OFFSET(pf_id) (IRO[30].base + \
+	((pf_id) * IRO[30].m1))
+#define TSTORM_ETH_RSS_UPDATE_SIZE (IRO[30].size)
 /* Xstorm queue zone */
-#define XSTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) (IRO[30].base + \
-	((queue_id) * IRO[30].m1))
-#define XSTORM_ETH_QUEUE_ZONE_SIZE (IRO[30].size)
+#define XSTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) (IRO[31].base + \
+	((queue_id) * IRO[31].m1))
+#define XSTORM_ETH_QUEUE_ZONE_SIZE (IRO[31].size)
 /* Ystorm cqe producer */
-#define YSTORM_TOE_CQ_PROD_OFFSET(rss_id) (IRO[31].base + \
-	((rss_id) * IRO[31].m1))
-#define YSTORM_TOE_CQ_PROD_SIZE (IRO[31].size)
-/* Ustorm cqe producer */
-#define USTORM_TOE_CQ_PROD_OFFSET(rss_id) (IRO[32].base + \
+#define YSTORM_TOE_CQ_PROD_OFFSET(rss_id) (IRO[32].base + \
 	((rss_id) * IRO[32].m1))
-#define USTORM_TOE_CQ_PROD_SIZE (IRO[32].size)
+#define YSTORM_TOE_CQ_PROD_SIZE (IRO[32].size)
+/* Ustorm cqe producer */
+#define USTORM_TOE_CQ_PROD_OFFSET(rss_id) (IRO[33].base + \
+	((rss_id) * IRO[33].m1))
+#define USTORM_TOE_CQ_PROD_SIZE (IRO[33].size)
 /* Ustorm grq producer */
-#define USTORM_TOE_GRQ_PROD_OFFSET(pf_id) (IRO[33].base + \
-	((pf_id) * IRO[33].m1))
-#define USTORM_TOE_GRQ_PROD_SIZE (IRO[33].size)
+#define USTORM_TOE_GRQ_PROD_OFFSET(pf_id) (IRO[34].base + \
+	((pf_id) * IRO[34].m1))
+#define USTORM_TOE_GRQ_PROD_SIZE (IRO[34].size)
 /* Tstorm cmdq-cons of given command queue-id */
-#define TSTORM_SCSI_CMDQ_CONS_OFFSET(cmdq_queue_id) (IRO[34].base + \
-	((cmdq_queue_id) * IRO[34].m1))
-#define TSTORM_SCSI_CMDQ_CONS_SIZE (IRO[34].size)
+#define TSTORM_SCSI_CMDQ_CONS_OFFSET(cmdq_queue_id) (IRO[35].base + \
+	((cmdq_queue_id) * IRO[35].m1))
+#define TSTORM_SCSI_CMDQ_CONS_SIZE (IRO[35].size)
 /* Tstorm (reflects M-Storm) bdq-external-producer of given function ID,
  * BDqueue-id
  */
-#define TSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) (IRO[35].base + \
-	((func_id) * IRO[35].m1) + ((bdq_id) * IRO[35].m2))
-#define TSTORM_SCSI_BDQ_EXT_PROD_SIZE (IRO[35].size)
-/* Mstorm bdq-external-producer of given BDQ resource ID, BDqueue-id */
-#define MSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) (IRO[36].base + \
+#define TSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) (IRO[36].base + \
 	((func_id) * IRO[36].m1) + ((bdq_id) * IRO[36].m2))
-#define MSTORM_SCSI_BDQ_EXT_PROD_SIZE (IRO[36].size)
+#define TSTORM_SCSI_BDQ_EXT_PROD_SIZE (IRO[36].size)
+/* Mstorm bdq-external-producer of given BDQ resource ID, BDqueue-id */
+#define MSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) (IRO[37].base + \
+	((func_id) * IRO[37].m1) + ((bdq_id) * IRO[37].m2))
+#define MSTORM_SCSI_BDQ_EXT_PROD_SIZE (IRO[37].size)
 /* Tstorm iSCSI RX stats */
-#define TSTORM_ISCSI_RX_STATS_OFFSET(pf_id) (IRO[37].base + \
-	((pf_id) * IRO[37].m1))
-#define TSTORM_ISCSI_RX_STATS_SIZE (IRO[37].size)
-/* Mstorm iSCSI RX stats */
-#define MSTORM_ISCSI_RX_STATS_OFFSET(pf_id) (IRO[38].base + \
+#define TSTORM_ISCSI_RX_STATS_OFFSET(pf_id) (IRO[38].base + \
 	((pf_id) * IRO[38].m1))
-#define MSTORM_ISCSI_RX_STATS_SIZE (IRO[38].size)
-/* Ustorm iSCSI RX stats */
-#define USTORM_ISCSI_RX_STATS_OFFSET(pf_id) (IRO[39].base + \
+#define TSTORM_ISCSI_RX_STATS_SIZE (IRO[38].size)
+/* Mstorm iSCSI RX stats */
+#define MSTORM_ISCSI_RX_STATS_OFFSET(pf_id) (IRO[39].base + \
 	((pf_id) * IRO[39].m1))
-#define USTORM_ISCSI_RX_STATS_SIZE (IRO[39].size)
-/* Xstorm iSCSI TX stats */
-#define XSTORM_ISCSI_TX_STATS_OFFSET(pf_id) (IRO[40].base + \
+#define MSTORM_ISCSI_RX_STATS_SIZE (IRO[39].size)
+/* Ustorm iSCSI RX stats */
+#define USTORM_ISCSI_RX_STATS_OFFSET(pf_id) (IRO[40].base + \
 	((pf_id) * IRO[40].m1))
-#define XSTORM_ISCSI_TX_STATS_SIZE (IRO[40].size)
-/* Ystorm iSCSI TX stats */
-#define YSTORM_ISCSI_TX_STATS_OFFSET(pf_id) (IRO[41].base + \
+#define USTORM_ISCSI_RX_STATS_SIZE (IRO[40].size)
+/* Xstorm iSCSI TX stats */
+#define XSTORM_ISCSI_TX_STATS_OFFSET(pf_id) (IRO[41].base + \
 	((pf_id) * IRO[41].m1))
-#define YSTORM_ISCSI_TX_STATS_SIZE (IRO[41].size)
-/* Pstorm iSCSI TX stats */
-#define PSTORM_ISCSI_TX_STATS_OFFSET(pf_id) (IRO[42].base + \
+#define XSTORM_ISCSI_TX_STATS_SIZE (IRO[41].size)
+/* Ystorm iSCSI TX stats */
+#define YSTORM_ISCSI_TX_STATS_OFFSET(pf_id) (IRO[42].base + \
 	((pf_id) * IRO[42].m1))
-#define PSTORM_ISCSI_TX_STATS_SIZE (IRO[42].size)
-/* Tstorm FCoE RX stats */
-#define TSTORM_FCOE_RX_STATS_OFFSET(pf_id) (IRO[43].base + \
+#define YSTORM_ISCSI_TX_STATS_SIZE (IRO[42].size)
+/* Pstorm iSCSI TX stats */
+#define PSTORM_ISCSI_TX_STATS_OFFSET(pf_id) (IRO[43].base + \
 	((pf_id) * IRO[43].m1))
-#define TSTORM_FCOE_RX_STATS_SIZE (IRO[43].size)
-/* Pstorm FCoE TX stats */
-#define PSTORM_FCOE_TX_STATS_OFFSET(pf_id) (IRO[44].base + \
+#define PSTORM_ISCSI_TX_STATS_SIZE (IRO[43].size)
+/* Tstorm FCoE RX stats */
+#define TSTORM_FCOE_RX_STATS_OFFSET(pf_id) (IRO[44].base + \
 	((pf_id) * IRO[44].m1))
-#define PSTORM_FCOE_TX_STATS_SIZE (IRO[44].size)
+#define TSTORM_FCOE_RX_STATS_SIZE (IRO[44].size)
+/* Pstorm FCoE TX stats */
+#define PSTORM_FCOE_TX_STATS_OFFSET(pf_id) (IRO[45].base + \
+	((pf_id) * IRO[45].m1))
+#define PSTORM_FCOE_TX_STATS_SIZE (IRO[45].size)
 /* Pstorm RDMA queue statistics */
-#define PSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) \
-	(IRO[45].base + ((rdma_stat_counter_id) * IRO[45].m1))
-#define PSTORM_RDMA_QUEUE_STAT_SIZE (IRO[45].size)
-/* Tstorm RDMA queue statistics */
-#define TSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) (IRO[46].base + \
+#define PSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) (IRO[46].base + \
 	((rdma_stat_counter_id) * IRO[46].m1))
-#define TSTORM_RDMA_QUEUE_STAT_SIZE (IRO[46].size)
+#define PSTORM_RDMA_QUEUE_STAT_SIZE (IRO[46].size)
+/* Tstorm RDMA queue statistics */
+#define TSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) (IRO[47].base + \
+	((rdma_stat_counter_id) * IRO[47].m1))
+#define TSTORM_RDMA_QUEUE_STAT_SIZE (IRO[47].size)
+/* Xstorm error level for assert */
+#define XSTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) (IRO[48].base + \
+	((pf_id) * IRO[48].m1))
+#define XSTORM_RDMA_ASSERT_LEVEL_SIZE (IRO[48].size)
+/* Ystorm error level for assert */
+#define YSTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) (IRO[49].base + \
+	((pf_id) * IRO[49].m1))
+#define YSTORM_RDMA_ASSERT_LEVEL_SIZE (IRO[49].size)
+/* Pstorm error level for assert */
+#define PSTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) (IRO[50].base + \
+	((pf_id) * IRO[50].m1))
+#define PSTORM_RDMA_ASSERT_LEVEL_SIZE (IRO[50].size)
+/* Tstorm error level for assert */
+#define TSTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) (IRO[51].base + \
+	((pf_id) * IRO[51].m1))
+#define TSTORM_RDMA_ASSERT_LEVEL_SIZE (IRO[51].size)
+/* Mstorm error level for assert */
+#define MSTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) (IRO[52].base + \
+	((pf_id) * IRO[52].m1))
+#define MSTORM_RDMA_ASSERT_LEVEL_SIZE (IRO[52].size)
+/* Ustorm error level for assert */
+#define USTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) (IRO[53].base + \
+	((pf_id) * IRO[53].m1))
+#define USTORM_RDMA_ASSERT_LEVEL_SIZE (IRO[53].size)
 /* Xstorm iWARP rxmit stats */
-#define XSTORM_IWARP_RXMIT_STATS_OFFSET(pf_id) (IRO[47].base + \
-	((pf_id) * IRO[47].m1))
-#define XSTORM_IWARP_RXMIT_STATS_SIZE (IRO[47].size)
+#define XSTORM_IWARP_RXMIT_STATS_OFFSET(pf_id) (IRO[54].base + \
+	((pf_id) * IRO[54].m1))
+#define XSTORM_IWARP_RXMIT_STATS_SIZE (IRO[54].size)
 /* Tstorm RoCE Event Statistics */
-#define TSTORM_ROCE_EVENTS_STAT_OFFSET(roce_pf_id) (IRO[48].base + \
-	((roce_pf_id) * IRO[48].m1))
-#define TSTORM_ROCE_EVENTS_STAT_SIZE (IRO[48].size)
+#define TSTORM_ROCE_EVENTS_STAT_OFFSET(roce_pf_id) (IRO[55].base + \
+	((roce_pf_id) * IRO[55].m1))
+#define TSTORM_ROCE_EVENTS_STAT_SIZE (IRO[55].size)
 /* DCQCN Received Statistics */
-#define YSTORM_ROCE_DCQCN_RECEIVED_STATS_OFFSET(roce_pf_id) (IRO[49].base + \
-	((roce_pf_id) * IRO[49].m1))
-#define YSTORM_ROCE_DCQCN_RECEIVED_STATS_SIZE (IRO[49].size)
+#define YSTORM_ROCE_DCQCN_RECEIVED_STATS_OFFSET(roce_pf_id) (IRO[56].base + \
+	((roce_pf_id) * IRO[56].m1))
+#define YSTORM_ROCE_DCQCN_RECEIVED_STATS_SIZE (IRO[56].size)
+/* RoCE Error Statistics */
+#define YSTORM_ROCE_ERROR_STATS_OFFSET(roce_pf_id) (IRO[57].base + \
+	((roce_pf_id) * IRO[57].m1))
+#define YSTORM_ROCE_ERROR_STATS_SIZE (IRO[57].size)
 /* DCQCN Sent Statistics */
-#define PSTORM_ROCE_DCQCN_SENT_STATS_OFFSET(roce_pf_id) (IRO[50].base + \
-	((roce_pf_id) * IRO[50].m1))
-#define PSTORM_ROCE_DCQCN_SENT_STATS_SIZE (IRO[50].size)
+#define PSTORM_ROCE_DCQCN_SENT_STATS_OFFSET(roce_pf_id) (IRO[58].base + \
+	((roce_pf_id) * IRO[58].m1))
+#define PSTORM_ROCE_DCQCN_SENT_STATS_SIZE (IRO[58].size)
+/* RoCE CQEs Statistics */
+#define USTORM_ROCE_CQE_STATS_OFFSET(roce_pf_id) (IRO[59].base + \
+	((roce_pf_id) * IRO[59].m1))
+#define USTORM_ROCE_CQE_STATS_SIZE (IRO[59].size)
 
 #endif /* __IRO_H__ */
diff --git a/drivers/net/qede/base/ecore_iro_values.h b/drivers/net/qede/base/ecore_iro_values.h
index 685fa2e8..30e632ce 100644
--- a/drivers/net/qede/base/ecore_iro_values.h
+++ b/drivers/net/qede/base/ecore_iro_values.h
@@ -7,7 +7,7 @@
 #ifndef __IRO_VALUES_H__
 #define __IRO_VALUES_H__
 
-static const struct iro iro_arr[51] = {
+static const struct iro iro_arr[60] = {
 /* YSTORM_FLOW_CONTROL_MODE_OFFSET */
 	{      0x0,      0x0,      0x0,      0x0,      0x8},
 /* TSTORM_PORT_STAT_OFFSET(port_id) */
@@ -29,7 +29,7 @@ static const struct iro iro_arr[51] = {
 /* YSTORM_INTEG_TEST_DATA_OFFSET */
 	{   0x3e38,      0x0,      0x0,      0x0,     0x78},
 /* PSTORM_INTEG_TEST_DATA_OFFSET */
-	{   0x2b78,      0x0,      0x0,      0x0,     0x78},
+	{   0x3ef8,      0x0,      0x0,      0x0,     0x78},
 /* TSTORM_INTEG_TEST_DATA_OFFSET */
 	{   0x4c40,      0x0,      0x0,      0x0,     0x78},
 /* MSTORM_INTEG_TEST_DATA_OFFSET */
@@ -43,7 +43,7 @@ static const struct iro iro_arr[51] = {
 /* CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) */
 	{   0xb820,     0x30,      0x0,      0x0,     0x30},
 /* CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(core_tx_stats_id) */
-	{   0x96c0,     0x30,      0x0,      0x0,     0x30},
+	{   0xa990,     0x30,      0x0,      0x0,     0x30},
 /* MSTORM_QUEUE_STAT_OFFSET(stat_counter_id) */
 	{   0x4b68,     0x80,      0x0,      0x0,     0x40},
 /* MSTORM_ETH_PF_PRODS_OFFSET(queue_id) */
@@ -59,15 +59,17 @@ static const struct iro iro_arr[51] = {
 /* USTORM_ETH_PF_STAT_OFFSET(pf_id) */
 	{   0xe770,     0x60,      0x0,      0x0,     0x60},
 /* PSTORM_QUEUE_STAT_OFFSET(stat_counter_id) */
-	{   0x2d10,     0x80,      0x0,      0x0,     0x38},
+	{   0x4090,     0x80,      0x0,      0x0,     0x38},
 /* PSTORM_ETH_PF_STAT_OFFSET(pf_id) */
-	{   0xf2b8,     0x78,      0x0,      0x0,     0x78},
+	{   0xfea8,     0x78,      0x0,      0x0,     0x78},
 /* PSTORM_CTL_FRAME_ETHTYPE_OFFSET(ethType_id) */
 	{    0x1f8,      0x4,      0x0,      0x0,      0x4},
 /* TSTORM_ETH_PRS_INPUT_OFFSET */
 	{   0xaf20,      0x0,      0x0,      0x0,     0xf0},
 /* ETH_RX_RATE_LIMIT_OFFSET(pf_id) */
 	{   0xb010,      0x8,      0x0,      0x0,      0x8},
+/* TSTORM_ETH_RSS_UPDATE_OFFSET(pf_id) */
+	{    0xc00,      0x8,      0x0,      0x0,      0x8},
 /* XSTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) */
 	{    0x1f8,      0x8,      0x0,      0x0,      0x8},
 /* YSTORM_TOE_CQ_PROD_OFFSET(rss_id) */
@@ -91,25 +93,41 @@ static const struct iro iro_arr[51] = {
 /* XSTORM_ISCSI_TX_STATS_OFFSET(pf_id) */
 	{   0xa588,     0x50,      0x0,      0x0,     0x20},
 /* YSTORM_ISCSI_TX_STATS_OFFSET(pf_id) */
-	{   0x8700,     0x40,      0x0,      0x0,     0x28},
+	{   0x8f00,     0x40,      0x0,      0x0,     0x28},
 /* PSTORM_ISCSI_TX_STATS_OFFSET(pf_id) */
-	{  0x10300,     0x18,      0x0,      0x0,     0x10},
+	{  0x10e30,     0x18,      0x0,      0x0,     0x10},
 /* TSTORM_FCOE_RX_STATS_OFFSET(pf_id) */
 	{   0xde48,     0x48,      0x0,      0x0,     0x38},
 /* PSTORM_FCOE_TX_STATS_OFFSET(pf_id) */
-	{  0x10768,     0x20,      0x0,      0x0,     0x20},
+	{  0x11298,     0x20,      0x0,      0x0,     0x20},
 /* PSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) */
-	{   0x2d48,     0x80,      0x0,      0x0,     0x10},
+	{   0x40c8,     0x80,      0x0,      0x0,     0x10},
 /* TSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) */
 	{   0x5048,     0x10,      0x0,      0x0,     0x10},
+/* XSTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) */
+	{   0xa928,      0x8,      0x0,      0x0,      0x1},
+/* YSTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) */
+	{   0xa128,      0x8,      0x0,      0x0,      0x1},
+/* PSTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) */
+	{  0x11a30,      0x8,      0x0,      0x0,      0x1},
+/* TSTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) */
+	{   0xf030,      0x8,      0x0,      0x0,      0x1},
+/* MSTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) */
+	{  0x13028,      0x8,      0x0,      0x0,      0x1},
+/* USTORM_RDMA_ASSERT_LEVEL_OFFSET(pf_id) */
+	{  0x12c58,      0x8,      0x0,      0x0,      0x1},
 /* XSTORM_IWARP_RXMIT_STATS_OFFSET(pf_id) */
 	{   0xc9b8,     0x30,      0x0,      0x0,     0x10},
 /* TSTORM_ROCE_EVENTS_STAT_OFFSET(roce_pf_id) */
-	{   0xed90,     0x10,      0x0,      0x0,     0x10},
+	{   0xed90,     0x28,      0x0,      0x0,     0x28},
 /* YSTORM_ROCE_DCQCN_RECEIVED_STATS_OFFSET(roce_pf_id) */
-	{   0xa520,     0x10,      0x0,      0x0,     0x10},
+	{   0xad20,     0x18,      0x0,      0x0,     0x18},
+/* YSTORM_ROCE_ERROR_STATS_OFFSET(roce_pf_id) */
+	{   0xaea0,      0x8,      0x0,      0x0,      0x8},
 /* PSTORM_ROCE_DCQCN_SENT_STATS_OFFSET(roce_pf_id) */
-	{  0x13108,      0x8,      0x0,      0x0,      0x8},
+	{  0x13c38,      0x8,      0x0,      0x0,      0x8},
+/* USTORM_ROCE_CQE_STATS_OFFSET(roce_pf_id) */
+	{  0x13c50,     0x18,      0x0,      0x0,     0x18},
 };
 
 #endif /* __IRO_VALUES_H__ */
diff --git a/drivers/net/qede/base/ecore_l2.c b/drivers/net/qede/base/ecore_l2.c
index d71f4616..8b9817eb 100644
--- a/drivers/net/qede/base/ecore_l2.c
+++ b/drivers/net/qede/base/ecore_l2.c
@@ -608,6 +608,9 @@ ecore_sp_update_accept_mode(struct ecore_hwfn *p_hwfn,
 		SET_FIELD(state, ETH_VPORT_RX_MODE_BCAST_ACCEPT_ALL,
 			  !!(accept_filter & ECORE_ACCEPT_BCAST));
 
+		SET_FIELD(state, ETH_VPORT_RX_MODE_ACCEPT_ANY_VNI,
+			  !!(accept_filter & ECORE_ACCEPT_ANY_VNI));
+
 		p_ramrod->rx_mode.state = OSAL_CPU_TO_LE16(state);
 		DP_VERBOSE(p_hwfn, ECORE_MSG_SP,
 			   "vport[%02x] p_ramrod->rx_mode.state = 0x%x\n",
@@ -783,6 +786,11 @@ ecore_sp_vport_update(struct ecore_hwfn *p_hwfn,
 		return rc;
 	}
 
+	if (p_params->update_ctl_frame_check) {
+		p_cmn->ctl_frame_mac_check_en = p_params->mac_chk_en;
+		p_cmn->ctl_frame_ethtype_check_en = p_params->ethtype_chk_en;
+	}
+
 	/* Update mcast bins for VFs, PF doesn't use this functionality */
 	ecore_sp_update_mcast_bin(p_ramrod, p_params);
 
@@ -2084,6 +2092,24 @@ void ecore_reset_vport_stats(struct ecore_dev *p_dev)
 	}
 }
 
+static enum gft_profile_type
+ecore_arfs_mode_to_hsi(enum ecore_filter_config_mode mode)
+{
+	if (mode == ECORE_FILTER_CONFIG_MODE_5_TUPLE)
+		return GFT_PROFILE_TYPE_4_TUPLE;
+
+	if (mode == ECORE_FILTER_CONFIG_MODE_IP_DEST)
+		return GFT_PROFILE_TYPE_IP_DST_ADDR;
+
+	if (mode == ECORE_FILTER_CONFIG_MODE_TUNN_TYPE)
+		return GFT_PROFILE_TYPE_TUNNEL_TYPE;
+
+	if (mode == ECORE_FILTER_CONFIG_MODE_IP_SRC)
+		return GFT_PROFILE_TYPE_IP_SRC_ADDR;
+
+	return GFT_PROFILE_TYPE_L4_DST_PORT;
+}
+
 void ecore_arfs_mode_configure(struct ecore_hwfn *p_hwfn,
 			       struct ecore_ptt *p_ptt,
 			       struct ecore_arfs_config_params *p_cfg_params)
@@ -2091,13 +2117,13 @@ void ecore_arfs_mode_configure(struct ecore_hwfn *p_hwfn,
 	if (OSAL_TEST_BIT(ECORE_MF_DISABLE_ARFS, &p_hwfn->p_dev->mf_bits))
 		return;
 
-	if (p_cfg_params->arfs_enable) {
+	if (p_cfg_params->mode != ECORE_FILTER_CONFIG_MODE_DISABLE) {
 		ecore_gft_config(p_hwfn, p_ptt, p_hwfn->rel_pf_id,
 				 p_cfg_params->tcp,
 				 p_cfg_params->udp,
 				 p_cfg_params->ipv4,
 				 p_cfg_params->ipv6,
-				 GFT_PROFILE_TYPE_4_TUPLE);
+				 ecore_arfs_mode_to_hsi(p_cfg_params->mode));
 		DP_VERBOSE(p_hwfn, ECORE_MSG_SP,
 			   "tcp = %s, udp = %s, ipv4 = %s, ipv6 =%s\n",
 			   p_cfg_params->tcp ? "Enable" : "Disable",
@@ -2107,8 +2133,8 @@ void ecore_arfs_mode_configure(struct ecore_hwfn *p_hwfn,
 	} else {
 		ecore_gft_disable(p_hwfn, p_ptt, p_hwfn->rel_pf_id);
 	}
-	DP_VERBOSE(p_hwfn, ECORE_MSG_SP, "Configured ARFS mode : %s\n",
-		   p_cfg_params->arfs_enable ? "Enable" : "Disable");
+	DP_VERBOSE(p_hwfn, ECORE_MSG_SP, "Configured ARFS mode : %d\n",
+		   (int)p_cfg_params->mode);
 }
 
 enum _ecore_status_t
@@ -2179,10 +2205,10 @@ ecore_configure_rfs_ntuple_filter(struct ecore_hwfn *p_hwfn,
 	return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL);
 }
 
-int ecore_get_rxq_coalesce(struct ecore_hwfn *p_hwfn,
-			   struct ecore_ptt *p_ptt,
-			   struct ecore_queue_cid *p_cid,
-			   u16 *p_rx_coal)
+enum _ecore_status_t ecore_get_rxq_coalesce(struct ecore_hwfn *p_hwfn,
+					    struct ecore_ptt *p_ptt,
+					    struct ecore_queue_cid *p_cid,
+					    u16 *p_rx_coal)
 {
 	u32 coalesce, address, is_valid;
 	struct cau_sb_entry sb_entry;
@@ -2191,7 +2217,8 @@ int ecore_get_rxq_coalesce(struct ecore_hwfn *p_hwfn,
 
 	rc = ecore_dmae_grc2host(p_hwfn, p_ptt, CAU_REG_SB_VAR_MEMORY +
 				 p_cid->sb_igu_id * sizeof(u64),
-				 (u64)(osal_uintptr_t)&sb_entry, 2, 0);
+				 (u64)(osal_uintptr_t)&sb_entry, 2,
+				 OSAL_NULL /* default parameters */);
 	if (rc != ECORE_SUCCESS) {
 		DP_ERR(p_hwfn, "dmae_grc2host failed %d\n", rc);
 		return rc;
@@ -2213,10 +2240,10 @@ int ecore_get_rxq_coalesce(struct ecore_hwfn *p_hwfn,
 	return ECORE_SUCCESS;
 }
 
-int ecore_get_txq_coalesce(struct ecore_hwfn *p_hwfn,
-			   struct ecore_ptt *p_ptt,
-			   struct ecore_queue_cid *p_cid,
-			   u16 *p_tx_coal)
+enum _ecore_status_t ecore_get_txq_coalesce(struct ecore_hwfn *p_hwfn,
+					    struct ecore_ptt *p_ptt,
+					    struct ecore_queue_cid *p_cid,
+					    u16 *p_tx_coal)
 {
 	u32 coalesce, address, is_valid;
 	struct cau_sb_entry sb_entry;
@@ -2225,7 +2252,8 @@ int ecore_get_txq_coalesce(struct ecore_hwfn *p_hwfn,
 
 	rc = ecore_dmae_grc2host(p_hwfn, p_ptt, CAU_REG_SB_VAR_MEMORY +
 				 p_cid->sb_igu_id * sizeof(u64),
-				 (u64)(osal_uintptr_t)&sb_entry, 2, 0);
+				 (u64)(osal_uintptr_t)&sb_entry, 2,
+				 OSAL_NULL /* default parameters */);
 	if (rc != ECORE_SUCCESS) {
 		DP_ERR(p_hwfn, "dmae_grc2host failed %d\n", rc);
 		return rc;
@@ -2302,3 +2330,55 @@ ecore_eth_tx_queue_maxrate(struct ecore_hwfn *p_hwfn,
 	return ecore_init_vport_rl(p_hwfn, p_ptt, vport, rate,
 				   p_link->speed);
 }
+
+#define RSS_TSTORM_UPDATE_STATUS_MAX_POLL_COUNT    100
+#define RSS_TSTORM_UPDATE_STATUS_POLL_PERIOD_US    1
+
+enum _ecore_status_t
+ecore_update_eth_rss_ind_table_entry(struct ecore_hwfn *p_hwfn,
+				     u8 vport_id,
+				     u8 ind_table_index,
+				     u16 ind_table_value)
+{
+	struct eth_tstorm_rss_update_data update_data = { 0 };
+	void OSAL_IOMEM *addr = OSAL_NULL;
+	enum _ecore_status_t rc;
+	u8 abs_vport_id;
+	u32 cnt = 0;
+
+	OSAL_BUILD_BUG_ON(sizeof(update_data) != sizeof(u64));
+
+	rc = ecore_fw_vport(p_hwfn, vport_id, &abs_vport_id);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	addr = (u8 OSAL_IOMEM *)p_hwfn->regview +
+	       GTT_BAR0_MAP_REG_TSDM_RAM +
+	       TSTORM_ETH_RSS_UPDATE_OFFSET(p_hwfn->rel_pf_id);
+
+	*(u64 *)(&update_data) = DIRECT_REG_RD64(p_hwfn, addr);
+
+	for (cnt = 0; update_data.valid &&
+	     cnt < RSS_TSTORM_UPDATE_STATUS_MAX_POLL_COUNT; cnt++) {
+		OSAL_UDELAY(RSS_TSTORM_UPDATE_STATUS_POLL_PERIOD_US);
+		*(u64 *)(&update_data) = DIRECT_REG_RD64(p_hwfn, addr);
+	}
+
+	if (update_data.valid) {
+		DP_NOTICE(p_hwfn, true,
+			  "rss update valid status is not clear! valid=0x%x vport id=%d ind_Table_idx=%d ind_table_value=%d.\n",
+			  update_data.valid, vport_id, ind_table_index,
+			  ind_table_value);
+
+		return ECORE_AGAIN;
+	}
+
+	update_data.valid	    = 1;
+	update_data.ind_table_index = ind_table_index;
+	update_data.ind_table_value = ind_table_value;
+	update_data.vport_id	    = abs_vport_id;
+
+	DIRECT_REG_WR64(p_hwfn, addr, *(u64 *)(&update_data));
+
+	return ECORE_SUCCESS;
+}
diff --git a/drivers/net/qede/base/ecore_l2_api.h b/drivers/net/qede/base/ecore_l2_api.h
index 575b9e3a..004fb61b 100644
--- a/drivers/net/qede/base/ecore_l2_api.h
+++ b/drivers/net/qede/base/ecore_l2_api.h
@@ -137,6 +137,16 @@ struct ecore_filter_accept_flags {
 #define ECORE_ACCEPT_MCAST_MATCHED	0x08
 #define ECORE_ACCEPT_MCAST_UNMATCHED	0x10
 #define ECORE_ACCEPT_BCAST		0x20
+#define ECORE_ACCEPT_ANY_VNI		0x40
+};
+
+enum ecore_filter_config_mode {
+	ECORE_FILTER_CONFIG_MODE_DISABLE,
+	ECORE_FILTER_CONFIG_MODE_5_TUPLE,
+	ECORE_FILTER_CONFIG_MODE_L4_PORT,
+	ECORE_FILTER_CONFIG_MODE_IP_DEST,
+	ECORE_FILTER_CONFIG_MODE_TUNN_TYPE,
+	ECORE_FILTER_CONFIG_MODE_IP_SRC,
 };
 
 struct ecore_arfs_config_params {
@@ -144,7 +154,7 @@ struct ecore_arfs_config_params {
 	bool udp;
 	bool ipv4;
 	bool ipv6;
-	bool arfs_enable;	/* Enable or disable arfs mode */
+	enum ecore_filter_config_mode mode;
 };
 
 /* Add / remove / move / remove-all unicast MAC-VLAN filters.
@@ -337,7 +347,10 @@ struct ecore_sp_vport_update_params {
 	/* MTU change - notice this requires the vport to be disabled.
 	 * If non-zero, value would be used.
 	 */
-	u16 mtu;
+	u16                     mtu;
+	u8			update_ctl_frame_check;
+	u8			mac_chk_en;
+	u8			ethtype_chk_en;
 };
 
 /**
@@ -460,4 +473,28 @@ ecore_configure_rfs_ntuple_filter(struct ecore_hwfn *p_hwfn,
 				  dma_addr_t p_addr, u16 length,
 				  u16 qid, u8 vport_id,
 				  bool b_is_add);
+
+/**
+ * @brief - ecore_update_eth_rss_ind_table_entry
+ *
+ * This function being used to update RSS indirection table entry to FW RAM
+ * instead of using the SP vport update ramrod with rss params.
+ *
+ * Notice:
+ * This function supports only one outstanding command per engine. Ecore
+ * clients which use this function should call ecore_mcp_ind_table_lock() prior
+ * to it and ecore_mcp_ind_table_unlock() after it.
+ *
+ * @params p_hwfn
+ * @params vport_id
+ * @params ind_table_index
+ * @params ind_table_value
+ *
+ * @return enum _ecore_status_t
+ */
+enum _ecore_status_t
+ecore_update_eth_rss_ind_table_entry(struct ecore_hwfn *p_hwfn,
+				     u8 vport_id,
+				     u8 ind_table_index,
+				     u16 ind_table_value);
 #endif
diff --git a/drivers/net/qede/base/ecore_mcp.c b/drivers/net/qede/base/ecore_mcp.c
index ea14c172..6c656068 100644
--- a/drivers/net/qede/base/ecore_mcp.c
+++ b/drivers/net/qede/base/ecore_mcp.c
@@ -177,10 +177,16 @@ enum _ecore_status_t ecore_mcp_free(struct ecore_hwfn *p_hwfn)
 	return ECORE_SUCCESS;
 }
 
+/* Maximum of 1 sec to wait for the SHMEM ready indication */
+#define ECORE_MCP_SHMEM_RDY_MAX_RETRIES	20
+#define ECORE_MCP_SHMEM_RDY_ITER_MS	50
+
 static enum _ecore_status_t ecore_load_mcp_offsets(struct ecore_hwfn *p_hwfn,
 						   struct ecore_ptt *p_ptt)
 {
 	struct ecore_mcp_info *p_info = p_hwfn->mcp_info;
+	u8 cnt = ECORE_MCP_SHMEM_RDY_MAX_RETRIES;
+	u8 msec = ECORE_MCP_SHMEM_RDY_ITER_MS;
 	u32 drv_mb_offsize, mfw_mb_offsize;
 	u32 mcp_pf_id = MCP_PF_ID(p_hwfn);
 
@@ -198,6 +204,35 @@ static enum _ecore_status_t ecore_load_mcp_offsets(struct ecore_hwfn *p_hwfn,
 
 	p_info->public_base |= GRCBASE_MCP;
 
+	/* Get the MFW MB address and number of supported messages */
+	mfw_mb_offsize = ecore_rd(p_hwfn, p_ptt,
+				  SECTION_OFFSIZE_ADDR(p_info->public_base,
+				  PUBLIC_MFW_MB));
+	p_info->mfw_mb_addr = SECTION_ADDR(mfw_mb_offsize, mcp_pf_id);
+	p_info->mfw_mb_length = (u16)ecore_rd(p_hwfn, p_ptt,
+					      p_info->mfw_mb_addr);
+
+	/* @@@TBD:
+	 * The driver can notify that there was an MCP reset, and read the SHMEM
+	 * values before the MFW has completed initializing them.
+	 * As a temporary solution, the "sup_msgs" field is used as a data ready
+	 * indication.
+	 * This should be replaced with an actual indication when it is provided
+	 * by the MFW.
+	 */
+	while (!p_info->mfw_mb_length && cnt--) {
+		OSAL_MSLEEP(msec);
+		p_info->mfw_mb_length = (u16)ecore_rd(p_hwfn, p_ptt,
+						      p_info->mfw_mb_addr);
+	}
+
+	if (!cnt) {
+		DP_NOTICE(p_hwfn, false,
+			  "Failed to get the SHMEM ready notification after %d msec\n",
+			  ECORE_MCP_SHMEM_RDY_MAX_RETRIES * msec);
+		return ECORE_TIMEOUT;
+	}
+
 	/* Calculate the driver and MFW mailbox address */
 	drv_mb_offsize = ecore_rd(p_hwfn, p_ptt,
 				  SECTION_OFFSIZE_ADDR(p_info->public_base,
@@ -208,14 +243,6 @@ static enum _ecore_status_t ecore_load_mcp_offsets(struct ecore_hwfn *p_hwfn,
 		   " mcp_pf_id = 0x%x\n",
 		   drv_mb_offsize, p_info->drv_mb_addr, mcp_pf_id);
 
-	/* Set the MFW MB address */
-	mfw_mb_offsize = ecore_rd(p_hwfn, p_ptt,
-				  SECTION_OFFSIZE_ADDR(p_info->public_base,
-						       PUBLIC_MFW_MB));
-	p_info->mfw_mb_addr = SECTION_ADDR(mfw_mb_offsize, mcp_pf_id);
-	p_info->mfw_mb_length = (u16)ecore_rd(p_hwfn, p_ptt,
-					       p_info->mfw_mb_addr);
-
 	/* Get the current driver mailbox sequence before sending
 	 * the first command
 	 */
@@ -1656,6 +1683,49 @@ ecore_mcp_update_bw(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
 		      &param);
 }
 
+static void ecore_mcp_update_stag(struct ecore_hwfn *p_hwfn,
+				  struct ecore_ptt *p_ptt)
+{
+	struct public_func shmem_info;
+	u32 resp = 0, param = 0;
+
+	ecore_mcp_get_shmem_func(p_hwfn, p_ptt, &shmem_info,
+				 MCP_PF_ID(p_hwfn));
+
+	p_hwfn->mcp_info->func_info.ovlan = (u16)shmem_info.ovlan_stag &
+						 FUNC_MF_CFG_OV_STAG_MASK;
+	p_hwfn->hw_info.ovlan = p_hwfn->mcp_info->func_info.ovlan;
+	if (OSAL_TEST_BIT(ECORE_MF_OVLAN_CLSS, &p_hwfn->p_dev->mf_bits)) {
+		if (p_hwfn->hw_info.ovlan != ECORE_MCP_VLAN_UNSET) {
+			ecore_wr(p_hwfn, p_ptt, NIG_REG_LLH_FUNC_TAG_VALUE,
+				 p_hwfn->hw_info.ovlan);
+			ecore_wr(p_hwfn, p_ptt, NIG_REG_LLH_FUNC_TAG_EN, 1);
+
+			/* Configure DB to add external vlan to EDPM packets */
+			ecore_wr(p_hwfn, p_ptt, DORQ_REG_TAG1_OVRD_MODE, 1);
+			ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_EXT_VID_BB_K2,
+				 p_hwfn->hw_info.ovlan);
+		} else {
+			ecore_wr(p_hwfn, p_ptt, NIG_REG_LLH_FUNC_TAG_EN, 0);
+			ecore_wr(p_hwfn, p_ptt, NIG_REG_LLH_FUNC_TAG_VALUE, 0);
+
+			/* Configure DB to add external vlan to EDPM packets */
+			ecore_wr(p_hwfn, p_ptt, DORQ_REG_TAG1_OVRD_MODE, 0);
+			ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_EXT_VID_BB_K2, 0);
+		}
+
+		ecore_sp_pf_update_stag(p_hwfn);
+	}
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_SP, "ovlan  = %d hw_mode = 0x%x\n",
+		   p_hwfn->mcp_info->func_info.ovlan, p_hwfn->hw_info.hw_mode);
+	OSAL_HW_INFO_CHANGE(p_hwfn, ECORE_HW_INFO_CHANGE_OVLAN);
+
+	/* Acknowledge the MFW */
+	ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_S_TAG_UPDATE_ACK, 0,
+		      &resp, &param);
+}
+
 static void ecore_mcp_handle_fan_failure(struct ecore_hwfn *p_hwfn)
 {
 	/* A single notification should be sent to upper driver in CMT mode */
@@ -1946,7 +2016,7 @@ ecore_mcp_read_ufp_config(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt)
 		DP_NOTICE(p_hwfn, false, "Unknown Host priority control %d\n",
 			  val);
 
-	DP_VERBOSE(p_hwfn, ECORE_MSG_DCB,
+	DP_VERBOSE(p_hwfn, ECORE_MSG_SP,
 		   "UFP shmem config: mode = %d tc = %d pri_type = %d\n",
 		   p_hwfn->ufp_info.mode, p_hwfn->ufp_info.tc,
 		   p_hwfn->ufp_info.pri_type);
@@ -2041,6 +2111,9 @@ enum _ecore_status_t ecore_mcp_handle_events(struct ecore_hwfn *p_hwfn,
 		case MFW_DRV_MSG_BW_UPDATE:
 			ecore_mcp_update_bw(p_hwfn, p_ptt);
 			break;
+		case MFW_DRV_MSG_S_TAG_UPDATE:
+			ecore_mcp_update_stag(p_hwfn, p_ptt);
+			break;
 		case MFW_DRV_MSG_FAILURE_DETECTED:
 			ecore_mcp_handle_fan_failure(p_hwfn);
 			break;
@@ -2155,8 +2228,10 @@ enum _ecore_status_t ecore_mcp_get_media_type(struct ecore_hwfn *p_hwfn,
 
 enum _ecore_status_t ecore_mcp_get_transceiver_data(struct ecore_hwfn *p_hwfn,
 						    struct ecore_ptt *p_ptt,
-						    u32 *p_tranceiver_type)
+						    u32 *p_transceiver_state,
+						    u32 *p_transceiver_type)
 {
+	u32 transceiver_info;
 	enum _ecore_status_t rc = ECORE_SUCCESS;
 
 	/* TODO - Add support for VFs */
@@ -2167,14 +2242,23 @@ enum _ecore_status_t ecore_mcp_get_transceiver_data(struct ecore_hwfn *p_hwfn,
 		DP_NOTICE(p_hwfn, false, "MFW is not initialized!\n");
 		return ECORE_BUSY;
 	}
-	if (!p_ptt) {
-		*p_tranceiver_type = ETH_TRANSCEIVER_TYPE_NONE;
-		rc = ECORE_INVAL;
+
+	*p_transceiver_type = ETH_TRANSCEIVER_TYPE_NONE;
+	*p_transceiver_state = ETH_TRANSCEIVER_STATE_UPDATING;
+
+	transceiver_info = ecore_rd(p_hwfn, p_ptt,
+				    p_hwfn->mcp_info->port_addr +
+				    offsetof(struct public_port,
+				    transceiver_data));
+
+	*p_transceiver_state = GET_MFW_FIELD(transceiver_info,
+					     ETH_TRANSCEIVER_STATE);
+
+	if (*p_transceiver_state == ETH_TRANSCEIVER_STATE_PRESENT) {
+		*p_transceiver_type = GET_MFW_FIELD(transceiver_info,
+					    ETH_TRANSCEIVER_TYPE);
 	} else {
-		*p_tranceiver_type = ecore_rd(p_hwfn, p_ptt,
-				p_hwfn->mcp_info->port_addr +
-				offsetof(struct public_port,
-					transceiver_data));
+		*p_transceiver_type = ETH_TRANSCEIVER_TYPE_UNKNOWN;
 	}
 
 	return rc;
@@ -2194,15 +2278,11 @@ enum _ecore_status_t ecore_mcp_trans_speed_mask(struct ecore_hwfn *p_hwfn,
 						struct ecore_ptt *p_ptt,
 						u32 *p_speed_mask)
 {
-	u32 transceiver_data, transceiver_type, transceiver_state;
+	u32 transceiver_type, transceiver_state;
 
-	ecore_mcp_get_transceiver_data(p_hwfn, p_ptt, &transceiver_data);
+	ecore_mcp_get_transceiver_data(p_hwfn, p_ptt, &transceiver_state,
+				       &transceiver_type);
 
-	transceiver_state = GET_MFW_FIELD(transceiver_data,
-			    ETH_TRANSCEIVER_STATE);
-
-	transceiver_type = GET_MFW_FIELD(transceiver_data,
-			   ETH_TRANSCEIVER_TYPE);
 
 	if (is_transceiver_ready(transceiver_state, transceiver_type) == 0)
 		return ECORE_INVAL;
@@ -2823,10 +2903,72 @@ ecore_mcp_ov_get_fc_npiv(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 }
 
 enum _ecore_status_t
-ecore_mcp_ov_update_mtu(struct ecore_hwfn *p_hwfn,
-			struct ecore_ptt *p_ptt, u16 mtu)
+ecore_mcp_ov_update_mtu(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			u16 mtu)
 {
-	return 0;
+	u32 resp = 0, param = 0, drv_mb_param = 0;
+	enum _ecore_status_t rc;
+
+	SET_MFW_FIELD(drv_mb_param, DRV_MB_PARAM_OV_MTU_SIZE, (u32)mtu);
+	rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_OV_UPDATE_MTU,
+			   drv_mb_param, &resp, &param);
+	if (rc != ECORE_SUCCESS)
+		DP_ERR(p_hwfn, "Failed to send mtu value, rc = %d\n", rc);
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_mcp_ov_update_mac(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			u8 *mac)
+{
+	struct ecore_mcp_mb_params mb_params;
+	union drv_union_data union_data;
+	enum _ecore_status_t rc;
+
+	OSAL_MEM_ZERO(&mb_params, sizeof(mb_params));
+	mb_params.cmd = DRV_MSG_CODE_SET_VMAC;
+	SET_MFW_FIELD(mb_params.param, DRV_MSG_CODE_VMAC_TYPE,
+		      DRV_MSG_CODE_VMAC_TYPE_MAC);
+	mb_params.param |= MCP_PF_ID(p_hwfn);
+	OSAL_MEMCPY(&union_data.raw_data, mac, ETH_ALEN);
+	mb_params.p_data_src = &union_data;
+	rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params);
+	if (rc != ECORE_SUCCESS)
+		DP_ERR(p_hwfn, "Failed to send mac address, rc = %d\n", rc);
+
+	return rc;
+}
+
+enum _ecore_status_t
+ecore_mcp_ov_update_eswitch(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			    enum ecore_ov_eswitch eswitch)
+{
+	enum _ecore_status_t rc;
+	u32 resp = 0, param = 0;
+	u32 drv_mb_param;
+
+	switch (eswitch) {
+	case ECORE_OV_ESWITCH_NONE:
+		drv_mb_param = DRV_MB_PARAM_ESWITCH_MODE_NONE;
+		break;
+	case ECORE_OV_ESWITCH_VEB:
+		drv_mb_param = DRV_MB_PARAM_ESWITCH_MODE_VEB;
+		break;
+	case ECORE_OV_ESWITCH_VEPA:
+		drv_mb_param = DRV_MB_PARAM_ESWITCH_MODE_VEPA;
+		break;
+	default:
+		DP_ERR(p_hwfn, "Invalid eswitch mode %d\n", eswitch);
+		return ECORE_INVAL;
+	}
+
+	rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_OV_UPDATE_ESWITCH_MODE,
+			   drv_mb_param, &resp, &param);
+	if (rc != ECORE_SUCCESS)
+		DP_ERR(p_hwfn, "Failed to send eswitch mode, rc = %d\n", rc);
+
+	return rc;
 }
 
 enum _ecore_status_t ecore_mcp_set_led(struct ecore_hwfn *p_hwfn,
@@ -2938,11 +3080,11 @@ enum _ecore_status_t ecore_mcp_nvm_read(struct ecore_dev *p_dev, u32 addr,
 }
 
 enum _ecore_status_t ecore_mcp_phy_read(struct ecore_dev *p_dev, u32 cmd,
-					u32 addr, u8 *p_buf, u32 len)
+					u32 addr, u8 *p_buf, u32 *p_len)
 {
 	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
 	struct ecore_ptt *p_ptt;
-	u32 resp, param;
+	u32 resp = 0, param;
 	enum _ecore_status_t rc;
 
 	p_ptt = ecore_ptt_acquire(p_hwfn);
@@ -2953,7 +3095,7 @@ enum _ecore_status_t ecore_mcp_phy_read(struct ecore_dev *p_dev, u32 cmd,
 				  (cmd == ECORE_PHY_CORE_READ) ?
 				  DRV_MSG_CODE_PHY_CORE_READ :
 				  DRV_MSG_CODE_PHY_RAW_READ,
-				  addr, &resp, &param, &len, (u32 *)p_buf);
+				  addr, &resp, &param, p_len, (u32 *)p_buf);
 	if (rc != ECORE_SUCCESS)
 		DP_NOTICE(p_dev, false, "MCP command rc = %d\n", rc);
 
@@ -2982,7 +3124,7 @@ enum _ecore_status_t ecore_mcp_nvm_del_file(struct ecore_dev *p_dev, u32 addr)
 {
 	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
 	struct ecore_ptt *p_ptt;
-	u32 resp, param;
+	u32 resp = 0, param;
 	enum _ecore_status_t rc;
 
 	p_ptt = ecore_ptt_acquire(p_hwfn);
@@ -3001,7 +3143,7 @@ enum _ecore_status_t ecore_mcp_nvm_put_file_begin(struct ecore_dev *p_dev,
 {
 	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
 	struct ecore_ptt *p_ptt;
-	u32 resp, param;
+	u32 resp = 0, param;
 	enum _ecore_status_t rc;
 
 	p_ptt = ecore_ptt_acquire(p_hwfn);
@@ -3095,8 +3237,8 @@ enum _ecore_status_t ecore_mcp_phy_write(struct ecore_dev *p_dev, u32 cmd,
 					 u32 addr, u8 *p_buf, u32 len)
 {
 	struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev);
+	u32 resp = 0, param, nvm_cmd;
 	struct ecore_ptt *p_ptt;
-	u32 resp, param, nvm_cmd;
 	enum _ecore_status_t rc;
 
 	p_ptt = ecore_ptt_acquire(p_hwfn);
@@ -4002,13 +4144,83 @@ ecore_mcp_drv_attribute(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 	return ECORE_SUCCESS;
 }
 
+enum _ecore_status_t ecore_mcp_get_engine_config(struct ecore_hwfn *p_hwfn,
+						 struct ecore_ptt *p_ptt)
+{
+	struct ecore_dev *p_dev = p_hwfn->p_dev;
+	struct ecore_mcp_mb_params mb_params;
+	u8 fir_valid, l2_valid;
+	enum _ecore_status_t rc;
+
+	OSAL_MEM_ZERO(&mb_params, sizeof(mb_params));
+	mb_params.cmd = DRV_MSG_CODE_GET_ENGINE_CONFIG;
+	rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	if (mb_params.mcp_resp == FW_MSG_CODE_UNSUPPORTED) {
+		DP_INFO(p_hwfn,
+			"The get_engine_config command is unsupported by the MFW\n");
+		return ECORE_NOTIMPL;
+	}
+
+	fir_valid = GET_MFW_FIELD(mb_params.mcp_param,
+				  FW_MB_PARAM_ENG_CFG_FIR_AFFIN_VALID);
+	if (fir_valid)
+		p_dev->fir_affin =
+			GET_MFW_FIELD(mb_params.mcp_param,
+				      FW_MB_PARAM_ENG_CFG_FIR_AFFIN_VALUE);
+
+	l2_valid = GET_MFW_FIELD(mb_params.mcp_param,
+				 FW_MB_PARAM_ENG_CFG_L2_AFFIN_VALID);
+	if (l2_valid)
+		p_dev->l2_affin_hint =
+			GET_MFW_FIELD(mb_params.mcp_param,
+				      FW_MB_PARAM_ENG_CFG_L2_AFFIN_VALUE);
+
+	DP_INFO(p_hwfn,
+		"Engine affinity config: FIR={valid %hhd, value %hhd}, L2_hint={valid %hhd, value %hhd}\n",
+		fir_valid, p_dev->fir_affin, l2_valid, p_dev->l2_affin_hint);
+
+	return ECORE_SUCCESS;
+}
+
+enum _ecore_status_t ecore_mcp_get_ppfid_bitmap(struct ecore_hwfn *p_hwfn,
+						struct ecore_ptt *p_ptt)
+{
+	struct ecore_dev *p_dev = p_hwfn->p_dev;
+	struct ecore_mcp_mb_params mb_params;
+	enum _ecore_status_t rc;
+
+	OSAL_MEM_ZERO(&mb_params, sizeof(mb_params));
+	mb_params.cmd = DRV_MSG_CODE_GET_PPFID_BITMAP;
+	rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	if (mb_params.mcp_resp == FW_MSG_CODE_UNSUPPORTED) {
+		DP_INFO(p_hwfn,
+			"The get_ppfid_bitmap command is unsupported by the MFW\n");
+		return ECORE_NOTIMPL;
+	}
+
+	p_dev->ppfid_bitmap = GET_MFW_FIELD(mb_params.mcp_param,
+					    FW_MB_PARAM_PPFID_BITMAP);
+
+	DP_VERBOSE(p_hwfn, ECORE_MSG_SP, "PPFID bitmap 0x%hhx\n",
+		   p_dev->ppfid_bitmap);
+
+	return ECORE_SUCCESS;
+}
+
 void ecore_mcp_wol_wr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 		      u32 offset, u32 val)
 {
-	struct ecore_mcp_mb_params mb_params = {0};
 	enum _ecore_status_t	   rc = ECORE_SUCCESS;
 	u32			   dword = val;
+	struct ecore_mcp_mb_params mb_params;
 
+	OSAL_MEMSET(&mb_params, 0, sizeof(struct ecore_mcp_mb_params));
 	mb_params.cmd = DRV_MSG_CODE_WRITE_WOL_REG;
 	mb_params.param = offset;
 	mb_params.p_data_src = &dword;
diff --git a/drivers/net/qede/base/ecore_mcp.h b/drivers/net/qede/base/ecore_mcp.h
index 8e125310..2c052b7f 100644
--- a/drivers/net/qede/base/ecore_mcp.h
+++ b/drivers/net/qede/base/ecore_mcp.h
@@ -25,9 +25,6 @@
 					     rel_pfid)
 #define MCP_PF_ID(p_hwfn)	MCP_PF_ID_BY_REL(p_hwfn, (p_hwfn)->rel_pf_id)
 
-#define MFW_PORT(_p_hwfn)	((_p_hwfn)->abs_pf_id % \
-				 ecore_device_num_ports((_p_hwfn)->p_dev))
-
 struct ecore_mcp_info {
 	/* List for mailbox commands which were sent and wait for a response */
 	osal_list_t cmd_list;
@@ -566,4 +563,22 @@ ecore_mcp_read_ufp_config(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt);
 void ecore_mcp_wol_wr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
 		      u32 offset, u32 val);
 
+/**
+ * @brief Get the engine affinity configuration.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ */
+enum _ecore_status_t ecore_mcp_get_engine_config(struct ecore_hwfn *p_hwfn,
+						 struct ecore_ptt *p_ptt);
+
+/**
+ * @brief Get the PPFID bitmap.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ */
+enum _ecore_status_t ecore_mcp_get_ppfid_bitmap(struct ecore_hwfn *p_hwfn,
+						struct ecore_ptt *p_ptt);
+
 #endif /* __ECORE_MCP_H__ */
diff --git a/drivers/net/qede/base/ecore_mcp_api.h b/drivers/net/qede/base/ecore_mcp_api.h
index cfb9f99d..7327074f 100644
--- a/drivers/net/qede/base/ecore_mcp_api.h
+++ b/drivers/net/qede/base/ecore_mcp_api.h
@@ -185,6 +185,12 @@ enum ecore_ov_driver_state {
 	ECORE_OV_DRIVER_STATE_ACTIVE
 };
 
+enum ecore_ov_eswitch {
+	ECORE_OV_ESWITCH_NONE,
+	ECORE_OV_ESWITCH_VEB,
+	ECORE_OV_ESWITCH_VEPA
+};
+
 #define ECORE_MAX_NPIV_ENTRIES 128
 #define ECORE_WWN_SIZE 8
 struct ecore_fc_npiv_tbl {
@@ -521,6 +527,10 @@ union ecore_mfw_tlv_data {
 	struct ecore_mfw_tlv_iscsi iscsi;
 };
 
+enum ecore_hw_info_change {
+	ECORE_HW_INFO_CHANGE_OVLAN,
+};
+
 /**
  * @brief - returns the link params of the hw function
  *
@@ -597,6 +607,7 @@ enum _ecore_status_t ecore_mcp_get_media_type(struct ecore_hwfn *p_hwfn,
  *
  * @param p_dev      - ecore dev pointer
  * @param p_ptt
+ * @param p_transceiver_state - transceiver state.
  * @param p_transceiver_type - media type value
  *
  * @return enum _ecore_status_t -
@@ -605,6 +616,7 @@ enum _ecore_status_t ecore_mcp_get_media_type(struct ecore_hwfn *p_hwfn,
  */
 enum _ecore_status_t ecore_mcp_get_transceiver_data(struct ecore_hwfn *p_hwfn,
 						    struct ecore_ptt *p_ptt,
+						    u32 *p_transceiver_state,
 						    u32 *p_tranceiver_type);
 
 /**
@@ -810,6 +822,32 @@ enum _ecore_status_t ecore_mcp_ov_update_mtu(struct ecore_hwfn *p_hwfn,
 					     struct ecore_ptt *p_ptt, u16 mtu);
 
 /**
+ * @brief Send MAC address to MFW
+ *
+ *  @param p_hwfn
+ *  @param p_ptt
+ *  @param mac - MAC address
+ *
+ * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful.
+ */
+enum _ecore_status_t
+ecore_mcp_ov_update_mac(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			u8 *mac);
+
+/**
+ * @brief Send eswitch mode to MFW
+ *
+ *  @param p_hwfn
+ *  @param p_ptt
+ *  @param eswitch - eswitch mode
+ *
+ * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful.
+ */
+enum _ecore_status_t
+ecore_mcp_ov_update_eswitch(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt,
+			    enum ecore_ov_eswitch eswitch);
+
+/**
  * @brief Set LED status
  *
  *  @param p_hwfn
@@ -905,7 +943,7 @@ enum _ecore_status_t ecore_mcp_nvm_resp(struct ecore_dev *p_dev, u8 *p_buf);
  * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful.
  */
 enum _ecore_status_t ecore_mcp_phy_read(struct ecore_dev *p_dev, u32 cmd,
-					u32 addr, u8 *p_buf, u32 len);
+					u32 addr, u8 *p_buf, u32 *p_len);
 
 /**
  * @brief Read from nvm
diff --git a/drivers/net/qede/base/ecore_rt_defs.h b/drivers/net/qede/base/ecore_rt_defs.h
index 721b8c15..3860e1a5 100644
--- a/drivers/net/qede/base/ecore_rt_defs.h
+++ b/drivers/net/qede/base/ecore_rt_defs.h
@@ -390,147 +390,146 @@
 #define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET                   39769
 #define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_SIZE                     16
 #define NIG_REG_TX_EDPM_CTRL_RT_OFFSET                              39785
-#define NIG_REG_ROCE_DUPLICATE_TO_HOST_RT_OFFSET                    39786
-#define NIG_REG_PPF_TO_ENGINE_SEL_RT_OFFSET                         39787
+#define NIG_REG_PPF_TO_ENGINE_SEL_RT_OFFSET                         39786
 #define NIG_REG_PPF_TO_ENGINE_SEL_RT_SIZE                           8
-#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_VALUE_RT_OFFSET              39795
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_VALUE_RT_OFFSET              39794
 #define NIG_REG_LLH_PF_CLS_FUNC_FILTER_VALUE_RT_SIZE                1024
-#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_EN_RT_OFFSET                 40819
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_EN_RT_OFFSET                 40818
 #define NIG_REG_LLH_PF_CLS_FUNC_FILTER_EN_RT_SIZE                   512
-#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_MODE_RT_OFFSET               41331
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_MODE_RT_OFFSET               41330
 #define NIG_REG_LLH_PF_CLS_FUNC_FILTER_MODE_RT_SIZE                 512
-#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET      41843
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET      41842
 #define NIG_REG_LLH_PF_CLS_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE        512
-#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_HDR_SEL_RT_OFFSET            42355
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_HDR_SEL_RT_OFFSET            42354
 #define NIG_REG_LLH_PF_CLS_FUNC_FILTER_HDR_SEL_RT_SIZE              512
-#define NIG_REG_LLH_PF_CLS_FILTERS_MAP_RT_OFFSET                    42867
+#define NIG_REG_LLH_PF_CLS_FILTERS_MAP_RT_OFFSET                    42866
 #define NIG_REG_LLH_PF_CLS_FILTERS_MAP_RT_SIZE                      32
-#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET                           42899
-#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET                           42900
-#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET                           42901
-#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET                       42902
-#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET                       42903
-#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET                       42904
-#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET                       42905
-#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET                    42906
-#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET                    42907
-#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET                    42908
-#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET                    42909
-#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET                        42910
-#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET                     42911
-#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET                           42912
-#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET                      42913
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET                    42914
-#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET                       42915
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET                42916
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET                    42917
-#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET                       42918
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET                42919
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET                    42920
-#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET                       42921
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET                42922
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET                    42923
-#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET                       42924
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET                42925
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET                    42926
-#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET                       42927
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET                42928
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET                    42929
-#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET                       42930
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET                42931
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET                    42932
-#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET                       42933
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET                42934
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET                    42935
-#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET                       42936
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET                42937
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET                    42938
-#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET                       42939
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET                42940
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET                    42941
-#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET                       42942
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET                42943
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET                   42944
-#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET                      42945
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET               42946
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET                   42947
-#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET                      42948
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET               42949
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET                   42950
-#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET                      42951
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET               42952
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET                   42953
-#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET                      42954
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET               42955
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET                   42956
-#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET                      42957
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET               42958
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET                   42959
-#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET                      42960
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET               42961
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET                   42962
-#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET                      42963
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET               42964
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET                   42965
-#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET                      42966
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET               42967
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET                   42968
-#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET                      42969
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET               42970
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET                   42971
-#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET                      42972
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET               42973
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ20_RT_OFFSET                   42974
-#define PBF_REG_BTB_GUARANTEED_VOQ20_RT_OFFSET                      42975
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ20_RT_OFFSET               42976
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ21_RT_OFFSET                   42977
-#define PBF_REG_BTB_GUARANTEED_VOQ21_RT_OFFSET                      42978
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ21_RT_OFFSET               42979
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ22_RT_OFFSET                   42980
-#define PBF_REG_BTB_GUARANTEED_VOQ22_RT_OFFSET                      42981
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ22_RT_OFFSET               42982
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ23_RT_OFFSET                   42983
-#define PBF_REG_BTB_GUARANTEED_VOQ23_RT_OFFSET                      42984
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ23_RT_OFFSET               42985
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ24_RT_OFFSET                   42986
-#define PBF_REG_BTB_GUARANTEED_VOQ24_RT_OFFSET                      42987
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ24_RT_OFFSET               42988
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ25_RT_OFFSET                   42989
-#define PBF_REG_BTB_GUARANTEED_VOQ25_RT_OFFSET                      42990
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ25_RT_OFFSET               42991
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ26_RT_OFFSET                   42992
-#define PBF_REG_BTB_GUARANTEED_VOQ26_RT_OFFSET                      42993
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ26_RT_OFFSET               42994
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ27_RT_OFFSET                   42995
-#define PBF_REG_BTB_GUARANTEED_VOQ27_RT_OFFSET                      42996
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ27_RT_OFFSET               42997
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ28_RT_OFFSET                   42998
-#define PBF_REG_BTB_GUARANTEED_VOQ28_RT_OFFSET                      42999
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ28_RT_OFFSET               43000
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ29_RT_OFFSET                   43001
-#define PBF_REG_BTB_GUARANTEED_VOQ29_RT_OFFSET                      43002
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ29_RT_OFFSET               43003
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ30_RT_OFFSET                   43004
-#define PBF_REG_BTB_GUARANTEED_VOQ30_RT_OFFSET                      43005
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ30_RT_OFFSET               43006
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ31_RT_OFFSET                   43007
-#define PBF_REG_BTB_GUARANTEED_VOQ31_RT_OFFSET                      43008
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ31_RT_OFFSET               43009
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ32_RT_OFFSET                   43010
-#define PBF_REG_BTB_GUARANTEED_VOQ32_RT_OFFSET                      43011
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ32_RT_OFFSET               43012
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ33_RT_OFFSET                   43013
-#define PBF_REG_BTB_GUARANTEED_VOQ33_RT_OFFSET                      43014
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ33_RT_OFFSET               43015
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ34_RT_OFFSET                   43016
-#define PBF_REG_BTB_GUARANTEED_VOQ34_RT_OFFSET                      43017
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ34_RT_OFFSET               43018
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ35_RT_OFFSET                   43019
-#define PBF_REG_BTB_GUARANTEED_VOQ35_RT_OFFSET                      43020
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ35_RT_OFFSET               43021
-#define XCM_REG_CON_PHY_Q3_RT_OFFSET                                43022
+#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET                           42898
+#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET                           42899
+#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET                           42900
+#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET                       42901
+#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET                       42902
+#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET                       42903
+#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET                       42904
+#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET                    42905
+#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET                    42906
+#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET                    42907
+#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET                    42908
+#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET                        42909
+#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET                     42910
+#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET                           42911
+#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET                      42912
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET                    42913
+#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET                       42914
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET                42915
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET                    42916
+#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET                       42917
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET                42918
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET                    42919
+#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET                       42920
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET                42921
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET                    42922
+#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET                       42923
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET                42924
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET                    42925
+#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET                       42926
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET                42927
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET                    42928
+#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET                       42929
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET                42930
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET                    42931
+#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET                       42932
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET                42933
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET                    42934
+#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET                       42935
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET                42936
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET                    42937
+#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET                       42938
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET                42939
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET                    42940
+#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET                       42941
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET                42942
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET                   42943
+#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET                      42944
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET               42945
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET                   42946
+#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET                      42947
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET               42948
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET                   42949
+#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET                      42950
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET               42951
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET                   42952
+#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET                      42953
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET               42954
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET                   42955
+#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET                      42956
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET               42957
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET                   42958
+#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET                      42959
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET               42960
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET                   42961
+#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET                      42962
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET               42963
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET                   42964
+#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET                      42965
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET               42966
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET                   42967
+#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET                      42968
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET               42969
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET                   42970
+#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET                      42971
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET               42972
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ20_RT_OFFSET                   42973
+#define PBF_REG_BTB_GUARANTEED_VOQ20_RT_OFFSET                      42974
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ20_RT_OFFSET               42975
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ21_RT_OFFSET                   42976
+#define PBF_REG_BTB_GUARANTEED_VOQ21_RT_OFFSET                      42977
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ21_RT_OFFSET               42978
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ22_RT_OFFSET                   42979
+#define PBF_REG_BTB_GUARANTEED_VOQ22_RT_OFFSET                      42980
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ22_RT_OFFSET               42981
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ23_RT_OFFSET                   42982
+#define PBF_REG_BTB_GUARANTEED_VOQ23_RT_OFFSET                      42983
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ23_RT_OFFSET               42984
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ24_RT_OFFSET                   42985
+#define PBF_REG_BTB_GUARANTEED_VOQ24_RT_OFFSET                      42986
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ24_RT_OFFSET               42987
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ25_RT_OFFSET                   42988
+#define PBF_REG_BTB_GUARANTEED_VOQ25_RT_OFFSET                      42989
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ25_RT_OFFSET               42990
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ26_RT_OFFSET                   42991
+#define PBF_REG_BTB_GUARANTEED_VOQ26_RT_OFFSET                      42992
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ26_RT_OFFSET               42993
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ27_RT_OFFSET                   42994
+#define PBF_REG_BTB_GUARANTEED_VOQ27_RT_OFFSET                      42995
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ27_RT_OFFSET               42996
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ28_RT_OFFSET                   42997
+#define PBF_REG_BTB_GUARANTEED_VOQ28_RT_OFFSET                      42998
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ28_RT_OFFSET               42999
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ29_RT_OFFSET                   43000
+#define PBF_REG_BTB_GUARANTEED_VOQ29_RT_OFFSET                      43001
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ29_RT_OFFSET               43002
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ30_RT_OFFSET                   43003
+#define PBF_REG_BTB_GUARANTEED_VOQ30_RT_OFFSET                      43004
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ30_RT_OFFSET               43005
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ31_RT_OFFSET                   43006
+#define PBF_REG_BTB_GUARANTEED_VOQ31_RT_OFFSET                      43007
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ31_RT_OFFSET               43008
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ32_RT_OFFSET                   43009
+#define PBF_REG_BTB_GUARANTEED_VOQ32_RT_OFFSET                      43010
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ32_RT_OFFSET               43011
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ33_RT_OFFSET                   43012
+#define PBF_REG_BTB_GUARANTEED_VOQ33_RT_OFFSET                      43013
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ33_RT_OFFSET               43014
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ34_RT_OFFSET                   43015
+#define PBF_REG_BTB_GUARANTEED_VOQ34_RT_OFFSET                      43016
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ34_RT_OFFSET               43017
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ35_RT_OFFSET                   43018
+#define PBF_REG_BTB_GUARANTEED_VOQ35_RT_OFFSET                      43019
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ35_RT_OFFSET               43020
+#define XCM_REG_CON_PHY_Q3_RT_OFFSET                                43021
 
-#define RUNTIME_ARRAY_SIZE 43023
+#define RUNTIME_ARRAY_SIZE 43022
 
 /* Init Callbacks */
 #define DMAE_READY_CB                                               0
diff --git a/drivers/net/qede/base/ecore_sp_commands.c b/drivers/net/qede/base/ecore_sp_commands.c
index b43baf9d..49a5ff55 100644
--- a/drivers/net/qede/base/ecore_sp_commands.c
+++ b/drivers/net/qede/base/ecore_sp_commands.c
@@ -515,6 +515,10 @@ enum _ecore_status_t ecore_sp_rl_update(struct ecore_hwfn *p_hwfn,
 	rl_update->rl_id_first = params->rl_id_first;
 	rl_update->rl_id_last = params->rl_id_last;
 	rl_update->rl_dc_qcn_flg = params->rl_dc_qcn_flg;
+	rl_update->dcqcn_reset_alpha_on_idle =
+		params->dcqcn_reset_alpha_on_idle;
+	rl_update->rl_bc_stage_th = params->rl_bc_stage_th;
+	rl_update->rl_timer_stage_th = params->rl_timer_stage_th;
 	rl_update->rl_bc_rate = OSAL_CPU_TO_LE32(params->rl_bc_rate);
 	rl_update->rl_max_rate =
 		OSAL_CPU_TO_LE16(ecore_sp_rl_mb_to_qm(params->rl_max_rate));
@@ -529,12 +533,14 @@ enum _ecore_status_t ecore_sp_rl_update(struct ecore_hwfn *p_hwfn,
 		OSAL_CPU_TO_LE32(params->dcqcn_timeuot_us);
 	rl_update->qcn_timeuot_us = OSAL_CPU_TO_LE32(params->qcn_timeuot_us);
 
-	DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "rl_params: qcn_update_param_flg %x, dcqcn_update_param_flg %x, rl_init_flg %x, rl_start_flg %x, rl_stop_flg %x, rl_id_first %x, rl_id_last %x, rl_dc_qcn_flg %x, rl_bc_rate %x, rl_max_rate %x, rl_r_ai %x, rl_r_hai %x, dcqcn_g %x, dcqcn_k_us %x, dcqcn_timeuot_us %x, qcn_timeuot_us %x\n",
+	DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "rl_params: qcn_update_param_flg %x, dcqcn_update_param_flg %x, rl_init_flg %x, rl_start_flg %x, rl_stop_flg %x, rl_id_first %x, rl_id_last %x, rl_dc_qcn_flg %x,dcqcn_reset_alpha_on_idle %x, rl_bc_stage_th %x, rl_timer_stage_th %x, rl_bc_rate %x, rl_max_rate %x, rl_r_ai %x, rl_r_hai %x, dcqcn_g %x, dcqcn_k_us %x, dcqcn_timeuot_us %x, qcn_timeuot_us %x\n",
 		   rl_update->qcn_update_param_flg,
 		   rl_update->dcqcn_update_param_flg,
 		   rl_update->rl_init_flg, rl_update->rl_start_flg,
 		   rl_update->rl_stop_flg, rl_update->rl_id_first,
 		   rl_update->rl_id_last, rl_update->rl_dc_qcn_flg,
+		   rl_update->dcqcn_reset_alpha_on_idle,
+		   rl_update->rl_bc_stage_th, rl_update->rl_timer_stage_th,
 		   rl_update->rl_bc_rate, rl_update->rl_max_rate,
 		   rl_update->rl_r_ai, rl_update->rl_r_hai,
 		   rl_update->dcqcn_g, rl_update->dcqcn_k_us,
diff --git a/drivers/net/qede/base/ecore_sp_commands.h b/drivers/net/qede/base/ecore_sp_commands.h
index e57414cf..524fe57a 100644
--- a/drivers/net/qede/base/ecore_sp_commands.h
+++ b/drivers/net/qede/base/ecore_sp_commands.h
@@ -119,6 +119,9 @@ struct ecore_rl_update_params {
 	u8 rl_stop_flg;
 	u8 rl_id_first;
 	u8 rl_id_last;
+	u8 dcqcn_reset_alpha_on_idle;
+	u8 rl_bc_stage_th;
+	u8 rl_timer_stage_th;
 	u8 rl_dc_qcn_flg; /* If set, RL will used for DCQCN */
 	u32 rl_bc_rate; /* Byte Counter Limit */
 	u32 rl_max_rate; /* Maximum rate in Mbps resolution */
diff --git a/drivers/net/qede/base/ecore_spq.c b/drivers/net/qede/base/ecore_spq.c
index 776c86f7..88ad961e 100644
--- a/drivers/net/qede/base/ecore_spq.c
+++ b/drivers/net/qede/base/ecore_spq.c
@@ -282,20 +282,30 @@ ecore_async_event_completion(struct ecore_hwfn *p_hwfn,
 			     struct event_ring_entry *p_eqe)
 {
 	ecore_spq_async_comp_cb cb;
+	enum _ecore_status_t rc;
 
-	if (!p_hwfn->p_spq || (p_eqe->protocol_id >= MAX_PROTOCOL_TYPE))
+	if (p_eqe->protocol_id >= MAX_PROTOCOL_TYPE) {
+		DP_ERR(p_hwfn, "Wrong protocol: %d\n", p_eqe->protocol_id);
 		return ECORE_INVAL;
+	}
 
 	cb = p_hwfn->p_spq->async_comp_cb[p_eqe->protocol_id];
-	if (cb) {
-		return cb(p_hwfn, p_eqe->opcode, p_eqe->echo,
-			  &p_eqe->data, p_eqe->fw_return_code);
-	} else {
+	if (!cb) {
 		DP_NOTICE(p_hwfn,
 			  true, "Unknown Async completion for protocol: %d\n",
 			  p_eqe->protocol_id);
 		return ECORE_INVAL;
 	}
+
+	rc = cb(p_hwfn, p_eqe->opcode, p_eqe->echo,
+		&p_eqe->data, p_eqe->fw_return_code);
+	if (rc != ECORE_SUCCESS)
+		DP_NOTICE(p_hwfn, true,
+			  "Async completion callback failed, rc = %d [opcode %x, echo %x, fw_return_code %x]",
+			  rc, p_eqe->opcode, p_eqe->echo,
+			  p_eqe->fw_return_code);
+
+	return rc;
 }
 
 enum _ecore_status_t
@@ -339,10 +349,16 @@ enum _ecore_status_t ecore_eq_completion(struct ecore_hwfn *p_hwfn,
 {
 	struct ecore_eq *p_eq = cookie;
 	struct ecore_chain *p_chain = &p_eq->chain;
-	enum _ecore_status_t rc = 0;
+	u16 fw_cons_idx             = 0;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+
+	if (!p_hwfn->p_spq) {
+		DP_ERR(p_hwfn, "Unexpected NULL p_spq\n");
+		return ECORE_INVAL;
+	}
 
 	/* take a snapshot of the FW consumer */
-	u16 fw_cons_idx = OSAL_LE16_TO_CPU(*p_eq->p_fw_cons);
+	fw_cons_idx = OSAL_LE16_TO_CPU(*p_eq->p_fw_cons);
 
 	DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "fw_cons_idx %x\n", fw_cons_idx);
 
@@ -358,7 +374,8 @@ enum _ecore_status_t ecore_eq_completion(struct ecore_hwfn *p_hwfn,
 	while (fw_cons_idx != ecore_chain_get_cons_idx(p_chain)) {
 		struct event_ring_entry *p_eqe = ecore_chain_consume(p_chain);
 		if (!p_eqe) {
-			rc = ECORE_INVAL;
+			DP_ERR(p_hwfn,
+			       "Unexpected NULL chain consumer entry\n");
 			break;
 		}
 
@@ -374,15 +391,13 @@ enum _ecore_status_t ecore_eq_completion(struct ecore_hwfn *p_hwfn,
 						      */
 			   p_eqe->flags);
 
-		if (GET_FIELD(p_eqe->flags, EVENT_RING_ENTRY_ASYNC)) {
-			if (ecore_async_event_completion(p_hwfn, p_eqe))
-				rc = ECORE_INVAL;
-		} else if (ecore_spq_completion(p_hwfn,
-						p_eqe->echo,
-						p_eqe->fw_return_code,
-						&p_eqe->data)) {
-			rc = ECORE_INVAL;
-		}
+		if (GET_FIELD(p_eqe->flags, EVENT_RING_ENTRY_ASYNC))
+			ecore_async_event_completion(p_hwfn, p_eqe);
+		else
+			ecore_spq_completion(p_hwfn,
+					     p_eqe->echo,
+					     p_eqe->fw_return_code,
+					     &p_eqe->data);
 
 		ecore_chain_recycle_consumed(p_chain);
 	}
@@ -928,12 +943,11 @@ enum _ecore_status_t ecore_spq_completion(struct ecore_hwfn *p_hwfn,
 	struct ecore_spq_entry *found = OSAL_NULL;
 	enum _ecore_status_t rc;
 
-	if (!p_hwfn)
-		return ECORE_INVAL;
-
 	p_spq = p_hwfn->p_spq;
-	if (!p_spq)
+	if (!p_spq) {
+		DP_ERR(p_hwfn, "Unexpected NULL p_spq\n");
 		return ECORE_INVAL;
+	}
 
 	OSAL_SPIN_LOCK(&p_spq->lock);
 	OSAL_LIST_FOR_EACH_ENTRY_SAFE(p_ent,
diff --git a/drivers/net/qede/base/ecore_sriov.c b/drivers/net/qede/base/ecore_sriov.c
index f7ebf7ad..7d73ef9f 100644
--- a/drivers/net/qede/base/ecore_sriov.c
+++ b/drivers/net/qede/base/ecore_sriov.c
@@ -31,7 +31,7 @@ static enum _ecore_status_t ecore_sriov_eqe_event(struct ecore_hwfn *p_hwfn,
 						  union event_ring_data *data,
 						  u8 fw_return_code);
 
-const char *ecore_channel_tlvs_string[] = {
+const char *qede_ecore_channel_tlvs_string[] = {
 	"CHANNEL_TLV_NONE",	/* ends tlv sequence */
 	"CHANNEL_TLV_ACQUIRE",
 	"CHANNEL_TLV_VPORT_START",
@@ -218,7 +218,7 @@ struct ecore_vf_info *ecore_iov_get_vf_info(struct ecore_hwfn *p_hwfn,
 static struct ecore_queue_cid *
 ecore_iov_get_vf_rx_queue_cid(struct ecore_vf_queue *p_queue)
 {
-	int i;
+	u32 i;
 
 	for (i = 0; i < MAX_QUEUES_PER_QZONE; i++) {
 		if (p_queue->cids[i].p_cid &&
@@ -240,7 +240,7 @@ static bool ecore_iov_validate_queue_mode(struct ecore_vf_info *p_vf,
 					  enum ecore_iov_validate_q_mode mode,
 					  bool b_is_tx)
 {
-	int i;
+	u32 i;
 
 	if (mode == ECORE_IOV_VALIDATE_Q_NA)
 		return true;
@@ -979,10 +979,12 @@ static u8 ecore_iov_alloc_vf_igu_sbs(struct ecore_hwfn *p_hwfn,
 		ecore_init_cau_sb_entry(p_hwfn, &sb_entry,
 					p_hwfn->rel_pf_id,
 					vf->abs_vf_id, 1);
+
 		ecore_dmae_host2grc(p_hwfn, p_ptt,
 				    (u64)(osal_uintptr_t)&sb_entry,
 				    CAU_REG_SB_VAR_MEMORY +
-				    p_block->igu_sb_id * sizeof(u64), 2, 0);
+				    p_block->igu_sb_id * sizeof(u64), 2,
+				    OSAL_NULL /* default parameters */);
 	}
 
 	vf->num_sbs = (u8)num_rx_queues;
@@ -1278,7 +1280,7 @@ static void ecore_iov_lock_vf_pf_channel(struct ecore_hwfn *p_hwfn,
 			   ECORE_MSG_IOV,
 			   "VF[%d]: vf pf channel locked by %s\n",
 			   vf->abs_vf_id,
-			   ecore_channel_tlvs_string[tlv]);
+			   qede_ecore_channel_tlvs_string[tlv]);
 	else
 		DP_VERBOSE(p_hwfn,
 			   ECORE_MSG_IOV,
@@ -1296,7 +1298,7 @@ static void ecore_iov_unlock_vf_pf_channel(struct ecore_hwfn *p_hwfn,
 			   ECORE_MSG_IOV,
 			   "VF[%d]: vf pf channel unlocked by %s\n",
 			   vf->abs_vf_id,
-			   ecore_channel_tlvs_string[expected_tlv]);
+			   qede_ecore_channel_tlvs_string[expected_tlv]);
 	else
 		DP_VERBOSE(p_hwfn,
 			   ECORE_MSG_IOV,
@@ -1336,7 +1338,7 @@ void ecore_dp_tlv_list(struct ecore_hwfn *p_hwfn, void *tlvs_list)
 		if (ecore_iov_tlv_supported(tlv->type))
 			DP_VERBOSE(p_hwfn, ECORE_MSG_IOV,
 				   "TLV number %d: type %s, length %d\n",
-				   i, ecore_channel_tlvs_string[tlv->type],
+				   i, qede_ecore_channel_tlvs_string[tlv->type],
 				   tlv->length);
 		else
 			DP_VERBOSE(p_hwfn, ECORE_MSG_IOV,
@@ -1968,7 +1970,8 @@ ecore_iov_configure_vport_forced(struct ecore_hwfn *p_hwfn,
 		return ECORE_INVAL;
 
 	if ((events & (1 << MAC_ADDR_FORCED)) ||
-	    p_hwfn->pf_params.eth_pf_params.allow_vf_mac_change) {
+	    p_hwfn->pf_params.eth_pf_params.allow_vf_mac_change ||
+	    p_vf->p_vf_info.is_trusted_configured) {
 		/* Since there's no way [currently] of removing the MAC,
 		 * we can always assume this means we need to force it.
 		 */
@@ -1989,7 +1992,8 @@ ecore_iov_configure_vport_forced(struct ecore_hwfn *p_hwfn,
 			return rc;
 		}
 
-		if (p_hwfn->pf_params.eth_pf_params.allow_vf_mac_change)
+		if (p_hwfn->pf_params.eth_pf_params.allow_vf_mac_change ||
+		    p_vf->p_vf_info.is_trusted_configured)
 			p_vf->configured_features |=
 				1 << VFPF_BULLETIN_MAC_ADDR;
 		else
@@ -2085,8 +2089,8 @@ static void ecore_iov_vf_mbx_start_vport(struct ecore_hwfn *p_hwfn,
 					 struct ecore_ptt *p_ptt,
 					 struct ecore_vf_info *vf)
 {
-	struct ecore_sp_vport_start_params params = { 0 };
 	struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx;
+	struct ecore_sp_vport_start_params params;
 	struct vfpf_vport_start_tlv *start;
 	u8 status = PFVF_STATUS_SUCCESS;
 	struct ecore_vf_info *vf_info;
@@ -2137,6 +2141,7 @@ static void ecore_iov_vf_mbx_start_vport(struct ecore_hwfn *p_hwfn,
 		*p_bitmap |= 1 << VFPF_BULLETIN_UNTAGGED_DEFAULT;
 	}
 
+	OSAL_MEMSET(&params, 0, sizeof(struct ecore_sp_vport_start_params));
 	params.tpa_mode = start->tpa_mode;
 	params.remove_inner_vlan = start->inner_vlan_removal;
 	params.tx_switching = true;
@@ -2156,7 +2161,9 @@ static void ecore_iov_vf_mbx_start_vport(struct ecore_hwfn *p_hwfn,
 	params.vport_id = vf->vport_id;
 	params.max_buffers_per_cqe = start->max_buffers_per_cqe;
 	params.mtu = vf->mtu;
-	params.check_mac = true;
+
+	/* Non trusted VFs should enable control frame filtering */
+	params.check_mac = !vf->p_vf_info.is_trusted_configured;
 
 	rc = ecore_sp_eth_vport_start(p_hwfn, &params);
 	if (rc != ECORE_SUCCESS) {
@@ -2912,7 +2919,7 @@ void *ecore_iov_search_list_tlvs(struct ecore_hwfn *p_hwfn,
 		if (p_tlv->type == req_type) {
 			DP_VERBOSE(p_hwfn, ECORE_MSG_IOV,
 				   "Extended tlv type %s, length %d found\n",
-				   ecore_channel_tlvs_string[p_tlv->type],
+				   qede_ecore_channel_tlvs_string[p_tlv->type],
 				   p_tlv->length);
 			return p_tlv;
 		}
@@ -3351,6 +3358,15 @@ ecore_iov_vf_update_mac_shadow(struct ecore_hwfn *p_hwfn,
 	if (p_vf->bulletin.p_virt->valid_bitmap & (1 << MAC_ADDR_FORCED))
 		return ECORE_SUCCESS;
 
+	/* Since we don't have the implementation of the logic for removing
+	 * a forced MAC and restoring shadow MAC, let's not worry about
+	 * processing shadow copies of MAC as long as VF trust mode is ON,
+	 * to keep things simple.
+	 */
+	if (p_hwfn->pf_params.eth_pf_params.allow_vf_mac_change ||
+	    p_vf->p_vf_info.is_trusted_configured)
+		return ECORE_SUCCESS;
+
 	/* First remove entries and then add new ones */
 	if (p_params->opcode == ECORE_FILTER_REMOVE) {
 		for (i = 0; i < ECORE_ETH_VF_NUM_MAC_FILTERS; i++) {
@@ -3653,7 +3669,7 @@ static void ecore_iov_vf_pf_set_coalesce(struct ecore_hwfn *p_hwfn,
 	struct ecore_queue_cid *p_cid;
 	u16 rx_coal, tx_coal;
 	u16 qid;
-	int i;
+	u32 i;
 
 	req = &mbx->req_virt->update_coalesce;
 
@@ -3733,7 +3749,8 @@ ecore_iov_pf_configure_vf_queue_coalesce(struct ecore_hwfn *p_hwfn,
 	struct ecore_queue_cid *p_cid;
 	struct ecore_vf_info *vf;
 	struct ecore_ptt *p_ptt;
-	int i, rc = 0;
+	int rc = 0;
+	u32 i;
 
 	if (!ecore_iov_is_valid_vfid(p_hwfn, vf_id, true, true)) {
 		DP_NOTICE(p_hwfn, true,
@@ -4415,17 +4432,23 @@ void ecore_iov_bulletin_set_forced_mac(struct ecore_hwfn *p_hwfn,
 		return;
 	}
 
-	if (p_hwfn->pf_params.eth_pf_params.allow_vf_mac_change)
+	if (p_hwfn->pf_params.eth_pf_params.allow_vf_mac_change ||
+	    vf_info->p_vf_info.is_trusted_configured) {
 		feature = 1 << VFPF_BULLETIN_MAC_ADDR;
-	else
+		/* Trust mode will disable Forced MAC */
+		vf_info->bulletin.p_virt->valid_bitmap &=
+			~(1 << MAC_ADDR_FORCED);
+	} else {
 		feature = 1 << MAC_ADDR_FORCED;
+		/* Forced MAC will disable MAC_ADDR */
+		vf_info->bulletin.p_virt->valid_bitmap &=
+			~(1 << VFPF_BULLETIN_MAC_ADDR);
+	}
 
-	OSAL_MEMCPY(vf_info->bulletin.p_virt->mac, mac, ETH_ALEN);
+	OSAL_MEMCPY(vf_info->bulletin.p_virt->mac,
+		    mac, ETH_ALEN);
 
 	vf_info->bulletin.p_virt->valid_bitmap |= feature;
-	/* Forced MAC will disable MAC_ADDR */
-	vf_info->bulletin.p_virt->valid_bitmap &=
-	    ~(1 << VFPF_BULLETIN_MAC_ADDR);
 
 	ecore_iov_configure_vport_forced(p_hwfn, vf_info, feature);
 }
@@ -4460,7 +4483,8 @@ enum _ecore_status_t ecore_iov_bulletin_set_mac(struct ecore_hwfn *p_hwfn,
 
 	vf_info->bulletin.p_virt->valid_bitmap |= feature;
 
-	if (p_hwfn->pf_params.eth_pf_params.allow_vf_mac_change)
+	if (p_hwfn->pf_params.eth_pf_params.allow_vf_mac_change ||
+	    vf_info->p_vf_info.is_trusted_configured)
 		ecore_iov_configure_vport_forced(p_hwfn, vf_info, feature);
 
 	return ECORE_SUCCESS;
@@ -4780,6 +4804,32 @@ enum _ecore_status_t ecore_iov_configure_tx_rate(struct ecore_hwfn *p_hwfn,
 				   p_link->speed);
 }
 
+enum _ecore_status_t ecore_iov_configure_min_tx_rate(struct ecore_dev *p_dev,
+						     int vfid, u32 rate)
+{
+	struct ecore_vf_info *vf;
+	int i;
+
+	for_each_hwfn(p_dev, i) {
+		struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i];
+
+		if (!ecore_iov_pf_sanity_check(p_hwfn, vfid)) {
+			DP_NOTICE(p_hwfn, true,
+				  "SR-IOV sanity check failed, can't set min rate\n");
+			return ECORE_INVAL;
+		}
+	}
+
+	vf = ecore_iov_get_vf_info(ECORE_LEADING_HWFN(p_dev), (u16)vfid, true);
+	if (!vf) {
+		DP_NOTICE(p_dev, true,
+			  "Getting vf info failed, can't set min rate\n");
+		return ECORE_INVAL;
+	}
+
+	return ecore_configure_vport_wfq(p_dev, vf->vport_id, rate);
+}
+
 enum _ecore_status_t ecore_iov_get_vf_stats(struct ecore_hwfn *p_hwfn,
 					    struct ecore_ptt *p_ptt,
 					    int vfid,
@@ -4890,7 +4940,7 @@ bool ecore_iov_is_vf_started(struct ecore_hwfn *p_hwfn,
 	return (p_vf->state != VF_FREE && p_vf->state != VF_STOPPED);
 }
 
-enum _ecore_status_t
+int
 ecore_iov_get_vf_min_rate(struct ecore_hwfn *p_hwfn, int vfid)
 {
 	struct ecore_wfq_data *vf_vp_wfq;
diff --git a/drivers/net/qede/base/ecore_vf.c b/drivers/net/qede/base/ecore_vf.c
index d2213f79..3ba6a0cf 100644
--- a/drivers/net/qede/base/ecore_vf.c
+++ b/drivers/net/qede/base/ecore_vf.c
@@ -32,7 +32,7 @@ static void *ecore_vf_pf_prep(struct ecore_hwfn *p_hwfn, u16 type, u16 length)
 
 	DP_VERBOSE(p_hwfn, ECORE_MSG_IOV,
 		   "preparing to send %s tlv over vf pf channel\n",
-		   ecore_channel_tlvs_string[type]);
+		   qede_ecore_channel_tlvs_string[type]);
 
 	/* Reset Request offset */
 	p_iov->offset = (u8 *)(p_iov->vf2pf_request);
@@ -565,13 +565,20 @@ enum _ecore_status_t ecore_vf_hw_prepare(struct ecore_hwfn *p_hwfn)
 							   phys,
 							   p_iov->bulletin.
 							   size);
+	if (!p_iov->bulletin.p_virt) {
+		DP_NOTICE(p_hwfn, false, "Failed to alloc bulletin memory\n");
+		goto free_pf2vf_reply;
+	}
 	DP_VERBOSE(p_hwfn, ECORE_MSG_IOV,
 		   "VF's bulletin Board [%p virt 0x%lx phys 0x%08x bytes]\n",
 		   p_iov->bulletin.p_virt, (unsigned long)p_iov->bulletin.phys,
 		   p_iov->bulletin.size);
 
 #ifdef CONFIG_ECORE_LOCK_ALLOC
-	OSAL_MUTEX_ALLOC(p_hwfn, &p_iov->mutex);
+	if (OSAL_MUTEX_ALLOC(p_hwfn, &p_iov->mutex)) {
+		DP_NOTICE(p_hwfn, false, "Failed to allocate p_iov->mutex\n");
+		goto free_bulletin_mem;
+	}
 #endif
 	OSAL_MUTEX_INIT(&p_iov->mutex);
 
@@ -609,6 +616,16 @@ enum _ecore_status_t ecore_vf_hw_prepare(struct ecore_hwfn *p_hwfn)
 
 	return rc;
 
+#ifdef CONFIG_ECORE_LOCK_ALLOC
+free_bulletin_mem:
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_iov->bulletin.p_virt,
+			       p_iov->bulletin.phys,
+			       p_iov->bulletin.size);
+#endif
+free_pf2vf_reply:
+	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_iov->pf2vf_reply,
+			       p_iov->pf2vf_reply_phys,
+			       sizeof(union pfvf_tlvs));
 free_vf2pf_request:
 	OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_iov->vf2pf_request,
 			       p_iov->vf2pf_request_phys,
@@ -1167,7 +1184,7 @@ ecore_vf_handle_vp_update_is_needed(struct ecore_hwfn *p_hwfn,
 		return !!p_data->sge_tpa_params;
 	default:
 		DP_INFO(p_hwfn, "Unexpected vport-update TLV[%d] %s\n",
-			tlv, ecore_channel_tlvs_string[tlv]);
+			tlv, qede_ecore_channel_tlvs_string[tlv]);
 		return false;
 	}
 }
@@ -1191,7 +1208,7 @@ ecore_vf_handle_vp_update_tlvs_resp(struct ecore_hwfn *p_hwfn,
 		if (p_resp && p_resp->hdr.status)
 			DP_VERBOSE(p_hwfn, ECORE_MSG_IOV,
 				   "TLV[%d] type %s Configuration %s\n",
-				   tlv, ecore_channel_tlvs_string[tlv],
+				   tlv, qede_ecore_channel_tlvs_string[tlv],
 				   (p_resp && p_resp->hdr.status) ? "succeeded"
 								  : "failed");
 	}
diff --git a/drivers/net/qede/base/ecore_vfpf_if.h b/drivers/net/qede/base/ecore_vfpf_if.h
index c30677ab..c7ecb01c 100644
--- a/drivers/net/qede/base/ecore_vfpf_if.h
+++ b/drivers/net/qede/base/ecore_vfpf_if.h
@@ -698,6 +698,6 @@ enum {
 
 /*!!!!! Make sure to update STRINGS structure accordingly !!!!!*/
 };
-extern const char *ecore_channel_tlvs_string[];
+extern const char *qede_ecore_channel_tlvs_string[];
 
 #endif /* __ECORE_VF_PF_IF_H__ */
diff --git a/drivers/net/qede/base/eth_common.h b/drivers/net/qede/base/eth_common.h
index abfa6854..9a401ed4 100644
--- a/drivers/net/qede/base/eth_common.h
+++ b/drivers/net/qede/base/eth_common.h
@@ -178,6 +178,11 @@ struct eth_tx_1st_bd_flags {
 #define ETH_TX_1ST_BD_FLAGS_TUNN_IP_CSUM_SHIFT    6
 /* Recalculate Tunnel UDP/GRE Checksum (Depending on Tunnel Type) */
 #define ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_MASK     0x1
+/* Recalculate Tunnel UDP/GRE Checksum (Depending on Tunnel Type). In case of
+ * GRE tunnel, this flag means GRE CSO, and in this case GRE checksum field
+ * Must be present.
+ */
+#define ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_MASK     0x1
 #define ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_SHIFT    7
 };
 
diff --git a/drivers/net/qede/base/mcp_public.h b/drivers/net/qede/base/mcp_public.h
index 81aa88e7..13c2e2d1 100644
--- a/drivers/net/qede/base/mcp_public.h
+++ b/drivers/net/qede/base/mcp_public.h
@@ -1258,6 +1258,17 @@ struct public_drv_mb {
  */
 #define DRV_MSG_GET_RESOURCE_ALLOC_MSG		0x34000000
 #define DRV_MSG_SET_RESOURCE_VALUE_MSG		0x35000000
+#define DRV_MSG_CODE_OV_UPDATE_WOL		0x38000000
+#define DRV_MSG_CODE_OV_UPDATE_ESWITCH_MODE	0x39000000
+#define DRV_MSG_CODE_S_TAG_UPDATE_ACK		0x3b000000
+#define DRV_MSG_CODE_OEM_UPDATE_FCOE_CVID	0x3c000000
+#define DRV_MSG_CODE_OEM_UPDATE_FCOE_FABRIC_NAME	0x3d000000
+#define DRV_MSG_CODE_OEM_UPDATE_BOOT_CFG	0x3e000000
+#define DRV_MSG_CODE_OEM_RESET_TO_DEFAULT	0x3f000000
+#define DRV_MSG_CODE_OV_GET_CURR_CFG		0x40000000
+#define DRV_MSG_CODE_GET_OEM_UPDATES		0x41000000
+/* params [31:8] - reserved, [7:0] - bitmap */
+#define DRV_MSG_CODE_GET_PPFID_BITMAP		0x43000000
 
 /*deprecated don't use*/
 #define DRV_MSG_CODE_INITIATE_FLR_DEPRECATED    0x02000000
@@ -1467,6 +1478,7 @@ struct public_drv_mb {
 
 /* Param: Password len. Union: Plain Password */
 #define DRV_MSG_CODE_ENCRYPT_PASSWORD		0x00360000
+#define DRV_MSG_CODE_GET_ENGINE_CONFIG		0x00370000 /* Param: None */
 
 #define DRV_MSG_SEQ_NUMBER_MASK                 0x0000ffff
 
@@ -1582,6 +1594,16 @@ struct public_drv_mb {
 #define DRV_MB_PARAM_OV_MTU_SIZE_OFFSET		0
 #define DRV_MB_PARAM_OV_MTU_SIZE_MASK		0xFFFFFFFF
 
+#define DRV_MB_PARAM_ESWITCH_MODE_MASK  (DRV_MB_PARAM_ESWITCH_MODE_NONE | \
+					 DRV_MB_PARAM_ESWITCH_MODE_VEB |   \
+					 DRV_MB_PARAM_ESWITCH_MODE_VEPA)
+#define DRV_MB_PARAM_ESWITCH_MODE_NONE  0x0
+#define DRV_MB_PARAM_ESWITCH_MODE_VEB   0x1
+#define DRV_MB_PARAM_ESWITCH_MODE_VEPA  0x2
+
+#define DRV_MB_PARAM_DUMMY_OEM_UPDATES_MASK     0x1
+#define DRV_MB_PARAM_DUMMY_OEM_UPDATES_OFFSET   0
+
 #define DRV_MB_PARAM_SET_LED_MODE_OPER		0x0
 #define DRV_MB_PARAM_SET_LED_MODE_ON		0x1
 #define DRV_MB_PARAM_SET_LED_MODE_OFF		0x2
@@ -1677,6 +1699,8 @@ struct public_drv_mb {
 #define FW_MSG_CODE_RESOURCE_ALLOC_UNKNOWN      0x35000000
 #define FW_MSG_CODE_RESOURCE_ALLOC_DEPRECATED   0x36000000
 #define FW_MSG_CODE_RESOURCE_ALLOC_GEN_ERR      0x37000000
+#define FW_MSG_CODE_GET_OEM_UPDATES_DONE	0x41000000
+
 #define FW_MSG_CODE_NIG_DRAIN_DONE              0x30000000
 #define FW_MSG_CODE_VF_DISABLED_DONE            0xb0000000
 #define FW_MSG_CODE_DRV_CFG_VF_MSIX_DONE        0xb0010000
@@ -1778,11 +1802,31 @@ struct public_drv_mb {
 #define FW_MB_PARAM_FEATURE_SUPPORT_EEE         0x00000002
 /* MFW supports DRV_LOAD Timeout */
 #define FW_MB_PARAM_FEATURE_SUPPORT_DRV_LOAD_TO  0x00000004
+/* MFW support complete IGU cleanup upon FLR */
+#define FW_MB_PARAM_FEATURE_SUPPORT_IGU_CLEANUP	0x00000080
 /* MFW supports virtual link */
 #define FW_MB_PARAM_FEATURE_SUPPORT_VLINK       0x00010000
 
 #define FW_MB_PARAM_LOAD_DONE_DID_EFUSE_ERROR	(1 << 0)
 
+#define FW_MB_PARAM_OEM_UPDATE_MASK		0xFF
+#define FW_MB_PARAM_OEM_UPDATE_OFFSET		0
+#define FW_MB_PARAM_OEM_UPDATE_BW		0x01
+#define FW_MB_PARAM_OEM_UPDATE_S_TAG		0x02
+#define FW_MB_PARAM_OEM_UPDATE_CFG		0x04
+
+#define FW_MB_PARAM_ENG_CFG_FIR_AFFIN_VALID_MASK   0x00000001
+#define FW_MB_PARAM_ENG_CFG_FIR_AFFIN_VALID_OFFSET 0
+#define FW_MB_PARAM_ENG_CFG_FIR_AFFIN_VALUE_MASK   0x00000002
+#define FW_MB_PARAM_ENG_CFG_FIR_AFFIN_VALUE_OFFSET 1
+#define FW_MB_PARAM_ENG_CFG_L2_AFFIN_VALID_MASK    0x00000004
+#define FW_MB_PARAM_ENG_CFG_L2_AFFIN_VALID_OFFSET  2
+#define FW_MB_PARAM_ENG_CFG_L2_AFFIN_VALUE_MASK    0x00000008
+#define FW_MB_PARAM_ENG_CFG_L2_AFFIN_VALUE_OFFSET  3
+
+#define FW_MB_PARAM_PPFID_BITMAP_MASK   0xFF
+#define FW_MB_PARAM_PPFID_BITMAP_OFFSET    0
+
 	u32 drv_pulse_mb;
 #define DRV_PULSE_SEQ_MASK                      0x00007fff
 #define DRV_PULSE_SYSTEM_TIME_MASK              0xffff0000
diff --git a/drivers/net/qede/base/meson.build b/drivers/net/qede/base/meson.build
new file mode 100644
index 00000000..71b89737
--- /dev/null
+++ b/drivers/net/qede/base/meson.build
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
+
+sources = [
+	'bcm_osal.c',
+	'ecore_cxt.c',
+	'ecore_dcbx.c',
+	'ecore_dev.c',
+	'ecore_hw.c',
+	'ecore_init_fw_funcs.c',
+	'ecore_init_ops.c',
+	'ecore_int.c',
+	'ecore_l2.c',
+	'ecore_mcp.c',
+	'ecore_sp_commands.c',
+	'ecore_spq.c',
+	'ecore_sriov.c',
+	'ecore_vf.c',
+]
+
+
+error_cflags = [
+	'-Wno-unused-parameter',
+	'-Wno-sign-compare',
+	'-Wno-missing-prototypes',
+	'-Wno-cast-qual',
+	'-Wno-unused-function',
+	'-Wno-unused-variable',
+	'-Wno-strict-aliasing',
+	'-Wno-missing-prototypes',
+	'-Wno-unused-value',
+	'-Wno-format-nonliteral',
+	'-Wno-shift-negative-value',
+	'-Wno-unused-but-set-variable',
+	'-Wno-missing-declarations',
+	'-Wno-maybe-uninitialized',
+	'-Wno-strict-prototypes',
+	'-Wno-shift-negative-value',
+	'-Wno-implicit-fallthrough',
+	'-Wno-format-extra-args',
+	'-Wno-visibility',
+	'-Wno-empty-body',
+	'-Wno-invalid-source-encoding',
+	'-Wno-sometimes-uninitialized',
+	'-Wno-pointer-bool-conversion',
+]
+c_args = cflags
+if allow_experimental_apis
+	c_args += '-DALLOW_EXPERIMENTAL_API'
+endif
+foreach flag: error_cflags
+        if cc.has_argument(flag)
+                c_args += flag
+        endif
+endforeach
+
+base_lib = static_library('qede_base', sources,
+	dependencies: static_rte_net,
+	c_args: c_args)
+base_objs = base_lib.extract_all_objects()
diff --git a/drivers/net/qede/base/reg_addr.h b/drivers/net/qede/base/reg_addr.h
index 402f6204..be59f773 100644
--- a/drivers/net/qede/base/reg_addr.h
+++ b/drivers/net/qede/base/reg_addr.h
@@ -8,13 +8,13 @@
 	0
 
 #define  CDU_REG_CID_ADDR_PARAMS_CONTEXT_SIZE		( \
-		0xfff << 0)
+		0xfffUL << 0)
 
 #define  CDU_REG_CID_ADDR_PARAMS_BLOCK_WASTE_SHIFT \
 	12
 
 #define  CDU_REG_CID_ADDR_PARAMS_BLOCK_WASTE		( \
-		0xfff << 12)
+		0xfffUL << 12)
 
 #define  CDU_REG_CID_ADDR_PARAMS_NCIB_SHIFT \
 	24
@@ -322,6 +322,21 @@
 	0x180820UL
 #define  IGU_REG_ATTN_MSG_ADDR_H	\
 	0x180824UL
+#define IGU_REG_LEADING_EDGE_LATCH \
+	0x18082cUL
+#define IGU_REG_TRAILING_EDGE_LATCH \
+	0x180830UL
+#define IGU_REG_ATTENTION_ACK_BITS \
+	0x180838UL
+#define IGU_REG_PBA_STS_PF \
+	0x180d20UL
+#define IGU_REG_PF_FUNCTIONAL_CLEANUP \
+	0x181210UL
+#define IGU_REG_STATISTIC_NUM_OF_INTA_ASSERTED \
+	0x18042cUL
+#define IGU_REG_PBA_STS_PF_SIZE 5
+#define IGU_REG_PBA_STS_PF \
+	0x180d20UL
 #define  MISC_REG_AEU_GENERAL_ATTN_0 \
 	0x008400UL
 #define  CAU_REG_SB_ADDR_MEMORY \
@@ -351,9 +366,9 @@
 #define  IGU_REG_COMMAND_REG_CTRL \
 	0x180848UL
 #define  IGU_REG_BLOCK_CONFIGURATION_VF_CLEANUP_EN	( \
-		0x1 << 1)
+		0x1UL << 1)
 #define  IGU_REG_BLOCK_CONFIGURATION_PXP_TPH_INTERFACE_EN	( \
-		0x1 << 0)
+		0x1UL << 0)
 #define  IGU_REG_MAPPING_MEMORY \
 	0x184000UL
 #define  MISCS_REG_GENERIC_POR_0	\
@@ -361,7 +376,7 @@
 #define  MCP_REG_NVM_CFG4 \
 	0xe0642cUL
 #define  MCP_REG_NVM_CFG4_FLASH_SIZE	( \
-		0x7 << 0)
+		0x7UL << 0)
 #define  MCP_REG_NVM_CFG4_FLASH_SIZE_SHIFT \
 	0
 #define CCFC_REG_STRONG_ENABLE_VF 0x2e070cUL
@@ -394,7 +409,7 @@
 #define XMAC_REG_TX_CTRL_LO 0x210020UL
 #define XMAC_REG_CTRL 0x210000UL
 #define XMAC_REG_RX_CTRL 0x210030UL
-#define XMAC_REG_RX_CTRL_PROCESS_VARIABLE_PREAMBLE (0x1 << 12)
+#define XMAC_REG_RX_CTRL_PROCESS_VARIABLE_PREAMBLE (0x1UL << 12)
 #define MISC_REG_CLK_100G_MODE 0x008c10UL
 #define MISC_REG_OPTE_MODE 0x008c0cUL
 #define NIG_REG_LLH_ENG_CLS_TCP_4_TUPLE_SEARCH 0x501b84UL
@@ -424,16 +439,16 @@
 #define NIG_REG_LLH_FUNC_FILTER_EN 0x501a80UL
 #define NIG_REG_LLH_FUNC_FILTER_EN_SIZE 16
 #define NIG_REG_LLH_FUNC_FILTER_VALUE 0x501a00UL
-#define XMAC_REG_CTRL_TX_EN (0x1 << 0)
-#define XMAC_REG_CTRL_RX_EN (0x1 << 1)
+#define XMAC_REG_CTRL_TX_EN (0x1UL << 0)
+#define XMAC_REG_CTRL_RX_EN (0x1UL << 1)
 #define CDU_REG_SEGMENT0_PARAMS_T0_TID_SIZE (0xffUL << 24) /* @DPDK */
-#define CDU_REG_SEGMENT0_PARAMS_T0_TID_BLOCK_WASTE (0xff << 16)
+#define CDU_REG_SEGMENT0_PARAMS_T0_TID_BLOCK_WASTE (0xffUL << 16)
 #define CDU_REG_SEGMENT0_PARAMS_T0_TID_BLOCK_WASTE_SHIFT 16
-#define CDU_REG_SEGMENT1_PARAMS_T1_TID_BLOCK_WASTE (0xff << 16)
+#define CDU_REG_SEGMENT1_PARAMS_T1_TID_BLOCK_WASTE (0xffUL << 16)
 #define CDU_REG_SEGMENT1_PARAMS_T1_TID_SIZE (0xffUL << 24) /* @DPDK */
-#define CDU_REG_SEGMENT1_PARAMS_T1_NUM_TIDS_IN_BLOCK (0xfff << 0)
+#define CDU_REG_SEGMENT1_PARAMS_T1_NUM_TIDS_IN_BLOCK (0xfffUL << 0)
 #define CDU_REG_SEGMENT1_PARAMS_T1_NUM_TIDS_IN_BLOCK_SHIFT 0
-#define CDU_REG_SEGMENT0_PARAMS_T0_NUM_TIDS_IN_BLOCK (0xfff << 0)
+#define CDU_REG_SEGMENT0_PARAMS_T0_NUM_TIDS_IN_BLOCK (0xfffUL << 0)
 #define CDU_REG_SEGMENT0_PARAMS_T0_NUM_TIDS_IN_BLOCK_SHIFT 0
 #define PSWRQ2_REG_ILT_MEMORY 0x260000UL
 #define QM_REG_WFQPFWEIGHT 0x2f4e80UL
@@ -521,7 +536,7 @@
 #define MISC_REG_AEU_GENERAL_ATTN_35 0x00848cUL
 #define MCP_REG_CPU_STATE 0xe05004UL
 #define MCP_REG_CPU_MODE 0xe05000UL
-#define MCP_REG_CPU_MODE_SOFT_HALT (0x1 << 10)
+#define MCP_REG_CPU_MODE_SOFT_HALT (0x1UL << 10)
 #define MCP_REG_CPU_EVENT_MASK 0xe05008UL
 #define PSWHST_REG_VF_DISABLED_ERROR_VALID 0x2a0060UL
 #define PSWHST_REG_VF_DISABLED_ERROR_ADDRESS 0x2a0064UL
@@ -550,15 +565,15 @@
 #define PGLUE_B_REG_VF_ILT_ERR_ADD_63_32 0x2aae78UL
 #define PGLUE_B_REG_VF_ILT_ERR_DETAILS 0x2aae7cUL
 #define PGLUE_B_REG_LATCHED_ERRORS_CLR 0x2aa3bcUL
-#define NIG_REG_INT_MASK_3_P0_LB_TC1_PAUSE_TOO_LONG_INT (0x1 << 10)
+#define NIG_REG_INT_MASK_3_P0_LB_TC1_PAUSE_TOO_LONG_INT (0x1UL << 10)
 #define DORQ_REG_DB_DROP_REASON 0x100a2cUL
 #define DORQ_REG_DB_DROP_DETAILS 0x100a24UL
 #define TM_REG_INT_STS_1 0x2c0190UL
-#define TM_REG_INT_STS_1_PEND_TASK_SCAN (0x1 << 6)
-#define TM_REG_INT_STS_1_PEND_CONN_SCAN (0x1 << 5)
+#define TM_REG_INT_STS_1_PEND_TASK_SCAN (0x1UL << 6)
+#define TM_REG_INT_STS_1_PEND_CONN_SCAN (0x1UL << 5)
 #define TM_REG_INT_MASK_1 0x2c0194UL
-#define TM_REG_INT_MASK_1_PEND_CONN_SCAN (0x1 << 5)
-#define TM_REG_INT_MASK_1_PEND_TASK_SCAN (0x1 << 6)
+#define TM_REG_INT_MASK_1_PEND_CONN_SCAN (0x1UL << 5)
+#define TM_REG_INT_MASK_1_PEND_TASK_SCAN (0x1UL << 6)
 #define MISC_REG_AEU_AFTER_INVERT_1_IGU 0x0087b4UL
 #define MISC_REG_AEU_ENABLE4_IGU_OUT_0 0x0084a8UL
 #define MISC_REG_AEU_ENABLE3_IGU_OUT_0 0x0084a4UL
@@ -1172,10 +1187,10 @@
 #define XMAC_REG_RX_MAX_SIZE_BB  0x210040UL
 #define XMAC_REG_TX_CTRL_LO_BB 0x210020UL
 #define XMAC_REG_CTRL_BB 0x210000UL
-#define XMAC_REG_CTRL_TX_EN_BB (0x1 << 0)
-#define XMAC_REG_CTRL_RX_EN_BB (0x1 << 1)
+#define XMAC_REG_CTRL_TX_EN_BB (0x1UL << 0)
+#define XMAC_REG_CTRL_RX_EN_BB (0x1UL << 1)
 #define XMAC_REG_RX_CTRL_BB 0x210030UL
-#define XMAC_REG_RX_CTRL_PROCESS_VARIABLE_PREAMBLE_BB (0x1 << 12)
+#define XMAC_REG_RX_CTRL_PROCESS_VARIABLE_PREAMBLE_BB (0x1UL << 12)
 
 #define PGLUE_B_REG_PGL_ADDR_E8_F0_K2_E5 0x2aaf98UL
 #define PGLUE_B_REG_PGL_ADDR_EC_F0_K2_E5 0x2aaf9cUL
@@ -1202,15 +1217,26 @@
 #define DORQ_REG_DPM_FORCE_ABORT 0x1009d8UL
 #define DORQ_REG_PF_OVFL_STICKY 0x1009d0UL
 #define DORQ_REG_INT_STS 0x100180UL
-  #define DORQ_REG_INT_STS_DB_DROP (0x1 << 1)
-  #define DORQ_REG_INT_STS_DORQ_FIFO_OVFL_ERR (0x1 << 2)
-  #define DORQ_REG_INT_STS_DORQ_FIFO_AFULL (0x1 << 3)
+  #define DORQ_REG_INT_STS_DB_DROP (0x1UL << 1)
+  #define DORQ_REG_INT_STS_DORQ_FIFO_OVFL_ERR (0x1UL << 2)
+  #define DORQ_REG_INT_STS_DORQ_FIFO_AFULL (0x1UL << 3)
 #define DORQ_REG_DB_DROP_DETAILS_REL 0x100a28UL
 #define DORQ_REG_INT_STS_WR 0x100188UL
 #define DORQ_REG_DB_DROP_DETAILS_REASON 0x100a20UL
 #define MCP_REG_CPU_PROGRAM_COUNTER 0xe0501cUL
-  #define MCP_REG_CPU_STATE_SOFT_HALTED (0x1 << 10)
+  #define MCP_REG_CPU_STATE_SOFT_HALTED (0x1UL << 10)
 #define PRS_REG_SEARCH_TENANT_ID 0x1f044cUL
 #define PGLUE_B_REG_VF_BAR1_SIZE 0x2aae68UL
 
 #define RSS_REG_RSS_RAM_MASK 0x238c10UL
+
+#define NIG_REG_LLH_FUNC_TAG_EN 0x5019b0UL
+#define NIG_REG_LLH_FUNC_TAG_VALUE 0x5019d0UL
+#define DORQ_REG_TAG1_OVRD_MODE 0x1008b4UL
+#define DORQ_REG_PF_PCP_BB_K2 0x1008c4UL
+#define DORQ_REG_PF_EXT_VID_BB_K2 0x1008c8UL
+#define PRS_REG_SEARCH_NON_IP_AS_GFT 0x1f11c0UL
+#define NIG_REG_LLH_PPFID2PFID_TBL_0 0x501970UL
+#define NIG_REG_PPF_TO_ENGINE_SEL 0x508900UL
+#define NIG_REG_LLH_ENG_CLS_ROCE_QP_SEL 0x501b98UL
+#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_BB_K2 0x501b40UL
diff --git a/drivers/net/qede/meson.build b/drivers/net/qede/meson.build
new file mode 100644
index 00000000..12388a68
--- /dev/null
+++ b/drivers/net/qede/meson.build
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
+
+subdir('base')
+objs = [base_objs]
+
+sources = files(
+	'qede_ethdev.c',
+	'qede_filter.c',
+	'qede_main.c',
+	'qede_rxtx.c',
+)
diff --git a/drivers/net/qede/qede_ethdev.c b/drivers/net/qede/qede_ethdev.c
index df52ea92..518673dc 100644
--- a/drivers/net/qede/qede_ethdev.c
+++ b/drivers/net/qede/qede_ethdev.c
@@ -14,112 +14,10 @@ int qede_logtype_init;
 int qede_logtype_driver;
 
 static const struct qed_eth_ops *qed_ops;
-#define QEDE_SP_TIMER_PERIOD	10000 /* 100ms */
+static int qede_eth_dev_uninit(struct rte_eth_dev *eth_dev);
+static int qede_eth_dev_init(struct rte_eth_dev *eth_dev);
 
-/* VXLAN tunnel classification mapping */
-const struct _qede_udp_tunn_types {
-	uint16_t rte_filter_type;
-	enum ecore_filter_ucast_type qede_type;
-	enum ecore_tunn_clss qede_tunn_clss;
-	const char *string;
-} qede_tunn_types[] = {
-	{
-		ETH_TUNNEL_FILTER_OMAC,
-		ECORE_FILTER_MAC,
-		ECORE_TUNN_CLSS_MAC_VLAN,
-		"outer-mac"
-	},
-	{
-		ETH_TUNNEL_FILTER_TENID,
-		ECORE_FILTER_VNI,
-		ECORE_TUNN_CLSS_MAC_VNI,
-		"vni"
-	},
-	{
-		ETH_TUNNEL_FILTER_IMAC,
-		ECORE_FILTER_INNER_MAC,
-		ECORE_TUNN_CLSS_INNER_MAC_VLAN,
-		"inner-mac"
-	},
-	{
-		ETH_TUNNEL_FILTER_IVLAN,
-		ECORE_FILTER_INNER_VLAN,
-		ECORE_TUNN_CLSS_INNER_MAC_VLAN,
-		"inner-vlan"
-	},
-	{
-		ETH_TUNNEL_FILTER_OMAC | ETH_TUNNEL_FILTER_TENID,
-		ECORE_FILTER_MAC_VNI_PAIR,
-		ECORE_TUNN_CLSS_MAC_VNI,
-		"outer-mac and vni"
-	},
-	{
-		ETH_TUNNEL_FILTER_OMAC | ETH_TUNNEL_FILTER_IMAC,
-		ECORE_FILTER_UNUSED,
-		MAX_ECORE_TUNN_CLSS,
-		"outer-mac and inner-mac"
-	},
-	{
-		ETH_TUNNEL_FILTER_OMAC | ETH_TUNNEL_FILTER_IVLAN,
-		ECORE_FILTER_UNUSED,
-		MAX_ECORE_TUNN_CLSS,
-		"outer-mac and inner-vlan"
-	},
-	{
-		ETH_TUNNEL_FILTER_TENID | ETH_TUNNEL_FILTER_IMAC,
-		ECORE_FILTER_INNER_MAC_VNI_PAIR,
-		ECORE_TUNN_CLSS_INNER_MAC_VNI,
-		"vni and inner-mac",
-	},
-	{
-		ETH_TUNNEL_FILTER_TENID | ETH_TUNNEL_FILTER_IVLAN,
-		ECORE_FILTER_UNUSED,
-		MAX_ECORE_TUNN_CLSS,
-		"vni and inner-vlan",
-	},
-	{
-		ETH_TUNNEL_FILTER_IMAC | ETH_TUNNEL_FILTER_IVLAN,
-		ECORE_FILTER_INNER_PAIR,
-		ECORE_TUNN_CLSS_INNER_MAC_VLAN,
-		"inner-mac and inner-vlan",
-	},
-	{
-		ETH_TUNNEL_FILTER_OIP,
-		ECORE_FILTER_UNUSED,
-		MAX_ECORE_TUNN_CLSS,
-		"outer-IP"
-	},
-	{
-		ETH_TUNNEL_FILTER_IIP,
-		ECORE_FILTER_UNUSED,
-		MAX_ECORE_TUNN_CLSS,
-		"inner-IP"
-	},
-	{
-		RTE_TUNNEL_FILTER_IMAC_IVLAN,
-		ECORE_FILTER_UNUSED,
-		MAX_ECORE_TUNN_CLSS,
-		"IMAC_IVLAN"
-	},
-	{
-		RTE_TUNNEL_FILTER_IMAC_IVLAN_TENID,
-		ECORE_FILTER_UNUSED,
-		MAX_ECORE_TUNN_CLSS,
-		"IMAC_IVLAN_TENID"
-	},
-	{
-		RTE_TUNNEL_FILTER_IMAC_TENID,
-		ECORE_FILTER_UNUSED,
-		MAX_ECORE_TUNN_CLSS,
-		"IMAC_TENID"
-	},
-	{
-		RTE_TUNNEL_FILTER_OMAC_TENID_IMAC,
-		ECORE_FILTER_UNUSED,
-		MAX_ECORE_TUNN_CLSS,
-		"OMAC_TENID_IMAC"
-	},
-};
+#define QEDE_SP_TIMER_PERIOD	10000 /* 100ms */
 
 struct rte_qede_xstats_name_off {
 	char name[RTE_ETH_XSTATS_NAME_SIZE];
@@ -399,7 +297,7 @@ static void qede_print_adapter_info(struct qede_dev *qdev)
 		(info->mfw_rev >> 16) & 0xff,
 		(info->mfw_rev >> 8) & 0xff, (info->mfw_rev) & 0xff);
 	DP_INFO(edev, " Management Firmware version : %s\n", ver_str);
-	DP_INFO(edev, " Firmware file : %s\n", fw_file);
+	DP_INFO(edev, " Firmware file : %s\n", qede_fw_file);
 	DP_INFO(edev, "*********************************\n");
 }
 
@@ -614,14 +512,6 @@ int qede_enable_tpa(struct rte_eth_dev *eth_dev, bool flg)
 	return 0;
 }
 
-static void qede_set_ucast_cmn_params(struct ecore_filter_ucast *ucast)
-{
-	memset(ucast, 0, sizeof(struct ecore_filter_ucast));
-	ucast->is_rx_filter = true;
-	ucast->is_tx_filter = true;
-	/* ucast->assert_on_error = true; - For debug */
-}
-
 static int
 qed_configure_filter_rx_mode(struct rte_eth_dev *eth_dev,
 			     enum qed_filter_rx_mode_type type)
@@ -660,167 +550,7 @@ qed_configure_filter_rx_mode(struct rte_eth_dev *eth_dev,
 			ECORE_SPQ_MODE_CB, NULL);
 }
 
-static int
-qede_tunnel_update(struct qede_dev *qdev,
-		   struct ecore_tunnel_info *tunn_info)
-{
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	enum _ecore_status_t rc = ECORE_INVAL;
-	struct ecore_hwfn *p_hwfn;
-	struct ecore_ptt *p_ptt;
-	int i;
-
-	for_each_hwfn(edev, i) {
-		p_hwfn = &edev->hwfns[i];
-		if (IS_PF(edev)) {
-			p_ptt = ecore_ptt_acquire(p_hwfn);
-			if (!p_ptt) {
-				DP_ERR(p_hwfn, "Can't acquire PTT\n");
-				return -EAGAIN;
-			}
-		} else {
-			p_ptt = NULL;
-		}
-
-		rc = ecore_sp_pf_update_tunn_cfg(p_hwfn, p_ptt,
-				tunn_info, ECORE_SPQ_MODE_CB, NULL);
-		if (IS_PF(edev))
-			ecore_ptt_release(p_hwfn, p_ptt);
-
-		if (rc != ECORE_SUCCESS)
-			break;
-	}
-
-	return rc;
-}
-
-static int
-qede_vxlan_enable(struct rte_eth_dev *eth_dev, uint8_t clss,
-		  bool enable)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	enum _ecore_status_t rc = ECORE_INVAL;
-	struct ecore_tunnel_info tunn;
-
-	if (qdev->vxlan.enable == enable)
-		return ECORE_SUCCESS;
-
-	memset(&tunn, 0, sizeof(struct ecore_tunnel_info));
-	tunn.vxlan.b_update_mode = true;
-	tunn.vxlan.b_mode_enabled = enable;
-	tunn.b_update_rx_cls = true;
-	tunn.b_update_tx_cls = true;
-	tunn.vxlan.tun_cls = clss;
-
-	tunn.vxlan_port.b_update_port = true;
-	tunn.vxlan_port.port = enable ? QEDE_VXLAN_DEF_PORT : 0;
-
-	rc = qede_tunnel_update(qdev, &tunn);
-	if (rc == ECORE_SUCCESS) {
-		qdev->vxlan.enable = enable;
-		qdev->vxlan.udp_port = (enable) ? QEDE_VXLAN_DEF_PORT : 0;
-		DP_INFO(edev, "vxlan is %s, UDP port = %d\n",
-			enable ? "enabled" : "disabled", qdev->vxlan.udp_port);
-	} else {
-		DP_ERR(edev, "Failed to update tunn_clss %u\n",
-		       tunn.vxlan.tun_cls);
-	}
-
-	return rc;
-}
-
-static int
-qede_geneve_enable(struct rte_eth_dev *eth_dev, uint8_t clss,
-		  bool enable)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	enum _ecore_status_t rc = ECORE_INVAL;
-	struct ecore_tunnel_info tunn;
-
-	memset(&tunn, 0, sizeof(struct ecore_tunnel_info));
-	tunn.l2_geneve.b_update_mode = true;
-	tunn.l2_geneve.b_mode_enabled = enable;
-	tunn.ip_geneve.b_update_mode = true;
-	tunn.ip_geneve.b_mode_enabled = enable;
-	tunn.l2_geneve.tun_cls = clss;
-	tunn.ip_geneve.tun_cls = clss;
-	tunn.b_update_rx_cls = true;
-	tunn.b_update_tx_cls = true;
-
-	tunn.geneve_port.b_update_port = true;
-	tunn.geneve_port.port = enable ? QEDE_GENEVE_DEF_PORT : 0;
-
-	rc = qede_tunnel_update(qdev, &tunn);
-	if (rc == ECORE_SUCCESS) {
-		qdev->geneve.enable = enable;
-		qdev->geneve.udp_port = (enable) ? QEDE_GENEVE_DEF_PORT : 0;
-		DP_INFO(edev, "GENEVE is %s, UDP port = %d\n",
-			enable ? "enabled" : "disabled", qdev->geneve.udp_port);
-	} else {
-		DP_ERR(edev, "Failed to update tunn_clss %u\n",
-		       clss);
-	}
-
-	return rc;
-}
-
-static int
-qede_ipgre_enable(struct rte_eth_dev *eth_dev, uint8_t clss,
-		  bool enable)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	enum _ecore_status_t rc = ECORE_INVAL;
-	struct ecore_tunnel_info tunn;
-
-	memset(&tunn, 0, sizeof(struct ecore_tunnel_info));
-	tunn.ip_gre.b_update_mode = true;
-	tunn.ip_gre.b_mode_enabled = enable;
-	tunn.ip_gre.tun_cls = clss;
-	tunn.ip_gre.tun_cls = clss;
-	tunn.b_update_rx_cls = true;
-	tunn.b_update_tx_cls = true;
-
-	rc = qede_tunnel_update(qdev, &tunn);
-	if (rc == ECORE_SUCCESS) {
-		qdev->ipgre.enable = enable;
-		DP_INFO(edev, "IPGRE is %s\n",
-			enable ? "enabled" : "disabled");
-	} else {
-		DP_ERR(edev, "Failed to update tunn_clss %u\n",
-		       clss);
-	}
-
-	return rc;
-}
-
-static int
-qede_tunn_enable(struct rte_eth_dev *eth_dev, uint8_t clss,
-		 enum rte_eth_tunnel_type tunn_type, bool enable)
-{
-	int rc = -EINVAL;
-
-	switch (tunn_type) {
-	case RTE_TUNNEL_TYPE_VXLAN:
-		rc = qede_vxlan_enable(eth_dev, clss, enable);
-		break;
-	case RTE_TUNNEL_TYPE_GENEVE:
-		rc = qede_geneve_enable(eth_dev, clss, enable);
-		break;
-	case RTE_TUNNEL_TYPE_IP_IN_GRE:
-		rc = qede_ipgre_enable(eth_dev, clss, enable);
-		break;
-	default:
-		rc = -EINVAL;
-		break;
-	}
-
-	return rc;
-}
-
-static int
+int
 qede_ucast_filter(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *ucast,
 		  bool add)
 {
@@ -941,7 +671,7 @@ static int qede_del_mcast_filters(struct rte_eth_dev *eth_dev)
 	return 0;
 }
 
-static enum _ecore_status_t
+enum _ecore_status_t
 qede_mac_int_ops(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *ucast,
 		 bool add)
 {
@@ -1033,7 +763,7 @@ qede_mac_addr_set(struct rte_eth_dev *eth_dev, struct ether_addr *mac_addr)
 	return qede_mac_addr_add(eth_dev, mac_addr, 0, 0);
 }
 
-static void qede_config_accept_any_vlan(struct qede_dev *qdev, bool flg)
+void qede_config_accept_any_vlan(struct qede_dev *qdev, bool flg)
 {
 	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 	struct ecore_sp_vport_update_params params;
@@ -1359,7 +1089,7 @@ static void qede_dev_stop(struct rte_eth_dev *eth_dev)
 	DP_INFO(edev, "Device is stopped\n");
 }
 
-const char *valid_args[] = {
+static const char * const valid_args[] = {
 	QEDE_NPAR_TX_SWITCHING,
 	QEDE_VF_TX_SWITCHING,
 	NULL,
@@ -1483,7 +1213,7 @@ static int qede_dev_configure(struct rte_eth_dev *eth_dev)
 	if (rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 		eth_dev->data->mtu =
 			eth_dev->data->dev_conf.rxmode.max_rx_pkt_len -
-			ETHER_HDR_LEN - ETHER_CRC_LEN;
+			ETHER_HDR_LEN - QEDE_ETH_OVERHEAD;
 
 	if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER)
 		eth_dev->data->scattered_rx = 1;
@@ -1554,7 +1284,6 @@ qede_dev_info_get(struct rte_eth_dev *eth_dev,
 				     DEV_RX_OFFLOAD_TCP_CKSUM	|
 				     DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
 				     DEV_RX_OFFLOAD_TCP_LRO	|
-				     DEV_RX_OFFLOAD_CRC_STRIP	|
 				     DEV_RX_OFFLOAD_KEEP_CRC    |
 				     DEV_RX_OFFLOAD_SCATTER	|
 				     DEV_RX_OFFLOAD_JUMBO_FRAME |
@@ -1651,14 +1380,11 @@ qede_link_update(struct rte_eth_dev *eth_dev, __rte_unused int wait_to_complete)
 
 static void qede_promiscuous_enable(struct rte_eth_dev *eth_dev)
 {
-#ifdef RTE_LIBRTE_QEDE_DEBUG_INIT
 	struct qede_dev *qdev = eth_dev->data->dev_private;
 	struct ecore_dev *edev = &qdev->edev;
+	enum qed_filter_rx_mode_type type = QED_FILTER_RX_MODE_TYPE_PROMISC;
 
 	PMD_INIT_FUNC_TRACE(edev);
-#endif
-
-	enum qed_filter_rx_mode_type type = QED_FILTER_RX_MODE_TYPE_PROMISC;
 
 	if (rte_eth_allmulticast_get(eth_dev->data->port_id) == 1)
 		type |= QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC;
@@ -1668,12 +1394,10 @@ static void qede_promiscuous_enable(struct rte_eth_dev *eth_dev)
 
 static void qede_promiscuous_disable(struct rte_eth_dev *eth_dev)
 {
-#ifdef RTE_LIBRTE_QEDE_DEBUG_INIT
 	struct qede_dev *qdev = eth_dev->data->dev_private;
 	struct ecore_dev *edev = &qdev->edev;
 
 	PMD_INIT_FUNC_TRACE(edev);
-#endif
 
 	if (rte_eth_allmulticast_get(eth_dev->data->port_id) == 1)
 		qed_configure_filter_rx_mode(eth_dev,
@@ -2499,19 +2223,18 @@ static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	struct qede_fastpath *fp;
 	uint32_t max_rx_pkt_len;
 	uint32_t frame_size;
-	uint16_t rx_buf_size;
 	uint16_t bufsz;
 	bool restart = false;
-	int i;
+	int i, rc;
 
 	PMD_INIT_FUNC_TRACE(edev);
 	qede_dev_info_get(dev, &dev_info);
-	max_rx_pkt_len = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
-	frame_size = max_rx_pkt_len + QEDE_ETH_OVERHEAD;
+	max_rx_pkt_len = mtu + QEDE_MAX_ETHER_HDR_LEN;
+	frame_size = max_rx_pkt_len;
 	if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen)) {
 		DP_ERR(edev, "MTU %u out of range, %u is maximum allowable\n",
 		       mtu, dev_info.max_rx_pktlen - ETHER_HDR_LEN -
-			ETHER_CRC_LEN - QEDE_ETH_OVERHEAD);
+		       QEDE_ETH_OVERHEAD);
 		return -EINVAL;
 	}
 	if (!dev->data->scattered_rx &&
@@ -2539,14 +2262,15 @@ static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 		if (fp->rxq != NULL) {
 			bufsz = (uint16_t)rte_pktmbuf_data_room_size(
 				fp->rxq->mb_pool) - RTE_PKTMBUF_HEADROOM;
-			if (dev->data->scattered_rx)
-				rx_buf_size = bufsz + ETHER_HDR_LEN +
-					      ETHER_CRC_LEN + QEDE_ETH_OVERHEAD;
-			else
-				rx_buf_size = frame_size;
-			rx_buf_size = QEDE_CEIL_TO_CACHE_LINE_SIZE(rx_buf_size);
-			fp->rxq->rx_buf_size = rx_buf_size;
-			DP_INFO(edev, "RX buffer size %u\n", rx_buf_size);
+			/* cache align the mbuf size to simplfy rx_buf_size
+			 * calculation
+			 */
+			bufsz = QEDE_FLOOR_TO_CACHE_LINE_SIZE(bufsz);
+			rc = qede_calc_rx_buf_size(dev, bufsz, frame_size);
+			if (rc < 0)
+				return rc;
+
+			fp->rxq->rx_buf_size = rc;
 		}
 	}
 	if (max_rx_pkt_len > ETHER_MAX_LEN)
@@ -2569,411 +2293,15 @@ static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 }
 
 static int
-qede_udp_dst_port_del(struct rte_eth_dev *eth_dev,
-		      struct rte_eth_udp_tunnel *tunnel_udp)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	struct ecore_tunnel_info tunn; /* @DPDK */
-	uint16_t udp_port;
-	int rc;
-
-	PMD_INIT_FUNC_TRACE(edev);
-
-	memset(&tunn, 0, sizeof(tunn));
-
-	switch (tunnel_udp->prot_type) {
-	case RTE_TUNNEL_TYPE_VXLAN:
-		if (qdev->vxlan.udp_port != tunnel_udp->udp_port) {
-			DP_ERR(edev, "UDP port %u doesn't exist\n",
-				tunnel_udp->udp_port);
-			return ECORE_INVAL;
-		}
-		udp_port = 0;
-
-		tunn.vxlan_port.b_update_port = true;
-		tunn.vxlan_port.port = udp_port;
-
-		rc = qede_tunnel_update(qdev, &tunn);
-		if (rc != ECORE_SUCCESS) {
-			DP_ERR(edev, "Unable to config UDP port %u\n",
-			       tunn.vxlan_port.port);
-			return rc;
-		}
-
-		qdev->vxlan.udp_port = udp_port;
-		/* If the request is to delete UDP port and if the number of
-		 * VXLAN filters have reached 0 then VxLAN offload can be be
-		 * disabled.
-		 */
-		if (qdev->vxlan.enable && qdev->vxlan.num_filters == 0)
-			return qede_vxlan_enable(eth_dev,
-					ECORE_TUNN_CLSS_MAC_VLAN, false);
-
-		break;
-	case RTE_TUNNEL_TYPE_GENEVE:
-		if (qdev->geneve.udp_port != tunnel_udp->udp_port) {
-			DP_ERR(edev, "UDP port %u doesn't exist\n",
-				tunnel_udp->udp_port);
-			return ECORE_INVAL;
-		}
-
-		udp_port = 0;
-
-		tunn.geneve_port.b_update_port = true;
-		tunn.geneve_port.port = udp_port;
-
-		rc = qede_tunnel_update(qdev, &tunn);
-		if (rc != ECORE_SUCCESS) {
-			DP_ERR(edev, "Unable to config UDP port %u\n",
-			       tunn.vxlan_port.port);
-			return rc;
-		}
-
-		qdev->vxlan.udp_port = udp_port;
-		/* If the request is to delete UDP port and if the number of
-		 * GENEVE filters have reached 0 then GENEVE offload can be be
-		 * disabled.
-		 */
-		if (qdev->geneve.enable && qdev->geneve.num_filters == 0)
-			return qede_geneve_enable(eth_dev,
-					ECORE_TUNN_CLSS_MAC_VLAN, false);
-
-		break;
-
-	default:
-		return ECORE_INVAL;
-	}
-
-	return 0;
-
-}
-static int
-qede_udp_dst_port_add(struct rte_eth_dev *eth_dev,
-		      struct rte_eth_udp_tunnel *tunnel_udp)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	struct ecore_tunnel_info tunn; /* @DPDK */
-	uint16_t udp_port;
-	int rc;
-
-	PMD_INIT_FUNC_TRACE(edev);
-
-	memset(&tunn, 0, sizeof(tunn));
-
-	switch (tunnel_udp->prot_type) {
-	case RTE_TUNNEL_TYPE_VXLAN:
-		if (qdev->vxlan.udp_port == tunnel_udp->udp_port) {
-			DP_INFO(edev,
-				"UDP port %u for VXLAN was already configured\n",
-				tunnel_udp->udp_port);
-			return ECORE_SUCCESS;
-		}
-
-		/* Enable VxLAN tunnel with default MAC/VLAN classification if
-		 * it was not enabled while adding VXLAN filter before UDP port
-		 * update.
-		 */
-		if (!qdev->vxlan.enable) {
-			rc = qede_vxlan_enable(eth_dev,
-				ECORE_TUNN_CLSS_MAC_VLAN, true);
-			if (rc != ECORE_SUCCESS) {
-				DP_ERR(edev, "Failed to enable VXLAN "
-					"prior to updating UDP port\n");
-				return rc;
-			}
-		}
-		udp_port = tunnel_udp->udp_port;
-
-		tunn.vxlan_port.b_update_port = true;
-		tunn.vxlan_port.port = udp_port;
-
-		rc = qede_tunnel_update(qdev, &tunn);
-		if (rc != ECORE_SUCCESS) {
-			DP_ERR(edev, "Unable to config UDP port %u for VXLAN\n",
-			       udp_port);
-			return rc;
-		}
-
-		DP_INFO(edev, "Updated UDP port %u for VXLAN\n", udp_port);
-
-		qdev->vxlan.udp_port = udp_port;
-		break;
-	case RTE_TUNNEL_TYPE_GENEVE:
-		if (qdev->geneve.udp_port == tunnel_udp->udp_port) {
-			DP_INFO(edev,
-				"UDP port %u for GENEVE was already configured\n",
-				tunnel_udp->udp_port);
-			return ECORE_SUCCESS;
-		}
-
-		/* Enable GENEVE tunnel with default MAC/VLAN classification if
-		 * it was not enabled while adding GENEVE filter before UDP port
-		 * update.
-		 */
-		if (!qdev->geneve.enable) {
-			rc = qede_geneve_enable(eth_dev,
-				ECORE_TUNN_CLSS_MAC_VLAN, true);
-			if (rc != ECORE_SUCCESS) {
-				DP_ERR(edev, "Failed to enable GENEVE "
-					"prior to updating UDP port\n");
-				return rc;
-			}
-		}
-		udp_port = tunnel_udp->udp_port;
-
-		tunn.geneve_port.b_update_port = true;
-		tunn.geneve_port.port = udp_port;
-
-		rc = qede_tunnel_update(qdev, &tunn);
-		if (rc != ECORE_SUCCESS) {
-			DP_ERR(edev, "Unable to config UDP port %u for GENEVE\n",
-			       udp_port);
-			return rc;
-		}
-
-		DP_INFO(edev, "Updated UDP port %u for GENEVE\n", udp_port);
-
-		qdev->geneve.udp_port = udp_port;
-		break;
-	default:
-		return ECORE_INVAL;
-	}
-
-	return 0;
-}
-
-static void qede_get_ecore_tunn_params(uint32_t filter, uint32_t *type,
-				       uint32_t *clss, char *str)
-{
-	uint16_t j;
-	*clss = MAX_ECORE_TUNN_CLSS;
-
-	for (j = 0; j < RTE_DIM(qede_tunn_types); j++) {
-		if (filter == qede_tunn_types[j].rte_filter_type) {
-			*type = qede_tunn_types[j].qede_type;
-			*clss = qede_tunn_types[j].qede_tunn_clss;
-			strcpy(str, qede_tunn_types[j].string);
-			return;
-		}
-	}
-}
-
-static int
-qede_set_ucast_tunn_cmn_param(struct ecore_filter_ucast *ucast,
-			      const struct rte_eth_tunnel_filter_conf *conf,
-			      uint32_t type)
-{
-	/* Init commmon ucast params first */
-	qede_set_ucast_cmn_params(ucast);
-
-	/* Copy out the required fields based on classification type */
-	ucast->type = type;
-
-	switch (type) {
-	case ECORE_FILTER_VNI:
-		ucast->vni = conf->tenant_id;
-	break;
-	case ECORE_FILTER_INNER_VLAN:
-		ucast->vlan = conf->inner_vlan;
-	break;
-	case ECORE_FILTER_MAC:
-		memcpy(ucast->mac, conf->outer_mac.addr_bytes,
-		       ETHER_ADDR_LEN);
-	break;
-	case ECORE_FILTER_INNER_MAC:
-		memcpy(ucast->mac, conf->inner_mac.addr_bytes,
-		       ETHER_ADDR_LEN);
-	break;
-	case ECORE_FILTER_MAC_VNI_PAIR:
-		memcpy(ucast->mac, conf->outer_mac.addr_bytes,
-			ETHER_ADDR_LEN);
-		ucast->vni = conf->tenant_id;
-	break;
-	case ECORE_FILTER_INNER_MAC_VNI_PAIR:
-		memcpy(ucast->mac, conf->inner_mac.addr_bytes,
-			ETHER_ADDR_LEN);
-		ucast->vni = conf->tenant_id;
-	break;
-	case ECORE_FILTER_INNER_PAIR:
-		memcpy(ucast->mac, conf->inner_mac.addr_bytes,
-			ETHER_ADDR_LEN);
-		ucast->vlan = conf->inner_vlan;
-	break;
-	default:
-		return -EINVAL;
-	}
-
-	return ECORE_SUCCESS;
-}
-
-static int
-_qede_tunn_filter_config(struct rte_eth_dev *eth_dev,
-			 const struct rte_eth_tunnel_filter_conf *conf,
-			 __attribute__((unused)) enum rte_filter_op filter_op,
-			 enum ecore_tunn_clss *clss,
-			 bool add)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	struct ecore_filter_ucast ucast = {0};
-	enum ecore_filter_ucast_type type;
-	uint16_t filter_type = 0;
-	char str[80];
-	int rc;
-
-	filter_type = conf->filter_type;
-	/* Determine if the given filter classification is supported */
-	qede_get_ecore_tunn_params(filter_type, &type, clss, str);
-	if (*clss == MAX_ECORE_TUNN_CLSS) {
-		DP_ERR(edev, "Unsupported filter type\n");
-		return -EINVAL;
-	}
-	/* Init tunnel ucast params */
-	rc = qede_set_ucast_tunn_cmn_param(&ucast, conf, type);
-	if (rc != ECORE_SUCCESS) {
-		DP_ERR(edev, "Unsupported Tunnel filter type 0x%x\n",
-		conf->filter_type);
-		return rc;
-	}
-	DP_INFO(edev, "Rule: \"%s\", op %d, type 0x%x\n",
-		str, filter_op, ucast.type);
-
-	ucast.opcode = add ? ECORE_FILTER_ADD : ECORE_FILTER_REMOVE;
-
-	/* Skip MAC/VLAN if filter is based on VNI */
-	if (!(filter_type & ETH_TUNNEL_FILTER_TENID)) {
-		rc = qede_mac_int_ops(eth_dev, &ucast, add);
-		if ((rc == 0) && add) {
-			/* Enable accept anyvlan */
-			qede_config_accept_any_vlan(qdev, true);
-		}
-	} else {
-		rc = qede_ucast_filter(eth_dev, &ucast, add);
-		if (rc == 0)
-			rc = ecore_filter_ucast_cmd(edev, &ucast,
-					    ECORE_SPQ_MODE_CB, NULL);
-	}
-
-	return rc;
-}
-
-static int
-qede_tunn_filter_config(struct rte_eth_dev *eth_dev,
-			enum rte_filter_op filter_op,
-			const struct rte_eth_tunnel_filter_conf *conf)
+qede_dev_reset(struct rte_eth_dev *dev)
 {
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	enum ecore_tunn_clss clss = MAX_ECORE_TUNN_CLSS;
-	bool add;
-	int rc;
-
-	PMD_INIT_FUNC_TRACE(edev);
-
-	switch (filter_op) {
-	case RTE_ETH_FILTER_ADD:
-		add = true;
-		break;
-	case RTE_ETH_FILTER_DELETE:
-		add = false;
-		break;
-	default:
-		DP_ERR(edev, "Unsupported operation %d\n", filter_op);
-		return -EINVAL;
-	}
-
-	if (IS_VF(edev))
-		return qede_tunn_enable(eth_dev,
-					ECORE_TUNN_CLSS_MAC_VLAN,
-					conf->tunnel_type, add);
-
-	rc = _qede_tunn_filter_config(eth_dev, conf, filter_op, &clss, add);
-	if (rc != ECORE_SUCCESS)
-		return rc;
-
-	if (add) {
-		if (conf->tunnel_type == RTE_TUNNEL_TYPE_VXLAN) {
-			qdev->vxlan.num_filters++;
-			qdev->vxlan.filter_type = conf->filter_type;
-		} else { /* GENEVE */
-			qdev->geneve.num_filters++;
-			qdev->geneve.filter_type = conf->filter_type;
-		}
-
-		if (!qdev->vxlan.enable || !qdev->geneve.enable ||
-		    !qdev->ipgre.enable)
-			return qede_tunn_enable(eth_dev, clss,
-						conf->tunnel_type,
-						true);
-	} else {
-		if (conf->tunnel_type == RTE_TUNNEL_TYPE_VXLAN)
-			qdev->vxlan.num_filters--;
-		else /*GENEVE*/
-			qdev->geneve.num_filters--;
-
-		/* Disable VXLAN if VXLAN filters become 0 */
-		if ((qdev->vxlan.num_filters == 0) ||
-		    (qdev->geneve.num_filters == 0))
-			return qede_tunn_enable(eth_dev, clss,
-						conf->tunnel_type,
-						false);
-	}
-
-	return 0;
-}
+	int ret;
 
-int qede_dev_filter_ctrl(struct rte_eth_dev *eth_dev,
-			 enum rte_filter_type filter_type,
-			 enum rte_filter_op filter_op,
-			 void *arg)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	struct rte_eth_tunnel_filter_conf *filter_conf =
-			(struct rte_eth_tunnel_filter_conf *)arg;
-
-	switch (filter_type) {
-	case RTE_ETH_FILTER_TUNNEL:
-		switch (filter_conf->tunnel_type) {
-		case RTE_TUNNEL_TYPE_VXLAN:
-		case RTE_TUNNEL_TYPE_GENEVE:
-		case RTE_TUNNEL_TYPE_IP_IN_GRE:
-			DP_INFO(edev,
-				"Packet steering to the specified Rx queue"
-				" is not supported with UDP tunneling");
-			return(qede_tunn_filter_config(eth_dev, filter_op,
-						      filter_conf));
-		case RTE_TUNNEL_TYPE_TEREDO:
-		case RTE_TUNNEL_TYPE_NVGRE:
-		case RTE_L2_TUNNEL_TYPE_E_TAG:
-			DP_ERR(edev, "Unsupported tunnel type %d\n",
-				filter_conf->tunnel_type);
-			return -EINVAL;
-		case RTE_TUNNEL_TYPE_NONE:
-		default:
-			return 0;
-		}
-		break;
-	case RTE_ETH_FILTER_FDIR:
-		return qede_fdir_filter_conf(eth_dev, filter_op, arg);
-	case RTE_ETH_FILTER_NTUPLE:
-		return qede_ntuple_filter_conf(eth_dev, filter_op, arg);
-	case RTE_ETH_FILTER_MACVLAN:
-	case RTE_ETH_FILTER_ETHERTYPE:
-	case RTE_ETH_FILTER_FLEXIBLE:
-	case RTE_ETH_FILTER_SYN:
-	case RTE_ETH_FILTER_HASH:
-	case RTE_ETH_FILTER_L2_TUNNEL:
-	case RTE_ETH_FILTER_MAX:
-	default:
-		DP_ERR(edev, "Unsupported filter type %d\n",
-			filter_type);
-		return -EINVAL;
-	}
+	ret = qede_eth_dev_uninit(dev);
+	if (ret)
+		return ret;
 
-	return 0;
+	return qede_eth_dev_init(dev);
 }
 
 static const struct eth_dev_ops qede_eth_dev_ops = {
@@ -2981,9 +2309,11 @@ static const struct eth_dev_ops qede_eth_dev_ops = {
 	.dev_infos_get = qede_dev_info_get,
 	.rx_queue_setup = qede_rx_queue_setup,
 	.rx_queue_release = qede_rx_queue_release,
+	.rx_descriptor_status = qede_rx_descriptor_status,
 	.tx_queue_setup = qede_tx_queue_setup,
 	.tx_queue_release = qede_tx_queue_release,
 	.dev_start = qede_dev_start,
+	.dev_reset = qede_dev_reset,
 	.dev_set_link_up = qede_dev_set_link_up,
 	.dev_set_link_down = qede_dev_set_link_down,
 	.link_update = qede_link_update,
@@ -3022,9 +2352,11 @@ static const struct eth_dev_ops qede_eth_vf_dev_ops = {
 	.dev_infos_get = qede_dev_info_get,
 	.rx_queue_setup = qede_rx_queue_setup,
 	.rx_queue_release = qede_rx_queue_release,
+	.rx_descriptor_status = qede_rx_descriptor_status,
 	.tx_queue_setup = qede_tx_queue_setup,
 	.tx_queue_release = qede_tx_queue_release,
 	.dev_start = qede_dev_start,
+	.dev_reset = qede_dev_reset,
 	.dev_set_link_up = qede_dev_set_link_up,
 	.dev_set_link_down = qede_dev_set_link_down,
 	.link_update = qede_link_update,
@@ -3257,7 +2589,7 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf)
 
 	adapter->num_tx_queues = 0;
 	adapter->num_rx_queues = 0;
-	SLIST_INIT(&adapter->fdir_info.fdir_list_head);
+	SLIST_INIT(&adapter->arfs_info.arfs_list_head);
 	SLIST_INIT(&adapter->vlan_list_head);
 	SLIST_INIT(&adapter->uc_list_head);
 	SLIST_INIT(&adapter->mc_list_head);
@@ -3311,12 +2643,10 @@ static int qede_eth_dev_init(struct rte_eth_dev *eth_dev)
 
 static int qede_dev_common_uninit(struct rte_eth_dev *eth_dev)
 {
-#ifdef RTE_LIBRTE_QEDE_DEBUG_INIT
 	struct qede_dev *qdev = eth_dev->data->dev_private;
 	struct ecore_dev *edev = &qdev->edev;
 
 	PMD_INIT_FUNC_TRACE(edev);
-#endif
 
 	/* only uninitialize in the primary process */
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
@@ -3329,11 +2659,6 @@ static int qede_dev_common_uninit(struct rte_eth_dev *eth_dev)
 	eth_dev->rx_pkt_burst = NULL;
 	eth_dev->tx_pkt_burst = NULL;
 
-	if (eth_dev->data->mac_addrs)
-		rte_free(eth_dev->data->mac_addrs);
-
-	eth_dev->data->mac_addrs = NULL;
-
 	return 0;
 }
 
diff --git a/drivers/net/qede/qede_ethdev.h b/drivers/net/qede/qede_ethdev.h
index 6e9a5b4b..c06274d9 100644
--- a/drivers/net/qede/qede_ethdev.h
+++ b/drivers/net/qede/qede_ethdev.h
@@ -44,7 +44,7 @@
 /* Driver versions */
 #define QEDE_PMD_VER_PREFIX		"QEDE PMD"
 #define QEDE_PMD_VERSION_MAJOR		2
-#define QEDE_PMD_VERSION_MINOR	        9
+#define QEDE_PMD_VERSION_MINOR	        10
 #define QEDE_PMD_VERSION_REVISION       0
 #define QEDE_PMD_VERSION_PATCH	        1
 
@@ -118,7 +118,7 @@
 
 
 
-extern char fw_file[];
+extern char qede_fw_file[];
 
 /* Number of PF connections - 32 RX + 32 TX */
 #define QEDE_PF_NUM_CONNS		(64)
@@ -151,18 +151,48 @@ struct qede_ucast_entry {
 	SLIST_ENTRY(qede_ucast_entry) list;
 };
 
-struct qede_fdir_entry {
+#ifndef IPV6_ADDR_LEN
+#define IPV6_ADDR_LEN				(16)
+#endif
+
+struct qede_arfs_tuple {
+	union {
+		uint32_t src_ipv4;
+		uint8_t src_ipv6[IPV6_ADDR_LEN];
+	};
+
+	union {
+		uint32_t dst_ipv4;
+		uint8_t dst_ipv6[IPV6_ADDR_LEN];
+	};
+
+	uint16_t	src_port;
+	uint16_t	dst_port;
+	uint16_t	eth_proto;
+	uint8_t		ip_proto;
+
+	/* Describe filtering mode needed for this kind of filter */
+	enum ecore_filter_config_mode mode;
+};
+
+struct qede_arfs_entry {
 	uint32_t soft_id; /* unused for now */
 	uint16_t pkt_len; /* actual packet length to match */
 	uint16_t rx_queue; /* queue to be steered to */
 	const struct rte_memzone *mz; /* mz used to hold L2 frame */
-	SLIST_ENTRY(qede_fdir_entry) list;
+	struct qede_arfs_tuple tuple;
+	SLIST_ENTRY(qede_arfs_entry) list;
 };
 
-struct qede_fdir_info {
+/* Opaque handle for rte flow managed by PMD */
+struct rte_flow {
+	struct qede_arfs_entry entry;
+};
+
+struct qede_arfs_info {
 	struct ecore_arfs_config_params arfs;
 	uint16_t filter_count;
-	SLIST_HEAD(fdir_list_head, qede_fdir_entry)fdir_list_head;
+	SLIST_HEAD(arfs_list_head, qede_arfs_entry)arfs_list_head;
 };
 
 /* IANA assigned default UDP ports for encapsulation protocols */
@@ -207,7 +237,7 @@ struct qede_dev {
 	struct qede_tunn_params vxlan;
 	struct qede_tunn_params geneve;
 	struct qede_tunn_params ipgre;
-	struct qede_fdir_info fdir_info;
+	struct qede_arfs_info arfs_info;
 	bool vlan_strip_flg;
 	char drv_ver[QEDE_PMD_DRV_VER_STR_SIZE];
 	bool vport_started;
@@ -215,6 +245,15 @@ struct qede_dev {
 	void *ethdev;
 };
 
+static inline void qede_set_ucast_cmn_params(struct ecore_filter_ucast *ucast)
+{
+	memset(ucast, 0, sizeof(struct ecore_filter_ucast));
+	ucast->is_rx_filter = true;
+	ucast->is_tx_filter = true;
+	/* ucast->assert_on_error = true; - For debug */
+}
+
+
 /* Non-static functions */
 int qede_config_rss(struct rte_eth_dev *eth_dev);
 
@@ -235,9 +274,6 @@ int qede_link_update(struct rte_eth_dev *eth_dev,
 int qede_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_type type,
 			 enum rte_filter_op op, void *arg);
 
-int qede_fdir_filter_conf(struct rte_eth_dev *eth_dev,
-			  enum rte_filter_op filter_op, void *arg);
-
 int qede_ntuple_filter_conf(struct rte_eth_dev *eth_dev,
 			    enum rte_filter_op filter_op, void *arg);
 
@@ -255,5 +291,16 @@ int qede_activate_vport(struct rte_eth_dev *eth_dev, bool flg);
 int qede_update_mtu(struct rte_eth_dev *eth_dev, uint16_t mtu);
 
 int qede_enable_tpa(struct rte_eth_dev *eth_dev, bool flg);
-
+int qede_udp_dst_port_del(struct rte_eth_dev *eth_dev,
+			  struct rte_eth_udp_tunnel *tunnel_udp);
+int qede_udp_dst_port_add(struct rte_eth_dev *eth_dev,
+			  struct rte_eth_udp_tunnel *tunnel_udp);
+
+enum _ecore_status_t
+qede_mac_int_ops(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *ucast,
+		 bool add);
+void qede_config_accept_any_vlan(struct qede_dev *qdev, bool flg);
+int qede_ucast_filter(struct rte_eth_dev *eth_dev,
+		      struct ecore_filter_ucast *ucast,
+		      bool add);
 #endif /* _QEDE_ETHDEV_H_ */
diff --git a/drivers/net/qede/qede_fdir.c b/drivers/net/qede/qede_fdir.c
deleted file mode 100644
index 83580d04..00000000
--- a/drivers/net/qede/qede_fdir.c
+++ /dev/null
@@ -1,470 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2017 Cavium Inc.
- * All rights reserved.
- * www.cavium.com
- */
-
-#include <rte_udp.h>
-#include <rte_tcp.h>
-#include <rte_sctp.h>
-#include <rte_errno.h>
-
-#include "qede_ethdev.h"
-
-#define IP_VERSION				(0x40)
-#define IP_HDRLEN				(0x5)
-#define QEDE_FDIR_IP_DEFAULT_VERSION_IHL	(IP_VERSION | IP_HDRLEN)
-#define QEDE_FDIR_TCP_DEFAULT_DATAOFF		(0x50)
-#define QEDE_FDIR_IPV4_DEF_TTL			(64)
-
-/* Sum of length of header types of L2, L3, L4.
- * L2 : ether_hdr + vlan_hdr + vxlan_hdr
- * L3 : ipv6_hdr
- * L4 : tcp_hdr
- */
-#define QEDE_MAX_FDIR_PKT_LEN			(86)
-
-#ifndef IPV6_ADDR_LEN
-#define IPV6_ADDR_LEN				(16)
-#endif
-
-#define QEDE_VALID_FLOW(flow_type) \
-	((flow_type) == RTE_ETH_FLOW_NONFRAG_IPV4_TCP	|| \
-	(flow_type) == RTE_ETH_FLOW_NONFRAG_IPV4_UDP	|| \
-	(flow_type) == RTE_ETH_FLOW_NONFRAG_IPV6_TCP	|| \
-	(flow_type) == RTE_ETH_FLOW_NONFRAG_IPV6_UDP)
-
-/* Note: Flowdir support is only partial.
- * For ex: drop_queue, FDIR masks, flex_conf are not supported.
- * Parameters like pballoc/status fields are irrelevant here.
- */
-int qede_check_fdir_support(struct rte_eth_dev *eth_dev)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	struct rte_fdir_conf *fdir = &eth_dev->data->dev_conf.fdir_conf;
-
-	/* check FDIR modes */
-	switch (fdir->mode) {
-	case RTE_FDIR_MODE_NONE:
-		qdev->fdir_info.arfs.arfs_enable = false;
-		DP_INFO(edev, "flowdir is disabled\n");
-	break;
-	case RTE_FDIR_MODE_PERFECT:
-		if (ECORE_IS_CMT(edev)) {
-			DP_ERR(edev, "flowdir is not supported in 100G mode\n");
-			qdev->fdir_info.arfs.arfs_enable = false;
-			return -ENOTSUP;
-		}
-		qdev->fdir_info.arfs.arfs_enable = true;
-		DP_INFO(edev, "flowdir is enabled\n");
-	break;
-	case RTE_FDIR_MODE_PERFECT_TUNNEL:
-	case RTE_FDIR_MODE_SIGNATURE:
-	case RTE_FDIR_MODE_PERFECT_MAC_VLAN:
-		DP_ERR(edev, "Unsupported flowdir mode %d\n", fdir->mode);
-		return -ENOTSUP;
-	}
-
-	return 0;
-}
-
-void qede_fdir_dealloc_resc(struct rte_eth_dev *eth_dev)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct qede_fdir_entry *tmp = NULL;
-
-	SLIST_FOREACH(tmp, &qdev->fdir_info.fdir_list_head, list) {
-		if (tmp) {
-			if (tmp->mz)
-				rte_memzone_free(tmp->mz);
-			SLIST_REMOVE(&qdev->fdir_info.fdir_list_head, tmp,
-				     qede_fdir_entry, list);
-			rte_free(tmp);
-		}
-	}
-}
-
-static int
-qede_config_cmn_fdir_filter(struct rte_eth_dev *eth_dev,
-			    struct rte_eth_fdir_filter *fdir_filter,
-			    bool add)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	char mz_name[RTE_MEMZONE_NAMESIZE] = {0};
-	struct qede_fdir_entry *tmp = NULL;
-	struct qede_fdir_entry *fdir = NULL;
-	const struct rte_memzone *mz;
-	struct ecore_hwfn *p_hwfn;
-	enum _ecore_status_t rc;
-	uint16_t pkt_len;
-	void *pkt;
-
-	if (add) {
-		if (qdev->fdir_info.filter_count == QEDE_RFS_MAX_FLTR - 1) {
-			DP_ERR(edev, "Reached max flowdir filter limit\n");
-			return -EINVAL;
-		}
-		fdir = rte_malloc(NULL, sizeof(struct qede_fdir_entry),
-				  RTE_CACHE_LINE_SIZE);
-		if (!fdir) {
-			DP_ERR(edev, "Did not allocate memory for fdir\n");
-			return -ENOMEM;
-		}
-	}
-	/* soft_id could have been used as memzone string, but soft_id is
-	 * not currently used so it has no significance.
-	 */
-	snprintf(mz_name, sizeof(mz_name) - 1, "%lx",
-		 (unsigned long)rte_get_timer_cycles());
-	mz = rte_memzone_reserve_aligned(mz_name, QEDE_MAX_FDIR_PKT_LEN,
-					 SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
-	if (!mz) {
-		DP_ERR(edev, "Failed to allocate memzone for fdir, err = %s\n",
-		       rte_strerror(rte_errno));
-		rc = -rte_errno;
-		goto err1;
-	}
-
-	pkt = mz->addr;
-	memset(pkt, 0, QEDE_MAX_FDIR_PKT_LEN);
-	pkt_len = qede_fdir_construct_pkt(eth_dev, fdir_filter, pkt,
-					  &qdev->fdir_info.arfs);
-	if (pkt_len == 0) {
-		rc = -EINVAL;
-		goto err2;
-	}
-	DP_INFO(edev, "pkt_len = %u memzone = %s\n", pkt_len, mz_name);
-	if (add) {
-		SLIST_FOREACH(tmp, &qdev->fdir_info.fdir_list_head, list) {
-			if (memcmp(tmp->mz->addr, pkt, pkt_len) == 0) {
-				DP_INFO(edev, "flowdir filter exist\n");
-				rc = 0;
-				goto err2;
-			}
-		}
-	} else {
-		SLIST_FOREACH(tmp, &qdev->fdir_info.fdir_list_head, list) {
-			if (memcmp(tmp->mz->addr, pkt, pkt_len) == 0)
-				break;
-		}
-		if (!tmp) {
-			DP_ERR(edev, "flowdir filter does not exist\n");
-			rc = -EEXIST;
-			goto err2;
-		}
-	}
-	p_hwfn = ECORE_LEADING_HWFN(edev);
-	if (add) {
-		if (!qdev->fdir_info.arfs.arfs_enable) {
-			/* Force update */
-			eth_dev->data->dev_conf.fdir_conf.mode =
-						RTE_FDIR_MODE_PERFECT;
-			qdev->fdir_info.arfs.arfs_enable = true;
-			DP_INFO(edev, "Force enable flowdir in perfect mode\n");
-		}
-		/* Enable ARFS searcher with updated flow_types */
-		ecore_arfs_mode_configure(p_hwfn, p_hwfn->p_arfs_ptt,
-					  &qdev->fdir_info.arfs);
-	}
-	/* configure filter with ECORE_SPQ_MODE_EBLOCK */
-	rc = ecore_configure_rfs_ntuple_filter(p_hwfn, NULL,
-					       (dma_addr_t)mz->iova,
-					       pkt_len,
-					       fdir_filter->action.rx_queue,
-					       0, add);
-	if (rc == ECORE_SUCCESS) {
-		if (add) {
-			fdir->rx_queue = fdir_filter->action.rx_queue;
-			fdir->pkt_len = pkt_len;
-			fdir->mz = mz;
-			SLIST_INSERT_HEAD(&qdev->fdir_info.fdir_list_head,
-					  fdir, list);
-			qdev->fdir_info.filter_count++;
-			DP_INFO(edev, "flowdir filter added, count = %d\n",
-				qdev->fdir_info.filter_count);
-		} else {
-			rte_memzone_free(tmp->mz);
-			SLIST_REMOVE(&qdev->fdir_info.fdir_list_head, tmp,
-				     qede_fdir_entry, list);
-			rte_free(tmp); /* the node deleted */
-			rte_memzone_free(mz); /* temp node allocated */
-			qdev->fdir_info.filter_count--;
-			DP_INFO(edev, "Fdir filter deleted, count = %d\n",
-				qdev->fdir_info.filter_count);
-		}
-	} else {
-		DP_ERR(edev, "flowdir filter failed, rc=%d filter_count=%d\n",
-		       rc, qdev->fdir_info.filter_count);
-	}
-
-	/* Disable ARFS searcher if there are no more filters */
-	if (qdev->fdir_info.filter_count == 0) {
-		memset(&qdev->fdir_info.arfs, 0,
-		       sizeof(struct ecore_arfs_config_params));
-		DP_INFO(edev, "Disabling flowdir\n");
-		qdev->fdir_info.arfs.arfs_enable = false;
-		ecore_arfs_mode_configure(p_hwfn, p_hwfn->p_arfs_ptt,
-					  &qdev->fdir_info.arfs);
-	}
-	return 0;
-
-err2:
-	rte_memzone_free(mz);
-err1:
-	if (add)
-		rte_free(fdir);
-	return rc;
-}
-
-static int
-qede_fdir_filter_add(struct rte_eth_dev *eth_dev,
-		     struct rte_eth_fdir_filter *fdir,
-		     bool add)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-
-	if (!QEDE_VALID_FLOW(fdir->input.flow_type)) {
-		DP_ERR(edev, "invalid flow_type input\n");
-		return -EINVAL;
-	}
-
-	if (fdir->action.rx_queue >= QEDE_RSS_COUNT(qdev)) {
-		DP_ERR(edev, "invalid queue number %u\n",
-		       fdir->action.rx_queue);
-		return -EINVAL;
-	}
-
-	if (fdir->input.flow_ext.is_vf) {
-		DP_ERR(edev, "flowdir is not supported over VF\n");
-		return -EINVAL;
-	}
-
-	return qede_config_cmn_fdir_filter(eth_dev, fdir, add);
-}
-
-/* Fills the L3/L4 headers and returns the actual length  of flowdir packet */
-uint16_t
-qede_fdir_construct_pkt(struct rte_eth_dev *eth_dev,
-			struct rte_eth_fdir_filter *fdir,
-			void *buff,
-			struct ecore_arfs_config_params *params)
-
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	uint16_t *ether_type;
-	uint8_t *raw_pkt;
-	struct rte_eth_fdir_input *input;
-	static uint8_t vlan_frame[] = {0x81, 0, 0, 0};
-	struct ipv4_hdr *ip;
-	struct ipv6_hdr *ip6;
-	struct udp_hdr *udp;
-	struct tcp_hdr *tcp;
-	uint16_t len;
-	static const uint8_t next_proto[] = {
-		[RTE_ETH_FLOW_NONFRAG_IPV4_TCP] = IPPROTO_TCP,
-		[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] = IPPROTO_UDP,
-		[RTE_ETH_FLOW_NONFRAG_IPV6_TCP] = IPPROTO_TCP,
-		[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] = IPPROTO_UDP,
-	};
-	raw_pkt = (uint8_t *)buff;
-	input = &fdir->input;
-	DP_INFO(edev, "flow_type %d\n", input->flow_type);
-
-	len =  2 * sizeof(struct ether_addr);
-	raw_pkt += 2 * sizeof(struct ether_addr);
-	if (input->flow_ext.vlan_tci) {
-		DP_INFO(edev, "adding VLAN header\n");
-		rte_memcpy(raw_pkt, vlan_frame, sizeof(vlan_frame));
-		rte_memcpy(raw_pkt + sizeof(uint16_t),
-			   &input->flow_ext.vlan_tci,
-			   sizeof(uint16_t));
-		raw_pkt += sizeof(vlan_frame);
-		len += sizeof(vlan_frame);
-	}
-	ether_type = (uint16_t *)raw_pkt;
-	raw_pkt += sizeof(uint16_t);
-	len += sizeof(uint16_t);
-
-	switch (input->flow_type) {
-	case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
-	case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
-		/* fill the common ip header */
-		ip = (struct ipv4_hdr *)raw_pkt;
-		*ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4);
-		ip->version_ihl = QEDE_FDIR_IP_DEFAULT_VERSION_IHL;
-		ip->total_length = sizeof(struct ipv4_hdr);
-		ip->next_proto_id = input->flow.ip4_flow.proto ?
-				    input->flow.ip4_flow.proto :
-				    next_proto[input->flow_type];
-		ip->time_to_live = input->flow.ip4_flow.ttl ?
-				   input->flow.ip4_flow.ttl :
-				   QEDE_FDIR_IPV4_DEF_TTL;
-		ip->type_of_service = input->flow.ip4_flow.tos;
-		ip->dst_addr = input->flow.ip4_flow.dst_ip;
-		ip->src_addr = input->flow.ip4_flow.src_ip;
-		len += sizeof(struct ipv4_hdr);
-		params->ipv4 = true;
-
-		raw_pkt = (uint8_t *)buff;
-		/* UDP */
-		if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_UDP) {
-			udp = (struct udp_hdr *)(raw_pkt + len);
-			udp->dst_port = input->flow.udp4_flow.dst_port;
-			udp->src_port = input->flow.udp4_flow.src_port;
-			udp->dgram_len = sizeof(struct udp_hdr);
-			len += sizeof(struct udp_hdr);
-			/* adjust ip total_length */
-			ip->total_length += sizeof(struct udp_hdr);
-			params->udp = true;
-		} else { /* TCP */
-			tcp = (struct tcp_hdr *)(raw_pkt + len);
-			tcp->src_port = input->flow.tcp4_flow.src_port;
-			tcp->dst_port = input->flow.tcp4_flow.dst_port;
-			tcp->data_off = QEDE_FDIR_TCP_DEFAULT_DATAOFF;
-			len += sizeof(struct tcp_hdr);
-			/* adjust ip total_length */
-			ip->total_length += sizeof(struct tcp_hdr);
-			params->tcp = true;
-		}
-		break;
-	case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
-	case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
-		ip6 = (struct ipv6_hdr *)raw_pkt;
-		*ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv6);
-		ip6->proto = input->flow.ipv6_flow.proto ?
-					input->flow.ipv6_flow.proto :
-					next_proto[input->flow_type];
-		rte_memcpy(&ip6->src_addr, &input->flow.ipv6_flow.dst_ip,
-			   IPV6_ADDR_LEN);
-		rte_memcpy(&ip6->dst_addr, &input->flow.ipv6_flow.src_ip,
-			   IPV6_ADDR_LEN);
-		len += sizeof(struct ipv6_hdr);
-
-		raw_pkt = (uint8_t *)buff;
-		/* UDP */
-		if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_UDP) {
-			udp = (struct udp_hdr *)(raw_pkt + len);
-			udp->src_port = input->flow.udp6_flow.dst_port;
-			udp->dst_port = input->flow.udp6_flow.src_port;
-			len += sizeof(struct udp_hdr);
-			params->udp = true;
-		} else { /* TCP */
-			tcp = (struct tcp_hdr *)(raw_pkt + len);
-			tcp->src_port = input->flow.tcp4_flow.src_port;
-			tcp->dst_port = input->flow.tcp4_flow.dst_port;
-			tcp->data_off = QEDE_FDIR_TCP_DEFAULT_DATAOFF;
-			len += sizeof(struct tcp_hdr);
-			params->tcp = true;
-		}
-		break;
-	default:
-		DP_ERR(edev, "Unsupported flow_type %u\n",
-		       input->flow_type);
-		return 0;
-	}
-
-	return len;
-}
-
-int
-qede_fdir_filter_conf(struct rte_eth_dev *eth_dev,
-		      enum rte_filter_op filter_op,
-		      void *arg)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	struct rte_eth_fdir_filter *fdir;
-	int ret;
-
-	fdir = (struct rte_eth_fdir_filter *)arg;
-	switch (filter_op) {
-	case RTE_ETH_FILTER_NOP:
-		/* Typically used to query flowdir support */
-		if (ECORE_IS_CMT(edev)) {
-			DP_ERR(edev, "flowdir is not supported in 100G mode\n");
-			return -ENOTSUP;
-		}
-		return 0; /* means supported */
-	case RTE_ETH_FILTER_ADD:
-		ret = qede_fdir_filter_add(eth_dev, fdir, 1);
-	break;
-	case RTE_ETH_FILTER_DELETE:
-		ret = qede_fdir_filter_add(eth_dev, fdir, 0);
-	break;
-	case RTE_ETH_FILTER_FLUSH:
-	case RTE_ETH_FILTER_UPDATE:
-	case RTE_ETH_FILTER_INFO:
-		return -ENOTSUP;
-	break;
-	default:
-		DP_ERR(edev, "unknown operation %u", filter_op);
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-int qede_ntuple_filter_conf(struct rte_eth_dev *eth_dev,
-			    enum rte_filter_op filter_op,
-			    void *arg)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	struct rte_eth_ntuple_filter *ntuple;
-	struct rte_eth_fdir_filter fdir_entry;
-	struct rte_eth_tcpv4_flow *tcpv4_flow;
-	struct rte_eth_udpv4_flow *udpv4_flow;
-	bool add = false;
-
-	switch (filter_op) {
-	case RTE_ETH_FILTER_NOP:
-		/* Typically used to query fdir support */
-		if (ECORE_IS_CMT(edev)) {
-			DP_ERR(edev, "flowdir is not supported in 100G mode\n");
-			return -ENOTSUP;
-		}
-		return 0; /* means supported */
-	case RTE_ETH_FILTER_ADD:
-		add = true;
-	break;
-	case RTE_ETH_FILTER_DELETE:
-	break;
-	case RTE_ETH_FILTER_INFO:
-	case RTE_ETH_FILTER_GET:
-	case RTE_ETH_FILTER_UPDATE:
-	case RTE_ETH_FILTER_FLUSH:
-	case RTE_ETH_FILTER_SET:
-	case RTE_ETH_FILTER_STATS:
-	case RTE_ETH_FILTER_OP_MAX:
-		DP_ERR(edev, "Unsupported filter_op %d\n", filter_op);
-		return -ENOTSUP;
-	}
-	ntuple = (struct rte_eth_ntuple_filter *)arg;
-	/* Internally convert ntuple to fdir entry */
-	memset(&fdir_entry, 0, sizeof(fdir_entry));
-	if (ntuple->proto == IPPROTO_TCP) {
-		fdir_entry.input.flow_type = RTE_ETH_FLOW_NONFRAG_IPV4_TCP;
-		tcpv4_flow = &fdir_entry.input.flow.tcp4_flow;
-		tcpv4_flow->ip.src_ip = ntuple->src_ip;
-		tcpv4_flow->ip.dst_ip = ntuple->dst_ip;
-		tcpv4_flow->ip.proto = IPPROTO_TCP;
-		tcpv4_flow->src_port = ntuple->src_port;
-		tcpv4_flow->dst_port = ntuple->dst_port;
-	} else {
-		fdir_entry.input.flow_type = RTE_ETH_FLOW_NONFRAG_IPV4_UDP;
-		udpv4_flow = &fdir_entry.input.flow.udp4_flow;
-		udpv4_flow->ip.src_ip = ntuple->src_ip;
-		udpv4_flow->ip.dst_ip = ntuple->dst_ip;
-		udpv4_flow->ip.proto = IPPROTO_TCP;
-		udpv4_flow->src_port = ntuple->src_port;
-		udpv4_flow->dst_port = ntuple->dst_port;
-	}
-
-	fdir_entry.action.rx_queue = ntuple->queue;
-
-	return qede_config_cmn_fdir_filter(eth_dev, &fdir_entry, add);
-}
diff --git a/drivers/net/qede/qede_filter.c b/drivers/net/qede/qede_filter.c
new file mode 100644
index 00000000..5e6571ca
--- /dev/null
+++ b/drivers/net/qede/qede_filter.c
@@ -0,0 +1,1546 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2017 Cavium Inc.
+ * All rights reserved.
+ * www.cavium.com
+ */
+
+#include <rte_udp.h>
+#include <rte_tcp.h>
+#include <rte_sctp.h>
+#include <rte_errno.h>
+#include <rte_flow_driver.h>
+
+#include "qede_ethdev.h"
+
+/* VXLAN tunnel classification mapping */
+const struct _qede_udp_tunn_types {
+	uint16_t rte_filter_type;
+	enum ecore_filter_ucast_type qede_type;
+	enum ecore_tunn_clss qede_tunn_clss;
+	const char *string;
+} qede_tunn_types[] = {
+	{
+		ETH_TUNNEL_FILTER_OMAC,
+		ECORE_FILTER_MAC,
+		ECORE_TUNN_CLSS_MAC_VLAN,
+		"outer-mac"
+	},
+	{
+		ETH_TUNNEL_FILTER_TENID,
+		ECORE_FILTER_VNI,
+		ECORE_TUNN_CLSS_MAC_VNI,
+		"vni"
+	},
+	{
+		ETH_TUNNEL_FILTER_IMAC,
+		ECORE_FILTER_INNER_MAC,
+		ECORE_TUNN_CLSS_INNER_MAC_VLAN,
+		"inner-mac"
+	},
+	{
+		ETH_TUNNEL_FILTER_IVLAN,
+		ECORE_FILTER_INNER_VLAN,
+		ECORE_TUNN_CLSS_INNER_MAC_VLAN,
+		"inner-vlan"
+	},
+	{
+		ETH_TUNNEL_FILTER_OMAC | ETH_TUNNEL_FILTER_TENID,
+		ECORE_FILTER_MAC_VNI_PAIR,
+		ECORE_TUNN_CLSS_MAC_VNI,
+		"outer-mac and vni"
+	},
+	{
+		ETH_TUNNEL_FILTER_OMAC | ETH_TUNNEL_FILTER_IMAC,
+		ECORE_FILTER_UNUSED,
+		MAX_ECORE_TUNN_CLSS,
+		"outer-mac and inner-mac"
+	},
+	{
+		ETH_TUNNEL_FILTER_OMAC | ETH_TUNNEL_FILTER_IVLAN,
+		ECORE_FILTER_UNUSED,
+		MAX_ECORE_TUNN_CLSS,
+		"outer-mac and inner-vlan"
+	},
+	{
+		ETH_TUNNEL_FILTER_TENID | ETH_TUNNEL_FILTER_IMAC,
+		ECORE_FILTER_INNER_MAC_VNI_PAIR,
+		ECORE_TUNN_CLSS_INNER_MAC_VNI,
+		"vni and inner-mac",
+	},
+	{
+		ETH_TUNNEL_FILTER_TENID | ETH_TUNNEL_FILTER_IVLAN,
+		ECORE_FILTER_UNUSED,
+		MAX_ECORE_TUNN_CLSS,
+		"vni and inner-vlan",
+	},
+	{
+		ETH_TUNNEL_FILTER_IMAC | ETH_TUNNEL_FILTER_IVLAN,
+		ECORE_FILTER_INNER_PAIR,
+		ECORE_TUNN_CLSS_INNER_MAC_VLAN,
+		"inner-mac and inner-vlan",
+	},
+	{
+		ETH_TUNNEL_FILTER_OIP,
+		ECORE_FILTER_UNUSED,
+		MAX_ECORE_TUNN_CLSS,
+		"outer-IP"
+	},
+	{
+		ETH_TUNNEL_FILTER_IIP,
+		ECORE_FILTER_UNUSED,
+		MAX_ECORE_TUNN_CLSS,
+		"inner-IP"
+	},
+	{
+		RTE_TUNNEL_FILTER_IMAC_IVLAN,
+		ECORE_FILTER_UNUSED,
+		MAX_ECORE_TUNN_CLSS,
+		"IMAC_IVLAN"
+	},
+	{
+		RTE_TUNNEL_FILTER_IMAC_IVLAN_TENID,
+		ECORE_FILTER_UNUSED,
+		MAX_ECORE_TUNN_CLSS,
+		"IMAC_IVLAN_TENID"
+	},
+	{
+		RTE_TUNNEL_FILTER_IMAC_TENID,
+		ECORE_FILTER_UNUSED,
+		MAX_ECORE_TUNN_CLSS,
+		"IMAC_TENID"
+	},
+	{
+		RTE_TUNNEL_FILTER_OMAC_TENID_IMAC,
+		ECORE_FILTER_UNUSED,
+		MAX_ECORE_TUNN_CLSS,
+		"OMAC_TENID_IMAC"
+	},
+};
+
+#define IP_VERSION				(0x40)
+#define IP_HDRLEN				(0x5)
+#define QEDE_FDIR_IP_DEFAULT_VERSION_IHL	(IP_VERSION | IP_HDRLEN)
+#define QEDE_FDIR_TCP_DEFAULT_DATAOFF		(0x50)
+#define QEDE_FDIR_IPV4_DEF_TTL			(64)
+#define QEDE_FDIR_IPV6_DEFAULT_VTC_FLOW		(0x60000000)
+/* Sum of length of header types of L2, L3, L4.
+ * L2 : ether_hdr + vlan_hdr + vxlan_hdr
+ * L3 : ipv6_hdr
+ * L4 : tcp_hdr
+ */
+#define QEDE_MAX_FDIR_PKT_LEN			(86)
+
+static inline bool qede_valid_flow(uint16_t flow_type)
+{
+	return  ((flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_TCP) ||
+		 (flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_UDP) ||
+		 (flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_TCP) ||
+		 (flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_UDP));
+}
+
+static uint16_t
+qede_arfs_construct_pkt(struct rte_eth_dev *eth_dev,
+			struct qede_arfs_entry *arfs,
+			void *buff,
+			struct ecore_arfs_config_params *params);
+
+/* Note: Flowdir support is only partial.
+ * For ex: drop_queue, FDIR masks, flex_conf are not supported.
+ * Parameters like pballoc/status fields are irrelevant here.
+ */
+int qede_check_fdir_support(struct rte_eth_dev *eth_dev)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	struct rte_fdir_conf *fdir = &eth_dev->data->dev_conf.fdir_conf;
+
+	/* check FDIR modes */
+	switch (fdir->mode) {
+	case RTE_FDIR_MODE_NONE:
+		qdev->arfs_info.arfs.mode = ECORE_FILTER_CONFIG_MODE_DISABLE;
+		DP_INFO(edev, "flowdir is disabled\n");
+	break;
+	case RTE_FDIR_MODE_PERFECT:
+		if (ECORE_IS_CMT(edev)) {
+			DP_ERR(edev, "flowdir is not supported in 100G mode\n");
+			qdev->arfs_info.arfs.mode =
+				ECORE_FILTER_CONFIG_MODE_DISABLE;
+			return -ENOTSUP;
+		}
+		qdev->arfs_info.arfs.mode =
+				ECORE_FILTER_CONFIG_MODE_5_TUPLE;
+		DP_INFO(edev, "flowdir is enabled (5 Tuple mode)\n");
+	break;
+	case RTE_FDIR_MODE_PERFECT_TUNNEL:
+	case RTE_FDIR_MODE_SIGNATURE:
+	case RTE_FDIR_MODE_PERFECT_MAC_VLAN:
+		DP_ERR(edev, "Unsupported flowdir mode %d\n", fdir->mode);
+		return -ENOTSUP;
+	}
+
+	return 0;
+}
+
+void qede_fdir_dealloc_resc(struct rte_eth_dev *eth_dev)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct qede_arfs_entry *tmp = NULL;
+
+	SLIST_FOREACH(tmp, &qdev->arfs_info.arfs_list_head, list) {
+		if (tmp) {
+			if (tmp->mz)
+				rte_memzone_free(tmp->mz);
+			SLIST_REMOVE(&qdev->arfs_info.arfs_list_head, tmp,
+				     qede_arfs_entry, list);
+			rte_free(tmp);
+		}
+	}
+}
+
+static int
+qede_fdir_to_arfs_filter(struct rte_eth_dev *eth_dev,
+			 struct rte_eth_fdir_filter *fdir,
+			 struct qede_arfs_entry *arfs)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	struct rte_eth_fdir_input *input;
+
+	static const uint8_t next_proto[] = {
+		[RTE_ETH_FLOW_NONFRAG_IPV4_TCP] = IPPROTO_TCP,
+		[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] = IPPROTO_UDP,
+		[RTE_ETH_FLOW_NONFRAG_IPV6_TCP] = IPPROTO_TCP,
+		[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] = IPPROTO_UDP,
+	};
+
+	input = &fdir->input;
+
+	DP_INFO(edev, "flow_type %d\n", input->flow_type);
+
+	switch (input->flow_type) {
+	case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
+	case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
+		/* fill the common ip header */
+		arfs->tuple.eth_proto = ETHER_TYPE_IPv4;
+		arfs->tuple.dst_ipv4 = input->flow.ip4_flow.dst_ip;
+		arfs->tuple.src_ipv4 = input->flow.ip4_flow.src_ip;
+		arfs->tuple.ip_proto = next_proto[input->flow_type];
+
+		/* UDP */
+		if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_UDP) {
+			arfs->tuple.dst_port = input->flow.udp4_flow.dst_port;
+			arfs->tuple.src_port = input->flow.udp4_flow.src_port;
+		} else { /* TCP */
+			arfs->tuple.dst_port = input->flow.tcp4_flow.dst_port;
+			arfs->tuple.src_port = input->flow.tcp4_flow.src_port;
+		}
+		break;
+	case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
+	case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
+		arfs->tuple.eth_proto = ETHER_TYPE_IPv6;
+		arfs->tuple.ip_proto = next_proto[input->flow_type];
+		rte_memcpy(arfs->tuple.dst_ipv6,
+			   &input->flow.ipv6_flow.dst_ip,
+			   IPV6_ADDR_LEN);
+		rte_memcpy(arfs->tuple.src_ipv6,
+			   &input->flow.ipv6_flow.src_ip,
+			   IPV6_ADDR_LEN);
+
+		/* UDP */
+		if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_UDP) {
+			arfs->tuple.dst_port = input->flow.udp6_flow.dst_port;
+			arfs->tuple.src_port = input->flow.udp6_flow.src_port;
+		} else { /* TCP */
+			arfs->tuple.dst_port = input->flow.tcp6_flow.dst_port;
+			arfs->tuple.src_port = input->flow.tcp6_flow.src_port;
+		}
+		break;
+	default:
+		DP_ERR(edev, "Unsupported flow_type %u\n",
+		       input->flow_type);
+		return -ENOTSUP;
+	}
+
+	arfs->rx_queue = fdir->action.rx_queue;
+	return 0;
+}
+
+static int
+qede_config_arfs_filter(struct rte_eth_dev *eth_dev,
+			struct qede_arfs_entry *arfs,
+			bool add)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	char mz_name[RTE_MEMZONE_NAMESIZE] = {0};
+	struct qede_arfs_entry *tmp = NULL;
+	const struct rte_memzone *mz;
+	struct ecore_hwfn *p_hwfn;
+	enum _ecore_status_t rc;
+	uint16_t pkt_len;
+	void *pkt;
+
+	if (add) {
+		if (qdev->arfs_info.filter_count == QEDE_RFS_MAX_FLTR - 1) {
+			DP_ERR(edev, "Reached max flowdir filter limit\n");
+			return -EINVAL;
+		}
+	}
+
+	/* soft_id could have been used as memzone string, but soft_id is
+	 * not currently used so it has no significance.
+	 */
+	snprintf(mz_name, sizeof(mz_name) - 1, "%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	mz = rte_memzone_reserve_aligned(mz_name, QEDE_MAX_FDIR_PKT_LEN,
+					 SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
+	if (!mz) {
+		DP_ERR(edev, "Failed to allocate memzone for fdir, err = %s\n",
+		       rte_strerror(rte_errno));
+		return -rte_errno;
+	}
+
+	pkt = mz->addr;
+	memset(pkt, 0, QEDE_MAX_FDIR_PKT_LEN);
+	pkt_len = qede_arfs_construct_pkt(eth_dev, arfs, pkt,
+					  &qdev->arfs_info.arfs);
+	if (pkt_len == 0) {
+		rc = -EINVAL;
+		goto err1;
+	}
+
+	DP_INFO(edev, "pkt_len = %u memzone = %s\n", pkt_len, mz_name);
+	if (add) {
+		SLIST_FOREACH(tmp, &qdev->arfs_info.arfs_list_head, list) {
+			if (memcmp(tmp->mz->addr, pkt, pkt_len) == 0) {
+				DP_INFO(edev, "flowdir filter exist\n");
+				rc = -EEXIST;
+				goto err1;
+			}
+		}
+	} else {
+		SLIST_FOREACH(tmp, &qdev->arfs_info.arfs_list_head, list) {
+			if (memcmp(tmp->mz->addr, pkt, pkt_len) == 0)
+				break;
+		}
+		if (!tmp) {
+			DP_ERR(edev, "flowdir filter does not exist\n");
+			rc = -EEXIST;
+			goto err1;
+		}
+	}
+	p_hwfn = ECORE_LEADING_HWFN(edev);
+	if (add) {
+		if (qdev->arfs_info.arfs.mode ==
+			ECORE_FILTER_CONFIG_MODE_DISABLE) {
+			/* Force update */
+			eth_dev->data->dev_conf.fdir_conf.mode =
+						RTE_FDIR_MODE_PERFECT;
+			qdev->arfs_info.arfs.mode =
+					ECORE_FILTER_CONFIG_MODE_5_TUPLE;
+			DP_INFO(edev, "Force enable flowdir in perfect mode\n");
+		}
+		/* Enable ARFS searcher with updated flow_types */
+		ecore_arfs_mode_configure(p_hwfn, p_hwfn->p_arfs_ptt,
+					  &qdev->arfs_info.arfs);
+	}
+	/* configure filter with ECORE_SPQ_MODE_EBLOCK */
+	rc = ecore_configure_rfs_ntuple_filter(p_hwfn, NULL,
+					       (dma_addr_t)mz->iova,
+					       pkt_len,
+					       arfs->rx_queue,
+					       0, add);
+	if (rc == ECORE_SUCCESS) {
+		if (add) {
+			arfs->pkt_len = pkt_len;
+			arfs->mz = mz;
+			SLIST_INSERT_HEAD(&qdev->arfs_info.arfs_list_head,
+					  arfs, list);
+			qdev->arfs_info.filter_count++;
+			DP_INFO(edev, "flowdir filter added, count = %d\n",
+				qdev->arfs_info.filter_count);
+		} else {
+			rte_memzone_free(tmp->mz);
+			SLIST_REMOVE(&qdev->arfs_info.arfs_list_head, tmp,
+				     qede_arfs_entry, list);
+			rte_free(tmp); /* the node deleted */
+			rte_memzone_free(mz); /* temp node allocated */
+			qdev->arfs_info.filter_count--;
+			DP_INFO(edev, "Fdir filter deleted, count = %d\n",
+				qdev->arfs_info.filter_count);
+		}
+	} else {
+		DP_ERR(edev, "flowdir filter failed, rc=%d filter_count=%d\n",
+		       rc, qdev->arfs_info.filter_count);
+	}
+
+	/* Disable ARFS searcher if there are no more filters */
+	if (qdev->arfs_info.filter_count == 0) {
+		memset(&qdev->arfs_info.arfs, 0,
+		       sizeof(struct ecore_arfs_config_params));
+		DP_INFO(edev, "Disabling flowdir\n");
+		qdev->arfs_info.arfs.mode = ECORE_FILTER_CONFIG_MODE_DISABLE;
+		ecore_arfs_mode_configure(p_hwfn, p_hwfn->p_arfs_ptt,
+					  &qdev->arfs_info.arfs);
+	}
+	return 0;
+
+err1:
+	rte_memzone_free(mz);
+	return rc;
+}
+
+static int
+qede_config_cmn_fdir_filter(struct rte_eth_dev *eth_dev,
+			    struct rte_eth_fdir_filter *fdir_filter,
+			    bool add)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	struct qede_arfs_entry *arfs = NULL;
+	int rc = 0;
+
+	arfs = rte_malloc(NULL, sizeof(struct qede_arfs_entry),
+				  RTE_CACHE_LINE_SIZE);
+	if (!arfs) {
+		DP_ERR(edev, "Did not allocate memory for arfs\n");
+		return -ENOMEM;
+	}
+
+	rc = qede_fdir_to_arfs_filter(eth_dev, fdir_filter, arfs);
+	if (rc < 0)
+		return rc;
+
+	rc = qede_config_arfs_filter(eth_dev, arfs, add);
+	if (rc < 0)
+		rte_free(arfs);
+
+	return rc;
+}
+
+static int
+qede_fdir_filter_add(struct rte_eth_dev *eth_dev,
+		     struct rte_eth_fdir_filter *fdir,
+		     bool add)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+
+	if (!qede_valid_flow(fdir->input.flow_type)) {
+		DP_ERR(edev, "invalid flow_type input\n");
+		return -EINVAL;
+	}
+
+	if (fdir->action.rx_queue >= QEDE_RSS_COUNT(qdev)) {
+		DP_ERR(edev, "invalid queue number %u\n",
+		       fdir->action.rx_queue);
+		return -EINVAL;
+	}
+
+	if (fdir->input.flow_ext.is_vf) {
+		DP_ERR(edev, "flowdir is not supported over VF\n");
+		return -EINVAL;
+	}
+
+	return qede_config_cmn_fdir_filter(eth_dev, fdir, add);
+}
+
+/* Fills the L3/L4 headers and returns the actual length  of flowdir packet */
+static uint16_t
+qede_arfs_construct_pkt(struct rte_eth_dev *eth_dev,
+			struct qede_arfs_entry *arfs,
+			void *buff,
+			struct ecore_arfs_config_params *params)
+
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	uint16_t *ether_type;
+	uint8_t *raw_pkt;
+	struct ipv4_hdr *ip;
+	struct ipv6_hdr *ip6;
+	struct udp_hdr *udp;
+	struct tcp_hdr *tcp;
+	uint16_t len;
+
+	raw_pkt = (uint8_t *)buff;
+
+	len =  2 * sizeof(struct ether_addr);
+	raw_pkt += 2 * sizeof(struct ether_addr);
+	ether_type = (uint16_t *)raw_pkt;
+	raw_pkt += sizeof(uint16_t);
+	len += sizeof(uint16_t);
+
+	*ether_type = rte_cpu_to_be_16(arfs->tuple.eth_proto);
+	switch (arfs->tuple.eth_proto) {
+	case ETHER_TYPE_IPv4:
+		ip = (struct ipv4_hdr *)raw_pkt;
+		ip->version_ihl = QEDE_FDIR_IP_DEFAULT_VERSION_IHL;
+		ip->total_length = sizeof(struct ipv4_hdr);
+		ip->next_proto_id = arfs->tuple.ip_proto;
+		ip->time_to_live = QEDE_FDIR_IPV4_DEF_TTL;
+		ip->dst_addr = arfs->tuple.dst_ipv4;
+		ip->src_addr = arfs->tuple.src_ipv4;
+		len += sizeof(struct ipv4_hdr);
+		params->ipv4 = true;
+
+		raw_pkt = (uint8_t *)buff;
+		/* UDP */
+		if (arfs->tuple.ip_proto == IPPROTO_UDP) {
+			udp = (struct udp_hdr *)(raw_pkt + len);
+			udp->dst_port = arfs->tuple.dst_port;
+			udp->src_port = arfs->tuple.src_port;
+			udp->dgram_len = sizeof(struct udp_hdr);
+			len += sizeof(struct udp_hdr);
+			/* adjust ip total_length */
+			ip->total_length += sizeof(struct udp_hdr);
+			params->udp = true;
+		} else { /* TCP */
+			tcp = (struct tcp_hdr *)(raw_pkt + len);
+			tcp->src_port = arfs->tuple.src_port;
+			tcp->dst_port = arfs->tuple.dst_port;
+			tcp->data_off = QEDE_FDIR_TCP_DEFAULT_DATAOFF;
+			len += sizeof(struct tcp_hdr);
+			/* adjust ip total_length */
+			ip->total_length += sizeof(struct tcp_hdr);
+			params->tcp = true;
+		}
+		break;
+	case ETHER_TYPE_IPv6:
+		ip6 = (struct ipv6_hdr *)raw_pkt;
+		ip6->proto = arfs->tuple.ip_proto;
+		ip6->vtc_flow =
+			rte_cpu_to_be_32(QEDE_FDIR_IPV6_DEFAULT_VTC_FLOW);
+
+		rte_memcpy(&ip6->src_addr, arfs->tuple.src_ipv6,
+			   IPV6_ADDR_LEN);
+		rte_memcpy(&ip6->dst_addr, arfs->tuple.dst_ipv6,
+			   IPV6_ADDR_LEN);
+		len += sizeof(struct ipv6_hdr);
+		params->ipv6 = true;
+
+		raw_pkt = (uint8_t *)buff;
+		/* UDP */
+		if (arfs->tuple.ip_proto == IPPROTO_UDP) {
+			udp = (struct udp_hdr *)(raw_pkt + len);
+			udp->src_port = arfs->tuple.src_port;
+			udp->dst_port = arfs->tuple.dst_port;
+			len += sizeof(struct udp_hdr);
+			params->udp = true;
+		} else { /* TCP */
+			tcp = (struct tcp_hdr *)(raw_pkt + len);
+			tcp->src_port = arfs->tuple.src_port;
+			tcp->dst_port = arfs->tuple.dst_port;
+			tcp->data_off = QEDE_FDIR_TCP_DEFAULT_DATAOFF;
+			len += sizeof(struct tcp_hdr);
+			params->tcp = true;
+		}
+		break;
+	default:
+		DP_ERR(edev, "Unsupported eth_proto %u\n",
+		       arfs->tuple.eth_proto);
+		return 0;
+	}
+
+	return len;
+}
+
+static int
+qede_fdir_filter_conf(struct rte_eth_dev *eth_dev,
+		      enum rte_filter_op filter_op,
+		      void *arg)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	struct rte_eth_fdir_filter *fdir;
+	int ret;
+
+	fdir = (struct rte_eth_fdir_filter *)arg;
+	switch (filter_op) {
+	case RTE_ETH_FILTER_NOP:
+		/* Typically used to query flowdir support */
+		if (ECORE_IS_CMT(edev)) {
+			DP_ERR(edev, "flowdir is not supported in 100G mode\n");
+			return -ENOTSUP;
+		}
+		return 0; /* means supported */
+	case RTE_ETH_FILTER_ADD:
+		ret = qede_fdir_filter_add(eth_dev, fdir, 1);
+	break;
+	case RTE_ETH_FILTER_DELETE:
+		ret = qede_fdir_filter_add(eth_dev, fdir, 0);
+	break;
+	case RTE_ETH_FILTER_FLUSH:
+	case RTE_ETH_FILTER_UPDATE:
+	case RTE_ETH_FILTER_INFO:
+		return -ENOTSUP;
+	break;
+	default:
+		DP_ERR(edev, "unknown operation %u", filter_op);
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+int qede_ntuple_filter_conf(struct rte_eth_dev *eth_dev,
+			    enum rte_filter_op filter_op,
+			    void *arg)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	struct rte_eth_ntuple_filter *ntuple;
+	struct rte_eth_fdir_filter fdir_entry;
+	struct rte_eth_tcpv4_flow *tcpv4_flow;
+	struct rte_eth_udpv4_flow *udpv4_flow;
+	bool add = false;
+
+	switch (filter_op) {
+	case RTE_ETH_FILTER_NOP:
+		/* Typically used to query fdir support */
+		if (ECORE_IS_CMT(edev)) {
+			DP_ERR(edev, "flowdir is not supported in 100G mode\n");
+			return -ENOTSUP;
+		}
+		return 0; /* means supported */
+	case RTE_ETH_FILTER_ADD:
+		add = true;
+	break;
+	case RTE_ETH_FILTER_DELETE:
+	break;
+	case RTE_ETH_FILTER_INFO:
+	case RTE_ETH_FILTER_GET:
+	case RTE_ETH_FILTER_UPDATE:
+	case RTE_ETH_FILTER_FLUSH:
+	case RTE_ETH_FILTER_SET:
+	case RTE_ETH_FILTER_STATS:
+	case RTE_ETH_FILTER_OP_MAX:
+		DP_ERR(edev, "Unsupported filter_op %d\n", filter_op);
+		return -ENOTSUP;
+	}
+	ntuple = (struct rte_eth_ntuple_filter *)arg;
+	/* Internally convert ntuple to fdir entry */
+	memset(&fdir_entry, 0, sizeof(fdir_entry));
+	if (ntuple->proto == IPPROTO_TCP) {
+		fdir_entry.input.flow_type = RTE_ETH_FLOW_NONFRAG_IPV4_TCP;
+		tcpv4_flow = &fdir_entry.input.flow.tcp4_flow;
+		tcpv4_flow->ip.src_ip = ntuple->src_ip;
+		tcpv4_flow->ip.dst_ip = ntuple->dst_ip;
+		tcpv4_flow->ip.proto = IPPROTO_TCP;
+		tcpv4_flow->src_port = ntuple->src_port;
+		tcpv4_flow->dst_port = ntuple->dst_port;
+	} else {
+		fdir_entry.input.flow_type = RTE_ETH_FLOW_NONFRAG_IPV4_UDP;
+		udpv4_flow = &fdir_entry.input.flow.udp4_flow;
+		udpv4_flow->ip.src_ip = ntuple->src_ip;
+		udpv4_flow->ip.dst_ip = ntuple->dst_ip;
+		udpv4_flow->ip.proto = IPPROTO_TCP;
+		udpv4_flow->src_port = ntuple->src_port;
+		udpv4_flow->dst_port = ntuple->dst_port;
+	}
+
+	fdir_entry.action.rx_queue = ntuple->queue;
+
+	return qede_config_cmn_fdir_filter(eth_dev, &fdir_entry, add);
+}
+
+static int
+qede_tunnel_update(struct qede_dev *qdev,
+		   struct ecore_tunnel_info *tunn_info)
+{
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	enum _ecore_status_t rc = ECORE_INVAL;
+	struct ecore_hwfn *p_hwfn;
+	struct ecore_ptt *p_ptt;
+	int i;
+
+	for_each_hwfn(edev, i) {
+		p_hwfn = &edev->hwfns[i];
+		if (IS_PF(edev)) {
+			p_ptt = ecore_ptt_acquire(p_hwfn);
+			if (!p_ptt) {
+				DP_ERR(p_hwfn, "Can't acquire PTT\n");
+				return -EAGAIN;
+			}
+		} else {
+			p_ptt = NULL;
+		}
+
+		rc = ecore_sp_pf_update_tunn_cfg(p_hwfn, p_ptt,
+				tunn_info, ECORE_SPQ_MODE_CB, NULL);
+		if (IS_PF(edev))
+			ecore_ptt_release(p_hwfn, p_ptt);
+
+		if (rc != ECORE_SUCCESS)
+			break;
+	}
+
+	return rc;
+}
+
+static int
+qede_vxlan_enable(struct rte_eth_dev *eth_dev, uint8_t clss,
+		  bool enable)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	enum _ecore_status_t rc = ECORE_INVAL;
+	struct ecore_tunnel_info tunn;
+
+	if (qdev->vxlan.enable == enable)
+		return ECORE_SUCCESS;
+
+	memset(&tunn, 0, sizeof(struct ecore_tunnel_info));
+	tunn.vxlan.b_update_mode = true;
+	tunn.vxlan.b_mode_enabled = enable;
+	tunn.b_update_rx_cls = true;
+	tunn.b_update_tx_cls = true;
+	tunn.vxlan.tun_cls = clss;
+
+	tunn.vxlan_port.b_update_port = true;
+	tunn.vxlan_port.port = enable ? QEDE_VXLAN_DEF_PORT : 0;
+
+	rc = qede_tunnel_update(qdev, &tunn);
+	if (rc == ECORE_SUCCESS) {
+		qdev->vxlan.enable = enable;
+		qdev->vxlan.udp_port = (enable) ? QEDE_VXLAN_DEF_PORT : 0;
+		DP_INFO(edev, "vxlan is %s, UDP port = %d\n",
+			enable ? "enabled" : "disabled", qdev->vxlan.udp_port);
+	} else {
+		DP_ERR(edev, "Failed to update tunn_clss %u\n",
+		       tunn.vxlan.tun_cls);
+	}
+
+	return rc;
+}
+
+static int
+qede_geneve_enable(struct rte_eth_dev *eth_dev, uint8_t clss,
+		  bool enable)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	enum _ecore_status_t rc = ECORE_INVAL;
+	struct ecore_tunnel_info tunn;
+
+	memset(&tunn, 0, sizeof(struct ecore_tunnel_info));
+	tunn.l2_geneve.b_update_mode = true;
+	tunn.l2_geneve.b_mode_enabled = enable;
+	tunn.ip_geneve.b_update_mode = true;
+	tunn.ip_geneve.b_mode_enabled = enable;
+	tunn.l2_geneve.tun_cls = clss;
+	tunn.ip_geneve.tun_cls = clss;
+	tunn.b_update_rx_cls = true;
+	tunn.b_update_tx_cls = true;
+
+	tunn.geneve_port.b_update_port = true;
+	tunn.geneve_port.port = enable ? QEDE_GENEVE_DEF_PORT : 0;
+
+	rc = qede_tunnel_update(qdev, &tunn);
+	if (rc == ECORE_SUCCESS) {
+		qdev->geneve.enable = enable;
+		qdev->geneve.udp_port = (enable) ? QEDE_GENEVE_DEF_PORT : 0;
+		DP_INFO(edev, "GENEVE is %s, UDP port = %d\n",
+			enable ? "enabled" : "disabled", qdev->geneve.udp_port);
+	} else {
+		DP_ERR(edev, "Failed to update tunn_clss %u\n",
+		       clss);
+	}
+
+	return rc;
+}
+
+static int
+qede_ipgre_enable(struct rte_eth_dev *eth_dev, uint8_t clss,
+		  bool enable)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	enum _ecore_status_t rc = ECORE_INVAL;
+	struct ecore_tunnel_info tunn;
+
+	memset(&tunn, 0, sizeof(struct ecore_tunnel_info));
+	tunn.ip_gre.b_update_mode = true;
+	tunn.ip_gre.b_mode_enabled = enable;
+	tunn.ip_gre.tun_cls = clss;
+	tunn.ip_gre.tun_cls = clss;
+	tunn.b_update_rx_cls = true;
+	tunn.b_update_tx_cls = true;
+
+	rc = qede_tunnel_update(qdev, &tunn);
+	if (rc == ECORE_SUCCESS) {
+		qdev->ipgre.enable = enable;
+		DP_INFO(edev, "IPGRE is %s\n",
+			enable ? "enabled" : "disabled");
+	} else {
+		DP_ERR(edev, "Failed to update tunn_clss %u\n",
+		       clss);
+	}
+
+	return rc;
+}
+
+int
+qede_udp_dst_port_del(struct rte_eth_dev *eth_dev,
+		      struct rte_eth_udp_tunnel *tunnel_udp)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	struct ecore_tunnel_info tunn; /* @DPDK */
+	uint16_t udp_port;
+	int rc;
+
+	PMD_INIT_FUNC_TRACE(edev);
+
+	memset(&tunn, 0, sizeof(tunn));
+
+	switch (tunnel_udp->prot_type) {
+	case RTE_TUNNEL_TYPE_VXLAN:
+		if (qdev->vxlan.udp_port != tunnel_udp->udp_port) {
+			DP_ERR(edev, "UDP port %u doesn't exist\n",
+				tunnel_udp->udp_port);
+			return ECORE_INVAL;
+		}
+		udp_port = 0;
+
+		tunn.vxlan_port.b_update_port = true;
+		tunn.vxlan_port.port = udp_port;
+
+		rc = qede_tunnel_update(qdev, &tunn);
+		if (rc != ECORE_SUCCESS) {
+			DP_ERR(edev, "Unable to config UDP port %u\n",
+			       tunn.vxlan_port.port);
+			return rc;
+		}
+
+		qdev->vxlan.udp_port = udp_port;
+		/* If the request is to delete UDP port and if the number of
+		 * VXLAN filters have reached 0 then VxLAN offload can be be
+		 * disabled.
+		 */
+		if (qdev->vxlan.enable && qdev->vxlan.num_filters == 0)
+			return qede_vxlan_enable(eth_dev,
+					ECORE_TUNN_CLSS_MAC_VLAN, false);
+
+		break;
+	case RTE_TUNNEL_TYPE_GENEVE:
+		if (qdev->geneve.udp_port != tunnel_udp->udp_port) {
+			DP_ERR(edev, "UDP port %u doesn't exist\n",
+				tunnel_udp->udp_port);
+			return ECORE_INVAL;
+		}
+
+		udp_port = 0;
+
+		tunn.geneve_port.b_update_port = true;
+		tunn.geneve_port.port = udp_port;
+
+		rc = qede_tunnel_update(qdev, &tunn);
+		if (rc != ECORE_SUCCESS) {
+			DP_ERR(edev, "Unable to config UDP port %u\n",
+			       tunn.vxlan_port.port);
+			return rc;
+		}
+
+		qdev->vxlan.udp_port = udp_port;
+		/* If the request is to delete UDP port and if the number of
+		 * GENEVE filters have reached 0 then GENEVE offload can be be
+		 * disabled.
+		 */
+		if (qdev->geneve.enable && qdev->geneve.num_filters == 0)
+			return qede_geneve_enable(eth_dev,
+					ECORE_TUNN_CLSS_MAC_VLAN, false);
+
+		break;
+
+	default:
+		return ECORE_INVAL;
+	}
+
+	return 0;
+}
+
+int
+qede_udp_dst_port_add(struct rte_eth_dev *eth_dev,
+		      struct rte_eth_udp_tunnel *tunnel_udp)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	struct ecore_tunnel_info tunn; /* @DPDK */
+	uint16_t udp_port;
+	int rc;
+
+	PMD_INIT_FUNC_TRACE(edev);
+
+	memset(&tunn, 0, sizeof(tunn));
+
+	switch (tunnel_udp->prot_type) {
+	case RTE_TUNNEL_TYPE_VXLAN:
+		if (qdev->vxlan.udp_port == tunnel_udp->udp_port) {
+			DP_INFO(edev,
+				"UDP port %u for VXLAN was already configured\n",
+				tunnel_udp->udp_port);
+			return ECORE_SUCCESS;
+		}
+
+		/* Enable VxLAN tunnel with default MAC/VLAN classification if
+		 * it was not enabled while adding VXLAN filter before UDP port
+		 * update.
+		 */
+		if (!qdev->vxlan.enable) {
+			rc = qede_vxlan_enable(eth_dev,
+				ECORE_TUNN_CLSS_MAC_VLAN, true);
+			if (rc != ECORE_SUCCESS) {
+				DP_ERR(edev, "Failed to enable VXLAN "
+					"prior to updating UDP port\n");
+				return rc;
+			}
+		}
+		udp_port = tunnel_udp->udp_port;
+
+		tunn.vxlan_port.b_update_port = true;
+		tunn.vxlan_port.port = udp_port;
+
+		rc = qede_tunnel_update(qdev, &tunn);
+		if (rc != ECORE_SUCCESS) {
+			DP_ERR(edev, "Unable to config UDP port %u for VXLAN\n",
+			       udp_port);
+			return rc;
+		}
+
+		DP_INFO(edev, "Updated UDP port %u for VXLAN\n", udp_port);
+
+		qdev->vxlan.udp_port = udp_port;
+		break;
+	case RTE_TUNNEL_TYPE_GENEVE:
+		if (qdev->geneve.udp_port == tunnel_udp->udp_port) {
+			DP_INFO(edev,
+				"UDP port %u for GENEVE was already configured\n",
+				tunnel_udp->udp_port);
+			return ECORE_SUCCESS;
+		}
+
+		/* Enable GENEVE tunnel with default MAC/VLAN classification if
+		 * it was not enabled while adding GENEVE filter before UDP port
+		 * update.
+		 */
+		if (!qdev->geneve.enable) {
+			rc = qede_geneve_enable(eth_dev,
+				ECORE_TUNN_CLSS_MAC_VLAN, true);
+			if (rc != ECORE_SUCCESS) {
+				DP_ERR(edev, "Failed to enable GENEVE "
+					"prior to updating UDP port\n");
+				return rc;
+			}
+		}
+		udp_port = tunnel_udp->udp_port;
+
+		tunn.geneve_port.b_update_port = true;
+		tunn.geneve_port.port = udp_port;
+
+		rc = qede_tunnel_update(qdev, &tunn);
+		if (rc != ECORE_SUCCESS) {
+			DP_ERR(edev, "Unable to config UDP port %u for GENEVE\n",
+			       udp_port);
+			return rc;
+		}
+
+		DP_INFO(edev, "Updated UDP port %u for GENEVE\n", udp_port);
+
+		qdev->geneve.udp_port = udp_port;
+		break;
+	default:
+		return ECORE_INVAL;
+	}
+
+	return 0;
+}
+
+static void qede_get_ecore_tunn_params(uint32_t filter, uint32_t *type,
+				       uint32_t *clss, char *str)
+{
+	uint16_t j;
+	*clss = MAX_ECORE_TUNN_CLSS;
+
+	for (j = 0; j < RTE_DIM(qede_tunn_types); j++) {
+		if (filter == qede_tunn_types[j].rte_filter_type) {
+			*type = qede_tunn_types[j].qede_type;
+			*clss = qede_tunn_types[j].qede_tunn_clss;
+			strcpy(str, qede_tunn_types[j].string);
+			return;
+		}
+	}
+}
+
+static int
+qede_set_ucast_tunn_cmn_param(struct ecore_filter_ucast *ucast,
+			      const struct rte_eth_tunnel_filter_conf *conf,
+			      uint32_t type)
+{
+	/* Init commmon ucast params first */
+	qede_set_ucast_cmn_params(ucast);
+
+	/* Copy out the required fields based on classification type */
+	ucast->type = type;
+
+	switch (type) {
+	case ECORE_FILTER_VNI:
+		ucast->vni = conf->tenant_id;
+	break;
+	case ECORE_FILTER_INNER_VLAN:
+		ucast->vlan = conf->inner_vlan;
+	break;
+	case ECORE_FILTER_MAC:
+		memcpy(ucast->mac, conf->outer_mac.addr_bytes,
+		       ETHER_ADDR_LEN);
+	break;
+	case ECORE_FILTER_INNER_MAC:
+		memcpy(ucast->mac, conf->inner_mac.addr_bytes,
+		       ETHER_ADDR_LEN);
+	break;
+	case ECORE_FILTER_MAC_VNI_PAIR:
+		memcpy(ucast->mac, conf->outer_mac.addr_bytes,
+			ETHER_ADDR_LEN);
+		ucast->vni = conf->tenant_id;
+	break;
+	case ECORE_FILTER_INNER_MAC_VNI_PAIR:
+		memcpy(ucast->mac, conf->inner_mac.addr_bytes,
+			ETHER_ADDR_LEN);
+		ucast->vni = conf->tenant_id;
+	break;
+	case ECORE_FILTER_INNER_PAIR:
+		memcpy(ucast->mac, conf->inner_mac.addr_bytes,
+			ETHER_ADDR_LEN);
+		ucast->vlan = conf->inner_vlan;
+	break;
+	default:
+		return -EINVAL;
+	}
+
+	return ECORE_SUCCESS;
+}
+
+static int
+_qede_tunn_filter_config(struct rte_eth_dev *eth_dev,
+			 const struct rte_eth_tunnel_filter_conf *conf,
+			 __attribute__((unused)) enum rte_filter_op filter_op,
+			 enum ecore_tunn_clss *clss,
+			 bool add)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	struct ecore_filter_ucast ucast = {0};
+	enum ecore_filter_ucast_type type;
+	uint16_t filter_type = 0;
+	char str[80];
+	int rc;
+
+	filter_type = conf->filter_type;
+	/* Determine if the given filter classification is supported */
+	qede_get_ecore_tunn_params(filter_type, &type, clss, str);
+	if (*clss == MAX_ECORE_TUNN_CLSS) {
+		DP_ERR(edev, "Unsupported filter type\n");
+		return -EINVAL;
+	}
+	/* Init tunnel ucast params */
+	rc = qede_set_ucast_tunn_cmn_param(&ucast, conf, type);
+	if (rc != ECORE_SUCCESS) {
+		DP_ERR(edev, "Unsupported Tunnel filter type 0x%x\n",
+		conf->filter_type);
+		return rc;
+	}
+	DP_INFO(edev, "Rule: \"%s\", op %d, type 0x%x\n",
+		str, filter_op, ucast.type);
+
+	ucast.opcode = add ? ECORE_FILTER_ADD : ECORE_FILTER_REMOVE;
+
+	/* Skip MAC/VLAN if filter is based on VNI */
+	if (!(filter_type & ETH_TUNNEL_FILTER_TENID)) {
+		rc = qede_mac_int_ops(eth_dev, &ucast, add);
+		if (rc == 0 && add) {
+			/* Enable accept anyvlan */
+			qede_config_accept_any_vlan(qdev, true);
+		}
+	} else {
+		rc = qede_ucast_filter(eth_dev, &ucast, add);
+		if (rc == 0)
+			rc = ecore_filter_ucast_cmd(edev, &ucast,
+					    ECORE_SPQ_MODE_CB, NULL);
+	}
+
+	return rc;
+}
+
+static int
+qede_tunn_enable(struct rte_eth_dev *eth_dev, uint8_t clss,
+		 enum rte_eth_tunnel_type tunn_type, bool enable)
+{
+	int rc = -EINVAL;
+
+	switch (tunn_type) {
+	case RTE_TUNNEL_TYPE_VXLAN:
+		rc = qede_vxlan_enable(eth_dev, clss, enable);
+		break;
+	case RTE_TUNNEL_TYPE_GENEVE:
+		rc = qede_geneve_enable(eth_dev, clss, enable);
+		break;
+	case RTE_TUNNEL_TYPE_IP_IN_GRE:
+		rc = qede_ipgre_enable(eth_dev, clss, enable);
+		break;
+	default:
+		rc = -EINVAL;
+		break;
+	}
+
+	return rc;
+}
+
+static int
+qede_tunn_filter_config(struct rte_eth_dev *eth_dev,
+			enum rte_filter_op filter_op,
+			const struct rte_eth_tunnel_filter_conf *conf)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	enum ecore_tunn_clss clss = MAX_ECORE_TUNN_CLSS;
+	bool add;
+	int rc;
+
+	PMD_INIT_FUNC_TRACE(edev);
+
+	switch (filter_op) {
+	case RTE_ETH_FILTER_ADD:
+		add = true;
+		break;
+	case RTE_ETH_FILTER_DELETE:
+		add = false;
+		break;
+	default:
+		DP_ERR(edev, "Unsupported operation %d\n", filter_op);
+		return -EINVAL;
+	}
+
+	if (IS_VF(edev))
+		return qede_tunn_enable(eth_dev,
+					ECORE_TUNN_CLSS_MAC_VLAN,
+					conf->tunnel_type, add);
+
+	rc = _qede_tunn_filter_config(eth_dev, conf, filter_op, &clss, add);
+	if (rc != ECORE_SUCCESS)
+		return rc;
+
+	if (add) {
+		if (conf->tunnel_type == RTE_TUNNEL_TYPE_VXLAN) {
+			qdev->vxlan.num_filters++;
+			qdev->vxlan.filter_type = conf->filter_type;
+		} else { /* GENEVE */
+			qdev->geneve.num_filters++;
+			qdev->geneve.filter_type = conf->filter_type;
+		}
+
+		if (!qdev->vxlan.enable || !qdev->geneve.enable ||
+		    !qdev->ipgre.enable)
+			return qede_tunn_enable(eth_dev, clss,
+						conf->tunnel_type,
+						true);
+	} else {
+		if (conf->tunnel_type == RTE_TUNNEL_TYPE_VXLAN)
+			qdev->vxlan.num_filters--;
+		else /*GENEVE*/
+			qdev->geneve.num_filters--;
+
+		/* Disable VXLAN if VXLAN filters become 0 */
+		if (qdev->vxlan.num_filters == 0 ||
+		    qdev->geneve.num_filters == 0)
+			return qede_tunn_enable(eth_dev, clss,
+						conf->tunnel_type,
+						false);
+	}
+
+	return 0;
+}
+
+static int
+qede_flow_validate_attr(__attribute__((unused))struct rte_eth_dev *dev,
+			const struct rte_flow_attr *attr,
+			struct rte_flow_error *error)
+{
+	if (attr == NULL) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ATTR, NULL,
+				   "NULL attribute");
+		return -rte_errno;
+	}
+
+	if (attr->group != 0) {
+		rte_flow_error_set(error, ENOTSUP,
+				   RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr,
+				   "Groups are not supported");
+		return -rte_errno;
+	}
+
+	if (attr->priority != 0) {
+		rte_flow_error_set(error, ENOTSUP,
+				   RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, attr,
+				   "Priorities are not supported");
+		return -rte_errno;
+	}
+
+	if (attr->egress != 0) {
+		rte_flow_error_set(error, ENOTSUP,
+				   RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attr,
+				   "Egress is not supported");
+		return -rte_errno;
+	}
+
+	if (attr->transfer != 0) {
+		rte_flow_error_set(error, ENOTSUP,
+				   RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, attr,
+				   "Transfer is not supported");
+		return -rte_errno;
+	}
+
+	if (attr->ingress == 0) {
+		rte_flow_error_set(error, ENOTSUP,
+				   RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, attr,
+				   "Only ingress is supported");
+		return -rte_errno;
+	}
+
+	return 0;
+}
+
+static int
+qede_flow_parse_pattern(__attribute__((unused))struct rte_eth_dev *dev,
+			const struct rte_flow_item pattern[],
+			struct rte_flow_error *error,
+			struct rte_flow *flow)
+{
+	bool l3 = false, l4 = false;
+
+	if (pattern == NULL) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL,
+				   "NULL pattern");
+		return -rte_errno;
+	}
+
+	for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
+		if (!pattern->spec) {
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   pattern,
+					   "Item spec not defined");
+			return -rte_errno;
+		}
+
+		if (pattern->last) {
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   pattern,
+					   "Item last not supported");
+			return -rte_errno;
+		}
+
+		if (pattern->mask) {
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   pattern,
+					   "Item mask not supported");
+			return -rte_errno;
+		}
+
+		/* Below validation is only for 4 tuple flow
+		 * (GFT_PROFILE_TYPE_4_TUPLE)
+		 * - src and dst L3 address (IPv4 or IPv6)
+		 * - src and dst L4 port (TCP or UDP)
+		 */
+
+		switch (pattern->type) {
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			l3 = true;
+
+			if (flow) {
+				const struct rte_flow_item_ipv4 *spec;
+
+				spec = pattern->spec;
+				flow->entry.tuple.src_ipv4 = spec->hdr.src_addr;
+				flow->entry.tuple.dst_ipv4 = spec->hdr.dst_addr;
+				flow->entry.tuple.eth_proto = ETHER_TYPE_IPv4;
+			}
+			break;
+
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			l3 = true;
+
+			if (flow) {
+				const struct rte_flow_item_ipv6 *spec;
+
+				spec = pattern->spec;
+				rte_memcpy(flow->entry.tuple.src_ipv6,
+					   spec->hdr.src_addr,
+					   IPV6_ADDR_LEN);
+				rte_memcpy(flow->entry.tuple.dst_ipv6,
+					   spec->hdr.dst_addr,
+					   IPV6_ADDR_LEN);
+				flow->entry.tuple.eth_proto = ETHER_TYPE_IPv6;
+			}
+			break;
+
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			l4 = true;
+
+			if (flow) {
+				const struct rte_flow_item_udp *spec;
+
+				spec = pattern->spec;
+				flow->entry.tuple.src_port =
+						spec->hdr.src_port;
+				flow->entry.tuple.dst_port =
+						spec->hdr.dst_port;
+				flow->entry.tuple.ip_proto = IPPROTO_UDP;
+			}
+			break;
+
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			l4 = true;
+
+			if (flow) {
+				const struct rte_flow_item_tcp *spec;
+
+				spec = pattern->spec;
+				flow->entry.tuple.src_port =
+						spec->hdr.src_port;
+				flow->entry.tuple.dst_port =
+						spec->hdr.dst_port;
+				flow->entry.tuple.ip_proto = IPPROTO_TCP;
+			}
+
+			break;
+		default:
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   pattern,
+					   "Only 4 tuple (IPV4, IPV6, UDP and TCP) item types supported");
+			return -rte_errno;
+		}
+	}
+
+	if (!(l3 && l4)) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ITEM,
+				   pattern,
+				   "Item types need to have both L3 and L4 protocols");
+		return -rte_errno;
+	}
+
+	return 0;
+}
+
+static int
+qede_flow_parse_actions(struct rte_eth_dev *dev,
+			const struct rte_flow_action actions[],
+			struct rte_flow_error *error,
+			struct rte_flow *flow)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(dev);
+	const struct rte_flow_action_queue *queue;
+
+	if (actions == NULL) {
+		rte_flow_error_set(error, EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL,
+				   "NULL actions");
+		return -rte_errno;
+	}
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			queue = actions->conf;
+
+			if (queue->index >= QEDE_RSS_COUNT(qdev)) {
+				rte_flow_error_set(error, EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ACTION,
+						   actions,
+						   "Bad QUEUE action");
+				return -rte_errno;
+			}
+
+			if (flow)
+				flow->entry.rx_queue = queue->index;
+
+			break;
+
+		default:
+			rte_flow_error_set(error, ENOTSUP,
+					   RTE_FLOW_ERROR_TYPE_ACTION,
+					   actions,
+					   "Action is not supported - only ACTION_TYPE_QUEUE supported");
+			return -rte_errno;
+		}
+	}
+
+	return 0;
+}
+
+static int
+qede_flow_parse(struct rte_eth_dev *dev,
+		const struct rte_flow_attr *attr,
+		const struct rte_flow_item patterns[],
+		const struct rte_flow_action actions[],
+		struct rte_flow_error *error,
+		struct rte_flow *flow)
+
+{
+	int rc = 0;
+
+	rc = qede_flow_validate_attr(dev, attr, error);
+	if (rc)
+		return rc;
+
+	/* parse and validate item pattern and actions.
+	 * Given item list and actions will be translate to qede PMD
+	 * specific arfs structure.
+	 */
+	rc = qede_flow_parse_pattern(dev, patterns, error, flow);
+	if (rc)
+		return rc;
+
+	rc = qede_flow_parse_actions(dev, actions, error, flow);
+
+	return rc;
+}
+
+static int
+qede_flow_validate(struct rte_eth_dev *dev,
+		   const struct rte_flow_attr *attr,
+		   const struct rte_flow_item patterns[],
+		   const struct rte_flow_action actions[],
+		   struct rte_flow_error *error)
+{
+	return qede_flow_parse(dev, attr, patterns, actions, error, NULL);
+}
+
+static struct rte_flow *
+qede_flow_create(struct rte_eth_dev *dev,
+		 const struct rte_flow_attr *attr,
+		 const struct rte_flow_item pattern[],
+		 const struct rte_flow_action actions[],
+		 struct rte_flow_error *error)
+{
+	struct rte_flow *flow = NULL;
+	int rc;
+
+	flow = rte_zmalloc("qede_rte_flow", sizeof(*flow), 0);
+	if (flow == NULL) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "Failed to allocate memory");
+		return NULL;
+	}
+
+	rc = qede_flow_parse(dev, attr, pattern, actions, error, flow);
+	if (rc < 0) {
+		rte_free(flow);
+		return NULL;
+	}
+
+	rc = qede_config_arfs_filter(dev, &flow->entry, true);
+	if (rc < 0) {
+		rte_flow_error_set(error, rc,
+				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+				   "Failed to configure flow filter");
+		rte_free(flow);
+		return NULL;
+	}
+
+	return flow;
+}
+
+static int
+qede_flow_destroy(struct rte_eth_dev *eth_dev,
+		  struct rte_flow *flow,
+		  struct rte_flow_error *error)
+{
+	int rc = 0;
+
+	rc = qede_config_arfs_filter(eth_dev, &flow->entry, false);
+	if (rc < 0) {
+		rte_flow_error_set(error, rc,
+				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+				   "Failed to delete flow filter");
+		rte_free(flow);
+	}
+
+	return rc;
+}
+
+const struct rte_flow_ops qede_flow_ops = {
+	.validate = qede_flow_validate,
+	.create = qede_flow_create,
+	.destroy = qede_flow_destroy,
+};
+
+int qede_dev_filter_ctrl(struct rte_eth_dev *eth_dev,
+			 enum rte_filter_type filter_type,
+			 enum rte_filter_op filter_op,
+			 void *arg)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	struct rte_eth_tunnel_filter_conf *filter_conf =
+			(struct rte_eth_tunnel_filter_conf *)arg;
+
+	switch (filter_type) {
+	case RTE_ETH_FILTER_TUNNEL:
+		switch (filter_conf->tunnel_type) {
+		case RTE_TUNNEL_TYPE_VXLAN:
+		case RTE_TUNNEL_TYPE_GENEVE:
+		case RTE_TUNNEL_TYPE_IP_IN_GRE:
+			DP_INFO(edev,
+				"Packet steering to the specified Rx queue"
+				" is not supported with UDP tunneling");
+			return(qede_tunn_filter_config(eth_dev, filter_op,
+						      filter_conf));
+		case RTE_TUNNEL_TYPE_TEREDO:
+		case RTE_TUNNEL_TYPE_NVGRE:
+		case RTE_L2_TUNNEL_TYPE_E_TAG:
+			DP_ERR(edev, "Unsupported tunnel type %d\n",
+				filter_conf->tunnel_type);
+			return -EINVAL;
+		case RTE_TUNNEL_TYPE_NONE:
+		default:
+			return 0;
+		}
+		break;
+	case RTE_ETH_FILTER_FDIR:
+		return qede_fdir_filter_conf(eth_dev, filter_op, arg);
+	case RTE_ETH_FILTER_NTUPLE:
+		return qede_ntuple_filter_conf(eth_dev, filter_op, arg);
+	case RTE_ETH_FILTER_GENERIC:
+		if (ECORE_IS_CMT(edev)) {
+			DP_ERR(edev, "flowdir is not supported in 100G mode\n");
+			return -ENOTSUP;
+		}
+
+		if (filter_op != RTE_ETH_FILTER_GET)
+			return -EINVAL;
+
+		*(const void **)arg = &qede_flow_ops;
+		return 0;
+	case RTE_ETH_FILTER_MACVLAN:
+	case RTE_ETH_FILTER_ETHERTYPE:
+	case RTE_ETH_FILTER_FLEXIBLE:
+	case RTE_ETH_FILTER_SYN:
+	case RTE_ETH_FILTER_HASH:
+	case RTE_ETH_FILTER_L2_TUNNEL:
+	case RTE_ETH_FILTER_MAX:
+	default:
+		DP_ERR(edev, "Unsupported filter type %d\n",
+			filter_type);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
diff --git a/drivers/net/qede/qede_main.c b/drivers/net/qede/qede_main.c
index 46fa8371..df83666f 100644
--- a/drivers/net/qede/qede_main.c
+++ b/drivers/net/qede/qede_main.c
@@ -15,10 +15,10 @@
 #define QEDE_ALARM_TIMEOUT_US 100000
 
 /* Global variable to hold absolute path of fw file */
-char fw_file[PATH_MAX];
+char qede_fw_file[PATH_MAX];
 
-const char *QEDE_DEFAULT_FIRMWARE =
-	"/lib/firmware/qed/qed_init_values-8.33.12.0.bin";
+static const char * const QEDE_DEFAULT_FIRMWARE =
+	"/lib/firmware/qed/qed_init_values-8.37.7.0.bin";
 
 static void
 qed_update_pf_params(struct ecore_dev *edev, struct ecore_pf_params *params)
@@ -126,11 +126,11 @@ static int qed_load_firmware_data(struct ecore_dev *edev)
 	const char *fw = RTE_LIBRTE_QEDE_FW;
 
 	if (strcmp(fw, "") == 0)
-		strcpy(fw_file, QEDE_DEFAULT_FIRMWARE);
+		strcpy(qede_fw_file, QEDE_DEFAULT_FIRMWARE);
 	else
-		strcpy(fw_file, fw);
+		strcpy(qede_fw_file, fw);
 
-	fd = open(fw_file, O_RDONLY);
+	fd = open(qede_fw_file, O_RDONLY);
 	if (fd < 0) {
 		DP_ERR(edev, "Can't open firmware file\n");
 		return -ENOENT;
@@ -234,7 +234,8 @@ static int qed_slowpath_start(struct ecore_dev *edev,
 #ifdef CONFIG_ECORE_BINARY_FW
 		rc = qed_load_firmware_data(edev);
 		if (rc) {
-			DP_ERR(edev, "Failed to find fw file %s\n", fw_file);
+			DP_ERR(edev, "Failed to find fw file %s\n",
+				qede_fw_file);
 			goto err;
 		}
 #endif
diff --git a/drivers/net/qede/qede_rxtx.c b/drivers/net/qede/qede_rxtx.c
index 0f157ded..8a4772f4 100644
--- a/drivers/net/qede/qede_rxtx.c
+++ b/drivers/net/qede/qede_rxtx.c
@@ -35,6 +35,49 @@ static inline int qede_alloc_rx_buffer(struct qede_rx_queue *rxq)
 	return 0;
 }
 
+/* Criterias for calculating Rx buffer size -
+ * 1) rx_buf_size should not exceed the size of mbuf
+ * 2) In scattered_rx mode - minimum rx_buf_size should be
+ *    (MTU + Maximum L2 Header Size + 2) / ETH_RX_MAX_BUFF_PER_PKT
+ * 3) In regular mode - minimum rx_buf_size should be
+ *    (MTU + Maximum L2 Header Size + 2)
+ *    In above cases +2 corrosponds to 2 bytes padding in front of L2
+ *    header.
+ * 4) rx_buf_size should be cacheline-size aligned. So considering
+ *    criteria 1, we need to adjust the size to floor instead of ceil,
+ *    so that we don't exceed mbuf size while ceiling rx_buf_size.
+ */
+int
+qede_calc_rx_buf_size(struct rte_eth_dev *dev, uint16_t mbufsz,
+		      uint16_t max_frame_size)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	int rx_buf_size;
+
+	if (dev->data->scattered_rx) {
+		/* per HW limitation, only ETH_RX_MAX_BUFF_PER_PKT number of
+		 * bufferes can be used for single packet. So need to make sure
+		 * mbuf size is sufficient enough for this.
+		 */
+		if ((mbufsz * ETH_RX_MAX_BUFF_PER_PKT) <
+		     (max_frame_size + QEDE_ETH_OVERHEAD)) {
+			DP_ERR(edev, "mbuf %d size is not enough to hold max fragments (%d) for max rx packet length (%d)\n",
+			       mbufsz, ETH_RX_MAX_BUFF_PER_PKT, max_frame_size);
+			return -EINVAL;
+		}
+
+		rx_buf_size = RTE_MAX(mbufsz,
+				      (max_frame_size + QEDE_ETH_OVERHEAD) /
+				       ETH_RX_MAX_BUFF_PER_PKT);
+	} else {
+		rx_buf_size = max_frame_size + QEDE_ETH_OVERHEAD;
+	}
+
+	/* Align to cache-line size if needed */
+	return QEDE_FLOOR_TO_CACHE_LINE_SIZE(rx_buf_size);
+}
+
 int
 qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 		    uint16_t nb_desc, unsigned int socket_id,
@@ -85,6 +128,8 @@ qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 
 	/* Fix up RX buffer size */
 	bufsz = (uint16_t)rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM;
+	/* cache align the mbuf size to simplfy rx_buf_size calculation */
+	bufsz = QEDE_FLOOR_TO_CACHE_LINE_SIZE(bufsz);
 	if ((rxmode->offloads & DEV_RX_OFFLOAD_SCATTER)	||
 	    (max_rx_pkt_len + QEDE_ETH_OVERHEAD) > bufsz) {
 		if (!dev->data->scattered_rx) {
@@ -93,13 +138,13 @@ qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 		}
 	}
 
-	if (dev->data->scattered_rx)
-		rxq->rx_buf_size = bufsz + ETHER_HDR_LEN +
-				   ETHER_CRC_LEN + QEDE_ETH_OVERHEAD;
-	else
-		rxq->rx_buf_size = max_rx_pkt_len + QEDE_ETH_OVERHEAD;
-	/* Align to cache-line size if needed */
-	rxq->rx_buf_size = QEDE_CEIL_TO_CACHE_LINE_SIZE(rxq->rx_buf_size);
+	rc = qede_calc_rx_buf_size(dev, bufsz, max_rx_pkt_len);
+	if (rc < 0) {
+		rte_free(rxq);
+		return rc;
+	}
+
+	rxq->rx_buf_size = rc;
 
 	DP_INFO(edev, "mtu %u mbufsz %u bd_max_bytes %u scatter_mode %d\n",
 		qdev->mtu, bufsz, rxq->rx_buf_size, dev->data->scattered_rx);
@@ -2106,3 +2151,84 @@ qede_rxtx_pkts_dummy(__rte_unused void *p_rxq,
 {
 	return 0;
 }
+
+
+/* this function does a fake walk through over completion queue
+ * to calculate number of BDs used by HW.
+ * At the end, it restores the state of completion queue.
+ */
+static uint16_t
+qede_parse_fp_cqe(struct qede_rx_queue *rxq)
+{
+	uint16_t hw_comp_cons, sw_comp_cons, bd_count = 0;
+	union eth_rx_cqe *cqe, *orig_cqe = NULL;
+
+	hw_comp_cons = rte_le_to_cpu_16(*rxq->hw_cons_ptr);
+	sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
+
+	if (hw_comp_cons == sw_comp_cons)
+		return 0;
+
+	/* Get the CQE from the completion ring */
+	cqe = (union eth_rx_cqe *)ecore_chain_consume(&rxq->rx_comp_ring);
+	orig_cqe = cqe;
+
+	while (sw_comp_cons != hw_comp_cons) {
+		switch (cqe->fast_path_regular.type) {
+		case ETH_RX_CQE_TYPE_REGULAR:
+			bd_count += cqe->fast_path_regular.bd_num;
+			break;
+		case ETH_RX_CQE_TYPE_TPA_END:
+			bd_count += cqe->fast_path_tpa_end.num_of_bds;
+			break;
+		default:
+			break;
+		}
+
+		cqe =
+		(union eth_rx_cqe *)ecore_chain_consume(&rxq->rx_comp_ring);
+		sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
+	}
+
+	/* revert comp_ring to original state */
+	ecore_chain_set_cons(&rxq->rx_comp_ring, sw_comp_cons, orig_cqe);
+
+	return bd_count;
+}
+
+int
+qede_rx_descriptor_status(void *p_rxq, uint16_t offset)
+{
+	uint16_t hw_bd_cons, sw_bd_cons, sw_bd_prod;
+	uint16_t produced, consumed;
+	struct qede_rx_queue *rxq = p_rxq;
+
+	if (offset > rxq->nb_rx_desc)
+		return -EINVAL;
+
+	sw_bd_cons = ecore_chain_get_cons_idx(&rxq->rx_bd_ring);
+	sw_bd_prod = ecore_chain_get_prod_idx(&rxq->rx_bd_ring);
+
+	/* find BDs used by HW from completion queue elements */
+	hw_bd_cons = sw_bd_cons + qede_parse_fp_cqe(rxq);
+
+	if (hw_bd_cons < sw_bd_cons)
+		/* wraparound case */
+		consumed = (0xffff - sw_bd_cons) + hw_bd_cons;
+	else
+		consumed = hw_bd_cons - sw_bd_cons;
+
+	if (offset <= consumed)
+		return RTE_ETH_RX_DESC_DONE;
+
+	if (sw_bd_prod < sw_bd_cons)
+		/* wraparound case */
+		produced = (0xffff - sw_bd_cons) + sw_bd_prod;
+	else
+		produced = sw_bd_prod - sw_bd_cons;
+
+	if (offset <= produced)
+		return RTE_ETH_RX_DESC_AVAIL;
+
+	return RTE_ETH_RX_DESC_UNAVAIL;
+}
diff --git a/drivers/net/qede/qede_rxtx.h b/drivers/net/qede/qede_rxtx.h
index e710fbae..d3a41e92 100644
--- a/drivers/net/qede/qede_rxtx.h
+++ b/drivers/net/qede/qede_rxtx.h
@@ -61,9 +61,16 @@
 #define QEDE_FW_RX_ALIGN_END	(1UL << QEDE_RX_ALIGN_SHIFT)
 #define QEDE_CEIL_TO_CACHE_LINE_SIZE(n) (((n) + (QEDE_FW_RX_ALIGN_END - 1)) & \
 					~(QEDE_FW_RX_ALIGN_END - 1))
-/* Note: QEDE_LLC_SNAP_HDR_LEN is optional */
-#define QEDE_ETH_OVERHEAD	(((2 * QEDE_VLAN_TAG_SIZE)) - (ETHER_CRC_LEN) \
-				+ (QEDE_LLC_SNAP_HDR_LEN))
+#define QEDE_FLOOR_TO_CACHE_LINE_SIZE(n) RTE_ALIGN_FLOOR(n, \
+							 QEDE_FW_RX_ALIGN_END)
+
+/* Note: QEDE_LLC_SNAP_HDR_LEN is optional,
+ * +2 is for padding in front of L2 header
+ */
+#define QEDE_ETH_OVERHEAD	(((2 * QEDE_VLAN_TAG_SIZE)) \
+				 + (QEDE_LLC_SNAP_HDR_LEN) + 2)
+
+#define QEDE_MAX_ETHER_HDR_LEN	(ETHER_HDR_LEN + QEDE_ETH_OVERHEAD)
 
 #define QEDE_RSS_OFFLOAD_ALL    (ETH_RSS_IPV4			|\
 				 ETH_RSS_NONFRAG_IPV4_TCP	|\
@@ -267,6 +274,10 @@ uint16_t qede_rxtx_pkts_dummy(void *p_rxq,
 int qede_start_queues(struct rte_eth_dev *eth_dev);
 
 void qede_stop_queues(struct rte_eth_dev *eth_dev);
+int qede_calc_rx_buf_size(struct rte_eth_dev *dev, uint16_t mbufsz,
+			  uint16_t max_frame_size);
+int
+qede_rx_descriptor_status(void *rxq, uint16_t offset);
 
 /* Fastpath resource alloc/dealloc helpers */
 int qede_alloc_fp_resc(struct qede_dev *qdev);
diff --git a/drivers/net/ring/rte_eth_ring.c b/drivers/net/ring/rte_eth_ring.c
index 791deb0b..aeb48f5e 100644
--- a/drivers/net/ring/rte_eth_ring.c
+++ b/drivers/net/ring/rte_eth_ring.c
@@ -164,7 +164,6 @@ eth_dev_info(struct rte_eth_dev *dev,
 	dev_info->max_rx_queues = (uint16_t)internals->max_rx_queues;
 	dev_info->max_tx_queues = (uint16_t)internals->max_tx_queues;
 	dev_info->min_rx_bufsize = 0;
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -667,10 +666,8 @@ rte_pmd_ring_remove(struct rte_vdev_device *dev)
 		}
 	}
 
-	rte_free(eth_dev->data->rx_queues);
-	rte_free(eth_dev->data->tx_queues);
-	rte_free(eth_dev->data->dev_private);
-
+	/* mac_addrs must not be freed alone because part of dev_private */
+	eth_dev->data->mac_addrs = NULL;
 	rte_eth_dev_release_port(eth_dev);
 	return 0;
 }
diff --git a/drivers/net/sfc/base/ef10_ev.c b/drivers/net/sfc/base/ef10_ev.c
index 7f89a7bf..cdf835f0 100644
--- a/drivers/net/sfc/base/ef10_ev.c
+++ b/drivers/net/sfc/base/ef10_ev.c
@@ -73,11 +73,10 @@ efx_mcdi_set_evq_tmr(
 	__in		uint32_t timer_ns)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_SET_EVQ_TMR_IN_LEN,
-			    MC_CMD_SET_EVQ_TMR_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_EVQ_TMR_IN_LEN,
+		MC_CMD_SET_EVQ_TMR_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_SET_EVQ_TMR;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_SET_EVQ_TMR_IN_LEN;
@@ -123,9 +122,9 @@ efx_mcdi_init_evq(
 	__in		boolean_t low_latency)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[
-	    MAX(MC_CMD_INIT_EVQ_IN_LEN(EFX_EVQ_NBUFS(EFX_EVQ_MAXNEVS)),
-		MC_CMD_INIT_EVQ_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload,
+		MC_CMD_INIT_EVQ_IN_LEN(EFX_EVQ_NBUFS(EFX_EVQ_MAXNEVS)),
+		MC_CMD_INIT_EVQ_OUT_LEN);
 	efx_qword_t *dma_addr;
 	uint64_t addr;
 	int npages;
@@ -140,7 +139,6 @@ efx_mcdi_init_evq(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_INIT_EVQ;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_INIT_EVQ_IN_LEN(npages);
@@ -260,9 +258,9 @@ efx_mcdi_init_evq_v2(
 	__in		uint32_t flags)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[
-		MAX(MC_CMD_INIT_EVQ_V2_IN_LEN(EFX_EVQ_NBUFS(EFX_EVQ_MAXNEVS)),
-		    MC_CMD_INIT_EVQ_V2_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload,
+		MC_CMD_INIT_EVQ_V2_IN_LEN(EFX_EVQ_NBUFS(EFX_EVQ_MAXNEVS)),
+		MC_CMD_INIT_EVQ_V2_OUT_LEN);
 	boolean_t interrupting;
 	unsigned int evq_type;
 	efx_qword_t *dma_addr;
@@ -277,7 +275,6 @@ efx_mcdi_init_evq_v2(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_INIT_EVQ;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_INIT_EVQ_V2_IN_LEN(npages);
@@ -384,11 +381,10 @@ efx_mcdi_fini_evq(
 	__in		uint32_t instance)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_FINI_EVQ_IN_LEN,
-			    MC_CMD_FINI_EVQ_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_EVQ_IN_LEN,
+		MC_CMD_FINI_EVQ_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_FINI_EVQ;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_FINI_EVQ_IN_LEN;
@@ -603,8 +599,8 @@ efx_mcdi_driver_event(
 	__in		efx_qword_t data)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_DRIVER_EVENT_IN_LEN,
-			    MC_CMD_DRIVER_EVENT_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_DRIVER_EVENT_IN_LEN,
+		MC_CMD_DRIVER_EVENT_OUT_LEN);
 	efx_rc_t rc;
 
 	req.emr_cmd = MC_CMD_DRIVER_EVENT;
@@ -867,8 +863,9 @@ ef10_ev_rx(
 
 	EFX_EV_QSTAT_INCR(eep, EV_RX);
 
-	/* Discard events after RXQ/TXQ errors */
-	if (enp->en_reset_flags & (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR))
+	/* Discard events after RXQ/TXQ errors, or hardware not available */
+	if (enp->en_reset_flags &
+	    (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR | EFX_RESET_HW_UNAVAIL))
 		return (B_FALSE);
 
 	/* Basic packet information */
@@ -1068,8 +1065,9 @@ ef10_ev_tx(
 
 	EFX_EV_QSTAT_INCR(eep, EV_TX);
 
-	/* Discard events after RXQ/TXQ errors */
-	if (enp->en_reset_flags & (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR))
+	/* Discard events after RXQ/TXQ errors, or hardware not available */
+	if (enp->en_reset_flags &
+	    (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR | EFX_RESET_HW_UNAVAIL))
 		return (B_FALSE);
 
 	if (EFX_QWORD_FIELD(*eqp, ESF_DZ_TX_DROP_EVENT) != 0) {
diff --git a/drivers/net/sfc/base/ef10_filter.c b/drivers/net/sfc/base/ef10_filter.c
index ae872853..afe4064d 100644
--- a/drivers/net/sfc/base/ef10_filter.c
+++ b/drivers/net/sfc/base/ef10_filter.c
@@ -172,12 +172,11 @@ efx_mcdi_filter_op_add(
 	__inout		ef10_filter_handle_t *handle)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_FILTER_OP_V3_IN_LEN,
-			    MC_CMD_FILTER_OP_EXT_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FILTER_OP_V3_IN_LEN,
+		MC_CMD_FILTER_OP_EXT_OUT_LEN);
 	efx_filter_match_flags_t match_flags;
 	efx_rc_t rc;
 
-	memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_FILTER_OP;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_FILTER_OP_V3_IN_LEN;
@@ -376,11 +375,10 @@ efx_mcdi_filter_op_delete(
 	__inout		ef10_filter_handle_t *handle)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_FILTER_OP_EXT_IN_LEN,
-			    MC_CMD_FILTER_OP_EXT_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FILTER_OP_EXT_IN_LEN,
+		MC_CMD_FILTER_OP_EXT_OUT_LEN);
 	efx_rc_t rc;
 
-	memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_FILTER_OP;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_FILTER_OP_EXT_IN_LEN;
@@ -950,13 +948,12 @@ efx_mcdi_get_parser_disp_info(
 	__out				size_t *list_lengthp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_PARSER_DISP_INFO_IN_LEN,
-			    MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN,
+		MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX);
 	size_t matches_count;
 	size_t list_size;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_PARSER_DISP_INFO;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_PARSER_DISP_INFO_IN_LEN;
@@ -1144,12 +1141,15 @@ ef10_filter_insert_unicast(
 	efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
 	    filter_flags,
 	    eftp->eft_default_rxq);
-	efx_filter_spec_set_eth_local(&spec, EFX_FILTER_SPEC_VID_UNSPEC, addr);
+	rc = efx_filter_spec_set_eth_local(&spec, EFX_FILTER_SPEC_VID_UNSPEC,
+	    addr);
+	if (rc != 0)
+		goto fail1;
 
 	rc = ef10_filter_add_internal(enp, &spec, B_TRUE,
 	    &eftp->eft_unicst_filter_indexes[eftp->eft_unicst_filter_count]);
 	if (rc != 0)
-		goto fail1;
+		goto fail2;
 
 	eftp->eft_unicst_filter_count++;
 	EFSYS_ASSERT(eftp->eft_unicst_filter_count <=
@@ -1157,6 +1157,8 @@ ef10_filter_insert_unicast(
 
 	return (0);
 
+fail2:
+	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 	return (rc);
@@ -1175,11 +1177,13 @@ ef10_filter_insert_all_unicast(
 	efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
 	    filter_flags,
 	    eftp->eft_default_rxq);
-	efx_filter_spec_set_uc_def(&spec);
+	rc = efx_filter_spec_set_uc_def(&spec);
+	if (rc != 0)
+		goto fail1;
 	rc = ef10_filter_add_internal(enp, &spec, B_TRUE,
 	    &eftp->eft_unicst_filter_indexes[eftp->eft_unicst_filter_count]);
 	if (rc != 0)
-		goto fail1;
+		goto fail2;
 
 	eftp->eft_unicst_filter_count++;
 	EFSYS_ASSERT(eftp->eft_unicst_filter_count <=
@@ -1187,6 +1191,8 @@ ef10_filter_insert_all_unicast(
 
 	return (0);
 
+fail2:
+	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 	return (rc);
@@ -1228,9 +1234,21 @@ ef10_filter_insert_multicast_list(
 		    filter_flags,
 		    eftp->eft_default_rxq);
 
-		efx_filter_spec_set_eth_local(&spec,
+		rc = efx_filter_spec_set_eth_local(&spec,
 		    EFX_FILTER_SPEC_VID_UNSPEC,
 		    &addrs[i * EFX_MAC_ADDR_LEN]);
+		if (rc != 0) {
+			if (rollback == B_TRUE) {
+				/* Only stop upon failure if told to rollback */
+				goto rollback;
+			} else {
+				/*
+				 * Don't try to add a filter with a corrupt
+				 * specification.
+				 */
+				continue;
+			}
+		}
 
 		rc = ef10_filter_add_internal(enp, &spec, B_TRUE,
 					    &filter_index);
@@ -1253,8 +1271,12 @@ ef10_filter_insert_multicast_list(
 		    eftp->eft_default_rxq);
 
 		EFX_MAC_BROADCAST_ADDR_SET(addr);
-		efx_filter_spec_set_eth_local(&spec, EFX_FILTER_SPEC_VID_UNSPEC,
-		    addr);
+		rc = efx_filter_spec_set_eth_local(&spec,
+		    EFX_FILTER_SPEC_VID_UNSPEC, addr);
+		if ((rc != 0) && (rollback == B_TRUE)) {
+			/* Only stop upon failure if told to rollback */
+			goto rollback;
+		}
 
 		rc = ef10_filter_add_internal(enp, &spec, B_TRUE,
 					    &filter_index);
@@ -1302,12 +1324,14 @@ ef10_filter_insert_all_multicast(
 	efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
 	    filter_flags,
 	    eftp->eft_default_rxq);
-	efx_filter_spec_set_mc_def(&spec);
+	rc = efx_filter_spec_set_mc_def(&spec);
+	if (rc != 0)
+		goto fail1;
 
 	rc = ef10_filter_add_internal(enp, &spec, B_TRUE,
 	    &eftp->eft_mulcst_filter_indexes[0]);
 	if (rc != 0)
-		goto fail1;
+		goto fail2;
 
 	eftp->eft_mulcst_filter_count = 1;
 	eftp->eft_using_all_mulcst = B_TRUE;
@@ -1318,6 +1342,8 @@ ef10_filter_insert_all_multicast(
 
 	return (0);
 
+fail2:
+	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
@@ -1552,7 +1578,7 @@ ef10_filter_reconfigure(
 	/*
 	 * Insert or renew unicast filters.
 	 *
-	 * Frimware does not perform chaining on unicast filters. As traffic is
+	 * Firmware does not perform chaining on unicast filters. As traffic is
 	 * therefore only delivered to the first matching filter, we should
 	 * always insert the specific filter for our MAC address, to try and
 	 * ensure we get that traffic.
diff --git a/drivers/net/sfc/base/ef10_image.c b/drivers/net/sfc/base/ef10_image.c
index 6fb7e476..c035e0df 100644
--- a/drivers/net/sfc/base/ef10_image.c
+++ b/drivers/net/sfc/base/ef10_image.c
@@ -577,7 +577,8 @@ fail1:
 
 	__checkReturn	efx_rc_t
 efx_build_signed_image_write_buffer(
-	__out		uint8_t			*bufferp,
+	__out_bcount(buffer_size)
+			uint8_t			*bufferp,
 	__in		uint32_t		buffer_size,
 	__in		efx_image_info_t	*infop,
 	__out		efx_image_header_t	**headerpp)
@@ -704,7 +705,7 @@ efx_build_signed_image_write_buffer(
 	 * results in the layout used for the data chunks and chunk headers.
 	 */
 	/* END CSTYLED */
-	memset(bufferp, buffer_size, 0xFF);
+	memset(bufferp, 0xFF, buffer_size);
 
 	EFX_STATIC_ASSERT(sizeof (chunk_hdr) == SIGNED_IMAGE_CHUNK_HDR_LEN);
 	memset(&chunk_hdr, 0, SIGNED_IMAGE_CHUNK_HDR_LEN);
diff --git a/drivers/net/sfc/base/ef10_impl.h b/drivers/net/sfc/base/ef10_impl.h
index 4751faf1..f971063a 100644
--- a/drivers/net/sfc/base/ef10_impl.h
+++ b/drivers/net/sfc/base/ef10_impl.h
@@ -190,6 +190,14 @@ extern	__checkReturn	efx_rc_t
 ef10_nic_init(
 	__in		efx_nic_t *enp);
 
+extern	__checkReturn	boolean_t
+ef10_nic_hw_unavailable(
+	__in		efx_nic_t *enp);
+
+extern			void
+ef10_nic_set_hw_unavailable(
+	__in		efx_nic_t *enp);
+
 #if EFSYS_OPT_DIAG
 
 extern	__checkReturn	efx_rc_t
@@ -453,7 +461,7 @@ ef10_nvram_partn_write(
 	__in			efx_nic_t *enp,
 	__in			uint32_t partn,
 	__in			unsigned int offset,
-	__out_bcount(size)	caddr_t data,
+	__in_bcount(size)	caddr_t data,
 	__in			size_t size);
 
 extern	__checkReturn		efx_rc_t
@@ -477,17 +485,21 @@ ef10_nvram_partn_set_version(
 
 extern	__checkReturn		efx_rc_t
 ef10_nvram_buffer_validate(
-	__in			efx_nic_t *enp,
 	__in			uint32_t partn,
 	__in_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size);
 
+extern			void
+ef10_nvram_buffer_init(
+	__out_bcount(buffer_size)
+				caddr_t bufferp,
+	__in			size_t buffer_size);
+
 extern	__checkReturn		efx_rc_t
 ef10_nvram_buffer_create(
-	__in			efx_nic_t *enp,
-	__in			uint16_t partn_type,
-	__in_bcount(buffer_size)
+	__in			uint32_t partn_type,
+	__out_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size);
 
@@ -516,15 +528,26 @@ ef10_nvram_buffer_find_item(
 	__out			uint32_t *lengthp);
 
 extern	__checkReturn		efx_rc_t
+ef10_nvram_buffer_peek_item(
+	__in_bcount(buffer_size)
+				caddr_t bufferp,
+	__in			size_t buffer_size,
+	__in			uint32_t offset,
+	__out			uint32_t *tagp,
+	__out			uint32_t *lengthp,
+	__out			uint32_t *value_offsetp);
+
+extern	__checkReturn		efx_rc_t
 ef10_nvram_buffer_get_item(
 	__in_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size,
 	__in			uint32_t offset,
 	__in			uint32_t length,
-	__out_bcount_part(item_max_size, *lengthp)
-				caddr_t itemp,
-	__in			size_t item_max_size,
+	__out			uint32_t *tagp,
+	__out_bcount_part(value_max_size, *lengthp)
+				caddr_t valuep,
+	__in			size_t value_max_size,
 	__out			uint32_t *lengthp);
 
 extern	__checkReturn		efx_rc_t
@@ -533,7 +556,19 @@ ef10_nvram_buffer_insert_item(
 				caddr_t bufferp,
 	__in			size_t buffer_size,
 	__in			uint32_t offset,
-	__in_bcount(length)	caddr_t keyp,
+	__in			uint32_t tag,
+	__in_bcount(length)	caddr_t valuep,
+	__in			uint32_t length,
+	__out			uint32_t *lengthp);
+
+extern	__checkReturn		efx_rc_t
+ef10_nvram_buffer_modify_item(
+	__in_bcount(buffer_size)
+				caddr_t bufferp,
+	__in			size_t buffer_size,
+	__in			uint32_t offset,
+	__in			uint32_t tag,
+	__in_bcount(length)	caddr_t valuep,
 	__in			uint32_t length,
 	__out			uint32_t *lengthp);
 
@@ -558,10 +593,7 @@ ef10_nvram_buffer_finish(
 /* PHY */
 
 typedef struct ef10_link_state_s {
-	uint32_t		els_adv_cap_mask;
-	uint32_t		els_lp_cap_mask;
-	unsigned int		els_fcntl;
-	efx_link_mode_t		els_link_mode;
+	efx_phy_link_state_t	epls;
 #if EFSYS_OPT_LOOPBACK
 	efx_loopback_type_t	els_loopback;
 #endif
@@ -597,6 +629,11 @@ ef10_phy_oui_get(
 	__in		efx_nic_t *enp,
 	__out		uint32_t *ouip);
 
+extern	__checkReturn	efx_rc_t
+ef10_phy_link_state_get(
+	__in		efx_nic_t *enp,
+	__out		efx_phy_link_state_t *eplsp);
+
 #if EFSYS_OPT_PHY_STATS
 
 extern	__checkReturn			efx_rc_t
@@ -1128,11 +1165,12 @@ extern	__checkReturn	efx_rc_t
 efx_mcdi_get_port_modes(
 	__in		efx_nic_t *enp,
 	__out		uint32_t *modesp,
-	__out_opt	uint32_t *current_modep);
+	__out_opt	uint32_t *current_modep,
+	__out_opt	uint32_t *default_modep);
 
 extern	__checkReturn	efx_rc_t
 ef10_nic_get_port_mode_bandwidth(
-	__in		uint32_t port_mode,
+	__in		efx_nic_t *enp,
 	__out		uint32_t *bandwidth_mbpsp);
 
 extern	__checkReturn	efx_rc_t
diff --git a/drivers/net/sfc/base/ef10_intr.c b/drivers/net/sfc/base/ef10_intr.c
index 1ffe266b..efa15712 100644
--- a/drivers/net/sfc/base/ef10_intr.c
+++ b/drivers/net/sfc/base/ef10_intr.c
@@ -51,8 +51,8 @@ efx_mcdi_trigger_interrupt(
 	__in		unsigned int level)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_TRIGGER_INTERRUPT_IN_LEN,
-			    MC_CMD_TRIGGER_INTERRUPT_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_TRIGGER_INTERRUPT_IN_LEN,
+		MC_CMD_TRIGGER_INTERRUPT_OUT_LEN);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
@@ -64,7 +64,6 @@ efx_mcdi_trigger_interrupt(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_TRIGGER_INTERRUPT;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_TRIGGER_INTERRUPT_IN_LEN;
diff --git a/drivers/net/sfc/base/ef10_mac.c b/drivers/net/sfc/base/ef10_mac.c
index 1031e836..9f10f6f7 100644
--- a/drivers/net/sfc/base/ef10_mac.c
+++ b/drivers/net/sfc/base/ef10_mac.c
@@ -22,10 +22,10 @@ ef10_mac_poll(
 	if ((rc = ef10_phy_get_link(enp, &els)) != 0)
 		goto fail1;
 
-	epp->ep_adv_cap_mask = els.els_adv_cap_mask;
-	epp->ep_fcntl = els.els_fcntl;
+	epp->ep_adv_cap_mask = els.epls.epls_adv_cap_mask;
+	epp->ep_fcntl = els.epls.epls_fcntl;
 
-	*link_modep = els.els_link_mode;
+	*link_modep = els.epls.epls_link_mode;
 
 	return (0);
 
@@ -75,11 +75,10 @@ efx_mcdi_vadapter_set_mac(
 {
 	efx_port_t *epp = &(enp->en_port);
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_VADAPTOR_SET_MAC_IN_LEN,
-			    MC_CMD_VADAPTOR_SET_MAC_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_VADAPTOR_SET_MAC_IN_LEN,
+		MC_CMD_VADAPTOR_SET_MAC_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_VADAPTOR_SET_MAC;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_VADAPTOR_SET_MAC_IN_LEN;
@@ -141,11 +140,10 @@ efx_mcdi_mtu_set(
 	__in		uint32_t mtu)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_SET_MAC_EXT_IN_LEN,
-			    MC_CMD_SET_MAC_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_MAC_EXT_IN_LEN,
+		MC_CMD_SET_MAC_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_SET_MAC;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_SET_MAC_EXT_IN_LEN;
@@ -178,11 +176,10 @@ efx_mcdi_mtu_get(
 	__out		size_t *mtu)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_SET_MAC_EXT_IN_LEN,
-			    MC_CMD_SET_MAC_V2_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_MAC_EXT_IN_LEN,
+		MC_CMD_SET_MAC_V2_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_SET_MAC;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_SET_MAC_EXT_IN_LEN;
@@ -274,11 +271,10 @@ ef10_mac_reconfigure(
 {
 	efx_port_t *epp = &(enp->en_port);
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_SET_MAC_IN_LEN,
-			    MC_CMD_SET_MAC_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_MAC_IN_LEN,
+		MC_CMD_SET_MAC_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_SET_MAC;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_SET_MAC_IN_LEN;
@@ -412,7 +408,7 @@ ef10_mac_filter_default_rxq_clear(
 
 	ef10_filter_default_rxq_clear(enp);
 
-	efx_filter_reconfigure(enp, epp->ep_mac_addr,
+	(void) efx_filter_reconfigure(enp, epp->ep_mac_addr,
 				    epp->ep_all_unicst, epp->ep_mulcst,
 				    epp->ep_all_mulcst, epp->ep_brdcst,
 				    epp->ep_mulcst_addr_list,
@@ -654,7 +650,7 @@ ef10_mac_stats_update(
 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_LT64_PKTS, &value);
 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value);
 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_64_PKTS, &value);
-	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value);
+	EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value);
 
 	EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_65_TO_127_PKTS, &value);
 	EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_65_TO_127_PKTS]), &value);
diff --git a/drivers/net/sfc/base/ef10_nic.c b/drivers/net/sfc/base/ef10_nic.c
index 7dbf843b..50e23b7d 100644
--- a/drivers/net/sfc/base/ef10_nic.c
+++ b/drivers/net/sfc/base/ef10_nic.c
@@ -20,15 +20,14 @@ efx_mcdi_get_port_assignment(
 	__out		uint32_t *portp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_PORT_ASSIGNMENT_IN_LEN,
-			    MC_CMD_GET_PORT_ASSIGNMENT_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_PORT_ASSIGNMENT_IN_LEN,
+		MC_CMD_GET_PORT_ASSIGNMENT_OUT_LEN);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
 	    enp->en_family == EFX_FAMILY_MEDFORD ||
 	    enp->en_family == EFX_FAMILY_MEDFORD2);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_PORT_ASSIGNMENT;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_PORT_ASSIGNMENT_IN_LEN;
@@ -63,18 +62,18 @@ fail1:
 efx_mcdi_get_port_modes(
 	__in		efx_nic_t *enp,
 	__out		uint32_t *modesp,
-	__out_opt	uint32_t *current_modep)
+	__out_opt	uint32_t *current_modep,
+	__out_opt	uint32_t *default_modep)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_PORT_MODES_IN_LEN,
-			    MC_CMD_GET_PORT_MODES_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_PORT_MODES_IN_LEN,
+		MC_CMD_GET_PORT_MODES_OUT_LEN);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
 	    enp->en_family == EFX_FAMILY_MEDFORD ||
 	    enp->en_family == EFX_FAMILY_MEDFORD2);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_PORT_MODES;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_PORT_MODES_IN_LEN;
@@ -110,6 +109,11 @@ efx_mcdi_get_port_modes(
 					    GET_PORT_MODES_OUT_CURRENT_MODE);
 	}
 
+	if (default_modep != NULL) {
+		*default_modep = MCDI_OUT_DWORD(req,
+					    GET_PORT_MODES_OUT_DEFAULT_MODE);
+	}
+
 	return (0);
 
 fail3:
@@ -124,44 +128,99 @@ fail1:
 
 	__checkReturn	efx_rc_t
 ef10_nic_get_port_mode_bandwidth(
-	__in		uint32_t port_mode,
+	__in		efx_nic_t *enp,
 	__out		uint32_t *bandwidth_mbpsp)
 {
+	uint32_t port_modes;
+	uint32_t current_mode;
+	efx_port_t *epp = &(enp->en_port);
+
+	uint32_t single_lane;
+	uint32_t dual_lane;
+	uint32_t quad_lane;
 	uint32_t bandwidth;
 	efx_rc_t rc;
 
-	switch (port_mode) {
-	case TLV_PORT_MODE_10G:
-		bandwidth = 10000;
+	if ((rc = efx_mcdi_get_port_modes(enp, &port_modes,
+				    &current_mode, NULL)) != 0) {
+		/* No port mode info available. */
+		goto fail1;
+	}
+
+	if (epp->ep_phy_cap_mask & (1 << EFX_PHY_CAP_25000FDX))
+		single_lane = 25000;
+	else
+		single_lane = 10000;
+
+	if (epp->ep_phy_cap_mask & (1 << EFX_PHY_CAP_50000FDX))
+		dual_lane = 50000;
+	else
+		dual_lane = 20000;
+
+	if (epp->ep_phy_cap_mask & (1 << EFX_PHY_CAP_100000FDX))
+		quad_lane = 100000;
+	else
+		quad_lane = 40000;
+
+	switch (current_mode) {
+	case TLV_PORT_MODE_1x1_NA:			/* mode 0 */
+		bandwidth = single_lane;
+		break;
+	case TLV_PORT_MODE_1x2_NA:			/* mode 10 */
+	case TLV_PORT_MODE_NA_1x2:			/* mode 11 */
+		bandwidth = dual_lane;
+		break;
+	case TLV_PORT_MODE_1x1_1x1:			/* mode 2 */
+		bandwidth = single_lane + single_lane;
+		break;
+	case TLV_PORT_MODE_4x1_NA:			/* mode 4 */
+	case TLV_PORT_MODE_NA_4x1:			/* mode 8 */
+		bandwidth = 4 * single_lane;
+		break;
+	case TLV_PORT_MODE_2x1_2x1:			/* mode 5 */
+		bandwidth = (2 * single_lane) + (2 * single_lane);
+		break;
+	case TLV_PORT_MODE_1x2_1x2:			/* mode 12 */
+		bandwidth = dual_lane + dual_lane;
+		break;
+	case TLV_PORT_MODE_1x2_2x1:			/* mode 17 */
+	case TLV_PORT_MODE_2x1_1x2:			/* mode 18 */
+		bandwidth = dual_lane + (2 * single_lane);
 		break;
-	case TLV_PORT_MODE_10G_10G:
-		bandwidth = 10000 * 2;
+	/* Legacy Medford-only mode. Do not use (see bug63270) */
+	case TLV_PORT_MODE_10G_10G_10G_10G_Q1_Q2:	/* mode 9 */
+		bandwidth = 4 * single_lane;
 		break;
-	case TLV_PORT_MODE_10G_10G_10G_10G:
-	case TLV_PORT_MODE_10G_10G_10G_10G_Q:
-	case TLV_PORT_MODE_10G_10G_10G_10G_Q1_Q2:
-	case TLV_PORT_MODE_10G_10G_10G_10G_Q2:
-		bandwidth = 10000 * 4;
+	case TLV_PORT_MODE_1x4_NA:			/* mode 1 */
+	case TLV_PORT_MODE_NA_1x4:			/* mode 22 */
+		bandwidth = quad_lane;
 		break;
-	case TLV_PORT_MODE_40G:
-		bandwidth = 40000;
+	case TLV_PORT_MODE_2x2_NA:			/* mode 13 */
+	case TLV_PORT_MODE_NA_2x2:			/* mode 14 */
+		bandwidth = 2 * dual_lane;
 		break;
-	case TLV_PORT_MODE_40G_40G:
-		bandwidth = 40000 * 2;
+	case TLV_PORT_MODE_1x4_2x1:			/* mode 6 */
+	case TLV_PORT_MODE_2x1_1x4:			/* mode 7 */
+		bandwidth = quad_lane + (2 * single_lane);
 		break;
-	case TLV_PORT_MODE_40G_10G_10G:
-	case TLV_PORT_MODE_10G_10G_40G:
-		bandwidth = 40000 + (10000 * 2);
+	case TLV_PORT_MODE_1x4_1x2:			/* mode 15 */
+	case TLV_PORT_MODE_1x2_1x4:			/* mode 16 */
+		bandwidth = quad_lane + dual_lane;
+		break;
+	case TLV_PORT_MODE_1x4_1x4:			/* mode 3 */
+		bandwidth = quad_lane + quad_lane;
 		break;
 	default:
 		rc = EINVAL;
-		goto fail1;
+		goto fail2;
 	}
 
 	*bandwidth_mbpsp = bandwidth;
 
 	return (0);
 
+fail2:
+	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
@@ -174,13 +233,12 @@ efx_mcdi_vadaptor_alloc(
 	__in			uint32_t port_id)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_VADAPTOR_ALLOC_IN_LEN,
-			    MC_CMD_VADAPTOR_ALLOC_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_VADAPTOR_ALLOC_IN_LEN,
+		MC_CMD_VADAPTOR_ALLOC_OUT_LEN);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT3U(enp->en_vport_id, ==, EVB_PORT_ID_NULL);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_VADAPTOR_ALLOC;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_VADAPTOR_ALLOC_IN_LEN;
@@ -213,11 +271,10 @@ efx_mcdi_vadaptor_free(
 	__in			uint32_t port_id)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_VADAPTOR_FREE_IN_LEN,
-			    MC_CMD_VADAPTOR_FREE_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_VADAPTOR_FREE_IN_LEN,
+		MC_CMD_VADAPTOR_FREE_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_VADAPTOR_FREE;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_VADAPTOR_FREE_IN_LEN;
@@ -247,15 +304,14 @@ efx_mcdi_get_mac_address_pf(
 	__out_ecount_opt(6)	uint8_t mac_addrp[6])
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_MAC_ADDRESSES_IN_LEN,
-			    MC_CMD_GET_MAC_ADDRESSES_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_MAC_ADDRESSES_IN_LEN,
+		MC_CMD_GET_MAC_ADDRESSES_OUT_LEN);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
 	    enp->en_family == EFX_FAMILY_MEDFORD ||
 	    enp->en_family == EFX_FAMILY_MEDFORD2);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_MAC_ADDRESSES;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_MAC_ADDRESSES_IN_LEN;
@@ -306,15 +362,14 @@ efx_mcdi_get_mac_address_vf(
 	__out_ecount_opt(6)	uint8_t mac_addrp[6])
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_LEN,
-			    MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMAX)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_LEN,
+		MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMAX);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
 	    enp->en_family == EFX_FAMILY_MEDFORD ||
 	    enp->en_family == EFX_FAMILY_MEDFORD2);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_VPORT_GET_MAC_ADDRESSES;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_LEN;
@@ -371,15 +426,14 @@ efx_mcdi_get_clock(
 	__out		uint32_t *dpcpu_freqp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_CLOCK_IN_LEN,
-			    MC_CMD_GET_CLOCK_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_CLOCK_IN_LEN,
+		MC_CMD_GET_CLOCK_OUT_LEN);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
 	    enp->en_family == EFX_FAMILY_MEDFORD ||
 	    enp->en_family == EFX_FAMILY_MEDFORD2);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_CLOCK;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_CLOCK_IN_LEN;
@@ -429,12 +483,11 @@ efx_mcdi_get_rxdp_config(
 	__out		uint32_t *end_paddingp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_RXDP_CONFIG_IN_LEN,
-			    MC_CMD_GET_RXDP_CONFIG_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_RXDP_CONFIG_IN_LEN,
+		MC_CMD_GET_RXDP_CONFIG_OUT_LEN);
 	uint32_t end_padding;
 	efx_rc_t rc;
 
-	memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_RXDP_CONFIG;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_RXDP_CONFIG_IN_LEN;
@@ -489,11 +542,10 @@ efx_mcdi_get_vector_cfg(
 	__out_opt	uint32_t *vf_nvecp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_VECTOR_CFG_IN_LEN,
-			    MC_CMD_GET_VECTOR_CFG_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_VECTOR_CFG_IN_LEN,
+		MC_CMD_GET_VECTOR_CFG_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_VECTOR_CFG;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_VECTOR_CFG_IN_LEN;
@@ -539,8 +591,8 @@ efx_mcdi_alloc_vis(
 	__out		uint32_t *vi_shiftp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_ALLOC_VIS_IN_LEN,
-			    MC_CMD_ALLOC_VIS_EXT_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_ALLOC_VIS_IN_LEN,
+		MC_CMD_ALLOC_VIS_EXT_OUT_LEN);
 	efx_rc_t rc;
 
 	if (vi_countp == NULL) {
@@ -548,7 +600,6 @@ efx_mcdi_alloc_vis(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_ALLOC_VIS;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_ALLOC_VIS_IN_LEN;
@@ -631,8 +682,8 @@ efx_mcdi_alloc_piobuf(
 	__out		efx_piobuf_handle_t *handlep)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_ALLOC_PIOBUF_IN_LEN,
-			    MC_CMD_ALLOC_PIOBUF_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_ALLOC_PIOBUF_IN_LEN,
+		MC_CMD_ALLOC_PIOBUF_OUT_LEN);
 	efx_rc_t rc;
 
 	if (handlep == NULL) {
@@ -640,7 +691,6 @@ efx_mcdi_alloc_piobuf(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_ALLOC_PIOBUF;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_ALLOC_PIOBUF_IN_LEN;
@@ -679,11 +729,10 @@ efx_mcdi_free_piobuf(
 	__in		efx_piobuf_handle_t handle)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_FREE_PIOBUF_IN_LEN,
-			    MC_CMD_FREE_PIOBUF_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FREE_PIOBUF_IN_LEN,
+		MC_CMD_FREE_PIOBUF_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_FREE_PIOBUF;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_FREE_PIOBUF_IN_LEN;
@@ -714,11 +763,10 @@ efx_mcdi_link_piobuf(
 	__in		efx_piobuf_handle_t handle)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_LINK_PIOBUF_IN_LEN,
-			    MC_CMD_LINK_PIOBUF_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_LINK_PIOBUF_IN_LEN,
+		MC_CMD_LINK_PIOBUF_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_LINK_PIOBUF;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_LINK_PIOBUF_IN_LEN;
@@ -749,11 +797,10 @@ efx_mcdi_unlink_piobuf(
 	__in		uint32_t vi_index)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_UNLINK_PIOBUF_IN_LEN,
-			    MC_CMD_UNLINK_PIOBUF_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_UNLINK_PIOBUF_IN_LEN,
+		MC_CMD_UNLINK_PIOBUF_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_UNLINK_PIOBUF;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_UNLINK_PIOBUF_IN_LEN;
@@ -806,7 +853,7 @@ fail1:
 	for (i = 0; i < enp->en_arch.ef10.ena_piobuf_count; i++) {
 		handlep = &enp->en_arch.ef10.ena_piobuf_handle[i];
 
-		efx_mcdi_free_piobuf(enp, *handlep);
+		(void) efx_mcdi_free_piobuf(enp, *handlep);
 		*handlep = EFX_PIOBUF_HANDLE_INVALID;
 	}
 	enp->en_arch.ef10.ena_piobuf_count = 0;
@@ -823,7 +870,7 @@ ef10_nic_free_piobufs(
 	for (i = 0; i < enp->en_arch.ef10.ena_piobuf_count; i++) {
 		handlep = &enp->en_arch.ef10.ena_piobuf_handle[i];
 
-		efx_mcdi_free_piobuf(enp, *handlep);
+		(void) efx_mcdi_free_piobuf(enp, *handlep);
 		*handlep = EFX_PIOBUF_HANDLE_INVALID;
 	}
 	enp->en_arch.ef10.ena_piobuf_count = 0;
@@ -951,11 +998,10 @@ ef10_mcdi_get_pf_count(
 	__out		uint32_t *pf_countp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_PF_COUNT_IN_LEN,
-			    MC_CMD_GET_PF_COUNT_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_PF_COUNT_IN_LEN,
+		MC_CMD_GET_PF_COUNT_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_PF_COUNT;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_PF_COUNT_IN_LEN;
@@ -995,15 +1041,14 @@ ef10_get_datapath_caps(
 {
 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_CAPABILITIES_IN_LEN,
-			    MC_CMD_GET_CAPABILITIES_V5_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_CAPABILITIES_IN_LEN,
+		MC_CMD_GET_CAPABILITIES_V5_OUT_LEN);
 	efx_rc_t rc;
 
 	if ((rc = ef10_mcdi_get_pf_count(enp, &encp->enc_hw_pf_count)) != 0)
 		goto fail1;
 
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_CAPABILITIES;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_CAPABILITIES_IN_LEN;
@@ -1041,11 +1086,13 @@ ef10_get_datapath_caps(
 	}
 	encp->enc_rx_prefix_size = 14;
 
+#if EFSYS_OPT_RX_SCALE
 	/* Check if the firmware supports additional RSS modes */
 	if (CAP_FLAGS1(req, ADDITIONAL_RSS_MODES))
 		encp->enc_rx_scale_additional_modes_supported = B_TRUE;
 	else
 		encp->enc_rx_scale_additional_modes_supported = B_FALSE;
+#endif /* EFSYS_OPT_RX_SCALE */
 
 	/* Check if the firmware supports TSO */
 	if (CAP_FLAGS1(req, TX_TSO))
@@ -1251,6 +1298,7 @@ ef10_get_datapath_caps(
 	else
 		encp->enc_hlb_counters = B_FALSE;
 
+#if EFSYS_OPT_RX_SCALE
 	if (CAP_FLAGS1(req, RX_RSS_LIMITED)) {
 		/* Only one exclusive RSS context is available per port. */
 		encp->enc_rx_scale_max_exclusive_contexts = 1;
@@ -1300,6 +1348,8 @@ ef10_get_datapath_caps(
 		 */
 		encp->enc_rx_scale_l4_hash_supported = B_TRUE;
 	}
+#endif /* EFSYS_OPT_RX_SCALE */
+
 	/* Check if the firmware supports "FLAG" and "MARK" filter actions */
 	if (CAP_FLAGS2(req, FILTER_ACTION_FLAG))
 		encp->enc_filter_action_flag_supported = B_TRUE;
@@ -1323,8 +1373,10 @@ ef10_get_datapath_caps(
 
 	return (0);
 
+#if EFSYS_OPT_RX_SCALE
 fail5:
 	EFSYS_PROBE(fail5);
+#endif /* EFSYS_OPT_RX_SCALE */
 fail4:
 	EFSYS_PROBE(fail4);
 fail3:
@@ -1478,8 +1530,8 @@ static struct ef10_external_port_map_s {
 	 */
 	{
 		EFX_FAMILY_MEDFORD,
-		(1U << TLV_PORT_MODE_10G) |			/* mode 0 */
-		(1U << TLV_PORT_MODE_10G_10G),			/* mode 2 */
+		(1U << TLV_PORT_MODE_1x1_NA) |			/* mode 0 */
+		(1U << TLV_PORT_MODE_1x1_1x1),			/* mode 2 */
 		1,	/* ports per cage */
 		1	/* first cage */
 	},
@@ -1493,10 +1545,10 @@ static struct ef10_external_port_map_s {
 	 */
 	{
 		EFX_FAMILY_MEDFORD,
-		(1U << TLV_PORT_MODE_40G) |			/* mode 1 */
-		(1U << TLV_PORT_MODE_40G_40G) |			/* mode 3 */
-		(1U << TLV_PORT_MODE_40G_10G_10G) |		/* mode 6 */
-		(1U << TLV_PORT_MODE_10G_10G_40G) |		/* mode 7 */
+		(1U << TLV_PORT_MODE_1x4_NA) |			/* mode 1 */
+		(1U << TLV_PORT_MODE_1x4_1x4) |			/* mode 3 */
+		(1U << TLV_PORT_MODE_1x4_2x1) |			/* mode 6 */
+		(1U << TLV_PORT_MODE_2x1_1x4) |			/* mode 7 */
 		/* Do not use 10G_10G_10G_10G_Q1_Q2 (see bug63270) */
 		(1U << TLV_PORT_MODE_10G_10G_10G_10G_Q1_Q2),	/* mode 9 */
 		2,	/* ports per cage */
@@ -1512,9 +1564,9 @@ static struct ef10_external_port_map_s {
 	 */
 	{
 		EFX_FAMILY_MEDFORD,
-		(1U << TLV_PORT_MODE_10G_10G_10G_10G_Q) |	/* mode 5 */
+		(1U << TLV_PORT_MODE_2x1_2x1) |			/* mode 5 */
 		/* Do not use 10G_10G_10G_10G_Q1 (see bug63270) */
-		(1U << TLV_PORT_MODE_10G_10G_10G_10G_Q1),	/* mode 4 */
+		(1U << TLV_PORT_MODE_4x1_NA),			/* mode 4 */
 		4,	/* ports per cage */
 		1	/* first cage */
 	},
@@ -1528,7 +1580,7 @@ static struct ef10_external_port_map_s {
 	 */
 	{
 		EFX_FAMILY_MEDFORD,
-		(1U << TLV_PORT_MODE_10G_10G_10G_10G_Q2),	/* mode 8 */
+		(1U << TLV_PORT_MODE_NA_4x1),			/* mode 8 */
 		4,	/* ports per cage */
 		2	/* first cage */
 	},
@@ -1635,13 +1687,14 @@ ef10_external_port_mapping(
 	int32_t count = 1; /* Default 1-1 mapping */
 	int32_t offset = 1; /* Default starting external port number */
 
-	if ((rc = efx_mcdi_get_port_modes(enp, &port_modes, &current)) != 0) {
+	if ((rc = efx_mcdi_get_port_modes(enp, &port_modes, &current,
+		    NULL)) != 0) {
 		/*
 		 * No current port mode information (i.e. Huntington)
 		 * - infer mapping from available modes
 		 */
 		if ((rc = efx_mcdi_get_port_modes(enp,
-			    &port_modes, NULL)) != 0) {
+			    &port_modes, NULL, NULL)) != 0) {
 			/*
 			 * No port mode information available
 			 * - use default mapping
@@ -1781,11 +1834,26 @@ ef10_nic_board_cfg(
 	if ((rc = efx_mcdi_get_phy_cfg(enp)) != 0)
 		goto fail6;
 
+	/*
+	 * Firmware with support for *_FEC capability bits does not
+	 * report that the corresponding *_FEC_REQUESTED bits are supported.
+	 * Add them here so that drivers understand that they are supported.
+	 */
+	if (epp->ep_phy_cap_mask & (1u << EFX_PHY_CAP_BASER_FEC))
+		epp->ep_phy_cap_mask |=
+		    (1u << EFX_PHY_CAP_BASER_FEC_REQUESTED);
+	if (epp->ep_phy_cap_mask & (1u << EFX_PHY_CAP_RS_FEC))
+		epp->ep_phy_cap_mask |=
+		    (1u << EFX_PHY_CAP_RS_FEC_REQUESTED);
+	if (epp->ep_phy_cap_mask & (1u << EFX_PHY_CAP_25G_BASER_FEC))
+		epp->ep_phy_cap_mask |=
+		    (1u << EFX_PHY_CAP_25G_BASER_FEC_REQUESTED);
+
 	/* Obtain the default PHY advertised capabilities */
 	if ((rc = ef10_phy_get_link(enp, &els)) != 0)
 		goto fail7;
-	epp->ep_default_adv_cap_mask = els.els_adv_cap_mask;
-	epp->ep_adv_cap_mask = els.els_adv_cap_mask;
+	epp->ep_default_adv_cap_mask = els.epls.epls_adv_cap_mask;
+	epp->ep_adv_cap_mask = els.epls.epls_adv_cap_mask;
 
 	/* Check capabilities of running datapath firmware */
 	if ((rc = ef10_get_datapath_caps(enp)) != 0)
@@ -2039,8 +2107,8 @@ ef10_nic_reset(
 	__in		efx_nic_t *enp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_ENTITY_RESET_IN_LEN,
-			    MC_CMD_ENTITY_RESET_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_ENTITY_RESET_IN_LEN,
+		MC_CMD_ENTITY_RESET_OUT_LEN);
 	efx_rc_t rc;
 
 	/* ef10_nic_reset() is called to recover from BADASSERT failures. */
@@ -2049,7 +2117,6 @@ ef10_nic_reset(
 	if ((rc = efx_mcdi_exit_assertion_handler(enp)) != 0)
 		goto fail2;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_ENTITY_RESET;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_ENTITY_RESET_IN_LEN;
@@ -2314,6 +2381,36 @@ fail1:
 	return (rc);
 }
 
+	__checkReturn	boolean_t
+ef10_nic_hw_unavailable(
+	__in		efx_nic_t *enp)
+{
+	efx_dword_t dword;
+
+	if (enp->en_reset_flags & EFX_RESET_HW_UNAVAIL)
+		return (B_TRUE);
+
+	EFX_BAR_READD(enp, ER_DZ_BIU_MC_SFT_STATUS_REG, &dword, B_FALSE);
+	if (EFX_DWORD_FIELD(dword, EFX_DWORD_0) == 0xffffffff)
+		goto unavail;
+
+	return (B_FALSE);
+
+unavail:
+	ef10_nic_set_hw_unavailable(enp);
+
+	return (B_TRUE);
+}
+
+			void
+ef10_nic_set_hw_unavailable(
+	__in		efx_nic_t *enp)
+{
+	EFSYS_PROBE(hw_unavail);
+	enp->en_reset_flags |= EFX_RESET_HW_UNAVAIL;
+}
+
+
 			void
 ef10_nic_fini(
 	__in		efx_nic_t *enp)
@@ -2386,11 +2483,10 @@ efx_mcdi_get_nic_global(
 	__out		uint32_t *valuep)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_NIC_GLOBAL_IN_LEN,
-			    MC_CMD_GET_NIC_GLOBAL_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_NIC_GLOBAL_IN_LEN,
+		MC_CMD_GET_NIC_GLOBAL_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_NIC_GLOBAL;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_NIC_GLOBAL_IN_LEN;
@@ -2430,10 +2526,9 @@ efx_mcdi_set_nic_global(
 	__in		uint32_t value)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MC_CMD_SET_NIC_GLOBAL_IN_LEN];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_NIC_GLOBAL_IN_LEN, 0);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_SET_NIC_GLOBAL;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_SET_NIC_GLOBAL_IN_LEN;
diff --git a/drivers/net/sfc/base/ef10_nvram.c b/drivers/net/sfc/base/ef10_nvram.c
index 2883ec8f..8d1b64f2 100644
--- a/drivers/net/sfc/base/ef10_nvram.c
+++ b/drivers/net/sfc/base/ef10_nvram.c
@@ -203,14 +203,14 @@ tlv_validate_state(
 
 	if (tlv_tag(cursor) != TLV_TAG_END) {
 		/* Check current item has space for tag and length */
-		if (cursor->current > (cursor->limit - 2)) {
+		if (cursor->current > (cursor->limit - 1)) {
 			cursor->current = NULL;
 			rc = EFAULT;
 			goto fail3;
 		}
 
-		/* Check we have value data for current item and another tag */
-		if (tlv_next_item_ptr(cursor) > (cursor->limit - 1)) {
+		/* Check we have value data for current item and an END tag */
+		if (tlv_next_item_ptr(cursor) > cursor->limit) {
 			cursor->current = NULL;
 			rc = EFAULT;
 			goto fail4;
@@ -635,7 +635,6 @@ fail1:
 /* Validate buffer contents (before writing to flash) */
 	__checkReturn		efx_rc_t
 ef10_nvram_buffer_validate(
-	__in			efx_nic_t *enp,
 	__in			uint32_t partn,
 	__in_bcount(partn_size)	caddr_t partn_data,
 	__in			size_t partn_size)
@@ -648,7 +647,6 @@ ef10_nvram_buffer_validate(
 	int pos;
 	efx_rc_t rc;
 
-	_NOTE(ARGUNUSED(enp, partn))
 	EFX_STATIC_ASSERT(sizeof (*header) <= EF10_NVRAM_CHUNK);
 
 	if ((partn_data == NULL) || (partn_size == 0)) {
@@ -675,26 +673,32 @@ ef10_nvram_buffer_validate(
 		goto fail4;
 	}
 
+	/* Check partition header matches partn */
+	if (__LE_TO_CPU_16(header->type_id) != partn) {
+		rc = EINVAL;
+		goto fail5;
+	}
+
 	/* Check partition ends with PARTITION_TRAILER and END tags */
 	if ((rc = tlv_find(&cursor, TLV_TAG_PARTITION_TRAILER)) != 0) {
 		rc = EINVAL;
-		goto fail5;
+		goto fail6;
 	}
 	trailer = (struct tlv_partition_trailer *)tlv_item(&cursor);
 
 	if ((rc = tlv_advance(&cursor)) != 0) {
 		rc = EINVAL;
-		goto fail6;
+		goto fail7;
 	}
 	if (tlv_tag(&cursor) != TLV_TAG_END) {
 		rc = EINVAL;
-		goto fail7;
+		goto fail8;
 	}
 
 	/* Check generation counts are consistent */
 	if (trailer->generation != header->generation) {
 		rc = EINVAL;
-		goto fail8;
+		goto fail9;
 	}
 
 	/* Verify partition checksum */
@@ -704,11 +708,13 @@ ef10_nvram_buffer_validate(
 	}
 	if (cksum != 0) {
 		rc = EINVAL;
-		goto fail9;
+		goto fail10;
 	}
 
 	return (0);
 
+fail10:
+	EFSYS_PROBE(fail10);
 fail9:
 	EFSYS_PROBE(fail9);
 fail8:
@@ -731,13 +737,24 @@ fail1:
 	return (rc);
 }
 
+			void
+ef10_nvram_buffer_init(
+	__out_bcount(buffer_size)
+				caddr_t bufferp,
+	__in			size_t buffer_size)
+{
+	uint32_t *buf = (uint32_t *)bufferp;
+
+	memset(buf, 0xff, buffer_size);
 
+	tlv_init_block(buf);
+}
 
 	__checkReturn		efx_rc_t
 ef10_nvram_buffer_create(
-	__in			efx_nic_t *enp,
-	__in			uint16_t partn_type,
-	__in_bcount(partn_size)	caddr_t partn_data,
+	__in			uint32_t partn_type,
+	__out_bcount(partn_size)
+				caddr_t partn_data,
 	__in			size_t partn_size)
 {
 	uint32_t *buf = (uint32_t *)partn_data;
@@ -753,9 +770,8 @@ ef10_nvram_buffer_create(
 		goto fail1;
 	}
 
-	memset(buf, 0xff, partn_size);
+	ef10_nvram_buffer_init(partn_data, partn_size);
 
-	tlv_init_block(buf);
 	if ((rc = tlv_init_cursor(&cursor, buf,
 	    (uint32_t *)((uint8_t *)buf + partn_size),
 	    buf)) != 0) {
@@ -787,7 +803,7 @@ ef10_nvram_buffer_create(
 		goto fail6;
 
 	/* Check that the partition is valid. */
-	if ((rc = ef10_nvram_buffer_validate(enp, partn_type,
+	if ((rc = ef10_nvram_buffer_validate(partn_type,
 	    partn_data, partn_size)) != 0)
 		goto fail7;
 
@@ -959,22 +975,65 @@ ef10_nvram_buffer_find_item(
 }
 
 	__checkReturn		efx_rc_t
+ef10_nvram_buffer_peek_item(
+	__in_bcount(buffer_size)
+				caddr_t bufferp,
+	__in			size_t buffer_size,
+	__in			uint32_t offset,
+	__out			uint32_t *tagp,
+	__out			uint32_t *lengthp,
+	__out			uint32_t *value_offsetp)
+{
+	efx_rc_t rc;
+	tlv_cursor_t cursor;
+	uint32_t tag;
+
+	if ((rc = tlv_init_cursor_at_offset(&cursor, (uint8_t *)bufferp,
+			buffer_size, offset)) != 0) {
+		goto fail1;
+	}
+
+	tag = tlv_tag(&cursor);
+	*tagp = tag;
+	if (tag == TLV_TAG_END) {
+		/*
+		 * To allow stepping over the END tag, report the full tag
+		 * length and a zero length value.
+		 */
+		*lengthp = sizeof (tag);
+		*value_offsetp = sizeof (tag);
+	} else {
+		*lengthp = byte_offset(tlv_next_item_ptr(&cursor),
+			    cursor.current);
+		*value_offsetp = byte_offset((uint32_t *)tlv_value(&cursor),
+			    cursor.current);
+	}
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+	__checkReturn		efx_rc_t
 ef10_nvram_buffer_get_item(
 	__in_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size,
 	__in			uint32_t offset,
 	__in			uint32_t length,
-	__out_bcount_part(item_max_size, *lengthp)
-				caddr_t itemp,
-	__in			size_t item_max_size,
+	__out			uint32_t *tagp,
+	__out_bcount_part(value_max_size, *lengthp)
+				caddr_t valuep,
+	__in			size_t value_max_size,
 	__out			uint32_t *lengthp)
 {
 	efx_rc_t rc;
 	tlv_cursor_t cursor;
-	uint32_t item_length;
+	uint32_t value_length;
 
-	if (item_max_size < length) {
+	if (buffer_size < (offset + length)) {
 		rc = ENOSPC;
 		goto fail1;
 	}
@@ -984,14 +1043,15 @@ ef10_nvram_buffer_get_item(
 		goto fail2;
 	}
 
-	item_length = tlv_length(&cursor);
-	if (length < item_length) {
+	value_length = tlv_length(&cursor);
+	if (value_max_size < value_length) {
 		rc = ENOSPC;
 		goto fail3;
 	}
-	memcpy(itemp, tlv_value(&cursor), item_length);
+	memcpy(valuep, tlv_value(&cursor), value_length);
 
-	*lengthp = item_length;
+	*tagp = tlv_tag(&cursor);
+	*lengthp = value_length;
 
 	return (0);
 
@@ -1011,7 +1071,45 @@ ef10_nvram_buffer_insert_item(
 				caddr_t bufferp,
 	__in			size_t buffer_size,
 	__in			uint32_t offset,
-	__in_bcount(length)	caddr_t keyp,
+	__in			uint32_t tag,
+	__in_bcount(length)	caddr_t valuep,
+	__in			uint32_t length,
+	__out			uint32_t *lengthp)
+{
+	efx_rc_t rc;
+	tlv_cursor_t cursor;
+
+	if ((rc = tlv_init_cursor_at_offset(&cursor, (uint8_t *)bufferp,
+			buffer_size, offset)) != 0) {
+		goto fail1;
+	}
+
+	rc = tlv_insert(&cursor, tag, (uint8_t *)valuep, length);
+
+	if (rc != 0)
+		goto fail2;
+
+	*lengthp = byte_offset(tlv_next_item_ptr(&cursor),
+		    cursor.current);
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+	__checkReturn		efx_rc_t
+ef10_nvram_buffer_modify_item(
+	__in_bcount(buffer_size)
+				caddr_t bufferp,
+	__in			size_t buffer_size,
+	__in			uint32_t offset,
+	__in			uint32_t tag,
+	__in_bcount(length)	caddr_t valuep,
 	__in			uint32_t length,
 	__out			uint32_t *lengthp)
 {
@@ -1023,7 +1121,7 @@ ef10_nvram_buffer_insert_item(
 		goto fail1;
 	}
 
-	rc = tlv_insert(&cursor, TLV_TAG_LICENSE, (uint8_t *)keyp, length);
+	rc = tlv_modify(&cursor, tag, (uint8_t *)valuep, length);
 
 	if (rc != 0) {
 		goto fail2;
@@ -1042,6 +1140,7 @@ fail1:
 	return (rc);
 }
 
+
 	__checkReturn		efx_rc_t
 ef10_nvram_buffer_delete_item(
 	__in_bcount(buffer_size)
@@ -1808,7 +1907,7 @@ ef10_nvram_partn_write_segment_tlv(
 		goto fail7;
 
 	/* Unlock the partition */
-	ef10_nvram_partn_unlock(enp, partn, NULL);
+	(void) ef10_nvram_partn_unlock(enp, partn, NULL);
 
 	EFSYS_KMEM_FREE(enp->en_esip, partn_size, partn_data);
 
@@ -1823,7 +1922,7 @@ fail5:
 fail4:
 	EFSYS_PROBE(fail4);
 
-	ef10_nvram_partn_unlock(enp, partn, NULL);
+	(void) ef10_nvram_partn_unlock(enp, partn, NULL);
 fail3:
 	EFSYS_PROBE(fail3);
 
@@ -2000,7 +2099,7 @@ ef10_nvram_partn_write(
 	__in			efx_nic_t *enp,
 	__in			uint32_t partn,
 	__in			unsigned int offset,
-	__out_bcount(size)	caddr_t data,
+	__in_bcount(size)	caddr_t data,
 	__in			size_t size)
 {
 	size_t chunk;
@@ -2168,6 +2267,8 @@ static ef10_parttbl_entry_t medford2_parttbl[] = {
 	PARTN_MAP_ENTRY(LICENSE,		ALL,	LICENSE),
 	PARTN_MAP_ENTRY(EXPANSION_UEFI,		ALL,	UEFIROM),
 	PARTN_MAP_ENTRY(MUM_FIRMWARE,		ALL,	MUM_FIRMWARE),
+	PARTN_MAP_ENTRY(DYNCONFIG_DEFAULTS,	ALL,	DYNCONFIG_DEFAULTS),
+	PARTN_MAP_ENTRY(ROMCONFIG_DEFAULTS,	ALL,	ROMCONFIG_DEFAULTS),
 };
 
 static	__checkReturn		efx_rc_t
diff --git a/drivers/net/sfc/base/ef10_phy.c b/drivers/net/sfc/base/ef10_phy.c
index 84acb70a..84ccdde5 100644
--- a/drivers/net/sfc/base/ef10_phy.c
+++ b/drivers/net/sfc/base/ef10_phy.c
@@ -98,8 +98,10 @@ mcdi_phy_decode_link_mode(
 	__in		uint32_t link_flags,
 	__in		unsigned int speed,
 	__in		unsigned int fcntl,
+	__in		uint32_t fec,
 	__out		efx_link_mode_t *link_modep,
-	__out		unsigned int *fcntlp)
+	__out		unsigned int *fcntlp,
+	__out		efx_phy_fec_type_t *fecp)
 {
 	boolean_t fd = !!(link_flags &
 		    (1 << MC_CMD_GET_LINK_OUT_FULL_DUPLEX_LBN));
@@ -141,6 +143,22 @@ mcdi_phy_decode_link_mode(
 		EFSYS_PROBE1(mc_pcol_error, int, fcntl);
 		*fcntlp = 0;
 	}
+
+	switch (fec) {
+	case MC_CMD_FEC_NONE:
+		*fecp = EFX_PHY_FEC_NONE;
+		break;
+	case MC_CMD_FEC_BASER:
+		*fecp = EFX_PHY_FEC_BASER;
+		break;
+	case MC_CMD_FEC_RS:
+		*fecp = EFX_PHY_FEC_RS;
+		break;
+	default:
+		EFSYS_PROBE1(mc_pcol_error, int, fec);
+		*fecp = EFX_PHY_FEC_NONE;
+		break;
+	}
 }
 
 
@@ -154,6 +172,7 @@ ef10_phy_link_ev(
 	unsigned int link_flags;
 	unsigned int speed;
 	unsigned int fcntl;
+	efx_phy_fec_type_t fec = MC_CMD_FEC_NONE;
 	efx_link_mode_t link_mode;
 	uint32_t lp_cap_mask;
 
@@ -191,7 +210,8 @@ ef10_phy_link_ev(
 	link_flags = MCDI_EV_FIELD(eqp, LINKCHANGE_LINK_FLAGS);
 	mcdi_phy_decode_link_mode(enp, link_flags, speed,
 				    MCDI_EV_FIELD(eqp, LINKCHANGE_FCNTL),
-				    &link_mode, &fcntl);
+				    MC_CMD_FEC_NONE, &link_mode,
+				    &fcntl, &fec);
 	mcdi_phy_decode_cap(MCDI_EV_FIELD(eqp, LINKCHANGE_LP_CAP),
 			    &lp_cap_mask);
 
@@ -242,16 +262,16 @@ ef10_phy_get_link(
 	__out		ef10_link_state_t *elsp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_LINK_IN_LEN,
-			    MC_CMD_GET_LINK_OUT_LEN)];
+	uint32_t fec;
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_LINK_IN_LEN,
+		MC_CMD_GET_LINK_OUT_V2_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_LINK;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_LINK_IN_LEN;
 	req.emr_out_buf = payload;
-	req.emr_out_length = MC_CMD_GET_LINK_OUT_LEN;
+	req.emr_out_length = MC_CMD_GET_LINK_OUT_V2_LEN;
 
 	efx_mcdi_execute(enp, &req);
 
@@ -266,14 +286,28 @@ ef10_phy_get_link(
 	}
 
 	mcdi_phy_decode_cap(MCDI_OUT_DWORD(req, GET_LINK_OUT_CAP),
-			    &elsp->els_adv_cap_mask);
+			    &elsp->epls.epls_adv_cap_mask);
 	mcdi_phy_decode_cap(MCDI_OUT_DWORD(req, GET_LINK_OUT_LP_CAP),
-			    &elsp->els_lp_cap_mask);
+			    &elsp->epls.epls_lp_cap_mask);
+
+	if (req.emr_out_length_used < MC_CMD_GET_LINK_OUT_V2_LEN)
+		fec = MC_CMD_FEC_NONE;
+	else
+		fec = MCDI_OUT_DWORD(req, GET_LINK_OUT_V2_FEC_TYPE);
 
 	mcdi_phy_decode_link_mode(enp, MCDI_OUT_DWORD(req, GET_LINK_OUT_FLAGS),
 			    MCDI_OUT_DWORD(req, GET_LINK_OUT_LINK_SPEED),
 			    MCDI_OUT_DWORD(req, GET_LINK_OUT_FCNTL),
-			    &elsp->els_link_mode, &elsp->els_fcntl);
+			    fec, &elsp->epls.epls_link_mode,
+			    &elsp->epls.epls_fcntl, &elsp->epls.epls_fec);
+
+	if (req.emr_out_length_used < MC_CMD_GET_LINK_OUT_V2_LEN) {
+		elsp->epls.epls_ld_cap_mask = 0;
+	} else {
+		mcdi_phy_decode_cap(MCDI_OUT_DWORD(req, GET_LINK_OUT_V2_LD_CAP),
+				    &elsp->epls.epls_ld_cap_mask);
+	}
+
 
 #if EFSYS_OPT_LOOPBACK
 	/*
@@ -301,8 +335,8 @@ ef10_phy_reconfigure(
 {
 	efx_port_t *epp = &(enp->en_port);
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_SET_LINK_IN_LEN,
-			    MC_CMD_SET_LINK_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SET_LINK_IN_LEN,
+		MC_CMD_SET_LINK_OUT_LEN);
 	uint32_t cap_mask;
 #if EFSYS_OPT_PHY_LED_CONTROL
 	unsigned int led_mode;
@@ -316,7 +350,6 @@ ef10_phy_reconfigure(
 	if (supported == B_FALSE)
 		goto out;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_SET_LINK;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_SET_LINK_IN_LEN;
@@ -464,12 +497,11 @@ ef10_phy_verify(
 	__in		efx_nic_t *enp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_PHY_STATE_IN_LEN,
-			    MC_CMD_GET_PHY_STATE_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_PHY_STATE_IN_LEN,
+		MC_CMD_GET_PHY_STATE_OUT_LEN);
 	uint32_t state;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_PHY_STATE;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_PHY_STATE_IN_LEN;
@@ -518,6 +550,29 @@ ef10_phy_oui_get(
 	return (ENOTSUP);
 }
 
+	__checkReturn	efx_rc_t
+ef10_phy_link_state_get(
+	__in		efx_nic_t *enp,
+	__out		efx_phy_link_state_t  *eplsp)
+{
+	efx_rc_t rc;
+	ef10_link_state_t els;
+
+	/* Obtain the active link state */
+	if ((rc = ef10_phy_get_link(enp, &els)) != 0)
+		goto fail1;
+
+	*eplsp = els.epls;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+
 #if EFSYS_OPT_PHY_STATS
 
 	__checkReturn				efx_rc_t
@@ -583,22 +638,34 @@ ef10_bist_poll(
 	unsigned long *valuesp,
 	__in			size_t count)
 {
+	/*
+	 * MCDI_CTL_SDU_LEN_MAX_V1 is large enough cover all BIST results,
+	 * whilst not wasting stack.
+	 */
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_POLL_BIST_IN_LEN,
+		MCDI_CTL_SDU_LEN_MAX_V1);
 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_POLL_BIST_IN_LEN,
-			    MCDI_CTL_SDU_LEN_MAX)];
 	uint32_t value_mask = 0;
 	uint32_t result;
 	efx_rc_t rc;
 
+	EFX_STATIC_ASSERT(MC_CMD_POLL_BIST_OUT_LEN <=
+	    MCDI_CTL_SDU_LEN_MAX_V1);
+	EFX_STATIC_ASSERT(MC_CMD_POLL_BIST_OUT_SFT9001_LEN <=
+	    MCDI_CTL_SDU_LEN_MAX_V1);
+	EFX_STATIC_ASSERT(MC_CMD_POLL_BIST_OUT_MRSFP_LEN <=
+	    MCDI_CTL_SDU_LEN_MAX_V1);
+	EFX_STATIC_ASSERT(MC_CMD_POLL_BIST_OUT_MEM_LEN <=
+	    MCDI_CTL_SDU_LEN_MAX_V1);
+
 	_NOTE(ARGUNUSED(type))
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_POLL_BIST;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_POLL_BIST_IN_LEN;
 	req.emr_out_buf = payload;
-	req.emr_out_length = MCDI_CTL_SDU_LEN_MAX;
+	req.emr_out_length = MCDI_CTL_SDU_LEN_MAX_V1;
 
 	efx_mcdi_execute(enp, &req);
 
diff --git a/drivers/net/sfc/base/ef10_rx.c b/drivers/net/sfc/base/ef10_rx.c
index 313a3691..3c8f4f3b 100644
--- a/drivers/net/sfc/base/ef10_rx.c
+++ b/drivers/net/sfc/base/ef10_rx.c
@@ -29,8 +29,8 @@ efx_mcdi_init_rxq(
 {
 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_INIT_RXQ_V3_IN_LEN,
-			    MC_CMD_INIT_RXQ_V3_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_INIT_RXQ_V3_IN_LEN,
+		MC_CMD_INIT_RXQ_V3_OUT_LEN);
 	int npages = EFX_RXQ_NBUFS(ndescs);
 	int i;
 	efx_qword_t *dma_addr;
@@ -73,7 +73,6 @@ efx_mcdi_init_rxq(
 		want_outer_classes = B_FALSE;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_INIT_RXQ;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_INIT_RXQ_V3_IN_LEN;
@@ -146,11 +145,10 @@ efx_mcdi_fini_rxq(
 	__in		uint32_t instance)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
-			    MC_CMD_FINI_RXQ_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_RXQ_IN_LEN,
+		MC_CMD_FINI_RXQ_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_FINI_RXQ;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
@@ -188,8 +186,8 @@ efx_mcdi_rss_context_alloc(
 	__out		uint32_t *rss_contextp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
-			    MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
+		MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
 	uint32_t rss_context;
 	uint32_t context_type;
 	efx_rc_t rc;
@@ -211,7 +209,6 @@ efx_mcdi_rss_context_alloc(
 		goto fail2;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
@@ -274,8 +271,8 @@ efx_mcdi_rss_context_free(
 	__in		uint32_t rss_context)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
-			    MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
+		MC_CMD_RSS_CONTEXT_FREE_OUT_LEN);
 	efx_rc_t rc;
 
 	if (rss_context == EF10_RSS_CONTEXT_INVALID) {
@@ -283,7 +280,6 @@ efx_mcdi_rss_context_free(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
@@ -318,14 +314,9 @@ efx_mcdi_rss_context_set_flags(
 	__in		efx_rx_hash_type_t type)
 {
 	efx_nic_cfg_t *encp = &enp->en_nic_cfg;
-	efx_rx_hash_type_t type_ipv4;
-	efx_rx_hash_type_t type_ipv4_tcp;
-	efx_rx_hash_type_t type_ipv6;
-	efx_rx_hash_type_t type_ipv6_tcp;
-	efx_rx_hash_type_t modes;
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
-			    MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
+		MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN);
 	efx_rc_t rc;
 
 	EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN ==
@@ -350,7 +341,6 @@ efx_mcdi_rss_context_set_flags(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
@@ -360,57 +350,38 @@ efx_mcdi_rss_context_set_flags(
 	MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 	    rss_context);
 
-	type_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) | EFX_RX_HASH(IPV4_TCP, 2TUPLE) |
-		    EFX_RX_HASH(IPV4_UDP, 2TUPLE);
-	type_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE);
-	type_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) | EFX_RX_HASH(IPV6_TCP, 2TUPLE) |
-		    EFX_RX_HASH(IPV6_UDP, 2TUPLE);
-	type_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE);
-
-	/*
-	 * Create a copy of the original hash type.
-	 * The copy will be used to fill in RSS_MODE bits and
-	 * may be cleared beforehand. The original variable
-	 * and, thus, EN bits will remain unaffected.
-	 */
-	modes = type;
-
 	/*
 	 * If the firmware lacks support for additional modes, RSS_MODE
 	 * fields must contain zeros, otherwise the operation will fail.
 	 */
 	if (encp->enc_rx_scale_additional_modes_supported == B_FALSE)
-		modes = 0;
-
-#define	EXTRACT_RSS_MODE(_type, _class)		\
-	(EFX_EXTRACT_NATIVE(_type, 0, 31,	\
-	EFX_LOW_BIT(EFX_RX_CLASS_##_class),	\
-	EFX_HIGH_BIT(EFX_RX_CLASS_##_class)) &	\
-	EFX_MASK32(EFX_RX_CLASS_##_class))
+		type &= EFX_RX_HASH_LEGACY_MASK;
 
 	MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 	    RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
-	    ((type & type_ipv4) == type_ipv4) ? 1 : 0,
+	    (type & EFX_RX_HASH_IPV4) ? 1 : 0,
 	    RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
-	    ((type & type_ipv4_tcp) == type_ipv4_tcp) ? 1 : 0,
+	    (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
 	    RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
-	    ((type & type_ipv6) == type_ipv6) ? 1 : 0,
+	    (type & EFX_RX_HASH_IPV6) ? 1 : 0,
 	    RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
-	    ((type & type_ipv6_tcp) == type_ipv6_tcp) ? 1 : 0,
+	    (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0,
 	    RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE,
-	    EXTRACT_RSS_MODE(modes, IPV4_TCP),
+	    (type >> EFX_RX_CLASS_IPV4_TCP_LBN) &
+	    EFX_MASK32(EFX_RX_CLASS_IPV4_TCP),
 	    RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE,
-	    EXTRACT_RSS_MODE(modes, IPV4_UDP),
+	    (type >> EFX_RX_CLASS_IPV4_UDP_LBN) &
+	    EFX_MASK32(EFX_RX_CLASS_IPV4_UDP),
 	    RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE,
-	    EXTRACT_RSS_MODE(modes, IPV4),
+	    (type >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4),
 	    RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE,
-	    EXTRACT_RSS_MODE(modes, IPV6_TCP),
+	    (type >> EFX_RX_CLASS_IPV6_TCP_LBN) &
+	    EFX_MASK32(EFX_RX_CLASS_IPV6_TCP),
 	    RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE,
-	    EXTRACT_RSS_MODE(modes, IPV6_UDP),
+	    (type >> EFX_RX_CLASS_IPV6_UDP_LBN) &
+	    EFX_MASK32(EFX_RX_CLASS_IPV6_UDP),
 	    RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE,
-	    EXTRACT_RSS_MODE(modes, IPV6));
-
-#undef EXTRACT_RSS_MODE
+	    (type >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6));
 
 	efx_mcdi_execute(enp, &req);
 
@@ -439,8 +410,8 @@ efx_mcdi_rss_context_set_key(
 	__in		size_t n)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
-			    MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
+		MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN);
 	efx_rc_t rc;
 
 	if (rss_context == EF10_RSS_CONTEXT_INVALID) {
@@ -448,7 +419,6 @@ efx_mcdi_rss_context_set_key(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
@@ -496,8 +466,8 @@ efx_mcdi_rss_context_set_table(
 	__in		size_t n)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
-			    MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
+		MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN);
 	uint8_t *req_table;
 	int i, rc;
 
@@ -506,7 +476,6 @@ efx_mcdi_rss_context_set_table(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
diff --git a/drivers/net/sfc/base/ef10_signed_image_layout.h b/drivers/net/sfc/base/ef10_signed_image_layout.h
index a35d1601..c25ffe2f 100644
--- a/drivers/net/sfc/base/ef10_signed_image_layout.h
+++ b/drivers/net/sfc/base/ef10_signed_image_layout.h
@@ -4,6 +4,14 @@
  * All rights reserved.
  */
 
+/*
+ * This is NOT the original source file. Do NOT edit it.
+ * To update the image layout headers, please edit the copy in
+ * the sfregistry repo and then, in that repo,
+ * "make layout_headers" or "make export" to
+ * regenerate and export all types of headers.
+ */
+
 /* These structures define the layouts for the signed firmware image binary
  * saved in NVRAM. The original image is in the Cryptographic message
  * syntax (CMS) format which contains the bootable firmware binary plus the
diff --git a/drivers/net/sfc/base/ef10_tx.c b/drivers/net/sfc/base/ef10_tx.c
index 7d27f710..5f3df42b 100644
--- a/drivers/net/sfc/base/ef10_tx.c
+++ b/drivers/net/sfc/base/ef10_tx.c
@@ -31,8 +31,8 @@ efx_mcdi_init_txq(
 	__in		efsys_mem_t *esmp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_INIT_TXQ_IN_LEN(EFX_TXQ_MAX_BUFS),
-			    MC_CMD_INIT_TXQ_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_INIT_TXQ_IN_LEN(EFX_TXQ_MAX_BUFS),
+		MC_CMD_INIT_TXQ_OUT_LEN);
 	efx_qword_t *dma_addr;
 	uint64_t addr;
 	int npages;
@@ -53,7 +53,6 @@ efx_mcdi_init_txq(
 		goto fail2;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_INIT_TXQ;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_INIT_TXQ_IN_LEN(npages);
@@ -120,11 +119,10 @@ efx_mcdi_fini_txq(
 	__in		uint32_t instance)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_FINI_TXQ_IN_LEN,
-			    MC_CMD_FINI_TXQ_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_TXQ_IN_LEN,
+		MC_CMD_FINI_TXQ_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_FINI_TXQ;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_FINI_TXQ_IN_LEN;
@@ -278,7 +276,7 @@ ef10_tx_qpio_enable(
 
 fail3:
 	EFSYS_PROBE(fail3);
-	ef10_nic_pio_free(enp, etp->et_pio_bufnum, etp->et_pio_blknum);
+	(void) ef10_nic_pio_free(enp, etp->et_pio_bufnum, etp->et_pio_blknum);
 fail2:
 	EFSYS_PROBE(fail2);
 	etp->et_pio_size = 0;
@@ -296,10 +294,12 @@ ef10_tx_qpio_disable(
 
 	if (etp->et_pio_size != 0) {
 		/* Unlink the piobuf from this TXQ */
-		ef10_nic_pio_unlink(enp, etp->et_index);
+		if (ef10_nic_pio_unlink(enp, etp->et_index) != 0)
+			return;
 
 		/* Free the sub-allocated PIO block */
-		ef10_nic_pio_free(enp, etp->et_pio_bufnum, etp->et_pio_blknum);
+		(void) ef10_nic_pio_free(enp, etp->et_pio_bufnum,
+		    etp->et_pio_blknum);
 		etp->et_pio_size = 0;
 		etp->et_pio_write_offset = 0;
 	}
@@ -539,12 +539,9 @@ ef10_tx_qdesc_post(
 {
 	unsigned int added = *addedp;
 	unsigned int i;
-	efx_rc_t rc;
 
-	if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
-		rc = ENOSPC;
-		goto fail1;
-	}
+	if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
+		return (ENOSPC);
 
 	for (i = 0; i < ndescs; i++) {
 		efx_desc_t *edp = &ed[i];
@@ -564,11 +561,6 @@ ef10_tx_qdesc_post(
 
 	*addedp = added;
 	return (0);
-
-fail1:
-	EFSYS_PROBE1(fail1, efx_rc_t, rc);
-
-	return (rc);
 }
 
 	void
@@ -637,22 +629,22 @@ ef10_tx_qdesc_tso2_create(
 
 	EFSYS_ASSERT(count >= EFX_TX_FATSOV2_OPT_NDESCS);
 
-	EFX_POPULATE_QWORD_6(edp[0].ed_eq,
+	EFX_POPULATE_QWORD_5(edp[0].ed_eq,
 			    ESF_DZ_TX_DESC_IS_OPT, 1,
 			    ESF_DZ_TX_OPTION_TYPE,
 			    ESE_DZ_TX_OPTION_DESC_TSO,
 			    ESF_DZ_TX_TSO_OPTION_TYPE,
 			    ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A,
 			    ESF_DZ_TX_TSO_IP_ID, ipv4_id,
-			    ESF_DZ_TX_TSO_OUTER_IPID, outer_ipv4_id,
 			    ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq);
-	EFX_POPULATE_QWORD_4(edp[1].ed_eq,
+	EFX_POPULATE_QWORD_5(edp[1].ed_eq,
 			    ESF_DZ_TX_DESC_IS_OPT, 1,
 			    ESF_DZ_TX_OPTION_TYPE,
 			    ESE_DZ_TX_OPTION_DESC_TSO,
 			    ESF_DZ_TX_TSO_OPTION_TYPE,
 			    ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B,
-			    ESF_DZ_TX_TSO_TCP_MSS, tcp_mss);
+			    ESF_DZ_TX_TSO_TCP_MSS, tcp_mss,
+			    ESF_DZ_TX_TSO_OUTER_IPID, outer_ipv4_id);
 }
 
 	void
diff --git a/drivers/net/sfc/base/efx.h b/drivers/net/sfc/base/efx.h
index 5108b9b1..8e10e893 100644
--- a/drivers/net/sfc/base/efx.h
+++ b/drivers/net/sfc/base/efx.h
@@ -7,6 +7,7 @@
 #ifndef	_SYS_EFX_H
 #define	_SYS_EFX_H
 
+#include "efx_annote.h"
 #include "efsys.h"
 #include "efx_check.h"
 #include "efx_phy_ids.h"
@@ -154,6 +155,14 @@ extern	__checkReturn	efx_rc_t
 efx_nic_reset(
 	__in		efx_nic_t *enp);
 
+extern	__checkReturn	boolean_t
+efx_nic_hw_unavailable(
+	__in		efx_nic_t *enp);
+
+extern			void
+efx_nic_set_hw_unavailable(
+	__in		efx_nic_t *enp);
+
 #if EFSYS_OPT_DIAG
 
 extern	__checkReturn	efx_rc_t
@@ -661,77 +670,74 @@ efx_mon_init(
 #define	EFX_MON_STATS_PAGE_SIZE 0x100
 #define	EFX_MON_MASK_ELEMENT_SIZE 32
 
-/* START MKCONFIG GENERATED MonitorHeaderStatsBlock 400fdb0517af1fca */
+/* START MKCONFIG GENERATED MonitorHeaderStatsBlock 78b65c8d5af9747b */
 typedef enum efx_mon_stat_e {
-	EFX_MON_STAT_2_5V,
-	EFX_MON_STAT_VCCP1,
-	EFX_MON_STAT_VCC,
-	EFX_MON_STAT_5V,
-	EFX_MON_STAT_12V,
-	EFX_MON_STAT_VCCP2,
-	EFX_MON_STAT_EXT_TEMP,
-	EFX_MON_STAT_INT_TEMP,
-	EFX_MON_STAT_AIN1,
-	EFX_MON_STAT_AIN2,
-	EFX_MON_STAT_INT_COOLING,
-	EFX_MON_STAT_EXT_COOLING,
-	EFX_MON_STAT_1V,
-	EFX_MON_STAT_1_2V,
-	EFX_MON_STAT_1_8V,
-	EFX_MON_STAT_3_3V,
-	EFX_MON_STAT_1_2VA,
-	EFX_MON_STAT_VREF,
-	EFX_MON_STAT_VAOE,
+	EFX_MON_STAT_CONTROLLER_TEMP,
+	EFX_MON_STAT_PHY_COMMON_TEMP,
+	EFX_MON_STAT_CONTROLLER_COOLING,
+	EFX_MON_STAT_PHY0_TEMP,
+	EFX_MON_STAT_PHY0_COOLING,
+	EFX_MON_STAT_PHY1_TEMP,
+	EFX_MON_STAT_PHY1_COOLING,
+	EFX_MON_STAT_IN_1V0,
+	EFX_MON_STAT_IN_1V2,
+	EFX_MON_STAT_IN_1V8,
+	EFX_MON_STAT_IN_2V5,
+	EFX_MON_STAT_IN_3V3,
+	EFX_MON_STAT_IN_12V0,
+	EFX_MON_STAT_IN_1V2A,
+	EFX_MON_STAT_IN_VREF,
+	EFX_MON_STAT_OUT_VAOE,
 	EFX_MON_STAT_AOE_TEMP,
 	EFX_MON_STAT_PSU_AOE_TEMP,
 	EFX_MON_STAT_PSU_TEMP,
-	EFX_MON_STAT_FAN0,
-	EFX_MON_STAT_FAN1,
-	EFX_MON_STAT_FAN2,
-	EFX_MON_STAT_FAN3,
-	EFX_MON_STAT_FAN4,
-	EFX_MON_STAT_VAOE_IN,
-	EFX_MON_STAT_IAOE,
-	EFX_MON_STAT_IAOE_IN,
+	EFX_MON_STAT_FAN_0,
+	EFX_MON_STAT_FAN_1,
+	EFX_MON_STAT_FAN_2,
+	EFX_MON_STAT_FAN_3,
+	EFX_MON_STAT_FAN_4,
+	EFX_MON_STAT_IN_VAOE,
+	EFX_MON_STAT_OUT_IAOE,
+	EFX_MON_STAT_IN_IAOE,
 	EFX_MON_STAT_NIC_POWER,
-	EFX_MON_STAT_0_9V,
-	EFX_MON_STAT_I0_9V,
-	EFX_MON_STAT_I1_2V,
-	EFX_MON_STAT_0_9V_ADC,
-	EFX_MON_STAT_INT_TEMP2,
-	EFX_MON_STAT_VREG_TEMP,
-	EFX_MON_STAT_VREG_0_9V_TEMP,
-	EFX_MON_STAT_VREG_1_2V_TEMP,
-	EFX_MON_STAT_INT_VPTAT,
-	EFX_MON_STAT_INT_ADC_TEMP,
-	EFX_MON_STAT_EXT_VPTAT,
-	EFX_MON_STAT_EXT_ADC_TEMP,
+	EFX_MON_STAT_IN_0V9,
+	EFX_MON_STAT_IN_I0V9,
+	EFX_MON_STAT_IN_I1V2,
+	EFX_MON_STAT_IN_0V9_ADC,
+	EFX_MON_STAT_CONTROLLER_2_TEMP,
+	EFX_MON_STAT_VREG_INTERNAL_TEMP,
+	EFX_MON_STAT_VREG_0V9_TEMP,
+	EFX_MON_STAT_VREG_1V2_TEMP,
+	EFX_MON_STAT_CONTROLLER_VPTAT,
+	EFX_MON_STAT_CONTROLLER_INTERNAL_TEMP,
+	EFX_MON_STAT_CONTROLLER_VPTAT_EXTADC,
+	EFX_MON_STAT_CONTROLLER_INTERNAL_TEMP_EXTADC,
 	EFX_MON_STAT_AMBIENT_TEMP,
 	EFX_MON_STAT_AIRFLOW,
 	EFX_MON_STAT_VDD08D_VSS08D_CSR,
 	EFX_MON_STAT_VDD08D_VSS08D_CSR_EXTADC,
 	EFX_MON_STAT_HOTPOINT_TEMP,
-	EFX_MON_STAT_PHY_POWER_SWITCH_PORT0,
-	EFX_MON_STAT_PHY_POWER_SWITCH_PORT1,
+	EFX_MON_STAT_PHY_POWER_PORT0,
+	EFX_MON_STAT_PHY_POWER_PORT1,
 	EFX_MON_STAT_MUM_VCC,
-	EFX_MON_STAT_0V9_A,
-	EFX_MON_STAT_I0V9_A,
-	EFX_MON_STAT_0V9_A_TEMP,
-	EFX_MON_STAT_0V9_B,
-	EFX_MON_STAT_I0V9_B,
-	EFX_MON_STAT_0V9_B_TEMP,
+	EFX_MON_STAT_IN_0V9_A,
+	EFX_MON_STAT_IN_I0V9_A,
+	EFX_MON_STAT_VREG_0V9_A_TEMP,
+	EFX_MON_STAT_IN_0V9_B,
+	EFX_MON_STAT_IN_I0V9_B,
+	EFX_MON_STAT_VREG_0V9_B_TEMP,
 	EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY,
-	EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY_EXT_ADC,
+	EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY_EXTADC,
 	EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY,
-	EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY_EXT_ADC,
+	EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY_EXTADC,
 	EFX_MON_STAT_CONTROLLER_MASTER_VPTAT,
 	EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP,
-	EFX_MON_STAT_CONTROLLER_MASTER_VPTAT_EXT_ADC,
-	EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP_EXT_ADC,
+	EFX_MON_STAT_CONTROLLER_MASTER_VPTAT_EXTADC,
+	EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP_EXTADC,
 	EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT,
 	EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP,
-	EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT_EXT_ADC,
-	EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP_EXT_ADC,
+	EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT_EXTADC,
+	EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP_EXTADC,
 	EFX_MON_STAT_SODIMM_VOUT,
 	EFX_MON_STAT_SODIMM_0_TEMP,
 	EFX_MON_STAT_SODIMM_1_TEMP,
@@ -740,12 +746,12 @@ typedef enum efx_mon_stat_e {
 	EFX_MON_STAT_CONTROLLER_TDIODE_TEMP,
 	EFX_MON_STAT_BOARD_FRONT_TEMP,
 	EFX_MON_STAT_BOARD_BACK_TEMP,
-	EFX_MON_STAT_I1V8,
-	EFX_MON_STAT_I2V5,
-	EFX_MON_STAT_I3V3,
-	EFX_MON_STAT_I12V0,
-	EFX_MON_STAT_1_3V,
-	EFX_MON_STAT_I1V3,
+	EFX_MON_STAT_IN_I1V8,
+	EFX_MON_STAT_IN_I2V5,
+	EFX_MON_STAT_IN_I3V3,
+	EFX_MON_STAT_IN_I12V0,
+	EFX_MON_STAT_IN_1V3,
+	EFX_MON_STAT_IN_I1V3,
 	EFX_MON_NSTATS
 } efx_mon_stat_t;
 
@@ -759,11 +765,40 @@ typedef enum efx_mon_stat_state_e {
 	EFX_MON_STAT_STATE_NO_READING = 4,
 } efx_mon_stat_state_t;
 
+typedef enum efx_mon_stat_unit_e {
+	EFX_MON_STAT_UNIT_UNKNOWN = 0,
+	EFX_MON_STAT_UNIT_BOOL,
+	EFX_MON_STAT_UNIT_TEMP_C,
+	EFX_MON_STAT_UNIT_VOLTAGE_MV,
+	EFX_MON_STAT_UNIT_CURRENT_MA,
+	EFX_MON_STAT_UNIT_POWER_W,
+	EFX_MON_STAT_UNIT_RPM,
+	EFX_MON_NUNITS
+} efx_mon_stat_unit_t;
+
 typedef struct efx_mon_stat_value_s {
-	uint16_t	emsv_value;
-	uint16_t	emsv_state;
+	uint16_t		emsv_value;
+	efx_mon_stat_state_t	emsv_state;
+	efx_mon_stat_unit_t	emsv_unit;
 } efx_mon_stat_value_t;
 
+typedef struct efx_mon_limit_value_s {
+	uint16_t			emlv_warning_min;
+	uint16_t			emlv_warning_max;
+	uint16_t			emlv_fatal_min;
+	uint16_t			emlv_fatal_max;
+} efx_mon_stat_limits_t;
+
+typedef enum efx_mon_stat_portmask_e {
+	EFX_MON_STAT_PORTMAP_NONE = 0,
+	EFX_MON_STAT_PORTMAP_PORT0 = 1,
+	EFX_MON_STAT_PORTMAP_PORT1 = 2,
+	EFX_MON_STAT_PORTMAP_PORT2 = 3,
+	EFX_MON_STAT_PORTMAP_PORT3 = 4,
+	EFX_MON_STAT_PORTMAP_ALL = (-1),
+	EFX_MON_STAT_PORTMAP_UNKNOWN = (-2)
+} efx_mon_stat_portmask_t;
+
 #if EFSYS_OPT_NAMES
 
 extern					const char *
@@ -771,14 +806,39 @@ efx_mon_stat_name(
 	__in				efx_nic_t *enp,
 	__in				efx_mon_stat_t id);
 
+extern					const char *
+efx_mon_stat_description(
+	__in				efx_nic_t *enp,
+	__in				efx_mon_stat_t id);
+
 #endif	/* EFSYS_OPT_NAMES */
 
+extern	__checkReturn			boolean_t
+efx_mon_mcdi_to_efx_stat(
+	__in				int mcdi_index,
+	__out				efx_mon_stat_t *statp);
+
+extern	__checkReturn			boolean_t
+efx_mon_get_stat_unit(
+	__in				efx_mon_stat_t stat,
+	__out				efx_mon_stat_unit_t *unitp);
+
+extern	__checkReturn			boolean_t
+efx_mon_get_stat_portmap(
+	__in				efx_mon_stat_t stat,
+	__out				efx_mon_stat_portmask_t *maskp);
+
 extern	__checkReturn			efx_rc_t
 efx_mon_stats_update(
 	__in				efx_nic_t *enp,
 	__in				efsys_mem_t *esmp,
 	__inout_ecount(EFX_MON_NSTATS)	efx_mon_stat_value_t *values);
 
+extern	__checkReturn			efx_rc_t
+efx_mon_limits_update(
+	__in				efx_nic_t *enp,
+	__inout_ecount(EFX_MON_NSTATS)	efx_mon_stat_limits_t *values);
+
 #endif	/* EFSYS_OPT_MON_STATS */
 
 extern		void
@@ -970,12 +1030,39 @@ efx_phy_media_type_get(
 	__in		efx_nic_t *enp,
 	__out		efx_phy_media_type_t *typep);
 
+/*
+ * 2-wire device address of the base information in accordance with SFF-8472
+ * Diagnostic Monitoring Interface for Optical Transceivers section
+ * 4 Memory Organization.
+ */
+#define	EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_BASE	0xA0
+
+/*
+ * 2-wire device address of the digital diagnostics monitoring interface
+ * in accordance with SFF-8472 Diagnostic Monitoring Interface for Optical
+ * Transceivers section 4 Memory Organization.
+ */
+#define	EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_DDM	0xA2
+
+/*
+ * Hard wired 2-wire device address for QSFP+ in accordance with SFF-8436
+ * QSFP+ 10 Gbs 4X PLUGGABLE TRANSCEIVER section 7.4 Device Addressing and
+ * Operation.
+ */
+#define	EFX_PHY_MEDIA_INFO_DEV_ADDR_QSFP	0xA0
+
+/*
+ * Maximum accessible data offset for PHY module information.
+ */
+#define	EFX_PHY_MEDIA_INFO_MAX_OFFSET		0x100
+
+
 extern	__checkReturn		efx_rc_t
 efx_phy_module_get_info(
 	__in			efx_nic_t *enp,
 	__in			uint8_t dev_addr,
-	__in			uint8_t offset,
-	__in			uint8_t len,
+	__in			size_t offset,
+	__in			size_t len,
 	__out_bcount(len)	uint8_t *data);
 
 #if EFSYS_OPT_PHY_STATS
@@ -1194,6 +1281,7 @@ typedef struct efx_nic_cfg_s {
 	uint32_t		enc_rx_prefix_size;
 	uint32_t		enc_rx_buf_align_start;
 	uint32_t		enc_rx_buf_align_end;
+#if EFSYS_OPT_RX_SCALE
 	uint32_t		enc_rx_scale_max_exclusive_contexts;
 	/*
 	 * Mask of supported hash algorithms.
@@ -1206,6 +1294,7 @@ typedef struct efx_nic_cfg_s {
 	 */
 	boolean_t		enc_rx_scale_l4_hash_supported;
 	boolean_t		enc_rx_scale_additional_modes_supported;
+#endif /* EFSYS_OPT_RX_SCALE */
 #if EFSYS_OPT_LOOPBACK
 	efx_qword_t		enc_loopback_types[EFX_LINK_NMODES];
 #endif	/* EFSYS_OPT_LOOPBACK */
@@ -1240,6 +1329,7 @@ typedef struct efx_nic_cfg_s {
 	boolean_t		enc_bug35388_workaround;
 	boolean_t		enc_bug41750_workaround;
 	boolean_t		enc_bug61265_workaround;
+	boolean_t		enc_bug61297_workaround;
 	boolean_t		enc_rx_batching_enabled;
 	/* Maximum number of descriptors completed in an rx event. */
 	uint32_t		enc_rx_batch_max;
@@ -1483,6 +1573,8 @@ typedef enum efx_nvram_type_e {
 	EFX_NVRAM_LICENSE,
 	EFX_NVRAM_UEFIROM,
 	EFX_NVRAM_MUM_FIRMWARE,
+	EFX_NVRAM_DYNCONFIG_DEFAULTS,
+	EFX_NVRAM_ROMCONFIG_DEFAULTS,
 	EFX_NVRAM_NTYPES,
 } efx_nvram_type_t;
 
@@ -1608,6 +1700,87 @@ efx_bootcfg_write(
 	__in_bcount(size)	uint8_t *data,
 	__in			size_t size);
 
+
+/*
+ * Processing routines for buffers arranged in the DHCP/BOOTP option format
+ * (see https://tools.ietf.org/html/rfc1533)
+ *
+ * Summarising the format: the buffer is a sequence of options. All options
+ * begin with a tag octet, which uniquely identifies the option.  Fixed-
+ * length options without data consist of only a tag octet.  Only options PAD
+ * (0) and END (255) are fixed length.  All other options are variable-length
+ * with a length octet following the tag octet.  The value of the length
+ * octet does not include the two octets specifying the tag and length.  The
+ * length octet is followed by "length" octets of data.
+ *
+ * Option data may be a sequence of sub-options in the same format. The data
+ * content of the encapsulating option is one or more encapsulated sub-options,
+ * with no terminating END tag is required.
+ *
+ * To be valid, the top-level sequence of options should be terminated by an
+ * END tag. The buffer should be padded with the PAD byte.
+ *
+ * When stored to NVRAM, the DHCP option format buffer is preceded by a
+ * checksum octet. The full buffer (including after the END tag) contributes
+ * to the checksum, hence the need to fill the buffer to the end with PAD.
+ */
+
+#define	EFX_DHCP_END ((uint8_t)0xff)
+#define	EFX_DHCP_PAD ((uint8_t)0)
+
+#define	EFX_DHCP_ENCAP_OPT(encapsulator, encapsulated) \
+  (uint16_t)(((encapsulator) << 8) | (encapsulated))
+
+extern	__checkReturn		uint8_t
+efx_dhcp_csum(
+	__in_bcount(size)	uint8_t const *data,
+	__in			size_t size);
+
+extern	__checkReturn		efx_rc_t
+efx_dhcp_verify(
+	__in_bcount(size)	uint8_t const *data,
+	__in			size_t size,
+	__out_opt		size_t *usedp);
+
+extern	__checkReturn	efx_rc_t
+efx_dhcp_find_tag(
+	__in_bcount(buffer_length)	uint8_t *bufferp,
+	__in				size_t buffer_length,
+	__in				uint16_t opt,
+	__deref_out			uint8_t **valuepp,
+	__out				size_t *value_lengthp);
+
+extern	__checkReturn	efx_rc_t
+efx_dhcp_find_end(
+	__in_bcount(buffer_length)	uint8_t *bufferp,
+	__in				size_t buffer_length,
+	__deref_out			uint8_t **endpp);
+
+
+extern	__checkReturn	efx_rc_t
+efx_dhcp_delete_tag(
+	__inout_bcount(buffer_length)	uint8_t *bufferp,
+	__in				size_t buffer_length,
+	__in				uint16_t opt);
+
+extern	__checkReturn	efx_rc_t
+efx_dhcp_add_tag(
+	__inout_bcount(buffer_length)	uint8_t *bufferp,
+	__in				size_t buffer_length,
+	__in				uint16_t opt,
+	__in_bcount_opt(value_length)	uint8_t *valuep,
+	__in				size_t value_length);
+
+extern	__checkReturn	efx_rc_t
+efx_dhcp_update_tag(
+	__inout_bcount(buffer_length)	uint8_t *bufferp,
+	__in				size_t buffer_length,
+	__in				uint16_t opt,
+	__in				uint8_t *value_locationp,
+	__in_bcount_opt(value_length)	uint8_t *valuep,
+	__in				size_t value_length);
+
+
 #endif	/* EFSYS_OPT_BOOTCFG */
 
 #if EFSYS_OPT_IMAGE_LAYOUT
@@ -1689,7 +1862,8 @@ efx_check_reflash_image(
 
 extern	__checkReturn	efx_rc_t
 efx_build_signed_image_write_buffer(
-	__out		uint8_t			*bufferp,
+	__out_bcount(buffer_size)
+			uint8_t			*bufferp,
 	__in		uint32_t		buffer_size,
 	__in		efx_image_info_t	*infop,
 	__out		efx_image_header_t	**headerpp);
@@ -2122,7 +2296,7 @@ typedef enum efx_rx_hash_alg_e {
  *  - a combination of legacy flags
  *  - a combination of EFX_RX_HASH() flags
  */
-typedef unsigned int efx_rx_hash_type_t;
+typedef uint32_t efx_rx_hash_type_t;
 
 typedef enum efx_rx_hash_support_e {
 	EFX_RX_HASH_UNAVAILABLE = 0,	/* Hardware hash not inserted */
@@ -2223,7 +2397,8 @@ extern	__checkReturn				efx_rc_t
 efx_rx_scale_hash_flags_get(
 	__in					efx_nic_t *enp,
 	__in					efx_rx_hash_alg_t hash_alg,
-	__inout_ecount(EFX_RX_HASH_NFLAGS)	unsigned int *flags,
+	__out_ecount_part(max_nflags, *nflagsp)	unsigned int *flagsp,
+	__in					unsigned int max_nflags,
 	__out					unsigned int *nflagsp);
 
 extern	__checkReturn	efx_rc_t
@@ -2813,9 +2988,23 @@ efx_filter_spec_set_encap_type(
 	__in		efx_filter_inner_frame_match_t inner_frame_match);
 
 extern	__checkReturn	efx_rc_t
-efx_filter_spec_set_vxlan_full(
+efx_filter_spec_set_vxlan(
+	__inout		efx_filter_spec_t *spec,
+	__in		const uint8_t *vni,
+	__in		const uint8_t *inner_addr,
+	__in		const uint8_t *outer_addr);
+
+extern	__checkReturn	efx_rc_t
+efx_filter_spec_set_geneve(
+	__inout		efx_filter_spec_t *spec,
+	__in		const uint8_t *vni,
+	__in		const uint8_t *inner_addr,
+	__in		const uint8_t *outer_addr);
+
+extern	__checkReturn	efx_rc_t
+efx_filter_spec_set_nvgre(
 	__inout		efx_filter_spec_t *spec,
-	__in		const uint8_t *vxlan_id,
+	__in		const uint8_t *vsid,
 	__in		const uint8_t *inner_addr,
 	__in		const uint8_t *outer_addr);
 
@@ -3057,6 +3246,32 @@ efx_nic_set_fw_subvariant(
 
 #endif	/* EFSYS_OPT_FW_SUBVARIANT_AWARE */
 
+typedef enum efx_phy_fec_type_e {
+	EFX_PHY_FEC_NONE = 0,
+	EFX_PHY_FEC_BASER,
+	EFX_PHY_FEC_RS
+} efx_phy_fec_type_t;
+
+extern	__checkReturn	efx_rc_t
+efx_phy_fec_type_get(
+	__in		efx_nic_t *enp,
+	__out		efx_phy_fec_type_t *typep);
+
+typedef struct efx_phy_link_state_s {
+	uint32_t		epls_adv_cap_mask;
+	uint32_t		epls_lp_cap_mask;
+	uint32_t		epls_ld_cap_mask;
+	unsigned int		epls_fcntl;
+	efx_phy_fec_type_t	epls_fec;
+	efx_link_mode_t		epls_link_mode;
+} efx_phy_link_state_t;
+
+extern	__checkReturn	efx_rc_t
+efx_phy_link_state_get(
+	__in		efx_nic_t *enp,
+	__out		efx_phy_link_state_t  *eplsp);
+
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/drivers/net/sfc/base/efx_annote.h b/drivers/net/sfc/base/efx_annote.h
new file mode 100644
index 00000000..607b43c7
--- /dev/null
+++ b/drivers/net/sfc/base/efx_annote.h
@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 2018 Solarflare Communications Inc.
+ * All rights reserved.
+ */
+
+#ifndef	_SYS_EFX_ANNOTE_H
+#define	_SYS_EFX_ANNOTE_H
+
+#if defined(_WIN32) || defined(_WIN64)
+#define	EFX_HAVE_WINDOWS_ANNOTATIONS 1
+#else
+#define	EFX_HAVE_WINDOWS_ANNOTATIONS 0
+#endif	/* defined(_WIN32) || defined(_WIN64) */
+
+#if defined(__sun)
+#define	EFX_HAVE_SOLARIS_ANNOTATIONS 1
+#else
+#define	EFX_HAVE_SOLARIS_ANNOTATIONS 0
+#endif	/* defined(__sun) */
+
+#if !EFX_HAVE_WINDOWS_ANNOTATIONS
+
+/* Ignore Windows SAL annotations on other platforms */
+#define	__in
+#define	__in_opt
+#define	__in_ecount(_n)
+#define	__in_ecount_opt(_n)
+#define	__in_bcount(_n)
+#define	__in_bcount_opt(_n)
+
+#define	__out
+#define	__out_opt
+#define	__out_ecount(_n)
+#define	__out_ecount_opt(_n)
+#define	__out_ecount_part(_n, _l)
+#define	__out_bcount(_n)
+#define	__out_bcount_opt(_n)
+#define	__out_bcount_part(_n, _l)
+#define	__out_bcount_part_opt(_n, _l)
+
+#define	__deref_out
+#define	__deref_inout
+
+#define	__inout
+#define	__inout_opt
+#define	__inout_ecount(_n)
+#define	__inout_ecount_opt(_n)
+#define	__inout_bcount(_n)
+#define	__inout_bcount_opt(_n)
+#define	__inout_bcount_full_opt(_n)
+
+#define	__deref_out_bcount_opt(n)
+
+#define	__checkReturn
+#define	__success(_x)
+
+#define	__drv_when(_p, _c)
+
+#endif	/* !EFX_HAVE_WINDOWS_ANNOTATIONS */
+
+#if !EFX_HAVE_SOLARIS_ANNOTATIONS
+
+#if EFX_HAVE_WINDOWS_ANNOTATIONS
+
+/*
+ * Support some SunOS/Solaris style _NOTE() annotations
+ *
+ * At present with the facilities provided in the WDL and the SAL we can only
+ * easily act upon _NOTE(ARGUNUSED(arglist)) annotations.
+ *
+ * Intermediate macros to expand individual _NOTE annotation types into
+ * something the WDK or SAL can understand.  They shouldn't be used directly,
+ * for example EFX_NOTE_ARGUNUSED() is only used as an intermediate step on the
+ * transformation of _NOTE(ARGUNSED(arg1, arg2)) into
+ * UNREFERENCED_PARAMETER((arg1, arg2));
+ */
+#define	EFX_NOTE_ALIGNMENT(_fname, _n)
+#define	EFX_NOTE_ARGUNUSED(...)		UNREFERENCED_PARAMETER((__VA_ARGS__));
+#define	EFX_NOTE_CONSTANTCONDITION
+#define	EFX_NOTE_CONSTCOND
+#define	EFX_NOTE_EMPTY
+#define	EFX_NOTE_FALLTHROUGH
+#define	EFX_NOTE_FALLTHRU
+#define	EFX_NOTE_LINTED(_msg)
+#define	EFX_NOTE_NOTREACHED
+#define	EFX_NOTE_PRINTFLIKE(_n)
+#define	EFX_NOTE_SCANFLIKE(_n)
+#define	EFX_NOTE_VARARGS(_n)
+
+#define	_NOTE(_annotation)		EFX_NOTE_ ## _annotation
+
+#else
+
+/* Ignore Solaris annotations on other platforms */
+
+#define	_NOTE(_annotation)
+
+#endif	/* EFX_HAVE_WINDOWS_ANNOTATIONS */
+
+#endif	/* !EFX_HAVE_SOLARIS_ANNOTATIONS */
+
+#endif	/* _SYS_EFX_ANNOTE_H */
diff --git a/drivers/net/sfc/base/efx_bootcfg.c b/drivers/net/sfc/base/efx_bootcfg.c
index 715e18e8..3b0401e8 100644
--- a/drivers/net/sfc/base/efx_bootcfg.c
+++ b/drivers/net/sfc/base/efx_bootcfg.c
@@ -19,8 +19,33 @@
 #define	BOOTCFG_PER_PF   0x800
 #define	BOOTCFG_PF_COUNT 16
 
-#define	DHCP_END ((uint8_t)0xff)
-#define	DHCP_PAD ((uint8_t)0)
+#define	DHCP_OPT_HAS_VALUE(opt) \
+	(((opt) > EFX_DHCP_PAD) && ((opt) < EFX_DHCP_END))
+
+#define	DHCP_MAX_VALUE 255
+
+#define	DHCP_ENCAPSULATOR(encap_opt) ((encap_opt) >> 8)
+#define	DHCP_ENCAPSULATED(encap_opt) ((encap_opt) & 0xff)
+#define	DHCP_IS_ENCAP_OPT(opt) DHCP_OPT_HAS_VALUE(DHCP_ENCAPSULATOR(opt))
+
+typedef struct efx_dhcp_tag_hdr_s {
+	uint8_t		tag;
+	uint8_t		length;
+} efx_dhcp_tag_hdr_t;
+
+/*
+ * Length calculations for tags with value field. PAD and END
+ * have a fixed length of 1, with no length or value field.
+ */
+#define	DHCP_FULL_TAG_LENGTH(hdr) \
+	(sizeof (efx_dhcp_tag_hdr_t) + (hdr)->length)
+
+#define	DHCP_NEXT_TAG(hdr) \
+	((efx_dhcp_tag_hdr_t *)(((uint8_t *)(hdr)) + \
+	DHCP_FULL_TAG_LENGTH((hdr))))
+
+#define	DHCP_CALC_TAG_LENGTH(payload_len) \
+	((payload_len) + sizeof (efx_dhcp_tag_hdr_t))
 
 
 /* Report the layout of bootcfg sectors in NVRAM partition. */
@@ -110,14 +135,11 @@ fail1:
 }
 
 
-static	__checkReturn		uint8_t
-efx_bootcfg_csum(
-	__in			efx_nic_t *enp,
+	__checkReturn		uint8_t
+efx_dhcp_csum(
 	__in_bcount(size)	uint8_t const *data,
 	__in			size_t size)
 {
-	_NOTE(ARGUNUSED(enp))
-
 	unsigned int pos;
 	uint8_t checksum = 0;
 
@@ -126,9 +148,8 @@ efx_bootcfg_csum(
 	return (checksum);
 }
 
-static	__checkReturn		efx_rc_t
-efx_bootcfg_verify(
-	__in			efx_nic_t *enp,
+	__checkReturn		efx_rc_t
+efx_dhcp_verify(
 	__in_bcount(size)	uint8_t const *data,
 	__in			size_t size,
 	__out_opt		size_t *usedp)
@@ -144,12 +165,12 @@ efx_bootcfg_verify(
 
 		/* Consume tag */
 		tag = data[offset];
-		if (tag == DHCP_END) {
+		if (tag == EFX_DHCP_END) {
 			offset++;
 			used = offset;
 			break;
 		}
-		if (tag == DHCP_PAD) {
+		if (tag == EFX_DHCP_PAD) {
 			offset++;
 			continue;
 		}
@@ -171,8 +192,8 @@ efx_bootcfg_verify(
 		used = offset;
 	}
 
-	/* Checksum the entire sector, including bytes after any DHCP_END */
-	if (efx_bootcfg_csum(enp, data, size) != 0) {
+	/* Checksum the entire sector, including bytes after any EFX_DHCP_END */
+	if (efx_dhcp_csum(data, size) != 0) {
 		rc = EINVAL;
 		goto fail3;
 	}
@@ -193,6 +214,516 @@ fail1:
 }
 
 /*
+ * Walk the entire tag set looking for option. The sought option may be
+ * encapsulated. ENOENT indicates the walk completed without finding the
+ * option. If we run out of buffer during the walk the function will return
+ * ENOSPC.
+ */
+static	efx_rc_t
+efx_dhcp_walk_tags(
+	__deref_inout	uint8_t **tagpp,
+	__inout		size_t *buffer_sizep,
+	__in		uint16_t opt)
+{
+	efx_rc_t rc = 0;
+	boolean_t is_encap = B_FALSE;
+
+	if (DHCP_IS_ENCAP_OPT(opt)) {
+		/*
+		 * Look for the encapsulator and, if found, limit ourselves
+		 * to its payload. If it's not found then the entire tag
+		 * cannot be found, so the encapsulated opt search is
+		 * skipped.
+		 */
+		rc = efx_dhcp_walk_tags(tagpp, buffer_sizep,
+		    DHCP_ENCAPSULATOR(opt));
+		if (rc == 0) {
+			*buffer_sizep = ((efx_dhcp_tag_hdr_t *)*tagpp)->length;
+			(*tagpp) += sizeof (efx_dhcp_tag_hdr_t);
+		}
+		opt = DHCP_ENCAPSULATED(opt);
+		is_encap = B_TRUE;
+	}
+
+	EFSYS_ASSERT(!DHCP_IS_ENCAP_OPT(opt));
+
+	while (rc == 0) {
+		size_t size;
+
+		if (*buffer_sizep == 0) {
+			rc = ENOSPC;
+			goto fail1;
+		}
+
+		if (DHCP_ENCAPSULATED(**tagpp) == opt)
+			break;
+
+		if ((**tagpp) == EFX_DHCP_END) {
+			rc = ENOENT;
+			break;
+		} else if ((**tagpp) == EFX_DHCP_PAD) {
+			size = 1;
+		} else {
+			if (*buffer_sizep < sizeof (efx_dhcp_tag_hdr_t)) {
+				rc = ENOSPC;
+				goto fail2;
+			}
+
+			size =
+			    DHCP_FULL_TAG_LENGTH((efx_dhcp_tag_hdr_t *)*tagpp);
+		}
+
+		if (size > *buffer_sizep) {
+			rc = ENOSPC;
+			goto fail3;
+		}
+
+		(*tagpp) += size;
+		(*buffer_sizep) -= size;
+
+		if ((*buffer_sizep == 0) && is_encap) {
+			/* Search within encapulator tag finished */
+			rc = ENOENT;
+			break;
+		}
+	}
+
+	/*
+	 * Returns 0 if found otherwise ENOENT indicating search finished
+	 * correctly
+	 */
+	return (rc);
+
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+/*
+ * Locate value buffer for option in the given buffer.
+ * Returns 0 if found, ENOENT indicating search finished
+ * correctly, otherwise search failed before completion.
+ */
+	__checkReturn	efx_rc_t
+efx_dhcp_find_tag(
+	__in_bcount(buffer_length)	uint8_t *bufferp,
+	__in				size_t buffer_length,
+	__in				uint16_t opt,
+	__deref_out			uint8_t **valuepp,
+	__out				size_t *value_lengthp)
+{
+	efx_rc_t rc;
+	uint8_t *tagp = bufferp;
+	size_t len = buffer_length;
+
+	rc = efx_dhcp_walk_tags(&tagp, &len, opt);
+	if (rc == 0) {
+		efx_dhcp_tag_hdr_t *hdrp;
+
+		hdrp = (efx_dhcp_tag_hdr_t *)tagp;
+		*valuepp = (uint8_t *)(&hdrp[1]);
+		*value_lengthp = hdrp->length;
+	} else if (rc != ENOENT) {
+		goto fail1;
+	}
+
+	return (rc);
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+/*
+ * Locate the end tag in the given buffer.
+ * Returns 0 if found, ENOENT indicating search finished
+ * correctly but end tag was not found; otherwise search
+ * failed before completion.
+ */
+	__checkReturn	efx_rc_t
+efx_dhcp_find_end(
+	__in_bcount(buffer_length)	uint8_t *bufferp,
+	__in				size_t buffer_length,
+	__deref_out			uint8_t **endpp)
+{
+	efx_rc_t rc;
+	uint8_t *endp = bufferp;
+	size_t len = buffer_length;
+
+	rc = efx_dhcp_walk_tags(&endp, &len, EFX_DHCP_END);
+	if (rc == 0)
+		*endpp = endp;
+	else if (rc != ENOENT)
+		goto fail1;
+
+	return (rc);
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+
+/*
+ * Delete the given tag from anywhere in the buffer. Copes with
+ * encapsulated tags, and updates or deletes the encapsulating opt as
+ * necessary.
+ */
+	__checkReturn	efx_rc_t
+efx_dhcp_delete_tag(
+	__inout_bcount(buffer_length)	uint8_t *bufferp,
+	__in				size_t buffer_length,
+	__in				uint16_t opt)
+{
+	efx_rc_t rc;
+	efx_dhcp_tag_hdr_t *hdrp;
+	size_t len;
+	uint8_t *startp;
+	uint8_t *endp;
+
+	len = buffer_length;
+	startp = bufferp;
+
+	if (!DHCP_OPT_HAS_VALUE(DHCP_ENCAPSULATED(opt))) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	rc = efx_dhcp_walk_tags(&startp, &len, opt);
+	if (rc != 0)
+		goto fail1;
+
+	hdrp = (efx_dhcp_tag_hdr_t *)startp;
+
+	if (DHCP_IS_ENCAP_OPT(opt)) {
+		uint8_t tag_length = DHCP_FULL_TAG_LENGTH(hdrp);
+		uint8_t *encapp = bufferp;
+		efx_dhcp_tag_hdr_t *encap_hdrp;
+
+		len = buffer_length;
+		rc = efx_dhcp_walk_tags(&encapp, &len,
+		    DHCP_ENCAPSULATOR(opt));
+		if (rc != 0)
+			goto fail2;
+
+		encap_hdrp = (efx_dhcp_tag_hdr_t *)encapp;
+		if (encap_hdrp->length > tag_length) {
+			encap_hdrp->length = (uint8_t)(
+			    (size_t)encap_hdrp->length - tag_length);
+		} else {
+			/* delete the encapsulating tag */
+			hdrp = encap_hdrp;
+		}
+	}
+
+	startp = (uint8_t *)hdrp;
+	endp = (uint8_t *)DHCP_NEXT_TAG(hdrp);
+
+	if (startp < bufferp) {
+		rc = EINVAL;
+		goto fail3;
+	}
+
+	if (endp > &bufferp[buffer_length]) {
+		rc = EINVAL;
+		goto fail4;
+	}
+
+	memmove(startp, endp,
+		buffer_length - (endp - bufferp));
+
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+/*
+ * Write the tag header into write_pointp and optionally copies the payload
+ * into the space following.
+ */
+static	void
+efx_dhcp_write_tag(
+	__in		uint8_t *write_pointp,
+	__in		uint16_t opt,
+	__in_bcount_opt(value_length)
+			uint8_t *valuep,
+	__in		size_t value_length)
+{
+	efx_dhcp_tag_hdr_t *hdrp = (efx_dhcp_tag_hdr_t *)write_pointp;
+	hdrp->tag = DHCP_ENCAPSULATED(opt);
+	hdrp->length = (uint8_t)value_length;
+	if ((value_length > 0) && (valuep != NULL))
+		memcpy(&hdrp[1], valuep, value_length);
+}
+
+/*
+ * Add the given tag to the end of the buffer. Copes with creating an
+ * encapsulated tag, and updates or creates the encapsulating opt as
+ * necessary.
+ */
+	__checkReturn	efx_rc_t
+efx_dhcp_add_tag(
+	__inout_bcount(buffer_length)	uint8_t *bufferp,
+	__in				size_t buffer_length,
+	__in				uint16_t opt,
+	__in_bcount_opt(value_length)	uint8_t *valuep,
+	__in				size_t value_length)
+{
+	efx_rc_t rc;
+	efx_dhcp_tag_hdr_t *encap_hdrp = NULL;
+	uint8_t *insert_pointp = NULL;
+	uint8_t *endp;
+	size_t available_space;
+	size_t added_length;
+	size_t search_size;
+	uint8_t *searchp;
+
+	if (!DHCP_OPT_HAS_VALUE(DHCP_ENCAPSULATED(opt))) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if (value_length > DHCP_MAX_VALUE) {
+		rc = EINVAL;
+		goto fail2;
+	}
+
+	if ((value_length > 0) && (valuep == NULL)) {
+		rc = EINVAL;
+		goto fail3;
+	}
+
+	endp = bufferp;
+	available_space = buffer_length;
+	rc = efx_dhcp_walk_tags(&endp, &available_space, EFX_DHCP_END);
+	if (rc != 0)
+		goto fail4;
+
+	searchp = bufferp;
+	search_size = buffer_length;
+	if (DHCP_IS_ENCAP_OPT(opt)) {
+		rc = efx_dhcp_walk_tags(&searchp, &search_size,
+		    DHCP_ENCAPSULATOR(opt));
+		if (rc == 0) {
+			encap_hdrp = (efx_dhcp_tag_hdr_t *)searchp;
+
+			/* Check encapsulated tag is not present */
+			search_size = encap_hdrp->length;
+			rc = efx_dhcp_walk_tags(&searchp, &search_size,
+			    opt);
+			if (rc != ENOENT) {
+				rc = EINVAL;
+				goto fail5;
+			}
+
+			/* Check encapsulator will not overflow */
+			if (((size_t)encap_hdrp->length +
+			    DHCP_CALC_TAG_LENGTH(value_length)) >
+			    DHCP_MAX_VALUE) {
+				rc = E2BIG;
+				goto fail6;
+			}
+
+			/* Insert at start of existing encapsulator */
+			insert_pointp = (uint8_t *)&encap_hdrp[1];
+			opt = DHCP_ENCAPSULATED(opt);
+		} else if (rc == ENOENT) {
+			encap_hdrp = NULL;
+		} else {
+			goto fail7;
+		}
+	} else {
+		/* Check unencapsulated tag is not present */
+		rc = efx_dhcp_walk_tags(&searchp, &search_size,
+		    opt);
+		if (rc != ENOENT) {
+			rc = EINVAL;
+			goto fail8;
+		}
+	}
+
+	if (insert_pointp == NULL) {
+		/* Insert at end of existing tags */
+		insert_pointp = endp;
+	}
+
+	/* Includes the new encapsulator tag hdr if required */
+	added_length = DHCP_CALC_TAG_LENGTH(value_length) +
+	    (DHCP_IS_ENCAP_OPT(opt) ? sizeof (efx_dhcp_tag_hdr_t) : 0);
+
+	if (available_space <= added_length) {
+		rc = ENOMEM;
+		goto fail9;
+	}
+
+	memmove(insert_pointp + added_length, insert_pointp,
+	    available_space - added_length);
+
+	if (DHCP_IS_ENCAP_OPT(opt)) {
+		/* Create new encapsulator header */
+		added_length -= sizeof (efx_dhcp_tag_hdr_t);
+		efx_dhcp_write_tag(insert_pointp,
+		    DHCP_ENCAPSULATOR(opt), NULL, added_length);
+		insert_pointp += sizeof (efx_dhcp_tag_hdr_t);
+	} else if (encap_hdrp)
+		/* Modify existing encapsulator header */
+		encap_hdrp->length +=
+		    ((uint8_t)DHCP_CALC_TAG_LENGTH(value_length));
+
+	efx_dhcp_write_tag(insert_pointp, opt, valuep, value_length);
+
+	return (0);
+
+fail9:
+	EFSYS_PROBE(fail9);
+fail8:
+	EFSYS_PROBE(fail8);
+fail7:
+	EFSYS_PROBE(fail7);
+fail6:
+	EFSYS_PROBE(fail6);
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+/*
+ * Update an existing tag to the new value. Copes with encapsulated
+ * tags, and updates the encapsulating opt as necessary.
+ */
+	__checkReturn	efx_rc_t
+efx_dhcp_update_tag(
+	__inout_bcount(buffer_length)	uint8_t *bufferp,
+	__in				size_t buffer_length,
+	__in				uint16_t opt,
+	__in				uint8_t *value_locationp,
+	__in_bcount_opt(value_length)	uint8_t *valuep,
+	__in				size_t value_length)
+{
+	efx_rc_t rc;
+	uint8_t *write_pointp = value_locationp - sizeof (efx_dhcp_tag_hdr_t);
+	efx_dhcp_tag_hdr_t *hdrp = (efx_dhcp_tag_hdr_t *)write_pointp;
+	efx_dhcp_tag_hdr_t *encap_hdrp = NULL;
+	size_t old_length;
+
+	if (!DHCP_OPT_HAS_VALUE(DHCP_ENCAPSULATED(opt))) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if (value_length > DHCP_MAX_VALUE) {
+		rc = EINVAL;
+		goto fail2;
+	}
+
+	if ((value_length > 0) && (valuep == NULL)) {
+		rc = EINVAL;
+		goto fail3;
+	}
+
+	old_length = hdrp->length;
+
+	if (old_length < value_length) {
+		uint8_t *endp = bufferp;
+		size_t available_space = buffer_length;
+
+		rc = efx_dhcp_walk_tags(&endp, &available_space,
+		    EFX_DHCP_END);
+		if (rc != 0)
+			goto fail4;
+
+		if (available_space < (value_length - old_length)) {
+			rc = EINVAL;
+			goto fail5;
+		}
+	}
+
+	if (DHCP_IS_ENCAP_OPT(opt)) {
+		uint8_t *encapp = bufferp;
+		size_t following_encap = buffer_length;
+		size_t new_length;
+
+		rc = efx_dhcp_walk_tags(&encapp, &following_encap,
+		    DHCP_ENCAPSULATOR(opt));
+		if (rc != 0)
+			goto fail6;
+
+		encap_hdrp = (efx_dhcp_tag_hdr_t *)encapp;
+
+		new_length = ((size_t)encap_hdrp->length +
+		    value_length - old_length);
+		/* Check encapsulator will not overflow */
+		if (new_length > DHCP_MAX_VALUE) {
+			rc = E2BIG;
+			goto fail7;
+		}
+
+		encap_hdrp->length = (uint8_t)new_length;
+	}
+
+	/*
+	 * Move the following data up/down to accomodate the new payload
+	 * length.
+	 */
+	if (old_length != value_length) {
+		uint8_t *destp = (uint8_t *)DHCP_NEXT_TAG(hdrp) +
+		    value_length - old_length;
+		size_t count = &bufferp[buffer_length] -
+		    (uint8_t *)DHCP_NEXT_TAG(hdrp);
+
+		memmove(destp, DHCP_NEXT_TAG(hdrp), count);
+	}
+
+	EFSYS_ASSERT(hdrp->tag == DHCP_ENCAPSULATED(opt));
+	efx_dhcp_write_tag(write_pointp, opt, valuep, value_length);
+
+	return (0);
+
+fail7:
+	EFSYS_PROBE(fail7);
+fail6:
+	EFSYS_PROBE(fail6);
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+
+/*
  * Copy bootcfg sector data to a target buffer which may differ in size.
  * Optionally corrects format errors in source buffer.
  */
@@ -206,17 +737,19 @@ efx_bootcfg_copy_sector(
 	__in			size_t data_size,
 	__in			boolean_t handle_format_errors)
 {
+	_NOTE(ARGUNUSED(enp))
+
 	size_t used_bytes;
 	efx_rc_t rc;
 
-	/* Minimum buffer is checksum byte and DHCP_END terminator */
+	/* Minimum buffer is checksum byte and EFX_DHCP_END terminator */
 	if (data_size < 2) {
 		rc = ENOSPC;
 		goto fail1;
 	}
 
 	/* Verify that the area is correctly formatted and checksummed */
-	rc = efx_bootcfg_verify(enp, sector, sector_length,
+	rc = efx_dhcp_verify(sector, sector_length,
 				    &used_bytes);
 
 	if (!handle_format_errors) {
@@ -224,8 +757,8 @@ efx_bootcfg_copy_sector(
 			goto fail2;
 
 		if ((used_bytes < 2) ||
-		    (sector[used_bytes - 1] != DHCP_END)) {
-			/* Block too short, or DHCP_END missing */
+		    (sector[used_bytes - 1] != EFX_DHCP_END)) {
+			/* Block too short, or EFX_DHCP_END missing */
 			rc = ENOENT;
 			goto fail3;
 		}
@@ -234,24 +767,24 @@ efx_bootcfg_copy_sector(
 	/* Synthesize empty format on verification failure */
 	if (rc != 0 || used_bytes == 0) {
 		sector[0] = 0;
-		sector[1] = DHCP_END;
+		sector[1] = EFX_DHCP_END;
 		used_bytes = 2;
 	}
-	EFSYS_ASSERT(used_bytes >= 2);	/* checksum and DHCP_END */
+	EFSYS_ASSERT(used_bytes >= 2);	/* checksum and EFX_DHCP_END */
 	EFSYS_ASSERT(used_bytes <= sector_length);
 	EFSYS_ASSERT(sector_length >= 2);
 
 	/*
-	 * Legacy bootcfg sectors don't terminate with a DHCP_END character.
-	 * Modify the returned payload so it does.
+	 * Legacy bootcfg sectors don't terminate with an EFX_DHCP_END
+	 * character. Modify the returned payload so it does.
 	 * Reinitialise the sector if there isn't room for the character.
 	 */
-	if (sector[used_bytes - 1] != DHCP_END) {
+	if (sector[used_bytes - 1] != EFX_DHCP_END) {
 		if (used_bytes >= sector_length) {
 			sector[0] = 0;
 			used_bytes = 1;
 		}
-		sector[used_bytes] = DHCP_END;
+		sector[used_bytes] = EFX_DHCP_END;
 		++used_bytes;
 	}
 
@@ -274,10 +807,11 @@ efx_bootcfg_copy_sector(
 		(void) memset(data + used_bytes, 0, data_size - used_bytes);
 
 	/*
-	 * The checksum includes trailing data after any DHCP_END character,
-	 * which we've just modified (by truncation or appending DHCP_END).
+	 * The checksum includes trailing data after any EFX_DHCP_END
+	 * character, which we've just modified (by truncation or appending
+	 * EFX_DHCP_END).
 	 */
-	data[0] -= efx_bootcfg_csum(enp, data, data_size);
+	data[0] -= efx_dhcp_csum(data, data_size);
 
 	return (0);
 
@@ -307,7 +841,7 @@ efx_bootcfg_read(
 	efx_rc_t rc;
 	uint32_t sector_number;
 
-	/* Minimum buffer is checksum byte and DHCP_END terminator */
+	/* Minimum buffer is checksum byte and EFX_DHCP_END terminator */
 	if (size < 2) {
 		rc = ENOSPC;
 		goto fail1;
@@ -343,10 +877,10 @@ efx_bootcfg_read(
 	}
 
 	/*
-	 * We need to read the entire BOOTCFG sector to ensure we read all the
-	 * tags, because legacy bootcfg sectors are not guaranteed to end with
-	 * a DHCP_END character. If the user hasn't supplied a sufficiently
-	 * large buffer then use our own buffer.
+	 * We need to read the entire BOOTCFG sector to ensure we read all
+	 * tags, because legacy bootcfg sectors are not guaranteed to end
+	 * with an EFX_DHCP_END character. If the user hasn't supplied a
+	 * sufficiently large buffer then use our own buffer.
 	 */
 	if (sector_length > size) {
 		EFSYS_KMEM_ALLOC(enp->en_esip, sector_length, payload);
@@ -370,28 +904,29 @@ efx_bootcfg_read(
 		goto fail9;
 
 	/* Verify that the area is correctly formatted and checksummed */
-	rc = efx_bootcfg_verify(enp, payload, sector_length,
+	rc = efx_dhcp_verify(payload, sector_length,
 	    &used_bytes);
 	if (rc != 0 || used_bytes == 0) {
 		payload[0] = 0;
-		payload[1] = DHCP_END;
+		payload[1] = EFX_DHCP_END;
 		used_bytes = 2;
 	}
 
-	EFSYS_ASSERT(used_bytes >= 2);	/* checksum and DHCP_END */
+	EFSYS_ASSERT(used_bytes >= 2);	/* checksum and EFX_DHCP_END */
 	EFSYS_ASSERT(used_bytes <= sector_length);
 
 	/*
-	 * Legacy bootcfg sectors don't terminate with a DHCP_END character.
-	 * Modify the returned payload so it does. BOOTCFG_MAX_SIZE is by
-	 * definition large enough for any valid (per-port) bootcfg sector,
-	 * so reinitialise the sector if there isn't room for the character.
+	 * Legacy bootcfg sectors don't terminate with an EFX_DHCP_END
+	 * character. Modify the returned payload so it does.
+	 * BOOTCFG_MAX_SIZE is by definition large enough for any valid
+	 * (per-port) bootcfg sector, so reinitialise the sector if there
+	 * isn't room for the character.
 	 */
-	if (payload[used_bytes - 1] != DHCP_END) {
+	if (payload[used_bytes - 1] != EFX_DHCP_END) {
 		if (used_bytes >= sector_length)
 			used_bytes = 1;
 
-		payload[used_bytes] = DHCP_END;
+		payload[used_bytes] = EFX_DHCP_END;
 		++used_bytes;
 	}
 
@@ -417,10 +952,10 @@ efx_bootcfg_read(
 		(void) memset(data + used_bytes, 0, size - used_bytes);
 
 	/*
-	 * The checksum includes trailing data after any DHCP_END character,
-	 * which we've just modified (by truncation or appending DHCP_END).
+	 * The checksum includes trailing data after any EFX_DHCP_END character,
+	 * which we've just modified (by truncation or appending EFX_DHCP_END).
 	 */
-	data[0] -= efx_bootcfg_csum(enp, data, size);
+	data[0] -= efx_dhcp_csum(data, size);
 
 	return (0);
 
@@ -490,12 +1025,16 @@ efx_bootcfg_write(
 		goto fail3;
 	}
 
-	if ((rc = efx_bootcfg_verify(enp, data, size, &used_bytes)) != 0)
+	if ((rc = efx_dhcp_verify(data, size, &used_bytes)) != 0)
 		goto fail4;
 
-	/* The caller *must* terminate their block with a DHCP_END character */
-	if ((used_bytes < 2) || ((uint8_t)data[used_bytes - 1] != DHCP_END)) {
-		/* Block too short or DHCP_END missing */
+	/*
+	 * The caller *must* terminate their block with a EFX_DHCP_END
+	 * character
+	 */
+	if ((used_bytes < 2) || ((uint8_t)data[used_bytes - 1] !=
+	    EFX_DHCP_END)) {
+		/* Block too short or EFX_DHCP_END missing */
 		rc = ENOENT;
 		goto fail5;
 	}
@@ -528,13 +1067,13 @@ efx_bootcfg_write(
 		goto fail9;
 
 	/*
-	 * Insert the BOOTCFG sector into the partition, Zero out all data after
-	 * the DHCP_END tag, and adjust the checksum.
+	 * Insert the BOOTCFG sector into the partition, Zero out all data
+	 * after the EFX_DHCP_END tag, and adjust the checksum.
 	 */
 	(void) memset(partn_data + sector_offset, 0x0, sector_length);
 	(void) memcpy(partn_data + sector_offset, data, used_bytes);
 
-	checksum = efx_bootcfg_csum(enp, data, used_bytes);
+	checksum = efx_dhcp_csum(data, used_bytes);
 	partn_data[sector_offset] -= checksum;
 
 	if ((rc = efx_nvram_erase(enp, EFX_NVRAM_BOOTROM_CFG)) != 0)
diff --git a/drivers/net/sfc/base/efx_filter.c b/drivers/net/sfc/base/efx_filter.c
index 412298ac..a7523b38 100644
--- a/drivers/net/sfc/base/efx_filter.c
+++ b/drivers/net/sfc/base/efx_filter.c
@@ -490,27 +490,42 @@ fail1:
 }
 
 /*
- * Specify inner and outer Ethernet address and VXLAN ID in filter
+ * Specify inner and outer Ethernet address and VNI or VSID in tunnel filter
  * specification.
  */
-	__checkReturn	efx_rc_t
-efx_filter_spec_set_vxlan_full(
-	__inout		efx_filter_spec_t *spec,
-	__in		const uint8_t *vxlan_id,
+static	__checkReturn	efx_rc_t
+efx_filter_spec_set_tunnel(
+	__inout	efx_filter_spec_t *spec,
+	__in		efx_tunnel_protocol_t encap_type,
+	__in		const uint8_t *vni_or_vsid,
 	__in		const uint8_t *inner_addr,
 	__in		const uint8_t *outer_addr)
 {
+	efx_rc_t rc;
+
 	EFSYS_ASSERT3P(spec, !=, NULL);
-	EFSYS_ASSERT3P(vxlan_id, !=, NULL);
+	EFSYS_ASSERT3P(vni_or_vsid, !=, NULL);
 	EFSYS_ASSERT3P(inner_addr, !=, NULL);
 	EFSYS_ASSERT3P(outer_addr, !=, NULL);
 
-	if ((inner_addr == NULL) && (outer_addr == NULL))
-		return (EINVAL);
+	switch (encap_type) {
+	case EFX_TUNNEL_PROTOCOL_VXLAN:
+	case EFX_TUNNEL_PROTOCOL_GENEVE:
+	case EFX_TUNNEL_PROTOCOL_NVGRE:
+		break;
+	default:
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	if ((inner_addr == NULL) && (outer_addr == NULL)) {
+		rc = EINVAL;
+		goto fail2;
+	}
 
-	if (vxlan_id != NULL) {
+	if (vni_or_vsid != NULL) {
 		spec->efs_match_flags |= EFX_FILTER_MATCH_VNI_OR_VSID;
-		memcpy(spec->efs_vni_or_vsid, vxlan_id, EFX_VNI_OR_VSID_LEN);
+		memcpy(spec->efs_vni_or_vsid, vni_or_vsid, EFX_VNI_OR_VSID_LEN);
 	}
 	if (outer_addr != NULL) {
 		spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
@@ -520,10 +535,63 @@ efx_filter_spec_set_vxlan_full(
 		spec->efs_match_flags |= EFX_FILTER_MATCH_IFRM_LOC_MAC;
 		memcpy(spec->efs_ifrm_loc_mac, inner_addr, EFX_MAC_ADDR_LEN);
 	}
+
 	spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
-	spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_VXLAN;
+	spec->efs_encap_type = encap_type;
 
 	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+/*
+ * Specify inner and outer Ethernet address and VNI in VXLAN filter
+ * specification.
+ */
+__checkReturn		efx_rc_t
+efx_filter_spec_set_vxlan(
+	__inout		efx_filter_spec_t *spec,
+	__in		const uint8_t *vni,
+	__in		const uint8_t *inner_addr,
+	__in		const uint8_t *outer_addr)
+{
+	return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_VXLAN,
+	    vni, inner_addr, outer_addr);
+}
+
+/*
+ * Specify inner and outer Ethernet address and VNI in Geneve filter
+ * specification.
+ */
+__checkReturn		efx_rc_t
+efx_filter_spec_set_geneve(
+	__inout		efx_filter_spec_t *spec,
+	__in		const uint8_t *vni,
+	__in		const uint8_t *inner_addr,
+	__in		const uint8_t *outer_addr)
+{
+	return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_GENEVE,
+	    vni, inner_addr, outer_addr);
+}
+
+/*
+ * Specify inner and outer Ethernet address and vsid in NVGRE filter
+ * specification.
+ */
+__checkReturn		efx_rc_t
+efx_filter_spec_set_nvgre(
+	__inout		efx_filter_spec_t *spec,
+	__in		const uint8_t *vsid,
+	__in		const uint8_t *inner_addr,
+	__in		const uint8_t *outer_addr)
+{
+	return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_NVGRE,
+	    vsid, inner_addr, outer_addr);
 }
 
 #if EFSYS_OPT_RX_SCALE
diff --git a/drivers/net/sfc/base/efx_impl.h b/drivers/net/sfc/base/efx_impl.h
index 548834f9..bad23f81 100644
--- a/drivers/net/sfc/base/efx_impl.h
+++ b/drivers/net/sfc/base/efx_impl.h
@@ -59,6 +59,7 @@ extern "C" {
 #define	EFX_RESET_PHY		0x00000001
 #define	EFX_RESET_RXQ_ERR	0x00000002
 #define	EFX_RESET_TXQ_ERR	0x00000004
+#define	EFX_RESET_HW_UNAVAIL	0x00000008
 
 typedef enum efx_mac_type_e {
 	EFX_MAC_INVALID = 0,
@@ -223,6 +224,7 @@ typedef struct efx_phy_ops_s {
 	efx_rc_t	(*epo_reconfigure)(efx_nic_t *);
 	efx_rc_t	(*epo_verify)(efx_nic_t *);
 	efx_rc_t	(*epo_oui_get)(efx_nic_t *, uint32_t *);
+	efx_rc_t	(*epo_link_state_get)(efx_nic_t *, efx_phy_link_state_t *);
 #if EFSYS_OPT_PHY_STATS
 	efx_rc_t	(*epo_stats_update)(efx_nic_t *, efsys_mem_t *,
 					    uint32_t *);
@@ -317,6 +319,8 @@ typedef struct efx_mon_ops_s {
 #if EFSYS_OPT_MON_STATS
 	efx_rc_t	(*emo_stats_update)(efx_nic_t *, efsys_mem_t *,
 					    efx_mon_stat_value_t *);
+	efx_rc_t	(*emo_limits_update)(efx_nic_t *,
+					     efx_mon_stat_limits_t *);
 #endif	/* EFSYS_OPT_MON_STATS */
 } efx_mon_ops_t;
 
@@ -354,6 +358,8 @@ typedef struct efx_nic_ops_s {
 	efx_rc_t	(*eno_get_vi_pool)(efx_nic_t *, uint32_t *);
 	efx_rc_t	(*eno_get_bar_region)(efx_nic_t *, efx_nic_region_t,
 					uint32_t *, size_t *);
+	boolean_t	(*eno_hw_unavailable)(efx_nic_t *);
+	void		(*eno_set_hw_unavailable)(efx_nic_t *);
 #if EFSYS_OPT_DIAG
 	efx_rc_t	(*eno_register_test)(efx_nic_t *);
 #endif	/* EFSYS_OPT_DIAG */
@@ -507,7 +513,7 @@ typedef struct efx_nvram_ops_s {
 					    uint32_t *, uint16_t *);
 	efx_rc_t	(*envo_partn_set_version)(efx_nic_t *, uint32_t,
 					    uint16_t *);
-	efx_rc_t	(*envo_buffer_validate)(efx_nic_t *, uint32_t,
+	efx_rc_t	(*envo_buffer_validate)(uint32_t,
 					    caddr_t, size_t);
 } efx_nvram_ops_t;
 #endif /* EFSYS_OPT_NVRAM */
@@ -583,7 +589,7 @@ efx_mcdi_nvram_write(
 	__in			efx_nic_t *enp,
 	__in			uint32_t partn,
 	__in			uint32_t offset,
-	__out_bcount(size)	caddr_t data,
+	__in_bcount(size)	caddr_t data,
 	__in			size_t size);
 
 	__checkReturn		efx_rc_t
diff --git a/drivers/net/sfc/base/efx_lic.c b/drivers/net/sfc/base/efx_lic.c
index 49c00347..4081aef1 100644
--- a/drivers/net/sfc/base/efx_lic.c
+++ b/drivers/net/sfc/base/efx_lic.c
@@ -301,12 +301,11 @@ efx_mcdi_fc_license_update_license(
 	__in		efx_nic_t *enp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MC_CMD_FC_IN_LICENSE_LEN];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FC_IN_LICENSE_LEN, 0);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_FC;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_FC_IN_LICENSE_LEN;
@@ -347,13 +346,12 @@ efx_mcdi_fc_license_get_key_stats(
 	__out		efx_key_stats_t *eksp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_FC_IN_LICENSE_LEN,
-			    MC_CMD_FC_OUT_LICENSE_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FC_IN_LICENSE_LEN,
+		MC_CMD_FC_OUT_LICENSE_LEN);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_FC;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_FC_IN_LICENSE_LEN;
@@ -663,8 +661,8 @@ efx_mcdi_licensed_app_state(
 	__out		boolean_t *licensedp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_LICENSED_APP_STATE_IN_LEN,
-			    MC_CMD_GET_LICENSED_APP_STATE_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_LICENSED_APP_STATE_IN_LEN,
+		MC_CMD_GET_LICENSED_APP_STATE_OUT_LEN);
 	uint32_t app_state;
 	efx_rc_t rc;
 
@@ -676,7 +674,6 @@ efx_mcdi_licensed_app_state(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_LICENSED_APP_STATE;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_LICENSED_APP_STATE_IN_LEN;
@@ -722,12 +719,11 @@ efx_mcdi_licensing_update_licenses(
 	__in		efx_nic_t *enp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MC_CMD_LICENSING_IN_LEN];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_LICENSING_IN_LEN, 0);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_LICENSING;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_LICENSING_IN_LEN;
@@ -765,13 +761,12 @@ efx_mcdi_licensing_get_key_stats(
 	__out		efx_key_stats_t *eksp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_LICENSING_IN_LEN,
-			    MC_CMD_LICENSING_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_LICENSING_IN_LEN,
+		MC_CMD_LICENSING_OUT_LEN);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_LICENSING;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_LICENSING_IN_LEN;
@@ -829,13 +824,12 @@ efx_mcdi_licensing_v3_update_licenses(
 	__in		efx_nic_t *enp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MC_CMD_LICENSING_V3_IN_LEN];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_LICENSING_V3_IN_LEN, 0);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT((enp->en_family == EFX_FAMILY_MEDFORD) ||
 	    (enp->en_family == EFX_FAMILY_MEDFORD2));
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_LICENSING_V3;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_LICENSING_V3_IN_LEN;
@@ -866,14 +860,13 @@ efx_mcdi_licensing_v3_report_license(
 	__out		efx_key_stats_t *eksp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_LICENSING_V3_IN_LEN,
-			    MC_CMD_LICENSING_V3_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_LICENSING_V3_IN_LEN,
+		MC_CMD_LICENSING_V3_OUT_LEN);
 	efx_rc_t rc;
 
 	EFSYS_ASSERT((enp->en_family == EFX_FAMILY_MEDFORD) ||
 	    (enp->en_family == EFX_FAMILY_MEDFORD2));
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_LICENSING_V3;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_LICENSING_V3_IN_LEN;
@@ -930,15 +923,14 @@ efx_mcdi_licensing_v3_app_state(
 	__out		boolean_t *licensedp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_LICENSED_V3_APP_STATE_IN_LEN,
-			    MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_LICENSED_V3_APP_STATE_IN_LEN,
+		MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_LEN);
 	uint32_t app_state;
 	efx_rc_t rc;
 
 	EFSYS_ASSERT((enp->en_family == EFX_FAMILY_MEDFORD) ||
 	    (enp->en_family == EFX_FAMILY_MEDFORD2));
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_LICENSED_V3_APP_STATE;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_LICENSED_V3_APP_STATE_IN_LEN;
@@ -990,28 +982,15 @@ efx_mcdi_licensing_v3_get_id(
 			uint8_t *bufferp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_LICENSING_GET_ID_V3_IN_LEN,
-			    MC_CMD_LICENSING_GET_ID_V3_OUT_LENMIN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_LICENSING_GET_ID_V3_IN_LEN,
+		MC_CMD_LICENSING_GET_ID_V3_OUT_LENMAX);
 	efx_rc_t rc;
 
 	req.emr_cmd = MC_CMD_LICENSING_GET_ID_V3;
-
-	if (bufferp == NULL) {
-		/* Request id type and length only */
-		req.emr_in_buf = bufferp;
-		req.emr_in_length = MC_CMD_LICENSING_GET_ID_V3_IN_LEN;
-		req.emr_out_buf = bufferp;
-		req.emr_out_length = MC_CMD_LICENSING_GET_ID_V3_OUT_LENMIN;
-		(void) memset(payload, 0, sizeof (payload));
-	} else {
-		/* Request full buffer */
-		req.emr_in_buf = bufferp;
-		req.emr_in_length = MC_CMD_LICENSING_GET_ID_V3_IN_LEN;
-		req.emr_out_buf = bufferp;
-		req.emr_out_length =
-		    MIN(buffer_size, MC_CMD_LICENSING_GET_ID_V3_OUT_LENMAX);
-		(void) memset(bufferp, 0, req.emr_out_length);
-	}
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_LICENSING_GET_ID_V3_IN_LEN;
+	req.emr_out_buf = payload;
+	req.emr_out_length = MC_CMD_LICENSING_GET_ID_V3_OUT_LENMAX;
 
 	efx_mcdi_execute_quiet(enp, &req);
 
@@ -1029,19 +1008,10 @@ efx_mcdi_licensing_v3_get_id(
 	*lengthp =
 	    MCDI_OUT_DWORD(req, LICENSING_GET_ID_V3_OUT_LICENSE_ID_LENGTH);
 
-	if (bufferp == NULL) {
-		/*
-		 * Modify length requirements to indicate to caller the extra
-		 * buffering needed to read the complete output.
-		 */
-		*lengthp += MC_CMD_LICENSING_GET_ID_V3_OUT_LENMIN;
-	} else {
-		/* Shift ID down to start of buffer */
-		memmove(bufferp,
-		    bufferp + MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_OFST,
-		    *lengthp);
-		memset(bufferp + (*lengthp), 0,
-		    MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_OFST);
+	if (bufferp != NULL) {
+		memcpy(bufferp,
+		    payload + MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_OFST,
+		    MIN(buffer_size, *lengthp));
 	}
 
 	return (0);
@@ -1158,10 +1128,12 @@ efx_lic_v3_read_key(
 	__in			size_t key_max_size,
 	__out			uint32_t *lengthp)
 {
+	uint32_t tag;
+
 	_NOTE(ARGUNUSED(enp))
 
 	return ef10_nvram_buffer_get_item(bufferp, buffer_size,
-		    offset, length, keyp, key_max_size, lengthp);
+		    offset, length, &tag, keyp, key_max_size, lengthp);
 }
 
 	__checkReturn		efx_rc_t
@@ -1179,7 +1151,7 @@ efx_lic_v3_write_key(
 	EFSYS_ASSERT(length <= EFX_LICENSE_V3_KEY_LENGTH_MAX);
 
 	return ef10_nvram_buffer_insert_item(bufferp, buffer_size,
-		    offset, keyp, length, lengthp);
+		    offset, TLV_TAG_LICENSE, keyp, length, lengthp);
 }
 
 	__checkReturn		efx_rc_t
@@ -1221,8 +1193,10 @@ efx_lic_v3_create_partition(
 {
 	efx_rc_t rc;
 
+	_NOTE(ARGUNUSED(enp))
+
 	/* Construct empty partition */
-	if ((rc = ef10_nvram_buffer_create(enp,
+	if ((rc = ef10_nvram_buffer_create(
 	    NVRAM_PARTITION_TYPE_LICENSE,
 	    bufferp, buffer_size)) != 0) {
 		rc = EFAULT;
@@ -1246,13 +1220,16 @@ efx_lic_v3_finish_partition(
 {
 	efx_rc_t rc;
 
+	_NOTE(ARGUNUSED(enp))
+
 	if ((rc = ef10_nvram_buffer_finish(bufferp,
 			buffer_size)) != 0) {
 		goto fail1;
 	}
 
 	/* Validate completed partition */
-	if ((rc = ef10_nvram_buffer_validate(enp, NVRAM_PARTITION_TYPE_LICENSE,
+	if ((rc = ef10_nvram_buffer_validate(
+					NVRAM_PARTITION_TYPE_LICENSE,
 					bufferp, buffer_size)) != 0) {
 		goto fail2;
 	}
diff --git a/drivers/net/sfc/base/efx_mcdi.c b/drivers/net/sfc/base/efx_mcdi.c
index d4ebcf26..c896aa0b 100644
--- a/drivers/net/sfc/base/efx_mcdi.c
+++ b/drivers/net/sfc/base/efx_mcdi.c
@@ -496,6 +496,12 @@ efx_mcdi_request_poll(
 	EFSYS_ASSERT(!emip->emi_ev_cpl);
 	emrp = emip->emi_pending_req;
 
+	/* Check if hardware is unavailable */
+	if (efx_nic_hw_unavailable(enp)) {
+		EFSYS_UNLOCK(enp->en_eslp, state);
+		return (B_FALSE);
+	}
+
 	/* Check for reboot atomically w.r.t efx_mcdi_request_start */
 	if (emip->emi_poll_cnt++ == 0) {
 		if ((rc = efx_mcdi_poll_reboot(enp)) != 0) {
@@ -900,10 +906,10 @@ efx_mcdi_version(
 	__out_opt		efx_mcdi_boot_t *statusp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MAX(MC_CMD_GET_VERSION_IN_LEN,
-				MC_CMD_GET_VERSION_OUT_LEN),
-			    MAX(MC_CMD_GET_BOOT_STATUS_IN_LEN,
-				MC_CMD_GET_BOOT_STATUS_OUT_LEN))];
+	EFX_MCDI_DECLARE_BUF(payload,
+		MAX(MC_CMD_GET_VERSION_IN_LEN, MC_CMD_GET_BOOT_STATUS_IN_LEN),
+		MAX(MC_CMD_GET_VERSION_OUT_LEN,
+			MC_CMD_GET_BOOT_STATUS_OUT_LEN));
 	efx_word_t *ver_words;
 	uint16_t version[4];
 	uint32_t build;
@@ -912,7 +918,6 @@ efx_mcdi_version(
 
 	EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_VERSION;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_VERSION_IN_LEN;
@@ -1018,12 +1023,11 @@ efx_mcdi_get_capabilities(
 	__out_opt	uint32_t *tso2ncp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_CAPABILITIES_IN_LEN,
-			    MC_CMD_GET_CAPABILITIES_V2_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_CAPABILITIES_IN_LEN,
+		MC_CMD_GET_CAPABILITIES_V2_OUT_LEN);
 	boolean_t v2_capable;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_CAPABILITIES;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_CAPABILITIES_IN_LEN;
@@ -1086,7 +1090,8 @@ efx_mcdi_do_reboot(
 	__in		efx_nic_t *enp,
 	__in		boolean_t after_assertion)
 {
-	uint8_t payload[MAX(MC_CMD_REBOOT_IN_LEN, MC_CMD_REBOOT_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_REBOOT_IN_LEN,
+		MC_CMD_REBOOT_OUT_LEN);
 	efx_mcdi_req_t req;
 	efx_rc_t rc;
 
@@ -1099,7 +1104,6 @@ efx_mcdi_do_reboot(
 	 */
 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_REBOOT;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_REBOOT_IN_LEN;
@@ -1150,8 +1154,8 @@ efx_mcdi_read_assertion(
 	__in		efx_nic_t *enp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_ASSERTS_IN_LEN,
-			    MC_CMD_GET_ASSERTS_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_ASSERTS_IN_LEN,
+		MC_CMD_GET_ASSERTS_OUT_LEN);
 	const char *reason;
 	unsigned int flags;
 	unsigned int index;
@@ -1252,11 +1256,10 @@ efx_mcdi_drv_attach(
 	__in		boolean_t attach)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_DRV_ATTACH_IN_LEN,
-			    MC_CMD_DRV_ATTACH_EXT_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_DRV_ATTACH_IN_LEN,
+		MC_CMD_DRV_ATTACH_EXT_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_DRV_ATTACH;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_DRV_ATTACH_IN_LEN;
@@ -1311,11 +1314,10 @@ efx_mcdi_get_board_cfg(
 {
 	efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip);
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_BOARD_CFG_IN_LEN,
-			    MC_CMD_GET_BOARD_CFG_OUT_LENMIN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_BOARD_CFG_IN_LEN,
+		MC_CMD_GET_BOARD_CFG_OUT_LENMIN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_BOARD_CFG;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_BOARD_CFG_IN_LEN;
@@ -1391,11 +1393,10 @@ efx_mcdi_get_resource_limits(
 	__out_opt	uint32_t *ntxqp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_RESOURCE_LIMITS_IN_LEN,
-			    MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_RESOURCE_LIMITS_IN_LEN,
+		MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_RESOURCE_LIMITS;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_RESOURCE_LIMITS_IN_LEN;
@@ -1438,8 +1439,8 @@ efx_mcdi_get_phy_cfg(
 	efx_port_t *epp = &(enp->en_port);
 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_PHY_CFG_IN_LEN,
-			    MC_CMD_GET_PHY_CFG_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_PHY_CFG_IN_LEN,
+		MC_CMD_GET_PHY_CFG_OUT_LEN);
 #if EFSYS_OPT_NAMES
 	const char *namep;
 	size_t namelen;
@@ -1447,7 +1448,6 @@ efx_mcdi_get_phy_cfg(
 	uint32_t phy_media_type;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_PHY_CFG;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_PHY_CFG_IN_LEN;
@@ -1686,11 +1686,10 @@ efx_mcdi_bist_start(
 	__in			efx_bist_type_t type)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_START_BIST_IN_LEN,
-			    MC_CMD_START_BIST_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_START_BIST_IN_LEN,
+		MC_CMD_START_BIST_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_START_BIST;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_START_BIST_IN_LEN;
@@ -1749,11 +1748,10 @@ efx_mcdi_log_ctrl(
 	__in		efx_nic_t *enp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_LOG_CTRL_IN_LEN,
-			    MC_CMD_LOG_CTRL_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_LOG_CTRL_IN_LEN,
+		MC_CMD_LOG_CTRL_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_LOG_CTRL;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_LOG_CTRL_IN_LEN;
@@ -1798,8 +1796,8 @@ efx_mcdi_mac_stats(
 	__in		uint16_t period_ms)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_MAC_STATS_IN_LEN,
-			    MC_CMD_MAC_STATS_V2_OUT_DMA_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_MAC_STATS_IN_LEN,
+		MC_CMD_MAC_STATS_V2_OUT_DMA_LEN);
 	int clear = (action == EFX_STATS_CLEAR);
 	int upload = (action == EFX_STATS_UPLOAD);
 	int enable = (action == EFX_STATS_ENABLE_NOEVENTS);
@@ -1807,7 +1805,6 @@ efx_mcdi_mac_stats(
 	int disable = (action == EFX_STATS_DISABLE);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_MAC_STATS;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_MAC_STATS_IN_LEN;
@@ -1979,11 +1976,10 @@ efx_mcdi_get_function_info(
 	__out_opt		uint32_t *vfp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_FUNCTION_INFO_IN_LEN,
-			    MC_CMD_GET_FUNCTION_INFO_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_FUNCTION_INFO_IN_LEN,
+		MC_CMD_GET_FUNCTION_INFO_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_FUNCTION_INFO;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_FUNCTION_INFO_IN_LEN;
@@ -2024,11 +2020,10 @@ efx_mcdi_privilege_mask(
 	__out			uint32_t *maskp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_PRIVILEGE_MASK_IN_LEN,
-			    MC_CMD_PRIVILEGE_MASK_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_PRIVILEGE_MASK_IN_LEN,
+		MC_CMD_PRIVILEGE_MASK_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_PRIVILEGE_MASK;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_PRIVILEGE_MASK_IN_LEN;
@@ -2073,11 +2068,10 @@ efx_mcdi_set_workaround(
 	__out_opt		uint32_t *flagsp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_WORKAROUND_IN_LEN,
-			    MC_CMD_WORKAROUND_EXT_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_WORKAROUND_IN_LEN,
+		MC_CMD_WORKAROUND_EXT_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_WORKAROUND;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_WORKAROUND_IN_LEN;
@@ -2117,10 +2111,9 @@ efx_mcdi_get_workarounds(
 	__out_opt		uint32_t *enabledp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MC_CMD_GET_WORKAROUNDS_OUT_LEN];
+	EFX_MCDI_DECLARE_BUF(payload, 0, MC_CMD_GET_WORKAROUNDS_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_WORKAROUNDS;
 	req.emr_in_buf = NULL;
 	req.emr_in_length = 0;
@@ -2157,6 +2150,14 @@ fail1:
  */
 #define	EFX_PHY_MEDIA_INFO_PAGE_SIZE		0x80
 
+/*
+ * Transceiver identifiers from SFF-8024 Table 4-1.
+ */
+#define	EFX_SFF_TRANSCEIVER_ID_SFP		0x03 /* SFP/SFP+/SFP28 */
+#define	EFX_SFF_TRANSCEIVER_ID_QSFP		0x0c /* QSFP */
+#define	EFX_SFF_TRANSCEIVER_ID_QSFP_PLUS	0x0d /* QSFP+ or later */
+#define	EFX_SFF_TRANSCEIVER_ID_QSFP28		0x11 /* QSFP28 or later */
+
 static	__checkReturn		efx_rc_t
 efx_mcdi_get_phy_media_info(
 	__in			efx_nic_t *enp,
@@ -2166,14 +2167,13 @@ efx_mcdi_get_phy_media_info(
 	__out_bcount(len)	uint8_t *data)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN,
-			    MC_CMD_GET_PHY_MEDIA_INFO_OUT_LEN(
-				EFX_PHY_MEDIA_INFO_PAGE_SIZE))];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN,
+		MC_CMD_GET_PHY_MEDIA_INFO_OUT_LEN(
+			EFX_PHY_MEDIA_INFO_PAGE_SIZE));
 	efx_rc_t rc;
 
 	EFSYS_ASSERT((uint32_t)offset + len <= EFX_PHY_MEDIA_INFO_PAGE_SIZE);
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_PHY_MEDIA_INFO;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN;
@@ -2218,39 +2218,19 @@ fail1:
 	return (rc);
 }
 
-/*
- * 2-wire device address of the base information in accordance with SFF-8472
- * Diagnostic Monitoring Interface for Optical Transceivers section
- * 4 Memory Organization.
- */
-#define	EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_BASE	0xA0
-
-/*
- * 2-wire device address of the digital diagnostics monitoring interface
- * in accordance with SFF-8472 Diagnostic Monitoring Interface for Optical
- * Transceivers section 4 Memory Organization.
- */
-#define	EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_DDM	0xA2
-
-/*
- * Hard wired 2-wire device address for QSFP+ in accordance with SFF-8436
- * QSFP+ 10 Gbs 4X PLUGGABLE TRANSCEIVER section 7.4 Device Addressing and
- * Operation.
- */
-#define	EFX_PHY_MEDIA_INFO_DEV_ADDR_QSFP	0xA0
-
 	__checkReturn		efx_rc_t
 efx_mcdi_phy_module_get_info(
 	__in			efx_nic_t *enp,
 	__in			uint8_t dev_addr,
-	__in			uint8_t offset,
-	__in			uint8_t len,
+	__in			size_t offset,
+	__in			size_t len,
 	__out_bcount(len)	uint8_t *data)
 {
 	efx_port_t *epp = &(enp->en_port);
 	efx_rc_t rc;
 	uint32_t mcdi_lower_page;
 	uint32_t mcdi_upper_page;
+	uint8_t id;
 
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 
@@ -2264,6 +2244,26 @@ efx_mcdi_phy_module_get_info(
 	 */
 	switch (epp->ep_fixed_port_type) {
 	case EFX_PHY_MEDIA_SFP_PLUS:
+	case EFX_PHY_MEDIA_QSFP_PLUS:
+		/* Port type supports modules */
+		break;
+	default:
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	/*
+	 * For all supported port types, MCDI page 0 offset 0 holds the
+	 * transceiver identifier. Probe to determine the data layout.
+	 * Definitions from SFF-8024 Table 4-1.
+	 */
+	rc = efx_mcdi_get_phy_media_info(enp,
+		    0, 0, sizeof(id), &id);
+	if (rc != 0)
+		goto fail2;
+
+	switch (id) {
+	case EFX_SFF_TRANSCEIVER_ID_SFP:
 		/*
 		 * In accordance with SFF-8472 Diagnostic Monitoring
 		 * Interface for Optical Transceivers section 4 Memory
@@ -2298,10 +2298,12 @@ efx_mcdi_phy_module_get_info(
 			break;
 		default:
 			rc = ENOTSUP;
-			goto fail1;
+			goto fail3;
 		}
 		break;
-	case EFX_PHY_MEDIA_QSFP_PLUS:
+	case EFX_SFF_TRANSCEIVER_ID_QSFP:
+	case EFX_SFF_TRANSCEIVER_ID_QSFP_PLUS:
+	case EFX_SFF_TRANSCEIVER_ID_QSFP28:
 		switch (dev_addr) {
 		case EFX_PHY_MEDIA_INFO_DEV_ADDR_QSFP:
 			/*
@@ -2317,22 +2319,24 @@ efx_mcdi_phy_module_get_info(
 			break;
 		default:
 			rc = ENOTSUP;
-			goto fail1;
+			goto fail3;
 		}
 		break;
 	default:
 		rc = ENOTSUP;
-		goto fail1;
+		goto fail3;
 	}
 
+	EFX_STATIC_ASSERT(EFX_PHY_MEDIA_INFO_PAGE_SIZE <= 0xFF);
+
 	if (offset < EFX_PHY_MEDIA_INFO_PAGE_SIZE) {
-		uint8_t read_len =
+		size_t read_len =
 		    MIN(len, EFX_PHY_MEDIA_INFO_PAGE_SIZE - offset);
 
 		rc = efx_mcdi_get_phy_media_info(enp,
-		    mcdi_lower_page, offset, read_len, data);
+		    mcdi_lower_page, (uint8_t)offset, (uint8_t)read_len, data);
 		if (rc != 0)
-			goto fail2;
+			goto fail4;
 
 		data += read_len;
 		len -= read_len;
@@ -2347,13 +2351,17 @@ efx_mcdi_phy_module_get_info(
 		EFSYS_ASSERT3U(offset, <, EFX_PHY_MEDIA_INFO_PAGE_SIZE);
 
 		rc = efx_mcdi_get_phy_media_info(enp,
-		    mcdi_upper_page, offset, len, data);
+		    mcdi_upper_page, (uint8_t)offset, (uint8_t)len, data);
 		if (rc != 0)
-			goto fail3;
+			goto fail5;
 	}
 
 	return (0);
 
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
 fail3:
 	EFSYS_PROBE(fail3);
 fail2:
diff --git a/drivers/net/sfc/base/efx_mcdi.h b/drivers/net/sfc/base/efx_mcdi.h
index 253a9e60..ddf91c11 100644
--- a/drivers/net/sfc/base/efx_mcdi.h
+++ b/drivers/net/sfc/base/efx_mcdi.h
@@ -10,6 +10,10 @@
 #include "efx.h"
 #include "efx_regs_mcdi.h"
 
+#if EFSYS_OPT_NAMES
+#include "mc_driver_pcol_strs.h"
+#endif /* EFSYS_OPT_NAMES */
+
 #ifdef	__cplusplus
 extern "C" {
 #endif
@@ -215,8 +219,8 @@ extern	__checkReturn	efx_rc_t
 efx_mcdi_phy_module_get_info(
 	__in			efx_nic_t *enp,
 	__in			uint8_t dev_addr,
-	__in			uint8_t offset,
-	__in			uint8_t len,
+	__in			size_t offset,
+	__in			size_t len,
 	__out_bcount(len)	uint8_t *data);
 
 #define	MCDI_IN(_emr, _type, _ofst)					\
@@ -380,6 +384,17 @@ efx_mcdi_phy_module_get_info(
 	(((mask) & (MC_CMD_PRIVILEGE_MASK_IN_GRP_ ## priv)) ==		\
 	(MC_CMD_PRIVILEGE_MASK_IN_GRP_ ## priv))
 
+/*
+ * The buffer size must be a multiple of dword to ensure that MCDI works
+ * properly with Siena based boards (which use on-chip buffer). Also, it
+ * should be at minimum the size of two dwords to allow space for extended
+ * error responses if the request/response buffer sizes are smaller.
+ */
+#define EFX_MCDI_DECLARE_BUF(_name, _in_len, _out_len)			\
+	uint8_t _name[P2ROUNDUP(MAX(MAX(_in_len, _out_len),		\
+				    (2 * sizeof (efx_dword_t))),	\
+				sizeof (efx_dword_t))] = {0}
+
 typedef enum efx_mcdi_feature_id_e {
 	EFX_MCDI_FEATURE_FW_UPDATE = 0,
 	EFX_MCDI_FEATURE_LINK_CONTROL,
diff --git a/drivers/net/sfc/base/efx_mon.c b/drivers/net/sfc/base/efx_mon.c
index 9fc268ec..f28775d0 100644
--- a/drivers/net/sfc/base/efx_mon.c
+++ b/drivers/net/sfc/base/efx_mon.c
@@ -38,7 +38,8 @@ efx_mon_name(
 #if EFSYS_OPT_MON_MCDI
 static const efx_mon_ops_t	__efx_mon_mcdi_ops = {
 #if EFSYS_OPT_MON_STATS
-	mcdi_mon_stats_update		/* emo_stats_update */
+	mcdi_mon_stats_update,		/* emo_stats_update */
+	mcdi_mon_limits_update,		/* emo_limits_update */
 #endif	/* EFSYS_OPT_MON_STATS */
 };
 #endif
@@ -99,77 +100,74 @@ fail1:
 
 #if EFSYS_OPT_NAMES
 
-/* START MKCONFIG GENERATED MonitorStatNamesBlock 8150a068198c0f96 */
+/* START MKCONFIG GENERATED MonitorStatNamesBlock 277c17eda1a6d1a4 */
 static const char * const __mon_stat_name[] = {
-	"value_2_5v",
-	"value_vccp1",
-	"value_vcc",
-	"value_5v",
-	"value_12v",
-	"value_vccp2",
-	"value_ext_temp",
-	"value_int_temp",
-	"value_ain1",
-	"value_ain2",
+	"controller_temp",
+	"phy_common_temp",
 	"controller_cooling",
-	"ext_cooling",
-	"1v",
-	"1_2v",
-	"1_8v",
-	"3_3v",
-	"1_2va",
-	"vref",
-	"vaoe",
-	"aoe_temperature",
-	"psu_aoe_temperature",
-	"psu_temperature",
-	"fan0",
-	"fan1",
-	"fan2",
-	"fan3",
-	"fan4",
-	"vaoe_in",
-	"iaoe",
-	"iaoe_in",
+	"phy0_temp",
+	"phy0_cooling",
+	"phy1_temp",
+	"phy1_cooling",
+	"in_1v0",
+	"in_1v2",
+	"in_1v8",
+	"in_2v5",
+	"in_3v3",
+	"in_12v0",
+	"in_1v2a",
+	"in_vref",
+	"out_vaoe",
+	"aoe_temp",
+	"psu_aoe_temp",
+	"psu_temp",
+	"fan_0",
+	"fan_1",
+	"fan_2",
+	"fan_3",
+	"fan_4",
+	"in_vaoe",
+	"out_iaoe",
+	"in_iaoe",
 	"nic_power",
-	"0_9v",
-	"i0_9v",
-	"i1_2v",
-	"0_9v_adc",
-	"controller_temperature2",
-	"vreg_temperature",
-	"vreg_0_9v_temperature",
-	"vreg_1_2v_temperature",
-	"int_vptat",
-	"controller_internal_adc_temperature",
-	"ext_vptat",
-	"controller_external_adc_temperature",
-	"ambient_temperature",
+	"in_0v9",
+	"in_i0v9",
+	"in_i1v2",
+	"in_0v9_adc",
+	"controller_2_temp",
+	"vreg_internal_temp",
+	"vreg_0v9_temp",
+	"vreg_1v2_temp",
+	"controller_vptat",
+	"controller_internal_temp",
+	"controller_vptat_extadc",
+	"controller_internal_temp_extadc",
+	"ambient_temp",
 	"airflow",
 	"vdd08d_vss08d_csr",
 	"vdd08d_vss08d_csr_extadc",
-	"hotpoint_temperature",
-	"phy_power_switch_port0",
-	"phy_power_switch_port1",
+	"hotpoint_temp",
+	"phy_power_port0",
+	"phy_power_port1",
 	"mum_vcc",
-	"0v9_a",
-	"i0v9_a",
-	"0v9_a_temp",
-	"0v9_b",
-	"i0v9_b",
-	"0v9_b_temp",
+	"in_0v9_a",
+	"in_i0v9_a",
+	"vreg_0v9_a_temp",
+	"in_0v9_b",
+	"in_i0v9_b",
+	"vreg_0v9_b_temp",
 	"ccom_avreg_1v2_supply",
-	"ccom_avreg_1v2_supply_ext_adc",
+	"ccom_avreg_1v2_supply_extadc",
 	"ccom_avreg_1v8_supply",
-	"ccom_avreg_1v8_supply_ext_adc",
+	"ccom_avreg_1v8_supply_extadc",
 	"controller_master_vptat",
 	"controller_master_internal_temp",
-	"controller_master_vptat_ext_adc",
-	"controller_master_internal_temp_ext_adc",
+	"controller_master_vptat_extadc",
+	"controller_master_internal_temp_extadc",
 	"controller_slave_vptat",
 	"controller_slave_internal_temp",
-	"controller_slave_vptat_ext_adc",
-	"controller_slave_internal_temp_ext_adc",
+	"controller_slave_vptat_extadc",
+	"controller_slave_internal_temp_extadc",
 	"sodimm_vout",
 	"sodimm_0_temp",
 	"sodimm_1_temp",
@@ -178,17 +176,17 @@ static const char * const __mon_stat_name[] = {
 	"controller_tdiode_temp",
 	"board_front_temp",
 	"board_back_temp",
-	"i1v8",
-	"i2v5",
-	"i3v3",
-	"i12v0",
-	"1v3",
-	"i1v3",
+	"in_i1v8",
+	"in_i2v5",
+	"in_i3v3",
+	"in_i12v0",
+	"in_1v3",
+	"in_i1v3",
 };
 
 /* END MKCONFIG GENERATED MonitorStatNamesBlock */
 
-extern					const char *
+					const char *
 efx_mon_stat_name(
 	__in				efx_nic_t *enp,
 	__in				efx_mon_stat_t id)
@@ -200,8 +198,609 @@ efx_mon_stat_name(
 	return (__mon_stat_name[id]);
 }
 
+typedef struct _stat_description_t {
+	efx_mon_stat_t	stat;
+	const char	*desc;
+} stat_description_t;
+
+/* START MKCONFIG GENERATED MonitorStatDescriptionsBlock f072138f16d2e1f8 */
+static const char *__mon_stat_description[] = {
+	MC_CMD_SENSOR_CONTROLLER_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_PHY_COMMON_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_COOLING_ENUM_STR,
+	MC_CMD_SENSOR_PHY0_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_PHY0_COOLING_ENUM_STR,
+	MC_CMD_SENSOR_PHY1_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_PHY1_COOLING_ENUM_STR,
+	MC_CMD_SENSOR_IN_1V0_ENUM_STR,
+	MC_CMD_SENSOR_IN_1V2_ENUM_STR,
+	MC_CMD_SENSOR_IN_1V8_ENUM_STR,
+	MC_CMD_SENSOR_IN_2V5_ENUM_STR,
+	MC_CMD_SENSOR_IN_3V3_ENUM_STR,
+	MC_CMD_SENSOR_IN_12V0_ENUM_STR,
+	MC_CMD_SENSOR_IN_1V2A_ENUM_STR,
+	MC_CMD_SENSOR_IN_VREF_ENUM_STR,
+	MC_CMD_SENSOR_OUT_VAOE_ENUM_STR,
+	MC_CMD_SENSOR_AOE_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_PSU_AOE_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_PSU_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_FAN_0_ENUM_STR,
+	MC_CMD_SENSOR_FAN_1_ENUM_STR,
+	MC_CMD_SENSOR_FAN_2_ENUM_STR,
+	MC_CMD_SENSOR_FAN_3_ENUM_STR,
+	MC_CMD_SENSOR_FAN_4_ENUM_STR,
+	MC_CMD_SENSOR_IN_VAOE_ENUM_STR,
+	MC_CMD_SENSOR_OUT_IAOE_ENUM_STR,
+	MC_CMD_SENSOR_IN_IAOE_ENUM_STR,
+	MC_CMD_SENSOR_NIC_POWER_ENUM_STR,
+	MC_CMD_SENSOR_IN_0V9_ENUM_STR,
+	MC_CMD_SENSOR_IN_I0V9_ENUM_STR,
+	MC_CMD_SENSOR_IN_I1V2_ENUM_STR,
+	MC_CMD_SENSOR_IN_0V9_ADC_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_2_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_VREG_INTERNAL_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_VREG_0V9_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_VREG_1V2_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_VPTAT_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_INTERNAL_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_VPTAT_EXTADC_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_INTERNAL_TEMP_EXTADC_ENUM_STR,
+	MC_CMD_SENSOR_AMBIENT_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_AIRFLOW_ENUM_STR,
+	MC_CMD_SENSOR_VDD08D_VSS08D_CSR_ENUM_STR,
+	MC_CMD_SENSOR_VDD08D_VSS08D_CSR_EXTADC_ENUM_STR,
+	MC_CMD_SENSOR_HOTPOINT_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_PHY_POWER_PORT0_ENUM_STR,
+	MC_CMD_SENSOR_PHY_POWER_PORT1_ENUM_STR,
+	MC_CMD_SENSOR_MUM_VCC_ENUM_STR,
+	MC_CMD_SENSOR_IN_0V9_A_ENUM_STR,
+	MC_CMD_SENSOR_IN_I0V9_A_ENUM_STR,
+	MC_CMD_SENSOR_VREG_0V9_A_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_IN_0V9_B_ENUM_STR,
+	MC_CMD_SENSOR_IN_I0V9_B_ENUM_STR,
+	MC_CMD_SENSOR_VREG_0V9_B_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_CCOM_AVREG_1V2_SUPPLY_ENUM_STR,
+	MC_CMD_SENSOR_CCOM_AVREG_1V2_SUPPLY_EXTADC_ENUM_STR,
+	MC_CMD_SENSOR_CCOM_AVREG_1V8_SUPPLY_ENUM_STR,
+	MC_CMD_SENSOR_CCOM_AVREG_1V8_SUPPLY_EXTADC_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_MASTER_VPTAT_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_MASTER_INTERNAL_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_MASTER_VPTAT_EXTADC_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_MASTER_INTERNAL_TEMP_EXTADC_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_SLAVE_VPTAT_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_SLAVE_INTERNAL_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_SLAVE_VPTAT_EXTADC_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_SLAVE_INTERNAL_TEMP_EXTADC_ENUM_STR,
+	MC_CMD_SENSOR_SODIMM_VOUT_ENUM_STR,
+	MC_CMD_SENSOR_SODIMM_0_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_SODIMM_1_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_PHY0_VCC_ENUM_STR,
+	MC_CMD_SENSOR_PHY1_VCC_ENUM_STR,
+	MC_CMD_SENSOR_CONTROLLER_TDIODE_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_BOARD_FRONT_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_BOARD_BACK_TEMP_ENUM_STR,
+	MC_CMD_SENSOR_IN_I1V8_ENUM_STR,
+	MC_CMD_SENSOR_IN_I2V5_ENUM_STR,
+	MC_CMD_SENSOR_IN_I3V3_ENUM_STR,
+	MC_CMD_SENSOR_IN_I12V0_ENUM_STR,
+	MC_CMD_SENSOR_IN_1V3_ENUM_STR,
+	MC_CMD_SENSOR_IN_I1V3_ENUM_STR,
+};
+
+/* END MKCONFIG GENERATED MonitorStatDescriptionsBlock */
+
+					const char *
+efx_mon_stat_description(
+	__in				efx_nic_t *enp,
+	__in				efx_mon_stat_t id)
+{
+	_NOTE(ARGUNUSED(enp))
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+
+	EFSYS_ASSERT3U(id, <, EFX_MON_NSTATS);
+	return (__mon_stat_description[id]);
+}
+
 #endif	/* EFSYS_OPT_NAMES */
 
+/* START MKCONFIG GENERATED MonitorMcdiMappingBlock 173eee0a5599996a */
+	__checkReturn			boolean_t
+efx_mon_mcdi_to_efx_stat(
+	__in				int mcdi_index,
+	__out				efx_mon_stat_t *statp)
+{
+
+	if ((mcdi_index % (MC_CMD_SENSOR_PAGE0_NEXT + 1)) ==
+	    MC_CMD_SENSOR_PAGE0_NEXT) {
+		*statp = EFX_MON_NSTATS;
+		return (B_FALSE);
+	}
+
+	switch (mcdi_index) {
+	case MC_CMD_SENSOR_IN_I0V9:
+		*statp = EFX_MON_STAT_IN_I0V9;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_SLAVE_VPTAT_EXTADC:
+		*statp = EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT_EXTADC;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_SLAVE_VPTAT:
+		*statp = EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT;
+		break;
+	case MC_CMD_SENSOR_PSU_TEMP:
+		*statp = EFX_MON_STAT_PSU_TEMP;
+		break;
+	case MC_CMD_SENSOR_FAN_2:
+		*statp = EFX_MON_STAT_FAN_2;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_INTERNAL_TEMP_EXTADC:
+		*statp = EFX_MON_STAT_CONTROLLER_INTERNAL_TEMP_EXTADC;
+		break;
+	case MC_CMD_SENSOR_BOARD_BACK_TEMP:
+		*statp = EFX_MON_STAT_BOARD_BACK_TEMP;
+		break;
+	case MC_CMD_SENSOR_IN_1V3:
+		*statp = EFX_MON_STAT_IN_1V3;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_TDIODE_TEMP:
+		*statp = EFX_MON_STAT_CONTROLLER_TDIODE_TEMP;
+		break;
+	case MC_CMD_SENSOR_IN_2V5:
+		*statp = EFX_MON_STAT_IN_2V5;
+		break;
+	case MC_CMD_SENSOR_PHY_COMMON_TEMP:
+		*statp = EFX_MON_STAT_PHY_COMMON_TEMP;
+		break;
+	case MC_CMD_SENSOR_PHY1_TEMP:
+		*statp = EFX_MON_STAT_PHY1_TEMP;
+		break;
+	case MC_CMD_SENSOR_VREG_INTERNAL_TEMP:
+		*statp = EFX_MON_STAT_VREG_INTERNAL_TEMP;
+		break;
+	case MC_CMD_SENSOR_IN_1V0:
+		*statp = EFX_MON_STAT_IN_1V0;
+		break;
+	case MC_CMD_SENSOR_FAN_1:
+		*statp = EFX_MON_STAT_FAN_1;
+		break;
+	case MC_CMD_SENSOR_IN_1V2:
+		*statp = EFX_MON_STAT_IN_1V2;
+		break;
+	case MC_CMD_SENSOR_FAN_3:
+		*statp = EFX_MON_STAT_FAN_3;
+		break;
+	case MC_CMD_SENSOR_IN_1V2A:
+		*statp = EFX_MON_STAT_IN_1V2A;
+		break;
+	case MC_CMD_SENSOR_SODIMM_0_TEMP:
+		*statp = EFX_MON_STAT_SODIMM_0_TEMP;
+		break;
+	case MC_CMD_SENSOR_IN_1V8:
+		*statp = EFX_MON_STAT_IN_1V8;
+		break;
+	case MC_CMD_SENSOR_IN_VREF:
+		*statp = EFX_MON_STAT_IN_VREF;
+		break;
+	case MC_CMD_SENSOR_SODIMM_VOUT:
+		*statp = EFX_MON_STAT_SODIMM_VOUT;
+		break;
+	case MC_CMD_SENSOR_CCOM_AVREG_1V2_SUPPLY:
+		*statp = EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY;
+		break;
+	case MC_CMD_SENSOR_IN_I1V2:
+		*statp = EFX_MON_STAT_IN_I1V2;
+		break;
+	case MC_CMD_SENSOR_IN_I1V3:
+		*statp = EFX_MON_STAT_IN_I1V3;
+		break;
+	case MC_CMD_SENSOR_AIRFLOW:
+		*statp = EFX_MON_STAT_AIRFLOW;
+		break;
+	case MC_CMD_SENSOR_HOTPOINT_TEMP:
+		*statp = EFX_MON_STAT_HOTPOINT_TEMP;
+		break;
+	case MC_CMD_SENSOR_VDD08D_VSS08D_CSR:
+		*statp = EFX_MON_STAT_VDD08D_VSS08D_CSR;
+		break;
+	case MC_CMD_SENSOR_AOE_TEMP:
+		*statp = EFX_MON_STAT_AOE_TEMP;
+		break;
+	case MC_CMD_SENSOR_IN_I1V8:
+		*statp = EFX_MON_STAT_IN_I1V8;
+		break;
+	case MC_CMD_SENSOR_IN_I2V5:
+		*statp = EFX_MON_STAT_IN_I2V5;
+		break;
+	case MC_CMD_SENSOR_PHY1_COOLING:
+		*statp = EFX_MON_STAT_PHY1_COOLING;
+		break;
+	case MC_CMD_SENSOR_CCOM_AVREG_1V8_SUPPLY_EXTADC:
+		*statp = EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY_EXTADC;
+		break;
+	case MC_CMD_SENSOR_IN_0V9_ADC:
+		*statp = EFX_MON_STAT_IN_0V9_ADC;
+		break;
+	case MC_CMD_SENSOR_VREG_0V9_A_TEMP:
+		*statp = EFX_MON_STAT_VREG_0V9_A_TEMP;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_MASTER_VPTAT:
+		*statp = EFX_MON_STAT_CONTROLLER_MASTER_VPTAT;
+		break;
+	case MC_CMD_SENSOR_PHY0_VCC:
+		*statp = EFX_MON_STAT_PHY0_VCC;
+		break;
+	case MC_CMD_SENSOR_PHY0_COOLING:
+		*statp = EFX_MON_STAT_PHY0_COOLING;
+		break;
+	case MC_CMD_SENSOR_PSU_AOE_TEMP:
+		*statp = EFX_MON_STAT_PSU_AOE_TEMP;
+		break;
+	case MC_CMD_SENSOR_VREG_0V9_TEMP:
+		*statp = EFX_MON_STAT_VREG_0V9_TEMP;
+		break;
+	case MC_CMD_SENSOR_IN_I0V9_A:
+		*statp = EFX_MON_STAT_IN_I0V9_A;
+		break;
+	case MC_CMD_SENSOR_IN_I3V3:
+		*statp = EFX_MON_STAT_IN_I3V3;
+		break;
+	case MC_CMD_SENSOR_BOARD_FRONT_TEMP:
+		*statp = EFX_MON_STAT_BOARD_FRONT_TEMP;
+		break;
+	case MC_CMD_SENSOR_OUT_VAOE:
+		*statp = EFX_MON_STAT_OUT_VAOE;
+		break;
+	case MC_CMD_SENSOR_VDD08D_VSS08D_CSR_EXTADC:
+		*statp = EFX_MON_STAT_VDD08D_VSS08D_CSR_EXTADC;
+		break;
+	case MC_CMD_SENSOR_IN_I12V0:
+		*statp = EFX_MON_STAT_IN_I12V0;
+		break;
+	case MC_CMD_SENSOR_PHY_POWER_PORT1:
+		*statp = EFX_MON_STAT_PHY_POWER_PORT1;
+		break;
+	case MC_CMD_SENSOR_PHY_POWER_PORT0:
+		*statp = EFX_MON_STAT_PHY_POWER_PORT0;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_SLAVE_INTERNAL_TEMP_EXTADC:
+		*statp = EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP_EXTADC;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_MASTER_INTERNAL_TEMP:
+		*statp = EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_TEMP:
+		*statp = EFX_MON_STAT_CONTROLLER_TEMP;
+		break;
+	case MC_CMD_SENSOR_IN_IAOE:
+		*statp = EFX_MON_STAT_IN_IAOE;
+		break;
+	case MC_CMD_SENSOR_IN_VAOE:
+		*statp = EFX_MON_STAT_IN_VAOE;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_MASTER_VPTAT_EXTADC:
+		*statp = EFX_MON_STAT_CONTROLLER_MASTER_VPTAT_EXTADC;
+		break;
+	case MC_CMD_SENSOR_CCOM_AVREG_1V8_SUPPLY:
+		*statp = EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY;
+		break;
+	case MC_CMD_SENSOR_PHY1_VCC:
+		*statp = EFX_MON_STAT_PHY1_VCC;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_COOLING:
+		*statp = EFX_MON_STAT_CONTROLLER_COOLING;
+		break;
+	case MC_CMD_SENSOR_AMBIENT_TEMP:
+		*statp = EFX_MON_STAT_AMBIENT_TEMP;
+		break;
+	case MC_CMD_SENSOR_IN_3V3:
+		*statp = EFX_MON_STAT_IN_3V3;
+		break;
+	case MC_CMD_SENSOR_PHY0_TEMP:
+		*statp = EFX_MON_STAT_PHY0_TEMP;
+		break;
+	case MC_CMD_SENSOR_SODIMM_1_TEMP:
+		*statp = EFX_MON_STAT_SODIMM_1_TEMP;
+		break;
+	case MC_CMD_SENSOR_MUM_VCC:
+		*statp = EFX_MON_STAT_MUM_VCC;
+		break;
+	case MC_CMD_SENSOR_VREG_0V9_B_TEMP:
+		*statp = EFX_MON_STAT_VREG_0V9_B_TEMP;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_SLAVE_INTERNAL_TEMP:
+		*statp = EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP;
+		break;
+	case MC_CMD_SENSOR_FAN_4:
+		*statp = EFX_MON_STAT_FAN_4;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_2_TEMP:
+		*statp = EFX_MON_STAT_CONTROLLER_2_TEMP;
+		break;
+	case MC_CMD_SENSOR_CCOM_AVREG_1V2_SUPPLY_EXTADC:
+		*statp = EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY_EXTADC;
+		break;
+	case MC_CMD_SENSOR_IN_0V9_A:
+		*statp = EFX_MON_STAT_IN_0V9_A;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_VPTAT_EXTADC:
+		*statp = EFX_MON_STAT_CONTROLLER_VPTAT_EXTADC;
+		break;
+	case MC_CMD_SENSOR_IN_0V9:
+		*statp = EFX_MON_STAT_IN_0V9;
+		break;
+	case MC_CMD_SENSOR_IN_I0V9_B:
+		*statp = EFX_MON_STAT_IN_I0V9_B;
+		break;
+	case MC_CMD_SENSOR_NIC_POWER:
+		*statp = EFX_MON_STAT_NIC_POWER;
+		break;
+	case MC_CMD_SENSOR_IN_12V0:
+		*statp = EFX_MON_STAT_IN_12V0;
+		break;
+	case MC_CMD_SENSOR_OUT_IAOE:
+		*statp = EFX_MON_STAT_OUT_IAOE;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_VPTAT:
+		*statp = EFX_MON_STAT_CONTROLLER_VPTAT;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_MASTER_INTERNAL_TEMP_EXTADC:
+		*statp = EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP_EXTADC;
+		break;
+	case MC_CMD_SENSOR_CONTROLLER_INTERNAL_TEMP:
+		*statp = EFX_MON_STAT_CONTROLLER_INTERNAL_TEMP;
+		break;
+	case MC_CMD_SENSOR_FAN_0:
+		*statp = EFX_MON_STAT_FAN_0;
+		break;
+	case MC_CMD_SENSOR_VREG_1V2_TEMP:
+		*statp = EFX_MON_STAT_VREG_1V2_TEMP;
+		break;
+	case MC_CMD_SENSOR_IN_0V9_B:
+		*statp = EFX_MON_STAT_IN_0V9_B;
+		break;
+	default:
+		*statp = EFX_MON_NSTATS;
+		break;
+	};
+
+	if (*statp == EFX_MON_NSTATS)
+		goto fail1;
+
+	return (B_TRUE);
+
+fail1:
+	EFSYS_PROBE1(fail1, boolean_t, B_TRUE);
+	return (B_FALSE);
+};
+
+/* END MKCONFIG GENERATED MonitorMcdiMappingBlock */
+
+/* START MKCONFIG GENERATED MonitorStatisticUnitsBlock 2d447c656cc2d01d */
+	__checkReturn			boolean_t
+efx_mon_get_stat_unit(
+	__in				efx_mon_stat_t stat,
+	__out				efx_mon_stat_unit_t *unitp)
+{
+	switch (stat) {
+	case EFX_MON_STAT_IN_1V0:
+	case EFX_MON_STAT_IN_1V2:
+	case EFX_MON_STAT_IN_1V8:
+	case EFX_MON_STAT_IN_2V5:
+	case EFX_MON_STAT_IN_3V3:
+	case EFX_MON_STAT_IN_12V0:
+	case EFX_MON_STAT_IN_1V2A:
+	case EFX_MON_STAT_IN_VREF:
+	case EFX_MON_STAT_OUT_VAOE:
+	case EFX_MON_STAT_IN_VAOE:
+	case EFX_MON_STAT_IN_0V9:
+	case EFX_MON_STAT_IN_0V9_ADC:
+	case EFX_MON_STAT_CONTROLLER_VPTAT_EXTADC:
+	case EFX_MON_STAT_VDD08D_VSS08D_CSR:
+	case EFX_MON_STAT_VDD08D_VSS08D_CSR_EXTADC:
+	case EFX_MON_STAT_MUM_VCC:
+	case EFX_MON_STAT_IN_0V9_A:
+	case EFX_MON_STAT_IN_0V9_B:
+	case EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY:
+	case EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY_EXTADC:
+	case EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY:
+	case EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY_EXTADC:
+	case EFX_MON_STAT_CONTROLLER_MASTER_VPTAT:
+	case EFX_MON_STAT_CONTROLLER_MASTER_VPTAT_EXTADC:
+	case EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT:
+	case EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT_EXTADC:
+	case EFX_MON_STAT_SODIMM_VOUT:
+	case EFX_MON_STAT_PHY0_VCC:
+	case EFX_MON_STAT_PHY1_VCC:
+	case EFX_MON_STAT_IN_1V3:
+		*unitp = EFX_MON_STAT_UNIT_VOLTAGE_MV;
+		break;
+	case EFX_MON_STAT_CONTROLLER_TEMP:
+	case EFX_MON_STAT_PHY_COMMON_TEMP:
+	case EFX_MON_STAT_PHY0_TEMP:
+	case EFX_MON_STAT_PHY1_TEMP:
+	case EFX_MON_STAT_AOE_TEMP:
+	case EFX_MON_STAT_PSU_AOE_TEMP:
+	case EFX_MON_STAT_PSU_TEMP:
+	case EFX_MON_STAT_CONTROLLER_2_TEMP:
+	case EFX_MON_STAT_VREG_INTERNAL_TEMP:
+	case EFX_MON_STAT_VREG_0V9_TEMP:
+	case EFX_MON_STAT_VREG_1V2_TEMP:
+	case EFX_MON_STAT_CONTROLLER_VPTAT:
+	case EFX_MON_STAT_CONTROLLER_INTERNAL_TEMP:
+	case EFX_MON_STAT_CONTROLLER_INTERNAL_TEMP_EXTADC:
+	case EFX_MON_STAT_AMBIENT_TEMP:
+	case EFX_MON_STAT_HOTPOINT_TEMP:
+	case EFX_MON_STAT_VREG_0V9_A_TEMP:
+	case EFX_MON_STAT_VREG_0V9_B_TEMP:
+	case EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP:
+	case EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP_EXTADC:
+	case EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP:
+	case EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP_EXTADC:
+	case EFX_MON_STAT_SODIMM_0_TEMP:
+	case EFX_MON_STAT_SODIMM_1_TEMP:
+	case EFX_MON_STAT_CONTROLLER_TDIODE_TEMP:
+	case EFX_MON_STAT_BOARD_FRONT_TEMP:
+	case EFX_MON_STAT_BOARD_BACK_TEMP:
+		*unitp = EFX_MON_STAT_UNIT_TEMP_C;
+		break;
+	case EFX_MON_STAT_CONTROLLER_COOLING:
+	case EFX_MON_STAT_PHY0_COOLING:
+	case EFX_MON_STAT_PHY1_COOLING:
+	case EFX_MON_STAT_AIRFLOW:
+	case EFX_MON_STAT_PHY_POWER_PORT0:
+	case EFX_MON_STAT_PHY_POWER_PORT1:
+		*unitp = EFX_MON_STAT_UNIT_BOOL;
+		break;
+	case EFX_MON_STAT_NIC_POWER:
+		*unitp = EFX_MON_STAT_UNIT_POWER_W;
+		break;
+	case EFX_MON_STAT_OUT_IAOE:
+	case EFX_MON_STAT_IN_IAOE:
+	case EFX_MON_STAT_IN_I0V9:
+	case EFX_MON_STAT_IN_I1V2:
+	case EFX_MON_STAT_IN_I0V9_A:
+	case EFX_MON_STAT_IN_I0V9_B:
+	case EFX_MON_STAT_IN_I1V8:
+	case EFX_MON_STAT_IN_I2V5:
+	case EFX_MON_STAT_IN_I3V3:
+	case EFX_MON_STAT_IN_I12V0:
+	case EFX_MON_STAT_IN_I1V3:
+		*unitp = EFX_MON_STAT_UNIT_CURRENT_MA;
+		break;
+	case EFX_MON_STAT_FAN_0:
+	case EFX_MON_STAT_FAN_1:
+	case EFX_MON_STAT_FAN_2:
+	case EFX_MON_STAT_FAN_3:
+	case EFX_MON_STAT_FAN_4:
+		*unitp = EFX_MON_STAT_UNIT_RPM;
+		break;
+	default:
+		*unitp = EFX_MON_STAT_UNIT_UNKNOWN;
+		break;
+	};
+
+	if (*unitp == EFX_MON_STAT_UNIT_UNKNOWN)
+		goto fail1;
+
+	return (B_TRUE);
+
+fail1:
+	EFSYS_PROBE1(fail1, boolean_t, B_TRUE);
+	return (B_FALSE);
+};
+
+/* END MKCONFIG GENERATED MonitorStatisticUnitsBlock */
+
+/* START MKCONFIG GENERATED MonitorStatisticPortsBlock 1719b751d842534f */
+	__checkReturn			boolean_t
+efx_mon_get_stat_portmap(
+	__in				efx_mon_stat_t stat,
+	__out				efx_mon_stat_portmask_t *maskp)
+{
+
+	switch (stat) {
+	case EFX_MON_STAT_PHY1_TEMP:
+	case EFX_MON_STAT_PHY1_COOLING:
+	case EFX_MON_STAT_PHY_POWER_PORT1:
+		*maskp = EFX_MON_STAT_PORTMAP_PORT1;
+		break;
+	case EFX_MON_STAT_CONTROLLER_TEMP:
+	case EFX_MON_STAT_PHY_COMMON_TEMP:
+	case EFX_MON_STAT_CONTROLLER_COOLING:
+	case EFX_MON_STAT_IN_1V0:
+	case EFX_MON_STAT_IN_1V2:
+	case EFX_MON_STAT_IN_1V8:
+	case EFX_MON_STAT_IN_2V5:
+	case EFX_MON_STAT_IN_3V3:
+	case EFX_MON_STAT_IN_12V0:
+	case EFX_MON_STAT_IN_1V2A:
+	case EFX_MON_STAT_IN_VREF:
+	case EFX_MON_STAT_OUT_VAOE:
+	case EFX_MON_STAT_AOE_TEMP:
+	case EFX_MON_STAT_PSU_AOE_TEMP:
+	case EFX_MON_STAT_PSU_TEMP:
+	case EFX_MON_STAT_FAN_0:
+	case EFX_MON_STAT_FAN_1:
+	case EFX_MON_STAT_FAN_2:
+	case EFX_MON_STAT_FAN_3:
+	case EFX_MON_STAT_FAN_4:
+	case EFX_MON_STAT_IN_VAOE:
+	case EFX_MON_STAT_OUT_IAOE:
+	case EFX_MON_STAT_IN_IAOE:
+	case EFX_MON_STAT_NIC_POWER:
+	case EFX_MON_STAT_IN_0V9:
+	case EFX_MON_STAT_IN_I0V9:
+	case EFX_MON_STAT_IN_I1V2:
+	case EFX_MON_STAT_IN_0V9_ADC:
+	case EFX_MON_STAT_CONTROLLER_2_TEMP:
+	case EFX_MON_STAT_VREG_INTERNAL_TEMP:
+	case EFX_MON_STAT_VREG_0V9_TEMP:
+	case EFX_MON_STAT_VREG_1V2_TEMP:
+	case EFX_MON_STAT_CONTROLLER_VPTAT:
+	case EFX_MON_STAT_CONTROLLER_INTERNAL_TEMP:
+	case EFX_MON_STAT_CONTROLLER_VPTAT_EXTADC:
+	case EFX_MON_STAT_CONTROLLER_INTERNAL_TEMP_EXTADC:
+	case EFX_MON_STAT_AMBIENT_TEMP:
+	case EFX_MON_STAT_AIRFLOW:
+	case EFX_MON_STAT_VDD08D_VSS08D_CSR:
+	case EFX_MON_STAT_VDD08D_VSS08D_CSR_EXTADC:
+	case EFX_MON_STAT_HOTPOINT_TEMP:
+	case EFX_MON_STAT_MUM_VCC:
+	case EFX_MON_STAT_IN_0V9_A:
+	case EFX_MON_STAT_IN_I0V9_A:
+	case EFX_MON_STAT_VREG_0V9_A_TEMP:
+	case EFX_MON_STAT_IN_0V9_B:
+	case EFX_MON_STAT_IN_I0V9_B:
+	case EFX_MON_STAT_VREG_0V9_B_TEMP:
+	case EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY:
+	case EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY_EXTADC:
+	case EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY:
+	case EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY_EXTADC:
+	case EFX_MON_STAT_CONTROLLER_MASTER_VPTAT:
+	case EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP:
+	case EFX_MON_STAT_CONTROLLER_MASTER_VPTAT_EXTADC:
+	case EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP_EXTADC:
+	case EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT:
+	case EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP:
+	case EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT_EXTADC:
+	case EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP_EXTADC:
+	case EFX_MON_STAT_SODIMM_VOUT:
+	case EFX_MON_STAT_SODIMM_0_TEMP:
+	case EFX_MON_STAT_SODIMM_1_TEMP:
+	case EFX_MON_STAT_PHY0_VCC:
+	case EFX_MON_STAT_PHY1_VCC:
+	case EFX_MON_STAT_CONTROLLER_TDIODE_TEMP:
+	case EFX_MON_STAT_BOARD_FRONT_TEMP:
+	case EFX_MON_STAT_BOARD_BACK_TEMP:
+	case EFX_MON_STAT_IN_I1V8:
+	case EFX_MON_STAT_IN_I2V5:
+	case EFX_MON_STAT_IN_I3V3:
+	case EFX_MON_STAT_IN_I12V0:
+	case EFX_MON_STAT_IN_1V3:
+	case EFX_MON_STAT_IN_I1V3:
+		*maskp = EFX_MON_STAT_PORTMAP_ALL;
+		break;
+	case EFX_MON_STAT_PHY0_TEMP:
+	case EFX_MON_STAT_PHY0_COOLING:
+	case EFX_MON_STAT_PHY_POWER_PORT0:
+		*maskp = EFX_MON_STAT_PORTMAP_PORT0;
+		break;
+	default:
+		*maskp = EFX_MON_STAT_PORTMAP_UNKNOWN;
+		break;
+	};
+
+	if (*maskp == EFX_MON_STAT_PORTMAP_UNKNOWN)
+		goto fail1;
+
+	return (B_TRUE);
+
+fail1:
+	EFSYS_PROBE1(fail1, boolean_t, B_TRUE);
+	return (B_FALSE);
+};
+
+/* END MKCONFIG GENERATED MonitorStatisticPortsBlock */
+
 	__checkReturn			efx_rc_t
 efx_mon_stats_update(
 	__in				efx_nic_t *enp,
@@ -217,6 +816,20 @@ efx_mon_stats_update(
 	return (emop->emo_stats_update(enp, esmp, values));
 }
 
+	__checkReturn			efx_rc_t
+efx_mon_limits_update(
+	__in				efx_nic_t *enp,
+	__inout_ecount(EFX_MON_NSTATS)	efx_mon_stat_limits_t *values)
+{
+	efx_mon_t *emp = &(enp->en_mon);
+	const efx_mon_ops_t *emop = emp->em_emop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MON);
+
+	return (emop->emo_limits_update(enp, values));
+}
+
 #endif	/* EFSYS_OPT_MON_STATS */
 
 		void
diff --git a/drivers/net/sfc/base/efx_nic.c b/drivers/net/sfc/base/efx_nic.c
index 6c162e03..cea32b79 100644
--- a/drivers/net/sfc/base/efx_nic.c
+++ b/drivers/net/sfc/base/efx_nic.c
@@ -100,6 +100,8 @@ static const efx_nic_ops_t	__efx_nic_siena_ops = {
 	siena_nic_init,			/* eno_init */
 	NULL,				/* eno_get_vi_pool */
 	NULL,				/* eno_get_bar_region */
+	NULL,				/* eno_hw_unavailable */
+	NULL,				/* eno_set_hw_unavailable */
 #if EFSYS_OPT_DIAG
 	siena_nic_register_test,	/* eno_register_test */
 #endif	/* EFSYS_OPT_DIAG */
@@ -119,6 +121,8 @@ static const efx_nic_ops_t	__efx_nic_hunt_ops = {
 	ef10_nic_init,			/* eno_init */
 	ef10_nic_get_vi_pool,		/* eno_get_vi_pool */
 	ef10_nic_get_bar_region,	/* eno_get_bar_region */
+	ef10_nic_hw_unavailable,	/* eno_hw_unavailable */
+	ef10_nic_set_hw_unavailable,	/* eno_set_hw_unavailable */
 #if EFSYS_OPT_DIAG
 	ef10_nic_register_test,		/* eno_register_test */
 #endif	/* EFSYS_OPT_DIAG */
@@ -138,6 +142,8 @@ static const efx_nic_ops_t	__efx_nic_medford_ops = {
 	ef10_nic_init,			/* eno_init */
 	ef10_nic_get_vi_pool,		/* eno_get_vi_pool */
 	ef10_nic_get_bar_region,	/* eno_get_bar_region */
+	ef10_nic_hw_unavailable,	/* eno_hw_unavailable */
+	ef10_nic_set_hw_unavailable,	/* eno_set_hw_unavailable */
 #if EFSYS_OPT_DIAG
 	ef10_nic_register_test,		/* eno_register_test */
 #endif	/* EFSYS_OPT_DIAG */
@@ -157,6 +163,8 @@ static const efx_nic_ops_t	__efx_nic_medford2_ops = {
 	ef10_nic_init,			/* eno_init */
 	ef10_nic_get_vi_pool,		/* eno_get_vi_pool */
 	ef10_nic_get_bar_region,	/* eno_get_bar_region */
+	ef10_nic_hw_unavailable,	/* eno_hw_unavailable */
+	ef10_nic_set_hw_unavailable,	/* eno_set_hw_unavailable */
 #if EFSYS_OPT_DIAG
 	ef10_nic_register_test,		/* eno_register_test */
 #endif	/* EFSYS_OPT_DIAG */
@@ -549,7 +557,7 @@ efx_nic_reset(
 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_PROBE);
 	/*
-	 * All modules except the MCDI, PROBE, NVRAM, VPD, MON
+	 * All modules except the MCDI, PROBE, NVRAM, VPD, MON, TUNNEL
 	 * (which we do not reset here) must have been shut down or never
 	 * initialized.
 	 *
@@ -559,7 +567,10 @@ efx_nic_reset(
 	 */
 	mod_flags = enp->en_mod_flags;
 	mod_flags &= ~(EFX_MOD_MCDI | EFX_MOD_PROBE | EFX_MOD_NVRAM |
-		    EFX_MOD_VPD | EFX_MOD_MON);
+	    EFX_MOD_VPD | EFX_MOD_MON);
+#if EFSYS_OPT_TUNNEL
+	mod_flags &= ~EFX_MOD_TUNNEL;
+#endif /* EFSYS_OPT_TUNNEL */
 	EFSYS_ASSERT3U(mod_flags, ==, 0);
 	if (mod_flags != 0) {
 		rc = EINVAL;
@@ -584,6 +595,7 @@ efx_nic_cfg_get(
 	__in		efx_nic_t *enp)
 {
 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 
 	return (&(enp->en_nic_cfg));
 }
@@ -649,6 +661,39 @@ fail1:
 	return (rc);
 }
 
+	__checkReturn	boolean_t
+efx_nic_hw_unavailable(
+	__in		efx_nic_t *enp)
+{
+	const efx_nic_ops_t *enop = enp->en_enop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	/* NOTE: can be used by MCDI before NIC probe */
+
+	if (enop->eno_hw_unavailable != NULL) {
+		if ((enop->eno_hw_unavailable)(enp) != B_FALSE)
+			goto unavail;
+	}
+
+	return (B_FALSE);
+
+unavail:
+	return (B_TRUE);
+}
+
+			void
+efx_nic_set_hw_unavailable(
+	__in		efx_nic_t *enp)
+{
+	const efx_nic_ops_t *enop = enp->en_enop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+
+	if (enop->eno_set_hw_unavailable != NULL)
+		enop->eno_set_hw_unavailable(enp);
+}
+
+
 #if EFSYS_OPT_DIAG
 
 	__checkReturn	efx_rc_t
@@ -786,13 +831,12 @@ efx_mcdi_get_loopback_modes(
 {
 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_LOOPBACK_MODES_IN_LEN,
-			    MC_CMD_GET_LOOPBACK_MODES_OUT_V2_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_LOOPBACK_MODES_IN_LEN,
+		MC_CMD_GET_LOOPBACK_MODES_OUT_V2_LEN);
 	efx_qword_t mask;
 	efx_qword_t modes;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_LOOPBACK_MODES;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_LOOPBACK_MODES_IN_LEN;
diff --git a/drivers/net/sfc/base/efx_nvram.c b/drivers/net/sfc/base/efx_nvram.c
index be409c3a..5296c59b 100644
--- a/drivers/net/sfc/base/efx_nvram.c
+++ b/drivers/net/sfc/base/efx_nvram.c
@@ -468,7 +468,7 @@ efx_nvram_validate(
 		goto fail1;
 
 	if (envop->envo_buffer_validate != NULL) {
-		if ((rc = envop->envo_buffer_validate(enp, partn,
+		if ((rc = envop->envo_buffer_validate(partn,
 			    partn_data, partn_size)) != 0)
 			goto fail2;
 	}
@@ -514,12 +514,11 @@ efx_mcdi_nvram_partitions(
 	__out			unsigned int *npartnp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_NVRAM_PARTITIONS_IN_LEN,
-			    MC_CMD_NVRAM_PARTITIONS_OUT_LENMAX)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_NVRAM_PARTITIONS_IN_LEN,
+		MC_CMD_NVRAM_PARTITIONS_OUT_LENMAX);
 	unsigned int npartn;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_NVRAM_PARTITIONS;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_NVRAM_PARTITIONS_IN_LEN;
@@ -577,11 +576,10 @@ efx_mcdi_nvram_metadata(
 	__in			size_t size)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_NVRAM_METADATA_IN_LEN,
-			    MC_CMD_NVRAM_METADATA_OUT_LENMAX)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_NVRAM_METADATA_IN_LEN,
+		MC_CMD_NVRAM_METADATA_OUT_LENMAX);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_NVRAM_METADATA;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_NVRAM_METADATA_IN_LEN;
@@ -667,12 +665,11 @@ efx_mcdi_nvram_info(
 	__out_opt		uint32_t *erase_sizep,
 	__out_opt		uint32_t *write_sizep)
 {
-	uint8_t payload[MAX(MC_CMD_NVRAM_INFO_IN_LEN,
-			    MC_CMD_NVRAM_INFO_V2_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_NVRAM_INFO_IN_LEN,
+		MC_CMD_NVRAM_INFO_V2_OUT_LEN);
 	efx_mcdi_req_t req;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_NVRAM_INFO;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_NVRAM_INFO_IN_LEN;
@@ -728,12 +725,11 @@ efx_mcdi_nvram_update_start(
 	__in			efx_nic_t *enp,
 	__in			uint32_t partn)
 {
-	uint8_t payload[MAX(MC_CMD_NVRAM_UPDATE_START_V2_IN_LEN,
-			    MC_CMD_NVRAM_UPDATE_START_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_NVRAM_UPDATE_START_V2_IN_LEN,
+		MC_CMD_NVRAM_UPDATE_START_OUT_LEN);
 	efx_mcdi_req_t req;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_NVRAM_UPDATE_START;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_NVRAM_UPDATE_START_V2_IN_LEN;
@@ -770,8 +766,8 @@ efx_mcdi_nvram_read(
 	__in			uint32_t mode)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_NVRAM_READ_IN_V2_LEN,
-			    MC_CMD_NVRAM_READ_OUT_LENMAX)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_NVRAM_READ_IN_V2_LEN,
+		MC_CMD_NVRAM_READ_OUT_LENMAX);
 	efx_rc_t rc;
 
 	if (size > MC_CMD_NVRAM_READ_OUT_LENMAX) {
@@ -779,7 +775,6 @@ efx_mcdi_nvram_read(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_NVRAM_READ;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_NVRAM_READ_IN_V2_LEN;
@@ -825,11 +820,10 @@ efx_mcdi_nvram_erase(
 	__in			size_t size)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_NVRAM_ERASE_IN_LEN,
-			    MC_CMD_NVRAM_ERASE_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_NVRAM_ERASE_IN_LEN,
+		MC_CMD_NVRAM_ERASE_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_NVRAM_ERASE;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_NVRAM_ERASE_IN_LEN;
@@ -865,27 +859,31 @@ efx_mcdi_nvram_write(
 	__in			efx_nic_t *enp,
 	__in			uint32_t partn,
 	__in			uint32_t offset,
-	__out_bcount(size)	caddr_t data,
+	__in_bcount(size)	caddr_t data,
 	__in			size_t size)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MCDI_CTL_SDU_LEN_MAX_V1,
-			    MCDI_CTL_SDU_LEN_MAX_V2)];
+	uint8_t *payload;
 	efx_rc_t rc;
 	size_t max_data_size;
+	size_t payload_len = enp->en_nic_cfg.enc_mcdi_max_payload_length;
 
-	max_data_size = enp->en_nic_cfg.enc_mcdi_max_payload_length
-	    - MC_CMD_NVRAM_WRITE_IN_LEN(0);
-	EFSYS_ASSERT3U(enp->en_nic_cfg.enc_mcdi_max_payload_length, >, 0);
-	EFSYS_ASSERT3U(max_data_size, <,
-		    enp->en_nic_cfg.enc_mcdi_max_payload_length);
+	max_data_size = payload_len - MC_CMD_NVRAM_WRITE_IN_LEN(0);
+	EFSYS_ASSERT3U(payload_len, >, 0);
+	EFSYS_ASSERT3U(max_data_size, <, payload_len);
 
 	if (size > max_data_size) {
 		rc = EINVAL;
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
+	EFSYS_KMEM_ALLOC(enp->en_esip, payload_len, payload);
+	if (payload == NULL) {
+		rc = ENOMEM;
+		goto fail2;
+	}
+
+	(void) memset(payload, 0, payload_len);
 	req.emr_cmd = MC_CMD_NVRAM_WRITE;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_NVRAM_WRITE_IN_LEN(size);
@@ -903,11 +901,16 @@ efx_mcdi_nvram_write(
 
 	if (req.emr_rc != 0) {
 		rc = req.emr_rc;
-		goto fail2;
+		goto fail3;
 	}
 
+	EFSYS_KMEM_FREE(enp->en_esip, payload_len, payload);
+
 	return (0);
 
+fail3:
+	EFSYS_PROBE(fail3);
+	EFSYS_KMEM_FREE(enp->en_esip, payload_len, payload);
 fail2:
 	EFSYS_PROBE(fail2);
 fail1:
@@ -930,12 +933,11 @@ efx_mcdi_nvram_update_finish(
 {
 	const efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_LEN,
-			    MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_LEN,
+		MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT_LEN);
 	uint32_t verify_result = MC_CMD_NVRAM_VERIFY_RC_UNKNOWN;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_NVRAM_UPDATE_FINISH;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_LEN;
@@ -1001,12 +1003,11 @@ efx_mcdi_nvram_test(
 	__in			uint32_t partn)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_NVRAM_TEST_IN_LEN,
-			    MC_CMD_NVRAM_TEST_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_NVRAM_TEST_IN_LEN,
+		MC_CMD_NVRAM_TEST_OUT_LEN);
 	int result;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_NVRAM_TEST;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_NVRAM_TEST_IN_LEN;
diff --git a/drivers/net/sfc/base/efx_phy.c b/drivers/net/sfc/base/efx_phy.c
index ba2f51c1..36a7bbd3 100644
--- a/drivers/net/sfc/base/efx_phy.c
+++ b/drivers/net/sfc/base/efx_phy.c
@@ -15,6 +15,7 @@ static const efx_phy_ops_t	__efx_phy_siena_ops = {
 	siena_phy_reconfigure,		/* epo_reconfigure */
 	siena_phy_verify,		/* epo_verify */
 	siena_phy_oui_get,		/* epo_oui_get */
+	NULL,				/* epo_link_state_get */
 #if EFSYS_OPT_PHY_STATS
 	siena_phy_stats_update,		/* epo_stats_update */
 #endif	/* EFSYS_OPT_PHY_STATS */
@@ -34,6 +35,7 @@ static const efx_phy_ops_t	__efx_phy_ef10_ops = {
 	ef10_phy_reconfigure,		/* epo_reconfigure */
 	ef10_phy_verify,		/* epo_verify */
 	ef10_phy_oui_get,		/* epo_oui_get */
+	ef10_phy_link_state_get,	/* epo_link_state_get */
 #if EFSYS_OPT_PHY_STATS
 	ef10_phy_stats_update,		/* epo_stats_update */
 #endif	/* EFSYS_OPT_PHY_STATS */
@@ -286,8 +288,8 @@ efx_phy_media_type_get(
 efx_phy_module_get_info(
 	__in			efx_nic_t *enp,
 	__in			uint8_t dev_addr,
-	__in			uint8_t offset,
-	__in			uint8_t len,
+	__in			size_t offset,
+	__in			size_t len,
 	__out_bcount(len)	uint8_t *data)
 {
 	efx_rc_t rc;
@@ -295,7 +297,8 @@ efx_phy_module_get_info(
 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 	EFSYS_ASSERT(data != NULL);
 
-	if ((uint32_t)offset + len > 0xff) {
+	if ((offset > EFX_PHY_MEDIA_INFO_MAX_OFFSET) ||
+	    ((offset + len) > EFX_PHY_MEDIA_INFO_MAX_OFFSET)) {
 		rc = EINVAL;
 		goto fail1;
 	}
@@ -314,6 +317,57 @@ fail1:
 	return (rc);
 }
 
+	__checkReturn		efx_rc_t
+efx_phy_fec_type_get(
+	__in		efx_nic_t *enp,
+	__out		efx_phy_fec_type_t *typep)
+{
+	efx_rc_t rc;
+	efx_phy_link_state_t epls;
+
+	if ((rc = efx_phy_link_state_get(enp, &epls)) != 0)
+		goto fail1;
+
+	*typep = epls.epls_fec;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+	__checkReturn		efx_rc_t
+efx_phy_link_state_get(
+	__in		efx_nic_t *enp,
+	__out		efx_phy_link_state_t *eplsp)
+{
+	efx_port_t *epp = &(enp->en_port);
+	const efx_phy_ops_t *epop = epp->ep_epop;
+	efx_rc_t rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
+
+	if (epop->epo_link_state_get == NULL) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+
+	if ((rc = epop->epo_link_state_get(enp, eplsp)) != 0)
+		goto fail2;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
 #if EFSYS_OPT_PHY_STATS
 
 #if EFSYS_OPT_NAMES
diff --git a/drivers/net/sfc/base/efx_port.c b/drivers/net/sfc/base/efx_port.c
index 33a1a084..5fff932b 100644
--- a/drivers/net/sfc/base/efx_port.c
+++ b/drivers/net/sfc/base/efx_port.c
@@ -37,7 +37,7 @@ efx_port_init(
 	epp->ep_emop->emo_reconfigure(enp);
 
 	/* Pick up current phy capababilities */
-	efx_port_poll(enp, NULL);
+	(void) efx_port_poll(enp, NULL);
 
 	/*
 	 * Turn on the PHY if available, otherwise reset it, and
diff --git a/drivers/net/sfc/base/efx_rx.c b/drivers/net/sfc/base/efx_rx.c
index 4fd73bab..04bc7aed 100644
--- a/drivers/net/sfc/base/efx_rx.c
+++ b/drivers/net/sfc/base/efx_rx.c
@@ -298,84 +298,104 @@ fail1:
 efx_rx_scale_hash_flags_get(
 	__in					efx_nic_t *enp,
 	__in					efx_rx_hash_alg_t hash_alg,
-	__inout_ecount(EFX_RX_HASH_NFLAGS)	unsigned int *flags,
+	__out_ecount_part(max_nflags, *nflagsp)	unsigned int *flagsp,
+	__in					unsigned int max_nflags,
 	__out					unsigned int *nflagsp)
 {
 	efx_nic_cfg_t *encp = &enp->en_nic_cfg;
-	boolean_t l4;
-	boolean_t additional_modes;
-	unsigned int *entryp = flags;
+	unsigned int nflags = 0;
 	efx_rc_t rc;
 
-	if (flags == NULL || nflagsp == NULL) {
+	if (flagsp == NULL || nflagsp == NULL) {
 		rc = EINVAL;
 		goto fail1;
 	}
 
-	l4 = encp->enc_rx_scale_l4_hash_supported;
-	additional_modes = encp->enc_rx_scale_additional_modes_supported;
+	if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0) {
+		nflags = 0;
+		goto done;
+	}
 
-#define	LIST_FLAGS(_entryp, _class, _l4_hashing, _additional_modes)	\
-	do {								\
-		if (_l4_hashing) {					\
-			*(_entryp++) = EFX_RX_HASH(_class, 4TUPLE);	\
-									\
-			if (_additional_modes) {			\
-				*(_entryp++) =				\
-				    EFX_RX_HASH(_class, 2TUPLE_DST);	\
-				*(_entryp++) =				\
-				    EFX_RX_HASH(_class, 2TUPLE_SRC);	\
-			}						\
-		}							\
-									\
-		*(_entryp++) = EFX_RX_HASH(_class, 2TUPLE);		\
-									\
-		if (_additional_modes) {				\
-			*(_entryp++) = EFX_RX_HASH(_class, 1TUPLE_DST);	\
-			*(_entryp++) = EFX_RX_HASH(_class, 1TUPLE_SRC);	\
-		}							\
-									\
-		*(_entryp++) = EFX_RX_HASH(_class, DISABLE);		\
-									\
-		_NOTE(CONSTANTCONDITION)				\
+	/* Helper to add flags word to flags array without buffer overflow */
+#define	INSERT_FLAGS(_flags)			\
+	do {					\
+		if (nflags >= max_nflags) {	\
+			rc = E2BIG;		\
+			goto fail2;		\
+		}				\
+		*(flagsp + nflags) = (_flags);	\
+		nflags++;			\
+						\
+		_NOTE(CONSTANTCONDITION)	\
 	} while (B_FALSE)
 
-	switch (hash_alg) {
-	case EFX_RX_HASHALG_PACKED_STREAM:
-		if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0)
-			break;
-		/* FALLTHRU */
-	case EFX_RX_HASHALG_TOEPLITZ:
-		if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0)
-			break;
+	if (encp->enc_rx_scale_l4_hash_supported != B_FALSE) {
+		INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 4TUPLE));
+		INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 4TUPLE));
+	}
 
-		LIST_FLAGS(entryp, IPV4_TCP, l4, additional_modes);
-		LIST_FLAGS(entryp, IPV6_TCP, l4, additional_modes);
+	if ((encp->enc_rx_scale_l4_hash_supported != B_FALSE) &&
+	    (encp->enc_rx_scale_additional_modes_supported != B_FALSE)) {
+		INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_DST));
+		INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_SRC));
 
-		if (additional_modes) {
-			LIST_FLAGS(entryp, IPV4_UDP, l4, additional_modes);
-			LIST_FLAGS(entryp, IPV6_UDP, l4, additional_modes);
-		}
+		INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_DST));
+		INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_SRC));
 
-		LIST_FLAGS(entryp, IPV4, B_FALSE, additional_modes);
-		LIST_FLAGS(entryp, IPV6, B_FALSE, additional_modes);
-		break;
+		INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 4TUPLE));
+		INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_DST));
+		INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_SRC));
 
-	default:
-		rc = EINVAL;
-		goto fail2;
+		INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 4TUPLE));
+		INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_DST));
+		INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_SRC));
+	}
+
+	INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE));
+	INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE));
+
+	INSERT_FLAGS(EFX_RX_HASH(IPV4, 2TUPLE));
+	INSERT_FLAGS(EFX_RX_HASH(IPV6, 2TUPLE));
+
+	if (encp->enc_rx_scale_additional_modes_supported != B_FALSE) {
+		INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_DST));
+		INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_SRC));
+
+		INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_DST));
+		INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_SRC));
+
+		INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE));
+		INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_DST));
+		INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_SRC));
+
+		INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE));
+		INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_DST));
+		INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_SRC));
+
+		INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_DST));
+		INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_SRC));
+
+		INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_DST));
+		INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_SRC));
 	}
 
-#undef LIST_FLAGS
+	INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, DISABLE));
+	INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, DISABLE));
+
+	INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, DISABLE));
+	INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, DISABLE));
 
-	*nflagsp = (unsigned int)(entryp - flags);
-	EFSYS_ASSERT3U(*nflagsp, <=, EFX_RX_HASH_NFLAGS);
+	INSERT_FLAGS(EFX_RX_HASH(IPV4, DISABLE));
+	INSERT_FLAGS(EFX_RX_HASH(IPV6, DISABLE));
 
+#undef INSERT_FLAGS
+
+done:
+	*nflagsp = nflags;
 	return (0);
 
 fail2:
 	EFSYS_PROBE(fail2);
-
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
@@ -512,9 +532,8 @@ efx_rx_scale_mode_set(
 	__in		efx_rx_hash_type_t type,
 	__in		boolean_t insert)
 {
+	efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 	const efx_rx_ops_t *erxop = enp->en_erxop;
-	unsigned int type_flags[EFX_RX_HASH_NFLAGS];
-	unsigned int type_nflags;
 	efx_rx_hash_type_t type_check;
 	unsigned int i;
 	efx_rc_t rc;
@@ -533,46 +552,60 @@ efx_rx_scale_mode_set(
 	}
 
 	/*
-	 * Translate legacy flags to the new representation
-	 * so that chip-specific handlers will consider the
-	 * new flags only.
+	 * If RSS hash type is represented by additional bits
+	 * in the value, the latter need to be verified since
+	 * not all bit combinations are valid RSS modes. Also,
+	 * depending on the firmware, some valid combinations
+	 * may be unsupported. Discern additional bits in the
+	 * type value and try to recognise valid combinations.
+	 * If some bits remain unrecognised, report the error.
 	 */
-	if (type & EFX_RX_HASH_IPV4) {
-		type |= EFX_RX_HASH(IPV4, 2TUPLE);
-		type |= EFX_RX_HASH(IPV4_TCP, 2TUPLE);
-		type |= EFX_RX_HASH(IPV4_UDP, 2TUPLE);
-	}
-
-	if (type & EFX_RX_HASH_TCPIPV4)
-		type |= EFX_RX_HASH(IPV4_TCP, 4TUPLE);
+	type_check = type & ~EFX_RX_HASH_LEGACY_MASK;
+	if (type_check != 0) {
+		unsigned int type_flags[EFX_RX_HASH_NFLAGS];
+		unsigned int type_nflags;
 
-	if (type & EFX_RX_HASH_IPV6) {
-		type |= EFX_RX_HASH(IPV6, 2TUPLE);
-		type |= EFX_RX_HASH(IPV6_TCP, 2TUPLE);
-		type |= EFX_RX_HASH(IPV6_UDP, 2TUPLE);
-	}
+		rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags,
+				    EFX_ARRAY_SIZE(type_flags), &type_nflags);
+		if (rc != 0)
+			goto fail2;
 
-	if (type & EFX_RX_HASH_TCPIPV6)
-		type |= EFX_RX_HASH(IPV6_TCP, 4TUPLE);
+		for (i = 0; i < type_nflags; ++i) {
+			if ((type_check & type_flags[i]) == type_flags[i])
+				type_check &= ~(type_flags[i]);
+		}
 
-	type &= ~EFX_RX_HASH_LEGACY_MASK;
-	type_check = type;
+		if (type_check != 0) {
+			rc = EINVAL;
+			goto fail3;
+		}
+	}
 
 	/*
-	 * Get the list of supported hash flags and sanitise the input.
+	 * Translate EFX_RX_HASH() flags to their legacy counterparts
+	 * provided that the FW claims no support for additional modes.
 	 */
-	rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags, &type_nflags);
-	if (rc != 0)
-		goto fail2;
-
-	for (i = 0; i < type_nflags; ++i) {
-		if ((type_check & type_flags[i]) == type_flags[i])
-			type_check &= ~(type_flags[i]);
-	}
+	if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) {
+		efx_rx_hash_type_t t_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) |
+					    EFX_RX_HASH(IPV4_TCP, 2TUPLE);
+		efx_rx_hash_type_t t_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) |
+					    EFX_RX_HASH(IPV6_TCP, 2TUPLE);
+		efx_rx_hash_type_t t_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE);
+		efx_rx_hash_type_t t_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE);
+
+		if ((type & t_ipv4) == t_ipv4)
+			type |= EFX_RX_HASH_IPV4;
+		if ((type & t_ipv6) == t_ipv6)
+			type |= EFX_RX_HASH_IPV6;
+
+		if (encp->enc_rx_scale_l4_hash_supported == B_TRUE) {
+			if ((type & t_ipv4_tcp) == t_ipv4_tcp)
+				type |= EFX_RX_HASH_TCPIPV4;
+			if ((type & t_ipv6_tcp) == t_ipv6_tcp)
+				type |= EFX_RX_HASH_TCPIPV6;
+		}
 
-	if (type_check != 0) {
-		rc = EINVAL;
-		goto fail3;
+		type &= EFX_RX_HASH_LEGACY_MASK;
 	}
 
 	if (erxop->erxo_scale_mode_set != NULL) {
@@ -831,7 +864,7 @@ efx_rx_qcreate_packed_stream(
 {
 	efx_rxq_type_data_t type_data;
 
-	memset(&type_data, 0, sizeof(type_data));
+	memset(&type_data, 0, sizeof (type_data));
 
 	type_data.ertd_packed_stream.eps_buf_size = ps_buf_size;
 
@@ -867,7 +900,7 @@ efx_rx_qcreate_es_super_buffer(
 		goto fail1;
 	}
 
-	memset(&type_data, 0, sizeof(type_data));
+	memset(&type_data, 0, sizeof (type_data));
 
 	type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc;
 	type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len;
@@ -1090,10 +1123,6 @@ siena_rx_scale_mode_set(
 	__in		efx_rx_hash_type_t type,
 	__in		boolean_t insert)
 {
-	efx_rx_hash_type_t type_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE);
-	efx_rx_hash_type_t type_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE);
-	efx_rx_hash_type_t type_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE);
-	efx_rx_hash_type_t type_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE);
 	efx_rc_t rc;
 
 	if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
@@ -1108,12 +1137,12 @@ siena_rx_scale_mode_set(
 
 	case EFX_RX_HASHALG_TOEPLITZ:
 		EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
-		    (type & type_ipv4) == type_ipv4,
-		    (type & type_ipv4_tcp) == type_ipv4_tcp);
+		    (type & EFX_RX_HASH_IPV4) ? B_TRUE : B_FALSE,
+		    (type & EFX_RX_HASH_TCPIPV4) ? B_TRUE : B_FALSE);
 
 		EFX_RX_TOEPLITZ_IPV6_HASH(enp,
-		    (type & type_ipv6) == type_ipv6,
-		    (type & type_ipv6_tcp) == type_ipv6_tcp,
+		    (type & EFX_RX_HASH_IPV6) ? B_TRUE : B_FALSE,
+		    (type & EFX_RX_HASH_TCPIPV6) ? B_TRUE : B_FALSE,
 		    rc);
 		if (rc != 0)
 			goto fail2;
diff --git a/drivers/net/sfc/base/efx_tunnel.c b/drivers/net/sfc/base/efx_tunnel.c
index 399fd540..edb6be02 100644
--- a/drivers/net/sfc/base/efx_tunnel.c
+++ b/drivers/net/sfc/base/efx_tunnel.c
@@ -40,8 +40,9 @@ efx_mcdi_set_tunnel_encap_udp_ports(
 	__out			boolean_t *resetting)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LENMAX,
-			    MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload,
+		MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LENMAX,
+		MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_LEN);
 	efx_word_t flags;
 	efx_rc_t rc;
 	unsigned int i;
@@ -52,7 +53,6 @@ efx_mcdi_set_tunnel_encap_udp_ports(
 	else
 		entries_num = etcp->etc_udp_entries_num;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS;
 	req.emr_in_buf = payload;
 	req.emr_in_length =
diff --git a/drivers/net/sfc/base/efx_tx.c b/drivers/net/sfc/base/efx_tx.c
index da37580a..bf1180a1 100644
--- a/drivers/net/sfc/base/efx_tx.c
+++ b/drivers/net/sfc/base/efx_tx.c
@@ -572,19 +572,10 @@ efx_tx_qdesc_post(
 {
 	efx_nic_t *enp = etp->et_enp;
 	const efx_tx_ops_t *etxop = enp->en_etxop;
-	efx_rc_t rc;
 
 	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
 
-	if ((rc = etxop->etxo_qdesc_post(etp, ed,
-	    ndescs, completed, addedp)) != 0)
-		goto fail1;
-
-	return (0);
-
-fail1:
-	EFSYS_PROBE1(fail1, efx_rc_t, rc);
-	return (rc);
+	return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp));
 }
 
 	void
@@ -763,10 +754,9 @@ siena_tx_qpost(
 {
 	unsigned int added = *addedp;
 	unsigned int i;
-	int rc = ENOSPC;
 
 	if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
-		goto fail1;
+		return (ENOSPC);
 
 	for (i = 0; i < ndescs; i++) {
 		efx_buffer_t *ebp = &eb[i];
@@ -788,11 +778,6 @@ siena_tx_qpost(
 
 	*addedp = added;
 	return (0);
-
-fail1:
-	EFSYS_PROBE1(fail1, efx_rc_t, rc);
-
-	return (rc);
 }
 
 static		void
diff --git a/drivers/net/sfc/base/hunt_nic.c b/drivers/net/sfc/base/hunt_nic.c
index 16ea81d2..ca30e90f 100644
--- a/drivers/net/sfc/base/hunt_nic.c
+++ b/drivers/net/sfc/base/hunt_nic.c
@@ -20,7 +20,6 @@ hunt_nic_get_required_pcie_bandwidth(
 	__out		uint32_t *bandwidth_mbpsp)
 {
 	uint32_t port_modes;
-	uint32_t max_port_mode;
 	uint32_t bandwidth;
 	efx_rc_t rc;
 
@@ -30,7 +29,8 @@ hunt_nic_get_required_pcie_bandwidth(
 	 * capable mode is in use.
 	 */
 
-	if ((rc = efx_mcdi_get_port_modes(enp, &port_modes, NULL)) != 0) {
+	if ((rc = efx_mcdi_get_port_modes(enp, &port_modes,
+		    NULL, NULL)) != 0) {
 		/* No port mode info available */
 		bandwidth = 0;
 		goto out;
@@ -46,17 +46,13 @@ hunt_nic_get_required_pcie_bandwidth(
 			goto fail1;
 	} else {
 		if (port_modes & (1U << TLV_PORT_MODE_40G)) {
-			max_port_mode = TLV_PORT_MODE_40G;
+			bandwidth = 40000;
 		} else if (port_modes & (1U << TLV_PORT_MODE_10G_10G_10G_10G)) {
-			max_port_mode = TLV_PORT_MODE_10G_10G_10G_10G;
+			bandwidth = 4 * 10000;
 		} else {
 			/* Assume two 10G ports */
-			max_port_mode = TLV_PORT_MODE_10G_10G;
+			bandwidth = 2 * 10000;
 		}
-
-		if ((rc = ef10_nic_get_port_mode_bandwidth(max_port_mode,
-							    &bandwidth)) != 0)
-			goto fail2;
 	}
 
 out:
@@ -64,8 +60,6 @@ out:
 
 	return (0);
 
-fail2:
-	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
@@ -189,6 +183,9 @@ hunt_board_cfg(
 
 	encp->enc_bug61265_workaround = B_FALSE; /* Medford only */
 
+	/* Checksums for TSO sends can be incorrect on Huntington. */
+	encp->enc_bug61297_workaround = B_TRUE;
+
 	/* Alignment for receive packet DMA buffers */
 	encp->enc_rx_buf_align_start = 1;
 	encp->enc_rx_buf_align_end = 64; /* RX DMA end padding */
diff --git a/drivers/net/sfc/base/mc_driver_pcol_strs.h b/drivers/net/sfc/base/mc_driver_pcol_strs.h
new file mode 100644
index 00000000..73d633cb
--- /dev/null
+++ b/drivers/net/sfc/base/mc_driver_pcol_strs.h
@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright 2008-2018 Solarflare Communications Inc.
+ * All rights reserved.
+ */
+
+/*
+ * This file is automatically generated. DO NOT EDIT IT.
+ * To make changes, edit the .yml files under firmwaresrc doc/mcdi/ and
+ * rebuild this file with "make -C doc mcdiheaders".
+ *
+ * The version of this file has MCDI strings really used in the libefx.
+ */
+
+#ifndef _MC_DRIVER_PCOL_STRS_H
+#define _MC_DRIVER_PCOL_STRS_H
+
+#define MC_CMD_SENSOR_CONTROLLER_TEMP_ENUM_STR "Controller temperature: degC"
+#define MC_CMD_SENSOR_PHY_COMMON_TEMP_ENUM_STR "Phy common temperature: degC"
+#define MC_CMD_SENSOR_CONTROLLER_COOLING_ENUM_STR "Controller cooling: bool"
+#define MC_CMD_SENSOR_PHY0_TEMP_ENUM_STR "Phy 0 temperature: degC"
+#define MC_CMD_SENSOR_PHY0_COOLING_ENUM_STR "Phy 0 cooling: bool"
+#define MC_CMD_SENSOR_PHY1_TEMP_ENUM_STR "Phy 1 temperature: degC"
+#define MC_CMD_SENSOR_PHY1_COOLING_ENUM_STR "Phy 1 cooling: bool"
+#define MC_CMD_SENSOR_IN_1V0_ENUM_STR "1.0v power: mV"
+#define MC_CMD_SENSOR_IN_1V2_ENUM_STR "1.2v power: mV"
+#define MC_CMD_SENSOR_IN_1V8_ENUM_STR "1.8v power: mV"
+#define MC_CMD_SENSOR_IN_2V5_ENUM_STR "2.5v power: mV"
+#define MC_CMD_SENSOR_IN_3V3_ENUM_STR "3.3v power: mV"
+#define MC_CMD_SENSOR_IN_12V0_ENUM_STR "12v power: mV"
+#define MC_CMD_SENSOR_IN_1V2A_ENUM_STR "1.2v analogue power: mV"
+#define MC_CMD_SENSOR_IN_VREF_ENUM_STR "reference voltage: mV"
+#define MC_CMD_SENSOR_OUT_VAOE_ENUM_STR "AOE FPGA power: mV"
+#define MC_CMD_SENSOR_AOE_TEMP_ENUM_STR "AOE FPGA temperature: degC"
+#define MC_CMD_SENSOR_PSU_AOE_TEMP_ENUM_STR "AOE FPGA PSU temperature: degC"
+#define MC_CMD_SENSOR_PSU_TEMP_ENUM_STR "AOE PSU temperature: degC"
+#define MC_CMD_SENSOR_FAN_0_ENUM_STR "Fan 0 speed: RPM"
+#define MC_CMD_SENSOR_FAN_1_ENUM_STR "Fan 1 speed: RPM"
+#define MC_CMD_SENSOR_FAN_2_ENUM_STR "Fan 2 speed: RPM"
+#define MC_CMD_SENSOR_FAN_3_ENUM_STR "Fan 3 speed: RPM"
+#define MC_CMD_SENSOR_FAN_4_ENUM_STR "Fan 4 speed: RPM"
+#define MC_CMD_SENSOR_IN_VAOE_ENUM_STR "AOE FPGA input power: mV"
+#define MC_CMD_SENSOR_OUT_IAOE_ENUM_STR "AOE FPGA current: mA"
+#define MC_CMD_SENSOR_IN_IAOE_ENUM_STR "AOE FPGA input current: mA"
+#define MC_CMD_SENSOR_NIC_POWER_ENUM_STR "NIC power consumption: W"
+#define MC_CMD_SENSOR_IN_0V9_ENUM_STR "0.9v power voltage: mV"
+#define MC_CMD_SENSOR_IN_I0V9_ENUM_STR "0.9v power current: mA"
+#define MC_CMD_SENSOR_IN_I1V2_ENUM_STR "1.2v power current: mA"
+#define MC_CMD_SENSOR_PAGE0_NEXT_ENUM_STR "Not a sensor: reserved for the next page flag"
+#define MC_CMD_SENSOR_IN_0V9_ADC_ENUM_STR "0.9v power voltage (at ADC): mV"
+#define MC_CMD_SENSOR_CONTROLLER_2_TEMP_ENUM_STR "Controller temperature 2: degC"
+#define MC_CMD_SENSOR_VREG_INTERNAL_TEMP_ENUM_STR "Voltage regulator internal temperature: degC"
+#define MC_CMD_SENSOR_VREG_0V9_TEMP_ENUM_STR "0.9V voltage regulator temperature: degC"
+#define MC_CMD_SENSOR_VREG_1V2_TEMP_ENUM_STR "1.2V voltage regulator temperature: degC"
+#define MC_CMD_SENSOR_CONTROLLER_VPTAT_ENUM_STR "controller internal temperature sensor voltage (internal ADC): mV"
+#define MC_CMD_SENSOR_CONTROLLER_INTERNAL_TEMP_ENUM_STR "controller internal temperature (internal ADC): degC"
+#define MC_CMD_SENSOR_CONTROLLER_VPTAT_EXTADC_ENUM_STR "controller internal temperature sensor voltage (external ADC): mV"
+#define MC_CMD_SENSOR_CONTROLLER_INTERNAL_TEMP_EXTADC_ENUM_STR "controller internal temperature (external ADC): degC"
+#define MC_CMD_SENSOR_AMBIENT_TEMP_ENUM_STR "ambient temperature: degC"
+#define MC_CMD_SENSOR_AIRFLOW_ENUM_STR "air flow: bool"
+#define MC_CMD_SENSOR_VDD08D_VSS08D_CSR_ENUM_STR "voltage between VSS08D and VSS08D at CSR: mV"
+#define MC_CMD_SENSOR_VDD08D_VSS08D_CSR_EXTADC_ENUM_STR "voltage between VSS08D and VSS08D at CSR (external ADC): mV"
+#define MC_CMD_SENSOR_HOTPOINT_TEMP_ENUM_STR "Hotpoint temperature: degC"
+#define MC_CMD_SENSOR_PHY_POWER_PORT0_ENUM_STR "Port 0 PHY power switch over-current: bool"
+#define MC_CMD_SENSOR_PHY_POWER_PORT1_ENUM_STR "Port 1 PHY power switch over-current: bool"
+#define MC_CMD_SENSOR_MUM_VCC_ENUM_STR "Mop-up microcontroller reference voltage: mV"
+#define MC_CMD_SENSOR_IN_0V9_A_ENUM_STR "0.9v power phase A voltage: mV"
+#define MC_CMD_SENSOR_IN_I0V9_A_ENUM_STR "0.9v power phase A current: mA"
+#define MC_CMD_SENSOR_VREG_0V9_A_TEMP_ENUM_STR "0.9V voltage regulator phase A temperature: degC"
+#define MC_CMD_SENSOR_IN_0V9_B_ENUM_STR "0.9v power phase B voltage: mV"
+#define MC_CMD_SENSOR_IN_I0V9_B_ENUM_STR "0.9v power phase B current: mA"
+#define MC_CMD_SENSOR_VREG_0V9_B_TEMP_ENUM_STR "0.9V voltage regulator phase B temperature: degC"
+#define MC_CMD_SENSOR_CCOM_AVREG_1V2_SUPPLY_ENUM_STR "CCOM AVREG 1v2 supply (interval ADC): mV"
+#define MC_CMD_SENSOR_CCOM_AVREG_1V2_SUPPLY_EXTADC_ENUM_STR "CCOM AVREG 1v2 supply (external ADC): mV"
+#define MC_CMD_SENSOR_CCOM_AVREG_1V8_SUPPLY_ENUM_STR "CCOM AVREG 1v8 supply (interval ADC): mV"
+#define MC_CMD_SENSOR_CCOM_AVREG_1V8_SUPPLY_EXTADC_ENUM_STR "CCOM AVREG 1v8 supply (external ADC): mV"
+#define MC_CMD_SENSOR_CONTROLLER_RTS_ENUM_STR "CCOM RTS temperature: degC"
+#define MC_CMD_SENSOR_PAGE1_NEXT_ENUM_STR "Not a sensor: reserved for the next page flag"
+#define MC_CMD_SENSOR_CONTROLLER_MASTER_VPTAT_ENUM_STR "controller internal temperature sensor voltage on master core (internal ADC): mV"
+#define MC_CMD_SENSOR_CONTROLLER_MASTER_INTERNAL_TEMP_ENUM_STR "controller internal temperature on master core (internal ADC): degC"
+#define MC_CMD_SENSOR_CONTROLLER_MASTER_VPTAT_EXTADC_ENUM_STR "controller internal temperature sensor voltage on master core (external ADC): mV"
+#define MC_CMD_SENSOR_CONTROLLER_MASTER_INTERNAL_TEMP_EXTADC_ENUM_STR "controller internal temperature on master core (external ADC): degC"
+#define MC_CMD_SENSOR_CONTROLLER_SLAVE_VPTAT_ENUM_STR "controller internal temperature on slave core sensor voltage (internal ADC): mV"
+#define MC_CMD_SENSOR_CONTROLLER_SLAVE_INTERNAL_TEMP_ENUM_STR "controller internal temperature on slave core (internal ADC): degC"
+#define MC_CMD_SENSOR_CONTROLLER_SLAVE_VPTAT_EXTADC_ENUM_STR "controller internal temperature on slave core sensor voltage (external ADC): mV"
+#define MC_CMD_SENSOR_CONTROLLER_SLAVE_INTERNAL_TEMP_EXTADC_ENUM_STR "controller internal temperature on slave core (external ADC): degC"
+#define MC_CMD_SENSOR_SODIMM_VOUT_ENUM_STR "Voltage supplied to the SODIMMs from their power supply: mV"
+#define MC_CMD_SENSOR_SODIMM_0_TEMP_ENUM_STR "Temperature of SODIMM 0 (if installed): degC"
+#define MC_CMD_SENSOR_SODIMM_1_TEMP_ENUM_STR "Temperature of SODIMM 1 (if installed): degC"
+#define MC_CMD_SENSOR_PHY0_VCC_ENUM_STR "Voltage supplied to the QSFP #0 from their power supply: mV"
+#define MC_CMD_SENSOR_PHY1_VCC_ENUM_STR "Voltage supplied to the QSFP #1 from their power supply: mV"
+#define MC_CMD_SENSOR_CONTROLLER_TDIODE_TEMP_ENUM_STR "Controller die temperature (TDIODE): degC"
+#define MC_CMD_SENSOR_BOARD_FRONT_TEMP_ENUM_STR "Board temperature (front): degC"
+#define MC_CMD_SENSOR_BOARD_BACK_TEMP_ENUM_STR "Board temperature (back): degC"
+#define MC_CMD_SENSOR_IN_I1V8_ENUM_STR "1.8v power current: mA"
+#define MC_CMD_SENSOR_IN_I2V5_ENUM_STR "2.5v power current: mA"
+#define MC_CMD_SENSOR_IN_I3V3_ENUM_STR "3.3v power current: mA"
+#define MC_CMD_SENSOR_IN_I12V0_ENUM_STR "12v power current: mA"
+#define MC_CMD_SENSOR_IN_1V3_ENUM_STR "1.3v power: mV"
+#define MC_CMD_SENSOR_IN_I1V3_ENUM_STR "1.3v power current: mA"
+
+#endif /* _MC_DRIVER_PCOL_STRS_H */
diff --git a/drivers/net/sfc/base/mcdi_mon.c b/drivers/net/sfc/base/mcdi_mon.c
index 940bd026..b53de0d6 100644
--- a/drivers/net/sfc/base/mcdi_mon.c
+++ b/drivers/net/sfc/base/mcdi_mon.c
@@ -6,141 +6,15 @@
 
 #include "efx.h"
 #include "efx_impl.h"
+#include "mcdi_mon.h"
 
 #if EFSYS_OPT_MON_MCDI
 
 #if EFSYS_OPT_MON_STATS
 
-#define	MCDI_MON_NEXT_PAGE  ((uint16_t)0xfffe)
-#define	MCDI_MON_INVALID_SENSOR ((uint16_t)0xfffd)
-#define	MCDI_MON_PAGE_SIZE 0x20
-
-/* Bitmasks of valid port(s) for each sensor */
-#define	MCDI_MON_PORT_NONE	(0x00)
-#define	MCDI_MON_PORT_P1	(0x01)
-#define	MCDI_MON_PORT_P2	(0x02)
-#define	MCDI_MON_PORT_P3	(0x04)
-#define	MCDI_MON_PORT_P4	(0x08)
-#define	MCDI_MON_PORT_Px	(0xFFFF)
-
 /* Get port mask from one-based MCDI port number */
 #define	MCDI_MON_PORT_MASK(_emip) (1U << ((_emip)->emi_port - 1))
 
-/* Entry for MCDI sensor in sensor map */
-#define	STAT(portmask, stat)	\
-	{ (MCDI_MON_PORT_##portmask), (EFX_MON_STAT_##stat) }
-
-/* Entry for sensor next page flag in sensor map */
-#define	STAT_NEXT_PAGE()	\
-	{ MCDI_MON_PORT_NONE, MCDI_MON_NEXT_PAGE }
-
-/* Placeholder for gaps in the array */
-#define	STAT_NO_SENSOR()	\
-	{ MCDI_MON_PORT_NONE, MCDI_MON_INVALID_SENSOR }
-
-/* Map from MC sensors to monitor statistics */
-static const struct mcdi_sensor_map_s {
-	uint16_t	msm_port_mask;
-	uint16_t	msm_stat;
-} mcdi_sensor_map[] = {
-	/* Sensor page 0		MC_CMD_SENSOR_xxx */
-	STAT(Px, INT_TEMP),		/* 0x00 CONTROLLER_TEMP */
-	STAT(Px, EXT_TEMP),		/* 0x01 PHY_COMMON_TEMP */
-	STAT(Px, INT_COOLING),		/* 0x02 CONTROLLER_COOLING */
-	STAT(P1, EXT_TEMP),		/* 0x03 PHY0_TEMP */
-	STAT(P1, EXT_COOLING),		/* 0x04 PHY0_COOLING */
-	STAT(P2, EXT_TEMP),		/* 0x05 PHY1_TEMP */
-	STAT(P2, EXT_COOLING),		/* 0x06 PHY1_COOLING */
-	STAT(Px, 1V),			/* 0x07 IN_1V0 */
-	STAT(Px, 1_2V),			/* 0x08 IN_1V2 */
-	STAT(Px, 1_8V),			/* 0x09 IN_1V8 */
-	STAT(Px, 2_5V),			/* 0x0a IN_2V5 */
-	STAT(Px, 3_3V),			/* 0x0b IN_3V3 */
-	STAT(Px, 12V),			/* 0x0c IN_12V0 */
-	STAT(Px, 1_2VA),		/* 0x0d IN_1V2A */
-	STAT(Px, VREF),			/* 0x0e IN_VREF */
-	STAT(Px, VAOE),			/* 0x0f OUT_VAOE */
-	STAT(Px, AOE_TEMP),		/* 0x10 AOE_TEMP */
-	STAT(Px, PSU_AOE_TEMP),		/* 0x11 PSU_AOE_TEMP */
-	STAT(Px, PSU_TEMP),		/* 0x12 PSU_TEMP */
-	STAT(Px, FAN0),			/* 0x13 FAN_0 */
-	STAT(Px, FAN1),			/* 0x14 FAN_1 */
-	STAT(Px, FAN2),			/* 0x15 FAN_2 */
-	STAT(Px, FAN3),			/* 0x16 FAN_3 */
-	STAT(Px, FAN4),			/* 0x17 FAN_4 */
-	STAT(Px, VAOE_IN),		/* 0x18 IN_VAOE */
-	STAT(Px, IAOE),			/* 0x19 OUT_IAOE */
-	STAT(Px, IAOE_IN),		/* 0x1a IN_IAOE */
-	STAT(Px, NIC_POWER),		/* 0x1b NIC_POWER */
-	STAT(Px, 0_9V),			/* 0x1c IN_0V9 */
-	STAT(Px, I0_9V),		/* 0x1d IN_I0V9 */
-	STAT(Px, I1_2V),		/* 0x1e IN_I1V2 */
-	STAT_NEXT_PAGE(),		/* 0x1f Next page flag (not a sensor) */
-
-	/* Sensor page 1		MC_CMD_SENSOR_xxx */
-	STAT(Px, 0_9V_ADC),		/* 0x20 IN_0V9_ADC */
-	STAT(Px, INT_TEMP2),		/* 0x21 CONTROLLER_2_TEMP */
-	STAT(Px, VREG_TEMP),		/* 0x22 VREG_INTERNAL_TEMP */
-	STAT(Px, VREG_0_9V_TEMP),	/* 0x23 VREG_0V9_TEMP */
-	STAT(Px, VREG_1_2V_TEMP),	/* 0x24 VREG_1V2_TEMP */
-	STAT(Px, INT_VPTAT),		/* 0x25 CTRLR. VPTAT */
-	STAT(Px, INT_ADC_TEMP),		/* 0x26 CTRLR. INTERNAL_TEMP */
-	STAT(Px, EXT_VPTAT),		/* 0x27 CTRLR. VPTAT_EXTADC */
-	STAT(Px, EXT_ADC_TEMP),		/* 0x28 CTRLR. INTERNAL_TEMP_EXTADC */
-	STAT(Px, AMBIENT_TEMP),		/* 0x29 AMBIENT_TEMP */
-	STAT(Px, AIRFLOW),		/* 0x2a AIRFLOW */
-	STAT(Px, VDD08D_VSS08D_CSR),	/* 0x2b VDD08D_VSS08D_CSR */
-	STAT(Px, VDD08D_VSS08D_CSR_EXTADC), /* 0x2c VDD08D_VSS08D_CSR_EXTADC */
-	STAT(Px, HOTPOINT_TEMP),	/* 0x2d HOTPOINT_TEMP */
-	STAT(P1, PHY_POWER_SWITCH_PORT0),   /* 0x2e PHY_POWER_SWITCH_PORT0 */
-	STAT(P2, PHY_POWER_SWITCH_PORT1),   /* 0x2f PHY_POWER_SWITCH_PORT1 */
-	STAT(Px, MUM_VCC),		/* 0x30 MUM_VCC */
-	STAT(Px, 0V9_A),		/* 0x31 0V9_A */
-	STAT(Px, I0V9_A),		/* 0x32 I0V9_A */
-	STAT(Px, 0V9_A_TEMP),		/* 0x33 0V9_A_TEMP */
-	STAT(Px, 0V9_B),		/* 0x34 0V9_B */
-	STAT(Px, I0V9_B),		/* 0x35 I0V9_B */
-	STAT(Px, 0V9_B_TEMP),		/* 0x36 0V9_B_TEMP */
-	STAT(Px, CCOM_AVREG_1V2_SUPPLY),  /* 0x37 CCOM_AVREG_1V2_SUPPLY */
-	STAT(Px, CCOM_AVREG_1V2_SUPPLY_EXT_ADC),
-					/* 0x38 CCOM_AVREG_1V2_SUPPLY_EXT_ADC */
-	STAT(Px, CCOM_AVREG_1V8_SUPPLY),  /* 0x39 CCOM_AVREG_1V8_SUPPLY */
-	STAT(Px, CCOM_AVREG_1V8_SUPPLY_EXT_ADC),
-					/* 0x3a CCOM_AVREG_1V8_SUPPLY_EXT_ADC */
-	STAT_NO_SENSOR(),		/* 0x3b (no sensor) */
-	STAT_NO_SENSOR(),		/* 0x3c (no sensor) */
-	STAT_NO_SENSOR(),		/* 0x3d (no sensor) */
-	STAT_NO_SENSOR(),		/* 0x3e (no sensor) */
-	STAT_NEXT_PAGE(),		/* 0x3f Next page flag (not a sensor) */
-
-	/* Sensor page 2		MC_CMD_SENSOR_xxx */
-	STAT(Px, CONTROLLER_MASTER_VPTAT),	   /* 0x40 MASTER_VPTAT */
-	STAT(Px, CONTROLLER_MASTER_INTERNAL_TEMP), /* 0x41 MASTER_INT_TEMP */
-	STAT(Px, CONTROLLER_MASTER_VPTAT_EXT_ADC), /* 0x42 MAST_VPTAT_EXT_ADC */
-	STAT(Px, CONTROLLER_MASTER_INTERNAL_TEMP_EXT_ADC),
-					/* 0x43 MASTER_INTERNAL_TEMP_EXT_ADC */
-	STAT(Px, CONTROLLER_SLAVE_VPTAT),	  /* 0x44 SLAVE_VPTAT */
-	STAT(Px, CONTROLLER_SLAVE_INTERNAL_TEMP), /* 0x45 SLAVE_INTERNAL_TEMP */
-	STAT(Px, CONTROLLER_SLAVE_VPTAT_EXT_ADC), /* 0x46 SLAVE_VPTAT_EXT_ADC */
-	STAT(Px, CONTROLLER_SLAVE_INTERNAL_TEMP_EXT_ADC),
-					/* 0x47 SLAVE_INTERNAL_TEMP_EXT_ADC */
-	STAT_NO_SENSOR(),		/* 0x48 (no sensor) */
-	STAT(Px, SODIMM_VOUT),		/* 0x49 SODIMM_VOUT */
-	STAT(Px, SODIMM_0_TEMP),	/* 0x4a SODIMM_0_TEMP */
-	STAT(Px, SODIMM_1_TEMP),	/* 0x4b SODIMM_1_TEMP */
-	STAT(Px, PHY0_VCC),		/* 0x4c PHY0_VCC */
-	STAT(Px, PHY1_VCC),		/* 0x4d PHY1_VCC */
-	STAT(Px, CONTROLLER_TDIODE_TEMP), /* 0x4e CONTROLLER_TDIODE_TEMP */
-	STAT(Px, BOARD_FRONT_TEMP),	/* 0x4f BOARD_FRONT_TEMP */
-	STAT(Px, BOARD_BACK_TEMP),	/* 0x50 BOARD_BACK_TEMP */
-	STAT(Px, I1V8),			/* 0x51 IN_I1V8 */
-	STAT(Px, I2V5),			/* 0x52 IN_I2V5 */
-	STAT(Px, I3V3),			/* 0x53 IN_I3V3 */
-	STAT(Px, I12V0),		/* 0x54 IN_I12V0 */
-	STAT(Px, 1_3V),			/* 0x55 IN_1V3 */
-	STAT(Px, I1V3),			/* 0x56 IN_I1V3 */
-};
-
 #define	MCDI_STATIC_SENSOR_ASSERT(_field)				\
 	EFX_STATIC_ASSERT(MC_CMD_SENSOR_STATE_ ## _field		\
 			    == EFX_MON_STAT_STATE_ ## _field)
@@ -155,10 +29,10 @@ mcdi_mon_decode_stats(
 	__inout_ecount_opt(EFX_MON_NSTATS)	efx_mon_stat_value_t *stat)
 {
 	efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip);
-	uint16_t port_mask;
+	efx_mon_stat_portmask_t port_mask;
 	uint16_t sensor;
 	size_t sensor_max;
-	uint32_t stat_mask[(EFX_ARRAY_SIZE(mcdi_sensor_map) + 31) / 32];
+	uint32_t stat_mask[(EFX_MON_NSTATS + 31) / 32];
 	uint32_t idx = 0;
 	uint32_t page = 0;
 
@@ -169,13 +43,10 @@ mcdi_mon_decode_stats(
 	MCDI_STATIC_SENSOR_ASSERT(BROKEN);
 	MCDI_STATIC_SENSOR_ASSERT(NO_READING);
 
-	EFX_STATIC_ASSERT(sizeof (stat_mask[0]) * 8 ==
-	    EFX_MON_MASK_ELEMENT_SIZE);
-	sensor_max =
-	    MIN((8 * sensor_mask_size), EFX_ARRAY_SIZE(mcdi_sensor_map));
+	sensor_max = 8 * sensor_mask_size;
 
 	EFSYS_ASSERT(emip->emi_port > 0); /* MCDI port number is one-based */
-	port_mask = MCDI_MON_PORT_MASK(emip);
+	port_mask = (efx_mon_stat_portmask_t)MCDI_MON_PORT_MASK(emip);
 
 	memset(stat_mask, 0, sizeof (stat_mask));
 
@@ -190,19 +61,36 @@ mcdi_mon_decode_stats(
 	 * does not understand.
 	 */
 	for (sensor = 0; sensor < sensor_max; ++sensor) {
-		efx_mon_stat_t id = mcdi_sensor_map[sensor].msm_stat;
+		efx_mon_stat_t id;
+		efx_mon_stat_portmask_t stat_portmask = 0;
+		boolean_t decode_ok;
+		efx_mon_stat_unit_t stat_unit;
 
-		if ((sensor % MCDI_MON_PAGE_SIZE) == MC_CMD_SENSOR_PAGE0_NEXT) {
-			EFSYS_ASSERT3U(id, ==, MCDI_MON_NEXT_PAGE);
+		if ((sensor % (MC_CMD_SENSOR_PAGE0_NEXT + 1)) ==
+		    MC_CMD_SENSOR_PAGE0_NEXT) {
 			page++;
 			continue;
+			/* This sensor is one of the page boundary bits. */
 		}
+
 		if (~(sensor_mask[page]) & (1U << sensor))
 			continue;
+		/* This sensor not in DMA buffer */
+
 		idx++;
+		/*
+		 * Valid stat in DMA buffer that we need to increment over, even
+		 * if we couldn't look up the id
+		 */
+
+		decode_ok = efx_mon_mcdi_to_efx_stat(sensor, &id);
+		decode_ok =
+		    decode_ok && efx_mon_get_stat_portmap(id, &stat_portmask);
 
-		if ((port_mask & mcdi_sensor_map[sensor].msm_port_mask) == 0)
+		if (!(decode_ok && (stat_portmask & port_mask)))
 			continue;
+		/* Either bad decode, or don't know what port stat is on */
+
 		EFSYS_ASSERT(id < EFX_MON_NSTATS);
 
 		/*
@@ -228,6 +116,10 @@ mcdi_mon_decode_stats(
 
 			stat[id].emsv_state = (uint16_t)EFX_DWORD_FIELD(dword,
 			    MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_STATE);
+
+			stat[id].emsv_unit =
+			    efx_mon_get_stat_unit(id, &stat_unit) ?
+			    stat_unit : EFX_MON_STAT_UNIT_UNKNOWN;
 		}
 	}
 
@@ -244,7 +136,7 @@ mcdi_mon_ev(
 	__out				efx_mon_stat_value_t *valuep)
 {
 	efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip);
-	uint16_t port_mask;
+	efx_mon_stat_portmask_t port_mask, sensor_port_mask;
 	uint16_t sensor;
 	uint16_t state;
 	uint16_t value;
@@ -261,20 +153,22 @@ mcdi_mon_ev(
 	/* Hardware must support this MCDI sensor */
 	EFSYS_ASSERT3U(sensor, <,
 	    (8 * enp->en_nic_cfg.enc_mcdi_sensor_mask_size));
-	EFSYS_ASSERT((sensor % MCDI_MON_PAGE_SIZE) != MC_CMD_SENSOR_PAGE0_NEXT);
+	EFSYS_ASSERT((sensor % (MC_CMD_SENSOR_PAGE0_NEXT + 1)) !=
+	    MC_CMD_SENSOR_PAGE0_NEXT);
 	EFSYS_ASSERT(enp->en_nic_cfg.enc_mcdi_sensor_maskp != NULL);
-	EFSYS_ASSERT(
-	    (enp->en_nic_cfg.enc_mcdi_sensor_maskp[sensor/MCDI_MON_PAGE_SIZE] &
-	    (1U << (sensor % MCDI_MON_PAGE_SIZE))) != 0);
+	EFSYS_ASSERT((enp->en_nic_cfg.enc_mcdi_sensor_maskp[
+		    sensor / (MC_CMD_SENSOR_PAGE0_NEXT + 1)] &
+		(1U << (sensor % (MC_CMD_SENSOR_PAGE0_NEXT + 1)))) != 0);
 
-	/* But we don't have to understand it */
-	if (sensor >= EFX_ARRAY_SIZE(mcdi_sensor_map)) {
+	/* And we need to understand it, to get port-map */
+	if (!efx_mon_mcdi_to_efx_stat(sensor, &id)) {
 		rc = ENOTSUP;
 		goto fail1;
 	}
-	id = mcdi_sensor_map[sensor].msm_stat;
-	if ((port_mask & mcdi_sensor_map[sensor].msm_port_mask) == 0)
+	if (!(efx_mon_get_stat_portmap(id, &sensor_port_mask) &&
+		(port_mask && sensor_port_mask))) {
 		return (ENODEV);
+	}
 	EFSYS_ASSERT(id < EFX_MON_NSTATS);
 
 	*idp = id;
@@ -297,9 +191,15 @@ efx_mcdi_read_sensors(
 	__in		uint32_t size)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_READ_SENSORS_EXT_IN_LEN,
-			    MC_CMD_READ_SENSORS_EXT_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_READ_SENSORS_EXT_IN_LEN,
+		MC_CMD_READ_SENSORS_EXT_OUT_LEN);
 	uint32_t addr_lo, addr_hi;
+	efx_rc_t rc;
+
+	if (EFSYS_MEM_SIZE(esmp) < size) {
+		rc = EINVAL;
+		goto fail1;
+	}
 
 	req.emr_cmd = MC_CMD_READ_SENSORS;
 	req.emr_in_buf = payload;
@@ -317,6 +217,11 @@ efx_mcdi_read_sensors(
 	efx_mcdi_execute(enp, &req);
 
 	return (req.emr_rc);
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
 }
 
 static	__checkReturn	efx_rc_t
@@ -325,8 +230,8 @@ efx_mcdi_sensor_info_npages(
 	__out		uint32_t *npagesp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_SENSOR_INFO_EXT_IN_LEN,
-			    MC_CMD_SENSOR_INFO_OUT_LENMAX)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SENSOR_INFO_EXT_IN_LEN,
+		MC_CMD_SENSOR_INFO_OUT_LENMAX);
 	int page;
 	efx_rc_t rc;
 
@@ -369,8 +274,8 @@ efx_mcdi_sensor_info(
 	__in			size_t npages)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_SENSOR_INFO_EXT_IN_LEN,
-			    MC_CMD_SENSOR_INFO_OUT_LENMAX)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SENSOR_INFO_EXT_IN_LEN,
+		MC_CMD_SENSOR_INFO_OUT_LENMAX);
 	uint32_t page;
 	efx_rc_t rc;
 
@@ -429,6 +334,86 @@ fail1:
 	return (rc);
 }
 
+static	__checkReturn		efx_rc_t
+efx_mcdi_sensor_info_page(
+	__in			efx_nic_t *enp,
+	__in			uint32_t page,
+	__out			uint32_t *mask_part,
+	__out_ecount((sizeof (*mask_part) * 8) - 1)
+				efx_mon_stat_limits_t *limits)
+{
+	efx_mcdi_req_t req;
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_SENSOR_INFO_EXT_IN_LEN,
+		MC_CMD_SENSOR_INFO_OUT_LENMAX);
+	efx_rc_t rc;
+	uint32_t mask_copy;
+	efx_dword_t *maskp;
+	efx_qword_t *limit_info;
+
+	EFSYS_ASSERT(mask_part != NULL);
+	EFSYS_ASSERT(limits != NULL);
+
+	memset(limits, 0,
+	    ((sizeof (*mask_part) * 8) - 1) * sizeof (efx_mon_stat_limits_t));
+
+	req.emr_cmd = MC_CMD_SENSOR_INFO;
+	req.emr_in_buf = payload;
+	req.emr_in_length = MC_CMD_SENSOR_INFO_EXT_IN_LEN;
+	req.emr_out_buf = payload;
+	req.emr_out_length = MC_CMD_SENSOR_INFO_OUT_LENMAX;
+
+	MCDI_IN_SET_DWORD(req, SENSOR_INFO_EXT_IN_PAGE, page);
+
+	efx_mcdi_execute(enp, &req);
+
+	rc = req.emr_rc;
+
+	if (rc != 0)
+		goto fail1;
+
+	EFSYS_ASSERT(sizeof (*limit_info) ==
+	    MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_LEN);
+	maskp = MCDI_OUT2(req, efx_dword_t, SENSOR_INFO_OUT_MASK);
+	limit_info = (efx_qword_t *)(maskp + 1);
+
+	*mask_part = maskp->ed_u32[0];
+	mask_copy = *mask_part;
+
+	/* Copy an entry for all but the highest bit set. */
+	while (mask_copy) {
+
+		if (mask_copy == (1U << MC_CMD_SENSOR_PAGE0_NEXT)) {
+			/* Only next page bit set. */
+			mask_copy = 0;
+		} else {
+			/* Clear lowest bit */
+			mask_copy = mask_copy & ~(mask_copy ^ (mask_copy - 1));
+			/* And copy out limit entry into buffer */
+			limits->emlv_warning_min = EFX_QWORD_FIELD(*limit_info,
+			    MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN1);
+
+			limits->emlv_warning_max = EFX_QWORD_FIELD(*limit_info,
+			    MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX1);
+
+			limits->emlv_fatal_min = EFX_QWORD_FIELD(*limit_info,
+			    MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN2);
+
+			limits->emlv_fatal_max = EFX_QWORD_FIELD(*limit_info,
+			    MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX2);
+
+			limits++;
+			limit_info++;
+		}
+	}
+
+	return (rc);
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
 	__checkReturn			efx_rc_t
 mcdi_mon_stats_update(
 	__in				efx_nic_t *enp,
@@ -457,6 +442,96 @@ fail1:
 	return (rc);
 }
 
+static		void
+lowest_set_bit(
+	__in	uint32_t input_mask,
+	__out	uint32_t *lowest_bit_mask,
+	__out	uint32_t *lowest_bit_num
+)
+{
+	uint32_t x;
+	uint32_t set_bit, bit_index;
+
+	x = (input_mask ^ (input_mask - 1));
+	set_bit = (x + 1) >> 1;
+	if (!set_bit)
+		set_bit = (1U << 31U);
+
+	bit_index = 0;
+	if (set_bit & 0xFFFF0000)
+		bit_index += 16;
+	if (set_bit & 0xFF00FF00)
+		bit_index += 8;
+	if (set_bit & 0xF0F0F0F0)
+		bit_index += 4;
+	if (set_bit & 0xCCCCCCCC)
+		bit_index += 2;
+	if (set_bit & 0xAAAAAAAA)
+		bit_index += 1;
+
+	*lowest_bit_mask = set_bit;
+	*lowest_bit_num = bit_index;
+}
+
+	__checkReturn			efx_rc_t
+mcdi_mon_limits_update(
+	__in				efx_nic_t *enp,
+	__inout_ecount(EFX_MON_NSTATS)	efx_mon_stat_limits_t *values)
+{
+	efx_rc_t rc;
+	uint32_t page;
+	uint32_t page_mask;
+	uint32_t limit_index;
+	efx_mon_stat_limits_t limits[sizeof (page_mask) * 8];
+	efx_mon_stat_t stat;
+
+	page = 0;
+	page--;
+	do {
+		page++;
+
+		rc = efx_mcdi_sensor_info_page(enp, page, &page_mask, limits);
+		if (rc != 0)
+			goto fail1;
+
+		limit_index = 0;
+		while (page_mask) {
+			uint32_t set_bit;
+			uint32_t page_index;
+			uint32_t mcdi_index;
+
+			if (page_mask == (1U << MC_CMD_SENSOR_PAGE0_NEXT))
+				break;
+
+			lowest_set_bit(page_mask, &set_bit, &page_index);
+			page_mask = page_mask & ~set_bit;
+
+			mcdi_index =
+			    page_index + (sizeof (page_mask) * 8 * page);
+
+			/*
+			 * This can fail if MCDI reports newer stats than the
+			 * drivers understand, or the bit is the next page bit.
+			 *
+			 * Driver needs to be tolerant of this.
+			 */
+			if (!efx_mon_mcdi_to_efx_stat(mcdi_index, &stat))
+				continue;
+
+			values[stat] = limits[limit_index];
+			limit_index++;
+		}
+
+	} while (page_mask & (1U << MC_CMD_SENSOR_PAGE0_NEXT));
+
+	return (rc);
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
 	__checkReturn	efx_rc_t
 mcdi_mon_cfg_build(
 	__in		efx_nic_t *enp)
diff --git a/drivers/net/sfc/base/mcdi_mon.h b/drivers/net/sfc/base/mcdi_mon.h
index 5aa6a6a2..5eba0901 100644
--- a/drivers/net/sfc/base/mcdi_mon.h
+++ b/drivers/net/sfc/base/mcdi_mon.h
@@ -39,6 +39,11 @@ mcdi_mon_stats_update(
 	__in				efsys_mem_t *esmp,
 	__inout_ecount(EFX_MON_NSTATS)	efx_mon_stat_value_t *values);
 
+extern	__checkReturn			efx_rc_t
+mcdi_mon_limits_update(
+	__in				efx_nic_t *enp,
+	__inout_ecount(EFX_MON_NSTATS)	efx_mon_stat_limits_t *values);
+
 #endif	/* EFSYS_OPT_MON_STATS */
 
 #endif /* EFSYS_OPT_MON_MCDI */
diff --git a/drivers/net/sfc/base/medford2_nic.c b/drivers/net/sfc/base/medford2_nic.c
index 7f5ad175..6bc1e87c 100644
--- a/drivers/net/sfc/base/medford2_nic.c
+++ b/drivers/net/sfc/base/medford2_nic.c
@@ -15,25 +15,15 @@ medford2_nic_get_required_pcie_bandwidth(
 	__in		efx_nic_t *enp,
 	__out		uint32_t *bandwidth_mbpsp)
 {
-	uint32_t port_modes;
-	uint32_t current_mode;
 	uint32_t bandwidth;
 	efx_rc_t rc;
 
 	/* FIXME: support new Medford2 dynamic port modes */
 
-	if ((rc = efx_mcdi_get_port_modes(enp, &port_modes,
-				    &current_mode)) != 0) {
-		/* No port mode info available. */
-		bandwidth = 0;
-		goto out;
-	}
-
-	if ((rc = ef10_nic_get_port_mode_bandwidth(current_mode,
+	if ((rc = ef10_nic_get_port_mode_bandwidth(enp,
 						    &bandwidth)) != 0)
 		goto fail1;
 
-out:
 	*bandwidth_mbpsp = bandwidth;
 
 	return (0);
@@ -96,6 +86,9 @@ medford2_board_cfg(
 	else
 		goto fail1;
 
+	/* Checksums for TSO sends should always be correct on Medford2. */
+	encp->enc_bug61297_workaround = B_FALSE;
+
 	/* Get clock frequencies (in MHz). */
 	if ((rc = efx_mcdi_get_clock(enp, &sysclk, &dpcpu_clk)) != 0)
 		goto fail2;
diff --git a/drivers/net/sfc/base/medford_nic.c b/drivers/net/sfc/base/medford_nic.c
index 6dc895f5..bfe01ca9 100644
--- a/drivers/net/sfc/base/medford_nic.c
+++ b/drivers/net/sfc/base/medford_nic.c
@@ -15,23 +15,13 @@ medford_nic_get_required_pcie_bandwidth(
 	__in		efx_nic_t *enp,
 	__out		uint32_t *bandwidth_mbpsp)
 {
-	uint32_t port_modes;
-	uint32_t current_mode;
 	uint32_t bandwidth;
 	efx_rc_t rc;
 
-	if ((rc = efx_mcdi_get_port_modes(enp, &port_modes,
-				    &current_mode)) != 0) {
-		/* No port mode info available. */
-		bandwidth = 0;
-		goto out;
-	}
-
-	if ((rc = ef10_nic_get_port_mode_bandwidth(current_mode,
+	if ((rc = ef10_nic_get_port_mode_bandwidth(enp,
 						    &bandwidth)) != 0)
 		goto fail1;
 
-out:
 	*bandwidth_mbpsp = bandwidth;
 
 	return (0);
@@ -94,6 +84,9 @@ medford_board_cfg(
 	else
 		goto fail1;
 
+	/* Checksums for TSO sends can be incorrect on Medford. */
+	encp->enc_bug61297_workaround = B_TRUE;
+
 	/* Get clock frequencies (in MHz). */
 	if ((rc = efx_mcdi_get_clock(enp, &sysclk, &dpcpu_clk)) != 0)
 		goto fail2;
diff --git a/drivers/net/sfc/base/meson.build b/drivers/net/sfc/base/meson.build
index da2bf44d..ab66f32f 100644
--- a/drivers/net/sfc/base/meson.build
+++ b/drivers/net/sfc/base/meson.build
@@ -58,6 +58,9 @@ extra_flags = [
 ]
 
 c_args = cflags
+if allow_experimental_apis
+	c_args += '-DALLOW_EXPERIMENTAL_API'
+endif
 foreach flag: extra_flags
 	if cc.has_argument(flag)
 		c_args += flag
diff --git a/drivers/net/sfc/base/siena_mac.c b/drivers/net/sfc/base/siena_mac.c
index f8857cdd..928dfc34 100644
--- a/drivers/net/sfc/base/siena_mac.c
+++ b/drivers/net/sfc/base/siena_mac.c
@@ -68,14 +68,13 @@ siena_mac_reconfigure(
 	efx_port_t *epp = &(enp->en_port);
 	efx_oword_t multicast_hash[2];
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MAX(MC_CMD_SET_MAC_IN_LEN,
-				MC_CMD_SET_MAC_OUT_LEN),
-			    MAX(MC_CMD_SET_MCAST_HASH_IN_LEN,
-				MC_CMD_SET_MCAST_HASH_OUT_LEN))];
+	EFX_MCDI_DECLARE_BUF(payload,
+		MAX(MC_CMD_SET_MAC_IN_LEN, MC_CMD_SET_MCAST_HASH_IN_LEN),
+		MAX(MC_CMD_SET_MAC_OUT_LEN, MC_CMD_SET_MCAST_HASH_OUT_LEN));
+
 	unsigned int fcntl;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_SET_MAC;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_SET_MAC_IN_LEN;
diff --git a/drivers/net/sfc/base/siena_nic.c b/drivers/net/sfc/base/siena_nic.c
index 31eef80b..fca17171 100644
--- a/drivers/net/sfc/base/siena_nic.c
+++ b/drivers/net/sfc/base/siena_nic.c
@@ -18,11 +18,10 @@ siena_nic_get_partn_mask(
 	__out			unsigned int *maskp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_NVRAM_TYPES_IN_LEN,
-			    MC_CMD_NVRAM_TYPES_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_NVRAM_TYPES_IN_LEN,
+		MC_CMD_NVRAM_TYPES_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_NVRAM_TYPES;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_NVRAM_TYPES_IN_LEN;
@@ -115,6 +114,7 @@ siena_board_cfg(
 	/* Alignment for WPTR updates */
 	encp->enc_rx_push_align = 1;
 
+#if EFSYS_OPT_RX_SCALE
 	/* There is one RSS context per function */
 	encp->enc_rx_scale_max_exclusive_contexts = 1;
 
@@ -129,6 +129,7 @@ siena_board_cfg(
 
 	/* There is no support for additional RSS modes */
 	encp->enc_rx_scale_additional_modes_supported = B_FALSE;
+#endif /* EFSYS_OPT_RX_SCALE */
 
 	encp->enc_tx_dma_desc_size_max = EFX_MASK32(FSF_AZ_TX_KER_BYTE_COUNT);
 	/* Fragments must not span 4k boundaries. */
diff --git a/drivers/net/sfc/base/siena_nvram.c b/drivers/net/sfc/base/siena_nvram.c
index 8cdd2df7..b8ea8a75 100644
--- a/drivers/net/sfc/base/siena_nvram.c
+++ b/drivers/net/sfc/base/siena_nvram.c
@@ -418,12 +418,11 @@ siena_nvram_get_subtype(
 	__out			uint32_t *subtypep)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_BOARD_CFG_IN_LEN,
-			    MC_CMD_GET_BOARD_CFG_OUT_LENMAX)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_BOARD_CFG_IN_LEN,
+		MC_CMD_GET_BOARD_CFG_OUT_LENMAX);
 	efx_word_t *fw_list;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_BOARD_CFG;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_BOARD_CFG_IN_LEN;
diff --git a/drivers/net/sfc/base/siena_phy.c b/drivers/net/sfc/base/siena_phy.c
index 4b2190d3..7eec9c74 100644
--- a/drivers/net/sfc/base/siena_phy.c
+++ b/drivers/net/sfc/base/siena_phy.c
@@ -169,11 +169,10 @@ siena_phy_get_link(
 	__out		siena_link_state_t *slsp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_LINK_IN_LEN,
-			    MC_CMD_GET_LINK_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_LINK_IN_LEN,
+		MC_CMD_GET_LINK_OUT_LEN);
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_LINK;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_LINK_IN_LEN;
@@ -244,10 +243,9 @@ siena_phy_reconfigure(
 {
 	efx_port_t *epp = &(enp->en_port);
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MAX(MC_CMD_SET_ID_LED_IN_LEN,
-				MC_CMD_SET_ID_LED_OUT_LEN),
-			    MAX(MC_CMD_SET_LINK_IN_LEN,
-				MC_CMD_SET_LINK_OUT_LEN))];
+	EFX_MCDI_DECLARE_BUF(payload,
+		MAX(MC_CMD_SET_ID_LED_IN_LEN, MC_CMD_SET_LINK_IN_LEN),
+		MAX(MC_CMD_SET_ID_LED_OUT_LEN, MC_CMD_SET_LINK_OUT_LEN));
 	uint32_t cap_mask;
 #if EFSYS_OPT_PHY_LED_CONTROL
 	unsigned int led_mode;
@@ -255,7 +253,6 @@ siena_phy_reconfigure(
 	unsigned int speed;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_SET_LINK;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_SET_LINK_IN_LEN;
@@ -361,12 +358,11 @@ siena_phy_verify(
 	__in		efx_nic_t *enp)
 {
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_GET_PHY_STATE_IN_LEN,
-			    MC_CMD_GET_PHY_STATE_OUT_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_PHY_STATE_IN_LEN,
+		MC_CMD_GET_PHY_STATE_OUT_LEN);
 	uint32_t state;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_PHY_STATE;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_PHY_STATE_IN_LEN;
@@ -530,8 +526,8 @@ siena_phy_stats_update(
 	uint32_t vmask = encp->enc_mcdi_phy_stat_mask;
 	uint64_t smask;
 	efx_mcdi_req_t req;
-	uint8_t payload[MAX(MC_CMD_PHY_STATS_IN_LEN,
-			    MC_CMD_PHY_STATS_OUT_DMA_LEN)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_PHY_STATS_IN_LEN,
+		MC_CMD_PHY_STATS_OUT_DMA_LEN);
 	efx_rc_t rc;
 
 	if ((esmp == NULL) || (EFSYS_MEM_SIZE(esmp) < EFX_PHY_STATS_SIZE)) {
@@ -539,7 +535,6 @@ siena_phy_stats_update(
 		goto fail1;
 	}
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_PHY_STATS;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_PHY_STATS_IN_LEN;
@@ -626,14 +621,13 @@ siena_phy_bist_poll(
 	__in			size_t count)
 {
 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
-	uint8_t payload[MAX(MC_CMD_POLL_BIST_IN_LEN,
-			    MCDI_CTL_SDU_LEN_MAX)];
+	EFX_MCDI_DECLARE_BUF(payload, MC_CMD_POLL_BIST_IN_LEN,
+		MCDI_CTL_SDU_LEN_MAX);
 	uint32_t value_mask = 0;
 	efx_mcdi_req_t req;
 	uint32_t result;
 	efx_rc_t rc;
 
-	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_POLL_BIST;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_POLL_BIST_IN_LEN;
diff --git a/drivers/net/sfc/efsys.h b/drivers/net/sfc/efsys.h
index b9d2df58..0b4795da 100644
--- a/drivers/net/sfc/efsys.h
+++ b/drivers/net/sfc/efsys.h
@@ -48,10 +48,6 @@ extern "C" {
 #include "efx_types.h"
 
 
-#ifndef _NOTE
-#define _NOTE(s)
-#endif
-
 typedef bool boolean_t;
 
 #ifndef B_FALSE
@@ -106,40 +102,6 @@ prefetch_read_once(const volatile void *addr)
 	rte_prefetch_non_temporal(addr);
 }
 
-/* Modifiers used for Windows builds */
-#define __in
-#define __in_opt
-#define __in_ecount(_n)
-#define __in_ecount_opt(_n)
-#define __in_bcount(_n)
-#define __in_bcount_opt(_n)
-
-#define __out
-#define __out_opt
-#define __out_ecount(_n)
-#define __out_ecount_opt(_n)
-#define __out_bcount(_n)
-#define __out_bcount_opt(_n)
-#define __out_bcount_part(_n, _l)
-#define __out_bcount_part_opt(_n, _l)
-
-#define __deref_out
-
-#define __inout
-#define __inout_opt
-#define __inout_ecount(_n)
-#define __inout_ecount_opt(_n)
-#define __inout_bcount(_n)
-#define __inout_bcount_opt(_n)
-#define __inout_bcount_full_opt(_n)
-
-#define __deref_out_bcount_opt(n)
-
-#define __checkReturn
-#define __success(_x)
-
-#define __drv_when(_p, _c)
-
 /* Code inclusion options */
 
 
diff --git a/drivers/net/sfc/sfc_dp_tx.h b/drivers/net/sfc/sfc_dp_tx.h
index eda9676c..c246871c 100644
--- a/drivers/net/sfc/sfc_dp_tx.h
+++ b/drivers/net/sfc/sfc_dp_tx.h
@@ -57,6 +57,11 @@ struct sfc_dp_tx_qcreate_info {
 	volatile void		*mem_bar;
 	/** VI window size shift */
 	unsigned int		vi_window_shift;
+	/**
+	 * Maximum number of bytes into the packet the TCP header can start for
+	 * the hardware to apply TSO packet edits.
+	 */
+	uint16_t		tso_tcp_header_offset_limit;
 };
 
 /**
diff --git a/drivers/net/sfc/sfc_ef10_essb_rx.c b/drivers/net/sfc/sfc_ef10_essb_rx.c
index 81c8f7fb..a24f54e7 100644
--- a/drivers/net/sfc/sfc_ef10_essb_rx.c
+++ b/drivers/net/sfc/sfc_ef10_essb_rx.c
@@ -123,14 +123,22 @@ static struct rte_mbuf *
 sfc_ef10_essb_next_mbuf(const struct sfc_ef10_essb_rxq *rxq,
 			struct rte_mbuf *mbuf)
 {
-	return (struct rte_mbuf *)((uintptr_t)mbuf + rxq->buf_stride);
+	struct rte_mbuf *m;
+
+	m = (struct rte_mbuf *)((uintptr_t)mbuf + rxq->buf_stride);
+	MBUF_RAW_ALLOC_CHECK(m);
+	return m;
 }
 
 static struct rte_mbuf *
 sfc_ef10_essb_mbuf_by_index(const struct sfc_ef10_essb_rxq *rxq,
 			    struct rte_mbuf *mbuf, unsigned int idx)
 {
-	return (struct rte_mbuf *)((uintptr_t)mbuf + idx * rxq->buf_stride);
+	struct rte_mbuf *m;
+
+	m = (struct rte_mbuf *)((uintptr_t)mbuf + idx * rxq->buf_stride);
+	MBUF_RAW_ALLOC_CHECK(m);
+	return m;
 }
 
 static struct rte_mbuf *
@@ -324,7 +332,7 @@ sfc_ef10_essb_rx_get_pending(struct sfc_ef10_essb_rxq *rxq,
 
 			/* Buffers to be discarded have 0 in packet type */
 			if (unlikely(m->packet_type == 0)) {
-				rte_mempool_put(rxq->refill_mb_pool, m);
+				rte_mbuf_raw_free(m);
 				goto next_buf;
 			}
 
@@ -687,7 +695,7 @@ sfc_ef10_essb_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 		m = sfc_ef10_essb_mbuf_by_index(rxq, rxd->first_mbuf,
 				rxq->block_size - rxq->left_in_completed);
 		while (rxq->left_in_completed > 0) {
-			rte_mempool_put(rxq->refill_mb_pool, m);
+			rte_mbuf_raw_free(m);
 			m = sfc_ef10_essb_next_mbuf(rxq, m);
 			rxq->left_in_completed--;
 		}
diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
index 6a5052b9..77ca580b 100644
--- a/drivers/net/sfc/sfc_ef10_rx.c
+++ b/drivers/net/sfc/sfc_ef10_rx.c
@@ -57,12 +57,13 @@ struct sfc_ef10_rxq {
 #define SFC_EF10_RXQ_EXCEPTION		0x4
 #define SFC_EF10_RXQ_RSS_HASH		0x8
 	unsigned int			ptr_mask;
-	unsigned int			prepared;
+	unsigned int			pending;
 	unsigned int			completed;
 	unsigned int			evq_read_ptr;
 	efx_qword_t			*evq_hw_ring;
 	struct sfc_ef10_rx_sw_desc	*sw_ring;
 	uint64_t			rearm_data;
+	struct rte_mbuf			*scatter_pkt;
 	uint16_t			prefix_size;
 
 	/* Used on refill */
@@ -133,6 +134,8 @@ sfc_ef10_rx_qrefill(struct sfc_ef10_rxq *rxq)
 			struct sfc_ef10_rx_sw_desc *rxd;
 			rte_iova_t phys_addr;
 
+			MBUF_RAW_ALLOC_CHECK(m);
+
 			SFC_ASSERT((id & ~ptr_mask) == 0);
 			rxd = &rxq->sw_ring[id];
 			rxd->mbuf = m;
@@ -184,21 +187,26 @@ sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id)
 	}
 }
 
-static uint16_t
-sfc_ef10_rx_prepared(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts,
-		     uint16_t nb_pkts)
+static struct rte_mbuf **
+sfc_ef10_rx_pending(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts,
+		    uint16_t nb_pkts)
 {
-	uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->prepared);
-	unsigned int completed = rxq->completed;
-	unsigned int i;
+	uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->pending - rxq->completed);
 
-	rxq->prepared -= n_rx_pkts;
-	rxq->completed = completed + n_rx_pkts;
+	SFC_ASSERT(rxq->pending == rxq->completed || rxq->scatter_pkt == NULL);
 
-	for (i = 0; i < n_rx_pkts; ++i, ++completed)
-		rx_pkts[i] = rxq->sw_ring[completed & rxq->ptr_mask].mbuf;
+	if (n_rx_pkts != 0) {
+		unsigned int completed = rxq->completed;
+
+		rxq->completed = completed + n_rx_pkts;
+
+		do {
+			*rx_pkts++ =
+				rxq->sw_ring[completed++ & rxq->ptr_mask].mbuf;
+		} while (completed != rxq->completed);
+	}
 
-	return n_rx_pkts;
+	return rx_pkts;
 }
 
 static uint16_t
@@ -213,47 +221,80 @@ sfc_ef10_rx_pseudo_hdr_get_hash(const uint8_t *pseudo_hdr)
 	return rte_le_to_cpu_32(*(const uint32_t *)pseudo_hdr);
 }
 
-static uint16_t
+static struct rte_mbuf **
 sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
-			  struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+			  struct rte_mbuf **rx_pkts,
+			  struct rte_mbuf ** const rx_pkts_end)
 {
 	const unsigned int ptr_mask = rxq->ptr_mask;
-	unsigned int completed = rxq->completed;
+	unsigned int pending = rxq->pending;
 	unsigned int ready;
 	struct sfc_ef10_rx_sw_desc *rxd;
 	struct rte_mbuf *m;
 	struct rte_mbuf *m0;
-	uint16_t n_rx_pkts;
 	const uint8_t *pseudo_hdr;
-	uint16_t pkt_len;
+	uint16_t seg_len;
 
-	ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - completed) &
+	ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - pending) &
 		EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
-	SFC_ASSERT(ready > 0);
+
+	if (ready == 0) {
+		/* Rx abort - it was no enough descriptors for Rx packet */
+		rte_pktmbuf_free(rxq->scatter_pkt);
+		rxq->scatter_pkt = NULL;
+		return rx_pkts;
+	}
+
+	rxq->pending = pending + ready;
 
 	if (rx_ev.eq_u64[0] &
 	    rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) |
 			     (1ull << ESF_DZ_RX_ECRC_ERR_LBN))) {
-		SFC_ASSERT(rxq->prepared == 0);
-		rxq->completed += ready;
-		while (ready-- > 0) {
-			rxd = &rxq->sw_ring[completed++ & ptr_mask];
-			rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf);
-		}
-		return 0;
+		SFC_ASSERT(rxq->completed == pending);
+		do {
+			rxd = &rxq->sw_ring[pending++ & ptr_mask];
+			rte_mbuf_raw_free(rxd->mbuf);
+		} while (pending != rxq->pending);
+		rxq->completed = pending;
+		return rx_pkts;
 	}
 
-	n_rx_pkts = RTE_MIN(ready, nb_pkts);
-	rxq->prepared = ready - n_rx_pkts;
-	rxq->completed += n_rx_pkts;
+	/* If scattered packet is in progress */
+	if (rxq->scatter_pkt != NULL) {
+		/* Events for scattered packet frags are not merged */
+		SFC_ASSERT(ready == 1);
+		SFC_ASSERT(rxq->completed == pending);
 
-	rxd = &rxq->sw_ring[completed++ & ptr_mask];
+		/* There is no pseudo-header in scatter segments. */
+		seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES);
 
-	sfc_ef10_rx_prefetch_next(rxq, completed & ptr_mask);
+		rxd = &rxq->sw_ring[pending++ & ptr_mask];
+		m = rxd->mbuf;
 
-	m = rxd->mbuf;
+		MBUF_RAW_ALLOC_CHECK(m);
+
+		m->data_off = RTE_PKTMBUF_HEADROOM;
+		rte_pktmbuf_data_len(m) = seg_len;
+		rte_pktmbuf_pkt_len(m) = seg_len;
 
-	*rx_pkts++ = m;
+		rxq->scatter_pkt->nb_segs++;
+		rte_pktmbuf_pkt_len(rxq->scatter_pkt) += seg_len;
+		rte_pktmbuf_lastseg(rxq->scatter_pkt)->next = m;
+
+		if (~rx_ev.eq_u64[0] &
+		    rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) {
+			*rx_pkts++ = rxq->scatter_pkt;
+			rxq->scatter_pkt = NULL;
+		}
+		rxq->completed = pending;
+		return rx_pkts;
+	}
+
+	rxd = &rxq->sw_ring[pending++ & ptr_mask];
+
+	sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask);
+
+	m = rxd->mbuf;
 
 	RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data));
 	m->rearm_data[0] = rxq->rearm_data;
@@ -275,27 +316,40 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 	m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr);
 
 	if (ready == 1)
-		pkt_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) -
+		seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) -
 			rxq->prefix_size;
 	else
-		pkt_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
-	SFC_ASSERT(pkt_len > 0);
-	rte_pktmbuf_data_len(m) = pkt_len;
-	rte_pktmbuf_pkt_len(m) = pkt_len;
+		seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
+	SFC_ASSERT(seg_len > 0);
+	rte_pktmbuf_data_len(m) = seg_len;
+	rte_pktmbuf_pkt_len(m) = seg_len;
 
 	SFC_ASSERT(m->next == NULL);
 
+	if (~rx_ev.eq_u64[0] & rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) {
+		*rx_pkts++ = m;
+		rxq->completed = pending;
+	} else {
+		/* Events with CONT bit are not merged */
+		SFC_ASSERT(ready == 1);
+		rxq->scatter_pkt = m;
+		rxq->completed = pending;
+		return rx_pkts;
+	}
+
 	/* Remember mbuf to copy offload flags and packet type from */
 	m0 = m;
-	for (--ready; ready > 0; --ready) {
-		rxd = &rxq->sw_ring[completed++ & ptr_mask];
+	while (pending != rxq->pending) {
+		rxd = &rxq->sw_ring[pending++ & ptr_mask];
 
-		sfc_ef10_rx_prefetch_next(rxq, completed & ptr_mask);
+		sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask);
 
 		m = rxd->mbuf;
 
-		if (ready > rxq->prepared)
+		if (rx_pkts != rx_pkts_end) {
 			*rx_pkts++ = m;
+			rxq->completed = pending;
+		}
 
 		RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) !=
 				 sizeof(rxq->rearm_data));
@@ -315,15 +369,15 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 		 */
 		m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr);
 
-		pkt_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
-		SFC_ASSERT(pkt_len > 0);
-		rte_pktmbuf_data_len(m) = pkt_len;
-		rte_pktmbuf_pkt_len(m) = pkt_len;
+		seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
+		SFC_ASSERT(seg_len > 0);
+		rte_pktmbuf_data_len(m) = seg_len;
+		rte_pktmbuf_pkt_len(m) = seg_len;
 
 		SFC_ASSERT(m->next == NULL);
 	}
 
-	return n_rx_pkts;
+	return rx_pkts;
 }
 
 static bool
@@ -355,26 +409,25 @@ static uint16_t
 sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 {
 	struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(rx_queue);
+	struct rte_mbuf ** const rx_pkts_end = &rx_pkts[nb_pkts];
 	unsigned int evq_old_read_ptr;
-	uint16_t n_rx_pkts;
 	efx_qword_t rx_ev;
 
+	rx_pkts = sfc_ef10_rx_pending(rxq, rx_pkts, nb_pkts);
+
 	if (unlikely(rxq->flags &
 		     (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION)))
-		return 0;
-
-	n_rx_pkts = sfc_ef10_rx_prepared(rxq, rx_pkts, nb_pkts);
+		goto done;
 
 	evq_old_read_ptr = rxq->evq_read_ptr;
-	while (n_rx_pkts != nb_pkts && sfc_ef10_rx_get_event(rxq, &rx_ev)) {
+	while (rx_pkts != rx_pkts_end && sfc_ef10_rx_get_event(rxq, &rx_ev)) {
 		/*
 		 * DROP_EVENT is an internal to the NIC, software should
 		 * never see it and, therefore, may ignore it.
 		 */
 
-		n_rx_pkts += sfc_ef10_rx_process_event(rxq, rx_ev,
-						       rx_pkts + n_rx_pkts,
-						       nb_pkts - n_rx_pkts);
+		rx_pkts = sfc_ef10_rx_process_event(rxq, rx_ev,
+						    rx_pkts, rx_pkts_end);
 	}
 
 	sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->ptr_mask, evq_old_read_ptr,
@@ -383,7 +436,8 @@ sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	/* It is not a problem if we refill in the case of exception */
 	sfc_ef10_rx_qrefill(rxq);
 
-	return n_rx_pkts;
+done:
+	return nb_pkts - (rx_pkts_end - rx_pkts);
 }
 
 const uint32_t *
@@ -446,21 +500,53 @@ sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps)
 
 static sfc_dp_rx_qdesc_npending_t sfc_ef10_rx_qdesc_npending;
 static unsigned int
-sfc_ef10_rx_qdesc_npending(__rte_unused struct sfc_dp_rxq *dp_rxq)
+sfc_ef10_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq)
 {
+	struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
+	efx_qword_t rx_ev;
+	const unsigned int evq_old_read_ptr = rxq->evq_read_ptr;
+	unsigned int pending = rxq->pending;
+	unsigned int ready;
+
+	if (unlikely(rxq->flags &
+		     (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION)))
+		goto done;
+
+	while (sfc_ef10_rx_get_event(rxq, &rx_ev)) {
+		ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) -
+			 pending) &
+			EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
+		pending += ready;
+	}
+
 	/*
-	 * Correct implementation requires EvQ polling and events
-	 * processing (keeping all ready mbufs in prepared).
+	 * The function does not process events, so return event queue read
+	 * pointer to the original position to allow the events that were
+	 * read to be processed later
 	 */
-	return -ENOTSUP;
+	rxq->evq_read_ptr = evq_old_read_ptr;
+
+done:
+	return pending - rxq->completed;
 }
 
 static sfc_dp_rx_qdesc_status_t sfc_ef10_rx_qdesc_status;
 static int
-sfc_ef10_rx_qdesc_status(__rte_unused struct sfc_dp_rxq *dp_rxq,
-			 __rte_unused uint16_t offset)
+sfc_ef10_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset)
 {
-	return -ENOTSUP;
+	struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
+	unsigned int npending = sfc_ef10_rx_qdesc_npending(dp_rxq);
+
+	if (unlikely(offset > rxq->ptr_mask))
+		return -EINVAL;
+
+	if (offset < npending)
+		return RTE_ETH_RX_DESC_DONE;
+
+	if (offset < (rxq->added - rxq->completed))
+		return RTE_ETH_RX_DESC_AVAIL;
+
+	return RTE_ETH_RX_DESC_UNAVAIL;
 }
 
 
@@ -594,8 +680,9 @@ sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr)
 {
 	struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 
-	rxq->prepared = 0;
-	rxq->completed = rxq->added = 0;
+	SFC_ASSERT(rxq->completed == 0);
+	SFC_ASSERT(rxq->pending == 0);
+	SFC_ASSERT(rxq->added == 0);
 
 	sfc_ef10_rx_qrefill(rxq);
 
@@ -642,12 +729,17 @@ sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 	unsigned int i;
 	struct sfc_ef10_rx_sw_desc *rxd;
 
+	rte_pktmbuf_free(rxq->scatter_pkt);
+	rxq->scatter_pkt = NULL;
+
 	for (i = rxq->completed; i != rxq->added; ++i) {
 		rxd = &rxq->sw_ring[i & rxq->ptr_mask];
-		rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf);
+		rte_mbuf_raw_free(rxd->mbuf);
 		rxd->mbuf = NULL;
 	}
 
+	rxq->completed = rxq->pending = rxq->added = 0;
+
 	rxq->flags &= ~SFC_EF10_RXQ_STARTED;
 }
 
@@ -657,7 +749,8 @@ struct sfc_dp_rx sfc_ef10_rx = {
 		.type		= SFC_DP_RX,
 		.hw_fw_caps	= SFC_DP_HW_FW_CAP_EF10,
 	},
-	.features		= SFC_DP_RX_FEAT_MULTI_PROCESS |
+	.features		= SFC_DP_RX_FEAT_SCATTER |
+				  SFC_DP_RX_FEAT_MULTI_PROCESS |
 				  SFC_DP_RX_FEAT_TUNNELS |
 				  SFC_DP_RX_FEAT_CHECKSUM,
 	.get_dev_info		= sfc_ef10_rx_get_dev_info,
diff --git a/drivers/net/sfc/sfc_ef10_tx.c b/drivers/net/sfc/sfc_ef10_tx.c
index d0daa3b3..bcd3153f 100644
--- a/drivers/net/sfc/sfc_ef10_tx.c
+++ b/drivers/net/sfc/sfc_ef10_tx.c
@@ -11,6 +11,8 @@
 
 #include <rte_mbuf.h>
 #include <rte_io.h>
+#include <rte_ip.h>
+#include <rte_tcp.h>
 
 #include "efx.h"
 #include "efx_types.h"
@@ -21,6 +23,7 @@
 #include "sfc_tweak.h"
 #include "sfc_kvargs.h"
 #include "sfc_ef10.h"
+#include "sfc_tso.h"
 
 #define sfc_ef10_tx_err(dpq, ...) \
 	SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__)
@@ -62,6 +65,9 @@ struct sfc_ef10_txq {
 	efx_qword_t			*txq_hw_ring;
 	volatile void			*doorbell;
 	efx_qword_t			*evq_hw_ring;
+	uint8_t				*tsoh;
+	rte_iova_t			tsoh_iova;
+	uint16_t			tso_tcp_header_offset_limit;
 
 	/* Datapath transmit queue anchor */
 	struct sfc_dp_txq		dp;
@@ -184,6 +190,30 @@ sfc_ef10_tx_qdesc_dma_create(rte_iova_t addr, uint16_t size, bool eop,
 			     ESF_DZ_TX_KER_BUF_ADDR, addr);
 }
 
+static void
+sfc_ef10_tx_qdesc_tso2_create(struct sfc_ef10_txq * const txq,
+			      unsigned int added, uint16_t ipv4_id,
+			      uint16_t outer_ipv4_id, uint32_t tcp_seq,
+			      uint16_t tcp_mss)
+{
+	EFX_POPULATE_QWORD_5(txq->txq_hw_ring[added & txq->ptr_mask],
+			    ESF_DZ_TX_DESC_IS_OPT, 1,
+			    ESF_DZ_TX_OPTION_TYPE,
+			    ESE_DZ_TX_OPTION_DESC_TSO,
+			    ESF_DZ_TX_TSO_OPTION_TYPE,
+			    ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A,
+			    ESF_DZ_TX_TSO_IP_ID, ipv4_id,
+			    ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq);
+	EFX_POPULATE_QWORD_5(txq->txq_hw_ring[(added + 1) & txq->ptr_mask],
+			    ESF_DZ_TX_DESC_IS_OPT, 1,
+			    ESF_DZ_TX_OPTION_TYPE,
+			    ESE_DZ_TX_OPTION_DESC_TSO,
+			    ESF_DZ_TX_TSO_OPTION_TYPE,
+			    ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B,
+			    ESF_DZ_TX_TSO_TCP_MSS, tcp_mss,
+			    ESF_DZ_TX_TSO_OUTER_IPID, outer_ipv4_id);
+}
+
 static inline void
 sfc_ef10_tx_qpush(struct sfc_ef10_txq *txq, unsigned int added,
 		  unsigned int pushed)
@@ -263,6 +293,252 @@ sfc_ef10_tx_pkt_descs_max(const struct rte_mbuf *m)
 				    extra_descs_per_pkt);
 }
 
+static bool
+sfc_ef10_try_reap(struct sfc_ef10_txq * const txq, unsigned int added,
+		  unsigned int needed_desc, unsigned int *dma_desc_space,
+		  bool *reap_done)
+{
+	if (*reap_done)
+		return false;
+
+	if (added != txq->added) {
+		sfc_ef10_tx_qpush(txq, added, txq->added);
+		txq->added = added;
+	}
+
+	sfc_ef10_tx_reap(txq);
+	*reap_done = true;
+
+	/*
+	 * Recalculate DMA descriptor space since Tx reap may change
+	 * the number of completed descriptors
+	 */
+	*dma_desc_space = txq->max_fill_level -
+		(added - txq->completed);
+
+	return (needed_desc <= *dma_desc_space);
+}
+
+static int
+sfc_ef10_xmit_tso_pkt(struct sfc_ef10_txq * const txq, struct rte_mbuf *m_seg,
+		      unsigned int *added, unsigned int *dma_desc_space,
+		      bool *reap_done)
+{
+	size_t iph_off = m_seg->l2_len;
+	size_t tcph_off = m_seg->l2_len + m_seg->l3_len;
+	size_t header_len = m_seg->l2_len + m_seg->l3_len + m_seg->l4_len;
+	/* Offset of the payload in the last segment that contains the header */
+	size_t in_off = 0;
+	const struct tcp_hdr *th;
+	uint16_t packet_id;
+	uint32_t sent_seq;
+	uint8_t *hdr_addr;
+	rte_iova_t hdr_iova;
+	struct rte_mbuf *first_m_seg = m_seg;
+	unsigned int pkt_start = *added;
+	unsigned int needed_desc;
+	struct rte_mbuf *m_seg_to_free_up_to = first_m_seg;
+	bool eop;
+
+	/* Both checks may be done, so use bit OR to have only one branching */
+	if (unlikely((header_len > SFC_TSOH_STD_LEN) |
+		     (tcph_off > txq->tso_tcp_header_offset_limit)))
+		return EMSGSIZE;
+
+	/*
+	 * Preliminary estimation of required DMA descriptors, including extra
+	 * descriptor for TSO header that is needed when the header is
+	 * separated from payload in one segment. It does not include
+	 * extra descriptors that may appear when a big segment is split across
+	 * several descriptors.
+	 */
+	needed_desc = m_seg->nb_segs +
+			(unsigned int)SFC_TSO_OPT_DESCS_NUM +
+			(unsigned int)SFC_TSO_HDR_DESCS_NUM;
+
+	if (needed_desc > *dma_desc_space &&
+	    !sfc_ef10_try_reap(txq, pkt_start, needed_desc,
+			       dma_desc_space, reap_done)) {
+		/*
+		 * If a future Tx reap may increase available DMA descriptor
+		 * space, do not try to send the packet.
+		 */
+		if (txq->completed != pkt_start)
+			return ENOSPC;
+		/*
+		 * Do not allow to send packet if the maximum DMA
+		 * descriptor space is not sufficient to hold TSO
+		 * descriptors, header descriptor and at least 1
+		 * segment descriptor.
+		 */
+		if (*dma_desc_space < SFC_TSO_OPT_DESCS_NUM +
+				SFC_TSO_HDR_DESCS_NUM + 1)
+			return EMSGSIZE;
+	}
+
+	/* Check if the header is not fragmented */
+	if (rte_pktmbuf_data_len(m_seg) >= header_len) {
+		hdr_addr = rte_pktmbuf_mtod(m_seg, uint8_t *);
+		hdr_iova = rte_mbuf_data_iova(m_seg);
+		if (rte_pktmbuf_data_len(m_seg) == header_len) {
+			/*
+			 * Associate header mbuf with header descriptor
+			 * which is located after TSO descriptors.
+			 */
+			txq->sw_ring[(pkt_start + SFC_TSO_OPT_DESCS_NUM) &
+				     txq->ptr_mask].mbuf = m_seg;
+			m_seg = m_seg->next;
+			in_off = 0;
+
+			/*
+			 * If there is no payload offset (payload starts at the
+			 * beginning of a segment) then an extra descriptor for
+			 * separated header is not needed.
+			 */
+			needed_desc--;
+		} else {
+			in_off = header_len;
+		}
+	} else {
+		unsigned int copied_segs;
+		unsigned int hdr_addr_off = (*added & txq->ptr_mask) *
+				SFC_TSOH_STD_LEN;
+
+		hdr_addr = txq->tsoh + hdr_addr_off;
+		hdr_iova = txq->tsoh_iova + hdr_addr_off;
+		copied_segs = sfc_tso_prepare_header(hdr_addr, header_len,
+						     &m_seg, &in_off);
+
+		m_seg_to_free_up_to = m_seg;
+		/*
+		 * Reduce the number of needed descriptors by the number of
+		 * segments that entirely consist of header data.
+		 */
+		needed_desc -= copied_segs;
+
+		/* Extra descriptor for separated header is not needed */
+		if (in_off == 0)
+			needed_desc--;
+	}
+
+	switch (first_m_seg->ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)) {
+	case PKT_TX_IPV4: {
+		const struct ipv4_hdr *iphe4;
+
+		iphe4 = (const struct ipv4_hdr *)(hdr_addr + iph_off);
+		rte_memcpy(&packet_id, &iphe4->packet_id, sizeof(uint16_t));
+		packet_id = rte_be_to_cpu_16(packet_id);
+		break;
+	}
+	case PKT_TX_IPV6:
+		packet_id = 0;
+		break;
+	default:
+		return EINVAL;
+	}
+
+	th = (const struct tcp_hdr *)(hdr_addr + tcph_off);
+	rte_memcpy(&sent_seq, &th->sent_seq, sizeof(uint32_t));
+	sent_seq = rte_be_to_cpu_32(sent_seq);
+
+	sfc_ef10_tx_qdesc_tso2_create(txq, *added, packet_id, 0, sent_seq,
+			first_m_seg->tso_segsz);
+	(*added) += SFC_TSO_OPT_DESCS_NUM;
+
+	sfc_ef10_tx_qdesc_dma_create(hdr_iova, header_len, false,
+			&txq->txq_hw_ring[(*added) & txq->ptr_mask]);
+	(*added)++;
+
+	do {
+		rte_iova_t next_frag = rte_mbuf_data_iova(m_seg);
+		unsigned int seg_len = rte_pktmbuf_data_len(m_seg);
+		unsigned int id;
+
+		next_frag += in_off;
+		seg_len -= in_off;
+		in_off = 0;
+
+		do {
+			rte_iova_t frag_addr = next_frag;
+			size_t frag_len;
+
+			frag_len = RTE_MIN(seg_len,
+					   SFC_EF10_TX_DMA_DESC_LEN_MAX);
+
+			next_frag += frag_len;
+			seg_len -= frag_len;
+
+			eop = (seg_len == 0 && m_seg->next == NULL);
+
+			id = (*added) & txq->ptr_mask;
+			(*added)++;
+
+			/*
+			 * Initially we assume that one DMA descriptor is needed
+			 * for every segment. When the segment is split across
+			 * several DMA descriptors, increase the estimation.
+			 */
+			needed_desc += (seg_len != 0);
+
+			/*
+			 * When no more descriptors can be added, but not all
+			 * segments are processed.
+			 */
+			if (*added - pkt_start == *dma_desc_space &&
+			    !eop &&
+			    !sfc_ef10_try_reap(txq, pkt_start, needed_desc,
+						dma_desc_space, reap_done)) {
+				struct rte_mbuf *m;
+				struct rte_mbuf *m_next;
+
+				if (txq->completed != pkt_start) {
+					unsigned int i;
+
+					/*
+					 * Reset mbuf associations with added
+					 * descriptors.
+					 */
+					for (i = pkt_start; i != *added; i++) {
+						id = i & txq->ptr_mask;
+						txq->sw_ring[id].mbuf = NULL;
+					}
+					return ENOSPC;
+				}
+
+				/* Free the segments that cannot be sent */
+				for (m = m_seg->next; m != NULL; m = m_next) {
+					m_next = m->next;
+					rte_pktmbuf_free_seg(m);
+				}
+				eop = true;
+				/* Ignore the rest of the segment */
+				seg_len = 0;
+			}
+
+			sfc_ef10_tx_qdesc_dma_create(frag_addr, frag_len,
+					eop, &txq->txq_hw_ring[id]);
+
+		} while (seg_len != 0);
+
+		txq->sw_ring[id].mbuf = m_seg;
+
+		m_seg = m_seg->next;
+	} while (!eop);
+
+	/*
+	 * Free segments which content was entirely copied to the TSO header
+	 * memory space of Tx queue
+	 */
+	for (m_seg = first_m_seg; m_seg != m_seg_to_free_up_to;) {
+		struct rte_mbuf *seg_to_free = m_seg;
+
+		m_seg = m_seg->next;
+		rte_pktmbuf_free_seg(seg_to_free);
+	}
+
+	return 0;
+}
+
 static uint16_t
 sfc_ef10_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 {
@@ -296,6 +572,30 @@ sfc_ef10_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		if (likely(pktp + 1 != pktp_end))
 			rte_mbuf_prefetch_part1(pktp[1]);
 
+		if (m_seg->ol_flags & PKT_TX_TCP_SEG) {
+			int rc;
+
+			rc = sfc_ef10_xmit_tso_pkt(txq, m_seg, &added,
+					&dma_desc_space, &reap_done);
+			if (rc != 0) {
+				added = pkt_start;
+
+				/* Packet can be sent in following xmit calls */
+				if (likely(rc == ENOSPC))
+					break;
+
+				/*
+				 * Packet cannot be sent, tell RTE that
+				 * it is sent, but actually drop it and
+				 * continue with another packet
+				 */
+				rte_pktmbuf_free(*pktp);
+				continue;
+			}
+
+			goto dma_desc_space_update;
+		}
+
 		if (sfc_ef10_tx_pkt_descs_max(m_seg) > dma_desc_space) {
 			if (reap_done)
 				break;
@@ -349,6 +649,7 @@ sfc_ef10_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 
 		} while ((m_seg = m_seg->next) != 0);
 
+dma_desc_space_update:
 		dma_desc_space -= (added - pkt_start);
 	}
 
@@ -524,6 +825,18 @@ sfc_ef10_tx_qcreate(uint16_t port_id, uint16_t queue_id,
 	if (txq->sw_ring == NULL)
 		goto fail_sw_ring_alloc;
 
+	if (info->offloads & DEV_TX_OFFLOAD_TCP_TSO) {
+		txq->tsoh = rte_calloc_socket("sfc-ef10-txq-tsoh",
+					      info->txq_entries,
+					      SFC_TSOH_STD_LEN,
+					      RTE_CACHE_LINE_SIZE,
+					      socket_id);
+		if (txq->tsoh == NULL)
+			goto fail_tsoh_alloc;
+
+		txq->tsoh_iova = rte_malloc_virt2iova(txq->tsoh);
+	}
+
 	txq->flags = SFC_EF10_TXQ_NOT_RUNNING;
 	txq->ptr_mask = info->txq_entries - 1;
 	txq->max_fill_level = info->max_fill_level;
@@ -533,10 +846,14 @@ sfc_ef10_tx_qcreate(uint16_t port_id, uint16_t queue_id,
 			ER_DZ_TX_DESC_UPD_REG_OFST +
 			(info->hw_index << info->vi_window_shift);
 	txq->evq_hw_ring = info->evq_hw_ring;
+	txq->tso_tcp_header_offset_limit = info->tso_tcp_header_offset_limit;
 
 	*dp_txqp = &txq->dp;
 	return 0;
 
+fail_tsoh_alloc:
+	rte_free(txq->sw_ring);
+
 fail_sw_ring_alloc:
 	rte_free(txq);
 
@@ -551,6 +868,7 @@ sfc_ef10_tx_qdestroy(struct sfc_dp_txq *dp_txq)
 {
 	struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
 
+	rte_free(txq->tsoh);
 	rte_free(txq->sw_ring);
 	rte_free(txq);
 }
@@ -618,12 +936,49 @@ sfc_ef10_tx_qreap(struct sfc_dp_txq *dp_txq)
 	txq->flags &= ~SFC_EF10_TXQ_STARTED;
 }
 
+static unsigned int
+sfc_ef10_tx_qdesc_npending(struct sfc_ef10_txq *txq)
+{
+	const unsigned int curr_done = txq->completed - 1;
+	unsigned int anew_done = curr_done;
+	efx_qword_t tx_ev;
+	const unsigned int evq_old_read_ptr = txq->evq_read_ptr;
+
+	if (unlikely(txq->flags &
+		     (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION)))
+		return 0;
+
+	while (sfc_ef10_tx_get_event(txq, &tx_ev))
+		anew_done = EFX_QWORD_FIELD(tx_ev, ESF_DZ_TX_DESCR_INDX);
+
+	/*
+	 * The function does not process events, so return event queue read
+	 * pointer to the original position to allow the events that were
+	 * read to be processed later
+	 */
+	txq->evq_read_ptr = evq_old_read_ptr;
+
+	return (anew_done - curr_done) & txq->ptr_mask;
+}
+
 static sfc_dp_tx_qdesc_status_t sfc_ef10_tx_qdesc_status;
 static int
-sfc_ef10_tx_qdesc_status(__rte_unused struct sfc_dp_txq *dp_txq,
-			 __rte_unused uint16_t offset)
+sfc_ef10_tx_qdesc_status(struct sfc_dp_txq *dp_txq,
+			 uint16_t offset)
 {
-	return -ENOTSUP;
+	struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
+	unsigned int npending = sfc_ef10_tx_qdesc_npending(txq);
+
+	if (unlikely(offset > txq->ptr_mask))
+		return -EINVAL;
+
+	if (unlikely(offset >= txq->max_fill_level))
+		return RTE_ETH_TX_DESC_UNAVAIL;
+
+	if (unlikely(offset < npending))
+		return RTE_ETH_TX_DESC_FULL;
+
+	return RTE_ETH_TX_DESC_DONE;
 }
 
 struct sfc_dp_tx sfc_ef10_tx = {
@@ -632,7 +987,8 @@ struct sfc_dp_tx sfc_ef10_tx = {
 		.type		= SFC_DP_TX,
 		.hw_fw_caps	= SFC_DP_HW_FW_CAP_EF10,
 	},
-	.features		= SFC_DP_TX_FEAT_MULTI_SEG |
+	.features		= SFC_DP_TX_FEAT_TSO |
+				  SFC_DP_TX_FEAT_MULTI_SEG |
 				  SFC_DP_TX_FEAT_MULTI_POOL |
 				  SFC_DP_TX_FEAT_REFCNT |
 				  SFC_DP_TX_FEAT_MULTI_PROCESS,
diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c
index 9decbf5a..3886daf7 100644
--- a/drivers/net/sfc/sfc_ethdev.c
+++ b/drivers/net/sfc/sfc_ethdev.c
@@ -171,6 +171,9 @@ sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 		sa->dp_rx->get_dev_info(dev_info);
 	if (sa->dp_tx->get_dev_info != NULL)
 		sa->dp_tx->get_dev_info(dev_info);
+
+	dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP |
+			     RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP;
 }
 
 static const uint32_t *
@@ -441,8 +444,6 @@ sfc_rx_queue_release(void *queue)
 
 	sfc_log_init(sa, "RxQ=%u", sw_index);
 
-	sa->eth_dev->data->rx_queues[sw_index] = NULL;
-
 	sfc_rx_qfini(sa, sw_index);
 
 	sfc_adapter_unlock(sa);
@@ -497,9 +498,6 @@ sfc_tx_queue_release(void *queue)
 
 	sfc_adapter_lock(sa);
 
-	SFC_ASSERT(sw_index < sa->eth_dev->data->nb_tx_queues);
-	sa->eth_dev->data->tx_queues[sw_index] = NULL;
-
 	sfc_tx_qfini(sa, sw_index);
 
 	sfc_adapter_unlock(sa);
@@ -1143,6 +1141,9 @@ sfc_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	if (sa->state != SFC_ADAPTER_STARTED)
 		goto fail_not_started;
 
+	if (sa->rxq_info[rx_queue_id].rxq == NULL)
+		goto fail_not_setup;
+
 	rc = sfc_rx_qstart(sa, rx_queue_id);
 	if (rc != 0)
 		goto fail_rx_qstart;
@@ -1154,6 +1155,7 @@ sfc_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	return 0;
 
 fail_rx_qstart:
+fail_not_setup:
 fail_not_started:
 	sfc_adapter_unlock(sa);
 	SFC_ASSERT(rc > 0);
@@ -1191,6 +1193,9 @@ sfc_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	if (sa->state != SFC_ADAPTER_STARTED)
 		goto fail_not_started;
 
+	if (sa->txq_info[tx_queue_id].txq == NULL)
+		goto fail_not_setup;
+
 	rc = sfc_tx_qstart(sa, tx_queue_id);
 	if (rc != 0)
 		goto fail_tx_qstart;
@@ -1202,6 +1207,7 @@ sfc_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 
 fail_tx_qstart:
 
+fail_not_setup:
 fail_not_started:
 	sfc_adapter_unlock(sa);
 	SFC_ASSERT(rc > 0);
@@ -1348,14 +1354,10 @@ sfc_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 {
 	struct sfc_adapter *sa = dev->data->dev_private;
 	struct sfc_rss *rss = &sa->rss;
-	struct sfc_port *port = &sa->port;
 
-	if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE || port->isolated)
+	if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE)
 		return -ENOTSUP;
 
-	if (rss->channels == 0)
-		return -EINVAL;
-
 	sfc_adapter_lock(sa);
 
 	/*
@@ -2033,9 +2035,6 @@ sfc_eth_dev_uninit(struct rte_eth_dev *dev)
 	sfc_detach(sa);
 	sfc_unprobe(sa);
 
-	rte_free(dev->data->mac_addrs);
-	dev->data->mac_addrs = NULL;
-
 	sfc_kvargs_cleanup(sa);
 
 	sfc_adapter_unlock(sa);
diff --git a/drivers/net/sfc/sfc_rx.c b/drivers/net/sfc/sfc_rx.c
index d8503e20..c792e0b2 100644
--- a/drivers/net/sfc/sfc_rx.c
+++ b/drivers/net/sfc/sfc_rx.c
@@ -96,13 +96,12 @@ sfc_efx_rx_qrefill(struct sfc_efx_rxq *rxq)
 		     ++i, id = (id + 1) & rxq->ptr_mask) {
 			m = objs[i];
 
+			MBUF_RAW_ALLOC_CHECK(m);
+
 			rxd = &rxq->sw_desc[id];
 			rxd->mbuf = m;
 
-			SFC_ASSERT(rte_mbuf_refcnt_read(m) == 1);
 			m->data_off = RTE_PKTMBUF_HEADROOM;
-			SFC_ASSERT(m->next == NULL);
-			SFC_ASSERT(m->nb_segs == 1);
 			m->port = port_id;
 
 			addr[i] = rte_pktmbuf_iova(m);
@@ -296,7 +295,7 @@ sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 discard:
 		discard_next = ((desc_flags & EFX_PKT_CONT) != 0);
-		rte_mempool_put(rxq->refill_mb_pool, m);
+		rte_mbuf_raw_free(m);
 		rxd->mbuf = NULL;
 	}
 
@@ -498,7 +497,7 @@ sfc_efx_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 
 	for (i = rxq->completed; i != rxq->added; ++i) {
 		rxd = &rxq->sw_desc[i & rxq->ptr_mask];
-		rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf);
+		rte_mbuf_raw_free(rxd->mbuf);
 		rxd->mbuf = NULL;
 		/* Packed stream relies on 0 in inactive SW desc.
 		 * Rx queue stop is not performance critical, so
@@ -673,6 +672,7 @@ sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index)
 
 	rxq_info = &sa->rxq_info[sw_index];
 	rxq = rxq_info->rxq;
+	SFC_ASSERT(rxq != NULL);
 	SFC_ASSERT(rxq->state == SFC_RXQ_INITIALIZED);
 
 	evq = rxq->evq;
@@ -763,7 +763,7 @@ sfc_rx_qstop(struct sfc_adapter *sa, unsigned int sw_index)
 	rxq_info = &sa->rxq_info[sw_index];
 	rxq = rxq_info->rxq;
 
-	if (rxq->state == SFC_RXQ_INITIALIZED)
+	if (rxq == NULL || rxq->state == SFC_RXQ_INITIALIZED)
 		return;
 	SFC_ASSERT(rxq->state & SFC_RXQ_STARTED);
 
@@ -792,7 +792,6 @@ sfc_rx_get_dev_offload_caps(struct sfc_adapter *sa)
 	uint64_t caps = 0;
 
 	caps |= DEV_RX_OFFLOAD_JUMBO_FRAME;
-	caps |= DEV_RX_OFFLOAD_CRC_STRIP;
 
 	if (sa->dp_rx->features & SFC_DP_RX_FEAT_CHECKSUM) {
 		caps |= DEV_RX_OFFLOAD_IPV4_CKSUM;
@@ -1103,6 +1102,7 @@ sfc_rx_qfini(struct sfc_adapter *sa, unsigned int sw_index)
 	struct sfc_rxq *rxq;
 
 	SFC_ASSERT(sw_index < sa->rxq_count);
+	sa->eth_dev->data->rx_queues[sw_index] = NULL;
 
 	rxq_info = &sa->rxq_info[sw_index];
 
@@ -1126,7 +1126,7 @@ sfc_rx_qfini(struct sfc_adapter *sa, unsigned int sw_index)
 /*
  * Mapping between RTE RSS hash functions and their EFX counterparts.
  */
-struct sfc_rss_hf_rte_to_efx sfc_rss_hf_map[] = {
+static const struct sfc_rss_hf_rte_to_efx sfc_rss_hf_map[] = {
 	{ ETH_RSS_NONFRAG_IPV4_TCP,
 	  EFX_RX_HASH(IPV4_TCP, 4TUPLE) },
 	{ ETH_RSS_NONFRAG_IPV4_UDP,
@@ -1200,7 +1200,7 @@ sfc_rx_hash_init(struct sfc_adapter *sa)
 		return EINVAL;
 
 	rc = efx_rx_scale_hash_flags_get(sa->nic, alg, flags_supp,
-					 &nb_flags_supp);
+					 RTE_DIM(flags_supp), &nb_flags_supp);
 	if (rc != 0)
 		return rc;
 
@@ -1363,7 +1363,8 @@ sfc_rx_start(struct sfc_adapter *sa)
 		goto fail_rss_config;
 
 	for (sw_index = 0; sw_index < sa->rxq_count; ++sw_index) {
-		if ((!sa->rxq_info[sw_index].deferred_start ||
+		if (sa->rxq_info[sw_index].rxq != NULL &&
+		    (!sa->rxq_info[sw_index].deferred_start ||
 		     sa->rxq_info[sw_index].deferred_started)) {
 			rc = sfc_rx_qstart(sa, sw_index);
 			if (rc != 0)
@@ -1439,14 +1440,6 @@ sfc_rx_check_mode(struct sfc_adapter *sa, struct rte_eth_rxmode *rxmode)
 		rc = EINVAL;
 	}
 
-	/* KEEP_CRC offload flag is not supported by PMD
-	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
-	 */
-	if (rte_eth_dev_must_keep_crc(rxmode->offloads)) {
-		sfc_warn(sa, "FCS stripping cannot be disabled - always on");
-		rxmode->offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
-	}
-
 	/*
 	 * Requested offloads are validated against supported by ethdev,
 	 * so unsupported offloads cannot be added as the result of
@@ -1511,7 +1504,7 @@ sfc_rx_configure(struct sfc_adapter *sa)
 		goto fail_check_mode;
 
 	if (nb_rx_queues == sa->rxq_count)
-		goto done;
+		goto configure_rss;
 
 	if (sa->rxq_info == NULL) {
 		rc = ENOMEM;
@@ -1548,6 +1541,7 @@ sfc_rx_configure(struct sfc_adapter *sa)
 		sa->rxq_count++;
 	}
 
+configure_rss:
 	rss->channels = (dev_conf->rxmode.mq_mode == ETH_MQ_RX_RSS) ?
 			 MIN(sa->rxq_count, EFX_MAXRSS) : 0;
 
@@ -1564,7 +1558,6 @@ sfc_rx_configure(struct sfc_adapter *sa)
 			goto fail_rx_process_adv_conf_rss;
 	}
 
-done:
 	return 0;
 
 fail_rx_process_adv_conf_rss:
diff --git a/drivers/net/sfc/sfc_tso.c b/drivers/net/sfc/sfc_tso.c
index effe9853..076a25d4 100644
--- a/drivers/net/sfc/sfc_tso.c
+++ b/drivers/net/sfc/sfc_tso.c
@@ -14,12 +14,7 @@
 #include "sfc_debug.h"
 #include "sfc_tx.h"
 #include "sfc_ev.h"
-
-/** Standard TSO header length */
-#define SFC_TSOH_STD_LEN        256
-
-/** The number of TSO option descriptors that precede the packet descriptors */
-#define SFC_TSO_OPDESCS_IDX_SHIFT	2
+#include "sfc_tso.h"
 
 int
 sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
@@ -57,13 +52,14 @@ sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
 	}
 }
 
-static void
-sfc_efx_tso_prepare_header(struct sfc_efx_txq *txq, struct rte_mbuf **in_seg,
-			   size_t *in_off, unsigned int idx, size_t bytes_left)
+unsigned int
+sfc_tso_prepare_header(uint8_t *tsoh, size_t header_len,
+		       struct rte_mbuf **in_seg, size_t *in_off)
 {
 	struct rte_mbuf *m = *in_seg;
 	size_t bytes_to_copy = 0;
-	uint8_t *tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh;
+	size_t bytes_left = header_len;
+	unsigned int segments_copied = 0;
 
 	do {
 		bytes_to_copy = MIN(bytes_left, m->data_len);
@@ -77,16 +73,20 @@ sfc_efx_tso_prepare_header(struct sfc_efx_txq *txq, struct rte_mbuf **in_seg,
 		if (bytes_left > 0) {
 			m = m->next;
 			SFC_ASSERT(m != NULL);
+			segments_copied++;
 		}
 	} while (bytes_left > 0);
 
 	if (bytes_to_copy == m->data_len) {
 		*in_seg = m->next;
 		*in_off = 0;
+		segments_copied++;
 	} else {
 		*in_seg = m;
 		*in_off = bytes_to_copy;
 	}
+
+	return segments_copied;
 }
 
 int
@@ -105,7 +105,7 @@ sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx,
 	size_t header_len = m->l2_len + m->l3_len + m->l4_len;
 	const efx_nic_cfg_t *encp = efx_nic_cfg_get(txq->evq->sa->nic);
 
-	idx += SFC_TSO_OPDESCS_IDX_SHIFT;
+	idx += SFC_TSO_OPT_DESCS_NUM;
 
 	/* Packets which have too big headers should be discarded */
 	if (unlikely(header_len > SFC_TSOH_STD_LEN))
@@ -129,9 +129,8 @@ sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx,
 	 * limitations on address boundaries crossing by DMA descriptor data.
 	 */
 	if (m->data_len < header_len) {
-		sfc_efx_tso_prepare_header(txq, in_seg, in_off, idx,
-					   header_len);
 		tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh;
+		sfc_tso_prepare_header(tsoh, header_len, in_seg, in_off);
 
 		header_paddr = rte_malloc_virt2iova((void *)tsoh);
 	} else {
diff --git a/drivers/net/sfc/sfc_tso.h b/drivers/net/sfc/sfc_tso.h
new file mode 100644
index 00000000..3d2faf54
--- /dev/null
+++ b/drivers/net/sfc/sfc_tso.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 2018 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ */
+
+/** Standard TSO header length */
+#define SFC_TSOH_STD_LEN	256
+
+/** The number of TSO option descriptors that precede the packet descriptors */
+#define SFC_TSO_OPT_DESCS_NUM	2
+
+/**
+ * The number of DMA descriptors for TSO header that may or may not precede the
+ * packet's payload descriptors
+ */
+#define SFC_TSO_HDR_DESCS_NUM	1
+
+unsigned int sfc_tso_prepare_header(uint8_t *tsoh, size_t header_len,
+				    struct rte_mbuf **in_seg, size_t *in_off);
diff --git a/drivers/net/sfc/sfc_tx.c b/drivers/net/sfc/sfc_tx.c
index 6d42a1a6..147f9336 100644
--- a/drivers/net/sfc/sfc_tx.c
+++ b/drivers/net/sfc/sfc_tx.c
@@ -190,6 +190,8 @@ sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
 	info.hw_index = txq->hw_index;
 	info.mem_bar = sa->mem_bar.esb_base;
 	info.vi_window_shift = encp->enc_vi_window_shift;
+	info.tso_tcp_header_offset_limit =
+		encp->enc_tx_tso_tcp_header_offset_limit;
 
 	rc = sa->dp_tx->qcreate(sa->eth_dev->data->port_id, sw_index,
 				&RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr,
@@ -233,6 +235,8 @@ sfc_tx_qfini(struct sfc_adapter *sa, unsigned int sw_index)
 	sfc_log_init(sa, "TxQ = %u", sw_index);
 
 	SFC_ASSERT(sw_index < sa->txq_count);
+	sa->eth_dev->data->tx_queues[sw_index] = NULL;
+
 	txq_info = &sa->txq_info[sw_index];
 
 	txq = txq_info->txq;
@@ -421,6 +425,7 @@ sfc_tx_qstart(struct sfc_adapter *sa, unsigned int sw_index)
 
 	txq = txq_info->txq;
 
+	SFC_ASSERT(txq != NULL);
 	SFC_ASSERT(txq->state == SFC_TXQ_INITIALIZED);
 
 	evq = txq->evq;
@@ -501,7 +506,7 @@ sfc_tx_qstop(struct sfc_adapter *sa, unsigned int sw_index)
 
 	txq = txq_info->txq;
 
-	if (txq->state == SFC_TXQ_INITIALIZED)
+	if (txq == NULL || txq->state == SFC_TXQ_INITIALIZED)
 		return;
 
 	SFC_ASSERT(txq->state & SFC_TXQ_STARTED);
@@ -578,8 +583,9 @@ sfc_tx_start(struct sfc_adapter *sa)
 		goto fail_efx_tx_init;
 
 	for (sw_index = 0; sw_index < sa->txq_count; ++sw_index) {
-		if (!(sa->txq_info[sw_index].deferred_start) ||
-		    sa->txq_info[sw_index].deferred_started) {
+		if (sa->txq_info[sw_index].txq != NULL &&
+		    (!(sa->txq_info[sw_index].deferred_start) ||
+		     sa->txq_info[sw_index].deferred_started)) {
 			rc = sfc_tx_qstart(sa, sw_index);
 			if (rc != 0)
 				goto fail_tx_qstart;
diff --git a/drivers/net/softnic/Makefile b/drivers/net/softnic/Makefile
index ea9b65f4..484e76cd 100644
--- a/drivers/net/softnic/Makefile
+++ b/drivers/net/softnic/Makefile
@@ -14,6 +14,7 @@ CFLAGS += $(WERROR_FLAGS)
 LDLIBS += -lrte_pipeline -lrte_port -lrte_table
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_sched
+LDLIBS += -lrte_cryptodev
 LDLIBS += -lrte_bus_vdev
 
 EXPORT_MAP := rte_pmd_softnic_version.map
@@ -33,6 +34,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_action.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_pipeline.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_thread.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_cli.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_flow.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_meter.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_cryptodev.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += parser.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += conn.c
 
diff --git a/drivers/net/softnic/conn.c b/drivers/net/softnic/conn.c
index 990cf40f..8b665808 100644
--- a/drivers/net/softnic/conn.c
+++ b/drivers/net/softnic/conn.c
@@ -8,7 +8,6 @@
 #include <unistd.h>
 #include <sys/types.h>
 
-#define __USE_GNU
 #include <sys/socket.h>
 
 #include <sys/epoll.h>
diff --git a/drivers/net/softnic/hash_func.h b/drivers/net/softnic/hash_func.h
deleted file mode 100644
index 198d2b20..00000000
--- a/drivers/net/softnic/hash_func.h
+++ /dev/null
@@ -1,359 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2018 Intel Corporation
- */
-
-#ifndef __INCLUDE_HASH_FUNC_H__
-#define __INCLUDE_HASH_FUNC_H__
-
-#include <rte_common.h>
-
-static inline uint64_t
-hash_xor_key8(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0;
-
-	xor0 = seed ^ (k[0] & m[0]);
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key16(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key24(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-
-	xor0 ^= k[2] & m[2];
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key32(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0, xor1;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
-
-	xor0 ^= xor1;
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key40(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0, xor1;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
-
-	xor0 ^= xor1;
-
-	xor0 ^= k[4] & m[4];
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key48(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0, xor1, xor2;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
-	xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
-
-	xor0 ^= xor1;
-
-	xor0 ^= xor2;
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key56(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0, xor1, xor2;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
-	xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
-
-	xor0 ^= xor1;
-	xor2 ^= k[6] & m[6];
-
-	xor0 ^= xor2;
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key64(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0, xor1, xor2, xor3;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
-	xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
-	xor3 = (k[6] & m[6]) ^ (k[7] & m[7]);
-
-	xor0 ^= xor1;
-	xor2 ^= xor3;
-
-	xor0 ^= xor2;
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-#if defined(RTE_ARCH_X86_64)
-
-#include <x86intrin.h>
-
-static inline uint64_t
-hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t crc0;
-
-	crc0 = _mm_crc32_u64(seed, k[0] & m[0]);
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, crc0, crc1;
-
-	k0 = k[0] & m[0];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, crc0, crc1;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc0 = _mm_crc32_u64(crc0, k2);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-
-	crc0 = _mm_crc32_u64(crc0, crc1);
-	crc1 = _mm_crc32_u64(crc2, crc3);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
-	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
-
-	crc0 = _mm_crc32_u64(crc0, crc1);
-	crc1 = _mm_crc32_u64(crc2, crc3);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, k5, crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
-	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
-
-	crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
-	crc1 = _mm_crc32_u64(crc3, k5);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
-	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
-
-	crc4 = _mm_crc32_u64(k5, k[6] & m[6]);
-	crc5 = k5 >> 32;
-
-	crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
-	crc1 = _mm_crc32_u64(crc3, (crc4 << 32) ^ crc5);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
-	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
-
-	crc4 = _mm_crc32_u64(k5, k[6] & m[6]);
-	crc5 = _mm_crc32_u64(k5 >> 32, k[7] & m[7]);
-
-	crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
-	crc1 = _mm_crc32_u64(crc3, (crc4 << 32) ^ crc5);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-#define hash_default_key8			hash_crc_key8
-#define hash_default_key16			hash_crc_key16
-#define hash_default_key24			hash_crc_key24
-#define hash_default_key32			hash_crc_key32
-#define hash_default_key40			hash_crc_key40
-#define hash_default_key48			hash_crc_key48
-#define hash_default_key56			hash_crc_key56
-#define hash_default_key64			hash_crc_key64
-
-#elif defined(RTE_ARCH_ARM64)
-#include "hash_func_arm64.h"
-#else
-
-#define hash_default_key8			hash_xor_key8
-#define hash_default_key16			hash_xor_key16
-#define hash_default_key24			hash_xor_key24
-#define hash_default_key32			hash_xor_key32
-#define hash_default_key40			hash_xor_key40
-#define hash_default_key48			hash_xor_key48
-#define hash_default_key56			hash_xor_key56
-#define hash_default_key64			hash_xor_key64
-
-#endif
-
-#endif
diff --git a/drivers/net/softnic/hash_func_arm64.h b/drivers/net/softnic/hash_func_arm64.h
deleted file mode 100644
index ae6c0f41..00000000
--- a/drivers/net/softnic/hash_func_arm64.h
+++ /dev/null
@@ -1,261 +0,0 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2017 Linaro Limited. All rights reserved.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#ifndef __HASH_FUNC_ARM64_H__
-#define __HASH_FUNC_ARM64_H__
-
-#define _CRC32CX(crc, val)	\
-	__asm__("crc32cx %w[c], %w[c], %x[v]":[c] "+r" (crc):[v] "r" (val))
-
-static inline uint64_t
-hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint32_t crc0;
-
-	crc0 = seed;
-	_CRC32CX(crc0, k[0] & m[0]);
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1;
-
-	k0 = k[0] & m[0];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	_CRC32CX(crc0, k2);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc2 = k2;
-	_CRC32CX(crc2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-
-	_CRC32CX(crc0, crc1);
-	_CRC32CX(crc2, crc3);
-
-	crc0 ^= crc2;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc2 = k2;
-	_CRC32CX(crc2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-	_CRC32CX(crc3, k[4] & m[4]);
-
-	_CRC32CX(crc0, crc1);
-	_CRC32CX(crc2, crc3);
-
-	crc0 ^= crc2;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2, k5;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc2 = k2;
-	_CRC32CX(crc2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-	_CRC32CX(crc3, k[4] & m[4]);
-
-	_CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2);
-	_CRC32CX(crc3, k5);
-
-	crc0 ^= crc3;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2, k5;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1, crc2, crc3, crc4, crc5;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc2 = k2;
-	_CRC32CX(crc2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-	_CRC32CX(crc3, k[4] & m[4]);
-
-	crc4 = k5;
-	 _CRC32CX(crc4, k[6] & m[6]);
-	crc5 = k5 >> 32;
-
-	_CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2);
-	_CRC32CX(crc3, ((uint64_t)crc4 << 32) ^ crc5);
-
-	crc0 ^= crc3;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2, k5;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1, crc2, crc3, crc4, crc5;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc2 = k2;
-	_CRC32CX(crc2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-	_CRC32CX(crc3, k[4] & m[4]);
-
-	crc4 = k5;
-	 _CRC32CX(crc4, k[6] & m[6]);
-	crc5 = k5 >> 32;
-	_CRC32CX(crc5, k[7] & m[7]);
-
-	_CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2);
-	_CRC32CX(crc3, ((uint64_t)crc4 << 32) ^ crc5);
-
-	crc0 ^= crc3;
-
-	return crc0;
-}
-
-#define hash_default_key8			hash_crc_key8
-#define hash_default_key16			hash_crc_key16
-#define hash_default_key24			hash_crc_key24
-#define hash_default_key32			hash_crc_key32
-#define hash_default_key40			hash_crc_key40
-#define hash_default_key48			hash_crc_key48
-#define hash_default_key56			hash_crc_key56
-#define hash_default_key64			hash_crc_key64
-
-#endif
diff --git a/drivers/net/softnic/meson.build b/drivers/net/softnic/meson.build
index ff982274..da249c06 100644
--- a/drivers/net/softnic/meson.build
+++ b/drivers/net/softnic/meson.build
@@ -1,6 +1,9 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2018 Intel Corporation
 
+if host_machine.system() != 'linux'
+        build = false
+endif
 allow_experimental_apis = true
 install_headers('rte_eth_softnic.h')
 sources = files('rte_eth_softnic_tm.c',
@@ -13,6 +16,9 @@ sources = files('rte_eth_softnic_tm.c',
 	'rte_eth_softnic_pipeline.c',
 	'rte_eth_softnic_thread.c',
 	'rte_eth_softnic_cli.c',
+	'rte_eth_softnic_flow.c',
+	'rte_eth_softnic_meter.c',
+	'rte_eth_softnic_cryptodev.c',
 	'parser.c',
 	'conn.c')
-deps += ['pipeline', 'port', 'table', 'sched']
+deps += ['pipeline', 'port', 'table', 'sched', 'cryptodev']
diff --git a/drivers/net/softnic/rte_eth_softnic.c b/drivers/net/softnic/rte_eth_softnic.c
index 30fb3952..743a7c58 100644
--- a/drivers/net/softnic/rte_eth_softnic.c
+++ b/drivers/net/softnic/rte_eth_softnic.c
@@ -14,6 +14,7 @@
 #include <rte_errno.h>
 #include <rte_ring.h>
 #include <rte_tm_driver.h>
+#include <rte_mtr_driver.h>
 
 #include "rte_eth_softnic.h"
 #include "rte_eth_softnic_internals.h"
@@ -27,7 +28,7 @@
 #define PMD_PARAM_TM_QSIZE2                                "tm_qsize2"
 #define PMD_PARAM_TM_QSIZE3                                "tm_qsize3"
 
-static const char *pmd_valid_args[] = {
+static const char * const pmd_valid_args[] = {
 	PMD_PARAM_FIRMWARE,
 	PMD_PARAM_CONN_PORT,
 	PMD_PARAM_CPU_ID,
@@ -46,7 +47,7 @@ static const char welcome[] =
 
 static const char prompt[] = "softnic> ";
 
-struct softnic_conn_params conn_params_default = {
+static const struct softnic_conn_params conn_params_default = {
 	.welcome = welcome,
 	.prompt = prompt,
 	.addr = "0.0.0.0",
@@ -73,7 +74,6 @@ static const struct rte_eth_dev_info pmd_dev_info = {
 		.nb_min = 0,
 		.nb_align = 1,
 	},
-	.rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP,
 };
 
 static int pmd_softnic_logtype;
@@ -190,6 +190,7 @@ pmd_dev_stop(struct rte_eth_dev *dev)
 	softnic_mempool_free(p);
 
 	tm_hierarchy_free(p);
+	softnic_mtr_free(p);
 }
 
 static void
@@ -206,6 +207,21 @@ pmd_link_update(struct rte_eth_dev *dev __rte_unused,
 }
 
 static int
+pmd_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
+		enum rte_filter_type filter_type,
+		enum rte_filter_op filter_op,
+		void *arg)
+{
+	if (filter_type == RTE_ETH_FILTER_GENERIC &&
+			filter_op == RTE_ETH_FILTER_GET) {
+		*(const void **)arg = &pmd_flow_ops;
+		return 0;
+	}
+
+	return -ENOTSUP;
+}
+
+static int
 pmd_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
 {
 	*(const struct rte_tm_ops **)arg = &pmd_tm_ops;
@@ -213,6 +229,14 @@ pmd_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
 	return 0;
 }
 
+static int
+pmd_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
+{
+	*(const struct rte_mtr_ops **)arg = &pmd_mtr_ops;
+
+	return 0;
+}
+
 static const struct eth_dev_ops pmd_ops = {
 	.dev_configure = pmd_dev_configure,
 	.dev_start = pmd_dev_start,
@@ -222,7 +246,9 @@ static const struct eth_dev_ops pmd_ops = {
 	.dev_infos_get = pmd_dev_infos_get,
 	.rx_queue_setup = pmd_rx_queue_setup,
 	.tx_queue_setup = pmd_tx_queue_setup,
+	.filter_ctrl = pmd_filter_ctrl,
 	.tm_ops_get = pmd_tm_ops_get,
+	.mtr_ops_get = pmd_mtr_ops_get,
 };
 
 static uint16_t
@@ -265,12 +291,14 @@ pmd_init(struct pmd_params *params)
 
 	/* Resources */
 	tm_hierarchy_init(p);
+	softnic_mtr_init(p);
 
 	softnic_mempool_init(p);
 	softnic_swq_init(p);
 	softnic_link_init(p);
 	softnic_tmgr_init(p);
 	softnic_tap_init(p);
+	softnic_cryptodev_init(p);
 	softnic_port_in_action_profile_init(p);
 	softnic_table_action_profile_init(p);
 	softnic_pipeline_init(p);
@@ -319,6 +347,7 @@ pmd_free(struct pmd_internals *p)
 	softnic_mempool_free(p);
 
 	tm_hierarchy_free(p);
+	softnic_mtr_free(p);
 
 	rte_free(p);
 }
@@ -528,7 +557,6 @@ static int
 pmd_remove(struct rte_vdev_device *vdev)
 {
 	struct rte_eth_dev *dev = NULL;
-	struct pmd_internals *p;
 
 	if (!vdev)
 		return -EINVAL;
@@ -539,12 +567,12 @@ pmd_remove(struct rte_vdev_device *vdev)
 	dev = rte_eth_dev_allocated(rte_vdev_device_name(vdev));
 	if (dev == NULL)
 		return -ENODEV;
-	p = dev->data->dev_private;
 
 	/* Free device data structures*/
-	rte_free(dev->data);
+	pmd_free(dev->data->dev_private);
+	dev->data->dev_private = NULL; /* already freed */
+	dev->data->mac_addrs = NULL; /* statically allocated */
 	rte_eth_dev_release_port(dev);
-	pmd_free(p);
 
 	return 0;
 }
diff --git a/drivers/net/softnic/rte_eth_softnic_action.c b/drivers/net/softnic/rte_eth_softnic_action.c
index c25f4dd9..92c744dc 100644
--- a/drivers/net/softnic/rte_eth_softnic_action.c
+++ b/drivers/net/softnic/rte_eth_softnic_action.c
@@ -7,8 +7,8 @@
 #include <string.h>
 
 #include <rte_string_fns.h>
+#include <rte_table_hash_func.h>
 
-#include "hash_func.h"
 #include "rte_eth_softnic_internals.h"
 
 /**
@@ -72,35 +72,35 @@ softnic_port_in_action_profile_create(struct pmd_internals *p,
 		params->lb.f_hash == NULL) {
 		switch (params->lb.key_size) {
 		case  8:
-			params->lb.f_hash = hash_default_key8;
+			params->lb.f_hash = rte_table_hash_crc_key8;
 			break;
 
 		case 16:
-			params->lb.f_hash = hash_default_key16;
+			params->lb.f_hash = rte_table_hash_crc_key16;
 			break;
 
 		case 24:
-			params->lb.f_hash = hash_default_key24;
+			params->lb.f_hash = rte_table_hash_crc_key24;
 			break;
 
 		case 32:
-			params->lb.f_hash = hash_default_key32;
+			params->lb.f_hash = rte_table_hash_crc_key32;
 			break;
 
 		case 40:
-			params->lb.f_hash = hash_default_key40;
+			params->lb.f_hash = rte_table_hash_crc_key40;
 			break;
 
 		case 48:
-			params->lb.f_hash = hash_default_key48;
+			params->lb.f_hash = rte_table_hash_crc_key48;
 			break;
 
 		case 56:
-			params->lb.f_hash = hash_default_key56;
+			params->lb.f_hash = rte_table_hash_crc_key56;
 			break;
 
 		case 64:
-			params->lb.f_hash = hash_default_key64;
+			params->lb.f_hash = rte_table_hash_crc_key64;
 			break;
 
 		default:
@@ -223,35 +223,35 @@ softnic_table_action_profile_create(struct pmd_internals *p,
 		params->lb.f_hash == NULL) {
 		switch (params->lb.key_size) {
 		case 8:
-			params->lb.f_hash = hash_default_key8;
+			params->lb.f_hash = rte_table_hash_crc_key8;
 			break;
 
 		case 16:
-			params->lb.f_hash = hash_default_key16;
+			params->lb.f_hash = rte_table_hash_crc_key16;
 			break;
 
 		case 24:
-			params->lb.f_hash = hash_default_key24;
+			params->lb.f_hash = rte_table_hash_crc_key24;
 			break;
 
 		case 32:
-			params->lb.f_hash = hash_default_key32;
+			params->lb.f_hash = rte_table_hash_crc_key32;
 			break;
 
 		case 40:
-			params->lb.f_hash = hash_default_key40;
+			params->lb.f_hash = rte_table_hash_crc_key40;
 			break;
 
 		case 48:
-			params->lb.f_hash = hash_default_key48;
+			params->lb.f_hash = rte_table_hash_crc_key48;
 			break;
 
 		case 56:
-			params->lb.f_hash = hash_default_key56;
+			params->lb.f_hash = rte_table_hash_crc_key56;
 			break;
 
 		case 64:
-			params->lb.f_hash = hash_default_key64;
+			params->lb.f_hash = rte_table_hash_crc_key64;
 			break;
 
 		default:
@@ -364,6 +364,39 @@ softnic_table_action_profile_create(struct pmd_internals *p,
 		}
 	}
 
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_TAG,
+			NULL);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_DECAP,
+			NULL);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_SYM_CRYPTO,
+			&params->sym_crypto);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
 	status = rte_table_action_profile_freeze(ap);
 	if (status) {
 		rte_table_action_profile_free(ap);
diff --git a/drivers/net/softnic/rte_eth_softnic_cli.c b/drivers/net/softnic/rte_eth_softnic_cli.c
index 0c7448cc..c6640d65 100644
--- a/drivers/net/softnic/rte_eth_softnic_cli.c
+++ b/drivers/net/softnic/rte_eth_softnic_cli.c
@@ -9,6 +9,8 @@
 
 #include <rte_common.h>
 #include <rte_cycles.h>
+#include <rte_string_fns.h>
+#include <rte_cryptodev.h>
 
 #include "rte_eth_softnic_internals.h"
 #include "parser.h"
@@ -1089,6 +1091,67 @@ cmd_tap(struct pmd_internals *softnic,
 }
 
 /**
+ * cryptodev <tap_name> dev <device_name> | dev_id <device_id>
+ * queue <n_queues> <queue_size>
+ **/
+
+static void
+cmd_cryptodev(struct pmd_internals *softnic,
+		char **tokens,
+		uint32_t n_tokens,
+		char *out,
+		size_t out_size)
+{
+	struct softnic_cryptodev_params params;
+	char *name;
+
+	memset(&params, 0, sizeof(params));
+	if (n_tokens != 7) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	name = tokens[1];
+
+	if (strcmp(tokens[2], "dev") == 0)
+		params.dev_name = tokens[3];
+	else if (strcmp(tokens[2], "dev_id") == 0) {
+		if (softnic_parser_read_uint32(&params.dev_id, tokens[3]) < 0) {
+			snprintf(out, out_size,	MSG_ARG_INVALID,
+				"dev_id");
+			return;
+		}
+	} else {
+		snprintf(out, out_size,	MSG_ARG_INVALID,
+			"cryptodev");
+		return;
+	}
+
+	if (strcmp(tokens[4], "queue")) {
+		snprintf(out, out_size,	MSG_ARG_NOT_FOUND,
+			"4");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&params.n_queues, tokens[5]) < 0) {
+		snprintf(out, out_size,	MSG_ARG_INVALID,
+			"q");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&params.queue_size, tokens[6]) < 0) {
+		snprintf(out, out_size,	MSG_ARG_INVALID,
+			"queue_size");
+		return;
+	}
+
+	if (softnic_cryptodev_create(softnic, name, &params) == NULL) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
  * port in action profile <profile_name>
  *  [filter match | mismatch offset <key_offset> mask <key_mask> key <key_value> port <port_id>]
  *  [balance offset <key_offset> mask <key_mask> port <port_id0> ... <port_id15>]
@@ -1272,13 +1335,17 @@ cmd_port_in_action_profile(struct pmd_internals *softnic,
  *      tc <n_tc>
  *      stats none | pkts | bytes | both]
  *  [tm spp <n_subports_per_port> pps <n_pipes_per_subport>]
- *  [encap ether | vlan | qinq | mpls | pppoe]
+ *  [encap ether | vlan | qinq | mpls | pppoe |
+ *      vxlan offset <ether_offset> ipv4 | ipv6 vlan on | off]
  *  [nat src | dst
  *      proto udp | tcp]
  *  [ttl drop | fwd
  *      stats none | pkts]
  *  [stats pkts | bytes | both]
  *  [time]
+ *  [tag]
+ *  [decap]
+ *
  */
 static void
 cmd_table_action_profile(struct pmd_internals *softnic,
@@ -1478,6 +1545,8 @@ cmd_table_action_profile(struct pmd_internals *softnic,
 
 	if (t0 < n_tokens &&
 		(strcmp(tokens[t0], "encap") == 0)) {
+		uint32_t n_extra_tokens = 0;
+
 		if (n_tokens < t0 + 2) {
 			snprintf(out, out_size, MSG_ARG_MISMATCH,
 				"action profile encap");
@@ -1494,13 +1563,61 @@ cmd_table_action_profile(struct pmd_internals *softnic,
 			p.encap.encap_mask = 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS;
 		} else if (strcmp(tokens[t0 + 1], "pppoe") == 0) {
 			p.encap.encap_mask = 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE;
+		} else if (strcmp(tokens[t0 + 1], "vxlan") == 0) {
+			if (n_tokens < t0 + 2 + 5) {
+				snprintf(out, out_size, MSG_ARG_MISMATCH,
+					"action profile encap vxlan");
+				return;
+			}
+
+			if (strcmp(tokens[t0 + 2], "offset") != 0) {
+				snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+					"vxlan: offset");
+				return;
+			}
+
+			if (softnic_parser_read_uint32(&p.encap.vxlan.data_offset,
+				tokens[t0 + 2 + 1]) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"vxlan: ether_offset");
+				return;
+			}
+
+			if (strcmp(tokens[t0 + 2 + 2], "ipv4") == 0)
+				p.encap.vxlan.ip_version = 1;
+			else if (strcmp(tokens[t0 + 2 + 2], "ipv6") == 0)
+				p.encap.vxlan.ip_version = 0;
+			else {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"vxlan: ipv4 or ipv6");
+				return;
+			}
+
+			if (strcmp(tokens[t0 + 2 + 3], "vlan") != 0) {
+				snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+					"vxlan: vlan");
+				return;
+			}
+
+			if (strcmp(tokens[t0 + 2 + 4], "on") == 0)
+				p.encap.vxlan.vlan = 1;
+			else if (strcmp(tokens[t0 + 2 + 4], "off") == 0)
+				p.encap.vxlan.vlan = 0;
+			else {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"vxlan: on or off");
+				return;
+			}
+
+			p.encap.encap_mask = 1LLU << RTE_TABLE_ACTION_ENCAP_VXLAN;
+			n_extra_tokens = 5;
+
 		} else {
 			snprintf(out, out_size, MSG_ARG_MISMATCH, "encap");
 			return;
 		}
-
 		p.action_mask |= 1LLU << RTE_TABLE_ACTION_ENCAP;
-		t0 += 2;
+		t0 += 2 + n_extra_tokens;
 	} /* encap */
 
 	if (t0 < n_tokens &&
@@ -1610,6 +1727,18 @@ cmd_table_action_profile(struct pmd_internals *softnic,
 		t0 += 1;
 	} /* time */
 
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "tag") == 0)) {
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_TAG;
+		t0 += 1;
+	} /* tag */
+
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "decap") == 0)) {
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_DECAP;
+		t0 += 1;
+	} /* decap */
+
 	if (t0 < n_tokens) {
 		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
 		return;
@@ -1682,6 +1811,7 @@ cmd_pipeline(struct pmd_internals *softnic,
  *  | tmgr <tmgr_name>
  *  | tap <tap_name> mempool <mempool_name> mtu <mtu>
  *  | source mempool <mempool_name> file <file_name> bpp <n_bytes_per_pkt>
+ *  | cryptodev <cryptodev_name> rxq <queue_id>
  *  [action <port_in_action_profile_name>]
  *  [disabled]
  */
@@ -1697,6 +1827,8 @@ cmd_pipeline_port_in(struct pmd_internals *softnic,
 	uint32_t t0;
 	int enabled, status;
 
+	memset(&p, 0, sizeof(p));
+
 	if (n_tokens < 7) {
 		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
 		return;
@@ -1735,7 +1867,7 @@ cmd_pipeline_port_in(struct pmd_internals *softnic,
 
 		p.type = PORT_IN_RXQ;
 
-		p.dev_name = tokens[t0 + 1];
+		strcpy(p.dev_name, tokens[t0 + 1]);
 
 		if (strcmp(tokens[t0 + 2], "rxq") != 0) {
 			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rxq");
@@ -1758,7 +1890,7 @@ cmd_pipeline_port_in(struct pmd_internals *softnic,
 
 		p.type = PORT_IN_SWQ;
 
-		p.dev_name = tokens[t0 + 1];
+		strcpy(p.dev_name, tokens[t0 + 1]);
 
 		t0 += 2;
 	} else if (strcmp(tokens[t0], "tmgr") == 0) {
@@ -1770,7 +1902,7 @@ cmd_pipeline_port_in(struct pmd_internals *softnic,
 
 		p.type = PORT_IN_TMGR;
 
-		p.dev_name = tokens[t0 + 1];
+		strcpy(p.dev_name, tokens[t0 + 1]);
 
 		t0 += 2;
 	} else if (strcmp(tokens[t0], "tap") == 0) {
@@ -1782,7 +1914,7 @@ cmd_pipeline_port_in(struct pmd_internals *softnic,
 
 		p.type = PORT_IN_TAP;
 
-		p.dev_name = tokens[t0 + 1];
+		strcpy(p.dev_name, tokens[t0 + 1]);
 
 		if (strcmp(tokens[t0 + 2], "mempool") != 0) {
 			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
@@ -1814,8 +1946,6 @@ cmd_pipeline_port_in(struct pmd_internals *softnic,
 
 		p.type = PORT_IN_SOURCE;
 
-		p.dev_name = NULL;
-
 		if (strcmp(tokens[t0 + 1], "mempool") != 0) {
 			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
 				"mempool");
@@ -1846,12 +1976,32 @@ cmd_pipeline_port_in(struct pmd_internals *softnic,
 		}
 
 		t0 += 7;
+	} else if (strcmp(tokens[t0], "cryptodev") == 0) {
+		if (n_tokens < t0 + 3) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port in cryptodev");
+			return;
+		}
+
+		p.type = PORT_IN_CRYPTODEV;
+
+		strlcpy(p.dev_name, tokens[t0 + 1], sizeof(p.dev_name));
+		if (softnic_parser_read_uint16(&p.rxq.queue_id,
+				tokens[t0 + 3]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"rxq");
+			return;
+		}
+
+		p.cryptodev.arg_callback = NULL;
+		p.cryptodev.f_callback = NULL;
+
+		t0 += 4;
 	} else {
 		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
 		return;
 	}
 
-	p.action_profile_name = NULL;
 	if (n_tokens > t0 &&
 		(strcmp(tokens[t0], "action") == 0)) {
 		if (n_tokens < t0 + 2) {
@@ -1859,7 +2009,7 @@ cmd_pipeline_port_in(struct pmd_internals *softnic,
 			return;
 		}
 
-		p.action_profile_name = tokens[t0 + 1];
+		strcpy(p.action_profile_name, tokens[t0 + 1]);
 
 		t0 += 2;
 	}
@@ -1895,6 +2045,7 @@ cmd_pipeline_port_in(struct pmd_internals *softnic,
  *  | tmgr <tmgr_name>
  *  | tap <tap_name>
  *  | sink [file <file_name> pkts <max_n_pkts>]
+ *  | cryptodev <cryptodev_name> txq <txq_id> offset <crypto_op_offset>
  */
 static void
 cmd_pipeline_port_out(struct pmd_internals *softnic,
@@ -1945,7 +2096,7 @@ cmd_pipeline_port_out(struct pmd_internals *softnic,
 
 		p.type = PORT_OUT_TXQ;
 
-		p.dev_name = tokens[7];
+		strcpy(p.dev_name, tokens[7]);
 
 		if (strcmp(tokens[8], "txq") != 0) {
 			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "txq");
@@ -1966,7 +2117,7 @@ cmd_pipeline_port_out(struct pmd_internals *softnic,
 
 		p.type = PORT_OUT_SWQ;
 
-		p.dev_name = tokens[7];
+		strcpy(p.dev_name, tokens[7]);
 	} else if (strcmp(tokens[6], "tmgr") == 0) {
 		if (n_tokens != 8) {
 			snprintf(out, out_size, MSG_ARG_MISMATCH,
@@ -1976,7 +2127,7 @@ cmd_pipeline_port_out(struct pmd_internals *softnic,
 
 		p.type = PORT_OUT_TMGR;
 
-		p.dev_name = tokens[7];
+		strcpy(p.dev_name, tokens[7]);
 	} else if (strcmp(tokens[6], "tap") == 0) {
 		if (n_tokens != 8) {
 			snprintf(out, out_size, MSG_ARG_MISMATCH,
@@ -1986,7 +2137,7 @@ cmd_pipeline_port_out(struct pmd_internals *softnic,
 
 		p.type = PORT_OUT_TAP;
 
-		p.dev_name = tokens[7];
+		strcpy(p.dev_name, tokens[7]);
 	} else if (strcmp(tokens[6], "sink") == 0) {
 		if ((n_tokens != 7) && (n_tokens != 11)) {
 			snprintf(out, out_size, MSG_ARG_MISMATCH,
@@ -1996,8 +2147,6 @@ cmd_pipeline_port_out(struct pmd_internals *softnic,
 
 		p.type = PORT_OUT_SINK;
 
-		p.dev_name = NULL;
-
 		if (n_tokens == 7) {
 			p.sink.file_name = NULL;
 			p.sink.max_n_pkts = 0;
@@ -2021,6 +2170,40 @@ cmd_pipeline_port_out(struct pmd_internals *softnic,
 				return;
 			}
 		}
+	} else if (strcmp(tokens[6], "cryptodev") == 0) {
+		if (n_tokens != 12) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port out cryptodev");
+			return;
+		}
+
+		p.type = PORT_OUT_CRYPTODEV;
+
+		strlcpy(p.dev_name, tokens[7], sizeof(p.dev_name));
+
+		if (strcmp(tokens[8], "txq")) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port out cryptodev");
+			return;
+		}
+
+		if (softnic_parser_read_uint16(&p.cryptodev.queue_id, tokens[9])
+				!= 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "queue_id");
+			return;
+		}
+
+		if (strcmp(tokens[10], "offset")) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port out cryptodev");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.cryptodev.op_offset,
+				tokens[11]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "queue_id");
+			return;
+		}
 	} else {
 		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
 		return;
@@ -2064,12 +2247,13 @@ cmd_pipeline_table(struct pmd_internals *softnic,
 	char *out,
 	size_t out_size)
 {
-	uint8_t key_mask[TABLE_RULE_MATCH_SIZE_MAX];
 	struct softnic_table_params p;
 	char *pipeline_name;
 	uint32_t t0;
 	int status;
 
+	memset(&p, 0, sizeof(p));
+
 	if (n_tokens < 5) {
 		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
 		return;
@@ -2203,12 +2387,11 @@ cmd_pipeline_table(struct pmd_internals *softnic,
 		}
 
 		if ((softnic_parse_hex_string(tokens[t0 + 5],
-			key_mask, &key_mask_size) != 0) ||
+			p.match.hash.key_mask, &key_mask_size) != 0) ||
 			key_mask_size != p.match.hash.key_size) {
 			snprintf(out, out_size, MSG_ARG_INVALID, "key_mask");
 			return;
 		}
-		p.match.hash.key_mask = key_mask;
 
 		if (strcmp(tokens[t0 + 6], "offset") != 0) {
 			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "offset");
@@ -2295,7 +2478,6 @@ cmd_pipeline_table(struct pmd_internals *softnic,
 		return;
 	}
 
-	p.action_profile_name = NULL;
 	if (n_tokens > t0 &&
 		(strcmp(tokens[t0], "action") == 0)) {
 		if (n_tokens < t0 + 2) {
@@ -2303,7 +2485,7 @@ cmd_pipeline_table(struct pmd_internals *softnic,
 			return;
 		}
 
-		p.action_profile_name = tokens[t0 + 1];
+		strcpy(p.action_profile_name, tokens[t0 + 1]);
 
 		t0 += 2;
 	}
@@ -3176,10 +3358,30 @@ parse_match(char **tokens,
  *          [label2 <label> <tc> <ttl>
  *          [label3 <label> <tc> <ttl>]]]
  *       | pppoe <da> <sa> <session_id>]
+ *       | vxlan ether <da> <sa>
+ *          [vlan <pcp> <dei> <vid>]
+ *          ipv4 <sa> <da> <dscp> <ttl>
+ *          | ipv6 <sa> <da> <flow_label> <dscp> <hop_limit>
+ *          udp <sp> <dp>
+ *          vxlan <vni>]
  *    [nat ipv4 | ipv6 <addr> <port>]
  *    [ttl dec | keep]
  *    [stats]
  *    [time]
+ *    [tag <tag>]
+ *    [decap <n>]
+ *    [sym_crypto
+ *       encrypt | decrypt
+ *       type
+ *       | cipher
+ *          cipher_algo <algo> cipher_key <key> cipher_iv <iv>
+ *       | cipher_auth
+ *          cipher_algo <algo> cipher_key <key> cipher_iv <iv>
+ *          auth_algo <algo> auth_key <key> digest_size <size>
+ *       | aead
+ *          aead_algo <algo> aead_key <key> aead_iv <iv> aead_aad <aad>
+ *          digest_size <size>
+ *       data_offset <data_offset>]
  *
  * where:
  *    <pa> ::= g | y | r | drop
@@ -3575,6 +3777,122 @@ parse_table_action_encap(char **tokens,
 		return 1 + 4;
 	}
 
+	/* vxlan */
+	if (n_tokens && (strcmp(tokens[0], "vxlan") == 0)) {
+		uint32_t n = 0;
+
+		n_tokens--;
+		tokens++;
+		n++;
+
+		/* ether <da> <sa> */
+		if ((n_tokens < 3) ||
+			strcmp(tokens[0], "ether") ||
+			softnic_parse_mac_addr(tokens[1], &a->encap.vxlan.ether.da) ||
+			softnic_parse_mac_addr(tokens[2], &a->encap.vxlan.ether.sa))
+			return 0;
+
+		n_tokens -= 3;
+		tokens += 3;
+		n += 3;
+
+		/* [vlan <pcp> <dei> <vid>] */
+		if (strcmp(tokens[0], "vlan") == 0) {
+			uint32_t pcp, dei, vid;
+
+			if ((n_tokens < 4) ||
+				softnic_parser_read_uint32(&pcp, tokens[1]) ||
+				(pcp > 7) ||
+				softnic_parser_read_uint32(&dei, tokens[2]) ||
+				(dei > 1) ||
+				softnic_parser_read_uint32(&vid, tokens[3]) ||
+				(vid > 0xFFF))
+				return 0;
+
+			a->encap.vxlan.vlan.pcp = pcp;
+			a->encap.vxlan.vlan.dei = dei;
+			a->encap.vxlan.vlan.vid = vid;
+
+			n_tokens -= 4;
+			tokens += 4;
+			n += 4;
+		}
+
+		/* ipv4 <sa> <da> <dscp> <ttl>
+		   | ipv6 <sa> <da> <flow_label> <dscp> <hop_limit> */
+		if (strcmp(tokens[0], "ipv4") == 0) {
+			struct in_addr sa, da;
+			uint8_t dscp, ttl;
+
+			if ((n_tokens < 5) ||
+				softnic_parse_ipv4_addr(tokens[1], &sa) ||
+				softnic_parse_ipv4_addr(tokens[2], &da) ||
+				softnic_parser_read_uint8(&dscp, tokens[3]) ||
+				(dscp > 64) ||
+				softnic_parser_read_uint8(&ttl, tokens[4]))
+				return 0;
+
+			a->encap.vxlan.ipv4.sa = rte_be_to_cpu_32(sa.s_addr);
+			a->encap.vxlan.ipv4.da = rte_be_to_cpu_32(da.s_addr);
+			a->encap.vxlan.ipv4.dscp = dscp;
+			a->encap.vxlan.ipv4.ttl = ttl;
+
+			n_tokens -= 5;
+			tokens += 5;
+			n += 5;
+		} else if (strcmp(tokens[0], "ipv6") == 0) {
+			struct in6_addr sa, da;
+			uint32_t flow_label;
+			uint8_t dscp, hop_limit;
+
+			if ((n_tokens < 6) ||
+				softnic_parse_ipv6_addr(tokens[1], &sa) ||
+				softnic_parse_ipv6_addr(tokens[2], &da) ||
+				softnic_parser_read_uint32(&flow_label, tokens[3]) ||
+				softnic_parser_read_uint8(&dscp, tokens[4]) ||
+				(dscp > 64) ||
+				softnic_parser_read_uint8(&hop_limit, tokens[5]))
+				return 0;
+
+			memcpy(a->encap.vxlan.ipv6.sa, sa.s6_addr, 16);
+			memcpy(a->encap.vxlan.ipv6.da, da.s6_addr, 16);
+			a->encap.vxlan.ipv6.flow_label = flow_label;
+			a->encap.vxlan.ipv6.dscp = dscp;
+			a->encap.vxlan.ipv6.hop_limit = hop_limit;
+
+			n_tokens -= 6;
+			tokens += 6;
+			n += 6;
+		} else
+			return 0;
+
+		/* udp <sp> <dp> */
+		if ((n_tokens < 3) ||
+			strcmp(tokens[0], "udp") ||
+			softnic_parser_read_uint16(&a->encap.vxlan.udp.sp, tokens[1]) ||
+			softnic_parser_read_uint16(&a->encap.vxlan.udp.dp, tokens[2]))
+			return 0;
+
+		n_tokens -= 3;
+		tokens += 3;
+		n += 3;
+
+		/* vxlan <vni> */
+		if ((n_tokens < 2) ||
+			strcmp(tokens[0], "vxlan") ||
+			softnic_parser_read_uint32(&a->encap.vxlan.vxlan.vni, tokens[1]) ||
+			(a->encap.vxlan.vxlan.vni > 0xFFFFFF))
+			return 0;
+
+		n_tokens -= 2;
+		tokens += 2;
+		n += 2;
+
+		a->encap.type = RTE_TABLE_ACTION_ENCAP_VXLAN;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_ENCAP;
+		return 1 + n;
+	}
+
 	return 0;
 }
 
@@ -3669,6 +3987,400 @@ parse_table_action_time(char **tokens,
 	return 1;
 }
 
+static void
+parse_free_sym_crypto_param_data(struct rte_table_action_sym_crypto_params *p)
+{
+	struct rte_crypto_sym_xform *xform[2] = {NULL};
+	uint32_t i;
+
+	xform[0] = p->xform;
+	if (xform[0])
+		xform[1] = xform[0]->next;
+
+	for (i = 0; i < 2; i++) {
+		if (xform[i] == NULL)
+			continue;
+
+		switch (xform[i]->type) {
+		case RTE_CRYPTO_SYM_XFORM_CIPHER:
+			if (xform[i]->cipher.key.data)
+				free(xform[i]->cipher.key.data);
+			if (p->cipher_auth.cipher_iv.val)
+				free(p->cipher_auth.cipher_iv.val);
+			if (p->cipher_auth.cipher_iv_update.val)
+				free(p->cipher_auth.cipher_iv_update.val);
+			break;
+		case RTE_CRYPTO_SYM_XFORM_AUTH:
+			if (xform[i]->auth.key.data)
+				free(xform[i]->cipher.key.data);
+			if (p->cipher_auth.auth_iv.val)
+				free(p->cipher_auth.cipher_iv.val);
+			if (p->cipher_auth.auth_iv_update.val)
+				free(p->cipher_auth.cipher_iv_update.val);
+			break;
+		case RTE_CRYPTO_SYM_XFORM_AEAD:
+			if (xform[i]->aead.key.data)
+				free(xform[i]->cipher.key.data);
+			if (p->aead.iv.val)
+				free(p->aead.iv.val);
+			if (p->aead.aad.val)
+				free(p->aead.aad.val);
+			break;
+		default:
+			continue;
+		}
+	}
+
+}
+
+static struct rte_crypto_sym_xform *
+parse_table_action_cipher(struct rte_table_action_sym_crypto_params *p,
+		char **tokens, uint32_t n_tokens, uint32_t encrypt,
+		uint32_t *used_n_tokens)
+{
+	struct rte_crypto_sym_xform *xform_cipher;
+	int status;
+	size_t len;
+
+	if (n_tokens < 7 || strcmp(tokens[1], "cipher_algo") ||
+			strcmp(tokens[3], "cipher_key") ||
+			strcmp(tokens[5], "cipher_iv"))
+		return NULL;
+
+	xform_cipher = calloc(1, sizeof(*xform_cipher));
+	if (xform_cipher == NULL)
+		return NULL;
+
+	xform_cipher->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+	xform_cipher->cipher.op = encrypt ? RTE_CRYPTO_CIPHER_OP_ENCRYPT :
+			RTE_CRYPTO_CIPHER_OP_DECRYPT;
+
+	/* cipher_algo */
+	status = rte_cryptodev_get_cipher_algo_enum(
+			&xform_cipher->cipher.algo, tokens[2]);
+	if (status < 0)
+		goto error_exit;
+
+	/* cipher_key */
+	len = strlen(tokens[4]);
+	xform_cipher->cipher.key.data = calloc(1, len / 2 + 1);
+	if (xform_cipher->cipher.key.data == NULL)
+		goto error_exit;
+
+	status = softnic_parse_hex_string(tokens[4],
+			xform_cipher->cipher.key.data,
+			(uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_cipher->cipher.key.length = (uint16_t)len;
+
+	/* cipher_iv */
+	len = strlen(tokens[6]);
+
+	p->cipher_auth.cipher_iv.val = calloc(1, len / 2 + 1);
+	if (p->cipher_auth.cipher_iv.val == NULL)
+		goto error_exit;
+
+	status = softnic_parse_hex_string(tokens[6],
+			p->cipher_auth.cipher_iv.val,
+			(uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_cipher->cipher.iv.length = (uint16_t)len;
+	xform_cipher->cipher.iv.offset = RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET;
+	p->cipher_auth.cipher_iv.length = (uint32_t)len;
+	*used_n_tokens = 7;
+
+	return xform_cipher;
+
+error_exit:
+	if (xform_cipher->cipher.key.data)
+		free(xform_cipher->cipher.key.data);
+
+	if (p->cipher_auth.cipher_iv.val) {
+		free(p->cipher_auth.cipher_iv.val);
+		p->cipher_auth.cipher_iv.val = NULL;
+	}
+
+	free(xform_cipher);
+
+	return NULL;
+}
+
+static struct rte_crypto_sym_xform *
+parse_table_action_cipher_auth(struct rte_table_action_sym_crypto_params *p,
+		char **tokens, uint32_t n_tokens, uint32_t encrypt,
+		uint32_t *used_n_tokens)
+{
+	struct rte_crypto_sym_xform *xform_cipher;
+	struct rte_crypto_sym_xform *xform_auth;
+	int status;
+	size_t len;
+
+	if (n_tokens < 13 ||
+			strcmp(tokens[7], "auth_algo") ||
+			strcmp(tokens[9], "auth_key") ||
+			strcmp(tokens[11], "digest_size"))
+		return NULL;
+
+	xform_auth = calloc(1, sizeof(*xform_auth));
+	if (xform_auth == NULL)
+		return NULL;
+
+	xform_auth->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+	xform_auth->auth.op = encrypt ? RTE_CRYPTO_AUTH_OP_GENERATE :
+			RTE_CRYPTO_AUTH_OP_VERIFY;
+
+	/* auth_algo */
+	status = rte_cryptodev_get_auth_algo_enum(&xform_auth->auth.algo,
+			tokens[8]);
+	if (status < 0)
+		goto error_exit;
+
+	/* auth_key */
+	len = strlen(tokens[10]);
+	xform_auth->auth.key.data = calloc(1, len / 2 + 1);
+	if (xform_auth->auth.key.data == NULL)
+		goto error_exit;
+
+	status = softnic_parse_hex_string(tokens[10],
+			xform_auth->auth.key.data, (uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_auth->auth.key.length = (uint16_t)len;
+
+	if (strcmp(tokens[11], "digest_size"))
+		goto error_exit;
+
+	status = softnic_parser_read_uint16(&xform_auth->auth.digest_length,
+			tokens[12]);
+	if (status < 0)
+		goto error_exit;
+
+	xform_cipher = parse_table_action_cipher(p, tokens, 7, encrypt,
+			used_n_tokens);
+	if (xform_cipher == NULL)
+		goto error_exit;
+
+	*used_n_tokens += 6;
+
+	if (encrypt) {
+		xform_cipher->next = xform_auth;
+		return xform_cipher;
+	} else {
+		xform_auth->next = xform_cipher;
+		return xform_auth;
+	}
+
+error_exit:
+	if (xform_auth->auth.key.data)
+		free(xform_auth->auth.key.data);
+	if (p->cipher_auth.auth_iv.val) {
+		free(p->cipher_auth.auth_iv.val);
+		p->cipher_auth.auth_iv.val = 0;
+	}
+
+	free(xform_auth);
+
+	return NULL;
+}
+
+static struct rte_crypto_sym_xform *
+parse_table_action_aead(struct rte_table_action_sym_crypto_params *p,
+		char **tokens, uint32_t n_tokens, uint32_t encrypt,
+		uint32_t *used_n_tokens)
+{
+	struct rte_crypto_sym_xform *xform_aead;
+	int status;
+	size_t len;
+
+	if (n_tokens < 11 || strcmp(tokens[1], "aead_algo") ||
+			strcmp(tokens[3], "aead_key") ||
+			strcmp(tokens[5], "aead_iv") ||
+			strcmp(tokens[7], "aead_aad") ||
+			strcmp(tokens[9], "digest_size"))
+		return NULL;
+
+	xform_aead = calloc(1, sizeof(*xform_aead));
+	if (xform_aead == NULL)
+		return NULL;
+
+	xform_aead->type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	xform_aead->aead.op = encrypt ? RTE_CRYPTO_AEAD_OP_ENCRYPT :
+			RTE_CRYPTO_AEAD_OP_DECRYPT;
+
+	/* aead_algo */
+	status = rte_cryptodev_get_aead_algo_enum(&xform_aead->aead.algo,
+			tokens[2]);
+	if (status < 0)
+		goto error_exit;
+
+	/* aead_key */
+	len = strlen(tokens[4]);
+	xform_aead->aead.key.data = calloc(1, len / 2 + 1);
+	if (xform_aead->aead.key.data == NULL)
+		goto error_exit;
+
+	status = softnic_parse_hex_string(tokens[4], xform_aead->aead.key.data,
+			(uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_aead->aead.key.length = (uint16_t)len;
+
+	/* aead_iv */
+	len = strlen(tokens[6]);
+	p->aead.iv.val = calloc(1, len / 2 + 1);
+	if (p->aead.iv.val == NULL)
+		goto error_exit;
+
+	status = softnic_parse_hex_string(tokens[6], p->aead.iv.val,
+			(uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_aead->aead.iv.length = (uint16_t)len;
+	xform_aead->aead.iv.offset = RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET;
+	p->aead.iv.length = (uint32_t)len;
+
+	/* aead_aad */
+	len = strlen(tokens[8]);
+	p->aead.aad.val = calloc(1, len / 2 + 1);
+	if (p->aead.aad.val == NULL)
+		goto error_exit;
+
+	status = softnic_parse_hex_string(tokens[8], p->aead.aad.val, (uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_aead->aead.aad_length = (uint16_t)len;
+	p->aead.aad.length = (uint32_t)len;
+
+	/* digest_size */
+	status = softnic_parser_read_uint16(&xform_aead->aead.digest_length,
+			tokens[10]);
+	if (status < 0)
+		goto error_exit;
+
+	*used_n_tokens = 11;
+
+	return xform_aead;
+
+error_exit:
+	if (xform_aead->aead.key.data)
+		free(xform_aead->aead.key.data);
+	if (p->aead.iv.val) {
+		free(p->aead.iv.val);
+		p->aead.iv.val = NULL;
+	}
+	if (p->aead.aad.val) {
+		free(p->aead.aad.val);
+		p->aead.aad.val = NULL;
+	}
+
+	free(xform_aead);
+
+	return NULL;
+}
+
+
+static uint32_t
+parse_table_action_sym_crypto(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	struct rte_table_action_sym_crypto_params *p = &a->sym_crypto;
+	struct rte_crypto_sym_xform *xform = NULL;
+	uint32_t used_n_tokens;
+	uint32_t encrypt;
+	int status;
+
+	if ((n_tokens < 12) ||
+		strcmp(tokens[0], "sym_crypto") ||
+		strcmp(tokens[2], "type"))
+		return 0;
+
+	memset(p, 0, sizeof(*p));
+
+	if (strcmp(tokens[1], "encrypt") == 0)
+		encrypt = 1;
+	else
+		encrypt = 0;
+
+	status = softnic_parser_read_uint32(&p->data_offset, tokens[n_tokens - 1]);
+	if (status < 0)
+		return 0;
+
+	if (strcmp(tokens[3], "cipher") == 0) {
+		tokens += 3;
+		n_tokens -= 3;
+
+		xform = parse_table_action_cipher(p, tokens, n_tokens, encrypt,
+				&used_n_tokens);
+	} else if (strcmp(tokens[3], "cipher_auth") == 0) {
+		tokens += 3;
+		n_tokens -= 3;
+
+		xform = parse_table_action_cipher_auth(p, tokens, n_tokens,
+				encrypt, &used_n_tokens);
+	} else if (strcmp(tokens[3], "aead") == 0) {
+		tokens += 3;
+		n_tokens -= 3;
+
+		xform = parse_table_action_aead(p, tokens, n_tokens, encrypt,
+				&used_n_tokens);
+	}
+
+	if (xform == NULL)
+		return 0;
+
+	p->xform = xform;
+
+	if (strcmp(tokens[used_n_tokens], "data_offset")) {
+		parse_free_sym_crypto_param_data(p);
+		return 0;
+	}
+
+	a->action_mask |= 1 << RTE_TABLE_ACTION_SYM_CRYPTO;
+
+	return used_n_tokens + 5;
+}
+
+static uint32_t
+parse_table_action_tag(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	if (n_tokens < 2 ||
+		strcmp(tokens[0], "tag"))
+		return 0;
+
+	if (softnic_parser_read_uint32(&a->tag.tag, tokens[1]))
+		return 0;
+
+	a->action_mask |= 1 << RTE_TABLE_ACTION_TAG;
+	return 2;
+}
+
+static uint32_t
+parse_table_action_decap(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	if (n_tokens < 2 ||
+		strcmp(tokens[0], "decap"))
+		return 0;
+
+	if (softnic_parser_read_uint16(&a->decap.n, tokens[1]))
+		return 0;
+
+	a->action_mask |= 1 << RTE_TABLE_ACTION_DECAP;
+	return 2;
+}
+
 static uint32_t
 parse_table_action(char **tokens,
 	uint32_t n_tokens,
@@ -3813,6 +4525,47 @@ parse_table_action(char **tokens,
 		n_tokens -= n;
 	}
 
+	if (n_tokens && (strcmp(tokens[0], "tag") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_tag(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action tag");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "decap") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_decap(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action decap");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "sym_crypto") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_sym_crypto(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action sym_crypto");
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
 	if (n_tokens0 - n_tokens == 1) {
 		snprintf(out, out_size, MSG_ARG_INVALID, "action");
 		return 0;
@@ -4797,6 +5550,81 @@ cmd_softnic_thread_pipeline_disable(struct pmd_internals *softnic,
 	}
 }
 
+/**
+ * flowapi map
+ *  group <group_id>
+ *  ingress | egress
+ *  pipeline <pipeline_name>
+ *  table <table_id>
+ */
+static void
+cmd_softnic_flowapi_map(struct pmd_internals *softnic,
+		char **tokens,
+		uint32_t n_tokens,
+		char *out,
+		size_t out_size)
+{
+	char *pipeline_name;
+	uint32_t group_id, table_id;
+	int ingress, status;
+
+	if (n_tokens != 9) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	if (strcmp(tokens[1], "map") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "map");
+		return;
+	}
+
+	if (strcmp(tokens[2], "group") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "group");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&group_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "group_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "ingress") == 0) {
+		ingress = 1;
+	} else if (strcmp(tokens[4], "egress") == 0) {
+		ingress = 0;
+	} else {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "ingress | egress");
+		return;
+	}
+
+	if (strcmp(tokens[5], "pipeline") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pipeline");
+		return;
+	}
+
+	pipeline_name = tokens[6];
+
+	if (strcmp(tokens[7], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&table_id, tokens[8]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	status = flow_attr_map_set(softnic,
+			group_id,
+			ingress,
+			pipeline_name,
+			table_id);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
 void
 softnic_cli_process(char *in, char *out, size_t out_size, void *arg)
 {
@@ -4877,6 +5705,11 @@ softnic_cli_process(char *in, char *out, size_t out_size, void *arg)
 		return;
 	}
 
+	if (strcmp(tokens[0], "cryptodev") == 0) {
+		cmd_cryptodev(softnic, tokens, n_tokens, out, out_size);
+		return;
+	}
+
 	if (strcmp(tokens[0], "port") == 0) {
 		cmd_port_in_action_profile(softnic, tokens, n_tokens, out, out_size);
 		return;
@@ -5089,6 +5922,12 @@ softnic_cli_process(char *in, char *out, size_t out_size, void *arg)
 		}
 	}
 
+	if (strcmp(tokens[0], "flowapi") == 0) {
+		cmd_softnic_flowapi_map(softnic, tokens, n_tokens, out,
+					out_size);
+		return;
+	}
+
 	snprintf(out, out_size, MSG_CMD_UNKNOWN, tokens[0]);
 }
 
diff --git a/drivers/net/softnic/rte_eth_softnic_cryptodev.c b/drivers/net/softnic/rte_eth_softnic_cryptodev.c
new file mode 100644
index 00000000..1480f6dd
--- /dev/null
+++ b/drivers/net/softnic/rte_eth_softnic_cryptodev.c
@@ -0,0 +1,125 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_string_fns.h>
+
+#include "rte_eth_softnic_internals.h"
+
+int
+softnic_cryptodev_init(struct pmd_internals *p)
+{
+	TAILQ_INIT(&p->cryptodev_list);
+
+	return 0;
+}
+
+void
+softnic_cryptodev_free(struct pmd_internals *p)
+{
+	for ( ; ; ) {
+		struct softnic_cryptodev *cryptodev;
+
+		cryptodev = TAILQ_FIRST(&p->cryptodev_list);
+		if (cryptodev == NULL)
+			break;
+
+		TAILQ_REMOVE(&p->cryptodev_list, cryptodev, node);
+		free(cryptodev);
+	}
+}
+
+struct softnic_cryptodev *
+softnic_cryptodev_find(struct pmd_internals *p,
+	const char *name)
+{
+	struct softnic_cryptodev *cryptodev;
+
+	if (name == NULL)
+		return NULL;
+
+	TAILQ_FOREACH(cryptodev, &p->cryptodev_list, node)
+		if (strcmp(cryptodev->name, name) == 0)
+			return cryptodev;
+
+	return NULL;
+}
+
+struct softnic_cryptodev *
+softnic_cryptodev_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_cryptodev_params *params)
+{
+	struct rte_cryptodev_info dev_info;
+	struct rte_cryptodev_config dev_conf;
+	struct rte_cryptodev_qp_conf queue_conf;
+	struct softnic_cryptodev *cryptodev;
+	uint32_t dev_id, i;
+	uint32_t socket_id;
+	int status;
+
+	/* Check input params */
+	if ((name == NULL) ||
+		softnic_cryptodev_find(p, name) ||
+		(params->n_queues == 0) ||
+		(params->queue_size == 0))
+		return NULL;
+
+	if (params->dev_name) {
+		status = rte_cryptodev_get_dev_id(params->dev_name);
+		if (status == -1)
+			return NULL;
+
+		dev_id = (uint32_t)status;
+	} else {
+		if (rte_cryptodev_pmd_is_valid_dev(params->dev_id) == 0)
+			return NULL;
+
+		dev_id = params->dev_id;
+	}
+
+	socket_id = rte_cryptodev_socket_id(dev_id);
+	rte_cryptodev_info_get(dev_id, &dev_info);
+
+	if (dev_info.max_nb_queue_pairs < params->n_queues)
+		return NULL;
+	if (dev_info.feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED)
+		return NULL;
+
+	dev_conf.socket_id = socket_id;
+	dev_conf.nb_queue_pairs = params->n_queues;
+
+	status = rte_cryptodev_configure(dev_id, &dev_conf);
+	if (status < 0)
+		return NULL;
+
+	queue_conf.nb_descriptors = params->queue_size;
+	for (i = 0; i < params->n_queues; i++) {
+		status = rte_cryptodev_queue_pair_setup(dev_id, i,
+				&queue_conf, socket_id, NULL);
+		if (status < 0)
+			return NULL;
+	}
+
+	if (rte_cryptodev_start(dev_id) < 0)
+		return NULL;
+
+	cryptodev = calloc(1, sizeof(struct softnic_cryptodev));
+	if (cryptodev == NULL) {
+		rte_cryptodev_stop(dev_id);
+		return NULL;
+	}
+
+	strlcpy(cryptodev->name, name, sizeof(cryptodev->name));
+	cryptodev->dev_id = dev_id;
+	cryptodev->n_queues = params->n_queues;
+
+	TAILQ_INSERT_TAIL(&p->cryptodev_list, cryptodev, node);
+
+	return cryptodev;
+}
diff --git a/drivers/net/softnic/rte_eth_softnic_flow.c b/drivers/net/softnic/rte_eth_softnic_flow.c
new file mode 100644
index 00000000..285af462
--- /dev/null
+++ b/drivers/net/softnic/rte_eth_softnic_flow.c
@@ -0,0 +1,2287 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_malloc.h>
+#include <rte_string_fns.h>
+#include <rte_flow.h>
+#include <rte_flow_driver.h>
+#include <rte_tailq.h>
+
+#include "rte_eth_softnic_internals.h"
+#include "rte_eth_softnic.h"
+
+#define rte_htons rte_cpu_to_be_16
+#define rte_htonl rte_cpu_to_be_32
+
+#define rte_ntohs rte_be_to_cpu_16
+#define rte_ntohl rte_be_to_cpu_32
+
+static struct rte_flow *
+softnic_flow_find(struct softnic_table *table,
+	struct softnic_table_rule_match *rule_match)
+{
+	struct rte_flow *flow;
+
+	TAILQ_FOREACH(flow, &table->flows, node)
+		if (memcmp(&flow->match, rule_match, sizeof(*rule_match)) == 0)
+			return flow;
+
+	return NULL;
+}
+
+int
+flow_attr_map_set(struct pmd_internals *softnic,
+		uint32_t group_id,
+		int ingress,
+		const char *pipeline_name,
+		uint32_t table_id)
+{
+	struct pipeline *pipeline;
+	struct flow_attr_map *map;
+
+	if (group_id >= SOFTNIC_FLOW_MAX_GROUPS ||
+			pipeline_name == NULL)
+		return -1;
+
+	pipeline = softnic_pipeline_find(softnic, pipeline_name);
+	if (pipeline == NULL ||
+			table_id >= pipeline->n_tables)
+		return -1;
+
+	map = (ingress) ? &softnic->flow.ingress_map[group_id] :
+		&softnic->flow.egress_map[group_id];
+	strcpy(map->pipeline_name, pipeline_name);
+	map->table_id = table_id;
+	map->valid = 1;
+
+	return 0;
+}
+
+struct flow_attr_map *
+flow_attr_map_get(struct pmd_internals *softnic,
+		uint32_t group_id,
+		int ingress)
+{
+	if (group_id >= SOFTNIC_FLOW_MAX_GROUPS)
+		return NULL;
+
+	return (ingress) ? &softnic->flow.ingress_map[group_id] :
+		&softnic->flow.egress_map[group_id];
+}
+
+static int
+flow_pipeline_table_get(struct pmd_internals *softnic,
+		const struct rte_flow_attr *attr,
+		const char **pipeline_name,
+		uint32_t *table_id,
+		struct rte_flow_error *error)
+{
+	struct flow_attr_map *map;
+
+	if (attr == NULL)
+		return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ATTR,
+				NULL,
+				"Null attr");
+
+	if (!attr->ingress && !attr->egress)
+		return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+				attr,
+				"Ingress/egress not specified");
+
+	if (attr->ingress && attr->egress)
+		return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+				attr,
+				"Setting both ingress and egress is not allowed");
+
+	map = flow_attr_map_get(softnic,
+			attr->group,
+			attr->ingress);
+	if (map == NULL ||
+			map->valid == 0)
+		return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+				attr,
+				"Invalid group ID");
+
+	if (pipeline_name)
+		*pipeline_name = map->pipeline_name;
+
+	if (table_id)
+		*table_id = map->table_id;
+
+	return 0;
+}
+
+union flow_item {
+	uint8_t raw[TABLE_RULE_MATCH_SIZE_MAX];
+	struct rte_flow_item_eth eth;
+	struct rte_flow_item_vlan vlan;
+	struct rte_flow_item_ipv4 ipv4;
+	struct rte_flow_item_ipv6 ipv6;
+	struct rte_flow_item_icmp icmp;
+	struct rte_flow_item_udp udp;
+	struct rte_flow_item_tcp tcp;
+	struct rte_flow_item_sctp sctp;
+	struct rte_flow_item_vxlan vxlan;
+	struct rte_flow_item_e_tag e_tag;
+	struct rte_flow_item_nvgre nvgre;
+	struct rte_flow_item_mpls mpls;
+	struct rte_flow_item_gre gre;
+	struct rte_flow_item_gtp gtp;
+	struct rte_flow_item_esp esp;
+	struct rte_flow_item_geneve geneve;
+	struct rte_flow_item_vxlan_gpe vxlan_gpe;
+	struct rte_flow_item_arp_eth_ipv4 arp_eth_ipv4;
+	struct rte_flow_item_ipv6_ext ipv6_ext;
+	struct rte_flow_item_icmp6 icmp6;
+	struct rte_flow_item_icmp6_nd_ns icmp6_nd_ns;
+	struct rte_flow_item_icmp6_nd_na icmp6_nd_na;
+	struct rte_flow_item_icmp6_nd_opt icmp6_nd_opt;
+	struct rte_flow_item_icmp6_nd_opt_sla_eth icmp6_nd_opt_sla_eth;
+	struct rte_flow_item_icmp6_nd_opt_tla_eth icmp6_nd_opt_tla_eth;
+};
+
+static const union flow_item flow_item_raw_mask;
+
+static int
+flow_item_is_proto(enum rte_flow_item_type type,
+	const void **mask,
+	size_t *size)
+{
+	switch (type) {
+	case RTE_FLOW_ITEM_TYPE_RAW:
+		*mask = &flow_item_raw_mask;
+		*size = sizeof(flow_item_raw_mask);
+		return 1; /* TRUE */
+
+	case RTE_FLOW_ITEM_TYPE_ETH:
+		*mask = &rte_flow_item_eth_mask;
+		*size = sizeof(struct rte_flow_item_eth);
+		return 1; /* TRUE */
+
+	case RTE_FLOW_ITEM_TYPE_VLAN:
+		*mask = &rte_flow_item_vlan_mask;
+		*size = sizeof(struct rte_flow_item_vlan);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_IPV4:
+		*mask = &rte_flow_item_ipv4_mask;
+		*size = sizeof(struct rte_flow_item_ipv4);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_IPV6:
+		*mask = &rte_flow_item_ipv6_mask;
+		*size = sizeof(struct rte_flow_item_ipv6);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_ICMP:
+		*mask = &rte_flow_item_icmp_mask;
+		*size = sizeof(struct rte_flow_item_icmp);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_UDP:
+		*mask = &rte_flow_item_udp_mask;
+		*size = sizeof(struct rte_flow_item_udp);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_TCP:
+		*mask = &rte_flow_item_tcp_mask;
+		*size = sizeof(struct rte_flow_item_tcp);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_SCTP:
+		*mask = &rte_flow_item_sctp_mask;
+		*size = sizeof(struct rte_flow_item_sctp);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_VXLAN:
+		*mask = &rte_flow_item_vxlan_mask;
+		*size = sizeof(struct rte_flow_item_vxlan);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_E_TAG:
+		*mask = &rte_flow_item_e_tag_mask;
+		*size = sizeof(struct rte_flow_item_e_tag);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_NVGRE:
+		*mask = &rte_flow_item_nvgre_mask;
+		*size = sizeof(struct rte_flow_item_nvgre);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_MPLS:
+		*mask = &rte_flow_item_mpls_mask;
+		*size = sizeof(struct rte_flow_item_mpls);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_GRE:
+		*mask = &rte_flow_item_gre_mask;
+		*size = sizeof(struct rte_flow_item_gre);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_GTP:
+	case RTE_FLOW_ITEM_TYPE_GTPC:
+	case RTE_FLOW_ITEM_TYPE_GTPU:
+		*mask = &rte_flow_item_gtp_mask;
+		*size = sizeof(struct rte_flow_item_gtp);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_ESP:
+		*mask = &rte_flow_item_esp_mask;
+		*size = sizeof(struct rte_flow_item_esp);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_GENEVE:
+		*mask = &rte_flow_item_geneve_mask;
+		*size = sizeof(struct rte_flow_item_geneve);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+		*mask = &rte_flow_item_vxlan_gpe_mask;
+		*size = sizeof(struct rte_flow_item_vxlan_gpe);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_ARP_ETH_IPV4:
+		*mask = &rte_flow_item_arp_eth_ipv4_mask;
+		*size = sizeof(struct rte_flow_item_arp_eth_ipv4);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_IPV6_EXT:
+		*mask = &rte_flow_item_ipv6_ext_mask;
+		*size = sizeof(struct rte_flow_item_ipv6_ext);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_ICMP6:
+		*mask = &rte_flow_item_icmp6_mask;
+		*size = sizeof(struct rte_flow_item_icmp6);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_ICMP6_ND_NS:
+		*mask = &rte_flow_item_icmp6_nd_ns_mask;
+		*size = sizeof(struct rte_flow_item_icmp6_nd_ns);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_ICMP6_ND_NA:
+		*mask = &rte_flow_item_icmp6_nd_na_mask;
+		*size = sizeof(struct rte_flow_item_icmp6_nd_na);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_ICMP6_ND_OPT:
+		*mask = &rte_flow_item_icmp6_nd_opt_mask;
+		*size = sizeof(struct rte_flow_item_icmp6_nd_opt);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_ICMP6_ND_OPT_SLA_ETH:
+		*mask = &rte_flow_item_icmp6_nd_opt_sla_eth_mask;
+		*size = sizeof(struct rte_flow_item_icmp6_nd_opt_sla_eth);
+		return 1;
+
+	case RTE_FLOW_ITEM_TYPE_ICMP6_ND_OPT_TLA_ETH:
+		*mask = &rte_flow_item_icmp6_nd_opt_tla_eth_mask;
+		*size = sizeof(struct rte_flow_item_icmp6_nd_opt_tla_eth);
+		return 1;
+
+	default: return 0; /* FALSE */
+	}
+}
+
+static int
+flow_item_raw_preprocess(const struct rte_flow_item *item,
+	union flow_item *item_spec,
+	union flow_item *item_mask,
+	size_t *item_size,
+	int *item_disabled,
+	struct rte_flow_error *error)
+{
+	const struct rte_flow_item_raw *item_raw_spec = item->spec;
+	const struct rte_flow_item_raw *item_raw_mask = item->mask;
+	const uint8_t *pattern;
+	const uint8_t *pattern_mask;
+	uint8_t *spec = (uint8_t *)item_spec;
+	uint8_t *mask = (uint8_t *)item_mask;
+	size_t pattern_length, pattern_offset, i;
+	int disabled;
+
+	if (!item->spec)
+		return rte_flow_error_set(error,
+			ENOTSUP,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"RAW: Null specification");
+
+	if (item->last)
+		return rte_flow_error_set(error,
+			ENOTSUP,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"RAW: Range not allowed (last must be NULL)");
+
+	if (item_raw_spec->relative == 0)
+		return rte_flow_error_set(error,
+			ENOTSUP,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"RAW: Absolute offset not supported");
+
+	if (item_raw_spec->search)
+		return rte_flow_error_set(error,
+			ENOTSUP,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"RAW: Search not supported");
+
+	if (item_raw_spec->offset < 0)
+		return rte_flow_error_set(error,
+			ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"RAW: Negative offset not supported");
+
+	if (item_raw_spec->length == 0)
+		return rte_flow_error_set(error,
+			ENOTSUP,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"RAW: Zero pattern length");
+
+	if (item_raw_spec->offset + item_raw_spec->length >
+		TABLE_RULE_MATCH_SIZE_MAX)
+		return rte_flow_error_set(error,
+			ENOTSUP,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"RAW: Item too big");
+
+	if (!item_raw_spec->pattern && item_raw_mask && item_raw_mask->pattern)
+		return rte_flow_error_set(error,
+			ENOTSUP,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"RAW: Non-NULL pattern mask not allowed with NULL pattern");
+
+	pattern = item_raw_spec->pattern;
+	pattern_mask = (item_raw_mask) ? item_raw_mask->pattern : NULL;
+	pattern_length = (size_t)item_raw_spec->length;
+	pattern_offset = (size_t)item_raw_spec->offset;
+
+	disabled = 0;
+	if (pattern_mask == NULL)
+		disabled = 1;
+	else
+		for (i = 0; i < pattern_length; i++)
+			if ((pattern)[i])
+				disabled = 1;
+
+	memset(spec, 0, TABLE_RULE_MATCH_SIZE_MAX);
+	if (pattern)
+		memcpy(&spec[pattern_offset], pattern, pattern_length);
+
+	memset(mask, 0, TABLE_RULE_MATCH_SIZE_MAX);
+	if (pattern_mask)
+		memcpy(&mask[pattern_offset], pattern_mask, pattern_length);
+
+	*item_size = pattern_offset + pattern_length;
+	*item_disabled = disabled;
+
+	return 0;
+}
+
+static int
+flow_item_proto_preprocess(const struct rte_flow_item *item,
+	union flow_item *item_spec,
+	union flow_item *item_mask,
+	size_t *item_size,
+	int *item_disabled,
+	struct rte_flow_error *error)
+{
+	const void *mask_default;
+	uint8_t *spec = (uint8_t *)item_spec;
+	uint8_t *mask = (uint8_t *)item_mask;
+	size_t size, i;
+
+	if (!flow_item_is_proto(item->type, &mask_default, &size))
+		return rte_flow_error_set(error,
+			ENOTSUP,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"Item type not supported");
+
+	if (item->type == RTE_FLOW_ITEM_TYPE_RAW)
+		return flow_item_raw_preprocess(item,
+			item_spec,
+			item_mask,
+			item_size,
+			item_disabled,
+			error);
+
+	/* spec */
+	if (!item->spec) {
+		/* If spec is NULL, then last and mask also have to be NULL. */
+		if (item->last || item->mask)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"Invalid item (NULL spec with non-NULL last or mask)");
+
+		memset(item_spec, 0, size);
+		memset(item_mask, 0, size);
+		*item_size = size;
+		*item_disabled = 1; /* TRUE */
+		return 0;
+	}
+
+	memcpy(spec, item->spec, size);
+	*item_size = size;
+
+	/* mask */
+	if (item->mask)
+		memcpy(mask, item->mask, size);
+	else
+		memcpy(mask, mask_default, size);
+
+	/* disabled */
+	for (i = 0; i < size; i++)
+		if (mask[i])
+			break;
+	*item_disabled = (i == size) ? 1 : 0;
+
+	/* Apply mask over spec. */
+	for (i = 0; i < size; i++)
+		spec[i] &= mask[i];
+
+	/* last */
+	if (item->last) {
+		uint8_t last[size];
+
+		/* init last */
+		memcpy(last, item->last, size);
+		for (i = 0; i < size; i++)
+			last[i] &= mask[i];
+
+		/* check for range */
+		for (i = 0; i < size; i++)
+			if (last[i] != spec[i])
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_ITEM,
+					item,
+					"Range not supported");
+	}
+
+	return 0;
+}
+
+/***
+ * Skip disabled protocol items and VOID items
+ * until any of the mutually exclusive conditions
+ * from the list below takes place:
+ *    (A) A protocol present in the proto_mask
+ *        is met (either ENABLED or DISABLED);
+ *    (B) A protocol NOT present in the proto_mask is met in ENABLED state;
+ *    (C) The END item is met.
+ */
+static int
+flow_item_skip_disabled_protos(const struct rte_flow_item **item,
+	uint64_t proto_mask,
+	size_t *length,
+	struct rte_flow_error *error)
+{
+	size_t len = 0;
+
+	for ( ; (*item)->type != RTE_FLOW_ITEM_TYPE_END; (*item)++) {
+		union flow_item spec, mask;
+		size_t size;
+		int disabled = 0, status;
+
+		if ((*item)->type == RTE_FLOW_ITEM_TYPE_VOID)
+			continue;
+
+		status = flow_item_proto_preprocess(*item,
+				&spec,
+				&mask,
+				&size,
+				&disabled,
+				error);
+		if (status)
+			return status;
+
+		if ((proto_mask & (1LLU << (*item)->type)) ||
+				!disabled)
+			break;
+
+		len += size;
+	}
+
+	if (length)
+		*length = len;
+
+	return 0;
+}
+
+#define FLOW_ITEM_PROTO_IP \
+	((1LLU << RTE_FLOW_ITEM_TYPE_IPV4) | \
+	 (1LLU << RTE_FLOW_ITEM_TYPE_IPV6))
+
+static void
+flow_item_skip_void(const struct rte_flow_item **item)
+{
+	for ( ; ; (*item)++)
+		if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
+			return;
+}
+
+#define IP_PROTOCOL_TCP 0x06
+#define IP_PROTOCOL_UDP 0x11
+#define IP_PROTOCOL_SCTP 0x84
+
+static int
+mask_to_depth(uint64_t mask,
+		uint32_t *depth)
+{
+	uint64_t n;
+
+	if (mask == UINT64_MAX) {
+		if (depth)
+			*depth = 64;
+
+		return 0;
+	}
+
+	mask = ~mask;
+
+	if (mask & (mask + 1))
+		return -1;
+
+	n = __builtin_popcountll(mask);
+	if (depth)
+		*depth = (uint32_t)(64 - n);
+
+	return 0;
+}
+
+static int
+ipv4_mask_to_depth(uint32_t mask,
+		uint32_t *depth)
+{
+	uint32_t d;
+	int status;
+
+	status = mask_to_depth(mask | (UINT64_MAX << 32), &d);
+	if (status)
+		return status;
+
+	d -= 32;
+	if (depth)
+		*depth = d;
+
+	return 0;
+}
+
+static int
+ipv6_mask_to_depth(uint8_t *mask,
+	uint32_t *depth)
+{
+	uint64_t *m = (uint64_t *)mask;
+	uint64_t m0 = rte_be_to_cpu_64(m[0]);
+	uint64_t m1 = rte_be_to_cpu_64(m[1]);
+	uint32_t d0, d1;
+	int status;
+
+	status = mask_to_depth(m0, &d0);
+	if (status)
+		return status;
+
+	status = mask_to_depth(m1, &d1);
+	if (status)
+		return status;
+
+	if (d0 < 64 && d1)
+		return -1;
+
+	if (depth)
+		*depth = d0 + d1;
+
+	return 0;
+}
+
+static int
+port_mask_to_range(uint16_t port,
+	uint16_t port_mask,
+	uint16_t *port0,
+	uint16_t *port1)
+{
+	int status;
+	uint16_t p0, p1;
+
+	status = mask_to_depth(port_mask | (UINT64_MAX << 16), NULL);
+	if (status)
+		return -1;
+
+	p0 = port & port_mask;
+	p1 = p0 | ~port_mask;
+
+	if (port0)
+		*port0 = p0;
+
+	if (port1)
+		*port1 = p1;
+
+	return 0;
+}
+
+static int
+flow_rule_match_acl_get(struct pmd_internals *softnic __rte_unused,
+		struct pipeline *pipeline __rte_unused,
+		struct softnic_table *table __rte_unused,
+		const struct rte_flow_attr *attr,
+		const struct rte_flow_item *item,
+		struct softnic_table_rule_match *rule_match,
+		struct rte_flow_error *error)
+{
+	union flow_item spec, mask;
+	size_t size, length = 0;
+	int disabled = 0, status;
+	uint8_t ip_proto, ip_proto_mask;
+
+	memset(rule_match, 0, sizeof(*rule_match));
+	rule_match->match_type = TABLE_ACL;
+	rule_match->match.acl.priority = attr->priority;
+
+	/* VOID or disabled protos only, if any. */
+	status = flow_item_skip_disabled_protos(&item,
+			FLOW_ITEM_PROTO_IP, &length, error);
+	if (status)
+		return status;
+
+	/* IP only. */
+	status = flow_item_proto_preprocess(item, &spec, &mask,
+			&size, &disabled, error);
+	if (status)
+		return status;
+
+	switch (item->type) {
+	case RTE_FLOW_ITEM_TYPE_IPV4:
+	{
+		uint32_t sa_depth, da_depth;
+
+		status = ipv4_mask_to_depth(rte_ntohl(mask.ipv4.hdr.src_addr),
+				&sa_depth);
+		if (status)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Illegal IPv4 header source address mask");
+
+		status = ipv4_mask_to_depth(rte_ntohl(mask.ipv4.hdr.dst_addr),
+				&da_depth);
+		if (status)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Illegal IPv4 header destination address mask");
+
+		ip_proto = spec.ipv4.hdr.next_proto_id;
+		ip_proto_mask = mask.ipv4.hdr.next_proto_id;
+
+		rule_match->match.acl.ip_version = 1;
+		rule_match->match.acl.ipv4.sa =
+			rte_ntohl(spec.ipv4.hdr.src_addr);
+		rule_match->match.acl.ipv4.da =
+			rte_ntohl(spec.ipv4.hdr.dst_addr);
+		rule_match->match.acl.sa_depth = sa_depth;
+		rule_match->match.acl.da_depth = da_depth;
+		rule_match->match.acl.proto = ip_proto;
+		rule_match->match.acl.proto_mask = ip_proto_mask;
+		break;
+	} /* RTE_FLOW_ITEM_TYPE_IPV4 */
+
+	case RTE_FLOW_ITEM_TYPE_IPV6:
+	{
+		uint32_t sa_depth, da_depth;
+
+		status = ipv6_mask_to_depth(mask.ipv6.hdr.src_addr, &sa_depth);
+		if (status)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Illegal IPv6 header source address mask");
+
+		status = ipv6_mask_to_depth(mask.ipv6.hdr.dst_addr, &da_depth);
+		if (status)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Illegal IPv6 header destination address mask");
+
+		ip_proto = spec.ipv6.hdr.proto;
+		ip_proto_mask = mask.ipv6.hdr.proto;
+
+		rule_match->match.acl.ip_version = 0;
+		memcpy(rule_match->match.acl.ipv6.sa,
+			spec.ipv6.hdr.src_addr,
+			sizeof(spec.ipv6.hdr.src_addr));
+		memcpy(rule_match->match.acl.ipv6.da,
+			spec.ipv6.hdr.dst_addr,
+			sizeof(spec.ipv6.hdr.dst_addr));
+		rule_match->match.acl.sa_depth = sa_depth;
+		rule_match->match.acl.da_depth = da_depth;
+		rule_match->match.acl.proto = ip_proto;
+		rule_match->match.acl.proto_mask = ip_proto_mask;
+		break;
+	} /* RTE_FLOW_ITEM_TYPE_IPV6 */
+
+	default:
+		return rte_flow_error_set(error,
+			ENOTSUP,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"ACL: IP protocol required");
+	} /* switch */
+
+	if (ip_proto_mask != UINT8_MAX)
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"ACL: Illegal IP protocol mask");
+
+	item++;
+
+	/* VOID only, if any. */
+	flow_item_skip_void(&item);
+
+	/* TCP/UDP/SCTP only. */
+	status = flow_item_proto_preprocess(item, &spec, &mask,
+			&size, &disabled, error);
+	if (status)
+		return status;
+
+	switch (item->type) {
+	case RTE_FLOW_ITEM_TYPE_TCP:
+	{
+		uint16_t sp0, sp1, dp0, dp1;
+
+		if (ip_proto != IP_PROTOCOL_TCP)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Item type is TCP, but IP protocol is not");
+
+		status = port_mask_to_range(rte_ntohs(spec.tcp.hdr.src_port),
+				rte_ntohs(mask.tcp.hdr.src_port),
+				&sp0,
+				&sp1);
+
+		if (status)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Illegal TCP source port mask");
+
+		status = port_mask_to_range(rte_ntohs(spec.tcp.hdr.dst_port),
+				rte_ntohs(mask.tcp.hdr.dst_port),
+				&dp0,
+				&dp1);
+
+		if (status)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Illegal TCP destination port mask");
+
+		rule_match->match.acl.sp0 = sp0;
+		rule_match->match.acl.sp1 = sp1;
+		rule_match->match.acl.dp0 = dp0;
+		rule_match->match.acl.dp1 = dp1;
+
+		break;
+	} /* RTE_FLOW_ITEM_TYPE_TCP */
+
+	case RTE_FLOW_ITEM_TYPE_UDP:
+	{
+		uint16_t sp0, sp1, dp0, dp1;
+
+		if (ip_proto != IP_PROTOCOL_UDP)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Item type is UDP, but IP protocol is not");
+
+		status = port_mask_to_range(rte_ntohs(spec.udp.hdr.src_port),
+			rte_ntohs(mask.udp.hdr.src_port),
+			&sp0,
+			&sp1);
+		if (status)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Illegal UDP source port mask");
+
+		status = port_mask_to_range(rte_ntohs(spec.udp.hdr.dst_port),
+			rte_ntohs(mask.udp.hdr.dst_port),
+			&dp0,
+			&dp1);
+		if (status)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Illegal UDP destination port mask");
+
+		rule_match->match.acl.sp0 = sp0;
+		rule_match->match.acl.sp1 = sp1;
+		rule_match->match.acl.dp0 = dp0;
+		rule_match->match.acl.dp1 = dp1;
+
+		break;
+	} /* RTE_FLOW_ITEM_TYPE_UDP */
+
+	case RTE_FLOW_ITEM_TYPE_SCTP:
+	{
+		uint16_t sp0, sp1, dp0, dp1;
+
+		if (ip_proto != IP_PROTOCOL_SCTP)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Item type is SCTP, but IP protocol is not");
+
+		status = port_mask_to_range(rte_ntohs(spec.sctp.hdr.src_port),
+			rte_ntohs(mask.sctp.hdr.src_port),
+			&sp0,
+			&sp1);
+
+		if (status)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Illegal SCTP source port mask");
+
+		status = port_mask_to_range(rte_ntohs(spec.sctp.hdr.dst_port),
+			rte_ntohs(mask.sctp.hdr.dst_port),
+			&dp0,
+			&dp1);
+		if (status)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"ACL: Illegal SCTP destination port mask");
+
+		rule_match->match.acl.sp0 = sp0;
+		rule_match->match.acl.sp1 = sp1;
+		rule_match->match.acl.dp0 = dp0;
+		rule_match->match.acl.dp1 = dp1;
+
+		break;
+	} /* RTE_FLOW_ITEM_TYPE_SCTP */
+
+	default:
+		return rte_flow_error_set(error,
+			ENOTSUP,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"ACL: TCP/UDP/SCTP required");
+	} /* switch */
+
+	item++;
+
+	/* VOID or disabled protos only, if any. */
+	status = flow_item_skip_disabled_protos(&item, 0, NULL, error);
+	if (status)
+		return status;
+
+	/* END only. */
+	if (item->type != RTE_FLOW_ITEM_TYPE_END)
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"ACL: Expecting END item");
+
+	return 0;
+}
+
+/***
+ * Both *tmask* and *fmask* are byte arrays of size *tsize* and *fsize*
+ * respectively.
+ * They are located within a larger buffer at offsets *toffset* and *foffset*
+ * respectivelly. Both *tmask* and *fmask* represent bitmasks for the larger
+ * buffer.
+ * Question: are the two masks equivalent?
+ *
+ * Notes:
+ * 1. Offset basically indicates that the first offset bytes in the buffer
+ *    are "don't care", so offset is equivalent to pre-pending an "all-zeros"
+ *    array of *offset* bytes to the *mask*.
+ * 2. Each *mask* might contain a number of zero bytes at the beginning or
+ *    at the end.
+ * 3. Bytes in the larger buffer after the end of the *mask* are also considered
+ *    "don't care", so they are equivalent to appending an "all-zeros" array of
+ *    bytes to the *mask*.
+ *
+ * Example:
+ * Buffer = [xx xx xx xx xx xx xx xx], buffer size = 8 bytes
+ * tmask = [00 22 00 33 00], toffset = 2, tsize = 5
+ *    => buffer mask = [00 00 00 22 00 33 00 00]
+ * fmask = [22 00 33], foffset = 3, fsize = 3 =>
+ *    => buffer mask = [00 00 00 22 00 33 00 00]
+ * Therefore, the tmask and fmask from this example are equivalent.
+ */
+static int
+hash_key_mask_is_same(uint8_t *tmask,
+	size_t toffset,
+	size_t tsize,
+	uint8_t *fmask,
+	size_t foffset,
+	size_t fsize,
+	size_t *toffset_plus,
+	size_t *foffset_plus)
+{
+	size_t tpos; /* Position of first non-zero byte in the tmask buffer. */
+	size_t fpos; /* Position of first non-zero byte in the fmask buffer. */
+
+	/* Compute tpos and fpos. */
+	for (tpos = 0; tmask[tpos] == 0; tpos++)
+		;
+	for (fpos = 0; fmask[fpos] == 0; fpos++)
+		;
+
+	if (toffset + tpos != foffset + fpos)
+		return 0; /* FALSE */
+
+	tsize -= tpos;
+	fsize -= fpos;
+
+	if (tsize < fsize) {
+		size_t i;
+
+		for (i = 0; i < tsize; i++)
+			if (tmask[tpos + i] != fmask[fpos + i])
+				return 0; /* FALSE */
+
+		for ( ; i < fsize; i++)
+			if (fmask[fpos + i])
+				return 0; /* FALSE */
+	} else {
+		size_t i;
+
+		for (i = 0; i < fsize; i++)
+			if (tmask[tpos + i] != fmask[fpos + i])
+				return 0; /* FALSE */
+
+		for ( ; i < tsize; i++)
+			if (tmask[tpos + i])
+				return 0; /* FALSE */
+	}
+
+	if (toffset_plus)
+		*toffset_plus = tpos;
+
+	if (foffset_plus)
+		*foffset_plus = fpos;
+
+	return 1; /* TRUE */
+}
+
+static int
+flow_rule_match_hash_get(struct pmd_internals *softnic __rte_unused,
+	struct pipeline *pipeline __rte_unused,
+	struct softnic_table *table,
+	const struct rte_flow_attr *attr __rte_unused,
+	const struct rte_flow_item *item,
+	struct softnic_table_rule_match *rule_match,
+	struct rte_flow_error *error)
+{
+	struct softnic_table_rule_match_hash key, key_mask;
+	struct softnic_table_hash_params *params = &table->params.match.hash;
+	size_t offset = 0, length = 0, tpos, fpos;
+	int status;
+
+	memset(&key, 0, sizeof(key));
+	memset(&key_mask, 0, sizeof(key_mask));
+
+	/* VOID or disabled protos only, if any. */
+	status = flow_item_skip_disabled_protos(&item, 0, &offset, error);
+	if (status)
+		return status;
+
+	if (item->type == RTE_FLOW_ITEM_TYPE_END)
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			item,
+			"HASH: END detected too early");
+
+	/* VOID or any protocols (enabled or disabled). */
+	for ( ; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
+		union flow_item spec, mask;
+		size_t size;
+		int disabled, status;
+
+		if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
+			continue;
+
+		status = flow_item_proto_preprocess(item,
+			&spec,
+			&mask,
+			&size,
+			&disabled,
+			error);
+		if (status)
+			return status;
+
+		if (length + size > sizeof(key)) {
+			if (disabled)
+				break;
+
+			return rte_flow_error_set(error,
+				ENOTSUP,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"HASH: Item too big");
+		}
+
+		memcpy(&key.key[length], &spec, size);
+		memcpy(&key_mask.key[length], &mask, size);
+		length += size;
+	}
+
+	if (item->type != RTE_FLOW_ITEM_TYPE_END) {
+		/* VOID or disabled protos only, if any. */
+		status = flow_item_skip_disabled_protos(&item, 0, NULL, error);
+		if (status)
+			return status;
+
+		/* END only. */
+		if (item->type != RTE_FLOW_ITEM_TYPE_END)
+			return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				item,
+				"HASH: Expecting END item");
+	}
+
+	/* Compare flow key mask against table key mask. */
+	offset += sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM;
+
+	if (!hash_key_mask_is_same(params->key_mask,
+		params->key_offset,
+		params->key_size,
+		key_mask.key,
+		offset,
+		length,
+		&tpos,
+		&fpos))
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"HASH: Item list is not observing the match format");
+
+	/* Rule match. */
+	memset(rule_match, 0, sizeof(*rule_match));
+	rule_match->match_type = TABLE_HASH;
+	memcpy(&rule_match->match.hash.key[tpos],
+		&key.key[fpos],
+		RTE_MIN(sizeof(rule_match->match.hash.key) - tpos,
+			length - fpos));
+
+	return 0;
+}
+
+static int
+flow_rule_match_get(struct pmd_internals *softnic,
+		struct pipeline *pipeline,
+		struct softnic_table *table,
+		const struct rte_flow_attr *attr,
+		const struct rte_flow_item *item,
+		struct softnic_table_rule_match *rule_match,
+		struct rte_flow_error *error)
+{
+	switch (table->params.match_type) {
+	case TABLE_ACL:
+		return flow_rule_match_acl_get(softnic,
+			pipeline,
+			table,
+			attr,
+			item,
+			rule_match,
+			error);
+
+		/* FALLTHROUGH */
+
+	case TABLE_HASH:
+		return flow_rule_match_hash_get(softnic,
+			pipeline,
+			table,
+			attr,
+			item,
+			rule_match,
+			error);
+
+		/* FALLTHROUGH */
+
+	default:
+		return rte_flow_error_set(error,
+			ENOTSUP,
+			RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Unsupported pipeline table match type");
+	}
+}
+
+static int
+flow_rule_action_get(struct pmd_internals *softnic,
+	struct pipeline *pipeline,
+	struct softnic_table *table,
+	const struct rte_flow_attr *attr,
+	const struct rte_flow_action *action,
+	struct softnic_table_rule_action *rule_action,
+	struct rte_flow_error *error)
+{
+	struct softnic_table_action_profile *profile;
+	struct softnic_table_action_profile_params *params;
+	int n_jump_queue_rss_drop = 0;
+	int n_count = 0;
+	int n_mark = 0;
+	int n_vxlan_decap = 0;
+
+	profile = softnic_table_action_profile_find(softnic,
+		table->params.action_profile_name);
+	if (profile == NULL)
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			action,
+			"JUMP: Table action profile");
+
+	params = &profile->params;
+
+	for ( ; action->type != RTE_FLOW_ACTION_TYPE_END; action++) {
+		if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
+			continue;
+
+		switch (action->type) {
+		case RTE_FLOW_ACTION_TYPE_JUMP:
+		{
+			const struct rte_flow_action_jump *conf = action->conf;
+			struct flow_attr_map *map;
+
+			if (conf == NULL)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"JUMP: Null configuration");
+
+			if (n_jump_queue_rss_drop)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"Only one termination action is"
+					" allowed per flow");
+
+			if ((params->action_mask &
+				(1LLU << RTE_TABLE_ACTION_FWD)) == 0)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"JUMP action not enabled for this table");
+
+			n_jump_queue_rss_drop = 1;
+
+			map = flow_attr_map_get(softnic,
+				conf->group,
+				attr->ingress);
+			if (map == NULL || map->valid == 0)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"JUMP: Invalid group mapping");
+
+			if (strcmp(pipeline->name, map->pipeline_name) != 0)
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"JUMP: Jump to table in different pipeline");
+
+			/* RTE_TABLE_ACTION_FWD */
+			rule_action->fwd.action = RTE_PIPELINE_ACTION_TABLE;
+			rule_action->fwd.id = map->table_id;
+			rule_action->action_mask |= 1 << RTE_TABLE_ACTION_FWD;
+			break;
+		} /* RTE_FLOW_ACTION_TYPE_JUMP */
+
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+		{
+			char name[NAME_SIZE];
+			struct rte_eth_dev *dev;
+			const struct rte_flow_action_queue *conf = action->conf;
+			uint32_t port_id;
+			int status;
+
+			if (conf == NULL)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"QUEUE: Null configuration");
+
+			if (n_jump_queue_rss_drop)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"Only one termination action is allowed"
+					" per flow");
+
+			if ((params->action_mask &
+				(1LLU << RTE_TABLE_ACTION_FWD)) == 0)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"QUEUE action not enabled for this table");
+
+			n_jump_queue_rss_drop = 1;
+
+			dev = ETHDEV(softnic);
+			if (dev == NULL ||
+				conf->index >= dev->data->nb_rx_queues)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"QUEUE: Invalid RX queue ID");
+
+			sprintf(name, "RXQ%u", (uint32_t)conf->index);
+
+			status = softnic_pipeline_port_out_find(softnic,
+				pipeline->name,
+				name,
+				&port_id);
+			if (status)
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"QUEUE: RX queue not accessible from this pipeline");
+
+			/* RTE_TABLE_ACTION_FWD */
+			rule_action->fwd.action = RTE_PIPELINE_ACTION_PORT;
+			rule_action->fwd.id = port_id;
+			rule_action->action_mask |= 1 << RTE_TABLE_ACTION_FWD;
+			break;
+		} /*RTE_FLOW_ACTION_TYPE_QUEUE */
+
+		case RTE_FLOW_ACTION_TYPE_RSS:
+		{
+			const struct rte_flow_action_rss *conf = action->conf;
+			uint32_t i;
+
+			if (conf == NULL)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"RSS: Null configuration");
+
+			if (!rte_is_power_of_2(conf->queue_num))
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					conf,
+					"RSS: Number of queues must be a power of 2");
+
+			if (conf->queue_num > RTE_DIM(rule_action->lb.out))
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					conf,
+					"RSS: Number of queues too big");
+
+			if (n_jump_queue_rss_drop)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"Only one termination action is allowed per flow");
+
+			if (((params->action_mask &
+				(1LLU << RTE_TABLE_ACTION_FWD)) == 0) ||
+				((params->action_mask &
+				(1LLU << RTE_TABLE_ACTION_LB)) == 0))
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"RSS action not supported by this table");
+
+			if (params->lb.out_offset !=
+				pipeline->params.offset_port_id)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"RSS action not supported by this pipeline");
+
+			n_jump_queue_rss_drop = 1;
+
+			/* RTE_TABLE_ACTION_LB */
+			for (i = 0; i < conf->queue_num; i++) {
+				char name[NAME_SIZE];
+				struct rte_eth_dev *dev;
+				uint32_t port_id;
+				int status;
+
+				dev = ETHDEV(softnic);
+				if (dev == NULL ||
+					conf->queue[i] >=
+						dev->data->nb_rx_queues)
+					return rte_flow_error_set(error,
+						EINVAL,
+						RTE_FLOW_ERROR_TYPE_ACTION,
+						action,
+						"RSS: Invalid RX queue ID");
+
+				sprintf(name, "RXQ%u",
+					(uint32_t)conf->queue[i]);
+
+				status = softnic_pipeline_port_out_find(softnic,
+					pipeline->name,
+					name,
+					&port_id);
+				if (status)
+					return rte_flow_error_set(error,
+						ENOTSUP,
+						RTE_FLOW_ERROR_TYPE_ACTION,
+						action,
+						"RSS: RX queue not accessible from this pipeline");
+
+				rule_action->lb.out[i] = port_id;
+			}
+
+			for ( ; i < RTE_DIM(rule_action->lb.out); i++)
+				rule_action->lb.out[i] =
+				rule_action->lb.out[i % conf->queue_num];
+
+			rule_action->action_mask |= 1 << RTE_TABLE_ACTION_LB;
+
+			/* RTE_TABLE_ACTION_FWD */
+			rule_action->fwd.action = RTE_PIPELINE_ACTION_PORT_META;
+			rule_action->action_mask |= 1 << RTE_TABLE_ACTION_FWD;
+			break;
+		} /* RTE_FLOW_ACTION_TYPE_RSS */
+
+		case RTE_FLOW_ACTION_TYPE_DROP:
+		{
+			const void *conf = action->conf;
+
+			if (conf != NULL)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"DROP: No configuration required");
+
+			if (n_jump_queue_rss_drop)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"Only one termination action is allowed per flow");
+			if ((params->action_mask &
+				(1LLU << RTE_TABLE_ACTION_FWD)) == 0)
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"DROP action not supported by this table");
+
+			n_jump_queue_rss_drop = 1;
+
+			/* RTE_TABLE_ACTION_FWD */
+			rule_action->fwd.action = RTE_PIPELINE_ACTION_DROP;
+			rule_action->action_mask |= 1 << RTE_TABLE_ACTION_FWD;
+			break;
+		} /* RTE_FLOW_ACTION_TYPE_DROP */
+
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+		{
+			const struct rte_flow_action_count *conf = action->conf;
+
+			if (conf == NULL)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"COUNT: Null configuration");
+
+			if (conf->shared)
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					conf,
+					"COUNT: Shared counters not supported");
+
+			if (n_count)
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"Only one COUNT action per flow");
+
+			if ((params->action_mask &
+				(1LLU << RTE_TABLE_ACTION_STATS)) == 0)
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"COUNT action not supported by this table");
+
+			n_count = 1;
+
+			/* RTE_TABLE_ACTION_STATS */
+			rule_action->stats.n_packets = 0;
+			rule_action->stats.n_bytes = 0;
+			rule_action->action_mask |= 1 << RTE_TABLE_ACTION_STATS;
+			break;
+		} /* RTE_FLOW_ACTION_TYPE_COUNT */
+
+		case RTE_FLOW_ACTION_TYPE_MARK:
+		{
+			const struct rte_flow_action_mark *conf = action->conf;
+
+			if (conf == NULL)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"MARK: Null configuration");
+
+			if (n_mark)
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"Only one MARK action per flow");
+
+			if ((params->action_mask &
+				(1LLU << RTE_TABLE_ACTION_TAG)) == 0)
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"MARK action not supported by this table");
+
+			n_mark = 1;
+
+			/* RTE_TABLE_ACTION_TAG */
+			rule_action->tag.tag = conf->id;
+			rule_action->action_mask |= 1 << RTE_TABLE_ACTION_TAG;
+			break;
+		} /* RTE_FLOW_ACTION_TYPE_MARK */
+
+		case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
+		{
+			const struct rte_flow_action_mark *conf = action->conf;
+
+			if (conf)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"VXLAN DECAP: Non-null configuration");
+
+			if (n_vxlan_decap)
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"Only one VXLAN DECAP action per flow");
+
+			if ((params->action_mask &
+				(1LLU << RTE_TABLE_ACTION_DECAP)) == 0)
+				return rte_flow_error_set(error,
+					ENOTSUP,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"VXLAN DECAP action not supported by this table");
+
+			n_vxlan_decap = 1;
+
+			/* RTE_TABLE_ACTION_DECAP */
+			rule_action->decap.n = 50; /* Ether/IPv4/UDP/VXLAN */
+			rule_action->action_mask |= 1 << RTE_TABLE_ACTION_DECAP;
+			break;
+		} /* RTE_FLOW_ACTION_TYPE_VXLAN_DECAP */
+
+		case RTE_FLOW_ACTION_TYPE_METER:
+		{
+			const struct rte_flow_action_meter *conf = action->conf;
+			struct softnic_mtr_meter_profile *mp;
+			struct softnic_mtr *m;
+			uint32_t table_id = table - pipeline->table;
+			uint32_t meter_profile_id;
+			int status;
+
+			if ((params->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"METER: Table action not supported");
+
+			if (params->mtr.n_tc != 1)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"METER: Multiple TCs not supported");
+
+			if (conf == NULL)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"METER: Null configuration");
+
+			m = softnic_mtr_find(softnic, conf->mtr_id);
+
+			if (m == NULL)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					NULL,
+					"METER: Invalid meter ID");
+
+			if (m->flow)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					NULL,
+					"METER: Meter already attached to a flow");
+
+			meter_profile_id = m->params.meter_profile_id;
+			mp = softnic_mtr_meter_profile_find(softnic, meter_profile_id);
+
+			/* Add meter profile to pipeline table */
+			if (!softnic_pipeline_table_meter_profile_find(table,
+					meter_profile_id)) {
+				struct rte_table_action_meter_profile profile;
+
+				memset(&profile, 0, sizeof(profile));
+				profile.alg = RTE_TABLE_ACTION_METER_TRTCM;
+				profile.trtcm.cir = mp->params.trtcm_rfc2698.cir;
+				profile.trtcm.pir = mp->params.trtcm_rfc2698.pir;
+				profile.trtcm.cbs = mp->params.trtcm_rfc2698.cbs;
+				profile.trtcm.pbs = mp->params.trtcm_rfc2698.pbs;
+
+				status = softnic_pipeline_table_mtr_profile_add(softnic,
+						pipeline->name,
+						table_id,
+						meter_profile_id,
+						&profile);
+				if (status) {
+					rte_flow_error_set(error,
+						EINVAL,
+						RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+						NULL,
+						"METER: Table meter profile add failed");
+					return -1;
+				}
+			}
+
+			/* RTE_TABLE_ACTION_METER */
+			rule_action->mtr.mtr[0].meter_profile_id = meter_profile_id;
+			rule_action->mtr.mtr[0].policer[e_RTE_METER_GREEN] =
+				(enum rte_table_action_policer)m->params.action[RTE_MTR_GREEN];
+			rule_action->mtr.mtr[0].policer[e_RTE_METER_YELLOW] =
+				(enum rte_table_action_policer)m->params.action[RTE_MTR_YELLOW];
+			rule_action->mtr.mtr[0].policer[e_RTE_METER_RED] =
+				(enum rte_table_action_policer)m->params.action[RTE_MTR_RED];
+			rule_action->mtr.tc_mask = 1;
+			rule_action->action_mask |= 1 << RTE_TABLE_ACTION_MTR;
+			break;
+		} /* RTE_FLOW_ACTION_TYPE_METER */
+
+		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
+		{
+			const struct rte_flow_action_vxlan_encap *conf =
+				action->conf;
+			const struct rte_flow_item *item;
+			union flow_item spec, mask;
+			int disabled = 0, status;
+			size_t size;
+
+			if (conf == NULL)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"VXLAN ENCAP: Null configuration");
+
+			item = conf->definition;
+			if (item == NULL)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION,
+					action,
+					"VXLAN ENCAP: Null configuration definition");
+
+			if (!(params->action_mask &
+					(1LLU << RTE_TABLE_ACTION_ENCAP)))
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"VXLAN ENCAP: Encap action not enabled for this table");
+
+			/* Check for Ether. */
+			flow_item_skip_void(&item);
+			status = flow_item_proto_preprocess(item, &spec, &mask,
+				&size, &disabled, error);
+			if (status)
+				return status;
+
+			if (item->type != RTE_FLOW_ITEM_TYPE_ETH) {
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ITEM,
+					item,
+					"VXLAN ENCAP: first encap item should be ether");
+			}
+			ether_addr_copy(&spec.eth.dst,
+					&rule_action->encap.vxlan.ether.da);
+			ether_addr_copy(&spec.eth.src,
+					&rule_action->encap.vxlan.ether.sa);
+
+			item++;
+
+			/* Check for VLAN. */
+			flow_item_skip_void(&item);
+			status = flow_item_proto_preprocess(item, &spec, &mask,
+					&size, &disabled, error);
+			if (status)
+				return status;
+
+			if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
+				if (!params->encap.vxlan.vlan)
+					return rte_flow_error_set(error,
+						ENOTSUP,
+						RTE_FLOW_ERROR_TYPE_ITEM,
+						item,
+						"VXLAN ENCAP: vlan encap not supported by table");
+
+				uint16_t tci = rte_ntohs(spec.vlan.tci);
+				rule_action->encap.vxlan.vlan.pcp =
+					tci >> 13;
+				rule_action->encap.vxlan.vlan.dei =
+					(tci >> 12) & 0x1;
+				rule_action->encap.vxlan.vlan.vid =
+					tci & 0xfff;
+
+				item++;
+
+				flow_item_skip_void(&item);
+				status = flow_item_proto_preprocess(item, &spec,
+						&mask, &size, &disabled, error);
+				if (status)
+					return status;
+			} else {
+				if (params->encap.vxlan.vlan)
+					return rte_flow_error_set(error,
+						ENOTSUP,
+						RTE_FLOW_ERROR_TYPE_ITEM,
+						item,
+						"VXLAN ENCAP: expecting vlan encap item");
+			}
+
+			/* Check for IPV4/IPV6. */
+			switch (item->type) {
+			case RTE_FLOW_ITEM_TYPE_IPV4:
+			{
+				rule_action->encap.vxlan.ipv4.sa =
+					rte_ntohl(spec.ipv4.hdr.src_addr);
+				rule_action->encap.vxlan.ipv4.da =
+					rte_ntohl(spec.ipv4.hdr.dst_addr);
+				rule_action->encap.vxlan.ipv4.dscp =
+					spec.ipv4.hdr.type_of_service >> 2;
+				rule_action->encap.vxlan.ipv4.ttl =
+					spec.ipv4.hdr.time_to_live;
+				break;
+			}
+			case RTE_FLOW_ITEM_TYPE_IPV6:
+			{
+				uint32_t vtc_flow;
+
+				memcpy(&rule_action->encap.vxlan.ipv6.sa,
+						&spec.ipv6.hdr.src_addr,
+						sizeof(spec.ipv6.hdr.src_addr));
+				memcpy(&rule_action->encap.vxlan.ipv6.da,
+						&spec.ipv6.hdr.dst_addr,
+						sizeof(spec.ipv6.hdr.dst_addr));
+				vtc_flow = rte_ntohl(spec.ipv6.hdr.vtc_flow);
+				rule_action->encap.vxlan.ipv6.flow_label =
+						vtc_flow & 0xfffff;
+				rule_action->encap.vxlan.ipv6.dscp =
+						(vtc_flow >> 22) & 0x3f;
+				rule_action->encap.vxlan.ipv6.hop_limit =
+					spec.ipv6.hdr.hop_limits;
+				break;
+			}
+			default:
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ITEM,
+					item,
+					"VXLAN ENCAP: encap item after ether should be ipv4/ipv6");
+			}
+
+			item++;
+
+			/* Check for UDP. */
+			flow_item_skip_void(&item);
+			status = flow_item_proto_preprocess(item, &spec, &mask,
+					&size, &disabled, error);
+			if (status)
+				return status;
+
+			if (item->type != RTE_FLOW_ITEM_TYPE_UDP) {
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ITEM,
+					item,
+					"VXLAN ENCAP: encap item after ipv4/ipv6 should be udp");
+			}
+			rule_action->encap.vxlan.udp.sp =
+				rte_ntohs(spec.udp.hdr.src_port);
+			rule_action->encap.vxlan.udp.dp =
+				rte_ntohs(spec.udp.hdr.dst_port);
+
+			item++;
+
+			/* Check for VXLAN. */
+			flow_item_skip_void(&item);
+			status = flow_item_proto_preprocess(item, &spec, &mask,
+					&size, &disabled, error);
+			if (status)
+				return status;
+
+			if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN) {
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ITEM,
+					item,
+					"VXLAN ENCAP: encap item after udp should be vxlan");
+			}
+			rule_action->encap.vxlan.vxlan.vni =
+				(spec.vxlan.vni[0] << 16U |
+					spec.vxlan.vni[1] << 8U
+					| spec.vxlan.vni[2]);
+
+			item++;
+
+			/* Check for END. */
+			flow_item_skip_void(&item);
+
+			if (item->type != RTE_FLOW_ITEM_TYPE_END)
+				return rte_flow_error_set(error,
+					EINVAL,
+					RTE_FLOW_ERROR_TYPE_ITEM,
+					item,
+					"VXLAN ENCAP: expecting END item");
+
+			rule_action->encap.type = RTE_TABLE_ACTION_ENCAP_VXLAN;
+			rule_action->action_mask |= 1 << RTE_TABLE_ACTION_ENCAP;
+			break;
+		} /* RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP */
+
+		default:
+			return -ENOTSUP;
+		}
+	}
+
+	if (n_jump_queue_rss_drop == 0)
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_ACTION,
+			action,
+			"Flow does not have any terminating action");
+
+	return 0;
+}
+
+static int
+pmd_flow_validate(struct rte_eth_dev *dev,
+		const struct rte_flow_attr *attr,
+		const struct rte_flow_item item[],
+		const struct rte_flow_action action[],
+		struct rte_flow_error *error)
+{
+	struct softnic_table_rule_match rule_match;
+	struct softnic_table_rule_action rule_action;
+
+	struct pmd_internals *softnic = dev->data->dev_private;
+	struct pipeline *pipeline;
+	struct softnic_table *table;
+	const char *pipeline_name = NULL;
+	uint32_t table_id = 0;
+	int status;
+
+	/* Check input parameters. */
+	if (attr == NULL)
+		return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ATTR,
+				NULL, "Null attr");
+
+	if (item == NULL)
+		return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ITEM,
+				NULL,
+				"Null item");
+
+	if (action == NULL)
+		return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_ACTION,
+				NULL,
+				"Null action");
+
+	/* Identify the pipeline table to add this flow to. */
+	status = flow_pipeline_table_get(softnic, attr, &pipeline_name,
+					&table_id, error);
+	if (status)
+		return status;
+
+	pipeline = softnic_pipeline_find(softnic, pipeline_name);
+	if (pipeline == NULL)
+		return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				NULL,
+				"Invalid pipeline name");
+
+	if (table_id >= pipeline->n_tables)
+		return rte_flow_error_set(error,
+				EINVAL,
+				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				NULL,
+				"Invalid pipeline table ID");
+
+	table = &pipeline->table[table_id];
+
+	/* Rule match. */
+	memset(&rule_match, 0, sizeof(rule_match));
+	status = flow_rule_match_get(softnic,
+			pipeline,
+			table,
+			attr,
+			item,
+			&rule_match,
+			error);
+	if (status)
+		return status;
+
+	/* Rule action. */
+	memset(&rule_action, 0, sizeof(rule_action));
+	status = flow_rule_action_get(softnic,
+		pipeline,
+		table,
+		attr,
+		action,
+		&rule_action,
+		error);
+	if (status)
+		return status;
+
+	return 0;
+}
+
+static struct softnic_mtr *
+flow_action_meter_get(struct pmd_internals *softnic,
+	const struct rte_flow_action *action)
+{
+	for ( ; action->type != RTE_FLOW_ACTION_TYPE_END; action++)
+		if (action->type == RTE_FLOW_ACTION_TYPE_METER) {
+			const struct rte_flow_action_meter *conf = action->conf;
+
+			if (conf == NULL)
+				return NULL;
+
+			return softnic_mtr_find(softnic, conf->mtr_id);
+		}
+
+	return NULL;
+}
+
+static void
+flow_meter_owner_reset(struct pmd_internals *softnic,
+	struct rte_flow *flow)
+{
+	struct softnic_mtr_list *ml = &softnic->mtr.mtrs;
+	struct softnic_mtr *m;
+
+	TAILQ_FOREACH(m, ml, node)
+		if (m->flow == flow) {
+			m->flow = NULL;
+			break;
+		}
+}
+
+static void
+flow_meter_owner_set(struct pmd_internals *softnic,
+	struct rte_flow *flow,
+	struct softnic_mtr *mtr)
+{
+	/* Reset current flow meter  */
+	flow_meter_owner_reset(softnic, flow);
+
+	/* Set new flow meter */
+	mtr->flow = flow;
+}
+
+static int
+is_meter_action_enable(struct pmd_internals *softnic,
+	struct softnic_table *table)
+{
+	struct softnic_table_action_profile *profile =
+		softnic_table_action_profile_find(softnic,
+			table->params.action_profile_name);
+	struct softnic_table_action_profile_params *params = &profile->params;
+
+	return (params->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) ? 1 : 0;
+}
+
+static struct rte_flow *
+pmd_flow_create(struct rte_eth_dev *dev,
+	const struct rte_flow_attr *attr,
+	const struct rte_flow_item item[],
+	const struct rte_flow_action action[],
+	struct rte_flow_error *error)
+{
+	struct softnic_table_rule_match rule_match;
+	struct softnic_table_rule_action rule_action;
+	void *rule_data;
+
+	struct pmd_internals *softnic = dev->data->dev_private;
+	struct pipeline *pipeline;
+	struct softnic_table *table;
+	struct rte_flow *flow;
+	struct softnic_mtr *mtr;
+	const char *pipeline_name = NULL;
+	uint32_t table_id = 0;
+	int new_flow, status;
+
+	/* Check input parameters. */
+	if (attr == NULL) {
+		rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_ATTR,
+			NULL,
+			"Null attr");
+		return NULL;
+	}
+
+	if (item == NULL) {
+		rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_ITEM,
+			NULL,
+			"Null item");
+		return NULL;
+	}
+
+	if (action == NULL) {
+		rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_ACTION,
+			NULL,
+			"Null action");
+		return NULL;
+	}
+
+	/* Identify the pipeline table to add this flow to. */
+	status = flow_pipeline_table_get(softnic, attr, &pipeline_name,
+					&table_id, error);
+	if (status)
+		return NULL;
+
+	pipeline = softnic_pipeline_find(softnic, pipeline_name);
+	if (pipeline == NULL) {
+		rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Invalid pipeline name");
+		return NULL;
+	}
+
+	if (table_id >= pipeline->n_tables) {
+		rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Invalid pipeline table ID");
+		return NULL;
+	}
+
+	table = &pipeline->table[table_id];
+
+	/* Rule match. */
+	memset(&rule_match, 0, sizeof(rule_match));
+	status = flow_rule_match_get(softnic,
+		pipeline,
+		table,
+		attr,
+		item,
+		&rule_match,
+		error);
+	if (status)
+		return NULL;
+
+	/* Rule action. */
+	memset(&rule_action, 0, sizeof(rule_action));
+	status = flow_rule_action_get(softnic,
+		pipeline,
+		table,
+		attr,
+		action,
+		&rule_action,
+		error);
+	if (status)
+		return NULL;
+
+	/* Flow find/allocate. */
+	new_flow = 0;
+	flow = softnic_flow_find(table, &rule_match);
+	if (flow == NULL) {
+		new_flow = 1;
+		flow = calloc(1, sizeof(struct rte_flow));
+		if (flow == NULL) {
+			rte_flow_error_set(error,
+				ENOMEM,
+				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				NULL,
+				"Not enough memory for new flow");
+			return NULL;
+		}
+	}
+
+	/* Rule add. */
+	status = softnic_pipeline_table_rule_add(softnic,
+		pipeline_name,
+		table_id,
+		&rule_match,
+		&rule_action,
+		&rule_data);
+	if (status) {
+		if (new_flow)
+			free(flow);
+
+		rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Pipeline table rule add failed");
+		return NULL;
+	}
+
+	/* Flow fill in. */
+	memcpy(&flow->match, &rule_match, sizeof(rule_match));
+	memcpy(&flow->action, &rule_action, sizeof(rule_action));
+	flow->data = rule_data;
+	flow->pipeline = pipeline;
+	flow->table_id = table_id;
+
+	mtr = flow_action_meter_get(softnic, action);
+	if (mtr)
+		flow_meter_owner_set(softnic, flow, mtr);
+
+	/* Flow add to list. */
+	if (new_flow)
+		TAILQ_INSERT_TAIL(&table->flows, flow, node);
+
+	return flow;
+}
+
+static int
+pmd_flow_destroy(struct rte_eth_dev *dev,
+	struct rte_flow *flow,
+	struct rte_flow_error *error)
+{
+	struct pmd_internals *softnic = dev->data->dev_private;
+	struct softnic_table *table;
+	int status;
+
+	/* Check input parameters. */
+	if (flow == NULL)
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_HANDLE,
+			NULL,
+			"Null flow");
+
+	table = &flow->pipeline->table[flow->table_id];
+
+	/* Rule delete. */
+	status = softnic_pipeline_table_rule_delete(softnic,
+		flow->pipeline->name,
+		flow->table_id,
+		&flow->match);
+	if (status)
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Pipeline table rule delete failed");
+
+	/* Update dependencies */
+	if (is_meter_action_enable(softnic, table))
+		flow_meter_owner_reset(softnic, flow);
+
+	/* Flow delete. */
+	TAILQ_REMOVE(&table->flows, flow, node);
+	free(flow);
+
+	return 0;
+}
+
+static int
+pmd_flow_flush(struct rte_eth_dev *dev,
+	struct rte_flow_error *error)
+{
+	struct pmd_internals *softnic = dev->data->dev_private;
+	struct pipeline *pipeline;
+	int fail_to_del_rule = 0;
+	uint32_t i;
+
+	TAILQ_FOREACH(pipeline, &softnic->pipeline_list, node) {
+		/* Remove all the flows added to the tables. */
+		for (i = 0; i < pipeline->n_tables; i++) {
+			struct softnic_table *table = &pipeline->table[i];
+			struct rte_flow *flow;
+			void *temp;
+			int status;
+
+			TAILQ_FOREACH_SAFE(flow, &table->flows, node, temp) {
+				/* Rule delete. */
+				status = softnic_pipeline_table_rule_delete
+						(softnic,
+						pipeline->name,
+						i,
+						&flow->match);
+				if (status)
+					fail_to_del_rule = 1;
+				/* Update dependencies */
+				if (is_meter_action_enable(softnic, table))
+					flow_meter_owner_reset(softnic, flow);
+
+				/* Flow delete. */
+				TAILQ_REMOVE(&table->flows, flow, node);
+				free(flow);
+			}
+		}
+	}
+
+	if (fail_to_del_rule)
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Some of the rules could not be deleted");
+
+	return 0;
+}
+
+static int
+pmd_flow_query(struct rte_eth_dev *dev __rte_unused,
+	struct rte_flow *flow,
+	const struct rte_flow_action *action __rte_unused,
+	void *data,
+	struct rte_flow_error *error)
+{
+	struct rte_table_action_stats_counters stats;
+	struct softnic_table *table;
+	struct rte_flow_query_count *flow_stats = data;
+	int status;
+
+	/* Check input parameters. */
+	if (flow == NULL)
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_HANDLE,
+			NULL,
+			"Null flow");
+
+	if (data == NULL)
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Null data");
+
+	table = &flow->pipeline->table[flow->table_id];
+
+	/* Rule stats read. */
+	status = rte_table_action_stats_read(table->a,
+		flow->data,
+		&stats,
+		flow_stats->reset);
+	if (status)
+		return rte_flow_error_set(error,
+			EINVAL,
+			RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Pipeline table rule stats read failed");
+
+	/* Fill in flow stats. */
+	flow_stats->hits_set =
+		(table->ap->params.stats.n_packets_enabled) ? 1 : 0;
+	flow_stats->bytes_set =
+		(table->ap->params.stats.n_bytes_enabled) ? 1 : 0;
+	flow_stats->hits = stats.n_packets;
+	flow_stats->bytes = stats.n_bytes;
+
+	return 0;
+}
+
+const struct rte_flow_ops pmd_flow_ops = {
+	.validate = pmd_flow_validate,
+	.create = pmd_flow_create,
+	.destroy = pmd_flow_destroy,
+	.flush = pmd_flow_flush,
+	.query = pmd_flow_query,
+	.isolate = NULL,
+};
diff --git a/drivers/net/softnic/rte_eth_softnic_internals.h b/drivers/net/softnic/rte_eth_softnic_internals.h
index a25eb874..e12b8ae4 100644
--- a/drivers/net/softnic/rte_eth_softnic_internals.h
+++ b/drivers/net/softnic/rte_eth_softnic_internals.h
@@ -18,8 +18,11 @@
 #include <rte_table_action.h>
 #include <rte_pipeline.h>
 
+#include <rte_ethdev_core.h>
 #include <rte_ethdev_driver.h>
 #include <rte_tm_driver.h>
+#include <rte_flow_driver.h>
+#include <rte_mtr_driver.h>
 
 #include "rte_eth_softnic.h"
 #include "conn.h"
@@ -44,6 +47,57 @@ struct pmd_params {
 };
 
 /**
+ * Ethdev Flow API
+ */
+struct rte_flow;
+
+TAILQ_HEAD(flow_list, rte_flow);
+
+struct flow_attr_map {
+	char pipeline_name[NAME_SIZE];
+	uint32_t table_id;
+	int valid;
+};
+
+#ifndef SOFTNIC_FLOW_MAX_GROUPS
+#define SOFTNIC_FLOW_MAX_GROUPS                            64
+#endif
+
+struct flow_internals {
+	struct flow_attr_map ingress_map[SOFTNIC_FLOW_MAX_GROUPS];
+	struct flow_attr_map egress_map[SOFTNIC_FLOW_MAX_GROUPS];
+};
+
+/**
+ * Meter
+ */
+
+/* MTR meter profile */
+struct softnic_mtr_meter_profile {
+	TAILQ_ENTRY(softnic_mtr_meter_profile) node;
+	uint32_t meter_profile_id;
+	struct rte_mtr_meter_profile params;
+	uint32_t n_users;
+};
+
+TAILQ_HEAD(softnic_mtr_meter_profile_list, softnic_mtr_meter_profile);
+
+/* MTR meter object */
+struct softnic_mtr {
+	TAILQ_ENTRY(softnic_mtr) node;
+	uint32_t mtr_id;
+	struct rte_mtr_params params;
+	struct rte_flow *flow;
+};
+
+TAILQ_HEAD(softnic_mtr_list, softnic_mtr);
+
+struct mtr_internals {
+	struct softnic_mtr_meter_profile_list meter_profiles;
+	struct softnic_mtr_list mtrs;
+};
+
+/**
  * MEMPOOL
  */
 struct softnic_mempool_params {
@@ -225,6 +279,25 @@ struct softnic_tap {
 TAILQ_HEAD(softnic_tap_list, softnic_tap);
 
 /**
+ * Cryptodev
+ */
+struct softnic_cryptodev_params {
+	const char *dev_name;
+	uint32_t dev_id; /**< Valid only when *dev_name* is NULL. */
+	uint32_t n_queues;
+	uint32_t queue_size;
+};
+
+struct softnic_cryptodev {
+	TAILQ_ENTRY(softnic_cryptodev) node;
+	char name[NAME_SIZE];
+	uint16_t dev_id;
+	uint32_t n_queues;
+};
+
+TAILQ_HEAD(softnic_cryptodev_list, softnic_cryptodev);
+
+/**
  * Input port action
  */
 struct softnic_port_in_action_profile_params {
@@ -255,6 +328,7 @@ struct softnic_table_action_profile_params {
 	struct rte_table_action_nat_config nat;
 	struct rte_table_action_ttl_config ttl;
 	struct rte_table_action_stats_config stats;
+	struct rte_table_action_sym_crypto_config sym_crypto;
 };
 
 struct softnic_table_action_profile {
@@ -266,6 +340,15 @@ struct softnic_table_action_profile {
 
 TAILQ_HEAD(softnic_table_action_profile_list, softnic_table_action_profile);
 
+struct softnic_table_meter_profile {
+	TAILQ_ENTRY(softnic_table_meter_profile) node;
+	uint32_t meter_profile_id;
+	struct rte_table_action_meter_profile profile;
+};
+
+TAILQ_HEAD(softnic_table_meter_profile_list,
+	softnic_table_meter_profile);
+
 /**
  * Pipeline
  */
@@ -280,12 +363,13 @@ enum softnic_port_in_type {
 	PORT_IN_TMGR,
 	PORT_IN_TAP,
 	PORT_IN_SOURCE,
+	PORT_IN_CRYPTODEV,
 };
 
 struct softnic_port_in_params {
 	/* Read */
 	enum softnic_port_in_type type;
-	const char *dev_name;
+	char dev_name[NAME_SIZE];
 	union {
 		struct {
 			uint16_t queue_id;
@@ -301,11 +385,17 @@ struct softnic_port_in_params {
 			const char *file_name;
 			uint32_t n_bytes_per_pkt;
 		} source;
+
+		struct {
+			uint16_t queue_id;
+			void *f_callback;
+			void *arg_callback;
+		} cryptodev;
 	};
 	uint32_t burst_size;
 
 	/* Action */
-	const char *action_profile_name;
+	char action_profile_name[NAME_SIZE];
 };
 
 enum softnic_port_out_type {
@@ -314,11 +404,12 @@ enum softnic_port_out_type {
 	PORT_OUT_TMGR,
 	PORT_OUT_TAP,
 	PORT_OUT_SINK,
+	PORT_OUT_CRYPTODEV,
 };
 
 struct softnic_port_out_params {
 	enum softnic_port_out_type type;
-	const char *dev_name;
+	char dev_name[NAME_SIZE];
 	union {
 		struct {
 			uint16_t queue_id;
@@ -328,6 +419,11 @@ struct softnic_port_out_params {
 			const char *file_name;
 			uint32_t max_n_pkts;
 		} sink;
+
+		struct {
+			uint16_t queue_id;
+			uint32_t op_offset;
+		} cryptodev;
 	};
 	uint32_t burst_size;
 	int retry;
@@ -353,11 +449,15 @@ struct softnic_table_array_params {
 	uint32_t key_offset;
 };
 
+#ifndef TABLE_RULE_MATCH_SIZE_MAX
+#define TABLE_RULE_MATCH_SIZE_MAX                          256
+#endif
+
 struct softnic_table_hash_params {
 	uint32_t n_keys;
 	uint32_t key_offset;
 	uint32_t key_size;
-	uint8_t *key_mask;
+	uint8_t key_mask[TABLE_RULE_MATCH_SIZE_MAX];
 	uint32_t n_buckets;
 	int extendable_bucket;
 };
@@ -379,7 +479,7 @@ struct softnic_table_params {
 	} match;
 
 	/* Action */
-	const char *action_profile_name;
+	char action_profile_name[NAME_SIZE];
 };
 
 struct softnic_port_in {
@@ -388,10 +488,17 @@ struct softnic_port_in {
 	struct rte_port_in_action *a;
 };
 
+struct softnic_port_out {
+	struct softnic_port_out_params params;
+};
+
 struct softnic_table {
 	struct softnic_table_params params;
 	struct softnic_table_action_profile *ap;
 	struct rte_table_action *a;
+	struct flow_list flows;
+	struct rte_table_action_dscp_table dscp_table;
+	struct softnic_table_meter_profile_list meter_profiles;
 };
 
 struct pipeline {
@@ -399,7 +506,9 @@ struct pipeline {
 	char name[NAME_SIZE];
 
 	struct rte_pipeline *p;
+	struct pipeline_params params;
 	struct softnic_port_in port_in[RTE_PIPELINE_PORT_IN_MAX];
+	struct softnic_port_out port_out[RTE_PIPELINE_PORT_OUT_MAX];
 	struct softnic_table table[RTE_PIPELINE_TABLE_MAX];
 	uint32_t n_ports_in;
 	uint32_t n_ports_out;
@@ -489,12 +598,16 @@ struct pmd_internals {
 		struct tm_internals tm; /**< Traffic Management */
 	} soft;
 
+	struct flow_internals flow;
+	struct mtr_internals mtr;
+
 	struct softnic_conn *conn;
 	struct softnic_mempool_list mempool_list;
 	struct softnic_swq_list swq_list;
 	struct softnic_link_list link_list;
 	struct softnic_tmgr_port_list tmgr_port_list;
 	struct softnic_tap_list tap_list;
+	struct softnic_cryptodev_list cryptodev_list;
 	struct softnic_port_in_action_profile_list port_in_action_profile_list;
 	struct softnic_table_action_profile_list table_action_profile_list;
 	struct pipeline_list pipeline_list;
@@ -502,6 +615,58 @@ struct pmd_internals {
 	struct softnic_thread_data thread_data[RTE_MAX_LCORE];
 };
 
+static inline struct rte_eth_dev *
+ETHDEV(struct pmd_internals *softnic)
+{
+	uint16_t port_id;
+	int status;
+
+	if (softnic == NULL)
+		return NULL;
+
+	status = rte_eth_dev_get_port_by_name(softnic->params.name, &port_id);
+	if (status)
+		return NULL;
+
+	return &rte_eth_devices[port_id];
+}
+
+/**
+ * Ethdev Flow API
+ */
+int
+flow_attr_map_set(struct pmd_internals *softnic,
+		uint32_t group_id,
+		int ingress,
+		const char *pipeline_name,
+		uint32_t table_id);
+
+struct flow_attr_map *
+flow_attr_map_get(struct pmd_internals *softnic,
+		uint32_t group_id,
+		int ingress);
+
+extern const struct rte_flow_ops pmd_flow_ops;
+
+/**
+ * Meter
+ */
+int
+softnic_mtr_init(struct pmd_internals *p);
+
+void
+softnic_mtr_free(struct pmd_internals *p);
+
+struct softnic_mtr *
+softnic_mtr_find(struct pmd_internals *p,
+	uint32_t mtr_id);
+
+struct softnic_mtr_meter_profile *
+softnic_mtr_meter_profile_find(struct pmd_internals *p,
+	uint32_t meter_profile_id);
+
+extern const struct rte_mtr_ops pmd_mtr_ops;
+
 /**
  * MEMPOOL
  */
@@ -610,6 +775,24 @@ softnic_tap_create(struct pmd_internals *p,
 	const char *name);
 
 /**
+ * Sym Crypto
+ */
+int
+softnic_cryptodev_init(struct pmd_internals *p);
+
+void
+softnic_cryptodev_free(struct pmd_internals *p);
+
+struct softnic_cryptodev *
+softnic_cryptodev_find(struct pmd_internals *p,
+	const char *name);
+
+struct softnic_cryptodev *
+softnic_cryptodev_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_cryptodev_params *params);
+
+/**
  * Input port action
  */
 int
@@ -683,10 +866,20 @@ softnic_pipeline_port_out_create(struct pmd_internals *p,
 	struct softnic_port_out_params *params);
 
 int
+softnic_pipeline_port_out_find(struct pmd_internals *softnic,
+		const char *pipeline_name,
+		const char *name,
+		uint32_t *port_id);
+
+int
 softnic_pipeline_table_create(struct pmd_internals *p,
 	const char *pipeline_name,
 	struct softnic_table_params *params);
 
+struct softnic_table_meter_profile *
+softnic_pipeline_table_meter_profile_find(struct softnic_table *table,
+	uint32_t meter_profile_id);
+
 struct softnic_table_rule_match_acl {
 	int ip_version;
 
@@ -718,10 +911,6 @@ struct softnic_table_rule_match_array {
 	uint32_t pos;
 };
 
-#ifndef TABLE_RULE_MATCH_SIZE_MAX
-#define TABLE_RULE_MATCH_SIZE_MAX                          256
-#endif
-
 struct softnic_table_rule_match_hash {
 	uint8_t key[TABLE_RULE_MATCH_SIZE_MAX];
 };
@@ -760,6 +949,18 @@ struct softnic_table_rule_action {
 	struct rte_table_action_ttl_params ttl;
 	struct rte_table_action_stats_params stats;
 	struct rte_table_action_time_params time;
+	struct rte_table_action_tag_params tag;
+	struct rte_table_action_decap_params decap;
+	struct rte_table_action_sym_crypto_params sym_crypto;
+};
+
+struct rte_flow {
+	TAILQ_ENTRY(rte_flow) node;
+	struct softnic_table_rule_match match;
+	struct softnic_table_rule_action action;
+	void *data;
+	struct pipeline *pipeline;
+	uint32_t table_id;
 };
 
 int
diff --git a/drivers/net/softnic/rte_eth_softnic_meter.c b/drivers/net/softnic/rte_eth_softnic_meter.c
new file mode 100644
index 00000000..73ecf3b1
--- /dev/null
+++ b/drivers/net/softnic/rte_eth_softnic_meter.c
@@ -0,0 +1,728 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_mtr.h>
+#include <rte_mtr_driver.h>
+
+#include "rte_eth_softnic_internals.h"
+
+int
+softnic_mtr_init(struct pmd_internals *p)
+{
+	/* Initialize meter profiles list */
+	TAILQ_INIT(&p->mtr.meter_profiles);
+
+	/* Initialize MTR objects list */
+	TAILQ_INIT(&p->mtr.mtrs);
+
+	return 0;
+}
+
+void
+softnic_mtr_free(struct pmd_internals *p)
+{
+	/* Remove MTR objects */
+	for ( ; ; ) {
+		struct softnic_mtr *m;
+
+		m = TAILQ_FIRST(&p->mtr.mtrs);
+		if (m == NULL)
+			break;
+
+		TAILQ_REMOVE(&p->mtr.mtrs, m, node);
+		free(m);
+	}
+
+	/* Remove meter profiles */
+	for ( ; ; ) {
+		struct softnic_mtr_meter_profile *mp;
+
+		mp = TAILQ_FIRST(&p->mtr.meter_profiles);
+		if (mp == NULL)
+			break;
+
+		TAILQ_REMOVE(&p->mtr.meter_profiles, mp, node);
+		free(mp);
+	}
+}
+
+struct softnic_mtr_meter_profile *
+softnic_mtr_meter_profile_find(struct pmd_internals *p,
+	uint32_t meter_profile_id)
+{
+	struct softnic_mtr_meter_profile_list *mpl = &p->mtr.meter_profiles;
+	struct softnic_mtr_meter_profile *mp;
+
+	TAILQ_FOREACH(mp, mpl, node)
+		if (meter_profile_id == mp->meter_profile_id)
+			return mp;
+
+	return NULL;
+}
+
+static int
+meter_profile_check(struct rte_eth_dev *dev,
+	uint32_t meter_profile_id,
+	struct rte_mtr_meter_profile *profile,
+	struct rte_mtr_error *error)
+{
+	struct pmd_internals *p = dev->data->dev_private;
+	struct softnic_mtr_meter_profile *mp;
+
+	/* Meter profile ID must be valid. */
+	if (meter_profile_id == UINT32_MAX)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+			NULL,
+			"Meter profile id not valid");
+
+	/* Meter profile must not exist. */
+	mp = softnic_mtr_meter_profile_find(p, meter_profile_id);
+	if (mp)
+		return -rte_mtr_error_set(error,
+			EEXIST,
+			RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+			NULL,
+			"Meter prfile already exists");
+
+	/* Profile must not be NULL. */
+	if (profile == NULL)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_METER_PROFILE,
+			NULL,
+			"profile null");
+
+	/* Traffic metering algorithm : TRTCM_RFC2698 */
+	if (profile->alg != RTE_MTR_TRTCM_RFC2698)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_METER_PROFILE,
+			NULL,
+			"Metering alg not supported");
+
+	return 0;
+}
+
+/* MTR meter profile add */
+static int
+pmd_mtr_meter_profile_add(struct rte_eth_dev *dev,
+	uint32_t meter_profile_id,
+	struct rte_mtr_meter_profile *profile,
+	struct rte_mtr_error *error)
+{
+	struct pmd_internals *p = dev->data->dev_private;
+	struct softnic_mtr_meter_profile_list *mpl = &p->mtr.meter_profiles;
+	struct softnic_mtr_meter_profile *mp;
+	int status;
+
+	/* Check input params */
+	status = meter_profile_check(dev, meter_profile_id, profile, error);
+	if (status)
+		return status;
+
+	/* Memory allocation */
+	mp = calloc(1, sizeof(struct softnic_mtr_meter_profile));
+	if (mp == NULL)
+		return -rte_mtr_error_set(error,
+			ENOMEM,
+			RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Memory alloc failed");
+
+	/* Fill in */
+	mp->meter_profile_id = meter_profile_id;
+	memcpy(&mp->params, profile, sizeof(mp->params));
+
+	/* Add to list */
+	TAILQ_INSERT_TAIL(mpl, mp, node);
+
+	return 0;
+}
+
+/* MTR meter profile delete */
+static int
+pmd_mtr_meter_profile_delete(struct rte_eth_dev *dev,
+	uint32_t meter_profile_id,
+	struct rte_mtr_error *error)
+{
+	struct pmd_internals *p = dev->data->dev_private;
+	struct softnic_mtr_meter_profile *mp;
+
+	/* Meter profile must exist */
+	mp = softnic_mtr_meter_profile_find(p, meter_profile_id);
+	if (mp == NULL)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+			NULL,
+			"Meter profile id invalid");
+
+	/* Check unused */
+	if (mp->n_users)
+		return -rte_mtr_error_set(error,
+			EBUSY,
+			RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+			NULL,
+			"Meter profile in use");
+
+	/* Remove from list */
+	TAILQ_REMOVE(&p->mtr.meter_profiles, mp, node);
+	free(mp);
+
+	return 0;
+}
+
+struct softnic_mtr *
+softnic_mtr_find(struct pmd_internals *p, uint32_t mtr_id)
+{
+	struct softnic_mtr_list *ml = &p->mtr.mtrs;
+	struct softnic_mtr *m;
+
+	TAILQ_FOREACH(m, ml, node)
+		if (m->mtr_id == mtr_id)
+			return m;
+
+	return NULL;
+}
+
+
+static int
+mtr_check(struct pmd_internals *p,
+	uint32_t mtr_id,
+	struct rte_mtr_params *params,
+	int shared,
+	struct rte_mtr_error *error)
+{
+	/* MTR id valid  */
+	if (softnic_mtr_find(p, mtr_id))
+		return -rte_mtr_error_set(error,
+			EEXIST,
+			RTE_MTR_ERROR_TYPE_MTR_ID,
+			NULL,
+			"MTR object already exists");
+
+	/* MTR params must not be NULL */
+	if (params == NULL)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_MTR_PARAMS,
+			NULL,
+			"MTR object params null");
+
+	/* Previous meter color not supported */
+	if (params->use_prev_mtr_color)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_MTR_PARAMS,
+			NULL,
+			"Previous meter color not supported");
+
+	/* Shared MTR object not supported */
+	if (shared)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_SHARED,
+			NULL,
+			"Shared MTR object not supported");
+
+	return 0;
+}
+
+/* MTR object create */
+static int
+pmd_mtr_create(struct rte_eth_dev *dev,
+	uint32_t mtr_id,
+	struct rte_mtr_params *params,
+	int shared,
+	struct rte_mtr_error *error)
+{
+	struct pmd_internals *p = dev->data->dev_private;
+	struct softnic_mtr_list *ml = &p->mtr.mtrs;
+	struct softnic_mtr_meter_profile *mp;
+	struct softnic_mtr *m;
+	int status;
+
+	/* Check parameters */
+	status = mtr_check(p, mtr_id, params, shared, error);
+	if (status)
+		return status;
+
+	/* Meter profile must exist */
+	mp = softnic_mtr_meter_profile_find(p, params->meter_profile_id);
+	if (mp == NULL)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+			NULL,
+			"Meter profile id not valid");
+
+	/* Memory allocation */
+	m = calloc(1, sizeof(struct softnic_mtr));
+	if (m == NULL)
+		return -rte_mtr_error_set(error,
+			ENOMEM,
+			RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Memory alloc failed");
+
+	/* Fill in */
+	m->mtr_id = mtr_id;
+	memcpy(&m->params, params, sizeof(m->params));
+
+	/* Add to list */
+	TAILQ_INSERT_TAIL(ml, m, node);
+
+	/* Update dependencies */
+	mp->n_users++;
+
+	return 0;
+}
+
+/* MTR object destroy */
+static int
+pmd_mtr_destroy(struct rte_eth_dev *dev,
+	uint32_t mtr_id,
+	struct rte_mtr_error *error)
+{
+	struct pmd_internals *p = dev->data->dev_private;
+	struct softnic_mtr_list *ml = &p->mtr.mtrs;
+	struct softnic_mtr_meter_profile *mp;
+	struct softnic_mtr *m;
+
+	/* MTR object must exist */
+	m = softnic_mtr_find(p, mtr_id);
+	if (m == NULL)
+		return -rte_mtr_error_set(error,
+			EEXIST,
+			RTE_MTR_ERROR_TYPE_MTR_ID,
+			NULL,
+			"MTR object id not valid");
+
+	/* MTR object must not have any owner */
+	if (m->flow != NULL)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"MTR object is being used");
+
+	/* Get meter profile */
+	mp = softnic_mtr_meter_profile_find(p, m->params.meter_profile_id);
+	if (mp == NULL)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+			NULL,
+			"MTR object meter profile invalid");
+
+	/* Update dependencies */
+	mp->n_users--;
+
+	/* Remove from list */
+	TAILQ_REMOVE(ml, m, node);
+	free(m);
+
+	return 0;
+}
+
+/* MTR object meter profile update */
+static int
+pmd_mtr_meter_profile_update(struct rte_eth_dev *dev,
+	uint32_t mtr_id,
+	uint32_t meter_profile_id,
+	struct rte_mtr_error *error)
+{
+	struct pmd_internals *p = dev->data->dev_private;
+	struct softnic_mtr_meter_profile *mp_new, *mp_old;
+	struct softnic_mtr *m;
+	int status;
+
+	/* MTR object id must be valid */
+	m = softnic_mtr_find(p, mtr_id);
+	if (m == NULL)
+		return -rte_mtr_error_set(error,
+			EEXIST,
+			RTE_MTR_ERROR_TYPE_MTR_ID,
+			NULL,
+			"MTR object id not valid");
+
+	/* Meter profile id must be valid */
+	mp_new = softnic_mtr_meter_profile_find(p, meter_profile_id);
+	if (mp_new == NULL)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
+			NULL,
+			"Meter profile not valid");
+
+	/* MTR object already set to meter profile id */
+	if (m->params.meter_profile_id == meter_profile_id)
+		return 0;
+
+	/*  MTR object owner table update */
+	if (m->flow) {
+		uint32_t table_id = m->flow->table_id;
+		struct softnic_table *table = &m->flow->pipeline->table[table_id];
+		struct softnic_table_rule_action action;
+
+		if (!softnic_pipeline_table_meter_profile_find(table,
+			meter_profile_id)) {
+			struct rte_table_action_meter_profile profile;
+
+			memset(&profile, 0, sizeof(profile));
+
+			profile.alg = RTE_TABLE_ACTION_METER_TRTCM;
+			profile.trtcm.cir = mp_new->params.trtcm_rfc2698.cir;
+			profile.trtcm.pir = mp_new->params.trtcm_rfc2698.pir;
+			profile.trtcm.cbs = mp_new->params.trtcm_rfc2698.cbs;
+			profile.trtcm.pbs = mp_new->params.trtcm_rfc2698.pbs;
+
+			/* Add meter profile to pipeline table */
+			status = softnic_pipeline_table_mtr_profile_add(p,
+					m->flow->pipeline->name,
+					table_id,
+					meter_profile_id,
+					&profile);
+			if (status)
+				return -rte_mtr_error_set(error,
+					EINVAL,
+					RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"Table meter profile add failed");
+		}
+
+		/* Set meter action */
+		memcpy(&action, &m->flow->action, sizeof(action));
+
+		action.mtr.mtr[0].meter_profile_id = meter_profile_id;
+
+		/* Re-add rule */
+		status = softnic_pipeline_table_rule_add(p,
+			m->flow->pipeline->name,
+			table_id,
+			&m->flow->match,
+			&action,
+			&m->flow->data);
+		if (status)
+			return -rte_mtr_error_set(error,
+				EINVAL,
+				RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+				NULL,
+				"Pipeline table rule add failed");
+
+		/* Flow: update meter action */
+		memcpy(&m->flow->action, &action, sizeof(m->flow->action));
+	}
+
+	mp_old = softnic_mtr_meter_profile_find(p, m->params.meter_profile_id);
+
+	/* Meter: Set meter profile */
+	m->params.meter_profile_id = meter_profile_id;
+
+	/* Update dependencies*/
+	mp_old->n_users--;
+	mp_new->n_users++;
+
+	return 0;
+}
+
+/* MTR object meter DSCP table update */
+static int
+pmd_mtr_meter_dscp_table_update(struct rte_eth_dev *dev,
+	uint32_t mtr_id,
+	enum rte_mtr_color *dscp_table,
+	struct rte_mtr_error *error)
+{
+	struct pmd_internals *p = dev->data->dev_private;
+	struct rte_table_action_dscp_table dt;
+	struct pipeline *pipeline;
+	struct softnic_table *table;
+	struct softnic_mtr *m;
+	uint32_t table_id, i;
+	int status;
+
+	/* MTR object id must be valid */
+	m = softnic_mtr_find(p, mtr_id);
+	if (m == NULL)
+		return -rte_mtr_error_set(error,
+			EEXIST,
+			RTE_MTR_ERROR_TYPE_MTR_ID,
+			NULL,
+			"MTR object id not valid");
+
+	/* MTR object owner valid? */
+	if (m->flow == NULL)
+		return 0;
+
+	pipeline = m->flow->pipeline;
+	table_id = m->flow->table_id;
+	table = &pipeline->table[table_id];
+
+	memcpy(&dt, &table->dscp_table, sizeof(dt));
+	for (i = 0; i < RTE_DIM(dt.entry); i++)
+		dt.entry[i].color = (enum rte_meter_color)dscp_table[i];
+
+	/* Update table */
+	status = softnic_pipeline_table_dscp_table_update(p,
+			pipeline->name,
+			table_id,
+			UINT64_MAX,
+			&dt);
+	if (status)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Table action dscp table update failed");
+
+	return 0;
+}
+
+/* MTR object policer action update */
+static int
+pmd_mtr_policer_actions_update(struct rte_eth_dev *dev,
+	uint32_t mtr_id,
+	uint32_t action_mask,
+	enum rte_mtr_policer_action *actions,
+	struct rte_mtr_error *error)
+{
+	struct pmd_internals *p = dev->data->dev_private;
+	struct softnic_mtr *m;
+	uint32_t i;
+	int status;
+
+	/* MTR object id must be valid */
+	m = softnic_mtr_find(p, mtr_id);
+	if (m == NULL)
+		return -rte_mtr_error_set(error,
+			EEXIST,
+			RTE_MTR_ERROR_TYPE_MTR_ID,
+			NULL,
+			"MTR object id not valid");
+
+	/* Valid policer actions */
+	if (actions == NULL)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Invalid actions");
+
+	for (i = 0; i < RTE_MTR_COLORS; i++) {
+		if (action_mask & (1 << i)) {
+			if (actions[i] != MTR_POLICER_ACTION_COLOR_GREEN  &&
+				actions[i] != MTR_POLICER_ACTION_COLOR_YELLOW &&
+				actions[i] != MTR_POLICER_ACTION_COLOR_RED &&
+				actions[i] != MTR_POLICER_ACTION_DROP) {
+				return -rte_mtr_error_set(error,
+					EINVAL,
+					RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					" Invalid action value");
+			}
+		}
+	}
+
+	/* MTR object owner valid? */
+	if (m->flow) {
+		struct pipeline *pipeline = m->flow->pipeline;
+		struct softnic_table *table = &pipeline->table[m->flow->table_id];
+		struct softnic_table_rule_action action;
+
+		memcpy(&action, &m->flow->action, sizeof(action));
+
+		/* Set action */
+		for (i = 0; i < RTE_MTR_COLORS; i++)
+			if (action_mask & (1 << i))
+				action.mtr.mtr[0].policer[i] =
+					(enum rte_table_action_policer)actions[i];
+
+		/* Re-add the rule */
+		status = softnic_pipeline_table_rule_add(p,
+			pipeline->name,
+			m->flow->table_id,
+			&m->flow->match,
+			&action,
+			&m->flow->data);
+		if (status)
+			return -rte_mtr_error_set(error,
+				EINVAL,
+				RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+				NULL,
+				"Pipeline table rule re-add failed");
+
+		/* Flow: Update meter action */
+		memcpy(&m->flow->action, &action, sizeof(m->flow->action));
+
+		/* Reset the meter stats */
+		rte_table_action_meter_read(table->a, m->flow->data,
+			1, NULL, 1);
+	}
+
+	/* Meter: Update policer actions */
+	for (i = 0; i < RTE_MTR_COLORS; i++)
+		if (action_mask & (1 << i))
+			m->params.action[i] = actions[i];
+
+	return 0;
+}
+
+#define MTR_STATS_PKTS_DEFAULT (RTE_MTR_STATS_N_PKTS_GREEN | \
+				RTE_MTR_STATS_N_PKTS_YELLOW | \
+				RTE_MTR_STATS_N_PKTS_RED | \
+				RTE_MTR_STATS_N_PKTS_DROPPED)
+
+#define MTR_STATS_BYTES_DEFAULT (RTE_MTR_STATS_N_BYTES_GREEN | \
+				RTE_MTR_STATS_N_BYTES_YELLOW | \
+				RTE_MTR_STATS_N_BYTES_RED | \
+				RTE_MTR_STATS_N_BYTES_DROPPED)
+
+/* MTR object stats read */
+static void
+mtr_stats_convert(struct softnic_mtr *m,
+	struct rte_table_action_mtr_counters_tc *in,
+	struct rte_mtr_stats *out,
+	uint64_t *out_mask)
+{
+	memset(&out, 0, sizeof(out));
+	*out_mask = 0;
+
+	if (in->n_packets_valid) {
+		uint32_t i;
+
+		for (i = 0; i < RTE_MTR_COLORS; i++) {
+			if (m->params.action[i] == MTR_POLICER_ACTION_COLOR_GREEN)
+				out->n_pkts[RTE_MTR_GREEN] += in->n_packets[i];
+
+			if (m->params.action[i] == MTR_POLICER_ACTION_COLOR_YELLOW)
+				out->n_pkts[RTE_MTR_YELLOW] += in->n_packets[i];
+
+			if (m->params.action[i] == MTR_POLICER_ACTION_COLOR_RED)
+				out->n_pkts[RTE_MTR_RED] += in->n_packets[i];
+
+			if (m->params.action[i] == MTR_POLICER_ACTION_DROP)
+				out->n_pkts_dropped += in->n_packets[i];
+		}
+
+		*out_mask |= MTR_STATS_PKTS_DEFAULT;
+	}
+
+	if (in->n_bytes_valid) {
+		uint32_t i;
+
+		for (i = 0; i < RTE_MTR_COLORS; i++) {
+			if (m->params.action[i] == MTR_POLICER_ACTION_COLOR_GREEN)
+				out->n_bytes[RTE_MTR_GREEN] += in->n_bytes[i];
+
+			if (m->params.action[i] == MTR_POLICER_ACTION_COLOR_YELLOW)
+				out->n_bytes[RTE_MTR_YELLOW] += in->n_bytes[i];
+
+			if (m->params.action[i] == MTR_POLICER_ACTION_COLOR_RED)
+				out->n_bytes[RTE_MTR_RED] += in->n_bytes[i];
+
+			if (m->params.action[i] == MTR_POLICER_ACTION_DROP)
+				out->n_bytes_dropped += in->n_bytes[i];
+		}
+
+		*out_mask |= MTR_STATS_BYTES_DEFAULT;
+	}
+}
+
+/* MTR object stats read */
+static int
+pmd_mtr_stats_read(struct rte_eth_dev *dev,
+	uint32_t mtr_id,
+	struct rte_mtr_stats *stats,
+	uint64_t *stats_mask,
+	int clear,
+	struct rte_mtr_error *error)
+{
+	struct pmd_internals *p = dev->data->dev_private;
+	struct rte_table_action_mtr_counters counters;
+	struct pipeline *pipeline;
+	struct softnic_table *table;
+	struct softnic_mtr *m;
+	int status;
+
+	/* MTR object id must be valid */
+	m = softnic_mtr_find(p, mtr_id);
+	if (m == NULL)
+		return -rte_mtr_error_set(error,
+			EEXIST,
+			RTE_MTR_ERROR_TYPE_MTR_ID,
+			NULL,
+			"MTR object id not valid");
+
+	/* MTR meter object owner valid? */
+	if (m->flow == NULL) {
+		if (stats != NULL)
+			memset(stats, 0, sizeof(*stats));
+
+		if (stats_mask)
+			*stats_mask = MTR_STATS_PKTS_DEFAULT |
+				MTR_STATS_BYTES_DEFAULT;
+
+		return 0;
+	}
+
+	pipeline = m->flow->pipeline;
+	table = &pipeline->table[m->flow->table_id];
+
+	/* Meter stats read. */
+	status = rte_table_action_meter_read(table->a,
+		m->flow->data,
+		1,
+		&counters,
+		clear);
+	if (status)
+		return -rte_mtr_error_set(error,
+			EINVAL,
+			RTE_MTR_ERROR_TYPE_UNSPECIFIED,
+			NULL,
+			"Meter stats read failed");
+
+	/* Stats format conversion. */
+	if (stats || stats_mask) {
+		struct rte_mtr_stats s;
+		uint64_t s_mask = 0;
+
+		mtr_stats_convert(m,
+			&counters.stats[0],
+			&s,
+			&s_mask);
+
+		if (stats)
+			memcpy(stats, &s, sizeof(*stats));
+
+		if (stats_mask)
+			*stats_mask = s_mask;
+	}
+
+	return 0;
+}
+
+const struct rte_mtr_ops pmd_mtr_ops = {
+	.capabilities_get = NULL,
+
+	.meter_profile_add = pmd_mtr_meter_profile_add,
+	.meter_profile_delete = pmd_mtr_meter_profile_delete,
+
+	.create = pmd_mtr_create,
+	.destroy = pmd_mtr_destroy,
+	.meter_enable = NULL,
+	.meter_disable = NULL,
+
+	.meter_profile_update = pmd_mtr_meter_profile_update,
+	.meter_dscp_table_update = pmd_mtr_meter_dscp_table_update,
+	.policer_actions_update = pmd_mtr_policer_actions_update,
+	.stats_update = NULL,
+
+	.stats_read = pmd_mtr_stats_read,
+};
diff --git a/drivers/net/softnic/rte_eth_softnic_pipeline.c b/drivers/net/softnic/rte_eth_softnic_pipeline.c
index 45136a4a..5e180f8f 100644
--- a/drivers/net/softnic/rte_eth_softnic_pipeline.c
+++ b/drivers/net/softnic/rte_eth_softnic_pipeline.c
@@ -15,18 +15,18 @@
 #include <rte_port_source_sink.h>
 #include <rte_port_fd.h>
 #include <rte_port_sched.h>
+#include <rte_port_sym_crypto.h>
 
 #include <rte_table_acl.h>
 #include <rte_table_array.h>
 #include <rte_table_hash.h>
+#include <rte_table_hash_func.h>
 #include <rte_table_lpm.h>
 #include <rte_table_lpm_ipv6.h>
 #include <rte_table_stub.h>
 
 #include "rte_eth_softnic_internals.h"
 
-#include "hash_func.h"
-
 #ifndef PIPELINE_MSGQ_SIZE
 #define PIPELINE_MSGQ_SIZE                                 64
 #endif
@@ -43,17 +43,52 @@ softnic_pipeline_init(struct pmd_internals *p)
 	return 0;
 }
 
+static void
+softnic_pipeline_table_free(struct softnic_table *table)
+{
+	for ( ; ; ) {
+		struct rte_flow *flow;
+
+		flow = TAILQ_FIRST(&table->flows);
+		if (flow == NULL)
+			break;
+
+		TAILQ_REMOVE(&table->flows, flow, node);
+		free(flow);
+	}
+
+	for ( ; ; ) {
+		struct softnic_table_meter_profile *mp;
+
+		mp = TAILQ_FIRST(&table->meter_profiles);
+		if (mp == NULL)
+			break;
+
+		TAILQ_REMOVE(&table->meter_profiles, mp, node);
+		free(mp);
+	}
+}
+
 void
 softnic_pipeline_free(struct pmd_internals *p)
 {
 	for ( ; ; ) {
 		struct pipeline *pipeline;
+		uint32_t table_id;
 
 		pipeline = TAILQ_FIRST(&p->pipeline_list);
 		if (pipeline == NULL)
 			break;
 
 		TAILQ_REMOVE(&p->pipeline_list, pipeline, node);
+
+		for (table_id = 0; table_id < pipeline->n_tables; table_id++) {
+			struct softnic_table *table =
+				&pipeline->table[table_id];
+
+			softnic_pipeline_table_free(table);
+		}
+
 		rte_ring_free(pipeline->msgq_req);
 		rte_ring_free(pipeline->msgq_rsp);
 		rte_pipeline_free(pipeline->p);
@@ -160,6 +195,7 @@ softnic_pipeline_create(struct pmd_internals *softnic,
 	/* Node fill in */
 	strlcpy(pipeline->name, name, sizeof(pipeline->name));
 	pipeline->p = p;
+	memcpy(&pipeline->params, params, sizeof(*params));
 	pipeline->n_ports_in = 0;
 	pipeline->n_ports_out = 0;
 	pipeline->n_tables = 0;
@@ -189,6 +225,7 @@ softnic_pipeline_port_in_create(struct pmd_internals *softnic,
 		struct rte_port_sched_reader_params sched;
 		struct rte_port_fd_reader_params fd;
 		struct rte_port_source_params source;
+		struct rte_port_sym_crypto_reader_params cryptodev;
 	} pp;
 
 	struct pipeline *pipeline;
@@ -213,7 +250,7 @@ softnic_pipeline_port_in_create(struct pmd_internals *softnic,
 		return -1;
 
 	ap = NULL;
-	if (params->action_profile_name) {
+	if (strlen(params->action_profile_name)) {
 		ap = softnic_port_in_action_profile_find(softnic,
 			params->action_profile_name);
 		if (ap == NULL)
@@ -306,6 +343,23 @@ softnic_pipeline_port_in_create(struct pmd_internals *softnic,
 		break;
 	}
 
+	case PORT_IN_CRYPTODEV:
+	{
+		struct softnic_cryptodev *cryptodev;
+
+		cryptodev = softnic_cryptodev_find(softnic, params->dev_name);
+		if (cryptodev == NULL)
+			return -1;
+
+		pp.cryptodev.cryptodev_id = cryptodev->dev_id;
+		pp.cryptodev.queue_id = params->cryptodev.queue_id;
+		pp.cryptodev.f_callback = params->cryptodev.f_callback;
+		pp.cryptodev.arg_callback = params->cryptodev.arg_callback;
+		p.ops = &rte_port_sym_crypto_reader_ops;
+		p.arg_create = &pp.cryptodev;
+		break;
+	}
+
 	default:
 		return -1;
 	}
@@ -392,15 +446,18 @@ softnic_pipeline_port_out_create(struct pmd_internals *softnic,
 		struct rte_port_sched_writer_params sched;
 		struct rte_port_fd_writer_params fd;
 		struct rte_port_sink_params sink;
+		struct rte_port_sym_crypto_writer_params cryptodev;
 	} pp;
 
 	union {
 		struct rte_port_ethdev_writer_nodrop_params ethdev;
 		struct rte_port_ring_writer_nodrop_params ring;
 		struct rte_port_fd_writer_nodrop_params fd;
+		struct rte_port_sym_crypto_writer_nodrop_params cryptodev;
 	} pp_nodrop;
 
 	struct pipeline *pipeline;
+	struct softnic_port_out *port_out;
 	uint32_t port_id;
 	int status;
 
@@ -526,6 +583,40 @@ softnic_pipeline_port_out_create(struct pmd_internals *softnic,
 		break;
 	}
 
+	case PORT_OUT_CRYPTODEV:
+	{
+		struct softnic_cryptodev *cryptodev;
+
+		cryptodev = softnic_cryptodev_find(softnic, params->dev_name);
+		if (cryptodev == NULL)
+			return -1;
+
+		if (params->cryptodev.queue_id >= cryptodev->n_queues)
+			return -1;
+
+		pp.cryptodev.cryptodev_id = cryptodev->dev_id;
+		pp.cryptodev.queue_id = params->cryptodev.queue_id;
+		pp.cryptodev.tx_burst_sz = params->burst_size;
+		pp.cryptodev.crypto_op_offset = params->cryptodev.op_offset;
+
+		pp_nodrop.cryptodev.cryptodev_id = cryptodev->dev_id;
+		pp_nodrop.cryptodev.queue_id = params->cryptodev.queue_id;
+		pp_nodrop.cryptodev.tx_burst_sz = params->burst_size;
+		pp_nodrop.cryptodev.n_retries = params->retry;
+		pp_nodrop.cryptodev.crypto_op_offset =
+				params->cryptodev.op_offset;
+
+		if (params->retry == 0) {
+			p.ops = &rte_port_sym_crypto_writer_ops;
+			p.arg_create = &pp.cryptodev;
+		} else {
+			p.ops = &rte_port_sym_crypto_writer_nodrop_ops;
+			p.arg_create = &pp_nodrop.cryptodev;
+		}
+
+		break;
+	}
+
 	default:
 		return -1;
 	}
@@ -542,6 +633,8 @@ softnic_pipeline_port_out_create(struct pmd_internals *softnic,
 		return -1;
 
 	/* Pipeline */
+	port_out = &pipeline->port_out[pipeline->n_ports_out];
+	memcpy(&port_out->params, params, sizeof(*params));
 	pipeline->n_ports_out++;
 
 	return 0;
@@ -730,7 +823,7 @@ softnic_pipeline_table_create(struct pmd_internals *softnic,
 		return -1;
 
 	ap = NULL;
-	if (params->action_profile_name) {
+	if (strlen(params->action_profile_name)) {
 		ap = softnic_table_action_profile_find(softnic,
 			params->action_profile_name);
 		if (ap == NULL)
@@ -797,28 +890,28 @@ softnic_pipeline_table_create(struct pmd_internals *softnic,
 
 		switch (params->match.hash.key_size) {
 		case  8:
-			f_hash = hash_default_key8;
+			f_hash = rte_table_hash_crc_key8;
 			break;
 		case 16:
-			f_hash = hash_default_key16;
+			f_hash = rte_table_hash_crc_key16;
 			break;
 		case 24:
-			f_hash = hash_default_key24;
+			f_hash = rte_table_hash_crc_key24;
 			break;
 		case 32:
-			f_hash = hash_default_key32;
+			f_hash = rte_table_hash_crc_key32;
 			break;
 		case 40:
-			f_hash = hash_default_key40;
+			f_hash = rte_table_hash_crc_key40;
 			break;
 		case 48:
-			f_hash = hash_default_key48;
+			f_hash = rte_table_hash_crc_key48;
 			break;
 		case 56:
-			f_hash = hash_default_key56;
+			f_hash = rte_table_hash_crc_key56;
 			break;
 		case 64:
-			f_hash = hash_default_key64;
+			f_hash = rte_table_hash_crc_key64;
 			break;
 		default:
 			return -1;
@@ -960,7 +1053,51 @@ softnic_pipeline_table_create(struct pmd_internals *softnic,
 	memcpy(&table->params, params, sizeof(*params));
 	table->ap = ap;
 	table->a = action;
+	TAILQ_INIT(&table->flows);
+	TAILQ_INIT(&table->meter_profiles);
+	memset(&table->dscp_table, 0, sizeof(table->dscp_table));
 	pipeline->n_tables++;
 
 	return 0;
 }
+
+int
+softnic_pipeline_port_out_find(struct pmd_internals *softnic,
+		const char *pipeline_name,
+		const char *name,
+		uint32_t *port_id)
+{
+	struct pipeline *pipeline;
+	uint32_t i;
+
+	if (softnic == NULL ||
+			pipeline_name == NULL ||
+			name == NULL ||
+			port_id == NULL)
+		return -1;
+
+	pipeline = softnic_pipeline_find(softnic, pipeline_name);
+	if (pipeline == NULL)
+		return -1;
+
+	for (i = 0; i < pipeline->n_ports_out; i++)
+		if (strcmp(pipeline->port_out[i].params.dev_name, name) == 0) {
+			*port_id = i;
+			return 0;
+		}
+
+	return -1;
+}
+
+struct softnic_table_meter_profile *
+softnic_pipeline_table_meter_profile_find(struct softnic_table *table,
+	uint32_t meter_profile_id)
+{
+	struct softnic_table_meter_profile *mp;
+
+	TAILQ_FOREACH(mp, &table->meter_profiles, node)
+		if (mp->meter_profile_id == meter_profile_id)
+			return mp;
+
+	return NULL;
+}
diff --git a/drivers/net/softnic/rte_eth_softnic_thread.c b/drivers/net/softnic/rte_eth_softnic_thread.c
index 8a150903..4572adfa 100644
--- a/drivers/net/softnic/rte_eth_softnic_thread.c
+++ b/drivers/net/softnic/rte_eth_softnic_thread.c
@@ -1680,6 +1680,8 @@ softnic_pipeline_table_mtr_profile_add(struct pmd_internals *softnic,
 	struct pipeline *p;
 	struct pipeline_msg_req *req;
 	struct pipeline_msg_rsp *rsp;
+	struct softnic_table *table;
+	struct softnic_table_meter_profile *mp;
 	int status;
 
 	/* Check input params */
@@ -1692,20 +1694,40 @@ softnic_pipeline_table_mtr_profile_add(struct pmd_internals *softnic,
 		table_id >= p->n_tables)
 		return -1;
 
-	if (!pipeline_is_running(p)) {
-		struct rte_table_action *a = p->table[table_id].a;
+	table = &p->table[table_id];
+	mp = softnic_pipeline_table_meter_profile_find(table, meter_profile_id);
+	if (mp)
+		return -1;
 
-		status = rte_table_action_meter_profile_add(a,
+	/* Resource Allocation */
+	mp = calloc(1, sizeof(struct softnic_table_meter_profile));
+	if (mp == NULL)
+		return -1;
+
+	mp->meter_profile_id = meter_profile_id;
+	memcpy(&mp->profile, profile, sizeof(mp->profile));
+
+	if (!pipeline_is_running(p)) {
+		status = rte_table_action_meter_profile_add(table->a,
 			meter_profile_id,
 			profile);
+		if (status) {
+			free(mp);
+			return status;
+		}
+
+		/* Add profile to the table. */
+		TAILQ_INSERT_TAIL(&table->meter_profiles, mp, node);
 
 		return status;
 	}
 
 	/* Allocate request */
 	req = pipeline_msg_alloc();
-	if (req == NULL)
+	if (req == NULL) {
+		free(mp);
 		return -1;
+	}
 
 	/* Write request */
 	req->type = PIPELINE_REQ_TABLE_MTR_PROFILE_ADD;
@@ -1715,11 +1737,17 @@ softnic_pipeline_table_mtr_profile_add(struct pmd_internals *softnic,
 
 	/* Send request and wait for response */
 	rsp = pipeline_msg_send_recv(p, req);
-	if (rsp == NULL)
+	if (rsp == NULL) {
+		free(mp);
 		return -1;
+	}
 
 	/* Read response */
 	status = rsp->status;
+	if (status == 0)
+		TAILQ_INSERT_TAIL(&table->meter_profiles, mp, node);
+	else
+		free(mp);
 
 	/* Free response */
 	pipeline_msg_free(rsp);
@@ -1874,6 +1902,11 @@ softnic_pipeline_table_dscp_table_update(struct pmd_internals *softnic,
 				dscp_mask,
 				dscp_table);
 
+		/* Update table dscp table */
+		if (!status)
+			memcpy(&p->table[table_id].dscp_table, dscp_table,
+				sizeof(p->table[table_id].dscp_table));
+
 		return status;
 	}
 
@@ -1897,6 +1930,11 @@ softnic_pipeline_table_dscp_table_update(struct pmd_internals *softnic,
 	/* Read response */
 	status = rsp->status;
 
+	/* Update table dscp table */
+	if (!status)
+		memcpy(&p->table[table_id].dscp_table, dscp_table,
+			sizeof(p->table[table_id].dscp_table));
+
 	/* Free response */
 	pipeline_msg_free(rsp);
 
@@ -2202,29 +2240,37 @@ match_convert(struct softnic_table_rule_match *mh,
 				ml->acl_add.field_value[0].mask_range.u8 =
 					mh->match.acl.proto_mask;
 
-				ml->acl_add.field_value[1].value.u32 = sa32[0];
+				ml->acl_add.field_value[1].value.u32 =
+					rte_be_to_cpu_32(sa32[0]);
 				ml->acl_add.field_value[1].mask_range.u32 =
 					sa32_depth[0];
-				ml->acl_add.field_value[2].value.u32 = sa32[1];
+				ml->acl_add.field_value[2].value.u32 =
+					rte_be_to_cpu_32(sa32[1]);
 				ml->acl_add.field_value[2].mask_range.u32 =
 					sa32_depth[1];
-				ml->acl_add.field_value[3].value.u32 = sa32[2];
+				ml->acl_add.field_value[3].value.u32 =
+					rte_be_to_cpu_32(sa32[2]);
 				ml->acl_add.field_value[3].mask_range.u32 =
 					sa32_depth[2];
-				ml->acl_add.field_value[4].value.u32 = sa32[3];
+				ml->acl_add.field_value[4].value.u32 =
+					rte_be_to_cpu_32(sa32[3]);
 				ml->acl_add.field_value[4].mask_range.u32 =
 					sa32_depth[3];
 
-				ml->acl_add.field_value[5].value.u32 = da32[0];
+				ml->acl_add.field_value[5].value.u32 =
+					rte_be_to_cpu_32(da32[0]);
 				ml->acl_add.field_value[5].mask_range.u32 =
 					da32_depth[0];
-				ml->acl_add.field_value[6].value.u32 = da32[1];
+				ml->acl_add.field_value[6].value.u32 =
+					rte_be_to_cpu_32(da32[1]);
 				ml->acl_add.field_value[6].mask_range.u32 =
 					da32_depth[1];
-				ml->acl_add.field_value[7].value.u32 = da32[2];
+				ml->acl_add.field_value[7].value.u32 =
+					rte_be_to_cpu_32(da32[2]);
 				ml->acl_add.field_value[7].mask_range.u32 =
 					da32_depth[2];
-				ml->acl_add.field_value[8].value.u32 = da32[3];
+				ml->acl_add.field_value[8].value.u32 =
+					rte_be_to_cpu_32(da32[3]);
 				ml->acl_add.field_value[8].mask_range.u32 =
 					da32_depth[3];
 
@@ -2264,36 +2310,36 @@ match_convert(struct softnic_table_rule_match *mh,
 					mh->match.acl.proto_mask;
 
 				ml->acl_delete.field_value[1].value.u32 =
-					sa32[0];
+					rte_be_to_cpu_32(sa32[0]);
 				ml->acl_delete.field_value[1].mask_range.u32 =
 					sa32_depth[0];
 				ml->acl_delete.field_value[2].value.u32 =
-					sa32[1];
+					rte_be_to_cpu_32(sa32[1]);
 				ml->acl_delete.field_value[2].mask_range.u32 =
 					sa32_depth[1];
 				ml->acl_delete.field_value[3].value.u32 =
-					sa32[2];
+					rte_be_to_cpu_32(sa32[2]);
 				ml->acl_delete.field_value[3].mask_range.u32 =
 					sa32_depth[2];
 				ml->acl_delete.field_value[4].value.u32 =
-					sa32[3];
+					rte_be_to_cpu_32(sa32[3]);
 				ml->acl_delete.field_value[4].mask_range.u32 =
 					sa32_depth[3];
 
 				ml->acl_delete.field_value[5].value.u32 =
-					da32[0];
+					rte_be_to_cpu_32(da32[0]);
 				ml->acl_delete.field_value[5].mask_range.u32 =
 					da32_depth[0];
 				ml->acl_delete.field_value[6].value.u32 =
-					da32[1];
+					rte_be_to_cpu_32(da32[1]);
 				ml->acl_delete.field_value[6].mask_range.u32 =
 					da32_depth[1];
 				ml->acl_delete.field_value[7].value.u32 =
-					da32[2];
+					rte_be_to_cpu_32(da32[2]);
 				ml->acl_delete.field_value[7].mask_range.u32 =
 					da32_depth[2];
 				ml->acl_delete.field_value[8].value.u32 =
-					da32[3];
+					rte_be_to_cpu_32(da32[3]);
 				ml->acl_delete.field_value[8].mask_range.u32 =
 					da32_depth[3];
 
@@ -2432,6 +2478,36 @@ action_convert(struct rte_table_action *a,
 			return status;
 	}
 
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_TAG,
+			&action->tag);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_DECAP,
+			&action->decap);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_SYM_CRYPTO,
+			&action->sym_crypto);
+
+		if (status)
+			return status;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/net/szedata2/rte_eth_szedata2.c b/drivers/net/szedata2/rte_eth_szedata2.c
index 1d20cb51..88448eff 100644
--- a/drivers/net/szedata2/rte_eth_szedata2.c
+++ b/drivers/net/szedata2/rte_eth_szedata2.c
@@ -1056,8 +1056,7 @@ eth_dev_info(struct rte_eth_dev *dev,
 	dev_info->max_rx_queues = internals->max_rx_queues;
 	dev_info->max_tx_queues = internals->max_tx_queues;
 	dev_info->min_rx_bufsize = 0;
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_SCATTER |
-				    DEV_RX_OFFLOAD_CRC_STRIP;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_SCATTER;
 	dev_info->tx_offload_capa = 0;
 	dev_info->rx_queue_offload_capa = 0;
 	dev_info->tx_queue_offload_capa = 0;
@@ -1475,7 +1474,7 @@ rte_szedata2_eth_dev_init(struct rte_eth_dev *dev, struct port_info *pi)
 	PMD_INIT_FUNC_TRACE();
 
 	PMD_INIT_LOG(INFO, "Initializing eth_dev %s (driver %s)", data->name,
-			dev->device->driver->name);
+			RTE_STR(RTE_SZEDATA2_DRIVER_NAME));
 
 	/* Fill internal private structure. */
 	internals->dev = dev;
@@ -1526,7 +1525,7 @@ rte_szedata2_eth_dev_init(struct rte_eth_dev *dev, struct port_info *pi)
 	ether_addr_copy(&eth_addr, data->mac_addrs);
 
 	PMD_INIT_LOG(INFO, "%s device %s successfully initialized",
-			dev->device->driver->name, data->name);
+			RTE_STR(RTE_SZEDATA2_DRIVER_NAME), data->name);
 
 	return 0;
 }
@@ -1545,10 +1544,9 @@ rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev)
 	PMD_INIT_FUNC_TRACE();
 
 	free(internals->sze_dev_path);
-	rte_free(dev->data->mac_addrs);
 
 	PMD_DRV_LOG(INFO, "%s device %s successfully uninitialized",
-			dev->device->driver->name, dev->data->name);
+			RTE_STR(RTE_SZEDATA2_DRIVER_NAME), dev->data->name);
 
 	return 0;
 }
diff --git a/drivers/net/tap/Makefile b/drivers/net/tap/Makefile
index 32433653..77482838 100644
--- a/drivers/net/tap/Makefile
+++ b/drivers/net/tap/Makefile
@@ -22,6 +22,7 @@ CFLAGS += -O3
 CFLAGS += -I$(SRCDIR)
 CFLAGS += -I.
 CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash
 LDLIBS += -lrte_bus_vdev -lrte_gso
diff --git a/drivers/net/tap/meson.build b/drivers/net/tap/meson.build
new file mode 100644
index 00000000..9cb7142a
--- /dev/null
+++ b/drivers/net/tap/meson.build
@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2018 Luca Boccassi <bluca@debian.org>
+
+if host_machine.system() != 'linux'
+        build = false
+endif
+sources = files(
+	'rte_eth_tap.c',
+	'tap_bpf_api.c',
+	'tap_flow.c',
+	'tap_intr.c',
+	'tap_netlink.c',
+	'tap_tcmsgs.c',
+)
+
+deps = ['bus_vdev', 'gso', 'hash']
+
+cflags += '-DTAP_MAX_QUEUES=16'
+
+# To maintain the compatibility with the make build system
+# tap_autoconf.h file is still generated.
+# input array for meson symbol search:
+# [ "MACRO to define if found", "header for the search",
+#   "enum/define", "symbol to search" ]
+#
+args = [
+	[ 'HAVE_TC_FLOWER', 'linux/pkt_cls.h',
+	  'TCA_FLOWER_UNSPEC' ],
+	[ 'HAVE_TC_VLAN_ID', 'linux/pkt_cls.h',
+	  'TCA_FLOWER_KEY_VLAN_PRIO' ],
+	[ 'HAVE_TC_BPF', 'linux/pkt_cls.h',
+	  'TCA_BPF_UNSPEC' ],
+	[ 'HAVE_TC_BPF_FD', 'linux/pkt_cls.h',
+	  'TCA_BPF_FD' ],
+	[ 'HAVE_TC_ACT_BPF', 'linux/tc_act/tc_bpf.h',
+	  'TCA_ACT_BPF_UNSPEC' ],
+	[ 'HAVE_TC_ACT_BPF_FD', 'linux/tc_act/tc_bpf.h',
+	  'TCA_ACT_BPF_FD' ],
+]
+config = configuration_data()
+allow_experimental_apis = true
+foreach arg:args
+	config.set(arg[0], cc.has_header_symbol(arg[1], arg[2]))
+endforeach
+configure_file(output : 'tap_autoconf.h', configuration : config)
diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index feb92b48..e7817e89 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -16,6 +16,8 @@
 #include <rte_debug.h>
 #include <rte_ip.h>
 #include <rte_string_fns.h>
+#include <rte_ethdev.h>
+#include <rte_errno.h>
 
 #include <assert.h>
 #include <sys/types.h>
@@ -62,6 +64,10 @@
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
+/* IPC key for queue fds sync */
+#define TAP_MP_KEY "tap_mp_sync_queues"
+
+static int tap_devices_count;
 static struct rte_vdev_driver pmd_tap_drv;
 static struct rte_vdev_driver pmd_tun_drv;
 
@@ -100,6 +106,17 @@ enum ioctl_mode {
 	REMOTE_ONLY,
 };
 
+/* Message header to synchronize queues via IPC */
+struct ipc_queues {
+	char port_name[RTE_DEV_NAME_MAX_LEN];
+	int rxq_count;
+	int txq_count;
+	/*
+	 * The file descriptors are in the dedicated part
+	 * of the Unix message to be translated by the kernel.
+	 */
+};
+
 static int tap_intr_handle_set(struct rte_eth_dev *dev, int set);
 
 /**
@@ -305,8 +322,7 @@ tap_rx_offload_get_queue_capa(void)
 	return DEV_RX_OFFLOAD_SCATTER |
 	       DEV_RX_OFFLOAD_IPV4_CKSUM |
 	       DEV_RX_OFFLOAD_UDP_CKSUM |
-	       DEV_RX_OFFLOAD_TCP_CKSUM |
-	       DEV_RX_OFFLOAD_CRC_STRIP;
+	       DEV_RX_OFFLOAD_TCP_CKSUM;
 }
 
 /* Callback to handle the rx burst of packets to the correct interface and
@@ -316,6 +332,7 @@ static uint16_t
 pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 	uint16_t num_rx;
 	unsigned long num_rx_bytes = 0;
 	uint32_t trigger = tap_trigger;
@@ -324,6 +341,7 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		return 0;
 	if (trigger)
 		rxq->trigger_seen = trigger;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
 	rte_compiler_barrier();
 	for (num_rx = 0; num_rx < nb_pkts; ) {
 		struct rte_mbuf *mbuf = rxq->pool;
@@ -332,9 +350,9 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		uint16_t data_off = rte_pktmbuf_headroom(mbuf);
 		int len;
 
-		len = readv(rxq->fd, *rxq->iovecs,
-			    1 +
-			    (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
+		len = readv(process_private->rxq_fds[rxq->queue_id],
+			*rxq->iovecs,
+			1 + (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
 			     rxq->nb_rx_desc : 1));
 		if (len < (int)sizeof(struct tun_pi))
 			break;
@@ -495,6 +513,9 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 {
 	int i;
 	uint16_t l234_hlen;
+	struct pmd_process_private *process_private;
+
+	process_private = rte_eth_devices[txq->out_port].process_private;
 
 	for (i = 0; i < num_mbufs; i++) {
 		struct rte_mbuf *mbuf = pmbufs[i];
@@ -596,7 +617,7 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 			tap_tx_l4_cksum(l4_cksum, l4_phdr_cksum, l4_raw_cksum);
 
 		/* copy the tx frame data */
-		n = writev(txq->fd, iovecs, j);
+		n = writev(process_private->txq_fds[txq->queue_id], iovecs, j);
 		if (n <= 0)
 			break;
 		(*num_packets)++;
@@ -686,7 +707,7 @@ pmd_tx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 	txq->stats.errs += nb_pkts - num_tx;
 	txq->stats.obytes += num_tx_bytes;
 
-	return num_tx;
+	return num_packets;
 }
 
 static const char *
@@ -971,19 +992,20 @@ tap_dev_close(struct rte_eth_dev *dev)
 {
 	int i;
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	tap_link_set_down(dev);
 	tap_flow_flush(dev, NULL);
 	tap_flow_implicit_flush(internals, NULL);
 
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
@@ -1007,10 +1029,14 @@ static void
 tap_rx_queue_release(void *queue)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 
-	if (rxq && (rxq->fd > 0)) {
-		close(rxq->fd);
-		rxq->fd = -1;
+	if (!rxq)
+		return;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
+	if (process_private->rxq_fds[rxq->queue_id] > 0) {
+		close(process_private->rxq_fds[rxq->queue_id]);
+		process_private->rxq_fds[rxq->queue_id] = -1;
 		rte_pktmbuf_free(rxq->pool);
 		rte_free(rxq->iovecs);
 		rxq->pool = NULL;
@@ -1022,10 +1048,15 @@ static void
 tap_tx_queue_release(void *queue)
 {
 	struct tx_queue *txq = queue;
+	struct pmd_process_private *process_private;
 
-	if (txq && (txq->fd > 0)) {
-		close(txq->fd);
-		txq->fd = -1;
+	if (!txq)
+		return;
+	process_private = rte_eth_devices[txq->out_port].process_private;
+
+	if (process_private->txq_fds[txq->queue_id] > 0) {
+		close(process_private->txq_fds[txq->queue_id]);
+		process_private->txq_fds[txq->queue_id] = -1;
 	}
 }
 
@@ -1210,18 +1241,19 @@ tap_setup_queue(struct rte_eth_dev *dev,
 	int *other_fd;
 	const char *dir;
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rx = &internals->rxq[qid];
 	struct tx_queue *tx = &internals->txq[qid];
 	struct rte_gso_ctx *gso_ctx;
 
 	if (is_rx) {
-		fd = &rx->fd;
-		other_fd = &tx->fd;
+		fd = &process_private->rxq_fds[qid];
+		other_fd = &process_private->txq_fds[qid];
 		dir = "rx";
 		gso_ctx = NULL;
 	} else {
-		fd = &tx->fd;
-		other_fd = &rx->fd;
+		fd = &process_private->txq_fds[qid];
+		other_fd = &process_private->rxq_fds[qid];
 		dir = "tx";
 		gso_ctx = &tx->gso_ctx;
 	}
@@ -1274,6 +1306,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 		   struct rte_mempool *mp)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rxq = &internals->rxq[rx_queue_id];
 	struct rte_mbuf **tmp = &rxq->pool;
 	long iov_max = sysconf(_SC_IOV_MAX);
@@ -1294,6 +1327,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->mp = mp;
 	rxq->trigger_seen = 1; /* force initial burst */
 	rxq->in_port = dev->data->port_id;
+	rxq->queue_id = rx_queue_id;
 	rxq->nb_rx_desc = nb_desc;
 	iovecs = rte_zmalloc_socket(dev->device->name, sizeof(*iovecs), 0,
 				    socket_id);
@@ -1332,7 +1366,8 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	}
 
 	TAP_LOG(DEBUG, "  RX TUNTAP device name %s, qid %d on fd %d",
-		internals->name, rx_queue_id, internals->rxq[rx_queue_id].fd);
+		internals->name, rx_queue_id,
+		process_private->rxq_fds[rx_queue_id]);
 
 	return 0;
 
@@ -1352,6 +1387,7 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		   const struct rte_eth_txconf *tx_conf)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct tx_queue *txq;
 	int ret;
 	uint64_t offloads;
@@ -1360,6 +1396,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	dev->data->tx_queues[tx_queue_id] = &internals->txq[tx_queue_id];
 	txq = dev->data->tx_queues[tx_queue_id];
+	txq->out_port = dev->data->port_id;
+	txq->queue_id = tx_queue_id;
 
 	offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
 	txq->csum = !!(offloads &
@@ -1372,7 +1410,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	TAP_LOG(DEBUG,
 		"  TX TUNTAP device name %s, qid %d on fd %d csum %s",
-		internals->name, tx_queue_id, internals->txq[tx_queue_id].fd,
+		internals->name, tx_queue_id,
+		process_private->txq_fds[tx_queue_id],
 		txq->csum ? "on" : "off");
 
 	return 0;
@@ -1620,6 +1659,7 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	int numa_node = rte_socket_id();
 	struct rte_eth_dev *dev;
 	struct pmd_internals *pmd;
+	struct pmd_process_private *process_private;
 	struct rte_eth_dev_data *data;
 	struct ifreq ifr;
 	int i;
@@ -1634,7 +1674,16 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 		goto error_exit_nodev;
 	}
 
+	process_private = (struct pmd_process_private *)
+		rte_zmalloc_socket(tap_name, sizeof(struct pmd_process_private),
+			RTE_CACHE_LINE_SIZE, dev->device->numa_node);
+
+	if (process_private == NULL) {
+		TAP_LOG(ERR, "Failed to alloc memory for process private");
+		return -1;
+	}
 	pmd = dev->data->dev_private;
+	dev->process_private = process_private;
 	pmd->dev = dev;
 	snprintf(pmd->name, sizeof(pmd->name), "%s", tap_name);
 	pmd->type = type;
@@ -1670,8 +1719,8 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	/* Presetup the fds to -1 as being not valid */
 	pmd->ka_fd = -1;
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		pmd->rxq[i].fd = -1;
-		pmd->txq[i].fd = -1;
+		process_private->rxq_fds[i] = -1;
+		process_private->txq_fds[i] = -1;
 	}
 
 	if (pmd->type == ETH_TUNTAP_TYPE_TAP) {
@@ -1809,6 +1858,8 @@ error_remote:
 error_exit:
 	if (pmd->ioctl_sock > 0)
 		close(pmd->ioctl_sock);
+	/* mac_addrs must not be freed alone because part of dev_private */
+	dev->data->mac_addrs = NULL;
 	rte_eth_dev_release_port(dev);
 
 error_exit_nodev:
@@ -1974,6 +2025,102 @@ leave:
 	return ret;
 }
 
+/* Request queue file descriptors from secondary to primary. */
+static int
+tap_mp_attach_queues(const char *port_name, struct rte_eth_dev *dev)
+{
+	int ret;
+	struct timespec timeout = {.tv_sec = 1, .tv_nsec = 0};
+	struct rte_mp_msg request, *reply;
+	struct rte_mp_reply replies;
+	struct ipc_queues *request_param = (struct ipc_queues *)request.param;
+	struct ipc_queues *reply_param;
+	struct pmd_process_private *process_private = dev->process_private;
+	int queue, fd_iterator;
+
+	/* Prepare the request */
+	strlcpy(request.name, TAP_MP_KEY, sizeof(request.name));
+	strlcpy(request_param->port_name, port_name,
+		sizeof(request_param->port_name));
+	request.len_param = sizeof(*request_param);
+	/* Send request and receive reply */
+	ret = rte_mp_request_sync(&request, &replies, &timeout);
+	if (ret < 0) {
+		TAP_LOG(ERR, "Failed to request queues from primary: %d",
+			rte_errno);
+		return -1;
+	}
+	reply = &replies.msgs[0];
+	reply_param = (struct ipc_queues *)reply->param;
+	TAP_LOG(DEBUG, "Received IPC reply for %s", reply_param->port_name);
+
+	/* Attach the queues from received file descriptors */
+	dev->data->nb_rx_queues = reply_param->rxq_count;
+	dev->data->nb_tx_queues = reply_param->txq_count;
+	fd_iterator = 0;
+	for (queue = 0; queue < reply_param->rxq_count; queue++)
+		process_private->rxq_fds[queue] = reply->fds[fd_iterator++];
+	for (queue = 0; queue < reply_param->txq_count; queue++)
+		process_private->txq_fds[queue] = reply->fds[fd_iterator++];
+
+	return 0;
+}
+
+/* Send the queue file descriptors from the primary process to secondary. */
+static int
+tap_mp_sync_queues(const struct rte_mp_msg *request, const void *peer)
+{
+	struct rte_eth_dev *dev;
+	struct pmd_process_private *process_private;
+	struct rte_mp_msg reply;
+	const struct ipc_queues *request_param =
+		(const struct ipc_queues *)request->param;
+	struct ipc_queues *reply_param =
+		(struct ipc_queues *)reply.param;
+	uint16_t port_id;
+	int queue;
+	int ret;
+
+	/* Get requested port */
+	TAP_LOG(DEBUG, "Received IPC request for %s", request_param->port_name);
+	ret = rte_eth_dev_get_port_by_name(request_param->port_name, &port_id);
+	if (ret) {
+		TAP_LOG(ERR, "Failed to get port id for %s",
+			request_param->port_name);
+		return -1;
+	}
+	dev = &rte_eth_devices[port_id];
+	process_private = dev->process_private;
+
+	/* Fill file descriptors for all queues */
+	reply.num_fds = 0;
+	reply_param->rxq_count = 0;
+	for (queue = 0; queue < dev->data->nb_rx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->rxq_fds[queue];
+		reply_param->rxq_count++;
+	}
+	RTE_ASSERT(reply_param->rxq_count == dev->data->nb_rx_queues);
+	RTE_ASSERT(reply_param->txq_count == dev->data->nb_tx_queues);
+	RTE_ASSERT(reply.num_fds <= RTE_MP_MAX_FD_NUM);
+
+	reply_param->txq_count = 0;
+	for (queue = 0; queue < dev->data->nb_tx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->txq_fds[queue];
+		reply_param->txq_count++;
+	}
+
+	/* Send reply */
+	strlcpy(reply.name, request->name, sizeof(reply.name));
+	strlcpy(reply_param->port_name, request_param->port_name,
+		sizeof(reply_param->port_name));
+	reply.len_param = sizeof(*reply_param);
+	if (rte_mp_reply(&reply, peer) < 0) {
+		TAP_LOG(ERR, "Failed to reply an IPC request to sync queues");
+		return -1;
+	}
+	return 0;
+}
+
 /* Open a TAP interface device.
  */
 static int
@@ -1987,22 +2134,41 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	char remote_iface[RTE_ETH_NAME_MAX_LEN];
 	struct ether_addr user_mac = { .addr_bytes = {0} };
 	struct rte_eth_dev *eth_dev;
+	int tap_devices_count_increased = 0;
 
 	strcpy(tuntap_name, "TAP");
 
 	name = rte_vdev_device_name(dev);
 	params = rte_vdev_device_args(dev);
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
-	    strlen(params) == 0) {
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 		eth_dev = rte_eth_dev_attach_secondary(name);
 		if (!eth_dev) {
 			TAP_LOG(ERR, "Failed to probe %s", name);
 			return -1;
 		}
-		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &ops;
 		eth_dev->device = &dev->device;
+		eth_dev->rx_pkt_burst = pmd_rx_burst;
+		eth_dev->tx_pkt_burst = pmd_tx_burst;
+		if (!rte_eal_primary_proc_alive(NULL)) {
+			TAP_LOG(ERR, "Primary process is missing");
+			return -1;
+		}
+		eth_dev->process_private = (struct pmd_process_private *)
+			rte_zmalloc_socket(name,
+				sizeof(struct pmd_process_private),
+				RTE_CACHE_LINE_SIZE,
+				eth_dev->device->numa_node);
+		if (eth_dev->process_private == NULL) {
+			TAP_LOG(ERR,
+				"Failed to alloc memory for process private");
+			return -1;
+		}
+
+		ret = tap_mp_attach_queues(name, eth_dev);
+		if (ret != 0)
+			return -1;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -2050,6 +2216,17 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
 		name, tap_name);
 
+	/* Register IPC feed callback */
+	if (!tap_devices_count) {
+		ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
+		if (ret < 0) {
+			TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
+				tuntap_name, strerror(rte_errno));
+			goto leave;
+		}
+	}
+	tap_devices_count++;
+	tap_devices_count_increased = 1;
 	ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
 		ETH_TUNTAP_TYPE_TAP);
 
@@ -2057,6 +2234,11 @@ leave:
 	if (ret == -1) {
 		TAP_LOG(ERR, "Failed to create pmd for %s as %s",
 			name, tap_name);
+		if (tap_devices_count_increased == 1) {
+			if (tap_devices_count == 1)
+				rte_mp_action_unregister(TAP_MP_KEY);
+			tap_devices_count--;
+		}
 		tap_unit--;		/* Restore the unit number */
 	}
 	rte_kvargs_free(kvlist);
@@ -2071,14 +2253,22 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 {
 	struct rte_eth_dev *eth_dev = NULL;
 	struct pmd_internals *internals;
+	struct pmd_process_private *process_private;
 	int i;
 
 	/* find the ethdev entry */
 	eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(dev));
 	if (!eth_dev)
-		return 0;
+		return -ENODEV;
+
+	/* mac_addrs must not be freed alone because part of dev_private */
+	eth_dev->data->mac_addrs = NULL;
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return rte_eth_dev_release_port(eth_dev);
 
 	internals = eth_dev->data->dev_private;
+	process_private = eth_dev->process_private;
 
 	TAP_LOG(DEBUG, "Closing %s Ethernet device on numa %u",
 		(internals->type == ETH_TUNTAP_TYPE_TAP) ? "TAP" : "TUN",
@@ -2090,18 +2280,21 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 		tap_nl_final(internals->nlsk_fd);
 	}
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
 	close(internals->ioctl_sock);
-	rte_free(eth_dev->data->dev_private);
+	rte_free(eth_dev->process_private);
+	if (tap_devices_count == 1)
+		rte_mp_action_unregister(TAP_MP_KEY);
+	tap_devices_count--;
 	rte_eth_dev_release_port(eth_dev);
 
 	if (internals->ka_fd != -1) {
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 44e2773f..dc3579ac 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -46,7 +46,7 @@ struct rx_queue {
 	struct rte_mempool *mp;         /* Mempool for RX packets */
 	uint32_t trigger_seen;          /* Last seen Rx trigger value */
 	uint16_t in_port;               /* Port ID */
-	int fd;
+	uint16_t queue_id;		/* queue ID*/
 	struct pkt_stats stats;         /* Stats for this RX queue */
 	uint16_t nb_rx_desc;            /* max number of mbufs available */
 	struct rte_eth_rxmode *rxmode;  /* RX features */
@@ -56,12 +56,13 @@ struct rx_queue {
 };
 
 struct tx_queue {
-	int fd;
 	int type;                       /* Type field - TUN|TAP */
 	uint16_t *mtu;                  /* Pointer to MTU from dev_data */
 	uint16_t csum:1;                /* Enable checksum offloading */
 	struct pkt_stats stats;         /* Stats for this TX queue */
 	struct rte_gso_ctx gso_ctx;     /* GSO context */
+	uint16_t out_port;              /* Port ID */
+	uint16_t queue_id;		/* queue ID*/
 };
 
 struct pmd_internals {
@@ -92,6 +93,11 @@ struct pmd_internals {
 	int ka_fd;                        /* keep-alive file descriptor */
 };
 
+struct pmd_process_private {
+	int rxq_fds[RTE_PMD_TAP_MAX_QUEUES];
+	int txq_fds[RTE_PMD_TAP_MAX_QUEUES];
+};
+
 /* tap_intr.c */
 
 int tap_rx_intr_vec_set(struct rte_eth_dev *dev, int set);
diff --git a/drivers/net/tap/tap_bpf_insns.h b/drivers/net/tap/tap_bpf_insns.h
index 79e3e66b..1a91bbad 100644
--- a/drivers/net/tap/tap_bpf_insns.h
+++ b/drivers/net/tap/tap_bpf_insns.h
@@ -5,7 +5,7 @@
 #include <tap_bpf.h>
 
 /* bpf_insn array matching cls_q section. See tap_bpf_program.c file */
-struct bpf_insn cls_q_insns[] = {
+static struct bpf_insn cls_q_insns[] = {
 	{0x61,    2,    1,       52, 0x00000000},
 	{0x18,    3,    0,        0, 0xdeadbeef},
 	{0x00,    0,    0,        0, 0x00000000},
@@ -24,7 +24,7 @@ struct bpf_insn cls_q_insns[] = {
 };
 
 /* bpf_insn array matching l3_l4 section. see tap_bpf_program.c file */
-struct bpf_insn l3_l4_hash_insns[] = {
+static struct bpf_insn l3_l4_hash_insns[] = {
 	{0xbf,    7,    1,        0, 0x00000000},
 	{0x61,    8,    7,       16, 0x00000000},
 	{0x61,    6,    7,       76, 0x00000000},
diff --git a/drivers/net/tap/tap_flow.c b/drivers/net/tap/tap_flow.c
index 0e01af62..d155618f 100644
--- a/drivers/net/tap/tap_flow.c
+++ b/drivers/net/tap/tap_flow.c
@@ -1567,6 +1567,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 		 struct rte_flow_error *error __rte_unused)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	/* normalize 'set' variable to contain 0 or 1 values */
 	if (set)
@@ -1580,7 +1581,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 	 * If netdevice is there, setup appropriate flow rules immediately.
 	 * Otherwise it will be set when bringing up the netdevice (tun_alloc).
 	 */
-	if (!pmd->rxq[0].fd)
+	if (!process_private->rxq_fds[0])
 		return 0;
 	if (set) {
 		struct rte_flow *remote_flow;
@@ -1810,7 +1811,7 @@ tap_flow_implicit_flush(struct pmd_internals *pmd, struct rte_flow_error *error)
 #define KEY_IDX_OFFSET (3 * MAX_RSS_KEYS)
 #define SEC_NAME_CLS_Q "cls_q"
 
-const char *sec_name[SEC_MAX] = {
+static const char *sec_name[SEC_MAX] = {
 	[SEC_L3_L4] = "l3_l4",
 };
 
diff --git a/drivers/net/tap/tap_intr.c b/drivers/net/tap/tap_intr.c
index fc590181..7af0010e 100644
--- a/drivers/net/tap/tap_intr.c
+++ b/drivers/net/tap/tap_intr.c
@@ -51,6 +51,7 @@ static int
 tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	unsigned int rxqs_n = pmd->dev->data->nb_rx_queues;
 	struct rte_intr_handle *intr_handle = &pmd->intr_handle;
 	unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
@@ -71,7 +72,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 		struct rx_queue *rxq = pmd->dev->data->rx_queues[i];
 
 		/* Skip queues that cannot request interrupts. */
-		if (!rxq || rxq->fd <= 0) {
+		if (!rxq || process_private->rxq_fds[i] <= 0) {
 			/* Use invalid intr_vec[] index to disable entry. */
 			intr_handle->intr_vec[i] =
 				RTE_INTR_VEC_RXTX_OFFSET +
@@ -79,7 +80,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 			continue;
 		}
 		intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
-		intr_handle->efds[count] = rxq->fd;
+		intr_handle->efds[count] = process_private->rxq_fds[i];
 		count++;
 	}
 	if (!count)
diff --git a/drivers/net/thunderx/base/meson.build b/drivers/net/thunderx/base/meson.build
index c9d5a8f4..bf4e8608 100644
--- a/drivers/net/thunderx/base/meson.build
+++ b/drivers/net/thunderx/base/meson.build
@@ -7,8 +7,12 @@ sources = [
 	'nicvf_bsvf.c'
 ]
 
+c_args = cflags
+if allow_experimental_apis
+	c_args += '-DALLOW_EXPERIMENTAL_API'
+endif
 base_lib = static_library('nicvf_base', sources,
-	c_args: cflags,
+	c_args: c_args,
 	dependencies: static_rte_ethdev
 )
 
diff --git a/drivers/net/thunderx/nicvf_ethdev.c b/drivers/net/thunderx/nicvf_ethdev.c
index a55c3ca6..879d8899 100644
--- a/drivers/net/thunderx/nicvf_ethdev.c
+++ b/drivers/net/thunderx/nicvf_ethdev.c
@@ -1431,7 +1431,6 @@ nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	dev_info->default_rxconf = (struct rte_eth_rxconf) {
 		.rx_free_thresh = NICVF_DEFAULT_RX_FREE_THRESH,
 		.rx_drop_en = 0,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	};
 
 	dev_info->default_txconf = (struct rte_eth_txconf) {
@@ -1916,14 +1915,6 @@ nicvf_dev_configure(struct rte_eth_dev *dev)
 		return -EINVAL;
 	}
 
-	/* KEEP_CRC offload flag is not supported by PMD
-	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
-	 */
-	if (rte_eth_dev_must_keep_crc(rxmode->offloads)) {
-		PMD_INIT_LOG(NOTICE, "Can't disable hw crc strip");
-		rxmode->offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
-	}
-
 	if (txmode->mq_mode) {
 		PMD_INIT_LOG(INFO, "Tx mq_mode DCB or VMDq not supported");
 		return -EINVAL;
diff --git a/drivers/net/thunderx/nicvf_ethdev.h b/drivers/net/thunderx/nicvf_ethdev.h
index ae440fef..c0bfbf84 100644
--- a/drivers/net/thunderx/nicvf_ethdev.h
+++ b/drivers/net/thunderx/nicvf_ethdev.h
@@ -40,7 +40,6 @@
 #define NICVF_RX_OFFLOAD_CAPA ( \
 	DEV_RX_OFFLOAD_CHECKSUM    | \
 	DEV_RX_OFFLOAD_VLAN_STRIP  | \
-	DEV_RX_OFFLOAD_CRC_STRIP   | \
 	DEV_RX_OFFLOAD_JUMBO_FRAME | \
 	DEV_RX_OFFLOAD_SCATTER)
 
diff --git a/drivers/net/vdev_netvsc/meson.build b/drivers/net/vdev_netvsc/meson.build
new file mode 100644
index 00000000..d3ada878
--- /dev/null
+++ b/drivers/net/vdev_netvsc/meson.build
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
+
+if host_machine.system() != 'linux'
+        build = false
+endif
+sources = files('vdev_netvsc.c')
+
+allow_experimental_apis = true
+
+cflags_options = [
+        '-Wall',
+        '-Wextra',
+        '-D_BSD_SOURCE',
+        '-D_DEFAULT_SOURCE',
+        '-D_XOPEN_SOURCE=600'
+]
+foreach option:cflags_options
+        if cc.has_argument(option)
+                cflags += option
+        endif
+endforeach
diff --git a/drivers/net/vdev_netvsc/vdev_netvsc.c b/drivers/net/vdev_netvsc/vdev_netvsc.c
index 48717f2f..16303ef5 100644
--- a/drivers/net/vdev_netvsc/vdev_netvsc.c
+++ b/drivers/net/vdev_netvsc/vdev_netvsc.c
@@ -789,7 +789,7 @@ RTE_PMD_REGISTER_PARAM_STRING(net_vdev_netvsc,
 /** Initialize driver log type. */
 RTE_INIT(vdev_netvsc_init_log)
 {
-	vdev_netvsc_logtype = rte_log_register("pmd.vdev_netvsc");
+	vdev_netvsc_logtype = rte_log_register("pmd.net.vdev_netvsc");
 	if (vdev_netvsc_logtype >= 0)
 		rte_log_set_level(vdev_netvsc_logtype, RTE_LOG_NOTICE);
 }
diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index e58f3221..b38a4b6b 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -30,6 +30,7 @@ enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM};
 #define ETH_VHOST_CLIENT_ARG		"client"
 #define ETH_VHOST_DEQUEUE_ZERO_COPY	"dequeue-zero-copy"
 #define ETH_VHOST_IOMMU_SUPPORT		"iommu-support"
+#define ETH_VHOST_POSTCOPY_SUPPORT	"postcopy-support"
 #define VHOST_MAX_PKT_BURST 32
 
 static const char *valid_arguments[] = {
@@ -38,6 +39,7 @@ static const char *valid_arguments[] = {
 	ETH_VHOST_CLIENT_ARG,
 	ETH_VHOST_DEQUEUE_ZERO_COPY,
 	ETH_VHOST_IOMMU_SUPPORT,
+	ETH_VHOST_POSTCOPY_SUPPORT,
 	NULL
 };
 
@@ -1070,8 +1072,7 @@ eth_dev_info(struct rte_eth_dev *dev,
 
 	dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
 				DEV_TX_OFFLOAD_VLAN_INSERT;
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
-				    DEV_RX_OFFLOAD_CRC_STRIP;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
 }
 
 static int
@@ -1221,10 +1222,12 @@ eth_dev_vhost_create(struct rte_vdev_device *dev, char *iface_name,
 	eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internal));
 	if (eth_dev == NULL)
 		goto error;
+	data = eth_dev->data;
 
 	eth_addr = rte_zmalloc_socket(name, sizeof(*eth_addr), 0, numa_node);
 	if (eth_addr == NULL)
 		goto error;
+	data->mac_addrs = eth_addr;
 	*eth_addr = base_eth_addr;
 	eth_addr->addr_bytes[5] = eth_dev->data->port_id;
 
@@ -1254,13 +1257,11 @@ eth_dev_vhost_create(struct rte_vdev_device *dev, char *iface_name,
 	rte_spinlock_init(&vring_state->lock);
 	vring_states[eth_dev->data->port_id] = vring_state;
 
-	data = eth_dev->data;
 	data->nb_rx_queues = queues;
 	data->nb_tx_queues = queues;
 	internal->max_queues = queues;
 	internal->vid = -1;
 	data->dev_link = pmd_link;
-	data->mac_addrs = eth_addr;
 	data->dev_flags = RTE_ETH_DEV_INTR_LSC;
 
 	eth_dev->dev_ops = &ops;
@@ -1292,10 +1293,7 @@ error:
 		free(internal->dev_name);
 	}
 	rte_free(vring_state);
-	rte_free(eth_addr);
-	if (eth_dev)
-		rte_eth_dev_release_port(eth_dev);
-	rte_free(internal);
+	rte_eth_dev_release_port(eth_dev);
 	rte_free(list);
 
 	return -1;
@@ -1340,13 +1338,13 @@ rte_pmd_vhost_probe(struct rte_vdev_device *dev)
 	int client_mode = 0;
 	int dequeue_zero_copy = 0;
 	int iommu_support = 0;
+	int postcopy_support = 0;
 	struct rte_eth_dev *eth_dev;
 	const char *name = rte_vdev_device_name(dev);
 
 	VHOST_LOG(INFO, "Initializing pmd_vhost for %s\n", name);
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
-	    strlen(rte_vdev_device_args(dev)) == 0) {
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 		eth_dev = rte_eth_dev_attach_secondary(name);
 		if (!eth_dev) {
 			VHOST_LOG(ERR, "Failed to probe %s\n", name);
@@ -1412,6 +1410,16 @@ rte_pmd_vhost_probe(struct rte_vdev_device *dev)
 			flags |= RTE_VHOST_USER_IOMMU_SUPPORT;
 	}
 
+	if (rte_kvargs_count(kvlist, ETH_VHOST_POSTCOPY_SUPPORT) == 1) {
+		ret = rte_kvargs_process(kvlist, ETH_VHOST_POSTCOPY_SUPPORT,
+					 &open_int, &postcopy_support);
+		if (ret < 0)
+			goto out_free;
+
+		if (postcopy_support)
+			flags |= RTE_VHOST_USER_POSTCOPY_SUPPORT;
+	}
+
 	if (dev->device.numa_node == SOCKET_ID_ANY)
 		dev->device.numa_node = rte_socket_id();
 
@@ -1437,6 +1445,9 @@ rte_pmd_vhost_remove(struct rte_vdev_device *dev)
 	if (eth_dev == NULL)
 		return -ENODEV;
 
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return rte_eth_dev_release_port(eth_dev);
+
 	eth_dev_close(eth_dev);
 
 	rte_free(vring_states[eth_dev->data->port_id]);
diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
index 614357da..10a7e3fc 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1697,7 +1697,7 @@ eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev)
 	PMD_INIT_FUNC_TRACE();
 
 	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
-		return -EPERM;
+		return 0;
 
 	virtio_dev_stop(eth_dev);
 	virtio_dev_close(eth_dev);
@@ -1706,9 +1706,6 @@ eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev)
 	eth_dev->tx_pkt_burst = NULL;
 	eth_dev->rx_pkt_burst = NULL;
 
-	rte_free(eth_dev->data->mac_addrs);
-	eth_dev->data->mac_addrs = NULL;
-
 	/* reset interrupt callback  */
 	if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
 		rte_intr_callback_unregister(eth_dev->intr_handle,
@@ -2166,8 +2163,7 @@ virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS;
 
 	host_features = VTPCI_OPS(hw)->get_features(hw);
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
-				    DEV_RX_OFFLOAD_CRC_STRIP;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
 	if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
 		dev_info->rx_offload_capa |=
 			DEV_RX_OFFLOAD_TCP_CKSUM |
diff --git a/drivers/net/virtio/virtio_ethdev.h b/drivers/net/virtio/virtio_ethdev.h
index b726ad10..e0f80e5a 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -40,7 +40,10 @@
 	(VIRTIO_PMD_DEFAULT_GUEST_FEATURES |	\
 	 1u << VIRTIO_NET_F_GUEST_CSUM	   |	\
 	 1u << VIRTIO_NET_F_GUEST_TSO4     |	\
-	 1u << VIRTIO_NET_F_GUEST_TSO6)
+	 1u << VIRTIO_NET_F_GUEST_TSO6     |	\
+	 1u << VIRTIO_NET_F_CSUM           |	\
+	 1u << VIRTIO_NET_F_HOST_TSO4      |	\
+	 1u << VIRTIO_NET_F_HOST_TSO6)
 
 /*
  * CQ function prototype
diff --git a/drivers/net/virtio/virtio_pci.c b/drivers/net/virtio/virtio_pci.c
index 6bd22e54..b6a3c80b 100644
--- a/drivers/net/virtio/virtio_pci.c
+++ b/drivers/net/virtio/virtio_pci.c
@@ -567,16 +567,18 @@ virtio_read_caps(struct rte_pci_device *dev, struct virtio_hw *hw)
 	}
 
 	ret = rte_pci_read_config(dev, &pos, 1, PCI_CAPABILITY_LIST);
-	if (ret < 0) {
-		PMD_INIT_LOG(DEBUG, "failed to read pci capability list");
+	if (ret != 1) {
+		PMD_INIT_LOG(DEBUG,
+			     "failed to read pci capability list, ret %d", ret);
 		return -1;
 	}
 
 	while (pos) {
-		ret = rte_pci_read_config(dev, &cap, sizeof(cap), pos);
-		if (ret < 0) {
-			PMD_INIT_LOG(ERR,
-				"failed to read pci cap at pos: %x", pos);
+		ret = rte_pci_read_config(dev, &cap, 2, pos);
+		if (ret != 2) {
+			PMD_INIT_LOG(DEBUG,
+				     "failed to read pci cap at pos: %x ret %d",
+				     pos, ret);
 			break;
 		}
 
@@ -586,7 +588,16 @@ virtio_read_caps(struct rte_pci_device *dev, struct virtio_hw *hw)
 			 * 1st byte is cap ID; 2nd byte is the position of next
 			 * cap; next two bytes are the flags.
 			 */
-			uint16_t flags = ((uint16_t *)&cap)[1];
+			uint16_t flags;
+
+			ret = rte_pci_read_config(dev, &flags, sizeof(flags),
+					pos + 2);
+			if (ret != sizeof(flags)) {
+				PMD_INIT_LOG(DEBUG,
+					     "failed to read pci cap at pos:"
+					     " %x ret %d", pos + 2, ret);
+				break;
+			}
 
 			if (flags & PCI_MSIX_ENABLE)
 				hw->use_msix = VIRTIO_MSIX_ENABLED;
@@ -601,6 +612,14 @@ virtio_read_caps(struct rte_pci_device *dev, struct virtio_hw *hw)
 			goto next;
 		}
 
+		ret = rte_pci_read_config(dev, &cap, sizeof(cap), pos);
+		if (ret != sizeof(cap)) {
+			PMD_INIT_LOG(DEBUG,
+				     "failed to read pci cap at pos: %x ret %d",
+				     pos, ret);
+			break;
+		}
+
 		PMD_INIT_LOG(DEBUG,
 			"[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u",
 			pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
@@ -689,25 +708,37 @@ enum virtio_msix_status
 vtpci_msix_detect(struct rte_pci_device *dev)
 {
 	uint8_t pos;
-	struct virtio_pci_cap cap;
 	int ret;
 
 	ret = rte_pci_read_config(dev, &pos, 1, PCI_CAPABILITY_LIST);
-	if (ret < 0) {
-		PMD_INIT_LOG(DEBUG, "failed to read pci capability list");
+	if (ret != 1) {
+		PMD_INIT_LOG(DEBUG,
+			     "failed to read pci capability list, ret %d", ret);
 		return VIRTIO_MSIX_NONE;
 	}
 
 	while (pos) {
-		ret = rte_pci_read_config(dev, &cap, sizeof(cap), pos);
-		if (ret < 0) {
-			PMD_INIT_LOG(ERR,
-				"failed to read pci cap at pos: %x", pos);
+		uint8_t cap[2];
+
+		ret = rte_pci_read_config(dev, cap, sizeof(cap), pos);
+		if (ret != sizeof(cap)) {
+			PMD_INIT_LOG(DEBUG,
+				     "failed to read pci cap at pos: %x ret %d",
+				     pos, ret);
 			break;
 		}
 
-		if (cap.cap_vndr == PCI_CAP_ID_MSIX) {
-			uint16_t flags = ((uint16_t *)&cap)[1];
+		if (cap[0] == PCI_CAP_ID_MSIX) {
+			uint16_t flags;
+
+			ret = rte_pci_read_config(dev, &flags, sizeof(flags),
+					pos + sizeof(cap));
+			if (ret != sizeof(flags)) {
+				PMD_INIT_LOG(DEBUG,
+					     "failed to read pci cap at pos:"
+					     " %x ret %d", pos + 2, ret);
+				break;
+			}
 
 			if (flags & PCI_MSIX_ENABLE)
 				return VIRTIO_MSIX_ENABLED;
@@ -715,7 +746,7 @@ vtpci_msix_detect(struct rte_pci_device *dev)
 				return VIRTIO_MSIX_DISABLED;
 		}
 
-		pos = cap.cap_next;
+		pos = cap[1];
 	}
 
 	return VIRTIO_MSIX_NONE;
diff --git a/drivers/net/virtio/virtio_rxtx_simple.c b/drivers/net/virtio/virtio_rxtx_simple.c
index 31e565b4..f8bcbaa1 100644
--- a/drivers/net/virtio/virtio_rxtx_simple.c
+++ b/drivers/net/virtio/virtio_rxtx_simple.c
@@ -47,7 +47,7 @@ virtio_rxq_vec_setup(struct virtnet_rx *rxq)
 }
 
 /* Stub for linkage when arch specific implementation is not available */
-uint16_t __attribute__((weak))
+__rte_weak uint16_t
 virtio_recv_pkts_vec(void *rx_queue __rte_unused,
 		     struct rte_mbuf **rx_pkts __rte_unused,
 		     uint16_t nb_pkts __rte_unused)
diff --git a/drivers/net/virtio/virtio_user/vhost.h b/drivers/net/virtio/virtio_user/vhost.h
index 668cc99f..83a85cc6 100644
--- a/drivers/net/virtio/virtio_user/vhost.h
+++ b/drivers/net/virtio/virtio_user/vhost.h
@@ -88,7 +88,7 @@ struct virtio_user_backend_ops {
 			 int enable);
 };
 
-struct virtio_user_backend_ops ops_user;
-struct virtio_user_backend_ops ops_kernel;
+extern struct virtio_user_backend_ops virtio_ops_user;
+extern struct virtio_user_backend_ops virtio_ops_kernel;
 
 #endif
diff --git a/drivers/net/virtio/virtio_user/vhost_kernel.c b/drivers/net/virtio/virtio_user/vhost_kernel.c
index b2444096..6b19180d 100644
--- a/drivers/net/virtio/virtio_user/vhost_kernel.c
+++ b/drivers/net/virtio/virtio_user/vhost_kernel.c
@@ -70,41 +70,44 @@ static uint64_t vhost_req_user_to_kernel[] = {
 	[VHOST_USER_SET_MEM_TABLE] = VHOST_SET_MEM_TABLE,
 };
 
-struct walk_arg {
-	struct vhost_memory_kernel *vm;
-	uint32_t region_nr;
-};
 static int
-add_memory_region(const struct rte_memseg_list *msl __rte_unused,
-		const struct rte_memseg *ms, size_t len, void *arg)
+add_memseg_list(const struct rte_memseg_list *msl, void *arg)
 {
-	struct walk_arg *wa = arg;
+	struct vhost_memory_kernel *vm = arg;
 	struct vhost_memory_region *mr;
 	void *start_addr;
+	uint64_t len;
 
-	if (wa->region_nr >= max_regions)
+	if (msl->external)
+		return 0;
+
+	if (vm->nregions >= max_regions)
 		return -1;
 
-	mr = &wa->vm->regions[wa->region_nr++];
-	start_addr = ms->addr;
+	start_addr = msl->base_va;
+	len = msl->page_sz * msl->memseg_arr.len;
+
+	mr = &vm->regions[vm->nregions++];
 
 	mr->guest_phys_addr = (uint64_t)(uintptr_t)start_addr;
 	mr->userspace_addr = (uint64_t)(uintptr_t)start_addr;
 	mr->memory_size = len;
-	mr->mmap_offset = 0;
+	mr->mmap_offset = 0; /* flags_padding */
+
+	PMD_DRV_LOG(DEBUG, "index=%u addr=%p len=%" PRIu64,
+			vm->nregions - 1, start_addr, len);
 
 	return 0;
 }
 
-/* By default, vhost kernel module allows 64 regions, but DPDK allows
- * 256 segments. As a relief, below function merges those virtually
- * adjacent memsegs into one region.
+/* By default, vhost kernel module allows 64 regions, but DPDK may
+ * have much more memory regions. Below function will treat each
+ * contiguous memory space reserved by DPDK as one region.
  */
 static struct vhost_memory_kernel *
 prepare_vhost_memory_kernel(void)
 {
 	struct vhost_memory_kernel *vm;
-	struct walk_arg wa;
 
 	vm = malloc(sizeof(struct vhost_memory_kernel) +
 			max_regions *
@@ -112,16 +115,18 @@ prepare_vhost_memory_kernel(void)
 	if (!vm)
 		return NULL;
 
-	wa.region_nr = 0;
-	wa.vm = vm;
+	vm->nregions = 0;
+	vm->padding = 0;
 
-	if (rte_memseg_contig_walk(add_memory_region, &wa) < 0) {
+	/*
+	 * The memory lock has already been taken by memory subsystem
+	 * or virtio_user_start_device().
+	 */
+	if (rte_memseg_list_walk_thread_unsafe(add_memseg_list, vm) < 0) {
 		free(vm);
 		return NULL;
 	}
 
-	vm->nregions = wa.region_nr;
-	vm->padding = 0;
 	return vm;
 }
 
@@ -147,8 +152,8 @@ prepare_vhost_memory_kernel(void)
 	 (1ULL << VIRTIO_NET_F_HOST_TSO6) |	\
 	 (1ULL << VIRTIO_NET_F_CSUM))
 
-static int
-tap_supporte_mq(void)
+static unsigned int
+tap_support_features(void)
 {
 	int tapfd;
 	unsigned int tap_features;
@@ -167,7 +172,7 @@ tap_supporte_mq(void)
 	}
 
 	close(tapfd);
-	return tap_features & IFF_MULTI_QUEUE;
+	return tap_features;
 }
 
 static int
@@ -181,6 +186,7 @@ vhost_kernel_ioctl(struct virtio_user_dev *dev,
 	struct vhost_memory_kernel *vm = NULL;
 	int vhostfd;
 	unsigned int queue_sel;
+	unsigned int features;
 
 	PMD_DRV_LOG(INFO, "%s", vhost_msg_strings[req]);
 
@@ -234,17 +240,20 @@ vhost_kernel_ioctl(struct virtio_user_dev *dev,
 	}
 
 	if (!ret && req_kernel == VHOST_GET_FEATURES) {
+		features = tap_support_features();
 		/* with tap as the backend, all these features are supported
 		 * but not claimed by vhost-net, so we add them back when
 		 * reporting to upper layer.
 		 */
-		*((uint64_t *)arg) |= VHOST_KERNEL_GUEST_OFFLOADS_MASK;
-		*((uint64_t *)arg) |= VHOST_KERNEL_HOST_OFFLOADS_MASK;
+		if (features & IFF_VNET_HDR) {
+			*((uint64_t *)arg) |= VHOST_KERNEL_GUEST_OFFLOADS_MASK;
+			*((uint64_t *)arg) |= VHOST_KERNEL_HOST_OFFLOADS_MASK;
+		}
 
 		/* vhost_kernel will not declare this feature, but it does
 		 * support multi-queue.
 		 */
-		if (tap_supporte_mq())
+		if (features & IFF_MULTI_QUEUE)
 			*(uint64_t *)arg |= (1ull << VIRTIO_NET_F_MQ);
 	}
 
@@ -339,7 +348,7 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
 		hdr_size = sizeof(struct virtio_net_hdr);
 
 	tapfd = vhost_kernel_open_tap(&dev->ifname, hdr_size, req_mq,
-			 (char *)dev->mac_addr);
+			 (char *)dev->mac_addr, dev->features);
 	if (tapfd < 0) {
 		PMD_DRV_LOG(ERR, "fail to open tap for vhost kernel");
 		return -1;
@@ -355,7 +364,7 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
 	return 0;
 }
 
-struct virtio_user_backend_ops ops_kernel = {
+struct virtio_user_backend_ops virtio_ops_kernel = {
 	.setup = vhost_kernel_setup,
 	.send_request = vhost_kernel_ioctl,
 	.enable_qp = vhost_kernel_enable_queue_pair
diff --git a/drivers/net/virtio/virtio_user/vhost_kernel_tap.c b/drivers/net/virtio/virtio_user/vhost_kernel_tap.c
index 9ea7ade7..a3faf1d0 100644
--- a/drivers/net/virtio/virtio_user/vhost_kernel_tap.c
+++ b/drivers/net/virtio/virtio_user/vhost_kernel_tap.c
@@ -16,21 +16,55 @@
 
 #include "vhost_kernel_tap.h"
 #include "../virtio_logs.h"
+#include "../virtio_pci.h"
+
+static int
+vhost_kernel_tap_set_offload(int fd, uint64_t features)
+{
+	unsigned int offload = 0;
+
+	if (features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
+		offload |= TUN_F_CSUM;
+		if (features & (1ULL << VIRTIO_NET_F_GUEST_TSO4))
+			offload |= TUN_F_TSO4;
+		if (features & (1ULL << VIRTIO_NET_F_GUEST_TSO6))
+			offload |= TUN_F_TSO6;
+		if (features & ((1ULL << VIRTIO_NET_F_GUEST_TSO4) |
+			(1ULL << VIRTIO_NET_F_GUEST_TSO6)) &&
+			(features & (1ULL << VIRTIO_NET_F_GUEST_ECN)))
+			offload |= TUN_F_TSO_ECN;
+		if (features & (1ULL << VIRTIO_NET_F_GUEST_UFO))
+			offload |= TUN_F_UFO;
+	}
+
+	if (offload != 0) {
+		/* Check if our kernel supports TUNSETOFFLOAD */
+		if (ioctl(fd, TUNSETOFFLOAD, 0) != 0 && errno == EINVAL) {
+			PMD_DRV_LOG(ERR, "Kernel does't support TUNSETOFFLOAD\n");
+			return -ENOTSUP;
+		}
+
+		if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) {
+			offload &= ~TUN_F_UFO;
+			if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) {
+				PMD_DRV_LOG(ERR, "TUNSETOFFLOAD ioctl() failed: %s\n",
+					strerror(errno));
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
 
 int
 vhost_kernel_open_tap(char **p_ifname, int hdr_size, int req_mq,
-			 const char *mac)
+			 const char *mac, uint64_t features)
 {
 	unsigned int tap_features;
 	int sndbuf = INT_MAX;
 	struct ifreq ifr;
 	int tapfd;
-	unsigned int offload =
-			TUN_F_CSUM |
-			TUN_F_TSO4 |
-			TUN_F_TSO6 |
-			TUN_F_TSO_ECN |
-			TUN_F_UFO;
 
 	/* TODO:
 	 * 1. verify we can get/set vnet_hdr_len, tap_probe_vnet_hdr_len
@@ -90,13 +124,7 @@ vhost_kernel_open_tap(char **p_ifname, int hdr_size, int req_mq,
 		goto error;
 	}
 
-	/* TODO: before set the offload capabilities, we'd better (1) check
-	 * negotiated features to see if necessary to offload; (2) query tap
-	 * to see if it supports the offload capabilities.
-	 */
-	if (ioctl(tapfd, TUNSETOFFLOAD, offload) != 0)
-		PMD_DRV_LOG(ERR, "TUNSETOFFLOAD ioctl() failed: %s",
-			   strerror(errno));
+	vhost_kernel_tap_set_offload(tapfd, features);
 
 	memset(&ifr, 0, sizeof(ifr));
 	ifr.ifr_hwaddr.sa_family = ARPHRD_ETHER;
diff --git a/drivers/net/virtio/virtio_user/vhost_kernel_tap.h b/drivers/net/virtio/virtio_user/vhost_kernel_tap.h
index 01a026f5..e0e95b4f 100644
--- a/drivers/net/virtio/virtio_user/vhost_kernel_tap.h
+++ b/drivers/net/virtio/virtio_user/vhost_kernel_tap.h
@@ -36,4 +36,4 @@
 #define PATH_NET_TUN	"/dev/net/tun"
 
 int vhost_kernel_open_tap(char **p_ifname, int hdr_size, int req_mq,
-			 const char *mac);
+			 const char *mac, uint64_t features);
diff --git a/drivers/net/virtio/virtio_user/vhost_user.c b/drivers/net/virtio/virtio_user/vhost_user.c
index ef6e43df..2c6eba0a 100644
--- a/drivers/net/virtio/virtio_user/vhost_user.c
+++ b/drivers/net/virtio/virtio_user/vhost_user.c
@@ -11,6 +11,9 @@
 #include <string.h>
 #include <errno.h>
 
+#include <rte_fbarray.h>
+#include <rte_eal_memconfig.h>
+
 #include "vhost.h"
 #include "virtio_user_dev.h"
 
@@ -121,133 +124,103 @@ fail:
 	return -1;
 }
 
-struct hugepage_file_info {
-	uint64_t addr;            /**< virtual addr */
-	size_t   size;            /**< the file size */
-	char     path[PATH_MAX];  /**< path to backing file */
+struct walk_arg {
+	struct vhost_memory *vm;
+	int *fds;
+	int region_nr;
 };
 
-/* Two possible options:
- * 1. Match HUGEPAGE_INFO_FMT to find the file storing struct hugepage_file
- * array. This is simple but cannot be used in secondary process because
- * secondary process will close and munmap that file.
- * 2. Match HUGEFILE_FMT to find hugepage files directly.
- *
- * We choose option 2.
- */
 static int
-get_hugepage_file_info(struct hugepage_file_info huges[], int max)
+update_memory_region(const struct rte_memseg_list *msl __rte_unused,
+		const struct rte_memseg *ms, void *arg)
 {
-	int idx, k, exist;
-	FILE *f;
-	char buf[BUFSIZ], *tmp, *tail;
-	char *str_underline, *str_start;
-	int huge_index;
-	uint64_t v_start, v_end;
-	struct stat stats;
-
-	f = fopen("/proc/self/maps", "r");
-	if (!f) {
-		PMD_DRV_LOG(ERR, "cannot open /proc/self/maps");
+	struct walk_arg *wa = arg;
+	struct vhost_memory_region *mr;
+	uint64_t start_addr, end_addr;
+	size_t offset;
+	int i, fd;
+
+	fd = rte_memseg_get_fd_thread_unsafe(ms);
+	if (fd < 0) {
+		PMD_DRV_LOG(ERR, "Failed to get fd, ms=%p rte_errno=%d",
+			ms, rte_errno);
 		return -1;
 	}
 
-	idx = 0;
-	while (fgets(buf, sizeof(buf), f) != NULL) {
-		if (sscanf(buf, "%" PRIx64 "-%" PRIx64, &v_start, &v_end) < 2) {
-			PMD_DRV_LOG(ERR, "Failed to parse address");
-			goto error;
-		}
+	if (rte_memseg_get_fd_offset_thread_unsafe(ms, &offset) < 0) {
+		PMD_DRV_LOG(ERR, "Failed to get offset, ms=%p rte_errno=%d",
+			ms, rte_errno);
+		return -1;
+	}
 
-		tmp = strchr(buf, ' ') + 1; /** skip address */
-		tmp = strchr(tmp, ' ') + 1; /** skip perm */
-		tmp = strchr(tmp, ' ') + 1; /** skip offset */
-		tmp = strchr(tmp, ' ') + 1; /** skip dev */
-		tmp = strchr(tmp, ' ') + 1; /** skip inode */
-		while (*tmp == ' ')         /** skip spaces */
-			tmp++;
-		tail = strrchr(tmp, '\n');  /** remove newline if exists */
-		if (tail)
-			*tail = '\0';
-
-		/* Match HUGEFILE_FMT, aka "%s/%smap_%d",
-		 * which is defined in eal_filesystem.h
-		 */
-		str_underline = strrchr(tmp, '_');
-		if (!str_underline)
-			continue;
+	start_addr = (uint64_t)(uintptr_t)ms->addr;
+	end_addr = start_addr + ms->len;
 
-		str_start = str_underline - strlen("map");
-		if (str_start < tmp)
+	for (i = 0; i < wa->region_nr; i++) {
+		if (wa->fds[i] != fd)
 			continue;
 
-		if (sscanf(str_start, "map_%d", &huge_index) != 1)
-			continue;
+		mr = &wa->vm->regions[i];
 
-		/* skip duplicated file which is mapped to different regions */
-		for (k = 0, exist = -1; k < idx; ++k) {
-			if (!strcmp(huges[k].path, tmp)) {
-				exist = k;
-				break;
-			}
-		}
-		if (exist >= 0)
-			continue;
+		if (mr->userspace_addr + mr->memory_size < end_addr)
+			mr->memory_size = end_addr - mr->userspace_addr;
 
-		if (idx >= max) {
-			PMD_DRV_LOG(ERR, "Exceed maximum of %d", max);
-			goto error;
+		if (mr->userspace_addr > start_addr) {
+			mr->userspace_addr = start_addr;
+			mr->guest_phys_addr = start_addr;
 		}
 
-		huges[idx].addr = v_start;
-		huges[idx].size = v_end - v_start; /* To be corrected later */
-		snprintf(huges[idx].path, PATH_MAX, "%s", tmp);
-		idx++;
+		if (mr->mmap_offset > offset)
+			mr->mmap_offset = offset;
+
+		PMD_DRV_LOG(DEBUG, "index=%d fd=%d offset=0x%" PRIx64
+			" addr=0x%" PRIx64 " len=%" PRIu64, i, fd,
+			mr->mmap_offset, mr->userspace_addr,
+			mr->memory_size);
+
+		return 0;
 	}
 
-	/* correct the size for files who have many regions */
-	for (k = 0; k < idx; ++k) {
-		if (stat(huges[k].path, &stats) < 0) {
-			PMD_DRV_LOG(ERR, "Failed to stat %s, %s\n",
-				    huges[k].path, strerror(errno));
-			continue;
-		}
-		huges[k].size = stats.st_size;
-		PMD_DRV_LOG(INFO, "file %s, size %zx\n",
-			    huges[k].path, huges[k].size);
+	if (i >= VHOST_MEMORY_MAX_NREGIONS) {
+		PMD_DRV_LOG(ERR, "Too many memory regions");
+		return -1;
 	}
 
-	fclose(f);
-	return idx;
+	mr = &wa->vm->regions[i];
+	wa->fds[i] = fd;
 
-error:
-	fclose(f);
-	return -1;
+	mr->guest_phys_addr = start_addr;
+	mr->userspace_addr = start_addr;
+	mr->memory_size = ms->len;
+	mr->mmap_offset = offset;
+
+	PMD_DRV_LOG(DEBUG, "index=%d fd=%d offset=0x%" PRIx64
+		" addr=0x%" PRIx64 " len=%" PRIu64, i, fd,
+		mr->mmap_offset, mr->userspace_addr,
+		mr->memory_size);
+
+	wa->region_nr++;
+
+	return 0;
 }
 
 static int
 prepare_vhost_memory_user(struct vhost_user_msg *msg, int fds[])
 {
-	int i, num;
-	struct hugepage_file_info huges[VHOST_MEMORY_MAX_NREGIONS];
-	struct vhost_memory_region *mr;
+	struct walk_arg wa;
 
-	num = get_hugepage_file_info(huges, VHOST_MEMORY_MAX_NREGIONS);
-	if (num < 0) {
-		PMD_INIT_LOG(ERR, "Failed to prepare memory for vhost-user");
-		return -1;
-	}
+	wa.region_nr = 0;
+	wa.vm = &msg->payload.memory;
+	wa.fds = fds;
 
-	for (i = 0; i < num; ++i) {
-		mr = &msg->payload.memory.regions[i];
-		mr->guest_phys_addr = huges[i].addr; /* use vaddr! */
-		mr->userspace_addr = huges[i].addr;
-		mr->memory_size = huges[i].size;
-		mr->mmap_offset = 0;
-		fds[i] = open(huges[i].path, O_RDWR);
-	}
+	/*
+	 * The memory lock has already been taken by memory subsystem
+	 * or virtio_user_start_device().
+	 */
+	if (rte_memseg_walk_thread_unsafe(update_memory_region, &wa) < 0)
+		return -1;
 
-	msg->payload.memory.nregions = num;
+	msg->payload.memory.nregions = wa.region_nr;
 	msg->payload.memory.padding = 0;
 
 	return 0;
@@ -280,7 +253,7 @@ vhost_user_sock(struct virtio_user_dev *dev,
 	int need_reply = 0;
 	int fds[VHOST_MEMORY_MAX_NREGIONS];
 	int fd_num = 0;
-	int i, len;
+	int len;
 	int vhostfd = dev->vhostfd;
 
 	RTE_SET_USED(m);
@@ -364,10 +337,6 @@ vhost_user_sock(struct virtio_user_dev *dev,
 		return -1;
 	}
 
-	if (req == VHOST_USER_SET_MEM_TABLE)
-		for (i = 0; i < fd_num; ++i)
-			close(fds[i]);
-
 	if (need_reply) {
 		if (vhost_user_read(vhostfd, &msg) < 0) {
 			PMD_DRV_LOG(ERR, "Received msg failed: %s",
@@ -497,7 +466,7 @@ vhost_user_enable_queue_pair(struct virtio_user_dev *dev,
 	return 0;
 }
 
-struct virtio_user_backend_ops ops_user = {
+struct virtio_user_backend_ops virtio_ops_user = {
 	.setup = vhost_user_setup,
 	.send_request = vhost_user_sock,
 	.enable_qp = vhost_user_enable_queue_pair
diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c b/drivers/net/virtio/virtio_user/virtio_user_dev.c
index 7df600b0..b4997ee3 100644
--- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
+++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
@@ -13,6 +13,8 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#include <rte_eal_memconfig.h>
+
 #include "vhost.h"
 #include "virtio_user_dev.h"
 #include "../virtio_ethdev.h"
@@ -109,9 +111,24 @@ is_vhost_user_by_type(const char *path)
 int
 virtio_user_start_device(struct virtio_user_dev *dev)
 {
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 	uint64_t features;
 	int ret;
 
+	/*
+	 * XXX workaround!
+	 *
+	 * We need to make sure that the locks will be
+	 * taken in the correct order to avoid deadlocks.
+	 *
+	 * Before releasing this lock, this thread should
+	 * not trigger any memory hotplug events.
+	 *
+	 * This is a temporary workaround, and should be
+	 * replaced when we get proper supports from the
+	 * memory subsystem in the future.
+	 */
+	rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
 	pthread_mutex_lock(&dev->mutex);
 
 	if (is_vhost_user_by_type(dev->path) && dev->vhostfd < 0)
@@ -152,10 +169,12 @@ virtio_user_start_device(struct virtio_user_dev *dev)
 
 	dev->started = true;
 	pthread_mutex_unlock(&dev->mutex);
+	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
 
 	return 0;
 error:
 	pthread_mutex_unlock(&dev->mutex);
+	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
 	/* TODO: free resource here or caller to check */
 	return -1;
 }
@@ -282,8 +301,14 @@ virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused,
 						 void *arg)
 {
 	struct virtio_user_dev *dev = arg;
+	struct rte_memseg_list *msl;
 	uint16_t i;
 
+	/* ignore externally allocated memory */
+	msl = rte_mem_virt2memseg_list(addr);
+	if (msl->external)
+		return;
+
 	pthread_mutex_lock(&dev->mutex);
 
 	if (dev->started == false)
@@ -319,12 +344,12 @@ virtio_user_dev_setup(struct virtio_user_dev *dev)
 			PMD_DRV_LOG(ERR, "Server mode doesn't support vhost-kernel!");
 			return -1;
 		}
-		dev->ops = &ops_user;
+		dev->ops = &virtio_ops_user;
 	} else {
 		if (is_vhost_user_by_type(dev->path)) {
-			dev->ops = &ops_user;
+			dev->ops = &virtio_ops_user;
 		} else {
-			dev->ops = &ops_kernel;
+			dev->ops = &virtio_ops_kernel;
 
 			dev->vhostfds = malloc(dev->max_queue_pairs *
 					       sizeof(int));
@@ -530,13 +555,11 @@ virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs)
 	/* Server mode can't enable queue pairs if vhostfd is invalid,
 	 * always return 0 in this case.
 	 */
-	if (dev->vhostfd >= 0) {
+	if (!dev->is_server || dev->vhostfd >= 0) {
 		for (i = 0; i < q_pairs; ++i)
 			ret |= dev->ops->enable_qp(dev, i, 1);
 		for (i = q_pairs; i < dev->max_queue_pairs; ++i)
 			ret |= dev->ops->enable_qp(dev, i, 0);
-	} else if (!dev->is_server) {
-		ret = ~0;
 	}
 	dev->queue_pairs = q_pairs;
 
diff --git a/drivers/net/virtio/virtio_user_ethdev.c b/drivers/net/virtio/virtio_user_ethdev.c
index 525d16ca..b51cbc85 100644
--- a/drivers/net/virtio/virtio_user_ethdev.c
+++ b/drivers/net/virtio/virtio_user_ethdev.c
@@ -422,7 +422,6 @@ virtio_user_eth_dev_alloc(struct rte_vdev_device *vdev)
 	if (!dev) {
 		PMD_INIT_LOG(ERR, "malloc virtio_user_dev failed");
 		rte_eth_dev_release_port(eth_dev);
-		rte_free(hw);
 		return NULL;
 	}
 
@@ -449,7 +448,6 @@ virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev)
 	struct virtio_hw *hw = data->dev_private;
 
 	rte_free(hw->virtio_user_dev);
-	rte_free(hw);
 	rte_eth_dev_release_port(eth_dev);
 }
 
@@ -637,7 +635,6 @@ end:
 	return ret;
 }
 
-/** Called by rte_eth_dev_detach() */
 static int
 virtio_user_pmd_remove(struct rte_vdev_device *vdev)
 {
@@ -662,7 +659,6 @@ virtio_user_pmd_remove(struct rte_vdev_device *vdev)
 	dev = hw->virtio_user_dev;
 	virtio_user_dev_uninit(dev);
 
-	rte_free(eth_dev->data->dev_private);
 	rte_eth_dev_release_port(eth_dev);
 
 	return 0;
diff --git a/drivers/net/vmxnet3/meson.build b/drivers/net/vmxnet3/meson.build
new file mode 100644
index 00000000..a92bd286
--- /dev/null
+++ b/drivers/net/vmxnet3/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
+
+allow_experimental_apis = true
+sources += files(
+	'vmxnet3_ethdev.c',
+	'vmxnet3_rxtx.c',
+)
+
+error_cflags = [
+		'-Wno-unused-parameter', '-Wno-unused-value',
+                '-Wno-strict-aliasing', '-Wno-format-extra-args',
+]
+foreach flag: error_cflags
+        if cc.has_argument(flag)
+                cflags += flag
+        endif
+endforeach
diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.c b/drivers/net/vmxnet3/vmxnet3_ethdev.c
index 2613cd13..41bcd450 100644
--- a/drivers/net/vmxnet3/vmxnet3_ethdev.c
+++ b/drivers/net/vmxnet3/vmxnet3_ethdev.c
@@ -57,8 +57,7 @@
 	 DEV_RX_OFFLOAD_UDP_CKSUM |	\
 	 DEV_RX_OFFLOAD_TCP_CKSUM |	\
 	 DEV_RX_OFFLOAD_TCP_LRO |	\
-	 DEV_RX_OFFLOAD_JUMBO_FRAME |   \
-	 DEV_RX_OFFLOAD_CRC_STRIP)
+	 DEV_RX_OFFLOAD_JUMBO_FRAME)
 
 static int eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev);
 static int eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev);
@@ -78,6 +77,7 @@ static int vmxnet3_dev_link_update(struct rte_eth_dev *dev,
 static void vmxnet3_hw_stats_save(struct vmxnet3_hw *hw);
 static int vmxnet3_dev_stats_get(struct rte_eth_dev *dev,
 				  struct rte_eth_stats *stats);
+static void vmxnet3_dev_stats_reset(struct rte_eth_dev *dev);
 static int vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev,
 					struct rte_eth_xstat_name *xstats,
 					unsigned int n);
@@ -120,6 +120,7 @@ static const struct eth_dev_ops vmxnet3_eth_dev_ops = {
 	.stats_get            = vmxnet3_dev_stats_get,
 	.xstats_get_names     = vmxnet3_dev_xstats_get_names,
 	.xstats_get           = vmxnet3_dev_xstats_get,
+	.stats_reset          = vmxnet3_dev_stats_reset,
 	.mac_addr_set         = vmxnet3_mac_addr_set,
 	.dev_infos_get        = vmxnet3_dev_info_get,
 	.dev_supported_ptypes_get = vmxnet3_dev_supported_ptypes_get,
@@ -160,8 +161,8 @@ gpa_zone_reserve(struct rte_eth_dev *dev, uint32_t size,
 	char z_name[RTE_MEMZONE_NAMESIZE];
 	const struct rte_memzone *mz;
 
-	snprintf(z_name, sizeof(z_name), "%s_%d_%s",
-		 dev->device->driver->name, dev->data->port_id, post_string);
+	snprintf(z_name, sizeof(z_name), "eth_p%d_%s",
+			dev->data->port_id, post_string);
 
 	mz = rte_memzone_lookup(z_name);
 	if (!reuse) {
@@ -335,6 +336,10 @@ eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev)
 	memset(hw->saved_tx_stats, 0, sizeof(hw->saved_tx_stats));
 	memset(hw->saved_rx_stats, 0, sizeof(hw->saved_rx_stats));
 
+	/* clear snapshot stats */
+	memset(hw->snapshot_tx_stats, 0, sizeof(hw->snapshot_tx_stats));
+	memset(hw->snapshot_rx_stats, 0, sizeof(hw->snapshot_rx_stats));
+
 	/* set the initial link status */
 	memset(&link, 0, sizeof(link));
 	link.link_duplex = ETH_LINK_FULL_DUPLEX;
@@ -363,9 +368,6 @@ eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev)
 	eth_dev->tx_pkt_burst = NULL;
 	eth_dev->tx_pkt_prepare = NULL;
 
-	rte_free(eth_dev->data->mac_addrs);
-	eth_dev->data->mac_addrs = NULL;
-
 	return 0;
 }
 
@@ -890,7 +892,49 @@ vmxnet3_hw_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q,
 	VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxError, res);
 	VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxOutOfBuf, res);
 
-#undef VMXNET3_UPDATE_RX_STATS
+#undef VMXNET3_UPDATE_RX_STAT
+}
+
+static void
+vmxnet3_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q,
+					struct UPT1_TxStats *res)
+{
+		vmxnet3_hw_tx_stats_get(hw, q, res);
+
+#define VMXNET3_REDUCE_SNAPSHOT_TX_STAT(h, i, f, r)	\
+		((r)->f -= (h)->snapshot_tx_stats[(i)].f)
+
+	VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, ucastPktsTxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, mcastPktsTxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, bcastPktsTxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, ucastBytesTxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, mcastBytesTxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, bcastBytesTxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, pktsTxError, res);
+	VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, pktsTxDiscard, res);
+
+#undef VMXNET3_REDUCE_SNAPSHOT_TX_STAT
+}
+
+static void
+vmxnet3_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q,
+					struct UPT1_RxStats *res)
+{
+		vmxnet3_hw_rx_stats_get(hw, q, res);
+
+#define VMXNET3_REDUCE_SNAPSHOT_RX_STAT(h, i, f, r)	\
+		((r)->f -= (h)->snapshot_rx_stats[(i)].f)
+
+	VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, ucastPktsRxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, mcastPktsRxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, bcastPktsRxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, ucastBytesRxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, mcastBytesRxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, bcastBytesRxOK, res);
+	VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, pktsRxError, res);
+	VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, pktsRxOutOfBuf, res);
+
+#undef VMXNET3_REDUCE_SNAPSHOT_RX_STAT
 }
 
 static void
@@ -1005,7 +1049,7 @@ vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 
 	RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES);
 	for (i = 0; i < hw->num_tx_queues; i++) {
-		vmxnet3_hw_tx_stats_get(hw, i, &txStats);
+		vmxnet3_tx_stats_get(hw, i, &txStats);
 
 		stats->q_opackets[i] = txStats.ucastPktsTxOK +
 			txStats.mcastPktsTxOK +
@@ -1022,7 +1066,7 @@ vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 
 	RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_RX_QUEUES);
 	for (i = 0; i < hw->num_rx_queues; i++) {
-		vmxnet3_hw_rx_stats_get(hw, i, &rxStats);
+		vmxnet3_rx_stats_get(hw, i, &rxStats);
 
 		stats->q_ipackets[i] = rxStats.ucastPktsRxOK +
 			rxStats.mcastPktsRxOK +
@@ -1044,6 +1088,30 @@ vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 }
 
 static void
+vmxnet3_dev_stats_reset(struct rte_eth_dev *dev)
+{
+	unsigned int i;
+	struct vmxnet3_hw *hw = dev->data->dev_private;
+	struct UPT1_TxStats txStats;
+	struct UPT1_RxStats rxStats;
+
+	VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
+
+	RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES);
+
+	for (i = 0; i < hw->num_tx_queues; i++) {
+		vmxnet3_hw_tx_stats_get(hw, i, &txStats);
+		memcpy(&hw->snapshot_tx_stats[i], &txStats,
+			sizeof(hw->snapshot_tx_stats[0]));
+	}
+	for (i = 0; i < hw->num_rx_queues; i++) {
+		vmxnet3_hw_rx_stats_get(hw, i, &rxStats);
+		memcpy(&hw->snapshot_rx_stats[i], &rxStats,
+			sizeof(hw->snapshot_rx_stats[0]));
+	}
+}
+
+static void
 vmxnet3_dev_info_get(struct rte_eth_dev *dev __rte_unused,
 		     struct rte_eth_dev_info *dev_info)
 {
diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.h b/drivers/net/vmxnet3/vmxnet3_ethdev.h
index d3f2b352..5bc3a84c 100644
--- a/drivers/net/vmxnet3/vmxnet3_ethdev.h
+++ b/drivers/net/vmxnet3/vmxnet3_ethdev.h
@@ -98,6 +98,9 @@ struct vmxnet3_hw {
 #define VMXNET3_VFT_TABLE_SIZE     (VMXNET3_VFT_SIZE * sizeof(uint32_t))
 	UPT1_TxStats	      saved_tx_stats[VMXNET3_MAX_TX_QUEUES];
 	UPT1_RxStats	      saved_rx_stats[VMXNET3_MAX_RX_QUEUES];
+
+	UPT1_TxStats          snapshot_tx_stats[VMXNET3_MAX_TX_QUEUES];
+	UPT1_RxStats          snapshot_rx_stats[VMXNET3_MAX_RX_QUEUES];
 };
 
 #define VMXNET3_REV_3		2		/* Vmxnet3 Rev. 3 */
diff --git a/drivers/raw/dpaa2_cmdif/Makefile b/drivers/raw/dpaa2_cmdif/Makefile
index 9b863dda..9bd5ff22 100644
--- a/drivers/raw/dpaa2_cmdif/Makefile
+++ b/drivers/raw/dpaa2_cmdif/Makefile
@@ -21,10 +21,11 @@ LDLIBS += -lrte_eal
 LDLIBS += -lrte_kvargs
 LDLIBS += -lrte_mempool_dpaa2
 LDLIBS += -lrte_rawdev
+LDLIBS += -lrte_common_dpaax
 
 EXPORT_MAP := rte_pmd_dpaa2_cmdif_version.map
 
-LIBABIVER := 1
+LIBABIVER := 2
 
 #
 # all source are stored in SRCS-y
diff --git a/drivers/raw/dpaa2_cmdif/meson.build b/drivers/raw/dpaa2_cmdif/meson.build
index 1d146872..37bb24a1 100644
--- a/drivers/raw/dpaa2_cmdif/meson.build
+++ b/drivers/raw/dpaa2_cmdif/meson.build
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright 2018 NXP
 
+version = 2
+
 build = dpdk_conf.has('RTE_LIBRTE_DPAA2_MEMPOOL')
 deps += ['rawdev', 'mempool_dpaa2', 'bus_vdev']
 sources = files('dpaa2_cmdif.c')
diff --git a/drivers/raw/dpaa2_qdma/Makefile b/drivers/raw/dpaa2_qdma/Makefile
index d88809ea..bdd99c97 100644
--- a/drivers/raw/dpaa2_qdma/Makefile
+++ b/drivers/raw/dpaa2_qdma/Makefile
@@ -22,10 +22,11 @@ LDLIBS += -lrte_mempool
 LDLIBS += -lrte_mempool_dpaa2
 LDLIBS += -lrte_rawdev
 LDLIBS += -lrte_ring
+LDLIBS += -lrte_common_dpaax
 
 EXPORT_MAP := rte_pmd_dpaa2_qdma_version.map
 
-LIBABIVER := 1
+LIBABIVER := 2
 
 #
 # all source are stored in SRCS-y
diff --git a/drivers/raw/dpaa2_qdma/dpaa2_qdma.c b/drivers/raw/dpaa2_qdma/dpaa2_qdma.c
index 2787d302..f474442d 100644
--- a/drivers/raw/dpaa2_qdma/dpaa2_qdma.c
+++ b/drivers/raw/dpaa2_qdma/dpaa2_qdma.c
@@ -34,10 +34,10 @@ static struct qdma_hw_queue_list qdma_queue_list
 	= TAILQ_HEAD_INITIALIZER(qdma_queue_list);
 
 /* QDMA Virtual Queues */
-struct qdma_virt_queue *qdma_vqs;
+static struct qdma_virt_queue *qdma_vqs;
 
 /* QDMA per core data */
-struct qdma_per_core_info qdma_core_info[RTE_MAX_LCORE];
+static struct qdma_per_core_info qdma_core_info[RTE_MAX_LCORE];
 
 static struct qdma_hw_queue *
 alloc_hw_queue(uint32_t lcore_id)
@@ -805,7 +805,7 @@ dpaa2_dpdmai_dev_uninit(struct rte_rawdev *rawdev)
 		DPAA2_QDMA_ERR("dmdmai disable failed");
 
 	/* Set up the DQRR storage for Rx */
-	for (i = 0; i < DPDMAI_PRIO_NUM; i++) {
+	for (i = 0; i < dpdmai_dev->num_queues; i++) {
 		struct dpaa2_queue *rxq = &(dpdmai_dev->rx_queue[i]);
 
 		if (rxq->q_storage) {
@@ -856,17 +856,17 @@ dpaa2_dpdmai_dev_init(struct rte_rawdev *rawdev, int dpdmai_id)
 			       ret);
 		goto init_err;
 	}
-	dpdmai_dev->num_queues = attr.num_of_priorities;
+	dpdmai_dev->num_queues = attr.num_of_queues;
 
 	/* Set up Rx Queues */
-	for (i = 0; i < attr.num_of_priorities; i++) {
+	for (i = 0; i < dpdmai_dev->num_queues; i++) {
 		struct dpaa2_queue *rxq;
 
 		memset(&rx_queue_cfg, 0, sizeof(struct dpdmai_rx_queue_cfg));
 		ret = dpdmai_set_rx_queue(&dpdmai_dev->dpdmai,
 					  CMD_PRI_LOW,
 					  dpdmai_dev->token,
-					  i, &rx_queue_cfg);
+					  i, 0, &rx_queue_cfg);
 		if (ret) {
 			DPAA2_QDMA_ERR("Setting Rx queue failed with err: %d",
 				       ret);
@@ -893,9 +893,9 @@ dpaa2_dpdmai_dev_init(struct rte_rawdev *rawdev, int dpdmai_id)
 	}
 
 	/* Get Rx and Tx queues FQID's */
-	for (i = 0; i < DPDMAI_PRIO_NUM; i++) {
+	for (i = 0; i < dpdmai_dev->num_queues; i++) {
 		ret = dpdmai_get_rx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
-					  dpdmai_dev->token, i, &rx_attr);
+					  dpdmai_dev->token, i, 0, &rx_attr);
 		if (ret) {
 			DPAA2_QDMA_ERR("Reading device failed with err: %d",
 				       ret);
@@ -904,7 +904,7 @@ dpaa2_dpdmai_dev_init(struct rte_rawdev *rawdev, int dpdmai_id)
 		dpdmai_dev->rx_queue[i].fqid = rx_attr.fqid;
 
 		ret = dpdmai_get_tx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
-					  dpdmai_dev->token, i, &tx_attr);
+					  dpdmai_dev->token, i, 0, &tx_attr);
 		if (ret) {
 			DPAA2_QDMA_ERR("Reading device failed with err: %d",
 				       ret);
diff --git a/drivers/raw/dpaa2_qdma/dpaa2_qdma.h b/drivers/raw/dpaa2_qdma/dpaa2_qdma.h
index c6a05780..0cbe9025 100644
--- a/drivers/raw/dpaa2_qdma/dpaa2_qdma.h
+++ b/drivers/raw/dpaa2_qdma/dpaa2_qdma.h
@@ -11,6 +11,8 @@ struct qdma_io_meta;
 #define DPAA2_QDMA_MAX_FLE 3
 #define DPAA2_QDMA_MAX_SDD 2
 
+#define DPAA2_DPDMAI_MAX_QUEUES	8
+
 /** FLE pool size: 3 Frame list + 2 source/destination descriptor */
 #define QDMA_FLE_POOL_SIZE (sizeof(struct qdma_io_meta) + \
 		sizeof(struct qbman_fle) * DPAA2_QDMA_MAX_FLE + \
@@ -142,9 +144,9 @@ struct dpaa2_dpdmai_dev {
 	/** Number of queue in this DPDMAI device */
 	uint8_t num_queues;
 	/** RX queues */
-	struct dpaa2_queue rx_queue[DPDMAI_PRIO_NUM];
+	struct dpaa2_queue rx_queue[DPAA2_DPDMAI_MAX_QUEUES];
 	/** TX queues */
-	struct dpaa2_queue tx_queue[DPDMAI_PRIO_NUM];
+	struct dpaa2_queue tx_queue[DPAA2_DPDMAI_MAX_QUEUES];
 };
 
 #endif /* __DPAA2_QDMA_H__ */
diff --git a/drivers/raw/dpaa2_qdma/meson.build b/drivers/raw/dpaa2_qdma/meson.build
index b6a081f1..2a4b69c1 100644
--- a/drivers/raw/dpaa2_qdma/meson.build
+++ b/drivers/raw/dpaa2_qdma/meson.build
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright 2018 NXP
 
+version = 2
+
 build = dpdk_conf.has('RTE_LIBRTE_DPAA2_MEMPOOL')
 deps += ['rawdev', 'mempool_dpaa2', 'ring']
 sources = files('dpaa2_qdma.c')
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_enumerate.c b/drivers/raw/ifpga_rawdev/base/ifpga_enumerate.c
index f0939dc3..848e5183 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_enumerate.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_enumerate.c
@@ -104,14 +104,14 @@ static struct feature_info port_features[] = {
 		.resource_size = sizeof(struct feature_port_header),
 		.feature_index = PORT_FEATURE_ID_HEADER,
 		.revision_id = PORT_HEADER_REVISION,
-		.ops = &port_hdr_ops,
+		.ops = &ifpga_rawdev_port_hdr_ops,
 	},
 	{
 		.name = PORT_FEATURE_ERR,
 		.resource_size = sizeof(struct feature_port_error),
 		.feature_index = PORT_FEATURE_ID_ERROR,
 		.revision_id = PORT_ERR_REVISION,
-		.ops = &port_error_ops,
+		.ops = &ifpga_rawdev_port_error_ops,
 	},
 	{
 		.name = PORT_FEATURE_UMSG,
@@ -124,14 +124,14 @@ static struct feature_info port_features[] = {
 		.resource_size = sizeof(struct feature_port_uint),
 		.feature_index = PORT_FEATURE_ID_UINT,
 		.revision_id = PORT_UINT_REVISION,
-		.ops = &port_uint_ops,
+		.ops = &ifpga_rawdev_port_uint_ops,
 	},
 	{
 		.name = PORT_FEATURE_STP,
 		.resource_size = PORT_FEATURE_STP_REGION_SIZE,
 		.feature_index = PORT_FEATURE_ID_STP,
 		.revision_id = PORT_STP_REVISION,
-		.ops = &port_stp_ops,
+		.ops = &ifpga_rawdev_port_stp_ops,
 	},
 	{
 		.name = PORT_FEATURE_UAFU,
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
index 7a39a580..4391f2fd 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
@@ -156,10 +156,10 @@ struct fpga_uafu_irq_set {
 
 int port_set_irq(struct ifpga_port_hw *port, u32 feature_id, void *irq_set);
 
-extern struct feature_ops port_hdr_ops;
-extern struct feature_ops port_error_ops;
-extern struct feature_ops port_stp_ops;
-extern struct feature_ops port_uint_ops;
+extern struct feature_ops ifpga_rawdev_port_hdr_ops;
+extern struct feature_ops ifpga_rawdev_port_error_ops;
+extern struct feature_ops ifpga_rawdev_port_stp_ops;
+extern struct feature_ops ifpga_rawdev_port_uint_ops;
 
 /* help functions for feature ops */
 int fpga_msix_set_block(struct feature *feature, unsigned int start,
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_port.c b/drivers/raw/ifpga_rawdev/base/ifpga_port.c
index a962f5b4..8b5668d4 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_port.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_port.c
@@ -326,7 +326,7 @@ static int port_hdr_set_prop(struct feature *feature, struct feature_prop *prop)
 	return -ENOENT;
 }
 
-struct feature_ops port_hdr_ops = {
+struct feature_ops ifpga_rawdev_port_hdr_ops = {
 	.init = port_hdr_init,
 	.uinit = port_hdr_uinit,
 	.get_prop = port_hdr_get_prop,
@@ -354,7 +354,7 @@ static void port_stp_uinit(struct feature *feature)
 	dev_info(NULL, "port stp uinit.\n");
 }
 
-struct feature_ops port_stp_ops = {
+struct feature_ops ifpga_rawdev_port_stp_ops = {
 	.init = port_stp_init,
 	.uinit = port_stp_uinit,
 };
@@ -382,7 +382,7 @@ static void port_uint_uinit(struct feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
-struct feature_ops port_uint_ops = {
+struct feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
 };
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
index 23db562b..9dd1cf59 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
@@ -136,7 +136,7 @@ static int port_error_set_prop(struct feature *feature,
 	return -ENOENT;
 }
 
-struct feature_ops port_error_ops = {
+struct feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
diff --git a/drivers/raw/ifpga_rawdev/base/meson.build b/drivers/raw/ifpga_rawdev/base/meson.build
index cb655352..03f5112c 100644
--- a/drivers/raw/ifpga_rawdev/base/meson.build
+++ b/drivers/raw/ifpga_rawdev/base/meson.build
@@ -18,8 +18,8 @@ sources = [
 ]
 
 error_cflags = ['-Wno-sign-compare', '-Wno-unused-value',
-		'-Wno-format', '-Wno-unused-but-set-variable',
-		'-Wno-strict-aliasing'
+		'-Wno-format', '-Wno-error=format-security',
+		'-Wno-strict-aliasing', '-Wno-unused-but-set-variable'
 ]
 c_args = cflags
 foreach flag: error_cflags
diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
index 3fed0578..32e318fc 100644
--- a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
@@ -542,6 +542,7 @@ ifpga_cfg_probe(struct rte_vdev_device *dev)
 	int port;
 	char *name = NULL;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
+	int ret = -1;
 
 	devargs = dev->device.devargs;
 
@@ -583,7 +584,7 @@ ifpga_cfg_probe(struct rte_vdev_device *dev)
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
-	rte_eal_hotplug_add(RTE_STR(IFPGA_BUS_NAME),
+	ret = rte_eal_hotplug_add(RTE_STR(IFPGA_BUS_NAME),
 			dev_name, devargs->args);
 end:
 	if (kvlist)
@@ -591,7 +592,7 @@ end:
 	if (name)
 		free(name);
 
-	return 0;
+	return ret;
 }
 
 static int
diff --git a/drivers/raw/skeleton_rawdev/skeleton_rawdev.c b/drivers/raw/skeleton_rawdev/skeleton_rawdev.c
index 6518a2d9..d7630fc6 100644
--- a/drivers/raw/skeleton_rawdev/skeleton_rawdev.c
+++ b/drivers/raw/skeleton_rawdev/skeleton_rawdev.c
@@ -32,7 +32,7 @@
 int skeleton_pmd_logtype;
 
 /* Count of instances */
-uint16_t skeldev_init_once;
+static uint16_t skeldev_init_once;
 
 /**< Rawdev Skeleton dummy driver name */
 #define SKELETON_PMD_RAWDEV_NAME rawdev_skeleton
diff --git a/drivers/raw/skeleton_rawdev/skeleton_rawdev_test.c b/drivers/raw/skeleton_rawdev/skeleton_rawdev_test.c
index 3405b898..359c9e29 100644
--- a/drivers/raw/skeleton_rawdev/skeleton_rawdev_test.c
+++ b/drivers/raw/skeleton_rawdev/skeleton_rawdev_test.c
@@ -294,13 +294,14 @@ test_rawdev_attr_set_get(void)
 			      "Attribute (Test1) not set correctly (%" PRIu64 ")",
 			      ret_value);
 
+	free(dummy_value);
+
 	ret_value = 0;
 	ret = rte_rawdev_get_attr(TEST_DEV_ID, "Test2", &ret_value);
 	RTE_TEST_ASSERT_EQUAL(*((int *)(uintptr_t)ret_value), 200,
 			      "Attribute (Test2) not set correctly (%" PRIu64 ")",
 			      ret_value);
 
-	free(dummy_value);
 	return TEST_SUCCESS;
 }