Merge branch 'upstream' into 18.08.x

Change-Id: Ifbda2d554199dd4d11e01f0090881b5f0103ae12
Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>

1020 files changed, 79105 insertions, 25608 deletions

diff --git a/MAINTAINERS b/MAINTAINERS
index 4667fa7f..9fd258fa 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -32,7 +32,8 @@ M: Ferruh Yigit <ferruh.yigit@intel.com>
 T: git://dpdk.org/next/dpdk-next-net
 
 Next-net-intel Tree
-M: Helin Zhang <helin.zhang@intel.com>
+M: Qi Zhang <qi.z.zhang@intel.com>
+M: Beilei Xing <beilei.xing@intel.com>
 T: git://dpdk.org/next/dpdk-next-net-intel
 
 Next-net-mlx Tree
@@ -41,6 +42,7 @@ T: git://dpdk.org/next/dpdk-next-net-mlx
 
 Next-virtio Tree
 M: Maxime Coquelin <maxime.coquelin@redhat.com>
+M: Tiwei Bie <tiwei.bie@intel.com>
 T: git://dpdk.org/next/dpdk-next-virtio
 
 Next-crypto Tree
@@ -122,6 +124,7 @@ M: Neil Horman <nhorman@tuxdriver.com>
 F: lib/librte_compat/
 F: doc/guides/rel_notes/deprecation.rst
 F: devtools/validate-abi.sh
+F: devtools/check-symbol-change.sh
 F: buildtools/check-experimental-syms.sh
 
 Driver information
@@ -175,6 +178,7 @@ F: lib/librte_eal/linuxapp/eal/eal_mem*
 F: lib/librte_eal/bsdapp/eal/eal_mem*
 F: doc/guides/prog_guide/env_abstraction_layer.rst
 F: test/test/test_func_reentrancy.c
+F: test/test/test_fbarray.c
 F: test/test/test_malloc.c
 F: test/test/test_memory.c
 F: test/test/test_memzone.c
@@ -209,13 +213,13 @@ F: test/test/test_bitmap.c
 
 ARM v7
 M: Jan Viktorin <viktorin@rehivetech.com>
-M: Jianbo Liu <jianbo.liu@arm.com>
+M: Gavin Hu <gavin.hu@arm.com>
 F: lib/librte_eal/common/arch/arm/
 F: lib/librte_eal/common/include/arch/arm/
 
 ARM v8
 M: Jerin Jacob <jerin.jacob@caviumnetworks.com>
-M: Jianbo Liu <jianbo.liu@arm.com>
+M: Gavin Hu <gavin.hu@arm.com>
 F: lib/librte_eal/common/include/arch/arm/*_64.h
 F: lib/librte_net/net_crc_neon.h
 F: lib/librte_acl/acl_run_neon.*
@@ -275,6 +279,7 @@ Core Libraries
 
 Memory pool
 M: Olivier Matz <olivier.matz@6wind.com>
+M: Andrew Rybchenko <arybchenko@solarflare.com>
 F: lib/librte_mempool/
 F: drivers/mempool/Makefile
 F: drivers/mempool/ring/
@@ -298,6 +303,8 @@ F: test/test/test_mbuf.c
 
 Ethernet API
 M: Thomas Monjalon <thomas@monjalon.net>
+M: Ferruh Yigit <ferruh.yigit@intel.com>
+M: Andrew Rybchenko <arybchenko@solarflare.com>
 T: git://dpdk.org/next/dpdk-next-net
 F: lib/librte_ethdev/
 F: devtools/test-null.sh
@@ -421,6 +428,10 @@ F: drivers/bus/pci/
 VDEV bus driver
 F: drivers/bus/vdev/
 
+VMBUS bus driver
+M: Stephen Hemminger <sthemmin@microsoft.com>
+F: drivers/bus/vmbus/
+
 
 Networking Drivers
 ------------------
@@ -563,28 +574,27 @@ Intel ifc
 M: Xiao Wang <xiao.w.wang@intel.com>
 T: git://dpdk.org/next/dpdk-next-net-intel
 F: drivers/net/ifc/
-F: doc/guides/nics/ifcvf.rst
-F: doc/guides/nics/features/ifcvf.ini
+F: doc/guides/nics/ifc.rst
+F: doc/guides/nics/features/ifc*.ini
 
 Marvell mvpp2
 M: Tomasz Duszynski <tdu@semihalf.com>
 M: Dmitri Epshtein <dima@marvell.com>
 M: Natalie Samsonov <nsamsono@marvell.com>
-M: Jianbo Liu <jianbo.liu@arm.com>
 F: drivers/net/mvpp2/
 F: doc/guides/nics/mvpp2.rst
 F: doc/guides/nics/features/mvpp2.ini
 
 Mellanox mlx4
-M: Adrien Mazarguil <adrien.mazarguil@6wind.com>
+M: Matan Azrad <matan@mellanox.com>
+M: Shahaf Shuler <shahafs@mellanox.com>
 T: git://dpdk.org/next/dpdk-next-net-mlx
 F: drivers/net/mlx4/
 F: doc/guides/nics/mlx4.rst
 F: doc/guides/nics/features/mlx4.ini
 
 Mellanox mlx5
-M: Adrien Mazarguil <adrien.mazarguil@6wind.com>
-M: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
+M: Shahaf Shuler <shahafs@mellanox.com>
 M: Yongseok Koh <yskoh@mellanox.com>
 T: git://dpdk.org/next/dpdk-next-net-mlx
 F: drivers/net/mlx5/
@@ -597,8 +607,16 @@ F: drivers/net/vdev_netvsc/
 F: doc/guides/nics/vdev_netvsc.rst
 F: doc/guides/nics/features/vdev_netvsc.ini
 
+Microsoft Hyper-V netvsc - EXPERIMENTAL
+M: Stephen Hemminger <sthemmin@microsoft.com>
+M: K. Y. Srinivasan <kys@microsoft.com>
+M: Haiyang Zhang <haiyangz@microsoft.com>
+F: drivers/net/netvsc/
+F: doc/guides/nics/netvsc.rst
+F: doc/guides/nics/features/netvsc.ini
+
 Netcope szedata2
-M: Matej Vido <vido@cesnet.cz>
+M: Jan Remes <remes@netcope.com>
 F: drivers/net/szedata2/
 F: doc/guides/nics/szedata2.rst
 F: doc/guides/nics/features/szedata2.ini
@@ -654,6 +672,8 @@ F: doc/guides/nics/features/vmxnet3.ini
 
 Vhost-user
 M: Maxime Coquelin <maxime.coquelin@redhat.com>
+M: Tiwei Bie <tiwei.bie@intel.com>
+M: Zhihong Wang <zhihong.wang@intel.com>
 T: git://dpdk.org/next/dpdk-next-virtio
 F: lib/librte_vhost/
 F: doc/guides/prog_guide/vhost_lib.rst
@@ -664,8 +684,9 @@ F: doc/guides/sample_app_ug/vhost_scsi.rst
 F: examples/vhost_crypto/
 
 Vhost PMD
-M: Tetsuya Mukawa <mtetsuyah@gmail.com>
 M: Maxime Coquelin <maxime.coquelin@redhat.com>
+M: Tiwei Bie <tiwei.bie@intel.com>
+M: Zhihong Wang <zhihong.wang@intel.com>
 T: git://dpdk.org/next/dpdk-next-virtio
 F: drivers/net/vhost/
 F: doc/guides/nics/vhost.rst
@@ -674,6 +695,7 @@ F: doc/guides/nics/features/vhost.ini
 Virtio PMD
 M: Maxime Coquelin <maxime.coquelin@redhat.com>
 M: Tiwei Bie <tiwei.bie@intel.com>
+M: Zhihong Wang <zhihong.wang@intel.com>
 T: git://dpdk.org/next/dpdk-next-virtio
 F: drivers/net/virtio/
 F: doc/guides/nics/virtio.rst
@@ -727,6 +749,8 @@ Softnic PMD
 M: Jasvinder Singh <jasvinder.singh@intel.com>
 M: Cristian Dumitrescu <cristian.dumitrescu@intel.com>
 F: drivers/net/softnic/
+F: doc/guides/nics/features/softnic.ini
+F: doc/guides/nics/softnic.rst
 
 
 Crypto Drivers
@@ -769,6 +793,7 @@ M: John Griffin <john.griffin@intel.com>
 M: Fiona Trahe <fiona.trahe@intel.com>
 M: Deepak Kumar Jain <deepak.k.jain@intel.com>
 F: drivers/crypto/qat/
+F: drivers/common/qat/
 F: doc/guides/cryptodevs/qat.rst
 F: doc/guides/cryptodevs/features/qat.ini
 
@@ -782,7 +807,6 @@ Marvell Mrvl
 M: Tomasz Duszynski <tdu@semihalf.com>
 M: Dmitri Epshtein <dima@marvell.com>
 M: Natalie Samsonov <nsamsono@marvell.com>
-M: Jianbo Liu <jianbo.liu@arm.com>
 F: drivers/crypto/mvsam/
 F: doc/guides/cryptodevs/mvsam.rst
 F: doc/guides/cryptodevs/features/mvsam.ini
@@ -837,12 +861,29 @@ Compression Drivers
 M: Pablo de Lara <pablo.de.lara.guarch@intel.com>
 T: git://dpdk.org/next/dpdk-next-crypto
 
+Cavium OCTEONTX zipvf
+M: Ashish Gupta <ashish.gupta@cavium.com>
+F: drivers/compress/octeontx/
+F: doc/guides/compressdevs/octeontx.rst
+F: doc/guides/compressdevs/features/octeontx.ini
+
+Intel QuickAssist
+M: Fiona Trahe <fiona.trahe@intel.com>
+F: drivers/compress/qat/
+F: drivers/common/qat/
+
 ISA-L
 M: Lee Daly <lee.daly@intel.com>
 F: drivers/compress/isal/
 F: doc/guides/compressdevs/isal.rst
 F: doc/guides/compressdevs/features/isal.ini
 
+ZLIB
+M: Sunila Sahu <sunila.sahu@caviumnetworks.com>
+F: drivers/compress/zlib/
+F: doc/guides/compressdevs/zlib.rst
+F: doc/guides/compressdevs/features/zlib.ini
+
 
 Eventdev Drivers
 ----------------
@@ -1237,7 +1278,7 @@ M: John McNamara <john.mcnamara@intel.com>
 F: examples/skeleton/
 F: doc/guides/sample_app_ug/skeleton.rst
 
-M: Jijiang Liu <jijiang.liu@intel.com>
+M: Xiaoyun Li <xiaoyun.li@intel.com>
 F: examples/tep_termination/
 
 F: examples/vmdq/
diff --git a/app/pdump/main.c b/app/pdump/main.c
index 6bcf8c49..ac228712 100644
--- a/app/pdump/main.c
+++ b/app/pdump/main.c
@@ -38,8 +38,9 @@
 #define PDUMP_MSIZE_ARG "mbuf-size"
 #define PDUMP_NUM_MBUFS_ARG "total-num-mbufs"
 
-#define VDEV_PCAP "net_pcap_%s_%d,tx_pcap=%s"
-#define VDEV_IFACE "net_pcap_%s_%d,tx_iface=%s"
+#define VDEV_NAME_FMT "net_pcap_%s_%d"
+#define VDEV_PCAP_ARGS_FMT "tx_pcap=%s"
+#define VDEV_IFACE_ARGS_FMT "tx_iface=%s"
 #define TX_STREAM_SIZE 64
 
 #define MP_NAME "pdump_pool_%d"
@@ -570,6 +571,7 @@ create_mp_ring_vdev(void)
 	uint16_t portid;
 	struct pdump_tuples *pt = NULL;
 	struct rte_mempool *mbuf_pool = NULL;
+	char vdev_name[SIZE];
 	char vdev_args[SIZE];
 	char ring_name[SIZE];
 	char mempool_name[SIZE];
@@ -619,17 +621,28 @@ create_mp_ring_vdev(void)
 			}
 
 			/* create vdevs */
+			snprintf(vdev_name, sizeof(vdev_name),
+				 VDEV_NAME_FMT, RX_STR, i);
 			(pt->rx_vdev_stream_type == IFACE) ?
-			snprintf(vdev_args, SIZE, VDEV_IFACE, RX_STR, i,
-			pt->rx_dev) :
-			snprintf(vdev_args, SIZE, VDEV_PCAP, RX_STR, i,
-			pt->rx_dev);
-			if (rte_eth_dev_attach(vdev_args, &portid) < 0) {
+			snprintf(vdev_args, sizeof(vdev_args),
+				 VDEV_IFACE_ARGS_FMT, pt->rx_dev) :
+			snprintf(vdev_args, sizeof(vdev_args),
+				 VDEV_PCAP_ARGS_FMT, pt->rx_dev);
+			if (rte_eal_hotplug_add("vdev", vdev_name,
+						vdev_args) < 0) {
 				cleanup_rings();
 				rte_exit(EXIT_FAILURE,
 					"vdev creation failed:%s:%d\n",
 					__func__, __LINE__);
 			}
+			if (rte_eth_dev_get_port_by_name(vdev_name,
+							 &portid) != 0) {
+				rte_eal_hotplug_remove("vdev", vdev_name);
+				cleanup_rings();
+				rte_exit(EXIT_FAILURE,
+					"cannot find added vdev %s:%s:%d\n",
+					vdev_name, __func__, __LINE__);
+			}
 			pt->rx_vdev_id = portid;
 
 			/* configure vdev */
@@ -638,18 +651,29 @@ create_mp_ring_vdev(void)
 			if (pt->single_pdump_dev)
 				pt->tx_vdev_id = portid;
 			else {
-				(pt->tx_vdev_stream_type == IFACE) ?
-				snprintf(vdev_args, SIZE, VDEV_IFACE, TX_STR, i,
-				pt->tx_dev) :
-				snprintf(vdev_args, SIZE, VDEV_PCAP, TX_STR, i,
-				pt->tx_dev);
-				if (rte_eth_dev_attach(vdev_args,
-							&portid) < 0) {
+				snprintf(vdev_name, sizeof(vdev_name),
+					 VDEV_NAME_FMT, TX_STR, i);
+				(pt->rx_vdev_stream_type == IFACE) ?
+				snprintf(vdev_args, sizeof(vdev_args),
+					 VDEV_IFACE_ARGS_FMT, pt->tx_dev) :
+				snprintf(vdev_args, sizeof(vdev_args),
+					 VDEV_PCAP_ARGS_FMT, pt->tx_dev);
+				if (rte_eal_hotplug_add("vdev", vdev_name,
+							vdev_args) < 0) {
 					cleanup_rings();
 					rte_exit(EXIT_FAILURE,
 						"vdev creation failed:"
 						"%s:%d\n", __func__, __LINE__);
 				}
+				if (rte_eth_dev_get_port_by_name(vdev_name,
+						&portid) != 0) {
+					rte_eal_hotplug_remove("vdev",
+							       vdev_name);
+					cleanup_rings();
+					rte_exit(EXIT_FAILURE,
+						"cannot find added vdev %s:%s:%d\n",
+						vdev_name, __func__, __LINE__);
+				}
 				pt->tx_vdev_id = portid;
 
 				/* configure vdev */
@@ -667,17 +691,28 @@ create_mp_ring_vdev(void)
 					rte_strerror(rte_errno));
 			}
 
+			snprintf(vdev_name, sizeof(vdev_name),
+				 VDEV_NAME_FMT, RX_STR, i);
 			(pt->rx_vdev_stream_type == IFACE) ?
-			snprintf(vdev_args, SIZE, VDEV_IFACE, RX_STR, i,
-				pt->rx_dev) :
-			snprintf(vdev_args, SIZE, VDEV_PCAP, RX_STR, i,
-				pt->rx_dev);
-			if (rte_eth_dev_attach(vdev_args, &portid) < 0) {
+			snprintf(vdev_args, sizeof(vdev_args),
+				 VDEV_IFACE_ARGS_FMT, pt->rx_dev) :
+			snprintf(vdev_args, sizeof(vdev_args),
+				 VDEV_PCAP_ARGS_FMT, pt->rx_dev);
+			if (rte_eal_hotplug_add("vdev", vdev_name,
+						vdev_args) < 0) {
 				cleanup_rings();
 				rte_exit(EXIT_FAILURE,
 					"vdev creation failed:%s:%d\n",
 					__func__, __LINE__);
 			}
+			if (rte_eth_dev_get_port_by_name(vdev_name,
+							 &portid) != 0) {
+				rte_eal_hotplug_remove("vdev", vdev_name);
+				cleanup_rings();
+				rte_exit(EXIT_FAILURE,
+					"cannot find added vdev %s:%s:%d\n",
+					vdev_name, __func__, __LINE__);
+			}
 			pt->rx_vdev_id = portid;
 			/* configure vdev */
 			configure_vdev(pt->rx_vdev_id);
@@ -693,16 +728,27 @@ create_mp_ring_vdev(void)
 					rte_strerror(rte_errno));
 			}
 
+			snprintf(vdev_name, sizeof(vdev_name),
+				 VDEV_NAME_FMT, TX_STR, i);
 			(pt->tx_vdev_stream_type == IFACE) ?
-			snprintf(vdev_args, SIZE, VDEV_IFACE, TX_STR, i,
-				pt->tx_dev) :
-			snprintf(vdev_args, SIZE, VDEV_PCAP, TX_STR, i,
-				pt->tx_dev);
-			if (rte_eth_dev_attach(vdev_args, &portid) < 0) {
+			snprintf(vdev_args, sizeof(vdev_args),
+				 VDEV_IFACE_ARGS_FMT, pt->tx_dev) :
+			snprintf(vdev_args, sizeof(vdev_args),
+				 VDEV_PCAP_ARGS_FMT, pt->tx_dev);
+			if (rte_eal_hotplug_add("vdev", vdev_name,
+						vdev_args) < 0) {
 				cleanup_rings();
 				rte_exit(EXIT_FAILURE,
 					"vdev creation failed\n");
 			}
+			if (rte_eth_dev_get_port_by_name(vdev_name,
+							 &portid) != 0) {
+				rte_eal_hotplug_remove("vdev", vdev_name);
+				cleanup_rings();
+				rte_exit(EXIT_FAILURE,
+					"cannot find added vdev %s:%s:%d\n",
+					vdev_name, __func__, __LINE__);
+			}
 			pt->tx_vdev_id = portid;
 
 			/* configure vdev */
diff --git a/app/test-crypto-perf/Makefile b/app/test-crypto-perf/Makefile
index 28a0cd03..78135f38 100644
--- a/app/test-crypto-perf/Makefile
+++ b/app/test-crypto-perf/Makefile
@@ -8,6 +8,7 @@ APP = dpdk-test-crypto-perf
 
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -DALLOW_EXPERIMENTAL_API
+CFLAGS += -O3
 
 # all source are stored in SRCS-y
 SRCS-y := main.c
diff --git a/app/test-crypto-perf/cperf_ops.c b/app/test-crypto-perf/cperf_ops.c
index 8f320099..44808f50 100644
--- a/app/test-crypto-perf/cperf_ops.c
+++ b/app/test-crypto-perf/cperf_ops.c
@@ -514,6 +514,7 @@ cperf_create_session(struct rte_mempool *sess_mp,
 		auth_xform.next = NULL;
 		auth_xform.auth.algo = options->auth_algo;
 		auth_xform.auth.op = options->auth_op;
+		auth_xform.auth.iv.offset = iv_offset;
 
 		/* auth different than null */
 		if (options->auth_algo != RTE_CRYPTO_AUTH_NULL) {
@@ -568,6 +569,8 @@ cperf_create_session(struct rte_mempool *sess_mp,
 		auth_xform.next = NULL;
 		auth_xform.auth.algo = options->auth_algo;
 		auth_xform.auth.op = options->auth_op;
+		auth_xform.auth.iv.offset = iv_offset +
+			cipher_xform.cipher.iv.length;
 
 		/* auth different than null */
 		if (options->auth_algo != RTE_CRYPTO_AUTH_NULL) {
diff --git a/app/test-crypto-perf/cperf_options.h b/app/test-crypto-perf/cperf_options.h
index 350ad7e1..f5bf03c8 100644
--- a/app/test-crypto-perf/cperf_options.h
+++ b/app/test-crypto-perf/cperf_options.h
@@ -76,6 +76,8 @@ struct cperf_options {
 
 	uint32_t pool_sz;
 	uint32_t total_ops;
+	uint32_t headroom_sz;
+	uint32_t tailroom_sz;
 	uint32_t segment_sz;
 	uint32_t test_buffer_size;
 	uint32_t *imix_buffer_sizes;
diff --git a/app/test-crypto-perf/cperf_test_common.c b/app/test-crypto-perf/cperf_test_common.c
index 423782cb..e803dc10 100644
--- a/app/test-crypto-perf/cperf_test_common.c
+++ b/app/test-crypto-perf/cperf_test_common.c
@@ -11,12 +11,15 @@ struct obj_params {
 	uint32_t src_buf_offset;
 	uint32_t dst_buf_offset;
 	uint16_t segment_sz;
+	uint16_t headroom_sz;
+	uint16_t data_len;
 	uint16_t segments_nb;
 };
 
 static void
 fill_single_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp,
-		void *obj, uint32_t mbuf_offset, uint16_t segment_sz)
+		void *obj, uint32_t mbuf_offset, uint16_t segment_sz,
+		uint16_t headroom, uint16_t data_len)
 {
 	uint32_t mbuf_hdr_size = sizeof(struct rte_mbuf);
 
@@ -26,10 +29,10 @@ fill_single_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp,
 	m->buf_iova = rte_mempool_virt2iova(obj) +
 		mbuf_offset + mbuf_hdr_size;
 	m->buf_len = segment_sz;
-	m->data_len = segment_sz;
+	m->data_len = data_len;
 
-	/* No headroom needed for the buffer */
-	m->data_off = 0;
+	/* Use headroom specified for the buffer */
+	m->data_off = headroom;
 
 	/* init some constant fields */
 	m->pool = mp;
@@ -42,7 +45,7 @@ fill_single_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp,
 static void
 fill_multi_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp,
 		void *obj, uint32_t mbuf_offset, uint16_t segment_sz,
-		uint16_t segments_nb)
+		uint16_t headroom, uint16_t data_len, uint16_t segments_nb)
 {
 	uint16_t mbuf_hdr_size = sizeof(struct rte_mbuf);
 	uint16_t remaining_segments = segments_nb;
@@ -57,10 +60,10 @@ fill_multi_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp,
 		m->buf_iova = next_seg_phys_addr;
 		next_seg_phys_addr += mbuf_hdr_size + segment_sz;
 		m->buf_len = segment_sz;
-		m->data_len = segment_sz;
+		m->data_len = data_len;
 
-		/* No headroom needed for the buffer */
-		m->data_off = 0;
+		/* Use headroom specified for the buffer */
+		m->data_off = headroom;
 
 		/* init some constant fields */
 		m->pool = mp;
@@ -99,10 +102,12 @@ mempool_obj_init(struct rte_mempool *mp,
 	op->sym->m_src = m;
 	if (params->segments_nb == 1)
 		fill_single_seg_mbuf(m, mp, obj, params->src_buf_offset,
-				params->segment_sz);
+				params->segment_sz, params->headroom_sz,
+				params->data_len);
 	else
 		fill_multi_seg_mbuf(m, mp, obj, params->src_buf_offset,
-				params->segment_sz, params->segments_nb);
+				params->segment_sz, params->headroom_sz,
+				params->data_len, params->segments_nb);
 
 
 	/* Set destination buffer */
@@ -110,7 +115,8 @@ mempool_obj_init(struct rte_mempool *mp,
 		m = (struct rte_mbuf *) ((uint8_t *) obj +
 				params->dst_buf_offset);
 		fill_single_seg_mbuf(m, mp, obj, params->dst_buf_offset,
-				params->segment_sz);
+				params->segment_sz, params->headroom_sz,
+				params->data_len);
 		op->sym->m_dst = m;
 	} else
 		op->sym->m_dst = NULL;
@@ -172,6 +178,11 @@ cperf_alloc_common_memory(const struct cperf_options *options,
 
 	struct obj_params params = {
 		.segment_sz = options->segment_sz,
+		.headroom_sz = options->headroom_sz,
+		/* Data len = segment size - (headroom + tailroom) */
+		.data_len = options->segment_sz -
+			    options->headroom_sz -
+			    options->tailroom_sz,
 		.segments_nb = segments_nb,
 		.src_buf_offset = crypto_op_total_size_padded,
 		.dst_buf_offset = 0
diff --git a/app/test-crypto-perf/main.c b/app/test-crypto-perf/main.c
index 4ae14390..5c7dadb6 100644
--- a/app/test-crypto-perf/main.c
+++ b/app/test-crypto-perf/main.c
@@ -21,8 +21,6 @@
 #include "cperf_test_verify.h"
 #include "cperf_test_pmd_cyclecount.h"
 
-#define NUM_SESSIONS 2048
-#define SESS_MEMPOOL_CACHE_SIZE 64
 
 const char *cperf_test_type_strs[] = {
 	[CPERF_TEST_TYPE_THROUGHPUT] = "throughput",
@@ -67,6 +65,7 @@ cperf_initialize_cryptodev(struct cperf_options *opts, uint8_t *enabled_cdevs,
 			struct rte_mempool *session_pool_socket[])
 {
 	uint8_t enabled_cdev_count = 0, nb_lcores, cdev_id;
+	uint32_t sessions_needed = 0;
 	unsigned int i, j;
 	int ret;
 
@@ -149,17 +148,62 @@ cperf_initialize_cryptodev(struct cperf_options *opts, uint8_t *enabled_cdevs,
 			.nb_descriptors = opts->nb_descriptors
 		};
 
+		/**
+		 * Device info specifies the min headroom and tailroom
+		 * requirement for the crypto PMD. This need to be honoured
+		 * by the application, while creating mbuf.
+		 */
+		if (opts->headroom_sz < cdev_info.min_mbuf_headroom_req) {
+			/* Update headroom */
+			opts->headroom_sz = cdev_info.min_mbuf_headroom_req;
+		}
+		if (opts->tailroom_sz < cdev_info.min_mbuf_tailroom_req) {
+			/* Update tailroom */
+			opts->tailroom_sz = cdev_info.min_mbuf_tailroom_req;
+		}
+
+		/* Update segment size to include headroom & tailroom */
+		opts->segment_sz += (opts->headroom_sz + opts->tailroom_sz);
+
+		uint32_t dev_max_nb_sess = cdev_info.sym.max_nb_sessions;
+		/*
+		 * Two sessions objects are required for each session
+		 * (one for the header, one for the private data)
+		 */
+		if (!strcmp((const char *)opts->device_type,
+					"crypto_scheduler")) {
+#ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER
+			uint32_t nb_slaves =
+				rte_cryptodev_scheduler_slaves_get(cdev_id,
+								NULL);
+
+			sessions_needed = 2 * enabled_cdev_count *
+				opts->nb_qps * nb_slaves;
+#endif
+		} else
+			sessions_needed = 2 * enabled_cdev_count *
+						opts->nb_qps;
+
+		/*
+		 * A single session is required per queue pair
+		 * in each device
+		 */
+		if (dev_max_nb_sess != 0 && dev_max_nb_sess < opts->nb_qps) {
+			RTE_LOG(ERR, USER1,
+				"Device does not support at least "
+				"%u sessions\n", opts->nb_qps);
+			return -ENOTSUP;
+		}
 		if (session_pool_socket[socket_id] == NULL) {
 			char mp_name[RTE_MEMPOOL_NAMESIZE];
 			struct rte_mempool *sess_mp;
 
 			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
 				"sess_mp_%u", socket_id);
-
 			sess_mp = rte_mempool_create(mp_name,
-						NUM_SESSIONS,
+						sessions_needed,
 						max_sess_size,
-						SESS_MEMPOOL_CACHE_SIZE,
+						0,
 						0, NULL, NULL, NULL,
 						NULL, socket_id,
 						0);
diff --git a/app/test-eventdev/evt_main.c b/app/test-eventdev/evt_main.c
index 57bb9457..a8d304ba 100644
--- a/app/test-eventdev/evt_main.c
+++ b/app/test-eventdev/evt_main.c
@@ -20,29 +20,41 @@ struct evt_test *test;
 static void
 signal_handler(int signum)
 {
-	if (signum == SIGINT || signum == SIGTERM) {
+	int i;
+	static uint8_t once;
+
+	if ((signum == SIGINT || signum == SIGTERM) && !once) {
+		once = true;
 		printf("\nSignal %d received, preparing to exit...\n",
 				signum);
-		/* request all lcores to exit from the main loop */
-		*(int *)test->test_priv = true;
-		rte_wmb();
 
-		rte_eal_mp_wait_lcore();
+		if (test != NULL) {
+			/* request all lcores to exit from the main loop */
+			*(int *)test->test_priv = true;
+			rte_wmb();
+
+			if (test->ops.ethdev_destroy)
+				test->ops.ethdev_destroy(test, &opt);
 
-		if (test->ops.test_result)
-			test->ops.test_result(test, &opt);
+			rte_eal_mp_wait_lcore();
 
-		if (test->ops.eventdev_destroy)
-			test->ops.eventdev_destroy(test, &opt);
+			if (test->ops.test_result)
+				test->ops.test_result(test, &opt);
 
-		if (test->ops.ethdev_destroy)
-			test->ops.ethdev_destroy(test, &opt);
+			if (opt.prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) {
+				RTE_ETH_FOREACH_DEV(i)
+					rte_eth_dev_close(i);
+			}
 
-		if (test->ops.mempool_destroy)
-			test->ops.mempool_destroy(test, &opt);
+			if (test->ops.eventdev_destroy)
+				test->ops.eventdev_destroy(test, &opt);
 
-		if (test->ops.test_destroy)
-			test->ops.test_destroy(test, &opt);
+			if (test->ops.mempool_destroy)
+				test->ops.mempool_destroy(test, &opt);
+
+			if (test->ops.test_destroy)
+				test->ops.test_destroy(test, &opt);
+		}
 
 		/* exit with the expected status */
 		signal(signum, SIG_DFL);
diff --git a/app/test-eventdev/evt_test.h b/app/test-eventdev/evt_test.h
index 7477a325..f07d2c33 100644
--- a/app/test-eventdev/evt_test.h
+++ b/app/test-eventdev/evt_test.h
@@ -77,8 +77,7 @@ void evt_test_dump_names(void);
 
 #define EVT_TEST_REGISTER(nm)                         \
 static struct evt_test_entry _evt_test_entry_ ##nm;   \
-RTE_INIT(evt_test_ ##nm);                             \
-static void evt_test_ ##nm(void)                      \
+RTE_INIT(evt_test_ ##nm)                              \
 {                                                     \
 	_evt_test_entry_ ##nm.test.name = RTE_STR(nm);\
 	memcpy(&_evt_test_entry_ ##nm.test.ops, &nm,  \
diff --git a/app/test-eventdev/test_order_atq.c b/app/test-eventdev/test_order_atq.c
index c57fbbfa..35debcfd 100644
--- a/app/test-eventdev/test_order_atq.c
+++ b/app/test-eventdev/test_order_atq.c
@@ -151,10 +151,14 @@ order_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
 	if (ret)
 		return ret;
 
-	ret = evt_service_setup(opt->dev_id);
-	if (ret) {
-		evt_err("No service lcore found to run event dev.");
-		return ret;
+	if (!evt_has_distributed_sched(opt->dev_id)) {
+		uint32_t service_id;
+		rte_event_dev_service_id_get(opt->dev_id, &service_id);
+		ret = evt_service_setup(service_id);
+		if (ret) {
+			evt_err("No service lcore found to run event dev.");
+			return ret;
+		}
 	}
 
 	ret = rte_event_dev_start(opt->dev_id);
diff --git a/app/test-eventdev/test_order_queue.c b/app/test-eventdev/test_order_queue.c
index f603a023..17f7b984 100644
--- a/app/test-eventdev/test_order_queue.c
+++ b/app/test-eventdev/test_order_queue.c
@@ -164,10 +164,14 @@ order_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
 	if (ret)
 		return ret;
 
-	ret = evt_service_setup(opt->dev_id);
-	if (ret) {
-		evt_err("No service lcore found to run event dev.");
-		return ret;
+	if (!evt_has_distributed_sched(opt->dev_id)) {
+		uint32_t service_id;
+		rte_event_dev_service_id_get(opt->dev_id, &service_id);
+		ret = evt_service_setup(service_id);
+		if (ret) {
+			evt_err("No service lcore found to run event dev.");
+			return ret;
+		}
 	}
 
 	ret = rte_event_dev_start(opt->dev_id);
diff --git a/app/test-eventdev/test_perf_common.c b/app/test-eventdev/test_perf_common.c
index d00f9180..d33cb2cd 100644
--- a/app/test-eventdev/test_perf_common.c
+++ b/app/test-eventdev/test_perf_common.c
@@ -680,13 +680,7 @@ perf_ethdev_setup(struct evt_test *test, struct evt_options *opt)
 			.mq_mode = ETH_MQ_RX_RSS,
 			.max_rx_pkt_len = ETHER_MAX_LEN,
 			.split_hdr_size = 0,
-			.header_split   = 0,
-			.hw_ip_checksum = 0,
-			.hw_vlan_filter = 0,
-			.hw_vlan_strip  = 0,
-			.hw_vlan_extend = 0,
-			.jumbo_frame    = 0,
-			.hw_strip_crc   = 1,
+			.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 		},
 		.rx_adv_conf = {
 			.rss_conf = {
@@ -706,10 +700,23 @@ perf_ethdev_setup(struct evt_test *test, struct evt_options *opt)
 	}
 
 	RTE_ETH_FOREACH_DEV(i) {
+		struct rte_eth_dev_info dev_info;
+		struct rte_eth_conf local_port_conf = port_conf;
+
+		rte_eth_dev_info_get(i, &dev_info);
+
+		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+			dev_info.flow_type_rss_offloads;
+		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+				port_conf.rx_adv_conf.rss_conf.rss_hf) {
+			evt_info("Port %u modified RSS hash function based on hardware support,"
+				"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+				i,
+				port_conf.rx_adv_conf.rss_conf.rss_hf,
+				local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+		}
 
-		if (rte_eth_dev_configure(i, 1, 1,
-					&port_conf)
-				< 0) {
+		if (rte_eth_dev_configure(i, 1, 1, &local_port_conf) < 0) {
 			evt_err("Failed to configure eth port [%d]", i);
 			return -EINVAL;
 		}
@@ -743,7 +750,6 @@ void perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt)
 		RTE_ETH_FOREACH_DEV(i) {
 			rte_event_eth_rx_adapter_stop(i);
 			rte_eth_dev_stop(i);
-			rte_eth_dev_close(i);
 		}
 	}
 }
diff --git a/app/test-eventdev/test_pipeline_common.c b/app/test-eventdev/test_pipeline_common.c
index 719518ff..a54068df 100644
--- a/app/test-eventdev/test_pipeline_common.c
+++ b/app/test-eventdev/test_pipeline_common.c
@@ -223,7 +223,6 @@ pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt)
 			.mq_mode = ETH_MQ_RX_RSS,
 			.max_rx_pkt_len = ETHER_MAX_LEN,
 			.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
-			.ignore_offload_bitfield = 1,
 		},
 		.rx_adv_conf = {
 			.rss_conf = {
@@ -241,16 +240,27 @@ pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt)
 
 	RTE_ETH_FOREACH_DEV(i) {
 		struct rte_eth_dev_info dev_info;
+		struct rte_eth_conf local_port_conf = port_conf;
 
-		memset(&dev_info, 0, sizeof(struct rte_eth_dev_info));
 		rte_eth_dev_info_get(i, &dev_info);
 		mt_state = !(dev_info.tx_offload_capa &
 				DEV_TX_OFFLOAD_MT_LOCKFREE);
 		rx_conf = dev_info.default_rxconf;
 		rx_conf.offloads = port_conf.rxmode.offloads;
 
+		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+			dev_info.flow_type_rss_offloads;
+		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+				port_conf.rx_adv_conf.rss_conf.rss_hf) {
+			evt_info("Port %u modified RSS hash function based on hardware support,"
+				"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+				i,
+				port_conf.rx_adv_conf.rss_conf.rss_hf,
+				local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+		}
+
 		if (rte_eth_dev_configure(i, nb_queues, nb_queues,
-					&port_conf)
+					&local_port_conf)
 				< 0) {
 			evt_err("Failed to configure eth port [%d]\n", i);
 			return -EINVAL;
@@ -467,7 +477,6 @@ pipeline_ethdev_destroy(struct evt_test *test, struct evt_options *opt)
 	RTE_ETH_FOREACH_DEV(i) {
 		rte_event_eth_rx_adapter_stop(i);
 		rte_eth_dev_stop(i);
-		rte_eth_dev_close(i);
 	}
 }
 
diff --git a/app/test-pmd/Makefile b/app/test-pmd/Makefile
index a5a827bb..2b4d604b 100644
--- a/app/test-pmd/Makefile
+++ b/app/test-pmd/Makefile
@@ -13,6 +13,7 @@ APP = testpmd
 CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -Wno-deprecated-declarations
 
 #
 # all source are stored in SRCS-y
@@ -35,8 +36,8 @@ SRCS-y += icmpecho.c
 SRCS-$(CONFIG_RTE_LIBRTE_IEEE1588) += ieee1588fwd.c
 SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_cmd.c
 
-ifeq ($(CONFIG_RTE_LIBRTE_PMD_SOFTNIC)$(CONFIG_RTE_LIBRTE_SCHED),yy)
-SRCS-y += tm.c
+ifeq ($(CONFIG_RTE_LIBRTE_PMD_SOFTNIC), y)
+SRCS-y += softnicfwd.c
 endif
 
 ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y)
diff --git a/app/test-pmd/bpf_cmd.c b/app/test-pmd/bpf_cmd.c
index 584fad90..830bfc13 100644
--- a/app/test-pmd/bpf_cmd.c
+++ b/app/test-pmd/bpf_cmd.c
@@ -19,12 +19,35 @@ static const struct rte_bpf_xsym bpf_xsym[] = {
 	{
 		.name = RTE_STR(stdout),
 		.type = RTE_BPF_XTYPE_VAR,
-		.var = &stdout,
+		.var = {
+			.val = &stdout,
+			.desc = {
+				.type = RTE_BPF_ARG_PTR,
+				.size = sizeof(stdout),
+			},
+		},
 	},
 	{
 		.name = RTE_STR(rte_pktmbuf_dump),
 		.type = RTE_BPF_XTYPE_FUNC,
-		.func = (void *)rte_pktmbuf_dump,
+		.func = {
+			.val = (void *)rte_pktmbuf_dump,
+			.nb_args = 3,
+			.args = {
+				[0] = {
+					.type = RTE_BPF_ARG_RAW,
+					.size = sizeof(uintptr_t),
+				},
+				[1] = {
+					.type = RTE_BPF_ARG_PTR_MBUF,
+					.size = sizeof(struct rte_mbuf),
+				},
+				[2] = {
+					.type = RTE_BPF_ARG_RAW,
+					.size = sizeof(uint32_t),
+				},
+			},
+		},
 	},
 };
 
diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index 27e2aa8c..589121d6 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -781,6 +781,27 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"port tm hierarchy commit (port_id) (clean_on_fail)\n"
 			"	Commit tm hierarchy.\n\n"
 
+			"vxlan ip-version (ipv4|ipv6) vni (vni) udp-src"
+			" (udp-src) udp-dst (udp-dst) ip-src (ip-src) ip-dst"
+			" (ip-dst) eth-src (eth-src) eth-dst (eth-dst)\n"
+			"       Configure the VXLAN encapsulation for flows.\n\n"
+
+			"vxlan-with-vlan ip-version (ipv4|ipv6) vni (vni)"
+			" udp-src (udp-src) udp-dst (udp-dst) ip-src (ip-src)"
+			" ip-dst (ip-dst) vlan-tci (vlan-tci) eth-src (eth-src)"
+			" eth-dst (eth-dst)\n"
+			"       Configure the VXLAN encapsulation for flows.\n\n"
+
+			"nvgre ip-version (ipv4|ipv6) tni (tni) ip-src"
+			" (ip-src) ip-dst (ip-dst) eth-src (eth-src) eth-dst"
+			" (eth-dst)\n"
+			"       Configure the NVGRE encapsulation for flows.\n\n"
+
+			"nvgre-with-vlan ip-version (ipv4|ipv6) tni (tni)"
+			" ip-src (ip-src) ip-dst (ip-dst) vlan-tci (vlan-tci)"
+			" eth-src (eth-src) eth-dst (eth-dst)\n"
+			"       Configure the NVGRE encapsulation for flows.\n\n"
+
 			, list_pkt_forwarding_modes()
 		);
 	}
@@ -1876,11 +1897,13 @@ cmd_config_rx_mode_flag_parsed(void *parsed_result,
 		port = &ports[pid];
 		rx_offloads = port->dev_conf.rxmode.offloads;
 		if (!strcmp(res->name, "crc-strip")) {
-			if (!strcmp(res->value, "on"))
+			if (!strcmp(res->value, "on")) {
 				rx_offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
-			else if (!strcmp(res->value, "off"))
+				rx_offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC;
+			} else if (!strcmp(res->value, "off")) {
+				rx_offloads |= DEV_RX_OFFLOAD_KEEP_CRC;
 				rx_offloads &= ~DEV_RX_OFFLOAD_CRC_STRIP;
-			else {
+			} else {
 				printf("Unknown parameter\n");
 				return;
 			}
@@ -2058,11 +2081,21 @@ cmd_config_rss_parsed(void *parsed_result,
 	rss_conf.rss_key = NULL;
 	/* Update global configuration for RSS types. */
 	RTE_ETH_FOREACH_DEV(i) {
-		if (use_default) {
-			rte_eth_dev_info_get(i, &dev_info);
+		struct rte_eth_rss_conf local_rss_conf;
+
+		rte_eth_dev_info_get(i, &dev_info);
+		if (use_default)
 			rss_conf.rss_hf = dev_info.flow_type_rss_offloads;
+
+		local_rss_conf = rss_conf;
+		local_rss_conf.rss_hf = rss_conf.rss_hf &
+			dev_info.flow_type_rss_offloads;
+		if (local_rss_conf.rss_hf != rss_conf.rss_hf) {
+			printf("Port %u modified RSS hash function based on hardware support,"
+				"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+				i, rss_conf.rss_hf, local_rss_conf.rss_hf);
 		}
-		diag = rte_eth_dev_rss_hash_update(i, &rss_conf);
+		diag = rte_eth_dev_rss_hash_update(i, &local_rss_conf);
 		if (diag < 0) {
 			all_updated = 0;
 			printf("Configuration of RSS hash at ethernet port %d "
@@ -4792,8 +4825,9 @@ cmd_gso_show_parsed(void *parsed_result,
 		if (gso_ports[res->cmd_pid].enable) {
 			printf("Max GSO'd packet size: %uB\n"
 					"Supported GSO types: TCP/IPv4, "
-					"VxLAN with inner TCP/IPv4 packet, "
-					"GRE with inner TCP/IPv4  packet\n",
+					"UDP/IPv4, VxLAN with inner "
+					"TCP/IPv4 packet, GRE with inner "
+					"TCP/IPv4 packet\n",
 					gso_max_segment_size);
 		} else
 			printf("GSO is not enabled on Port %u\n", res->cmd_pid);
@@ -14806,20 +14840,14 @@ static void cmd_set_port_tm_hierarchy_default_parsed(void *parsed_result,
 
 	p = &ports[port_id];
 
-	/* Port tm flag */
-	if (p->softport.tm_flag == 0) {
-		printf("  tm not enabled on port %u (error)\n", port_id);
-		return;
-	}
-
 	/* Forward mode: tm */
-	if (strcmp(cur_fwd_config.fwd_eng->fwd_mode_name, "tm")) {
-		printf("  tm mode not enabled(error)\n");
+	if (strcmp(cur_fwd_config.fwd_eng->fwd_mode_name, "softnic")) {
+		printf("  softnicfwd mode not enabled(error)\n");
 		return;
 	}
 
 	/* Set the default tm hierarchy */
-	p->softport.tm.default_hierarchy_enable = 1;
+	p->softport.default_tm_hierarchy_enable = 1;
 }
 
 cmdline_parse_inst_t cmd_set_port_tm_hierarchy_default = {
@@ -14838,6 +14866,326 @@ cmdline_parse_inst_t cmd_set_port_tm_hierarchy_default = {
 };
 #endif
 
+/** Set VXLAN encapsulation details */
+struct cmd_set_vxlan_result {
+	cmdline_fixed_string_t set;
+	cmdline_fixed_string_t vxlan;
+	cmdline_fixed_string_t pos_token;
+	cmdline_fixed_string_t ip_version;
+	uint32_t vlan_present:1;
+	uint32_t vni;
+	uint16_t udp_src;
+	uint16_t udp_dst;
+	cmdline_ipaddr_t ip_src;
+	cmdline_ipaddr_t ip_dst;
+	uint16_t tci;
+	struct ether_addr eth_src;
+	struct ether_addr eth_dst;
+};
+
+cmdline_parse_token_string_t cmd_set_vxlan_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, set, "set");
+cmdline_parse_token_string_t cmd_set_vxlan_vxlan =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, vxlan, "vxlan");
+cmdline_parse_token_string_t cmd_set_vxlan_vxlan_with_vlan =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, vxlan,
+				 "vxlan-with-vlan");
+cmdline_parse_token_string_t cmd_set_vxlan_ip_version =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token,
+				 "ip-version");
+cmdline_parse_token_string_t cmd_set_vxlan_ip_version_value =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, ip_version,
+				 "ipv4#ipv6");
+cmdline_parse_token_string_t cmd_set_vxlan_vni =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token,
+				 "vni");
+cmdline_parse_token_num_t cmd_set_vxlan_vni_value =
+	TOKEN_NUM_INITIALIZER(struct cmd_set_vxlan_result, vni, UINT32);
+cmdline_parse_token_string_t cmd_set_vxlan_udp_src =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token,
+				 "udp-src");
+cmdline_parse_token_num_t cmd_set_vxlan_udp_src_value =
+	TOKEN_NUM_INITIALIZER(struct cmd_set_vxlan_result, udp_src, UINT16);
+cmdline_parse_token_string_t cmd_set_vxlan_udp_dst =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token,
+				 "udp-dst");
+cmdline_parse_token_num_t cmd_set_vxlan_udp_dst_value =
+	TOKEN_NUM_INITIALIZER(struct cmd_set_vxlan_result, udp_dst, UINT16);
+cmdline_parse_token_string_t cmd_set_vxlan_ip_src =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token,
+				 "ip-src");
+cmdline_parse_token_ipaddr_t cmd_set_vxlan_ip_src_value =
+	TOKEN_IPADDR_INITIALIZER(struct cmd_set_vxlan_result, ip_src);
+cmdline_parse_token_string_t cmd_set_vxlan_ip_dst =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token,
+				 "ip-dst");
+cmdline_parse_token_ipaddr_t cmd_set_vxlan_ip_dst_value =
+	TOKEN_IPADDR_INITIALIZER(struct cmd_set_vxlan_result, ip_dst);
+cmdline_parse_token_string_t cmd_set_vxlan_vlan =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token,
+				 "vlan-tci");
+cmdline_parse_token_num_t cmd_set_vxlan_vlan_value =
+	TOKEN_NUM_INITIALIZER(struct cmd_set_vxlan_result, tci, UINT16);
+cmdline_parse_token_string_t cmd_set_vxlan_eth_src =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token,
+				 "eth-src");
+cmdline_parse_token_etheraddr_t cmd_set_vxlan_eth_src_value =
+	TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_vxlan_result, eth_src);
+cmdline_parse_token_string_t cmd_set_vxlan_eth_dst =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token,
+				 "eth-dst");
+cmdline_parse_token_etheraddr_t cmd_set_vxlan_eth_dst_value =
+	TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_vxlan_result, eth_dst);
+
+static void cmd_set_vxlan_parsed(void *parsed_result,
+	__attribute__((unused)) struct cmdline *cl,
+	__attribute__((unused)) void *data)
+{
+	struct cmd_set_vxlan_result *res = parsed_result;
+	union {
+		uint32_t vxlan_id;
+		uint8_t vni[4];
+	} id = {
+		.vxlan_id = rte_cpu_to_be_32(res->vni) & RTE_BE32(0x00ffffff),
+	};
+
+	if (strcmp(res->vxlan, "vxlan") == 0)
+		vxlan_encap_conf.select_vlan = 0;
+	else if (strcmp(res->vxlan, "vxlan-with-vlan") == 0)
+		vxlan_encap_conf.select_vlan = 1;
+	if (strcmp(res->ip_version, "ipv4") == 0)
+		vxlan_encap_conf.select_ipv4 = 1;
+	else if (strcmp(res->ip_version, "ipv6") == 0)
+		vxlan_encap_conf.select_ipv4 = 0;
+	else
+		return;
+	rte_memcpy(vxlan_encap_conf.vni, &id.vni[1], 3);
+	vxlan_encap_conf.udp_src = rte_cpu_to_be_16(res->udp_src);
+	vxlan_encap_conf.udp_dst = rte_cpu_to_be_16(res->udp_dst);
+	if (vxlan_encap_conf.select_ipv4) {
+		IPV4_ADDR_TO_UINT(res->ip_src, vxlan_encap_conf.ipv4_src);
+		IPV4_ADDR_TO_UINT(res->ip_dst, vxlan_encap_conf.ipv4_dst);
+	} else {
+		IPV6_ADDR_TO_ARRAY(res->ip_src, vxlan_encap_conf.ipv6_src);
+		IPV6_ADDR_TO_ARRAY(res->ip_dst, vxlan_encap_conf.ipv6_dst);
+	}
+	if (vxlan_encap_conf.select_vlan)
+		vxlan_encap_conf.vlan_tci = rte_cpu_to_be_16(res->tci);
+	rte_memcpy(vxlan_encap_conf.eth_src, res->eth_src.addr_bytes,
+		   ETHER_ADDR_LEN);
+	rte_memcpy(vxlan_encap_conf.eth_dst, res->eth_dst.addr_bytes,
+		   ETHER_ADDR_LEN);
+}
+
+cmdline_parse_inst_t cmd_set_vxlan = {
+	.f = cmd_set_vxlan_parsed,
+	.data = NULL,
+	.help_str = "set vxlan ip-version ipv4|ipv6 vni <vni> udp-src"
+		" <udp-src> udp-dst <udp-dst> ip-src <ip-src> ip-dst <ip-dst>"
+		" eth-src <eth-src> eth-dst <eth-dst>",
+	.tokens = {
+		(void *)&cmd_set_vxlan_set,
+		(void *)&cmd_set_vxlan_vxlan,
+		(void *)&cmd_set_vxlan_ip_version,
+		(void *)&cmd_set_vxlan_ip_version_value,
+		(void *)&cmd_set_vxlan_vni,
+		(void *)&cmd_set_vxlan_vni_value,
+		(void *)&cmd_set_vxlan_udp_src,
+		(void *)&cmd_set_vxlan_udp_src_value,
+		(void *)&cmd_set_vxlan_udp_dst,
+		(void *)&cmd_set_vxlan_udp_dst_value,
+		(void *)&cmd_set_vxlan_ip_src,
+		(void *)&cmd_set_vxlan_ip_src_value,
+		(void *)&cmd_set_vxlan_ip_dst,
+		(void *)&cmd_set_vxlan_ip_dst_value,
+		(void *)&cmd_set_vxlan_eth_src,
+		(void *)&cmd_set_vxlan_eth_src_value,
+		(void *)&cmd_set_vxlan_eth_dst,
+		(void *)&cmd_set_vxlan_eth_dst_value,
+		NULL,
+	},
+};
+
+cmdline_parse_inst_t cmd_set_vxlan_with_vlan = {
+	.f = cmd_set_vxlan_parsed,
+	.data = NULL,
+	.help_str = "set vxlan-with-vlan ip-version ipv4|ipv6 vni <vni>"
+		" udp-src <udp-src> udp-dst <udp-dst> ip-src <ip-src> ip-dst"
+		" <ip-dst> vlan-tci <vlan-tci> eth-src <eth-src> eth-dst"
+		" <eth-dst>",
+	.tokens = {
+		(void *)&cmd_set_vxlan_set,
+		(void *)&cmd_set_vxlan_vxlan_with_vlan,
+		(void *)&cmd_set_vxlan_ip_version,
+		(void *)&cmd_set_vxlan_ip_version_value,
+		(void *)&cmd_set_vxlan_vni,
+		(void *)&cmd_set_vxlan_vni_value,
+		(void *)&cmd_set_vxlan_udp_src,
+		(void *)&cmd_set_vxlan_udp_src_value,
+		(void *)&cmd_set_vxlan_udp_dst,
+		(void *)&cmd_set_vxlan_udp_dst_value,
+		(void *)&cmd_set_vxlan_ip_src,
+		(void *)&cmd_set_vxlan_ip_src_value,
+		(void *)&cmd_set_vxlan_ip_dst,
+		(void *)&cmd_set_vxlan_ip_dst_value,
+		(void *)&cmd_set_vxlan_vlan,
+		(void *)&cmd_set_vxlan_vlan_value,
+		(void *)&cmd_set_vxlan_eth_src,
+		(void *)&cmd_set_vxlan_eth_src_value,
+		(void *)&cmd_set_vxlan_eth_dst,
+		(void *)&cmd_set_vxlan_eth_dst_value,
+		NULL,
+	},
+};
+
+/** Set NVGRE encapsulation details */
+struct cmd_set_nvgre_result {
+	cmdline_fixed_string_t set;
+	cmdline_fixed_string_t nvgre;
+	cmdline_fixed_string_t pos_token;
+	cmdline_fixed_string_t ip_version;
+	uint32_t tni;
+	cmdline_ipaddr_t ip_src;
+	cmdline_ipaddr_t ip_dst;
+	uint16_t tci;
+	struct ether_addr eth_src;
+	struct ether_addr eth_dst;
+};
+
+cmdline_parse_token_string_t cmd_set_nvgre_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, set, "set");
+cmdline_parse_token_string_t cmd_set_nvgre_nvgre =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, nvgre, "nvgre");
+cmdline_parse_token_string_t cmd_set_nvgre_nvgre_with_vlan =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, nvgre,
+				 "nvgre-with-vlan");
+cmdline_parse_token_string_t cmd_set_nvgre_ip_version =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token,
+				 "ip-version");
+cmdline_parse_token_string_t cmd_set_nvgre_ip_version_value =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, ip_version,
+				 "ipv4#ipv6");
+cmdline_parse_token_string_t cmd_set_nvgre_tni =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token,
+				 "tni");
+cmdline_parse_token_num_t cmd_set_nvgre_tni_value =
+	TOKEN_NUM_INITIALIZER(struct cmd_set_nvgre_result, tni, UINT32);
+cmdline_parse_token_string_t cmd_set_nvgre_ip_src =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token,
+				 "ip-src");
+cmdline_parse_token_num_t cmd_set_nvgre_ip_src_value =
+	TOKEN_IPADDR_INITIALIZER(struct cmd_set_nvgre_result, ip_src);
+cmdline_parse_token_string_t cmd_set_nvgre_ip_dst =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token,
+				 "ip-dst");
+cmdline_parse_token_ipaddr_t cmd_set_nvgre_ip_dst_value =
+	TOKEN_IPADDR_INITIALIZER(struct cmd_set_nvgre_result, ip_dst);
+cmdline_parse_token_string_t cmd_set_nvgre_vlan =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token,
+				 "vlan-tci");
+cmdline_parse_token_num_t cmd_set_nvgre_vlan_value =
+	TOKEN_NUM_INITIALIZER(struct cmd_set_nvgre_result, tci, UINT16);
+cmdline_parse_token_string_t cmd_set_nvgre_eth_src =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token,
+				 "eth-src");
+cmdline_parse_token_etheraddr_t cmd_set_nvgre_eth_src_value =
+	TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_nvgre_result, eth_src);
+cmdline_parse_token_string_t cmd_set_nvgre_eth_dst =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token,
+				 "eth-dst");
+cmdline_parse_token_etheraddr_t cmd_set_nvgre_eth_dst_value =
+	TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_nvgre_result, eth_dst);
+
+static void cmd_set_nvgre_parsed(void *parsed_result,
+	__attribute__((unused)) struct cmdline *cl,
+	__attribute__((unused)) void *data)
+{
+	struct cmd_set_nvgre_result *res = parsed_result;
+	union {
+		uint32_t nvgre_tni;
+		uint8_t tni[4];
+	} id = {
+		.nvgre_tni = rte_cpu_to_be_32(res->tni) & RTE_BE32(0x00ffffff),
+	};
+
+	if (strcmp(res->nvgre, "nvgre") == 0)
+		nvgre_encap_conf.select_vlan = 0;
+	else if (strcmp(res->nvgre, "nvgre-with-vlan") == 0)
+		nvgre_encap_conf.select_vlan = 1;
+	if (strcmp(res->ip_version, "ipv4") == 0)
+		nvgre_encap_conf.select_ipv4 = 1;
+	else if (strcmp(res->ip_version, "ipv6") == 0)
+		nvgre_encap_conf.select_ipv4 = 0;
+	else
+		return;
+	rte_memcpy(nvgre_encap_conf.tni, &id.tni[1], 3);
+	if (nvgre_encap_conf.select_ipv4) {
+		IPV4_ADDR_TO_UINT(res->ip_src, nvgre_encap_conf.ipv4_src);
+		IPV4_ADDR_TO_UINT(res->ip_dst, nvgre_encap_conf.ipv4_dst);
+	} else {
+		IPV6_ADDR_TO_ARRAY(res->ip_src, nvgre_encap_conf.ipv6_src);
+		IPV6_ADDR_TO_ARRAY(res->ip_dst, nvgre_encap_conf.ipv6_dst);
+	}
+	if (nvgre_encap_conf.select_vlan)
+		nvgre_encap_conf.vlan_tci = rte_cpu_to_be_16(res->tci);
+	rte_memcpy(nvgre_encap_conf.eth_src, res->eth_src.addr_bytes,
+		   ETHER_ADDR_LEN);
+	rte_memcpy(nvgre_encap_conf.eth_dst, res->eth_dst.addr_bytes,
+		   ETHER_ADDR_LEN);
+}
+
+cmdline_parse_inst_t cmd_set_nvgre = {
+	.f = cmd_set_nvgre_parsed,
+	.data = NULL,
+	.help_str = "set nvgre ip-version <ipv4|ipv6> tni <tni> ip-src"
+		" <ip-src> ip-dst <ip-dst> eth-src <eth-src>"
+		" eth-dst <eth-dst>",
+	.tokens = {
+		(void *)&cmd_set_nvgre_set,
+		(void *)&cmd_set_nvgre_nvgre,
+		(void *)&cmd_set_nvgre_ip_version,
+		(void *)&cmd_set_nvgre_ip_version_value,
+		(void *)&cmd_set_nvgre_tni,
+		(void *)&cmd_set_nvgre_tni_value,
+		(void *)&cmd_set_nvgre_ip_src,
+		(void *)&cmd_set_nvgre_ip_src_value,
+		(void *)&cmd_set_nvgre_ip_dst,
+		(void *)&cmd_set_nvgre_ip_dst_value,
+		(void *)&cmd_set_nvgre_eth_src,
+		(void *)&cmd_set_nvgre_eth_src_value,
+		(void *)&cmd_set_nvgre_eth_dst,
+		(void *)&cmd_set_nvgre_eth_dst_value,
+		NULL,
+	},
+};
+
+cmdline_parse_inst_t cmd_set_nvgre_with_vlan = {
+	.f = cmd_set_nvgre_parsed,
+	.data = NULL,
+	.help_str = "set nvgre-with-vlan ip-version <ipv4|ipv6> tni <tni>"
+		" ip-src <ip-src> ip-dst <ip-dst> vlan-tci <vlan-tci>"
+		" eth-src <eth-src> eth-dst <eth-dst>",
+	.tokens = {
+		(void *)&cmd_set_nvgre_set,
+		(void *)&cmd_set_nvgre_nvgre_with_vlan,
+		(void *)&cmd_set_nvgre_ip_version,
+		(void *)&cmd_set_nvgre_ip_version_value,
+		(void *)&cmd_set_nvgre_tni,
+		(void *)&cmd_set_nvgre_tni_value,
+		(void *)&cmd_set_nvgre_ip_src,
+		(void *)&cmd_set_nvgre_ip_src_value,
+		(void *)&cmd_set_nvgre_ip_dst,
+		(void *)&cmd_set_nvgre_ip_dst_value,
+		(void *)&cmd_set_nvgre_vlan,
+		(void *)&cmd_set_nvgre_vlan_value,
+		(void *)&cmd_set_nvgre_eth_src,
+		(void *)&cmd_set_nvgre_eth_src_value,
+		(void *)&cmd_set_nvgre_eth_dst,
+		(void *)&cmd_set_nvgre_eth_dst_value,
+		NULL,
+	},
+};
+
 /* Strict link priority scheduling mode setting */
 static void
 cmd_strict_link_prio_parsed(
@@ -16657,7 +17005,7 @@ cmdline_parse_token_string_t cmd_config_per_port_rx_offload_result_offload =
 		 offload, "vlan_strip#ipv4_cksum#udp_cksum#tcp_cksum#tcp_lro#"
 			   "qinq_strip#outer_ipv4_cksum#macsec_strip#"
 			   "header_split#vlan_filter#vlan_extend#jumbo_frame#"
-			   "crc_strip#scatter#timestamp#security");
+			   "crc_strip#scatter#timestamp#security#keep_crc");
 cmdline_parse_token_string_t cmd_config_per_port_rx_offload_result_on_off =
 	TOKEN_STRING_INITIALIZER
 		(struct cmd_config_per_port_rx_offload_result,
@@ -16736,7 +17084,7 @@ cmdline_parse_inst_t cmd_config_per_port_rx_offload = {
 	.help_str = "port config <port_id> rx_offload vlan_strip|ipv4_cksum|"
 		    "udp_cksum|tcp_cksum|tcp_lro|qinq_strip|outer_ipv4_cksum|"
 		    "macsec_strip|header_split|vlan_filter|vlan_extend|"
-		    "jumbo_frame|crc_strip|scatter|timestamp|security "
+		    "jumbo_frame|crc_strip|scatter|timestamp|security|keep_crc "
 		    "on|off",
 	.tokens = {
 		(void *)&cmd_config_per_port_rx_offload_result_port,
@@ -16786,7 +17134,7 @@ cmdline_parse_token_string_t cmd_config_per_queue_rx_offload_result_offload =
 		 offload, "vlan_strip#ipv4_cksum#udp_cksum#tcp_cksum#tcp_lro#"
 			   "qinq_strip#outer_ipv4_cksum#macsec_strip#"
 			   "header_split#vlan_filter#vlan_extend#jumbo_frame#"
-			   "crc_strip#scatter#timestamp#security");
+			   "crc_strip#scatter#timestamp#security#keep_crc");
 cmdline_parse_token_string_t cmd_config_per_queue_rx_offload_result_on_off =
 	TOKEN_STRING_INITIALIZER
 		(struct cmd_config_per_queue_rx_offload_result,
@@ -16838,7 +17186,7 @@ cmdline_parse_inst_t cmd_config_per_queue_rx_offload = {
 		    "vlan_strip|ipv4_cksum|"
 		    "udp_cksum|tcp_cksum|tcp_lro|qinq_strip|outer_ipv4_cksum|"
 		    "macsec_strip|header_split|vlan_filter|vlan_extend|"
-		    "jumbo_frame|crc_strip|scatter|timestamp|security "
+		    "jumbo_frame|crc_strip|scatter|timestamp|security|keep_crc "
 		    "on|off",
 	.tokens = {
 		(void *)&cmd_config_per_queue_rx_offload_result_port,
@@ -17051,11 +17399,11 @@ cmdline_parse_token_string_t cmd_config_per_port_tx_offload_result_tx_offload =
 cmdline_parse_token_string_t cmd_config_per_port_tx_offload_result_offload =
 	TOKEN_STRING_INITIALIZER
 		(struct cmd_config_per_port_tx_offload_result,
-		 offload, "vlan_insert#ipv4_cksum#udp_cksum#udp_cksum#"
+		 offload, "vlan_insert#ipv4_cksum#udp_cksum#tcp_cksum#"
 			  "sctp_cksum#tcp_tso#udp_tso#outer_ipv4_cksum#"
 			  "qinq_insert#vxlan_tnl_tso#gre_tnl_tso#"
 			  "ipip_tnl_tso#geneve_tnl_tso#macsec_insert#"
-			  "mt_lockfree#multi_segs#fast_free#security");
+			  "mt_lockfree#multi_segs#mbuf_fast_free#security");
 cmdline_parse_token_string_t cmd_config_per_port_tx_offload_result_on_off =
 	TOKEN_STRING_INITIALIZER
 		(struct cmd_config_per_port_tx_offload_result,
@@ -17132,11 +17480,11 @@ cmdline_parse_inst_t cmd_config_per_port_tx_offload = {
 	.f = cmd_config_per_port_tx_offload_parsed,
 	.data = NULL,
 	.help_str = "port config <port_id> tx_offload "
-		    "vlan_insert|ipv4_cksum|udp_cksum|udp_cksum|"
+		    "vlan_insert|ipv4_cksum|udp_cksum|tcp_cksum|"
 		    "sctp_cksum|tcp_tso|udp_tso|outer_ipv4_cksum|"
 		    "qinq_insert|vxlan_tnl_tso|gre_tnl_tso|"
 		    "ipip_tnl_tso|geneve_tnl_tso|macsec_insert|"
-		    "mt_lockfree|multi_segs|fast_free|security "
+		    "mt_lockfree|multi_segs|mbuf_fast_free|security "
 		    "on|off",
 	.tokens = {
 		(void *)&cmd_config_per_port_tx_offload_result_port,
@@ -17183,11 +17531,11 @@ cmdline_parse_token_string_t cmd_config_per_queue_tx_offload_result_txoffload =
 cmdline_parse_token_string_t cmd_config_per_queue_tx_offload_result_offload =
 	TOKEN_STRING_INITIALIZER
 		(struct cmd_config_per_queue_tx_offload_result,
-		 offload, "vlan_insert#ipv4_cksum#udp_cksum#udp_cksum#"
+		 offload, "vlan_insert#ipv4_cksum#udp_cksum#tcp_cksum#"
 			  "sctp_cksum#tcp_tso#udp_tso#outer_ipv4_cksum#"
 			  "qinq_insert#vxlan_tnl_tso#gre_tnl_tso#"
 			  "ipip_tnl_tso#geneve_tnl_tso#macsec_insert#"
-			  "mt_lockfree#multi_segs#fast_free#security");
+			  "mt_lockfree#multi_segs#mbuf_fast_free#security");
 cmdline_parse_token_string_t cmd_config_per_queue_tx_offload_result_on_off =
 	TOKEN_STRING_INITIALIZER
 		(struct cmd_config_per_queue_tx_offload_result,
@@ -17236,11 +17584,11 @@ cmdline_parse_inst_t cmd_config_per_queue_tx_offload = {
 	.f = cmd_config_per_queue_tx_offload_parsed,
 	.data = NULL,
 	.help_str = "port <port_id> txq <queue_id> tx_offload "
-		    "vlan_insert|ipv4_cksum|udp_cksum|udp_cksum|"
+		    "vlan_insert|ipv4_cksum|udp_cksum|tcp_cksum|"
 		    "sctp_cksum|tcp_tso|udp_tso|outer_ipv4_cksum|"
 		    "qinq_insert|vxlan_tnl_tso|gre_tnl_tso|"
 		    "ipip_tnl_tso|geneve_tnl_tso|macsec_insert|"
-		    "mt_lockfree|multi_segs|fast_free|security "
+		    "mt_lockfree|multi_segs|mbuf_fast_free|security "
 		    "on|off",
 	.tokens = {
 		(void *)&cmd_config_per_queue_tx_offload_result_port,
@@ -17462,6 +17810,10 @@ cmdline_parse_ctx_t main_ctx[] = {
 #if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED
 	(cmdline_parse_inst_t *)&cmd_set_port_tm_hierarchy_default,
 #endif
+	(cmdline_parse_inst_t *)&cmd_set_vxlan,
+	(cmdline_parse_inst_t *)&cmd_set_vxlan_with_vlan,
+	(cmdline_parse_inst_t *)&cmd_set_nvgre,
+	(cmdline_parse_inst_t *)&cmd_set_nvgre_with_vlan,
 	(cmdline_parse_inst_t *)&cmd_ddp_add,
 	(cmdline_parse_inst_t *)&cmd_ddp_del,
 	(cmdline_parse_inst_t *)&cmd_ddp_get_list,
diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
index 9918d7fd..f9260600 100644
--- a/app/test-pmd/cmdline_flow.c
+++ b/app/test-pmd/cmdline_flow.c
@@ -194,6 +194,8 @@ enum index {
 	ACTION_QUEUE_INDEX,
 	ACTION_DROP,
 	ACTION_COUNT,
+	ACTION_COUNT_SHARED,
+	ACTION_COUNT_ID,
 	ACTION_RSS,
 	ACTION_RSS_FUNC,
 	ACTION_RSS_LEVEL,
@@ -237,6 +239,10 @@ enum index {
 	ACTION_OF_POP_MPLS_ETHERTYPE,
 	ACTION_OF_PUSH_MPLS,
 	ACTION_OF_PUSH_MPLS_ETHERTYPE,
+	ACTION_VXLAN_ENCAP,
+	ACTION_VXLAN_DECAP,
+	ACTION_NVGRE_ENCAP,
+	ACTION_NVGRE_DECAP,
 };
 
 /** Maximum size for pattern in struct rte_flow_item_raw. */
@@ -256,6 +262,39 @@ struct action_rss_data {
 	uint16_t queue[ACTION_RSS_QUEUE_NUM];
 };
 
+/** Maximum number of items in struct rte_flow_action_vxlan_encap. */
+#define ACTION_VXLAN_ENCAP_ITEMS_NUM 6
+
+/** Storage for struct rte_flow_action_vxlan_encap including external data. */
+struct action_vxlan_encap_data {
+	struct rte_flow_action_vxlan_encap conf;
+	struct rte_flow_item items[ACTION_VXLAN_ENCAP_ITEMS_NUM];
+	struct rte_flow_item_eth item_eth;
+	struct rte_flow_item_vlan item_vlan;
+	union {
+		struct rte_flow_item_ipv4 item_ipv4;
+		struct rte_flow_item_ipv6 item_ipv6;
+	};
+	struct rte_flow_item_udp item_udp;
+	struct rte_flow_item_vxlan item_vxlan;
+};
+
+/** Maximum number of items in struct rte_flow_action_nvgre_encap. */
+#define ACTION_NVGRE_ENCAP_ITEMS_NUM 5
+
+/** Storage for struct rte_flow_action_nvgre_encap including external data. */
+struct action_nvgre_encap_data {
+	struct rte_flow_action_nvgre_encap conf;
+	struct rte_flow_item items[ACTION_NVGRE_ENCAP_ITEMS_NUM];
+	struct rte_flow_item_eth item_eth;
+	struct rte_flow_item_vlan item_vlan;
+	union {
+		struct rte_flow_item_ipv4 item_ipv4;
+		struct rte_flow_item_ipv6 item_ipv6;
+	};
+	struct rte_flow_item_nvgre item_nvgre;
+};
+
 /** Maximum number of subsequent tokens and arguments on the stack. */
 #define CTX_STACK_SIZE 16
 
@@ -773,6 +812,10 @@ static const enum index next_action[] = {
 	ACTION_OF_SET_VLAN_PCP,
 	ACTION_OF_POP_MPLS,
 	ACTION_OF_PUSH_MPLS,
+	ACTION_VXLAN_ENCAP,
+	ACTION_VXLAN_DECAP,
+	ACTION_NVGRE_ENCAP,
+	ACTION_NVGRE_DECAP,
 	ZERO,
 };
 
@@ -788,6 +831,13 @@ static const enum index action_queue[] = {
 	ZERO,
 };
 
+static const enum index action_count[] = {
+	ACTION_COUNT_ID,
+	ACTION_COUNT_SHARED,
+	ACTION_NEXT,
+	ZERO,
+};
+
 static const enum index action_rss[] = {
 	ACTION_RSS_FUNC,
 	ACTION_RSS_LEVEL,
@@ -896,6 +946,12 @@ static int parse_vc_action_rss_type(struct context *, const struct token *,
 static int parse_vc_action_rss_queue(struct context *, const struct token *,
 				     const char *, unsigned int, void *,
 				     unsigned int);
+static int parse_vc_action_vxlan_encap(struct context *, const struct token *,
+				       const char *, unsigned int, void *,
+				       unsigned int);
+static int parse_vc_action_nvgre_encap(struct context *, const struct token *,
+				       const char *, unsigned int, void *,
+				       unsigned int);
 static int parse_destroy(struct context *, const struct token *,
 			 const char *, unsigned int,
 			 void *, unsigned int);
@@ -2022,10 +2078,26 @@ static const struct token token_list[] = {
 	[ACTION_COUNT] = {
 		.name = "count",
 		.help = "enable counters for this rule",
-		.priv = PRIV_ACTION(COUNT, 0),
-		.next = NEXT(NEXT_ENTRY(ACTION_NEXT)),
+		.priv = PRIV_ACTION(COUNT,
+				    sizeof(struct rte_flow_action_count)),
+		.next = NEXT(action_count),
 		.call = parse_vc,
 	},
+	[ACTION_COUNT_ID] = {
+		.name = "identifier",
+		.help = "counter identifier to use",
+		.next = NEXT(action_count, NEXT_ENTRY(UNSIGNED)),
+		.args = ARGS(ARGS_ENTRY(struct rte_flow_action_count, id)),
+		.call = parse_vc_conf,
+	},
+	[ACTION_COUNT_SHARED] = {
+		.name = "shared",
+		.help = "shared counter",
+		.next = NEXT(action_count, NEXT_ENTRY(BOOLEAN)),
+		.args = ARGS(ARGS_ENTRY_BF(struct rte_flow_action_count,
+					   shared, 1)),
+		.call = parse_vc_conf,
+	},
 	[ACTION_RSS] = {
 		.name = "rss",
 		.help = "spread packets among several queues",
@@ -2362,6 +2434,42 @@ static const struct token token_list[] = {
 			      ethertype)),
 		.call = parse_vc_conf,
 	},
+	[ACTION_VXLAN_ENCAP] = {
+		.name = "vxlan_encap",
+		.help = "VXLAN encapsulation, uses configuration set by \"set"
+			" vxlan\"",
+		.priv = PRIV_ACTION(VXLAN_ENCAP,
+				    sizeof(struct action_vxlan_encap_data)),
+		.next = NEXT(NEXT_ENTRY(ACTION_NEXT)),
+		.call = parse_vc_action_vxlan_encap,
+	},
+	[ACTION_VXLAN_DECAP] = {
+		.name = "vxlan_decap",
+		.help = "Performs a decapsulation action by stripping all"
+			" headers of the VXLAN tunnel network overlay from the"
+			" matched flow.",
+		.priv = PRIV_ACTION(VXLAN_DECAP, 0),
+		.next = NEXT(NEXT_ENTRY(ACTION_NEXT)),
+		.call = parse_vc,
+	},
+	[ACTION_NVGRE_ENCAP] = {
+		.name = "nvgre_encap",
+		.help = "NVGRE encapsulation, uses configuration set by \"set"
+			" nvgre\"",
+		.priv = PRIV_ACTION(NVGRE_ENCAP,
+				    sizeof(struct action_nvgre_encap_data)),
+		.next = NEXT(NEXT_ENTRY(ACTION_NEXT)),
+		.call = parse_vc_action_nvgre_encap,
+	},
+	[ACTION_NVGRE_DECAP] = {
+		.name = "nvgre_decap",
+		.help = "Performs a decapsulation action by stripping all"
+			" headers of the NVGRE tunnel network overlay from the"
+			" matched flow.",
+		.priv = PRIV_ACTION(NVGRE_DECAP, 0),
+		.next = NEXT(NEXT_ENTRY(ACTION_NEXT)),
+		.call = parse_vc,
+	},
 };
 
 /** Remove and return last entry from argument stack. */
@@ -2926,6 +3034,197 @@ end:
 	return len;
 }
 
+/** Parse VXLAN encap action. */
+static int
+parse_vc_action_vxlan_encap(struct context *ctx, const struct token *token,
+			    const char *str, unsigned int len,
+			    void *buf, unsigned int size)
+{
+	struct buffer *out = buf;
+	struct rte_flow_action *action;
+	struct action_vxlan_encap_data *action_vxlan_encap_data;
+	int ret;
+
+	ret = parse_vc(ctx, token, str, len, buf, size);
+	if (ret < 0)
+		return ret;
+	/* Nothing else to do if there is no buffer. */
+	if (!out)
+		return ret;
+	if (!out->args.vc.actions_n)
+		return -1;
+	action = &out->args.vc.actions[out->args.vc.actions_n - 1];
+	/* Point to selected object. */
+	ctx->object = out->args.vc.data;
+	ctx->objmask = NULL;
+	/* Set up default configuration. */
+	action_vxlan_encap_data = ctx->object;
+	*action_vxlan_encap_data = (struct action_vxlan_encap_data){
+		.conf = (struct rte_flow_action_vxlan_encap){
+			.definition = action_vxlan_encap_data->items,
+		},
+		.items = {
+			{
+				.type = RTE_FLOW_ITEM_TYPE_ETH,
+				.spec = &action_vxlan_encap_data->item_eth,
+				.mask = &rte_flow_item_eth_mask,
+			},
+			{
+				.type = RTE_FLOW_ITEM_TYPE_VLAN,
+				.spec = &action_vxlan_encap_data->item_vlan,
+				.mask = &rte_flow_item_vlan_mask,
+			},
+			{
+				.type = RTE_FLOW_ITEM_TYPE_IPV4,
+				.spec = &action_vxlan_encap_data->item_ipv4,
+				.mask = &rte_flow_item_ipv4_mask,
+			},
+			{
+				.type = RTE_FLOW_ITEM_TYPE_UDP,
+				.spec = &action_vxlan_encap_data->item_udp,
+				.mask = &rte_flow_item_udp_mask,
+			},
+			{
+				.type = RTE_FLOW_ITEM_TYPE_VXLAN,
+				.spec = &action_vxlan_encap_data->item_vxlan,
+				.mask = &rte_flow_item_vxlan_mask,
+			},
+			{
+				.type = RTE_FLOW_ITEM_TYPE_END,
+			},
+		},
+		.item_eth.type = 0,
+		.item_vlan = {
+			.tci = vxlan_encap_conf.vlan_tci,
+			.inner_type = 0,
+		},
+		.item_ipv4.hdr = {
+			.src_addr = vxlan_encap_conf.ipv4_src,
+			.dst_addr = vxlan_encap_conf.ipv4_dst,
+		},
+		.item_udp.hdr = {
+			.src_port = vxlan_encap_conf.udp_src,
+			.dst_port = vxlan_encap_conf.udp_dst,
+		},
+		.item_vxlan.flags = 0,
+	};
+	memcpy(action_vxlan_encap_data->item_eth.dst.addr_bytes,
+	       vxlan_encap_conf.eth_dst, ETHER_ADDR_LEN);
+	memcpy(action_vxlan_encap_data->item_eth.src.addr_bytes,
+	       vxlan_encap_conf.eth_src, ETHER_ADDR_LEN);
+	if (!vxlan_encap_conf.select_ipv4) {
+		memcpy(&action_vxlan_encap_data->item_ipv6.hdr.src_addr,
+		       &vxlan_encap_conf.ipv6_src,
+		       sizeof(vxlan_encap_conf.ipv6_src));
+		memcpy(&action_vxlan_encap_data->item_ipv6.hdr.dst_addr,
+		       &vxlan_encap_conf.ipv6_dst,
+		       sizeof(vxlan_encap_conf.ipv6_dst));
+		action_vxlan_encap_data->items[2] = (struct rte_flow_item){
+			.type = RTE_FLOW_ITEM_TYPE_IPV6,
+			.spec = &action_vxlan_encap_data->item_ipv6,
+			.mask = &rte_flow_item_ipv6_mask,
+		};
+	}
+	if (!vxlan_encap_conf.select_vlan)
+		action_vxlan_encap_data->items[1].type =
+			RTE_FLOW_ITEM_TYPE_VOID;
+	memcpy(action_vxlan_encap_data->item_vxlan.vni, vxlan_encap_conf.vni,
+	       RTE_DIM(vxlan_encap_conf.vni));
+	action->conf = &action_vxlan_encap_data->conf;
+	return ret;
+}
+
+/** Parse NVGRE encap action. */
+static int
+parse_vc_action_nvgre_encap(struct context *ctx, const struct token *token,
+			    const char *str, unsigned int len,
+			    void *buf, unsigned int size)
+{
+	struct buffer *out = buf;
+	struct rte_flow_action *action;
+	struct action_nvgre_encap_data *action_nvgre_encap_data;
+	int ret;
+
+	ret = parse_vc(ctx, token, str, len, buf, size);
+	if (ret < 0)
+		return ret;
+	/* Nothing else to do if there is no buffer. */
+	if (!out)
+		return ret;
+	if (!out->args.vc.actions_n)
+		return -1;
+	action = &out->args.vc.actions[out->args.vc.actions_n - 1];
+	/* Point to selected object. */
+	ctx->object = out->args.vc.data;
+	ctx->objmask = NULL;
+	/* Set up default configuration. */
+	action_nvgre_encap_data = ctx->object;
+	*action_nvgre_encap_data = (struct action_nvgre_encap_data){
+		.conf = (struct rte_flow_action_nvgre_encap){
+			.definition = action_nvgre_encap_data->items,
+		},
+		.items = {
+			{
+				.type = RTE_FLOW_ITEM_TYPE_ETH,
+				.spec = &action_nvgre_encap_data->item_eth,
+				.mask = &rte_flow_item_eth_mask,
+			},
+			{
+				.type = RTE_FLOW_ITEM_TYPE_VLAN,
+				.spec = &action_nvgre_encap_data->item_vlan,
+				.mask = &rte_flow_item_vlan_mask,
+			},
+			{
+				.type = RTE_FLOW_ITEM_TYPE_IPV4,
+				.spec = &action_nvgre_encap_data->item_ipv4,
+				.mask = &rte_flow_item_ipv4_mask,
+			},
+			{
+				.type = RTE_FLOW_ITEM_TYPE_NVGRE,
+				.spec = &action_nvgre_encap_data->item_nvgre,
+				.mask = &rte_flow_item_nvgre_mask,
+			},
+			{
+				.type = RTE_FLOW_ITEM_TYPE_END,
+			},
+		},
+		.item_eth.type = 0,
+		.item_vlan = {
+			.tci = nvgre_encap_conf.vlan_tci,
+			.inner_type = 0,
+		},
+		.item_ipv4.hdr = {
+		       .src_addr = nvgre_encap_conf.ipv4_src,
+		       .dst_addr = nvgre_encap_conf.ipv4_dst,
+		},
+		.item_nvgre.flow_id = 0,
+	};
+	memcpy(action_nvgre_encap_data->item_eth.dst.addr_bytes,
+	       nvgre_encap_conf.eth_dst, ETHER_ADDR_LEN);
+	memcpy(action_nvgre_encap_data->item_eth.src.addr_bytes,
+	       nvgre_encap_conf.eth_src, ETHER_ADDR_LEN);
+	if (!nvgre_encap_conf.select_ipv4) {
+		memcpy(&action_nvgre_encap_data->item_ipv6.hdr.src_addr,
+		       &nvgre_encap_conf.ipv6_src,
+		       sizeof(nvgre_encap_conf.ipv6_src));
+		memcpy(&action_nvgre_encap_data->item_ipv6.hdr.dst_addr,
+		       &nvgre_encap_conf.ipv6_dst,
+		       sizeof(nvgre_encap_conf.ipv6_dst));
+		action_nvgre_encap_data->items[2] = (struct rte_flow_item){
+			.type = RTE_FLOW_ITEM_TYPE_IPV6,
+			.spec = &action_nvgre_encap_data->item_ipv6,
+			.mask = &rte_flow_item_ipv6_mask,
+		};
+	}
+	if (!nvgre_encap_conf.select_vlan)
+		action_nvgre_encap_data->items[1].type =
+			RTE_FLOW_ITEM_TYPE_VOID;
+	memcpy(action_nvgre_encap_data->item_nvgre.tni, nvgre_encap_conf.tni,
+	       RTE_DIM(nvgre_encap_conf.tni));
+	action->conf = &action_nvgre_encap_data->conf;
+	return ret;
+}
+
 /** Parse tokens for destroy command. */
 static int
 parse_destroy(struct context *ctx, const struct token *token,
diff --git a/app/test-pmd/cmdline_tm.c b/app/test-pmd/cmdline_tm.c
index c904e44f..631f1799 100644
--- a/app/test-pmd/cmdline_tm.c
+++ b/app/test-pmd/cmdline_tm.c
@@ -234,6 +234,7 @@ static void cmd_show_port_tm_cap_parsed(void *parsed_result,
 		return;
 
 	memset(&cap, 0, sizeof(struct rte_tm_capabilities));
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	ret = rte_tm_capabilities_get(port_id, &cap, &error);
 	if (ret) {
 		print_err_msg(&error);
@@ -374,6 +375,7 @@ static void cmd_show_port_tm_level_cap_parsed(void *parsed_result,
 		return;
 
 	memset(&lcap, 0, sizeof(struct rte_tm_level_capabilities));
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	ret = rte_tm_level_capabilities_get(port_id, level_id, &lcap, &error);
 	if (ret) {
 		print_err_msg(&error);
@@ -498,6 +500,7 @@ static void cmd_show_port_tm_node_cap_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	/* Node id must be valid */
 	ret = rte_tm_node_type_get(port_id, node_id, &is_leaf, &error);
 	if (ret != 0) {
@@ -615,6 +618,7 @@ static void cmd_show_port_tm_node_stats_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	/* Port status */
 	if (!port_is_started(port_id)) {
 		printf(" Port %u not started (error)\n", port_id);
@@ -727,6 +731,7 @@ static void cmd_show_port_tm_node_type_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	ret = rte_tm_node_type_get(port_id, node_id, &is_leaf, &error);
 	if (ret != 0) {
 		print_err_msg(&error);
@@ -832,6 +837,7 @@ static void cmd_add_port_tm_node_shaper_profile_parsed(void *parsed_result,
 
 	/* Private shaper profile params */
 	memset(&sp, 0, sizeof(struct rte_tm_shaper_params));
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	sp.peak.rate = res->tb_rate;
 	sp.peak.size = res->tb_size;
 	sp.pkt_length_adjust = pkt_len_adjust;
@@ -919,6 +925,7 @@ static void cmd_del_port_tm_node_shaper_profile_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	ret = rte_tm_shaper_profile_delete(port_id, shaper_id, &error);
 	if (ret != 0) {
 		print_err_msg(&error);
@@ -1004,6 +1011,7 @@ static void cmd_add_port_tm_node_shared_shaper_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	/* Command type: add */
 	if ((strcmp(res->cmd_type, "add") == 0) &&
 		(port_is_started(port_id))) {
@@ -1098,6 +1106,7 @@ static void cmd_del_port_tm_node_shared_shaper_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	ret = rte_tm_shared_shaper_delete(port_id, shared_shaper_id, &error);
 	if (ret != 0) {
 		print_err_msg(&error);
@@ -1254,6 +1263,7 @@ static void cmd_add_port_tm_node_wred_profile_parsed(void *parsed_result,
 		return;
 
 	memset(&wp, 0, sizeof(struct rte_tm_wred_params));
+	memset(&error, 0, sizeof(struct rte_tm_error));
 
 	/* WRED Params  (Green Color)*/
 	color = RTE_TM_GREEN;
@@ -1369,6 +1379,7 @@ static void cmd_del_port_tm_node_wred_profile_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	ret = rte_tm_wred_profile_delete(port_id, wred_profile_id, &error);
 	if (ret != 0) {
 		print_err_msg(&error);
@@ -1455,6 +1466,7 @@ static void cmd_set_port_tm_node_shaper_profile_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	/* Port status */
 	if (!port_is_started(port_id)) {
 		printf(" Port %u not started (error)\n", port_id);
@@ -1571,6 +1583,7 @@ static void cmd_add_port_tm_nonleaf_node_parsed(void *parsed_result,
 		return;
 
 	memset(&np, 0, sizeof(struct rte_tm_node_params));
+	memset(&error, 0, sizeof(struct rte_tm_error));
 
 	/* Node parameters */
 	if (res->parent_node_id < 0)
@@ -1599,10 +1612,12 @@ static void cmd_add_port_tm_nonleaf_node_parsed(void *parsed_result,
 		np.shaper_profile_id = res->shaper_profile_id;
 
 	np.n_shared_shapers = n_shared_shapers;
-	if (np.n_shared_shapers)
+	if (np.n_shared_shapers) {
 		np.shared_shaper_id = &shared_shaper_id[0];
-	else
-		np.shared_shaper_id = NULL;
+	} else {
+		free(shared_shaper_id);
+		shared_shaper_id = NULL;
+	}
 
 	np.nonleaf.n_sp_priorities = res->n_sp_priorities;
 	np.stats_mask = res->stats_mask;
@@ -1729,6 +1744,7 @@ static void cmd_add_port_tm_leaf_node_parsed(void *parsed_result,
 		return;
 
 	memset(&np, 0, sizeof(struct rte_tm_node_params));
+	memset(&error, 0, sizeof(struct rte_tm_error));
 
 	/* Node parameters */
 	if (res->parent_node_id < 0)
@@ -1758,10 +1774,12 @@ static void cmd_add_port_tm_leaf_node_parsed(void *parsed_result,
 
 	np.n_shared_shapers = n_shared_shapers;
 
-	if (np.n_shared_shapers)
+	if (np.n_shared_shapers) {
 		np.shared_shaper_id = &shared_shaper_id[0];
-	else
-		np.shared_shaper_id = NULL;
+	} else {
+		free(shared_shaper_id);
+		shared_shaper_id = NULL;
+	}
 
 	np.leaf.cman = res->cman_mode;
 	np.leaf.wred.wred_profile_id = res->wred_profile_id;
@@ -1844,6 +1862,7 @@ static void cmd_del_port_tm_node_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	/* Port status */
 	if (port_is_started(port_id)) {
 		printf(" Port %u not stopped (error)\n", port_id);
@@ -1933,6 +1952,7 @@ static void cmd_set_port_tm_node_parent_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	/* Port status */
 	if (!port_is_started(port_id)) {
 		printf(" Port %u not started (error)\n", port_id);
@@ -2008,6 +2028,7 @@ static void cmd_suspend_port_tm_node_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	ret = rte_tm_node_suspend(port_id, node_id, &error);
 	if (ret != 0) {
 		print_err_msg(&error);
@@ -2072,6 +2093,7 @@ static void cmd_resume_port_tm_node_parsed(void *parsed_result,
 	if (port_id_is_invalid(port_id, ENABLED_WARN))
 		return;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	ret = rte_tm_node_resume(port_id, node_id, &error);
 	if (ret != 0) {
 		print_err_msg(&error);
@@ -2143,6 +2165,7 @@ static void cmd_port_tm_hierarchy_commit_parsed(void *parsed_result,
 	else
 		clean_on_fail = 0;
 
+	memset(&error, 0, sizeof(struct rte_tm_error));
 	ret = rte_tm_hierarchy_commit(port_id, clean_on_fail, &error);
 	if (ret != 0) {
 		print_err_msg(&error);
@@ -2153,7 +2176,7 @@ static void cmd_port_tm_hierarchy_commit_parsed(void *parsed_result,
 cmdline_parse_inst_t cmd_port_tm_hierarchy_commit = {
 	.f = cmd_port_tm_hierarchy_commit_parsed,
 	.data = NULL,
-	.help_str = "Set port tm node shaper profile",
+	.help_str = "Commit port tm hierarchy",
 	.tokens = {
 		(void *)&cmd_port_tm_hierarchy_commit_port,
 		(void *)&cmd_port_tm_hierarchy_commit_tm,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index 97020fb3..14ccd686 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -2177,15 +2177,11 @@ rss_fwd_config_setup(void)
 		fs->tx_queue = rxq;
 		fs->peer_addr = fs->tx_port;
 		fs->retry_enabled = retry_enabled;
-		rxq = (queueid_t) (rxq + 1);
-		if (rxq < nb_q)
-			continue;
-		/*
-		 * rxq == nb_q
-		 * Restart from RX queue 0 on next RX port
-		 */
-		rxq = 0;
 		rxp++;
+		if (rxp < nb_fwd_ports)
+			continue;
+		rxp = 0;
+		rxq++;
 	}
 }
 
@@ -2332,6 +2328,55 @@ icmp_echo_config_setup(void)
 	}
 }
 
+#if defined RTE_LIBRTE_PMD_SOFTNIC
+static void
+softnic_fwd_config_setup(void)
+{
+	struct rte_port *port;
+	portid_t pid, softnic_portid;
+	queueid_t i;
+	uint8_t softnic_enable = 0;
+
+	RTE_ETH_FOREACH_DEV(pid) {
+			port = &ports[pid];
+			const char *driver = port->dev_info.driver_name;
+
+			if (strcmp(driver, "net_softnic") == 0) {
+				softnic_portid = pid;
+				softnic_enable = 1;
+				break;
+			}
+	}
+
+	if (softnic_enable == 0) {
+		printf("Softnic mode not configured(%s)!\n", __func__);
+		return;
+	}
+
+	cur_fwd_config.nb_fwd_ports = 1;
+	cur_fwd_config.nb_fwd_streams = (streamid_t) nb_rxq;
+
+	/* Re-initialize forwarding streams */
+	init_fwd_streams();
+
+	/*
+	 * In the softnic forwarding test, the number of forwarding cores
+	 * is set to one and remaining are used for softnic packet processing.
+	 */
+	cur_fwd_config.nb_fwd_lcores = 1;
+	setup_fwd_config_of_each_lcore(&cur_fwd_config);
+
+	for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) {
+		fwd_streams[i]->rx_port   = softnic_portid;
+		fwd_streams[i]->rx_queue  = i;
+		fwd_streams[i]->tx_port   = softnic_portid;
+		fwd_streams[i]->tx_queue  = i;
+		fwd_streams[i]->peer_addr = fwd_streams[i]->tx_port;
+		fwd_streams[i]->retry_enabled = retry_enabled;
+	}
+}
+#endif
+
 void
 fwd_config_setup(void)
 {
@@ -2340,6 +2385,14 @@ fwd_config_setup(void)
 		icmp_echo_config_setup();
 		return;
 	}
+
+#if defined RTE_LIBRTE_PMD_SOFTNIC
+	if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
+		softnic_fwd_config_setup();
+		return;
+	}
+#endif
+
 	if ((nb_rxq > 1) && (nb_txq > 1)){
 		if (dcb_config)
 			dcb_fwd_config_setup();
diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
index 0bb88cf7..49482926 100644
--- a/app/test-pmd/csumonly.c
+++ b/app/test-pmd/csumonly.c
@@ -411,6 +411,8 @@ process_inner_cksums(void *l3_hdr, const struct testpmd_offload_info *info,
 						info->ethertype);
 			}
 		}
+		if (info->gso_enable)
+			ol_flags |= PKT_TX_UDP_SEG;
 	} else if (info->l4_proto == IPPROTO_TCP) {
 		tcp_hdr = (struct tcp_hdr *)((char *)l3_hdr + info->l3_len);
 		tcp_hdr->cksum = 0;
diff --git a/app/test-pmd/meson.build b/app/test-pmd/meson.build
index a51514b0..a0b3be07 100644
--- a/app/test-pmd/meson.build
+++ b/app/test-pmd/meson.build
@@ -4,6 +4,7 @@
 # override default name to drop the hyphen
 name = 'testpmd'
 allow_experimental_apis = true
+cflags += '-Wno-deprecated-declarations'
 sources = files('cmdline.c',
 	'cmdline_flow.c',
 	'cmdline_mtr.c',
@@ -25,6 +26,9 @@ deps = ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'meter', 'bus_pci']
 if dpdk_conf.has('RTE_LIBRTE_PDUMP')
 	deps += 'pdump'
 endif
+if dpdk_conf.has('RTE_LIBRTE_BNXT_PMD')
+	deps += 'pmd_bnxt'
+endif
 if dpdk_conf.has('RTE_LIBRTE_I40E_PMD')
 	deps += 'pmd_i40e'
 endif
@@ -32,7 +36,7 @@ if dpdk_conf.has('RTE_LIBRTE_IXGBE_PMD')
 	deps += 'pmd_ixgbe'
 endif
 if dpdk_conf.has('RTE_LIBRTE_SOFTNIC_PMD')
-	sources += files('tm.c')
+	sources += files('softnicfwd.c')
 	deps += 'pmd_softnic'
 endif
 if dpdk_conf.has('RTE_LIBRTE_DPAA_PMD')
diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
index 75807623..962fad78 100644
--- a/app/test-pmd/parameters.c
+++ b/app/test-pmd/parameters.c
@@ -878,8 +878,10 @@ launch_args_parse(int argc, char** argv)
 						 " must be >= 0\n", n);
 			}
 #endif
-			if (!strcmp(lgopts[opt_idx].name, "disable-crc-strip"))
+			if (!strcmp(lgopts[opt_idx].name, "disable-crc-strip")) {
 				rx_offloads &= ~DEV_RX_OFFLOAD_CRC_STRIP;
+				rx_offloads |= DEV_RX_OFFLOAD_KEEP_CRC;
+			}
 			if (!strcmp(lgopts[opt_idx].name, "enable-lro"))
 				rx_offloads |= DEV_RX_OFFLOAD_TCP_LRO;
 			if (!strcmp(lgopts[opt_idx].name, "enable-scatter"))
diff --git a/app/test-pmd/tm.c b/app/test-pmd/softnicfwd.c
index 7231552a..7ff62280 100644
--- a/app/test-pmd/tm.c
+++ b/app/test-pmd/softnicfwd.c
@@ -6,6 +6,7 @@
 
 #include <rte_cycles.h>
 #include <rte_mbuf.h>
+#include <rte_malloc.h>
 #include <rte_ethdev.h>
 #include <rte_flow.h>
 #include <rte_meter.h>
@@ -71,170 +72,17 @@ struct tm_hierarchy {
 	uint32_t n_shapers;
 };
 
-#define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr)	\
-({								\
-	uint64_t slab = *((uint64_t *) &byte_array[slab_pos]);	\
-	uint64_t val =				\
-		(rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr;	\
-	val;						\
-})
-
-#define RTE_SCHED_PORT_HIERARCHY(subport, pipe,           \
-	traffic_class, queue, color)                          \
-	((((uint64_t) (queue)) & 0x3) |                       \
-	((((uint64_t) (traffic_class)) & 0x3) << 2) |         \
-	((((uint64_t) (color)) & 0x3) << 4) |                 \
-	((((uint64_t) (subport)) & 0xFFFF) << 16) |           \
-	((((uint64_t) (pipe)) & 0xFFFFFFFF) << 32))
-
-
-static void
-pkt_metadata_set(struct rte_port *p, struct rte_mbuf **pkts,
-	uint32_t n_pkts)
-{
-	struct softnic_port_tm *tm = &p->softport.tm;
-	uint32_t i;
-
-	for (i = 0; i < (n_pkts & (~0x3)); i += 4) {
-		struct rte_mbuf *pkt0 = pkts[i];
-		struct rte_mbuf *pkt1 = pkts[i + 1];
-		struct rte_mbuf *pkt2 = pkts[i + 2];
-		struct rte_mbuf *pkt3 = pkts[i + 3];
-
-		uint8_t *pkt0_data = rte_pktmbuf_mtod(pkt0, uint8_t *);
-		uint8_t *pkt1_data = rte_pktmbuf_mtod(pkt1, uint8_t *);
-		uint8_t *pkt2_data = rte_pktmbuf_mtod(pkt2, uint8_t *);
-		uint8_t *pkt3_data = rte_pktmbuf_mtod(pkt3, uint8_t *);
-
-		uint64_t pkt0_subport = BITFIELD(pkt0_data,
-					tm->tm_pktfield0_slabpos,
-					tm->tm_pktfield0_slabmask,
-					tm->tm_pktfield0_slabshr);
-		uint64_t pkt0_pipe = BITFIELD(pkt0_data,
-					tm->tm_pktfield1_slabpos,
-					tm->tm_pktfield1_slabmask,
-					tm->tm_pktfield1_slabshr);
-		uint64_t pkt0_dscp = BITFIELD(pkt0_data,
-					tm->tm_pktfield2_slabpos,
-					tm->tm_pktfield2_slabmask,
-					tm->tm_pktfield2_slabshr);
-		uint32_t pkt0_tc = tm->tm_tc_table[pkt0_dscp & 0x3F] >> 2;
-		uint32_t pkt0_tc_q = tm->tm_tc_table[pkt0_dscp & 0x3F] & 0x3;
-		uint64_t pkt1_subport = BITFIELD(pkt1_data,
-					tm->tm_pktfield0_slabpos,
-					tm->tm_pktfield0_slabmask,
-					tm->tm_pktfield0_slabshr);
-		uint64_t pkt1_pipe = BITFIELD(pkt1_data,
-					tm->tm_pktfield1_slabpos,
-					tm->tm_pktfield1_slabmask,
-					tm->tm_pktfield1_slabshr);
-		uint64_t pkt1_dscp = BITFIELD(pkt1_data,
-					tm->tm_pktfield2_slabpos,
-					tm->tm_pktfield2_slabmask,
-					tm->tm_pktfield2_slabshr);
-		uint32_t pkt1_tc = tm->tm_tc_table[pkt1_dscp & 0x3F] >> 2;
-		uint32_t pkt1_tc_q = tm->tm_tc_table[pkt1_dscp & 0x3F] & 0x3;
-
-		uint64_t pkt2_subport = BITFIELD(pkt2_data,
-					tm->tm_pktfield0_slabpos,
-					tm->tm_pktfield0_slabmask,
-					tm->tm_pktfield0_slabshr);
-		uint64_t pkt2_pipe = BITFIELD(pkt2_data,
-					tm->tm_pktfield1_slabpos,
-					tm->tm_pktfield1_slabmask,
-					tm->tm_pktfield1_slabshr);
-		uint64_t pkt2_dscp = BITFIELD(pkt2_data,
-					tm->tm_pktfield2_slabpos,
-					tm->tm_pktfield2_slabmask,
-					tm->tm_pktfield2_slabshr);
-		uint32_t pkt2_tc = tm->tm_tc_table[pkt2_dscp & 0x3F] >> 2;
-		uint32_t pkt2_tc_q = tm->tm_tc_table[pkt2_dscp & 0x3F] & 0x3;
-
-		uint64_t pkt3_subport = BITFIELD(pkt3_data,
-					tm->tm_pktfield0_slabpos,
-					tm->tm_pktfield0_slabmask,
-					tm->tm_pktfield0_slabshr);
-		uint64_t pkt3_pipe = BITFIELD(pkt3_data,
-					tm->tm_pktfield1_slabpos,
-					tm->tm_pktfield1_slabmask,
-					tm->tm_pktfield1_slabshr);
-		uint64_t pkt3_dscp = BITFIELD(pkt3_data,
-					tm->tm_pktfield2_slabpos,
-					tm->tm_pktfield2_slabmask,
-					tm->tm_pktfield2_slabshr);
-		uint32_t pkt3_tc = tm->tm_tc_table[pkt3_dscp & 0x3F] >> 2;
-		uint32_t pkt3_tc_q = tm->tm_tc_table[pkt3_dscp & 0x3F] & 0x3;
-
-		uint64_t pkt0_sched = RTE_SCHED_PORT_HIERARCHY(pkt0_subport,
-						pkt0_pipe,
-						pkt0_tc,
-						pkt0_tc_q,
-						0);
-		uint64_t pkt1_sched = RTE_SCHED_PORT_HIERARCHY(pkt1_subport,
-						pkt1_pipe,
-						pkt1_tc,
-						pkt1_tc_q,
-						0);
-		uint64_t pkt2_sched = RTE_SCHED_PORT_HIERARCHY(pkt2_subport,
-						pkt2_pipe,
-						pkt2_tc,
-						pkt2_tc_q,
-						0);
-		uint64_t pkt3_sched = RTE_SCHED_PORT_HIERARCHY(pkt3_subport,
-						pkt3_pipe,
-						pkt3_tc,
-						pkt3_tc_q,
-						0);
-
-		pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
-		pkt0->hash.sched.hi = pkt0_sched >> 32;
-		pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
-		pkt1->hash.sched.hi = pkt1_sched >> 32;
-		pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
-		pkt2->hash.sched.hi = pkt2_sched >> 32;
-		pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
-		pkt3->hash.sched.hi = pkt3_sched >> 32;
-	}
-
-	for (; i < n_pkts; i++)	{
-		struct rte_mbuf *pkt = pkts[i];
-
-		uint8_t *pkt_data = rte_pktmbuf_mtod(pkt, uint8_t *);
-
-		uint64_t pkt_subport = BITFIELD(pkt_data,
-					tm->tm_pktfield0_slabpos,
-					tm->tm_pktfield0_slabmask,
-					tm->tm_pktfield0_slabshr);
-		uint64_t pkt_pipe = BITFIELD(pkt_data,
-					tm->tm_pktfield1_slabpos,
-					tm->tm_pktfield1_slabmask,
-					tm->tm_pktfield1_slabshr);
-		uint64_t pkt_dscp = BITFIELD(pkt_data,
-					tm->tm_pktfield2_slabpos,
-					tm->tm_pktfield2_slabmask,
-					tm->tm_pktfield2_slabshr);
-		uint32_t pkt_tc = tm->tm_tc_table[pkt_dscp & 0x3F] >> 2;
-		uint32_t pkt_tc_q = tm->tm_tc_table[pkt_dscp & 0x3F] & 0x3;
-
-		uint64_t pkt_sched = RTE_SCHED_PORT_HIERARCHY(pkt_subport,
-						pkt_pipe,
-						pkt_tc,
-						pkt_tc_q,
-						0);
-
-		pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
-		pkt->hash.sched.hi = pkt_sched >> 32;
-	}
-}
+static struct fwd_lcore *softnic_fwd_lcore;
+static uint16_t softnic_port_id;
+struct fwd_engine softnic_fwd_engine;
 
 /*
- * Soft port packet forward
+ * Softnic packet forward
  */
 static void
-softport_packet_fwd(struct fwd_stream *fs)
+softnic_fwd(struct fwd_stream *fs)
 {
 	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
-	struct rte_port *rte_tx_port = &ports[fs->tx_port];
 	uint16_t nb_rx;
 	uint16_t nb_tx;
 	uint32_t retry;
@@ -258,14 +106,6 @@ softport_packet_fwd(struct fwd_stream *fs)
 	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
 #endif
 
-	if (rte_tx_port->softnic_enable) {
-		/* Set packet metadata if tm flag enabled */
-		if (rte_tx_port->softport.tm_flag)
-			pkt_metadata_set(rte_tx_port, pkts_burst, nb_rx);
-
-		/* Softport run */
-		rte_pmd_softnic_run(fs->tx_port);
-	}
 	nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
 			pkts_burst, nb_rx);
 
@@ -298,7 +138,34 @@ softport_packet_fwd(struct fwd_stream *fs)
 }
 
 static void
-set_tm_hiearchy_nodes_shaper_rate(portid_t port_id, struct tm_hierarchy *h)
+softnic_fwd_run(struct fwd_stream *fs)
+{
+	rte_pmd_softnic_run(softnic_port_id);
+	softnic_fwd(fs);
+}
+
+/**
+ * Softnic init
+ */
+static int
+softnic_begin(void *arg __rte_unused)
+{
+	for (;;) {
+		if (!softnic_fwd_lcore->stopped)
+			break;
+	}
+
+	do {
+		/* Run softnic */
+		rte_pmd_softnic_run(softnic_port_id);
+	} while (!softnic_fwd_lcore->stopped);
+
+	return 0;
+}
+
+static void
+set_tm_hiearchy_nodes_shaper_rate(portid_t port_id,
+	struct tm_hierarchy *h)
 {
 	struct rte_eth_link link_params;
 	uint64_t tm_port_rate;
@@ -306,10 +173,7 @@ set_tm_hiearchy_nodes_shaper_rate(portid_t port_id, struct tm_hierarchy *h)
 	memset(&link_params, 0, sizeof(link_params));
 
 	rte_eth_link_get(port_id, &link_params);
-	tm_port_rate = (uint64_t)link_params.link_speed * BYTES_IN_MBPS;
-
-	if (tm_port_rate > UINT32_MAX)
-		tm_port_rate = UINT32_MAX;
+	tm_port_rate = (uint64_t)ETH_SPEED_NUM_10G * BYTES_IN_MBPS;
 
 	/* Set tm hierarchy shapers rate */
 	h->root_node_shaper_rate = tm_port_rate;
@@ -374,7 +238,8 @@ softport_tm_root_node_add(portid_t port_id, struct tm_hierarchy *h,
 }
 
 static int
-softport_tm_subport_node_add(portid_t port_id, struct tm_hierarchy *h,
+softport_tm_subport_node_add(portid_t port_id,
+	struct tm_hierarchy *h,
 	struct rte_tm_error *error)
 {
 	uint32_t subport_parent_node_id, subport_node_id = 0;
@@ -442,7 +307,8 @@ softport_tm_subport_node_add(portid_t port_id, struct tm_hierarchy *h,
 }
 
 static int
-softport_tm_pipe_node_add(portid_t port_id, struct tm_hierarchy *h,
+softport_tm_pipe_node_add(portid_t port_id,
+	struct tm_hierarchy *h,
 	struct rte_tm_error *error)
 {
 	uint32_t pipe_parent_node_id;
@@ -511,7 +377,8 @@ softport_tm_pipe_node_add(portid_t port_id, struct tm_hierarchy *h,
 }
 
 static int
-softport_tm_tc_node_add(portid_t port_id, struct tm_hierarchy *h,
+softport_tm_tc_node_add(portid_t port_id,
+	struct tm_hierarchy *h,
 	struct rte_tm_error *error)
 {
 	uint32_t tc_parent_node_id;
@@ -674,63 +541,9 @@ softport_tm_queue_node_add(portid_t port_id, struct tm_hierarchy *h,
 	return 0;
 }
 
-/*
- * TM Packet Field Setup
- */
-static void
-softport_tm_pktfield_setup(portid_t port_id)
-{
-	struct rte_port *p = &ports[port_id];
-	uint64_t pktfield0_mask = 0;
-	uint64_t pktfield1_mask = 0x0000000FFF000000LLU;
-	uint64_t pktfield2_mask = 0x00000000000000FCLLU;
-
-	p->softport.tm = (struct softnic_port_tm) {
-		.n_subports_per_port = SUBPORT_NODES_PER_PORT,
-		.n_pipes_per_subport = PIPE_NODES_PER_SUBPORT,
-
-		/* Packet field to identify subport
-		 *
-		 * Default configuration assumes only one subport, thus
-		 * the subport ID is hardcoded to 0
-		 */
-		.tm_pktfield0_slabpos = 0,
-		.tm_pktfield0_slabmask = pktfield0_mask,
-		.tm_pktfield0_slabshr =
-			__builtin_ctzll(pktfield0_mask),
-
-		/* Packet field to identify pipe.
-		 *
-		 * Default value assumes Ethernet/IPv4/UDP packets,
-		 * UDP payload bits 12 .. 23
-		 */
-		.tm_pktfield1_slabpos = 40,
-		.tm_pktfield1_slabmask = pktfield1_mask,
-		.tm_pktfield1_slabshr =
-			__builtin_ctzll(pktfield1_mask),
-
-		/* Packet field used as index into TC translation table
-		 * to identify the traffic class and queue.
-		 *
-		 * Default value assumes Ethernet/IPv4 packets, IPv4
-		 * DSCP field
-		 */
-		.tm_pktfield2_slabpos = 8,
-		.tm_pktfield2_slabmask = pktfield2_mask,
-		.tm_pktfield2_slabshr =
-			__builtin_ctzll(pktfield2_mask),
-
-		.tm_tc_table = {
-			0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
-			0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
-			0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
-			0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
-		}, /**< TC translation table */
-	};
-}
-
 static int
-softport_tm_hierarchy_specify(portid_t port_id, struct rte_tm_error *error)
+softport_tm_hierarchy_specify(portid_t port_id,
+	struct rte_tm_error *error)
 {
 
 	struct tm_hierarchy h;
@@ -766,75 +579,96 @@ softport_tm_hierarchy_specify(portid_t port_id, struct rte_tm_error *error)
 	if (status)
 		return status;
 
-	/* TM packet fields setup */
-	softport_tm_pktfield_setup(port_id);
-
 	return 0;
 }
 
 /*
- * Soft port Init
+ * Softnic TM default configuration
  */
 static void
-softport_tm_begin(portid_t pi)
+softnic_tm_default_config(portid_t pi)
 {
 	struct rte_port *port = &ports[pi];
+	struct rte_tm_error error;
+	int status;
 
-	/* Soft port TM flag */
-	if (port->softport.tm_flag == 1) {
-		printf("\n\n  TM feature available on port %u\n", pi);
-
-		/* Soft port TM hierarchy configuration */
-		if ((port->softport.tm.hierarchy_config == 0) &&
-			(port->softport.tm.default_hierarchy_enable == 1)) {
-			struct rte_tm_error error;
-			int status;
-
-			/* Stop port */
-			rte_eth_dev_stop(pi);
-
-			/* TM hierarchy specification */
-			status = softport_tm_hierarchy_specify(pi, &error);
-			if (status) {
-				printf("  TM Hierarchy built error(%d) - %s\n",
-					error.type, error.message);
-				return;
-			}
-			printf("\n  TM Hierarchy Specified!\n\v");
-
-			/* TM hierarchy commit */
-			status = rte_tm_hierarchy_commit(pi, 0, &error);
-			if (status) {
-				printf("  Hierarchy commit error(%d) - %s\n",
-					error.type, error.message);
-				return;
-			}
-			printf("  Hierarchy Committed (port %u)!", pi);
-			port->softport.tm.hierarchy_config = 1;
-
-			/* Start port */
-			status = rte_eth_dev_start(pi);
-			if (status) {
-				printf("\n  Port %u start error!\n", pi);
-				return;
-			}
-			printf("\n  Port %u started!\n", pi);
-			return;
-		}
+	/* Stop port */
+	rte_eth_dev_stop(pi);
+
+	/* TM hierarchy specification */
+	status = softport_tm_hierarchy_specify(pi, &error);
+	if (status) {
+		printf("  TM Hierarchy built error(%d) - %s\n",
+			error.type, error.message);
+		return;
+	}
+	printf("\n  TM Hierarchy Specified!\n");
+
+	/* TM hierarchy commit */
+	status = rte_tm_hierarchy_commit(pi, 0, &error);
+	if (status) {
+		printf("  Hierarchy commit error(%d) - %s\n",
+			error.type, error.message);
+		return;
+	}
+	printf("  Hierarchy Committed (port %u)!\n", pi);
+
+	/* Start port */
+	status = rte_eth_dev_start(pi);
+	if (status) {
+		printf("\n  Port %u start error!\n", pi);
+		return;
 	}
-	printf("\n  TM feature not available on port %u", pi);
+
+	/* Reset the default hierarchy flag */
+	port->softport.default_tm_hierarchy_enable = 0;
 }
 
-struct fwd_engine softnic_tm_engine = {
-	.fwd_mode_name  = "tm",
-	.port_fwd_begin = softport_tm_begin,
-	.port_fwd_end   = NULL,
-	.packet_fwd     = softport_packet_fwd,
-};
+/*
+ * Softnic forwarding init
+ */
+static void
+softnic_fwd_begin(portid_t pi)
+{
+	struct rte_port *port = &ports[pi];
+	uint32_t lcore, fwd_core_present = 0, softnic_run_launch = 0;
+	int	status;
+
+	softnic_fwd_lcore = port->softport.fwd_lcore_arg[0];
+	softnic_port_id = pi;
+
+	/* Launch softnic_run function on lcores */
+	for (lcore = 0; lcore < RTE_MAX_LCORE; lcore++) {
+		if (!rte_lcore_is_enabled(lcore))
+			continue;
+
+		if (lcore == rte_get_master_lcore())
+			continue;
+
+		if (fwd_core_present == 0) {
+			fwd_core_present++;
+			continue;
+		}
+
+		status = rte_eal_remote_launch(softnic_begin, NULL, lcore);
+		if (status)
+			printf("softnic launch on lcore %u failed (%d)\n",
+				       lcore, status);
+
+		softnic_run_launch = 1;
+	}
+
+	if (!softnic_run_launch)
+		softnic_fwd_engine.packet_fwd = softnic_fwd_run;
+
+	/* Softnic TM default configuration */
+	if (port->softport.default_tm_hierarchy_enable == 1)
+		softnic_tm_default_config(pi);
+}
 
-struct fwd_engine softnic_tm_bypass_engine = {
-	.fwd_mode_name  = "tm-bypass",
-	.port_fwd_begin = NULL,
+struct fwd_engine softnic_fwd_engine = {
+	.fwd_mode_name  = "softnic",
+	.port_fwd_begin = softnic_fwd_begin,
 	.port_fwd_end   = NULL,
-	.packet_fwd     = softport_packet_fwd,
+	.packet_fwd     = softnic_fwd,
 };
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 35cf2667..ee48db2a 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -127,6 +127,8 @@ portid_t nb_ports;             /**< Number of probed ethernet ports. */
 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 
+portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
+
 /*
  * Test Forwarding Configuration.
  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
@@ -155,9 +157,8 @@ struct fwd_engine * fwd_engines[] = {
 	&tx_only_engine,
 	&csum_fwd_engine,
 	&icmp_echo_engine,
-#if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED
-	&softnic_tm_engine,
-	&softnic_tm_bypass_engine,
+#if defined RTE_LIBRTE_PMD_SOFTNIC
+	&softnic_fwd_engine,
 #endif
 #ifdef RTE_LIBRTE_IEEE1588
 	&ieee1588_fwd_engine,
@@ -334,7 +335,6 @@ lcoreid_t latencystats_lcore_id = -1;
 struct rte_eth_rxmode rx_mode = {
 	.max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
 	.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
-	.ignore_offload_bitfield = 1,
 };
 
 struct rte_eth_txmode tx_mode = {
@@ -346,7 +346,7 @@ struct rte_fdir_conf fdir_conf = {
 	.pballoc = RTE_FDIR_PBALLOC_64K,
 	.status = RTE_FDIR_REPORT_STATUS,
 	.mask = {
-		.vlan_tci_mask = 0x0,
+		.vlan_tci_mask = 0xFFEF,
 		.ipv4_mask     = {
 			.src_ip = 0xFFFFFFFF,
 			.dst_ip = 0xFFFFFFFF,
@@ -393,6 +393,38 @@ uint8_t bitrate_enabled;
 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 
+struct vxlan_encap_conf vxlan_encap_conf = {
+	.select_ipv4 = 1,
+	.select_vlan = 0,
+	.vni = "\x00\x00\x00",
+	.udp_src = 0,
+	.udp_dst = RTE_BE16(4789),
+	.ipv4_src = IPv4(127, 0, 0, 1),
+	.ipv4_dst = IPv4(255, 255, 255, 255),
+	.ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
+		"\x00\x00\x00\x00\x00\x00\x00\x01",
+	.ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
+		"\x00\x00\x00\x00\x00\x00\x11\x11",
+	.vlan_tci = 0,
+	.eth_src = "\x00\x00\x00\x00\x00\x00",
+	.eth_dst = "\xff\xff\xff\xff\xff\xff",
+};
+
+struct nvgre_encap_conf nvgre_encap_conf = {
+	.select_ipv4 = 1,
+	.select_vlan = 0,
+	.tni = "\x00\x00\x00",
+	.ipv4_src = IPv4(127, 0, 0, 1),
+	.ipv4_dst = IPv4(255, 255, 255, 255),
+	.ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00"
+		"\x00\x00\x00\x00\x00\x00\x00\x01",
+	.ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00"
+		"\x00\x00\x00\x00\x00\x00\x11\x11",
+	.vlan_tci = 0,
+	.eth_src = "\x00\x00\x00\x00\x00\x00",
+	.eth_dst = "\xff\xff\xff\xff\xff\xff",
+};
+
 /* Forward function declarations */
 static void map_port_queue_stats_mapping_registers(portid_t pi,
 						   struct rte_port *port);
@@ -777,7 +809,7 @@ init_config(void)
 	init_port_config();
 
 	gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
-		DEV_TX_OFFLOAD_GRE_TNL_TSO;
+		DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
 	/*
 	 * Records which Mbuf pool to use by each logical core, if needed.
 	 */
@@ -816,6 +848,19 @@ init_config(void)
 					"rte_gro_ctx_create() failed\n");
 		}
 	}
+
+#if defined RTE_LIBRTE_PMD_SOFTNIC
+	if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
+		RTE_ETH_FOREACH_DEV(pid) {
+			port = &ports[pid];
+			const char *driver = port->dev_info.driver_name;
+
+			if (strcmp(driver, "net_softnic") == 0)
+				port->softport.fwd_lcore_arg = fwd_lcores;
+		}
+	}
+#endif
+
 }
 
 
@@ -1148,8 +1193,9 @@ run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
 	uint64_t tics_per_1sec;
 	uint64_t tics_datum;
 	uint64_t tics_current;
-	uint16_t idx_port;
+	uint16_t i, cnt_ports;
 
+	cnt_ports = nb_ports;
 	tics_datum = rte_rdtsc();
 	tics_per_1sec = rte_get_timer_hz();
 #endif
@@ -1164,9 +1210,9 @@ run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
 			tics_current = rte_rdtsc();
 			if (tics_current - tics_datum >= tics_per_1sec) {
 				/* Periodic bitrate calculation */
-				RTE_ETH_FOREACH_DEV(idx_port)
+				for (i = 0; i < cnt_ports; i++)
 					rte_stats_bitrate_calc(bitrate_data,
-						idx_port);
+						ports_ids[i]);
 				tics_datum = tics_current;
 			}
 		}
@@ -1645,8 +1691,6 @@ start_port(portid_t pid)
 			port->need_reconfig_queues = 0;
 			/* setup tx queues */
 			for (qi = 0; qi < nb_txq; qi++) {
-				port->tx_conf[qi].txq_flags =
-					ETH_TXQ_FLAGS_IGNORE;
 				if ((numa_support) &&
 					(txring_numa[pi] != NUMA_NO_CONFIG))
 					diag = rte_eth_tx_queue_setup(pi, qi,
@@ -1971,6 +2015,7 @@ attach_port(char *identifier)
 	reconfig(pi, socket_id);
 	rte_eth_promiscuous_enable(pi);
 
+	ports_ids[nb_ports] = pi;
 	nb_ports = rte_eth_dev_count_avail();
 
 	ports[pi].port_status = RTE_PORT_STOPPED;
@@ -1985,6 +2030,7 @@ void
 detach_port(portid_t port_id)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
+	uint16_t i;
 
 	printf("Detaching a port...\n");
 
@@ -2001,6 +2047,13 @@ detach_port(portid_t port_id)
 		return;
 	}
 
+	for (i = 0; i < nb_ports; i++) {
+		if (ports_ids[i] == port_id) {
+			ports_ids[i] = ports_ids[nb_ports-1];
+			ports_ids[nb_ports-1] = 0;
+			break;
+		}
+	}
 	nb_ports = rte_eth_dev_count_avail();
 
 	update_fwd_ports(RTE_MAX_ETHPORTS);
@@ -2355,16 +2408,15 @@ init_port_config(void)
 {
 	portid_t pid;
 	struct rte_port *port;
-	struct rte_eth_dev_info dev_info;
 
 	RTE_ETH_FOREACH_DEV(pid) {
 		port = &ports[pid];
 		port->dev_conf.fdir_conf = fdir_conf;
+		rte_eth_dev_info_get(pid, &port->dev_info);
 		if (nb_rxq > 1) {
-			rte_eth_dev_info_get(pid, &dev_info);
 			port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
 			port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
-				rss_hf & dev_info.flow_type_rss_offloads;
+				rss_hf & port->dev_info.flow_type_rss_offloads;
 		} else {
 			port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
 			port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
@@ -2394,17 +2446,6 @@ init_port_config(void)
 		    (rte_eth_devices[pid].data->dev_flags &
 		     RTE_ETH_DEV_INTR_RMV))
 			port->dev_conf.intr_conf.rmv = 1;
-
-#if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED
-		/* Detect softnic port */
-		if (!strcmp(port->dev_info.driver_name, "net_softnic")) {
-			port->softnic_enable = 1;
-			memset(&port->softport, 0, sizeof(struct softnic_port));
-
-			if (!strcmp(cur_fwd_eng->fwd_mode_name, "tm"))
-				port->softport.tm_flag = 1;
-		}
-#endif
 	}
 }
 
@@ -2443,12 +2484,14 @@ const uint16_t vlan_tags[] = {
 };
 
 static  int
-get_eth_dcb_conf(struct rte_eth_conf *eth_conf,
+get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
 		 enum dcb_mode_enable dcb_mode,
 		 enum rte_eth_nb_tcs num_tcs,
 		 uint8_t pfc_en)
 {
 	uint8_t i;
+	int32_t rc;
+	struct rte_eth_rss_conf rss_conf;
 
 	/*
 	 * Builds up the correct configuration for dcb+vt based on the vlan tags array
@@ -2488,6 +2531,10 @@ get_eth_dcb_conf(struct rte_eth_conf *eth_conf,
 		struct rte_eth_dcb_tx_conf *tx_conf =
 				&eth_conf->tx_adv_conf.dcb_tx_conf;
 
+		rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
+		if (rc != 0)
+			return rc;
+
 		rx_conf->nb_tcs = num_tcs;
 		tx_conf->nb_tcs = num_tcs;
 
@@ -2495,8 +2542,9 @@ get_eth_dcb_conf(struct rte_eth_conf *eth_conf,
 			rx_conf->dcb_tc[i] = i % num_tcs;
 			tx_conf->dcb_tc[i] = i % num_tcs;
 		}
+
 		eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
-		eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf;
+		eth_conf->rx_adv_conf.rss_conf = rss_conf;
 		eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
 	}
 
@@ -2530,7 +2578,7 @@ init_port_dcb_config(portid_t pid,
 	port_conf.txmode = rte_port->dev_conf.txmode;
 
 	/*set configuration of DCB in vt mode and DCB in non-vt mode*/
-	retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en);
+	retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
 	if (retval < 0)
 		return retval;
 	port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
@@ -2653,6 +2701,7 @@ main(int argc, char** argv)
 {
 	int diag;
 	portid_t port_id;
+	uint16_t count;
 	int ret;
 
 	signal(SIGINT, signal_handler);
@@ -2672,7 +2721,12 @@ main(int argc, char** argv)
 	rte_pdump_init(NULL);
 #endif
 
-	nb_ports = (portid_t) rte_eth_dev_count_avail();
+	count = 0;
+	RTE_ETH_FOREACH_DEV(port_id) {
+		ports_ids[count] = port_id;
+		count++;
+	}
+	nb_ports = (portid_t) count;
 	if (nb_ports == 0)
 		TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
 
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index f51cd9dd..a1f66147 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -57,10 +57,10 @@ typedef uint16_t streamid_t;
 
 #define MAX_QUEUE_ID ((1 << (sizeof(queueid_t) * 8)) - 1)
 
-#if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED
-#define TM_MODE			1
+#if defined RTE_LIBRTE_PMD_SOFTNIC
+#define SOFTNIC			1
 #else
-#define TM_MODE			0
+#define SOFTNIC			0
 #endif
 
 enum {
@@ -135,35 +135,13 @@ struct port_flow {
 	uint8_t data[]; /**< Storage for pattern/actions. */
 };
 
-#ifdef TM_MODE
-/**
- * Soft port tm related parameters
- */
-struct softnic_port_tm {
-	uint32_t default_hierarchy_enable; /**< def hierarchy enable flag */
-	uint32_t hierarchy_config;  /**< set to 1 if hierarchy configured */
-
-	uint32_t n_subports_per_port;  /**< Num of subport nodes per port */
-	uint32_t n_pipes_per_subport;  /**< Num of pipe nodes per subport */
-
-	uint64_t tm_pktfield0_slabpos;	/**< Pkt field position for subport */
-	uint64_t tm_pktfield0_slabmask; /**< Pkt field mask for the subport */
-	uint64_t tm_pktfield0_slabshr;
-	uint64_t tm_pktfield1_slabpos; /**< Pkt field position for the pipe */
-	uint64_t tm_pktfield1_slabmask; /**< Pkt field mask for the pipe */
-	uint64_t tm_pktfield1_slabshr;
-	uint64_t tm_pktfield2_slabpos; /**< Pkt field position table index */
-	uint64_t tm_pktfield2_slabmask;	/**< Pkt field mask for tc table idx */
-	uint64_t tm_pktfield2_slabshr;
-	uint64_t tm_tc_table[64];  /**< TC translation table */
-};
-
+#ifdef SOFTNIC
 /**
  * The data structure associate with softnic port
  */
 struct softnic_port {
-	unsigned int tm_flag;	/**< set to 1 if tm feature is enabled */
-	struct softnic_port_tm tm;	/**< softnic port tm parameters */
+	uint32_t default_tm_hierarchy_enable; /**< default tm hierarchy */
+	struct fwd_lcore **fwd_lcore_arg; /**< softnic fwd core parameters */
 };
 #endif
 
@@ -202,9 +180,8 @@ struct rte_port {
 	uint32_t                mc_addr_nb; /**< nb. of addr. in mc_addr_pool */
 	uint8_t                 slave_flag; /**< bonding slave port */
 	struct port_flow        *flow_list; /**< Associated flows. */
-#ifdef TM_MODE
-	unsigned int			softnic_enable;	/**< softnic flag */
-	struct softnic_port     softport;  /**< softnic port params */
+#ifdef SOFTNIC
+	struct softnic_port     softport;  /**< softnic params */
 #endif
 };
 
@@ -266,9 +243,8 @@ extern struct fwd_engine rx_only_engine;
 extern struct fwd_engine tx_only_engine;
 extern struct fwd_engine csum_fwd_engine;
 extern struct fwd_engine icmp_echo_engine;
-#ifdef TM_MODE
-extern struct fwd_engine softnic_tm_engine;
-extern struct fwd_engine softnic_tm_bypass_engine;
+#ifdef SOFTNIC
+extern struct fwd_engine softnic_fwd_engine;
 #endif
 #ifdef RTE_LIBRTE_IEEE1588
 extern struct fwd_engine ieee1588_fwd_engine;
@@ -479,6 +455,38 @@ struct gso_status {
 extern struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 extern uint16_t gso_max_segment_size;
 
+/* VXLAN encap/decap parameters. */
+struct vxlan_encap_conf {
+	uint32_t select_ipv4:1;
+	uint32_t select_vlan:1;
+	uint8_t vni[3];
+	rte_be16_t udp_src;
+	rte_be16_t udp_dst;
+	rte_be32_t ipv4_src;
+	rte_be32_t ipv4_dst;
+	uint8_t ipv6_src[16];
+	uint8_t ipv6_dst[16];
+	rte_be16_t vlan_tci;
+	uint8_t eth_src[ETHER_ADDR_LEN];
+	uint8_t eth_dst[ETHER_ADDR_LEN];
+};
+struct vxlan_encap_conf vxlan_encap_conf;
+
+/* NVGRE encap/decap parameters. */
+struct nvgre_encap_conf {
+	uint32_t select_ipv4:1;
+	uint32_t select_vlan:1;
+	uint8_t tni[3];
+	rte_be32_t ipv4_src;
+	rte_be32_t ipv4_dst;
+	uint8_t ipv6_src[16];
+	uint8_t ipv6_dst[16];
+	rte_be16_t vlan_tci;
+	uint8_t eth_src[ETHER_ADDR_LEN];
+	uint8_t eth_dst[ETHER_ADDR_LEN];
+};
+struct nvgre_encap_conf nvgre_encap_conf;
+
 static inline unsigned int
 lcore_num(void)
 {
diff --git a/buildtools/Makefile b/buildtools/Makefile
index 35a42ff5..7f76fd7d 100644
--- a/buildtools/Makefile
+++ b/buildtools/Makefile
@@ -1,33 +1,6 @@
-#   BSD LICENSE
-#
-#   Copyright(c) 2016 Neil Horman. All rights reserved.
-#   All rights reserved.
-#
-#   Redistribution and use in source and binary forms, with or without
-#   modification, are permitted provided that the following conditions
-#   are met:
-#
-#     * Redistributions of source code must retain the above copyright
-#       notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright
-#       notice, this list of conditions and the following disclaimer in
-#       the documentation and/or other materials provided with the
-#       distribution.
-#     * Neither the name of Intel Corporation nor the names of its
-#       contributors may be used to endorse or promote products derived
-#       from this software without specific prior written permission.
-#
-#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2016 Neil Horman <nhorman@tuxdriver.com>
+# All rights reserved.
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
diff --git a/buildtools/pmdinfogen/Makefile b/buildtools/pmdinfogen/Makefile
index bf07b6f2..a97a7648 100644
--- a/buildtools/pmdinfogen/Makefile
+++ b/buildtools/pmdinfogen/Makefile
@@ -1,33 +1,6 @@
-#   BSD LICENSE
-#
-#   Copyright(c) 2016 Neil Horman. All rights reserved.
-#   All rights reserved.
-#
-#   Redistribution and use in source and binary forms, with or without
-#   modification, are permitted provided that the following conditions
-#   are met:
-#
-#     * Redistributions of source code must retain the above copyright
-#       notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright
-#       notice, this list of conditions and the following disclaimer in
-#       the documentation and/or other materials provided with the
-#       distribution.
-#     * Neither the name of Intel Corporation nor the names of its
-#       contributors may be used to endorse or promote products derived
-#       from this software without specific prior written permission.
-#
-#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2016 Neil Horman <nhorman@tuxdriver.com>
+# All rights reserved.
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
@@ -41,7 +14,7 @@ HOSTAPP = dpdk-pmdinfogen
 #
 SRCS-y += pmdinfogen.c
 
-HOST_CFLAGS += $(WERROR_FLAGS) -g
+HOST_CFLAGS += $(HOST_WERROR_FLAGS) -g
 HOST_CFLAGS += -I$(RTE_OUTPUT)/include
 
 include $(RTE_SDK)/mk/rte.hostapp.mk
diff --git a/config/arm/arm64_thunderx_linuxapp_gcc b/config/arm/arm64_thunderx_linuxapp_gcc
index 7ff34af7..967d9d46 100644
--- a/config/arm/arm64_thunderx_linuxapp_gcc
+++ b/config/arm/arm64_thunderx_linuxapp_gcc
@@ -2,6 +2,7 @@
 c = 'aarch64-linux-gnu-gcc'
 cpp = 'aarch64-linux-gnu-cpp'
 ar = 'aarch64-linux-gnu-gcc-ar'
+strip = 'aarch64-linux-gnu-strip'
 
 [host_machine]
 system = 'linux'
diff --git a/config/arm/meson.build b/config/arm/meson.build
index 5ad926ed..40dbc87f 100644
--- a/config/arm/meson.build
+++ b/config/arm/meson.build
@@ -64,7 +64,7 @@ flags_dpaa2 = [
 	['RTE_CACHE_LINE_SIZE', 64],
 	['RTE_MAX_NUMA_NODES', 1],
 	['RTE_MAX_LCORE', 16],
-	['RTE_LIBRTE_DPAA2_USE_PHYS_IOVA', 'n']]
+	['RTE_LIBRTE_DPAA2_USE_PHYS_IOVA', false]]
 
 ## Arm implementer ID (ARM DDI 0487C.a, Section G7.2.106, Page G7-5321)
 impl_generic = ['Generic armv8', flags_generic, machine_args_generic]
diff --git a/config/common_base b/config/common_base
index 6b0d1cbb..4bcbaf92 100644
--- a/config/common_base
+++ b/config/common_base
@@ -264,8 +264,6 @@ CONFIG_RTE_LIBRTE_I40E_INC_VECTOR=y
 CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC=n
 CONFIG_RTE_LIBRTE_I40E_QUEUE_NUM_PER_PF=64
 CONFIG_RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM=4
-# interval up to 8160 us, aligned to 2 (or default value)
-CONFIG_RTE_LIBRTE_I40E_ITR_INTERVAL=-1
 
 #
 # Compile burst-oriented FM10K PMD
@@ -402,6 +400,19 @@ CONFIG_RTE_LIBRTE_PMD_FAILSAFE=y
 CONFIG_RTE_LIBRTE_MVPP2_PMD=n
 
 #
+# Compile support for VMBus library
+#
+CONFIG_RTE_LIBRTE_VMBUS=n
+
+#
+# Compile native PMD for Hyper-V/Azure
+#
+CONFIG_RTE_LIBRTE_NETVSC_PMD=n
+CONFIG_RTE_LIBRTE_NETVSC_DEBUG_RX=n
+CONFIG_RTE_LIBRTE_NETVSC_DEBUG_TX=n
+CONFIG_RTE_LIBRTE_NETVSC_DEBUG_DUMP=n
+
+#
 # Compile virtual device driver for NetVSC on Hyper-V/Azure
 #
 CONFIG_RTE_LIBRTE_VDEV_NETVSC_PMD=n
@@ -426,7 +437,7 @@ CONFIG_RTE_PMD_RING_MAX_TX_RINGS=16
 #
 # Compile SOFTNIC PMD
 #
-CONFIG_RTE_LIBRTE_PMD_SOFTNIC=y
+CONFIG_RTE_LIBRTE_PMD_SOFTNIC=n
 
 #
 # Compile the TAP PMD
@@ -460,7 +471,6 @@ CONFIG_RTE_LIBRTE_PMD_BBDEV_TURBO_SW=n
 # Compile generic crypto device library
 #
 CONFIG_RTE_LIBRTE_CRYPTODEV=y
-CONFIG_RTE_LIBRTE_CRYPTODEV_DEBUG=n
 CONFIG_RTE_CRYPTO_MAX_DEVS=64
 
 #
@@ -473,28 +483,23 @@ CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO_DEBUG=n
 # Compile NXP DPAA2 crypto sec driver for CAAM HW
 #
 CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC=n
-CONFIG_RTE_DPAA2_SEC_PMD_MAX_NB_SESSIONS=2048
 
 #
 # NXP DPAA caam - crypto driver
 #
 CONFIG_RTE_LIBRTE_PMD_DPAA_SEC=n
 CONFIG_RTE_LIBRTE_DPAA_MAX_CRYPTODEV=4
-CONFIG_RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS=2048
 
 #
-# Compile PMD for QuickAssist based devices
+# Compile PMD for QuickAssist based devices - see docs for details
 #
-CONFIG_RTE_LIBRTE_PMD_QAT=n
-CONFIG_RTE_LIBRTE_PMD_QAT_DEBUG_INIT=n
-CONFIG_RTE_LIBRTE_PMD_QAT_DEBUG_TX=n
-CONFIG_RTE_LIBRTE_PMD_QAT_DEBUG_RX=n
-CONFIG_RTE_LIBRTE_PMD_QAT_DEBUG_DRIVER=n
+CONFIG_RTE_LIBRTE_PMD_QAT=y
+CONFIG_RTE_LIBRTE_PMD_QAT_SYM=n
 #
-# Number of sessions to create in the session memory pool
-# on a single QuickAssist device.
+# Max. number of QuickAssist devices, which can be detected and attached
 #
-CONFIG_RTE_QAT_PMD_MAX_NB_SESSIONS=2048
+CONFIG_RTE_PMD_QAT_MAX_PCI_DEVICES=48
+CONFIG_RTE_PMD_QAT_COMP_SGL_MAX_SEGMENTS=16
 
 #
 # Compile PMD for virtio crypto devices
@@ -504,29 +509,21 @@ CONFIG_RTE_LIBRTE_PMD_VIRTIO_CRYPTO=y
 # Number of maximum virtio crypto devices
 #
 CONFIG_RTE_MAX_VIRTIO_CRYPTO=32
-#
-# Number of sessions to create in the session memory pool
-# on a single virtio crypto device.
-#
-CONFIG_RTE_VIRTIO_CRYPTO_PMD_MAX_NB_SESSIONS=1024
 
 #
 # Compile PMD for AESNI backed device
 #
 CONFIG_RTE_LIBRTE_PMD_AESNI_MB=n
-CONFIG_RTE_LIBRTE_PMD_AESNI_MB_DEBUG=n
 
 #
 # Compile PMD for Software backed device
 #
 CONFIG_RTE_LIBRTE_PMD_OPENSSL=n
-CONFIG_RTE_LIBRTE_PMD_OPENSSL_DEBUG=n
 
 #
 # Compile PMD for AESNI GCM device
 #
 CONFIG_RTE_LIBRTE_PMD_AESNI_GCM=n
-CONFIG_RTE_LIBRTE_PMD_AESNI_GCM_DEBUG=n
 
 #
 # Compile PMD for SNOW 3G device
@@ -538,19 +535,15 @@ CONFIG_RTE_LIBRTE_PMD_SNOW3G_DEBUG=n
 # Compile PMD for KASUMI device
 #
 CONFIG_RTE_LIBRTE_PMD_KASUMI=n
-CONFIG_RTE_LIBRTE_PMD_KASUMI_DEBUG=n
 
 #
 # Compile PMD for ZUC device
 #
 CONFIG_RTE_LIBRTE_PMD_ZUC=n
-CONFIG_RTE_LIBRTE_PMD_ZUC_DEBUG=n
 
-#
 # Compile PMD for Crypto Scheduler device
 #
 CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER=y
-CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER_DEBUG=n
 
 #
 # Compile PMD for NULL Crypto device
@@ -585,11 +578,21 @@ CONFIG_RTE_COMPRESS_MAX_DEVS=64
 CONFIG_RTE_COMPRESSDEV_TEST=n
 
 #
+# Compile PMD for Octeontx ZIPVF compression device
+#
+CONFIG_RTE_LIBRTE_PMD_OCTEONTX_ZIPVF=y
+
+#
 # Compile PMD for ISA-L compression device
 #
 CONFIG_RTE_LIBRTE_PMD_ISAL=n
 
 #
+# Compile PMD for ZLIB compression device
+#
+CONFIG_RTE_LIBRTE_PMD_ZLIB=n
+
+#
 # Compile generic event device library
 #
 CONFIG_RTE_LIBRTE_EVENTDEV=y
@@ -597,6 +600,7 @@ CONFIG_RTE_LIBRTE_EVENTDEV_DEBUG=n
 CONFIG_RTE_EVENT_MAX_DEVS=16
 CONFIG_RTE_EVENT_MAX_QUEUES_PER_DEV=64
 CONFIG_RTE_EVENT_TIMER_ADAPTER_NUM_MAX=32
+CONFIG_RTE_EVENT_ETH_INTR_RING_SIZE=1024
 CONFIG_RTE_EVENT_CRYPTO_ADAPTER_MAX_INSTANCE=32
 
 #
@@ -861,11 +865,11 @@ CONFIG_RTE_LIBRTE_VHOST_DEBUG=n
 CONFIG_RTE_LIBRTE_PMD_VHOST=n
 
 #
-# Compile IFCVF driver
+# Compile IFC driver
 # To compile, CONFIG_RTE_LIBRTE_VHOST and CONFIG_RTE_EAL_VFIO
 # should be enabled.
 #
-CONFIG_RTE_LIBRTE_IFCVF_VDPA_PMD=n
+CONFIG_RTE_LIBRTE_IFC_PMD=n
 
 #
 # Compile librte_bpf
diff --git a/config/common_linuxapp b/config/common_linuxapp
index 5c68cc0f..9c5ea9d8 100644
--- a/config/common_linuxapp
+++ b/config/common_linuxapp
@@ -15,8 +15,9 @@ CONFIG_RTE_LIBRTE_PMD_KNI=y
 CONFIG_RTE_LIBRTE_VHOST=y
 CONFIG_RTE_LIBRTE_VHOST_NUMA=y
 CONFIG_RTE_LIBRTE_PMD_VHOST=y
-CONFIG_RTE_LIBRTE_IFCVF_VDPA_PMD=y
+CONFIG_RTE_LIBRTE_IFC_PMD=y
 CONFIG_RTE_LIBRTE_PMD_AF_PACKET=y
+CONFIG_RTE_LIBRTE_PMD_SOFTNIC=y
 CONFIG_RTE_LIBRTE_PMD_TAP=y
 CONFIG_RTE_LIBRTE_AVP_PMD=y
 CONFIG_RTE_LIBRTE_VDEV_NETVSC_PMD=y
@@ -25,6 +26,9 @@ CONFIG_RTE_LIBRTE_POWER=y
 CONFIG_RTE_VIRTIO_USER=y
 CONFIG_RTE_PROC_INFO=y
 
+CONFIG_RTE_LIBRTE_VMBUS=y
+CONFIG_RTE_LIBRTE_NETVSC_PMD=y
+
 # NXP DPAA BUS and drivers
 CONFIG_RTE_LIBRTE_DPAA_BUS=y
 CONFIG_RTE_LIBRTE_DPAA_MEMPOOL=y
diff --git a/config/meson.build b/config/meson.build
index 50081b57..4d755323 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -57,9 +57,12 @@ add_project_arguments('-include', 'rte_config.h', language: 'c')
 warning_flags = [
 	'-Wsign-compare',
 	'-Wcast-qual',
-	'-Wno-address-of-packed-member',
-	'-Wno-format-truncation'
+	'-Wno-address-of-packed-member'
 ]
+if cc.sizeof('void *') == 4
+# for 32-bit, don't warn about casting a 32-bit pointer to 64-bit int - it's fine!!
+	warning_flags += '-Wno-pointer-to-int-cast'
+endif
 foreach arg: warning_flags
 	if cc.has_argument(arg)
 		add_project_arguments(arg, language: 'c')
@@ -74,6 +77,7 @@ dpdk_conf.set('RTE_EAL_ALLOW_INV_SOCKET_ID', get_option('allow_invalid_socket_id
 # values which have defaults which may be overridden
 dpdk_conf.set('RTE_MAX_VFIO_GROUPS', 64)
 dpdk_conf.set('RTE_DRIVER_MEMPOOL_BUCKET_SIZE_KB', 64)
+dpdk_conf.set('RTE_LIBRTE_DPAA2_USE_PHYS_IOVA', true)
 
 compile_time_cpuflags = []
 if host_machine.cpu_family().startswith('x86')
diff --git a/config/rte_config.h b/config/rte_config.h
index a1d01759..a8e47977 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -64,6 +64,7 @@
 #define RTE_EVENT_MAX_DEVS 16
 #define RTE_EVENT_MAX_QUEUES_PER_DEV 64
 #define RTE_EVENT_TIMER_ADAPTER_NUM_MAX 32
+#define RTE_EVENT_ETH_INTR_RING_SIZE 1024
 #define RTE_EVENT_CRYPTO_ADAPTER_MAX_INSTANCE 32
 
 /* rawdev defines */
@@ -85,23 +86,14 @@
 
 /****** driver defines ********/
 
-/*
- * Number of sessions to create in the session memory pool
- * on a single instance of crypto HW device.
- */
 /* QuickAssist device */
-#define RTE_QAT_PMD_MAX_NB_SESSIONS 2048
+/* Max. number of QuickAssist devices which can be attached */
+#define RTE_PMD_QAT_MAX_PCI_DEVICES 48
+#define RTE_PMD_QAT_COMP_SGL_MAX_SEGMENTS 16
 
 /* virtio crypto defines */
-#define RTE_VIRTIO_CRYPTO_PMD_MAX_NB_SESSIONS 1024
 #define RTE_MAX_VIRTIO_CRYPTO 32
 
-/* DPAA2_SEC */
-#define RTE_DPAA2_SEC_PMD_MAX_NB_SESSIONS 2048
-
-/* DPAA_SEC */
-#define RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS 2048
-
 /* DPAA SEC max cryptodev devices*/
 #define RTE_LIBRTE_DPAA_MAX_CRYPTODEV	4
 
diff --git a/devtools/check-symbol-change.sh b/devtools/check-symbol-change.sh
new file mode 100755
index 00000000..daaf45e1
--- /dev/null
+++ b/devtools/check-symbol-change.sh
@@ -0,0 +1,159 @@
+#!/bin/sh
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Neil Horman <nhorman@tuxdriver.com>
+
+build_map_changes()
+{
+	local fname="$1"
+	local mapdb="$2"
+
+	cat "$fname" | awk '
+		# Initialize our variables
+		BEGIN {map="";sym="";ar="";sec=""; in_sec=0; in_map=0}
+
+		# Anything that starts with + or -, followed by an a
+		# and ends in the string .map is the name of our map file
+		# This may appear multiple times in a patch if multiple
+		# map files are altered, and all section/symbol names
+		# appearing between a triggering of this rule and the
+		# next trigger of this rule are associated with this file
+		/[-+] a\/.*\.map/ {map=$2; in_map=1}
+
+		# Same pattern as above, only it matches on anything that
+		# does not end in 'map', indicating we have left the map chunk.
+		# When we hit this, turn off the in_map variable, which
+		# supresses the subordonate rules below
+		/[-+] a\/.*\.^(map)/ {in_map=0}
+
+		# Triggering this rule, which starts a line with a + and ends it
+		# with a { identifies a versioned section.  The section name is
+		# the rest of the line with the + and { symbols remvoed.
+		# Triggering this rule sets in_sec to 1, which actives the
+		# symbol rule below
+		/+.*{/ {gsub("+","");
+			if (in_map == 1) {
+				sec=$1; in_sec=1;
+			}
+		}
+
+		# This rule idenfies the end of a section, and disables the
+		# symbol rule
+		/.*}/ {in_sec=0}
+
+		# This rule matches on a + followed by any characters except a :
+		# (which denotes a global vs local segment), and ends with a ;.
+		# The semicolon is removed and the symbol is printed with its
+		# association file name and version section, along with an
+		# indicator that the symbol is a new addition.  Note this rule
+		# only works if we have found a version section in the rule
+		# above (hence the in_sec check) And found a map file (the
+		# in_map check).  If we are not in a map chunk, do nothing.  If
+		# we are in a map chunk but not a section chunk, record it as
+		# unknown.
+		/^+[^}].*[^:*];/ {gsub(";","");sym=$2;
+			if (in_map == 1) {
+				if (in_sec == 1) {
+					print map " " sym " " sec " add"
+				} else {
+					print map " " sym " unknown add"
+				}
+			}
+		}
+
+		# This is the same rule as above, but the rule matches on a
+		# leading - rather than a +, denoting that the symbol is being
+		# removed.
+		/^-[^}].*[^:*];/ {gsub(";","");sym=$2;
+			if (in_map == 1) {
+				if (in_sec == 1) {
+					print map " " sym " " sec " del"
+				} else {
+					print map " " sym " unknown del"
+				}
+			}
+		}' > "$mapdb"
+
+		sort -u "$mapdb" > "$mapdb.2"
+		mv -f "$mapdb.2" "$mapdb"
+
+}
+
+check_for_rule_violations()
+{
+	local mapdb="$1"
+	local mname
+	local symname
+	local secname
+	local ar
+	local ret=0
+
+	while read mname symname secname ar
+	do
+		if [ "$ar" = "add" ]
+		then
+
+			if [ "$secname" = "unknown" ]
+			then
+				# Just inform the user of this occurrence, but
+				# don't flag it as an error
+				echo -n "INFO: symbol $syname is added but "
+				echo -n "patch has insuficient context "
+				echo -n "to determine the section name "
+				echo -n "please ensure the version is "
+				echo "EXPERIMENTAL"
+				continue
+			fi
+
+			if [ "$secname" != "EXPERIMENTAL" ]
+			then
+				# Symbols that are getting added in a section
+				# other than the experimental section
+				# to be moving from an already supported
+				# section or its a violation
+				grep -q \
+				"$mname $symname [^EXPERIMENTAL] del" "$mapdb"
+				if [ $? -ne 0 ]
+				then
+					echo -n "ERROR: symbol $symname "
+					echo -n "is added in a section "
+					echo -n "other than the EXPERIMENTAL "
+					echo "section of the version map"
+					ret=1
+				fi
+			fi
+		else
+
+			if [ "$secname" != "EXPERIMENTAL" ]
+			then
+				# Just inform users that non-experimenal
+				# symbols need to go through a deprecation
+				# process
+				echo -n "INFO: symbol $symname is being "
+				echo -n "removed, ensure that it has "
+				echo "gone through the deprecation process"
+			fi
+		fi
+	done < "$mapdb"
+
+	return $ret
+}
+
+trap clean_and_exit_on_sig EXIT
+
+mapfile=`mktemp mapdb.XXXXXX`
+patch=$1
+exit_code=1
+
+clean_and_exit_on_sig()
+{
+	rm -f "$mapfile"
+	exit $exit_code
+}
+
+build_map_changes "$patch" "$mapfile"
+check_for_rule_violations "$mapfile"
+exit_code=$?
+
+rm -f "$mapfile"
+
+exit $exit_code
diff --git a/devtools/checkpatches.sh b/devtools/checkpatches.sh
index 245d5ab1..ba795ad1 100755
--- a/devtools/checkpatches.sh
+++ b/devtools/checkpatches.sh
@@ -7,6 +7,8 @@
 # - DPDK_CHECKPATCH_LINE_LENGTH
 . $(dirname $(readlink -e $0))/load-devel-config
 
+VALIDATE_NEW_API=$(dirname $(readlink -e $0))/check-symbol-change.sh
+
 length=${DPDK_CHECKPATCH_LINE_LENGTH:-80}
 
 # override default Linux options
@@ -14,13 +16,21 @@ options="--no-tree"
 options="$options --max-line-length=$length"
 options="$options --show-types"
 options="$options --ignore=LINUX_VERSION_CODE,\
-FILE_PATH_CHANGES,MAINTAINERS_STYLE,\
+FILE_PATH_CHANGES,MAINTAINERS_STYLE,SPDX_LICENSE_TAG,\
 VOLATILE,PREFER_PACKED,PREFER_ALIGNED,PREFER_PRINTF,\
 PREFER_KERNEL_TYPES,BIT_MACRO,CONST_STRUCT,\
 SPLIT_STRING,LONG_LINE_STRING,\
 LINE_SPACING,PARENTHESIS_ALIGNMENT,NETWORKING_BLOCK_COMMENT_STYLE,\
 NEW_TYPEDEFS,COMPARISON_TO_NULL"
 
+clean_tmp_files() {
+	if echo $tmpinput | grep -q '^checkpatches\.' ; then
+		rm -f "$tmpinput"
+	fi
+}
+
+trap "clean_tmp_files" INT
+
 print_usage () {
 	cat <<- END_OF_HELP
 	usage: $(basename $0) [-q] [-v] [-nX|patch1 [patch2] ...]]
@@ -33,6 +43,88 @@ print_usage () {
 	END_OF_HELP
 }
 
+check_forbidden_additions() {
+    # This awk script receives a list of expressions to monitor
+    # and a list of folders to search these expressions in
+    # - No search is done inside comments
+    # - Both additions and removals of the expressions are checked
+    #   A positive balance of additions fails the check
+	read -d '' awk_script << 'EOF'
+	BEGIN {
+		split(FOLDERS,deny_folders," ");
+		split(EXPRESSIONS,deny_expr," ");
+		in_file=0;
+		in_comment=0;
+		count=0;
+		comment_start="/*"
+		comment_end="*/"
+	}
+	# search for add/remove instances in current file
+	# state machine assumes the comments structure is enforced by
+	# checkpatches.pl
+	(in_file) {
+		# comment start
+		if (index($0,comment_start) > 0) {
+			in_comment = 1
+		}
+		# non comment code
+		if (in_comment == 0) {
+			for (i in deny_expr) {
+				forbidden_added = "^\+.*" deny_expr[i];
+				forbidden_removed="^-.*" deny_expr[i];
+				current = expressions[deny_expr[i]]
+				if ($0 ~ forbidden_added) {
+					count = count + 1;
+					expressions[deny_expr[i]] = current + 1
+				}
+				if ($0 ~ forbidden_removed) {
+					count = count - 1;
+					expressions[deny_expr[i]] = current - 1
+				}
+			}
+		}
+		# comment end
+		if (index($0,comment_end) > 0) {
+			in_comment = 0
+		}
+	}
+	# switch to next file , check if the balance of add/remove
+	# of previous filehad new additions
+	($0 ~ "^\+\+\+ b/") {
+		in_file = 0;
+		if (count > 0) {
+			exit;
+		}
+		for (i in deny_folders) {
+			re = "^\+\+\+ b/" deny_folders[i];
+			if ($0 ~ deny_folders[i]) {
+				in_file = 1
+				last_file = $0
+			}
+		}
+	}
+	END {
+		if (count > 0) {
+			print "Warning in " substr(last_file,6) ":"
+			print "are you sure you want to add the following:"
+			for (key in expressions) {
+				if (expressions[key] > 0) {
+					print key
+				}
+			}
+			exit RET_ON_FAIL
+		}
+	}
+EOF
+	# ---------------------------------
+	# refrain from new additions of rte_panic() and rte_exit()
+	# multiple folders and expressions are separated by spaces
+	awk -v FOLDERS="lib drivers" \
+		-v EXPRESSIONS="rte_panic\\\( rte_exit\\\(" \
+		-v RET_ON_FAIL=1 \
+		"$awk_script" -
+}
+
 number=0
 quiet=false
 verbose=false
@@ -58,19 +150,45 @@ total=0
 status=0
 
 check () { # <patch> <commit> <title>
+	local ret=0
+
 	total=$(($total + 1))
 	! $verbose || printf '\n### %s\n\n' "$3"
 	if [ -n "$1" ] ; then
-		report=$($DPDK_CHECKPATCH_PATH $options "$1" 2>/dev/null)
+		tmpinput=$1
 	elif [ -n "$2" ] ; then
-		report=$(git format-patch --find-renames --no-stat --stdout -1 $commit |
-			$DPDK_CHECKPATCH_PATH $options - 2>/dev/null)
+		tmpinput=$(mktemp checkpatches.XXXXXX)
+		git format-patch --find-renames \
+		--no-stat --stdout -1 $commit > "$tmpinput"
 	else
-		report=$($DPDK_CHECKPATCH_PATH $options - 2>/dev/null)
+		tmpinput=$(mktemp checkpatches.XXXXXX)
+		cat > "$tmpinput"
 	fi
-	[ $? -ne 0 ] || return 0
-	$verbose || printf '\n### %s\n\n' "$3"
-	printf '%s\n' "$report" | sed -n '1,/^total:.*lines checked$/p'
+
+	report=$($DPDK_CHECKPATCH_PATH $options "$tmpinput" 2>/dev/null)
+	if [ $? -ne 0 ] ; then
+		$verbose || printf '\n### %s\n\n' "$3"
+		printf '%s\n' "$report" | sed -n '1,/^total:.*lines checked$/p'
+		ret=1
+	fi
+
+	! $verbose || printf '\nChecking API additions/removals:\n'
+	report=$($VALIDATE_NEW_API "$tmpinput")
+	if [ $? -ne 0 ] ; then
+		printf '%s\n' "$report"
+		ret=1
+	fi
+
+	! $verbose || printf '\nChecking forbidden tokens additions:\n'
+	report=$(check_forbidden_additions <"$tmpinput")
+	if [ $? -ne 0 ] ; then
+		printf '%s\n' "$report"
+		ret=1
+	fi
+
+	clean_tmp_files
+	[ $ret -eq 0 ] && return 0
+
 	status=$(($status + 1))
 }
 
diff --git a/devtools/test-build.sh b/devtools/test-build.sh
index 3ea82bb4..1eee2417 100755
--- a/devtools/test-build.sh
+++ b/devtools/test-build.sh
@@ -153,7 +153,8 @@ config () # <directory> <target> <options>
 		sed -ri       's,(PMD_SZEDATA2=)n,\1y,' $1/.config
 		test "$DPDK_DEP_ZLIB" != y || \
 		sed -ri          's,(BNX2X_PMD=)n,\1y,' $1/.config
-		sed -ri            's,(NFP_PMD=)n,\1y,' $1/.config
+		test "$DPDK_DEP_ZLIB" != y || \
+		sed -ri   's,(COMPRESSDEV_TEST=)n,\1y,' $1/.config
 		test "$DPDK_DEP_PCAP" != y || \
 		sed -ri               's,(PCAP=)n,\1y,' $1/.config
 		test -z "$ARMV8_CRYPTO_LIB_PATH" || \
diff --git a/devtools/test-meson-builds.sh b/devtools/test-meson-builds.sh
index 9868c325..951c9067 100755
--- a/devtools/test-meson-builds.sh
+++ b/devtools/test-meson-builds.sh
@@ -10,6 +10,15 @@
 srcdir=$(dirname $(readlink -m $0))/..
 MESON=${MESON:-meson}
 
+if command -v ninja >/dev/null 2>&1 ; then
+	ninja_cmd=ninja
+elif command -v ninja-build >/dev/null 2>&1 ; then
+	ninja_cmd=ninja-build
+else
+	echo "ERROR: ninja is not found" >&2
+	exit 1
+fi
+
 build () # <directory> <meson options>
 {
 	builddir=$1
@@ -20,8 +29,8 @@ build () # <directory> <meson options>
 		$MESON $options $srcdir $builddir
 		unset CC
 	fi
-	echo "ninja -C $builddir"
-	ninja -C $builddir
+	echo "$ninja_cmd -C $builddir"
+	$ninja_cmd -C $builddir
 }
 
 # shared and static linked builds with gcc and clang
@@ -36,11 +45,16 @@ done
 build build-x86-default -Dmachine=nehalem
 
 # enable cross compilation if gcc cross-compiler is found
-for f in config/arm/arm*gcc ; do
-	c=aarch64-linux-gnu-gcc
-	if ! command -v $c >/dev/null 2>&1 ; then
-		continue
-	fi
-	export CC="ccache $c"
-	build build-$(basename $f | tr '_' '-' | cut -d'-' -f-2) --cross-file $f
-done
+c=aarch64-linux-gnu-gcc
+if command -v $c >/dev/null 2>&1 ; then
+	# compile the general v8a also for clang to increase coverage
+	export CC="ccache clang"
+	build build-arm64-host-clang --cross-file \
+		config/arm/arm64_armv8_linuxapp_gcc
+
+	for f in config/arm/arm*gcc ; do
+		export CC="ccache gcc"
+		build build-$(basename $f | tr '_' '-' | cut -d'-' -f-2) \
+			--cross-file $f
+	done
+fi
diff --git a/doc/guides/bbdevs/turbo_sw.rst b/doc/guides/bbdevs/turbo_sw.rst
index 9661a306..0b96fbb1 100644
--- a/doc/guides/bbdevs/turbo_sw.rst
+++ b/doc/guides/bbdevs/turbo_sw.rst
@@ -47,8 +47,8 @@ Design Center <https://www.intel.com/content/www/us/en/design/resource-design-ce
 needs to be registered.
 
 Once registered, the user needs to log in, and look for
-*Intel FlexRAN Software Release Package 1_4_0* to download or directly through
-this `link <https://cdrdv2.intel.com/v1/dl/getContent/576288>`_.
+*Intel FlexRAN Software Release Package -1-6-0* to download or directly through
+this `link <https://cdrdv2.intel.com/v1/dl/getContent/600609>`_.
 
 After download is complete, the user needs to unpack and compile on their
 system before building DPDK.
@@ -64,6 +64,7 @@ The following table maps DPDK versions with past FlexRAN SDK releases:
    =====================  ============================
    18.02                  1.3.0
    18.05                  1.4.0
+   18.08                  1.6.0
    =====================  ============================
 
 FlexRAN SDK Installation
@@ -71,8 +72,8 @@ FlexRAN SDK Installation
 
 The following are pre-requisites for building FlexRAN SDK Libraries:
  (a) An AVX2 supporting machine
- (b) Windriver TS 2 or CentOS 7 operating systems
- (c) Intel ICC 17.0.3 compiler installed
+ (b) CentOS Linux release 7.2.1511 (Core) operating system
+ (c) Intel ICC 18.0.1 20171018 compiler installed
 
 The following instructions should be followed in this exact order:
 
@@ -82,29 +83,26 @@ The following instructions should be followed in this exact order:
 
         source <path-to-icc-compiler-install-folder>/linux/bin/compilervars.sh intel64 -platform linux
 
-
-#. Extract the ``576288-576288-intel-swa-sw-flexran-release-package-r1-4-0.zip``
-   package, then run the SDK extractor script and accept the license:
+#. Extract the ``flexran-1-6-0-tar.gz.zip`` package:
 
     .. code-block:: console
 
-        cd <path-to-workspace>/FlexRAN-1.4.0/
-        ./SDK-R1.4.0.sh
+        unzip flexran-1-6-0-tar.gz.zip
+        tar xvzf flexran-1-6-0-tar.gz -C FlexRAN-1.6.0/
 
-#. Generate makefiles based on system configuration:
+#. Run the SDK extractor script and accept the license:
 
     .. code-block:: console
 
-        cd <path-to-workspace>/FlexRAN-1.4.0/SDK-R1.4.0/sdk/
-        ./create-makefiles-linux.sh
+        cd <path-to-workspace>/FlexRAN-1.6.0/
+        ./SDK-R1.6.0.sh
 
-#. If build as a shared library is required, then add ``"-fPIC"`` option to file
-   ``<path-to-workspace>/FlexRAN-1.4.0/SDK-R1.4.0/sdk/cmake/intel-compile-options.cmake``
-   as follows:
+#. Generate makefiles based on system configuration:
 
     .. code-block:: console
 
-        add_compile_options("-fPIC")
+        cd <path-to-workspace>/FlexRAN-1.6.0/SDK-R1.6.0/sdk/
+        ./create-makefiles-linux.sh
 
 #. A build folder is generated in this form ``build-<ISA>-<CC>``, enter that
    folder and install:
@@ -112,7 +110,7 @@ The following instructions should be followed in this exact order:
     .. code-block:: console
 
         cd build-avx2-icc/
-        make install
+        make && make install
 
 
 Initialization
@@ -130,8 +128,8 @@ Example:
 
 .. code-block:: console
 
-    export FLEXRAN_SDK=<path-to-workspace>/FlexRAN-1.4.0/SDK-R1.4.0/sdk/build-avx2-icc/install
-    export DIR_WIRELESS_SDK=<path-to-workspace>/FlexRAN-1.4.0/SDK-R1.4.0/sdk/
+    export FLEXRAN_SDK=<path-to-workspace>/FlexRAN-1.6.0/SDK-R1.6.0/sdk/build-avx2-icc/install
+    export DIR_WIRELESS_SDK=<path-to-workspace>/FlexRAN-1.6.0/SDK-R1.6.0/sdk/
 
 
 * Set ``CONFIG_RTE_LIBRTE_PMD_BBDEV_TURBO_SW=y`` in DPDK common configuration
@@ -156,4 +154,4 @@ Example:
 .. code-block:: console
 
     ./test-bbdev.py -e="--vdev=baseband_turbo_sw,socket_id=0,max_nb_queues=8" \
-    -c validation -v ./test_vectors/bbdev_vector_t?_default.data
+    -c validation -v ./turbo_*_default.data
diff --git a/doc/guides/compressdevs/features/default.ini b/doc/guides/compressdevs/features/default.ini
index 795fc557..829e4df6 100644
--- a/doc/guides/compressdevs/features/default.ini
+++ b/doc/guides/compressdevs/features/default.ini
@@ -6,19 +6,21 @@
 ; the features table in the documentation.
 ;
 [Features]
-HW Accelerated =
-CPU SSE        =
-CPU AVX        =
-CPU AVX2       =
-CPU AVX512     =
-CPU NEON       =
-Stateful       =
-By-Pass        =
-Chained mbufs  =
-Deflate        =
-LZS            =
-Adler32        =
-Crc32          =
-Adler32&Crc32  =
-Fixed          =
-Dynamic        =
+HW Accelerated      =
+CPU SSE             =
+CPU AVX             =
+CPU AVX2            =
+CPU AVX512          =
+CPU NEON            =
+Stateful            =
+Pass-through        =
+OOP SGL In SGL Out  =
+OOP SGL In LB  Out  =
+OOP LB  In SGL Out  =
+Deflate             =
+LZS                 =
+Adler32             =
+Crc32               =
+Adler32&Crc32       =
+Fixed               =
+Dynamic             =
diff --git a/doc/guides/compressdevs/features/isal.ini b/doc/guides/compressdevs/features/isal.ini
index ad2718df..919cf705 100644
--- a/doc/guides/compressdevs/features/isal.ini
+++ b/doc/guides/compressdevs/features/isal.ini
@@ -4,19 +4,13 @@
 ; Supported features of 'ISA-L' compression driver.
 ;
 [Features]
-HW Accelerated =
-CPU SSE        = Y
-CPU AVX        = Y
-CPU AVX2       = Y
-CPU AVX512     = Y
-CPU NEON       =
-Stateful       =
-By-Pass        =
-Chained mbufs  =
-Deflate        = Y
-LZS            =
-Adler32        =
-Crc32          =
-Adler32&Crc32  =
-Fixed          = Y
-Dynamic        = Y
+CPU SSE            = Y
+CPU AVX            = Y
+CPU AVX2           = Y
+CPU AVX512         = Y
+OOP SGL In SGL Out = Y
+OOP SGL In LB  Out = Y
+OOP LB  In SGL Out = Y
+Deflate            = Y
+Fixed              = Y
+Dynamic            = Y
diff --git a/doc/guides/compressdevs/features/octeontx.ini b/doc/guides/compressdevs/features/octeontx.ini
new file mode 100644
index 00000000..884a8b07
--- /dev/null
+++ b/doc/guides/compressdevs/features/octeontx.ini
@@ -0,0 +1,10 @@
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+; Supported features of 'OCTEONTX ZIP' compression driver.
+;
+[Features]
+HW Accelerated = Y
+Deflate        = Y
+Fixed          = Y
+Dynamic        = Y
diff --git a/doc/guides/compressdevs/features/qat.ini b/doc/guides/compressdevs/features/qat.ini
new file mode 100644
index 00000000..5cd4524b
--- /dev/null
+++ b/doc/guides/compressdevs/features/qat.ini
@@ -0,0 +1,15 @@
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+; Supported features of 'QAT' compression driver.
+;
+[Features]
+HW Accelerated      = Y
+OOP SGL In SGL Out  = Y
+OOP SGL In LB  Out  = Y
+OOP LB  In SGL Out  = Y
+Deflate             = Y
+Adler32             = Y
+Crc32               = Y
+Adler32&Crc32       = Y
+Fixed               = Y
diff --git a/doc/guides/compressdevs/features/zlib.ini b/doc/guides/compressdevs/features/zlib.ini
new file mode 100644
index 00000000..58a4ee3a
--- /dev/null
+++ b/doc/guides/compressdevs/features/zlib.ini
@@ -0,0 +1,10 @@
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+; Supported features of 'ZLIB' compression driver.
+;
+[Features]
+Pass-through   = Y
+Deflate        = Y
+Fixed          = Y
+Dynamic        = Y
diff --git a/doc/guides/compressdevs/index.rst b/doc/guides/compressdevs/index.rst
index bc59ce81..1f37e260 100644
--- a/doc/guides/compressdevs/index.rst
+++ b/doc/guides/compressdevs/index.rst
@@ -11,3 +11,6 @@ Compression Device Drivers
 
     overview
     isal
+    octeontx
+    qat_comp
+    zlib
diff --git a/doc/guides/compressdevs/isal.rst b/doc/guides/compressdevs/isal.rst
index 276ff063..3bc30229 100644
--- a/doc/guides/compressdevs/isal.rst
+++ b/doc/guides/compressdevs/isal.rst
@@ -78,8 +78,6 @@ As a result the level mappings from the API to the PMD are shown below.
 Limitations
 -----------
 
-* Chained mbufs will not be supported until a future release, meaning max data size being passed to PMD through a single mbuf is 64K - 1. If data is larger than this it will need to be split up and sent as multiple operations.
-
 * Compressdev level 0, no compression, is not supported.
 
 * Checksums will not be supported until future release.
diff --git a/doc/guides/compressdevs/octeontx.rst b/doc/guides/compressdevs/octeontx.rst
new file mode 100644
index 00000000..5a32d5d1
--- /dev/null
+++ b/doc/guides/compressdevs/octeontx.rst
@@ -0,0 +1,105 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2018 Cavium Networks.
+
+Octeontx ZIP Compression Poll Mode Driver
+=========================================
+
+The Octeontx ZIP PMD (**librte_pmd_octeontx_zip**) provides poll mode
+compression & decompression driver for ZIP HW offload device, found in
+**Cavium OCTEONTX** SoC family.
+
+More information can be found at `Cavium, Inc Official Website
+<http://www.cavium.com/OCTEON-TX_ARM_Processors.html>`_.
+
+Features
+--------
+
+Octeontx ZIP PMD has support for:
+
+Compression/Decompression algorithm:
+
+* DEFLATE
+
+Huffman code type:
+
+* FIXED
+* DYNAMIC
+
+Window size support:
+
+* 2 to 2^14
+
+Limitations
+-----------
+
+* Chained mbufs are not supported.
+
+Supported OCTEONTX SoCs
+-----------------------
+
+- CN83xx
+
+Steps To Setup Platform
+-----------------------
+
+   Octeontx SDK includes kernel image which provides Octeontx ZIP PF
+   driver to manage configuration of ZIPVF device
+   Required version of SDK is "OCTEONTX-SDK-6.2.0-build35" or above.
+
+   SDK can be install by using below command.
+   #rpm -ivh CTEONTX-SDK-6.2.0-build35.x86_64.rpm --force --nodeps
+   It will install OCTEONTX-SDK at following default location
+   /usr/local/Cavium_Networks/OCTEONTX-SDK/
+
+   For more information on building and booting linux kernel on OCTEONTX
+   please refer /usr/local/Cavium_Networks/OCTEONTX-SDK/docs/OcteonTX-SDK-UG_6.2.0.pdf.
+
+   SDK and related information can be obtained from: `Cavium support site <https://support.cavium.com/>`_.
+
+Installation
+------------
+
+Driver Compilation
+~~~~~~~~~~~~~~~~~~
+
+To compile the OCTEONTX ZIP PMD for Linux arm64 gcc target, run the
+following ``make`` command:
+
+   .. code-block:: console
+
+      cd <DPDK-source-directory>
+      make config T=arm64-thunderx-linuxapp-gcc install
+
+
+Initialization
+--------------
+
+The octeontx zip is exposed as pci device which consists of a set of
+PCIe VF devices. On EAL initialization, ZIP PCIe VF devices will be
+probed. To use the PMD in an application, user must:
+
+* run dev_bind script to bind eight ZIP PCIe VFs to the ``vfio-pci`` driver:
+
+   .. code-block:: console
+
+      ./usertools/dpdk-devbind.py -b vfio-pci 0001:04:00.1
+      ./usertools/dpdk-devbind.py -b vfio-pci 0001:04:00.2
+      ./usertools/dpdk-devbind.py -b vfio-pci 0001:04:00.3
+      ./usertools/dpdk-devbind.py -b vfio-pci 0001:04:00.4
+      ./usertools/dpdk-devbind.py -b vfio-pci 0001:04:00.5
+      ./usertools/dpdk-devbind.py -b vfio-pci 0001:04:00.6
+      ./usertools/dpdk-devbind.py -b vfio-pci 0001:04:00.7
+      ./usertools/dpdk-devbind.py -b vfio-pci 0001:04:01.0
+
+* The unit test cases can be tested as below:
+
+   .. code-block:: console
+
+      reserve enough huge pages
+      cd to the top-level DPDK directory
+      export RTE_TARGET=arm64-thunderx-linuxapp-gcc
+      export RTE_SDK=`pwd`
+      cd to test/test
+      type the command "make" to compile
+      run the tests with "./test"
+      type the command "compressdev_autotest" to test
diff --git a/doc/guides/compressdevs/overview.rst b/doc/guides/compressdevs/overview.rst
index ca37de17..70bbe82b 100644
--- a/doc/guides/compressdevs/overview.rst
+++ b/doc/guides/compressdevs/overview.rst
@@ -10,3 +10,23 @@ Supported Feature Flags
 .. _table_compression_pmd_features:
 
 .. include:: overview_feature_table.txt
+
+.. Note::
+
+   - "Pass-through" feature flag refers to the ability of the PMD
+     to let input buffers pass-through it, copying the input to the output,
+     without making any modifications to it (no compression done).
+
+   - "OOP SGL In SGL Out" feature flag stands for
+     "Out-of-place Scatter-gather list Input, Scatter-gater list Output",
+     which means PMD supports different scatter-gather styled input and output buffers
+     (i.e. both can consists of multiple segments).
+
+   - "OOP SGL In LB Out" feature flag stands for
+     "Out-of-place Scatter-gather list Input, Linear Buffers Output",
+     which means PMD supports input from scatter-gathered styled buffers, outputting linear buffers
+     (i.e. single segment).
+
+   - "OOP LB In SGL Out" feature flag stands for
+     "Out-of-place Linear Buffers Input, Scatter-gather list Output",
+     which means PMD supports input from linear buffer, outputting scatter-gathered styled buffers.
diff --git a/doc/guides/compressdevs/qat_comp.rst b/doc/guides/compressdevs/qat_comp.rst
new file mode 100644
index 00000000..8b1270b7
--- /dev/null
+++ b/doc/guides/compressdevs/qat_comp.rst
@@ -0,0 +1,47 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2018 Intel Corporation.
+
+Intel(R) QuickAssist (QAT) Compression Poll Mode Driver
+=======================================================
+
+The QAT compression PMD provides poll mode compression & decompression driver
+support for the following hardware accelerator devices:
+
+* ``Intel QuickAssist Technology C62x``
+* ``Intel QuickAssist Technology C3xxx``
+
+
+Features
+--------
+
+QAT compression PMD has support for:
+
+Compression/Decompression algorithm:
+
+    * DEFLATE
+
+Huffman code type:
+
+    * FIXED
+
+Window size support:
+
+    * 32K
+
+Checksum generation:
+
+    * CRC32, Adler and combined checksum
+
+Limitations
+-----------
+
+* Compressdev level 0, no compression, is not supported.
+
+* Dynamic Huffman encoding is not yet supported.
+
+Installation
+------------
+
+The QAT compression PMD is built by default with a standard DPDK build.
+
+It depends on a QAT kernel driver, see :ref:`qat_kernel_installation`.
diff --git a/doc/guides/compressdevs/zlib.rst b/doc/guides/compressdevs/zlib.rst
new file mode 100644
index 00000000..986c59d4
--- /dev/null
+++ b/doc/guides/compressdevs/zlib.rst
@@ -0,0 +1,69 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2018 Cavium Networks.
+
+ZLIB Compression Poll Mode Driver
+==================================
+
+The ZLIB PMD (**librte_pmd_zlib**) provides poll mode compression &
+decompression driver based on SW zlib library,
+
+Features
+--------
+
+ZLIB PMD has support for:
+
+Compression/Decompression algorithm:
+
+* DEFLATE
+
+Huffman code type:
+
+* FIXED
+* DYNAMIC
+
+Window size support:
+
+* Min - 256 bytes
+* Max - 32K
+
+Limitations
+-----------
+
+* Scatter-Gather and Stateful not supported.
+
+Installation
+------------
+
+* To build DPDK with ZLIB library, the user is required to download the ``libz`` library.
+* Use following command for installation.
+
+* For Fedora users::
+     sudo yum install zlib-devel
+* For Ubuntu users::
+     sudo apt-get install zlib1g-dev
+
+* Once downloaded, the user needs to build the library.
+
+* To build from sources
+  download zlib sources from http://zlib.net/ and do following before building DPDK::
+
+    make
+    sudo make install
+
+Initialization
+--------------
+
+In order to enable this virtual compression PMD, user must:
+
+* Set ``CONFIG_RTE_LIBRTE_PMD_ZLIB=y`` in config/common_base.
+
+To use the PMD in an application, user must:
+
+* Call ``rte_vdev_init("compress_zlib")`` within the application.
+
+* Use ``--vdev="compress_zlib"`` in the EAL options, which will call ``rte_vdev_init()`` internally.
+
+The following parameter (optional) can be provided in the previous two calls:
+
+* ``socket_id:`` Specify the socket where the memory for the device is going to be allocated
+  (by default, socket_id will be the socket where the core that is creating the PMD is running on).
diff --git a/doc/guides/contributing/cheatsheet.rst b/doc/guides/contributing/cheatsheet.rst
index 7bc07715..0debd118 100644
--- a/doc/guides/contributing/cheatsheet.rst
+++ b/doc/guides/contributing/cheatsheet.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
 Patch Cheatsheet
 ================
 
diff --git a/doc/guides/contributing/coding_style.rst b/doc/guides/contributing/coding_style.rst
index 25876059..b1bf0d15 100644
--- a/doc/guides/contributing/coding_style.rst
+++ b/doc/guides/contributing/coding_style.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
 .. _coding_style:
 
 DPDK Coding Style
diff --git a/doc/guides/contributing/design.rst b/doc/guides/contributing/design.rst
index 88d3a433..651fd224 100644
--- a/doc/guides/contributing/design.rst
+++ b/doc/guides/contributing/design.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
 Design
 ======
 
diff --git a/doc/guides/contributing/documentation.rst b/doc/guides/contributing/documentation.rst
index 82f2e1bb..6a075553 100644
--- a/doc/guides/contributing/documentation.rst
+++ b/doc/guides/contributing/documentation.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
 .. _doc_guidelines:
 
 DPDK Documentation Guidelines
diff --git a/doc/guides/contributing/index.rst b/doc/guides/contributing/index.rst
index 329b678a..f90df451 100644
--- a/doc/guides/contributing/index.rst
+++ b/doc/guides/contributing/index.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
 Contributor's Guidelines
 ========================
 
diff --git a/doc/guides/contributing/patches.rst b/doc/guides/contributing/patches.rst
index e4e0cb59..a3d78802 100644
--- a/doc/guides/contributing/patches.rst
+++ b/doc/guides/contributing/patches.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
 .. submitting_patches:
 
 Contributing Code to DPDK
diff --git a/doc/guides/contributing/stable.rst b/doc/guides/contributing/stable.rst
index 0f2f1f37..1746c046 100644
--- a/doc/guides/contributing/stable.rst
+++ b/doc/guides/contributing/stable.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
 .. stable_lts_releases:
 
 DPDK Stable Releases and Long Term Support
@@ -57,7 +60,25 @@ that a tagged release has been tested.
 What changes should be backported
 ---------------------------------
 
-Backporting should be limited to bug fixes.
+Backporting should be limited to bug fixes. All patches accepted on the master
+branch with a Fixes: tag should be backported to the relevant stable/LTS
+branches, unless the submitter indicates otherwise. If there are exceptions,
+they will be discussed on the mailing lists.
+
+Fixes suitable for backport should have a ``Cc: stable@dpdk.org`` tag in the
+commit message body as follows::
+
+     doc: fix some parameter description
+
+     Update the docs, fixing description of some parameter.
+
+     Fixes: abcdefgh1234 ("doc: add some parameter")
+     Cc: stable@dpdk.org
+
+     Signed-off-by: Alex Smith <alex.smith@example.com>
+
+
+Fixes not suitable for backport should not include the ``Cc: stable@dpdk.org`` tag.
 
 Features should not be backported to stable releases. It may be acceptable, in
 limited cases, to back port features for the LTS release where:
diff --git a/doc/guides/contributing/versioning.rst b/doc/guides/contributing/versioning.rst
index c495294d..01b36247 100644
--- a/doc/guides/contributing/versioning.rst
+++ b/doc/guides/contributing/versioning.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
 Managing ABI updates
 ====================
 
@@ -43,29 +46,6 @@ ABI versions are set at the time of major release labeling, and the ABI may
 change multiple times, without warning, between the last release label and the
 HEAD label of the git tree.
 
-APIs marked as ``experimental`` are not considered part of the ABI and may
-change without warning at any time.  Since changes to APIs are most likely
-immediately after their introduction, as users begin to take advantage of
-those new APIs and start finding issues with them, new DPDK APIs will be
-automatically marked as ``experimental`` to allow for a period of stabilization
-before they become part of a tracked ABI.
-
-Note that marking an API as experimental is a multi step process.
-To mark an API as experimental, the symbols which are desired to be exported
-must be placed in an EXPERIMENTAL version block in the corresponding libraries'
-version map script.
-Secondly, the corresponding definitions of those exported functions, and
-their forward declarations (in the development header files), must be marked
-with the ``__rte_experimental`` tag (see ``rte_compat.h``).
-The DPDK build makefiles perform a check to ensure that the map file and the
-C code reflect the same list of symbols.
-This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
-during compilation in the corresponding library Makefile.
-
-In addition to tagging the code with ``__rte_experimental``,
-the doxygen markup must also contain the EXPERIMENTAL string,
-and the MAINTAINERS file should note the EXPERIMENTAL libraries.
-
 ABI versions, once released, are available until such time as their
 deprecation has been noted in the Release Notes for at least one major release
 cycle. For example consider the case where the ABI for DPDK 2.0 has been
@@ -119,6 +99,37 @@ readability purposes should be avoided.
    follow the relevant deprecation policy procedures as above: 3 acks and
    announcement at least one release in advance.
 
+Experimental APIs
+~~~~~~~~~~~~~~~~~
+
+APIs marked as ``experimental`` are not considered part of the ABI and may
+change without warning at any time.  Since changes to APIs are most likely
+immediately after their introduction, as users begin to take advantage of
+those new APIs and start finding issues with them, new DPDK APIs will be
+automatically marked as ``experimental`` to allow for a period of stabilization
+before they become part of a tracked ABI.
+
+Note that marking an API as experimental is a multi step process.
+To mark an API as experimental, the symbols which are desired to be exported
+must be placed in an EXPERIMENTAL version block in the corresponding libraries'
+version map script.
+Secondly, the corresponding definitions of those exported functions, and
+their forward declarations (in the development header files), must be marked
+with the ``__rte_experimental`` tag (see ``rte_compat.h``).
+The DPDK build makefiles perform a check to ensure that the map file and the
+C code reflect the same list of symbols.
+This check can be circumvented by defining ``ALLOW_EXPERIMENTAL_API``
+during compilation in the corresponding library Makefile.
+
+In addition to tagging the code with ``__rte_experimental``,
+the doxygen markup must also contain the EXPERIMENTAL string,
+and the MAINTAINERS file should note the EXPERIMENTAL libraries.
+
+For removing the experimental tag associated with an API, deprecation notice
+is not required. Though, an API should remain in experimental state for at least
+one release. Thereafter, normal process of posting patch for review to mailing
+list can be followed.
+
 Examples of Deprecation Notices
 -------------------------------
 
diff --git a/doc/guides/cryptodevs/aesni_gcm.rst b/doc/guides/cryptodevs/aesni_gcm.rst
index 01590e85..e0346080 100644
--- a/doc/guides/cryptodevs/aesni_gcm.rst
+++ b/doc/guides/cryptodevs/aesni_gcm.rst
@@ -36,8 +36,8 @@ Installation
 To build DPDK with the AESNI_GCM_PMD the user is required to download the multi-buffer
 library from `here <https://github.com/01org/intel-ipsec-mb>`_
 and compile it on their user system before building DPDK.
-The latest version of the library supported by this PMD is v0.49, which
-can be downloaded in `<https://github.com/01org/intel-ipsec-mb/archive/v0.49.zip>`_.
+The latest version of the library supported by this PMD is v0.50, which
+can be downloaded in `<https://github.com/01org/intel-ipsec-mb/archive/v0.50.zip>`_.
 
 .. code-block:: console
 
@@ -57,7 +57,7 @@ and the external crypto libraries supported by them:
    16.04 - 16.11  Multi-buffer library 0.43 - 0.44
    17.02 - 17.05  ISA-L Crypto v2.18
    17.08 - 18.02  Multi-buffer library 0.46 - 0.48
-   18.05          Multi-buffer library 0.49
+   18.05+         Multi-buffer library 0.49+
    =============  ================================
 
 
diff --git a/doc/guides/cryptodevs/aesni_mb.rst b/doc/guides/cryptodevs/aesni_mb.rst
index 236828c0..c2929500 100644
--- a/doc/guides/cryptodevs/aesni_mb.rst
+++ b/doc/guides/cryptodevs/aesni_mb.rst
@@ -27,6 +27,7 @@ Cipher algorithms:
 * RTE_CRYPTO_CIPHER_AES256_CTR
 * RTE_CRYPTO_CIPHER_AES_DOCSISBPI
 * RTE_CRYPTO_CIPHER_DES_CBC
+* RTE_CRYPTO_CIPHER_3DES_CBC
 * RTE_CRYPTO_CIPHER_DES_DOCSISBPI
 
 Hash algorithms:
@@ -57,8 +58,8 @@ Installation
 To build DPDK with the AESNI_MB_PMD the user is required to download the multi-buffer
 library from `here <https://github.com/01org/intel-ipsec-mb>`_
 and compile it on their user system before building DPDK.
-The latest version of the library supported by this PMD is v0.49, which
-can be downloaded from `<https://github.com/01org/intel-ipsec-mb/archive/v0.49.zip>`_.
+The latest version of the library supported by this PMD is v0.50, which
+can be downloaded from `<https://github.com/01org/intel-ipsec-mb/archive/v0.50.zip>`_.
 
 .. code-block:: console
 
@@ -80,7 +81,7 @@ and the Multi-Buffer library version supported by them:
    17.05 - 17.08   0.45 - 0.48
    17.11           0.47 - 0.48
    18.02           0.48
-   18.05           0.49
+   18.05+          0.49+
    ==============  ============================
 
 
diff --git a/doc/guides/cryptodevs/dpaa2_sec.rst b/doc/guides/cryptodevs/dpaa2_sec.rst
index 3ea24c8a..9191704e 100644
--- a/doc/guides/cryptodevs/dpaa2_sec.rst
+++ b/doc/guides/cryptodevs/dpaa2_sec.rst
@@ -148,7 +148,6 @@ Where x is the device object id as configured in resource container.
 Limitations
 -----------
 
-* Chained mbufs are not supported.
 * Hash followed by Cipher mode is not supported
 * Only supports the session-oriented API implementation (session-less APIs are not supported).
 
@@ -200,11 +199,6 @@ Please note that enabling debugging options may affect system performance.
   By default it is only enabled in defconfig_arm64-dpaa2-* config.
   Toggle compilation of the ``librte_pmd_dpaa2_sec`` driver.
 
-* ``CONFIG_RTE_DPAA2_SEC_PMD_MAX_NB_SESSIONS``
-  By default it is set as 2048 in defconfig_arm64-dpaa2-* config.
-  It indicates Number of sessions to create in the session memory pool
-  on a single DPAA2 SEC device.
-
 Installations
 -------------
 To compile the DPAA2_SEC PMD for Linux arm64 gcc target, run the
diff --git a/doc/guides/cryptodevs/dpaa_sec.rst b/doc/guides/cryptodevs/dpaa_sec.rst
index c14d6d7b..dd683894 100644
--- a/doc/guides/cryptodevs/dpaa_sec.rst
+++ b/doc/guides/cryptodevs/dpaa_sec.rst
@@ -94,7 +94,6 @@ For blacklisting a DPAA device, following commands can be used.
 Limitations
 -----------
 
-* Chained mbufs are not supported.
 * Hash followed by Cipher mode is not supported
 * Only supports the session-oriented API implementation (session-less APIs are not supported).
 
@@ -145,11 +144,6 @@ Please note that enabling debugging options may affect system performance.
   By default it is only enabled in defconfig_arm64-dpaa-* config.
   Toggle compilation of the ``librte_pmd_dpaa_sec`` driver.
 
-* ``CONFIG_RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS``
-  By default it is set as 2048 in defconfig_arm64-dpaa-* config.
-  It indicates Number of sessions to create in the session memory pool
-  on a single DPAA SEC device.
-
 Installations
 -------------
 To compile the DPAA_SEC PMD for Linux arm64 gcc target, run the
diff --git a/doc/guides/cryptodevs/features/aesni_gcm.ini b/doc/guides/cryptodevs/features/aesni_gcm.ini
index 920b6b6a..b9e9c906 100644
--- a/doc/guides/cryptodevs/features/aesni_gcm.ini
+++ b/doc/guides/cryptodevs/features/aesni_gcm.ini
@@ -10,7 +10,8 @@ CPU AESNI              = Y
 CPU SSE                = Y
 CPU AVX                = Y
 CPU AVX2               = Y
-Mbuf scatter gather    = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In LB  Out     = Y
 ;
 ; Supported crypto algorithms of the 'aesni_gcm' crypto driver.
 ;
diff --git a/doc/guides/cryptodevs/features/aesni_mb.ini b/doc/guides/cryptodevs/features/aesni_mb.ini
index 1e263c2b..f7295745 100644
--- a/doc/guides/cryptodevs/features/aesni_mb.ini
+++ b/doc/guides/cryptodevs/features/aesni_mb.ini
@@ -24,6 +24,7 @@ AES CTR (192)  = Y
 AES CTR (256)  = Y
 AES DOCSIS BPI = Y
 DES CBC        = Y
+3DES CBC       = Y
 DES DOCSIS BPI = Y
 
 ;
diff --git a/doc/guides/cryptodevs/features/default.ini b/doc/guides/cryptodevs/features/default.ini
index 42783887..92a7ccf3 100644
--- a/doc/guides/cryptodevs/features/default.ini
+++ b/doc/guides/cryptodevs/features/default.ini
@@ -18,7 +18,11 @@ CPU AVX512             =
 CPU AESNI              =
 CPU NEON               =
 CPU ARM CE             =
-Mbuf scatter gather    =
+In Place SGL           =
+OOP SGL In SGL Out     =
+OOP SGL In LB  Out     =
+OOP LB  In SGL Out     =
+OOP LB  In LB  Out     =
 
 ;
 ; Supported crypto algorithms of a default crypto driver.
diff --git a/doc/guides/cryptodevs/features/dpaa2_sec.ini b/doc/guides/cryptodevs/features/dpaa2_sec.ini
index 68c9960d..69700df4 100644
--- a/doc/guides/cryptodevs/features/dpaa2_sec.ini
+++ b/doc/guides/cryptodevs/features/dpaa2_sec.ini
@@ -8,7 +8,11 @@ Symmetric crypto       = Y
 Sym operation chaining = Y
 HW Accelerated         = Y
 Protocol offload       = Y
-Mbuf scatter gather    = Y
+In Place SGL           = Y
+OOP SGL In SGL Out     = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In SGL Out     = Y
+OOP LB  In LB  Out     = Y
 
 ;
 ; Supported crypto algorithms of the 'dpaa2_sec' crypto driver.
diff --git a/doc/guides/cryptodevs/features/dpaa_sec.ini b/doc/guides/cryptodevs/features/dpaa_sec.ini
index 260fae72..937b621c 100644
--- a/doc/guides/cryptodevs/features/dpaa_sec.ini
+++ b/doc/guides/cryptodevs/features/dpaa_sec.ini
@@ -8,7 +8,11 @@ Symmetric crypto       = Y
 Sym operation chaining = Y
 HW Accelerated         = Y
 Protocol offload       = Y
-Mbuf scatter gather    = Y
+In Place SGL           = Y
+OOP SGL In SGL Out     = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In SGL Out     = Y
+OOP LB  In LB  Out     = Y
 
 ;
 ; Supported crypto algorithms of the 'dpaa_sec' crypto driver.
diff --git a/doc/guides/cryptodevs/features/null.ini b/doc/guides/cryptodevs/features/null.ini
index a9e172da..ecf5779a 100644
--- a/doc/guides/cryptodevs/features/null.ini
+++ b/doc/guides/cryptodevs/features/null.ini
@@ -6,7 +6,7 @@
 [Features]
 Symmetric crypto       = Y
 Sym operation chaining = Y
-Mbuf scatter gather    = Y
+In Place SGL           = Y
 
 ;
 ; Supported crypto algorithms of the 'null' crypto driver.
diff --git a/doc/guides/cryptodevs/features/openssl.ini b/doc/guides/cryptodevs/features/openssl.ini
index 69156586..b9c0bdcc 100644
--- a/doc/guides/cryptodevs/features/openssl.ini
+++ b/doc/guides/cryptodevs/features/openssl.ini
@@ -6,7 +6,9 @@
 [Features]
 Symmetric crypto       = Y
 Sym operation chaining = Y
-Mbuf scatter gather    = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In LB  Out     = Y
+Asymmetric crypto      = Y
 
 ;
 ; Supported crypto algorithms of the 'openssl' crypto driver.
@@ -49,3 +51,13 @@ AES GCM (256) = Y
 AES CCM (128) = Y
 AES CCM (192) = Y
 AES CCM (256) = Y
+
+;
+; Supported Asymmetric algorithms of the 'openssl' crypto driver.
+;
+[Asymmetric]
+RSA = Y
+DSA = Y
+Modular Exponentiation = Y
+Modular Inversion = Y
+Diffie-hellman = Y
diff --git a/doc/guides/cryptodevs/features/qat.ini b/doc/guides/cryptodevs/features/qat.ini
index 51ed5967..29d865e0 100644
--- a/doc/guides/cryptodevs/features/qat.ini
+++ b/doc/guides/cryptodevs/features/qat.ini
@@ -7,7 +7,11 @@
 Symmetric crypto       = Y
 Sym operation chaining = Y
 HW Accelerated         = Y
-Mbuf scatter gather    = Y
+In Place SGL           = Y
+OOP SGL In SGL Out     = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In SGL Out     = Y
+OOP LB  In LB  Out     = Y
 
 ;
 ; Supported crypto algorithms of the 'qat' crypto driver.
diff --git a/doc/guides/cryptodevs/openssl.rst b/doc/guides/cryptodevs/openssl.rst
index 427fc807..bdc30f66 100644
--- a/doc/guides/cryptodevs/openssl.rst
+++ b/doc/guides/cryptodevs/openssl.rst
@@ -80,6 +80,7 @@ crypto processing.
 
 Test name is cryptodev_openssl_autotest.
 For performance test cryptodev_openssl_perftest can be used.
+For asymmetric crypto operations testing, run cryptodev_openssl_asym_autotest.
 
 To verify real traffic l2fwd-crypto example can be used with this command:
 
diff --git a/doc/guides/cryptodevs/overview.rst b/doc/guides/cryptodevs/overview.rst
index 493cd5f5..3f776f07 100644
--- a/doc/guides/cryptodevs/overview.rst
+++ b/doc/guides/cryptodevs/overview.rst
@@ -11,14 +11,31 @@ Supported Feature Flags
 
 .. include:: overview_feature_table.txt
 
-Note, the mbuf scatter gather feature (aka chained mbufs, scatter-gather-lists
-or SGLs) indicate all following combinations are supported unless otherwise called
-out in the Limitations section of each PMD.
-
-* In place operation, input buffer as multiple segments, same buffer used for output
-* Out of place operation, input buffer as single segment and output as multiple segments
-* Out of place operation, input buffer as multiple segments and output as single segment
-* Out of place operation, input buffer as multiple segments and output as multiple segments
+.. Note::
+
+   - "In Place SGL" feature flag stands for "In place Scatter-gather list",
+     which means that an input buffer can consist of multiple segments,
+     being the operation in-place (input address = output address).
+
+   - "OOP SGL In SGL Out" feature flag stands for
+     "Out-of-place Scatter-gather list Input, Scatter-gater list Output",
+     which means pmd supports different scatter-gather styled input and output buffers
+     (i.e. both can consists of multiple segments).
+
+   - "OOP SGL In LB Out" feature flag stands for
+     "Out-of-place Scatter-gather list Input, Linear Buffers Output",
+     which means PMD supports input from scatter-gathered styled buffers,
+     outputting linear buffers (i.e. single segment).
+
+   - "OOP LB In SGL Out" feature flag stands for
+     "Out-of-place Linear Buffers Input, Scatter-gather list Output",
+     which means PMD supports input from linear buffer, outputting
+     scatter-gathered styled buffers.
+
+   - "OOP LB In LB Out" feature flag stands for
+     "Out-of-place Linear Buffers Input, Scatter-gather list Output",
+     which means that Out-of-place operation is supported,
+     with linear input and output buffers.
 
 
 Supported Cipher Algorithms
diff --git a/doc/guides/cryptodevs/qat.rst b/doc/guides/cryptodevs/qat.rst
index 8c8fefaa..bdc58eb2 100644
--- a/doc/guides/cryptodevs/qat.rst
+++ b/doc/guides/cryptodevs/qat.rst
@@ -68,12 +68,32 @@ Limitations
 * Queue pairs are not thread-safe (that is, within a single queue pair, RX and TX from different lcores is not supported).
 
 
-Installation
-------------
+Extra notes on KASUMI F9
+------------------------
+
+When using KASUMI F9 authentication algorithm, the input buffer must be
+constructed according to the 3GPP KASUMI specifications (section 4.4, page 13):
+`<http://cryptome.org/3gpp/35201-900.pdf>`_.
+Input buffer has to have COUNT (4 bytes), FRESH (4 bytes), MESSAGE and DIRECTION (1 bit)
+concatenated. After the DIRECTION bit, a single '1' bit is appended, followed by
+between 0 and 7 '0' bits, so that the total length of the buffer is multiple of 8 bits.
+Note that the actual message can be any length, specified in bits.
+
+Once this buffer is passed this way, when creating the crypto operation,
+length of data to authenticate (op.sym.auth.data.length) must be the length
+of all the items described above, including the padding at the end.
+Also, offset of data to authenticate (op.sym.auth.data.offset)
+must be such that points at the start of the COUNT bytes.
+
+
+Building the DPDK QAT cryptodev PMD
+-----------------------------------
+
 
-To enable QAT in DPDK, follow the instructions for modifying the compile-time
+To enable QAT crypto in DPDK, follow the instructions for modifying the compile-time
 configuration file as described `here <http://dpdk.org/doc/guides/linux_gsg/build_dpdk.html>`_.
 
+
 Quick instructions are as follows:
 
 .. code-block:: console
@@ -81,29 +101,95 @@ Quick instructions are as follows:
 	cd to the top-level DPDK directory
 	make config T=x86_64-native-linuxapp-gcc
 	sed -i 's,\(CONFIG_RTE_LIBRTE_PMD_QAT\)=n,\1=y,' build/.config
+	sed -i 's,\(CONFIG_RTE_LIBRTE_PMD_QAT_SYM\)=n,\1=y,' build/.config
 	make
 
-To use the DPDK QAT PMD an SRIOV-enabled QAT kernel driver is required. The VF
-devices exposed by this driver will be used by the QAT PMD. The devices and
-available kernel drivers and device ids are :
+
+.. _qat_kernel_installation:
+
+Dependency on the QAT kernel driver
+-----------------------------------
+
+To use the QAT PMD an SRIOV-enabled QAT kernel driver is required. The VF
+devices created and initialised by this driver will be used by the QAT PMD.
+
+Instructions for installation are below, but first an explanation of the
+relationships between the PF/VF devices and the PMDs visible to
+DPDK applications.
+
+
+Acceleration services - cryptography and compression - are provided to DPDK
+applications via PMDs which register to implement the corresponding
+cryptodev and compressdev APIs.
+
+Each QuickAssist VF device can expose one cryptodev PMD and/or one compressdev PMD.
+These QAT PMDs share the same underlying device and pci-mgmt code, but are
+enumerated independently on their respective APIs and appear as independent
+devices to applications.
+
+.. Note::
+
+   Each VF can only be used by one DPDK process. It is not possible to share
+   the same VF across multiple processes, even if these processes are using
+   different acceleration services.
+
+   Conversely one DPDK process can use one or more QAT VFs and can expose both
+   cryptodev and compressdev instances on each of those VFs.
+
+
+
+Device and driver naming
+------------------------
+
+* The qat cryptodev driver name is "crypto_qat".
+  The rte_cryptodev_devices_get() returns the devices exposed by this driver.
+
+* Each qat crypto device has a unique name, in format
+  <pci bdf>_<service>, e.g. "0000:41:01.0_qat_sym".
+  This name can be passed to rte_cryptodev_get_dev_id() to get the device_id.
+
+.. Note::
+
+	The qat crypto driver name is passed to the dpdk-test-crypto-perf tool in the -devtype parameter.
+
+	The qat crypto device name is in the format of the slave parameter passed to the crypto scheduler.
+
+* The qat compressdev driver name is "comp_qat".
+  The rte_compressdev_devices_get() returns the devices exposed by this driver.
+
+* Each qat compression device has a unique name, in format
+  <pci bdf>_<service>, e.g. "0000:41:01.0_qat_comp".
+  This name can be passed to rte_compressdev_get_dev_id() to get the device_id.
+
+
+Available kernel drivers
+------------------------
+
+Kernel drivers for each device are listed in the following table. Scroll right
+to check that the driver and device supports the servic you require.
+
 
 .. _table_qat_pmds_drivers:
 
 .. table:: QAT device generations, devices and drivers
 
-   +-----+----------+--------+---------------+------------+--------+------+--------+--------+
-   | Gen | Device   | Driver | Kernel Module | Pci Driver | PF Did | #PFs | Vf Did | VFs/PF |
-   +=====+==========+========+===============+============+========+======+========+========+
-   | 1   | DH895xCC | 01.org | icp_qa_al     | n/a        | 435    | 1    | 443    | 32     |
-   +-----+----------+--------+---------------+------------+--------+------+--------+--------+
-   | 1   | DH895xCC | 4.4+   | qat_dh895xcc  | dh895xcc   | 435    | 1    | 443    | 32     |
-   +-----+----------+--------+---------------+------------+--------+------+--------+--------+
-   | 2   | C62x     | 4.5+   | qat_c62x      | c6xx       | 37c8   | 3    | 37c9   | 16     |
-   +-----+----------+--------+---------------+------------+--------+------+--------+--------+
-   | 2   | C3xxx    | 4.5+   | qat_c3xxx     | c3xxx      | 19e2   | 1    | 19e3   | 16     |
-   +-----+----------+--------+---------------+------------+--------+------+--------+--------+
-   | 2   | D15xx    | p      | qat_d15xx     | d15xx      | 6f54   | 1    | 6f55   | 16     |
-   +-----+----------+--------+---------------+------------+--------+------+--------+--------+
+   +-----+----------+---------------+---------------+------------+--------+------+--------+--------+-----------+-------------+
+   | Gen | Device   | Driver/ver    | Kernel Module | Pci Driver | PF Did | #PFs | VF Did | VFs/PF | cryptodev | compressdev |
+   +=====+==========+===============+===============+============+========+======+========+========+===========+=============+
+   | 1   | DH895xCC | linux/4.4+    | qat_dh895xcc  | dh895xcc   | 435    | 1    | 443    | 32     | Yes       | No          |
+   +-----+----------+---------------+---------------+------------+--------+------+--------+--------+-----------+-------------+
+   | "   | "        | 01.org/4.2.0+ | "             | "          | "      | "    | "      | "      | Yes       | No          |
+   +-----+----------+---------------+---------------+------------+--------+------+--------+--------+-----------+-------------+
+   | 2   | C62x     | linux/4.5+    | qat_c62x      | c6xx       | 37c8   | 3    | 37c9   | 16     | Yes       | No          |
+   +-----+----------+---------------+---------------+------------+--------+------+--------+--------+-----------+-------------+
+   | "   | "        | 01.org/4.2.0+ | "             | "          | "      | "    | "      | "      | Yes       | Yes         |
+   +-----+----------+---------------+---------------+------------+--------+------+--------+--------+-----------+-------------+
+   | 2   | C3xxx    | linux/4.5+    | qat_c3xxx     | c3xxx      | 19e2   | 1    | 19e3   | 16     | Yes       | No          |
+   +-----+----------+---------------+---------------+------------+--------+------+--------+--------+-----------+-------------+
+   | "   | "        | 01.org/4.2.0+ | "             | "          | "      | "    | "      | "      | Yes       | Yes         |
+   +-----+----------+---------------+---------------+------------+--------+------+--------+--------+-----------+-------------+
+   | 2   | D15xx    | p             | qat_d15xx     | d15xx      | 6f54   | 1    | 6f55   | 16     | Yes       | No          |
+   +-----+----------+---------------+---------------+------------+--------+------+--------+--------+-----------+-------------+
 
 
 The ``Driver`` column indicates either the Linux kernel version in which
@@ -196,9 +282,9 @@ Consult the *Getting Started Guide* at the same URL for further information.
 
 The steps below assume you are:
 
-* Building on a platform with one ``DH895xCC`` device.
-* Using package ``qatmux.l.2.3.0-34.tgz``.
-* On Fedora21 kernel ``3.17.4-301.fc21.x86_64``.
+* Building on a platform with one ``C62x`` device.
+* Using package ``qat1.7.l.4.2.0-000xx.tar.gz``.
+* On Fedora26 kernel ``4.11.11-300.fc26.x86_64``.
 
 In the BIOS ensure that SRIOV is enabled and VT-d is disabled.
 
@@ -206,21 +292,30 @@ Uninstall any existing QAT driver, for example by running:
 
 * ``./installer.sh uninstall`` in the directory where originally installed.
 
-* or ``rmmod qat_dh895xcc; rmmod intel_qat``.
 
 Build and install the SRIOV-enabled QAT driver::
 
     mkdir /QAT
     cd /QAT
 
-    # Copy qatmux.l.2.3.0-34.tgz to this location
-    tar zxof qatmux.l.2.3.0-34.tgz
+    # Copy the package to this location and unpack
+    tar zxof qat1.7.l.4.2.0-000xx.tar.gz
+
+    ./configure --enable-icp-sriov=host
+    make install
+
+You can use ``cat /sys/kernel/debug/qat<your device type and bdf>/version/fw`` to confirm the driver is correctly installed and is using firmware version 4.2.0.
+You can use ``lspci -d:37c9`` to confirm the presence of the 16 VF devices available per ``C62x`` PF.
+
+Confirm the driver is correctly installed and is using firmware version 4.2.0::
+
+    cat /sys/kernel/debug/qat<your device type and bdf>/version/fw
 
-    export ICP_WITHOUT_IOMMU=1
-    ./installer.sh install QAT1.6 host
 
-You can use ``cat /proc/icp_dh895xcc_dev0/version`` to confirm the driver is correctly installed.
-You can use ``lspci -d:443`` to confirm the  of the 32 VF devices available per ``DH895xCC`` device.
+Confirm the presence of 48 VF devices - 16 per PF::
+
+    lspci -d:37c9
+
 
 To complete the installation - follow instructions in `Binding the available VFs to the DPDK UIO driver`_.
 
@@ -261,6 +356,7 @@ To complete the installation - follow instructions in `Binding the available VFs
 
       sudo yum install zlib-devel
       sudo yum install openssl-devel
+      sudo yum install libudev-devel
 
 .. Note::
 
@@ -343,19 +439,28 @@ Another way to bind the VFs to the DPDK UIO driver is by using the
     ./usertools/dpdk-devbind.py -b igb_uio 0000:03:01.1
 
 
-Extra notes on KASUMI F9
-------------------------
+Debugging
+----------------------------------------
 
-When using KASUMI F9 authentication algorithm, the input buffer must be
-constructed according to the 3GPP KASUMI specifications (section 4.4, page 13):
-`<http://cryptome.org/3gpp/35201-900.pdf>`_.
-Input buffer has to have COUNT (4 bytes), FRESH (4 bytes), MESSAGE and DIRECTION (1 bit)
-concatenated. After the DIRECTION bit, a single '1' bit is appended, followed by
-between 0 and 7 '0' bits, so that the total length of the buffer is multiple of 8 bits.
-Note that the actual message can be any length, specified in bits.
+There are 2 sets of trace available via the dynamic logging feature:
 
-Once this buffer is passed this way, when creating the crypto operation,
-length of data to authenticate (op.sym.auth.data.length) must be the length
-of all the items described above, including the padding at the end.
-Also, offset of data to authenticate (op.sym.auth.data.offset)
-must be such that points at the start of the COUNT bytes.
+* pmd.qat_dp exposes trace on the data-path.
+* pmd.qat_general exposes all other trace.
+
+pmd.qat exposes both sets of traces.
+They can be enabled using the log-level option (where 8=maximum log level) on
+the process cmdline, e.g. using any of the following::
+
+    --log-level="pmd.qat_general,8"
+    --log-level="pmd.qat_dp,8"
+    --log-level="pmd.qat,8"
+
+.. Note::
+
+    The global RTE_LOG_DP_LEVEL overrides data-path trace so must be set to
+    RTE_LOG_DEBUG to see all the trace. This variable is in config/rte_config.h
+    for meson build and config/common_base for gnu make.
+    Also the dynamic global log level overrides both sets of trace, so e.g. no
+    QAT trace would display in this case::
+
+	--log-level="7" --log-level="pmd.qat_general,8"
diff --git a/doc/guides/cryptodevs/scheduler.rst b/doc/guides/cryptodevs/scheduler.rst
index d67894d5..a754a27e 100644
--- a/doc/guides/cryptodevs/scheduler.rst
+++ b/doc/guides/cryptodevs/scheduler.rst
@@ -71,6 +71,11 @@ two calls:
   mode parameter values are specified in the "Cryptodev Scheduler Modes
   Overview" section.
 
+* mode_param: Specify the mode-specific parameter. Some scheduling modes
+  may be initialized with specific parameters other than the default ones,
+  such as the **threshold** packet size of **packet-size-distr** mode. This
+  parameter fulfills the purpose.
+
 * ordering: Specify the status of the crypto operations ordering feature.
   The value of this parameter can be "enable" or "disable". This feature
   is disabled by default.
@@ -132,7 +137,12 @@ operation:
    **option_type** must be **CDEV_SCHED_OPTION_THRESHOLD** and **option** should
    point to a rte_cryptodev_scheduler_threshold_option structure filled with
    appropriate threshold value. Please NOTE this threshold has be a power-of-2
-   unsigned integer.
+   unsigned integer. It is possible to use **mode_param** initialization
+   parameter to achieve the same purpose. For example:
+
+   ... --vdev "crypto_scheduler,mode=packet-size-distr,mode_param=threshold:512" ...
+
+   The above parameter will overwrite the threshold value to 512.
 
 *   **CDEV_SCHED_MODE_FAILOVER:**
 
diff --git a/doc/guides/eventdevs/octeontx.rst b/doc/guides/eventdevs/octeontx.rst
index 3dcf6afb..18cfc7a9 100644
--- a/doc/guides/eventdevs/octeontx.rst
+++ b/doc/guides/eventdevs/octeontx.rst
@@ -96,7 +96,7 @@ The tests are run once the vdev creation is successfully complete.
 
 .. code-block:: console
 
-    --vdev="event_octeontx,self_test=1"
+    --vdev="event_octeontx,selftest=1"
 
 
 Enable TIMvf stats
diff --git a/doc/guides/linux_gsg/build_sample_apps.rst b/doc/guides/linux_gsg/build_sample_apps.rst
index 3623ddf4..332424e0 100644
--- a/doc/guides/linux_gsg/build_sample_apps.rst
+++ b/doc/guides/linux_gsg/build_sample_apps.rst
@@ -114,6 +114,10 @@ The EAL options are as follows:
   this memory will also be pinned (i.e. not released back to the system until
   application closes).
 
+* ``--socket-limit``:
+  Limit maximum memory available for allocation on each socket. Does not support
+  legacy memory mode.
+
 * ``-d``:
   Add a driver or driver directory to be loaded.
   The application should use this option to load the pmd drivers
diff --git a/doc/guides/linux_gsg/cross_build_dpdk_for_arm64.rst b/doc/guides/linux_gsg/cross_build_dpdk_for_arm64.rst
new file mode 100644
index 00000000..9d1f0fa0
--- /dev/null
+++ b/doc/guides/linux_gsg/cross_build_dpdk_for_arm64.rst
@@ -0,0 +1,132 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2018 ARM Corporation.
+
+Cross compile DPDK for ARM64
+============================
+This chapter describes how to cross compile DPDK for ARM64 from x86 build hosts.
+
+.. note::
+
+   Whilst it is recommended to natively build DPDK on ARM64 (just
+   like with x86), it is also possible to cross-build DPDK for ARM64. An
+   ARM64 cross compile GNU toolchain is used for this.
+
+Obtain the cross tool chain
+---------------------------
+The latest cross compile tool chain can be downloaded from:
+https://releases.linaro.org/components/toolchain/binaries/latest/aarch64-linux-gnu/.
+
+Following is the step to get the version 7.2.1, latest one at the time of this writing.
+
+.. code-block:: console
+
+   wget https://releases.linaro.org/components/toolchain/binaries/latest/aarch64-linux-gnu/gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu.tar.xz
+
+Unzip and add into the PATH
+---------------------------
+
+.. code-block:: console
+
+   tar -xvf gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu.tar.xz
+   export PATH=$PATH:<cross_install_dir>/gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu/bin
+
+.. note::
+
+   For the host requirements and other info, refer to the release note section: https://releases.linaro.org/components/toolchain/binaries/latest/
+
+Getting the prerequisite library
+--------------------------------
+
+NUMA is required by most modern machines, not needed for non-NUMA architectures.
+
+.. note::
+
+   For compiling the NUMA lib, run libtool --version to ensure the libtool version >= 2.2,
+   otherwise the compilation will fail with errors.
+
+.. code-block:: console
+
+   git clone https://github.com/numactl/numactl.git
+   cd numactl
+   git checkout v2.0.11 -b v2.0.11
+   ./autogen.sh
+   autoconf -i
+   ./configure --host=aarch64-linux-gnu CC=aarch64-linux-gnu-gcc --prefix=<numa install dir>
+   make install
+
+The numa header files and lib file is generated in the include and lib folder respectively under <numa install dir>.
+
+.. _augment_the_cross_toolchain_with_numa_support:
+
+Augment the cross toolchain with NUMA support
+---------------------------------------------
+
+.. note::
+
+   This way is optional, an alternative is to use extra CFLAGS and LDFLAGS, depicted in :ref:`configure_and_cross_compile_dpdk_build` below.
+
+Copy the NUMA header files and lib to the cross compiler's directories:
+
+.. code-block:: console
+
+   cp <numa_install_dir>/include/numa*.h <cross_install_dir>/gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu/bin/../aarch64-linux-gnu/libc/usr/include/
+   cp <numa_install_dir>/lib/libnuma.a <cross_install_dir>/gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu/lib/gcc/aarch64-linux-gnu/7.2.1/
+
+.. _configure_and_cross_compile_dpdk_build:
+
+Configure and cross compile DPDK Build
+--------------------------------------
+To configure a build, choose one of the target configurations, like arm64-dpaa2-linuxapp-gcc and arm64-thunderx-linuxapp-gcc.
+
+.. code-block:: console
+
+   make config T=arm64-armv8a-linuxapp-gcc
+
+To cross-compile, without compiling the kernel modules, use the following command:
+
+.. code-block:: console
+
+   make -j CROSS=aarch64-linux-gnu- CONFIG_RTE_KNI_KMOD=n CONFIG_RTE_EAL_IGB_UIO=n
+
+To cross-compile, including the kernel modules, the kernel source tree needs to be specified by setting
+RTE_KERNELDIR:
+
+.. code-block:: console
+
+   make -j CROSS=aarch64-linux-gnu- RTE_KERNELDIR=<kernel_src_rootdir> CROSS_COMPILE=aarch64-linux-gnu-
+
+To compile for non-NUMA targets, without compiling the kernel modules, use the following command:
+
+.. code-block:: console
+
+   make -j CROSS=aarch64-linux-gnu- CONFIG_RTE_KNI_KMOD=n CONFIG_RTE_EAL_IGB_UIO=n CONFIG_RTE_LIBRTE_VHOST_NUMA=n CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
+
+.. note::
+
+   1. EXTRA_CFLAGS and EXTRA_LDFLAGS should be added to include the NUMA headers and link the library respectively,
+   if the above step :ref:`augment_the_cross_toolchain_with_numa_support` was skipped therefore the toolchain was not
+   augmented with NUMA support.
+
+   2. "-isystem <numa_install_dir>/include" should be add to EXTRA_CFLAGS, otherwise the numa.h file will get a lot of compiling
+   errors of Werror=cast-qual, Werror=strict-prototypes and Werror=old-style-definition.
+
+   An example is given below:
+
+   .. code-block:: console
+
+      make -j CROSS=aarch64-linux-gnu- CONFIG_RTE_KNI_KMOD=n CONFIG_RTE_EAL_IGB_UIO=n EXTRA_CFLAGS="-isystem <numa_install_dir>/include" EXTRA_LDFLAGS="-L<numa_install_dir>/lib -lnuma"
+
+Meson Cross Compiling DPDK
+--------------------------
+
+To cross-compile DPDK on a desired target machine we can use the following
+command::
+
+	meson cross-build --cross-file <target_machine_configuration>
+	ninja -C cross-build
+
+For example if the target machine is arm64 we can use the following
+command::
+
+	meson arm64-build --cross-file config/arm/arm64_armv8_linuxapp_gcc
+	ninja -C arm64-build
diff --git a/doc/guides/linux_gsg/index.rst b/doc/guides/linux_gsg/index.rst
index 2a7bdfe9..077f9302 100644
--- a/doc/guides/linux_gsg/index.rst
+++ b/doc/guides/linux_gsg/index.rst
@@ -13,6 +13,7 @@ Getting Started Guide for Linux
     intro
     sys_reqs
     build_dpdk
+    cross_build_dpdk_for_arm64
     linux_drivers
     build_sample_apps
     enable_func
diff --git a/doc/guides/nics/cxgbe.rst b/doc/guides/nics/cxgbe.rst
index 78e39147..58d88eef 100644
--- a/doc/guides/nics/cxgbe.rst
+++ b/doc/guides/nics/cxgbe.rst
@@ -30,6 +30,7 @@ CXGBE and CXGBEVF PMD has support for:
 - All multicast mode
 - Port hardware statistics
 - Jumbo frames
+- Flow API - Support for both Wildcard (LE-TCAM) and Exact (HASH) match filters.
 
 Limitations
 -----------
diff --git a/doc/guides/nics/enic.rst b/doc/guides/nics/enic.rst
index d650ba0f..438a83d5 100644
--- a/doc/guides/nics/enic.rst
+++ b/doc/guides/nics/enic.rst
@@ -1,32 +1,7 @@
-..  BSD LICENSE
+..  SPDX-License-Identifier: BSD-3-Clause
     Copyright (c) 2017, Cisco Systems, Inc.
     All rights reserved.
 
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions
-    are met:
-
-    1. Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-
-    2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in
-    the documentation and/or other materials provided with the
-    distribution.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-    FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-    COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-    INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-    BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-    LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-    LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-    ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-    POSSIBILITY OF SUCH DAMAGE.
-
 ENIC Poll Mode Driver
 =====================
 
@@ -336,6 +311,40 @@ it, set ``devargs`` parameter ``disable-overlay=1``. For example::
 
     -w 12:00.0,disable-overlay=1
 
+By default, the NIC uses 4789 as the VXLAN port. The user may change
+it through ``rte_eth_dev_udp_tunnel_port_{add,delete}``. However, as
+the current NIC has a single VXLAN port number, the user cannot
+configure multiple port numbers.
+
+Ingress VLAN Rewrite
+--------------------
+
+VIC adapters can tag, untag, or modify the VLAN headers of ingress
+packets. The ingress VLAN rewrite mode controls this behavior. By
+default, it is set to pass-through, where the NIC does not modify the
+VLAN header in any way so that the application can see the original
+header. This mode is sufficient for many applications, but may not be
+suitable for others. Such applications may change the mode by setting
+``devargs`` parameter ``ig-vlan-rewrite`` to one of the following.
+
+- ``pass``: Pass-through mode. The NIC does not modify the VLAN
+  header. This is the default mode.
+
+- ``priority``: Priority-tag default VLAN mode. If the ingress packet
+  is tagged with the default VLAN, the NIC replaces its VLAN header
+  with the priority tag (VLAN ID 0).
+
+- ``trunk``: Default trunk mode. The NIC tags untagged ingress packets
+  with the default VLAN. Tagged ingress packets are not modified. To
+  the application, every packet appears as tagged.
+
+- ``untag``: Untag default VLAN mode. If the ingress packet is tagged
+  with the default VLAN, the NIC removes or untags its VLAN header so
+  that the application sees an untagged packet. As a result, the
+  default VLAN becomes `untagged`. This mode can be useful for
+  applications such as OVS-DPDK performance benchmarks that utilize
+  only the default VLAN and want to see only untagged packets.
+
 .. _enic_limitations:
 
 Limitations
@@ -351,9 +360,10 @@ Limitations
   In test setups where an Ethernet port of a Cisco adapter in TRUNK mode is
   connected point-to-point to another adapter port or connected though a router
   instead of a switch, all ingress packets will be VLAN tagged. Programs such
-  as l3fwd which do not account for VLAN tags in packets will misbehave. The
-  solution is to enable VLAN stripping on ingress. The following code fragment is
-  an example of how to accomplish this:
+  as l3fwd may not account for VLAN tags in packets and may misbehave. One
+  solution is to enable VLAN stripping on ingress so the VLAN tag is removed
+  from the packet and put into the mbuf->vlan_tci field. Here is an example
+  of how to accomplish this:
 
 .. code-block:: console
 
@@ -361,6 +371,14 @@ Limitations
      vlan_offload |= ETH_VLAN_STRIP_OFFLOAD;
      rte_eth_dev_set_vlan_offload(port, vlan_offload);
 
+Another alternative is modify the adapter's ingress VLAN rewrite mode so that
+packets with the default VLAN tag are stripped by the adapter and presented to
+DPDK as untagged packets. In this case mbuf->vlan_tci and the PKT_RX_VLAN and
+PKT_RX_VLAN_STRIPPED mbuf flags would not be set. This mode is enabled with the
+``devargs`` parameter ``ig-vlan-rewrite=untag``. For example::
+
+    -w 12:00.0,ig-vlan-rewrite=untag
+
 - Limited flow director support on 1200 series and 1300 series Cisco VIC
   adapters with old firmware. Please see :ref:`enic-flow-director`.
 
@@ -396,10 +414,14 @@ Limitations
 
   - ``rx_good_bytes`` (ibytes) always includes VLAN header (4B) and CRC bytes (4B).
     This behavior applies to 1300 and older series VIC adapters.
+    1400 series VICs do not count CRC bytes, and count VLAN header only when VLAN
+    stripping is disabled.
   - When the NIC drops a packet because the Rx queue has no free buffers,
     ``rx_good_bytes`` still increments by 4B if the packet is not VLAN tagged or
     VLAN stripping is disabled, or by 8B if the packet is VLAN tagged and stripping
-    is enabled. This behavior applies to 1300 and older series VIC adapters.
+    is enabled.
+    This behavior applies to 1300 and older series VIC adapters. 1400 series VICs
+    do not increment this byte counter when packets are dropped.
 
 - **RSS Hashing**
 
@@ -425,6 +447,7 @@ ENIC PMD supports all recent generations of Cisco VIC adapters including:
 
 - VIC 1200 series
 - VIC 1300 series
+- VIC 1400 series
 
 Supported Operating Systems
 ---------------------------
@@ -448,7 +471,7 @@ Supported features
 - VLAN filtering (supported via UCSM/CIMC only)
 - Execution of application by unprivileged system users
 - IPV4, IPV6 and TCP RSS hashing
-- UDP RSS hashing (support for upcoming adapters)
+- UDP RSS hashing (1400 series and later adapters)
 - Scattered Rx
 - MTU update
 - SR-IOV on UCS managed servers connected to Fabric Interconnects
@@ -525,4 +548,4 @@ Any questions or bugs should be reported to DPDK community and to the ENIC PMD
 maintainers:
 
 - John Daley <johndale@cisco.com>
-- Nelson Escobar <neescoba@cisco.com>
+- Hyong Youb Kim <hyonkim@cisco.com>
diff --git a/doc/guides/nics/features.rst b/doc/guides/nics/features.rst
index ee89d6bf..cddc877d 100644
--- a/doc/guides/nics/features.rst
+++ b/doc/guides/nics/features.rst
@@ -514,7 +514,7 @@ CRC offload
 
 Supports CRC stripping by hardware.
 
-* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_CRC_STRIP``.
+* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_CRC_STRIP,DEV_RX_OFFLOAD_KEEP_CRC``.
 
 
 .. _nic_features_vlan_offload:
@@ -577,7 +577,6 @@ Supports L4 checksum offload.
 
 * **[uses]     rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_UDP_CKSUM,DEV_RX_OFFLOAD_TCP_CKSUM``.
 * **[uses]     rte_eth_txconf,rte_eth_txmode**: ``offloads:DEV_TX_OFFLOAD_UDP_CKSUM,DEV_TX_OFFLOAD_TCP_CKSUM,DEV_TX_OFFLOAD_SCTP_CKSUM``.
-* **[uses]     user config**: ``dev_conf.rxmode.hw_ip_checksum``.
 * **[uses]     mbuf**: ``mbuf.ol_flags:PKT_TX_IPV4`` | ``PKT_TX_IPV6``,
   ``mbuf.ol_flags:PKT_TX_L4_NO_CKSUM`` | ``PKT_TX_TCP_CKSUM`` |
   ``PKT_TX_SCTP_CKSUM`` | ``PKT_TX_UDP_CKSUM``.
diff --git a/doc/guides/nics/features/cxgbe.ini b/doc/guides/nics/features/cxgbe.ini
index 6cf5c13f..88f2f92b 100644
--- a/doc/guides/nics/features/cxgbe.ini
+++ b/doc/guides/nics/features/cxgbe.ini
@@ -16,6 +16,7 @@ Allmulticast mode    = Y
 RSS hash             = Y
 RSS key update       = Y
 Flow control         = Y
+Flow API             = Y
 CRC offload          = Y
 VLAN offload         = Y
 L3 checksum offload  = Y
diff --git a/doc/guides/nics/features/enic.ini b/doc/guides/nics/features/enic.ini
index ae46d299..8a4bad29 100644
--- a/doc/guides/nics/features/enic.ini
+++ b/doc/guides/nics/features/enic.ini
@@ -19,6 +19,7 @@ Multicast MAC filter =
 RSS hash             = Y
 RSS key update       = Y
 RSS reta update      = Y
+Inner RSS            = Y
 SR-IOV               = Y
 CRC offload          = Y
 VLAN offload         = Y
@@ -26,6 +27,8 @@ Flow director        = Y
 Flow API             = Y
 L3 checksum offload  = Y
 L4 checksum offload  = Y
+Inner L3 checksum    = Y
+Inner L4 checksum    = Y
 Packet type parsing  = Y
 Basic stats          = Y
 Multiprocess aware   = Y
diff --git a/doc/guides/nics/features/fm10k.ini b/doc/guides/nics/features/fm10k.ini
index 58f58b99..0acdf0d3 100644
--- a/doc/guides/nics/features/fm10k.ini
+++ b/doc/guides/nics/features/fm10k.ini
@@ -26,6 +26,8 @@ VLAN offload         = Y
 L3 checksum offload  = Y
 L4 checksum offload  = Y
 Packet type parsing  = Y
+Rx descriptor status = Y
+Tx descriptor status = Y
 Basic stats          = Y
 Extended stats       = Y
 Stats per queue      = Y
diff --git a/doc/guides/nics/features/mlx4.ini b/doc/guides/nics/features/mlx4.ini
index f6efd21d..98a3f611 100644
--- a/doc/guides/nics/features/mlx4.ini
+++ b/doc/guides/nics/features/mlx4.ini
@@ -13,6 +13,7 @@ Queue start/stop     = Y
 MTU update           = Y
 Jumbo frame          = Y
 Scattered Rx         = Y
+TSO                  = Y
 Promiscuous mode     = Y
 Allmulticast mode    = Y
 Unicast MAC filter   = Y
diff --git a/doc/guides/nics/features/mlx5.ini b/doc/guides/nics/features/mlx5.ini
index e75b14bd..b28b43e5 100644
--- a/doc/guides/nics/features/mlx5.ini
+++ b/doc/guides/nics/features/mlx5.ini
@@ -43,5 +43,6 @@ Multiprocess aware   = Y
 Other kdrv           = Y
 ARMv8                = Y
 Power8               = Y
+x86-32               = Y
 x86-64               = Y
 Usage doc            = Y
diff --git a/doc/guides/nics/features/netvsc.ini b/doc/guides/nics/features/netvsc.ini
new file mode 100644
index 00000000..2ff6042b
--- /dev/null
+++ b/doc/guides/nics/features/netvsc.ini
@@ -0,0 +1,23 @@
+;
+; Supported features of the 'netvsc' network poll mode driver.
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+[Features]
+Speed capabilities   = P
+Link status          = Y
+Queue start/stop     = Y
+Scattered Rx         = Y
+Promiscuous mode     = Y
+Allmulticast mode    = Y
+Basic stats          = Y
+Stats per queue      = Y
+Extended stats       = Y
+Multiprocess aware   = Y
+Other kdrv           = Y
+ARMv7                = Y
+ARMv8                = Y
+x86-32               = Y
+x86-64               = Y
+Usage doc            = Y
+MTU update           = Y
diff --git a/doc/guides/nics/features/softnic.ini b/doc/guides/nics/features/softnic.ini
new file mode 100644
index 00000000..0583381c
--- /dev/null
+++ b/doc/guides/nics/features/softnic.ini
@@ -0,0 +1,9 @@
+;
+; Supported features of the 'softnic' poll mode driver.
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+[Features]
+x86-32               = Y
+x86-64               = Y
+Usage doc            = Y
diff --git a/doc/guides/nics/i40e.rst b/doc/guides/nics/i40e.rst
index 18549bf5..65d87f86 100644
--- a/doc/guides/nics/i40e.rst
+++ b/doc/guides/nics/i40e.rst
@@ -58,6 +58,36 @@ Prerequisites
 - Upgrade the NVM/FW version following the `Intel® Ethernet NVM Update Tool Quick Usage Guide for Linux
   <https://www-ssl.intel.com/content/www/us/en/embedded/products/networking/nvm-update-tool-quick-linux-usage-guide.html>`_ and `Intel® Ethernet NVM Update Tool: Quick Usage Guide for EFI <https://www.intel.com/content/www/us/en/embedded/products/networking/nvm-update-tool-quick-efi-usage-guide.html>`_ if needed.
 
+Recommended Matching List
+-------------------------
+
+It is highly recommended to upgrade the i40e kernel driver and firmware to
+avoid the compatibility issues with i40e PMD. Here is the suggested matching
+list which has been tested and verified. The detailed information can refer
+to chapter Tested Platforms/Tested NICs in release notes.
+
+   +--------------+-----------------------+------------------+
+   | DPDK version | Kernel driver version | Firmware version |
+   +==============+=======================+==================+
+   |    18.05     |         2.4.6         |       6.01       |
+   +--------------+-----------------------+------------------+
+   |    18.02     |         2.4.3         |       6.01       |
+   +--------------+-----------------------+------------------+
+   |    17.11     |         2.1.26        |       6.01       |
+   +--------------+-----------------------+------------------+
+   |    17.08     |         2.0.19        |       6.01       |
+   +--------------+-----------------------+------------------+
+   |    17.05     |         1.5.23        |       5.05       |
+   +--------------+-----------------------+------------------+
+   |    17.02     |         1.5.23        |       5.05       |
+   +--------------+-----------------------+------------------+
+   |    16.11     |         1.5.23        |       5.05       |
+   +--------------+-----------------------+------------------+
+   |    16.07     |         1.4.25        |       5.04       |
+   +--------------+-----------------------+------------------+
+   |    16.04     |         1.4.25        |       5.02       |
+   +--------------+-----------------------+------------------+
+
 Pre-Installation Configuration
 ------------------------------
 
@@ -96,11 +126,6 @@ Please note that enabling debugging options may affect system performance.
 
   Number of queues reserved for each VMDQ Pool.
 
-- ``CONFIG_RTE_LIBRTE_I40E_ITR_INTERVAL`` (default ``-1``)
-
-  Interrupt Throttling interval.
-
-
 Runtime Config Options
 ~~~~~~~~~~~~~~~~~~~~~~
 
@@ -595,16 +620,6 @@ Use 16 Bytes RX Descriptor Size
 As i40e PMD supports both 16 and 32 bytes RX descriptor sizes, and 16 bytes size can provide helps to high performance of small packets.
 Configuration of ``CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC`` in config files can be changed to use 16 bytes size RX descriptors.
 
-High Performance and per Packet Latency Tradeoff
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Due to the hardware design, the interrupt signal inside NIC is needed for per
-packet descriptor write-back. The minimum interval of interrupts could be set
-at compile time by ``CONFIG_RTE_LIBRTE_I40E_ITR_INTERVAL`` in configuration files.
-Though there is a default configuration, the interval could be tuned by the
-users with that configuration item depends on what the user cares about more,
-performance or per packet latency.
-
 Example of getting best performance with l3fwd example
 ------------------------------------------------------
 
diff --git a/doc/guides/nics/ifcvf.rst b/doc/guides/nics/ifc.rst
index 48f9adf1..48f9adf1 100644
--- a/doc/guides/nics/ifcvf.rst
+++ b/doc/guides/nics/ifc.rst
diff --git a/doc/guides/nics/index.rst b/doc/guides/nics/index.rst
index ddb9eb7a..59f6063d 100644
--- a/doc/guides/nics/index.rst
+++ b/doc/guides/nics/index.rst
@@ -24,7 +24,7 @@ Network Interface Controller Drivers
     enic
     fm10k
     i40e
-    ifcvf
+    ifc
     igb
     ixgbe
     intel_vf
@@ -33,10 +33,12 @@ Network Interface Controller Drivers
     mlx4
     mlx5
     mvpp2
+    netvsc
     nfp
     octeontx
     qede
     sfc_efx
+    softnic
     szedata2
     tap
     thunderx
diff --git a/doc/guides/nics/mlx4.rst b/doc/guides/nics/mlx4.rst
index 491106a9..4a57c7a6 100644
--- a/doc/guides/nics/mlx4.rst
+++ b/doc/guides/nics/mlx4.rst
@@ -142,6 +142,9 @@ Limitations
   The ability to enable/disable CRC stripping requires OFED version
   4.3-1.5.0.0 and above  or rdma-core version v18 and above.
 
+- TSO (Transmit Segmentation Offload) is supported in OFED version
+  4.4 and above.
+
 Prerequisites
 -------------
 
@@ -210,7 +213,7 @@ Current RDMA core package and Linux kernel (recommended)
 Mellanox OFED as a fallback
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- `Mellanox OFED`_ version: **4.3**.
+- `Mellanox OFED`_ version: **4.3, 4.4**.
 - firmware version: **2.42.5000** and above.
 
 .. _`Mellanox OFED`: http://www.mellanox.com/page/products_dyn?product_family=26&mtag=linux_sw_drivers
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 79c982e2..52e1213c 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -49,7 +49,7 @@ libibverbs.
 Features
 --------
 
-- Multi arch support: x86_64, POWER8, ARMv8.
+- Multi arch support: x86_64, POWER8, ARMv8, i686.
 - Multiple TX and RX queues.
 - Support for scattered TX and RX frames.
 - IPv4, IPv6, TCPv4, TCPv6, UDPv4 and UDPv6 RSS on any number of queues.
@@ -132,6 +132,10 @@ Limitations
   the external buffers will be freed by PMD and the application which still
   holds the external buffers may be corrupted.
 
+- If Multi-Packet Rx queue is configured (``mprq_en``) and Rx CQE compression is
+  enabled (``rxq_cqe_comp_en``) at the same time, RSS hash result is not fully
+  supported. Some Rx packets may not have PKT_RX_RSS_HASH.
+
 Statistics
 ----------
 
@@ -388,6 +392,18 @@ Run-time configuration
 
   Disabled by default.
 
+- ``representor`` parameter [list]
+
+  This parameter can be used to instantiate DPDK Ethernet devices from
+  existing port (or VF) representors configured on the device.
+
+  It is a standard parameter whose format is described in
+  :ref:`ethernet_device_standard_device_arguments`.
+
+  For instance, to probe port representors 0 through 2::
+
+    representor=[0-2]
+
 Firmware configuration
 ~~~~~~~~~~~~~~~~~~~~~~
 
@@ -432,6 +448,12 @@ DPDK and must be installed separately:
   This library basically implements send/receive calls to the hardware
   queues.
 
+- **libmnl**
+
+  Minimalistic Netlink library mainly relied on to manage E-Switch flow
+  rules (i.e. those with the "transfer" attribute and typically involving
+  port representors).
+
 - **Kernel modules**
 
   They provide the kernel-side Verbs API and low level device drivers that
@@ -473,6 +495,10 @@ RMDA Core with Linux Kernel
 - Minimal kernel version : v4.14 or the most recent 4.14-rc (see `Linux installation documentation`_)
 - Minimal rdma-core version: v15+ commit 0c5f5765213a ("Merge pull request #227 from yishaih/tm")
   (see `RDMA Core installation documentation`_)
+- When building for i686 use:
+
+  - rdma-core version 18.0 or above built with 32bit support.
+  - Kernel version 4.14.41 or above.
 
 .. _`Linux installation documentation`: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/plain/Documentation/admin-guide/README.rst
 .. _`RDMA Core installation documentation`: https://raw.githubusercontent.com/linux-rdma/rdma-core/master/README.md
@@ -480,7 +506,7 @@ RMDA Core with Linux Kernel
 Mellanox OFED
 ^^^^^^^^^^^^^
 
-- Mellanox OFED version: **4.2, 4.3**.
+- Mellanox OFED version: **4.3, 4.4**.
 - firmware version:
 
   - ConnectX-4: **12.21.1000** and above.
@@ -506,6 +532,19 @@ required from that distribution.
    this DPDK release was developed and tested against is strongly
    recommended. Please check the `prerequisites`_.
 
+Libmnl
+^^^^^^
+
+Minimal version for libmnl is **1.0.3**.
+
+As a dependency of the **iproute2** suite, this library is often installed
+by default. It is otherwise readily available through standard system
+packages.
+
+Its development headers must be installed in order to compile this PMD.
+These packages are usually named **libmnl-dev** or **libmnl-devel**
+depending on the Linux distribution.
+
 Supported NICs
 --------------
 
diff --git a/doc/guides/nics/netvsc.rst b/doc/guides/nics/netvsc.rst
new file mode 100644
index 00000000..345f393c
--- /dev/null
+++ b/doc/guides/nics/netvsc.rst
@@ -0,0 +1,105 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) Microsoft Corporation.
+
+Netvsc poll mode driver
+=======================
+
+The Netvsc Poll Mode driver (PMD) provides support for the paravirtualized
+network device for Microsoft Hyper-V. It can be used with
+Window Server 2008/2012/2016, Windows 10.
+The device offers multi-queue support (if kernel and host support it),
+checksum and segmentation offloads.
+
+
+Features and Limitations of Hyper-V PMD
+---------------------------------------
+
+In this release, the hyper PMD driver provides the basic functionality of packet reception and transmission.
+
+*   It supports merge-able buffers per packet when receiving packets and scattered buffer per packet
+    when transmitting packets. The packet size supported is from 64 to 65536.
+
+*   The PMD supports multicast packets and promiscuous mode subject to restrictions on the host.
+    In order to this to work, the guest network configuration on Hyper-V must be configured to allow MAC address
+    spoofing.
+
+*   The device has only a single MAC address.
+    Hyper-V driver does not support MAC or VLAN filtering because the Hyper-V host does not support it.
+
+*   VLAN tags are always stripped and presented in mbuf tci field.
+
+*   The Hyper-V driver does not use or support Link State or Rx interrupt.
+
+*   The maximum number of queues is limited by the host (currently 64).
+    When used with 4.16 kernel only a single queue is available.
+
+.. note::
+   This driver is intended for use with **Hyper-V only** and is
+   not recommended for use on Azure because accelerated Networking
+   (SR-IOV) is not supported.
+
+   On Azure, use the :doc:`vdev_netvsc` which
+   automatically configures the necessary TAP and failsave drivers.
+
+
+Installation
+------------
+
+The Netvsc PMD is a standalone driver, similar to virtio and vmxnet3.
+Using Netvsc PMD requires that the associated VMBUS device be bound to the userspace
+I/O device driver for Hyper-V (uio_hv_generic). By default, all netvsc devices
+will be bound to the Linux kernel driver; in order to use netvsc PMD the
+device must first be overridden.
+
+The first step is to identify the network device to override.
+VMBUS uses Universal Unique Identifiers
+(`UUID`_) to identify devices on the bus similar to how PCI uses Domain:Bus:Function.
+The UUID associated with a Linux kernel network device can be determined
+by looking at the sysfs information. To find the UUID for eth1 and
+store it in a shell variable:
+
+    .. code-block:: console
+
+	DEV_UUID=$(basename $(readlink /sys/class/net/eth1/device))
+
+
+.. _`UUID`: https://en.wikipedia.org/wiki/Universally_unique_identifier
+
+There are several possible ways to assign the uio device driver for a device.
+The easiest way (but only on 4.18 or later)
+is to use the `driverctl Device Driver control utility`_ to override
+the normal kernel device.
+
+    .. code-block:: console
+
+	driverctl -b vmbus set-override $DEV_UUID uio_hv_generic
+
+.. _`driverctl Device Driver control utility`: https://gitlab.com/driverctl/driverctl
+
+Any settings done with driverctl are by default persistent and will be reapplied
+on reboot.
+
+On older kernels, the same effect can be had by manual sysfs bind and unbind
+operations:
+
+    .. code-block:: console
+
+	NET_UUID="f8615163-df3e-46c5-913f-f2d2f965ed0e"
+	modprobe uio_hv_generic
+	echo $NET_UUID > /sys/bus/vmbus/drivers/uio_hv_generic/new_id
+	echo $DEV_UUID > /sys/bus/vmbus/drivers/hv_netvsc/unbind
+	echo $DEV_UUID > /sys/bus/vmbus/drivers/uio_hv_generic/bind
+
+.. Note::
+
+   The dpkd-devbind.py script can not be used since it only handles PCI devices.
+
+
+Prerequisites
+-------------
+
+The following prerequisites apply:
+
+*   Linux kernel support for UIO on vmbus is done with the uio_hv_generic driver.
+    Full support of multiple queues requires the 4.17 kernel. It is possible
+    to use the netvsc PMD with 4.16 kernel but it is limited to a single queue.
diff --git a/doc/guides/nics/pcap_ring.rst b/doc/guides/nics/pcap_ring.rst
index 7fd063c9..879e5430 100644
--- a/doc/guides/nics/pcap_ring.rst
+++ b/doc/guides/nics/pcap_ring.rst
@@ -71,11 +71,19 @@ The different stream types are:
         tx_pcap=/path/to/file.pcap
 
 *   rx_iface: Defines a reception stream based on a network interface name.
-    The driver reads packets coming from the given interface using the Linux kernel driver for that interface.
+    The driver reads packets from the given interface using the Linux kernel driver for that interface.
+    The driver captures both the incoming and outgoing packets on that interface.
     The value is an interface name.
 
         rx_iface=eth0
 
+*   rx_iface_in: Defines a reception stream based on a network interface name.
+    The driver reads packets from the given interface using the Linux kernel driver for that interface.
+    The driver captures only the incoming packets on that interface.
+    The value is an interface name.
+
+        rx_iface_in=eth0
+
 *   tx_iface: Defines a transmission stream based on a network interface name.
     The driver sends packets to the given interface using the Linux kernel driver for that interface.
     The value is an interface name.
@@ -122,6 +130,21 @@ Forward packets through two network interfaces:
     $RTE_TARGET/app/testpmd -l 0-3 -n 4 \
         --vdev 'net_pcap0,iface=eth0' --vdev='net_pcap1;iface=eth1'
 
+Enable 2 tx queues on a network interface:
+
+.. code-block:: console
+
+    $RTE_TARGET/app/testpmd -l 0-3 -n 4 \
+        --vdev 'net_pcap0,rx_iface=eth1,tx_iface=eth1,tx_iface=eth1' \
+        -- --txq 2
+
+Read only incoming packets from a network interface and write them back to the same network interface:
+
+.. code-block:: console
+
+    $RTE_TARGET/app/testpmd -l 0-3 -n 4 \
+        --vdev 'net_pcap0,rx_iface_in=eth1,tx_iface=eth1'
+
 Using libpcap-based PMD with the testpmd Application
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/doc/guides/nics/qede.rst b/doc/guides/nics/qede.rst
index a4adf756..cba38868 100644
--- a/doc/guides/nics/qede.rst
+++ b/doc/guides/nics/qede.rst
@@ -66,12 +66,13 @@ Prerequisites
   or `QLogic Driver Download Center <http://driverdownloads.qlogic.com/QLogicDriverDownloads_UI/DefaultNewSearch.aspx>`_.
   To download firmware file from QLogic website, select adapter category, model and DPDK Poll Mode Driver.
 
-- Requires management firmware (MFW) version **8.34.x.x** or higher to be
-  flashed on to the adapter. If the required management firmware is not
-  available then download from
-  `QLogic Driver Download Center <http://driverdownloads.qlogic.com/QLogicDriverDownloads_UI/DefaultNewSearch.aspx>`_.
-  For downloading firmware upgrade utility, select adapter category, model and Linux distro.
-  To flash the management firmware refer to the instructions in the QLogic Firmware Upgrade Utility Readme document.
+- Requires the NIC be updated minimally with **8.30.x.x** Management firmware(MFW) version supported for that NIC.
+  It is highly recommended that the NIC be updated with the latest available management firmware version to get latest feature  set.
+  Management Firmware and Firmware Upgrade Utility for Cavium FastLinQ(r) branded adapters can be downloaded from
+  `Driver Download Center <http://driverdownloads.qlogic.com/QLogicDriverDownloads_UI/DefaultNewSearch.aspx>`_.
+  For downloading Firmware Upgrade Utility, select NIC category, model and Linux distro.
+  To update the management firmware, refer to the instructions in the Firmware Upgrade Utility Readme document.
+  For OEM branded adapters please follow the instruction provided by the OEM to update the Management Firmware on the NIC.
 
 - SR-IOV requires Linux PF driver version **8.20.x.x** or higher.
   If the required PF driver is not available then download it from
diff --git a/doc/guides/nics/softnic.rst b/doc/guides/nics/softnic.rst
new file mode 100644
index 00000000..6c2287a1
--- /dev/null
+++ b/doc/guides/nics/softnic.rst
@@ -0,0 +1,250 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2018 Intel Corporation.
+
+Soft NIC Poll Mode Driver
+=========================
+
+The Soft NIC allows building custom NIC pipelines in software. The Soft NIC pipeline
+is DIY and reconfigurable through ``firmware`` (DPDK Packet Framework script).
+
+The Soft NIC leverages the DPDK Packet Framework libraries (librte_port,
+librte_table and librte_pipeline) to make it modular, flexible and extensible
+with new functionality. Please refer to DPDK Programmer's Guide, Chapter
+``Packet Framework`` and DPDK Sample Application User Guide,
+Chapter ``IP Pipeline Application`` for more details.
+
+The Soft NIC is configured through the standard DPDK ethdev API (ethdev, flow,
+QoS, security). The internal framework is not externally visible.
+
+Key benefits:
+ - Can be used to augment missing features to HW NICs.
+ - Allows consumption of advanced DPDK features without application redesign.
+ - Allows out-of-the-box performance boost of DPDK consumers applications simply by
+   instantiating this type of Ethernet device.
+
+Flow
+----
+* ``Device creation``: Each Soft NIC instance is a virtual device.
+
+* ``Device start``: The Soft NIC firmware script is executed every time the device
+  is started. The firmware script typically creates several internal objects,
+  such as: memory pools, SW queues, traffic manager, action profiles, pipelines,
+  etc.
+
+* ``Device stop``: All the internal objects that were previously created by the
+  firmware script during device start are now destroyed.
+
+* ``Device run``: Each Soft NIC device needs one or several CPU cores to run.
+  The firmware script maps each internal pipeline to a CPU core. Multiple
+  pipelines can be mapped to the same CPU core. In order for a given pipeline
+  assigned to CPU core X to run, the application needs to periodically call on
+  CPU core X the `rte_pmd_softnic_run()` function for the current Soft NIC
+  device.
+
+* ``Application run``: The application reads packets from the Soft NIC device RX
+  queues and writes packets to the Soft NIC device TX queues.
+
+Supported Operating Systems
+---------------------------
+
+Any Linux distribution fulfilling the conditions described in ``System Requirements``
+section of :ref:`the DPDK documentation <linux_gsg>` or refer to *DPDK
+Release Notes*.
+
+Build options
+-------------
+
+The default PMD configuration available in the common_linuxapp configuration file:
+
+CONFIG_RTE_LIBRTE_PMD_SOFTNIC=y
+
+Once the DPDK is built, all the DPDK applications include support for the
+Soft NIC PMD.
+
+Soft NIC PMD arguments
+----------------------
+
+The user can specify below arguments in EAL ``--vdev`` options to create the
+Soft NIC device instance:
+
+        --vdev "net_softnic0,firmware=firmware.cli,conn_port=8086"
+
+#.  ``firmware``: path to the firmware script used for Soft NIC configuration.
+    The example "firmware" script is provided at `drivers/net/softnic/`.
+    (Optional: No, Default = NA)
+
+#.  ``conn_port``: tcp connection port (non-zero value) used by remote client
+    (for examples- telnet, netcat, etc.) to connect and configure Soft NIC device in run-time.
+    (Optional: yes, Default value: 0, no connection with external client)
+
+#.  ``cpu_id``: numa node id. (Optional: yes, Default value: 0)
+
+#.  ``tm_n_queues``: number of traffic manager's scheduler queues. The traffic manager
+    is based on DPDK *librte_sched* library. (Optional: yes, Default value: 65,536 queues)
+
+#.  ``tm_qsize0``: size of scheduler queue 0 per traffic class of the pipes/subscribers.
+    (Optional: yes, Default: 64)
+
+#.  ``tm_qsize1``: size of scheduler queue 1 per traffic class of the pipes/subscribers.
+    (Optional: yes, Default: 64)
+
+#.  ``tm_qsize2``: size of scheduler queue 2 per traffic class of the pipes/subscribers.
+    (Optional: yes, Default: 64)
+
+#.  ``tm_qsize3``: size of scheduler queue 3 per traffic class of the pipes/subscribers.
+    (Optional: yes, Default: 64)
+
+
+Soft NIC testing
+----------------
+
+* Run testpmd application in Soft NIC forwarding mode with loopback feature
+  enabled on Soft NIC port:
+
+    .. code-block:: console
+
+         ./testpmd -c 0x3 --vdev 'net_softnic0,firmware=<script path>/firmware.cli,cpu_id=0,conn_port=8086' -- -i
+              --forward-mode=softnic --portmask=0x2
+
+    .. code-block:: console
+
+        ...
+        Interactive-mode selected
+        Set softnic packet forwarding mode
+        ...
+        Configuring Port 0 (socket 0)
+        Port 0: 90:E2:BA:37:9D:DC
+        Configuring Port 1 (socket 0)
+
+        ; SPDX-License-Identifier: BSD-3-Clause
+        ; Copyright(c) 2018 Intel Corporation
+
+        link LINK dev 0000:02:00.0
+
+        pipeline RX period 10 offset_port_id 0
+        pipeline RX port in bsz 32 link LINK rxq 0
+        pipeline RX port out bsz 32 swq RXQ0
+        pipeline RX table match stub
+        pipeline RX port in 0 table 0
+
+        pipeline TX period 10 offset_port_id 0
+        pipeline TX port in bsz 32 swq TXQ0
+        pipeline TX port out bsz 32 link LINK txq 0
+        pipeline TX table match stub
+        pipeline TX port in 0 table 0
+
+        thread 1 pipeline RX enable
+        thread 1 pipeline TX enable
+        Port 1: 00:00:00:00:00:00
+        Checking link statuses...
+        Done
+        testpmd>
+
+* Start forwarding
+
+    .. code-block:: console
+
+         testpmd> start
+         softnic packet forwarding - ports=1 - cores=1 - streams=1 - NUMA support enabled, MP over anonymous pages disabled
+         Logical Core 1 (socket 0) forwards packets on 1 streams:
+         RX P=2/Q=0 (socket 0) -> TX P=2/Q=0 (socket 0) peer=02:00:00:00:00:02
+
+         softnic packet forwarding packets/burst=32
+         nb forwarding cores=1 - nb forwarding ports=1
+         port 0: RX queue number: 1 Tx queue number: 1
+         Rx offloads=0x1000 Tx offloads=0x0
+         RX queue: 0
+         RX desc=512 - RX free threshold=32
+         RX threshold registers: pthresh=8 hthresh=8  wthresh=0
+         RX Offloads=0x0
+         TX queue: 0
+         TX desc=512 - TX free threshold=32
+         TX threshold registers: pthresh=32 hthresh=0  wthresh=0
+         TX offloads=0x0 - TX RS bit threshold=32
+         port 1: RX queue number: 1 Tx queue number: 1
+         Rx offloads=0x0 Tx offloads=0x0
+         RX queue: 0
+         RX desc=0 - RX free threshold=0
+         RX threshold registers: pthresh=0 hthresh=0  wthresh=0
+         RX Offloads=0x0
+         TX queue: 0
+         TX desc=0 - TX free threshold=0
+         TX threshold registers: pthresh=0 hthresh=0  wthresh=0
+         TX offloads=0x0 - TX RS bit threshold=0
+
+* Start remote client (e.g. telnet) to communicate with the softnic device:
+
+    .. code-block:: console
+
+        $ telnet 127.0.0.1 8086
+        Trying 127.0.0.1...
+        Connected to 127.0.0.1.
+        Escape character is '^]'.
+
+        Welcome to Soft NIC!
+
+        softnic>
+
+* Add/update Soft NIC pipeline table match-action entries from telnet client:
+
+    .. code-block:: console
+
+        softnic> pipeline RX table 0 rule add match default action fwd port 0
+        softnic> pipeline TX table 0 rule add match default action fwd port 0
+
+Soft NIC Firmware
+-----------------
+
+The Soft NIC firmware, for example- `softnic/firmware.cli`, consists of following CLI commands
+for creating and managing software based NIC pipelines. For more details, please refer to CLI
+command description provided in `softnic/rte_eth_softnic_cli.c`.
+
+* Physical port for packets send/receive:
+
+    .. code-block:: console
+
+        link LINK dev 0000:02:00.0
+
+* Pipeline create:
+
+    .. code-block:: console
+
+        pipeline RX period 10 offset_port_id 0           (Soft NIC rx-path pipeline)
+        pipeline TX period 10 offset_port_id 0           (Soft NIC tx-path pipeline)
+
+* Pipeline input/output port create
+
+    .. code-block:: console
+
+        pipeline RX port in bsz 32 link LINK rxq 0      (Soft NIC rx pipeline input port)
+        pipeline RX port out bsz 32 swq RXQ0            (Soft NIC rx pipeline output port)
+        pipeline TX port in bsz 32 swq TXQ0             (Soft NIC tx pipeline input port)
+        pipeline TX port out bsz 32 link LINK txq 0     (Soft NIC tx pipeline output port)
+
+* Pipeline table create
+
+    .. code-block:: console
+
+        pipeline RX table match stub             (Soft NIC rx pipeline match-action table)
+        pipeline TX table match stub             (Soft NIC tx pipeline match-action table)
+
+* Pipeline input port connection with table
+
+    .. code-block:: console
+
+        pipeline RX port in 0 table 0          (Soft NIC rx pipeline input port 0 connection with table 0)
+        pipeline TX port in 0 table 0          (Soft NIC tx pipeline input port 0 connection with table 0)
+
+* Pipeline table match-action rules add
+
+    .. code-block:: console
+
+        pipeline RX table 0 rule add match default action fwd port 0        (Soft NIC rx pipeline table 0 rule)
+        pipeline TX table 0 rule add match default action fwd port 0        (Soft NIC tx pipeline table 0 rule)
+
+* Enable pipeline on CPU thread
+
+    .. code-block:: console
+
+        thread 1 pipeline RX enable        (Soft NIC rx pipeline enable on cpu thread id 1)
+        thread 1 pipeline TX enable        (Soft NIC tx pipeline enable on cpu thread id 1)
diff --git a/doc/guides/nics/thunderx.rst b/doc/guides/nics/thunderx.rst
index 26423747..e84eaafe 100644
--- a/doc/guides/nics/thunderx.rst
+++ b/doc/guides/nics/thunderx.rst
@@ -30,6 +30,7 @@ Features of the ThunderX PMD are:
 - SR-IOV VF
 - NUMA support
 - Multi queue set support (up to 96 queues (12 queue sets)) per port
+- Skip data bytes
 
 Supported ThunderX SoCs
 -----------------------
@@ -312,6 +313,21 @@ We will choose four secondary queue sets from the ending of the list (0002:01:01
 
 The nicvf thunderx driver will make use of attached secondary VFs automatically during the interface configuration stage.
 
+
+Module params
+--------------
+
+skip_data_bytes
+~~~~~~~~~~~~~~~
+This feature is used to create a hole between HEADROOM and actual data. Size of hole is specified
+in bytes as module param("skip_data_bytes") to pmd.
+This scheme is useful when application would like to insert vlan header without disturbing HEADROOM.
+
+Example:
+   .. code-block:: console
+
+      -w 0002:01:00.2,skip_data_bytes=8
+
 Limitations
 -----------
 
@@ -335,3 +351,8 @@ Maximum packet segments
 The ThunderX SoC family NICs support up to 12 segments per packet when working
 in scatter/gather mode. So, setting MTU will result with ``EINVAL`` when the
 frame size does not fit in the maximum number of segments.
+
+skip_data_bytes
+~~~~~~~~~~~~~~~
+
+Maximum limit of skip_data_bytes is 128 bytes and number of bytes should be multiple of 8.
diff --git a/doc/guides/nics/vdev_netvsc.rst b/doc/guides/nics/vdev_netvsc.rst
index e02ae9ac..d1da0711 100644
--- a/doc/guides/nics/vdev_netvsc.rst
+++ b/doc/guides/nics/vdev_netvsc.rst
@@ -93,7 +93,7 @@ The following device parameters are supported:
 
 - ``ignore`` [int]
 
-  If nonzero, ignores the driver runnig (actually used to disable the
+  If nonzero, ignores the driver running (actually used to disable the
   auto-detection in Hyper-V VM).
 
 .. note::
diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst
index a42d1bb3..7c099fb7 100644
--- a/doc/guides/nics/virtio.rst
+++ b/doc/guides/nics/virtio.rst
@@ -201,7 +201,7 @@ The packet transmission flow is:
 Virtio PMD Rx/Tx Callbacks
 --------------------------
 
-Virtio driver has 3 Rx callbacks and 2 Tx callbacks.
+Virtio driver has 4 Rx callbacks and 3 Tx callbacks.
 
 Rx callbacks:
 
@@ -215,6 +215,9 @@ Rx callbacks:
    Vector version without mergeable Rx buffer support, also fixes the available
    ring indexes and uses vector instructions to optimize performance.
 
+#. ``virtio_recv_mergeable_pkts_inorder``:
+   In-order version with mergeable Rx buffer support.
+
 Tx callbacks:
 
 #. ``virtio_xmit_pkts``:
@@ -223,6 +226,8 @@ Tx callbacks:
 #. ``virtio_xmit_pkts_simple``:
    Vector version fixes the available ring indexes to optimize performance.
 
+#. ``virtio_xmit_pkts_inorder``:
+   In-order version.
 
 By default, the non-vector callbacks are used:
 
@@ -254,6 +259,12 @@ Example of using the vector version of the virtio poll mode driver in
 
    testpmd -l 0-2 -n 4 -- -i --tx-offloads=0x0 --rxq=1 --txq=1 --nb-cores=1
 
+In-order callbacks only work on simulated virtio user vdev.
+
+*   For Rx: If mergeable Rx buffers is enabled and in-order is enabled then
+    ``virtio_xmit_pkts_inorder`` is used.
+
+*   For Tx: If in-order is enabled then ``virtio_xmit_pkts_inorder`` is used.
 
 Interrupt mode
 --------------
@@ -331,3 +342,13 @@ The user can specify below argument in devargs.
     driver, and works as a HW vhost backend. This argument is used to specify
     a virtio device needs to work in vDPA mode.
     (Default: 0 (disabled))
+
+#. ``mrg_rxbuf``:
+
+    It is used to enable virtio device mergeable Rx buffer feature.
+    (Default: 1 (enabled))
+
+#. ``in_order``:
+
+    It is used to enable virtio device in-order feature.
+    (Default: 1 (enabled))
diff --git a/doc/guides/prog_guide/bbdev.rst b/doc/guides/prog_guide/bbdev.rst
index b0f44cc0..9de14443 100644
--- a/doc/guides/prog_guide/bbdev.rst
+++ b/doc/guides/prog_guide/bbdev.rst
@@ -566,6 +566,15 @@ The case when one CB belongs to TB and is being enqueued individually to BBDEV,
 this case is considered as a special case of partial TB where its number of CBs
 is 1. Therefore, it requires to get processed in TB-mode.
 
+The figure below visualizes the encoding of CBs using BBDEV interface in
+TB-mode. CB-mode is a reduced version, where only one CB exists:
+
+.. _figure_turbo_tb_encode:
+
+.. figure:: img/turbo_tb_encode.*
+
+    Turbo encoding of Code Blocks in mbuf structure
+
 
 BBDEV Turbo Decode Operation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -651,6 +660,15 @@ is 1. Therefore, it requires to get processed in TB-mode.
 The output mbuf data structure is expected to be allocated by the application
 with enough room for the output data.
 
+The figure below visualizes the decoding of CBs using BBDEV interface in
+TB-mode. CB-mode is a reduced version, where only one CB exists:
+
+.. _figure_turbo_tb_decode:
+
+.. figure:: img/turbo_tb_decode.*
+
+    Turbo decoding of Code Blocks in mbuf structure
+
 
 Sample code
 -----------
diff --git a/doc/guides/prog_guide/cryptodev_lib.rst b/doc/guides/prog_guide/cryptodev_lib.rst
index 30f0bcf7..90d01e93 100644
--- a/doc/guides/prog_guide/cryptodev_lib.rst
+++ b/doc/guides/prog_guide/cryptodev_lib.rst
@@ -8,7 +8,7 @@ The cryptodev library provides a Crypto device framework for management and
 provisioning of hardware and software Crypto poll mode drivers, defining generic
 APIs which support a number of different Crypto operations. The framework
 currently only supports cipher, authentication, chained cipher/authentication
-and AEAD symmetric Crypto operations.
+and AEAD symmetric and asymmetric Crypto operations.
 
 
 Design Principles
@@ -41,7 +41,7 @@ From the command line using the --vdev EAL option
 
 .. code-block:: console
 
-   --vdev  'crypto_aesni_mb0,max_nb_queue_pairs=2,max_nb_sessions=1024,socket_id=0'
+   --vdev  'crypto_aesni_mb0,max_nb_queue_pairs=2,socket_id=0'
 
 .. Note::
 
@@ -57,12 +57,11 @@ Our using the rte_vdev_init API within the application code.
 .. code-block:: c
 
    rte_vdev_init("crypto_aesni_mb",
-                     "max_nb_queue_pairs=2,max_nb_sessions=1024,socket_id=0")
+                     "max_nb_queue_pairs=2,socket_id=0")
 
 All virtual Crypto devices support the following initialization parameters:
 
 * ``max_nb_queue_pairs`` - maximum number of queue pairs supported by the device.
-* ``max_nb_sessions`` - maximum number of sessions supported by the device
 * ``socket_id`` - socket on which to allocate the device resources on.
 
 
@@ -159,8 +158,8 @@ Device Features and Capabilities
 Crypto devices define their functionality through two mechanisms, global device
 features and algorithm capabilities. Global devices features identify device
 wide level features which are applicable to the whole device such as
-the device having hardware acceleration or supporting symmetric Crypto
-operations,
+the device having hardware acceleration or supporting symmetric and/or asymmetric
+Crypto operations.
 
 The capabilities mechanism defines the individual algorithms/functions which
 the device supports, such as a specific symmetric Crypto cipher,
@@ -269,7 +268,7 @@ relevant information for the device.
     struct rte_cryptodev_info {
         const char *driver_name;
         uint8_t driver_id;
-        struct rte_pci_device *pci_dev;
+        struct rte_device *device;
 
         uint64_t feature_flags;
 
@@ -302,24 +301,24 @@ enqueue call.
 Private data
 ~~~~~~~~~~~~
 For session-based operations, the set and get API provides a mechanism for an
-application to store and retrieve the private data information stored along with
-the crypto session.
+application to store and retrieve the private user data information stored along
+with the crypto session.
 
 For example, suppose an application is submitting a crypto operation with a session
-associated and wants to indicate private data information which is required to be
+associated and wants to indicate private user data information which is required to be
 used after completion of the crypto operation. In this case, the application can use
-the set API to set the private data and retrieve it using get API.
+the set API to set the user data and retrieve it using get API.
 
 .. code-block:: c
 
-	int rte_cryptodev_sym_session_set_private_data(
+	int rte_cryptodev_sym_session_set_user_data(
 		struct rte_cryptodev_sym_session *sess,	void *data, uint16_t size);
 
-	void * rte_cryptodev_sym_session_get_private_data(
+	void * rte_cryptodev_sym_session_get_user_data(
 		struct rte_cryptodev_sym_session *sess);
 
 
-For session-less mode, the private data information can be placed along with the
+For session-less mode, the private user data information can be placed along with the
 ``struct rte_crypto_op``. The ``rte_crypto_op::private_data_offset`` indicates the
 start of private data information. The offset is counted from the start of the
 rte_crypto_op including other crypto information such as the IVs (since there can
@@ -446,7 +445,7 @@ Crypto workloads.
 
 .. figure:: img/cryptodev_sym_sess.*
 
-The Crypto device framework provides APIs to allocate and initizalize sessions
+The Crypto device framework provides APIs to allocate and initialize sessions
 for crypto devices, where sessions are mempool objects.
 It is the application's responsibility to create and manage the session mempools.
 This approach allows for different scenarios such as having a single session
@@ -788,14 +787,260 @@ using one of the crypto PMDs available in DPDK.
                                             num_dequeued_ops);
     } while (total_num_dequeued_ops < num_enqueued_ops);
 
-
 Asymmetric Cryptography
 -----------------------
 
-Asymmetric functionality is currently not supported by the cryptodev API.
+The cryptodev library currently provides support for the following asymmetric
+Crypto operations; RSA, Modular exponentiation and inversion, Diffie-Hellman
+public and/or private key generation and shared secret compute, DSA Signature
+generation and verification.
+
+Session and Session Management
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sessions are used in asymmetric cryptographic processing to store the immutable
+data defined in asymmetric cryptographic transform which is further used in the
+operation processing. Sessions typically stores information, such as, public
+and private key information or domain params or prime modulus data i.e. immutable
+across data sets. Crypto sessions cache this immutable data in a optimal way for the
+underlying PMD and this allows further acceleration of the offload of Crypto workloads.
+
+Like symmetric, the Crypto device framework provides APIs to allocate and initialize
+asymmetric sessions for crypto devices, where sessions are mempool objects.
+It is the application's responsibility to create and manage the session mempools.
+Application using both symmetric and asymmetric sessions should allocate and maintain
+different sessions pools for each type.
+
+An application can use ``rte_cryptodev_get_asym_session_private_size()`` to
+get the private size of asymmetric session on a given crypto device. This
+function would allow an application to calculate the max device asymmetric
+session size of all crypto devices to create a single session mempool.
+If instead an application creates multiple asymmetric session mempools,
+the Crypto device framework also provides ``rte_cryptodev_asym_get_header_session_size()`` to get
+the size of an uninitialized session.
+
+Once the session mempools have been created, ``rte_cryptodev_asym_session_create()``
+is used to allocate an uninitialized asymmetric session from the given mempool.
+The session then must be initialized using ``rte_cryptodev_asym_session_init()``
+for each of the required crypto devices. An asymmetric transform chain
+is used to specify the operation and its parameters. See the section below for
+details on transforms.
+
+When a session is no longer used, user must call ``rte_cryptodev_asym_session_clear()``
+for each of the crypto devices that are using the session, to free all driver
+private asymmetric session data. Once this is done, session should be freed using
+``rte_cryptodev_asym_session_free()`` which returns them to their mempool.
+
+Asymmetric Sessionless Support
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Currently asymmetric crypto framework does not support sessionless.
+
+Transforms and Transform Chaining
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Asymmetric Crypto transforms (``rte_crypto_asym_xform``) are the mechanism used
+to specify the details of the asymmetric Crypto operation. Next pointer within
+xform allows transform to be chained together. Also it is important to note that
+the order in which the transforms are passed indicates the order of the chaining.
+
+Not all asymmetric crypto xforms are supported for chaining. Currently supported
+asymmetric crypto chaining is Diffie-Hellman private key generation followed by
+public generation. Also, currently API does not support chaining of symmetric and
+asymmetric crypto xfroms.
+
+Each xform defines specific asymmetric crypto algo. Currently supported are:
+* RSA
+* Modular operations (Exponentiation and Inverse)
+* Diffie-Hellman
+* DSA
+* None - special case where PMD may support a passthrough mode. More for diagnostic purpose
+
+See *DPDK API Reference* for details on each rte_crypto_xxx_xform struct
+
+Asymmetric Operations
+~~~~~~~~~~~~~~~~~~~~~
+
+The asymmetric Crypto operation structure contains all the mutable data relating
+to asymmetric cryptographic processing on an input data buffer. It uses either
+RSA, Modular, Diffie-Hellman or DSA operations depending upon session it is attached
+to.
+
+Every operation must carry a valid session handle which further carries information
+on xform or xform-chain to be performed on op. Every xform type defines its own set
+of operational params in their respective rte_crypto_xxx_op_param struct. Depending
+on xform information within session, PMD picks up and process respective op_param
+struct.
+Unlike symmetric, asymmetric operations do not use mbufs for input/output.
+They operate on data buffer of type ``rte_crypto_param``.
+
+See *DPDK API Reference* for details on each rte_crypto_xxx_op_param struct
+
+Asymmetric crypto Sample code
+-----------------------------
+
+There's a unit test application test_cryptodev_asym.c inside unit test framework that
+show how to setup and process asymmetric operations using cryptodev library.
+
+The following sample code shows the basic steps to compute modular exponentiation
+using 1024-bit modulus length using openssl PMD available in DPDK (performing other
+crypto operations is similar except change to respective op and xform setup).
+
+.. code-block:: c
+
+    /*
+     * Simple example to compute modular exponentiation with 1024-bit key
+     *
+     */
+    #define MAX_ASYM_SESSIONS	10
+    #define NUM_ASYM_BUFS	10
+
+    struct rte_mempool *crypto_op_pool, *asym_session_pool;
+    unsigned int asym_session_size;
+    int ret;
+
+    /* Initialize EAL. */
+    ret = rte_eal_init(argc, argv);
+    if (ret < 0)
+        rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
+
+    uint8_t socket_id = rte_socket_id();
+
+    /* Create crypto operation pool. */
+    crypto_op_pool = rte_crypto_op_pool_create(
+                                    "crypto_op_pool",
+                                    RTE_CRYPTO_OP_TYPE_ASYMMETRIC,
+                                    NUM_ASYM_BUFS, 0, 0,
+                                    socket_id);
+    if (crypto_op_pool == NULL)
+        rte_exit(EXIT_FAILURE, "Cannot create crypto op pool\n");
+
+    /* Create the virtual crypto device. */
+    char args[128];
+    const char *crypto_name = "crypto_openssl";
+    snprintf(args, sizeof(args), "socket_id=%d", socket_id);
+    ret = rte_vdev_init(crypto_name, args);
+    if (ret != 0)
+        rte_exit(EXIT_FAILURE, "Cannot create virtual device");
+
+    uint8_t cdev_id = rte_cryptodev_get_dev_id(crypto_name);
+
+    /* Get private asym session data size. */
+    asym_session_size = rte_cryptodev_get_asym_private_session_size(cdev_id);
 
+    /*
+     * Create session mempool, with two objects per session,
+     * one for the session header and another one for the
+     * private asym session data for the crypto device.
+     */
+    asym_session_pool = rte_mempool_create("asym_session_pool",
+                                    MAX_ASYM_SESSIONS * 2,
+                                    asym_session_size,
+                                    0,
+                                    0, NULL, NULL, NULL,
+                                    NULL, socket_id,
+                                    0);
+
+    /* Configure the crypto device. */
+    struct rte_cryptodev_config conf = {
+        .nb_queue_pairs = 1,
+        .socket_id = socket_id
+    };
+    struct rte_cryptodev_qp_conf qp_conf = {
+        .nb_descriptors = 2048
+    };
+
+    if (rte_cryptodev_configure(cdev_id, &conf) < 0)
+        rte_exit(EXIT_FAILURE, "Failed to configure cryptodev %u", cdev_id);
+
+    if (rte_cryptodev_queue_pair_setup(cdev_id, 0, &qp_conf,
+                            socket_id, asym_session_pool) < 0)
+        rte_exit(EXIT_FAILURE, "Failed to setup queue pair\n");
 
-Crypto Device API
-~~~~~~~~~~~~~~~~~
+    if (rte_cryptodev_start(cdev_id) < 0)
+        rte_exit(EXIT_FAILURE, "Failed to start device\n");
+
+    /* Setup crypto xform to do modular exponentiation with 1024 bit
+	 * length modulus
+	 */
+    struct rte_crypto_asym_xform modex_xform = {
+		.next = NULL,
+		.xform_type = RTE_CRYPTO_ASYM_XFORM_MODEX,
+		.modex = {
+			.modulus = {
+				.data =
+				(uint8_t *)
+				("\xb3\xa1\xaf\xb7\x13\x08\x00\x0a\x35\xdc\x2b\x20\x8d"
+				"\xa1\xb5\xce\x47\x8a\xc3\x80\xf4\x7d\x4a\xa2\x62\xfd\x61\x7f"
+				"\xb5\xa8\xde\x0a\x17\x97\xa0\xbf\xdf\x56\x5a\x3d\x51\x56\x4f"
+				"\x70\x70\x3f\x63\x6a\x44\x5b\xad\x84\x0d\x3f\x27\x6e\x3b\x34"
+				"\x91\x60\x14\xb9\xaa\x72\xfd\xa3\x64\xd2\x03\xa7\x53\x87\x9e"
+				"\x88\x0b\xc1\x14\x93\x1a\x62\xff\xb1\x5d\x74\xcd\x59\x63\x18"
+				"\x11\x3d\x4f\xba\x75\xd4\x33\x4e\x23\x6b\x7b\x57\x44\xe1\xd3"
+				"\x03\x13\xa6\xf0\x8b\x60\xb0\x9e\xee\x75\x08\x9d\x71\x63\x13"
+				"\xcb\xa6\x81\x92\x14\x03\x22\x2d\xde\x55"),
+				.length = 128
+			},
+			.exponent = {
+				.data = (uint8_t *)("\x01\x00\x01"),
+				.length = 3
+			}
+		}
+    };
+    /* Create asym crypto session and initialize it for the crypto device. */
+    struct rte_cryptodev_asym_session *asym_session;
+    asym_session = rte_cryptodev_asym_session_create(asym_session_pool);
+    if (asym_session == NULL)
+        rte_exit(EXIT_FAILURE, "Session could not be created\n");
+
+    if (rte_cryptodev_asym_session_init(cdev_id, asym_session,
+                    &modex_xform, asym_session_pool) < 0)
+        rte_exit(EXIT_FAILURE, "Session could not be initialized "
+                    "for the crypto device\n");
+
+    /* Get a burst of crypto operations. */
+    struct rte_crypto_op *crypto_ops[1];
+    if (rte_crypto_op_bulk_alloc(crypto_op_pool,
+                            RTE_CRYPTO_OP_TYPE_ASYMMETRIC,
+                            crypto_ops, 1) == 0)
+        rte_exit(EXIT_FAILURE, "Not enough crypto operations available\n");
+
+    /* Set up the crypto operations. */
+    struct rte_crypto_asym_op *asym_op = crypto_ops[0]->asym;
+
+	/* calculate mod exp of value 0xf8 */
+    static unsigned char base[] = {0xF8};
+    asym_op->modex.base.data = base;
+    asym_op->modex.base.length = sizeof(base);
+	asym_op->modex.base.iova = base;
+
+    /* Attach the asym crypto session to the operation */
+    rte_crypto_op_attach_asym_session(op, asym_session);
+
+    /* Enqueue the crypto operations in the crypto device. */
+    uint16_t num_enqueued_ops = rte_cryptodev_enqueue_burst(cdev_id, 0,
+                                            crypto_ops, 1);
+
+    /*
+     * Dequeue the crypto operations until all the operations
+     * are processed in the crypto device.
+     */
+    uint16_t num_dequeued_ops, total_num_dequeued_ops = 0;
+    do {
+        struct rte_crypto_op *dequeued_ops[1];
+        num_dequeued_ops = rte_cryptodev_dequeue_burst(cdev_id, 0,
+                                        dequeued_ops, 1);
+        total_num_dequeued_ops += num_dequeued_ops;
+
+        /* Check if operation was processed successfully */
+        if (dequeued_ops[0]->status != RTE_CRYPTO_OP_STATUS_SUCCESS)
+                rte_exit(EXIT_FAILURE,
+                        "Some operations were not processed correctly");
+
+    } while (total_num_dequeued_ops < num_enqueued_ops);
+
+
+Asymmetric Crypto Device API
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The cryptodev Library API is described in the *DPDK API Reference* document.
+The cryptodev Library API is described in the
+`DPDK API Reference <http://dpdk.org/doc/api/>`_
diff --git a/doc/guides/prog_guide/env_abstraction_layer.rst b/doc/guides/prog_guide/env_abstraction_layer.rst
index a22640d2..d362c920 100644
--- a/doc/guides/prog_guide/env_abstraction_layer.rst
+++ b/doc/guides/prog_guide/env_abstraction_layer.rst
@@ -147,6 +147,10 @@ notified about memory allocations above specified threshold (and have a chance
 to deny them), allocation validator callbacks are also available via
 ``rte_mem_alloc_validator_callback_register()`` function.
 
+A default validator callback is provided by EAL, which can be enabled with a
+``--socket-limit`` command-line option, for a simple way to limit maximum amount
+of memory that can be used by DPDK application.
+
 .. note::
 
     In multiprocess scenario, all related processes (i.e. primary process, and
@@ -445,13 +449,17 @@ Known Issues
     Bypassing this constraint may cause the 2nd pthread to spin until the 1st one is scheduled again.
     Moreover, if the 1st pthread is preempted by a context that has an higher priority, it may even cause a dead lock.
 
-  This does not mean it cannot be used, simply, there is a need to narrow down the situation when it is used by multi-pthread on the same core.
+  This means, use cases involving preemptible pthreads should consider using rte_ring carefully.
+
+  1. It CAN be used for preemptible single-producer and single-consumer use case.
+
+  2. It CAN be used for non-preemptible multi-producer and preemptible single-consumer use case.
 
-  1. It CAN be used for any single-producer or single-consumer situation.
+  3. It CAN be used for preemptible single-producer and non-preemptible multi-consumer use case.
 
-  2. It MAY be used by multi-producer/consumer pthread whose scheduling policy are all SCHED_OTHER(cfs). User SHOULD be aware of the performance penalty before using it.
+  4. It MAY be used by preemptible multi-producer and/or preemptible multi-consumer pthreads whose scheduling policy are all SCHED_OTHER(cfs), SCHED_IDLE or SCHED_BATCH. User SHOULD be aware of the performance penalty before using it.
 
-  3. It MUST not be used by multi-producer/consumer pthreads, whose scheduling policies are SCHED_FIFO or SCHED_RR.
+  5. It MUST not be used by multi-producer/consumer pthreads, whose scheduling policies are SCHED_FIFO or SCHED_RR.
 
 + rte_timer
 
diff --git a/doc/guides/prog_guide/event_crypto_adapter.rst b/doc/guides/prog_guide/event_crypto_adapter.rst
index 5c1354de..9fe09c80 100644
--- a/doc/guides/prog_guide/event_crypto_adapter.rst
+++ b/doc/guides/prog_guide/event_crypto_adapter.rst
@@ -223,9 +223,9 @@ crypto security session or at an offset in the ``struct rte_crypto_op``.
 The ``rte_crypto_op::private_data_offset`` is used to locate the request/
 response in the ``rte_crypto_op``.
 
-For crypto session, ``rte_cryptodev_sym_session_set_private_data()`` API
+For crypto session, ``rte_cryptodev_sym_session_set_user_data()`` API
 will be used to set request/response data. The same data will be obtained
-by ``rte_cryptodev_sym_session_get_private_data()`` API.  The
+by ``rte_cryptodev_sym_session_get_user_data()`` API.  The
 RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA capability indicates
 whether HW or SW supports this feature.
 
@@ -257,7 +257,7 @@ the ``rte_crypto_op``.
                 m_data.request_info.cdev_id = cdev_id;
                 m_data.request_info.queue_pair_id = qp_id;
                 /* Call set API to store private data information */
-                rte_cryptodev_sym_session_set_private_data(
+                rte_cryptodev_sym_session_set_user_data(
                         op->sym->session,
                         &m_data,
                         sizeof(m_data));
diff --git a/doc/guides/prog_guide/event_ethernet_rx_adapter.rst b/doc/guides/prog_guide/event_ethernet_rx_adapter.rst
index 319e4f05..0166bb45 100644
--- a/doc/guides/prog_guide/event_ethernet_rx_adapter.rst
+++ b/doc/guides/prog_guide/event_ethernet_rx_adapter.rst
@@ -144,3 +144,44 @@ enqueued event counts are a sum of the counts from the eventdev PMD callbacks
 if the callback is supported, and the counts maintained by the service function,
 if one exists. The service function also maintains a count of cycles for which
 it was not able to enqueue to the event device.
+
+Interrupt Based Rx Queues
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The service core function is typically set up to poll ethernet Rx queues for
+packets. Certain queues may have low packet rates and it would be more
+efficient to enable the Rx queue interrupt and read packets after receiving
+the interrupt.
+
+The servicing_weight member of struct rte_event_eth_rx_adapter_queue_conf
+is applicable when the adapter uses a service core function. The application
+has to enable Rx queue interrupts when configuring the ethernet device
+using the ``rte_eth_dev_configure()`` function and then use a servicing_weight
+of zero when addding the Rx queue to the adapter.
+
+The adapter creates a thread blocked on the interrupt, on an interrupt this
+thread enqueues the port id and the queue id to a ring buffer. The adapter
+service function dequeues the port id and queue id from the ring buffer,
+invokes the ``rte_eth_rx_burst()`` to receive packets on the queue and
+converts the received packets to events in the same manner as packets
+received on a polled Rx queue. The interrupt thread is affinitized to the same
+CPUs as the lcores of the Rx adapter service function, if the Rx adapter
+service function has not been mapped to any lcores, the interrupt thread
+is mapped to the master lcore.
+
+Rx Callback for SW Rx Adapter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For SW based packet transfers, i.e., when the
+``RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT`` is not set in the adapter's
+capabilities flags for a particular ethernet device, the service function
+temporarily enqueues mbufs to an event buffer before batch enqueueing these
+to the event device. If the buffer fills up, the service function stops
+dequeueing packets from the ethernet device. The application may want to
+monitor the buffer fill level and instruct the service function to selectively
+enqueue packets to the event device. The application may also use some other
+criteria to decide which packets should enter the event device even when
+the event buffer fill level is low. The
+``rte_event_eth_rx_adapter_cb_register()`` function allow the application
+to register a callback that selects which packets to enqueue to the event
+device.
diff --git a/doc/guides/prog_guide/eventdev.rst b/doc/guides/prog_guide/eventdev.rst
index ce19997d..8fcae546 100644
--- a/doc/guides/prog_guide/eventdev.rst
+++ b/doc/guides/prog_guide/eventdev.rst
@@ -1,5 +1,6 @@
 ..  SPDX-License-Identifier: BSD-3-Clause
     Copyright(c) 2017 Intel Corporation.
+    Copyright(c) 2018 Arm Limited.
 
 Event Device Library
 ====================
@@ -129,8 +130,10 @@ API Walk-through
 
 This section will introduce the reader to the eventdev API, showing how to
 create and configure an eventdev and use it for a two-stage atomic pipeline
-with a single core for TX. The diagram below shows the final state of the
-application after this walk-through:
+with one core each for RX and TX. RX and TX cores are shown here for
+illustration, refer to Eventdev Adapter documentation for further details.
+The diagram below shows the final state of the application after this
+walk-through:
 
 .. _figure_eventdev-usage1:
 
@@ -196,23 +199,29 @@ calling the setup function. Repeat this step for each queue, starting from
                 .nb_atomic_flows = 1024,
                 .nb_atomic_order_sequences = 1024,
         };
+        struct rte_event_queue_conf single_link_conf = {
+                .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK,
+        };
         int dev_id = 0;
-        int queue_id = 0;
-        int err = rte_event_queue_setup(dev_id, queue_id, &atomic_conf);
+        int atomic_q_1 = 0;
+        int atomic_q_2 = 1;
+        int single_link_q = 2;
+        int err = rte_event_queue_setup(dev_id, atomic_q_1, &atomic_conf);
+        int err = rte_event_queue_setup(dev_id, atomic_q_2, &atomic_conf);
+        int err = rte_event_queue_setup(dev_id, single_link_q, &single_link_conf);
 
-The remainder of this walk-through assumes that the queues are configured as
-follows:
+As shown above, queue IDs are as follows:
 
  * id 0, atomic queue #1
  * id 1, atomic queue #2
  * id 2, single-link queue
 
+These queues are used for the remainder of this walk-through.
+
 Setting up Ports
 ~~~~~~~~~~~~~~~~
 
-Once queues are set up successfully, create the ports as required. Each port
-should be set up with its corresponding port_conf type, worker for worker cores,
-rx and tx for the RX and TX cores:
+Once queues are set up successfully, create the ports as required.
 
 .. code-block:: c
 
@@ -232,15 +241,24 @@ rx and tx for the RX and TX cores:
                 .new_event_threshold = 4096,
         };
         int dev_id = 0;
-        int port_id = 0;
-        int err = rte_event_port_setup(dev_id, port_id, &CORE_FUNCTION_conf);
+        int rx_port_id = 0;
+        int err = rte_event_port_setup(dev_id, rx_port_id, &rx_conf);
+
+        for(int worker_port_id = 1; worker_port_id <= 4; worker_port_id++) {
+	        int err = rte_event_port_setup(dev_id, worker_port_id, &worker_conf);
+        }
 
-It is now assumed that:
+        int tx_port_id = 5;
+	int err = rte_event_port_setup(dev_id, tx_port_id, &tx_conf);
+
+As shown above:
 
  * port 0: RX core
  * ports 1,2,3,4: Workers
  * port 5: TX core
 
+These ports are used for the remainder of this walk-through.
+
 Linking Queues and Ports
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -254,15 +272,14 @@ can be achieved like this:
 
 .. code-block:: c
 
-        uint8_t port_id = 0;
+        uint8_t rx_port_id = 0;
+        uint8_t tx_port_id = 5;
         uint8_t atomic_qs[] = {0, 1};
         uint8_t single_link_q = 2;
-        uint8_t tx_port_id = 5;
         uin8t_t priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
 
-        for(int i = 0; i < 4; i++) {
-                int worker_port = i + 1;
-                int links_made = rte_event_port_link(dev_id, worker_port, atomic_qs, NULL, 2);
+        for(int worker_port_id = 1; worker_port_id <= 4; worker_port_id++) {
+                int links_made = rte_event_port_link(dev_id, worker_port_id, atomic_qs, NULL, 2);
         }
         int links_made = rte_event_port_link(dev_id, tx_port_id, &single_link_q, &priority, 1);
 
@@ -295,14 +312,14 @@ The following code shows how those packets can be enqueued into the eventdev:
                 ev[i].flow_id = mbufs[i]->hash.rss;
                 ev[i].op = RTE_EVENT_OP_NEW;
                 ev[i].sched_type = RTE_SCHED_TYPE_ATOMIC;
-                ev[i].queue_id = 0;
+                ev[i].queue_id = atomic_q_1;
                 ev[i].event_type = RTE_EVENT_TYPE_ETHDEV;
                 ev[i].sub_event_type = 0;
                 ev[i].priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
                 ev[i].mbuf = mbufs[i];
         }
 
-        const int nb_tx = rte_event_enqueue_burst(dev_id, port_id, ev, nb_rx);
+        const int nb_tx = rte_event_enqueue_burst(dev_id, rx_port_id, ev, nb_rx);
         if (nb_tx != nb_rx) {
                 for(i = nb_tx; i < nb_rx; i++)
                         rte_pktmbuf_free(mbufs[i]);
@@ -334,7 +351,7 @@ the event to the next stage in the pipeline.
                 events[i].queue_id++;
         }
 
-        uint16_t nb_tx = rte_event_enqueue_burst(dev_id, port_id, events, nb_rx);
+        uint16_t nb_tx = rte_event_enqueue_burst(dev_id, worker_port_id, events, nb_rx);
 
 
 Egress of Events
diff --git a/doc/guides/prog_guide/generic_segmentation_offload_lib.rst b/doc/guides/prog_guide/generic_segmentation_offload_lib.rst
index 9959f0d2..0cfc1198 100644
--- a/doc/guides/prog_guide/generic_segmentation_offload_lib.rst
+++ b/doc/guides/prog_guide/generic_segmentation_offload_lib.rst
@@ -43,6 +43,7 @@ Limitations
 #. Currently, the GSO library supports the following IPv4 packet types:
 
  - TCP
+ - UDP
  - VxLAN
  - GRE
 
@@ -146,6 +147,15 @@ TCP/IPv4 GSO
 TCP/IPv4 GSO supports segmentation of suitably large TCP/IPv4 packets, which
 may also contain an optional VLAN tag.
 
+UDP/IPv4 GSO
+~~~~~~~~~~~~
+UDP/IPv4 GSO supports segmentation of suitably large UDP/IPv4 packets, which
+may also contain an optional VLAN tag. UDP GSO is the same as IP fragmentation.
+Specifically, UDP GSO treats the UDP header as a part of the payload and
+does not modify it during segmentation. Therefore, after UDP GSO, only the
+first output packet has the original UDP header, and others just have l2
+and l3 headers.
+
 VxLAN GSO
 ~~~~~~~~~
 VxLAN packets GSO supports segmentation of suitably large VxLAN packets,
diff --git a/doc/guides/prog_guide/hash_lib.rst b/doc/guides/prog_guide/hash_lib.rst
index 180c776e..76a1f323 100644
--- a/doc/guides/prog_guide/hash_lib.rst
+++ b/doc/guides/prog_guide/hash_lib.rst
@@ -19,6 +19,8 @@ The main configuration parameters for the hash are:
 
 *   Size of the key in bytes
 
+*   An extra flag used to describe additional settings, for example the multithreading mode of operation (as will be described later)
+
 The hash also allows the configuration of some low-level implementation related parameters such as:
 
 *   Hash function to translate the key into a bucket index
@@ -49,8 +51,7 @@ Apart from these method explained above, the API allows the user three more opti
 Also, the API contains a method to allow the user to look up entries in bursts, achieving higher performance
 than looking up individual entries, as the function prefetches next entries at the time it is operating
 with the first ones, which reduces significantly the impact of the necessary memory accesses.
-Notice that this method uses a pipeline of 8 entries (4 stages of 2 entries), so it is highly recommended
-to use at least 8 entries per burst.
+
 
 The actual data associated with each key can be either managed by the user using a separate table that
 mirrors the hash in terms of number of entries and position of each entry,
@@ -63,11 +64,33 @@ However, this table could also be used for more sophisticated features and provi
 Multi-process support
 ---------------------
 
-The hash library can be used in a multi-process environment, minding that only lookups are thread-safe.
+The hash library can be used in a multi-process environment.
 The only function that can only be used in single-process mode is rte_hash_set_cmp_func(), which sets up
 a custom compare function, which is assigned to a function pointer (therefore, it is not supported in
 multi-process mode).
 
+
+Multi-thread support
+---------------------
+
+The hash library supports multithreading, and the user specifies the needed mode of operation at the creation time of the hash table
+by appropriately setting the flag. In all modes of operation lookups are thread-safe meaning lookups can be called from multiple
+threads concurrently.
+
+For concurrent writes, and concurrent reads and writes the following flag values define the corresponding modes of operation:
+
+*  If the multi-writer flag (RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD) is set, multiple threads writing to the table is allowed.
+   Key add, delete, and table reset are protected from other writer threads. With only this flag set, readers are not protected from ongoing writes.
+
+*  If the read/write concurrency (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY) is set, multithread read/write operation is safe
+   (i.e., no need to stop the readers from accessing the hash table until writers finish their updates. Reads and writes can operate table concurrently).
+
+*  In addition to these two flag values, if the transactional memory flag (RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT) is also set,
+   hardware transactional memory will be used to guarantee the thread safety as long as it is supported by the hardware (for example the Intel® TSX support).
+
+If the platform supports Intel® TSX, it is advised to set the transactional memory flag, as this will speed up concurrent table operations.
+Otherwise concurrent operations will be slower because of the overhead associated with the software locking mechanisms.
+
 Implementation Details
 ----------------------
 
diff --git a/doc/guides/prog_guide/img/eventdev_usage.svg b/doc/guides/prog_guide/img/eventdev_usage.svg
index 7765649b..c19818b9 100644
--- a/doc/guides/prog_guide/img/eventdev_usage.svg
+++ b/doc/guides/prog_guide/img/eventdev_usage.svg
@@ -1,994 +1,549 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by Microsoft Visio, SVG Export eventdev_usage.svg Page-1 -->
+<!-- SPDX-License-Identifier: BSD-3-Clause -->
+<!-- Copyright(c) 2018 Arm -->
 
-<svg
-   xmlns:v="http://schemas.microsoft.com/visio/2003/SVGExtensions/"
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:xlink="http://www.w3.org/1999/xlink"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   width="683.12061"
-   height="184.672"
-   viewBox="0 0 546.49648 147.7376"
-   xml:space="preserve"
-   color-interpolation-filters="sRGB"
-   class="st9"
-   id="svg2"
-   version="1.1"
-   inkscape:version="0.48.4 r9939"
-   sodipodi:docname="eventdev_usage.svg"
-   style="font-size:12px;fill:none;stroke-linecap:square;stroke-miterlimit:3;overflow:visible"><metadata
-   id="metadata214"><rdf:RDF><cc:Work
-       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
-         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /></cc:Work></rdf:RDF></metadata><sodipodi:namedview
-   pagecolor="#ffffff"
-   bordercolor="#666666"
-   borderopacity="1"
-   objecttolerance="10"
-   gridtolerance="10"
-   guidetolerance="10"
-   inkscape:pageopacity="0"
-   inkscape:pageshadow="2"
-   inkscape:window-width="1920"
-   inkscape:window-height="1017"
-   id="namedview212"
-   showgrid="false"
-   fit-margin-top="2"
-   fit-margin-left="2"
-   fit-margin-right="2"
-   fit-margin-bottom="2"
-   inkscape:zoom="1.2339869"
-   inkscape:cx="501.15554"
-   inkscape:cy="164.17693"
-   inkscape:window-x="-8"
-   inkscape:window-y="406"
-   inkscape:window-maximized="1"
-   inkscape:current-layer="g17" />
-	<v:documentProperties
-   v:langID="1033"
-   v:viewMarkup="false">
-		<v:userDefs>
-			<v:ud
-   v:nameU="msvSubprocessMaster"
-   v:prompt=""
-   v:val="VT4(Rectangle)" />
-			<v:ud
-   v:nameU="msvNoAutoConnect"
-   v:val="VT0(1):26" />
-		</v:userDefs>
-	</v:documentProperties>
-
-	<style
-   type="text/css"
-   id="style4">
-
-		.st1 {visibility:visible}
-		.st2 {fill:#5b9bd5;fill-opacity:0.22;filter:url(#filter_2);stroke:#5b9bd5;stroke-opacity:0.22}
-		.st3 {fill:#5b9bd5;stroke:#c7c8c8;stroke-width:0.25}
-		.st4 {fill:#feffff;font-family:Calibri;font-size:0.833336em}
-		.st5 {font-size:1em}
-		.st6 {fill:#ffd965;stroke:#c7c8c8;stroke-width:0.25}
-		.st7 {marker-end:url(#mrkr4-33);stroke:#5b9bd5;stroke-linecap:round;stroke-linejoin:round;stroke-width:1}
-		.st8 {fill:#5b9bd5;fill-opacity:1;stroke:#5b9bd5;stroke-opacity:1;stroke-width:0.28409090909091}
-		.st9 {fill:none;fill-rule:evenodd;font-size:12px;overflow:visible;stroke-linecap:square;stroke-miterlimit:3}
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ev="http://www.w3.org/2001/xml-events"
+		xmlns:v="http://schemas.microsoft.com/visio/2003/SVGExtensions/" width="9.04167in" height="1.84602in"
+		viewBox="0 0 651 132.913" xml:space="preserve" color-interpolation-filters="sRGB" class="st12">
+	<v:documentProperties v:langID="1033" v:viewMarkup="false"/>
 
+	<style type="text/css">
+	<![CDATA[
+		.st1 {fill:#3c63ac;stroke:#30518f;stroke-width:0.75}
+		.st2 {fill:#feffff;font-family:Calibri;font-size:0.833336em}
+		.st3 {fill:none;stroke:#203864;stroke-width:0.25}
+		.st4 {stroke:#203864;stroke-linecap:round;stroke-linejoin:round;stroke-width:1}
+		.st5 {fill:#ffd965;stroke:#203864;stroke-width:0.25}
+		.st6 {fill:#000000;font-family:Calibri;font-size:1.16666em}
+		.st7 {font-size:1em}
+		.st8 {fill:none;stroke:none;stroke-width:0.25}
+		.st9 {fill:#000000;font-family:Calibri;font-size:0.833336em}
+		.st10 {fill:#000000;font-family:Calibri;font-size:1.00001em}
+		.st11 {font-size:1.16665em}
+		.st12 {fill:none;fill-rule:evenodd;font-size:12px;overflow:visible;stroke-linecap:square;stroke-miterlimit:3}
+	]]>
 	</style>
 
-	<defs
-   id="Markers">
-		<g
-   id="lend4">
-			<path
-   d="M 2,1 0,0 2,-1 2,1"
-   style="stroke:none"
-   id="path8"
-   inkscape:connector-curvature="0" />
-		</g>
-		<marker
-   id="mrkr4-33"
-   class="st8"
-   v:arrowType="4"
-   v:arrowSize="2"
-   v:setback="7.04"
-   refX="-7.04"
-   orient="auto"
-   markerUnits="strokeWidth"
-   overflow="visible"
-   style="fill:#5b9bd5;fill-opacity:1;stroke:#5b9bd5;stroke-width:0.28409091;stroke-opacity:1;overflow:visible">
-			<use
-   xlink:href="#lend4"
-   transform="scale(-3.52,-3.52)"
-   id="use11"
-   x="0"
-   y="0"
-   width="3"
-   height="3" />
-		</marker>
-	<filter
-   id="filter_2-7"
-   color-interpolation-filters="sRGB"><feGaussianBlur
-     stdDeviation="2"
-     id="feGaussianBlur15-1" /></filter><marker
-   id="mrkr4-33-2"
-   class="st8"
-   v:arrowType="4"
-   v:arrowSize="2"
-   v:setback="7.04"
-   refX="-7.04"
-   orient="auto"
-   markerUnits="strokeWidth"
-   overflow="visible"
-   style="fill:#5b9bd5;fill-opacity:1;stroke:#5b9bd5;stroke-width:0.28409091;stroke-opacity:1;overflow:visible"><use
-     xlink:href="#lend4"
-     transform="scale(-3.52,-3.52)"
-     id="use11-3"
-     x="0"
-     y="0"
-     width="3"
-     height="3" /></marker><marker
-   id="mrkr4-33-6"
-   class="st8"
-   v:arrowType="4"
-   v:arrowSize="2"
-   v:setback="7.04"
-   refX="-7.04"
-   orient="auto"
-   markerUnits="strokeWidth"
-   overflow="visible"
-   style="fill:#5b9bd5;fill-opacity:1;stroke:#5b9bd5;stroke-width:0.28409091;stroke-opacity:1;overflow:visible"><use
-     xlink:href="#lend4"
-     transform="scale(-3.52,-3.52)"
-     id="use11-8"
-     x="0"
-     y="0"
-     width="3"
-     height="3" /></marker></defs>
-	<defs
-   id="Filters">
-		<filter
-   id="filter_2"
-   color-interpolation-filters="sRGB">
-			<feGaussianBlur
-   stdDeviation="2"
-   id="feGaussianBlur15" />
-		</filter>
-	</defs>
-	<g
-   v:mID="0"
-   v:index="1"
-   v:groupContext="foregroundPage"
-   id="g17"
-   transform="translate(-47.323579,-90.784072)">
-		<v:userDefs>
-			<v:ud
-   v:nameU="msvThemeOrder"
-   v:val="VT0(0):26" />
-		</v:userDefs>
-		<title
-   id="title19">Page-1</title>
-		<v:pageProperties
-   v:drawingScale="1"
-   v:pageScale="1"
-   v:drawingUnits="0"
-   v:shadowOffsetX="9"
-   v:shadowOffsetY="-9" />
-		<v:layer
-   v:name="Connector"
-   v:index="0" />
-		<g
-   id="shape1-1"
-   v:mID="1"
-   v:groupContext="shape"
-   transform="matrix(0.77644652,0,0,0.77644652,128.62352,-288.18843)">
-			<title
-   id="title22">Square</title>
-			<desc
-   id="desc24">Atomic Queue #1</desc>
-			<v:userDefs>
-				<v:ud
-   v:nameU="visVersion"
-   v:val="VT0(15):26" />
-			</v:userDefs>
-			<v:textBlock
-   v:margins="rect(4,4,4,4)" />
-			<v:textRect
-   cx="30.75"
-   cy="581.25"
-   width="61.5"
-   height="61.5" />
-			<g
-   id="shadow1-2"
-   v:groupContext="shadow"
-   v:shadowOffsetX="0.345598"
-   v:shadowOffsetY="-1.97279"
-   v:shadowType="1"
-   transform="translate(0.345598,1.97279)"
-   class="st1"
-   style="visibility:visible">
-				<rect
-   x="0"
-   y="550.5"
-   width="61.5"
-   height="61.5"
-   class="st2"
-   id="rect27"
-   style="fill:#5b9bd5;fill-opacity:0.22000002;stroke:#5b9bd5;stroke-opacity:0.22000002;filter:url(#filter_2)" />
-			</g>
-			<rect
-   x="0"
-   y="550.5"
-   width="61.5"
-   height="61.5"
-   class="st3"
-   id="rect29"
-   style="fill:#5b9bd5;stroke:#c7c8c8;stroke-width:0.25" />
-
-		</g>
-		<g
-   id="shape3-8"
-   v:mID="3"
-   v:groupContext="shape"
-   transform="matrix(0.77644652,0,0,0.77644652,297.37175,-288.18843)">
-			<title
-   id="title36">Square.3</title>
-			<desc
-   id="desc38">Atomic Queue #2</desc>
-			<v:userDefs>
-				<v:ud
-   v:nameU="visVersion"
-   v:val="VT0(15):26" />
-			</v:userDefs>
-			<v:textBlock
-   v:margins="rect(4,4,4,4)" />
-			<v:textRect
-   cx="30.75"
-   cy="581.25"
-   width="61.5"
-   height="61.5" />
-			<g
-   id="shadow3-9"
-   v:groupContext="shadow"
-   v:shadowOffsetX="0.345598"
-   v:shadowOffsetY="-1.97279"
-   v:shadowType="1"
-   transform="translate(0.345598,1.97279)"
-   class="st1"
-   style="visibility:visible">
-				<rect
-   x="0"
-   y="550.5"
-   width="61.5"
-   height="61.5"
-   class="st2"
-   id="rect41"
-   style="fill:#5b9bd5;fill-opacity:0.22000002;stroke:#5b9bd5;stroke-opacity:0.22000002;filter:url(#filter_2)" />
-			</g>
-			<rect
-   x="0"
-   y="550.5"
-   width="61.5"
-   height="61.5"
-   class="st3"
-   id="rect43"
-   style="fill:#5b9bd5;stroke:#c7c8c8;stroke-width:0.25" />
-
-		</g>
-		<g
-   id="shape4-15"
-   v:mID="4"
-   v:groupContext="shape"
-   transform="matrix(0.77644652,0,0,0.77644652,466.1192,-288.18843)">
-			<title
-   id="title50">Square.4</title>
-			<desc
-   id="desc52">Single Link Queue # 1</desc>
-			<v:userDefs>
-				<v:ud
-   v:nameU="visVersion"
-   v:val="VT0(15):26" />
-			</v:userDefs>
-			<v:textBlock
-   v:margins="rect(4,4,4,4)" />
-			<v:textRect
-   cx="30.75"
-   cy="581.25"
-   width="61.5"
-   height="61.5" />
-			<g
-   id="shadow4-16"
-   v:groupContext="shadow"
-   v:shadowOffsetX="0.345598"
-   v:shadowOffsetY="-1.97279"
-   v:shadowType="1"
-   transform="translate(0.345598,1.97279)"
-   class="st1"
-   style="visibility:visible">
-				<rect
-   x="0"
-   y="550.5"
-   width="61.5"
-   height="61.5"
-   class="st2"
-   id="rect55"
-   style="fill:#5b9bd5;fill-opacity:0.22000002;stroke:#5b9bd5;stroke-opacity:0.22000002;filter:url(#filter_2)" />
-			</g>
-			<rect
-   x="0"
-   y="550.5"
-   width="61.5"
-   height="61.5"
-   class="st3"
-   id="rect57"
-   style="fill:#5b9bd5;stroke:#c7c8c8;stroke-width:0.25" />
-
-		</g>
-		<g
-   id="shape5-22"
-   v:mID="5"
-   v:groupContext="shape"
-   transform="matrix(0.77644652,0,0,0.77644652,52.208527,-296.14701)">
-			<title
-   id="title64">Circle</title>
-			<desc
-   id="desc66">RX</desc>
-			<v:userDefs>
-				<v:ud
-   v:nameU="visVersion"
-   v:val="VT0(15):26" />
-			</v:userDefs>
-			<v:textBlock
-   v:margins="rect(4,4,4,4)" />
-			<v:textRect
-   cx="20.5"
-   cy="591.5"
-   width="35.88"
-   height="30.75" />
-			<g
-   id="shadow5-23"
-   v:groupContext="shadow"
-   v:shadowOffsetX="0.345598"
-   v:shadowOffsetY="-1.97279"
-   v:shadowType="1"
-   transform="translate(0.345598,1.97279)"
-   class="st1"
-   style="visibility:visible">
-				<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st2"
-   id="path69"
-   inkscape:connector-curvature="0"
-   style="fill:#5b9bd5;fill-opacity:0.22000002;stroke:#5b9bd5;stroke-opacity:0.22000002;filter:url(#filter_2)" />
-			</g>
-			<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st6"
-   id="path71"
-   inkscape:connector-curvature="0"
-   style="fill:#ffd965;stroke:#c7c8c8;stroke-width:0.25" />
-			<text
-   x="15.19"
-   y="594.5"
-   class="st4"
-   v:langID="1033"
-   id="text73"
-   style="fill:#feffff;font-family:Calibri"><v:paragraph
-   v:horizAlign="1" /><v:tabList />RX</text>
-
-		</g>
-		<g
-   id="shape6-28"
-   v:mID="6"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,84.042834,-305.07614)">
-			<title
-   id="title76">Dynamic connector</title>
-			<path
-   d="m 0,603 50.38,0"
-   class="st7"
-   id="path78"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape7-34"
-   v:mID="7"
-   v:groupContext="shape"
-   transform="matrix(0.77644652,0,0,0.77644652,220.95621,-296.14701)">
-			<title
-   id="title81">Circle.7</title>
-			<desc
-   id="desc83">W ..</desc>
-			<v:userDefs>
-				<v:ud
-   v:nameU="visVersion"
-   v:val="VT0(15):26" />
-			</v:userDefs>
-			<v:textBlock
-   v:margins="rect(4,4,4,4)" />
-			<v:textRect
-   cx="20.5"
-   cy="591.5"
-   width="35.88"
-   height="30.75" />
-			<g
-   id="shadow7-35"
-   v:groupContext="shadow"
-   v:shadowOffsetX="0.345598"
-   v:shadowOffsetY="-1.97279"
-   v:shadowType="1"
-   transform="translate(0.345598,1.97279)"
-   class="st1"
-   style="visibility:visible">
-				<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st2"
-   id="path86"
-   inkscape:connector-curvature="0"
-   style="fill:#5b9bd5;fill-opacity:0.22000002;stroke:#5b9bd5;stroke-opacity:0.22000002;filter:url(#filter_2)" />
-			</g>
-			<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st6"
-   id="path88"
-   inkscape:connector-curvature="0"
-   style="fill:#ffd965;stroke:#c7c8c8;stroke-width:0.25" />
-			<text
-   x="12.4"
-   y="594.5"
-   class="st4"
-   v:langID="1033"
-   id="text90"
-   style="fill:#feffff;font-family:Calibri"><v:paragraph
-   v:horizAlign="1" /><v:tabList />W ..</text>
-
-		</g>
-		<g
-   id="shape9-40"
-   v:mID="9"
-   v:groupContext="shape"
-   transform="matrix(0.77644652,0,0,0.77644652,220.95621,-243.34865)">
-			<title
-   id="title93">Circle.9</title>
-			<desc
-   id="desc95">W N</desc>
-			<v:userDefs>
-				<v:ud
-   v:nameU="visVersion"
-   v:val="VT0(15):26" />
-			</v:userDefs>
-			<v:textBlock
-   v:margins="rect(4,4,4,4)" />
-			<v:textRect
-   cx="20.5"
-   cy="591.5"
-   width="35.88"
-   height="30.75" />
-			<g
-   id="shadow9-41"
-   v:groupContext="shadow"
-   v:shadowOffsetX="0.345598"
-   v:shadowOffsetY="-1.97279"
-   v:shadowType="1"
-   transform="translate(0.345598,1.97279)"
-   class="st1"
-   style="visibility:visible">
-				<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st2"
-   id="path98"
-   inkscape:connector-curvature="0"
-   style="fill:#5b9bd5;fill-opacity:0.22000002;stroke:#5b9bd5;stroke-opacity:0.22000002;filter:url(#filter_2)" />
+	<g v:mID="0" v:index="1" v:groupContext="foregroundPage">
+		<title>Page-1</title>
+		<v:pageProperties v:drawingScale="1" v:pageScale="1" v:drawingUnits="0" v:shadowOffsetX="9" v:shadowOffsetY="-9"/>
+		<v:layer v:name="Connector" v:index="0"/>
+		<g id="group1068-1" transform="translate(0.75,-0.25)" v:mID="1068" v:groupContext="group">
+			<title>Sheet.1068</title>
+			<g id="shape3-2" v:mID="3" v:groupContext="shape" transform="translate(63,184.827) rotate(180)">
+				<title>Simple Arrow</title>
+				<desc>In Intf</desc>
+				<v:userDefs>
+					<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+					<v:ud v:nameU="ArrowType" v:prompt="" v:val="VT0(2):26"/>
+				</v:userDefs>
+				<v:textBlock v:margins="rect(4,4,4,4)"/>
+				<v:textRect cx="31.5" cy="132.913" width="63.01" height="0" transform="rotate(180)"/>
+				<path d="M0 132.91 L12 120.92 L12 126.92 L63 126.92 L63 132.91 L63 138.91 L12 138.91 L12 144.91 L0 132.91 Z"
+						class="st1"/>
+				<text x="-43.6" y="-129.91" transform="rotate(180)" class="st2" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>In Intf</text>			</g>
+			<g id="group1010-5" transform="translate(130.5,-73.3167)" v:mID="1010" v:groupContext="group">
+				<title>Sheet.1010</title>
+				<g id="group1000-6" transform="translate(-2.19824E-14,-0.0534178)" v:mID="1000" v:groupContext="group">
+					<title>Sheet.1000</title>
+					<g id="shape1001-7" v:mID="1001" v:groupContext="shape" transform="translate(0,-4.86)">
+						<title>Rectangle.38</title>
+						<v:userDefs>
+							<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+						</v:userDefs>
+						<rect x="0" y="123.913" width="22.3966" height="8.99999" class="st3"/>
+					</g>
+					<g id="shape1002-9" v:mID="1002" v:groupContext="shape" v:layerMember="0" transform="translate(2.19832,-18.18)">
+						<title>Dynamic connector.162</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1003-12" v:mID="1003" v:groupContext="shape" v:layerMember="0"
+							transform="translate(7.79747,-18.18)">
+						<title>Dynamic connector.163</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1004-15" v:mID="1004" v:groupContext="shape" v:layerMember="0" transform="translate(-3.40084,-18)">
+						<title>Dynamic connector.164</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+				</g>
+				<g id="group1005-18" transform="translate(22.6034,0)" v:mID="1005" v:groupContext="group">
+					<title>Sheet.1005</title>
+					<g id="shape1006-19" v:mID="1006" v:groupContext="shape" transform="translate(0,-4.86)">
+						<title>Rectangle.38</title>
+						<v:userDefs>
+							<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+						</v:userDefs>
+						<rect x="0" y="123.913" width="22.3966" height="8.99999" class="st3"/>
+					</g>
+					<g id="shape1007-21" v:mID="1007" v:groupContext="shape" v:layerMember="0"
+							transform="translate(2.19832,-18.18)">
+						<title>Dynamic connector.162</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1008-24" v:mID="1008" v:groupContext="shape" v:layerMember="0"
+							transform="translate(7.79747,-18.18)">
+						<title>Dynamic connector.163</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1009-27" v:mID="1009" v:groupContext="shape" v:layerMember="0" transform="translate(-3.40084,-18)">
+						<title>Dynamic connector.164</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+				</g>
 			</g>
-			<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st6"
-   id="path100"
-   inkscape:connector-curvature="0"
-   style="fill:#ffd965;stroke:#c7c8c8;stroke-width:0.25" />
-			<text
-   x="11.69"
-   y="594.5"
-   class="st4"
-   v:langID="1033"
-   id="text102"
-   style="fill:#feffff;font-family:Calibri"><v:paragraph
-   v:horizAlign="1" /><v:tabList />W N</text>
-
-		</g>
-		<g
-   id="shape10-46"
-   v:mID="10"
-   v:groupContext="shape"
-   transform="matrix(0.77644652,0,0,0.77644652,220.95621,-348.94537)">
-			<title
-   id="title105">Circle.10</title>
-			<desc
-   id="desc107">W 1</desc>
-			<v:userDefs>
-				<v:ud
-   v:nameU="visVersion"
-   v:val="VT0(15):26" />
-			</v:userDefs>
-			<v:textBlock
-   v:margins="rect(4,4,4,4)" />
-			<v:textRect
-   cx="20.5"
-   cy="591.5"
-   width="35.88"
-   height="30.75" />
-			<g
-   id="shadow10-47"
-   v:groupContext="shadow"
-   v:shadowOffsetX="0.345598"
-   v:shadowOffsetY="-1.97279"
-   v:shadowType="1"
-   transform="translate(0.345598,1.97279)"
-   class="st1"
-   style="visibility:visible">
-				<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st2"
-   id="path110"
-   inkscape:connector-curvature="0"
-   style="fill:#5b9bd5;fill-opacity:0.22000002;stroke:#5b9bd5;stroke-opacity:0.22000002;filter:url(#filter_2)" />
+			<g id="group1016-30" transform="translate(301.5,-73.3167)" v:mID="1016" v:groupContext="group">
+				<title>Sheet.1016</title>
+				<g id="group1017-31" transform="translate(-2.19824E-14,-0.0534178)" v:mID="1017" v:groupContext="group">
+					<title>Sheet.1017</title>
+					<g id="shape1018-32" v:mID="1018" v:groupContext="shape" transform="translate(0,-4.86)">
+						<title>Rectangle.38</title>
+						<v:userDefs>
+							<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+						</v:userDefs>
+						<rect x="0" y="123.913" width="22.3966" height="8.99999" class="st3"/>
+					</g>
+					<g id="shape1019-34" v:mID="1019" v:groupContext="shape" v:layerMember="0"
+							transform="translate(2.19832,-18.18)">
+						<title>Dynamic connector.162</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1020-37" v:mID="1020" v:groupContext="shape" v:layerMember="0"
+							transform="translate(7.79747,-18.18)">
+						<title>Dynamic connector.163</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1021-40" v:mID="1021" v:groupContext="shape" v:layerMember="0" transform="translate(-3.40084,-18)">
+						<title>Dynamic connector.164</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+				</g>
+				<g id="group1022-43" transform="translate(22.6034,0)" v:mID="1022" v:groupContext="group">
+					<title>Sheet.1022</title>
+					<g id="shape1023-44" v:mID="1023" v:groupContext="shape" transform="translate(0,-4.86)">
+						<title>Rectangle.38</title>
+						<v:userDefs>
+							<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+						</v:userDefs>
+						<rect x="0" y="123.913" width="22.3966" height="8.99999" class="st3"/>
+					</g>
+					<g id="shape1024-46" v:mID="1024" v:groupContext="shape" v:layerMember="0"
+							transform="translate(2.19832,-18.18)">
+						<title>Dynamic connector.162</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1025-49" v:mID="1025" v:groupContext="shape" v:layerMember="0"
+							transform="translate(7.79747,-18.18)">
+						<title>Dynamic connector.163</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1026-52" v:mID="1026" v:groupContext="shape" v:layerMember="0" transform="translate(-3.40084,-18)">
+						<title>Dynamic connector.164</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+				</g>
 			</g>
-			<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st6"
-   id="path112"
-   inkscape:connector-curvature="0"
-   style="fill:#ffd965;stroke:#c7c8c8;stroke-width:0.25" />
-			<text
-   x="12.39"
-   y="594.5"
-   class="st4"
-   v:langID="1033"
-   id="text114"
-   style="fill:#feffff;font-family:Calibri"><v:paragraph
-   v:horizAlign="1" /><v:tabList />W 1</text>
-
-		</g>
-		<g
-   id="shape11-52"
-   v:mID="11"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,195.91581,-312.06416)">
-			<title
-   id="title117">Dynamic connector.11</title>
-			<path
-   d="m 0,612 0,-68 25.21,0"
-   class="st7"
-   id="path119"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape12-57"
-   v:mID="12"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,176.37498,-305.07614)">
-			<title
-   id="title122">Dynamic connector.12</title>
-			<path
-   d="m 0,603 50.38,0"
-   class="st7"
-   id="path124"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape13-62"
-   v:mID="13"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,176.37498,-312.06416)">
-			<title
-   id="title127">Dynamic connector.13</title>
-			<path
-   d="m 0,612 25.17,0 0,68 25.21,0"
-   class="st7"
-   id="path129"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape14-67"
-   v:mID="14"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,252.79052,-259.2658)">
-			<title
-   id="title132">Dynamic connector.14</title>
-			<path
-   d="m 0,612 26.88,0 0,-68 23.5,0"
-   class="st7"
-   id="path134"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape15-72"
-   v:mID="15"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,252.79052,-305.07614)">
-			<title
-   id="title137">Dynamic connector.15</title>
-			<path
-   d="m 0,603 50.38,0"
-   class="st7"
-   id="path139"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape19-77"
-   v:mID="19"
-   v:groupContext="shape"
-   transform="matrix(0.77644652,0,0,0.77644652,389.70366,-296.14701)">
-			<title
-   id="title142">Circle.19</title>
-			<desc
-   id="desc144">W ..</desc>
-			<v:userDefs>
-				<v:ud
-   v:nameU="visVersion"
-   v:val="VT0(15):26" />
-			</v:userDefs>
-			<v:textBlock
-   v:margins="rect(4,4,4,4)" />
-			<v:textRect
-   cx="20.5"
-   cy="591.5"
-   width="35.88"
-   height="30.75" />
-			<g
-   id="shadow19-78"
-   v:groupContext="shadow"
-   v:shadowOffsetX="0.345598"
-   v:shadowOffsetY="-1.97279"
-   v:shadowType="1"
-   transform="translate(0.345598,1.97279)"
-   class="st1"
-   style="visibility:visible">
-				<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st2"
-   id="path147"
-   inkscape:connector-curvature="0"
-   style="fill:#5b9bd5;fill-opacity:0.22000002;stroke:#5b9bd5;stroke-opacity:0.22000002;filter:url(#filter_2)" />
+			<g id="group1032-55" transform="translate(468,-73.2)" v:mID="1032" v:groupContext="group">
+				<title>Sheet.1032</title>
+				<g id="group1033-56" transform="translate(-2.19824E-14,-0.0534178)" v:mID="1033" v:groupContext="group">
+					<title>Sheet.1033</title>
+					<g id="shape1034-57" v:mID="1034" v:groupContext="shape" transform="translate(0,-4.86)">
+						<title>Rectangle.38</title>
+						<v:userDefs>
+							<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+						</v:userDefs>
+						<rect x="0" y="123.913" width="22.3966" height="8.99999" class="st3"/>
+					</g>
+					<g id="shape1035-59" v:mID="1035" v:groupContext="shape" v:layerMember="0"
+							transform="translate(2.19832,-18.18)">
+						<title>Dynamic connector.162</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1036-62" v:mID="1036" v:groupContext="shape" v:layerMember="0"
+							transform="translate(7.79747,-18.18)">
+						<title>Dynamic connector.163</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1037-65" v:mID="1037" v:groupContext="shape" v:layerMember="0" transform="translate(-3.40084,-18)">
+						<title>Dynamic connector.164</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+				</g>
+				<g id="group1038-68" transform="translate(22.6034,0)" v:mID="1038" v:groupContext="group">
+					<title>Sheet.1038</title>
+					<g id="shape1039-69" v:mID="1039" v:groupContext="shape" transform="translate(0,-4.86)">
+						<title>Rectangle.38</title>
+						<v:userDefs>
+							<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+						</v:userDefs>
+						<rect x="0" y="123.913" width="22.3966" height="8.99999" class="st3"/>
+					</g>
+					<g id="shape1040-71" v:mID="1040" v:groupContext="shape" v:layerMember="0"
+							transform="translate(2.19832,-18.18)">
+						<title>Dynamic connector.162</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1041-74" v:mID="1041" v:groupContext="shape" v:layerMember="0"
+							transform="translate(7.79747,-18.18)">
+						<title>Dynamic connector.163</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+					<g id="shape1042-77" v:mID="1042" v:groupContext="shape" v:layerMember="0" transform="translate(-3.40084,-18)">
+						<title>Dynamic connector.164</title>
+						<path d="M9 137.41 L9 146.41" class="st4"/>
+					</g>
+				</g>
 			</g>
-			<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st6"
-   id="path149"
-   inkscape:connector-curvature="0"
-   style="fill:#ffd965;stroke:#c7c8c8;stroke-width:0.25" />
-			<text
-   x="12.4"
-   y="594.5"
-   class="st4"
-   v:langID="1033"
-   id="text151"
-   style="fill:#feffff;font-family:Calibri"><v:paragraph
-   v:horizAlign="1" /><v:tabList />W ..</text>
-
-		</g>
-		<g
-   id="shape20-83"
-   v:mID="20"
-   v:groupContext="shape"
-   transform="matrix(0.77644652,0,0,0.77644652,389.70366,-243.34865)">
-			<title
-   id="title154">Circle.20</title>
-			<desc
-   id="desc156">W N</desc>
-			<v:userDefs>
-				<v:ud
-   v:nameU="visVersion"
-   v:val="VT0(15):26" />
-			</v:userDefs>
-			<v:textBlock
-   v:margins="rect(4,4,4,4)" />
-			<v:textRect
-   cx="20.5"
-   cy="591.5"
-   width="35.88"
-   height="30.75" />
-			<g
-   id="shadow20-84"
-   v:groupContext="shadow"
-   v:shadowOffsetX="0.345598"
-   v:shadowOffsetY="-1.97279"
-   v:shadowType="1"
-   transform="translate(0.345598,1.97279)"
-   class="st1"
-   style="visibility:visible">
-				<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st2"
-   id="path159"
-   inkscape:connector-curvature="0"
-   style="fill:#5b9bd5;fill-opacity:0.22000002;stroke:#5b9bd5;stroke-opacity:0.22000002;filter:url(#filter_2)" />
+			<g id="shape1044-80" v:mID="1044" v:groupContext="shape" transform="translate(651.291,179.381) rotate(179.228)">
+				<title>Simple Arrow.1044</title>
+				<desc>Out Intf</desc>
+				<v:userDefs>
+					<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+					<v:ud v:nameU="ArrowType" v:prompt="" v:val="VT0(2):26"/>
+				</v:userDefs>
+				<v:textBlock v:margins="rect(4,4,4,4)"/>
+				<v:textRect cx="30.3028" cy="132.913" width="60.61" height="0" transform="rotate(180)"/>
+				<path d="M0 132.91 L12 120.92 L12 126.92 L60.61 126.92 L60.61 132.91 L60.61 138.91 L12 138.91 L12 144.91 L0 132.91
+							 Z" class="st1"/>
+				<text x="-46.13" y="-129.91" transform="rotate(180)" class="st2" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Out Intf</text>			</g>
+			<g id="shape1045-83" v:mID="1045" v:groupContext="shape" transform="translate(67.8,-50.9334)">
+				<title>Rounded Rectangle.1045</title>
+				<desc>RX Core</desc>
+				<v:userDefs>
+					<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+					<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
+					<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+					<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
+					<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(1):5"/>
+					<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(1):5"/>
+					<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
+					<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+					<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+					<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+					<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+					<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
+					<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.0703125):1"/>
+					<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.0703125):1"/>
+					<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.0703125):1"/>
+					<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.0703125):1"/>
+				</v:userDefs>
+				<v:textBlock v:margins="rect(4,4,4,4)"/>
+				<v:textRect cx="25.3125" cy="101.413" width="50.63" height="63"/>
+				<path d="M5.06 132.91 L45.56 132.91 A5.06242 5.06242 -180 0 0 50.62 127.85 L50.62 74.98 A5.06242 5.06242 -180 0 0
+							 45.56 69.91 L5.06 69.91 A5.06242 5.06242 -180 0 0 0 74.98 L0 127.85 A5.06242 5.06242 -180 0 0 5.06 132.91
+							 Z" class="st5"/>
+				<text x="17.88" y="88.81" class="st6" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>RX<v:newlineChar/><v:newlineChar/><tspan
+							x="11.97" dy="2.4em" class="st7">Core</tspan></text>			</g>
+			<g id="shape1056-87" v:mID="1056" v:groupContext="shape" transform="translate(532.5,-54)">
+				<title>Rounded Rectangle.1056</title>
+				<desc>TX Core</desc>
+				<v:userDefs>
+					<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+					<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
+					<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+					<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
+					<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(1):5"/>
+					<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(1):5"/>
+					<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
+					<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+					<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+					<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+					<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+					<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
+					<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.0703125):1"/>
+					<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.0703125):1"/>
+					<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.0703125):1"/>
+					<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.0703125):1"/>
+				</v:userDefs>
+				<v:textBlock v:margins="rect(4,4,4,4)"/>
+				<v:textRect cx="25.3125" cy="101.413" width="50.63" height="63"/>
+				<path d="M5.06 132.91 L45.56 132.91 A5.06242 5.06242 -180 0 0 50.62 127.85 L50.62 74.98 A5.06242 5.06242 -180 0 0
+							 45.56 69.91 L5.06 69.91 A5.06242 5.06242 -180 0 0 0 74.98 L0 127.85 A5.06242 5.06242 -180 0 0 5.06 132.91
+							 Z" class="st5"/>
+				<text x="18.27" y="88.81" class="st6" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>TX<v:newlineChar/><v:newlineChar/><tspan
+							x="11.97" dy="2.4em" class="st7">Core</tspan></text>			</g>
+			<g id="shape1057-91" v:mID="1057" v:groupContext="shape" transform="translate(123.188,-59.0334)">
+				<title>Rectangle.1057</title>
+				<desc>Atomic Q 1</desc>
+				<v:userDefs>
+					<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+				</v:userDefs>
+				<v:textBlock v:margins="rect(4,4,4,4)"/>
+				<v:textRect cx="29.8125" cy="123.797" width="59.63" height="18.2334"/>
+				<rect x="0" y="114.68" width="59.625" height="18.2334" class="st8"/>
+				<text x="7.19" y="126.8" class="st9" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Atomic Q 1</text>			</g>
+			<g id="shape1058-94" v:mID="1058" v:groupContext="shape" transform="translate(295.5,-59.4)">
+				<title>Rectangle.1058</title>
+				<desc>Atomic Q 2</desc>
+				<v:userDefs>
+					<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+				</v:userDefs>
+				<v:textBlock v:margins="rect(4,4,4,4)"/>
+				<v:textRect cx="29.8125" cy="123.797" width="59.63" height="18.2334"/>
+				<rect x="0" y="114.68" width="59.625" height="18.2334" class="st8"/>
+				<text x="7.19" y="126.8" class="st9" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Atomic Q 2</text>			</g>
+			<g id="shape1059-97" v:mID="1059" v:groupContext="shape" transform="translate(460.687,-58.3167)">
+				<title>Rectangle.1059</title>
+				<desc>Single Link</desc>
+				<v:userDefs>
+					<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+				</v:userDefs>
+				<v:textBlock v:margins="rect(4,4,4,4)"/>
+				<v:textRect cx="29.8125" cy="123.797" width="59.63" height="18.2334"/>
+				<rect x="0" y="114.68" width="59.625" height="18.2334" class="st8"/>
+				<text x="8.47" y="126.8" class="st9" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Single Link</text>			</g>
+			<g id="shape1060-100" v:mID="1060" v:groupContext="shape" transform="translate(198,-1.2)">
+				<title>Rectangle.1060</title>
+				<desc>Stage 1</desc>
+				<v:userDefs>
+					<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+				</v:userDefs>
+				<v:textBlock v:margins="rect(4,4,4,4)"/>
+				<v:textRect cx="29.8125" cy="123.797" width="59.63" height="18.2334"/>
+				<rect x="0" y="114.68" width="59.625" height="18.2334" class="st8"/>
+				<text x="14.94" y="126.8" class="st9" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Stage 1</text>			</g>
+			<g id="shape1061-103" v:mID="1061" v:groupContext="shape" transform="translate(366.188,0)">
+				<title>Rectangle.1061</title>
+				<desc>Stage 2</desc>
+				<v:userDefs>
+					<v:ud v:nameU="visVersion" v:val="VT0(15):26"/>
+				</v:userDefs>
+				<v:textBlock v:margins="rect(4,4,4,4)"/>
+				<v:textRect cx="29.8125" cy="123.797" width="59.63" height="18.2334"/>
+				<rect x="0" y="114.68" width="59.625" height="18.2334" class="st8"/>
+				<text x="14.94" y="126.8" class="st9" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Stage 2</text>			</g>
+			<g id="group1062-106" transform="translate(199.2,-18.7134)" v:mID="1062" v:groupContext="group">
+				<title>Sheet.1062</title>
+				<g id="shape1052-107" v:mID="1052" v:groupContext="shape" transform="translate(18.66,-50.7)">
+					<title>Rounded Rectangle.1049</title>
+					<desc>Worker4 Core</desc>
+					<v:userDefs>
+						<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.097416666666665):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.097416666666665):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.097416666666665):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.097416666666665):1"/>
+					</v:userDefs>
+					<v:textBlock v:margins="rect(4,4,4,4)" v:verticalAlign="0"/>
+					<v:textRect cx="35.07" cy="101.413" width="70.14" height="63"/>
+					<path d="M7.01 132.91 L63.13 132.91 A7.01389 7.01389 -180 0 0 70.14 125.9 L70.14 76.93 A7.01389 7.01389 -180
+								 0 0 63.13 69.91 L7.01 69.91 A7.01389 7.01389 -180 0 0 0 76.93 L0 125.9 A7.01389 7.01389 -180 0 0
+								 7.01 132.91 Z" class="st5"/>
+					<text x="13.63" y="84.71" class="st10" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Worker4<v:newlineChar/><v:newlineChar/><tspan
+								x="21.72" dy="2.357em" class="st11">Core</tspan></text>				</g>
+				<g id="shape1053-111" v:mID="1053" v:groupContext="shape" transform="translate(12.9,-33.6)">
+					<title>Rounded Rectangle.1048</title>
+					<desc>Worker3 Core</desc>
+					<v:userDefs>
+						<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.099166666666667):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.099166666666667):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.099166666666667):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.099166666666667):1"/>
+					</v:userDefs>
+					<v:textBlock v:margins="rect(4,4,4,4)" v:verticalAlign="0"/>
+					<v:textRect cx="35.7" cy="101.413" width="71.4" height="63"/>
+					<path d="M7.14 132.91 L64.26 132.91 A7.13988 7.13988 -180 0 0 71.4 125.77 L71.4 77.05 A7.13988 7.13988 -180 0
+								 0 64.26 69.91 L7.14 69.91 A7.13988 7.13988 -180 0 0 0 77.05 L0 125.77 A7.13988 7.13988 -180 0 0
+								 7.14 132.91 Z" class="st5"/>
+					<text x="14.26" y="84.71" class="st10" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Worker3<v:newlineChar/><v:newlineChar/><tspan
+								x="22.35" dy="2.357em" class="st11">Core</tspan></text>				</g>
+				<g id="shape1054-115" v:mID="1054" v:groupContext="shape" transform="translate(5.89875,-16.8)">
+					<title>Rounded Rectangle.1047</title>
+					<desc>Worker2 Core</desc>
+					<v:userDefs>
+						<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.097953125):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.097953125):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.097953125):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.097953125):1"/>
+					</v:userDefs>
+					<v:textBlock v:margins="rect(4,4,4,4)" v:verticalAlign="0"/>
+					<v:textRect cx="35.2631" cy="101.413" width="70.53" height="63"/>
+					<path d="M7.05 132.91 L63.47 132.91 A7.05251 7.05251 -180 0 0 70.53 125.86 L70.53 76.97 A7.05251 7.05251 -180
+								 0 0 63.47 69.91 L7.05 69.91 A7.05251 7.05251 -180 0 0 0 76.97 L0 125.86 A7.05251 7.05251 -180 0
+								 0 7.05 132.91 Z" class="st5"/>
+					<text x="13.82" y="84.71" class="st10" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Worker2<v:newlineChar/><v:newlineChar/><tspan
+								x="21.92" dy="2.357em" class="st11">Core</tspan></text>				</g>
+				<g id="shape1055-119" v:mID="1055" v:groupContext="shape">
+					<title>Rounded Rectangle.1046</title>
+					<desc>Worker1 Core</desc>
+					<v:userDefs>
+						<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.098333333333333):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.098333333333333):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.098333333333333):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.098333333333333):1"/>
+					</v:userDefs>
+					<v:textBlock v:margins="rect(4,4,4,4)" v:verticalAlign="0"/>
+					<v:textRect cx="35.4" cy="101.413" width="70.8" height="63"/>
+					<path d="M7.08 132.91 L63.72 132.91 A7.07988 7.07988 -180 0 0 70.8 125.83 L70.8 76.99 A7.07988 7.07988 -180 0
+								 0 63.72 69.91 L7.08 69.91 A7.07988 7.07988 -180 0 0 0 76.99 L0 125.83 A7.07988 7.07988 -180 0 0
+								 7.08 132.91 Z" class="st5"/>
+					<text x="13.96" y="84.71" class="st10" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Worker1<v:newlineChar/><v:newlineChar/><tspan
+								x="22.05" dy="2.357em" class="st11">Core</tspan></text>				</g>
 			</g>
-			<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st6"
-   id="path161"
-   inkscape:connector-curvature="0"
-   style="fill:#ffd965;stroke:#c7c8c8;stroke-width:0.25" />
-			<text
-   x="11.69"
-   y="594.5"
-   class="st4"
-   v:langID="1033"
-   id="text163"
-   style="fill:#feffff;font-family:Calibri"><v:paragraph
-   v:horizAlign="1" /><v:tabList />W N</text>
-
-		</g>
-		<g
-   id="shape21-89"
-   v:mID="21"
-   v:groupContext="shape"
-   transform="matrix(0.77644652,0,0,0.77644652,389.70366,-348.94537)">
-			<title
-   id="title166">Circle.21</title>
-			<desc
-   id="desc168">W 1</desc>
-			<v:userDefs>
-				<v:ud
-   v:nameU="visVersion"
-   v:val="VT0(15):26" />
-			</v:userDefs>
-			<v:textBlock
-   v:margins="rect(4,4,4,4)" />
-			<v:textRect
-   cx="20.5"
-   cy="591.5"
-   width="35.88"
-   height="30.75" />
-			<g
-   id="shadow21-90"
-   v:groupContext="shadow"
-   v:shadowOffsetX="0.345598"
-   v:shadowOffsetY="-1.97279"
-   v:shadowType="1"
-   transform="translate(0.345598,1.97279)"
-   class="st1"
-   style="visibility:visible">
-				<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st2"
-   id="path171"
-   inkscape:connector-curvature="0"
-   style="fill:#5b9bd5;fill-opacity:0.22000002;stroke:#5b9bd5;stroke-opacity:0.22000002;filter:url(#filter_2)" />
+			<g id="group1063-123" transform="translate(369.6,-18.6)" v:mID="1063" v:groupContext="group">
+				<title>Sheet.1063</title>
+				<g id="shape1064-124" v:mID="1064" v:groupContext="shape" transform="translate(18.66,-50.7)">
+					<title>Rounded Rectangle.1049</title>
+					<desc>Worker4 Core</desc>
+					<v:userDefs>
+						<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.097416666666665):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.097416666666665):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.097416666666665):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.097416666666665):1"/>
+					</v:userDefs>
+					<v:textBlock v:margins="rect(4,4,4,4)" v:verticalAlign="0"/>
+					<v:textRect cx="35.07" cy="101.413" width="70.14" height="63"/>
+					<path d="M7.01 132.91 L63.13 132.91 A7.01389 7.01389 -180 0 0 70.14 125.9 L70.14 76.93 A7.01389 7.01389 -180
+								 0 0 63.13 69.91 L7.01 69.91 A7.01389 7.01389 -180 0 0 0 76.93 L0 125.9 A7.01389 7.01389 -180 0 0
+								 7.01 132.91 Z" class="st5"/>
+					<text x="13.63" y="84.71" class="st10" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Worker4<v:newlineChar/><v:newlineChar/><tspan
+								x="21.72" dy="2.357em" class="st11">Core</tspan></text>				</g>
+				<g id="shape1065-128" v:mID="1065" v:groupContext="shape" transform="translate(12.9,-33.6)">
+					<title>Rounded Rectangle.1048</title>
+					<desc>Worker3 Core</desc>
+					<v:userDefs>
+						<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.099166666666667):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.099166666666667):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.099166666666667):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.099166666666667):1"/>
+					</v:userDefs>
+					<v:textBlock v:margins="rect(4,4,4,4)" v:verticalAlign="0"/>
+					<v:textRect cx="35.7" cy="101.413" width="71.4" height="63"/>
+					<path d="M7.14 132.91 L64.26 132.91 A7.13988 7.13988 -180 0 0 71.4 125.77 L71.4 77.05 A7.13988 7.13988 -180 0
+								 0 64.26 69.91 L7.14 69.91 A7.13988 7.13988 -180 0 0 0 77.05 L0 125.77 A7.13988 7.13988 -180 0 0
+								 7.14 132.91 Z" class="st5"/>
+					<text x="14.26" y="84.71" class="st10" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Worker3<v:newlineChar/><v:newlineChar/><tspan
+								x="22.35" dy="2.357em" class="st11">Core</tspan></text>				</g>
+				<g id="shape1066-132" v:mID="1066" v:groupContext="shape" transform="translate(5.89875,-16.8)">
+					<title>Rounded Rectangle.1047</title>
+					<desc>Worker2 Core</desc>
+					<v:userDefs>
+						<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.097953125):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.097953125):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.097953125):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.097953125):1"/>
+					</v:userDefs>
+					<v:textBlock v:margins="rect(4,4,4,4)" v:verticalAlign="0"/>
+					<v:textRect cx="35.2631" cy="101.413" width="70.53" height="63"/>
+					<path d="M7.05 132.91 L63.47 132.91 A7.05251 7.05251 -180 0 0 70.53 125.86 L70.53 76.97 A7.05251 7.05251 -180
+								 0 0 63.47 69.91 L7.05 69.91 A7.05251 7.05251 -180 0 0 0 76.97 L0 125.86 A7.05251 7.05251 -180 0
+								 0 7.05 132.91 Z" class="st5"/>
+					<text x="13.82" y="84.71" class="st10" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Worker2<v:newlineChar/><v:newlineChar/><tspan
+								x="21.92" dy="2.357em" class="st11">Core</tspan></text>				</g>
+				<g id="shape1067-136" v:mID="1067" v:groupContext="shape">
+					<title>Rounded Rectangle.1046</title>
+					<desc>Worker1 Core</desc>
+					<v:userDefs>
+						<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(1):5"/>
+						<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15):1"/>
+						<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
+						<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.098333333333333):1"/>
+						<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.098333333333333):1"/>
+						<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.098333333333333):1"/>
+						<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.098333333333333):1"/>
+					</v:userDefs>
+					<v:textBlock v:margins="rect(4,4,4,4)" v:verticalAlign="0"/>
+					<v:textRect cx="35.4" cy="101.413" width="70.8" height="63"/>
+					<path d="M7.08 132.91 L63.72 132.91 A7.07988 7.07988 -180 0 0 70.8 125.83 L70.8 76.99 A7.07988 7.07988 -180 0
+								 0 63.72 69.91 L7.08 69.91 A7.07988 7.07988 -180 0 0 0 76.99 L0 125.83 A7.07988 7.07988 -180 0 0
+								 7.08 132.91 Z" class="st5"/>
+					<text x="13.96" y="84.71" class="st10" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Worker1<v:newlineChar/><v:newlineChar/><tspan
+								x="22.05" dy="2.357em" class="st11">Core</tspan></text>				</g>
 			</g>
-			<path
-   d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-   class="st6"
-   id="path173"
-   inkscape:connector-curvature="0"
-   style="fill:#ffd965;stroke:#c7c8c8;stroke-width:0.25" />
-			<text
-   x="12.39"
-   y="594.5"
-   class="st4"
-   v:langID="1033"
-   id="text175"
-   style="fill:#feffff;font-family:Calibri"><v:paragraph
-   v:horizAlign="1" /><v:tabList />W 1</text>
-
-		</g>
-		<g
-   id="shape28-95"
-   v:mID="28"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,345.12321,-305.07614)">
-			<title
-   id="title178">Dynamic connector.28</title>
-			<path
-   d="m 0,603 50.38,0"
-   class="st7"
-   id="path180"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape29-100"
-   v:mID="29"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,345.12321,-312.06416)">
-			<title
-   id="title183">Dynamic connector.29</title>
-			<path
-   d="m 0,612 28.33,0 0,-68 22.05,0"
-   class="st7"
-   id="path185"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape30-105"
-   v:mID="30"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,345.12321,-312.06416)">
-			<title
-   id="title188">Dynamic connector.30</title>
-			<path
-   d="m 0,612 28.33,0 0,68 22.05,0"
-   class="st7"
-   id="path190"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape31-110"
-   v:mID="31"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,421.53797,-259.2658)">
-			<title
-   id="title193">Dynamic connector.31</title>
-			<path
-   d="m 0,612 24.42,0 0,-68 25.96,0"
-   class="st7"
-   id="path195"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape32-115"
-   v:mID="32"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,421.53797,-305.07614)">
-			<title
-   id="title198">Dynamic connector.32</title>
-			<path
-   d="m 0,603 50.38,0"
-   class="st7"
-   id="path200"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape33-120"
-   v:mID="33"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,421.53797,-364.86253)">
-			<title
-   id="title203">Dynamic connector.33</title>
-			<path
-   d="m 0,612 24.42,0 0,68 25.96,0"
-   class="st7"
-   id="path205"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
-		</g>
-		<g
-   id="shape34-125"
-   v:mID="34"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,252.79052,-364.86253)">
-			<title
-   id="title208">Dynamic connector.34</title>
-			<path
-   d="m 0,612 26.88,0 0,68 23.5,0"
-   class="st7"
-   id="path210"
-   inkscape:connector-curvature="0"
-   style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" />
 		</g>
-	<text
-   xml:space="preserve"
-   style="font-size:24.84628868px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
-   x="153.38116"
-   y="165.90149"
-   id="text3106"
-   sodipodi:linespacing="125%"><tspan
-     sodipodi:role="line"
-     x="153.38116"
-     y="165.90149"
-     id="tspan3110"
-     style="font-size:8.69620132px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#ffffff;fill-opacity:1;font-family:Sans;-inkscape-font-specification:Sans">Atomic #1</tspan></text>
-<text
-   xml:space="preserve"
-   style="font-size:24.84628868px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;overflow:visible;font-family:Sans"
-   x="322.12939"
-   y="165.90149"
-   id="text3106-1"
-   sodipodi:linespacing="125%"><tspan
-     sodipodi:role="line"
-     x="322.12939"
-     y="165.90149"
-     id="tspan3110-4"
-     style="font-size:8.69620132px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#ffffff;fill-opacity:1;font-family:Sans;-inkscape-font-specification:Sans">Atomic #2</tspan></text>
-<text
-   xml:space="preserve"
-   style="font-size:24.84628868px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;overflow:visible;font-family:Sans"
-   x="491.82089"
-   y="172.79289"
-   id="text3106-0"
-   sodipodi:linespacing="125%"><tspan
-     sodipodi:role="line"
-     x="491.82089"
-     y="172.79289"
-     style="font-size:8.69620132px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#ffffff;fill-opacity:1;font-family:Sans;-inkscape-font-specification:Sans"
-     id="tspan3923" /></text>
-<text
-   xml:space="preserve"
-   style="font-size:24.84628868px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;overflow:visible;font-family:Sans"
-   x="491.02899"
-   y="165.03951"
-   id="text3106-8-5"
-   sodipodi:linespacing="125%"><tspan
-     sodipodi:role="line"
-     x="491.02899"
-     y="165.03951"
-     id="tspan3110-2-1"
-     style="font-size:8.69620132px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#ffffff;fill-opacity:1;font-family:Sans;-inkscape-font-specification:Sans">Single Link</tspan></text>
-<g
-   style="font-size:12px;fill:none;stroke-linecap:square;stroke-miterlimit:3;overflow:visible"
-   id="shape5-22-1"
-   v:mID="5"
-   v:groupContext="shape"
-   transform="matrix(0.77644652,0,0,0.77644652,556.00223,-296.89447)"><title
-     id="title64-5">Circle</title><desc
-     id="desc66-2">RX</desc><v:userDefs><v:ud
-       v:nameU="visVersion"
-       v:val="VT0(15):26" /></v:userDefs><v:textBlock
-     v:margins="rect(4,4,4,4)" /><v:textRect
-     cx="20.5"
-     cy="591.5"
-     width="35.88"
-     height="30.75" /><g
-     id="shadow5-23-7"
-     v:groupContext="shadow"
-     v:shadowOffsetX="0.345598"
-     v:shadowOffsetY="-1.97279"
-     v:shadowType="1"
-     transform="translate(0.345598,1.97279)"
-     class="st1"
-     style="visibility:visible"><path
-       d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-       class="st2"
-       id="path69-6"
-       inkscape:connector-curvature="0"
-       style="fill:#5b9bd5;fill-opacity:0.22000002;stroke:#5b9bd5;stroke-opacity:0.22000002;filter:url(#filter_2-7)" /></g><path
-     d="m 0,591.5 a 20.5,20.5 0 0 1 41,0 20.5,20.5 0 1 1 -41,0 z"
-     class="st6"
-     id="path71-1"
-     inkscape:connector-curvature="0"
-     style="fill:#ffd965;stroke:#c7c8c8;stroke-width:0.25" /><text
-     x="11.06866"
-     y="596.56067"
-     class="st4"
-     v:langID="1033"
-     id="text73-4"
-     style="fill:#feffff;font-family:Calibri"> TX</text>
-</g><g
-   style="font-size:12px;fill:none;stroke-linecap:square;stroke-miterlimit:3;overflow:visible"
-   id="shape28-95-5"
-   v:mID="28"
-   v:groupContext="shape"
-   v:layerMember="0"
-   transform="matrix(0.77644652,0,0,0.77644652,512.00213,-305.42637)"><title
-     id="title178-7">Dynamic connector.28</title><path
-     d="m 0,603 50.38,0"
-     class="st7"
-     id="path180-6"
-     inkscape:connector-curvature="0"
-     style="stroke:#5b9bd5;stroke-width:1;stroke-linecap:round;stroke-linejoin:round;marker-end:url(#mrkr4-33)" /></g></g>
+	</g>
 </svg>
diff --git a/doc/guides/prog_guide/img/turbo_tb_decode.svg b/doc/guides/prog_guide/img/turbo_tb_decode.svg
new file mode 100644
index 00000000..a259f458
--- /dev/null
+++ b/doc/guides/prog_guide/img/turbo_tb_decode.svg
@@ -0,0 +1,1471 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<!-- SPDX-License-Identifier: BSD-3-Clause -->
+<!-- Copyright(c) 2018 Intel Corporation -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="194.21973mm"
+   height="163.25349mm"
+   viewBox="0 0 194.21973 163.25349"
+   version="1.1"
+   id="svg8"
+   inkscape:version="0.92.3 (2405546, 2018-03-11)"
+   sodipodi:docname="turbo_tb_decode.svg">
+  <defs
+     id="defs2">
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker8474"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Mend">
+      <path
+         transform="scale(-0.6)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         id="path8472"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path6507"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5140"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Mstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5122"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(0.4,0,0,0.4,4,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Lstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Lstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5134"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="matrix(1.1,0,0,1.1,1.1,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5116"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(0.8,0,0,0.8,10,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Sstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Sstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5128"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(0.2,0,0,0.2,1.2,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend"
+       style="overflow:visible"
+       inkscape:isstock="true"
+       inkscape:collect="always">
+      <path
+         id="path5143"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Send"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5131"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5119"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <filter
+       id="filter_2">
+      <feGaussianBlur
+         stdDeviation="2"
+         id="feGaussianBlur4" />
+    </filter>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-0"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-5"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-1"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-0"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-4"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-9"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-11"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-2"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-0-1"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-5-6"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-1-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-0-1"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker8474-2"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Mend">
+      <path
+         inkscape:connector-curvature="0"
+         transform="scale(-0.6)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         id="path8472-4" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.98994949"
+     inkscape:cx="148.9027"
+     inkscape:cy="256.96386"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:snap-text-baseline="true"
+     inkscape:window-width="1920"
+     inkscape:window-height="1137"
+     inkscape:window-x="1072"
+     inkscape:window-y="185"
+     inkscape:window-maximized="1"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     style="display:inline"
+     transform="translate(-9.7553377,-54.351435)">
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="41.159508"
+       y="61.13464"
+       id="text873"><tspan
+         sodipodi:role="line"
+         id="tspan871"
+         x="41.159508"
+         y="64.996841"
+         style="font-size:4.23333311px;stroke-width:0.26458332" /></text>
+    <g
+       id="g4997">
+      <a
+         id="a990">
+        <rect
+           style="opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.1950596;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           id="rect854"
+           width="44.771584"
+           height="14.03559"
+           x="18.573261"
+           y="54.450935" />
+      </a>
+      <text
+         id="text877"
+         y="62.738258"
+         x="21.648832"
+         style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:3.52777767px;stroke-width:0.26458332"
+           y="62.738258"
+           x="21.648832"
+           id="tspan875"
+           sodipodi:role="line">w<tspan
+   id="tspan885"
+   style="font-size:64.99999762%;baseline-shift:sub">k</tspan> LLR circular buffer</tspan></text>
+    </g>
+    <rect
+       style="opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.1981452;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967"
+       width="25.196077"
+       height="14.03559"
+       x="63.344845"
+       y="54.450935" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="73.52343"
+       y="62.445942"
+       id="text877-8"><tspan
+         sodipodi:role="line"
+         id="tspan875-6"
+         x="73.52343"
+         y="62.445942"
+         style="font-size:3.52777767px;stroke-width:0.26458332">...</tspan></text>
+    <rect
+       style="opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.1950596;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect854-5"
+       width="44.771584"
+       height="14.03559"
+       x="88.540924"
+       y="54.450935" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="91.90699"
+       y="62.531521"
+       id="text877-4"><tspan
+         sodipodi:role="line"
+         id="tspan875-0"
+         x="91.90699"
+         y="62.531521"
+         style="font-size:3.52777767px;stroke-width:0.26458332">w<tspan
+   style="font-size:2.2930553px;baseline-shift:sub;stroke-width:0.26458332"
+   id="tspan885-9">k</tspan> LLR circular buffer</tspan></text>
+    <rect
+       style="opacity:1;fill:#d8d8d8;fill-opacity:1;stroke:#000000;stroke-width:0.199;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect4735"
+       width="6.6797671"
+       height="14.033618"
+       x="11.893495"
+       y="54.450935" />
+    <g
+       id="g4807"
+       transform="translate(0,0.188984)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741"
+         d="m 11.929873,70.237907 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5"
+         d="m 18.686199,70.237907 c 0,0 -0.4016,2.480469 -1.370164,2.456847 -0.968564,-0.02363 -2.007999,-0.02363 -2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760"
+         d="m 15.308036,72.671124 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <g
+       id="g5063">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-0"
+         d="m 18.686199,70.426891 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-8"
+         d="m 132.59878,70.332401 c 0,0 -0.4016,2.480469 -1.37017,2.456847 -0.96856,-0.02363 -2.008,-0.02363 -2.008,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-2"
+         d="m 75.831475,72.954606 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4809"
+         d="M 22.064362,72.860108 129.22061,72.765618"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <flowRoot
+       xml:space="preserve"
+       id="flowRoot4811"
+       style="font-style:normal;font-weight:normal;font-size:13.33333302px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none"
+       transform="scale(0.26458333)"><flowRegion
+         id="flowRegion4813"><rect
+           id="rect4815"
+           width="41.785713"
+           height="14.642858"
+           x="39.285713"
+           y="287.16254" /></flowRegion><flowPara
+         id="flowPara4817">offse</flowPara></flowRoot>    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="74.16684"
+       y="75.043541"
+       id="text4821-3"
+       transform="scale(0.95903923,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0"
+         x="74.16684"
+         y="75.043541"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">length</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="11.603812"
+       y="75.449066"
+       id="text4821-3-9"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0"
+         x="11.603812"
+         y="75.449066"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">offset</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#7f8085;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="150.53461"
+       y="58.039307"
+       id="text4821-3-6"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-1"
+         x="150.53461"
+         y="58.039307"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144">The encoded TB is given as a</tspan><tspan
+         sodipodi:role="line"
+         x="150.53461"
+         y="61.97533"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan4877">contiguous buffer</tspan></text>
+    <path
+       style="fill:none;stroke:#7f7f7f;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:2.1199999, 0.26499999;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 10.423511,83.31801 H 72.162772"
+       id="path4885"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#818181;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:2.1199999, 0.26499999;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 82.319012,83.31801 176.93243,83.184377"
+       id="path4887"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:5.39796209px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#818181;fill-opacity:1;stroke:none;stroke-width:0.40484715"
+       x="77.368088"
+       y="81.855705"
+       id="text4891"
+       transform="scale(0.96100989,1.040572)"><tspan
+         sodipodi:role="line"
+         id="tspan4889"
+         x="77.368088"
+         y="81.855705"
+         style="fill:#818181;fill-opacity:1;stroke-width:0.40484715">or</tspan></text>
+    <path
+       style="fill:none;stroke:#7f7f7f;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:2.11999992, 0.26499999;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 9.7553377,181.2723 H 71.494599"
+       id="path4885-9"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#818181;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:2.11999992, 0.26499999;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 81.786039,181.24224 94.613421,-0.13363"
+       id="path4887-9"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:5.39796209px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#818181;fill-opacity:1;stroke:none;stroke-width:0.40484715"
+       x="76.813484"
+       y="175.96187"
+       id="text4891-9"
+       transform="scale(0.9610099,1.040572)"><tspan
+         sodipodi:role="line"
+         id="tspan4889-7"
+         x="76.813484"
+         y="175.96187"
+         style="fill:#818181;fill-opacity:1;stroke-width:0.40484715">or</tspan></text>
+    <rect
+       style="opacity:1;fill:#d8d8d8;fill-opacity:1;stroke:#000000;stroke-width:0.199;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect4735-7"
+       width="6.6797671"
+       height="14.033618"
+       x="10.825401"
+       y="101.15727" />
+    <g
+       id="g4807-3"
+       transform="translate(-0.98393573,46.759016)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-2"
+         d="m 11.929873,70.237907 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-81"
+         d="m 18.686199,70.237907 c 0,0 -0.4016,2.480469 -1.370164,2.456847 -0.968564,-0.02363 -2.007999,-0.02363 -2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-3"
+         d="m 15.308036,72.671124 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="10.57785"
+       y="120.11156"
+       id="text4821-3-9-7"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0-3"
+         x="10.57785"
+         y="120.11156"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">offset</tspan></text>
+    <g
+       transform="matrix(0.99106501,0,0,1.0000618,-0.90257595,46.700562)"
+       id="g4997-1">
+      <a
+         id="a990-9">
+        <rect
+           style="opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.1950596;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           id="rect854-4"
+           width="44.771584"
+           height="14.03559"
+           x="18.573261"
+           y="54.450935" />
+      </a>
+      <text
+         id="text877-1"
+         y="62.738258"
+         x="21.648832"
+         style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:3.52777767px;stroke-width:0.26458332"
+           y="62.738258"
+           x="21.648832"
+           id="tspan875-9"
+           sodipodi:role="line">w<tspan
+   id="tspan885-5"
+   style="font-size:64.99999762%;baseline-shift:sub">k</tspan> LLR circular buffer</tspan></text>
+    </g>
+    <g
+       transform="matrix(0.98808659,0,0,1.0000825,72.450284,46.566642)"
+       id="g4997-1-4">
+      <a
+         id="a990-9-5">
+        <rect
+           style="opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.1950596;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+           id="rect854-4-6"
+           width="44.771584"
+           height="14.03559"
+           x="18.573261"
+           y="54.450935" />
+      </a>
+      <text
+         id="text877-1-0"
+         y="62.738258"
+         x="21.648832"
+         style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:3.52777767px;stroke-width:0.26458332"
+           y="62.738258"
+           x="21.648832"
+           id="tspan875-9-8"
+           sodipodi:role="line">w<tspan
+   id="tspan885-5-7"
+   style="font-size:64.99999762%;baseline-shift:sub">k</tspan> LLR circular buffer</tspan></text>
+    </g>
+    <g
+       transform="matrix(1.0292712,0,0,0.99978365,-1.5276486,46.585803)"
+       id="g5063-5">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-0-0"
+         d="m 18.686199,70.426891 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-8-2"
+         d="m 132.59878,70.332401 c 0,0 -0.4016,2.480469 -1.37017,2.456847 -0.96856,-0.02363 -2.008,-0.02363 -2.008,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-2-4"
+         d="m 75.831475,72.954606 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4809-8"
+         d="M 22.064362,72.860108 129.22061,72.765618"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="74.729973"
+       y="119.73718"
+       id="text4821-3-3"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7"
+         x="74.729973"
+         y="119.73718"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">length</tspan></text>
+    <path
+       style="fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 61.876283,101.15486 9.985811,-0.03 v 4.04245 l -2.53906,1.46999 5.57925,2.87314 -3.006781,1.60362 v 4.04245 l -10.01922,0.0348 z"
+       id="path5575"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 90.802274,101.02207 -9.953242,0.003 -0.03341,3.94222 -2.639287,1.63703 5.679477,2.87314 -2.939965,1.57021 v 4.00905 l 9.886426,0.003 z"
+       id="path5581"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="64.84626"
+       y="109.37679"
+       id="text5879"><tspan
+         sodipodi:role="line"
+         id="tspan5877"
+         x="64.84626"
+         y="109.37679"
+         style="stroke-width:0.26458332">..</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="85.559654"
+       y="109.51042"
+       id="text5883"><tspan
+         sodipodi:role="line"
+         id="tspan5881"
+         x="85.559654"
+         y="109.51042"
+         style="stroke-width:0.26458332">..</tspan></text>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow2Mend)"
+       d="m 66.783973,101.05803 c 0,0 5.144937,-13.096212 18.942727,-0.33409"
+       id="path5899"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.06663418px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#7f8085;fill-opacity:1;stroke:none;stroke-width:0.22999756"
+       x="155.38319"
+       y="97.845207"
+       id="text4821-3-6-2"
+       transform="scale(0.93400804,1.0706546)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-1-0"
+         x="155.38319"
+         y="97.845207"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.22999756">The encoded TB is given as a</tspan><tspan
+         sodipodi:role="line"
+         x="155.38319"
+         y="101.6785"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.22999756"
+         id="tspan4877-1">&quot;scattered&quot; buffer through a</tspan><tspan
+         sodipodi:role="line"
+         x="155.38319"
+         y="105.5118"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.22999756"
+         id="tspan5992">chained mbuf</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.06663418px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#7f8085;fill-opacity:1;stroke:none;stroke-width:0.22999756"
+       x="147.15628"
+       y="145.52435"
+       id="text4821-3-6-2-4"
+       transform="scale(0.93400803,1.0706546)"><tspan
+         sodipodi:role="line"
+         x="147.15628"
+         y="145.52435"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.22999756"
+         id="tspan5992-2">Result is decoded back into the given output</tspan><tspan
+         sodipodi:role="line"
+         x="147.15628"
+         y="149.35765"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.22999756"
+         id="tspan6023">mbuf as one contiguous buffer with no </tspan><tspan
+         sodipodi:role="line"
+         x="147.15628"
+         y="153.19093"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.22999756"
+         id="tspan6025">CRC24B retaining</tspan></text>
+    <g
+       id="g6253">
+      <g
+         transform="translate(10.356694,1.2027129)"
+         id="g6079">
+        <path
+           style="fill:#fec000;fill-opacity:1;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           d="m 64.144685,140.61392 -0.03341,-11.65963 h 4.209497 l -0.03341,11.65963 1.971111,-0.0334 -4.042449,3.90882 -4.04245,-3.90882 z"
+           id="path6066"
+           inkscape:connector-curvature="0" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+           x="-142.49652"
+           y="67.226768"
+           id="text6074"
+           transform="rotate(-90)"><tspan
+             sodipodi:role="line"
+             id="tspan6072"
+             x="-142.49652"
+             y="67.226768"
+             style="font-size:2.82222223px;stroke-width:0.26458332">decode</tspan></text>
+      </g>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.06663418px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#7f8085;fill-opacity:1;stroke:none;stroke-width:0.22999756"
+       x="146.80908"
+       y="180.29805"
+       id="text4821-3-6-2-4-6"
+       transform="scale(0.93400803,1.0706546)"><tspan
+         sodipodi:role="line"
+         x="146.80908"
+         y="180.29805"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.22999756"
+         id="tspan6025-0">Result is decoded back into the given output</tspan><tspan
+         sodipodi:role="line"
+         x="146.80908"
+         y="184.13135"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.22999756"
+         id="tspan6110">mbuf as one contiguous buffer with CRC24B</tspan><tspan
+         sodipodi:role="line"
+         x="146.80908"
+         y="187.96463"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.22999756"
+         id="tspan6112">retained in place when</tspan><tspan
+         sodipodi:role="line"
+         x="146.80908"
+         y="191.79793"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.22999756"
+         id="tspan6114">RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP</tspan><tspan
+         sodipodi:role="line"
+         x="146.80908"
+         y="195.63123"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.22999756"
+         id="tspan6116">is set in op_flags</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="25.9951"
+       y="169.83803"
+       id="text4821-3-9-7-1"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0-3-0"
+         x="25.9951"
+         y="169.83803"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">offset</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="75.794968"
+       y="168.951"
+       id="text4821-3-3-0"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8"
+         x="75.794968"
+         y="168.951"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">length</tspan></text>
+    <path
+       style="display:inline;fill:none;stroke:#a8d08d;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart);marker-end:url(#marker6509)"
+       d="m 32.606882,149.73449 c 19.777945,-0.0668 19.777945,-0.0668 19.777945,-0.0668"
+       id="path6255"
+       inkscape:connector-curvature="0" />
+    <g
+       style="display:inline"
+       id="g4807-5"
+       transform="translate(13.985119,97.480562)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-6"
+         d="m 11.929873,70.237907 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-88"
+         d="m 18.686199,70.237907 c 0,0 -0.4016,2.480469 -1.370164,2.456847 -0.968564,-0.02363 -2.007999,-0.02363 -2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-9"
+         d="m 15.308036,72.671124 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <g
+       style="display:inline"
+       transform="matrix(0.78752716,0,0,1.0016782,17.928141,97.168708)"
+       id="g5063-5-2">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-0-0-0"
+         d="m 18.686199,70.426891 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-8-2-1"
+         d="m 132.59878,70.332401 c 0,0 -0.4016,2.480469 -1.37017,2.456847 -0.96856,-0.02363 -2.008,-0.02363 -2.008,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-2-4-5"
+         d="m 75.831475,72.954606 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4809-8-8"
+         d="M 22.064362,72.860108 129.22061,72.765618"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <rect
+       style="display:inline;opacity:1;fill:#d8d8d8;fill-opacity:1;stroke:#000000;stroke-width:0.199;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect4735-7-3"
+       width="6.6797671"
+       height="14.033618"
+       x="25.781357"
+       y="152.21487" />
+    <g
+       id="g8093">
+      <rect
+         y="152.21487"
+         x="32.461124"
+         height="14.0336"
+         width="16.248745"
+         id="rect6210"
+         style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         id="text6214"
+         y="158.15347"
+         x="40.321487"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+           y="158.15347"
+           x="40.321487"
+           id="tspan6212"
+           sodipodi:role="line">CB<tspan
+   id="tspan6218"
+   style="font-size:64.99999762%;text-align:center;baseline-shift:sub;text-anchor:middle">1</tspan></tspan><tspan
+           id="tspan6216"
+           style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+           y="162.5632"
+           x="40.321487"
+           sodipodi:role="line">hard</tspan></text>
+    </g>
+    <g
+       id="g8100">
+      <rect
+         y="152.21487"
+         x="48.709869"
+         height="14.0336"
+         width="16.248745"
+         id="rect6210-1"
+         style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         id="text6214-7"
+         y="158.15347"
+         x="56.570232"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+           y="158.15347"
+           x="56.570232"
+           id="tspan6212-1"
+           sodipodi:role="line">CB<tspan
+   id="tspan6247"
+   style="font-size:64.99999762%;baseline-shift:sub">2</tspan></tspan><tspan
+           id="tspan6216-4"
+           style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+           y="162.5632"
+           x="56.570232"
+           sodipodi:role="line">hard</tspan></text>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="39.47636"
+       y="140.81966"
+       id="text4821-3-3-0-5"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3"
+         x="39.47636"
+         y="140.81966"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">k_neg</tspan></text>
+    <g
+       id="g8252">
+      <rect
+         y="152.21487"
+         x="64.958618"
+         height="14.033598"
+         width="24.977577"
+         id="rect6693"
+         style="opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.26701048;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         id="text6697"
+         y="160.55891"
+         x="75.637276"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="stroke-width:0.26458332"
+           y="160.55891"
+           x="75.637276"
+           id="tspan6695"
+           sodipodi:role="line">...</tspan></text>
+    </g>
+    <rect
+       style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+       id="rect6210-8"
+       width="16.248745"
+       height="14.0336"
+       x="89.936195"
+       y="152.21487" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="98.231049"
+       y="158.29669"
+       id="text6214-78"><tspan
+         sodipodi:role="line"
+         id="tspan6212-2"
+         x="98.231049"
+         y="158.29669"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;text-align:center;baseline-shift:sub;text-anchor:middle;stroke-width:0.26458332"
+   id="tspan6218-3">c-1</tspan></tspan><tspan
+         sodipodi:role="line"
+         x="98.231049"
+         y="162.70642"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+         id="tspan6216-2">hard</tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.23060164;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+       id="rect6210-8-3"
+       width="12.2741"
+       height="14.067998"
+       x="106.16774"
+       y="152.19768" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="112.17137"
+       y="158.42459"
+       id="text6214-78-6"><tspan
+         sodipodi:role="line"
+         id="tspan6212-2-9"
+         x="112.17137"
+         y="158.42459"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;text-align:center;baseline-shift:sub;text-anchor:middle;stroke-width:0.26458332"
+   id="tspan6218-3-7">c</tspan></tspan><tspan
+         sodipodi:role="line"
+         x="112.17137"
+         y="162.83432"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+         id="tspan6216-2-9">hard</tspan></text>
+    <g
+       id="g6838"
+       transform="translate(-4.1092682)">
+      <rect
+         y="152.22496"
+         x="122.55111"
+         height="14.040706"
+         width="4.0010114"
+         id="rect6777"
+         style="opacity:1;fill:#375623;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781"
+         y="125.5383"
+         x="-164.82439"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="125.5383"
+           x="-164.82439"
+           id="tspan6779"
+           sodipodi:role="line">CRC24A</tspan></text>
+    </g>
+    <path
+       style="display:inline;fill:none;stroke:#a8d08d;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart-0);marker-end:url(#marker6509-1)"
+       d="m 105.83873,149.28245 c 19.77795,-0.0668 19.77795,-0.0668 19.77795,-0.0668"
+       id="path6255-2"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="115.83596"
+       y="140.38614"
+       id="text4821-3-3-0-5-1"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3-7"
+         x="115.83596"
+         y="140.38614"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">k_pos</tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#d8d8d8;fill-opacity:1;stroke:#000000;stroke-width:0.199;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect4735-7-3-8"
+       width="6.6797671"
+       height="14.033618"
+       x="17.908045"
+       y="192.83072" />
+    <g
+       style="display:inline"
+       id="g4807-5-7"
+       transform="translate(6.0254188,138.42182)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-6-7"
+         d="m 11.929873,70.237907 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-88-6"
+         d="m 18.686199,70.237907 c 0,0 -0.4016,2.480469 -1.370164,2.456847 -0.968564,-0.02363 -2.007999,-0.02363 -2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-9-4"
+         d="m 15.308036,72.671124 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="17.775927"
+       y="208.62222"
+       id="text4821-3-9-7-1-7"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0-3-0-0"
+         x="17.775927"
+         y="208.62222"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">offset</tspan></text>
+    <path
+       style="display:inline;fill:none;stroke:#a8d08d;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart-4);marker-end:url(#marker6509-11)"
+       d="m 24.97933,190.59164 c 19.777946,-0.0668 19.777946,-0.0668 19.777946,-0.0668"
+       id="path6255-0"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="31.523031"
+       y="180.00327"
+       id="text4821-3-3-0-5-5"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3-9"
+         x="31.523031"
+         y="180.00327"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">k_neg</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="75.525093"
+       y="208.04131"
+       id="text4821-3-3-0-9"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-8"
+         x="75.525093"
+         y="208.04131"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">length</tspan></text>
+    <g
+       style="display:inline"
+       transform="matrix(0.91917288,0,0,1.0006169,7.5499955,138.19202)"
+       id="g5063-5-2-9">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-0-0-0-9"
+         d="m 18.686199,70.426891 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-8-2-1-3"
+         d="m 132.59878,70.332401 c 0,0 -0.4016,2.480469 -1.37017,2.456847 -0.96856,-0.02363 -2.008,-0.02363 -2.008,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-2-4-5-3"
+         d="m 75.831475,72.954606 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4809-8-8-4"
+         d="M 22.064362,72.860108 129.22061,72.765618"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <path
+       style="display:inline;fill:none;stroke:#a8d08d;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart-0-1);marker-end:url(#marker6509-1-3)"
+       d="m 109.83498,190.21366 c 19.77795,-0.0668 19.77795,-0.0668 19.77795,-0.0668"
+       id="path6255-2-4"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="120.00289"
+       y="179.64078"
+       id="text4821-3-3-0-5-1-5"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3-7-6"
+         x="120.00289"
+         y="179.64078"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">k_pos</tspan></text>
+    <g
+       transform="translate(-7.873312,40.61586)"
+       style="display:inline"
+       id="g8093-1">
+      <rect
+         y="152.21487"
+         x="32.461124"
+         height="14.0336"
+         width="16.248745"
+         id="rect6210-9"
+         style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         id="text6214-9"
+         y="158.15347"
+         x="40.321487"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+           y="158.15347"
+           x="40.321487"
+           id="tspan6212-7"
+           sodipodi:role="line">CB<tspan
+   id="tspan6218-8"
+   style="font-size:64.99999762%;text-align:center;baseline-shift:sub;text-anchor:middle">1</tspan></tspan><tspan
+           id="tspan6216-0"
+           style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+           y="162.5632"
+           x="40.321487"
+           sodipodi:role="line">hard</tspan></text>
+    </g>
+    <g
+       style="display:inline"
+       id="g6838-7"
+       transform="translate(-81.714552,40.598663)">
+      <rect
+         y="152.22496"
+         x="122.55111"
+         height="14.040706"
+         width="4.0010114"
+         id="rect6777-5"
+         style="opacity:1;fill:#375623;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-1"
+         y="125.5383"
+         x="-164.82439"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="125.5383"
+           x="-164.82439"
+           id="tspan6779-7"
+           sodipodi:role="line">CRC24B</tspan></text>
+    </g>
+    <g
+       transform="translate(-3.8722974,40.608749)"
+       style="display:inline"
+       id="g8100-3">
+      <rect
+         y="152.21487"
+         x="48.709869"
+         height="14.0336"
+         width="16.248745"
+         id="rect6210-1-8"
+         style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         id="text6214-7-4"
+         y="158.15347"
+         x="56.570232"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+           y="158.15347"
+           x="56.570232"
+           id="tspan6212-1-0"
+           sodipodi:role="line">CB<tspan
+   id="tspan6247-0"
+   style="font-size:64.99999762%;baseline-shift:sub">2</tspan></tspan><tspan
+           id="tspan6216-4-1"
+           style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+           y="162.5632"
+           x="56.570232"
+           sodipodi:role="line">hard</tspan></text>
+    </g>
+    <g
+       style="display:inline"
+       id="g6838-7-2"
+       transform="translate(-61.464789,40.591552)">
+      <rect
+         y="152.22496"
+         x="122.55111"
+         height="14.040706"
+         width="4.0010114"
+         id="rect6777-5-2"
+         style="opacity:1;fill:#375623;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-1-7"
+         y="125.5383"
+         x="-164.82439"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="125.5383"
+           x="-164.82439"
+           id="tspan6779-7-1"
+           sodipodi:role="line">CRC24B</tspan></text>
+    </g>
+    <g
+       transform="translate(0.12871686,40.608749)"
+       style="display:inline"
+       id="g8252-9">
+      <rect
+         y="152.21487"
+         x="64.958618"
+         height="14.033598"
+         width="24.977577"
+         id="rect6693-0"
+         style="opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.26701048;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         id="text6697-7"
+         y="160.55891"
+         x="75.637276"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="stroke-width:0.26458332"
+           y="160.55891"
+           x="75.637276"
+           id="tspan6695-1"
+           sodipodi:role="line">...</tspan></text>
+    </g>
+    <rect
+       style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+       id="rect6210-8-5"
+       width="16.248745"
+       height="14.0336"
+       x="90.064911"
+       y="192.82362" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="98.359756"
+       y="198.90544"
+       id="text6214-78-69"><tspan
+         sodipodi:role="line"
+         id="tspan6212-2-98"
+         x="98.359756"
+         y="198.90544"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;text-align:center;baseline-shift:sub;text-anchor:middle;stroke-width:0.26458332"
+   id="tspan6218-3-1">c-1</tspan></tspan><tspan
+         sodipodi:role="line"
+         x="98.359756"
+         y="203.31517"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+         id="tspan6216-2-8">hard</tspan></text>
+    <g
+       style="display:inline"
+       id="g6838-7-2-9"
+       transform="translate(-16.23745,40.591553)">
+      <rect
+         y="152.22496"
+         x="122.55111"
+         height="14.040706"
+         width="4.0010114"
+         id="rect6777-5-2-0"
+         style="opacity:1;fill:#375623;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-1-7-2"
+         y="125.5383"
+         x="-164.82439"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="125.5383"
+           x="-164.82439"
+           id="tspan6779-7-1-9"
+           sodipodi:role="line">CRC24B</tspan></text>
+    </g>
+    <rect
+       style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.23060165;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+       id="rect6210-8-3-2"
+       width="12.2741"
+       height="14.067998"
+       x="110.31467"
+       y="192.81651" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="116.3183"
+       y="199.04343"
+       id="text6214-78-6-8"><tspan
+         sodipodi:role="line"
+         id="tspan6212-2-9-7"
+         x="116.3183"
+         y="199.04343"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;text-align:center;baseline-shift:sub;text-anchor:middle;stroke-width:0.26458332"
+   id="tspan6218-3-7-9">c</tspan></tspan><tspan
+         sodipodi:role="line"
+         x="116.3183"
+         y="203.45316"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+         id="tspan6216-2-9-1">hard</tspan></text>
+    <g
+       style="display:inline"
+       id="g6838-8"
+       transform="translate(0.03765869,40.591553)">
+      <rect
+         y="152.22496"
+         x="122.55111"
+         height="14.040706"
+         width="4.0010114"
+         id="rect6777-6"
+         style="opacity:1;fill:#375623;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-15"
+         y="125.5383"
+         x="-164.82439"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="125.5383"
+           x="-164.82439"
+           id="tspan6779-6"
+           sodipodi:role="line">CRC24A</tspan></text>
+    </g>
+    <g
+       style="display:inline"
+       id="g6838-7-2-7"
+       transform="translate(4.0386734,40.591553)">
+      <rect
+         y="152.22496"
+         x="122.55111"
+         height="14.040706"
+         width="4.0010114"
+         id="rect6777-5-2-04"
+         style="opacity:1;fill:#375623;fill-opacity:1;stroke:#000000;stroke-width:0.26499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-1-7-5"
+         y="125.5383"
+         x="-164.82439"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="125.5383"
+           x="-164.82439"
+           id="tspan6779-7-1-0"
+           sodipodi:role="line">CRC24B</tspan></text>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="27.138876"
+       y="85.186432"
+       id="text4821-3-9-0"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0-1"
+         x="27.138876"
+         y="85.186432"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">mbuf seg 1</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="112.34333"
+       y="85.141403"
+       id="text4821-3-9-0-8"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0-1-0"
+         x="112.34333"
+         y="85.141403"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">mbuf seg 2</tspan></text>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker8474)"
+       d="m 34.076864,90.734741 5.946746,5.746295"
+       id="path8464"
+       inkscape:connector-curvature="0" />
+    <path
+       style="display:inline;fill:none;stroke:#000000;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker8474-2)"
+       d="m 115.35315,91.80841 -5.7463,5.946752"
+       id="path8464-8"
+       inkscape:connector-curvature="0" />
+  </g>
+</svg>
diff --git a/doc/guides/prog_guide/img/turbo_tb_encode.svg b/doc/guides/prog_guide/img/turbo_tb_encode.svg
new file mode 100644
index 00000000..e3708a93
--- /dev/null
+++ b/doc/guides/prog_guide/img/turbo_tb_encode.svg
@@ -0,0 +1,1948 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<!-- SPDX-License-Identifier: BSD-3-Clause -->
+<!-- Copyright(c) 2018 Intel Corporation -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="189.87321mm"
+   height="166.91023mm"
+   viewBox="0 0 189.87321 166.91023"
+   version="1.1"
+   id="svg8"
+   inkscape:version="0.92.3 (2405546, 2018-03-11)"
+   sodipodi:docname="turbo_tb_encode.svg">
+  <defs
+     id="defs2">
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker8474"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Mend">
+      <path
+         transform="scale(-0.6)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         id="path8472"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path6507"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5140"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Mstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5122"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(0.4,0,0,0.4,4,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Lstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Lstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5134"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="matrix(1.1,0,0,1.1,1.1,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5116"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(0.8,0,0,0.8,10,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Sstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Sstart"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5128"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(0.2,0,0,0.2,1.2,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Send"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5131"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path5119"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <filter
+       id="filter_2">
+      <feGaussianBlur
+         stdDeviation="2"
+         id="feGaussianBlur4" />
+    </filter>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-0"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-5"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-1"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-0"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-4"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-9"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-11"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-2"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-0-1"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-5-6"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-1-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-0-1"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker8474-2"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Mend">
+      <path
+         inkscape:connector-curvature="0"
+         transform="scale(-0.6)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         id="path8472-4" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-0-9"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-5-7"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-1-38"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-0-6"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-1"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-6"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-22"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-1-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-6-4"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-7-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-22-7"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-0-9-1"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-5-7-6"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-1-38-0"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-0-6-0"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-1-2-5"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-6-4-6"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-7-7-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-22-7-9"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-0-9-1-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-5-7-6-8"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-1-38-0-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-0-6-0-8"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-0-9-1-4"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-5-7-6-2"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-1-38-0-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-0-6-0-5"
+         style="fill:#9cc3e5;fill-opacity:1;fill-rule:evenodd;stroke:#9cc3e5;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker8474-2-2"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Mend">
+      <path
+         inkscape:connector-curvature="0"
+         transform="scale(-0.6)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:#818181;fill-opacity:1;fill-rule:evenodd;stroke:#818181;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         id="path8472-4-6" />
+    </marker>
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="marker8474-3"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Mend">
+      <path
+         inkscape:connector-curvature="0"
+         transform="scale(-0.6)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:#818181;fill-opacity:1;fill-rule:evenodd;stroke:#818181;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         id="path8472-3" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-1"
+       style="overflow:visible"
+       inkscape:isstock="true"
+       inkscape:collect="always">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5143-7"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-0-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-5-77"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-1-39"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-0-9"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mstart"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mstart-8"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path5140-99"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker6509-78"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path6507-02"
+         style="fill:#a8d08d;fill-opacity:1;fill-rule:evenodd;stroke:#a8d08d;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.98994949"
+     inkscape:cx="213.35065"
+     inkscape:cy="360.88227"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:snap-text-baseline="true"
+     inkscape:window-width="1920"
+     inkscape:window-height="1137"
+     inkscape:window-x="1072"
+     inkscape:window-y="185"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:groupmode="layer"
+     id="layer3"
+     inkscape:label="bgImage"
+     style="display:inline"
+     transform="translate(-10.86151,-57.361626)" />
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     style="display:inline"
+     transform="translate(-10.86151,-57.361626)">
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="41.159508"
+       y="61.13464"
+       id="text873"><tspan
+         sodipodi:role="line"
+         id="tspan871"
+         x="41.159508"
+         y="64.996841"
+         style="font-size:4.23333311px;stroke-width:0.26458332" /></text>
+    <flowRoot
+       xml:space="preserve"
+       id="flowRoot4811"
+       style="font-style:normal;font-weight:normal;font-size:13.33333302px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none"
+       transform="scale(0.26458333)"><flowRegion
+         id="flowRegion4813"><rect
+           id="rect4815"
+           width="41.785713"
+           height="14.642858"
+           x="39.285713"
+           y="287.16254" /></flowRegion><flowPara
+         id="flowPara4817">offse</flowPara></flowRoot>    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="16.351753"
+       y="215.03786"
+       id="text4821-3-9-7-1-7"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0-3-0-0"
+         x="16.351753"
+         y="215.03786"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">offset</tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#d8d8d8;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect4735-7-3-8-5"
+       width="6.6797671"
+       height="14.033618"
+       x="13.480058"
+       y="65.465332" />
+    <g
+       style="display:inline"
+       id="g4807-4"
+       transform="translate(1.6626143,11.103676)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-8"
+         d="m 11.929873,70.237907 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-5"
+         d="m 18.686199,70.237907 c 0,0 -0.4016,2.480469 -1.370164,2.456847 -0.968564,-0.02363 -2.007999,-0.02363 -2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-5"
+         d="m 15.308036,72.671124 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <g
+       style="display:inline"
+       id="g5063-4"
+       transform="matrix(0.96955809,0,0,1.0002284,2.2280641,10.898039)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-0-01"
+         d="m 18.686199,70.426891 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-8-9"
+         d="m 132.59878,70.332401 c 0,0 -0.4016,2.480469 -1.37017,2.456847 -0.96856,-0.02363 -2.008,-0.02363 -2.008,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-2-0"
+         d="m 75.831475,72.954606 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4809-2"
+         d="M 22.064362,72.860108 129.22061,72.765618"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="74.602684"
+       y="85.144012"
+       id="text4821-3-0"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-5"
+         x="74.602684"
+         y="85.144012"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">length</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="13.386705"
+       y="85.690132"
+       id="text4821-3-9-4"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0-4"
+         x="13.386705"
+         y="85.690132"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">offset</tspan></text>
+    <g
+       transform="translate(209.08086,-15.131588)"
+       style="display:inline"
+       id="g10789-0">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4885-7"
+         d="m -198.21935,107.88646 h 61.73926"
+         style="fill:none;stroke:#7f7f7f;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:2.1199999, 0.26499999;stroke-dashoffset:0;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4887-5"
+         d="m -126.32385,107.88646 94.613422,-0.13363"
+         style="fill:none;stroke:#818181;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:2.1199999, 0.26499999;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="scale(0.9610099,1.040572)"
+         id="text4891-3"
+         y="105.46623"
+         x="-139.73984"
+         style="font-style:normal;font-weight:normal;font-size:5.39796209px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#818181;fill-opacity:1;stroke:none;stroke-width:0.40484715"
+         xml:space="preserve"><tspan
+           style="fill:#818181;fill-opacity:1;stroke-width:0.40484715"
+           y="105.46623"
+           x="-139.73984"
+           id="tspan4889-9"
+           sodipodi:role="line">or</tspan></text>
+    </g>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15867083;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6"
+       width="16.144258"
+       height="14.033618"
+       x="20.159824"
+       y="65.465332" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="25.127815"
+       y="73.846748"
+       id="text877-8-6"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8"
+         x="25.127815"
+         y="73.846748"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:64.99999762%;baseline-shift:sub"
+   id="tspan10851">1</tspan></tspan></text>
+    <g
+       id="g10891">
+      <rect
+         y="65.443756"
+         x="36.304085"
+         height="14.074809"
+         width="4.0010114"
+         id="rect6777-5-2-04-3"
+         style="opacity:1;fill:#0070c0;fill-opacity:1;stroke:#000000;stroke-width:0.15919298;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-1-7-5-6"
+         y="39.324684"
+         x="-78.343857"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="39.324684"
+           x="-78.343857"
+           id="tspan6779-7-1-0-4"
+           sodipodi:role="line">CRC24B</tspan></text>
+    </g>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6-1"
+       width="16.163868"
+       height="14.074809"
+       x="40.305096"
+       y="65.443756" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="44.893223"
+       y="73.780594"
+       id="text877-8-6-7"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8-8"
+         x="44.893223"
+         y="73.780594"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:64.99999762%;baseline-shift:sub"
+   id="tspan10935">2</tspan></tspan></text>
+    <g
+       transform="translate(20.164879)"
+       style="display:inline"
+       id="g10891-1">
+      <rect
+         y="65.443756"
+         x="36.304085"
+         height="14.074809"
+         width="4.0010114"
+         id="rect6777-5-2-04-3-5"
+         style="opacity:1;fill:#0070c0;fill-opacity:1;stroke:#000000;stroke-width:0.15919298;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-1-7-5-6-4"
+         y="39.324684"
+         x="-78.343857"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="39.324684"
+           x="-78.343857"
+           id="tspan6779-7-1-0-4-5"
+           sodipodi:role="line">CRC24B</tspan></text>
+    </g>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-8"
+       width="30.921177"
+       height="14.014396"
+       x="60.480572"
+       y="65.493568" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="73.468872"
+       y="73.892609"
+       id="text877-8-4"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-3"
+         x="73.468872"
+         y="73.892609"
+         style="font-size:3.52777767px;stroke-width:0.26458332">...</tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6-0"
+       width="16.163868"
+       height="14.074809"
+       x="91.401749"
+       y="65.433159" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="95.206711"
+       y="73.84742"
+       id="text877-8-6-1"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8-9"
+         x="95.206711"
+         y="73.84742"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;baseline-shift:sub;stroke-width:0.26458332"
+   id="tspan10851-4">c-1</tspan></tspan></text>
+    <g
+       transform="translate(71.261528,-0.01059723)"
+       style="display:inline"
+       id="g10891-1-1">
+      <rect
+         y="65.443756"
+         x="36.304085"
+         height="14.074809"
+         width="4.0010114"
+         id="rect6777-5-2-04-3-5-4"
+         style="opacity:1;fill:#0070c0;fill-opacity:1;stroke:#000000;stroke-width:0.15919298;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-1-7-5-6-4-0"
+         y="39.324684"
+         x="-78.343857"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="39.324684"
+           x="-78.343857"
+           id="tspan6779-7-1-0-4-5-4"
+           sodipodi:role="line">CRC24B</tspan></text>
+    </g>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6-0-8"
+       width="11.644219"
+       height="14.098742"
+       x="111.55466"
+       y="65.421196" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="114.16887"
+       y="73.780602"
+       id="text877-8-6-1-0"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8-9-9"
+         x="114.16887"
+         y="73.780602"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;baseline-shift:sub;stroke-width:0.26458332"
+   id="tspan10851-4-4">c</tspan></tspan></text>
+    <g
+       transform="translate(86.894791,0.00137329)"
+       style="display:inline"
+       id="g10891-1-1-1">
+      <rect
+         y="65.443756"
+         x="36.304085"
+         height="14.074809"
+         width="4.0010114"
+         id="rect6777-5-2-04-3-5-4-9"
+         style="opacity:1;fill:#0070c0;fill-opacity:1;stroke:#000000;stroke-width:0.15919298;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-1-7-5-6-4-0-4"
+         y="39.324684"
+         x="-78.343857"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="39.324684"
+           x="-78.343857"
+           id="tspan6779-7-1-0-4-5-4-7"
+           sodipodi:role="line">CRC24A</tspan></text>
+    </g>
+    <g
+       transform="translate(90.895802,0.00137329)"
+       style="display:inline"
+       id="g10891-1-1-6">
+      <rect
+         y="65.443756"
+         x="36.304085"
+         height="14.074809"
+         width="4.0010114"
+         id="rect6777-5-2-04-3-5-4-95"
+         style="opacity:1;fill:#0070c0;fill-opacity:1;stroke:#000000;stroke-width:0.15919298;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-1-7-5-6-4-0-6"
+         y="39.324684"
+         x="-78.343857"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="39.324684"
+           x="-78.343857"
+           id="tspan6779-7-1-0-4-5-4-2"
+           sodipodi:role="line">CRC24B</tspan></text>
+    </g>
+    <path
+       style="display:inline;fill:none;stroke:#9cc3e5;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart-1);marker-end:url(#marker6509-7)"
+       d="m 20.359726,62.756584 c 19.77795,-0.0668 19.77795,-0.0668 19.77795,-0.0668"
+       id="path6255-4"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="26.706127"
+       y="57.404415"
+       id="text4821-3-3-0-5-4"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3-95"
+         x="26.706127"
+         y="57.404415"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">k_neg</tspan></text>
+    <path
+       style="display:inline;fill:none;stroke:#9cc3e5;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart-0-9);marker-end:url(#marker6509-1-38)"
+       d="m 111.58503,62.795193 c 19.77795,-0.0668 19.77795,-0.0668 19.77795,-0.0668"
+       id="path6255-2-3"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="121.82767"
+       y="57.441448"
+       id="text4821-3-3-0-5-1-6"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3-7-9"
+         x="121.82767"
+         y="57.441448"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">k_pos</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#7f8085;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="144.51123"
+       y="64.972511"
+       id="text4821-3-6-0"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         x="144.51123"
+         y="64.972511"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan4877-6">- CRC24B &amp; CRC24A were pre-calculated</tspan><tspan
+         sodipodi:role="line"
+         x="144.51123"
+         y="68.908539"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan11806">by the application</tspan><tspan
+         sodipodi:role="line"
+         x="144.51123"
+         y="72.844559"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan11808">- The raw TB is given as a contiguous</tspan><tspan
+         sodipodi:role="line"
+         x="144.51123"
+         y="76.780586"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan11810">buffer</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#7f8085;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="143.92094"
+       y="97.043495"
+       id="text4821-3-6-0-2"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         x="143.92094"
+         y="97.043495"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan11810-4">- Only CRC24A was pre-calculated by the</tspan><tspan
+         sodipodi:role="line"
+         x="143.92094"
+         y="100.97952"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan11848">application, therefore</tspan><tspan
+         sodipodi:role="line"
+         x="143.92094"
+         y="104.91554"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan11850">RTE_BBDEV_TURBO_CRC_24B_ATTACH</tspan><tspan
+         sodipodi:role="line"
+         x="143.92094"
+         y="108.85157"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan11852">is set in op_flags</tspan><tspan
+         sodipodi:role="line"
+         x="143.92094"
+         y="112.78759"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan11854">- The raw TB is given as a contiguous</tspan><tspan
+         sodipodi:role="line"
+         x="143.92094"
+         y="116.72362"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan11856">buffer</tspan></text>
+    <g
+       style="display:inline"
+       id="g4807-4-6"
+       transform="translate(9.4628222,49.06356)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-8-0"
+         d="m 11.929873,70.237907 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-5-5"
+         d="m 18.686199,70.237907 c 0,0 -0.4016,2.480469 -1.370164,2.456847 -0.968564,-0.02363 -2.007999,-0.02363 -2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-5-6"
+         d="m 15.308036,72.671124 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="21.520061"
+       y="122.09515"
+       id="text4821-3-9-4-8"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0-4-4"
+         x="21.520061"
+         y="122.09515"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">offset</tspan></text>
+    <g
+       style="display:inline"
+       id="g5063-4-7"
+       transform="matrix(0.83046983,0,0,1.0013214,12.615148,48.778811)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-0-01-3"
+         d="m 18.686199,70.426891 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-8-9-4"
+         d="m 132.59878,70.332401 c 0,0 -0.4016,2.480469 -1.37017,2.456847 -0.96856,-0.02363 -2.008,-0.02363 -2.008,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-2-0-6"
+         d="m 75.831475,72.954606 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4809-2-9"
+         d="M 22.064362,72.860108 129.22061,72.765618"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="72.49247"
+       y="121.91208"
+       id="text4821-3-0-1"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-5-7"
+         x="72.49247"
+         y="121.91208"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">length</tspan></text>
+    <g
+       transform="translate(213.42759,24.366924)"
+       style="display:inline"
+       id="g10789-0-0">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4885-7-2"
+         d="m -198.21935,107.88646 h 61.73926"
+         style="fill:none;stroke:#7f7f7f;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:2.1199999, 0.26499999;stroke-dashoffset:0;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4887-5-0"
+         d="m -126.32385,107.88646 94.613422,-0.13363"
+         style="fill:none;stroke:#818181;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:2.1199999, 0.26499999;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="scale(0.9610099,1.040572)"
+         id="text4891-3-9"
+         y="105.46623"
+         x="-139.73984"
+         style="font-style:normal;font-weight:normal;font-size:5.39796209px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#818181;fill-opacity:1;stroke:none;stroke-width:0.40484715"
+         xml:space="preserve"><tspan
+           style="fill:#818181;fill-opacity:1;stroke-width:0.40484715"
+           y="105.46623"
+           x="-139.73984"
+           id="tspan4889-9-9"
+           sodipodi:role="line">or</tspan></text>
+    </g>
+    <rect
+       style="display:inline;opacity:1;fill:#d8d8d8;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect4735-7-3-8-5-5"
+       width="6.6797671"
+       height="14.033618"
+       x="20.985983"
+       y="103.46108" />
+    <path
+       style="display:inline;fill:none;stroke:#9cc3e5;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart-1-2);marker-end:url(#marker6509-7-7)"
+       d="m 28.597837,100.50577 c 19.777951,-0.0668 19.777951,-0.0668 19.777951,-0.0668"
+       id="path6255-4-7"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="35.296089"
+       y="93.607361"
+       id="text4821-3-3-0-5-4-7"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3-95-5"
+         x="35.296089"
+         y="93.607361"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">k_neg</tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6-3"
+       width="16.163868"
+       height="14.074809"
+       x="27.665751"
+       y="103.46108" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="32.131104"
+       y="111.79969"
+       id="text877-8-6-9"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8-0"
+         x="32.131104"
+         y="111.79969"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;baseline-shift:sub;stroke-width:0.26458332"
+   id="tspan10851-5">1</tspan></tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6-1-0"
+       width="16.163868"
+       height="14.074809"
+       x="43.82962"
+       y="103.46108" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="48.417747"
+       y="111.79792"
+       id="text877-8-6-7-1"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8-8-5"
+         x="48.417747"
+         y="111.79792"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;baseline-shift:sub;stroke-width:0.26458332"
+   id="tspan10935-8">2</tspan></tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-8-8"
+       width="30.921177"
+       height="14.014396"
+       x="59.993488"
+       y="103.46108" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="73.38308"
+       y="111.96056"
+       id="text877-8-4-6"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-3-9"
+         x="73.38308"
+         y="111.96056"
+         style="font-size:3.52777767px;stroke-width:0.26458332">...</tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6-0-7"
+       width="16.163868"
+       height="14.074809"
+       x="90.914665"
+       y="103.46108" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="94.719627"
+       y="111.87534"
+       id="text877-8-6-1-5"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8-9-4"
+         x="94.719627"
+         y="111.87534"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;baseline-shift:sub;stroke-width:0.26458332"
+   id="tspan10851-4-2">c-1</tspan></tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6-0-8-6"
+       width="12.207969"
+       height="14.095527"
+       x="107.08015"
+       y="103.43876" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="109.69276"
+       y="111.79655"
+       id="text877-8-6-1-0-1"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8-9-9-0"
+         x="109.69276"
+         y="111.79655"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;baseline-shift:sub;stroke-width:0.26458332"
+   id="tspan10851-4-4-5">c</tspan></tspan></text>
+    <g
+       transform="translate(82.984032,37.995003)"
+       style="display:inline"
+       id="g10891-1-1-1-9">
+      <rect
+         y="65.443756"
+         x="36.304085"
+         height="14.074809"
+         width="4.0010114"
+         id="rect6777-5-2-04-3-5-4-9-2"
+         style="opacity:1;fill:#0070c0;fill-opacity:1;stroke:#000000;stroke-width:0.15919298;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-1-7-5-6-4-0-4-0"
+         y="39.324684"
+         x="-78.343857"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="39.324684"
+           x="-78.343857"
+           id="tspan6779-7-1-0-4-5-4-7-9"
+           sodipodi:role="line">CRC24A</tspan></text>
+    </g>
+    <path
+       style="display:inline;fill:none;stroke:#9cc3e5;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart-0-9-1);marker-end:url(#marker6509-1-38-0)"
+       d="m 107.37813,100.63331 c 19.77795,-0.0668 19.77795,-0.0668 19.77795,-0.0668"
+       id="path6255-2-3-9"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="117.44109"
+       y="93.729691"
+       id="text4821-3-3-0-5-1-6-2"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3-7-9-7"
+         x="117.44109"
+         y="93.729691"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">k_pos</tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#d8d8d8;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect4735-7-3-8-5-5-2"
+       width="6.6797671"
+       height="14.033618"
+       x="13.230828"
+       y="152.20575" />
+    <path
+       style="display:inline;fill:none;stroke:#9cc3e5;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart-1-2-5);marker-end:url(#marker6509-7-7-6)"
+       d="m 20.312481,149.54973 c 19.777952,-0.0668 19.777952,-0.0668 19.777952,-0.0668"
+       id="path6255-4-7-7"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="26.656864"
+       y="140.64244"
+       id="text4821-3-3-0-5-4-7-0"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3-95-5-0"
+         x="26.656864"
+         y="140.64244"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">k_neg</tspan></text>
+    <path
+       style="display:inline;fill:none;stroke:#9cc3e5;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart-0-9-1-4);marker-end:url(#marker6509-1-38-0-7)"
+       d="m 114.65863,149.41609 c 19.77795,-0.0668 19.77795,-0.0668 19.77795,-0.0668"
+       id="path6255-2-3-9-5"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="125.03253"
+       y="140.5143"
+       id="text4821-3-3-0-5-1-6-2-0"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3-7-9-7-4"
+         x="125.03253"
+         y="140.5143"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">k_pos</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="26.321789"
+       y="132.12051"
+       id="text4821-3-9-0-4"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0-1-04"
+         x="26.321789"
+         y="132.12051"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">mbuf seg 1</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="111.52624"
+       y="132.07547"
+       id="text4821-3-9-0-8-6"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0-1-0-4"
+         x="111.52624"
+         y="132.07547"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">mbuf seg 2</tspan></text>
+    <path
+       style="display:inline;fill:none;stroke:#818181;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker8474-3)"
+       d="m 33.293244,139.67339 5.946743,5.7463"
+       id="path8464-83"
+       inkscape:connector-curvature="0" />
+    <path
+       style="display:inline;fill:none;stroke:#818181;stroke-width:0.26499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#marker8474-2-2)"
+       d="m 114.56953,140.74706 -5.7463,5.94675"
+       id="path8464-8-8"
+       inkscape:connector-curvature="0" />
+    <g
+       style="display:inline"
+       id="g4807-4-6-1"
+       transform="translate(1.5234255,97.894043)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-8-0-9"
+         d="m 11.929873,70.237907 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-5-5-9"
+         d="m 18.686199,70.237907 c 0,0 -0.4016,2.480469 -1.370164,2.456847 -0.968564,-0.02363 -2.007999,-0.02363 -2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-5-6-4"
+         d="m 15.308036,72.671124 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="13.241572"
+       y="168.92551"
+       id="text4821-3-9-4-8-3"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-0-4-4-2"
+         x="13.241572"
+         y="168.92551"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">offset</tspan></text>
+    <g
+       style="display:inline"
+       id="g5063-4-7-3"
+       transform="matrix(0.96708187,0,0,1.0002483,2.1378366,97.68752)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-0-01-3-5"
+         d="m 18.686199,70.426891 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-8-9-4-4"
+         d="m 132.59878,70.332401 c 0,0 -0.4016,2.480469 -1.37017,2.456847 -0.96856,-0.02363 -2.008,-0.02363 -2.008,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-2-0-6-6"
+         d="m 75.831475,72.954606 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4809-2-9-4"
+         d="M 22.064362,72.860108 129.22061,72.765618"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#9cc3e5;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="74.165497"
+       y="168.19868"
+       id="text4821-3-0-1-2"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-5-7-5"
+         x="74.165497"
+         y="168.19868"
+         style="fill:#9cc3e5;fill-opacity:1;stroke-width:0.23616144">length</tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6-3-0"
+       width="16.163868"
+       height="14.074809"
+       x="19.910595"
+       y="152.20575" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="24.37595"
+       y="160.54436"
+       id="text877-8-6-9-1"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8-0-4"
+         x="24.37595"
+         y="160.54436"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;baseline-shift:sub;stroke-width:0.26458332"
+   id="tspan10851-5-1">1</tspan></tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6-1-0-2"
+       width="16.163868"
+       height="14.074809"
+       x="36.074467"
+       y="152.20575" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="40.662594"
+       y="160.54259"
+       id="text877-8-6-7-1-7"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8-8-5-6"
+         x="40.662594"
+         y="160.54259"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;baseline-shift:sub;stroke-width:0.26458332"
+   id="tspan10935-8-3">2</tspan></tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-8-8-7"
+       width="8.9107389"
+       height="14.115565"
+       x="52.187752"
+       y="152.21559" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="54.513344"
+       y="160.1346"
+       id="text877-8-4-6-7"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-3-9-3"
+         x="54.513344"
+         y="160.1346"
+         style="font-size:3.52777767px;stroke-width:0.26458332">...</tspan></text>
+    <path
+       style="display:inline;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="m 61.098492,152.29471 9.985814,-0.03 v 4.04245 l -2.539064,1.46999 5.579254,2.87314 -3.00678,1.60362 v 4.04245 l -10.019224,0.0348 z"
+       id="path5575-6"
+       inkscape:connector-curvature="0" />
+    <path
+       style="display:inline;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="m 90.024487,152.16192 -9.953251,0.003 -0.0334,3.94222 -2.63928,1.63703 5.67947,2.87314 -2.93996,1.57021 v 4.00905 l 9.886431,0.003 z"
+       id="path5581-0"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="62.273087"
+       y="160.61116"
+       id="text5879-0"><tspan
+         sodipodi:role="line"
+         id="tspan5877-5"
+         x="62.273087"
+         y="160.61116"
+         style="stroke-width:0.26458332">CB<tspan
+   style="font-size:64.99999762%;baseline-shift:sub"
+   id="tspan15311">N</tspan></tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="83.078026"
+       y="160.61731"
+       id="text5883-5"><tspan
+         sodipodi:role="line"
+         id="tspan5881-6"
+         x="83.078026"
+         y="160.61731"
+         style="stroke-width:0.26458332">CB<tspan
+   style="font-size:64.99999762%;baseline-shift:sub"
+   id="tspan15313">N</tspan></tspan></text>
+    <path
+       style="display:inline;fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow2Mend-1)"
+       d="m 66.006182,152.19788 c 0,0 5.144934,-13.09621 18.942724,-0.3341"
+       id="path5899-9"
+       inkscape:connector-curvature="0" />
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-8-8-7-8"
+       width="8.9107389"
+       height="14.115566"
+       x="90.024498"
+       y="152.084" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="92.247215"
+       y="160.20142"
+       id="text877-8-4-6-7-9"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-3-9-3-4"
+         x="92.247215"
+         y="160.20142"
+         style="font-size:3.52777767px;stroke-width:0.26458332">...</tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6-3-0-3"
+       width="16.163868"
+       height="14.074809"
+       x="98.935234"
+       y="152.12476" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="102.77936"
+       y="160.27509"
+       id="text877-8-6-9-1-0"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8-0-4-7"
+         x="102.77936"
+         y="160.27509"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;baseline-shift:sub;stroke-width:0.26458332"
+   id="tspan10851-5-1-1">c-1</tspan></tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#9cc3e5;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect967-6-0-8-6-9"
+       width="12.207969"
+       height="14.095527"
+       x="115.09911"
+       y="152.10405" />
+    <g
+       transform="translate(91.002987,86.660285)"
+       style="display:inline"
+       id="g10891-1-1-1-9-9">
+      <rect
+         y="65.443756"
+         x="36.304085"
+         height="14.074809"
+         width="4.0010114"
+         id="rect6777-5-2-04-3-5-4-9-2-3"
+         style="opacity:1;fill:#0070c0;fill-opacity:1;stroke:#000000;stroke-width:0.15919298;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <text
+         transform="rotate(-90)"
+         id="text6781-1-7-5-6-4-0-4-0-6"
+         y="39.324684"
+         x="-78.343857"
+         style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-size:2.82222223px;fill:#ffffff;fill-opacity:1;stroke-width:0.26458332"
+           y="39.324684"
+           x="-78.343857"
+           id="tspan6779-7-1-0-4-5-4-7-9-7"
+           sodipodi:role="line">CRC24A</tspan></text>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.23333311px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="118.12826"
+       y="160.26109"
+       id="text877-8-6-1-0-1-8"><tspan
+         sodipodi:role="line"
+         id="tspan875-6-8-9-9-0-1"
+         x="118.12826"
+         y="160.26109"
+         style="font-size:3.52777767px;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;baseline-shift:sub;stroke-width:0.26458332"
+   id="tspan10851-4-4-5-0">c</tspan></tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#7f8085;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="144.46991"
+       y="144.41747"
+       id="text4821-3-6-0-2-7"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         x="144.46991"
+         y="144.41747"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan11856-1">- CRC24A was pre-calculated and </tspan><tspan
+         sodipodi:role="line"
+         x="144.46991"
+         y="148.35349"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan15485">RTE_BBDEV_TURBO_CRC_24B_ATTACH</tspan><tspan
+         sodipodi:role="line"
+         x="144.46991"
+         y="152.28952"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan15487">is set in op_flags</tspan><tspan
+         sodipodi:role="line"
+         x="144.46991"
+         y="156.22554"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan15489">- The raw TB is given as a &quot;scattered&quot;</tspan><tspan
+         sodipodi:role="line"
+         x="144.46991"
+         y="160.16156"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan15491">buffer through a chained mbuf</tspan></text>
+    <path
+       style="fill:#fec000;fill-opacity:1;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 73.879064,190.90497 -0.03341,-11.65962 h 4.209497 l -0.03341,11.65962 1.971111,-0.0334 -4.042449,3.90882 -4.04245,-3.90882 z"
+       id="path6066-6"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="-190.93837"
+       y="76.819031"
+       id="text6074-7"
+       transform="rotate(-90)"><tspan
+         sodipodi:role="line"
+         id="tspan6072-6"
+         x="-190.93837"
+         y="76.819031"
+         style="font-size:2.82222223px;stroke-width:0.26458332">encode</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="74.319527"
+       y="213.86391"
+       id="text4821-3-3-0-8"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-6"
+         x="74.319527"
+         y="213.86391"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">length</tspan></text>
+    <path
+       style="display:inline;fill:none;stroke:#a8d08d;stroke-width:0.25237256;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart-8);marker-end:url(#marker6509-78)"
+       d="m 22.905754,197.23808 c 15.904037,-0.0753 15.904037,-0.0753 15.904037,-0.0753"
+       id="path6255-22"
+       inkscape:connector-curvature="0" />
+    <g
+       style="display:inline"
+       id="g4807-5-2"
+       transform="translate(4.1511086,145.38078)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-6-72"
+         d="m 11.929873,70.237907 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-88-63"
+         d="m 18.686199,70.237907 c 0,0 -0.4016,2.480469 -1.370164,2.456847 -0.968564,-0.02363 -2.007999,-0.02363 -2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-9-0"
+         d="m 15.308036,72.671124 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <g
+       style="display:inline"
+       transform="matrix(0.9338915,0,0,1.0005032,5.402228,145.15927)"
+       id="g5063-5-2-4">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-0-0-0-2"
+         d="m 18.686199,70.426891 c 0,0 0.4016,2.480469 1.370164,2.456847 0.968564,-0.02363 2.007999,-0.02363 2.007999,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4741-5-8-2-1-6"
+         d="m 132.59878,70.332401 c 0,0 -0.4016,2.480469 -1.37017,2.456847 -0.96856,-0.02363 -2.008,-0.02363 -2.008,-0.02363"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4760-2-4-5-5"
+         d="m 75.831475,72.954606 -0.02362,2.527721"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4809-8-8-9"
+         d="M 22.064362,72.860108 129.22061,72.765618"
+         style="fill:none;stroke:#000000;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+    <rect
+       style="display:inline;opacity:1;fill:#d8d8d8;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       id="rect4735-7-3-3"
+       width="6.6797671"
+       height="14.033618"
+       x="15.753516"
+       y="199.83667" />
+    <rect
+       style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+       id="rect6210-97"
+       width="16.248745"
+       height="14.0336"
+       x="22.433283"
+       y="199.83669" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="30.577127"
+       y="208.42111"
+       id="text6214-8"><tspan
+         sodipodi:role="line"
+         id="tspan6212-4"
+         x="30.577127"
+         y="208.42111"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332">CB<tspan
+   style="font-size:64.99999762%;text-align:center;baseline-shift:sub;text-anchor:middle"
+   id="tspan6218-7">1</tspan></tspan><tspan
+         sodipodi:role="line"
+         x="30.577127"
+         y="212.83084"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+         id="tspan6216-3" /></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="29.919048"
+       y="186.11703"
+       id="text4821-3-3-0-5-7"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3-0"
+         x="29.919048"
+         y="186.11703"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">ea</tspan></text>
+    <rect
+       style="opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.15930426;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+       id="rect6693-01"
+       width="42.577148"
+       height="13.993384"
+       x="54.930775"
+       y="199.87691" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="74.028419"
+       y="208.31436"
+       id="text6697-5"><tspan
+         sodipodi:role="line"
+         id="tspan6695-3"
+         x="74.028419"
+         y="208.31436"
+         style="stroke-width:0.26458332">...</tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.15922768;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+       id="rect6210-8-1"
+       width="16.248745"
+       height="14.073822"
+       x="97.507919"
+       y="199.79646" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="105.80277"
+       y="207.61554"
+       id="text6214-78-1"><tspan
+         sodipodi:role="line"
+         id="tspan6212-2-2"
+         x="105.80277"
+         y="207.61554"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;text-align:center;baseline-shift:sub;text-anchor:middle;stroke-width:0.26458332"
+   id="tspan6218-3-0">c-1</tspan></tspan><tspan
+         sodipodi:role="line"
+         x="105.80277"
+         y="212.02527"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+         id="tspan6216-2-3" /></text>
+    <rect
+       style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.15929575;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+       id="rect6210-8-3-28"
+       width="15.999747"
+       height="14.073821"
+       x="113.75667"
+       y="199.79646" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="121.63119"
+       y="208.12714"
+       id="text6214-78-6-0"><tspan
+         sodipodi:role="line"
+         id="tspan6212-2-9-72"
+         x="121.63119"
+         y="208.12714"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332">CB<tspan
+   style="font-size:2.2930553px;text-align:center;baseline-shift:sub;text-anchor:middle;stroke-width:0.26458332"
+   id="tspan6218-3-7-8">c</tspan></tspan><tspan
+         sodipodi:role="line"
+         x="121.63119"
+         y="212.53687"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+         id="tspan6216-2-9-4" /></text>
+    <path
+       style="display:inline;fill:none;stroke:#a8d08d;stroke-width:0.25187415;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Mstart-0-6);marker-end:url(#marker6509-1-39)"
+       d="m 114.04452,197.18711 c 15.77071,-0.0757 15.77071,-0.0757 15.77071,-0.0757"
+       id="path6255-2-9"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#a8d08d;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="125.36855"
+       y="186.19617"
+       id="text4821-3-3-0-5-1-9"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         id="tspan4819-0-7-8-3-7-5"
+         x="125.36855"
+         y="186.19617"
+         style="fill:#a8d08d;fill-opacity:1;stroke-width:0.23616144">eb</tspan></text>
+    <rect
+       style="display:inline;opacity:1;fill:#a8d08d;fill-opacity:1;stroke:#000000;stroke-width:0.15899999;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+       id="rect6210-97-3"
+       width="16.248745"
+       height="14.0336"
+       x="38.68203"
+       y="199.83669" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.52777767px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;display:inline;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="46.825874"
+       y="208.42111"
+       id="text6214-8-2"><tspan
+         sodipodi:role="line"
+         id="tspan6212-4-7"
+         x="46.825874"
+         y="208.42111"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332">CB<tspan
+   style="font-size:64.99999762%;text-align:center;baseline-shift:sub;text-anchor:middle"
+   id="tspan6218-7-0">2</tspan></tspan><tspan
+         sodipodi:role="line"
+         x="46.825874"
+         y="212.83084"
+         style="text-align:center;text-anchor:middle;stroke-width:0.26458332"
+         id="tspan6216-3-5" /></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.14881921px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;display:inline;fill:#7f8085;fill-opacity:1;stroke:none;stroke-width:0.23616144"
+       x="145.06511"
+       y="197.72243"
+       id="text4821-3-6-0-2-7-2"
+       transform="scale(0.95903924,1.0427102)"><tspan
+         sodipodi:role="line"
+         x="145.06511"
+         y="197.72243"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan15491-4">Result is encoded back into the given</tspan><tspan
+         sodipodi:role="line"
+         x="145.06511"
+         y="201.65845"
+         style="fill:#7f8085;fill-opacity:1;stroke-width:0.23616144"
+         id="tspan16671">output mbuf as one contiguous buffer</tspan></text>
+  </g>
+</svg>
diff --git a/doc/guides/prog_guide/multi_proc_support.rst b/doc/guides/prog_guide/multi_proc_support.rst
index 46a00ec1..1384fe33 100644
--- a/doc/guides/prog_guide/multi_proc_support.rst
+++ b/doc/guides/prog_guide/multi_proc_support.rst
@@ -220,8 +220,8 @@ way communication mechanism, with the requester expecting a response from the
 other side.
 
 Both messages and requests will trigger a named callback on the receiver side.
-These callbacks will be called from within a dedicated IPC thread that is not
-part of EAL lcore threads.
+These callbacks will be called from within a dedicated IPC or interrupt thread
+that are not part of EAL lcore threads.
 
 Registering for incoming messages
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -280,6 +280,13 @@ For asynchronous requests, a function pointer to the callback function must be
 provided instead. This callback will be called when the request either has timed
 out, or will have received a response to all the messages that were sent.
 
+.. warning::
+
+    When an asynchronous request times out, the callback will be called not by
+    a dedicated IPC thread, but rather from EAL interrupt thread. Because of
+    this, it may not be possible for DPDK to trigger another interrupt-based
+    event (such as an alarm) while handling asynchronous IPC callback.
+
 When the callback is called, the original request descriptor will be provided
 (so that it would be possible to determine for which sent message this is a
 callback to), along with a response descriptor like the one described above.
@@ -311,6 +318,12 @@ supported. However, since sending messages (not requests) does not involve an
 IPC thread, sending messages while processing another message or request is
 supported.
 
+Asynchronous request callbacks may be triggered either from IPC thread or from
+interrupt thread, depending on whether the request has timed out. It is
+therefore suggested to avoid waiting for interrupt-based events (such as alarms)
+inside asynchronous IPC request callbacks. This limitation does not apply to
+messages or synchronous requests.
+
 If callbacks spend a long time processing the incoming requests, the requestor
 might time out, so setting the right timeout value on the requestor side is
 imperative.
diff --git a/doc/guides/prog_guide/poll_mode_drv.rst b/doc/guides/prog_guide/poll_mode_drv.rst
index af82352a..b2cf4835 100644
--- a/doc/guides/prog_guide/poll_mode_drv.rst
+++ b/doc/guides/prog_guide/poll_mode_drv.rst
@@ -328,11 +328,6 @@ A newly added offloads in ``[rt]x_conf->offloads`` to ``rte_eth_[rt]x_queue_setu
 is the one which hasn't been enabled in ``rte_eth_dev_configure()`` and is requested to be enabled
 in ``rte_eth_[rt]x_queue_setup()``. It must be per-queue type, otherwise trigger an error log.
 
-For an application to use the Tx offloads API it should set the ``ETH_TXQ_FLAGS_IGNORE`` flag in the ``txq_flags`` field located in ``rte_eth_txconf`` struct.
-In such cases it is not required to set other flags in ``txq_flags``.
-For an application to use the Rx offloads API it should set the ``ignore_offload_bitfield`` bit in the ``rte_eth_rxmode`` struct.
-In such cases it is not required to set other bitfield offloads in the ``rxmode`` struct.
-
 Poll Mode Driver API
 --------------------
 
@@ -365,6 +360,8 @@ Ethernet Device API
 
 The Ethernet device API exported by the Ethernet PMDs is described in the *DPDK API Reference*.
 
+.. _ethernet_device_standard_device_arguments:
+
 Ethernet Device Standard Device Arguments
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 1ce692ea..e2dbee31 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
 ABI and API Deprecation
 =======================
 
@@ -8,17 +11,22 @@ API and ABI deprecation notices are to be posted here.
 Deprecation Notices
 -------------------
 
-* eal: DPDK runtime configuration file (located at
-  ``/var/run/.<prefix>_config``) will be moved. The new path will be as follows:
-
-  - if DPDK is running as root, path will be set to
-    ``/var/run/dpdk/<prefix>/config``
-  - if DPDK is not running as root and $XDG_RUNTIME_DIR is set, path will be set
-    to ``$XDG_RUNTIME_DIR/dpdk/<prefix>/config``
-  - if DPDK is not running as root and $XDG_RUNTIME_DIR is not set, path will be
-    set to ``/tmp/dpdk/<prefix>/config``
-
-* eal: both declaring and identifying devices will be streamlined in v18.08.
+* eal: certain structures will change in EAL on account of upcoming external
+  memory support. Aside from internal changes leading to an ABI break, the
+  following externally visible changes will also be implemented:
+
+  - ``rte_memseg_list`` will change to include a boolean flag indicating
+    whether a particular memseg list is externally allocated. This will have
+    implications for any users of memseg-walk-related functions, as they will
+    now have to skip externally allocated segments in most cases if the intent
+    is to only iterate over internal DPDK memory.
+  - ``socket_id`` parameter across the entire DPDK will gain additional meaning,
+    as some socket ID's will now be representing externally allocated memory. No
+    changes will be required for existing code as backwards compatibility will
+    be kept, and those who do not use this feature will not see these extra
+    socket ID's.
+
+* eal: both declaring and identifying devices will be streamlined in v18.11.
   New functions will appear to query a specific port from buses, classes of
   device and device drivers. Device declaration will be made coherent with the
   new scheme of device identification.
@@ -28,7 +36,6 @@ Deprecation Notices
   - Functions previously deprecated will change or disappear:
 
     + ``rte_eal_devargs_type_count``
-    + ``rte_eal_devargs_parse`` will change its format and use.
 
 * pci: Several exposed functions are misnamed.
   The following functions are deprecated starting from v17.11 and are replaced:
@@ -37,15 +44,6 @@ Deprecation Notices
   - ``eal_parse_pci_DomBDF`` replaced by ``rte_pci_addr_parse``
   - ``rte_eal_compare_pci_addr`` replaced by ``rte_pci_addr_cmp``
 
-* eal: a new set of mbuf mempool ops name APIs for user, platform and best
-  mempool names have been defined in ``rte_mbuf`` in v18.02. The uses of
-  ``rte_eal_mbuf_default_mempool_ops`` shall be replaced by
-  ``rte_mbuf_best_mempool_ops``.
-  The following function is deprecated since 18.05, and will be removed
-  in 18.08:
-
-  - ``rte_eal_mbuf_default_mempool_ops``
-
 * mbuf: The opaque ``mbuf->hash.sched`` field will be updated to support generic
   definition in line with the ethdev TM and MTR APIs. Currently, this field
   is defined in librte_sched in a non-generic way. The new generic format
@@ -58,25 +56,6 @@ Deprecation Notices
   experimental API ``rte_pktmbuf_attach_extbuf()`` is used. Removal of the macro
   is to fix this semantic inconsistency.
 
-* ethdev: a new Tx and Rx offload API was introduced on 17.11.
-  In the new API, offloads are divided into per-port and per-queue offloads.
-  Offloads are disabled by default and enabled per application request.
-
-  In later releases the old offloading API will be deprecated, which will include:
-  - removal of ``ETH_TXQ_FLAGS_NO*`` flags.
-  - removal of ``txq_flags`` field from ``rte_eth_txconf`` struct.
-  - removal of the offloads bit-field from ``rte_eth_rxmode`` struct.
-
-* ethdev: A new offloading flag ``DEV_RX_OFFLOAD_KEEP_CRC`` will be added in v18.08,
-  This will replace the usage of not setting ``DEV_RX_OFFLOAD_CRC_STRIP`` flag
-  and will be implemented in PMDs accordingly.
-  In v18.08 both ``DEV_RX_OFFLOAD_KEEP_CRC`` and ``DEV_RX_OFFLOAD_CRC_STRIP`` flags
-  will be available, usage will be:
-  ``CRC_STRIP``: Strip CRC from packet
-  ``KEEP_CRC``: Keep CRC in packet
-  Both ``CRC_STRIP`` & ``KEEP_CRC``: Invalid
-  No flag: Keep CRC in packet
-
 * ethdev: In v18.11 ``DEV_RX_OFFLOAD_CRC_STRIP`` offload flag will be removed, default
   behavior without any flag will be changed to CRC strip.
   To keep CRC ``DEV_RX_OFFLOAD_KEEP_CRC`` flag is required.
@@ -91,6 +70,18 @@ Deprecation Notices
   Target release for removal of the legacy API will be defined once most
   PMDs have switched to rte_flow.
 
+* ethdev: In v18.11 ``rte_eth_dev_attach()`` and ``rte_eth_dev_detach()``
+  will be removed.
+  Hotplug functions ``rte_eal_hotplug_add()`` and ``rte_eal_hotplug_remove()``
+  should be used instread.
+  Function ``rte_eth_dev_get_port_by_name()`` may be used to find
+  identifier of the added port.
+
+* eal: In v18.11 ``rte_eal_dev_attach()`` and ``rte_eal_dev_detach()``
+  will be removed.
+  Hotplug functions ``rte_eal_hotplug_add()`` and ``rte_eal_hotplug_remove()``
+  should be used directly.
+
 * pdump: As we changed to use generic IPC, some changes in APIs and structure
   are expected in subsequent release.
 
@@ -98,27 +89,9 @@ Deprecation Notices
   - The parameter, ``path``, of ``rte_pdump_init`` will be removed;
   - The enum ``rte_pdump_socktype`` will be removed.
 
-* cryptodev: The following changes will be made in the library
-  for 18.08:
-
-  - Removal of ``sym`` structure in ``rte_cryptodev_info`` structure,
-    containing fields not relevant anymore since the session mempool
-    is not internal in the crypto device anymore.
-  - Replacement of ``pci_dev`` field with the more generic ``rte_device``
-    structure.
-  - Functions ``rte_cryptodev_queue_pair_attach_sym_session()`` and
-    ``rte_cryptodev_queue_pair_dettach_sym_session()`` will be deprecated from
-    18.05 and removed in 18.08, as there are no drivers doing anything useful
-    with them.
-  - Functions ``rte_cryptodev_queue_pair_start()`` and
-    ``rte_cryptodev_queue_pair_stop()`` will be deprecated from 18.05
-    and removed in 18.08, as there are no drivers doing anything useful
-    with them.
-  - Some feature flags such as ``RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER`` are ambiguous,
-    so some will be replaced by more explicit flags.
-  - Function ``rte_cryptodev_get_header_session_size()`` will be deprecated
-    in 18.05, and it gets replaced with ``rte_cryptodev_sym_get_header_session_size()``.
-    It will be removed in 18.08.
-  - Function ``rte_cryptodev_get_private_session_size()`` will be deprecated
-    in 18.05, and it gets replaced with ``rte_cryptodev_sym_get_private_session_size()``.
-    It will be removed in 18.08.
+* ethdev: flow API function ``rte_flow_copy()`` will be deprecated in v18.11
+  in favor of ``rte_flow_conv()`` (which will appear in that version) and
+  subsequently removed for v19.02.
+
+  This is due to a lack of flexibility and reliance on a type unusable with
+  C++ programs (struct rte_flow_desc).
diff --git a/doc/guides/rel_notes/index.rst b/doc/guides/rel_notes/index.rst
index eb82a0e0..d125342c 100644
--- a/doc/guides/rel_notes/index.rst
+++ b/doc/guides/rel_notes/index.rst
@@ -9,6 +9,7 @@ Release Notes
     :numbered:
 
     rel_description
+    release_18_08
     release_18_05
     release_18_02
     release_17_11
diff --git a/doc/guides/rel_notes/known_issues.rst b/doc/guides/rel_notes/known_issues.rst
index 716bd736..95e4ce69 100644
--- a/doc/guides/rel_notes/known_issues.rst
+++ b/doc/guides/rel_notes/known_issues.rst
@@ -739,3 +739,23 @@ Linux kernel 4.10.0 iommu attribute read error
 
 **Driver/Module**:
    ``vfio-pci`` module.
+
+Netvsc driver and application restart
+-------------------------------------
+
+**Description**:
+   The Linux kernel uio_hv_generic driver does not completely shutdown and clean up
+   resources properly if application using Netvsc PMD exits.
+
+**Implication**:
+   When application using Netvsc PMD is restarted it can not complete initialization
+   handshake sequence with the host.
+
+**Resolution/Workaround**:
+   Either reboot the guest or remove and reinsert the hv_uio_generic module.
+
+**Affected Environment/Platform**:
+   Linux Hyper-V.
+
+**Driver/Module**:
+   ``uio_hv_generic`` module.
diff --git a/doc/guides/rel_notes/release_16_04.rst b/doc/guides/rel_notes/release_16_04.rst
index d0a09eff..e9f1e6ff 100644
--- a/doc/guides/rel_notes/release_16_04.rst
+++ b/doc/guides/rel_notes/release_16_04.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2016 The DPDK contributors
+
 DPDK Release 16.04
 ==================
 
diff --git a/doc/guides/rel_notes/release_16_07.rst b/doc/guides/rel_notes/release_16_07.rst
index a8a3fc11..2904aacf 100644
--- a/doc/guides/rel_notes/release_16_07.rst
+++ b/doc/guides/rel_notes/release_16_07.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2016 The DPDK contributors
+
 DPDK Release 16.07
 ==================
 
diff --git a/doc/guides/rel_notes/release_16_11.rst b/doc/guides/rel_notes/release_16_11.rst
index 8c9ec65c..92e0ec69 100644
--- a/doc/guides/rel_notes/release_16_11.rst
+++ b/doc/guides/rel_notes/release_16_11.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2016 The DPDK contributors
+
 DPDK Release 16.11
 ==================
 
diff --git a/doc/guides/rel_notes/release_17_02.rst b/doc/guides/rel_notes/release_17_02.rst
index 357965ac..d6c1c567 100644
--- a/doc/guides/rel_notes/release_17_02.rst
+++ b/doc/guides/rel_notes/release_17_02.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2017 The DPDK contributors
+
 DPDK Release 17.02
 ==================
 
diff --git a/doc/guides/rel_notes/release_17_05.rst b/doc/guides/rel_notes/release_17_05.rst
index 68922840..64182406 100644
--- a/doc/guides/rel_notes/release_17_05.rst
+++ b/doc/guides/rel_notes/release_17_05.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2017 The DPDK contributors
+
 DPDK Release 17.05
 ==================
 
diff --git a/doc/guides/rel_notes/release_17_08.rst b/doc/guides/rel_notes/release_17_08.rst
index 0bcdfb7b..dc622409 100644
--- a/doc/guides/rel_notes/release_17_08.rst
+++ b/doc/guides/rel_notes/release_17_08.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2017 The DPDK contributors
+
 DPDK Release 17.08
 ==================
 
diff --git a/doc/guides/rel_notes/release_17_11.rst b/doc/guides/rel_notes/release_17_11.rst
index 5176d695..2a93af32 100644
--- a/doc/guides/rel_notes/release_17_11.rst
+++ b/doc/guides/rel_notes/release_17_11.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2017 The DPDK contributors
+
 DPDK Release 17.11
 ==================
 
diff --git a/doc/guides/rel_notes/release_18_02.rst b/doc/guides/rel_notes/release_18_02.rst
index 44b7de59..8e403118 100644
--- a/doc/guides/rel_notes/release_18_02.rst
+++ b/doc/guides/rel_notes/release_18_02.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
 DPDK Release 18.02
 ==================
 
diff --git a/doc/guides/rel_notes/release_18_05.rst b/doc/guides/rel_notes/release_18_05.rst
index 6b364932..8dc22b01 100644
--- a/doc/guides/rel_notes/release_18_05.rst
+++ b/doc/guides/rel_notes/release_18_05.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
 DPDK Release 18.05
 ==================
 
diff --git a/doc/guides/rel_notes/release_18_08.rst b/doc/guides/rel_notes/release_18_08.rst
new file mode 100644
index 00000000..321fa845
--- /dev/null
+++ b/doc/guides/rel_notes/release_18_08.rst
@@ -0,0 +1,549 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2018 The DPDK contributors
+
+DPDK Release 18.08
+==================
+
+.. **Read this first.**
+
+   The text in the sections below explains how to update the release notes.
+
+   Use proper spelling, capitalization and punctuation in all sections.
+
+   Variable and config names should be quoted as fixed width text:
+   ``LIKE_THIS``.
+
+   Build the docs and view the output file to ensure the changes are correct::
+
+      make doc-guides-html
+
+      xdg-open build/doc/html/guides/rel_notes/release_18_08.html
+
+
+New Features
+------------
+
+.. This section should contain new features added in this release.
+   Sample format:
+
+   * **Add a title in the past tense with a full stop.**
+
+     Add a short 1-2 sentence description in the past tense.
+     The description should be enough to allow someone scanning
+     the release notes to understand the new feature.
+
+     If the feature adds a lot of sub-features you can use a bullet list
+     like this:
+
+     * Added feature foo to do something.
+     * Enhanced feature bar to do something else.
+
+     Refer to the previous release notes for examples.
+
+     This section is a comment. Do not overwrite or remove it.
+     Also, make sure to start the actual text at the margin.
+     =========================================================
+
+* **Added support for Hyper-V netvsc PMD.**
+
+  The new ``netvsc`` poll mode driver provides native support for
+  networking on Hyper-V. See the :doc:`../nics/netvsc` NIC driver guide
+  for more details on this new driver.
+
+* **Added Flow API support for CXGBE PMD.**
+
+  Flow API support has been added to CXGBE Poll Mode Driver to offload
+  flows to Chelsio T5/T6 NICs. Support added for:
+
+  * Wildcard (LE-TCAM) and Exact (HASH) match filters.
+  * Match items: physical ingress port, IPv4, IPv6, TCP and UDP.
+  * Action items: queue, drop, count, and physical egress port redirect.
+
+* **Added ixgbe preferred Rx/Tx parameters.**
+
+  Rather than applications providing explicit Rx and Tx parameters such as
+  queue and burst sizes, they can request that the EAL instead uses preferred
+  values provided by the PMD, falling back to defaults within the EAL if the
+  PMD does not provide any. The provision of such tuned values now includes
+  the ixgbe PMD.
+
+* **Added descriptor status check support for fm10k.**
+
+  The ``rte_eth_rx_descriptor_status`` and ``rte_eth_tx_descriptor_status``
+  APIs are now supported by fm10K.
+
+* **Updated the enic driver.**
+
+  * Add low cycle count Tx handler for no-offload Tx.
+  * Add low cycle count Rx handler for non-scattered Rx.
+  * Minor performance improvements to scattered Rx handler.
+  * Add handlers to add/delete VxLAN port number.
+  * Add devarg to specify ingress VLAN rewrite mode.
+
+* **Updated mlx5 driver.**
+
+  Updated the mlx5 driver including the following changes:
+
+  * Added port representors support.
+  * Added Flow API support for e-switch rules.
+    Added support for ACTION_PORT_ID, ACTION_DROP, ACTION_OF_POP_VLAN,
+    ACTION_OF_PUSH_VLAN, ACTION_OF_SET_VLAN_VID, ACTION_OF_SET_VLAN_PCP
+    and ITEM_PORT_ID.
+  * Added support for 32-bit compilation.
+
+* **Added TSO support for the mlx4 driver.**
+
+  Added TSO support for the mlx4 drivers from MLNX_OFED_4.4 and above.
+
+* **SoftNIC PMD rework.**
+
+  The SoftNIC PMD infrastructure has been restructured to use the Packet
+  Framework, which makes it more flexible, modular and easier to add new
+  functionality in the future.
+
+* **Updated the AESNI MB PMD.**
+
+  The AESNI MB PMD has been updated with additional support for:
+
+  * 3DES for 8, 16 and 24 byte keys.
+
+* **Added a new compression PMD using Intel's QuickAssist (QAT) device family.**
+
+  Added the new ``QAT`` compression driver, for compression and decompression
+  operations in software. See the :doc:`../compressdevs/qat_comp` compression
+  driver guide for details on this new driver.
+
+* **Updated the ISA-L PMD.**
+
+  Added support for chained mbufs (input and output).
+
+
+API Changes
+-----------
+
+.. This section should contain API changes. Sample format:
+
+   * Add a short 1-2 sentence description of the API change.
+     Use fixed width quotes for ``function_names`` or ``struct_names``.
+     Use the past tense.
+
+   This section is a comment. Do not overwrite or remove it.
+   Also, make sure to start the actual text at the margin.
+   =========================================================
+
+* The path to the runtime config file has changed. The new path is determined
+  as follows:
+
+  - If DPDK is running as root, ``/var/run/dpdk/<prefix>/config``
+  - If DPDK is not running as root:
+
+    * If ``$XDG_RUNTIME_DIR`` is set, ``${XDG_RUNTIME_DIR}/dpdk/<prefix>/config``
+    * Otherwise, ``/tmp/dpdk/<prefix>/config``
+
+* eal: The function ``rte_eal_mbuf_default_mempool_ops`` was deprecated
+  and is removed in 18.08. It shall be replaced by
+  ``rte_mbuf_best_mempool_ops``.
+
+* mempool: Following functions were deprecated and are removed in 18.08:
+
+  - ``rte_mempool_populate_iova_tab``
+  - ``rte_mempool_populate_phys_tab``
+  - ``rte_mempool_populate_phys`` (``rte_mempool_populate_iova`` should be used)
+  - ``rte_mempool_virt2phy`` (``rte_mempool_virt2iova`` should be used)
+  - ``rte_mempool_xmem_create``
+  - ``rte_mempool_xmem_size``
+  - ``rte_mempool_xmem_usage``
+
+* ethdev: The old offload API is removed:
+
+  - Rx per-port ``rte_eth_conf.rxmode.[bit-fields]``
+  - Tx per-queue ``rte_eth_txconf.txq_flags``
+  - ``ETH_TXQ_FLAGS_NO*``
+
+  The transition bits are removed:
+
+  - ``rte_eth_conf.rxmode.ignore_offload_bitfield``
+  - ``ETH_TXQ_FLAGS_IGNORE``
+
+* cryptodev: The following API changes have been made in 18.08:
+
+  - In struct ``struct rte_cryptodev_info``, field ``rte_pci_device *pci_dev``
+    has been replaced with field ``struct rte_device *device``.
+  - Value 0 is accepted in ``sym.max_nb_sessions``, meaning that a device
+    supports an unlimited number of sessions.
+  - Two new fields of type ``uint16_t`` have been added:
+    ``min_mbuf_headroom_req`` and ``min_mbuf_tailroom_req``.  These parameters
+    specify the recommended headroom and tailroom for mbufs to be processed by
+    the PMD.
+
+* cryptodev: The following functions were deprecated and are removed in 18.08:
+
+  - ``rte_cryptodev_queue_pair_start``
+  - ``rte_cryptodev_queue_pair_stop``
+  - ``rte_cryptodev_queue_pair_attach_sym_session``
+  - ``rte_cryptodev_queue_pair_detach_sym_session``
+
+* cryptodev: The following functions were deprecated and are replaced by other
+  functions in 18.08:
+
+  - ``rte_cryptodev_get_header_session_size`` is replaced with
+    ``rte_cryptodev_sym_get_header_session_size``
+  - ``rte_cryptodev_get_private_session_size`` is replaced with
+    ``rte_cryptodev_sym_get_private_session_size``
+
+* cryptodev: Feature flag ``RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER`` is
+  replaced with the following more explicit flags:
+
+  - ``RTE_CRYPTODEV_FF_IN_PLACE_SGL``
+  - ``RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT``
+  - ``RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT``
+  - ``RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT``
+  - ``RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT``
+
+* cryptodev: Renamed cryptodev experimental APIs:
+
+  Used user_data instead of private_data in following APIs to avoid confusion
+  with the existing session parameter ``sess_private_data[]`` and related APIs.
+
+  - ``rte_cryptodev_sym_session_set_private_data()`` changed to
+    ``rte_cryptodev_sym_session_set_user_data()``
+  - ``rte_cryptodev_sym_session_get_private_data()`` changed to
+    ``rte_cryptodev_sym_session_get_user_data()``
+
+* compressdev: Feature flag ``RTE_COMP_FF_MBUF_SCATTER_GATHER`` is
+  replaced with the following more explicit flags:
+
+  - ``RTE_COMP_FF_OOP_SGL_IN_SGL_OUT``
+  - ``RTE_COMP_FF_OOP_SGL_IN_LB_OUT``
+  - ``RTE_COMP_FF_OOP_LB_IN_SGL_OUT``
+
+
+Shared Library Versions
+-----------------------
+
+.. Update any library version updated in this release
+   and prepend with a ``+`` sign, like this:
+
+     librte_acl.so.2
+   + librte_cfgfile.so.2
+     librte_cmdline.so.2
+
+   This section is a comment. Do not overwrite or remove it.
+   =========================================================
+
+The libraries prepended with a plus sign were incremented in this version.
+
+.. code-block:: diff
+
+     librte_acl.so.2
+     librte_bbdev.so.1
+     librte_bitratestats.so.2
+     librte_bpf.so.1
+     librte_bus_dpaa.so.1
+     librte_bus_fslmc.so.1
+     librte_bus_pci.so.1
+     librte_bus_vdev.so.1
+   + librte_bus_vmbus.so.1
+     librte_cfgfile.so.2
+     librte_cmdline.so.2
+     librte_common_octeontx.so.1
+     librte_compressdev.so.1
+   + librte_cryptodev.so.5
+     librte_distributor.so.1
+   + librte_eal.so.8
+   + librte_ethdev.so.10
+     librte_eventdev.so.4
+     librte_flow_classify.so.1
+     librte_gro.so.1
+     librte_gso.so.1
+     librte_hash.so.2
+     librte_ip_frag.so.1
+     librte_jobstats.so.1
+     librte_kni.so.2
+     librte_kvargs.so.1
+     librte_latencystats.so.1
+     librte_lpm.so.2
+     librte_mbuf.so.4
+   + librte_mempool.so.5
+     librte_meter.so.2
+     librte_metrics.so.1
+     librte_net.so.1
+     librte_pci.so.1
+     librte_pdump.so.2
+     librte_pipeline.so.3
+     librte_pmd_bnxt.so.2
+     librte_pmd_bond.so.2
+     librte_pmd_i40e.so.2
+     librte_pmd_ixgbe.so.2
+     librte_pmd_dpaa2_cmdif.so.1
+     librte_pmd_dpaa2_qdma.so.1
+     librte_pmd_ring.so.2
+     librte_pmd_softnic.so.1
+     librte_pmd_vhost.so.2
+     librte_port.so.3
+     librte_power.so.1
+     librte_rawdev.so.1
+     librte_reorder.so.1
+     librte_ring.so.2
+     librte_sched.so.1
+     librte_security.so.1
+     librte_table.so.3
+     librte_timer.so.1
+     librte_vhost.so.3
+
+
+Tested Platforms
+----------------
+
+.. This section should contain a list of platforms that were tested
+   with this release.
+
+   The format is:
+
+   * <vendor> platform with <vendor> <type of devices> combinations
+
+     * List of CPU
+     * List of OS
+     * List of devices
+     * Other relevant details...
+
+   This section is a comment. Do not overwrite or remove it.
+   Also, make sure to start the actual text at the margin.
+   =========================================================
+
+* Intel(R) platforms with Intel(R) NICs combinations
+
+   * CPU
+
+     * Intel(R) Atom(TM) CPU C3858 @ 2.00GHz
+     * Intel(R) Xeon(R) CPU D-1541 @ 2.10GHz
+     * Intel(R) Xeon(R) CPU E5-4667 v3 @ 2.00GHz
+     * Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz
+     * Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz
+     * Intel(R) Xeon(R) CPU E5-2695 v4 @ 2.10GHz
+     * Intel(R) Xeon(R) CPU E5-2658 v2 @ 2.40GHz
+     * Intel(R) Xeon(R) CPU E5-2658 v3 @ 2.20GHz
+     * Intel(R) Xeon(R) Platinum 8180 CPU @ 2.50GHz
+
+   * OS:
+
+     * CentOS 7.4
+     * Fedora 25
+     * Fedora 27
+     * Fedora 28
+     * FreeBSD 11.1
+     * Red Hat Enterprise Linux Server release 7.5
+     * SUSE Enterprise Linux 12
+     * Wind River Linux 8
+     * Ubuntu 14.04
+     * Ubuntu 16.04
+     * Ubuntu 16.10
+     * Ubuntu 17.10
+     * Ubuntu 18.04
+
+   * NICs:
+
+     * Intel(R) 82599ES 10 Gigabit Ethernet Controller
+
+       * Firmware version: 0x61bf0001
+       * Device id (pf/vf): 8086:10fb / 8086:10ed
+       * Driver version: 5.2.3 (ixgbe)
+
+     * Intel(R) Corporation Ethernet Connection X552/X557-AT 10GBASE-T
+
+       * Firmware version: 0x800003e7
+       * Device id (pf/vf): 8086:15ad / 8086:15a8
+       * Driver version: 4.4.6 (ixgbe)
+
+     * Intel(R) Ethernet Converged Network Adapter X710-DA4 (4x10G)
+
+       * Firmware version: 6.01 0x80003221
+       * Device id (pf/vf): 8086:1572 / 8086:154c
+       * Driver version: 2.4.6 (i40e)
+
+     * Intel Corporation Ethernet Connection X722 for 10GbE SFP+ (4x10G)
+
+       * Firmware version: 3.33 0x80000fd5 0.0.0
+       * Device id (pf/vf): 8086:37d0 / 8086:37cd
+       * Driver version: 2.4.3 (i40e)
+
+     * Intel(R) Ethernet Converged Network Adapter XXV710-DA2 (2x25G)
+
+       * Firmware version: 6.01 0x80003221
+       * Device id (pf/vf): 8086:158b / 8086:154c
+       * Driver version: 2.4.6 (i40e)
+
+     * Intel(R) Ethernet Converged Network Adapter XL710-QDA2 (2X40G)
+
+       * Firmware version: 6.01 0x8000321c
+       * Device id (pf/vf): 8086:1583 / 8086:154c
+       * Driver version: 2.4.6 (i40e)
+
+     * Intel(R) Corporation I350 Gigabit Network Connection
+
+       * Firmware version: 1.63, 0x80000dda
+       * Device id (pf/vf): 8086:1521 / 8086:1520
+       * Driver version: 5.4.0-k (igb)
+
+* Intel(R) platforms with Mellanox(R) NICs combinations
+
+   * CPU:
+
+     * Intel(R) Xeon(R) Gold 6154 CPU @ 3.00GHz
+     * Intel(R) Xeon(R) CPU E5-2697A v4 @ 2.60GHz
+     * Intel(R) Xeon(R) CPU E5-2697 v3 @ 2.60GHz
+     * Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz
+     * Intel(R) Xeon(R) CPU E5-2650 v4 @ 2.20GHz
+     * Intel(R) Xeon(R) CPU E5-2640 @ 2.50GHz
+     * Intel(R) Xeon(R) CPU E5-2620 v4 @ 2.10GHz
+
+   * OS:
+
+     * Red Hat Enterprise Linux Server release 7.5 (Maipo)
+     * Red Hat Enterprise Linux Server release 7.4 (Maipo)
+     * Red Hat Enterprise Linux Server release 7.3 (Maipo)
+     * Red Hat Enterprise Linux Server release 7.2 (Maipo)
+     * Ubuntu 18.04
+     * Ubuntu 17.10
+     * Ubuntu 16.04
+     * SUSE Linux Enterprise Server 15
+
+   * MLNX_OFED: 4.3-2.0.2.0
+   * MLNX_OFED: 4.4-2.0.1.0
+
+   * NICs:
+
+     * Mellanox(R) ConnectX(R)-3 Pro 40G MCX354A-FCC_Ax (2x40G)
+
+       * Host interface: PCI Express 3.0 x8
+       * Device ID: 15b3:1007
+       * Firmware version: 2.42.5000
+
+     * Mellanox(R) ConnectX(R)-4 10G MCX4111A-XCAT (1x10G)
+
+       * Host interface: PCI Express 3.0 x8
+       * Device ID: 15b3:1013
+       * Firmware version: 12.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 10G MCX4121A-XCAT (2x10G)
+
+       * Host interface: PCI Express 3.0 x8
+       * Device ID: 15b3:1013
+       * Firmware version: 12.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 25G MCX4111A-ACAT (1x25G)
+
+       * Host interface: PCI Express 3.0 x8
+       * Device ID: 15b3:1013
+       * Firmware version: 12.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 25G MCX4121A-ACAT (2x25G)
+
+       * Host interface: PCI Express 3.0 x8
+       * Device ID: 15b3:1013
+       * Firmware version: 12.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 40G MCX4131A-BCAT/MCX413A-BCAT (1x40G)
+
+       * Host interface: PCI Express 3.0 x8
+       * Device ID: 15b3:1013
+       * Firmware version: 12.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 40G MCX415A-BCAT (1x40G)
+
+       * Host interface: PCI Express 3.0 x16
+       * Device ID: 15b3:1013
+       * Firmware version: 12.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 50G MCX4131A-GCAT/MCX413A-GCAT (1x50G)
+
+       * Host interface: PCI Express 3.0 x8
+       * Device ID: 15b3:1013
+       * Firmware version: 12.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 50G MCX414A-BCAT (2x50G)
+
+       * Host interface: PCI Express 3.0 x8
+       * Device ID: 15b3:1013
+       * Firmware version: 12.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 50G MCX415A-GCAT/MCX416A-BCAT/MCX416A-GCAT (2x50G)
+
+       * Host interface: PCI Express 3.0 x16
+       * Device ID: 15b3:1013
+       * Firmware version: 12.21.1000 and above
+       * Firmware version: 12.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 50G MCX415A-CCAT (1x100G)
+
+       * Host interface: PCI Express 3.0 x16
+       * Device ID: 15b3:1013
+       * Firmware version: 12.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 100G MCX416A-CCAT (2x100G)
+
+       * Host interface: PCI Express 3.0 x16
+       * Device ID: 15b3:1013
+       * Firmware version: 12.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 Lx 10G MCX4121A-XCAT (2x10G)
+
+       * Host interface: PCI Express 3.0 x8
+       * Device ID: 15b3:1015
+       * Firmware version: 14.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-4 Lx 25G MCX4121A-ACAT (2x25G)
+
+       * Host interface: PCI Express 3.0 x8
+       * Device ID: 15b3:1015
+       * Firmware version: 14.21.1000 and above
+
+     * Mellanox(R) ConnectX(R)-5 100G MCX556A-ECAT (2x100G)
+
+       * Host interface: PCI Express 3.0 x16
+       * Device ID: 15b3:1017
+       * Firmware version: 16.21.1000 and above
+
+     * Mellanox(R) ConnectX-5 Ex EN 100G MCX516A-CDAT (2x100G)
+
+       * Host interface: PCI Express 4.0 x16
+       * Device ID: 15b3:1019
+       * Firmware version: 16.21.1000 and above
+
+* ARM platforms with Mellanox(R) NICs combinations
+
+   * CPU:
+
+     * Qualcomm ARM 1.1 2500MHz
+
+   * OS:
+
+     * Red Hat Enterprise Linux Server release 7.5 (Maipo)
+
+   * NICs:
+
+     * Mellanox(R) ConnectX(R)-4 Lx 25G MCX4121A-ACAT (2x25G)
+
+       * Host interface: PCI Express 3.0 x8
+       * Device ID: 15b3:1015
+       * Firmware version: 14.23.1000
+
+     * Mellanox(R) ConnectX(R)-5 100G MCX556A-ECAT (2x100G)
+
+       * Host interface: PCI Express 3.0 x16
+       * Device ID: 15b3:1017
+       * Firmware version: 16.23.1000
+
+* Mellanox BlueField SmartNIC
+
+   * Mellanox(R) BlueField SmartNIC MT416842 (2x25G)
+       * Host interface: PCI Express 3.0 x16
+       * Device ID: 15b3:a2d2
+       * Firmware version: 18.99.3950
+
+   * SoC ARM cores running OS:
+     * CentOS Linux release 7.4.1708 (AltArch)
+     * Mellanox MLNX_OFED 4.2-1.4.21.0
+
+  * DPDK application running on ARM cores inside SmartNIC
+  * Bluefield representors support planned for next release.
diff --git a/doc/guides/rel_notes/release_2_2.rst b/doc/guides/rel_notes/release_2_2.rst
index bb7d15a6..cea5c874 100644
--- a/doc/guides/rel_notes/release_2_2.rst
+++ b/doc/guides/rel_notes/release_2_2.rst
@@ -1,3 +1,6 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright 2016 The DPDK contributors
+
 DPDK Release 2.2
 ================
 
diff --git a/doc/guides/sample_app_ug/flow_filtering.rst b/doc/guides/sample_app_ug/flow_filtering.rst
index bbf8af0e..bd0ae1e2 100644
--- a/doc/guides/sample_app_ug/flow_filtering.rst
+++ b/doc/guides/sample_app_ug/flow_filtering.rst
@@ -139,7 +139,6 @@ application is shown below:
            struct rte_eth_conf port_conf = {
                    .rxmode = {
                            .split_hdr_size = 0,
-                           .ignore_offload_bitfield = 1,
                            .offloads = DEV_RX_OFFLOAD_CRC_STRIP,
                            },
                    .txmode = {
@@ -216,7 +215,6 @@ The Ethernet port is configured with default settings using the
    struct rte_eth_conf port_conf = {
            .rxmode = {
                    .split_hdr_size = 0,
-                   .ignore_offload_bitfield = 1,
                    .offloads = DEV_RX_OFFLOAD_CRC_STRIP,
                    },
            .txmode = {
diff --git a/doc/guides/sample_app_ug/img/master_slave_proc.png b/doc/guides/sample_app_ug/img/master_slave_proc.png
deleted file mode 100644
index a0065e85..00000000
--- a/doc/guides/sample_app_ug/img/master_slave_proc.png
+++ /dev/null
diff --git a/doc/guides/sample_app_ug/img/slave_proc_recov.png b/doc/guides/sample_app_ug/img/slave_proc_recov.png
deleted file mode 100644
index 62f1f088..00000000
--- a/doc/guides/sample_app_ug/img/slave_proc_recov.png
+++ /dev/null
diff --git a/doc/guides/sample_app_ug/index.rst b/doc/guides/sample_app_ug/index.rst
index 57e8354d..5bedf4f6 100644
--- a/doc/guides/sample_app_ug/index.rst
+++ b/doc/guides/sample_app_ug/index.rst
@@ -83,10 +83,6 @@ Sample Applications User Guides
 
 :numref:`figure_client_svr_sym_multi_proc_app` :ref:`figure_client_svr_sym_multi_proc_app`
 
-:numref:`figure_master_slave_proc` :ref:`figure_master_slave_proc`
-
-:numref:`figure_slave_proc_recov` :ref:`figure_slave_proc_recov`
-
 :numref:`figure_qos_sched_app_arch` :ref:`figure_qos_sched_app_arch`
 
 :numref:`figure_pipeline_overview` :ref:`figure_pipeline_overview`
diff --git a/doc/guides/sample_app_ug/ip_pipeline.rst b/doc/guides/sample_app_ug/ip_pipeline.rst
index 660798d5..b75509a0 100644
--- a/doc/guides/sample_app_ug/ip_pipeline.rst
+++ b/doc/guides/sample_app_ug/ip_pipeline.rst
@@ -222,6 +222,8 @@ Link
     txq <n_queues> <queue_size> promiscuous on | off
     [rss <qid_0> ... <qid_n>]
 
+ Note: The PCI device name must be specified in the Domain:Bus:Device.Function format.
+
 
 Mempool
 ~~~~~~~
diff --git a/doc/guides/sample_app_ug/kernel_nic_interface.rst b/doc/guides/sample_app_ug/kernel_nic_interface.rst
index fc110997..1b3ee9a5 100644
--- a/doc/guides/sample_app_ug/kernel_nic_interface.rst
+++ b/doc/guides/sample_app_ug/kernel_nic_interface.rst
@@ -202,74 +202,8 @@ Setup of mbuf pool, driver and queues is similar to the setup done in the :doc:`
 In addition, one or more kernel NIC interfaces are allocated for each
 of the configured ports according to the command line parameters.
 
-The code for allocating the kernel NIC interfaces for a specific port is as follows:
-
-.. code-block:: c
-
-    static int
-    kni_alloc(uint16_t port_id)
-    {
-        uint8_t i;
-        struct rte_kni *kni;
-        struct rte_kni_conf conf;
-        struct kni_port_params **params = kni_port_params_array;
-
-        if (port_id >= RTE_MAX_ETHPORTS || !params[port_id])
-            return -1;
-
-        params[port_id]->nb_kni = params[port_id]->nb_lcore_k ? params[port_id]->nb_lcore_k : 1;
-
-        for (i = 0; i < params[port_id]->nb_kni; i++) {
-
-            /* Clear conf at first */
-
-            memset(&conf, 0, sizeof(conf));
-            if (params[port_id]->nb_lcore_k) {
-                snprintf(conf.name, RTE_KNI_NAMESIZE, "vEth%u_%u", port_id, i);
-                conf.core_id = params[port_id]->lcore_k[i];
-                conf.force_bind = 1;
-            } else
-                snprintf(conf.name, RTE_KNI_NAMESIZE, "vEth%u", port_id);
-                conf.group_id = (uint16_t)port_id;
-                conf.mbuf_size = MAX_PACKET_SZ;
-
-                /*
-                 *   The first KNI device associated to a port
-                 *   is the master, for multiple kernel thread
-                 *   environment.
-                 */
-
-                if (i == 0) {
-                    struct rte_kni_ops ops;
-                    struct rte_eth_dev_info dev_info;
-
-                    memset(&dev_info, 0, sizeof(dev_info)); rte_eth_dev_info_get(port_id, &dev_info);
-
-                    conf.addr = dev_info.pci_dev->addr;
-                    conf.id = dev_info.pci_dev->id;
-
-                    /* Get the interface default mac address */
-                    rte_eth_macaddr_get(port_id, (struct ether_addr *)&conf.mac_addr);
-
-                    memset(&ops, 0, sizeof(ops));
-
-                    ops.port_id = port_id;
-                    ops.change_mtu = kni_change_mtu;
-                    ops.config_network_if = kni_config_network_interface;
-                    ops.config_mac_address = kni_config_mac_address;
-
-                    kni = rte_kni_alloc(pktmbuf_pool, &conf, &ops);
-                } else
-                    kni = rte_kni_alloc(pktmbuf_pool, &conf, NULL);
-
-                if (!kni)
-                    rte_exit(EXIT_FAILURE, "Fail to create kni for "
-                            "port: %d\n", port_id);
-
-                params[port_id]->kni[i] = kni;
-            }
-        return 0;
-   }
+The code for allocating the kernel NIC interfaces for a specific port is
+in the function ``kni_alloc``.
 
 The other step in the initialization process that is unique to this sample application
 is the association of each port with lcores for RX, TX and kernel threads.
@@ -280,105 +214,8 @@ is the association of each port with lcores for RX, TX and kernel threads.
 
 *   Other lcores for pinning the kernel threads on one by one
 
-This is done by using the`kni_port_params_array[]` array, which is indexed by the port ID.
-The code is as follows:
-
-.. code-block:: console
-
-    static int
-    parse_config(const char *arg)
-    {
-        const char *p, *p0 = arg;
-        char s[256], *end;
-        unsigned size;
-        enum fieldnames {
-            FLD_PORT = 0,
-            FLD_LCORE_RX,
-            FLD_LCORE_TX,
-            _NUM_FLD = KNI_MAX_KTHREAD + 3,
-        };
-        int i, j, nb_token;
-        char *str_fld[_NUM_FLD];
-        unsigned long int_fld[_NUM_FLD];
-        uint16_t port_id, nb_kni_port_params = 0;
-
-        memset(&kni_port_params_array, 0, sizeof(kni_port_params_array));
-
-        while (((p = strchr(p0, '(')) != NULL) && nb_kni_port_params < RTE_MAX_ETHPORTS) {
-            p++;
-            if ((p0 = strchr(p, ')')) == NULL)
-                goto fail;
-
-            size = p0 - p;
-
-            if (size >= sizeof(s)) {
-                printf("Invalid config parameters\n");
-                goto fail;
-            }
-
-            snprintf(s, sizeof(s), "%.*s", size, p);
-            nb_token = rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',');
-
-            if (nb_token <= FLD_LCORE_TX) {
-                printf("Invalid config parameters\n");
-                goto fail;
-            }
-
-            for (i = 0; i < nb_token; i++) {
-                errno = 0;
-                int_fld[i] = strtoul(str_fld[i], &end, 0);
-                if (errno != 0 || end == str_fld[i]) {
-                    printf("Invalid config parameters\n");
-                    goto fail;
-                }
-            }
-
-            i = 0;
-            port_id = (uint8_t)int_fld[i++];
-
-            if (port_id >= RTE_MAX_ETHPORTS) {
-                printf("Port ID %u could not exceed the maximum %u\n", port_id, RTE_MAX_ETHPORTS);
-                goto fail;
-            }
-
-            if (kni_port_params_array[port_id]) {
-                printf("Port %u has been configured\n", port_id);
-                goto fail;
-            }
-
-            kni_port_params_array[port_id] = (struct kni_port_params*)rte_zmalloc("KNI_port_params", sizeof(struct kni_port_params), RTE_CACHE_LINE_SIZE);
-            kni_port_params_array[port_id]->port_id = port_id;
-            kni_port_params_array[port_id]->lcore_rx = (uint8_t)int_fld[i++];
-            kni_port_params_array[port_id]->lcore_tx = (uint8_t)int_fld[i++];
-
-            if (kni_port_params_array[port_id]->lcore_rx >= RTE_MAX_LCORE || kni_port_params_array[port_id]->lcore_tx >= RTE_MAX_LCORE) {
-                printf("lcore_rx %u or lcore_tx %u ID could not "
-                        "exceed the maximum %u\n",
-                        kni_port_params_array[port_id]->lcore_rx, kni_port_params_array[port_id]->lcore_tx, RTE_MAX_LCORE);
-                goto fail;
-           }
-
-        for (j = 0; i < nb_token && j < KNI_MAX_KTHREAD; i++, j++)
-            kni_port_params_array[port_id]->lcore_k[j] = (uint8_t)int_fld[i];
-            kni_port_params_array[port_id]->nb_lcore_k = j;
-        }
-
-        print_config();
-
-        return 0;
-
-    fail:
-
-        for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
-            if (kni_port_params_array[i]) {
-                rte_free(kni_port_params_array[i]);
-                kni_port_params_array[i] = NULL;
-            }
-        }
-
-        return -1;
-
-    }
+This is done by using the ``kni_port_params_array[]`` array, which is indexed by the port ID.
+The code is in the function ``parse_config``.
 
 Packet Forwarding
 ~~~~~~~~~~~~~~~~~
@@ -387,99 +224,18 @@ After the initialization steps are completed, the main_loop() function is run on
 This function first checks the lcore_id against the user provided lcore_rx and lcore_tx
 to see if this lcore is reading from or writing to kernel NIC interfaces.
 
-For the case that reads from a NIC port and writes to the kernel NIC interfaces,
+For the case that reads from a NIC port and writes to the kernel NIC interfaces (``kni_ingress``),
 the packet reception is the same as in L2 Forwarding sample application
 (see :ref:`l2_fwd_app_rx_tx_packets`).
 The packet transmission is done by sending mbufs into the kernel NIC interfaces by rte_kni_tx_burst().
 The KNI library automatically frees the mbufs after the kernel successfully copied the mbufs.
 
-.. code-block:: c
-
-    /**
-     *   Interface to burst rx and enqueue mbufs into rx_q
-     */
-
-    static void
-    kni_ingress(struct kni_port_params *p)
-    {
-        uint8_t i, nb_kni, port_id;
-        unsigned nb_rx, num;
-        struct rte_mbuf *pkts_burst[PKT_BURST_SZ];
-
-        if (p == NULL)
-            return;
-
-        nb_kni = p->nb_kni;
-        port_id = p->port_id;
-
-        for (i = 0; i < nb_kni; i++) {
-            /* Burst rx from eth */
-            nb_rx = rte_eth_rx_burst(port_id, 0, pkts_burst, PKT_BURST_SZ);
-            if (unlikely(nb_rx > PKT_BURST_SZ)) {
-                RTE_LOG(ERR, APP, "Error receiving from eth\n");
-                return;
-            }
-
-            /* Burst tx to kni */
-            num = rte_kni_tx_burst(p->kni[i], pkts_burst, nb_rx);
-            kni_stats[port_id].rx_packets += num;
-            rte_kni_handle_request(p->kni[i]);
-
-            if (unlikely(num < nb_rx)) {
-                /* Free mbufs not tx to kni interface */
-                kni_burst_free_mbufs(&pkts_burst[num], nb_rx - num);
-                kni_stats[port_id].rx_dropped += nb_rx - num;
-            }
-        }
-    }
-
-For the other case that reads from kernel NIC interfaces and writes to a physical NIC port, packets are retrieved by reading
-mbufs from kernel NIC interfaces by `rte_kni_rx_burst()`.
+For the other case that reads from kernel NIC interfaces
+and writes to a physical NIC port (``kni_egress``),
+packets are retrieved by reading mbufs from kernel NIC interfaces by ``rte_kni_rx_burst()``.
 The packet transmission is the same as in the L2 Forwarding sample application
 (see :ref:`l2_fwd_app_rx_tx_packets`).
 
-.. code-block:: c
-
-    /**
-     *   Interface to dequeue mbufs from tx_q and burst tx
-     */
-
-    static void
-
-    kni_egress(struct kni_port_params *p)
-    {
-        uint8_t i, nb_kni, port_id;
-        unsigned nb_tx, num;
-        struct rte_mbuf *pkts_burst[PKT_BURST_SZ];
-
-        if (p == NULL)
-            return;
-
-        nb_kni = p->nb_kni;
-        port_id = p->port_id;
-
-        for (i = 0; i < nb_kni; i++) {
-            /* Burst rx from kni */
-            num = rte_kni_rx_burst(p->kni[i], pkts_burst, PKT_BURST_SZ);
-            if (unlikely(num > PKT_BURST_SZ)) {
-                RTE_LOG(ERR, APP, "Error receiving from KNI\n");
-                return;
-            }
-
-            /* Burst tx to eth */
-
-            nb_tx = rte_eth_tx_burst(port_id, 0, pkts_burst, (uint16_t)num);
-
-            kni_stats[port_id].tx_packets += nb_tx;
-
-            if (unlikely(nb_tx < num)) {
-                /* Free mbufs not tx to NIC */
-                kni_burst_free_mbufs(&pkts_burst[nb_tx], num - nb_tx);
-                kni_stats[port_id].tx_dropped += num - nb_tx;
-            }
-        }
-    }
-
 Callbacks for Kernel Requests
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -492,96 +248,3 @@ Application may choose to not implement following callbacks:
 
 - ``config_mac_address``
 - ``config_promiscusity``
-
-
-.. code-block:: c
-
-    static struct rte_kni_ops kni_ops = {
-        .change_mtu = kni_change_mtu,
-        .config_network_if = kni_config_network_interface,
-        .config_mac_address = kni_config_mac_address,
-        .config_promiscusity = kni_config_promiscusity,
-    };
-
-    /* Callback for request of changing MTU */
-
-    static int
-    kni_change_mtu(uint16_t port_id, unsigned new_mtu)
-    {
-        int ret;
-        struct rte_eth_conf conf;
-
-        RTE_LOG(INFO, APP, "Change MTU of port %d to %u\n", port_id, new_mtu);
-
-        /* Stop specific port */
-
-        rte_eth_dev_stop(port_id);
-
-        memcpy(&conf, &port_conf, sizeof(conf));
-
-        /* Set new MTU */
-
-        if (new_mtu > ETHER_MAX_LEN)
-            conf.rxmode.jumbo_frame = 1;
-        else
-            conf.rxmode.jumbo_frame = 0;
-
-        /* mtu + length of header + length of FCS = max pkt length */
-
-        conf.rxmode.max_rx_pkt_len = new_mtu + KNI_ENET_HEADER_SIZE + KNI_ENET_FCS_SIZE;
-
-        ret = rte_eth_dev_configure(port_id, 1, 1, &conf);
-        if (ret < 0) {
-            RTE_LOG(ERR, APP, "Fail to reconfigure port %d\n", port_id);
-            return ret;
-        }
-
-        /* Restart specific port */
-
-        ret = rte_eth_dev_start(port_id);
-        if (ret < 0) {
-             RTE_LOG(ERR, APP, "Fail to restart port %d\n", port_id);
-            return ret;
-        }
-
-        return 0;
-    }
-
-    /* Callback for request of configuring network interface up/down */
-
-    static int
-    kni_config_network_interface(uint16_t port_id, uint8_t if_up)
-    {
-        int ret = 0;
-
-        RTE_LOG(INFO, APP, "Configure network interface of %d %s\n",
-
-        port_id, if_up ? "up" : "down");
-
-        if (if_up != 0) {
-            /* Configure network interface up */
-            rte_eth_dev_stop(port_id);
-            ret = rte_eth_dev_start(port_id);
-        } else /* Configure network interface down */
-            rte_eth_dev_stop(port_id);
-
-        if (ret < 0)
-            RTE_LOG(ERR, APP, "Failed to start port %d\n", port_id);
-        return ret;
-    }
-
-    /* Callback for request of configuring device mac address */
-
-    static int
-    kni_config_mac_address(uint16_t port_id, uint8_t mac_addr[])
-    {
-        .....
-    }
-
-    /* Callback for request of configuring promiscuous mode */
-
-    static int
-    kni_config_promiscusity(uint16_t port_id, uint8_t to_on)
-    {
-        .....
-    }
diff --git a/doc/guides/sample_app_ug/l2_forward_job_stats.rst b/doc/guides/sample_app_ug/l2_forward_job_stats.rst
index d7219a29..ba73d855 100644
--- a/doc/guides/sample_app_ug/l2_forward_job_stats.rst
+++ b/doc/guides/sample_app_ug/l2_forward_job_stats.rst
@@ -218,25 +218,6 @@ The rte_eth_dev_configure() function is used to configure the number of queues f
             "err=%d, port=%u\n",
             ret, portid);
 
-The global configuration is stored in a static structure:
-
-.. code-block:: c
-
-    static const struct rte_eth_conf port_conf = {
-        .rxmode = {
-            .split_hdr_size = 0,
-            .header_split = 0,   /**< Header Split disabled */
-            .hw_ip_checksum = 0, /**< IP checksum offload disabled */
-            .hw_vlan_filter = 0, /**< VLAN filtering disabled */
-            .jumbo_frame = 0,    /**< Jumbo Frame Support disabled */
-            .hw_strip_crc= 0,    /**< CRC stripped by hardware */
-        },
-
-        .txmode = {
-            .mq_mode = ETH_DCB_NONE
-        },
-    };
-
 RX Queue Initialization
 ~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/guides/sample_app_ug/l2_forward_real_virtual.rst b/doc/guides/sample_app_ug/l2_forward_real_virtual.rst
index b459d4ef..2b2d5afa 100644
--- a/doc/guides/sample_app_ug/l2_forward_real_virtual.rst
+++ b/doc/guides/sample_app_ug/l2_forward_real_virtual.rst
@@ -245,25 +245,6 @@ The rte_eth_dev_configure() function is used to configure the number of queues f
             "err=%d, port=%u\n",
             ret, portid);
 
-The global configuration is stored in a static structure:
-
-.. code-block:: c
-
-    static const struct rte_eth_conf port_conf = {
-        .rxmode = {
-            .split_hdr_size = 0,
-            .header_split = 0,   /**< Header Split disabled */
-            .hw_ip_checksum = 0, /**< IP checksum offload disabled */
-            .hw_vlan_filter = 0, /**< VLAN filtering disabled */
-            .jumbo_frame = 0,    /**< Jumbo Frame Support disabled */
-            .hw_strip_crc= 0,    /**< CRC stripped by hardware */
-        },
-
-        .txmode = {
-            .mq_mode = ETH_DCB_NONE
-        },
-    };
-
 .. _l2_fwd_app_rx_init:
 
 RX Queue Initialization
diff --git a/doc/guides/sample_app_ug/link_status_intr.rst b/doc/guides/sample_app_ug/link_status_intr.rst
index 8c11ba4a..c7665fe5 100644
--- a/doc/guides/sample_app_ug/link_status_intr.rst
+++ b/doc/guides/sample_app_ug/link_status_intr.rst
@@ -137,10 +137,7 @@ The global configuration is stored in a static structure:
     static const struct rte_eth_conf port_conf = {
         .rxmode = {
             .split_hdr_size = 0,
-            .header_split = 0,   /**< Header Split disabled */
-            .hw_ip_checksum = 0, /**< IP checksum offload disabled */
-            .hw_vlan_filter = 0, /**< VLAN filtering disabled */
-            .hw_strip_crc= 0,    /**< CRC stripped by hardware */
+            .offloads = DEV_RX_OFFLOAD_CRC_STRIP,
         },
         .txmode = {},
         .intr_conf = {
diff --git a/doc/guides/sample_app_ug/multi_process.rst b/doc/guides/sample_app_ug/multi_process.rst
index b84e9a74..9c374da6 100644
--- a/doc/guides/sample_app_ug/multi_process.rst
+++ b/doc/guides/sample_app_ug/multi_process.rst
@@ -321,409 +321,3 @@ In both the server and the client processes, outgoing packets are buffered befor
 so as to allow the sending of multiple packets in a single burst to improve efficiency.
 For example, the client process will buffer packets to send,
 until either the buffer is full or until we receive no further packets from the server.
-
-Master-slave Multi-process Example
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The fourth example of DPDK multi-process support demonstrates a master-slave model that
-provide the capability of application recovery if a slave process crashes or  meets unexpected conditions.
-In addition, it also demonstrates the floating process,
-which can run among different cores in contrast to the traditional way of binding a process/thread to a specific CPU core,
-using the local cache mechanism of mempool structures.
-
-This application performs the same functionality as the L2 Forwarding sample application,
-therefore this chapter does not cover that part but describes functionality that is introduced in this multi-process example only.
-Please refer to :doc:`l2_forward_real_virtual` for more information.
-
-Unlike previous examples where all processes are started from the command line with input arguments, in this example,
-only one process is spawned from the command line and that process creates other processes.
-The following section describes this in more detail.
-
-Master-slave Process Models
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The process spawned from the command line is called the *master process* in this document.
-A process created by the master is called a *slave process*.
-The application has only one master process, but could have multiple slave processes.
-
-Once the master process begins to run, it tries to initialize all the resources such as
-memory, CPU cores, driver, ports, and so on, as the other examples do.
-Thereafter, it creates slave processes, as shown in the following figure.
-
-.. _figure_master_slave_proc:
-
-.. figure:: img/master_slave_proc.*
-
-   Master-slave Process Workflow
-
-
-The master process calls the rte_eal_mp_remote_launch() EAL function to launch an application function for each pinned thread through the pipe.
-Then, it waits to check if any slave processes have exited.
-If so, the process tries to re-initialize the resources that belong to that slave and launch them in the pinned thread entry again.
-The following section describes the recovery procedures in more detail.
-
-For each pinned thread in EAL, after reading any data from the pipe, it tries to call the function that the application specified.
-In this master specified function, a fork() call creates a slave process that performs the L2 forwarding task.
-Then, the function waits until the slave exits, is killed or crashes. Thereafter, it notifies the master of this event and returns.
-Finally, the EAL pinned thread waits until the new function is launched.
-
-After discussing the master-slave model, it is necessary to mention another issue, global and static variables.
-
-For multiple-thread cases, all global and static variables have only one copy and they can be accessed by any thread if applicable.
-So, they can be used to sync or share data among threads.
-
-In the previous examples, each process has separate global and static variables in memory and are independent of each other.
-If it is necessary to share the knowledge, some communication mechanism should be deployed, such as, memzone, ring, shared memory, and so on.
-The global or static variables are not a valid approach to share data among processes.
-For variables in this example, on the one hand, the slave process inherits all the knowledge of these variables after being created by the master.
-On the other hand, other processes cannot know if one or more processes modifies them after slave creation since that
-is the nature of a multiple process address space.
-But this does not mean that these variables cannot be used to share or sync data; it depends on the use case.
-The following are the possible use cases:
-
-#.  The master process starts and initializes a variable and it will never be changed after slave processes created. This case is OK.
-
-#.  After the slave processes are created, the master or slave cores need to change a variable, but other processes do not need to know the change.
-    This case is also OK.
-
-#.  After the slave processes are created, the master or a slave needs to change a variable.
-    In the meantime, one or more other process needs to be aware of the change.
-    In this case, global and static variables cannot be used to share knowledge. Another communication mechanism is needed.
-    A simple approach without lock protection can be a heap buffer allocated by rte_malloc or mem zone.
-
-Slave Process Recovery Mechanism
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Before talking about the recovery mechanism, it is necessary to know what is needed before a new slave instance can run if a previous one exited.
-
-When a slave process exits, the system returns all the resources allocated for this process automatically.
-However, this does not include the resources that were allocated by the DPDK. All the hardware resources are shared among the processes,
-which include memzone, mempool, ring, a heap buffer allocated by the rte_malloc library, and so on.
-If the new instance runs and the allocated resource is not returned, either resource allocation failed or the hardware resource is lost forever.
-
-When a slave process runs, it may have dependencies on other processes.
-They could have execution sequence orders; they could share the ring to communicate; they could share the same port for reception and forwarding;
-they could use lock structures to do exclusive access in some critical path.
-What happens to the dependent process(es) if the peer leaves?
-The consequence are varied since the dependency cases are complex.
-It depends on what the processed had shared.
-However, it is necessary to notify the peer(s) if one slave exited.
-Then, the peer(s) will be aware of that and wait until the new instance begins to run.
-
-Therefore, to provide the capability to resume the new slave instance if the previous one exited, it is necessary to provide several mechanisms:
-
-#.  Keep a resource list for each slave process.
-    Before a slave process run, the master should prepare a resource list.
-    After it exits, the master could either delete the allocated resources and create new ones,
-    or re-initialize those for use by the new instance.
-
-#.  Set up a notification mechanism for slave process exit cases. After the specific slave leaves,
-    the master should be notified and then help to create a new instance.
-    This mechanism is provided in Section `Master-slave Process Models`_.
-
-#.  Use a synchronization mechanism among dependent processes.
-    The master should have the capability to stop or kill slave processes that have a dependency on the one that has exited.
-    Then, after the new instance of exited slave process begins to run, the dependency ones could resume or run from the start.
-    The example sends a STOP command to slave processes dependent on the exited one, then they will exit.
-    Thereafter, the master creates new instances for the exited slave processes.
-
-The following diagram describes slave process recovery.
-
-.. _figure_slave_proc_recov:
-
-.. figure:: img/slave_proc_recov.*
-
-   Slave Process Recovery Process Flow
-
-
-Floating Process Support
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-When the DPDK application runs, there is always a -c option passed in to indicate the cores that are enabled.
-Then, the DPDK creates a thread for each enabled core.
-By doing so, it creates a 1:1 mapping between the enabled core and each thread.
-The enabled core always has an ID, therefore, each thread has a unique core ID in the DPDK execution environment.
-With the ID, each thread can easily access the structures or resources exclusively belonging to it without using function parameter passing.
-It can easily use the rte_lcore_id() function to get the value in every function that is called.
-
-For threads/processes not created in that way, either pinned to a core or not, they will not own a unique ID and the
-rte_lcore_id() function will not work in the correct way.
-However, sometimes these threads/processes still need the unique ID mechanism to do easy access on structures or resources.
-For example, the DPDK mempool library provides a local cache mechanism
-(refer to :ref:`mempool_local_cache`)
-for fast element allocation and freeing.
-If using a non-unique ID or a fake one,
-a race condition occurs if two or more threads/ processes with the same core ID try to use the local cache.
-
-Therefore, unused core IDs from the passing of parameters with the -c option are used to organize the core ID allocation array.
-Once the floating process is spawned, it tries to allocate a unique core ID from the array and release it on exit.
-
-A natural way to spawn a floating process is to use the fork() function and allocate a unique core ID from the unused core ID array.
-However, it is necessary to write new code to provide a notification mechanism for slave exit
-and make sure the process recovery mechanism can work with it.
-
-To avoid producing redundant code, the Master-Slave process model is still used to spawn floating processes,
-then cancel the affinity to specific cores.
-Besides that, clear the core ID assigned to the DPDK spawning a thread that has a 1:1 mapping with the core mask.
-Thereafter, get a new core ID from the unused core ID allocation array.
-
-Run the Application
-^^^^^^^^^^^^^^^^^^^
-
-This example has a command line similar to the L2 Forwarding sample application with a few differences.
-
-To run the application, start one copy of the l2fwd_fork binary in one terminal.
-Unlike the L2 Forwarding example,
-this example requires at least three cores since the master process will wait and be accountable for slave process recovery.
-The command is as follows:
-
-.. code-block:: console
-
-    #./build/l2fwd_fork -l 2-4 -n 4 -- -p 3 -f
-
-This example provides another -f option to specify the use of floating process.
-If not specified, the example will use a pinned process to perform the L2 forwarding task.
-
-To verify the recovery mechanism, proceed as follows: First, check the PID of the slave processes:
-
-.. code-block:: console
-
-    #ps -fe | grep l2fwd_fork
-    root 5136 4843 29 11:11 pts/1 00:00:05 ./build/l2fwd_fork
-    root 5145 5136 98 11:11 pts/1 00:00:11 ./build/l2fwd_fork
-    root 5146 5136 98 11:11 pts/1 00:00:11 ./build/l2fwd_fork
-
-Then, kill one of the slaves:
-
-.. code-block:: console
-
-    #kill -9 5145
-
-After 1 or 2 seconds, check whether the slave has resumed:
-
-.. code-block:: console
-
-    #ps -fe | grep l2fwd_fork
-    root 5136 4843 3 11:11 pts/1 00:00:06 ./build/l2fwd_fork
-    root 5247 5136 99 11:14 pts/1 00:00:01 ./build/l2fwd_fork
-    root 5248 5136 99 11:14 pts/1 00:00:01 ./build/l2fwd_fork
-
-It can also monitor the traffic generator statics to see whether slave processes have resumed.
-
-Explanation
-^^^^^^^^^^^
-
-As described in previous sections,
-not all global and static variables need to change to be accessible in multiple processes;
-it depends on how they are used.
-In this example,
-the statics info on packets dropped/forwarded/received count needs to be updated by the slave process,
-and the master needs to see the update and print them out.
-So, it needs to allocate a heap buffer using rte_zmalloc.
-In addition, if the -f option is specified,
-an array is needed to store the allocated core ID for the floating process so that the master can return it
-after a slave has exited accidentally.
-
-.. code-block:: c
-
-    static int
-    l2fwd_malloc_shared_struct(void)
-    {
-        port_statistics = rte_zmalloc("port_stat", sizeof(struct l2fwd_port_statistics) * RTE_MAX_ETHPORTS, 0);
-
-        if (port_statistics == NULL)
-            return -1;
-
-        /* allocate mapping_id array */
-
-        if (float_proc) {
-            int i;
-
-            mapping_id = rte_malloc("mapping_id", sizeof(unsigned) * RTE_MAX_LCORE, 0);
-            if (mapping_id == NULL)
-                return -1;
-
-            for (i = 0 ;i < RTE_MAX_LCORE; i++)
-                mapping_id[i] = INVALID_MAPPING_ID;
-
-        }
-        return 0;
-    }
-
-For each slave process, packets are received from one port and forwarded to another port that another slave is operating on.
-If the other slave exits accidentally, the port it is operating on may not work normally,
-so the first slave cannot forward packets to that port.
-There is a dependency on the port in this case. So, the master should recognize the dependency.
-The following is the code to detect this dependency:
-
-.. code-block:: c
-
-    for (portid = 0; portid < nb_ports; portid++) {
-        /* skip ports that are not enabled */
-
-        if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
-            continue;
-
-        /* Find pair ports' lcores */
-
-        find_lcore = find_pair_lcore = 0;
-        pair_port = l2fwd_dst_ports[portid];
-
-        for (i = 0; i < RTE_MAX_LCORE; i++) {
-            if (!rte_lcore_is_enabled(i))
-                continue;
-
-            for (j = 0; j < lcore_queue_conf[i].n_rx_port;j++) {
-                if (lcore_queue_conf[i].rx_port_list[j] == portid) {
-                    lcore = i;
-                    find_lcore = 1;
-                    break;
-                }
-
-                if (lcore_queue_conf[i].rx_port_list[j] == pair_port) {
-                    pair_lcore = i;
-                    find_pair_lcore = 1;
-                    break;
-                }
-            }
-
-            if (find_lcore && find_pair_lcore)
-                break;
-        }
-
-        if (!find_lcore || !find_pair_lcore)
-            rte_exit(EXIT_FAILURE, "Not find port=%d pair\\n", portid);
-
-        printf("lcore %u and %u paired\\n", lcore, pair_lcore);
-
-        lcore_resource[lcore].pair_id = pair_lcore;
-        lcore_resource[pair_lcore].pair_id = lcore;
-    }
-
-Before launching the slave process,
-it is necessary to set up the communication channel between the master and slave so that
-the master can notify the slave if its peer process with the dependency exited.
-In addition, the master needs to register a callback function in the case where a specific slave exited.
-
-.. code-block:: c
-
-    for (i = 0; i < RTE_MAX_LCORE; i++) {
-        if (lcore_resource[i].enabled) {
-            /* Create ring for master and slave communication */
-
-            ret = create_ms_ring(i);
-            if (ret != 0)
-                rte_exit(EXIT_FAILURE, "Create ring for lcore=%u failed",i);
-
-            if (flib_register_slave_exit_notify(i,slave_exit_cb) != 0)
-                rte_exit(EXIT_FAILURE, "Register master_trace_slave_exit failed");
-        }
-    }
-
-After launching the slave process, the master waits and prints out the port statics periodically.
-If an event indicating that a slave process exited is detected,
-it sends the STOP command to the peer and waits until it has also exited.
-Then, it tries to clean up the execution environment and prepare new resources.
-Finally, the new slave instance is launched.
-
-.. code-block:: c
-
-    while (1) {
-        sleep(1);
-        cur_tsc = rte_rdtsc();
-        diff_tsc = cur_tsc - prev_tsc;
-
-        /* if timer is enabled */
-
-        if (timer_period > 0) {
-            /* advance the timer */
-            timer_tsc += diff_tsc;
-
-            /* if timer has reached its timeout */
-            if (unlikely(timer_tsc >= (uint64_t) timer_period)) {
-                print_stats();
-
-                /* reset the timer */
-                timer_tsc = 0;
-            }
-        }
-
-        prev_tsc = cur_tsc;
-
-        /* Check any slave need restart or recreate */
-
-        rte_spinlock_lock(&res_lock);
-
-        for (i = 0; i < RTE_MAX_LCORE; i++) {
-            struct lcore_resource_struct *res = &lcore_resource[i];
-            struct lcore_resource_struct *pair = &lcore_resource[res->pair_id];
-
-            /* If find slave exited, try to reset pair */
-
-            if (res->enabled && res->flags && pair->enabled) {
-                if (!pair->flags) {
-                    master_sendcmd_with_ack(pair->lcore_id, CMD_STOP);
-                    rte_spinlock_unlock(&res_lock);
-                    sleep(1);
-                    rte_spinlock_lock(&res_lock);
-                    if (pair->flags)
-                        continue;
-                }
-
-                if (reset_pair(res->lcore_id, pair->lcore_id) != 0)
-                    rte_exit(EXIT_FAILURE, "failed to reset slave");
-
-                res->flags = 0;
-                pair->flags = 0;
-            }
-        }
-        rte_spinlock_unlock(&res_lock);
-    }
-
-When the slave process is spawned and starts to run, it checks whether the floating process option is applied.
-If so, it clears the affinity to a specific core and also sets the unique core ID to 0.
-Then, it tries to allocate a new core ID.
-Since the core ID has changed, the resource allocated by the master cannot work,
-so it remaps the resource to the new core ID slot.
-
-.. code-block:: c
-
-    static int
-    l2fwd_launch_one_lcore( attribute ((unused)) void *dummy)
-    {
-        unsigned lcore_id = rte_lcore_id();
-
-        if (float_proc) {
-            unsigned flcore_id;
-
-            /* Change it to floating process, also change it's lcore_id */
-
-            clear_cpu_affinity();
-
-            RTE_PER_LCORE(_lcore_id) = 0;
-
-            /* Get a lcore_id */
-
-            if (flib_assign_lcore_id() < 0 ) {
-                printf("flib_assign_lcore_id failed\n");
-                return -1;
-            }
-
-            flcore_id = rte_lcore_id();
-
-            /* Set mapping id, so master can return it after slave exited */
-
-            mapping_id[lcore_id] = flcore_id;
-            printf("Org lcore_id = %u, cur lcore_id = %u\n",lcore_id, flcore_id);
-            remapping_slave_resource(lcore_id, flcore_id);
-        }
-
-        l2fwd_main_loop();
-
-        /* return lcore_id before return */
-        if (float_proc) {
-            flib_free_lcore_id(rte_lcore_id());
-            mapping_id[lcore_id] = INVALID_MAPPING_ID;
-        }
-        return 0;
-    }
diff --git a/doc/guides/sample_app_ug/vhost.rst b/doc/guides/sample_app_ug/vhost.rst
index 9112e789..fd42cb3f 100644
--- a/doc/guides/sample_app_ug/vhost.rst
+++ b/doc/guides/sample_app_ug/vhost.rst
@@ -158,6 +158,10 @@ when disabling VLAN strip. Such feature, which heavily depends on hardware,
 should be removed from this example to reduce confusion. Now, VLAN strip is
 enabled and cannot be disabled.
 
+**--builtin-net-driver**
+A very simple vhost-user net driver which demonstrates how to use the generic
+vhost APIs will be used when this option is given. It is disabled by default.
+
 Common Issues
 -------------
 
@@ -192,3 +196,8 @@ Common Issues
   according to the NIC's property. ::
 
       make EXTRA_CFLAGS="-DMAX_QUEUES=320"
+
+* Option "builtin-net-driver" is incompatible with QEMU
+
+  QEMU vhost net device start will fail if protocol feature is not negotiated.
+  DPDK virtio-user pmd can be the replacement of QEMU.
diff --git a/doc/guides/sample_app_ug/vm_power_management.rst b/doc/guides/sample_app_ug/vm_power_management.rst
index cd7c7f34..855570d6 100644
--- a/doc/guides/sample_app_ug/vm_power_management.rst
+++ b/doc/guides/sample_app_ug/vm_power_management.rst
@@ -20,7 +20,7 @@ running on Virtual Machines(VMs).
 
 The Virtual Machine Power Management solution shows an example of
 how a DPDK application can indicate its processing requirements using VM local
-only information(vCPU/lcore) to a Host based Monitor which is responsible
+only information(vCPU/lcore, etc.) to a Host based Monitor which is responsible
 for accepting requests for frequency changes for a vCPU, translating the vCPU
 to a pCPU via libvirt and affecting the change in frequency.
 
@@ -38,6 +38,26 @@ The solution is comprised of two high-level components:
    to the librte_power ACPI cpufreq sysfs based library.
    The Host Application relies on both qemu-kvm and libvirt to function.
 
+   This monitoring application is responsible for:
+
+   - Accepting requests from client applications: Client applications can
+     request frequency changes for a vCPU, translating
+     the vCPU to a pCPU via libvirt and affecting the change in frequency.
+
+   - Accepting policies from client applications: Client application can
+     send a policy to the host application. The
+     host application will then apply the rules of the policy independent
+     of the application. For example, the policy can contain time-of-day
+     information for busy/quiet periods, and the host application can scale
+     up/down the relevant cores when required. See the details of the guest
+     application below for more information on setting the policy values.
+
+   - Out-of-band monitoring of workloads via cores hardware event counters:
+     The host application can manage power for an application in a virtualised
+     OR non-virtualised environment by looking at the event counters of the
+     cores and taking action based on the branch hit/miss ratio. See the host
+     application '--core-list' command line parameter below.
+
 #. librte_power for Virtual Machines
 
    Using an alternate implementation for the librte_power API, requests for
@@ -174,13 +194,20 @@ Compiling and Running the Host Application
 Compiling
 ~~~~~~~~~
 
-Compiling the Application
--------------------------
-
-To compile the sample application see :doc:`compiling`.
+For information on compiling DPDK and the sample applications
+see :doc:`compiling`.
 
 The application is located in the ``vm_power_manager`` sub-directory.
 
+To build just the ``vm_power_manager`` application:
+
+.. code-block:: console
+
+  export RTE_SDK=/path/to/rte_sdk
+  export RTE_TARGET=build
+  cd ${RTE_SDK}/examples/vm_power_manager/
+  make
+
 Running
 ~~~~~~~
 
@@ -287,38 +314,129 @@ Manual control and inspection can also be carried in relation CPU frequency scal
 
     set_cpu_freq {core_num} up|down|min|max
 
+There are also some command line parameters for enabling the out-of-band
+monitoring of branch ratio on cores doing busy polling via PMDs.
+
+  .. code-block:: console
+
+    --core-list {list of cores}
+
+  When this parameter is used, the list of cores specified will monitor the ratio
+  between branch hits and branch misses. A tightly polling PMD thread will have a
+  very low branch ratio, so the core frequency will be scaled down to the minimim
+  allowed value. When packets are received, the code path will alter, causing the
+  branch ratio to increase. When the ratio goes above the ratio threshold, the
+  core frequency will be scaled up to the maximum allowed value.
+
+  .. code-block:: console
+
+    --branch-ratio {ratio}
+
+  The branch ratio is a floating point number that specifies the threshold at which
+  to scale up or down for the given workload. The default branch ratio is 0.01,
+  and will need to be adjusted for different workloads.
+
+
 Compiling and Running the Guest Applications
 --------------------------------------------
 
-For compiling and running l3fwd-power, see :doc:`l3_forward_power_man`.
+l3fwd-power is one sample application that can be used with vm_power_manager.
 
 A guest CLI is also provided for validating the setup.
 
 For both l3fwd-power and guest CLI, the channels for the VM must be monitored by the
-host application using the *add_channels* command on the host.
+host application using the *add_channels* command on the host. This typically uses
+the following commands in the host application:
+
+.. code-block:: console
+
+  vm_power> add_vm vmname
+  vm_power> add_channels vmname all
+  vm_power> set_channel_status vmname all enabled
+  vm_power> show_vm vmname
+
 
 Compiling
 ~~~~~~~~~
 
-#. export RTE_SDK=/path/to/rte_sdk
-#. cd ${RTE_SDK}/examples/vm_power_manager/guest_cli
-#. make
+For information on compiling DPDK and the sample applications
+see :doc:`compiling`.
+
+For compiling and running l3fwd-power, see :doc:`l3_forward_power_man`.
+
+The application is located in the ``guest_cli`` sub-directory under ``vm_power_manager``.
+
+To build just the ``guest_vm_power_manager`` application:
+
+.. code-block:: console
+
+  export RTE_SDK=/path/to/rte_sdk
+  export RTE_TARGET=build
+  cd ${RTE_SDK}/examples/vm_power_manager/guest_cli/
+  make
 
 Running
 ~~~~~~~
 
-The application does not have any specific command line options other than *EAL*:
+The standard *EAL* command line parameters are required:
 
 .. code-block:: console
 
- ./build/vm_power_mgr [EAL options]
+ ./build/guest_vm_power_mgr [EAL options] -- [guest options]
 
-The application for example purposes uses a channel for each lcore enabled,
-for example to run on cores 0,1,2,3 on a system with 4 memory channels:
+The guest example uses a channel for each lcore enabled. For example,
+to run on cores 0,1,2,3:
 
 .. code-block:: console
 
- ./build/guest_vm_power_mgr -l 0-3 -n 4
+ ./build/guest_vm_power_mgr -l 0-3
+
+Optionally, there is a list of command line parameter should the user wish to send a power
+policy down to the host application. These parameters are as follows:
+
+  .. code-block:: console
+
+    --vm-name {name of guest vm}
+
+  This parameter allows the user to change the Virtual Machine name passed down to the
+  host application via the power policy. The default is "ubuntu2"
+
+  .. code-block:: console
+
+    --vcpu-list {list vm cores}
+
+  A comma-separated list of cores in the VM that the user wants the host application to
+  monitor. The list of cores in any vm starts at zero, and these are mapped to the
+  physical cores by the host application once the policy is passed down.
+  Valid syntax includes individial cores '2,3,4', or a range of cores '2-4', or a
+  combination of both '1,3,5-7'
+
+  .. code-block:: console
+
+    --busy-hours {list of busy hours}
+
+  A comma-separated list of hours within which to set the core frequency to maximum.
+  Valid syntax includes individial hours '2,3,4', or a range of hours '2-4', or a
+  combination of both '1,3,5-7'. Valid hours are 0 to 23.
+
+  .. code-block:: console
+
+    --quiet-hours {list of quiet hours}
+
+  A comma-separated list of hours within which to set the core frequency to minimum.
+  Valid syntax includes individial hours '2,3,4', or a range of hours '2-4', or a
+  combination of both '1,3,5-7'. Valid hours are 0 to 23.
+
+  .. code-block:: console
+
+    --policy {policy type}
+
+  The type of policy. This can be one of the following values:
+  TRAFFIC - based on incoming traffic rates on the NIC.
+  TIME - busy/quiet hours policy.
+  BRANCH_RATIO - uses branch ratio counters to determine core busyness.
+  Not all parameters are needed for all policy types. For example, BRANCH_RATIO
+  only needs the vcpu-list parameter, not any of the hours.
 
 
 After successful initialization the user is presented with VM Power Manager Guest CLI:
@@ -333,3 +451,20 @@ Where {core_num} is the lcore and channel to change frequency by scaling up/down
 .. code-block:: console
 
   set_cpu_freq {core_num} up|down|min|max
+
+To start the application and configure the power policy, and send it to the host:
+
+.. code-block:: console
+
+ ./build/guest_vm_power_mgr -l 0-3 -n 4 -- --vm-name=ubuntu --policy=BRANCH_RATIO --vcpu-list=2-4
+
+Once the VM Power Manager Guest CLI appears, issuing the 'send_policy now' command
+will send the policy to the host:
+
+.. code-block:: console
+
+  send_policy now
+
+Once the policy is sent to the host, the host application takes over the power monitoring
+of the specified cores in the policy.
+
diff --git a/doc/guides/sample_app_ug/vmdq_dcb_forwarding.rst b/doc/guides/sample_app_ug/vmdq_dcb_forwarding.rst
index 83fcdf63..0e9da970 100644
--- a/doc/guides/sample_app_ug/vmdq_dcb_forwarding.rst
+++ b/doc/guides/sample_app_ug/vmdq_dcb_forwarding.rst
@@ -102,10 +102,6 @@ a default structure is provided for VMDQ and DCB configuration to be filled in l
         .rxmode = {
             .mq_mode        = ETH_MQ_RX_VMDQ_DCB,
             .split_hdr_size = 0,
-            .header_split   = 0, /**< Header Split disabled */
-            .hw_ip_checksum = 0, /**< IP checksum offload disabled */
-            .hw_vlan_filter = 0, /**< VLAN filtering disabled */
-            .jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
         },
         .txmode = {
             .mq_mode = ETH_MQ_TX_VMDQ_DCB,
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 0d6fd50c..dde205a2 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -320,12 +320,8 @@ The available information categories are:
 
 * ``ieee1588``: Demonstrate L2 IEEE1588 V2 PTP timestamping for RX and TX. Requires ``CONFIG_RTE_LIBRTE_IEEE1588=y``.
 
-* ``tm``: Traffic Management forwarding mode
-  Demonstrates the use of ethdev traffic management APIs and softnic PMD for
-  QoS traffic management. In this mode, 5-level hierarchical QoS scheduler is
-  available as an default option that can be enabled through CLI. The user can
-  also modify the default hierarchy or specify the new hierarchy through CLI for
-  implementing QoS scheduler.  Requires ``CONFIG_RTE_LIBRTE_PMD_SOFTNIC=y`` ``CONFIG_RTE_LIBRTE_SCHED=y``.
+* ``softnic``: Demonstrates the softnic forwarding operation. In this mode, packet forwarding is
+  similar to I/O mode except for the fact that packets are loopback to the softnic ports only. Therefore, portmask parameter should be set to softnic port only. The various software based custom NIC pipelines specified through the softnic firmware (DPDK packet framework script) can be tested in this mode. Furthermore, it allows to build 5-level hierarchical QoS scheduler as a default option that can be enabled through CLI once testpmd application is initialised. The user can modify the default scheduler hierarchy or can specify the new QoS Scheduler hierarchy through CLI. Requires ``CONFIG_RTE_LIBRTE_PMD_SOFTNIC=y``.
 
 Example::
 
@@ -1059,6 +1055,13 @@ By default, GSO is disabled for all ports.
 
    testpmd> csum set tcp hw <port_id>
 
+   UDP GSO is the same as IP fragmentation, which treats the UDP header
+   as the payload and does not modify it during segmentation. That is,
+   after UDP GSO, only the first output fragment has the original UDP
+   header. Therefore, users need to enable HW IP checksum calculation
+   and SW UDP checksum calculation for GSO-enabled ports, if they want
+   correct checksums for UDP/IPv4 packets.
+
 set gso segsz
 ~~~~~~~~~~~~~
 
@@ -1483,7 +1486,7 @@ Enable or disable a per port Rx offloading on all Rx queues of a port::
                   vlan_strip, ipv4_cksum, udp_cksum, tcp_cksum, tcp_lro,
                   qinq_strip, outer_ipv4_cksum, macsec_strip,
                   header_split, vlan_filter, vlan_extend, jumbo_frame,
-                  crc_strip, scatter, timestamp, security
+                  crc_strip, scatter, timestamp, security, keep_crc
 
 This command should be run when the port is stopped, or else it will fail.
 
@@ -1498,7 +1501,7 @@ Enable or disable a per queue Rx offloading only on a specific Rx queue::
                   vlan_strip, ipv4_cksum, udp_cksum, tcp_cksum, tcp_lro,
                   qinq_strip, outer_ipv4_cksum, macsec_strip,
                   header_split, vlan_filter, vlan_extend, jumbo_frame,
-                  crc_strip, scatter, timestamp, security
+                  crc_strip, scatter, timestamp, security, keep_crc
 
 This command should be run when the port is stopped, or else it will fail.
 
@@ -1510,11 +1513,11 @@ Enable or disable a per port Tx offloading on all Tx queues of a port::
    testpmd> port config (port_id) tx_offload (offloading) on|off
 
 * ``offloading``: can be any of these offloading capability:
-                  vlan_insert, ipv4_cksum, udp_cksum, udp_cksum,
+                  vlan_insert, ipv4_cksum, udp_cksum, tcp_cksum,
                   sctp_cksum, tcp_tso, udp_tso, outer_ipv4_cksum,
                   qinq_insert, vxlan_tnl_tso, gre_tnl_tso,
                   ipip_tnl_tso, geneve_tnl_tso, macsec_insert,
-                  mt_lockfree, multi_segs, fast_free, security
+                  mt_lockfree, multi_segs, mbuf_fast_free, security
 
 This command should be run when the port is stopped, or else it will fail.
 
@@ -1526,14 +1529,46 @@ Enable or disable a per queue Tx offloading only on a specific Tx queue::
    testpmd> port (port_id) txq (queue_id) tx_offload (offloading) on|off
 
 * ``offloading``: can be any of these offloading capability:
-                  vlan_insert, ipv4_cksum, udp_cksum, udp_cksum,
+                  vlan_insert, ipv4_cksum, udp_cksum, tcp_cksum,
                   sctp_cksum, tcp_tso, udp_tso, outer_ipv4_cksum,
                   qinq_insert, vxlan_tnl_tso, gre_tnl_tso,
                   ipip_tnl_tso, geneve_tnl_tso, macsec_insert,
-                  mt_lockfree, multi_segs, fast_free, security
+                  mt_lockfree, multi_segs, mbuf_fast_free, security
 
 This command should be run when the port is stopped, or else it will fail.
 
+Config VXLAN Encap outer layers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Configure the outer layer to encapsulate a packet inside a VXLAN tunnel::
+
+ set vxlan ip-version (ipv4|ipv6) vni (vni) udp-src (udp-src) \
+ udp-dst (udp-dst) ip-src (ip-src) ip-dst (ip-dst) eth-src (eth-src) \
+ eth-dst (eth-dst)
+
+ set vxlan-with-vlan ip-version (ipv4|ipv6) vni (vni) udp-src (udp-src) \
+ udp-dst (udp-dst) ip-src (ip-src) ip-dst (ip-dst) vlan-tci (vlan-tci) \
+ eth-src (eth-src) eth-dst (eth-dst)
+
+Those command will set an internal configuration inside testpmd, any following
+flow rule using the action vxlan_encap will use the last configuration set.
+To have a different encapsulation header, one of those commands must be called
+before the flow rule creation.
+
+Config NVGRE Encap outer layers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Configure the outer layer to encapsulate a packet inside a NVGRE tunnel::
+
+ set nvgre ip-version (ipv4|ipv6) tni (tni) ip-src (ip-src) ip-dst (ip-dst) \
+        eth-src (eth-src) eth-dst (eth-dst)
+ set nvgre-with-vlan ip-version (ipv4|ipv6) tni (tni) ip-src (ip-src) \
+        ip-dst (ip-dst) vlan-tci (vlan-tci) eth-src (eth-src) eth-dst (eth-dst)
+
+Those command will set an internal configuration inside testpmd, any following
+flow rule using the action nvgre_encap will use the last configuration set.
+To have a different encapsulation header, one of those commands must be called
+before the flow rule creation.
 
 Port Functions
 --------------
@@ -1986,7 +2021,7 @@ Create a new bonding device::
 
 For example, to create a bonded device in mode 1 on socket 0::
 
-   testpmd> create bonded 1 0
+   testpmd> create bonded device 1 0
    created new bonded device (port X)
 
 add bonding slave
@@ -2621,8 +2656,8 @@ where:
   call failure. On the other hand, hierarchy is preserved when this parameter
   is equal to zero.
 
-Set port traffic management default hierarchy (tm forwarding mode)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Set port traffic management default hierarchy (softnic forwarding mode)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 set the traffic management default hierarchy on the port::
 
@@ -3650,6 +3685,18 @@ This section lists supported actions and their attributes, if any.
 
   - ``ethertype``: Ethertype.
 
+- ``vxlan_encap``: Performs a VXLAN encapsulation, outer layer configuration
+  is done through `Config VXLAN Encap outer layers`_.
+
+- ``vxlan_decap``: Performs a decapsulation action by stripping all headers of
+  the VXLAN tunnel network overlay from the matched flow.
+
+- ``nvgre_encap``: Performs a NVGRE encapsulation, outer layer configuration
+  is done through `Config NVGRE Encap outer layers`_.
+
+- ``nvgre_decap``: Performs a decapsulation action by stripping all headers of
+  the NVGRE tunnel network overlay from the matched flow.
+
 Destroying flow rules
 ~~~~~~~~~~~~~~~~~~~~~
 
@@ -3915,6 +3962,69 @@ Validate and create a QinQ rule on port 0 to steer traffic to a queue on the hos
    0       0       0       i-      ETH VLAN VLAN=>VF QUEUE
    1       0       0       i-      ETH VLAN VLAN=>PF QUEUE
 
+Sample VXLAN encapsulation rule
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+VXLAN encapsulation outer layer has default value pre-configured in testpmd
+source code, those can be changed by using the following commands
+
+IPv4 VXLAN outer header::
+
+ testpmd> set vxlan ip-version ipv4 vni 4 udp-src 4 udp-dst 4 ip-src 127.0.0.1
+        ip-dst 128.0.0.1 eth-src 11:11:11:11:11:11 eth-dst 22:22:22:22:22:22
+ testpmd> flow create 0 ingress pattern end actions vxlan_encap /
+        queue index 0 / end
+
+ testpmd> set vxlan-with-vlan ip-version ipv4 vni 4 udp-src 4 udp-dst 4 ip-src
+         127.0.0.1 ip-dst 128.0.0.1 vlan-tci 34 eth-src 11:11:11:11:11:11
+         eth-dst 22:22:22:22:22:22
+ testpmd> flow create 0 ingress pattern end actions vxlan_encap /
+         queue index 0 / end
+
+IPv6 VXLAN outer header::
+
+ testpmd> set vxlan ip-version ipv6 vni 4 udp-src 4 udp-dst 4 ip-src ::1
+        ip-dst ::2222 eth-src 11:11:11:11:11:11 eth-dst 22:22:22:22:22:22
+ testpmd> flow create 0 ingress pattern end actions vxlan_encap /
+         queue index 0 / end
+
+ testpmd> set vxlan-with-vlan ip-version ipv6 vni 4 udp-src 4 udp-dst 4
+         ip-src ::1 ip-dst ::2222 vlan-tci 34 eth-src 11:11:11:11:11:11
+         eth-dst 22:22:22:22:22:22
+ testpmd> flow create 0 ingress pattern end actions vxlan_encap /
+         queue index 0 / end
+
+Sample NVGRE encapsulation rule
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+NVGRE encapsulation outer layer has default value pre-configured in testpmd
+source code, those can be changed by using the following commands
+
+IPv4 NVGRE outer header::
+
+ testpmd> set nvgre ip-version ipv4 tni 4 ip-src 127.0.0.1 ip-dst 128.0.0.1
+        eth-src 11:11:11:11:11:11 eth-dst 22:22:22:22:22:22
+ testpmd> flow create 0 ingress pattern end actions nvgre_encap /
+        queue index 0 / end
+
+ testpmd> set nvgre-with-vlan ip-version ipv4 tni 4 ip-src 127.0.0.1
+         ip-dst 128.0.0.1 vlan-tci 34 eth-src 11:11:11:11:11:11
+         eth-dst 22:22:22:22:22:22
+ testpmd> flow create 0 ingress pattern end actions nvgre_encap /
+         queue index 0 / end
+
+IPv6 NVGRE outer header::
+
+ testpmd> set nvgre ip-version ipv6 tni 4 ip-src ::1 ip-dst ::2222
+        eth-src 11:11:11:11:11:11 eth-dst 22:22:22:22:22:22
+ testpmd> flow create 0 ingress pattern end actions nvgre_encap /
+        queue index 0 / end
+
+ testpmd> set nvgre-with-vlan ip-version ipv6 tni 4 ip-src ::1 ip-dst ::2222
+        vlan-tci 34 eth-src 11:11:11:11:11:11 eth-dst 22:22:22:22:22:22
+ testpmd> flow create 0 ingress pattern end actions nvgre_encap /
+        queue index 0 / end
+
 BPF Functions
 --------------
 
diff --git a/drivers/Makefile b/drivers/Makefile
index c88638c8..75660765 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -13,6 +13,8 @@ DIRS-$(CONFIG_RTE_LIBRTE_BBDEV) += baseband
 DEPDIRS-baseband := common bus mempool
 DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += crypto
 DEPDIRS-crypto := common bus mempool
+DIRS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += common/qat
+DEPDIRS-common/qat := bus mempool
 DIRS-$(CONFIG_RTE_LIBRTE_COMPRESSDEV) += compress
 DEPDIRS-compress := bus mempool
 DIRS-$(CONFIG_RTE_LIBRTE_EVENTDEV) += event
diff --git a/drivers/baseband/null/bbdev_null.c b/drivers/baseband/null/bbdev_null.c
index 76fc9d7a..2f251510 100644
--- a/drivers/baseband/null/bbdev_null.c
+++ b/drivers/baseband/null/bbdev_null.c
@@ -348,9 +348,7 @@ RTE_PMD_REGISTER_PARAM_STRING(DRIVER_NAME,
 	BBDEV_NULL_SOCKET_ID_ARG"=<int>");
 RTE_PMD_REGISTER_ALIAS(DRIVER_NAME, bbdev_null);
 
-RTE_INIT(null_bbdev_init_log);
-static void
-null_bbdev_init_log(void)
+RTE_INIT(null_bbdev_init_log)
 {
 	bbdev_null_logtype = rte_log_register("pmd.bb.null");
 	if (bbdev_null_logtype >= 0)
diff --git a/drivers/baseband/turbo_sw/bbdev_turbo_software.c b/drivers/baseband/turbo_sw/bbdev_turbo_software.c
index 05e95ed9..8ceb2769 100644
--- a/drivers/baseband/turbo_sw/bbdev_turbo_software.c
+++ b/drivers/baseband/turbo_sw/bbdev_turbo_software.c
@@ -490,8 +490,8 @@ process_enc_cb(struct turbo_sw_queue *q, struct rte_bbdev_enc_op *op,
 			return;
 		}
 		crc_req.data = in;
-		crc_req.len = (k - 24) >> 3;
-		/* Check if there is a room for CRC bits. If not use
+		crc_req.len = k - 24;
+		/* Check if there is a room for CRC bits if not use
 		 * the temporary buffer.
 		 */
 		if (rte_pktmbuf_append(m_in, 3) == NULL) {
@@ -522,8 +522,8 @@ process_enc_cb(struct turbo_sw_queue *q, struct rte_bbdev_enc_op *op,
 			return;
 		}
 		crc_req.data = in;
-		crc_req.len = (k - 24) >> 3;
-		/* Check if there is a room for CRC bits. If this is the last
+		crc_req.len = k - 24;
+		/* Check if there is a room for CRC bits if this is the last
 		 * CB in TB. If not use temporary buffer.
 		 */
 		if ((c - r == 1) && (rte_pktmbuf_append(m_in, 3) == NULL)) {
@@ -1299,9 +1299,7 @@ RTE_PMD_REGISTER_PARAM_STRING(DRIVER_NAME,
 	TURBO_SW_SOCKET_ID_ARG"=<int>");
 RTE_PMD_REGISTER_ALIAS(DRIVER_NAME, turbo_sw);
 
-RTE_INIT(null_bbdev_init_log);
-static void
-null_bbdev_init_log(void)
+RTE_INIT(turbo_sw_bbdev_init_log)
 {
 	bbdev_turbo_sw_logtype = rte_log_register("pmd.bb.turbo_sw");
 	if (bbdev_turbo_sw_logtype >= 0)
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index ef7f2475..cea3b55e 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -10,5 +10,6 @@ endif
 DIRS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += ifpga
 DIRS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci
 DIRS-$(CONFIG_RTE_LIBRTE_VDEV_BUS) += vdev
+DIRS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus
 
 include $(RTE_SDK)/mk/rte.subdir.mk
diff --git a/drivers/bus/dpaa/base/fman/fman_hw.c b/drivers/bus/dpaa/base/fman/fman_hw.c
index 0148b98e..4ebbc3d3 100644
--- a/drivers/bus/dpaa/base/fman/fman_hw.c
+++ b/drivers/bus/dpaa/base/fman/fman_hw.c
@@ -16,6 +16,9 @@
 #include <fsl_fman_crc64.h>
 #include <fsl_bman.h>
 
+#define FMAN_SP_SG_DISABLE                          0x80000000
+#define FMAN_SP_EXT_BUF_MARG_START_SHIFT            16
+
 /* Instantiate the global variable that the inline CRC64 implementation (in
  * <fsl_fman.h>) depends on.
  */
@@ -422,20 +425,16 @@ fman_if_set_fc_quanta(struct fman_if *fm_if, u16 pause_quanta)
 int
 fman_if_get_fdoff(struct fman_if *fm_if)
 {
-	u32 fmbm_ricp;
+	u32 fmbm_rebm;
 	int fdoff;
-	int iceof_mask = 0x001f0000;
-	int icsz_mask = 0x0000001f;
 
 	struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if);
 
 	assert(fman_ccsr_map_fd != -1);
 
-	fmbm_ricp =
-		   in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_ricp);
-	/*iceof + icsz*/
-	fdoff = ((fmbm_ricp & iceof_mask) >> 16) * 16 +
-		(fmbm_ricp & icsz_mask) * 16;
+	fmbm_rebm = in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm);
+
+	fdoff = (fmbm_rebm >> FMAN_SP_EXT_BUF_MARG_START_SHIFT) & 0x1ff;
 
 	return fdoff;
 }
@@ -502,12 +501,16 @@ fman_if_set_fdoff(struct fman_if *fm_if, uint32_t fd_offset)
 {
 	struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if);
 	unsigned int *fmbm_rebm;
+	int val = 0;
+	int fmbm_mask = 0x01ff0000;
+
+	val = fd_offset << FMAN_SP_EXT_BUF_MARG_START_SHIFT;
 
 	assert(fman_ccsr_map_fd != -1);
 
 	fmbm_rebm = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm;
 
-	out_be32(fmbm_rebm, in_be32(fmbm_rebm) | (fd_offset << 16));
+	out_be32(fmbm_rebm, (in_be32(fmbm_rebm) & ~fmbm_mask) | val);
 }
 
 void
@@ -536,6 +539,47 @@ fman_if_get_maxfrm(struct fman_if *fm_if)
 	return (in_be32(reg_maxfrm) | 0x0000FFFF);
 }
 
+/* MSB in fmbm_rebm register
+ * 0 - If BMI cannot store the frame in a single buffer it may select a buffer
+ *     of smaller size and store the frame in scatter gather (S/G) buffers
+ * 1 - Scatter gather format is not enabled for frame storage. If BMI cannot
+ *     store the frame in a single buffer, the frame is discarded.
+ */
+
+int
+fman_if_get_sg_enable(struct fman_if *fm_if)
+{
+	u32 fmbm_rebm;
+
+	struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if);
+
+	assert(fman_ccsr_map_fd != -1);
+
+	fmbm_rebm = in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm);
+
+	return (fmbm_rebm & FMAN_SP_SG_DISABLE) ? 0 : 1;
+}
+
+void
+fman_if_set_sg(struct fman_if *fm_if, int enable)
+{
+	struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if);
+	unsigned int *fmbm_rebm;
+	int val;
+	int fmbm_mask = FMAN_SP_SG_DISABLE;
+
+	if (enable)
+		val = 0;
+	else
+		val = FMAN_SP_SG_DISABLE;
+
+	assert(fman_ccsr_map_fd != -1);
+
+	fmbm_rebm = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm;
+
+	out_be32(fmbm_rebm, (in_be32(fmbm_rebm) & ~fmbm_mask) | val);
+}
+
 void
 fman_if_set_dnia(struct fman_if *fm_if, uint32_t nia)
 {
diff --git a/drivers/bus/dpaa/base/fman/netcfg_layer.c b/drivers/bus/dpaa/base/fman/netcfg_layer.c
index 3e956ce1..031c6f1a 100644
--- a/drivers/bus/dpaa/base/fman/netcfg_layer.c
+++ b/drivers/bus/dpaa/base/fman/netcfg_layer.c
@@ -18,11 +18,6 @@
 #include <rte_dpaa_logs.h>
 #include <netcfg.h>
 
-/* Structure contains information about all the interfaces given by user
- * on command line.
- */
-struct netcfg_interface *netcfg_interface;
-
 /* This data structure contaings all configurations information
  * related to usages of DPA devices.
  */
diff --git a/drivers/bus/dpaa/base/fman/of.c b/drivers/bus/dpaa/base/fman/of.c
index 1b2dbe26..a7f3174e 100644
--- a/drivers/bus/dpaa/base/fman/of.c
+++ b/drivers/bus/dpaa/base/fman/of.c
@@ -182,6 +182,11 @@ linear_dir(struct dt_dir *d)
 				DPAA_BUS_LOG(DEBUG, "Duplicate lphandle in %s",
 					     d->node.node.full_name);
 			d->lphandle = f;
+		} else if (!strcmp(f->node.node.name, "phandle")) {
+			if (d->lphandle)
+				DPAA_BUS_LOG(DEBUG, "Duplicate lphandle in %s",
+					     d->node.node.full_name);
+			d->lphandle = f;
 		} else if (!strcmp(f->node.node.name, "#address-cells")) {
 			if (d->a_cells)
 				DPAA_BUS_LOG(DEBUG, "Duplicate a_cells in %s",
@@ -541,3 +546,42 @@ of_device_is_compatible(const struct device_node *dev_node,
 		return true;
 	return false;
 }
+
+static const void *of_get_mac_addr(const struct device_node *np,
+		const char *name)
+{
+	return of_get_property(np, name, NULL);
+}
+
+/**
+ * Search the device tree for the best MAC address to use.  'mac-address' is
+ * checked first, because that is supposed to contain to "most recent" MAC
+ * address. If that isn't set, then 'local-mac-address' is checked next,
+ * because that is the default address.  If that isn't set, then the obsolete
+ * 'address' is checked, just in case we're using an old device tree.
+ *
+ * Note that the 'address' property is supposed to contain a virtual address of
+ * the register set, but some DTS files have redefined that property to be the
+ * MAC address.
+ *
+ * All-zero MAC addresses are rejected, because those could be properties that
+ * exist in the device tree, but were not set by U-Boot.  For example, the
+ * DTS could define 'mac-address' and 'local-mac-address', with zero MAC
+ * addresses.  Some older U-Boots only initialized 'local-mac-address'.  In
+ * this case, the real MAC is in 'local-mac-address', and 'mac-address' exists
+ * but is all zeros.
+ */
+const void *of_get_mac_address(const struct device_node *np)
+{
+	const void *addr;
+
+	addr = of_get_mac_addr(np, "mac-address");
+	if (addr)
+		return addr;
+
+	addr = of_get_mac_addr(np, "local-mac-address");
+	if (addr)
+		return addr;
+
+	return of_get_mac_addr(np, "address");
+}
diff --git a/drivers/bus/dpaa/base/qbman/bman_driver.c b/drivers/bus/dpaa/base/qbman/bman_driver.c
index 1381da36..b14b5905 100644
--- a/drivers/bus/dpaa/base/qbman/bman_driver.c
+++ b/drivers/bus/dpaa/base/qbman/bman_driver.c
@@ -15,9 +15,9 @@
 /*
  * Global variables of the max portal/pool number this bman version supported
  */
-u16 bman_ip_rev;
+static u16 bman_ip_rev;
 u16 bman_pool_max;
-void *bman_ccsr_map;
+static void *bman_ccsr_map;
 
 /*****************/
 /* Portal driver */
diff --git a/drivers/bus/dpaa/base/qbman/qman.c b/drivers/bus/dpaa/base/qbman/qman.c
index 27d98cc1..7c17027f 100644
--- a/drivers/bus/dpaa/base/qbman/qman.c
+++ b/drivers/bus/dpaa/base/qbman/qman.c
@@ -625,7 +625,7 @@ fail_eqcr:
 
 #define MAX_GLOBAL_PORTALS 8
 static struct qman_portal global_portals[MAX_GLOBAL_PORTALS];
-rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS];
+static rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS];
 
 static struct qman_portal *
 qman_alloc_global_portal(void)
@@ -1058,7 +1058,7 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit,
 	struct qm_portal *portal = &p->p;
 	register struct qm_dqrr *dqrr = &portal->dqrr;
 	struct qm_dqrr_entry *dq[QM_DQRR_SIZE], *shadow[QM_DQRR_SIZE];
-	struct qman_fq *fq[QM_DQRR_SIZE];
+	struct qman_fq *fq;
 	unsigned int limit = 0, rx_number = 0;
 	uint32_t consume = 0;
 
@@ -1092,14 +1092,13 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit,
 
 		/* SDQCR: context_b points to the FQ */
 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
-		fq[rx_number] = qman_fq_lookup_table[be32_to_cpu(
-						dq[rx_number]->contextB)];
+		fq = qman_fq_lookup_table[be32_to_cpu(dq[rx_number]->contextB)];
 #else
-		fq[rx_number] = (void *)be32_to_cpu(
-						dq[rx_number]->contextB);
+		fq = (void *)be32_to_cpu(dq[rx_number]->contextB);
 #endif
-		fq[rx_number]->cb.dqrr_prepare(shadow[rx_number],
-						 &bufs[rx_number]);
+		if (fq->cb.dqrr_prepare)
+			fq->cb.dqrr_prepare(shadow[rx_number],
+					    &bufs[rx_number]);
 
 		consume |= (1 << (31 - DQRR_PTR2IDX(shadow[rx_number])));
 		rx_number++;
@@ -1107,7 +1106,7 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit,
 	} while (++limit < poll_limit);
 
 	if (rx_number)
-		fq[0]->cb.dqrr_dpdk_pull_cb(fq, shadow, bufs, rx_number);
+		fq->cb.dqrr_dpdk_pull_cb(&fq, shadow, bufs, rx_number);
 
 	/* Consume all the DQRR enries together */
 	qm_out(DQRR_DCAP, (1 << 8) | consume);
@@ -2003,13 +2002,13 @@ int qman_query_congestion(struct qm_mcr_querycongestion *congestion)
 	return 0;
 }
 
-int qman_set_vdq(struct qman_fq *fq, u16 num)
+int qman_set_vdq(struct qman_fq *fq, u16 num, uint32_t vdqcr_flags)
 {
 	struct qman_portal *p = get_affine_portal();
 	uint32_t vdqcr;
 	int ret = -EBUSY;
 
-	vdqcr = QM_VDQCR_EXACT;
+	vdqcr = vdqcr_flags;
 	vdqcr |= QM_VDQCR_NUMFRAMES_SET(num);
 
 	if ((fq->state != qman_fq_state_parked) &&
diff --git a/drivers/bus/dpaa/base/qbman/qman_driver.c b/drivers/bus/dpaa/base/qbman/qman_driver.c
index 07b29d55..f6ecd6b2 100644
--- a/drivers/bus/dpaa/base/qbman/qman_driver.c
+++ b/drivers/bus/dpaa/base/qbman/qman_driver.c
@@ -20,9 +20,9 @@ u16 qm_channel_caam = QMAN_CHANNEL_CAAM;
 u16 qm_channel_pme = QMAN_CHANNEL_PME;
 
 /* Ccsr map address to access ccsrbased register */
-void *qman_ccsr_map;
+static void *qman_ccsr_map;
 /* The qman clock frequency */
-u32 qman_clk;
+static u32 qman_clk;
 
 static __thread int qmfd = -1;
 static __thread struct qm_portal_config qpcfg;
diff --git a/drivers/bus/dpaa/base/qbman/qman_priv.h b/drivers/bus/dpaa/base/qbman/qman_priv.h
index 9e4471e6..02f6301f 100644
--- a/drivers/bus/dpaa/base/qbman/qman_priv.h
+++ b/drivers/bus/dpaa/base/qbman/qman_priv.h
@@ -139,7 +139,6 @@ struct qm_portal_config {
 #define QMAN_REV31 0x0301
 #define QMAN_REV32 0x0302
 extern u16 qman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */
-extern u32 qman_clk;
 
 int qm_set_wpm(int wpm);
 int qm_get_wpm(int *wpm);
diff --git a/drivers/bus/dpaa/dpaa_bus.c b/drivers/bus/dpaa/dpaa_bus.c
index 20462065..16fabd1b 100644
--- a/drivers/bus/dpaa/dpaa_bus.c
+++ b/drivers/bus/dpaa/dpaa_bus.c
@@ -50,7 +50,7 @@ struct rte_dpaa_bus rte_dpaa_bus;
 struct netcfg_info *dpaa_netcfg;
 
 /* define a variable to hold the portal_key, once created.*/
-pthread_key_t dpaa_portal_key;
+static pthread_key_t dpaa_portal_key;
 
 unsigned int dpaa_svr_family;
 
@@ -539,6 +539,13 @@ rte_dpaa_bus_probe(void)
 	unsigned int svr_ver;
 	int probe_all = rte_dpaa_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST;
 
+	svr_file = fopen(DPAA_SOC_ID_FILE, "r");
+	if (svr_file) {
+		if (fscanf(svr_file, "svr:%x", &svr_ver) > 0)
+			dpaa_svr_family = svr_ver & SVR_MASK;
+		fclose(svr_file);
+	}
+
 	/* For each registered driver, and device, call the driver->probe */
 	TAILQ_FOREACH(dev, &rte_dpaa_bus.device_list, next) {
 		TAILQ_FOREACH(drv, &rte_dpaa_bus.driver_list, next) {
@@ -569,13 +576,6 @@ rte_dpaa_bus_probe(void)
 	if (!TAILQ_EMPTY(&rte_dpaa_bus.device_list))
 		rte_mbuf_set_platform_mempool_ops(DPAA_MEMPOOL_OPS_NAME);
 
-	svr_file = fopen(DPAA_SOC_ID_FILE, "r");
-	if (svr_file) {
-		if (fscanf(svr_file, "svr:%x", &svr_ver) > 0)
-			dpaa_svr_family = svr_ver & SVR_MASK;
-		fclose(svr_file);
-	}
-
 	return 0;
 }
 
@@ -626,9 +626,7 @@ struct rte_dpaa_bus rte_dpaa_bus = {
 
 RTE_REGISTER_BUS(FSL_DPAA_BUS_NAME, rte_dpaa_bus.bus);
 
-RTE_INIT(dpaa_init_log);
-static void
-dpaa_init_log(void)
+RTE_INIT(dpaa_init_log)
 {
 	dpaa_logtype_bus = rte_log_register("bus.dpaa");
 	if (dpaa_logtype_bus >= 0)
diff --git a/drivers/bus/dpaa/include/compat.h b/drivers/bus/dpaa/include/compat.h
index e4b57021..92241d23 100644
--- a/drivers/bus/dpaa/include/compat.h
+++ b/drivers/bus/dpaa/include/compat.h
@@ -48,9 +48,15 @@
  */
 
 /* Required compiler attributes */
+#ifndef __maybe_unused
 #define __maybe_unused	__rte_unused
+#endif
+#ifndef __always_unused
 #define __always_unused	__rte_unused
+#endif
+#ifndef __packed
 #define __packed	__rte_packed
+#endif
 #define noinline	__attribute__((noinline))
 
 #define L1_CACHE_BYTES 64
diff --git a/drivers/bus/dpaa/include/fsl_fman.h b/drivers/bus/dpaa/include/fsl_fman.h
index c0ef1bff..1d1ce867 100644
--- a/drivers/bus/dpaa/include/fsl_fman.h
+++ b/drivers/bus/dpaa/include/fsl_fman.h
@@ -108,6 +108,12 @@ int fman_if_get_fdoff(struct fman_if *fm_if);
 /* Set interface fd->offset value */
 void fman_if_set_fdoff(struct fman_if *fm_if, uint32_t fd_offset);
 
+/* Get interface SG enable status value */
+int fman_if_get_sg_enable(struct fman_if *fm_if);
+
+/* Set interface SG support mode */
+void fman_if_set_sg(struct fman_if *fm_if, int enable);
+
 /* Get interface Max Frame length (MTU) */
 uint16_t fman_if_get_maxfrm(struct fman_if *fm_if);
 
diff --git a/drivers/bus/dpaa/include/fsl_qman.h b/drivers/bus/dpaa/include/fsl_qman.h
index e4ad7ae4..b18cf037 100644
--- a/drivers/bus/dpaa/include/fsl_qman.h
+++ b/drivers/bus/dpaa/include/fsl_qman.h
@@ -1332,10 +1332,11 @@ unsigned int qman_portal_poll_rx(unsigned int poll_limit,
  * qman_set_vdq - Issue a volatile dequeue command
  * @fq: Frame Queue on which the volatile dequeue command is issued
  * @num: Number of Frames requested for volatile dequeue
+ * @vdqcr_flags: QM_VDQCR_EXACT flag to for VDQCR command
  *
  * This function will issue a volatile dequeue command to the QMAN.
  */
-int qman_set_vdq(struct qman_fq *fq, u16 num);
+int qman_set_vdq(struct qman_fq *fq, u16 num, uint32_t vdqcr_flags);
 
 /**
  * qman_dequeue - Get the DQRR entry after volatile dequeue command
diff --git a/drivers/bus/dpaa/include/of.h b/drivers/bus/dpaa/include/of.h
index 151be5a3..7ea7608f 100644
--- a/drivers/bus/dpaa/include/of.h
+++ b/drivers/bus/dpaa/include/of.h
@@ -109,6 +109,8 @@ const struct device_node *of_get_parent(const struct device_node *dev_node);
 const struct device_node *of_get_next_child(const struct device_node *dev_node,
 					    const struct device_node *prev);
 
+const void *of_get_mac_address(const struct device_node *np);
+
 #define for_each_child_node(parent, child) \
 	for (child = of_get_next_child(parent, NULL); child != NULL; \
 			child = of_get_next_child(parent, child))
diff --git a/drivers/bus/dpaa/rte_bus_dpaa_version.map b/drivers/bus/dpaa/rte_bus_dpaa_version.map
index 8d902854..7d6d6243 100644
--- a/drivers/bus/dpaa/rte_bus_dpaa_version.map
+++ b/drivers/bus/dpaa/rte_bus_dpaa_version.map
@@ -92,3 +92,13 @@ DPDK_18.02 {
 
 	local: *;
 } DPDK_17.11;
+
+DPDK_18.08 {
+	global:
+
+	fman_if_get_sg_enable;
+	fman_if_set_sg;
+	of_get_mac_address;
+
+	local: *;
+} DPDK_18.02;
diff --git a/drivers/bus/dpaa/rte_dpaa_bus.h b/drivers/bus/dpaa/rte_dpaa_bus.h
index 8573bd6e..15dc6a4a 100644
--- a/drivers/bus/dpaa/rte_dpaa_bus.h
+++ b/drivers/bus/dpaa/rte_dpaa_bus.h
@@ -164,8 +164,7 @@ void dpaa_portal_finish(void *arg);
 
 /** Helper for DPAA device registration from driver (eth, crypto) instance */
 #define RTE_PMD_REGISTER_DPAA(nm, dpaa_drv) \
-RTE_INIT(dpaainitfn_ ##nm); \
-static void dpaainitfn_ ##nm(void) \
+RTE_INIT(dpaainitfn_ ##nm) \
 {\
 	(dpaa_drv).driver.name = RTE_STR(nm);\
 	rte_dpaa_driver_register(&dpaa_drv); \
diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index c6301b20..d2900edc 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -519,9 +519,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
 
 RTE_REGISTER_BUS(FSLMC_BUS_NAME, rte_fslmc_bus.bus);
 
-RTE_INIT(fslmc_init_log);
-static void
-fslmc_init_log(void)
+RTE_INIT(fslmc_init_log)
 {
 	/* Bus level logs */
 	dpaa2_logtype_bus = rte_log_register("bus.fslmc");
diff --git a/drivers/bus/fslmc/fslmc_logs.h b/drivers/bus/fslmc/fslmc_logs.h
index 9750b8c8..dd74cb7d 100644
--- a/drivers/bus/fslmc/fslmc_logs.h
+++ b/drivers/bus/fslmc/fslmc_logs.h
@@ -18,7 +18,7 @@ extern int dpaa2_logtype_bus;
 	rte_log(RTE_LOG_DEBUG, dpaa2_logtype_bus, "fslmc: %s(): " fmt "\n", \
 		__func__, ##args)
 
-#define BUS_INIT_FUNC_TRACE() DPAA2_BUS_LOG(DEBUG, " >>")
+#define BUS_INIT_FUNC_TRACE() DPAA2_BUS_DEBUG(" >>")
 
 #define DPAA2_BUS_INFO(fmt, args...) \
 	DPAA2_BUS_LOG(INFO, fmt, ## args)
diff --git a/drivers/bus/fslmc/qbman/qbman_portal.c b/drivers/bus/fslmc/qbman/qbman_portal.c
index 713ec965..07145005 100644
--- a/drivers/bus/fslmc/qbman/qbman_portal.c
+++ b/drivers/bus/fslmc/qbman/qbman_portal.c
@@ -122,8 +122,7 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
 	p->vdq.valid_bit = QB_VALID_BIT;
 	p->dqrr.next_idx = 0;
 	p->dqrr.valid_bit = QB_VALID_BIT;
-	qman_version = p->desc.qman_version;
-	if ((qman_version & 0xFFFF0000) < QMAN_REV_4100) {
+	if ((p->desc.qman_version & 0xFFFF0000) < QMAN_REV_4100) {
 		p->dqrr.dqrr_size = 4;
 		p->dqrr.reset_bug = 1;
 	} else {
diff --git a/drivers/bus/fslmc/qbman/qbman_portal.h b/drivers/bus/fslmc/qbman/qbman_portal.h
index 8bff0b4f..dbea22a1 100644
--- a/drivers/bus/fslmc/qbman/qbman_portal.h
+++ b/drivers/bus/fslmc/qbman/qbman_portal.h
@@ -7,7 +7,6 @@
 #include "qbman_sys.h"
 #include <fsl_qbman_portal.h>
 
-uint32_t qman_version;
 #define QMAN_REV_4000   0x04000000
 #define QMAN_REV_4100   0x04010000
 #define QMAN_REV_4101   0x04010001
diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
index 33552b48..cea5b78f 100644
--- a/drivers/bus/fslmc/rte_fslmc.h
+++ b/drivers/bus/fslmc/rte_fslmc.h
@@ -173,8 +173,7 @@ void rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver);
 
 /** Helper for DPAA2 device registration from driver (eth, crypto) instance */
 #define RTE_PMD_REGISTER_DPAA2(nm, dpaa2_drv) \
-RTE_INIT(dpaa2initfn_ ##nm); \
-static void dpaa2initfn_ ##nm(void) \
+RTE_INIT(dpaa2initfn_ ##nm) \
 {\
 	(dpaa2_drv).driver.name = RTE_STR(nm);\
 	rte_fslmc_driver_register(&dpaa2_drv); \
@@ -203,8 +202,7 @@ uint32_t rte_fslmc_get_device_count(enum rte_dpaa2_dev_type device_type);
 
 /** Helper for DPAA2 object registration */
 #define RTE_PMD_REGISTER_DPAA2_OBJECT(nm, dpaa2_obj) \
-RTE_INIT(dpaa2objinitfn_ ##nm); \
-static void dpaa2objinitfn_ ##nm(void) \
+RTE_INIT(dpaa2objinitfn_ ##nm) \
 {\
 	(dpaa2_obj).name = RTE_STR(nm);\
 	rte_fslmc_object_register(&dpaa2_obj); \
diff --git a/drivers/bus/ifpga/rte_bus_ifpga.h b/drivers/bus/ifpga/rte_bus_ifpga.h
index 981bc352..51d5ae0d 100644
--- a/drivers/bus/ifpga/rte_bus_ifpga.h
+++ b/drivers/bus/ifpga/rte_bus_ifpga.h
@@ -134,9 +134,8 @@ void rte_ifpga_driver_register(struct rte_afu_driver *driver);
 void rte_ifpga_driver_unregister(struct rte_afu_driver *driver);
 
 #define RTE_PMD_REGISTER_AFU(nm, afudrv)\
-RTE_INIT(afudrvinitfn_ ##afudrv);\
 static const char *afudrvinit_ ## nm ## _alias;\
-static void afudrvinitfn_ ##afudrv(void)\
+RTE_INIT(afudrvinitfn_ ##afudrv)\
 {\
 	(afudrv).driver.name = RTE_STR(nm);\
 	(afudrv).driver.alias = afudrvinit_ ## nm ## _alias;\
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 52c755dc..80de2d91 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['dpaa', 'fslmc', 'ifpga', 'pci', 'vdev']
+drivers = ['dpaa', 'fslmc', 'ifpga', 'pci', 'vdev', 'vmbus']
 std_deps = ['eal']
 config_flag_fmt = 'RTE_LIBRTE_@0@_BUS'
 driver_name_fmt = 'rte_bus_@0@'
diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c
index 004600f1..04648ac9 100644
--- a/drivers/bus/pci/linux/pci.c
+++ b/drivers/bus/pci/linux/pci.c
@@ -15,7 +15,6 @@
 #include <rte_memcpy.h>
 #include <rte_vfio.h>
 
-#include "eal_private.h"
 #include "eal_filesystem.h"
 
 #include "private.h"
diff --git a/drivers/bus/pci/linux/pci_uio.c b/drivers/bus/pci/linux/pci_uio.c
index d423e4bb..a7c14421 100644
--- a/drivers/bus/pci/linux/pci_uio.c
+++ b/drivers/bus/pci/linux/pci_uio.c
@@ -282,22 +282,19 @@ int
 pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx,
 		struct mapped_pci_resource *uio_res, int map_idx)
 {
-	int fd;
+	int fd = -1;
 	char devname[PATH_MAX];
 	void *mapaddr;
 	struct rte_pci_addr *loc;
 	struct pci_map *maps;
+	int wc_activate = 0;
+
+	if (dev->driver != NULL)
+		wc_activate = dev->driver->drv_flags & RTE_PCI_DRV_WC_ACTIVATE;
 
 	loc = &dev->addr;
 	maps = uio_res->maps;
 
-	/* update devname for mmap  */
-	snprintf(devname, sizeof(devname),
-			"%s/" PCI_PRI_FMT "/resource%d",
-			rte_pci_get_sysfs_path(),
-			loc->domain, loc->bus, loc->devid,
-			loc->function, res_idx);
-
 	/* allocate memory to keep path */
 	maps[map_idx].path = rte_malloc(NULL, strlen(devname) + 1, 0);
 	if (maps[map_idx].path == NULL) {
@@ -309,11 +306,37 @@ pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx,
 	/*
 	 * open resource file, to mmap it
 	 */
-	fd = open(devname, O_RDWR);
-	if (fd < 0) {
-		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
+	if (wc_activate) {
+		/* update devname for mmap  */
+		snprintf(devname, sizeof(devname),
+			"%s/" PCI_PRI_FMT "/resource%d_wc",
+			rte_pci_get_sysfs_path(),
+			loc->domain, loc->bus, loc->devid,
+			loc->function, res_idx);
+
+		if (access(devname, R_OK|W_OK) != -1) {
+			fd = open(devname, O_RDWR);
+			if (fd < 0)
+				RTE_LOG(INFO, EAL, "%s cannot be mapped. "
+					"Fall-back to non prefetchable mode.\n",
+					devname);
+		}
+	}
+
+	if (!wc_activate || fd < 0) {
+		snprintf(devname, sizeof(devname),
+			"%s/" PCI_PRI_FMT "/resource%d",
+			rte_pci_get_sysfs_path(),
+			loc->domain, loc->bus, loc->devid,
+			loc->function, res_idx);
+
+		/* then try to map resource file */
+		fd = open(devname, O_RDWR);
+		if (fd < 0) {
+			RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
 				devname, strerror(errno));
-		goto error;
+			goto error;
+		}
 	}
 
 	/* try mapping somewhere close to the end of hugepages */
diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index aeeaa9ed..686386d6 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -584,6 +584,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		dev->mem_resource[i].addr = maps[i].addr;
 	}
 
+	/* we need save vfio_dev_fd, so it can be used during release */
+	dev->intr_handle.vfio_dev_fd = vfio_dev_fd;
+
 	return 0;
 err_vfio_dev_fd:
 	close(vfio_dev_fd);
@@ -603,22 +606,58 @@ pci_vfio_map_resource(struct rte_pci_device *dev)
 		return pci_vfio_map_resource_secondary(dev);
 }
 
-int
-pci_vfio_unmap_resource(struct rte_pci_device *dev)
+static struct mapped_pci_resource *
+find_and_unmap_vfio_resource(struct mapped_pci_res_list *vfio_res_list,
+			struct rte_pci_device *dev,
+			const char *pci_addr)
+{
+	struct mapped_pci_resource *vfio_res = NULL;
+	struct pci_map *maps;
+	int i;
+
+	/* Get vfio_res */
+	TAILQ_FOREACH(vfio_res, vfio_res_list, next) {
+		if (rte_pci_addr_cmp(&vfio_res->pci_addr, &dev->addr))
+			continue;
+		break;
+	}
+
+	if  (vfio_res == NULL)
+		return vfio_res;
+
+	RTE_LOG(INFO, EAL, "Releasing pci mapped resource for %s\n",
+		pci_addr);
+
+	maps = vfio_res->maps;
+	for (i = 0; i < (int) vfio_res->nb_maps; i++) {
+
+		/*
+		 * We do not need to be aware of MSI-X table BAR mappings as
+		 * when mapping. Just using current maps array is enough
+		 */
+		if (maps[i].addr) {
+			RTE_LOG(INFO, EAL, "Calling pci_unmap_resource for %s at %p\n",
+				pci_addr, maps[i].addr);
+			pci_unmap_resource(maps[i].addr, maps[i].size);
+		}
+	}
+
+	return vfio_res;
+}
+
+static int
+pci_vfio_unmap_resource_primary(struct rte_pci_device *dev)
 {
 	char pci_addr[PATH_MAX] = {0};
 	struct rte_pci_addr *loc = &dev->addr;
-	int i, ret;
 	struct mapped_pci_resource *vfio_res = NULL;
 	struct mapped_pci_res_list *vfio_res_list;
-
-	struct pci_map *maps;
+	int ret;
 
 	/* store PCI address string */
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
-
 	if (close(dev->intr_handle.fd) < 0) {
 		RTE_LOG(INFO, EAL, "Error when closing eventfd file descriptor for %s\n",
 			pci_addr);
@@ -639,13 +678,10 @@ pci_vfio_unmap_resource(struct rte_pci_device *dev)
 		return ret;
 	}
 
-	vfio_res_list = RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list);
-	/* Get vfio_res */
-	TAILQ_FOREACH(vfio_res, vfio_res_list, next) {
-		if (memcmp(&vfio_res->pci_addr, &dev->addr, sizeof(dev->addr)))
-			continue;
-		break;
-	}
+	vfio_res_list =
+		RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list);
+	vfio_res = find_and_unmap_vfio_resource(vfio_res_list, dev, pci_addr);
+
 	/* if we haven't found our tailq entry, something's wrong */
 	if (vfio_res == NULL) {
 		RTE_LOG(ERR, EAL, "  %s cannot find TAILQ entry for PCI device!\n",
@@ -653,30 +689,56 @@ pci_vfio_unmap_resource(struct rte_pci_device *dev)
 		return -1;
 	}
 
-	/* unmap BARs */
-	maps = vfio_res->maps;
+	TAILQ_REMOVE(vfio_res_list, vfio_res, next);
 
-	RTE_LOG(INFO, EAL, "Releasing pci mapped resource for %s\n",
-		pci_addr);
-	for (i = 0; i < (int) vfio_res->nb_maps; i++) {
+	return 0;
+}
 
-		/*
-		 * We do not need to be aware of MSI-X table BAR mappings as
-		 * when mapping. Just using current maps array is enough
-		 */
-		if (maps[i].addr) {
-			RTE_LOG(INFO, EAL, "Calling pci_unmap_resource for %s at %p\n",
-				pci_addr, maps[i].addr);
-			pci_unmap_resource(maps[i].addr, maps[i].size);
-		}
+static int
+pci_vfio_unmap_resource_secondary(struct rte_pci_device *dev)
+{
+	char pci_addr[PATH_MAX] = {0};
+	struct rte_pci_addr *loc = &dev->addr;
+	struct mapped_pci_resource *vfio_res = NULL;
+	struct mapped_pci_res_list *vfio_res_list;
+	int ret;
+
+	/* store PCI address string */
+	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
+			loc->domain, loc->bus, loc->devid, loc->function);
+
+	ret = rte_vfio_release_device(rte_pci_get_sysfs_path(), pci_addr,
+				  dev->intr_handle.vfio_dev_fd);
+	if (ret < 0) {
+		RTE_LOG(ERR, EAL,
+			"%s(): cannot release device\n", __func__);
+		return ret;
 	}
 
-	TAILQ_REMOVE(vfio_res_list, vfio_res, next);
+	vfio_res_list =
+		RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list);
+	vfio_res = find_and_unmap_vfio_resource(vfio_res_list, dev, pci_addr);
+
+	/* if we haven't found our tailq entry, something's wrong */
+	if (vfio_res == NULL) {
+		RTE_LOG(ERR, EAL, "  %s cannot find TAILQ entry for PCI device!\n",
+				pci_addr);
+		return -1;
+	}
 
 	return 0;
 }
 
 int
+pci_vfio_unmap_resource(struct rte_pci_device *dev)
+{
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+		return pci_vfio_unmap_resource_primary(dev);
+	else
+		return pci_vfio_unmap_resource_secondary(dev);
+}
+
+int
 pci_vfio_ioport_map(struct rte_pci_device *dev, int bar,
 		    struct rte_pci_ioport *p)
 {
diff --git a/drivers/bus/pci/pci_common.c b/drivers/bus/pci/pci_common.c
index 7215aaec..7736b3f9 100644
--- a/drivers/bus/pci/pci_common.c
+++ b/drivers/bus/pci/pci_common.c
@@ -26,6 +26,7 @@
 
 #include "private.h"
 
+
 extern struct rte_pci_bus rte_pci_bus;
 
 #define SYSFS_PCI_DEVICES "/sys/bus/pci/devices"
@@ -69,7 +70,7 @@ pci_name_set(struct rte_pci_device *dev)
 	 */
 	if (devargs != NULL)
 		/* If an rte_devargs exists, the generic rte_device uses the
-		 * given name as its namea
+		 * given name as its name.
 		 */
 		dev->device.name = dev->device.devargs->name;
 	else
@@ -155,17 +156,24 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr,
 	RTE_LOG(INFO, EAL, "  probe driver: %x:%x %s\n", dev->id.vendor_id,
 		dev->id.device_id, dr->driver.name);
 
+	/*
+	 * reference driver structure
+	 * This needs to be before rte_pci_map_device(), as it enables to use
+	 * driver flags for adjusting configuration.
+	 */
+	dev->driver = dr;
+	dev->device.driver = &dr->driver;
+
 	if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) {
 		/* map resources for devices that use igb_uio */
 		ret = rte_pci_map_device(dev);
-		if (ret != 0)
+		if (ret != 0) {
+			dev->driver = NULL;
+			dev->device.driver = NULL;
 			return ret;
+		}
 	}
 
-	/* reference driver structure */
-	dev->driver = dr;
-	dev->device.driver = &dr->driver;
-
 	/* call the driver probe() function */
 	ret = dr->probe(dr, dev);
 	if (ret) {
@@ -255,81 +263,6 @@ pci_probe_all_drivers(struct rte_pci_device *dev)
 }
 
 /*
- * Find the pci device specified by pci address, then invoke probe function of
- * the driver of the device.
- */
-int
-rte_pci_probe_one(const struct rte_pci_addr *addr)
-{
-	struct rte_pci_device *dev = NULL;
-
-	int ret = 0;
-
-	if (addr == NULL)
-		return -1;
-
-	/* update current pci device in global list, kernel bindings might have
-	 * changed since last time we looked at it.
-	 */
-	if (pci_update_device(addr) < 0)
-		goto err_return;
-
-	FOREACH_DEVICE_ON_PCIBUS(dev) {
-		if (rte_pci_addr_cmp(&dev->addr, addr))
-			continue;
-
-		ret = pci_probe_all_drivers(dev);
-		if (ret)
-			goto err_return;
-		return 0;
-	}
-	return -1;
-
-err_return:
-	RTE_LOG(WARNING, EAL,
-		"Requested device " PCI_PRI_FMT " cannot be used\n",
-		addr->domain, addr->bus, addr->devid, addr->function);
-	return -1;
-}
-
-/*
- * Detach device specified by its pci address.
- */
-int
-rte_pci_detach(const struct rte_pci_addr *addr)
-{
-	struct rte_pci_device *dev = NULL;
-	int ret = 0;
-
-	if (addr == NULL)
-		return -1;
-
-	FOREACH_DEVICE_ON_PCIBUS(dev) {
-		if (rte_pci_addr_cmp(&dev->addr, addr))
-			continue;
-
-		ret = rte_pci_detach_dev(dev);
-		if (ret < 0)
-			/* negative value is an error */
-			goto err_return;
-		if (ret > 0)
-			/* positive value means driver doesn't support it */
-			continue;
-
-		rte_pci_remove_device(dev);
-		free(dev);
-		return 0;
-	}
-	return -1;
-
-err_return:
-	RTE_LOG(WARNING, EAL, "Requested device " PCI_PRI_FMT
-			" cannot be used\n", dev->addr.domain, dev->addr.bus,
-			dev->addr.devid, dev->addr.function);
-	return -1;
-}
-
-/*
  * Scan the content of the PCI bus, and call the probe() function for
  * all registered drivers that have a matching entry in its id_table
  * for discovered devices.
@@ -445,7 +378,7 @@ rte_pci_insert_device(struct rte_pci_device *exist_pci_dev,
 }
 
 /* Remove a device from PCI bus */
-void
+static void
 rte_pci_remove_device(struct rte_pci_device *pci_dev)
 {
 	TAILQ_REMOVE(&rte_pci_bus.device_list, pci_dev, next);
diff --git a/drivers/bus/pci/private.h b/drivers/bus/pci/private.h
index 88fa587e..8ddd03e1 100644
--- a/drivers/bus/pci/private.h
+++ b/drivers/bus/pci/private.h
@@ -33,36 +33,6 @@ rte_pci_probe(void);
 int rte_pci_scan(void);
 
 /**
- * Probe the single PCI device.
- *
- * Scan the content of the PCI bus, and find the pci device specified by pci
- * address, then call the probe() function for registered driver that has a
- * matching entry in its id_table for discovered device.
- *
- * @param addr
- *	The PCI Bus-Device-Function address to probe.
- * @return
- *   - 0 on success.
- *   - Negative on error.
- */
-int rte_pci_probe_one(const struct rte_pci_addr *addr);
-
-/**
- * Close the single PCI device.
- *
- * Scan the content of the PCI bus, and find the pci device specified by pci
- * address, then call the remove() function for registered driver that has a
- * matching entry in its id_table for discovered device.
- *
- * @param addr
- *	The PCI Bus-Device-Function address to close.
- * @return
- *   - 0 on success.
- *   - Negative on error.
- */
-int rte_pci_detach(const struct rte_pci_addr *addr);
-
-/**
  * Find the name of a PCI device.
  */
 void
@@ -94,16 +64,6 @@ void rte_pci_insert_device(struct rte_pci_device *exist_pci_dev,
 		struct rte_pci_device *new_pci_dev);
 
 /**
- * Remove a PCI device from the PCI Bus. This sets to NULL the bus references
- * in the PCI device object as well as the generic device object.
- *
- * @param pci_device
- *	PCI device to be removed from PCI Bus
- * @return void
- */
-void rte_pci_remove_device(struct rte_pci_device *pci_device);
-
-/**
  * Update a pci device object by asking the kernel for the latest information.
  *
  * This function is private to EAL.
@@ -117,16 +77,6 @@ void rte_pci_remove_device(struct rte_pci_device *pci_device);
 int pci_update_device(const struct rte_pci_addr *addr);
 
 /**
- * Unbind kernel driver for this device
- *
- * This function is private to EAL.
- *
- * @return
- *   0 on success, negative on error
- */
-int pci_unbind_kernel_driver(struct rte_pci_device *dev);
-
-/**
  * Map the PCI resource of a PCI device in virtual memory
  *
  * This function is private to EAL.
diff --git a/drivers/bus/pci/rte_bus_pci.h b/drivers/bus/pci/rte_bus_pci.h
index 458e6d07..0d1955ff 100644
--- a/drivers/bus/pci/rte_bus_pci.h
+++ b/drivers/bus/pci/rte_bus_pci.h
@@ -135,6 +135,8 @@ struct rte_pci_bus {
 
 /** Device needs PCI BAR mapping (done with either IGB_UIO or VFIO) */
 #define RTE_PCI_DRV_NEED_MAPPING 0x0001
+/** Device needs PCI BAR mapping with enabled write combining (wc) */
+#define RTE_PCI_DRV_WC_ACTIVATE 0x0002
 /** Device driver supports link state interrupt */
 #define RTE_PCI_DRV_INTR_LSC	0x0008
 /** Device driver supports device removal interrupt */
@@ -189,8 +191,7 @@ void rte_pci_register(struct rte_pci_driver *driver);
 
 /** Helper for PCI device registration from driver (eth, crypto) instance */
 #define RTE_PMD_REGISTER_PCI(nm, pci_drv) \
-RTE_INIT(pciinitfn_ ##nm); \
-static void pciinitfn_ ##nm(void) \
+RTE_INIT(pciinitfn_ ##nm) \
 {\
 	(pci_drv).driver.name = RTE_STR(nm);\
 	rte_pci_register(&pci_drv); \
diff --git a/drivers/bus/vdev/rte_bus_vdev.h b/drivers/bus/vdev/rte_bus_vdev.h
index f9b5eb59..9ae3eaae 100644
--- a/drivers/bus/vdev/rte_bus_vdev.h
+++ b/drivers/bus/vdev/rte_bus_vdev.h
@@ -86,9 +86,8 @@ void rte_vdev_register(struct rte_vdev_driver *driver);
 void rte_vdev_unregister(struct rte_vdev_driver *driver);
 
 #define RTE_PMD_REGISTER_VDEV(nm, vdrv)\
-RTE_INIT(vdrvinitfn_ ##vdrv);\
 static const char *vdrvinit_ ## nm ## _alias;\
-static void vdrvinitfn_ ##vdrv(void)\
+RTE_INIT(vdrvinitfn_ ##vdrv)\
 {\
 	(vdrv).driver.name = RTE_STR(nm);\
 	(vdrv).driver.alias = vdrvinit_ ## nm ## _alias;\
diff --git a/drivers/bus/vmbus/Makefile b/drivers/bus/vmbus/Makefile
new file mode 100644
index 00000000..deee9dd1
--- /dev/null
+++ b/drivers/bus/vmbus/Makefile
@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+LIB = librte_bus_vmbus.a
+LIBABIVER := 1
+EXPORT_MAP := rte_bus_vmbus_version.map
+
+CFLAGS += -I$(SRCDIR)
+CFLAGS += -O3 $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),)
+SYSTEM := linux
+endif
+ifneq ($(CONFIG_RTE_EXEC_ENV_BSDAPP),)
+$(error "VMBUS not implemented for BSD yet")
+endif
+
+CFLAGS += -I$(RTE_SDK)/drivers/bus/vmbus/$(SYSTEM)
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/$(SYSTEM)app/eal
+
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_ethdev
+
+include $(RTE_SDK)/drivers/bus/vmbus/$(SYSTEM)/Makefile
+SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) := $(addprefix $(SYSTEM)/,$(SRCS))
+SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus_common.c
+SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus_channel.c vmbus_bufring.c
+SRCS-$(CONFIG_RTE_LIBRTE_VMBUS) += vmbus_common_uio.c
+
+SYMLINK-$(CONFIG_RTE_LIBRTE_VMBUS)-include += rte_bus_vmbus.h
+SYMLINK-$(CONFIG_RTE_LIBRTE_VMBUS)-include += rte_vmbus_reg.h
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/bus/vmbus/linux/Makefile b/drivers/bus/vmbus/linux/Makefile
new file mode 100644
index 00000000..ef0d30b2
--- /dev/null
+++ b/drivers/bus/vmbus/linux/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+SRCS += vmbus_bus.c vmbus_uio.c
diff --git a/drivers/bus/vmbus/linux/vmbus_bus.c b/drivers/bus/vmbus/linux/vmbus_bus.c
new file mode 100644
index 00000000..52d6a3c0
--- /dev/null
+++ b/drivers/bus/vmbus/linux/vmbus_bus.c
@@ -0,0 +1,355 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#include <string.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+
+#include <rte_eal.h>
+#include <rte_uuid.h>
+#include <rte_tailq.h>
+#include <rte_log.h>
+#include <rte_devargs.h>
+#include <rte_memory.h>
+#include <rte_malloc.h>
+#include <rte_bus_vmbus.h>
+
+#include "eal_filesystem.h"
+#include "private.h"
+
+/** Pathname of VMBUS devices directory. */
+#define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices"
+
+extern struct rte_vmbus_bus rte_vmbus_bus;
+
+/* Read sysfs file to get UUID */
+static int
+parse_sysfs_uuid(const char *filename, rte_uuid_t uu)
+{
+	char buf[BUFSIZ];
+	char *cp, *in = buf;
+	FILE *f;
+
+	f = fopen(filename, "r");
+	if (f == NULL) {
+		VMBUS_LOG(ERR, "cannot open sysfs value %s: %s",
+			  filename, strerror(errno));
+		return -1;
+	}
+
+	if (fgets(buf, sizeof(buf), f) == NULL) {
+		VMBUS_LOG(ERR, "cannot read sysfs value %s",
+				filename);
+		fclose(f);
+		return -1;
+	}
+	fclose(f);
+
+	cp = strchr(buf, '\n');
+	if (cp)
+		*cp = '\0';
+
+	/* strip { } notation */
+	if (buf[0] == '{') {
+		in = buf + 1;
+		cp = strchr(in, '}');
+		if (cp)
+			*cp = '\0';
+	}
+
+	if (rte_uuid_parse(in, uu) < 0) {
+		VMBUS_LOG(ERR, "%s %s not a valid UUID",
+			filename, buf);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+get_sysfs_string(const char *filename, char *buf, size_t buflen)
+{
+	char *cp;
+	FILE *f;
+
+	f = fopen(filename, "r");
+	if (f == NULL) {
+		VMBUS_LOG(ERR, "cannot open sysfs value %s:%s",
+			  filename, strerror(errno));
+		return -1;
+	}
+
+	if (fgets(buf, buflen, f) == NULL) {
+		VMBUS_LOG(ERR, "cannot read sysfs value %s",
+				filename);
+		fclose(f);
+		return -1;
+	}
+	fclose(f);
+
+	/* remove trailing newline */
+	cp = memchr(buf, '\n', buflen);
+	if (cp)
+		*cp = '\0';
+
+	return 0;
+}
+
+static int
+vmbus_get_uio_dev(const struct rte_vmbus_device *dev,
+		  char *dstbuf, size_t buflen)
+{
+	char dirname[PATH_MAX];
+	unsigned int uio_num;
+	struct dirent *e;
+	DIR *dir;
+
+	/* Assume recent kernel where uio is in uio/uioX */
+	snprintf(dirname, sizeof(dirname),
+		 SYSFS_VMBUS_DEVICES "/%s/uio", dev->device.name);
+
+	dir = opendir(dirname);
+	if (dir == NULL)
+		return -1; /* Not a UIO device */
+
+	/* take the first file starting with "uio" */
+	while ((e = readdir(dir)) != NULL) {
+		const int prefix_len = 3;
+		char *endptr;
+
+		if (strncmp(e->d_name, "uio", prefix_len) != 0)
+			continue;
+
+		/* try uio%d */
+		errno = 0;
+		uio_num = strtoull(e->d_name + prefix_len, &endptr, 10);
+		if (errno == 0 && endptr != (e->d_name + prefix_len)) {
+			snprintf(dstbuf, buflen, "%s/uio%u", dirname, uio_num);
+			break;
+		}
+	}
+	closedir(dir);
+
+	if (e == NULL)
+		return -1;
+
+	return uio_num;
+}
+
+/* Check map names with kernel names */
+static const char *map_names[VMBUS_MAX_RESOURCE] = {
+	[HV_TXRX_RING_MAP] = "txrx_rings",
+	[HV_INT_PAGE_MAP]  = "int_page",
+	[HV_MON_PAGE_MAP]  = "monitor_page",
+	[HV_RECV_BUF_MAP]  = "recv:",
+	[HV_SEND_BUF_MAP]  = "send:",
+};
+
+
+/* map the resources of a vmbus device in virtual memory */
+int
+rte_vmbus_map_device(struct rte_vmbus_device *dev)
+{
+	char uioname[PATH_MAX], filename[PATH_MAX];
+	char dirname[PATH_MAX], mapname[64];
+	int i;
+
+	dev->uio_num = vmbus_get_uio_dev(dev, uioname, sizeof(uioname));
+	if (dev->uio_num < 0) {
+		VMBUS_LOG(DEBUG, "Not managed by UIO driver, skipped");
+		return 1;
+	}
+
+	/* Extract resource value */
+	for (i = 0; i < VMBUS_MAX_RESOURCE; i++) {
+		struct rte_mem_resource *res = &dev->resource[i];
+		unsigned long len, gpad = 0;
+		char *cp;
+
+		snprintf(dirname, sizeof(dirname),
+			 "%s/maps/map%d", uioname, i);
+
+		snprintf(filename, sizeof(filename),
+			 "%s/name", dirname);
+
+		if (get_sysfs_string(filename, mapname, sizeof(mapname)) < 0) {
+			VMBUS_LOG(ERR, "could not read %s", filename);
+			return -1;
+		}
+
+		if (strncmp(map_names[i], mapname, strlen(map_names[i])) != 0) {
+			VMBUS_LOG(ERR,
+				"unexpected resource %s (expected %s)",
+				mapname, map_names[i]);
+			return -1;
+		}
+
+		snprintf(filename, sizeof(filename),
+			 "%s/size", dirname);
+		if (eal_parse_sysfs_value(filename, &len) < 0) {
+			VMBUS_LOG(ERR,
+				"could not read %s", filename);
+			return -1;
+		}
+		res->len = len;
+
+		/* both send and receive buffers have gpad in name */
+		cp = memchr(mapname, ':', sizeof(mapname));
+		if (cp)
+			gpad = strtoul(cp+1, NULL, 0);
+
+		/* put the GPAD value in physical address */
+		res->phys_addr = gpad;
+	}
+
+	return vmbus_uio_map_resource(dev);
+}
+
+void
+rte_vmbus_unmap_device(struct rte_vmbus_device *dev)
+{
+	vmbus_uio_unmap_resource(dev);
+}
+
+/* Scan one vmbus sysfs entry, and fill the devices list from it. */
+static int
+vmbus_scan_one(const char *name)
+{
+	struct rte_vmbus_device *dev, *dev2;
+	char filename[PATH_MAX];
+	char dirname[PATH_MAX];
+	unsigned long tmp;
+
+	dev = calloc(1, sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	dev->device.name = strdup(name);
+	if (!dev->device.name)
+		goto error;
+
+	/* sysfs base directory
+	 *   /sys/bus/vmbus/devices/7a08391f-f5a0-4ac0-9802-d13fd964f8df
+	 * or on older kernel
+	 *   /sys/bus/vmbus/devices/vmbus_1
+	 */
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 SYSFS_VMBUS_DEVICES, name);
+
+	/* get device id */
+	snprintf(filename, sizeof(filename), "%s/device_id", dirname);
+	if (parse_sysfs_uuid(filename, dev->device_id) < 0)
+		goto error;
+
+	/* get device class  */
+	snprintf(filename, sizeof(filename), "%s/class_id", dirname);
+	if (parse_sysfs_uuid(filename, dev->class_id) < 0)
+		goto error;
+
+	/* get relid */
+	snprintf(filename, sizeof(filename), "%s/id", dirname);
+	if (eal_parse_sysfs_value(filename, &tmp) < 0)
+		goto error;
+	dev->relid = tmp;
+
+	/* get monitor id */
+	snprintf(filename, sizeof(filename), "%s/monitor_id", dirname);
+	if (eal_parse_sysfs_value(filename, &tmp) < 0)
+		goto error;
+	dev->monitor_id = tmp;
+
+	/* get numa node (if present) */
+	snprintf(filename, sizeof(filename), "%s/numa_node",
+		 dirname);
+
+	if (access(filename, R_OK) == 0) {
+		if (eal_parse_sysfs_value(filename, &tmp) < 0)
+			goto error;
+		dev->device.numa_node = tmp;
+	} else {
+		/* if no NUMA support, set default to 0 */
+		dev->device.numa_node = SOCKET_ID_ANY;
+	}
+
+	/* device is valid, add in list (sorted) */
+	VMBUS_LOG(DEBUG, "Adding vmbus device %s", name);
+
+	TAILQ_FOREACH(dev2, &rte_vmbus_bus.device_list, next) {
+		int ret;
+
+		ret = rte_uuid_compare(dev->device_id, dev2->device_id);
+		if (ret > 0)
+			continue;
+
+		if (ret < 0) {
+			vmbus_insert_device(dev2, dev);
+		} else { /* already registered */
+			VMBUS_LOG(NOTICE,
+				"%s already registered", name);
+			free(dev);
+		}
+		return 0;
+	}
+
+	vmbus_add_device(dev);
+	return 0;
+error:
+	VMBUS_LOG(DEBUG, "failed");
+
+	free(dev);
+	return -1;
+}
+
+/*
+ * Scan the content of the vmbus, and the devices in the devices list
+ */
+int
+rte_vmbus_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+
+	dir = opendir(SYSFS_VMBUS_DEVICES);
+	if (dir == NULL) {
+		if (errno == ENOENT)
+			return 0;
+
+		VMBUS_LOG(ERR, "opendir %s failed: %s",
+			  SYSFS_VMBUS_DEVICES, strerror(errno));
+		return -1;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (vmbus_scan_one(e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
+
+void rte_vmbus_irq_mask(struct rte_vmbus_device *device)
+{
+	vmbus_uio_irq_control(device, 1);
+}
+
+void rte_vmbus_irq_unmask(struct rte_vmbus_device *device)
+{
+	vmbus_uio_irq_control(device, 0);
+}
+
+int rte_vmbus_irq_read(struct rte_vmbus_device *device)
+{
+	return vmbus_uio_irq_read(device);
+}
diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c
new file mode 100644
index 00000000..856c6d66
--- /dev/null
+++ b/drivers/bus/vmbus/linux/vmbus_uio.c
@@ -0,0 +1,398 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <inttypes.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_memory.h>
+#include <rte_eal_memconfig.h>
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_bus_vmbus.h>
+#include <rte_string_fns.h>
+
+#include "private.h"
+
+/** Pathname of VMBUS devices directory. */
+#define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices"
+
+static void *vmbus_map_addr;
+
+/* Control interrupts */
+void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff)
+{
+	if (write(dev->intr_handle.fd, &onoff, sizeof(onoff)) < 0) {
+		VMBUS_LOG(ERR, "cannot write to %d:%s",
+			dev->intr_handle.fd, strerror(errno));
+	}
+}
+
+int vmbus_uio_irq_read(struct rte_vmbus_device *dev)
+{
+	int32_t count;
+	int cc;
+
+	cc = read(dev->intr_handle.fd, &count, sizeof(count));
+	if (cc < (int)sizeof(count)) {
+		if (cc < 0) {
+			VMBUS_LOG(ERR, "IRQ read failed %s",
+				  strerror(errno));
+			return -errno;
+		}
+		VMBUS_LOG(ERR, "can't read IRQ count");
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+void
+vmbus_uio_free_resource(struct rte_vmbus_device *dev,
+		struct mapped_vmbus_resource *uio_res)
+{
+	rte_free(uio_res);
+
+	if (dev->intr_handle.uio_cfg_fd >= 0) {
+		close(dev->intr_handle.uio_cfg_fd);
+		dev->intr_handle.uio_cfg_fd = -1;
+	}
+
+	if (dev->intr_handle.fd >= 0) {
+		close(dev->intr_handle.fd);
+		dev->intr_handle.fd = -1;
+		dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
+	}
+}
+
+int
+vmbus_uio_alloc_resource(struct rte_vmbus_device *dev,
+			 struct mapped_vmbus_resource **uio_res)
+{
+	char devname[PATH_MAX]; /* contains the /dev/uioX */
+
+	/* save fd if in primary process */
+	snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num);
+	dev->intr_handle.fd = open(devname, O_RDWR);
+	if (dev->intr_handle.fd < 0) {
+		VMBUS_LOG(ERR, "Cannot open %s: %s",
+			devname, strerror(errno));
+		goto error;
+	}
+	dev->intr_handle.type = RTE_INTR_HANDLE_UIO_INTX;
+
+	/* allocate the mapping details for secondary processes*/
+	*uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0);
+	if (*uio_res == NULL) {
+		VMBUS_LOG(ERR, "cannot store uio mmap details");
+		goto error;
+	}
+
+	strlcpy((*uio_res)->path, devname, PATH_MAX);
+	rte_uuid_copy((*uio_res)->id, dev->device_id);
+
+	return 0;
+
+error:
+	vmbus_uio_free_resource(dev, *uio_res);
+	return -1;
+}
+
+static int
+find_max_end_va(const struct rte_memseg_list *msl, void *arg)
+{
+	size_t sz = msl->memseg_arr.len * msl->page_sz;
+	void *end_va = RTE_PTR_ADD(msl->base_va, sz);
+	void **max_va = arg;
+
+	if (*max_va < end_va)
+		*max_va = end_va;
+	return 0;
+}
+
+/*
+ * TODO: this should be part of memseg api.
+ *       code is duplicated from PCI.
+ */
+static void *
+vmbus_find_max_end_va(void)
+{
+	void *va = NULL;
+
+	rte_memseg_list_walk(find_max_end_va, &va);
+	return va;
+}
+
+int
+vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int idx,
+				struct mapped_vmbus_resource *uio_res,
+				int flags)
+{
+	size_t size = dev->resource[idx].len;
+	struct vmbus_map *maps = uio_res->maps;
+	void *mapaddr;
+	off_t offset;
+	int fd;
+
+	/* devname for mmap  */
+	fd = open(uio_res->path, O_RDWR);
+	if (fd < 0) {
+		VMBUS_LOG(ERR, "Cannot open %s: %s",
+			  uio_res->path, strerror(errno));
+		return -1;
+	}
+
+	/* try mapping somewhere close to the end of hugepages */
+	if (vmbus_map_addr == NULL)
+		vmbus_map_addr = vmbus_find_max_end_va();
+
+	/* offset is special in uio it indicates which resource */
+	offset = idx * PAGE_SIZE;
+
+	mapaddr = vmbus_map_resource(vmbus_map_addr, fd, offset, size, flags);
+	close(fd);
+
+	if (mapaddr == MAP_FAILED)
+		return -1;
+
+	dev->resource[idx].addr = mapaddr;
+	vmbus_map_addr = RTE_PTR_ADD(mapaddr, size);
+
+	/* Record result of sucessful mapping for use by secondary */
+	maps[idx].addr = mapaddr;
+	maps[idx].size = size;
+
+	return 0;
+}
+
+static int vmbus_uio_map_primary(struct vmbus_channel *chan,
+				 void **ring_buf, uint32_t *ring_size)
+{
+	struct mapped_vmbus_resource *uio_res;
+
+	uio_res = vmbus_uio_find_resource(chan->device);
+	if (!uio_res) {
+		VMBUS_LOG(ERR, "can not find resources!");
+		return -ENOMEM;
+	}
+
+	if (uio_res->nb_maps < VMBUS_MAX_RESOURCE) {
+		VMBUS_LOG(ERR, "VMBUS: only %u resources found!",
+			  uio_res->nb_maps);
+		return -EINVAL;
+	}
+
+	*ring_size = uio_res->maps[HV_TXRX_RING_MAP].size / 2;
+	*ring_buf  = uio_res->maps[HV_TXRX_RING_MAP].addr;
+	return 0;
+}
+
+static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev,
+				 const struct vmbus_channel *chan,
+				 void **ring_buf, uint32_t *ring_size)
+{
+	char ring_path[PATH_MAX];
+	size_t file_size;
+	struct stat sb;
+	int fd;
+
+	snprintf(ring_path, sizeof(ring_path),
+		 "%s/%s/channels/%u/ring",
+		 SYSFS_VMBUS_DEVICES, dev->device.name,
+		 chan->relid);
+
+	fd = open(ring_path, O_RDWR);
+	if (fd < 0) {
+		VMBUS_LOG(ERR, "Cannot open %s: %s",
+			  ring_path, strerror(errno));
+		return -errno;
+	}
+
+	if (fstat(fd, &sb) < 0) {
+		VMBUS_LOG(ERR, "Cannot state %s: %s",
+			  ring_path, strerror(errno));
+		close(fd);
+		return -errno;
+	}
+	file_size = sb.st_size;
+
+	if (file_size == 0 || (file_size & (PAGE_SIZE - 1))) {
+		VMBUS_LOG(ERR, "incorrect size %s: %zu",
+			  ring_path, file_size);
+
+		close(fd);
+		return -EINVAL;
+	}
+
+	*ring_size = file_size / 2;
+	*ring_buf = vmbus_map_resource(vmbus_map_addr, fd,
+				       0, sb.st_size, 0);
+	close(fd);
+
+	if (ring_buf == MAP_FAILED)
+		return -EIO;
+
+	vmbus_map_addr = RTE_PTR_ADD(ring_buf, file_size);
+	return 0;
+}
+
+int vmbus_uio_map_rings(struct vmbus_channel *chan)
+{
+	const struct rte_vmbus_device *dev = chan->device;
+	uint32_t ring_size;
+	void *ring_buf;
+	int ret;
+
+	/* Primary channel */
+	if (chan->subchannel_id == 0)
+		ret = vmbus_uio_map_primary(chan, &ring_buf, &ring_size);
+	else
+		ret = vmbus_uio_map_subchan(dev, chan, &ring_buf, &ring_size);
+
+	if (ret)
+		return ret;
+
+	vmbus_br_setup(&chan->txbr, ring_buf, ring_size);
+	vmbus_br_setup(&chan->rxbr, (char *)ring_buf + ring_size, ring_size);
+	return 0;
+}
+
+static int vmbus_uio_sysfs_read(const char *dir, const char *name,
+				unsigned long *val, unsigned long max_range)
+{
+	char path[PATH_MAX];
+	FILE *f;
+	int ret;
+
+	snprintf(path, sizeof(path), "%s/%s", dir, name);
+	f = fopen(path, "r");
+	if (!f) {
+		VMBUS_LOG(ERR, "can't open %s:%s",
+			  path, strerror(errno));
+		return -errno;
+	}
+
+	if (fscanf(f, "%lu", val) != 1)
+		ret = -EIO;
+	else if (*val > max_range)
+		ret = -ERANGE;
+	else
+		ret = 0;
+	fclose(f);
+
+	return ret;
+}
+
+static bool vmbus_uio_ring_present(const struct rte_vmbus_device *dev,
+				   uint32_t relid)
+{
+	char ring_path[PATH_MAX];
+
+	/* Check if kernel has subchannel sysfs files */
+	snprintf(ring_path, sizeof(ring_path),
+		 "%s/%s/channels/%u/ring",
+		 SYSFS_VMBUS_DEVICES, dev->device.name, relid);
+
+	return access(ring_path, R_OK|W_OK) == 0;
+}
+
+bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev,
+				     const struct vmbus_channel *chan)
+{
+	return vmbus_uio_ring_present(dev, chan->relid);
+}
+
+static bool vmbus_isnew_subchannel(struct vmbus_channel *primary,
+				   unsigned long id)
+{
+	const struct vmbus_channel *c;
+
+	STAILQ_FOREACH(c, &primary->subchannel_list, next) {
+		if (c->relid == id)
+			return false;
+	}
+	return true;
+}
+
+int vmbus_uio_get_subchan(struct vmbus_channel *primary,
+			  struct vmbus_channel **subchan)
+{
+	const struct rte_vmbus_device *dev = primary->device;
+	char chan_path[PATH_MAX], subchan_path[PATH_MAX];
+	struct dirent *ent;
+	DIR *chan_dir;
+
+	snprintf(chan_path, sizeof(chan_path),
+		 "%s/%s/channels",
+		 SYSFS_VMBUS_DEVICES, dev->device.name);
+
+	chan_dir = opendir(chan_path);
+	if (!chan_dir) {
+		VMBUS_LOG(ERR, "cannot open %s: %s",
+			  chan_path, strerror(errno));
+		return -errno;
+	}
+
+	while ((ent = readdir(chan_dir))) {
+		unsigned long relid, subid, monid;
+		char *endp;
+		int err;
+
+		if (ent->d_name[0] == '.')
+			continue;
+
+		errno = 0;
+		relid = strtoul(ent->d_name, &endp, 0);
+		if (*endp || errno != 0 || relid > UINT16_MAX) {
+			VMBUS_LOG(NOTICE, "not a valid channel relid: %s",
+				  ent->d_name);
+			continue;
+		}
+
+		snprintf(subchan_path, sizeof(subchan_path), "%s/%lu",
+			 chan_path, relid);
+		err = vmbus_uio_sysfs_read(subchan_path, "subchannel_id",
+					   &subid, UINT16_MAX);
+		if (err) {
+			VMBUS_LOG(NOTICE, "invalid subchannel id %lu",
+				  subid);
+			closedir(chan_dir);
+			return err;
+		}
+
+		if (subid == 0)
+			continue;	/* skip primary channel */
+
+		if (!vmbus_isnew_subchannel(primary, relid))
+			continue;
+
+		if (!vmbus_uio_ring_present(dev, relid))
+			continue;	/* Ring may not be ready yet */
+
+		err = vmbus_uio_sysfs_read(subchan_path, "monitor_id",
+					   &monid, UINT8_MAX);
+		if (err) {
+			VMBUS_LOG(NOTICE, "invalid monitor id %lu",
+				  monid);
+			return err;
+		}
+
+		err = vmbus_chan_create(dev, relid, subid, monid, subchan);
+		if (err) {
+			VMBUS_LOG(NOTICE, "subchannel setup failed");
+			return err;
+		}
+		break;
+	}
+	closedir(chan_dir);
+
+	return (ent == NULL) ? -ENOENT : 0;
+}
diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build
new file mode 100644
index 00000000..18daabec
--- /dev/null
+++ b/drivers/bus/vmbus/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+allow_experimental_apis = true
+
+install_headers('rte_bus_vmbus.h','rte_vmbus_reg.h')
+
+sources = files('vmbus_common.c',
+		'vmbus_channel.c',
+		'vmbus_bufring.c',
+		'vmbus_common_uio.c')
+
+if host_machine.system() == 'linux'
+	sources += files('linux/vmbus_bus.c',
+			'linux/vmbus_uio.c')
+	includes += include_directories('linux')
+else
+	build = false
+endif
diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h
new file mode 100644
index 00000000..9964fc42
--- /dev/null
+++ b/drivers/bus/vmbus/private.h
@@ -0,0 +1,132 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#ifndef _VMBUS_PRIVATE_H_
+#define _VMBUS_PRIVATE_H_
+
+#include <stdbool.h>
+#include <sys/uio.h>
+#include <rte_log.h>
+#include <rte_vmbus_reg.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE	4096
+#endif
+
+extern int vmbus_logtype_bus;
+#define VMBUS_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, vmbus_logtype_bus, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+struct vmbus_br {
+	struct vmbus_bufring *vbr;
+	uint32_t	dsize;
+	uint32_t	windex; /* next available location */
+};
+
+#define UIO_NAME_MAX 64
+
+struct vmbus_map {
+	void *addr;	/* user mmap of resource */
+	uint64_t size;	/* length */
+};
+
+/*
+ * For multi-process we need to reproduce all vmbus mappings in secondary
+ * processes, so save them in a tailq.
+ */
+struct mapped_vmbus_resource {
+	TAILQ_ENTRY(mapped_vmbus_resource) next;
+
+	rte_uuid_t id;
+	int nb_maps;
+	struct vmbus_map maps[VMBUS_MAX_RESOURCE];
+	char path[PATH_MAX];
+};
+
+TAILQ_HEAD(mapped_vmbus_res_list, mapped_vmbus_resource);
+
+#define HV_MON_TRIG_LEN	32
+#define HV_MON_TRIG_MAX	4
+
+struct vmbus_channel {
+	STAILQ_HEAD(, vmbus_channel) subchannel_list;
+	STAILQ_ENTRY(vmbus_channel) next;
+	const struct rte_vmbus_device *device;
+
+	struct vmbus_br rxbr;
+	struct vmbus_br txbr;
+
+	uint16_t relid;
+	uint16_t subchannel_id;
+	uint8_t monitor_id;
+};
+
+#define VMBUS_MAX_CHANNELS	64
+
+int vmbus_chan_create(const struct rte_vmbus_device *device,
+		      uint16_t relid, uint16_t subid, uint8_t monitor_id,
+		      struct vmbus_channel **new_chan);
+
+void vmbus_add_device(struct rte_vmbus_device *vmbus_dev);
+void vmbus_insert_device(struct rte_vmbus_device *exist_vmbus_dev,
+			 struct rte_vmbus_device *new_vmbus_dev);
+void vmbus_remove_device(struct rte_vmbus_device *vmbus_device);
+
+void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff);
+int vmbus_uio_irq_read(struct rte_vmbus_device *dev);
+
+int vmbus_uio_map_resource(struct rte_vmbus_device *dev);
+void vmbus_uio_unmap_resource(struct rte_vmbus_device *dev);
+
+int vmbus_uio_alloc_resource(struct rte_vmbus_device *dev,
+		struct mapped_vmbus_resource **uio_res);
+void vmbus_uio_free_resource(struct rte_vmbus_device *dev,
+		struct mapped_vmbus_resource *uio_res);
+
+struct mapped_vmbus_resource *
+vmbus_uio_find_resource(const struct rte_vmbus_device *dev);
+int vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int res_idx,
+				    struct mapped_vmbus_resource *uio_res,
+				    int flags);
+
+void *vmbus_map_resource(void *requested_addr, int fd, off_t offset,
+		size_t size, int additional_flags);
+void vmbus_unmap_resource(void *requested_addr, size_t size);
+
+bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev,
+				     const struct vmbus_channel *chan);
+int vmbus_uio_get_subchan(struct vmbus_channel *primary,
+			  struct vmbus_channel **subchan);
+int vmbus_uio_map_rings(struct vmbus_channel *chan);
+
+void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen);
+
+/* Amount of space available for write */
+static inline uint32_t
+vmbus_br_availwrite(const struct vmbus_br *br, uint32_t windex)
+{
+	uint32_t rindex = br->vbr->rindex;
+
+	if (windex >= rindex)
+		return br->dsize - (windex - rindex);
+	else
+		return rindex - windex;
+}
+
+static inline uint32_t
+vmbus_br_availread(const struct vmbus_br *br)
+{
+	return br->dsize - vmbus_br_availwrite(br, br->vbr->windex);
+}
+
+int vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen,
+		     bool *need_sig);
+
+int vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen);
+
+int vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t hlen);
+
+#endif /* _VMBUS_PRIVATE_H_ */
diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h b/drivers/bus/vmbus/rte_bus_vmbus.h
new file mode 100644
index 00000000..4a2c1f6f
--- /dev/null
+++ b/drivers/bus/vmbus/rte_bus_vmbus.h
@@ -0,0 +1,407 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#ifndef _VMBUS_H_
+#define _VMBUS_H_
+
+/**
+ * @file
+ *
+ * VMBUS Interface
+ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_compat.h>
+#include <rte_uuid.h>
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_vmbus_reg.h>
+
+/* Forward declarations */
+struct rte_vmbus_device;
+struct rte_vmbus_driver;
+struct rte_vmbus_bus;
+struct vmbus_channel;
+struct vmbus_mon_page;
+
+TAILQ_HEAD(rte_vmbus_device_list, rte_vmbus_device);
+TAILQ_HEAD(rte_vmbus_driver_list, rte_vmbus_driver);
+
+/* VMBus iterators */
+#define FOREACH_DEVICE_ON_VMBUS(p)	\
+	TAILQ_FOREACH(p, &(rte_vmbus_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_VMBUS(p)	\
+	TAILQ_FOREACH(p, &(rte_vmbus_bus.driver_list), next)
+
+/** Maximum number of VMBUS resources. */
+enum hv_uio_map {
+	HV_TXRX_RING_MAP = 0,
+	HV_INT_PAGE_MAP,
+	HV_MON_PAGE_MAP,
+	HV_RECV_BUF_MAP,
+	HV_SEND_BUF_MAP
+};
+#define VMBUS_MAX_RESOURCE 5
+
+/**
+ * A structure describing a VMBUS device.
+ */
+struct rte_vmbus_device {
+	TAILQ_ENTRY(rte_vmbus_device) next;    /**< Next probed VMBUS device */
+	const struct rte_vmbus_driver *driver; /**< Associated driver */
+	struct rte_device device;              /**< Inherit core device */
+	rte_uuid_t device_id;		       /**< VMBUS device id */
+	rte_uuid_t class_id;		       /**< VMBUS device type */
+	uint32_t relid;			       /**< id for primary */
+	uint8_t monitor_id;		       /**< monitor page */
+	int uio_num;			       /**< UIO device number */
+	uint32_t *int_page;		       /**< VMBUS interrupt page */
+	struct vmbus_channel *primary;	       /**< VMBUS primary channel */
+	struct vmbus_mon_page *monitor_page;   /**< VMBUS monitor page */
+
+	struct rte_intr_handle intr_handle;    /**< Interrupt handle */
+	struct rte_mem_resource resource[VMBUS_MAX_RESOURCE];
+};
+
+/**
+ * Initialization function for the driver called during VMBUS probing.
+ */
+typedef int (vmbus_probe_t)(struct rte_vmbus_driver *,
+			    struct rte_vmbus_device *);
+
+/**
+ * Initialization function for the driver called during hot plugging.
+ */
+typedef int (vmbus_remove_t)(struct rte_vmbus_device *);
+
+/**
+ * A structure describing a VMBUS driver.
+ */
+struct rte_vmbus_driver {
+	TAILQ_ENTRY(rte_vmbus_driver) next; /**< Next in list. */
+	struct rte_driver driver;
+	struct rte_vmbus_bus *bus;          /**< VM bus reference. */
+	vmbus_probe_t *probe;               /**< Device Probe function. */
+	vmbus_remove_t *remove;             /**< Device Remove function. */
+
+	const rte_uuid_t *id_table;	    /**< ID table. */
+};
+
+
+/**
+ * Structure describing the VM bus
+ */
+struct rte_vmbus_bus {
+	struct rte_bus bus;               /**< Inherit the generic class */
+	struct rte_vmbus_device_list device_list;  /**< List of devices */
+	struct rte_vmbus_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * Scan the content of the VMBUS bus, and the devices in the devices
+ * list
+ *
+ * @return
+ *  0 on success, negative on error
+ */
+int rte_vmbus_scan(void);
+
+/**
+ * Probe the VMBUS bus
+ *
+ * @return
+ *   - 0 on success.
+ *   - !0 on error.
+ */
+int rte_vmbus_probe(void);
+
+/**
+ * Map the VMBUS device resources in user space virtual memory address
+ *
+ * @param dev
+ *   A pointer to a rte_vmbus_device structure describing the device
+ *   to use
+ *
+ * @return
+ *   0 on success, negative on error and positive if no driver
+ *   is found for the device.
+ */
+int rte_vmbus_map_device(struct rte_vmbus_device *dev);
+
+/**
+ * Unmap this device
+ *
+ * @param dev
+ *   A pointer to a rte_vmbus_device structure describing the device
+ *   to use
+ */
+void rte_vmbus_unmap_device(struct rte_vmbus_device *dev);
+
+/**
+ * Get connection to primary VMBUS channel
+ *
+ * @param device
+ *   A pointer to a rte_vmbus_device structure describing the device
+ * @param chan
+ *   A pointer to a VMBUS channel pointer that will be filled.
+ * @return
+ *   - 0 Success; channel opened.
+ *   - -ENOMEM: Not enough memory available.
+ *   - -EINVAL: Regions could not be mapped.
+ */
+int rte_vmbus_chan_open(struct rte_vmbus_device *device,
+			struct vmbus_channel **chan);
+
+/**
+ * Free connection to VMBUS channel
+ *
+ * @param chan
+ *    VMBUS channel
+ */
+void rte_vmbus_chan_close(struct vmbus_channel *chan);
+
+/**
+ * Gets the maximum number of channels supported on device
+ *
+ * @param device
+ *   A pointer to a rte_vmbus_device structure describing the device
+ * @return
+ *   Number of channels available.
+ */
+int rte_vmbus_max_channels(const struct rte_vmbus_device *device);
+
+/**
+ * Get a connection to new secondary vmbus channel
+ *
+ * @param primary
+ *   A pointer to primary VMBUS channel
+ * @param chan
+ *   A pointer to a secondary VMBUS channel pointer that will be filled.
+ * @return
+ *   - 0 Success; channel opened.
+ *   - -ENOMEM: Not enough memory available.
+ *   - -EINVAL: Regions could not be mapped.
+ */
+int rte_vmbus_subchan_open(struct vmbus_channel *primary,
+			   struct vmbus_channel **new_chan);
+
+/**
+ * Disable IRQ for device
+ *
+ * @param device
+ *    VMBUS device
+ */
+void rte_vmbus_irq_mask(struct rte_vmbus_device *device);
+
+/**
+ * Enable IRQ for device
+ *
+ * @param device
+ *    VMBUS device
+ */
+void rte_vmbus_irq_unmask(struct rte_vmbus_device *device);
+
+/**
+ * Read (and wait) for IRQ
+ *
+ * @param device
+ *    VMBUS device
+ */
+int rte_vmbus_irq_read(struct rte_vmbus_device *device);
+
+/**
+ * Test if channel is empty
+ *
+ * @param channel
+ *	Pointer to vmbus_channel structure.
+ * @return
+ *	Return true if no data present in incoming ring.
+ */
+bool rte_vmbus_chan_rx_empty(const struct vmbus_channel *channel);
+
+/**
+ * Send the specified buffer on the given channel
+ *
+ * @param channel
+ *	Pointer to vmbus_channel structure.
+ * @param type
+ *	Type of packet that is being send e.g. negotiate, time
+ *	packet etc.
+ * @param data
+ *	Pointer to the buffer to send
+ * @param dlen
+ *	Number of bytes of data to send
+ * @param xact
+ *	Identifier of the request
+ * @param flags
+ *	Message type inband, rxbuf, gpa
+ * @param need_sig
+ *	Is host signal tx is required (optional)
+ *
+ * Sends data in buffer directly to hyper-v via the vmbus
+ */
+int rte_vmbus_chan_send(struct vmbus_channel *channel, uint16_t type,
+			void *data, uint32_t dlen,
+			uint64_t xact, uint32_t flags, bool *need_sig);
+
+/**
+ * Explicitly signal host that data is available
+ *
+ * @param
+ *	Pointer to vmbus_channel structure.
+ *
+ * Used when batching multiple sends and only signaling host
+ * after the last send.
+ */
+void rte_vmbus_chan_signal_tx(const struct vmbus_channel *channel);
+
+/* Structure for scatter/gather I/O */
+struct iova_list {
+	rte_iova_t	addr;
+	uint32_t	len;
+};
+#define MAX_PAGE_BUFFER_COUNT		32
+
+/**
+ * Send a scattered buffer on the given channel
+ *
+ * @param channel
+ *	Pointer to vmbus_channel structure.
+ * @param type
+ *	Type of packet that is being send e.g. negotiate, time
+ *	packet etc.
+ * @param gpa
+ *	Array of buffers to send
+ * @param gpacnt
+ *	Number of elements in iov
+ * @param data
+ *	Pointer to the buffer additional data to send
+ * @param dlen
+ *	 Maximum size of what the the buffer will hold
+ * @param xact
+ *	Identifier of the request
+ * @param flags
+ *	Message type inband, rxbuf, gpa
+ * @param need_sig
+ *	Is host signal tx is required (optional)
+ *
+ * Sends data in buffer directly to hyper-v via the vmbus
+ */
+int rte_vmbus_chan_send_sglist(struct vmbus_channel *channel,
+			       struct vmbus_gpa gpa[], uint32_t gpacnt,
+			       void *data, uint32_t dlen,
+			       uint64_t xact, bool *need_sig);
+/**
+ * Receive response to request on the given channel
+ * skips the channel header.
+ *
+ * @param channel
+ *	Pointer to vmbus_channel structure.
+ * @param data
+ *	Pointer to the buffer you want to receive the data into.
+ * @param len
+ *	Pointer to size of receive buffer (in/out)
+ * @param
+ *	Pointer to received transaction_id
+ * @return
+ *   On success, returns 0
+ *   On failure, returns negative errno.
+ */
+int rte_vmbus_chan_recv(struct vmbus_channel *chan,
+			void *data, uint32_t *len,
+			uint64_t *request_id);
+
+/**
+ * Receive response to request on the given channel
+ * includes the channel header.
+ *
+ * @param channel
+ *	Pointer to vmbus_channel structure.
+ * @param data
+ *	Pointer to the buffer you want to receive the data into.
+ * @param len
+ *	Pointer to size of receive buffer (in/out)
+ * @return
+ *   On success, returns number of bytes read.
+ *   On failure, returns negative errno.
+ */
+int rte_vmbus_chan_recv_raw(struct vmbus_channel *chan,
+			    void *data, uint32_t *len);
+
+/**
+ * Notify host of bytes read (after recv_raw)
+ * Signals host if required.
+ *
+ * @param channel
+ *	Pointer to vmbus_channel structure.
+ * @param bytes_read
+ *	Number of bytes read since last signal
+ */
+void rte_vmbus_chan_signal_read(struct vmbus_channel *chan, uint32_t bytes_read);
+
+/**
+ * Determine sub channel index of the given channel
+ *
+ * @param channel
+ *	Pointer to vmbus_channel structure.
+ * @return
+ *   Sub channel index (0 for primary)
+ */
+uint16_t rte_vmbus_sub_channel_index(const struct vmbus_channel *chan);
+
+/**
+ * Register a VMBUS driver.
+ *
+ * @param driver
+ *   A pointer to a rte_vmbus_driver structure describing the driver
+ *   to be registered.
+ */
+void rte_vmbus_register(struct rte_vmbus_driver *driver);
+
+/**
+ * For debug dump contents of ring buffer.
+ *
+ * @param channel
+ *	Pointer to vmbus_channel structure.
+ */
+void rte_vmbus_chan_dump(FILE *f, const struct vmbus_channel *chan);
+
+/**
+ * Unregister a VMBUS driver.
+ *
+ * @param driver
+ *   A pointer to a rte_vmbus_driver structure describing the driver
+ *   to be unregistered.
+ */
+void rte_vmbus_unregister(struct rte_vmbus_driver *driver);
+
+/** Helper for VMBUS device registration from driver instance */
+#define RTE_PMD_REGISTER_VMBUS(nm, vmbus_drv)		\
+	RTE_INIT(vmbusinitfn_ ##nm);			\
+	static void vmbusinitfn_ ##nm(void)		\
+	{						\
+		(vmbus_drv).driver.name = RTE_STR(nm);	\
+		rte_vmbus_register(&vmbus_drv);		\
+	}						\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _VMBUS_H_ */
diff --git a/drivers/bus/vmbus/rte_bus_vmbus_version.map b/drivers/bus/vmbus/rte_bus_vmbus_version.map
new file mode 100644
index 00000000..dabb9203
--- /dev/null
+++ b/drivers/bus/vmbus/rte_bus_vmbus_version.map
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+
+DPDK_18.08 {
+	global:
+
+	rte_vmbus_chan_close;
+	rte_vmbus_chan_open;
+	rte_vmbus_chan_recv;
+	rte_vmbus_chan_recv_raw;
+	rte_vmbus_chan_rx_empty;
+	rte_vmbus_chan_send;
+	rte_vmbus_chan_send_sglist;
+	rte_vmbus_chan_signal_read;
+	rte_vmbus_chan_signal_tx;
+	rte_vmbus_irq_mask;
+	rte_vmbus_irq_read;
+	rte_vmbus_irq_unmask;
+	rte_vmbus_map_device;
+	rte_vmbus_max_channels;
+	rte_vmbus_probe;
+	rte_vmbus_register;
+	rte_vmbus_scan;
+	rte_vmbus_sub_channel_index;
+	rte_vmbus_subchan_open;
+	rte_vmbus_unmap_device;
+	rte_vmbus_unregister;
+
+	local: *;
+};
diff --git a/drivers/bus/vmbus/rte_vmbus_reg.h b/drivers/bus/vmbus/rte_vmbus_reg.h
new file mode 100644
index 00000000..f5a0693d
--- /dev/null
+++ b/drivers/bus/vmbus/rte_vmbus_reg.h
@@ -0,0 +1,344 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#ifndef _VMBUS_REG_H_
+#define _VMBUS_REG_H_
+
+/*
+ * Hyper-V SynIC message format.
+ */
+#define VMBUS_MSG_DSIZE_MAX		240
+#define VMBUS_MSG_SIZE			256
+
+struct vmbus_message {
+	uint32_t	type;	/* HYPERV_MSGTYPE_ */
+	uint8_t		dsize;	/* data size */
+	uint8_t		flags;	/* VMBUS_MSGFLAG_ */
+	uint16_t	rsvd;
+	uint64_t	id;
+	uint8_t		data[VMBUS_MSG_DSIZE_MAX];
+} __rte_packed;
+
+#define VMBUS_MSGFLAG_PENDING		0x01
+
+/*
+ * Hyper-V Monitor Notification Facility
+ */
+
+struct vmbus_mon_trig {
+	uint32_t	pending;
+	uint32_t	armed;
+} __rte_packed;
+
+#define VMBUS_MONTRIGS_MAX	4
+#define VMBUS_MONTRIG_LEN	32
+
+/*
+ * Hyper-V Monitor Notification Facility
+ */
+struct hyperv_mon_param {
+	uint32_t	connid;
+	uint16_t	evtflag_ofs;
+	uint16_t	rsvd;
+} __rte_packed;
+
+struct vmbus_mon_page {
+	uint32_t	state;
+	uint32_t	rsvd1;
+
+	struct vmbus_mon_trig trigs[VMBUS_MONTRIGS_MAX];
+	uint8_t		rsvd2[536];
+
+	uint16_t	lat[VMBUS_MONTRIGS_MAX][VMBUS_MONTRIG_LEN];
+	uint8_t		rsvd3[256];
+
+	struct hyperv_mon_param
+			param[VMBUS_MONTRIGS_MAX][VMBUS_MONTRIG_LEN];
+	uint8_t		rsvd4[1984];
+} __rte_packed;
+
+/*
+ * Buffer ring
+ */
+
+struct vmbus_bufring {
+	volatile uint32_t windex;
+	volatile uint32_t rindex;
+
+	/*
+	 * Interrupt mask {0,1}
+	 *
+	 * For TX bufring, host set this to 1, when it is processing
+	 * the TX bufring, so that we can safely skip the TX event
+	 * notification to host.
+	 *
+	 * For RX bufring, once this is set to 1 by us, host will not
+	 * further dispatch interrupts to us, even if there are data
+	 * pending on the RX bufring.  This effectively disables the
+	 * interrupt of the channel to which this RX bufring is attached.
+	 */
+	volatile uint32_t imask;
+
+	/*
+	 * Win8 uses some of the reserved bits to implement
+	 * interrupt driven flow management. On the send side
+	 * we can request that the receiver interrupt the sender
+	 * when the ring transitions from being full to being able
+	 * to handle a message of size "pending_send_sz".
+	 *
+	 * Add necessary state for this enhancement.
+	 */
+	volatile uint32_t pending_send;
+	uint32_t reserved1[12];
+
+	union {
+		struct {
+			uint32_t feat_pending_send_sz:1;
+		};
+		uint32_t value;
+	} feature_bits;
+
+	/* Pad it to PAGE_SIZE so that data starts on page boundary */
+	uint8_t	reserved2[4028];
+
+	/*
+	 * Ring data starts here + RingDataStartOffset
+	 * !!! DO NOT place any fields below this !!!
+	 */
+	uint8_t data[0];
+} __rte_packed;
+
+/*
+ * Channel packets
+ */
+
+/* Channel packet flags */
+#define VMBUS_CHANPKT_TYPE_INBAND      0x0006
+#define VMBUS_CHANPKT_TYPE_RXBUF       0x0007
+#define VMBUS_CHANPKT_TYPE_GPA         0x0009
+#define VMBUS_CHANPKT_TYPE_COMP        0x000b
+
+#define VMBUS_CHANPKT_FLAG_NONE        0
+#define VMBUS_CHANPKT_FLAG_RC          0x0001  /* report completion */
+
+#define VMBUS_CHANPKT_SIZE_SHIFT	3
+#define VMBUS_CHANPKT_SIZE_ALIGN	(1 << VMBUS_CHANPKT_SIZE_SHIFT)
+#define VMBUS_CHANPKT_HLEN_MIN		\
+	(sizeof(struct vmbus_chanpkt_hdr) >> VMBUS_CHANPKT_SIZE_SHIFT)
+
+static inline uint32_t
+vmbus_chanpkt_getlen(uint16_t pktlen)
+{
+	return (uint32_t)pktlen << VMBUS_CHANPKT_SIZE_SHIFT;
+}
+
+/*
+ * GPA stuffs.
+ */
+struct vmbus_gpa_range {
+	uint32_t       len;
+	uint32_t       ofs;
+	uint64_t       page[0];
+} __rte_packed;
+
+/* This is actually vmbus_gpa_range.gpa_page[1] */
+struct vmbus_gpa {
+	uint32_t	len;
+	uint32_t	ofs;
+	uint64_t	page;
+} __rte_packed;
+
+struct vmbus_chanpkt_hdr {
+	uint16_t	type;	/* VMBUS_CHANPKT_TYPE_ */
+	uint16_t	hlen;	/* header len, in 8 bytes */
+	uint16_t	tlen;	/* total len, in 8 bytes */
+	uint16_t	flags;	/* VMBUS_CHANPKT_FLAG_ */
+	uint64_t	xactid;
+} __rte_packed;
+
+static inline uint32_t
+vmbus_chanpkt_datalen(const struct vmbus_chanpkt_hdr *pkt)
+{
+	return vmbus_chanpkt_getlen(pkt->tlen)
+		- vmbus_chanpkt_getlen(pkt->hlen);
+}
+
+struct vmbus_chanpkt {
+	struct vmbus_chanpkt_hdr hdr;
+} __rte_packed;
+
+struct vmbus_rxbuf_desc {
+	uint32_t	len;
+	uint32_t	ofs;
+} __rte_packed;
+
+struct vmbus_chanpkt_rxbuf {
+	struct vmbus_chanpkt_hdr hdr;
+	uint16_t	rxbuf_id;
+	uint16_t	rsvd;
+	uint32_t	rxbuf_cnt;
+	struct vmbus_rxbuf_desc rxbuf[];
+} __rte_packed;
+
+struct vmbus_chanpkt_sglist {
+	struct vmbus_chanpkt_hdr hdr;
+	uint32_t	rsvd;
+	uint32_t	gpa_cnt;
+	struct vmbus_gpa gpa[];
+} __rte_packed;
+
+/*
+ * Channel messages
+ * - Embedded in vmbus_message.msg_data, e.g. response and notification.
+ * - Embedded in hypercall_postmsg_in.hc_data, e.g. request.
+ */
+
+#define VMBUS_CHANMSG_TYPE_CHOFFER		1	/* NOTE */
+#define VMBUS_CHANMSG_TYPE_CHRESCIND		2	/* NOTE */
+#define VMBUS_CHANMSG_TYPE_CHREQUEST		3	/* REQ */
+#define VMBUS_CHANMSG_TYPE_CHOFFER_DONE		4	/* NOTE */
+#define VMBUS_CHANMSG_TYPE_CHOPEN		5	/* REQ */
+#define VMBUS_CHANMSG_TYPE_CHOPEN_RESP		6	/* RESP */
+#define VMBUS_CHANMSG_TYPE_CHCLOSE		7	/* REQ */
+#define VMBUS_CHANMSG_TYPE_GPADL_CONN		8	/* REQ */
+#define VMBUS_CHANMSG_TYPE_GPADL_SUBCONN	9	/* REQ */
+#define VMBUS_CHANMSG_TYPE_GPADL_CONNRESP	10	/* RESP */
+#define VMBUS_CHANMSG_TYPE_GPADL_DISCONN	11	/* REQ */
+#define VMBUS_CHANMSG_TYPE_GPADL_DISCONNRESP	12	/* RESP */
+#define VMBUS_CHANMSG_TYPE_CHFREE		13	/* REQ */
+#define VMBUS_CHANMSG_TYPE_CONNECT		14	/* REQ */
+#define VMBUS_CHANMSG_TYPE_CONNECT_RESP		15	/* RESP */
+#define VMBUS_CHANMSG_TYPE_DISCONNECT		16	/* REQ */
+#define VMBUS_CHANMSG_TYPE_MAX			22
+
+struct vmbus_chanmsg_hdr {
+	uint32_t	type;	/* VMBUS_CHANMSG_TYPE_ */
+	uint32_t	rsvd;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CONNECT */
+struct vmbus_chanmsg_connect {
+	struct vmbus_chanmsg_hdr hdr;
+	uint32_t	ver;
+	uint32_t	rsvd;
+	uint64_t	evtflags;
+	uint64_t	mnf1;
+	uint64_t	mnf2;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CONNECT_RESP */
+struct vmbus_chanmsg_connect_resp {
+	struct vmbus_chanmsg_hdr hdr;
+	uint8_t		done;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHREQUEST */
+struct vmbus_chanmsg_chrequest {
+	struct vmbus_chanmsg_hdr hdr;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_DISCONNECT */
+struct vmbus_chanmsg_disconnect {
+	struct vmbus_chanmsg_hdr hdr;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHOPEN */
+struct vmbus_chanmsg_chopen {
+	struct vmbus_chanmsg_hdr hdr;
+	uint32_t	chanid;
+	uint32_t	openid;
+	uint32_t	gpadl;
+	uint32_t	vcpuid;
+	uint32_t	txbr_pgcnt;
+#define VMBUS_CHANMSG_CHOPEN_UDATA_SIZE	120
+	uint8_t		udata[VMBUS_CHANMSG_CHOPEN_UDATA_SIZE];
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHOPEN_RESP */
+struct vmbus_chanmsg_chopen_resp {
+	struct vmbus_chanmsg_hdr hdr;
+	uint32_t	chanid;
+	uint32_t	openid;
+	uint32_t	status;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_GPADL_CONN */
+struct vmbus_chanmsg_gpadl_conn {
+	struct vmbus_chanmsg_hdr hdr;
+	uint32_t	chanid;
+	uint32_t	gpadl;
+	uint16_t	range_len;
+	uint16_t	range_cnt;
+	struct vmbus_gpa_range range;
+} __rte_packed;
+
+#define VMBUS_CHANMSG_GPADL_CONN_PGMAX		26
+
+/* VMBUS_CHANMSG_TYPE_GPADL_SUBCONN */
+struct vmbus_chanmsg_gpadl_subconn {
+	struct vmbus_chanmsg_hdr hdr;
+	uint32_t	msgno;
+	uint32_t	gpadl;
+	uint64_t	gpa_page[];
+} __rte_packed;
+
+#define VMBUS_CHANMSG_GPADL_SUBCONN_PGMAX	28
+
+/* VMBUS_CHANMSG_TYPE_GPADL_CONNRESP */
+struct vmbus_chanmsg_gpadl_connresp {
+	struct vmbus_chanmsg_hdr hdr;
+	uint32_t	chanid;
+	uint32_t	gpadl;
+	uint32_t	status;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHCLOSE */
+struct vmbus_chanmsg_chclose {
+	struct vmbus_chanmsg_hdr hdr;
+	uint32_t	chanid;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_GPADL_DISCONN */
+struct vmbus_chanmsg_gpadl_disconn {
+	struct vmbus_chanmsg_hdr hdr;
+	uint32_t	chanid;
+	uint32_t	gpadl;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHFREE */
+struct vmbus_chanmsg_chfree {
+	struct vmbus_chanmsg_hdr hdr;
+	uint32_t	chanid;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHRESCIND */
+struct vmbus_chanmsg_chrescind {
+	struct vmbus_chanmsg_hdr hdr;
+	uint32_t	chanid;
+} __rte_packed;
+
+/* VMBUS_CHANMSG_TYPE_CHOFFER */
+struct vmbus_chanmsg_choffer {
+	struct vmbus_chanmsg_hdr hdr;
+	rte_uuid_t	chtype;
+	rte_uuid_t	chinst;
+	uint64_t	chlat;	/* unit: 100ns */
+	uint32_t	chrev;
+	uint32_t	svrctx_sz;
+	uint16_t	chflags;
+	uint16_t	mmio_sz;	/* unit: MB */
+	uint8_t		udata[120];
+	uint16_t	subidx;
+	uint16_t	rsvd;
+	uint32_t	chanid;
+	uint8_t		montrig;
+	uint8_t		flags1;	/* VMBUS_CHOFFER_FLAG1_ */
+	uint16_t	flags2;
+	uint32_t	connid;
+} __rte_packed;
+
+#define VMBUS_CHOFFER_FLAG1_HASMNF	0x01
+
+#endif	/* !_VMBUS_REG_H_ */
diff --git a/drivers/bus/vmbus/vmbus_bufring.c b/drivers/bus/vmbus/vmbus_bufring.c
new file mode 100644
index 00000000..c8800160
--- /dev/null
+++ b/drivers/bus/vmbus/vmbus_bufring.c
@@ -0,0 +1,244 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2009-2012,2016 Microsoft Corp.
+ * Copyright (c) 2012 NetApp Inc.
+ * Copyright (c) 2012 Citrix Inc.
+ * All rights reserved.
+ */
+
+#include <unistd.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <sys/uio.h>
+
+#include <rte_eal.h>
+#include <rte_tailq.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_bus.h>
+#include <rte_atomic.h>
+#include <rte_memory.h>
+#include <rte_pause.h>
+#include <rte_bus_vmbus.h>
+
+#include "private.h"
+
+/* Increase bufring index by inc with wraparound */
+static inline uint32_t vmbus_br_idxinc(uint32_t idx, uint32_t inc, uint32_t sz)
+{
+	idx += inc;
+	if (idx >= sz)
+		idx -= sz;
+
+	return idx;
+}
+
+void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen)
+{
+	br->vbr = buf;
+	br->windex = br->vbr->windex;
+	br->dsize = blen - sizeof(struct vmbus_bufring);
+}
+
+/*
+ * When we write to the ring buffer, check if the host needs to be
+ * signaled.
+ *
+ * The contract:
+ * - The host guarantees that while it is draining the TX bufring,
+ *   it will set the br_imask to indicate it does not need to be
+ *   interrupted when new data are added.
+ * - The host guarantees that it will completely drain the TX bufring
+ *   before exiting the read loop.  Further, once the TX bufring is
+ *   empty, it will clear the br_imask and re-check to see if new
+ *   data have arrived.
+ */
+static inline bool
+vmbus_txbr_need_signal(const struct vmbus_br *tbr, uint32_t old_windex)
+{
+	rte_smp_mb();
+	if (tbr->vbr->imask)
+		return false;
+
+	rte_smp_rmb();
+
+	/*
+	 * This is the only case we need to signal when the
+	 * ring transitions from being empty to non-empty.
+	 */
+	return old_windex == tbr->vbr->rindex;
+}
+
+static inline uint32_t
+vmbus_txbr_copyto(const struct vmbus_br *tbr, uint32_t windex,
+		  const void *src0, uint32_t cplen)
+{
+	uint8_t *br_data = tbr->vbr->data;
+	uint32_t br_dsize = tbr->dsize;
+	const uint8_t *src = src0;
+
+	/* XXX use double mapping like Linux kernel? */
+	if (cplen > br_dsize - windex) {
+		uint32_t fraglen = br_dsize - windex;
+
+		/* Wrap-around detected */
+		memcpy(br_data + windex, src, fraglen);
+		memcpy(br_data, src + fraglen, cplen - fraglen);
+	} else {
+		memcpy(br_data + windex, src, cplen);
+	}
+
+	return vmbus_br_idxinc(windex, cplen, br_dsize);
+}
+
+/*
+ * Write scattered channel packet to TX bufring.
+ *
+ * The offset of this channel packet is written as a 64bits value
+ * immediately after this channel packet.
+ *
+ * The write goes through three stages:
+ *  1. Reserve space in ring buffer for the new data.
+ *     Writer atomically moves priv_write_index.
+ *  2. Copy the new data into the ring.
+ *  3. Update the tail of the ring (visible to host) that indicates
+ *     next read location. Writer updates write_index
+ */
+int
+vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen,
+		 bool *need_sig)
+{
+	struct vmbus_bufring *vbr = tbr->vbr;
+	uint32_t ring_size = tbr->dsize;
+	uint32_t old_windex, next_windex, windex, total;
+	uint64_t save_windex;
+	int i;
+
+	total = 0;
+	for (i = 0; i < iovlen; i++)
+		total += iov[i].iov_len;
+	total += sizeof(save_windex);
+
+	/* Reserve space in ring */
+	do {
+		uint32_t avail;
+
+		/* Get current free location */
+		old_windex = tbr->windex;
+
+		/* Prevent compiler reordering this with calculation */
+		rte_compiler_barrier();
+
+		avail = vmbus_br_availwrite(tbr, old_windex);
+
+		/* If not enough space in ring, then tell caller. */
+		if (avail <= total)
+			return -EAGAIN;
+
+		next_windex = vmbus_br_idxinc(old_windex, total, ring_size);
+
+		/* Atomic update of next write_index for other threads */
+	} while (!rte_atomic32_cmpset(&tbr->windex, old_windex, next_windex));
+
+	/* Space from old..new is now reserved */
+	windex = old_windex;
+	for (i = 0; i < iovlen; i++) {
+		windex = vmbus_txbr_copyto(tbr, windex,
+					   iov[i].iov_base, iov[i].iov_len);
+	}
+
+	/* Set the offset of the current channel packet. */
+	save_windex = ((uint64_t)old_windex) << 32;
+	windex = vmbus_txbr_copyto(tbr, windex, &save_windex,
+				   sizeof(save_windex));
+
+	/* The region reserved should match region used */
+	RTE_ASSERT(windex == next_windex);
+
+	/* Ensure that data is available before updating host index */
+	rte_smp_wmb();
+
+	/* Checkin for our reservation. wait for our turn to update host */
+	while (!rte_atomic32_cmpset(&vbr->windex, old_windex, next_windex))
+		rte_pause();
+
+	/* If host had read all data before this, then need to signal */
+	*need_sig |= vmbus_txbr_need_signal(tbr, old_windex);
+	return 0;
+}
+
+static inline uint32_t
+vmbus_rxbr_copyfrom(const struct vmbus_br *rbr, uint32_t rindex,
+		    void *dst0, size_t cplen)
+{
+	const uint8_t *br_data = rbr->vbr->data;
+	uint32_t br_dsize = rbr->dsize;
+	uint8_t *dst = dst0;
+
+	if (cplen > br_dsize - rindex) {
+		uint32_t fraglen = br_dsize - rindex;
+
+		/* Wrap-around detected. */
+		memcpy(dst, br_data + rindex, fraglen);
+		memcpy(dst + fraglen, br_data, cplen - fraglen);
+	} else {
+		memcpy(dst, br_data + rindex, cplen);
+	}
+
+	return vmbus_br_idxinc(rindex, cplen, br_dsize);
+}
+
+/* Copy data from receive ring but don't change index */
+int
+vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen)
+{
+	uint32_t avail;
+
+	/*
+	 * The requested data and the 64bits channel packet
+	 * offset should be there at least.
+	 */
+	avail = vmbus_br_availread(rbr);
+	if (avail < dlen + sizeof(uint64_t))
+		return -EAGAIN;
+
+	vmbus_rxbr_copyfrom(rbr, rbr->vbr->rindex, data, dlen);
+	return 0;
+}
+
+/*
+ * Copy data from receive ring and change index
+ * NOTE:
+ * We assume (dlen + skip) == sizeof(channel packet).
+ */
+int
+vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t skip)
+{
+	struct vmbus_bufring *vbr = rbr->vbr;
+	uint32_t br_dsize = rbr->dsize;
+	uint32_t rindex;
+
+	if (vmbus_br_availread(rbr) < dlen + skip + sizeof(uint64_t))
+		return -EAGAIN;
+
+	/* Record where host was when we started read (for debug) */
+	rbr->windex = rbr->vbr->windex;
+
+	/*
+	 * Copy channel packet from RX bufring.
+	 */
+	rindex = vmbus_br_idxinc(rbr->vbr->rindex, skip, br_dsize);
+	rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen);
+
+	/*
+	 * Discard this channel packet's 64bits offset, which is useless to us.
+	 */
+	rindex = vmbus_br_idxinc(rindex, sizeof(uint64_t), br_dsize);
+
+	/* Update the read index _after_ the channel packet is fetched.	 */
+	rte_compiler_barrier();
+
+	vbr->rindex = rindex;
+
+	return 0;
+}
diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c
new file mode 100644
index 00000000..cc5f3e83
--- /dev/null
+++ b/drivers/bus/vmbus/vmbus_channel.c
@@ -0,0 +1,405 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#include <unistd.h>
+#include <stdint.h>
+#include <string.h>
+#include <sys/uio.h>
+
+#include <rte_eal.h>
+#include <rte_tailq.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_bus.h>
+#include <rte_atomic.h>
+#include <rte_memory.h>
+#include <rte_bus_vmbus.h>
+
+#include "private.h"
+
+static inline void
+vmbus_sync_set_bit(volatile uint32_t *addr, uint32_t mask)
+{
+	/* Use GCC builtin which atomic does atomic OR operation */
+	__sync_or_and_fetch(addr, mask);
+}
+
+static inline void
+vmbus_send_interrupt(const struct rte_vmbus_device *dev, uint32_t relid)
+{
+	uint32_t *int_addr;
+	uint32_t int_mask;
+
+	int_addr = dev->int_page + relid / 32;
+	int_mask = 1u << (relid % 32);
+
+	vmbus_sync_set_bit(int_addr, int_mask);
+}
+
+static inline void
+vmbus_set_monitor(const struct rte_vmbus_device *dev, uint32_t monitor_id)
+{
+	uint32_t *monitor_addr, monitor_mask;
+	unsigned int trigger_index;
+
+	trigger_index = monitor_id / HV_MON_TRIG_LEN;
+	monitor_mask = 1u << (monitor_id % HV_MON_TRIG_LEN);
+
+	monitor_addr = &dev->monitor_page->trigs[trigger_index].pending;
+	vmbus_sync_set_bit(monitor_addr, monitor_mask);
+}
+
+static void
+vmbus_set_event(const struct rte_vmbus_device *dev,
+		const struct vmbus_channel *chan)
+{
+	vmbus_send_interrupt(dev, chan->relid);
+	vmbus_set_monitor(dev, chan->monitor_id);
+}
+
+/*
+ * Notify host that there are data pending on our TX bufring.
+ *
+ * Since this in userspace, rely on the monitor page.
+ * Can't do a hypercall from userspace.
+ */
+void
+rte_vmbus_chan_signal_tx(const struct vmbus_channel *chan)
+{
+	const struct rte_vmbus_device *dev = chan->device;
+	const struct vmbus_br *tbr = &chan->txbr;
+
+	/* Make sure all updates are done before signaling host */
+	rte_smp_wmb();
+
+	/* If host is ignoring interrupts? */
+	if (tbr->vbr->imask)
+		return;
+
+	vmbus_set_event(dev, chan);
+}
+
+
+/* Do a simple send directly using transmit ring. */
+int rte_vmbus_chan_send(struct vmbus_channel *chan, uint16_t type,
+			void *data, uint32_t dlen,
+			uint64_t xactid, uint32_t flags, bool *need_sig)
+{
+	struct vmbus_chanpkt pkt;
+	unsigned int pktlen, pad_pktlen;
+	const uint32_t hlen = sizeof(pkt);
+	bool send_evt = false;
+	uint64_t pad = 0;
+	struct iovec iov[3];
+	int error;
+
+	pktlen = hlen + dlen;
+	pad_pktlen = RTE_ALIGN(pktlen, sizeof(uint64_t));
+
+	pkt.hdr.type = type;
+	pkt.hdr.flags = flags;
+	pkt.hdr.hlen = hlen >> VMBUS_CHANPKT_SIZE_SHIFT;
+	pkt.hdr.tlen = pad_pktlen >> VMBUS_CHANPKT_SIZE_SHIFT;
+	pkt.hdr.xactid = xactid;
+
+	iov[0].iov_base = &pkt;
+	iov[0].iov_len = hlen;
+	iov[1].iov_base = data;
+	iov[1].iov_len = dlen;
+	iov[2].iov_base = &pad;
+	iov[2].iov_len = pad_pktlen - pktlen;
+
+	error = vmbus_txbr_write(&chan->txbr, iov, 3, &send_evt);
+
+	/*
+	 * caller sets need_sig to non-NULL if it will handle
+	 * signaling if required later.
+	 * if need_sig is NULL, signal now if needed.
+	 */
+	if (need_sig)
+		*need_sig |= send_evt;
+	else if (error == 0 && send_evt)
+		rte_vmbus_chan_signal_tx(chan);
+	return error;
+}
+
+/* Do a scatter/gather send where the descriptor points to data. */
+int rte_vmbus_chan_send_sglist(struct vmbus_channel *chan,
+			       struct vmbus_gpa sg[], uint32_t sglen,
+			       void *data, uint32_t dlen,
+			       uint64_t xactid, bool *need_sig)
+{
+	struct vmbus_chanpkt_sglist pkt;
+	unsigned int pktlen, pad_pktlen, hlen;
+	bool send_evt = false;
+	struct iovec iov[4];
+	uint64_t pad = 0;
+	int error;
+
+	hlen = offsetof(struct vmbus_chanpkt_sglist, gpa[sglen]);
+	pktlen = hlen + dlen;
+	pad_pktlen = RTE_ALIGN(pktlen, sizeof(uint64_t));
+
+	pkt.hdr.type = VMBUS_CHANPKT_TYPE_GPA;
+	pkt.hdr.flags = VMBUS_CHANPKT_FLAG_RC;
+	pkt.hdr.hlen = hlen >> VMBUS_CHANPKT_SIZE_SHIFT;
+	pkt.hdr.tlen = pad_pktlen >> VMBUS_CHANPKT_SIZE_SHIFT;
+	pkt.hdr.xactid = xactid;
+	pkt.rsvd = 0;
+	pkt.gpa_cnt = sglen;
+
+	iov[0].iov_base = &pkt;
+	iov[0].iov_len = sizeof(pkt);
+	iov[1].iov_base = sg;
+	iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen;
+	iov[2].iov_base = data;
+	iov[2].iov_len = dlen;
+	iov[3].iov_base = &pad;
+	iov[3].iov_len = pad_pktlen - pktlen;
+
+	error = vmbus_txbr_write(&chan->txbr, iov, 4, &send_evt);
+
+	/* if caller is batching, just propagate the status */
+	if (need_sig)
+		*need_sig |= send_evt;
+	else if (error == 0 && send_evt)
+		rte_vmbus_chan_signal_tx(chan);
+	return error;
+}
+
+bool rte_vmbus_chan_rx_empty(const struct vmbus_channel *channel)
+{
+	const struct vmbus_br *br = &channel->rxbr;
+
+	return br->vbr->rindex == br->vbr->windex;
+}
+
+/* Signal host after reading N bytes */
+void rte_vmbus_chan_signal_read(struct vmbus_channel *chan, uint32_t bytes_read)
+{
+	struct vmbus_br *rbr = &chan->rxbr;
+	uint32_t write_sz, pending_sz;
+
+	/* No need for signaling on older versions */
+	if (!rbr->vbr->feature_bits.feat_pending_send_sz)
+		return;
+
+	/* Make sure reading of pending happens after new read index */
+	rte_mb();
+
+	pending_sz = rbr->vbr->pending_send;
+	if (!pending_sz)
+		return;
+
+	rte_smp_rmb();
+	write_sz = vmbus_br_availwrite(rbr, rbr->vbr->windex);
+
+	/* If there was space before then host was not blocked */
+	if (write_sz - bytes_read > pending_sz)
+		return;
+
+	/* If pending write will not fit */
+	if (write_sz <= pending_sz)
+		return;
+
+	vmbus_set_event(chan->device, chan);
+}
+
+int rte_vmbus_chan_recv(struct vmbus_channel *chan, void *data, uint32_t *len,
+			uint64_t *request_id)
+{
+	struct vmbus_chanpkt_hdr pkt;
+	uint32_t dlen, hlen, bufferlen = *len;
+	int error;
+
+	*len = 0;
+
+	error = vmbus_rxbr_peek(&chan->rxbr, &pkt, sizeof(pkt));
+	if (error)
+		return error;
+
+	if (unlikely(pkt.hlen < VMBUS_CHANPKT_HLEN_MIN)) {
+		VMBUS_LOG(ERR, "VMBUS recv, invalid hlen %u", pkt.hlen);
+		/* XXX this channel is dead actually. */
+		return -EIO;
+	}
+
+	if (unlikely(pkt.hlen > pkt.tlen)) {
+		VMBUS_LOG(ERR, "VMBUS recv,invalid hlen %u and tlen %u",
+			  pkt.hlen, pkt.tlen);
+		return -EIO;
+	}
+
+	/* Length are in quad words */
+	hlen = pkt.hlen << VMBUS_CHANPKT_SIZE_SHIFT;
+	dlen = (pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT) - hlen;
+	*len = dlen;
+
+	/* If caller buffer is not large enough */
+	if (unlikely(dlen > bufferlen))
+		return -ENOBUFS;
+
+	if (request_id)
+		*request_id = pkt.xactid;
+
+	/* Read data and skip packet header */
+	error = vmbus_rxbr_read(&chan->rxbr, data, dlen, hlen);
+	if (error)
+		return error;
+
+	rte_vmbus_chan_signal_read(chan, dlen + hlen + sizeof(uint64_t));
+	return 0;
+}
+
+/* TODO: replace this with inplace ring buffer (no copy) */
+int rte_vmbus_chan_recv_raw(struct vmbus_channel *chan,
+			    void *data, uint32_t *len)
+{
+	struct vmbus_chanpkt_hdr pkt;
+	uint32_t dlen, bufferlen = *len;
+	int error;
+
+	error = vmbus_rxbr_peek(&chan->rxbr, &pkt, sizeof(pkt));
+	if (error)
+		return error;
+
+	if (unlikely(pkt.hlen < VMBUS_CHANPKT_HLEN_MIN)) {
+		VMBUS_LOG(ERR, "VMBUS recv, invalid hlen %u", pkt.hlen);
+		/* XXX this channel is dead actually. */
+		return -EIO;
+	}
+
+	if (unlikely(pkt.hlen > pkt.tlen)) {
+		VMBUS_LOG(ERR, "VMBUS recv,invalid hlen %u and tlen %u",
+			pkt.hlen, pkt.tlen);
+		return -EIO;
+	}
+
+	/* Length are in quad words */
+	dlen = pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT;
+	*len = dlen;
+
+	/* If caller buffer is not large enough */
+	if (unlikely(dlen > bufferlen))
+		return -ENOBUFS;
+
+	/* Read data and skip packet header */
+	error = vmbus_rxbr_read(&chan->rxbr, data, dlen, 0);
+	if (error)
+		return error;
+
+	/* Return the number of bytes read */
+	return dlen + sizeof(uint64_t);
+}
+
+int vmbus_chan_create(const struct rte_vmbus_device *device,
+		      uint16_t relid, uint16_t subid, uint8_t monitor_id,
+		      struct vmbus_channel **new_chan)
+{
+	struct vmbus_channel *chan;
+	int err;
+
+	chan = rte_zmalloc_socket("VMBUS", sizeof(*chan), RTE_CACHE_LINE_SIZE,
+				  device->device.numa_node);
+	if (!chan)
+		return -ENOMEM;
+
+	STAILQ_INIT(&chan->subchannel_list);
+	chan->device = device;
+	chan->subchannel_id = subid;
+	chan->relid = relid;
+	chan->monitor_id = monitor_id;
+	*new_chan = chan;
+
+	err = vmbus_uio_map_rings(chan);
+	if (err) {
+		rte_free(chan);
+		return err;
+	}
+
+	return 0;
+}
+
+/* Setup the primary channel */
+int rte_vmbus_chan_open(struct rte_vmbus_device *device,
+			struct vmbus_channel **new_chan)
+{
+	int err;
+
+	err = vmbus_chan_create(device, device->relid, 0,
+				device->monitor_id, new_chan);
+	if (!err)
+		device->primary = *new_chan;
+
+	return err;
+}
+
+int rte_vmbus_max_channels(const struct rte_vmbus_device *device)
+{
+	if (vmbus_uio_subchannels_supported(device, device->primary))
+		return VMBUS_MAX_CHANNELS;
+	else
+		return 1;
+}
+
+/* Setup secondary channel */
+int rte_vmbus_subchan_open(struct vmbus_channel *primary,
+			   struct vmbus_channel **new_chan)
+{
+	struct vmbus_channel *chan;
+	int err;
+
+	err = vmbus_uio_get_subchan(primary, &chan);
+	if (err)
+		return err;
+
+	STAILQ_INSERT_TAIL(&primary->subchannel_list, chan, next);
+	*new_chan = chan;
+	return 0;
+}
+
+uint16_t rte_vmbus_sub_channel_index(const struct vmbus_channel *chan)
+{
+	return chan->subchannel_id;
+}
+
+void rte_vmbus_chan_close(struct vmbus_channel *chan)
+{
+	const struct rte_vmbus_device *device = chan->device;
+	struct vmbus_channel *primary = device->primary;
+
+	if (chan != primary)
+		STAILQ_REMOVE(&primary->subchannel_list, chan,
+			      vmbus_channel, next);
+
+	rte_free(chan);
+}
+
+static void vmbus_dump_ring(FILE *f, const char *id, const struct vmbus_br *br)
+{
+	const struct vmbus_bufring *vbr = br->vbr;
+	struct vmbus_chanpkt_hdr pkt;
+
+	fprintf(f, "%s windex=%u rindex=%u mask=%u pending=%u feature=%#x\n",
+		id, vbr->windex, vbr->rindex, vbr->imask,
+		vbr->pending_send, vbr->feature_bits.value);
+	fprintf(f, " size=%u avail write=%u read=%u\n",
+		br->dsize, vmbus_br_availwrite(br, vbr->windex),
+		vmbus_br_availread(br));
+
+	if (vmbus_rxbr_peek(br, &pkt, sizeof(pkt)) == 0)
+		fprintf(f, "  pkt type %#x len %u flags %#x xactid %#"PRIx64"\n",
+			pkt.type,
+			pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT,
+			pkt.flags, pkt.xactid);
+}
+
+void rte_vmbus_chan_dump(FILE *f, const struct vmbus_channel *chan)
+{
+	fprintf(f, "channel[%u] relid=%u monitor=%u\n",
+		chan->subchannel_id, chan->relid, chan->monitor_id);
+	vmbus_dump_ring(f, "rxbr", &chan->rxbr);
+	vmbus_dump_ring(f, "txbr", &chan->txbr);
+}
diff --git a/drivers/bus/vmbus/vmbus_common.c b/drivers/bus/vmbus/vmbus_common.c
new file mode 100644
index 00000000..c7165ad5
--- /dev/null
+++ b/drivers/bus/vmbus/vmbus_common.c
@@ -0,0 +1,286 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#include <string.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_eal.h>
+#include <rte_tailq.h>
+#include <rte_devargs.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include <rte_memory.h>
+#include <rte_bus_vmbus.h>
+
+#include "private.h"
+
+int vmbus_logtype_bus;
+extern struct rte_vmbus_bus rte_vmbus_bus;
+
+/* map a particular resource from a file */
+void *
+vmbus_map_resource(void *requested_addr, int fd, off_t offset, size_t size,
+		   int flags)
+{
+	void *mapaddr;
+
+	/* Map the memory resource of device */
+	mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE,
+		       MAP_SHARED | flags, fd, offset);
+	if (mapaddr == MAP_FAILED) {
+		VMBUS_LOG(ERR,
+			  "mmap(%d, %p, %zu, %ld) failed: %s",
+			  fd, requested_addr, size, (long)offset,
+			  strerror(errno));
+	}
+	return mapaddr;
+}
+
+/* unmap a particular resource */
+void
+vmbus_unmap_resource(void *requested_addr, size_t size)
+{
+	if (requested_addr == NULL)
+		return;
+
+	/* Unmap the VMBUS memory resource of device */
+	if (munmap(requested_addr, size)) {
+		VMBUS_LOG(ERR, "munmap(%p, 0x%lx) failed: %s",
+			requested_addr, (unsigned long)size,
+			strerror(errno));
+	} else
+		VMBUS_LOG(DEBUG, "  VMBUS memory unmapped at %p",
+			  requested_addr);
+}
+
+/**
+ * Match the VMBUS driver and device using UUID table
+ *
+ * @param drv
+ *	VMBUS driver from which ID table would be extracted
+ * @param pci_dev
+ *	VMBUS device to match against the driver
+ * @return
+ *	true for successful match
+ *	false for unsuccessful match
+ */
+static bool
+vmbus_match(const struct rte_vmbus_driver *dr,
+	    const struct rte_vmbus_device *dev)
+{
+	const rte_uuid_t *id_table;
+
+	for (id_table = dr->id_table; !rte_uuid_is_null(*id_table); ++id_table) {
+		if (rte_uuid_compare(*id_table, dev->class_id) == 0)
+			return true;
+	}
+
+	return false;
+}
+
+/*
+ * If device ID match, call the devinit() function of the driver.
+ */
+static int
+vmbus_probe_one_driver(struct rte_vmbus_driver *dr,
+		       struct rte_vmbus_device *dev)
+{
+	char guid[RTE_UUID_STRLEN];
+	int ret;
+
+	if (!vmbus_match(dr, dev))
+		return 1;	 /* not supported */
+
+	rte_uuid_unparse(dev->device_id, guid, sizeof(guid));
+	VMBUS_LOG(INFO, "VMBUS device %s on NUMA socket %i",
+		  guid, dev->device.numa_node);
+
+	/* TODO add blacklisted */
+
+	/* map resources for device */
+	ret = rte_vmbus_map_device(dev);
+	if (ret != 0)
+		return ret;
+
+	/* reference driver structure */
+	dev->driver = dr;
+	dev->device.driver = &dr->driver;
+
+	if (dev->device.numa_node < 0) {
+		VMBUS_LOG(WARNING, "  Invalid NUMA socket, default to 0");
+		dev->device.numa_node = 0;
+	}
+
+	/* call the driver probe() function */
+	VMBUS_LOG(INFO, "  probe driver: %s", dr->driver.name);
+	ret = dr->probe(dr, dev);
+	if (ret) {
+		dev->driver = NULL;
+		rte_vmbus_unmap_device(dev);
+	}
+
+	return ret;
+}
+
+/*
+ * IF device class GUID mathces, call the probe function of
+ * registere drivers for the vmbus device.
+ * Return -1 if initialization failed,
+ * and 1 if no driver found for this device.
+ */
+static int
+vmbus_probe_all_drivers(struct rte_vmbus_device *dev)
+{
+	struct rte_vmbus_driver *dr;
+	int rc;
+
+	/* Check if a driver is already loaded */
+	if (dev->driver != NULL) {
+		VMBUS_LOG(DEBUG, "VMBUS driver already loaded");
+		return 0;
+	}
+
+	FOREACH_DRIVER_ON_VMBUS(dr) {
+		rc = vmbus_probe_one_driver(dr, dev);
+		if (rc < 0) /* negative is an error */
+			return -1;
+
+		if (rc > 0) /* positive driver doesn't support it */
+			continue;
+
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the vmbus, and call the devinit() function for
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+int
+rte_vmbus_probe(void)
+{
+	struct rte_vmbus_device *dev;
+	size_t probed = 0, failed = 0;
+	char ubuf[RTE_UUID_STRLEN];
+
+	FOREACH_DEVICE_ON_VMBUS(dev) {
+		probed++;
+
+		rte_uuid_unparse(dev->device_id, ubuf, sizeof(ubuf));
+
+		/* TODO: add whitelist/blacklist */
+
+		if (vmbus_probe_all_drivers(dev) < 0) {
+			VMBUS_LOG(NOTICE,
+				"Requested device %s cannot be used", ubuf);
+			rte_errno = errno;
+			failed++;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+vmbus_parse(const char *name, void *addr)
+{
+	rte_uuid_t guid;
+	int ret;
+
+	ret = rte_uuid_parse(name, guid);
+	if (ret == 0 && addr)
+		memcpy(addr, &guid, sizeof(guid));
+
+	return ret;
+}
+
+/* register vmbus driver */
+void
+rte_vmbus_register(struct rte_vmbus_driver *driver)
+{
+	VMBUS_LOG(DEBUG,
+		"Registered driver %s", driver->driver.name);
+
+	TAILQ_INSERT_TAIL(&rte_vmbus_bus.driver_list, driver, next);
+	driver->bus = &rte_vmbus_bus;
+}
+
+/* unregister vmbus driver */
+void
+rte_vmbus_unregister(struct rte_vmbus_driver *driver)
+{
+	TAILQ_REMOVE(&rte_vmbus_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to VMBUS bus */
+void
+vmbus_add_device(struct rte_vmbus_device *vmbus_dev)
+{
+	TAILQ_INSERT_TAIL(&rte_vmbus_bus.device_list, vmbus_dev, next);
+}
+
+/* Insert a device into a predefined position in VMBUS bus */
+void
+vmbus_insert_device(struct rte_vmbus_device *exist_vmbus_dev,
+		      struct rte_vmbus_device *new_vmbus_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_vmbus_dev, new_vmbus_dev, next);
+}
+
+/* Remove a device from VMBUS bus */
+void
+vmbus_remove_device(struct rte_vmbus_device *vmbus_dev)
+{
+	TAILQ_REMOVE(&rte_vmbus_bus.device_list, vmbus_dev, next);
+}
+
+/* VMBUS doesn't support hotplug */
+static struct rte_device *
+vmbus_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		  const void *data)
+{
+	struct rte_vmbus_device *dev;
+
+	FOREACH_DEVICE_ON_VMBUS(dev) {
+		if (start && &dev->device == start) {
+			start = NULL;
+			continue;
+		}
+		if (cmp(&dev->device, data) == 0)
+			return &dev->device;
+	}
+
+	return NULL;
+}
+
+
+struct rte_vmbus_bus rte_vmbus_bus = {
+	.bus = {
+		.scan = rte_vmbus_scan,
+		.probe = rte_vmbus_probe,
+		.find_device = vmbus_find_device,
+		.parse = vmbus_parse,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(rte_vmbus_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(rte_vmbus_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(vmbus, rte_vmbus_bus.bus);
+
+RTE_INIT(vmbus_init_log)
+{
+	vmbus_logtype_bus = rte_log_register("bus.vmbus");
+	if (vmbus_logtype_bus >= 0)
+		rte_log_set_level(vmbus_logtype_bus, RTE_LOG_NOTICE);
+}
diff --git a/drivers/bus/vmbus/vmbus_common_uio.c b/drivers/bus/vmbus/vmbus_common_uio.c
new file mode 100644
index 00000000..5ddd36ab
--- /dev/null
+++ b/drivers/bus/vmbus/vmbus_common_uio.c
@@ -0,0 +1,232 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018, Microsoft Corporation.
+ * All Rights Reserved.
+ */
+
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+
+#include <rte_eal.h>
+#include <rte_tailq.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_bus.h>
+#include <rte_bus_vmbus.h>
+
+#include "private.h"
+
+static struct rte_tailq_elem vmbus_tailq = {
+	.name = "VMBUS_RESOURCE_LIST",
+};
+EAL_REGISTER_TAILQ(vmbus_tailq)
+
+static int
+vmbus_uio_map_secondary(struct rte_vmbus_device *dev)
+{
+	int fd, i;
+	struct mapped_vmbus_resource *uio_res;
+	struct mapped_vmbus_res_list *uio_res_list
+		= RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list);
+
+	TAILQ_FOREACH(uio_res, uio_res_list, next) {
+
+		/* skip this element if it doesn't match our UUID */
+		if (rte_uuid_compare(uio_res->id, dev->device_id) != 0)
+			continue;
+
+		/* open /dev/uioX */
+		fd = open(uio_res->path, O_RDWR);
+		if (fd < 0) {
+			VMBUS_LOG(ERR, "Cannot open %s: %s",
+				  uio_res->path, strerror(errno));
+			return -1;
+		}
+
+		for (i = 0; i != uio_res->nb_maps; i++) {
+			void *mapaddr;
+
+			mapaddr = vmbus_map_resource(uio_res->maps[i].addr,
+						     fd, 0,
+						     uio_res->maps[i].size, 0);
+
+			if (mapaddr == uio_res->maps[i].addr)
+				continue;
+
+			VMBUS_LOG(ERR,
+				  "Cannot mmap device resource file %s to address: %p",
+				  uio_res->path, uio_res->maps[i].addr);
+
+			if (mapaddr != MAP_FAILED)
+				/* unmap addr wrongly mapped */
+				vmbus_unmap_resource(mapaddr,
+						     (size_t)uio_res->maps[i].size);
+
+			/* unmap addrs correctly mapped */
+			while (--i >= 0)
+				vmbus_unmap_resource(uio_res->maps[i].addr,
+						     (size_t)uio_res->maps[i].size);
+
+			close(fd);
+			return -1;
+		}
+
+		/* fd is not needed in slave process, close it */
+		close(fd);
+		return 0;
+	}
+
+	VMBUS_LOG(ERR,  "Cannot find resource for device");
+	return 1;
+}
+
+static int
+vmbus_uio_map_primary(struct rte_vmbus_device *dev)
+{
+	int i, ret;
+	struct mapped_vmbus_resource *uio_res = NULL;
+	struct mapped_vmbus_res_list *uio_res_list =
+		RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list);
+
+	/* allocate uio resource */
+	ret = vmbus_uio_alloc_resource(dev, &uio_res);
+	if (ret)
+		return ret;
+
+	/* Map the resources */
+	for (i = 0; i < VMBUS_MAX_RESOURCE; i++) {
+		/* skip empty BAR */
+		if (dev->resource[i].len == 0)
+			continue;
+
+		ret = vmbus_uio_map_resource_by_index(dev, i, uio_res, 0);
+		if (ret)
+			goto error;
+	}
+
+	uio_res->nb_maps = i;
+
+	TAILQ_INSERT_TAIL(uio_res_list, uio_res, next);
+
+	return 0;
+error:
+	while (--i >= 0) {
+		vmbus_unmap_resource(uio_res->maps[i].addr,
+				(size_t)uio_res->maps[i].size);
+	}
+	vmbus_uio_free_resource(dev, uio_res);
+	return -1;
+}
+
+
+struct mapped_vmbus_resource *
+vmbus_uio_find_resource(const struct rte_vmbus_device *dev)
+{
+	struct mapped_vmbus_resource *uio_res;
+	struct mapped_vmbus_res_list *uio_res_list =
+			RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list);
+
+	if (dev == NULL)
+		return NULL;
+
+	TAILQ_FOREACH(uio_res, uio_res_list, next) {
+		/* skip this element if it doesn't match our VMBUS address */
+		if (rte_uuid_compare(uio_res->id, dev->device_id) == 0)
+			return uio_res;
+	}
+	return NULL;
+}
+
+/* map the VMBUS resource of a VMBUS device in virtual memory */
+int
+vmbus_uio_map_resource(struct rte_vmbus_device *dev)
+{
+	struct mapped_vmbus_resource *uio_res;
+	int ret;
+
+	/* TODO: handle rescind */
+	dev->intr_handle.fd = -1;
+	dev->intr_handle.uio_cfg_fd = -1;
+	dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
+
+	/* secondary processes - use already recorded details */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		ret = vmbus_uio_map_secondary(dev);
+	else
+		ret = vmbus_uio_map_primary(dev);
+
+	if (ret != 0)
+		return ret;
+
+	uio_res = vmbus_uio_find_resource(dev);
+	if (!uio_res) {
+		VMBUS_LOG(ERR, "can not find resources!");
+		return -EIO;
+	}
+
+	if (uio_res->nb_maps <= HV_MON_PAGE_MAP) {
+		VMBUS_LOG(ERR, "VMBUS: only %u resources found!",
+			uio_res->nb_maps);
+		return -EINVAL;
+	}
+
+	dev->int_page = (uint32_t *)((char *)uio_res->maps[HV_INT_PAGE_MAP].addr
+				     + (PAGE_SIZE >> 1));
+	dev->monitor_page = uio_res->maps[HV_MON_PAGE_MAP].addr;
+	return 0;
+}
+
+static void
+vmbus_uio_unmap(struct mapped_vmbus_resource *uio_res)
+{
+	int i;
+
+	if (uio_res == NULL)
+		return;
+
+	for (i = 0; i != uio_res->nb_maps; i++) {
+		vmbus_unmap_resource(uio_res->maps[i].addr,
+				     (size_t)uio_res->maps[i].size);
+	}
+}
+
+/* unmap the VMBUS resource of a VMBUS device in virtual memory */
+void
+vmbus_uio_unmap_resource(struct rte_vmbus_device *dev)
+{
+	struct mapped_vmbus_resource *uio_res;
+	struct mapped_vmbus_res_list *uio_res_list =
+			RTE_TAILQ_CAST(vmbus_tailq.head, mapped_vmbus_res_list);
+
+	if (dev == NULL)
+		return;
+
+	/* find an entry for the device */
+	uio_res = vmbus_uio_find_resource(dev);
+	if (uio_res == NULL)
+		return;
+
+	/* secondary processes - just free maps */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return vmbus_uio_unmap(uio_res);
+
+	TAILQ_REMOVE(uio_res_list, uio_res, next);
+
+	/* unmap all resources */
+	vmbus_uio_unmap(uio_res);
+
+	/* free uio resource */
+	rte_free(uio_res);
+
+	/* close fd if in primary process */
+	close(dev->intr_handle.fd);
+	if (dev->intr_handle.uio_cfg_fd >= 0) {
+		close(dev->intr_handle.uio_cfg_fd);
+		dev->intr_handle.uio_cfg_fd = -1;
+	}
+
+	dev->intr_handle.fd = -1;
+	dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
+}
diff --git a/drivers/common/meson.build b/drivers/common/meson.build
index 5f6341b8..d7b7d8cf 100644
--- a/drivers/common/meson.build
+++ b/drivers/common/meson.build
@@ -2,6 +2,6 @@
 # Copyright(c) 2018 Cavium, Inc
 
 std_deps = ['eal']
-drivers = ['octeontx']
+drivers = ['octeontx', 'qat']
 config_flag_fmt = 'RTE_LIBRTE_@0@_COMMON'
 driver_name_fmt = 'rte_common_@0@'
diff --git a/drivers/common/octeontx/octeontx_mbox.c b/drivers/common/octeontx/octeontx_mbox.c
index 93e6e857..880f8a40 100644
--- a/drivers/common/octeontx/octeontx_mbox.c
+++ b/drivers/common/octeontx/octeontx_mbox.c
@@ -61,9 +61,7 @@ struct mbox_ram_hdr {
 
 int octeontx_logtype_mbox;
 
-RTE_INIT(otx_init_log);
-static void
-otx_init_log(void)
+RTE_INIT(otx_init_log)
 {
 	octeontx_logtype_mbox = rte_log_register("pmd.octeontx.mbox");
 	if (octeontx_logtype_mbox >= 0)
diff --git a/drivers/common/qat/Makefile b/drivers/common/qat/Makefile
new file mode 100644
index 00000000..c68a032a
--- /dev/null
+++ b/drivers/common/qat/Makefile
@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2015-2018 Intel Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# build directories
+QAT_CRYPTO_DIR := $(RTE_SDK)/drivers/crypto/qat
+QAT_COMPRESS_DIR := $(RTE_SDK)/drivers/compress/qat
+VPATH=$(QAT_CRYPTO_DIR):$(QAT_COMPRESS_DIR)
+
+# external library include paths
+CFLAGS += -I$(SRCDIR)/qat_adf
+CFLAGS += -I$(SRCDIR)
+CFLAGS += -I$(QAT_CRYPTO_DIR)
+CFLAGS += -I$(QAT_COMPRESS_DIR)
+
+
+ifeq ($(CONFIG_RTE_LIBRTE_COMPRESSDEV),y)
+	CFLAGS += -DALLOW_EXPERIMENTAL_API
+	LDLIBS += -lrte_compressdev
+	SRCS-y += qat_comp.c
+	SRCS-y += qat_comp_pmd.c
+	build_qat = yes
+endif
+
+# library symmetric crypto source files
+ifeq ($(CONFIG_RTE_LIBRTE_CRYPTODEV),y)
+ifeq ($(CONFIG_RTE_LIBRTE_PMD_QAT_SYM),y)
+	LDLIBS += -lrte_cryptodev
+	LDLIBS += -lcrypto
+	CFLAGS += -DBUILD_QAT_SYM
+	SRCS-y += qat_sym.c
+	SRCS-y += qat_sym_session.c
+	SRCS-y += qat_sym_pmd.c
+	build_qat = yes
+endif
+endif
+
+ifdef build_qat
+
+	# library name
+	LIB = librte_pmd_qat.a
+
+	# library version
+	LIBABIVER := 1
+	# build flags
+	CFLAGS += $(WERROR_FLAGS)
+	CFLAGS += -O3
+
+	# library common source files
+	SRCS-y += qat_device.c
+	SRCS-y += qat_common.c
+	SRCS-y += qat_logs.c
+	SRCS-y += qat_qp.c
+
+	LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool
+	LDLIBS += -lrte_pci -lrte_bus_pci
+
+	# export include files
+	SYMLINK-y-include +=
+
+	# versioning export map
+	EXPORT_MAP := ../../compress/qat/rte_pmd_qat_version.map
+endif
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/common/qat/meson.build b/drivers/common/qat/meson.build
new file mode 100644
index 00000000..80b6b25a
--- /dev/null
+++ b/drivers/common/qat/meson.build
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2017-2018 Intel Corporation
+
+# This does not build a driver, but instead holds common files for
+# the crypto and compression drivers.
+build = false
+qat_deps = ['bus_pci']
+qat_sources = files('qat_common.c',
+		'qat_qp.c',
+		'qat_device.c',
+		'qat_logs.c')
+qat_includes = [include_directories('.', 'qat_adf')]
+qat_ext_deps = []
+qat_cflags = []
diff --git a/drivers/crypto/qat/qat_adf/adf_transport_access_macros.h b/drivers/common/qat/qat_adf/adf_transport_access_macros.h
index bfdbc979..1eef5513 100644
--- a/drivers/crypto/qat/qat_adf/adf_transport_access_macros.h
+++ b/drivers/common/qat/qat_adf/adf_transport_access_macros.h
@@ -50,8 +50,10 @@
 #define ADF_MAX_RING_SIZE ADF_RING_SIZE_4M
 #define ADF_DEFAULT_RING_SIZE ADF_RING_SIZE_16K
 
-#define ADF_NUM_BUNDLES_PER_DEV         1
-#define ADF_NUM_SYM_QPS_PER_BUNDLE      2
+/* Maximum number of qps on a device for any service type */
+#define ADF_MAX_QPS_ON_ANY_SERVICE	2
+#define ADF_RING_DIR_TX			0
+#define ADF_RING_DIR_RX			1
 
 /* Valid internal msg size values */
 #define ADF_MSG_SIZE_32 0x01
@@ -130,4 +132,5 @@ do { \
 #define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \
 	ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \
 			ADF_RING_CSR_INT_FLAG_AND_COL, value)
-#endif
+
+#endif /*ADF_TRANSPORT_ACCESS_MACROS_H */
diff --git a/drivers/crypto/qat/qat_adf/icp_qat_fw.h b/drivers/common/qat/qat_adf/icp_qat_fw.h
index ae39b7f1..8f7cb37b 100644
--- a/drivers/crypto/qat/qat_adf/icp_qat_fw.h
+++ b/drivers/common/qat/qat_adf/icp_qat_fw.h
@@ -117,6 +117,10 @@ struct icp_qat_fw_comn_resp {
 #define ICP_QAT_FW_COMN_VALID_FLAG_BITPOS 7
 #define ICP_QAT_FW_COMN_VALID_FLAG_MASK 0x1
 #define ICP_QAT_FW_COMN_HDR_RESRVD_FLD_MASK 0x7F
+#define ICP_QAT_FW_COMN_CNV_FLAG_BITPOS 6
+#define ICP_QAT_FW_COMN_CNV_FLAG_MASK 0x1
+#define ICP_QAT_FW_COMN_CNVNR_FLAG_BITPOS 5
+#define ICP_QAT_FW_COMN_CNVNR_FLAG_MASK 0x1
 
 #define ICP_QAT_FW_COMN_OV_SRV_TYPE_GET(icp_qat_fw_comn_req_hdr_t) \
 	icp_qat_fw_comn_req_hdr_t.service_type
@@ -133,6 +137,16 @@ struct icp_qat_fw_comn_resp {
 #define ICP_QAT_FW_COMN_HDR_VALID_FLAG_GET(hdr_t) \
 	ICP_QAT_FW_COMN_VALID_FLAG_GET(hdr_t.hdr_flags)
 
+#define ICP_QAT_FW_COMN_HDR_CNVNR_FLAG_GET(hdr_flags) \
+	QAT_FIELD_GET(hdr_flags, \
+		ICP_QAT_FW_COMN_CNVNR_FLAG_BITPOS, \
+		ICP_QAT_FW_COMN_CNVNR_FLAG_MASK)
+
+#define ICP_QAT_FW_COMN_HDR_CNV_FLAG_GET(hdr_flags) \
+	QAT_FIELD_GET(hdr_flags, \
+		ICP_QAT_FW_COMN_CNV_FLAG_BITPOS, \
+		ICP_QAT_FW_COMN_CNV_FLAG_MASK)
+
 #define ICP_QAT_FW_COMN_HDR_VALID_FLAG_SET(hdr_t, val) \
 	ICP_QAT_FW_COMN_VALID_FLAG_SET(hdr_t, val)
 
@@ -204,29 +218,44 @@ struct icp_qat_fw_comn_resp {
 	& ICP_QAT_FW_COMN_NEXT_ID_MASK) | \
 	((val) & ICP_QAT_FW_COMN_CURR_ID_MASK)); }
 
+#define ICP_QAT_FW_COMN_NEXT_ID_SET_2(next_curr_id, val)                       \
+	do {                                                                   \
+		(next_curr_id) =                                               \
+		    (((next_curr_id) & ICP_QAT_FW_COMN_CURR_ID_MASK) |         \
+		     (((val) << ICP_QAT_FW_COMN_NEXT_ID_BITPOS) &              \
+		      ICP_QAT_FW_COMN_NEXT_ID_MASK))                           \
+	} while (0)
+
+#define ICP_QAT_FW_COMN_CURR_ID_SET_2(next_curr_id, val)                       \
+	do {                                                                   \
+		(next_curr_id) =                                               \
+		    (((next_curr_id) & ICP_QAT_FW_COMN_NEXT_ID_MASK) |         \
+		     ((val) & ICP_QAT_FW_COMN_CURR_ID_MASK))                   \
+	} while (0)
+
 #define QAT_COMN_RESP_CRYPTO_STATUS_BITPOS 7
 #define QAT_COMN_RESP_CRYPTO_STATUS_MASK 0x1
+#define QAT_COMN_RESP_PKE_STATUS_BITPOS 6
+#define QAT_COMN_RESP_PKE_STATUS_MASK 0x1
 #define QAT_COMN_RESP_CMP_STATUS_BITPOS 5
 #define QAT_COMN_RESP_CMP_STATUS_MASK 0x1
 #define QAT_COMN_RESP_XLAT_STATUS_BITPOS 4
 #define QAT_COMN_RESP_XLAT_STATUS_MASK 0x1
 #define QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS 3
 #define QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK 0x1
-
-#define ICP_QAT_FW_COMN_RESP_STATUS_BUILD(crypto, comp, xlat, eolb) \
-	((((crypto) & QAT_COMN_RESP_CRYPTO_STATUS_MASK) << \
-	QAT_COMN_RESP_CRYPTO_STATUS_BITPOS) | \
-	(((comp) & QAT_COMN_RESP_CMP_STATUS_MASK) << \
-	QAT_COMN_RESP_CMP_STATUS_BITPOS) | \
-	(((xlat) & QAT_COMN_RESP_XLAT_STATUS_MASK) << \
-	QAT_COMN_RESP_XLAT_STATUS_BITPOS) | \
-	(((eolb) & QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK) << \
-	QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS))
+#define QAT_COMN_RESP_UNSUPPORTED_REQUEST_BITPOS 2
+#define QAT_COMN_RESP_UNSUPPORTED_REQUEST_MASK 0x1
+#define QAT_COMN_RESP_XLT_WA_APPLIED_BITPOS 0
+#define QAT_COMN_RESP_XLT_WA_APPLIED_MASK 0x1
 
 #define ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(status) \
 	QAT_FIELD_GET(status, QAT_COMN_RESP_CRYPTO_STATUS_BITPOS, \
 	QAT_COMN_RESP_CRYPTO_STATUS_MASK)
 
+#define ICP_QAT_FW_COMN_RESP_PKE_STAT_GET(status) \
+	QAT_FIELD_GET(status, QAT_COMN_RESP_PKE_STATUS_BITPOS, \
+	QAT_COMN_RESP_PKE_STATUS_MASK)
+
 #define ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(status) \
 	QAT_FIELD_GET(status, QAT_COMN_RESP_CMP_STATUS_BITPOS, \
 	QAT_COMN_RESP_CMP_STATUS_MASK)
@@ -235,10 +264,18 @@ struct icp_qat_fw_comn_resp {
 	QAT_FIELD_GET(status, QAT_COMN_RESP_XLAT_STATUS_BITPOS, \
 	QAT_COMN_RESP_XLAT_STATUS_MASK)
 
+#define ICP_QAT_FW_COMN_RESP_XLT_WA_APPLIED_GET(status) \
+	QAT_FIELD_GET(status, QAT_COMN_RESP_XLT_WA_APPLIED_BITPOS, \
+	QAT_COMN_RESP_XLT_WA_APPLIED_MASK)
+
 #define ICP_QAT_FW_COMN_RESP_CMP_END_OF_LAST_BLK_FLAG_GET(status) \
 	QAT_FIELD_GET(status, QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS, \
 	QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK)
 
+#define ICP_QAT_FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET(status) \
+	QAT_FIELD_GET(status, QAT_COMN_RESP_UNSUPPORTED_REQUEST_BITPOS, \
+	QAT_COMN_RESP_UNSUPPORTED_REQUEST_MASK)
+
 #define ICP_QAT_FW_COMN_STATUS_FLAG_OK 0
 #define ICP_QAT_FW_COMN_STATUS_FLAG_ERROR 1
 #define ICP_QAT_FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_CLR 0
@@ -257,8 +294,16 @@ struct icp_qat_fw_comn_resp {
 #define ERR_CODE_OVERFLOW_ERROR -11
 #define ERR_CODE_SOFT_ERROR -12
 #define ERR_CODE_FATAL_ERROR -13
-#define ERR_CODE_SSM_ERROR -14
-#define ERR_CODE_ENDPOINT_ERROR -15
+#define ERR_CODE_COMP_OUTPUT_CORRUPTION -14
+#define ERR_CODE_HW_INCOMPLETE_FILE -15
+#define ERR_CODE_SSM_ERROR -16
+#define ERR_CODE_ENDPOINT_ERROR -17
+#define ERR_CODE_CNV_ERROR -18
+#define ERR_CODE_EMPTY_DYM_BLOCK -19
+#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_INVALID_HANDLE -20
+#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_HMAC_FAILED -21
+#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_INVALID_WRAPPING_ALGO -22
+#define ERR_CODE_KPT_DRNG_SEED_NOT_LOAD -23
 
 enum icp_qat_fw_slice {
 	ICP_QAT_FW_SLICE_NULL = 0,
diff --git a/drivers/common/qat/qat_adf/icp_qat_fw_comp.h b/drivers/common/qat/qat_adf/icp_qat_fw_comp.h
new file mode 100644
index 00000000..81381772
--- /dev/null
+++ b/drivers/common/qat/qat_adf/icp_qat_fw_comp.h
@@ -0,0 +1,482 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+ * Copyright(c) 2015-2018 Intel Corporation
+ */
+#ifndef _ICP_QAT_FW_COMP_H_
+#define _ICP_QAT_FW_COMP_H_
+
+#include "icp_qat_fw.h"
+
+enum icp_qat_fw_comp_cmd_id {
+	ICP_QAT_FW_COMP_CMD_STATIC = 0,
+	/*!< Static Compress Request */
+
+	ICP_QAT_FW_COMP_CMD_DYNAMIC = 1,
+	/*!< Dynamic Compress Request */
+
+	ICP_QAT_FW_COMP_CMD_DECOMPRESS = 2,
+	/*!< Decompress Request */
+
+	ICP_QAT_FW_COMP_CMD_DELIMITER
+	/**< Delimiter type */
+};
+
+/**< Flag usage */
+
+#define ICP_QAT_FW_COMP_STATELESS_SESSION 0
+/**< @ingroup icp_qat_fw_comp
+ *  Flag representing that session is stateless
+ */
+
+#define ICP_QAT_FW_COMP_STATEFUL_SESSION 1
+/**< @ingroup icp_qat_fw_comp
+ *  Flag representing that session is stateful
+ */
+
+#define ICP_QAT_FW_COMP_NOT_AUTO_SELECT_BEST 0
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing that autoselectbest is NOT used
+ */
+
+#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST 1
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing that autoselectbest is used
+ */
+
+#define ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST 0
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing that enhanced autoselectbest is NOT used
+ */
+
+#define ICP_QAT_FW_COMP_ENH_AUTO_SELECT_BEST 1
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing that enhanced autoselectbest is used
+ */
+
+#define ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST 0
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing that enhanced autoselectbest is NOT used
+ */
+
+#define ICP_QAT_FW_COMP_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST 1
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing that enhanced autoselectbest is used
+ */
+
+#define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_USED_AS_INTMD_BUF 1
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing secure RAM from being used as
+ * an intermediate buffer is DISABLED.
+ */
+
+#define ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF 0
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing secure RAM from being used as
+ * an intermediate buffer is ENABLED.
+ */
+
+/**< Flag mask & bit position */
+
+#define ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS 2
+/**< @ingroup icp_qat_fw_comp
+ * Starting bit position for the session type
+ */
+
+#define ICP_QAT_FW_COMP_SESSION_TYPE_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ * One bit mask used to determine the session type
+ */
+
+#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_BITPOS 3
+/**< @ingroup icp_qat_fw_comp
+ * Starting bit position for auto select best
+ */
+
+#define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ * One bit mask for auto select best
+ */
+
+#define ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_BITPOS 4
+/**< @ingroup icp_qat_fw_comp
+ * Starting bit position for enhanced auto select best
+ */
+
+#define ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ * One bit mask for enhanced auto select best
+ */
+
+#define ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_BITPOS 5
+/**< @ingroup icp_qat_fw_comp
+ * Starting bit position for disabling type zero header write back
+ * when Enhanced autoselect best is enabled. If set firmware does
+ * not return type0 store block header, only copies src to dest.
+ * (if best output is Type0)
+ */
+
+#define ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ * One bit mask for auto select best
+ */
+
+#define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS 7
+/**< @ingroup icp_qat_fw_comp
+ * Starting bit position for flag used to disable secure ram from
+ *  being used as an intermediate buffer.
+ */
+
+#define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ * One bit mask for disable secure ram for use as an intermediate
+ * buffer.
+ */
+
+#define ICP_QAT_FW_COMP_FLAGS_BUILD(sesstype, autoselect, enhanced_asb,        \
+				    ret_uncomp, secure_ram)                    \
+	((((sesstype)&ICP_QAT_FW_COMP_SESSION_TYPE_MASK)                       \
+	  << ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS) |                            \
+	 (((autoselect)&ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MASK)                 \
+	  << ICP_QAT_FW_COMP_AUTO_SELECT_BEST_BITPOS) |                        \
+	 (((enhanced_asb)&ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_MASK)      \
+	  << ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_BITPOS) |               \
+	 (((ret_uncomp)&ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_MASK)    \
+	  << ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_BITPOS) |           \
+	 (((secure_ram)&ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK)  \
+	  << ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS))
+
+union icp_qat_fw_comp_req_hdr_cd_pars {
+	/**< LWs 2-5 */
+	struct {
+		uint64_t content_desc_addr;
+		/**< Address of the content descriptor */
+
+		uint16_t content_desc_resrvd1;
+		/**< Content descriptor reserved field */
+
+		uint8_t content_desc_params_sz;
+		/**< Size of the content descriptor parameters in quad words.
+		 * These parameters describe the session setup configuration
+		 * info for the slices that this request relies upon i.e.
+		 * the configuration word and cipher key needed by the cipher
+		 * slice if there is a request for cipher processing.
+		 */
+
+		uint8_t content_desc_hdr_resrvd2;
+		/**< Content descriptor reserved field */
+
+		uint32_t content_desc_resrvd3;
+		/**< Content descriptor reserved field */
+	} s;
+
+	struct {
+		uint32_t comp_slice_cfg_word[ICP_QAT_FW_NUM_LONGWORDS_2];
+		/* Compression Slice Config Word */
+
+		uint32_t content_desc_resrvd4;
+		/**< Content descriptor reserved field */
+
+	} sl;
+
+};
+
+struct icp_qat_fw_comp_req_params {
+	/**< LW 14 */
+	uint32_t comp_len;
+	/**< Size of input to process in bytes Note:  Only EOP requests can be
+	 * odd for decompression. IA must set LSB to zero for odd sized
+	 * intermediate inputs
+	 */
+
+	/**< LW 15 */
+	uint32_t out_buffer_sz;
+	/**< Size of output buffer in bytes */
+
+	/**< LW 16 */
+	uint32_t initial_crc32;
+	/**< CRC of previously processed bytes */
+
+	/**< LW 17 */
+	uint32_t initial_adler;
+	/**< Adler of previously processed bytes */
+
+	/**< LW 18 */
+	uint32_t req_par_flags;
+
+	/**< LW 19 */
+	uint32_t rsrvd;
+};
+
+#define ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(sop, eop, bfinal, cnv, cnvnr)    \
+	((((sop)&ICP_QAT_FW_COMP_SOP_MASK) << ICP_QAT_FW_COMP_SOP_BITPOS) |    \
+	 (((eop)&ICP_QAT_FW_COMP_EOP_MASK) << ICP_QAT_FW_COMP_EOP_BITPOS) |    \
+	 (((bfinal)&ICP_QAT_FW_COMP_BFINAL_MASK)                               \
+	  << ICP_QAT_FW_COMP_BFINAL_BITPOS) |                                  \
+	 ((cnv & ICP_QAT_FW_COMP_CNV_MASK) << ICP_QAT_FW_COMP_CNV_BITPOS) |    \
+	 ((cnvnr & ICP_QAT_FW_COMP_CNV_RECOVERY_MASK)                          \
+	  << ICP_QAT_FW_COMP_CNV_RECOVERY_BITPOS))
+
+#define ICP_QAT_FW_COMP_NOT_SOP 0
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing that a request is NOT Start of Packet
+ */
+
+#define ICP_QAT_FW_COMP_SOP 1
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing that a request IS Start of Packet
+ */
+
+#define ICP_QAT_FW_COMP_NOT_EOP 0
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing that a request is NOT Start of Packet
+ */
+
+#define ICP_QAT_FW_COMP_EOP 1
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing that a request IS End of Packet
+ */
+
+#define ICP_QAT_FW_COMP_NOT_BFINAL 0
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing to indicate firmware this is not the last block
+ */
+
+#define ICP_QAT_FW_COMP_BFINAL 1
+/**< @ingroup icp_qat_fw_comp
+ * Flag representing to indicate firmware this is the last block
+ */
+
+#define ICP_QAT_FW_COMP_NO_CNV 0
+/**< @ingroup icp_qat_fw_comp
+ * Flag indicating that NO cnv check is to be performed on the request
+ */
+
+#define ICP_QAT_FW_COMP_CNV 1
+/**< @ingroup icp_qat_fw_comp
+ * Flag indicating that a cnv check IS to be performed on the request
+ */
+
+#define ICP_QAT_FW_COMP_NO_CNV_RECOVERY 0
+/**< @ingroup icp_qat_fw_comp
+ * Flag indicating that NO cnv recovery is to be performed on the request
+ */
+
+#define ICP_QAT_FW_COMP_CNV_RECOVERY 1
+/**< @ingroup icp_qat_fw_comp
+ * Flag indicating that a cnv recovery is to be performed on the request
+ */
+
+#define ICP_QAT_FW_COMP_SOP_BITPOS 0
+/**< @ingroup icp_qat_fw_comp
+ * Starting bit position for SOP
+ */
+
+#define ICP_QAT_FW_COMP_SOP_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ *  One bit mask used to determine SOP
+ */
+
+#define ICP_QAT_FW_COMP_EOP_BITPOS 1
+/**< @ingroup icp_qat_fw_comp
+ *  Starting bit position for EOP
+ */
+
+#define ICP_QAT_FW_COMP_EOP_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ *  One bit mask used to determine EOP
+ */
+
+#define ICP_QAT_FW_COMP_BFINAL_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ *  One bit mask for the bfinal bit
+ */
+
+#define ICP_QAT_FW_COMP_BFINAL_BITPOS 6
+/**< @ingroup icp_qat_fw_comp
+ *  Starting bit position for the bfinal bit
+ */
+
+#define ICP_QAT_FW_COMP_CNV_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ * One bit mask for the CNV bit
+ */
+
+#define ICP_QAT_FW_COMP_CNV_BITPOS 16
+/**< @ingroup icp_qat_fw_comp
+ * Starting bit position for the CNV bit
+ */
+
+#define ICP_QAT_FW_COMP_CNV_RECOVERY_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ * One bit mask for the CNV Recovery bit
+ */
+
+#define ICP_QAT_FW_COMP_CNV_RECOVERY_BITPOS 17
+/**< @ingroup icp_qat_fw_comp
+ * Starting bit position for the CNV Recovery bit
+ */
+
+struct icp_qat_fw_xlt_req_params {
+	/**< LWs 20-21 */
+	uint64_t inter_buff_ptr;
+	/**< This field specifies the physical address of an intermediate
+	 *  buffer SGL array. The array contains a pair of 64-bit
+	 *  intermediate buffer pointers to SGL buffer descriptors, one pair
+	 *  per CPM. Please refer to the CPM1.6 Firmware Interface HLD
+	 *  specification for more details.
+	 */
+};
+
+
+struct icp_qat_fw_comp_cd_hdr {
+	/**< LW 24 */
+	uint16_t ram_bank_flags;
+	/**< Flags to show which ram banks to access */
+
+	uint8_t comp_cfg_offset;
+	/**< Quad word offset from the content descriptor parameters address
+	 * to the parameters for the compression processing
+	 */
+
+	uint8_t next_curr_id;
+	/**< This field combines the next and current id (each four bits) -
+	 * the next id is the most significant nibble.
+	 * Next Id:  Set to the next slice to pass the compressed data through.
+	 * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through
+	 * anymore slices after compression
+	 * Current Id: Initialised with the compression slice type
+	 */
+
+	/**< LW 25 */
+	uint32_t resrvd;
+	/**< LWs 26-27 */
+
+	uint64_t comp_state_addr;
+	/**< Pointer to compression state */
+
+	/**< LWs 28-29 */
+	uint64_t ram_banks_addr;
+	/**< Pointer to banks */
+
+};
+
+
+struct icp_qat_fw_xlt_cd_hdr {
+	/**< LW 30 */
+	uint16_t resrvd1;
+	/**< Reserved field and assumed set to 0 */
+
+	uint8_t resrvd2;
+	/**< Reserved field and assumed set to 0 */
+
+	uint8_t next_curr_id;
+	/**< This field combines the next and current id (each four bits) -
+	 * the next id is the most significant nibble.
+	 * Next Id:  Set to the next slice to pass the translated data through.
+	 * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through
+	 * any more slices after compression
+	 * Current Id: Initialised with the translation slice type
+	 */
+
+	/**< LW 31 */
+	uint32_t resrvd3;
+	/**< Reserved and should be set to zero, needed for quadword
+	 * alignment
+	 */
+};
+
+struct icp_qat_fw_comp_req {
+	/**< LWs 0-1 */
+	struct icp_qat_fw_comn_req_hdr comn_hdr;
+	/**< Common request header - for Service Command Id,
+	 * use service-specific Compression Command Id.
+	 * Service Specific Flags - use Compression Command Flags
+	 */
+
+	/**< LWs 2-5 */
+	union icp_qat_fw_comp_req_hdr_cd_pars cd_pars;
+	/**< Compression service-specific content descriptor field which points
+	 * either to a content descriptor parameter block or contains the
+	 * compression slice config word.
+	 */
+
+	/**< LWs 6-13 */
+	struct icp_qat_fw_comn_req_mid comn_mid;
+	/**< Common request middle section */
+
+	/**< LWs 14-19 */
+	struct icp_qat_fw_comp_req_params comp_pars;
+	/**< Compression request Parameters block */
+
+	/**< LWs 20-21 */
+	union {
+		struct icp_qat_fw_xlt_req_params xlt_pars;
+		/**< Translation request Parameters block */
+		uint32_t resrvd1[ICP_QAT_FW_NUM_LONGWORDS_2];
+		/**< Reserved if not used for translation */
+
+	} u1;
+
+	/**< LWs 22-23 */
+	union {
+		uint32_t resrvd2[ICP_QAT_FW_NUM_LONGWORDS_2];
+		/**< Reserved - not used if Batch and Pack is disabled.*/
+
+		uint64_t bnp_res_table_addr;
+		/**< A generic pointer to the unbounded list of
+		 * icp_qat_fw_resp_comp_pars members. This pointer is only
+		 * used when the Batch and Pack is enabled.
+		 */
+	} u3;
+
+	/**< LWs 24-29 */
+	struct icp_qat_fw_comp_cd_hdr comp_cd_ctrl;
+	/**< Compression request content descriptor control block header */
+
+	/**< LWs 30-31 */
+	union {
+		struct icp_qat_fw_xlt_cd_hdr xlt_cd_ctrl;
+		/**< Translation request content descriptor
+		 * control block header
+		 */
+
+		uint32_t resrvd3[ICP_QAT_FW_NUM_LONGWORDS_2];
+		/**< Reserved if not used for translation */
+	} u2;
+};
+
+struct icp_qat_fw_resp_comp_pars {
+	/**< LW 4 */
+	uint32_t input_byte_counter;
+	/**< Input byte counter */
+
+	/**< LW 5 */
+	uint32_t output_byte_counter;
+	/**< Output byte counter */
+
+	/**< LW 6 & 7*/
+	union {
+		uint64_t curr_chksum;
+		struct {
+			/**< LW 6 */
+			uint32_t curr_crc32;
+			/**< LW 7 */
+			uint32_t curr_adler_32;
+		};
+	};
+};
+
+struct icp_qat_fw_comp_resp {
+	/**< LWs 0-1 */
+	struct icp_qat_fw_comn_resp_hdr comn_resp;
+	/**< Common interface response format see icp_qat_fw.h */
+
+	/**< LWs 2-3 */
+	uint64_t opaque_data;
+	/**< Opaque data passed from the request to the response message */
+
+	/**< LWs 4-7 */
+	struct icp_qat_fw_resp_comp_pars comp_resp_pars;
+	/**< Common response params (checksums and byte counts) */
+};
+
+#endif
diff --git a/drivers/crypto/qat/qat_adf/icp_qat_fw_la.h b/drivers/common/qat/qat_adf/icp_qat_fw_la.h
index c33bc3fe..c33bc3fe 100644
--- a/drivers/crypto/qat/qat_adf/icp_qat_fw_la.h
+++ b/drivers/common/qat/qat_adf/icp_qat_fw_la.h
diff --git a/drivers/crypto/qat/qat_adf/icp_qat_hw.h b/drivers/common/qat/qat_adf/icp_qat_hw.h
index 56e3cf79..e7961dba 100644
--- a/drivers/crypto/qat/qat_adf/icp_qat_hw.h
+++ b/drivers/common/qat/qat_adf/icp_qat_hw.h
@@ -72,19 +72,44 @@ struct icp_qat_hw_auth_config {
 #define QAT_AUTH_ALGO_MASK 0xF
 #define QAT_AUTH_CMP_BITPOS 8
 #define QAT_AUTH_CMP_MASK 0x7F
-#define QAT_AUTH_SHA3_PADDING_BITPOS 16
-#define QAT_AUTH_SHA3_PADDING_MASK 0x1
+#define QAT_AUTH_SHA3_PADDING_DISABLE_BITPOS 16
+#define QAT_AUTH_SHA3_PADDING_DISABLE_MASK 0x1
+#define QAT_AUTH_SHA3_PADDING_OVERRIDE_BITPOS 17
+#define QAT_AUTH_SHA3_PADDING_OVERRIDE_MASK 0x1
 #define QAT_AUTH_ALGO_SHA3_BITPOS 22
 #define QAT_AUTH_ALGO_SHA3_MASK 0x3
-#define ICP_QAT_HW_AUTH_CONFIG_BUILD(mode, algo, cmp_len) \
-	(((mode & QAT_AUTH_MODE_MASK) << QAT_AUTH_MODE_BITPOS) | \
-	((algo & QAT_AUTH_ALGO_MASK) << QAT_AUTH_ALGO_BITPOS) | \
-	(((algo >> 4) & QAT_AUTH_ALGO_SHA3_MASK) << \
-	 QAT_AUTH_ALGO_SHA3_BITPOS) | \
-	 (((((algo == ICP_QAT_HW_AUTH_ALGO_SHA3_256) || \
-	(algo == ICP_QAT_HW_AUTH_ALGO_SHA3_512)) ? 1 : 0) \
-	& QAT_AUTH_SHA3_PADDING_MASK) << QAT_AUTH_SHA3_PADDING_BITPOS) | \
-	((cmp_len & QAT_AUTH_CMP_MASK) << QAT_AUTH_CMP_BITPOS))
+#define QAT_AUTH_SHA3_PROG_PADDING_POSTFIX_BITPOS 16
+#define QAT_AUTH_SHA3_PROG_PADDING_POSTFIX_MASK 0xF
+#define QAT_AUTH_SHA3_PROG_PADDING_PREFIX_BITPOS 24
+#define QAT_AUTH_SHA3_PROG_PADDING_PREFIX_MASK 0xFF
+#define QAT_AUTH_SHA3_HW_PADDING_ENABLE 0
+#define QAT_AUTH_SHA3_HW_PADDING_DISABLE 1
+#define QAT_AUTH_SHA3_PADDING_DISABLE_USE_DEFAULT 0
+#define QAT_AUTH_SHA3_PADDING_OVERRIDE_USE_DEFAULT 0
+#define QAT_AUTH_SHA3_PADDING_OVERRIDE_PROGRAMMABLE 1
+#define QAT_AUTH_SHA3_PROG_PADDING_POSTFIX_RESERVED 0
+#define QAT_AUTH_SHA3_PROG_PADDING_PREFIX_RESERVED 0
+
+#define ICP_QAT_HW_AUTH_CONFIG_BUILD(mode, algo, cmp_len)                      \
+	((((mode) & QAT_AUTH_MODE_MASK) << QAT_AUTH_MODE_BITPOS) |             \
+	 (((algo) & QAT_AUTH_ALGO_MASK) << QAT_AUTH_ALGO_BITPOS) |             \
+	 (((algo >> 4) & QAT_AUTH_ALGO_SHA3_MASK)                              \
+			<< QAT_AUTH_ALGO_SHA3_BITPOS) |                        \
+	 (((QAT_AUTH_SHA3_PADDING_DISABLE_USE_DEFAULT) &                       \
+			QAT_AUTH_SHA3_PADDING_DISABLE_MASK)                    \
+			<< QAT_AUTH_SHA3_PADDING_DISABLE_BITPOS) |             \
+	 (((QAT_AUTH_SHA3_PADDING_OVERRIDE_USE_DEFAULT) &                      \
+			QAT_AUTH_SHA3_PADDING_OVERRIDE_MASK)                   \
+			<< QAT_AUTH_SHA3_PADDING_OVERRIDE_BITPOS) |            \
+	 (((cmp_len) & QAT_AUTH_CMP_MASK) << QAT_AUTH_CMP_BITPOS))
+
+#define ICP_QAT_HW_AUTH_CONFIG_BUILD_UPPER                                     \
+	((((QAT_AUTH_SHA3_PROG_PADDING_POSTFIX_RESERVED) &                     \
+		QAT_AUTH_SHA3_PROG_PADDING_POSTFIX_MASK)                       \
+		<< QAT_AUTH_SHA3_PROG_PADDING_POSTFIX_BITPOS) |                \
+	 (((QAT_AUTH_SHA3_PROG_PADDING_PREFIX_RESERVED) &                      \
+		QAT_AUTH_SHA3_PROG_PADDING_PREFIX_MASK)                        \
+		<< QAT_AUTH_SHA3_PROG_PADDING_PREFIX_BITPOS))
 
 struct icp_qat_hw_auth_counter {
 	uint32_t counter;
@@ -107,13 +132,13 @@ struct icp_qat_hw_auth_setup {
 #define ICP_QAT_HW_MD5_STATE1_SZ 16
 #define ICP_QAT_HW_SHA1_STATE1_SZ 20
 #define ICP_QAT_HW_SHA224_STATE1_SZ 32
+#define ICP_QAT_HW_SHA3_224_STATE1_SZ 28
 #define ICP_QAT_HW_SHA256_STATE1_SZ 32
 #define ICP_QAT_HW_SHA3_256_STATE1_SZ 32
 #define ICP_QAT_HW_SHA384_STATE1_SZ 64
+#define ICP_QAT_HW_SHA3_384_STATE1_SZ 48
 #define ICP_QAT_HW_SHA512_STATE1_SZ 64
 #define ICP_QAT_HW_SHA3_512_STATE1_SZ 64
-#define ICP_QAT_HW_SHA3_224_STATE1_SZ 28
-#define ICP_QAT_HW_SHA3_384_STATE1_SZ 48
 #define ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ 16
 #define ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ 16
 #define ICP_QAT_HW_AES_F9_STATE1_SZ 32
@@ -121,17 +146,18 @@ struct icp_qat_hw_auth_setup {
 #define ICP_QAT_HW_GALOIS_128_STATE1_SZ 16
 #define ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ 8
 #define ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ 8
+
 #define ICP_QAT_HW_NULL_STATE2_SZ 32
 #define ICP_QAT_HW_MD5_STATE2_SZ 16
 #define ICP_QAT_HW_SHA1_STATE2_SZ 20
 #define ICP_QAT_HW_SHA224_STATE2_SZ 32
+#define ICP_QAT_HW_SHA3_224_STATE2_SZ 0
 #define ICP_QAT_HW_SHA256_STATE2_SZ 32
 #define ICP_QAT_HW_SHA3_256_STATE2_SZ 0
 #define ICP_QAT_HW_SHA384_STATE2_SZ 64
+#define ICP_QAT_HW_SHA3_384_STATE2_SZ 0
 #define ICP_QAT_HW_SHA512_STATE2_SZ 64
 #define ICP_QAT_HW_SHA3_512_STATE2_SZ 0
-#define ICP_QAT_HW_SHA3_224_STATE2_SZ 0
-#define ICP_QAT_HW_SHA3_384_STATE2_SZ 0
 #define ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ 48
 #define ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ 16
 #define ICP_QAT_HW_AES_CBC_MAC_KEY_SZ 16
@@ -154,6 +180,12 @@ struct icp_qat_hw_auth_sha512 {
 	uint8_t state2[ICP_QAT_HW_SHA512_STATE2_SZ];
 };
 
+struct icp_qat_hw_auth_sha3_512 {
+	struct icp_qat_hw_auth_setup inner_setup;
+	uint8_t state1[ICP_QAT_HW_SHA3_512_STATE1_SZ];
+	struct icp_qat_hw_auth_setup outer_setup;
+};
+
 struct icp_qat_hw_auth_algo_blk {
 	struct icp_qat_hw_auth_sha512 sha;
 };
@@ -283,4 +315,72 @@ struct icp_qat_hw_cipher_algo_blk {
 	uint8_t key[ICP_QAT_HW_CIPHER_MAX_KEY_SZ];
 } __rte_cache_aligned;
 
+/* ========================================================================= */
+/*                COMPRESSION SLICE                                          */
+/* ========================================================================= */
+
+enum icp_qat_hw_compression_direction {
+	ICP_QAT_HW_COMPRESSION_DIR_COMPRESS = 0,
+	ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS = 1,
+	ICP_QAT_HW_COMPRESSION_DIR_DELIMITER = 2
+};
+
+enum icp_qat_hw_compression_delayed_match {
+	ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED = 0,
+	ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED = 1,
+	ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DELIMITER = 2
+};
+
+enum icp_qat_hw_compression_algo {
+	ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE = 0,
+	ICP_QAT_HW_COMPRESSION_ALGO_LZS = 1,
+	ICP_QAT_HW_COMPRESSION_ALGO_DELIMITER = 2
+};
+
+
+enum icp_qat_hw_compression_depth {
+	ICP_QAT_HW_COMPRESSION_DEPTH_1 = 0,
+	ICP_QAT_HW_COMPRESSION_DEPTH_4 = 1,
+	ICP_QAT_HW_COMPRESSION_DEPTH_8 = 2,
+	ICP_QAT_HW_COMPRESSION_DEPTH_16 = 3,
+	ICP_QAT_HW_COMPRESSION_DEPTH_DELIMITER = 4
+};
+
+enum icp_qat_hw_compression_file_type {
+	ICP_QAT_HW_COMPRESSION_FILE_TYPE_0 = 0,
+	ICP_QAT_HW_COMPRESSION_FILE_TYPE_1 = 1,
+	ICP_QAT_HW_COMPRESSION_FILE_TYPE_2 = 2,
+	ICP_QAT_HW_COMPRESSION_FILE_TYPE_3 = 3,
+	ICP_QAT_HW_COMPRESSION_FILE_TYPE_4 = 4,
+	ICP_QAT_HW_COMPRESSION_FILE_TYPE_DELIMITER = 5
+};
+
+struct icp_qat_hw_compression_config {
+	uint32_t val;
+	uint32_t reserved;
+};
+
+#define QAT_COMPRESSION_DIR_BITPOS 4
+#define QAT_COMPRESSION_DIR_MASK 0x7
+#define QAT_COMPRESSION_DELAYED_MATCH_BITPOS 16
+#define QAT_COMPRESSION_DELAYED_MATCH_MASK 0x1
+#define QAT_COMPRESSION_ALGO_BITPOS 31
+#define QAT_COMPRESSION_ALGO_MASK 0x1
+#define QAT_COMPRESSION_DEPTH_BITPOS 28
+#define QAT_COMPRESSION_DEPTH_MASK 0x7
+#define QAT_COMPRESSION_FILE_TYPE_BITPOS 24
+#define QAT_COMPRESSION_FILE_TYPE_MASK 0xF
+
+#define ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(                                   \
+	dir, delayed, algo, depth, filetype)                                   \
+	((((dir) & QAT_COMPRESSION_DIR_MASK) << QAT_COMPRESSION_DIR_BITPOS) |  \
+	 (((delayed) & QAT_COMPRESSION_DELAYED_MATCH_MASK)                     \
+	  << QAT_COMPRESSION_DELAYED_MATCH_BITPOS) |                           \
+	 (((algo) & QAT_COMPRESSION_ALGO_MASK)                                 \
+	  << QAT_COMPRESSION_ALGO_BITPOS) |                                    \
+	 (((depth) & QAT_COMPRESSION_DEPTH_MASK)                               \
+	  << QAT_COMPRESSION_DEPTH_BITPOS) |                                   \
+	 (((filetype) & QAT_COMPRESSION_FILE_TYPE_MASK)                        \
+	  << QAT_COMPRESSION_FILE_TYPE_BITPOS))
+
 #endif
diff --git a/drivers/common/qat/qat_common.c b/drivers/common/qat/qat_common.c
new file mode 100644
index 00000000..47538669
--- /dev/null
+++ b/drivers/common/qat/qat_common.c
@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include "qat_common.h"
+#include "qat_device.h"
+#include "qat_logs.h"
+
+int
+qat_sgl_fill_array(struct rte_mbuf *buf, int64_t offset,
+		void *list_in, uint32_t data_len,
+		const uint16_t max_segs)
+{
+	int res = -EINVAL;
+	uint32_t buf_len, nr;
+	struct qat_sgl *list = (struct qat_sgl *)list_in;
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+	uint8_t *virt_addr[max_segs];
+#endif
+
+	for (nr = buf_len = 0; buf &&  nr < max_segs; buf = buf->next)  {
+		if (offset >= rte_pktmbuf_data_len(buf)) {
+			offset -= rte_pktmbuf_data_len(buf);
+			continue;
+		}
+
+		list->buffers[nr].len = rte_pktmbuf_data_len(buf) - offset;
+		list->buffers[nr].resrvd = 0;
+		list->buffers[nr].addr = rte_pktmbuf_iova_offset(buf, offset);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		virt_addr[nr] = rte_pktmbuf_mtod_offset(buf, uint8_t*, offset);
+#endif
+		offset = 0;
+		buf_len += list->buffers[nr].len;
+
+		if (buf_len >= data_len) {
+			list->buffers[nr].len -= buf_len - data_len;
+			res = 0;
+			break;
+		}
+		++nr;
+	}
+
+	if (unlikely(res != 0)) {
+		if (nr == max_segs) {
+			QAT_DP_LOG(ERR, "Exceeded max segments in QAT SGL (%u)",
+				   max_segs);
+		} else {
+			QAT_DP_LOG(ERR, "Mbuf chain is too short");
+		}
+	} else {
+
+		list->num_bufs = ++nr;
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_LOG(INFO, "SGL with %d buffers:", list->num_bufs);
+		for (nr = 0; nr < list->num_bufs; nr++) {
+			QAT_DP_LOG(INFO,
+				"QAT SGL buf %d, len = %d, iova = 0x%012"PRIx64,
+				 nr, list->buffers[nr].len,
+				 list->buffers[nr].addr);
+			QAT_DP_HEXDUMP_LOG(DEBUG, "qat SGL",
+					   virt_addr[nr],
+					   list->buffers[nr].len);
+		}
+#endif
+	}
+
+	return res;
+}
+
+void qat_stats_get(struct qat_pci_device *dev,
+		struct qat_common_stats *stats,
+		enum qat_service_type service)
+{
+	int i;
+	struct qat_qp **qp;
+
+	if (stats == NULL || dev == NULL || service >= QAT_SERVICE_INVALID) {
+		QAT_LOG(ERR, "invalid param: stats %p, dev %p, service %d",
+				stats, dev, service);
+		return;
+	}
+
+	qp = dev->qps_in_use[service];
+	for (i = 0; i < ADF_MAX_QPS_ON_ANY_SERVICE; i++) {
+		if (qp[i] == NULL) {
+			QAT_LOG(DEBUG, "Service %d Uninitialised qp %d",
+					service, i);
+			continue;
+		}
+
+		stats->enqueued_count += qp[i]->stats.enqueued_count;
+		stats->dequeued_count += qp[i]->stats.dequeued_count;
+		stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
+		stats->dequeue_err_count += qp[i]->stats.dequeue_err_count;
+	}
+}
+
+void qat_stats_reset(struct qat_pci_device *dev,
+		enum qat_service_type service)
+{
+	int i;
+	struct qat_qp **qp;
+
+	if (dev == NULL || service >= QAT_SERVICE_INVALID) {
+		QAT_LOG(ERR, "invalid param: dev %p, service %d",
+				dev, service);
+		return;
+	}
+
+	qp = dev->qps_in_use[service];
+	for (i = 0; i < ADF_MAX_QPS_ON_ANY_SERVICE; i++) {
+		if (qp[i] == NULL) {
+			QAT_LOG(DEBUG, "Service %d Uninitialised qp %d",
+					service, i);
+			continue;
+		}
+		memset(&(qp[i]->stats), 0, sizeof(qp[i]->stats));
+	}
+
+	QAT_LOG(DEBUG, "QAT: %d stats cleared", service);
+}
diff --git a/drivers/common/qat/qat_common.h b/drivers/common/qat/qat_common.h
new file mode 100644
index 00000000..d4bef539
--- /dev/null
+++ b/drivers/common/qat/qat_common.h
@@ -0,0 +1,79 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+#ifndef _QAT_COMMON_H_
+#define _QAT_COMMON_H_
+
+#include <stdint.h>
+
+#include <rte_mbuf.h>
+
+/**< Intel(R) QAT device name for PCI registration */
+#define QAT_PCI_NAME	qat
+#define QAT_64_BTYE_ALIGN_MASK (~0x3f)
+
+/* Intel(R) QuickAssist Technology device generation is enumerated
+ * from one according to the generation of the device
+ */
+enum qat_device_gen {
+	QAT_GEN1 = 1,
+	QAT_GEN2
+};
+
+enum qat_service_type {
+	QAT_SERVICE_ASYMMETRIC = 0,
+	QAT_SERVICE_SYMMETRIC,
+	QAT_SERVICE_COMPRESSION,
+	QAT_SERVICE_INVALID
+};
+
+#define QAT_MAX_SERVICES		(QAT_SERVICE_INVALID)
+
+/**< Common struct for scatter-gather list operations */
+struct qat_flat_buf {
+	uint32_t len;
+	uint32_t resrvd;
+	uint64_t addr;
+} __rte_packed;
+
+#define qat_sgl_hdr  struct { \
+	uint64_t resrvd; \
+	uint32_t num_bufs; \
+	uint32_t num_mapped_bufs; \
+}
+
+__extension__
+struct qat_sgl {
+	qat_sgl_hdr;
+	/* flexible array of flat buffers*/
+	struct qat_flat_buf buffers[0];
+} __rte_packed __rte_cache_aligned;
+
+/** Common, i.e. not service-specific, statistics */
+struct qat_common_stats {
+	uint64_t enqueued_count;
+	/**< Count of all operations enqueued */
+	uint64_t dequeued_count;
+	/**< Count of all operations dequeued */
+
+	uint64_t enqueue_err_count;
+	/**< Total error count on operations enqueued */
+	uint64_t dequeue_err_count;
+	/**< Total error count on operations dequeued */
+};
+
+struct qat_pci_device;
+
+int
+qat_sgl_fill_array(struct rte_mbuf *buf, int64_t offset,
+		void *list_in, uint32_t data_len,
+		const uint16_t max_segs);
+void
+qat_stats_get(struct qat_pci_device *dev,
+		struct qat_common_stats *stats,
+		enum qat_service_type service);
+void
+qat_stats_reset(struct qat_pci_device *dev,
+		enum qat_service_type service);
+
+#endif /* _QAT_COMMON_H_ */
diff --git a/drivers/common/qat/qat_device.c b/drivers/common/qat/qat_device.c
new file mode 100644
index 00000000..f32d7235
--- /dev/null
+++ b/drivers/common/qat/qat_device.c
@@ -0,0 +1,279 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <rte_string_fns.h>
+
+#include "qat_device.h"
+#include "adf_transport_access_macros.h"
+#include "qat_sym_pmd.h"
+
+/* Hardware device information per generation */
+__extension__
+struct qat_gen_hw_data qat_gen_config[] =  {
+	[QAT_GEN1] = {
+		.dev_gen = QAT_GEN1,
+		.qp_hw_data = qat_gen1_qps,
+	},
+	[QAT_GEN2] = {
+		.dev_gen = QAT_GEN2,
+		.qp_hw_data = qat_gen1_qps,
+		/* gen2 has same ring layout as gen1 */
+	},
+};
+
+
+static struct qat_pci_device qat_pci_devices[RTE_PMD_QAT_MAX_PCI_DEVICES];
+static int qat_nb_pci_devices;
+
+/*
+ * The set of PCI devices this driver supports
+ */
+
+static const struct rte_pci_id pci_id_qat_map[] = {
+		{
+			RTE_PCI_DEVICE(0x8086, 0x0443),
+		},
+		{
+			RTE_PCI_DEVICE(0x8086, 0x37c9),
+		},
+		{
+			RTE_PCI_DEVICE(0x8086, 0x19e3),
+		},
+		{
+			RTE_PCI_DEVICE(0x8086, 0x6f55),
+		},
+		{.device_id = 0},
+};
+
+
+static struct qat_pci_device *
+qat_pci_get_dev(uint8_t dev_id)
+{
+	return &qat_pci_devices[dev_id];
+}
+
+static struct qat_pci_device *
+qat_pci_get_named_dev(const char *name)
+{
+	struct qat_pci_device *dev;
+	unsigned int i;
+
+	if (name == NULL)
+		return NULL;
+
+	for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) {
+		dev = &qat_pci_devices[i];
+
+		if ((dev->attached == QAT_ATTACHED) &&
+				(strcmp(dev->name, name) == 0))
+			return dev;
+	}
+
+	return NULL;
+}
+
+static uint8_t
+qat_pci_find_free_device_index(void)
+{
+	uint8_t dev_id;
+
+	for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES; dev_id++) {
+		if (qat_pci_devices[dev_id].attached == QAT_DETACHED)
+			break;
+	}
+	return dev_id;
+}
+
+struct qat_pci_device *
+qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev)
+{
+	char name[QAT_DEV_NAME_MAX_LEN];
+
+	rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
+
+	return qat_pci_get_named_dev(name);
+}
+
+struct qat_pci_device *
+qat_pci_device_allocate(struct rte_pci_device *pci_dev)
+{
+	struct qat_pci_device *qat_dev;
+	uint8_t qat_dev_id;
+	char name[QAT_DEV_NAME_MAX_LEN];
+
+	rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
+	snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
+	if (qat_pci_get_named_dev(name) != NULL) {
+		QAT_LOG(ERR, "QAT device with name %s already allocated!",
+				name);
+		return NULL;
+	}
+
+	qat_dev_id = qat_pci_find_free_device_index();
+	if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) {
+		QAT_LOG(ERR, "Reached maximum number of QAT devices");
+		return NULL;
+	}
+
+	qat_dev = qat_pci_get_dev(qat_dev_id);
+	memset(qat_dev, 0, sizeof(*qat_dev));
+	strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN);
+	qat_dev->qat_dev_id = qat_dev_id;
+	qat_dev->pci_dev = pci_dev;
+	switch (qat_dev->pci_dev->id.device_id) {
+	case 0x0443:
+		qat_dev->qat_dev_gen = QAT_GEN1;
+		break;
+	case 0x37c9:
+	case 0x19e3:
+	case 0x6f55:
+		qat_dev->qat_dev_gen = QAT_GEN2;
+		break;
+	default:
+		QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
+		return NULL;
+	}
+
+	rte_spinlock_init(&qat_dev->arb_csr_lock);
+
+	qat_dev->attached = QAT_ATTACHED;
+
+	qat_nb_pci_devices++;
+
+	QAT_LOG(DEBUG, "QAT device %d allocated, name %s, total QATs %d",
+			qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
+
+	return qat_dev;
+}
+
+int
+qat_pci_device_release(struct rte_pci_device *pci_dev)
+{
+	struct qat_pci_device *qat_dev;
+	char name[QAT_DEV_NAME_MAX_LEN];
+
+	if (pci_dev == NULL)
+		return -EINVAL;
+
+	rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
+	snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
+	qat_dev = qat_pci_get_named_dev(name);
+	if (qat_dev != NULL) {
+
+		/* Check that there are no service devs still on pci device */
+		if (qat_dev->sym_dev != NULL)
+			return -EBUSY;
+
+		qat_dev->attached = QAT_DETACHED;
+		qat_nb_pci_devices--;
+	}
+	QAT_LOG(DEBUG, "QAT device %s released, total QATs %d",
+				name, qat_nb_pci_devices);
+	return 0;
+}
+
+static int
+qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev,
+		struct rte_pci_device *pci_dev)
+{
+	qat_sym_dev_destroy(qat_pci_dev);
+	qat_comp_dev_destroy(qat_pci_dev);
+	qat_asym_dev_destroy(qat_pci_dev);
+	return qat_pci_device_release(pci_dev);
+}
+
+static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+		struct rte_pci_device *pci_dev)
+{
+	int ret = 0;
+	struct qat_pci_device *qat_pci_dev;
+
+	QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
+			pci_dev->addr.bus,
+			pci_dev->addr.devid,
+			pci_dev->addr.function);
+
+	qat_pci_dev = qat_pci_device_allocate(pci_dev);
+	if (qat_pci_dev == NULL)
+		return -ENODEV;
+
+	ret = qat_sym_dev_create(qat_pci_dev);
+	if (ret != 0)
+		goto error_out;
+
+	ret = qat_comp_dev_create(qat_pci_dev);
+	if (ret != 0)
+		goto error_out;
+
+	ret = qat_asym_dev_create(qat_pci_dev);
+	if (ret != 0)
+		goto error_out;
+
+	return 0;
+
+error_out:
+	qat_pci_dev_destroy(qat_pci_dev, pci_dev);
+	return ret;
+
+}
+
+static int qat_pci_remove(struct rte_pci_device *pci_dev)
+{
+	struct qat_pci_device *qat_pci_dev;
+
+	if (pci_dev == NULL)
+		return -EINVAL;
+
+	qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev);
+	if (qat_pci_dev == NULL)
+		return 0;
+
+	return qat_pci_dev_destroy(qat_pci_dev, pci_dev);
+}
+
+static struct rte_pci_driver rte_qat_pmd = {
+	.id_table = pci_id_qat_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = qat_pci_probe,
+	.remove = qat_pci_remove
+};
+
+__attribute__((weak)) int
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
+{
+	return 0;
+}
+
+__attribute__((weak)) int
+qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
+{
+	return 0;
+}
+
+__attribute__((weak)) int
+qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
+{
+	return 0;
+}
+
+__attribute__((weak)) int
+qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
+{
+	return 0;
+}
+
+__attribute__((weak)) int
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
+{
+	return 0;
+}
+
+__attribute__((weak)) int
+qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
+{
+	return 0;
+}
+
+RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map);
diff --git a/drivers/common/qat/qat_device.h b/drivers/common/qat/qat_device.h
new file mode 100644
index 00000000..9599fc59
--- /dev/null
+++ b/drivers/common/qat/qat_device.h
@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+#ifndef _QAT_DEVICE_H_
+#define _QAT_DEVICE_H_
+
+#include <rte_bus_pci.h>
+
+#include "qat_common.h"
+#include "qat_logs.h"
+#include "adf_transport_access_macros.h"
+#include "qat_qp.h"
+
+#define QAT_DETACHED  (0)
+#define QAT_ATTACHED  (1)
+
+#define QAT_DEV_NAME_MAX_LEN	64
+
+/*
+ * This struct holds all the data about a QAT pci device
+ * including data about all services it supports.
+ * It contains
+ *  - hw_data
+ *  - config data
+ *  - runtime data
+ */
+struct qat_sym_dev_private;
+struct qat_comp_dev_private;
+
+struct qat_pci_device {
+
+	/* Data used by all services */
+	char name[QAT_DEV_NAME_MAX_LEN];
+	/**< Name of qat pci device */
+	uint8_t qat_dev_id;
+	/**< Device instance for this qat pci device */
+	struct rte_pci_device *pci_dev;
+	/**< PCI information. */
+	enum qat_device_gen qat_dev_gen;
+	/**< QAT device generation */
+	rte_spinlock_t arb_csr_lock;
+	/**< lock to protect accesses to the arbiter CSR */
+	__extension__
+	uint8_t attached : 1;
+	/**< Flag indicating the device is attached */
+
+	struct qat_qp *qps_in_use[QAT_MAX_SERVICES][ADF_MAX_QPS_ON_ANY_SERVICE];
+	/**< links to qps set up for each service, index same as on API */
+
+	/* Data relating to symmetric crypto service */
+	struct qat_sym_dev_private *sym_dev;
+	/**< link back to cryptodev private data */
+	struct rte_device sym_rte_dev;
+	/**< This represents the crypto subset of this pci device.
+	 * Register with this rather than with the one in
+	 * pci_dev so that its driver can have a crypto-specific name
+	 */
+
+	/* Data relating to compression service */
+	struct qat_comp_dev_private *comp_dev;
+	/**< link back to compressdev private data */
+
+	/* Data relating to asymmetric crypto service */
+
+};
+
+struct qat_gen_hw_data {
+	enum qat_device_gen dev_gen;
+	const struct qat_qp_hw_data (*qp_hw_data)[ADF_MAX_QPS_ON_ANY_SERVICE];
+};
+
+extern struct qat_gen_hw_data qat_gen_config[];
+
+struct qat_pci_device *
+qat_pci_device_allocate(struct rte_pci_device *pci_dev);
+
+int
+qat_pci_device_release(struct rte_pci_device *pci_dev);
+
+struct qat_pci_device *
+qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev);
+
+/* declaration needed for weak functions */
+int
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused);
+
+int
+qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused);
+
+int
+qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);
+
+int
+qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);
+
+int
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused);
+
+int
+qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);
+
+#endif /* _QAT_DEVICE_H_ */
diff --git a/drivers/common/qat/qat_logs.c b/drivers/common/qat/qat_logs.c
new file mode 100644
index 00000000..7a861709
--- /dev/null
+++ b/drivers/common/qat/qat_logs.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <rte_log.h>
+#include <rte_hexdump.h>
+
+#include "qat_logs.h"
+
+int qat_gen_logtype;
+int qat_dp_logtype;
+
+int
+qat_hexdump_log(uint32_t level, uint32_t logtype, const char *title,
+		const void *buf, unsigned int len)
+{
+	if (level > rte_log_get_global_level())
+		return 0;
+	if (level > (uint32_t)(rte_log_get_level(logtype)))
+		return 0;
+
+	rte_hexdump(rte_logs.file == NULL ? stderr : rte_logs.file,
+				title, buf, len);
+	return 0;
+}
+
+RTE_INIT(qat_pci_init_log)
+{
+	/* Non-data-path logging for pci device and all services */
+	qat_gen_logtype = rte_log_register("pmd.qat_general");
+	if (qat_gen_logtype >= 0)
+		rte_log_set_level(qat_gen_logtype, RTE_LOG_NOTICE);
+
+	/* data-path logging for all services */
+	qat_dp_logtype = rte_log_register("pmd.qat_dp");
+	if (qat_dp_logtype >= 0)
+		rte_log_set_level(qat_dp_logtype, RTE_LOG_NOTICE);
+}
diff --git a/drivers/common/qat/qat_logs.h b/drivers/common/qat/qat_logs.h
new file mode 100644
index 00000000..4baea12c
--- /dev/null
+++ b/drivers/common/qat/qat_logs.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2018 Intel Corporation
+ */
+
+#ifndef _QAT_LOGS_H_
+#define _QAT_LOGS_H_
+
+extern int qat_gen_logtype;
+extern int qat_dp_logtype;
+
+#define QAT_LOG(level, fmt, args...)			\
+	rte_log(RTE_LOG_ ## level, qat_gen_logtype,		\
+			"%s(): " fmt "\n", __func__, ## args)
+
+#define QAT_DP_LOG(level, fmt, args...)			\
+	rte_log(RTE_LOG_ ## level, qat_dp_logtype,		\
+			"%s(): " fmt "\n", __func__, ## args)
+
+#define QAT_DP_HEXDUMP_LOG(level, title, buf, len)		\
+	qat_hexdump_log(RTE_LOG_ ## level, qat_dp_logtype, title, buf, len)
+
+/**
+ * qat_hexdump_log - Dump out memory in a special hex dump format.
+ *
+ * Dump out the message buffer in a special hex dump output format with
+ * characters printed for each line of 16 hex values. The message will be sent
+ * to the stream defined by rte_logs.file or to stderr in case of rte_logs.file
+ * is undefined.
+ */
+int
+qat_hexdump_log(uint32_t level, uint32_t logtype, const char *title,
+		const void *buf, unsigned int len);
+
+#endif /* _QAT_LOGS_H_ */
diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c
new file mode 100644
index 00000000..7ca7a45e
--- /dev/null
+++ b/drivers/common/qat/qat_qp.c
@@ -0,0 +1,642 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2018 Intel Corporation
+ */
+
+#include <rte_common.h>
+#include <rte_dev.h>
+#include <rte_malloc.h>
+#include <rte_memzone.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_atomic.h>
+#include <rte_prefetch.h>
+
+#include "qat_logs.h"
+#include "qat_device.h"
+#include "qat_qp.h"
+#include "qat_sym.h"
+#include "qat_comp.h"
+#include "adf_transport_access_macros.h"
+
+
+#define ADF_MAX_DESC				4096
+#define ADF_MIN_DESC				128
+
+#define ADF_ARB_REG_SLOT			0x1000
+#define ADF_ARB_RINGSRVARBEN_OFFSET		0x19C
+
+#define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
+	ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
+	(ADF_ARB_REG_SLOT * index), value)
+
+__extension__
+const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
+					 [ADF_MAX_QPS_ON_ANY_SERVICE] = {
+	/* queue pairs which provide an asymmetric crypto service */
+	[QAT_SERVICE_ASYMMETRIC] = {
+		{
+			.service_type = QAT_SERVICE_ASYMMETRIC,
+			.hw_bundle_num = 0,
+			.tx_ring_num = 0,
+			.rx_ring_num = 8,
+			.tx_msg_size = 64,
+			.rx_msg_size = 32,
+
+		}, {
+			.service_type = QAT_SERVICE_ASYMMETRIC,
+			.hw_bundle_num = 0,
+			.tx_ring_num = 1,
+			.rx_ring_num = 9,
+			.tx_msg_size = 64,
+			.rx_msg_size = 32,
+		}
+	},
+	/* queue pairs which provide a symmetric crypto service */
+	[QAT_SERVICE_SYMMETRIC] = {
+		{
+			.service_type = QAT_SERVICE_SYMMETRIC,
+			.hw_bundle_num = 0,
+			.tx_ring_num = 2,
+			.rx_ring_num = 10,
+			.tx_msg_size = 128,
+			.rx_msg_size = 32,
+		},
+		{
+			.service_type = QAT_SERVICE_SYMMETRIC,
+			.hw_bundle_num = 0,
+			.tx_ring_num = 3,
+			.rx_ring_num = 11,
+			.tx_msg_size = 128,
+			.rx_msg_size = 32,
+		}
+	},
+	/* queue pairs which provide a compression service */
+	[QAT_SERVICE_COMPRESSION] = {
+		{
+			.service_type = QAT_SERVICE_COMPRESSION,
+			.hw_bundle_num = 0,
+			.tx_ring_num = 6,
+			.rx_ring_num = 14,
+			.tx_msg_size = 128,
+			.rx_msg_size = 32,
+		}, {
+			.service_type = QAT_SERVICE_COMPRESSION,
+			.hw_bundle_num = 0,
+			.tx_ring_num = 7,
+			.rx_ring_num = 15,
+			.tx_msg_size = 128,
+			.rx_msg_size = 32,
+		}
+	}
+};
+
+static int qat_qp_check_queue_alignment(uint64_t phys_addr,
+	uint32_t queue_size_bytes);
+static void qat_queue_delete(struct qat_queue *queue);
+static int qat_queue_create(struct qat_pci_device *qat_dev,
+	struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
+static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
+	uint32_t *queue_size_for_csr);
+static void adf_configure_queues(struct qat_qp *queue);
+static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
+	rte_spinlock_t *lock);
+static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
+	rte_spinlock_t *lock);
+
+
+int qat_qps_per_service(const struct qat_qp_hw_data *qp_hw_data,
+		enum qat_service_type service)
+{
+	int i, count;
+
+	for (i = 0, count = 0; i < ADF_MAX_QPS_ON_ANY_SERVICE; i++)
+		if (qp_hw_data[i].service_type == service)
+			count++;
+	return count;
+}
+
+static const struct rte_memzone *
+queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
+			int socket_id)
+{
+	const struct rte_memzone *mz;
+
+	mz = rte_memzone_lookup(queue_name);
+	if (mz != 0) {
+		if (((size_t)queue_size <= mz->len) &&
+				((socket_id == SOCKET_ID_ANY) ||
+					(socket_id == mz->socket_id))) {
+			QAT_LOG(DEBUG, "re-use memzone already "
+					"allocated for %s", queue_name);
+			return mz;
+		}
+
+		QAT_LOG(ERR, "Incompatible memzone already "
+				"allocated %s, size %u, socket %d. "
+				"Requested size %u, socket %u",
+				queue_name, (uint32_t)mz->len,
+				mz->socket_id, queue_size, socket_id);
+		return NULL;
+	}
+
+	QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
+					queue_name, queue_size, socket_id);
+	return rte_memzone_reserve_aligned(queue_name, queue_size,
+		socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
+}
+
+int qat_qp_setup(struct qat_pci_device *qat_dev,
+		struct qat_qp **qp_addr,
+		uint16_t queue_pair_id,
+		struct qat_qp_config *qat_qp_conf)
+
+{
+	struct qat_qp *qp;
+	struct rte_pci_device *pci_dev = qat_dev->pci_dev;
+	char op_cookie_pool_name[RTE_RING_NAMESIZE];
+	uint32_t i;
+
+	QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
+		queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
+
+	if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
+		(qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
+		QAT_LOG(ERR, "Can't create qp for %u descriptors",
+				qat_qp_conf->nb_descriptors);
+		return -EINVAL;
+	}
+
+	if (pci_dev->mem_resource[0].addr == NULL) {
+		QAT_LOG(ERR, "Could not find VF config space "
+				"(UIO driver attached?).");
+		return -EINVAL;
+	}
+
+	/* Allocate the queue pair data structure. */
+	qp = rte_zmalloc("qat PMD qp metadata",
+			sizeof(*qp), RTE_CACHE_LINE_SIZE);
+	if (qp == NULL) {
+		QAT_LOG(ERR, "Failed to alloc mem for qp struct");
+		return -ENOMEM;
+	}
+	qp->nb_descriptors = qat_qp_conf->nb_descriptors;
+	qp->op_cookies = rte_zmalloc("qat PMD op cookie pointer",
+			qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
+			RTE_CACHE_LINE_SIZE);
+	if (qp->op_cookies == NULL) {
+		QAT_LOG(ERR, "Failed to alloc mem for cookie");
+		rte_free(qp);
+		return -ENOMEM;
+	}
+
+	qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
+	qp->inflights16 = 0;
+
+	if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
+					ADF_RING_DIR_TX) != 0) {
+		QAT_LOG(ERR, "Tx queue create failed "
+				"queue_pair_id=%u", queue_pair_id);
+		goto create_err;
+	}
+
+	if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
+					ADF_RING_DIR_RX) != 0) {
+		QAT_LOG(ERR, "Rx queue create failed "
+				"queue_pair_id=%hu", queue_pair_id);
+		qat_queue_delete(&(qp->tx_q));
+		goto create_err;
+	}
+
+	adf_configure_queues(qp);
+	adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr,
+					&qat_dev->arb_csr_lock);
+
+	snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
+					"%s%d_cookies_%s_qp%hu",
+		pci_dev->driver->driver.name, qat_dev->qat_dev_id,
+		qat_qp_conf->service_str, queue_pair_id);
+
+	QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
+	qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
+	if (qp->op_cookie_pool == NULL)
+		qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
+				qp->nb_descriptors,
+				qat_qp_conf->cookie_size, 64, 0,
+				NULL, NULL, NULL, NULL, qat_qp_conf->socket_id,
+				0);
+	if (!qp->op_cookie_pool) {
+		QAT_LOG(ERR, "QAT PMD Cannot create"
+				" op mempool");
+		goto create_err;
+	}
+
+	for (i = 0; i < qp->nb_descriptors; i++) {
+		if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
+			QAT_LOG(ERR, "QAT PMD Cannot get op_cookie");
+			goto create_err;
+		}
+	}
+
+	qp->qat_dev_gen = qat_dev->qat_dev_gen;
+	qp->build_request = qat_qp_conf->build_request;
+	qp->service_type = qat_qp_conf->hw->service_type;
+	qp->qat_dev = qat_dev;
+
+	QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
+			queue_pair_id, op_cookie_pool_name);
+
+	*qp_addr = qp;
+	return 0;
+
+create_err:
+	if (qp->op_cookie_pool)
+		rte_mempool_free(qp->op_cookie_pool);
+	rte_free(qp->op_cookies);
+	rte_free(qp);
+	return -EFAULT;
+}
+
+int qat_qp_release(struct qat_qp **qp_addr)
+{
+	struct qat_qp *qp = *qp_addr;
+	uint32_t i;
+
+	if (qp == NULL) {
+		QAT_LOG(DEBUG, "qp already freed");
+		return 0;
+	}
+
+	QAT_LOG(DEBUG, "Free qp on qat_pci device %d",
+				qp->qat_dev->qat_dev_id);
+
+	/* Don't free memory if there are still responses to be processed */
+	if (qp->inflights16 == 0) {
+		qat_queue_delete(&(qp->tx_q));
+		qat_queue_delete(&(qp->rx_q));
+	} else {
+		return -EAGAIN;
+	}
+
+	adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr,
+					&qp->qat_dev->arb_csr_lock);
+
+	for (i = 0; i < qp->nb_descriptors; i++)
+		rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
+
+	if (qp->op_cookie_pool)
+		rte_mempool_free(qp->op_cookie_pool);
+
+	rte_free(qp->op_cookies);
+	rte_free(qp);
+	*qp_addr = NULL;
+	return 0;
+}
+
+
+static void qat_queue_delete(struct qat_queue *queue)
+{
+	const struct rte_memzone *mz;
+	int status = 0;
+
+	if (queue == NULL) {
+		QAT_LOG(DEBUG, "Invalid queue");
+		return;
+	}
+	QAT_LOG(DEBUG, "Free ring %d, memzone: %s",
+			queue->hw_queue_number, queue->memz_name);
+
+	mz = rte_memzone_lookup(queue->memz_name);
+	if (mz != NULL)	{
+		/* Write an unused pattern to the queue memory. */
+		memset(queue->base_addr, 0x7F, queue->queue_size);
+		status = rte_memzone_free(mz);
+		if (status != 0)
+			QAT_LOG(ERR, "Error %d on freeing queue %s",
+					status, queue->memz_name);
+	} else {
+		QAT_LOG(DEBUG, "queue %s doesn't exist",
+				queue->memz_name);
+	}
+}
+
+static int
+qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
+		struct qat_qp_config *qp_conf, uint8_t dir)
+{
+	uint64_t queue_base;
+	void *io_addr;
+	const struct rte_memzone *qp_mz;
+	struct rte_pci_device *pci_dev = qat_dev->pci_dev;
+	int ret = 0;
+	uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
+			qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
+	uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
+
+	queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
+	queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
+			qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
+
+	if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
+		QAT_LOG(ERR, "Invalid descriptor size %d", desc_size);
+		return -EINVAL;
+	}
+
+	/*
+	 * Allocate a memzone for the queue - create a unique name.
+	 */
+	snprintf(queue->memz_name, sizeof(queue->memz_name),
+			"%s_%d_%s_%s_%d_%d",
+		pci_dev->driver->driver.name, qat_dev->qat_dev_id,
+		qp_conf->service_str, "qp_mem",
+		queue->hw_bundle_number, queue->hw_queue_number);
+	qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
+			qp_conf->socket_id);
+	if (qp_mz == NULL) {
+		QAT_LOG(ERR, "Failed to allocate ring memzone");
+		return -ENOMEM;
+	}
+
+	queue->base_addr = (char *)qp_mz->addr;
+	queue->base_phys_addr = qp_mz->iova;
+	if (qat_qp_check_queue_alignment(queue->base_phys_addr,
+			queue_size_bytes)) {
+		QAT_LOG(ERR, "Invalid alignment on queue create "
+					" 0x%"PRIx64"\n",
+					queue->base_phys_addr);
+		ret = -EFAULT;
+		goto queue_create_err;
+	}
+
+	if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
+			&(queue->queue_size)) != 0) {
+		QAT_LOG(ERR, "Invalid num inflights");
+		ret = -EINVAL;
+		goto queue_create_err;
+	}
+
+	queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size,
+					ADF_BYTES_TO_MSG_SIZE(desc_size));
+	queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
+
+	if (queue->max_inflights < 2) {
+		QAT_LOG(ERR, "Invalid num inflights");
+		ret = -EINVAL;
+		goto queue_create_err;
+	}
+	queue->head = 0;
+	queue->tail = 0;
+	queue->msg_size = desc_size;
+
+	/*
+	 * Write an unused pattern to the queue memory.
+	 */
+	memset(queue->base_addr, 0x7F, queue_size_bytes);
+
+	queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
+					queue->queue_size);
+
+	io_addr = pci_dev->mem_resource[0].addr;
+
+	WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
+			queue->hw_queue_number, queue_base);
+
+	QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
+		" nb msgs %u, msg_size %u, max_inflights %u modulo mask %u",
+			queue->memz_name,
+			queue->queue_size, queue_size_bytes,
+			qp_conf->nb_descriptors, desc_size,
+			queue->max_inflights, queue->modulo_mask);
+
+	return 0;
+
+queue_create_err:
+	rte_memzone_free(qp_mz);
+	return ret;
+}
+
+static int qat_qp_check_queue_alignment(uint64_t phys_addr,
+					uint32_t queue_size_bytes)
+{
+	if (((queue_size_bytes - 1) & phys_addr) != 0)
+		return -EINVAL;
+	return 0;
+}
+
+static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
+	uint32_t *p_queue_size_for_csr)
+{
+	uint8_t i = ADF_MIN_RING_SIZE;
+
+	for (; i <= ADF_MAX_RING_SIZE; i++)
+		if ((msg_size * msg_num) ==
+				(uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
+			*p_queue_size_for_csr = i;
+			return 0;
+		}
+	QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
+	return -EINVAL;
+}
+
+static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
+					rte_spinlock_t *lock)
+{
+	uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
+					(ADF_ARB_REG_SLOT *
+							txq->hw_bundle_number);
+	uint32_t value;
+
+	rte_spinlock_lock(lock);
+	value = ADF_CSR_RD(base_addr, arb_csr_offset);
+	value |= (0x01 << txq->hw_queue_number);
+	ADF_CSR_WR(base_addr, arb_csr_offset, value);
+	rte_spinlock_unlock(lock);
+}
+
+static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
+					rte_spinlock_t *lock)
+{
+	uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
+					(ADF_ARB_REG_SLOT *
+							txq->hw_bundle_number);
+	uint32_t value;
+
+	rte_spinlock_lock(lock);
+	value = ADF_CSR_RD(base_addr, arb_csr_offset);
+	value &= ~(0x01 << txq->hw_queue_number);
+	ADF_CSR_WR(base_addr, arb_csr_offset, value);
+	rte_spinlock_unlock(lock);
+}
+
+static void adf_configure_queues(struct qat_qp *qp)
+{
+	uint32_t queue_config;
+	struct qat_queue *queue = &qp->tx_q;
+
+	queue_config = BUILD_RING_CONFIG(queue->queue_size);
+
+	WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
+			queue->hw_queue_number, queue_config);
+
+	queue = &qp->rx_q;
+	queue_config =
+			BUILD_RESP_RING_CONFIG(queue->queue_size,
+					ADF_RING_NEAR_WATERMARK_512,
+					ADF_RING_NEAR_WATERMARK_0);
+
+	WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
+			queue->hw_queue_number, queue_config);
+}
+
+static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
+{
+	return data & modulo_mask;
+}
+
+static inline void
+txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
+	WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
+			q->hw_queue_number, q->tail);
+	q->nb_pending_requests = 0;
+	q->csr_tail = q->tail;
+}
+
+static inline
+void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
+{
+	uint32_t old_head, new_head;
+	uint32_t max_head;
+
+	old_head = q->csr_head;
+	new_head = q->head;
+	max_head = qp->nb_descriptors * q->msg_size;
+
+	/* write out free descriptors */
+	void *cur_desc = (uint8_t *)q->base_addr + old_head;
+
+	if (new_head < old_head) {
+		memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
+		memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
+	} else {
+		memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
+	}
+	q->nb_processed_responses = 0;
+	q->csr_head = new_head;
+
+	/* write current head to CSR */
+	WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
+			    q->hw_queue_number, new_head);
+}
+
+uint16_t
+qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+{
+	register struct qat_queue *queue;
+	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
+	register uint32_t nb_ops_sent = 0;
+	register int ret;
+	uint16_t nb_ops_possible = nb_ops;
+	register uint8_t *base_addr;
+	register uint32_t tail;
+	int overflow;
+
+	if (unlikely(nb_ops == 0))
+		return 0;
+
+	/* read params used a lot in main loop into registers */
+	queue = &(tmp_qp->tx_q);
+	base_addr = (uint8_t *)queue->base_addr;
+	tail = queue->tail;
+
+	/* Find how many can actually fit on the ring */
+	tmp_qp->inflights16 += nb_ops;
+	overflow = tmp_qp->inflights16 - queue->max_inflights;
+	if (overflow > 0) {
+		tmp_qp->inflights16 -= overflow;
+		nb_ops_possible = nb_ops - overflow;
+		if (nb_ops_possible == 0)
+			return 0;
+	}
+
+	while (nb_ops_sent != nb_ops_possible) {
+		ret = tmp_qp->build_request(*ops, base_addr + tail,
+				tmp_qp->op_cookies[tail / queue->msg_size],
+				tmp_qp->qat_dev_gen);
+		if (ret != 0) {
+			tmp_qp->stats.enqueue_err_count++;
+			/*
+			 * This message cannot be enqueued,
+			 * decrease number of ops that wasn't sent
+			 */
+			tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent;
+			if (nb_ops_sent == 0)
+				return 0;
+			goto kick_tail;
+		}
+
+		tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
+		ops++;
+		nb_ops_sent++;
+	}
+kick_tail:
+	queue->tail = tail;
+	tmp_qp->stats.enqueued_count += nb_ops_sent;
+	queue->nb_pending_requests += nb_ops_sent;
+	if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH ||
+		    queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) {
+		txq_write_tail(tmp_qp, queue);
+	}
+	return nb_ops_sent;
+}
+
+uint16_t
+qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
+{
+	struct qat_queue *rx_queue, *tx_queue;
+	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
+	uint32_t head;
+	uint32_t resp_counter = 0;
+	uint8_t *resp_msg;
+
+	rx_queue = &(tmp_qp->rx_q);
+	tx_queue = &(tmp_qp->tx_q);
+	head = rx_queue->head;
+	resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
+
+	while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
+			resp_counter != nb_ops) {
+
+		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
+			qat_sym_process_response(ops, resp_msg);
+		else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
+			qat_comp_process_response(ops, resp_msg);
+
+		head = adf_modulo(head + rx_queue->msg_size,
+				  rx_queue->modulo_mask);
+
+		resp_msg = (uint8_t *)rx_queue->base_addr + head;
+		ops++;
+		resp_counter++;
+	}
+	if (resp_counter > 0) {
+		rx_queue->head = head;
+		tmp_qp->stats.dequeued_count += resp_counter;
+		rx_queue->nb_processed_responses += resp_counter;
+		tmp_qp->inflights16 -= resp_counter;
+
+		if (rx_queue->nb_processed_responses >
+						QAT_CSR_HEAD_WRITE_THRESH)
+			rxq_free_desc(tmp_qp, rx_queue);
+	}
+	/* also check if tail needs to be advanced */
+	if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH &&
+		tx_queue->tail != tx_queue->csr_tail) {
+		txq_write_tail(tmp_qp, tx_queue);
+	}
+	return resp_counter;
+}
+
+__attribute__((weak)) int
+qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused)
+{
+	return  0;
+}
diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h
new file mode 100644
index 00000000..69f8a613
--- /dev/null
+++ b/drivers/common/qat/qat_qp.h
@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+#ifndef _QAT_QP_H_
+#define _QAT_QP_H_
+
+#include "qat_common.h"
+#include "adf_transport_access_macros.h"
+
+struct qat_pci_device;
+
+#define QAT_CSR_HEAD_WRITE_THRESH 32U
+/* number of requests to accumulate before writing head CSR */
+#define QAT_CSR_TAIL_WRITE_THRESH 32U
+/* number of requests to accumulate before writing tail CSR */
+#define QAT_CSR_TAIL_FORCE_WRITE_THRESH 256U
+/* number of inflights below which no tail write coalescing should occur */
+
+typedef int (*build_request_t)(void *op,
+		uint8_t *req, void *op_cookie,
+		enum qat_device_gen qat_dev_gen);
+/**< Build a request from an op. */
+
+/**
+ * Structure with data needed for creation of queue pair.
+ */
+struct qat_qp_hw_data {
+	enum qat_service_type service_type;
+	uint8_t hw_bundle_num;
+	uint8_t tx_ring_num;
+	uint8_t rx_ring_num;
+	uint16_t tx_msg_size;
+	uint16_t rx_msg_size;
+};
+/**
+ * Structure with data needed for creation of queue pair.
+ */
+struct qat_qp_config {
+	const struct qat_qp_hw_data *hw;
+	uint32_t nb_descriptors;
+	uint32_t cookie_size;
+	int socket_id;
+	build_request_t build_request;
+	const char *service_str;
+};
+
+/**
+ * Structure associated with each queue.
+ */
+struct qat_queue {
+	char		memz_name[RTE_MEMZONE_NAMESIZE];
+	void		*base_addr;		/* Base address */
+	rte_iova_t	base_phys_addr;		/* Queue physical address */
+	uint32_t	head;			/* Shadow copy of the head */
+	uint32_t	tail;			/* Shadow copy of the tail */
+	uint32_t	modulo_mask;
+	uint32_t	msg_size;
+	uint16_t	max_inflights;
+	uint32_t	queue_size;
+	uint8_t		hw_bundle_number;
+	uint8_t		hw_queue_number;
+	/* HW queue aka ring offset on bundle */
+	uint32_t	csr_head;		/* last written head value */
+	uint32_t	csr_tail;		/* last written tail value */
+	uint16_t	nb_processed_responses;
+	/* number of responses processed since last CSR head write */
+	uint16_t	nb_pending_requests;
+	/* number of requests pending since last CSR tail write */
+};
+
+struct qat_qp {
+	void			*mmap_bar_addr;
+	uint16_t		inflights16;
+	struct qat_queue	tx_q;
+	struct qat_queue	rx_q;
+	struct qat_common_stats stats;
+	struct rte_mempool *op_cookie_pool;
+	void **op_cookies;
+	uint32_t nb_descriptors;
+	enum qat_device_gen qat_dev_gen;
+	build_request_t build_request;
+	enum qat_service_type service_type;
+	struct qat_pci_device *qat_dev;
+	/**< qat device this qp is on */
+} __rte_cache_aligned;
+
+extern const struct qat_qp_hw_data qat_gen1_qps[][ADF_MAX_QPS_ON_ANY_SERVICE];
+
+uint16_t
+qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops);
+
+uint16_t
+qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops);
+
+int
+qat_qp_release(struct qat_qp **qp_addr);
+
+int
+qat_qp_setup(struct qat_pci_device *qat_dev,
+		struct qat_qp **qp_addr, uint16_t queue_pair_id,
+		struct qat_qp_config *qat_qp_conf);
+
+int
+qat_qps_per_service(const struct qat_qp_hw_data *qp_hw_data,
+			enum qat_service_type service);
+
+/* Needed for weak function*/
+int
+qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused);
+
+#endif /* _QAT_QP_H_ */
diff --git a/drivers/compress/Makefile b/drivers/compress/Makefile
index 592497f5..286ea6ee 100644
--- a/drivers/compress/Makefile
+++ b/drivers/compress/Makefile
@@ -4,5 +4,7 @@
 include $(RTE_SDK)/mk/rte.vars.mk
 
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_ISAL) += isal
+DIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_ZIPVF) += octeontx
+DIRS-$(CONFIG_RTE_LIBRTE_PMD_ZLIB) += zlib
 
 include $(RTE_SDK)/mk/rte.subdir.mk
diff --git a/drivers/compress/isal/isal_compress_pmd.c b/drivers/compress/isal/isal_compress_pmd.c
index 0f025a3b..e943336b 100644
--- a/drivers/compress/isal/isal_compress_pmd.c
+++ b/drivers/compress/isal/isal_compress_pmd.c
@@ -188,6 +188,179 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 	return 0;
 }
 
+/* Compression using chained mbufs for input/output data */
+static int
+chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp)
+{
+	int ret;
+	uint32_t remaining_offset;
+	uint32_t remaining_data = op->src.length;
+	struct rte_mbuf *src = op->m_src;
+	struct rte_mbuf *dst = op->m_dst;
+
+	/* check for source/destination offset passing multiple segments
+	 * and point compression stream to input/output buffer.
+	 */
+	remaining_offset = op->src.offset;
+	while (remaining_offset >= src->data_len) {
+		remaining_offset -= src->data_len;
+		src = src->next;
+	}
+	qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset,
+			op->src.length);
+	qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
+			remaining_offset);
+
+	remaining_offset = op->dst.offset;
+	while (remaining_offset >= dst->data_len) {
+		remaining_offset -= dst->data_len;
+		dst = dst->next;
+	}
+	qp->stream->avail_out = dst->data_len - remaining_offset;
+	qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
+			remaining_offset);
+
+	if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
+		ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return -1;
+	}
+
+	while (qp->stream->internal_state.state != ZSTATE_END) {
+		/* Last segment of data */
+		if (remaining_data <= src->data_len)
+			qp->stream->end_of_stream = 1;
+
+		/* Execute compression operation */
+		ret = isal_deflate(qp->stream);
+
+		remaining_data = op->src.length - qp->stream->total_in;
+
+		if (ret != COMP_OK) {
+			ISAL_PMD_LOG(ERR, "Compression operation failed\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+
+		if (qp->stream->avail_in == 0 &&
+				qp->stream->total_in != op->src.length) {
+			if (src->next != NULL) {
+				src = src->next;
+				qp->stream->next_in =
+						rte_pktmbuf_mtod(src, uint8_t *);
+				qp->stream->avail_in =
+					RTE_MIN(remaining_data, src->data_len);
+			} else {
+				ISAL_PMD_LOG(ERR,
+				"Not enough input buffer segments\n");
+				op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+				return -1;
+			}
+		}
+
+		if (qp->stream->avail_out == 0 &&
+				qp->stream->internal_state.state != ZSTATE_END) {
+			if (dst->next != NULL) {
+				dst = dst->next;
+				qp->stream->next_out =
+						rte_pktmbuf_mtod(dst, uint8_t *);
+				qp->stream->avail_out = dst->data_len;
+			} else {
+				ISAL_PMD_LOG(ERR,
+				"Not enough output buffer segments\n");
+				op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+/* Decompression using chained mbufs for input/output data */
+static int
+chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp)
+{
+	int ret;
+	uint32_t consumed_data, src_remaining_offset, dst_remaining_offset;
+	uint32_t remaining_data = op->src.length;
+	struct rte_mbuf *src = op->m_src;
+	struct rte_mbuf *dst = op->m_dst;
+
+	/* check for offset passing multiple segments
+	 * and point decompression state to input/output buffer
+	 */
+	src_remaining_offset = op->src.offset;
+	while (src_remaining_offset >= src->data_len) {
+		src_remaining_offset -= src->data_len;
+		src = src->next;
+	}
+	qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset,
+			op->src.length);
+	qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
+			src_remaining_offset);
+
+	dst_remaining_offset = op->dst.offset;
+	while (dst_remaining_offset >= dst->data_len) {
+		dst_remaining_offset -= dst->data_len;
+		dst = dst->next;
+	}
+	qp->state->avail_out = dst->data_len - dst_remaining_offset;
+	qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
+			dst_remaining_offset);
+
+	while (qp->state->block_state != ISAL_BLOCK_FINISH) {
+
+		ret = isal_inflate(qp->state);
+
+		/* Check for first segment, offset needs to be accounted for */
+		if (remaining_data == op->src.length) {
+			consumed_data = src->data_len - qp->state->avail_in -
+					src_remaining_offset;
+		} else
+			consumed_data = src->data_len - qp->state->avail_in;
+
+		op->consumed += consumed_data;
+		remaining_data -= consumed_data;
+
+		if (ret != ISAL_DECOMP_OK) {
+			ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+
+		if (qp->state->avail_in == 0
+				&& op->consumed != op->src.length) {
+			if (src->next != NULL) {
+				src = src->next;
+				qp->state->next_in =
+						rte_pktmbuf_mtod(src, uint8_t *);
+				qp->state->avail_in =
+					RTE_MIN(remaining_data, src->data_len);
+			}
+		}
+
+		if (qp->state->avail_out == 0 &&
+				qp->state->block_state != ISAL_BLOCK_FINISH) {
+			if (dst->next != NULL) {
+				dst = dst->next;
+				qp->state->next_out =
+						rte_pktmbuf_mtod(dst, uint8_t *);
+				qp->state->avail_out = dst->data_len;
+			} else {
+				ISAL_PMD_LOG(ERR,
+				"Not enough output buffer segments\n");
+				op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
 /* Stateless Compression Function */
 static int
 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
@@ -207,23 +380,10 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 	/* Stateless operation, input will be consumed in one go */
 	qp->stream->flush = NO_FLUSH;
 
-	/* set op level & intermediate level buffer */
+	/* set compression level & intermediate level buffer size */
 	qp->stream->level = priv_xform->compress.level;
 	qp->stream->level_buf_size = priv_xform->level_buffer_size;
 
-	/* Point compression stream structure to input/output buffers */
-	qp->stream->avail_in = op->src.length;
-	qp->stream->next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
-	qp->stream->avail_out = op->m_dst->data_len;
-	qp->stream->next_out  = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
-	qp->stream->end_of_stream = 1; /* All input consumed in one go */
-
-	if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
-		ISAL_PMD_LOG(ERR, "Invalid source or destination buffers\n");
-		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
-		return -1;
-	}
-
 	/* Set op huffman code */
 	if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
 		isal_deflate_set_hufftables(qp->stream, NULL,
@@ -238,30 +398,70 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
 		isal_deflate_set_hufftables(qp->stream, NULL,
 				IGZIP_HUFFTABLE_DEFAULT);
 
-	/* Execute compression operation */
-	ret =  isal_deflate_stateless(qp->stream);
-
-	/* Check that output buffer did not run out of space */
-	if (ret == STATELESS_OVERFLOW) {
-		ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
-		op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
-		return ret;
+	if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
+		ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return -1;
 	}
 
-	/* Check that input buffer has been fully consumed */
-	if (qp->stream->avail_in != (uint32_t)0) {
-		ISAL_PMD_LOG(ERR, "Input buffer could not be read entirely\n");
-		op->status = RTE_COMP_OP_STATUS_ERROR;
+	if (op->dst.offset >= op->m_dst->pkt_len) {
+		ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough"
+				" for offset provided.\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 		return -1;
 	}
 
-	if (ret != COMP_OK) {
-		op->status = RTE_COMP_OP_STATUS_ERROR;
-		return ret;
-	}
+	/* Chained mbufs */
+	if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
+		ret = chained_mbuf_compression(op, qp);
+		if (ret < 0)
+			return ret;
+	} else {
+		/* Linear buffer */
+		qp->stream->end_of_stream = 1; /* All input consumed in one */
+		/* Point compression stream to input buffer */
+		qp->stream->avail_in = op->src.length;
+		qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src,
+				uint8_t *, op->src.offset);
+
+		/*  Point compression stream to output buffer */
+		qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
+		qp->stream->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
+				uint8_t *, op->dst.offset);
+
+		if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
+			ISAL_PMD_LOG(ERR, "Invalid source or destination"
+					" buffers\n");
+			op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+			return -1;
+		}
 
-	op->consumed = qp->stream->total_in;
-	op->produced = qp->stream->total_out;
+		/* Execute compression operation */
+		ret =  isal_deflate_stateless(qp->stream);
+
+		/* Check that output buffer did not run out of space */
+		if (ret == STATELESS_OVERFLOW) {
+			ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
+			op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+			return ret;
+		}
+
+		/* Check that input buffer has been fully consumed */
+		if (qp->stream->avail_in != (uint32_t)0) {
+			ISAL_PMD_LOG(ERR, "Input buffer could not be read"
+					" entirely\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return -1;
+		}
+
+		if (ret != COMP_OK) {
+			ISAL_PMD_LOG(ERR, "Compression operation failed\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+	}
+		op->consumed = qp->stream->total_in;
+		op->produced = qp->stream->total_out;
 
 	return ret;
 }
@@ -277,43 +477,69 @@ process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp)
 	/* Initialize decompression state */
 	isal_inflate_init(qp->state);
 
-	/* Point decompression state structure to input/output buffers */
-	qp->state->avail_in = op->src.length;
-	qp->state->next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
-	qp->state->avail_out = op->m_dst->data_len;
-	qp->state->next_out  = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
+	if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
+		ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return -1;
+	}
 
-	if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
-		ISAL_PMD_LOG(ERR, "Invalid source or destination buffers\n");
+	if (op->dst.offset >= op->m_dst->pkt_len) {
+		ISAL_PMD_LOG(ERR, "Output mbuf not big enough for "
+				"offset provided.\n");
 		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
 		return -1;
 	}
 
-	/* Execute decompression operation */
-	ret = isal_inflate_stateless(qp->state);
+	/* Chained mbufs */
+	if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
+		ret = chained_mbuf_decompression(op, qp);
+		if (ret !=  0)
+			return ret;
+	} else {
+		/* Linear buffer */
+		/* Point decompression state to input buffer */
+		qp->state->avail_in = op->src.length;
+		qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src,
+				uint8_t *, op->src.offset);
+
+		/* Point decompression state to output buffer */
+		qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
+		qp->state->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
+				uint8_t *, op->dst.offset);
+
+		if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
+			ISAL_PMD_LOG(ERR, "Invalid source or destination"
+					" buffers\n");
+			op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+			return -1;
+		}
 
-	if (ret == ISAL_OUT_OVERFLOW) {
-		ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
-		op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
-		return ret;
-	}
+		/* Execute decompression operation */
+		ret = isal_inflate_stateless(qp->state);
 
-	/* Check that input buffer has been fully consumed */
-	if (qp->state->avail_in != (uint32_t)0) {
-		ISAL_PMD_LOG(ERR, "Input buffer could not be read entirely\n");
-		op->status = RTE_COMP_OP_STATUS_ERROR;
-		return -1;
-	}
+		if (ret == ISAL_OUT_OVERFLOW) {
+			ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
+			op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+			return ret;
+		}
 
-	if (ret != ISAL_DECOMP_OK) {
-		op->status = RTE_COMP_OP_STATUS_ERROR;
-		return ret;
-	}
+		/* Check that input buffer has been fully consumed */
+		if (qp->state->avail_in != (uint32_t)0) {
+			ISAL_PMD_LOG(ERR, "Input buffer could not be read"
+					" entirely\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return -1;
+		}
 
-	op->consumed = op->src.length - qp->state->avail_in;
-	op->produced = qp->state->total_out;
+		if (ret != ISAL_DECOMP_OK) {
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+		op->consumed = op->src.length - qp->state->avail_in;
+	}
+		op->produced = qp->state->total_out;
 
-return ret;
+	return ret;
 }
 
 /* Process compression/decompression operation */
@@ -460,12 +686,9 @@ RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv);
 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD,
 	"socket_id=<int>");
 
-RTE_INIT(isal_init_log);
-
-static void
-isal_init_log(void)
+RTE_INIT(isal_init_log)
 {
-	isal_logtype_driver = rte_log_register("comp_isal");
+	isal_logtype_driver = rte_log_register("pmd.compress.isal");
 	if (isal_logtype_driver >= 0)
 		rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO);
 }
diff --git a/drivers/compress/isal/isal_compress_pmd_ops.c b/drivers/compress/isal/isal_compress_pmd_ops.c
index 970a0413..41cade87 100644
--- a/drivers/compress/isal/isal_compress_pmd_ops.c
+++ b/drivers/compress/isal/isal_compress_pmd_ops.c
@@ -12,7 +12,12 @@
 static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = {
 	{
 		.algo = RTE_COMP_ALGO_DEFLATE,
-		.comp_feature_flags =	RTE_COMP_FF_SHAREABLE_PRIV_XFORM,
+		.comp_feature_flags =	RTE_COMP_FF_OOP_SGL_IN_SGL_OUT |
+					RTE_COMP_FF_OOP_SGL_IN_LB_OUT |
+					RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
+					RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
+					RTE_COMP_FF_HUFFMAN_FIXED |
+					RTE_COMP_FF_HUFFMAN_DYNAMIC,
 		.window_size = {
 			.min = 15,
 			.max = 15,
@@ -165,8 +170,11 @@ isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
 	if (qp->state != NULL)
 		rte_free(qp->state);
 
-	if (dev->data->queue_pairs[qp_id] != NULL)
-		rte_free(dev->data->queue_pairs[qp_id]);
+	if (qp->processed_pkts != NULL)
+		rte_ring_free(qp->processed_pkts);
+
+	rte_free(qp);
+	dev->data->queue_pairs[qp_id] = NULL;
 
 	return 0;
 }
diff --git a/drivers/compress/meson.build b/drivers/compress/meson.build
index fb136e1b..817ef3be 100644
--- a/drivers/compress/meson.build
+++ b/drivers/compress/meson.build
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2018 Intel Corporation
 
-drivers = ['isal']
+drivers = ['isal', 'octeontx', 'qat', 'zlib']
 
 std_deps = ['compressdev'] # compressdev pulls in all other needed deps
 config_flag_fmt = 'RTE_LIBRTE_@0@_PMD'
diff --git a/drivers/compress/octeontx/Makefile b/drivers/compress/octeontx/Makefile
new file mode 100644
index 00000000..f34424c8
--- /dev/null
+++ b/drivers/compress/octeontx/Makefile
@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Cavium, Inc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_pmd_octeontx_zip.a
+
+# library version
+LIBABIVER := 1
+
+# build flags
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -O3
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+CFLAGS += -I$(RTE_SDK)/drivers/compress/octeontx/include
+
+# external library include paths
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_compressdev
+LDLIBS += -lrte_pci -lrte_bus_pci
+
+# library source files
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_ZIPVF) += otx_zip_pmd.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_ZIPVF) += otx_zip.c
+
+# versioning export map
+EXPORT_MAP := rte_pmd_octeontx_compress_version.map
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/compress/octeontx/include/zip_regs.h b/drivers/compress/octeontx/include/zip_regs.h
new file mode 100644
index 00000000..1e74db43
--- /dev/null
+++ b/drivers/compress/octeontx/include/zip_regs.h
@@ -0,0 +1,711 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _RTE_OCTEONTX_ZIP_REGS_H_
+#define _RTE_OCTEONTX_ZIP_REGS_H_
+
+
+/**
+ * Enumeration zip_cc
+ *
+ * ZIP compression coding Enumeration
+ * Enumerates ZIP_INST_S[CC].
+ */
+enum {
+	ZIP_CC_DEFAULT = 0,
+	ZIP_CC_DYN_HUFF,
+	ZIP_CC_FIXED_HUFF,
+	ZIP_CC_LZS
+} zip_cc;
+
+/**
+ * Register (NCB) zip_vq#_ena
+ *
+ * ZIP VF Queue Enable Register
+ * If a queue is disabled, ZIP CTL stops fetching instructions from the queue.
+ */
+typedef union {
+	uint64_t u;
+	struct zip_vqx_ena_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+		uint64_t reserved_1_63         : 63;
+		uint64_t ena                   : 1;
+#else /* Word 0 - Little Endian */
+		uint64_t ena                   : 1;
+		uint64_t reserved_1_63         : 63;
+#endif /* Word 0 - End */
+	} s;
+	/* struct zip_vqx_ena_s cn; */
+} zip_vqx_ena_t;
+
+/**
+ * Register (NCB) zip_vq#_sbuf_addr
+ *
+ * ZIP VF Queue Starting Buffer Address Registers
+ * These registers set the buffer parameters for the instruction queues.
+ * When quiescent (i.e.
+ * outstanding doorbell count is 0), it is safe to rewrite this register
+ * to effectively reset the
+ * command buffer state machine.
+ * These registers must be programmed after software programs the
+ * corresponding ZIP_QUE()_SBUF_CTL.
+ */
+typedef union {
+	uint64_t u;
+	struct zip_vqx_sbuf_addr_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+		uint64_t reserved_49_63        : 15;
+		uint64_t ptr                   : 42;
+		uint64_t off                   : 7;
+#else /* Word 0 - Little Endian */
+		uint64_t off                   : 7;
+		uint64_t ptr                   : 42;
+		uint64_t reserved_49_63        : 15;
+#endif /* Word 0 - End */
+	} s;
+	/* struct zip_vqx_sbuf_addr_s cn; */
+} zip_vqx_sbuf_addr_t;
+
+/**
+ * Register (NCB) zip_que#_doorbell
+ *
+ * ZIP Queue Doorbell Registers
+ * Doorbells for the ZIP instruction queues.
+ */
+typedef union {
+	uint64_t u;
+	struct zip_quex_doorbell_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+		uint64_t reserved_20_63        : 44;
+		uint64_t dbell_cnt             : 20;
+#else /* Word 0 - Little Endian */
+		uint64_t dbell_cnt             : 20;
+		uint64_t reserved_20_63        : 44;
+#endif /* Word 0 - End */
+	} s;
+	/* struct zip_quex_doorbell_s cn; */
+} zip_quex_doorbell_t;
+
+/**
+ * Structure zip_nptr_s
+ *
+ * ZIP Instruction Next-Chunk-Buffer Pointer (NPTR) Structure
+ * This structure is used to chain all the ZIP instruction buffers
+ * together. ZIP instruction buffers are managed
+ * (allocated and released) by software.
+ */
+union zip_nptr_s {
+	uint64_t u;
+	struct zip_nptr_s_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+		uint64_t addr                  : 64;
+#else /* Word 0 - Little Endian */
+		uint64_t addr                  : 64;
+#endif /* Word 0 - End */
+	} s;
+	/* struct zip_nptr_s_s cn83xx; */
+};
+
+/**
+ * generic ptr address
+ */
+union zip_zptr_addr_s {
+	/** This field can be used to set/clear all bits, or do bitwise
+	 * operations over the entire structure.
+	 */
+	uint64_t u;
+	/** generic ptr address */
+	struct {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+		uint64_t addr : 64;
+#else /* Word 0 - Little Endian */
+		uint64_t addr : 64;
+#endif /* Word 0 - End */
+	} s;
+};
+
+/**
+ * generic ptr ctl
+ */
+union zip_zptr_ctl_s {
+	/** This field can be used to set/clear all bits, or do bitwise
+	 * operations over the entire structure.
+	 */
+	uint64_t u;
+	/** generic ptr ctl */
+	struct {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 1 - Big Endian */
+		uint64_t reserved_112_127      : 16;
+		uint64_t length                : 16;
+		uint64_t reserved_67_95        : 29;
+		uint64_t fw                    : 1;
+		uint64_t nc                    : 1;
+		uint64_t data_be               : 1;
+#else /* Word 1 - Little Endian */
+		uint64_t data_be               : 1;
+		uint64_t nc                    : 1;
+		uint64_t fw                    : 1;
+		uint64_t reserved_67_95        : 29;
+		uint64_t length                : 16;
+		uint64_t reserved_112_127      : 16;
+#endif /* Word 1 - End */
+	} s;
+
+};
+
+/**
+ * Structure zip_inst_s
+ *
+ * ZIP Instruction Structure
+ * Each ZIP instruction has 16 words (they are called IWORD0 to IWORD15
+ * within the structure).
+ */
+union zip_inst_s {
+	/** This field can be used to set/clear all bits, or do bitwise
+	 * operations over the entire structure.
+	 */
+	uint64_t u[16];
+	/** ZIP Instruction Structure */
+	struct zip_inst_s_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+		/** Done interrupt */
+		uint64_t doneint               : 1;
+		/** reserved */
+		uint64_t reserved_56_62        : 7;
+		/**  Total output length */
+		uint64_t totaloutputlength     : 24;
+		/** reserved */
+		uint64_t reserved_27_31        : 5;
+		/** EXNUM */
+		uint64_t exn                   : 3;
+		/**  HASH IV */
+		uint64_t iv                    : 1;
+		/** EXBITS */
+		uint64_t exbits                : 7;
+		/** Hash more-in-file */
+		uint64_t hmif                  : 1;
+		/** Hash Algorithm and enable */
+		uint64_t halg                  : 3;
+		/** Sync flush*/
+		uint64_t sf                    : 1;
+		/** Compression speed/storage */
+		uint64_t ss                    : 2;
+		/** Compression coding */
+		uint64_t cc                    : 2;
+		/** End of input data */
+		uint64_t ef                    : 1;
+		/** Beginning of file */
+		uint64_t bf                    : 1;
+		// uint64_t reserved_3_4          : 2;
+		/** Comp/decomp operation */
+		uint64_t op                    : 2;
+		/** Data sactter */
+		uint64_t ds                    : 1;
+		/** Data gather */
+		uint64_t dg                    : 1;
+		/** History gather */
+		uint64_t hg                    : 1;
+#else /* Word 0 - Little Endian */
+		uint64_t hg                    : 1;
+		uint64_t dg                    : 1;
+		uint64_t ds                    : 1;
+		//uint64_t reserved_3_4          : 2;
+		uint64_t op                    : 2;
+		uint64_t bf                    : 1;
+		uint64_t ef                    : 1;
+		uint64_t cc                    : 2;
+		uint64_t ss                    : 2;
+		uint64_t sf                    : 1;
+		uint64_t halg                  : 3;
+		uint64_t hmif                  : 1;
+		uint64_t exbits                : 7;
+		uint64_t iv                    : 1;
+		uint64_t exn                   : 3;
+		uint64_t reserved_27_31        : 5;
+		uint64_t totaloutputlength     : 24;
+		uint64_t reserved_56_62        : 7;
+		uint64_t doneint               : 1;
+
+#endif /* Word 0 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 1 - Big Endian */
+		/** History length */
+		uint64_t historylength         : 16;
+		/** reserved */
+		uint64_t reserved_96_111       : 16;
+		/** adler/crc32 checksum*/
+		uint64_t adlercrc32            : 32;
+#else /* Word 1 - Little Endian */
+		uint64_t adlercrc32            : 32;
+		uint64_t reserved_96_111       : 16;
+		uint64_t historylength         : 16;
+#endif /* Word 1 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */
+		/** Decompression Context Pointer Address */
+		union zip_zptr_addr_s  ctx_ptr_addr;
+#else /* Word 2 - Little Endian */
+		union zip_zptr_addr_s  ctx_ptr_addr;
+#endif /* Word 2 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD)
+		/** Decompression Context Pointer Control */
+		union zip_zptr_ctl_s   ctx_ptr_ctl;
+#else /* Word 3 - Little Endian */
+		union zip_zptr_ctl_s   ctx_ptr_ctl;
+#endif /* Word 3 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD)
+		/** Decompression history pointer address */
+		union zip_zptr_addr_s  his_ptr_addr;
+#else /* Word 4 - Little Endian */
+		union zip_zptr_addr_s  his_ptr_addr;
+#endif /* Word 4 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD)
+		/** Decompression history pointer control */
+		union zip_zptr_ctl_s   his_ptr_ctl;
+#else /* Word 5 - Little Endian */
+		union zip_zptr_ctl_s   his_ptr_ctl;
+#endif /* Word 5 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD)
+		/** Input and compression history pointer address */
+		union zip_zptr_addr_s  inp_ptr_addr;
+#else /* Word 6 - Little Endian */
+		union zip_zptr_addr_s  inp_ptr_addr;
+#endif /* Word 6 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD)
+		/** Input and compression history pointer control */
+		union zip_zptr_ctl_s   inp_ptr_ctl;
+#else /* Word 7 - Little Endian */
+		union zip_zptr_ctl_s   inp_ptr_ctl;
+#endif /* Word 7 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD)
+		/** Output pointer address */
+		union zip_zptr_addr_s  out_ptr_addr;
+#else /* Word 8 - Little Endian */
+		union zip_zptr_addr_s  out_ptr_addr;
+#endif /* Word 8 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD)
+		/** Output pointer control */
+		union zip_zptr_ctl_s   out_ptr_ctl;
+#else /* Word 9 - Little Endian */
+		union zip_zptr_ctl_s   out_ptr_ctl;
+#endif /* Word 9 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD)
+		/** Result pointer address */
+		union zip_zptr_addr_s  res_ptr_addr;
+#else /* Word 10 - Little Endian */
+		union zip_zptr_addr_s  res_ptr_addr;
+#endif /* Word 10 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD)
+		/** Result pointer control */
+		union zip_zptr_ctl_s   res_ptr_ctl;
+#else /* Word 11 - Little Endian */
+		union zip_zptr_ctl_s   res_ptr_ctl;
+#endif /* Word 11 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 12 - Big Endian */
+		/** reserved */
+		uint64_t reserved_812_831      : 20;
+		/** SSO guest group */
+		uint64_t ggrp                  : 10;
+		/** SSO tag type */
+		uint64_t tt                    : 2;
+		/** SSO tag */
+		uint64_t tag                   : 32;
+#else /* Word 12 - Little Endian */
+		uint64_t tag                   : 32;
+		uint64_t tt                    : 2;
+		uint64_t ggrp                  : 10;
+		uint64_t reserved_812_831      : 20;
+#endif /* Word 12 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 13 - Big Endian */
+		/** Work queue entry pointer */
+		uint64_t wq_ptr                : 64;
+#else /* Word 13 - Little Endian */
+		uint64_t wq_ptr                : 64;
+#endif /* Word 13 - End */
+
+#if defined(__BIG_ENDIAN_BITFIELD)
+		/** reserved */
+		uint64_t reserved_896_959      : 64;
+#else /* Word 14 - Little Endian */
+		uint64_t reserved_896_959      : 64;
+#endif /* Word 14 - End */
+#if defined(__BIG_ENDIAN_BITFIELD)
+		/** Hash structure pointer */
+		uint64_t hash_ptr              : 64;
+#else /* Word 15 - Little Endian */
+		uint64_t hash_ptr              : 64;
+#endif /* Word 15 - End */
+	} /** ZIP 88xx Instruction Structure */zip88xx;
+
+	/** ZIP Instruction Structure */
+	struct zip_inst_s_cn83xx {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+		/** Done interrupt */
+		uint64_t doneint               : 1;
+		/** reserved */
+		uint64_t reserved_56_62        : 7;
+		/**  Total output length */
+		uint64_t totaloutputlength     : 24;
+		/** reserved */
+		uint64_t reserved_27_31        : 5;
+		/** EXNUM */
+		uint64_t exn                   : 3;
+		/**  HASH IV */
+		uint64_t iv                    : 1;
+		/** EXBITS */
+		uint64_t exbits                : 7;
+		/** Hash more-in-file */
+		uint64_t hmif                  : 1;
+		/** Hash Algorithm and enable */
+		uint64_t halg                  : 3;
+		/** Sync flush*/
+		uint64_t sf                    : 1;
+		/** Compression speed/storage */
+		uint64_t ss                    : 2;
+		/** Compression coding */
+		uint64_t cc                    : 2;
+		/** End of input data */
+		uint64_t ef                    : 1;
+		/** Beginning of file */
+		uint64_t bf                    : 1;
+		/** Comp/decomp operation */
+		uint64_t op                    : 2;
+		/** Data sactter */
+		uint64_t ds                    : 1;
+		/** Data gather */
+		uint64_t dg                    : 1;
+		/** History gather */
+		uint64_t hg                    : 1;
+#else /* Word 0 - Little Endian */
+		uint64_t hg                    : 1;
+		uint64_t dg                    : 1;
+		uint64_t ds                    : 1;
+		uint64_t op                    : 2;
+		uint64_t bf                    : 1;
+		uint64_t ef                    : 1;
+		uint64_t cc                    : 2;
+		uint64_t ss                    : 2;
+		uint64_t sf                    : 1;
+		uint64_t halg                  : 3;
+		uint64_t hmif                  : 1;
+		uint64_t exbits                : 7;
+		uint64_t iv                    : 1;
+		uint64_t exn                   : 3;
+		uint64_t reserved_27_31        : 5;
+		uint64_t totaloutputlength     : 24;
+		uint64_t reserved_56_62        : 7;
+		uint64_t doneint               : 1;
+#endif /* Word 0 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 1 - Big Endian */
+		/** History length */
+		uint64_t historylength         : 16;
+		/** reserved */
+		uint64_t reserved_96_111       : 16;
+		/** adler/crc32 checksum*/
+		uint64_t adlercrc32            : 32;
+#else /* Word 1 - Little Endian */
+		uint64_t adlercrc32            : 32;
+		uint64_t reserved_96_111       : 16;
+		uint64_t historylength         : 16;
+#endif /* Word 1 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */
+		/** Decompression Context Pointer Address */
+		union zip_zptr_addr_s  ctx_ptr_addr;
+#else /* Word 2 - Little Endian */
+		union zip_zptr_addr_s  ctx_ptr_addr;
+#endif /* Word 2 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 3 - Big Endian */
+		/** Decompression Context Pointer Control */
+		union zip_zptr_ctl_s   ctx_ptr_ctl;
+#else /* Word 3 - Little Endian */
+		union zip_zptr_ctl_s   ctx_ptr_ctl;
+#endif /* Word 3 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 4 - Big Endian */
+		/** Decompression history pointer address */
+		union zip_zptr_addr_s  his_ptr_addr;
+#else /* Word 4 - Little Endian */
+		union zip_zptr_addr_s  his_ptr_addr;
+#endif /* Word 4 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 5 - Big Endian */
+		/** Decompression history pointer control */
+		union zip_zptr_ctl_s   his_ptr_ctl;
+#else /* Word 5 - Little Endian */
+		union zip_zptr_ctl_s   his_ptr_ctl;
+#endif /* Word 5 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 6 - Big Endian */
+		/** Input and compression history pointer address */
+		union zip_zptr_addr_s  inp_ptr_addr;
+#else /* Word 6 - Little Endian */
+		union zip_zptr_addr_s  inp_ptr_addr;
+#endif /* Word 6 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 7 - Big Endian */
+		/** Input and compression history pointer control */
+		union zip_zptr_ctl_s   inp_ptr_ctl;
+#else /* Word 7 - Little Endian */
+		union zip_zptr_ctl_s   inp_ptr_ctl;
+#endif /* Word 7 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 8 - Big Endian */
+		/** Output pointer address */
+		union zip_zptr_addr_s  out_ptr_addr;
+#else /* Word 8 - Little Endian */
+		union zip_zptr_addr_s  out_ptr_addr;
+#endif /* Word 8 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 9 - Big Endian */
+		/** Output pointer control */
+		union zip_zptr_ctl_s   out_ptr_ctl;
+#else /* Word 9 - Little Endian */
+		union zip_zptr_ctl_s   out_ptr_ctl;
+#endif /* Word 9 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 10 - Big Endian */
+		/** Result pointer address */
+		union zip_zptr_addr_s  res_ptr_addr;
+#else /* Word 10 - Little Endian */
+		union zip_zptr_addr_s  res_ptr_addr;
+#endif /* Word 10 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 11 - Big Endian */
+		/** Result pointer control */
+		union zip_zptr_ctl_s   res_ptr_ctl;
+#else /* Word 11 - Little Endian */
+		union zip_zptr_ctl_s   res_ptr_ctl;
+#endif /* Word 11 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 12 - Big Endian */
+		/** reserved */
+		uint64_t reserved_812_831      : 20;
+		/** SSO guest group */
+		uint64_t ggrp                  : 10;
+		/** SSO tag type */
+		uint64_t tt                    : 2;
+		/** SSO tag */
+		uint64_t tag                   : 32;
+#else /* Word 12 - Little Endian */
+		uint64_t tag                   : 32;
+		uint64_t tt                    : 2;
+		uint64_t ggrp                  : 10;
+		uint64_t reserved_812_831      : 20;
+#endif /* Word 12 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 13 - Big Endian */
+		/** Work queue entry pointer */
+		uint64_t wq_ptr                : 64;
+#else /* Word 13 - Little Endian */
+		uint64_t wq_ptr                : 64;
+#endif /* Word 13 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 14 - Big Endian */
+		/** reserved */
+		uint64_t reserved_896_959      : 64;
+#else /* Word 14 - Little Endian */
+		uint64_t reserved_896_959      : 64;
+#endif /* Word 14 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 15 - Big Endian */
+		/** Hash structure pointer */
+		uint64_t hash_ptr              : 64;
+#else /* Word 15 - Little Endian */
+		uint64_t hash_ptr              : 64;
+#endif /* Word 15 - End */
+	} /** ZIP 83xx Instruction Structure */s;
+};
+
+/**
+ * Structure zip_zres_s
+ *
+ * ZIP Result Structure
+ * The ZIP coprocessor writes the result structure after it completes the
+ * invocation. The result structure is exactly 24 bytes, and each invocation
+ * of the ZIP coprocessor produces exactly one result structure.
+ */
+union zip_zres_s {
+	/** This field can be used to set/clear all bits, or do bitwise
+	 * operations over the entire structure.
+	 */
+	uint64_t u[8];
+	/** ZIP Result Structure */
+	struct zip_zres_s_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+		/** crc32 checksum of uncompressed stream */
+		uint64_t crc32                 : 32;
+		/** adler32 checksum of uncompressed stream*/
+		uint64_t adler32               : 32;
+#else /* Word 0 - Little Endian */
+		uint64_t adler32               : 32;
+		uint64_t crc32                 : 32;
+#endif /* Word 0 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 1 - Big Endian */
+		/** Total numer of Bytes produced in output stream */
+		uint64_t totalbyteswritten     : 32;
+		/** Total number of bytes processed from the input stream */
+		uint64_t totalbytesread        : 32;
+#else /* Word 1 - Little Endian */
+		uint64_t totalbytesread        : 32;
+		uint64_t totalbyteswritten     : 32;
+#endif /* Word 1 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */
+		/** Total number of compressed input bits
+		 * consumed to decompress all blocks in the file
+		 */
+		uint64_t totalbitsprocessed    : 32;
+		/** Done interrupt*/
+		uint64_t doneint               : 1;
+		/** reserved */
+		uint64_t reserved_155_158      : 4;
+		/** EXNUM */
+		uint64_t exn                   : 3;
+		/** reserved */
+		uint64_t reserved_151          : 1;
+		/** EXBITS */
+		uint64_t exbits                : 7;
+		/** reserved */
+		uint64_t reserved_137_143      : 7;
+		/** End of file */
+		uint64_t ef                    : 1;
+		/** Completion/error code */
+		uint64_t compcode              : 8;
+#else /* Word 2 - Little Endian */
+		uint64_t compcode              : 8;
+		uint64_t ef                    : 1;
+		uint64_t reserved_137_143      : 7;
+		uint64_t exbits                : 7;
+		uint64_t reserved_151          : 1;
+		uint64_t exn                   : 3;
+		uint64_t reserved_155_158      : 4;
+		uint64_t doneint               : 1;
+		uint64_t totalbitsprocessed    : 32;
+#endif /* Word 2 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 3 - Big Endian */
+		/** reserved */
+		uint64_t reserved_253_255      : 3;
+		/** Hash length in bytes */
+		uint64_t hshlen                : 61;
+#else /* Word 3 - Little Endian */
+		uint64_t hshlen                : 61;
+		uint64_t reserved_253_255      : 3;
+#endif /* Word 3 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 4 - Big Endian */
+		/** Double-word 0 of computed hash */
+		uint64_t hash0                 : 64;
+#else /* Word 4 - Little Endian */
+		uint64_t hash0                 : 64;
+#endif /* Word 4 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 5 - Big Endian */
+		/** Double-word 1 of computed hash */
+		uint64_t hash1                 : 64;
+#else /* Word 5 - Little Endian */
+		uint64_t hash1                 : 64;
+#endif /* Word 5 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 6 - Big Endian */
+		/** Double-word 2 of computed hash */
+		uint64_t hash2                 : 64;
+#else /* Word 6 - Little Endian */
+		uint64_t hash2                 : 64;
+#endif /* Word 6 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 7 - Big Endian */
+		/** Double-word 3 of computed hash */
+		uint64_t hash3                 : 64;
+#else /* Word 7 - Little Endian */
+		uint64_t hash3                 : 64;
+#endif /* Word 7 - End */
+	} /** ZIP Result Structure */s;
+
+	/* struct zip_zres_s_s cn83xx; */
+};
+
+/**
+ * Structure zip_zptr_s
+ *
+ * ZIP Generic Pointer Structure
+ * This structure is the generic format of pointers in ZIP_INST_S.
+ */
+union zip_zptr_s {
+	/** This field can be used to set/clear all bits, or do bitwise
+	 * operations over the entire structure.
+	 */
+	uint64_t u[2];
+	/** ZIP Generic Pointer Structure */
+	struct zip_zptr_s_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+		/** Pointer to Data or scatter-gather list */
+		uint64_t addr                  : 64;
+#else /* Word 0 - Little Endian */
+		uint64_t addr                  : 64;
+#endif /* Word 0 - End */
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 1 - Big Endian */
+		/** reserved */
+		uint64_t reserved_112_127      : 16;
+		/** Length of Data or scatter-gather list*/
+		uint64_t length                : 16;
+		/** reserved */
+		uint64_t reserved_67_95        : 29;
+		/** Full-block write */
+		uint64_t fw                    : 1;
+		/** No cache allocation */
+		uint64_t nc                    : 1;
+		/** reserved */
+		uint64_t data_be               : 1;
+#else /* Word 1 - Little Endian */
+		uint64_t data_be               : 1;
+		uint64_t nc                    : 1;
+		uint64_t fw                    : 1;
+		uint64_t reserved_67_95        : 29;
+		uint64_t length                : 16;
+		uint64_t reserved_112_127      : 16;
+#endif /* Word 1 - End */
+	} /** ZIP Generic Pointer Structure */s;
+};
+
+/**
+ * Enumeration zip_comp_e
+ *
+ * ZIP Completion Enumeration
+ * Enumerates the values of ZIP_ZRES_S[COMPCODE].
+ */
+#define ZIP_COMP_E_NOTDONE       (0)
+#define ZIP_COMP_E_SUCCESS       (1)
+#define ZIP_COMP_E_DTRUNC        (2)
+#define ZIP_COMP_E_DSTOP         (3)
+#define ZIP_COMP_E_ITRUNC        (4)
+#define ZIP_COMP_E_RBLOCK        (5)
+#define ZIP_COMP_E_NLEN          (6)
+#define ZIP_COMP_E_BADCODE       (7)
+#define ZIP_COMP_E_BADCODE2      (8)
+#define ZIP_COMP_E_ZERO_LEN      (9)
+#define ZIP_COMP_E_PARITY        (0xa)
+#define ZIP_COMP_E_FATAL         (0xb)
+#define ZIP_COMP_E_TIMEOUT       (0xc)
+#define ZIP_COMP_E_INSTR_ERR     (0xd)
+#define ZIP_COMP_E_HCTX_ERR      (0xe)
+#define ZIP_COMP_E_STOP          (3)
+
+/**
+ * Enumeration zip_op_e
+ *
+ * ZIP Operation Enumeration
+ * Enumerates ZIP_INST_S[OP].
+ * Internal:
+ */
+#define ZIP_OP_E_DECOMP   (0)
+#define ZIP_OP_E_NOCOMP   (1)
+#define ZIP_OP_E_COMP     (2)
+
+/**
+ * Enumeration zip compression levels
+ *
+ * ZIP Compression Level Enumeration
+ * Enumerates ZIP_INST_S[SS].
+ * Internal:
+ */
+#define ZIP_COMP_E_LEVEL_MAX  (0)
+#define ZIP_COMP_E_LEVEL_MED  (1)
+#define ZIP_COMP_E_LEVEL_LOW  (2)
+#define ZIP_COMP_E_LEVEL_MIN  (3)
+
+#endif	/* _RTE_ZIP_REGS_H_ */
diff --git a/drivers/compress/octeontx/meson.build b/drivers/compress/octeontx/meson.build
new file mode 100644
index 00000000..7cd202d0
--- /dev/null
+++ b/drivers/compress/octeontx/meson.build
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Cavium, Inc
+
+name = 'octeontx_compress'
+sources = files('otx_zip.c', 'otx_zip_pmd.c')
+allow_experimental_apis = true
+includes += include_directories('include')
+deps += ['mempool_octeontx', 'bus_pci']
+ext_deps += dep
diff --git a/drivers/compress/octeontx/otx_zip.c b/drivers/compress/octeontx/otx_zip.c
new file mode 100644
index 00000000..a9046ff3
--- /dev/null
+++ b/drivers/compress/octeontx/otx_zip.c
@@ -0,0 +1,180 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#include "otx_zip.h"
+
+uint64_t
+zip_reg_read64(uint8_t *hw_addr, uint64_t offset)
+{
+	uint8_t *base = hw_addr;
+	return *(volatile uint64_t *)(base + offset);
+}
+
+void
+zip_reg_write64(uint8_t *hw_addr, uint64_t offset, uint64_t val)
+{
+	uint8_t *base = hw_addr;
+	*(uint64_t *)(base + offset) = val;
+}
+
+static void
+zip_q_enable(struct zipvf_qp *qp)
+{
+	zip_vqx_ena_t que_ena;
+
+	/*ZIP VFx command queue init*/
+	que_ena.u = 0ull;
+	que_ena.s.ena = 1;
+
+	zip_reg_write64(qp->vf->vbar0, ZIP_VQ_ENA, que_ena.u);
+	rte_wmb();
+}
+
+/* initialize given qp on zip device */
+int
+zipvf_q_init(struct zipvf_qp *qp)
+{
+	zip_vqx_sbuf_addr_t que_sbuf_addr;
+
+	uint64_t size;
+	void *cmdq_addr;
+	uint64_t iova;
+	struct zipvf_cmdq *cmdq = &qp->cmdq;
+	struct zip_vf *vf = qp->vf;
+
+	/* allocate and setup instruction queue */
+	size = ZIP_MAX_CMDQ_SIZE;
+	size = ZIP_ALIGN_ROUNDUP(size, ZIP_CMDQ_ALIGN);
+
+	cmdq_addr = rte_zmalloc(qp->name, size, ZIP_CMDQ_ALIGN);
+	if (cmdq_addr == NULL)
+		return -1;
+
+	cmdq->sw_head = (uint64_t *)cmdq_addr;
+	cmdq->va = (uint8_t *)cmdq_addr;
+	iova = rte_mem_virt2iova(cmdq_addr);
+
+	cmdq->iova = iova;
+
+	que_sbuf_addr.u = 0ull;
+	que_sbuf_addr.s.ptr = (cmdq->iova >> 7);
+	zip_reg_write64(vf->vbar0, ZIP_VQ_SBUF_ADDR, que_sbuf_addr.u);
+
+	zip_q_enable(qp);
+
+	memset(cmdq->va, 0, ZIP_MAX_CMDQ_SIZE);
+	rte_spinlock_init(&cmdq->qlock);
+
+	return 0;
+}
+
+int
+zipvf_q_term(struct zipvf_qp *qp)
+{
+	struct zipvf_cmdq *cmdq = &qp->cmdq;
+	zip_vqx_ena_t que_ena;
+	struct zip_vf *vf = qp->vf;
+
+	if (cmdq->va != NULL) {
+		memset(cmdq->va, 0, ZIP_MAX_CMDQ_SIZE);
+		rte_free(cmdq->va);
+	}
+
+	/*Disabling the ZIP queue*/
+	que_ena.u = 0ull;
+	zip_reg_write64(vf->vbar0, ZIP_VQ_ENA, que_ena.u);
+
+	return 0;
+}
+
+void
+zipvf_push_command(struct zipvf_qp *qp, union zip_inst_s *cmd)
+{
+	zip_quex_doorbell_t dbell;
+	union zip_nptr_s ncp;
+	uint64_t *ncb_ptr;
+	struct zipvf_cmdq *cmdq = &qp->cmdq;
+	void *reg_base = qp->vf->vbar0;
+
+	/*Held queue lock*/
+	rte_spinlock_lock(&(cmdq->qlock));
+
+	/* Check space availability in zip cmd queue */
+	if ((((cmdq->sw_head - (uint64_t *)cmdq->va) * sizeof(uint64_t *)) +
+		ZIP_CMD_SIZE) == (ZIP_MAX_CMDQ_SIZE - ZIP_MAX_NCBP_SIZE)) {
+		/*Last buffer of the command queue*/
+		memcpy((uint8_t *)cmdq->sw_head,
+			(uint8_t *)cmd,
+			sizeof(union zip_inst_s));
+		/* move pointer to next loc in unit of 64-bit word */
+		cmdq->sw_head += ZIP_CMD_SIZE_WORDS;
+
+		/* now, point the "Next-Chunk Buffer Ptr" to sw_head */
+		ncb_ptr = cmdq->sw_head;
+		/* Pointing head again to cmdqueue base*/
+		cmdq->sw_head = (uint64_t *)cmdq->va;
+
+		ncp.u = 0ull;
+		ncp.s.addr = cmdq->iova;
+		*ncb_ptr = ncp.u;
+	} else {
+		/*Enough buffers available in the command queue*/
+		memcpy((uint8_t *)cmdq->sw_head,
+			(uint8_t *)cmd,
+			sizeof(union zip_inst_s));
+		cmdq->sw_head += ZIP_CMD_SIZE_WORDS;
+	}
+
+	rte_wmb();
+
+	/* Ringing ZIP VF doorbell */
+	dbell.u = 0ull;
+	dbell.s.dbell_cnt = 1;
+	zip_reg_write64(reg_base, ZIP_VQ_DOORBELL, dbell.u);
+
+	rte_spinlock_unlock(&(cmdq->qlock));
+}
+
+int
+zipvf_create(struct rte_compressdev *compressdev)
+{
+	struct   rte_pci_device *pdev = RTE_DEV_TO_PCI(compressdev->device);
+	struct   zip_vf *zipvf = NULL;
+	char     *dev_name = compressdev->data->name;
+	void     *vbar0;
+	uint64_t reg;
+
+	if (pdev->mem_resource[0].phys_addr == 0ULL)
+		return -EIO;
+
+	vbar0 = pdev->mem_resource[0].addr;
+	if (!vbar0) {
+		ZIP_PMD_ERR("Failed to map BAR0 of %s", dev_name);
+		return -ENODEV;
+	}
+
+	zipvf = (struct zip_vf *)(compressdev->data->dev_private);
+
+	if (!zipvf)
+		return -ENOMEM;
+
+	zipvf->vbar0 = vbar0;
+	reg = zip_reg_read64(zipvf->vbar0, ZIP_VF_PF_MBOXX(0));
+	/* Storing domain in local to ZIP VF */
+	zipvf->dom_sdom = reg;
+	zipvf->pdev = pdev;
+	zipvf->max_nb_queue_pairs = ZIP_MAX_VF_QUEUE;
+	return 0;
+}
+
+int
+zipvf_destroy(struct rte_compressdev *compressdev)
+{
+	struct zip_vf *vf = (struct zip_vf *)(compressdev->data->dev_private);
+
+	/* Rewriting the domain_id in ZIP_VF_MBOX for app rerun */
+	zip_reg_write64(vf->vbar0, ZIP_VF_PF_MBOXX(0), vf->dom_sdom);
+
+	return 0;
+}
diff --git a/drivers/compress/octeontx/otx_zip.h b/drivers/compress/octeontx/otx_zip.h
new file mode 100644
index 00000000..99a38d00
--- /dev/null
+++ b/drivers/compress/octeontx/otx_zip.h
@@ -0,0 +1,277 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _RTE_OCTEONTX_ZIP_VF_H_
+#define _RTE_OCTEONTX_ZIP_VF_H_
+
+#include <unistd.h>
+
+#include <rte_bus_pci.h>
+#include <rte_comp.h>
+#include <rte_compressdev.h>
+#include <rte_compressdev_pmd.h>
+#include <rte_malloc.h>
+#include <rte_memory.h>
+#include <rte_spinlock.h>
+
+#include <zip_regs.h>
+
+int octtx_zip_logtype_driver;
+
+/* ZIP VF Control/Status registers (CSRs): */
+/* VF_BAR0: */
+#define ZIP_VQ_ENA              (0x10)
+#define ZIP_VQ_SBUF_ADDR        (0x20)
+#define ZIP_VF_PF_MBOXX(x)      (0x400 | (x)<<3)
+#define ZIP_VQ_DOORBELL         (0x1000)
+
+/**< Vendor ID */
+#define PCI_VENDOR_ID_CAVIUM	0x177D
+/**< PCI device id of ZIP VF */
+#define PCI_DEVICE_ID_OCTEONTX_ZIPVF	0xA037
+
+/* maxmum number of zip vf devices */
+#define ZIP_MAX_VFS 8
+
+/* max size of one chunk */
+#define ZIP_MAX_CHUNK_SIZE	8192
+
+/* each instruction is fixed 128 bytes */
+#define ZIP_CMD_SIZE		128
+
+#define ZIP_CMD_SIZE_WORDS	(ZIP_CMD_SIZE >> 3) /* 16 64_bit words */
+
+/* size of next chunk buffer pointer */
+#define ZIP_MAX_NCBP_SIZE	8
+
+/* size of instruction queue in units of instruction size */
+#define ZIP_MAX_NUM_CMDS	((ZIP_MAX_CHUNK_SIZE - ZIP_MAX_NCBP_SIZE) / \
+				ZIP_CMD_SIZE) /* 63 */
+
+/* size of instruct queue in bytes */
+#define ZIP_MAX_CMDQ_SIZE	((ZIP_MAX_NUM_CMDS * ZIP_CMD_SIZE) + \
+				ZIP_MAX_NCBP_SIZE)/* ~8072ull */
+
+#define ZIP_BUF_SIZE	256
+
+#define ZIP_SGPTR_ALIGN	16
+#define ZIP_CMDQ_ALIGN	128
+#define MAX_SG_LEN	((ZIP_BUF_SIZE - ZIP_SGPTR_ALIGN) / sizeof(void *))
+
+/**< ZIP PMD specified queue pairs */
+#define ZIP_MAX_VF_QUEUE	1
+
+#define ZIP_ALIGN_ROUNDUP(x, _align) \
+	((_align) * (((x) + (_align) - 1) / (_align)))
+
+/**< ZIP PMD device name */
+#define COMPRESSDEV_NAME_ZIP_PMD	compress_octeonx
+
+#define ZIP_PMD_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, \
+	octtx_zip_logtype_driver, "%s(): "fmt "\n", \
+	__func__, ##args)
+
+#define ZIP_PMD_INFO(fmt, args...) \
+	ZIP_PMD_LOG(INFO, fmt, ## args)
+#define ZIP_PMD_ERR(fmt, args...) \
+	ZIP_PMD_LOG(ERR, fmt, ## args)
+
+/* resources required to process stream */
+enum {
+	RES_BUF = 0,
+	CMD_BUF,
+	HASH_CTX_BUF,
+	DECOMP_CTX_BUF,
+	IN_DATA_BUF,
+	OUT_DATA_BUF,
+	HISTORY_DATA_BUF,
+	MAX_BUFS_PER_STREAM
+} NUM_BUFS_PER_STREAM;
+
+struct zip_stream;
+struct zipvf_qp;
+
+/* Algorithm handler function prototype */
+typedef int (*comp_func_t)(struct rte_comp_op *op,
+			   struct zipvf_qp *qp, struct zip_stream *zstrm);
+
+/**
+ * ZIP private stream structure
+ */
+struct zip_stream {
+	union zip_inst_s *inst;
+	/* zip instruction pointer */
+	comp_func_t func;
+	/* function to process comp operation */
+	void *bufs[MAX_BUFS_PER_STREAM];
+} _rte_cache_aligned;
+
+
+/**
+ * ZIP instruction Queue
+ */
+struct zipvf_cmdq {
+	rte_spinlock_t qlock;
+	/* queue lock */
+	uint64_t *sw_head;
+	/* pointer to start of 8-byte word length queue-head */
+	uint8_t *va;
+	/* pointer to instruction queue virtual address */
+	rte_iova_t iova;
+	/* iova addr of cmdq head*/
+};
+
+/**
+ * ZIP device queue structure
+ */
+struct zipvf_qp {
+	struct zipvf_cmdq cmdq;
+	/* Hardware instruction queue structure */
+	struct rte_ring *processed_pkts;
+	/* Ring for placing processed packets */
+	struct rte_compressdev_stats qp_stats;
+	/* Queue pair statistics */
+	uint16_t id;
+	/* Queue Pair Identifier */
+	const char *name;
+	/* Unique Queue Pair Name */
+	struct zip_vf *vf;
+	/* pointer to device, queue belongs to */
+} __rte_cache_aligned;
+
+/**
+ * ZIP VF device structure.
+ */
+struct zip_vf {
+	int vfid;
+	/* vf index */
+	struct rte_pci_device *pdev;
+	/* pci device */
+	void *vbar0;
+	/* CSR base address for underlying BAR0 VF.*/
+	uint64_t dom_sdom;
+	/* Storing mbox domain and subdomain id for app rerun*/
+	uint32_t  max_nb_queue_pairs;
+	/* pointer to device qps */
+	struct rte_mempool *zip_mp;
+	/* pointer to pools */
+} __rte_cache_aligned;
+
+
+static inline void
+zipvf_prepare_in_buf(struct zip_stream *zstrm, struct rte_comp_op *op)
+{
+	uint32_t offset, inlen;
+	struct rte_mbuf *m_src;
+	union zip_inst_s *inst = zstrm->inst;
+
+	inlen = op->src.length;
+	offset = op->src.offset;
+	m_src = op->m_src;
+
+	/* Prepare direct input data pointer */
+	inst->s.dg = 0;
+	inst->s.inp_ptr_addr.s.addr =
+			rte_pktmbuf_iova_offset(m_src, offset);
+	inst->s.inp_ptr_ctl.s.length = inlen;
+}
+
+static inline void
+zipvf_prepare_out_buf(struct zip_stream *zstrm, struct rte_comp_op *op)
+{
+	uint32_t offset;
+	struct rte_mbuf *m_dst;
+	union zip_inst_s *inst = zstrm->inst;
+
+	offset = op->dst.offset;
+	m_dst = op->m_dst;
+
+	/* Prepare direct input data pointer */
+	inst->s.ds = 0;
+	inst->s.out_ptr_addr.s.addr =
+			rte_pktmbuf_iova_offset(m_dst, offset);
+	inst->s.totaloutputlength = rte_pktmbuf_pkt_len(m_dst) -
+			op->dst.offset;
+	inst->s.out_ptr_ctl.s.length = inst->s.totaloutputlength;
+}
+
+static inline void
+zipvf_prepare_cmd_stateless(struct rte_comp_op *op, struct zip_stream *zstrm)
+{
+	union zip_inst_s *inst = zstrm->inst;
+
+	/* set flush flag to always 1*/
+	inst->s.ef = 1;
+
+	if (inst->s.op == ZIP_OP_E_DECOMP)
+		inst->s.sf = 1;
+	else
+		inst->s.sf = 0;
+
+	/* Set input checksum */
+	inst->s.adlercrc32 = op->input_chksum;
+
+	/* Prepare gather buffers */
+	zipvf_prepare_in_buf(zstrm, op);
+	zipvf_prepare_out_buf(zstrm, op);
+}
+
+#ifdef ZIP_DBG
+static inline void
+zip_dump_instruction(void *inst)
+{
+	union zip_inst_s *cmd83 = (union zip_inst_s *)inst;
+	printf("####### START ########\n");
+	printf("doneint:%d totaloutputlength:%d\n", cmd83->s.doneint,
+		cmd83->s.totaloutputlength);
+	printf("exnum:%d iv:%d exbits:%d hmif:%d halg:%d\n", cmd83->s.exn,
+		cmd83->s.iv, cmd83->s.exbits, cmd83->s.hmif, cmd83->s.halg);
+	printf("flush:%d speed:%d cc:%d\n", cmd83->s.sf,
+		cmd83->s.ss, cmd83->s.cc);
+	printf("eof:%d bof:%d op:%d dscatter:%d dgather:%d hgather:%d\n",
+		cmd83->s.ef, cmd83->s.bf, cmd83->s.op, cmd83->s.ds,
+		cmd83->s.dg, cmd83->s.hg);
+	printf("historylength:%d adler32:%d\n", cmd83->s.historylength,
+		cmd83->s.adlercrc32);
+	printf("ctx_ptr.addr:0x%"PRIx64"\n", cmd83->s.ctx_ptr_addr.s.addr);
+	printf("ctx_ptr.len:%d\n", cmd83->s.ctx_ptr_ctl.s.length);
+	printf("history_ptr.addr:0x%"PRIx64"\n", cmd83->s.his_ptr_addr.s.addr);
+	printf("history_ptr.len:%d\n", cmd83->s.his_ptr_ctl.s.length);
+	printf("inp_ptr.addr:0x%"PRIx64"\n", cmd83->s.inp_ptr_addr.s.addr);
+	printf("inp_ptr.len:%d\n", cmd83->s.inp_ptr_ctl.s.length);
+	printf("out_ptr.addr:0x%"PRIx64"\n", cmd83->s.out_ptr_addr.s.addr);
+	printf("out_ptr.len:%d\n", cmd83->s.out_ptr_ctl.s.length);
+	printf("result_ptr.len:%d\n", cmd83->s.res_ptr_ctl.s.length);
+	printf("####### END ########\n");
+}
+#endif
+
+int
+zipvf_create(struct rte_compressdev *compressdev);
+
+int
+zipvf_destroy(struct rte_compressdev *compressdev);
+
+int
+zipvf_q_init(struct zipvf_qp *qp);
+
+int
+zipvf_q_term(struct zipvf_qp *qp);
+
+void
+zipvf_push_command(struct zipvf_qp *qp, union zip_inst_s *zcmd);
+
+int
+zip_process_op(struct rte_comp_op *op,
+				struct zipvf_qp *qp,
+				struct zip_stream *zstrm);
+
+uint64_t
+zip_reg_read64(uint8_t *hw_addr, uint64_t offset);
+
+void
+zip_reg_write64(uint8_t *hw_addr, uint64_t offset, uint64_t val);
+
+#endif /* _RTE_ZIP_VF_H_ */
diff --git a/drivers/compress/octeontx/otx_zip_pmd.c b/drivers/compress/octeontx/otx_zip_pmd.c
new file mode 100644
index 00000000..9d13f933
--- /dev/null
+++ b/drivers/compress/octeontx/otx_zip_pmd.c
@@ -0,0 +1,658 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#include <string.h>
+
+#include <rte_byteorder.h>
+#include <rte_common.h>
+#include <rte_cpuflags.h>
+#include <rte_malloc.h>
+
+#include "otx_zip.h"
+
+static const struct rte_compressdev_capabilities
+				octtx_zip_pmd_capabilities[] = {
+	{	.algo = RTE_COMP_ALGO_DEFLATE,
+		/* Deflate */
+		.comp_feature_flags =	RTE_COMP_FF_HUFFMAN_FIXED |
+					RTE_COMP_FF_HUFFMAN_DYNAMIC,
+		/* Non sharable Priv XFORM and Stateless */
+		.window_size = {
+				.min = 1,
+				.max = 14,
+				.increment = 1
+				/* size supported 2^1 to 2^14 */
+		},
+	},
+	RTE_COMP_END_OF_CAPABILITIES_LIST()
+};
+
+/*
+ * Reset session to default state for next set of stateless operation
+ */
+static inline void
+reset_stream(struct zip_stream *z_stream)
+{
+	union zip_inst_s *inst = (union zip_inst_s *)(z_stream->inst);
+
+	inst->s.bf = 1;
+	inst->s.ef = 0;
+}
+
+int
+zip_process_op(struct rte_comp_op *op,
+		struct zipvf_qp *qp,
+		struct zip_stream *zstrm)
+{
+	union zip_inst_s *inst = zstrm->inst;
+	volatile union zip_zres_s *zresult = NULL;
+
+
+	if ((op->m_src->nb_segs > 1) || (op->m_dst->nb_segs > 1) ||
+			(op->src.offset > rte_pktmbuf_pkt_len(op->m_src)) ||
+			(op->dst.offset > rte_pktmbuf_pkt_len(op->m_dst))) {
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		ZIP_PMD_ERR("Segmented packet is not supported\n");
+		return 0;
+	}
+
+	zipvf_prepare_cmd_stateless(op, zstrm);
+
+	zresult = (union zip_zres_s *)zstrm->bufs[RES_BUF];
+	zresult->s.compcode = 0;
+
+#ifdef ZIP_DBG
+	zip_dump_instruction(inst);
+#endif
+
+	/* Submit zip command */
+	zipvf_push_command(qp, (void *)inst);
+
+	/* Check and Process results in sync mode */
+	do {
+	} while (!zresult->s.compcode);
+
+	if (zresult->s.compcode == ZIP_COMP_E_SUCCESS) {
+		op->status = RTE_COMP_OP_STATUS_SUCCESS;
+	} else {
+		/* FATAL error cannot do anything */
+		ZIP_PMD_ERR("operation failed with error code:%d\n",
+			zresult->s.compcode);
+		if (zresult->s.compcode == ZIP_COMP_E_DSTOP)
+			op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+		else
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+	}
+
+	ZIP_PMD_INFO("written %d\n", zresult->s.totalbyteswritten);
+
+	/* Update op stats */
+	switch (op->status) {
+	case RTE_COMP_OP_STATUS_SUCCESS:
+		op->consumed = zresult->s.totalbytesread;
+	/* Fall-through */
+	case RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED:
+		op->produced = zresult->s.totalbyteswritten;
+		break;
+	default:
+		ZIP_PMD_ERR("stats not updated for status:%d\n",
+				op->status);
+		break;
+	}
+	/* zstream is reset irrespective of result */
+	reset_stream(zstrm);
+
+	zresult->s.compcode = ZIP_COMP_E_NOTDONE;
+	return 0;
+}
+
+/** Parse xform parameters and setup a stream */
+static int
+zip_set_stream_parameters(struct rte_compressdev *dev,
+			const struct rte_comp_xform *xform,
+			struct zip_stream *z_stream)
+{
+	int ret;
+	union zip_inst_s *inst;
+	struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private;
+	void *res;
+
+	/* Allocate resources required by a stream */
+	ret = rte_mempool_get_bulk(vf->zip_mp,
+			z_stream->bufs, MAX_BUFS_PER_STREAM);
+	if (ret < 0)
+		return -1;
+
+	/* get one command buffer from pool and set up */
+	inst = (union zip_inst_s *)z_stream->bufs[CMD_BUF];
+	res = z_stream->bufs[RES_BUF];
+
+	memset(inst->u, 0, sizeof(inst->u));
+
+	/* set bf for only first ops of stream */
+	inst->s.bf = 1;
+
+	if (xform->type == RTE_COMP_COMPRESS) {
+		inst->s.op = ZIP_OP_E_COMP;
+
+		switch (xform->compress.deflate.huffman) {
+		case RTE_COMP_HUFFMAN_DEFAULT:
+			inst->s.cc = ZIP_CC_DEFAULT;
+			break;
+		case RTE_COMP_HUFFMAN_FIXED:
+			inst->s.cc = ZIP_CC_FIXED_HUFF;
+			break;
+		case RTE_COMP_HUFFMAN_DYNAMIC:
+			inst->s.cc = ZIP_CC_DYN_HUFF;
+			break;
+		default:
+			ret = -1;
+			goto err;
+		}
+
+		switch (xform->compress.level) {
+		case RTE_COMP_LEVEL_MIN:
+			inst->s.ss = ZIP_COMP_E_LEVEL_MIN;
+			break;
+		case RTE_COMP_LEVEL_MAX:
+			inst->s.ss = ZIP_COMP_E_LEVEL_MAX;
+			break;
+		case RTE_COMP_LEVEL_NONE:
+			ZIP_PMD_ERR("Compression level not supported");
+			ret = -1;
+			goto err;
+		default:
+			/* for any value between min and max , choose
+			 * PMD default.
+			 */
+			inst->s.ss = ZIP_COMP_E_LEVEL_MED; /** PMD default **/
+			break;
+		}
+	} else if (xform->type == RTE_COMP_DECOMPRESS) {
+		inst->s.op = ZIP_OP_E_DECOMP;
+		/* from HRM,
+		 * For DEFLATE decompression, [CC] must be 0x0.
+		 * For decompression, [SS] must be 0x0
+		 */
+		inst->s.cc = 0;
+		/* Speed bit should not be set for decompression */
+		inst->s.ss = 0;
+		/* decompression context is supported only for STATEFUL
+		 * operations. Currently we support STATELESS ONLY so
+		 * skip setting of ctx pointer
+		 */
+
+	} else {
+		ZIP_PMD_ERR("\nxform type not supported");
+		ret = -1;
+		goto err;
+	}
+
+	inst->s.res_ptr_addr.s.addr = rte_mempool_virt2iova(res);
+	inst->s.res_ptr_ctl.s.length = 0;
+
+	z_stream->inst = inst;
+	z_stream->func = zip_process_op;
+
+	return 0;
+
+err:
+	rte_mempool_put_bulk(vf->zip_mp,
+			     (void *)&(z_stream->bufs[0]),
+			     MAX_BUFS_PER_STREAM);
+
+	return ret;
+}
+
+/** Configure device */
+static int
+zip_pmd_config(struct rte_compressdev *dev,
+		struct rte_compressdev_config *config)
+{
+	int nb_streams;
+	char res_pool[RTE_MEMZONE_NAMESIZE];
+	struct zip_vf *vf;
+	struct rte_mempool *zip_buf_mp;
+
+	if (!config || !dev)
+		return -EIO;
+
+	vf = (struct zip_vf *)(dev->data->dev_private);
+
+	/* create pool with maximum numbers of resources
+	 * required by streams
+	 */
+
+	/* use common pool for non-shareable priv_xform and stream */
+	nb_streams = config->max_nb_priv_xforms + config->max_nb_streams;
+
+	snprintf(res_pool, RTE_MEMZONE_NAMESIZE, "octtx_zip_res_pool%u",
+		 dev->data->dev_id);
+
+	/** TBD Should we use the per core object cache for stream resources */
+	zip_buf_mp = rte_mempool_create(
+			res_pool,
+			nb_streams * MAX_BUFS_PER_STREAM,
+			ZIP_BUF_SIZE,
+			0,
+			0,
+			NULL,
+			NULL,
+			NULL,
+			NULL,
+			SOCKET_ID_ANY,
+			0);
+
+	if (zip_buf_mp == NULL) {
+		ZIP_PMD_ERR(
+			"Failed to create buf mempool octtx_zip_res_pool%u",
+			dev->data->dev_id);
+		return -1;
+	}
+
+	vf->zip_mp = zip_buf_mp;
+
+	return 0;
+}
+
+/** Start device */
+static int
+zip_pmd_start(__rte_unused struct rte_compressdev *dev)
+{
+	return 0;
+}
+
+/** Stop device */
+static void
+zip_pmd_stop(__rte_unused struct rte_compressdev *dev)
+{
+
+}
+
+/** Close device */
+static int
+zip_pmd_close(struct rte_compressdev *dev)
+{
+	if (dev == NULL)
+		return -1;
+
+	struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private;
+	rte_mempool_free(vf->zip_mp);
+
+	return 0;
+}
+
+/** Get device statistics */
+static void
+zip_pmd_stats_get(struct rte_compressdev *dev,
+		struct rte_compressdev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct zipvf_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->qp_stats.enqueued_count;
+		stats->dequeued_count += qp->qp_stats.dequeued_count;
+
+		stats->enqueue_err_count += qp->qp_stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->qp_stats.dequeue_err_count;
+	}
+}
+
+/** Reset device statistics */
+static void
+zip_pmd_stats_reset(struct rte_compressdev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct zipvf_qp *qp = dev->data->queue_pairs[qp_id];
+		memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
+	}
+}
+
+/** Get device info */
+static void
+zip_pmd_info_get(struct rte_compressdev *dev,
+		struct rte_compressdev_info *dev_info)
+{
+	struct zip_vf *vf = (struct zip_vf *)dev->data->dev_private;
+
+	if (dev_info != NULL) {
+		dev_info->driver_name = dev->device->driver->name;
+		dev_info->feature_flags = dev->feature_flags;
+		dev_info->capabilities = octtx_zip_pmd_capabilities;
+		dev_info->max_nb_queue_pairs = vf->max_nb_queue_pairs;
+	}
+}
+
+/** Release queue pair */
+static int
+zip_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
+{
+	struct zipvf_qp *qp = dev->data->queue_pairs[qp_id];
+
+	if (qp != NULL) {
+		zipvf_q_term(qp);
+
+		if (qp->processed_pkts)
+			rte_ring_free(qp->processed_pkts);
+
+		rte_free(qp);
+		dev->data->queue_pairs[qp_id] = NULL;
+	}
+	return 0;
+}
+
+/** Create a ring to place process packets on */
+static struct rte_ring *
+zip_pmd_qp_create_processed_pkts_ring(struct zipvf_qp *qp,
+		unsigned int ring_size, int socket_id)
+{
+	struct rte_ring *r;
+
+	r = rte_ring_lookup(qp->name);
+	if (r) {
+		if (rte_ring_get_size(r) >= ring_size) {
+			ZIP_PMD_INFO("Reusing existing ring %s for processed"
+					" packets", qp->name);
+			return r;
+		}
+
+		ZIP_PMD_ERR("Unable to reuse existing ring %s for processed"
+				" packets", qp->name);
+		return NULL;
+	}
+
+	return rte_ring_create(qp->name, ring_size, socket_id,
+						RING_F_EXACT_SZ);
+}
+
+/** Setup a queue pair */
+static int
+zip_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
+		uint32_t max_inflight_ops, int socket_id)
+{
+	struct zipvf_qp *qp = NULL;
+	struct zip_vf *vf;
+	char *name;
+	int ret;
+
+	if (!dev)
+		return -1;
+
+	vf = (struct zip_vf *) (dev->data->dev_private);
+
+	/* Free memory prior to re-allocation if needed. */
+	if (dev->data->queue_pairs[qp_id] != NULL) {
+		ZIP_PMD_INFO("Using existing queue pair %d ", qp_id);
+		return 0;
+	}
+
+	name =  rte_malloc(NULL, RTE_COMPRESSDEV_NAME_MAX_LEN, 0);
+	snprintf(name, RTE_COMPRESSDEV_NAME_MAX_LEN,
+		 "zip_pmd_%u_qp_%u",
+		 dev->data->dev_id, qp_id);
+
+	/* Allocate the queue pair data structure. */
+	qp = rte_zmalloc_socket(name, sizeof(*qp),
+				RTE_CACHE_LINE_SIZE, socket_id);
+	if (qp == NULL)
+		return (-ENOMEM);
+
+	qp->name = name;
+
+	/* Create completion queue upto max_inflight_ops */
+	qp->processed_pkts = zip_pmd_qp_create_processed_pkts_ring(qp,
+						max_inflight_ops, socket_id);
+	if (qp->processed_pkts == NULL)
+		goto qp_setup_cleanup;
+
+	qp->id = qp_id;
+	qp->vf = vf;
+
+	ret = zipvf_q_init(qp);
+	if (ret < 0)
+		goto qp_setup_cleanup;
+
+	dev->data->queue_pairs[qp_id] = qp;
+
+	memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
+	return 0;
+
+qp_setup_cleanup:
+	if (qp->processed_pkts)
+		rte_ring_free(qp->processed_pkts);
+	if (qp)
+		rte_free(qp);
+	return -1;
+}
+
+static int
+zip_pmd_stream_create(struct rte_compressdev *dev,
+		const struct rte_comp_xform *xform, void **stream)
+{
+	int ret;
+	struct zip_stream *strm = NULL;
+
+	strm = rte_malloc(NULL,
+			sizeof(struct zip_stream), 0);
+
+	if (strm == NULL)
+		return (-ENOMEM);
+
+	ret = zip_set_stream_parameters(dev, xform, strm);
+	if (ret < 0) {
+		ZIP_PMD_ERR("failed configure xform parameters");
+		rte_free(strm);
+		return ret;
+	}
+	*stream = strm;
+	return 0;
+}
+
+static int
+zip_pmd_stream_free(struct rte_compressdev *dev, void *stream)
+{
+	struct zip_vf *vf = (struct zip_vf *) (dev->data->dev_private);
+	struct zip_stream *z_stream;
+
+	if (stream == NULL)
+		return 0;
+
+	z_stream = (struct zip_stream *)stream;
+
+	/* Free resources back to pool */
+	rte_mempool_put_bulk(vf->zip_mp,
+				(void *)&(z_stream->bufs[0]),
+				MAX_BUFS_PER_STREAM);
+
+	/* Zero out the whole structure */
+	memset(stream, 0, sizeof(struct zip_stream));
+	rte_free(stream);
+
+	return 0;
+}
+
+
+static uint16_t
+zip_pmd_enqueue_burst_sync(void *queue_pair,
+		struct rte_comp_op **ops, uint16_t nb_ops)
+{
+	struct zipvf_qp *qp = queue_pair;
+	struct rte_comp_op *op;
+	struct zip_stream *zstrm;
+	int i, ret = 0;
+	uint16_t enqd = 0;
+
+	for (i = 0; i < nb_ops; i++) {
+		op = ops[i];
+
+		if (op->op_type == RTE_COMP_OP_STATEFUL) {
+			op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		} else {
+			/* process stateless ops */
+			zstrm = (struct zip_stream *)op->private_xform;
+			if (unlikely(zstrm == NULL))
+				op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+			else
+				ret = zstrm->func(op, qp, zstrm);
+		}
+
+		/* Whatever is out of op, put it into completion queue with
+		 * its status
+		 */
+		if (!ret)
+			ret = rte_ring_enqueue(qp->processed_pkts, (void *)op);
+
+		if (unlikely(ret < 0)) {
+			/* increment count if failed to enqueue op*/
+			qp->qp_stats.enqueue_err_count++;
+		} else {
+			qp->qp_stats.enqueued_count++;
+			enqd++;
+		}
+	}
+	return enqd;
+}
+
+static uint16_t
+zip_pmd_dequeue_burst_sync(void *queue_pair,
+		struct rte_comp_op **ops, uint16_t nb_ops)
+{
+	struct zipvf_qp *qp = queue_pair;
+
+	unsigned int nb_dequeued = 0;
+
+	nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts,
+			(void **)ops, nb_ops, NULL);
+	qp->qp_stats.dequeued_count += nb_dequeued;
+
+	return nb_dequeued;
+}
+
+struct rte_compressdev_ops octtx_zip_pmd_ops = {
+		.dev_configure		= zip_pmd_config,
+		.dev_start		= zip_pmd_start,
+		.dev_stop		= zip_pmd_stop,
+		.dev_close		= zip_pmd_close,
+
+		.stats_get		= zip_pmd_stats_get,
+		.stats_reset		= zip_pmd_stats_reset,
+
+		.dev_infos_get		= zip_pmd_info_get,
+
+		.queue_pair_setup	= zip_pmd_qp_setup,
+		.queue_pair_release	= zip_pmd_qp_release,
+
+		.private_xform_create	= zip_pmd_stream_create,
+		.private_xform_free	= zip_pmd_stream_free,
+		.stream_create		= NULL,
+		.stream_free		= NULL
+};
+
+static int
+zip_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+	struct rte_pci_device *pci_dev)
+{
+	int ret = 0;
+	char compressdev_name[RTE_COMPRESSDEV_NAME_MAX_LEN];
+	struct rte_compressdev *compressdev;
+	struct rte_compressdev_pmd_init_params init_params = {
+		"",
+		rte_socket_id(),
+	};
+
+	ZIP_PMD_INFO("vendor_id=0x%x device_id=0x%x",
+			(unsigned int)pci_dev->id.vendor_id,
+			(unsigned int)pci_dev->id.device_id);
+
+	rte_pci_device_name(&pci_dev->addr, compressdev_name,
+			    sizeof(compressdev_name));
+
+	compressdev = rte_compressdev_pmd_create(compressdev_name,
+		&pci_dev->device, sizeof(struct zip_vf), &init_params);
+	if (compressdev == NULL) {
+		ZIP_PMD_ERR("driver %s: create failed", init_params.name);
+		return -ENODEV;
+	}
+
+	/*
+	 * create only if proc_type is primary.
+	 */
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		/*  create vf dev with given pmd dev id */
+		ret = zipvf_create(compressdev);
+		if (ret < 0) {
+			ZIP_PMD_ERR("Device creation failed");
+			rte_compressdev_pmd_destroy(compressdev);
+			return ret;
+		}
+	}
+
+	compressdev->dev_ops = &octtx_zip_pmd_ops;
+	/* register rx/tx burst functions for data path */
+	compressdev->dequeue_burst = zip_pmd_dequeue_burst_sync;
+	compressdev->enqueue_burst = zip_pmd_enqueue_burst_sync;
+	compressdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED;
+	return ret;
+}
+
+static int
+zip_pci_remove(struct rte_pci_device *pci_dev)
+{
+	struct rte_compressdev *compressdev;
+	char compressdev_name[RTE_COMPRESSDEV_NAME_MAX_LEN];
+
+	if (pci_dev == NULL) {
+		ZIP_PMD_ERR(" Invalid PCI Device\n");
+		return -EINVAL;
+	}
+	rte_pci_device_name(&pci_dev->addr, compressdev_name,
+			sizeof(compressdev_name));
+
+	compressdev = rte_compressdev_pmd_get_named_dev(compressdev_name);
+	if (compressdev == NULL)
+		return -ENODEV;
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		if (zipvf_destroy(compressdev) < 0)
+			return -ENODEV;
+	}
+	return rte_compressdev_pmd_destroy(compressdev);
+}
+
+static struct rte_pci_id pci_id_octtx_zipvf_table[] = {
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
+			PCI_DEVICE_ID_OCTEONTX_ZIPVF),
+	},
+	{
+		.device_id = 0
+	},
+};
+
+/**
+ * Structure that represents a PCI driver
+ */
+static struct rte_pci_driver octtx_zip_pmd = {
+	.id_table    = pci_id_octtx_zipvf_table,
+	.drv_flags   = RTE_PCI_DRV_NEED_MAPPING,
+	.probe       = zip_pci_probe,
+	.remove      = zip_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(COMPRESSDEV_NAME_ZIP_PMD, octtx_zip_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(COMPRESSDEV_NAME_ZIP_PMD, pci_id_octtx_zipvf_table);
+
+RTE_INIT(octtx_zip_init_log);
+
+static void
+octtx_zip_init_log(void)
+{
+	octtx_zip_logtype_driver = rte_log_register("pmd.compress.octeontx");
+	if (octtx_zip_logtype_driver >= 0)
+		rte_log_set_level(octtx_zip_logtype_driver, RTE_LOG_INFO);
+}
diff --git a/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map b/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map
new file mode 100644
index 00000000..ad6e191e
--- /dev/null
+++ b/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map
@@ -0,0 +1,3 @@
+DPDK_18.08 {
+	local: *;
+};
diff --git a/drivers/compress/qat/meson.build b/drivers/compress/qat/meson.build
new file mode 100644
index 00000000..9d15076d
--- /dev/null
+++ b/drivers/compress/qat/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2017-2018 Intel Corporation
+
+
+# Add our sources files to the list
+allow_experimental_apis = true
+qat_sources += files('qat_comp_pmd.c',
+		     'qat_comp.c')
+qat_includes += include_directories('.')
+qat_deps += 'compressdev'
+qat_ext_deps += dep
+
+# build the whole driver
+sources += qat_sources
+cflags += qat_cflags
+deps += qat_deps
+ext_deps += qat_ext_deps
+includes += qat_includes
diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c
new file mode 100644
index 00000000..38c8a5b8
--- /dev/null
+++ b/drivers/compress/qat/qat_comp.c
@@ -0,0 +1,393 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_hexdump.h>
+#include <rte_comp.h>
+#include <rte_bus_pci.h>
+#include <rte_byteorder.h>
+#include <rte_memcpy.h>
+#include <rte_common.h>
+#include <rte_spinlock.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+
+#include "qat_logs.h"
+#include "qat_comp.h"
+#include "qat_comp_pmd.h"
+
+
+int
+qat_comp_build_request(void *in_op, uint8_t *out_msg,
+		       void *op_cookie,
+		       enum qat_device_gen qat_dev_gen __rte_unused)
+{
+	struct rte_comp_op *op = in_op;
+	struct qat_comp_op_cookie *cookie =
+			(struct qat_comp_op_cookie *)op_cookie;
+	struct qat_comp_xform *qat_xform = op->private_xform;
+	const uint8_t *tmpl = (uint8_t *)&qat_xform->qat_comp_req_tmpl;
+	struct icp_qat_fw_comp_req *comp_req =
+	    (struct icp_qat_fw_comp_req *)out_msg;
+
+	if (unlikely(op->op_type != RTE_COMP_OP_STATELESS)) {
+		QAT_DP_LOG(ERR, "QAT PMD only supports stateless compression "
+				"operation requests, op (%p) is not a "
+				"stateless operation.", op);
+		return -EINVAL;
+	}
+
+	rte_mov128(out_msg, tmpl);
+	comp_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;
+
+	/* common for sgl and flat buffers */
+	comp_req->comp_pars.comp_len = op->src.length;
+	comp_req->comp_pars.out_buffer_sz = rte_pktmbuf_pkt_len(op->m_dst) -
+			op->dst.offset;
+
+	if (op->m_src->next != NULL || op->m_dst->next != NULL) {
+		/* sgl */
+		int ret = 0;
+
+		ICP_QAT_FW_COMN_PTR_TYPE_SET(comp_req->comn_hdr.comn_req_flags,
+				QAT_COMN_PTR_TYPE_SGL);
+
+		ret = qat_sgl_fill_array(op->m_src,
+				op->src.offset,
+				&cookie->qat_sgl_src,
+				op->src.length,
+				RTE_PMD_QAT_COMP_SGL_MAX_SEGMENTS);
+		if (ret) {
+			QAT_DP_LOG(ERR, "QAT PMD Cannot fill source sgl array");
+			return ret;
+		}
+
+		ret = qat_sgl_fill_array(op->m_dst,
+				op->dst.offset,
+				&cookie->qat_sgl_dst,
+				comp_req->comp_pars.out_buffer_sz,
+				RTE_PMD_QAT_COMP_SGL_MAX_SEGMENTS);
+		if (ret) {
+			QAT_DP_LOG(ERR, "QAT PMD Cannot fill dest. sgl array");
+			return ret;
+		}
+
+		comp_req->comn_mid.src_data_addr =
+				cookie->qat_sgl_src_phys_addr;
+		comp_req->comn_mid.dest_data_addr =
+				cookie->qat_sgl_dst_phys_addr;
+		comp_req->comn_mid.src_length = 0;
+		comp_req->comn_mid.dst_length = 0;
+
+	} else {
+		/* flat aka linear buffer */
+		ICP_QAT_FW_COMN_PTR_TYPE_SET(comp_req->comn_hdr.comn_req_flags,
+				QAT_COMN_PTR_TYPE_FLAT);
+		comp_req->comn_mid.src_length = op->src.length;
+		comp_req->comn_mid.dst_length =
+				comp_req->comp_pars.out_buffer_sz;
+
+		comp_req->comn_mid.src_data_addr =
+		    rte_pktmbuf_mtophys_offset(op->m_src, op->src.offset);
+		comp_req->comn_mid.dest_data_addr =
+		    rte_pktmbuf_mtophys_offset(op->m_dst, op->dst.offset);
+	}
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+	QAT_DP_LOG(DEBUG, "Direction: %s",
+	    qat_xform->qat_comp_request_type == QAT_COMP_REQUEST_DECOMPRESS ?
+			    "decompression" : "compression");
+	QAT_DP_HEXDUMP_LOG(DEBUG, "qat compression message:", comp_req,
+		    sizeof(struct icp_qat_fw_comp_req));
+#endif
+	return 0;
+}
+
+int
+qat_comp_process_response(void **op, uint8_t *resp)
+{
+	struct icp_qat_fw_comp_resp *resp_msg =
+			(struct icp_qat_fw_comp_resp *)resp;
+	struct rte_comp_op *rx_op = (struct rte_comp_op *)(uintptr_t)
+			(resp_msg->opaque_data);
+	struct qat_comp_xform *qat_xform = (struct qat_comp_xform *)
+				(rx_op->private_xform);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+	QAT_DP_LOG(DEBUG, "Direction: %s",
+	    qat_xform->qat_comp_request_type == QAT_COMP_REQUEST_DECOMPRESS ?
+	    "decompression" : "compression");
+	QAT_DP_HEXDUMP_LOG(DEBUG,  "qat_response:", (uint8_t *)resp_msg,
+			sizeof(struct icp_qat_fw_comp_resp));
+#endif
+
+	if (likely(qat_xform->qat_comp_request_type
+			!= QAT_COMP_REQUEST_DECOMPRESS)) {
+		if (unlikely(ICP_QAT_FW_COMN_HDR_CNV_FLAG_GET(
+				resp_msg->comn_resp.hdr_flags)
+					== ICP_QAT_FW_COMP_NO_CNV)) {
+			rx_op->status = RTE_COMP_OP_STATUS_ERROR;
+			rx_op->debug_status = ERR_CODE_QAT_COMP_WRONG_FW;
+			*op = (void *)rx_op;
+			QAT_DP_LOG(ERR, "QAT has wrong firmware");
+			return 0;
+		}
+	}
+
+	if ((ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(resp_msg->comn_resp.comn_status)
+		| ICP_QAT_FW_COMN_RESP_XLAT_STAT_GET(
+				resp_msg->comn_resp.comn_status)) !=
+				ICP_QAT_FW_COMN_STATUS_FLAG_OK) {
+
+		rx_op->status = RTE_COMP_OP_STATUS_ERROR;
+		rx_op->debug_status =
+			*((uint16_t *)(&resp_msg->comn_resp.comn_error));
+	} else {
+		struct qat_comp_xform *qat_xform = rx_op->private_xform;
+		struct icp_qat_fw_resp_comp_pars *comp_resp =
+		  (struct icp_qat_fw_resp_comp_pars *)&resp_msg->comp_resp_pars;
+
+		rx_op->status = RTE_COMP_OP_STATUS_SUCCESS;
+		rx_op->consumed = comp_resp->input_byte_counter;
+		rx_op->produced = comp_resp->output_byte_counter;
+
+		if (qat_xform->checksum_type != RTE_COMP_CHECKSUM_NONE) {
+			if (qat_xform->checksum_type == RTE_COMP_CHECKSUM_CRC32)
+				rx_op->output_chksum = comp_resp->curr_crc32;
+			else if (qat_xform->checksum_type ==
+					RTE_COMP_CHECKSUM_ADLER32)
+				rx_op->output_chksum = comp_resp->curr_adler_32;
+			else
+				rx_op->output_chksum = comp_resp->curr_chksum;
+		}
+	}
+	*op = (void *)rx_op;
+
+	return 0;
+}
+
+unsigned int
+qat_comp_xform_size(void)
+{
+	return RTE_ALIGN_CEIL(sizeof(struct qat_comp_xform), 8);
+}
+
+static void qat_comp_create_req_hdr(struct icp_qat_fw_comn_req_hdr *header,
+				    enum qat_comp_request_type request)
+{
+	if (request == QAT_COMP_REQUEST_FIXED_COMP_STATELESS)
+		header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC;
+	else if (request == QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS)
+		header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC;
+	else if (request == QAT_COMP_REQUEST_DECOMPRESS)
+		header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS;
+
+	header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_COMP;
+	header->hdr_flags =
+	    ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
+
+	header->comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(
+	    QAT_COMN_CD_FLD_TYPE_16BYTE_DATA, QAT_COMN_PTR_TYPE_FLAT);
+}
+
+static int qat_comp_create_templates(struct qat_comp_xform *qat_xform,
+			const struct rte_memzone *interm_buff_mz __rte_unused,
+			const struct rte_comp_xform *xform)
+{
+	struct icp_qat_fw_comp_req *comp_req;
+	int comp_level, algo;
+	uint32_t req_par_flags;
+	int direction = ICP_QAT_HW_COMPRESSION_DIR_COMPRESS;
+
+	if (unlikely(qat_xform == NULL)) {
+		QAT_LOG(ERR, "Session was not created for this device");
+		return -EINVAL;
+	}
+
+	if (qat_xform->qat_comp_request_type == QAT_COMP_REQUEST_DECOMPRESS) {
+		direction = ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS;
+		comp_level = ICP_QAT_HW_COMPRESSION_DEPTH_1;
+		req_par_flags = ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(
+				ICP_QAT_FW_COMP_SOP, ICP_QAT_FW_COMP_EOP,
+				ICP_QAT_FW_COMP_BFINAL, ICP_QAT_FW_COMP_NO_CNV,
+				ICP_QAT_FW_COMP_NO_CNV_RECOVERY);
+
+	} else {
+		if (xform->compress.level == RTE_COMP_LEVEL_PMD_DEFAULT)
+			comp_level = ICP_QAT_HW_COMPRESSION_DEPTH_8;
+		else if (xform->compress.level == 1)
+			comp_level = ICP_QAT_HW_COMPRESSION_DEPTH_1;
+		else if (xform->compress.level == 2)
+			comp_level = ICP_QAT_HW_COMPRESSION_DEPTH_4;
+		else if (xform->compress.level == 3)
+			comp_level = ICP_QAT_HW_COMPRESSION_DEPTH_8;
+		else if (xform->compress.level >= 4 &&
+			 xform->compress.level <= 9)
+			comp_level = ICP_QAT_HW_COMPRESSION_DEPTH_16;
+		else {
+			QAT_LOG(ERR, "compression level not supported");
+			return -EINVAL;
+		}
+		req_par_flags = ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(
+				ICP_QAT_FW_COMP_SOP, ICP_QAT_FW_COMP_EOP,
+				ICP_QAT_FW_COMP_BFINAL, ICP_QAT_FW_COMP_CNV,
+				ICP_QAT_FW_COMP_CNV_RECOVERY);
+	}
+
+	switch (xform->compress.algo) {
+	case RTE_COMP_ALGO_DEFLATE:
+		algo = ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE;
+		break;
+	case RTE_COMP_ALGO_LZS:
+	default:
+		/* RTE_COMP_NULL */
+		QAT_LOG(ERR, "compression algorithm not supported");
+		return -EINVAL;
+	}
+
+	comp_req = &qat_xform->qat_comp_req_tmpl;
+
+	/* Initialize header */
+	qat_comp_create_req_hdr(&comp_req->comn_hdr,
+					qat_xform->qat_comp_request_type);
+
+	comp_req->comn_hdr.serv_specif_flags = ICP_QAT_FW_COMP_FLAGS_BUILD(
+	    ICP_QAT_FW_COMP_STATELESS_SESSION,
+	    ICP_QAT_FW_COMP_NOT_AUTO_SELECT_BEST,
+	    ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST,
+	    ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST,
+	    ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_USED_AS_INTMD_BUF);
+
+	comp_req->cd_pars.sl.comp_slice_cfg_word[0] =
+	    ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(
+		direction,
+		/* In CPM 1.6 only valid mode ! */
+		ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED, algo,
+		/* Translate level to depth */
+		comp_level, ICP_QAT_HW_COMPRESSION_FILE_TYPE_0);
+
+	comp_req->comp_pars.initial_adler = 1;
+	comp_req->comp_pars.initial_crc32 = 0;
+	comp_req->comp_pars.req_par_flags = req_par_flags;
+
+
+	if (qat_xform->qat_comp_request_type ==
+			QAT_COMP_REQUEST_FIXED_COMP_STATELESS ||
+	    qat_xform->qat_comp_request_type == QAT_COMP_REQUEST_DECOMPRESS) {
+		ICP_QAT_FW_COMN_NEXT_ID_SET(&comp_req->comp_cd_ctrl,
+					    ICP_QAT_FW_SLICE_DRAM_WR);
+		ICP_QAT_FW_COMN_CURR_ID_SET(&comp_req->comp_cd_ctrl,
+					    ICP_QAT_FW_SLICE_COMP);
+	} else if (qat_xform->qat_comp_request_type ==
+		   QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS) {
+
+		QAT_LOG(ERR, "Dynamic huffman encoding not supported");
+		return -EINVAL;
+	}
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+	QAT_DP_HEXDUMP_LOG(DEBUG, "qat compression message template:", comp_req,
+		    sizeof(struct icp_qat_fw_comp_req));
+#endif
+	return 0;
+}
+
+/**
+ * Create driver private_xform data.
+ *
+ * @param dev
+ *   Compressdev device
+ * @param xform
+ *   xform data from application
+ * @param private_xform
+ *   ptr where handle of pmd's private_xform data should be stored
+ * @return
+ *  - if successful returns 0
+ *    and valid private_xform handle
+ *  - <0 in error cases
+ *  - Returns -EINVAL if input parameters are invalid.
+ *  - Returns -ENOTSUP if comp device does not support the comp transform.
+ *  - Returns -ENOMEM if the private_xform could not be allocated.
+ */
+int
+qat_comp_private_xform_create(struct rte_compressdev *dev,
+			      const struct rte_comp_xform *xform,
+			      void **private_xform)
+{
+	struct qat_comp_dev_private *qat = dev->data->dev_private;
+
+	if (unlikely(private_xform == NULL)) {
+		QAT_LOG(ERR, "QAT: private_xform parameter is NULL");
+		return -EINVAL;
+	}
+	if (unlikely(qat->xformpool == NULL)) {
+		QAT_LOG(ERR, "QAT device has no private_xform mempool");
+		return -ENOMEM;
+	}
+	if (rte_mempool_get(qat->xformpool, private_xform)) {
+		QAT_LOG(ERR, "Couldn't get object from qat xform mempool");
+		return -ENOMEM;
+	}
+
+	struct qat_comp_xform *qat_xform =
+			(struct qat_comp_xform *)*private_xform;
+
+	if (xform->type == RTE_COMP_COMPRESS) {
+		if (xform->compress.deflate.huffman ==
+				RTE_COMP_HUFFMAN_DYNAMIC) {
+			QAT_LOG(ERR,
+			"QAT device doesn't support dynamic compression");
+			return -ENOTSUP;
+		}
+
+		if (xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED ||
+		  ((xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_DEFAULT)
+				   && qat->interm_buff_mz == NULL))
+
+			qat_xform->qat_comp_request_type =
+					QAT_COMP_REQUEST_FIXED_COMP_STATELESS;
+
+
+	} else {
+		qat_xform->qat_comp_request_type = QAT_COMP_REQUEST_DECOMPRESS;
+	}
+
+	qat_xform->checksum_type = xform->compress.chksum;
+
+	if (qat_comp_create_templates(qat_xform, qat->interm_buff_mz, xform)) {
+		QAT_LOG(ERR, "QAT: Problem with setting compression");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+/**
+ * Free driver private_xform data.
+ *
+ * @param dev
+ *   Compressdev device
+ * @param private_xform
+ *   handle of pmd's private_xform data
+ * @return
+ *  - 0 if successful
+ *  - <0 in error cases
+ *  - Returns -EINVAL if input parameters are invalid.
+ */
+int
+qat_comp_private_xform_free(struct rte_compressdev *dev __rte_unused,
+			    void *private_xform)
+{
+	struct qat_comp_xform *qat_xform =
+			(struct qat_comp_xform *)private_xform;
+
+	if (qat_xform) {
+		memset(qat_xform, 0, qat_comp_xform_size());
+		struct rte_mempool *mp = rte_mempool_from_obj(qat_xform);
+
+		rte_mempool_put(mp, qat_xform);
+		return 0;
+	}
+	return -EINVAL;
+}
diff --git a/drivers/compress/qat/qat_comp.h b/drivers/compress/qat/qat_comp.h
new file mode 100644
index 00000000..8d315efb
--- /dev/null
+++ b/drivers/compress/qat/qat_comp.h
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2018 Intel Corporation
+ */
+
+#ifndef _QAT_COMP_H_
+#define _QAT_COMP_H_
+
+#ifdef RTE_LIBRTE_COMPRESSDEV
+
+#include <rte_compressdev.h>
+#include <rte_compressdev_pmd.h>
+
+#include "qat_common.h"
+#include "icp_qat_hw.h"
+#include "icp_qat_fw_comp.h"
+#include "icp_qat_fw_la.h"
+
+#define ERR_CODE_QAT_COMP_WRONG_FW -99
+
+enum qat_comp_request_type {
+	QAT_COMP_REQUEST_FIXED_COMP_STATELESS,
+	QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS,
+	QAT_COMP_REQUEST_DECOMPRESS,
+	REQ_COMP_END
+};
+
+struct qat_comp_sgl {
+	qat_sgl_hdr;
+	struct qat_flat_buf buffers[RTE_PMD_QAT_COMP_SGL_MAX_SEGMENTS];
+} __rte_packed __rte_cache_aligned;
+
+struct qat_comp_op_cookie {
+	struct qat_comp_sgl qat_sgl_src;
+	struct qat_comp_sgl qat_sgl_dst;
+	phys_addr_t qat_sgl_src_phys_addr;
+	phys_addr_t qat_sgl_dst_phys_addr;
+};
+
+struct qat_comp_xform {
+	struct icp_qat_fw_comp_req qat_comp_req_tmpl;
+	enum qat_comp_request_type qat_comp_request_type;
+	enum rte_comp_checksum_type checksum_type;
+};
+
+int
+qat_comp_build_request(void *in_op, uint8_t *out_msg, void *op_cookie,
+		       enum qat_device_gen qat_dev_gen __rte_unused);
+
+int
+qat_comp_process_response(void **op, uint8_t *resp);
+
+
+int
+qat_comp_private_xform_create(struct rte_compressdev *dev,
+			      const struct rte_comp_xform *xform,
+			      void **private_xform);
+
+int
+qat_comp_private_xform_free(struct rte_compressdev *dev, void *private_xform);
+
+unsigned int
+qat_comp_xform_size(void);
+
+#endif
+#endif
diff --git a/drivers/compress/qat/qat_comp_pmd.c b/drivers/compress/qat/qat_comp_pmd.c
new file mode 100644
index 00000000..b89975fc
--- /dev/null
+++ b/drivers/compress/qat/qat_comp_pmd.c
@@ -0,0 +1,429 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2018 Intel Corporation
+ */
+
+#include "qat_comp.h"
+#include "qat_comp_pmd.h"
+
+static const struct rte_compressdev_capabilities qat_comp_gen_capabilities[] = {
+	{/* COMPRESSION - deflate */
+	 .algo = RTE_COMP_ALGO_DEFLATE,
+	 .comp_feature_flags = RTE_COMP_FF_MULTI_PKT_CHECKSUM |
+				RTE_COMP_FF_CRC32_CHECKSUM |
+				RTE_COMP_FF_ADLER32_CHECKSUM |
+				RTE_COMP_FF_CRC32_ADLER32_CHECKSUM |
+				RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
+				RTE_COMP_FF_HUFFMAN_FIXED |
+				RTE_COMP_FF_OOP_SGL_IN_SGL_OUT |
+				RTE_COMP_FF_OOP_SGL_IN_LB_OUT |
+				RTE_COMP_FF_OOP_LB_IN_SGL_OUT,
+	 .window_size = {.min = 15, .max = 15, .increment = 0} },
+	{RTE_COMP_ALGO_LIST_END, 0, {0, 0, 0} } };
+
+static void
+qat_comp_stats_get(struct rte_compressdev *dev,
+		struct rte_compressdev_stats *stats)
+{
+	struct qat_common_stats qat_stats = {0};
+	struct qat_comp_dev_private *qat_priv;
+
+	if (stats == NULL || dev == NULL) {
+		QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
+		return;
+	}
+	qat_priv = dev->data->dev_private;
+
+	qat_stats_get(qat_priv->qat_dev, &qat_stats, QAT_SERVICE_COMPRESSION);
+	stats->enqueued_count = qat_stats.enqueued_count;
+	stats->dequeued_count = qat_stats.dequeued_count;
+	stats->enqueue_err_count = qat_stats.enqueue_err_count;
+	stats->dequeue_err_count = qat_stats.dequeue_err_count;
+}
+
+static void
+qat_comp_stats_reset(struct rte_compressdev *dev)
+{
+	struct qat_comp_dev_private *qat_priv;
+
+	if (dev == NULL) {
+		QAT_LOG(ERR, "invalid compressdev ptr %p", dev);
+		return;
+	}
+	qat_priv = dev->data->dev_private;
+
+	qat_stats_reset(qat_priv->qat_dev, QAT_SERVICE_COMPRESSION);
+
+}
+
+static int
+qat_comp_qp_release(struct rte_compressdev *dev, uint16_t queue_pair_id)
+{
+	struct qat_comp_dev_private *qat_private = dev->data->dev_private;
+
+	QAT_LOG(DEBUG, "Release comp qp %u on device %d",
+				queue_pair_id, dev->data->dev_id);
+
+	qat_private->qat_dev->qps_in_use[QAT_SERVICE_COMPRESSION][queue_pair_id]
+						= NULL;
+
+	return qat_qp_release((struct qat_qp **)
+			&(dev->data->queue_pairs[queue_pair_id]));
+}
+
+static int
+qat_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
+		  uint32_t max_inflight_ops, int socket_id)
+{
+	struct qat_qp *qp;
+	int ret = 0;
+	uint32_t i;
+	struct qat_qp_config qat_qp_conf;
+
+	struct qat_qp **qp_addr =
+			(struct qat_qp **)&(dev->data->queue_pairs[qp_id]);
+	struct qat_comp_dev_private *qat_private = dev->data->dev_private;
+	const struct qat_qp_hw_data *comp_hw_qps =
+			qat_gen_config[qat_private->qat_dev->qat_dev_gen]
+				      .qp_hw_data[QAT_SERVICE_COMPRESSION];
+	const struct qat_qp_hw_data *qp_hw_data = comp_hw_qps + qp_id;
+
+	/* If qp is already in use free ring memory and qp metadata. */
+	if (*qp_addr != NULL) {
+		ret = qat_comp_qp_release(dev, qp_id);
+		if (ret < 0)
+			return ret;
+	}
+	if (qp_id >= qat_qps_per_service(comp_hw_qps,
+					 QAT_SERVICE_COMPRESSION)) {
+		QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
+		return -EINVAL;
+	}
+
+	qat_qp_conf.hw = qp_hw_data;
+	qat_qp_conf.build_request = qat_comp_build_request;
+	qat_qp_conf.cookie_size = sizeof(struct qat_comp_op_cookie);
+	qat_qp_conf.nb_descriptors = max_inflight_ops;
+	qat_qp_conf.socket_id = socket_id;
+	qat_qp_conf.service_str = "comp";
+
+	ret = qat_qp_setup(qat_private->qat_dev, qp_addr, qp_id, &qat_qp_conf);
+	if (ret != 0)
+		return ret;
+
+	/* store a link to the qp in the qat_pci_device */
+	qat_private->qat_dev->qps_in_use[QAT_SERVICE_COMPRESSION][qp_id]
+							= *qp_addr;
+
+	qp = (struct qat_qp *)*qp_addr;
+
+	for (i = 0; i < qp->nb_descriptors; i++) {
+
+		struct qat_comp_op_cookie *cookie =
+				qp->op_cookies[i];
+
+		cookie->qat_sgl_src_phys_addr =
+				rte_mempool_virt2iova(cookie) +
+				offsetof(struct qat_comp_op_cookie,
+				qat_sgl_src);
+
+		cookie->qat_sgl_dst_phys_addr =
+				rte_mempool_virt2iova(cookie) +
+				offsetof(struct qat_comp_op_cookie,
+				qat_sgl_dst);
+	}
+
+	return ret;
+}
+
+static struct rte_mempool *
+qat_comp_create_xform_pool(struct qat_comp_dev_private *comp_dev,
+			      uint32_t num_elements)
+{
+	char xform_pool_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *mp;
+
+	snprintf(xform_pool_name, RTE_MEMPOOL_NAMESIZE,
+			"%s_xforms", comp_dev->qat_dev->name);
+
+	QAT_LOG(DEBUG, "xformpool: %s", xform_pool_name);
+	mp = rte_mempool_lookup(xform_pool_name);
+
+	if (mp != NULL) {
+		QAT_LOG(DEBUG, "xformpool already created");
+		if (mp->size != num_elements) {
+			QAT_LOG(DEBUG, "xformpool wrong size - delete it");
+			rte_mempool_free(mp);
+			mp = NULL;
+			comp_dev->xformpool = NULL;
+		}
+	}
+
+	if (mp == NULL)
+		mp = rte_mempool_create(xform_pool_name,
+				num_elements,
+				qat_comp_xform_size(), 0, 0,
+				NULL, NULL, NULL, NULL, rte_socket_id(),
+				0);
+	if (mp == NULL) {
+		QAT_LOG(ERR, "Err creating mempool %s w %d elements of size %d",
+			xform_pool_name, num_elements, qat_comp_xform_size());
+		return NULL;
+	}
+
+	return mp;
+}
+
+static void
+_qat_comp_dev_config_clear(struct qat_comp_dev_private *comp_dev)
+{
+	/* Free private_xform pool */
+	if (comp_dev->xformpool) {
+		/* Free internal mempool for private xforms */
+		rte_mempool_free(comp_dev->xformpool);
+		comp_dev->xformpool = NULL;
+	}
+}
+
+static int
+qat_comp_dev_config(struct rte_compressdev *dev,
+		struct rte_compressdev_config *config)
+{
+	struct qat_comp_dev_private *comp_dev = dev->data->dev_private;
+	int ret = 0;
+
+	if (config->max_nb_streams != 0) {
+		QAT_LOG(ERR,
+	"QAT device does not support STATEFUL so max_nb_streams must be 0");
+		return -EINVAL;
+	}
+
+	comp_dev->xformpool = qat_comp_create_xform_pool(comp_dev,
+					config->max_nb_priv_xforms);
+	if (comp_dev->xformpool == NULL) {
+
+		ret = -ENOMEM;
+		goto error_out;
+	}
+	return 0;
+
+error_out:
+	_qat_comp_dev_config_clear(comp_dev);
+	return ret;
+}
+
+static int
+qat_comp_dev_start(struct rte_compressdev *dev __rte_unused)
+{
+	return 0;
+}
+
+static void
+qat_comp_dev_stop(struct rte_compressdev *dev __rte_unused)
+{
+
+}
+
+static int
+qat_comp_dev_close(struct rte_compressdev *dev)
+{
+	int i;
+	int ret = 0;
+	struct qat_comp_dev_private *comp_dev = dev->data->dev_private;
+
+	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
+		ret = qat_comp_qp_release(dev, i);
+		if (ret < 0)
+			return ret;
+	}
+
+	_qat_comp_dev_config_clear(comp_dev);
+
+	return ret;
+}
+
+
+static void
+qat_comp_dev_info_get(struct rte_compressdev *dev,
+			struct rte_compressdev_info *info)
+{
+	struct qat_comp_dev_private *comp_dev = dev->data->dev_private;
+	const struct qat_qp_hw_data *comp_hw_qps =
+		qat_gen_config[comp_dev->qat_dev->qat_dev_gen]
+			      .qp_hw_data[QAT_SERVICE_COMPRESSION];
+
+	if (info != NULL) {
+		info->max_nb_queue_pairs =
+			qat_qps_per_service(comp_hw_qps,
+					    QAT_SERVICE_COMPRESSION);
+		info->feature_flags = dev->feature_flags;
+		info->capabilities = comp_dev->qat_dev_capabilities;
+	}
+}
+
+static uint16_t
+qat_comp_pmd_enqueue_op_burst(void *qp, struct rte_comp_op **ops,
+		uint16_t nb_ops)
+{
+	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+}
+
+static uint16_t
+qat_comp_pmd_dequeue_op_burst(void *qp, struct rte_comp_op **ops,
+			      uint16_t nb_ops)
+{
+	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+}
+
+static uint16_t
+qat_comp_pmd_enq_deq_dummy_op_burst(void *qp __rte_unused,
+				    struct rte_comp_op **ops __rte_unused,
+				    uint16_t nb_ops __rte_unused)
+{
+	QAT_DP_LOG(ERR, "QAT PMD detected wrong FW version !");
+	return 0;
+}
+
+static struct rte_compressdev_ops compress_qat_dummy_ops = {
+
+	/* Device related operations */
+	.dev_configure		= NULL,
+	.dev_start		= NULL,
+	.dev_stop		= qat_comp_dev_stop,
+	.dev_close		= qat_comp_dev_close,
+	.dev_infos_get		= NULL,
+
+	.stats_get		= NULL,
+	.stats_reset		= qat_comp_stats_reset,
+	.queue_pair_setup	= NULL,
+	.queue_pair_release	= qat_comp_qp_release,
+
+	/* Compression related operations */
+	.private_xform_create	= NULL,
+	.private_xform_free	= qat_comp_private_xform_free
+};
+
+static uint16_t
+qat_comp_pmd_dequeue_frst_op_burst(void *qp, struct rte_comp_op **ops,
+				   uint16_t nb_ops)
+{
+	uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
+
+	if (ret) {
+		if ((*ops)->debug_status ==
+				(uint64_t)ERR_CODE_QAT_COMP_WRONG_FW) {
+			tmp_qp->qat_dev->comp_dev->compressdev->enqueue_burst =
+					qat_comp_pmd_enq_deq_dummy_op_burst;
+			tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
+					qat_comp_pmd_enq_deq_dummy_op_burst;
+
+			tmp_qp->qat_dev->comp_dev->compressdev->dev_ops =
+					&compress_qat_dummy_ops;
+			QAT_LOG(ERR, "QAT PMD detected wrong FW version !");
+
+		} else {
+			tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
+					qat_comp_pmd_dequeue_op_burst;
+		}
+	}
+	return ret;
+}
+
+static struct rte_compressdev_ops compress_qat_ops = {
+
+	/* Device related operations */
+	.dev_configure		= qat_comp_dev_config,
+	.dev_start		= qat_comp_dev_start,
+	.dev_stop		= qat_comp_dev_stop,
+	.dev_close		= qat_comp_dev_close,
+	.dev_infos_get		= qat_comp_dev_info_get,
+
+	.stats_get		= qat_comp_stats_get,
+	.stats_reset		= qat_comp_stats_reset,
+	.queue_pair_setup	= qat_comp_qp_setup,
+	.queue_pair_release	= qat_comp_qp_release,
+
+	/* Compression related operations */
+	.private_xform_create	= qat_comp_private_xform_create,
+	.private_xform_free	= qat_comp_private_xform_free
+};
+
+int
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
+{
+	if (qat_pci_dev->qat_dev_gen == QAT_GEN1) {
+		QAT_LOG(ERR, "Compression PMD not supported on QAT dh895xcc");
+		return 0;
+	}
+
+	struct rte_compressdev_pmd_init_params init_params = {
+		.name = "",
+		.socket_id = qat_pci_dev->pci_dev->device.numa_node,
+	};
+	char name[RTE_COMPRESSDEV_NAME_MAX_LEN];
+	struct rte_compressdev *compressdev;
+	struct qat_comp_dev_private *comp_dev;
+
+	snprintf(name, RTE_COMPRESSDEV_NAME_MAX_LEN, "%s_%s",
+			qat_pci_dev->name, "comp");
+	QAT_LOG(DEBUG, "Creating QAT COMP device %s", name);
+
+	compressdev = rte_compressdev_pmd_create(name,
+			&qat_pci_dev->pci_dev->device,
+			sizeof(struct qat_comp_dev_private),
+			&init_params);
+
+	if (compressdev == NULL)
+		return -ENODEV;
+
+	compressdev->dev_ops = &compress_qat_ops;
+
+	compressdev->enqueue_burst = qat_comp_pmd_enqueue_op_burst;
+	compressdev->dequeue_burst = qat_comp_pmd_dequeue_frst_op_burst;
+
+	compressdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED;
+
+	comp_dev = compressdev->data->dev_private;
+	comp_dev->qat_dev = qat_pci_dev;
+	comp_dev->compressdev = compressdev;
+	qat_pci_dev->comp_dev = comp_dev;
+
+	switch (qat_pci_dev->qat_dev_gen) {
+	case QAT_GEN1:
+	case QAT_GEN2:
+		comp_dev->qat_dev_capabilities = qat_comp_gen_capabilities;
+		break;
+	default:
+		comp_dev->qat_dev_capabilities = qat_comp_gen_capabilities;
+		QAT_LOG(DEBUG,
+			"QAT gen %d capabilities unknown, default to GEN1",
+					qat_pci_dev->qat_dev_gen);
+		break;
+	}
+
+	QAT_LOG(DEBUG,
+		    "Created QAT COMP device %s as compressdev instance %d",
+			name, compressdev->data->dev_id);
+	return 0;
+}
+
+int
+qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev)
+{
+	struct qat_comp_dev_private *comp_dev;
+
+	if (qat_pci_dev == NULL)
+		return -ENODEV;
+
+	comp_dev = qat_pci_dev->comp_dev;
+	if (comp_dev == NULL)
+		return 0;
+
+	/* clean up any resources used by the device */
+	qat_comp_dev_close(comp_dev->compressdev);
+
+	rte_compressdev_pmd_destroy(comp_dev->compressdev);
+	qat_pci_dev->comp_dev = NULL;
+
+	return 0;
+}
diff --git a/drivers/compress/qat/qat_comp_pmd.h b/drivers/compress/qat/qat_comp_pmd.h
new file mode 100644
index 00000000..9ad2a283
--- /dev/null
+++ b/drivers/compress/qat/qat_comp_pmd.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2018 Intel Corporation
+ */
+
+#ifndef _QAT_COMP_PMD_H_
+#define _QAT_COMP_PMD_H_
+
+#ifdef RTE_LIBRTE_COMPRESSDEV
+
+#include <rte_compressdev.h>
+#include <rte_compressdev_pmd.h>
+
+#include "qat_device.h"
+
+/** private data structure for a QAT compression device.
+ * This QAT device is a device offering only a compression service,
+ * there can be one of these on each qat_pci_device (VF).
+ */
+struct qat_comp_dev_private {
+	struct qat_pci_device *qat_dev;
+	/**< The qat pci device hosting the service */
+	struct rte_compressdev *compressdev;
+	/**< The pointer to this compression device structure */
+	const struct rte_compressdev_capabilities *qat_dev_capabilities;
+	/* QAT device compression capabilities */
+	const struct rte_memzone *interm_buff_mz;
+	/**< The device's memory for intermediate buffers */
+	struct rte_mempool *xformpool;
+	/**< The device's pool for qat_comp_xforms */
+};
+
+int
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev);
+
+int
+qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev);
+
+#endif
+#endif /* _QAT_COMP_PMD_H_ */
diff --git a/drivers/compress/qat/rte_pmd_qat_version.map b/drivers/compress/qat/rte_pmd_qat_version.map
new file mode 100644
index 00000000..ad6e191e
--- /dev/null
+++ b/drivers/compress/qat/rte_pmd_qat_version.map
@@ -0,0 +1,3 @@
+DPDK_18.08 {
+	local: *;
+};
diff --git a/drivers/compress/zlib/Makefile b/drivers/compress/zlib/Makefile
new file mode 100644
index 00000000..5cf8de6f
--- /dev/null
+++ b/drivers/compress/zlib/Makefile
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Cavium Networks
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_pmd_zlib.a
+
+# build flags
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+# library version
+LIBABIVER := 1
+
+# versioning export map
+EXPORT_MAP := rte_pmd_zlib_version.map
+
+# external library dependencies
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring -lz
+LDLIBS += -lrte_compressdev
+LDLIBS += -lrte_bus_vdev
+
+# library source files
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_ZLIB) += zlib_pmd.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_ZLIB) += zlib_pmd_ops.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/compress/zlib/meson.build b/drivers/compress/zlib/meson.build
new file mode 100644
index 00000000..7748de2d
--- /dev/null
+++ b/drivers/compress/zlib/meson.build
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Cavium Networks
+
+dep = dependency('zlib', required: false)
+if not dep.found()
+	build = false
+endif
+
+deps += 'bus_vdev'
+sources = files('zlib_pmd.c', 'zlib_pmd_ops.c')
+ext_deps += dep
+pkgconfig_extra_libs += '-lz'
+
+allow_experimental_apis = true
diff --git a/drivers/compress/zlib/rte_pmd_zlib_version.map b/drivers/compress/zlib/rte_pmd_zlib_version.map
new file mode 100644
index 00000000..ad6e191e
--- /dev/null
+++ b/drivers/compress/zlib/rte_pmd_zlib_version.map
@@ -0,0 +1,3 @@
+DPDK_18.08 {
+	local: *;
+};
diff --git a/drivers/compress/zlib/zlib_pmd.c b/drivers/compress/zlib/zlib_pmd.c
new file mode 100644
index 00000000..7d6871b1
--- /dev/null
+++ b/drivers/compress/zlib/zlib_pmd.c
@@ -0,0 +1,436 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium Networks
+ */
+
+#include <rte_bus_vdev.h>
+#include <rte_common.h>
+
+#include "zlib_pmd_private.h"
+
+/** Compute next mbuf in the list, assign data buffer and length,
+ *  returns 0 if mbuf is NULL
+ */
+#define COMPUTE_BUF(mbuf, data, len)		\
+		((mbuf = mbuf->next) ?		\
+		(data = rte_pktmbuf_mtod(mbuf, uint8_t *)),	\
+		(len = rte_pktmbuf_data_len(mbuf)) : 0)
+
+static void
+process_zlib_deflate(struct rte_comp_op *op, z_stream *strm)
+{
+	int ret, flush, fin_flush;
+	struct rte_mbuf *mbuf_src = op->m_src;
+	struct rte_mbuf *mbuf_dst = op->m_dst;
+
+	switch (op->flush_flag) {
+	case RTE_COMP_FLUSH_FULL:
+	case RTE_COMP_FLUSH_FINAL:
+		fin_flush = Z_FINISH;
+		break;
+	default:
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		ZLIB_PMD_ERR("Invalid flush value\n");
+	}
+
+	if (unlikely(!strm)) {
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		ZLIB_PMD_ERR("Invalid z_stream\n");
+		return;
+	}
+	/* Update z_stream with the inputs provided by application */
+	strm->next_in = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
+			op->src.offset);
+
+	strm->avail_in = rte_pktmbuf_data_len(mbuf_src) - op->src.offset;
+
+	strm->next_out = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
+			op->dst.offset);
+
+	strm->avail_out = rte_pktmbuf_data_len(mbuf_dst) - op->dst.offset;
+
+	/* Set flush value to NO_FLUSH unless it is last mbuf */
+	flush = Z_NO_FLUSH;
+	/* Initialize status to SUCCESS */
+	op->status = RTE_COMP_OP_STATUS_SUCCESS;
+
+	do {
+		/* Set flush value to Z_FINISH for last block */
+		if ((op->src.length - strm->total_in) <= strm->avail_in) {
+			strm->avail_in = (op->src.length - strm->total_in);
+			flush = fin_flush;
+		}
+		do {
+			ret = deflate(strm, flush);
+			if (unlikely(ret == Z_STREAM_ERROR)) {
+				/* error return, do not process further */
+				op->status =  RTE_COMP_OP_STATUS_ERROR;
+				goto def_end;
+			}
+			/* Break if Z_STREAM_END is encountered */
+			if (ret == Z_STREAM_END)
+				goto def_end;
+
+		/* Keep looping until input mbuf is consumed.
+		 * Exit if destination mbuf gets exhausted.
+		 */
+		} while ((strm->avail_out == 0) &&
+			COMPUTE_BUF(mbuf_dst, strm->next_out, strm->avail_out));
+
+		if (!strm->avail_out) {
+			/* there is no space for compressed output */
+			op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+			break;
+		}
+
+	/* Update source buffer to next mbuf
+	 * Exit if input buffers are fully consumed
+	 */
+	} while (COMPUTE_BUF(mbuf_src, strm->next_in, strm->avail_in));
+
+def_end:
+	/* Update op stats */
+	switch (op->status) {
+	case RTE_COMP_OP_STATUS_SUCCESS:
+		op->consumed += strm->total_in;
+	/* Fall-through */
+	case RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED:
+		op->produced += strm->total_out;
+		break;
+	default:
+		ZLIB_PMD_ERR("stats not updated for status:%d\n",
+				op->status);
+	}
+
+	deflateReset(strm);
+}
+
+static void
+process_zlib_inflate(struct rte_comp_op *op, z_stream *strm)
+{
+	int ret, flush;
+	struct rte_mbuf *mbuf_src = op->m_src;
+	struct rte_mbuf *mbuf_dst = op->m_dst;
+
+	if (unlikely(!strm)) {
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		ZLIB_PMD_ERR("Invalid z_stream\n");
+		return;
+	}
+	strm->next_in = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
+			op->src.offset);
+
+	strm->avail_in = rte_pktmbuf_data_len(mbuf_src) - op->src.offset;
+
+	strm->next_out = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
+			op->dst.offset);
+
+	strm->avail_out = rte_pktmbuf_data_len(mbuf_dst) - op->dst.offset;
+
+	/** Ignoring flush value provided from application for decompression */
+	flush = Z_NO_FLUSH;
+	/* initialize status to SUCCESS */
+	op->status = RTE_COMP_OP_STATUS_SUCCESS;
+
+	do {
+		do {
+			ret = inflate(strm, flush);
+
+			switch (ret) {
+			/* Fall-through */
+			case Z_NEED_DICT:
+				ret = Z_DATA_ERROR;
+			/* Fall-through */
+			case Z_DATA_ERROR:
+			/* Fall-through */
+			case Z_MEM_ERROR:
+			/* Fall-through */
+			case Z_STREAM_ERROR:
+				op->status = RTE_COMP_OP_STATUS_ERROR;
+			/* Fall-through */
+			case Z_STREAM_END:
+				/* no further computation needed if
+				 * Z_STREAM_END is encountered
+				 */
+				goto inf_end;
+			default:
+				/* success */
+				break;
+
+			}
+		/* Keep looping until input mbuf is consumed.
+		 * Exit if destination mbuf gets exhausted.
+		 */
+		} while ((strm->avail_out == 0) &&
+			COMPUTE_BUF(mbuf_dst, strm->next_out, strm->avail_out));
+
+		if (!strm->avail_out) {
+			/* there is no more space for decompressed output */
+			op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+			break;
+		}
+	/* Read next input buffer to be processed, exit if compressed
+	 * blocks are fully read
+	 */
+	} while (COMPUTE_BUF(mbuf_src, strm->next_in, strm->avail_in));
+
+inf_end:
+	/* Update op stats */
+	switch (op->status) {
+	case RTE_COMP_OP_STATUS_SUCCESS:
+		op->consumed += strm->total_in;
+	/* Fall-through */
+	case RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED:
+		op->produced += strm->total_out;
+		break;
+	default:
+		ZLIB_PMD_ERR("stats not produced for status:%d\n",
+				op->status);
+	}
+
+	inflateReset(strm);
+}
+
+/** Process comp operation for mbuf */
+static inline int
+process_zlib_op(struct zlib_qp *qp, struct rte_comp_op *op)
+{
+	struct zlib_stream *stream;
+	struct zlib_priv_xform *private_xform;
+
+	if ((op->op_type == RTE_COMP_OP_STATEFUL) ||
+			(op->src.offset > rte_pktmbuf_data_len(op->m_src)) ||
+			(op->dst.offset > rte_pktmbuf_data_len(op->m_dst))) {
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		ZLIB_PMD_ERR("Invalid source or destination buffers or "
+			     "invalid Operation requested\n");
+	} else {
+		private_xform = (struct zlib_priv_xform *)op->private_xform;
+		stream = &private_xform->stream;
+		stream->comp(op, &stream->strm);
+	}
+	/* whatever is out of op, put it into completion queue with
+	 * its status
+	 */
+	return rte_ring_enqueue(qp->processed_pkts, (void *)op);
+}
+
+/** Parse comp xform and set private xform/Stream parameters */
+int
+zlib_set_stream_parameters(const struct rte_comp_xform *xform,
+		struct zlib_stream *stream)
+{
+	int strategy, level, wbits;
+	z_stream *strm = &stream->strm;
+
+	/* allocate deflate state */
+	strm->zalloc = Z_NULL;
+	strm->zfree = Z_NULL;
+	strm->opaque = Z_NULL;
+
+	switch (xform->type) {
+	case RTE_COMP_COMPRESS:
+		stream->comp = process_zlib_deflate;
+		stream->free = deflateEnd;
+		/** Compression window bits */
+		switch (xform->compress.algo) {
+		case RTE_COMP_ALGO_DEFLATE:
+			wbits = -(xform->compress.window_size);
+			break;
+		default:
+			ZLIB_PMD_ERR("Compression algorithm not supported\n");
+			return -1;
+		}
+		/** Compression Level */
+		switch (xform->compress.level) {
+		case RTE_COMP_LEVEL_PMD_DEFAULT:
+			level = Z_DEFAULT_COMPRESSION;
+			break;
+		case RTE_COMP_LEVEL_NONE:
+			level = Z_NO_COMPRESSION;
+			break;
+		case RTE_COMP_LEVEL_MIN:
+			level = Z_BEST_SPEED;
+			break;
+		case RTE_COMP_LEVEL_MAX:
+			level = Z_BEST_COMPRESSION;
+			break;
+		default:
+			level = xform->compress.level;
+			if (level < RTE_COMP_LEVEL_MIN ||
+					level > RTE_COMP_LEVEL_MAX) {
+				ZLIB_PMD_ERR("Compression level %d "
+						"not supported\n",
+						level);
+				return -1;
+			}
+			break;
+		}
+		/** Compression strategy */
+		switch (xform->compress.deflate.huffman) {
+		case RTE_COMP_HUFFMAN_DEFAULT:
+			strategy = Z_DEFAULT_STRATEGY;
+			break;
+		case RTE_COMP_HUFFMAN_FIXED:
+			strategy = Z_FIXED;
+			break;
+		case RTE_COMP_HUFFMAN_DYNAMIC:
+			strategy = Z_DEFAULT_STRATEGY;
+			break;
+		default:
+			ZLIB_PMD_ERR("Compression strategy not supported\n");
+			return -1;
+		}
+		if (deflateInit2(strm, level,
+					Z_DEFLATED, wbits,
+					DEF_MEM_LEVEL, strategy) != Z_OK) {
+			ZLIB_PMD_ERR("Deflate init failed\n");
+			return -1;
+		}
+		break;
+
+	case RTE_COMP_DECOMPRESS:
+		stream->comp = process_zlib_inflate;
+		stream->free = inflateEnd;
+		/** window bits */
+		switch (xform->decompress.algo) {
+		case RTE_COMP_ALGO_DEFLATE:
+			wbits = -(xform->decompress.window_size);
+			break;
+		default:
+			ZLIB_PMD_ERR("Compression algorithm not supported\n");
+			return -1;
+		}
+
+		if (inflateInit2(strm, wbits) != Z_OK) {
+			ZLIB_PMD_ERR("Inflate init failed\n");
+			return -1;
+		}
+		break;
+	default:
+		return -1;
+	}
+	return 0;
+}
+
+static uint16_t
+zlib_pmd_enqueue_burst(void *queue_pair,
+			struct rte_comp_op **ops, uint16_t nb_ops)
+{
+	struct zlib_qp *qp = queue_pair;
+	int ret;
+	uint16_t i;
+	uint16_t enqd = 0;
+	for (i = 0; i < nb_ops; i++) {
+		ret = process_zlib_op(qp, ops[i]);
+		if (unlikely(ret < 0)) {
+			/* increment count if failed to push to completion
+			 * queue
+			 */
+			qp->qp_stats.enqueue_err_count++;
+		} else {
+			qp->qp_stats.enqueued_count++;
+			enqd++;
+		}
+	}
+	return enqd;
+}
+
+static uint16_t
+zlib_pmd_dequeue_burst(void *queue_pair,
+			struct rte_comp_op **ops, uint16_t nb_ops)
+{
+	struct zlib_qp *qp = queue_pair;
+
+	unsigned int nb_dequeued = 0;
+
+	nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts,
+			(void **)ops, nb_ops, NULL);
+	qp->qp_stats.dequeued_count += nb_dequeued;
+
+	return nb_dequeued;
+}
+
+static int
+zlib_create(const char *name,
+		struct rte_vdev_device *vdev,
+		struct rte_compressdev_pmd_init_params *init_params)
+{
+	struct rte_compressdev *dev;
+
+	dev = rte_compressdev_pmd_create(name, &vdev->device,
+			sizeof(struct zlib_private), init_params);
+	if (dev == NULL) {
+		ZLIB_PMD_ERR("driver %s: create failed", init_params->name);
+		return -ENODEV;
+	}
+
+	dev->dev_ops = rte_zlib_pmd_ops;
+
+	/* register rx/tx burst functions for data path */
+	dev->dequeue_burst = zlib_pmd_dequeue_burst;
+	dev->enqueue_burst = zlib_pmd_enqueue_burst;
+
+	return 0;
+}
+
+static int
+zlib_probe(struct rte_vdev_device *vdev)
+{
+	struct rte_compressdev_pmd_init_params init_params = {
+		"",
+		rte_socket_id()
+	};
+	const char *name;
+	const char *input_args;
+	int retval;
+
+	name = rte_vdev_device_name(vdev);
+
+	if (name == NULL)
+		return -EINVAL;
+
+	input_args = rte_vdev_device_args(vdev);
+
+	retval = rte_compressdev_pmd_parse_input_args(&init_params, input_args);
+	if (retval < 0) {
+		ZLIB_PMD_LOG(ERR,
+			"Failed to parse initialisation arguments[%s]\n",
+			input_args);
+		return -EINVAL;
+	}
+
+	return zlib_create(name, vdev, &init_params);
+}
+
+static int
+zlib_remove(struct rte_vdev_device *vdev)
+{
+	struct rte_compressdev *compressdev;
+	const char *name;
+
+	name = rte_vdev_device_name(vdev);
+	if (name == NULL)
+		return -EINVAL;
+
+	compressdev = rte_compressdev_pmd_get_named_dev(name);
+	if (compressdev == NULL)
+		return -ENODEV;
+
+	return rte_compressdev_pmd_destroy(compressdev);
+}
+
+static struct rte_vdev_driver zlib_pmd_drv = {
+	.probe = zlib_probe,
+	.remove = zlib_remove
+};
+
+RTE_PMD_REGISTER_VDEV(COMPRESSDEV_NAME_ZLIB_PMD, zlib_pmd_drv);
+RTE_INIT(zlib_init_log);
+
+static void
+zlib_init_log(void)
+{
+	zlib_logtype_driver = rte_log_register("pmd.compress.zlib");
+	if (zlib_logtype_driver >= 0)
+		rte_log_set_level(zlib_logtype_driver, RTE_LOG_INFO);
+}
diff --git a/drivers/compress/zlib/zlib_pmd_ops.c b/drivers/compress/zlib/zlib_pmd_ops.c
new file mode 100644
index 00000000..0a73aed9
--- /dev/null
+++ b/drivers/compress/zlib/zlib_pmd_ops.c
@@ -0,0 +1,307 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium Networks
+ */
+
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_malloc.h>
+
+#include "zlib_pmd_private.h"
+
+static const struct rte_compressdev_capabilities zlib_pmd_capabilities[] = {
+	{   /* Deflate */
+		.algo = RTE_COMP_ALGO_DEFLATE,
+		.comp_feature_flags = (RTE_COMP_FF_NONCOMPRESSED_BLOCKS |
+					RTE_COMP_FF_HUFFMAN_FIXED |
+					RTE_COMP_FF_HUFFMAN_DYNAMIC),
+		.window_size = {
+			.min = 8,
+			.max = 15,
+			.increment = 1
+		},
+	},
+
+	RTE_COMP_END_OF_CAPABILITIES_LIST()
+
+};
+
+/** Configure device */
+static int
+zlib_pmd_config(struct rte_compressdev *dev,
+		struct rte_compressdev_config *config)
+{
+	struct rte_mempool *mp;
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct zlib_private *internals = dev->data->dev_private;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"stream_mp_%u", dev->data->dev_id);
+	mp = internals->mp;
+	if (mp == NULL) {
+		mp = rte_mempool_create(mp_name,
+				config->max_nb_priv_xforms +
+				config->max_nb_streams,
+				sizeof(struct zlib_priv_xform),
+				0, 0, NULL, NULL, NULL,
+				NULL, config->socket_id,
+				0);
+		if (mp == NULL) {
+			ZLIB_PMD_ERR("Cannot create private xform pool on "
+			"socket %d\n", config->socket_id);
+			return -ENOMEM;
+		}
+		internals->mp = mp;
+	}
+	return 0;
+}
+
+/** Start device */
+static int
+zlib_pmd_start(__rte_unused struct rte_compressdev *dev)
+{
+	return 0;
+}
+
+/** Stop device */
+static void
+zlib_pmd_stop(__rte_unused struct rte_compressdev *dev)
+{
+}
+
+/** Close device */
+static int
+zlib_pmd_close(struct rte_compressdev *dev)
+{
+	struct zlib_private *internals = dev->data->dev_private;
+	rte_mempool_free(internals->mp);
+	internals->mp = NULL;
+	return 0;
+}
+
+/** Get device statistics */
+static void
+zlib_pmd_stats_get(struct rte_compressdev *dev,
+		struct rte_compressdev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct zlib_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->qp_stats.enqueued_count;
+		stats->dequeued_count += qp->qp_stats.dequeued_count;
+
+		stats->enqueue_err_count += qp->qp_stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->qp_stats.dequeue_err_count;
+	}
+}
+
+/** Reset device statistics */
+static void
+zlib_pmd_stats_reset(struct rte_compressdev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct zlib_qp *qp = dev->data->queue_pairs[qp_id];
+
+		memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
+	}
+}
+
+/** Get device info */
+static void
+zlib_pmd_info_get(struct rte_compressdev *dev,
+		struct rte_compressdev_info *dev_info)
+{
+	if (dev_info != NULL) {
+		dev_info->driver_name = dev->device->name;
+		dev_info->feature_flags = dev->feature_flags;
+		dev_info->capabilities = zlib_pmd_capabilities;
+	}
+}
+
+/** Release queue pair */
+static int
+zlib_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
+{
+	struct zlib_qp *qp = dev->data->queue_pairs[qp_id];
+
+	if (qp != NULL) {
+		rte_ring_free(qp->processed_pkts);
+		rte_free(qp);
+		dev->data->queue_pairs[qp_id] = NULL;
+	}
+	return 0;
+}
+
+/** set a unique name for the queue pair based on its name, dev_id and qp_id */
+static int
+zlib_pmd_qp_set_unique_name(struct rte_compressdev *dev,
+		struct zlib_qp *qp)
+{
+	unsigned int n = snprintf(qp->name, sizeof(qp->name),
+				"zlib_pmd_%u_qp_%u",
+				dev->data->dev_id, qp->id);
+
+	if (n >= sizeof(qp->name))
+		return -1;
+
+	return 0;
+}
+
+/** Create a ring to place process packets on */
+static struct rte_ring *
+zlib_pmd_qp_create_processed_pkts_ring(struct zlib_qp *qp,
+		unsigned int ring_size, int socket_id)
+{
+	struct rte_ring *r = qp->processed_pkts;
+
+	if (r) {
+		if (rte_ring_get_size(r) >= ring_size) {
+			ZLIB_PMD_INFO("Reusing existing ring %s for processed"
+					" packets", qp->name);
+			return r;
+		}
+
+		ZLIB_PMD_ERR("Unable to reuse existing ring %s for processed"
+				" packets", qp->name);
+		return NULL;
+	}
+
+	return rte_ring_create(qp->name, ring_size, socket_id,
+						RING_F_EXACT_SZ);
+}
+
+/** Setup a queue pair */
+static int
+zlib_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
+		uint32_t max_inflight_ops, int socket_id)
+{
+	struct zlib_qp *qp = NULL;
+
+	/* Free memory prior to re-allocation if needed. */
+	if (dev->data->queue_pairs[qp_id] != NULL)
+		zlib_pmd_qp_release(dev, qp_id);
+
+	/* Allocate the queue pair data structure. */
+	qp = rte_zmalloc_socket("ZLIB PMD Queue Pair", sizeof(*qp),
+					RTE_CACHE_LINE_SIZE, socket_id);
+	if (qp == NULL)
+		return (-ENOMEM);
+
+	qp->id = qp_id;
+	dev->data->queue_pairs[qp_id] = qp;
+
+	if (zlib_pmd_qp_set_unique_name(dev, qp))
+		goto qp_setup_cleanup;
+
+	qp->processed_pkts = zlib_pmd_qp_create_processed_pkts_ring(qp,
+			max_inflight_ops, socket_id);
+	if (qp->processed_pkts == NULL)
+		goto qp_setup_cleanup;
+
+	memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
+	return 0;
+
+qp_setup_cleanup:
+	if (qp) {
+		rte_free(qp);
+		qp = NULL;
+	}
+	return -1;
+}
+
+/** Configure stream */
+static int
+zlib_pmd_stream_create(struct rte_compressdev *dev,
+		const struct rte_comp_xform *xform,
+		void **zstream)
+{
+	int ret = 0;
+	struct zlib_stream *stream;
+	struct zlib_private *internals = dev->data->dev_private;
+
+	if (xform == NULL) {
+		ZLIB_PMD_ERR("invalid xform struct");
+		return -EINVAL;
+	}
+
+	if (rte_mempool_get(internals->mp, zstream)) {
+		ZLIB_PMD_ERR("Couldn't get object from session mempool");
+		return -ENOMEM;
+	}
+	stream = *((struct zlib_stream **)zstream);
+
+	ret = zlib_set_stream_parameters(xform, stream);
+
+	if (ret < 0) {
+		ZLIB_PMD_ERR("failed configure session parameters");
+
+		memset(stream, 0, sizeof(struct zlib_stream));
+		/* Return session to mempool */
+		rte_mempool_put(internals->mp, stream);
+		return ret;
+	}
+
+	return 0;
+}
+
+/** Configure private xform */
+static int
+zlib_pmd_private_xform_create(struct rte_compressdev *dev,
+		const struct rte_comp_xform *xform,
+		void **private_xform)
+{
+	return zlib_pmd_stream_create(dev, xform, private_xform);
+}
+
+/** Clear the memory of stream so it doesn't leave key material behind */
+static int
+zlib_pmd_stream_free(__rte_unused struct rte_compressdev *dev,
+		void *zstream)
+{
+	struct zlib_stream *stream = (struct zlib_stream *)zstream;
+	if (!stream)
+		return -EINVAL;
+
+	stream->free(&stream->strm);
+	/* Zero out the whole structure */
+	memset(stream, 0, sizeof(struct zlib_stream));
+	struct rte_mempool *mp = rte_mempool_from_obj(stream);
+	rte_mempool_put(mp, stream);
+
+	return 0;
+}
+
+/** Clear the memory of stream so it doesn't leave key material behind */
+static int
+zlib_pmd_private_xform_free(struct rte_compressdev *dev,
+		void *private_xform)
+{
+	return zlib_pmd_stream_free(dev, private_xform);
+}
+
+struct rte_compressdev_ops zlib_pmd_ops = {
+		.dev_configure		= zlib_pmd_config,
+		.dev_start		= zlib_pmd_start,
+		.dev_stop		= zlib_pmd_stop,
+		.dev_close		= zlib_pmd_close,
+
+		.stats_get		= zlib_pmd_stats_get,
+		.stats_reset		= zlib_pmd_stats_reset,
+
+		.dev_infos_get		= zlib_pmd_info_get,
+
+		.queue_pair_setup	= zlib_pmd_qp_setup,
+		.queue_pair_release	= zlib_pmd_qp_release,
+
+		.private_xform_create	= zlib_pmd_private_xform_create,
+		.private_xform_free	= zlib_pmd_private_xform_free,
+
+		.stream_create	= NULL,
+		.stream_free	= NULL
+};
+
+struct rte_compressdev_ops *rte_zlib_pmd_ops = &zlib_pmd_ops;
diff --git a/drivers/compress/zlib/zlib_pmd_private.h b/drivers/compress/zlib/zlib_pmd_private.h
new file mode 100644
index 00000000..2c6e83d4
--- /dev/null
+++ b/drivers/compress/zlib/zlib_pmd_private.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium Networks
+ */
+
+#ifndef _RTE_ZLIB_PMD_PRIVATE_H_
+#define _RTE_ZLIB_PMD_PRIVATE_H_
+
+#include <zlib.h>
+#include <rte_compressdev.h>
+#include <rte_compressdev_pmd.h>
+
+#define COMPRESSDEV_NAME_ZLIB_PMD	compress_zlib
+/**< ZLIB PMD device name */
+
+#define DEF_MEM_LEVEL			8
+
+int zlib_logtype_driver;
+#define ZLIB_PMD_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, zlib_logtype_driver, "%s(): "fmt "\n", \
+			__func__, ##args)
+
+#define ZLIB_PMD_INFO(fmt, args...) \
+	ZLIB_PMD_LOG(INFO, fmt, ## args)
+#define ZLIB_PMD_ERR(fmt, args...) \
+	ZLIB_PMD_LOG(ERR, fmt, ## args)
+#define ZLIB_PMD_WARN(fmt, args...) \
+	ZLIB_PMD_LOG(WARNING, fmt, ## args)
+
+struct zlib_private {
+	struct rte_mempool *mp;
+};
+
+struct zlib_qp {
+	struct rte_ring *processed_pkts;
+	/**< Ring for placing process packets */
+	struct rte_compressdev_stats qp_stats;
+	/**< Queue pair statistics */
+	uint16_t id;
+	/**< Queue Pair Identifier */
+	char name[RTE_COMPRESSDEV_NAME_MAX_LEN];
+	/**< Unique Queue Pair Name */
+} __rte_cache_aligned;
+
+/* Algorithm handler function prototype */
+typedef void (*comp_func_t)(struct rte_comp_op *op, z_stream *strm);
+
+typedef int (*comp_free_t)(z_stream *strm);
+
+/** ZLIB Stream structure */
+struct zlib_stream {
+	z_stream strm;
+	/**< zlib stream structure */
+	comp_func_t comp;
+	/**< Operation (compression/decompression) */
+	comp_free_t free;
+	/**< Free Operation (compression/decompression) */
+} __rte_cache_aligned;
+
+/** ZLIB private xform structure */
+struct zlib_priv_xform {
+	struct zlib_stream stream;
+} __rte_cache_aligned;
+
+int
+zlib_set_stream_parameters(const struct rte_comp_xform *xform,
+		struct zlib_stream *stream);
+
+/** Device specific operations function pointer structure */
+extern struct rte_compressdev_ops *rte_zlib_pmd_ops;
+
+#endif /* _RTE_ZLIB_PMD_PRIVATE_H_ */
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 1d0c88ef..c480cbd3 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -8,7 +8,6 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += aesni_mb
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO) += armv8
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_CCP) += ccp
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_OPENSSL) += openssl
-DIRS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += qat
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += snow3g
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += kasumi
diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c
index 80360dd9..752e0cd6 100644
--- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c
+++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c
@@ -31,8 +31,8 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops,
 	if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 		auth_xform = xform;
 		if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC) {
-			GCM_LOG_ERR("Only AES GMAC is supported as an "
-					"authentication only algorithm");
+			AESNI_GCM_LOG(ERR, "Only AES GMAC is supported as an "
+				"authentication only algorithm");
 			return -ENOTSUP;
 		}
 		/* Set IV parameters */
@@ -54,7 +54,7 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops,
 		aead_xform = xform;
 
 		if (aead_xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM) {
-			GCM_LOG_ERR("The only combined operation "
+			AESNI_GCM_LOG(ERR, "The only combined operation "
 						"supported is AES GCM");
 			return -ENOTSUP;
 		}
@@ -75,7 +75,7 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops,
 		sess->aad_length = aead_xform->aead.aad_length;
 		digest_length = aead_xform->aead.digest_length;
 	} else {
-		GCM_LOG_ERR("Wrong xform type, has to be AEAD or authentication");
+		AESNI_GCM_LOG(ERR, "Wrong xform type, has to be AEAD or authentication");
 		return -ENOTSUP;
 	}
 
@@ -83,7 +83,7 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops,
 	/* IV check */
 	if (sess->iv.length != 16 && sess->iv.length != 12 &&
 			sess->iv.length != 0) {
-		GCM_LOG_ERR("Wrong IV length");
+		AESNI_GCM_LOG(ERR, "Wrong IV length");
 		return -EINVAL;
 	}
 
@@ -99,7 +99,7 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops,
 		sess->key = AESNI_GCM_KEY_256;
 		break;
 	default:
-		GCM_LOG_ERR("Invalid key length");
+		AESNI_GCM_LOG(ERR, "Invalid key length");
 		return -EINVAL;
 	}
 
@@ -109,7 +109,7 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops,
 	if (digest_length != 16 &&
 			digest_length != 12 &&
 			digest_length != 8) {
-		GCM_LOG_ERR("digest");
+		AESNI_GCM_LOG(ERR, "Invalid digest length");
 		return -EINVAL;
 	}
 	sess->digest_length = digest_length;
@@ -127,7 +127,7 @@ aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_op *op)
 	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 		if (likely(sym_op->session != NULL))
 			sess = (struct aesni_gcm_session *)
-					get_session_private_data(
+					get_sym_session_private_data(
 					sym_op->session,
 					cryptodev_driver_id);
 	} else  {
@@ -149,8 +149,8 @@ aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_op *op)
 			sess = NULL;
 		}
 		sym_op->session = (struct rte_cryptodev_sym_session *)_sess;
-		set_session_private_data(sym_op->session, cryptodev_driver_id,
-			_sess_private_data);
+		set_sym_session_private_data(sym_op->session,
+				cryptodev_driver_id, _sess_private_data);
 	}
 
 	if (unlikely(sess == NULL))
@@ -464,13 +464,13 @@ aesni_gcm_create(const char *name,
 
 	/* Check CPU for support for AES instruction set */
 	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
-		GCM_LOG_ERR("AES instructions not supported by CPU");
+		AESNI_GCM_LOG(ERR, "AES instructions not supported by CPU");
 		return -EFAULT;
 	}
-
 	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
 	if (dev == NULL) {
-		GCM_LOG_ERR("driver %s: create failed", init_params->name);
+		AESNI_GCM_LOG(ERR, "driver %s: create failed",
+			init_params->name);
 		return -ENODEV;
 	}
 
@@ -492,7 +492,8 @@ aesni_gcm_create(const char *name,
 	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 			RTE_CRYPTODEV_FF_CPU_AESNI |
-			RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
 
 	switch (vector_mode) {
 	case RTE_AESNI_GCM_SSE:
@@ -513,7 +514,13 @@ aesni_gcm_create(const char *name,
 	internals->vector_mode = vector_mode;
 
 	internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
-	internals->max_nb_sessions = init_params->max_nb_sessions;
+
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+	AESNI_GCM_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
+			imb_get_version_str());
+#else
+	AESNI_GCM_LOG(INFO, "IPSec Multi-buffer library version used: 0.49.0\n");
+#endif
 
 	return 0;
 }
@@ -525,8 +532,7 @@ aesni_gcm_probe(struct rte_vdev_device *vdev)
 		"",
 		sizeof(struct aesni_gcm_private),
 		rte_socket_id(),
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
+		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 	};
 	const char *name;
 	const char *input_args;
@@ -568,7 +574,12 @@ RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_GCM_PMD, aesni_gcm_pmd_drv);
 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_GCM_PMD, cryptodev_aesni_gcm_pmd);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD,
 	"max_nb_queue_pairs=<int> "
-	"max_nb_sessions=<int> "
 	"socket_id=<int>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_crypto_drv, aesni_gcm_pmd_drv.driver,
 		cryptodev_driver_id);
+
+
+RTE_INIT(aesni_gcm_init_log)
+{
+	aesni_gcm_logtype_driver = rte_log_register("pmd.crypto.aesni_gcm");
+}
diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c
index 6f542137..b6b4dd02 100644
--- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c
+++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c
@@ -143,7 +143,8 @@ aesni_gcm_pmd_info_get(struct rte_cryptodev *dev,
 		dev_info->capabilities = aesni_gcm_pmd_capabilities;
 
 		dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
-		dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
+		/* No limit of number of sessions */
+		dev_info->sym.max_nb_sessions = 0;
 	}
 }
 
@@ -183,12 +184,11 @@ aesni_gcm_pmd_qp_create_processed_pkts_ring(struct aesni_gcm_qp *qp,
 	r = rte_ring_lookup(qp->name);
 	if (r) {
 		if (rte_ring_get_size(r) >= ring_size) {
-			GCM_LOG_INFO("Reusing existing ring %s for processed"
-					" packets", qp->name);
+			AESNI_GCM_LOG(INFO, "Reusing existing ring %s for processed"
+				" packets", qp->name);
 			return r;
 		}
-
-		GCM_LOG_ERR("Unable to reuse existing ring %s for processed"
+		AESNI_GCM_LOG(ERR, "Unable to reuse existing ring %s for processed"
 				" packets", qp->name);
 		return NULL;
 	}
@@ -242,22 +242,6 @@ qp_setup_cleanup:
 	return -1;
 }
 
-/** Start queue pair */
-static int
-aesni_gcm_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
-/** Stop queue pair */
-static int
-aesni_gcm_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
 /** Return the number of allocated queue pairs */
 static uint32_t
 aesni_gcm_pmd_qp_count(struct rte_cryptodev *dev)
@@ -267,14 +251,14 @@ aesni_gcm_pmd_qp_count(struct rte_cryptodev *dev)
 
 /** Returns the size of the aesni gcm session structure */
 static unsigned
-aesni_gcm_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
+aesni_gcm_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	return sizeof(struct aesni_gcm_session);
 }
 
 /** Configure a aesni gcm session from a crypto xform chain */
 static int
-aesni_gcm_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
+aesni_gcm_pmd_sym_session_configure(struct rte_cryptodev *dev __rte_unused,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *sess,
 		struct rte_mempool *mempool)
@@ -284,26 +268,26 @@ aesni_gcm_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
 	struct aesni_gcm_private *internals = dev->data->dev_private;
 
 	if (unlikely(sess == NULL)) {
-		GCM_LOG_ERR("invalid session struct");
+		AESNI_GCM_LOG(ERR, "invalid session struct");
 		return -EINVAL;
 	}
 
 	if (rte_mempool_get(mempool, &sess_private_data)) {
-		CDEV_LOG_ERR(
-			"Couldn't get object from session mempool");
+		AESNI_GCM_LOG(ERR,
+				"Couldn't get object from session mempool");
 		return -ENOMEM;
 	}
 	ret = aesni_gcm_set_session_parameters(gcm_ops[internals->vector_mode],
 				sess_private_data, xform);
 	if (ret != 0) {
-		GCM_LOG_ERR("failed configure session parameters");
+		AESNI_GCM_LOG(ERR, "failed configure session parameters");
 
 		/* Return session to mempool */
 		rte_mempool_put(mempool, sess_private_data);
 		return ret;
 	}
 
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 			sess_private_data);
 
 	return 0;
@@ -311,17 +295,17 @@ aesni_gcm_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
 
 /** Clear the memory of session so it doesn't leave key material behind */
 static void
-aesni_gcm_pmd_session_clear(struct rte_cryptodev *dev,
+aesni_gcm_pmd_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 
 	/* Zero out the whole structure */
 	if (sess_priv) {
 		memset(sess_priv, 0, sizeof(struct aesni_gcm_session));
 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
 		rte_mempool_put(sess_mp, sess_priv);
 	}
 }
@@ -339,13 +323,11 @@ struct rte_cryptodev_ops aesni_gcm_pmd_ops = {
 
 		.queue_pair_setup	= aesni_gcm_pmd_qp_setup,
 		.queue_pair_release	= aesni_gcm_pmd_qp_release,
-		.queue_pair_start	= aesni_gcm_pmd_qp_start,
-		.queue_pair_stop	= aesni_gcm_pmd_qp_stop,
 		.queue_pair_count	= aesni_gcm_pmd_qp_count,
 
-		.session_get_size	= aesni_gcm_pmd_session_get_size,
-		.session_configure	= aesni_gcm_pmd_session_configure,
-		.session_clear		= aesni_gcm_pmd_session_clear
+		.sym_session_get_size	= aesni_gcm_pmd_sym_session_get_size,
+		.sym_session_configure	= aesni_gcm_pmd_sym_session_configure,
+		.sym_session_clear	= aesni_gcm_pmd_sym_session_clear
 };
 
 struct rte_cryptodev_ops *rte_aesni_gcm_pmd_ops = &aesni_gcm_pmd_ops;
diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h
index 3d60583b..c13a12a5 100644
--- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h
+++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h
@@ -7,28 +7,24 @@
 
 #include "aesni_gcm_ops.h"
 
+/*
+ * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50,
+ * so if macro is not defined, it means that the version is 0.49.
+ */
+#if !defined(IMB_VERSION_NUM)
+#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
+#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0)
+#endif
+
 #define CRYPTODEV_NAME_AESNI_GCM_PMD	crypto_aesni_gcm
 /**< AES-NI GCM PMD device name */
 
-#define GCM_LOG_ERR(fmt, args...) \
-	RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",  \
-			RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD), \
-			__func__, __LINE__, ## args)
-
-#ifdef RTE_LIBRTE_AESNI_MB_DEBUG
-#define GCM_LOG_INFO(fmt, args...) \
-	RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD), \
-			__func__, __LINE__, ## args)
-
-#define GCM_LOG_DBG(fmt, args...) \
-	RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD), \
-			__func__, __LINE__, ## args)
-#else
-#define GCM_LOG_INFO(fmt, args...)
-#define GCM_LOG_DBG(fmt, args...)
-#endif
+/** AES-NI GCM PMD  LOGTYPE DRIVER */
+int aesni_gcm_logtype_driver;
+#define AESNI_GCM_LOG(level, fmt, ...) \
+	rte_log(RTE_LOG_ ## level, aesni_gcm_logtype_driver,	\
+			"%s() line %u: "fmt "\n", __func__, __LINE__,	\
+					## __VA_ARGS__)
 
 /* Maximum length for digest */
 #define DIGEST_LENGTH_MAX 16
@@ -39,8 +35,6 @@ struct aesni_gcm_private {
 	/**< Vector mode */
 	unsigned max_nb_queue_pairs;
 	/**< Max number of queue pairs supported by device */
-	unsigned max_nb_sessions;
-	/**< Max number of sessions supported by device */
 };
 
 struct aesni_gcm_qp {
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
index bb35c66a..93dc7a44 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
@@ -108,7 +108,7 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 	}
 
 	if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
-		MB_LOG_ERR("Crypto xform struct not of type auth");
+		AESNI_MB_LOG(ERR, "Crypto xform struct not of type auth");
 		return -1;
 	}
 
@@ -161,7 +161,7 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 		hash_oneblock_fn = mb_ops->aux.one_block.sha512;
 		break;
 	default:
-		MB_LOG_ERR("Unsupported authentication algorithm selection");
+		AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
 		return -ENOTSUP;
 	}
 
@@ -182,6 +182,7 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
 		const struct rte_crypto_sym_xform *xform)
 {
 	uint8_t is_aes = 0;
+	uint8_t is_3DES = 0;
 	aes_keyexp_t aes_keyexp_fn;
 
 	if (xform == NULL) {
@@ -190,7 +191,7 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
 	}
 
 	if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
-		MB_LOG_ERR("Crypto xform struct not of type cipher");
+		AESNI_MB_LOG(ERR, "Crypto xform struct not of type cipher");
 		return -EINVAL;
 	}
 
@@ -203,7 +204,7 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
 		sess->cipher.direction = DECRYPT;
 		break;
 	default:
-		MB_LOG_ERR("Invalid cipher operation parameter");
+		AESNI_MB_LOG(ERR, "Invalid cipher operation parameter");
 		return -EINVAL;
 	}
 
@@ -227,8 +228,12 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
 	case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 		sess->cipher.mode = DOCSIS_DES;
 		break;
+	case RTE_CRYPTO_CIPHER_3DES_CBC:
+		sess->cipher.mode = DES3;
+		is_3DES = 1;
+		break;
 	default:
-		MB_LOG_ERR("Unsupported cipher mode parameter");
+		AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
 		return -ENOTSUP;
 	}
 
@@ -252,7 +257,7 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
 			aes_keyexp_fn = mb_ops->aux.keyexp.aes256;
 			break;
 		default:
-			MB_LOG_ERR("Invalid cipher key length");
+			AESNI_MB_LOG(ERR, "Invalid cipher key length");
 			return -EINVAL;
 		}
 
@@ -261,9 +266,52 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
 				sess->cipher.expanded_aes_keys.encode,
 				sess->cipher.expanded_aes_keys.decode);
 
+	} else if (is_3DES) {
+		uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
+				sess->cipher.exp_3des_keys.key[1],
+				sess->cipher.exp_3des_keys.key[2]};
+
+		switch (xform->cipher.key.length) {
+		case  24:
+			des_key_schedule(keys[0], xform->cipher.key.data);
+			des_key_schedule(keys[1], xform->cipher.key.data+8);
+			des_key_schedule(keys[2], xform->cipher.key.data+16);
+
+			/* Initialize keys - 24 bytes: [K1-K2-K3] */
+			sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
+			sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
+			sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
+			break;
+		case 16:
+			des_key_schedule(keys[0], xform->cipher.key.data);
+			des_key_schedule(keys[1], xform->cipher.key.data+8);
+
+			/* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
+			sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
+			sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
+			sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
+			break;
+		case 8:
+			des_key_schedule(keys[0], xform->cipher.key.data);
+
+			/* Initialize keys - 8 bytes: [K1 = K2 = K3] */
+			sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
+			sess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];
+			sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
+			break;
+		default:
+			AESNI_MB_LOG(ERR, "Invalid cipher key length");
+			return -EINVAL;
+		}
+
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		sess->cipher.key_length_in_bytes = 24;
+#else
+		sess->cipher.key_length_in_bytes = 8;
+#endif
 	} else {
 		if (xform->cipher.key.length != 8) {
-			MB_LOG_ERR("Invalid cipher key length");
+			AESNI_MB_LOG(ERR, "Invalid cipher key length");
 			return -EINVAL;
 		}
 		sess->cipher.key_length_in_bytes = 8;
@@ -294,7 +342,7 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
 		sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 		break;
 	default:
-		MB_LOG_ERR("Invalid aead operation parameter");
+		AESNI_MB_LOG(ERR, "Invalid aead operation parameter");
 		return -EINVAL;
 	}
 
@@ -304,7 +352,7 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
 		sess->auth.algo = AES_CCM;
 		break;
 	default:
-		MB_LOG_ERR("Unsupported aead mode parameter");
+		AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
 		return -ENOTSUP;
 	}
 
@@ -320,7 +368,7 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
 		aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
 		break;
 	default:
-		MB_LOG_ERR("Invalid cipher key length");
+		AESNI_MB_LOG(ERR, "Invalid cipher key length");
 		return -EINVAL;
 	}
 
@@ -391,7 +439,7 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
 		break;
 	case AESNI_MB_OP_NOT_SUPPORTED:
 	default:
-		MB_LOG_ERR("Unsupported operation chain order parameter");
+		AESNI_MB_LOG(ERR, "Unsupported operation chain order parameter");
 		return -ENOTSUP;
 	}
 
@@ -400,14 +448,14 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
 
 	ret = aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform);
 	if (ret != 0) {
-		MB_LOG_ERR("Invalid/unsupported authentication parameters");
+		AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
 		return ret;
 	}
 
 	ret = aesni_mb_set_session_cipher_parameters(mb_ops, sess,
 			cipher_xform);
 	if (ret != 0) {
-		MB_LOG_ERR("Invalid/unsupported cipher parameters");
+		AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
 		return ret;
 	}
 
@@ -415,7 +463,7 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
 		ret = aesni_mb_set_session_aead_parameters(mb_ops, sess,
 				aead_xform);
 		if (ret != 0) {
-			MB_LOG_ERR("Invalid/unsupported aead parameters");
+			AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
 			return ret;
 		}
 	}
@@ -458,7 +506,7 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
 	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 		if (likely(op->sym->session != NULL))
 			sess = (struct aesni_mb_session *)
-					get_session_private_data(
+					get_sym_session_private_data(
 					op->sym->session,
 					cryptodev_driver_id);
 	} else {
@@ -480,8 +528,8 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
 			sess = NULL;
 		}
 		op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
-		set_session_private_data(op->sym->session, cryptodev_driver_id,
-			_sess_private_data);
+		set_sym_session_private_data(op->sym->session,
+				cryptodev_driver_id, _sess_private_data);
 	}
 
 	if (unlikely(sess == NULL))
@@ -524,8 +572,20 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
 	job->cipher_mode = session->cipher.mode;
 
 	job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
-	job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
-	job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode;
+
+	if (job->cipher_mode == DES3) {
+		job->aes_enc_key_expanded =
+			session->cipher.exp_3des_keys.ks_ptr;
+		job->aes_dec_key_expanded =
+			session->cipher.exp_3des_keys.ks_ptr;
+	} else {
+		job->aes_enc_key_expanded =
+			session->cipher.expanded_aes_keys.encode;
+		job->aes_dec_key_expanded =
+			session->cipher.expanded_aes_keys.decode;
+	}
+
+
 
 
 	/* Set authentication parameters */
@@ -555,7 +615,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
 		char *odata = rte_pktmbuf_append(m_dst,
 				rte_pktmbuf_data_len(op->sym->m_src));
 		if (odata == NULL) {
-			MB_LOG_ERR("failed to allocate space in destination "
+			AESNI_MB_LOG(ERR, "failed to allocate space in destination "
 					"mbuf for source data");
 			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 			return -1;
@@ -658,7 +718,7 @@ static inline struct rte_crypto_op *
 post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
 {
 	struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
-	struct aesni_mb_session *sess = get_session_private_data(
+	struct aesni_mb_session *sess = get_sym_session_private_data(
 							op->sym->session,
 							cryptodev_driver_id);
 
@@ -721,7 +781,7 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
 		if (processed_jobs == nb_ops)
 			break;
 
-		job = (*qp->op_fns->job.get_completed_job)(&qp->mb_mgr);
+		job = (*qp->op_fns->job.get_completed_job)(qp->mb_mgr);
 	}
 
 	return processed_jobs;
@@ -734,7 +794,7 @@ flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
 	int processed_ops = 0;
 
 	/* Flush the remaining jobs */
-	JOB_AES_HMAC *job = (*qp->op_fns->job.flush_job)(&qp->mb_mgr);
+	JOB_AES_HMAC *job = (*qp->op_fns->job.flush_job)(qp->mb_mgr);
 
 	if (job)
 		processed_ops += handle_completed_jobs(qp, job,
@@ -779,14 +839,14 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 			break;
 
 		/* Get next free mb job struct from mb manager */
-		job = (*qp->op_fns->job.get_next)(&qp->mb_mgr);
+		job = (*qp->op_fns->job.get_next)(qp->mb_mgr);
 		if (unlikely(job == NULL)) {
 			/* if no free mb job structs we need to flush mb_mgr */
 			processed_jobs += flush_mb_mgr(qp,
 					&ops[processed_jobs],
 					(nb_ops - processed_jobs) - 1);
 
-			job = (*qp->op_fns->job.get_next)(&qp->mb_mgr);
+			job = (*qp->op_fns->job.get_next)(qp->mb_mgr);
 		}
 
 		retval = set_mb_job_params(job, qp, op, &digest_idx);
@@ -796,7 +856,7 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		}
 
 		/* Submit job to multi-buffer for processing */
-		job = (*qp->op_fns->job.submit)(&qp->mb_mgr);
+		job = (*qp->op_fns->job.submit)(qp->mb_mgr);
 
 		/*
 		 * If submit returns a processed job then handle it,
@@ -832,13 +892,13 @@ cryptodev_aesni_mb_create(const char *name,
 
 	/* Check CPU for support for AES instruction set */
 	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
-		MB_LOG_ERR("AES instructions not supported by CPU");
+		AESNI_MB_LOG(ERR, "AES instructions not supported by CPU");
 		return -EFAULT;
 	}
 
 	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
 	if (dev == NULL) {
-		MB_LOG_ERR("failed to create cryptodev vdev");
+		AESNI_MB_LOG(ERR, "failed to create cryptodev vdev");
 		return -ENODEV;
 	}
 
@@ -885,7 +945,13 @@ cryptodev_aesni_mb_create(const char *name,
 
 	internals->vector_mode = vector_mode;
 	internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
-	internals->max_nb_sessions = init_params->max_nb_sessions;
+
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+	AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
+			imb_get_version_str());
+#else
+	AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: 0.49.0\n");
+#endif
 
 	return 0;
 }
@@ -897,8 +963,7 @@ cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
 		"",
 		sizeof(struct aesni_mb_private),
 		rte_socket_id(),
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
+		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 	};
 	const char *name, *args;
 	int retval;
@@ -911,7 +976,7 @@ cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
 
 	retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
 	if (retval) {
-		MB_LOG_ERR("Failed to parse initialisation arguments[%s]\n",
+		AESNI_MB_LOG(ERR, "Failed to parse initialisation arguments[%s]",
 				args);
 		return -EINVAL;
 	}
@@ -947,8 +1012,12 @@ RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
 	"max_nb_queue_pairs=<int> "
-	"max_nb_sessions=<int> "
 	"socket_id=<int>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
 		cryptodev_aesni_mb_pmd_drv.driver,
 		cryptodev_driver_id);
+
+RTE_INIT(aesni_mb_init_log)
+{
+	aesni_mb_logtype_driver = rte_log_register("pmd.crypto.aesni_mb");
+}
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
index 01530523..ab26e5ae 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
@@ -239,6 +239,26 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
 			}, }
 		}, }
 	},
+	{	/*  3DES CBC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_3DES_CBC,
+				.block_size = 8,
+				.key_size = {
+					.min = 8,
+					.max = 24,
+					.increment = 8
+				},
+				.iv_size = {
+					.min = 8,
+					.max = 8,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
 	{	/* DES DOCSIS BPI */
 		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 		{.sym = {
@@ -387,7 +407,8 @@ aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
 		dev_info->feature_flags = dev->feature_flags;
 		dev_info->capabilities = aesni_mb_pmd_capabilities;
 		dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
-		dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
+		/* No limit of number of sessions */
+		dev_info->sym.max_nb_sessions = 0;
 	}
 }
 
@@ -402,6 +423,8 @@ aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 		r = rte_ring_lookup(qp->name);
 		if (r)
 			rte_ring_free(r);
+		if (qp->mb_mgr)
+			free_mb_mgr(qp->mb_mgr);
 		rte_free(qp);
 		dev->data->queue_pairs[qp_id] = NULL;
 	}
@@ -441,12 +464,12 @@ aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
 	r = rte_ring_lookup(ring_name);
 	if (r) {
 		if (rte_ring_get_size(r) >= ring_size) {
-			MB_LOG_INFO("Reusing existing ring %s for processed ops",
+			AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops",
 			ring_name);
 			return r;
 		}
 
-		MB_LOG_ERR("Unable to reuse existing ring %s for processed ops",
+		AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
 			ring_name);
 		return NULL;
 	}
@@ -463,6 +486,7 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 {
 	struct aesni_mb_qp *qp = NULL;
 	struct aesni_mb_private *internals = dev->data->dev_private;
+	int ret = -1;
 
 	/* Free memory prior to re-allocation if needed. */
 	if (dev->data->queue_pairs[qp_id] != NULL)
@@ -481,12 +505,20 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		goto qp_setup_cleanup;
 
 
+	qp->mb_mgr = alloc_mb_mgr(0);
+	if (qp->mb_mgr == NULL) {
+		ret = -ENOMEM;
+		goto qp_setup_cleanup;
+	}
+
 	qp->op_fns = &job_ops[internals->vector_mode];
 
 	qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
 			"ingress", qp_conf->nb_descriptors, socket_id);
-	if (qp->ingress_queue == NULL)
+	if (qp->ingress_queue == NULL) {
+		ret = -1;
 		goto qp_setup_cleanup;
+	}
 
 	qp->sess_mp = session_pool;
 
@@ -498,30 +530,17 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 				"digest_mp_%u_%u", dev->data->dev_id, qp_id);
 
 	/* Initialise multi-buffer manager */
-	(*qp->op_fns->job.init_mgr)(&qp->mb_mgr);
+	(*qp->op_fns->job.init_mgr)(qp->mb_mgr);
 	return 0;
 
 qp_setup_cleanup:
-	if (qp)
+	if (qp) {
+		if (qp->mb_mgr == NULL)
+			free_mb_mgr(qp->mb_mgr);
 		rte_free(qp);
+	}
 
-	return -1;
-}
-
-/** Start queue pair */
-static int
-aesni_mb_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
-/** Stop queue pair */
-static int
-aesni_mb_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
+	return ret;
 }
 
 /** Return the number of allocated queue pairs */
@@ -533,14 +552,14 @@ aesni_mb_pmd_qp_count(struct rte_cryptodev *dev)
 
 /** Returns the size of the aesni multi-buffer session structure */
 static unsigned
-aesni_mb_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
+aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	return sizeof(struct aesni_mb_session);
 }
 
 /** Configure a aesni multi-buffer session from a crypto xform chain */
 static int
-aesni_mb_pmd_session_configure(struct rte_cryptodev *dev,
+aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *sess,
 		struct rte_mempool *mempool)
@@ -550,27 +569,27 @@ aesni_mb_pmd_session_configure(struct rte_cryptodev *dev,
 	int ret;
 
 	if (unlikely(sess == NULL)) {
-		MB_LOG_ERR("invalid session struct");
+		AESNI_MB_LOG(ERR, "invalid session struct");
 		return -EINVAL;
 	}
 
 	if (rte_mempool_get(mempool, &sess_private_data)) {
-		CDEV_LOG_ERR(
-			"Couldn't get object from session mempool");
+		AESNI_MB_LOG(ERR,
+				"Couldn't get object from session mempool");
 		return -ENOMEM;
 	}
 
 	ret = aesni_mb_set_session_parameters(&job_ops[internals->vector_mode],
 			sess_private_data, xform);
 	if (ret != 0) {
-		MB_LOG_ERR("failed configure session parameters");
+		AESNI_MB_LOG(ERR, "failed configure session parameters");
 
 		/* Return session to mempool */
 		rte_mempool_put(mempool, sess_private_data);
 		return ret;
 	}
 
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 			sess_private_data);
 
 	return 0;
@@ -578,17 +597,17 @@ aesni_mb_pmd_session_configure(struct rte_cryptodev *dev,
 
 /** Clear the memory of session so it doesn't leave key material behind */
 static void
-aesni_mb_pmd_session_clear(struct rte_cryptodev *dev,
+aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 
 	/* Zero out the whole structure */
 	if (sess_priv) {
 		memset(sess_priv, 0, sizeof(struct aesni_mb_session));
 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
 		rte_mempool_put(sess_mp, sess_priv);
 	}
 }
@@ -606,13 +625,11 @@ struct rte_cryptodev_ops aesni_mb_pmd_ops = {
 
 		.queue_pair_setup	= aesni_mb_pmd_qp_setup,
 		.queue_pair_release	= aesni_mb_pmd_qp_release,
-		.queue_pair_start	= aesni_mb_pmd_qp_start,
-		.queue_pair_stop	= aesni_mb_pmd_qp_stop,
 		.queue_pair_count	= aesni_mb_pmd_qp_count,
 
-		.session_get_size	= aesni_mb_pmd_session_get_size,
-		.session_configure	= aesni_mb_pmd_session_configure,
-		.session_clear		= aesni_mb_pmd_session_clear
+		.sym_session_get_size	= aesni_mb_pmd_sym_session_get_size,
+		.sym_session_configure	= aesni_mb_pmd_sym_session_configure,
+		.sym_session_clear	= aesni_mb_pmd_sym_session_clear
 };
 
 struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
index a33b2f69..70e9d18e 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
@@ -7,28 +7,26 @@
 
 #include "aesni_mb_ops.h"
 
+/*
+ * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50,
+ * so if macro is not defined, it means that the version is 0.49.
+ */
+#if !defined(IMB_VERSION_NUM)
+#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
+#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0)
+#endif
+
 #define CRYPTODEV_NAME_AESNI_MB_PMD	crypto_aesni_mb
 /**< AES-NI Multi buffer PMD device name */
 
-#define MB_LOG_ERR(fmt, args...) \
-	RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",  \
-			RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD), \
-			__func__, __LINE__, ## args)
-
-#ifdef RTE_LIBRTE_AESNI_MB_DEBUG
-#define MB_LOG_INFO(fmt, args...) \
-	RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			CRYPTODEV_NAME_AESNI_MB_PMD, \
-			__func__, __LINE__, ## args)
-
-#define MB_LOG_DBG(fmt, args...) \
-	RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			CRYPTODEV_NAME_AESNI_MB_PMD, \
-			__func__, __LINE__, ## args)
-#else
-#define MB_LOG_INFO(fmt, args...)
-#define MB_LOG_DBG(fmt, args...)
-#endif
+/** AESNI_MB PMD LOGTYPE DRIVER */
+int aesni_mb_logtype_driver;
+
+#define AESNI_MB_LOG(level, fmt, ...)  \
+	rte_log(RTE_LOG_ ## level, aesni_mb_logtype_driver,  \
+			"%s() line %u: " fmt "\n", __func__, __LINE__,  \
+					## __VA_ARGS__)
+
 
 #define HMAC_IPAD_VALUE			(0x36)
 #define HMAC_OPAD_VALUE			(0x5C)
@@ -124,8 +122,6 @@ struct aesni_mb_private {
 	/**< CPU vector instruction set mode */
 	unsigned max_nb_queue_pairs;
 	/**< Max number of queue pairs supported by device */
-	unsigned max_nb_sessions;
-	/**< Max number of sessions supported by device */
 };
 
 /** AESNI Multi buffer queue pair */
@@ -136,7 +132,7 @@ struct aesni_mb_qp {
 	/**< Unique Queue Pair Name */
 	const struct aesni_mb_op_fns *op_fns;
 	/**< Vector mode dependent pointer table of the multi-buffer APIs */
-	MB_MGR mb_mgr;
+	MB_MGR *mb_mgr;
 	/**< Multi-buffer instance */
 	struct rte_ring *ingress_queue;
        /**< Ring for placing operations ready for processing */
@@ -173,12 +169,18 @@ struct aesni_mb_session {
 
 		uint64_t key_length_in_bytes;
 
-		struct {
-			uint32_t encode[60] __rte_aligned(16);
-			/**< encode key */
-			uint32_t decode[60] __rte_aligned(16);
-			/**< decode key */
-		} expanded_aes_keys;
+		union {
+			struct {
+				uint32_t encode[60] __rte_aligned(16);
+				/**< encode key */
+				uint32_t decode[60] __rte_aligned(16);
+				/**< decode key */
+			} expanded_aes_keys;
+			struct {
+				const void *ks_ptr[3];
+				uint64_t key[3][16];
+			} exp_3des_keys;
+		};
 		/**< Expanded AES keys - Allocating space to
 		 * contain the maximum expanded key size which
 		 * is 240 bytes for 256 bit AES, calculate by:
diff --git a/drivers/crypto/armv8/rte_armv8_pmd.c b/drivers/crypto/armv8/rte_armv8_pmd.c
index fbb08f72..9d15fee5 100644
--- a/drivers/crypto/armv8/rte_armv8_pmd.c
+++ b/drivers/crypto/armv8/rte_armv8_pmd.c
@@ -502,7 +502,7 @@ get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op)
 		/* get existing session */
 		if (likely(op->sym->session != NULL)) {
 			sess = (struct armv8_crypto_session *)
-					get_session_private_data(
+					get_sym_session_private_data(
 					op->sym->session,
 					cryptodev_driver_id);
 		}
@@ -526,8 +526,8 @@ get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op)
 			sess = NULL;
 		}
 		op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
-		set_session_private_data(op->sym->session, cryptodev_driver_id,
-			_sess_private_data);
+		set_sym_session_private_data(op->sym->session,
+				cryptodev_driver_id, _sess_private_data);
 	}
 
 	if (unlikely(sess == NULL))
@@ -779,7 +779,6 @@ cryptodev_armv8_crypto_create(const char *name,
 	internals = dev->data->dev_private;
 
 	internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
-	internals->max_nb_sessions = init_params->max_nb_sessions;
 
 	return 0;
 
@@ -800,8 +799,7 @@ cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev)
 		"",
 		sizeof(struct armv8_crypto_private),
 		rte_socket_id(),
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
+		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 	};
 	const char *name;
 	const char *input_args;
@@ -848,7 +846,6 @@ RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_pmd_drv);
 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD,
 	"max_nb_queue_pairs=<int> "
-	"max_nb_sessions=<int> "
 	"socket_id=<int>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, armv8_crypto_pmd_drv.driver,
 		cryptodev_driver_id);
diff --git a/drivers/crypto/armv8/rte_armv8_pmd_ops.c b/drivers/crypto/armv8/rte_armv8_pmd_ops.c
index c64aef09..ae03117e 100644
--- a/drivers/crypto/armv8/rte_armv8_pmd_ops.c
+++ b/drivers/crypto/armv8/rte_armv8_pmd_ops.c
@@ -154,7 +154,8 @@ armv8_crypto_pmd_info_get(struct rte_cryptodev *dev,
 		dev_info->feature_flags = dev->feature_flags;
 		dev_info->capabilities = armv8_crypto_pmd_capabilities;
 		dev_info->max_nb_queue_pairs = internals->max_nb_qpairs;
-		dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
+		/* No limit of number of sessions */
+		dev_info->sym.max_nb_sessions = 0;
 	}
 }
 
@@ -257,22 +258,6 @@ qp_setup_cleanup:
 	return -1;
 }
 
-/** Start queue pair */
-static int
-armv8_crypto_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
-/** Stop queue pair */
-static int
-armv8_crypto_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
 /** Return the number of allocated queue pairs */
 static uint32_t
 armv8_crypto_pmd_qp_count(struct rte_cryptodev *dev)
@@ -282,14 +267,14 @@ armv8_crypto_pmd_qp_count(struct rte_cryptodev *dev)
 
 /** Returns the size of the session structure */
 static unsigned
-armv8_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
+armv8_crypto_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	return sizeof(struct armv8_crypto_session);
 }
 
 /** Configure the session from a crypto xform chain */
 static int
-armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev,
+armv8_crypto_pmd_sym_session_configure(struct rte_cryptodev *dev,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *sess,
 		struct rte_mempool *mempool)
@@ -317,7 +302,7 @@ armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev,
 		return ret;
 	}
 
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 			sess_private_data);
 
 	return 0;
@@ -325,17 +310,17 @@ armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev,
 
 /** Clear the memory of session so it doesn't leave key material behind */
 static void
-armv8_crypto_pmd_session_clear(struct rte_cryptodev *dev,
+armv8_crypto_pmd_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 
 	/* Zero out the whole structure */
 	if (sess_priv) {
 		memset(sess_priv, 0, sizeof(struct armv8_crypto_session));
 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
 		rte_mempool_put(sess_mp, sess_priv);
 	}
 }
@@ -353,13 +338,11 @@ struct rte_cryptodev_ops armv8_crypto_pmd_ops = {
 
 		.queue_pair_setup	= armv8_crypto_pmd_qp_setup,
 		.queue_pair_release	= armv8_crypto_pmd_qp_release,
-		.queue_pair_start	= armv8_crypto_pmd_qp_start,
-		.queue_pair_stop	= armv8_crypto_pmd_qp_stop,
 		.queue_pair_count	= armv8_crypto_pmd_qp_count,
 
-		.session_get_size	= armv8_crypto_pmd_session_get_size,
-		.session_configure	= armv8_crypto_pmd_session_configure,
-		.session_clear		= armv8_crypto_pmd_session_clear
+		.sym_session_get_size	= armv8_crypto_pmd_sym_session_get_size,
+		.sym_session_configure	= armv8_crypto_pmd_sym_session_configure,
+		.sym_session_clear	= armv8_crypto_pmd_sym_session_clear
 };
 
 struct rte_cryptodev_ops *rte_armv8_crypto_pmd_ops = &armv8_crypto_pmd_ops;
diff --git a/drivers/crypto/armv8/rte_armv8_pmd_private.h b/drivers/crypto/armv8/rte_armv8_pmd_private.h
index b8966e93..7feb021d 100644
--- a/drivers/crypto/armv8/rte_armv8_pmd_private.h
+++ b/drivers/crypto/armv8/rte_armv8_pmd_private.h
@@ -106,8 +106,6 @@ typedef void (*crypto_key_sched_t)(uint8_t *, const uint8_t *);
 struct armv8_crypto_private {
 	unsigned int max_nb_qpairs;
 	/**< Max number of queue pairs */
-	unsigned int max_nb_sessions;
-	/**< Max number of sessions */
 };
 
 /** ARMv8 crypto queue pair */
diff --git a/drivers/crypto/ccp/ccp_crypto.c b/drivers/crypto/ccp/ccp_crypto.c
index 3ce0f39f..19ae9153 100644
--- a/drivers/crypto/ccp/ccp_crypto.c
+++ b/drivers/crypto/ccp/ccp_crypto.c
@@ -1566,7 +1566,7 @@ ccp_perform_hmac(struct rte_crypto_op *op,
 	void *append_ptr;
 	uint8_t *addr;
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					 op->sym->session,
 					 ccp_cryptodev_driver_id);
 	addr = session->auth.pre_compute;
@@ -1739,7 +1739,7 @@ ccp_perform_sha(struct rte_crypto_op *op,
 	void *append_ptr;
 	uint64_t auth_msg_bits;
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					 op->sym->session,
 					ccp_cryptodev_driver_id);
 
@@ -1828,7 +1828,7 @@ ccp_perform_sha3_hmac(struct rte_crypto_op *op,
 	uint32_t tail;
 	phys_addr_t src_addr, dest_addr, ctx_paddr, dest_addr_t;
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					 op->sym->session,
 					ccp_cryptodev_driver_id);
 
@@ -1968,7 +1968,7 @@ ccp_perform_sha3(struct rte_crypto_op *op,
 	uint32_t tail;
 	phys_addr_t src_addr, dest_addr, ctx_paddr;
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					 op->sym->session,
 					ccp_cryptodev_driver_id);
 
@@ -2036,7 +2036,7 @@ ccp_perform_aes_cmac(struct rte_crypto_op *op,
 	phys_addr_t src_addr, dest_addr, key_addr;
 	int length, non_align_len;
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					 op->sym->session,
 					ccp_cryptodev_driver_id);
 	key_addr = rte_mem_virt2phy(session->auth.key_ccp);
@@ -2188,7 +2188,7 @@ ccp_perform_aes(struct rte_crypto_op *op,
 	phys_addr_t src_addr, dest_addr, key_addr;
 	uint8_t *iv;
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					 op->sym->session,
 					ccp_cryptodev_driver_id);
 	function.raw = 0;
@@ -2276,7 +2276,7 @@ ccp_perform_3des(struct rte_crypto_op *op,
 	uint8_t *iv;
 	phys_addr_t src_addr, dest_addr, key_addr;
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					 op->sym->session,
 					ccp_cryptodev_driver_id);
 
@@ -2379,7 +2379,7 @@ ccp_perform_aes_gcm(struct rte_crypto_op *op, struct ccp_queue *cmd_q)
 	phys_addr_t digest_dest_addr;
 	int length, non_align_len;
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					 op->sym->session,
 					 ccp_cryptodev_driver_id);
 	iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
@@ -2546,7 +2546,7 @@ ccp_crypto_cipher(struct rte_crypto_op *op,
 	int result = 0;
 	struct ccp_session *session;
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					 op->sym->session,
 					 ccp_cryptodev_driver_id);
 
@@ -2584,7 +2584,7 @@ ccp_crypto_auth(struct rte_crypto_op *op,
 	int result = 0;
 	struct ccp_session *session;
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					 op->sym->session,
 					ccp_cryptodev_driver_id);
 
@@ -2654,7 +2654,7 @@ ccp_crypto_aead(struct rte_crypto_op *op,
 	int result = 0;
 	struct ccp_session *session;
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					op->sym->session,
 					ccp_cryptodev_driver_id);
 
@@ -2711,7 +2711,7 @@ process_ops_to_enqueue(struct ccp_qp *qp,
 	b_info->head_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx *
 					 Q_DESC_SIZE);
 	for (i = 0; i < nb_ops; i++) {
-		session = (struct ccp_session *)get_session_private_data(
+		session = (struct ccp_session *)get_sym_session_private_data(
 						 op[i]->sym->session,
 						 ccp_cryptodev_driver_id);
 		switch (session->cmd_id) {
@@ -2787,7 +2787,7 @@ static inline void ccp_auth_dq_prepare(struct rte_crypto_op *op)
 	int offset, digest_offset;
 	uint8_t digest_le[64];
 
-	session = (struct ccp_session *)get_session_private_data(
+	session = (struct ccp_session *)get_sym_session_private_data(
 					 op->sym->session,
 					ccp_cryptodev_driver_id);
 
@@ -2863,7 +2863,7 @@ ccp_prepare_ops(struct ccp_qp *qp,
 
 	for (i = 0; i < min_ops; i++) {
 		op_d[i] = b_info->op[b_info->op_idx++];
-		session = (struct ccp_session *)get_session_private_data(
+		session = (struct ccp_session *)get_sym_session_private_data(
 						 op_d[i]->sym->session,
 						ccp_cryptodev_driver_id);
 		switch (session->cmd_id) {
diff --git a/drivers/crypto/ccp/ccp_pmd_ops.c b/drivers/crypto/ccp/ccp_pmd_ops.c
index 80b75ccb..6984913f 100644
--- a/drivers/crypto/ccp/ccp_pmd_ops.c
+++ b/drivers/crypto/ccp/ccp_pmd_ops.c
@@ -624,7 +624,8 @@ ccp_pmd_info_get(struct rte_cryptodev *dev,
 		if (internals->auth_opt == 1)
 			dev_info->capabilities = ccp_crypto_cap_complete;
 		dev_info->max_nb_queue_pairs = internals->max_nb_qpairs;
-		dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
+		/* No limit of number of sessions */
+		dev_info->sym.max_nb_sessions = 0;
 	}
 }
 
@@ -747,20 +748,6 @@ qp_setup_cleanup:
 	return -1;
 }
 
-static int
-ccp_pmd_qp_start(struct rte_cryptodev *dev __rte_unused,
-		 uint16_t queue_pair_id __rte_unused)
-{
-	return -ENOTSUP;
-}
-
-static int
-ccp_pmd_qp_stop(struct rte_cryptodev *dev __rte_unused,
-		uint16_t queue_pair_id __rte_unused)
-{
-	return -ENOTSUP;
-}
-
 static uint32_t
 ccp_pmd_qp_count(struct rte_cryptodev *dev)
 {
@@ -768,13 +755,13 @@ ccp_pmd_qp_count(struct rte_cryptodev *dev)
 }
 
 static unsigned
-ccp_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
+ccp_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	return sizeof(struct ccp_session);
 }
 
 static int
-ccp_pmd_session_configure(struct rte_cryptodev *dev,
+ccp_pmd_sym_session_configure(struct rte_cryptodev *dev,
 			  struct rte_crypto_sym_xform *xform,
 			  struct rte_cryptodev_sym_session *sess,
 			  struct rte_mempool *mempool)
@@ -801,25 +788,25 @@ ccp_pmd_session_configure(struct rte_cryptodev *dev,
 		rte_mempool_put(mempool, sess_private_data);
 		return ret;
 	}
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 				 sess_private_data);
 
 	return 0;
 }
 
 static void
-ccp_pmd_session_clear(struct rte_cryptodev *dev,
+ccp_pmd_sym_session_clear(struct rte_cryptodev *dev,
 		      struct rte_cryptodev_sym_session *sess)
 {
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 
 	if (sess_priv) {
 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 
 		rte_mempool_put(sess_mp, sess_priv);
 		memset(sess_priv, 0, sizeof(struct ccp_session));
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
 	}
 }
 
@@ -836,13 +823,11 @@ struct rte_cryptodev_ops ccp_ops = {
 
 		.queue_pair_setup	= ccp_pmd_qp_setup,
 		.queue_pair_release	= ccp_pmd_qp_release,
-		.queue_pair_start	= ccp_pmd_qp_start,
-		.queue_pair_stop	= ccp_pmd_qp_stop,
 		.queue_pair_count	= ccp_pmd_qp_count,
 
-		.session_get_size	= ccp_pmd_session_get_size,
-		.session_configure	= ccp_pmd_session_configure,
-		.session_clear		= ccp_pmd_session_clear,
+		.sym_session_get_size	= ccp_pmd_sym_session_get_size,
+		.sym_session_configure	= ccp_pmd_sym_session_configure,
+		.sym_session_clear	= ccp_pmd_sym_session_clear,
 };
 
 struct rte_cryptodev_ops *ccp_pmd_ops = &ccp_ops;
diff --git a/drivers/crypto/ccp/ccp_pmd_private.h b/drivers/crypto/ccp/ccp_pmd_private.h
index f4498048..79752f68 100644
--- a/drivers/crypto/ccp/ccp_pmd_private.h
+++ b/drivers/crypto/ccp/ccp_pmd_private.h
@@ -40,7 +40,6 @@
 /* private data structure for each CCP crypto device */
 struct ccp_private {
 	unsigned int max_nb_qpairs;	/**< Max number of queue pairs */
-	unsigned int max_nb_sessions;	/**< Max number of sessions */
 	uint8_t crypto_num_dev;		/**< Number of working crypto devices */
 	bool auth_opt;			/**< Authentication offload option */
 	struct ccp_device *last_dev;	/**< Last working crypto device */
diff --git a/drivers/crypto/ccp/rte_ccp_pmd.c b/drivers/crypto/ccp/rte_ccp_pmd.c
index 2061f465..92d8a955 100644
--- a/drivers/crypto/ccp/rte_ccp_pmd.c
+++ b/drivers/crypto/ccp/rte_ccp_pmd.c
@@ -30,14 +30,12 @@ struct ccp_pmd_init_params {
 #define CCP_CRYPTODEV_PARAM_NAME		("name")
 #define CCP_CRYPTODEV_PARAM_SOCKET_ID		("socket_id")
 #define CCP_CRYPTODEV_PARAM_MAX_NB_QP		("max_nb_queue_pairs")
-#define CCP_CRYPTODEV_PARAM_MAX_NB_SESS		("max_nb_sessions")
 #define CCP_CRYPTODEV_PARAM_AUTH_OPT		("ccp_auth_opt")
 
 const char *ccp_pmd_valid_params[] = {
 	CCP_CRYPTODEV_PARAM_NAME,
 	CCP_CRYPTODEV_PARAM_SOCKET_ID,
 	CCP_CRYPTODEV_PARAM_MAX_NB_QP,
-	CCP_CRYPTODEV_PARAM_MAX_NB_SESS,
 	CCP_CRYPTODEV_PARAM_AUTH_OPT,
 };
 
@@ -125,13 +123,6 @@ ccp_pmd_parse_input_args(struct ccp_pmd_init_params *params,
 			goto free_kvlist;
 
 		ret = rte_kvargs_process(kvlist,
-					 CCP_CRYPTODEV_PARAM_MAX_NB_SESS,
-					 &parse_integer_arg,
-					 &params->def_p.max_nb_sessions);
-		if (ret < 0)
-			goto free_kvlist;
-
-		ret = rte_kvargs_process(kvlist,
 					 CCP_CRYPTODEV_PARAM_SOCKET_ID,
 					 &parse_integer_arg,
 					 &params->def_p.socket_id);
@@ -169,7 +160,7 @@ get_ccp_session(struct ccp_qp *qp, struct rte_crypto_op *op)
 			return NULL;
 
 		sess = (struct ccp_session *)
-			get_session_private_data(
+			get_sym_session_private_data(
 				op->sym->session,
 				ccp_cryptodev_driver_id);
 	} else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
@@ -192,7 +183,7 @@ get_ccp_session(struct ccp_qp *qp, struct rte_crypto_op *op)
 			sess = NULL;
 		}
 		op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
-		set_session_private_data(op->sym->session,
+		set_sym_session_private_data(op->sym->session,
 					 ccp_cryptodev_driver_id,
 					 _sess_private_data);
 	}
@@ -334,7 +325,6 @@ cryptodev_ccp_create(const char *name,
 	internals = dev->data->dev_private;
 
 	internals->max_nb_qpairs = init_params->def_p.max_nb_queue_pairs;
-	internals->max_nb_sessions = init_params->def_p.max_nb_sessions;
 	internals->auth_opt = init_params->auth_opt;
 	internals->crypto_num_dev = cryptodev_cnt;
 
@@ -359,8 +349,7 @@ cryptodev_ccp_probe(struct rte_vdev_device *vdev)
 			"",
 			sizeof(struct ccp_private),
 			rte_socket_id(),
-			CCP_PMD_MAX_QUEUE_PAIRS,
-			RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
+			CCP_PMD_MAX_QUEUE_PAIRS
 		},
 		.auth_opt = CCP_PMD_AUTH_OPT_CCP,
 	};
@@ -382,8 +371,6 @@ cryptodev_ccp_probe(struct rte_vdev_device *vdev)
 		init_params.def_p.socket_id);
 	RTE_LOG(INFO, PMD, "Max number of queue pairs = %d\n",
 		init_params.def_p.max_nb_queue_pairs);
-	RTE_LOG(INFO, PMD, "Max number of sessions = %d\n",
-		init_params.def_p.max_nb_sessions);
 	RTE_LOG(INFO, PMD, "Authentication offload to %s\n",
 		((init_params.auth_opt == 0) ? "CCP" : "CPU"));
 
@@ -404,7 +391,6 @@ static struct cryptodev_driver ccp_crypto_drv;
 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CCP_PMD, cryptodev_ccp_pmd_drv);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CCP_PMD,
 	"max_nb_queue_pairs=<int> "
-	"max_nb_sessions=<int> "
 	"socket_id=<int> "
 	"ccp_auth_opt=<int>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(ccp_crypto_drv, cryptodev_ccp_pmd_drv.driver,
diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
index 56fa969d..2a3c61c6 100644
--- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
+++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
@@ -1080,7 +1080,7 @@ build_sec_fd(struct rte_crypto_op *op,
 	PMD_INIT_FUNC_TRACE();
 
 	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
-		sess = (dpaa2_sec_session *)get_session_private_data(
+		sess = (dpaa2_sec_session *)get_sym_session_private_data(
 				op->sym->session, cryptodev_driver_id);
 	else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
 		sess = (dpaa2_sec_session *)get_sec_session_private_data(
@@ -1470,26 +1470,6 @@ dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return retcode;
 }
 
-/** Start queue pair */
-static int
-dpaa2_sec_queue_pair_start(__rte_unused struct rte_cryptodev *dev,
-			   __rte_unused uint16_t queue_pair_id)
-{
-	PMD_INIT_FUNC_TRACE();
-
-	return 0;
-}
-
-/** Stop queue pair */
-static int
-dpaa2_sec_queue_pair_stop(__rte_unused struct rte_cryptodev *dev,
-			  __rte_unused uint16_t queue_pair_id)
-{
-	PMD_INIT_FUNC_TRACE();
-
-	return 0;
-}
-
 /** Return the number of allocated queue pairs */
 static uint32_t
 dpaa2_sec_queue_pair_count(struct rte_cryptodev *dev)
@@ -1501,7 +1481,7 @@ dpaa2_sec_queue_pair_count(struct rte_cryptodev *dev)
 
 /** Returns the size of the aesni gcm session structure */
 static unsigned int
-dpaa2_sec_session_get_size(struct rte_cryptodev *dev __rte_unused)
+dpaa2_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	PMD_INIT_FUNC_TRACE();
 
@@ -2456,7 +2436,7 @@ dpaa2_sec_security_session_destroy(void *dev __rte_unused,
 }
 
 static int
-dpaa2_sec_session_configure(struct rte_cryptodev *dev,
+dpaa2_sec_sym_session_configure(struct rte_cryptodev *dev,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *sess,
 		struct rte_mempool *mempool)
@@ -2477,7 +2457,7 @@ dpaa2_sec_session_configure(struct rte_cryptodev *dev,
 		return ret;
 	}
 
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 		sess_private_data);
 
 	return 0;
@@ -2485,12 +2465,12 @@ dpaa2_sec_session_configure(struct rte_cryptodev *dev,
 
 /** Clear the memory of session so it doesn't leave key material behind */
 static void
-dpaa2_sec_session_clear(struct rte_cryptodev *dev,
+dpaa2_sec_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 	PMD_INIT_FUNC_TRACE();
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 	dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
 
 	if (sess_priv) {
@@ -2499,7 +2479,7 @@ dpaa2_sec_session_clear(struct rte_cryptodev *dev,
 		rte_free(s->auth_key.data);
 		memset(sess, 0, sizeof(dpaa2_sec_session));
 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
 		rte_mempool_put(sess_mp, sess_priv);
 	}
 }
@@ -2626,7 +2606,8 @@ dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev,
 		info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
 		info->feature_flags = dev->feature_flags;
 		info->capabilities = dpaa2_sec_capabilities;
-		info->sym.max_nb_sessions = internals->max_nb_sessions;
+		/* No limit of number of sessions */
+		info->sym.max_nb_sessions = 0;
 		info->driver_id = cryptodev_driver_id;
 	}
 }
@@ -2715,12 +2696,10 @@ static struct rte_cryptodev_ops crypto_ops = {
 	.stats_reset	      = dpaa2_sec_stats_reset,
 	.queue_pair_setup     = dpaa2_sec_queue_pair_setup,
 	.queue_pair_release   = dpaa2_sec_queue_pair_release,
-	.queue_pair_start     = dpaa2_sec_queue_pair_start,
-	.queue_pair_stop      = dpaa2_sec_queue_pair_stop,
 	.queue_pair_count     = dpaa2_sec_queue_pair_count,
-	.session_get_size     = dpaa2_sec_session_get_size,
-	.session_configure    = dpaa2_sec_session_configure,
-	.session_clear        = dpaa2_sec_session_clear,
+	.sym_session_get_size     = dpaa2_sec_sym_session_get_size,
+	.sym_session_configure    = dpaa2_sec_sym_session_configure,
+	.sym_session_clear        = dpaa2_sec_sym_session_clear,
 };
 
 static const struct rte_security_capability *
@@ -2783,10 +2762,13 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev)
 			RTE_CRYPTODEV_FF_HW_ACCELERATED |
 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 			RTE_CRYPTODEV_FF_SECURITY |
-			RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
+			RTE_CRYPTODEV_FF_IN_PLACE_SGL |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
 
 	internals = cryptodev->data->dev_private;
-	internals->max_nb_sessions = RTE_DPAA2_SEC_PMD_MAX_NB_SESSIONS;
 
 	/*
 	 * For secondary processes, we don't initialise any further as primary
@@ -2940,9 +2922,7 @@ RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver);
 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa2_sec_crypto_drv,
 		rte_dpaa2_sec_driver.driver, cryptodev_driver_id);
 
-RTE_INIT(dpaa2_sec_init_log);
-static void
-dpaa2_sec_init_log(void)
+RTE_INIT(dpaa2_sec_init_log)
 {
 	/* Bus level logs */
 	dpaa2_logtype_sec = rte_log_register("pmd.crypto.dpaa2");
diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h b/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h
index 7c1f5e73..8a990442 100644
--- a/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h
+++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h
@@ -18,7 +18,7 @@ extern int dpaa2_logtype_sec;
 	rte_log(RTE_LOG_DEBUG, dpaa2_logtype_sec, "dpaa2_sec: %s(): " \
 		fmt "\n", __func__, ##args)
 
-#define PMD_INIT_FUNC_TRACE() DPAA2_SEC_LOG(DEBUG, " >>")
+#define PMD_INIT_FUNC_TRACE() DPAA2_SEC_DEBUG(">>")
 
 #define DPAA2_SEC_INFO(fmt, args...) \
 	DPAA2_SEC_LOG(INFO, fmt, ## args)
diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h
index a9d83ebc..d015be1e 100644
--- a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h
+++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h
@@ -23,8 +23,6 @@ struct dpaa2_sec_dev_private {
 	uint16_t token; /**< Token required by DPxxx objects */
 	unsigned int max_nb_queue_pairs;
 	/**< Max number of queue pairs supported by device */
-	unsigned int max_nb_sessions;
-	/**< Max number of sessions supported by device */
 };
 
 struct dpaa2_sec_qp {
diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.c b/drivers/crypto/dpaa_sec/dpaa_sec.c
index 06f7e437..f571050b 100644
--- a/drivers/crypto/dpaa_sec/dpaa_sec.c
+++ b/drivers/crypto/dpaa_sec/dpaa_sec.c
@@ -526,12 +526,25 @@ dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops)
 {
 	struct qman_fq *fq;
 	unsigned int pkts = 0;
-	int ret;
+	int num_rx_bufs, ret;
 	struct qm_dqrr_entry *dq;
+	uint32_t vdqcr_flags = 0;
 
 	fq = &qp->outq;
-	ret = qman_set_vdq(fq, (nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES) ?
-				DPAA_MAX_DEQUEUE_NUM_FRAMES : nb_ops);
+	/*
+	 * Until request for four buffers, we provide exact number of buffers.
+	 * Otherwise we do not set the QM_VDQCR_EXACT flag.
+	 * Not setting QM_VDQCR_EXACT flag can provide two more buffers than
+	 * requested, so we request two less in this case.
+	 */
+	if (nb_ops < 4) {
+		vdqcr_flags = QM_VDQCR_EXACT;
+		num_rx_bufs = nb_ops;
+	} else {
+		num_rx_bufs = nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES ?
+			(DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_ops - 2);
+	}
+	ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags);
 	if (ret)
 		return 0;
 
@@ -1416,7 +1429,7 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
 			switch (op->sess_type) {
 			case RTE_CRYPTO_OP_WITH_SESSION:
 				ses = (dpaa_sec_session *)
-					get_session_private_data(
+					get_sym_session_private_data(
 							op->sym->session,
 							cryptodev_driver_id);
 				break;
@@ -1585,26 +1598,6 @@ dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return 0;
 }
 
-/** Start queue pair */
-static int
-dpaa_sec_queue_pair_start(__rte_unused struct rte_cryptodev *dev,
-			  __rte_unused uint16_t queue_pair_id)
-{
-	PMD_INIT_FUNC_TRACE();
-
-	return 0;
-}
-
-/** Stop queue pair */
-static int
-dpaa_sec_queue_pair_stop(__rte_unused struct rte_cryptodev *dev,
-			 __rte_unused uint16_t queue_pair_id)
-{
-	PMD_INIT_FUNC_TRACE();
-
-	return 0;
-}
-
 /** Return the number of allocated queue pairs */
 static uint32_t
 dpaa_sec_queue_pair_count(struct rte_cryptodev *dev)
@@ -1616,7 +1609,7 @@ dpaa_sec_queue_pair_count(struct rte_cryptodev *dev)
 
 /** Returns the size of session structure */
 static unsigned int
-dpaa_sec_session_get_size(struct rte_cryptodev *dev __rte_unused)
+dpaa_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	PMD_INIT_FUNC_TRACE();
 
@@ -1755,34 +1748,6 @@ dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess)
 }
 
 static int
-dpaa_sec_qp_attach_sess(struct rte_cryptodev *dev __rte_unused,
-			uint16_t qp_id __rte_unused,
-			void *ses __rte_unused)
-{
-	PMD_INIT_FUNC_TRACE();
-	return 0;
-}
-
-static int
-dpaa_sec_qp_detach_sess(struct rte_cryptodev *dev,
-			uint16_t qp_id  __rte_unused,
-			void *ses)
-{
-	dpaa_sec_session *sess = ses;
-	struct dpaa_sec_dev_private *qi = dev->data->dev_private;
-
-	PMD_INIT_FUNC_TRACE();
-
-	if (sess->inq)
-		dpaa_sec_detach_rxq(qi, sess->inq);
-	sess->inq = NULL;
-
-	sess->qp = NULL;
-
-	return 0;
-}
-
-static int
 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev,
 			    struct rte_crypto_sym_xform *xform,	void *sess)
 {
@@ -1859,7 +1824,7 @@ err1:
 }
 
 static int
-dpaa_sec_session_configure(struct rte_cryptodev *dev,
+dpaa_sec_sym_session_configure(struct rte_cryptodev *dev,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *sess,
 		struct rte_mempool *mempool)
@@ -1883,7 +1848,7 @@ dpaa_sec_session_configure(struct rte_cryptodev *dev,
 		return ret;
 	}
 
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 			sess_private_data);
 
 
@@ -1892,12 +1857,12 @@ dpaa_sec_session_configure(struct rte_cryptodev *dev,
 
 /** Clear the memory of session so it doesn't leave key material behind */
 static void
-dpaa_sec_session_clear(struct rte_cryptodev *dev,
+dpaa_sec_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 	struct dpaa_sec_dev_private *qi = dev->data->dev_private;
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -1911,7 +1876,7 @@ dpaa_sec_session_clear(struct rte_cryptodev *dev,
 		rte_free(s->cipher_key.data);
 		rte_free(s->auth_key.data);
 		memset(s, 0, sizeof(dpaa_sec_session));
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
 		rte_mempool_put(sess_mp, sess_priv);
 	}
 }
@@ -2215,9 +2180,6 @@ dpaa_sec_dev_infos_get(struct rte_cryptodev *dev,
 		info->feature_flags = dev->feature_flags;
 		info->capabilities = dpaa_sec_capabilities;
 		info->sym.max_nb_sessions = internals->max_nb_sessions;
-		info->sym.max_nb_sessions_per_qp =
-			RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS /
-			RTE_DPAA_MAX_NB_SEC_QPS;
 		info->driver_id = cryptodev_driver_id;
 	}
 }
@@ -2230,14 +2192,10 @@ static struct rte_cryptodev_ops crypto_ops = {
 	.dev_infos_get        = dpaa_sec_dev_infos_get,
 	.queue_pair_setup     = dpaa_sec_queue_pair_setup,
 	.queue_pair_release   = dpaa_sec_queue_pair_release,
-	.queue_pair_start     = dpaa_sec_queue_pair_start,
-	.queue_pair_stop      = dpaa_sec_queue_pair_stop,
 	.queue_pair_count     = dpaa_sec_queue_pair_count,
-	.session_get_size     = dpaa_sec_session_get_size,
-	.session_configure    = dpaa_sec_session_configure,
-	.session_clear        = dpaa_sec_session_clear,
-	.qp_attach_session    = dpaa_sec_qp_attach_sess,
-	.qp_detach_session    = dpaa_sec_qp_detach_sess,
+	.sym_session_get_size     = dpaa_sec_sym_session_get_size,
+	.sym_session_configure    = dpaa_sec_sym_session_configure,
+	.sym_session_clear        = dpaa_sec_sym_session_clear
 };
 
 static const struct rte_security_capability *
@@ -2296,7 +2254,11 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
 			RTE_CRYPTODEV_FF_HW_ACCELERATED |
 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 			RTE_CRYPTODEV_FF_SECURITY |
-			RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
+			RTE_CRYPTODEV_FF_IN_PLACE_SGL |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
 
 	internals = cryptodev->data->dev_private;
 	internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS;
@@ -2449,9 +2411,7 @@ RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver);
 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver,
 		cryptodev_driver_id);
 
-RTE_INIT(dpaa_sec_init_log);
-static void
-dpaa_sec_init_log(void)
+RTE_INIT(dpaa_sec_init_log)
 {
 	dpaa_logtype_sec = rte_log_register("pmd.crypto.dpaa");
 	if (dpaa_logtype_sec >= 0)
diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.h b/drivers/crypto/dpaa_sec/dpaa_sec.h
index e15e373f..ac6c00a6 100644
--- a/drivers/crypto/dpaa_sec/dpaa_sec.h
+++ b/drivers/crypto/dpaa_sec/dpaa_sec.h
@@ -26,6 +26,7 @@
 #define CTX_POOL_NUM_BUFS	32000
 #define CTX_POOL_BUF_SIZE	sizeof(struct dpaa_sec_op_ctx)
 #define CTX_POOL_CACHE_SIZE	512
+#define RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS 2048
 
 #define DIR_ENC                 1
 #define DIR_DEC                 0
diff --git a/drivers/crypto/dpaa_sec/dpaa_sec_log.h b/drivers/crypto/dpaa_sec/dpaa_sec_log.h
index 9784fcbf..fb895a8b 100644
--- a/drivers/crypto/dpaa_sec/dpaa_sec_log.h
+++ b/drivers/crypto/dpaa_sec/dpaa_sec_log.h
@@ -18,7 +18,7 @@ extern int dpaa_logtype_sec;
 	rte_log(RTE_LOG_DEBUG, dpaa_logtype_sec, "dpaa_sec: %s(): " \
 		fmt "\n", __func__, ##args)
 
-#define PMD_INIT_FUNC_TRACE() DPAA_SEC_LOG(DEBUG, " >>")
+#define PMD_INIT_FUNC_TRACE() DPAA_SEC_DEBUG(" >>")
 
 #define DPAA_SEC_INFO(fmt, args...) \
 	DPAA_SEC_LOG(INFO, fmt, ## args)
diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd.c b/drivers/crypto/kasumi/rte_kasumi_pmd.c
index 205dc1de..239a1cf4 100644
--- a/drivers/crypto/kasumi/rte_kasumi_pmd.c
+++ b/drivers/crypto/kasumi/rte_kasumi_pmd.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2016-2017 Intel Corporation
+ * Copyright(c) 2016-2018 Intel Corporation
  */
 
 #include <rte_common.h>
@@ -79,18 +79,20 @@ kasumi_set_session_parameters(struct kasumi_session *sess,
 		break;
 	case KASUMI_OP_NOT_SUPPORTED:
 	default:
-		KASUMI_LOG_ERR("Unsupported operation chain order parameter");
+		KASUMI_LOG(ERR, "Unsupported operation chain order parameter");
 		return -ENOTSUP;
 	}
 
 	if (cipher_xform) {
 		/* Only KASUMI F8 supported */
-		if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_KASUMI_F8)
+		if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_KASUMI_F8) {
+			KASUMI_LOG(ERR, "Unsupported cipher algorithm ");
 			return -ENOTSUP;
+		}
 
 		sess->cipher_iv_offset = cipher_xform->cipher.iv.offset;
 		if (cipher_xform->cipher.iv.length != KASUMI_IV_LENGTH) {
-			KASUMI_LOG_ERR("Wrong IV length");
+			KASUMI_LOG(ERR, "Wrong IV length");
 			return -EINVAL;
 		}
 
@@ -101,11 +103,13 @@ kasumi_set_session_parameters(struct kasumi_session *sess,
 
 	if (auth_xform) {
 		/* Only KASUMI F9 supported */
-		if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_KASUMI_F9)
+		if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_KASUMI_F9) {
+			KASUMI_LOG(ERR, "Unsupported authentication");
 			return -ENOTSUP;
+		}
 
 		if (auth_xform->auth.digest_length != KASUMI_DIGEST_LENGTH) {
-			KASUMI_LOG_ERR("Wrong digest length");
+			KASUMI_LOG(ERR, "Wrong digest length");
 			return -EINVAL;
 		}
 
@@ -131,7 +135,7 @@ kasumi_get_session(struct kasumi_qp *qp, struct rte_crypto_op *op)
 	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 		if (likely(op->sym->session != NULL))
 			sess = (struct kasumi_session *)
-					get_session_private_data(
+					get_sym_session_private_data(
 					op->sym->session,
 					cryptodev_driver_id);
 	} else {
@@ -153,8 +157,8 @@ kasumi_get_session(struct kasumi_qp *qp, struct rte_crypto_op *op)
 			sess = NULL;
 		}
 		op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
-		set_session_private_data(op->sym->session, cryptodev_driver_id,
-			_sess_private_data);
+		set_sym_session_private_data(op->sym->session,
+				cryptodev_driver_id, _sess_private_data);
 	}
 
 	if (unlikely(sess == NULL))
@@ -213,7 +217,7 @@ process_kasumi_cipher_op_bit(struct rte_crypto_op *op,
 	src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *);
 	if (op->sym->m_dst == NULL) {
 		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
-		KASUMI_LOG_ERR("bit-level in-place not supported\n");
+		KASUMI_LOG(ERR, "bit-level in-place not supported");
 		return 0;
 	}
 	dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *);
@@ -244,7 +248,7 @@ process_kasumi_hash_op(struct kasumi_qp *qp, struct rte_crypto_op **ops,
 		/* Data must be byte aligned */
 		if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) {
 			ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
-			KASUMI_LOG_ERR("offset");
+			KASUMI_LOG(ERR, "Invalid Offset");
 			break;
 		}
 
@@ -409,9 +413,9 @@ kasumi_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 				(curr_c_op->sym->m_dst != NULL &&
 				!rte_pktmbuf_is_contiguous(
 						curr_c_op->sym->m_dst))) {
-			KASUMI_LOG_ERR("PMD supports only contiguous mbufs, "
+			KASUMI_LOG(ERR, "PMD supports only contiguous mbufs, "
 				"op (%p) provides noncontiguous mbuf as "
-				"source/destination buffer.\n", curr_c_op);
+				"source/destination buffer.", curr_c_op);
 			curr_c_op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 			break;
 		}
@@ -531,7 +535,7 @@ cryptodev_kasumi_create(const char *name,
 
 	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
 	if (dev == NULL) {
-		KASUMI_LOG_ERR("failed to create cryptodev vdev");
+		KASUMI_LOG(ERR, "failed to create cryptodev vdev");
 		goto init_error;
 	}
 
@@ -555,11 +559,10 @@ cryptodev_kasumi_create(const char *name,
 	internals = dev->data->dev_private;
 
 	internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
-	internals->max_nb_sessions = init_params->max_nb_sessions;
 
 	return 0;
 init_error:
-	KASUMI_LOG_ERR("driver %s: cryptodev_kasumi_create failed",
+	KASUMI_LOG(ERR, "driver %s: failed",
 			init_params->name);
 
 	cryptodev_kasumi_remove(vdev);
@@ -573,8 +576,7 @@ cryptodev_kasumi_probe(struct rte_vdev_device *vdev)
 		"",
 		sizeof(struct kasumi_private),
 		rte_socket_id(),
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
+		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 	};
 	const char *name;
 	const char *input_args;
@@ -617,7 +619,11 @@ RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_KASUMI_PMD, cryptodev_kasumi_pmd_drv);
 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_KASUMI_PMD, cryptodev_kasumi_pmd);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_KASUMI_PMD,
 	"max_nb_queue_pairs=<int> "
-	"max_nb_sessions=<int> "
 	"socket_id=<int>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(kasumi_crypto_drv,
 		cryptodev_kasumi_pmd_drv.driver, cryptodev_driver_id);
+
+RTE_INIT(kasumi_init_log)
+{
+	kasumi_logtype_driver = rte_log_register("pmd.crypto.kasumi");
+}
diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c
index a388dbb6..9e4bf1b5 100644
--- a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c
+++ b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2016-2017 Intel Corporation
+ * Copyright(c) 2016-2018 Intel Corporation
  */
 
 #include <string.h>
@@ -126,7 +126,8 @@ kasumi_pmd_info_get(struct rte_cryptodev *dev,
 	if (dev_info != NULL) {
 		dev_info->driver_id = dev->driver_id;
 		dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
-		dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
+		/* No limit of number of sessions */
+		dev_info->sym.max_nb_sessions = 0;
 		dev_info->feature_flags = dev->feature_flags;
 		dev_info->capabilities = kasumi_pmd_capabilities;
 	}
@@ -171,13 +172,13 @@ kasumi_pmd_qp_create_processed_ops_ring(struct kasumi_qp *qp,
 	r = rte_ring_lookup(qp->name);
 	if (r) {
 		if (rte_ring_get_size(r) == ring_size) {
-			KASUMI_LOG_INFO("Reusing existing ring %s"
+			KASUMI_LOG(INFO, "Reusing existing ring %s"
 					" for processed packets",
 					 qp->name);
 			return r;
 		}
 
-		KASUMI_LOG_ERR("Unable to reuse existing ring %s"
+		KASUMI_LOG(ERR, "Unable to reuse existing ring %s"
 				" for processed packets",
 				 qp->name);
 		return NULL;
@@ -228,22 +229,6 @@ qp_setup_cleanup:
 	return -1;
 }
 
-/** Start queue pair */
-static int
-kasumi_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
-/** Stop queue pair */
-static int
-kasumi_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
 /** Return the number of allocated queue pairs */
 static uint32_t
 kasumi_pmd_qp_count(struct rte_cryptodev *dev)
@@ -253,14 +238,14 @@ kasumi_pmd_qp_count(struct rte_cryptodev *dev)
 
 /** Returns the size of the KASUMI session structure */
 static unsigned
-kasumi_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
+kasumi_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	return sizeof(struct kasumi_session);
 }
 
 /** Configure a KASUMI session from a crypto xform chain */
 static int
-kasumi_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
+kasumi_pmd_sym_session_configure(struct rte_cryptodev *dev __rte_unused,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *sess,
 		struct rte_mempool *mempool)
@@ -269,26 +254,26 @@ kasumi_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
 	int ret;
 
 	if (unlikely(sess == NULL)) {
-		KASUMI_LOG_ERR("invalid session struct");
+		KASUMI_LOG(ERR, "invalid session struct");
 		return -EINVAL;
 	}
 
 	if (rte_mempool_get(mempool, &sess_private_data)) {
-		CDEV_LOG_ERR(
-			"Couldn't get object from session mempool");
+		KASUMI_LOG(ERR,
+				"Couldn't get object from session mempool");
 		return -ENOMEM;
 	}
 
 	ret = kasumi_set_session_parameters(sess_private_data, xform);
 	if (ret != 0) {
-		KASUMI_LOG_ERR("failed configure session parameters");
+		KASUMI_LOG(ERR, "failed configure session parameters");
 
 		/* Return session to mempool */
 		rte_mempool_put(mempool, sess_private_data);
 		return ret;
 	}
 
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 		sess_private_data);
 
 	return 0;
@@ -296,17 +281,17 @@ kasumi_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
 
 /** Clear the memory of session so it doesn't leave key material behind */
 static void
-kasumi_pmd_session_clear(struct rte_cryptodev *dev,
+kasumi_pmd_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 
 	/* Zero out the whole structure */
 	if (sess_priv) {
 		memset(sess_priv, 0, sizeof(struct kasumi_session));
 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
 		rte_mempool_put(sess_mp, sess_priv);
 	}
 }
@@ -324,13 +309,11 @@ struct rte_cryptodev_ops kasumi_pmd_ops = {
 
 		.queue_pair_setup   = kasumi_pmd_qp_setup,
 		.queue_pair_release = kasumi_pmd_qp_release,
-		.queue_pair_start   = kasumi_pmd_qp_start,
-		.queue_pair_stop    = kasumi_pmd_qp_stop,
 		.queue_pair_count   = kasumi_pmd_qp_count,
 
-		.session_get_size   = kasumi_pmd_session_get_size,
-		.session_configure  = kasumi_pmd_session_configure,
-		.session_clear      = kasumi_pmd_session_clear
+		.sym_session_get_size   = kasumi_pmd_sym_session_get_size,
+		.sym_session_configure  = kasumi_pmd_sym_session_configure,
+		.sym_session_clear      = kasumi_pmd_sym_session_clear
 };
 
 struct rte_cryptodev_ops *rte_kasumi_pmd_ops = &kasumi_pmd_ops;
diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd_private.h b/drivers/crypto/kasumi/rte_kasumi_pmd_private.h
index a397bee6..488777ca 100644
--- a/drivers/crypto/kasumi/rte_kasumi_pmd_private.h
+++ b/drivers/crypto/kasumi/rte_kasumi_pmd_private.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2016-2017 Intel Corporation
+ * Copyright(c) 2016-2018 Intel Corporation
  */
 
 #ifndef _RTE_KASUMI_PMD_PRIVATE_H_
@@ -10,25 +10,13 @@
 #define CRYPTODEV_NAME_KASUMI_PMD	crypto_kasumi
 /**< KASUMI PMD device name */
 
-#define KASUMI_LOG_ERR(fmt, args...) \
-	RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",  \
-			RTE_STR(CRYPTODEV_NAME_KASUMI_PMD), \
-			__func__, __LINE__, ## args)
+/** KASUMI PMD LOGTYPE DRIVER */
+int kasumi_logtype_driver;
 
-#ifdef RTE_LIBRTE_KASUMI_DEBUG
-#define KASUMI_LOG_INFO(fmt, args...) \
-	RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_KASUMI_PMD), \
-			__func__, __LINE__, ## args)
-
-#define KASUMI_LOG_DBG(fmt, args...) \
-	RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_KASUMI_PMD), \
-			__func__, __LINE__, ## args)
-#else
-#define KASUMI_LOG_INFO(fmt, args...)
-#define KASUMI_LOG_DBG(fmt, args...)
-#endif
+#define KASUMI_LOG(level, fmt, ...)  \
+	rte_log(RTE_LOG_ ## level, kasumi_logtype_driver,  \
+			"%s() line %u: " fmt "\n", __func__, __LINE__,  \
+					## __VA_ARGS__)
 
 #define KASUMI_DIGEST_LENGTH 4
 
@@ -36,8 +24,6 @@
 struct kasumi_private {
 	unsigned max_nb_queue_pairs;
 	/**< Max number of queue pairs supported by device */
-	unsigned max_nb_sessions;
-	/**< Max number of sessions supported by device */
 };
 
 /** KASUMI buffer queue pair */
diff --git a/drivers/crypto/mvsam/rte_mrvl_pmd.c b/drivers/crypto/mvsam/rte_mrvl_pmd.c
index 1b6029a5..73eff757 100644
--- a/drivers/crypto/mvsam/rte_mrvl_pmd.c
+++ b/drivers/crypto/mvsam/rte_mrvl_pmd.c
@@ -16,8 +16,23 @@
 
 #define MRVL_MUSDK_DMA_MEMSIZE 41943040
 
+#define MRVL_PMD_MAX_NB_SESS_ARG		("max_nb_sessions")
+#define MRVL_PMD_DEFAULT_MAX_NB_SESSIONS	2048
+
 static uint8_t cryptodev_driver_id;
 
+struct mrvl_pmd_init_params {
+	struct rte_cryptodev_pmd_init_params common;
+	uint32_t max_nb_sessions;
+};
+
+const char *mrvl_pmd_valid_params[] = {
+	RTE_CRYPTODEV_PMD_NAME_ARG,
+	RTE_CRYPTODEV_PMD_MAX_NB_QP_ARG,
+	RTE_CRYPTODEV_PMD_SOCKET_ID_ARG,
+	MRVL_PMD_MAX_NB_SESS_ARG
+};
+
 /**
  * Flag if particular crypto algorithm is supported by PMD/MUSDK.
  *
@@ -432,7 +447,7 @@ mrvl_request_prepare(struct sam_cio_op_params *request,
 		return -EINVAL;
 	}
 
-	sess = (struct mrvl_crypto_session *)get_session_private_data(
+	sess = (struct mrvl_crypto_session *)get_sym_session_private_data(
 			op->sym->session, cryptodev_driver_id);
 	if (unlikely(sess == NULL)) {
 		MRVL_CRYPTO_LOG_ERR("Session was not created for this device");
@@ -691,14 +706,15 @@ mrvl_crypto_pmd_dequeue_burst(void *queue_pair,
 static int
 cryptodev_mrvl_crypto_create(const char *name,
 		struct rte_vdev_device *vdev,
-		struct rte_cryptodev_pmd_init_params *init_params)
+		struct mrvl_pmd_init_params *init_params)
 {
 	struct rte_cryptodev *dev;
 	struct mrvl_crypto_private *internals;
 	struct sam_init_params	sam_params;
 	int ret;
 
-	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
+	dev = rte_cryptodev_pmd_create(name, &vdev->device,
+			&init_params->common);
 	if (dev == NULL) {
 		MRVL_CRYPTO_LOG_ERR("failed to create cryptodev vdev");
 		goto init_error;
@@ -718,7 +734,7 @@ cryptodev_mrvl_crypto_create(const char *name,
 	/* Set vector instructions mode supported */
 	internals = dev->data->dev_private;
 
-	internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
+	internals->max_nb_qpairs = init_params->common.max_nb_queue_pairs;
 	internals->max_nb_sessions = init_params->max_nb_sessions;
 
 	/*
@@ -740,12 +756,99 @@ cryptodev_mrvl_crypto_create(const char *name,
 
 init_error:
 	MRVL_CRYPTO_LOG_ERR(
-		"driver %s: %s failed", init_params->name, __func__);
+		"driver %s: %s failed", init_params->common.name, __func__);
 
 	cryptodev_mrvl_crypto_uninit(vdev);
 	return -EFAULT;
 }
 
+/** Parse integer from integer argument */
+static int
+parse_integer_arg(const char *key __rte_unused,
+		const char *value, void *extra_args)
+{
+	int *i = (int *) extra_args;
+
+	*i = atoi(value);
+	if (*i < 0) {
+		MRVL_CRYPTO_LOG_ERR("Argument has to be positive.\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/** Parse name */
+static int
+parse_name_arg(const char *key __rte_unused,
+		const char *value, void *extra_args)
+{
+	struct rte_cryptodev_pmd_init_params *params = extra_args;
+
+	if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
+		MRVL_CRYPTO_LOG_ERR("Invalid name %s, should be less than "
+				"%u bytes.\n", value,
+				RTE_CRYPTODEV_NAME_MAX_LEN - 1);
+		return -EINVAL;
+	}
+
+	strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
+
+	return 0;
+}
+
+static int
+mrvl_pmd_parse_input_args(struct mrvl_pmd_init_params *params,
+			 const char *input_args)
+{
+	struct rte_kvargs *kvlist = NULL;
+	int ret = 0;
+
+	if (params == NULL)
+		return -EINVAL;
+
+	if (input_args) {
+		kvlist = rte_kvargs_parse(input_args,
+					  mrvl_pmd_valid_params);
+		if (kvlist == NULL)
+			return -1;
+
+		/* Common VDEV parameters */
+		ret = rte_kvargs_process(kvlist,
+					 RTE_CRYPTODEV_PMD_MAX_NB_QP_ARG,
+					 &parse_integer_arg,
+					 &params->common.max_nb_queue_pairs);
+		if (ret < 0)
+			goto free_kvlist;
+
+		ret = rte_kvargs_process(kvlist,
+					 RTE_CRYPTODEV_PMD_SOCKET_ID_ARG,
+					 &parse_integer_arg,
+					 &params->common.socket_id);
+		if (ret < 0)
+			goto free_kvlist;
+
+		ret = rte_kvargs_process(kvlist,
+					 RTE_CRYPTODEV_PMD_NAME_ARG,
+					 &parse_name_arg,
+					 &params->common);
+		if (ret < 0)
+			goto free_kvlist;
+
+		ret = rte_kvargs_process(kvlist,
+					 MRVL_PMD_MAX_NB_SESS_ARG,
+					 &parse_integer_arg,
+					 params);
+		if (ret < 0)
+			goto free_kvlist;
+
+	}
+
+free_kvlist:
+	rte_kvargs_free(kvlist);
+	return ret;
+}
+
 /**
  * Initialize the crypto device.
  *
@@ -755,7 +858,18 @@ init_error:
 static int
 cryptodev_mrvl_crypto_init(struct rte_vdev_device *vdev)
 {
-	struct rte_cryptodev_pmd_init_params init_params = { };
+	struct mrvl_pmd_init_params init_params = {
+		.common = {
+			.name = "",
+			.private_data_size =
+				sizeof(struct mrvl_crypto_private),
+			.max_nb_queue_pairs =
+				sam_get_num_inst() * SAM_HW_RING_NUM,
+			.socket_id = rte_socket_id()
+		},
+		.max_nb_sessions = MRVL_PMD_DEFAULT_MAX_NB_SESSIONS
+	};
+
 	const char *name, *args;
 	int ret;
 
@@ -764,13 +878,7 @@ cryptodev_mrvl_crypto_init(struct rte_vdev_device *vdev)
 		return -EINVAL;
 	args = rte_vdev_device_args(vdev);
 
-	init_params.private_data_size = sizeof(struct mrvl_crypto_private);
-	init_params.max_nb_queue_pairs = sam_get_num_inst() * SAM_HW_RING_NUM;
-	init_params.max_nb_sessions =
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS;
-	init_params.socket_id = rte_socket_id();
-
-	ret = rte_cryptodev_pmd_parse_input_args(&init_params, args);
+	ret = mrvl_pmd_parse_input_args(&init_params, args);
 	if (ret) {
 		RTE_LOG(ERR, PMD,
 			"Failed to parse initialisation arguments[%s]\n",
diff --git a/drivers/crypto/mvsam/rte_mrvl_pmd_ops.c b/drivers/crypto/mvsam/rte_mrvl_pmd_ops.c
index 3f8de37b..c045562c 100644
--- a/drivers/crypto/mvsam/rte_mrvl_pmd_ops.c
+++ b/drivers/crypto/mvsam/rte_mrvl_pmd_ops.c
@@ -595,32 +595,6 @@ mrvl_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return -1;
 }
 
-/** Start queue pair (PMD ops callback) - not supported.
- *
- * @param dev Pointer to the device structure.
- * @param qp_id ID of the Queue Pair.
- * @returns -ENOTSUP. Always.
- */
-static int
-mrvl_crypto_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
-/** Stop queue pair (PMD ops callback) - not supported.
- *
- * @param dev Pointer to the device structure.
- * @param qp_id ID of the Queue Pair.
- * @returns -ENOTSUP. Always.
- */
-static int
-mrvl_crypto_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
 /** Return the number of allocated queue pairs (PMD ops callback).
  *
  * @param dev Pointer to the device structure.
@@ -638,7 +612,7 @@ mrvl_crypto_pmd_qp_count(struct rte_cryptodev *dev)
  * @returns Size of Marvell crypto session.
  */
 static unsigned
-mrvl_crypto_pmd_session_get_size(__rte_unused struct rte_cryptodev *dev)
+mrvl_crypto_pmd_sym_session_get_size(__rte_unused struct rte_cryptodev *dev)
 {
 	return sizeof(struct mrvl_crypto_session);
 }
@@ -651,7 +625,7 @@ mrvl_crypto_pmd_session_get_size(__rte_unused struct rte_cryptodev *dev)
  * @returns 0 upon success, negative value otherwise.
  */
 static int
-mrvl_crypto_pmd_session_configure(__rte_unused struct rte_cryptodev *dev,
+mrvl_crypto_pmd_sym_session_configure(__rte_unused struct rte_cryptodev *dev,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *sess,
 		struct rte_mempool *mp)
@@ -679,7 +653,7 @@ mrvl_crypto_pmd_session_configure(__rte_unused struct rte_cryptodev *dev,
 		return ret;
 	}
 
-	set_session_private_data(sess, dev->driver_id, sess_private_data);
+	set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
 
 	mrvl_sess = (struct mrvl_crypto_session *)sess_private_data;
 	if (sam_session_create(&mrvl_sess->sam_sess_params,
@@ -698,12 +672,12 @@ mrvl_crypto_pmd_session_configure(__rte_unused struct rte_cryptodev *dev,
  * @returns 0. Always.
  */
 static void
-mrvl_crypto_pmd_session_clear(struct rte_cryptodev *dev,
+mrvl_crypto_pmd_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 
 	/* Zero out the whole structure */
 	if (sess_priv) {
@@ -717,7 +691,7 @@ mrvl_crypto_pmd_session_clear(struct rte_cryptodev *dev,
 
 		memset(sess, 0, sizeof(struct mrvl_crypto_session));
 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
 		rte_mempool_put(sess_mp, sess_priv);
 	}
 }
@@ -738,13 +712,11 @@ static struct rte_cryptodev_ops mrvl_crypto_pmd_ops = {
 
 		.queue_pair_setup	= mrvl_crypto_pmd_qp_setup,
 		.queue_pair_release	= mrvl_crypto_pmd_qp_release,
-		.queue_pair_start	= mrvl_crypto_pmd_qp_start,
-		.queue_pair_stop	= mrvl_crypto_pmd_qp_stop,
 		.queue_pair_count	= mrvl_crypto_pmd_qp_count,
 
-		.session_get_size	= mrvl_crypto_pmd_session_get_size,
-		.session_configure	= mrvl_crypto_pmd_session_configure,
-		.session_clear		= mrvl_crypto_pmd_session_clear
+		.sym_session_get_size	= mrvl_crypto_pmd_sym_session_get_size,
+		.sym_session_configure	= mrvl_crypto_pmd_sym_session_configure,
+		.sym_session_clear	= mrvl_crypto_pmd_sym_session_clear
 };
 
 struct rte_cryptodev_ops *rte_mrvl_crypto_pmd_ops = &mrvl_crypto_pmd_ops;
diff --git a/drivers/crypto/null/null_crypto_pmd.c b/drivers/crypto/null/null_crypto_pmd.c
index 052b6546..6e29a21a 100644
--- a/drivers/crypto/null/null_crypto_pmd.c
+++ b/drivers/crypto/null/null_crypto_pmd.c
@@ -78,7 +78,7 @@ get_session(struct null_crypto_qp *qp, struct rte_crypto_op *op)
 	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 		if (likely(sym_op->session != NULL))
 			sess = (struct null_crypto_session *)
-					get_session_private_data(
+					get_sym_session_private_data(
 					sym_op->session, cryptodev_driver_id);
 	} else {
 		void *_sess = NULL;
@@ -99,8 +99,8 @@ get_session(struct null_crypto_qp *qp, struct rte_crypto_op *op)
 			sess = NULL;
 		}
 		sym_op->session = (struct rte_cryptodev_sym_session *)_sess;
-		set_session_private_data(sym_op->session, cryptodev_driver_id,
-			_sess_private_data);
+		set_sym_session_private_data(op->sym->session,
+				cryptodev_driver_id, _sess_private_data);
 	}
 
 	return sess;
@@ -161,10 +161,9 @@ cryptodev_null_create(const char *name,
 {
 	struct rte_cryptodev *dev;
 	struct null_crypto_private *internals;
-
 	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
 	if (dev == NULL) {
-		NULL_CRYPTO_LOG_ERR("failed to create cryptodev vdev");
+		NULL_LOG(ERR, "failed to create cryptodev vdev");
 		return -EFAULT;
 	}
 
@@ -177,12 +176,11 @@ cryptodev_null_create(const char *name,
 
 	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
-			RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
+			RTE_CRYPTODEV_FF_IN_PLACE_SGL;
 
 	internals = dev->data->dev_private;
 
 	internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
-	internals->max_nb_sessions = init_params->max_nb_sessions;
 
 	return 0;
 }
@@ -195,8 +193,7 @@ cryptodev_null_probe(struct rte_vdev_device *dev)
 		"",
 		sizeof(struct null_crypto_private),
 		rte_socket_id(),
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
+		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 	};
 	const char *name, *args;
 	int retval;
@@ -209,8 +206,9 @@ cryptodev_null_probe(struct rte_vdev_device *dev)
 
 	retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
 	if (retval) {
-		RTE_LOG(ERR, PMD,
-			"Failed to parse initialisation arguments[%s]\n", args);
+		NULL_LOG(ERR,
+				"Failed to parse initialisation arguments[%s]",
+				args);
 		return -EINVAL;
 	}
 
@@ -245,7 +243,11 @@ RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_NULL_PMD, cryptodev_null_pmd_drv);
 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_NULL_PMD, cryptodev_null_pmd);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_NULL_PMD,
 	"max_nb_queue_pairs=<int> "
-	"max_nb_sessions=<int> "
 	"socket_id=<int>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(null_crypto_drv, cryptodev_null_pmd_drv.driver,
 		cryptodev_driver_id);
+
+RTE_INIT(null_init_log)
+{
+	null_logtype_driver = rte_log_register("pmd.crypto.null");
+}
diff --git a/drivers/crypto/null/null_crypto_pmd_ops.c b/drivers/crypto/null/null_crypto_pmd_ops.c
index f8e5f61f..bb2b6e14 100644
--- a/drivers/crypto/null/null_crypto_pmd_ops.c
+++ b/drivers/crypto/null/null_crypto_pmd_ops.c
@@ -121,7 +121,8 @@ null_crypto_pmd_info_get(struct rte_cryptodev *dev,
 	if (dev_info != NULL) {
 		dev_info->driver_id = dev->driver_id;
 		dev_info->max_nb_queue_pairs = internals->max_nb_qpairs;
-		dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
+		/* No limit of number of sessions */
+		dev_info->sym.max_nb_sessions = 0;
 		dev_info->feature_flags = dev->feature_flags;
 		dev_info->capabilities = null_crypto_pmd_capabilities;
 	}
@@ -163,15 +164,15 @@ null_crypto_pmd_qp_create_processed_pkts_ring(struct null_crypto_qp *qp,
 	r = rte_ring_lookup(qp->name);
 	if (r) {
 		if (rte_ring_get_size(r) >= ring_size) {
-			NULL_CRYPTO_LOG_INFO(
-				"Reusing existing ring %s for processed packets",
-				qp->name);
+			NULL_LOG(INFO,
+					"Reusing existing ring %s for "
+					" processed packets", qp->name);
 			return r;
 		}
 
-		NULL_CRYPTO_LOG_INFO(
-			"Unable to reuse existing ring %s for processed packets",
-			 qp->name);
+		NULL_LOG(INFO,
+				"Unable to reuse existing ring %s for "
+				" processed packets", qp->name);
 		return NULL;
 	}
 
@@ -190,7 +191,7 @@ null_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	int retval;
 
 	if (qp_id >= internals->max_nb_qpairs) {
-		NULL_CRYPTO_LOG_ERR("Invalid qp_id %u, greater than maximum "
+		NULL_LOG(ERR, "Invalid qp_id %u, greater than maximum "
 				"number of queue pairs supported (%u).",
 				qp_id, internals->max_nb_qpairs);
 		return (-EINVAL);
@@ -204,7 +205,7 @@ null_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	qp = rte_zmalloc_socket("Null Crypto PMD Queue Pair", sizeof(*qp),
 					RTE_CACHE_LINE_SIZE, socket_id);
 	if (qp == NULL) {
-		NULL_CRYPTO_LOG_ERR("Failed to allocate queue pair memory");
+		NULL_LOG(ERR, "Failed to allocate queue pair memory");
 		return (-ENOMEM);
 	}
 
@@ -213,15 +214,16 @@ null_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 	retval = null_crypto_pmd_qp_set_unique_name(dev, qp);
 	if (retval) {
-		NULL_CRYPTO_LOG_ERR("Failed to create unique name for null "
+		NULL_LOG(ERR, "Failed to create unique name for null "
 				"crypto device");
+
 		goto qp_setup_cleanup;
 	}
 
 	qp->processed_pkts = null_crypto_pmd_qp_create_processed_pkts_ring(qp,
 			qp_conf->nb_descriptors, socket_id);
 	if (qp->processed_pkts == NULL) {
-		NULL_CRYPTO_LOG_ERR("Failed to create unique name for null "
+		NULL_LOG(ERR, "Failed to create unique name for null "
 				"crypto device");
 		goto qp_setup_cleanup;
 	}
@@ -239,22 +241,6 @@ qp_setup_cleanup:
 	return -1;
 }
 
-/** Start queue pair */
-static int
-null_crypto_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
-/** Stop queue pair */
-static int
-null_crypto_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
 /** Return the number of allocated queue pairs */
 static uint32_t
 null_crypto_pmd_qp_count(struct rte_cryptodev *dev)
@@ -264,14 +250,14 @@ null_crypto_pmd_qp_count(struct rte_cryptodev *dev)
 
 /** Returns the size of the NULL crypto session structure */
 static unsigned
-null_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
+null_crypto_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	return sizeof(struct null_crypto_session);
 }
 
 /** Configure a null crypto session from a crypto xform chain */
 static int
-null_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
+null_crypto_pmd_sym_session_configure(struct rte_cryptodev *dev __rte_unused,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *sess,
 		struct rte_mempool *mp)
@@ -280,26 +266,26 @@ null_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
 	int ret;
 
 	if (unlikely(sess == NULL)) {
-		NULL_CRYPTO_LOG_ERR("invalid session struct");
+		NULL_LOG(ERR, "invalid session struct");
 		return -EINVAL;
 	}
 
 	if (rte_mempool_get(mp, &sess_private_data)) {
-		CDEV_LOG_ERR(
-			"Couldn't get object from session mempool");
+		NULL_LOG(ERR,
+				"Couldn't get object from session mempool");
 		return -ENOMEM;
 	}
 
 	ret = null_crypto_set_session_parameters(sess_private_data, xform);
 	if (ret != 0) {
-		NULL_CRYPTO_LOG_ERR("failed configure session parameters");
+		NULL_LOG(ERR, "failed configure session parameters");
 
 		/* Return session to mempool */
 		rte_mempool_put(mp, sess_private_data);
 		return ret;
 	}
 
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 		sess_private_data);
 
 	return 0;
@@ -307,17 +293,17 @@ null_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
 
 /** Clear the memory of session so it doesn't leave key material behind */
 static void
-null_crypto_pmd_session_clear(struct rte_cryptodev *dev,
+null_crypto_pmd_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 
 	/* Zero out the whole structure */
 	if (sess_priv) {
 		memset(sess_priv, 0, sizeof(struct null_crypto_session));
 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
 		rte_mempool_put(sess_mp, sess_priv);
 	}
 }
@@ -335,13 +321,11 @@ struct rte_cryptodev_ops pmd_ops = {
 
 		.queue_pair_setup	= null_crypto_pmd_qp_setup,
 		.queue_pair_release	= null_crypto_pmd_qp_release,
-		.queue_pair_start	= null_crypto_pmd_qp_start,
-		.queue_pair_stop	= null_crypto_pmd_qp_stop,
 		.queue_pair_count	= null_crypto_pmd_qp_count,
 
-		.session_get_size	= null_crypto_pmd_session_get_size,
-		.session_configure	= null_crypto_pmd_session_configure,
-		.session_clear		= null_crypto_pmd_session_clear
+		.sym_session_get_size	= null_crypto_pmd_sym_session_get_size,
+		.sym_session_configure	= null_crypto_pmd_sym_session_configure,
+		.sym_session_clear	= null_crypto_pmd_sym_session_clear
 };
 
 struct rte_cryptodev_ops *null_crypto_pmd_ops = &pmd_ops;
diff --git a/drivers/crypto/null/null_crypto_pmd_private.h b/drivers/crypto/null/null_crypto_pmd_private.h
index 0fd13362..d5905afd 100644
--- a/drivers/crypto/null/null_crypto_pmd_private.h
+++ b/drivers/crypto/null/null_crypto_pmd_private.h
@@ -8,31 +8,17 @@
 #define CRYPTODEV_NAME_NULL_PMD		crypto_null
 /**< Null crypto PMD device name */
 
-#define NULL_CRYPTO_LOG_ERR(fmt, args...) \
-	RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",  \
-			RTE_STR(CRYPTODEV_NAME_NULL_PMD), \
-			__func__, __LINE__, ## args)
+int null_logtype_driver;
 
-#ifdef RTE_LIBRTE_NULL_CRYPTO_DEBUG
-#define NULL_CRYPTO_LOG_INFO(fmt, args...) \
-	RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_NULL_PMD), \
-			__func__, __LINE__, ## args)
-
-#define NULL_CRYPTO_LOG_DBG(fmt, args...) \
-	RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_NULL_PMD), \
-			__func__, __LINE__, ## args)
-#else
-#define NULL_CRYPTO_LOG_INFO(fmt, args...)
-#define NULL_CRYPTO_LOG_DBG(fmt, args...)
-#endif
+#define NULL_LOG(level, fmt, ...)  \
+	rte_log(RTE_LOG_ ## level, null_logtype_driver,  \
+			"%s() line %u: "fmt "\n", __func__, __LINE__,  \
+					## __VA_ARGS__)
 
 
 /** private data structure for each NULL crypto device */
 struct null_crypto_private {
 	unsigned max_nb_qpairs;		/**< Max number of queue pairs */
-	unsigned max_nb_sessions;	/**< Max number of sessions */
 };
 
 /** NULL crypto queue pair */
diff --git a/drivers/crypto/openssl/compat.h b/drivers/crypto/openssl/compat.h
new file mode 100644
index 00000000..45f9a33d
--- /dev/null
+++ b/drivers/crypto/openssl/compat.h
@@ -0,0 +1,108 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium Networks
+ */
+
+#ifndef __RTA_COMPAT_H__
+#define __RTA_COMPAT_H__
+
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L)
+
+#define set_rsa_params(rsa, p, q, ret) \
+	do {rsa->p = p; rsa->q = q; ret = 0; } while (0)
+
+#define set_rsa_crt_params(rsa, dmp1, dmq1, iqmp, ret) \
+	do { \
+		rsa->dmp1 = dmp1; \
+		rsa->dmq1 = dmq1; \
+		rsa->iqmp = iqmp; \
+		ret = 0; \
+	} while (0)
+
+#define set_rsa_keys(rsa, n, e, d, ret) \
+	do { \
+		rsa->n = n; rsa->e = e; rsa->d = d; ret = 0; \
+	} while (0)
+
+#define set_dh_params(dh, p, g, ret) \
+	do { \
+		dh->p = p; \
+		dh->q = NULL; \
+		dh->g = g; \
+		ret = 0; \
+	} while (0)
+
+#define set_dh_priv_key(dh, priv_key, ret) \
+	do { dh->priv_key = priv_key; ret = 0; } while (0)
+
+#define set_dsa_params(dsa, p, q, g, ret) \
+	do { dsa->p = p; dsa->q = q; dsa->g = g; ret = 0; } while (0)
+
+#define get_dh_pub_key(dh, pub_key) \
+	(pub_key = dh->pub_key)
+
+#define get_dh_priv_key(dh, priv_key) \
+	(priv_key = dh->priv_key)
+
+#define set_dsa_sign(sign, r, s) \
+	do { sign->r = r; sign->s = s; } while (0)
+
+#define get_dsa_sign(sign, r, s) \
+	do { r = sign->r; s = sign->s; } while (0)
+
+#define set_dsa_keys(dsa, pub, priv, ret) \
+	do { dsa->pub_key = pub; dsa->priv_key = priv; ret = 0; } while (0)
+
+#define set_dsa_pub_key(dsa, pub_key) \
+	(dsa->pub_key = pub_key)
+
+#define get_dsa_priv_key(dsa, priv_key) \
+	(priv_key = dsa->priv_key)
+
+#else
+
+#define set_rsa_params(rsa, p, q, ret) \
+	(ret = !RSA_set0_factors(rsa, p, q))
+
+#define set_rsa_crt_params(rsa, dmp1, dmq1, iqmp, ret) \
+	(ret = !RSA_set0_crt_params(rsa, dmp1, dmq1, iqmp))
+
+/* n, e must be non-null, d can be NULL */
+#define set_rsa_keys(rsa, n, e, d, ret) \
+	(ret = !RSA_set0_key(rsa, n, e, d))
+
+#define set_dh_params(dh, p, g, ret) \
+	(ret = !DH_set0_pqg(dh, p, NULL, g))
+
+#define set_dh_priv_key(dh, priv_key, ret) \
+	(ret = !DH_set0_key(dh, NULL, priv_key))
+
+#define get_dh_pub_key(dh, pub_key) \
+	(DH_get0_key(dh_key, &pub_key, NULL))
+
+#define get_dh_priv_key(dh, priv_key) \
+	(DH_get0_key(dh_key, NULL, &priv_key))
+
+#define set_dsa_params(dsa, p, q, g, ret) \
+	(ret = !DSA_set0_pqg(dsa, p, q, g))
+
+#define set_dsa_priv_key(dsa, priv_key) \
+	(DSA_set0_key(dsa, NULL, priv_key))
+
+#define set_dsa_sign(sign, r, s) \
+	(DSA_SIG_set0(sign, r, s))
+
+#define get_dsa_sign(sign, r, s) \
+	(DSA_SIG_get0(sign, &r, &s))
+
+#define set_dsa_keys(dsa, pub, priv, ret) \
+	(ret = !DSA_set0_key(dsa, pub, priv))
+
+#define set_dsa_pub_key(dsa, pub_key) \
+	(DSA_set0_key(dsa, pub_key, NULL))
+
+#define get_dsa_priv_key(dsa, priv_key) \
+	(DSA_get0_key(dsa, NULL, &priv_key))
+
+#endif /* version < 10100000 */
+
+#endif /* __RTA_COMPAT_H__ */
diff --git a/drivers/crypto/openssl/rte_openssl_pmd.c b/drivers/crypto/openssl/rte_openssl_pmd.c
index 93c6d7e5..7d263aba 100644
--- a/drivers/crypto/openssl/rte_openssl_pmd.c
+++ b/drivers/crypto/openssl/rte_openssl_pmd.c
@@ -14,6 +14,7 @@
 #include <openssl/evp.h>
 
 #include "rte_openssl_pmd_private.h"
+#include "compat.h"
 
 #define DES_BLOCK_SIZE 8
 
@@ -119,7 +120,7 @@ get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede)
 		memcpy(key_ede + 16, key, 8);
 		break;
 	default:
-		OPENSSL_LOG_ERR("Unsupported key size");
+		OPENSSL_LOG(ERR, "Unsupported key size");
 		res = -EINVAL;
 	}
 
@@ -137,6 +138,9 @@ get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
 		switch (sess_algo) {
 		case RTE_CRYPTO_CIPHER_3DES_CBC:
 			switch (keylen) {
+			case 8:
+				*algo = EVP_des_cbc();
+				break;
 			case 16:
 				*algo = EVP_des_ede_cbc();
 				break;
@@ -677,7 +681,7 @@ openssl_set_session_parameters(struct openssl_session *sess,
 		ret = openssl_set_session_cipher_parameters(
 				sess, cipher_xform);
 		if (ret != 0) {
-			OPENSSL_LOG_ERR(
+			OPENSSL_LOG(ERR,
 				"Invalid/unsupported cipher parameters");
 			return ret;
 		}
@@ -686,7 +690,7 @@ openssl_set_session_parameters(struct openssl_session *sess,
 	if (auth_xform) {
 		ret = openssl_set_session_auth_parameters(sess, auth_xform);
 		if (ret != 0) {
-			OPENSSL_LOG_ERR(
+			OPENSSL_LOG(ERR,
 				"Invalid/unsupported auth parameters");
 			return ret;
 		}
@@ -695,7 +699,7 @@ openssl_set_session_parameters(struct openssl_session *sess,
 	if (aead_xform) {
 		ret = openssl_set_session_aead_parameters(sess, aead_xform);
 		if (ret != 0) {
-			OPENSSL_LOG_ERR(
+			OPENSSL_LOG(ERR,
 				"Invalid/unsupported AEAD parameters");
 			return ret;
 		}
@@ -727,19 +731,36 @@ openssl_reset_session(struct openssl_session *sess)
 }
 
 /** Provide session for operation */
-static struct openssl_session *
+static void *
 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
 {
 	struct openssl_session *sess = NULL;
+	struct openssl_asym_session *asym_sess = NULL;
 
 	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
-		/* get existing session */
-		if (likely(op->sym->session != NULL))
-			sess = (struct openssl_session *)
-					get_session_private_data(
-					op->sym->session,
-					cryptodev_driver_id);
+		if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
+			/* get existing session */
+			if (likely(op->sym->session != NULL))
+				sess = (struct openssl_session *)
+						get_sym_session_private_data(
+						op->sym->session,
+						cryptodev_driver_id);
+		} else {
+			if (likely(op->asym->session != NULL))
+				asym_sess = (struct openssl_asym_session *)
+						get_asym_session_private_data(
+						op->asym->session,
+						cryptodev_driver_id);
+			if (asym_sess == NULL)
+				op->status =
+					RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+			return asym_sess;
+		}
 	} else {
+		/* sessionless asymmetric not supported */
+		if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
+			return NULL;
+
 		/* provide internal session */
 		void *_sess = NULL;
 		void *_sess_private_data = NULL;
@@ -759,8 +780,8 @@ get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
 			sess = NULL;
 		}
 		op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
-		set_session_private_data(op->sym->session, cryptodev_driver_id,
-			_sess_private_data);
+		set_sym_session_private_data(op->sym->session,
+				cryptodev_driver_id, _sess_private_data);
 	}
 
 	if (sess == NULL)
@@ -884,7 +905,7 @@ process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 	return 0;
 
 process_cipher_encrypt_err:
-	OPENSSL_LOG_ERR("Process openssl cipher encrypt failed");
+	OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed");
 	return -EINVAL;
 }
 
@@ -908,7 +929,7 @@ process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
 	return 0;
 
 process_cipher_encrypt_err:
-	OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed");
+	OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed");
 	return -EINVAL;
 }
 /** Process standard openssl cipher decryption */
@@ -932,7 +953,7 @@ process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 	return 0;
 
 process_cipher_decrypt_err:
-	OPENSSL_LOG_ERR("Process openssl cipher decrypt failed");
+	OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed");
 	return -EINVAL;
 }
 
@@ -989,7 +1010,7 @@ process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
 	return 0;
 
 process_cipher_des3ctr_err:
-	OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed");
+	OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed");
 	return -EINVAL;
 }
 
@@ -1027,7 +1048,7 @@ process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
 	return 0;
 
 process_auth_encryption_gcm_err:
-	OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed");
+	OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed");
 	return -EINVAL;
 }
 
@@ -1068,7 +1089,7 @@ process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset,
 	return 0;
 
 process_auth_encryption_ccm_err:
-	OPENSSL_LOG_ERR("Process openssl auth encryption ccm failed");
+	OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed");
 	return -EINVAL;
 }
 
@@ -1106,7 +1127,7 @@ process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
 	return 0;
 
 process_auth_decryption_gcm_err:
-	OPENSSL_LOG_ERR("Process openssl auth decryption gcm failed");
+	OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed");
 	return -EINVAL;
 }
 
@@ -1145,7 +1166,7 @@ process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset,
 	return 0;
 
 process_auth_decryption_ccm_err:
-	OPENSSL_LOG_ERR("Process openssl auth decryption ccm failed");
+	OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed");
 	return -EINVAL;
 }
 
@@ -1198,7 +1219,7 @@ process_auth_final:
 	return 0;
 
 process_auth_err:
-	OPENSSL_LOG_ERR("Process openssl auth failed");
+	OPENSSL_LOG(ERR, "Process openssl auth failed");
 	return -EINVAL;
 }
 
@@ -1251,7 +1272,7 @@ process_auth_final:
 	return 0;
 
 process_auth_err:
-	OPENSSL_LOG_ERR("Process openssl auth failed");
+	OPENSSL_LOG(ERR, "Process openssl auth failed");
 	return -EINVAL;
 }
 
@@ -1525,6 +1546,433 @@ process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 }
 
+/* process dsa sign operation */
+static int
+process_openssl_dsa_sign_op(struct rte_crypto_op *cop,
+		struct openssl_asym_session *sess)
+{
+	struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
+	DSA *dsa = sess->u.s.dsa;
+	DSA_SIG *sign = NULL;
+
+	sign = DSA_do_sign(op->message.data,
+			op->message.length,
+			dsa);
+
+	if (sign == NULL) {
+		OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__);
+		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	} else {
+		const BIGNUM *r = NULL, *s = NULL;
+		get_dsa_sign(sign, r, s);
+
+		op->r.length = BN_bn2bin(r, op->r.data);
+		op->s.length = BN_bn2bin(s, op->s.data);
+		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+	}
+
+	DSA_SIG_free(sign);
+
+	return 0;
+}
+
+/* process dsa verify operation */
+static int
+process_openssl_dsa_verify_op(struct rte_crypto_op *cop,
+		struct openssl_asym_session *sess)
+{
+	struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
+	DSA *dsa = sess->u.s.dsa;
+	int ret;
+	DSA_SIG *sign = DSA_SIG_new();
+	BIGNUM *r = NULL, *s = NULL;
+	BIGNUM *pub_key = NULL;
+
+	if (sign == NULL) {
+		OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__);
+		cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+		return -1;
+	}
+
+	r = BN_bin2bn(op->r.data,
+			op->r.length,
+			r);
+	s = BN_bin2bn(op->s.data,
+			op->s.length,
+			s);
+	pub_key = BN_bin2bn(op->y.data,
+			op->y.length,
+			pub_key);
+	if (!r || !s || !pub_key) {
+		if (r)
+			BN_free(r);
+		if (s)
+			BN_free(s);
+		if (pub_key)
+			BN_free(pub_key);
+
+		cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+		return -1;
+	}
+	set_dsa_sign(sign, r, s);
+	set_dsa_pub_key(dsa, pub_key);
+
+	ret = DSA_do_verify(op->message.data,
+			op->message.length,
+			sign,
+			dsa);
+
+	if (ret != 1)
+		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	else
+		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+
+	DSA_SIG_free(sign);
+
+	return 0;
+}
+
+/* process dh operation */
+static int
+process_openssl_dh_op(struct rte_crypto_op *cop,
+		struct openssl_asym_session *sess)
+{
+	struct rte_crypto_dh_op_param *op = &cop->asym->dh;
+	DH *dh_key = sess->u.dh.dh_key;
+	BIGNUM *priv_key = NULL;
+	int ret = 0;
+
+	if (sess->u.dh.key_op &
+			(1 << RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE)) {
+		/* compute shared secret using peer public key
+		 * and current private key
+		 * shared secret = peer_key ^ priv_key mod p
+		 */
+		BIGNUM *peer_key = NULL;
+
+		/* copy private key and peer key and compute shared secret */
+		peer_key = BN_bin2bn(op->pub_key.data,
+				op->pub_key.length,
+				peer_key);
+		if (peer_key == NULL) {
+			cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+			return -1;
+		}
+		priv_key = BN_bin2bn(op->priv_key.data,
+				op->priv_key.length,
+				priv_key);
+		if (priv_key == NULL) {
+			BN_free(peer_key);
+			cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+			return -1;
+		}
+		set_dh_priv_key(dh_key, priv_key, ret);
+		if (ret) {
+			OPENSSL_LOG(ERR, "Failed to set private key\n");
+			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			BN_free(peer_key);
+			BN_free(priv_key);
+			return 0;
+		}
+
+		ret = DH_compute_key(
+				op->shared_secret.data,
+				peer_key, dh_key);
+		if (ret < 0) {
+			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			BN_free(peer_key);
+			/* priv key is already loaded into dh,
+			 * let's not free that directly here.
+			 * DH_free() will auto free it later.
+			 */
+			return 0;
+		}
+		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		op->shared_secret.length = ret;
+		BN_free(peer_key);
+		return 0;
+	}
+
+	/*
+	 * other options are public and private key generations.
+	 *
+	 * if user provides private key,
+	 * then first set DH with user provided private key
+	 */
+	if ((sess->u.dh.key_op &
+			(1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) &&
+			!(sess->u.dh.key_op &
+			(1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE))) {
+		/* generate public key using user-provided private key
+		 * pub_key = g ^ priv_key mod p
+		 */
+
+		/* load private key into DH */
+		priv_key = BN_bin2bn(op->priv_key.data,
+				op->priv_key.length,
+				priv_key);
+		if (priv_key == NULL) {
+			cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+			return -1;
+		}
+		set_dh_priv_key(dh_key, priv_key, ret);
+		if (ret) {
+			OPENSSL_LOG(ERR, "Failed to set private key\n");
+			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			BN_free(priv_key);
+			return 0;
+		}
+	}
+
+	/* generate public and private key pair.
+	 *
+	 * if private key already set, generates only public key.
+	 *
+	 * if private key is not already set, then set it to random value
+	 * and update internal private key.
+	 */
+	if (!DH_generate_key(dh_key)) {
+		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return 0;
+	}
+
+	if (sess->u.dh.key_op & (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) {
+		const BIGNUM *pub_key = NULL;
+
+		OPENSSL_LOG(DEBUG, "%s:%d update public key\n",
+				__func__, __LINE__);
+
+		/* get the generated keys */
+		get_dh_pub_key(dh_key, pub_key);
+
+		/* output public key */
+		op->pub_key.length = BN_bn2bin(pub_key,
+				op->pub_key.data);
+	}
+
+	if (sess->u.dh.key_op &
+			(1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE)) {
+		const BIGNUM *priv_key = NULL;
+
+		OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n",
+				__func__, __LINE__);
+
+		/* get the generated keys */
+		get_dh_priv_key(dh_key, priv_key);
+
+		/* provide generated private key back to user */
+		op->priv_key.length = BN_bn2bin(priv_key,
+				op->priv_key.data);
+	}
+
+	cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+
+	return 0;
+}
+
+/* process modinv operation */
+static int
+process_openssl_modinv_op(struct rte_crypto_op *cop,
+		struct openssl_asym_session *sess)
+{
+	struct rte_crypto_asym_op *op = cop->asym;
+	BIGNUM *base = BN_CTX_get(sess->u.m.ctx);
+	BIGNUM *res = BN_CTX_get(sess->u.m.ctx);
+
+	if (unlikely(base == NULL || res == NULL)) {
+		if (base)
+			BN_free(base);
+		if (res)
+			BN_free(res);
+		cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+		return -1;
+	}
+
+	base = BN_bin2bn((const unsigned char *)op->modinv.base.data,
+			op->modinv.base.length, base);
+
+	if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) {
+		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		op->modinv.base.length = BN_bn2bin(res, op->modinv.base.data);
+	} else {
+		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	}
+
+	return 0;
+}
+
+/* process modexp operation */
+static int
+process_openssl_modexp_op(struct rte_crypto_op *cop,
+		struct openssl_asym_session *sess)
+{
+	struct rte_crypto_asym_op *op = cop->asym;
+	BIGNUM *base = BN_CTX_get(sess->u.e.ctx);
+	BIGNUM *res = BN_CTX_get(sess->u.e.ctx);
+
+	if (unlikely(base == NULL || res == NULL)) {
+		if (base)
+			BN_free(base);
+		if (res)
+			BN_free(res);
+		cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+		return -1;
+	}
+
+	base = BN_bin2bn((const unsigned char *)op->modinv.base.data,
+			op->modinv.base.length, base);
+
+	if (BN_mod_exp(res, base, sess->u.e.exp,
+				sess->u.e.mod, sess->u.e.ctx)) {
+		op->modinv.base.length = BN_bn2bin(res, op->modinv.base.data);
+		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+	} else {
+		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	}
+
+	return 0;
+}
+
+/* process rsa operations */
+static int
+process_openssl_rsa_op(struct rte_crypto_op *cop,
+		struct openssl_asym_session *sess)
+{
+	int ret = 0;
+	struct rte_crypto_asym_op *op = cop->asym;
+	RSA *rsa = sess->u.r.rsa;
+	uint32_t pad = (op->rsa.pad);
+
+	switch (pad) {
+	case RTE_CRYPTO_RSA_PKCS1_V1_5_BT0:
+	case RTE_CRYPTO_RSA_PKCS1_V1_5_BT1:
+	case RTE_CRYPTO_RSA_PKCS1_V1_5_BT2:
+		pad = RSA_PKCS1_PADDING;
+		break;
+	case RTE_CRYPTO_RSA_PADDING_NONE:
+		pad = RSA_NO_PADDING;
+		break;
+	default:
+		cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		OPENSSL_LOG(ERR,
+				"rsa pad type not supported %d\n", pad);
+		return 0;
+	}
+
+	switch (op->rsa.op_type) {
+	case RTE_CRYPTO_ASYM_OP_ENCRYPT:
+		ret = RSA_public_encrypt(op->rsa.message.length,
+				op->rsa.message.data,
+				op->rsa.message.data,
+				rsa,
+				pad);
+
+		if (ret > 0)
+			op->rsa.message.length = ret;
+		OPENSSL_LOG(DEBUG,
+				"length of encrypted text %d\n", ret);
+		break;
+
+	case RTE_CRYPTO_ASYM_OP_DECRYPT:
+		ret = RSA_private_decrypt(op->rsa.message.length,
+				op->rsa.message.data,
+				op->rsa.message.data,
+				rsa,
+				pad);
+		if (ret > 0)
+			op->rsa.message.length = ret;
+		break;
+
+	case RTE_CRYPTO_ASYM_OP_SIGN:
+		ret = RSA_private_encrypt(op->rsa.message.length,
+				op->rsa.message.data,
+				op->rsa.sign.data,
+				rsa,
+				pad);
+		if (ret > 0)
+			op->rsa.sign.length = ret;
+		break;
+
+	case RTE_CRYPTO_ASYM_OP_VERIFY:
+		ret = RSA_public_decrypt(op->rsa.sign.length,
+				op->rsa.sign.data,
+				op->rsa.sign.data,
+				rsa,
+				pad);
+
+		OPENSSL_LOG(DEBUG,
+				"Length of public_decrypt %d "
+				"length of message %zd\n",
+				ret, op->rsa.message.length);
+
+		if (memcmp(op->rsa.sign.data, op->rsa.message.data,
+					op->rsa.message.length)) {
+			OPENSSL_LOG(ERR,
+					"RSA sign Verification failed");
+			return -1;
+		}
+		break;
+
+	default:
+		/* allow ops with invalid args to be pushed to
+		 * completion queue
+		 */
+		cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		break;
+	}
+
+	if (ret < 0)
+		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+
+	return 0;
+}
+
+static int
+process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op,
+		struct openssl_asym_session *sess)
+{
+	int retval = 0;
+
+	op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+
+	switch (sess->xfrm_type) {
+	case RTE_CRYPTO_ASYM_XFORM_RSA:
+		retval = process_openssl_rsa_op(op, sess);
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_MODEX:
+		retval = process_openssl_modexp_op(op, sess);
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_MODINV:
+		retval = process_openssl_modinv_op(op, sess);
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_DH:
+		retval = process_openssl_dh_op(op, sess);
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_DSA:
+		if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN)
+			retval = process_openssl_dsa_sign_op(op, sess);
+		else if (op->asym->dsa.op_type ==
+				RTE_CRYPTO_ASYM_OP_VERIFY)
+			retval =
+				process_openssl_dsa_verify_op(op, sess);
+		else
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		break;
+	default:
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+		break;
+	}
+	if (!retval) {
+		/* op processed so push to completion queue as processed */
+		retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
+		if (retval)
+			/* return error if failed to put in completion queue */
+			retval = -1;
+	}
+
+	return retval;
+}
+
 /** Process crypto operation for mbuf */
 static int
 process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
@@ -1597,7 +2045,7 @@ static uint16_t
 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
 {
-	struct openssl_session *sess;
+	void *sess;
 	struct openssl_qp *qp = queue_pair;
 	int i, retval;
 
@@ -1606,7 +2054,12 @@ openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		if (unlikely(sess == NULL))
 			goto enqueue_err;
 
-		retval = process_op(qp, ops[i], sess);
+		if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
+			retval = process_op(qp, ops[i],
+					(struct openssl_session *) sess);
+		else
+			retval = process_asym_op(qp, ops[i],
+					(struct openssl_asym_session *) sess);
 		if (unlikely(retval < 0))
 			goto enqueue_err;
 	}
@@ -1646,7 +2099,7 @@ cryptodev_openssl_create(const char *name,
 
 	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
 	if (dev == NULL) {
-		OPENSSL_LOG_ERR("failed to create cryptodev vdev");
+		OPENSSL_LOG(ERR, "failed to create cryptodev vdev");
 		goto init_error;
 	}
 
@@ -1660,18 +2113,19 @@ cryptodev_openssl_create(const char *name,
 	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 			RTE_CRYPTODEV_FF_CPU_AESNI |
-			RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
+			RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO;
 
 	/* Set vector instructions mode supported */
 	internals = dev->data->dev_private;
 
 	internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
-	internals->max_nb_sessions = init_params->max_nb_sessions;
 
 	return 0;
 
 init_error:
-	OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed",
+	OPENSSL_LOG(ERR, "driver %s: create failed",
 			init_params->name);
 
 	cryptodev_openssl_remove(vdev);
@@ -1686,8 +2140,7 @@ cryptodev_openssl_probe(struct rte_vdev_device *vdev)
 		"",
 		sizeof(struct openssl_private),
 		rte_socket_id(),
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
+		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 	};
 	const char *name;
 	const char *input_args;
@@ -1731,7 +2184,11 @@ RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
 	cryptodev_openssl_pmd_drv);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
 	"max_nb_queue_pairs=<int> "
-	"max_nb_sessions=<int> "
 	"socket_id=<int>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv,
 		cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id);
+
+RTE_INIT(openssl_init_log)
+{
+	openssl_logtype_driver = rte_log_register("pmd.crypto.openssl");
+}
diff --git a/drivers/crypto/openssl/rte_openssl_pmd_ops.c b/drivers/crypto/openssl/rte_openssl_pmd_ops.c
index 1cb87d59..de228439 100644
--- a/drivers/crypto/openssl/rte_openssl_pmd_ops.c
+++ b/drivers/crypto/openssl/rte_openssl_pmd_ops.c
@@ -9,6 +9,7 @@
 #include <rte_cryptodev_pmd.h>
 
 #include "rte_openssl_pmd_private.h"
+#include "compat.h"
 
 
 static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = {
@@ -397,7 +398,7 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = {
 				.algo = RTE_CRYPTO_CIPHER_3DES_CBC,
 				.block_size = 8,
 				.key_size = {
-					.min = 16,
+					.min = 8,
 					.max = 24,
 					.increment = 8
 				},
@@ -469,6 +470,105 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = {
 			}, }
 		}, }
 	},
+	{	/* RSA */
+		.op = RTE_CRYPTO_OP_TYPE_ASYMMETRIC,
+		{.asym = {
+			.xform_capa = {
+				.xform_type = RTE_CRYPTO_ASYM_XFORM_RSA,
+				.op_types = ((1 << RTE_CRYPTO_ASYM_OP_SIGN) |
+					(1 << RTE_CRYPTO_ASYM_OP_VERIFY) |
+					(1 << RTE_CRYPTO_ASYM_OP_ENCRYPT) |
+					(1 << RTE_CRYPTO_ASYM_OP_DECRYPT)),
+				{
+				.modlen = {
+				/* min length is based on openssl rsa keygen */
+				.min = 30,
+				/* value 0 symbolizes no limit on max length */
+				.max = 0,
+				.increment = 1
+				}, }
+			}
+		},
+		}
+	},
+	{	/* modexp */
+		.op = RTE_CRYPTO_OP_TYPE_ASYMMETRIC,
+		{.asym = {
+			.xform_capa = {
+				.xform_type = RTE_CRYPTO_ASYM_XFORM_MODEX,
+				.op_types = 0,
+				{
+				.modlen = {
+				/* value 0 symbolizes no limit on min length */
+				.min = 0,
+				/* value 0 symbolizes no limit on max length */
+				.max = 0,
+				.increment = 1
+				}, }
+			}
+		},
+		}
+	},
+	{	/* modinv */
+		.op = RTE_CRYPTO_OP_TYPE_ASYMMETRIC,
+		{.asym = {
+			.xform_capa = {
+				.xform_type = RTE_CRYPTO_ASYM_XFORM_MODINV,
+				.op_types = 0,
+				{
+				.modlen = {
+				/* value 0 symbolizes no limit on min length */
+				.min = 0,
+				/* value 0 symbolizes no limit on max length */
+				.max = 0,
+				.increment = 1
+				}, }
+			}
+		},
+		}
+	},
+	{	/* dh */
+		.op = RTE_CRYPTO_OP_TYPE_ASYMMETRIC,
+		{.asym = {
+			.xform_capa = {
+				.xform_type = RTE_CRYPTO_ASYM_XFORM_DH,
+				.op_types =
+				((1<<RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE) |
+				(1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE |
+				(1 <<
+				RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE))),
+				{
+				.modlen = {
+				/* value 0 symbolizes no limit on min length */
+				.min = 0,
+				/* value 0 symbolizes no limit on max length */
+				.max = 0,
+				.increment = 1
+				}, }
+			}
+		},
+		}
+	},
+	{	/* dsa */
+		.op = RTE_CRYPTO_OP_TYPE_ASYMMETRIC,
+		{.asym = {
+			.xform_capa = {
+				.xform_type = RTE_CRYPTO_ASYM_XFORM_DSA,
+				.op_types =
+				((1<<RTE_CRYPTO_ASYM_OP_SIGN) |
+				(1 << RTE_CRYPTO_ASYM_OP_VERIFY)),
+				{
+				.modlen = {
+				/* value 0 symbolizes no limit on min length */
+				.min = 0,
+				/* value 0 symbolizes no limit on max length */
+				.max = 0,
+				.increment = 1
+				}, }
+			}
+		},
+		}
+	},
 
 	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 };
@@ -547,7 +647,8 @@ openssl_pmd_info_get(struct rte_cryptodev *dev,
 		dev_info->feature_flags = dev->feature_flags;
 		dev_info->capabilities = openssl_pmd_capabilities;
 		dev_info->max_nb_queue_pairs = internals->max_nb_qpairs;
-		dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
+		/* No limit of number of sessions */
+		dev_info->sym.max_nb_sessions = 0;
 	}
 }
 
@@ -588,14 +689,14 @@ openssl_pmd_qp_create_processed_ops_ring(struct openssl_qp *qp,
 	r = rte_ring_lookup(qp->name);
 	if (r) {
 		if (rte_ring_get_size(r) >= ring_size) {
-			OPENSSL_LOG_INFO(
-				"Reusing existing ring %s for processed ops",
+			OPENSSL_LOG(INFO,
+					"Reusing existing ring %s for processed ops",
 				 qp->name);
 			return r;
 		}
 
-		OPENSSL_LOG_ERR(
-			"Unable to reuse existing ring %s for processed ops",
+		OPENSSL_LOG(ERR,
+				"Unable to reuse existing ring %s for processed ops",
 			 qp->name);
 		return NULL;
 	}
@@ -647,22 +748,6 @@ qp_setup_cleanup:
 	return -1;
 }
 
-/** Start queue pair */
-static int
-openssl_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
-/** Stop queue pair */
-static int
-openssl_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
 /** Return the number of allocated queue pairs */
 static uint32_t
 openssl_pmd_qp_count(struct rte_cryptodev *dev)
@@ -670,16 +755,23 @@ openssl_pmd_qp_count(struct rte_cryptodev *dev)
 	return dev->data->nb_queue_pairs;
 }
 
-/** Returns the size of the session structure */
+/** Returns the size of the symmetric session structure */
 static unsigned
-openssl_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
+openssl_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	return sizeof(struct openssl_session);
 }
 
+/** Returns the size of the asymmetric session structure */
+static unsigned
+openssl_pmd_asym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct openssl_asym_session);
+}
+
 /** Configure the session from a crypto xform chain */
 static int
-openssl_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
+openssl_pmd_sym_session_configure(struct rte_cryptodev *dev __rte_unused,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *sess,
 		struct rte_mempool *mempool)
@@ -688,46 +780,460 @@ openssl_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
 	int ret;
 
 	if (unlikely(sess == NULL)) {
-		OPENSSL_LOG_ERR("invalid session struct");
+		OPENSSL_LOG(ERR, "invalid session struct");
 		return -EINVAL;
 	}
 
 	if (rte_mempool_get(mempool, &sess_private_data)) {
-		CDEV_LOG_ERR(
+		OPENSSL_LOG(ERR,
 			"Couldn't get object from session mempool");
 		return -ENOMEM;
 	}
 
 	ret = openssl_set_session_parameters(sess_private_data, xform);
 	if (ret != 0) {
-		OPENSSL_LOG_ERR("failed configure session parameters");
+		OPENSSL_LOG(ERR, "failed configure session parameters");
 
 		/* Return session to mempool */
 		rte_mempool_put(mempool, sess_private_data);
 		return ret;
 	}
 
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 			sess_private_data);
 
 	return 0;
 }
 
+static int openssl_set_asym_session_parameters(
+		struct openssl_asym_session *asym_session,
+		struct rte_crypto_asym_xform *xform)
+{
+	int ret = 0;
+
+	if ((xform->xform_type != RTE_CRYPTO_ASYM_XFORM_DH) &&
+		(xform->next != NULL)) {
+		OPENSSL_LOG(ERR, "chained xfrms are not supported on %s",
+			rte_crypto_asym_xform_strings[xform->xform_type]);
+		return -1;
+	}
+
+	switch (xform->xform_type) {
+	case RTE_CRYPTO_ASYM_XFORM_RSA:
+	{
+		BIGNUM *n = NULL;
+		BIGNUM *e = NULL;
+		BIGNUM *d = NULL;
+		BIGNUM *p = NULL, *q = NULL, *dmp1 = NULL;
+		BIGNUM *iqmp = NULL, *dmq1 = NULL;
+
+		/* copy xfrm data into rsa struct */
+		n = BN_bin2bn((const unsigned char *)xform->rsa.n.data,
+				xform->rsa.n.length, n);
+		e = BN_bin2bn((const unsigned char *)xform->rsa.e.data,
+				xform->rsa.e.length, e);
+
+		if (!n || !e)
+			goto err_rsa;
+
+		RSA *rsa = RSA_new();
+		if (rsa == NULL)
+			goto err_rsa;
+
+		if (xform->rsa.key_type == RTE_RSA_KEY_TYPE_EXP) {
+			d = BN_bin2bn(
+			(const unsigned char *)xform->rsa.d.data,
+			xform->rsa.d.length,
+			d);
+			if (!d) {
+				RSA_free(rsa);
+				goto err_rsa;
+			}
+		} else {
+			p = BN_bin2bn((const unsigned char *)
+					xform->rsa.qt.p.data,
+					xform->rsa.qt.p.length,
+					p);
+			q = BN_bin2bn((const unsigned char *)
+					xform->rsa.qt.q.data,
+					xform->rsa.qt.q.length,
+					q);
+			dmp1 = BN_bin2bn((const unsigned char *)
+					xform->rsa.qt.dP.data,
+					xform->rsa.qt.dP.length,
+					dmp1);
+			dmq1 = BN_bin2bn((const unsigned char *)
+					xform->rsa.qt.dQ.data,
+					xform->rsa.qt.dQ.length,
+					dmq1);
+			iqmp = BN_bin2bn((const unsigned char *)
+					xform->rsa.qt.qInv.data,
+					xform->rsa.qt.qInv.length,
+					iqmp);
+
+			if (!p || !q || !dmp1 || !dmq1 || !iqmp) {
+				RSA_free(rsa);
+				goto err_rsa;
+			}
+			set_rsa_params(rsa, p, q, ret);
+			if (ret) {
+				OPENSSL_LOG(ERR,
+					"failed to set rsa params\n");
+				RSA_free(rsa);
+				goto err_rsa;
+			}
+			set_rsa_crt_params(rsa, dmp1, dmq1, iqmp, ret);
+			if (ret) {
+				OPENSSL_LOG(ERR,
+					"failed to set crt params\n");
+				RSA_free(rsa);
+				/*
+				 * set already populated params to NULL
+				 * as its freed by call to RSA_free
+				 */
+				p = q = NULL;
+				goto err_rsa;
+			}
+		}
+
+		set_rsa_keys(rsa, n, e, d, ret);
+		if (ret) {
+			OPENSSL_LOG(ERR, "Failed to load rsa keys\n");
+			RSA_free(rsa);
+			return -1;
+		}
+		asym_session->u.r.rsa = rsa;
+		asym_session->xfrm_type = RTE_CRYPTO_ASYM_XFORM_RSA;
+		break;
+err_rsa:
+		if (n)
+			BN_free(n);
+		if (e)
+			BN_free(e);
+		if (d)
+			BN_free(d);
+		if (p)
+			BN_free(p);
+		if (q)
+			BN_free(q);
+		if (dmp1)
+			BN_free(dmp1);
+		if (dmq1)
+			BN_free(dmq1);
+		if (iqmp)
+			BN_free(iqmp);
+
+		return -1;
+	}
+	case RTE_CRYPTO_ASYM_XFORM_MODEX:
+	{
+		struct rte_crypto_modex_xform *xfrm = &(xform->modex);
+
+		BN_CTX *ctx = BN_CTX_new();
+		if (ctx == NULL) {
+			OPENSSL_LOG(ERR,
+				" failed to allocate resources\n");
+			return -1;
+		}
+		BN_CTX_start(ctx);
+		BIGNUM *mod = BN_CTX_get(ctx);
+		BIGNUM *exp = BN_CTX_get(ctx);
+		if (mod == NULL || exp == NULL) {
+			BN_CTX_end(ctx);
+			BN_CTX_free(ctx);
+			return -1;
+		}
+
+		mod = BN_bin2bn((const unsigned char *)
+				xfrm->modulus.data,
+				xfrm->modulus.length, mod);
+		exp = BN_bin2bn((const unsigned char *)
+				xfrm->exponent.data,
+				xfrm->exponent.length, exp);
+		asym_session->u.e.ctx = ctx;
+		asym_session->u.e.mod = mod;
+		asym_session->u.e.exp = exp;
+		asym_session->xfrm_type = RTE_CRYPTO_ASYM_XFORM_MODEX;
+		break;
+	}
+	case RTE_CRYPTO_ASYM_XFORM_MODINV:
+	{
+		struct rte_crypto_modinv_xform *xfrm = &(xform->modinv);
+
+		BN_CTX *ctx = BN_CTX_new();
+		if (ctx == NULL) {
+			OPENSSL_LOG(ERR,
+				" failed to allocate resources\n");
+			return -1;
+		}
+		BN_CTX_start(ctx);
+		BIGNUM *mod = BN_CTX_get(ctx);
+		if (mod == NULL) {
+			BN_CTX_end(ctx);
+			BN_CTX_free(ctx);
+			return -1;
+		}
+
+		mod = BN_bin2bn((const unsigned char *)
+				xfrm->modulus.data,
+				xfrm->modulus.length,
+				mod);
+		asym_session->u.m.ctx = ctx;
+		asym_session->u.m.modulus = mod;
+		asym_session->xfrm_type = RTE_CRYPTO_ASYM_XFORM_MODINV;
+		break;
+	}
+	case RTE_CRYPTO_ASYM_XFORM_DH:
+	{
+		BIGNUM *p = NULL;
+		BIGNUM *g = NULL;
+
+		p = BN_bin2bn((const unsigned char *)
+				xform->dh.p.data,
+				xform->dh.p.length,
+				p);
+		g = BN_bin2bn((const unsigned char *)
+				xform->dh.g.data,
+				xform->dh.g.length,
+				g);
+		if (!p || !g)
+			goto err_dh;
+
+		DH *dh = DH_new();
+		if (dh == NULL) {
+			OPENSSL_LOG(ERR,
+				"failed to allocate resources\n");
+			goto err_dh;
+		}
+		set_dh_params(dh, p, g, ret);
+		if (ret) {
+			DH_free(dh);
+			goto err_dh;
+		}
+
+		/*
+		 * setup xfrom for
+		 * public key generate, or
+		 * DH Priv key generate, or both
+		 * public and private key generate
+		 */
+		asym_session->u.dh.key_op = (1 << xform->dh.type);
+
+		if (xform->dh.type ==
+			RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE) {
+			/* check if next is pubkey */
+			if ((xform->next != NULL) &&
+				(xform->next->xform_type ==
+				RTE_CRYPTO_ASYM_XFORM_DH) &&
+				(xform->next->dh.type ==
+				RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)
+				) {
+				/*
+				 * setup op as pub/priv key
+				 * pair generationi
+				 */
+				asym_session->u.dh.key_op |=
+				(1 <<
+				RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE);
+			}
+		}
+		asym_session->u.dh.dh_key = dh;
+		asym_session->xfrm_type = RTE_CRYPTO_ASYM_XFORM_DH;
+		break;
+
+err_dh:
+		OPENSSL_LOG(ERR, " failed to set dh params\n");
+		if (p)
+			BN_free(p);
+		if (g)
+			BN_free(g);
+		return -1;
+	}
+	case RTE_CRYPTO_ASYM_XFORM_DSA:
+	{
+		BIGNUM *p = NULL, *g = NULL;
+		BIGNUM *q = NULL, *priv_key = NULL;
+		BIGNUM *pub_key = BN_new();
+		BN_zero(pub_key);
+
+		p = BN_bin2bn((const unsigned char *)
+				xform->dsa.p.data,
+				xform->dsa.p.length,
+				p);
+
+		g = BN_bin2bn((const unsigned char *)
+				xform->dsa.g.data,
+				xform->dsa.g.length,
+				g);
+
+		q = BN_bin2bn((const unsigned char *)
+				xform->dsa.q.data,
+				xform->dsa.q.length,
+				q);
+		if (!p || !q || !g)
+			goto err_dsa;
+
+		priv_key = BN_bin2bn((const unsigned char *)
+				xform->dsa.x.data,
+				xform->dsa.x.length,
+				priv_key);
+		if (priv_key == NULL)
+			goto err_dsa;
+
+		DSA *dsa = DSA_new();
+		if (dsa == NULL) {
+			OPENSSL_LOG(ERR,
+				" failed to allocate resources\n");
+			goto err_dsa;
+		}
+
+		set_dsa_params(dsa, p, q, g, ret);
+		if (ret) {
+			DSA_free(dsa);
+			OPENSSL_LOG(ERR, "Failed to dsa params\n");
+			goto err_dsa;
+		}
+
+		/*
+		 * openssl 1.1.0 mandate that public key can't be
+		 * NULL in very first call. so set a dummy pub key.
+		 * to keep consistency, lets follow same approach for
+		 * both versions
+		 */
+		/* just set dummy public for very 1st call */
+		set_dsa_keys(dsa, pub_key, priv_key, ret);
+		if (ret) {
+			DSA_free(dsa);
+			OPENSSL_LOG(ERR, "Failed to set keys\n");
+			return -1;
+		}
+		asym_session->u.s.dsa = dsa;
+		asym_session->xfrm_type = RTE_CRYPTO_ASYM_XFORM_DSA;
+		break;
+
+err_dsa:
+		if (p)
+			BN_free(p);
+		if (q)
+			BN_free(q);
+		if (g)
+			BN_free(g);
+		if (priv_key)
+			BN_free(priv_key);
+		if (pub_key)
+			BN_free(pub_key);
+		return -1;
+	}
+	default:
+		return -1;
+	}
+
+	return 0;
+}
+
+/** Configure the session from a crypto xform chain */
+static int
+openssl_pmd_asym_session_configure(struct rte_cryptodev *dev __rte_unused,
+		struct rte_crypto_asym_xform *xform,
+		struct rte_cryptodev_asym_session *sess,
+		struct rte_mempool *mempool)
+{
+	void *asym_sess_private_data;
+	int ret;
+
+	if (unlikely(sess == NULL)) {
+		OPENSSL_LOG(ERR, "invalid asymmetric session struct");
+		return -EINVAL;
+	}
+
+	if (rte_mempool_get(mempool, &asym_sess_private_data)) {
+		CDEV_LOG_ERR(
+			"Couldn't get object from session mempool");
+		return -ENOMEM;
+	}
+
+	ret = openssl_set_asym_session_parameters(asym_sess_private_data,
+			xform);
+	if (ret != 0) {
+		OPENSSL_LOG(ERR, "failed configure session parameters");
+
+		/* Return session to mempool */
+		rte_mempool_put(mempool, asym_sess_private_data);
+		return ret;
+	}
+
+	set_asym_session_private_data(sess, dev->driver_id,
+			asym_sess_private_data);
+
+	return 0;
+}
 
 /** Clear the memory of session so it doesn't leave key material behind */
 static void
-openssl_pmd_session_clear(struct rte_cryptodev *dev,
+openssl_pmd_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 
 	/* Zero out the whole structure */
 	if (sess_priv) {
 		openssl_reset_session(sess_priv);
 		memset(sess_priv, 0, sizeof(struct openssl_session));
 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
+		rte_mempool_put(sess_mp, sess_priv);
+	}
+}
+
+static void openssl_reset_asym_session(struct openssl_asym_session *sess)
+{
+	switch (sess->xfrm_type) {
+	case RTE_CRYPTO_ASYM_XFORM_RSA:
+		if (sess->u.r.rsa)
+			RSA_free(sess->u.r.rsa);
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_MODEX:
+		if (sess->u.e.ctx) {
+			BN_CTX_end(sess->u.e.ctx);
+			BN_CTX_free(sess->u.e.ctx);
+		}
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_MODINV:
+		if (sess->u.m.ctx) {
+			BN_CTX_end(sess->u.m.ctx);
+			BN_CTX_free(sess->u.m.ctx);
+		}
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_DH:
+		if (sess->u.dh.dh_key)
+			DH_free(sess->u.dh.dh_key);
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_DSA:
+		if (sess->u.s.dsa)
+			DSA_free(sess->u.s.dsa);
+		break;
+	default:
+		break;
+	}
+}
+
+/** Clear the memory of asymmetric session
+ * so it doesn't leave key material behind
+ */
+static void
+openssl_pmd_asym_session_clear(struct rte_cryptodev *dev,
+		struct rte_cryptodev_asym_session *sess)
+{
+	uint8_t index = dev->driver_id;
+	void *sess_priv = get_asym_session_private_data(sess, index);
+
+	/* Zero out the whole structure */
+	if (sess_priv) {
+		openssl_reset_asym_session(sess_priv);
+		memset(sess_priv, 0, sizeof(struct openssl_asym_session));
+		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+		set_asym_session_private_data(sess, index, NULL);
 		rte_mempool_put(sess_mp, sess_priv);
 	}
 }
@@ -745,13 +1251,14 @@ struct rte_cryptodev_ops openssl_pmd_ops = {
 
 		.queue_pair_setup	= openssl_pmd_qp_setup,
 		.queue_pair_release	= openssl_pmd_qp_release,
-		.queue_pair_start	= openssl_pmd_qp_start,
-		.queue_pair_stop	= openssl_pmd_qp_stop,
 		.queue_pair_count	= openssl_pmd_qp_count,
 
-		.session_get_size	= openssl_pmd_session_get_size,
-		.session_configure	= openssl_pmd_session_configure,
-		.session_clear		= openssl_pmd_session_clear
+		.sym_session_get_size	= openssl_pmd_sym_session_get_size,
+		.asym_session_get_size	= openssl_pmd_asym_session_get_size,
+		.sym_session_configure	= openssl_pmd_sym_session_configure,
+		.asym_session_configure	= openssl_pmd_asym_session_configure,
+		.sym_session_clear	= openssl_pmd_sym_session_clear,
+		.asym_session_clear	= openssl_pmd_asym_session_clear
 };
 
 struct rte_cryptodev_ops *rte_openssl_pmd_ops = &openssl_pmd_ops;
diff --git a/drivers/crypto/openssl/rte_openssl_pmd_private.h b/drivers/crypto/openssl/rte_openssl_pmd_private.h
index bc8dc7cd..a8f2c848 100644
--- a/drivers/crypto/openssl/rte_openssl_pmd_private.h
+++ b/drivers/crypto/openssl/rte_openssl_pmd_private.h
@@ -8,29 +8,19 @@
 #include <openssl/evp.h>
 #include <openssl/hmac.h>
 #include <openssl/des.h>
+#include <openssl/rsa.h>
+#include <openssl/dh.h>
+#include <openssl/dsa.h>
 
 #define CRYPTODEV_NAME_OPENSSL_PMD	crypto_openssl
 /**< Open SSL Crypto PMD device name */
 
-#define OPENSSL_LOG_ERR(fmt, args...) \
-	RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",  \
-			RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD), \
-			__func__, __LINE__, ## args)
-
-#ifdef RTE_LIBRTE_OPENSSL_DEBUG
-#define OPENSSL_LOG_INFO(fmt, args...) \
-	RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD), \
-			__func__, __LINE__, ## args)
-
-#define OPENSSL_LOG_DBG(fmt, args...) \
-	RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD), \
-			__func__, __LINE__, ## args)
-#else
-#define OPENSSL_LOG_INFO(fmt, args...)
-#define OPENSSL_LOG_DBG(fmt, args...)
-#endif
+/** OPENSSL PMD LOGTYPE DRIVER */
+int openssl_logtype_driver;
+#define OPENSSL_LOG(level, fmt, ...)  \
+	rte_log(RTE_LOG_ ## level, openssl_logtype_driver,  \
+			"%s() line %u: " fmt "\n", __func__, __LINE__,  \
+					## __VA_ARGS__)
 
 /* Maximum length for digest (SHA-512 needs 64 bytes) */
 #define DIGEST_LENGTH_MAX 64
@@ -62,8 +52,6 @@ enum openssl_auth_mode {
 struct openssl_private {
 	unsigned int max_nb_qpairs;
 	/**< Max number of queue pairs */
-	unsigned int max_nb_sessions;
-	/**< Max number of sessions */
 };
 
 /** OPENSSL crypto queue pair */
@@ -157,6 +145,31 @@ struct openssl_session {
 
 } __rte_cache_aligned;
 
+/** OPENSSL crypto private asymmetric session structure */
+struct openssl_asym_session {
+	enum rte_crypto_asym_xform_type xfrm_type;
+	union {
+		struct rsa {
+			RSA *rsa;
+		} r;
+		struct exp {
+			BIGNUM *exp;
+			BIGNUM *mod;
+			BN_CTX *ctx;
+		} e;
+		struct mod {
+			BIGNUM *modulus;
+			BN_CTX *ctx;
+		} m;
+		struct dh {
+			DH *dh_key;
+			uint32_t key_op;
+		} dh;
+		struct {
+			DSA *dsa;
+		} s;
+	} u;
+} __rte_cache_aligned;
 /** Set and validate OPENSSL crypto session parameters */
 extern int
 openssl_set_session_parameters(struct openssl_session *sess,
diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/qat/Makefile
deleted file mode 100644
index 07266a5e..00000000
--- a/drivers/crypto/qat/Makefile
+++ /dev/null
@@ -1,35 +0,0 @@
-# SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2015-2018 Intel Corporation
-
-include $(RTE_SDK)/mk/rte.vars.mk
-
-# library name
-LIB = librte_pmd_qat.a
-
-# library version
-LIBABIVER := 1
-
-# build flags
-CFLAGS += $(WERROR_FLAGS)
-CFLAGS += -O3
-
-# external library include paths
-CFLAGS += -I$(SRCDIR)/qat_adf
-LDLIBS += -lcrypto
-LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
-LDLIBS += -lrte_cryptodev
-LDLIBS += -lrte_pci -lrte_bus_pci
-
-# library source files
-SRCS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += qat_crypto.c
-SRCS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += qat_qp.c
-SRCS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += qat_adf/qat_algs_build_desc.c
-SRCS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += rte_qat_cryptodev.c
-
-# export include files
-SYMLINK-y-include +=
-
-# versioning export map
-EXPORT_MAP := rte_pmd_qat_version.map
-
-include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/crypto/qat/README b/drivers/crypto/qat/README
new file mode 100644
index 00000000..444ae605
--- /dev/null
+++ b/drivers/crypto/qat/README
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2015-2018 Intel Corporation
+
+Makefile for crypto QAT PMD is in common/qat directory.
+The build for the QAT driver is done from there as only one library is built for the
+whole QAT pci device and that library includes all the services (crypto, compression)
+which are enabled on the device.
diff --git a/drivers/crypto/qat/meson.build b/drivers/crypto/qat/meson.build
index 006cd655..9cc98d2c 100644
--- a/drivers/crypto/qat/meson.build
+++ b/drivers/crypto/qat/meson.build
@@ -1,14 +1,18 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017-2018 Intel Corporation
 
+# this does not build the QAT driver, instead that is done in the compression
+# driver which comes later. Here we just add our sources files to the list
+build = false
 dep = dependency('libcrypto', required: false)
-if not dep.found()
-	build = false
+qat_includes += include_directories('.')
+qat_deps += 'cryptodev'
+if dep.found()
+	# Add our sources files to the list
+	qat_sources += files('qat_sym_pmd.c',
+			     'qat_sym.c',
+			     'qat_sym_session.c')
+	qat_ext_deps += dep
+	pkgconfig_extra_libs += '-lcrypto'
+	qat_cflags += '-DBUILD_QAT_SYM'
 endif
-sources = files('qat_crypto.c', 'qat_qp.c',
-		'qat_adf/qat_algs_build_desc.c',
-		'rte_qat_cryptodev.c')
-includes += include_directories('qat_adf')
-deps += ['bus_pci']
-ext_deps += dep
-pkgconfig_extra_libs += '-lcrypto'
diff --git a/drivers/crypto/qat/qat_adf/qat_algs.h b/drivers/crypto/qat/qat_adf/qat_algs.h
deleted file mode 100644
index 88bd5f00..00000000
--- a/drivers/crypto/qat/qat_adf/qat_algs.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
- * Copyright(c) 2015-2018 Intel Corporation
- */
-#ifndef _ICP_QAT_ALGS_H_
-#define _ICP_QAT_ALGS_H_
-#include <rte_memory.h>
-#include <rte_crypto.h>
-#include "icp_qat_hw.h"
-#include "icp_qat_fw.h"
-#include "icp_qat_fw_la.h"
-#include "../qat_crypto.h"
-
-/*
- * Key Modifier (KM) value used in KASUMI algorithm in F9 mode to XOR
- * Integrity Key (IK)
- */
-#define KASUMI_F9_KEY_MODIFIER_4_BYTES   0xAAAAAAAA
-
-#define KASUMI_F8_KEY_MODIFIER_4_BYTES   0x55555555
-
-/* 3DES key sizes */
-#define QAT_3DES_KEY_SZ_OPT1 24 /* Keys are independent */
-#define QAT_3DES_KEY_SZ_OPT2 16 /* K3=K1 */
-
-#define QAT_AES_HW_CONFIG_CBC_ENC(alg) \
-	ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
-					ICP_QAT_HW_CIPHER_NO_CONVERT, \
-					ICP_QAT_HW_CIPHER_ENCRYPT)
-
-#define QAT_AES_HW_CONFIG_CBC_DEC(alg) \
-	ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
-					ICP_QAT_HW_CIPHER_KEY_CONVERT, \
-					ICP_QAT_HW_CIPHER_DECRYPT)
-
-struct qat_alg_buf {
-	uint32_t len;
-	uint32_t resrvd;
-	uint64_t addr;
-} __rte_packed;
-
-enum qat_crypto_proto_flag {
-	QAT_CRYPTO_PROTO_FLAG_NONE = 0,
-	QAT_CRYPTO_PROTO_FLAG_CCM = 1,
-	QAT_CRYPTO_PROTO_FLAG_GCM = 2,
-	QAT_CRYPTO_PROTO_FLAG_SNOW3G = 3,
-	QAT_CRYPTO_PROTO_FLAG_ZUC = 4
-};
-
-/*
- * Maximum number of SGL entries
- */
-#define QAT_SGL_MAX_NUMBER	16
-
-struct qat_alg_buf_list {
-	uint64_t resrvd;
-	uint32_t num_bufs;
-	uint32_t num_mapped_bufs;
-	struct qat_alg_buf bufers[QAT_SGL_MAX_NUMBER];
-} __rte_packed __rte_cache_aligned;
-
-struct qat_crypto_op_cookie {
-	struct qat_alg_buf_list qat_sgl_list_src;
-	struct qat_alg_buf_list qat_sgl_list_dst;
-	rte_iova_t qat_sgl_src_phys_addr;
-	rte_iova_t qat_sgl_dst_phys_addr;
-};
-
-/* Common content descriptor */
-struct qat_alg_cd {
-	struct icp_qat_hw_cipher_algo_blk cipher;
-	struct icp_qat_hw_auth_algo_blk hash;
-} __rte_packed __rte_cache_aligned;
-
-struct qat_session {
-	enum icp_qat_fw_la_cmd_id qat_cmd;
-	enum icp_qat_hw_cipher_algo qat_cipher_alg;
-	enum icp_qat_hw_cipher_dir qat_dir;
-	enum icp_qat_hw_cipher_mode qat_mode;
-	enum icp_qat_hw_auth_algo qat_hash_alg;
-	enum icp_qat_hw_auth_op auth_op;
-	void *bpi_ctx;
-	struct qat_alg_cd cd;
-	uint8_t *cd_cur_ptr;
-	rte_iova_t cd_paddr;
-	struct icp_qat_fw_la_bulk_req fw_req;
-	uint8_t aad_len;
-	struct qat_crypto_instance *inst;
-	struct {
-		uint16_t offset;
-		uint16_t length;
-	} cipher_iv;
-	struct {
-		uint16_t offset;
-		uint16_t length;
-	} auth_iv;
-	uint16_t digest_length;
-	rte_spinlock_t lock;	/* protects this struct */
-	enum qat_device_gen min_qat_dev_gen;
-};
-
-int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg);
-
-int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cd,
-						uint8_t *enckey,
-						uint32_t enckeylen);
-
-int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc,
-						uint8_t *authkey,
-						uint32_t authkeylen,
-						uint32_t aad_length,
-						uint32_t digestsize,
-						unsigned int operation);
-
-void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header,
-					enum qat_crypto_proto_flag proto_flags);
-
-int qat_alg_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
-int qat_alg_validate_aes_docsisbpi_key(int key_len,
-					enum icp_qat_hw_cipher_algo *alg);
-int qat_alg_validate_snow3g_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
-int qat_alg_validate_kasumi_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
-int qat_alg_validate_3des_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
-int qat_alg_validate_des_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
-int qat_cipher_get_block_size(enum icp_qat_hw_cipher_algo qat_cipher_alg);
-int qat_alg_validate_zuc_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
-#endif
diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c
deleted file mode 100644
index d9ce2a13..00000000
--- a/drivers/crypto/qat/qat_crypto.c
+++ /dev/null
@@ -1,1696 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2018 Intel Corporation
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <string.h>
-#include <inttypes.h>
-#include <errno.h>
-#include <sys/queue.h>
-#include <stdarg.h>
-
-#include <rte_common.h>
-#include <rte_log.h>
-#include <rte_debug.h>
-#include <rte_memory.h>
-#include <rte_tailq.h>
-#include <rte_malloc.h>
-#include <rte_launch.h>
-#include <rte_eal.h>
-#include <rte_per_lcore.h>
-#include <rte_lcore.h>
-#include <rte_branch_prediction.h>
-#include <rte_mempool.h>
-#include <rte_mbuf.h>
-#include <rte_string_fns.h>
-#include <rte_spinlock.h>
-#include <rte_hexdump.h>
-#include <rte_crypto_sym.h>
-#include <rte_byteorder.h>
-#include <rte_pci.h>
-#include <rte_bus_pci.h>
-
-#include <openssl/evp.h>
-
-#include "qat_logs.h"
-#include "qat_algs.h"
-#include "qat_crypto.h"
-#include "adf_transport_access_macros.h"
-
-#define BYTE_LENGTH    8
-/* bpi is only used for partial blocks of DES and AES
- * so AES block len can be assumed as max len for iv, src and dst
- */
-#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
-
-static int
-qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo,
-		struct qat_pmd_private *internals) {
-	int i = 0;
-	const struct rte_cryptodev_capabilities *capability;
-
-	while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
-			RTE_CRYPTO_OP_TYPE_UNDEFINED) {
-		if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
-			continue;
-
-		if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
-			continue;
-
-		if (capability->sym.cipher.algo == algo)
-			return 1;
-	}
-	return 0;
-}
-
-static int
-qat_is_auth_alg_supported(enum rte_crypto_auth_algorithm algo,
-		struct qat_pmd_private *internals) {
-	int i = 0;
-	const struct rte_cryptodev_capabilities *capability;
-
-	while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
-			RTE_CRYPTO_OP_TYPE_UNDEFINED) {
-		if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
-			continue;
-
-		if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
-			continue;
-
-		if (capability->sym.auth.algo == algo)
-			return 1;
-	}
-	return 0;
-}
-
-/** Encrypt a single partial block
- *  Depends on openssl libcrypto
- *  Uses ECB+XOR to do CFB encryption, same result, more performant
- */
-static inline int
-bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
-		uint8_t *iv, int ivlen, int srclen,
-		void *bpi_ctx)
-{
-	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
-	int encrypted_ivlen;
-	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
-	uint8_t *encr = encrypted_iv;
-
-	/* ECB method: encrypt the IV, then XOR this with plaintext */
-	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
-								<= 0)
-		goto cipher_encrypt_err;
-
-	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
-		*dst = *src ^ *encr;
-
-	return 0;
-
-cipher_encrypt_err:
-	PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
-	return -EINVAL;
-}
-
-/** Decrypt a single partial block
- *  Depends on openssl libcrypto
- *  Uses ECB+XOR to do CFB encryption, same result, more performant
- */
-static inline int
-bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
-		uint8_t *iv, int ivlen, int srclen,
-		void *bpi_ctx)
-{
-	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
-	int encrypted_ivlen;
-	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
-	uint8_t *encr = encrypted_iv;
-
-	/* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
-	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
-								<= 0)
-		goto cipher_decrypt_err;
-
-	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
-		*dst = *src ^ *encr;
-
-	return 0;
-
-cipher_decrypt_err:
-	PMD_DRV_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
-	return -EINVAL;
-}
-
-/** Creates a context in either AES or DES in ECB mode
- *  Depends on openssl libcrypto
- */
-static int
-bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo,
-		enum rte_crypto_cipher_operation direction __rte_unused,
-		uint8_t *key, void **ctx)
-{
-	const EVP_CIPHER *algo = NULL;
-	int ret;
-	*ctx = EVP_CIPHER_CTX_new();
-
-	if (*ctx == NULL) {
-		ret = -ENOMEM;
-		goto ctx_init_err;
-	}
-
-	if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI)
-		algo = EVP_des_ecb();
-	else
-		algo = EVP_aes_128_ecb();
-
-	/* IV will be ECB encrypted whether direction is encrypt or decrypt*/
-	if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) {
-		ret = -EINVAL;
-		goto ctx_init_err;
-	}
-
-	return 0;
-
-ctx_init_err:
-	if (*ctx != NULL)
-		EVP_CIPHER_CTX_free(*ctx);
-	return ret;
-}
-
-/** Frees a context previously created
- *  Depends on openssl libcrypto
- */
-static void
-bpi_cipher_ctx_free(void *bpi_ctx)
-{
-	if (bpi_ctx != NULL)
-		EVP_CIPHER_CTX_free((EVP_CIPHER_CTX *)bpi_ctx);
-}
-
-static inline uint32_t
-adf_modulo(uint32_t data, uint32_t shift);
-
-static inline int
-qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg,
-		struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp);
-
-void
-qat_crypto_sym_clear_session(struct rte_cryptodev *dev,
-		struct rte_cryptodev_sym_session *sess)
-{
-	PMD_INIT_FUNC_TRACE();
-	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
-	struct qat_session *s = (struct qat_session *)sess_priv;
-
-	if (sess_priv) {
-		if (s->bpi_ctx)
-			bpi_cipher_ctx_free(s->bpi_ctx);
-		memset(s, 0, qat_crypto_sym_get_session_private_size(dev));
-		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-		set_session_private_data(sess, index, NULL);
-		rte_mempool_put(sess_mp, sess_priv);
-	}
-}
-
-static int
-qat_get_cmd_id(const struct rte_crypto_sym_xform *xform)
-{
-	/* Cipher Only */
-	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL)
-		return ICP_QAT_FW_LA_CMD_CIPHER;
-
-	/* Authentication Only */
-	if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL)
-		return ICP_QAT_FW_LA_CMD_AUTH;
-
-	/* AEAD */
-	if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
-		/* AES-GCM and AES-CCM works with different direction
-		 * GCM first encrypts and generate hash where AES-CCM
-		 * first generate hash and encrypts. Similar relation
-		 * applies to decryption.
-		 */
-		if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
-			if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
-				return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
-			else
-				return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
-		else
-			if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
-				return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
-			else
-				return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
-	}
-
-	if (xform->next == NULL)
-		return -1;
-
-	/* Cipher then Authenticate */
-	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
-			xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
-		return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
-
-	/* Authenticate then Cipher */
-	if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
-			xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
-		return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
-
-	return -1;
-}
-
-static struct rte_crypto_auth_xform *
-qat_get_auth_xform(struct rte_crypto_sym_xform *xform)
-{
-	do {
-		if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
-			return &xform->auth;
-
-		xform = xform->next;
-	} while (xform);
-
-	return NULL;
-}
-
-static struct rte_crypto_cipher_xform *
-qat_get_cipher_xform(struct rte_crypto_sym_xform *xform)
-{
-	do {
-		if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
-			return &xform->cipher;
-
-		xform = xform->next;
-	} while (xform);
-
-	return NULL;
-}
-
-int
-qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev,
-		struct rte_crypto_sym_xform *xform,
-		struct qat_session *session)
-{
-	struct qat_pmd_private *internals = dev->data->dev_private;
-	struct rte_crypto_cipher_xform *cipher_xform = NULL;
-	int ret;
-
-	/* Get cipher xform from crypto xform chain */
-	cipher_xform = qat_get_cipher_xform(xform);
-
-	session->cipher_iv.offset = cipher_xform->iv.offset;
-	session->cipher_iv.length = cipher_xform->iv.length;
-
-	switch (cipher_xform->algo) {
-	case RTE_CRYPTO_CIPHER_AES_CBC:
-		if (qat_alg_validate_aes_key(cipher_xform->key.length,
-				&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
-			ret = -EINVAL;
-			goto error_out;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
-		break;
-	case RTE_CRYPTO_CIPHER_AES_CTR:
-		if (qat_alg_validate_aes_key(cipher_xform->key.length,
-				&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
-			ret = -EINVAL;
-			goto error_out;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
-		break;
-	case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
-		if (qat_alg_validate_snow3g_key(cipher_xform->key.length,
-					&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid SNOW 3G cipher key size");
-			ret = -EINVAL;
-			goto error_out;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
-		break;
-	case RTE_CRYPTO_CIPHER_NULL:
-		session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
-		break;
-	case RTE_CRYPTO_CIPHER_KASUMI_F8:
-		if (qat_alg_validate_kasumi_key(cipher_xform->key.length,
-					&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid KASUMI cipher key size");
-			ret = -EINVAL;
-			goto error_out;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE;
-		break;
-	case RTE_CRYPTO_CIPHER_3DES_CBC:
-		if (qat_alg_validate_3des_key(cipher_xform->key.length,
-				&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size");
-			ret = -EINVAL;
-			goto error_out;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
-		break;
-	case RTE_CRYPTO_CIPHER_DES_CBC:
-		if (qat_alg_validate_des_key(cipher_xform->key.length,
-				&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid DES cipher key size");
-			ret = -EINVAL;
-			goto error_out;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
-		break;
-	case RTE_CRYPTO_CIPHER_3DES_CTR:
-		if (qat_alg_validate_3des_key(cipher_xform->key.length,
-				&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size");
-			ret = -EINVAL;
-			goto error_out;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
-		break;
-	case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
-		ret = bpi_cipher_ctx_init(
-					cipher_xform->algo,
-					cipher_xform->op,
-					cipher_xform->key.data,
-					&session->bpi_ctx);
-		if (ret != 0) {
-			PMD_DRV_LOG(ERR, "failed to create DES BPI ctx");
-			goto error_out;
-		}
-		if (qat_alg_validate_des_key(cipher_xform->key.length,
-				&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid DES cipher key size");
-			ret = -EINVAL;
-			goto error_out;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
-		break;
-	case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
-		ret = bpi_cipher_ctx_init(
-					cipher_xform->algo,
-					cipher_xform->op,
-					cipher_xform->key.data,
-					&session->bpi_ctx);
-		if (ret != 0) {
-			PMD_DRV_LOG(ERR, "failed to create AES BPI ctx");
-			goto error_out;
-		}
-		if (qat_alg_validate_aes_docsisbpi_key(cipher_xform->key.length,
-				&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid AES DOCSISBPI key size");
-			ret = -EINVAL;
-			goto error_out;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
-		break;
-	case RTE_CRYPTO_CIPHER_ZUC_EEA3:
-		if (!qat_is_cipher_alg_supported(
-			cipher_xform->algo, internals)) {
-			PMD_DRV_LOG(ERR, "%s not supported on this device",
-				rte_crypto_cipher_algorithm_strings
-					[cipher_xform->algo]);
-			ret = -ENOTSUP;
-			goto error_out;
-		}
-		if (qat_alg_validate_zuc_key(cipher_xform->key.length,
-				&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid ZUC cipher key size");
-			ret = -EINVAL;
-			goto error_out;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
-		break;
-	case RTE_CRYPTO_CIPHER_3DES_ECB:
-	case RTE_CRYPTO_CIPHER_AES_ECB:
-	case RTE_CRYPTO_CIPHER_AES_F8:
-	case RTE_CRYPTO_CIPHER_AES_XTS:
-	case RTE_CRYPTO_CIPHER_ARC4:
-		PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported Cipher alg %u",
-				cipher_xform->algo);
-		ret = -ENOTSUP;
-		goto error_out;
-	default:
-		PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n",
-				cipher_xform->algo);
-		ret = -EINVAL;
-		goto error_out;
-	}
-
-	if (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
-		session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
-	else
-		session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
-
-	if (qat_alg_aead_session_create_content_desc_cipher(session,
-						cipher_xform->key.data,
-						cipher_xform->key.length)) {
-		ret = -EINVAL;
-		goto error_out;
-	}
-
-	return 0;
-
-error_out:
-	if (session->bpi_ctx) {
-		bpi_cipher_ctx_free(session->bpi_ctx);
-		session->bpi_ctx = NULL;
-	}
-	return ret;
-}
-
-int
-qat_crypto_sym_configure_session(struct rte_cryptodev *dev,
-		struct rte_crypto_sym_xform *xform,
-		struct rte_cryptodev_sym_session *sess,
-		struct rte_mempool *mempool)
-{
-	void *sess_private_data;
-	int ret;
-
-	if (rte_mempool_get(mempool, &sess_private_data)) {
-		CDEV_LOG_ERR(
-			"Couldn't get object from session mempool");
-		return -ENOMEM;
-	}
-
-	ret = qat_crypto_set_session_parameters(dev, xform, sess_private_data);
-	if (ret != 0) {
-		PMD_DRV_LOG(ERR, "Crypto QAT PMD: failed to configure "
-				"session parameters");
-
-		/* Return session to mempool */
-		rte_mempool_put(mempool, sess_private_data);
-		return ret;
-	}
-
-	set_session_private_data(sess, dev->driver_id,
-		sess_private_data);
-
-	return 0;
-}
-
-int
-qat_crypto_set_session_parameters(struct rte_cryptodev *dev,
-		struct rte_crypto_sym_xform *xform, void *session_private)
-{
-	struct qat_session *session = session_private;
-	int ret;
-
-	int qat_cmd_id;
-	PMD_INIT_FUNC_TRACE();
-
-	/* Set context descriptor physical address */
-	session->cd_paddr = rte_mempool_virt2iova(session) +
-			offsetof(struct qat_session, cd);
-
-	session->min_qat_dev_gen = QAT_GEN1;
-
-	/* Get requested QAT command id */
-	qat_cmd_id = qat_get_cmd_id(xform);
-	if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) {
-		PMD_DRV_LOG(ERR, "Unsupported xform chain requested");
-		return -ENOTSUP;
-	}
-	session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
-	switch (session->qat_cmd) {
-	case ICP_QAT_FW_LA_CMD_CIPHER:
-		ret = qat_crypto_sym_configure_session_cipher(dev, xform, session);
-		if (ret < 0)
-			return ret;
-		break;
-	case ICP_QAT_FW_LA_CMD_AUTH:
-		ret = qat_crypto_sym_configure_session_auth(dev, xform, session);
-		if (ret < 0)
-			return ret;
-		break;
-	case ICP_QAT_FW_LA_CMD_CIPHER_HASH:
-		if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
-			ret = qat_crypto_sym_configure_session_aead(xform,
-					session);
-			if (ret < 0)
-				return ret;
-		} else {
-			ret = qat_crypto_sym_configure_session_cipher(dev,
-					xform, session);
-			if (ret < 0)
-				return ret;
-			ret = qat_crypto_sym_configure_session_auth(dev,
-					xform, session);
-			if (ret < 0)
-				return ret;
-		}
-		break;
-	case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
-		if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
-			ret = qat_crypto_sym_configure_session_aead(xform,
-					session);
-			if (ret < 0)
-				return ret;
-		} else {
-			ret = qat_crypto_sym_configure_session_auth(dev,
-					xform, session);
-			if (ret < 0)
-				return ret;
-			ret = qat_crypto_sym_configure_session_cipher(dev,
-					xform, session);
-			if (ret < 0)
-				return ret;
-		}
-		break;
-	case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
-	case ICP_QAT_FW_LA_CMD_TRNG_TEST:
-	case ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE:
-	case ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE:
-	case ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE:
-	case ICP_QAT_FW_LA_CMD_MGF1:
-	case ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP:
-	case ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP:
-	case ICP_QAT_FW_LA_CMD_DELIMITER:
-	PMD_DRV_LOG(ERR, "Unsupported Service %u",
-		session->qat_cmd);
-		return -ENOTSUP;
-	default:
-	PMD_DRV_LOG(ERR, "Unsupported Service %u",
-		session->qat_cmd);
-		return -ENOTSUP;
-	}
-
-	return 0;
-}
-
-int
-qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev,
-				struct rte_crypto_sym_xform *xform,
-				struct qat_session *session)
-{
-	struct rte_crypto_auth_xform *auth_xform = NULL;
-	struct qat_pmd_private *internals = dev->data->dev_private;
-	auth_xform = qat_get_auth_xform(xform);
-	uint8_t *key_data = auth_xform->key.data;
-	uint8_t key_length = auth_xform->key.length;
-
-	switch (auth_xform->algo) {
-	case RTE_CRYPTO_AUTH_SHA1_HMAC:
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
-		break;
-	case RTE_CRYPTO_AUTH_SHA224_HMAC:
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA224;
-		break;
-	case RTE_CRYPTO_AUTH_SHA256_HMAC:
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256;
-		break;
-	case RTE_CRYPTO_AUTH_SHA384_HMAC:
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA384;
-		break;
-	case RTE_CRYPTO_AUTH_SHA512_HMAC:
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512;
-		break;
-	case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
-		break;
-	case RTE_CRYPTO_AUTH_AES_GMAC:
-		if (qat_alg_validate_aes_key(auth_xform->key.length,
-				&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid AES key size");
-			return -EINVAL;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
-
-		break;
-	case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2;
-		break;
-	case RTE_CRYPTO_AUTH_MD5_HMAC:
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_MD5;
-		break;
-	case RTE_CRYPTO_AUTH_NULL:
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_NULL;
-		break;
-	case RTE_CRYPTO_AUTH_KASUMI_F9:
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_KASUMI_F9;
-		break;
-	case RTE_CRYPTO_AUTH_ZUC_EIA3:
-		if (!qat_is_auth_alg_supported(auth_xform->algo, internals)) {
-			PMD_DRV_LOG(ERR, "%s not supported on this device",
-				rte_crypto_auth_algorithm_strings
-				[auth_xform->algo]);
-			return -ENOTSUP;
-		}
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3;
-		break;
-	case RTE_CRYPTO_AUTH_SHA1:
-	case RTE_CRYPTO_AUTH_SHA256:
-	case RTE_CRYPTO_AUTH_SHA512:
-	case RTE_CRYPTO_AUTH_SHA224:
-	case RTE_CRYPTO_AUTH_SHA384:
-	case RTE_CRYPTO_AUTH_MD5:
-	case RTE_CRYPTO_AUTH_AES_CMAC:
-	case RTE_CRYPTO_AUTH_AES_CBC_MAC:
-		PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u",
-				auth_xform->algo);
-		return -ENOTSUP;
-	default:
-		PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified",
-				auth_xform->algo);
-		return -EINVAL;
-	}
-
-	session->auth_iv.offset = auth_xform->iv.offset;
-	session->auth_iv.length = auth_xform->iv.length;
-
-	if (auth_xform->algo == RTE_CRYPTO_AUTH_AES_GMAC) {
-		if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) {
-			session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
-			session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
-			/*
-			 * It needs to create cipher desc content first,
-			 * then authentication
-			 */
-			if (qat_alg_aead_session_create_content_desc_cipher(session,
-						auth_xform->key.data,
-						auth_xform->key.length))
-				return -EINVAL;
-
-			if (qat_alg_aead_session_create_content_desc_auth(session,
-						key_data,
-						key_length,
-						0,
-						auth_xform->digest_length,
-						auth_xform->op))
-				return -EINVAL;
-		} else {
-			session->qat_cmd = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
-			session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
-			/*
-			 * It needs to create authentication desc content first,
-			 * then cipher
-			 */
-			if (qat_alg_aead_session_create_content_desc_auth(session,
-					key_data,
-					key_length,
-					0,
-					auth_xform->digest_length,
-					auth_xform->op))
-				return -EINVAL;
-
-			if (qat_alg_aead_session_create_content_desc_cipher(session,
-						auth_xform->key.data,
-						auth_xform->key.length))
-				return -EINVAL;
-		}
-		/* Restore to authentication only only */
-		session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH;
-	} else {
-		if (qat_alg_aead_session_create_content_desc_auth(session,
-				key_data,
-				key_length,
-				0,
-				auth_xform->digest_length,
-				auth_xform->op))
-			return -EINVAL;
-	}
-
-	session->digest_length = auth_xform->digest_length;
-	return 0;
-}
-
-int
-qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform,
-				struct qat_session *session)
-{
-	struct rte_crypto_aead_xform *aead_xform = &xform->aead;
-	enum rte_crypto_auth_operation crypto_operation;
-
-	/*
-	 * Store AEAD IV parameters as cipher IV,
-	 * to avoid unnecessary memory usage
-	 */
-	session->cipher_iv.offset = xform->aead.iv.offset;
-	session->cipher_iv.length = xform->aead.iv.length;
-
-	switch (aead_xform->algo) {
-	case RTE_CRYPTO_AEAD_AES_GCM:
-		if (qat_alg_validate_aes_key(aead_xform->key.length,
-				&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid AES key size");
-			return -EINVAL;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
-		break;
-	case RTE_CRYPTO_AEAD_AES_CCM:
-		if (qat_alg_validate_aes_key(aead_xform->key.length,
-				&session->qat_cipher_alg) != 0) {
-			PMD_DRV_LOG(ERR, "Invalid AES key size");
-			return -EINVAL;
-		}
-		session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
-		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC;
-		break;
-	default:
-		PMD_DRV_LOG(ERR, "Crypto: Undefined AEAD specified %u\n",
-				aead_xform->algo);
-		return -EINVAL;
-	}
-
-	if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT &&
-			aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) ||
-			(aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT &&
-			aead_xform->algo == RTE_CRYPTO_AEAD_AES_CCM)) {
-		session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
-		/*
-		 * It needs to create cipher desc content first,
-		 * then authentication
-		 */
-
-		crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
-			RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY;
-
-		if (qat_alg_aead_session_create_content_desc_cipher(session,
-					aead_xform->key.data,
-					aead_xform->key.length))
-			return -EINVAL;
-
-		if (qat_alg_aead_session_create_content_desc_auth(session,
-					aead_xform->key.data,
-					aead_xform->key.length,
-					aead_xform->aad_length,
-					aead_xform->digest_length,
-					crypto_operation))
-			return -EINVAL;
-	} else {
-		session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
-		/*
-		 * It needs to create authentication desc content first,
-		 * then cipher
-		 */
-
-		crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
-			RTE_CRYPTO_AUTH_OP_VERIFY : RTE_CRYPTO_AUTH_OP_GENERATE;
-
-		if (qat_alg_aead_session_create_content_desc_auth(session,
-					aead_xform->key.data,
-					aead_xform->key.length,
-					aead_xform->aad_length,
-					aead_xform->digest_length,
-					crypto_operation))
-			return -EINVAL;
-
-		if (qat_alg_aead_session_create_content_desc_cipher(session,
-					aead_xform->key.data,
-					aead_xform->key.length))
-			return -EINVAL;
-	}
-
-	session->digest_length = aead_xform->digest_length;
-	return 0;
-}
-
-unsigned qat_crypto_sym_get_session_private_size(
-		struct rte_cryptodev *dev __rte_unused)
-{
-	return RTE_ALIGN_CEIL(sizeof(struct qat_session), 8);
-}
-
-static inline uint32_t
-qat_bpicipher_preprocess(struct qat_session *ctx,
-				struct rte_crypto_op *op)
-{
-	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
-	struct rte_crypto_sym_op *sym_op = op->sym;
-	uint8_t last_block_len = block_len > 0 ?
-			sym_op->cipher.data.length % block_len : 0;
-
-	if (last_block_len &&
-			ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
-
-		/* Decrypt last block */
-		uint8_t *last_block, *dst, *iv;
-		uint32_t last_block_offset = sym_op->cipher.data.offset +
-				sym_op->cipher.data.length - last_block_len;
-		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
-				uint8_t *, last_block_offset);
-
-		if (unlikely(sym_op->m_dst != NULL))
-			/* out-of-place operation (OOP) */
-			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
-						uint8_t *, last_block_offset);
-		else
-			dst = last_block;
-
-		if (last_block_len < sym_op->cipher.data.length)
-			/* use previous block ciphertext as IV */
-			iv = last_block - block_len;
-		else
-			/* runt block, i.e. less than one full block */
-			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
-					ctx->cipher_iv.offset);
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
-		rte_hexdump(stdout, "BPI: src before pre-process:", last_block,
-			last_block_len);
-		if (sym_op->m_dst != NULL)
-			rte_hexdump(stdout, "BPI: dst before pre-process:", dst,
-				last_block_len);
-#endif
-		bpi_cipher_decrypt(last_block, dst, iv, block_len,
-				last_block_len, ctx->bpi_ctx);
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
-		rte_hexdump(stdout, "BPI: src after pre-process:", last_block,
-			last_block_len);
-		if (sym_op->m_dst != NULL)
-			rte_hexdump(stdout, "BPI: dst after pre-process:", dst,
-				last_block_len);
-#endif
-	}
-
-	return sym_op->cipher.data.length - last_block_len;
-}
-
-static inline uint32_t
-qat_bpicipher_postprocess(struct qat_session *ctx,
-				struct rte_crypto_op *op)
-{
-	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
-	struct rte_crypto_sym_op *sym_op = op->sym;
-	uint8_t last_block_len = block_len > 0 ?
-			sym_op->cipher.data.length % block_len : 0;
-
-	if (last_block_len > 0 &&
-			ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
-
-		/* Encrypt last block */
-		uint8_t *last_block, *dst, *iv;
-		uint32_t last_block_offset;
-
-		last_block_offset = sym_op->cipher.data.offset +
-				sym_op->cipher.data.length - last_block_len;
-		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
-				uint8_t *, last_block_offset);
-
-		if (unlikely(sym_op->m_dst != NULL))
-			/* out-of-place operation (OOP) */
-			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
-						uint8_t *, last_block_offset);
-		else
-			dst = last_block;
-
-		if (last_block_len < sym_op->cipher.data.length)
-			/* use previous block ciphertext as IV */
-			iv = dst - block_len;
-		else
-			/* runt block, i.e. less than one full block */
-			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
-					ctx->cipher_iv.offset);
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
-		rte_hexdump(stdout, "BPI: src before post-process:", last_block,
-			last_block_len);
-		if (sym_op->m_dst != NULL)
-			rte_hexdump(stdout, "BPI: dst before post-process:",
-					dst, last_block_len);
-#endif
-		bpi_cipher_encrypt(last_block, dst, iv, block_len,
-				last_block_len, ctx->bpi_ctx);
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
-		rte_hexdump(stdout, "BPI: src after post-process:", last_block,
-			last_block_len);
-		if (sym_op->m_dst != NULL)
-			rte_hexdump(stdout, "BPI: dst after post-process:", dst,
-				last_block_len);
-#endif
-	}
-	return sym_op->cipher.data.length - last_block_len;
-}
-
-static inline void
-txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
-	WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
-			q->hw_queue_number, q->tail);
-	q->nb_pending_requests = 0;
-	q->csr_tail = q->tail;
-}
-
-uint16_t
-qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
-		uint16_t nb_ops)
-{
-	register struct qat_queue *queue;
-	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
-	register uint32_t nb_ops_sent = 0;
-	register struct rte_crypto_op **cur_op = ops;
-	register int ret;
-	uint16_t nb_ops_possible = nb_ops;
-	register uint8_t *base_addr;
-	register uint32_t tail;
-	int overflow;
-
-	if (unlikely(nb_ops == 0))
-		return 0;
-
-	/* read params used a lot in main loop into registers */
-	queue = &(tmp_qp->tx_q);
-	base_addr = (uint8_t *)queue->base_addr;
-	tail = queue->tail;
-
-	/* Find how many can actually fit on the ring */
-	tmp_qp->inflights16 += nb_ops;
-	overflow = tmp_qp->inflights16 - queue->max_inflights;
-	if (overflow > 0) {
-		tmp_qp->inflights16 -= overflow;
-		nb_ops_possible = nb_ops - overflow;
-		if (nb_ops_possible == 0)
-			return 0;
-	}
-
-	while (nb_ops_sent != nb_ops_possible) {
-		ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail,
-			tmp_qp->op_cookies[tail / queue->msg_size], tmp_qp);
-		if (ret != 0) {
-			tmp_qp->stats.enqueue_err_count++;
-			/*
-			 * This message cannot be enqueued,
-			 * decrease number of ops that wasn't sent
-			 */
-			tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent;
-			if (nb_ops_sent == 0)
-				return 0;
-			goto kick_tail;
-		}
-
-		tail = adf_modulo(tail + queue->msg_size, queue->modulo);
-		nb_ops_sent++;
-		cur_op++;
-	}
-kick_tail:
-	queue->tail = tail;
-	tmp_qp->stats.enqueued_count += nb_ops_sent;
-	queue->nb_pending_requests += nb_ops_sent;
-	if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH ||
-			queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) {
-		txq_write_tail(tmp_qp, queue);
-	}
-	return nb_ops_sent;
-}
-
-static inline
-void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
-{
-	uint32_t old_head, new_head;
-	uint32_t max_head;
-
-	old_head = q->csr_head;
-	new_head = q->head;
-	max_head = qp->nb_descriptors * q->msg_size;
-
-	/* write out free descriptors */
-	void *cur_desc = (uint8_t *)q->base_addr + old_head;
-
-	if (new_head < old_head) {
-		memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
-		memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
-	} else {
-		memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
-	}
-	q->nb_processed_responses = 0;
-	q->csr_head = new_head;
-
-	/* write current head to CSR */
-	WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
-			    q->hw_queue_number, new_head);
-}
-
-uint16_t
-qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
-		uint16_t nb_ops)
-{
-	struct qat_queue *rx_queue, *tx_queue;
-	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
-	uint32_t msg_counter = 0;
-	struct rte_crypto_op *rx_op;
-	struct icp_qat_fw_comn_resp *resp_msg;
-	uint32_t head;
-
-	rx_queue = &(tmp_qp->rx_q);
-	tx_queue = &(tmp_qp->tx_q);
-	head = rx_queue->head;
-	resp_msg = (struct icp_qat_fw_comn_resp *)
-			((uint8_t *)rx_queue->base_addr + head);
-
-	while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
-			msg_counter != nb_ops) {
-		rx_op = (struct rte_crypto_op *)(uintptr_t)
-				(resp_msg->opaque_data);
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
-		rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
-			sizeof(struct icp_qat_fw_comn_resp));
-#endif
-		if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
-				ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
-					resp_msg->comn_hdr.comn_status)) {
-			rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
-		} else {
-			struct qat_session *sess = (struct qat_session *)
-					get_session_private_data(
-					rx_op->sym->session,
-					cryptodev_qat_driver_id);
-
-			if (sess->bpi_ctx)
-				qat_bpicipher_postprocess(sess, rx_op);
-			rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
-		}
-
-		head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo);
-		resp_msg = (struct icp_qat_fw_comn_resp *)
-				((uint8_t *)rx_queue->base_addr + head);
-		*ops = rx_op;
-		ops++;
-		msg_counter++;
-	}
-	if (msg_counter > 0) {
-		rx_queue->head = head;
-		tmp_qp->stats.dequeued_count += msg_counter;
-		rx_queue->nb_processed_responses += msg_counter;
-		tmp_qp->inflights16 -= msg_counter;
-
-		if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
-			rxq_free_desc(tmp_qp, rx_queue);
-	}
-	/* also check if tail needs to be advanced */
-	if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH &&
-			tx_queue->tail != tx_queue->csr_tail) {
-		txq_write_tail(tmp_qp, tx_queue);
-	}
-	return msg_counter;
-}
-
-static inline int
-qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start,
-		struct qat_alg_buf_list *list, uint32_t data_len)
-{
-	int nr = 1;
-
-	uint32_t buf_len = rte_pktmbuf_iova(buf) -
-			buff_start + rte_pktmbuf_data_len(buf);
-
-	list->bufers[0].addr = buff_start;
-	list->bufers[0].resrvd = 0;
-	list->bufers[0].len = buf_len;
-
-	if (data_len <= buf_len) {
-		list->num_bufs = nr;
-		list->bufers[0].len = data_len;
-		return 0;
-	}
-
-	buf = buf->next;
-	while (buf) {
-		if (unlikely(nr == QAT_SGL_MAX_NUMBER)) {
-			PMD_DRV_LOG(ERR, "QAT PMD exceeded size of QAT SGL"
-					" entry(%u)",
-					QAT_SGL_MAX_NUMBER);
-			return -EINVAL;
-		}
-
-		list->bufers[nr].len = rte_pktmbuf_data_len(buf);
-		list->bufers[nr].resrvd = 0;
-		list->bufers[nr].addr = rte_pktmbuf_iova(buf);
-
-		buf_len += list->bufers[nr].len;
-		buf = buf->next;
-
-		if (buf_len > data_len) {
-			list->bufers[nr].len -=
-				buf_len - data_len;
-			buf = NULL;
-		}
-		++nr;
-	}
-	list->num_bufs = nr;
-
-	return 0;
-}
-
-static inline void
-set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
-		struct icp_qat_fw_la_cipher_req_params *cipher_param,
-		struct rte_crypto_op *op,
-		struct icp_qat_fw_la_bulk_req *qat_req)
-{
-	/* copy IV into request if it fits */
-	if (iv_length <= sizeof(cipher_param->u.cipher_IV_array)) {
-		rte_memcpy(cipher_param->u.cipher_IV_array,
-				rte_crypto_op_ctod_offset(op, uint8_t *,
-					iv_offset),
-				iv_length);
-	} else {
-		ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
-				qat_req->comn_hdr.serv_specif_flags,
-				ICP_QAT_FW_CIPH_IV_64BIT_PTR);
-		cipher_param->u.s.cipher_IV_ptr =
-				rte_crypto_op_ctophys_offset(op,
-					iv_offset);
-	}
-}
-
-/** Set IV for CCM is special case, 0th byte is set to q-1
- *  where q is padding of nonce in 16 byte block
- */
-static inline void
-set_cipher_iv_ccm(uint16_t iv_length, uint16_t iv_offset,
-		struct icp_qat_fw_la_cipher_req_params *cipher_param,
-		struct rte_crypto_op *op, uint8_t q, uint8_t aad_len_field_sz)
-{
-	rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array) +
-			ICP_QAT_HW_CCM_NONCE_OFFSET,
-			rte_crypto_op_ctod_offset(op, uint8_t *,
-				iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET,
-			iv_length);
-	*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
-			q - ICP_QAT_HW_CCM_NONCE_OFFSET;
-
-	if (aad_len_field_sz)
-		rte_memcpy(&op->sym->aead.aad.data[ICP_QAT_HW_CCM_NONCE_OFFSET],
-			rte_crypto_op_ctod_offset(op, uint8_t *,
-				iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET,
-			iv_length);
-}
-
-static inline int
-qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg,
-		struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp)
-{
-	int ret = 0;
-	struct qat_session *ctx;
-	struct icp_qat_fw_la_cipher_req_params *cipher_param;
-	struct icp_qat_fw_la_auth_req_params *auth_param;
-	register struct icp_qat_fw_la_bulk_req *qat_req;
-	uint8_t do_auth = 0, do_cipher = 0, do_aead = 0;
-	uint32_t cipher_len = 0, cipher_ofs = 0;
-	uint32_t auth_len = 0, auth_ofs = 0;
-	uint32_t min_ofs = 0;
-	uint64_t src_buf_start = 0, dst_buf_start = 0;
-	uint8_t do_sgl = 0;
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
-	if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) {
-		PMD_DRV_LOG(ERR, "QAT PMD only supports symmetric crypto "
-				"operation requests, op (%p) is not a "
-				"symmetric operation.", op);
-		return -EINVAL;
-	}
-#endif
-	if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
-		PMD_DRV_LOG(ERR, "QAT PMD only supports session oriented"
-				" requests, op (%p) is sessionless.", op);
-		return -EINVAL;
-	}
-
-	ctx = (struct qat_session *)get_session_private_data(
-			op->sym->session, cryptodev_qat_driver_id);
-
-	if (unlikely(ctx == NULL)) {
-		PMD_DRV_LOG(ERR, "Session was not created for this device");
-		return -EINVAL;
-	}
-
-	if (unlikely(ctx->min_qat_dev_gen > qp->qat_dev_gen)) {
-		PMD_DRV_LOG(ERR, "Session alg not supported on this device gen");
-		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
-		return -EINVAL;
-	}
-
-
-
-	qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
-	rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
-	qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;
-	cipher_param = (void *)&qat_req->serv_specif_rqpars;
-	auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
-
-	if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
-			ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
-		/* AES-GCM or AES-CCM */
-		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
-				ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||
-				(ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128
-				&& ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE
-				&& ctx->qat_hash_alg ==
-						ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
-			do_aead = 1;
-		} else {
-			do_auth = 1;
-			do_cipher = 1;
-		}
-	} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) {
-		do_auth = 1;
-		do_cipher = 0;
-	} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
-		do_auth = 0;
-		do_cipher = 1;
-	}
-
-	if (do_cipher) {
-
-		if (ctx->qat_cipher_alg ==
-					 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
-			ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
-			ctx->qat_cipher_alg ==
-				ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
-
-			if (unlikely(
-				(cipher_param->cipher_length % BYTE_LENGTH != 0)
-				 || (cipher_param->cipher_offset
-							% BYTE_LENGTH != 0))) {
-				PMD_DRV_LOG(ERR,
-		  "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
-				op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
-				return -EINVAL;
-			}
-			cipher_len = op->sym->cipher.data.length >> 3;
-			cipher_ofs = op->sym->cipher.data.offset >> 3;
-
-		} else if (ctx->bpi_ctx) {
-			/* DOCSIS - only send complete blocks to device
-			 * Process any partial block using CFB mode.
-			 * Even if 0 complete blocks, still send this to device
-			 * to get into rx queue for post-process and dequeuing
-			 */
-			cipher_len = qat_bpicipher_preprocess(ctx, op);
-			cipher_ofs = op->sym->cipher.data.offset;
-		} else {
-			cipher_len = op->sym->cipher.data.length;
-			cipher_ofs = op->sym->cipher.data.offset;
-		}
-
-		set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset,
-				cipher_param, op, qat_req);
-		min_ofs = cipher_ofs;
-	}
-
-	if (do_auth) {
-
-		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
-			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
-			ctx->qat_hash_alg ==
-				ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
-			if (unlikely((auth_param->auth_off % BYTE_LENGTH != 0)
-				|| (auth_param->auth_len % BYTE_LENGTH != 0))) {
-				PMD_DRV_LOG(ERR,
-		"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
-				op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
-				return -EINVAL;
-			}
-			auth_ofs = op->sym->auth.data.offset >> 3;
-			auth_len = op->sym->auth.data.length >> 3;
-
-			auth_param->u1.aad_adr =
-					rte_crypto_op_ctophys_offset(op,
-							ctx->auth_iv.offset);
-
-		} else if (ctx->qat_hash_alg ==
-					ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
-				ctx->qat_hash_alg ==
-					ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
-			/* AES-GMAC */
-			set_cipher_iv(ctx->auth_iv.length,
-				ctx->auth_iv.offset,
-				cipher_param, op, qat_req);
-			auth_ofs = op->sym->auth.data.offset;
-			auth_len = op->sym->auth.data.length;
-
-			auth_param->u1.aad_adr = 0;
-			auth_param->u2.aad_sz = 0;
-
-			/*
-			 * If len(iv)==12B fw computes J0
-			 */
-			if (ctx->auth_iv.length == 12) {
-				ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
-					qat_req->comn_hdr.serv_specif_flags,
-					ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
-
-			}
-		} else {
-			auth_ofs = op->sym->auth.data.offset;
-			auth_len = op->sym->auth.data.length;
-
-		}
-		min_ofs = auth_ofs;
-
-		if (likely(ctx->qat_hash_alg != ICP_QAT_HW_AUTH_ALGO_NULL))
-			auth_param->auth_res_addr =
-					op->sym->auth.digest.phys_addr;
-
-	}
-
-	if (do_aead) {
-		/*
-		 * This address may used for setting AAD physical pointer
-		 * into IV offset from op
-		 */
-		rte_iova_t aad_phys_addr_aead = op->sym->aead.aad.phys_addr;
-		if (ctx->qat_hash_alg ==
-				ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
-				ctx->qat_hash_alg ==
-					ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
-			/*
-			 * If len(iv)==12B fw computes J0
-			 */
-			if (ctx->cipher_iv.length == 12) {
-				ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
-					qat_req->comn_hdr.serv_specif_flags,
-					ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
-			}
-
-			set_cipher_iv(ctx->cipher_iv.length,
-					ctx->cipher_iv.offset,
-					cipher_param, op, qat_req);
-
-		} else if (ctx->qat_hash_alg ==
-				ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC) {
-
-			/* In case of AES-CCM this may point to user selected memory
-			 * or iv offset in cypto_op
-			 */
-			uint8_t *aad_data = op->sym->aead.aad.data;
-			/* This is true AAD length, it not includes 18 bytes of
-			 * preceding data
-			 */
-			uint8_t aad_ccm_real_len = 0;
-
-			uint8_t aad_len_field_sz = 0;
-			uint32_t msg_len_be =
-					rte_bswap32(op->sym->aead.data.length);
-
-			if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
-				aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
-				aad_ccm_real_len = ctx->aad_len -
-					ICP_QAT_HW_CCM_AAD_B0_LEN -
-					ICP_QAT_HW_CCM_AAD_LEN_INFO;
-			} else {
-				/*
-				 * aad_len not greater than 18, so no actual aad data,
-				 * then use IV after op for B0 block
-				 */
-				aad_data = rte_crypto_op_ctod_offset(op, uint8_t *,
-						ctx->cipher_iv.offset);
-				aad_phys_addr_aead =
-						rte_crypto_op_ctophys_offset(op,
-								ctx->cipher_iv.offset);
-			}
-
-			uint8_t q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
-
-			aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(aad_len_field_sz,
-							ctx->digest_length, q);
-
-			if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
-				memcpy(aad_data	+ ctx->cipher_iv.length +
-					ICP_QAT_HW_CCM_NONCE_OFFSET
-					+ (q - ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
-					(uint8_t *)&msg_len_be,
-					ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
-			} else {
-				memcpy(aad_data	+ ctx->cipher_iv.length +
-					ICP_QAT_HW_CCM_NONCE_OFFSET,
-					(uint8_t *)&msg_len_be
-					+ (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
-					- q), q);
-			}
-
-			if (aad_len_field_sz > 0) {
-				*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN]
-						= rte_bswap16(aad_ccm_real_len);
-
-				if ((aad_ccm_real_len + aad_len_field_sz)
-						% ICP_QAT_HW_CCM_AAD_B0_LEN) {
-					uint8_t pad_len = 0;
-					uint8_t pad_idx = 0;
-
-					pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
-						((aad_ccm_real_len + aad_len_field_sz) %
-							ICP_QAT_HW_CCM_AAD_B0_LEN);
-					pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
-						aad_ccm_real_len + aad_len_field_sz;
-					memset(&aad_data[pad_idx],
-							0, pad_len);
-				}
-
-			}
-
-			set_cipher_iv_ccm(ctx->cipher_iv.length,
-					ctx->cipher_iv.offset,
-					cipher_param, op, q,
-					aad_len_field_sz);
-
-		}
-
-		cipher_len = op->sym->aead.data.length;
-		cipher_ofs = op->sym->aead.data.offset;
-		auth_len = op->sym->aead.data.length;
-		auth_ofs = op->sym->aead.data.offset;
-
-		auth_param->u1.aad_adr = aad_phys_addr_aead;
-		auth_param->auth_res_addr = op->sym->aead.digest.phys_addr;
-		min_ofs = op->sym->aead.data.offset;
-	}
-
-	if (op->sym->m_src->next || (op->sym->m_dst && op->sym->m_dst->next))
-		do_sgl = 1;
-
-	/* adjust for chain case */
-	if (do_cipher && do_auth)
-		min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
-
-	if (unlikely(min_ofs >= rte_pktmbuf_data_len(op->sym->m_src) && do_sgl))
-		min_ofs = 0;
-
-	if (unlikely(op->sym->m_dst != NULL)) {
-		/* Out-of-place operation (OOP)
-		 * Don't align DMA start. DMA the minimum data-set
-		 * so as not to overwrite data in dest buffer
-		 */
-		src_buf_start =
-			rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs);
-		dst_buf_start =
-			rte_pktmbuf_iova_offset(op->sym->m_dst, min_ofs);
-
-	} else {
-		/* In-place operation
-		 * Start DMA at nearest aligned address below min_ofs
-		 */
-		src_buf_start =
-			rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs)
-						& QAT_64_BTYE_ALIGN_MASK;
-
-		if (unlikely((rte_pktmbuf_iova(op->sym->m_src) -
-					rte_pktmbuf_headroom(op->sym->m_src))
-							> src_buf_start)) {
-			/* alignment has pushed addr ahead of start of mbuf
-			 * so revert and take the performance hit
-			 */
-			src_buf_start =
-				rte_pktmbuf_iova_offset(op->sym->m_src,
-								min_ofs);
-		}
-		dst_buf_start = src_buf_start;
-	}
-
-	if (do_cipher || do_aead) {
-		cipher_param->cipher_offset =
-				(uint32_t)rte_pktmbuf_iova_offset(
-				op->sym->m_src, cipher_ofs) - src_buf_start;
-		cipher_param->cipher_length = cipher_len;
-	} else {
-		cipher_param->cipher_offset = 0;
-		cipher_param->cipher_length = 0;
-	}
-
-	if (do_auth || do_aead) {
-		auth_param->auth_off = (uint32_t)rte_pktmbuf_iova_offset(
-				op->sym->m_src, auth_ofs) - src_buf_start;
-		auth_param->auth_len = auth_len;
-	} else {
-		auth_param->auth_off = 0;
-		auth_param->auth_len = 0;
-	}
-
-	qat_req->comn_mid.dst_length =
-		qat_req->comn_mid.src_length =
-		(cipher_param->cipher_offset + cipher_param->cipher_length)
-		> (auth_param->auth_off + auth_param->auth_len) ?
-		(cipher_param->cipher_offset + cipher_param->cipher_length)
-		: (auth_param->auth_off + auth_param->auth_len);
-
-	if (do_sgl) {
-
-		ICP_QAT_FW_COMN_PTR_TYPE_SET(qat_req->comn_hdr.comn_req_flags,
-				QAT_COMN_PTR_TYPE_SGL);
-		ret = qat_sgl_fill_array(op->sym->m_src, src_buf_start,
-				&qat_op_cookie->qat_sgl_list_src,
-				qat_req->comn_mid.src_length);
-		if (ret) {
-			PMD_DRV_LOG(ERR, "QAT PMD Cannot fill sgl array");
-			return ret;
-		}
-
-		if (likely(op->sym->m_dst == NULL))
-			qat_req->comn_mid.dest_data_addr =
-				qat_req->comn_mid.src_data_addr =
-				qat_op_cookie->qat_sgl_src_phys_addr;
-		else {
-			ret = qat_sgl_fill_array(op->sym->m_dst,
-					dst_buf_start,
-					&qat_op_cookie->qat_sgl_list_dst,
-						qat_req->comn_mid.dst_length);
-
-			if (ret) {
-				PMD_DRV_LOG(ERR, "QAT PMD Cannot "
-						"fill sgl array");
-				return ret;
-			}
-
-			qat_req->comn_mid.src_data_addr =
-				qat_op_cookie->qat_sgl_src_phys_addr;
-			qat_req->comn_mid.dest_data_addr =
-					qat_op_cookie->qat_sgl_dst_phys_addr;
-		}
-	} else {
-		qat_req->comn_mid.src_data_addr = src_buf_start;
-		qat_req->comn_mid.dest_data_addr = dst_buf_start;
-	}
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
-	rte_hexdump(stdout, "qat_req:", qat_req,
-			sizeof(struct icp_qat_fw_la_bulk_req));
-	rte_hexdump(stdout, "src_data:",
-			rte_pktmbuf_mtod(op->sym->m_src, uint8_t*),
-			rte_pktmbuf_data_len(op->sym->m_src));
-	if (do_cipher) {
-		uint8_t *cipher_iv_ptr = rte_crypto_op_ctod_offset(op,
-						uint8_t *,
-						ctx->cipher_iv.offset);
-		rte_hexdump(stdout, "cipher iv:", cipher_iv_ptr,
-				ctx->cipher_iv.length);
-	}
-
-	if (do_auth) {
-		if (ctx->auth_iv.length) {
-			uint8_t *auth_iv_ptr = rte_crypto_op_ctod_offset(op,
-							uint8_t *,
-							ctx->auth_iv.offset);
-			rte_hexdump(stdout, "auth iv:", auth_iv_ptr,
-						ctx->auth_iv.length);
-		}
-		rte_hexdump(stdout, "digest:", op->sym->auth.digest.data,
-				ctx->digest_length);
-	}
-
-	if (do_aead) {
-		rte_hexdump(stdout, "digest:", op->sym->aead.digest.data,
-				ctx->digest_length);
-		rte_hexdump(stdout, "aad:", op->sym->aead.aad.data,
-				ctx->aad_len);
-	}
-#endif
-	return 0;
-}
-
-static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
-{
-	uint32_t div = data >> shift;
-	uint32_t mult = div << shift;
-
-	return data - mult;
-}
-
-int qat_dev_config(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused struct rte_cryptodev_config *config)
-{
-	PMD_INIT_FUNC_TRACE();
-	return 0;
-}
-
-int qat_dev_start(__rte_unused struct rte_cryptodev *dev)
-{
-	PMD_INIT_FUNC_TRACE();
-	return 0;
-}
-
-void qat_dev_stop(__rte_unused struct rte_cryptodev *dev)
-{
-	PMD_INIT_FUNC_TRACE();
-}
-
-int qat_dev_close(struct rte_cryptodev *dev)
-{
-	int i, ret;
-
-	PMD_INIT_FUNC_TRACE();
-
-	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
-		ret = qat_crypto_sym_qp_release(dev, i);
-		if (ret < 0)
-			return ret;
-	}
-
-	return 0;
-}
-
-void qat_dev_info_get(struct rte_cryptodev *dev,
-			struct rte_cryptodev_info *info)
-{
-	struct qat_pmd_private *internals = dev->data->dev_private;
-
-	PMD_INIT_FUNC_TRACE();
-	if (info != NULL) {
-		info->max_nb_queue_pairs =
-				ADF_NUM_SYM_QPS_PER_BUNDLE *
-				ADF_NUM_BUNDLES_PER_DEV;
-		info->feature_flags = dev->feature_flags;
-		info->capabilities = internals->qat_dev_capabilities;
-		info->sym.max_nb_sessions = internals->max_nb_sessions;
-		info->driver_id = cryptodev_qat_driver_id;
-		info->pci_dev = RTE_DEV_TO_PCI(dev->device);
-	}
-}
-
-void qat_crypto_sym_stats_get(struct rte_cryptodev *dev,
-		struct rte_cryptodev_stats *stats)
-{
-	int i;
-	struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
-
-	PMD_INIT_FUNC_TRACE();
-	if (stats == NULL) {
-		PMD_DRV_LOG(ERR, "invalid stats ptr NULL");
-		return;
-	}
-	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
-		if (qp[i] == NULL) {
-			PMD_DRV_LOG(DEBUG, "Uninitialised queue pair");
-			continue;
-		}
-
-		stats->enqueued_count += qp[i]->stats.enqueued_count;
-		stats->dequeued_count += qp[i]->stats.dequeued_count;
-		stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
-		stats->dequeue_err_count += qp[i]->stats.dequeue_err_count;
-	}
-}
-
-void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev)
-{
-	int i;
-	struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
-
-	PMD_INIT_FUNC_TRACE();
-	for (i = 0; i < dev->data->nb_queue_pairs; i++)
-		memset(&(qp[i]->stats), 0, sizeof(qp[i]->stats));
-	PMD_DRV_LOG(DEBUG, "QAT crypto: stats cleared");
-}
diff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h
deleted file mode 100644
index 281a142b..00000000
--- a/drivers/crypto/qat/qat_crypto.h
+++ /dev/null
@@ -1,150 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2018 Intel Corporation
- */
-
-#ifndef _QAT_CRYPTO_H_
-#define _QAT_CRYPTO_H_
-
-#include <rte_cryptodev_pmd.h>
-#include <rte_memzone.h>
-
-#include "qat_crypto_capabilities.h"
-
-#define CRYPTODEV_NAME_QAT_SYM_PMD	crypto_qat
-/**< Intel QAT Symmetric Crypto PMD device name */
-
-/*
- * This macro rounds up a number to a be a multiple of
- * the alignment when the alignment is a power of 2
- */
-#define ALIGN_POW2_ROUNDUP(num, align) \
-	(((num) + (align) - 1) & ~((align) - 1))
-#define QAT_64_BTYE_ALIGN_MASK (~0x3f)
-
-#define QAT_CSR_HEAD_WRITE_THRESH 32U
-/* number of requests to accumulate before writing head CSR */
-#define QAT_CSR_TAIL_WRITE_THRESH 32U
-/* number of requests to accumulate before writing tail CSR */
-#define QAT_CSR_TAIL_FORCE_WRITE_THRESH 256U
-/* number of inflights below which no tail write coalescing should occur */
-
-struct qat_session;
-
-enum qat_device_gen {
-	QAT_GEN1 = 1,
-	QAT_GEN2,
-};
-
-/**
- * Structure associated with each queue.
- */
-struct qat_queue {
-	char		memz_name[RTE_MEMZONE_NAMESIZE];
-	void		*base_addr;		/* Base address */
-	rte_iova_t	base_phys_addr;		/* Queue physical address */
-	uint32_t	head;			/* Shadow copy of the head */
-	uint32_t	tail;			/* Shadow copy of the tail */
-	uint32_t	modulo;
-	uint32_t	msg_size;
-	uint16_t	max_inflights;
-	uint32_t	queue_size;
-	uint8_t		hw_bundle_number;
-	uint8_t		hw_queue_number;
-	/* HW queue aka ring offset on bundle */
-	uint32_t	csr_head;		/* last written head value */
-	uint32_t	csr_tail;		/* last written tail value */
-	uint16_t	nb_processed_responses;
-	/* number of responses processed since last CSR head write */
-	uint16_t	nb_pending_requests;
-	/* number of requests pending since last CSR tail write */
-};
-
-struct qat_qp {
-	void			*mmap_bar_addr;
-	uint16_t		inflights16;
-	struct	qat_queue	tx_q;
-	struct	qat_queue	rx_q;
-	struct	rte_cryptodev_stats stats;
-	struct rte_mempool *op_cookie_pool;
-	void **op_cookies;
-	uint32_t nb_descriptors;
-	enum qat_device_gen qat_dev_gen;
-} __rte_cache_aligned;
-
-/** private data structure for each QAT device */
-struct qat_pmd_private {
-	unsigned max_nb_queue_pairs;
-	/**< Max number of queue pairs supported by device */
-	unsigned max_nb_sessions;
-	/**< Max number of sessions supported by device */
-	enum qat_device_gen qat_dev_gen;
-	/**< QAT device generation */
-	const struct rte_cryptodev_capabilities *qat_dev_capabilities;
-};
-
-extern uint8_t cryptodev_qat_driver_id;
-
-int qat_dev_config(struct rte_cryptodev *dev,
-		struct rte_cryptodev_config *config);
-int qat_dev_start(struct rte_cryptodev *dev);
-void qat_dev_stop(struct rte_cryptodev *dev);
-int qat_dev_close(struct rte_cryptodev *dev);
-void qat_dev_info_get(struct rte_cryptodev *dev,
-	struct rte_cryptodev_info *info);
-
-void qat_crypto_sym_stats_get(struct rte_cryptodev *dev,
-	struct rte_cryptodev_stats *stats);
-void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev);
-
-int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
-	const struct rte_cryptodev_qp_conf *rx_conf, int socket_id,
-	struct rte_mempool *session_pool);
-int qat_crypto_sym_qp_release(struct rte_cryptodev *dev,
-	uint16_t queue_pair_id);
-
-int
-qat_pmd_session_mempool_create(struct rte_cryptodev *dev,
-	unsigned nb_objs, unsigned obj_cache_size, int socket_id);
-
-extern unsigned
-qat_crypto_sym_get_session_private_size(struct rte_cryptodev *dev);
-
-extern int
-qat_crypto_sym_configure_session(struct rte_cryptodev *dev,
-		struct rte_crypto_sym_xform *xform,
-		struct rte_cryptodev_sym_session *sess,
-		struct rte_mempool *mempool);
-
-
-int
-qat_crypto_set_session_parameters(struct rte_cryptodev *dev,
-		struct rte_crypto_sym_xform *xform, void *session_private);
-
-int
-qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform,
-				struct qat_session *session);
-
-int
-qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev,
-				struct rte_crypto_sym_xform *xform,
-				struct qat_session *session);
-
-int
-qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev,
-		struct rte_crypto_sym_xform *xform,
-		struct qat_session *session);
-
-
-extern void
-qat_crypto_sym_clear_session(struct rte_cryptodev *dev,
-		struct rte_cryptodev_sym_session *session);
-
-extern uint16_t
-qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
-		uint16_t nb_ops);
-
-extern uint16_t
-qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
-		uint16_t nb_ops);
-
-#endif /* _QAT_CRYPTO_H_ */
diff --git a/drivers/crypto/qat/qat_logs.h b/drivers/crypto/qat/qat_logs.h
deleted file mode 100644
index c9144bf6..00000000
--- a/drivers/crypto/qat/qat_logs.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2018 Intel Corporation
- */
-
-#ifndef _QAT_LOGS_H_
-#define _QAT_LOGS_H_
-
-#define PMD_INIT_LOG(level, fmt, args...) \
-	rte_log(RTE_LOG_ ## level, RTE_LOGTYPE_PMD, \
-		"PMD: %s(): " fmt "\n", __func__, ##args)
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_INIT
-#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
-#else
-#define PMD_INIT_FUNC_TRACE() do { } while (0)
-#endif
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
-#define PMD_RX_LOG(level, fmt, args...) \
-	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
-#else
-#define PMD_RX_LOG(level, fmt, args...) do { } while (0)
-#endif
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
-#define PMD_TX_LOG(level, fmt, args...) \
-	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
-#else
-#define PMD_TX_LOG(level, fmt, args...) do { } while (0)
-#endif
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX_FREE
-#define PMD_TX_FREE_LOG(level, fmt, args...) \
-	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
-#else
-#define PMD_TX_FREE_LOG(level, fmt, args...) do { } while (0)
-#endif
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_DRIVER
-#define PMD_DRV_LOG_RAW(level, fmt, args...) \
-	RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args)
-#else
-#define PMD_DRV_LOG_RAW(level, fmt, args...) do { } while (0)
-#endif
-
-#define PMD_DRV_LOG(level, fmt, args...) \
-	PMD_DRV_LOG_RAW(level, fmt "\n", ## args)
-
-#endif /* _QAT_LOGS_H_ */
diff --git a/drivers/crypto/qat/qat_qp.c b/drivers/crypto/qat/qat_qp.c
deleted file mode 100644
index 7fea10c7..00000000
--- a/drivers/crypto/qat/qat_qp.c
+++ /dev/null
@@ -1,451 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2018 Intel Corporation
- */
-
-#include <rte_common.h>
-#include <rte_dev.h>
-#include <rte_malloc.h>
-#include <rte_memzone.h>
-#include <rte_cryptodev_pmd.h>
-#include <rte_pci.h>
-#include <rte_bus_pci.h>
-#include <rte_atomic.h>
-#include <rte_prefetch.h>
-
-#include "qat_logs.h"
-#include "qat_crypto.h"
-#include "qat_algs.h"
-#include "adf_transport_access_macros.h"
-
-#define ADF_MAX_SYM_DESC			4096
-#define ADF_MIN_SYM_DESC			128
-#define ADF_SYM_TX_RING_DESC_SIZE		128
-#define ADF_SYM_RX_RING_DESC_SIZE		32
-#define ADF_SYM_TX_QUEUE_STARTOFF		2
-/* Offset from bundle start to 1st Sym Tx queue */
-#define ADF_SYM_RX_QUEUE_STARTOFF		10
-#define ADF_ARB_REG_SLOT			0x1000
-#define ADF_ARB_RINGSRVARBEN_OFFSET		0x19C
-
-#define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
-	ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
-	(ADF_ARB_REG_SLOT * index), value)
-
-static int qat_qp_check_queue_alignment(uint64_t phys_addr,
-	uint32_t queue_size_bytes);
-static int qat_tx_queue_create(struct rte_cryptodev *dev,
-	struct qat_queue *queue, uint8_t id, uint32_t nb_desc,
-	int socket_id);
-static int qat_rx_queue_create(struct rte_cryptodev *dev,
-	struct qat_queue *queue, uint8_t id, uint32_t nb_desc,
-	int socket_id);
-static void qat_queue_delete(struct qat_queue *queue);
-static int qat_queue_create(struct rte_cryptodev *dev,
-	struct qat_queue *queue, uint32_t nb_desc, uint8_t desc_size,
-	int socket_id);
-static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
-	uint32_t *queue_size_for_csr);
-static void adf_configure_queues(struct qat_qp *queue);
-static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr);
-static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr);
-
-static const struct rte_memzone *
-queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
-			int socket_id)
-{
-	const struct rte_memzone *mz;
-
-	PMD_INIT_FUNC_TRACE();
-	mz = rte_memzone_lookup(queue_name);
-	if (mz != 0) {
-		if (((size_t)queue_size <= mz->len) &&
-				((socket_id == SOCKET_ID_ANY) ||
-					(socket_id == mz->socket_id))) {
-			PMD_DRV_LOG(DEBUG, "re-use memzone already "
-					"allocated for %s", queue_name);
-			return mz;
-		}
-
-		PMD_DRV_LOG(ERR, "Incompatible memzone already "
-				"allocated %s, size %u, socket %d. "
-				"Requested size %u, socket %u",
-				queue_name, (uint32_t)mz->len,
-				mz->socket_id, queue_size, socket_id);
-		return NULL;
-	}
-
-	PMD_DRV_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
-					queue_name, queue_size, socket_id);
-	return rte_memzone_reserve_aligned(queue_name, queue_size,
-		socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
-}
-
-int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
-	const struct rte_cryptodev_qp_conf *qp_conf,
-	int socket_id, struct rte_mempool *session_pool __rte_unused)
-{
-	struct qat_qp *qp;
-	struct rte_pci_device *pci_dev;
-	int ret;
-	char op_cookie_pool_name[RTE_RING_NAMESIZE];
-	uint32_t i;
-
-	PMD_INIT_FUNC_TRACE();
-
-	/* If qp is already in use free ring memory and qp metadata. */
-	if (dev->data->queue_pairs[queue_pair_id] != NULL) {
-		ret = qat_crypto_sym_qp_release(dev, queue_pair_id);
-		if (ret < 0)
-			return ret;
-	}
-
-	if ((qp_conf->nb_descriptors > ADF_MAX_SYM_DESC) ||
-		(qp_conf->nb_descriptors < ADF_MIN_SYM_DESC)) {
-		PMD_DRV_LOG(ERR, "Can't create qp for %u descriptors",
-				qp_conf->nb_descriptors);
-		return -EINVAL;
-	}
-
-	pci_dev = RTE_DEV_TO_PCI(dev->device);
-
-	if (pci_dev->mem_resource[0].addr == NULL) {
-		PMD_DRV_LOG(ERR, "Could not find VF config space "
-				"(UIO driver attached?).");
-		return -EINVAL;
-	}
-
-	if (queue_pair_id >=
-			(ADF_NUM_SYM_QPS_PER_BUNDLE *
-					ADF_NUM_BUNDLES_PER_DEV)) {
-		PMD_DRV_LOG(ERR, "qp_id %u invalid for this device",
-				queue_pair_id);
-		return -EINVAL;
-	}
-	/* Allocate the queue pair data structure. */
-	qp = rte_zmalloc("qat PMD qp metadata",
-			sizeof(*qp), RTE_CACHE_LINE_SIZE);
-	if (qp == NULL) {
-		PMD_DRV_LOG(ERR, "Failed to alloc mem for qp struct");
-		return -ENOMEM;
-	}
-	qp->nb_descriptors = qp_conf->nb_descriptors;
-	qp->op_cookies = rte_zmalloc("qat PMD op cookie pointer",
-			qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
-			RTE_CACHE_LINE_SIZE);
-	if (qp->op_cookies == NULL) {
-		PMD_DRV_LOG(ERR, "Failed to alloc mem for cookie");
-		rte_free(qp);
-		return -ENOMEM;
-	}
-
-	qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
-	qp->inflights16 = 0;
-
-	if (qat_tx_queue_create(dev, &(qp->tx_q),
-		queue_pair_id, qp_conf->nb_descriptors, socket_id) != 0) {
-		PMD_INIT_LOG(ERR, "Tx queue create failed "
-				"queue_pair_id=%u", queue_pair_id);
-		goto create_err;
-	}
-
-	if (qat_rx_queue_create(dev, &(qp->rx_q),
-		queue_pair_id, qp_conf->nb_descriptors, socket_id) != 0) {
-		PMD_DRV_LOG(ERR, "Rx queue create failed "
-				"queue_pair_id=%hu", queue_pair_id);
-		qat_queue_delete(&(qp->tx_q));
-		goto create_err;
-	}
-
-	adf_configure_queues(qp);
-	adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr);
-	snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE, "%s_qp_op_%d_%hu",
-		pci_dev->driver->driver.name, dev->data->dev_id,
-		queue_pair_id);
-
-	qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
-	if (qp->op_cookie_pool == NULL)
-		qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
-				qp->nb_descriptors,
-				sizeof(struct qat_crypto_op_cookie), 64, 0,
-				NULL, NULL, NULL, NULL, socket_id,
-				0);
-	if (!qp->op_cookie_pool) {
-		PMD_DRV_LOG(ERR, "QAT PMD Cannot create"
-				" op mempool");
-		goto create_err;
-	}
-
-	for (i = 0; i < qp->nb_descriptors; i++) {
-		if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
-			PMD_DRV_LOG(ERR, "QAT PMD Cannot get op_cookie");
-			goto create_err;
-		}
-
-		struct qat_crypto_op_cookie *sql_cookie =
-				qp->op_cookies[i];
-
-		sql_cookie->qat_sgl_src_phys_addr =
-				rte_mempool_virt2iova(sql_cookie) +
-				offsetof(struct qat_crypto_op_cookie,
-				qat_sgl_list_src);
-
-		sql_cookie->qat_sgl_dst_phys_addr =
-				rte_mempool_virt2iova(sql_cookie) +
-				offsetof(struct qat_crypto_op_cookie,
-				qat_sgl_list_dst);
-	}
-
-	struct qat_pmd_private *internals
-		= dev->data->dev_private;
-	qp->qat_dev_gen = internals->qat_dev_gen;
-
-	dev->data->queue_pairs[queue_pair_id] = qp;
-	return 0;
-
-create_err:
-	if (qp->op_cookie_pool)
-		rte_mempool_free(qp->op_cookie_pool);
-	rte_free(qp->op_cookies);
-	rte_free(qp);
-	return -EFAULT;
-}
-
-int qat_crypto_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
-{
-	struct qat_qp *qp =
-			(struct qat_qp *)dev->data->queue_pairs[queue_pair_id];
-	uint32_t i;
-
-	PMD_INIT_FUNC_TRACE();
-	if (qp == NULL) {
-		PMD_DRV_LOG(DEBUG, "qp already freed");
-		return 0;
-	}
-
-	/* Don't free memory if there are still responses to be processed */
-	if (qp->inflights16 == 0) {
-		qat_queue_delete(&(qp->tx_q));
-		qat_queue_delete(&(qp->rx_q));
-	} else {
-		return -EAGAIN;
-	}
-
-	adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr);
-
-	for (i = 0; i < qp->nb_descriptors; i++)
-		rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
-
-	if (qp->op_cookie_pool)
-		rte_mempool_free(qp->op_cookie_pool);
-
-	rte_free(qp->op_cookies);
-	rte_free(qp);
-	dev->data->queue_pairs[queue_pair_id] = NULL;
-	return 0;
-}
-
-static int qat_tx_queue_create(struct rte_cryptodev *dev,
-	struct qat_queue *queue, uint8_t qp_id,
-	uint32_t nb_desc, int socket_id)
-{
-	PMD_INIT_FUNC_TRACE();
-	queue->hw_bundle_number = qp_id/ADF_NUM_SYM_QPS_PER_BUNDLE;
-	queue->hw_queue_number = (qp_id%ADF_NUM_SYM_QPS_PER_BUNDLE) +
-						ADF_SYM_TX_QUEUE_STARTOFF;
-	PMD_DRV_LOG(DEBUG, "TX ring for %u msgs: qp_id %d, bundle %u, ring %u",
-		nb_desc, qp_id, queue->hw_bundle_number,
-		queue->hw_queue_number);
-
-	return qat_queue_create(dev, queue, nb_desc,
-				ADF_SYM_TX_RING_DESC_SIZE, socket_id);
-}
-
-static int qat_rx_queue_create(struct rte_cryptodev *dev,
-		struct qat_queue *queue, uint8_t qp_id, uint32_t nb_desc,
-		int socket_id)
-{
-	PMD_INIT_FUNC_TRACE();
-	queue->hw_bundle_number = qp_id/ADF_NUM_SYM_QPS_PER_BUNDLE;
-	queue->hw_queue_number = (qp_id%ADF_NUM_SYM_QPS_PER_BUNDLE) +
-						ADF_SYM_RX_QUEUE_STARTOFF;
-
-	PMD_DRV_LOG(DEBUG, "RX ring for %u msgs: qp id %d, bundle %u, ring %u",
-		nb_desc, qp_id, queue->hw_bundle_number,
-		queue->hw_queue_number);
-	return qat_queue_create(dev, queue, nb_desc,
-				ADF_SYM_RX_RING_DESC_SIZE, socket_id);
-}
-
-static void qat_queue_delete(struct qat_queue *queue)
-{
-	const struct rte_memzone *mz;
-	int status = 0;
-
-	if (queue == NULL) {
-		PMD_DRV_LOG(DEBUG, "Invalid queue");
-		return;
-	}
-	mz = rte_memzone_lookup(queue->memz_name);
-	if (mz != NULL)	{
-		/* Write an unused pattern to the queue memory. */
-		memset(queue->base_addr, 0x7F, queue->queue_size);
-		status = rte_memzone_free(mz);
-		if (status != 0)
-			PMD_DRV_LOG(ERR, "Error %d on freeing queue %s",
-					status, queue->memz_name);
-	} else {
-		PMD_DRV_LOG(DEBUG, "queue %s doesn't exist",
-				queue->memz_name);
-	}
-}
-
-static int
-qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue,
-		uint32_t nb_desc, uint8_t desc_size, int socket_id)
-{
-	uint64_t queue_base;
-	void *io_addr;
-	const struct rte_memzone *qp_mz;
-	uint32_t queue_size_bytes = nb_desc*desc_size;
-	struct rte_pci_device *pci_dev;
-
-	PMD_INIT_FUNC_TRACE();
-	if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
-		PMD_DRV_LOG(ERR, "Invalid descriptor size %d", desc_size);
-		return -EINVAL;
-	}
-
-	pci_dev = RTE_DEV_TO_PCI(dev->device);
-
-	/*
-	 * Allocate a memzone for the queue - create a unique name.
-	 */
-	snprintf(queue->memz_name, sizeof(queue->memz_name), "%s_%s_%d_%d_%d",
-		pci_dev->driver->driver.name, "qp_mem", dev->data->dev_id,
-		queue->hw_bundle_number, queue->hw_queue_number);
-	qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
-			socket_id);
-	if (qp_mz == NULL) {
-		PMD_DRV_LOG(ERR, "Failed to allocate ring memzone");
-		return -ENOMEM;
-	}
-
-	queue->base_addr = (char *)qp_mz->addr;
-	queue->base_phys_addr = qp_mz->iova;
-	if (qat_qp_check_queue_alignment(queue->base_phys_addr,
-			queue_size_bytes)) {
-		PMD_DRV_LOG(ERR, "Invalid alignment on queue create "
-					" 0x%"PRIx64"\n",
-					queue->base_phys_addr);
-		return -EFAULT;
-	}
-
-	if (adf_verify_queue_size(desc_size, nb_desc, &(queue->queue_size))
-			!= 0) {
-		PMD_DRV_LOG(ERR, "Invalid num inflights");
-		return -EINVAL;
-	}
-
-	queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size,
-					ADF_BYTES_TO_MSG_SIZE(desc_size));
-	queue->modulo = ADF_RING_SIZE_MODULO(queue->queue_size);
-	PMD_DRV_LOG(DEBUG, "RING size in CSR: %u, in bytes %u, nb msgs %u,"
-				" msg_size %u, max_inflights %u modulo %u",
-				queue->queue_size, queue_size_bytes,
-				nb_desc, desc_size, queue->max_inflights,
-				queue->modulo);
-
-	if (queue->max_inflights < 2) {
-		PMD_DRV_LOG(ERR, "Invalid num inflights");
-		return -EINVAL;
-	}
-	queue->head = 0;
-	queue->tail = 0;
-	queue->msg_size = desc_size;
-
-	/*
-	 * Write an unused pattern to the queue memory.
-	 */
-	memset(queue->base_addr, 0x7F, queue_size_bytes);
-
-	queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
-					queue->queue_size);
-
-	io_addr = pci_dev->mem_resource[0].addr;
-
-	WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
-			queue->hw_queue_number, queue_base);
-	return 0;
-}
-
-static int qat_qp_check_queue_alignment(uint64_t phys_addr,
-					uint32_t queue_size_bytes)
-{
-	PMD_INIT_FUNC_TRACE();
-	if (((queue_size_bytes - 1) & phys_addr) != 0)
-		return -EINVAL;
-	return 0;
-}
-
-static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
-	uint32_t *p_queue_size_for_csr)
-{
-	uint8_t i = ADF_MIN_RING_SIZE;
-
-	PMD_INIT_FUNC_TRACE();
-	for (; i <= ADF_MAX_RING_SIZE; i++)
-		if ((msg_size * msg_num) ==
-				(uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
-			*p_queue_size_for_csr = i;
-			return 0;
-		}
-	PMD_DRV_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
-	return -EINVAL;
-}
-
-static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr)
-{
-	uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
-					(ADF_ARB_REG_SLOT *
-							txq->hw_bundle_number);
-	uint32_t value;
-
-	PMD_INIT_FUNC_TRACE();
-	value = ADF_CSR_RD(base_addr, arb_csr_offset);
-	value |= (0x01 << txq->hw_queue_number);
-	ADF_CSR_WR(base_addr, arb_csr_offset, value);
-}
-
-static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr)
-{
-	uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
-					(ADF_ARB_REG_SLOT *
-							txq->hw_bundle_number);
-	uint32_t value;
-
-	PMD_INIT_FUNC_TRACE();
-	value = ADF_CSR_RD(base_addr, arb_csr_offset);
-	value ^= (0x01 << txq->hw_queue_number);
-	ADF_CSR_WR(base_addr, arb_csr_offset, value);
-}
-
-static void adf_configure_queues(struct qat_qp *qp)
-{
-	uint32_t queue_config;
-	struct qat_queue *queue = &qp->tx_q;
-
-	PMD_INIT_FUNC_TRACE();
-	queue_config = BUILD_RING_CONFIG(queue->queue_size);
-
-	WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
-			queue->hw_queue_number, queue_config);
-
-	queue = &qp->rx_q;
-	queue_config =
-			BUILD_RESP_RING_CONFIG(queue->queue_size,
-					ADF_RING_NEAR_WATERMARK_512,
-					ADF_RING_NEAR_WATERMARK_0);
-
-	WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
-			queue->hw_queue_number, queue_config);
-}
diff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c
new file mode 100644
index 00000000..10cdf2e1
--- /dev/null
+++ b/drivers/crypto/qat/qat_sym.c
@@ -0,0 +1,569 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2018 Intel Corporation
+ */
+
+#include <openssl/evp.h>
+
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_crypto_sym.h>
+#include <rte_bus_pci.h>
+#include <rte_byteorder.h>
+
+#include "qat_sym.h"
+
+/** Decrypt a single partial block
+ *  Depends on openssl libcrypto
+ *  Uses ECB+XOR to do CFB encryption, same result, more performant
+ */
+static inline int
+bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_decrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_decrypt_err:
+	QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
+	return -EINVAL;
+}
+
+
+static inline uint32_t
+qat_bpicipher_preprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len &&
+			ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
+
+		/* Decrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely(sym_op->m_dst != NULL))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = last_block - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI:dst before pre-process:",
+			dst, last_block_len);
+#endif
+		bpi_cipher_decrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
+			dst, last_block_len);
+#endif
+	}
+
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static inline void
+set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
+		struct icp_qat_fw_la_cipher_req_params *cipher_param,
+		struct rte_crypto_op *op,
+		struct icp_qat_fw_la_bulk_req *qat_req)
+{
+	/* copy IV into request if it fits */
+	if (iv_length <= sizeof(cipher_param->u.cipher_IV_array)) {
+		rte_memcpy(cipher_param->u.cipher_IV_array,
+				rte_crypto_op_ctod_offset(op, uint8_t *,
+					iv_offset),
+				iv_length);
+	} else {
+		ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
+				qat_req->comn_hdr.serv_specif_flags,
+				ICP_QAT_FW_CIPH_IV_64BIT_PTR);
+		cipher_param->u.s.cipher_IV_ptr =
+				rte_crypto_op_ctophys_offset(op,
+					iv_offset);
+	}
+}
+
+/** Set IV for CCM is special case, 0th byte is set to q-1
+ *  where q is padding of nonce in 16 byte block
+ */
+static inline void
+set_cipher_iv_ccm(uint16_t iv_length, uint16_t iv_offset,
+		struct icp_qat_fw_la_cipher_req_params *cipher_param,
+		struct rte_crypto_op *op, uint8_t q, uint8_t aad_len_field_sz)
+{
+	rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array) +
+			ICP_QAT_HW_CCM_NONCE_OFFSET,
+			rte_crypto_op_ctod_offset(op, uint8_t *,
+				iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET,
+			iv_length);
+	*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
+			q - ICP_QAT_HW_CCM_NONCE_OFFSET;
+
+	if (aad_len_field_sz)
+		rte_memcpy(&op->sym->aead.aad.data[ICP_QAT_HW_CCM_NONCE_OFFSET],
+			rte_crypto_op_ctod_offset(op, uint8_t *,
+				iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET,
+			iv_length);
+}
+
+int
+qat_sym_build_request(void *in_op, uint8_t *out_msg,
+		void *op_cookie, enum qat_device_gen qat_dev_gen)
+{
+	int ret = 0;
+	struct qat_sym_session *ctx;
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+	register struct icp_qat_fw_la_bulk_req *qat_req;
+	uint8_t do_auth = 0, do_cipher = 0, do_aead = 0;
+	uint32_t cipher_len = 0, cipher_ofs = 0;
+	uint32_t auth_len = 0, auth_ofs = 0;
+	uint32_t min_ofs = 0;
+	uint64_t src_buf_start = 0, dst_buf_start = 0;
+	uint8_t do_sgl = 0;
+	struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
+	struct qat_sym_op_cookie *cookie =
+				(struct qat_sym_op_cookie *)op_cookie;
+
+	if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) {
+		QAT_DP_LOG(ERR, "QAT PMD only supports symmetric crypto "
+				"operation requests, op (%p) is not a "
+				"symmetric operation.", op);
+		return -EINVAL;
+	}
+
+	if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
+		QAT_DP_LOG(ERR, "QAT PMD only supports session oriented"
+				" requests, op (%p) is sessionless.", op);
+		return -EINVAL;
+	}
+
+	ctx = (struct qat_sym_session *)get_sym_session_private_data(
+			op->sym->session, cryptodev_qat_driver_id);
+
+	if (unlikely(ctx == NULL)) {
+		QAT_DP_LOG(ERR, "Session was not created for this device");
+		return -EINVAL;
+	}
+
+	if (unlikely(ctx->min_qat_dev_gen > qat_dev_gen)) {
+		QAT_DP_LOG(ERR, "Session alg not supported on this device gen");
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+		return -EINVAL;
+	}
+
+	qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
+	rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
+	qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;
+	cipher_param = (void *)&qat_req->serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
+
+	if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
+			ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
+		/* AES-GCM or AES-CCM */
+		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||
+			(ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128
+			&& ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE
+			&& ctx->qat_hash_alg ==
+					ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
+			do_aead = 1;
+		} else {
+			do_auth = 1;
+			do_cipher = 1;
+		}
+	} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) {
+		do_auth = 1;
+		do_cipher = 0;
+	} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
+		do_auth = 0;
+		do_cipher = 1;
+	}
+
+	if (do_cipher) {
+
+		if (ctx->qat_cipher_alg ==
+					 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
+			ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
+			ctx->qat_cipher_alg ==
+				ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
+
+			if (unlikely(
+			    (op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
+			    (op->sym->cipher.data.offset % BYTE_LENGTH != 0))) {
+				QAT_DP_LOG(ERR,
+		  "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
+				op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+				return -EINVAL;
+			}
+			cipher_len = op->sym->cipher.data.length >> 3;
+			cipher_ofs = op->sym->cipher.data.offset >> 3;
+
+		} else if (ctx->bpi_ctx) {
+			/* DOCSIS - only send complete blocks to device
+			 * Process any partial block using CFB mode.
+			 * Even if 0 complete blocks, still send this to device
+			 * to get into rx queue for post-process and dequeuing
+			 */
+			cipher_len = qat_bpicipher_preprocess(ctx, op);
+			cipher_ofs = op->sym->cipher.data.offset;
+		} else {
+			cipher_len = op->sym->cipher.data.length;
+			cipher_ofs = op->sym->cipher.data.offset;
+		}
+
+		set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset,
+				cipher_param, op, qat_req);
+		min_ofs = cipher_ofs;
+	}
+
+	if (do_auth) {
+
+		if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
+			ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
+			ctx->qat_hash_alg ==
+				ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
+			if (unlikely(
+			    (op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
+			    (op->sym->auth.data.length % BYTE_LENGTH != 0))) {
+				QAT_DP_LOG(ERR,
+		"For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
+				op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+				return -EINVAL;
+			}
+			auth_ofs = op->sym->auth.data.offset >> 3;
+			auth_len = op->sym->auth.data.length >> 3;
+
+			auth_param->u1.aad_adr =
+					rte_crypto_op_ctophys_offset(op,
+							ctx->auth_iv.offset);
+
+		} else if (ctx->qat_hash_alg ==
+					ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+				ctx->qat_hash_alg ==
+					ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+			/* AES-GMAC */
+			set_cipher_iv(ctx->auth_iv.length,
+				ctx->auth_iv.offset,
+				cipher_param, op, qat_req);
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+
+			auth_param->u1.aad_adr = 0;
+			auth_param->u2.aad_sz = 0;
+
+			/*
+			 * If len(iv)==12B fw computes J0
+			 */
+			if (ctx->auth_iv.length == 12) {
+				ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+					qat_req->comn_hdr.serv_specif_flags,
+					ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+
+			}
+		} else {
+			auth_ofs = op->sym->auth.data.offset;
+			auth_len = op->sym->auth.data.length;
+
+		}
+		min_ofs = auth_ofs;
+
+		if (likely(ctx->qat_hash_alg != ICP_QAT_HW_AUTH_ALGO_NULL))
+			auth_param->auth_res_addr =
+					op->sym->auth.digest.phys_addr;
+
+	}
+
+	if (do_aead) {
+		/*
+		 * This address may used for setting AAD physical pointer
+		 * into IV offset from op
+		 */
+		rte_iova_t aad_phys_addr_aead = op->sym->aead.aad.phys_addr;
+		if (ctx->qat_hash_alg ==
+				ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
+				ctx->qat_hash_alg ==
+					ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
+			/*
+			 * If len(iv)==12B fw computes J0
+			 */
+			if (ctx->cipher_iv.length == 12) {
+				ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+					qat_req->comn_hdr.serv_specif_flags,
+					ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+			}
+			set_cipher_iv(ctx->cipher_iv.length,
+					ctx->cipher_iv.offset,
+					cipher_param, op, qat_req);
+
+		} else if (ctx->qat_hash_alg ==
+				ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC) {
+
+			/* In case of AES-CCM this may point to user selected
+			 * memory or iv offset in cypto_op
+			 */
+			uint8_t *aad_data = op->sym->aead.aad.data;
+			/* This is true AAD length, it not includes 18 bytes of
+			 * preceding data
+			 */
+			uint8_t aad_ccm_real_len = 0;
+			uint8_t aad_len_field_sz = 0;
+			uint32_t msg_len_be =
+					rte_bswap32(op->sym->aead.data.length);
+
+			if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
+				aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
+				aad_ccm_real_len = ctx->aad_len -
+					ICP_QAT_HW_CCM_AAD_B0_LEN -
+					ICP_QAT_HW_CCM_AAD_LEN_INFO;
+			} else {
+				/*
+				 * aad_len not greater than 18, so no actual aad
+				 *  data, then use IV after op for B0 block
+				 */
+				aad_data = rte_crypto_op_ctod_offset(op,
+						uint8_t *,
+						ctx->cipher_iv.offset);
+				aad_phys_addr_aead =
+						rte_crypto_op_ctophys_offset(op,
+							ctx->cipher_iv.offset);
+			}
+
+			uint8_t q = ICP_QAT_HW_CCM_NQ_CONST -
+							ctx->cipher_iv.length;
+
+			aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
+							aad_len_field_sz,
+							ctx->digest_length, q);
+
+			if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
+				memcpy(aad_data	+ ctx->cipher_iv.length +
+				    ICP_QAT_HW_CCM_NONCE_OFFSET +
+				    (q - ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
+				    (uint8_t *)&msg_len_be,
+				    ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
+			} else {
+				memcpy(aad_data	+ ctx->cipher_iv.length +
+				    ICP_QAT_HW_CCM_NONCE_OFFSET,
+				    (uint8_t *)&msg_len_be
+				    + (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
+				    - q), q);
+			}
+
+			if (aad_len_field_sz > 0) {
+				*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN]
+						= rte_bswap16(aad_ccm_real_len);
+
+				if ((aad_ccm_real_len + aad_len_field_sz)
+						% ICP_QAT_HW_CCM_AAD_B0_LEN) {
+					uint8_t pad_len = 0;
+					uint8_t pad_idx = 0;
+
+					pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
+					((aad_ccm_real_len + aad_len_field_sz) %
+						ICP_QAT_HW_CCM_AAD_B0_LEN);
+					pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
+					    aad_ccm_real_len + aad_len_field_sz;
+					memset(&aad_data[pad_idx],
+							0, pad_len);
+				}
+
+			}
+
+			set_cipher_iv_ccm(ctx->cipher_iv.length,
+					ctx->cipher_iv.offset,
+					cipher_param, op, q,
+					aad_len_field_sz);
+
+		}
+
+		cipher_len = op->sym->aead.data.length;
+		cipher_ofs = op->sym->aead.data.offset;
+		auth_len = op->sym->aead.data.length;
+		auth_ofs = op->sym->aead.data.offset;
+
+		auth_param->u1.aad_adr = aad_phys_addr_aead;
+		auth_param->auth_res_addr = op->sym->aead.digest.phys_addr;
+		min_ofs = op->sym->aead.data.offset;
+	}
+
+	if (op->sym->m_src->next || (op->sym->m_dst && op->sym->m_dst->next))
+		do_sgl = 1;
+
+	/* adjust for chain case */
+	if (do_cipher && do_auth)
+		min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
+
+	if (unlikely(min_ofs >= rte_pktmbuf_data_len(op->sym->m_src) && do_sgl))
+		min_ofs = 0;
+
+	if (unlikely(op->sym->m_dst != NULL)) {
+		/* Out-of-place operation (OOP)
+		 * Don't align DMA start. DMA the minimum data-set
+		 * so as not to overwrite data in dest buffer
+		 */
+		src_buf_start =
+			rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs);
+		dst_buf_start =
+			rte_pktmbuf_iova_offset(op->sym->m_dst, min_ofs);
+
+	} else {
+		/* In-place operation
+		 * Start DMA at nearest aligned address below min_ofs
+		 */
+		src_buf_start =
+			rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs)
+						& QAT_64_BTYE_ALIGN_MASK;
+
+		if (unlikely((rte_pktmbuf_iova(op->sym->m_src) -
+					rte_pktmbuf_headroom(op->sym->m_src))
+							> src_buf_start)) {
+			/* alignment has pushed addr ahead of start of mbuf
+			 * so revert and take the performance hit
+			 */
+			src_buf_start =
+				rte_pktmbuf_iova_offset(op->sym->m_src,
+								min_ofs);
+		}
+		dst_buf_start = src_buf_start;
+	}
+
+	if (do_cipher || do_aead) {
+		cipher_param->cipher_offset =
+				(uint32_t)rte_pktmbuf_iova_offset(
+				op->sym->m_src, cipher_ofs) - src_buf_start;
+		cipher_param->cipher_length = cipher_len;
+	} else {
+		cipher_param->cipher_offset = 0;
+		cipher_param->cipher_length = 0;
+	}
+
+	if (do_auth || do_aead) {
+		auth_param->auth_off = (uint32_t)rte_pktmbuf_iova_offset(
+				op->sym->m_src, auth_ofs) - src_buf_start;
+		auth_param->auth_len = auth_len;
+	} else {
+		auth_param->auth_off = 0;
+		auth_param->auth_len = 0;
+	}
+
+	qat_req->comn_mid.dst_length =
+		qat_req->comn_mid.src_length =
+		(cipher_param->cipher_offset + cipher_param->cipher_length)
+		> (auth_param->auth_off + auth_param->auth_len) ?
+		(cipher_param->cipher_offset + cipher_param->cipher_length)
+		: (auth_param->auth_off + auth_param->auth_len);
+
+	if (do_sgl) {
+
+		ICP_QAT_FW_COMN_PTR_TYPE_SET(qat_req->comn_hdr.comn_req_flags,
+				QAT_COMN_PTR_TYPE_SGL);
+		ret = qat_sgl_fill_array(op->sym->m_src,
+		   (int64_t)(src_buf_start - rte_pktmbuf_iova(op->sym->m_src)),
+		   &cookie->qat_sgl_src,
+		   qat_req->comn_mid.src_length,
+		   QAT_SYM_SGL_MAX_NUMBER);
+
+		if (unlikely(ret)) {
+			QAT_DP_LOG(ERR, "QAT PMD Cannot fill sgl array");
+			return ret;
+		}
+
+		if (likely(op->sym->m_dst == NULL))
+			qat_req->comn_mid.dest_data_addr =
+				qat_req->comn_mid.src_data_addr =
+				cookie->qat_sgl_src_phys_addr;
+		else {
+			ret = qat_sgl_fill_array(op->sym->m_dst,
+				(int64_t)(dst_buf_start -
+					  rte_pktmbuf_iova(op->sym->m_dst)),
+				 &cookie->qat_sgl_dst,
+				 qat_req->comn_mid.dst_length,
+				 QAT_SYM_SGL_MAX_NUMBER);
+
+			if (unlikely(ret)) {
+				QAT_DP_LOG(ERR, "QAT PMD can't fill sgl array");
+				return ret;
+			}
+
+			qat_req->comn_mid.src_data_addr =
+				cookie->qat_sgl_src_phys_addr;
+			qat_req->comn_mid.dest_data_addr =
+					cookie->qat_sgl_dst_phys_addr;
+		}
+	} else {
+		qat_req->comn_mid.src_data_addr = src_buf_start;
+		qat_req->comn_mid.dest_data_addr = dst_buf_start;
+	}
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+	QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
+			sizeof(struct icp_qat_fw_la_bulk_req));
+	QAT_DP_HEXDUMP_LOG(DEBUG, "src_data:",
+			rte_pktmbuf_mtod(op->sym->m_src, uint8_t*),
+			rte_pktmbuf_data_len(op->sym->m_src));
+	if (do_cipher) {
+		uint8_t *cipher_iv_ptr = rte_crypto_op_ctod_offset(op,
+						uint8_t *,
+						ctx->cipher_iv.offset);
+		QAT_DP_HEXDUMP_LOG(DEBUG, "cipher iv:", cipher_iv_ptr,
+				ctx->cipher_iv.length);
+	}
+
+	if (do_auth) {
+		if (ctx->auth_iv.length) {
+			uint8_t *auth_iv_ptr = rte_crypto_op_ctod_offset(op,
+							uint8_t *,
+							ctx->auth_iv.offset);
+			QAT_DP_HEXDUMP_LOG(DEBUG, "auth iv:", auth_iv_ptr,
+						ctx->auth_iv.length);
+		}
+		QAT_DP_HEXDUMP_LOG(DEBUG, "digest:", op->sym->auth.digest.data,
+				ctx->digest_length);
+	}
+
+	if (do_aead) {
+		QAT_DP_HEXDUMP_LOG(DEBUG, "digest:", op->sym->aead.digest.data,
+				ctx->digest_length);
+		QAT_DP_HEXDUMP_LOG(DEBUG, "aad:", op->sym->aead.aad.data,
+				ctx->aad_len);
+	}
+#endif
+	return 0;
+}
diff --git a/drivers/crypto/qat/qat_sym.h b/drivers/crypto/qat/qat_sym.h
new file mode 100644
index 00000000..bc6426c3
--- /dev/null
+++ b/drivers/crypto/qat/qat_sym.h
@@ -0,0 +1,174 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2018 Intel Corporation
+ */
+
+#ifndef _QAT_SYM_H_
+#define _QAT_SYM_H_
+
+#include <rte_cryptodev_pmd.h>
+
+#ifdef BUILD_QAT_SYM
+#include <openssl/evp.h>
+
+#include "qat_common.h"
+#include "qat_sym_session.h"
+#include "qat_sym_pmd.h"
+#include "qat_logs.h"
+
+#define BYTE_LENGTH    8
+/* bpi is only used for partial blocks of DES and AES
+ * so AES block len can be assumed as max len for iv, src and dst
+ */
+#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
+
+/*
+ * Maximum number of SGL entries
+ */
+#define QAT_SYM_SGL_MAX_NUMBER	16
+
+struct qat_sym_session;
+
+struct qat_sym_sgl {
+	qat_sgl_hdr;
+	struct qat_flat_buf buffers[QAT_SYM_SGL_MAX_NUMBER];
+} __rte_packed __rte_cache_aligned;
+
+struct qat_sym_op_cookie {
+	struct qat_sym_sgl qat_sgl_src;
+	struct qat_sym_sgl qat_sgl_dst;
+	phys_addr_t qat_sgl_src_phys_addr;
+	phys_addr_t qat_sgl_dst_phys_addr;
+};
+
+int
+qat_sym_build_request(void *in_op, uint8_t *out_msg,
+		void *op_cookie, enum qat_device_gen qat_dev_gen);
+
+
+/** Encrypt a single partial block
+ *  Depends on openssl libcrypto
+ *  Uses ECB+XOR to do CFB encryption, same result, more performant
+ */
+static inline int
+bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt the IV, then XOR this with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_encrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_encrypt_err:
+	QAT_DP_LOG(ERR, "libcrypto ECB cipher encrypt failed");
+	return -EINVAL;
+}
+
+static inline uint32_t
+qat_bpicipher_postprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len > 0 &&
+			ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
+
+		/* Encrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset;
+
+		last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely(sym_op->m_dst != NULL))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = dst - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before post-process:",
+			last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG,
+				"BPI: dst before post-process:",
+				dst, last_block_len);
+#endif
+		bpi_cipher_encrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after post-process:",
+				last_block, last_block_len);
+		if (sym_op->m_dst != NULL)
+			QAT_DP_HEXDUMP_LOG(DEBUG,
+				"BPI: dst after post-process:",
+				dst, last_block_len);
+#endif
+	}
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static inline void
+qat_sym_process_response(void **op, uint8_t *resp)
+{
+
+	struct icp_qat_fw_comn_resp *resp_msg =
+			(struct icp_qat_fw_comn_resp *)resp;
+	struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
+			(resp_msg->opaque_data);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+	QAT_DP_HEXDUMP_LOG(DEBUG, "qat_response:", (uint8_t *)resp_msg,
+			sizeof(struct icp_qat_fw_comn_resp));
+#endif
+
+	if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
+			ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
+			resp_msg->comn_hdr.comn_status)) {
+
+		rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+	} else {
+		struct qat_sym_session *sess = (struct qat_sym_session *)
+						get_sym_session_private_data(
+						rx_op->sym->session,
+						cryptodev_qat_driver_id);
+
+
+		if (sess->bpi_ctx)
+			qat_bpicipher_postprocess(sess, rx_op);
+		rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+	}
+	*op = (void *)rx_op;
+}
+#else
+
+static inline void
+qat_sym_process_response(void **op __rte_unused, uint8_t *resp __rte_unused)
+{
+}
+#endif
+#endif /* _QAT_SYM_H_ */
diff --git a/drivers/crypto/qat/qat_crypto_capabilities.h b/drivers/crypto/qat/qat_sym_capabilities.h
index 001c32c5..eea08bc7 100644
--- a/drivers/crypto/qat/qat_crypto_capabilities.h
+++ b/drivers/crypto/qat/qat_sym_capabilities.h
@@ -2,8 +2,8 @@
  * Copyright(c) 2017-2018 Intel Corporation
  */
 
-#ifndef _QAT_CRYPTO_CAPABILITIES_H_
-#define _QAT_CRYPTO_CAPABILITIES_H_
+#ifndef _QAT_SYM_CAPABILITIES_H_
+#define _QAT_SYM_CAPABILITIES_H_
 
 #define QAT_BASE_GEN1_SYM_CAPABILITIES					\
 	{	/* SHA1 HMAC */						\
@@ -434,7 +434,7 @@
 				.algo = RTE_CRYPTO_CIPHER_3DES_CBC,	\
 				.block_size = 8,			\
 				.key_size = {				\
-					.min = 16,			\
+					.min = 8,			\
 					.max = 24,			\
 					.increment = 8			\
 				},					\
@@ -554,4 +554,4 @@
 		}, }							\
 	}
 
-#endif /* _QAT_CRYPTO_CAPABILITIES_H_ */
+#endif /* _QAT_SYM_CAPABILITIES_H_ */
diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c
new file mode 100644
index 00000000..96f442e8
--- /dev/null
+++ b/drivers/crypto/qat/qat_sym_pmd.c
@@ -0,0 +1,331 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2018 Intel Corporation
+ */
+
+#include <rte_bus_pci.h>
+#include <rte_common.h>
+#include <rte_dev.h>
+#include <rte_malloc.h>
+#include <rte_pci.h>
+#include <rte_cryptodev_pmd.h>
+
+#include "qat_logs.h"
+#include "qat_sym.h"
+#include "qat_sym_session.h"
+#include "qat_sym_pmd.h"
+
+uint8_t cryptodev_qat_driver_id;
+
+static const struct rte_cryptodev_capabilities qat_gen1_sym_capabilities[] = {
+	QAT_BASE_GEN1_SYM_CAPABILITIES,
+	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+static const struct rte_cryptodev_capabilities qat_gen2_sym_capabilities[] = {
+	QAT_BASE_GEN1_SYM_CAPABILITIES,
+	QAT_EXTRA_GEN2_SYM_CAPABILITIES,
+	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+static int qat_sym_qp_release(struct rte_cryptodev *dev,
+	uint16_t queue_pair_id);
+
+static int qat_sym_dev_config(__rte_unused struct rte_cryptodev *dev,
+		__rte_unused struct rte_cryptodev_config *config)
+{
+	return 0;
+}
+
+static int qat_sym_dev_start(__rte_unused struct rte_cryptodev *dev)
+{
+	return 0;
+}
+
+static void qat_sym_dev_stop(__rte_unused struct rte_cryptodev *dev)
+{
+	return;
+}
+
+static int qat_sym_dev_close(struct rte_cryptodev *dev)
+{
+	int i, ret;
+
+	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
+		ret = qat_sym_qp_release(dev, i);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void qat_sym_dev_info_get(struct rte_cryptodev *dev,
+			struct rte_cryptodev_info *info)
+{
+	struct qat_sym_dev_private *internals = dev->data->dev_private;
+	const struct qat_qp_hw_data *sym_hw_qps =
+		qat_gen_config[internals->qat_dev->qat_dev_gen]
+			      .qp_hw_data[QAT_SERVICE_SYMMETRIC];
+
+	if (info != NULL) {
+		info->max_nb_queue_pairs =
+			qat_qps_per_service(sym_hw_qps, QAT_SERVICE_SYMMETRIC);
+		info->feature_flags = dev->feature_flags;
+		info->capabilities = internals->qat_dev_capabilities;
+		info->driver_id = cryptodev_qat_driver_id;
+		/* No limit of number of sessions */
+		info->sym.max_nb_sessions = 0;
+	}
+}
+
+static void qat_sym_stats_get(struct rte_cryptodev *dev,
+		struct rte_cryptodev_stats *stats)
+{
+	struct qat_common_stats qat_stats = {0};
+	struct qat_sym_dev_private *qat_priv;
+
+	if (stats == NULL || dev == NULL) {
+		QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
+		return;
+	}
+	qat_priv = dev->data->dev_private;
+
+	qat_stats_get(qat_priv->qat_dev, &qat_stats, QAT_SERVICE_SYMMETRIC);
+	stats->enqueued_count = qat_stats.enqueued_count;
+	stats->dequeued_count = qat_stats.dequeued_count;
+	stats->enqueue_err_count = qat_stats.enqueue_err_count;
+	stats->dequeue_err_count = qat_stats.dequeue_err_count;
+}
+
+static void qat_sym_stats_reset(struct rte_cryptodev *dev)
+{
+	struct qat_sym_dev_private *qat_priv;
+
+	if (dev == NULL) {
+		QAT_LOG(ERR, "invalid cryptodev ptr %p", dev);
+		return;
+	}
+	qat_priv = dev->data->dev_private;
+
+	qat_stats_reset(qat_priv->qat_dev, QAT_SERVICE_SYMMETRIC);
+
+}
+
+static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
+{
+	struct qat_sym_dev_private *qat_private = dev->data->dev_private;
+
+	QAT_LOG(DEBUG, "Release sym qp %u on device %d",
+				queue_pair_id, dev->data->dev_id);
+
+	qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][queue_pair_id]
+						= NULL;
+
+	return qat_qp_release((struct qat_qp **)
+			&(dev->data->queue_pairs[queue_pair_id]));
+}
+
+static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+	const struct rte_cryptodev_qp_conf *qp_conf,
+	int socket_id, struct rte_mempool *session_pool __rte_unused)
+{
+	struct qat_qp *qp;
+	int ret = 0;
+	uint32_t i;
+	struct qat_qp_config qat_qp_conf;
+
+	struct qat_qp **qp_addr =
+			(struct qat_qp **)&(dev->data->queue_pairs[qp_id]);
+	struct qat_sym_dev_private *qat_private = dev->data->dev_private;
+	const struct qat_qp_hw_data *sym_hw_qps =
+			qat_gen_config[qat_private->qat_dev->qat_dev_gen]
+				      .qp_hw_data[QAT_SERVICE_SYMMETRIC];
+	const struct qat_qp_hw_data *qp_hw_data = sym_hw_qps + qp_id;
+
+	/* If qp is already in use free ring memory and qp metadata. */
+	if (*qp_addr != NULL) {
+		ret = qat_sym_qp_release(dev, qp_id);
+		if (ret < 0)
+			return ret;
+	}
+	if (qp_id >= qat_qps_per_service(sym_hw_qps, QAT_SERVICE_SYMMETRIC)) {
+		QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
+		return -EINVAL;
+	}
+
+	qat_qp_conf.hw = qp_hw_data;
+	qat_qp_conf.build_request = qat_sym_build_request;
+	qat_qp_conf.cookie_size = sizeof(struct qat_sym_op_cookie);
+	qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
+	qat_qp_conf.socket_id = socket_id;
+	qat_qp_conf.service_str = "sym";
+
+	ret = qat_qp_setup(qat_private->qat_dev, qp_addr, qp_id, &qat_qp_conf);
+	if (ret != 0)
+		return ret;
+
+	/* store a link to the qp in the qat_pci_device */
+	qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][qp_id]
+							= *qp_addr;
+
+	qp = (struct qat_qp *)*qp_addr;
+
+	for (i = 0; i < qp->nb_descriptors; i++) {
+
+		struct qat_sym_op_cookie *cookie =
+				qp->op_cookies[i];
+
+		cookie->qat_sgl_src_phys_addr =
+				rte_mempool_virt2iova(cookie) +
+				offsetof(struct qat_sym_op_cookie,
+				qat_sgl_src);
+
+		cookie->qat_sgl_dst_phys_addr =
+				rte_mempool_virt2iova(cookie) +
+				offsetof(struct qat_sym_op_cookie,
+				qat_sgl_dst);
+	}
+
+	return ret;
+}
+
+static struct rte_cryptodev_ops crypto_qat_ops = {
+
+		/* Device related operations */
+		.dev_configure		= qat_sym_dev_config,
+		.dev_start		= qat_sym_dev_start,
+		.dev_stop		= qat_sym_dev_stop,
+		.dev_close		= qat_sym_dev_close,
+		.dev_infos_get		= qat_sym_dev_info_get,
+
+		.stats_get		= qat_sym_stats_get,
+		.stats_reset		= qat_sym_stats_reset,
+		.queue_pair_setup	= qat_sym_qp_setup,
+		.queue_pair_release	= qat_sym_qp_release,
+		.queue_pair_count	= NULL,
+
+		/* Crypto related operations */
+		.sym_session_get_size	= qat_sym_session_get_private_size,
+		.sym_session_configure	= qat_sym_session_configure,
+		.sym_session_clear	= qat_sym_session_clear
+};
+
+static uint16_t
+qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops)
+{
+	return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
+}
+
+static uint16_t
+qat_sym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
+		uint16_t nb_ops)
+{
+	return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
+}
+
+/* An rte_driver is needed in the registration of both the device and the driver
+ * with cryptodev.
+ * The actual qat pci's rte_driver can't be used as its name represents
+ * the whole pci device with all services. Think of this as a holder for a name
+ * for the crypto part of the pci device.
+ */
+static const char qat_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD);
+static const struct rte_driver cryptodev_qat_sym_driver = {
+	.name = qat_sym_drv_name,
+	.alias = qat_sym_drv_name
+};
+
+int
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev)
+{
+	struct rte_cryptodev_pmd_init_params init_params = {
+			.name = "",
+			.socket_id = qat_pci_dev->pci_dev->device.numa_node,
+			.private_data_size = sizeof(struct qat_sym_dev_private)
+	};
+	char name[RTE_CRYPTODEV_NAME_MAX_LEN];
+	struct rte_cryptodev *cryptodev;
+	struct qat_sym_dev_private *internals;
+
+	snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
+			qat_pci_dev->name, "sym");
+	QAT_LOG(DEBUG, "Creating QAT SYM device %s", name);
+
+	/* Populate subset device to use in cryptodev device creation */
+	qat_pci_dev->sym_rte_dev.driver = &cryptodev_qat_sym_driver;
+	qat_pci_dev->sym_rte_dev.numa_node =
+				qat_pci_dev->pci_dev->device.numa_node;
+	qat_pci_dev->sym_rte_dev.devargs = NULL;
+
+	cryptodev = rte_cryptodev_pmd_create(name,
+			&(qat_pci_dev->sym_rte_dev), &init_params);
+
+	if (cryptodev == NULL)
+		return -ENODEV;
+
+	qat_pci_dev->sym_rte_dev.name = cryptodev->data->name;
+	cryptodev->driver_id = cryptodev_qat_driver_id;
+	cryptodev->dev_ops = &crypto_qat_ops;
+
+	cryptodev->enqueue_burst = qat_sym_pmd_enqueue_op_burst;
+	cryptodev->dequeue_burst = qat_sym_pmd_dequeue_op_burst;
+
+	cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+			RTE_CRYPTODEV_FF_HW_ACCELERATED |
+			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
+			RTE_CRYPTODEV_FF_IN_PLACE_SGL |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
+
+	internals = cryptodev->data->dev_private;
+	internals->qat_dev = qat_pci_dev;
+	qat_pci_dev->sym_dev = internals;
+
+	internals->sym_dev_id = cryptodev->data->dev_id;
+	switch (qat_pci_dev->qat_dev_gen) {
+	case QAT_GEN1:
+		internals->qat_dev_capabilities = qat_gen1_sym_capabilities;
+		break;
+	case QAT_GEN2:
+		internals->qat_dev_capabilities = qat_gen2_sym_capabilities;
+		break;
+	default:
+		internals->qat_dev_capabilities = qat_gen2_sym_capabilities;
+		QAT_LOG(DEBUG,
+			"QAT gen %d capabilities unknown, default to GEN2",
+					qat_pci_dev->qat_dev_gen);
+		break;
+	}
+
+	QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d",
+			cryptodev->data->name, internals->sym_dev_id);
+	return 0;
+}
+
+int
+qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)
+{
+	struct rte_cryptodev *cryptodev;
+
+	if (qat_pci_dev == NULL)
+		return -ENODEV;
+	if (qat_pci_dev->sym_dev == NULL)
+		return 0;
+
+	/* free crypto device */
+	cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->sym_dev_id);
+	rte_cryptodev_pmd_destroy(cryptodev);
+	qat_pci_dev->sym_rte_dev.name = NULL;
+	qat_pci_dev->sym_dev = NULL;
+
+	return 0;
+}
+
+
+static struct cryptodev_driver qat_crypto_drv;
+RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
+		cryptodev_qat_sym_driver,
+		cryptodev_qat_driver_id);
diff --git a/drivers/crypto/qat/qat_sym_pmd.h b/drivers/crypto/qat/qat_sym_pmd.h
new file mode 100644
index 00000000..d3432854
--- /dev/null
+++ b/drivers/crypto/qat/qat_sym_pmd.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2018 Intel Corporation
+ */
+
+#ifndef _QAT_SYM_PMD_H_
+#define _QAT_SYM_PMD_H_
+
+#ifdef BUILD_QAT_SYM
+
+#include <rte_cryptodev.h>
+
+#include "qat_sym_capabilities.h"
+#include "qat_device.h"
+
+/**< Intel(R) QAT Symmetric Crypto PMD device name */
+#define CRYPTODEV_NAME_QAT_SYM_PMD	crypto_qat
+
+extern uint8_t cryptodev_qat_driver_id;
+
+/** private data structure for a QAT device.
+ * This QAT device is a device offering only symmetric crypto service,
+ * there can be one of these on each qat_pci_device (VF),
+ * in future there may also be private data structures for other services.
+ */
+struct qat_sym_dev_private {
+	struct qat_pci_device *qat_dev;
+	/**< The qat pci device hosting the service */
+	uint8_t sym_dev_id;
+	/**< Device instance for this rte_cryptodev */
+	const struct rte_cryptodev_capabilities *qat_dev_capabilities;
+	/* QAT device symmetric crypto capabilities */
+};
+
+int
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev);
+
+int
+qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev);
+
+#endif
+#endif /* _QAT_SYM_PMD_H_ */
diff --git a/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c b/drivers/crypto/qat/qat_sym_session.c
index c87ed40f..1d58220a 100644
--- a/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c
+++ b/drivers/crypto/qat/qat_sym_session.c
@@ -1,6 +1,12 @@
 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
  * Copyright(c) 2015-2018 Intel Corporation
  */
+
+#include <openssl/sha.h>	/* Needed to calculate pre-compute values */
+#include <openssl/aes.h>	/* Needed to calculate pre-compute values */
+#include <openssl/md5.h>	/* Needed to calculate pre-compute values */
+#include <openssl/evp.h>	/* Needed for bpi runt block processing */
+
 #include <rte_memcpy.h>
 #include <rte_common.h>
 #include <rte_spinlock.h>
@@ -9,13 +15,714 @@
 #include <rte_malloc.h>
 #include <rte_crypto_sym.h>
 
-#include "../qat_logs.h"
+#include "qat_logs.h"
+#include "qat_sym_session.h"
+#include "qat_sym_pmd.h"
 
-#include <openssl/sha.h>	/* Needed to calculate pre-compute values */
-#include <openssl/aes.h>	/* Needed to calculate pre-compute values */
-#include <openssl/md5.h>	/* Needed to calculate pre-compute values */
+/** Frees a context previously created
+ *  Depends on openssl libcrypto
+ */
+static void
+bpi_cipher_ctx_free(void *bpi_ctx)
+{
+	if (bpi_ctx != NULL)
+		EVP_CIPHER_CTX_free((EVP_CIPHER_CTX *)bpi_ctx);
+}
+
+/** Creates a context in either AES or DES in ECB mode
+ *  Depends on openssl libcrypto
+ */
+static int
+bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo,
+		enum rte_crypto_cipher_operation direction __rte_unused,
+		uint8_t *key, void **ctx)
+{
+	const EVP_CIPHER *algo = NULL;
+	int ret;
+	*ctx = EVP_CIPHER_CTX_new();
+
+	if (*ctx == NULL) {
+		ret = -ENOMEM;
+		goto ctx_init_err;
+	}
+
+	if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI)
+		algo = EVP_des_ecb();
+	else
+		algo = EVP_aes_128_ecb();
+
+	/* IV will be ECB encrypted whether direction is encrypt or decrypt*/
+	if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) {
+		ret = -EINVAL;
+		goto ctx_init_err;
+	}
+
+	return 0;
+
+ctx_init_err:
+	if (*ctx != NULL)
+		EVP_CIPHER_CTX_free(*ctx);
+	return ret;
+}
+
+static int
+qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo,
+		struct qat_sym_dev_private *internals)
+{
+	int i = 0;
+	const struct rte_cryptodev_capabilities *capability;
+
+	while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
+			RTE_CRYPTO_OP_TYPE_UNDEFINED) {
+		if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
+			continue;
+
+		if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
+			continue;
+
+		if (capability->sym.cipher.algo == algo)
+			return 1;
+	}
+	return 0;
+}
+
+static int
+qat_is_auth_alg_supported(enum rte_crypto_auth_algorithm algo,
+		struct qat_sym_dev_private *internals)
+{
+	int i = 0;
+	const struct rte_cryptodev_capabilities *capability;
+
+	while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
+			RTE_CRYPTO_OP_TYPE_UNDEFINED) {
+		if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
+			continue;
+
+		if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
+			continue;
+
+		if (capability->sym.auth.algo == algo)
+			return 1;
+	}
+	return 0;
+}
+
+void
+qat_sym_session_clear(struct rte_cryptodev *dev,
+		struct rte_cryptodev_sym_session *sess)
+{
+	uint8_t index = dev->driver_id;
+	void *sess_priv = get_sym_session_private_data(sess, index);
+	struct qat_sym_session *s = (struct qat_sym_session *)sess_priv;
+
+	if (sess_priv) {
+		if (s->bpi_ctx)
+			bpi_cipher_ctx_free(s->bpi_ctx);
+		memset(s, 0, qat_sym_session_get_private_size(dev));
+		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+
+		set_sym_session_private_data(sess, index, NULL);
+		rte_mempool_put(sess_mp, sess_priv);
+	}
+}
+
+static int
+qat_get_cmd_id(const struct rte_crypto_sym_xform *xform)
+{
+	/* Cipher Only */
+	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL)
+		return ICP_QAT_FW_LA_CMD_CIPHER;
+
+	/* Authentication Only */
+	if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL)
+		return ICP_QAT_FW_LA_CMD_AUTH;
+
+	/* AEAD */
+	if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
+		/* AES-GCM and AES-CCM works with different direction
+		 * GCM first encrypts and generate hash where AES-CCM
+		 * first generate hash and encrypts. Similar relation
+		 * applies to decryption.
+		 */
+		if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
+			if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
+				return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
+			else
+				return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
+		else
+			if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
+				return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
+			else
+				return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
+	}
+
+	if (xform->next == NULL)
+		return -1;
+
+	/* Cipher then Authenticate */
+	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
+			xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
+		return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
+
+	/* Authenticate then Cipher */
+	if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
+			xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
+		return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
+
+	return -1;
+}
+
+static struct rte_crypto_auth_xform *
+qat_get_auth_xform(struct rte_crypto_sym_xform *xform)
+{
+	do {
+		if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
+			return &xform->auth;
+
+		xform = xform->next;
+	} while (xform);
+
+	return NULL;
+}
+
+static struct rte_crypto_cipher_xform *
+qat_get_cipher_xform(struct rte_crypto_sym_xform *xform)
+{
+	do {
+		if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
+			return &xform->cipher;
+
+		xform = xform->next;
+	} while (xform);
+
+	return NULL;
+}
+
+int
+qat_sym_session_configure_cipher(struct rte_cryptodev *dev,
+		struct rte_crypto_sym_xform *xform,
+		struct qat_sym_session *session)
+{
+	struct qat_sym_dev_private *internals = dev->data->dev_private;
+	struct rte_crypto_cipher_xform *cipher_xform = NULL;
+	int ret;
+
+	/* Get cipher xform from crypto xform chain */
+	cipher_xform = qat_get_cipher_xform(xform);
+
+	session->cipher_iv.offset = cipher_xform->iv.offset;
+	session->cipher_iv.length = cipher_xform->iv.length;
+
+	switch (cipher_xform->algo) {
+	case RTE_CRYPTO_CIPHER_AES_CBC:
+		if (qat_sym_validate_aes_key(cipher_xform->key.length,
+				&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid AES cipher key size");
+			ret = -EINVAL;
+			goto error_out;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
+		break;
+	case RTE_CRYPTO_CIPHER_AES_CTR:
+		if (qat_sym_validate_aes_key(cipher_xform->key.length,
+				&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid AES cipher key size");
+			ret = -EINVAL;
+			goto error_out;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
+		break;
+	case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
+		if (qat_sym_validate_snow3g_key(cipher_xform->key.length,
+					&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid SNOW 3G cipher key size");
+			ret = -EINVAL;
+			goto error_out;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
+		break;
+	case RTE_CRYPTO_CIPHER_NULL:
+		session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
+		break;
+	case RTE_CRYPTO_CIPHER_KASUMI_F8:
+		if (qat_sym_validate_kasumi_key(cipher_xform->key.length,
+					&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid KASUMI cipher key size");
+			ret = -EINVAL;
+			goto error_out;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE;
+		break;
+	case RTE_CRYPTO_CIPHER_3DES_CBC:
+		if (qat_sym_validate_3des_key(cipher_xform->key.length,
+				&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid 3DES cipher key size");
+			ret = -EINVAL;
+			goto error_out;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
+		break;
+	case RTE_CRYPTO_CIPHER_DES_CBC:
+		if (qat_sym_validate_des_key(cipher_xform->key.length,
+				&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid DES cipher key size");
+			ret = -EINVAL;
+			goto error_out;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
+		break;
+	case RTE_CRYPTO_CIPHER_3DES_CTR:
+		if (qat_sym_validate_3des_key(cipher_xform->key.length,
+				&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid 3DES cipher key size");
+			ret = -EINVAL;
+			goto error_out;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
+		break;
+	case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
+		ret = bpi_cipher_ctx_init(
+					cipher_xform->algo,
+					cipher_xform->op,
+					cipher_xform->key.data,
+					&session->bpi_ctx);
+		if (ret != 0) {
+			QAT_LOG(ERR, "failed to create DES BPI ctx");
+			goto error_out;
+		}
+		if (qat_sym_validate_des_key(cipher_xform->key.length,
+				&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid DES cipher key size");
+			ret = -EINVAL;
+			goto error_out;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
+		break;
+	case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
+		ret = bpi_cipher_ctx_init(
+					cipher_xform->algo,
+					cipher_xform->op,
+					cipher_xform->key.data,
+					&session->bpi_ctx);
+		if (ret != 0) {
+			QAT_LOG(ERR, "failed to create AES BPI ctx");
+			goto error_out;
+		}
+		if (qat_sym_validate_aes_docsisbpi_key(cipher_xform->key.length,
+				&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid AES DOCSISBPI key size");
+			ret = -EINVAL;
+			goto error_out;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
+		break;
+	case RTE_CRYPTO_CIPHER_ZUC_EEA3:
+		if (!qat_is_cipher_alg_supported(
+			cipher_xform->algo, internals)) {
+			QAT_LOG(ERR, "%s not supported on this device",
+				rte_crypto_cipher_algorithm_strings
+					[cipher_xform->algo]);
+			ret = -ENOTSUP;
+			goto error_out;
+		}
+		if (qat_sym_validate_zuc_key(cipher_xform->key.length,
+				&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid ZUC cipher key size");
+			ret = -EINVAL;
+			goto error_out;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
+		break;
+	case RTE_CRYPTO_CIPHER_3DES_ECB:
+	case RTE_CRYPTO_CIPHER_AES_ECB:
+	case RTE_CRYPTO_CIPHER_AES_F8:
+	case RTE_CRYPTO_CIPHER_AES_XTS:
+	case RTE_CRYPTO_CIPHER_ARC4:
+		QAT_LOG(ERR, "Crypto QAT PMD: Unsupported Cipher alg %u",
+				cipher_xform->algo);
+		ret = -ENOTSUP;
+		goto error_out;
+	default:
+		QAT_LOG(ERR, "Crypto: Undefined Cipher specified %u\n",
+				cipher_xform->algo);
+		ret = -EINVAL;
+		goto error_out;
+	}
+
+	if (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+		session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
+	else
+		session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
+
+	if (qat_sym_session_aead_create_cd_cipher(session,
+						cipher_xform->key.data,
+						cipher_xform->key.length)) {
+		ret = -EINVAL;
+		goto error_out;
+	}
+
+	return 0;
+
+error_out:
+	if (session->bpi_ctx) {
+		bpi_cipher_ctx_free(session->bpi_ctx);
+		session->bpi_ctx = NULL;
+	}
+	return ret;
+}
 
-#include "qat_algs.h"
+int
+qat_sym_session_configure(struct rte_cryptodev *dev,
+		struct rte_crypto_sym_xform *xform,
+		struct rte_cryptodev_sym_session *sess,
+		struct rte_mempool *mempool)
+{
+	void *sess_private_data;
+	int ret;
+
+	if (rte_mempool_get(mempool, &sess_private_data)) {
+		CDEV_LOG_ERR(
+			"Couldn't get object from session mempool");
+		return -ENOMEM;
+	}
+
+	ret = qat_sym_session_set_parameters(dev, xform, sess_private_data);
+	if (ret != 0) {
+		QAT_LOG(ERR,
+		    "Crypto QAT PMD: failed to configure session parameters");
+
+		/* Return session to mempool */
+		rte_mempool_put(mempool, sess_private_data);
+		return ret;
+	}
+
+	set_sym_session_private_data(sess, dev->driver_id,
+		sess_private_data);
+
+	return 0;
+}
+
+int
+qat_sym_session_set_parameters(struct rte_cryptodev *dev,
+		struct rte_crypto_sym_xform *xform, void *session_private)
+{
+	struct qat_sym_session *session = session_private;
+	int ret;
+	int qat_cmd_id;
+
+	/* Set context descriptor physical address */
+	session->cd_paddr = rte_mempool_virt2iova(session) +
+			offsetof(struct qat_sym_session, cd);
+
+	session->min_qat_dev_gen = QAT_GEN1;
+
+	/* Get requested QAT command id */
+	qat_cmd_id = qat_get_cmd_id(xform);
+	if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) {
+		QAT_LOG(ERR, "Unsupported xform chain requested");
+		return -ENOTSUP;
+	}
+	session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
+	switch (session->qat_cmd) {
+	case ICP_QAT_FW_LA_CMD_CIPHER:
+		ret = qat_sym_session_configure_cipher(dev, xform, session);
+		if (ret < 0)
+			return ret;
+		break;
+	case ICP_QAT_FW_LA_CMD_AUTH:
+		ret = qat_sym_session_configure_auth(dev, xform, session);
+		if (ret < 0)
+			return ret;
+		break;
+	case ICP_QAT_FW_LA_CMD_CIPHER_HASH:
+		if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
+			ret = qat_sym_session_configure_aead(xform,
+					session);
+			if (ret < 0)
+				return ret;
+		} else {
+			ret = qat_sym_session_configure_cipher(dev,
+					xform, session);
+			if (ret < 0)
+				return ret;
+			ret = qat_sym_session_configure_auth(dev,
+					xform, session);
+			if (ret < 0)
+				return ret;
+		}
+		break;
+	case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
+		if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
+			ret = qat_sym_session_configure_aead(xform,
+					session);
+			if (ret < 0)
+				return ret;
+		} else {
+			ret = qat_sym_session_configure_auth(dev,
+					xform, session);
+			if (ret < 0)
+				return ret;
+			ret = qat_sym_session_configure_cipher(dev,
+					xform, session);
+			if (ret < 0)
+				return ret;
+		}
+		break;
+	case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
+	case ICP_QAT_FW_LA_CMD_TRNG_TEST:
+	case ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE:
+	case ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE:
+	case ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE:
+	case ICP_QAT_FW_LA_CMD_MGF1:
+	case ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP:
+	case ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP:
+	case ICP_QAT_FW_LA_CMD_DELIMITER:
+	QAT_LOG(ERR, "Unsupported Service %u",
+		session->qat_cmd);
+		return -ENOTSUP;
+	default:
+	QAT_LOG(ERR, "Unsupported Service %u",
+		session->qat_cmd);
+		return -ENOTSUP;
+	}
+
+	return 0;
+}
+
+int
+qat_sym_session_configure_auth(struct rte_cryptodev *dev,
+				struct rte_crypto_sym_xform *xform,
+				struct qat_sym_session *session)
+{
+	struct rte_crypto_auth_xform *auth_xform = qat_get_auth_xform(xform);
+	struct qat_sym_dev_private *internals = dev->data->dev_private;
+	uint8_t *key_data = auth_xform->key.data;
+	uint8_t key_length = auth_xform->key.length;
+
+	switch (auth_xform->algo) {
+	case RTE_CRYPTO_AUTH_SHA1_HMAC:
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
+		break;
+	case RTE_CRYPTO_AUTH_SHA224_HMAC:
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA224;
+		break;
+	case RTE_CRYPTO_AUTH_SHA256_HMAC:
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256;
+		break;
+	case RTE_CRYPTO_AUTH_SHA384_HMAC:
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA384;
+		break;
+	case RTE_CRYPTO_AUTH_SHA512_HMAC:
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512;
+		break;
+	case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
+		break;
+	case RTE_CRYPTO_AUTH_AES_GMAC:
+		if (qat_sym_validate_aes_key(auth_xform->key.length,
+				&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid AES key size");
+			return -EINVAL;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
+
+		break;
+	case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2;
+		break;
+	case RTE_CRYPTO_AUTH_MD5_HMAC:
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_MD5;
+		break;
+	case RTE_CRYPTO_AUTH_NULL:
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_NULL;
+		break;
+	case RTE_CRYPTO_AUTH_KASUMI_F9:
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_KASUMI_F9;
+		break;
+	case RTE_CRYPTO_AUTH_ZUC_EIA3:
+		if (!qat_is_auth_alg_supported(auth_xform->algo, internals)) {
+			QAT_LOG(ERR, "%s not supported on this device",
+				rte_crypto_auth_algorithm_strings
+				[auth_xform->algo]);
+			return -ENOTSUP;
+		}
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3;
+		break;
+	case RTE_CRYPTO_AUTH_SHA1:
+	case RTE_CRYPTO_AUTH_SHA256:
+	case RTE_CRYPTO_AUTH_SHA512:
+	case RTE_CRYPTO_AUTH_SHA224:
+	case RTE_CRYPTO_AUTH_SHA384:
+	case RTE_CRYPTO_AUTH_MD5:
+	case RTE_CRYPTO_AUTH_AES_CMAC:
+	case RTE_CRYPTO_AUTH_AES_CBC_MAC:
+		QAT_LOG(ERR, "Crypto: Unsupported hash alg %u",
+				auth_xform->algo);
+		return -ENOTSUP;
+	default:
+		QAT_LOG(ERR, "Crypto: Undefined Hash algo %u specified",
+				auth_xform->algo);
+		return -EINVAL;
+	}
+
+	session->auth_iv.offset = auth_xform->iv.offset;
+	session->auth_iv.length = auth_xform->iv.length;
+
+	if (auth_xform->algo == RTE_CRYPTO_AUTH_AES_GMAC) {
+		if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) {
+			session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
+			session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
+			/*
+			 * It needs to create cipher desc content first,
+			 * then authentication
+			 */
+
+			if (qat_sym_session_aead_create_cd_cipher(session,
+						auth_xform->key.data,
+						auth_xform->key.length))
+				return -EINVAL;
+
+			if (qat_sym_session_aead_create_cd_auth(session,
+						key_data,
+						key_length,
+						0,
+						auth_xform->digest_length,
+						auth_xform->op))
+				return -EINVAL;
+		} else {
+			session->qat_cmd = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
+			session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
+			/*
+			 * It needs to create authentication desc content first,
+			 * then cipher
+			 */
+
+			if (qat_sym_session_aead_create_cd_auth(session,
+					key_data,
+					key_length,
+					0,
+					auth_xform->digest_length,
+					auth_xform->op))
+				return -EINVAL;
+
+			if (qat_sym_session_aead_create_cd_cipher(session,
+						auth_xform->key.data,
+						auth_xform->key.length))
+				return -EINVAL;
+		}
+		/* Restore to authentication only only */
+		session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH;
+	} else {
+		if (qat_sym_session_aead_create_cd_auth(session,
+				key_data,
+				key_length,
+				0,
+				auth_xform->digest_length,
+				auth_xform->op))
+			return -EINVAL;
+	}
+
+	session->digest_length = auth_xform->digest_length;
+	return 0;
+}
+
+int
+qat_sym_session_configure_aead(struct rte_crypto_sym_xform *xform,
+				struct qat_sym_session *session)
+{
+	struct rte_crypto_aead_xform *aead_xform = &xform->aead;
+	enum rte_crypto_auth_operation crypto_operation;
+
+	/*
+	 * Store AEAD IV parameters as cipher IV,
+	 * to avoid unnecessary memory usage
+	 */
+	session->cipher_iv.offset = xform->aead.iv.offset;
+	session->cipher_iv.length = xform->aead.iv.length;
+
+	switch (aead_xform->algo) {
+	case RTE_CRYPTO_AEAD_AES_GCM:
+		if (qat_sym_validate_aes_key(aead_xform->key.length,
+				&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid AES key size");
+			return -EINVAL;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
+		break;
+	case RTE_CRYPTO_AEAD_AES_CCM:
+		if (qat_sym_validate_aes_key(aead_xform->key.length,
+				&session->qat_cipher_alg) != 0) {
+			QAT_LOG(ERR, "Invalid AES key size");
+			return -EINVAL;
+		}
+		session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
+		session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC;
+		break;
+	default:
+		QAT_LOG(ERR, "Crypto: Undefined AEAD specified %u\n",
+				aead_xform->algo);
+		return -EINVAL;
+	}
+
+	if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT &&
+			aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) ||
+			(aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT &&
+			aead_xform->algo == RTE_CRYPTO_AEAD_AES_CCM)) {
+		session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
+		/*
+		 * It needs to create cipher desc content first,
+		 * then authentication
+		 */
+		crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
+			RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY;
+
+		if (qat_sym_session_aead_create_cd_cipher(session,
+					aead_xform->key.data,
+					aead_xform->key.length))
+			return -EINVAL;
+
+		if (qat_sym_session_aead_create_cd_auth(session,
+					aead_xform->key.data,
+					aead_xform->key.length,
+					aead_xform->aad_length,
+					aead_xform->digest_length,
+					crypto_operation))
+			return -EINVAL;
+	} else {
+		session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
+		/*
+		 * It needs to create authentication desc content first,
+		 * then cipher
+		 */
+
+		crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
+			RTE_CRYPTO_AUTH_OP_VERIFY : RTE_CRYPTO_AUTH_OP_GENERATE;
+
+		if (qat_sym_session_aead_create_cd_auth(session,
+					aead_xform->key.data,
+					aead_xform->key.length,
+					aead_xform->aad_length,
+					aead_xform->digest_length,
+					crypto_operation))
+			return -EINVAL;
+
+		if (qat_sym_session_aead_create_cd_cipher(session,
+					aead_xform->key.data,
+					aead_xform->key.length))
+			return -EINVAL;
+	}
+
+	session->digest_length = aead_xform->digest_length;
+	return 0;
+}
+
+unsigned int qat_sym_session_get_private_size(
+		struct rte_cryptodev *dev __rte_unused)
+{
+	return RTE_ALIGN_CEIL(sizeof(struct qat_sym_session), 8);
+}
 
 /* returns block size in bytes per cipher algo */
 int qat_cipher_get_block_size(enum icp_qat_hw_cipher_algo qat_cipher_alg)
@@ -30,7 +737,7 @@ int qat_cipher_get_block_size(enum icp_qat_hw_cipher_algo qat_cipher_alg)
 	case ICP_QAT_HW_CIPHER_ALGO_AES256:
 		return ICP_QAT_HW_AES_BLK_SZ;
 	default:
-		PMD_DRV_LOG(ERR, "invalid block cipher alg %u", qat_cipher_alg);
+		QAT_LOG(ERR, "invalid block cipher alg %u", qat_cipher_alg);
 		return -EFAULT;
 	};
 	return -EFAULT;
@@ -77,18 +784,18 @@ static int qat_hash_get_state1_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 	case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
 		return QAT_HW_ROUND_UP(ICP_QAT_HW_KASUMI_F9_STATE1_SZ,
 						QAT_HW_DEFAULT_ALIGNMENT);
-	case ICP_QAT_HW_AUTH_ALGO_NULL:
-		return QAT_HW_ROUND_UP(ICP_QAT_HW_NULL_STATE1_SZ,
-						QAT_HW_DEFAULT_ALIGNMENT);
 	case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
 		return QAT_HW_ROUND_UP(ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ,
 						QAT_HW_DEFAULT_ALIGNMENT);
+	case ICP_QAT_HW_AUTH_ALGO_NULL:
+		return QAT_HW_ROUND_UP(ICP_QAT_HW_NULL_STATE1_SZ,
+						QAT_HW_DEFAULT_ALIGNMENT);
 	case ICP_QAT_HW_AUTH_ALGO_DELIMITER:
 		/* return maximum state1 size in this case */
 		return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA512_STATE1_SZ,
 						QAT_HW_DEFAULT_ALIGNMENT);
 	default:
-		PMD_DRV_LOG(ERR, "invalid hash alg %u", qat_hash_alg);
+		QAT_LOG(ERR, "invalid hash alg %u", qat_hash_alg);
 		return -EFAULT;
 	};
 	return -EFAULT;
@@ -114,7 +821,7 @@ static int qat_hash_get_digest_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 		/* return maximum digest size in this case */
 		return ICP_QAT_HW_SHA512_STATE1_SZ;
 	default:
-		PMD_DRV_LOG(ERR, "invalid hash alg %u", qat_hash_alg);
+		QAT_LOG(ERR, "invalid hash alg %u", qat_hash_alg);
 		return -EFAULT;
 	};
 	return -EFAULT;
@@ -142,7 +849,7 @@ static int qat_hash_get_block_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 		/* return maximum block size in this case */
 		return SHA512_CBLOCK;
 	default:
-		PMD_DRV_LOG(ERR, "invalid hash alg %u", qat_hash_alg);
+		QAT_LOG(ERR, "invalid hash alg %u", qat_hash_alg);
 		return -EFAULT;
 	};
 	return -EFAULT;
@@ -226,7 +933,6 @@ static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg,
 	uint64_t *hash_state_out_be64;
 	int i;
 
-	PMD_INIT_FUNC_TRACE();
 	digest_size = qat_hash_get_digest_size(hash_alg);
 	if (digest_size <= 0)
 		return -EFAULT;
@@ -275,7 +981,7 @@ static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg,
 			return -EFAULT;
 		break;
 	default:
-		PMD_DRV_LOG(ERR, "invalid hash alg %u", hash_alg);
+		QAT_LOG(ERR, "invalid hash alg %u", hash_alg);
 		return -EFAULT;
 	}
 
@@ -285,7 +991,7 @@ static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg,
 #define HMAC_OPAD_VALUE	0x5c
 #define HASH_XCBC_PRECOMP_KEY_NUM 3
 
-static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
+static int qat_sym_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 				const uint8_t *auth_key,
 				uint16_t auth_keylen,
 				uint8_t *p_state_buf,
@@ -296,7 +1002,6 @@ static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 	uint8_t opad[qat_hash_get_block_size(ICP_QAT_HW_AUTH_ALGO_DELIMITER)];
 	int i;
 
-	PMD_INIT_FUNC_TRACE();
 	if (hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC) {
 		static uint8_t qat_aes_xcbc_key_seed[
 					ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ] = {
@@ -316,7 +1021,7 @@ static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 		in = rte_zmalloc("working mem for key",
 				ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ, 16);
 		if (in == NULL) {
-			PMD_DRV_LOG(ERR, "Failed to alloc memory");
+			QAT_LOG(ERR, "Failed to alloc memory");
 			return -ENOMEM;
 		}
 
@@ -351,7 +1056,7 @@ static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 		in = rte_zmalloc("working mem for key",
 				ICP_QAT_HW_GALOIS_H_SZ, 16);
 		if (in == NULL) {
-			PMD_DRV_LOG(ERR, "Failed to alloc memory");
+			QAT_LOG(ERR, "Failed to alloc memory");
 			return -ENOMEM;
 		}
 
@@ -376,7 +1081,7 @@ static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 	memset(opad, 0, block_size);
 
 	if (auth_keylen > (unsigned int)block_size) {
-		PMD_DRV_LOG(ERR, "invalid keylen %u", auth_keylen);
+		QAT_LOG(ERR, "invalid keylen %u", auth_keylen);
 		return -EFAULT;
 	}
 	rte_memcpy(ipad, auth_key, auth_keylen);
@@ -393,7 +1098,7 @@ static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 	if (partial_hash_compute(hash_alg, ipad, p_state_buf)) {
 		memset(ipad, 0, block_size);
 		memset(opad, 0, block_size);
-		PMD_DRV_LOG(ERR, "ipad precompute failed");
+		QAT_LOG(ERR, "ipad precompute failed");
 		return -EFAULT;
 	}
 
@@ -405,7 +1110,7 @@ static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 	if (partial_hash_compute(hash_alg, opad, p_state_buf + *p_state_len)) {
 		memset(ipad, 0, block_size);
 		memset(opad, 0, block_size);
-		PMD_DRV_LOG(ERR, "opad precompute failed");
+		QAT_LOG(ERR, "opad precompute failed");
 		return -EFAULT;
 	}
 
@@ -415,10 +1120,10 @@ static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 	return 0;
 }
 
-void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header,
-		enum qat_crypto_proto_flag proto_flags)
+static void
+qat_sym_session_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header,
+		enum qat_sym_proto_flag proto_flags)
 {
-	PMD_INIT_FUNC_TRACE();
 	header->hdr_flags =
 		ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
 	header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
@@ -464,11 +1169,11 @@ void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header,
  *	and set its protocol flag in both cipher and auth part of content
  *	descriptor building function
  */
-static enum qat_crypto_proto_flag
+static enum qat_sym_proto_flag
 qat_get_crypto_proto_flag(uint16_t flags)
 {
 	int proto = ICP_QAT_FW_LA_PROTO_GET(flags);
-	enum qat_crypto_proto_flag qat_proto_flag =
+	enum qat_sym_proto_flag qat_proto_flag =
 			QAT_CRYPTO_PROTO_FLAG_NONE;
 
 	switch (proto) {
@@ -483,7 +1188,7 @@ qat_get_crypto_proto_flag(uint16_t flags)
 	return qat_proto_flag;
 }
 
-int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc,
+int qat_sym_session_aead_create_cd_cipher(struct qat_sym_session *cdesc,
 						uint8_t *cipherkey,
 						uint32_t cipherkeylen)
 {
@@ -495,13 +1200,12 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc,
 	struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
 	struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
 	enum icp_qat_hw_cipher_convert key_convert;
-	enum qat_crypto_proto_flag qat_proto_flag =
+	enum qat_sym_proto_flag qat_proto_flag =
 		QAT_CRYPTO_PROTO_FLAG_NONE;
 	uint32_t total_key_size;
 	uint16_t cipher_offset, cd_size;
 	uint32_t wordIndex  = 0;
 	uint32_t *temp_key = NULL;
-	PMD_INIT_FUNC_TRACE();
 
 	if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
 		cd_pars->u.s.content_desc_addr = cdesc->cd_paddr;
@@ -526,7 +1230,7 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc,
 					ICP_QAT_FW_SLICE_DRAM_WR);
 		cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd;
 	} else if (cdesc->qat_cmd != ICP_QAT_FW_LA_CMD_HASH_CIPHER) {
-		PMD_DRV_LOG(ERR, "Invalid param, must be a cipher command.");
+		QAT_LOG(ERR, "Invalid param, must be a cipher command.");
 		return -EFAULT;
 	}
 
@@ -587,7 +1291,7 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc,
 	cipher_cd_ctrl->cipher_cfg_offset = cipher_offset >> 3;
 
 	header->service_cmd_id = cdesc->qat_cmd;
-	qat_alg_init_common_hdr(header, qat_proto_flag);
+	qat_sym_session_init_common_hdr(header, qat_proto_flag);
 
 	cipher = (struct icp_qat_hw_cipher_algo_blk *)cdesc->cd_cur_ptr;
 	cipher->cipher_config.val =
@@ -618,11 +1322,19 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc,
 	if (total_key_size > cipherkeylen) {
 		uint32_t padding_size =  total_key_size-cipherkeylen;
 		if ((cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES)
-			&& (cipherkeylen == QAT_3DES_KEY_SZ_OPT2))
+			&& (cipherkeylen == QAT_3DES_KEY_SZ_OPT2)) {
 			/* K3 not provided so use K1 = K3*/
 			memcpy(cdesc->cd_cur_ptr, cipherkey, padding_size);
-		else
+		} else if ((cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES)
+			&& (cipherkeylen == QAT_3DES_KEY_SZ_OPT3)) {
+			/* K2 and K3 not provided so use K1 = K2 = K3*/
+			memcpy(cdesc->cd_cur_ptr, cipherkey,
+				cipherkeylen);
+			memcpy(cdesc->cd_cur_ptr+cipherkeylen,
+				cipherkey, cipherkeylen);
+		} else
 			memset(cdesc->cd_cur_ptr, 0, padding_size);
+
 		cdesc->cd_cur_ptr += padding_size;
 	}
 	cd_size = cdesc->cd_cur_ptr-(uint8_t *)&cdesc->cd;
@@ -631,7 +1343,7 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc,
 	return 0;
 }
 
-int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc,
+int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
 						uint8_t *authkey,
 						uint32_t authkeylen,
 						uint32_t aad_length,
@@ -655,11 +1367,9 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc,
 	uint32_t *aad_len = NULL;
 	uint32_t wordIndex  = 0;
 	uint32_t *pTempKey;
-	enum qat_crypto_proto_flag qat_proto_flag =
+	enum qat_sym_proto_flag qat_proto_flag =
 		QAT_CRYPTO_PROTO_FLAG_NONE;
 
-	PMD_INIT_FUNC_TRACE();
-
 	if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) {
 		ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl,
 					ICP_QAT_FW_SLICE_AUTH);
@@ -677,7 +1387,7 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc,
 				ICP_QAT_FW_SLICE_DRAM_WR);
 		cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd;
 	} else if (cdesc->qat_cmd != ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
-		PMD_DRV_LOG(ERR, "Invalid param, must be a hash command.");
+		QAT_LOG(ERR, "Invalid param, must be a hash command.");
 		return -EFAULT;
 	}
 
@@ -720,51 +1430,51 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc,
 	 */
 	switch (cdesc->qat_hash_alg) {
 	case ICP_QAT_HW_AUTH_ALGO_SHA1:
-		if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA1,
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA1,
 			authkey, authkeylen, cdesc->cd_cur_ptr,	&state1_size)) {
-			PMD_DRV_LOG(ERR, "(SHA)precompute failed");
+			QAT_LOG(ERR, "(SHA)precompute failed");
 			return -EFAULT;
 		}
 		state2_size = RTE_ALIGN_CEIL(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_SHA224:
-		if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA224,
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA224,
 			authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) {
-			PMD_DRV_LOG(ERR, "(SHA)precompute failed");
+			QAT_LOG(ERR, "(SHA)precompute failed");
 			return -EFAULT;
 		}
 		state2_size = ICP_QAT_HW_SHA224_STATE2_SZ;
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_SHA256:
-		if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA256,
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA256,
 			authkey, authkeylen, cdesc->cd_cur_ptr,	&state1_size)) {
-			PMD_DRV_LOG(ERR, "(SHA)precompute failed");
+			QAT_LOG(ERR, "(SHA)precompute failed");
 			return -EFAULT;
 		}
 		state2_size = ICP_QAT_HW_SHA256_STATE2_SZ;
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_SHA384:
-		if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA384,
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA384,
 			authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) {
-			PMD_DRV_LOG(ERR, "(SHA)precompute failed");
+			QAT_LOG(ERR, "(SHA)precompute failed");
 			return -EFAULT;
 		}
 		state2_size = ICP_QAT_HW_SHA384_STATE2_SZ;
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_SHA512:
-		if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA512,
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA512,
 			authkey, authkeylen, cdesc->cd_cur_ptr,	&state1_size)) {
-			PMD_DRV_LOG(ERR, "(SHA)precompute failed");
+			QAT_LOG(ERR, "(SHA)precompute failed");
 			return -EFAULT;
 		}
 		state2_size = ICP_QAT_HW_SHA512_STATE2_SZ;
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC:
 		state1_size = ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ;
-		if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC,
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC,
 			authkey, authkeylen, cdesc->cd_cur_ptr + state1_size,
 			&state2_size)) {
-			PMD_DRV_LOG(ERR, "(XCBC)precompute failed");
+			QAT_LOG(ERR, "(XCBC)precompute failed");
 			return -EFAULT;
 		}
 		break;
@@ -772,10 +1482,10 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc,
 	case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
 		qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM;
 		state1_size = ICP_QAT_HW_GALOIS_128_STATE1_SZ;
-		if (qat_alg_do_precomputes(cdesc->qat_hash_alg,
+		if (qat_sym_do_precomputes(cdesc->qat_hash_alg,
 			authkey, authkeylen, cdesc->cd_cur_ptr + state1_size,
 			&state2_size)) {
-			PMD_DRV_LOG(ERR, "(GCM)precompute failed");
+			QAT_LOG(ERR, "(GCM)precompute failed");
 			return -EFAULT;
 		}
 		/*
@@ -832,10 +1542,10 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc,
 
 		break;
 	case ICP_QAT_HW_AUTH_ALGO_MD5:
-		if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_MD5,
+		if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_MD5,
 			authkey, authkeylen, cdesc->cd_cur_ptr,
 			&state1_size)) {
-			PMD_DRV_LOG(ERR, "(MD5)precompute failed");
+			QAT_LOG(ERR, "(MD5)precompute failed");
 			return -EFAULT;
 		}
 		state2_size = ICP_QAT_HW_MD5_STATE2_SZ;
@@ -854,14 +1564,13 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc,
 
 		if (aad_length > 0) {
 			aad_length += ICP_QAT_HW_CCM_AAD_B0_LEN +
-				ICP_QAT_HW_CCM_AAD_LEN_INFO;
+			ICP_QAT_HW_CCM_AAD_LEN_INFO;
 			auth_param->u2.aad_sz =
-					RTE_ALIGN_CEIL(aad_length,
-					ICP_QAT_HW_CCM_AAD_ALIGNMENT);
+			RTE_ALIGN_CEIL(aad_length,
+			ICP_QAT_HW_CCM_AAD_ALIGNMENT);
 		} else {
 			auth_param->u2.aad_sz = ICP_QAT_HW_CCM_AAD_B0_LEN;
 		}
-
 		cdesc->aad_len = aad_length;
 		hash->auth_counter.counter = 0;
 
@@ -891,12 +1600,12 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc,
 			pTempKey[wordIndex] ^= KASUMI_F9_KEY_MODIFIER_4_BYTES;
 		break;
 	default:
-		PMD_DRV_LOG(ERR, "Invalid HASH alg %u", cdesc->qat_hash_alg);
+		QAT_LOG(ERR, "Invalid HASH alg %u", cdesc->qat_hash_alg);
 		return -EFAULT;
 	}
 
 	/* Request template setup */
-	qat_alg_init_common_hdr(header, qat_proto_flag);
+	qat_sym_session_init_common_hdr(header, qat_proto_flag);
 	header->service_cmd_id = cdesc->qat_cmd;
 
 	/* Auth CD config setup */
@@ -922,7 +1631,7 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc,
 	return 0;
 }
 
-int qat_alg_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
+int qat_sym_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
 {
 	switch (key_len) {
 	case ICP_QAT_HW_AES_128_KEY_SZ:
@@ -940,7 +1649,7 @@ int qat_alg_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
 	return 0;
 }
 
-int qat_alg_validate_aes_docsisbpi_key(int key_len,
+int qat_sym_validate_aes_docsisbpi_key(int key_len,
 		enum icp_qat_hw_cipher_algo *alg)
 {
 	switch (key_len) {
@@ -953,7 +1662,7 @@ int qat_alg_validate_aes_docsisbpi_key(int key_len,
 	return 0;
 }
 
-int qat_alg_validate_snow3g_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
+int qat_sym_validate_snow3g_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
 {
 	switch (key_len) {
 	case ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ:
@@ -965,7 +1674,7 @@ int qat_alg_validate_snow3g_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
 	return 0;
 }
 
-int qat_alg_validate_kasumi_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
+int qat_sym_validate_kasumi_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
 {
 	switch (key_len) {
 	case ICP_QAT_HW_KASUMI_KEY_SZ:
@@ -977,7 +1686,7 @@ int qat_alg_validate_kasumi_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
 	return 0;
 }
 
-int qat_alg_validate_des_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
+int qat_sym_validate_des_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
 {
 	switch (key_len) {
 	case ICP_QAT_HW_DES_KEY_SZ:
@@ -989,11 +1698,12 @@ int qat_alg_validate_des_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
 	return 0;
 }
 
-int qat_alg_validate_3des_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
+int qat_sym_validate_3des_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
 {
 	switch (key_len) {
 	case QAT_3DES_KEY_SZ_OPT1:
 	case QAT_3DES_KEY_SZ_OPT2:
+	case QAT_3DES_KEY_SZ_OPT3:
 		*alg = ICP_QAT_HW_CIPHER_ALGO_3DES;
 		break;
 	default:
@@ -1002,7 +1712,7 @@ int qat_alg_validate_3des_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
 	return 0;
 }
 
-int qat_alg_validate_zuc_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
+int qat_sym_validate_zuc_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
 {
 	switch (key_len) {
 	case ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ:
diff --git a/drivers/crypto/qat/qat_sym_session.h b/drivers/crypto/qat/qat_sym_session.h
new file mode 100644
index 00000000..e8f51e5b
--- /dev/null
+++ b/drivers/crypto/qat/qat_sym_session.h
@@ -0,0 +1,145 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2018 Intel Corporation
+ */
+#ifndef _QAT_SYM_SESSION_H_
+#define _QAT_SYM_SESSION_H_
+
+#include <rte_crypto.h>
+#include <rte_cryptodev_pmd.h>
+
+#include "qat_common.h"
+#include "icp_qat_hw.h"
+#include "icp_qat_fw.h"
+#include "icp_qat_fw_la.h"
+
+/*
+ * Key Modifier (KM) value used in KASUMI algorithm in F9 mode to XOR
+ * Integrity Key (IK)
+ */
+#define KASUMI_F9_KEY_MODIFIER_4_BYTES   0xAAAAAAAA
+
+#define KASUMI_F8_KEY_MODIFIER_4_BYTES   0x55555555
+
+/* 3DES key sizes */
+#define QAT_3DES_KEY_SZ_OPT1 24 /* Keys are independent */
+#define QAT_3DES_KEY_SZ_OPT2 16 /* K3=K1 */
+#define QAT_3DES_KEY_SZ_OPT3 8 /* K1=K2=K3 */
+
+
+#define QAT_AES_HW_CONFIG_CBC_ENC(alg) \
+	ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
+					ICP_QAT_HW_CIPHER_NO_CONVERT, \
+					ICP_QAT_HW_CIPHER_ENCRYPT)
+
+#define QAT_AES_HW_CONFIG_CBC_DEC(alg) \
+	ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
+					ICP_QAT_HW_CIPHER_KEY_CONVERT, \
+					ICP_QAT_HW_CIPHER_DECRYPT)
+
+enum qat_sym_proto_flag {
+	QAT_CRYPTO_PROTO_FLAG_NONE = 0,
+	QAT_CRYPTO_PROTO_FLAG_CCM = 1,
+	QAT_CRYPTO_PROTO_FLAG_GCM = 2,
+	QAT_CRYPTO_PROTO_FLAG_SNOW3G = 3,
+	QAT_CRYPTO_PROTO_FLAG_ZUC = 4
+};
+
+/* Common content descriptor */
+struct qat_sym_cd {
+	struct icp_qat_hw_cipher_algo_blk cipher;
+	struct icp_qat_hw_auth_algo_blk hash;
+} __rte_packed __rte_cache_aligned;
+
+struct qat_sym_session {
+	enum icp_qat_fw_la_cmd_id qat_cmd;
+	enum icp_qat_hw_cipher_algo qat_cipher_alg;
+	enum icp_qat_hw_cipher_dir qat_dir;
+	enum icp_qat_hw_cipher_mode qat_mode;
+	enum icp_qat_hw_auth_algo qat_hash_alg;
+	enum icp_qat_hw_auth_op auth_op;
+	void *bpi_ctx;
+	struct qat_sym_cd cd;
+	uint8_t *cd_cur_ptr;
+	phys_addr_t cd_paddr;
+	struct icp_qat_fw_la_bulk_req fw_req;
+	uint8_t aad_len;
+	struct qat_crypto_instance *inst;
+	struct {
+		uint16_t offset;
+		uint16_t length;
+	} cipher_iv;
+	struct {
+		uint16_t offset;
+		uint16_t length;
+	} auth_iv;
+	uint16_t digest_length;
+	rte_spinlock_t lock;	/* protects this struct */
+	enum qat_device_gen min_qat_dev_gen;
+};
+
+int
+qat_sym_session_configure(struct rte_cryptodev *dev,
+		struct rte_crypto_sym_xform *xform,
+		struct rte_cryptodev_sym_session *sess,
+		struct rte_mempool *mempool);
+
+int
+qat_sym_session_set_parameters(struct rte_cryptodev *dev,
+		struct rte_crypto_sym_xform *xform, void *session_private);
+
+int
+qat_sym_session_configure_aead(struct rte_crypto_sym_xform *xform,
+				struct qat_sym_session *session);
+
+int
+qat_sym_session_configure_cipher(struct rte_cryptodev *dev,
+		struct rte_crypto_sym_xform *xform,
+		struct qat_sym_session *session);
+
+int
+qat_sym_session_configure_auth(struct rte_cryptodev *dev,
+				struct rte_crypto_sym_xform *xform,
+				struct qat_sym_session *session);
+
+int
+qat_sym_session_aead_create_cd_cipher(struct qat_sym_session *cd,
+						uint8_t *enckey,
+						uint32_t enckeylen);
+
+int
+qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
+						uint8_t *authkey,
+						uint32_t authkeylen,
+						uint32_t aad_length,
+						uint32_t digestsize,
+						unsigned int operation);
+
+void
+qat_sym_session_clear(struct rte_cryptodev *dev,
+		struct rte_cryptodev_sym_session *session);
+
+unsigned int
+qat_sym_session_get_private_size(struct rte_cryptodev *dev);
+
+void
+qat_sym_sesssion_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header,
+					enum qat_sym_proto_flag proto_flags);
+int
+qat_sym_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
+int
+qat_sym_validate_aes_docsisbpi_key(int key_len,
+					enum icp_qat_hw_cipher_algo *alg);
+int
+qat_sym_validate_snow3g_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
+int
+qat_sym_validate_kasumi_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
+int
+qat_sym_validate_3des_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
+int
+qat_sym_validate_des_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
+int
+qat_cipher_get_block_size(enum icp_qat_hw_cipher_algo qat_cipher_alg);
+int
+qat_sym_validate_zuc_key(int key_len, enum icp_qat_hw_cipher_algo *alg);
+
+#endif /* _QAT_SYM_SESSION_H_ */
diff --git a/drivers/crypto/qat/rte_pmd_qat_version.map b/drivers/crypto/qat/rte_pmd_qat_version.map
deleted file mode 100644
index bbaf1c85..00000000
--- a/drivers/crypto/qat/rte_pmd_qat_version.map
+++ /dev/null
@@ -1,3 +0,0 @@
-DPDK_2.2 {
-	local: *;
-};
\ No newline at end of file
diff --git a/drivers/crypto/qat/rte_qat_cryptodev.c b/drivers/crypto/qat/rte_qat_cryptodev.c
deleted file mode 100644
index c8da07af..00000000
--- a/drivers/crypto/qat/rte_qat_cryptodev.c
+++ /dev/null
@@ -1,180 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2018 Intel Corporation
- */
-
-#include <rte_bus_pci.h>
-#include <rte_common.h>
-#include <rte_dev.h>
-#include <rte_malloc.h>
-#include <rte_pci.h>
-#include <rte_cryptodev_pmd.h>
-
-#include "qat_crypto.h"
-#include "qat_logs.h"
-
-uint8_t cryptodev_qat_driver_id;
-
-static const struct rte_cryptodev_capabilities qat_gen1_capabilities[] = {
-	QAT_BASE_GEN1_SYM_CAPABILITIES,
-	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
-};
-
-static const struct rte_cryptodev_capabilities qat_gen2_capabilities[] = {
-	QAT_BASE_GEN1_SYM_CAPABILITIES,
-	QAT_EXTRA_GEN2_SYM_CAPABILITIES,
-	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
-};
-
-static struct rte_cryptodev_ops crypto_qat_ops = {
-
-		/* Device related operations */
-		.dev_configure		= qat_dev_config,
-		.dev_start		= qat_dev_start,
-		.dev_stop		= qat_dev_stop,
-		.dev_close		= qat_dev_close,
-		.dev_infos_get		= qat_dev_info_get,
-
-		.stats_get		= qat_crypto_sym_stats_get,
-		.stats_reset		= qat_crypto_sym_stats_reset,
-		.queue_pair_setup	= qat_crypto_sym_qp_setup,
-		.queue_pair_release	= qat_crypto_sym_qp_release,
-		.queue_pair_start	= NULL,
-		.queue_pair_stop	= NULL,
-		.queue_pair_count	= NULL,
-
-		/* Crypto related operations */
-		.session_get_size	= qat_crypto_sym_get_session_private_size,
-		.session_configure	= qat_crypto_sym_configure_session,
-		.session_clear		= qat_crypto_sym_clear_session
-};
-
-/*
- * The set of PCI devices this driver supports
- */
-
-static const struct rte_pci_id pci_id_qat_map[] = {
-		{
-			RTE_PCI_DEVICE(0x8086, 0x0443),
-		},
-		{
-			RTE_PCI_DEVICE(0x8086, 0x37c9),
-		},
-		{
-			RTE_PCI_DEVICE(0x8086, 0x19e3),
-		},
-		{
-			RTE_PCI_DEVICE(0x8086, 0x6f55),
-		},
-		{.device_id = 0},
-};
-
-static int
-crypto_qat_create(const char *name, struct rte_pci_device *pci_dev,
-		struct rte_cryptodev_pmd_init_params *init_params)
-{
-	struct rte_cryptodev *cryptodev;
-	struct qat_pmd_private *internals;
-
-	PMD_INIT_FUNC_TRACE();
-
-	cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device,
-			init_params);
-	if (cryptodev == NULL)
-		return -ENODEV;
-
-	cryptodev->driver_id = cryptodev_qat_driver_id;
-	cryptodev->dev_ops = &crypto_qat_ops;
-
-	cryptodev->enqueue_burst = qat_pmd_enqueue_op_burst;
-	cryptodev->dequeue_burst = qat_pmd_dequeue_op_burst;
-
-	cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
-			RTE_CRYPTODEV_FF_HW_ACCELERATED |
-			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
-			RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
-
-	internals = cryptodev->data->dev_private;
-	internals->max_nb_sessions = init_params->max_nb_sessions;
-	switch (pci_dev->id.device_id) {
-	case 0x0443:
-		internals->qat_dev_gen = QAT_GEN1;
-		internals->qat_dev_capabilities = qat_gen1_capabilities;
-		break;
-	case 0x37c9:
-	case 0x19e3:
-	case 0x6f55:
-		internals->qat_dev_gen = QAT_GEN2;
-		internals->qat_dev_capabilities = qat_gen2_capabilities;
-		break;
-	default:
-		PMD_DRV_LOG(ERR,
-			"Invalid dev_id, can't determine capabilities");
-		break;
-	}
-
-	/*
-	 * For secondary processes, we don't initialise any further as primary
-	 * has already done this work. Only check we don't need a different
-	 * RX function
-	 */
-	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
-		PMD_DRV_LOG(DEBUG, "Device already initialised by primary process");
-		return 0;
-	}
-
-	return 0;
-}
-
-static int crypto_qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
-		struct rte_pci_device *pci_dev)
-{
-	struct rte_cryptodev_pmd_init_params init_params = {
-		.name = "",
-		.socket_id = pci_dev->device.numa_node,
-		.private_data_size = sizeof(struct qat_pmd_private),
-		.max_nb_sessions = RTE_QAT_PMD_MAX_NB_SESSIONS
-	};
-	char name[RTE_CRYPTODEV_NAME_MAX_LEN];
-
-	PMD_DRV_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
-			pci_dev->addr.bus,
-			pci_dev->addr.devid,
-			pci_dev->addr.function);
-
-	rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
-
-	return crypto_qat_create(name, pci_dev, &init_params);
-}
-
-static int crypto_qat_pci_remove(struct rte_pci_device *pci_dev)
-{
-	struct rte_cryptodev *cryptodev;
-	char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
-
-	if (pci_dev == NULL)
-		return -EINVAL;
-
-	rte_pci_device_name(&pci_dev->addr, cryptodev_name,
-			sizeof(cryptodev_name));
-
-	cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name);
-	if (cryptodev == NULL)
-		return -ENODEV;
-
-	/* free crypto device */
-	return rte_cryptodev_pmd_destroy(cryptodev);
-}
-
-static struct rte_pci_driver rte_qat_pmd = {
-	.id_table = pci_id_qat_map,
-	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
-	.probe = crypto_qat_pci_probe,
-	.remove = crypto_qat_pci_remove
-};
-
-static struct cryptodev_driver qat_crypto_drv;
-
-RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_QAT_SYM_PMD, rte_qat_pmd);
-RTE_PMD_REGISTER_PCI_TABLE(CRYPTODEV_NAME_QAT_SYM_PMD, pci_id_qat_map);
-RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv, rte_qat_pmd.driver,
-		cryptodev_qat_driver_id);
diff --git a/drivers/crypto/scheduler/rte_cryptodev_scheduler.c b/drivers/crypto/scheduler/rte_cryptodev_scheduler.c
index ed574cc1..6e4919c4 100644
--- a/drivers/crypto/scheduler/rte_cryptodev_scheduler.c
+++ b/drivers/crypto/scheduler/rte_cryptodev_scheduler.c
@@ -9,6 +9,8 @@
 #include "rte_cryptodev_scheduler.h"
 #include "scheduler_pmd_private.h"
 
+int scheduler_logtype_driver;
+
 /** update the scheduler pmd's capability with attaching device's
  *  capability.
  *  For each device to be attached, the scheduler's capability should be
@@ -168,30 +170,30 @@ rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id)
 	uint32_t i;
 
 	if (!dev) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->driver_id != cryptodev_driver_id) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->data->dev_started) {
-		CS_LOG_ERR("Illegal operation");
+		CR_SCHED_LOG(ERR, "Illegal operation");
 		return -EBUSY;
 	}
 
 	sched_ctx = dev->data->dev_private;
 	if (sched_ctx->nb_slaves >=
 			RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
-		CS_LOG_ERR("Too many slaves attached");
+		CR_SCHED_LOG(ERR, "Too many slaves attached");
 		return -ENOMEM;
 	}
 
 	for (i = 0; i < sched_ctx->nb_slaves; i++)
 		if (sched_ctx->slaves[i].dev_id == slave_id) {
-			CS_LOG_ERR("Slave already added");
+			CR_SCHED_LOG(ERR, "Slave already added");
 			return -ENOTSUP;
 		}
 
@@ -208,7 +210,7 @@ rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id)
 		slave->driver_id = 0;
 		sched_ctx->nb_slaves--;
 
-		CS_LOG_ERR("capabilities update failed");
+		CR_SCHED_LOG(ERR, "capabilities update failed");
 		return -ENOTSUP;
 	}
 
@@ -227,17 +229,17 @@ rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id)
 	uint32_t i, slave_pos;
 
 	if (!dev) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->driver_id != cryptodev_driver_id) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->data->dev_started) {
-		CS_LOG_ERR("Illegal operation");
+		CR_SCHED_LOG(ERR, "Illegal operation");
 		return -EBUSY;
 	}
 
@@ -247,12 +249,12 @@ rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id)
 		if (sched_ctx->slaves[slave_pos].dev_id == slave_id)
 			break;
 	if (slave_pos == sched_ctx->nb_slaves) {
-		CS_LOG_ERR("Cannot find slave");
+		CR_SCHED_LOG(ERR, "Cannot find slave");
 		return -ENOTSUP;
 	}
 
 	if (sched_ctx->ops.slave_detach(dev, slave_id) < 0) {
-		CS_LOG_ERR("Failed to detach slave");
+		CR_SCHED_LOG(ERR, "Failed to detach slave");
 		return -ENOTSUP;
 	}
 
@@ -265,7 +267,7 @@ rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id)
 	sched_ctx->nb_slaves--;
 
 	if (update_scheduler_capability(sched_ctx) < 0) {
-		CS_LOG_ERR("capabilities update failed");
+		CR_SCHED_LOG(ERR, "capabilities update failed");
 		return -ENOTSUP;
 	}
 
@@ -284,17 +286,17 @@ rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id,
 	struct scheduler_ctx *sched_ctx;
 
 	if (!dev) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->driver_id != cryptodev_driver_id) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->data->dev_started) {
-		CS_LOG_ERR("Illegal operation");
+		CR_SCHED_LOG(ERR, "Illegal operation");
 		return -EBUSY;
 	}
 
@@ -307,33 +309,33 @@ rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id,
 	case CDEV_SCHED_MODE_ROUNDROBIN:
 		if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 				roundrobin_scheduler) < 0) {
-			CS_LOG_ERR("Failed to load scheduler");
+			CR_SCHED_LOG(ERR, "Failed to load scheduler");
 			return -1;
 		}
 		break;
 	case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
 		if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 				pkt_size_based_distr_scheduler) < 0) {
-			CS_LOG_ERR("Failed to load scheduler");
+			CR_SCHED_LOG(ERR, "Failed to load scheduler");
 			return -1;
 		}
 		break;
 	case CDEV_SCHED_MODE_FAILOVER:
 		if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 				failover_scheduler) < 0) {
-			CS_LOG_ERR("Failed to load scheduler");
+			CR_SCHED_LOG(ERR, "Failed to load scheduler");
 			return -1;
 		}
 		break;
 	case CDEV_SCHED_MODE_MULTICORE:
 		if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 				multicore_scheduler) < 0) {
-			CS_LOG_ERR("Failed to load scheduler");
+			CR_SCHED_LOG(ERR, "Failed to load scheduler");
 			return -1;
 		}
 		break;
 	default:
-		CS_LOG_ERR("Not yet supported");
+		CR_SCHED_LOG(ERR, "Not yet supported");
 		return -ENOTSUP;
 	}
 
@@ -347,12 +349,12 @@ rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id)
 	struct scheduler_ctx *sched_ctx;
 
 	if (!dev) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->driver_id != cryptodev_driver_id) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
@@ -369,17 +371,17 @@ rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id,
 	struct scheduler_ctx *sched_ctx;
 
 	if (!dev) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->driver_id != cryptodev_driver_id) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->data->dev_started) {
-		CS_LOG_ERR("Illegal operation");
+		CR_SCHED_LOG(ERR, "Illegal operation");
 		return -EBUSY;
 	}
 
@@ -397,12 +399,12 @@ rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id)
 	struct scheduler_ctx *sched_ctx;
 
 	if (!dev) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->driver_id != cryptodev_driver_id) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
@@ -419,25 +421,25 @@ rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id,
 	struct scheduler_ctx *sched_ctx;
 
 	if (!dev) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->driver_id != cryptodev_driver_id) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->data->dev_started) {
-		CS_LOG_ERR("Illegal operation");
+		CR_SCHED_LOG(ERR, "Illegal operation");
 		return -EBUSY;
 	}
 
 	sched_ctx = dev->data->dev_private;
 
 	if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
-		CS_LOG_ERR("Invalid name %s, should be less than "
-				"%u bytes.\n", scheduler->name,
+		CR_SCHED_LOG(ERR, "Invalid name %s, should be less than "
+				"%u bytes.", scheduler->name,
 				RTE_CRYPTODEV_NAME_MAX_LEN);
 		return -EINVAL;
 	}
@@ -446,8 +448,8 @@ rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id,
 
 	if (strlen(scheduler->description) >
 			RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) {
-		CS_LOG_ERR("Invalid description %s, should be less than "
-				"%u bytes.\n", scheduler->description,
+		CR_SCHED_LOG(ERR, "Invalid description %s, should be less than "
+				"%u bytes.", scheduler->description,
 				RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1);
 		return -EINVAL;
 	}
@@ -473,7 +475,7 @@ rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id,
 		int ret = (*sched_ctx->ops.create_private_ctx)(dev);
 
 		if (ret < 0) {
-			CS_LOG_ERR("Unable to create scheduler private "
+			CR_SCHED_LOG(ERR, "Unable to create scheduler private "
 					"context");
 			return ret;
 		}
@@ -492,12 +494,12 @@ rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves)
 	uint32_t nb_slaves = 0;
 
 	if (!dev) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (dev->driver_id != cryptodev_driver_id) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
@@ -525,17 +527,17 @@ rte_cryptodev_scheduler_option_set(uint8_t scheduler_id,
 
 	if (option_type == CDEV_SCHED_OPTION_NOT_SET ||
 			option_type >= CDEV_SCHED_OPTION_COUNT) {
-		CS_LOG_ERR("Invalid option parameter");
+		CR_SCHED_LOG(ERR, "Invalid option parameter");
 		return -EINVAL;
 	}
 
 	if (!option) {
-		CS_LOG_ERR("Invalid option parameter");
+		CR_SCHED_LOG(ERR, "Invalid option parameter");
 		return -EINVAL;
 	}
 
 	if (dev->data->dev_started) {
-		CS_LOG_ERR("Illegal operation");
+		CR_SCHED_LOG(ERR, "Illegal operation");
 		return -EBUSY;
 	}
 
@@ -555,17 +557,17 @@ rte_cryptodev_scheduler_option_get(uint8_t scheduler_id,
 	struct scheduler_ctx *sched_ctx;
 
 	if (!dev) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
 	if (!option) {
-		CS_LOG_ERR("Invalid option parameter");
+		CR_SCHED_LOG(ERR, "Invalid option parameter");
 		return -EINVAL;
 	}
 
 	if (dev->driver_id != cryptodev_driver_id) {
-		CS_LOG_ERR("Operation not supported");
+		CR_SCHED_LOG(ERR, "Operation not supported");
 		return -ENOTSUP;
 	}
 
@@ -575,3 +577,8 @@ rte_cryptodev_scheduler_option_get(uint8_t scheduler_id,
 
 	return (*sched_ctx->ops.option_get)(dev, option_type, option);
 }
+
+RTE_INIT(scheduler_init_log)
+{
+	scheduler_logtype_driver = rte_log_register("pmd.crypto.scheduler");
+}
diff --git a/drivers/crypto/scheduler/rte_cryptodev_scheduler.h b/drivers/crypto/scheduler/rte_cryptodev_scheduler.h
index 1c164da7..3faea409 100644
--- a/drivers/crypto/scheduler/rte_cryptodev_scheduler.h
+++ b/drivers/crypto/scheduler/rte_cryptodev_scheduler.h
@@ -76,6 +76,7 @@ enum rte_cryptodev_schedule_option_type {
 /**
  * Threshold option structure
  */
+#define RTE_CRYPTODEV_SCHEDULER_PARAM_THRES	"threshold"
 struct rte_cryptodev_scheduler_threshold_option {
 	uint32_t threshold;	/**< Threshold for packet-size mode */
 };
diff --git a/drivers/crypto/scheduler/scheduler_failover.c b/drivers/crypto/scheduler/scheduler_failover.c
index 005b1638..ddfb5b81 100644
--- a/drivers/crypto/scheduler/scheduler_failover.c
+++ b/drivers/crypto/scheduler/scheduler_failover.c
@@ -139,7 +139,7 @@ scheduler_start(struct rte_cryptodev *dev)
 	uint16_t i;
 
 	if (sched_ctx->nb_slaves < 2) {
-		CS_LOG_ERR("Number of slaves shall no less than 2");
+		CR_SCHED_LOG(ERR, "Number of slaves shall no less than 2");
 		return -ENOMEM;
 	}
 
@@ -182,7 +182,7 @@ scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
 	fo_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*fo_qp_ctx), 0,
 			rte_socket_id());
 	if (!fo_qp_ctx) {
-		CS_LOG_ERR("failed allocate memory for private queue pair");
+		CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair");
 		return -ENOMEM;
 	}
 
diff --git a/drivers/crypto/scheduler/scheduler_multicore.c b/drivers/crypto/scheduler/scheduler_multicore.c
index 91fb0668..d410e69d 100644
--- a/drivers/crypto/scheduler/scheduler_multicore.c
+++ b/drivers/crypto/scheduler/scheduler_multicore.c
@@ -178,7 +178,8 @@ mc_scheduler_worker(struct rte_cryptodev *dev)
 		}
 	}
 	if (worker_idx == -1) {
-		CS_LOG_ERR("worker on core %u:cannot find worker index!\n", core_id);
+		CR_SCHED_LOG(ERR, "worker on core %u:cannot find worker index!",
+			core_id);
 		return -1;
 	}
 
@@ -313,7 +314,7 @@ scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
 	mc_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*mc_qp_ctx), 0,
 			rte_socket_id());
 	if (!mc_qp_ctx) {
-		CS_LOG_ERR("failed allocate memory for private queue pair");
+		CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair");
 		return -ENOMEM;
 	}
 
@@ -339,7 +340,7 @@ scheduler_create_private_ctx(struct rte_cryptodev *dev)
 	mc_ctx = rte_zmalloc_socket(NULL, sizeof(struct mc_scheduler_ctx), 0,
 			rte_socket_id());
 	if (!mc_ctx) {
-		CS_LOG_ERR("failed allocate memory");
+		CR_SCHED_LOG(ERR, "failed allocate memory");
 		return -ENOMEM;
 	}
 
@@ -356,7 +357,7 @@ scheduler_create_private_ctx(struct rte_cryptodev *dev)
 						rte_socket_id(),
 						RING_F_SC_DEQ | RING_F_SP_ENQ);
 			if (!mc_ctx->sched_enq_ring[i]) {
-				CS_LOG_ERR("Cannot create ring for worker %u",
+				CR_SCHED_LOG(ERR, "Cannot create ring for worker %u",
 					   i);
 				goto exit;
 			}
@@ -370,7 +371,7 @@ scheduler_create_private_ctx(struct rte_cryptodev *dev)
 						rte_socket_id(),
 						RING_F_SC_DEQ | RING_F_SP_ENQ);
 			if (!mc_ctx->sched_deq_ring[i]) {
-				CS_LOG_ERR("Cannot create ring for worker %u",
+				CR_SCHED_LOG(ERR, "Cannot create ring for worker %u",
 					   i);
 				goto exit;
 			}
diff --git a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c
index d09e849a..74129b66 100644
--- a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c
+++ b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c
@@ -258,7 +258,7 @@ scheduler_start(struct rte_cryptodev *dev)
 
 	/* for packet size based scheduler, nb_slaves have to >= 2 */
 	if (sched_ctx->nb_slaves < NB_PKT_SIZE_SLAVES) {
-		CS_LOG_ERR("not enough slaves to start");
+		CR_SCHED_LOG(ERR, "not enough slaves to start");
 		return -1;
 	}
 
@@ -302,7 +302,7 @@ scheduler_stop(struct rte_cryptodev *dev)
 
 		if (ps_qp_ctx->primary_slave.nb_inflight_cops +
 				ps_qp_ctx->secondary_slave.nb_inflight_cops) {
-			CS_LOG_ERR("Some crypto ops left in slave queue");
+			CR_SCHED_LOG(ERR, "Some crypto ops left in slave queue");
 			return -1;
 		}
 	}
@@ -319,7 +319,7 @@ scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
 	ps_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*ps_qp_ctx), 0,
 			rte_socket_id());
 	if (!ps_qp_ctx) {
-		CS_LOG_ERR("failed allocate memory for private queue pair");
+		CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair");
 		return -ENOMEM;
 	}
 
@@ -342,7 +342,7 @@ scheduler_create_private_ctx(struct rte_cryptodev *dev)
 	psd_ctx = rte_zmalloc_socket(NULL, sizeof(struct psd_scheduler_ctx), 0,
 			rte_socket_id());
 	if (!psd_ctx) {
-		CS_LOG_ERR("failed allocate memory");
+		CR_SCHED_LOG(ERR, "failed allocate memory");
 		return -ENOMEM;
 	}
 
@@ -362,14 +362,14 @@ scheduler_option_set(struct rte_cryptodev *dev, uint32_t option_type,
 
 	if ((enum rte_cryptodev_schedule_option_type)option_type !=
 			CDEV_SCHED_OPTION_THRESHOLD) {
-		CS_LOG_ERR("Option not supported");
+		CR_SCHED_LOG(ERR, "Option not supported");
 		return -EINVAL;
 	}
 
 	threshold = ((struct rte_cryptodev_scheduler_threshold_option *)
 			option)->threshold;
 	if (!rte_is_power_of_2(threshold)) {
-		CS_LOG_ERR("Threshold is not power of 2");
+		CR_SCHED_LOG(ERR, "Threshold is not power of 2");
 		return -EINVAL;
 	}
 
@@ -388,7 +388,7 @@ scheduler_option_get(struct rte_cryptodev *dev, uint32_t option_type,
 
 	if ((enum rte_cryptodev_schedule_option_type)option_type !=
 			CDEV_SCHED_OPTION_THRESHOLD) {
-		CS_LOG_ERR("Option not supported");
+		CR_SCHED_LOG(ERR, "Option not supported");
 		return -EINVAL;
 	}
 
diff --git a/drivers/crypto/scheduler/scheduler_pmd.c b/drivers/crypto/scheduler/scheduler_pmd.c
index 25d6409f..a9221a94 100644
--- a/drivers/crypto/scheduler/scheduler_pmd.c
+++ b/drivers/crypto/scheduler/scheduler_pmd.c
@@ -9,6 +9,7 @@
 #include <rte_malloc.h>
 #include <rte_cpuflags.h>
 #include <rte_reorder.h>
+#include <rte_string_fns.h>
 
 #include "rte_cryptodev_scheduler.h"
 #include "scheduler_pmd_private.h"
@@ -19,6 +20,7 @@ struct scheduler_init_params {
 	struct rte_cryptodev_pmd_init_params def_p;
 	uint32_t nb_slaves;
 	enum rte_cryptodev_scheduler_mode mode;
+	char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
 	uint32_t enable_ordering;
 	uint16_t wc_pool[RTE_MAX_LCORE];
 	uint16_t nb_wc;
@@ -29,9 +31,9 @@ struct scheduler_init_params {
 #define RTE_CRYPTODEV_VDEV_NAME			("name")
 #define RTE_CRYPTODEV_VDEV_SLAVE		("slave")
 #define RTE_CRYPTODEV_VDEV_MODE			("mode")
+#define RTE_CRYPTODEV_VDEV_MODE_PARAM		("mode_param")
 #define RTE_CRYPTODEV_VDEV_ORDERING		("ordering")
 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG	("max_nb_queue_pairs")
-#define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG	("max_nb_sessions")
 #define RTE_CRYPTODEV_VDEV_SOCKET_ID		("socket_id")
 #define RTE_CRYPTODEV_VDEV_COREMASK		("coremask")
 #define RTE_CRYPTODEV_VDEV_CORELIST		("corelist")
@@ -40,9 +42,9 @@ const char *scheduler_valid_params[] = {
 	RTE_CRYPTODEV_VDEV_NAME,
 	RTE_CRYPTODEV_VDEV_SLAVE,
 	RTE_CRYPTODEV_VDEV_MODE,
+	RTE_CRYPTODEV_VDEV_MODE_PARAM,
 	RTE_CRYPTODEV_VDEV_ORDERING,
 	RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
-	RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
 	RTE_CRYPTODEV_VDEV_SOCKET_ID,
 	RTE_CRYPTODEV_VDEV_COREMASK,
 	RTE_CRYPTODEV_VDEV_CORELIST
@@ -69,6 +71,8 @@ const struct scheduler_parse_map scheduler_ordering_map[] = {
 		{"disable", 0}
 };
 
+#define CDEV_SCHED_MODE_PARAM_SEP_CHAR		':'
+
 static int
 cryptodev_scheduler_create(const char *name,
 		struct rte_vdev_device *vdev,
@@ -82,7 +86,7 @@ cryptodev_scheduler_create(const char *name,
 	dev = rte_cryptodev_pmd_create(name, &vdev->device,
 			&init_params->def_p);
 	if (dev == NULL) {
-		CS_LOG_ERR("driver %s: failed to create cryptodev vdev",
+		CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev",
 			name);
 		return -EFAULT;
 	}
@@ -101,13 +105,22 @@ cryptodev_scheduler_create(const char *name,
 
 		for (i = 0; i < sched_ctx->nb_wc; i++) {
 			sched_ctx->wc_pool[i] = init_params->wc_pool[i];
-			RTE_LOG(INFO, PMD, "  Worker core[%u]=%u added\n",
+			CR_SCHED_LOG(INFO, "  Worker core[%u]=%u added",
 				i, sched_ctx->wc_pool[i]);
 		}
 	}
 
 	if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
 			init_params->mode < CDEV_SCHED_MODE_COUNT) {
+		union {
+			struct rte_cryptodev_scheduler_threshold_option
+					threshold_option;
+		} option;
+		enum rte_cryptodev_schedule_option_type option_type;
+		char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
+		char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
+		char *s, *end;
+
 		ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
 			init_params->mode);
 		if (ret < 0) {
@@ -119,10 +132,52 @@ cryptodev_scheduler_create(const char *name,
 			if (scheduler_mode_map[i].val != sched_ctx->mode)
 				continue;
 
-			RTE_LOG(INFO, PMD, "  Scheduling mode = %s\n",
+			CR_SCHED_LOG(INFO, "  Scheduling mode = %s",
 					scheduler_mode_map[i].name);
 			break;
 		}
+
+		if (strlen(init_params->mode_param_str) > 0) {
+			s = strchr(init_params->mode_param_str,
+					CDEV_SCHED_MODE_PARAM_SEP_CHAR);
+			if (s == NULL) {
+				CR_SCHED_LOG(ERR, "Invalid mode param");
+				return -EINVAL;
+			}
+
+			strlcpy(param_name, init_params->mode_param_str,
+					s - init_params->mode_param_str + 1);
+			s++;
+			strlcpy(param_val, s,
+					RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
+
+			switch (init_params->mode) {
+			case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
+				if (strcmp(param_name,
+					RTE_CRYPTODEV_SCHEDULER_PARAM_THRES)
+						!= 0) {
+					CR_SCHED_LOG(ERR, "Invalid mode param");
+					return -EINVAL;
+				}
+				option_type = CDEV_SCHED_OPTION_THRESHOLD;
+
+				option.threshold_option.threshold =
+						strtoul(param_val, &end, 0);
+				break;
+			default:
+				CR_SCHED_LOG(ERR, "Invalid mode param");
+				return -EINVAL;
+			}
+
+			if (sched_ctx->ops.option_set(dev, option_type,
+					(void *)&option) < 0) {
+				CR_SCHED_LOG(ERR, "Invalid mode param");
+				return -EINVAL;
+			}
+
+			RTE_LOG(INFO, PMD, "  Sched mode param (%s = %s)\n",
+					param_name, param_val);
+		}
 	}
 
 	sched_ctx->reordering_enabled = init_params->enable_ordering;
@@ -132,7 +187,7 @@ cryptodev_scheduler_create(const char *name,
 				sched_ctx->reordering_enabled)
 			continue;
 
-		RTE_LOG(INFO, PMD, "  Packet ordering = %s\n",
+		CR_SCHED_LOG(INFO, "  Packet ordering = %s",
 				scheduler_ordering_map[i].name);
 
 		break;
@@ -147,7 +202,7 @@ cryptodev_scheduler_create(const char *name,
 
 		if (!sched_ctx->init_slave_names[
 				sched_ctx->nb_init_slaves]) {
-			CS_LOG_ERR("driver %s: Insufficient memory",
+			CR_SCHED_LOG(ERR, "driver %s: Insufficient memory",
 					name);
 			return -ENOMEM;
 		}
@@ -169,8 +224,8 @@ cryptodev_scheduler_create(const char *name,
 			0, SOCKET_ID_ANY);
 
 	if (!sched_ctx->capabilities) {
-		RTE_LOG(ERR, PMD, "Not enough memory for capability "
-				"information\n");
+		CR_SCHED_LOG(ERR, "Not enough memory for capability "
+				"information");
 		return -ENOMEM;
 	}
 
@@ -214,7 +269,7 @@ parse_integer_arg(const char *key __rte_unused,
 
 	*i = atoi(value);
 	if (*i < 0) {
-		CS_LOG_ERR("Argument has to be positive.\n");
+		CR_SCHED_LOG(ERR, "Argument has to be positive.");
 		return -EINVAL;
 	}
 
@@ -287,8 +342,8 @@ parse_corelist_arg(const char *key __rte_unused,
 		unsigned int core = strtoul(token, &rval, 10);
 
 		if (core >= RTE_MAX_LCORE) {
-			CS_LOG_ERR("Invalid worker core %u, should be smaller "
-				   "than %u.\n", core, RTE_MAX_LCORE);
+			CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller "
+				   "than %u.", core, RTE_MAX_LCORE);
 		}
 		params->wc_pool[params->nb_wc++] = (uint16_t)core;
 		token = (const char *)rval;
@@ -308,8 +363,8 @@ parse_name_arg(const char *key __rte_unused,
 	struct rte_cryptodev_pmd_init_params *params = extra_args;
 
 	if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
-		CS_LOG_ERR("Invalid name %s, should be less than "
-				"%u bytes.\n", value,
+		CR_SCHED_LOG(ERR, "Invalid name %s, should be less than "
+				"%u bytes.", value,
 				RTE_CRYPTODEV_NAME_MAX_LEN - 1);
 		return -EINVAL;
 	}
@@ -327,7 +382,7 @@ parse_slave_arg(const char *key __rte_unused,
 	struct scheduler_init_params *param = extra_args;
 
 	if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
-		CS_LOG_ERR("Too many slaves.\n");
+		CR_SCHED_LOG(ERR, "Too many slaves.");
 		return -ENOMEM;
 	}
 
@@ -348,12 +403,13 @@ parse_mode_arg(const char *key __rte_unused,
 		if (strcmp(value, scheduler_mode_map[i].name) == 0) {
 			param->mode = (enum rte_cryptodev_scheduler_mode)
 					scheduler_mode_map[i].val;
+
 			break;
 		}
 	}
 
 	if (i == RTE_DIM(scheduler_mode_map)) {
-		CS_LOG_ERR("Unrecognized input.\n");
+		CR_SCHED_LOG(ERR, "Unrecognized input.");
 		return -EINVAL;
 	}
 
@@ -361,6 +417,18 @@ parse_mode_arg(const char *key __rte_unused,
 }
 
 static int
+parse_mode_param_arg(const char *key __rte_unused,
+		const char *value, void *extra_args)
+{
+	struct scheduler_init_params *param = extra_args;
+
+	strlcpy(param->mode_param_str, value,
+			RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
+
+	return 0;
+}
+
+static int
 parse_ordering_arg(const char *key __rte_unused,
 		const char *value, void *extra_args)
 {
@@ -376,7 +444,7 @@ parse_ordering_arg(const char *key __rte_unused,
 	}
 
 	if (i == RTE_DIM(scheduler_ordering_map)) {
-		CS_LOG_ERR("Unrecognized input.\n");
+		CR_SCHED_LOG(ERR, "Unrecognized input.");
 		return -EINVAL;
 	}
 
@@ -406,13 +474,6 @@ scheduler_parse_init_params(struct scheduler_init_params *params,
 		if (ret < 0)
 			goto free_kvlist;
 
-		ret = rte_kvargs_process(kvlist,
-				RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
-				&parse_integer_arg,
-				&params->def_p.max_nb_sessions);
-		if (ret < 0)
-			goto free_kvlist;
-
 		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
 				&parse_integer_arg,
 				&params->def_p.socket_id);
@@ -447,6 +508,11 @@ scheduler_parse_init_params(struct scheduler_init_params *params,
 		if (ret < 0)
 			goto free_kvlist;
 
+		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM,
+				&parse_mode_param_arg, params);
+		if (ret < 0)
+			goto free_kvlist;
+
 		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
 				&parse_ordering_arg, params);
 		if (ret < 0)
@@ -466,8 +532,7 @@ cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
 			"",
 			sizeof(struct scheduler_ctx),
 			rte_socket_id(),
-			RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
-			RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
+			RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 		},
 		.nb_slaves = 0,
 		.mode = CDEV_SCHED_MODE_NOT_SET,
@@ -500,7 +565,6 @@ RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
 	cryptodev_scheduler_pmd_drv);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
 	"max_nb_queue_pairs=<int> "
-	"max_nb_sessions=<int> "
 	"socket_id=<int> "
 	"slave=<name>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
diff --git a/drivers/crypto/scheduler/scheduler_pmd_ops.c b/drivers/crypto/scheduler/scheduler_pmd_ops.c
index 147dc51e..778071ca 100644
--- a/drivers/crypto/scheduler/scheduler_pmd_ops.c
+++ b/drivers/crypto/scheduler/scheduler_pmd_ops.c
@@ -27,7 +27,7 @@ scheduler_attach_init_slave(struct rte_cryptodev *dev)
 		int status;
 
 		if (!slave_dev) {
-			CS_LOG_ERR("Failed to locate slave dev %s",
+			CR_SCHED_LOG(ERR, "Failed to locate slave dev %s",
 					dev_name);
 			return -EINVAL;
 		}
@@ -36,12 +36,12 @@ scheduler_attach_init_slave(struct rte_cryptodev *dev)
 				scheduler_id, slave_dev->data->dev_id);
 
 		if (status < 0) {
-			CS_LOG_ERR("Failed to attach slave cryptodev %u",
+			CR_SCHED_LOG(ERR, "Failed to attach slave cryptodev %u",
 					slave_dev->data->dev_id);
 			return status;
 		}
 
-		CS_LOG_INFO("Scheduler %s attached slave %s\n",
+		CR_SCHED_LOG(INFO, "Scheduler %s attached slave %s",
 				dev->data->name,
 				sched_ctx->init_slave_names[i]);
 
@@ -102,7 +102,7 @@ update_order_ring(struct rte_cryptodev *dev, uint16_t qp_id)
 		if (snprintf(order_ring_name, RTE_CRYPTODEV_NAME_MAX_LEN,
 			"%s_rb_%u_%u", RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD),
 			dev->data->dev_id, qp_id) < 0) {
-			CS_LOG_ERR("failed to create unique reorder buffer "
+			CR_SCHED_LOG(ERR, "failed to create unique reorder buffer"
 					"name");
 			return -ENOMEM;
 		}
@@ -111,7 +111,7 @@ update_order_ring(struct rte_cryptodev *dev, uint16_t qp_id)
 				buff_size, rte_socket_id(),
 				RING_F_SP_ENQ | RING_F_SC_DEQ);
 		if (!qp_ctx->order_ring) {
-			CS_LOG_ERR("failed to create order ring");
+			CR_SCHED_LOG(ERR, "failed to create order ring");
 			return -ENOMEM;
 		}
 	} else {
@@ -145,18 +145,18 @@ scheduler_pmd_start(struct rte_cryptodev *dev)
 	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 		ret = update_order_ring(dev, i);
 		if (ret < 0) {
-			CS_LOG_ERR("Failed to update reorder buffer");
+			CR_SCHED_LOG(ERR, "Failed to update reorder buffer");
 			return ret;
 		}
 	}
 
 	if (sched_ctx->mode == CDEV_SCHED_MODE_NOT_SET) {
-		CS_LOG_ERR("Scheduler mode is not set");
+		CR_SCHED_LOG(ERR, "Scheduler mode is not set");
 		return -1;
 	}
 
 	if (!sched_ctx->nb_slaves) {
-		CS_LOG_ERR("No slave in the scheduler");
+		CR_SCHED_LOG(ERR, "No slave in the scheduler");
 		return -1;
 	}
 
@@ -166,7 +166,7 @@ scheduler_pmd_start(struct rte_cryptodev *dev)
 		uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 
 		if ((*sched_ctx->ops.slave_attach)(dev, slave_dev_id) < 0) {
-			CS_LOG_ERR("Failed to attach slave");
+			CR_SCHED_LOG(ERR, "Failed to attach slave");
 			return -ENOTSUP;
 		}
 	}
@@ -174,7 +174,7 @@ scheduler_pmd_start(struct rte_cryptodev *dev)
 	RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.scheduler_start, -ENOTSUP);
 
 	if ((*sched_ctx->ops.scheduler_start)(dev) < 0) {
-		CS_LOG_ERR("Scheduler start failed");
+		CR_SCHED_LOG(ERR, "Scheduler start failed");
 		return -1;
 	}
 
@@ -186,7 +186,7 @@ scheduler_pmd_start(struct rte_cryptodev *dev)
 
 		ret = (*slave_dev->dev_ops->dev_start)(slave_dev);
 		if (ret < 0) {
-			CS_LOG_ERR("Failed to start slave dev %u",
+			CR_SCHED_LOG(ERR, "Failed to start slave dev %u",
 					slave_dev_id);
 			return ret;
 		}
@@ -321,8 +321,9 @@ scheduler_pmd_info_get(struct rte_cryptodev *dev,
 		struct rte_cryptodev_info *dev_info)
 {
 	struct scheduler_ctx *sched_ctx = dev->data->dev_private;
-	uint32_t max_nb_sessions = sched_ctx->nb_slaves ?
-			UINT32_MAX : RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS;
+	uint32_t max_nb_sess = 0;
+	uint16_t headroom_sz = 0;
+	uint16_t tailroom_sz = 0;
 	uint32_t i;
 
 	if (!dev_info)
@@ -338,17 +339,32 @@ scheduler_pmd_info_get(struct rte_cryptodev *dev,
 		struct rte_cryptodev_info slave_info;
 
 		rte_cryptodev_info_get(slave_dev_id, &slave_info);
-		max_nb_sessions = slave_info.sym.max_nb_sessions <
-				max_nb_sessions ?
-				slave_info.sym.max_nb_sessions :
-				max_nb_sessions;
+		uint32_t dev_max_sess = slave_info.sym.max_nb_sessions;
+		if (dev_max_sess != 0) {
+			if (max_nb_sess == 0 ||	dev_max_sess < max_nb_sess)
+				max_nb_sess = slave_info.sym.max_nb_sessions;
+		}
+
+		/* Get the max headroom requirement among slave PMDs */
+		headroom_sz = slave_info.min_mbuf_headroom_req >
+				headroom_sz ?
+				slave_info.min_mbuf_headroom_req :
+				headroom_sz;
+
+		/* Get the max tailroom requirement among slave PMDs */
+		tailroom_sz = slave_info.min_mbuf_tailroom_req >
+				tailroom_sz ?
+				slave_info.min_mbuf_tailroom_req :
+				tailroom_sz;
 	}
 
 	dev_info->driver_id = dev->driver_id;
 	dev_info->feature_flags = dev->feature_flags;
 	dev_info->capabilities = sched_ctx->capabilities;
 	dev_info->max_nb_queue_pairs = sched_ctx->max_nb_queue_pairs;
-	dev_info->sym.max_nb_sessions = max_nb_sessions;
+	dev_info->min_mbuf_headroom_req = headroom_sz;
+	dev_info->min_mbuf_tailroom_req = tailroom_sz;
+	dev_info->sym.max_nb_sessions = max_nb_sess;
 }
 
 /** Release queue pair */
@@ -386,7 +402,7 @@ scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	if (snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN,
 			"CRYTO_SCHE PMD %u QP %u",
 			dev->data->dev_id, qp_id) < 0) {
-		CS_LOG_ERR("Failed to create unique queue pair name");
+		CR_SCHED_LOG(ERR, "Failed to create unique queue pair name");
 		return -EFAULT;
 	}
 
@@ -424,14 +440,14 @@ scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	 */
 	ret = scheduler_attach_init_slave(dev);
 	if (ret < 0) {
-		CS_LOG_ERR("Failed to attach slave");
+		CR_SCHED_LOG(ERR, "Failed to attach slave");
 		scheduler_pmd_qp_release(dev, qp_id);
 		return ret;
 	}
 
 	if (*sched_ctx->ops.config_queue_pair) {
 		if ((*sched_ctx->ops.config_queue_pair)(dev, qp_id) < 0) {
-			CS_LOG_ERR("Unable to configure queue pair");
+			CR_SCHED_LOG(ERR, "Unable to configure queue pair");
 			return -1;
 		}
 	}
@@ -439,22 +455,6 @@ scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return 0;
 }
 
-/** Start queue pair */
-static int
-scheduler_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
-/** Stop queue pair */
-static int
-scheduler_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
 /** Return the number of allocated queue pairs */
 static uint32_t
 scheduler_pmd_qp_count(struct rte_cryptodev *dev)
@@ -463,7 +463,7 @@ scheduler_pmd_qp_count(struct rte_cryptodev *dev)
 }
 
 static uint32_t
-scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
+scheduler_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 	uint8_t i = 0;
@@ -473,7 +473,7 @@ scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
 	for (i = 0; i < sched_ctx->nb_slaves; i++) {
 		uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 		struct rte_cryptodev *dev = &rte_cryptodevs[slave_dev_id];
-		uint32_t priv_sess_size = (*dev->dev_ops->session_get_size)(dev);
+		uint32_t priv_sess_size = (*dev->dev_ops->sym_session_get_size)(dev);
 
 		if (max_priv_sess_size < priv_sess_size)
 			max_priv_sess_size = priv_sess_size;
@@ -483,7 +483,7 @@ scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
 }
 
 static int
-scheduler_pmd_session_configure(struct rte_cryptodev *dev,
+scheduler_pmd_sym_session_configure(struct rte_cryptodev *dev,
 	struct rte_crypto_sym_xform *xform,
 	struct rte_cryptodev_sym_session *sess,
 	struct rte_mempool *mempool)
@@ -498,7 +498,7 @@ scheduler_pmd_session_configure(struct rte_cryptodev *dev,
 		ret = rte_cryptodev_sym_session_init(slave->dev_id, sess,
 					xform, mempool);
 		if (ret < 0) {
-			CS_LOG_ERR("unabled to config sym session");
+			CR_SCHED_LOG(ERR, "unable to config sym session");
 			return ret;
 		}
 	}
@@ -508,7 +508,7 @@ scheduler_pmd_session_configure(struct rte_cryptodev *dev,
 
 /** Clear the memory of session so it doesn't leave key material behind */
 static void
-scheduler_pmd_session_clear(struct rte_cryptodev *dev,
+scheduler_pmd_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 	struct scheduler_ctx *sched_ctx = dev->data->dev_private;
@@ -535,13 +535,11 @@ struct rte_cryptodev_ops scheduler_pmd_ops = {
 
 		.queue_pair_setup	= scheduler_pmd_qp_setup,
 		.queue_pair_release	= scheduler_pmd_qp_release,
-		.queue_pair_start	= scheduler_pmd_qp_start,
-		.queue_pair_stop	= scheduler_pmd_qp_stop,
 		.queue_pair_count	= scheduler_pmd_qp_count,
 
-		.session_get_size	= scheduler_pmd_session_get_size,
-		.session_configure	= scheduler_pmd_session_configure,
-		.session_clear		= scheduler_pmd_session_clear,
+		.sym_session_get_size	= scheduler_pmd_sym_session_get_size,
+		.sym_session_configure	= scheduler_pmd_sym_session_configure,
+		.sym_session_clear	= scheduler_pmd_sym_session_clear,
 };
 
 struct rte_cryptodev_ops *rte_crypto_scheduler_pmd_ops = &scheduler_pmd_ops;
diff --git a/drivers/crypto/scheduler/scheduler_pmd_private.h b/drivers/crypto/scheduler/scheduler_pmd_private.h
index 12410b48..d5e602a2 100644
--- a/drivers/crypto/scheduler/scheduler_pmd_private.h
+++ b/drivers/crypto/scheduler/scheduler_pmd_private.h
@@ -12,25 +12,11 @@
 
 #define PER_SLAVE_BUFF_SIZE			(256)
 
-#define CS_LOG_ERR(fmt, args...)					\
-	RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",		\
-		RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD),			\
-		__func__, __LINE__, ## args)
-
-#ifdef RTE_LIBRTE_CRYPTO_SCHEDULER_DEBUG
-#define CS_LOG_INFO(fmt, args...)					\
-	RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",	\
-		RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD),			\
-		__func__, __LINE__, ## args)
-
-#define CS_LOG_DBG(fmt, args...)					\
-	RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",	\
-		RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD),			\
-		__func__, __LINE__, ## args)
-#else
-#define CS_LOG_INFO(fmt, args...)
-#define CS_LOG_DBG(fmt, args...)
-#endif
+extern int scheduler_logtype_driver;
+
+#define CR_SCHED_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, scheduler_logtype_driver,		\
+			"%s() line %u: "fmt "\n", __func__, __LINE__, ##args)
 
 struct scheduler_slave {
 	uint8_t dev_id;
diff --git a/drivers/crypto/scheduler/scheduler_roundrobin.c b/drivers/crypto/scheduler/scheduler_roundrobin.c
index c6e03e21..c7082a64 100644
--- a/drivers/crypto/scheduler/scheduler_roundrobin.c
+++ b/drivers/crypto/scheduler/scheduler_roundrobin.c
@@ -175,7 +175,7 @@ scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
 	rr_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*rr_qp_ctx), 0,
 			rte_socket_id());
 	if (!rr_qp_ctx) {
-		CS_LOG_ERR("failed allocate memory for private queue pair");
+		CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair");
 		return -ENOMEM;
 	}
 
diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd.c b/drivers/crypto/snow3g/rte_snow3g_pmd.c
index 72751e35..a17536b7 100644
--- a/drivers/crypto/snow3g/rte_snow3g_pmd.c
+++ b/drivers/crypto/snow3g/rte_snow3g_pmd.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2016-2017 Intel Corporation
+ * Copyright(c) 2016-2018 Intel Corporation
  */
 
 #include <rte_common.h>
@@ -79,7 +79,7 @@ snow3g_set_session_parameters(struct snow3g_session *sess,
 		break;
 	case SNOW3G_OP_NOT_SUPPORTED:
 	default:
-		SNOW3G_LOG_ERR("Unsupported operation chain order parameter");
+		SNOW3G_LOG(ERR, "Unsupported operation chain order parameter");
 		return -ENOTSUP;
 	}
 
@@ -89,7 +89,7 @@ snow3g_set_session_parameters(struct snow3g_session *sess,
 			return -ENOTSUP;
 
 		if (cipher_xform->cipher.iv.length != SNOW3G_IV_LENGTH) {
-			SNOW3G_LOG_ERR("Wrong IV length");
+			SNOW3G_LOG(ERR, "Wrong IV length");
 			return -EINVAL;
 		}
 		sess->cipher_iv_offset = cipher_xform->cipher.iv.offset;
@@ -105,14 +105,14 @@ snow3g_set_session_parameters(struct snow3g_session *sess,
 			return -ENOTSUP;
 
 		if (auth_xform->auth.digest_length != SNOW3G_DIGEST_LENGTH) {
-			SNOW3G_LOG_ERR("Wrong digest length");
+			SNOW3G_LOG(ERR, "Wrong digest length");
 			return -EINVAL;
 		}
 
 		sess->auth_op = auth_xform->auth.op;
 
 		if (auth_xform->auth.iv.length != SNOW3G_IV_LENGTH) {
-			SNOW3G_LOG_ERR("Wrong IV length");
+			SNOW3G_LOG(ERR, "Wrong IV length");
 			return -EINVAL;
 		}
 		sess->auth_iv_offset = auth_xform->auth.iv.offset;
@@ -137,7 +137,7 @@ snow3g_get_session(struct snow3g_qp *qp, struct rte_crypto_op *op)
 	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 		if (likely(op->sym->session != NULL))
 			sess = (struct snow3g_session *)
-					get_session_private_data(
+					get_sym_session_private_data(
 					op->sym->session,
 					cryptodev_driver_id);
 	} else {
@@ -159,8 +159,8 @@ snow3g_get_session(struct snow3g_qp *qp, struct rte_crypto_op *op)
 			sess = NULL;
 		}
 		op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
-		set_session_private_data(op->sym->session, cryptodev_driver_id,
-			_sess_private_data);
+		set_sym_session_private_data(op->sym->session,
+				cryptodev_driver_id, _sess_private_data);
 	}
 
 	if (unlikely(sess == NULL))
@@ -216,7 +216,7 @@ process_snow3g_cipher_op_bit(struct rte_crypto_op *op,
 	src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *);
 	if (op->sym->m_dst == NULL) {
 		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
-		SNOW3G_LOG_ERR("bit-level in-place not supported\n");
+		SNOW3G_LOG(ERR, "bit-level in-place not supported\n");
 		return 0;
 	}
 	dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *);
@@ -246,7 +246,7 @@ process_snow3g_hash_op(struct snow3g_qp *qp, struct rte_crypto_op **ops,
 		/* Data must be byte aligned */
 		if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) {
 			ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
-			SNOW3G_LOG_ERR("Offset");
+			SNOW3G_LOG(ERR, "Offset");
 			break;
 		}
 
@@ -295,7 +295,7 @@ process_ops(struct rte_crypto_op **ops, struct snow3g_session *session,
 				(ops[i]->sym->m_dst != NULL &&
 				!rte_pktmbuf_is_contiguous(
 						ops[i]->sym->m_dst))) {
-			SNOW3G_LOG_ERR("PMD supports only contiguous mbufs, "
+			SNOW3G_LOG(ERR, "PMD supports only contiguous mbufs, "
 				"op (%p) provides noncontiguous mbuf as "
 				"source/destination buffer.\n", ops[i]);
 			ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
@@ -537,7 +537,7 @@ cryptodev_snow3g_create(const char *name,
 
 	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
 	if (dev == NULL) {
-		SNOW3G_LOG_ERR("failed to create cryptodev vdev");
+		SNOW3G_LOG(ERR, "failed to create cryptodev vdev");
 		goto init_error;
 	}
 
@@ -555,11 +555,10 @@ cryptodev_snow3g_create(const char *name,
 	internals = dev->data->dev_private;
 
 	internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
-	internals->max_nb_sessions = init_params->max_nb_sessions;
 
 	return 0;
 init_error:
-	SNOW3G_LOG_ERR("driver %s: cryptodev_snow3g_create failed",
+	SNOW3G_LOG(ERR, "driver %s: cryptodev_snow3g_create failed",
 			init_params->name);
 
 	cryptodev_snow3g_remove(vdev);
@@ -573,8 +572,7 @@ cryptodev_snow3g_probe(struct rte_vdev_device *vdev)
 		"",
 		sizeof(struct snow3g_private),
 		rte_socket_id(),
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
+		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 	};
 	const char *name;
 	const char *input_args;
@@ -617,7 +615,11 @@ RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd_drv);
 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SNOW3G_PMD,
 	"max_nb_queue_pairs=<int> "
-	"max_nb_sessions=<int> "
 	"socket_id=<int>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(snow3g_crypto_drv,
 		cryptodev_snow3g_pmd_drv.driver, cryptodev_driver_id);
+
+RTE_INIT(snow3g_init_log)
+{
+	snow3g_logtype_driver = rte_log_register("pmd.crypto.snow3g");
+}
diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c
index f60b4759..cfbc9522 100644
--- a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c
+++ b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2016-2017 Intel Corporation
+ * Copyright(c) 2016-2018 Intel Corporation
  */
 
 #include <string.h>
@@ -130,7 +130,8 @@ snow3g_pmd_info_get(struct rte_cryptodev *dev,
 	if (dev_info != NULL) {
 		dev_info->driver_id = dev->driver_id;
 		dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
-		dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
+		/* No limit of number of sessions */
+		dev_info->sym.max_nb_sessions = 0;
 		dev_info->feature_flags = dev->feature_flags;
 		dev_info->capabilities = snow3g_pmd_capabilities;
 	}
@@ -172,13 +173,13 @@ snow3g_pmd_qp_create_processed_ops_ring(struct snow3g_qp *qp,
 	r = rte_ring_lookup(qp->name);
 	if (r) {
 		if (rte_ring_get_size(r) >= ring_size) {
-			SNOW3G_LOG_INFO("Reusing existing ring %s"
+			SNOW3G_LOG(INFO, "Reusing existing ring %s"
 					" for processed packets",
 					 qp->name);
 			return r;
 		}
 
-		SNOW3G_LOG_ERR("Unable to reuse existing ring %s"
+		SNOW3G_LOG(ERR, "Unable to reuse existing ring %s"
 				" for processed packets",
 				 qp->name);
 		return NULL;
@@ -230,22 +231,6 @@ qp_setup_cleanup:
 	return -1;
 }
 
-/** Start queue pair */
-static int
-snow3g_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
-/** Stop queue pair */
-static int
-snow3g_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
 /** Return the number of allocated queue pairs */
 static uint32_t
 snow3g_pmd_qp_count(struct rte_cryptodev *dev)
@@ -255,14 +240,14 @@ snow3g_pmd_qp_count(struct rte_cryptodev *dev)
 
 /** Returns the size of the SNOW 3G session structure */
 static unsigned
-snow3g_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
+snow3g_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	return sizeof(struct snow3g_session);
 }
 
 /** Configure a SNOW 3G session from a crypto xform chain */
 static int
-snow3g_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
+snow3g_pmd_sym_session_configure(struct rte_cryptodev *dev __rte_unused,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *sess,
 		struct rte_mempool *mempool)
@@ -271,26 +256,26 @@ snow3g_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
 	int ret;
 
 	if (unlikely(sess == NULL)) {
-		SNOW3G_LOG_ERR("invalid session struct");
+		SNOW3G_LOG(ERR, "invalid session struct");
 		return -EINVAL;
 	}
 
 	if (rte_mempool_get(mempool, &sess_private_data)) {
-		CDEV_LOG_ERR(
+		SNOW3G_LOG(ERR,
 			"Couldn't get object from session mempool");
 		return -ENOMEM;
 	}
 
 	ret = snow3g_set_session_parameters(sess_private_data, xform);
 	if (ret != 0) {
-		SNOW3G_LOG_ERR("failed configure session parameters");
+		SNOW3G_LOG(ERR, "failed configure session parameters");
 
 		/* Return session to mempool */
 		rte_mempool_put(mempool, sess_private_data);
 		return ret;
 	}
 
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 		sess_private_data);
 
 	return 0;
@@ -298,17 +283,17 @@ snow3g_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
 
 /** Clear the memory of session so it doesn't leave key material behind */
 static void
-snow3g_pmd_session_clear(struct rte_cryptodev *dev,
+snow3g_pmd_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 
 	/* Zero out the whole structure */
 	if (sess_priv) {
 		memset(sess_priv, 0, sizeof(struct snow3g_session));
 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
 		rte_mempool_put(sess_mp, sess_priv);
 	}
 }
@@ -326,13 +311,11 @@ struct rte_cryptodev_ops snow3g_pmd_ops = {
 
 		.queue_pair_setup   = snow3g_pmd_qp_setup,
 		.queue_pair_release = snow3g_pmd_qp_release,
-		.queue_pair_start   = snow3g_pmd_qp_start,
-		.queue_pair_stop    = snow3g_pmd_qp_stop,
 		.queue_pair_count   = snow3g_pmd_qp_count,
 
-		.session_get_size   = snow3g_pmd_session_get_size,
-		.session_configure  = snow3g_pmd_session_configure,
-		.session_clear      = snow3g_pmd_session_clear
+		.sym_session_get_size   = snow3g_pmd_sym_session_get_size,
+		.sym_session_configure  = snow3g_pmd_sym_session_configure,
+		.sym_session_clear      = snow3g_pmd_sym_session_clear
 };
 
 struct rte_cryptodev_ops *rte_snow3g_pmd_ops = &snow3g_pmd_ops;
diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd_private.h b/drivers/crypto/snow3g/rte_snow3g_pmd_private.h
index eea900e0..b7807b62 100644
--- a/drivers/crypto/snow3g/rte_snow3g_pmd_private.h
+++ b/drivers/crypto/snow3g/rte_snow3g_pmd_private.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2016-2017 Intel Corporation
+ * Copyright(c) 2016-2018 Intel Corporation
  */
 
 #ifndef _RTE_SNOW3G_PMD_PRIVATE_H_
@@ -10,25 +10,13 @@
 #define CRYPTODEV_NAME_SNOW3G_PMD	crypto_snow3g
 /**< SNOW 3G PMD device name */
 
-#define SNOW3G_LOG_ERR(fmt, args...) \
-	RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",  \
-			RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD), \
-			__func__, __LINE__, ## args)
-
-#ifdef RTE_LIBRTE_SNOW3G_DEBUG
-#define SNOW3G_LOG_INFO(fmt, args...) \
-	RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD), \
-			__func__, __LINE__, ## args)
-
-#define SNOW3G_LOG_DBG(fmt, args...) \
-	RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD), \
-			__func__, __LINE__, ## args)
-#else
-#define SNOW3G_LOG_INFO(fmt, args...)
-#define SNOW3G_LOG_DBG(fmt, args...)
-#endif
+/** SNOW 3G PMD LOGTYPE DRIVER */
+int snow3g_logtype_driver;
+
+#define SNOW3G_LOG(level, fmt, ...)  \
+	rte_log(RTE_LOG_ ## level, snow3g_logtype_driver,  \
+			"%s() line %u: " fmt "\n", __func__, __LINE__,  \
+					## __VA_ARGS__)
 
 #define SNOW3G_DIGEST_LENGTH 4
 
@@ -36,8 +24,6 @@
 struct snow3g_private {
 	unsigned max_nb_queue_pairs;
 	/**< Max number of queue pairs supported by device */
-	unsigned max_nb_sessions;
-	/**< Max number of sessions supported by device */
 };
 
 /** SNOW 3G buffer queue pair */
diff --git a/drivers/crypto/virtio/virtio_cryptodev.c b/drivers/crypto/virtio/virtio_cryptodev.c
index df88953f..568b5a40 100644
--- a/drivers/crypto/virtio/virtio_cryptodev.c
+++ b/drivers/crypto/virtio/virtio_cryptodev.c
@@ -515,16 +515,12 @@ static struct rte_cryptodev_ops virtio_crypto_dev_ops = {
 
 	.queue_pair_setup                = virtio_crypto_qp_setup,
 	.queue_pair_release              = virtio_crypto_qp_release,
-	.queue_pair_start                = NULL,
-	.queue_pair_stop                 = NULL,
 	.queue_pair_count                = NULL,
 
 	/* Crypto related operations */
-	.session_get_size	= virtio_crypto_sym_get_session_private_size,
-	.session_configure	= virtio_crypto_sym_configure_session,
-	.session_clear		= virtio_crypto_sym_clear_session,
-	.qp_attach_session = NULL,
-	.qp_detach_session = NULL
+	.sym_session_get_size		= virtio_crypto_sym_get_session_private_size,
+	.sym_session_configure		= virtio_crypto_sym_configure_session,
+	.sym_session_clear		= virtio_crypto_sym_clear_session
 };
 
 static void
@@ -962,7 +958,7 @@ virtio_crypto_sym_clear_session(
 
 	hw = dev->data->dev_private;
 	vq = hw->cvq;
-	session = (struct virtio_crypto_session *)get_session_private_data(
+	session = (struct virtio_crypto_session *)get_sym_session_private_data(
 		sess, cryptodev_virtio_driver_id);
 	if (session == NULL) {
 		VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid session parameter");
@@ -1080,7 +1076,10 @@ virtio_crypto_sym_clear_session(
 	VIRTIO_CRYPTO_SESSION_LOG_INFO("Close session %"PRIu64" successfully ",
 			session->session_id);
 
-	memset(sess, 0, sizeof(struct virtio_crypto_session));
+	memset(session, 0, sizeof(struct virtio_crypto_session));
+	struct rte_mempool *sess_mp = rte_mempool_from_obj(session);
+	set_sym_session_private_data(sess, cryptodev_virtio_driver_id, NULL);
+	rte_mempool_put(sess_mp, session);
 	rte_free(malloc_virt_addr);
 }
 
@@ -1223,6 +1222,12 @@ virtio_crypto_sym_pad_op_ctrl_req(
 	/* Get cipher xform from crypto xform chain */
 	cipher_xform = virtio_crypto_get_cipher_xform(xform);
 	if (cipher_xform) {
+		if (cipher_xform->iv.length > VIRTIO_CRYPTO_MAX_IV_SIZE) {
+			VIRTIO_CRYPTO_SESSION_LOG_ERR(
+				"cipher IV size cannot be longer than %u",
+				VIRTIO_CRYPTO_MAX_IV_SIZE);
+			return -1;
+		}
 		if (is_chainned)
 			ret = virtio_crypto_sym_pad_cipher_param(
 				&ctrl->u.sym_create_session.u.chain.para
@@ -1390,7 +1395,7 @@ virtio_crypto_sym_configure_session(
 		goto error_out;
 	}
 
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 		session_private);
 
 	return 0;
@@ -1409,11 +1414,10 @@ virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
 
 	if (info != NULL) {
 		info->driver_id = cryptodev_virtio_driver_id;
-		info->pci_dev = RTE_DEV_TO_PCI(dev->device);
 		info->feature_flags = dev->feature_flags;
 		info->max_nb_queue_pairs = hw->max_dataqueues;
-		info->sym.max_nb_sessions =
-			RTE_VIRTIO_CRYPTO_PMD_MAX_NB_SESSIONS;
+		/* No limit of number of sessions */
+		info->sym.max_nb_sessions = 0;
 		info->capabilities = hw->virtio_dev_capabilities;
 	}
 }
@@ -1426,8 +1430,7 @@ crypto_virtio_pci_probe(
 	struct rte_cryptodev_pmd_init_params init_params = {
 		.name = "",
 		.socket_id = rte_socket_id(),
-		.private_data_size = sizeof(struct virtio_crypto_hw),
-		.max_nb_sessions = RTE_VIRTIO_CRYPTO_PMD_MAX_NB_SESSIONS
+		.private_data_size = sizeof(struct virtio_crypto_hw)
 	};
 	char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 
@@ -1475,9 +1478,7 @@ RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv,
 	rte_virtio_crypto_driver.driver,
 	cryptodev_virtio_driver_id);
 
-RTE_INIT(virtio_crypto_init_log);
-static void
-virtio_crypto_init_log(void)
+RTE_INIT(virtio_crypto_init_log)
 {
 	virtio_crypto_logtype_init = rte_log_register("pmd.crypto.virtio.init");
 	if (virtio_crypto_logtype_init >= 0)
diff --git a/drivers/crypto/virtio/virtio_cryptodev.h b/drivers/crypto/virtio/virtio_cryptodev.h
index e402c030..0fd7b722 100644
--- a/drivers/crypto/virtio/virtio_cryptodev.h
+++ b/drivers/crypto/virtio/virtio_cryptodev.h
@@ -16,6 +16,8 @@
 
 #define NUM_ENTRY_VIRTIO_CRYPTO_OP 7
 
+#define VIRTIO_CRYPTO_MAX_IV_SIZE 16
+
 extern uint8_t cryptodev_virtio_driver_id;
 
 enum virtio_crypto_cmd_id {
@@ -29,6 +31,7 @@ struct virtio_crypto_op_cookie {
 	struct virtio_crypto_op_data_req data_req;
 	struct virtio_crypto_inhdr inhdr;
 	struct vring_desc desc[NUM_ENTRY_VIRTIO_CRYPTO_OP];
+	uint8_t iv[VIRTIO_CRYPTO_MAX_IV_SIZE];
 };
 
 /*
diff --git a/drivers/crypto/virtio/virtio_rxtx.c b/drivers/crypto/virtio/virtio_rxtx.c
index 45039284..e32a1ecd 100644
--- a/drivers/crypto/virtio/virtio_rxtx.c
+++ b/drivers/crypto/virtio/virtio_rxtx.c
@@ -203,9 +203,11 @@ virtqueue_crypto_sym_enqueue_xmit(
 	uint16_t req_data_len = sizeof(struct virtio_crypto_op_data_req);
 	uint32_t indirect_vring_addr_offset = req_data_len +
 		sizeof(struct virtio_crypto_inhdr);
+	uint32_t indirect_iv_addr_offset = indirect_vring_addr_offset +
+			sizeof(struct vring_desc) * NUM_ENTRY_VIRTIO_CRYPTO_OP;
 	struct rte_crypto_sym_op *sym_op = cop->sym;
 	struct virtio_crypto_session *session =
-		(struct virtio_crypto_session *)get_session_private_data(
+		(struct virtio_crypto_session *)get_sym_session_private_data(
 		cop->sym->session, cryptodev_virtio_driver_id);
 	struct virtio_crypto_op_data_req *op_data_req;
 	uint32_t hash_result_len = 0;
@@ -259,7 +261,17 @@ virtqueue_crypto_sym_enqueue_xmit(
 
 	/* indirect vring: iv of cipher */
 	if (session->iv.length) {
-		desc[idx].addr = cop->phys_addr + session->iv.offset;
+		if (cop->phys_addr)
+			desc[idx].addr = cop->phys_addr + session->iv.offset;
+		else {
+			rte_memcpy(crypto_op_cookie->iv,
+					rte_crypto_op_ctod_offset(cop,
+					uint8_t *, session->iv.offset),
+					session->iv.length);
+			desc[idx].addr = indirect_op_data_req_phys_addr +
+				indirect_iv_addr_offset;
+		}
+
 		desc[idx].len = session->iv.length;
 		desc[idx++].flags = VRING_DESC_F_NEXT;
 	}
diff --git a/drivers/crypto/zuc/rte_zuc_pmd.c b/drivers/crypto/zuc/rte_zuc_pmd.c
index a805b227..313f4590 100644
--- a/drivers/crypto/zuc/rte_zuc_pmd.c
+++ b/drivers/crypto/zuc/rte_zuc_pmd.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2016-2017 Intel Corporation
+ * Copyright(c) 2016-2018 Intel Corporation
  */
 
 #include <rte_common.h>
@@ -11,7 +11,6 @@
 #include <rte_cpuflags.h>
 
 #include "rte_zuc_pmd_private.h"
-
 #define ZUC_MAX_BURST 4
 #define BYTE_LEN 8
 
@@ -78,7 +77,7 @@ zuc_set_session_parameters(struct zuc_session *sess,
 		break;
 	case ZUC_OP_NOT_SUPPORTED:
 	default:
-		ZUC_LOG_ERR("Unsupported operation chain order parameter");
+		ZUC_LOG(ERR, "Unsupported operation chain order parameter");
 		return -ENOTSUP;
 	}
 
@@ -88,7 +87,7 @@ zuc_set_session_parameters(struct zuc_session *sess,
 			return -ENOTSUP;
 
 		if (cipher_xform->cipher.iv.length != ZUC_IV_KEY_LENGTH) {
-			ZUC_LOG_ERR("Wrong IV length");
+			ZUC_LOG(ERR, "Wrong IV length");
 			return -EINVAL;
 		}
 		sess->cipher_iv_offset = cipher_xform->cipher.iv.offset;
@@ -104,14 +103,14 @@ zuc_set_session_parameters(struct zuc_session *sess,
 			return -ENOTSUP;
 
 		if (auth_xform->auth.digest_length != ZUC_DIGEST_LENGTH) {
-			ZUC_LOG_ERR("Wrong digest length");
+			ZUC_LOG(ERR, "Wrong digest length");
 			return -EINVAL;
 		}
 
 		sess->auth_op = auth_xform->auth.op;
 
 		if (auth_xform->auth.iv.length != ZUC_IV_KEY_LENGTH) {
-			ZUC_LOG_ERR("Wrong IV length");
+			ZUC_LOG(ERR, "Wrong IV length");
 			return -EINVAL;
 		}
 		sess->auth_iv_offset = auth_xform->auth.iv.offset;
@@ -135,7 +134,7 @@ zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op)
 
 	if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 		if (likely(op->sym->session != NULL))
-			sess = (struct zuc_session *)get_session_private_data(
+			sess = (struct zuc_session *)get_sym_session_private_data(
 					op->sym->session,
 					cryptodev_driver_id);
 	} else {
@@ -157,8 +156,8 @@ zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op)
 			sess = NULL;
 		}
 		op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
-		set_session_private_data(op->sym->session, cryptodev_driver_id,
-			_sess_private_data);
+		set_sym_session_private_data(op->sym->session,
+				cryptodev_driver_id, _sess_private_data);
 	}
 
 	if (unlikely(sess == NULL))
@@ -187,7 +186,7 @@ process_zuc_cipher_op(struct rte_crypto_op **ops,
 				|| ((ops[i]->sym->cipher.data.offset
 					% BYTE_LEN) != 0)) {
 			ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
-			ZUC_LOG_ERR("Data Length or offset");
+			ZUC_LOG(ERR, "Data Length or offset");
 			break;
 		}
 
@@ -198,7 +197,7 @@ process_zuc_cipher_op(struct rte_crypto_op **ops,
 				(ops[i]->sym->m_dst != NULL &&
 				!rte_pktmbuf_is_contiguous(
 						ops[i]->sym->m_dst))) {
-			ZUC_LOG_ERR("PMD supports only contiguous mbufs, "
+			ZUC_LOG(ERR, "PMD supports only contiguous mbufs, "
 				"op (%p) provides noncontiguous mbuf as "
 				"source/destination buffer.\n", ops[i]);
 			ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
@@ -246,7 +245,7 @@ process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops,
 		/* Data must be byte aligned */
 		if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) {
 			ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
-			ZUC_LOG_ERR("Offset");
+			ZUC_LOG(ERR, "Offset");
 			break;
 		}
 
@@ -461,7 +460,7 @@ cryptodev_zuc_create(const char *name,
 
 	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
 	if (dev == NULL) {
-		ZUC_LOG_ERR("failed to create cryptodev vdev");
+		ZUC_LOG(ERR, "failed to create cryptodev vdev");
 		goto init_error;
 	}
 
@@ -479,11 +478,10 @@ cryptodev_zuc_create(const char *name,
 	internals = dev->data->dev_private;
 
 	internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
-	internals->max_nb_sessions = init_params->max_nb_sessions;
 
 	return 0;
 init_error:
-	ZUC_LOG_ERR("driver %s: cryptodev_zuc_create failed",
+	ZUC_LOG(ERR, "driver %s: failed",
 			init_params->name);
 
 	cryptodev_zuc_remove(vdev);
@@ -497,8 +495,7 @@ cryptodev_zuc_probe(struct rte_vdev_device *vdev)
 		"",
 		sizeof(struct zuc_private),
 		rte_socket_id(),
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
-		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
+		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 	};
 	const char *name;
 	const char *input_args;
@@ -541,7 +538,11 @@ static struct cryptodev_driver zuc_crypto_drv;
 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ZUC_PMD, cryptodev_zuc_pmd_drv);
 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ZUC_PMD,
 	"max_nb_queue_pairs=<int> "
-	"max_nb_sessions=<int> "
 	"socket_id=<int>");
 RTE_PMD_REGISTER_CRYPTO_DRIVER(zuc_crypto_drv, cryptodev_zuc_pmd_drv.driver,
 		cryptodev_driver_id);
+
+RTE_INIT(zuc_init_log)
+{
+	zuc_logtype_driver = rte_log_register("pmd.crypto.zuc");
+}
diff --git a/drivers/crypto/zuc/rte_zuc_pmd_ops.c b/drivers/crypto/zuc/rte_zuc_pmd_ops.c
index 8abac898..6da39654 100644
--- a/drivers/crypto/zuc/rte_zuc_pmd_ops.c
+++ b/drivers/crypto/zuc/rte_zuc_pmd_ops.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2016-2017 Intel Corporation
+ * Copyright(c) 2016-2018 Intel Corporation
  */
 
 #include <string.h>
@@ -130,7 +130,8 @@ zuc_pmd_info_get(struct rte_cryptodev *dev,
 	if (dev_info != NULL) {
 		dev_info->driver_id = dev->driver_id;
 		dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
-		dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
+		/* No limit of number of sessions */
+		dev_info->sym.max_nb_sessions = 0;
 		dev_info->feature_flags = dev->feature_flags;
 		dev_info->capabilities = zuc_pmd_capabilities;
 	}
@@ -172,13 +173,13 @@ zuc_pmd_qp_create_processed_ops_ring(struct zuc_qp *qp,
 	r = rte_ring_lookup(qp->name);
 	if (r) {
 		if (rte_ring_get_size(r) >= ring_size) {
-			ZUC_LOG_INFO("Reusing existing ring %s"
+			ZUC_LOG(INFO, "Reusing existing ring %s"
 					" for processed packets",
 					 qp->name);
 			return r;
 		}
 
-		ZUC_LOG_ERR("Unable to reuse existing ring %s"
+		ZUC_LOG(ERR, "Unable to reuse existing ring %s"
 				" for processed packets",
 				 qp->name);
 		return NULL;
@@ -230,22 +231,6 @@ qp_setup_cleanup:
 	return -1;
 }
 
-/** Start queue pair */
-static int
-zuc_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
-/** Stop queue pair */
-static int
-zuc_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
-		__rte_unused uint16_t queue_pair_id)
-{
-	return -ENOTSUP;
-}
-
 /** Return the number of allocated queue pairs */
 static uint32_t
 zuc_pmd_qp_count(struct rte_cryptodev *dev)
@@ -255,14 +240,14 @@ zuc_pmd_qp_count(struct rte_cryptodev *dev)
 
 /** Returns the size of the ZUC session structure */
 static unsigned
-zuc_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
+zuc_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
 	return sizeof(struct zuc_session);
 }
 
 /** Configure a ZUC session from a crypto xform chain */
 static int
-zuc_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
+zuc_pmd_sym_session_configure(struct rte_cryptodev *dev __rte_unused,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *sess,
 		struct rte_mempool *mempool)
@@ -271,26 +256,27 @@ zuc_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
 	int ret;
 
 	if (unlikely(sess == NULL)) {
-		ZUC_LOG_ERR("invalid session struct");
+		ZUC_LOG(ERR, "invalid session struct");
 		return -EINVAL;
 	}
 
 	if (rte_mempool_get(mempool, &sess_private_data)) {
-		CDEV_LOG_ERR(
+		ZUC_LOG(ERR,
 			"Couldn't get object from session mempool");
+
 		return -ENOMEM;
 	}
 
 	ret = zuc_set_session_parameters(sess_private_data, xform);
 	if (ret != 0) {
-		ZUC_LOG_ERR("failed configure session parameters");
+		ZUC_LOG(ERR, "failed configure session parameters");
 
 		/* Return session to mempool */
 		rte_mempool_put(mempool, sess_private_data);
 		return ret;
 	}
 
-	set_session_private_data(sess, dev->driver_id,
+	set_sym_session_private_data(sess, dev->driver_id,
 		sess_private_data);
 
 	return 0;
@@ -298,17 +284,17 @@ zuc_pmd_session_configure(struct rte_cryptodev *dev __rte_unused,
 
 /** Clear the memory of session so it doesn't leave key material behind */
 static void
-zuc_pmd_session_clear(struct rte_cryptodev *dev,
+zuc_pmd_sym_session_clear(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess)
 {
 	uint8_t index = dev->driver_id;
-	void *sess_priv = get_session_private_data(sess, index);
+	void *sess_priv = get_sym_session_private_data(sess, index);
 
 	/* Zero out the whole structure */
 	if (sess_priv) {
 		memset(sess_priv, 0, sizeof(struct zuc_session));
 		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-		set_session_private_data(sess, index, NULL);
+		set_sym_session_private_data(sess, index, NULL);
 		rte_mempool_put(sess_mp, sess_priv);
 	}
 }
@@ -326,13 +312,11 @@ struct rte_cryptodev_ops zuc_pmd_ops = {
 
 		.queue_pair_setup   = zuc_pmd_qp_setup,
 		.queue_pair_release = zuc_pmd_qp_release,
-		.queue_pair_start   = zuc_pmd_qp_start,
-		.queue_pair_stop    = zuc_pmd_qp_stop,
 		.queue_pair_count   = zuc_pmd_qp_count,
 
-		.session_get_size   = zuc_pmd_session_get_size,
-		.session_configure  = zuc_pmd_session_configure,
-		.session_clear      = zuc_pmd_session_clear
+		.sym_session_get_size   = zuc_pmd_sym_session_get_size,
+		.sym_session_configure  = zuc_pmd_sym_session_configure,
+		.sym_session_clear      = zuc_pmd_sym_session_clear
 };
 
 struct rte_cryptodev_ops *rte_zuc_pmd_ops = &zuc_pmd_ops;
diff --git a/drivers/crypto/zuc/rte_zuc_pmd_private.h b/drivers/crypto/zuc/rte_zuc_pmd_private.h
index b83c4a04..5e5906dd 100644
--- a/drivers/crypto/zuc/rte_zuc_pmd_private.h
+++ b/drivers/crypto/zuc/rte_zuc_pmd_private.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2016-2017 Intel Corporation
+ * Copyright(c) 2016-2018 Intel Corporation
  */
 
 #ifndef _RTE_ZUC_PMD_PRIVATE_H_
@@ -10,25 +10,12 @@
 #define CRYPTODEV_NAME_ZUC_PMD		crypto_zuc
 /**< KASUMI PMD device name */
 
-#define ZUC_LOG_ERR(fmt, args...) \
-	RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",  \
-			RTE_STR(CRYPTODEV_NAME_ZUC_PMD), \
-			__func__, __LINE__, ## args)
-
-#ifdef RTE_LIBRTE_ZUC_DEBUG
-#define ZUC_LOG_INFO(fmt, args...) \
-	RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_ZUC_PMD), \
-			__func__, __LINE__, ## args)
-
-#define ZUC_LOG_DBG(fmt, args...) \
-	RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
-			RTE_STR(CRYPTODEV_NAME_ZUC_PMD), \
-			__func__, __LINE__, ## args)
-#else
-#define ZUC_LOG_INFO(fmt, args...)
-#define ZUC_LOG_DBG(fmt, args...)
-#endif
+/** ZUC PMD LOGTYPE DRIVER */
+int zuc_logtype_driver;
+#define ZUC_LOG(level, fmt, ...)  \
+	rte_log(RTE_LOG_ ## level, zuc_logtype_driver,  \
+			"%s()... line %u: " fmt "\n", __func__, __LINE__,  \
+				## __VA_ARGS__)
 
 #define ZUC_IV_KEY_LENGTH 16
 #define ZUC_DIGEST_LENGTH 4
@@ -37,8 +24,6 @@
 struct zuc_private {
 	unsigned max_nb_queue_pairs;
 	/**< Max number of queue pairs supported by device */
-	unsigned max_nb_sessions;
-	/**< Max number of sessions supported by device */
 };
 
 /** ZUC buffer queue pair */
diff --git a/drivers/event/dpaa2/dpaa2_eventdev.c b/drivers/event/dpaa2/dpaa2_eventdev.c
index cd801bfb..ea1e5cc6 100644
--- a/drivers/event/dpaa2/dpaa2_eventdev.c
+++ b/drivers/event/dpaa2/dpaa2_eventdev.c
@@ -822,9 +822,7 @@ static struct rte_vdev_driver vdev_eventdev_dpaa2_pmd = {
 
 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_DPAA2_PMD, vdev_eventdev_dpaa2_pmd);
 
-RTE_INIT(dpaa2_eventdev_init_log);
-static void
-dpaa2_eventdev_init_log(void)
+RTE_INIT(dpaa2_eventdev_init_log)
 {
 	dpaa2_logtype_event = rte_log_register("pmd.event.dpaa2");
 	if (dpaa2_logtype_event >= 0)
diff --git a/drivers/event/dpaa2/dpaa2_eventdev_logs.h b/drivers/event/dpaa2/dpaa2_eventdev_logs.h
index 48f1abd1..a2c2060c 100644
--- a/drivers/event/dpaa2/dpaa2_eventdev_logs.h
+++ b/drivers/event/dpaa2/dpaa2_eventdev_logs.h
@@ -16,7 +16,7 @@ extern int dpaa2_logtype_event;
 	rte_log(RTE_LOG_DEBUG, dpaa2_logtype_event, "dpaa2_event: %s(): " \
 		fmt "\n", __func__, ##args)
 
-#define EVENTDEV_INIT_FUNC_TRACE() DPAA2_EVENTDEV_LOG(DEBUG, " >>")
+#define EVENTDEV_INIT_FUNC_TRACE() DPAA2_EVENTDEV_DEBUG(" >>")
 
 #define DPAA2_EVENTDEV_INFO(fmt, args...) \
 	DPAA2_EVENTDEV_LOG(INFO, fmt, ## args)
diff --git a/drivers/event/octeontx/ssovf_evdev.c b/drivers/event/octeontx/ssovf_evdev.c
index 2df70b52..16a3a04b 100644
--- a/drivers/event/octeontx/ssovf_evdev.c
+++ b/drivers/event/octeontx/ssovf_evdev.c
@@ -23,9 +23,7 @@
 int otx_logtype_ssovf;
 static uint8_t timvf_enable_stats;
 
-RTE_INIT(otx_ssovf_init_log);
-static void
-otx_ssovf_init_log(void)
+RTE_INIT(otx_ssovf_init_log)
 {
 	otx_logtype_ssovf = rte_log_register("pmd.event.octeontx");
 	if (otx_logtype_ssovf >= 0)
@@ -476,14 +474,9 @@ static int
 ssovf_eth_rx_adapter_start(const struct rte_eventdev *dev,
 					const struct rte_eth_dev *eth_dev)
 {
-	int ret;
-	const struct octeontx_nic *nic = eth_dev->data->dev_private;
 	RTE_SET_USED(dev);
+	RTE_SET_USED(eth_dev);
 
-	ret = strncmp(eth_dev->data->name, "eth_octeontx", 12);
-	if (ret)
-		return 0;
-	octeontx_pki_port_start(nic->port_id);
 	return 0;
 }
 
@@ -492,14 +485,9 @@ static int
 ssovf_eth_rx_adapter_stop(const struct rte_eventdev *dev,
 		const struct rte_eth_dev *eth_dev)
 {
-	int ret;
-	const struct octeontx_nic *nic = eth_dev->data->dev_private;
 	RTE_SET_USED(dev);
+	RTE_SET_USED(eth_dev);
 
-	ret = strncmp(eth_dev->data->name, "eth_octeontx", 12);
-	if (ret)
-		return 0;
-	octeontx_pki_port_stop(nic->port_id);
 	return 0;
 }
 
diff --git a/drivers/event/octeontx/ssovf_worker.c b/drivers/event/octeontx/ssovf_worker.c
index d8bbc714..fffa9024 100644
--- a/drivers/event/octeontx/ssovf_worker.c
+++ b/drivers/event/octeontx/ssovf_worker.c
@@ -204,6 +204,8 @@ ssows_flush_events(struct ssows *ws, uint8_t queue_id,
 	uint32_t reg_off;
 	struct rte_event ev;
 	uint64_t enable, aq_cnt = 1, cq_ds_cnt = 1;
+	uint64_t get_work0, get_work1;
+	uint64_t sched_type_queue;
 	uint8_t *base = ssovf_bar(OCTEONTX_SSO_GROUP, queue_id, 0);
 
 	enable = ssovf_read64(base + SSO_VHGRP_QCTL);
@@ -219,7 +221,20 @@ ssows_flush_events(struct ssows *ws, uint8_t queue_id,
 		cq_ds_cnt = ssovf_read64(base + SSO_VHGRP_INT_CNT);
 		/* Extract cq and ds count */
 		cq_ds_cnt &= 0x1FFF1FFF0000;
-		ssows_get_work(ws, &ev);
+
+		ssovf_load_pair(get_work0, get_work1, ws->base + reg_off);
+
+		sched_type_queue = (get_work0 >> 32) & 0xfff;
+		ws->cur_tt = sched_type_queue & 0x3;
+		ws->cur_grp = sched_type_queue >> 2;
+		sched_type_queue = sched_type_queue << 38;
+		ev.event = sched_type_queue | (get_work0 & 0xffffffff);
+		if (get_work1 && ev.event_type == RTE_EVENT_TYPE_ETHDEV)
+			ev.mbuf = ssovf_octeontx_wqe_to_pkt(get_work1,
+					(ev.event >> 20) & 0x7F);
+		else
+			ev.u64 = get_work1;
+
 		if (fn != NULL && ev.u64 != 0)
 			fn(arg, ev);
 	}
diff --git a/drivers/event/octeontx/ssovf_worker.h b/drivers/event/octeontx/ssovf_worker.h
index d55018a9..7c7306b5 100644
--- a/drivers/event/octeontx/ssovf_worker.h
+++ b/drivers/event/octeontx/ssovf_worker.h
@@ -28,11 +28,11 @@ ssovf_octeontx_wqe_to_pkt(uint64_t work, uint16_t port_info)
 {
 	struct rte_mbuf *mbuf;
 	octtx_wqe_t *wqe = (octtx_wqe_t *)(uintptr_t)work;
-	rte_prefetch_non_temporal(wqe);
 
 	/* Get mbuf from wqe */
 	mbuf = (struct rte_mbuf *)((uintptr_t)wqe -
 			OCTTX_PACKET_WQE_SKIP);
+	rte_prefetch_non_temporal(mbuf);
 	mbuf->packet_type =
 		ptype_table[wqe->s.w2.lcty][wqe->s.w2.lety][wqe->s.w2.lfty];
 	mbuf->data_off = RTE_PTR_DIFF(wqe->s.w3.addr, mbuf->buf_addr);
diff --git a/drivers/event/octeontx/timvf_evdev.c b/drivers/event/octeontx/timvf_evdev.c
index c4fbd2d8..abbc9a77 100644
--- a/drivers/event/octeontx/timvf_evdev.c
+++ b/drivers/event/octeontx/timvf_evdev.c
@@ -6,9 +6,7 @@
 
 int otx_logtype_timvf;
 
-RTE_INIT(otx_timvf_init_log);
-static void
-otx_timvf_init_log(void)
+RTE_INIT(otx_timvf_init_log)
 {
 	otx_logtype_timvf = rte_log_register("pmd.event.octeontx.timer");
 	if (otx_logtype_timvf >= 0)
@@ -174,7 +172,7 @@ timvf_ring_start(const struct rte_event_timer_adapter *adptr)
 	if (use_fpa) {
 		pool = (uintptr_t)((struct rte_mempool *)
 				timr->chunk_pool)->pool_id;
-		ret = octeontx_fpa_bufpool_gpool(pool);
+		ret = octeontx_fpa_bufpool_gaura(pool);
 		if (ret < 0) {
 			timvf_log_dbg("Unable to get gaura id");
 			ret = -ENOMEM;
diff --git a/drivers/event/opdl/opdl_evdev.c b/drivers/event/opdl/opdl_evdev.c
index ef9fb30c..a4f0bc8b 100644
--- a/drivers/event/opdl/opdl_evdev.c
+++ b/drivers/event/opdl/opdl_evdev.c
@@ -753,10 +753,7 @@ static struct rte_vdev_driver evdev_opdl_pmd_drv = {
 	.remove = opdl_remove
 };
 
-RTE_INIT(opdl_init_log);
-
-static void
-opdl_init_log(void)
+RTE_INIT(opdl_init_log)
 {
 	opdl_logtype_driver = rte_log_register("pmd.event.opdl.driver");
 	if (opdl_logtype_driver >= 0)
diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
index 10f0e1ad..a6bb9138 100644
--- a/drivers/event/sw/sw_evdev.c
+++ b/drivers/event/sw/sw_evdev.c
@@ -361,9 +361,99 @@ sw_init_qid_iqs(struct sw_evdev *sw)
 	}
 }
 
+static int
+sw_qids_empty(struct sw_evdev *sw)
+{
+	unsigned int i, j;
+
+	for (i = 0; i < sw->qid_count; i++) {
+		for (j = 0; j < SW_IQS_MAX; j++) {
+			if (iq_count(&sw->qids[i].iq[j]))
+				return 0;
+		}
+	}
+
+	return 1;
+}
+
+static int
+sw_ports_empty(struct sw_evdev *sw)
+{
+	unsigned int i;
+
+	for (i = 0; i < sw->port_count; i++) {
+		if ((rte_event_ring_count(sw->ports[i].rx_worker_ring)) ||
+		     rte_event_ring_count(sw->ports[i].cq_worker_ring))
+			return 0;
+	}
+
+	return 1;
+}
+
 static void
-sw_clean_qid_iqs(struct sw_evdev *sw)
+sw_drain_ports(struct rte_eventdev *dev)
 {
+	struct sw_evdev *sw = sw_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	unsigned int i;
+	uint8_t dev_id;
+	void *arg;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	for (i = 0; i < sw->port_count; i++) {
+		struct rte_event ev;
+
+		while (rte_event_dequeue_burst(dev_id, i, &ev, 1, 0)) {
+			if (flush)
+				flush(dev_id, ev, arg);
+
+			ev.op = RTE_EVENT_OP_RELEASE;
+			rte_event_enqueue_burst(dev_id, i, &ev, 1);
+		}
+	}
+}
+
+static void
+sw_drain_queue(struct rte_eventdev *dev, struct sw_iq *iq)
+{
+	struct sw_evdev *sw = sw_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	uint8_t dev_id;
+	void *arg;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (iq_count(iq) > 0) {
+		struct rte_event ev;
+
+		iq_dequeue_burst(sw, iq, &ev, 1);
+
+		if (flush)
+			flush(dev_id, ev, arg);
+	}
+}
+
+static void
+sw_drain_queues(struct rte_eventdev *dev)
+{
+	struct sw_evdev *sw = sw_pmd_priv(dev);
+	unsigned int i, j;
+
+	for (i = 0; i < sw->qid_count; i++) {
+		for (j = 0; j < SW_IQS_MAX; j++)
+			sw_drain_queue(dev, &sw->qids[i].iq[j]);
+	}
+}
+
+static void
+sw_clean_qid_iqs(struct rte_eventdev *dev)
+{
+	struct sw_evdev *sw = sw_pmd_priv(dev);
 	int i, j;
 
 	/* Release the IQ memory of all configured qids */
@@ -729,10 +819,30 @@ static void
 sw_stop(struct rte_eventdev *dev)
 {
 	struct sw_evdev *sw = sw_pmd_priv(dev);
-	sw_clean_qid_iqs(sw);
+	int32_t runstate;
+
+	/* Stop the scheduler if it's running */
+	runstate = rte_service_runstate_get(sw->service_id);
+	if (runstate == 1)
+		rte_service_runstate_set(sw->service_id, 0);
+
+	while (rte_service_may_be_active(sw->service_id))
+		rte_pause();
+
+	/* Flush all events out of the device */
+	while (!(sw_qids_empty(sw) && sw_ports_empty(sw))) {
+		sw_event_schedule(dev);
+		sw_drain_ports(dev);
+		sw_drain_queues(dev);
+	}
+
+	sw_clean_qid_iqs(dev);
 	sw_xstats_uninit(sw);
 	sw->started = 0;
 	rte_smp_wmb();
+
+	if (runstate == 1)
+		rte_service_runstate_set(sw->service_id, 1);
 }
 
 static int
@@ -964,9 +1074,7 @@ RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> "
 /* declared extern in header, for access from other .c files */
 int eventdev_sw_log_level;
 
-RTE_INIT(evdev_sw_init_log);
-static void
-evdev_sw_init_log(void)
+RTE_INIT(evdev_sw_init_log)
 {
 	eventdev_sw_log_level = rte_log_register("pmd.event.sw");
 	if (eventdev_sw_log_level >= 0)
diff --git a/drivers/event/sw/sw_evdev_selftest.c b/drivers/event/sw/sw_evdev_selftest.c
index 78d30e07..c40912db 100644
--- a/drivers/event/sw/sw_evdev_selftest.c
+++ b/drivers/event/sw/sw_evdev_selftest.c
@@ -28,6 +28,7 @@
 #define MAX_PORTS 16
 #define MAX_QIDS 16
 #define NUM_PACKETS (1<<18)
+#define DEQUEUE_DEPTH 128
 
 static int evdev;
 
@@ -147,7 +148,7 @@ init(struct test *t, int nb_queues, int nb_ports)
 			.nb_event_ports = nb_ports,
 			.nb_event_queue_flows = 1024,
 			.nb_events_limit = 4096,
-			.nb_event_port_dequeue_depth = 128,
+			.nb_event_port_dequeue_depth = DEQUEUE_DEPTH,
 			.nb_event_port_enqueue_depth = 128,
 	};
 	int ret;
@@ -2807,6 +2808,78 @@ err:
 	return -1;
 }
 
+static void
+flush(uint8_t dev_id __rte_unused, struct rte_event event, void *arg)
+{
+	*((uint8_t *) arg) += (event.u64 == 0xCA11BACC) ? 1 : 0;
+}
+
+static int
+dev_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	const struct rte_event new_ev = {
+		.op = RTE_EVENT_OP_NEW,
+		.u64 = 0xCA11BACC,
+		.queue_id = 0
+	};
+	struct rte_event ev = new_ev;
+	uint8_t count = 0;
+	int i;
+
+	if (init(t, 1, 1) < 0 ||
+	    create_ports(t, 1) < 0 ||
+	    create_atomic_qids(t, 1) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	/* Link the queue so *_start() doesn't error out */
+	if (rte_event_port_link(evdev, t->port[0], NULL, NULL, 0) != 1) {
+		printf("%d: Error linking queue to port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < DEQUEUE_DEPTH + 1; i++) {
+		if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Schedule the events from the port to the IQ. At least one event
+	 * should be remaining in the queue.
+	 */
+	rte_service_run_iter_on_app_lcore(t->service_id, 1);
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, &count)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup(t);
+
+	if (count == 0) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	rte_event_dev_dump(evdev, stdout);
+	cleanup(t);
+	return -1;
+}
+
 static int
 worker_loopback_worker_fn(void *arg)
 {
@@ -3211,6 +3284,12 @@ test_sw_eventdev(void)
 		printf("ERROR - Head-of-line-blocking test FAILED.\n");
 		goto test_fail;
 	}
+	printf("*** Running Stop Flush test...\n");
+	ret = dev_stop_flush(t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		goto test_fail;
+	}
 	if (rte_lcore_count() >= 3) {
 		printf("*** Running Worker loopback test...\n");
 		ret = worker_loopback(t, 0);
diff --git a/drivers/mempool/dpaa2/dpaa2_hw_mempool.c b/drivers/mempool/dpaa2/dpaa2_hw_mempool.c
index e12a0ec8..7d0435f5 100644
--- a/drivers/mempool/dpaa2/dpaa2_hw_mempool.c
+++ b/drivers/mempool/dpaa2/dpaa2_hw_mempool.c
@@ -442,9 +442,7 @@ struct rte_mempool_ops dpaa2_mpool_ops = {
 
 MEMPOOL_REGISTER_OPS(dpaa2_mpool_ops);
 
-RTE_INIT(dpaa2_mempool_init_log);
-static void
-dpaa2_mempool_init_log(void)
+RTE_INIT(dpaa2_mempool_init_log)
 {
 	dpaa2_logtype_mempool = rte_log_register("mempool.dpaa2");
 	if (dpaa2_logtype_mempool >= 0)
diff --git a/drivers/mempool/octeontx/octeontx_fpavf.c b/drivers/mempool/octeontx/octeontx_fpavf.c
index 7aecaa85..4cf387e8 100644
--- a/drivers/mempool/octeontx/octeontx_fpavf.c
+++ b/drivers/mempool/octeontx/octeontx_fpavf.c
@@ -108,9 +108,7 @@ static struct octeontx_fpadev fpadev;
 int octeontx_logtype_fpavf;
 int octeontx_logtype_fpavf_mbox;
 
-RTE_INIT(otx_pool_init_log);
-static void
-otx_pool_init_log(void)
+RTE_INIT(otx_pool_init_log)
 {
 	octeontx_logtype_fpavf = rte_log_register("pmd.mempool.octeontx");
 	if (octeontx_logtype_fpavf >= 0)
@@ -243,7 +241,7 @@ octeontx_fpapf_pool_setup(unsigned int gpool, unsigned int buf_size,
 		POOL_LTYPE(0x2) | POOL_STYPE(0) | POOL_SET_NAT_ALIGN |
 		POOL_ENA;
 
-	cfg.aid = 0;
+	cfg.aid = FPA_AURA_IDX(gpool);
 	cfg.pool_cfg = reg;
 	cfg.pool_stack_base = phys_addr;
 	cfg.pool_stack_end = phys_addr + memsz;
@@ -327,7 +325,7 @@ octeontx_fpapf_aura_attach(unsigned int gpool_index)
 	hdr.vfid = gpool_index;
 	hdr.res_code = 0;
 	memset(&cfg, 0x0, sizeof(struct octeontx_mbox_fpa_cfg));
-	cfg.aid = gpool_index; /* gpool is guara */
+	cfg.aid = FPA_AURA_IDX(gpool_index);
 
 	ret = octeontx_mbox_send(&hdr, &cfg,
 					sizeof(struct octeontx_mbox_fpa_cfg),
@@ -335,7 +333,8 @@ octeontx_fpapf_aura_attach(unsigned int gpool_index)
 	if (ret < 0) {
 		fpavf_log_err("Could not attach fpa ");
 		fpavf_log_err("aura %d to pool %d. Err=%d. FuncErr=%d\n",
-			      gpool_index, gpool_index, ret, hdr.res_code);
+			      FPA_AURA_IDX(gpool_index), gpool_index, ret,
+			      hdr.res_code);
 		ret = -EACCES;
 		goto err;
 	}
@@ -355,14 +354,15 @@ octeontx_fpapf_aura_detach(unsigned int gpool_index)
 		goto err;
 	}
 
-	cfg.aid = gpool_index; /* gpool is gaura */
+	cfg.aid = FPA_AURA_IDX(gpool_index);
 	hdr.coproc = FPA_COPROC;
 	hdr.msg = FPA_DETACHAURA;
 	hdr.vfid = gpool_index;
 	ret = octeontx_mbox_send(&hdr, &cfg, sizeof(cfg), NULL, 0);
 	if (ret < 0) {
 		fpavf_log_err("Couldn't detach FPA aura %d Err=%d FuncErr=%d\n",
-			      gpool_index, ret, hdr.res_code);
+			      FPA_AURA_IDX(gpool_index), ret,
+			      hdr.res_code);
 		ret = -EINVAL;
 	}
 
@@ -469,6 +469,7 @@ octeontx_fpa_bufpool_free_count(uintptr_t handle)
 {
 	uint64_t cnt, limit, avail;
 	uint8_t gpool;
+	uint16_t gaura;
 	uintptr_t pool_bar;
 
 	if (unlikely(!octeontx_fpa_handle_valid(handle)))
@@ -476,14 +477,16 @@ octeontx_fpa_bufpool_free_count(uintptr_t handle)
 
 	/* get the gpool */
 	gpool = octeontx_fpa_bufpool_gpool(handle);
+	/* get the aura */
+	gaura = octeontx_fpa_bufpool_gaura(handle);
 
 	/* Get pool bar address from handle */
 	pool_bar = handle & ~(uint64_t)FPA_GPOOL_MASK;
 
 	cnt = fpavf_read64((void *)((uintptr_t)pool_bar +
-				FPA_VF_VHAURA_CNT(gpool)));
+				FPA_VF_VHAURA_CNT(gaura)));
 	limit = fpavf_read64((void *)((uintptr_t)pool_bar +
-				FPA_VF_VHAURA_CNT_LIMIT(gpool)));
+				FPA_VF_VHAURA_CNT_LIMIT(gaura)));
 
 	avail = fpavf_read64((void *)((uintptr_t)pool_bar +
 				FPA_VF_VHPOOL_AVAILABLE(gpool)));
@@ -496,6 +499,7 @@ octeontx_fpa_bufpool_create(unsigned int object_size, unsigned int object_count,
 				unsigned int buf_offset, int node_id)
 {
 	unsigned int gpool;
+	unsigned int gaura;
 	uintptr_t gpool_handle;
 	uintptr_t pool_bar;
 	int res;
@@ -545,16 +549,18 @@ octeontx_fpa_bufpool_create(unsigned int object_size, unsigned int object_count,
 		goto error_pool_destroy;
 	}
 
+	gaura = FPA_AURA_IDX(gpool);
+
 	/* Release lock */
 	rte_spinlock_unlock(&fpadev.lock);
 
 	/* populate AURA registers */
 	fpavf_write64(object_count, (void *)((uintptr_t)pool_bar +
-			 FPA_VF_VHAURA_CNT(gpool)));
+			 FPA_VF_VHAURA_CNT(gaura)));
 	fpavf_write64(object_count, (void *)((uintptr_t)pool_bar +
-			 FPA_VF_VHAURA_CNT_LIMIT(gpool)));
+			 FPA_VF_VHAURA_CNT_LIMIT(gaura)));
 	fpavf_write64(object_count + 1, (void *)((uintptr_t)pool_bar +
-			 FPA_VF_VHAURA_CNT_THRESHOLD(gpool)));
+			 FPA_VF_VHAURA_CNT_THRESHOLD(gaura)));
 
 	octeontx_fpapf_start_count(gpool);
 
@@ -581,6 +587,7 @@ octeontx_fpa_bufpool_destroy(uintptr_t handle, int node_id)
 	uint64_t sz;
 	uint64_t cnt, avail;
 	uint8_t gpool;
+	uint16_t gaura;
 	uintptr_t pool_bar;
 	int ret;
 
@@ -594,13 +601,15 @@ octeontx_fpa_bufpool_destroy(uintptr_t handle, int node_id)
 
 	/* get the pool */
 	gpool = octeontx_fpa_bufpool_gpool(handle);
+	/* get the aura */
+	gaura = octeontx_fpa_bufpool_gaura(handle);
 
 	/* Get pool bar address from handle */
 	pool_bar = handle & ~(uint64_t)FPA_GPOOL_MASK;
 
 	 /* Check for no outstanding buffers */
 	cnt = fpavf_read64((void *)((uintptr_t)pool_bar +
-					FPA_VF_VHAURA_CNT(gpool)));
+					FPA_VF_VHAURA_CNT(gaura)));
 	if (cnt) {
 		fpavf_log_dbg("buffer exist in pool cnt %" PRId64 "\n", cnt);
 		return -EBUSY;
@@ -613,9 +622,9 @@ octeontx_fpa_bufpool_destroy(uintptr_t handle, int node_id)
 
 	/* Prepare to empty the entire POOL */
 	fpavf_write64(avail, (void *)((uintptr_t)pool_bar +
-			 FPA_VF_VHAURA_CNT_LIMIT(gpool)));
+			 FPA_VF_VHAURA_CNT_LIMIT(gaura)));
 	fpavf_write64(avail + 1, (void *)((uintptr_t)pool_bar +
-			 FPA_VF_VHAURA_CNT_THRESHOLD(gpool)));
+			 FPA_VF_VHAURA_CNT_THRESHOLD(gaura)));
 
 	/* Empty the pool */
 	/* Invalidate the POOL */
@@ -627,11 +636,11 @@ octeontx_fpa_bufpool_destroy(uintptr_t handle, int node_id)
 		/* Yank a buffer from the pool */
 		node = (void *)(uintptr_t)
 			fpavf_read64((void *)
-				    (pool_bar + FPA_VF_VHAURA_OP_ALLOC(gpool)));
+				    (pool_bar + FPA_VF_VHAURA_OP_ALLOC(gaura)));
 
 		if (node == NULL) {
 			fpavf_log_err("GAURA[%u] missing %" PRIx64 " buf\n",
-				      gpool, avail);
+				      gaura, avail);
 			break;
 		}
 
@@ -665,9 +674,9 @@ octeontx_fpa_bufpool_destroy(uintptr_t handle, int node_id)
 
 	/* Deactivate the AURA */
 	fpavf_write64(0, (void *)((uintptr_t)pool_bar +
-			FPA_VF_VHAURA_CNT_LIMIT(gpool)));
+			FPA_VF_VHAURA_CNT_LIMIT(gaura)));
 	fpavf_write64(0, (void *)((uintptr_t)pool_bar +
-			FPA_VF_VHAURA_CNT_THRESHOLD(gpool)));
+			FPA_VF_VHAURA_CNT_THRESHOLD(gaura)));
 
 	ret = octeontx_fpapf_aura_detach(gpool);
 	if (ret) {
diff --git a/drivers/mempool/octeontx/octeontx_fpavf.h b/drivers/mempool/octeontx/octeontx_fpavf.h
index b76f40e7..b00be137 100644
--- a/drivers/mempool/octeontx/octeontx_fpavf.h
+++ b/drivers/mempool/octeontx/octeontx_fpavf.h
@@ -14,6 +14,7 @@
 
 #define	FPA_VF_MAX			32
 #define FPA_GPOOL_MASK			(FPA_VF_MAX-1)
+#define FPA_GAURA_SHIFT			4
 
 /* FPA VF register offsets */
 #define FPA_VF_INT(x)			(0x200ULL | ((x) << 22))
@@ -36,6 +37,7 @@
 #define FPA_VF_FREE_ADDRS_S(x, y, z)	\
 	((x) | (((y) & 0x1ff) << 3) | ((((z) & 1)) << 14))
 
+#define FPA_AURA_IDX(gpool)			(gpool << FPA_GAURA_SHIFT)
 /* FPA VF register offsets from VF_BAR4, size 2 MByte */
 #define	FPA_VF_MSIX_VEC_ADDR		0x00000
 #define	FPA_VF_MSIX_VEC_CTL		0x00008
@@ -102,4 +104,11 @@ octeontx_fpa_bufpool_gpool(uintptr_t handle)
 {
 	return (uint8_t)handle & FPA_GPOOL_MASK;
 }
+
+static __rte_always_inline uint16_t
+octeontx_fpa_bufpool_gaura(uintptr_t handle)
+{
+	return octeontx_fpa_bufpool_gpool(handle) << FPA_GAURA_SHIFT;
+}
+
 #endif	/* __OCTEONTX_FPAVF_H__ */
diff --git a/drivers/meson.build b/drivers/meson.build
index ac6c9729..f94e2fe6 100644
--- a/drivers/meson.build
+++ b/drivers/meson.build
@@ -11,6 +11,10 @@ driver_classes = ['common',
 	       'event',   # depends on common, bus, mempool and net.
 	       'raw']     # depends on common, bus, mempool, net and event.
 
+default_cflags = machine_args
+if cc.has_argument('-Wno-format-truncation')
+	default_cflags += '-Wno-format-truncation'
+endif
 foreach class:driver_classes
 	drivers = []
 	std_deps = []
@@ -31,7 +35,7 @@ foreach class:driver_classes
 		allow_experimental_apis = false
 		sources = []
 		objs = []
-		cflags = machine_args
+		cflags = default_cflags
 		includes = [include_directories(drv_path)]
 		# set up internal deps. Drivers can append/override as necessary
 		deps = std_deps
@@ -58,6 +62,10 @@ foreach class:driver_classes
 			shared_objs = []
 			static_objs = []
 			foreach d:deps
+				if not is_variable('shared_rte_' + d)
+					error('Missing dependency ' + d +
+						' for driver ' + lib_name)
+				endif
 				shared_objs += [get_variable('shared_rte_' + d)]
 				static_objs += [get_variable('static_rte_' + d)]
 			endforeach
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index 9f9da665..664398de 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -33,6 +33,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += liquidio
 DIRS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4
 DIRS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5
 DIRS-$(CONFIG_RTE_LIBRTE_MVPP2_PMD) += mvpp2
+DIRS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += netvsc
 DIRS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += nfp
 DIRS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += null
@@ -59,7 +60,7 @@ endif # $(CONFIG_RTE_LIBRTE_SCHED)
 ifeq ($(CONFIG_RTE_LIBRTE_VHOST),y)
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_VHOST) += vhost
 ifeq ($(CONFIG_RTE_EAL_VFIO),y)
-DIRS-$(CONFIG_RTE_LIBRTE_IFCVF_VDPA_PMD) += ifc
+DIRS-$(CONFIG_RTE_LIBRTE_IFC_PMD) += ifc
 endif
 endif # $(CONFIG_RTE_LIBRTE_VHOST)
 
diff --git a/drivers/net/af_packet/rte_eth_af_packet.c b/drivers/net/af_packet/rte_eth_af_packet.c
index ea47abbf..eb3cce3a 100644
--- a/drivers/net/af_packet/rte_eth_af_packet.c
+++ b/drivers/net/af_packet/rte_eth_af_packet.c
@@ -305,6 +305,7 @@ eth_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	dev_info->max_rx_queues = (uint16_t)internals->nb_queues;
 	dev_info->max_tx_queues = (uint16_t)internals->nb_queues;
 	dev_info->min_rx_bufsize = 0;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -935,6 +936,7 @@ rte_pmd_af_packet_probe(struct rte_vdev_device *dev)
 		}
 		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &ops;
+		eth_dev->device = &dev->device;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -1015,9 +1017,7 @@ RTE_PMD_REGISTER_PARAM_STRING(net_af_packet,
 	"framecnt=<int> "
 	"qdisc_bypass=<0|1>");
 
-RTE_INIT(af_packet_init_log);
-static void
-af_packet_init_log(void)
+RTE_INIT(af_packet_init_log)
 {
 	af_packet_logtype = rte_log_register("pmd.net.packet");
 	if (af_packet_logtype >= 0)
diff --git a/drivers/net/ark/Makefile b/drivers/net/ark/Makefile
index f1433bd2..2e232be8 100644
--- a/drivers/net/ark/Makefile
+++ b/drivers/net/ark/Makefile
@@ -1,33 +1,5 @@
-# BSD LICENSE
-#
-# Copyright (c) 2015-2017 Atomic Rules LLC
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# * Redistributions of source code must retain the above copyright
-#   notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above copyright
-#   notice, this list of conditions and the following disclaimer in
-#   the documentation and/or other materials provided with the
-#   distribution.
-# * Neither the name of copyright holder nor the names of its
-#   contributors may be used to endorse or promote products derived
-#   from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2015-2018 Atomic Rules LLC
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
diff --git a/drivers/net/ark/ark_ddm.c b/drivers/net/ark/ark_ddm.c
index 929dc7d1..eea388a1 100644
--- a/drivers/net/ark/ark_ddm.c
+++ b/drivers/net/ark/ark_ddm.c
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #include <unistd.h>
diff --git a/drivers/net/ark/ark_ddm.h b/drivers/net/ark/ark_ddm.h
index f67ad012..b37d1e09 100644
--- a/drivers/net/ark/ark_ddm.h
+++ b/drivers/net/ark/ark_ddm.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_DDM_H_
diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
index 834d8a9e..552ca01a 100644
--- a/drivers/net/ark/ark_ethdev.c
+++ b/drivers/net/ark/ark_ethdev.c
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #include <unistd.h>
diff --git a/drivers/net/ark/ark_ethdev_rx.c b/drivers/net/ark/ark_ethdev_rx.c
index 987d085e..16f0d11e 100644
--- a/drivers/net/ark/ark_ethdev_rx.c
+++ b/drivers/net/ark/ark_ethdev_rx.c
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #include <unistd.h>
diff --git a/drivers/net/ark/ark_ethdev_rx.h b/drivers/net/ark/ark_ethdev_rx.h
index 14678711..0fdd29b1 100644
--- a/drivers/net/ark/ark_ethdev_rx.h
+++ b/drivers/net/ark/ark_ethdev_rx.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_ETHDEV_RX_H_
diff --git a/drivers/net/ark/ark_ethdev_tx.c b/drivers/net/ark/ark_ethdev_tx.c
index 4ef55d10..57188c24 100644
--- a/drivers/net/ark/ark_ethdev_tx.c
+++ b/drivers/net/ark/ark_ethdev_tx.c
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #include <unistd.h>
diff --git a/drivers/net/ark/ark_ethdev_tx.h b/drivers/net/ark/ark_ethdev_tx.h
index 657f895a..e448ce22 100644
--- a/drivers/net/ark/ark_ethdev_tx.h
+++ b/drivers/net/ark/ark_ethdev_tx.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_ETHDEV_TX_H_
diff --git a/drivers/net/ark/ark_ext.h b/drivers/net/ark/ark_ext.h
index 031cfddc..f5af2153 100644
--- a/drivers/net/ark/ark_ext.h
+++ b/drivers/net/ark/ark_ext.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_EXT_H_
diff --git a/drivers/net/ark/ark_global.h b/drivers/net/ark/ark_global.h
index 41eb260e..f820091d 100644
--- a/drivers/net/ark/ark_global.h
+++ b/drivers/net/ark/ark_global.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_GLOBAL_H_
diff --git a/drivers/net/ark/ark_logs.h b/drivers/net/ark/ark_logs.h
index 8aff2963..b90e9f0a 100644
--- a/drivers/net/ark/ark_logs.h
+++ b/drivers/net/ark/ark_logs.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_DEBUG_H_
diff --git a/drivers/net/ark/ark_mpu.c b/drivers/net/ark/ark_mpu.c
index d4ba6dc7..21f840f3 100644
--- a/drivers/net/ark/ark_mpu.c
+++ b/drivers/net/ark/ark_mpu.c
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #include <unistd.h>
diff --git a/drivers/net/ark/ark_mpu.h b/drivers/net/ark/ark_mpu.h
index f6f6c808..92c3e67c 100644
--- a/drivers/net/ark/ark_mpu.h
+++ b/drivers/net/ark/ark_mpu.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_MPU_H_
diff --git a/drivers/net/ark/ark_pktchkr.c b/drivers/net/ark/ark_pktchkr.c
index 2cadaab8..c21003a0 100644
--- a/drivers/net/ark/ark_pktchkr.c
+++ b/drivers/net/ark/ark_pktchkr.c
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #include <getopt.h>
diff --git a/drivers/net/ark/ark_pktchkr.h b/drivers/net/ark/ark_pktchkr.h
index f4025dd6..a50f428b 100644
--- a/drivers/net/ark/ark_pktchkr.h
+++ b/drivers/net/ark/ark_pktchkr.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_PKTCHKR_H_
diff --git a/drivers/net/ark/ark_pktdir.c b/drivers/net/ark/ark_pktdir.c
index eb47dedb..1f2c8182 100644
--- a/drivers/net/ark/ark_pktdir.c
+++ b/drivers/net/ark/ark_pktdir.c
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #include <stdint.h>
diff --git a/drivers/net/ark/ark_pktdir.h b/drivers/net/ark/ark_pktdir.h
index e13fe821..314e6dea 100644
--- a/drivers/net/ark/ark_pktdir.h
+++ b/drivers/net/ark/ark_pktdir.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_PKTDIR_H_
diff --git a/drivers/net/ark/ark_pktgen.c b/drivers/net/ark/ark_pktgen.c
index d3c3dee1..2a2b428e 100644
--- a/drivers/net/ark/ark_pktgen.c
+++ b/drivers/net/ark/ark_pktgen.c
@@ -1,34 +1,5 @@
-/*
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #include <getopt.h>
diff --git a/drivers/net/ark/ark_pktgen.h b/drivers/net/ark/ark_pktgen.h
index bf5a241b..0e5f76aa 100644
--- a/drivers/net/ark/ark_pktgen.h
+++ b/drivers/net/ark/ark_pktgen.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_PKTGEN_H_
diff --git a/drivers/net/ark/ark_rqp.c b/drivers/net/ark/ark_rqp.c
index 41c497b0..bf1af4d6 100644
--- a/drivers/net/ark/ark_rqp.c
+++ b/drivers/net/ark/ark_rqp.c
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #include <unistd.h>
diff --git a/drivers/net/ark/ark_rqp.h b/drivers/net/ark/ark_rqp.h
index 0c380071..6c804606 100644
--- a/drivers/net/ark/ark_rqp.h
+++ b/drivers/net/ark/ark_rqp.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_RQP_H_
diff --git a/drivers/net/ark/ark_udm.c b/drivers/net/ark/ark_udm.c
index 7a429ac7..03f1922c 100644
--- a/drivers/net/ark/ark_udm.c
+++ b/drivers/net/ark/ark_udm.c
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #include <unistd.h>
diff --git a/drivers/net/ark/ark_udm.h b/drivers/net/ark/ark_udm.h
index 915343fe..5846c825 100644
--- a/drivers/net/ark/ark_udm.h
+++ b/drivers/net/ark/ark_udm.h
@@ -1,34 +1,5 @@
-/*-
- * BSD LICENSE
- *
- * Copyright (c) 2015-2017 Atomic Rules LLC
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of copyright holder nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2018 Atomic Rules LLC
  */
 
 #ifndef _ARK_UDM_H_
diff --git a/drivers/net/ark/meson.build b/drivers/net/ark/meson.build
new file mode 100644
index 00000000..99151bba
--- /dev/null
+++ b/drivers/net/ark/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+sources = files('ark_ddm.c',
+	'ark_ethdev.c',
+	'ark_ethdev_rx.c',
+	'ark_ethdev_tx.c',
+	'ark_mpu.c',
+	'ark_pktchkr.c',
+	'ark_pktdir.c',
+	'ark_pktgen.c',
+	'ark_rqp.c',
+	'ark_udm.c')
diff --git a/drivers/net/avf/avf_ethdev.c b/drivers/net/avf/avf_ethdev.c
index ad83a57e..3a2baaf2 100644
--- a/drivers/net/avf/avf_ethdev.c
+++ b/drivers/net/avf/avf_ethdev.c
@@ -518,16 +518,30 @@ avf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	dev_info->max_mac_addrs = AVF_NUM_MACADDR_MAX;
 	dev_info->rx_offload_capa =
 		DEV_RX_OFFLOAD_VLAN_STRIP |
+		DEV_RX_OFFLOAD_QINQ_STRIP |
 		DEV_RX_OFFLOAD_IPV4_CKSUM |
 		DEV_RX_OFFLOAD_UDP_CKSUM |
-		DEV_RX_OFFLOAD_TCP_CKSUM;
+		DEV_RX_OFFLOAD_TCP_CKSUM |
+		DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
+		DEV_RX_OFFLOAD_CRC_STRIP |
+		DEV_RX_OFFLOAD_KEEP_CRC |
+		DEV_RX_OFFLOAD_SCATTER |
+		DEV_RX_OFFLOAD_JUMBO_FRAME |
+		DEV_RX_OFFLOAD_VLAN_FILTER;
 	dev_info->tx_offload_capa =
 		DEV_TX_OFFLOAD_VLAN_INSERT |
+		DEV_TX_OFFLOAD_QINQ_INSERT |
 		DEV_TX_OFFLOAD_IPV4_CKSUM |
 		DEV_TX_OFFLOAD_UDP_CKSUM |
 		DEV_TX_OFFLOAD_TCP_CKSUM |
 		DEV_TX_OFFLOAD_SCTP_CKSUM |
-		DEV_TX_OFFLOAD_TCP_TSO;
+		DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
+		DEV_TX_OFFLOAD_TCP_TSO |
+		DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
+		DEV_TX_OFFLOAD_GRE_TNL_TSO |
+		DEV_TX_OFFLOAD_IPIP_TNL_TSO |
+		DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
+		DEV_TX_OFFLOAD_MULTI_SEGS;
 
 	dev_info->default_rxconf = (struct rte_eth_rxconf) {
 		.rx_free_thresh = AVF_DEFAULT_RX_FREE_THRESH,
@@ -1343,9 +1357,7 @@ static struct rte_pci_driver rte_avf_pmd = {
 RTE_PMD_REGISTER_PCI(net_avf, rte_avf_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_avf, pci_id_avf_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_avf, "* igb_uio | vfio-pci");
-RTE_INIT(avf_init_log);
-static void
-avf_init_log(void)
+RTE_INIT(avf_init_log)
 {
 	avf_logtype_init = rte_log_register("pmd.net.avf.init");
 	if (avf_logtype_init >= 0)
diff --git a/drivers/net/avp/avp_ethdev.c b/drivers/net/avp/avp_ethdev.c
index dc97e60e..761f6c1c 100644
--- a/drivers/net/avp/avp_ethdev.c
+++ b/drivers/net/avp/avp_ethdev.c
@@ -383,7 +383,7 @@ avp_dev_translate_address(struct rte_eth_dev *eth_dev,
 			(host_phys_addr < (map->phys_addr + map->length))) {
 			/* address is within this segment */
 			offset += (host_phys_addr - map->phys_addr);
-			addr = RTE_PTR_ADD(addr, offset);
+			addr = RTE_PTR_ADD(addr, (uintptr_t)offset);
 
 			PMD_DRV_LOG(DEBUG, "Translating host physical 0x%" PRIx64 " to guest virtual 0x%p\n",
 				    host_phys_addr, addr);
@@ -2170,6 +2170,7 @@ avp_dev_info_get(struct rte_eth_dev *eth_dev,
 		dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
 		dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT;
 	}
+	dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -2270,9 +2271,7 @@ avp_dev_stats_reset(struct rte_eth_dev *eth_dev)
 RTE_PMD_REGISTER_PCI(net_avp, rte_avp_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_avp, pci_id_avp_map);
 
-RTE_INIT(avp_init_log);
-static void
-avp_init_log(void)
+RTE_INIT(avp_init_log)
 {
 	avp_logtype_driver = rte_log_register("pmd.net.avp.driver");
 	if (avp_logtype_driver >= 0)
diff --git a/drivers/net/avp/meson.build b/drivers/net/avp/meson.build
new file mode 100644
index 00000000..6076c31b
--- /dev/null
+++ b/drivers/net/avp/meson.build
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+sources = files('avp_ethdev.c')
+install_headers('rte_avp_common.h', 'rte_avp_fifo.h')
diff --git a/drivers/net/axgbe/axgbe_ethdev.c b/drivers/net/axgbe/axgbe_ethdev.c
index 7a3ba2e7..9ae9f063 100644
--- a/drivers/net/axgbe/axgbe_ethdev.c
+++ b/drivers/net/axgbe/axgbe_ethdev.c
@@ -363,7 +363,9 @@ axgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	dev_info->rx_offload_capa =
 		DEV_RX_OFFLOAD_IPV4_CKSUM |
 		DEV_RX_OFFLOAD_UDP_CKSUM  |
-		DEV_RX_OFFLOAD_TCP_CKSUM;
+		DEV_RX_OFFLOAD_TCP_CKSUM  |
+		DEV_RX_OFFLOAD_CRC_STRIP  |
+		DEV_RX_OFFLOAD_KEEP_CRC;
 
 	dev_info->tx_offload_capa =
 		DEV_TX_OFFLOAD_IPV4_CKSUM  |
@@ -385,8 +387,6 @@ axgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 
 	dev_info->default_txconf = (struct rte_eth_txconf) {
 		.tx_free_thresh = AXGBE_TX_FREE_THRESH,
-		.txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
-				ETH_TXQ_FLAGS_NOOFFLOADS,
 	};
 }
 
@@ -759,9 +759,7 @@ RTE_PMD_REGISTER_PCI(net_axgbe, rte_axgbe_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_axgbe, pci_id_axgbe_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_axgbe, "* igb_uio | uio_pci_generic | vfio-pci");
 
-RTE_INIT(axgbe_init_log);
-static void
-axgbe_init_log(void)
+RTE_INIT(axgbe_init_log)
 {
 	axgbe_logtype_init = rte_log_register("pmd.net.axgbe.init");
 	if (axgbe_logtype_init >= 0)
diff --git a/drivers/net/axgbe/axgbe_rxtx.c b/drivers/net/axgbe/axgbe_rxtx.c
index b302bdd1..c5fd5f41 100644
--- a/drivers/net/axgbe/axgbe_rxtx.c
+++ b/drivers/net/axgbe/axgbe_rxtx.c
@@ -74,8 +74,10 @@ int axgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 		(DMA_CH_INC * rxq->queue_id));
 	rxq->dma_tail_reg = (volatile uint32_t *)((uint8_t *)rxq->dma_regs +
 						  DMA_CH_RDTR_LO);
-	rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.offloads &
-			DEV_RX_OFFLOAD_CRC_STRIP) ? 0 : ETHER_CRC_LEN);
+	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+		rxq->crc_len = ETHER_CRC_LEN;
+	else
+		rxq->crc_len = 0;
 
 	/* CRC strip in AXGBE supports per port not per queue */
 	pdata->crc_strip_enable = (rxq->crc_len == 0) ? 1 : 0;
@@ -369,10 +371,8 @@ int axgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	if (txq->nb_desc % txq->free_thresh != 0)
 		txq->vector_disable = 1;
 
-	if ((tx_conf->txq_flags & (uint32_t)ETH_TXQ_FLAGS_NOOFFLOADS) !=
-	    ETH_TXQ_FLAGS_NOOFFLOADS) {
+	if (tx_conf->offloads != 0)
 		txq->vector_disable = 1;
-	}
 
 	/* Allocate TX ring hardware descriptors */
 	tsize = txq->nb_desc * sizeof(struct axgbe_tx_desc);
diff --git a/drivers/net/bnx2x/LICENSE.bnx2x_pmd b/drivers/net/bnx2x/LICENSE.bnx2x_pmd
deleted file mode 100644
index 64c6ef2c..00000000
--- a/drivers/net/bnx2x/LICENSE.bnx2x_pmd
+++ /dev/null
@@ -1,3 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2014-2018 Cavium Inc.
- */
diff --git a/drivers/net/bnx2x/Makefile b/drivers/net/bnx2x/Makefile
index 150b4cfa..55d1ad6e 100644
--- a/drivers/net/bnx2x/Makefile
+++ b/drivers/net/bnx2x/Makefile
@@ -1,8 +1,7 @@
-#    Copyright (c) 2014 - 2018 Cavium Inc.
-#    All rights reserved.
-#    www.cavium.com
-#
-#    See LICENSE.bnx2x_pmd for copyright and licensing details.
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2014 - 2018 Cavium Inc.
+# All rights reserved.
+# www.cavium.com
 include $(RTE_SDK)/mk/rte.vars.mk
 
 #
diff --git a/drivers/net/bnx2x/bnx2x.c b/drivers/net/bnx2x/bnx2x.c
index 84ade5fb..4904eaf3 100644
--- a/drivers/net/bnx2x/bnx2x.c
+++ b/drivers/net/bnx2x/bnx2x.c
@@ -1,4 +1,4 @@
-/*-
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
@@ -9,8 +9,6 @@
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #define BNX2X_DRIVER_VERSION "1.78.18"
@@ -31,7 +29,7 @@
 #define BNX2X_PMD_VER_PREFIX "BNX2X PMD"
 #define BNX2X_PMD_VERSION_MAJOR 1
 #define BNX2X_PMD_VERSION_MINOR 0
-#define BNX2X_PMD_VERSION_REVISION 5
+#define BNX2X_PMD_VERSION_REVISION 6
 #define BNX2X_PMD_VERSION_PATCH 1
 
 static inline const char *
@@ -125,7 +123,6 @@ int bnx2x_nic_load(struct bnx2x_softc *sc);
 
 static int bnx2x_handle_sp_tq(struct bnx2x_softc *sc);
 static void bnx2x_handle_fp_tq(struct bnx2x_fastpath *fp, int scan_fp);
-static void bnx2x_periodic_stop(struct bnx2x_softc *sc);
 static void bnx2x_ack_sb(struct bnx2x_softc *sc, uint8_t igu_sb_id,
 			 uint8_t storm, uint16_t index, uint8_t op,
 			 uint8_t update);
@@ -1969,9 +1966,6 @@ bnx2x_nic_unload(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_link
 
 	PMD_DRV_LOG(DEBUG, "Starting NIC unload...");
 
-	/* stop the periodic callout */
-	bnx2x_periodic_stop(sc);
-
 	/* mark driver as unloaded in shmem2 */
 	if (IS_PF(sc) && SHMEM2_HAS(sc, drv_capabilities_flag)) {
 		val = SHMEM2_RD(sc, drv_capabilities_flag[SC_FW_MB_IDX(sc)]);
@@ -4490,6 +4484,8 @@ static void bnx2x_handle_fp_tq(struct bnx2x_fastpath *fp, int scan_fp)
 	struct bnx2x_softc *sc = fp->sc;
 	uint8_t more_rx = FALSE;
 
+	PMD_DRV_LOG(DEBUG, "---> FP TASK QUEUE (%d) <--", fp->index);
+
 	/* update the fastpath index */
 	bnx2x_update_fp_sb_idx(fp);
 
@@ -4506,7 +4502,7 @@ static void bnx2x_handle_fp_tq(struct bnx2x_fastpath *fp, int scan_fp)
 	}
 
 	bnx2x_ack_sb(sc, fp->igu_sb_id, USTORM_ID,
-		   le16toh(fp->fp_hc_idx), IGU_INT_DISABLE, 1);
+		   le16toh(fp->fp_hc_idx), IGU_INT_ENABLE, 1);
 }
 
 /*
@@ -6997,16 +6993,6 @@ void bnx2x_link_status_update(struct bnx2x_softc *sc)
 	}
 }
 
-static void bnx2x_periodic_start(struct bnx2x_softc *sc)
-{
-	atomic_store_rel_long(&sc->periodic_flags, PERIODIC_GO);
-}
-
-static void bnx2x_periodic_stop(struct bnx2x_softc *sc)
-{
-	atomic_store_rel_long(&sc->periodic_flags, PERIODIC_STOP);
-}
-
 static int bnx2x_initial_phy_init(struct bnx2x_softc *sc, int load_mode)
 {
 	int rc, cfg_idx = bnx2x_get_link_cfg_idx(sc);
@@ -7041,10 +7027,6 @@ static int bnx2x_initial_phy_init(struct bnx2x_softc *sc, int load_mode)
 		bnx2x_link_report(sc);
 	}
 
-	if (!CHIP_REV_IS_SLOW(sc)) {
-		bnx2x_periodic_start(sc);
-	}
-
 	sc->link_params.req_line_speed[cfg_idx] = req_line_speed;
 	return rc;
 }
diff --git a/drivers/net/bnx2x/bnx2x.h b/drivers/net/bnx2x/bnx2x.h
index 4150fd85..0f6024fb 100644
--- a/drivers/net/bnx2x/bnx2x.h
+++ b/drivers/net/bnx2x/bnx2x.h
@@ -1,4 +1,4 @@
-/*-
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
@@ -9,8 +9,6 @@
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef __BNX2X_H__
@@ -1930,6 +1928,7 @@ void bnx2x_link_status_update(struct bnx2x_softc *sc);
 int bnx2x_complete_sp(struct bnx2x_softc *sc);
 int bnx2x_set_storm_rx_mode(struct bnx2x_softc *sc);
 void bnx2x_periodic_callout(struct bnx2x_softc *sc);
+void bnx2x_periodic_stop(void *param);
 
 int bnx2x_vf_get_resources(struct bnx2x_softc *sc, uint8_t tx_count, uint8_t rx_count);
 void bnx2x_vf_close(struct bnx2x_softc *sc);
diff --git a/drivers/net/bnx2x/bnx2x_ethdev.c b/drivers/net/bnx2x/bnx2x_ethdev.c
index 6a9cd581..575271a8 100644
--- a/drivers/net/bnx2x/bnx2x_ethdev.c
+++ b/drivers/net/bnx2x/bnx2x_ethdev.c
@@ -1,11 +1,8 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
- *
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #include "bnx2x.h"
@@ -13,6 +10,7 @@
 
 #include <rte_dev.h>
 #include <rte_ethdev_pci.h>
+#include <rte_alarm.h>
 
 int bnx2x_logtype_init;
 int bnx2x_logtype_driver;
@@ -81,26 +79,31 @@ static const struct rte_bnx2x_xstats_name_off bnx2x_xstats_strings[] = {
 		offsetof(struct bnx2x_eth_stats, pfc_frames_received_lo)}
 };
 
-static void
+static int
 bnx2x_link_update(struct rte_eth_dev *dev)
 {
 	struct bnx2x_softc *sc = dev->data->dev_private;
+	struct rte_eth_link link;
 
 	PMD_INIT_FUNC_TRACE();
+
 	bnx2x_link_status_update(sc);
+	memset(&link, 0, sizeof(link));
 	mb();
-	dev->data->dev_link.link_speed = sc->link_vars.line_speed;
+	link.link_speed = sc->link_vars.line_speed;
 	switch (sc->link_vars.duplex) {
 		case DUPLEX_FULL:
-			dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
+			link.link_duplex = ETH_LINK_FULL_DUPLEX;
 			break;
 		case DUPLEX_HALF:
-			dev->data->dev_link.link_duplex = ETH_LINK_HALF_DUPLEX;
+			link.link_duplex = ETH_LINK_HALF_DUPLEX;
 			break;
 	}
-	dev->data->dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
+	link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 			ETH_LINK_SPEED_FIXED);
-	dev->data->dev_link.link_status = sc->link_vars.link_up;
+	link.link_status = sc->link_vars.link_up;
+
+	return rte_eth_linkstatus_set(dev, &link);
 }
 
 static void
@@ -109,8 +112,6 @@ bnx2x_interrupt_action(struct rte_eth_dev *dev)
 	struct bnx2x_softc *sc = dev->data->dev_private;
 	uint32_t link_status;
 
-	PMD_DEBUG_PERIODIC_LOG(INFO, "Interrupt handled");
-
 	bnx2x_intr_legacy(sc, 0);
 
 	if (sc->periodic_flags & PERIODIC_GO)
@@ -128,10 +129,41 @@ bnx2x_interrupt_handler(void *param)
 	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 	struct bnx2x_softc *sc = dev->data->dev_private;
 
+	PMD_DEBUG_PERIODIC_LOG(INFO, "Interrupt handled");
+
 	bnx2x_interrupt_action(dev);
 	rte_intr_enable(&sc->pci_dev->intr_handle);
 }
 
+static void bnx2x_periodic_start(void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+	struct bnx2x_softc *sc = dev->data->dev_private;
+	int ret = 0;
+
+	atomic_store_rel_long(&sc->periodic_flags, PERIODIC_GO);
+	bnx2x_interrupt_action(dev);
+	if (IS_PF(sc)) {
+		ret = rte_eal_alarm_set(BNX2X_SP_TIMER_PERIOD,
+					bnx2x_periodic_start, (void *)dev);
+		if (ret) {
+			PMD_DRV_LOG(ERR, "Unable to start periodic"
+					 " timer rc %d", ret);
+			assert(false && "Unable to start periodic timer");
+		}
+	}
+}
+
+void bnx2x_periodic_stop(void *param)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
+	struct bnx2x_softc *sc = dev->data->dev_private;
+
+	atomic_store_rel_long(&sc->periodic_flags, PERIODIC_STOP);
+
+	rte_eal_alarm_cancel(bnx2x_periodic_start, (void *)dev);
+}
+
 /*
  * Devops - helper functions can be called from user application
  */
@@ -187,6 +219,10 @@ bnx2x_dev_start(struct rte_eth_dev *dev)
 
 	PMD_INIT_FUNC_TRACE();
 
+	/* start the periodic callout */
+	if (sc->periodic_flags & PERIODIC_STOP)
+		bnx2x_periodic_start(dev);
+
 	ret = bnx2x_init(sc);
 	if (ret) {
 		PMD_DRV_LOG(DEBUG, "bnx2x_init failed (%d)", ret);
@@ -227,6 +263,9 @@ bnx2x_dev_stop(struct rte_eth_dev *dev)
 				bnx2x_interrupt_handler, (void *)dev);
 	}
 
+	/* stop the periodic callout */
+	bnx2x_periodic_stop(dev);
+
 	ret = bnx2x_nic_unload(sc, UNLOAD_NORMAL, FALSE);
 	if (ret) {
 		PMD_DRV_LOG(DEBUG, "bnx2x_nic_unload failed (%d)", ret);
@@ -309,20 +348,16 @@ bnx2x_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete
 {
 	PMD_INIT_FUNC_TRACE();
 
-	int old_link_status = dev->data->dev_link.link_status;
-
-	bnx2x_link_update(dev);
-
-	return old_link_status == dev->data->dev_link.link_status ? -1 : 0;
+	return bnx2x_link_update(dev);
 }
 
 static int
 bnx2xvf_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
 {
-	int old_link_status = dev->data->dev_link.link_status;
 	struct bnx2x_softc *sc = dev->data->dev_private;
+	int ret = 0;
 
-	bnx2x_link_update(dev);
+	ret = bnx2x_link_update(dev);
 
 	bnx2x_check_bull(sc);
 	if (sc->old_bulletin.valid_bitmap & (1 << CHANNEL_DOWN)) {
@@ -331,7 +366,7 @@ bnx2xvf_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_comple
 		dev->data->dev_link.link_status = ETH_LINK_DOWN;
 	}
 
-	return old_link_status == dev->data->dev_link.link_status ? -1 : 0;
+	return ret;
 }
 
 static int
@@ -585,6 +620,17 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf)
 		return ret;
 	}
 
+	/* schedule periodic poll for slowpath link events */
+	if (IS_PF(sc)) {
+		ret = rte_eal_alarm_set(BNX2X_SP_TIMER_PERIOD,
+					bnx2x_periodic_start, (void *)eth_dev);
+		if (ret) {
+			PMD_DRV_LOG(ERR, "Unable to start periodic"
+					  " timer rc %d", ret);
+			return -EINVAL;
+		}
+	}
+
 	eth_dev->data->mac_addrs = (struct ether_addr *)sc->link_params.mac_addr;
 
 	PMD_DRV_LOG(INFO, "pcie_bus=%d, pcie_device=%d",
@@ -599,18 +645,20 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf)
 	if (IS_VF(sc)) {
 		rte_spinlock_init(&sc->vf2pf_lock);
 
-		if (bnx2x_dma_alloc(sc, sizeof(struct bnx2x_vf_mbx_msg),
-				    &sc->vf2pf_mbox_mapping, "vf2pf_mbox",
-				    RTE_CACHE_LINE_SIZE) != 0)
-			return -ENOMEM;
+		ret = bnx2x_dma_alloc(sc, sizeof(struct bnx2x_vf_mbx_msg),
+				      &sc->vf2pf_mbox_mapping, "vf2pf_mbox",
+				      RTE_CACHE_LINE_SIZE);
+		if (ret)
+			goto out;
 
 		sc->vf2pf_mbox = (struct bnx2x_vf_mbx_msg *)
 					 sc->vf2pf_mbox_mapping.vaddr;
 
-		if (bnx2x_dma_alloc(sc, sizeof(struct bnx2x_vf_bulletin),
-				    &sc->pf2vf_bulletin_mapping, "vf2pf_bull",
-				    RTE_CACHE_LINE_SIZE) != 0)
-			return -ENOMEM;
+		ret = bnx2x_dma_alloc(sc, sizeof(struct bnx2x_vf_bulletin),
+				      &sc->pf2vf_bulletin_mapping, "vf2pf_bull",
+				      RTE_CACHE_LINE_SIZE);
+		if (ret)
+			goto out;
 
 		sc->pf2vf_bulletin = (struct bnx2x_vf_bulletin *)
 					     sc->pf2vf_bulletin_mapping.vaddr;
@@ -618,10 +666,14 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf)
 		ret = bnx2x_vf_get_resources(sc, sc->max_tx_queues,
 					     sc->max_rx_queues);
 		if (ret)
-			return ret;
+			goto out;
 	}
 
 	return 0;
+
+out:
+	bnx2x_periodic_stop(eth_dev);
+	return ret;
 }
 
 static int
@@ -683,9 +735,7 @@ RTE_PMD_REGISTER_PCI(net_bnx2xvf, rte_bnx2xvf_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_bnx2xvf, pci_id_bnx2xvf_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_bnx2xvf, "* igb_uio | vfio-pci");
 
-RTE_INIT(bnx2x_init_log);
-static void
-bnx2x_init_log(void)
+RTE_INIT(bnx2x_init_log)
 {
 	bnx2x_logtype_init = rte_log_register("pmd.net.bnx2x.init");
 	if (bnx2x_logtype_init >= 0)
diff --git a/drivers/net/bnx2x/bnx2x_ethdev.h b/drivers/net/bnx2x/bnx2x_ethdev.h
index f05be7ee..807ba178 100644
--- a/drivers/net/bnx2x/bnx2x_ethdev.h
+++ b/drivers/net/bnx2x/bnx2x_ethdev.h
@@ -1,11 +1,8 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
- *
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef PMD_BNX2X_ETHDEV_H
@@ -58,7 +55,6 @@
 #define wmb()   rte_wmb()
 #define rmb()   rte_rmb()
 
-
 #define MAX_QUEUES sysconf(_SC_NPROCESSORS_CONF)
 
 #define BNX2X_MIN_RX_BUF_SIZE 1024
@@ -72,6 +68,8 @@
 /* Maximum number of Rx packets to process at a time */
 #define BNX2X_RX_BUDGET 0xffffffff
 
+#define BNX2X_SP_TIMER_PERIOD US_PER_S /* 1 second */
+
 #endif
 
 /* MAC address operations */
diff --git a/drivers/net/bnx2x/bnx2x_logs.h b/drivers/net/bnx2x/bnx2x_logs.h
index 69a2fe1d..9e232a9b 100644
--- a/drivers/net/bnx2x/bnx2x_logs.h
+++ b/drivers/net/bnx2x/bnx2x_logs.h
@@ -1,11 +1,8 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
- *
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef _PMD_LOGS_H_
diff --git a/drivers/net/bnx2x/bnx2x_rxtx.c b/drivers/net/bnx2x/bnx2x_rxtx.c
index 331884cf..d9a4127d 100644
--- a/drivers/net/bnx2x/bnx2x_rxtx.c
+++ b/drivers/net/bnx2x/bnx2x_rxtx.c
@@ -1,11 +1,8 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
- *
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #include "bnx2x.h"
diff --git a/drivers/net/bnx2x/bnx2x_rxtx.h b/drivers/net/bnx2x/bnx2x_rxtx.h
index 94b9e1b6..6ad4928c 100644
--- a/drivers/net/bnx2x/bnx2x_rxtx.h
+++ b/drivers/net/bnx2x/bnx2x_rxtx.h
@@ -1,11 +1,8 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
- *
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef _BNX2X_RXTX_H_
diff --git a/drivers/net/bnx2x/bnx2x_stats.c b/drivers/net/bnx2x/bnx2x_stats.c
index e3880abe..edc86ccc 100644
--- a/drivers/net/bnx2x/bnx2x_stats.c
+++ b/drivers/net/bnx2x/bnx2x_stats.c
@@ -1,5 +1,5 @@
-/*-
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -9,8 +9,6 @@
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #include "bnx2x.h"
diff --git a/drivers/net/bnx2x/bnx2x_stats.h b/drivers/net/bnx2x/bnx2x_stats.h
index 6fcaf607..635412bd 100644
--- a/drivers/net/bnx2x/bnx2x_stats.h
+++ b/drivers/net/bnx2x/bnx2x_stats.h
@@ -1,5 +1,5 @@
-/*-
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -9,8 +9,6 @@
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef BNX2X_STATS_H
diff --git a/drivers/net/bnx2x/bnx2x_vfpf.c b/drivers/net/bnx2x/bnx2x_vfpf.c
index dacad771..50099d46 100644
--- a/drivers/net/bnx2x/bnx2x_vfpf.c
+++ b/drivers/net/bnx2x/bnx2x_vfpf.c
@@ -1,11 +1,8 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
- *
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #include "bnx2x.h"
diff --git a/drivers/net/bnx2x/bnx2x_vfpf.h b/drivers/net/bnx2x/bnx2x_vfpf.h
index c4675d4c..cc6fef95 100644
--- a/drivers/net/bnx2x/bnx2x_vfpf.h
+++ b/drivers/net/bnx2x/bnx2x_vfpf.h
@@ -1,11 +1,8 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
- *
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef BNX2X_VFPF_H
diff --git a/drivers/net/bnx2x/ecore_fw_defs.h b/drivers/net/bnx2x/ecore_fw_defs.h
index d10dd108..5984acd9 100644
--- a/drivers/net/bnx2x/ecore_fw_defs.h
+++ b/drivers/net/bnx2x/ecore_fw_defs.h
@@ -1,5 +1,5 @@
-/*-
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -8,8 +8,6 @@
  * Copyright (c) 2014-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef ECORE_FW_DEFS_H
diff --git a/drivers/net/bnx2x/ecore_hsi.h b/drivers/net/bnx2x/ecore_hsi.h
index 0220e5f9..57085ebb 100644
--- a/drivers/net/bnx2x/ecore_hsi.h
+++ b/drivers/net/bnx2x/ecore_hsi.h
@@ -1,5 +1,5 @@
-/*-
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -8,8 +8,6 @@
  * Copyright (c) 2014-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef ECORE_HSI_H
diff --git a/drivers/net/bnx2x/ecore_init.h b/drivers/net/bnx2x/ecore_init.h
index 8d00abb7..f2de07e5 100644
--- a/drivers/net/bnx2x/ecore_init.h
+++ b/drivers/net/bnx2x/ecore_init.h
@@ -1,5 +1,5 @@
-/*-
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -9,8 +9,6 @@
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef ECORE_INIT_H
diff --git a/drivers/net/bnx2x/ecore_init_ops.h b/drivers/net/bnx2x/ecore_init_ops.h
index dd5df3d5..2b003afb 100644
--- a/drivers/net/bnx2x/ecore_init_ops.h
+++ b/drivers/net/bnx2x/ecore_init_ops.h
@@ -1,5 +1,5 @@
-/*-
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -9,8 +9,6 @@
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef ECORE_INIT_OPS_H
diff --git a/drivers/net/bnx2x/ecore_mfw_req.h b/drivers/net/bnx2x/ecore_mfw_req.h
index c798c74c..fe945048 100644
--- a/drivers/net/bnx2x/ecore_mfw_req.h
+++ b/drivers/net/bnx2x/ecore_mfw_req.h
@@ -1,5 +1,5 @@
-/*-
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -8,8 +8,6 @@
  * Copyright (c) 2014-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef ECORE_MFW_REQ_H
diff --git a/drivers/net/bnx2x/ecore_reg.h b/drivers/net/bnx2x/ecore_reg.h
index 9800bafc..ae8a93bb 100644
--- a/drivers/net/bnx2x/ecore_reg.h
+++ b/drivers/net/bnx2x/ecore_reg.h
@@ -1,5 +1,5 @@
-/*-
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -8,8 +8,6 @@
  * Copyright (c) 2014-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef ECORE_REG_H
diff --git a/drivers/net/bnx2x/ecore_sp.c b/drivers/net/bnx2x/ecore_sp.c
index 75329672..0c8685c6 100644
--- a/drivers/net/bnx2x/ecore_sp.c
+++ b/drivers/net/bnx2x/ecore_sp.c
@@ -1,5 +1,5 @@
-/*-
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -9,8 +9,6 @@
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #include "bnx2x.h"
diff --git a/drivers/net/bnx2x/ecore_sp.h b/drivers/net/bnx2x/ecore_sp.h
index 772c8b1b..6b65a496 100644
--- a/drivers/net/bnx2x/ecore_sp.h
+++ b/drivers/net/bnx2x/ecore_sp.h
@@ -1,5 +1,5 @@
-/*-
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -9,8 +9,6 @@
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef ECORE_SP_H
diff --git a/drivers/net/bnx2x/elink.c b/drivers/net/bnx2x/elink.c
index 34a29373..b63fd23e 100644
--- a/drivers/net/bnx2x/elink.c
+++ b/drivers/net/bnx2x/elink.c
@@ -1,5 +1,5 @@
-/*
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -9,8 +9,6 @@
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #include "bnx2x.h"
diff --git a/drivers/net/bnx2x/elink.h b/drivers/net/bnx2x/elink.h
index 236f9367..40000c24 100644
--- a/drivers/net/bnx2x/elink.h
+++ b/drivers/net/bnx2x/elink.h
@@ -1,5 +1,5 @@
-/*
- * Copyright (c) 2007-2013 Cavium Inc. All rights reserved.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2007-2013 Broadcom Corporation.
  *
  * Eric Davis        <edavis@broadcom.com>
  * David Christensen <davidch@broadcom.com>
@@ -9,8 +9,6 @@
  * Copyright (c) 2015-2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.bnx2x_pmd for copyright and licensing details.
  */
 
 #ifndef ELINK_H
diff --git a/drivers/net/bnx2x/meson.build b/drivers/net/bnx2x/meson.build
new file mode 100644
index 00000000..e3c68886
--- /dev/null
+++ b/drivers/net/bnx2x/meson.build
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+dep = cc.find_library('z', required: false)
+build = dep.found()
+ext_deps += dep
+cflags += '-DZLIB_CONST'
+sources = files('bnx2x.c',
+	'bnx2x_ethdev.c',
+	'bnx2x_rxtx.c',
+	'bnx2x_stats.c',
+	'bnx2x_vfpf.c',
+	'ecore_sp.c',
+	'elink.c')
diff --git a/drivers/net/bnxt/Makefile b/drivers/net/bnxt/Makefile
index fd0cb523..8be3cb0e 100644
--- a/drivers/net/bnxt/Makefile
+++ b/drivers/net/bnxt/Makefile
@@ -29,6 +29,7 @@ EXPORT_MAP := rte_pmd_bnxt_version.map
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_cpr.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_filter.c
+SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_flow.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_hwrm.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_ring.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_rxq.c
@@ -38,6 +39,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_txq.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_txr.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_vnic.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_irq.c
+SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_util.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += rte_pmd_bnxt.c
 
 #
diff --git a/drivers/net/bnxt/bnxt.h b/drivers/net/bnxt/bnxt.h
index afaaf8c4..db5c4eb0 100644
--- a/drivers/net/bnxt/bnxt.h
+++ b/drivers/net/bnxt/bnxt.h
@@ -22,8 +22,24 @@
 
 #define BNXT_MAX_MTU		9500
 #define VLAN_TAG_SIZE		4
+#define BNXT_VF_RSV_NUM_RSS_CTX	1
+#define BNXT_VF_RSV_NUM_L2_CTX	4
+/* TODO: For now, do not support VMDq/RFS on VFs. */
+#define BNXT_VF_RSV_NUM_VNIC	1
 #define BNXT_MAX_LED		4
 #define BNXT_NUM_VLANS		2
+#define BNXT_MIN_RING_DESC	16
+#define BNXT_MAX_TX_RING_DESC	4096
+#define BNXT_MAX_RX_RING_DESC	8192
+#define BNXT_DB_SIZE		0x80
+
+#define BNXT_INT_LAT_TMR_MIN			75
+#define BNXT_INT_LAT_TMR_MAX			150
+#define BNXT_NUM_CMPL_AGGR_INT			36
+#define BNXT_CMPL_AGGR_DMA_TMR			37
+#define BNXT_NUM_CMPL_DMA_AGGR			36
+#define BNXT_CMPL_AGGR_DMA_TMR_DURING_INT	50
+#define BNXT_NUM_CMPL_DMA_AGGR_DURING_INT	12
 
 struct bnxt_led_info {
 	uint8_t      led_id;
@@ -98,6 +114,7 @@ struct bnxt_child_vf_info {
 struct bnxt_pf_info {
 #define BNXT_FIRST_PF_FID	1
 #define BNXT_MAX_VFS(bp)	(bp->pf.max_vfs)
+#define BNXT_TOTAL_VFS(bp)	((bp)->pf.total_vfs)
 #define BNXT_FIRST_VF_FID	128
 #define BNXT_PF_RINGS_USED(bp)	bnxt_get_num_queues(bp)
 #define BNXT_PF_RINGS_AVAIL(bp)	(bp->pf.max_cp_rings - BNXT_PF_RINGS_USED(bp))
@@ -105,6 +122,9 @@ struct bnxt_pf_info {
 	uint16_t		first_vf_id;
 	uint16_t		active_vfs;
 	uint16_t		max_vfs;
+	uint16_t		total_vfs; /* Total VFs possible.
+					    * Not necessarily enabled.
+					    */
 	uint32_t		func_cfg_flags;
 	void			*vf_req_buf;
 	rte_iova_t		vf_req_buf_dma_addr;
@@ -202,6 +222,16 @@ struct bnxt_ptp_cfg {
 	uint32_t			tx_mapped_regs[BNXT_PTP_TX_REGS];
 };
 
+struct bnxt_coal {
+	uint16_t			num_cmpl_aggr_int;
+	uint16_t			num_cmpl_dma_aggr;
+	uint16_t			num_cmpl_dma_aggr_during_int;
+	uint16_t			int_lat_tmr_max;
+	uint16_t			int_lat_tmr_min;
+	uint16_t			cmpl_aggr_dma_tmr;
+	uint16_t			cmpl_aggr_dma_tmr_during_int;
+};
+
 #define BNXT_HWRM_SHORT_REQ_LEN		sizeof(struct hwrm_short_input)
 struct bnxt {
 	void				*bar0;
@@ -302,12 +332,14 @@ struct bnxt {
 	struct bnxt_led_info	leds[BNXT_MAX_LED];
 	uint8_t			num_leds;
 	struct bnxt_ptp_cfg     *ptp_cfg;
+	uint16_t		vf_resv_strategy;
 };
 
 int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete);
 int bnxt_rcv_msg_from_vf(struct bnxt *bp, uint16_t vf_id, void *msg);
 
 bool is_bnxt_supported(struct rte_eth_dev *dev);
+bool bnxt_stratus_device(struct bnxt *bp);
 extern const struct rte_flow_ops bnxt_flow_ops;
 
 extern int bnxt_logtype_driver;
diff --git a/drivers/net/bnxt/bnxt_cpr.h b/drivers/net/bnxt/bnxt_cpr.h
index 6c1e6d2b..c7af5698 100644
--- a/drivers/net/bnxt/bnxt_cpr.h
+++ b/drivers/net/bnxt/bnxt_cpr.h
@@ -22,12 +22,20 @@
 #define ADV_RAW_CMP(idx, n)	((idx) + (n))
 #define NEXT_RAW_CMP(idx)	ADV_RAW_CMP(idx, 1)
 #define RING_CMP(ring, idx)	((idx) & (ring)->ring_mask)
+#define RING_CMPL(ring_mask, idx)	((idx) & (ring_mask))
 #define NEXT_CMP(idx)		RING_CMP(ADV_RAW_CMP(idx, 1))
 #define FLIP_VALID(cons, mask, val)	((cons) >= (mask) ? !(val) : (val))
 
 #define DB_CP_REARM_FLAGS	(DB_KEY_CP | DB_IDX_VALID)
 #define DB_CP_FLAGS		(DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 
+#define NEXT_CMPL(cpr, idx, v, inc)	do { \
+	(idx) += (inc); \
+	if (unlikely((idx) == (cpr)->cp_ring_struct->ring_size)) { \
+		(v) = !(v); \
+		(idx) = 0; \
+	} \
+} while (0)
 #define B_CP_DB_REARM(cpr, raw_cons)					\
 	rte_write32((DB_CP_REARM_FLAGS |				\
 		    RING_CMP(((cpr)->cp_ring_struct), raw_cons)),	\
@@ -50,6 +58,10 @@
 	rte_write32((DB_CP_FLAGS |					\
 		    RING_CMP(((cpr)->cp_ring_struct), raw_cons)),	\
 		    ((cpr)->cp_doorbell))
+#define B_CP_DB(cpr, raw_cons, ring_mask)				\
+	rte_write32((DB_CP_FLAGS |					\
+		    RING_CMPL((ring_mask), raw_cons)),	\
+		    ((cpr)->cp_doorbell))
 
 struct bnxt_ring;
 struct bnxt_cp_ring_info {
diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c
index 6e56bfd3..cc7e4391 100644
--- a/drivers/net/bnxt/bnxt_ethdev.c
+++ b/drivers/net/bnxt/bnxt_ethdev.c
@@ -26,6 +26,7 @@
 #include "bnxt_vnic.h"
 #include "hsi_struct_def_dpdk.h"
 #include "bnxt_nvm_defs.h"
+#include "bnxt_util.h"
 
 #define DRV_MODULE_NAME		"bnxt"
 static const char bnxt_version[] =
@@ -73,6 +74,7 @@ int bnxt_logtype_driver;
 #define BROADCOM_DEV_ID_58802 0xd802
 #define BROADCOM_DEV_ID_58804 0xd804
 #define BROADCOM_DEV_ID_58808 0x16f0
+#define BROADCOM_DEV_ID_58802_VF 0xd800
 
 static const struct rte_pci_id bnxt_pci_id_map[] = {
 	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM,
@@ -116,6 +118,7 @@ static const struct rte_pci_id bnxt_pci_id_map[] = {
 	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58802) },
 	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58804) },
 	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58808) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58802_VF) },
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
@@ -147,11 +150,13 @@ static const struct rte_pci_id bnxt_pci_id_map[] = {
 				     DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | \
 				     DEV_RX_OFFLOAD_JUMBO_FRAME | \
 				     DEV_RX_OFFLOAD_CRC_STRIP | \
+				     DEV_RX_OFFLOAD_KEEP_CRC | \
 				     DEV_RX_OFFLOAD_TCP_LRO)
 
 static int bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask);
 static void bnxt_print_link_info(struct rte_eth_dev *eth_dev);
 static int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu);
+static int bnxt_dev_uninit(struct rte_eth_dev *eth_dev);
 
 /***********************/
 
@@ -195,13 +200,14 @@ alloc_mem_err:
 
 static int bnxt_init_chip(struct bnxt *bp)
 {
-	unsigned int i;
+	struct bnxt_rx_queue *rxq;
 	struct rte_eth_link new;
 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(bp->eth_dev);
 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 	uint32_t intr_vector = 0;
 	uint32_t queue_id, base = BNXT_MISC_VEC_ID;
 	uint32_t vec = BNXT_MISC_VEC_ID;
+	unsigned int i, j;
 	int rc;
 
 	/* disable uio/vfio intr/eventfd mapping */
@@ -243,7 +249,19 @@ static int bnxt_init_chip(struct bnxt *bp)
 
 	/* VNIC configuration */
 	for (i = 0; i < bp->nr_vnics; i++) {
+		struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
+		uint32_t size = sizeof(*vnic->fw_grp_ids) * bp->max_ring_grps;
+
+		vnic->fw_grp_ids = rte_zmalloc("vnic_fw_grp_ids", size, 0);
+		if (!vnic->fw_grp_ids) {
+			PMD_DRV_LOG(ERR,
+				    "Failed to alloc %d bytes for group ids\n",
+				    size);
+			rc = -ENOMEM;
+			goto err_out;
+		}
+		memset(vnic->fw_grp_ids, -1, size);
 
 		rc = bnxt_hwrm_vnic_alloc(bp, vnic);
 		if (rc) {
@@ -252,12 +270,15 @@ static int bnxt_init_chip(struct bnxt *bp)
 			goto err_out;
 		}
 
-		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic);
-		if (rc) {
-			PMD_DRV_LOG(ERR,
-				"HWRM vnic %d ctx alloc failure rc: %x\n",
-				i, rc);
-			goto err_out;
+		/* Alloc RSS context only if RSS mode is enabled */
+		if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS) {
+			rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic);
+			if (rc) {
+				PMD_DRV_LOG(ERR,
+					"HWRM vnic %d ctx alloc failure rc: %x\n",
+					i, rc);
+				goto err_out;
+			}
 		}
 
 		rc = bnxt_hwrm_vnic_cfg(bp, vnic);
@@ -275,6 +296,13 @@ static int bnxt_init_chip(struct bnxt *bp)
 			goto err_out;
 		}
 
+		for (j = 0; j < bp->rx_nr_rings; j++) {
+			rxq = bp->eth_dev->data->rx_queues[j];
+
+			if (rxq->rx_deferred_start)
+				rxq->vnic->fw_grp_ids[j] = INVALID_HW_RING_ID;
+		}
+
 		rc = bnxt_vnic_rss_configure(bp, vnic);
 		if (rc) {
 			PMD_DRV_LOG(ERR,
@@ -410,7 +438,7 @@ static void bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev,
 	/* For the sake of symmetry, max_rx_queues = max_tx_queues */
 	dev_info->max_rx_queues = max_rx_rings;
 	dev_info->max_tx_queues = max_rx_rings;
-	dev_info->reta_size = bp->max_rsscos_ctx;
+	dev_info->reta_size = HW_HASH_INDEX_SIZE;
 	dev_info->hash_key_size = 40;
 	max_vnics = bp->max_vnics;
 
@@ -449,6 +477,10 @@ static void bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev,
 	eth_dev->data->dev_conf.intr_conf.lsc = 1;
 
 	eth_dev->data->dev_conf.intr_conf.rxq = 1;
+	dev_info->rx_desc_lim.nb_min = BNXT_MIN_RING_DESC;
+	dev_info->rx_desc_lim.nb_max = BNXT_MAX_RX_RING_DESC;
+	dev_info->tx_desc_lim.nb_min = BNXT_MIN_RING_DESC;
+	dev_info->tx_desc_lim.nb_max = BNXT_MAX_TX_RING_DESC;
 
 	/* *INDENT-ON* */
 
@@ -489,6 +521,7 @@ static int bnxt_dev_configure_op(struct rte_eth_dev *eth_dev)
 {
 	struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private;
 	uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads;
+	int rc;
 
 	bp->rx_queues = (void *)eth_dev->data->rx_queues;
 	bp->tx_queues = (void *)eth_dev->data->tx_queues;
@@ -496,19 +529,23 @@ static int bnxt_dev_configure_op(struct rte_eth_dev *eth_dev)
 	bp->rx_nr_rings = eth_dev->data->nb_rx_queues;
 
 	if (BNXT_VF(bp) && (bp->flags & BNXT_FLAG_NEW_RM)) {
-		int rc;
+		rc = bnxt_hwrm_check_vf_rings(bp);
+		if (rc) {
+			PMD_DRV_LOG(ERR, "HWRM insufficient resources\n");
+			return -ENOSPC;
+		}
 
-		rc = bnxt_hwrm_func_reserve_vf_resc(bp);
+		rc = bnxt_hwrm_func_reserve_vf_resc(bp, false);
 		if (rc) {
 			PMD_DRV_LOG(ERR, "HWRM resource alloc fail:%x\n", rc);
 			return -ENOSPC;
 		}
-
+	} else {
 		/* legacy driver needs to get updated values */
 		rc = bnxt_hwrm_func_qcaps(bp);
 		if (rc) {
 			PMD_DRV_LOG(ERR, "hwrm func qcaps fail:%d\n", rc);
-			return -ENOSPC;
+			return rc;
 		}
 	}
 
@@ -519,7 +556,9 @@ static int bnxt_dev_configure_op(struct rte_eth_dev *eth_dev)
 	    bp->max_cp_rings ||
 	    eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues >
 	    bp->max_stat_ctx ||
-	    (uint32_t)(eth_dev->data->nb_rx_queues) > bp->max_ring_grps) {
+	    (uint32_t)(eth_dev->data->nb_rx_queues) > bp->max_ring_grps ||
+	    (!(eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) &&
+	     bp->max_vnics < eth_dev->data->nb_rx_queues)) {
 		PMD_DRV_LOG(ERR,
 			"Insufficient resources to support requested config\n");
 		PMD_DRV_LOG(ERR,
@@ -527,9 +566,9 @@ static int bnxt_dev_configure_op(struct rte_eth_dev *eth_dev)
 			eth_dev->data->nb_tx_queues,
 			eth_dev->data->nb_rx_queues);
 		PMD_DRV_LOG(ERR,
-			"Res available: TxQ %d, RxQ %d, CQ %d Stat %d, Grp %d\n",
+			"MAX: TxQ %d, RxQ %d, CQ %d Stat %d, Grp %d, Vnic %d\n",
 			bp->max_tx_rings, bp->max_rx_rings, bp->max_cp_rings,
-			bp->max_stat_ctx, bp->max_ring_grps);
+			bp->max_stat_ctx, bp->max_ring_grps, bp->max_vnics);
 		return -ENOSPC;
 	}
 
@@ -664,6 +703,8 @@ static void bnxt_dev_close_op(struct rte_eth_dev *eth_dev)
 		rte_free(bp->grp_info);
 		bp->grp_info = NULL;
 	}
+
+	bnxt_dev_uninit(eth_dev);
 }
 
 static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev,
@@ -1287,9 +1328,9 @@ static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
 	struct bnxt_vnic_info *vnic;
 	unsigned int i;
 	int rc = 0;
-	uint32_t en = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN |
-		HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK;
-	uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN;
+	uint32_t en = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN |
+		HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK;
+	uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN;
 
 	/* Cycle through all VNICs */
 	for (i = 0; i < bp->nr_vnics; i++) {
@@ -1336,8 +1377,8 @@ static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
 				memcpy(new_filter->l2_addr, filter->l2_addr,
 				       ETHER_ADDR_LEN);
 				/* MAC + VLAN ID filter */
-				new_filter->l2_ovlan = vlan_id;
-				new_filter->l2_ovlan_mask = 0xF000;
+				new_filter->l2_ivlan = vlan_id;
+				new_filter->l2_ivlan_mask = 0xF000;
 				new_filter->enables |= en;
 				rc = bnxt_hwrm_set_l2_filter(bp,
 							     vnic->fw_vnic_id,
@@ -1563,6 +1604,7 @@ static int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu)
 
 	for (i = 0; i < bp->nr_vnics; i++) {
 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
+		uint16_t size = 0;
 
 		vnic->mru = bp->eth_dev->data->mtu + ETHER_HDR_LEN +
 					ETHER_CRC_LEN + VLAN_TAG_SIZE * 2;
@@ -1570,9 +1612,14 @@ static int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu)
 		if (rc)
 			break;
 
-		rc = bnxt_hwrm_vnic_plcmode_cfg(bp, vnic);
-		if (rc)
-			return rc;
+		size = rte_pktmbuf_data_room_size(bp->rx_queues[0]->mb_pool);
+		size -= RTE_PKTMBUF_HEADROOM;
+
+		if (size < new_mtu) {
+			rc = bnxt_hwrm_vnic_plcmode_cfg(bp, vnic);
+			if (rc)
+				return rc;
+		}
 	}
 
 	return rc;
@@ -3061,6 +3108,18 @@ static bool bnxt_vf_pciid(uint16_t id)
 	    id == BROADCOM_DEV_ID_5741X_VF ||
 	    id == BROADCOM_DEV_ID_57414_VF ||
 	    id == BROADCOM_DEV_ID_STRATUS_NIC_VF1 ||
+	    id == BROADCOM_DEV_ID_STRATUS_NIC_VF2 ||
+	    id == BROADCOM_DEV_ID_58802_VF)
+		return true;
+	return false;
+}
+
+bool bnxt_stratus_device(struct bnxt *bp)
+{
+	uint16_t id = bp->pdev->id.device_id;
+
+	if (id == BROADCOM_DEV_ID_STRATUS_NIC ||
+	    id == BROADCOM_DEV_ID_STRATUS_NIC_VF1 ||
 	    id == BROADCOM_DEV_ID_STRATUS_NIC_VF2)
 		return true;
 	return false;
@@ -3112,7 +3171,6 @@ init_err_disable:
 	return rc;
 }
 
-static int bnxt_dev_uninit(struct rte_eth_dev *eth_dev);
 
 #define ALLOW_FUNC(x)	\
 	{ \
@@ -3404,13 +3462,15 @@ error:
 }
 
 static int
-bnxt_dev_uninit(struct rte_eth_dev *eth_dev) {
+bnxt_dev_uninit(struct rte_eth_dev *eth_dev)
+{
 	struct bnxt *bp = eth_dev->data->dev_private;
 	int rc;
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return -EPERM;
 
+	PMD_DRV_LOG(DEBUG, "Calling Device uninit\n");
 	bnxt_disable_int(bp);
 	bnxt_free_int(bp);
 	bnxt_free_mem(bp);
@@ -3424,8 +3484,17 @@ bnxt_dev_uninit(struct rte_eth_dev *eth_dev) {
 	}
 	rc = bnxt_hwrm_func_driver_unregister(bp, 0);
 	bnxt_free_hwrm_resources(bp);
-	rte_memzone_free((const struct rte_memzone *)bp->tx_mem_zone);
-	rte_memzone_free((const struct rte_memzone *)bp->rx_mem_zone);
+
+	if (bp->tx_mem_zone) {
+		rte_memzone_free((const struct rte_memzone *)bp->tx_mem_zone);
+		bp->tx_mem_zone = NULL;
+	}
+
+	if (bp->rx_mem_zone) {
+		rte_memzone_free((const struct rte_memzone *)bp->rx_mem_zone);
+		bp->rx_mem_zone = NULL;
+	}
+
 	if (bp->dev_stopped == 0)
 		bnxt_dev_close_op(eth_dev);
 	if (bp->pf.vf_info)
@@ -3471,9 +3540,7 @@ bool is_bnxt_supported(struct rte_eth_dev *dev)
 	return is_device_supported(dev, &bnxt_rte_pmd);
 }
 
-RTE_INIT(bnxt_init_log);
-static void
-bnxt_init_log(void)
+RTE_INIT(bnxt_init_log)
 {
 	bnxt_logtype_driver = rte_log_register("pmd.bnxt.driver");
 	if (bnxt_logtype_driver >= 0)
diff --git a/drivers/net/bnxt/bnxt_filter.c b/drivers/net/bnxt/bnxt_filter.c
index e36da997..1038941e 100644
--- a/drivers/net/bnxt/bnxt_filter.c
+++ b/drivers/net/bnxt/bnxt_filter.c
@@ -117,16 +117,29 @@ void bnxt_free_filter_mem(struct bnxt *bp)
 	max_filters = bp->max_l2_ctx;
 	for (i = 0; i < max_filters; i++) {
 		filter = &bp->filter_info[i];
-		if (filter->fw_l2_filter_id != ((uint64_t)-1)) {
-			PMD_DRV_LOG(ERR, "HWRM filter is not freed??\n");
+		if (filter->fw_l2_filter_id != ((uint64_t)-1) &&
+		    filter->filter_type == HWRM_CFA_L2_FILTER) {
+			PMD_DRV_LOG(ERR, "L2 filter is not free\n");
 			/* Call HWRM to try to free filter again */
 			rc = bnxt_hwrm_clear_l2_filter(bp, filter);
 			if (rc)
 				PMD_DRV_LOG(ERR,
-				       "HWRM filter cannot be freed rc = %d\n",
-					rc);
+					    "Cannot free L2 filter: %d\n",
+					    rc);
 		}
 		filter->fw_l2_filter_id = UINT64_MAX;
+
+		if (filter->fw_ntuple_filter_id != ((uint64_t)-1) &&
+		    filter->filter_type == HWRM_CFA_NTUPLE_FILTER) {
+			PMD_DRV_LOG(ERR, "NTUPLE filter is not free\n");
+			/* Call HWRM to try to free filter again */
+			rc = bnxt_hwrm_clear_ntuple_filter(bp, filter);
+			if (rc)
+				PMD_DRV_LOG(ERR,
+					    "Cannot free NTUPLE filter: %d\n",
+					    rc);
+		}
+		filter->fw_ntuple_filter_id = UINT64_MAX;
 	}
 	STAILQ_INIT(&bp->free_filter_list);
 
@@ -180,1072 +193,3 @@ void bnxt_free_filter(struct bnxt *bp, struct bnxt_filter_info *filter)
 {
 	STAILQ_INSERT_TAIL(&bp->free_filter_list, filter, next);
 }
-
-static int
-bnxt_flow_agrs_validate(const struct rte_flow_attr *attr,
-			const struct rte_flow_item pattern[],
-			const struct rte_flow_action actions[],
-			struct rte_flow_error *error)
-{
-	if (!pattern) {
-		rte_flow_error_set(error, EINVAL,
-			RTE_FLOW_ERROR_TYPE_ITEM_NUM,
-			NULL, "NULL pattern.");
-		return -rte_errno;
-	}
-
-	if (!actions) {
-		rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_ACTION_NUM,
-				   NULL, "NULL action.");
-		return -rte_errno;
-	}
-
-	if (!attr) {
-		rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_ATTR,
-				   NULL, "NULL attribute.");
-		return -rte_errno;
-	}
-
-	return 0;
-}
-
-static const struct rte_flow_item *
-nxt_non_void_pattern(const struct rte_flow_item *cur)
-{
-	while (1) {
-		if (cur->type != RTE_FLOW_ITEM_TYPE_VOID)
-			return cur;
-		cur++;
-	}
-}
-
-static const struct rte_flow_action *
-nxt_non_void_action(const struct rte_flow_action *cur)
-{
-	while (1) {
-		if (cur->type != RTE_FLOW_ACTION_TYPE_VOID)
-			return cur;
-		cur++;
-	}
-}
-
-int bnxt_check_zero_bytes(const uint8_t *bytes, int len)
-{
-	int i;
-	for (i = 0; i < len; i++)
-		if (bytes[i] != 0x00)
-			return 0;
-	return 1;
-}
-
-static int
-bnxt_filter_type_check(const struct rte_flow_item pattern[],
-		       struct rte_flow_error *error __rte_unused)
-{
-	const struct rte_flow_item *item = nxt_non_void_pattern(pattern);
-	int use_ntuple = 1;
-
-	while (item->type != RTE_FLOW_ITEM_TYPE_END) {
-		switch (item->type) {
-		case RTE_FLOW_ITEM_TYPE_ETH:
-			use_ntuple = 1;
-			break;
-		case RTE_FLOW_ITEM_TYPE_VLAN:
-			use_ntuple = 0;
-			break;
-		case RTE_FLOW_ITEM_TYPE_IPV4:
-		case RTE_FLOW_ITEM_TYPE_IPV6:
-		case RTE_FLOW_ITEM_TYPE_TCP:
-		case RTE_FLOW_ITEM_TYPE_UDP:
-			/* FALLTHROUGH */
-			/* need ntuple match, reset exact match */
-			if (!use_ntuple) {
-				PMD_DRV_LOG(ERR,
-					"VLAN flow cannot use NTUPLE filter\n");
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   item,
-						   "Cannot use VLAN with NTUPLE");
-				return -rte_errno;
-			}
-			use_ntuple |= 1;
-			break;
-		default:
-			PMD_DRV_LOG(ERR, "Unknown Flow type");
-			use_ntuple |= 1;
-		}
-		item++;
-	}
-	return use_ntuple;
-}
-
-static int
-bnxt_validate_and_parse_flow_type(struct bnxt *bp,
-				  const struct rte_flow_attr *attr,
-				  const struct rte_flow_item pattern[],
-				  struct rte_flow_error *error,
-				  struct bnxt_filter_info *filter)
-{
-	const struct rte_flow_item *item = nxt_non_void_pattern(pattern);
-	const struct rte_flow_item_vlan *vlan_spec, *vlan_mask;
-	const struct rte_flow_item_ipv4 *ipv4_spec, *ipv4_mask;
-	const struct rte_flow_item_ipv6 *ipv6_spec, *ipv6_mask;
-	const struct rte_flow_item_tcp *tcp_spec, *tcp_mask;
-	const struct rte_flow_item_udp *udp_spec, *udp_mask;
-	const struct rte_flow_item_eth *eth_spec, *eth_mask;
-	const struct rte_flow_item_nvgre *nvgre_spec;
-	const struct rte_flow_item_nvgre *nvgre_mask;
-	const struct rte_flow_item_vxlan *vxlan_spec;
-	const struct rte_flow_item_vxlan *vxlan_mask;
-	uint8_t vni_mask[] = {0xFF, 0xFF, 0xFF};
-	uint8_t tni_mask[] = {0xFF, 0xFF, 0xFF};
-	const struct rte_flow_item_vf *vf_spec;
-	uint32_t tenant_id_be = 0;
-	bool vni_masked = 0;
-	bool tni_masked = 0;
-	uint32_t vf = 0;
-	int use_ntuple;
-	uint32_t en = 0;
-	uint32_t en_ethertype;
-	int dflt_vnic;
-
-	use_ntuple = bnxt_filter_type_check(pattern, error);
-	PMD_DRV_LOG(DEBUG, "Use NTUPLE %d\n", use_ntuple);
-	if (use_ntuple < 0)
-		return use_ntuple;
-
-	filter->filter_type = use_ntuple ?
-		HWRM_CFA_NTUPLE_FILTER : HWRM_CFA_EM_FILTER;
-	en_ethertype = use_ntuple ?
-		NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE :
-		EM_FLOW_ALLOC_INPUT_EN_ETHERTYPE;
-
-	while (item->type != RTE_FLOW_ITEM_TYPE_END) {
-		if (item->last) {
-			/* last or range is NOT supported as match criteria */
-			rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "No support for range");
-			return -rte_errno;
-		}
-		if (!item->spec || !item->mask) {
-			rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "spec/mask is NULL");
-			return -rte_errno;
-		}
-		switch (item->type) {
-		case RTE_FLOW_ITEM_TYPE_ETH:
-			eth_spec = item->spec;
-			eth_mask = item->mask;
-
-			/* Source MAC address mask cannot be partially set.
-			 * Should be All 0's or all 1's.
-			 * Destination MAC address mask must not be partially
-			 * set. Should be all 1's or all 0's.
-			 */
-			if ((!is_zero_ether_addr(&eth_mask->src) &&
-			     !is_broadcast_ether_addr(&eth_mask->src)) ||
-			    (!is_zero_ether_addr(&eth_mask->dst) &&
-			     !is_broadcast_ether_addr(&eth_mask->dst))) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   item,
-						   "MAC_addr mask not valid");
-				return -rte_errno;
-			}
-
-			/* Mask is not allowed. Only exact matches are */
-			if (eth_mask->type &&
-			    eth_mask->type != RTE_BE16(0xffff)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   item,
-						   "ethertype mask not valid");
-				return -rte_errno;
-			}
-
-			if (is_broadcast_ether_addr(&eth_mask->dst)) {
-				rte_memcpy(filter->dst_macaddr,
-					   &eth_spec->dst, 6);
-				en |= use_ntuple ?
-					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR :
-					EM_FLOW_ALLOC_INPUT_EN_DST_MACADDR;
-			}
-			if (is_broadcast_ether_addr(&eth_mask->src)) {
-				rte_memcpy(filter->src_macaddr,
-					   &eth_spec->src, 6);
-				en |= use_ntuple ?
-					NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_MACADDR :
-					EM_FLOW_ALLOC_INPUT_EN_SRC_MACADDR;
-			} /*
-			   * else {
-			   *  RTE_LOG(ERR, PMD, "Handle this condition\n");
-			   * }
-			   */
-			if (eth_mask->type) {
-				filter->ethertype =
-					rte_be_to_cpu_16(eth_spec->type);
-				en |= en_ethertype;
-			}
-
-			break;
-		case RTE_FLOW_ITEM_TYPE_VLAN:
-			vlan_spec = item->spec;
-			vlan_mask = item->mask;
-			if (en & en_ethertype) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   item,
-						   "VLAN TPID matching is not"
-						   " supported");
-				return -rte_errno;
-			}
-			if (vlan_mask->tci &&
-			    vlan_mask->tci == RTE_BE16(0x0fff)) {
-				/* Only the VLAN ID can be matched. */
-				filter->l2_ovlan =
-					rte_be_to_cpu_16(vlan_spec->tci &
-							 RTE_BE16(0x0fff));
-				en |= EM_FLOW_ALLOC_INPUT_EN_OVLAN_VID;
-			} else if (vlan_mask->tci) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   item,
-						   "VLAN mask is invalid");
-				return -rte_errno;
-			}
-			if (vlan_mask->inner_type &&
-			    vlan_mask->inner_type != RTE_BE16(0xffff)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   item,
-						   "inner ethertype mask not"
-						   " valid");
-				return -rte_errno;
-			}
-			if (vlan_mask->inner_type) {
-				filter->ethertype =
-					rte_be_to_cpu_16(vlan_spec->inner_type);
-				en |= en_ethertype;
-			}
-
-			break;
-		case RTE_FLOW_ITEM_TYPE_IPV4:
-			/* If mask is not involved, we could use EM filters. */
-			ipv4_spec = item->spec;
-			ipv4_mask = item->mask;
-			/* Only IP DST and SRC fields are maskable. */
-			if (ipv4_mask->hdr.version_ihl ||
-			    ipv4_mask->hdr.type_of_service ||
-			    ipv4_mask->hdr.total_length ||
-			    ipv4_mask->hdr.packet_id ||
-			    ipv4_mask->hdr.fragment_offset ||
-			    ipv4_mask->hdr.time_to_live ||
-			    ipv4_mask->hdr.next_proto_id ||
-			    ipv4_mask->hdr.hdr_checksum) {
-				rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Invalid IPv4 mask.");
-				return -rte_errno;
-			}
-			filter->dst_ipaddr[0] = ipv4_spec->hdr.dst_addr;
-			filter->src_ipaddr[0] = ipv4_spec->hdr.src_addr;
-			if (use_ntuple)
-				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR |
-					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
-			else
-				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_IPADDR |
-					EM_FLOW_ALLOC_INPUT_EN_DST_IPADDR;
-			if (ipv4_mask->hdr.src_addr) {
-				filter->src_ipaddr_mask[0] =
-					ipv4_mask->hdr.src_addr;
-				en |= !use_ntuple ? 0 :
-				     NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
-			}
-			if (ipv4_mask->hdr.dst_addr) {
-				filter->dst_ipaddr_mask[0] =
-					ipv4_mask->hdr.dst_addr;
-				en |= !use_ntuple ? 0 :
-				     NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
-			}
-			filter->ip_addr_type = use_ntuple ?
-			 HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV4 :
-			 HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV4;
-			if (ipv4_spec->hdr.next_proto_id) {
-				filter->ip_protocol =
-					ipv4_spec->hdr.next_proto_id;
-				if (use_ntuple)
-					en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
-				else
-					en |= EM_FLOW_ALLOC_INPUT_EN_IP_PROTO;
-			}
-			break;
-		case RTE_FLOW_ITEM_TYPE_IPV6:
-			ipv6_spec = item->spec;
-			ipv6_mask = item->mask;
-
-			/* Only IP DST and SRC fields are maskable. */
-			if (ipv6_mask->hdr.vtc_flow ||
-			    ipv6_mask->hdr.payload_len ||
-			    ipv6_mask->hdr.proto ||
-			    ipv6_mask->hdr.hop_limits) {
-				rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Invalid IPv6 mask.");
-				return -rte_errno;
-			}
-
-			if (use_ntuple)
-				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR |
-					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
-			else
-				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_IPADDR |
-					EM_FLOW_ALLOC_INPUT_EN_DST_IPADDR;
-			rte_memcpy(filter->src_ipaddr,
-				   ipv6_spec->hdr.src_addr, 16);
-			rte_memcpy(filter->dst_ipaddr,
-				   ipv6_spec->hdr.dst_addr, 16);
-			if (!bnxt_check_zero_bytes(ipv6_mask->hdr.src_addr,
-						   16)) {
-				rte_memcpy(filter->src_ipaddr_mask,
-					   ipv6_mask->hdr.src_addr, 16);
-				en |= !use_ntuple ? 0 :
-				    NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
-			}
-			if (!bnxt_check_zero_bytes(ipv6_mask->hdr.dst_addr,
-						   16)) {
-				rte_memcpy(filter->dst_ipaddr_mask,
-					   ipv6_mask->hdr.dst_addr, 16);
-				en |= !use_ntuple ? 0 :
-				     NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
-			}
-			filter->ip_addr_type = use_ntuple ?
-				NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 :
-				EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV6;
-			break;
-		case RTE_FLOW_ITEM_TYPE_TCP:
-			tcp_spec = item->spec;
-			tcp_mask = item->mask;
-
-			/* Check TCP mask. Only DST & SRC ports are maskable */
-			if (tcp_mask->hdr.sent_seq ||
-			    tcp_mask->hdr.recv_ack ||
-			    tcp_mask->hdr.data_off ||
-			    tcp_mask->hdr.tcp_flags ||
-			    tcp_mask->hdr.rx_win ||
-			    tcp_mask->hdr.cksum ||
-			    tcp_mask->hdr.tcp_urp) {
-				rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Invalid TCP mask");
-				return -rte_errno;
-			}
-			filter->src_port = tcp_spec->hdr.src_port;
-			filter->dst_port = tcp_spec->hdr.dst_port;
-			if (use_ntuple)
-				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT |
-					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
-			else
-				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_PORT |
-					EM_FLOW_ALLOC_INPUT_EN_DST_PORT;
-			if (tcp_mask->hdr.dst_port) {
-				filter->dst_port_mask = tcp_mask->hdr.dst_port;
-				en |= !use_ntuple ? 0 :
-				  NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
-			}
-			if (tcp_mask->hdr.src_port) {
-				filter->src_port_mask = tcp_mask->hdr.src_port;
-				en |= !use_ntuple ? 0 :
-				  NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
-			}
-			break;
-		case RTE_FLOW_ITEM_TYPE_UDP:
-			udp_spec = item->spec;
-			udp_mask = item->mask;
-
-			if (udp_mask->hdr.dgram_len ||
-			    udp_mask->hdr.dgram_cksum) {
-				rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Invalid UDP mask");
-				return -rte_errno;
-			}
-
-			filter->src_port = udp_spec->hdr.src_port;
-			filter->dst_port = udp_spec->hdr.dst_port;
-			if (use_ntuple)
-				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT |
-					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
-			else
-				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_PORT |
-					EM_FLOW_ALLOC_INPUT_EN_DST_PORT;
-
-			if (udp_mask->hdr.dst_port) {
-				filter->dst_port_mask = udp_mask->hdr.dst_port;
-				en |= !use_ntuple ? 0 :
-				  NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
-			}
-			if (udp_mask->hdr.src_port) {
-				filter->src_port_mask = udp_mask->hdr.src_port;
-				en |= !use_ntuple ? 0 :
-				  NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
-			}
-			break;
-		case RTE_FLOW_ITEM_TYPE_VXLAN:
-			vxlan_spec = item->spec;
-			vxlan_mask = item->mask;
-			/* Check if VXLAN item is used to describe protocol.
-			 * If yes, both spec and mask should be NULL.
-			 * If no, both spec and mask shouldn't be NULL.
-			 */
-			if ((!vxlan_spec && vxlan_mask) ||
-			    (vxlan_spec && !vxlan_mask)) {
-				rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Invalid VXLAN item");
-				return -rte_errno;
-			}
-
-			if (vxlan_spec->rsvd1 || vxlan_spec->rsvd0[0] ||
-			    vxlan_spec->rsvd0[1] || vxlan_spec->rsvd0[2] ||
-			    vxlan_spec->flags != 0x8) {
-				rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Invalid VXLAN item");
-				return -rte_errno;
-			}
-
-			/* Check if VNI is masked. */
-			if (vxlan_spec && vxlan_mask) {
-				vni_masked =
-					!!memcmp(vxlan_mask->vni, vni_mask,
-						 RTE_DIM(vni_mask));
-				if (vni_masked) {
-					rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   item,
-						   "Invalid VNI mask");
-					return -rte_errno;
-				}
-
-				rte_memcpy(((uint8_t *)&tenant_id_be + 1),
-					   vxlan_spec->vni, 3);
-				filter->vni =
-					rte_be_to_cpu_32(tenant_id_be);
-				filter->tunnel_type =
-				 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
-			}
-			break;
-		case RTE_FLOW_ITEM_TYPE_NVGRE:
-			nvgre_spec = item->spec;
-			nvgre_mask = item->mask;
-			/* Check if NVGRE item is used to describe protocol.
-			 * If yes, both spec and mask should be NULL.
-			 * If no, both spec and mask shouldn't be NULL.
-			 */
-			if ((!nvgre_spec && nvgre_mask) ||
-			    (nvgre_spec && !nvgre_mask)) {
-				rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Invalid NVGRE item");
-				return -rte_errno;
-			}
-
-			if (nvgre_spec->c_k_s_rsvd0_ver != 0x2000 ||
-			    nvgre_spec->protocol != 0x6558) {
-				rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Invalid NVGRE item");
-				return -rte_errno;
-			}
-
-			if (nvgre_spec && nvgre_mask) {
-				tni_masked =
-					!!memcmp(nvgre_mask->tni, tni_mask,
-						 RTE_DIM(tni_mask));
-				if (tni_masked) {
-					rte_flow_error_set(error, EINVAL,
-						       RTE_FLOW_ERROR_TYPE_ITEM,
-						       item,
-						       "Invalid TNI mask");
-					return -rte_errno;
-				}
-				rte_memcpy(((uint8_t *)&tenant_id_be + 1),
-					   nvgre_spec->tni, 3);
-				filter->vni =
-					rte_be_to_cpu_32(tenant_id_be);
-				filter->tunnel_type =
-				 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE;
-			}
-			break;
-		case RTE_FLOW_ITEM_TYPE_VF:
-			vf_spec = item->spec;
-			vf = vf_spec->id;
-			if (!BNXT_PF(bp)) {
-				rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Configuring on a VF!");
-				return -rte_errno;
-			}
-
-			if (vf >= bp->pdev->max_vfs) {
-				rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Incorrect VF id!");
-				return -rte_errno;
-			}
-
-			if (!attr->transfer) {
-				rte_flow_error_set(error, ENOTSUP,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Matching VF traffic without"
-					   " affecting it (transfer attribute)"
-					   " is unsupported");
-				return -rte_errno;
-			}
-
-			filter->mirror_vnic_id =
-			dflt_vnic = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf);
-			if (dflt_vnic < 0) {
-				/* This simply indicates there's no driver
-				 * loaded. This is not an error.
-				 */
-				rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ITEM,
-					   item,
-					   "Unable to get default VNIC for VF");
-				return -rte_errno;
-			}
-			filter->mirror_vnic_id = dflt_vnic;
-			en |= NTUPLE_FLTR_ALLOC_INPUT_EN_MIRROR_VNIC_ID;
-			break;
-		default:
-			break;
-		}
-		item++;
-	}
-	filter->enables = en;
-
-	return 0;
-}
-
-/* Parse attributes */
-static int
-bnxt_flow_parse_attr(const struct rte_flow_attr *attr,
-		     struct rte_flow_error *error)
-{
-	/* Must be input direction */
-	if (!attr->ingress) {
-		rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
-				   attr, "Only support ingress.");
-		return -rte_errno;
-	}
-
-	/* Not supported */
-	if (attr->egress) {
-		rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
-				   attr, "No support for egress.");
-		return -rte_errno;
-	}
-
-	/* Not supported */
-	if (attr->priority) {
-		rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
-				   attr, "No support for priority.");
-		return -rte_errno;
-	}
-
-	/* Not supported */
-	if (attr->group) {
-		rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
-				   attr, "No support for group.");
-		return -rte_errno;
-	}
-
-	return 0;
-}
-
-struct bnxt_filter_info *
-bnxt_get_l2_filter(struct bnxt *bp, struct bnxt_filter_info *nf,
-		   struct bnxt_vnic_info *vnic)
-{
-	struct bnxt_filter_info *filter1, *f0;
-	struct bnxt_vnic_info *vnic0;
-	int rc;
-
-	vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
-	f0 = STAILQ_FIRST(&vnic0->filter);
-
-	//This flow has same DST MAC as the port/l2 filter.
-	if (memcmp(f0->l2_addr, nf->dst_macaddr, ETHER_ADDR_LEN) == 0)
-		return f0;
-
-	//This flow needs DST MAC which is not same as port/l2
-	PMD_DRV_LOG(DEBUG, "Create L2 filter for DST MAC\n");
-	filter1 = bnxt_get_unused_filter(bp);
-	if (filter1 == NULL)
-		return NULL;
-	filter1->flags = HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX;
-	filter1->enables = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR |
-			L2_FILTER_ALLOC_INPUT_EN_L2_ADDR_MASK;
-	memcpy(filter1->l2_addr, nf->dst_macaddr, ETHER_ADDR_LEN);
-	memset(filter1->l2_addr_mask, 0xff, ETHER_ADDR_LEN);
-	rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id,
-				     filter1);
-	if (rc) {
-		bnxt_free_filter(bp, filter1);
-		return NULL;
-	}
-	return filter1;
-}
-
-static int
-bnxt_validate_and_parse_flow(struct rte_eth_dev *dev,
-			     const struct rte_flow_item pattern[],
-			     const struct rte_flow_action actions[],
-			     const struct rte_flow_attr *attr,
-			     struct rte_flow_error *error,
-			     struct bnxt_filter_info *filter)
-{
-	const struct rte_flow_action *act = nxt_non_void_action(actions);
-	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
-	const struct rte_flow_action_queue *act_q;
-	const struct rte_flow_action_vf *act_vf;
-	struct bnxt_vnic_info *vnic, *vnic0;
-	struct bnxt_filter_info *filter1;
-	uint32_t vf = 0;
-	int dflt_vnic;
-	int rc;
-
-	if (bp->eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) {
-		PMD_DRV_LOG(ERR, "Cannot create flow on RSS queues\n");
-		rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "Cannot create flow on RSS queues");
-		rc = -rte_errno;
-		goto ret;
-	}
-
-	rc = bnxt_validate_and_parse_flow_type(bp, attr, pattern, error,
-					       filter);
-	if (rc != 0)
-		goto ret;
-
-	rc = bnxt_flow_parse_attr(attr, error);
-	if (rc != 0)
-		goto ret;
-	//Since we support ingress attribute only - right now.
-	if (filter->filter_type == HWRM_CFA_EM_FILTER)
-		filter->flags = HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_RX;
-
-	switch (act->type) {
-	case RTE_FLOW_ACTION_TYPE_QUEUE:
-		/* Allow this flow. Redirect to a VNIC. */
-		act_q = (const struct rte_flow_action_queue *)act->conf;
-		if (act_q->index >= bp->rx_nr_rings) {
-			rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ACTION, act,
-					   "Invalid queue ID.");
-			rc = -rte_errno;
-			goto ret;
-		}
-		PMD_DRV_LOG(DEBUG, "Queue index %d\n", act_q->index);
-
-		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
-		vnic = STAILQ_FIRST(&bp->ff_pool[act_q->index]);
-		if (vnic == NULL) {
-			rte_flow_error_set(error, EINVAL,
-					   RTE_FLOW_ERROR_TYPE_ACTION, act,
-					   "No matching VNIC for queue ID.");
-			rc = -rte_errno;
-			goto ret;
-		}
-		filter->dst_id = vnic->fw_vnic_id;
-		filter1 = bnxt_get_l2_filter(bp, filter, vnic);
-		if (filter1 == NULL) {
-			rc = -ENOSPC;
-			goto ret;
-		}
-		filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
-		PMD_DRV_LOG(DEBUG, "VNIC found\n");
-		break;
-	case RTE_FLOW_ACTION_TYPE_DROP:
-		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
-		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
-		if (filter1 == NULL) {
-			rc = -ENOSPC;
-			goto ret;
-		}
-		filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
-		if (filter->filter_type == HWRM_CFA_EM_FILTER)
-			filter->flags =
-				HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DROP;
-		else
-			filter->flags =
-				HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP;
-		break;
-	case RTE_FLOW_ACTION_TYPE_COUNT:
-		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
-		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
-		if (filter1 == NULL) {
-			rc = -ENOSPC;
-			goto ret;
-		}
-		filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
-		filter->flags = HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_METER;
-		break;
-	case RTE_FLOW_ACTION_TYPE_VF:
-		act_vf = (const struct rte_flow_action_vf *)act->conf;
-		vf = act_vf->id;
-		if (!BNXT_PF(bp)) {
-			rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_ACTION,
-				   act,
-				   "Configuring on a VF!");
-			rc = -rte_errno;
-			goto ret;
-		}
-
-		if (vf >= bp->pdev->max_vfs) {
-			rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_ACTION,
-				   act,
-				   "Incorrect VF id!");
-			rc = -rte_errno;
-			goto ret;
-		}
-
-		filter->mirror_vnic_id =
-		dflt_vnic = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf);
-		if (dflt_vnic < 0) {
-			/* This simply indicates there's no driver loaded.
-			 * This is not an error.
-			 */
-			rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_ACTION,
-				   act,
-				   "Unable to get default VNIC for VF");
-			rc = -rte_errno;
-			goto ret;
-		}
-		filter->mirror_vnic_id = dflt_vnic;
-		filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_MIRROR_VNIC_ID;
-
-		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
-		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
-		if (filter1 == NULL) {
-			rc = -ENOSPC;
-			goto ret;
-		}
-		filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
-		break;
-
-	default:
-		rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_ACTION, act,
-				   "Invalid action.");
-		rc = -rte_errno;
-		goto ret;
-	}
-
-	act = nxt_non_void_action(++act);
-	if (act->type != RTE_FLOW_ACTION_TYPE_END) {
-		rte_flow_error_set(error, EINVAL,
-				   RTE_FLOW_ERROR_TYPE_ACTION,
-				   act, "Invalid action.");
-		rc = -rte_errno;
-		goto ret;
-	}
-ret:
-	return rc;
-}
-
-static int
-bnxt_flow_validate(struct rte_eth_dev *dev,
-		const struct rte_flow_attr *attr,
-		const struct rte_flow_item pattern[],
-		const struct rte_flow_action actions[],
-		struct rte_flow_error *error)
-{
-	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
-	struct bnxt_filter_info *filter;
-	int ret = 0;
-
-	ret = bnxt_flow_agrs_validate(attr, pattern, actions, error);
-	if (ret != 0)
-		return ret;
-
-	filter = bnxt_get_unused_filter(bp);
-	if (filter == NULL) {
-		PMD_DRV_LOG(ERR, "Not enough resources for a new flow.\n");
-		return -ENOMEM;
-	}
-
-	ret = bnxt_validate_and_parse_flow(dev, pattern, actions, attr,
-					   error, filter);
-	/* No need to hold on to this filter if we are just validating flow */
-	filter->fw_l2_filter_id = UINT64_MAX;
-	bnxt_free_filter(bp, filter);
-
-	return ret;
-}
-
-static int
-bnxt_match_filter(struct bnxt *bp, struct bnxt_filter_info *nf)
-{
-	struct bnxt_filter_info *mf;
-	struct rte_flow *flow;
-	int i;
-
-	for (i = bp->nr_vnics - 1; i >= 0; i--) {
-		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
-
-		STAILQ_FOREACH(flow, &vnic->flow_list, next) {
-			mf = flow->filter;
-
-			if (mf->filter_type == nf->filter_type &&
-			    mf->flags == nf->flags &&
-			    mf->src_port == nf->src_port &&
-			    mf->src_port_mask == nf->src_port_mask &&
-			    mf->dst_port == nf->dst_port &&
-			    mf->dst_port_mask == nf->dst_port_mask &&
-			    mf->ip_protocol == nf->ip_protocol &&
-			    mf->ip_addr_type == nf->ip_addr_type &&
-			    mf->ethertype == nf->ethertype &&
-			    mf->vni == nf->vni &&
-			    mf->tunnel_type == nf->tunnel_type &&
-			    mf->l2_ovlan == nf->l2_ovlan &&
-			    mf->l2_ovlan_mask == nf->l2_ovlan_mask &&
-			    mf->l2_ivlan == nf->l2_ivlan &&
-			    mf->l2_ivlan_mask == nf->l2_ivlan_mask &&
-			    !memcmp(mf->l2_addr, nf->l2_addr, ETHER_ADDR_LEN) &&
-			    !memcmp(mf->l2_addr_mask, nf->l2_addr_mask,
-				    ETHER_ADDR_LEN) &&
-			    !memcmp(mf->src_macaddr, nf->src_macaddr,
-				    ETHER_ADDR_LEN) &&
-			    !memcmp(mf->dst_macaddr, nf->dst_macaddr,
-				    ETHER_ADDR_LEN) &&
-			    !memcmp(mf->src_ipaddr, nf->src_ipaddr,
-				    sizeof(nf->src_ipaddr)) &&
-			    !memcmp(mf->src_ipaddr_mask, nf->src_ipaddr_mask,
-				    sizeof(nf->src_ipaddr_mask)) &&
-			    !memcmp(mf->dst_ipaddr, nf->dst_ipaddr,
-				    sizeof(nf->dst_ipaddr)) &&
-			    !memcmp(mf->dst_ipaddr_mask, nf->dst_ipaddr_mask,
-				    sizeof(nf->dst_ipaddr_mask))) {
-				if (mf->dst_id == nf->dst_id)
-					return -EEXIST;
-				/* Same Flow, Different queue
-				 * Clear the old ntuple filter
-				 */
-				if (nf->filter_type == HWRM_CFA_EM_FILTER)
-					bnxt_hwrm_clear_em_filter(bp, mf);
-				if (nf->filter_type == HWRM_CFA_NTUPLE_FILTER)
-					bnxt_hwrm_clear_ntuple_filter(bp, mf);
-				/* Free the old filter, update flow
-				 * with new filter
-				 */
-				bnxt_free_filter(bp, mf);
-				flow->filter = nf;
-				return -EXDEV;
-			}
-		}
-	}
-	return 0;
-}
-
-static struct rte_flow *
-bnxt_flow_create(struct rte_eth_dev *dev,
-		  const struct rte_flow_attr *attr,
-		  const struct rte_flow_item pattern[],
-		  const struct rte_flow_action actions[],
-		  struct rte_flow_error *error)
-{
-	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
-	struct bnxt_filter_info *filter;
-	struct bnxt_vnic_info *vnic = NULL;
-	bool update_flow = false;
-	struct rte_flow *flow;
-	unsigned int i;
-	int ret = 0;
-
-	flow = rte_zmalloc("bnxt_flow", sizeof(struct rte_flow), 0);
-	if (!flow) {
-		rte_flow_error_set(error, ENOMEM,
-				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
-				   "Failed to allocate memory");
-		return flow;
-	}
-
-	ret = bnxt_flow_agrs_validate(attr, pattern, actions, error);
-	if (ret != 0) {
-		PMD_DRV_LOG(ERR, "Not a validate flow.\n");
-		goto free_flow;
-	}
-
-	filter = bnxt_get_unused_filter(bp);
-	if (filter == NULL) {
-		PMD_DRV_LOG(ERR, "Not enough resources for a new flow.\n");
-		goto free_flow;
-	}
-
-	ret = bnxt_validate_and_parse_flow(dev, pattern, actions, attr,
-					   error, filter);
-	if (ret != 0)
-		goto free_filter;
-
-	ret = bnxt_match_filter(bp, filter);
-	if (ret == -EEXIST) {
-		PMD_DRV_LOG(DEBUG, "Flow already exists.\n");
-		/* Clear the filter that was created as part of
-		 * validate_and_parse_flow() above
-		 */
-		bnxt_hwrm_clear_l2_filter(bp, filter);
-		goto free_filter;
-	} else if (ret == -EXDEV) {
-		PMD_DRV_LOG(DEBUG, "Flow with same pattern exists");
-		PMD_DRV_LOG(DEBUG, "Updating with different destination\n");
-		update_flow = true;
-	}
-
-	if (filter->filter_type == HWRM_CFA_EM_FILTER) {
-		filter->enables |=
-			HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
-		ret = bnxt_hwrm_set_em_filter(bp, filter->dst_id, filter);
-	}
-	if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER) {
-		filter->enables |=
-			HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
-		ret = bnxt_hwrm_set_ntuple_filter(bp, filter->dst_id, filter);
-	}
-
-	for (i = 0; i < bp->nr_vnics; i++) {
-		vnic = &bp->vnic_info[i];
-		if (filter->dst_id == vnic->fw_vnic_id)
-			break;
-	}
-
-	if (!ret) {
-		flow->filter = filter;
-		flow->vnic = vnic;
-		if (update_flow) {
-			ret = -EXDEV;
-			goto free_flow;
-		}
-		PMD_DRV_LOG(ERR, "Successfully created flow.\n");
-		STAILQ_INSERT_TAIL(&vnic->flow_list, flow, next);
-		return flow;
-	}
-free_filter:
-	bnxt_free_filter(bp, filter);
-free_flow:
-	if (ret == -EEXIST)
-		rte_flow_error_set(error, ret,
-				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
-				   "Matching Flow exists.");
-	else if (ret == -EXDEV)
-		rte_flow_error_set(error, ret,
-				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
-				   "Flow with pattern exists, updating destination queue");
-	else
-		rte_flow_error_set(error, -ret,
-				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
-				   "Failed to create flow.");
-	rte_free(flow);
-	flow = NULL;
-	return flow;
-}
-
-static int
-bnxt_flow_destroy(struct rte_eth_dev *dev,
-		  struct rte_flow *flow,
-		  struct rte_flow_error *error)
-{
-	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
-	struct bnxt_filter_info *filter = flow->filter;
-	struct bnxt_vnic_info *vnic = flow->vnic;
-	int ret = 0;
-
-	ret = bnxt_match_filter(bp, filter);
-	if (ret == 0)
-		PMD_DRV_LOG(ERR, "Could not find matching flow\n");
-	if (filter->filter_type == HWRM_CFA_EM_FILTER)
-		ret = bnxt_hwrm_clear_em_filter(bp, filter);
-	if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
-		ret = bnxt_hwrm_clear_ntuple_filter(bp, filter);
-	else
-		ret = bnxt_hwrm_clear_l2_filter(bp, filter);
-	if (!ret) {
-		STAILQ_REMOVE(&vnic->flow_list, flow, rte_flow, next);
-		rte_free(flow);
-	} else {
-		rte_flow_error_set(error, -ret,
-				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
-				   "Failed to destroy flow.");
-	}
-
-	return ret;
-}
-
-static int
-bnxt_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
-{
-	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
-	struct bnxt_vnic_info *vnic;
-	struct rte_flow *flow;
-	unsigned int i;
-	int ret = 0;
-
-	for (i = 0; i < bp->nr_vnics; i++) {
-		vnic = &bp->vnic_info[i];
-		STAILQ_FOREACH(flow, &vnic->flow_list, next) {
-			struct bnxt_filter_info *filter = flow->filter;
-
-			if (filter->filter_type == HWRM_CFA_EM_FILTER)
-				ret = bnxt_hwrm_clear_em_filter(bp, filter);
-			if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
-				ret = bnxt_hwrm_clear_ntuple_filter(bp, filter);
-
-			if (ret) {
-				rte_flow_error_set(error, -ret,
-						   RTE_FLOW_ERROR_TYPE_HANDLE,
-						   NULL,
-						   "Failed to flush flow in HW.");
-				return -rte_errno;
-			}
-
-			STAILQ_REMOVE(&vnic->flow_list, flow,
-				      rte_flow, next);
-			rte_free(flow);
-		}
-	}
-
-	return ret;
-}
-
-const struct rte_flow_ops bnxt_flow_ops = {
-	.validate = bnxt_flow_validate,
-	.create = bnxt_flow_create,
-	.destroy = bnxt_flow_destroy,
-	.flush = bnxt_flow_flush,
-};
diff --git a/drivers/net/bnxt/bnxt_filter.h b/drivers/net/bnxt/bnxt_filter.h
index d27be703..a1ecfb19 100644
--- a/drivers/net/bnxt/bnxt_filter.h
+++ b/drivers/net/bnxt/bnxt_filter.h
@@ -69,7 +69,6 @@ struct bnxt_filter_info *bnxt_get_unused_filter(struct bnxt *bp);
 void bnxt_free_filter(struct bnxt *bp, struct bnxt_filter_info *filter);
 struct bnxt_filter_info *bnxt_get_l2_filter(struct bnxt *bp,
 		struct bnxt_filter_info *nf, struct bnxt_vnic_info *vnic);
-int bnxt_check_zero_bytes(const uint8_t *bytes, int len);
 
 #define NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_MACADDR	\
 	HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_MACADDR
diff --git a/drivers/net/bnxt/bnxt_flow.c b/drivers/net/bnxt/bnxt_flow.c
new file mode 100644
index 00000000..ac765674
--- /dev/null
+++ b/drivers/net/bnxt/bnxt_flow.c
@@ -0,0 +1,1171 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2018 Broadcom
+ * All rights reserved.
+ */
+
+#include <sys/queue.h>
+
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_flow.h>
+#include <rte_flow_driver.h>
+#include <rte_tailq.h>
+
+#include "bnxt.h"
+#include "bnxt_filter.h"
+#include "bnxt_hwrm.h"
+#include "bnxt_vnic.h"
+#include "bnxt_util.h"
+#include "hsi_struct_def_dpdk.h"
+
+static int
+bnxt_flow_args_validate(const struct rte_flow_attr *attr,
+			const struct rte_flow_item pattern[],
+			const struct rte_flow_action actions[],
+			struct rte_flow_error *error)
+{
+	if (!pattern) {
+		rte_flow_error_set(error,
+				   EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ITEM_NUM,
+				   NULL,
+				   "NULL pattern.");
+		return -rte_errno;
+	}
+
+	if (!actions) {
+		rte_flow_error_set(error,
+				   EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ACTION_NUM,
+				   NULL,
+				   "NULL action.");
+		return -rte_errno;
+	}
+
+	if (!attr) {
+		rte_flow_error_set(error,
+				   EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ATTR,
+				   NULL,
+				   "NULL attribute.");
+		return -rte_errno;
+	}
+
+	return 0;
+}
+
+static const struct rte_flow_item *
+bnxt_flow_non_void_item(const struct rte_flow_item *cur)
+{
+	while (1) {
+		if (cur->type != RTE_FLOW_ITEM_TYPE_VOID)
+			return cur;
+		cur++;
+	}
+}
+
+static const struct rte_flow_action *
+bnxt_flow_non_void_action(const struct rte_flow_action *cur)
+{
+	while (1) {
+		if (cur->type != RTE_FLOW_ACTION_TYPE_VOID)
+			return cur;
+		cur++;
+	}
+}
+
+static int
+bnxt_filter_type_check(const struct rte_flow_item pattern[],
+		       struct rte_flow_error *error __rte_unused)
+{
+	const struct rte_flow_item *item =
+		bnxt_flow_non_void_item(pattern);
+	int use_ntuple = 1;
+
+	while (item->type != RTE_FLOW_ITEM_TYPE_END) {
+		switch (item->type) {
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			use_ntuple = 1;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			use_ntuple = 0;
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+		case RTE_FLOW_ITEM_TYPE_TCP:
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			/* FALLTHROUGH */
+			/* need ntuple match, reset exact match */
+			if (!use_ntuple) {
+				PMD_DRV_LOG(ERR,
+					"VLAN flow cannot use NTUPLE filter\n");
+				rte_flow_error_set
+					(error,
+					 EINVAL,
+					 RTE_FLOW_ERROR_TYPE_ITEM,
+					 item,
+					 "Cannot use VLAN with NTUPLE");
+				return -rte_errno;
+			}
+			use_ntuple |= 1;
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "Unknown Flow type\n");
+			use_ntuple |= 1;
+		}
+		item++;
+	}
+	return use_ntuple;
+}
+
+static int
+bnxt_validate_and_parse_flow_type(struct bnxt *bp,
+				  const struct rte_flow_attr *attr,
+				  const struct rte_flow_item pattern[],
+				  struct rte_flow_error *error,
+				  struct bnxt_filter_info *filter)
+{
+	const struct rte_flow_item *item = bnxt_flow_non_void_item(pattern);
+	const struct rte_flow_item_vlan *vlan_spec, *vlan_mask;
+	const struct rte_flow_item_ipv4 *ipv4_spec, *ipv4_mask;
+	const struct rte_flow_item_ipv6 *ipv6_spec, *ipv6_mask;
+	const struct rte_flow_item_tcp *tcp_spec, *tcp_mask;
+	const struct rte_flow_item_udp *udp_spec, *udp_mask;
+	const struct rte_flow_item_eth *eth_spec, *eth_mask;
+	const struct rte_flow_item_nvgre *nvgre_spec;
+	const struct rte_flow_item_nvgre *nvgre_mask;
+	const struct rte_flow_item_vxlan *vxlan_spec;
+	const struct rte_flow_item_vxlan *vxlan_mask;
+	uint8_t vni_mask[] = {0xFF, 0xFF, 0xFF};
+	uint8_t tni_mask[] = {0xFF, 0xFF, 0xFF};
+	const struct rte_flow_item_vf *vf_spec;
+	uint32_t tenant_id_be = 0;
+	bool vni_masked = 0;
+	bool tni_masked = 0;
+	uint32_t vf = 0;
+	int use_ntuple;
+	uint32_t en = 0;
+	uint32_t en_ethertype;
+	int dflt_vnic;
+
+	use_ntuple = bnxt_filter_type_check(pattern, error);
+	PMD_DRV_LOG(DEBUG, "Use NTUPLE %d\n", use_ntuple);
+	if (use_ntuple < 0)
+		return use_ntuple;
+
+	filter->filter_type = use_ntuple ?
+		HWRM_CFA_NTUPLE_FILTER : HWRM_CFA_EM_FILTER;
+	en_ethertype = use_ntuple ?
+		NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE :
+		EM_FLOW_ALLOC_INPUT_EN_ETHERTYPE;
+
+	while (item->type != RTE_FLOW_ITEM_TYPE_END) {
+		if (item->last) {
+			/* last or range is NOT supported as match criteria */
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   item,
+					   "No support for range");
+			return -rte_errno;
+		}
+
+		if (!item->spec || !item->mask) {
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ITEM,
+					   item,
+					   "spec/mask is NULL");
+			return -rte_errno;
+		}
+
+		switch (item->type) {
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			eth_spec = item->spec;
+			eth_mask = item->mask;
+
+			/* Source MAC address mask cannot be partially set.
+			 * Should be All 0's or all 1's.
+			 * Destination MAC address mask must not be partially
+			 * set. Should be all 1's or all 0's.
+			 */
+			if ((!is_zero_ether_addr(&eth_mask->src) &&
+			     !is_broadcast_ether_addr(&eth_mask->src)) ||
+			    (!is_zero_ether_addr(&eth_mask->dst) &&
+			     !is_broadcast_ether_addr(&eth_mask->dst))) {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "MAC_addr mask not valid");
+				return -rte_errno;
+			}
+
+			/* Mask is not allowed. Only exact matches are */
+			if (eth_mask->type &&
+			    eth_mask->type != RTE_BE16(0xffff)) {
+				rte_flow_error_set(error, EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "ethertype mask not valid");
+				return -rte_errno;
+			}
+
+			if (is_broadcast_ether_addr(&eth_mask->dst)) {
+				rte_memcpy(filter->dst_macaddr,
+					   &eth_spec->dst, 6);
+				en |= use_ntuple ?
+					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR :
+					EM_FLOW_ALLOC_INPUT_EN_DST_MACADDR;
+			}
+
+			if (is_broadcast_ether_addr(&eth_mask->src)) {
+				rte_memcpy(filter->src_macaddr,
+					   &eth_spec->src, 6);
+				en |= use_ntuple ?
+					NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_MACADDR :
+					EM_FLOW_ALLOC_INPUT_EN_SRC_MACADDR;
+			} /*
+			   * else {
+			   *  PMD_DRV_LOG(ERR, "Handle this condition\n");
+			   * }
+			   */
+			if (eth_mask->type) {
+				filter->ethertype =
+					rte_be_to_cpu_16(eth_spec->type);
+				en |= en_ethertype;
+			}
+
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			vlan_spec = item->spec;
+			vlan_mask = item->mask;
+			if (en & en_ethertype) {
+				rte_flow_error_set(error, EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "VLAN TPID matching is not"
+						   " supported");
+				return -rte_errno;
+			}
+			if (vlan_mask->tci &&
+			    vlan_mask->tci == RTE_BE16(0x0fff)) {
+				/* Only the VLAN ID can be matched. */
+				filter->l2_ovlan =
+					rte_be_to_cpu_16(vlan_spec->tci &
+							 RTE_BE16(0x0fff));
+				en |= EM_FLOW_ALLOC_INPUT_EN_OVLAN_VID;
+			} else {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "VLAN mask is invalid");
+				return -rte_errno;
+			}
+			if (vlan_mask->inner_type &&
+			    vlan_mask->inner_type != RTE_BE16(0xffff)) {
+				rte_flow_error_set(error, EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "inner ethertype mask not"
+						   " valid");
+				return -rte_errno;
+			}
+			if (vlan_mask->inner_type) {
+				filter->ethertype =
+					rte_be_to_cpu_16(vlan_spec->inner_type);
+				en |= en_ethertype;
+			}
+
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			/* If mask is not involved, we could use EM filters. */
+			ipv4_spec = item->spec;
+			ipv4_mask = item->mask;
+			/* Only IP DST and SRC fields are maskable. */
+			if (ipv4_mask->hdr.version_ihl ||
+			    ipv4_mask->hdr.type_of_service ||
+			    ipv4_mask->hdr.total_length ||
+			    ipv4_mask->hdr.packet_id ||
+			    ipv4_mask->hdr.fragment_offset ||
+			    ipv4_mask->hdr.time_to_live ||
+			    ipv4_mask->hdr.next_proto_id ||
+			    ipv4_mask->hdr.hdr_checksum) {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Invalid IPv4 mask.");
+				return -rte_errno;
+			}
+
+			filter->dst_ipaddr[0] = ipv4_spec->hdr.dst_addr;
+			filter->src_ipaddr[0] = ipv4_spec->hdr.src_addr;
+
+			if (use_ntuple)
+				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR |
+					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
+			else
+				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_IPADDR |
+					EM_FLOW_ALLOC_INPUT_EN_DST_IPADDR;
+
+			if (ipv4_mask->hdr.src_addr) {
+				filter->src_ipaddr_mask[0] =
+					ipv4_mask->hdr.src_addr;
+				en |= !use_ntuple ? 0 :
+				     NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
+			}
+
+			if (ipv4_mask->hdr.dst_addr) {
+				filter->dst_ipaddr_mask[0] =
+					ipv4_mask->hdr.dst_addr;
+				en |= !use_ntuple ? 0 :
+				     NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
+			}
+
+			filter->ip_addr_type = use_ntuple ?
+			 HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV4 :
+			 HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV4;
+
+			if (ipv4_spec->hdr.next_proto_id) {
+				filter->ip_protocol =
+					ipv4_spec->hdr.next_proto_id;
+				if (use_ntuple)
+					en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
+				else
+					en |= EM_FLOW_ALLOC_INPUT_EN_IP_PROTO;
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			ipv6_spec = item->spec;
+			ipv6_mask = item->mask;
+
+			/* Only IP DST and SRC fields are maskable. */
+			if (ipv6_mask->hdr.vtc_flow ||
+			    ipv6_mask->hdr.payload_len ||
+			    ipv6_mask->hdr.proto ||
+			    ipv6_mask->hdr.hop_limits) {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Invalid IPv6 mask.");
+				return -rte_errno;
+			}
+
+			if (use_ntuple)
+				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR |
+					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
+			else
+				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_IPADDR |
+					EM_FLOW_ALLOC_INPUT_EN_DST_IPADDR;
+
+			rte_memcpy(filter->src_ipaddr,
+				   ipv6_spec->hdr.src_addr, 16);
+			rte_memcpy(filter->dst_ipaddr,
+				   ipv6_spec->hdr.dst_addr, 16);
+
+			if (!bnxt_check_zero_bytes(ipv6_mask->hdr.src_addr,
+						   16)) {
+				rte_memcpy(filter->src_ipaddr_mask,
+					   ipv6_mask->hdr.src_addr, 16);
+				en |= !use_ntuple ? 0 :
+				    NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
+			}
+
+			if (!bnxt_check_zero_bytes(ipv6_mask->hdr.dst_addr,
+						   16)) {
+				rte_memcpy(filter->dst_ipaddr_mask,
+					   ipv6_mask->hdr.dst_addr, 16);
+				en |= !use_ntuple ? 0 :
+				     NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
+			}
+
+			filter->ip_addr_type = use_ntuple ?
+				NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 :
+				EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV6;
+			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			tcp_spec = item->spec;
+			tcp_mask = item->mask;
+
+			/* Check TCP mask. Only DST & SRC ports are maskable */
+			if (tcp_mask->hdr.sent_seq ||
+			    tcp_mask->hdr.recv_ack ||
+			    tcp_mask->hdr.data_off ||
+			    tcp_mask->hdr.tcp_flags ||
+			    tcp_mask->hdr.rx_win ||
+			    tcp_mask->hdr.cksum ||
+			    tcp_mask->hdr.tcp_urp) {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Invalid TCP mask");
+				return -rte_errno;
+			}
+
+			filter->src_port = tcp_spec->hdr.src_port;
+			filter->dst_port = tcp_spec->hdr.dst_port;
+
+			if (use_ntuple)
+				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT |
+					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
+			else
+				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_PORT |
+					EM_FLOW_ALLOC_INPUT_EN_DST_PORT;
+
+			if (tcp_mask->hdr.dst_port) {
+				filter->dst_port_mask = tcp_mask->hdr.dst_port;
+				en |= !use_ntuple ? 0 :
+				  NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
+			}
+
+			if (tcp_mask->hdr.src_port) {
+				filter->src_port_mask = tcp_mask->hdr.src_port;
+				en |= !use_ntuple ? 0 :
+				  NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			udp_spec = item->spec;
+			udp_mask = item->mask;
+
+			if (udp_mask->hdr.dgram_len ||
+			    udp_mask->hdr.dgram_cksum) {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Invalid UDP mask");
+				return -rte_errno;
+			}
+
+			filter->src_port = udp_spec->hdr.src_port;
+			filter->dst_port = udp_spec->hdr.dst_port;
+
+			if (use_ntuple)
+				en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT |
+					NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
+			else
+				en |= EM_FLOW_ALLOC_INPUT_EN_SRC_PORT |
+					EM_FLOW_ALLOC_INPUT_EN_DST_PORT;
+
+			if (udp_mask->hdr.dst_port) {
+				filter->dst_port_mask = udp_mask->hdr.dst_port;
+				en |= !use_ntuple ? 0 :
+				  NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
+			}
+
+			if (udp_mask->hdr.src_port) {
+				filter->src_port_mask = udp_mask->hdr.src_port;
+				en |= !use_ntuple ? 0 :
+				  NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN:
+			vxlan_spec = item->spec;
+			vxlan_mask = item->mask;
+			/* Check if VXLAN item is used to describe protocol.
+			 * If yes, both spec and mask should be NULL.
+			 * If no, both spec and mask shouldn't be NULL.
+			 */
+			if ((!vxlan_spec && vxlan_mask) ||
+			    (vxlan_spec && !vxlan_mask)) {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Invalid VXLAN item");
+				return -rte_errno;
+			}
+
+			if (vxlan_spec->rsvd1 || vxlan_spec->rsvd0[0] ||
+			    vxlan_spec->rsvd0[1] || vxlan_spec->rsvd0[2] ||
+			    vxlan_spec->flags != 0x8) {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Invalid VXLAN item");
+				return -rte_errno;
+			}
+
+			/* Check if VNI is masked. */
+			if (vxlan_spec && vxlan_mask) {
+				vni_masked =
+					!!memcmp(vxlan_mask->vni, vni_mask,
+						 RTE_DIM(vni_mask));
+				if (vni_masked) {
+					rte_flow_error_set
+						(error,
+						 EINVAL,
+						 RTE_FLOW_ERROR_TYPE_ITEM,
+						 item,
+						 "Invalid VNI mask");
+					return -rte_errno;
+				}
+
+				rte_memcpy(((uint8_t *)&tenant_id_be + 1),
+					   vxlan_spec->vni, 3);
+				filter->vni =
+					rte_be_to_cpu_32(tenant_id_be);
+				filter->tunnel_type =
+				 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_NVGRE:
+			nvgre_spec = item->spec;
+			nvgre_mask = item->mask;
+			/* Check if NVGRE item is used to describe protocol.
+			 * If yes, both spec and mask should be NULL.
+			 * If no, both spec and mask shouldn't be NULL.
+			 */
+			if ((!nvgre_spec && nvgre_mask) ||
+			    (nvgre_spec && !nvgre_mask)) {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Invalid NVGRE item");
+				return -rte_errno;
+			}
+
+			if (nvgre_spec->c_k_s_rsvd0_ver != 0x2000 ||
+			    nvgre_spec->protocol != 0x6558) {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Invalid NVGRE item");
+				return -rte_errno;
+			}
+
+			if (nvgre_spec && nvgre_mask) {
+				tni_masked =
+					!!memcmp(nvgre_mask->tni, tni_mask,
+						 RTE_DIM(tni_mask));
+				if (tni_masked) {
+					rte_flow_error_set
+						(error,
+						 EINVAL,
+						 RTE_FLOW_ERROR_TYPE_ITEM,
+						 item,
+						 "Invalid TNI mask");
+					return -rte_errno;
+				}
+				rte_memcpy(((uint8_t *)&tenant_id_be + 1),
+					   nvgre_spec->tni, 3);
+				filter->vni =
+					rte_be_to_cpu_32(tenant_id_be);
+				filter->tunnel_type =
+				 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE;
+			}
+			break;
+		case RTE_FLOW_ITEM_TYPE_VF:
+			vf_spec = item->spec;
+			vf = vf_spec->id;
+
+			if (!BNXT_PF(bp)) {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Configuring on a VF!");
+				return -rte_errno;
+			}
+
+			if (vf >= bp->pdev->max_vfs) {
+				rte_flow_error_set(error,
+						   EINVAL,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Incorrect VF id!");
+				return -rte_errno;
+			}
+
+			if (!attr->transfer) {
+				rte_flow_error_set(error,
+						   ENOTSUP,
+						   RTE_FLOW_ERROR_TYPE_ITEM,
+						   item,
+						   "Matching VF traffic without"
+						   " affecting it (transfer attribute)"
+						   " is unsupported");
+				return -rte_errno;
+			}
+
+			filter->mirror_vnic_id =
+			dflt_vnic = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf);
+			if (dflt_vnic < 0) {
+				/* This simply indicates there's no driver
+				 * loaded. This is not an error.
+				 */
+				rte_flow_error_set
+					(error,
+					 EINVAL,
+					 RTE_FLOW_ERROR_TYPE_ITEM,
+					 item,
+					 "Unable to get default VNIC for VF");
+				return -rte_errno;
+			}
+
+			filter->mirror_vnic_id = dflt_vnic;
+			en |= NTUPLE_FLTR_ALLOC_INPUT_EN_MIRROR_VNIC_ID;
+			break;
+		default:
+			break;
+		}
+		item++;
+	}
+	filter->enables = en;
+
+	return 0;
+}
+
+/* Parse attributes */
+static int
+bnxt_flow_parse_attr(const struct rte_flow_attr *attr,
+		     struct rte_flow_error *error)
+{
+	/* Must be input direction */
+	if (!attr->ingress) {
+		rte_flow_error_set(error,
+				   EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+				   attr,
+				   "Only support ingress.");
+		return -rte_errno;
+	}
+
+	/* Not supported */
+	if (attr->egress) {
+		rte_flow_error_set(error,
+				   EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+				   attr,
+				   "No support for egress.");
+		return -rte_errno;
+	}
+
+	/* Not supported */
+	if (attr->priority) {
+		rte_flow_error_set(error,
+				   EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+				   attr,
+				   "No support for priority.");
+		return -rte_errno;
+	}
+
+	/* Not supported */
+	if (attr->group) {
+		rte_flow_error_set(error,
+				   EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+				   attr,
+				   "No support for group.");
+		return -rte_errno;
+	}
+
+	return 0;
+}
+
+struct bnxt_filter_info *
+bnxt_get_l2_filter(struct bnxt *bp, struct bnxt_filter_info *nf,
+		   struct bnxt_vnic_info *vnic)
+{
+	struct bnxt_filter_info *filter1, *f0;
+	struct bnxt_vnic_info *vnic0;
+	int rc;
+
+	vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
+	f0 = STAILQ_FIRST(&vnic0->filter);
+
+	/* This flow has same DST MAC as the port/l2 filter. */
+	if (memcmp(f0->l2_addr, nf->dst_macaddr, ETHER_ADDR_LEN) == 0)
+		return f0;
+
+	/* This flow needs DST MAC which is not same as port/l2 */
+	PMD_DRV_LOG(DEBUG, "Create L2 filter for DST MAC\n");
+	filter1 = bnxt_get_unused_filter(bp);
+	if (filter1 == NULL)
+		return NULL;
+
+	filter1->flags = HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX;
+	filter1->enables = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR |
+			L2_FILTER_ALLOC_INPUT_EN_L2_ADDR_MASK;
+	memcpy(filter1->l2_addr, nf->dst_macaddr, ETHER_ADDR_LEN);
+	memset(filter1->l2_addr_mask, 0xff, ETHER_ADDR_LEN);
+	rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id,
+				     filter1);
+	if (rc) {
+		bnxt_free_filter(bp, filter1);
+		return NULL;
+	}
+	return filter1;
+}
+
+static int
+bnxt_validate_and_parse_flow(struct rte_eth_dev *dev,
+			     const struct rte_flow_item pattern[],
+			     const struct rte_flow_action actions[],
+			     const struct rte_flow_attr *attr,
+			     struct rte_flow_error *error,
+			     struct bnxt_filter_info *filter)
+{
+	const struct rte_flow_action *act =
+		bnxt_flow_non_void_action(actions);
+	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
+	const struct rte_flow_action_queue *act_q;
+	const struct rte_flow_action_vf *act_vf;
+	struct bnxt_vnic_info *vnic, *vnic0;
+	struct bnxt_filter_info *filter1;
+	uint32_t vf = 0;
+	int dflt_vnic;
+	int rc;
+
+	if (bp->eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) {
+		PMD_DRV_LOG(ERR, "Cannot create flow on RSS queues\n");
+		rte_flow_error_set(error,
+				   EINVAL,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "Cannot create flow on RSS queues");
+		rc = -rte_errno;
+		goto ret;
+	}
+
+	rc =
+	bnxt_validate_and_parse_flow_type(bp, attr, pattern, error, filter);
+	if (rc != 0)
+		goto ret;
+
+	rc = bnxt_flow_parse_attr(attr, error);
+	if (rc != 0)
+		goto ret;
+
+	/* Since we support ingress attribute only - right now. */
+	if (filter->filter_type == HWRM_CFA_EM_FILTER)
+		filter->flags = HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_RX;
+
+	switch (act->type) {
+	case RTE_FLOW_ACTION_TYPE_QUEUE:
+		/* Allow this flow. Redirect to a VNIC. */
+		act_q = (const struct rte_flow_action_queue *)act->conf;
+		if (act_q->index >= bp->rx_nr_rings) {
+			rte_flow_error_set(error,
+					   EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ACTION,
+					   act,
+					   "Invalid queue ID.");
+			rc = -rte_errno;
+			goto ret;
+		}
+		PMD_DRV_LOG(DEBUG, "Queue index %d\n", act_q->index);
+
+		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
+		vnic = STAILQ_FIRST(&bp->ff_pool[act_q->index]);
+		if (vnic == NULL) {
+			rte_flow_error_set(error,
+					   EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ACTION,
+					   act,
+					   "No matching VNIC for queue ID.");
+			rc = -rte_errno;
+			goto ret;
+		}
+
+		filter->dst_id = vnic->fw_vnic_id;
+		filter1 = bnxt_get_l2_filter(bp, filter, vnic);
+		if (filter1 == NULL) {
+			rc = -ENOSPC;
+			goto ret;
+		}
+
+		filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
+		PMD_DRV_LOG(DEBUG, "VNIC found\n");
+		break;
+	case RTE_FLOW_ACTION_TYPE_DROP:
+		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
+		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
+		if (filter1 == NULL) {
+			rc = -ENOSPC;
+			goto ret;
+		}
+
+		filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
+		if (filter->filter_type == HWRM_CFA_EM_FILTER)
+			filter->flags =
+				HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DROP;
+		else
+			filter->flags =
+				HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP;
+		break;
+	case RTE_FLOW_ACTION_TYPE_COUNT:
+		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
+		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
+		if (filter1 == NULL) {
+			rc = -ENOSPC;
+			goto ret;
+		}
+
+		filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
+		filter->flags = HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_METER;
+		break;
+	case RTE_FLOW_ACTION_TYPE_VF:
+		act_vf = (const struct rte_flow_action_vf *)act->conf;
+		vf = act_vf->id;
+
+		if (!BNXT_PF(bp)) {
+			rte_flow_error_set(error,
+					   EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ACTION,
+					   act,
+					   "Configuring on a VF!");
+			rc = -rte_errno;
+			goto ret;
+		}
+
+		if (vf >= bp->pdev->max_vfs) {
+			rte_flow_error_set(error,
+					   EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ACTION,
+					   act,
+					   "Incorrect VF id!");
+			rc = -rte_errno;
+			goto ret;
+		}
+
+		filter->mirror_vnic_id =
+		dflt_vnic = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf);
+		if (dflt_vnic < 0) {
+			/* This simply indicates there's no driver loaded.
+			 * This is not an error.
+			 */
+			rte_flow_error_set(error,
+					   EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ACTION,
+					   act,
+					   "Unable to get default VNIC for VF");
+			rc = -rte_errno;
+			goto ret;
+		}
+
+		filter->mirror_vnic_id = dflt_vnic;
+		filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_MIRROR_VNIC_ID;
+
+		vnic0 = STAILQ_FIRST(&bp->ff_pool[0]);
+		filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
+		if (filter1 == NULL) {
+			rc = -ENOSPC;
+			goto ret;
+		}
+
+		filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
+		break;
+
+	default:
+		rte_flow_error_set(error,
+				   EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ACTION,
+				   act,
+				   "Invalid action.");
+		rc = -rte_errno;
+		goto ret;
+	}
+
+	if (filter1) {
+		bnxt_free_filter(bp, filter1);
+		filter1->fw_l2_filter_id = -1;
+	}
+
+	act = bnxt_flow_non_void_action(++act);
+	if (act->type != RTE_FLOW_ACTION_TYPE_END) {
+		rte_flow_error_set(error,
+				   EINVAL,
+				   RTE_FLOW_ERROR_TYPE_ACTION,
+				   act,
+				   "Invalid action.");
+		rc = -rte_errno;
+		goto ret;
+	}
+ret:
+	return rc;
+}
+
+static int
+bnxt_flow_validate(struct rte_eth_dev *dev,
+		   const struct rte_flow_attr *attr,
+		   const struct rte_flow_item pattern[],
+		   const struct rte_flow_action actions[],
+		   struct rte_flow_error *error)
+{
+	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
+	struct bnxt_filter_info *filter;
+	int ret = 0;
+
+	ret = bnxt_flow_args_validate(attr, pattern, actions, error);
+	if (ret != 0)
+		return ret;
+
+	filter = bnxt_get_unused_filter(bp);
+	if (filter == NULL) {
+		PMD_DRV_LOG(ERR, "Not enough resources for a new flow.\n");
+		return -ENOMEM;
+	}
+
+	ret = bnxt_validate_and_parse_flow(dev, pattern, actions, attr,
+					   error, filter);
+	/* No need to hold on to this filter if we are just validating flow */
+	filter->fw_l2_filter_id = UINT64_MAX;
+	bnxt_free_filter(bp, filter);
+
+	return ret;
+}
+
+static int
+bnxt_match_filter(struct bnxt *bp, struct bnxt_filter_info *nf)
+{
+	struct bnxt_filter_info *mf;
+	struct rte_flow *flow;
+	int i;
+
+	for (i = bp->nr_vnics - 1; i >= 0; i--) {
+		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
+
+		STAILQ_FOREACH(flow, &vnic->flow_list, next) {
+			mf = flow->filter;
+
+			if (mf->filter_type == nf->filter_type &&
+			    mf->flags == nf->flags &&
+			    mf->src_port == nf->src_port &&
+			    mf->src_port_mask == nf->src_port_mask &&
+			    mf->dst_port == nf->dst_port &&
+			    mf->dst_port_mask == nf->dst_port_mask &&
+			    mf->ip_protocol == nf->ip_protocol &&
+			    mf->ip_addr_type == nf->ip_addr_type &&
+			    mf->ethertype == nf->ethertype &&
+			    mf->vni == nf->vni &&
+			    mf->tunnel_type == nf->tunnel_type &&
+			    mf->l2_ovlan == nf->l2_ovlan &&
+			    mf->l2_ovlan_mask == nf->l2_ovlan_mask &&
+			    mf->l2_ivlan == nf->l2_ivlan &&
+			    mf->l2_ivlan_mask == nf->l2_ivlan_mask &&
+			    !memcmp(mf->l2_addr, nf->l2_addr, ETHER_ADDR_LEN) &&
+			    !memcmp(mf->l2_addr_mask, nf->l2_addr_mask,
+				    ETHER_ADDR_LEN) &&
+			    !memcmp(mf->src_macaddr, nf->src_macaddr,
+				    ETHER_ADDR_LEN) &&
+			    !memcmp(mf->dst_macaddr, nf->dst_macaddr,
+				    ETHER_ADDR_LEN) &&
+			    !memcmp(mf->src_ipaddr, nf->src_ipaddr,
+				    sizeof(nf->src_ipaddr)) &&
+			    !memcmp(mf->src_ipaddr_mask, nf->src_ipaddr_mask,
+				    sizeof(nf->src_ipaddr_mask)) &&
+			    !memcmp(mf->dst_ipaddr, nf->dst_ipaddr,
+				    sizeof(nf->dst_ipaddr)) &&
+			    !memcmp(mf->dst_ipaddr_mask, nf->dst_ipaddr_mask,
+				    sizeof(nf->dst_ipaddr_mask))) {
+				if (mf->dst_id == nf->dst_id)
+					return -EEXIST;
+				/*
+				 * Same Flow, Different queue
+				 * Clear the old ntuple filter
+				 * Reuse the matching L2 filter
+				 * ID for the new filter
+				 */
+				nf->fw_l2_filter_id = mf->fw_l2_filter_id;
+				if (nf->filter_type == HWRM_CFA_EM_FILTER)
+					bnxt_hwrm_clear_em_filter(bp, mf);
+				if (nf->filter_type == HWRM_CFA_NTUPLE_FILTER)
+					bnxt_hwrm_clear_ntuple_filter(bp, mf);
+				/* Free the old filter, update flow
+				 * with new filter
+				 */
+				bnxt_free_filter(bp, mf);
+				flow->filter = nf;
+				return -EXDEV;
+			}
+		}
+	}
+	return 0;
+}
+
+static struct rte_flow *
+bnxt_flow_create(struct rte_eth_dev *dev,
+		 const struct rte_flow_attr *attr,
+		 const struct rte_flow_item pattern[],
+		 const struct rte_flow_action actions[],
+		 struct rte_flow_error *error)
+{
+	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
+	struct bnxt_filter_info *filter;
+	struct bnxt_vnic_info *vnic = NULL;
+	bool update_flow = false;
+	struct rte_flow *flow;
+	unsigned int i;
+	int ret = 0;
+
+	flow = rte_zmalloc("bnxt_flow", sizeof(struct rte_flow), 0);
+	if (!flow) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+				   "Failed to allocate memory");
+		return flow;
+	}
+
+	ret = bnxt_flow_args_validate(attr, pattern, actions, error);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Not a validate flow.\n");
+		goto free_flow;
+	}
+
+	filter = bnxt_get_unused_filter(bp);
+	if (filter == NULL) {
+		PMD_DRV_LOG(ERR, "Not enough resources for a new flow.\n");
+		goto free_flow;
+	}
+
+	ret = bnxt_validate_and_parse_flow(dev, pattern, actions, attr,
+					   error, filter);
+	if (ret != 0)
+		goto free_filter;
+
+	ret = bnxt_match_filter(bp, filter);
+	if (ret == -EEXIST) {
+		PMD_DRV_LOG(DEBUG, "Flow already exists.\n");
+		/* Clear the filter that was created as part of
+		 * validate_and_parse_flow() above
+		 */
+		bnxt_hwrm_clear_l2_filter(bp, filter);
+		goto free_filter;
+	} else if (ret == -EXDEV) {
+		PMD_DRV_LOG(DEBUG, "Flow with same pattern exists\n");
+		PMD_DRV_LOG(DEBUG, "Updating with different destination\n");
+		update_flow = true;
+	}
+
+	if (filter->filter_type == HWRM_CFA_EM_FILTER) {
+		filter->enables |=
+			HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
+		ret = bnxt_hwrm_set_em_filter(bp, filter->dst_id, filter);
+	}
+
+	if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER) {
+		filter->enables |=
+			HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
+		ret = bnxt_hwrm_set_ntuple_filter(bp, filter->dst_id, filter);
+	}
+
+	for (i = 0; i < bp->nr_vnics; i++) {
+		vnic = &bp->vnic_info[i];
+		if (filter->dst_id == vnic->fw_vnic_id)
+			break;
+	}
+
+	if (!ret) {
+		flow->filter = filter;
+		flow->vnic = vnic;
+		if (update_flow) {
+			ret = -EXDEV;
+			goto free_flow;
+		}
+		PMD_DRV_LOG(ERR, "Successfully created flow.\n");
+		STAILQ_INSERT_TAIL(&vnic->flow_list, flow, next);
+		return flow;
+	}
+free_filter:
+	bnxt_free_filter(bp, filter);
+free_flow:
+	if (ret == -EEXIST)
+		rte_flow_error_set(error, ret,
+				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+				   "Matching Flow exists.");
+	else if (ret == -EXDEV)
+		rte_flow_error_set(error, ret,
+				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+				   "Flow with pattern exists, updating destination queue");
+	else
+		rte_flow_error_set(error, -ret,
+				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+				   "Failed to create flow.");
+	rte_free(flow);
+	flow = NULL;
+	return flow;
+}
+
+static int
+bnxt_flow_destroy(struct rte_eth_dev *dev,
+		  struct rte_flow *flow,
+		  struct rte_flow_error *error)
+{
+	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
+	struct bnxt_filter_info *filter = flow->filter;
+	struct bnxt_vnic_info *vnic = flow->vnic;
+	int ret = 0;
+
+	ret = bnxt_match_filter(bp, filter);
+	if (ret == 0)
+		PMD_DRV_LOG(ERR, "Could not find matching flow\n");
+	if (filter->filter_type == HWRM_CFA_EM_FILTER)
+		ret = bnxt_hwrm_clear_em_filter(bp, filter);
+	if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
+		ret = bnxt_hwrm_clear_ntuple_filter(bp, filter);
+	else
+		ret = bnxt_hwrm_clear_l2_filter(bp, filter);
+	if (!ret) {
+		STAILQ_REMOVE(&vnic->flow_list, flow, rte_flow, next);
+		rte_free(flow);
+	} else {
+		rte_flow_error_set(error, -ret,
+				   RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+				   "Failed to destroy flow.");
+	}
+
+	return ret;
+}
+
+static int
+bnxt_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
+{
+	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
+	struct bnxt_vnic_info *vnic;
+	struct rte_flow *flow;
+	unsigned int i;
+	int ret = 0;
+
+	for (i = 0; i < bp->nr_vnics; i++) {
+		vnic = &bp->vnic_info[i];
+		STAILQ_FOREACH(flow, &vnic->flow_list, next) {
+			struct bnxt_filter_info *filter = flow->filter;
+
+			if (filter->filter_type == HWRM_CFA_EM_FILTER)
+				ret = bnxt_hwrm_clear_em_filter(bp, filter);
+			if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER)
+				ret = bnxt_hwrm_clear_ntuple_filter(bp, filter);
+
+			if (ret) {
+				rte_flow_error_set
+					(error,
+					 -ret,
+					 RTE_FLOW_ERROR_TYPE_HANDLE,
+					 NULL,
+					 "Failed to flush flow in HW.");
+				return -rte_errno;
+			}
+
+			STAILQ_REMOVE(&vnic->flow_list, flow,
+				      rte_flow, next);
+			rte_free(flow);
+		}
+	}
+
+	return ret;
+}
+
+const struct rte_flow_ops bnxt_flow_ops = {
+	.validate = bnxt_flow_validate,
+	.create = bnxt_flow_create,
+	.destroy = bnxt_flow_destroy,
+	.flush = bnxt_flow_flush,
+};
diff --git a/drivers/net/bnxt/bnxt_hwrm.c b/drivers/net/bnxt/bnxt_hwrm.c
index d6fdc1b8..c682488a 100644
--- a/drivers/net/bnxt/bnxt_hwrm.c
+++ b/drivers/net/bnxt/bnxt_hwrm.c
@@ -166,10 +166,26 @@ err_ret:
 	req.resp_addr = rte_cpu_to_le_64(bp->hwrm_cmd_resp_dma_addr); \
 } while (0)
 
+#define HWRM_CHECK_RESULT_SILENT() do {\
+	if (rc) { \
+		rte_spinlock_unlock(&bp->hwrm_lock); \
+		return rc; \
+	} \
+	if (resp->error_code) { \
+		rc = rte_le_to_cpu_16(resp->error_code); \
+		rte_spinlock_unlock(&bp->hwrm_lock); \
+		return rc; \
+	} \
+} while (0)
+
 #define HWRM_CHECK_RESULT() do {\
 	if (rc) { \
 		PMD_DRV_LOG(ERR, "failed rc:%d\n", rc); \
 		rte_spinlock_unlock(&bp->hwrm_lock); \
+		if (rc == HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED) \
+			rc = -EACCES; \
+		else if (rc > 0) \
+			rc = -EINVAL; \
 		return rc; \
 	} \
 	if (resp->error_code) { \
@@ -188,6 +204,10 @@ err_ret:
 			PMD_DRV_LOG(ERR, "error %d\n", rc); \
 		} \
 		rte_spinlock_unlock(&bp->hwrm_lock); \
+		if (rc == HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED) \
+			rc = -EACCES; \
+		else if (rc > 0) \
+			rc = -EINVAL; \
 		return rc; \
 	} \
 } while (0)
@@ -376,13 +396,13 @@ int bnxt_hwrm_set_l2_filter(struct bnxt *bp,
 		req.l2_ovlan = filter->l2_ovlan;
 	if (enables &
 	    HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN)
-		req.l2_ovlan = filter->l2_ivlan;
+		req.l2_ivlan = filter->l2_ivlan;
 	if (enables &
 	    HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK)
 		req.l2_ovlan_mask = filter->l2_ovlan_mask;
 	if (enables &
 	    HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK)
-		req.l2_ovlan_mask = filter->l2_ivlan_mask;
+		req.l2_ivlan_mask = filter->l2_ivlan_mask;
 	if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_ID)
 		req.src_id = rte_cpu_to_le_32(filter->src_id);
 	if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_TYPE)
@@ -506,6 +526,7 @@ static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
 	if (BNXT_PF(bp)) {
 		bp->pf.port_id = resp->port_id;
 		bp->pf.first_vf_id = rte_le_to_cpu_16(resp->first_vf_id);
+		bp->pf.total_vfs = rte_le_to_cpu_16(resp->max_vfs);
 		new_max_vfs = bp->pdev->max_vfs;
 		if (new_max_vfs != bp->pf.max_vfs) {
 			if (bp->pf.vf_info)
@@ -657,9 +678,19 @@ int bnxt_hwrm_func_driver_register(struct bnxt *bp)
 	return rc;
 }
 
-int bnxt_hwrm_func_reserve_vf_resc(struct bnxt *bp)
+int bnxt_hwrm_check_vf_rings(struct bnxt *bp)
+{
+	if (!(BNXT_VF(bp) && (bp->flags & BNXT_FLAG_NEW_RM)))
+		return 0;
+
+	return bnxt_hwrm_func_reserve_vf_resc(bp, true);
+}
+
+int bnxt_hwrm_func_reserve_vf_resc(struct bnxt *bp, bool test)
 {
 	int rc;
+	uint32_t flags = 0;
+	uint32_t enables;
 	struct hwrm_func_vf_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 	struct hwrm_func_vf_cfg_input req = {0};
 
@@ -670,7 +701,8 @@ int bnxt_hwrm_func_reserve_vf_resc(struct bnxt *bp)
 			HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_TX_RINGS   |
 			HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_STAT_CTXS  |
 			HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_CMPL_RINGS |
-			HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_HW_RING_GRPS);
+			HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_HW_RING_GRPS |
+			HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_VNICS);
 
 	req.num_tx_rings = rte_cpu_to_le_16(bp->tx_nr_rings);
 	req.num_rx_rings = rte_cpu_to_le_16(bp->rx_nr_rings *
@@ -679,10 +711,35 @@ int bnxt_hwrm_func_reserve_vf_resc(struct bnxt *bp)
 	req.num_cmpl_rings = rte_cpu_to_le_16(bp->rx_nr_rings +
 					      bp->tx_nr_rings);
 	req.num_hw_ring_grps = rte_cpu_to_le_16(bp->rx_nr_rings);
+	req.num_vnics = rte_cpu_to_le_16(bp->rx_nr_rings);
+	if (bp->vf_resv_strategy ==
+	    HWRM_FUNC_RESOURCE_QCAPS_OUTPUT_VF_RESV_STRATEGY_MINIMAL_STATIC) {
+		enables = HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_VNICS |
+				HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_L2_CTXS |
+				HWRM_FUNC_VF_CFG_INPUT_ENABLES_NUM_RSSCOS_CTXS;
+		req.enables |= rte_cpu_to_le_32(enables);
+		req.num_rsscos_ctxs = rte_cpu_to_le_16(BNXT_VF_RSV_NUM_RSS_CTX);
+		req.num_l2_ctxs = rte_cpu_to_le_16(BNXT_VF_RSV_NUM_L2_CTX);
+		req.num_vnics = rte_cpu_to_le_16(BNXT_VF_RSV_NUM_VNIC);
+	}
+
+	if (test)
+		flags = HWRM_FUNC_VF_CFG_INPUT_FLAGS_TX_ASSETS_TEST |
+			HWRM_FUNC_VF_CFG_INPUT_FLAGS_RX_ASSETS_TEST |
+			HWRM_FUNC_VF_CFG_INPUT_FLAGS_CMPL_ASSETS_TEST |
+			HWRM_FUNC_VF_CFG_INPUT_FLAGS_RING_GRP_ASSETS_TEST |
+			HWRM_FUNC_VF_CFG_INPUT_FLAGS_STAT_CTX_ASSETS_TEST |
+			HWRM_FUNC_VF_CFG_INPUT_FLAGS_VNIC_ASSETS_TEST;
+
+	req.flags = rte_cpu_to_le_32(flags);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT();
+	if (test)
+		HWRM_CHECK_RESULT_SILENT();
+	else
+		HWRM_CHECK_RESULT();
+
 	HWRM_UNLOCK();
 	return rc;
 }
@@ -710,6 +767,11 @@ int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp)
 		bp->max_vnics = rte_le_to_cpu_16(resp->max_vnics);
 		bp->max_stat_ctx = rte_le_to_cpu_16(resp->max_stat_ctx);
 	}
+	bp->vf_resv_strategy = rte_le_to_cpu_16(resp->vf_reservation_strategy);
+	if (bp->vf_resv_strategy >
+	    HWRM_FUNC_RESOURCE_QCAPS_OUTPUT_VF_RESV_STRATEGY_MINIMAL_STATIC)
+		bp->vf_resv_strategy =
+		HWRM_FUNC_RESOURCE_QCAPS_OUTPUT_VF_RESERVATION_STRATEGY_MAXIMAL;
 
 	HWRM_UNLOCK();
 	return rc;
@@ -1265,8 +1327,9 @@ int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	/* map ring groups to this vnic */
 	PMD_DRV_LOG(DEBUG, "Alloc VNIC. Start %x, End %x\n",
 		vnic->start_grp_id, vnic->end_grp_id);
-	for (i = vnic->start_grp_id, j = 0; i <= vnic->end_grp_id; i++, j++)
+	for (i = vnic->start_grp_id, j = 0; i < vnic->end_grp_id; i++, j++)
 		vnic->fw_grp_ids[j] = bp->grp_info[i].fw_grp_id;
+
 	vnic->dflt_ring_grp = bp->grp_info[vnic->start_grp_id].fw_grp_id;
 	vnic->rss_rule = (uint16_t)HWRM_NA_SIGNATURE;
 	vnic->cos_rule = (uint16_t)HWRM_NA_SIGNATURE;
@@ -1559,6 +1622,11 @@ int bnxt_hwrm_vnic_plcmode_cfg(struct bnxt *bp,
 	struct hwrm_vnic_plcmodes_cfg_output *resp = bp->hwrm_cmd_resp_addr;
 	uint16_t size;
 
+	if (vnic->fw_vnic_id == INVALID_HW_RING_ID) {
+		PMD_DRV_LOG(DEBUG, "VNIC ID %x\n", vnic->fw_vnic_id);
+		return rc;
+	}
+
 	HWRM_PREP(req, VNIC_PLCMODES_CFG);
 
 	req.flags = rte_cpu_to_le_32(
@@ -1816,8 +1884,7 @@ int bnxt_free_all_hwrm_ring_grps(struct bnxt *bp)
 	return rc;
 }
 
-static void bnxt_free_cp_ring(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
-				unsigned int idx __rte_unused)
+static void bnxt_free_cp_ring(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
 {
 	struct bnxt_ring *cp_ring = cpr->cp_ring_struct;
 
@@ -1829,17 +1896,52 @@ static void bnxt_free_cp_ring(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 	cpr->cp_raw_cons = 0;
 }
 
+void bnxt_free_hwrm_rx_ring(struct bnxt *bp, int queue_index)
+{
+	struct bnxt_rx_queue *rxq = bp->rx_queues[queue_index];
+	struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
+	struct bnxt_ring *ring = rxr->rx_ring_struct;
+	struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
+
+	if (ring->fw_ring_id != INVALID_HW_RING_ID) {
+		bnxt_hwrm_ring_free(bp, ring,
+				    HWRM_RING_FREE_INPUT_RING_TYPE_RX);
+		ring->fw_ring_id = INVALID_HW_RING_ID;
+		bp->grp_info[queue_index].rx_fw_ring_id = INVALID_HW_RING_ID;
+		memset(rxr->rx_desc_ring, 0,
+		       rxr->rx_ring_struct->ring_size *
+		       sizeof(*rxr->rx_desc_ring));
+		memset(rxr->rx_buf_ring, 0,
+		       rxr->rx_ring_struct->ring_size *
+		       sizeof(*rxr->rx_buf_ring));
+		rxr->rx_prod = 0;
+	}
+	ring = rxr->ag_ring_struct;
+	if (ring->fw_ring_id != INVALID_HW_RING_ID) {
+		bnxt_hwrm_ring_free(bp, ring,
+				    HWRM_RING_FREE_INPUT_RING_TYPE_RX);
+		ring->fw_ring_id = INVALID_HW_RING_ID;
+		memset(rxr->ag_buf_ring, 0,
+		       rxr->ag_ring_struct->ring_size *
+		       sizeof(*rxr->ag_buf_ring));
+		rxr->ag_prod = 0;
+		bp->grp_info[queue_index].ag_fw_ring_id = INVALID_HW_RING_ID;
+	}
+	if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID)
+		bnxt_free_cp_ring(bp, cpr);
+
+	bp->grp_info[queue_index].cp_fw_ring_id = INVALID_HW_RING_ID;
+}
+
 int bnxt_free_all_hwrm_rings(struct bnxt *bp)
 {
 	unsigned int i;
-	int rc = 0;
 
 	for (i = 0; i < bp->tx_cp_nr_rings; i++) {
 		struct bnxt_tx_queue *txq = bp->tx_queues[i];
 		struct bnxt_tx_ring_info *txr = txq->tx_ring;
 		struct bnxt_ring *ring = txr->tx_ring_struct;
 		struct bnxt_cp_ring_info *cpr = txq->cp_ring;
-		unsigned int idx = bp->rx_cp_nr_rings + i;
 
 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
 			bnxt_hwrm_ring_free(bp, ring,
@@ -1855,59 +1957,15 @@ int bnxt_free_all_hwrm_rings(struct bnxt *bp)
 			txr->tx_cons = 0;
 		}
 		if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) {
-			bnxt_free_cp_ring(bp, cpr, idx);
-			cpr->cp_ring_struct->fw_ring_id = INVALID_HW_RING_ID;
-		}
-	}
-
-	for (i = 0; i < bp->rx_cp_nr_rings; i++) {
-		struct bnxt_rx_queue *rxq = bp->rx_queues[i];
-		struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
-		struct bnxt_ring *ring = rxr->rx_ring_struct;
-		struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
-
-		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
-			bnxt_hwrm_ring_free(bp, ring,
-					HWRM_RING_FREE_INPUT_RING_TYPE_RX);
-			ring->fw_ring_id = INVALID_HW_RING_ID;
-			bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
-			memset(rxr->rx_desc_ring, 0,
-					rxr->rx_ring_struct->ring_size *
-					sizeof(*rxr->rx_desc_ring));
-			memset(rxr->rx_buf_ring, 0,
-					rxr->rx_ring_struct->ring_size *
-					sizeof(*rxr->rx_buf_ring));
-			rxr->rx_prod = 0;
-		}
-		ring = rxr->ag_ring_struct;
-		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
-			bnxt_hwrm_ring_free(bp, ring,
-					    HWRM_RING_FREE_INPUT_RING_TYPE_RX);
-			ring->fw_ring_id = INVALID_HW_RING_ID;
-			memset(rxr->ag_buf_ring, 0,
-			       rxr->ag_ring_struct->ring_size *
-			       sizeof(*rxr->ag_buf_ring));
-			rxr->ag_prod = 0;
-			bp->grp_info[i].ag_fw_ring_id = INVALID_HW_RING_ID;
-		}
-		if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) {
-			bnxt_free_cp_ring(bp, cpr, i);
-			bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
+			bnxt_free_cp_ring(bp, cpr);
 			cpr->cp_ring_struct->fw_ring_id = INVALID_HW_RING_ID;
 		}
 	}
 
-	/* Default completion ring */
-	{
-		struct bnxt_cp_ring_info *cpr = bp->def_cp_ring;
-
-		if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) {
-			bnxt_free_cp_ring(bp, cpr, 0);
-			cpr->cp_ring_struct->fw_ring_id = INVALID_HW_RING_ID;
-		}
-	}
+	for (i = 0; i < bp->rx_cp_nr_rings; i++)
+		bnxt_free_hwrm_rx_ring(bp, i);
 
-	return rc;
+	return 0;
 }
 
 int bnxt_alloc_all_hwrm_ring_grps(struct bnxt *bp)
@@ -1970,6 +2028,7 @@ int bnxt_clear_hwrm_vnic_filters(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 			rc = bnxt_hwrm_clear_ntuple_filter(bp, filter);
 		else
 			rc = bnxt_hwrm_clear_l2_filter(bp, filter);
+		STAILQ_REMOVE(&vnic->filter, filter, bnxt_filter_info, next);
 		//if (rc)
 			//break;
 	}
@@ -2057,6 +2116,8 @@ void bnxt_free_all_hwrm_resources(struct bnxt *bp)
 		bnxt_hwrm_vnic_tpa_cfg(bp, vnic, false);
 
 		bnxt_hwrm_vnic_free(bp, vnic);
+
+		rte_free(vnic->fw_grp_ids);
 	}
 	/* Ring resources */
 	bnxt_free_all_hwrm_rings(bp);
@@ -3151,7 +3212,9 @@ int bnxt_hwrm_port_clr_stats(struct bnxt *bp)
 	struct bnxt_pf_info *pf = &bp->pf;
 	int rc;
 
-	if (!(bp->flags & BNXT_FLAG_PORT_STATS))
+	/* Not allowed on NS2 device, NPAR, MultiHost, VF */
+	if (!(bp->flags & BNXT_FLAG_PORT_STATS) || BNXT_VF(bp) ||
+	    BNXT_NPAR(bp) || BNXT_MH(bp) || BNXT_TOTAL_VFS(bp))
 		return 0;
 
 	HWRM_PREP(req, PORT_CLR_STATS);
@@ -3298,13 +3361,12 @@ int bnxt_get_nvram_directory(struct bnxt *bp, uint32_t len, uint8_t *data)
 	req.host_dest_addr = rte_cpu_to_le_64(dma_handle);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
-	HWRM_CHECK_RESULT();
-	HWRM_UNLOCK();
-
 	if (rc == 0)
 		memcpy(data, buf, len > buflen ? buflen : len);
 
 	rte_free(buf);
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
 
 	return rc;
 }
@@ -3336,12 +3398,13 @@ int bnxt_hwrm_get_nvram_item(struct bnxt *bp, uint32_t index,
 	req.offset = rte_cpu_to_le_32(offset);
 	req.len = rte_cpu_to_le_32(length);
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
-	HWRM_CHECK_RESULT();
-	HWRM_UNLOCK();
 	if (rc == 0)
 		memcpy(data, buf, length);
 
 	rte_free(buf);
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
+
 	return rc;
 }
 
@@ -3372,14 +3435,6 @@ int bnxt_hwrm_flash_nvram(struct bnxt *bp, uint16_t dir_type,
 	rte_iova_t dma_handle;
 	uint8_t *buf;
 
-	HWRM_PREP(req, NVM_WRITE);
-
-	req.dir_type = rte_cpu_to_le_16(dir_type);
-	req.dir_ordinal = rte_cpu_to_le_16(dir_ordinal);
-	req.dir_ext = rte_cpu_to_le_16(dir_ext);
-	req.dir_attr = rte_cpu_to_le_16(dir_attr);
-	req.dir_data_length = rte_cpu_to_le_32(data_len);
-
 	buf = rte_malloc("nvm_write", data_len, 0);
 	rte_mem_lock_page(buf);
 	if (!buf)
@@ -3392,14 +3447,22 @@ int bnxt_hwrm_flash_nvram(struct bnxt *bp, uint16_t dir_type,
 		return -ENOMEM;
 	}
 	memcpy(buf, data, data_len);
+
+	HWRM_PREP(req, NVM_WRITE);
+
+	req.dir_type = rte_cpu_to_le_16(dir_type);
+	req.dir_ordinal = rte_cpu_to_le_16(dir_ordinal);
+	req.dir_ext = rte_cpu_to_le_16(dir_ext);
+	req.dir_attr = rte_cpu_to_le_16(dir_attr);
+	req.dir_data_length = rte_cpu_to_le_32(data_len);
 	req.host_src_addr = rte_cpu_to_le_64(dma_handle);
 
 	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
 
+	rte_free(buf);
 	HWRM_CHECK_RESULT();
 	HWRM_UNLOCK();
 
-	rte_free(buf);
 	return rc;
 }
 
@@ -3800,7 +3863,6 @@ int bnxt_hwrm_clear_ntuple_filter(struct bnxt *bp,
 	HWRM_UNLOCK();
 
 	filter->fw_ntuple_filter_id = UINT64_MAX;
-	filter->fw_l2_filter_id = UINT64_MAX;
 
 	return 0;
 }
@@ -3832,3 +3894,54 @@ int bnxt_vnic_rss_configure(struct bnxt *bp, struct bnxt_vnic_info *vnic)
 	}
 	return 0;
 }
+
+static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal,
+	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
+{
+	uint16_t flags;
+
+	req->num_cmpl_aggr_int = rte_cpu_to_le_16(hw_coal->num_cmpl_aggr_int);
+
+	/* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */
+	req->num_cmpl_dma_aggr = rte_cpu_to_le_16(hw_coal->num_cmpl_dma_aggr);
+
+	/* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */
+	req->num_cmpl_dma_aggr_during_int =
+		rte_cpu_to_le_16(hw_coal->num_cmpl_dma_aggr_during_int);
+
+	req->int_lat_tmr_max = rte_cpu_to_le_16(hw_coal->int_lat_tmr_max);
+
+	/* min timer set to 1/2 of interrupt timer */
+	req->int_lat_tmr_min = rte_cpu_to_le_16(hw_coal->int_lat_tmr_min);
+
+	/* buf timer set to 1/4 of interrupt timer */
+	req->cmpl_aggr_dma_tmr = rte_cpu_to_le_16(hw_coal->cmpl_aggr_dma_tmr);
+
+	req->cmpl_aggr_dma_tmr_during_int =
+		rte_cpu_to_le_16(hw_coal->cmpl_aggr_dma_tmr_during_int);
+
+	flags = HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS_INPUT_FLAGS_TIMER_RESET |
+		HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS_INPUT_FLAGS_RING_IDLE;
+	req->flags = rte_cpu_to_le_16(flags);
+}
+
+int bnxt_hwrm_set_ring_coal(struct bnxt *bp,
+			struct bnxt_coal *coal, uint16_t ring_id)
+{
+	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
+	struct hwrm_ring_cmpl_ring_cfg_aggint_params_output *resp =
+						bp->hwrm_cmd_resp_addr;
+	int rc;
+
+	/* Set ring coalesce parameters only for Stratus 100G NIC */
+	if (!bnxt_stratus_device(bp))
+		return 0;
+
+	HWRM_PREP(req, RING_CMPL_RING_CFG_AGGINT_PARAMS);
+	bnxt_hwrm_set_coal_params(coal, &req);
+	req.ring_id = rte_cpu_to_le_16(ring_id);
+	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req));
+	HWRM_CHECK_RESULT();
+	HWRM_UNLOCK();
+	return 0;
+}
diff --git a/drivers/net/bnxt/bnxt_hwrm.h b/drivers/net/bnxt/bnxt_hwrm.h
index 60a4ab16..379aac6e 100644
--- a/drivers/net/bnxt/bnxt_hwrm.h
+++ b/drivers/net/bnxt/bnxt_hwrm.h
@@ -29,6 +29,9 @@ struct bnxt_cp_ring_info;
 #define HWRM_QUEUE_SERVICE_PROFILE_LOSSY \
 	HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSY
 
+#define HWRM_FUNC_RESOURCE_QCAPS_OUTPUT_VF_RESV_STRATEGY_MINIMAL_STATIC \
+	HWRM_FUNC_RESOURCE_QCAPS_OUTPUT_VF_RESERVATION_STRATEGY_MINIMAL_STATIC
+
 int bnxt_hwrm_cfa_l2_clear_rx_mask(struct bnxt *bp,
 				   struct bnxt_vnic_info *vnic);
 int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic,
@@ -107,12 +110,13 @@ int bnxt_set_hwrm_vnic_filters(struct bnxt *bp, struct bnxt_vnic_info *vnic);
 int bnxt_clear_hwrm_vnic_filters(struct bnxt *bp, struct bnxt_vnic_info *vnic);
 void bnxt_free_all_hwrm_resources(struct bnxt *bp);
 void bnxt_free_hwrm_resources(struct bnxt *bp);
+void bnxt_free_hwrm_rx_ring(struct bnxt *bp, int queue_index);
 int bnxt_alloc_hwrm_resources(struct bnxt *bp);
 int bnxt_get_hwrm_link_config(struct bnxt *bp, struct rte_eth_link *link);
 int bnxt_set_hwrm_link_config(struct bnxt *bp, bool link_up);
 int bnxt_hwrm_func_qcfg(struct bnxt *bp);
 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp);
-int bnxt_hwrm_func_reserve_vf_resc(struct bnxt *bp);
+int bnxt_hwrm_func_reserve_vf_resc(struct bnxt *bp, bool test);
 int bnxt_hwrm_allocate_pf_only(struct bnxt *bp);
 int bnxt_hwrm_allocate_vfs(struct bnxt *bp, int num_vfs);
 int bnxt_hwrm_func_vf_mac(struct bnxt *bp, uint16_t vf,
@@ -167,4 +171,7 @@ int bnxt_hwrm_flash_nvram(struct bnxt *bp, uint16_t dir_type,
 int bnxt_hwrm_ptp_cfg(struct bnxt *bp);
 int bnxt_vnic_rss_configure(struct bnxt *bp,
 			    struct bnxt_vnic_info *vnic);
+int bnxt_hwrm_set_ring_coal(struct bnxt *bp,
+			struct bnxt_coal *coal, uint16_t ring_id);
+int bnxt_hwrm_check_vf_rings(struct bnxt *bp);
 #endif
diff --git a/drivers/net/bnxt/bnxt_ring.c b/drivers/net/bnxt/bnxt_ring.c
index bb9f6d1c..fcbd6bc6 100644
--- a/drivers/net/bnxt/bnxt_ring.c
+++ b/drivers/net/bnxt/bnxt_ring.c
@@ -258,6 +258,116 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx,
 	return 0;
 }
 
+static void bnxt_init_dflt_coal(struct bnxt_coal *coal)
+{
+	/* Tick values in micro seconds.
+	 * 1 coal_buf x bufs_per_record = 1 completion record.
+	 */
+	coal->num_cmpl_aggr_int = BNXT_NUM_CMPL_AGGR_INT;
+	/* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */
+	coal->num_cmpl_dma_aggr = BNXT_NUM_CMPL_DMA_AGGR;
+	/* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */
+	coal->num_cmpl_dma_aggr_during_int = BNXT_NUM_CMPL_DMA_AGGR_DURING_INT;
+	coal->int_lat_tmr_max = BNXT_INT_LAT_TMR_MAX;
+	/* min timer set to 1/2 of interrupt timer */
+	coal->int_lat_tmr_min = BNXT_INT_LAT_TMR_MIN;
+	/* buf timer set to 1/4 of interrupt timer */
+	coal->cmpl_aggr_dma_tmr = BNXT_CMPL_AGGR_DMA_TMR;
+	coal->cmpl_aggr_dma_tmr_during_int = BNXT_CMPL_AGGR_DMA_TMR_DURING_INT;
+}
+
+int bnxt_alloc_hwrm_rx_ring(struct bnxt *bp, int queue_index)
+{
+	struct rte_pci_device *pci_dev = bp->pdev;
+	struct bnxt_rx_queue *rxq = bp->rx_queues[queue_index];
+	struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
+	struct bnxt_ring *cp_ring = cpr->cp_ring_struct;
+	struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
+	struct bnxt_ring *ring = rxr->rx_ring_struct;
+	unsigned int map_idx = queue_index + bp->rx_cp_nr_rings;
+	int rc = 0;
+
+	bp->grp_info[queue_index].fw_stats_ctx = cpr->hw_stats_ctx_id;
+
+	/* Rx cmpl */
+	rc = bnxt_hwrm_ring_alloc(bp, cp_ring,
+				  HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL,
+				  queue_index, HWRM_NA_SIGNATURE,
+				  HWRM_NA_SIGNATURE);
+	if (rc)
+		goto err_out;
+
+	cpr->cp_doorbell = (char *)pci_dev->mem_resource[2].addr +
+		queue_index * BNXT_DB_SIZE;
+	bp->grp_info[queue_index].cp_fw_ring_id = cp_ring->fw_ring_id;
+	B_CP_DIS_DB(cpr, cpr->cp_raw_cons);
+
+	if (!queue_index) {
+		/*
+		 * In order to save completion resources, use the first
+		 * completion ring from PF or VF as the default completion ring
+		 * for async event and HWRM forward response handling.
+		 */
+		bp->def_cp_ring = cpr;
+		rc = bnxt_hwrm_set_async_event_cr(bp);
+		if (rc)
+			goto err_out;
+	}
+	/* Rx ring */
+	rc = bnxt_hwrm_ring_alloc(bp, ring, HWRM_RING_ALLOC_INPUT_RING_TYPE_RX,
+				  queue_index, cpr->hw_stats_ctx_id,
+				  cp_ring->fw_ring_id);
+	if (rc)
+		goto err_out;
+
+	rxr->rx_prod = 0;
+	rxr->rx_doorbell = (char *)pci_dev->mem_resource[2].addr +
+		queue_index * BNXT_DB_SIZE;
+	bp->grp_info[queue_index].rx_fw_ring_id = ring->fw_ring_id;
+	B_RX_DB(rxr->rx_doorbell, rxr->rx_prod);
+
+	ring = rxr->ag_ring_struct;
+	/* Agg ring */
+	if (!ring)
+		PMD_DRV_LOG(ERR, "Alloc AGG Ring is NULL!\n");
+
+	rc = bnxt_hwrm_ring_alloc(bp, ring, HWRM_RING_ALLOC_INPUT_RING_TYPE_RX,
+				  map_idx, HWRM_NA_SIGNATURE,
+				  cp_ring->fw_ring_id);
+	if (rc)
+		goto err_out;
+
+	PMD_DRV_LOG(DEBUG, "Alloc AGG Done!\n");
+	rxr->ag_prod = 0;
+	rxr->ag_doorbell = (char *)pci_dev->mem_resource[2].addr +
+		map_idx * BNXT_DB_SIZE;
+	bp->grp_info[queue_index].ag_fw_ring_id = ring->fw_ring_id;
+	B_RX_DB(rxr->ag_doorbell, rxr->ag_prod);
+
+	rxq->rx_buf_use_size = BNXT_MAX_MTU + ETHER_HDR_LEN +
+		ETHER_CRC_LEN + (2 * VLAN_TAG_SIZE);
+
+	if (bp->eth_dev->data->rx_queue_state[queue_index] ==
+	    RTE_ETH_QUEUE_STATE_STARTED) {
+		if (bnxt_init_one_rx_ring(rxq)) {
+			RTE_LOG(ERR, PMD,
+				"bnxt_init_one_rx_ring failed!\n");
+			bnxt_rx_queue_release_op(rxq);
+			rc = -ENOMEM;
+			goto err_out;
+		}
+		B_RX_DB(rxr->rx_doorbell, rxr->rx_prod);
+		B_RX_DB(rxr->ag_doorbell, rxr->ag_prod);
+	}
+	rxq->index = queue_index;
+	PMD_DRV_LOG(INFO,
+		    "queue %d, rx_deferred_start %d, state %d!\n",
+		    queue_index, rxq->rx_deferred_start,
+		    bp->eth_dev->data->rx_queue_state[queue_index]);
+
+err_out:
+	return rc;
+}
 /* ring_grp usage:
  * [0] = default completion ring
  * [1 -> +rx_cp_nr_rings] = rx_cp, rx rings
@@ -265,9 +375,12 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx,
  */
 int bnxt_alloc_hwrm_rings(struct bnxt *bp)
 {
+	struct bnxt_coal coal;
 	unsigned int i;
 	int rc = 0;
 
+	bnxt_init_dflt_coal(&coal);
+
 	for (i = 0; i < bp->rx_cp_nr_rings; i++) {
 		struct bnxt_rx_queue *rxq = bp->rx_queues[i];
 		struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
@@ -291,6 +404,7 @@ int bnxt_alloc_hwrm_rings(struct bnxt *bp)
 		cpr->cp_doorbell = (char *)bp->doorbell_base + i * 0x80;
 		bp->grp_info[i].cp_fw_ring_id = cp_ring->fw_ring_id;
 		B_CP_DIS_DB(cpr, cpr->cp_raw_cons);
+		bnxt_hwrm_set_ring_coal(bp, &coal, cp_ring->fw_ring_id);
 
 		if (!i) {
 			/*
@@ -379,6 +493,7 @@ int bnxt_alloc_hwrm_rings(struct bnxt *bp)
 
 		txr->tx_doorbell = (char *)bp->doorbell_base + idx * 0x80;
 		txq->index = idx;
+		bnxt_hwrm_set_ring_coal(bp, &coal, cp_ring->fw_ring_id);
 	}
 
 err_out:
diff --git a/drivers/net/bnxt/bnxt_ring.h b/drivers/net/bnxt/bnxt_ring.h
index 65bf3e2f..1446d784 100644
--- a/drivers/net/bnxt/bnxt_ring.h
+++ b/drivers/net/bnxt/bnxt_ring.h
@@ -70,6 +70,7 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx,
 			    struct bnxt_rx_queue *rxq,
 			    struct bnxt_cp_ring_info *cp_ring_info,
 			    const char *suffix);
+int bnxt_alloc_hwrm_rx_ring(struct bnxt *bp, int queue_index);
 int bnxt_alloc_hwrm_rings(struct bnxt *bp);
 
 #endif
diff --git a/drivers/net/bnxt/bnxt_rxq.c b/drivers/net/bnxt/bnxt_rxq.c
index c55ddec4..832fc9ec 100644
--- a/drivers/net/bnxt/bnxt_rxq.c
+++ b/drivers/net/bnxt/bnxt_rxq.c
@@ -199,12 +199,14 @@ err_out:
 	return rc;
 }
 
-static void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq)
+void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq)
 {
 	struct bnxt_sw_rx_bd *sw_ring;
 	struct bnxt_tpa_info *tpa_info;
 	uint16_t i;
 
+	rte_spinlock_lock(&rxq->lock);
+
 	if (rxq) {
 		sw_ring = rxq->rx_ring->rx_buf_ring;
 		if (sw_ring) {
@@ -239,6 +241,8 @@ static void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq)
 			}
 		}
 	}
+
+	rte_spinlock_unlock(&rxq->lock);
 }
 
 void bnxt_free_rx_mbufs(struct bnxt *bp)
@@ -286,6 +290,7 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev,
 	uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads;
 	struct bnxt_rx_queue *rxq;
 	int rc = 0;
+	uint8_t queue_state;
 
 	if (queue_idx >= bp->max_rx_rings) {
 		PMD_DRV_LOG(ERR,
@@ -326,8 +331,8 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev,
 
 	rxq->queue_id = queue_idx;
 	rxq->port_id = eth_dev->data->port_id;
-	rxq->crc_len = rx_offloads & DEV_RX_OFFLOAD_CRC_STRIP ?
-		0 : ETHER_CRC_LEN;
+	rxq->crc_len = rte_eth_dev_must_keep_crc(rx_offloads) ?
+		ETHER_CRC_LEN : 0;
 
 	eth_dev->data->rx_queues[queue_idx] = rxq;
 	/* Allocate RX ring hardware descriptors */
@@ -341,6 +346,11 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev,
 	}
 	rte_atomic64_init(&rxq->rx_mbuf_alloc_fail);
 
+	rxq->rx_deferred_start = rx_conf->rx_deferred_start;
+	queue_state = rxq->rx_deferred_start ? RTE_ETH_QUEUE_STATE_STOPPED :
+						RTE_ETH_QUEUE_STATE_STARTED;
+	eth_dev->data->rx_queue_state[queue_idx] = queue_state;
+	rte_spinlock_init(&rxq->lock);
 out:
 	return rc;
 }
@@ -389,6 +399,7 @@ int bnxt_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
 	struct bnxt_rx_queue *rxq = bp->rx_queues[rx_queue_id];
 	struct bnxt_vnic_info *vnic = NULL;
+	int rc = 0;
 
 	if (rxq == NULL) {
 		PMD_DRV_LOG(ERR, "Invalid Rx queue %d\n", rx_queue_id);
@@ -396,28 +407,47 @@ int bnxt_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	}
 
 	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
-	rxq->rx_deferred_start = false;
+
+	bnxt_free_hwrm_rx_ring(bp, rx_queue_id);
+	bnxt_alloc_hwrm_rx_ring(bp, rx_queue_id);
 	PMD_DRV_LOG(INFO, "Rx queue started %d\n", rx_queue_id);
+
 	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
 		vnic = rxq->vnic;
+
 		if (vnic->fw_grp_ids[rx_queue_id] != INVALID_HW_RING_ID)
 			return 0;
-		PMD_DRV_LOG(DEBUG, "vnic = %p fw_grp_id = %d\n",
-			vnic, bp->grp_info[rx_queue_id + 1].fw_grp_id);
+
+		PMD_DRV_LOG(DEBUG,
+			    "vnic = %p fw_grp_id = %d\n",
+			    vnic, bp->grp_info[rx_queue_id].fw_grp_id);
+
 		vnic->fw_grp_ids[rx_queue_id] =
-					bp->grp_info[rx_queue_id + 1].fw_grp_id;
-		return bnxt_vnic_rss_configure(bp, vnic);
+					bp->grp_info[rx_queue_id].fw_grp_id;
+		rc = bnxt_vnic_rss_configure(bp, vnic);
 	}
 
-	return 0;
+	if (rc == 0)
+		rxq->rx_deferred_start = false;
+
+	return rc;
 }
 
 int bnxt_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 {
 	struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
 	struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
-	struct bnxt_rx_queue *rxq = bp->rx_queues[rx_queue_id];
 	struct bnxt_vnic_info *vnic = NULL;
+	struct bnxt_rx_queue *rxq = NULL;
+	int rc = 0;
+
+	/* Rx CQ 0 also works as Default CQ for async notifications */
+	if (!rx_queue_id) {
+		PMD_DRV_LOG(ERR, "Cannot stop Rx queue id %d\n", rx_queue_id);
+		return -EINVAL;
+	}
+
+	rxq = bp->rx_queues[rx_queue_id];
 
 	if (rxq == NULL) {
 		PMD_DRV_LOG(ERR, "Invalid Rx queue %d\n", rx_queue_id);
@@ -431,7 +461,11 @@ int bnxt_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
 		vnic = rxq->vnic;
 		vnic->fw_grp_ids[rx_queue_id] = INVALID_HW_RING_ID;
-		return bnxt_vnic_rss_configure(bp, vnic);
+		rc = bnxt_vnic_rss_configure(bp, vnic);
 	}
-	return 0;
+
+	if (rc == 0)
+		bnxt_rx_queue_release_mbufs(rxq);
+
+	return rc;
 }
diff --git a/drivers/net/bnxt/bnxt_rxq.h b/drivers/net/bnxt/bnxt_rxq.h
index 8307f603..e5d6001d 100644
--- a/drivers/net/bnxt/bnxt_rxq.h
+++ b/drivers/net/bnxt/bnxt_rxq.h
@@ -10,6 +10,9 @@ struct bnxt;
 struct bnxt_rx_ring_info;
 struct bnxt_cp_ring_info;
 struct bnxt_rx_queue {
+	rte_spinlock_t		lock;	/* Synchronize between rx_queue_stop
+					 * and fast path
+					 */
 	struct rte_mempool	*mb_pool; /* mbuf pool for RX ring */
 	struct rte_mbuf		*pkt_first_seg; /* 1st seg of pkt */
 	struct rte_mbuf		*pkt_last_seg; /* Last seg of pkt */
@@ -54,4 +57,5 @@ int bnxt_rx_queue_start(struct rte_eth_dev *dev,
 			uint16_t rx_queue_id);
 int bnxt_rx_queue_stop(struct rte_eth_dev *dev,
 		       uint16_t rx_queue_id);
+void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq);
 #endif
diff --git a/drivers/net/bnxt/bnxt_rxr.c b/drivers/net/bnxt/bnxt_rxr.c
index 9d884292..c7bc8848 100644
--- a/drivers/net/bnxt/bnxt_rxr.c
+++ b/drivers/net/bnxt/bnxt_rxr.c
@@ -540,8 +540,10 @@ uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 	int rc = 0;
 	bool evt = false;
 
-	/* If Rx Q was stopped return */
-	if (rxq->rx_deferred_start)
+	/* If Rx Q was stopped return. RxQ0 cannot be stopped. */
+	if (unlikely(((rxq->rx_deferred_start ||
+		       !rte_spinlock_trylock(&rxq->lock)) &&
+		      rxq->queue_id)))
 		return 0;
 
 	/* Handle RX burst request */
@@ -572,18 +574,20 @@ uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		raw_cons = NEXT_RAW_CMP(raw_cons);
 		if (nb_rx_pkts == nb_pkts || evt)
 			break;
+		/* Post some Rx buf early in case of larger burst processing */
+		if (nb_rx_pkts == BNXT_RX_POST_THRESH)
+			B_RX_DB(rxr->rx_doorbell, rxr->rx_prod);
 	}
 
 	cpr->cp_raw_cons = raw_cons;
-	if ((prod == rxr->rx_prod && ag_prod == rxr->ag_prod) && !evt) {
+	if (!nb_rx_pkts && !evt) {
 		/*
 		 * For PMD, there is no need to keep on pushing to REARM
 		 * the doorbell if there are no new completions
 		 */
-		return nb_rx_pkts;
+		goto done;
 	}
 
-	B_CP_DIS_DB(cpr, cpr->cp_raw_cons);
 	if (prod != rxr->rx_prod)
 		B_RX_DB(rxr->rx_doorbell, rxr->rx_prod);
 
@@ -591,6 +595,8 @@ uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 	if (ag_prod != rxr->ag_prod)
 		B_RX_DB(rxr->ag_doorbell, rxr->ag_prod);
 
+	B_CP_DIS_DB(cpr, cpr->cp_raw_cons);
+
 	/* Attempt to alloc Rx buf in case of a previous allocation failure. */
 	if (rc == -ENOMEM) {
 		int i;
@@ -614,16 +620,22 @@ uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		}
 	}
 
+done:
+	rte_spinlock_unlock(&rxq->lock);
+
 	return nb_rx_pkts;
 }
 
 void bnxt_free_rx_rings(struct bnxt *bp)
 {
 	int i;
+	struct bnxt_rx_queue *rxq;
 
-	for (i = 0; i < (int)bp->rx_nr_rings; i++) {
-		struct bnxt_rx_queue *rxq = bp->rx_queues[i];
+	if (!bp->rx_queues)
+		return;
 
+	for (i = 0; i < (int)bp->rx_nr_rings; i++) {
+		rxq = bp->rx_queues[i];
 		if (!rxq)
 			continue;
 
diff --git a/drivers/net/bnxt/bnxt_rxr.h b/drivers/net/bnxt/bnxt_rxr.h
index 5b28f032..3815a219 100644
--- a/drivers/net/bnxt/bnxt_rxr.h
+++ b/drivers/net/bnxt/bnxt_rxr.h
@@ -54,6 +54,8 @@
 #define RX_CMP_IP_CS_UNKNOWN(rxcmp1)					\
 		!((rxcmp1)->flags2 & RX_CMP_IP_CS_BITS)
 
+#define BNXT_RX_POST_THRESH	32
+
 enum pkt_hash_types {
 	PKT_HASH_TYPE_NONE,	/* Undefined type */
 	PKT_HASH_TYPE_L2,	/* Input: src_MAC, dest_MAC */
diff --git a/drivers/net/bnxt/bnxt_stats.c b/drivers/net/bnxt/bnxt_stats.c
index bbd4e78b..a5d3c866 100644
--- a/drivers/net/bnxt/bnxt_stats.c
+++ b/drivers/net/bnxt/bnxt_stats.c
@@ -278,6 +278,7 @@ int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev,
 	count = 0;
 	for (i = 0; i < RTE_DIM(bnxt_rx_stats_strings); i++) {
 		uint64_t *rx_stats = (uint64_t *)bp->hw_rx_port_stats;
+		xstats[count].id = count;
 		xstats[count].value = rte_le_to_cpu_64(
 				*(uint64_t *)((char *)rx_stats +
 				bnxt_rx_stats_strings[i].offset));
@@ -286,6 +287,7 @@ int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev,
 
 	for (i = 0; i < RTE_DIM(bnxt_tx_stats_strings); i++) {
 		uint64_t *tx_stats = (uint64_t *)bp->hw_tx_port_stats;
+		xstats[count].id = count;
 		xstats[count].value = rte_le_to_cpu_64(
 				 *(uint64_t *)((char *)tx_stats +
 				bnxt_tx_stats_strings[i].offset));
@@ -293,6 +295,7 @@ int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev,
 	}
 
 	/* The Tx drop pkts aka the Anti spoof coounter */
+	xstats[count].id = count;
 	xstats[count].value = rte_le_to_cpu_64(tx_drop_pkts);
 	count++;
 
diff --git a/drivers/net/bnxt/bnxt_txq.h b/drivers/net/bnxt/bnxt_txq.h
index 720ca90c..f2c712a7 100644
--- a/drivers/net/bnxt/bnxt_txq.h
+++ b/drivers/net/bnxt/bnxt_txq.h
@@ -24,6 +24,7 @@ struct bnxt_tx_queue {
 	uint8_t			wthresh; /* Write-back threshold reg */
 	uint32_t		ctx_curr; /* Hardware context states */
 	uint8_t			tx_deferred_start; /* not in global dev start */
+	uint8_t			cmpl_next; /* Next BD to trigger a compl */
 
 	struct bnxt		*bp;
 	int			index;
diff --git a/drivers/net/bnxt/bnxt_txr.c b/drivers/net/bnxt/bnxt_txr.c
index 470fddd5..67bb35e0 100644
--- a/drivers/net/bnxt/bnxt_txr.c
+++ b/drivers/net/bnxt/bnxt_txr.c
@@ -114,7 +114,9 @@ static inline uint32_t bnxt_tx_avail(struct bnxt_tx_ring_info *txr)
 }
 
 static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt,
-				struct bnxt_tx_queue *txq)
+				struct bnxt_tx_queue *txq,
+				uint16_t *coal_pkts,
+				uint16_t *cmpl_next)
 {
 	struct bnxt_tx_ring_info *txr = txq->tx_ring;
 	struct tx_bd_long *txbd;
@@ -133,7 +135,9 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt,
 
 	if (tx_pkt->ol_flags & (PKT_TX_TCP_SEG | PKT_TX_TCP_CKSUM |
 				PKT_TX_UDP_CKSUM | PKT_TX_IP_CKSUM |
-				PKT_TX_VLAN_PKT | PKT_TX_OUTER_IP_CKSUM))
+				PKT_TX_VLAN_PKT | PKT_TX_OUTER_IP_CKSUM |
+				PKT_TX_TUNNEL_GRE | PKT_TX_TUNNEL_VXLAN |
+				PKT_TX_TUNNEL_GENEVE))
 		long_bd = true;
 
 	tx_buf = &txr->tx_buf_ring[txr->tx_prod];
@@ -146,14 +150,21 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt,
 		return -ENOMEM;
 
 	txbd = &txr->tx_desc_ring[txr->tx_prod];
-	txbd->opaque = txr->tx_prod;
+	txbd->opaque = *coal_pkts;
 	txbd->flags_type = tx_buf->nr_bds << TX_BD_LONG_FLAGS_BD_CNT_SFT;
+	txbd->flags_type |= TX_BD_SHORT_FLAGS_COAL_NOW;
+	if (!*cmpl_next) {
+		txbd->flags_type |= TX_BD_LONG_FLAGS_NO_CMPL;
+	} else {
+		*coal_pkts = 0;
+		*cmpl_next = false;
+	}
 	txbd->len = tx_pkt->data_len;
-	if (txbd->len >= 2014)
+	if (tx_pkt->pkt_len >= 2014)
 		txbd->flags_type |= TX_BD_LONG_FLAGS_LHINT_GTE2K;
 	else
-		txbd->flags_type |= lhint_arr[txbd->len >> 9];
-	txbd->address = rte_cpu_to_le_32(rte_mbuf_data_iova(tx_buf->mbuf));
+		txbd->flags_type |= lhint_arr[tx_pkt->pkt_len >> 9];
+	txbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova(tx_buf->mbuf));
 
 	if (long_bd) {
 		txbd->flags_type |= TX_BD_LONG_TYPE_TX_BD_LONG;
@@ -194,16 +205,46 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt,
 			/* Outer IP, Inner IP, Inner TCP/UDP CSO */
 			txbd1->lflags |= TX_BD_FLG_TIP_IP_TCP_UDP_CHKSUM;
 			txbd1->mss = 0;
+		} else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_TCP_CKSUM) ==
+			   PKT_TX_OIP_IIP_TCP_CKSUM) {
+			/* Outer IP, Inner IP, Inner TCP/UDP CSO */
+			txbd1->lflags |= TX_BD_FLG_TIP_IP_TCP_UDP_CHKSUM;
+			txbd1->mss = 0;
+		} else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_UDP_CKSUM) ==
+			   PKT_TX_OIP_IIP_UDP_CKSUM) {
+			/* Outer IP, Inner IP, Inner TCP/UDP CSO */
+			txbd1->lflags |= TX_BD_FLG_TIP_IP_TCP_UDP_CHKSUM;
+			txbd1->mss = 0;
 		} else if ((tx_pkt->ol_flags & PKT_TX_IIP_TCP_UDP_CKSUM) ==
 			   PKT_TX_IIP_TCP_UDP_CKSUM) {
 			/* (Inner) IP, (Inner) TCP/UDP CSO */
 			txbd1->lflags |= TX_BD_FLG_IP_TCP_UDP_CHKSUM;
 			txbd1->mss = 0;
+		} else if ((tx_pkt->ol_flags & PKT_TX_IIP_UDP_CKSUM) ==
+			   PKT_TX_IIP_UDP_CKSUM) {
+			/* (Inner) IP, (Inner) TCP/UDP CSO */
+			txbd1->lflags |= TX_BD_FLG_IP_TCP_UDP_CHKSUM;
+			txbd1->mss = 0;
+		} else if ((tx_pkt->ol_flags & PKT_TX_IIP_TCP_CKSUM) ==
+			   PKT_TX_IIP_TCP_CKSUM) {
+			/* (Inner) IP, (Inner) TCP/UDP CSO */
+			txbd1->lflags |= TX_BD_FLG_IP_TCP_UDP_CHKSUM;
+			txbd1->mss = 0;
 		} else if ((tx_pkt->ol_flags & PKT_TX_OIP_TCP_UDP_CKSUM) ==
 			   PKT_TX_OIP_TCP_UDP_CKSUM) {
 			/* Outer IP, (Inner) TCP/UDP CSO */
 			txbd1->lflags |= TX_BD_FLG_TIP_TCP_UDP_CHKSUM;
 			txbd1->mss = 0;
+		} else if ((tx_pkt->ol_flags & PKT_TX_OIP_UDP_CKSUM) ==
+			   PKT_TX_OIP_UDP_CKSUM) {
+			/* Outer IP, (Inner) TCP/UDP CSO */
+			txbd1->lflags |= TX_BD_FLG_TIP_TCP_UDP_CHKSUM;
+			txbd1->mss = 0;
+		} else if ((tx_pkt->ol_flags & PKT_TX_OIP_TCP_CKSUM) ==
+			   PKT_TX_OIP_TCP_CKSUM) {
+			/* Outer IP, (Inner) TCP/UDP CSO */
+			txbd1->lflags |= TX_BD_FLG_TIP_TCP_UDP_CHKSUM;
+			txbd1->mss = 0;
 		} else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_CKSUM) ==
 			   PKT_TX_OIP_IIP_CKSUM) {
 			/* Outer IP, Inner IP CSO */
@@ -214,11 +255,23 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt,
 			/* TCP/UDP CSO */
 			txbd1->lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM;
 			txbd1->mss = 0;
-		} else if (tx_pkt->ol_flags & PKT_TX_IP_CKSUM) {
+		} else if ((tx_pkt->ol_flags & PKT_TX_TCP_CKSUM) ==
+			   PKT_TX_TCP_CKSUM) {
+			/* TCP/UDP CSO */
+			txbd1->lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM;
+			txbd1->mss = 0;
+		} else if ((tx_pkt->ol_flags & PKT_TX_UDP_CKSUM) ==
+			   PKT_TX_UDP_CKSUM) {
+			/* TCP/UDP CSO */
+			txbd1->lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM;
+			txbd1->mss = 0;
+		} else if ((tx_pkt->ol_flags & PKT_TX_IP_CKSUM) ==
+			   PKT_TX_IP_CKSUM) {
 			/* IP CSO */
 			txbd1->lflags |= TX_BD_LONG_LFLAGS_IP_CHKSUM;
 			txbd1->mss = 0;
-		} else if (tx_pkt->ol_flags & PKT_TX_OUTER_IP_CKSUM) {
+		} else if ((tx_pkt->ol_flags & PKT_TX_OUTER_IP_CKSUM) ==
+			   PKT_TX_OUTER_IP_CKSUM) {
 			/* IP CSO */
 			txbd1->lflags |= TX_BD_LONG_LFLAGS_T_IP_CHKSUM;
 			txbd1->mss = 0;
@@ -234,14 +287,15 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt,
 		tx_buf = &txr->tx_buf_ring[txr->tx_prod];
 
 		txbd = &txr->tx_desc_ring[txr->tx_prod];
-		txbd->address = rte_cpu_to_le_32(rte_mbuf_data_iova(m_seg));
-		txbd->flags_type = TX_BD_SHORT_TYPE_TX_BD_SHORT;
+		txbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova(m_seg));
+		txbd->flags_type |= TX_BD_SHORT_TYPE_TX_BD_SHORT;
 		txbd->len = m_seg->data_len;
 
 		m_seg = m_seg->next;
 	}
 
 	txbd->flags_type |= TX_BD_LONG_FLAGS_PACKET_END;
+	txbd1->lflags = rte_cpu_to_le_32(txbd1->lflags);
 
 	txr->tx_prod = RING_NEXT(txr->tx_ring_struct, txr->tx_prod);
 
@@ -278,35 +332,44 @@ static int bnxt_handle_tx_cp(struct bnxt_tx_queue *txq)
 	struct bnxt_cp_ring_info *cpr = txq->cp_ring;
 	uint32_t raw_cons = cpr->cp_raw_cons;
 	uint32_t cons;
-	int nb_tx_pkts = 0;
+	uint32_t nb_tx_pkts = 0;
 	struct tx_cmpl *txcmp;
+	struct cmpl_base *cp_desc_ring = cpr->cp_desc_ring;
+	struct bnxt_ring *cp_ring_struct = cpr->cp_ring_struct;
+	uint32_t ring_mask = cp_ring_struct->ring_mask;
+	uint32_t opaque = 0;
 
-	if ((txq->tx_ring->tx_ring_struct->ring_size -
-			(bnxt_tx_avail(txq->tx_ring))) >
-			txq->tx_free_thresh) {
-		while (1) {
-			cons = RING_CMP(cpr->cp_ring_struct, raw_cons);
-			txcmp = (struct tx_cmpl *)&cpr->cp_desc_ring[cons];
-
-			if (!CMP_VALID(txcmp, raw_cons, cpr->cp_ring_struct))
-				break;
-			cpr->valid = FLIP_VALID(cons,
-						cpr->cp_ring_struct->ring_mask,
-						cpr->valid);
-
-			if (CMP_TYPE(txcmp) == TX_CMPL_TYPE_TX_L2)
-				nb_tx_pkts++;
-			else
-				RTE_LOG_DP(DEBUG, PMD,
-						"Unhandled CMP type %02x\n",
-						CMP_TYPE(txcmp));
-			raw_cons = NEXT_RAW_CMP(raw_cons);
-		}
-		if (nb_tx_pkts)
-			bnxt_tx_cmp(txq, nb_tx_pkts);
+	if (((txq->tx_ring->tx_prod - txq->tx_ring->tx_cons) &
+		txq->tx_ring->tx_ring_struct->ring_mask) < txq->tx_free_thresh)
+		return 0;
+
+	do {
+		cons = RING_CMPL(ring_mask, raw_cons);
+		txcmp = (struct tx_cmpl *)&cpr->cp_desc_ring[cons];
+		rte_prefetch_non_temporal(&cp_desc_ring[(cons + 2) &
+							ring_mask]);
+
+		if (!CMPL_VALID(txcmp, cpr->valid))
+			break;
+		opaque = rte_cpu_to_le_32(txcmp->opaque);
+		NEXT_CMPL(cpr, cons, cpr->valid, 1);
+		rte_prefetch0(&cp_desc_ring[cons]);
+
+		if (CMP_TYPE(txcmp) == TX_CMPL_TYPE_TX_L2)
+			nb_tx_pkts += opaque;
+		else
+			RTE_LOG_DP(ERR, PMD,
+					"Unhandled CMP type %02x\n",
+					CMP_TYPE(txcmp));
+		raw_cons = cons;
+	} while (nb_tx_pkts < ring_mask);
+
+	if (nb_tx_pkts) {
+		bnxt_tx_cmp(txq, nb_tx_pkts);
 		cpr->cp_raw_cons = raw_cons;
-		B_CP_DIS_DB(cpr, cpr->cp_raw_cons);
+		B_CP_DB(cpr, cpr->cp_raw_cons, ring_mask);
 	}
+
 	return nb_tx_pkts;
 }
 
@@ -315,8 +378,8 @@ uint16_t bnxt_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 {
 	struct bnxt_tx_queue *txq = tx_queue;
 	uint16_t nb_tx_pkts = 0;
-	uint16_t db_mask = txq->tx_ring->tx_ring_struct->ring_size >> 2;
-	uint16_t last_db_mask = 0;
+	uint16_t coal_pkts = 0;
+	uint16_t cmpl_next = txq->cmpl_next;
 
 	/* Handle TX completions */
 	bnxt_handle_tx_cp(txq);
@@ -326,16 +389,25 @@ uint16_t bnxt_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		PMD_DRV_LOG(DEBUG, "Tx q stopped;return\n");
 		return 0;
 	}
+
+	txq->cmpl_next = 0;
 	/* Handle TX burst request */
 	for (nb_tx_pkts = 0; nb_tx_pkts < nb_pkts; nb_tx_pkts++) {
-		if (bnxt_start_xmit(tx_pkts[nb_tx_pkts], txq)) {
+		int rc;
+
+		/* Request a completion on first and last packet */
+		cmpl_next |= (nb_pkts == nb_tx_pkts + 1);
+		coal_pkts++;
+		rc = bnxt_start_xmit(tx_pkts[nb_tx_pkts], txq,
+				&coal_pkts, &cmpl_next);
+
+		if (unlikely(rc)) {
+			/* Request a completion in next cycle */
+			txq->cmpl_next = 1;
 			break;
-		} else if ((nb_tx_pkts & db_mask) != last_db_mask) {
-			B_TX_DB(txq->tx_ring->tx_doorbell,
-					txq->tx_ring->tx_prod);
-			last_db_mask = nb_tx_pkts & db_mask;
 		}
 	}
+
 	if (nb_tx_pkts)
 		B_TX_DB(txq->tx_ring->tx_doorbell, txq->tx_ring->tx_prod);
 
diff --git a/drivers/net/bnxt/bnxt_txr.h b/drivers/net/bnxt/bnxt_txr.h
index 15c7e5a0..7f3c7cdb 100644
--- a/drivers/net/bnxt/bnxt_txr.h
+++ b/drivers/net/bnxt/bnxt_txr.h
@@ -45,10 +45,20 @@ int bnxt_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id);
 
 #define PKT_TX_OIP_IIP_TCP_UDP_CKSUM	(PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | \
 					PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM)
+#define PKT_TX_OIP_IIP_UDP_CKSUM	(PKT_TX_UDP_CKSUM | \
+					PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM)
+#define PKT_TX_OIP_IIP_TCP_CKSUM	(PKT_TX_TCP_CKSUM | \
+					PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM)
 #define PKT_TX_IIP_TCP_UDP_CKSUM	(PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | \
 					PKT_TX_IP_CKSUM)
+#define PKT_TX_IIP_TCP_CKSUM		(PKT_TX_TCP_CKSUM | PKT_TX_IP_CKSUM)
+#define PKT_TX_IIP_UDP_CKSUM		(PKT_TX_UDP_CKSUM | PKT_TX_IP_CKSUM)
 #define PKT_TX_OIP_TCP_UDP_CKSUM	(PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | \
 					PKT_TX_OUTER_IP_CKSUM)
+#define PKT_TX_OIP_UDP_CKSUM		(PKT_TX_UDP_CKSUM | \
+					PKT_TX_OUTER_IP_CKSUM)
+#define PKT_TX_OIP_TCP_CKSUM		(PKT_TX_TCP_CKSUM | \
+					PKT_TX_OUTER_IP_CKSUM)
 #define PKT_TX_OIP_IIP_CKSUM		(PKT_TX_IP_CKSUM |	\
 					 PKT_TX_OUTER_IP_CKSUM)
 #define PKT_TX_TCP_UDP_CKSUM		(PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)
diff --git a/drivers/net/bnxt/bnxt_util.c b/drivers/net/bnxt/bnxt_util.c
new file mode 100644
index 00000000..7d334271
--- /dev/null
+++ b/drivers/net/bnxt/bnxt_util.c
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2018 Broadcom
+ * All rights reserved.
+ */
+
+#include <inttypes.h>
+
+#include "bnxt_util.h"
+
+int bnxt_check_zero_bytes(const uint8_t *bytes, int len)
+{
+	int i;
+
+	for (i = 0; i < len; i++)
+		if (bytes[i] != 0x00)
+			return 0;
+	return 1;
+}
diff --git a/drivers/net/bnxt/bnxt_util.h b/drivers/net/bnxt/bnxt_util.h
new file mode 100644
index 00000000..2378833c
--- /dev/null
+++ b/drivers/net/bnxt/bnxt_util.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2018 Broadcom
+ * All rights reserved.
+ */
+
+#ifndef _BNXT_UTIL_H_
+#define _BNXT_UTIL_H_
+
+int bnxt_check_zero_bytes(const uint8_t *bytes, int len);
+
+#endif /* _BNXT_UTIL_H_ */
diff --git a/drivers/net/bnxt/bnxt_vnic.c b/drivers/net/bnxt/bnxt_vnic.c
index 19d06af5..c0577cd7 100644
--- a/drivers/net/bnxt/bnxt_vnic.c
+++ b/drivers/net/bnxt/bnxt_vnic.c
@@ -39,7 +39,7 @@ void bnxt_init_vnics(struct bnxt *bp)
 {
 	struct bnxt_vnic_info *vnic;
 	uint16_t max_vnics;
-	int i, j;
+	int i;
 
 	max_vnics = bp->max_vnics;
 	STAILQ_INIT(&bp->free_vnic_list);
@@ -52,9 +52,6 @@ void bnxt_init_vnics(struct bnxt *bp)
 		vnic->hash_mode =
 			HWRM_VNIC_RSS_CFG_INPUT_HASH_MODE_FLAGS_DEFAULT;
 
-		for (j = 0; j < MAX_QUEUES_PER_VNIC; j++)
-			vnic->fw_grp_ids[j] = (uint16_t)HWRM_NA_SIGNATURE;
-
 		prandom_bytes(vnic->rss_hash_key, HW_HASH_KEY_SIZE);
 		STAILQ_INIT(&vnic->filter);
 		STAILQ_INIT(&vnic->flow_list);
diff --git a/drivers/net/bnxt/bnxt_vnic.h b/drivers/net/bnxt/bnxt_vnic.h
index c521d7e5..9029f78c 100644
--- a/drivers/net/bnxt/bnxt_vnic.h
+++ b/drivers/net/bnxt/bnxt_vnic.h
@@ -15,13 +15,9 @@ struct bnxt_vnic_info {
 
 	uint16_t	fw_vnic_id; /* returned by Chimp during alloc */
 	uint16_t	rss_rule;
-#define MAX_NUM_TRAFFIC_CLASSES		8
-#define MAX_NUM_RSS_QUEUES_PER_VNIC	16
-#define MAX_QUEUES_PER_VNIC	(MAX_NUM_RSS_QUEUES_PER_VNIC + \
-				 MAX_NUM_TRAFFIC_CLASSES)
 	uint16_t	start_grp_id;
 	uint16_t	end_grp_id;
-	uint16_t	fw_grp_ids[MAX_QUEUES_PER_VNIC];
+	uint16_t	*fw_grp_ids;
 	uint16_t	dflt_ring_grp;
 	uint16_t	mru;
 	uint16_t	hash_type;
diff --git a/drivers/net/bnxt/hsi_struct_def_dpdk.h b/drivers/net/bnxt/hsi_struct_def_dpdk.h
index fd6d8807..f5c7b422 100644
--- a/drivers/net/bnxt/hsi_struct_def_dpdk.h
+++ b/drivers/net/bnxt/hsi_struct_def_dpdk.h
@@ -686,8 +686,8 @@ struct hwrm_err_output {
 #define HWRM_VERSION_MINOR 9
 #define HWRM_VERSION_UPDATE 2
 /* non-zero means beta version */
-#define HWRM_VERSION_RSVD 6
-#define HWRM_VERSION_STR "1.9.2.6"
+#define HWRM_VERSION_RSVD 9
+#define HWRM_VERSION_STR "1.9.2.9"
 
 /****************
  * hwrm_ver_get *
@@ -3183,6 +3183,9 @@ struct hwrm_async_event_cmpl {
 	/* LLFC/PFC Configuration Change */
 	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LLFC_PFC_CHANGE \
 		UINT32_C(0x34)
+	/* Default VNIC Configuration Change */
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DEFAULT_VNIC_CHANGE \
+		UINT32_C(0x35)
 	/* HWRM Error */
 	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR \
 		UINT32_C(0xff)
@@ -3280,6 +3283,11 @@ struct hwrm_async_event_cmpl_link_status_change {
 		UINT32_C(0xffff0)
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_ID_SFT \
 		4
+	/* Indicates the physical function this event occured on. */
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PF_ID_MASK \
+		UINT32_C(0xff00000)
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PF_ID_SFT \
+		20
 } __attribute__((packed));
 
 /* hwrm_async_event_cmpl_link_mtu_change (size:128b/16B) */
@@ -4087,6 +4095,10 @@ struct hwrm_async_event_cmpl_vf_flr {
 	#define HWRM_ASYNC_EVENT_CMPL_VF_FLR_EVENT_DATA1_VF_ID_MASK \
 		UINT32_C(0xffff)
 	#define HWRM_ASYNC_EVENT_CMPL_VF_FLR_EVENT_DATA1_VF_ID_SFT 0
+	/* Indicates the physical function this event occured on. */
+	#define HWRM_ASYNC_EVENT_CMPL_VF_FLR_EVENT_DATA1_PF_ID_MASK \
+		UINT32_C(0xff0000)
+	#define HWRM_ASYNC_EVENT_CMPL_VF_FLR_EVENT_DATA1_PF_ID_SFT 16
 } __attribute__((packed));
 
 /* hwrm_async_event_cmpl_vf_mac_addr_change (size:128b/16B) */
@@ -4354,6 +4366,88 @@ struct hwrm_async_event_cmpl_llfc_pfc_change {
 		5
 } __attribute__((packed));
 
+/* hwrm_async_event_cmpl_default_vnic_change (size:128b/16B) */
+struct hwrm_async_event_cmpl_default_vnic_change {
+	uint16_t	type;
+	/*
+	 * This field indicates the exact type of the completion.
+	 * By convention, the LSB identifies the length of the
+	 * record in 16B units.  Even values indicate 16B
+	 * records.  Odd values indicate 32B
+	 * records.
+	 */
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_TYPE_MASK \
+		UINT32_C(0x3f)
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_TYPE_SFT \
+		0
+	/* HWRM Asynchronous Event Information */
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_TYPE_HWRM_ASYNC_EVENT \
+		UINT32_C(0x2e)
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_TYPE_LAST \
+		HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_TYPE_HWRM_ASYNC_EVENT
+	/* unused1 is 10 b */
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_UNUSED1_MASK \
+		UINT32_C(0xffc0)
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_UNUSED1_SFT \
+		6
+	/* Identifiers of events. */
+	uint16_t	event_id;
+	/* Notification of a default vnic allocaiton or free */
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_ID_ALLOC_FREE_NOTIFICATION \
+		UINT32_C(0x35)
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_ID_LAST \
+		HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_ID_ALLOC_FREE_NOTIFICATION
+	/* Event specific data */
+	uint32_t	event_data2;
+	uint8_t	opaque_v;
+	/*
+	 * This value is written by the NIC such that it will be different
+	 * for each pass through the completion queue.   The even passes
+	 * will write 1.  The odd passes will write 0.
+	 */
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_V \
+		UINT32_C(0x1)
+	/* opaque is 7 b */
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_OPAQUE_MASK \
+		UINT32_C(0xfe)
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_OPAQUE_SFT 1
+	/* 8-lsb timestamp from POR (100-msec resolution) */
+	uint8_t	timestamp_lo;
+	/* 16-lsb timestamp from POR (100-msec resolution) */
+	uint16_t	timestamp_hi;
+	/* Event specific data */
+	uint32_t	event_data1;
+	/* Indicates default vnic configuration change */
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_MASK \
+		UINT32_C(0x3)
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_SFT \
+		0
+	/*
+	 * If this field is set to 1, then it indicates that
+	 * a default VNIC has been allocate.
+	 */
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_DEF_VNIC_ALLOC \
+		UINT32_C(0x1)
+	/*
+	 * If this field is set to 2, then it indicates that
+	 * a default VNIC has been freed.
+	 */
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_DEF_VNIC_FREE \
+		UINT32_C(0x2)
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_LAST \
+		HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_DEF_VNIC_STATE_DEF_VNIC_FREE
+	/* Indicates the physical function this event occured on. */
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_PF_ID_MASK \
+		UINT32_C(0x3fc)
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_PF_ID_SFT \
+		2
+	/* Indicates the virtual function this event occured on */
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_VF_ID_MASK \
+		UINT32_C(0x3fffc00)
+	#define HWRM_ASYNC_EVENT_CMPL_DEFAULT_VNIC_CHANGE_EVENT_DATA1_VF_ID_SFT \
+		10
+} __attribute__((packed));
+
 /* hwrm_async_event_cmpl_hwrm_error (size:128b/16B) */
 struct hwrm_async_event_cmpl_hwrm_error {
 	uint16_t	type;
@@ -5197,6 +5291,21 @@ struct hwrm_func_qcaps_output {
 	#define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_PCIE_STATS_SUPPORTED \
 		UINT32_C(0x10000)
 	/*
+	 * If the query is for a VF, then this flag shall be ignored,
+	 * If this query is for a PF and this flag is set to 1,
+	 * then the PF has the capability to adopt the VF's belonging
+	 * to another PF.
+	 */
+	#define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ADOPTED_PF_SUPPORTED \
+		UINT32_C(0x20000)
+	/*
+	 * If the query is for a VF, then this flag shall be ignored,
+	 * If this query is for a PF and this flag is set to 1,
+	 * then the PF has the capability to administer another PF.
+	 */
+	#define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ADMIN_PF_SUPPORTED \
+		UINT32_C(0x40000)
+	/*
 	 * This value is current MAC address configured for this
 	 * function. A value of 00-00-00-00-00-00 indicates no
 	 * MAC address is currently configured.
diff --git a/drivers/net/bnxt/meson.build b/drivers/net/bnxt/meson.build
new file mode 100644
index 00000000..e130f271
--- /dev/null
+++ b/drivers/net/bnxt/meson.build
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+install_headers('rte_pmd_bnxt.h')
+version = 2
+sources = files('bnxt_cpr.c',
+	'bnxt_ethdev.c',
+	'bnxt_filter.c',
+	'bnxt_flow.c',
+	'bnxt_hwrm.c',
+	'bnxt_irq.c',
+	'bnxt_ring.c',
+	'bnxt_rxq.c',
+	'bnxt_rxr.c',
+	'bnxt_stats.c',
+	'bnxt_txq.c',
+	'bnxt_txr.c',
+	'bnxt_util.c',
+	'bnxt_vnic.c',
+	'rte_pmd_bnxt.c')
diff --git a/drivers/net/bonding/rte_eth_bond_api.c b/drivers/net/bonding/rte_eth_bond_api.c
index d558df8b..8bc04cfd 100644
--- a/drivers/net/bonding/rte_eth_bond_api.c
+++ b/drivers/net/bonding/rte_eth_bond_api.c
@@ -345,14 +345,6 @@ __eth_bond_slave_add_lock_free(uint16_t bonded_port_id, uint16_t slave_port_id)
 		internals->tx_queue_offload_capa &= dev_info.tx_queue_offload_capa;
 		internals->flow_type_rss_offloads &= dev_info.flow_type_rss_offloads;
 
-		if (link_properties_valid(bonded_eth_dev,
-				&slave_eth_dev->data->dev_link) != 0) {
-			RTE_BOND_LOG(ERR, "Invalid link properties for slave %d"
-					" in bonding mode %d", slave_port_id,
-					internals->mode);
-			return -1;
-		}
-
 		/* RETA size is GCD of all slaves RETA sizes, so, if all sizes will be
 		 * the power of 2, the lower one is GCD
 		 */
@@ -373,6 +365,13 @@ __eth_bond_slave_add_lock_free(uint16_t bonded_port_id, uint16_t slave_port_id)
 		return -1;
 	}
 
+	/* Add additional MAC addresses to the slave */
+	if (slave_add_mac_addresses(bonded_eth_dev, slave_port_id) != 0) {
+		RTE_BOND_LOG(ERR, "Failed to add mac address(es) to slave %hu",
+				slave_port_id);
+		return -1;
+	}
+
 	internals->slave_count++;
 
 	if (bonded_eth_dev->data->dev_started) {
@@ -387,7 +386,7 @@ __eth_bond_slave_add_lock_free(uint16_t bonded_port_id, uint16_t slave_port_id)
 	/* Add slave details to bonded device */
 	slave_eth_dev->data->dev_flags |= RTE_ETH_DEV_BONDED_SLAVE;
 
-	/* Update all slave devices MACs*/
+	/* Update all slave devices MACs */
 	mac_address_slaves_update(bonded_eth_dev);
 
 	/* Register link status change callback with bonded device pointer as
@@ -405,11 +404,6 @@ __eth_bond_slave_add_lock_free(uint16_t bonded_port_id, uint16_t slave_port_id)
 			    !internals->user_defined_primary_port)
 				bond_ethdev_primary_set(internals,
 							slave_port_id);
-
-			if (find_slave_by_id(internals->active_slaves,
-					     internals->active_slave_count,
-					     slave_port_id) == internals->active_slave_count)
-				activate_slave(bonded_eth_dev, slave_port_id);
 		}
 	}
 
@@ -491,6 +485,9 @@ __eth_bond_slave_remove_lock_free(uint16_t bonded_port_id,
 	rte_eth_dev_default_mac_addr_set(slave_port_id,
 			&(internals->slaves[slave_idx].persisted_mac_addr));
 
+	/* remove additional MAC addresses from the slave */
+	slave_remove_mac_addresses(bonded_eth_dev, slave_port_id);
+
 	/*
 	 * Remove bond device flows from slave device.
 	 * Note: don't restore flow isolate mode.
@@ -716,9 +713,21 @@ rte_eth_bond_mac_address_reset(uint16_t bonded_port_id)
 	internals->user_defined_mac = 0;
 
 	if (internals->slave_count > 0) {
+		int slave_port;
+		/* Get the primary slave location based on the primary port
+		 * number as, while slave_add(), we will keep the primary
+		 * slave based on slave_count,but not based on the primary port.
+		 */
+		for (slave_port = 0; slave_port < internals->slave_count;
+		     slave_port++) {
+			if (internals->slaves[slave_port].port_id ==
+			    internals->primary_port)
+				break;
+		}
+
 		/* Set MAC Address of Bonded Device */
 		if (mac_address_set(bonded_eth_dev,
-				&internals->slaves[internals->primary_port].persisted_mac_addr)
+			&internals->slaves[slave_port].persisted_mac_addr)
 				!= 0) {
 			RTE_BOND_LOG(ERR, "Failed to set MAC address on bonded device");
 			return -1;
diff --git a/drivers/net/bonding/rte_eth_bond_pmd.c b/drivers/net/bonding/rte_eth_bond_pmd.c
index 02d94b1b..58f7377c 100644
--- a/drivers/net/bonding/rte_eth_bond_pmd.c
+++ b/drivers/net/bonding/rte_eth_bond_pmd.c
@@ -25,6 +25,7 @@
 
 #define REORDER_PERIOD_MS 10
 #define DEFAULT_POLLING_INTERVAL_10_MS (10)
+#define BOND_MAX_MAC_ADDRS 16
 
 #define HASH_L4_PORTS(h) ((h)->src_port ^ (h)->dst_port)
 
@@ -1588,6 +1589,61 @@ mac_address_set(struct rte_eth_dev *eth_dev, struct ether_addr *new_mac_addr)
 	return 0;
 }
 
+static const struct ether_addr null_mac_addr;
+
+/*
+ * Add additional MAC addresses to the slave
+ */
+int
+slave_add_mac_addresses(struct rte_eth_dev *bonded_eth_dev,
+		uint16_t slave_port_id)
+{
+	int i, ret;
+	struct ether_addr *mac_addr;
+
+	for (i = 1; i < BOND_MAX_MAC_ADDRS; i++) {
+		mac_addr = &bonded_eth_dev->data->mac_addrs[i];
+		if (is_same_ether_addr(mac_addr, &null_mac_addr))
+			break;
+
+		ret = rte_eth_dev_mac_addr_add(slave_port_id, mac_addr, 0);
+		if (ret < 0) {
+			/* rollback */
+			for (i--; i > 0; i--)
+				rte_eth_dev_mac_addr_remove(slave_port_id,
+					&bonded_eth_dev->data->mac_addrs[i]);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * Remove additional MAC addresses from the slave
+ */
+int
+slave_remove_mac_addresses(struct rte_eth_dev *bonded_eth_dev,
+		uint16_t slave_port_id)
+{
+	int i, rc, ret;
+	struct ether_addr *mac_addr;
+
+	rc = 0;
+	for (i = 1; i < BOND_MAX_MAC_ADDRS; i++) {
+		mac_addr = &bonded_eth_dev->data->mac_addrs[i];
+		if (is_same_ether_addr(mac_addr, &null_mac_addr))
+			break;
+
+		ret = rte_eth_dev_mac_addr_remove(slave_port_id, mac_addr);
+		/* save only the first error */
+		if (ret < 0 && rc == 0)
+			rc = ret;
+	}
+
+	return rc;
+}
+
 int
 mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev)
 {
@@ -2057,10 +2113,6 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev)
 		}
 	}
 
-	/* Update all slave devices MACs*/
-	if (mac_address_slaves_update(eth_dev) != 0)
-		goto out_err;
-
 	/* If bonded device is configure in promiscuous mode then re-apply config */
 	if (internals->promiscuous_en)
 		bond_ethdev_promiscuous_enable(eth_dev);
@@ -2101,6 +2153,10 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev)
 			(void *)&rte_eth_devices[internals->port_id]);
 	}
 
+	/* Update all slave devices MACs*/
+	if (mac_address_slaves_update(eth_dev) != 0)
+		goto out_err;
+
 	if (internals->user_defined_primary_port)
 		bond_ethdev_primary_set(internals, internals->primary_port);
 
@@ -2173,7 +2229,6 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev)
 			tlb_last_obytets[internals->active_slaves[i]] = 0;
 	}
 
-	internals->active_slave_count = 0;
 	internals->link_status_polling_enabled = 0;
 	for (i = 0; i < internals->slave_count; i++)
 		internals->slaves[i].last_link_status = 0;
@@ -2219,7 +2274,7 @@ bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	uint16_t max_nb_rx_queues = UINT16_MAX;
 	uint16_t max_nb_tx_queues = UINT16_MAX;
 
-	dev_info->max_mac_addrs = 1;
+	dev_info->max_mac_addrs = BOND_MAX_MAC_ADDRS;
 
 	dev_info->max_rx_pktlen = internals->candidate_max_rx_pktlen ?
 			internals->candidate_max_rx_pktlen :
@@ -2664,10 +2719,8 @@ bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type,
 
 	rte_eth_link_get_nowait(port_id, &link);
 	if (link.link_status) {
-		if (active_pos < internals->active_slave_count) {
-			rte_spinlock_unlock(&internals->lsc_lock);
-			return rc;
-		}
+		if (active_pos < internals->active_slave_count)
+			goto link_update;
 
 		/* if no active slave ports then set this port to be primary port */
 		if (internals->active_slave_count < 1) {
@@ -2679,6 +2732,17 @@ bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type,
 			mac_address_slaves_update(bonded_eth_dev);
 		}
 
+		/* check link state properties if bonded link is up*/
+		if (bonded_eth_dev->data->dev_link.link_status == ETH_LINK_UP) {
+			if (link_properties_valid(bonded_eth_dev, &link) != 0)
+				RTE_BOND_LOG(ERR, "Invalid link properties "
+					     "for slave %d in bonding mode %d",
+					     port_id, internals->mode);
+		} else {
+			/* inherit slave link properties */
+			link_properties_set(bonded_eth_dev, &link);
+		}
+
 		activate_slave(bonded_eth_dev, port_id);
 
 		/* If user has defined the primary port then default to using it */
@@ -2686,10 +2750,8 @@ bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type,
 				internals->primary_port == port_id)
 			bond_ethdev_primary_set(internals, port_id);
 	} else {
-		if (active_pos == internals->active_slave_count) {
-			rte_spinlock_unlock(&internals->lsc_lock);
-			return rc;
-		}
+		if (active_pos == internals->active_slave_count)
+			goto link_update;
 
 		/* Remove from active slave list */
 		deactivate_slave(bonded_eth_dev, port_id);
@@ -2708,6 +2770,7 @@ bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type,
 		}
 	}
 
+link_update:
 	/**
 	 * Update bonded device link properties after any change to active
 	 * slaves
@@ -2745,7 +2808,7 @@ bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type,
 
 	rte_spinlock_unlock(&internals->lsc_lock);
 
-	return 0;
+	return rc;
 }
 
 static int
@@ -2905,6 +2968,68 @@ bond_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
 	return -ENOTSUP;
 }
 
+static int
+bond_ethdev_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
+				__rte_unused uint32_t index, uint32_t vmdq)
+{
+	struct rte_eth_dev *slave_eth_dev;
+	struct bond_dev_private *internals = dev->data->dev_private;
+	int ret, i;
+
+	rte_spinlock_lock(&internals->lock);
+
+	for (i = 0; i < internals->slave_count; i++) {
+		slave_eth_dev = &rte_eth_devices[internals->slaves[i].port_id];
+		if (*slave_eth_dev->dev_ops->mac_addr_add == NULL ||
+			 *slave_eth_dev->dev_ops->mac_addr_remove == NULL) {
+			ret = -ENOTSUP;
+			goto end;
+		}
+	}
+
+	for (i = 0; i < internals->slave_count; i++) {
+		ret = rte_eth_dev_mac_addr_add(internals->slaves[i].port_id,
+				mac_addr, vmdq);
+		if (ret < 0) {
+			/* rollback */
+			for (i--; i >= 0; i--)
+				rte_eth_dev_mac_addr_remove(
+					internals->slaves[i].port_id, mac_addr);
+			goto end;
+		}
+	}
+
+	ret = 0;
+end:
+	rte_spinlock_unlock(&internals->lock);
+	return ret;
+}
+
+static void
+bond_ethdev_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
+{
+	struct rte_eth_dev *slave_eth_dev;
+	struct bond_dev_private *internals = dev->data->dev_private;
+	int i;
+
+	rte_spinlock_lock(&internals->lock);
+
+	for (i = 0; i < internals->slave_count; i++) {
+		slave_eth_dev = &rte_eth_devices[internals->slaves[i].port_id];
+		if (*slave_eth_dev->dev_ops->mac_addr_remove == NULL)
+			goto end;
+	}
+
+	struct ether_addr *mac_addr = &dev->data->mac_addrs[index];
+
+	for (i = 0; i < internals->slave_count; i++)
+		rte_eth_dev_mac_addr_remove(internals->slaves[i].port_id,
+				mac_addr);
+
+end:
+	rte_spinlock_unlock(&internals->lock);
+}
+
 const struct eth_dev_ops default_dev_ops = {
 	.dev_start            = bond_ethdev_start,
 	.dev_stop             = bond_ethdev_stop,
@@ -2927,6 +3052,8 @@ const struct eth_dev_ops default_dev_ops = {
 	.rss_hash_conf_get    = bond_ethdev_rss_hash_conf_get,
 	.mtu_set              = bond_ethdev_mtu_set,
 	.mac_addr_set         = bond_ethdev_mac_address_set,
+	.mac_addr_add         = bond_ethdev_mac_addr_add,
+	.mac_addr_remove      = bond_ethdev_mac_addr_remove,
 	.filter_ctrl          = bond_filter_ctrl
 };
 
@@ -2954,10 +3081,13 @@ bond_alloc(struct rte_vdev_device *dev, uint8_t mode)
 	eth_dev->data->nb_rx_queues = (uint16_t)1;
 	eth_dev->data->nb_tx_queues = (uint16_t)1;
 
-	eth_dev->data->mac_addrs = rte_zmalloc_socket(name, ETHER_ADDR_LEN, 0,
-			socket_id);
+	/* Allocate memory for storing MAC addresses */
+	eth_dev->data->mac_addrs = rte_zmalloc_socket(name, ETHER_ADDR_LEN *
+			BOND_MAX_MAC_ADDRS, 0, socket_id);
 	if (eth_dev->data->mac_addrs == NULL) {
-		RTE_BOND_LOG(ERR, "Unable to malloc mac_addrs");
+		RTE_BOND_LOG(ERR,
+			     "Failed to allocate %u bytes needed to store MAC addresses",
+			     ETHER_ADDR_LEN * BOND_MAX_MAC_ADDRS);
 		goto err;
 	}
 
@@ -3065,6 +3195,7 @@ bond_probe(struct rte_vdev_device *dev)
 		}
 		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &default_dev_ops;
+		eth_dev->device = &dev->device;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -3119,6 +3250,7 @@ bond_probe(struct rte_vdev_device *dev)
 	internals = rte_eth_devices[port_id].data->dev_private;
 	internals->kvlist = kvlist;
 
+	rte_eth_dev_probing_finish(&rte_eth_devices[port_id]);
 
 	if (rte_kvargs_count(kvlist, PMD_BOND_AGG_MODE_KVARG) == 1) {
 		if (rte_kvargs_process(kvlist,
@@ -3138,7 +3270,6 @@ bond_probe(struct rte_vdev_device *dev)
 		rte_eth_bond_8023ad_agg_selection_set(port_id, AGG_STABLE);
 	}
 
-	rte_eth_dev_probing_finish(&rte_eth_devices[port_id]);
 	RTE_BOND_LOG(INFO, "Create bonded device %s on port %d in mode %u on "
 			"socket %u.",	name, port_id, bonding_mode, socket_id);
 	return 0;
@@ -3485,9 +3616,7 @@ RTE_PMD_REGISTER_PARAM_STRING(net_bonding,
 
 int bond_logtype;
 
-RTE_INIT(bond_init_log);
-static void
-bond_init_log(void)
+RTE_INIT(bond_init_log)
 {
 	bond_logtype = rte_log_register("pmd.net.bon");
 	if (bond_logtype >= 0)
diff --git a/drivers/net/bonding/rte_eth_bond_private.h b/drivers/net/bonding/rte_eth_bond_private.h
index 65445b86..43e0e448 100644
--- a/drivers/net/bonding/rte_eth_bond_private.h
+++ b/drivers/net/bonding/rte_eth_bond_private.h
@@ -231,6 +231,14 @@ int
 mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev);
 
 int
+slave_add_mac_addresses(struct rte_eth_dev *bonded_eth_dev,
+		uint16_t slave_port_id);
+
+int
+slave_remove_mac_addresses(struct rte_eth_dev *bonded_eth_dev,
+		uint16_t slave_port_id);
+
+int
 bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode);
 
 int
diff --git a/drivers/net/cxgbe/Makefile b/drivers/net/cxgbe/Makefile
index 79fdb6f0..5d66c4b3 100644
--- a/drivers/net/cxgbe/Makefile
+++ b/drivers/net/cxgbe/Makefile
@@ -49,7 +49,10 @@ SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbevf_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_main.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbevf_main.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += sge.c
+SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_filter.c
+SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_flow.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += t4_hw.c
+SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += clip_tbl.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += t4vf_hw.c
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/cxgbe/base/adapter.h b/drivers/net/cxgbe/base/adapter.h
index 55cb2e91..e98dd218 100644
--- a/drivers/net/cxgbe/base/adapter.h
+++ b/drivers/net/cxgbe/base/adapter.h
@@ -11,12 +11,16 @@
 #include <rte_bus_pci.h>
 #include <rte_mbuf.h>
 #include <rte_io.h>
+#include <rte_rwlock.h>
+#include <rte_ethdev.h>
 
 #include "cxgbe_compat.h"
 #include "t4_regs_values.h"
+#include "cxgbe_ofld.h"
 
 enum {
 	MAX_ETH_QSETS = 64,           /* # of Ethernet Tx/Rx queue sets */
+	MAX_CTRL_QUEUES = NCHAN,      /* # of control Tx queues */
 };
 
 struct adapter;
@@ -254,10 +258,20 @@ struct sge_eth_txq {                   /* state for an SGE Ethernet Tx queue */
 	unsigned int flags;            /* flags for state of the queue */
 } __rte_cache_aligned;
 
+struct sge_ctrl_txq {                /* State for an SGE control Tx queue */
+	struct sge_txq q;            /* txq */
+	struct adapter *adapter;     /* adapter associated with this queue */
+	rte_spinlock_t ctrlq_lock;   /* control queue lock */
+	u8 full;                     /* the Tx ring is full */
+	u64 txp;                     /* number of transmits */
+	struct rte_mempool *mb_pool; /* mempool to generate ctrl pkts */
+} __rte_cache_aligned;
+
 struct sge {
 	struct sge_eth_txq ethtxq[MAX_ETH_QSETS];
 	struct sge_eth_rxq ethrxq[MAX_ETH_QSETS];
 	struct sge_rspq fw_evtq __rte_cache_aligned;
+	struct sge_ctrl_txq ctrlq[MAX_CTRL_QUEUES];
 
 	u16 max_ethqsets;           /* # of available Ethernet queue sets */
 	u32 stat_len;               /* length of status page at ring end */
@@ -306,9 +320,54 @@ struct adapter {
 	unsigned int vpd_flag;
 
 	int use_unpacked_mode; /* unpacked rx mode state */
+	rte_spinlock_t win0_lock;
+
+	unsigned int clipt_start; /* CLIP table start */
+	unsigned int clipt_end;   /* CLIP table end */
+	struct clip_tbl *clipt;   /* CLIP table */
+
+	struct tid_info tids;     /* Info used to access TID related tables */
 };
 
 /**
+ * t4_os_rwlock_init - initialize rwlock
+ * @lock: the rwlock
+ */
+static inline void t4_os_rwlock_init(rte_rwlock_t *lock)
+{
+	rte_rwlock_init(lock);
+}
+
+/**
+ * t4_os_write_lock - get a write lock
+ * @lock: the rwlock
+ */
+static inline void t4_os_write_lock(rte_rwlock_t *lock)
+{
+	rte_rwlock_write_lock(lock);
+}
+
+/**
+ * t4_os_write_unlock - unlock a write lock
+ * @lock: the rwlock
+ */
+static inline void t4_os_write_unlock(rte_rwlock_t *lock)
+{
+	rte_rwlock_write_unlock(lock);
+}
+
+/**
+ * ethdev2pinfo - return the port_info structure associated with a rte_eth_dev
+ * @dev: the rte_eth_dev
+ *
+ * Return the struct port_info associated with a rte_eth_dev
+ */
+static inline struct port_info *ethdev2pinfo(const struct rte_eth_dev *dev)
+{
+	return (struct port_info *)dev->data->dev_private;
+}
+
+/**
  * adap2pinfo - return the port_info of a port
  * @adap: the adapter
  * @idx: the port index
@@ -320,6 +379,17 @@ static inline struct port_info *adap2pinfo(const struct adapter *adap, int idx)
 	return adap->port[idx];
 }
 
+/**
+ * ethdev2adap - return the adapter structure associated with a rte_eth_dev
+ * @dev: the rte_eth_dev
+ *
+ * Return the struct adapter associated with a rte_eth_dev
+ */
+static inline struct adapter *ethdev2adap(const struct rte_eth_dev *dev)
+{
+	return ethdev2pinfo(dev)->adapter;
+}
+
 #define CXGBE_PCI_REG(reg) rte_read32(reg)
 
 static inline uint64_t cxgbe_read_addr64(volatile void *addr)
@@ -680,6 +750,38 @@ static inline void t4_os_atomic_list_del(struct mbox_entry *entry,
 	t4_os_unlock(lock);
 }
 
+/**
+ * t4_init_completion - initialize completion
+ * @c: the completion context
+ */
+static inline void t4_init_completion(struct t4_completion *c)
+{
+	c->done = 0;
+	t4_os_lock_init(&c->lock);
+}
+
+/**
+ * t4_complete - set completion as done
+ * @c: the completion context
+ */
+static inline void t4_complete(struct t4_completion *c)
+{
+	t4_os_lock(&c->lock);
+	c->done = 1;
+	t4_os_unlock(&c->lock);
+}
+
+/**
+ * cxgbe_port_viid - get the VI id of a port
+ * @dev: the device for the port
+ *
+ * Return the VI id of the given port.
+ */
+static inline unsigned int cxgbe_port_viid(const struct rte_eth_dev *dev)
+{
+	return ethdev2pinfo(dev)->viid;
+}
+
 void *t4_alloc_mem(size_t size);
 void t4_free_mem(void *addr);
 #define t4_os_alloc(_size)     t4_alloc_mem((_size))
@@ -694,6 +796,7 @@ void t4_sge_tx_monitor_start(struct adapter *adap);
 void t4_sge_tx_monitor_stop(struct adapter *adap);
 int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf,
 		uint16_t nb_pkts);
+int t4_mgmt_tx(struct sge_ctrl_txq *txq, struct rte_mbuf *mbuf);
 int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
 		     const struct pkt_gl *gl);
 int t4_sge_init(struct adapter *adap);
@@ -701,6 +804,9 @@ int t4vf_sge_init(struct adapter *adap);
 int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
 			 struct rte_eth_dev *eth_dev, uint16_t queue_id,
 			 unsigned int iqid, int socket_id);
+int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
+			  struct rte_eth_dev *eth_dev, uint16_t queue_id,
+			  unsigned int iqid, int socket_id);
 int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *rspq, bool fwevtq,
 		     struct rte_eth_dev *eth_dev, int intr_idx,
 		     struct sge_fl *fl, rspq_handler_t handler,
diff --git a/drivers/net/cxgbe/base/common.h b/drivers/net/cxgbe/base/common.h
index 155a3028..157201da 100644
--- a/drivers/net/cxgbe/base/common.h
+++ b/drivers/net/cxgbe/base/common.h
@@ -18,6 +18,9 @@ extern "C" {
 
 #define CXGBE_PAGE_SIZE RTE_PGSIZE_4K
 
+#define T4_MEMORY_WRITE 0
+#define T4_MEMORY_READ  1
+
 enum {
 	MAX_NPORTS     = 4,     /* max # of ports */
 };
@@ -47,6 +50,8 @@ enum cc_fec {
 	FEC_BASER_RS = 1 << 2,    /* BaseR/Reed-Solomon */
 };
 
+enum { MEM_EDC0, MEM_EDC1, MEM_MC, MEM_MC0 = MEM_MC, MEM_MC1 };
+
 struct port_stats {
 	u64 tx_octets;            /* total # of octets in good frames */
 	u64 tx_frames;            /* all good frames */
@@ -151,6 +156,9 @@ struct tp_params {
 	int vnic_shift;
 	int port_shift;
 	int protocol_shift;
+	int ethertype_shift;
+
+	u64 hash_filter_mask;
 };
 
 struct vpd_params {
@@ -203,6 +211,14 @@ struct rss_params {
 };
 
 /*
+ * Maximum resources provisioned for a PCI PF.
+ */
+struct pf_resources {
+	unsigned int neq;      /* N egress Qs */
+	unsigned int niqflint; /* N ingress Qs/w free list(s) & intr */
+};
+
+/*
  * Maximum resources provisioned for a PCI VF.
  */
 struct vf_resources {
@@ -225,6 +241,7 @@ struct adapter_params {
 	struct pci_params pci;
 	struct devlog_params devlog;
 	struct rss_params rss;
+	struct pf_resources pfres;
 	struct vf_resources vfres;
 	enum pcie_memwin drv_memwin;
 
@@ -246,6 +263,8 @@ struct adapter_params {
 	unsigned char nports;             /* # of ethernet ports */
 	unsigned char portvec;
 
+	unsigned char hash_filter;
+
 	enum chip_type chip;              /* chip code */
 	struct arch_specific_params arch; /* chip specific params */
 
@@ -309,6 +328,11 @@ static inline int is_pf4(struct adapter *adap)
 #define for_each_port(adapter, iter) \
 	for (iter = 0; iter < (adapter)->params.nports; ++iter)
 
+static inline int is_hashfilter(const struct adapter *adap)
+{
+	return adap->params.hash_filter;
+}
+
 void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log);
 void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr,
 			    unsigned int mask, unsigned int val);
@@ -378,6 +402,8 @@ int t4_iq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 	       unsigned int fl0id, unsigned int fl1id);
 int t4_eth_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		   unsigned int vf, unsigned int eqid);
+int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
+		    unsigned int vf, unsigned int eqid);
 
 static inline unsigned int core_ticks_per_usec(const struct adapter *adap)
 {
@@ -449,6 +475,7 @@ void t4_write_indirect(struct adapter *adap, unsigned int addr_reg,
 		       unsigned int nregs, unsigned int start_idx);
 
 int t4_get_vpd_params(struct adapter *adapter, struct vpd_params *p);
+int t4_get_pfres(struct adapter *adapter);
 int t4_read_flash(struct adapter *adapter, unsigned int addr,
 		  unsigned int nwords, u32 *data, int byte_oriented);
 int t4_flash_cfg_addr(struct adapter *adapter);
@@ -500,5 +527,15 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size);
 int t4_seeprom_read(struct adapter *adapter, u32 addr, u32 *data);
 int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data);
 int t4_seeprom_wp(struct adapter *adapter, int enable);
+int t4_memory_rw_addr(struct adapter *adap, int win,
+		      u32 addr, u32 len, void *hbuf, int dir);
+int t4_memory_rw_mtype(struct adapter *adap, int win, int mtype, u32 maddr,
+		       u32 len, void *hbuf, int dir);
+static inline int t4_memory_rw(struct adapter *adap, int win,
+			       int mtype, u32 maddr, u32 len,
+			       void *hbuf, int dir)
+{
+	return t4_memory_rw_mtype(adap, win, mtype, maddr, len, hbuf, dir);
+}
 fw_port_cap32_t fwcaps16_to_caps32(fw_port_cap16_t caps16);
 #endif /* __CHELSIO_COMMON_H */
diff --git a/drivers/net/cxgbe/base/t4_hw.c b/drivers/net/cxgbe/base/t4_hw.c
index e5ef73b6..31762c9c 100644
--- a/drivers/net/cxgbe/base/t4_hw.c
+++ b/drivers/net/cxgbe/base/t4_hw.c
@@ -2480,6 +2480,46 @@ int t4_get_core_clock(struct adapter *adapter, struct vpd_params *p)
 	return 0;
 }
 
+/**
+ * t4_get_pfres - retrieve VF resource limits
+ * @adapter: the adapter
+ *
+ * Retrieves configured resource limits and capabilities for a physical
+ * function.  The results are stored in @adapter->pfres.
+ */
+int t4_get_pfres(struct adapter *adapter)
+{
+	struct pf_resources *pfres = &adapter->params.pfres;
+	struct fw_pfvf_cmd cmd, rpl;
+	u32 word;
+	int v;
+
+	/*
+	 * Execute PFVF Read command to get VF resource limits; bail out early
+	 * with error on command failure.
+	 */
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_PFVF_CMD) |
+				    F_FW_CMD_REQUEST |
+				    F_FW_CMD_READ |
+				    V_FW_PFVF_CMD_PFN(adapter->pf) |
+				    V_FW_PFVF_CMD_VFN(0));
+	cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd));
+	v = t4_wr_mbox(adapter, adapter->mbox, &cmd, sizeof(cmd), &rpl);
+	if (v != FW_SUCCESS)
+		return v;
+
+	/*
+	 * Extract PF resource limits and return success.
+	 */
+	word = be32_to_cpu(rpl.niqflint_niq);
+	pfres->niqflint = G_FW_PFVF_CMD_NIQFLINT(word);
+
+	word = be32_to_cpu(rpl.type_to_neq);
+	pfres->neq = G_FW_PFVF_CMD_NEQ(word);
+	return 0;
+}
+
 /* serial flash and firmware constants and flash config file constants */
 enum {
 	SF_ATTEMPTS = 10,             /* max retries for SF operations */
@@ -4491,6 +4531,31 @@ static void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl)
 }
 
 /**
+ * t4_ctrl_eq_free - free a control egress queue
+ * @adap: the adapter
+ * @mbox: mailbox to use for the FW command
+ * @pf: the PF owning the queue
+ * @vf: the VF owning the queue
+ * @eqid: egress queue id
+ *
+ * Frees a control egress queue.
+ */
+int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
+		    unsigned int vf, unsigned int eqid)
+{
+	struct fw_eq_ctrl_cmd c;
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) |
+				  F_FW_CMD_REQUEST | F_FW_CMD_EXEC |
+				  V_FW_EQ_CTRL_CMD_PFN(pf) |
+				  V_FW_EQ_CTRL_CMD_VFN(vf));
+	c.alloc_to_len16 = cpu_to_be32(F_FW_EQ_CTRL_CMD_FREE | FW_LEN16(c));
+	c.cmpliqid_eqid = cpu_to_be32(V_FW_EQ_CTRL_CMD_EQID(eqid));
+	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
+}
+
+/**
  * t4_handle_fw_rpl - process a FW reply message
  * @adap: the adapter
  * @rpl: start of the FW message
@@ -4616,9 +4681,8 @@ struct flash_desc {
 int t4_get_flash_params(struct adapter *adapter)
 {
 	/*
-	 * Table for non-Numonix supported flash parts.  Numonix parts are left
-	 * to the preexisting well-tested code.  All flash parts have 64KB
-	 * sectors.
+	 * Table for non-standard supported Flash parts.  Note, all Flash
+	 * parts must have 64KB sectors.
 	 */
 	static struct flash_desc supported_flash[] = {
 		{ 0x00150201, 4 << 20 },       /* Spansion 4MB S25FL032P */
@@ -4627,7 +4691,7 @@ int t4_get_flash_params(struct adapter *adapter)
 	int ret;
 	u32 flashid = 0;
 	unsigned int part, manufacturer;
-	unsigned int density, size;
+	unsigned int density, size = 0;
 
 	/**
 	 * Issue a Read ID Command to the Flash part.  We decode supported
@@ -4642,6 +4706,9 @@ int t4_get_flash_params(struct adapter *adapter)
 	if (ret < 0)
 		return ret;
 
+	/**
+	 * Check to see if it's one of our non-standard supported Flash parts.
+	 */
 	for (part = 0; part < ARRAY_SIZE(supported_flash); part++) {
 		if (supported_flash[part].vendor_and_model_id == flashid) {
 			adapter->params.sf_size =
@@ -4652,6 +4719,15 @@ int t4_get_flash_params(struct adapter *adapter)
 		}
 	}
 
+	/**
+	 * Decode Flash part size.  The code below looks repetative with
+	 * common encodings, but that's not guaranteed in the JEDEC
+	 * specification for the Read JADEC ID command.  The only thing that
+	 * we're guaranteed by the JADEC specification is where the
+	 * Manufacturer ID is in the returned result.  After that each
+	 * Manufacturer ~could~ encode things completely differently.
+	 * Note, all Flash parts must have 64KB sectors.
+	 */
 	manufacturer = flashid & 0xff;
 	switch (manufacturer) {
 	case 0x20: { /* Micron/Numonix */
@@ -4688,21 +4764,81 @@ int t4_get_flash_params(struct adapter *adapter)
 		case 0x22:
 			size = 1 << 28; /* 256MB */
 			break;
-		default:
-			dev_err(adapter, "Micron Flash Part has bad size, ID = %#x, Density code = %#x\n",
-				flashid, density);
-			return -EINVAL;
 		}
+		break;
+	}
 
-		adapter->params.sf_size = size;
-		adapter->params.sf_nsec = size / SF_SEC_SIZE;
+	case 0x9d: { /* ISSI -- Integrated Silicon Solution, Inc. */
+		/**
+		 * This Density -> Size decoding table is taken from ISSI
+		 * Data Sheets.
+		 */
+		density = (flashid >> 16) & 0xff;
+		switch (density) {
+		case 0x16:
+			size = 1 << 25; /* 32MB */
+			break;
+		case 0x17:
+			size = 1 << 26; /* 64MB */
+			break;
+		}
 		break;
 	}
-	default:
-		dev_err(adapter, "Unsupported Flash Part, ID = %#x\n", flashid);
-		return -EINVAL;
+
+	case 0xc2: { /* Macronix */
+		/**
+		 * This Density -> Size decoding table is taken from Macronix
+		 * Data Sheets.
+		 */
+		density = (flashid >> 16) & 0xff;
+		switch (density) {
+		case 0x17:
+			size = 1 << 23; /* 8MB */
+			break;
+		case 0x18:
+			size = 1 << 24; /* 16MB */
+			break;
+		}
+		break;
 	}
 
+	case 0xef: { /* Winbond */
+		/**
+		 * This Density -> Size decoding table is taken from Winbond
+		 * Data Sheets.
+		 */
+		density = (flashid >> 16) & 0xff;
+		switch (density) {
+		case 0x17:
+			size = 1 << 23; /* 8MB */
+			break;
+		case 0x18:
+			size = 1 << 24; /* 16MB */
+			break;
+		}
+		break;
+	}
+	}
+
+	/* If we didn't recognize the FLASH part, that's no real issue: the
+	 * Hardware/Software contract says that Hardware will _*ALWAYS*_
+	 * use a FLASH part which is at least 4MB in size and has 64KB
+	 * sectors.  The unrecognized FLASH part is likely to be much larger
+	 * than 4MB, but that's all we really need.
+	 */
+	if (size == 0) {
+		dev_warn(adapter,
+			 "Unknown Flash Part, ID = %#x, assuming 4MB\n",
+			 flashid);
+		size = 1 << 22;
+	}
+
+	/**
+	 * Store decoded Flash size and fall through into vetting code.
+	 */
+	adapter->params.sf_size = size;
+	adapter->params.sf_nsec = size / SF_SEC_SIZE;
+
 found:
 	/*
 	 * We should reject adapters with FLASHes which are too small. So, emit
@@ -5007,6 +5143,8 @@ int t4_init_tp_params(struct adapter *adap)
 	adap->params.tp.port_shift = t4_filter_field_shift(adap, F_PORT);
 	adap->params.tp.protocol_shift = t4_filter_field_shift(adap,
 							       F_PROTOCOL);
+	adap->params.tp.ethertype_shift = t4_filter_field_shift(adap,
+								F_ETHERTYPE);
 
 	/*
 	 * If TP_INGRESS_CONFIG.VNID == 0, then TP_VLAN_PRI_MAP.VNIC_ID
@@ -5015,6 +5153,11 @@ int t4_init_tp_params(struct adapter *adap)
 	if ((adap->params.tp.ingress_config & F_VNIC) == 0)
 		adap->params.tp.vnic_shift = -1;
 
+	v = t4_read_reg(adap, LE_3_DB_HASH_MASK_GEN_IPV4_T6_A);
+	adap->params.tp.hash_filter_mask = v;
+	v = t4_read_reg(adap, LE_4_DB_HASH_MASK_GEN_IPV4_T6_A);
+	adap->params.tp.hash_filter_mask |= ((u64)v << 32);
+
 	return 0;
 }
 
@@ -5190,3 +5333,212 @@ int t4_port_init(struct adapter *adap, int mbox, int pf, int vf)
 	}
 	return 0;
 }
+
+/**
+ * t4_memory_rw_addr - read/write adapter memory via PCIE memory window
+ * @adap: the adapter
+ * @win: PCI-E Memory Window to use
+ * @addr: address within adapter memory
+ * @len: amount of memory to transfer
+ * @hbuf: host memory buffer
+ * @dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0)
+ *
+ * Reads/writes an [almost] arbitrary memory region in the firmware: the
+ * firmware memory address and host buffer must be aligned on 32-bit
+ * boudaries; the length may be arbitrary.
+ *
+ * NOTES:
+ *  1. The memory is transferred as a raw byte sequence from/to the
+ *     firmware's memory.  If this memory contains data structures which
+ *     contain multi-byte integers, it's the caller's responsibility to
+ *     perform appropriate byte order conversions.
+ *
+ *  2. It is the Caller's responsibility to ensure that no other code
+ *     uses the specified PCI-E Memory Window while this routine is
+ *     using it.  This is typically done via the use of OS-specific
+ *     locks, etc.
+ */
+int t4_memory_rw_addr(struct adapter *adap, int win, u32 addr,
+		      u32 len, void *hbuf, int dir)
+{
+	u32 pos, offset, resid;
+	u32 win_pf, mem_reg, mem_aperture, mem_base;
+	u32 *buf;
+
+	/* Argument sanity checks ...*/
+	if (addr & 0x3 || (uintptr_t)hbuf & 0x3)
+		return -EINVAL;
+	buf = (u32 *)hbuf;
+
+	/* It's convenient to be able to handle lengths which aren't a
+	 * multiple of 32-bits because we often end up transferring files to
+	 * the firmware.  So we'll handle that by normalizing the length here
+	 * and then handling any residual transfer at the end.
+	 */
+	resid = len & 0x3;
+	len -= resid;
+
+	/* Each PCI-E Memory Window is programmed with a window size -- or
+	 * "aperture" -- which controls the granularity of its mapping onto
+	 * adapter memory.  We need to grab that aperture in order to know
+	 * how to use the specified window.  The window is also programmed
+	 * with the base address of the Memory Window in BAR0's address
+	 * space.  For T4 this is an absolute PCI-E Bus Address.  For T5
+	 * the address is relative to BAR0.
+	 */
+	mem_reg = t4_read_reg(adap,
+			      PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN,
+						  win));
+	mem_aperture = 1 << (G_WINDOW(mem_reg) + X_WINDOW_SHIFT);
+	mem_base = G_PCIEOFST(mem_reg) << X_PCIEOFST_SHIFT;
+
+	win_pf = is_t4(adap->params.chip) ? 0 : V_PFNUM(adap->pf);
+
+	/* Calculate our initial PCI-E Memory Window Position and Offset into
+	 * that Window.
+	 */
+	pos = addr & ~(mem_aperture - 1);
+	offset = addr - pos;
+
+	/* Set up initial PCI-E Memory Window to cover the start of our
+	 * transfer.  (Read it back to ensure that changes propagate before we
+	 * attempt to use the new value.)
+	 */
+	t4_write_reg(adap,
+		     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, win),
+		     pos | win_pf);
+	t4_read_reg(adap,
+		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, win));
+
+	/* Transfer data to/from the adapter as long as there's an integral
+	 * number of 32-bit transfers to complete.
+	 *
+	 * A note on Endianness issues:
+	 *
+	 * The "register" reads and writes below from/to the PCI-E Memory
+	 * Window invoke the standard adapter Big-Endian to PCI-E Link
+	 * Little-Endian "swizzel."  As a result, if we have the following
+	 * data in adapter memory:
+	 *
+	 *     Memory:  ... | b0 | b1 | b2 | b3 | ...
+	 *     Address:      i+0  i+1  i+2  i+3
+	 *
+	 * Then a read of the adapter memory via the PCI-E Memory Window
+	 * will yield:
+	 *
+	 *     x = readl(i)
+	 *         31                  0
+	 *         [ b3 | b2 | b1 | b0 ]
+	 *
+	 * If this value is stored into local memory on a Little-Endian system
+	 * it will show up correctly in local memory as:
+	 *
+	 *     ( ..., b0, b1, b2, b3, ... )
+	 *
+	 * But on a Big-Endian system, the store will show up in memory
+	 * incorrectly swizzled as:
+	 *
+	 *     ( ..., b3, b2, b1, b0, ... )
+	 *
+	 * So we need to account for this in the reads and writes to the
+	 * PCI-E Memory Window below by undoing the register read/write
+	 * swizzels.
+	 */
+	while (len > 0) {
+		if (dir == T4_MEMORY_READ)
+			*buf++ = le32_to_cpu((__le32)t4_read_reg(adap,
+								 mem_base +
+								 offset));
+		else
+			t4_write_reg(adap, mem_base + offset,
+				     (u32)cpu_to_le32(*buf++));
+		offset += sizeof(__be32);
+		len -= sizeof(__be32);
+
+		/* If we've reached the end of our current window aperture,
+		 * move the PCI-E Memory Window on to the next.  Note that
+		 * doing this here after "len" may be 0 allows us to set up
+		 * the PCI-E Memory Window for a possible final residual
+		 * transfer below ...
+		 */
+		if (offset == mem_aperture) {
+			pos += mem_aperture;
+			offset = 0;
+			t4_write_reg(adap,
+				PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET,
+						    win), pos | win_pf);
+			t4_read_reg(adap,
+				PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET,
+						    win));
+		}
+	}
+
+	/* If the original transfer had a length which wasn't a multiple of
+	 * 32-bits, now's where we need to finish off the transfer of the
+	 * residual amount.  The PCI-E Memory Window has already been moved
+	 * above (if necessary) to cover this final transfer.
+	 */
+	if (resid) {
+		union {
+			u32 word;
+			char byte[4];
+		} last;
+		unsigned char *bp;
+		int i;
+
+		if (dir == T4_MEMORY_READ) {
+			last.word = le32_to_cpu((__le32)t4_read_reg(adap,
+								    mem_base +
+								    offset));
+			for (bp = (unsigned char *)buf, i = resid; i < 4; i++)
+				bp[i] = last.byte[i];
+		} else {
+			last.word = *buf;
+			for (i = resid; i < 4; i++)
+				last.byte[i] = 0;
+			t4_write_reg(adap, mem_base + offset,
+				     (u32)cpu_to_le32(last.word));
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * t4_memory_rw_mtype -read/write EDC 0, EDC 1 or MC via PCIE memory window
+ * @adap: the adapter
+ * @win: PCI-E Memory Window to use
+ * @mtype: memory type: MEM_EDC0, MEM_EDC1 or MEM_MC
+ * @maddr: address within indicated memory type
+ * @len: amount of memory to transfer
+ * @hbuf: host memory buffer
+ * @dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0)
+ *
+ * Reads/writes adapter memory using t4_memory_rw_addr().  This routine
+ * provides an (memory type, address within memory type) interface.
+ */
+int t4_memory_rw_mtype(struct adapter *adap, int win, int mtype, u32 maddr,
+		       u32 len, void *hbuf, int dir)
+{
+	u32 mtype_offset;
+	u32 edc_size, mc_size;
+
+	/* Offset into the region of memory which is being accessed
+	 * MEM_EDC0 = 0
+	 * MEM_EDC1 = 1
+	 * MEM_MC   = 2 -- MEM_MC for chips with only 1 memory controller
+	 * MEM_MC1  = 3 -- for chips with 2 memory controllers (e.g. T5)
+	 */
+	edc_size  = G_EDRAM0_SIZE(t4_read_reg(adap, A_MA_EDRAM0_BAR));
+	if (mtype != MEM_MC1) {
+		mtype_offset = (mtype * (edc_size * 1024 * 1024));
+	} else {
+		mc_size = G_EXT_MEM0_SIZE(t4_read_reg(adap,
+						      A_MA_EXT_MEMORY0_BAR));
+		mtype_offset = (MEM_MC0 * edc_size + mc_size) * 1024 * 1024;
+	}
+
+	return t4_memory_rw_addr(adap, win,
+				 mtype_offset + maddr, len,
+				 hbuf, dir);
+}
diff --git a/drivers/net/cxgbe/base/t4_hw.h b/drivers/net/cxgbe/base/t4_hw.h
index ac12afc0..e77563df 100644
--- a/drivers/net/cxgbe/base/t4_hw.h
+++ b/drivers/net/cxgbe/base/t4_hw.h
@@ -42,6 +42,10 @@ enum {
 	SGE_MAX_WR_NDESC = SGE_MAX_WR_LEN / SGE_EQ_IDXSIZE,
 };
 
+enum {
+	TCB_SIZE        = 128,   /* TCB size */
+};
+
 struct sge_qstat {                /* data written to SGE queue status entries */
 	__be32 qid;
 	__be16 cidx;
diff --git a/drivers/net/cxgbe/base/t4_msg.h b/drivers/net/cxgbe/base/t4_msg.h
index 74b4fc19..5d433c91 100644
--- a/drivers/net/cxgbe/base/t4_msg.h
+++ b/drivers/net/cxgbe/base/t4_msg.h
@@ -7,6 +7,15 @@
 #define T4_MSG_H
 
 enum {
+	CPL_ACT_OPEN_REQ      = 0x3,
+	CPL_SET_TCB_FIELD     = 0x5,
+	CPL_ABORT_REQ         = 0xA,
+	CPL_ABORT_RPL         = 0xB,
+	CPL_TID_RELEASE       = 0x1A,
+	CPL_ACT_OPEN_RPL      = 0x25,
+	CPL_ABORT_RPL_RSS     = 0x2D,
+	CPL_SET_TCB_RPL       = 0x3A,
+	CPL_ACT_OPEN_REQ6     = 0x83,
 	CPL_SGE_EGR_UPDATE    = 0xA5,
 	CPL_FW4_MSG           = 0xC0,
 	CPL_FW6_MSG           = 0xE0,
@@ -14,6 +23,20 @@ enum {
 	CPL_TX_PKT_XT         = 0xEE,
 };
 
+enum CPL_error {
+	CPL_ERR_NONE               = 0,
+	CPL_ERR_TCAM_FULL          = 3,
+};
+
+enum {
+	ULP_MODE_NONE          = 0,
+};
+
+enum {
+	CPL_ABORT_SEND_RST = 0,
+	CPL_ABORT_NO_RST,
+};
+
 enum {                     /* TX_PKT_XT checksum types */
 	TX_CSUM_TCPIP  = 8,
 	TX_CSUM_UDPIP  = 9,
@@ -25,6 +48,24 @@ union opcode_tid {
 	__u8 opcode;
 };
 
+#define S_CPL_OPCODE    24
+#define V_CPL_OPCODE(x) ((x) << S_CPL_OPCODE)
+
+#define G_TID(x)    ((x) & 0xFFFFFF)
+
+/* tid is assumed to be 24-bits */
+#define MK_OPCODE_TID(opcode, tid) (V_CPL_OPCODE(opcode) | (tid))
+
+#define OPCODE_TID(cmd) ((cmd)->ot.opcode_tid)
+
+/* extract the TID from a CPL command */
+#define GET_TID(cmd) (G_TID(be32_to_cpu(OPCODE_TID(cmd))))
+
+/* partitioning of TID fields that also carry a queue id */
+#define S_TID_TID    0
+#define M_TID_TID    0x3fff
+#define G_TID_TID(x) (((x) >> S_TID_TID) & M_TID_TID)
+
 struct rss_header {
 	__u8 opcode;
 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
@@ -66,6 +107,169 @@ struct work_request_hdr {
 #define WR_HDR_SIZE 0
 #endif
 
+#define S_COOKIE    5
+#define M_COOKIE    0x7
+#define V_COOKIE(x) ((x) << S_COOKIE)
+#define G_COOKIE(x) (((x) >> S_COOKIE) & M_COOKIE)
+
+/* option 0 fields */
+#define S_TX_CHAN    2
+#define V_TX_CHAN(x) ((x) << S_TX_CHAN)
+
+#define S_DELACK    5
+#define V_DELACK(x) ((x) << S_DELACK)
+
+#define S_NON_OFFLOAD    7
+#define V_NON_OFFLOAD(x) ((x) << S_NON_OFFLOAD)
+#define F_NON_OFFLOAD    V_NON_OFFLOAD(1U)
+
+#define S_ULP_MODE    8
+#define V_ULP_MODE(x) ((x) << S_ULP_MODE)
+
+#define S_SMAC_SEL    28
+#define V_SMAC_SEL(x) ((__u64)(x) << S_SMAC_SEL)
+
+#define S_TCAM_BYPASS    48
+#define V_TCAM_BYPASS(x) ((__u64)(x) << S_TCAM_BYPASS)
+#define F_TCAM_BYPASS    V_TCAM_BYPASS(1ULL)
+
+/* option 2 fields */
+#define S_RSS_QUEUE    0
+#define V_RSS_QUEUE(x) ((x) << S_RSS_QUEUE)
+
+#define S_RSS_QUEUE_VALID    10
+#define V_RSS_QUEUE_VALID(x) ((x) << S_RSS_QUEUE_VALID)
+#define F_RSS_QUEUE_VALID    V_RSS_QUEUE_VALID(1U)
+
+#define S_CONG_CNTRL    14
+#define V_CONG_CNTRL(x) ((x) << S_CONG_CNTRL)
+
+#define S_RX_CHANNEL    26
+#define V_RX_CHANNEL(x) ((x) << S_RX_CHANNEL)
+#define F_RX_CHANNEL    V_RX_CHANNEL(1U)
+
+#define S_CCTRL_ECN    27
+#define V_CCTRL_ECN(x) ((x) << S_CCTRL_ECN)
+
+#define S_T5_OPT_2_VALID    31
+#define V_T5_OPT_2_VALID(x) ((x) << S_T5_OPT_2_VALID)
+#define F_T5_OPT_2_VALID    V_T5_OPT_2_VALID(1U)
+
+struct cpl_t6_act_open_req {
+	WR_HDR;
+	union opcode_tid ot;
+	__be16 local_port;
+	__be16 peer_port;
+	__be32 local_ip;
+	__be32 peer_ip;
+	__be64 opt0;
+	__be32 rsvd;
+	__be32 opt2;
+	__be64 params;
+	__be32 rsvd2;
+	__be32 opt3;
+};
+
+struct cpl_t6_act_open_req6 {
+	WR_HDR;
+	union opcode_tid ot;
+	__be16 local_port;
+	__be16 peer_port;
+	__be64 local_ip_hi;
+	__be64 local_ip_lo;
+	__be64 peer_ip_hi;
+	__be64 peer_ip_lo;
+	__be64 opt0;
+	__be32 rsvd;
+	__be32 opt2;
+	__be64 params;
+	__be32 rsvd2;
+	__be32 opt3;
+};
+
+#define S_FILTER_TUPLE	24
+#define V_FILTER_TUPLE(x) ((x) << S_FILTER_TUPLE)
+
+struct cpl_act_open_rpl {
+	RSS_HDR
+	union opcode_tid ot;
+	__be32 atid_status;
+};
+
+/* cpl_act_open_rpl.atid_status fields */
+#define S_AOPEN_STATUS    0
+#define M_AOPEN_STATUS    0xFF
+#define G_AOPEN_STATUS(x) (((x) >> S_AOPEN_STATUS) & M_AOPEN_STATUS)
+
+#define S_AOPEN_ATID    8
+#define M_AOPEN_ATID    0xFFFFFF
+#define G_AOPEN_ATID(x) (((x) >> S_AOPEN_ATID) & M_AOPEN_ATID)
+
+struct cpl_set_tcb_field {
+	WR_HDR;
+	union opcode_tid ot;
+	__be16 reply_ctrl;
+	__be16 word_cookie;
+	__be64 mask;
+	__be64 val;
+};
+
+/* cpl_set_tcb_field.word_cookie fields */
+#define S_WORD    0
+#define V_WORD(x) ((x) << S_WORD)
+
+/* cpl_get_tcb.reply_ctrl fields */
+#define S_QUEUENO    0
+#define V_QUEUENO(x) ((x) << S_QUEUENO)
+
+#define S_REPLY_CHAN    14
+#define V_REPLY_CHAN(x) ((x) << S_REPLY_CHAN)
+
+#define S_NO_REPLY    15
+#define V_NO_REPLY(x) ((x) << S_NO_REPLY)
+
+struct cpl_set_tcb_rpl {
+	RSS_HDR
+	union opcode_tid ot;
+	__be16 rsvd;
+	__u8   cookie;
+	__u8   status;
+	__be64 oldval;
+};
+
+/* cpl_abort_req status command code
+ */
+struct cpl_abort_req {
+	WR_HDR;
+	union opcode_tid ot;
+	__be32 rsvd0;
+	__u8  rsvd1;
+	__u8  cmd;
+	__u8  rsvd2[6];
+};
+
+struct cpl_abort_rpl_rss {
+	RSS_HDR
+	union opcode_tid ot;
+	__u8  rsvd[3];
+	__u8  status;
+};
+
+struct cpl_abort_rpl {
+	WR_HDR;
+	union opcode_tid ot;
+	__be32 rsvd0;
+	__u8  rsvd1;
+	__u8  cmd;
+	__u8  rsvd2[6];
+};
+
+struct cpl_tid_release {
+	WR_HDR;
+	union opcode_tid ot;
+	__be32 rsvd;
+};
+
 struct cpl_tx_data {
 	union opcode_tid ot;
 	__be32 len;
@@ -271,7 +475,13 @@ struct cpl_fw6_msg {
 	__be64 data[4];
 };
 
+/* ULP_TX opcodes */
+enum {
+	ULP_TX_PKT = 4
+};
+
 enum {
+	ULP_TX_SC_NOOP = 0x80,
 	ULP_TX_SC_IMM  = 0x81,
 	ULP_TX_SC_DSGL = 0x82,
 	ULP_TX_SC_ISGL = 0x83
diff --git a/drivers/net/cxgbe/base/t4_regs.h b/drivers/net/cxgbe/base/t4_regs.h
index c0d6ddca..6f872edc 100644
--- a/drivers/net/cxgbe/base/t4_regs.h
+++ b/drivers/net/cxgbe/base/t4_regs.h
@@ -458,6 +458,7 @@
 #define F_CRXPKTENC    V_CRXPKTENC(1U)
 
 #define TP_BASE_ADDR 0x7d00
+#define A_TP_CMM_TCB_BASE 0x7d10
 
 #define A_TP_TIMER_RESOLUTION 0x7d90
 
@@ -574,6 +575,21 @@
 #define S_RM_OVLAN	9
 #define V_RM_OVLAN(x)	((x) << S_RM_OVLAN)
 
+/* registers for module MA */
+#define A_MA_EDRAM0_BAR 0x77c0
+
+#define S_EDRAM0_SIZE    0
+#define M_EDRAM0_SIZE    0xfffU
+#define V_EDRAM0_SIZE(x) ((x) << S_EDRAM0_SIZE)
+#define G_EDRAM0_SIZE(x) (((x) >> S_EDRAM0_SIZE) & M_EDRAM0_SIZE)
+
+#define A_MA_EXT_MEMORY0_BAR 0x77c8
+
+#define S_EXT_MEM0_SIZE    0
+#define M_EXT_MEM0_SIZE    0xfffU
+#define V_EXT_MEM0_SIZE(x) ((x) << S_EXT_MEM0_SIZE)
+#define G_EXT_MEM0_SIZE(x) (((x) >> S_EXT_MEM0_SIZE) & M_EXT_MEM0_SIZE)
+
 /* registers for module MPS */
 #define MPS_BASE_ADDR 0x9000
 #define T4VF_MPS_BASE_ADDR 0x0100
@@ -783,8 +799,11 @@
 #define A_MPS_STAT_RX_BG_3_LB_TRUNC_FRAME_L 0x96b8
 #define A_MPS_STAT_RX_BG_3_LB_TRUNC_FRAME_H 0x96bc
 
+#define A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L 0x80
 #define A_MPS_VF_STAT_TX_VF_BCAST_FRAMES_L 0x88
+#define A_MPS_VF_STAT_TX_VF_MCAST_BYTES_L 0x90
 #define A_MPS_VF_STAT_TX_VF_MCAST_FRAMES_L 0x98
+#define A_MPS_VF_STAT_TX_VF_UCAST_BYTES_L 0xa0
 #define A_MPS_VF_STAT_TX_VF_UCAST_FRAMES_L 0xa8
 #define A_MPS_VF_STAT_TX_VF_DROP_FRAMES_L 0xb0
 #define A_MPS_VF_STAT_RX_VF_BCAST_FRAMES_L 0xd0
@@ -921,3 +940,15 @@
 #define M_REV    0xfU
 #define V_REV(x) ((x) << S_REV)
 #define G_REV(x) (((x) >> S_REV) & M_REV)
+
+/* registers for module LE */
+#define A_LE_DB_CONFIG 0x19c04
+
+#define S_HASHEN    20
+#define V_HASHEN(x) ((x) << S_HASHEN)
+#define F_HASHEN    V_HASHEN(1U)
+
+#define A_LE_DB_TID_HASHBASE 0x19df8
+
+#define LE_3_DB_HASH_MASK_GEN_IPV4_T6_A 0x19eac
+#define LE_4_DB_HASH_MASK_GEN_IPV4_T6_A 0x19eb0
diff --git a/drivers/net/cxgbe/base/t4_tcb.h b/drivers/net/cxgbe/base/t4_tcb.h
new file mode 100644
index 00000000..25435f9f
--- /dev/null
+++ b/drivers/net/cxgbe/base/t4_tcb.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+
+#ifndef _T4_TCB_DEFS_H
+#define _T4_TCB_DEFS_H
+
+/* 105:96 */
+#define W_TCB_RSS_INFO    3
+#define S_TCB_RSS_INFO    0
+#define M_TCB_RSS_INFO    0x3ffULL
+#define V_TCB_RSS_INFO(x) ((x) << S_TCB_RSS_INFO)
+
+/* 191:160 */
+#define W_TCB_TIMESTAMP    5
+#define S_TCB_TIMESTAMP    0
+#define M_TCB_TIMESTAMP    0xffffffffULL
+#define V_TCB_TIMESTAMP(x) ((x) << S_TCB_TIMESTAMP)
+
+/* 223:192 */
+#define S_TCB_T_RTT_TS_RECENT_AGE    0
+#define M_TCB_T_RTT_TS_RECENT_AGE    0xffffffffULL
+#define V_TCB_T_RTT_TS_RECENT_AGE(x) ((x) << S_TCB_T_RTT_TS_RECENT_AGE)
+
+#endif /* _T4_TCB_DEFS_H */
diff --git a/drivers/net/cxgbe/base/t4fw_interface.h b/drivers/net/cxgbe/base/t4fw_interface.h
index 852e8f3c..e80b58a3 100644
--- a/drivers/net/cxgbe/base/t4fw_interface.h
+++ b/drivers/net/cxgbe/base/t4fw_interface.h
@@ -54,6 +54,9 @@ enum fw_memtype {
  ********************************/
 
 enum fw_wr_opcodes {
+	FW_FILTER_WR		= 0x02,
+	FW_ULPTX_WR		= 0x04,
+	FW_TP_WR		= 0x05,
 	FW_ETH_TX_PKT_WR	= 0x08,
 	FW_ETH_TX_PKTS_WR	= 0x09,
 	FW_ETH_TX_PKT_VM_WR	= 0x11,
@@ -76,6 +79,11 @@ struct fw_wr_hdr {
 #define V_FW_WR_OP(x)		((x) << S_FW_WR_OP)
 #define G_FW_WR_OP(x)		(((x) >> S_FW_WR_OP) & M_FW_WR_OP)
 
+/* atomic flag (hi) - firmware encapsulates CPLs in CPL_BARRIER
+ */
+#define S_FW_WR_ATOMIC		23
+#define V_FW_WR_ATOMIC(x)	((x) << S_FW_WR_ATOMIC)
+
 /* work request immediate data length (hi)
  */
 #define S_FW_WR_IMMDLEN	0
@@ -92,6 +100,11 @@ struct fw_wr_hdr {
 #define G_FW_WR_EQUEQ(x)	(((x) >> S_FW_WR_EQUEQ) & M_FW_WR_EQUEQ)
 #define F_FW_WR_EQUEQ		V_FW_WR_EQUEQ(1U)
 
+/* flow context identifier (lo)
+ */
+#define S_FW_WR_FLOWID		8
+#define V_FW_WR_FLOWID(x)	((x) << S_FW_WR_FLOWID)
+
 /* length in units of 16-bytes (lo)
  */
 #define S_FW_WR_LEN16		0
@@ -143,6 +156,150 @@ struct fw_eth_tx_pkts_vm_wr {
 	__be16 vlantci;
 };
 
+/* filter wr reply code in cookie in CPL_SET_TCB_RPL */
+enum fw_filter_wr_cookie {
+	FW_FILTER_WR_SUCCESS,
+	FW_FILTER_WR_FLT_ADDED,
+	FW_FILTER_WR_FLT_DELETED,
+	FW_FILTER_WR_SMT_TBL_FULL,
+	FW_FILTER_WR_EINVAL,
+};
+
+struct fw_filter_wr {
+	__be32 op_pkd;
+	__be32 len16_pkd;
+	__be64 r3;
+	__be32 tid_to_iq;
+	__be32 del_filter_to_l2tix;
+	__be16 ethtype;
+	__be16 ethtypem;
+	__u8   frag_to_ovlan_vldm;
+	__u8   smac_sel;
+	__be16 rx_chan_rx_rpl_iq;
+	__be32 maci_to_matchtypem;
+	__u8   ptcl;
+	__u8   ptclm;
+	__u8   ttyp;
+	__u8   ttypm;
+	__be16 ivlan;
+	__be16 ivlanm;
+	__be16 ovlan;
+	__be16 ovlanm;
+	__u8   lip[16];
+	__u8   lipm[16];
+	__u8   fip[16];
+	__u8   fipm[16];
+	__be16 lp;
+	__be16 lpm;
+	__be16 fp;
+	__be16 fpm;
+	__be16 r7;
+	__u8   sma[6];
+};
+
+#define S_FW_FILTER_WR_TID	12
+#define V_FW_FILTER_WR_TID(x)	((x) << S_FW_FILTER_WR_TID)
+
+#define S_FW_FILTER_WR_RQTYPE		11
+#define V_FW_FILTER_WR_RQTYPE(x)	((x) << S_FW_FILTER_WR_RQTYPE)
+
+#define S_FW_FILTER_WR_NOREPLY		10
+#define V_FW_FILTER_WR_NOREPLY(x)	((x) << S_FW_FILTER_WR_NOREPLY)
+
+#define S_FW_FILTER_WR_IQ	0
+#define V_FW_FILTER_WR_IQ(x)	((x) << S_FW_FILTER_WR_IQ)
+
+#define S_FW_FILTER_WR_DEL_FILTER	31
+#define V_FW_FILTER_WR_DEL_FILTER(x)	((x) << S_FW_FILTER_WR_DEL_FILTER)
+#define F_FW_FILTER_WR_DEL_FILTER	V_FW_FILTER_WR_DEL_FILTER(1U)
+
+#define S_FW_FILTER_WR_RPTTID		25
+#define V_FW_FILTER_WR_RPTTID(x)	((x) << S_FW_FILTER_WR_RPTTID)
+
+#define S_FW_FILTER_WR_DROP	24
+#define V_FW_FILTER_WR_DROP(x)	((x) << S_FW_FILTER_WR_DROP)
+
+#define S_FW_FILTER_WR_DIRSTEER		23
+#define V_FW_FILTER_WR_DIRSTEER(x)	((x) << S_FW_FILTER_WR_DIRSTEER)
+
+#define S_FW_FILTER_WR_MASKHASH		22
+#define V_FW_FILTER_WR_MASKHASH(x)	((x) << S_FW_FILTER_WR_MASKHASH)
+
+#define S_FW_FILTER_WR_DIRSTEERHASH	21
+#define V_FW_FILTER_WR_DIRSTEERHASH(x)	((x) << S_FW_FILTER_WR_DIRSTEERHASH)
+
+#define S_FW_FILTER_WR_LPBK	20
+#define V_FW_FILTER_WR_LPBK(x)	((x) << S_FW_FILTER_WR_LPBK)
+
+#define S_FW_FILTER_WR_DMAC	19
+#define V_FW_FILTER_WR_DMAC(x)	((x) << S_FW_FILTER_WR_DMAC)
+
+#define S_FW_FILTER_WR_INSVLAN		17
+#define V_FW_FILTER_WR_INSVLAN(x)	((x) << S_FW_FILTER_WR_INSVLAN)
+
+#define S_FW_FILTER_WR_RMVLAN		16
+#define V_FW_FILTER_WR_RMVLAN(x)	((x) << S_FW_FILTER_WR_RMVLAN)
+
+#define S_FW_FILTER_WR_HITCNTS		15
+#define V_FW_FILTER_WR_HITCNTS(x)	((x) << S_FW_FILTER_WR_HITCNTS)
+
+#define S_FW_FILTER_WR_TXCHAN		13
+#define V_FW_FILTER_WR_TXCHAN(x)	((x) << S_FW_FILTER_WR_TXCHAN)
+
+#define S_FW_FILTER_WR_PRIO	12
+#define V_FW_FILTER_WR_PRIO(x)	((x) << S_FW_FILTER_WR_PRIO)
+
+#define S_FW_FILTER_WR_L2TIX	0
+#define V_FW_FILTER_WR_L2TIX(x)	((x) << S_FW_FILTER_WR_L2TIX)
+
+#define S_FW_FILTER_WR_FRAG	7
+#define V_FW_FILTER_WR_FRAG(x)	((x) << S_FW_FILTER_WR_FRAG)
+
+#define S_FW_FILTER_WR_FRAGM	6
+#define V_FW_FILTER_WR_FRAGM(x)	((x) << S_FW_FILTER_WR_FRAGM)
+
+#define S_FW_FILTER_WR_IVLAN_VLD	5
+#define V_FW_FILTER_WR_IVLAN_VLD(x)	((x) << S_FW_FILTER_WR_IVLAN_VLD)
+
+#define S_FW_FILTER_WR_OVLAN_VLD	4
+#define V_FW_FILTER_WR_OVLAN_VLD(x)	((x) << S_FW_FILTER_WR_OVLAN_VLD)
+
+#define S_FW_FILTER_WR_IVLAN_VLDM	3
+#define V_FW_FILTER_WR_IVLAN_VLDM(x)	((x) << S_FW_FILTER_WR_IVLAN_VLDM)
+
+#define S_FW_FILTER_WR_OVLAN_VLDM	2
+#define V_FW_FILTER_WR_OVLAN_VLDM(x)	((x) << S_FW_FILTER_WR_OVLAN_VLDM)
+
+#define S_FW_FILTER_WR_RX_CHAN		15
+#define V_FW_FILTER_WR_RX_CHAN(x)	((x) << S_FW_FILTER_WR_RX_CHAN)
+
+#define S_FW_FILTER_WR_RX_RPL_IQ	0
+#define V_FW_FILTER_WR_RX_RPL_IQ(x)	((x) << S_FW_FILTER_WR_RX_RPL_IQ)
+
+#define S_FW_FILTER_WR_MACI	23
+#define V_FW_FILTER_WR_MACI(x)	((x) << S_FW_FILTER_WR_MACI)
+
+#define S_FW_FILTER_WR_MACIM	14
+#define V_FW_FILTER_WR_MACIM(x)	((x) << S_FW_FILTER_WR_MACIM)
+
+#define S_FW_FILTER_WR_FCOE	13
+#define V_FW_FILTER_WR_FCOE(x)	((x) << S_FW_FILTER_WR_FCOE)
+
+#define S_FW_FILTER_WR_FCOEM	12
+#define V_FW_FILTER_WR_FCOEM(x)	((x) << S_FW_FILTER_WR_FCOEM)
+
+#define S_FW_FILTER_WR_PORT	9
+#define V_FW_FILTER_WR_PORT(x)	((x) << S_FW_FILTER_WR_PORT)
+
+#define S_FW_FILTER_WR_PORTM	6
+#define V_FW_FILTER_WR_PORTM(x)	((x) << S_FW_FILTER_WR_PORTM)
+
+#define S_FW_FILTER_WR_MATCHTYPE	3
+#define V_FW_FILTER_WR_MATCHTYPE(x)	((x) << S_FW_FILTER_WR_MATCHTYPE)
+
+#define S_FW_FILTER_WR_MATCHTYPEM	0
+#define V_FW_FILTER_WR_MATCHTYPEM(x)	((x) << S_FW_FILTER_WR_MATCHTYPEM)
+
 /******************************************************************************
  *  C O M M A N D s
  *********************/
@@ -178,6 +335,7 @@ enum fw_cmd_opcodes {
 	FW_PFVF_CMD		       = 0x09,
 	FW_IQ_CMD                      = 0x10,
 	FW_EQ_ETH_CMD                  = 0x12,
+	FW_EQ_CTRL_CMD                 = 0x13,
 	FW_VI_CMD                      = 0x14,
 	FW_VI_MAC_CMD                  = 0x15,
 	FW_VI_RXMODE_CMD               = 0x16,
@@ -187,6 +345,7 @@ enum fw_cmd_opcodes {
 	FW_RSS_IND_TBL_CMD             = 0x20,
 	FW_RSS_GLB_CONFIG_CMD	       = 0x22,
 	FW_RSS_VI_CONFIG_CMD           = 0x23,
+	FW_CLIP_CMD                    = 0x28,
 	FW_DEBUG_CMD                   = 0x81,
 };
 
@@ -489,6 +648,10 @@ enum fw_params_mnem {
 enum fw_params_param_dev {
 	FW_PARAMS_PARAM_DEV_CCLK	= 0x00, /* chip core clock in khz */
 	FW_PARAMS_PARAM_DEV_PORTVEC	= 0x01, /* the port vector */
+	FW_PARAMS_PARAM_DEV_NTID        = 0x02, /* reads the number of TIDs
+						 * allocated by the device's
+						 * Lookup Engine
+						 */
 	FW_PARAMS_PARAM_DEV_FWREV	= 0x0B, /* fw version */
 	FW_PARAMS_PARAM_DEV_TPREV	= 0x0C, /* tp version */
 	FW_PARAMS_PARAM_DEV_ULPTX_MEMWRITE_DSGL = 0x17,
@@ -498,6 +661,10 @@ enum fw_params_param_dev {
  * physical and virtual function parameters
  */
 enum fw_params_param_pfvf {
+	FW_PARAMS_PARAM_PFVF_CLIP_START = 0x03,
+	FW_PARAMS_PARAM_PFVF_CLIP_END = 0x04,
+	FW_PARAMS_PARAM_PFVF_FILTER_START = 0x05,
+	FW_PARAMS_PARAM_PFVF_FILTER_END = 0x06,
 	FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP = 0x31,
 	FW_PARAMS_PARAM_PFVF_PORT_CAPS32 = 0x3A
 };
@@ -577,6 +744,12 @@ struct fw_pfvf_cmd {
 	__be32 r4;
 };
 
+#define S_FW_PFVF_CMD_PFN		8
+#define V_FW_PFVF_CMD_PFN(x)		((x) << S_FW_PFVF_CMD_PFN)
+
+#define S_FW_PFVF_CMD_VFN		0
+#define V_FW_PFVF_CMD_VFN(x)		((x) << S_FW_PFVF_CMD_VFN)
+
 #define S_FW_PFVF_CMD_NIQFLINT          20
 #define M_FW_PFVF_CMD_NIQFLINT          0xfff
 #define G_FW_PFVF_CMD_NIQFLINT(x)       \
@@ -636,6 +809,11 @@ enum fw_iq_type {
 	FW_IQ_TYPE_FL_INT_CAP,
 };
 
+enum fw_iq_iqtype {
+	FW_IQ_IQTYPE_NIC = 1,
+	FW_IQ_IQTYPE_OFLD,
+};
+
 struct fw_iq_cmd {
 	__be32 op_to_vfn;
 	__be32 alloc_to_len16;
@@ -769,6 +947,9 @@ struct fw_iq_cmd {
 	(((x) >> S_FW_IQ_CMD_IQFLINTCONGEN) & M_FW_IQ_CMD_IQFLINTCONGEN)
 #define F_FW_IQ_CMD_IQFLINTCONGEN	V_FW_IQ_CMD_IQFLINTCONGEN(1U)
 
+#define S_FW_IQ_CMD_IQTYPE	24
+#define V_FW_IQ_CMD_IQTYPE(x)	((x) << S_FW_IQ_CMD_IQTYPE)
+
 #define S_FW_IQ_CMD_FL0CNGCHMAP		20
 #define M_FW_IQ_CMD_FL0CNGCHMAP		0xf
 #define V_FW_IQ_CMD_FL0CNGCHMAP(x)	((x) << S_FW_IQ_CMD_FL0CNGCHMAP)
@@ -954,6 +1135,75 @@ struct fw_eq_eth_cmd {
 #define G_FW_EQ_ETH_CMD_VIID(x)	\
 	(((x) >> S_FW_EQ_ETH_CMD_VIID) & M_FW_EQ_ETH_CMD_VIID)
 
+struct fw_eq_ctrl_cmd {
+	__be32 op_to_vfn;
+	__be32 alloc_to_len16;
+	__be32 cmpliqid_eqid;
+	__be32 physeqid_pkd;
+	__be32 fetchszm_to_iqid;
+	__be32 dcaen_to_eqsize;
+	__be64 eqaddr;
+};
+
+#define S_FW_EQ_CTRL_CMD_PFN		8
+#define V_FW_EQ_CTRL_CMD_PFN(x)		((x) << S_FW_EQ_CTRL_CMD_PFN)
+
+#define S_FW_EQ_CTRL_CMD_VFN		0
+#define V_FW_EQ_CTRL_CMD_VFN(x)		((x) << S_FW_EQ_CTRL_CMD_VFN)
+
+#define S_FW_EQ_CTRL_CMD_ALLOC		31
+#define V_FW_EQ_CTRL_CMD_ALLOC(x)	((x) << S_FW_EQ_CTRL_CMD_ALLOC)
+#define F_FW_EQ_CTRL_CMD_ALLOC		V_FW_EQ_CTRL_CMD_ALLOC(1U)
+
+#define S_FW_EQ_CTRL_CMD_FREE		30
+#define V_FW_EQ_CTRL_CMD_FREE(x)	((x) << S_FW_EQ_CTRL_CMD_FREE)
+#define F_FW_EQ_CTRL_CMD_FREE		V_FW_EQ_CTRL_CMD_FREE(1U)
+
+#define S_FW_EQ_CTRL_CMD_EQSTART	28
+#define V_FW_EQ_CTRL_CMD_EQSTART(x)	((x) << S_FW_EQ_CTRL_CMD_EQSTART)
+#define F_FW_EQ_CTRL_CMD_EQSTART	V_FW_EQ_CTRL_CMD_EQSTART(1U)
+
+#define S_FW_EQ_CTRL_CMD_CMPLIQID	20
+#define V_FW_EQ_CTRL_CMD_CMPLIQID(x)	((x) << S_FW_EQ_CTRL_CMD_CMPLIQID)
+
+#define S_FW_EQ_CTRL_CMD_EQID		0
+#define M_FW_EQ_CTRL_CMD_EQID		0xfffff
+#define V_FW_EQ_CTRL_CMD_EQID(x)	((x) << S_FW_EQ_CTRL_CMD_EQID)
+#define G_FW_EQ_CTRL_CMD_EQID(x)	\
+	(((x) >> S_FW_EQ_CTRL_CMD_EQID) & M_FW_EQ_CTRL_CMD_EQID)
+
+#define S_FW_EQ_CTRL_CMD_PHYSEQID       0
+#define M_FW_EQ_CTRL_CMD_PHYSEQID       0xfffff
+#define V_FW_EQ_CTRL_CMD_PHYSEQID(x)    ((x) << S_FW_EQ_CTRL_CMD_PHYSEQID)
+#define G_FW_EQ_CTRL_CMD_PHYSEQID(x)    \
+	(((x) >> S_FW_EQ_CTRL_CMD_PHYSEQID) & M_FW_EQ_CTRL_CMD_PHYSEQID)
+
+#define S_FW_EQ_CTRL_CMD_FETCHRO	22
+#define V_FW_EQ_CTRL_CMD_FETCHRO(x)	((x) << S_FW_EQ_CTRL_CMD_FETCHRO)
+#define F_FW_EQ_CTRL_CMD_FETCHRO	V_FW_EQ_CTRL_CMD_FETCHRO(1U)
+
+#define S_FW_EQ_CTRL_CMD_HOSTFCMODE	20
+#define M_FW_EQ_CTRL_CMD_HOSTFCMODE	0x3
+#define V_FW_EQ_CTRL_CMD_HOSTFCMODE(x)	((x) << S_FW_EQ_CTRL_CMD_HOSTFCMODE)
+
+#define S_FW_EQ_CTRL_CMD_PCIECHN	16
+#define V_FW_EQ_CTRL_CMD_PCIECHN(x)	((x) << S_FW_EQ_CTRL_CMD_PCIECHN)
+
+#define S_FW_EQ_CTRL_CMD_IQID		0
+#define V_FW_EQ_CTRL_CMD_IQID(x)	((x) << S_FW_EQ_CTRL_CMD_IQID)
+
+#define S_FW_EQ_CTRL_CMD_FBMIN		23
+#define V_FW_EQ_CTRL_CMD_FBMIN(x)	((x) << S_FW_EQ_CTRL_CMD_FBMIN)
+
+#define S_FW_EQ_CTRL_CMD_FBMAX		20
+#define V_FW_EQ_CTRL_CMD_FBMAX(x)	((x) << S_FW_EQ_CTRL_CMD_FBMAX)
+
+#define S_FW_EQ_CTRL_CMD_CIDXFTHRESH	16
+#define V_FW_EQ_CTRL_CMD_CIDXFTHRESH(x)	((x) << S_FW_EQ_CTRL_CMD_CIDXFTHRESH)
+
+#define S_FW_EQ_CTRL_CMD_EQSIZE		0
+#define V_FW_EQ_CTRL_CMD_EQSIZE(x)	((x) << S_FW_EQ_CTRL_CMD_EQSIZE)
+
 enum fw_vi_func {
 	FW_VI_FUNC_ETH,
 };
@@ -1946,6 +2196,22 @@ struct fw_rss_vi_config_cmd {
 	(((x) >> S_FW_RSS_VI_CONFIG_CMD_UDPEN) & M_FW_RSS_VI_CONFIG_CMD_UDPEN)
 #define F_FW_RSS_VI_CONFIG_CMD_UDPEN	V_FW_RSS_VI_CONFIG_CMD_UDPEN(1U)
 
+struct fw_clip_cmd {
+	__be32 op_to_write;
+	__be32 alloc_to_len16;
+	__be64 ip_hi;
+	__be64 ip_lo;
+	__be32 r4[2];
+};
+
+#define S_FW_CLIP_CMD_ALLOC		31
+#define V_FW_CLIP_CMD_ALLOC(x)		((x) << S_FW_CLIP_CMD_ALLOC)
+#define F_FW_CLIP_CMD_ALLOC		V_FW_CLIP_CMD_ALLOC(1U)
+
+#define S_FW_CLIP_CMD_FREE		30
+#define V_FW_CLIP_CMD_FREE(x)		((x) << S_FW_CLIP_CMD_FREE)
+#define F_FW_CLIP_CMD_FREE		V_FW_CLIP_CMD_FREE(1U)
+
 /******************************************************************************
  *   D E B U G   C O M M A N D s
  ******************************************************/
diff --git a/drivers/net/cxgbe/base/t4vf_hw.c b/drivers/net/cxgbe/base/t4vf_hw.c
index 9fd0b879..d96456bb 100644
--- a/drivers/net/cxgbe/base/t4vf_hw.c
+++ b/drivers/net/cxgbe/base/t4vf_hw.c
@@ -683,6 +683,9 @@ static int t4vf_get_port_stats_fw(struct adapter *adapter, int pidx,
 	/*
 	 * Translate firmware statistics into host native statistics.
 	 */
+	p->tx_octets = be64_to_cpu(fwstats.tx_bcast_bytes) +
+		       be64_to_cpu(fwstats.tx_mcast_bytes) +
+		       be64_to_cpu(fwstats.tx_ucast_bytes);
 	p->tx_bcast_frames = be64_to_cpu(fwstats.tx_bcast_frames);
 	p->tx_mcast_frames = be64_to_cpu(fwstats.tx_mcast_frames);
 	p->tx_ucast_frames = be64_to_cpu(fwstats.tx_ucast_frames);
@@ -722,6 +725,9 @@ void t4vf_get_port_stats(struct adapter *adapter, int pidx,
 #define GET_STAT(name) \
 	t4_read_reg64(adapter, \
 			T4VF_MPS_BASE_ADDR + A_MPS_VF_STAT_##name##_L)
+	p->tx_octets = GET_STAT(TX_VF_BCAST_BYTES) +
+		       GET_STAT(TX_VF_MCAST_BYTES) +
+		       GET_STAT(TX_VF_UCAST_BYTES);
 	p->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES);
 	p->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES);
 	p->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES);
diff --git a/drivers/net/cxgbe/clip_tbl.c b/drivers/net/cxgbe/clip_tbl.c
new file mode 100644
index 00000000..5e4dc527
--- /dev/null
+++ b/drivers/net/cxgbe/clip_tbl.c
@@ -0,0 +1,193 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+
+#include "common.h"
+#include "clip_tbl.h"
+
+/**
+ * Allocate clip entry in HW with associated IPV4/IPv6 address
+ */
+static int clip6_get_mbox(const struct rte_eth_dev *dev, const u32 *lip)
+{
+	struct adapter *adap = ethdev2adap(dev);
+	struct fw_clip_cmd c;
+	u64 hi = ((u64)lip[1]) << 32 | lip[0];
+	u64 lo = ((u64)lip[3]) << 32 | lip[2];
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_CLIP_CMD) |
+				    F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
+	c.alloc_to_len16 = cpu_to_be32(F_FW_CLIP_CMD_ALLOC | FW_LEN16(c));
+	c.ip_hi = hi;
+	c.ip_lo = lo;
+	return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
+}
+
+/**
+ * Delete clip entry in HW having the associated IPV4/IPV6 address
+ */
+static int clip6_release_mbox(const struct rte_eth_dev *dev, const u32 *lip)
+{
+	struct adapter *adap = ethdev2adap(dev);
+	struct fw_clip_cmd c;
+	u64 hi = ((u64)lip[1]) << 32 | lip[0];
+	u64 lo = ((u64)lip[3]) << 32 | lip[2];
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_CLIP_CMD) |
+				    F_FW_CMD_REQUEST | F_FW_CMD_READ);
+	c.alloc_to_len16 = cpu_to_be32(F_FW_CLIP_CMD_FREE | FW_LEN16(c));
+	c.ip_hi = hi;
+	c.ip_lo = lo;
+	return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
+}
+
+/**
+ * cxgbe_clip_release - Release associated CLIP entry
+ * @ce: clip entry to release
+ *
+ * Releases ref count and frees up a clip entry from CLIP table
+ */
+void cxgbe_clip_release(struct rte_eth_dev *dev, struct clip_entry *ce)
+{
+	int ret;
+
+	t4_os_lock(&ce->lock);
+	if (rte_atomic32_dec_and_test(&ce->refcnt)) {
+		ret = clip6_release_mbox(dev, ce->addr);
+		if (ret)
+			dev_debug(adap, "CLIP FW DEL CMD failed: %d", ret);
+	}
+	t4_os_unlock(&ce->lock);
+}
+
+/**
+ * find_or_alloc_clipe - Find/Allocate a free CLIP entry
+ * @c: CLIP table
+ * @lip: IPV4/IPV6 address to compare/add
+ * Returns pointer to the IPV4/IPV6 entry found/created
+ *
+ * Finds/Allocates an CLIP entry to be used for a filter rule.
+ */
+static struct clip_entry *find_or_alloc_clipe(struct clip_tbl *c,
+					      const u32 *lip)
+{
+	struct clip_entry *end, *e;
+	struct clip_entry *first_free = NULL;
+	unsigned int clipt_size = c->clipt_size;
+
+	for (e = &c->cl_list[0], end = &c->cl_list[clipt_size]; e != end; ++e) {
+		if (rte_atomic32_read(&e->refcnt) == 0) {
+			if (!first_free)
+				first_free = e;
+		} else {
+			if (memcmp(lip, e->addr, sizeof(e->addr)) == 0)
+				goto exists;
+		}
+	}
+
+	if (first_free) {
+		e = first_free;
+		goto exists;
+	}
+
+	return NULL;
+
+exists:
+	return e;
+}
+
+static struct clip_entry *t4_clip_alloc(struct rte_eth_dev *dev,
+					u32 *lip, u8 v6)
+{
+	struct adapter *adap = ethdev2adap(dev);
+	struct clip_tbl *ctbl = adap->clipt;
+	struct clip_entry *ce;
+	int ret = 0;
+
+	if (!ctbl)
+		return NULL;
+
+	t4_os_write_lock(&ctbl->lock);
+	ce = find_or_alloc_clipe(ctbl, lip);
+	if (ce) {
+		t4_os_lock(&ce->lock);
+		if (!rte_atomic32_read(&ce->refcnt)) {
+			rte_memcpy(ce->addr, lip, sizeof(ce->addr));
+			if (v6) {
+				ce->type = FILTER_TYPE_IPV6;
+				rte_atomic32_set(&ce->refcnt, 1);
+				ret = clip6_get_mbox(dev, lip);
+				if (ret)
+					dev_debug(adap,
+						  "CLIP FW ADD CMD failed: %d",
+						  ret);
+			} else {
+				ce->type = FILTER_TYPE_IPV4;
+			}
+		} else {
+			rte_atomic32_inc(&ce->refcnt);
+		}
+		t4_os_unlock(&ce->lock);
+	}
+	t4_os_write_unlock(&ctbl->lock);
+
+	return ret ? NULL : ce;
+}
+
+/**
+ * cxgbe_clip_alloc - Allocate a IPV6 CLIP entry
+ * @dev: rte_eth_dev pointer
+ * @lip: IPV6 address to add
+ * Returns pointer to the CLIP entry created
+ *
+ * Allocates a IPV6 CLIP entry to be used for a filter rule.
+ */
+struct clip_entry *cxgbe_clip_alloc(struct rte_eth_dev *dev, u32 *lip)
+{
+	return t4_clip_alloc(dev, lip, FILTER_TYPE_IPV6);
+}
+
+/**
+ * Initialize CLIP Table
+ */
+struct clip_tbl *t4_init_clip_tbl(unsigned int clipt_start,
+				  unsigned int clipt_end)
+{
+	unsigned int clipt_size;
+	struct clip_tbl *ctbl;
+	unsigned int i;
+
+	if (clipt_start >= clipt_end)
+		return NULL;
+
+	clipt_size = clipt_end - clipt_start + 1;
+
+	ctbl = t4_os_alloc(sizeof(*ctbl) +
+			   clipt_size * sizeof(struct clip_entry));
+	if (!ctbl)
+		return NULL;
+
+	ctbl->clipt_start = clipt_start;
+	ctbl->clipt_size = clipt_size;
+
+	t4_os_rwlock_init(&ctbl->lock);
+
+	for (i = 0; i < ctbl->clipt_size; i++) {
+		t4_os_lock_init(&ctbl->cl_list[i].lock);
+		rte_atomic32_set(&ctbl->cl_list[i].refcnt, 0);
+	}
+
+	return ctbl;
+}
+
+/**
+ * Cleanup CLIP Table
+ */
+void t4_cleanup_clip_tbl(struct adapter *adap)
+{
+	if (adap->clipt)
+		t4_os_free(adap->clipt);
+}
diff --git a/drivers/net/cxgbe/clip_tbl.h b/drivers/net/cxgbe/clip_tbl.h
new file mode 100644
index 00000000..737ccc69
--- /dev/null
+++ b/drivers/net/cxgbe/clip_tbl.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+
+#ifndef _CXGBE_CLIP_H_
+#define _CXGBE_CLIP_H_
+
+/*
+ * State for the corresponding entry of the HW CLIP table.
+ */
+struct clip_entry {
+	enum filter_type type;       /* entry type */
+	u32 addr[4];                 /* IPV4 or IPV6 address */
+	rte_spinlock_t lock;         /* entry lock */
+	rte_atomic32_t refcnt;       /* entry reference count */
+};
+
+struct clip_tbl {
+	unsigned int clipt_start;     /* start index of CLIP table */
+	unsigned int clipt_size;      /* size of CLIP table */
+	rte_rwlock_t lock;            /* table rw lock */
+	struct clip_entry cl_list[0]; /* MUST BE LAST */
+};
+
+struct clip_tbl *t4_init_clip_tbl(unsigned int clipt_start,
+				  unsigned int clipt_end);
+void t4_cleanup_clip_tbl(struct adapter *adap);
+struct clip_entry *cxgbe_clip_alloc(struct rte_eth_dev *dev, u32 *lip);
+void cxgbe_clip_release(struct rte_eth_dev *dev, struct clip_entry *ce);
+#endif /* _CXGBE_CLIP_H_ */
diff --git a/drivers/net/cxgbe/cxgbe.h b/drivers/net/cxgbe/cxgbe.h
index e4a52560..5e6f5c98 100644
--- a/drivers/net/cxgbe/cxgbe.h
+++ b/drivers/net/cxgbe/cxgbe.h
@@ -18,12 +18,21 @@
 #define CXGBE_MIN_RX_BUFSIZE ETHER_MIN_MTU /* min buf size */
 #define CXGBE_MAX_RX_PKTLEN (9000 + ETHER_HDR_LEN + ETHER_CRC_LEN) /* max pkt */
 
+/* Max poll time is 100 * 100msec = 10 sec */
+#define CXGBE_LINK_STATUS_POLL_MS 100 /* 100ms */
+#define CXGBE_LINK_STATUS_POLL_CNT 100 /* Max number of times to poll */
+
 #define CXGBE_DEFAULT_RSS_KEY_LEN     40 /* 320-bits */
-#define CXGBE_RSS_HF_ALL (ETH_RSS_IPV4 | ETH_RSS_IPV6 | \
-			  ETH_RSS_NONFRAG_IPV4_TCP | \
-			  ETH_RSS_NONFRAG_IPV4_UDP | \
-			  ETH_RSS_NONFRAG_IPV6_TCP | \
-			  ETH_RSS_NONFRAG_IPV6_UDP)
+#define CXGBE_RSS_HF_IPV4_MASK (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | \
+				ETH_RSS_NONFRAG_IPV4_OTHER)
+#define CXGBE_RSS_HF_IPV6_MASK (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | \
+				ETH_RSS_NONFRAG_IPV6_OTHER | \
+				ETH_RSS_IPV6_EX)
+#define CXGBE_RSS_HF_TCP_IPV6_MASK (ETH_RSS_NONFRAG_IPV6_TCP | \
+				    ETH_RSS_IPV6_TCP_EX)
+#define CXGBE_RSS_HF_UDP_IPV6_MASK (ETH_RSS_NONFRAG_IPV6_UDP | \
+				    ETH_RSS_IPV6_UDP_EX)
+#define CXGBE_RSS_HF_ALL (ETH_RSS_IP | ETH_RSS_TCP | ETH_RSS_UDP)
 
 #define CXGBE_DEVARG_KEEP_OVLAN "keep_ovlan"
 #define CXGBE_DEVARG_FORCE_LINK_UP "force_link_up"
@@ -32,16 +41,20 @@ bool force_linkup(struct adapter *adap);
 int cxgbe_probe(struct adapter *adapter);
 int cxgbevf_probe(struct adapter *adapter);
 void cxgbe_get_speed_caps(struct port_info *pi, u32 *speed_caps);
+int cxgbe_set_link_status(struct port_info *pi, bool status);
 int cxgbe_up(struct adapter *adap);
 int cxgbe_down(struct port_info *pi);
 void cxgbe_close(struct adapter *adapter);
 void cxgbe_stats_get(struct port_info *pi, struct port_stats *stats);
 void cxgbevf_stats_get(struct port_info *pi, struct port_stats *stats);
 void cxgbe_stats_reset(struct port_info *pi);
+int cxgbe_poll_for_completion(struct sge_rspq *q, unsigned int us,
+			      unsigned int cnt, struct t4_completion *c);
 int link_start(struct port_info *pi);
 void init_rspq(struct adapter *adap, struct sge_rspq *q, unsigned int us,
 	       unsigned int cnt, unsigned int size, unsigned int iqe_size);
 int setup_sge_fwevtq(struct adapter *adapter);
+int setup_sge_ctrl_txq(struct adapter *adapter);
 void cfg_queues(struct rte_eth_dev *eth_dev);
 int cfg_queue_count(struct rte_eth_dev *eth_dev);
 int init_rss(struct adapter *adap);
@@ -50,5 +63,6 @@ void cxgbe_enable_rx_queues(struct port_info *pi);
 void print_port_info(struct adapter *adap);
 void print_adapter_info(struct adapter *adap);
 int cxgbe_get_devargs(struct rte_devargs *devargs, const char *key);
+void configure_max_ethqsets(struct adapter *adapter);
 
 #endif /* _CXGBE_H_ */
diff --git a/drivers/net/cxgbe/cxgbe_compat.h b/drivers/net/cxgbe/cxgbe_compat.h
index 779bcf16..5d47c5f3 100644
--- a/drivers/net/cxgbe/cxgbe_compat.h
+++ b/drivers/net/cxgbe/cxgbe_compat.h
@@ -198,15 +198,6 @@ static inline int cxgbe_fls(int x)
 	return x ? sizeof(x) * 8 - __builtin_clz(x) : 0;
 }
 
-/**
- * cxgbe_ffs - find first bit set
- * @x: the word to search
- */
-static inline int cxgbe_ffs(int x)
-{
-	return x ? __builtin_ffs(x) : 0;
-}
-
 static inline unsigned long ilog2(unsigned long n)
 {
 	unsigned int e = 0;
@@ -250,4 +241,16 @@ static inline void writel_relaxed(unsigned int val, volatile void __iomem *addr)
 	rte_write32_relaxed(val, addr);
 }
 
+/*
+ * Multiplies an integer by a fraction, while avoiding unnecessary
+ * overflow or loss of precision.
+ */
+#define mult_frac(x, numer, denom)(                     \
+{                                                       \
+	typeof(x) quot = (x) / (denom);                 \
+	typeof(x) rem  = (x) % (denom);                 \
+	(quot * (numer)) + ((rem * (numer)) / (denom)); \
+}                                                       \
+)
+
 #endif /* _CXGBE_COMPAT_H_ */
diff --git a/drivers/net/cxgbe/cxgbe_ethdev.c b/drivers/net/cxgbe/cxgbe_ethdev.c
index 61115e26..4dcad7a2 100644
--- a/drivers/net/cxgbe/cxgbe_ethdev.c
+++ b/drivers/net/cxgbe/cxgbe_ethdev.c
@@ -36,6 +36,7 @@
 
 #include "cxgbe.h"
 #include "cxgbe_pfvf.h"
+#include "cxgbe_flow.h"
 
 /*
  * Macros needed to support the PCI Device ID Table ...
@@ -199,24 +200,89 @@ void cxgbe_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 }
 
 int cxgbe_dev_link_update(struct rte_eth_dev *eth_dev,
-			  __rte_unused int wait_to_complete)
+			  int wait_to_complete)
 {
 	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 	struct adapter *adapter = pi->adapter;
 	struct sge *s = &adapter->sge;
-	struct rte_eth_link new_link;
-	unsigned int work_done, budget = 4;
+	struct rte_eth_link new_link = { 0 };
+	unsigned int i, work_done, budget = 32;
+	u8 old_link = pi->link_cfg.link_ok;
 
-	cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done);
+	for (i = 0; i < CXGBE_LINK_STATUS_POLL_CNT; i++) {
+		cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done);
+
+		/* Exit if link status changed or always forced up */
+		if (pi->link_cfg.link_ok != old_link || force_linkup(adapter))
+			break;
+
+		if (!wait_to_complete)
+			break;
+
+		rte_delay_ms(CXGBE_LINK_STATUS_POLL_MS);
+	}
 
 	new_link.link_status = force_linkup(adapter) ?
 			       ETH_LINK_UP : pi->link_cfg.link_ok;
+	new_link.link_autoneg = pi->link_cfg.autoneg;
 	new_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 	new_link.link_speed = pi->link_cfg.speed;
 
 	return rte_eth_linkstatus_set(eth_dev, &new_link);
 }
 
+/**
+ * Set device link up.
+ */
+int cxgbe_dev_set_link_up(struct rte_eth_dev *dev)
+{
+	struct port_info *pi = (struct port_info *)(dev->data->dev_private);
+	struct adapter *adapter = pi->adapter;
+	unsigned int work_done, budget = 32;
+	struct sge *s = &adapter->sge;
+	int ret;
+
+	/* Flush all link events */
+	cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done);
+
+	/* If link already up, nothing to do */
+	if (pi->link_cfg.link_ok)
+		return 0;
+
+	ret = cxgbe_set_link_status(pi, true);
+	if (ret)
+		return ret;
+
+	cxgbe_dev_link_update(dev, 1);
+	return 0;
+}
+
+/**
+ * Set device link down.
+ */
+int cxgbe_dev_set_link_down(struct rte_eth_dev *dev)
+{
+	struct port_info *pi = (struct port_info *)(dev->data->dev_private);
+	struct adapter *adapter = pi->adapter;
+	unsigned int work_done, budget = 32;
+	struct sge *s = &adapter->sge;
+	int ret;
+
+	/* Flush all link events */
+	cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done);
+
+	/* If link already down, nothing to do */
+	if (!pi->link_cfg.link_ok)
+		return 0;
+
+	ret = cxgbe_set_link_status(pi, false);
+	if (ret)
+		return ret;
+
+	cxgbe_dev_link_update(dev, 0);
+	return 0;
+}
+
 int cxgbe_dev_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu)
 {
 	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
@@ -352,7 +418,11 @@ int cxgbe_dev_configure(struct rte_eth_dev *eth_dev)
 
 	CXGBE_FUNC_TRACE();
 	configured_offloads = eth_dev->data->dev_conf.rxmode.offloads;
-	if (!(configured_offloads & DEV_RX_OFFLOAD_CRC_STRIP)) {
+
+	/* KEEP_CRC offload flag is not supported by PMD
+	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
+	 */
+	if (rte_eth_dev_must_keep_crc(configured_offloads)) {
 		dev_info(adapter, "can't disable hw crc strip\n");
 		eth_dev->data->dev_conf.rxmode.offloads |=
 			DEV_RX_OFFLOAD_CRC_STRIP;
@@ -363,6 +433,11 @@ int cxgbe_dev_configure(struct rte_eth_dev *eth_dev)
 		if (err)
 			return err;
 		adapter->flags |= FW_QUEUE_BOUND;
+		if (is_pf4(adapter)) {
+			err = setup_sge_ctrl_txq(adapter);
+			if (err)
+				return err;
+		}
 	}
 
 	err = cfg_queue_count(eth_dev);
@@ -794,13 +869,13 @@ static int cxgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 		return err;
 
 	if (flags & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) {
-		rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
+		rss_hf |= CXGBE_RSS_HF_TCP_IPV6_MASK;
 		if (flags & F_FW_RSS_VI_CONFIG_CMD_UDPEN)
-			rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
+			rss_hf |= CXGBE_RSS_HF_UDP_IPV6_MASK;
 	}
 
 	if (flags & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
-		rss_hf |= ETH_RSS_IPV6;
+		rss_hf |= CXGBE_RSS_HF_IPV6_MASK;
 
 	if (flags & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) {
 		rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
@@ -809,7 +884,7 @@ static int cxgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 	}
 
 	if (flags & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
-		rss_hf |= ETH_RSS_IPV4;
+		rss_hf |= CXGBE_RSS_HF_IPV4_MASK;
 
 	rss_conf->rss_hf = rss_hf;
 
@@ -1026,6 +1101,8 @@ static const struct eth_dev_ops cxgbe_eth_dev_ops = {
 	.dev_infos_get		= cxgbe_dev_info_get,
 	.dev_supported_ptypes_get = cxgbe_dev_supported_ptypes_get,
 	.link_update		= cxgbe_dev_link_update,
+	.dev_set_link_up        = cxgbe_dev_set_link_up,
+	.dev_set_link_down      = cxgbe_dev_set_link_down,
 	.mtu_set		= cxgbe_dev_mtu_set,
 	.tx_queue_setup         = cxgbe_dev_tx_queue_setup,
 	.tx_queue_start		= cxgbe_dev_tx_queue_start,
@@ -1035,6 +1112,7 @@ static const struct eth_dev_ops cxgbe_eth_dev_ops = {
 	.rx_queue_start		= cxgbe_dev_rx_queue_start,
 	.rx_queue_stop		= cxgbe_dev_rx_queue_stop,
 	.rx_queue_release	= cxgbe_dev_rx_queue_release,
+	.filter_ctrl            = cxgbe_dev_filter_ctrl,
 	.stats_get		= cxgbe_dev_stats_get,
 	.stats_reset		= cxgbe_dev_stats_reset,
 	.flow_ctrl_get		= cxgbe_flow_ctrl_get,
diff --git a/drivers/net/cxgbe/cxgbe_filter.c b/drivers/net/cxgbe/cxgbe_filter.c
new file mode 100644
index 00000000..7f0d3800
--- /dev/null
+++ b/drivers/net/cxgbe/cxgbe_filter.c
@@ -0,0 +1,1252 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+#include <rte_net.h>
+#include "common.h"
+#include "t4_tcb.h"
+#include "t4_regs.h"
+#include "cxgbe_filter.h"
+#include "clip_tbl.h"
+
+/**
+ * Initialize Hash Filters
+ */
+int init_hash_filter(struct adapter *adap)
+{
+	unsigned int n_user_filters;
+	unsigned int user_filter_perc;
+	int ret;
+	u32 params[7], val[7];
+
+#define FW_PARAM_DEV(param) \
+	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
+	V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
+
+#define FW_PARAM_PFVF(param) \
+	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
+	V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) |  \
+	V_FW_PARAMS_PARAM_Y(0) | \
+	V_FW_PARAMS_PARAM_Z(0))
+
+	params[0] = FW_PARAM_DEV(NTID);
+	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1,
+			      params, val);
+	if (ret < 0)
+		return ret;
+	adap->tids.ntids = val[0];
+	adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
+
+	user_filter_perc = 100;
+	n_user_filters = mult_frac(adap->tids.nftids,
+				   user_filter_perc,
+				   100);
+
+	adap->tids.nftids = n_user_filters;
+	adap->params.hash_filter = 1;
+	return 0;
+}
+
+/**
+ * Validate if the requested filter specification can be set by checking
+ * if the requested features have been enabled
+ */
+int validate_filter(struct adapter *adapter, struct ch_filter_specification *fs)
+{
+	u32 fconf;
+
+	/*
+	 * Check for unconfigured fields being used.
+	 */
+	fconf = adapter->params.tp.vlan_pri_map;
+
+#define S(_field) \
+	(fs->val._field || fs->mask._field)
+#define U(_mask, _field) \
+	(!(fconf & (_mask)) && S(_field))
+
+	if (U(F_PORT, iport) || U(F_ETHERTYPE, ethtype) || U(F_PROTOCOL, proto))
+		return -EOPNOTSUPP;
+
+#undef S
+#undef U
+
+	/*
+	 * If the user is requesting that the filter action loop
+	 * matching packets back out one of our ports, make sure that
+	 * the egress port is in range.
+	 */
+	if (fs->action == FILTER_SWITCH &&
+	    fs->eport >= adapter->params.nports)
+		return -ERANGE;
+
+	/*
+	 * Don't allow various trivially obvious bogus out-of-range
+	 * values ...
+	 */
+	if (fs->val.iport >= adapter->params.nports)
+		return -ERANGE;
+
+	return 0;
+}
+
+/**
+ * Get the queue to which the traffic must be steered to.
+ */
+static unsigned int get_filter_steerq(struct rte_eth_dev *dev,
+				      struct ch_filter_specification *fs)
+{
+	struct port_info *pi = ethdev2pinfo(dev);
+	struct adapter *adapter = pi->adapter;
+	unsigned int iq;
+
+	/*
+	 * If the user has requested steering matching Ingress Packets
+	 * to a specific Queue Set, we need to make sure it's in range
+	 * for the port and map that into the Absolute Queue ID of the
+	 * Queue Set's Response Queue.
+	 */
+	if (!fs->dirsteer) {
+		iq = 0;
+	} else {
+		/*
+		 * If the iq id is greater than the number of qsets,
+		 * then assume it is an absolute qid.
+		 */
+		if (fs->iq < pi->n_rx_qsets)
+			iq = adapter->sge.ethrxq[pi->first_qset +
+						 fs->iq].rspq.abs_id;
+		else
+			iq = fs->iq;
+	}
+
+	return iq;
+}
+
+/* Return an error number if the indicated filter isn't writable ... */
+int writable_filter(struct filter_entry *f)
+{
+	if (f->locked)
+		return -EPERM;
+	if (f->pending)
+		return -EBUSY;
+
+	return 0;
+}
+
+/**
+ * Send CPL_SET_TCB_FIELD message
+ */
+static void set_tcb_field(struct adapter *adapter, unsigned int ftid,
+			  u16 word, u64 mask, u64 val, int no_reply)
+{
+	struct rte_mbuf *mbuf;
+	struct cpl_set_tcb_field *req;
+	struct sge_ctrl_txq *ctrlq;
+
+	ctrlq = &adapter->sge.ctrlq[0];
+	mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool);
+	WARN_ON(!mbuf);
+
+	mbuf->data_len = sizeof(*req);
+	mbuf->pkt_len = mbuf->data_len;
+
+	req = rte_pktmbuf_mtod(mbuf, struct cpl_set_tcb_field *);
+	memset(req, 0, sizeof(*req));
+	INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, ftid);
+	req->reply_ctrl = cpu_to_be16(V_REPLY_CHAN(0) |
+				      V_QUEUENO(adapter->sge.fw_evtq.abs_id) |
+				      V_NO_REPLY(no_reply));
+	req->word_cookie = cpu_to_be16(V_WORD(word) | V_COOKIE(ftid));
+	req->mask = cpu_to_be64(mask);
+	req->val = cpu_to_be64(val);
+
+	t4_mgmt_tx(ctrlq, mbuf);
+}
+
+/**
+ * Build a CPL_SET_TCB_FIELD message as payload of a ULP_TX_PKT command.
+ */
+static inline void mk_set_tcb_field_ulp(struct filter_entry *f,
+					struct cpl_set_tcb_field *req,
+					unsigned int word,
+					u64 mask, u64 val, u8 cookie,
+					int no_reply)
+{
+	struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
+	struct ulptx_idata *sc = (struct ulptx_idata *)(txpkt + 1);
+
+	txpkt->cmd_dest = cpu_to_be32(V_ULPTX_CMD(ULP_TX_PKT) |
+				      V_ULP_TXPKT_DEST(0));
+	txpkt->len = cpu_to_be32(DIV_ROUND_UP(sizeof(*req), 16));
+	sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_IMM));
+	sc->len = cpu_to_be32(sizeof(*req) - sizeof(struct work_request_hdr));
+	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, f->tid));
+	req->reply_ctrl = cpu_to_be16(V_NO_REPLY(no_reply) | V_REPLY_CHAN(0) |
+				      V_QUEUENO(0));
+	req->word_cookie = cpu_to_be16(V_WORD(word) | V_COOKIE(cookie));
+	req->mask = cpu_to_be64(mask);
+	req->val = cpu_to_be64(val);
+	sc = (struct ulptx_idata *)(req + 1);
+	sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
+	sc->len = cpu_to_be32(0);
+}
+
+/**
+ * Check if entry already filled.
+ */
+bool is_filter_set(struct tid_info *t, int fidx, int family)
+{
+	bool result = FALSE;
+	int i, max;
+
+	/* IPv6 requires four slots and IPv4 requires only 1 slot.
+	 * Ensure, there's enough slots available.
+	 */
+	max = family == FILTER_TYPE_IPV6 ? fidx + 3 : fidx;
+
+	t4_os_lock(&t->ftid_lock);
+	for (i = fidx; i <= max; i++) {
+		if (rte_bitmap_get(t->ftid_bmap, i)) {
+			result = TRUE;
+			break;
+		}
+	}
+	t4_os_unlock(&t->ftid_lock);
+	return result;
+}
+
+/**
+ * Allocate a available free entry
+ */
+int cxgbe_alloc_ftid(struct adapter *adap, unsigned int family)
+{
+	struct tid_info *t = &adap->tids;
+	int pos;
+	int size = t->nftids;
+
+	t4_os_lock(&t->ftid_lock);
+	if (family == FILTER_TYPE_IPV6)
+		pos = cxgbe_bitmap_find_free_region(t->ftid_bmap, size, 4);
+	else
+		pos = cxgbe_find_first_zero_bit(t->ftid_bmap, size);
+	t4_os_unlock(&t->ftid_lock);
+
+	return pos < size ? pos : -1;
+}
+
+/**
+ * Construct hash filter ntuple.
+ */
+static u64 hash_filter_ntuple(const struct filter_entry *f)
+{
+	struct adapter *adap = ethdev2adap(f->dev);
+	struct tp_params *tp = &adap->params.tp;
+	u64 ntuple = 0;
+	u16 tcp_proto = IPPROTO_TCP; /* TCP Protocol Number */
+
+	if (tp->port_shift >= 0)
+		ntuple |= (u64)f->fs.mask.iport << tp->port_shift;
+
+	if (tp->protocol_shift >= 0) {
+		if (!f->fs.val.proto)
+			ntuple |= (u64)tcp_proto << tp->protocol_shift;
+		else
+			ntuple |= (u64)f->fs.val.proto << tp->protocol_shift;
+	}
+
+	if (tp->ethertype_shift >= 0 && f->fs.mask.ethtype)
+		ntuple |= (u64)(f->fs.val.ethtype) << tp->ethertype_shift;
+
+	if (ntuple != tp->hash_filter_mask)
+		return 0;
+
+	return ntuple;
+}
+
+/**
+ * Build a CPL_ABORT_REQ message as payload of a ULP_TX_PKT command.
+ */
+static void mk_abort_req_ulp(struct cpl_abort_req *abort_req,
+			     unsigned int tid)
+{
+	struct ulp_txpkt *txpkt = (struct ulp_txpkt *)abort_req;
+	struct ulptx_idata *sc = (struct ulptx_idata *)(txpkt + 1);
+
+	txpkt->cmd_dest = cpu_to_be32(V_ULPTX_CMD(ULP_TX_PKT) |
+				      V_ULP_TXPKT_DEST(0));
+	txpkt->len = cpu_to_be32(DIV_ROUND_UP(sizeof(*abort_req), 16));
+	sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_IMM));
+	sc->len = cpu_to_be32(sizeof(*abort_req) -
+			      sizeof(struct work_request_hdr));
+	OPCODE_TID(abort_req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
+	abort_req->rsvd0 = cpu_to_be32(0);
+	abort_req->rsvd1 = 0;
+	abort_req->cmd = CPL_ABORT_NO_RST;
+	sc = (struct ulptx_idata *)(abort_req + 1);
+	sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
+	sc->len = cpu_to_be32(0);
+}
+
+/**
+ * Build a CPL_ABORT_RPL message as payload of a ULP_TX_PKT command.
+ */
+static void mk_abort_rpl_ulp(struct cpl_abort_rpl *abort_rpl,
+			     unsigned int tid)
+{
+	struct ulp_txpkt *txpkt = (struct ulp_txpkt *)abort_rpl;
+	struct ulptx_idata *sc = (struct ulptx_idata *)(txpkt + 1);
+
+	txpkt->cmd_dest = cpu_to_be32(V_ULPTX_CMD(ULP_TX_PKT) |
+				      V_ULP_TXPKT_DEST(0));
+	txpkt->len = cpu_to_be32(DIV_ROUND_UP(sizeof(*abort_rpl), 16));
+	sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_IMM));
+	sc->len = cpu_to_be32(sizeof(*abort_rpl) -
+			      sizeof(struct work_request_hdr));
+	OPCODE_TID(abort_rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
+	abort_rpl->rsvd0 = cpu_to_be32(0);
+	abort_rpl->rsvd1 = 0;
+	abort_rpl->cmd = CPL_ABORT_NO_RST;
+	sc = (struct ulptx_idata *)(abort_rpl + 1);
+	sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
+	sc->len = cpu_to_be32(0);
+}
+
+/**
+ * Delete the specified hash filter.
+ */
+static int cxgbe_del_hash_filter(struct rte_eth_dev *dev,
+				 unsigned int filter_id,
+				 struct filter_ctx *ctx)
+{
+	struct adapter *adapter = ethdev2adap(dev);
+	struct tid_info *t = &adapter->tids;
+	struct filter_entry *f;
+	struct sge_ctrl_txq *ctrlq;
+	unsigned int port_id = ethdev2pinfo(dev)->port_id;
+	int ret;
+
+	if (filter_id > adapter->tids.ntids)
+		return -E2BIG;
+
+	f = lookup_tid(t, filter_id);
+	if (!f) {
+		dev_err(adapter, "%s: no filter entry for filter_id = %d\n",
+			__func__, filter_id);
+		return -EINVAL;
+	}
+
+	ret = writable_filter(f);
+	if (ret)
+		return ret;
+
+	if (f->valid) {
+		unsigned int wrlen;
+		struct rte_mbuf *mbuf;
+		struct work_request_hdr *wr;
+		struct ulptx_idata *aligner;
+		struct cpl_set_tcb_field *req;
+		struct cpl_abort_req *abort_req;
+		struct cpl_abort_rpl *abort_rpl;
+
+		f->ctx = ctx;
+		f->pending = 1;
+
+		wrlen = cxgbe_roundup(sizeof(*wr) +
+				      (sizeof(*req) + sizeof(*aligner)) +
+				      sizeof(*abort_req) + sizeof(*abort_rpl),
+				      16);
+
+		ctrlq = &adapter->sge.ctrlq[port_id];
+		mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool);
+		if (!mbuf) {
+			dev_err(adapter, "%s: could not allocate skb ..\n",
+				__func__);
+			goto out_err;
+		}
+
+		mbuf->data_len = wrlen;
+		mbuf->pkt_len = mbuf->data_len;
+
+		req = rte_pktmbuf_mtod(mbuf, struct cpl_set_tcb_field *);
+		INIT_ULPTX_WR(req, wrlen, 0, 0);
+		wr = (struct work_request_hdr *)req;
+		wr++;
+		req = (struct cpl_set_tcb_field *)wr;
+		mk_set_tcb_field_ulp(f, req, W_TCB_RSS_INFO,
+				V_TCB_RSS_INFO(M_TCB_RSS_INFO),
+				V_TCB_RSS_INFO(adapter->sge.fw_evtq.abs_id),
+				0, 1);
+		aligner = (struct ulptx_idata *)(req + 1);
+		abort_req = (struct cpl_abort_req *)(aligner + 1);
+		mk_abort_req_ulp(abort_req, f->tid);
+		abort_rpl = (struct cpl_abort_rpl *)(abort_req + 1);
+		mk_abort_rpl_ulp(abort_rpl, f->tid);
+		t4_mgmt_tx(ctrlq, mbuf);
+	}
+	return 0;
+
+out_err:
+	return -ENOMEM;
+}
+
+/**
+ * Build a ACT_OPEN_REQ6 message for setting IPv6 hash filter.
+ */
+static void mk_act_open_req6(struct filter_entry *f, struct rte_mbuf *mbuf,
+			     unsigned int qid_filterid, struct adapter *adap)
+{
+	struct cpl_t6_act_open_req6 *req = NULL;
+	u64 local_lo, local_hi, peer_lo, peer_hi;
+	u32 *lip = (u32 *)f->fs.val.lip;
+	u32 *fip = (u32 *)f->fs.val.fip;
+
+	switch (CHELSIO_CHIP_VERSION(adap->params.chip)) {
+	case CHELSIO_T6:
+		req = rte_pktmbuf_mtod(mbuf, struct cpl_t6_act_open_req6 *);
+
+		INIT_TP_WR(req, 0);
+		break;
+	default:
+		dev_err(adap, "%s: unsupported chip type!\n", __func__);
+		return;
+	}
+
+	local_hi = ((u64)lip[1]) << 32 | lip[0];
+	local_lo = ((u64)lip[3]) << 32 | lip[2];
+	peer_hi = ((u64)fip[1]) << 32 | fip[0];
+	peer_lo = ((u64)fip[3]) << 32 | fip[2];
+
+	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
+						    qid_filterid));
+	req->local_port = cpu_to_be16(f->fs.val.lport);
+	req->peer_port = cpu_to_be16(f->fs.val.fport);
+	req->local_ip_hi = local_hi;
+	req->local_ip_lo = local_lo;
+	req->peer_ip_hi = peer_hi;
+	req->peer_ip_lo = peer_lo;
+	req->opt0 = cpu_to_be64(V_DELACK(f->fs.hitcnts) |
+				V_SMAC_SEL((cxgbe_port_viid(f->dev) & 0x7F)
+					   << 1) |
+				V_TX_CHAN(f->fs.eport) |
+				V_ULP_MODE(ULP_MODE_NONE) |
+				F_TCAM_BYPASS | F_NON_OFFLOAD);
+	req->params = cpu_to_be64(V_FILTER_TUPLE(hash_filter_ntuple(f)));
+	req->opt2 = cpu_to_be32(F_RSS_QUEUE_VALID |
+			    V_RSS_QUEUE(f->fs.iq) |
+			    F_T5_OPT_2_VALID |
+			    F_RX_CHANNEL |
+			    V_CONG_CNTRL((f->fs.action == FILTER_DROP) |
+					 (f->fs.dirsteer << 1)) |
+			    V_CCTRL_ECN(f->fs.action == FILTER_SWITCH));
+}
+
+/**
+ * Build a ACT_OPEN_REQ message for setting IPv4 hash filter.
+ */
+static void mk_act_open_req(struct filter_entry *f, struct rte_mbuf *mbuf,
+			    unsigned int qid_filterid, struct adapter *adap)
+{
+	struct cpl_t6_act_open_req *req = NULL;
+
+	switch (CHELSIO_CHIP_VERSION(adap->params.chip)) {
+	case CHELSIO_T6:
+		req = rte_pktmbuf_mtod(mbuf, struct cpl_t6_act_open_req *);
+
+		INIT_TP_WR(req, 0);
+		break;
+	default:
+		dev_err(adap, "%s: unsupported chip type!\n", __func__);
+		return;
+	}
+
+	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
+						    qid_filterid));
+	req->local_port = cpu_to_be16(f->fs.val.lport);
+	req->peer_port = cpu_to_be16(f->fs.val.fport);
+	req->local_ip = f->fs.val.lip[0] | f->fs.val.lip[1] << 8 |
+			f->fs.val.lip[2] << 16 | f->fs.val.lip[3] << 24;
+	req->peer_ip = f->fs.val.fip[0] | f->fs.val.fip[1] << 8 |
+			f->fs.val.fip[2] << 16 | f->fs.val.fip[3] << 24;
+	req->opt0 = cpu_to_be64(V_DELACK(f->fs.hitcnts) |
+				V_SMAC_SEL((cxgbe_port_viid(f->dev) & 0x7F)
+					   << 1) |
+				V_TX_CHAN(f->fs.eport) |
+				V_ULP_MODE(ULP_MODE_NONE) |
+				F_TCAM_BYPASS | F_NON_OFFLOAD);
+	req->params = cpu_to_be64(V_FILTER_TUPLE(hash_filter_ntuple(f)));
+	req->opt2 = cpu_to_be32(F_RSS_QUEUE_VALID |
+			    V_RSS_QUEUE(f->fs.iq) |
+			    F_T5_OPT_2_VALID |
+			    F_RX_CHANNEL |
+			    V_CONG_CNTRL((f->fs.action == FILTER_DROP) |
+					 (f->fs.dirsteer << 1)) |
+			    V_CCTRL_ECN(f->fs.action == FILTER_SWITCH));
+}
+
+/**
+ * Set the specified hash filter.
+ */
+static int cxgbe_set_hash_filter(struct rte_eth_dev *dev,
+				 struct ch_filter_specification *fs,
+				 struct filter_ctx *ctx)
+{
+	struct port_info *pi = ethdev2pinfo(dev);
+	struct adapter *adapter = pi->adapter;
+	struct tid_info *t = &adapter->tids;
+	struct filter_entry *f;
+	struct rte_mbuf *mbuf;
+	struct sge_ctrl_txq *ctrlq;
+	unsigned int iq;
+	int atid, size;
+	int ret = 0;
+
+	ret = validate_filter(adapter, fs);
+	if (ret)
+		return ret;
+
+	iq = get_filter_steerq(dev, fs);
+
+	ctrlq = &adapter->sge.ctrlq[pi->port_id];
+
+	f = t4_os_alloc(sizeof(*f));
+	if (!f)
+		goto out_err;
+
+	f->fs = *fs;
+	f->ctx = ctx;
+	f->dev = dev;
+	f->fs.iq = iq;
+
+	atid = cxgbe_alloc_atid(t, f);
+	if (atid < 0)
+		goto out_err;
+
+	if (f->fs.type) {
+		/* IPv6 hash filter */
+		f->clipt = cxgbe_clip_alloc(f->dev, (u32 *)&f->fs.val.lip);
+		if (!f->clipt)
+			goto free_atid;
+
+		size = sizeof(struct cpl_t6_act_open_req6);
+		mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool);
+		if (!mbuf) {
+			ret = -ENOMEM;
+			goto free_clip;
+		}
+
+		mbuf->data_len = size;
+		mbuf->pkt_len = mbuf->data_len;
+
+		mk_act_open_req6(f, mbuf,
+				 ((adapter->sge.fw_evtq.abs_id << 14) | atid),
+				 adapter);
+	} else {
+		/* IPv4 hash filter */
+		size = sizeof(struct cpl_t6_act_open_req);
+		mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool);
+		if (!mbuf) {
+			ret = -ENOMEM;
+			goto free_atid;
+		}
+
+		mbuf->data_len = size;
+		mbuf->pkt_len = mbuf->data_len;
+
+		mk_act_open_req(f, mbuf,
+				((adapter->sge.fw_evtq.abs_id << 14) | atid),
+				adapter);
+	}
+
+	f->pending = 1;
+	t4_mgmt_tx(ctrlq, mbuf);
+	return 0;
+
+free_clip:
+	cxgbe_clip_release(f->dev, f->clipt);
+free_atid:
+	cxgbe_free_atid(t, atid);
+
+out_err:
+	t4_os_free(f);
+	return ret;
+}
+
+/**
+ * Clear a filter and release any of its resources that we own.  This also
+ * clears the filter's "pending" status.
+ */
+void clear_filter(struct filter_entry *f)
+{
+	if (f->clipt)
+		cxgbe_clip_release(f->dev, f->clipt);
+
+	/*
+	 * The zeroing of the filter rule below clears the filter valid,
+	 * pending, locked flags etc. so it's all we need for
+	 * this operation.
+	 */
+	memset(f, 0, sizeof(*f));
+}
+
+/**
+ * t4_mk_filtdelwr - create a delete filter WR
+ * @ftid: the filter ID
+ * @wr: the filter work request to populate
+ * @qid: ingress queue to receive the delete notification
+ *
+ * Creates a filter work request to delete the supplied filter.  If @qid is
+ * negative the delete notification is suppressed.
+ */
+static void t4_mk_filtdelwr(unsigned int ftid, struct fw_filter_wr *wr, int qid)
+{
+	memset(wr, 0, sizeof(*wr));
+	wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER_WR));
+	wr->len16_pkd = cpu_to_be32(V_FW_WR_LEN16(sizeof(*wr) / 16));
+	wr->tid_to_iq = cpu_to_be32(V_FW_FILTER_WR_TID(ftid) |
+				    V_FW_FILTER_WR_NOREPLY(qid < 0));
+	wr->del_filter_to_l2tix = cpu_to_be32(F_FW_FILTER_WR_DEL_FILTER);
+	if (qid >= 0)
+		wr->rx_chan_rx_rpl_iq =
+				cpu_to_be16(V_FW_FILTER_WR_RX_RPL_IQ(qid));
+}
+
+/**
+ * Create FW work request to delete the filter at a specified index
+ */
+static int del_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
+{
+	struct adapter *adapter = ethdev2adap(dev);
+	struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
+	struct rte_mbuf *mbuf;
+	struct fw_filter_wr *fwr;
+	struct sge_ctrl_txq *ctrlq;
+	unsigned int port_id = ethdev2pinfo(dev)->port_id;
+
+	ctrlq = &adapter->sge.ctrlq[port_id];
+	mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool);
+	if (!mbuf)
+		return -ENOMEM;
+
+	mbuf->data_len = sizeof(*fwr);
+	mbuf->pkt_len = mbuf->data_len;
+
+	fwr = rte_pktmbuf_mtod(mbuf, struct fw_filter_wr *);
+	t4_mk_filtdelwr(f->tid, fwr, adapter->sge.fw_evtq.abs_id);
+
+	/*
+	 * Mark the filter as "pending" and ship off the Filter Work Request.
+	 * When we get the Work Request Reply we'll clear the pending status.
+	 */
+	f->pending = 1;
+	t4_mgmt_tx(ctrlq, mbuf);
+	return 0;
+}
+
+int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx)
+{
+	struct adapter *adapter = ethdev2adap(dev);
+	struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
+	struct rte_mbuf *mbuf;
+	struct fw_filter_wr *fwr;
+	struct sge_ctrl_txq *ctrlq;
+	unsigned int port_id = ethdev2pinfo(dev)->port_id;
+	int ret;
+
+	ctrlq = &adapter->sge.ctrlq[port_id];
+	mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool);
+	if (!mbuf) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	mbuf->data_len = sizeof(*fwr);
+	mbuf->pkt_len = mbuf->data_len;
+
+	fwr = rte_pktmbuf_mtod(mbuf, struct fw_filter_wr *);
+	memset(fwr, 0, sizeof(*fwr));
+
+	/*
+	 * Construct the work request to set the filter.
+	 */
+	fwr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_FILTER_WR));
+	fwr->len16_pkd = cpu_to_be32(V_FW_WR_LEN16(sizeof(*fwr) / 16));
+	fwr->tid_to_iq =
+		cpu_to_be32(V_FW_FILTER_WR_TID(f->tid) |
+			    V_FW_FILTER_WR_RQTYPE(f->fs.type) |
+			    V_FW_FILTER_WR_NOREPLY(0) |
+			    V_FW_FILTER_WR_IQ(f->fs.iq));
+	fwr->del_filter_to_l2tix =
+		cpu_to_be32(V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
+			    V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
+			    V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
+			    V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
+			    V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
+			    V_FW_FILTER_WR_PRIO(f->fs.prio));
+	fwr->ethtype = cpu_to_be16(f->fs.val.ethtype);
+	fwr->ethtypem = cpu_to_be16(f->fs.mask.ethtype);
+	fwr->smac_sel = 0;
+	fwr->rx_chan_rx_rpl_iq =
+		cpu_to_be16(V_FW_FILTER_WR_RX_CHAN(0) |
+			    V_FW_FILTER_WR_RX_RPL_IQ(adapter->sge.fw_evtq.abs_id
+						     ));
+	fwr->maci_to_matchtypem =
+		cpu_to_be32(V_FW_FILTER_WR_PORT(f->fs.val.iport) |
+			    V_FW_FILTER_WR_PORTM(f->fs.mask.iport));
+	fwr->ptcl = f->fs.val.proto;
+	fwr->ptclm = f->fs.mask.proto;
+	rte_memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
+	rte_memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
+	rte_memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
+	rte_memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
+	fwr->lp = cpu_to_be16(f->fs.val.lport);
+	fwr->lpm = cpu_to_be16(f->fs.mask.lport);
+	fwr->fp = cpu_to_be16(f->fs.val.fport);
+	fwr->fpm = cpu_to_be16(f->fs.mask.fport);
+
+	/*
+	 * Mark the filter as "pending" and ship off the Filter Work Request.
+	 * When we get the Work Request Reply we'll clear the pending status.
+	 */
+	f->pending = 1;
+	t4_mgmt_tx(ctrlq, mbuf);
+	return 0;
+
+out:
+	return ret;
+}
+
+/**
+ * Set the corresponding entry in the bitmap. 4 slots are
+ * marked for IPv6, whereas only 1 slot is marked for IPv4.
+ */
+static int cxgbe_set_ftid(struct tid_info *t, int fidx, int family)
+{
+	t4_os_lock(&t->ftid_lock);
+	if (rte_bitmap_get(t->ftid_bmap, fidx)) {
+		t4_os_unlock(&t->ftid_lock);
+		return -EBUSY;
+	}
+
+	if (family == FILTER_TYPE_IPV4) {
+		rte_bitmap_set(t->ftid_bmap, fidx);
+	} else {
+		rte_bitmap_set(t->ftid_bmap, fidx);
+		rte_bitmap_set(t->ftid_bmap, fidx + 1);
+		rte_bitmap_set(t->ftid_bmap, fidx + 2);
+		rte_bitmap_set(t->ftid_bmap, fidx + 3);
+	}
+	t4_os_unlock(&t->ftid_lock);
+	return 0;
+}
+
+/**
+ * Clear the corresponding entry in the bitmap. 4 slots are
+ * cleared for IPv6, whereas only 1 slot is cleared for IPv4.
+ */
+static void cxgbe_clear_ftid(struct tid_info *t, int fidx, int family)
+{
+	t4_os_lock(&t->ftid_lock);
+	if (family == FILTER_TYPE_IPV4) {
+		rte_bitmap_clear(t->ftid_bmap, fidx);
+	} else {
+		rte_bitmap_clear(t->ftid_bmap, fidx);
+		rte_bitmap_clear(t->ftid_bmap, fidx + 1);
+		rte_bitmap_clear(t->ftid_bmap, fidx + 2);
+		rte_bitmap_clear(t->ftid_bmap, fidx + 3);
+	}
+	t4_os_unlock(&t->ftid_lock);
+}
+
+/**
+ * Check a delete filter request for validity and send it to the hardware.
+ * Return 0 on success, an error number otherwise.  We attach any provided
+ * filter operation context to the internal filter specification in order to
+ * facilitate signaling completion of the operation.
+ */
+int cxgbe_del_filter(struct rte_eth_dev *dev, unsigned int filter_id,
+		     struct ch_filter_specification *fs,
+		     struct filter_ctx *ctx)
+{
+	struct port_info *pi = (struct port_info *)(dev->data->dev_private);
+	struct adapter *adapter = pi->adapter;
+	struct filter_entry *f;
+	unsigned int chip_ver;
+	int ret;
+
+	if (is_hashfilter(adapter) && fs->cap)
+		return cxgbe_del_hash_filter(dev, filter_id, ctx);
+
+	if (filter_id >= adapter->tids.nftids)
+		return -ERANGE;
+
+	chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
+
+	ret = is_filter_set(&adapter->tids, filter_id, fs->type);
+	if (!ret) {
+		dev_warn(adap, "%s: could not find filter entry: %u\n",
+			 __func__, filter_id);
+		return -EINVAL;
+	}
+
+	/*
+	 * Ensure filter id is aligned on the 2 slot boundary for T6,
+	 * and 4 slot boundary for cards below T6.
+	 */
+	if (fs->type) {
+		if (chip_ver < CHELSIO_T6)
+			filter_id &= ~(0x3);
+		else
+			filter_id &= ~(0x1);
+	}
+
+	f = &adapter->tids.ftid_tab[filter_id];
+	ret = writable_filter(f);
+	if (ret)
+		return ret;
+
+	if (f->valid) {
+		f->ctx = ctx;
+		cxgbe_clear_ftid(&adapter->tids,
+				 f->tid - adapter->tids.ftid_base,
+				 f->fs.type ? FILTER_TYPE_IPV6 :
+					      FILTER_TYPE_IPV4);
+		return del_filter_wr(dev, filter_id);
+	}
+
+	/*
+	 * If the caller has passed in a Completion Context then we need to
+	 * mark it as a successful completion so they don't stall waiting
+	 * for it.
+	 */
+	if (ctx) {
+		ctx->result = 0;
+		t4_complete(&ctx->completion);
+	}
+
+	return 0;
+}
+
+/**
+ * Check a Chelsio Filter Request for validity, convert it into our internal
+ * format and send it to the hardware.  Return 0 on success, an error number
+ * otherwise.  We attach any provided filter operation context to the internal
+ * filter specification in order to facilitate signaling completion of the
+ * operation.
+ */
+int cxgbe_set_filter(struct rte_eth_dev *dev, unsigned int filter_id,
+		     struct ch_filter_specification *fs,
+		     struct filter_ctx *ctx)
+{
+	struct port_info *pi = ethdev2pinfo(dev);
+	struct adapter *adapter = pi->adapter;
+	unsigned int fidx, iq, fid_bit = 0;
+	struct filter_entry *f;
+	unsigned int chip_ver;
+	uint8_t bitoff[16] = {0};
+	int ret;
+
+	if (is_hashfilter(adapter) && fs->cap)
+		return cxgbe_set_hash_filter(dev, fs, ctx);
+
+	if (filter_id >= adapter->tids.nftids)
+		return -ERANGE;
+
+	chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
+
+	ret = validate_filter(adapter, fs);
+	if (ret)
+		return ret;
+
+	/*
+	 * Ensure filter id is aligned on the 4 slot boundary for IPv6
+	 * maskfull filters.
+	 */
+	if (fs->type)
+		filter_id &= ~(0x3);
+
+	ret = is_filter_set(&adapter->tids, filter_id, fs->type);
+	if (ret)
+		return -EBUSY;
+
+	iq = get_filter_steerq(dev, fs);
+
+	/*
+	 * IPv6 filters occupy four slots and must be aligned on four-slot
+	 * boundaries for T5. On T6, IPv6 filters occupy two-slots and
+	 * must be aligned on two-slot boundaries.
+	 *
+	 * IPv4 filters only occupy a single slot and have no alignment
+	 * requirements but writing a new IPv4 filter into the middle
+	 * of an existing IPv6 filter requires clearing the old IPv6
+	 * filter.
+	 */
+	if (fs->type == FILTER_TYPE_IPV4) { /* IPv4 */
+		/*
+		 * For T6, If our IPv4 filter isn't being written to a
+		 * multiple of two filter index and there's an IPv6
+		 * filter at the multiple of 2 base slot, then we need
+		 * to delete that IPv6 filter ...
+		 * For adapters below T6, IPv6 filter occupies 4 entries.
+		 */
+		if (chip_ver < CHELSIO_T6)
+			fidx = filter_id & ~0x3;
+		else
+			fidx = filter_id & ~0x1;
+
+		if (fidx != filter_id && adapter->tids.ftid_tab[fidx].fs.type) {
+			f = &adapter->tids.ftid_tab[fidx];
+			if (f->valid)
+				return -EBUSY;
+		}
+	} else { /* IPv6 */
+		unsigned int max_filter_id;
+
+		if (chip_ver < CHELSIO_T6) {
+			/*
+			 * Ensure that the IPv6 filter is aligned on a
+			 * multiple of 4 boundary.
+			 */
+			if (filter_id & 0x3)
+				return -EINVAL;
+
+			max_filter_id = filter_id + 4;
+		} else {
+			/*
+			 * For T6, CLIP being enabled, IPv6 filter would occupy
+			 * 2 entries.
+			 */
+			if (filter_id & 0x1)
+				return -EINVAL;
+
+			max_filter_id = filter_id + 2;
+		}
+
+		/*
+		 * Check all except the base overlapping IPv4 filter
+		 * slots.
+		 */
+		for (fidx = filter_id + 1; fidx < max_filter_id; fidx++) {
+			f = &adapter->tids.ftid_tab[fidx];
+			if (f->valid)
+				return -EBUSY;
+		}
+	}
+
+	/*
+	 * Check to make sure that provided filter index is not
+	 * already in use by someone else
+	 */
+	f = &adapter->tids.ftid_tab[filter_id];
+	if (f->valid)
+		return -EBUSY;
+
+	fidx = adapter->tids.ftid_base + filter_id;
+	fid_bit = filter_id;
+	ret = cxgbe_set_ftid(&adapter->tids, fid_bit,
+			     fs->type ? FILTER_TYPE_IPV6 : FILTER_TYPE_IPV4);
+	if (ret)
+		return ret;
+
+	/*
+	 * Check to make sure the filter requested is writable ...
+	 */
+	ret = writable_filter(f);
+	if (ret) {
+		/* Clear the bits we have set above */
+		cxgbe_clear_ftid(&adapter->tids, fid_bit,
+				 fs->type ? FILTER_TYPE_IPV6 :
+					    FILTER_TYPE_IPV4);
+		return ret;
+	}
+
+	/*
+	 * Allocate a clip table entry only if we have non-zero IPv6 address
+	 */
+	if (chip_ver > CHELSIO_T5 && fs->type &&
+	    memcmp(fs->val.lip, bitoff, sizeof(bitoff))) {
+		f->clipt = cxgbe_clip_alloc(f->dev, (u32 *)&f->fs.val.lip);
+		if (!f->clipt)
+			goto free_tid;
+	}
+
+	/*
+	 * Convert the filter specification into our internal format.
+	 * We copy the PF/VF specification into the Outer VLAN field
+	 * here so the rest of the code -- including the interface to
+	 * the firmware -- doesn't have to constantly do these checks.
+	 */
+	f->fs = *fs;
+	f->fs.iq = iq;
+	f->dev = dev;
+
+	/*
+	 * Attempt to set the filter.  If we don't succeed, we clear
+	 * it and return the failure.
+	 */
+	f->ctx = ctx;
+	f->tid = fidx; /* Save the actual tid */
+	ret = set_filter_wr(dev, filter_id);
+	if (ret) {
+		fid_bit = f->tid - adapter->tids.ftid_base;
+		goto free_tid;
+	}
+
+	return ret;
+
+free_tid:
+	cxgbe_clear_ftid(&adapter->tids, fid_bit,
+			 fs->type ? FILTER_TYPE_IPV6 :
+				    FILTER_TYPE_IPV4);
+	clear_filter(f);
+	return ret;
+}
+
+/**
+ * Handle a Hash filter write reply.
+ */
+void hash_filter_rpl(struct adapter *adap, const struct cpl_act_open_rpl *rpl)
+{
+	struct tid_info *t = &adap->tids;
+	struct filter_entry *f;
+	struct filter_ctx *ctx = NULL;
+	unsigned int tid = GET_TID(rpl);
+	unsigned int ftid = G_TID_TID(G_AOPEN_ATID
+				      (be32_to_cpu(rpl->atid_status)));
+	unsigned int status  = G_AOPEN_STATUS(be32_to_cpu(rpl->atid_status));
+
+	f = lookup_atid(t, ftid);
+	if (!f) {
+		dev_warn(adap, "%s: could not find filter entry: %d\n",
+			 __func__, ftid);
+		return;
+	}
+
+	ctx = f->ctx;
+	f->ctx = NULL;
+
+	switch (status) {
+	case CPL_ERR_NONE: {
+		f->tid = tid;
+		f->pending = 0;  /* asynchronous setup completed */
+		f->valid = 1;
+
+		cxgbe_insert_tid(t, f, f->tid, 0);
+		cxgbe_free_atid(t, ftid);
+		if (ctx) {
+			ctx->tid = f->tid;
+			ctx->result = 0;
+		}
+		if (f->fs.hitcnts)
+			set_tcb_field(adap, tid,
+				      W_TCB_TIMESTAMP,
+				      V_TCB_TIMESTAMP(M_TCB_TIMESTAMP) |
+				      V_TCB_T_RTT_TS_RECENT_AGE
+					      (M_TCB_T_RTT_TS_RECENT_AGE),
+				      V_TCB_TIMESTAMP(0ULL) |
+				      V_TCB_T_RTT_TS_RECENT_AGE(0ULL),
+				      1);
+		break;
+	}
+	default:
+		dev_warn(adap, "%s: filter creation failed with status = %u\n",
+			 __func__, status);
+
+		if (ctx) {
+			if (status == CPL_ERR_TCAM_FULL)
+				ctx->result = -EAGAIN;
+			else
+				ctx->result = -EINVAL;
+		}
+
+		cxgbe_free_atid(t, ftid);
+		t4_os_free(f);
+	}
+
+	if (ctx)
+		t4_complete(&ctx->completion);
+}
+
+/**
+ * Handle a LE-TCAM filter write/deletion reply.
+ */
+void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
+{
+	struct filter_entry *f = NULL;
+	unsigned int tid = GET_TID(rpl);
+	int idx, max_fidx = adap->tids.nftids;
+
+	/* Get the corresponding filter entry for this tid */
+	if (adap->tids.ftid_tab) {
+		/* Check this in normal filter region */
+		idx = tid - adap->tids.ftid_base;
+		if (idx >= max_fidx)
+			return;
+
+		f = &adap->tids.ftid_tab[idx];
+		if (f->tid != tid)
+			return;
+	}
+
+	/* We found the filter entry for this tid */
+	if (f) {
+		unsigned int ret = G_COOKIE(rpl->cookie);
+		struct filter_ctx *ctx;
+
+		/*
+		 * Pull off any filter operation context attached to the
+		 * filter.
+		 */
+		ctx = f->ctx;
+		f->ctx = NULL;
+
+		if (ret == FW_FILTER_WR_FLT_ADDED) {
+			f->pending = 0;  /* asynchronous setup completed */
+			f->valid = 1;
+			if (ctx) {
+				ctx->tid = f->tid;
+				ctx->result = 0;
+			}
+		} else if (ret == FW_FILTER_WR_FLT_DELETED) {
+			/*
+			 * Clear the filter when we get confirmation from the
+			 * hardware that the filter has been deleted.
+			 */
+			clear_filter(f);
+			if (ctx)
+				ctx->result = 0;
+		} else {
+			/*
+			 * Something went wrong.  Issue a warning about the
+			 * problem and clear everything out.
+			 */
+			dev_warn(adap, "filter %u setup failed with error %u\n",
+				 idx, ret);
+			clear_filter(f);
+			if (ctx)
+				ctx->result = -EINVAL;
+		}
+
+		if (ctx)
+			t4_complete(&ctx->completion);
+	}
+}
+
+/*
+ * Retrieve the packet count for the specified filter.
+ */
+int cxgbe_get_filter_count(struct adapter *adapter, unsigned int fidx,
+			   u64 *c, int hash, bool get_byte)
+{
+	struct filter_entry *f;
+	unsigned int tcb_base, tcbaddr;
+	int ret;
+
+	tcb_base = t4_read_reg(adapter, A_TP_CMM_TCB_BASE);
+	if (is_hashfilter(adapter) && hash) {
+		if (fidx < adapter->tids.ntids) {
+			f = adapter->tids.tid_tab[fidx];
+			if (!f)
+				return -EINVAL;
+
+			if (is_t5(adapter->params.chip)) {
+				*c = 0;
+				return 0;
+			}
+			tcbaddr = tcb_base + (fidx * TCB_SIZE);
+			goto get_count;
+		} else {
+			return -ERANGE;
+		}
+	} else {
+		if (fidx >= adapter->tids.nftids)
+			return -ERANGE;
+
+		f = &adapter->tids.ftid_tab[fidx];
+		if (!f->valid)
+			return -EINVAL;
+
+		tcbaddr = tcb_base + f->tid * TCB_SIZE;
+	}
+
+	f = &adapter->tids.ftid_tab[fidx];
+	if (!f->valid)
+		return -EINVAL;
+
+get_count:
+	if (is_t5(adapter->params.chip) || is_t6(adapter->params.chip)) {
+		/*
+		 * For T5, the Filter Packet Hit Count is maintained as a
+		 * 32-bit Big Endian value in the TCB field {timestamp}.
+		 * Similar to the craziness above, instead of the filter hit
+		 * count showing up at offset 20 ((W_TCB_TIMESTAMP == 5) *
+		 * sizeof(u32)), it actually shows up at offset 24.  Whacky.
+		 */
+		if (get_byte) {
+			unsigned int word_offset = 4;
+			__be64 be64_byte_count;
+
+			t4_os_lock(&adapter->win0_lock);
+			ret = t4_memory_rw(adapter, MEMWIN_NIC, MEM_EDC0,
+					   tcbaddr +
+					   (word_offset * sizeof(__be32)),
+					   sizeof(be64_byte_count),
+					   &be64_byte_count,
+					   T4_MEMORY_READ);
+			t4_os_unlock(&adapter->win0_lock);
+			if (ret < 0)
+				return ret;
+			*c = be64_to_cpu(be64_byte_count);
+		} else {
+			unsigned int word_offset = 6;
+			__be32 be32_count;
+
+			t4_os_lock(&adapter->win0_lock);
+			ret = t4_memory_rw(adapter, MEMWIN_NIC, MEM_EDC0,
+					   tcbaddr +
+					   (word_offset * sizeof(__be32)),
+					   sizeof(be32_count), &be32_count,
+					   T4_MEMORY_READ);
+			t4_os_unlock(&adapter->win0_lock);
+			if (ret < 0)
+				return ret;
+			*c = (u64)be32_to_cpu(be32_count);
+		}
+	}
+	return 0;
+}
+
+/**
+ * Handle a Hash filter delete reply.
+ */
+void hash_del_filter_rpl(struct adapter *adap,
+			 const struct cpl_abort_rpl_rss *rpl)
+{
+	struct tid_info *t = &adap->tids;
+	struct filter_entry *f;
+	struct filter_ctx *ctx = NULL;
+	unsigned int tid = GET_TID(rpl);
+
+	f = lookup_tid(t, tid);
+	if (!f) {
+		dev_warn(adap, "%s: could not find filter entry: %u\n",
+			 __func__, tid);
+		return;
+	}
+
+	ctx = f->ctx;
+	f->ctx = NULL;
+
+	f->valid = 0;
+
+	if (f->clipt)
+		cxgbe_clip_release(f->dev, f->clipt);
+
+	cxgbe_remove_tid(t, 0, tid, 0);
+	t4_os_free(f);
+
+	if (ctx) {
+		ctx->result = 0;
+		t4_complete(&ctx->completion);
+	}
+}
diff --git a/drivers/net/cxgbe/cxgbe_filter.h b/drivers/net/cxgbe/cxgbe_filter.h
new file mode 100644
index 00000000..af8fa752
--- /dev/null
+++ b/drivers/net/cxgbe/cxgbe_filter.h
@@ -0,0 +1,235 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+
+#ifndef _CXGBE_FILTER_H_
+#define _CXGBE_FILTER_H_
+
+#include "t4_msg.h"
+/*
+ * Defined bit width of user definable filter tuples
+ */
+#define ETHTYPE_BITWIDTH 16
+#define FRAG_BITWIDTH 1
+#define MACIDX_BITWIDTH 9
+#define FCOE_BITWIDTH 1
+#define IPORT_BITWIDTH 3
+#define MATCHTYPE_BITWIDTH 3
+#define PROTO_BITWIDTH 8
+#define TOS_BITWIDTH 8
+#define PF_BITWIDTH 8
+#define VF_BITWIDTH 8
+#define IVLAN_BITWIDTH 16
+#define OVLAN_BITWIDTH 16
+
+/*
+ * Filter matching rules.  These consist of a set of ingress packet field
+ * (value, mask) tuples.  The associated ingress packet field matches the
+ * tuple when ((field & mask) == value).  (Thus a wildcard "don't care" field
+ * rule can be constructed by specifying a tuple of (0, 0).)  A filter rule
+ * matches an ingress packet when all of the individual individual field
+ * matching rules are true.
+ *
+ * Partial field masks are always valid, however, while it may be easy to
+ * understand their meanings for some fields (e.g. IP address to match a
+ * subnet), for others making sensible partial masks is less intuitive (e.g.
+ * MPS match type) ...
+ */
+struct ch_filter_tuple {
+	/*
+	 * Compressed header matching field rules.  The TP_VLAN_PRI_MAP
+	 * register selects which of these fields will participate in the
+	 * filter match rules -- up to a maximum of 36 bits.  Because
+	 * TP_VLAN_PRI_MAP is a global register, all filters must use the same
+	 * set of fields.
+	 */
+	uint32_t ethtype:ETHTYPE_BITWIDTH;	/* Ethernet type */
+	uint32_t frag:FRAG_BITWIDTH;		/* IP fragmentation header */
+	uint32_t ivlan_vld:1;			/* inner VLAN valid */
+	uint32_t ovlan_vld:1;			/* outer VLAN valid */
+	uint32_t pfvf_vld:1;			/* PF/VF valid */
+	uint32_t macidx:MACIDX_BITWIDTH;	/* exact match MAC index */
+	uint32_t fcoe:FCOE_BITWIDTH;		/* FCoE packet */
+	uint32_t iport:IPORT_BITWIDTH;		/* ingress port */
+	uint32_t matchtype:MATCHTYPE_BITWIDTH;	/* MPS match type */
+	uint32_t proto:PROTO_BITWIDTH;		/* protocol type */
+	uint32_t tos:TOS_BITWIDTH;		/* TOS/Traffic Type */
+	uint32_t pf:PF_BITWIDTH;		/* PCI-E PF ID */
+	uint32_t vf:VF_BITWIDTH;		/* PCI-E VF ID */
+	uint32_t ivlan:IVLAN_BITWIDTH;		/* inner VLAN */
+	uint32_t ovlan:OVLAN_BITWIDTH;		/* outer VLAN */
+
+	/*
+	 * Uncompressed header matching field rules.  These are always
+	 * available for field rules.
+	 */
+	uint8_t lip[16];	/* local IP address (IPv4 in [3:0]) */
+	uint8_t fip[16];	/* foreign IP address (IPv4 in [3:0]) */
+	uint16_t lport;		/* local port */
+	uint16_t fport;		/* foreign port */
+
+	/* reservations for future additions */
+	uint8_t rsvd[12];
+};
+
+/*
+ * Filter specification
+ */
+struct ch_filter_specification {
+	/* Administrative fields for filter. */
+	uint32_t hitcnts:1;	/* count filter hits in TCB */
+	uint32_t prio:1;	/* filter has priority over active/server */
+
+	/*
+	 * Fundamental filter typing.  This is the one element of filter
+	 * matching that doesn't exist as a (value, mask) tuple.
+	 */
+	uint32_t type:1;	/* 0 => IPv4, 1 => IPv6 */
+	uint32_t cap:1;		/* 0 => LE-TCAM, 1 => Hash */
+
+	/*
+	 * Packet dispatch information.  Ingress packets which match the
+	 * filter rules will be dropped, passed to the host or switched back
+	 * out as egress packets.
+	 */
+	uint32_t action:2;	/* drop, pass, switch */
+
+	uint32_t dirsteer:1;	/* 0 => RSS, 1 => steer to iq */
+	uint32_t iq:10;		/* ingress queue */
+
+	uint32_t eport:2;	/* egress port to switch packet out */
+
+	/* Filter rule value/mask pairs. */
+	struct ch_filter_tuple val;
+	struct ch_filter_tuple mask;
+};
+
+enum {
+	FILTER_PASS = 0,	/* default */
+	FILTER_DROP,
+	FILTER_SWITCH
+};
+
+enum filter_type {
+	FILTER_TYPE_IPV4 = 0,
+	FILTER_TYPE_IPV6,
+};
+
+struct t4_completion {
+	unsigned int done;       /* completion done (0 - No, 1 - Yes) */
+	rte_spinlock_t lock;     /* completion lock */
+};
+
+/*
+ * Filter operation context to allow callers to wait for
+ * an asynchronous completion.
+ */
+struct filter_ctx {
+	struct t4_completion completion; /* completion rendezvous */
+	int result;                      /* result of operation */
+	u32 tid;                         /* to store tid of hash filter */
+};
+
+/*
+ * Host shadow copy of ingress filter entry.  This is in host native format
+ * and doesn't match the ordering or bit order, etc. of the hardware or the
+ * firmware command.
+ */
+struct filter_entry {
+	/*
+	 * Administrative fields for filter.
+	 */
+	u32 valid:1;                /* filter allocated and valid */
+	u32 locked:1;               /* filter is administratively locked */
+	u32 pending:1;              /* filter action is pending FW reply */
+	struct filter_ctx *ctx;     /* caller's completion hook */
+	struct clip_entry *clipt;   /* CLIP Table entry for IPv6 */
+	struct rte_eth_dev *dev;    /* Port's rte eth device */
+	void *private;              /* For use by apps using filter_entry */
+
+	/* This will store the actual tid */
+	u32 tid;
+
+	/*
+	 * The filter itself.
+	 */
+	struct ch_filter_specification fs;
+};
+
+#define FILTER_ID_MAX   (~0U)
+
+struct tid_info;
+struct adapter;
+
+/**
+ * Find first clear bit in the bitmap.
+ */
+static inline unsigned int cxgbe_find_first_zero_bit(struct rte_bitmap *bmap,
+						     unsigned int size)
+{
+	unsigned int idx;
+
+	for (idx = 0; idx < size; idx++)
+		if (!rte_bitmap_get(bmap, idx))
+			break;
+
+	return idx;
+}
+
+/**
+ * Find a free region of 'num' consecutive entries.
+ */
+static inline unsigned int
+cxgbe_bitmap_find_free_region(struct rte_bitmap *bmap, unsigned int size,
+			      unsigned int num)
+{
+	unsigned int idx, j, free = 0;
+
+	if (num > size)
+		return size;
+
+	for (idx = 0; idx < size; idx += num) {
+		for (j = 0; j < num; j++) {
+			if (!rte_bitmap_get(bmap, idx + j)) {
+				free++;
+			} else {
+				free = 0;
+				break;
+			}
+		}
+
+		/* Found the Region */
+		if (free == num)
+			break;
+
+		/* Reached the end and still no region found */
+		if ((idx + num) > size) {
+			idx = size;
+			break;
+		}
+	}
+
+	return idx;
+}
+
+bool is_filter_set(struct tid_info *, int fidx, int family);
+void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl);
+void clear_filter(struct filter_entry *f);
+int set_filter_wr(struct rte_eth_dev *dev, unsigned int fidx);
+int writable_filter(struct filter_entry *f);
+int cxgbe_set_filter(struct rte_eth_dev *dev, unsigned int filter_id,
+		     struct ch_filter_specification *fs,
+		     struct filter_ctx *ctx);
+int cxgbe_del_filter(struct rte_eth_dev *dev, unsigned int filter_id,
+		     struct ch_filter_specification *fs,
+		     struct filter_ctx *ctx);
+int cxgbe_alloc_ftid(struct adapter *adap, unsigned int family);
+int init_hash_filter(struct adapter *adap);
+void hash_filter_rpl(struct adapter *adap, const struct cpl_act_open_rpl *rpl);
+void hash_del_filter_rpl(struct adapter *adap,
+			 const struct cpl_abort_rpl_rss *rpl);
+int validate_filter(struct adapter *adap, struct ch_filter_specification *fs);
+int cxgbe_get_filter_count(struct adapter *adapter, unsigned int fidx,
+			   u64 *c, int hash, bool get_byte);
+#endif /* _CXGBE_FILTER_H_ */
diff --git a/drivers/net/cxgbe/cxgbe_flow.c b/drivers/net/cxgbe/cxgbe_flow.c
new file mode 100644
index 00000000..01c945f1
--- /dev/null
+++ b/drivers/net/cxgbe/cxgbe_flow.c
@@ -0,0 +1,845 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+#include "common.h"
+#include "cxgbe_flow.h"
+
+#define __CXGBE_FILL_FS(__v, __m, fs, elem, e) \
+do { \
+	if (!((fs)->val.elem || (fs)->mask.elem)) { \
+		(fs)->val.elem = (__v); \
+		(fs)->mask.elem = (__m); \
+	} else { \
+		return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, \
+					  NULL, "a filter can be specified" \
+					  " only once"); \
+	} \
+} while (0)
+
+#define __CXGBE_FILL_FS_MEMCPY(__v, __m, fs, elem) \
+do { \
+	memcpy(&(fs)->val.elem, &(__v), sizeof(__v)); \
+	memcpy(&(fs)->mask.elem, &(__m), sizeof(__m)); \
+} while (0)
+
+#define CXGBE_FILL_FS(v, m, elem) \
+	__CXGBE_FILL_FS(v, m, fs, elem, e)
+
+#define CXGBE_FILL_FS_MEMCPY(v, m, elem) \
+	__CXGBE_FILL_FS_MEMCPY(v, m, fs, elem)
+
+static int
+cxgbe_validate_item(const struct rte_flow_item *i, struct rte_flow_error *e)
+{
+	/* rte_flow specification does not allow it. */
+	if (!i->spec && (i->mask ||  i->last))
+		return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
+				   i, "last or mask given without spec");
+	/*
+	 * We don't support it.
+	 * Although, we can support values in last as 0's or last == spec.
+	 * But this will not provide user with any additional functionality
+	 * and will only increase the complexity for us.
+	 */
+	if (i->last)
+		return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
+				   i, "last is not supported by chelsio pmd");
+	return 0;
+}
+
+static void
+cxgbe_fill_filter_region(struct adapter *adap,
+			 struct ch_filter_specification *fs)
+{
+	struct tp_params *tp = &adap->params.tp;
+	u64 hash_filter_mask = tp->hash_filter_mask;
+	u64 ntuple_mask = 0;
+
+	fs->cap = 0;
+
+	if (!is_hashfilter(adap))
+		return;
+
+	if (fs->type) {
+		uint8_t biton[16] = {0xff, 0xff, 0xff, 0xff,
+				     0xff, 0xff, 0xff, 0xff,
+				     0xff, 0xff, 0xff, 0xff,
+				     0xff, 0xff, 0xff, 0xff};
+		uint8_t bitoff[16] = {0};
+
+		if (!memcmp(fs->val.lip, bitoff, sizeof(bitoff)) ||
+		    !memcmp(fs->val.fip, bitoff, sizeof(bitoff)) ||
+		    memcmp(fs->mask.lip, biton, sizeof(biton)) ||
+		    memcmp(fs->mask.fip, biton, sizeof(biton)))
+			return;
+	} else {
+		uint32_t biton  = 0xffffffff;
+		uint32_t bitoff = 0x0U;
+
+		if (!memcmp(fs->val.lip, &bitoff, sizeof(bitoff)) ||
+		    !memcmp(fs->val.fip, &bitoff, sizeof(bitoff)) ||
+		    memcmp(fs->mask.lip, &biton, sizeof(biton)) ||
+		    memcmp(fs->mask.fip, &biton, sizeof(biton)))
+			return;
+	}
+
+	if (!fs->val.lport || fs->mask.lport != 0xffff)
+		return;
+	if (!fs->val.fport || fs->mask.fport != 0xffff)
+		return;
+
+	if (tp->protocol_shift >= 0)
+		ntuple_mask |= (u64)fs->mask.proto << tp->protocol_shift;
+	if (tp->ethertype_shift >= 0)
+		ntuple_mask |= (u64)fs->mask.ethtype << tp->ethertype_shift;
+	if (tp->port_shift >= 0)
+		ntuple_mask |= (u64)fs->mask.iport << tp->port_shift;
+
+	if (ntuple_mask != hash_filter_mask)
+		return;
+
+	fs->cap = 1;	/* use hash region */
+}
+
+static int
+ch_rte_parsetype_port(const void *dmask, const struct rte_flow_item *item,
+		      struct ch_filter_specification *fs,
+		      struct rte_flow_error *e)
+{
+	const struct rte_flow_item_phy_port *val = item->spec;
+	const struct rte_flow_item_phy_port *umask = item->mask;
+	const struct rte_flow_item_phy_port *mask;
+
+	mask = umask ? umask : (const struct rte_flow_item_phy_port *)dmask;
+
+	if (val->index > 0x7)
+		return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "port index upto 0x7 is supported");
+
+	CXGBE_FILL_FS(val->index, mask->index, iport);
+
+	return 0;
+}
+
+static int
+ch_rte_parsetype_udp(const void *dmask, const struct rte_flow_item *item,
+		     struct ch_filter_specification *fs,
+		     struct rte_flow_error *e)
+{
+	const struct rte_flow_item_udp *val = item->spec;
+	const struct rte_flow_item_udp *umask = item->mask;
+	const struct rte_flow_item_udp *mask;
+
+	mask = umask ? umask : (const struct rte_flow_item_udp *)dmask;
+
+	if (mask->hdr.dgram_len || mask->hdr.dgram_cksum)
+		return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "udp: only src/dst port supported");
+
+	CXGBE_FILL_FS(IPPROTO_UDP, 0xff, proto);
+	if (!val)
+		return 0;
+	CXGBE_FILL_FS(be16_to_cpu(val->hdr.src_port),
+		      be16_to_cpu(mask->hdr.src_port), fport);
+	CXGBE_FILL_FS(be16_to_cpu(val->hdr.dst_port),
+		      be16_to_cpu(mask->hdr.dst_port), lport);
+	return 0;
+}
+
+static int
+ch_rte_parsetype_tcp(const void *dmask, const struct rte_flow_item *item,
+		     struct ch_filter_specification *fs,
+		     struct rte_flow_error *e)
+{
+	const struct rte_flow_item_tcp *val = item->spec;
+	const struct rte_flow_item_tcp *umask = item->mask;
+	const struct rte_flow_item_tcp *mask;
+
+	mask = umask ? umask : (const struct rte_flow_item_tcp *)dmask;
+
+	if (mask->hdr.sent_seq || mask->hdr.recv_ack || mask->hdr.data_off ||
+	    mask->hdr.tcp_flags || mask->hdr.rx_win || mask->hdr.cksum ||
+	    mask->hdr.tcp_urp)
+		return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "tcp: only src/dst port supported");
+
+	CXGBE_FILL_FS(IPPROTO_TCP, 0xff, proto);
+	if (!val)
+		return 0;
+	CXGBE_FILL_FS(be16_to_cpu(val->hdr.src_port),
+		      be16_to_cpu(mask->hdr.src_port), fport);
+	CXGBE_FILL_FS(be16_to_cpu(val->hdr.dst_port),
+		      be16_to_cpu(mask->hdr.dst_port), lport);
+	return 0;
+}
+
+static int
+ch_rte_parsetype_ipv4(const void *dmask, const struct rte_flow_item *item,
+		      struct ch_filter_specification *fs,
+		      struct rte_flow_error *e)
+{
+	const struct rte_flow_item_ipv4 *val = item->spec;
+	const struct rte_flow_item_ipv4 *umask = item->mask;
+	const struct rte_flow_item_ipv4 *mask;
+
+	mask = umask ? umask : (const struct rte_flow_item_ipv4 *)dmask;
+
+	if (mask->hdr.time_to_live || mask->hdr.type_of_service)
+		return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
+					  item, "ttl/tos are not supported");
+
+	fs->type = FILTER_TYPE_IPV4;
+	CXGBE_FILL_FS(ETHER_TYPE_IPv4, 0xffff, ethtype);
+	if (!val)
+		return 0; /* ipv4 wild card */
+
+	CXGBE_FILL_FS(val->hdr.next_proto_id, mask->hdr.next_proto_id, proto);
+	CXGBE_FILL_FS_MEMCPY(val->hdr.dst_addr, mask->hdr.dst_addr, lip);
+	CXGBE_FILL_FS_MEMCPY(val->hdr.src_addr, mask->hdr.src_addr, fip);
+
+	return 0;
+}
+
+static int
+ch_rte_parsetype_ipv6(const void *dmask, const struct rte_flow_item *item,
+		      struct ch_filter_specification *fs,
+		      struct rte_flow_error *e)
+{
+	const struct rte_flow_item_ipv6 *val = item->spec;
+	const struct rte_flow_item_ipv6 *umask = item->mask;
+	const struct rte_flow_item_ipv6 *mask;
+
+	mask = umask ? umask : (const struct rte_flow_item_ipv6 *)dmask;
+
+	if (mask->hdr.vtc_flow ||
+	    mask->hdr.payload_len || mask->hdr.hop_limits)
+		return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "tc/flow/hop are not supported");
+
+	fs->type = FILTER_TYPE_IPV6;
+	CXGBE_FILL_FS(ETHER_TYPE_IPv6, 0xffff, ethtype);
+	if (!val)
+		return 0; /* ipv6 wild card */
+
+	CXGBE_FILL_FS(val->hdr.proto, mask->hdr.proto, proto);
+	CXGBE_FILL_FS_MEMCPY(val->hdr.dst_addr, mask->hdr.dst_addr, lip);
+	CXGBE_FILL_FS_MEMCPY(val->hdr.src_addr, mask->hdr.src_addr, fip);
+
+	return 0;
+}
+
+static int
+cxgbe_rtef_parse_attr(struct rte_flow *flow, const struct rte_flow_attr *attr,
+		      struct rte_flow_error *e)
+{
+	if (attr->egress)
+		return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR,
+					  attr, "attribute:<egress> is"
+					  " not supported !");
+	if (attr->group > 0)
+		return rte_flow_error_set(e, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR,
+					  attr, "group parameter is"
+					  " not supported.");
+
+	flow->fidx = attr->priority ? attr->priority - 1 : FILTER_ID_MAX;
+
+	return 0;
+}
+
+static inline int check_rxq(struct rte_eth_dev *dev, uint16_t rxq)
+{
+	struct port_info *pi = ethdev2pinfo(dev);
+
+	if (rxq > pi->n_rx_qsets)
+		return -EINVAL;
+	return 0;
+}
+
+static int cxgbe_validate_fidxondel(struct filter_entry *f, unsigned int fidx)
+{
+	struct adapter *adap = ethdev2adap(f->dev);
+	struct ch_filter_specification fs = f->fs;
+
+	if (fidx >= adap->tids.nftids) {
+		dev_err(adap, "invalid flow index %d.\n", fidx);
+		return -EINVAL;
+	}
+	if (!is_filter_set(&adap->tids, fidx, fs.type)) {
+		dev_err(adap, "Already free fidx:%d f:%p\n", fidx, f);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+cxgbe_validate_fidxonadd(struct ch_filter_specification *fs,
+			 struct adapter *adap, unsigned int fidx)
+{
+	if (is_filter_set(&adap->tids, fidx, fs->type)) {
+		dev_err(adap, "filter index: %d is busy.\n", fidx);
+		return -EBUSY;
+	}
+	if (fidx >= adap->tids.nftids) {
+		dev_err(adap, "filter index (%u) >= max(%u)\n",
+			fidx, adap->tids.nftids);
+		return -ERANGE;
+	}
+
+	return 0;
+}
+
+static int
+cxgbe_verify_fidx(struct rte_flow *flow, unsigned int fidx, uint8_t del)
+{
+	if (flow->fs.cap)
+		return 0; /* Hash filters */
+	return del ? cxgbe_validate_fidxondel(flow->f, fidx) :
+		cxgbe_validate_fidxonadd(&flow->fs,
+					 ethdev2adap(flow->dev), fidx);
+}
+
+static int cxgbe_get_fidx(struct rte_flow *flow, unsigned int *fidx)
+{
+	struct ch_filter_specification *fs = &flow->fs;
+	struct adapter *adap = ethdev2adap(flow->dev);
+
+	/* For tcam get the next available slot, if default value specified */
+	if (flow->fidx == FILTER_ID_MAX) {
+		int idx;
+
+		idx = cxgbe_alloc_ftid(adap, fs->type);
+		if (idx < 0) {
+			dev_err(adap, "unable to get a filter index in tcam\n");
+			return -ENOMEM;
+		}
+		*fidx = (unsigned int)idx;
+	} else {
+		*fidx = flow->fidx;
+	}
+
+	return 0;
+}
+
+static int
+ch_rte_parse_atype_switch(const struct rte_flow_action *a,
+			  struct ch_filter_specification *fs,
+			  struct rte_flow_error *e)
+{
+	const struct rte_flow_action_phy_port *port;
+
+	switch (a->type) {
+	case RTE_FLOW_ACTION_TYPE_PHY_PORT:
+		port = (const struct rte_flow_action_phy_port *)a->conf;
+		fs->eport = port->index;
+		break;
+	default:
+		/* We are not supposed to come here */
+		return rte_flow_error_set(e, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, a,
+					  "Action not supported");
+	}
+
+	return 0;
+}
+
+static int
+cxgbe_rtef_parse_actions(struct rte_flow *flow,
+			 const struct rte_flow_action action[],
+			 struct rte_flow_error *e)
+{
+	struct ch_filter_specification *fs = &flow->fs;
+	const struct rte_flow_action_queue *q;
+	const struct rte_flow_action *a;
+	char abit = 0;
+	int ret;
+
+	for (a = action; a->type != RTE_FLOW_ACTION_TYPE_END; a++) {
+		switch (a->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			continue;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			if (abit++)
+				return rte_flow_error_set(e, EINVAL,
+						RTE_FLOW_ERROR_TYPE_ACTION, a,
+						"specify only 1 pass/drop");
+			fs->action = FILTER_DROP;
+			break;
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			q = (const struct rte_flow_action_queue *)a->conf;
+			if (!q)
+				return rte_flow_error_set(e, EINVAL,
+						RTE_FLOW_ERROR_TYPE_ACTION, q,
+						"specify rx queue index");
+			if (check_rxq(flow->dev, q->index))
+				return rte_flow_error_set(e, EINVAL,
+						RTE_FLOW_ERROR_TYPE_ACTION, q,
+						"Invalid rx queue");
+			if (abit++)
+				return rte_flow_error_set(e, EINVAL,
+						RTE_FLOW_ERROR_TYPE_ACTION, a,
+						"specify only 1 pass/drop");
+			fs->action = FILTER_PASS;
+			fs->dirsteer = 1;
+			fs->iq = q->index;
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			fs->hitcnts = 1;
+			break;
+		case RTE_FLOW_ACTION_TYPE_PHY_PORT:
+			/* We allow multiple switch actions, but switch is
+			 * not compatible with either queue or drop
+			 */
+			if (abit++ && fs->action != FILTER_SWITCH)
+				return rte_flow_error_set(e, EINVAL,
+						RTE_FLOW_ERROR_TYPE_ACTION, a,
+						"overlapping action specified");
+			ret = ch_rte_parse_atype_switch(a, fs, e);
+			if (ret)
+				return ret;
+			fs->action = FILTER_SWITCH;
+			break;
+		default:
+			/* Not supported action : return error */
+			return rte_flow_error_set(e, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  a, "Action not supported");
+		}
+	}
+
+	return 0;
+}
+
+struct chrte_fparse parseitem[] = {
+		[RTE_FLOW_ITEM_TYPE_PHY_PORT] = {
+		.fptr = ch_rte_parsetype_port,
+		.dmask = &(const struct rte_flow_item_phy_port){
+			.index = 0x7,
+		}
+	},
+
+	[RTE_FLOW_ITEM_TYPE_IPV4] = {
+		.fptr  = ch_rte_parsetype_ipv4,
+		.dmask = &rte_flow_item_ipv4_mask,
+	},
+
+	[RTE_FLOW_ITEM_TYPE_IPV6] = {
+		.fptr  = ch_rte_parsetype_ipv6,
+		.dmask = &rte_flow_item_ipv6_mask,
+	},
+
+	[RTE_FLOW_ITEM_TYPE_UDP] = {
+		.fptr  = ch_rte_parsetype_udp,
+		.dmask = &rte_flow_item_udp_mask,
+	},
+
+	[RTE_FLOW_ITEM_TYPE_TCP] = {
+		.fptr  = ch_rte_parsetype_tcp,
+		.dmask = &rte_flow_item_tcp_mask,
+	},
+};
+
+static int
+cxgbe_rtef_parse_items(struct rte_flow *flow,
+		       const struct rte_flow_item items[],
+		       struct rte_flow_error *e)
+{
+	struct adapter *adap = ethdev2adap(flow->dev);
+	const struct rte_flow_item *i;
+	char repeat[ARRAY_SIZE(parseitem)] = {0};
+
+	for (i = items; i->type != RTE_FLOW_ITEM_TYPE_END; i++) {
+		struct chrte_fparse *idx = &flow->item_parser[i->type];
+		int ret;
+
+		if (i->type > ARRAY_SIZE(parseitem))
+			return rte_flow_error_set(e, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  i, "Item not supported");
+
+		switch (i->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			continue;
+		default:
+			/* check if item is repeated */
+			if (repeat[i->type])
+				return rte_flow_error_set(e, ENOTSUP,
+						RTE_FLOW_ERROR_TYPE_ITEM, i,
+						"parse items cannot be repeated (except void)");
+			repeat[i->type] = 1;
+
+			/* validate the item */
+			ret = cxgbe_validate_item(i, e);
+			if (ret)
+				return ret;
+
+			if (!idx || !idx->fptr) {
+				return rte_flow_error_set(e, ENOTSUP,
+						RTE_FLOW_ERROR_TYPE_ITEM, i,
+						"Item not supported");
+			} else {
+				ret = idx->fptr(idx->dmask, i, &flow->fs, e);
+				if (ret)
+					return ret;
+			}
+		}
+	}
+
+	cxgbe_fill_filter_region(adap, &flow->fs);
+
+	return 0;
+}
+
+static int
+cxgbe_flow_parse(struct rte_flow *flow,
+		 const struct rte_flow_attr *attr,
+		 const struct rte_flow_item item[],
+		 const struct rte_flow_action action[],
+		 struct rte_flow_error *e)
+{
+	int ret;
+
+	/* parse user request into ch_filter_specification */
+	ret = cxgbe_rtef_parse_attr(flow, attr, e);
+	if (ret)
+		return ret;
+	ret = cxgbe_rtef_parse_items(flow, item, e);
+	if (ret)
+		return ret;
+	return cxgbe_rtef_parse_actions(flow, action, e);
+}
+
+static int __cxgbe_flow_create(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct ch_filter_specification *fs = &flow->fs;
+	struct adapter *adap = ethdev2adap(dev);
+	struct tid_info *t = &adap->tids;
+	struct filter_ctx ctx;
+	unsigned int fidx;
+	int err;
+
+	if (cxgbe_get_fidx(flow, &fidx))
+		return -ENOMEM;
+	if (cxgbe_verify_fidx(flow, fidx, 0))
+		return -1;
+
+	t4_init_completion(&ctx.completion);
+	/* go create the filter */
+	err = cxgbe_set_filter(dev, fidx, fs, &ctx);
+	if (err) {
+		dev_err(adap, "Error %d while creating filter.\n", err);
+		return err;
+	}
+
+	/* Poll the FW for reply */
+	err = cxgbe_poll_for_completion(&adap->sge.fw_evtq,
+					CXGBE_FLOW_POLL_US,
+					CXGBE_FLOW_POLL_CNT,
+					&ctx.completion);
+	if (err) {
+		dev_err(adap, "Filter set operation timed out (%d)\n", err);
+		return err;
+	}
+	if (ctx.result) {
+		dev_err(adap, "Hardware error %d while creating the filter.\n",
+			ctx.result);
+		return ctx.result;
+	}
+
+	if (fs->cap) { /* to destroy the filter */
+		flow->fidx = ctx.tid;
+		flow->f = lookup_tid(t, ctx.tid);
+	} else {
+		flow->fidx = fidx;
+		flow->f = &adap->tids.ftid_tab[fidx];
+	}
+
+	return 0;
+}
+
+static struct rte_flow *
+cxgbe_flow_create(struct rte_eth_dev *dev,
+		  const struct rte_flow_attr *attr,
+		  const struct rte_flow_item item[],
+		  const struct rte_flow_action action[],
+		  struct rte_flow_error *e)
+{
+	struct rte_flow *flow;
+	int ret;
+
+	flow = t4_os_alloc(sizeof(struct rte_flow));
+	if (!flow) {
+		rte_flow_error_set(e, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
+				   NULL, "Unable to allocate memory for"
+				   " filter_entry");
+		return NULL;
+	}
+
+	flow->item_parser = parseitem;
+	flow->dev = dev;
+
+	if (cxgbe_flow_parse(flow, attr, item, action, e)) {
+		t4_os_free(flow);
+		return NULL;
+	}
+
+	/* go, interact with cxgbe_filter */
+	ret = __cxgbe_flow_create(dev, flow);
+	if (ret) {
+		rte_flow_error_set(e, ret, RTE_FLOW_ERROR_TYPE_HANDLE,
+				   NULL, "Unable to create flow rule");
+		t4_os_free(flow);
+		return NULL;
+	}
+
+	flow->f->private = flow; /* Will be used during flush */
+
+	return flow;
+}
+
+static int __cxgbe_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct adapter *adap = ethdev2adap(dev);
+	struct filter_entry *f = flow->f;
+	struct ch_filter_specification *fs;
+	struct filter_ctx ctx;
+	int err;
+
+	fs = &f->fs;
+	if (cxgbe_verify_fidx(flow, flow->fidx, 1))
+		return -1;
+
+	t4_init_completion(&ctx.completion);
+	err = cxgbe_del_filter(dev, flow->fidx, fs, &ctx);
+	if (err) {
+		dev_err(adap, "Error %d while deleting filter.\n", err);
+		return err;
+	}
+
+	/* Poll the FW for reply */
+	err = cxgbe_poll_for_completion(&adap->sge.fw_evtq,
+					CXGBE_FLOW_POLL_US,
+					CXGBE_FLOW_POLL_CNT,
+					&ctx.completion);
+	if (err) {
+		dev_err(adap, "Filter delete operation timed out (%d)\n", err);
+		return err;
+	}
+	if (ctx.result) {
+		dev_err(adap, "Hardware error %d while deleting the filter.\n",
+			ctx.result);
+		return ctx.result;
+	}
+
+	return 0;
+}
+
+static int
+cxgbe_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
+		   struct rte_flow_error *e)
+{
+	int ret;
+
+	ret = __cxgbe_flow_destroy(dev, flow);
+	if (ret)
+		return rte_flow_error_set(e, ret, RTE_FLOW_ERROR_TYPE_HANDLE,
+					  flow, "error destroying filter.");
+	t4_os_free(flow);
+	return 0;
+}
+
+static int __cxgbe_flow_query(struct rte_flow *flow, u64 *count,
+			      u64 *byte_count)
+{
+	struct adapter *adap = ethdev2adap(flow->dev);
+	struct ch_filter_specification fs = flow->f->fs;
+	unsigned int fidx = flow->fidx;
+	int ret = 0;
+
+	ret = cxgbe_get_filter_count(adap, fidx, count, fs.cap, 0);
+	if (ret)
+		return ret;
+	return cxgbe_get_filter_count(adap, fidx, byte_count, fs.cap, 1);
+}
+
+static int
+cxgbe_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
+		 const struct rte_flow_action *action, void *data,
+		 struct rte_flow_error *e)
+{
+	struct ch_filter_specification fs;
+	struct rte_flow_query_count *c;
+	struct filter_entry *f;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	f = flow->f;
+	fs = f->fs;
+
+	if (action->type != RTE_FLOW_ACTION_TYPE_COUNT)
+		return rte_flow_error_set(e, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "only count supported for query");
+
+	/*
+	 * This is a valid operation, Since we are allowed to do chelsio
+	 * specific operations in rte side of our code but not vise-versa
+	 *
+	 * So, fs can be queried/modified here BUT rte_flow_query_count
+	 * cannot be worked on by the lower layer since we want to maintain
+	 * it as rte_flow agnostic.
+	 */
+	if (!fs.hitcnts)
+		return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+					  &fs, "filter hit counters were not"
+					  " enabled during filter creation");
+
+	c = (struct rte_flow_query_count *)data;
+	ret = __cxgbe_flow_query(flow, &c->hits, &c->bytes);
+	if (ret)
+		return rte_flow_error_set(e, -ret, RTE_FLOW_ERROR_TYPE_ACTION,
+					  f, "cxgbe pmd failed to"
+					  " perform query");
+
+	/* Query was successful */
+	c->bytes_set = 1;
+	c->hits_set = 1;
+
+	return 0; /* success / partial_success */
+}
+
+static int
+cxgbe_flow_validate(struct rte_eth_dev *dev,
+		    const struct rte_flow_attr *attr,
+		    const struct rte_flow_item item[],
+		    const struct rte_flow_action action[],
+		    struct rte_flow_error *e)
+{
+	struct adapter *adap = ethdev2adap(dev);
+	struct rte_flow *flow;
+	unsigned int fidx;
+	int ret;
+
+	flow = t4_os_alloc(sizeof(struct rte_flow));
+	if (!flow)
+		return rte_flow_error_set(e, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
+				NULL,
+				"Unable to allocate memory for filter_entry");
+
+	flow->item_parser = parseitem;
+	flow->dev = dev;
+
+	ret = cxgbe_flow_parse(flow, attr, item, action, e);
+	if (ret) {
+		t4_os_free(flow);
+		return ret;
+	}
+
+	if (validate_filter(adap, &flow->fs)) {
+		t4_os_free(flow);
+		return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
+				NULL,
+				"validation failed. Check f/w config file.");
+	}
+
+	if (cxgbe_get_fidx(flow, &fidx)) {
+		t4_os_free(flow);
+		return rte_flow_error_set(e, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
+					  NULL, "no memory in tcam.");
+	}
+
+	if (cxgbe_verify_fidx(flow, fidx, 0)) {
+		t4_os_free(flow);
+		return rte_flow_error_set(e, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
+					  NULL, "validation failed");
+	}
+
+	t4_os_free(flow);
+	return 0;
+}
+
+/*
+ * @ret : > 0 filter destroyed succsesfully
+ *        < 0 error destroying filter
+ *        == 1 filter not active / not found
+ */
+static int
+cxgbe_check_n_destroy(struct filter_entry *f, struct rte_eth_dev *dev,
+		      struct rte_flow_error *e)
+{
+	if (f && (f->valid || f->pending) &&
+	    f->dev == dev && /* Only if user has asked for this port */
+	     f->private) /* We (rte_flow) created this filter */
+		return cxgbe_flow_destroy(dev, (struct rte_flow *)f->private,
+					  e);
+	return 1;
+}
+
+static int cxgbe_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *e)
+{
+	struct adapter *adap = ethdev2adap(dev);
+	unsigned int i;
+	int ret = 0;
+
+	if (adap->tids.ftid_tab) {
+		struct filter_entry *f = &adap->tids.ftid_tab[0];
+
+		for (i = 0; i < adap->tids.nftids; i++, f++) {
+			ret = cxgbe_check_n_destroy(f, dev, e);
+			if (ret < 0)
+				goto out;
+		}
+	}
+
+	if (is_hashfilter(adap) && adap->tids.tid_tab) {
+		struct filter_entry *f;
+
+		for (i = adap->tids.hash_base; i <= adap->tids.ntids; i++) {
+			f = (struct filter_entry *)adap->tids.tid_tab[i];
+
+			ret = cxgbe_check_n_destroy(f, dev, e);
+			if (ret < 0)
+				goto out;
+		}
+	}
+
+out:
+	return ret >= 0 ? 0 : ret;
+}
+
+static const struct rte_flow_ops cxgbe_flow_ops = {
+	.validate	= cxgbe_flow_validate,
+	.create		= cxgbe_flow_create,
+	.destroy	= cxgbe_flow_destroy,
+	.flush		= cxgbe_flow_flush,
+	.query		= cxgbe_flow_query,
+	.isolate	= NULL,
+};
+
+int
+cxgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
+		      enum rte_filter_type filter_type,
+		      enum rte_filter_op filter_op,
+		      void *arg)
+{
+	int ret = 0;
+
+	RTE_SET_USED(dev);
+	switch (filter_type) {
+	case RTE_ETH_FILTER_GENERIC:
+		if (filter_op != RTE_ETH_FILTER_GET)
+			return -EINVAL;
+		*(const void **)arg = &cxgbe_flow_ops;
+		break;
+	default:
+		ret = -ENOTSUP;
+		break;
+	}
+	return ret;
+}
diff --git a/drivers/net/cxgbe/cxgbe_flow.h b/drivers/net/cxgbe/cxgbe_flow.h
new file mode 100644
index 00000000..0f750474
--- /dev/null
+++ b/drivers/net/cxgbe/cxgbe_flow.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+#ifndef _CXGBE_FLOW_H_
+#define _CXGBE_FLOW_H_
+
+#include <rte_flow_driver.h>
+#include "cxgbe_filter.h"
+#include "cxgbe.h"
+
+#define CXGBE_FLOW_POLL_US  10
+#define CXGBE_FLOW_POLL_CNT 10
+
+struct chrte_fparse {
+	int (*fptr)(const void *mask, /* currently supported mask */
+		    const struct rte_flow_item *item, /* user input */
+		    struct ch_filter_specification *fs, /* where to parse */
+		    struct rte_flow_error *e);
+	const void *dmask; /* Specify what is supported by chelsio by default*/
+};
+
+struct rte_flow {
+	struct filter_entry *f;
+	struct ch_filter_specification fs; /* temp, to create filter */
+	struct chrte_fparse *item_parser;
+	/*
+	 * filter_entry doesn't store user priority.
+	 * Post creation of filter this will indicate the
+	 * flow index (fidx) for both hash and tcam filters
+	 */
+	unsigned int fidx;
+	struct rte_eth_dev *dev;
+};
+
+int
+cxgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
+		      enum rte_filter_type filter_type,
+		      enum rte_filter_op filter_op,
+		      void *arg);
+
+#endif /* _CXGBE_FLOW_H_ */
diff --git a/drivers/net/cxgbe/cxgbe_main.c b/drivers/net/cxgbe/cxgbe_main.c
index 54eb23df..c3938e8d 100644
--- a/drivers/net/cxgbe/cxgbe_main.c
+++ b/drivers/net/cxgbe/cxgbe_main.c
@@ -37,6 +37,23 @@
 #include "t4_regs.h"
 #include "t4_msg.h"
 #include "cxgbe.h"
+#include "clip_tbl.h"
+
+/**
+ * Allocate a chunk of memory. The allocated memory is cleared.
+ */
+void *t4_alloc_mem(size_t size)
+{
+	return rte_zmalloc(NULL, size, 0);
+}
+
+/**
+ * Free memory allocated through t4_alloc_mem().
+ */
+void t4_free_mem(void *addr)
+{
+	rte_free(addr);
+}
 
 /*
  * Response queue handler for the FW event queue.
@@ -70,6 +87,18 @@ static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 		const struct cpl_fw6_msg *msg = (const void *)rsp;
 
 		t4_handle_fw_rpl(q->adapter, msg->data);
+	} else if (opcode == CPL_ABORT_RPL_RSS) {
+		const struct cpl_abort_rpl_rss *p = (const void *)rsp;
+
+		hash_del_filter_rpl(q->adapter, p);
+	} else if (opcode == CPL_SET_TCB_RPL) {
+		const struct cpl_set_tcb_rpl *p = (const void *)rsp;
+
+		filter_rpl(q->adapter, p);
+	} else if (opcode == CPL_ACT_OPEN_RPL) {
+		const struct cpl_act_open_rpl *p = (const void *)rsp;
+
+		hash_filter_rpl(q->adapter, p);
 	} else {
 		dev_err(adapter, "unexpected CPL %#x on FW event queue\n",
 			opcode);
@@ -78,6 +107,79 @@ out:
 	return 0;
 }
 
+/**
+ * Setup sge control queues to pass control information.
+ */
+int setup_sge_ctrl_txq(struct adapter *adapter)
+{
+	struct sge *s = &adapter->sge;
+	int err = 0, i = 0;
+
+	for_each_port(adapter, i) {
+		char name[RTE_ETH_NAME_MAX_LEN];
+		struct sge_ctrl_txq *q = &s->ctrlq[i];
+
+		q->q.size = 1024;
+		err = t4_sge_alloc_ctrl_txq(adapter, q,
+					    adapter->eth_dev,  i,
+					    s->fw_evtq.cntxt_id,
+					    rte_socket_id());
+		if (err) {
+			dev_err(adapter, "Failed to alloc ctrl txq. Err: %d",
+				err);
+			goto out;
+		}
+		snprintf(name, sizeof(name), "cxgbe_ctrl_pool_%d", i);
+		q->mb_pool = rte_pktmbuf_pool_create(name, s->ctrlq[i].q.size,
+						     RTE_CACHE_LINE_SIZE,
+						     RTE_MBUF_PRIV_ALIGN,
+						     RTE_MBUF_DEFAULT_BUF_SIZE,
+						     SOCKET_ID_ANY);
+		if (!q->mb_pool) {
+			dev_err(adapter, "Can't create ctrl pool for port: %d",
+				i);
+			err = -ENOMEM;
+			goto out;
+		}
+	}
+	return 0;
+out:
+	t4_free_sge_resources(adapter);
+	return err;
+}
+
+/**
+ * cxgbe_poll_for_completion: Poll rxq for completion
+ * @q: rxq to poll
+ * @us: microseconds to delay
+ * @cnt: number of times to poll
+ * @c: completion to check for 'done' status
+ *
+ * Polls the rxq for reples until completion is done or the count
+ * expires.
+ */
+int cxgbe_poll_for_completion(struct sge_rspq *q, unsigned int us,
+			      unsigned int cnt, struct t4_completion *c)
+{
+	unsigned int i;
+	unsigned int work_done, budget = 4;
+
+	if (!c)
+		return -EINVAL;
+
+	for (i = 0; i < cnt; i++) {
+		cxgbe_poll(q, NULL, budget, &work_done);
+		t4_os_lock(&c->lock);
+		if (c->done) {
+			t4_os_unlock(&c->lock);
+			return 0;
+		}
+		t4_os_unlock(&c->lock);
+		udelay(us);
+	}
+	return -ETIMEDOUT;
+}
+
 int setup_sge_fwevtq(struct adapter *adapter)
 {
 	struct sge *s = &adapter->sge;
@@ -169,6 +271,174 @@ int cxgb4_set_rspq_intr_params(struct sge_rspq *q, unsigned int us,
 	return 0;
 }
 
+/**
+ * Allocate an active-open TID and set it to the supplied value.
+ */
+int cxgbe_alloc_atid(struct tid_info *t, void *data)
+{
+	int atid = -1;
+
+	t4_os_lock(&t->atid_lock);
+	if (t->afree) {
+		union aopen_entry *p = t->afree;
+
+		atid = p - t->atid_tab;
+		t->afree = p->next;
+		p->data = data;
+		t->atids_in_use++;
+	}
+	t4_os_unlock(&t->atid_lock);
+	return atid;
+}
+
+/**
+ * Release an active-open TID.
+ */
+void cxgbe_free_atid(struct tid_info *t, unsigned int atid)
+{
+	union aopen_entry *p = &t->atid_tab[atid];
+
+	t4_os_lock(&t->atid_lock);
+	p->next = t->afree;
+	t->afree = p;
+	t->atids_in_use--;
+	t4_os_unlock(&t->atid_lock);
+}
+
+/**
+ * Populate a TID_RELEASE WR.  Caller must properly size the skb.
+ */
+static void mk_tid_release(struct rte_mbuf *mbuf, unsigned int tid)
+{
+	struct cpl_tid_release *req;
+
+	req = rte_pktmbuf_mtod(mbuf, struct cpl_tid_release *);
+	INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid);
+}
+
+/**
+ * Release a TID and inform HW.  If we are unable to allocate the release
+ * message we defer to a work queue.
+ */
+void cxgbe_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid,
+		      unsigned short family)
+{
+	struct rte_mbuf *mbuf;
+	struct adapter *adap = container_of(t, struct adapter, tids);
+
+	WARN_ON(tid >= t->ntids);
+
+	if (t->tid_tab[tid]) {
+		t->tid_tab[tid] = NULL;
+		rte_atomic32_dec(&t->conns_in_use);
+		if (t->hash_base && tid >= t->hash_base) {
+			if (family == FILTER_TYPE_IPV4)
+				rte_atomic32_dec(&t->hash_tids_in_use);
+		} else {
+			if (family == FILTER_TYPE_IPV4)
+				rte_atomic32_dec(&t->tids_in_use);
+		}
+	}
+
+	mbuf = rte_pktmbuf_alloc((&adap->sge.ctrlq[chan])->mb_pool);
+	if (mbuf) {
+		mbuf->data_len = sizeof(struct cpl_tid_release);
+		mbuf->pkt_len = mbuf->data_len;
+		mk_tid_release(mbuf, tid);
+		t4_mgmt_tx(&adap->sge.ctrlq[chan], mbuf);
+	}
+}
+
+/**
+ * Insert a TID.
+ */
+void cxgbe_insert_tid(struct tid_info *t, void *data, unsigned int tid,
+		      unsigned short family)
+{
+	t->tid_tab[tid] = data;
+	if (t->hash_base && tid >= t->hash_base) {
+		if (family == FILTER_TYPE_IPV4)
+			rte_atomic32_inc(&t->hash_tids_in_use);
+	} else {
+		if (family == FILTER_TYPE_IPV4)
+			rte_atomic32_inc(&t->tids_in_use);
+	}
+
+	rte_atomic32_inc(&t->conns_in_use);
+}
+
+/**
+ * Free TID tables.
+ */
+static void tid_free(struct tid_info *t)
+{
+	if (t->tid_tab) {
+		if (t->ftid_bmap)
+			rte_bitmap_free(t->ftid_bmap);
+
+		if (t->ftid_bmap_array)
+			t4_os_free(t->ftid_bmap_array);
+
+		t4_os_free(t->tid_tab);
+	}
+
+	memset(t, 0, sizeof(struct tid_info));
+}
+
+/**
+ * Allocate and initialize the TID tables.  Returns 0 on success.
+ */
+static int tid_init(struct tid_info *t)
+{
+	size_t size;
+	unsigned int ftid_bmap_size;
+	unsigned int natids = t->natids;
+	unsigned int max_ftids = t->nftids;
+
+	ftid_bmap_size = rte_bitmap_get_memory_footprint(t->nftids);
+	size = t->ntids * sizeof(*t->tid_tab) +
+		max_ftids * sizeof(*t->ftid_tab) +
+		natids * sizeof(*t->atid_tab);
+
+	t->tid_tab = t4_os_alloc(size);
+	if (!t->tid_tab)
+		return -ENOMEM;
+
+	t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
+	t->ftid_tab = (struct filter_entry *)&t->tid_tab[t->natids];
+	t->ftid_bmap_array = t4_os_alloc(ftid_bmap_size);
+	if (!t->ftid_bmap_array) {
+		tid_free(t);
+		return -ENOMEM;
+	}
+
+	t4_os_lock_init(&t->atid_lock);
+	t4_os_lock_init(&t->ftid_lock);
+
+	t->afree = NULL;
+	t->atids_in_use = 0;
+	rte_atomic32_init(&t->tids_in_use);
+	rte_atomic32_set(&t->tids_in_use, 0);
+	rte_atomic32_init(&t->conns_in_use);
+	rte_atomic32_set(&t->conns_in_use, 0);
+
+	/* Setup the free list for atid_tab and clear the stid bitmap. */
+	if (natids) {
+		while (--natids)
+			t->atid_tab[natids - 1].next = &t->atid_tab[natids];
+		t->afree = t->atid_tab;
+	}
+
+	t->ftid_bmap = rte_bitmap_init(t->nftids, t->ftid_bmap_array,
+				       ftid_bmap_size);
+	if (!t->ftid_bmap) {
+		tid_free(t);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
 static inline bool is_x_1g_port(const struct link_config *lc)
 {
 	return (lc->pcaps & FW_PORT_CAP32_SPEED_1G) != 0;
@@ -243,7 +513,7 @@ void cfg_queues(struct rte_eth_dev *eth_dev)
 		 * We default up to # of cores queues per 1G/10G port.
 		 */
 		if (nb_ports)
-			q_per_port = (MAX_ETH_QSETS -
+			q_per_port = (s->max_ethqsets -
 				     (adap->params.nports - nb_ports)) /
 				     nb_ports;
 
@@ -267,8 +537,6 @@ void cfg_queues(struct rte_eth_dev *eth_dev)
 			qidx += pi->n_rx_qsets;
 		}
 
-		s->max_ethqsets = qidx;
-
 		for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
 			struct sge_eth_rxq *r = &s->ethrxq[i];
 
@@ -500,6 +768,40 @@ static void configure_pcie_ext_tag(struct adapter *adapter)
 	}
 }
 
+/* Figure out how many Queue Sets we can support */
+void configure_max_ethqsets(struct adapter *adapter)
+{
+	unsigned int ethqsets;
+
+	/*
+	 * We need to reserve an Ingress Queue for the Asynchronous Firmware
+	 * Event Queue.
+	 *
+	 * For each Queue Set, we'll need the ability to allocate two Egress
+	 * Contexts -- one for the Ingress Queue Free List and one for the TX
+	 * Ethernet Queue.
+	 */
+	if (is_pf4(adapter)) {
+		struct pf_resources *pfres = &adapter->params.pfres;
+
+		ethqsets = pfres->niqflint - 1;
+		if (pfres->neq < ethqsets * 2)
+			ethqsets = pfres->neq / 2;
+	} else {
+		struct vf_resources *vfres = &adapter->params.vfres;
+
+		ethqsets = vfres->niqflint - 1;
+		if (vfres->nethctrl != ethqsets)
+			ethqsets = min(vfres->nethctrl, ethqsets);
+		if (vfres->neq < ethqsets * 2)
+			ethqsets = vfres->neq / 2;
+	}
+
+	if (ethqsets > MAX_ETH_QSETS)
+		ethqsets = MAX_ETH_QSETS;
+	adapter->sge.max_ethqsets = ethqsets;
+}
+
 /*
  * Tweak configuration based on system architecture, etc.  Most of these have
  * defaults assigned to them by Firmware Configuration Files (if we're using
@@ -638,8 +940,7 @@ static int adap_init0_config(struct adapter *adapter, int reset)
 	 * This will allow the firmware to optimize aspects of the hardware
 	 * configuration which will result in improved performance.
 	 */
-	caps_cmd.niccaps &= cpu_to_be16(~(FW_CAPS_CONFIG_NIC_HASHFILTER |
-					  FW_CAPS_CONFIG_NIC_ETHOFLD));
+	caps_cmd.niccaps &= cpu_to_be16(~FW_CAPS_CONFIG_NIC_ETHOFLD);
 	caps_cmd.toecaps = 0;
 	caps_cmd.iscsicaps = 0;
 	caps_cmd.rdmacaps = 0;
@@ -706,6 +1007,7 @@ bye:
 
 static int adap_init0(struct adapter *adap)
 {
+	struct fw_caps_config_cmd caps_cmd;
 	int ret = 0;
 	u32 v, port_vec;
 	enum dev_state state;
@@ -781,6 +1083,17 @@ static int adap_init0(struct adapter *adap)
 		goto bye;
 	}
 
+	/* Now that we've successfully configured and initialized the adapter
+	 * (or found it already initialized), we can ask the Firmware what
+	 * resources it has provisioned for us.
+	 */
+	ret = t4_get_pfres(adap);
+	if (ret) {
+		dev_err(adap->pdev_dev,
+			"Unable to retrieve resource provisioning info\n");
+		goto bye;
+	}
+
 	/* Find out what ports are available to us. */
 	v = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);
@@ -822,6 +1135,50 @@ static int adap_init0(struct adapter *adap)
 	 V_FW_PARAMS_PARAM_Y(0) | \
 	 V_FW_PARAMS_PARAM_Z(0))
 
+	params[0] = FW_PARAM_PFVF(FILTER_START);
+	params[1] = FW_PARAM_PFVF(FILTER_END);
+	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
+	if (ret < 0)
+		goto bye;
+	adap->tids.ftid_base = val[0];
+	adap->tids.nftids = val[1] - val[0] + 1;
+
+	params[0] = FW_PARAM_PFVF(CLIP_START);
+	params[1] = FW_PARAM_PFVF(CLIP_END);
+	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
+	if (ret < 0)
+		goto bye;
+	adap->clipt_start = val[0];
+	adap->clipt_end = val[1];
+
+	/*
+	 * Get device capabilities so we can determine what resources we need
+	 * to manage.
+	 */
+	memset(&caps_cmd, 0, sizeof(caps_cmd));
+	caps_cmd.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
+				     F_FW_CMD_REQUEST | F_FW_CMD_READ);
+	caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
+	ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
+			 &caps_cmd);
+	if (ret < 0)
+		goto bye;
+
+	if ((caps_cmd.niccaps & cpu_to_be16(FW_CAPS_CONFIG_NIC_HASHFILTER)) &&
+	    is_t6(adap->params.chip)) {
+		if (init_hash_filter(adap) < 0)
+			goto bye;
+	}
+
+	/* query tid-related parameters */
+	params[0] = FW_PARAM_DEV(NTID);
+	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1,
+			      params, val);
+	if (ret < 0)
+		goto bye;
+	adap->tids.ntids = val[0];
+	adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
+
 	/* If we're running on newer firmware, let it know that we're
 	 * prepared to deal with encapsulated CPL messages.  Older
 	 * firmware won't understand this and we'll just get
@@ -887,6 +1244,7 @@ static int adap_init0(struct adapter *adap)
 	t4_init_tp_params(adap);
 	configure_pcie_ext_tag(adap);
 	configure_vlan_types(adap);
+	configure_max_ethqsets(adap);
 
 	adap->params.drv_memwin = MEMWIN_NIC;
 	adap->flags |= FW_OK;
@@ -1027,7 +1385,7 @@ int cxgbe_write_rss_conf(const struct port_info *pi, uint64_t rss_hf)
 	if (rss_hf & ~CXGBE_RSS_HF_ALL)
 		return -EINVAL;
 
-	if (rss_hf & ETH_RSS_IPV4)
+	if (rss_hf & CXGBE_RSS_HF_IPV4_MASK)
 		flags |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
 
 	if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
@@ -1037,14 +1395,16 @@ int cxgbe_write_rss_conf(const struct port_info *pi, uint64_t rss_hf)
 		flags |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
 			 F_FW_RSS_VI_CONFIG_CMD_UDPEN;
 
-	if (rss_hf & ETH_RSS_IPV6)
+	if (rss_hf & CXGBE_RSS_HF_IPV6_MASK)
 		flags |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
 
-	if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
-		flags |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
+	if (rss_hf & CXGBE_RSS_HF_TCP_IPV6_MASK)
+		flags |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
+			 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
 
-	if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
-		flags |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
+	if (rss_hf & CXGBE_RSS_HF_UDP_IPV6_MASK)
+		flags |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
+			 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
 			 F_FW_RSS_VI_CONFIG_CMD_UDPEN;
 
 	rxq = &adapter->sge.ethrxq[pi->first_qset];
@@ -1259,6 +1619,30 @@ void cxgbe_get_speed_caps(struct port_info *pi, u32 *speed_caps)
 }
 
 /**
+ * cxgbe_set_link_status - Set device link up or down.
+ * @pi: Underlying port's info
+ * @status: 0 - down, 1 - up
+ *
+ * Set the device link up or down.
+ */
+int cxgbe_set_link_status(struct port_info *pi, bool status)
+{
+	struct adapter *adapter = pi->adapter;
+	int err = 0;
+
+	err = t4_enable_vi(adapter, adapter->mbox, pi->viid, status, status);
+	if (err) {
+		dev_err(adapter, "%s: disable_vi failed: %d\n", __func__, err);
+		return err;
+	}
+
+	if (!status)
+		t4_reset_link_config(adapter, pi->pidx);
+
+	return 0;
+}
+
+/**
  * cxgb_up - enable the adapter
  * @adap: adapter being enabled
  *
@@ -1283,17 +1667,7 @@ int cxgbe_up(struct adapter *adap)
  */
 int cxgbe_down(struct port_info *pi)
 {
-	struct adapter *adapter = pi->adapter;
-	int err = 0;
-
-	err = t4_enable_vi(adapter, adapter->mbox, pi->viid, false, false);
-	if (err) {
-		dev_err(adapter, "%s: disable_vi failed: %d\n", __func__, err);
-		return err;
-	}
-
-	t4_reset_link_config(adapter, pi->pidx);
-	return 0;
+	return cxgbe_set_link_status(pi, false);
 }
 
 /*
@@ -1307,6 +1681,8 @@ void cxgbe_close(struct adapter *adapter)
 	if (adapter->flags & FULL_INIT_DONE) {
 		if (is_pf4(adapter))
 			t4_intr_disable(adapter);
+		tid_free(&adapter->tids);
+		t4_cleanup_clip_tbl(adapter);
 		t4_sge_tx_monitor_stop(adapter);
 		t4_free_sge_resources(adapter);
 		for_each_port(adapter, i) {
@@ -1350,6 +1726,7 @@ int cxgbe_probe(struct adapter *adapter)
 
 	t4_os_lock_init(&adapter->mbox_lock);
 	TAILQ_INIT(&adapter->mbox_list);
+	t4_os_lock_init(&adapter->win0_lock);
 
 	err = t4_prep_adapter(adapter);
 	if (err)
@@ -1469,6 +1846,35 @@ allocate_mac:
 	print_adapter_info(adapter);
 	print_port_info(adapter);
 
+	adapter->clipt = t4_init_clip_tbl(adapter->clipt_start,
+					  adapter->clipt_end);
+	if (!adapter->clipt) {
+		/* We tolerate a lack of clip_table, giving up some
+		 * functionality
+		 */
+		dev_warn(adapter, "could not allocate CLIP. Continuing\n");
+	}
+
+	if (tid_init(&adapter->tids) < 0) {
+		/* Disable filtering support */
+		dev_warn(adapter, "could not allocate TID table, "
+			 "filter support disabled. Continuing\n");
+	}
+
+	if (is_hashfilter(adapter)) {
+		if (t4_read_reg(adapter, A_LE_DB_CONFIG) & F_HASHEN) {
+			u32 hash_base, hash_reg;
+
+			hash_reg = A_LE_DB_TID_HASHBASE;
+			hash_base = t4_read_reg(adapter, hash_reg);
+			adapter->tids.hash_base = hash_base / 4;
+		}
+	} else {
+		/* Disable hash filtering support */
+		dev_warn(adapter,
+			 "Maskless filter support disabled. Continuing\n");
+	}
+
 	err = init_rss(adapter);
 	if (err)
 		goto out_free;
diff --git a/drivers/net/cxgbe/cxgbe_ofld.h b/drivers/net/cxgbe/cxgbe_ofld.h
new file mode 100644
index 00000000..50931ed0
--- /dev/null
+++ b/drivers/net/cxgbe/cxgbe_ofld.h
@@ -0,0 +1,89 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+
+#ifndef _CXGBE_OFLD_H_
+#define _CXGBE_OFLD_H_
+
+#include <rte_bitmap.h>
+
+#include "cxgbe_filter.h"
+
+#define INIT_TP_WR(w, tid) do { \
+	(w)->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_TP_WR) | \
+				V_FW_WR_IMMDLEN(sizeof(*w) - sizeof(w->wr))); \
+	(w)->wr.wr_mid = cpu_to_be32( \
+				V_FW_WR_LEN16(DIV_ROUND_UP(sizeof(*w), 16)) | \
+				V_FW_WR_FLOWID(tid)); \
+	(w)->wr.wr_lo = cpu_to_be64(0); \
+} while (0)
+
+#define INIT_TP_WR_MIT_CPL(w, cpl, tid) do { \
+	INIT_TP_WR(w, tid); \
+	OPCODE_TID(w) = cpu_to_be32(MK_OPCODE_TID(cpl, tid)); \
+} while (0)
+
+#define INIT_ULPTX_WR(w, wrlen, atomic, tid) do { \
+	(w)->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR) | \
+				    V_FW_WR_ATOMIC(atomic)); \
+	(w)->wr.wr_mid = cpu_to_be32(V_FW_WR_LEN16(DIV_ROUND_UP(wrlen, 16)) | \
+				     V_FW_WR_FLOWID(tid)); \
+	(w)->wr.wr_lo = cpu_to_be64(0); \
+} while (0)
+
+/*
+ * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
+ */
+#define MAX_ATIDS 8192U
+
+union aopen_entry {
+	void *data;
+	union aopen_entry *next;
+};
+
+/*
+ * Holds the size, base address, free list start, etc of filter TID.
+ * The tables themselves are allocated dynamically.
+ */
+struct tid_info {
+	void **tid_tab;
+	unsigned int ntids;
+	struct filter_entry *ftid_tab;	/* Normal filters */
+	union aopen_entry *atid_tab;
+	struct rte_bitmap *ftid_bmap;
+	uint8_t *ftid_bmap_array;
+	unsigned int nftids, natids;
+	unsigned int ftid_base, hash_base;
+
+	union aopen_entry *afree;
+	unsigned int atids_in_use;
+
+	/* TIDs in the TCAM */
+	rte_atomic32_t tids_in_use;
+	/* TIDs in the HASH */
+	rte_atomic32_t hash_tids_in_use;
+	rte_atomic32_t conns_in_use;
+
+	rte_spinlock_t atid_lock __rte_cache_aligned;
+	rte_spinlock_t ftid_lock;
+};
+
+static inline void *lookup_tid(const struct tid_info *t, unsigned int tid)
+{
+	return tid < t->ntids ? t->tid_tab[tid] : NULL;
+}
+
+static inline void *lookup_atid(const struct tid_info *t, unsigned int atid)
+{
+	return atid < t->natids ? t->atid_tab[atid].data : NULL;
+}
+
+int cxgbe_alloc_atid(struct tid_info *t, void *data);
+void cxgbe_free_atid(struct tid_info *t, unsigned int atid);
+void cxgbe_remove_tid(struct tid_info *t, unsigned int qid, unsigned int tid,
+		      unsigned short family);
+void cxgbe_insert_tid(struct tid_info *t, void *data, unsigned int tid,
+		      unsigned short family);
+
+#endif /* _CXGBE_OFLD_H_ */
diff --git a/drivers/net/cxgbe/cxgbe_pfvf.h b/drivers/net/cxgbe/cxgbe_pfvf.h
index 2bba9742..8d0a105a 100644
--- a/drivers/net/cxgbe/cxgbe_pfvf.h
+++ b/drivers/net/cxgbe/cxgbe_pfvf.h
@@ -35,6 +35,8 @@ int cxgbe_dev_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu);
 int cxgbe_dev_start(struct rte_eth_dev *eth_dev);
 int cxgbe_dev_link_update(struct rte_eth_dev *eth_dev,
 			  int wait_to_complete);
+int cxgbe_dev_set_link_up(struct rte_eth_dev *dev);
+int cxgbe_dev_set_link_down(struct rte_eth_dev *dev);
 uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 			 uint16_t nb_pkts);
 uint16_t cxgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
diff --git a/drivers/net/cxgbe/cxgbevf_ethdev.c b/drivers/net/cxgbe/cxgbevf_ethdev.c
index a942ba6b..3b32ca9d 100644
--- a/drivers/net/cxgbe/cxgbevf_ethdev.c
+++ b/drivers/net/cxgbe/cxgbevf_ethdev.c
@@ -50,6 +50,7 @@ static int cxgbevf_dev_stats_get(struct rte_eth_dev *eth_dev,
 	/* TX Stats */
 	eth_stats->opackets = ps.tx_bcast_frames + ps.tx_mcast_frames +
 			      ps.tx_ucast_frames;
+	eth_stats->obytes = ps.tx_octets;
 	eth_stats->oerrors  = ps.tx_drop;
 
 	for (i = 0; i < pi->n_rx_qsets; i++) {
@@ -85,6 +86,8 @@ static const struct eth_dev_ops cxgbevf_eth_dev_ops = {
 	.dev_infos_get          = cxgbe_dev_info_get,
 	.dev_supported_ptypes_get = cxgbe_dev_supported_ptypes_get,
 	.link_update            = cxgbe_dev_link_update,
+	.dev_set_link_up        = cxgbe_dev_set_link_up,
+	.dev_set_link_down      = cxgbe_dev_set_link_down,
 	.mtu_set                = cxgbe_dev_mtu_set,
 	.tx_queue_setup         = cxgbe_dev_tx_queue_setup,
 	.tx_queue_start         = cxgbe_dev_tx_queue_start,
diff --git a/drivers/net/cxgbe/cxgbevf_main.c b/drivers/net/cxgbe/cxgbevf_main.c
index 5b3fb539..4214d031 100644
--- a/drivers/net/cxgbe/cxgbevf_main.c
+++ b/drivers/net/cxgbe/cxgbevf_main.c
@@ -20,7 +20,7 @@
 static void size_nports_qsets(struct adapter *adapter)
 {
 	struct vf_resources *vfres = &adapter->params.vfres;
-	unsigned int ethqsets, pmask_nports;
+	unsigned int pmask_nports;
 
 	/*
 	 * The number of "ports" which we support is equal to the number of
@@ -49,23 +49,7 @@ static void size_nports_qsets(struct adapter *adapter)
 		adapter->params.nports = pmask_nports;
 	}
 
-	/*
-	 * We need to reserve an Ingress Queue for the Asynchronous Firmware
-	 * Event Queue.
-	 *
-	 * For each Queue Set, we'll need the ability to allocate two Egress
-	 * Contexts -- one for the Ingress Queue Free List and one for the TX
-	 * Ethernet Queue.
-	 */
-	ethqsets = vfres->niqflint - 1;
-	if (vfres->nethctrl != ethqsets)
-		ethqsets = min(vfres->nethctrl, ethqsets);
-	if (vfres->neq < ethqsets * 2)
-		ethqsets = vfres->neq / 2;
-	if (ethqsets > MAX_ETH_QSETS)
-		ethqsets = MAX_ETH_QSETS;
-	adapter->sge.max_ethqsets = ethqsets;
-
+	configure_max_ethqsets(adapter);
 	if (adapter->sge.max_ethqsets < adapter->params.nports) {
 		dev_warn(adapter->pdev_dev, "only using %d of %d available"
 			 " virtual interfaces (too few Queue Sets)\n",
diff --git a/drivers/net/cxgbe/meson.build b/drivers/net/cxgbe/meson.build
new file mode 100644
index 00000000..7c69a34b
--- /dev/null
+++ b/drivers/net/cxgbe/meson.build
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+sources = files('cxgbe_ethdev.c',
+	'cxgbe_main.c',
+	'cxgbevf_ethdev.c',
+	'cxgbevf_main.c',
+	'sge.c',
+	'cxgbe_filter.c',
+	'cxgbe_flow.c',
+	'clip_tbl.c',
+	'base/t4_hw.c',
+	'base/t4vf_hw.c')
+includes += include_directories('base')
diff --git a/drivers/net/cxgbe/sge.c b/drivers/net/cxgbe/sge.c
index b5d3611d..4ea40d19 100644
--- a/drivers/net/cxgbe/sge.c
+++ b/drivers/net/cxgbe/sge.c
@@ -55,6 +55,11 @@ static inline void ship_tx_pkt_coalesce_wr(struct adapter *adap,
 #define MAX_IMM_TX_PKT_LEN 256
 
 /*
+ * Max size of a WR sent through a control Tx queue.
+ */
+#define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN
+
+/*
  * Rx buffer sizes for "usembufs" Free List buffers (one ingress packet
  * per mbuf buffer).  We currently only support two sizes for 1500- and
  * 9000-byte MTUs. We could easily support more but there doesn't seem to be
@@ -1300,6 +1305,126 @@ out_free:
 }
 
 /**
+ * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
+ * @q: the SGE control Tx queue
+ *
+ * This is a variant of reclaim_completed_tx() that is used for Tx queues
+ * that send only immediate data (presently just the control queues) and
+ * thus do not have any mbufs to release.
+ */
+static inline void reclaim_completed_tx_imm(struct sge_txq *q)
+{
+	int hw_cidx = ntohs(q->stat->cidx);
+	int reclaim = hw_cidx - q->cidx;
+
+	if (reclaim < 0)
+		reclaim += q->size;
+
+	q->in_use -= reclaim;
+	q->cidx = hw_cidx;
+}
+
+/**
+ * is_imm - check whether a packet can be sent as immediate data
+ * @mbuf: the packet
+ *
+ * Returns true if a packet can be sent as a WR with immediate data.
+ */
+static inline int is_imm(const struct rte_mbuf *mbuf)
+{
+	return mbuf->pkt_len <= MAX_CTRL_WR_LEN;
+}
+
+/**
+ * inline_tx_mbuf: inline a packet's data into TX descriptors
+ * @q: the TX queue where the packet will be inlined
+ * @from: pointer to data portion of packet
+ * @to: pointer after cpl where data has to be inlined
+ * @len: length of data to inline
+ *
+ * Inline a packet's contents directly to TX descriptors, starting at
+ * the given position within the TX DMA ring.
+ * Most of the complexity of this operation is dealing with wrap arounds
+ * in the middle of the packet we want to inline.
+ */
+static void inline_tx_mbuf(const struct sge_txq *q, caddr_t from, caddr_t *to,
+			   int len)
+{
+	int left = RTE_PTR_DIFF(q->stat, *to);
+
+	if (likely((uintptr_t)*to + len <= (uintptr_t)q->stat)) {
+		rte_memcpy(*to, from, len);
+		*to = RTE_PTR_ADD(*to, len);
+	} else {
+		rte_memcpy(*to, from, left);
+		from = RTE_PTR_ADD(from, left);
+		left = len - left;
+		rte_memcpy((void *)q->desc, from, left);
+		*to = RTE_PTR_ADD((void *)q->desc, left);
+	}
+}
+
+/**
+ * ctrl_xmit - send a packet through an SGE control Tx queue
+ * @q: the control queue
+ * @mbuf: the packet
+ *
+ * Send a packet through an SGE control Tx queue.  Packets sent through
+ * a control queue must fit entirely as immediate data.
+ */
+static int ctrl_xmit(struct sge_ctrl_txq *q, struct rte_mbuf *mbuf)
+{
+	unsigned int ndesc;
+	struct fw_wr_hdr *wr;
+	caddr_t dst;
+
+	if (unlikely(!is_imm(mbuf))) {
+		WARN_ON(1);
+		rte_pktmbuf_free(mbuf);
+		return -1;
+	}
+
+	reclaim_completed_tx_imm(&q->q);
+	ndesc = DIV_ROUND_UP(mbuf->pkt_len, sizeof(struct tx_desc));
+	t4_os_lock(&q->ctrlq_lock);
+
+	q->full = txq_avail(&q->q) < ndesc ? 1 : 0;
+	if (unlikely(q->full)) {
+		t4_os_unlock(&q->ctrlq_lock);
+		return -1;
+	}
+
+	wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx];
+	dst = (void *)wr;
+	inline_tx_mbuf(&q->q, rte_pktmbuf_mtod(mbuf, caddr_t),
+		       &dst, mbuf->data_len);
+
+	txq_advance(&q->q, ndesc);
+	if (unlikely(txq_avail(&q->q) < 64))
+		wr->lo |= htonl(F_FW_WR_EQUEQ);
+
+	q->txp++;
+
+	ring_tx_db(q->adapter, &q->q);
+	t4_os_unlock(&q->ctrlq_lock);
+
+	rte_pktmbuf_free(mbuf);
+	return 0;
+}
+
+/**
+ * t4_mgmt_tx - send a management message
+ * @q: the control queue
+ * @mbuf: the packet containing the management message
+ *
+ * Send a management message through control queue.
+ */
+int t4_mgmt_tx(struct sge_ctrl_txq *q, struct rte_mbuf *mbuf)
+{
+	return ctrl_xmit(q, mbuf);
+}
+
+/**
  * alloc_ring - allocate resources for an SGE descriptor ring
  * @dev: the PCI device's core device
  * @nelem: the number of descriptors
@@ -1764,12 +1889,16 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
 			    F_FW_CMD_WRITE | F_FW_CMD_EXEC);
 
 	if (is_pf4(adap)) {
-		pciechan = cong > 0 ? cxgbe_ffs(cong) - 1 : pi->tx_chan;
+		pciechan = pi->tx_chan;
 		c.op_to_vfn |= htonl(V_FW_IQ_CMD_PFN(adap->pf) |
 				     V_FW_IQ_CMD_VFN(0));
 		if (cong >= 0)
-			c.iqns_to_fl0congen = htonl(F_FW_IQ_CMD_IQFLINTCONGEN |
-						    F_FW_IQ_CMD_IQRO);
+			c.iqns_to_fl0congen =
+				htonl(F_FW_IQ_CMD_IQFLINTCONGEN |
+				      V_FW_IQ_CMD_IQTYPE(cong ?
+							 FW_IQ_IQTYPE_NIC :
+							 FW_IQ_IQTYPE_OFLD) |
+				      F_FW_IQ_CMD_IQRO);
 	} else {
 		pciechan = pi->port_id;
 	}
@@ -2080,6 +2209,64 @@ int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
 	return 0;
 }
 
+int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
+			  struct rte_eth_dev *eth_dev, uint16_t queue_id,
+			  unsigned int iqid, int socket_id)
+{
+	int ret, nentries;
+	struct fw_eq_ctrl_cmd c;
+	struct sge *s = &adap->sge;
+	struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+	char z_name[RTE_MEMZONE_NAMESIZE];
+	char z_name_sw[RTE_MEMZONE_NAMESIZE];
+
+	/* Add status entries */
+	nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
+
+	snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
+		 eth_dev->device->driver->name, "ctrl_tx_ring",
+		 eth_dev->data->port_id, queue_id);
+	snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
+
+	txq->q.desc = alloc_ring(txq->q.size, sizeof(struct tx_desc),
+				 0, &txq->q.phys_addr,
+				 NULL, 0, queue_id,
+				 socket_id, z_name, z_name_sw);
+	if (!txq->q.desc)
+		return -ENOMEM;
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
+			    F_FW_CMD_WRITE | F_FW_CMD_EXEC |
+			    V_FW_EQ_CTRL_CMD_PFN(adap->pf) |
+			    V_FW_EQ_CTRL_CMD_VFN(0));
+	c.alloc_to_len16 = htonl(F_FW_EQ_CTRL_CMD_ALLOC |
+				 F_FW_EQ_CTRL_CMD_EQSTART | (sizeof(c) / 16));
+	c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(0));
+	c.physeqid_pkd = htonl(0);
+	c.fetchszm_to_iqid =
+		htonl(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
+		      V_FW_EQ_CTRL_CMD_PCIECHN(pi->tx_chan) |
+		      F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(iqid));
+	c.dcaen_to_eqsize =
+		htonl(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
+		      V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
+		      V_FW_EQ_CTRL_CMD_EQSIZE(nentries));
+	c.eqaddr = cpu_to_be64(txq->q.phys_addr);
+
+	ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
+	if (ret) {
+		txq->q.desc = NULL;
+		return ret;
+	}
+
+	init_txq(adap, &txq->q, G_FW_EQ_CTRL_CMD_EQID(ntohl(c.cmpliqid_eqid)),
+		 G_FW_EQ_CTRL_CMD_EQID(ntohl(c. physeqid_pkd)));
+	txq->adapter = adap;
+	txq->full = 0;
+	return 0;
+}
+
 static void free_txq(struct sge_txq *q)
 {
 	q->cntxt_id = 0;
@@ -2174,7 +2361,7 @@ void t4_sge_tx_monitor_stop(struct adapter *adap)
  */
 void t4_free_sge_resources(struct adapter *adap)
 {
-	int i;
+	unsigned int i;
 	struct sge_eth_rxq *rxq = &adap->sge.ethrxq[0];
 	struct sge_eth_txq *txq = &adap->sge.ethtxq[0];
 
@@ -2191,6 +2378,18 @@ void t4_free_sge_resources(struct adapter *adap)
 		}
 	}
 
+	/* clean up control Tx queues */
+	for (i = 0; i < ARRAY_SIZE(adap->sge.ctrlq); i++) {
+		struct sge_ctrl_txq *cq = &adap->sge.ctrlq[i];
+
+		if (cq->q.desc) {
+			reclaim_completed_tx_imm(&cq->q);
+			t4_ctrl_eq_free(adap, adap->mbox, adap->pf, 0,
+					cq->q.cntxt_id);
+			free_txq(&cq->q);
+		}
+	}
+
 	if (adap->sge.fw_evtq.desc)
 		free_rspq_fl(adap, &adap->sge.fw_evtq, NULL);
 }
diff --git a/drivers/net/dpaa/dpaa_ethdev.c b/drivers/net/dpaa/dpaa_ethdev.c
index d014a11a..7a950ac0 100644
--- a/drivers/net/dpaa/dpaa_ethdev.c
+++ b/drivers/net/dpaa/dpaa_ethdev.c
@@ -74,6 +74,7 @@ static uint64_t dev_tx_offloads_nodis =
 
 /* Keep track of whether QMAN and BMAN have been globally initialized */
 static int is_global_init;
+static int default_q;	/* use default queue - FMC is not executed*/
 /* At present we only allow up to 4 push mode queues as default - as each of
  * this queue need dedicated portal and we are short of portals.
  */
@@ -516,7 +517,15 @@ int dpaa_eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 
 	PMD_INIT_FUNC_TRACE();
 
-	DPAA_PMD_INFO("Rx queue setup for queue index: %d", queue_idx);
+	if (queue_idx >= dev->data->nb_rx_queues) {
+		rte_errno = EOVERFLOW;
+		DPAA_PMD_ERR("%p: queue index out of range (%u >= %u)",
+		      (void *)dev, queue_idx, dev->data->nb_rx_queues);
+		return -rte_errno;
+	}
+
+	DPAA_PMD_INFO("Rx queue setup for queue index: %d fq_id (0x%x)",
+			queue_idx, rxq->fqid);
 
 	if (!dpaa_intf->bp_info || dpaa_intf->bp_info->mp != mp) {
 		struct fman_if_ic_params icp;
@@ -580,9 +589,11 @@ int dpaa_eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 			opts.fqd.fq_ctrl |= QM_FQCTRL_CGE;
 		}
 		ret = qman_init_fq(rxq, flags, &opts);
-		if (ret)
-			DPAA_PMD_ERR("Channel/Queue association failed. fqid %d"
-				     " ret: %d", rxq->fqid, ret);
+		if (ret) {
+			DPAA_PMD_ERR("Channel/Q association failed. fqid 0x%x "
+				"ret:%d(%s)", rxq->fqid, ret, strerror(ret));
+			return ret;
+		}
 		rxq->cb.dqrr_dpdk_pull_cb = dpaa_rx_cb;
 		rxq->cb.dqrr_prepare = dpaa_rx_cb_prepare;
 		rxq->is_static = true;
@@ -606,7 +617,7 @@ int dpaa_eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	return 0;
 }
 
-int __rte_experimental
+int
 dpaa_eth_eventq_attach(const struct rte_eth_dev *dev,
 		int eth_rx_queue_id,
 		u16 ch_id,
@@ -657,8 +668,8 @@ dpaa_eth_eventq_attach(const struct rte_eth_dev *dev,
 
 	ret = qman_init_fq(rxq, flags, &opts);
 	if (ret) {
-		DPAA_PMD_ERR("Channel/Queue association failed. fqid %d ret:%d",
-			     rxq->fqid, ret);
+		DPAA_PMD_ERR("Ev-Channel/Q association failed. fqid 0x%x "
+				"ret:%d(%s)", rxq->fqid, ret, strerror(ret));
 		return ret;
 	}
 
@@ -669,7 +680,7 @@ dpaa_eth_eventq_attach(const struct rte_eth_dev *dev,
 	return ret;
 }
 
-int __rte_experimental
+int
 dpaa_eth_eventq_detach(const struct rte_eth_dev *dev,
 		int eth_rx_queue_id)
 {
@@ -715,7 +726,15 @@ int dpaa_eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 
 	PMD_INIT_FUNC_TRACE();
 
-	DPAA_PMD_INFO("Tx queue setup for queue index: %d", queue_idx);
+	if (queue_idx >= dev->data->nb_tx_queues) {
+		rte_errno = EOVERFLOW;
+		DPAA_PMD_ERR("%p: queue index out of range (%u >= %u)",
+		      (void *)dev, queue_idx, dev->data->nb_tx_queues);
+		return -rte_errno;
+	}
+
+	DPAA_PMD_INFO("Tx queue setup for queue index: %d fq_id (0x%x)",
+			queue_idx, dpaa_intf->tx_queues[queue_idx].fqid);
 	dev->data->tx_queues[queue_idx] = &dpaa_intf->tx_queues[queue_idx];
 	return 0;
 }
@@ -937,7 +956,7 @@ is_dpaa_supported(struct rte_eth_dev *dev)
 	return is_device_supported(dev, &rte_dpaa_pmd);
 }
 
-int __rte_experimental
+int
 rte_pmd_dpaa_set_tx_loopback(uint8_t port, uint8_t on)
 {
 	struct rte_eth_dev *dev;
@@ -1008,15 +1027,15 @@ static int dpaa_rx_queue_init(struct qman_fq *fq, struct qman_cgr *cgr_rx,
 
 	ret = qman_reserve_fqid(fqid);
 	if (ret) {
-		DPAA_PMD_ERR("reserve rx fqid %d failed with ret: %d",
+		DPAA_PMD_ERR("reserve rx fqid 0x%x failed with ret: %d",
 			     fqid, ret);
 		return -EINVAL;
 	}
 
-	DPAA_PMD_DEBUG("creating rx fq %p, fqid %d", fq, fqid);
+	DPAA_PMD_DEBUG("creating rx fq %p, fqid 0x%x", fq, fqid);
 	ret = qman_create_fq(fqid, QMAN_FQ_FLAG_NO_ENQUEUE, fq);
 	if (ret) {
-		DPAA_PMD_ERR("create rx fqid %d failed with ret: %d",
+		DPAA_PMD_ERR("create rx fqid 0x%x failed with ret: %d",
 			fqid, ret);
 		return ret;
 	}
@@ -1032,7 +1051,7 @@ static int dpaa_rx_queue_init(struct qman_fq *fq, struct qman_cgr *cgr_rx,
 				      &cgr_opts);
 		if (ret) {
 			DPAA_PMD_WARN(
-				"rx taildrop init fail on rx fqid %d (ret=%d)",
+				"rx taildrop init fail on rx fqid 0x%x(ret=%d)",
 				fqid, ret);
 			goto without_cgr;
 		}
@@ -1043,7 +1062,7 @@ static int dpaa_rx_queue_init(struct qman_fq *fq, struct qman_cgr *cgr_rx,
 without_cgr:
 	ret = qman_init_fq(fq, flags, &opts);
 	if (ret)
-		DPAA_PMD_ERR("init rx fqid %d failed with ret: %d", fqid, ret);
+		DPAA_PMD_ERR("init rx fqid 0x%x failed with ret:%d", fqid, ret);
 	return ret;
 }
 
@@ -1071,10 +1090,10 @@ static int dpaa_tx_queue_init(struct qman_fq *fq,
 	/* no tx-confirmation */
 	opts.fqd.context_a.hi = 0x80000000 | fman_dealloc_bufs_mask_hi;
 	opts.fqd.context_a.lo = 0 | fman_dealloc_bufs_mask_lo;
-	DPAA_PMD_DEBUG("init tx fq %p, fqid %d", fq, fq->fqid);
+	DPAA_PMD_DEBUG("init tx fq %p, fqid 0x%x", fq, fq->fqid);
 	ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts);
 	if (ret)
-		DPAA_PMD_ERR("init tx fqid %d failed %d", fq->fqid, ret);
+		DPAA_PMD_ERR("init tx fqid 0x%x failed %d", fq->fqid, ret);
 	return ret;
 }
 
@@ -1145,21 +1164,16 @@ dpaa_dev_init(struct rte_eth_dev *eth_dev)
 	dpaa_intf->cfg = cfg;
 
 	/* Initialize Rx FQ's */
-	if (getenv("DPAA_NUM_RX_QUEUES"))
-		num_rx_fqs = atoi(getenv("DPAA_NUM_RX_QUEUES"));
-	else
+	if (default_q) {
 		num_rx_fqs = DPAA_DEFAULT_NUM_PCD_QUEUES;
-
-	/* if push mode queues to be enabled. Currenly we are allowing only
-	 * one queue per thread.
-	 */
-	if (getenv("DPAA_PUSH_QUEUES_NUMBER")) {
-		dpaa_push_mode_max_queue =
-				atoi(getenv("DPAA_PUSH_QUEUES_NUMBER"));
-		if (dpaa_push_mode_max_queue > DPAA_MAX_PUSH_MODE_QUEUE)
-			dpaa_push_mode_max_queue = DPAA_MAX_PUSH_MODE_QUEUE;
+	} else {
+		if (getenv("DPAA_NUM_RX_QUEUES"))
+			num_rx_fqs = atoi(getenv("DPAA_NUM_RX_QUEUES"));
+		else
+			num_rx_fqs = DPAA_DEFAULT_NUM_PCD_QUEUES;
 	}
 
+
 	/* Each device can not have more than DPAA_MAX_NUM_PCD_QUEUES RX
 	 * queues.
 	 */
@@ -1196,8 +1210,11 @@ dpaa_dev_init(struct rte_eth_dev *eth_dev)
 	}
 
 	for (loop = 0; loop < num_rx_fqs; loop++) {
-		fqid = DPAA_PCD_FQID_START + dpaa_intf->ifid *
-			DPAA_PCD_FQID_MULTIPLIER + loop;
+		if (default_q)
+			fqid = cfg->rx_def;
+		else
+			fqid = DPAA_PCD_FQID_START + dpaa_intf->ifid *
+				DPAA_PCD_FQID_MULTIPLIER + loop;
 
 		if (dpaa_intf->cgr_rx)
 			dpaa_intf->cgr_rx[loop].cgrid = cgrid[loop];
@@ -1372,6 +1389,8 @@ rte_dpaa_probe(struct rte_dpaa_driver *dpaa_drv,
 		eth_dev = rte_eth_dev_attach_secondary(dpaa_dev->name);
 		if (!eth_dev)
 			return -ENOMEM;
+		eth_dev->device = &dpaa_dev->device;
+		eth_dev->dev_ops = &dpaa_devops;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -1391,6 +1410,26 @@ rte_dpaa_probe(struct rte_dpaa_driver *dpaa_drv,
 			return ret;
 		}
 
+		if (access("/tmp/fmc.bin", F_OK) == -1) {
+			RTE_LOG(INFO, PMD,
+				"* FMC not configured.Enabling default mode\n");
+			default_q = 1;
+		}
+
+		/* disabling the default push mode for LS1043 */
+		if (dpaa_svr_family == SVR_LS1043A_FAMILY)
+			dpaa_push_mode_max_queue = 0;
+
+		/* if push mode queues to be enabled. Currenly we are allowing
+		 * only one queue per thread.
+		 */
+		if (getenv("DPAA_PUSH_QUEUES_NUMBER")) {
+			dpaa_push_mode_max_queue =
+					atoi(getenv("DPAA_PUSH_QUEUES_NUMBER"));
+			if (dpaa_push_mode_max_queue > DPAA_MAX_PUSH_MODE_QUEUE)
+			    dpaa_push_mode_max_queue = DPAA_MAX_PUSH_MODE_QUEUE;
+		}
+
 		is_global_init = 1;
 	}
 
diff --git a/drivers/net/dpaa/dpaa_ethdev.h b/drivers/net/dpaa/dpaa_ethdev.h
index 1897b9e4..c79b9f86 100644
--- a/drivers/net/dpaa/dpaa_ethdev.h
+++ b/drivers/net/dpaa/dpaa_ethdev.h
@@ -160,12 +160,14 @@ struct dpaa_if_stats {
 	uint64_t tund;		/**<Tx Undersized */
 };
 
-int __rte_experimental dpaa_eth_eventq_attach(const struct rte_eth_dev *dev,
-			   int eth_rx_queue_id,
+int
+dpaa_eth_eventq_attach(const struct rte_eth_dev *dev,
+		int eth_rx_queue_id,
 		u16 ch_id,
 		const struct rte_event_eth_rx_adapter_queue_conf *queue_conf);
 
-int __rte_experimental dpaa_eth_eventq_detach(const struct rte_eth_dev *dev,
+int
+dpaa_eth_eventq_detach(const struct rte_eth_dev *dev,
 			   int eth_rx_queue_id);
 
 enum qman_cb_dqrr_result
diff --git a/drivers/net/dpaa/dpaa_rxtx.c b/drivers/net/dpaa/dpaa_rxtx.c
index 1316d2ad..168b77e4 100644
--- a/drivers/net/dpaa/dpaa_rxtx.c
+++ b/drivers/net/dpaa/dpaa_rxtx.c
@@ -431,7 +431,7 @@ dpaa_rx_cb(struct qman_fq **fq, struct qm_dqrr_entry **dqrr,
 		}
 
 		fd = &dqrr[i]->fd;
-		dpaa_intf = fq[i]->dpaa_intf;
+		dpaa_intf = fq[0]->dpaa_intf;
 
 		format = (fd->opaque & DPAA_FD_FORMAT_MASK) >>
 				DPAA_FD_FORMAT_SHIFT;
@@ -560,7 +560,8 @@ uint16_t dpaa_eth_queue_rx(void *q,
 	struct qman_fq *fq = q;
 	struct qm_dqrr_entry *dq;
 	uint32_t num_rx = 0, ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid;
-	int ret;
+	int num_rx_bufs, ret;
+	uint32_t vdqcr_flags = 0;
 
 	if (likely(fq->is_static))
 		return dpaa_eth_queue_portal_rx(fq, bufs, nb_bufs);
@@ -573,8 +574,19 @@ uint16_t dpaa_eth_queue_rx(void *q,
 		}
 	}
 
-	ret = qman_set_vdq(fq, (nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES) ?
-				DPAA_MAX_DEQUEUE_NUM_FRAMES : nb_bufs);
+	/* Until request for four buffers, we provide exact number of buffers.
+	 * Otherwise we do not set the QM_VDQCR_EXACT flag.
+	 * Not setting QM_VDQCR_EXACT flag can provide two more buffers than
+	 * requested, so we request two less in this case.
+	 */
+	if (nb_bufs < 4) {
+		vdqcr_flags = QM_VDQCR_EXACT;
+		num_rx_bufs = nb_bufs;
+	} else {
+		num_rx_bufs = nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES ?
+			(DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_bufs - 2);
+	}
+	ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags);
 	if (ret)
 		return 0;
 
diff --git a/drivers/net/dpaa/rte_pmd_dpaa.h b/drivers/net/dpaa/rte_pmd_dpaa.h
index 38405ec0..37eea9b0 100644
--- a/drivers/net/dpaa/rte_pmd_dpaa.h
+++ b/drivers/net/dpaa/rte_pmd_dpaa.h
@@ -18,9 +18,6 @@
 #include <rte_ethdev_driver.h>
 
 /**
- * @warning
- * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
- *
  * Enable/Disable TX loopback
  *
  * @param port
@@ -33,7 +30,7 @@
  *   - (-ENODEV) if *port* invalid.
  *   - (-EINVAL) if bad parameter.
  */
-int __rte_experimental
+int
 rte_pmd_dpaa_set_tx_loopback(uint8_t port, uint8_t on);
 
 #endif /* _PMD_DPAA_H_ */
diff --git a/drivers/net/dpaa/rte_pmd_dpaa_version.map b/drivers/net/dpaa/rte_pmd_dpaa_version.map
index c7ad4030..8cb4500b 100644
--- a/drivers/net/dpaa/rte_pmd_dpaa_version.map
+++ b/drivers/net/dpaa/rte_pmd_dpaa_version.map
@@ -3,10 +3,10 @@ DPDK_17.11 {
 	local: *;
 };
 
-EXPERIMENTAL {
+DPDK_18.08 {
 	global:
 
 	dpaa_eth_eventq_attach;
 	dpaa_eth_eventq_detach;
 	rte_pmd_dpaa_set_tx_loopback;
-};
+} DPDK_17.11;
diff --git a/drivers/net/dpaa2/dpaa2_ethdev.c b/drivers/net/dpaa2/dpaa2_ethdev.c
index 9297725d..c5047367 100644
--- a/drivers/net/dpaa2/dpaa2_ethdev.c
+++ b/drivers/net/dpaa2/dpaa2_ethdev.c
@@ -2053,9 +2053,7 @@ static struct rte_dpaa2_driver rte_dpaa2_pmd = {
 
 RTE_PMD_REGISTER_DPAA2(net_dpaa2, rte_dpaa2_pmd);
 
-RTE_INIT(dpaa2_pmd_init_log);
-static void
-dpaa2_pmd_init_log(void)
+RTE_INIT(dpaa2_pmd_init_log)
 {
 	dpaa2_logtype_pmd = rte_log_register("pmd.net.dpaa2");
 	if (dpaa2_logtype_pmd >= 0)
diff --git a/drivers/net/dpaa2/dpaa2_pmd_logs.h b/drivers/net/dpaa2/dpaa2_pmd_logs.h
index 98a48968..c04babdb 100644
--- a/drivers/net/dpaa2/dpaa2_pmd_logs.h
+++ b/drivers/net/dpaa2/dpaa2_pmd_logs.h
@@ -16,7 +16,7 @@ extern int dpaa2_logtype_pmd;
 	rte_log(RTE_LOG_DEBUG, dpaa2_logtype_pmd, "dpaa2_net: %s(): "\
 		fmt "\n", __func__, ##args)
 
-#define PMD_INIT_FUNC_TRACE() DPAA2_PMD_LOG(DEBUG, " >>")
+#define PMD_INIT_FUNC_TRACE() DPAA2_PMD_DEBUG(">>")
 
 #define DPAA2_PMD_CRIT(fmt, args...) \
 	DPAA2_PMD_LOG(CRIT, fmt, ## args)
diff --git a/drivers/net/dpaa2/dpaa2_rxtx.c b/drivers/net/dpaa2/dpaa2_rxtx.c
index dac086d6..ef109a62 100644
--- a/drivers/net/dpaa2/dpaa2_rxtx.c
+++ b/drivers/net/dpaa2/dpaa2_rxtx.c
@@ -447,6 +447,12 @@ eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf,
 return 0;
 }
 
+/* This function assumes that caller will be keep the same value for nb_pkts
+ * across calls per queue, if that is not the case, better use non-prefetch
+ * version of rx call.
+ * It will return the packets as requested in previous call without honoring
+ * the current nb_pkts or bufs space.
+ */
 uint16_t
 dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 {
@@ -454,7 +460,7 @@ dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 	struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 	struct qbman_result *dq_storage, *dq_storage1 = NULL;
 	uint32_t fqid = dpaa2_q->fqid;
-	int ret, num_rx = 0;
+	int ret, num_rx = 0, pull_size;
 	uint8_t pending, status;
 	struct qbman_swp *swp;
 	const struct qbman_fd *fd, *next_fd;
@@ -470,12 +476,12 @@ dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		}
 	}
 	swp = DPAA2_PER_LCORE_ETHRX_PORTAL;
-
+	pull_size = (nb_pkts > DPAA2_DQRR_RING_SIZE) ?
+					       DPAA2_DQRR_RING_SIZE : nb_pkts;
 	if (unlikely(!q_storage->active_dqs)) {
 		q_storage->toggle = 0;
 		dq_storage = q_storage->dq_storage[q_storage->toggle];
-		q_storage->last_num_pkts = (nb_pkts > DPAA2_DQRR_RING_SIZE) ?
-					       DPAA2_DQRR_RING_SIZE : nb_pkts;
+		q_storage->last_num_pkts = pull_size;
 		qbman_pull_desc_clear(&pulldesc);
 		qbman_pull_desc_set_numframes(&pulldesc,
 					      q_storage->last_num_pkts);
@@ -514,7 +520,7 @@ dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 	q_storage->toggle ^= 1;
 	dq_storage1 = q_storage->dq_storage[q_storage->toggle];
 	qbman_pull_desc_clear(&pulldesc);
-	qbman_pull_desc_set_numframes(&pulldesc, DPAA2_DQRR_RING_SIZE);
+	qbman_pull_desc_set_numframes(&pulldesc, pull_size);
 	qbman_pull_desc_set_fq(&pulldesc, fqid);
 	qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
 		(uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
diff --git a/drivers/net/dpaa2/mc/dpni.c b/drivers/net/dpaa2/mc/dpni.c
index 69cf119c..9f228169 100644
--- a/drivers/net/dpaa2/mc/dpni.c
+++ b/drivers/net/dpaa2/mc/dpni.c
@@ -198,7 +198,7 @@ int dpni_set_pools(struct fsl_mc_io *mc_io,
 					  token);
 	cmd_params = (struct dpni_cmd_set_pools *)cmd.params;
 	cmd_params->num_dpbp = cfg->num_dpbp;
-	for (i = 0; i < DPNI_MAX_DPBP; i++) {
+	for (i = 0; i < cmd_params->num_dpbp; i++) {
 		cmd_params->pool[i].dpbp_id =
 			cpu_to_le16(cfg->pools[i].dpbp_id);
 		cmd_params->pool[i].priority_mask =
diff --git a/drivers/net/e1000/em_ethdev.c b/drivers/net/e1000/em_ethdev.c
index 7039dc10..053e855b 100644
--- a/drivers/net/e1000/em_ethdev.c
+++ b/drivers/net/e1000/em_ethdev.c
@@ -1823,9 +1823,7 @@ RTE_PMD_REGISTER_PCI_TABLE(net_e1000_em, pci_id_em_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_em, "* igb_uio | uio_pci_generic | vfio-pci");
 
 /* see e1000_logs.c */
-RTE_INIT(igb_init_log);
-static void
-igb_init_log(void)
+RTE_INIT(igb_init_log)
 {
 	e1000_igb_init_log();
 }
diff --git a/drivers/net/e1000/em_rxtx.c b/drivers/net/e1000/em_rxtx.c
index a6b3e92a..7d2ac4eb 100644
--- a/drivers/net/e1000/em_rxtx.c
+++ b/drivers/net/e1000/em_rxtx.c
@@ -1364,6 +1364,7 @@ em_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
 		DEV_RX_OFFLOAD_UDP_CKSUM   |
 		DEV_RX_OFFLOAD_TCP_CKSUM   |
 		DEV_RX_OFFLOAD_CRC_STRIP   |
+		DEV_RX_OFFLOAD_KEEP_CRC    |
 		DEV_RX_OFFLOAD_SCATTER;
 	if (max_rx_pktlen > ETHER_MAX_LEN)
 		rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME;
@@ -1458,8 +1459,10 @@ eth_em_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->rx_free_thresh = rx_conf->rx_free_thresh;
 	rxq->queue_id = queue_idx;
 	rxq->port_id = dev->data->port_id;
-	rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.offloads &
-		DEV_RX_OFFLOAD_CRC_STRIP) ? 0 : ETHER_CRC_LEN);
+	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+		rxq->crc_len = ETHER_CRC_LEN;
+	else
+		rxq->crc_len = 0;
 
 	rxq->rdt_reg_addr = E1000_PCI_REG_ADDR(hw, E1000_RDT(queue_idx));
 	rxq->rdh_reg_addr = E1000_PCI_REG_ADDR(hw, E1000_RDH(queue_idx));
@@ -1792,9 +1795,10 @@ eth_em_rx_init(struct rte_eth_dev *dev)
 		 * Reset crc_len in case it was changed after queue setup by a
 		 *  call to configure
 		 */
-		rxq->crc_len =
-			(uint8_t)(dev->data->dev_conf.rxmode.offloads &
-				DEV_RX_OFFLOAD_CRC_STRIP ? 0 : ETHER_CRC_LEN);
+		if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+			rxq->crc_len = ETHER_CRC_LEN;
+		else
+			rxq->crc_len = 0;
 
 		bus_addr = rxq->rx_ring_phys_addr;
 		E1000_WRITE_REG(hw, E1000_RDLEN(i),
@@ -1873,10 +1877,10 @@ eth_em_rx_init(struct rte_eth_dev *dev)
 	}
 
 	/* Setup the Receive Control Register. */
-	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP)
-		rctl |= E1000_RCTL_SECRC; /* Strip Ethernet CRC. */
-	else
+	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
 		rctl &= ~E1000_RCTL_SECRC; /* Do not Strip Ethernet CRC. */
+	else
+		rctl |= E1000_RCTL_SECRC; /* Strip Ethernet CRC. */
 
 	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
 	rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
diff --git a/drivers/net/e1000/igb_ethdev.c b/drivers/net/e1000/igb_ethdev.c
index edc7be31..64dfe683 100644
--- a/drivers/net/e1000/igb_ethdev.c
+++ b/drivers/net/e1000/igb_ethdev.c
@@ -435,6 +435,9 @@ static const struct eth_dev_ops igbvf_eth_dev_ops = {
 	.dev_supported_ptypes_get = eth_igb_supported_ptypes_get,
 	.rx_queue_setup       = eth_igb_rx_queue_setup,
 	.rx_queue_release     = eth_igb_rx_queue_release,
+	.rx_descriptor_done   = eth_igb_rx_descriptor_done,
+	.rx_descriptor_status = eth_igb_rx_descriptor_status,
+	.tx_descriptor_status = eth_igb_tx_descriptor_status,
 	.tx_queue_setup       = eth_igb_tx_queue_setup,
 	.tx_queue_release     = eth_igb_tx_queue_release,
 	.set_mc_addr_list     = eth_igb_set_mc_addr_list,
@@ -3194,12 +3197,12 @@ igbvf_dev_configure(struct rte_eth_dev *dev)
 	 * Keep the persistent behavior the same as Host PF
 	 */
 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
-	if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP)) {
+	if (rte_eth_dev_must_keep_crc(conf->rxmode.offloads)) {
 		PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
 		conf->rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
 	}
 #else
-	if (conf->rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP) {
+	if (!rte_eth_dev_must_keep_crc(conf->rxmode.offloads)) {
 		PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
 		conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_CRC_STRIP;
 	}
@@ -5683,9 +5686,7 @@ RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci");
 
 /* see e1000_logs.c */
-RTE_INIT(e1000_init_log);
-static void
-e1000_init_log(void)
+RTE_INIT(e1000_init_log)
 {
 	e1000_igb_init_log();
 }
diff --git a/drivers/net/e1000/igb_rxtx.c b/drivers/net/e1000/igb_rxtx.c
index 5f729f27..b955068a 100644
--- a/drivers/net/e1000/igb_rxtx.c
+++ b/drivers/net/e1000/igb_rxtx.c
@@ -1639,6 +1639,7 @@ igb_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
 			  DEV_RX_OFFLOAD_TCP_CKSUM   |
 			  DEV_RX_OFFLOAD_JUMBO_FRAME |
 			  DEV_RX_OFFLOAD_CRC_STRIP   |
+			  DEV_RX_OFFLOAD_KEEP_CRC    |
 			  DEV_RX_OFFLOAD_SCATTER;
 
 	return rx_offload_capa;
@@ -1720,8 +1721,10 @@ eth_igb_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
 		queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
 	rxq->port_id = dev->data->port_id;
-	rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.offloads &
-			DEV_RX_OFFLOAD_CRC_STRIP) ? 0 : ETHER_CRC_LEN);
+	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+		rxq->crc_len = ETHER_CRC_LEN;
+	else
+		rxq->crc_len = 0;
 
 	/*
 	 *  Allocate RX ring hardware descriptors. A memzone large enough to
@@ -2371,8 +2374,10 @@ eth_igb_rx_init(struct rte_eth_dev *dev)
 		 * Reset crc_len in case it was changed after queue setup by a
 		 *  call to configure
 		 */
-		rxq->crc_len = (uint8_t)(dev->data->dev_conf.rxmode.offloads &
-				DEV_RX_OFFLOAD_CRC_STRIP ? 0 : ETHER_CRC_LEN);
+		if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+			rxq->crc_len = ETHER_CRC_LEN;
+		else
+			rxq->crc_len = 0;
 
 		bus_addr = rxq->rx_ring_phys_addr;
 		E1000_WRITE_REG(hw, E1000_RDLEN(rxq->reg_idx),
@@ -2501,10 +2506,10 @@ eth_igb_rx_init(struct rte_eth_dev *dev)
 	E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
 
 	/* Setup the Receive Control Register. */
-	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP) {
-		rctl |= E1000_RCTL_SECRC; /* Strip Ethernet CRC. */
+	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads)) {
+		rctl &= ~E1000_RCTL_SECRC; /* Do not Strip Ethernet CRC. */
 
-		/* set STRCRC bit in all queues */
+		/* clear STRCRC bit in all queues */
 		if (hw->mac.type == e1000_i350 ||
 		    hw->mac.type == e1000_i210 ||
 		    hw->mac.type == e1000_i211 ||
@@ -2513,14 +2518,14 @@ eth_igb_rx_init(struct rte_eth_dev *dev)
 				rxq = dev->data->rx_queues[i];
 				uint32_t dvmolr = E1000_READ_REG(hw,
 					E1000_DVMOLR(rxq->reg_idx));
-				dvmolr |= E1000_DVMOLR_STRCRC;
+				dvmolr &= ~E1000_DVMOLR_STRCRC;
 				E1000_WRITE_REG(hw, E1000_DVMOLR(rxq->reg_idx), dvmolr);
 			}
 		}
 	} else {
-		rctl &= ~E1000_RCTL_SECRC; /* Do not Strip Ethernet CRC. */
+		rctl |= E1000_RCTL_SECRC; /* Strip Ethernet CRC. */
 
-		/* clear STRCRC bit in all queues */
+		/* set STRCRC bit in all queues */
 		if (hw->mac.type == e1000_i350 ||
 		    hw->mac.type == e1000_i210 ||
 		    hw->mac.type == e1000_i211 ||
@@ -2529,7 +2534,7 @@ eth_igb_rx_init(struct rte_eth_dev *dev)
 				rxq = dev->data->rx_queues[i];
 				uint32_t dvmolr = E1000_READ_REG(hw,
 					E1000_DVMOLR(rxq->reg_idx));
-				dvmolr &= ~E1000_DVMOLR_STRCRC;
+				dvmolr |= E1000_DVMOLR_STRCRC;
 				E1000_WRITE_REG(hw, E1000_DVMOLR(rxq->reg_idx), dvmolr);
 			}
 		}
diff --git a/drivers/net/ena/Makefile b/drivers/net/ena/Makefile
index 43339f3b..ff9ce315 100644
--- a/drivers/net/ena/Makefile
+++ b/drivers/net/ena/Makefile
@@ -58,5 +58,6 @@ CFLAGS += $(INCLUDES)
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
+LDLIBS += -lrte_timer
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/ena/base/ena_com.c b/drivers/net/ena/base/ena_com.c
index 38a05877..4abf1a28 100644
--- a/drivers/net/ena/base/ena_com.c
+++ b/drivers/net/ena/base/ena_com.c
@@ -37,11 +37,19 @@
 /*****************************************************************************/
 
 /* Timeout in micro-sec */
-#define ADMIN_CMD_TIMEOUT_US (1000000)
+#define ADMIN_CMD_TIMEOUT_US (3000000)
 
-#define ENA_ASYNC_QUEUE_DEPTH 4
+#define ENA_ASYNC_QUEUE_DEPTH 16
 #define ENA_ADMIN_QUEUE_DEPTH 32
 
+#ifdef ENA_EXTENDED_STATS
+
+#define ENA_HISTOGRAM_ACTIVE_MASK_OFFSET 0xF08
+#define ENA_EXTENDED_STAT_GET_FUNCT(_funct_queue) (_funct_queue & 0xFFFF)
+#define ENA_EXTENDED_STAT_GET_QUEUE(_funct_queue) (_funct_queue >> 16)
+
+#endif /* ENA_EXTENDED_STATS */
+
 #define MIN_ENA_VER (((ENA_COMMON_SPEC_VERSION_MAJOR) << \
 		ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) \
 		| (ENA_COMMON_SPEC_VERSION_MINOR))
@@ -62,7 +70,9 @@
 
 #define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF
 
-static int ena_alloc_cnt;
+#define ENA_REGS_ADMIN_INTR_MASK 1
+
+#define ENA_POLL_MS	5
 
 /*****************************************************************************/
 /*****************************************************************************/
@@ -86,6 +96,11 @@ struct ena_comp_ctx {
 	bool occupied;
 };
 
+struct ena_com_stats_ctx {
+	struct ena_admin_aq_get_stats_cmd get_cmd;
+	struct ena_admin_acq_get_stats_resp get_resp;
+};
+
 static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
 				       struct ena_common_mem_addr *ena_addr,
 				       dma_addr_t addr)
@@ -95,50 +110,49 @@ static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
 		return ENA_COM_INVAL;
 	}
 
-	ena_addr->mem_addr_low = (u32)addr;
-	ena_addr->mem_addr_high =
-		((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 32)) >> 32);
+	ena_addr->mem_addr_low = lower_32_bits(addr);
+	ena_addr->mem_addr_high = (u16)upper_32_bits(addr);
 
 	return 0;
 }
 
 static int ena_com_admin_init_sq(struct ena_com_admin_queue *queue)
 {
-	ENA_MEM_ALLOC_COHERENT(queue->q_dmadev,
-			       ADMIN_SQ_SIZE(queue->q_depth),
-			       queue->sq.entries,
-			       queue->sq.dma_addr,
-			       queue->sq.mem_handle);
+	struct ena_com_admin_sq *sq = &queue->sq;
+	u16 size = ADMIN_SQ_SIZE(queue->q_depth);
 
-	if (!queue->sq.entries) {
+	ENA_MEM_ALLOC_COHERENT(queue->q_dmadev, size, sq->entries, sq->dma_addr,
+			       sq->mem_handle);
+
+	if (!sq->entries) {
 		ena_trc_err("memory allocation failed");
 		return ENA_COM_NO_MEM;
 	}
 
-	queue->sq.head = 0;
-	queue->sq.tail = 0;
-	queue->sq.phase = 1;
+	sq->head = 0;
+	sq->tail = 0;
+	sq->phase = 1;
 
-	queue->sq.db_addr = NULL;
+	sq->db_addr = NULL;
 
 	return 0;
 }
 
 static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue)
 {
-	ENA_MEM_ALLOC_COHERENT(queue->q_dmadev,
-			       ADMIN_CQ_SIZE(queue->q_depth),
-			       queue->cq.entries,
-			       queue->cq.dma_addr,
-			       queue->cq.mem_handle);
+	struct ena_com_admin_cq *cq = &queue->cq;
+	u16 size = ADMIN_CQ_SIZE(queue->q_depth);
+
+	ENA_MEM_ALLOC_COHERENT(queue->q_dmadev, size, cq->entries, cq->dma_addr,
+			       cq->mem_handle);
 
-	if (!queue->cq.entries)  {
+	if (!cq->entries)  {
 		ena_trc_err("memory allocation failed");
 		return ENA_COM_NO_MEM;
 	}
 
-	queue->cq.head = 0;
-	queue->cq.phase = 1;
+	cq->head = 0;
+	cq->phase = 1;
 
 	return 0;
 }
@@ -146,44 +160,44 @@ static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue)
 static int ena_com_admin_init_aenq(struct ena_com_dev *dev,
 				   struct ena_aenq_handlers *aenq_handlers)
 {
+	struct ena_com_aenq *aenq = &dev->aenq;
 	u32 addr_low, addr_high, aenq_caps;
+	u16 size;
 
 	dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH;
-	ENA_MEM_ALLOC_COHERENT(dev->dmadev,
-			       ADMIN_AENQ_SIZE(dev->aenq.q_depth),
-			       dev->aenq.entries,
-			       dev->aenq.dma_addr,
-			       dev->aenq.mem_handle);
+	size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH);
+	ENA_MEM_ALLOC_COHERENT(dev->dmadev, size,
+			aenq->entries,
+			aenq->dma_addr,
+			aenq->mem_handle);
 
-	if (!dev->aenq.entries) {
+	if (!aenq->entries) {
 		ena_trc_err("memory allocation failed");
 		return ENA_COM_NO_MEM;
 	}
 
-	dev->aenq.head = dev->aenq.q_depth;
-	dev->aenq.phase = 1;
+	aenq->head = aenq->q_depth;
+	aenq->phase = 1;
 
-	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(dev->aenq.dma_addr);
-	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(dev->aenq.dma_addr);
+	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr);
+	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr);
 
-	ENA_REG_WRITE32(addr_low, (unsigned char *)dev->reg_bar
-			+ ENA_REGS_AENQ_BASE_LO_OFF);
-	ENA_REG_WRITE32(addr_high, (unsigned char *)dev->reg_bar
-			+ ENA_REGS_AENQ_BASE_HI_OFF);
+	ENA_REG_WRITE32(dev->bus, addr_low, dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF);
+	ENA_REG_WRITE32(dev->bus, addr_high, dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF);
 
 	aenq_caps = 0;
 	aenq_caps |= dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
 	aenq_caps |= (sizeof(struct ena_admin_aenq_entry) <<
 		ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &
 		ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;
+	ENA_REG_WRITE32(dev->bus, aenq_caps, dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF);
 
-	ENA_REG_WRITE32(aenq_caps, (unsigned char *)dev->reg_bar
-			+ ENA_REGS_AENQ_CAPS_OFF);
-
-	if (unlikely(!aenq_handlers))
+	if (unlikely(!aenq_handlers)) {
 		ena_trc_err("aenq handlers pointer is NULL\n");
+		return ENA_COM_INVAL;
+	}
 
-	dev->aenq.aenq_handlers = aenq_handlers;
+	aenq->aenq_handlers = aenq_handlers;
 
 	return 0;
 }
@@ -217,12 +231,11 @@ static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *queue,
 	return &queue->comp_ctx[command_id];
 }
 
-static struct ena_comp_ctx *
-__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
-			   struct ena_admin_aq_entry *cmd,
-			   size_t cmd_size_in_bytes,
-			   struct ena_admin_acq_entry *comp,
-			   size_t comp_size_in_bytes)
+static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
+						       struct ena_admin_aq_entry *cmd,
+						       size_t cmd_size_in_bytes,
+						       struct ena_admin_acq_entry *comp,
+						       size_t comp_size_in_bytes)
 {
 	struct ena_comp_ctx *comp_ctx;
 	u16 tail_masked, cmd_id;
@@ -234,12 +247,9 @@ __ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
 	tail_masked = admin_queue->sq.tail & queue_size_mask;
 
 	/* In case of queue FULL */
-	cnt = admin_queue->sq.tail - admin_queue->sq.head;
+	cnt = ATOMIC32_READ(&admin_queue->outstanding_cmds);
 	if (cnt >= admin_queue->q_depth) {
-		ena_trc_dbg("admin queue is FULL (tail %d head %d depth: %d)\n",
-			    admin_queue->sq.tail,
-			    admin_queue->sq.head,
-			    admin_queue->q_depth);
+		ena_trc_dbg("admin queue is full.\n");
 		admin_queue->stats.out_of_space++;
 		return ERR_PTR(ENA_COM_NO_SPACE);
 	}
@@ -253,6 +263,8 @@ __ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
 		ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
 
 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);
+	if (unlikely(!comp_ctx))
+		return ERR_PTR(ENA_COM_INVAL);
 
 	comp_ctx->status = ENA_CMD_SUBMITTED;
 	comp_ctx->comp_size = (u32)comp_size_in_bytes;
@@ -272,7 +284,8 @@ __ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
 	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))
 		admin_queue->sq.phase = !admin_queue->sq.phase;
 
-	ENA_REG_WRITE32(admin_queue->sq.tail, admin_queue->sq.db_addr);
+	ENA_REG_WRITE32(admin_queue->bus, admin_queue->sq.tail,
+			admin_queue->sq.db_addr);
 
 	return comp_ctx;
 }
@@ -298,12 +311,11 @@ static inline int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue)
 	return 0;
 }
 
-static struct ena_comp_ctx *
-ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
-			 struct ena_admin_aq_entry *cmd,
-			 size_t cmd_size_in_bytes,
-			 struct ena_admin_acq_entry *comp,
-			 size_t comp_size_in_bytes)
+static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
+						     struct ena_admin_aq_entry *cmd,
+						     size_t cmd_size_in_bytes,
+						     struct ena_admin_acq_entry *comp,
+						     size_t comp_size_in_bytes)
 {
 	unsigned long flags = 0;
 	struct ena_comp_ctx *comp_ctx;
@@ -317,7 +329,7 @@ ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
 					      cmd_size_in_bytes,
 					      comp,
 					      comp_size_in_bytes);
-	if (unlikely(IS_ERR(comp_ctx)))
+	if (IS_ERR(comp_ctx))
 		admin_queue->running_state = false;
 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
 
@@ -331,9 +343,7 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
 	size_t size;
 	int dev_node = 0;
 
-	ENA_TOUCH(ctx);
-
-	memset(&io_sq->desc_addr, 0x0, sizeof(struct ena_com_io_desc_addr));
+	memset(&io_sq->desc_addr, 0x0, sizeof(io_sq->desc_addr));
 
 	io_sq->desc_entry_size =
 		(io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
@@ -347,23 +357,26 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
 					    size,
 					    io_sq->desc_addr.virt_addr,
 					    io_sq->desc_addr.phys_addr,
+					    io_sq->desc_addr.mem_handle,
 					    ctx->numa_node,
 					    dev_node);
-		if (!io_sq->desc_addr.virt_addr)
+		if (!io_sq->desc_addr.virt_addr) {
 			ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
 					       size,
 					       io_sq->desc_addr.virt_addr,
 					       io_sq->desc_addr.phys_addr,
 					       io_sq->desc_addr.mem_handle);
+		}
 	} else {
 		ENA_MEM_ALLOC_NODE(ena_dev->dmadev,
 				   size,
 				   io_sq->desc_addr.virt_addr,
 				   ctx->numa_node,
 				   dev_node);
-		if (!io_sq->desc_addr.virt_addr)
+		if (!io_sq->desc_addr.virt_addr) {
 			io_sq->desc_addr.virt_addr =
 				ENA_MEM_ALLOC(ena_dev->dmadev, size);
+		}
 	}
 
 	if (!io_sq->desc_addr.virt_addr) {
@@ -385,8 +398,7 @@ static int ena_com_init_io_cq(struct ena_com_dev *ena_dev,
 	size_t size;
 	int prev_node = 0;
 
-	ENA_TOUCH(ctx);
-	memset(&io_cq->cdesc_addr, 0x0, sizeof(struct ena_com_io_desc_addr));
+	memset(&io_cq->cdesc_addr, 0x0, sizeof(io_cq->cdesc_addr));
 
 	/* Use the basic completion descriptor for Rx */
 	io_cq->cdesc_entry_size_in_bytes =
@@ -397,17 +409,19 @@ static int ena_com_init_io_cq(struct ena_com_dev *ena_dev,
 	size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
 
 	ENA_MEM_ALLOC_COHERENT_NODE(ena_dev->dmadev,
-				    size,
-				    io_cq->cdesc_addr.virt_addr,
-				    io_cq->cdesc_addr.phys_addr,
-				    ctx->numa_node,
-				    prev_node);
-	if (!io_cq->cdesc_addr.virt_addr)
+			size,
+			io_cq->cdesc_addr.virt_addr,
+			io_cq->cdesc_addr.phys_addr,
+			io_cq->cdesc_addr.mem_handle,
+			ctx->numa_node,
+			prev_node);
+	if (!io_cq->cdesc_addr.virt_addr) {
 		ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
 				       size,
 				       io_cq->cdesc_addr.virt_addr,
 				       io_cq->cdesc_addr.phys_addr,
 				       io_cq->cdesc_addr.mem_handle);
+	}
 
 	if (!io_cq->cdesc_addr.virt_addr) {
 		ena_trc_err("memory allocation failed");
@@ -420,9 +434,8 @@ static int ena_com_init_io_cq(struct ena_com_dev *ena_dev,
 	return 0;
 }
 
-static void
-ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,
-				       struct ena_admin_acq_entry *cqe)
+static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,
+						   struct ena_admin_acq_entry *cqe)
 {
 	struct ena_comp_ctx *comp_ctx;
 	u16 cmd_id;
@@ -447,8 +460,7 @@ ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,
 		ENA_WAIT_EVENT_SIGNAL(comp_ctx->wait_event);
 }
 
-static void
-ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)
+static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)
 {
 	struct ena_admin_acq_entry *cqe = NULL;
 	u16 comp_num = 0;
@@ -499,7 +511,7 @@ static int ena_com_comp_status_to_errno(u8 comp_status)
 	case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
 		return ENA_COM_NO_MEM;
 	case ENA_ADMIN_UNSUPPORTED_OPCODE:
-		return ENA_COM_PERMISSION;
+		return ENA_COM_UNSUPPORTED;
 	case ENA_ADMIN_BAD_OPCODE:
 	case ENA_ADMIN_MALFORMED_REQUEST:
 	case ENA_ADMIN_ILLEGAL_PARAMETER:
@@ -510,20 +522,24 @@ static int ena_com_comp_status_to_errno(u8 comp_status)
 	return 0;
 }
 
-static int
-ena_com_wait_and_process_admin_cq_polling(
-		struct ena_comp_ctx *comp_ctx,
-		struct ena_com_admin_queue *admin_queue)
+static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_ctx,
+						     struct ena_com_admin_queue *admin_queue)
 {
 	unsigned long flags = 0;
-	u64 start_time;
+	unsigned long timeout;
 	int ret;
 
-	start_time = ENA_GET_SYSTEM_USECS();
+	timeout = ENA_GET_SYSTEM_TIMEOUT(admin_queue->completion_timeout);
+
+	while (1) {
+                ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
+                ena_com_handle_admin_completion(admin_queue);
+                ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
+
+                if (comp_ctx->status != ENA_CMD_SUBMITTED)
+			break;
 
-	while (comp_ctx->status == ENA_CMD_SUBMITTED) {
-		if ((ENA_GET_SYSTEM_USECS() - start_time) >
-		    ADMIN_CMD_TIMEOUT_US) {
+		if (ENA_TIME_EXPIRE(timeout)) {
 			ena_trc_err("Wait for completion (polling) timeout\n");
 			/* ENA didn't have any completion */
 			ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
@@ -535,9 +551,7 @@ ena_com_wait_and_process_admin_cq_polling(
 			goto err;
 		}
 
-		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
-		ena_com_handle_admin_completion(admin_queue);
-		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
+		ENA_MSLEEP(ENA_POLL_MS);
 	}
 
 	if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) {
@@ -549,8 +563,8 @@ ena_com_wait_and_process_admin_cq_polling(
 		goto err;
 	}
 
-	ENA_ASSERT(comp_ctx->status == ENA_CMD_COMPLETED,
-		   "Invalid comp status %d\n", comp_ctx->status);
+	ENA_WARN(comp_ctx->status != ENA_CMD_COMPLETED,
+		 "Invalid comp status %d\n", comp_ctx->status);
 
 	ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
 err:
@@ -558,16 +572,14 @@ err:
 	return ret;
 }
 
-static int
-ena_com_wait_and_process_admin_cq_interrupts(
-		struct ena_comp_ctx *comp_ctx,
-		struct ena_com_admin_queue *admin_queue)
+static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx,
+							struct ena_com_admin_queue *admin_queue)
 {
 	unsigned long flags = 0;
-	int ret = 0;
+	int ret;
 
 	ENA_WAIT_EVENT_WAIT(comp_ctx->wait_event,
-			    ADMIN_CMD_TIMEOUT_US);
+			    admin_queue->completion_timeout);
 
 	/* In case the command wasn't completed find out the root cause.
 	 * There might be 2 kinds of errors
@@ -607,16 +619,18 @@ static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
 	struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
 	volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp =
 		mmio_read->read_resp;
-	u32 mmio_read_reg, ret;
+	u32 mmio_read_reg, ret, i;
 	unsigned long flags = 0;
-	int i;
+	u32 timeout = mmio_read->reg_read_to;
 
 	ENA_MIGHT_SLEEP();
 
+	if (timeout == 0)
+		timeout = ENA_REG_READ_TIMEOUT;
+
 	/* If readless is disabled, perform regular read */
 	if (!mmio_read->readless_supported)
-		return ENA_REG_READ32((unsigned char *)ena_dev->reg_bar +
-				      offset);
+		return ENA_REG_READ32(ena_dev->bus, ena_dev->reg_bar + offset);
 
 	ENA_SPINLOCK_LOCK(mmio_read->lock, flags);
 	mmio_read->seq_num++;
@@ -632,17 +646,16 @@ static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
 	 */
 	wmb();
 
-	ENA_REG_WRITE32(mmio_read_reg, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_MMIO_REG_READ_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, mmio_read_reg, ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF);
 
-	for (i = 0; i < ENA_REG_READ_TIMEOUT; i++) {
+	for (i = 0; i < timeout; i++) {
 		if (read_resp->req_id == mmio_read->seq_num)
 			break;
 
 		ENA_UDELAY(1);
 	}
 
-	if (unlikely(i == ENA_REG_READ_TIMEOUT)) {
+	if (unlikely(i == timeout)) {
 		ena_trc_err("reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n",
 			    mmio_read->seq_num,
 			    offset,
@@ -653,7 +666,7 @@ static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
 	}
 
 	if (read_resp->reg_off != offset) {
-		ena_trc_err("reading failed for wrong offset value");
+		ena_trc_err("Read failure: wrong offset provided");
 		ret = ENA_MMIO_READ_TIMEOUT;
 	} else {
 		ret = read_resp->reg_val;
@@ -671,9 +684,8 @@ err:
  * It is expected that the IRQ called ena_com_handle_admin_completion
  * to mark the completions.
  */
-static int
-ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx,
-				  struct ena_com_admin_queue *admin_queue)
+static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx,
+					     struct ena_com_admin_queue *admin_queue)
 {
 	if (admin_queue->polling)
 		return ena_com_wait_and_process_admin_cq_polling(comp_ctx,
@@ -692,7 +704,7 @@ static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
 	u8 direction;
 	int ret;
 
-	memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd));
+	memset(&destroy_cmd, 0x0, sizeof(destroy_cmd));
 
 	if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
 		direction = ENA_ADMIN_SQ_DIRECTION_TX;
@@ -706,12 +718,11 @@ static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
 	destroy_cmd.sq.sq_idx = io_sq->idx;
 	destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ;
 
-	ret = ena_com_execute_admin_command(
-			admin_queue,
-			(struct ena_admin_aq_entry *)&destroy_cmd,
-			sizeof(destroy_cmd),
-			(struct ena_admin_acq_entry *)&destroy_resp,
-			sizeof(destroy_resp));
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&destroy_cmd,
+					    sizeof(destroy_cmd),
+					    (struct ena_admin_acq_entry *)&destroy_resp,
+					    sizeof(destroy_resp));
 
 	if (unlikely(ret && (ret != ENA_COM_NO_DEVICE)))
 		ena_trc_err("failed to destroy io sq error: %d\n", ret);
@@ -747,18 +758,20 @@ static void ena_com_io_queue_free(struct ena_com_dev *ena_dev,
 					      io_sq->desc_addr.phys_addr,
 					      io_sq->desc_addr.mem_handle);
 		else
-			ENA_MEM_FREE(ena_dev->dmadev,
-				     io_sq->desc_addr.virt_addr);
+			ENA_MEM_FREE(ena_dev->dmadev, io_sq->desc_addr.virt_addr);
 
 		io_sq->desc_addr.virt_addr = NULL;
 	}
 }
 
-static int wait_for_reset_state(struct ena_com_dev *ena_dev,
-				u32 timeout, u16 exp_state)
+static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout,
+				u16 exp_state)
 {
 	u32 val, i;
 
+	/* Convert timeout from resolution of 100ms to ENA_POLL_MS */
+	timeout = (timeout * 100) / ENA_POLL_MS;
+
 	for (i = 0; i < timeout; i++) {
 		val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
 
@@ -771,16 +784,14 @@ static int wait_for_reset_state(struct ena_com_dev *ena_dev,
 			exp_state)
 			return 0;
 
-		/* The resolution of the timeout is 100ms */
-		ENA_MSLEEP(100);
+		ENA_MSLEEP(ENA_POLL_MS);
 	}
 
 	return ENA_COM_TIMER_EXPIRED;
 }
 
-static bool
-ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev,
-				   enum ena_admin_aq_feature_id feature_id)
+static bool ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev,
+					       enum ena_admin_aq_feature_id feature_id)
 {
 	u32 feature_mask = 1 << feature_id;
 
@@ -802,14 +813,9 @@ static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
 	struct ena_admin_get_feat_cmd get_cmd;
 	int ret;
 
-	if (!ena_dev) {
-		ena_trc_err("%s : ena_dev is NULL\n", __func__);
-		return ENA_COM_NO_DEVICE;
-	}
-
 	if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) {
-		ena_trc_info("Feature %d isn't supported\n", feature_id);
-		return ENA_COM_PERMISSION;
+		ena_trc_dbg("Feature %d isn't supported\n", feature_id);
+		return ENA_COM_UNSUPPORTED;
 	}
 
 	memset(&get_cmd, 0x0, sizeof(get_cmd));
@@ -945,10 +951,10 @@ static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev,
 		sizeof(struct ena_admin_rss_ind_table_entry);
 
 	ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
-			       tbl_size,
-			       rss->rss_ind_tbl,
-			       rss->rss_ind_tbl_dma_addr,
-			       rss->rss_ind_tbl_mem_handle);
+			     tbl_size,
+			     rss->rss_ind_tbl,
+			     rss->rss_ind_tbl_dma_addr,
+			     rss->rss_ind_tbl_mem_handle);
 	if (unlikely(!rss->rss_ind_tbl))
 		goto mem_err1;
 
@@ -1005,7 +1011,7 @@ static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
 	u8 direction;
 	int ret;
 
-	memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_sq_cmd));
+	memset(&create_cmd, 0x0, sizeof(create_cmd));
 
 	create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_SQ;
 
@@ -1041,12 +1047,11 @@ static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
 		}
 	}
 
-	ret = ena_com_execute_admin_command(
-			admin_queue,
-			(struct ena_admin_aq_entry *)&create_cmd,
-			sizeof(create_cmd),
-			(struct ena_admin_acq_entry *)&cmd_completion,
-			sizeof(cmd_completion));
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&create_cmd,
+					    sizeof(create_cmd),
+					    (struct ena_admin_acq_entry *)&cmd_completion,
+					    sizeof(cmd_completion));
 	if (unlikely(ret)) {
 		ena_trc_err("Failed to create IO SQ. error: %d\n", ret);
 		return ret;
@@ -1133,9 +1138,8 @@ static int ena_com_init_interrupt_moderation_table(struct ena_com_dev *ena_dev)
 	return 0;
 }
 
-static void
-ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
-				     u16 intr_delay_resolution)
+static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
+						 u16 intr_delay_resolution)
 {
 	struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
 	unsigned int i;
@@ -1165,13 +1169,18 @@ int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
 				  size_t comp_size)
 {
 	struct ena_comp_ctx *comp_ctx;
-	int ret = 0;
+	int ret;
 
 	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,
 					    comp, comp_size);
-	if (unlikely(IS_ERR(comp_ctx))) {
-		ena_trc_err("Failed to submit command [%ld]\n",
-			    PTR_ERR(comp_ctx));
+	if (IS_ERR(comp_ctx)) {
+		if (comp_ctx == ERR_PTR(ENA_COM_NO_DEVICE))
+			ena_trc_dbg("Failed to submit command [%ld]\n",
+				    PTR_ERR(comp_ctx));
+		else
+			ena_trc_err("Failed to submit command [%ld]\n",
+				    PTR_ERR(comp_ctx));
+
 		return PTR_ERR(comp_ctx);
 	}
 
@@ -1195,7 +1204,7 @@ int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
 	struct ena_admin_acq_create_cq_resp_desc cmd_completion;
 	int ret;
 
-	memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_cq_cmd));
+	memset(&create_cmd, 0x0, sizeof(create_cmd));
 
 	create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_CQ;
 
@@ -1215,12 +1224,11 @@ int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
 		return ret;
 	}
 
-	ret = ena_com_execute_admin_command(
-			admin_queue,
-			(struct ena_admin_aq_entry *)&create_cmd,
-			sizeof(create_cmd),
-			(struct ena_admin_acq_entry *)&cmd_completion,
-			sizeof(cmd_completion));
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&create_cmd,
+					    sizeof(create_cmd),
+					    (struct ena_admin_acq_entry *)&cmd_completion,
+					    sizeof(cmd_completion));
 	if (unlikely(ret)) {
 		ena_trc_err("Failed to create IO CQ. error: %d\n", ret);
 		return ret;
@@ -1290,7 +1298,7 @@ void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev)
 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
 	while (ATOMIC32_READ(&admin_queue->outstanding_cmds) != 0) {
 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
-		ENA_MSLEEP(20);
+		ENA_MSLEEP(ENA_POLL_MS);
 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
 	}
 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
@@ -1304,17 +1312,16 @@ int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
 	struct ena_admin_acq_destroy_cq_resp_desc destroy_resp;
 	int ret;
 
-	memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd));
+	memset(&destroy_cmd, 0x0, sizeof(destroy_cmd));
 
 	destroy_cmd.cq_idx = io_cq->idx;
 	destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ;
 
-	ret = ena_com_execute_admin_command(
-			admin_queue,
-			(struct ena_admin_aq_entry *)&destroy_cmd,
-			sizeof(destroy_cmd),
-			(struct ena_admin_acq_entry *)&destroy_resp,
-			sizeof(destroy_resp));
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&destroy_cmd,
+					    sizeof(destroy_cmd),
+					    (struct ena_admin_acq_entry *)&destroy_resp,
+					    sizeof(destroy_resp));
 
 	if (unlikely(ret && (ret != ENA_COM_NO_DEVICE)))
 		ena_trc_err("Failed to destroy IO CQ. error: %d\n", ret);
@@ -1341,13 +1348,12 @@ void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev)
 {
 	u16 depth = ena_dev->aenq.q_depth;
 
-	ENA_ASSERT(ena_dev->aenq.head == depth, "Invalid AENQ state\n");
+	ENA_WARN(ena_dev->aenq.head != depth, "Invalid AENQ state\n");
 
 	/* Init head_db to mark that all entries in the queue
 	 * are initially available
 	 */
-	ENA_REG_WRITE32(depth, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_AENQ_HEAD_DB_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, depth, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
 }
 
 int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
@@ -1356,12 +1362,7 @@ int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
 	struct ena_admin_set_feat_cmd cmd;
 	struct ena_admin_set_feat_resp resp;
 	struct ena_admin_get_feat_resp get_resp;
-	int ret = 0;
-
-	if (unlikely(!ena_dev)) {
-		ena_trc_err("%s : ena_dev is NULL\n", __func__);
-		return ENA_COM_NO_DEVICE;
-	}
+	int ret;
 
 	ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG);
 	if (ret) {
@@ -1373,7 +1374,7 @@ int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
 		ena_trc_warn("Trying to set unsupported aenq events. supported flag: %x asked flag: %x\n",
 			     get_resp.u.aenq.supported_groups,
 			     groups_flag);
-		return ENA_COM_PERMISSION;
+		return ENA_COM_UNSUPPORTED;
 	}
 
 	memset(&cmd, 0x0, sizeof(cmd));
@@ -1476,41 +1477,42 @@ int ena_com_validate_version(struct ena_com_dev *ena_dev)
 void ena_com_admin_destroy(struct ena_com_dev *ena_dev)
 {
 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	struct ena_com_admin_cq *cq = &admin_queue->cq;
+	struct ena_com_admin_sq *sq = &admin_queue->sq;
+	struct ena_com_aenq *aenq = &ena_dev->aenq;
+	u16 size;
 
-	if (!admin_queue)
-		return;
-
+	ENA_WAIT_EVENT_DESTROY(admin_queue->comp_ctx->wait_event);
 	if (admin_queue->comp_ctx)
 		ENA_MEM_FREE(ena_dev->dmadev, admin_queue->comp_ctx);
 	admin_queue->comp_ctx = NULL;
-
-	if (admin_queue->sq.entries)
-		ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
-				      ADMIN_SQ_SIZE(admin_queue->q_depth),
-				      admin_queue->sq.entries,
-				      admin_queue->sq.dma_addr,
-				      admin_queue->sq.mem_handle);
-	admin_queue->sq.entries = NULL;
-
-	if (admin_queue->cq.entries)
-		ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
-				      ADMIN_CQ_SIZE(admin_queue->q_depth),
-				      admin_queue->cq.entries,
-				      admin_queue->cq.dma_addr,
-				      admin_queue->cq.mem_handle);
-	admin_queue->cq.entries = NULL;
-
+	size = ADMIN_SQ_SIZE(admin_queue->q_depth);
+	if (sq->entries)
+		ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, sq->entries,
+				      sq->dma_addr, sq->mem_handle);
+	sq->entries = NULL;
+
+	size = ADMIN_CQ_SIZE(admin_queue->q_depth);
+	if (cq->entries)
+		ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, cq->entries,
+				      cq->dma_addr, cq->mem_handle);
+	cq->entries = NULL;
+
+	size = ADMIN_AENQ_SIZE(aenq->q_depth);
 	if (ena_dev->aenq.entries)
-		ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
-				      ADMIN_AENQ_SIZE(ena_dev->aenq.q_depth),
-				      ena_dev->aenq.entries,
-				      ena_dev->aenq.dma_addr,
-				      ena_dev->aenq.mem_handle);
-	ena_dev->aenq.entries = NULL;
+		ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, aenq->entries,
+				      aenq->dma_addr, aenq->mem_handle);
+	aenq->entries = NULL;
 }
 
 void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling)
 {
+	u32 mask_value = 0;
+
+	if (polling)
+		mask_value = ENA_REGS_ADMIN_INTR_MASK;
+
+	ENA_REG_WRITE32(ena_dev->bus, mask_value, ena_dev->reg_bar + ENA_REGS_INTR_MASK_OFF);
 	ena_dev->admin_queue.polling = polling;
 }
 
@@ -1536,8 +1538,7 @@ int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev)
 	return 0;
 }
 
-void
-ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported)
+void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported)
 {
 	struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
 
@@ -1548,10 +1549,8 @@ void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev)
 {
 	struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
 
-	ENA_REG_WRITE32(0x0, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_MMIO_RESP_LO_OFF);
-	ENA_REG_WRITE32(0x0, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_MMIO_RESP_HI_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, 0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, 0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
 
 	ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
 			      sizeof(*mmio_read->read_resp),
@@ -1570,10 +1569,8 @@ void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev)
 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr);
 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr);
 
-	ENA_REG_WRITE32(addr_low, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_MMIO_RESP_LO_OFF);
-	ENA_REG_WRITE32(addr_high, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_MMIO_RESP_HI_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
 }
 
 int ena_com_admin_init(struct ena_com_dev *ena_dev,
@@ -1619,24 +1616,20 @@ int ena_com_admin_init(struct ena_com_dev *ena_dev,
 	if (ret)
 		goto error;
 
-	admin_queue->sq.db_addr = (u32 __iomem *)
-		((unsigned char *)ena_dev->reg_bar + ENA_REGS_AQ_DB_OFF);
+	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
+		ENA_REGS_AQ_DB_OFF);
 
 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);
 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);
 
-	ENA_REG_WRITE32(addr_low, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_AQ_BASE_LO_OFF);
-	ENA_REG_WRITE32(addr_high, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_AQ_BASE_HI_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_AQ_BASE_LO_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_AQ_BASE_HI_OFF);
 
 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);
 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);
 
-	ENA_REG_WRITE32(addr_low, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_ACQ_BASE_LO_OFF);
-	ENA_REG_WRITE32(addr_high, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_ACQ_BASE_HI_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_LO_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_HI_OFF);
 
 	aq_caps = 0;
 	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;
@@ -1650,10 +1643,8 @@ int ena_com_admin_init(struct ena_com_dev *ena_dev,
 		ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) &
 		ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK;
 
-	ENA_REG_WRITE32(aq_caps, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_AQ_CAPS_OFF);
-	ENA_REG_WRITE32(acq_caps, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_ACQ_CAPS_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, aq_caps, ena_dev->reg_bar + ENA_REGS_AQ_CAPS_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, acq_caps, ena_dev->reg_bar + ENA_REGS_ACQ_CAPS_OFF);
 	ret = ena_com_admin_init_aenq(ena_dev, aenq_handlers);
 	if (ret)
 		goto error;
@@ -1672,7 +1663,7 @@ int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
 {
 	struct ena_com_io_sq *io_sq;
 	struct ena_com_io_cq *io_cq;
-	int ret = 0;
+	int ret;
 
 	if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) {
 		ena_trc_err("Qid (%d) is bigger than max num of queues (%d)\n",
@@ -1683,8 +1674,8 @@ int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
 	io_sq = &ena_dev->io_sq_queues[ctx->qid];
 	io_cq = &ena_dev->io_cq_queues[ctx->qid];
 
-	memset(io_sq, 0x0, sizeof(struct ena_com_io_sq));
-	memset(io_cq, 0x0, sizeof(struct ena_com_io_cq));
+	memset(io_sq, 0x0, sizeof(*io_sq));
+	memset(io_cq, 0x0, sizeof(*io_cq));
 
 	/* Init CQ */
 	io_cq->q_depth = ctx->queue_size;
@@ -1794,6 +1785,19 @@ int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
 	memcpy(&get_feat_ctx->offload, &get_resp.u.offload,
 	       sizeof(get_resp.u.offload));
 
+	/* Driver hints isn't mandatory admin command. So in case the
+	 * command isn't supported set driver hints to 0
+	 */
+	rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_HW_HINTS);
+
+	if (!rc)
+		memcpy(&get_feat_ctx->hw_hints, &get_resp.u.hw_hints,
+		       sizeof(get_resp.u.hw_hints));
+	else if (rc == ENA_COM_UNSUPPORTED)
+		memset(&get_feat_ctx->hw_hints, 0x0, sizeof(get_feat_ctx->hw_hints));
+	else
+		return rc;
+
 	return 0;
 }
 
@@ -1826,6 +1830,7 @@ void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
 	struct ena_admin_aenq_common_desc *aenq_common;
 	struct ena_com_aenq *aenq  = &dev->aenq;
 	ena_aenq_handler handler_cb;
+	unsigned long long timestamp;
 	u16 masked_head, processed = 0;
 	u8 phase;
 
@@ -1837,11 +1842,13 @@ void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
 	/* Go over all the events */
 	while ((aenq_common->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) ==
 		phase) {
+		timestamp = (unsigned long long)aenq_common->timestamp_low |
+			((unsigned long long)aenq_common->timestamp_high << 32);
+		ENA_TOUCH(timestamp); /* In case debug is disabled */
 		ena_trc_dbg("AENQ! Group[%x] Syndrom[%x] timestamp: [%llus]\n",
 			    aenq_common->group,
 			    aenq_common->syndrom,
-			    (unsigned long long)aenq_common->timestamp_low +
-			    ((u64)aenq_common->timestamp_high << 32));
+			    timestamp);
 
 		/* Handle specific event*/
 		handler_cb = ena_com_get_specific_aenq_cb(dev,
@@ -1869,11 +1876,11 @@ void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
 
 	/* write the aenq doorbell after all AENQ descriptors were read */
 	mb();
-	ENA_REG_WRITE32((u32)aenq->head, (unsigned char *)dev->reg_bar
-			+ ENA_REGS_AENQ_HEAD_DB_OFF);
+	ENA_REG_WRITE32(dev->bus, (u32)aenq->head, dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
 }
 
-int ena_com_dev_reset(struct ena_com_dev *ena_dev)
+int ena_com_dev_reset(struct ena_com_dev *ena_dev,
+		      enum ena_regs_reset_reason_types reset_reason)
 {
 	u32 stat, timeout, cap, reset_val;
 	int rc;
@@ -1901,8 +1908,9 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev)
 
 	/* start reset */
 	reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK;
-	ENA_REG_WRITE32(reset_val, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_DEV_CTL_OFF);
+	reset_val |= (reset_reason << ENA_REGS_DEV_CTL_RESET_REASON_SHIFT) &
+			ENA_REGS_DEV_CTL_RESET_REASON_MASK;
+	ENA_REG_WRITE32(ena_dev->bus, reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
 
 	/* Write again the MMIO read request address */
 	ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
@@ -1915,29 +1923,32 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev)
 	}
 
 	/* reset done */
-	ENA_REG_WRITE32(0, (unsigned char *)ena_dev->reg_bar
-			+ ENA_REGS_DEV_CTL_OFF);
+	ENA_REG_WRITE32(ena_dev->bus, 0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
 	rc = wait_for_reset_state(ena_dev, timeout, 0);
 	if (rc != 0) {
 		ena_trc_err("Reset indication didn't turn off\n");
 		return rc;
 	}
 
+	timeout = (cap & ENA_REGS_CAPS_ADMIN_CMD_TO_MASK) >>
+		ENA_REGS_CAPS_ADMIN_CMD_TO_SHIFT;
+	if (timeout)
+		/* the resolution of timeout reg is 100ms */
+		ena_dev->admin_queue.completion_timeout = timeout * 100000;
+	else
+		ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;
+
 	return 0;
 }
 
 static int ena_get_dev_stats(struct ena_com_dev *ena_dev,
-			     struct ena_admin_aq_get_stats_cmd *get_cmd,
-			     struct ena_admin_acq_get_stats_resp *get_resp,
+			     struct ena_com_stats_ctx *ctx,
 			     enum ena_admin_get_stats_type type)
 {
+	struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd;
+	struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp;
 	struct ena_com_admin_queue *admin_queue;
-	int ret = 0;
-
-	if (!ena_dev) {
-		ena_trc_err("%s : ena_dev is NULL\n", __func__);
-		return ENA_COM_NO_DEVICE;
-	}
+	int ret;
 
 	admin_queue = &ena_dev->admin_queue;
 
@@ -1945,12 +1956,11 @@ static int ena_get_dev_stats(struct ena_com_dev *ena_dev,
 	get_cmd->aq_common_descriptor.flags = 0;
 	get_cmd->type = type;
 
-	ret =  ena_com_execute_admin_command(
-			admin_queue,
-			(struct ena_admin_aq_entry *)get_cmd,
-			sizeof(*get_cmd),
-			(struct ena_admin_acq_entry *)get_resp,
-			sizeof(*get_resp));
+	ret =  ena_com_execute_admin_command(admin_queue,
+					     (struct ena_admin_aq_entry *)get_cmd,
+					     sizeof(*get_cmd),
+					     (struct ena_admin_acq_entry *)get_resp,
+					     sizeof(*get_resp));
 
 	if (unlikely(ret))
 		ena_trc_err("Failed to get stats. error: %d\n", ret);
@@ -1961,78 +1971,28 @@ static int ena_get_dev_stats(struct ena_com_dev *ena_dev,
 int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev,
 				struct ena_admin_basic_stats *stats)
 {
-	int ret = 0;
-	struct ena_admin_aq_get_stats_cmd get_cmd;
-	struct ena_admin_acq_get_stats_resp get_resp;
+	struct ena_com_stats_ctx ctx;
+	int ret;
 
-	memset(&get_cmd, 0x0, sizeof(get_cmd));
-	ret = ena_get_dev_stats(ena_dev, &get_cmd, &get_resp,
-				ENA_ADMIN_GET_STATS_TYPE_BASIC);
+	memset(&ctx, 0x0, sizeof(ctx));
+	ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC);
 	if (likely(ret == 0))
-		memcpy(stats, &get_resp.basic_stats,
-		       sizeof(get_resp.basic_stats));
+		memcpy(stats, &ctx.get_resp.basic_stats,
+		       sizeof(ctx.get_resp.basic_stats));
 
 	return ret;
 }
 
-int ena_com_get_dev_extended_stats(struct ena_com_dev *ena_dev, char *buff,
-				   u32 len)
-{
-	int ret = 0;
-	struct ena_admin_aq_get_stats_cmd get_cmd;
-	struct ena_admin_acq_get_stats_resp get_resp;
-	ena_mem_handle_t mem_handle = 0;
-	void *virt_addr;
-	dma_addr_t phys_addr;
-
-	ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, len,
-			       virt_addr, phys_addr, mem_handle);
-	if (!virt_addr) {
-		ret = ENA_COM_NO_MEM;
-		goto done;
-	}
-	memset(&get_cmd, 0x0, sizeof(get_cmd));
-	ret = ena_com_mem_addr_set(ena_dev,
-				   &get_cmd.u.control_buffer.address,
-				   phys_addr);
-	if (unlikely(ret)) {
-		ena_trc_err("memory address set failed\n");
-		return ret;
-	}
-	get_cmd.u.control_buffer.length = len;
-
-	get_cmd.device_id = ena_dev->stats_func;
-	get_cmd.queue_idx = ena_dev->stats_queue;
-
-	ret = ena_get_dev_stats(ena_dev, &get_cmd, &get_resp,
-				ENA_ADMIN_GET_STATS_TYPE_EXTENDED);
-	if (ret < 0)
-		goto free_ext_stats_mem;
-
-	ret = snprintf(buff, len, "%s", (char *)virt_addr);
-
-free_ext_stats_mem:
-	ENA_MEM_FREE_COHERENT(ena_dev->dmadev, len, virt_addr, phys_addr,
-			      mem_handle);
-done:
-	return ret;
-}
-
 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
 {
 	struct ena_com_admin_queue *admin_queue;
 	struct ena_admin_set_feat_cmd cmd;
 	struct ena_admin_set_feat_resp resp;
-	int ret = 0;
-
-	if (unlikely(!ena_dev)) {
-		ena_trc_err("%s : ena_dev is NULL\n", __func__);
-		return ENA_COM_NO_DEVICE;
-	}
+	int ret;
 
 	if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) {
-		ena_trc_info("Feature %d isn't supported\n", ENA_ADMIN_MTU);
-		return ENA_COM_PERMISSION;
+		ena_trc_dbg("Feature %d isn't supported\n", ENA_ADMIN_MTU);
+		return ENA_COM_UNSUPPORTED;
 	}
 
 	memset(&cmd, 0x0, sizeof(cmd));
@@ -2049,11 +2009,10 @@ int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
 					    (struct ena_admin_acq_entry *)&resp,
 					    sizeof(resp));
 
-	if (unlikely(ret)) {
+	if (unlikely(ret))
 		ena_trc_err("Failed to set mtu %d. error: %d\n", mtu, ret);
-		return ENA_COM_INVAL;
-	}
-	return 0;
+
+	return ret;
 }
 
 int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
@@ -2066,7 +2025,7 @@ int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
 				  ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
 	if (unlikely(ret)) {
 		ena_trc_err("Failed to get offload capabilities %d\n", ret);
-		return ENA_COM_INVAL;
+		return ret;
 	}
 
 	memcpy(offload, &resp.u.offload, sizeof(resp.u.offload));
@@ -2085,9 +2044,9 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
 
 	if (!ena_com_check_supported_feature_id(ena_dev,
 						ENA_ADMIN_RSS_HASH_FUNCTION)) {
-		ena_trc_info("Feature %d isn't supported\n",
-			     ENA_ADMIN_RSS_HASH_FUNCTION);
-		return ENA_COM_PERMISSION;
+		ena_trc_dbg("Feature %d isn't supported\n",
+			    ENA_ADMIN_RSS_HASH_FUNCTION);
+		return ENA_COM_UNSUPPORTED;
 	}
 
 	/* Validate hash function is supported */
@@ -2099,7 +2058,7 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
 	if (get_resp.u.flow_hash_func.supported_func & (1 << rss->hash_func)) {
 		ena_trc_err("Func hash %d isn't supported by device, abort\n",
 			    rss->hash_func);
-		return ENA_COM_PERMISSION;
+		return ENA_COM_UNSUPPORTED;
 	}
 
 	memset(&cmd, 0x0, sizeof(cmd));
@@ -2158,7 +2117,7 @@ int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
 
 	if (!((1 << func) & get_resp.u.flow_hash_func.supported_func)) {
 		ena_trc_err("Flow hash function %d isn't supported\n", func);
-		return ENA_COM_PERMISSION;
+		return ENA_COM_UNSUPPORTED;
 	}
 
 	switch (func) {
@@ -2207,7 +2166,7 @@ int ena_com_get_hash_function(struct ena_com_dev *ena_dev,
 	if (unlikely(rc))
 		return rc;
 
-	rss->hash_func = (enum ena_admin_hash_functions)get_resp.u.flow_hash_func.selected_func;
+	rss->hash_func = get_resp.u.flow_hash_func.selected_func;
 	if (func)
 		*func = rss->hash_func;
 
@@ -2242,17 +2201,20 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
 {
 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
 	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
 	struct ena_admin_set_feat_cmd cmd;
 	struct ena_admin_set_feat_resp resp;
 	int ret;
 
 	if (!ena_com_check_supported_feature_id(ena_dev,
 						ENA_ADMIN_RSS_HASH_INPUT)) {
-		ena_trc_info("Feature %d isn't supported\n",
-			     ENA_ADMIN_RSS_HASH_INPUT);
-		return ENA_COM_PERMISSION;
+		ena_trc_dbg("Feature %d isn't supported\n",
+			    ENA_ADMIN_RSS_HASH_INPUT);
+		return ENA_COM_UNSUPPORTED;
 	}
 
+	memset(&cmd, 0x0, sizeof(cmd));
+
 	cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
 	cmd.aq_common_descriptor.flags =
 		ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
@@ -2268,20 +2230,17 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
 		ena_trc_err("memory address set failed\n");
 		return ret;
 	}
-	cmd.control_buffer.length =
-		sizeof(struct ena_admin_feature_rss_hash_control);
+	cmd.control_buffer.length = sizeof(*hash_ctrl);
 
 	ret = ena_com_execute_admin_command(admin_queue,
 					    (struct ena_admin_aq_entry *)&cmd,
 					    sizeof(cmd),
 					    (struct ena_admin_acq_entry *)&resp,
 					    sizeof(resp));
-	if (unlikely(ret)) {
+	if (unlikely(ret))
 		ena_trc_err("Failed to set hash input. error: %d\n", ret);
-		return ENA_COM_INVAL;
-	}
 
-	return 0;
+	return ret;
 }
 
 int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
@@ -2293,7 +2252,7 @@ int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
 	int rc, i;
 
 	/* Get the supported hash input */
-	rc = ena_com_get_hash_ctrl(ena_dev, (enum ena_admin_flow_hash_proto)0, NULL);
+	rc = ena_com_get_hash_ctrl(ena_dev, 0, NULL);
 	if (unlikely(rc))
 		return rc;
 
@@ -2322,7 +2281,7 @@ int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
 	hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
 		ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
 
-	hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
+	hash_ctrl->selected_fields[ENA_ADMIN_RSS_NOT_IP].fields =
 		ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA;
 
 	for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) {
@@ -2332,7 +2291,7 @@ int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
 			ena_trc_err("hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n",
 				    i, hash_ctrl->supported_fields[i].fields,
 				    hash_ctrl->selected_fields[i].fields);
-			return ENA_COM_PERMISSION;
+			return ENA_COM_UNSUPPORTED;
 		}
 	}
 
@@ -2340,7 +2299,7 @@ int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
 
 	/* In case of failure, restore the old hash ctrl */
 	if (unlikely(rc))
-		ena_com_get_hash_ctrl(ena_dev, (enum ena_admin_flow_hash_proto)0, NULL);
+		ena_com_get_hash_ctrl(ena_dev, 0, NULL);
 
 	return rc;
 }
@@ -2377,7 +2336,7 @@ int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev,
 
 	/* In case of failure, restore the old hash ctrl */
 	if (unlikely(rc))
-		ena_com_get_hash_ctrl(ena_dev, (enum ena_admin_flow_hash_proto)0, NULL);
+		ena_com_get_hash_ctrl(ena_dev, 0, NULL);
 
 	return 0;
 }
@@ -2404,14 +2363,13 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
 	struct ena_rss *rss = &ena_dev->rss;
 	struct ena_admin_set_feat_cmd cmd;
 	struct ena_admin_set_feat_resp resp;
-	int ret = 0;
+	int ret;
 
-	if (!ena_com_check_supported_feature_id(
-				ena_dev,
-				ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) {
-		ena_trc_info("Feature %d isn't supported\n",
-			     ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
-		return ENA_COM_PERMISSION;
+	if (!ena_com_check_supported_feature_id(ena_dev,
+						ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) {
+		ena_trc_dbg("Feature %d isn't supported\n",
+			    ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
+		return ENA_COM_UNSUPPORTED;
 	}
 
 	ret = ena_com_ind_tbl_convert_to_device(ena_dev);
@@ -2446,12 +2404,10 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
 					    (struct ena_admin_acq_entry *)&resp,
 					    sizeof(resp));
 
-	if (unlikely(ret)) {
+	if (unlikely(ret))
 		ena_trc_err("Failed to set indirect table. error: %d\n", ret);
-		return ENA_COM_INVAL;
-	}
 
-	return 0;
+	return ret;
 }
 
 int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl)
@@ -2538,17 +2494,18 @@ int ena_com_allocate_host_info(struct ena_com_dev *ena_dev)
 }
 
 int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev,
-				u32 debug_area_size) {
+				u32 debug_area_size)
+{
 	struct ena_host_attribute *host_attr = &ena_dev->host_attr;
 
-		ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
-				       debug_area_size,
-				       host_attr->debug_area_virt_addr,
-				       host_attr->debug_area_dma_addr,
-				       host_attr->debug_area_dma_handle);
-		if (unlikely(!host_attr->debug_area_virt_addr)) {
-			host_attr->debug_area_size = 0;
-			return ENA_COM_NO_MEM;
+	ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
+			       debug_area_size,
+			       host_attr->debug_area_virt_addr,
+			       host_attr->debug_area_dma_addr,
+			       host_attr->debug_area_dma_handle);
+	if (unlikely(!host_attr->debug_area_virt_addr)) {
+		host_attr->debug_area_size = 0;
+		return ENA_COM_NO_MEM;
 	}
 
 	host_attr->debug_area_size = debug_area_size;
@@ -2590,6 +2547,7 @@ int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
 	struct ena_com_admin_queue *admin_queue;
 	struct ena_admin_set_feat_cmd cmd;
 	struct ena_admin_set_feat_resp resp;
+
 	int ret;
 
 	/* Host attribute config is called before ena_com_get_dev_attr_feat
@@ -2635,14 +2593,12 @@ int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
 /* Interrupt moderation */
 bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev)
 {
-	return ena_com_check_supported_feature_id(
-			ena_dev,
-			ENA_ADMIN_INTERRUPT_MODERATION);
+	return ena_com_check_supported_feature_id(ena_dev,
+						  ENA_ADMIN_INTERRUPT_MODERATION);
 }
 
-int
-ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
-						  u32 tx_coalesce_usecs)
+int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
+						      u32 tx_coalesce_usecs)
 {
 	if (!ena_dev->intr_delay_resolution) {
 		ena_trc_err("Illegal interrupt delay granularity value\n");
@@ -2655,9 +2611,8 @@ ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
 	return 0;
 }
 
-int
-ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
-						  u32 rx_coalesce_usecs)
+int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
+						      u32 rx_coalesce_usecs)
 {
 	if (!ena_dev->intr_delay_resolution) {
 		ena_trc_err("Illegal interrupt delay granularity value\n");
@@ -2690,9 +2645,9 @@ int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
 				 ENA_ADMIN_INTERRUPT_MODERATION);
 
 	if (rc) {
-		if (rc == ENA_COM_PERMISSION) {
-			ena_trc_info("Feature %d isn't supported\n",
-				     ENA_ADMIN_INTERRUPT_MODERATION);
+		if (rc == ENA_COM_UNSUPPORTED) {
+			ena_trc_dbg("Feature %d isn't supported\n",
+				    ENA_ADMIN_INTERRUPT_MODERATION);
 			rc = 0;
 		} else {
 			ena_trc_err("Failed to get interrupt moderation admin cmd. rc: %d\n",
@@ -2719,8 +2674,7 @@ err:
 	return rc;
 }
 
-void
-ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev)
+void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev)
 {
 	struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
 
@@ -2763,14 +2717,12 @@ ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev)
 		ENA_INTR_HIGHEST_BYTES;
 }
 
-unsigned int
-ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev)
+unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev)
 {
 	return ena_dev->intr_moder_tx_interval;
 }
 
-unsigned int
-ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev)
+unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev)
 {
 	struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
 
@@ -2794,7 +2746,10 @@ void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev,
 		intr_moder_tbl[level].intr_moder_interval /=
 			ena_dev->intr_delay_resolution;
 	intr_moder_tbl[level].pkts_per_interval = entry->pkts_per_interval;
-	intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval;
+
+	/* use hardcoded value until ethtool supports bytecount parameter */
+	if (entry->bytes_per_interval != ENA_INTR_BYTE_COUNT_NOT_SUPPORTED)
+		intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval;
 }
 
 void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
diff --git a/drivers/net/ena/base/ena_com.h b/drivers/net/ena/base/ena_com.h
index e5345926..f58cd86a 100644
--- a/drivers/net/ena/base/ena_com.h
+++ b/drivers/net/ena/base/ena_com.h
@@ -35,15 +35,7 @@
 #define ENA_COM
 
 #include "ena_plat.h"
-#include "ena_common_defs.h"
-#include "ena_admin_defs.h"
-#include "ena_eth_io_defs.h"
-#include "ena_regs_defs.h"
-#if defined(__linux__) && !defined(__KERNEL__)
-#include <rte_lcore.h>
-#include <rte_spinlock.h>
-#define __iomem
-#endif
+#include "ena_includes.h"
 
 #define ENA_MAX_NUM_IO_QUEUES		128U
 /* We need to queues for each IO (on for Tx and one for Rx) */
@@ -89,6 +81,11 @@
 #define ENA_INTR_DELAY_OLD_VALUE_WEIGHT			6
 #define ENA_INTR_DELAY_NEW_VALUE_WEIGHT			4
 
+#define ENA_INTR_MODER_LEVEL_STRIDE			1
+#define ENA_INTR_BYTE_COUNT_NOT_SUPPORTED		0xFFFFFF
+
+#define ENA_HW_HINTS_NO_TIMEOUT				0xFFFF
+
 enum ena_intr_moder_level {
 	ENA_INTR_MODER_LOWEST = 0,
 	ENA_INTR_MODER_LOW,
@@ -120,8 +117,8 @@ struct ena_com_rx_buf_info {
 };
 
 struct ena_com_io_desc_addr {
-	u8  __iomem *pbuf_dev_addr; /* LLQ address */
-	u8  *virt_addr;
+	u8 __iomem *pbuf_dev_addr; /* LLQ address */
+	u8 *virt_addr;
 	dma_addr_t phys_addr;
 	ena_mem_handle_t mem_handle;
 };
@@ -130,13 +127,12 @@ struct ena_com_tx_meta {
 	u16 mss;
 	u16 l3_hdr_len;
 	u16 l3_hdr_offset;
-	u16 l3_outer_hdr_len; /* In words */
-	u16 l3_outer_hdr_offset;
 	u16 l4_hdr_len; /* In words */
 };
 
 struct ena_com_io_cq {
 	struct ena_com_io_desc_addr cdesc_addr;
+	void *bus;
 
 	/* Interrupt unmask register */
 	u32 __iomem *unmask_reg;
@@ -174,6 +170,7 @@ struct ena_com_io_cq {
 
 struct ena_com_io_sq {
 	struct ena_com_io_desc_addr desc_addr;
+	void *bus;
 
 	u32 __iomem *db_addr;
 	u8 __iomem *header_addr;
@@ -228,8 +225,11 @@ struct ena_com_stats_admin {
 
 struct ena_com_admin_queue {
 	void *q_dmadev;
+	void *bus;
 	ena_spinlock_t q_lock; /* spinlock for the admin queue */
+
 	struct ena_comp_ctx *comp_ctx;
+	u32 completion_timeout;
 	u16 q_depth;
 	struct ena_com_admin_cq cq;
 	struct ena_com_admin_sq sq;
@@ -266,6 +266,7 @@ struct ena_com_mmio_read {
 	struct ena_admin_ena_mmio_req_read_less_resp *read_resp;
 	dma_addr_t read_resp_dma_addr;
 	ena_mem_handle_t read_resp_mem_handle;
+	u32 reg_read_to; /* in us */
 	u16 seq_num;
 	bool readless_supported;
 	/* spin lock to ensure a single outstanding read */
@@ -316,6 +317,7 @@ struct ena_com_dev {
 	u8 __iomem *reg_bar;
 	void __iomem *mem_bar;
 	void *dmadev;
+	void *bus;
 
 	enum ena_admin_placement_policy_type tx_mem_queue_type;
 	u32 tx_max_header_size;
@@ -340,6 +342,7 @@ struct ena_com_dev_get_features_ctx {
 	struct ena_admin_device_attr_feature_desc dev_attr;
 	struct ena_admin_feature_aenq_desc aenq;
 	struct ena_admin_feature_offload_desc offload;
+	struct ena_admin_ena_hw_hints hw_hints;
 };
 
 struct ena_com_create_io_ctx {
@@ -379,7 +382,7 @@ int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev);
 
 /* ena_com_set_mmio_read_mode - Enable/disable the mmio reg read mechanism
  * @ena_dev: ENA communication layer struct
- * @realess_supported: readless mode (enable/disable)
+ * @readless_supported: readless mode (enable/disable)
  */
 void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev,
 				bool readless_supported);
@@ -421,14 +424,16 @@ void ena_com_admin_destroy(struct ena_com_dev *ena_dev);
 
 /* ena_com_dev_reset - Perform device FLR to the device.
  * @ena_dev: ENA communication layer struct
+ * @reset_reason: Specify what is the trigger for the reset in case of an error.
  *
  * @return - 0 on success, negative value on failure.
  */
-int ena_com_dev_reset(struct ena_com_dev *ena_dev);
+int ena_com_dev_reset(struct ena_com_dev *ena_dev,
+		      enum ena_regs_reset_reason_types reset_reason);
 
 /* ena_com_create_io_queue - Create io queue.
  * @ena_dev: ENA communication layer struct
- * ena_com_create_io_ctx - create context structure
+ * @ctx - create context structure
  *
  * Create the submission and the completion queues.
  *
@@ -437,8 +442,9 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev);
 int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
 			    struct ena_com_create_io_ctx *ctx);
 
-/* ena_com_admin_destroy - Destroy IO queue with the queue id - qid.
+/* ena_com_destroy_io_queue - Destroy IO queue with the queue id - qid.
  * @ena_dev: ENA communication layer struct
+ * @qid - the caller virtual queue id.
  */
 void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid);
 
@@ -581,9 +587,8 @@ int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag);
  *
  * @return: 0 on Success and negative value otherwise.
  */
-int
-ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
-			  struct ena_com_dev_get_features_ctx *get_feat_ctx);
+int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
+			      struct ena_com_dev_get_features_ctx *get_feat_ctx);
 
 /* ena_com_get_dev_basic_stats - Get device basic statistics
  * @ena_dev: ENA communication layer struct
@@ -608,9 +613,8 @@ int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu);
  *
  * @return: 0 on Success and negative value otherwise.
  */
-int
-ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
-			     struct ena_admin_feature_offload_desc *offload);
+int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
+				 struct ena_admin_feature_offload_desc *offload);
 
 /* ena_com_rss_init - Init RSS
  * @ena_dev: ENA communication layer struct
@@ -765,8 +769,8 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev);
  *
  * Retrieve the RSS indirection table from the device.
  *
- * @note: If the caller called ena_com_indirect_table_fill_entry but didn't
- * flash it to the device, the new configuration will be lost.
+ * @note: If the caller called ena_com_indirect_table_fill_entry but didn't flash
+ * it to the device, the new configuration will be lost.
  *
  * @return: 0 on Success and negative value otherwise.
  */
@@ -874,8 +878,7 @@ bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev);
  * moderation table back to the default parameters.
  * @ena_dev: ENA communication layer struct
  */
-void
-ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev);
+void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev);
 
 /* ena_com_update_nonadaptive_moderation_interval_tx - Update the
  * non-adaptive interval in Tx direction.
@@ -884,9 +887,8 @@ ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev);
  *
  * @return - 0 on success, negative value on failure.
  */
-int
-ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
-						  u32 tx_coalesce_usecs);
+int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
+						      u32 tx_coalesce_usecs);
 
 /* ena_com_update_nonadaptive_moderation_interval_rx - Update the
  * non-adaptive interval in Rx direction.
@@ -895,9 +897,8 @@ ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
  *
  * @return - 0 on success, negative value on failure.
  */
-int
-ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
-						  u32 rx_coalesce_usecs);
+int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
+						      u32 rx_coalesce_usecs);
 
 /* ena_com_get_nonadaptive_moderation_interval_tx - Retrieve the
  * non-adaptive interval in Tx direction.
@@ -905,8 +906,7 @@ ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
  *
  * @return - interval in usec
  */
-unsigned int
-ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev);
+unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev);
 
 /* ena_com_get_nonadaptive_moderation_interval_rx - Retrieve the
  * non-adaptive interval in Rx direction.
@@ -914,8 +914,7 @@ ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev);
  *
  * @return - interval in usec
  */
-unsigned int
-ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev);
+unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev);
 
 /* ena_com_init_intr_moderation_entry - Update a single entry in the interrupt
  * moderation table.
@@ -940,20 +939,17 @@ void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
 				       enum ena_intr_moder_level level,
 				       struct ena_intr_moder_entry *entry);
 
-static inline bool
-ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev)
+static inline bool ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev)
 {
 	return ena_dev->adaptive_coalescing;
 }
 
-static inline void
-ena_com_enable_adaptive_moderation(struct ena_com_dev *ena_dev)
+static inline void ena_com_enable_adaptive_moderation(struct ena_com_dev *ena_dev)
 {
 	ena_dev->adaptive_coalescing = true;
 }
 
-static inline void
-ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_dev)
+static inline void ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_dev)
 {
 	ena_dev->adaptive_coalescing = false;
 }
@@ -966,12 +962,11 @@ ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_dev)
  * @moder_tbl_idx: Current table level as input update new level as return
  * value.
  */
-static inline void
-ena_com_calculate_interrupt_delay(struct ena_com_dev *ena_dev,
-				  unsigned int pkts,
-				  unsigned int bytes,
-				  unsigned int *smoothed_interval,
-				  unsigned int *moder_tbl_idx)
+static inline void ena_com_calculate_interrupt_delay(struct ena_com_dev *ena_dev,
+						     unsigned int pkts,
+						     unsigned int bytes,
+						     unsigned int *smoothed_interval,
+						     unsigned int *moder_tbl_idx)
 {
 	enum ena_intr_moder_level curr_moder_idx, new_moder_idx;
 	struct ena_intr_moder_entry *curr_moder_entry;
@@ -1001,17 +996,20 @@ ena_com_calculate_interrupt_delay(struct ena_com_dev *ena_dev,
 	if (curr_moder_idx == ENA_INTR_MODER_LOWEST) {
 		if ((pkts > curr_moder_entry->pkts_per_interval) ||
 		    (bytes > curr_moder_entry->bytes_per_interval))
-			new_moder_idx = (enum ena_intr_moder_level)(curr_moder_idx + 1);
+			new_moder_idx =
+				(enum ena_intr_moder_level)(curr_moder_idx + ENA_INTR_MODER_LEVEL_STRIDE);
 	} else {
-		pred_moder_entry = &intr_moder_tbl[curr_moder_idx - 1];
+		pred_moder_entry = &intr_moder_tbl[curr_moder_idx - ENA_INTR_MODER_LEVEL_STRIDE];
 
 		if ((pkts <= pred_moder_entry->pkts_per_interval) ||
 		    (bytes <= pred_moder_entry->bytes_per_interval))
-			new_moder_idx = (enum ena_intr_moder_level)(curr_moder_idx - 1);
+			new_moder_idx =
+				(enum ena_intr_moder_level)(curr_moder_idx - ENA_INTR_MODER_LEVEL_STRIDE);
 		else if ((pkts > curr_moder_entry->pkts_per_interval) ||
 			 (bytes > curr_moder_entry->bytes_per_interval)) {
 			if (curr_moder_idx != ENA_INTR_MODER_HIGHEST)
-				new_moder_idx = (enum ena_intr_moder_level)(curr_moder_idx + 1);
+				new_moder_idx =
+					(enum ena_intr_moder_level)(curr_moder_idx + ENA_INTR_MODER_LEVEL_STRIDE);
 		}
 	}
 	new_moder_entry = &intr_moder_tbl[new_moder_idx];
@@ -1044,18 +1042,12 @@ static inline void ena_com_update_intr_reg(struct ena_eth_io_intr_reg *intr_reg,
 
 	intr_reg->intr_control |=
 		(tx_delay_interval << ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT)
-		& ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK;
+		& ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK;
 
 	if (unmask)
 		intr_reg->intr_control |= ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK;
 }
 
-int ena_com_get_dev_extended_stats(struct ena_com_dev *ena_dev, char *buff,
-				   u32 len);
-
-int ena_com_extended_stats_set_func_queue(struct ena_com_dev *ena_dev,
-					  u32 funct_queue);
-
 #if defined(__cplusplus)
 }
 #endif /* __cplusplus */
diff --git a/drivers/net/ena/base/ena_defs/ena_admin_defs.h b/drivers/net/ena/base/ena_defs/ena_admin_defs.h
index 7a031d90..04d4e9a5 100644
--- a/drivers/net/ena/base/ena_defs/ena_admin_defs.h
+++ b/drivers/net/ena/base/ena_defs/ena_admin_defs.h
@@ -34,174 +34,140 @@
 #ifndef _ENA_ADMIN_H_
 #define _ENA_ADMIN_H_
 
-/* admin commands opcodes */
 enum ena_admin_aq_opcode {
-	/* create submission queue */
-	ENA_ADMIN_CREATE_SQ = 1,
+	ENA_ADMIN_CREATE_SQ	= 1,
 
-	/* destroy submission queue */
-	ENA_ADMIN_DESTROY_SQ = 2,
+	ENA_ADMIN_DESTROY_SQ	= 2,
 
-	/* create completion queue */
-	ENA_ADMIN_CREATE_CQ = 3,
+	ENA_ADMIN_CREATE_CQ	= 3,
 
-	/* destroy completion queue */
-	ENA_ADMIN_DESTROY_CQ = 4,
+	ENA_ADMIN_DESTROY_CQ	= 4,
 
-	/* get capabilities of particular feature */
-	ENA_ADMIN_GET_FEATURE = 8,
+	ENA_ADMIN_GET_FEATURE	= 8,
 
-	/* get capabilities of particular feature */
-	ENA_ADMIN_SET_FEATURE = 9,
+	ENA_ADMIN_SET_FEATURE	= 9,
 
-	/* get statistics */
-	ENA_ADMIN_GET_STATS = 11,
+	ENA_ADMIN_GET_STATS	= 11,
 };
 
-/* admin command completion status codes */
 enum ena_admin_aq_completion_status {
-	/* Request completed successfully */
-	ENA_ADMIN_SUCCESS = 0,
+	ENA_ADMIN_SUCCESS			= 0,
 
-	/* no resources to satisfy request */
-	ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE = 1,
+	ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE	= 1,
 
-	/* Bad opcode in request descriptor */
-	ENA_ADMIN_BAD_OPCODE = 2,
+	ENA_ADMIN_BAD_OPCODE			= 2,
 
-	/* Unsupported opcode in request descriptor */
-	ENA_ADMIN_UNSUPPORTED_OPCODE = 3,
+	ENA_ADMIN_UNSUPPORTED_OPCODE		= 3,
 
-	/* Wrong request format */
-	ENA_ADMIN_MALFORMED_REQUEST = 4,
+	ENA_ADMIN_MALFORMED_REQUEST		= 4,
 
-	/* One of parameters is not valid. Provided in ACQ entry
-	 * extended_status
-	 */
-	ENA_ADMIN_ILLEGAL_PARAMETER = 5,
+	/* Additional status is provided in ACQ entry extended_status */
+	ENA_ADMIN_ILLEGAL_PARAMETER		= 5,
 
-	/* unexpected error */
-	ENA_ADMIN_UNKNOWN_ERROR = 6,
+	ENA_ADMIN_UNKNOWN_ERROR			= 6,
 };
 
-/* get/set feature subcommands opcodes */
 enum ena_admin_aq_feature_id {
-	/* list of all supported attributes/capabilities in the ENA */
-	ENA_ADMIN_DEVICE_ATTRIBUTES = 1,
+	ENA_ADMIN_DEVICE_ATTRIBUTES		= 1,
+
+	ENA_ADMIN_MAX_QUEUES_NUM		= 2,
 
-	/* max number of supported queues per for every queues type */
-	ENA_ADMIN_MAX_QUEUES_NUM = 2,
+	ENA_ADMIN_HW_HINTS			= 3,
 
-	/* Receive Side Scaling (RSS) function */
-	ENA_ADMIN_RSS_HASH_FUNCTION = 10,
+	ENA_ADMIN_RSS_HASH_FUNCTION		= 10,
 
-	/* stateless TCP/UDP/IP offload capabilities. */
-	ENA_ADMIN_STATELESS_OFFLOAD_CONFIG = 11,
+	ENA_ADMIN_STATELESS_OFFLOAD_CONFIG	= 11,
 
-	/* Multiple tuples flow table configuration */
-	ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG = 12,
+	ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG	= 12,
 
-	/* max MTU, current MTU */
-	ENA_ADMIN_MTU = 14,
+	ENA_ADMIN_MTU				= 14,
 
-	/* Receive Side Scaling (RSS) hash input */
-	ENA_ADMIN_RSS_HASH_INPUT = 18,
+	ENA_ADMIN_RSS_HASH_INPUT		= 18,
 
-	/* interrupt moderation parameters */
-	ENA_ADMIN_INTERRUPT_MODERATION = 20,
+	ENA_ADMIN_INTERRUPT_MODERATION		= 20,
 
-	/* AENQ configuration */
-	ENA_ADMIN_AENQ_CONFIG = 26,
+	ENA_ADMIN_AENQ_CONFIG			= 26,
 
-	/* Link configuration */
-	ENA_ADMIN_LINK_CONFIG = 27,
+	ENA_ADMIN_LINK_CONFIG			= 27,
 
-	/* Host attributes configuration */
-	ENA_ADMIN_HOST_ATTR_CONFIG = 28,
+	ENA_ADMIN_HOST_ATTR_CONFIG		= 28,
 
-	/* Number of valid opcodes */
-	ENA_ADMIN_FEATURES_OPCODE_NUM = 32,
+	ENA_ADMIN_FEATURES_OPCODE_NUM		= 32,
 };
 
-/* descriptors and headers placement */
 enum ena_admin_placement_policy_type {
-	/* descriptors and headers are in OS memory */
-	ENA_ADMIN_PLACEMENT_POLICY_HOST = 1,
+	/* descriptors and headers are in host memory */
+	ENA_ADMIN_PLACEMENT_POLICY_HOST	= 1,
 
-	/* descriptors and headers in device memory (a.k.a Low Latency
+	/* descriptors and headers are in device memory (a.k.a Low Latency
 	 * Queue)
 	 */
-	ENA_ADMIN_PLACEMENT_POLICY_DEV = 3,
+	ENA_ADMIN_PLACEMENT_POLICY_DEV	= 3,
 };
 
-/* link speeds */
 enum ena_admin_link_types {
-	ENA_ADMIN_LINK_SPEED_1G = 0x1,
+	ENA_ADMIN_LINK_SPEED_1G		= 0x1,
 
-	ENA_ADMIN_LINK_SPEED_2_HALF_G = 0x2,
+	ENA_ADMIN_LINK_SPEED_2_HALF_G	= 0x2,
 
-	ENA_ADMIN_LINK_SPEED_5G = 0x4,
+	ENA_ADMIN_LINK_SPEED_5G		= 0x4,
 
-	ENA_ADMIN_LINK_SPEED_10G = 0x8,
+	ENA_ADMIN_LINK_SPEED_10G	= 0x8,
 
-	ENA_ADMIN_LINK_SPEED_25G = 0x10,
+	ENA_ADMIN_LINK_SPEED_25G	= 0x10,
 
-	ENA_ADMIN_LINK_SPEED_40G = 0x20,
+	ENA_ADMIN_LINK_SPEED_40G	= 0x20,
 
-	ENA_ADMIN_LINK_SPEED_50G = 0x40,
+	ENA_ADMIN_LINK_SPEED_50G	= 0x40,
 
-	ENA_ADMIN_LINK_SPEED_100G = 0x80,
+	ENA_ADMIN_LINK_SPEED_100G	= 0x80,
 
-	ENA_ADMIN_LINK_SPEED_200G = 0x100,
+	ENA_ADMIN_LINK_SPEED_200G	= 0x100,
 
-	ENA_ADMIN_LINK_SPEED_400G = 0x200,
+	ENA_ADMIN_LINK_SPEED_400G	= 0x200,
 };
 
-/* completion queue update policy */
 enum ena_admin_completion_policy_type {
-	/* cqe for each sq descriptor */
-	ENA_ADMIN_COMPLETION_POLICY_DESC = 0,
+	/* completion queue entry for each sq descriptor */
+	ENA_ADMIN_COMPLETION_POLICY_DESC		= 0,
 
-	/* cqe upon request in sq descriptor */
-	ENA_ADMIN_COMPLETION_POLICY_DESC_ON_DEMAND = 1,
+	/* completion queue entry upon request in sq descriptor */
+	ENA_ADMIN_COMPLETION_POLICY_DESC_ON_DEMAND	= 1,
 
 	/* current queue head pointer is updated in OS memory upon sq
 	 * descriptor request
 	 */
-	ENA_ADMIN_COMPLETION_POLICY_HEAD_ON_DEMAND = 2,
+	ENA_ADMIN_COMPLETION_POLICY_HEAD_ON_DEMAND	= 2,
 
 	/* current queue head pointer is updated in OS memory for each sq
 	 * descriptor
 	 */
-	ENA_ADMIN_COMPLETION_POLICY_HEAD = 3,
+	ENA_ADMIN_COMPLETION_POLICY_HEAD		= 3,
 };
 
-/* type of get statistics command */
+/* basic stats return ena_admin_basic_stats while extanded stats return a
+ * buffer (string format) with additional statistics per queue and per
+ * device id
+ */
 enum ena_admin_get_stats_type {
-	/* Basic statistics */
-	ENA_ADMIN_GET_STATS_TYPE_BASIC = 0,
+	ENA_ADMIN_GET_STATS_TYPE_BASIC		= 0,
 
-	/* Extended statistics */
-	ENA_ADMIN_GET_STATS_TYPE_EXTENDED = 1,
+	ENA_ADMIN_GET_STATS_TYPE_EXTENDED	= 1,
 };
 
-/* scope of get statistics command */
 enum ena_admin_get_stats_scope {
-	ENA_ADMIN_SPECIFIC_QUEUE = 0,
+	ENA_ADMIN_SPECIFIC_QUEUE	= 0,
 
-	ENA_ADMIN_ETH_TRAFFIC = 1,
+	ENA_ADMIN_ETH_TRAFFIC		= 1,
 };
 
-/* ENA Admin Queue (AQ) common descriptor */
 struct ena_admin_aq_common_desc {
-	/* word 0 : */
-	/* command identificator to associate it with the completion
-	 * 11:0 : command_id
+	/* 11:0 : command_id
 	 * 15:12 : reserved12
 	 */
 	uint16_t command_id;
 
-	/* as appears in ena_aq_opcode */
+	/* as appears in ena_admin_aq_opcode */
 	uint8_t opcode;
 
 	/* 0 : phase
@@ -214,24 +180,17 @@ struct ena_admin_aq_common_desc {
 	uint8_t flags;
 };
 
-/* used in ena_aq_entry. Can point directly to control data, or to a page
- * list chunk. Used also at the end of indirect mode page list chunks, for
- * chaining.
+/* used in ena_admin_aq_entry. Can point directly to control data, or to a
+ * page list chunk. Used also at the end of indirect mode page list chunks,
+ * for chaining.
  */
 struct ena_admin_ctrl_buff_info {
-	/* word 0 : indicates length of the buffer pointed by
-	 * control_buffer_address.
-	 */
 	uint32_t length;
 
-	/* words 1:2 : points to control buffer (direct or indirect) */
 	struct ena_common_mem_addr address;
 };
 
-/* submission queue full identification */
 struct ena_admin_sq {
-	/* word 0 : */
-	/* queue id */
 	uint16_t sq_idx;
 
 	/* 4:0 : reserved
@@ -242,36 +201,25 @@ struct ena_admin_sq {
 	uint8_t reserved1;
 };
 
-/* AQ entry format */
 struct ena_admin_aq_entry {
-	/* words 0 :  */
 	struct ena_admin_aq_common_desc aq_common_descriptor;
 
-	/* words 1:3 :  */
 	union {
-		/* command specific inline data */
 		uint32_t inline_data_w1[3];
 
-		/* words 1:3 : points to control buffer (direct or
-		 * indirect, chained if needed)
-		 */
 		struct ena_admin_ctrl_buff_info control_buffer;
 	} u;
 
-	/* command specific inline data */
 	uint32_t inline_data_w4[12];
 };
 
-/* ENA Admin Completion Queue (ACQ) common descriptor */
 struct ena_admin_acq_common_desc {
-	/* word 0 : */
 	/* command identifier to associate it with the aq descriptor
 	 * 11:0 : command_id
 	 * 15:12 : reserved12
 	 */
 	uint16_t command;
 
-	/* status of request execution */
 	uint8_t status;
 
 	/* 0 : phase
@@ -279,33 +227,21 @@ struct ena_admin_acq_common_desc {
 	 */
 	uint8_t flags;
 
-	/* word 1 : */
-	/* provides additional info */
 	uint16_t extended_status;
 
-	/* submission queue head index, serves as a hint what AQ entries can
-	 *    be revoked
-	 */
+	/* serves as a hint what AQ entries can be revoked */
 	uint16_t sq_head_indx;
 };
 
-/* ACQ entry format */
 struct ena_admin_acq_entry {
-	/* words 0:1 :  */
 	struct ena_admin_acq_common_desc acq_common_descriptor;
 
-	/* response type specific data */
 	uint32_t response_specific_data[14];
 };
 
-/* ENA AQ Create Submission Queue command. Placed in control buffer pointed
- * by AQ entry
- */
 struct ena_admin_aq_create_sq_cmd {
-	/* words 0 :  */
 	struct ena_admin_aq_common_desc aq_common_descriptor;
 
-	/* word 1 : */
 	/* 4:0 : reserved0_w1
 	 * 7:5 : sq_direction - 0x1 - Tx, 0x2 - Rx
 	 */
@@ -337,7 +273,6 @@ struct ena_admin_aq_create_sq_cmd {
 	 */
 	uint8_t sq_caps_3;
 
-	/* word 2 : */
 	/* associated completion queue id. This CQ must be created prior to
 	 *    SQ creation
 	 */
@@ -346,85 +281,62 @@ struct ena_admin_aq_create_sq_cmd {
 	/* submission queue depth in entries */
 	uint16_t sq_depth;
 
-	/* words 3:4 : SQ physical base address in OS memory. This field
-	 * should not be used for Low Latency queues. Has to be page
-	 * aligned.
+	/* SQ physical base address in OS memory. This field should not be
+	 * used for Low Latency queues. Has to be page aligned.
 	 */
 	struct ena_common_mem_addr sq_ba;
 
-	/* words 5:6 : specifies queue head writeback location in OS
-	 * memory. Valid if completion_policy is set to
-	 * completion_policy_head_on_demand or completion_policy_head. Has
-	 * to be cache aligned
+	/* specifies queue head writeback location in OS memory. Valid if
+	 * completion_policy is set to completion_policy_head_on_demand or
+	 * completion_policy_head. Has to be cache aligned
 	 */
 	struct ena_common_mem_addr sq_head_writeback;
 
-	/* word 7 : reserved word */
 	uint32_t reserved0_w7;
 
-	/* word 8 : reserved word */
 	uint32_t reserved0_w8;
 };
 
-/* submission queue direction */
 enum ena_admin_sq_direction {
-	ENA_ADMIN_SQ_DIRECTION_TX = 1,
+	ENA_ADMIN_SQ_DIRECTION_TX	= 1,
 
-	ENA_ADMIN_SQ_DIRECTION_RX = 2,
+	ENA_ADMIN_SQ_DIRECTION_RX	= 2,
 };
 
-/* ENA Response for Create SQ Command. Appears in ACQ entry as
- * response_specific_data
- */
 struct ena_admin_acq_create_sq_resp_desc {
-	/* words 0:1 : Common Admin Queue completion descriptor */
 	struct ena_admin_acq_common_desc acq_common_desc;
 
-	/* word 2 : */
-	/* sq identifier */
 	uint16_t sq_idx;
 
 	uint16_t reserved;
 
-	/* word 3 : queue doorbell address as an offset to PCIe MMIO REG BAR */
+	/* queue doorbell address as an offset to PCIe MMIO REG BAR */
 	uint32_t sq_doorbell_offset;
 
-	/* word 4 : low latency queue ring base address as an offset to
-	 * PCIe MMIO LLQ_MEM BAR
+	/* low latency queue ring base address as an offset to PCIe MMIO
+	 * LLQ_MEM BAR
 	 */
 	uint32_t llq_descriptors_offset;
 
-	/* word 5 : low latency queue headers' memory as an offset to PCIe
-	 * MMIO LLQ_MEM BAR
+	/* low latency queue headers' memory as an offset to PCIe MMIO
+	 * LLQ_MEM BAR
 	 */
 	uint32_t llq_headers_offset;
 };
 
-/* ENA AQ Destroy Submission Queue command. Placed in control buffer
- * pointed by AQ entry
- */
 struct ena_admin_aq_destroy_sq_cmd {
-	/* words 0 :  */
 	struct ena_admin_aq_common_desc aq_common_descriptor;
 
-	/* words 1 :  */
 	struct ena_admin_sq sq;
 };
 
-/* ENA Response for Destroy SQ Command. Appears in ACQ entry as
- * response_specific_data
- */
 struct ena_admin_acq_destroy_sq_resp_desc {
-	/* words 0:1 : Common Admin Queue completion descriptor */
 	struct ena_admin_acq_common_desc acq_common_desc;
 };
 
-/* ENA AQ Create Completion Queue command */
 struct ena_admin_aq_create_cq_cmd {
-	/* words 0 :  */
 	struct ena_admin_aq_common_desc aq_common_descriptor;
 
-	/* word 1 : */
 	/* 4:0 : reserved5
 	 * 5 : interrupt_mode_enabled - if set, cq operates
 	 *    in interrupt mode, otherwise - polling
@@ -441,62 +353,39 @@ struct ena_admin_aq_create_cq_cmd {
 	/* completion queue depth in # of entries. must be power of 2 */
 	uint16_t cq_depth;
 
-	/* word 2 : msix vector assigned to this cq */
+	/* msix vector assigned to this cq */
 	uint32_t msix_vector;
 
-	/* words 3:4 : cq physical base address in OS memory. CQ must be
-	 * physically contiguous
+	/* cq physical base address in OS memory. CQ must be physically
+	 * contiguous
 	 */
 	struct ena_common_mem_addr cq_ba;
 };
 
-/* ENA Response for Create CQ Command. Appears in ACQ entry as response
- * specific data
- */
 struct ena_admin_acq_create_cq_resp_desc {
-	/* words 0:1 : Common Admin Queue completion descriptor */
 	struct ena_admin_acq_common_desc acq_common_desc;
 
-	/* word 2 : */
-	/* cq identifier */
 	uint16_t cq_idx;
 
-	/* actual cq depth in # of entries */
+	/* actual cq depth in number of entries */
 	uint16_t cq_actual_depth;
 
-	/* word 3 : cpu numa node address as an offset to PCIe MMIO REG BAR */
 	uint32_t numa_node_register_offset;
 
-	/* word 4 : completion head doorbell address as an offset to PCIe
-	 * MMIO REG BAR
-	 */
 	uint32_t cq_head_db_register_offset;
 
-	/* word 5 : interrupt unmask register address as an offset into
-	 * PCIe MMIO REG BAR
-	 */
 	uint32_t cq_interrupt_unmask_register_offset;
 };
 
-/* ENA AQ Destroy Completion Queue command. Placed in control buffer
- * pointed by AQ entry
- */
 struct ena_admin_aq_destroy_cq_cmd {
-	/* words 0 :  */
 	struct ena_admin_aq_common_desc aq_common_descriptor;
 
-	/* word 1 : */
-	/* associated queue id. */
 	uint16_t cq_idx;
 
 	uint16_t reserved1;
 };
 
-/* ENA Response for Destroy CQ Command. Appears in ACQ entry as
- * response_specific_data
- */
 struct ena_admin_acq_destroy_cq_resp_desc {
-	/* words 0:1 : Common Admin Queue completion descriptor */
 	struct ena_admin_acq_common_desc acq_common_desc;
 };
 
@@ -504,21 +393,15 @@ struct ena_admin_acq_destroy_cq_resp_desc {
  * buffer pointed by AQ entry
  */
 struct ena_admin_aq_get_stats_cmd {
-	/* words 0 :  */
 	struct ena_admin_aq_common_desc aq_common_descriptor;
 
-	/* words 1:3 :  */
 	union {
 		/* command specific inline data */
 		uint32_t inline_data_w1[3];
 
-		/* words 1:3 : points to control buffer (direct or
-		 * indirect, chained if needed)
-		 */
 		struct ena_admin_ctrl_buff_info control_buffer;
 	} u;
 
-	/* word 4 : */
 	/* stats type as defined in enum ena_admin_get_stats_type */
 	uint8_t type;
 
@@ -527,7 +410,6 @@ struct ena_admin_aq_get_stats_cmd {
 
 	uint16_t reserved3;
 
-	/* word 5 : */
 	/* queue id. used when scope is specific_queue */
 	uint16_t queue_idx;
 
@@ -539,89 +421,60 @@ struct ena_admin_aq_get_stats_cmd {
 
 /* Basic Statistics Command. */
 struct ena_admin_basic_stats {
-	/* word 0 :  */
 	uint32_t tx_bytes_low;
 
-	/* word 1 :  */
 	uint32_t tx_bytes_high;
 
-	/* word 2 :  */
 	uint32_t tx_pkts_low;
 
-	/* word 3 :  */
 	uint32_t tx_pkts_high;
 
-	/* word 4 :  */
 	uint32_t rx_bytes_low;
 
-	/* word 5 :  */
 	uint32_t rx_bytes_high;
 
-	/* word 6 :  */
 	uint32_t rx_pkts_low;
 
-	/* word 7 :  */
 	uint32_t rx_pkts_high;
 
-	/* word 8 :  */
 	uint32_t rx_drops_low;
 
-	/* word 9 :  */
 	uint32_t rx_drops_high;
 };
 
-/* ENA Response for Get Statistics Command. Appears in ACQ entry as
- * response_specific_data
- */
 struct ena_admin_acq_get_stats_resp {
-	/* words 0:1 : Common Admin Queue completion descriptor */
 	struct ena_admin_acq_common_desc acq_common_desc;
 
-	/* words 2:11 :  */
 	struct ena_admin_basic_stats basic_stats;
 };
 
-/* ENA Get/Set Feature common descriptor. Appears as inline word in
- * ena_aq_entry
- */
 struct ena_admin_get_set_feature_common_desc {
-	/* word 0 : */
 	/* 1:0 : select - 0x1 - current value; 0x3 - default
 	 *    value
 	 * 7:3 : reserved3
 	 */
 	uint8_t flags;
 
-	/* as appears in ena_feature_id */
+	/* as appears in ena_admin_aq_feature_id */
 	uint8_t feature_id;
 
-	/* reserved16 */
 	uint16_t reserved16;
 };
 
-/* ENA Device Attributes Feature descriptor. */
 struct ena_admin_device_attr_feature_desc {
-	/* word 0 : implementation id */
 	uint32_t impl_id;
 
-	/* word 1 : device version */
 	uint32_t device_version;
 
-	/* word 2 : bit map of which bits are supported value of 1
-	 * indicated that this feature is supported and can perform SET/GET
-	 * for it
-	 */
+	/* bitmap of ena_admin_aq_feature_id */
 	uint32_t supported_features;
 
-	/* word 3 :  */
 	uint32_t reserved3;
 
-	/* word 4 : Indicates how many bits are used physical address
-	 * access.
-	 */
+	/* Indicates how many bits are used physical address access. */
 	uint32_t phys_addr_width;
 
-	/* word 5 : Indicates how many bits are used virtual address access. */
+	/* Indicates how many bits are used virtual address access. */
 	uint32_t virt_addr_width;
 
 	/* unicast MAC address (in Network byte order) */
@@ -629,36 +482,27 @@ struct ena_admin_device_attr_feature_desc {
 
 	uint8_t reserved7[2];
 
-	/* word 8 : Max supported MTU value */
 	uint32_t max_mtu;
 };
 
-/* ENA Max Queues Feature descriptor. */
 struct ena_admin_queue_feature_desc {
-	/* word 0 : Max number of submission queues (including LLQs) */
+	/* including LLQs */
 	uint32_t max_sq_num;
 
-	/* word 1 : Max submission queue depth */
 	uint32_t max_sq_depth;
 
-	/* word 2 : Max number of completion queues */
 	uint32_t max_cq_num;
 
-	/* word 3 : Max completion queue depth */
 	uint32_t max_cq_depth;
 
-	/* word 4 : Max number of LLQ submission queues */
 	uint32_t max_llq_num;
 
-	/* word 5 : Max submission queue depth of LLQ */
 	uint32_t max_llq_depth;
 
-	/* word 6 : Max header size */
 	uint32_t max_header_size;
 
-	/* word 7 : */
-	/* Maximum Descriptors number, including meta descriptors, allowed
-	 *    for a single Tx packet
+	/* Maximum Descriptors number, including meta descriptor, allowed for
+	 *    a single Tx packet
 	 */
 	uint16_t max_packet_tx_descs;
 
@@ -666,86 +510,69 @@ struct ena_admin_queue_feature_desc {
 	uint16_t max_packet_rx_descs;
 };
 
-/* ENA MTU Set Feature descriptor. */
 struct ena_admin_set_feature_mtu_desc {
-	/* word 0 : mtu payload size (exclude L2) */
+	/* exclude L2 */
 	uint32_t mtu;
 };
 
-/* ENA host attributes Set Feature descriptor. */
 struct ena_admin_set_feature_host_attr_desc {
-	/* words 0:1 : host OS info base address in OS memory. host info is
-	 * 4KB of physically contiguous
+	/* host OS info base address in OS memory. host info is 4KB of
+	 * physically contiguous
 	 */
 	struct ena_common_mem_addr os_info_ba;
 
-	/* words 2:3 : host debug area base address in OS memory. debug
-	 * area must be physically contiguous
+	/* host debug area base address in OS memory. debug area must be
+	 * physically contiguous
 	 */
 	struct ena_common_mem_addr debug_ba;
 
-	/* word 4 : debug area size */
+	/* debug area size */
 	uint32_t debug_area_size;
 };
 
-/* ENA Interrupt Moderation Get Feature descriptor. */
 struct ena_admin_feature_intr_moder_desc {
-	/* word 0 : */
 	/* interrupt delay granularity in usec */
 	uint16_t intr_delay_resolution;
 
 	uint16_t reserved;
 };
 
-/* ENA Link Get Feature descriptor. */
 struct ena_admin_get_feature_link_desc {
-	/* word 0 : Link speed in Mb */
+	/* Link speed in Mb */
 	uint32_t speed;
 
-	/* word 1 : supported speeds (bit field of enum ena_admin_link
-	 * types)
-	 */
+	/* bit field of enum ena_admin_link types */
 	uint32_t supported;
 
-	/* word 2 : */
-	/* 0 : autoneg - auto negotiation
+	/* 0 : autoneg
 	 * 1 : duplex - Full Duplex
 	 * 31:2 : reserved2
 	 */
 	uint32_t flags;
 };
 
-/* ENA AENQ Feature descriptor. */
 struct ena_admin_feature_aenq_desc {
-	/* word 0 : bitmask for AENQ groups the device can report */
+	/* bitmask for AENQ groups the device can report */
 	uint32_t supported_groups;
 
-	/* word 1 : bitmask for AENQ groups to report */
+	/* bitmask for AENQ groups to report */
 	uint32_t enabled_groups;
 };
 
-/* ENA Stateless Offload Feature descriptor. */
 struct ena_admin_feature_offload_desc {
-	/* word 0 : */
-	/* Trasmit side stateless offload
-	 * 0 : TX_L3_csum_ipv4 - IPv4 checksum
-	 * 1 : TX_L4_ipv4_csum_part - TCP/UDP over IPv4
-	 *    checksum, the checksum field should be initialized
-	 *    with pseudo header checksum
-	 * 2 : TX_L4_ipv4_csum_full - TCP/UDP over IPv4
-	 *    checksum
-	 * 3 : TX_L4_ipv6_csum_part - TCP/UDP over IPv6
-	 *    checksum, the checksum field should be initialized
-	 *    with pseudo header checksum
-	 * 4 : TX_L4_ipv6_csum_full - TCP/UDP over IPv6
-	 *    checksum
-	 * 5 : tso_ipv4 - TCP/IPv4 Segmentation Offloading
-	 * 6 : tso_ipv6 - TCP/IPv6 Segmentation Offloading
-	 * 7 : tso_ecn - TCP Segmentation with ECN
+	/* 0 : TX_L3_csum_ipv4
+	 * 1 : TX_L4_ipv4_csum_part - The checksum field
+	 *    should be initialized with pseudo header checksum
+	 * 2 : TX_L4_ipv4_csum_full
+	 * 3 : TX_L4_ipv6_csum_part - The checksum field
+	 *    should be initialized with pseudo header checksum
+	 * 4 : TX_L4_ipv6_csum_full
+	 * 5 : tso_ipv4
+	 * 6 : tso_ipv6
+	 * 7 : tso_ecn
 	 */
 	uint32_t tx;
 
-	/* word 1 : */
 	/* Receive side supported stateless offload
 	 * 0 : RX_L3_csum_ipv4 - IPv4 checksum
 	 * 1 : RX_L4_ipv4_csum - TCP/UDP/IPv4 checksum
@@ -754,118 +581,94 @@ struct ena_admin_feature_offload_desc {
 	 */
 	uint32_t rx_supported;
 
-	/* word 2 : */
-	/* Receive side enabled stateless offload */
 	uint32_t rx_enabled;
 };
 
-/* hash functions */
 enum ena_admin_hash_functions {
-	/* Toeplitz hash */
-	ENA_ADMIN_TOEPLITZ = 1,
+	ENA_ADMIN_TOEPLITZ	= 1,
 
-	/* CRC32 hash */
-	ENA_ADMIN_CRC32 = 2,
+	ENA_ADMIN_CRC32		= 2,
 };
 
-/* ENA RSS flow hash control buffer structure */
 struct ena_admin_feature_rss_flow_hash_control {
-	/* word 0 : number of valid keys */
 	uint32_t keys_num;
 
-	/* word 1 :  */
 	uint32_t reserved;
 
-	/* Toeplitz keys */
 	uint32_t key[10];
 };
 
-/* ENA RSS Flow Hash Function */
 struct ena_admin_feature_rss_flow_hash_function {
-	/* word 0 : */
-	/* supported hash functions
-	 * 7:0 : funcs - supported hash functions (bitmask
-	 *    accroding to ena_admin_hash_functions)
-	 */
+	/* 7:0 : funcs - bitmask of ena_admin_hash_functions */
 	uint32_t supported_func;
 
-	/* word 1 : */
-	/* selected hash func
-	 * 7:0 : selected_func - selected hash function
-	 *    (bitmask accroding to ena_admin_hash_functions)
+	/* 7:0 : selected_func - bitmask of
+	 *    ena_admin_hash_functions
 	 */
 	uint32_t selected_func;
 
-	/* word 2 : initial value */
+	/* initial value */
 	uint32_t init_val;
 };
 
 /* RSS flow hash protocols */
 enum ena_admin_flow_hash_proto {
-	/* tcp/ipv4 */
-	ENA_ADMIN_RSS_TCP4 = 0,
+	ENA_ADMIN_RSS_TCP4	= 0,
 
-	/* udp/ipv4 */
-	ENA_ADMIN_RSS_UDP4 = 1,
+	ENA_ADMIN_RSS_UDP4	= 1,
 
-	/* tcp/ipv6 */
-	ENA_ADMIN_RSS_TCP6 = 2,
+	ENA_ADMIN_RSS_TCP6	= 2,
 
-	/* udp/ipv6 */
-	ENA_ADMIN_RSS_UDP6 = 3,
+	ENA_ADMIN_RSS_UDP6	= 3,
 
-	/* ipv4 not tcp/udp */
-	ENA_ADMIN_RSS_IP4 = 4,
+	ENA_ADMIN_RSS_IP4	= 4,
 
-	/* ipv6 not tcp/udp */
-	ENA_ADMIN_RSS_IP6 = 5,
+	ENA_ADMIN_RSS_IP6	= 5,
 
-	/* fragmented ipv4 */
-	ENA_ADMIN_RSS_IP4_FRAG = 6,
+	ENA_ADMIN_RSS_IP4_FRAG	= 6,
 
-	/* not ipv4/6 */
-	ENA_ADMIN_RSS_NOT_IP = 7,
+	ENA_ADMIN_RSS_NOT_IP	= 7,
 
-	/* max number of protocols */
-	ENA_ADMIN_RSS_PROTO_NUM = 16,
+	/* TCPv6 with extension header */
+	ENA_ADMIN_RSS_TCP6_EX	= 8,
+
+	/* IPv6 with extension header */
+	ENA_ADMIN_RSS_IP6_EX	= 9,
+
+	ENA_ADMIN_RSS_PROTO_NUM	= 16,
 };
 
 /* RSS flow hash fields */
 enum ena_admin_flow_hash_fields {
 	/* Ethernet Dest Addr */
-	ENA_ADMIN_RSS_L2_DA = 0,
+	ENA_ADMIN_RSS_L2_DA	= BIT(0),
 
 	/* Ethernet Src Addr */
-	ENA_ADMIN_RSS_L2_SA = 1,
+	ENA_ADMIN_RSS_L2_SA	= BIT(1),
 
 	/* ipv4/6 Dest Addr */
-	ENA_ADMIN_RSS_L3_DA = 2,
+	ENA_ADMIN_RSS_L3_DA	= BIT(2),
 
 	/* ipv4/6 Src Addr */
-	ENA_ADMIN_RSS_L3_SA = 5,
+	ENA_ADMIN_RSS_L3_SA	= BIT(3),
 
 	/* tcp/udp Dest Port */
-	ENA_ADMIN_RSS_L4_DP = 6,
+	ENA_ADMIN_RSS_L4_DP	= BIT(4),
 
 	/* tcp/udp Src Port */
-	ENA_ADMIN_RSS_L4_SP = 7,
+	ENA_ADMIN_RSS_L4_SP	= BIT(5),
 };
 
-/* hash input fields for flow protocol */
 struct ena_admin_proto_input {
-	/* word 0 : */
 	/* flow hash fields (bitwise according to ena_admin_flow_hash_fields) */
 	uint16_t fields;
 
 	uint16_t reserved2;
 };
 
-/* ENA RSS hash control buffer structure */
 struct ena_admin_feature_rss_hash_control {
-	/* supported input fields */
 	struct ena_admin_proto_input supported_fields[ENA_ADMIN_RSS_PROTO_NUM];
 
-	/* selected input fields */
 	struct ena_admin_proto_input selected_fields[ENA_ADMIN_RSS_PROTO_NUM];
 
 	struct ena_admin_proto_input reserved2[ENA_ADMIN_RSS_PROTO_NUM];
@@ -873,11 +676,9 @@ struct ena_admin_feature_rss_hash_control {
 	struct ena_admin_proto_input reserved3[ENA_ADMIN_RSS_PROTO_NUM];
 };
 
-/* ENA RSS flow hash input */
 struct ena_admin_feature_rss_flow_hash_input {
-	/* word 0 : */
 	/* supported hash input sorting
-	 * 1 : L3_sort - support swap L3 addresses if DA
+	 * 1 : L3_sort - support swap L3 addresses if DA is
 	 *    smaller than SA
 	 * 2 : L4_sort - support swap L4 ports if DP smaller
 	 *    SP
@@ -893,46 +694,37 @@ struct ena_admin_feature_rss_flow_hash_input {
 	uint16_t enabled_input_sort;
 };
 
-/* Operating system type */
 enum ena_admin_os_type {
-	/* Linux OS */
-	ENA_ADMIN_OS_LINUX = 1,
+	ENA_ADMIN_OS_LINUX	= 1,
 
-	/* Windows OS */
-	ENA_ADMIN_OS_WIN = 2,
+	ENA_ADMIN_OS_WIN	= 2,
 
-	/* DPDK OS */
-	ENA_ADMIN_OS_DPDK = 3,
+	ENA_ADMIN_OS_DPDK	= 3,
 
-	/* FreeBSD OS */
-	ENA_ADMIN_OS_FREEBSD = 4,
+	ENA_ADMIN_OS_FREEBSD	= 4,
 
-	/* PXE OS */
-	ENA_ADMIN_OS_IPXE = 5,
+	ENA_ADMIN_OS_IPXE	= 5,
 };
 
-/* host info */
 struct ena_admin_host_info {
-	/* word 0 : OS type defined in enum ena_os_type */
+	/* defined in enum ena_admin_os_type */
 	uint32_t os_type;
 
 	/* os distribution string format */
 	uint8_t os_dist_str[128];
 
-	/* word 33 : OS distribution numeric format */
+	/* OS distribution numeric format */
 	uint32_t os_dist;
 
 	/* kernel version string format */
 	uint8_t kernel_ver_str[32];
 
-	/* word 42 : Kernel version numeric format */
+	/* Kernel version numeric format */
 	uint32_t kernel_ver;
 
-	/* word 43 : */
-	/* driver version
-	 * 7:0 : major - major
-	 * 15:8 : minor - minor
-	 * 23:16 : sub_minor - sub minor
+	/* 7:0 : major
+	 * 15:8 : minor
+	 * 23:16 : sub_minor
 	 */
 	uint32_t driver_version;
 
@@ -940,220 +732,200 @@ struct ena_admin_host_info {
 	uint32_t supported_network_features[4];
 };
 
-/* ENA RSS indirection table entry */
 struct ena_admin_rss_ind_table_entry {
-	/* word 0 : */
-	/* cq identifier */
 	uint16_t cq_idx;
 
 	uint16_t reserved;
 };
 
-/* ENA RSS indirection table */
 struct ena_admin_feature_rss_ind_table {
-	/* word 0 : */
 	/* min supported table size (2^min_size) */
 	uint16_t min_size;
 
 	/* max supported table size (2^max_size) */
 	uint16_t max_size;
 
-	/* word 1 : */
 	/* table size (2^size) */
 	uint16_t size;
 
 	uint16_t reserved;
 
-	/* word 2 : index of the inline entry. 0xFFFFFFFF means invalid */
+	/* index of the inline entry. 0xFFFFFFFF means invalid */
 	uint32_t inline_index;
 
-	/* words 3 : used for updating single entry, ignored when setting
-	 * the entire table through the control buffer.
+	/* used for updating single entry, ignored when setting the entire
+	 * table through the control buffer.
 	 */
 	struct ena_admin_rss_ind_table_entry inline_entry;
 };
 
-/* ENA Get Feature command */
+/* When hint value is 0, driver should use it's own predefined value */
+struct ena_admin_ena_hw_hints {
+	/* value in ms */
+	uint16_t mmio_read_timeout;
+
+	/* value in ms */
+	uint16_t driver_watchdog_timeout;
+
+	/* Per packet tx completion timeout. value in ms */
+	uint16_t missing_tx_completion_timeout;
+
+	uint16_t missed_tx_completion_count_threshold_to_reset;
+
+	/* value in ms */
+	uint16_t admin_completion_tx_timeout;
+
+	uint16_t netdev_wd_timeout;
+
+	uint16_t max_tx_sgl_size;
+
+	uint16_t max_rx_sgl_size;
+
+	uint16_t reserved[8];
+};
+
 struct ena_admin_get_feat_cmd {
-	/* words 0 :  */
 	struct ena_admin_aq_common_desc aq_common_descriptor;
 
-	/* words 1:3 : points to control buffer (direct or indirect,
-	 * chained if needed)
-	 */
 	struct ena_admin_ctrl_buff_info control_buffer;
 
-	/* words 4 :  */
 	struct ena_admin_get_set_feature_common_desc feat_common;
 
-	/* words 5:15 :  */
-	union {
-		/* raw words */
-		uint32_t raw[11];
-	} u;
+	uint32_t raw[11];
 };
 
-/* ENA Get Feature command response */
 struct ena_admin_get_feat_resp {
-	/* words 0:1 :  */
 	struct ena_admin_acq_common_desc acq_common_desc;
 
-	/* words 2:15 :  */
 	union {
-		/* raw words */
 		uint32_t raw[14];
 
-		/* words 2:10 : Get Device Attributes */
 		struct ena_admin_device_attr_feature_desc dev_attr;
 
-		/* words 2:5 : Max queues num */
 		struct ena_admin_queue_feature_desc max_queue;
 
-		/* words 2:3 : AENQ configuration */
 		struct ena_admin_feature_aenq_desc aenq;
 
-		/* words 2:4 : Get Link configuration */
 		struct ena_admin_get_feature_link_desc link;
 
-		/* words 2:4 : offload configuration */
 		struct ena_admin_feature_offload_desc offload;
 
-		/* words 2:4 : rss flow hash function */
 		struct ena_admin_feature_rss_flow_hash_function flow_hash_func;
 
-		/* words 2 : rss flow hash input */
 		struct ena_admin_feature_rss_flow_hash_input flow_hash_input;
 
-		/* words 2:3 : rss indirection table */
 		struct ena_admin_feature_rss_ind_table ind_table;
 
-		/* words 2 : interrupt moderation configuration */
 		struct ena_admin_feature_intr_moder_desc intr_moderation;
+
+		struct ena_admin_ena_hw_hints hw_hints;
 	} u;
 };
 
-/* ENA Set Feature command */
 struct ena_admin_set_feat_cmd {
-	/* words 0 :  */
 	struct ena_admin_aq_common_desc aq_common_descriptor;
 
-	/* words 1:3 : points to control buffer (direct or indirect,
-	 * chained if needed)
-	 */
 	struct ena_admin_ctrl_buff_info control_buffer;
 
-	/* words 4 :  */
 	struct ena_admin_get_set_feature_common_desc feat_common;
 
-	/* words 5:15 :  */
 	union {
-		/* raw words */
 		uint32_t raw[11];
 
-		/* words 5 : mtu size */
+		/* mtu size */
 		struct ena_admin_set_feature_mtu_desc mtu;
 
-		/* words 5:7 : host attributes */
+		/* host attributes */
 		struct ena_admin_set_feature_host_attr_desc host_attr;
 
-		/* words 5:6 : AENQ configuration */
+		/* AENQ configuration */
 		struct ena_admin_feature_aenq_desc aenq;
 
-		/* words 5:7 : rss flow hash function */
+		/* rss flow hash function */
 		struct ena_admin_feature_rss_flow_hash_function flow_hash_func;
 
-		/* words 5 : rss flow hash input */
+		/* rss flow hash input */
 		struct ena_admin_feature_rss_flow_hash_input flow_hash_input;
 
-		/* words 5:6 : rss indirection table */
+		/* rss indirection table */
 		struct ena_admin_feature_rss_ind_table ind_table;
 	} u;
 };
 
-/* ENA Set Feature command response */
 struct ena_admin_set_feat_resp {
-	/* words 0:1 :  */
 	struct ena_admin_acq_common_desc acq_common_desc;
 
-	/* words 2:15 :  */
 	union {
-		/* raw words */
 		uint32_t raw[14];
 	} u;
 };
 
-/* ENA Asynchronous Event Notification Queue descriptor.  */
 struct ena_admin_aenq_common_desc {
-	/* word 0 : */
 	uint16_t group;
 
 	uint16_t syndrom;
 
-	/* word 1 : */
 	/* 0 : phase */
 	uint8_t flags;
 
 	uint8_t reserved1[3];
 
-	/* word 2 : Timestamp LSB */
 	uint32_t timestamp_low;
 
-	/* word 3 : Timestamp MSB */
 	uint32_t timestamp_high;
 };
 
 /* asynchronous event notification groups */
 enum ena_admin_aenq_group {
-	/* Link State Change */
-	ENA_ADMIN_LINK_CHANGE = 0,
+	ENA_ADMIN_LINK_CHANGE		= 0,
 
-	ENA_ADMIN_FATAL_ERROR = 1,
+	ENA_ADMIN_FATAL_ERROR		= 1,
 
-	ENA_ADMIN_WARNING = 2,
+	ENA_ADMIN_WARNING		= 2,
 
-	ENA_ADMIN_NOTIFICATION = 3,
+	ENA_ADMIN_NOTIFICATION		= 3,
 
-	ENA_ADMIN_KEEP_ALIVE = 4,
+	ENA_ADMIN_KEEP_ALIVE		= 4,
 
-	ENA_ADMIN_AENQ_GROUPS_NUM = 5,
+	ENA_ADMIN_AENQ_GROUPS_NUM	= 5,
 };
 
-/* syndorm of AENQ notification group */
 enum ena_admin_aenq_notification_syndrom {
-	ENA_ADMIN_SUSPEND = 0,
+	ENA_ADMIN_SUSPEND	= 0,
+
+	ENA_ADMIN_RESUME	= 1,
 
-	ENA_ADMIN_RESUME = 1,
+	ENA_ADMIN_UPDATE_HINTS	= 2,
 };
 
-/* ENA Asynchronous Event Notification generic descriptor.  */
 struct ena_admin_aenq_entry {
-	/* words 0:3 :  */
 	struct ena_admin_aenq_common_desc aenq_common_desc;
 
 	/* command specific inline data */
 	uint32_t inline_data_w4[12];
 };
 
-/* ENA Asynchronous Event Notification Queue Link Change descriptor.  */
 struct ena_admin_aenq_link_change_desc {
-	/* words 0:3 :  */
 	struct ena_admin_aenq_common_desc aenq_common_desc;
 
-	/* word 4 : */
 	/* 0 : link_status */
 	uint32_t flags;
 };
 
-/* ENA MMIO Readless response interface */
+struct ena_admin_aenq_keep_alive_desc {
+	struct ena_admin_aenq_common_desc aenq_common_desc;
+
+	uint32_t rx_drops_low;
+
+	uint32_t rx_drops_high;
+};
+
 struct ena_admin_ena_mmio_req_read_less_resp {
-	/* word 0 : */
-	/* request id */
 	uint16_t req_id;
 
-	/* register offset */
 	uint16_t reg_off;
 
-	/* word 1 : value is valid when poll is cleared */
+	/* value is valid when poll is cleared */
 	uint32_t reg_val;
 };
 
@@ -1220,8 +992,7 @@ struct ena_admin_ena_mmio_req_read_less_resp {
 
 /* feature_rss_flow_hash_function */
 #define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK GENMASK(7, 0)
-#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK \
-	GENMASK(7, 0)
+#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK GENMASK(7, 0)
 
 /* feature_rss_flow_hash_input */
 #define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT 1
@@ -1247,653 +1018,392 @@ struct ena_admin_ena_mmio_req_read_less_resp {
 #define ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK BIT(0)
 
 #if !defined(ENA_DEFS_LINUX_MAINLINE)
-static inline uint16_t
-get_ena_admin_aq_common_desc_command_id(
-		const struct ena_admin_aq_common_desc *p)
+static inline uint16_t get_ena_admin_aq_common_desc_command_id(const struct ena_admin_aq_common_desc *p)
 {
 	return p->command_id & ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
 }
 
-static inline void
-set_ena_admin_aq_common_desc_command_id(struct ena_admin_aq_common_desc *p,
-					uint16_t val)
+static inline void set_ena_admin_aq_common_desc_command_id(struct ena_admin_aq_common_desc *p, uint16_t val)
 {
 	p->command_id |= val & ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_aq_common_desc_phase(const struct ena_admin_aq_common_desc *p)
+static inline uint8_t get_ena_admin_aq_common_desc_phase(const struct ena_admin_aq_common_desc *p)
 {
 	return p->flags & ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;
 }
 
-static inline void
-set_ena_admin_aq_common_desc_phase(struct ena_admin_aq_common_desc *p,
-				   uint8_t val)
+static inline void set_ena_admin_aq_common_desc_phase(struct ena_admin_aq_common_desc *p, uint8_t val)
 {
 	p->flags |= val & ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_aq_common_desc_ctrl_data(
-		const struct ena_admin_aq_common_desc *p)
+static inline uint8_t get_ena_admin_aq_common_desc_ctrl_data(const struct ena_admin_aq_common_desc *p)
 {
-	return (p->flags & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK) >>
-	       ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT;
+	return (p->flags & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK) >> ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT;
 }
 
-static inline void
-set_ena_admin_aq_common_desc_ctrl_data(struct ena_admin_aq_common_desc *p,
-				       uint8_t val)
+static inline void set_ena_admin_aq_common_desc_ctrl_data(struct ena_admin_aq_common_desc *p, uint8_t val)
 {
-	p->flags |= (val << ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT)
-		     & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK;
+	p->flags |= (val << ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT) & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_aq_common_desc_ctrl_data_indirect(
-		const struct ena_admin_aq_common_desc *p)
+static inline uint8_t get_ena_admin_aq_common_desc_ctrl_data_indirect(const struct ena_admin_aq_common_desc *p)
 {
-	return (p->flags & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK)
-		>> ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT;
+	return (p->flags & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK) >> ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT;
 }
 
-static inline void
-set_ena_admin_aq_common_desc_ctrl_data_indirect(
-		struct ena_admin_aq_common_desc *p,
-		uint8_t val)
+static inline void set_ena_admin_aq_common_desc_ctrl_data_indirect(struct ena_admin_aq_common_desc *p, uint8_t val)
 {
-	p->flags |= (val << ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT)
-		     & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
+	p->flags |= (val << ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT) & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_sq_sq_direction(const struct ena_admin_sq *p)
+static inline uint8_t get_ena_admin_sq_sq_direction(const struct ena_admin_sq *p)
 {
-	return (p->sq_identity & ENA_ADMIN_SQ_SQ_DIRECTION_MASK)
-		>> ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT;
+	return (p->sq_identity & ENA_ADMIN_SQ_SQ_DIRECTION_MASK) >> ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT;
 }
 
-static inline void
-set_ena_admin_sq_sq_direction(struct ena_admin_sq *p, uint8_t val)
+static inline void set_ena_admin_sq_sq_direction(struct ena_admin_sq *p, uint8_t val)
 {
-	p->sq_identity |= (val << ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) &
-			  ENA_ADMIN_SQ_SQ_DIRECTION_MASK;
+	p->sq_identity |= (val << ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) & ENA_ADMIN_SQ_SQ_DIRECTION_MASK;
 }
 
-static inline uint16_t
-get_ena_admin_acq_common_desc_command_id(
-		const struct ena_admin_acq_common_desc *p)
+static inline uint16_t get_ena_admin_acq_common_desc_command_id(const struct ena_admin_acq_common_desc *p)
 {
 	return p->command & ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
 }
 
-static inline void
-set_ena_admin_acq_common_desc_command_id(struct ena_admin_acq_common_desc *p,
-					 uint16_t val)
+static inline void set_ena_admin_acq_common_desc_command_id(struct ena_admin_acq_common_desc *p, uint16_t val)
 {
 	p->command |= val & ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_acq_common_desc_phase(const struct ena_admin_acq_common_desc *p)
+static inline uint8_t get_ena_admin_acq_common_desc_phase(const struct ena_admin_acq_common_desc *p)
 {
 	return p->flags & ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK;
 }
 
-static inline void
-set_ena_admin_acq_common_desc_phase(struct ena_admin_acq_common_desc *p,
-				    uint8_t val)
+static inline void set_ena_admin_acq_common_desc_phase(struct ena_admin_acq_common_desc *p, uint8_t val)
 {
 	p->flags |= val & ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_aq_create_sq_cmd_sq_direction(
-		const struct ena_admin_aq_create_sq_cmd *p)
+static inline uint8_t get_ena_admin_aq_create_sq_cmd_sq_direction(const struct ena_admin_aq_create_sq_cmd *p)
 {
-	return (p->sq_identity & ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK)
-		>> ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT;
+	return (p->sq_identity & ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK) >> ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT;
 }
 
-static inline void
-set_ena_admin_aq_create_sq_cmd_sq_direction(
-		struct ena_admin_aq_create_sq_cmd *p,
-		uint8_t val)
+static inline void set_ena_admin_aq_create_sq_cmd_sq_direction(struct ena_admin_aq_create_sq_cmd *p, uint8_t val)
 {
-	p->sq_identity |= (val <<
-			   ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT)
-			  & ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK;
+	p->sq_identity |= (val << ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) & ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_aq_create_sq_cmd_placement_policy(
-		const struct ena_admin_aq_create_sq_cmd *p)
+static inline uint8_t get_ena_admin_aq_create_sq_cmd_placement_policy(const struct ena_admin_aq_create_sq_cmd *p)
 {
 	return p->sq_caps_2 & ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK;
 }
 
-static inline void
-set_ena_admin_aq_create_sq_cmd_placement_policy(
-		struct ena_admin_aq_create_sq_cmd *p,
-		uint8_t val)
+static inline void set_ena_admin_aq_create_sq_cmd_placement_policy(struct ena_admin_aq_create_sq_cmd *p, uint8_t val)
 {
 	p->sq_caps_2 |= val & ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_aq_create_sq_cmd_completion_policy(
-		const struct ena_admin_aq_create_sq_cmd *p)
+static inline uint8_t get_ena_admin_aq_create_sq_cmd_completion_policy(const struct ena_admin_aq_create_sq_cmd *p)
 {
-	return (p->sq_caps_2
-		& ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK)
-		>> ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT;
+	return (p->sq_caps_2 & ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK) >> ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT;
 }
 
-static inline void
-set_ena_admin_aq_create_sq_cmd_completion_policy(
-		struct ena_admin_aq_create_sq_cmd *p,
-		uint8_t val)
+static inline void set_ena_admin_aq_create_sq_cmd_completion_policy(struct ena_admin_aq_create_sq_cmd *p, uint8_t val)
 {
-	p->sq_caps_2 |=
-		(val << ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT)
-		& ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK;
+	p->sq_caps_2 |= (val << ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) & ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_aq_create_sq_cmd_is_physically_contiguous(
-		const struct ena_admin_aq_create_sq_cmd *p)
+static inline uint8_t get_ena_admin_aq_create_sq_cmd_is_physically_contiguous(const struct ena_admin_aq_create_sq_cmd *p)
 {
-	return p->sq_caps_3 &
-		ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK;
+	return p->sq_caps_3 & ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK;
 }
 
-static inline void
-set_ena_admin_aq_create_sq_cmd_is_physically_contiguous(
-		struct ena_admin_aq_create_sq_cmd *p,
-		uint8_t val)
+static inline void set_ena_admin_aq_create_sq_cmd_is_physically_contiguous(struct ena_admin_aq_create_sq_cmd *p, uint8_t val)
 {
-	p->sq_caps_3 |= val &
-		ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK;
+	p->sq_caps_3 |= val & ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_aq_create_cq_cmd_interrupt_mode_enabled(
-		const struct ena_admin_aq_create_cq_cmd *p)
+static inline uint8_t get_ena_admin_aq_create_cq_cmd_interrupt_mode_enabled(const struct ena_admin_aq_create_cq_cmd *p)
 {
-	return (p->cq_caps_1 &
-		ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK)
-		>> ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT;
+	return (p->cq_caps_1 & ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK) >> ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT;
 }
 
-static inline void
-set_ena_admin_aq_create_cq_cmd_interrupt_mode_enabled(
-		struct ena_admin_aq_create_cq_cmd *p,
-		uint8_t val)
+static inline void set_ena_admin_aq_create_cq_cmd_interrupt_mode_enabled(struct ena_admin_aq_create_cq_cmd *p, uint8_t val)
 {
-	p->cq_caps_1 |=
-	(val <<	ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT)
-	& ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK;
+	p->cq_caps_1 |= (val << ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT) & ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_aq_create_cq_cmd_cq_entry_size_words(
-		const struct ena_admin_aq_create_cq_cmd *p)
+static inline uint8_t get_ena_admin_aq_create_cq_cmd_cq_entry_size_words(const struct ena_admin_aq_create_cq_cmd *p)
 {
-	return p->cq_caps_2
-	       & ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
+	return p->cq_caps_2 & ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
 }
 
-static inline void
-set_ena_admin_aq_create_cq_cmd_cq_entry_size_words(
-		struct ena_admin_aq_create_cq_cmd *p,
-		uint8_t val)
+static inline void set_ena_admin_aq_create_cq_cmd_cq_entry_size_words(struct ena_admin_aq_create_cq_cmd *p, uint8_t val)
 {
-	p->cq_caps_2 |=
-		val & ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
+	p->cq_caps_2 |= val & ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_get_set_feature_common_desc_select(
-		const struct ena_admin_get_set_feature_common_desc *p)
+static inline uint8_t get_ena_admin_get_set_feature_common_desc_select(const struct ena_admin_get_set_feature_common_desc *p)
 {
 	return p->flags & ENA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK;
 }
 
-static inline void
-set_ena_admin_get_set_feature_common_desc_select(
-		struct ena_admin_get_set_feature_common_desc *p,
-		uint8_t val)
+static inline void set_ena_admin_get_set_feature_common_desc_select(struct ena_admin_get_set_feature_common_desc *p, uint8_t val)
 {
 	p->flags |= val & ENA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_get_feature_link_desc_autoneg(
-		const struct ena_admin_get_feature_link_desc *p)
+static inline uint32_t get_ena_admin_get_feature_link_desc_autoneg(const struct ena_admin_get_feature_link_desc *p)
 {
 	return p->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK;
 }
 
-static inline void
-set_ena_admin_get_feature_link_desc_autoneg(
-		struct ena_admin_get_feature_link_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_get_feature_link_desc_autoneg(struct ena_admin_get_feature_link_desc *p, uint32_t val)
 {
 	p->flags |= val & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_get_feature_link_desc_duplex(
-		const struct ena_admin_get_feature_link_desc *p)
+static inline uint32_t get_ena_admin_get_feature_link_desc_duplex(const struct ena_admin_get_feature_link_desc *p)
 {
-	return (p->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK)
-		>> ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT;
+	return (p->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK) >> ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT;
 }
 
-static inline void
-set_ena_admin_get_feature_link_desc_duplex(
-		struct ena_admin_get_feature_link_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_get_feature_link_desc_duplex(struct ena_admin_get_feature_link_desc *p, uint32_t val)
 {
-	p->flags |= (val << ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT)
-			& ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK;
+	p->flags |= (val << ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT) & ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_TX_L3_csum_ipv4(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_TX_L3_csum_ipv4(const struct ena_admin_feature_offload_desc *p)
 {
 	return p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_TX_L3_csum_ipv4(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_TX_L3_csum_ipv4(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
 	p->tx |= val & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_part(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_part(const struct ena_admin_feature_offload_desc *p)
 {
-	return (p->tx &
-		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)
-		>> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT;
+	return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_part(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_part(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
-	p->tx |= (val <<
-		  ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT)
-		 & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK;
+	p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_full(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_full(const struct ena_admin_feature_offload_desc *p)
 {
-	return (p->tx &
-		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK)
-		>> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT;
+	return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_full(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_full(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
-	p->tx |= (val <<
-		  ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT)
-		 & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK;
+	p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_part(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_part(const struct ena_admin_feature_offload_desc *p)
 {
-	return (p->tx &
-		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)
-		>> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT;
+	return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_part(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_part(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
-	p->tx |= (val <<
-		  ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT)
-		 & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK;
+	p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_full(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_full(const struct ena_admin_feature_offload_desc *p)
 {
-	return (p->tx &
-		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK)
-		>> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT;
+	return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_full(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_full(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
-	p->tx |= (val <<
-		  ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT)
-		 & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK;
+	p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_tso_ipv4(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_tso_ipv4(const struct ena_admin_feature_offload_desc *p)
 {
-	return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
-		>> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT;
+	return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_tso_ipv4(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_tso_ipv4(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
-	p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT)
-		 & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK;
+	p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_tso_ipv6(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_tso_ipv6(const struct ena_admin_feature_offload_desc *p)
 {
-	return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK)
-		>> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT;
+	return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_tso_ipv6(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_tso_ipv6(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
-	p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT)
-		 & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK;
+	p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_tso_ecn(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_tso_ecn(const struct ena_admin_feature_offload_desc *p)
 {
-	return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK)
-		>> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT;
+	return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_tso_ecn(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_tso_ecn(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
-	p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT)
-		 & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK;
+	p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_RX_L3_csum_ipv4(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_RX_L3_csum_ipv4(const struct ena_admin_feature_offload_desc *p)
 {
-	return p->rx_supported &
-		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK;
+	return p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_RX_L3_csum_ipv4(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_RX_L3_csum_ipv4(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
-	p->rx_supported |=
-		val & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK;
+	p->rx_supported |= val & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_RX_L4_ipv4_csum(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_RX_L4_ipv4_csum(const struct ena_admin_feature_offload_desc *p)
 {
-	return (p->rx_supported &
-		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)
-		>> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT;
+	return (p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_RX_L4_ipv4_csum(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_RX_L4_ipv4_csum(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
-	p->rx_supported |=
-		(val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT)
-		& ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK;
+	p->rx_supported |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_RX_L4_ipv6_csum(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_RX_L4_ipv6_csum(const struct ena_admin_feature_offload_desc *p)
 {
-	return (p->rx_supported &
-		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK)
-		>> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT;
+	return (p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_RX_L4_ipv6_csum(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_RX_L4_ipv6_csum(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
-	p->rx_supported |=
-		(val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT)
-		& ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK;
+	p->rx_supported |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_offload_desc_RX_hash(
-		const struct ena_admin_feature_offload_desc *p)
+static inline uint32_t get_ena_admin_feature_offload_desc_RX_hash(const struct ena_admin_feature_offload_desc *p)
 {
-	return (p->rx_supported &
-		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK)
-		>> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT;
+	return (p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_offload_desc_RX_hash(
-		struct ena_admin_feature_offload_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_offload_desc_RX_hash(struct ena_admin_feature_offload_desc *p, uint32_t val)
 {
-	p->rx_supported |=
-		(val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT)
-		& ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK;
+	p->rx_supported |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_rss_flow_hash_function_funcs(
-		const struct ena_admin_feature_rss_flow_hash_function *p)
+static inline uint32_t get_ena_admin_feature_rss_flow_hash_function_funcs(const struct ena_admin_feature_rss_flow_hash_function *p)
 {
-	return p->supported_func &
-		ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK;
+	return p->supported_func & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK;
 }
 
-static inline void
-set_ena_admin_feature_rss_flow_hash_function_funcs(
-		struct ena_admin_feature_rss_flow_hash_function *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_rss_flow_hash_function_funcs(struct ena_admin_feature_rss_flow_hash_function *p, uint32_t val)
 {
-	p->supported_func |=
-		val & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK;
+	p->supported_func |= val & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_feature_rss_flow_hash_function_selected_func(
-		const struct ena_admin_feature_rss_flow_hash_function *p)
+static inline uint32_t get_ena_admin_feature_rss_flow_hash_function_selected_func(const struct ena_admin_feature_rss_flow_hash_function *p)
 {
-	return p->selected_func &
-		ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK;
+	return p->selected_func & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK;
 }
 
-static inline void
-set_ena_admin_feature_rss_flow_hash_function_selected_func(
-		struct ena_admin_feature_rss_flow_hash_function *p,
-		uint32_t val)
+static inline void set_ena_admin_feature_rss_flow_hash_function_selected_func(struct ena_admin_feature_rss_flow_hash_function *p, uint32_t val)
 {
-	p->selected_func |=
-		val &
-		ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK;
+	p->selected_func |= val & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK;
 }
 
-static inline uint16_t
-get_ena_admin_feature_rss_flow_hash_input_L3_sort(
-		const struct ena_admin_feature_rss_flow_hash_input *p)
+static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_L3_sort(const struct ena_admin_feature_rss_flow_hash_input *p)
 {
-	return (p->supported_input_sort &
-			ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK)
-		>> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT;
+	return (p->supported_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_rss_flow_hash_input_L3_sort(
-		struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val)
+static inline void set_ena_admin_feature_rss_flow_hash_input_L3_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val)
 {
-	p->supported_input_sort |=
-		(val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT)
-		& ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK;
+	p->supported_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK;
 }
 
-static inline uint16_t
-get_ena_admin_feature_rss_flow_hash_input_L4_sort(
-		const struct ena_admin_feature_rss_flow_hash_input *p)
+static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_L4_sort(const struct ena_admin_feature_rss_flow_hash_input *p)
 {
-	return (p->supported_input_sort &
-		ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK)
-		>> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT;
+	return (p->supported_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_rss_flow_hash_input_L4_sort(
-		struct ena_admin_feature_rss_flow_hash_input *p,
-		uint16_t val)
+static inline void set_ena_admin_feature_rss_flow_hash_input_L4_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val)
 {
-	p->supported_input_sort |=
-		(val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT)
-		 & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK;
+	p->supported_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK;
 }
 
-static inline uint16_t
-get_ena_admin_feature_rss_flow_hash_input_enable_L3_sort(
-		const struct ena_admin_feature_rss_flow_hash_input *p)
+static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_enable_L3_sort(const struct ena_admin_feature_rss_flow_hash_input *p)
 {
-	return (p->enabled_input_sort &
-		ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK)
-		>> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT;
+	return (p->enabled_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_rss_flow_hash_input_enable_L3_sort(
-		struct ena_admin_feature_rss_flow_hash_input *p,
-		uint16_t val)
+static inline void set_ena_admin_feature_rss_flow_hash_input_enable_L3_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val)
 {
-	p->enabled_input_sort |=
-		(val <<
-		 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT)
-		& ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK;
+	p->enabled_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK;
 }
 
-static inline uint16_t
-get_ena_admin_feature_rss_flow_hash_input_enable_L4_sort(
-		const struct ena_admin_feature_rss_flow_hash_input *p)
+static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_enable_L4_sort(const struct ena_admin_feature_rss_flow_hash_input *p)
 {
-	return (p->enabled_input_sort &
-		ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK)
-		>> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT;
+	return (p->enabled_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT;
 }
 
-static inline void
-set_ena_admin_feature_rss_flow_hash_input_enable_L4_sort(
-		struct ena_admin_feature_rss_flow_hash_input *p,
-		uint16_t val)
+static inline void set_ena_admin_feature_rss_flow_hash_input_enable_L4_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val)
 {
-	p->enabled_input_sort |=
-		(val <<
-		 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT)
-		& ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK;
+	p->enabled_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_host_info_major(const struct ena_admin_host_info *p)
+static inline uint32_t get_ena_admin_host_info_major(const struct ena_admin_host_info *p)
 {
 	return p->driver_version & ENA_ADMIN_HOST_INFO_MAJOR_MASK;
 }
 
-static inline void
-set_ena_admin_host_info_major(struct ena_admin_host_info *p, uint32_t val)
+static inline void set_ena_admin_host_info_major(struct ena_admin_host_info *p, uint32_t val)
 {
 	p->driver_version |= val & ENA_ADMIN_HOST_INFO_MAJOR_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_host_info_minor(const struct ena_admin_host_info *p)
+static inline uint32_t get_ena_admin_host_info_minor(const struct ena_admin_host_info *p)
 {
-	return (p->driver_version & ENA_ADMIN_HOST_INFO_MINOR_MASK)
-		>> ENA_ADMIN_HOST_INFO_MINOR_SHIFT;
+	return (p->driver_version & ENA_ADMIN_HOST_INFO_MINOR_MASK) >> ENA_ADMIN_HOST_INFO_MINOR_SHIFT;
 }
 
-static inline void
-set_ena_admin_host_info_minor(struct ena_admin_host_info *p, uint32_t val)
+static inline void set_ena_admin_host_info_minor(struct ena_admin_host_info *p, uint32_t val)
 {
-	p->driver_version |= (val << ENA_ADMIN_HOST_INFO_MINOR_SHIFT)
-		& ENA_ADMIN_HOST_INFO_MINOR_MASK;
+	p->driver_version |= (val << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) & ENA_ADMIN_HOST_INFO_MINOR_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_host_info_sub_minor(const struct ena_admin_host_info *p)
+static inline uint32_t get_ena_admin_host_info_sub_minor(const struct ena_admin_host_info *p)
 {
-	return (p->driver_version & ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK)
-		>> ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT;
+	return (p->driver_version & ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK) >> ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT;
 }
 
-static inline void
-set_ena_admin_host_info_sub_minor(struct ena_admin_host_info *p, uint32_t val)
+static inline void set_ena_admin_host_info_sub_minor(struct ena_admin_host_info *p, uint32_t val)
 {
-	p->driver_version |= (val << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT)
-			     & ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK;
+	p->driver_version |= (val << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) & ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK;
 }
 
-static inline uint8_t
-get_ena_admin_aenq_common_desc_phase(
-		const struct ena_admin_aenq_common_desc *p)
+static inline uint8_t get_ena_admin_aenq_common_desc_phase(const struct ena_admin_aenq_common_desc *p)
 {
 	return p->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK;
 }
 
-static inline void
-set_ena_admin_aenq_common_desc_phase(
-		struct ena_admin_aenq_common_desc *p,
-		uint8_t val)
+static inline void set_ena_admin_aenq_common_desc_phase(struct ena_admin_aenq_common_desc *p, uint8_t val)
 {
 	p->flags |= val & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK;
 }
 
-static inline uint32_t
-get_ena_admin_aenq_link_change_desc_link_status(
-		const struct ena_admin_aenq_link_change_desc *p)
+static inline uint32_t get_ena_admin_aenq_link_change_desc_link_status(const struct ena_admin_aenq_link_change_desc *p)
 {
 	return p->flags & ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
 }
 
-static inline void
-set_ena_admin_aenq_link_change_desc_link_status(
-		struct ena_admin_aenq_link_change_desc *p,
-		uint32_t val)
+static inline void set_ena_admin_aenq_link_change_desc_link_status(struct ena_admin_aenq_link_change_desc *p, uint32_t val)
 {
 	p->flags |= val & ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
 }
diff --git a/drivers/net/ena/base/ena_defs/ena_common_defs.h b/drivers/net/ena/base/ena_defs/ena_common_defs.h
index 95e0f389..072e6c1f 100644
--- a/drivers/net/ena/base/ena_defs/ena_common_defs.h
+++ b/drivers/net/ena/base/ena_defs/ena_common_defs.h
@@ -34,17 +34,13 @@
 #ifndef _ENA_COMMON_H_
 #define _ENA_COMMON_H_
 
-/* spec version */
-#define ENA_COMMON_SPEC_VERSION_MAJOR	0 /* spec version major */
-#define ENA_COMMON_SPEC_VERSION_MINOR	10 /* spec version minor */
+#define ENA_COMMON_SPEC_VERSION_MAJOR	0 /*  */
+#define ENA_COMMON_SPEC_VERSION_MINOR	10 /*  */
 
 /* ENA operates with 48-bit memory addresses. ena_mem_addr_t */
 struct ena_common_mem_addr {
-	/* word 0 : low 32 bit of the memory address */
 	uint32_t mem_addr_low;
 
-	/* word 1 : */
-	/* high 16 bits of the memory address */
 	uint16_t mem_addr_high;
 
 	/* MBZ */
diff --git a/drivers/net/ena/base/ena_defs/ena_eth_io_defs.h b/drivers/net/ena/base/ena_defs/ena_eth_io_defs.h
index 6bc3d6a7..4cf0b205 100644
--- a/drivers/net/ena/base/ena_defs/ena_eth_io_defs.h
+++ b/drivers/net/ena/base/ena_defs/ena_eth_io_defs.h
@@ -34,35 +34,30 @@
 #ifndef _ENA_ETH_IO_H_
 #define _ENA_ETH_IO_H_
 
-/* Layer 3 protocol index */
 enum ena_eth_io_l3_proto_index {
-	ENA_ETH_IO_L3_PROTO_UNKNOWN = 0,
+	ENA_ETH_IO_L3_PROTO_UNKNOWN	= 0,
 
-	ENA_ETH_IO_L3_PROTO_IPV4 = 8,
+	ENA_ETH_IO_L3_PROTO_IPV4	= 8,
 
-	ENA_ETH_IO_L3_PROTO_IPV6 = 11,
+	ENA_ETH_IO_L3_PROTO_IPV6	= 11,
 
-	ENA_ETH_IO_L3_PROTO_FCOE = 21,
+	ENA_ETH_IO_L3_PROTO_FCOE	= 21,
 
-	ENA_ETH_IO_L3_PROTO_ROCE = 22,
+	ENA_ETH_IO_L3_PROTO_ROCE	= 22,
 };
 
-/* Layer 4 protocol index */
 enum ena_eth_io_l4_proto_index {
-	ENA_ETH_IO_L4_PROTO_UNKNOWN = 0,
+	ENA_ETH_IO_L4_PROTO_UNKNOWN		= 0,
 
-	ENA_ETH_IO_L4_PROTO_TCP = 12,
+	ENA_ETH_IO_L4_PROTO_TCP			= 12,
 
-	ENA_ETH_IO_L4_PROTO_UDP = 13,
+	ENA_ETH_IO_L4_PROTO_UDP			= 13,
 
-	ENA_ETH_IO_L4_PROTO_ROUTEABLE_ROCE = 23,
+	ENA_ETH_IO_L4_PROTO_ROUTEABLE_ROCE	= 23,
 };
 
-/* ENA IO Queue Tx descriptor */
 struct ena_eth_io_tx_desc {
-	/* word 0 : */
-	/* length, request id and control flags
-	 * 15:0 : length - Buffer length in bytes, must
+	/* 15:0 : length - Buffer length in bytes, must
 	 *    include any packet trailers that the ENA supposed
 	 *    to update like End-to-End CRC, Authentication GMAC
 	 *    etc. This length must not include the
@@ -85,9 +80,7 @@ struct ena_eth_io_tx_desc {
 	 */
 	uint32_t len_ctrl;
 
-	/* word 1 : */
-	/* ethernet control
-	 * 3:0 : l3_proto_idx - L3 protocol. This field
+	/* 3:0 : l3_proto_idx - L3 protocol. This field
 	 *    required when l3_csum_en,l3_csum or tso_en are set.
 	 * 4 : DF - IPv4 DF, must be 0 if packet is IPv4 and
 	 *    DF flags of the IPv4 header is 0. Otherwise must
@@ -119,10 +112,8 @@ struct ena_eth_io_tx_desc {
 	 */
 	uint32_t meta_ctrl;
 
-	/* word 2 : Buffer address bits[31:0] */
 	uint32_t buff_addr_lo;
 
-	/* word 3 : */
 	/* address high and header size
 	 * 15:0 : addr_hi - Buffer Pointer[47:32]
 	 * 23:16 : reserved16_w2
@@ -141,20 +132,16 @@ struct ena_eth_io_tx_desc {
 	uint32_t buff_addr_hi_hdr_sz;
 };
 
-/* ENA IO Queue Tx Meta descriptor */
 struct ena_eth_io_tx_meta_desc {
-	/* word 0 : */
-	/* length, request id and control flags
-	 * 9:0 : req_id_lo - Request ID[9:0]
+	/* 9:0 : req_id_lo - Request ID[9:0]
 	 * 11:10 : reserved10 - MBZ
 	 * 12 : reserved12 - MBZ
 	 * 13 : reserved13 - MBZ
 	 * 14 : ext_valid - if set, offset fields in Word2
-	 *    are valid Also MSS High in Word 0 and Outer L3
-	 *    Offset High in WORD 0 and bits [31:24] in Word 3
-	 * 15 : word3_valid - If set Crypto Info[23:0] of
-	 *    Word 3 is valid
-	 * 19:16 : mss_hi_ptp
+	 *    are valid Also MSS High in Word 0 and bits [31:24]
+	 *    in Word 3
+	 * 15 : reserved15
+	 * 19:16 : mss_hi
 	 * 20 : eth_meta_type - 0: Tx Metadata Descriptor, 1:
 	 *    Extended Metadata Descriptor
 	 * 21 : meta_store - Store extended metadata in queue
@@ -175,19 +162,13 @@ struct ena_eth_io_tx_meta_desc {
 	 */
 	uint32_t len_ctrl;
 
-	/* word 1 : */
-	/* word 1
-	 * 5:0 : req_id_hi
+	/* 5:0 : req_id_hi
 	 * 31:6 : reserved6 - MBZ
 	 */
 	uint32_t word1;
 
-	/* word 2 : */
-	/* word 2
-	 * 7:0 : l3_hdr_len - the header length L3 IP header.
-	 * 15:8 : l3_hdr_off - the offset of the first byte
-	 *    in the L3 header from the beginning of the to-be
-	 *    transmitted packet.
+	/* 7:0 : l3_hdr_len
+	 * 15:8 : l3_hdr_off
 	 * 21:16 : l4_hdr_len_in_words - counts the L4 header
 	 *    length in words. there is an explicit assumption
 	 *    that L4 header appears right after L3 header and
@@ -196,13 +177,10 @@ struct ena_eth_io_tx_meta_desc {
 	 */
 	uint32_t word2;
 
-	/* word 3 : */
 	uint32_t reserved;
 };
 
-/* ENA IO Queue Tx completions descriptor */
 struct ena_eth_io_tx_cdesc {
-	/* word 0 : */
 	/* Request ID[15:0] */
 	uint16_t req_id;
 
@@ -214,24 +192,19 @@ struct ena_eth_io_tx_cdesc {
 	 */
 	uint8_t flags;
 
-	/* word 1 : */
 	uint16_t sub_qid;
 
-	/* indicates location of submission queue head */
 	uint16_t sq_head_idx;
 };
 
-/* ENA IO Queue Rx descriptor */
 struct ena_eth_io_rx_desc {
-	/* word 0 : */
 	/* In bytes. 0 means 64KB */
 	uint16_t length;
 
 	/* MBZ */
 	uint8_t reserved2;
 
-	/* control flags
-	 * 0 : phase
+	/* 0 : phase
 	 * 1 : reserved1 - MBZ
 	 * 2 : first - Indicates first descriptor in
 	 *    transaction
@@ -242,32 +215,27 @@ struct ena_eth_io_rx_desc {
 	 */
 	uint8_t ctrl;
 
-	/* word 1 : */
 	uint16_t req_id;
 
 	/* MBZ */
 	uint16_t reserved6;
 
-	/* word 2 : Buffer address bits[31:0] */
 	uint32_t buff_addr_lo;
 
-	/* word 3 : */
-	/* Buffer Address bits[47:16] */
 	uint16_t buff_addr_hi;
 
 	/* MBZ */
 	uint16_t reserved16_w3;
 };
 
-/* ENA IO Queue Rx Completion Base Descriptor (4-word format). Note: all
- * ethernet parsing information are valid only when last=1
+/* 4-word format Note: all ethernet parsing information are valid only when
+ * last=1
  */
 struct ena_eth_io_rx_cdesc_base {
-	/* word 0 : */
-	/* 4:0 : l3_proto_idx - L3 protocol index
-	 * 6:5 : src_vlan_cnt - Source VLAN count
+	/* 4:0 : l3_proto_idx
+	 * 6:5 : src_vlan_cnt
 	 * 7 : reserved7 - MBZ
-	 * 12:8 : l4_proto_idx - L4 protocol index
+	 * 12:8 : l4_proto_idx
 	 * 13 : l3_csum_err - when set, either the L3
 	 *    checksum error detected, or, the controller didn't
 	 *    validate the checksum. This bit is valid only when
@@ -292,56 +260,43 @@ struct ena_eth_io_rx_cdesc_base {
 	 */
 	uint32_t status;
 
-	/* word 1 : */
 	uint16_t length;
 
 	uint16_t req_id;
 
-	/* word 2 : 32-bit hash result */
+	/* 32-bit hash result */
 	uint32_t hash;
 
-	/* word 3 : */
-	/* submission queue number */
 	uint16_t sub_qid;
 
 	uint16_t reserved;
 };
 
-/* ENA IO Queue Rx Completion Descriptor (8-word format) */
+/* 8-word format */
 struct ena_eth_io_rx_cdesc_ext {
-	/* words 0:3 : Rx Completion Extended */
 	struct ena_eth_io_rx_cdesc_base base;
 
-	/* word 4 : Completed Buffer address bits[31:0] */
 	uint32_t buff_addr_lo;
 
-	/* word 5 : */
-	/* the buffer address used bits[47:32] */
 	uint16_t buff_addr_hi;
 
 	uint16_t reserved16;
 
-	/* word 6 : Reserved */
 	uint32_t reserved_w6;
 
-	/* word 7 : Reserved */
 	uint32_t reserved_w7;
 };
 
-/* ENA Interrupt Unmask Register */
 struct ena_eth_io_intr_reg {
-	/* word 0 : */
-	/* 14:0 : rx_intr_delay - rx interrupt delay value
-	 * 29:15 : tx_intr_delay - tx interrupt delay value
-	 * 30 : intr_unmask - if set, unmasks interrupt
+	/* 14:0 : rx_intr_delay
+	 * 29:15 : tx_intr_delay
+	 * 30 : intr_unmask
 	 * 31 : reserved
 	 */
 	uint32_t intr_control;
 };
 
-/* ENA NUMA Node configuration register */
 struct ena_eth_io_numa_node_cfg_reg {
-	/* word 0 : */
 	/* 7:0 : numa
 	 * 30:8 : reserved
 	 * 31 : enabled
@@ -388,10 +343,8 @@ struct ena_eth_io_numa_node_cfg_reg {
 #define ENA_ETH_IO_TX_META_DESC_REQ_ID_LO_MASK GENMASK(9, 0)
 #define ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT 14
 #define ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK BIT(14)
-#define ENA_ETH_IO_TX_META_DESC_WORD3_VALID_SHIFT 15
-#define ENA_ETH_IO_TX_META_DESC_WORD3_VALID_MASK BIT(15)
-#define ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_SHIFT 16
-#define ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_MASK GENMASK(19, 16)
+#define ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT 16
+#define ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK GENMASK(19, 16)
 #define ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT 20
 #define ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK BIT(20)
 #define ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT 21
@@ -463,803 +416,544 @@ struct ena_eth_io_numa_node_cfg_reg {
 #define ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK BIT(31)
 
 #if !defined(ENA_DEFS_LINUX_MAINLINE)
-static inline uint32_t get_ena_eth_io_tx_desc_length(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_length(const struct ena_eth_io_tx_desc *p)
 {
 	return p->len_ctrl & ENA_ETH_IO_TX_DESC_LENGTH_MASK;
 }
 
-static inline void set_ena_eth_io_tx_desc_length(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_length(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
 	p->len_ctrl |= val & ENA_ETH_IO_TX_DESC_LENGTH_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_req_id_hi(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_req_id_hi(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK)
-		>> ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK) >> ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_req_id_hi(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_req_id_hi(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->len_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT)
-		& ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT) & ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_meta_desc(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_meta_desc(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_META_DESC_MASK)
-		>> ENA_ETH_IO_TX_DESC_META_DESC_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_META_DESC_MASK) >> ENA_ETH_IO_TX_DESC_META_DESC_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_meta_desc(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_meta_desc(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->len_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_META_DESC_SHIFT)
-		& ENA_ETH_IO_TX_DESC_META_DESC_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_META_DESC_SHIFT) & ENA_ETH_IO_TX_DESC_META_DESC_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_phase(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_phase(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_PHASE_MASK)
-		>> ENA_ETH_IO_TX_DESC_PHASE_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_PHASE_MASK) >> ENA_ETH_IO_TX_DESC_PHASE_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_phase(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_phase(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->len_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_PHASE_SHIFT)
-		& ENA_ETH_IO_TX_DESC_PHASE_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_PHASE_SHIFT) & ENA_ETH_IO_TX_DESC_PHASE_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_first(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_first(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_FIRST_MASK)
-		>> ENA_ETH_IO_TX_DESC_FIRST_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_FIRST_MASK) >> ENA_ETH_IO_TX_DESC_FIRST_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_first(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_first(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->len_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_FIRST_SHIFT)
-		& ENA_ETH_IO_TX_DESC_FIRST_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_FIRST_SHIFT) & ENA_ETH_IO_TX_DESC_FIRST_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_last(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_last(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_LAST_MASK)
-		>> ENA_ETH_IO_TX_DESC_LAST_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_LAST_MASK) >> ENA_ETH_IO_TX_DESC_LAST_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_last(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_last(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->len_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_LAST_SHIFT)
-		& ENA_ETH_IO_TX_DESC_LAST_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_LAST_SHIFT) & ENA_ETH_IO_TX_DESC_LAST_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_comp_req(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_comp_req(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_COMP_REQ_MASK)
-		>> ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_DESC_COMP_REQ_MASK) >> ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_comp_req(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_comp_req(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->len_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT)
-		& ENA_ETH_IO_TX_DESC_COMP_REQ_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT) & ENA_ETH_IO_TX_DESC_COMP_REQ_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_l3_proto_idx(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_l3_proto_idx(const struct ena_eth_io_tx_desc *p)
 {
 	return p->meta_ctrl & ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK;
 }
 
-static inline void set_ena_eth_io_tx_desc_l3_proto_idx(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_l3_proto_idx(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
 	p->meta_ctrl |= val & ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_DF(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_DF(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_DF_MASK)
-		>> ENA_ETH_IO_TX_DESC_DF_SHIFT;
+	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_DF_MASK) >> ENA_ETH_IO_TX_DESC_DF_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_DF(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_DF(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->meta_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_DF_SHIFT)
-		& ENA_ETH_IO_TX_DESC_DF_MASK;
+	p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_DF_SHIFT) & ENA_ETH_IO_TX_DESC_DF_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_tso_en(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_tso_en(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_TSO_EN_MASK)
-		>> ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT;
+	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_TSO_EN_MASK) >> ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_tso_en(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_tso_en(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->meta_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT)
-		& ENA_ETH_IO_TX_DESC_TSO_EN_MASK;
+	p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT) & ENA_ETH_IO_TX_DESC_TSO_EN_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_l4_proto_idx(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_l4_proto_idx(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK)
-		>> ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT;
+	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK) >> ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_l4_proto_idx(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_l4_proto_idx(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->meta_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT)
-		& ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK;
+	p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT) & ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_l3_csum_en(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_l3_csum_en(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK)
-		>> ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT;
+	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK) >> ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_l3_csum_en(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_l3_csum_en(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->meta_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT)
-		& ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK;
+	p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT) & ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_l4_csum_en(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_l4_csum_en(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK)
-		>> ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT;
+	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK) >> ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_l4_csum_en(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_l4_csum_en(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->meta_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT)
-		& ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK;
+	p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT) & ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_ethernet_fcs_dis(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_ethernet_fcs_dis(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK)
-		>> ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT;
+	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK) >> ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_ethernet_fcs_dis(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_ethernet_fcs_dis(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->meta_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT)
-		& ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK;
+	p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT) & ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_l4_csum_partial(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_l4_csum_partial(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK)
-		>> ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT;
+	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK) >> ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_l4_csum_partial(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_l4_csum_partial(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->meta_ctrl |=
-		(val << ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT)
-		& ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK;
+	p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT) & ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_req_id_lo(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_req_id_lo(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK)
-		>> ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT;
+	return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK) >> ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_req_id_lo(
-		struct ena_eth_io_tx_desc *p, uint32_t val)
+static inline void set_ena_eth_io_tx_desc_req_id_lo(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT)
-		& ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK;
+	p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT) & ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_addr_hi(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_addr_hi(const struct ena_eth_io_tx_desc *p)
 {
 	return p->buff_addr_hi_hdr_sz & ENA_ETH_IO_TX_DESC_ADDR_HI_MASK;
 }
 
-static inline void set_ena_eth_io_tx_desc_addr_hi(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_addr_hi(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
 	p->buff_addr_hi_hdr_sz |= val & ENA_ETH_IO_TX_DESC_ADDR_HI_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_desc_header_length(
-		const struct ena_eth_io_tx_desc *p)
+static inline uint32_t get_ena_eth_io_tx_desc_header_length(const struct ena_eth_io_tx_desc *p)
 {
-	return (p->buff_addr_hi_hdr_sz & ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK)
-		>> ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT;
+	return (p->buff_addr_hi_hdr_sz & ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK) >> ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_desc_header_length(
-		struct ena_eth_io_tx_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_desc_header_length(struct ena_eth_io_tx_desc *p, uint32_t val)
 {
-	p->buff_addr_hi_hdr_sz |=
-		(val << ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT)
-		& ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK;
+	p->buff_addr_hi_hdr_sz |= (val << ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT) & ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_req_id_lo(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_req_id_lo(const struct ena_eth_io_tx_meta_desc *p)
 {
 	return p->len_ctrl & ENA_ETH_IO_TX_META_DESC_REQ_ID_LO_MASK;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_req_id_lo(
-		struct ena_eth_io_tx_meta_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_req_id_lo(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
 	p->len_ctrl |= val & ENA_ETH_IO_TX_META_DESC_REQ_ID_LO_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_ext_valid(
-		const struct ena_eth_io_tx_meta_desc *p)
-{
-	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT;
-}
-
-static inline void set_ena_eth_io_tx_meta_desc_ext_valid(
-		struct ena_eth_io_tx_meta_desc *p, uint32_t val)
-{
-	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK;
-}
-
-static inline uint32_t get_ena_eth_io_tx_meta_desc_word3_valid(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_ext_valid(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_WORD3_VALID_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_WORD3_VALID_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK) >> ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_word3_valid(
-		struct ena_eth_io_tx_meta_desc *p, uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_ext_valid(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_WORD3_VALID_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_WORD3_VALID_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT) & ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_mss_hi_ptp(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_mss_hi(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK) >> ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_mss_hi_ptp(
-		struct ena_eth_io_tx_meta_desc *p, uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_mss_hi(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT) & ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_eth_meta_type(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_eth_meta_type(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK) >> ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_eth_meta_type(
-		struct ena_eth_io_tx_meta_desc *p, uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_eth_meta_type(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT) & ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_meta_store(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_meta_store(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_META_STORE_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_META_STORE_MASK) >> ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_meta_store(
-		struct ena_eth_io_tx_meta_desc *p, uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_meta_store(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT) & ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_meta_desc(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_meta_desc(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_META_DESC_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_META_DESC_MASK) >> ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_meta_desc(
-		struct ena_eth_io_tx_meta_desc *p, uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_meta_desc(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_META_DESC_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT) & ENA_ETH_IO_TX_META_DESC_META_DESC_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_phase(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_phase(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_PHASE_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_PHASE_MASK) >> ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_phase(
-		struct ena_eth_io_tx_meta_desc *p, uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_phase(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_PHASE_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT) & ENA_ETH_IO_TX_META_DESC_PHASE_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_first(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_first(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_FIRST_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_FIRST_MASK) >> ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_first(
-		struct ena_eth_io_tx_meta_desc *p, uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_first(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_FIRST_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT) & ENA_ETH_IO_TX_META_DESC_FIRST_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_last(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_last(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_LAST_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_LAST_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_LAST_MASK) >> ENA_ETH_IO_TX_META_DESC_LAST_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_last(
-		struct ena_eth_io_tx_meta_desc *p, uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_last(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_LAST_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_LAST_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_LAST_SHIFT) & ENA_ETH_IO_TX_META_DESC_LAST_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_comp_req(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_comp_req(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT;
+	return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK) >> ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_comp_req(
-		struct ena_eth_io_tx_meta_desc *p, uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_comp_req(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK;
+	p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT) & ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_req_id_hi(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_req_id_hi(const struct ena_eth_io_tx_meta_desc *p)
 {
 	return p->word1 & ENA_ETH_IO_TX_META_DESC_REQ_ID_HI_MASK;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_req_id_hi(
-		struct ena_eth_io_tx_meta_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_req_id_hi(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
 	p->word1 |= val & ENA_ETH_IO_TX_META_DESC_REQ_ID_HI_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_l3_hdr_len(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_l3_hdr_len(const struct ena_eth_io_tx_meta_desc *p)
 {
 	return p->word2 & ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_l3_hdr_len(
-		struct ena_eth_io_tx_meta_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_l3_hdr_len(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
 	p->word2 |= val & ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_l3_hdr_off(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_l3_hdr_off(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->word2 & ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT;
+	return (p->word2 & ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK) >> ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_l3_hdr_off(
-		struct ena_eth_io_tx_meta_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_l3_hdr_off(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->word2 |=
-		(val << ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK;
+	p->word2 |= (val << ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT) & ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_l4_hdr_len_in_words(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_l4_hdr_len_in_words(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->word2 & ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT;
+	return (p->word2 & ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK) >> ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_l4_hdr_len_in_words(
-		struct ena_eth_io_tx_meta_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_l4_hdr_len_in_words(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->word2 |=
-		(val << ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK;
+	p->word2 |= (val << ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT) & ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_tx_meta_desc_mss_lo(
-		const struct ena_eth_io_tx_meta_desc *p)
+static inline uint32_t get_ena_eth_io_tx_meta_desc_mss_lo(const struct ena_eth_io_tx_meta_desc *p)
 {
-	return (p->word2 & ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK)
-		>> ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT;
+	return (p->word2 & ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK) >> ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT;
 }
 
-static inline void set_ena_eth_io_tx_meta_desc_mss_lo(
-		struct ena_eth_io_tx_meta_desc *p,
-		uint32_t val)
+static inline void set_ena_eth_io_tx_meta_desc_mss_lo(struct ena_eth_io_tx_meta_desc *p, uint32_t val)
 {
-	p->word2 |=
-		(val << ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT)
-		& ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK;
+	p->word2 |= (val << ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT) & ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK;
 }
 
-static inline uint8_t get_ena_eth_io_tx_cdesc_phase(
-		const struct ena_eth_io_tx_cdesc *p)
+static inline uint8_t get_ena_eth_io_tx_cdesc_phase(const struct ena_eth_io_tx_cdesc *p)
 {
 	return p->flags & ENA_ETH_IO_TX_CDESC_PHASE_MASK;
 }
 
-static inline void set_ena_eth_io_tx_cdesc_phase(
-		struct ena_eth_io_tx_cdesc *p,
-		uint8_t val)
+static inline void set_ena_eth_io_tx_cdesc_phase(struct ena_eth_io_tx_cdesc *p, uint8_t val)
 {
 	p->flags |= val & ENA_ETH_IO_TX_CDESC_PHASE_MASK;
 }
 
-static inline uint8_t get_ena_eth_io_rx_desc_phase(
-		const struct ena_eth_io_rx_desc *p)
+static inline uint8_t get_ena_eth_io_rx_desc_phase(const struct ena_eth_io_rx_desc *p)
 {
 	return p->ctrl & ENA_ETH_IO_RX_DESC_PHASE_MASK;
 }
 
-static inline void set_ena_eth_io_rx_desc_phase(
-		struct ena_eth_io_rx_desc *p,
-		uint8_t val)
+static inline void set_ena_eth_io_rx_desc_phase(struct ena_eth_io_rx_desc *p, uint8_t val)
 {
 	p->ctrl |= val & ENA_ETH_IO_RX_DESC_PHASE_MASK;
 }
 
-static inline uint8_t get_ena_eth_io_rx_desc_first(
-		const struct ena_eth_io_rx_desc *p)
+static inline uint8_t get_ena_eth_io_rx_desc_first(const struct ena_eth_io_rx_desc *p)
 {
-	return (p->ctrl & ENA_ETH_IO_RX_DESC_FIRST_MASK)
-		>> ENA_ETH_IO_RX_DESC_FIRST_SHIFT;
+	return (p->ctrl & ENA_ETH_IO_RX_DESC_FIRST_MASK) >> ENA_ETH_IO_RX_DESC_FIRST_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_desc_first(
-		struct ena_eth_io_rx_desc *p,
-		uint8_t val)
+static inline void set_ena_eth_io_rx_desc_first(struct ena_eth_io_rx_desc *p, uint8_t val)
 {
-	p->ctrl |=
-		(val << ENA_ETH_IO_RX_DESC_FIRST_SHIFT)
-		& ENA_ETH_IO_RX_DESC_FIRST_MASK;
+	p->ctrl |= (val << ENA_ETH_IO_RX_DESC_FIRST_SHIFT) & ENA_ETH_IO_RX_DESC_FIRST_MASK;
 }
 
-static inline uint8_t get_ena_eth_io_rx_desc_last(
-		const struct ena_eth_io_rx_desc *p)
+static inline uint8_t get_ena_eth_io_rx_desc_last(const struct ena_eth_io_rx_desc *p)
 {
-	return (p->ctrl & ENA_ETH_IO_RX_DESC_LAST_MASK)
-		>> ENA_ETH_IO_RX_DESC_LAST_SHIFT;
+	return (p->ctrl & ENA_ETH_IO_RX_DESC_LAST_MASK) >> ENA_ETH_IO_RX_DESC_LAST_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_desc_last(
-		struct ena_eth_io_rx_desc *p,
-		uint8_t val)
+static inline void set_ena_eth_io_rx_desc_last(struct ena_eth_io_rx_desc *p, uint8_t val)
 {
-	p->ctrl |=
-		(val << ENA_ETH_IO_RX_DESC_LAST_SHIFT)
-		& ENA_ETH_IO_RX_DESC_LAST_MASK;
+	p->ctrl |= (val << ENA_ETH_IO_RX_DESC_LAST_SHIFT) & ENA_ETH_IO_RX_DESC_LAST_MASK;
 }
 
-static inline uint8_t get_ena_eth_io_rx_desc_comp_req(
-		const struct ena_eth_io_rx_desc *p)
+static inline uint8_t get_ena_eth_io_rx_desc_comp_req(const struct ena_eth_io_rx_desc *p)
 {
-	return (p->ctrl & ENA_ETH_IO_RX_DESC_COMP_REQ_MASK)
-		>> ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT;
+	return (p->ctrl & ENA_ETH_IO_RX_DESC_COMP_REQ_MASK) >> ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_desc_comp_req(
-		struct ena_eth_io_rx_desc *p,
-		uint8_t val)
+static inline void set_ena_eth_io_rx_desc_comp_req(struct ena_eth_io_rx_desc *p, uint8_t val)
 {
-	p->ctrl |=
-		(val << ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT)
-		& ENA_ETH_IO_RX_DESC_COMP_REQ_MASK;
+	p->ctrl |= (val << ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT) & ENA_ETH_IO_RX_DESC_COMP_REQ_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_proto_idx(
-		const struct ena_eth_io_rx_cdesc_base *p)
+static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_proto_idx(const struct ena_eth_io_rx_cdesc_base *p)
 {
 	return p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK;
 }
 
-static inline void set_ena_eth_io_rx_cdesc_base_l3_proto_idx(
-		struct ena_eth_io_rx_cdesc_base *p,
-		uint32_t val)
+static inline void set_ena_eth_io_rx_cdesc_base_l3_proto_idx(struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
 {
 	p->status |= val & ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_rx_cdesc_base_src_vlan_cnt(
-		const struct ena_eth_io_rx_cdesc_base *p)
+static inline uint32_t get_ena_eth_io_rx_cdesc_base_src_vlan_cnt(const struct ena_eth_io_rx_cdesc_base *p)
 {
-	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK)
-		>> ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT;
+	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_cdesc_base_src_vlan_cnt(
-		struct ena_eth_io_rx_cdesc_base *p,
-		uint32_t val)
+static inline void set_ena_eth_io_rx_cdesc_base_src_vlan_cnt(struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
 {
-	p->status |=
-		(val << ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT)
-		& ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK;
+	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_rx_cdesc_base_l4_proto_idx(
-		const struct ena_eth_io_rx_cdesc_base *p)
+static inline uint32_t get_ena_eth_io_rx_cdesc_base_l4_proto_idx(const struct ena_eth_io_rx_cdesc_base *p)
 {
-	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK)
-		>> ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT;
+	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_cdesc_base_l4_proto_idx(
-		struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
+static inline void set_ena_eth_io_rx_cdesc_base_l4_proto_idx(struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
 {
-	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT)
-		& ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK;
+	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_csum_err(
-		const struct ena_eth_io_rx_cdesc_base *p)
+static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_csum_err(const struct ena_eth_io_rx_cdesc_base *p)
 {
-	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK)
-		>> ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT;
+	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_cdesc_base_l3_csum_err(
-		struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
+static inline void set_ena_eth_io_rx_cdesc_base_l3_csum_err(struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
 {
-	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT)
-		& ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK;
+	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_rx_cdesc_base_l4_csum_err(
-		const struct ena_eth_io_rx_cdesc_base *p)
+static inline uint32_t get_ena_eth_io_rx_cdesc_base_l4_csum_err(const struct ena_eth_io_rx_cdesc_base *p)
 {
-	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK)
-		>> ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT;
+	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_cdesc_base_l4_csum_err(
-		struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
+static inline void set_ena_eth_io_rx_cdesc_base_l4_csum_err(struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
 {
-	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT)
-		& ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK;
+	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_rx_cdesc_base_ipv4_frag(
-		const struct ena_eth_io_rx_cdesc_base *p)
+static inline uint32_t get_ena_eth_io_rx_cdesc_base_ipv4_frag(const struct ena_eth_io_rx_cdesc_base *p)
 {
-	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK)
-		>> ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT;
+	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_cdesc_base_ipv4_frag(
-		struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
+static inline void set_ena_eth_io_rx_cdesc_base_ipv4_frag(struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
 {
-	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT)
-		& ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK;
+	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_rx_cdesc_base_phase(
-		const struct ena_eth_io_rx_cdesc_base *p)
+static inline uint32_t get_ena_eth_io_rx_cdesc_base_phase(const struct ena_eth_io_rx_cdesc_base *p)
 {
-	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK)
-		>> ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT;
+	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_cdesc_base_phase(
-		struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
+static inline void set_ena_eth_io_rx_cdesc_base_phase(struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
 {
-	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT)
-		& ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK;
+	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_csum2(
-		const struct ena_eth_io_rx_cdesc_base *p)
+static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_csum2(const struct ena_eth_io_rx_cdesc_base *p)
 {
-	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK)
-		>> ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT;
+	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_cdesc_base_l3_csum2(
-		struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
+static inline void set_ena_eth_io_rx_cdesc_base_l3_csum2(struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
 {
-	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT)
-		& ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK;
+	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_rx_cdesc_base_first(
-		const struct ena_eth_io_rx_cdesc_base *p)
+static inline uint32_t get_ena_eth_io_rx_cdesc_base_first(const struct ena_eth_io_rx_cdesc_base *p)
 {
-	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK)
-		>> ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT;
+	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_cdesc_base_first(
-		struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
+static inline void set_ena_eth_io_rx_cdesc_base_first(struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
 {
-	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT)
-		& ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK;
+	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_rx_cdesc_base_last(
-		const struct ena_eth_io_rx_cdesc_base *p)
+static inline uint32_t get_ena_eth_io_rx_cdesc_base_last(const struct ena_eth_io_rx_cdesc_base *p)
 {
-	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK)
-		>> ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT;
+	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_cdesc_base_last(
-		struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
+static inline void set_ena_eth_io_rx_cdesc_base_last(struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
 {
-	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT)
-		& ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK;
+	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_rx_cdesc_base_buffer(
-		const struct ena_eth_io_rx_cdesc_base *p)
+static inline uint32_t get_ena_eth_io_rx_cdesc_base_buffer(const struct ena_eth_io_rx_cdesc_base *p)
 {
-	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK)
-		>> ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT;
+	return (p->status & ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT;
 }
 
-static inline void set_ena_eth_io_rx_cdesc_base_buffer(
-		struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
+static inline void set_ena_eth_io_rx_cdesc_base_buffer(struct ena_eth_io_rx_cdesc_base *p, uint32_t val)
 {
-	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT)
-		& ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK;
+	p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_intr_reg_rx_intr_delay(
-		const struct ena_eth_io_intr_reg *p)
+static inline uint32_t get_ena_eth_io_intr_reg_rx_intr_delay(const struct ena_eth_io_intr_reg *p)
 {
 	return p->intr_control & ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK;
 }
 
-static inline void set_ena_eth_io_intr_reg_rx_intr_delay(
-		struct ena_eth_io_intr_reg *p, uint32_t val)
+static inline void set_ena_eth_io_intr_reg_rx_intr_delay(struct ena_eth_io_intr_reg *p, uint32_t val)
 {
 	p->intr_control |= val & ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_intr_reg_tx_intr_delay(
-		const struct ena_eth_io_intr_reg *p)
+static inline uint32_t get_ena_eth_io_intr_reg_tx_intr_delay(const struct ena_eth_io_intr_reg *p)
 {
-	return (p->intr_control & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK)
-		>> ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT;
+	return (p->intr_control & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK) >> ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT;
 }
 
-static inline void set_ena_eth_io_intr_reg_tx_intr_delay(
-		struct ena_eth_io_intr_reg *p, uint32_t val)
+static inline void set_ena_eth_io_intr_reg_tx_intr_delay(struct ena_eth_io_intr_reg *p, uint32_t val)
 {
-	p->intr_control |= (val << ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT)
-		& ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK;
+	p->intr_control |= (val << ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT) & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_intr_reg_intr_unmask(
-		const struct ena_eth_io_intr_reg *p)
+static inline uint32_t get_ena_eth_io_intr_reg_intr_unmask(const struct ena_eth_io_intr_reg *p)
 {
-	return (p->intr_control & ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK)
-		>> ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT;
+	return (p->intr_control & ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK) >> ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT;
 }
 
-static inline void set_ena_eth_io_intr_reg_intr_unmask(
-		struct ena_eth_io_intr_reg *p, uint32_t val)
+static inline void set_ena_eth_io_intr_reg_intr_unmask(struct ena_eth_io_intr_reg *p, uint32_t val)
 {
-	p->intr_control |= (val << ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT)
-		& ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK;
+	p->intr_control |= (val << ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT) & ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_numa(
-		const struct ena_eth_io_numa_node_cfg_reg *p)
+static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_numa(const struct ena_eth_io_numa_node_cfg_reg *p)
 {
 	return p->numa_cfg & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK;
 }
 
-static inline void set_ena_eth_io_numa_node_cfg_reg_numa(
-		struct ena_eth_io_numa_node_cfg_reg *p, uint32_t val)
+static inline void set_ena_eth_io_numa_node_cfg_reg_numa(struct ena_eth_io_numa_node_cfg_reg *p, uint32_t val)
 {
 	p->numa_cfg |= val & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK;
 }
 
-static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_enabled(
-		const struct ena_eth_io_numa_node_cfg_reg *p)
+static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_enabled(const struct ena_eth_io_numa_node_cfg_reg *p)
 {
-	return (p->numa_cfg & ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK)
-		>> ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT;
+	return (p->numa_cfg & ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK) >> ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT;
 }
 
-static inline void set_ena_eth_io_numa_node_cfg_reg_enabled(
-		struct ena_eth_io_numa_node_cfg_reg *p, uint32_t val)
+static inline void set_ena_eth_io_numa_node_cfg_reg_enabled(struct ena_eth_io_numa_node_cfg_reg *p, uint32_t val)
 {
-	p->numa_cfg |= (val << ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT)
-		& ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK;
+	p->numa_cfg |= (val << ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT) & ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK;
 }
 
 #endif /* !defined(ENA_DEFS_LINUX_MAINLINE) */
diff --git a/drivers/net/ena/base/ena_defs/ena_gen_info.h b/drivers/net/ena/base/ena_defs/ena_gen_info.h
index 3d252096..e87bcfd8 100644
--- a/drivers/net/ena/base/ena_defs/ena_gen_info.h
+++ b/drivers/net/ena/base/ena_defs/ena_gen_info.h
@@ -31,5 +31,5 @@
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
-#define	ENA_GEN_DATE	"Sun Jun  5 10:24:39 IDT 2016"
-#define	ENA_GEN_COMMIT	"17146ed"
+#define	ENA_GEN_DATE	"Sun Oct 23 12:27:32 IDT 2016"
+#define	ENA_GEN_COMMIT	"79d82fa"
diff --git a/drivers/net/ena/base/ena_defs/ena_includes.h b/drivers/net/ena/base/ena_defs/ena_includes.h
index a86c876f..30a920a8 100644
--- a/drivers/net/ena/base/ena_defs/ena_includes.h
+++ b/drivers/net/ena/base/ena_defs/ena_includes.h
@@ -35,5 +35,3 @@
 #include "ena_regs_defs.h"
 #include "ena_admin_defs.h"
 #include "ena_eth_io_defs.h"
-#include "ena_efa_admin_defs.h"
-#include "ena_efa_io_defs.h"
diff --git a/drivers/net/ena/base/ena_defs/ena_regs_defs.h b/drivers/net/ena/base/ena_defs/ena_regs_defs.h
index d0241278..b0870f25 100644
--- a/drivers/net/ena/base/ena_defs/ena_regs_defs.h
+++ b/drivers/net/ena/base/ena_defs/ena_regs_defs.h
@@ -34,6 +34,38 @@
 #ifndef _ENA_REGS_H_
 #define _ENA_REGS_H_
 
+enum ena_regs_reset_reason_types {
+	ENA_REGS_RESET_NORMAL			= 0,
+
+	ENA_REGS_RESET_KEEP_ALIVE_TO		= 1,
+
+	ENA_REGS_RESET_ADMIN_TO			= 2,
+
+	ENA_REGS_RESET_MISS_TX_CMPL		= 3,
+
+	ENA_REGS_RESET_INV_RX_REQ_ID		= 4,
+
+	ENA_REGS_RESET_INV_TX_REQ_ID		= 5,
+
+	ENA_REGS_RESET_TOO_MANY_RX_DESCS	= 6,
+
+	ENA_REGS_RESET_INIT_ERR			= 7,
+
+	ENA_REGS_RESET_DRIVER_INVALID_STATE	= 8,
+
+	ENA_REGS_RESET_OS_TRIGGER		= 9,
+
+	ENA_REGS_RESET_OS_NETDEV_WD		= 10,
+
+	ENA_REGS_RESET_SHUTDOWN			= 11,
+
+	ENA_REGS_RESET_USER_TRIGGER		= 12,
+
+	ENA_REGS_RESET_GENERIC			= 13,
+
+	ENA_REGS_RESET_MISS_INTERRUPT		= 14,
+};
+
 /* ena_registers offsets */
 #define ENA_REGS_VERSION_OFF		0x0
 #define ENA_REGS_CONTROLLER_VERSION_OFF		0x4
@@ -80,6 +112,8 @@
 #define ENA_REGS_CAPS_RESET_TIMEOUT_MASK		0x3e
 #define ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT		8
 #define ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK		0xff00
+#define ENA_REGS_CAPS_ADMIN_CMD_TO_SHIFT		16
+#define ENA_REGS_CAPS_ADMIN_CMD_TO_MASK		0xf0000
 
 /* aq_caps register */
 #define ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK		0xffff
@@ -104,6 +138,8 @@
 #define ENA_REGS_DEV_CTL_QUIESCENT_MASK		0x4
 #define ENA_REGS_DEV_CTL_IO_RESUME_SHIFT		3
 #define ENA_REGS_DEV_CTL_IO_RESUME_MASK		0x8
+#define ENA_REGS_DEV_CTL_RESET_REASON_SHIFT		28
+#define ENA_REGS_DEV_CTL_RESET_REASON_MASK		0xf0000000
 
 /* dev_sts register */
 #define ENA_REGS_DEV_STS_READY_MASK		0x1
diff --git a/drivers/net/ena/base/ena_eth_com.c b/drivers/net/ena/base/ena_eth_com.c
index 290a5666..4c4989a3 100644
--- a/drivers/net/ena/base/ena_eth_com.c
+++ b/drivers/net/ena/base/ena_eth_com.c
@@ -43,11 +43,10 @@ static inline struct ena_eth_io_rx_cdesc_base *ena_com_get_next_rx_cdesc(
 	head_masked = io_cq->head & (io_cq->q_depth - 1);
 	expected_phase = io_cq->phase;
 
-	cdesc = (struct ena_eth_io_rx_cdesc_base *)
-		((unsigned char *)io_cq->cdesc_addr.virt_addr
+	cdesc = (struct ena_eth_io_rx_cdesc_base *)(io_cq->cdesc_addr.virt_addr
 			+ (head_masked * io_cq->cdesc_entry_size_in_bytes));
 
-	desc_phase = (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) >>
+	desc_phase = (READ_ONCE(cdesc->status) & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) >>
 			ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT;
 
 	if (desc_phase != expected_phase)
@@ -74,7 +73,7 @@ static inline void *get_sq_desc(struct ena_com_io_sq *io_sq)
 
 	offset = tail_masked * io_sq->desc_entry_size;
 
-	return (unsigned char *)io_sq->desc_addr.virt_addr + offset;
+	return (void *)((uintptr_t)io_sq->desc_addr.virt_addr + offset);
 }
 
 static inline void ena_com_copy_curr_sq_desc_to_dev(struct ena_com_io_sq *io_sq)
@@ -86,8 +85,8 @@ static inline void ena_com_copy_curr_sq_desc_to_dev(struct ena_com_io_sq *io_sq)
 	if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
 		return;
 
-	memcpy_toio((unsigned char *)io_sq->desc_addr.pbuf_dev_addr + offset,
-		    (unsigned char *)io_sq->desc_addr.virt_addr + offset,
+	memcpy_toio(io_sq->desc_addr.pbuf_dev_addr + offset,
+		    io_sq->desc_addr.virt_addr + offset,
 		    io_sq->desc_entry_size);
 }
 
@@ -125,11 +124,11 @@ static inline struct ena_eth_io_rx_cdesc_base *
 {
 	idx &= (io_cq->q_depth - 1);
 	return (struct ena_eth_io_rx_cdesc_base *)
-		((unsigned char *)io_cq->cdesc_addr.virt_addr +
-			idx * io_cq->cdesc_entry_size_in_bytes);
+		((uintptr_t)io_cq->cdesc_addr.virt_addr +
+		idx * io_cq->cdesc_entry_size_in_bytes);
 }
 
-static inline int ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
+static inline u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
 					   u16 *first_cdesc_idx)
 {
 	struct ena_eth_io_rx_cdesc_base *cdesc;
@@ -143,7 +142,7 @@ static inline int ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
 
 		ena_com_cq_inc_head(io_cq);
 		count++;
-		last = (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >>
+		last = (READ_ONCE(cdesc->status) & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >>
 			ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT;
 	} while (!last);
 
@@ -183,9 +182,8 @@ static inline bool ena_com_meta_desc_changed(struct ena_com_io_sq *io_sq,
 	return false;
 }
 
-static inline void ena_com_create_and_store_tx_meta_desc(
-	struct ena_com_io_sq *io_sq,
-	struct ena_com_tx_ctx *ena_tx_ctx)
+static inline void ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq,
+							 struct ena_com_tx_ctx *ena_tx_ctx)
 {
 	struct ena_eth_io_tx_meta_desc *meta_desc = NULL;
 	struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
@@ -203,8 +201,8 @@ static inline void ena_com_create_and_store_tx_meta_desc(
 		ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK;
 	/* bits 10-13 of the mss */
 	meta_desc->len_ctrl |= ((ena_meta->mss >> 10) <<
-		ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_SHIFT) &
-		ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_MASK;
+		ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT) &
+		ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK;
 
 	/* Extended meta desc */
 	meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK;
@@ -237,11 +235,11 @@ static inline void ena_com_create_and_store_tx_meta_desc(
 static inline void ena_com_rx_set_flags(struct ena_com_rx_ctx *ena_rx_ctx,
 					struct ena_eth_io_rx_cdesc_base *cdesc)
 {
-	ena_rx_ctx->l3_proto = (enum ena_eth_io_l3_proto_index)(cdesc->status &
-		ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK);
-	ena_rx_ctx->l4_proto = (enum ena_eth_io_l4_proto_index)
-		((cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK) >>
-		ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT);
+	ena_rx_ctx->l3_proto = cdesc->status &
+		ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK;
+	ena_rx_ctx->l4_proto =
+		(cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK) >>
+		ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT;
 	ena_rx_ctx->l3_csum_err =
 		(cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK) >>
 		ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT;
@@ -280,8 +278,8 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
 	bool have_meta;
 	u64 addr_hi;
 
-	ENA_ASSERT(io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX,
-		   "wrong Q type");
+	ENA_WARN(io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_TX,
+		 "wrong Q type");
 
 	/* num_bufs +1 for potential meta desc */
 	if (ena_com_sq_empty_space(io_sq) < (num_bufs + 1)) {
@@ -410,8 +408,8 @@ int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
 	u16 nb_hw_desc;
 	u16 i;
 
-	ENA_ASSERT(io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_RX,
-		   "wrong Q type");
+	ENA_WARN(io_cq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX,
+		 "wrong Q type");
 
 	nb_hw_desc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx);
 	if (nb_hw_desc == 0) {
@@ -455,8 +453,8 @@ int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
 {
 	struct ena_eth_io_rx_desc *desc;
 
-	ENA_ASSERT(io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_RX,
-		   "wrong Q type");
+	ENA_WARN(io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX,
+		 "wrong Q type");
 
 	if (unlikely(ena_com_sq_empty_space(io_sq) == 0))
 		return ENA_COM_NO_SPACE;
@@ -475,8 +473,7 @@ int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
 
 	desc->buff_addr_lo = (u32)ena_buf->paddr;
 	desc->buff_addr_hi =
-		((ena_buf->paddr &
-		  GENMASK_ULL(io_sq->dma_addr_bits - 1, 32)) >> 32);
+		((ena_buf->paddr & GENMASK_ULL(io_sq->dma_addr_bits - 1, 32)) >> 32);
 
 	ena_com_sq_update_tail(io_sq);
 
@@ -493,20 +490,37 @@ int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id)
 	expected_phase = io_cq->phase;
 
 	cdesc = (struct ena_eth_io_tx_cdesc *)
-		((unsigned char *)io_cq->cdesc_addr.virt_addr
-		+ (masked_head * io_cq->cdesc_entry_size_in_bytes));
+		((uintptr_t)io_cq->cdesc_addr.virt_addr +
+		(masked_head * io_cq->cdesc_entry_size_in_bytes));
 
 	/* When the current completion descriptor phase isn't the same as the
 	 * expected, it mean that the device still didn't update
 	 * this completion.
 	 */
-	cdesc_phase = cdesc->flags & ENA_ETH_IO_TX_CDESC_PHASE_MASK;
+	cdesc_phase = READ_ONCE(cdesc->flags) & ENA_ETH_IO_TX_CDESC_PHASE_MASK;
 	if (cdesc_phase != expected_phase)
 		return ENA_COM_TRY_AGAIN;
 
+	if (unlikely(cdesc->req_id >= io_cq->q_depth)) {
+		ena_trc_err("Invalid req id %d\n", cdesc->req_id);
+		return ENA_COM_INVAL;
+	}
+
 	ena_com_cq_inc_head(io_cq);
 
-	*req_id = cdesc->req_id;
+	*req_id = READ_ONCE(cdesc->req_id);
 
 	return 0;
 }
+
+bool ena_com_cq_empty(struct ena_com_io_cq *io_cq)
+{
+	struct ena_eth_io_rx_cdesc_base *cdesc;
+
+	cdesc = ena_com_get_next_rx_cdesc(io_cq);
+	if (cdesc)
+		return false;
+	else
+		return true;
+}
+
diff --git a/drivers/net/ena/base/ena_eth_com.h b/drivers/net/ena/base/ena_eth_com.h
index 71a880c0..56ea4ae6 100644
--- a/drivers/net/ena/base/ena_eth_com.h
+++ b/drivers/net/ena/base/ena_eth_com.h
@@ -92,10 +92,12 @@ int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
 
 int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id);
 
+bool ena_com_cq_empty(struct ena_com_io_cq *io_cq);
+
 static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq,
 				       struct ena_eth_io_intr_reg *intr_reg)
 {
-	ENA_REG_WRITE32(intr_reg->intr_control, io_cq->unmask_reg);
+	ENA_REG_WRITE32(io_cq->bus, intr_reg->intr_control, io_cq->unmask_reg);
 }
 
 static inline int ena_com_sq_empty_space(struct ena_com_io_sq *io_sq)
@@ -118,7 +120,7 @@ static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)
 	ena_trc_dbg("write submission queue doorbell for queue: %d tail: %d\n",
 		    io_sq->qid, tail);
 
-	ENA_REG_WRITE32(tail, io_sq->db_addr);
+	ENA_REG_WRITE32(io_sq->bus, tail, io_sq->db_addr);
 
 	return 0;
 }
@@ -135,7 +137,7 @@ static inline int ena_com_update_dev_comp_head(struct ena_com_io_cq *io_cq)
 	if (io_cq->cq_head_db_reg && need_update) {
 		ena_trc_dbg("Write completion queue doorbell for queue %d: head: %d\n",
 			    io_cq->qid, head);
-		ENA_REG_WRITE32(head, io_cq->cq_head_db_reg);
+		ENA_REG_WRITE32(io_cq->bus, head, io_cq->cq_head_db_reg);
 		io_cq->last_head_update = head;
 	}
 
@@ -153,7 +155,7 @@ static inline void ena_com_update_numa_node(struct ena_com_io_cq *io_cq,
 	numa_cfg.numa_cfg = (numa_node & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK)
 		| ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK;
 
-	ENA_REG_WRITE32(numa_cfg.numa_cfg, io_cq->numa_node_cfg_reg);
+	ENA_REG_WRITE32(io_cq->bus, numa_cfg.numa_cfg, io_cq->numa_node_cfg_reg);
 }
 
 static inline void ena_com_comp_ack(struct ena_com_io_sq *io_sq, u16 elem)
diff --git a/drivers/net/ena/base/ena_plat.h b/drivers/net/ena/base/ena_plat.h
index b5b64545..f829936b 100644
--- a/drivers/net/ena/base/ena_plat.h
+++ b/drivers/net/ena/base/ena_plat.h
@@ -43,7 +43,11 @@
 #include "ena_plat_dpdk.h"
 #endif
 #elif defined(__FreeBSD__)
+#if defined(_KERNEL)
+#include "ena_plat_fbsd.h"
+#else
 #include "ena_plat_dpdk.h"
+#endif
 #elif defined(_WIN32)
 #include "ena_plat_windows.h"
 #else
diff --git a/drivers/net/ena/base/ena_plat_dpdk.h b/drivers/net/ena/base/ena_plat_dpdk.h
index 93345199..900ba1a6 100644
--- a/drivers/net/ena/base/ena_plat_dpdk.h
+++ b/drivers/net/ena/base/ena_plat_dpdk.h
@@ -73,10 +73,10 @@ typedef uint64_t dma_addr_t;
 #define ENA_COM_INVAL	-EINVAL
 #define ENA_COM_NO_SPACE	-ENOSPC
 #define ENA_COM_NO_DEVICE	-ENODEV
-#define ENA_COM_PERMISSION	-EPERM
 #define ENA_COM_TIMER_EXPIRED	-ETIME
 #define ENA_COM_FAULT	-EFAULT
 #define ENA_COM_TRY_AGAIN	-EAGAIN
+#define ENA_COM_UNSUPPORTED    -EOPNOTSUPP
 
 #define ____cacheline_aligned __rte_cache_aligned
 
@@ -116,11 +116,13 @@ typedef uint64_t dma_addr_t;
 #define ENA_MIN16(x, y) RTE_MIN((x), (y))
 #define ENA_MIN8(x, y) RTE_MIN((x), (y))
 
+#define BITS_PER_LONG_LONG (__SIZEOF_LONG_LONG__ * 8)
 #define U64_C(x) x ## ULL
 #define BIT(nr)         (1UL << (nr))
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) (((~0ULL) - (1ULL << (l)) + 1) & \
+			  (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 
 #ifdef RTE_LIBRTE_ENA_COM_DEBUG
 #define ena_trc_dbg(format, arg...)					\
@@ -138,6 +140,15 @@ typedef uint64_t dma_addr_t;
 #define ena_trc_err(format, arg...) do { } while (0)
 #endif /* RTE_LIBRTE_ENA_COM_DEBUG */
 
+#define ENA_WARN(cond, format, arg...)                                 \
+do {                                                                   \
+       if (unlikely(cond)) {                                           \
+               ena_trc_err(                                            \
+                       "Warn failed on %s:%s:%d:" format,              \
+                       __FILE__, __func__, __LINE__, ##arg);           \
+       }                                                               \
+} while (0)
+
 /* Spinlock related methods */
 #define ena_spinlock_t rte_spinlock_t
 #define ENA_SPINLOCK_INIT(spinlock) rte_spinlock_init(&spinlock)
@@ -177,10 +188,21 @@ typedef uint64_t dma_addr_t;
 #define ENA_WAIT_EVENT_SIGNAL(waitevent) pthread_cond_signal(&waitevent.cond)
 /* pthread condition doesn't need to be rearmed after usage */
 #define ENA_WAIT_EVENT_CLEAR(...)
+#define ENA_WAIT_EVENT_DESTROY(waitqueue) ((void)(waitqueue))
 
 #define ena_wait_event_t ena_wait_queue_t
 #define ENA_MIGHT_SLEEP()
 
+#define ENA_TIME_EXPIRE(timeout)  (timeout < rte_get_timer_cycles())
+#define ENA_GET_SYSTEM_TIMEOUT(timeout_us)                             \
+       (timeout_us * rte_get_timer_hz() / 1000000 + rte_get_timer_cycles())
+
+/*
+ * Each rte_memzone should have unique name.
+ * To satisfy it, count number of allocations and add it to name.
+ */
+extern uint32_t ena_alloc_cnt;
+
 #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, handle)	\
 	do {								\
 		const struct rte_memzone *mz;				\
@@ -190,17 +212,23 @@ typedef uint64_t dma_addr_t;
 				"ena_alloc_%d", ena_alloc_cnt++);	\
 		mz = rte_memzone_reserve(z_name, size, SOCKET_ID_ANY,	\
 				RTE_MEMZONE_IOVA_CONTIG);		\
-		memset(mz->addr, 0, size);				\
-		virt = mz->addr;					\
-		phys = mz->iova;					\
 		handle = mz;						\
+		if (mz == NULL) {					\
+			virt = NULL;					\
+			phys = 0;					\
+		} else {						\
+			memset(mz->addr, 0, size);			\
+			virt = mz->addr;				\
+			phys = mz->iova;				\
+		}							\
 	} while (0)
 #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, handle) 	\
 		({ ENA_TOUCH(size); ENA_TOUCH(phys);			\
 		   ENA_TOUCH(dmadev);					\
 		   rte_memzone_free(handle); })
 
-#define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, node, dev_node) \
+#define ENA_MEM_ALLOC_COHERENT_NODE(					\
+	dmadev, size, virt, phys, mem_handle, node, dev_node)		\
 	do {								\
 		const struct rte_memzone *mz;				\
 		char z_name[RTE_MEMZONE_NAMESIZE];			\
@@ -209,29 +237,30 @@ typedef uint64_t dma_addr_t;
 				"ena_alloc_%d", ena_alloc_cnt++);	\
 		mz = rte_memzone_reserve(z_name, size, node,		\
 				RTE_MEMZONE_IOVA_CONTIG);		\
-		memset(mz->addr, 0, size);				\
-		virt = mz->addr;					\
-		phys = mz->iova;					\
+		mem_handle = mz;					\
+		if (mz == NULL) {					\
+			virt = NULL;					\
+			phys = 0;					\
+		} else {						\
+			memset(mz->addr, 0, size);			\
+			virt = mz->addr;				\
+			phys = mz->iova;				\
+		}							\
 	} while (0)
 
 #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) \
 	do {								\
-		const struct rte_memzone *mz;				\
-		char z_name[RTE_MEMZONE_NAMESIZE];			\
 		ENA_TOUCH(dmadev); ENA_TOUCH(dev_node);			\
-		snprintf(z_name, sizeof(z_name),			\
-				"ena_alloc_%d", ena_alloc_cnt++);	\
-		mz = rte_memzone_reserve(z_name, size, node,		\
-				RTE_MEMZONE_IOVA_CONTIG);		\
-		memset(mz->addr, 0, size);				\
-		virt = mz->addr;					\
+		virt = rte_zmalloc_socket(NULL, size, 0, node);		\
 	} while (0)
 
 #define ENA_MEM_ALLOC(dmadev, size) rte_zmalloc(NULL, size, 1)
 #define ENA_MEM_FREE(dmadev, ptr) ({ENA_TOUCH(dmadev); rte_free(ptr); })
 
-#define ENA_REG_WRITE32(value, reg) rte_write32_relaxed((value), (reg))
-#define ENA_REG_READ32(reg) rte_read32_relaxed((reg))
+#define ENA_REG_WRITE32(bus, value, reg)				\
+	({ (void)(bus); rte_write32_relaxed((value), (reg)); })
+#define ENA_REG_READ32(bus, reg)					\
+	({ (void)(bus); rte_read32_relaxed((reg)); })
 
 #define ATOMIC32_INC(i32_ptr) rte_atomic32_inc(i32_ptr)
 #define ATOMIC32_DEC(i32_ptr) rte_atomic32_dec(i32_ptr)
@@ -247,4 +276,11 @@ typedef uint64_t dma_addr_t;
 #define PTR_ERR(error) ((long)(void *)error)
 #define might_sleep()
 
+#define lower_32_bits(x) ((uint32_t)(x))
+#define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16))
+
+#ifndef READ_ONCE
+#define READ_ONCE(var) (*((volatile typeof(var) *)(&(var))))
+#endif
+
 #endif /* DPDK_ENA_COM_ENA_PLAT_DPDK_H_ */
diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c
index c595cc7e..c255dc6d 100644
--- a/drivers/net/ena/ena_ethdev.c
+++ b/drivers/net/ena/ena_ethdev.c
@@ -54,7 +54,7 @@
 #include <ena_eth_io_defs.h>
 
 #define DRV_MODULE_VER_MAJOR	1
-#define DRV_MODULE_VER_MINOR	0
+#define DRV_MODULE_VER_MINOR	1
 #define DRV_MODULE_VER_SUBMINOR	0
 
 #define ENA_IO_TXQ_IDX(q)	(2 * (q))
@@ -85,6 +85,9 @@
 
 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
 
+#define ENA_MAX_RING_DESC	ENA_DEFAULT_RING_SIZE
+#define ENA_MIN_RING_DESC	128
+
 enum ethtool_stringset {
 	ETH_SS_TEST             = 0,
 	ETH_SS_STATS,
@@ -114,6 +117,12 @@ struct ena_stats {
 #define ENA_STAT_GLOBAL_ENTRY(stat) \
 	ENA_STAT_ENTRY(stat, dev)
 
+/*
+ * Each rte_memzone should have unique name.
+ * To satisfy it, count number of allocation and add it to name.
+ */
+uint32_t ena_alloc_cnt;
+
 static const struct ena_stats ena_stats_global_strings[] = {
 	ENA_STAT_GLOBAL_ENTRY(tx_timeout),
 	ENA_STAT_GLOBAL_ENTRY(io_suspend),
@@ -195,8 +204,11 @@ static const struct rte_pci_id pci_id_ena_map[] = {
 	{ .device_id = 0 },
 };
 
+static struct ena_aenq_handlers aenq_handlers;
+
 static int ena_device_init(struct ena_com_dev *ena_dev,
-			   struct ena_com_dev_get_features_ctx *get_feat_ctx);
+			   struct ena_com_dev_get_features_ctx *get_feat_ctx,
+			   bool *wd_state);
 static int ena_dev_configure(struct rte_eth_dev *dev);
 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
@@ -215,7 +227,9 @@ static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count);
 static void ena_init_rings(struct ena_adapter *adapter);
 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 static int ena_start(struct rte_eth_dev *dev);
+static void ena_stop(struct rte_eth_dev *dev);
 static void ena_close(struct rte_eth_dev *dev);
+static int ena_dev_reset(struct rte_eth_dev *dev);
 static int ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats);
 static void ena_rx_queue_release_all(struct rte_eth_dev *dev);
 static void ena_tx_queue_release_all(struct rte_eth_dev *dev);
@@ -238,6 +252,8 @@ static int ena_rss_reta_query(struct rte_eth_dev *dev,
 			      struct rte_eth_rss_reta_entry64 *reta_conf,
 			      uint16_t reta_size);
 static int ena_get_sset_count(struct rte_eth_dev *dev, int sset);
+static void ena_interrupt_handler_rte(void *cb_arg);
+static void ena_timer_wd_callback(struct rte_timer *timer, void *arg);
 
 static const struct eth_dev_ops ena_dev_ops = {
 	.dev_configure        = ena_dev_configure,
@@ -245,12 +261,14 @@ static const struct eth_dev_ops ena_dev_ops = {
 	.rx_queue_setup       = ena_rx_queue_setup,
 	.tx_queue_setup       = ena_tx_queue_setup,
 	.dev_start            = ena_start,
+	.dev_stop             = ena_stop,
 	.link_update          = ena_link_update,
 	.stats_get            = ena_stats_get,
 	.mtu_set              = ena_mtu_set,
 	.rx_queue_release     = ena_rx_queue_release,
 	.tx_queue_release     = ena_tx_queue_release,
 	.dev_close            = ena_close,
+	.dev_reset            = ena_dev_reset,
 	.reta_update          = ena_rss_reta_update,
 	.reta_query           = ena_rss_reta_query,
 };
@@ -346,9 +364,6 @@ static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf,
 		ena_meta->mss = mbuf->tso_segsz;
 		ena_meta->l3_hdr_len = mbuf->l3_len;
 		ena_meta->l3_hdr_offset = mbuf->l2_len;
-		/* this param needed only for TSO */
-		ena_meta->l3_outer_hdr_len = 0;
-		ena_meta->l3_outer_hdr_offset = 0;
 
 		ena_tx_ctx->meta_valid = true;
 	} else {
@@ -356,6 +371,40 @@ static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf,
 	}
 }
 
+static inline int validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id)
+{
+	if (likely(req_id < rx_ring->ring_size))
+		return 0;
+
+	RTE_LOG(ERR, PMD, "Invalid rx req_id: %hu\n", req_id);
+
+	rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
+	rx_ring->adapter->trigger_reset = true;
+
+	return -EFAULT;
+}
+
+static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
+{
+	struct ena_tx_buffer *tx_info = NULL;
+
+	if (likely(req_id < tx_ring->ring_size)) {
+		tx_info = &tx_ring->tx_buffer_info[req_id];
+		if (likely(tx_info->mbuf))
+			return 0;
+	}
+
+	if (tx_info)
+		RTE_LOG(ERR, PMD, "tx_info doesn't have valid mbuf\n");
+	else
+		RTE_LOG(ERR, PMD, "Invalid req_id: %hu\n", req_id);
+
+	/* Trigger device reset */
+	tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
+	tx_ring->adapter->trigger_reset	= true;
+	return -EFAULT;
+}
+
 static void ena_config_host_info(struct ena_com_dev *ena_dev)
 {
 	struct ena_admin_host_info *host_info;
@@ -387,9 +436,12 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev)
 
 	rc = ena_com_set_host_attributes(ena_dev);
 	if (rc) {
-		RTE_LOG(ERR, PMD, "Cannot set host attributes\n");
-		if (rc != -EPERM)
-			goto err;
+		if (rc == -ENA_COM_UNSUPPORTED)
+			RTE_LOG(WARNING, PMD, "Cannot set host attributes\n");
+		else
+			RTE_LOG(ERR, PMD, "Cannot set host attributes\n");
+
+		goto err;
 	}
 
 	return;
@@ -440,9 +492,12 @@ static void ena_config_debug_area(struct ena_adapter *adapter)
 
 	rc = ena_com_set_host_attributes(&adapter->ena_dev);
 	if (rc) {
-		RTE_LOG(WARNING, PMD, "Cannot set host attributes\n");
-		if (rc != -EPERM)
-			goto err;
+		if (rc == -ENA_COM_UNSUPPORTED)
+			RTE_LOG(WARNING, PMD, "Cannot set host attributes\n");
+		else
+			RTE_LOG(ERR, PMD, "Cannot set host attributes\n");
+
+		goto err;
 	}
 
 	return;
@@ -455,12 +510,76 @@ static void ena_close(struct rte_eth_dev *dev)
 	struct ena_adapter *adapter =
 		(struct ena_adapter *)(dev->data->dev_private);
 
-	adapter->state = ENA_ADAPTER_STATE_STOPPED;
+	ena_stop(dev);
+	adapter->state = ENA_ADAPTER_STATE_CLOSED;
 
 	ena_rx_queue_release_all(dev);
 	ena_tx_queue_release_all(dev);
 }
 
+static int
+ena_dev_reset(struct rte_eth_dev *dev)
+{
+	struct rte_mempool *mb_pool_rx[ENA_MAX_NUM_QUEUES];
+	struct rte_eth_dev *eth_dev;
+	struct rte_pci_device *pci_dev;
+	struct rte_intr_handle *intr_handle;
+	struct ena_com_dev *ena_dev;
+	struct ena_com_dev_get_features_ctx get_feat_ctx;
+	struct ena_adapter *adapter;
+	int nb_queues;
+	int rc, i;
+	bool wd_state;
+
+	adapter = (struct ena_adapter *)(dev->data->dev_private);
+	ena_dev = &adapter->ena_dev;
+	eth_dev = adapter->rte_dev;
+	pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+	intr_handle = &pci_dev->intr_handle;
+	nb_queues = eth_dev->data->nb_rx_queues;
+
+	ena_com_set_admin_running_state(ena_dev, false);
+
+	rc = ena_com_dev_reset(ena_dev, adapter->reset_reason);
+	if (rc)
+		RTE_LOG(ERR, PMD, "Device reset failed\n");
+
+	for (i = 0; i < nb_queues; i++)
+		mb_pool_rx[i] = adapter->rx_ring[i].mb_pool;
+
+	ena_rx_queue_release_all(eth_dev);
+	ena_tx_queue_release_all(eth_dev);
+
+	rte_intr_disable(intr_handle);
+
+	ena_com_abort_admin_commands(ena_dev);
+	ena_com_wait_for_abort_completion(ena_dev);
+	ena_com_admin_destroy(ena_dev);
+	ena_com_mmio_reg_read_request_destroy(ena_dev);
+
+	rc = ena_device_init(ena_dev, &get_feat_ctx, &wd_state);
+	if (rc) {
+		PMD_INIT_LOG(CRIT, "Cannot initialize device\n");
+		return rc;
+	}
+	adapter->wd_state = wd_state;
+
+	rte_intr_enable(intr_handle);
+	ena_com_set_admin_polling_mode(ena_dev, false);
+	ena_com_admin_aenq_enable(ena_dev);
+
+	for (i = 0; i < nb_queues; ++i)
+		ena_rx_queue_setup(eth_dev, i, adapter->rx_ring_size, 0, NULL,
+			mb_pool_rx[i]);
+
+	for (i = 0; i < nb_queues; ++i)
+		ena_tx_queue_setup(eth_dev, i, adapter->tx_ring_size, 0, NULL);
+
+	adapter->trigger_reset = false;
+
+	return 0;
+}
+
 static int ena_rss_reta_update(struct rte_eth_dev *dev,
 			       struct rte_eth_rss_reta_entry64 *reta_conf,
 			       uint16_t reta_size)
@@ -468,7 +587,7 @@ static int ena_rss_reta_update(struct rte_eth_dev *dev,
 	struct ena_adapter *adapter =
 		(struct ena_adapter *)(dev->data->dev_private);
 	struct ena_com_dev *ena_dev = &adapter->ena_dev;
-	int ret, i;
+	int rc, i;
 	u16 entry_value;
 	int conf_idx;
 	int idx;
@@ -480,8 +599,7 @@ static int ena_rss_reta_update(struct rte_eth_dev *dev,
 		RTE_LOG(WARNING, PMD,
 			"indirection table %d is bigger than supported (%d)\n",
 			reta_size, ENA_RX_RSS_TABLE_SIZE);
-		ret = -EINVAL;
-		goto err;
+		return -EINVAL;
 	}
 
 	for (i = 0 ; i < reta_size ; i++) {
@@ -493,29 +611,28 @@ static int ena_rss_reta_update(struct rte_eth_dev *dev,
 		if (TEST_BIT(reta_conf[conf_idx].mask, idx)) {
 			entry_value =
 				ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]);
-			ret = ena_com_indirect_table_fill_entry(ena_dev,
-								i,
-								entry_value);
-			if (unlikely(ret && (ret != ENA_COM_PERMISSION))) {
+
+			rc = ena_com_indirect_table_fill_entry(ena_dev,
+							       i,
+							       entry_value);
+			if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) {
 				RTE_LOG(ERR, PMD,
 					"Cannot fill indirect table\n");
-				ret = -ENOTSUP;
-				goto err;
+				return rc;
 			}
 		}
 	}
 
-	ret = ena_com_indirect_table_set(ena_dev);
-	if (unlikely(ret && (ret != ENA_COM_PERMISSION))) {
+	rc = ena_com_indirect_table_set(ena_dev);
+	if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) {
 		RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n");
-		ret = -ENOTSUP;
-		goto err;
+		return rc;
 	}
 
 	RTE_LOG(DEBUG, PMD, "%s(): RSS configured %d entries  for port %d\n",
 		__func__, reta_size, adapter->rte_dev->data->port_id);
-err:
-	return ret;
+
+	return 0;
 }
 
 /* Query redirection table. */
@@ -526,7 +643,7 @@ static int ena_rss_reta_query(struct rte_eth_dev *dev,
 	struct ena_adapter *adapter =
 		(struct ena_adapter *)(dev->data->dev_private);
 	struct ena_com_dev *ena_dev = &adapter->ena_dev;
-	int ret;
+	int rc;
 	int i;
 	u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0};
 	int reta_conf_idx;
@@ -536,11 +653,10 @@ static int ena_rss_reta_query(struct rte_eth_dev *dev,
 	    (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL)))
 		return -EINVAL;
 
-	ret = ena_com_indirect_table_get(ena_dev, indirect_table);
-	if (unlikely(ret && (ret != ENA_COM_PERMISSION))) {
+	rc = ena_com_indirect_table_get(ena_dev, indirect_table);
+	if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) {
 		RTE_LOG(ERR, PMD, "cannot get indirect table\n");
-		ret = -ENOTSUP;
-		goto err;
+		return -ENOTSUP;
 	}
 
 	for (i = 0 ; i < reta_size ; i++) {
@@ -550,8 +666,8 @@ static int ena_rss_reta_query(struct rte_eth_dev *dev,
 			reta_conf[reta_conf_idx].reta[reta_idx] =
 				ENA_IO_RXQ_IDX_REV(indirect_table[i]);
 	}
-err:
-	return ret;
+
+	return 0;
 }
 
 static int ena_rss_init_default(struct ena_adapter *adapter)
@@ -571,7 +687,7 @@ static int ena_rss_init_default(struct ena_adapter *adapter)
 		val = i % nb_rx_queues;
 		rc = ena_com_indirect_table_fill_entry(ena_dev, i,
 						       ENA_IO_RXQ_IDX(val));
-		if (unlikely(rc && (rc != ENA_COM_PERMISSION))) {
+		if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) {
 			RTE_LOG(ERR, PMD, "Cannot fill indirect table\n");
 			goto err_fill_indir;
 		}
@@ -579,19 +695,19 @@ static int ena_rss_init_default(struct ena_adapter *adapter)
 
 	rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL,
 					ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
-	if (unlikely(rc && (rc != ENA_COM_PERMISSION))) {
+	if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) {
 		RTE_LOG(INFO, PMD, "Cannot fill hash function\n");
 		goto err_fill_indir;
 	}
 
 	rc = ena_com_set_default_hash_ctrl(ena_dev);
-	if (unlikely(rc && (rc != ENA_COM_PERMISSION))) {
+	if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) {
 		RTE_LOG(INFO, PMD, "Cannot fill hash control\n");
 		goto err_fill_indir;
 	}
 
 	rc = ena_com_indirect_table_set(ena_dev);
-	if (unlikely(rc && (rc != ENA_COM_PERMISSION))) {
+	if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) {
 		RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n");
 		goto err_fill_indir;
 	}
@@ -650,6 +766,10 @@ static void ena_rx_queue_release(void *queue)
 		rte_free(ring->rx_buffer_info);
 	ring->rx_buffer_info = NULL;
 
+	if (ring->empty_rx_reqs)
+		rte_free(ring->empty_rx_reqs);
+	ring->empty_rx_reqs = NULL;
+
 	ring->configured = 0;
 
 	RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released\n",
@@ -723,9 +843,12 @@ static int ena_link_update(struct rte_eth_dev *dev,
 			   __rte_unused int wait_to_complete)
 {
 	struct rte_eth_link *link = &dev->data->dev_link;
+	struct ena_adapter *adapter;
+
+	adapter = (struct ena_adapter *)(dev->data->dev_private);
 
-	link->link_status = ETH_LINK_UP;
-	link->link_speed = ETH_SPEED_NUM_10G;
+	link->link_status = adapter->link_status ? ETH_LINK_UP : ETH_LINK_DOWN;
+	link->link_speed = ETH_SPEED_NUM_NONE;
 	link->link_duplex = ETH_LINK_FULL_DUPLEX;
 
 	return 0;
@@ -737,13 +860,18 @@ static int ena_queue_restart_all(struct rte_eth_dev *dev,
 	struct ena_adapter *adapter =
 		(struct ena_adapter *)(dev->data->dev_private);
 	struct ena_ring *queues = NULL;
+	int nb_queues;
 	int i = 0;
 	int rc = 0;
 
-	queues = (ring_type == ENA_RING_TYPE_RX) ?
-		adapter->rx_ring : adapter->tx_ring;
-
-	for (i = 0; i < adapter->num_queues; i++) {
+	if (ring_type == ENA_RING_TYPE_RX) {
+		queues = adapter->rx_ring;
+		nb_queues = dev->data->nb_rx_queues;
+	} else {
+		queues = adapter->tx_ring;
+		nb_queues = dev->data->nb_tx_queues;
+	}
+	for (i = 0; i < nb_queues; i++) {
 		if (queues[i].configured) {
 			if (ring_type == ENA_RING_TYPE_RX) {
 				ena_assert_msg(
@@ -761,7 +889,7 @@ static int ena_queue_restart_all(struct rte_eth_dev *dev,
 				PMD_INIT_LOG(ERR,
 					     "failed to restart queue %d type(%d)",
 					     i, ring_type);
-				return -1;
+				return rc;
 			}
 		}
 	}
@@ -785,9 +913,11 @@ static int ena_check_valid_conf(struct ena_adapter *adapter)
 {
 	uint32_t max_frame_len = ena_get_mtu_conf(adapter);
 
-	if (max_frame_len > adapter->max_mtu) {
-		PMD_INIT_LOG(ERR, "Unsupported MTU of %d", max_frame_len);
-		return -1;
+	if (max_frame_len > adapter->max_mtu || max_frame_len < ENA_MIN_MTU) {
+		PMD_INIT_LOG(ERR, "Unsupported MTU of %d. "
+				  "max mtu: %d, min mtu: %d\n",
+			     max_frame_len, adapter->max_mtu, ENA_MIN_MTU);
+		return ENA_COM_UNSUPPORTED;
 	}
 
 	return 0;
@@ -795,6 +925,7 @@ static int ena_check_valid_conf(struct ena_adapter *adapter)
 
 static int
 ena_calc_queue_size(struct ena_com_dev *ena_dev,
+		    u16 *max_tx_sgl_size,
 		    struct ena_com_dev_get_features_ctx *get_feat_ctx)
 {
 	uint32_t queue_size = ENA_DEFAULT_RING_SIZE;
@@ -812,11 +943,14 @@ ena_calc_queue_size(struct ena_com_dev *ena_dev,
 	if (!rte_is_power_of_2(queue_size))
 		queue_size = rte_align32pow2(queue_size >> 1);
 
-	if (queue_size == 0) {
+	if (unlikely(queue_size == 0)) {
 		PMD_INIT_LOG(ERR, "Invalid queue size");
 		return -EFAULT;
 	}
 
+	*max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS,
+		get_feat_ctx->max_queues.max_packet_tx_descs);
+
 	return queue_size;
 }
 
@@ -881,12 +1015,12 @@ static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 	ena_dev = &adapter->ena_dev;
 	ena_assert_msg(ena_dev != NULL, "Uninitialized device");
 
-	if (mtu > ena_get_mtu_conf(adapter)) {
+	if (mtu > ena_get_mtu_conf(adapter) || mtu < ENA_MIN_MTU) {
 		RTE_LOG(ERR, PMD,
-			"Given MTU (%d) exceeds maximum MTU supported (%d)\n",
-			mtu, ena_get_mtu_conf(adapter));
-		rc = -EINVAL;
-		goto err;
+			"Invalid MTU setting. new_mtu: %d "
+			"max mtu: %d min mtu: %d\n",
+			mtu, ena_get_mtu_conf(adapter), ENA_MIN_MTU);
+		return -EINVAL;
 	}
 
 	rc = ena_com_set_dev_mtu(ena_dev, mtu);
@@ -895,7 +1029,6 @@ static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 	else
 		RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu);
 
-err:
 	return rc;
 }
 
@@ -903,14 +1036,9 @@ static int ena_start(struct rte_eth_dev *dev)
 {
 	struct ena_adapter *adapter =
 		(struct ena_adapter *)(dev->data->dev_private);
+	uint64_t ticks;
 	int rc = 0;
 
-	if (!(adapter->state == ENA_ADAPTER_STATE_CONFIG ||
-	      adapter->state == ENA_ADAPTER_STATE_STOPPED)) {
-		PMD_INIT_LOG(ERR, "API violation");
-		return -1;
-	}
-
 	rc = ena_check_valid_conf(adapter);
 	if (rc)
 		return rc;
@@ -924,7 +1052,7 @@ static int ena_start(struct rte_eth_dev *dev)
 		return rc;
 
 	if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode &
-	    ETH_MQ_RX_RSS_FLAG) {
+	    ETH_MQ_RX_RSS_FLAG && adapter->rte_dev->data->nb_rx_queues > 0) {
 		rc = ena_rss_init_default(adapter);
 		if (rc)
 			return rc;
@@ -932,11 +1060,28 @@ static int ena_start(struct rte_eth_dev *dev)
 
 	ena_stats_restart(dev);
 
+	adapter->timestamp_wd = rte_get_timer_cycles();
+	adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT;
+
+	ticks = rte_get_timer_hz();
+	rte_timer_reset(&adapter->timer_wd, ticks, PERIODICAL, rte_lcore_id(),
+			ena_timer_wd_callback, adapter);
+
 	adapter->state = ENA_ADAPTER_STATE_RUNNING;
 
 	return 0;
 }
 
+static void ena_stop(struct rte_eth_dev *dev)
+{
+	struct ena_adapter *adapter =
+		(struct ena_adapter *)(dev->data->dev_private);
+
+	rte_timer_stop_sync(&adapter->timer_wd);
+
+	adapter->state = ENA_ADAPTER_STATE_STOPPED;
+}
+
 static int ena_queue_restart(struct ena_ring *ring)
 {
 	int rc, bufs_num;
@@ -954,7 +1099,7 @@ static int ena_queue_restart(struct ena_ring *ring)
 	rc = ena_populate_rx_queue(ring, bufs_num);
 	if (rc != bufs_num) {
 		PMD_INIT_LOG(ERR, "Failed to populate rx ring !");
-		return (-1);
+		return ENA_COM_FAULT;
 	}
 
 	return 0;
@@ -984,12 +1129,12 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,
 		RTE_LOG(CRIT, PMD,
 			"API violation. Queue %d is already configured\n",
 			queue_idx);
-		return -1;
+		return ENA_COM_FAULT;
 	}
 
 	if (!rte_is_power_of_2(nb_desc)) {
 		RTE_LOG(ERR, PMD,
-			"Unsupported size of RX queue: %d is not a power of 2.",
+			"Unsupported size of TX queue: %d is not a power of 2.",
 			nb_desc);
 		return -EINVAL;
 	}
@@ -1015,6 +1160,7 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,
 		RTE_LOG(ERR, PMD,
 			"failed to create io TX queue #%d (qid:%d) rc: %d\n",
 			queue_idx, ena_qid, rc);
+		return rc;
 	}
 	txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid];
 	txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid];
@@ -1026,10 +1172,11 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,
 		RTE_LOG(ERR, PMD,
 			"Failed to get TX queue handlers. TX queue num %d rc: %d\n",
 			queue_idx, rc);
-		ena_com_destroy_io_queue(ena_dev, ena_qid);
-		goto err;
+		goto err_destroy_io_queue;
 	}
 
+	ena_com_update_numa_node(txq->ena_com_io_cq, ctx.numa_node);
+
 	txq->port_id = dev->data->port_id;
 	txq->next_to_clean = 0;
 	txq->next_to_use = 0;
@@ -1041,7 +1188,8 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,
 					  RTE_CACHE_LINE_SIZE);
 	if (!txq->tx_buffer_info) {
 		RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n");
-		return -ENOMEM;
+		rc = -ENOMEM;
+		goto err_destroy_io_queue;
 	}
 
 	txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs",
@@ -1049,18 +1197,29 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,
 					 RTE_CACHE_LINE_SIZE);
 	if (!txq->empty_tx_reqs) {
 		RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n");
-		rte_free(txq->tx_buffer_info);
-		return -ENOMEM;
+		rc = -ENOMEM;
+		goto err_free;
 	}
+
 	for (i = 0; i < txq->ring_size; i++)
 		txq->empty_tx_reqs[i] = i;
 
-	txq->offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
+	if (tx_conf != NULL) {
+		txq->offloads =
+			tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
+	}
 
 	/* Store pointer to this queue in upper layer */
 	txq->configured = 1;
 	dev->data->tx_queues[queue_idx] = txq;
-err:
+
+	return 0;
+
+err_free:
+	rte_free(txq->tx_buffer_info);
+
+err_destroy_io_queue:
+	ena_com_destroy_io_queue(ena_dev, ena_qid);
 	return rc;
 }
 
@@ -1079,7 +1238,7 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev,
 		(struct ena_adapter *)(dev->data->dev_private);
 	struct ena_ring *rxq = NULL;
 	uint16_t ena_qid = 0;
-	int rc = 0;
+	int i, rc = 0;
 	struct ena_com_dev *ena_dev = &adapter->ena_dev;
 
 	rxq = &adapter->rx_ring[queue_idx];
@@ -1087,12 +1246,12 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev,
 		RTE_LOG(CRIT, PMD,
 			"API violation. Queue %d is already configured\n",
 			queue_idx);
-		return -1;
+		return ENA_COM_FAULT;
 	}
 
 	if (!rte_is_power_of_2(nb_desc)) {
 		RTE_LOG(ERR, PMD,
-			"Unsupported size of TX queue: %d is not a power of 2.",
+			"Unsupported size of RX queue: %d is not a power of 2.",
 			nb_desc);
 		return -EINVAL;
 	}
@@ -1114,9 +1273,11 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev,
 	ctx.numa_node = ena_cpu_to_node(queue_idx);
 
 	rc = ena_com_create_io_queue(ena_dev, &ctx);
-	if (rc)
+	if (rc) {
 		RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc: %d\n",
 			queue_idx, rc);
+		return rc;
+	}
 
 	rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid];
 	rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid];
@@ -1129,6 +1290,7 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev,
 			"Failed to get RX queue handlers. RX queue num %d rc: %d\n",
 			queue_idx, rc);
 		ena_com_destroy_io_queue(ena_dev, ena_qid);
+		return rc;
 	}
 
 	rxq->port_id = dev->data->port_id;
@@ -1142,9 +1304,24 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev,
 					  RTE_CACHE_LINE_SIZE);
 	if (!rxq->rx_buffer_info) {
 		RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n");
+		ena_com_destroy_io_queue(ena_dev, ena_qid);
+		return -ENOMEM;
+	}
+
+	rxq->empty_rx_reqs = rte_zmalloc("rxq->empty_rx_reqs",
+					 sizeof(uint16_t) * nb_desc,
+					 RTE_CACHE_LINE_SIZE);
+	if (!rxq->empty_rx_reqs) {
+		RTE_LOG(ERR, PMD, "failed to alloc mem for empty rx reqs\n");
+		rte_free(rxq->rx_buffer_info);
+		rxq->rx_buffer_info = NULL;
+		ena_com_destroy_io_queue(ena_dev, ena_qid);
 		return -ENOMEM;
 	}
 
+	for (i = 0; i < nb_desc; i++)
+		rxq->empty_tx_reqs[i] = i;
+
 	/* Store pointer to this queue in upper layer */
 	rxq->configured = 1;
 	dev->data->rx_queues[queue_idx] = rxq;
@@ -1159,7 +1336,7 @@ static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count)
 	uint16_t ring_size = rxq->ring_size;
 	uint16_t ring_mask = ring_size - 1;
 	uint16_t next_to_use = rxq->next_to_use;
-	uint16_t in_use;
+	uint16_t in_use, req_id;
 	struct rte_mbuf **mbufs = &rxq->rx_buffer_info[0];
 
 	if (unlikely(!count))
@@ -1187,12 +1364,18 @@ static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count)
 		struct ena_com_buf ebuf;
 
 		rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]);
+
+		req_id = rxq->empty_rx_reqs[next_to_use_masked];
+		rc = validate_rx_req_id(rxq, req_id);
+		if (unlikely(rc < 0))
+			break;
+
 		/* prepare physical address for DMA transaction */
 		ebuf.paddr = mbuf->buf_iova + RTE_PKTMBUF_HEADROOM;
 		ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM;
 		/* pass resource to device */
 		rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq,
-						&ebuf, next_to_use_masked);
+						&ebuf, req_id);
 		if (unlikely(rc)) {
 			rte_mempool_put_bulk(rxq->mb_pool, (void **)(&mbuf),
 					     count - i);
@@ -1202,9 +1385,17 @@ static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count)
 		next_to_use++;
 	}
 
+	if (unlikely(i < count))
+		RTE_LOG(WARNING, PMD, "refilled rx qid %d with only %d "
+			"buffers (from %d)\n", rxq->id, i, count);
+
 	/* When we submitted free recources to device... */
-	if (i > 0) {
-		/* ...let HW know that it can fill buffers with data */
+	if (likely(i > 0)) {
+		/* ...let HW know that it can fill buffers with data
+		 *
+		 * Add memory barrier to make sure the desc were written before
+		 * issue a doorbell
+		 */
 		rte_wmb();
 		ena_com_write_sq_doorbell(rxq->ena_com_io_sq);
 
@@ -1215,8 +1406,10 @@ static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count)
 }
 
 static int ena_device_init(struct ena_com_dev *ena_dev,
-			   struct ena_com_dev_get_features_ctx *get_feat_ctx)
+			   struct ena_com_dev_get_features_ctx *get_feat_ctx,
+			   bool *wd_state)
 {
+	uint32_t aenq_groups;
 	int rc;
 	bool readless_supported;
 
@@ -1236,7 +1429,7 @@ static int ena_device_init(struct ena_com_dev *ena_dev,
 	ena_com_set_mmio_read_mode(ena_dev, readless_supported);
 
 	/* reset device */
-	rc = ena_com_dev_reset(ena_dev);
+	rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
 	if (rc) {
 		RTE_LOG(ERR, PMD, "cannot reset device\n");
 		goto err_mmio_read_less;
@@ -1252,7 +1445,7 @@ static int ena_device_init(struct ena_com_dev *ena_dev,
 	ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev);
 
 	/* ENA device administration layer init */
-	rc = ena_com_admin_init(ena_dev, NULL, true);
+	rc = ena_com_admin_init(ena_dev, &aenq_handlers, true);
 	if (rc) {
 		RTE_LOG(ERR, PMD,
 			"cannot initialize ena admin queue with device\n");
@@ -1275,6 +1468,21 @@ static int ena_device_init(struct ena_com_dev *ena_dev,
 		goto err_admin_init;
 	}
 
+	aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
+		      BIT(ENA_ADMIN_NOTIFICATION) |
+		      BIT(ENA_ADMIN_KEEP_ALIVE) |
+		      BIT(ENA_ADMIN_FATAL_ERROR) |
+		      BIT(ENA_ADMIN_WARNING);
+
+	aenq_groups &= get_feat_ctx->aenq.supported_groups;
+	rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
+	if (rc) {
+		RTE_LOG(ERR, PMD, "Cannot configure aenq groups rc: %d\n", rc);
+		goto err_admin_init;
+	}
+
+	*wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
+
 	return 0;
 
 err_admin_init:
@@ -1286,16 +1494,89 @@ err_mmio_read_less:
 	return rc;
 }
 
+static void ena_interrupt_handler_rte(void *cb_arg)
+{
+	struct ena_adapter *adapter = (struct ena_adapter *)cb_arg;
+	struct ena_com_dev *ena_dev = &adapter->ena_dev;
+
+	ena_com_admin_q_comp_intr_handler(ena_dev);
+	if (likely(adapter->state != ENA_ADAPTER_STATE_CLOSED))
+		ena_com_aenq_intr_handler(ena_dev, adapter);
+}
+
+static void check_for_missing_keep_alive(struct ena_adapter *adapter)
+{
+	if (!adapter->wd_state)
+		return;
+
+	if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)
+		return;
+
+	if (unlikely((rte_get_timer_cycles() - adapter->timestamp_wd) >=
+	    adapter->keep_alive_timeout)) {
+		RTE_LOG(ERR, PMD, "Keep alive timeout\n");
+		adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;
+		adapter->trigger_reset = true;
+	}
+}
+
+/* Check if admin queue is enabled */
+static void check_for_admin_com_state(struct ena_adapter *adapter)
+{
+	if (unlikely(!ena_com_get_admin_running_state(&adapter->ena_dev))) {
+		RTE_LOG(ERR, PMD, "ENA admin queue is not in running state!\n");
+		adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO;
+		adapter->trigger_reset = true;
+	}
+}
+
+static void ena_timer_wd_callback(__rte_unused struct rte_timer *timer,
+				  void *arg)
+{
+	struct ena_adapter *adapter = (struct ena_adapter *)arg;
+	struct rte_eth_dev *dev = adapter->rte_dev;
+
+	check_for_missing_keep_alive(adapter);
+	check_for_admin_com_state(adapter);
+
+	if (unlikely(adapter->trigger_reset)) {
+		RTE_LOG(ERR, PMD, "Trigger reset is on\n");
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET,
+			NULL);
+	}
+}
+
+static int ena_calc_io_queue_num(__rte_unused struct ena_com_dev *ena_dev,
+				 struct ena_com_dev_get_features_ctx *get_feat_ctx)
+{
+	int io_sq_num, io_cq_num, io_queue_num;
+
+	io_sq_num = get_feat_ctx->max_queues.max_sq_num;
+	io_cq_num = get_feat_ctx->max_queues.max_cq_num;
+
+	io_queue_num = RTE_MIN(io_sq_num, io_cq_num);
+
+	if (unlikely(io_queue_num == 0)) {
+		RTE_LOG(ERR, PMD, "Number of IO queues should not be 0\n");
+		return -EFAULT;
+	}
+
+	return io_queue_num;
+}
+
 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev)
 {
 	struct rte_pci_device *pci_dev;
+	struct rte_intr_handle *intr_handle;
 	struct ena_adapter *adapter =
 		(struct ena_adapter *)(eth_dev->data->dev_private);
 	struct ena_com_dev *ena_dev = &adapter->ena_dev;
 	struct ena_com_dev_get_features_ctx get_feat_ctx;
 	int queue_size, rc;
+	u16 tx_sgl_size = 0;
 
 	static int adapters_found;
+	bool wd_state;
 
 	memset(adapter, 0, sizeof(struct ena_adapter));
 	ena_dev = &adapter->ena_dev;
@@ -1319,19 +1600,16 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev)
 		     pci_dev->addr.devid,
 		     pci_dev->addr.function);
 
+	intr_handle = &pci_dev->intr_handle;
+
 	adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr;
 	adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr;
 
-	/* Present ENA_MEM_BAR indicates available LLQ mode.
-	 * Use corresponding policy
-	 */
-	if (adapter->dev_mem_base)
-		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV;
-	else if (adapter->regs)
-		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
-	else
+	if (!adapter->regs) {
 		PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)",
 			     ENA_REGS_BAR);
+		return -ENXIO;
+	}
 
 	ena_dev->reg_bar = adapter->regs;
 	ena_dev->dmadev = adapter->pdev;
@@ -1342,36 +1620,28 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev)
 		 adapter->id_number);
 
 	/* device specific initialization routine */
-	rc = ena_device_init(ena_dev, &get_feat_ctx);
+	rc = ena_device_init(ena_dev, &get_feat_ctx, &wd_state);
 	if (rc) {
 		PMD_INIT_LOG(CRIT, "Failed to init ENA device");
-		return -1;
-	}
-
-	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
-		if (get_feat_ctx.max_queues.max_llq_num == 0) {
-			PMD_INIT_LOG(ERR,
-				     "Trying to use LLQ but llq_num is 0.\n"
-				     "Fall back into regular queues.");
-			ena_dev->tx_mem_queue_type =
-				ENA_ADMIN_PLACEMENT_POLICY_HOST;
-			adapter->num_queues =
-				get_feat_ctx.max_queues.max_sq_num;
-		} else {
-			adapter->num_queues =
-				get_feat_ctx.max_queues.max_llq_num;
-		}
-	} else {
-		adapter->num_queues = get_feat_ctx.max_queues.max_sq_num;
+		goto err;
 	}
+	adapter->wd_state = wd_state;
 
-	queue_size = ena_calc_queue_size(ena_dev, &get_feat_ctx);
-	if ((queue_size <= 0) || (adapter->num_queues <= 0))
-		return -EFAULT;
+	ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
+	adapter->num_queues = ena_calc_io_queue_num(ena_dev,
+						    &get_feat_ctx);
+
+	queue_size = ena_calc_queue_size(ena_dev, &tx_sgl_size, &get_feat_ctx);
+	if (queue_size <= 0 || adapter->num_queues <= 0) {
+		rc = -EFAULT;
+		goto err_device_destroy;
+	}
 
 	adapter->tx_ring_size = queue_size;
 	adapter->rx_ring_size = queue_size;
 
+	adapter->max_tx_sgl_size = tx_sgl_size;
+
 	/* prepare ring structures */
 	ena_init_rings(adapter);
 
@@ -1394,13 +1664,66 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev)
 					 RTE_CACHE_LINE_SIZE);
 	if (!adapter->drv_stats) {
 		RTE_LOG(ERR, PMD, "failed to alloc mem for adapter stats\n");
-		return -ENOMEM;
+		rc = -ENOMEM;
+		goto err_delete_debug_area;
 	}
 
+	rte_intr_callback_register(intr_handle,
+				   ena_interrupt_handler_rte,
+				   adapter);
+	rte_intr_enable(intr_handle);
+	ena_com_set_admin_polling_mode(ena_dev, false);
+	ena_com_admin_aenq_enable(ena_dev);
+
+	if (adapters_found == 0)
+		rte_timer_subsystem_init();
+	rte_timer_init(&adapter->timer_wd);
+
 	adapters_found++;
 	adapter->state = ENA_ADAPTER_STATE_INIT;
 
 	return 0;
+
+err_delete_debug_area:
+	ena_com_delete_debug_area(ena_dev);
+
+err_device_destroy:
+	ena_com_delete_host_info(ena_dev);
+	ena_com_admin_destroy(ena_dev);
+
+err:
+	return rc;
+}
+
+static int eth_ena_dev_uninit(struct rte_eth_dev *eth_dev)
+{
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
+	struct ena_adapter *adapter =
+		(struct ena_adapter *)(eth_dev->data->dev_private);
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return -EPERM;
+
+	if (adapter->state != ENA_ADAPTER_STATE_CLOSED)
+		ena_close(eth_dev);
+
+	eth_dev->dev_ops = NULL;
+	eth_dev->rx_pkt_burst = NULL;
+	eth_dev->tx_pkt_burst = NULL;
+	eth_dev->tx_pkt_prepare = NULL;
+
+	rte_free(adapter->drv_stats);
+	adapter->drv_stats = NULL;
+
+	rte_intr_disable(intr_handle);
+	rte_intr_callback_unregister(intr_handle,
+				     ena_interrupt_handler_rte,
+				     adapter);
+
+	adapter->state = ENA_ADAPTER_STATE_FREE;
+
+	return 0;
 }
 
 static int ena_dev_configure(struct rte_eth_dev *dev)
@@ -1408,25 +1731,7 @@ static int ena_dev_configure(struct rte_eth_dev *dev)
 	struct ena_adapter *adapter =
 		(struct ena_adapter *)(dev->data->dev_private);
 
-	if (!(adapter->state == ENA_ADAPTER_STATE_INIT ||
-	      adapter->state == ENA_ADAPTER_STATE_STOPPED)) {
-		PMD_INIT_LOG(ERR, "Illegal adapter state: %d",
-			     adapter->state);
-		return -1;
-	}
-
-	switch (adapter->state) {
-	case ENA_ADAPTER_STATE_INIT:
-	case ENA_ADAPTER_STATE_STOPPED:
-		adapter->state = ENA_ADAPTER_STATE_CONFIG;
-		break;
-	case ENA_ADAPTER_STATE_CONFIG:
-		RTE_LOG(WARNING, PMD,
-			"Ivalid driver state while trying to configure device\n");
-		break;
-	default:
-		break;
-	}
+	adapter->state = ENA_ADAPTER_STATE_CONFIG;
 
 	adapter->tx_selected_offloads = dev->data->dev_conf.txmode.offloads;
 	adapter->rx_selected_offloads = dev->data->dev_conf.rxmode.offloads;
@@ -1446,6 +1751,7 @@ static void ena_init_rings(struct ena_adapter *adapter)
 		ring->id = i;
 		ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type;
 		ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size;
+		ring->sgl_size = adapter->max_tx_sgl_size;
 	}
 
 	for (i = 0; i < adapter->num_queues; i++) {
@@ -1526,6 +1832,16 @@ static void ena_infos_get(struct rte_eth_dev *dev,
 
 	adapter->tx_supported_offloads = tx_feat;
 	adapter->rx_supported_offloads = rx_feat;
+
+	dev_info->rx_desc_lim.nb_max = ENA_MAX_RING_DESC;
+	dev_info->rx_desc_lim.nb_min = ENA_MIN_RING_DESC;
+
+	dev_info->tx_desc_lim.nb_max = ENA_MAX_RING_DESC;
+	dev_info->tx_desc_lim.nb_min = ENA_MIN_RING_DESC;
+	dev_info->tx_desc_lim.nb_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS,
+					feat.max_queues.max_packet_tx_descs);
+	dev_info->tx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS,
+					feat.max_queues.max_packet_tx_descs);
 }
 
 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
@@ -1536,6 +1852,7 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 	unsigned int ring_mask = ring_size - 1;
 	uint16_t next_to_clean = rx_ring->next_to_clean;
 	uint16_t desc_in_use = 0;
+	uint16_t req_id;
 	unsigned int recv_idx = 0;
 	struct rte_mbuf *mbuf = NULL;
 	struct rte_mbuf *mbuf_head = NULL;
@@ -1569,6 +1886,7 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 				    &ena_rx_ctx);
 		if (unlikely(rc)) {
 			RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d\n", rc);
+			rx_ring->adapter->trigger_reset = true;
 			return 0;
 		}
 
@@ -1576,12 +1894,17 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 			break;
 
 		while (segments < ena_rx_ctx.descs) {
-			mbuf = rx_buff_info[next_to_clean & ring_mask];
+			req_id = ena_rx_ctx.ena_bufs[segments].req_id;
+			rc = validate_rx_req_id(rx_ring, req_id);
+			if (unlikely(rc))
+				break;
+
+			mbuf = rx_buff_info[req_id];
 			mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len;
 			mbuf->data_off = RTE_PKTMBUF_HEADROOM;
 			mbuf->refcnt = 1;
 			mbuf->next = NULL;
-			if (segments == 0) {
+			if (unlikely(segments == 0)) {
 				mbuf->nb_segs = ena_rx_ctx.descs;
 				mbuf->port = rx_ring->port_id;
 				mbuf->pkt_len = 0;
@@ -1593,6 +1916,8 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 			mbuf_head->pkt_len += mbuf->data_len;
 
 			mbuf_prev = mbuf;
+			rx_ring->empty_rx_reqs[next_to_clean & ring_mask] =
+				req_id;
 			segments++;
 			next_to_clean++;
 		}
@@ -1686,6 +2011,46 @@ eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	return i;
 }
 
+static void ena_update_hints(struct ena_adapter *adapter,
+			     struct ena_admin_ena_hw_hints *hints)
+{
+	if (hints->admin_completion_tx_timeout)
+		adapter->ena_dev.admin_queue.completion_timeout =
+			hints->admin_completion_tx_timeout * 1000;
+
+	if (hints->mmio_read_timeout)
+		/* convert to usec */
+		adapter->ena_dev.mmio_read.reg_read_to =
+			hints->mmio_read_timeout * 1000;
+
+	if (hints->driver_watchdog_timeout) {
+		if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
+			adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
+		else
+			// Convert msecs to ticks
+			adapter->keep_alive_timeout =
+				(hints->driver_watchdog_timeout *
+				rte_get_timer_hz()) / 1000;
+	}
+}
+
+static int ena_check_and_linearize_mbuf(struct ena_ring *tx_ring,
+					struct rte_mbuf *mbuf)
+{
+	int num_segments, rc;
+
+	num_segments = mbuf->nb_segs;
+
+	if (likely(num_segments < tx_ring->sgl_size))
+		return 0;
+
+	rc = rte_pktmbuf_linearize(mbuf);
+	if (unlikely(rc))
+		RTE_LOG(WARNING, PMD, "Mbuf linearize failed\n");
+
+	return rc;
+}
+
 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts)
 {
@@ -1716,6 +2081,10 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) {
 		mbuf = tx_pkts[sent_idx];
 
+		rc = ena_check_and_linearize_mbuf(tx_ring, mbuf);
+		if (unlikely(rc))
+			break;
+
 		req_id = tx_ring->empty_tx_reqs[next_to_use & ring_mask];
 		tx_info = &tx_ring->tx_buffer_info[req_id];
 		tx_info->mbuf = mbuf;
@@ -1793,6 +2162,10 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 	/* Clear complete packets  */
 	while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) {
+		rc = validate_tx_req_id(tx_ring, req_id);
+		if (rc)
+			break;
+
 		/* Get Tx info & store how many descs were processed  */
 		tx_info = &tx_ring->tx_buffer_info[req_id];
 		total_tx_descs += tx_info->tx_descs;
@@ -1820,6 +2193,9 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	return sent_idx;
 }
 
+/*********************************************************************
+ *  PMD configuration
+ *********************************************************************/
 static int eth_ena_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	struct rte_pci_device *pci_dev)
 {
@@ -1829,12 +2205,13 @@ static int eth_ena_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 
 static int eth_ena_pci_remove(struct rte_pci_device *pci_dev)
 {
-	return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
+	return rte_eth_dev_pci_generic_remove(pci_dev, eth_ena_dev_uninit);
 }
 
 static struct rte_pci_driver rte_ena_pmd = {
 	.id_table = pci_id_ena_map,
-	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+		     RTE_PCI_DRV_WC_ACTIVATE,
 	.probe = eth_ena_pci_probe,
 	.remove = eth_ena_pci_remove,
 };
@@ -1843,9 +2220,7 @@ RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio-pci");
 
-RTE_INIT(ena_init_log);
-static void
-ena_init_log(void)
+RTE_INIT(ena_init_log)
 {
 	ena_logtype_init = rte_log_register("pmd.net.ena.init");
 	if (ena_logtype_init >= 0)
@@ -1854,3 +2229,75 @@ ena_init_log(void)
 	if (ena_logtype_driver >= 0)
 		rte_log_set_level(ena_logtype_driver, RTE_LOG_NOTICE);
 }
+
+/******************************************************************************
+ ******************************** AENQ Handlers *******************************
+ *****************************************************************************/
+static void ena_update_on_link_change(void *adapter_data,
+				      struct ena_admin_aenq_entry *aenq_e)
+{
+	struct rte_eth_dev *eth_dev;
+	struct ena_adapter *adapter;
+	struct ena_admin_aenq_link_change_desc *aenq_link_desc;
+	uint32_t status;
+
+	adapter = (struct ena_adapter *)adapter_data;
+	aenq_link_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e;
+	eth_dev = adapter->rte_dev;
+
+	status = get_ena_admin_aenq_link_change_desc_link_status(aenq_link_desc);
+	adapter->link_status = status;
+
+	ena_link_update(eth_dev, 0);
+	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
+}
+
+static void ena_notification(void *data,
+			     struct ena_admin_aenq_entry *aenq_e)
+{
+	struct ena_adapter *adapter = (struct ena_adapter *)data;
+	struct ena_admin_ena_hw_hints *hints;
+
+	if (aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION)
+		RTE_LOG(WARNING, PMD, "Invalid group(%x) expected %x\n",
+			aenq_e->aenq_common_desc.group,
+			ENA_ADMIN_NOTIFICATION);
+
+	switch (aenq_e->aenq_common_desc.syndrom) {
+	case ENA_ADMIN_UPDATE_HINTS:
+		hints = (struct ena_admin_ena_hw_hints *)
+			(&aenq_e->inline_data_w4);
+		ena_update_hints(adapter, hints);
+		break;
+	default:
+		RTE_LOG(ERR, PMD, "Invalid aenq notification link state %d\n",
+			aenq_e->aenq_common_desc.syndrom);
+	}
+}
+
+static void ena_keep_alive(void *adapter_data,
+			   __rte_unused struct ena_admin_aenq_entry *aenq_e)
+{
+	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
+
+	adapter->timestamp_wd = rte_get_timer_cycles();
+}
+
+/**
+ * This handler will called for unknown event group or unimplemented handlers
+ **/
+static void unimplemented_aenq_handler(__rte_unused void *data,
+				       __rte_unused struct ena_admin_aenq_entry *aenq_e)
+{
+	RTE_LOG(ERR, PMD, "Unknown event was received or event with "
+			  "unimplemented handler\n");
+}
+
+static struct ena_aenq_handlers aenq_handlers = {
+	.handlers = {
+		[ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
+		[ENA_ADMIN_NOTIFICATION] = ena_notification,
+		[ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive
+	},
+	.unimplemented_handler = unimplemented_aenq_handler
+};
diff --git a/drivers/net/ena/ena_ethdev.h b/drivers/net/ena/ena_ethdev.h
index 394d05e0..2dc8129e 100644
--- a/drivers/net/ena/ena_ethdev.h
+++ b/drivers/net/ena/ena_ethdev.h
@@ -34,8 +34,10 @@
 #ifndef _ENA_ETHDEV_H_
 #define _ENA_ETHDEV_H_
 
+#include <rte_cycles.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
+#include <rte_timer.h>
 
 #include "ena_com.h"
 
@@ -48,8 +50,13 @@
 #define ENA_NAME_MAX_LEN	20
 #define ENA_PKT_MAX_BUFS	17
 
+#define ENA_MIN_MTU		128
+
 #define ENA_MMIO_DISABLE_REG_READ	BIT(0)
 
+#define ENA_WD_TIMEOUT_SEC	3
+#define ENA_DEVICE_KALIVE_TIMEOUT (ENA_WD_TIMEOUT_SEC * rte_get_timer_hz())
+
 struct ena_adapter;
 
 enum ena_ring_type {
@@ -70,8 +77,12 @@ struct ena_ring {
 
 	enum ena_ring_type type;
 	enum ena_admin_placement_policy_type tx_mem_queue_type;
-	/* Holds the empty requests for TX OOO completions */
-	uint16_t *empty_tx_reqs;
+	/* Holds the empty requests for TX/RX OOO completions */
+	union {
+		uint16_t *empty_tx_reqs;
+		uint16_t *empty_rx_reqs;
+	};
+
 	union {
 		struct ena_tx_buffer *tx_buffer_info; /* contex of tx packet */
 		struct rte_mbuf **rx_buffer_info; /* contex of rx packet */
@@ -92,14 +103,16 @@ struct ena_ring {
 	int configured;
 	struct ena_adapter *adapter;
 	uint64_t offloads;
+	u16 sgl_size;
 } __rte_cache_aligned;
 
 enum ena_adapter_state {
 	ENA_ADAPTER_STATE_FREE    = 0,
 	ENA_ADAPTER_STATE_INIT    = 1,
-	ENA_ADAPTER_STATE_RUNNING  = 2,
+	ENA_ADAPTER_STATE_RUNNING = 2,
 	ENA_ADAPTER_STATE_STOPPED = 3,
 	ENA_ADAPTER_STATE_CONFIG  = 4,
+	ENA_ADAPTER_STATE_CLOSED  = 5,
 };
 
 struct ena_driver_stats {
@@ -157,6 +170,7 @@ struct ena_adapter {
 	/* TX */
 	struct ena_ring tx_ring[ENA_MAX_NUM_QUEUES] __rte_cache_aligned;
 	int tx_ring_size;
+	u16 max_tx_sgl_size;
 
 	/* RX */
 	struct ena_ring rx_ring[ENA_MAX_NUM_QUEUES] __rte_cache_aligned;
@@ -180,6 +194,18 @@ struct ena_adapter {
 	uint64_t tx_selected_offloads;
 	uint64_t rx_supported_offloads;
 	uint64_t rx_selected_offloads;
+
+	bool link_status;
+
+	enum ena_regs_reset_reason_types reset_reason;
+
+	struct rte_timer timer_wd;
+	uint64_t timestamp_wd;
+	uint64_t keep_alive_timeout;
+
+	bool trigger_reset;
+
+	bool wd_state;
 };
 
 #endif /* _ENA_ETHDEV_H_ */
diff --git a/drivers/net/ena/meson.build b/drivers/net/ena/meson.build
new file mode 100644
index 00000000..091ca6e3
--- /dev/null
+++ b/drivers/net/ena/meson.build
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+allow_experimental_apis = true
+sources = files('ena_ethdev.c',
+	'base/ena_com.c',
+	'base/ena_eth_com.c')
+
+deps += ['timer']
+
+includes += include_directories('base', 'base/ena_defs')
diff --git a/drivers/net/enic/base/cq_desc.h b/drivers/net/enic/base/cq_desc.h
index 7e138027..ae8847c6 100644
--- a/drivers/net/enic/base/cq_desc.h
+++ b/drivers/net/enic/base/cq_desc.h
@@ -38,6 +38,7 @@ struct cq_desc {
 #define CQ_DESC_TYPE_MASK        ((1 << CQ_DESC_TYPE_BITS) - 1)
 #define CQ_DESC_COLOR_MASK       1
 #define CQ_DESC_COLOR_SHIFT      7
+#define CQ_DESC_COLOR_MASK_NOSHIFT 0x80
 #define CQ_DESC_Q_NUM_BITS       10
 #define CQ_DESC_Q_NUM_MASK       ((1 << CQ_DESC_Q_NUM_BITS) - 1)
 #define CQ_DESC_COMP_NDX_BITS    12
diff --git a/drivers/net/enic/base/vnic_dev.c b/drivers/net/enic/base/vnic_dev.c
index 8483f76f..16e8814a 100644
--- a/drivers/net/enic/base/vnic_dev.c
+++ b/drivers/net/enic/base/vnic_dev.c
@@ -528,6 +528,22 @@ parse_max_level:
 	return 0;
 }
 
+void vnic_dev_capable_udp_rss_weak(struct vnic_dev *vdev, bool *cfg_chk,
+				   bool *weak)
+{
+	u64 a0 = CMD_NIC_CFG, a1 = 0;
+	int wait = 1000;
+	int err;
+
+	*cfg_chk = false;
+	*weak = false;
+	err = vnic_dev_cmd(vdev, CMD_CAPABILITY, &a0, &a1, wait);
+	if (err == 0 && a0 != 0 && a1 != 0) {
+		*cfg_chk = true;
+		*weak = !!((a1 >> 32) & CMD_NIC_CFG_CAPF_UDP_WEAK);
+	}
+}
+
 int vnic_dev_capable(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd)
 {
 	u64 a0 = (u32)cmd, a1 = 0;
diff --git a/drivers/net/enic/base/vnic_dev.h b/drivers/net/enic/base/vnic_dev.h
index 3c908430..270a47bd 100644
--- a/drivers/net/enic/base/vnic_dev.h
+++ b/drivers/net/enic/base/vnic_dev.h
@@ -6,6 +6,8 @@
 #ifndef _VNIC_DEV_H_
 #define _VNIC_DEV_H_
 
+#include <stdbool.h>
+
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
 
@@ -109,6 +111,8 @@ int vnic_dev_capable_adv_filters(struct vnic_dev *vdev);
 int vnic_dev_capable(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd);
 int vnic_dev_capable_filter_mode(struct vnic_dev *vdev, u32 *mode,
 				 u8 *filter_actions);
+void vnic_dev_capable_udp_rss_weak(struct vnic_dev *vdev, bool *cfg_chk,
+				   bool *weak);
 int vnic_dev_asic_info(struct vnic_dev *vdev, u16 *asic_type, u16 *asic_rev);
 int vnic_dev_spec(struct vnic_dev *vdev, unsigned int offset, size_t size,
 	void *value);
diff --git a/drivers/net/enic/base/vnic_devcmd.h b/drivers/net/enic/base/vnic_devcmd.h
index 2865eb4d..a22d8a76 100644
--- a/drivers/net/enic/base/vnic_devcmd.h
+++ b/drivers/net/enic/base/vnic_devcmd.h
@@ -138,9 +138,27 @@ enum vnic_devcmd_cmd {
 	/* del VLAN id in (u16)a0 */
 	CMD_VLAN_DEL            = _CMDCNW(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 15),
 
-	/* nic_cfg in (u32)a0 */
+	/*
+	 * nic_cfg in (u32)a0
+	 *
+	 * Capability query:
+	 * out: (u64) a0= 1 if a1 is valid
+	 *      (u64) a1= (NIC_CFG bits supported) | (flags << 32)
+	 *                              (flags are CMD_NIC_CFG_CAPF_xxx)
+	 */
 	CMD_NIC_CFG             = _CMDCNW(_CMD_DIR_WRITE, _CMD_VTYPE_ALL, 16),
 
+	/*
+	 * nic_cfg_chk  (same as nic_cfg, but may return error)
+	 * in (u32)a0
+	 *
+	 * Capability query:
+	 * out: (u64) a0= 1 if a1 is valid
+	 *      (u64) a1= (NIC_CFG bits supported) | (flags << 32)
+	 *                              (flags are CMD_NIC_CFG_CAPF_xxx)
+	 */
+	CMD_NIC_CFG_CHK         = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ALL, 16),
+
 	/* union vnic_rss_key in mem: (u64)a0=paddr, (u16)a1=len */
 	CMD_RSS_KEY             = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 17),
 
@@ -605,6 +623,9 @@ enum filter_cap_mode {
 /* flags for CMD_INIT */
 #define CMD_INITF_DEFAULT_MAC	0x1	/* init with default mac addr */
 
+/* flags for CMD_NIC_CFG */
+#define CMD_NIC_CFG_CAPF_UDP_WEAK	(1ULL << 0) /* Bodega-style UDP RSS */
+
 /* flags for CMD_PACKET_FILTER */
 #define CMD_PFILTER_DIRECTED		0x01
 #define CMD_PFILTER_MULTICAST		0x02
diff --git a/drivers/net/enic/base/vnic_enet.h b/drivers/net/enic/base/vnic_enet.h
index 49504a7d..901f3b46 100644
--- a/drivers/net/enic/base/vnic_enet.h
+++ b/drivers/net/enic/base/vnic_enet.h
@@ -53,9 +53,8 @@ struct vnic_enet_config {
 #define VENETF_NVGRE    0x20000 /* NVGRE offload */
 #define VENETF_GRPINTR  0x40000 /* group interrupt */
 #define VENETF_NICSWITCH        0x80000 /* NICSWITCH enabled */
-#define VENETF_RSSHASH_UDP_WEAK 0x100000 /* VIC has Bodega-style UDP RSS */
-#define VENETF_RSSHASH_UDPIPV4  0x200000 /* Hash on UDP + IPv4 fields */
-#define VENETF_RSSHASH_UDPIPV6  0x400000 /* Hash on UDP + IPv6 fields */
+#define VENETF_RSSHASH_UDPIPV4  0x100000 /* Hash on UDP + IPv4 fields */
+#define VENETF_RSSHASH_UDPIPV6  0x200000 /* Hash on UDP + IPv6 fields */
 
 #define VENET_INTR_TYPE_MIN	0	/* Timer specs min interrupt spacing */
 #define VENET_INTR_TYPE_IDLE	1	/* Timer specs idle time before irq */
diff --git a/drivers/net/enic/base/vnic_nic.h b/drivers/net/enic/base/vnic_nic.h
index e318d0cb..16040852 100644
--- a/drivers/net/enic/base/vnic_nic.h
+++ b/drivers/net/enic/base/vnic_nic.h
@@ -32,8 +32,8 @@
 #define NIC_CFG_RSS_HASH_TYPE_TCP_IPV4		(1 << 2)
 #define NIC_CFG_RSS_HASH_TYPE_IPV6		(1 << 3)
 #define NIC_CFG_RSS_HASH_TYPE_TCP_IPV6		(1 << 4)
-#define NIC_CFG_RSS_HASH_TYPE_IPV6_EX		(1 << 5)
-#define NIC_CFG_RSS_HASH_TYPE_TCP_IPV6_EX	(1 << 6)
+#define NIC_CFG_RSS_HASH_TYPE_RSVD1		(1 << 5)
+#define NIC_CFG_RSS_HASH_TYPE_RSVD2		(1 << 6)
 #define NIC_CFG_RSS_HASH_TYPE_UDP_IPV6		(1 << 7)
 
 static inline void vnic_set_nic_cfg(u32 *nic_cfg,
diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h
index 9619290d..d8e67f74 100644
--- a/drivers/net/enic/base/vnic_rq.h
+++ b/drivers/net/enic/base/vnic_rq.h
@@ -52,6 +52,8 @@ struct vnic_rq {
 	struct vnic_dev *vdev;
 	struct vnic_rq_ctrl __iomem *ctrl;	/* memory-mapped */
 	struct vnic_dev_ring ring;
+	struct rte_mbuf **free_mbufs;		/* reserve of free mbufs */
+	int num_free_mbufs;
 	struct rte_mbuf **mbuf_ring;		/* array of allocated mbufs */
 	unsigned int mbuf_next_idx;		/* next mb to consume */
 	void *os_buf_head;
diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c
index d61c4c6e..c9bf3572 100644
--- a/drivers/net/enic/base/vnic_wq.c
+++ b/drivers/net/enic/base/vnic_wq.c
@@ -32,8 +32,8 @@ static int vnic_wq_alloc_bufs(struct vnic_wq *wq)
 {
 	unsigned int count = wq->ring.desc_count;
        /* Allocate the mbuf ring */
-	wq->bufs = (struct vnic_wq_buf *)rte_zmalloc_socket("wq->bufs",
-		    sizeof(struct vnic_wq_buf) * count,
+	wq->bufs = (struct rte_mbuf **)rte_zmalloc_socket("wq->bufs",
+		    sizeof(struct rte_mbuf *) * count,
 		    RTE_CACHE_LINE_SIZE, wq->socket_id);
 	wq->head_idx = 0;
 	wq->tail_idx = 0;
@@ -113,6 +113,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
 	vnic_wq_init_start(wq, cq_index, 0, 0,
 		error_interrupt_enable,
 		error_interrupt_offset);
+	wq->cq_pend = 0;
 	wq->last_completed_index = 0;
 }
 
@@ -145,9 +146,9 @@ int vnic_wq_disable(struct vnic_wq *wq)
 }
 
 void vnic_wq_clean(struct vnic_wq *wq,
-		   void (*buf_clean)(struct vnic_wq_buf *buf))
+		   void (*buf_clean)(struct rte_mbuf **buf))
 {
-	struct vnic_wq_buf *buf;
+	struct rte_mbuf **buf;
 	unsigned int  to_clean = wq->tail_idx;
 
 	buf = &wq->bufs[to_clean];
diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h
index 0135bffc..236cf696 100644
--- a/drivers/net/enic/base/vnic_wq.h
+++ b/drivers/net/enic/base/vnic_wq.h
@@ -36,24 +36,20 @@ struct vnic_wq_ctrl {
 	u32 pad9;
 };
 
-/* 16 bytes */
-struct vnic_wq_buf {
-	struct rte_mempool *pool;
-	void *mb;
-};
-
 struct vnic_wq {
 	unsigned int index;
 	uint64_t tx_offload_notsup_mask;
 	struct vnic_dev *vdev;
 	struct vnic_wq_ctrl __iomem *ctrl;              /* memory-mapped */
 	struct vnic_dev_ring ring;
-	struct vnic_wq_buf *bufs;
+	struct rte_mbuf **bufs;
 	unsigned int head_idx;
+	unsigned int cq_pend;
 	unsigned int tail_idx;
 	unsigned int socket_id;
 	const struct rte_memzone *cqmsg_rz;
 	uint16_t last_completed_index;
+	uint64_t offloads;
 };
 
 static inline unsigned int vnic_wq_desc_avail(struct vnic_wq *wq)
@@ -164,5 +160,5 @@ unsigned int vnic_wq_error_status(struct vnic_wq *wq);
 void vnic_wq_enable(struct vnic_wq *wq);
 int vnic_wq_disable(struct vnic_wq *wq);
 void vnic_wq_clean(struct vnic_wq *wq,
-		   void (*buf_clean)(struct vnic_wq_buf *buf));
+		   void (*buf_clean)(struct rte_mbuf **buf));
 #endif /* _VNIC_WQ_H_ */
diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
index ee83fe57..7c27bd51 100644
--- a/drivers/net/enic/enic.h
+++ b/drivers/net/enic/enic.h
@@ -50,6 +50,9 @@
 
 #define ENICPMD_FDIR_MAX           64
 
+/* HW default VXLAN port */
+#define ENIC_DEFAULT_VXLAN_PORT	   4789
+
 /*
  * Interrupt 0: LSC and errors
  * Interrupt 1: rx queue 0
@@ -123,6 +126,10 @@ struct enic {
 	u8 filter_actions; /* HW supported actions */
 	bool vxlan;
 	bool disable_overlay; /* devargs disable_overlay=1 */
+	bool nic_cfg_chk;     /* NIC_CFG_CHK available */
+	bool udp_rss_weak;    /* Bodega style UDP RSS */
+	uint8_t ig_vlan_rewrite_mode; /* devargs ig-vlan-rewrite */
+	uint16_t vxlan_port;  /* current vxlan port pushed to NIC */
 
 	unsigned int flags;
 	unsigned int priv_flags;
@@ -176,6 +183,7 @@ struct enic {
 
 	uint64_t rx_offload_capa; /* DEV_RX_OFFLOAD flags */
 	uint64_t tx_offload_capa; /* DEV_TX_OFFLOAD flags */
+	uint64_t tx_queue_offload_capa; /* DEV_TX_OFFLOAD flags */
 	uint64_t tx_offload_mask; /* PKT_TX flags accepted */
 };
 
@@ -305,11 +313,15 @@ int enic_clsf_init(struct enic *enic);
 void enic_clsf_destroy(struct enic *enic);
 uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 			uint16_t nb_pkts);
+uint16_t enic_noscatter_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+				  uint16_t nb_pkts);
 uint16_t enic_dummy_recv_pkts(void *rx_queue,
 			      struct rte_mbuf **rx_pkts,
 			      uint16_t nb_pkts);
 uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 			uint16_t nb_pkts);
+uint16_t enic_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+			       uint16_t nb_pkts);
 uint16_t enic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 			uint16_t nb_pkts);
 int enic_set_mtu(struct enic *enic, uint16_t new_mtu);
diff --git a/drivers/net/enic/enic_compat.h b/drivers/net/enic/enic_compat.h
index c0af1ed2..ceb1b096 100644
--- a/drivers/net/enic/enic_compat.h
+++ b/drivers/net/enic/enic_compat.h
@@ -56,6 +56,11 @@
 #define dev_debug(x, args...) dev_printk(DEBUG, args)
 
 extern int enicpmd_logtype_flow;
+extern int enicpmd_logtype_init;
+
+#define PMD_INIT_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, enicpmd_logtype_init, \
+		"%s" fmt "\n", __func__, ##args)
 
 #define __le16 u16
 #define __le32 u32
diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c
index 28630892..b3d57771 100644
--- a/drivers/net/enic/enic_ethdev.c
+++ b/drivers/net/enic/enic_ethdev.c
@@ -24,10 +24,6 @@
 int enicpmd_logtype_init;
 int enicpmd_logtype_flow;
 
-#define PMD_INIT_LOG(level, fmt, args...) \
-	rte_log(RTE_LOG_ ## level, enicpmd_logtype_init, \
-		"%s" fmt "\n", __func__, ##args)
-
 #define ENICPMD_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
 
 /*
@@ -41,10 +37,9 @@ static const struct rte_pci_id pci_id_enic_map[] = {
 };
 
 #define ENIC_DEVARG_DISABLE_OVERLAY "disable-overlay"
+#define ENIC_DEVARG_IG_VLAN_REWRITE "ig-vlan-rewrite"
 
-RTE_INIT(enicpmd_init_log);
-static void
-enicpmd_init_log(void)
+RTE_INIT(enicpmd_init_log)
 {
 	enicpmd_logtype_init = rte_log_register("pmd.net.enic.init");
 	if (enicpmd_logtype_init >= 0)
@@ -184,17 +179,21 @@ static int enicpmd_dev_tx_queue_setup(struct rte_eth_dev *eth_dev,
 	uint16_t queue_idx,
 	uint16_t nb_desc,
 	unsigned int socket_id,
-	__rte_unused const struct rte_eth_txconf *tx_conf)
+	const struct rte_eth_txconf *tx_conf)
 {
 	int ret;
 	struct enic *enic = pmd_priv(eth_dev);
+	struct vnic_wq *wq;
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return -E_RTE_SECONDARY;
 
 	ENICPMD_FUNC_TRACE();
 	RTE_ASSERT(queue_idx < enic->conf_wq_count);
-	eth_dev->data->tx_queues[queue_idx] = (void *)&enic->wq[queue_idx];
+	wq = &enic->wq[queue_idx];
+	wq->offloads = tx_conf->offloads |
+		eth_dev->data->dev_conf.txmode.offloads;
+	eth_dev->data->tx_queues[queue_idx] = (void *)wq;
 
 	ret = enic_alloc_wq(enic, queue_idx, socket_id, nb_desc);
 	if (ret) {
@@ -476,12 +475,37 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev,
 	device_info->max_mac_addrs = ENIC_MAX_MAC_ADDR;
 	device_info->rx_offload_capa = enic->rx_offload_capa;
 	device_info->tx_offload_capa = enic->tx_offload_capa;
+	device_info->tx_queue_offload_capa = enic->tx_queue_offload_capa;
 	device_info->default_rxconf = (struct rte_eth_rxconf) {
 		.rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH
 	};
 	device_info->reta_size = enic->reta_size;
 	device_info->hash_key_size = enic->hash_key_size;
 	device_info->flow_type_rss_offloads = enic->flow_type_rss_offloads;
+	device_info->rx_desc_lim = (struct rte_eth_desc_lim) {
+		.nb_max = enic->config.rq_desc_count,
+		.nb_min = ENIC_MIN_RQ_DESCS,
+		.nb_align = ENIC_ALIGN_DESCS,
+	};
+	device_info->tx_desc_lim = (struct rte_eth_desc_lim) {
+		.nb_max = enic->config.wq_desc_count,
+		.nb_min = ENIC_MIN_WQ_DESCS,
+		.nb_align = ENIC_ALIGN_DESCS,
+		.nb_seg_max = ENIC_TX_XMIT_MAX,
+		.nb_mtu_seg_max = ENIC_NON_TSO_MAX_DESC,
+	};
+	device_info->default_rxportconf = (struct rte_eth_dev_portconf) {
+		.burst_size = ENIC_DEFAULT_RX_BURST,
+		.ring_size = RTE_MIN(device_info->rx_desc_lim.nb_max,
+			ENIC_DEFAULT_RX_RING_SIZE),
+		.nb_queues = ENIC_DEFAULT_RX_RINGS,
+	};
+	device_info->default_txportconf = (struct rte_eth_dev_portconf) {
+		.burst_size = ENIC_DEFAULT_TX_BURST,
+		.ring_size = RTE_MIN(device_info->tx_desc_lim.nb_max,
+			ENIC_DEFAULT_TX_RING_SIZE),
+		.nb_queues = ENIC_DEFAULT_TX_RINGS,
+	};
 }
 
 static const uint32_t *enicpmd_dev_supported_ptypes_get(struct rte_eth_dev *dev)
@@ -498,7 +522,8 @@ static const uint32_t *enicpmd_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 		RTE_PTYPE_UNKNOWN
 	};
 
-	if (dev->rx_pkt_burst == enic_recv_pkts)
+	if (dev->rx_pkt_burst == enic_recv_pkts ||
+	    dev->rx_pkt_burst == enic_noscatter_recv_pkts)
 		return ptypes;
 	return NULL;
 }
@@ -731,15 +756,16 @@ static void enicpmd_dev_rxq_info_get(struct rte_eth_dev *dev,
 }
 
 static void enicpmd_dev_txq_info_get(struct rte_eth_dev *dev,
-				     __rte_unused uint16_t tx_queue_id,
+				     uint16_t tx_queue_id,
 				     struct rte_eth_txq_info *qinfo)
 {
 	struct enic *enic = pmd_priv(dev);
+	struct vnic_wq *wq = &enic->wq[tx_queue_id];
 
 	ENICPMD_FUNC_TRACE();
-	qinfo->nb_desc = enic->config.wq_desc_count;
+	qinfo->nb_desc = wq->ring.desc_count;
 	memset(&qinfo->conf, 0, sizeof(qinfo->conf));
-	qinfo->conf.offloads = enic->tx_offload_capa;
+	qinfo->conf.offloads = wq->offloads;
 	/* tx_thresh, and all the other fields are not applicable for enic */
 }
 
@@ -763,6 +789,79 @@ static int enicpmd_dev_rx_queue_intr_disable(struct rte_eth_dev *eth_dev,
 	return 0;
 }
 
+static int udp_tunnel_common_check(struct enic *enic,
+				   struct rte_eth_udp_tunnel *tnl)
+{
+	if (tnl->prot_type != RTE_TUNNEL_TYPE_VXLAN)
+		return -ENOTSUP;
+	if (!enic->overlay_offload) {
+		PMD_INIT_LOG(DEBUG, " vxlan (overlay offload) is not "
+			     "supported\n");
+		return -ENOTSUP;
+	}
+	return 0;
+}
+
+static int update_vxlan_port(struct enic *enic, uint16_t port)
+{
+	if (vnic_dev_overlay_offload_cfg(enic->vdev,
+					 OVERLAY_CFG_VXLAN_PORT_UPDATE,
+					 port)) {
+		PMD_INIT_LOG(DEBUG, " failed to update vxlan port\n");
+		return -EINVAL;
+	}
+	PMD_INIT_LOG(DEBUG, " updated vxlan port to %u\n", port);
+	enic->vxlan_port = port;
+	return 0;
+}
+
+static int enicpmd_dev_udp_tunnel_port_add(struct rte_eth_dev *eth_dev,
+					   struct rte_eth_udp_tunnel *tnl)
+{
+	struct enic *enic = pmd_priv(eth_dev);
+	int ret;
+
+	ENICPMD_FUNC_TRACE();
+	ret = udp_tunnel_common_check(enic, tnl);
+	if (ret)
+		return ret;
+	/*
+	 * The NIC has 1 configurable VXLAN port number. "Adding" a new port
+	 * number replaces it.
+	 */
+	if (tnl->udp_port == enic->vxlan_port || tnl->udp_port == 0) {
+		PMD_INIT_LOG(DEBUG, " %u is already configured or invalid\n",
+			     tnl->udp_port);
+		return -EINVAL;
+	}
+	return update_vxlan_port(enic, tnl->udp_port);
+}
+
+static int enicpmd_dev_udp_tunnel_port_del(struct rte_eth_dev *eth_dev,
+					   struct rte_eth_udp_tunnel *tnl)
+{
+	struct enic *enic = pmd_priv(eth_dev);
+	int ret;
+
+	ENICPMD_FUNC_TRACE();
+	ret = udp_tunnel_common_check(enic, tnl);
+	if (ret)
+		return ret;
+	/*
+	 * Clear the previously set port number and restore the
+	 * hardware default port number. Some drivers disable VXLAN
+	 * offloads when there are no configured port numbers. But
+	 * enic does not do that as VXLAN is part of overlay offload,
+	 * which is tied to inner RSS and TSO.
+	 */
+	if (tnl->udp_port != enic->vxlan_port) {
+		PMD_INIT_LOG(DEBUG, " %u is not a configured vxlan port\n",
+			     tnl->udp_port);
+		return -EINVAL;
+	}
+	return update_vxlan_port(enic, ENIC_DEFAULT_VXLAN_PORT);
+}
+
 static const struct eth_dev_ops enicpmd_eth_dev_ops = {
 	.dev_configure        = enicpmd_dev_configure,
 	.dev_start            = enicpmd_dev_start,
@@ -812,6 +911,8 @@ static const struct eth_dev_ops enicpmd_eth_dev_ops = {
 	.reta_update          = enicpmd_dev_rss_reta_update,
 	.rss_hash_conf_get    = enicpmd_dev_rss_hash_conf_get,
 	.rss_hash_update      = enicpmd_dev_rss_hash_update,
+	.udp_tunnel_port_add  = enicpmd_dev_udp_tunnel_port_add,
+	.udp_tunnel_port_del  = enicpmd_dev_udp_tunnel_port_del,
 };
 
 static int enic_parse_disable_overlay(__rte_unused const char *key,
@@ -833,23 +934,61 @@ static int enic_parse_disable_overlay(__rte_unused const char *key,
 	return 0;
 }
 
+static int enic_parse_ig_vlan_rewrite(__rte_unused const char *key,
+				      const char *value,
+				      void *opaque)
+{
+	struct enic *enic;
+
+	enic = (struct enic *)opaque;
+	if (strcmp(value, "trunk") == 0) {
+		/* Trunk mode: always tag */
+		enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_DEFAULT_TRUNK;
+	} else if (strcmp(value, "untag") == 0) {
+		/* Untag default VLAN mode: untag if VLAN = default VLAN */
+		enic->ig_vlan_rewrite_mode =
+			IG_VLAN_REWRITE_MODE_UNTAG_DEFAULT_VLAN;
+	} else if (strcmp(value, "priority") == 0) {
+		/*
+		 * Priority-tag default VLAN mode: priority tag (VLAN header
+		 * with ID=0) if VLAN = default
+		 */
+		enic->ig_vlan_rewrite_mode =
+			IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN;
+	} else if (strcmp(value, "pass") == 0) {
+		/* Pass through mode: do not touch tags */
+		enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PASS_THRU;
+	} else {
+		dev_err(enic, "Invalid value for " ENIC_DEVARG_IG_VLAN_REWRITE
+			": expected=trunk|untag|priority|pass given=%s\n",
+			value);
+		return -EINVAL;
+	}
+	return 0;
+}
+
 static int enic_check_devargs(struct rte_eth_dev *dev)
 {
 	static const char *const valid_keys[] = {
-		ENIC_DEVARG_DISABLE_OVERLAY, NULL};
+		ENIC_DEVARG_DISABLE_OVERLAY,
+		ENIC_DEVARG_IG_VLAN_REWRITE,
+		NULL};
 	struct enic *enic = pmd_priv(dev);
 	struct rte_kvargs *kvlist;
 
 	ENICPMD_FUNC_TRACE();
 
 	enic->disable_overlay = false;
+	enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PASS_THRU;
 	if (!dev->device->devargs)
 		return 0;
 	kvlist = rte_kvargs_parse(dev->device->devargs->args, valid_keys);
 	if (!kvlist)
 		return -EINVAL;
 	if (rte_kvargs_process(kvlist, ENIC_DEVARG_DISABLE_OVERLAY,
-			       enic_parse_disable_overlay, enic) < 0) {
+			       enic_parse_disable_overlay, enic) < 0 ||
+	    rte_kvargs_process(kvlist, ENIC_DEVARG_IG_VLAN_REWRITE,
+			       enic_parse_ig_vlan_rewrite, enic) < 0) {
 		rte_kvargs_free(kvlist);
 		return -EINVAL;
 	}
@@ -914,4 +1053,5 @@ RTE_PMD_REGISTER_PCI(net_enic, rte_enic_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_enic, pci_id_enic_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_enic, "* igb_uio | uio_pci_generic | vfio-pci");
 RTE_PMD_REGISTER_PARAM_STRING(net_enic,
-			      ENIC_DEVARG_DISABLE_OVERLAY "=<0|1> ");
+	ENIC_DEVARG_DISABLE_OVERLAY "=0|1 "
+	ENIC_DEVARG_IG_VLAN_REWRITE "=trunk|untag|priority|pass");
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index a25d303d..fd940c58 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -69,12 +69,12 @@ enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
 	}
 }
 
-static void enic_free_wq_buf(struct vnic_wq_buf *buf)
+static void enic_free_wq_buf(struct rte_mbuf **buf)
 {
-	struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
+	struct rte_mbuf *mbuf = *buf;
 
 	rte_pktmbuf_free_seg(mbuf);
-	buf->mb = NULL;
+	*buf = NULL;
 }
 
 static void enic_log_q_error(struct enic *enic)
@@ -320,6 +320,8 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
 	 * enic_start_rq().
 	 */
 	rq->need_initial_post = true;
+	/* Initialize fetch index while RQ is disabled */
+	iowrite32(0, &rq->ctrl->fetch_index);
 	return 0;
 }
 
@@ -345,7 +347,6 @@ enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
 	dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
 		enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
 	iowrite32(rq->posted_index, &rq->ctrl->posted_index);
-	iowrite32(0, &rq->ctrl->fetch_index);
 	rte_rmb();
 	rq->need_initial_post = false;
 }
@@ -492,6 +493,42 @@ static void enic_rxq_intr_deinit(struct enic *enic)
 	}
 }
 
+static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
+{
+	struct wq_enet_desc *desc;
+	struct vnic_wq *wq;
+	unsigned int i;
+
+	/*
+	 * Fill WQ descriptor fields that never change. Every descriptor is
+	 * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
+	 * descriptors (i.e. request one completion update every 32 packets).
+	 */
+	wq = &enic->wq[queue_idx];
+	desc = (struct wq_enet_desc *)wq->ring.descs;
+	for (i = 0; i < wq->ring.desc_count; i++, desc++) {
+		desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
+		if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
+			desc->header_length_flags |=
+				(1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
+	}
+}
+
+static void pick_rx_handler(struct enic *enic)
+{
+	struct rte_eth_dev *eth_dev;
+
+	/* Use the non-scatter, simplified RX handler if possible. */
+	eth_dev = enic->rte_dev;
+	if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
+		PMD_INIT_LOG(DEBUG, " use the non-scatter Rx handler");
+		eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
+	} else {
+		PMD_INIT_LOG(DEBUG, " use the normal Rx handler");
+		eth_dev->rx_pkt_burst = &enic_recv_pkts;
+	}
+}
+
 int enic_enable(struct enic *enic)
 {
 	unsigned int index;
@@ -534,6 +571,22 @@ int enic_enable(struct enic *enic)
 		}
 	}
 
+	/*
+	 * Use the simple TX handler if possible. All offloads must be
+	 * disabled.
+	 */
+	if (eth_dev->data->dev_conf.txmode.offloads == 0) {
+		PMD_INIT_LOG(DEBUG, " use the simple tx handler");
+		eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
+		for (index = 0; index < enic->wq_count; index++)
+			enic_prep_wq_for_simple_tx(enic, index);
+	} else {
+		PMD_INIT_LOG(DEBUG, " use the default tx handler");
+		eth_dev->tx_pkt_burst = &enic_xmit_pkts;
+	}
+
+	pick_rx_handler(enic);
+
 	for (index = 0; index < enic->wq_count; index++)
 		enic_start_wq(enic, index);
 	for (index = 0; index < enic->rq_count; index++)
@@ -586,6 +639,19 @@ void enic_free_rq(void *rxq)
 	enic = vnic_dev_priv(rq_sop->vdev);
 	rq_data = &enic->rq[rq_sop->data_queue_idx];
 
+	if (rq_sop->free_mbufs) {
+		struct rte_mbuf **mb;
+		int i;
+
+		mb = rq_sop->free_mbufs;
+		for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
+		     i < ENIC_RX_BURST_MAX; i++)
+			rte_pktmbuf_free(mb[i]);
+		rte_free(rq_sop->free_mbufs);
+		rq_sop->free_mbufs = NULL;
+		rq_sop->num_free_mbufs = 0;
+	}
+
 	enic_rxmbuf_queue_release(enic, rq_sop);
 	if (rq_data->in_use)
 		enic_rxmbuf_queue_release(enic, rq_data);
@@ -742,20 +808,20 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 	}
 
 	/* number of descriptors have to be a multiple of 32 */
-	nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F;
-	nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F;
+	nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
+	nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
 
 	rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
 	rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
 
 	if (mbufs_per_pkt > 1) {
-		min_sop = 64;
+		min_sop = ENIC_RX_BURST_MAX;
 		max_sop = ((enic->config.rq_desc_count /
-			    (mbufs_per_pkt - 1)) & ~0x1F);
+			    (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
 		min_data = min_sop * (mbufs_per_pkt - 1);
 		max_data = enic->config.rq_desc_count;
 	} else {
-		min_sop = 64;
+		min_sop = ENIC_RX_BURST_MAX;
 		max_sop = enic->config.rq_desc_count;
 		min_data = 0;
 		max_data = 0;
@@ -826,10 +892,21 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
 			goto err_free_sop_mbuf;
 	}
 
+	rq_sop->free_mbufs = (struct rte_mbuf **)
+		rte_zmalloc_socket("rq->free_mbufs",
+				   sizeof(struct rte_mbuf *) *
+				   ENIC_RX_BURST_MAX,
+				   RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
+	if (rq_sop->free_mbufs == NULL)
+		goto err_free_data_mbuf;
+	rq_sop->num_free_mbufs = 0;
+
 	rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
 
 	return 0;
 
+err_free_data_mbuf:
+	rte_free(rq_data->mbuf_ring);
 err_free_sop_mbuf:
 	rte_free(rq_sop->mbuf_ring);
 err_free_cq:
@@ -869,25 +946,15 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 	static int instance;
 
 	wq->socket_id = socket_id;
-	if (nb_desc) {
-		if (nb_desc > enic->config.wq_desc_count) {
-			dev_warning(enic,
-				"WQ %d - number of tx desc in cmd line (%d)"\
-				"is greater than that in the UCSM/CIMC adapter"\
-				"policy.  Applying the value in the adapter "\
-				"policy (%d)\n",
-				queue_idx, nb_desc, enic->config.wq_desc_count);
-		} else if (nb_desc != enic->config.wq_desc_count) {
-			enic->config.wq_desc_count = nb_desc;
-			dev_info(enic,
-				"TX Queues - effective number of descs:%d\n",
-				nb_desc);
-		}
-	}
+	/*
+	 * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
+	 * print an info message for diagnostics.
+	 */
+	dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
 
 	/* Allocate queue resources */
 	err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
-		enic->config.wq_desc_count,
+		nb_desc,
 		sizeof(struct wq_enet_desc));
 	if (err) {
 		dev_err(enic, "error in allocation of wq\n");
@@ -895,7 +962,7 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 	}
 
 	err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
-		socket_id, enic->config.wq_desc_count,
+		socket_id, nb_desc,
 		sizeof(struct cq_enet_wq_desc));
 	if (err) {
 		vnic_wq_free(wq);
@@ -1197,7 +1264,7 @@ int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
 			rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
 		if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 			rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
-			if (ENIC_SETTING(enic, RSSHASH_UDP_WEAK)) {
+			if (enic->udp_rss_weak) {
 				/*
 				 * 'TCP' is not a typo. The "weak" version of
 				 * UDP RSS requires both the TCP and UDP bits
@@ -1213,7 +1280,7 @@ int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
 			rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
 		if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
 			rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
-			if (ENIC_SETTING(enic, RSSHASH_UDP_WEAK))
+			if (enic->udp_rss_weak)
 				rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
 		}
 	} else {
@@ -1237,8 +1304,11 @@ int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
 		enic->rss_hf = rss_hf;
 		enic->rss_hash_type = rss_hash_type;
 		enic->rss_enable = rss_enable;
+	} else {
+		dev_err(enic, "Failed to update RSS configurations."
+			" hash=0x%x\n", rss_hash_type);
 	}
-	return 0;
+	return ret;
 }
 
 int enic_set_vlan_strip(struct enic *enic)
@@ -1488,7 +1558,7 @@ int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
 
 	/* put back the real receive function */
 	rte_mb();
-	eth_dev->rx_pkt_burst = enic_recv_pkts;
+	pick_rx_handler(enic);
 	rte_mb();
 
 	/* restart Rx traffic */
@@ -1591,7 +1661,18 @@ static int enic_dev_init(struct enic *enic)
 			PKT_TX_OUTER_IP_CKSUM |
 			PKT_TX_TUNNEL_MASK;
 		enic->overlay_offload = true;
+		enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
 		dev_info(enic, "Overlay offload is enabled\n");
+		/*
+		 * Reset the vxlan port to the default, as the NIC firmware
+		 * does not reset it automatically and keeps the old setting.
+		 */
+		if (vnic_dev_overlay_offload_cfg(enic->vdev,
+						 OVERLAY_CFG_VXLAN_PORT_UPDATE,
+						 ENIC_DEFAULT_VXLAN_PORT)) {
+			dev_err(enic, "failed to update vxlan port\n");
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -1643,8 +1724,10 @@ int enic_probe(struct enic *enic)
 	}
 
 	/* Set ingress vlan rewrite mode before vnic initialization */
+	dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
+		  enic->ig_vlan_rewrite_mode);
 	err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
-		IG_VLAN_REWRITE_MODE_PASS_THRU);
+		enic->ig_vlan_rewrite_mode);
 	if (err) {
 		dev_err(enic,
 			"Failed to set ingress vlan rewrite mode, aborting.\n");
diff --git a/drivers/net/enic/enic_res.c b/drivers/net/enic/enic_res.c
index 6b404c3c..8d493ffe 100644
--- a/drivers/net/enic/enic_res.c
+++ b/drivers/net/enic/enic_res.c
@@ -82,6 +82,8 @@ int enic_get_vnic_config(struct enic *enic)
 			"Error getting filter modes, %d\n", err);
 		return err;
 	}
+	vnic_dev_capable_udp_rss_weak(enic->vdev, &enic->nic_cfg_chk,
+				      &enic->udp_rss_weak);
 
 	dev_info(enic, "Flow api filter mode: %s Actions: %s%s%s\n",
 		((enic->flow_filter_mode == FILTER_DPDK_1) ? "DPDK" :
@@ -124,7 +126,7 @@ int enic_get_vnic_config(struct enic *enic)
 		ENIC_SETTING(enic, RXCSUM) ? "yes" : "no",
 		ENIC_SETTING(enic, RSS) ?
 			(ENIC_SETTING(enic, RSSHASH_UDPIPV4) ? "+UDP" :
-			((ENIC_SETTING(enic, RSSHASH_UDP_WEAK) ? "+udp" :
+			((enic->udp_rss_weak ? "+udp" :
 			"yes"))) : "no",
 		c->intr_mode == VENET_INTR_MODE_INTX ? "INTx" :
 		c->intr_mode == VENET_INTR_MODE_MSI ? "MSI" :
@@ -161,7 +163,7 @@ int enic_get_vnic_config(struct enic *enic)
 	if (ENIC_SETTING(enic, RSSHASH_TCPIPV6))
 		enic->flow_type_rss_offloads |= ETH_RSS_NONFRAG_IPV6_TCP |
 			ETH_RSS_IPV6_TCP_EX;
-	if (ENIC_SETTING(enic, RSSHASH_UDP_WEAK))
+	if (enic->udp_rss_weak)
 		enic->flow_type_rss_offloads |=
 			ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP |
 			ETH_RSS_IPV6_UDP_EX;
@@ -181,7 +183,9 @@ int enic_get_vnic_config(struct enic *enic)
 	 * Default hardware capabilities. enic_dev_init() may add additional
 	 * flags if it enables overlay offloads.
 	 */
+	enic->tx_queue_offload_capa = 0;
 	enic->tx_offload_capa =
+		enic->tx_queue_offload_capa |
 		DEV_TX_OFFLOAD_MULTI_SEGS |
 		DEV_TX_OFFLOAD_VLAN_INSERT |
 		DEV_TX_OFFLOAD_IPV4_CKSUM |
@@ -235,6 +239,7 @@ int enic_set_nic_cfg(struct enic *enic, u8 rss_default_cpu, u8 rss_hash_type,
 	u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable, u8 tso_ipid_split_en,
 	u8 ig_vlan_strip_en)
 {
+	enum vnic_devcmd_cmd cmd;
 	u64 a0, a1;
 	u32 nic_cfg;
 	int wait = 1000;
@@ -245,8 +250,8 @@ int enic_set_nic_cfg(struct enic *enic, u8 rss_default_cpu, u8 rss_hash_type,
 
 	a0 = nic_cfg;
 	a1 = 0;
-
-	return vnic_dev_cmd(enic->vdev, CMD_NIC_CFG, &a0, &a1, wait);
+	cmd = enic->nic_cfg_chk ? CMD_NIC_CFG_CHK : CMD_NIC_CFG;
+	return vnic_dev_cmd(enic->vdev, cmd, &a0, &a1, wait);
 }
 
 int enic_set_rss_key(struct enic *enic, dma_addr_t key_pa, u64 len)
diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h
index e68f1307..3786bc0e 100644
--- a/drivers/net/enic/enic_res.h
+++ b/drivers/net/enic/enic_res.h
@@ -16,6 +16,13 @@
 #define ENIC_MIN_RQ_DESCS		64
 #define ENIC_MAX_RQ_DESCS		4096
 
+/* A descriptor ring has a multiple of 32 descriptors */
+#define ENIC_ALIGN_DESCS		32
+#define ENIC_ALIGN_DESCS_MASK		~(ENIC_ALIGN_DESCS - 1)
+
+/* Request a completion index every 32 buffers (roughly packets) */
+#define ENIC_WQ_CQ_THRESH		32
+
 #define ENIC_MIN_MTU			68
 
 /* Does not include (possible) inserted VLAN tag and FCS */
@@ -30,6 +37,15 @@
 #define ENIC_NON_TSO_MAX_DESC		16
 #define ENIC_DEFAULT_RX_FREE_THRESH	32
 #define ENIC_TX_XMIT_MAX		64
+#define ENIC_RX_BURST_MAX		64
+
+/* Defaults for dev_info.default_{rx,tx}portconf */
+#define ENIC_DEFAULT_RX_BURST		32
+#define ENIC_DEFAULT_RX_RINGS		1
+#define ENIC_DEFAULT_RX_RING_SIZE	512
+#define ENIC_DEFAULT_TX_BURST		32
+#define ENIC_DEFAULT_TX_RINGS		1
+#define ENIC_DEFAULT_TX_RING_SIZE	512
 
 #define ENIC_RSS_DEFAULT_CPU    0
 #define ENIC_RSS_BASE_CPU       0
diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c
index 8853a204..7129e121 100644
--- a/drivers/net/enic/enic_rxtx.c
+++ b/drivers/net/enic/enic_rxtx.c
@@ -137,51 +137,81 @@ enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd, uint8_t tnl)
 	 */
 	static const uint32_t cq_type_table[128] __rte_cache_aligned = {
 		[0x00] = RTE_PTYPE_UNKNOWN,
+		[0x01] = RTE_PTYPE_UNKNOWN |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER,
 		[0x20] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG,
 		[0x21] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG |
 			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
 			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 			 RTE_PTYPE_INNER_L4_NONFRAG,
 		[0x22] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
 		[0x23] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
 			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
 			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 			 RTE_PTYPE_INNER_L4_UDP,
 		[0x24] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
 		[0x25] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP |
 			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
 			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 			 RTE_PTYPE_INNER_L4_TCP,
 		[0x60] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
 		[0x61] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
 			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
 			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 			 RTE_PTYPE_INNER_L4_FRAG,
-		[0x62] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
-		[0x63] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
+		[0x62] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
+		[0x63] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
 			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
 			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_UDP,
-		[0x64] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
-		[0x65] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP |
+			 RTE_PTYPE_INNER_L4_FRAG,
+		[0x64] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
+		[0x65] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
 			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
 			 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-			 RTE_PTYPE_INNER_L4_TCP,
+			 RTE_PTYPE_INNER_L4_FRAG,
 		[0x10] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG,
 		[0x11] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG |
 			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
 			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 			 RTE_PTYPE_INNER_L4_NONFRAG,
 		[0x12] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
 		[0x13] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
 			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
 			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 			 RTE_PTYPE_INNER_L4_UDP,
 		[0x14] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
 		[0x15] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP |
 			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
 			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 			 RTE_PTYPE_INNER_L4_TCP,
+		[0x50] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
+		[0x51] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_FRAG,
+		[0x52] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
+		[0x53] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_FRAG,
+		[0x54] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
+		[0x55] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG |
+			 RTE_PTYPE_TUNNEL_GRENAT |
+			 RTE_PTYPE_INNER_L2_ETHER |
+			 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+			 RTE_PTYPE_INNER_L4_FRAG,
 		/* All others reserved */
 	};
 	cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
@@ -280,7 +310,7 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 	struct vnic_rq *rq;
 	struct enic *enic = vnic_dev_priv(sop_rq->vdev);
 	uint16_t cq_idx;
-	uint16_t rq_idx;
+	uint16_t rq_idx, max_rx;
 	uint16_t rq_num;
 	struct rte_mbuf *nmb, *rxmb;
 	uint16_t nb_rx = 0;
@@ -295,19 +325,23 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 	cq = &enic->cq[enic_cq_rq(enic, sop_rq->index)];
 	cq_idx = cq->to_clean;		/* index of cqd, rqd, mbuf_table */
 	cqd_ptr = (struct cq_desc *)(cq->ring.descs) + cq_idx;
+	color = cq->last_color;
 
 	data_rq = &enic->rq[sop_rq->data_queue_idx];
 
-	while (nb_rx < nb_pkts) {
+	/* Receive until the end of the ring, at most. */
+	max_rx = RTE_MIN(nb_pkts, cq->ring.desc_count - cq_idx);
+
+	while (max_rx) {
 		volatile struct rq_enet_desc *rqd_ptr;
 		struct cq_desc cqd;
 		uint8_t packet_error;
 		uint16_t ciflags;
 
+		max_rx--;
+
 		/* Check for pkts available */
-		color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
-			& CQ_DESC_COLOR_MASK;
-		if (color == cq->last_color)
+		if ((cqd_ptr->type_color & CQ_DESC_COLOR_MASK_NOSHIFT) == color)
 			break;
 
 		/* Get the cq descriptor and extract rq info from it */
@@ -331,13 +365,7 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		/* Get the mbuf to return and replace with one just allocated */
 		rxmb = rq->mbuf_ring[rq_idx];
 		rq->mbuf_ring[rq_idx] = nmb;
-
-		/* Increment cqd, rqd, mbuf_table index */
 		cq_idx++;
-		if (unlikely(cq_idx == cq->ring.desc_count)) {
-			cq_idx = 0;
-			cq->last_color = cq->last_color ? 0 : 1;
-		}
 
 		/* Prefetch next mbuf & desc while processing current one */
 		cqd_ptr = (struct cq_desc *)(cq->ring.descs) + cq_idx;
@@ -389,6 +417,7 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		first_seg->packet_type =
 			enic_cq_rx_flags_to_pkt_type(&cqd, tnl);
 		enic_cq_rx_to_pkt_flags(&cqd, first_seg);
+
 		/* Wipe the outer types set by enic_cq_rx_flags_to_pkt_type() */
 		if (tnl) {
 			first_seg->packet_type &= ~(RTE_PTYPE_L3_MASK |
@@ -408,6 +437,10 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		/* store the mbuf address into the next entry of the array */
 		rx_pkts[nb_rx++] = first_seg;
 	}
+	if (unlikely(cq_idx == cq->ring.desc_count)) {
+		cq_idx = 0;
+		cq->last_color ^= CQ_DESC_COLOR_MASK_NOSHIFT;
+	}
 
 	sop_rq->pkt_first_seg = first_seg;
 	sop_rq->pkt_last_seg = last_seg;
@@ -441,9 +474,123 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 	return nb_rx;
 }
 
+uint16_t
+enic_noscatter_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+			 uint16_t nb_pkts)
+{
+	struct rte_mbuf *mb, **rx, **rxmb;
+	uint16_t cq_idx, nb_rx, max_rx;
+	struct cq_enet_rq_desc *cqd;
+	struct rq_enet_desc *rqd;
+	unsigned int port_id;
+	struct vnic_cq *cq;
+	struct vnic_rq *rq;
+	struct enic *enic;
+	uint8_t color;
+	bool overlay;
+	bool tnl;
+
+	rq = rx_queue;
+	enic = vnic_dev_priv(rq->vdev);
+	cq = &enic->cq[enic_cq_rq(enic, rq->index)];
+	cq_idx = cq->to_clean;
+
+	/*
+	 * Fill up the reserve of free mbufs. Below, we restock the receive
+	 * ring with these mbufs to avoid allocation failures.
+	 */
+	if (rq->num_free_mbufs == 0) {
+		if (rte_mempool_get_bulk(rq->mp, (void **)rq->free_mbufs,
+					 ENIC_RX_BURST_MAX))
+			return 0;
+		rq->num_free_mbufs = ENIC_RX_BURST_MAX;
+	}
+
+	/* Receive until the end of the ring, at most. */
+	max_rx = RTE_MIN(nb_pkts, rq->num_free_mbufs);
+	max_rx = RTE_MIN(max_rx, cq->ring.desc_count - cq_idx);
+
+	cqd = (struct cq_enet_rq_desc *)(cq->ring.descs) + cq_idx;
+	color = cq->last_color;
+	rxmb = rq->mbuf_ring + cq_idx;
+	port_id = enic->port_id;
+	overlay = enic->overlay_offload;
+
+	rx = rx_pkts;
+	while (max_rx) {
+		max_rx--;
+		if ((cqd->type_color & CQ_DESC_COLOR_MASK_NOSHIFT) == color)
+			break;
+		if (unlikely(cqd->bytes_written_flags &
+			     CQ_ENET_RQ_DESC_FLAGS_TRUNCATED)) {
+			rte_pktmbuf_free(*rxmb++);
+			rte_atomic64_inc(&enic->soft_stats.rx_packet_errors);
+			cqd++;
+			continue;
+		}
+
+		mb = *rxmb++;
+		/* prefetch mbuf data for caller */
+		rte_packet_prefetch(RTE_PTR_ADD(mb->buf_addr,
+				    RTE_PKTMBUF_HEADROOM));
+		mb->data_len = cqd->bytes_written_flags &
+			CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
+		mb->pkt_len = mb->data_len;
+		mb->port = port_id;
+		tnl = overlay && (cqd->completed_index_flags &
+				  CQ_ENET_RQ_DESC_FLAGS_FCOE) != 0;
+		mb->packet_type =
+			enic_cq_rx_flags_to_pkt_type((struct cq_desc *)cqd,
+						     tnl);
+		enic_cq_rx_to_pkt_flags((struct cq_desc *)cqd, mb);
+		/* Wipe the outer types set by enic_cq_rx_flags_to_pkt_type() */
+		if (tnl) {
+			mb->packet_type &= ~(RTE_PTYPE_L3_MASK |
+					     RTE_PTYPE_L4_MASK);
+		}
+		cqd++;
+		*rx++ = mb;
+	}
+	/* Number of descriptors visited */
+	nb_rx = cqd - (struct cq_enet_rq_desc *)(cq->ring.descs) - cq_idx;
+	if (nb_rx == 0)
+		return 0;
+	rqd = ((struct rq_enet_desc *)rq->ring.descs) + cq_idx;
+	rxmb = rq->mbuf_ring + cq_idx;
+	cq_idx += nb_rx;
+	rq->rx_nb_hold += nb_rx;
+	if (unlikely(cq_idx == cq->ring.desc_count)) {
+		cq_idx = 0;
+		cq->last_color ^= CQ_DESC_COLOR_MASK_NOSHIFT;
+	}
+	cq->to_clean = cq_idx;
+
+	memcpy(rxmb, rq->free_mbufs + ENIC_RX_BURST_MAX - rq->num_free_mbufs,
+	       sizeof(struct rte_mbuf *) * nb_rx);
+	rq->num_free_mbufs -= nb_rx;
+	while (nb_rx) {
+		nb_rx--;
+		mb = *rxmb++;
+		mb->data_off = RTE_PKTMBUF_HEADROOM;
+		rqd->address = mb->buf_iova + RTE_PKTMBUF_HEADROOM;
+		rqd++;
+	}
+	if (rq->rx_nb_hold > rq->rx_free_thresh) {
+		rq->posted_index = enic_ring_add(rq->ring.desc_count,
+						 rq->posted_index,
+						 rq->rx_nb_hold);
+		rq->rx_nb_hold = 0;
+		rte_wmb();
+		iowrite32_relaxed(rq->posted_index,
+				  &rq->ctrl->posted_index);
+	}
+
+	return rx - rx_pkts;
+}
+
 static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index)
 {
-	struct vnic_wq_buf *buf;
+	struct rte_mbuf *buf;
 	struct rte_mbuf *m, *free[ENIC_MAX_WQ_DESCS];
 	unsigned int nb_to_free, nb_free = 0, i;
 	struct rte_mempool *pool;
@@ -453,13 +600,10 @@ static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index)
 	nb_to_free = enic_ring_sub(desc_count, wq->tail_idx, completed_index)
 				   + 1;
 	tail_idx = wq->tail_idx;
-	buf = &wq->bufs[tail_idx];
-	pool = ((struct rte_mbuf *)buf->mb)->pool;
+	pool = wq->bufs[tail_idx]->pool;
 	for (i = 0; i < nb_to_free; i++) {
-		buf = &wq->bufs[tail_idx];
-		m = rte_pktmbuf_prefree_seg((struct rte_mbuf *)(buf->mb));
-		buf->mb = NULL;
-
+		buf = wq->bufs[tail_idx];
+		m = rte_pktmbuf_prefree_seg(buf);
 		if (unlikely(m == NULL)) {
 			tail_idx = enic_ring_incr(desc_count, tail_idx);
 			continue;
@@ -508,6 +652,10 @@ uint16_t enic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 	for (i = 0; i != nb_pkts; i++) {
 		m = tx_pkts[i];
+		if (unlikely(m->pkt_len > ENIC_TX_MAX_PKT_SIZE)) {
+			rte_errno = EINVAL;
+			return i;
+		}
 		ol_flags = m->ol_flags;
 		if (ol_flags & wq->tx_offload_notsup_mask) {
 			rte_errno = ENOTSUP;
@@ -544,12 +692,11 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	uint64_t ol_flags_mask;
 	unsigned int wq_desc_avail;
 	int head_idx;
-	struct vnic_wq_buf *buf;
 	unsigned int desc_count;
 	struct wq_enet_desc *descs, *desc_p, desc_tmp;
 	uint16_t mss;
 	uint8_t vlan_tag_insert;
-	uint8_t eop;
+	uint8_t eop, cq;
 	uint64_t bus_addr;
 	uint8_t offload_mode;
 	uint16_t header_len;
@@ -632,15 +779,18 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 				break;
 			}
 		}
-
-
+		wq->cq_pend++;
+		cq = 0;
+		if (eop && wq->cq_pend >= ENIC_WQ_CQ_THRESH) {
+			cq = 1;
+			wq->cq_pend = 0;
+		}
 		wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, header_len,
-				 offload_mode, eop, eop, 0, vlan_tag_insert,
+				 offload_mode, eop, cq, 0, vlan_tag_insert,
 				 vlan_id, 0);
 
 		*desc_p = desc_tmp;
-		buf = &wq->bufs[head_idx];
-		buf->mb = (void *)tx_pkt;
+		wq->bufs[head_idx] = tx_pkt;
 		head_idx = enic_ring_incr(desc_count, head_idx);
 		wq_desc_avail--;
 
@@ -649,20 +799,26 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 			    tx_pkt->next) {
 				data_len = tx_pkt->data_len;
 
-				if (tx_pkt->next == NULL)
+				wq->cq_pend++;
+				cq = 0;
+				if (tx_pkt->next == NULL) {
 					eop = 1;
+					if (wq->cq_pend >= ENIC_WQ_CQ_THRESH) {
+						cq = 1;
+						wq->cq_pend = 0;
+					}
+				}
 				desc_p = descs + head_idx;
 				bus_addr = (dma_addr_t)(tx_pkt->buf_iova
 					   + tx_pkt->data_off);
 				wq_enet_desc_enc((struct wq_enet_desc *)
 						 &desc_tmp, bus_addr, data_len,
-						 mss, 0, offload_mode, eop, eop,
+						 mss, 0, offload_mode, eop, cq,
 						 0, vlan_tag_insert, vlan_id,
 						 0);
 
 				*desc_p = desc_tmp;
-				buf = &wq->bufs[head_idx];
-				buf->mb = (void *)tx_pkt;
+				wq->bufs[head_idx] = tx_pkt;
 				head_idx = enic_ring_incr(desc_count, head_idx);
 				wq_desc_avail--;
 			}
@@ -678,4 +834,81 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	return index;
 }
 
+static void enqueue_simple_pkts(struct rte_mbuf **pkts,
+				struct wq_enet_desc *desc,
+				uint16_t n,
+				struct enic *enic)
+{
+	struct rte_mbuf *p;
+
+	while (n) {
+		n--;
+		p = *pkts++;
+		desc->address = p->buf_iova + p->data_off;
+		desc->length = p->pkt_len;
+		/*
+		 * The app should not send oversized
+		 * packets. tx_pkt_prepare includes a check as
+		 * well. But some apps ignore the device max size and
+		 * tx_pkt_prepare. Oversized packets cause WQ errrors
+		 * and the NIC ends up disabling the whole WQ. So
+		 * truncate packets..
+		 */
+		if (unlikely(p->pkt_len > ENIC_TX_MAX_PKT_SIZE)) {
+			desc->length = ENIC_TX_MAX_PKT_SIZE;
+			rte_atomic64_inc(&enic->soft_stats.tx_oversized);
+		}
+		desc++;
+	}
+}
+
+uint16_t enic_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+			       uint16_t nb_pkts)
+{
+	unsigned int head_idx, desc_count;
+	struct wq_enet_desc *desc;
+	struct vnic_wq *wq;
+	struct enic *enic;
+	uint16_t rem, n;
+
+	wq = (struct vnic_wq *)tx_queue;
+	enic = vnic_dev_priv(wq->vdev);
+	enic_cleanup_wq(enic, wq);
+	/* Will enqueue this many packets in this call */
+	nb_pkts = RTE_MIN(nb_pkts, wq->ring.desc_avail);
+	if (nb_pkts == 0)
+		return 0;
+
+	head_idx = wq->head_idx;
+	desc_count = wq->ring.desc_count;
+
+	/* Descriptors until the end of the ring */
+	n = desc_count - head_idx;
+	n = RTE_MIN(nb_pkts, n);
+
+	/* Save mbuf pointers to free later */
+	memcpy(wq->bufs + head_idx, tx_pkts, sizeof(struct rte_mbuf *) * n);
+
+	/* Enqueue until the ring end */
+	rem = nb_pkts - n;
+	desc = ((struct wq_enet_desc *)wq->ring.descs) + head_idx;
+	enqueue_simple_pkts(tx_pkts, desc, n, enic);
+
+	/* Wrap to the start of the ring */
+	if (rem) {
+		tx_pkts += n;
+		memcpy(wq->bufs, tx_pkts, sizeof(struct rte_mbuf *) * rem);
+		desc = (struct wq_enet_desc *)wq->ring.descs;
+		enqueue_simple_pkts(tx_pkts, desc, rem, enic);
+	}
+	rte_wmb();
 
+	/* Update head_idx and desc_avail */
+	wq->ring.desc_avail -= nb_pkts;
+	head_idx += nb_pkts;
+	if (head_idx >= desc_count)
+		head_idx -= desc_count;
+	wq->head_idx = head_idx;
+	iowrite32_relaxed(head_idx, &wq->ctrl->posted_index);
+	return nb_pkts;
+}
diff --git a/drivers/net/failsafe/failsafe.c b/drivers/net/failsafe/failsafe.c
index eafbb75d..657919f9 100644
--- a/drivers/net/failsafe/failsafe.c
+++ b/drivers/net/failsafe/failsafe.c
@@ -328,6 +328,7 @@ rte_pmd_failsafe_probe(struct rte_vdev_device *vdev)
 		}
 		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &failsafe_ops;
+		eth_dev->device = &vdev->device;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -353,9 +354,7 @@ static struct rte_vdev_driver failsafe_drv = {
 RTE_PMD_REGISTER_VDEV(net_failsafe, failsafe_drv);
 RTE_PMD_REGISTER_PARAM_STRING(net_failsafe, PMD_FAILSAFE_PARAM_STRING);
 
-RTE_INIT(failsafe_init_log);
-static void
-failsafe_init_log(void)
+RTE_INIT(failsafe_init_log)
 {
 	failsafe_logtype = rte_log_register("pmd.net.failsafe");
 	if (failsafe_logtype >= 0)
diff --git a/drivers/net/failsafe/failsafe_args.c b/drivers/net/failsafe/failsafe_args.c
index 2c002b16..626883ce 100644
--- a/drivers/net/failsafe/failsafe_args.c
+++ b/drivers/net/failsafe/failsafe_args.c
@@ -63,7 +63,7 @@ fs_parse_device(struct sub_device *sdev, char *args)
 
 	d = &sdev->devargs;
 	DEBUG("%s", args);
-	ret = rte_devargs_parse(d, "%s", args);
+	ret = rte_devargs_parse(d, args);
 	if (ret) {
 		DEBUG("devargs parsing failed with code %d", ret);
 		return ret;
diff --git a/drivers/net/failsafe/failsafe_eal.c b/drivers/net/failsafe/failsafe_eal.c
index 5672f396..ce1633f1 100644
--- a/drivers/net/failsafe/failsafe_eal.c
+++ b/drivers/net/failsafe/failsafe_eal.c
@@ -86,7 +86,7 @@ fs_bus_init(struct rte_eth_dev *dev)
 			else
 				snprintf(devstr, sizeof(devstr), "%s",
 					 rte_eth_devices[pid].device->name);
-			ret = rte_devargs_parse(da, "%s", devstr);
+			ret = rte_devargs_parse(da, devstr);
 			if (ret) {
 				ERROR("Probed devargs parsing failed with code"
 				      " %d", ret);
diff --git a/drivers/net/failsafe/meson.build b/drivers/net/failsafe/meson.build
new file mode 100644
index 00000000..a249ff4a
--- /dev/null
+++ b/drivers/net/failsafe/meson.build
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+cflags += '-std=gnu99'
+cflags += '-D_DEFAULT_SOURCE'
+cflags += '-D_XOPEN_SOURCE=700'
+cflags += '-pedantic'
+if host_machine.system() == 'linux'
+	cflags += '-DLINUX'
+else
+	cflags += '-DBSD'
+endif
+
+allow_experimental_apis = true
+
+sources = files('failsafe_args.c',
+	'failsafe.c',
+	'failsafe_eal.c',
+	'failsafe_ether.c',
+	'failsafe_flow.c',
+	'failsafe_intr.c',
+	'failsafe_ops.c',
+	'failsafe_rxtx.c')
diff --git a/drivers/net/fm10k/fm10k.h b/drivers/net/fm10k/fm10k.h
index ef307809..dc814855 100644
--- a/drivers/net/fm10k/fm10k.h
+++ b/drivers/net/fm10k/fm10k.h
@@ -106,9 +106,6 @@
 #define FM10K_MISC_VEC_ID               RTE_INTR_VEC_ZERO_OFFSET
 #define FM10K_RX_VEC_START              RTE_INTR_VEC_RXTX_OFFSET
 
-#define FM10K_SIMPLE_TX_FLAG ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
-				ETH_TXQ_FLAGS_NOOFFLOADS)
-
 struct fm10k_macvlan_filter_info {
 	uint16_t vlan_num;       /* Total VLAN number */
 	uint16_t mac_num;        /* Total mac number */
@@ -329,6 +326,13 @@ uint16_t fm10k_recv_scattered_pkts(void *rx_queue,
 int
 fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
 
+int
+fm10k_dev_rx_descriptor_status(void *rx_queue, uint16_t offset);
+
+int
+fm10k_dev_tx_descriptor_status(void *rx_queue, uint16_t offset);
+
+
 uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	uint16_t nb_pkts);
 
diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c
index 3ff1b0e0..541a49b7 100644
--- a/drivers/net/fm10k/fm10k_ethdev.c
+++ b/drivers/net/fm10k/fm10k_ethdev.c
@@ -451,8 +451,10 @@ fm10k_dev_configure(struct rte_eth_dev *dev)
 
 	PMD_INIT_FUNC_TRACE();
 
-	if ((dev->data->dev_conf.rxmode.offloads &
-	     DEV_RX_OFFLOAD_CRC_STRIP) == 0)
+	/* KEEP_CRC offload flag is not supported by PMD
+	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
+	 */
+	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
 		PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 
 	/* multipe queue mode checking */
@@ -808,52 +810,50 @@ static int
 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 {
 	struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	int err = -1;
+	int err;
 	uint32_t reg;
 	struct fm10k_rx_queue *rxq;
 
 	PMD_INIT_FUNC_TRACE();
 
-	if (rx_queue_id < dev->data->nb_rx_queues) {
-		rxq = dev->data->rx_queues[rx_queue_id];
-		err = rx_queue_reset(rxq);
-		if (err == -ENOMEM) {
-			PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
-			return err;
-		} else if (err == -EINVAL) {
-			PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
-				" %d", err);
-			return err;
-		}
+	rxq = dev->data->rx_queues[rx_queue_id];
+	err = rx_queue_reset(rxq);
+	if (err == -ENOMEM) {
+		PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
+		return err;
+	} else if (err == -EINVAL) {
+		PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
+			" %d", err);
+		return err;
+	}
 
-		/* Setup the HW Rx Head and Tail Descriptor Pointers
-		 * Note: this must be done AFTER the queue is enabled on real
-		 * hardware, but BEFORE the queue is enabled when using the
-		 * emulation platform. Do it in both places for now and remove
-		 * this comment and the following two register writes when the
-		 * emulation platform is no longer being used.
-		 */
-		FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
-		FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
+	/* Setup the HW Rx Head and Tail Descriptor Pointers
+	 * Note: this must be done AFTER the queue is enabled on real
+	 * hardware, but BEFORE the queue is enabled when using the
+	 * emulation platform. Do it in both places for now and remove
+	 * this comment and the following two register writes when the
+	 * emulation platform is no longer being used.
+	 */
+	FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
+	FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 
-		/* Set PF ownership flag for PF devices */
-		reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
-		if (hw->mac.type == fm10k_mac_pf)
-			reg |= FM10K_RXQCTL_PF;
-		reg |= FM10K_RXQCTL_ENABLE;
-		/* enable RX queue */
-		FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
-		FM10K_WRITE_FLUSH(hw);
+	/* Set PF ownership flag for PF devices */
+	reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
+	if (hw->mac.type == fm10k_mac_pf)
+		reg |= FM10K_RXQCTL_PF;
+	reg |= FM10K_RXQCTL_ENABLE;
+	/* enable RX queue */
+	FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
+	FM10K_WRITE_FLUSH(hw);
 
-		/* Setup the HW Rx Head and Tail Descriptor Pointers
-		 * Note: this must be done AFTER the queue is enabled
-		 */
-		FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
-		FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
-		dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
-	}
+	/* Setup the HW Rx Head and Tail Descriptor Pointers
+	 * Note: this must be done AFTER the queue is enabled
+	 */
+	FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
+	FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 
-	return err;
+	return 0;
 }
 
 static int
@@ -863,14 +863,12 @@ fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 
 	PMD_INIT_FUNC_TRACE();
 
-	if (rx_queue_id < dev->data->nb_rx_queues) {
-		/* Disable RX queue */
-		rx_queue_disable(hw, rx_queue_id);
+	/* Disable RX queue */
+	rx_queue_disable(hw, rx_queue_id);
 
-		/* Free mbuf and clean HW ring */
-		rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
-		dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
-	}
+	/* Free mbuf and clean HW ring */
+	rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 
 	return 0;
 }
@@ -882,28 +880,23 @@ fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	/** @todo - this should be defined in the shared code */
 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY	0x00010000
 	uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
-	int err = 0;
+	struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 
 	PMD_INIT_FUNC_TRACE();
 
-	if (tx_queue_id < dev->data->nb_tx_queues) {
-		struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
-
-		q->ops->reset(q);
+	q->ops->reset(q);
 
-		/* reset head and tail pointers */
-		FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
-		FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
+	/* reset head and tail pointers */
+	FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
+	FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 
-		/* enable TX queue */
-		FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
-					FM10K_TXDCTL_ENABLE | txdctl);
-		FM10K_WRITE_FLUSH(hw);
-		dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
-	} else
-		err = -1;
+	/* enable TX queue */
+	FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
+				FM10K_TXDCTL_ENABLE | txdctl);
+	FM10K_WRITE_FLUSH(hw);
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 
-	return err;
+	return 0;
 }
 
 static int
@@ -913,11 +906,9 @@ fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 
 	PMD_INIT_FUNC_TRACE();
 
-	if (tx_queue_id < dev->data->nb_tx_queues) {
-		tx_queue_disable(hw, tx_queue_id);
-		tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
-		dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
-	}
+	tx_queue_disable(hw, tx_queue_id);
+	tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 
 	return 0;
 }
@@ -2837,6 +2828,8 @@ static const struct eth_dev_ops fm10k_eth_dev_ops = {
 	.tx_queue_setup		= fm10k_tx_queue_setup,
 	.tx_queue_release	= fm10k_tx_queue_release,
 	.rx_descriptor_done	= fm10k_dev_rx_descriptor_done,
+	.rx_descriptor_status = fm10k_dev_rx_descriptor_status,
+	.tx_descriptor_status = fm10k_dev_tx_descriptor_status,
 	.rx_queue_intr_enable	= fm10k_dev_rx_queue_intr_enable,
 	.rx_queue_intr_disable	= fm10k_dev_rx_queue_intr_disable,
 	.reta_update		= fm10k_reta_update,
@@ -3290,9 +3283,7 @@ RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
 
-RTE_INIT(fm10k_init_log);
-static void
-fm10k_init_log(void)
+RTE_INIT(fm10k_init_log)
 {
 	fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
 	if (fm10k_logtype_init >= 0)
diff --git a/drivers/net/fm10k/fm10k_rxtx.c b/drivers/net/fm10k/fm10k_rxtx.c
index 9320748c..4a5b46ec 100644
--- a/drivers/net/fm10k/fm10k_rxtx.c
+++ b/drivers/net/fm10k/fm10k_rxtx.c
@@ -389,6 +389,84 @@ fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
 	return ret;
 }
 
+int
+fm10k_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
+{
+	volatile union fm10k_rx_desc *rxdp;
+	struct fm10k_rx_queue *rxq = rx_queue;
+	uint16_t nb_hold, trigger_last;
+	uint16_t desc;
+	int ret;
+
+	if (unlikely(offset >= rxq->nb_desc)) {
+		PMD_DRV_LOG(ERR, "Invalid RX descriptor offset %u", offset);
+		return 0;
+	}
+
+	if (rxq->next_trigger < rxq->alloc_thresh)
+		trigger_last = rxq->next_trigger +
+					rxq->nb_desc - rxq->alloc_thresh;
+	else
+		trigger_last = rxq->next_trigger - rxq->alloc_thresh;
+
+	if (rxq->next_dd < trigger_last)
+		nb_hold = rxq->next_dd + rxq->nb_desc - trigger_last;
+	else
+		nb_hold = rxq->next_dd - trigger_last;
+
+	if (offset >= rxq->nb_desc - nb_hold)
+		return RTE_ETH_RX_DESC_UNAVAIL;
+
+	desc = rxq->next_dd + offset;
+	if (desc >= rxq->nb_desc)
+		desc -= rxq->nb_desc;
+
+	rxdp = &rxq->hw_ring[desc];
+
+	ret = !!(rxdp->w.status &
+			rte_cpu_to_le_16(FM10K_RXD_STATUS_DD));
+
+	return ret;
+}
+
+int
+fm10k_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
+{
+	volatile struct fm10k_tx_desc *txdp;
+	struct fm10k_tx_queue *txq = tx_queue;
+	uint16_t desc;
+	uint16_t next_rs = txq->nb_desc;
+	struct fifo rs_tracker = txq->rs_tracker;
+	struct fifo *r = &rs_tracker;
+
+	if (unlikely(offset >= txq->nb_desc))
+		return -EINVAL;
+
+	desc = txq->next_free + offset;
+	/* go to next desc that has the RS bit */
+	desc = (desc / txq->rs_thresh + 1) *
+		txq->rs_thresh - 1;
+
+	if (desc >= txq->nb_desc) {
+		desc -= txq->nb_desc;
+		if (desc >= txq->nb_desc)
+			desc -= txq->nb_desc;
+	}
+
+	r->head = r->list;
+	for ( ; r->head != r->endp; ) {
+		if (*r->head >= desc && *r->head < next_rs)
+			next_rs = *r->head;
+		++r->head;
+	}
+
+	txdp = &txq->hw_ring[next_rs];
+	if (txdp->flags & FM10K_TXD_FLAG_DONE)
+		return RTE_ETH_TX_DESC_DONE;
+
+	return RTE_ETH_TX_DESC_FULL;
+}
+
 /*
  * Free multiple TX mbuf at a time if they are in the same pool
  *
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 13c5d329..85a6a867 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -42,6 +42,8 @@
 
 #define ETH_I40E_FLOATING_VEB_ARG	"enable_floating_veb"
 #define ETH_I40E_FLOATING_VEB_LIST_ARG	"floating_veb_list"
+#define ETH_I40E_SUPPORT_MULTI_DRIVER	"support-multi-driver"
+#define ETH_I40E_QUEUE_NUM_PER_VF_ARG	"queue-num-per-vf"
 
 #define I40E_CLEAR_PXE_WAIT_MS     200
 
@@ -401,6 +403,13 @@ static void i40e_notify_all_vfs_link_status(struct rte_eth_dev *dev);
 int i40e_logtype_init;
 int i40e_logtype_driver;
 
+static const char *const valid_keys[] = {
+	ETH_I40E_FLOATING_VEB_ARG,
+	ETH_I40E_FLOATING_VEB_LIST_ARG,
+	ETH_I40E_SUPPORT_MULTI_DRIVER,
+	ETH_I40E_QUEUE_NUM_PER_VF_ARG,
+	NULL};
+
 static const struct rte_pci_id pci_id_i40e_map[] = {
 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710) },
 	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QEMU) },
@@ -698,12 +707,16 @@ i40e_write_global_rx_ctl(struct i40e_hw *hw, uint32_t reg_addr,
 			 uint32_t reg_val)
 {
 	uint32_t ori_reg_val;
+	struct rte_eth_dev *dev;
 
 	ori_reg_val = i40e_read_rx_ctl(hw, reg_addr);
+	dev = ((struct i40e_adapter *)hw->back)->eth_dev;
 	i40e_write_rx_ctl(hw, reg_addr, reg_val);
-	PMD_DRV_LOG(DEBUG,
-		    "Global register [0x%08x] original: 0x%08x, after: 0x%08x",
-		    reg_addr, ori_reg_val, reg_val);
+	if (ori_reg_val != reg_val)
+		PMD_DRV_LOG(WARNING,
+			    "i40e device %s changed global register [0x%08x]."
+			    " original: 0x%08x, new: 0x%08x",
+			    dev->device->name, reg_addr, ori_reg_val, reg_val);
 }
 
 RTE_PMD_REGISTER_PCI(net_i40e, rte_i40e_pmd);
@@ -730,7 +743,6 @@ static inline void i40e_GLQF_reg_init(struct i40e_hw *hw)
 	 */
 	I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(40), 0x00000029);
 	I40E_WRITE_GLB_REG(hw, I40E_GLQF_PIT(9), 0x00009420);
-	i40e_global_cfg_warning(I40E_WARNING_QINQ_PARSER);
 }
 
 static inline void i40e_config_automask(struct i40e_pf *pf)
@@ -849,7 +861,7 @@ config_vf_floating_veb(struct rte_devargs *devargs,
 	if (devargs == NULL)
 		return;
 
-	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	kvlist = rte_kvargs_parse(devargs->args, valid_keys);
 	if (kvlist == NULL)
 		return;
 
@@ -890,7 +902,7 @@ is_floating_veb_supported(struct rte_devargs *devargs)
 	if (devargs == NULL)
 		return 0;
 
-	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	kvlist = rte_kvargs_parse(devargs->args, valid_keys);
 	if (kvlist == NULL)
 		return 0;
 
@@ -1097,8 +1109,6 @@ i40e_init_queue_region_conf(struct rte_eth_dev *dev)
 	memset(info, 0, sizeof(struct i40e_queue_regions));
 }
 
-#define ETH_I40E_SUPPORT_MULTI_DRIVER	"support-multi-driver"
-
 static int
 i40e_parse_multi_drv_handler(__rte_unused const char *key,
 			       const char *value,
@@ -1130,9 +1140,8 @@ static int
 i40e_support_multi_driver(struct rte_eth_dev *dev)
 {
 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
-	static const char *const valid_keys[] = {
-		ETH_I40E_SUPPORT_MULTI_DRIVER, NULL};
 	struct rte_kvargs *kvlist;
+	int kvargs_count;
 
 	/* Enable global configuration by default */
 	pf->support_multi_driver = false;
@@ -1144,7 +1153,13 @@ i40e_support_multi_driver(struct rte_eth_dev *dev)
 	if (!kvlist)
 		return -EINVAL;
 
-	if (rte_kvargs_count(kvlist, ETH_I40E_SUPPORT_MULTI_DRIVER) > 1)
+	kvargs_count = rte_kvargs_count(kvlist, ETH_I40E_SUPPORT_MULTI_DRIVER);
+	if (!kvargs_count) {
+		rte_kvargs_free(kvlist);
+		return 0;
+	}
+
+	if (kvargs_count > 1)
 		PMD_DRV_LOG(WARNING, "More than one argument \"%s\" and only "
 			    "the first invalid or last valid one is used !",
 			    ETH_I40E_SUPPORT_MULTI_DRIVER);
@@ -1165,6 +1180,7 @@ i40e_aq_debug_write_global_register(struct i40e_hw *hw,
 				    struct i40e_asq_cmd_details *cmd_details)
 {
 	uint64_t ori_reg_val;
+	struct rte_eth_dev *dev;
 	int ret;
 
 	ret = i40e_aq_debug_read_register(hw, reg_addr, &ori_reg_val, NULL);
@@ -1174,11 +1190,13 @@ i40e_aq_debug_write_global_register(struct i40e_hw *hw,
 			    reg_addr);
 		return -EIO;
 	}
+	dev = ((struct i40e_adapter *)hw->back)->eth_dev;
 
-	PMD_DRV_LOG(DEBUG,
-		    "Global register [0x%08x] original: 0x%"PRIx64
-		    ", after: 0x%"PRIx64,
-		    reg_addr, ori_reg_val, reg_val);
+	if (ori_reg_val != reg_val)
+		PMD_DRV_LOG(WARNING,
+			    "i40e device %s changed global register [0x%08x]."
+			    " original: 0x%"PRIx64", after: 0x%"PRIx64,
+			    dev->device->name, reg_addr, ori_reg_val, reg_val);
 
 	return i40e_aq_debug_write_register(hw, reg_addr, reg_val, cmd_details);
 }
@@ -1236,6 +1254,13 @@ eth_i40e_dev_init(struct rte_eth_dev *dev, void *init_params __rte_unused)
 	hw->bus.func = pci_dev->addr.function;
 	hw->adapter_stopped = 0;
 
+	/*
+	 * Switch Tag value should not be identical to either the First Tag
+	 * or Second Tag values. So set something other than common Ethertype
+	 * for internal switching.
+	 */
+	hw->switch_tag = 0xffff;
+
 	/* Check if need to support multi-driver */
 	i40e_support_multi_driver(dev);
 
@@ -1299,7 +1324,6 @@ eth_i40e_dev_init(struct rte_eth_dev *dev, void *init_params __rte_unused)
 		PMD_INIT_LOG(DEBUG,
 			     "Global register 0x%08x is changed with 0x28",
 			     I40E_GLQF_L3_MAP(40));
-		i40e_global_cfg_warning(I40E_WARNING_QINQ_CLOUD_FILTER);
 	}
 
 	/* Need the special FW version to support floating VEB */
@@ -1586,7 +1610,6 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(33), 0x00000000);
 	I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(34), 0x00000000);
 	I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(35), 0x00000000);
-	i40e_global_cfg_warning(I40E_WARNING_DIS_FLX_PLD);
 }
 
 static int
@@ -1829,8 +1852,7 @@ __vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t msix_vect,
 	/* Write first RX queue to Link list register as the head element */
 	if (vsi->type != I40E_VSI_SRIOV) {
 		uint16_t interval =
-			i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL, 1,
-					       pf->support_multi_driver);
+			i40e_calc_itr_interval(1, pf->support_multi_driver);
 
 		if (msix_vect == I40E_MISC_VEC_ID) {
 			I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
@@ -2026,27 +2048,40 @@ i40e_phy_conf_link(struct i40e_hw *hw,
 	struct i40e_aq_get_phy_abilities_resp phy_ab;
 	struct i40e_aq_set_phy_config phy_conf;
 	enum i40e_aq_phy_type cnt;
+	uint8_t avail_speed;
 	uint32_t phy_type_mask = 0;
 
 	const uint8_t mask = I40E_AQ_PHY_FLAG_PAUSE_TX |
 			I40E_AQ_PHY_FLAG_PAUSE_RX |
 			I40E_AQ_PHY_FLAG_PAUSE_RX |
 			I40E_AQ_PHY_FLAG_LOW_POWER;
-	const uint8_t advt = I40E_LINK_SPEED_40GB |
-			I40E_LINK_SPEED_25GB |
-			I40E_LINK_SPEED_10GB |
-			I40E_LINK_SPEED_1GB |
-			I40E_LINK_SPEED_100MB;
 	int ret = -ENOTSUP;
 
+	/* To get phy capabilities of available speeds. */
+	status = i40e_aq_get_phy_capabilities(hw, false, true, &phy_ab,
+					      NULL);
+	if (status) {
+		PMD_DRV_LOG(ERR, "Failed to get PHY capabilities: %d\n",
+				status);
+		return ret;
+	}
+	avail_speed = phy_ab.link_speed;
 
+	/* To get the current phy config. */
 	status = i40e_aq_get_phy_capabilities(hw, false, false, &phy_ab,
 					      NULL);
-	if (status)
+	if (status) {
+		PMD_DRV_LOG(ERR, "Failed to get the current PHY config: %d\n",
+				status);
 		return ret;
+	}
 
-	/* If link already up, no need to set up again */
-	if (is_up && phy_ab.phy_type != 0)
+	/* If link needs to go up and it is in autoneg mode the speed is OK,
+	 * no need to set up again.
+	 */
+	if (is_up && phy_ab.phy_type != 0 &&
+		     abilities & I40E_AQ_PHY_AN_ENABLED &&
+		     phy_ab.link_speed != 0)
 		return I40E_SUCCESS;
 
 	memset(&phy_conf, 0, sizeof(phy_conf));
@@ -2055,18 +2090,20 @@ i40e_phy_conf_link(struct i40e_hw *hw,
 	abilities &= ~mask;
 	abilities |= phy_ab.abilities & mask;
 
-	/* update ablities and speed */
-	if (abilities & I40E_AQ_PHY_AN_ENABLED)
-		phy_conf.link_speed = advt;
-	else
-		phy_conf.link_speed = is_up ? force_speed : phy_ab.link_speed;
-
 	phy_conf.abilities = abilities;
 
+	/* If link needs to go up, but the force speed is not supported,
+	 * Warn users and config the default available speeds.
+	 */
+	if (is_up && !(force_speed & avail_speed)) {
+		PMD_DRV_LOG(WARNING, "Invalid speed setting, set to default!\n");
+		phy_conf.link_speed = avail_speed;
+	} else {
+		phy_conf.link_speed = is_up ? force_speed : avail_speed;
+	}
 
-
-	/* To enable link, phy_type mask needs to include each type */
-	for (cnt = I40E_PHY_TYPE_SGMII; cnt < I40E_PHY_TYPE_MAX; cnt++)
+	/* PHY type mask needs to include each type except PHY type extension */
+	for (cnt = I40E_PHY_TYPE_SGMII; cnt < I40E_PHY_TYPE_25GBASE_KR; cnt++)
 		phy_type_mask |= 1 << cnt;
 
 	/* use get_phy_abilities_resp value for the rest */
@@ -2099,11 +2136,18 @@ i40e_apply_link_speed(struct rte_eth_dev *dev)
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct rte_eth_conf *conf = &dev->data->dev_conf;
 
+	if (conf->link_speeds == ETH_LINK_SPEED_AUTONEG) {
+		conf->link_speeds = ETH_LINK_SPEED_40G |
+				    ETH_LINK_SPEED_25G |
+				    ETH_LINK_SPEED_20G |
+				    ETH_LINK_SPEED_10G |
+				    ETH_LINK_SPEED_1G |
+				    ETH_LINK_SPEED_100M;
+	}
 	speed = i40e_parse_link_speeds(conf->link_speeds);
-	abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
-	if (!(conf->link_speeds & ETH_LINK_SPEED_FIXED))
-		abilities |= I40E_AQ_PHY_AN_ENABLED;
-	abilities |= I40E_AQ_PHY_LINK_ENABLED;
+	abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK |
+		     I40E_AQ_PHY_AN_ENABLED |
+		     I40E_AQ_PHY_LINK_ENABLED;
 
 	return i40e_phy_conf_link(hw, abilities, speed, true);
 }
@@ -2220,13 +2264,6 @@ i40e_dev_start(struct rte_eth_dev *dev)
 	}
 
 	/* Apply link configure */
-	if (dev->data->dev_conf.link_speeds & ~(ETH_LINK_SPEED_100M |
-				ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G |
-				ETH_LINK_SPEED_20G | ETH_LINK_SPEED_25G |
-				ETH_LINK_SPEED_40G)) {
-		PMD_DRV_LOG(ERR, "Invalid link setting");
-		goto err_up;
-	}
 	ret = i40e_apply_link_speed(dev);
 	if (I40E_SUCCESS != ret) {
 		PMD_DRV_LOG(ERR, "Fail to apply link setting");
@@ -3327,6 +3364,7 @@ i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 		DEV_RX_OFFLOAD_TCP_CKSUM |
 		DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
 		DEV_RX_OFFLOAD_CRC_STRIP |
+		DEV_RX_OFFLOAD_KEEP_CRC |
 		DEV_RX_OFFLOAD_VLAN_EXTEND |
 		DEV_RX_OFFLOAD_VLAN_FILTER |
 		DEV_RX_OFFLOAD_JUMBO_FRAME;
@@ -3501,8 +3539,6 @@ i40e_vlan_tpid_set_by_registers(struct rte_eth_dev *dev,
 		    "Global register 0x%08x is changed with value 0x%08x",
 		    I40E_GL_SWT_L2TAGCTRL(reg_id), (uint32_t)reg_w);
 
-	i40e_global_cfg_warning(I40E_WARNING_TPID);
-
 	return 0;
 }
 
@@ -3797,7 +3833,6 @@ i40e_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 		I40E_WRITE_GLB_REG(hw, I40E_GLRPB_GLW,
 				   pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]
 				   << I40E_KILOSHIFT);
-		i40e_global_cfg_warning(I40E_WARNING_FLOW_CTL);
 	} else {
 		PMD_DRV_LOG(ERR,
 			    "Water marker configuration is not supported.");
@@ -4334,7 +4369,6 @@ i40e_get_cap(struct i40e_hw *hw)
 }
 
 #define RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF	4
-#define QUEUE_NUM_PER_VF_ARG			"queue-num-per-vf"
 
 static int i40e_pf_parse_vf_queue_number_handler(const char *key,
 		const char *value,
@@ -4368,9 +4402,9 @@ static int i40e_pf_parse_vf_queue_number_handler(const char *key,
 
 static int i40e_pf_config_vf_rxq_number(struct rte_eth_dev *dev)
 {
-	static const char * const valid_keys[] = {QUEUE_NUM_PER_VF_ARG, NULL};
 	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct rte_kvargs *kvlist;
+	int kvargs_count;
 
 	/* set default queue number per VF as 4 */
 	pf->vf_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
@@ -4382,12 +4416,18 @@ static int i40e_pf_config_vf_rxq_number(struct rte_eth_dev *dev)
 	if (kvlist == NULL)
 		return -(EINVAL);
 
-	if (rte_kvargs_count(kvlist, QUEUE_NUM_PER_VF_ARG) > 1)
+	kvargs_count = rte_kvargs_count(kvlist, ETH_I40E_QUEUE_NUM_PER_VF_ARG);
+	if (!kvargs_count) {
+		rte_kvargs_free(kvlist);
+		return 0;
+	}
+
+	if (kvargs_count > 1)
 		PMD_DRV_LOG(WARNING, "More than one argument \"%s\" and only "
 			    "the first invalid or last valid one is used !",
-			    QUEUE_NUM_PER_VF_ARG);
+			    ETH_I40E_QUEUE_NUM_PER_VF_ARG);
 
-	rte_kvargs_process(kvlist, QUEUE_NUM_PER_VF_ARG,
+	rte_kvargs_process(kvlist, ETH_I40E_QUEUE_NUM_PER_VF_ARG,
 			   i40e_pf_parse_vf_queue_number_handler, pf);
 
 	rte_kvargs_free(kvlist);
@@ -7565,6 +7605,7 @@ i40e_status_code i40e_replace_mpls_l1_filter(struct i40e_pf *pf)
 	struct i40e_aqc_replace_cloud_filters_cmd  filter_replace;
 	struct i40e_aqc_replace_cloud_filters_cmd_buf  filter_replace_buf;
 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
+	struct rte_eth_dev *dev = ((struct i40e_adapter *)hw->back)->eth_dev;
 	enum i40e_status_code status = I40E_SUCCESS;
 
 	if (pf->support_multi_driver) {
@@ -7608,13 +7649,14 @@ i40e_status_code i40e_replace_mpls_l1_filter(struct i40e_pf *pf)
 
 	status = i40e_aq_replace_cloud_filters(hw, &filter_replace,
 					       &filter_replace_buf);
-	if (!status) {
-		i40e_global_cfg_warning(I40E_WARNING_RPL_CLD_FILTER);
-		PMD_DRV_LOG(DEBUG, "Global configuration modification: "
-			    "cloud l1 type is changed from 0x%x to 0x%x",
+	if (!status && (filter_replace.old_filter_type !=
+			filter_replace.new_filter_type))
+		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud l1 type."
+			    " original: 0x%x, new: 0x%x",
+			    dev->device->name,
 			    filter_replace.old_filter_type,
 			    filter_replace.new_filter_type);
-	}
+
 	return status;
 }
 
@@ -7624,6 +7666,7 @@ i40e_status_code i40e_replace_mpls_cloud_filter(struct i40e_pf *pf)
 	struct i40e_aqc_replace_cloud_filters_cmd  filter_replace;
 	struct i40e_aqc_replace_cloud_filters_cmd_buf  filter_replace_buf;
 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
+	struct rte_eth_dev *dev = ((struct i40e_adapter *)hw->back)->eth_dev;
 	enum i40e_status_code status = I40E_SUCCESS;
 
 	if (pf->support_multi_driver) {
@@ -7652,10 +7695,13 @@ i40e_status_code i40e_replace_mpls_cloud_filter(struct i40e_pf *pf)
 					       &filter_replace_buf);
 	if (status < 0)
 		return status;
-	PMD_DRV_LOG(DEBUG, "Global configuration modification: "
-		    "cloud filter type is changed from 0x%x to 0x%x",
-		    filter_replace.old_filter_type,
-		    filter_replace.new_filter_type);
+	if (filter_replace.old_filter_type !=
+	    filter_replace.new_filter_type)
+		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud filter type."
+			    " original: 0x%x, new: 0x%x",
+			    dev->device->name,
+			    filter_replace.old_filter_type,
+			    filter_replace.new_filter_type);
 
 	/* For MPLSoGRE */
 	memset(&filter_replace, 0,
@@ -7678,13 +7724,14 @@ i40e_status_code i40e_replace_mpls_cloud_filter(struct i40e_pf *pf)
 
 	status = i40e_aq_replace_cloud_filters(hw, &filter_replace,
 					       &filter_replace_buf);
-	if (!status) {
-		i40e_global_cfg_warning(I40E_WARNING_RPL_CLD_FILTER);
-		PMD_DRV_LOG(DEBUG, "Global configuration modification: "
-			    "cloud filter type is changed from 0x%x to 0x%x",
+	if (!status && (filter_replace.old_filter_type !=
+			filter_replace.new_filter_type))
+		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud filter type."
+			    " original: 0x%x, new: 0x%x",
+			    dev->device->name,
 			    filter_replace.old_filter_type,
 			    filter_replace.new_filter_type);
-	}
+
 	return status;
 }
 
@@ -7694,6 +7741,7 @@ i40e_replace_gtp_l1_filter(struct i40e_pf *pf)
 	struct i40e_aqc_replace_cloud_filters_cmd  filter_replace;
 	struct i40e_aqc_replace_cloud_filters_cmd_buf  filter_replace_buf;
 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
+	struct rte_eth_dev *dev = ((struct i40e_adapter *)hw->back)->eth_dev;
 	enum i40e_status_code status = I40E_SUCCESS;
 
 	if (pf->support_multi_driver) {
@@ -7729,10 +7777,13 @@ i40e_replace_gtp_l1_filter(struct i40e_pf *pf)
 					       &filter_replace_buf);
 	if (status < 0)
 		return status;
-	PMD_DRV_LOG(DEBUG, "Global configuration modification: "
-		    "cloud l1 type is changed from 0x%x to 0x%x",
-		    filter_replace.old_filter_type,
-		    filter_replace.new_filter_type);
+	if (filter_replace.old_filter_type !=
+	    filter_replace.new_filter_type)
+		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud l1 type."
+			    " original: 0x%x, new: 0x%x",
+			    dev->device->name,
+			    filter_replace.old_filter_type,
+			    filter_replace.new_filter_type);
 
 	/* for GTP-U */
 	memset(&filter_replace, 0,
@@ -7761,13 +7812,14 @@ i40e_replace_gtp_l1_filter(struct i40e_pf *pf)
 
 	status = i40e_aq_replace_cloud_filters(hw, &filter_replace,
 					       &filter_replace_buf);
-	if (!status) {
-		i40e_global_cfg_warning(I40E_WARNING_RPL_CLD_FILTER);
-		PMD_DRV_LOG(DEBUG, "Global configuration modification: "
-			    "cloud l1 type is changed from 0x%x to 0x%x",
+	if (!status && (filter_replace.old_filter_type !=
+			filter_replace.new_filter_type))
+		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud l1 type."
+			    " original: 0x%x, new: 0x%x",
+			    dev->device->name,
 			    filter_replace.old_filter_type,
 			    filter_replace.new_filter_type);
-	}
+
 	return status;
 }
 
@@ -7777,6 +7829,7 @@ i40e_status_code i40e_replace_gtp_cloud_filter(struct i40e_pf *pf)
 	struct i40e_aqc_replace_cloud_filters_cmd  filter_replace;
 	struct i40e_aqc_replace_cloud_filters_cmd_buf  filter_replace_buf;
 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
+	struct rte_eth_dev *dev = ((struct i40e_adapter *)hw->back)->eth_dev;
 	enum i40e_status_code status = I40E_SUCCESS;
 
 	if (pf->support_multi_driver) {
@@ -7804,10 +7857,13 @@ i40e_status_code i40e_replace_gtp_cloud_filter(struct i40e_pf *pf)
 					       &filter_replace_buf);
 	if (status < 0)
 		return status;
-	PMD_DRV_LOG(DEBUG, "Global configuration modification: "
-		    "cloud filter type is changed from 0x%x to 0x%x",
-		    filter_replace.old_filter_type,
-		    filter_replace.new_filter_type);
+	if (filter_replace.old_filter_type !=
+	    filter_replace.new_filter_type)
+		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud filter type."
+			    " original: 0x%x, new: 0x%x",
+			    dev->device->name,
+			    filter_replace.old_filter_type,
+			    filter_replace.new_filter_type);
 
 	/* for GTP-U */
 	memset(&filter_replace, 0,
@@ -7829,13 +7885,14 @@ i40e_status_code i40e_replace_gtp_cloud_filter(struct i40e_pf *pf)
 
 	status = i40e_aq_replace_cloud_filters(hw, &filter_replace,
 					       &filter_replace_buf);
-	if (!status) {
-		i40e_global_cfg_warning(I40E_WARNING_RPL_CLD_FILTER);
-		PMD_DRV_LOG(DEBUG, "Global configuration modification: "
-			    "cloud filter type is changed from 0x%x to 0x%x",
+	if (!status && (filter_replace.old_filter_type !=
+			filter_replace.new_filter_type))
+		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud filter type."
+			    " original: 0x%x, new: 0x%x",
+			    dev->device->name,
 			    filter_replace.old_filter_type,
 			    filter_replace.new_filter_type);
-	}
+
 	return status;
 }
 
@@ -8395,7 +8452,6 @@ i40e_dev_set_gre_key_len(struct i40e_hw *hw, uint8_t len)
 		PMD_DRV_LOG(DEBUG, "Global register 0x%08x is changed "
 			    "with value 0x%08x",
 			    I40E_GL_PRS_FVBM(2), reg);
-		i40e_global_cfg_warning(I40E_WARNING_GRE_KEY_LEN);
 	} else {
 		ret = 0;
 	}
@@ -8661,7 +8717,6 @@ i40e_set_hash_filter_global_config(struct i40e_hw *hw,
 							  I40E_GLQF_HSYM(j),
 							  reg);
 			}
-			i40e_global_cfg_warning(I40E_WARNING_HSYM);
 		}
 	}
 
@@ -8687,7 +8742,6 @@ i40e_set_hash_filter_global_config(struct i40e_hw *hw,
 		goto out;
 
 	i40e_write_global_rx_ctl(hw, I40E_GLQF_CTL, reg);
-	i40e_global_cfg_warning(I40E_WARNING_QF_CTL);
 
 out:
 	I40E_WRITE_FLUSH(hw);
@@ -9280,12 +9334,17 @@ void
 i40e_check_write_global_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val)
 {
 	uint32_t reg = i40e_read_rx_ctl(hw, addr);
+	struct rte_eth_dev *dev;
 
-	if (reg != val)
+	dev = ((struct i40e_adapter *)hw->back)->eth_dev;
+	if (reg != val) {
 		i40e_write_rx_ctl(hw, addr, val);
-	PMD_DRV_LOG(DEBUG,
-		    "Global register [0x%08x] original: 0x%08x, after: 0x%08x",
-		    addr, reg, (uint32_t)i40e_read_rx_ctl(hw, addr));
+		PMD_DRV_LOG(WARNING,
+			    "i40e device %s changed global register [0x%08x]."
+			    " original: 0x%08x, new: 0x%08x",
+			    dev->device->name, addr, reg,
+			    (uint32_t)i40e_read_rx_ctl(hw, addr));
+	}
 }
 
 static void
@@ -9359,12 +9418,6 @@ i40e_filter_input_set_init(struct i40e_pf *pf)
 			pf->hash_input_set[pctype] = input_set;
 		pf->fdir.input_set[pctype] = input_set;
 	}
-
-	if (!pf->support_multi_driver) {
-		i40e_global_cfg_warning(I40E_WARNING_HASH_INSET);
-		i40e_global_cfg_warning(I40E_WARNING_FD_MSK);
-		i40e_global_cfg_warning(I40E_WARNING_HASH_MSK);
-	}
 }
 
 int
@@ -9430,7 +9483,6 @@ i40e_hash_filter_inset_select(struct i40e_hw *hw,
 	i40e_check_write_global_reg(hw, I40E_GLQF_HASH_INSET(1, pctype),
 				    (uint32_t)((inset_reg >>
 				    I40E_32_BIT_WIDTH) & UINT32_MAX));
-	i40e_global_cfg_warning(I40E_WARNING_HASH_INSET);
 
 	for (i = 0; i < num; i++)
 		i40e_check_write_global_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
@@ -9439,7 +9491,6 @@ i40e_hash_filter_inset_select(struct i40e_hw *hw,
 	for (i = num; i < I40E_INSET_MASK_NUM_REG; i++)
 		i40e_check_write_global_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
 					    0);
-	i40e_global_cfg_warning(I40E_WARNING_HASH_MSK);
 	I40E_WRITE_FLUSH(hw);
 
 	pf->hash_input_set[pctype] = input_set;
@@ -9520,7 +9571,6 @@ i40e_fdir_filter_inset_select(struct i40e_pf *pf,
 			i40e_check_write_global_reg(hw,
 						    I40E_GLQF_FD_MSK(i, pctype),
 						    0);
-		i40e_global_cfg_warning(I40E_WARNING_FD_MSK);
 	} else {
 		PMD_DRV_LOG(ERR, "FDIR bit mask is not supported.");
 	}
@@ -10003,6 +10053,60 @@ i40e_pctype_to_flowtype(const struct i40e_adapter *adapter,
 #define I40E_GL_SWR_PM_UP_THR_SF_VALUE   0x06060606
 #define I40E_GL_SWR_PM_UP_THR            0x269FBC
 
+/*
+ * GL_SWR_PM_UP_THR:
+ * The value is not impacted from the link speed, its value is set according
+ * to the total number of ports for a better pipe-monitor configuration.
+ */
+static bool
+i40e_get_swr_pm_cfg(struct i40e_hw *hw, uint32_t *value)
+{
+#define I40E_GL_SWR_PM_EF_DEVICE(dev) \
+		.device_id = (dev),   \
+		.val = I40E_GL_SWR_PM_UP_THR_EF_VALUE
+
+#define I40E_GL_SWR_PM_SF_DEVICE(dev) \
+		.device_id = (dev),   \
+		.val = I40E_GL_SWR_PM_UP_THR_SF_VALUE
+
+	static const struct {
+		uint16_t device_id;
+		uint32_t val;
+	} swr_pm_table[] = {
+		{ I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_SFP_XL710) },
+		{ I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_KX_C) },
+		{ I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_10G_BASE_T) },
+		{ I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_10G_BASE_T4) },
+
+		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_KX_B) },
+		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_QSFP_A) },
+		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_QSFP_B) },
+		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_20G_KR2) },
+		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_20G_KR2_A) },
+		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_25G_B) },
+		{ I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_25G_SFP28) },
+	};
+	uint32_t i;
+
+	if (value == NULL) {
+		PMD_DRV_LOG(ERR, "value is NULL");
+		return false;
+	}
+
+	for (i = 0; i < RTE_DIM(swr_pm_table); i++) {
+		if (hw->device_id == swr_pm_table[i].device_id) {
+			*value = swr_pm_table[i].val;
+
+			PMD_DRV_LOG(DEBUG, "Device 0x%x with GL_SWR_PM_UP_THR "
+				    "value - 0x%08x",
+				    hw->device_id, *value);
+			return true;
+		}
+	}
+
+	return false;
+}
+
 static int
 i40e_dev_sync_phy_type(struct i40e_hw *hw)
 {
@@ -10067,13 +10171,16 @@ i40e_configure_registers(struct i40e_hw *hw)
 		}
 
 		if (reg_table[i].addr == I40E_GL_SWR_PM_UP_THR) {
-			if (I40E_PHY_TYPE_SUPPORT_40G(hw->phy.phy_types) || /* For XL710 */
-			    I40E_PHY_TYPE_SUPPORT_25G(hw->phy.phy_types)) /* For XXV710 */
-				reg_table[i].val =
-					I40E_GL_SWR_PM_UP_THR_SF_VALUE;
-			else /* For X710 */
-				reg_table[i].val =
-					I40E_GL_SWR_PM_UP_THR_EF_VALUE;
+			uint32_t cfg_val;
+
+			if (!i40e_get_swr_pm_cfg(hw, &cfg_val)) {
+				PMD_DRV_LOG(DEBUG, "Device 0x%x skips "
+					    "GL_SWR_PM_UP_THR value fixup",
+					    hw->device_id);
+				continue;
+			}
+
+			reg_table[i].val = cfg_val;
 		}
 
 		ret = i40e_aq_debug_read_register(hw, reg_table[i].addr,
@@ -12070,7 +12177,8 @@ i40e_update_customized_ptype(struct rte_eth_dev *dev, uint8_t *pkg,
 					ptype_mapping[i].sw_ptype |=
 						RTE_PTYPE_TUNNEL_GRENAT;
 					in_tunnel = true;
-				} else if (!strncasecmp(name, "L2TPV2CTL", 9)) {
+				} else if (!strncasecmp(name, "L2TPV2CTL", 9) ||
+					   !strncasecmp(name, "L2TPV2", 6)) {
 					ptype_mapping[i].sw_ptype |=
 						RTE_PTYPE_TUNNEL_L2TP;
 					in_tunnel = true;
@@ -12214,6 +12322,7 @@ i40e_cloud_filter_qinq_create(struct i40e_pf *pf)
 	struct i40e_aqc_replace_cloud_filters_cmd  filter_replace;
 	struct i40e_aqc_replace_cloud_filters_cmd_buf  filter_replace_buf;
 	struct i40e_hw *hw = I40E_PF_TO_HW(pf);
+	struct rte_eth_dev *dev = ((struct i40e_adapter *)hw->back)->eth_dev;
 
 	if (pf->support_multi_driver) {
 		PMD_DRV_LOG(ERR, "Replace cloud filter is not supported.");
@@ -12250,10 +12359,14 @@ i40e_cloud_filter_qinq_create(struct i40e_pf *pf)
 			&filter_replace_buf);
 	if (ret != I40E_SUCCESS)
 		return ret;
-	PMD_DRV_LOG(DEBUG, "Global configuration modification: "
-		    "cloud l1 type is changed from 0x%x to 0x%x",
-		    filter_replace.old_filter_type,
-		    filter_replace.new_filter_type);
+
+	if (filter_replace.old_filter_type !=
+	    filter_replace.new_filter_type)
+		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud l1 type."
+			    " original: 0x%x, new: 0x%x",
+			    dev->device->name,
+			    filter_replace.old_filter_type,
+			    filter_replace.new_filter_type);
 
 	/* Apply the second L2 cloud filter */
 	memset(&filter_replace, 0,
@@ -12275,13 +12388,14 @@ i40e_cloud_filter_qinq_create(struct i40e_pf *pf)
 		I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED;
 	ret = i40e_aq_replace_cloud_filters(hw, &filter_replace,
 			&filter_replace_buf);
-	if (!ret) {
-		i40e_global_cfg_warning(I40E_WARNING_RPL_CLD_FILTER);
-		PMD_DRV_LOG(DEBUG, "Global configuration modification: "
-			    "cloud filter type is changed from 0x%x to 0x%x",
+	if (!ret && (filter_replace.old_filter_type !=
+		     filter_replace.new_filter_type))
+		PMD_DRV_LOG(WARNING, "i40e device %s changed cloud filter type."
+			    " original: 0x%x, new: 0x%x",
+			    dev->device->name,
 			    filter_replace.old_filter_type,
 			    filter_replace.new_filter_type);
-	}
+
 	return ret;
 }
 
@@ -12399,9 +12513,7 @@ i40e_config_rss_filter(struct i40e_pf *pf,
 	return 0;
 }
 
-RTE_INIT(i40e_init_log);
-static void
-i40e_init_log(void)
+RTE_INIT(i40e_init_log)
 {
 	i40e_logtype_init = rte_log_register("pmd.net.i40e.init");
 	if (i40e_logtype_init >= 0)
@@ -12412,5 +12524,7 @@ i40e_init_log(void)
 }
 
 RTE_PMD_REGISTER_PARAM_STRING(net_i40e,
-			      QUEUE_NUM_PER_VF_ARG "=1|2|4|8|16"
+			      ETH_I40E_FLOATING_VEB_ARG "=1"
+			      ETH_I40E_FLOATING_VEB_LIST_ARG "=<string>"
+			      ETH_I40E_QUEUE_NUM_PER_VF_ARG "=1|2|4|8|16"
 			      ETH_I40E_SUPPORT_MULTI_DRIVER "=1");
diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h
index 11c4c76b..3fffe5a5 100644
--- a/drivers/net/i40e/i40e_ethdev.h
+++ b/drivers/net/i40e/i40e_ethdev.h
@@ -28,6 +28,7 @@
 #define I40E_NUM_DESC_ALIGN       32
 #define I40E_BUF_SIZE_MIN         1024
 #define I40E_FRAME_SIZE_MAX       9728
+#define I40E_TSO_FRAME_SIZE_MAX   262144
 #define I40E_QUEUE_BASE_ADDR_UNIT 128
 /* number of VSIs and queue default setting */
 #define I40E_MAX_QP_NUM_PER_VF    16
@@ -87,12 +88,18 @@
 #define I40E_WRITE_GLB_REG(hw, reg, value)				\
 	do {								\
 		uint32_t ori_val;					\
+		struct rte_eth_dev *dev;				\
 		ori_val = I40E_READ_REG((hw), (reg));			\
+		dev = ((struct i40e_adapter *)hw->back)->eth_dev;	\
 		I40E_PCI_REG_WRITE(I40E_PCI_REG_ADDR((hw),		\
 						     (reg)), (value));	\
-		PMD_DRV_LOG(DEBUG, "global register [0x%08x] "		\
-			    "original: 0x%08x, after: 0x%08x ",		\
-			    (reg), (ori_val), (value));			\
+		if (ori_val != value)					\
+			PMD_DRV_LOG(WARNING,				\
+				    "i40e device %s changed global "	\
+				    "register [0x%08x]. original: 0x%08x, " \
+				    "new: 0x%08x ",			\
+				    (dev->device->name), (reg),		\
+				    (ori_val), (value));		\
 	} while (0)
 
 /* index flex payload per layer */
@@ -178,7 +185,7 @@ enum i40e_flxpld_layer_idx {
 #define I40E_ITR_INDEX_NONE             3
 #define I40E_QUEUE_ITR_INTERVAL_DEFAULT 32 /* 32 us */
 #define I40E_QUEUE_ITR_INTERVAL_MAX     8160 /* 8160 us */
-#define I40E_VF_QUEUE_ITR_INTERVAL_DEFAULT 8160 /* 8160 us */
+#define I40E_VF_QUEUE_ITR_INTERVAL_DEFAULT 32 /* 32 us */
 /* Special FW support this floating VEB feature */
 #define FLOATING_VEB_SUPPORTED_FW_MAJ 5
 #define FLOATING_VEB_SUPPORTED_FW_MIN 0
@@ -1108,22 +1115,6 @@ struct i40e_valid_pattern {
 	parse_filter_t parse_filter;
 };
 
-enum I40E_WARNING_IDX {
-	I40E_WARNING_DIS_FLX_PLD,
-	I40E_WARNING_ENA_FLX_PLD,
-	I40E_WARNING_QINQ_PARSER,
-	I40E_WARNING_QINQ_CLOUD_FILTER,
-	I40E_WARNING_TPID,
-	I40E_WARNING_FLOW_CTL,
-	I40E_WARNING_GRE_KEY_LEN,
-	I40E_WARNING_QF_CTL,
-	I40E_WARNING_HASH_INSET,
-	I40E_WARNING_HSYM,
-	I40E_WARNING_HASH_MSK,
-	I40E_WARNING_FD_MSK,
-	I40E_WARNING_RPL_CLD_FILTER,
-};
-
 int i40e_dev_switch_queues(struct i40e_pf *pf, bool on);
 int i40e_vsi_release(struct i40e_vsi *vsi);
 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf,
@@ -1328,50 +1319,23 @@ i40e_align_floor(int n)
 }
 
 static inline uint16_t
-i40e_calc_itr_interval(int16_t interval, bool is_pf, bool is_multi_drv)
+i40e_calc_itr_interval(bool is_pf, bool is_multi_drv)
 {
-	if (interval < 0 || interval > I40E_QUEUE_ITR_INTERVAL_MAX) {
-		if (is_multi_drv) {
-			interval = I40E_QUEUE_ITR_INTERVAL_MAX;
-		} else {
-			if (is_pf)
-				interval = I40E_QUEUE_ITR_INTERVAL_DEFAULT;
-			else
-				interval = I40E_VF_QUEUE_ITR_INTERVAL_DEFAULT;
-		}
+	uint16_t interval = 0;
+
+	if (is_multi_drv) {
+		interval = I40E_QUEUE_ITR_INTERVAL_MAX;
+	} else {
+		if (is_pf)
+			interval = I40E_QUEUE_ITR_INTERVAL_DEFAULT;
+		else
+			interval = I40E_VF_QUEUE_ITR_INTERVAL_DEFAULT;
 	}
 
 	/* Convert to hardware count, as writing each 1 represents 2 us */
 	return interval / 2;
 }
 
-static inline void
-i40e_global_cfg_warning(enum I40E_WARNING_IDX idx)
-{
-	const char *warning;
-	static const char *const warning_list[] = {
-		[I40E_WARNING_DIS_FLX_PLD] = "disable FDIR flexible payload",
-		[I40E_WARNING_ENA_FLX_PLD] = "enable FDIR flexible payload",
-		[I40E_WARNING_QINQ_PARSER] = "support QinQ parser",
-		[I40E_WARNING_QINQ_CLOUD_FILTER] = "support QinQ cloud filter",
-		[I40E_WARNING_TPID] = "support TPID configuration",
-		[I40E_WARNING_FLOW_CTL] = "configure water marker",
-		[I40E_WARNING_GRE_KEY_LEN] = "support GRE key length setting",
-		[I40E_WARNING_QF_CTL] = "support hash function setting",
-		[I40E_WARNING_HASH_INSET] = "configure hash input set",
-		[I40E_WARNING_HSYM] = "set symmetric hash",
-		[I40E_WARNING_HASH_MSK] = "configure hash mask",
-		[I40E_WARNING_FD_MSK] = "configure fdir mask",
-		[I40E_WARNING_RPL_CLD_FILTER] = "replace cloud filter",
-	};
-
-	warning = warning_list[idx];
-
-	RTE_LOG(WARNING, PMD,
-		"Global register is changed during %s\n",
-		warning);
-}
-
 #define I40E_VALID_FLOW(flow_type) \
 	((flow_type) == RTE_ETH_FLOW_FRAG_IPV4 || \
 	(flow_type) == RTE_ETH_FLOW_NONFRAG_IPV4_TCP || \
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 804e4453..001c301b 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -44,6 +44,8 @@
 #define I40EVF_BUSY_WAIT_COUNT 50
 #define MAX_RESET_WAIT_CNT     20
 
+#define I40EVF_ALARM_INTERVAL 50000 /* us */
+
 struct i40evf_arq_msg_info {
 	enum virtchnl_ops ops;
 	enum i40e_status_code result;
@@ -1133,7 +1135,7 @@ i40evf_init_vf(struct rte_eth_dev *dev)
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	uint16_t interval =
-		i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL, 0, 0);
+		i40e_calc_itr_interval(0, 0);
 
 	vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	vf->dev_data = dev->data;
@@ -1370,7 +1372,7 @@ i40evf_handle_aq_msg(struct rte_eth_dev *dev)
  *  void
  */
 static void
-i40evf_dev_interrupt_handler(void *param)
+i40evf_dev_alarm_handler(void *param)
 {
 	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
@@ -1399,6 +1401,8 @@ i40evf_dev_interrupt_handler(void *param)
 
 done:
 	i40evf_enable_irq0(hw);
+	rte_eal_alarm_set(I40EVF_ALARM_INTERVAL,
+			  i40evf_dev_alarm_handler, dev);
 }
 
 static int
@@ -1442,12 +1446,8 @@ i40evf_dev_init(struct rte_eth_dev *eth_dev)
 		return -1;
 	}
 
-	/* register callback func to eal lib */
-	rte_intr_callback_register(&pci_dev->intr_handle,
-		i40evf_dev_interrupt_handler, (void *)eth_dev);
-
-	/* enable uio intr after callback register */
-	rte_intr_enable(&pci_dev->intr_handle);
+	rte_eal_alarm_set(I40EVF_ALARM_INTERVAL,
+			  i40evf_dev_alarm_handler, eth_dev);
 
 	/* configure and enable device interrupt */
 	i40evf_enable_irq0(hw);
@@ -1536,7 +1536,7 @@ i40evf_dev_configure(struct rte_eth_dev *dev)
 	/* For non-DPDK PF drivers, VF has no ability to disable HW
 	 * CRC strip, and is implicitly enabled by the PF.
 	 */
-	if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP)) {
+	if (rte_eth_dev_must_keep_crc(conf->rxmode.offloads)) {
 		vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 		if ((vf->version_major == VIRTCHNL_VERSION_MAJOR) &&
 		    (vf->version_minor <= VIRTCHNL_VERSION_MINOR)) {
@@ -1583,37 +1583,35 @@ static int
 i40evf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 {
 	struct i40e_rx_queue *rxq;
-	int err = 0;
+	int err;
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
 	PMD_INIT_FUNC_TRACE();
 
-	if (rx_queue_id < dev->data->nb_rx_queues) {
-		rxq = dev->data->rx_queues[rx_queue_id];
+	rxq = dev->data->rx_queues[rx_queue_id];
 
-		err = i40e_alloc_rx_queue_mbufs(rxq);
-		if (err) {
-			PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
-			return err;
-		}
-
-		rte_wmb();
+	err = i40e_alloc_rx_queue_mbufs(rxq);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
+		return err;
+	}
 
-		/* Init the RX tail register. */
-		I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
-		I40EVF_WRITE_FLUSH(hw);
+	rte_wmb();
 
-		/* Ready to switch the queue on */
-		err = i40evf_switch_queue(dev, TRUE, rx_queue_id, TRUE);
+	/* Init the RX tail register. */
+	I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
+	I40EVF_WRITE_FLUSH(hw);
 
-		if (err)
-			PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
-				    rx_queue_id);
-		else
-			dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+	/* Ready to switch the queue on */
+	err = i40evf_switch_queue(dev, TRUE, rx_queue_id, TRUE);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
+			    rx_queue_id);
+		return err;
 	}
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 
-	return err;
+	return 0;
 }
 
 static int
@@ -1622,45 +1620,39 @@ i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	struct i40e_rx_queue *rxq;
 	int err;
 
-	if (rx_queue_id < dev->data->nb_rx_queues) {
-		rxq = dev->data->rx_queues[rx_queue_id];
-
-		err = i40evf_switch_queue(dev, TRUE, rx_queue_id, FALSE);
+	rxq = dev->data->rx_queues[rx_queue_id];
 
-		if (err) {
-			PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
-				    rx_queue_id);
-			return err;
-		}
-
-		i40e_rx_queue_release_mbufs(rxq);
-		i40e_reset_rx_queue(rxq);
-		dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+	err = i40evf_switch_queue(dev, TRUE, rx_queue_id, FALSE);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
+			    rx_queue_id);
+		return err;
 	}
 
+	i40e_rx_queue_release_mbufs(rxq);
+	i40e_reset_rx_queue(rxq);
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+
 	return 0;
 }
 
 static int
 i40evf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 {
-	int err = 0;
+	int err;
 
 	PMD_INIT_FUNC_TRACE();
 
-	if (tx_queue_id < dev->data->nb_tx_queues) {
-
-		/* Ready to switch the queue on */
-		err = i40evf_switch_queue(dev, FALSE, tx_queue_id, TRUE);
-
-		if (err)
-			PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
-				    tx_queue_id);
-		else
-			dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+	/* Ready to switch the queue on */
+	err = i40evf_switch_queue(dev, FALSE, tx_queue_id, TRUE);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
+			    tx_queue_id);
+		return err;
 	}
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 
-	return err;
+	return 0;
 }
 
 static int
@@ -1669,22 +1661,19 @@ i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	struct i40e_tx_queue *txq;
 	int err;
 
-	if (tx_queue_id < dev->data->nb_tx_queues) {
-		txq = dev->data->tx_queues[tx_queue_id];
-
-		err = i40evf_switch_queue(dev, FALSE, tx_queue_id, FALSE);
-
-		if (err) {
-			PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off",
-				    tx_queue_id);
-			return err;
-		}
+	txq = dev->data->tx_queues[tx_queue_id];
 
-		i40e_tx_queue_release_mbufs(txq);
-		i40e_reset_tx_queue(txq);
-		dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+	err = i40evf_switch_queue(dev, FALSE, tx_queue_id, FALSE);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off",
+			    tx_queue_id);
+		return err;
 	}
 
+	i40e_tx_queue_release_mbufs(txq);
+	i40e_reset_tx_queue(txq);
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+
 	return 0;
 }
 
@@ -1836,7 +1825,7 @@ i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	uint16_t interval =
-		i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL, 0, 0);
+		i40e_calc_itr_interval(0, 0);
 	uint16_t msix_intr;
 
 	msix_intr = intr_handle->intr_vec[queue_id];
@@ -1859,8 +1848,6 @@ i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
 
 	I40EVF_WRITE_FLUSH(hw);
 
-	rte_intr_enable(&pci_dev->intr_handle);
-
 	return 0;
 }
 
@@ -2016,17 +2003,9 @@ i40evf_dev_start(struct rte_eth_dev *dev)
 		goto err_mac;
 	}
 
-	/* When a VF port is bound to VFIO-PCI, only miscellaneous interrupt
-	 * is mapped to VFIO vector 0 in i40evf_dev_init( ).
-	 * If previous VFIO interrupt mapping set in i40evf_dev_init( ) is
-	 * not cleared, it will fail when rte_intr_enable( ) tries to map Rx
-	 * queue interrupt to other VFIO vectors.
-	 * So clear uio/vfio intr/evevnfd first to avoid failure.
-	 */
-	if (dev->data->dev_conf.intr_conf.rxq != 0) {
-		rte_intr_disable(intr_handle);
+	/* only enable interrupt in rx interrupt mode */
+	if (dev->data->dev_conf.intr_conf.rxq != 0)
 		rte_intr_enable(intr_handle);
-	}
 
 	i40evf_enable_queues_intr(dev);
 
@@ -2050,6 +2029,9 @@ i40evf_dev_stop(struct rte_eth_dev *dev)
 
 	PMD_INIT_FUNC_TRACE();
 
+	if (dev->data->dev_conf.intr_conf.rxq != 0)
+		rte_intr_disable(intr_handle);
+
 	if (hw->adapter_stopped == 1)
 		return;
 	i40evf_stop_queues(dev);
@@ -2182,7 +2164,6 @@ i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
 	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 
-	memset(dev_info, 0, sizeof(*dev_info));
 	dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
 	dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
 	dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
@@ -2200,6 +2181,7 @@ i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 		DEV_RX_OFFLOAD_TCP_CKSUM |
 		DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
 		DEV_RX_OFFLOAD_CRC_STRIP |
+		DEV_RX_OFFLOAD_KEEP_CRC |
 		DEV_RX_OFFLOAD_SCATTER |
 		DEV_RX_OFFLOAD_JUMBO_FRAME |
 		DEV_RX_OFFLOAD_VLAN_FILTER;
@@ -2285,9 +2267,8 @@ static void
 i40evf_dev_close(struct rte_eth_dev *dev)
 {
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
-	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 
+	rte_eal_alarm_cancel(i40evf_dev_alarm_handler, dev);
 	i40evf_dev_stop(dev);
 	i40e_dev_free_queues(dev);
 	/*
@@ -2300,12 +2281,6 @@ i40evf_dev_close(struct rte_eth_dev *dev)
 
 	i40evf_reset_vf(hw);
 	i40e_shutdown_adminq(hw);
-	/* disable uio intr before callback unregister */
-	rte_intr_disable(intr_handle);
-
-	/* unregister callback func from eal lib */
-	rte_intr_callback_unregister(intr_handle,
-				     i40evf_dev_interrupt_handler, dev);
 	i40evf_disable_irq0(hw);
 }
 
diff --git a/drivers/net/i40e/i40e_fdir.c b/drivers/net/i40e/i40e_fdir.c
index a4a61d1c..d41601a1 100644
--- a/drivers/net/i40e/i40e_fdir.c
+++ b/drivers/net/i40e/i40e_fdir.c
@@ -526,7 +526,6 @@ i40e_set_flx_pld_cfg(struct i40e_pf *pf,
 			  (num << I40E_GLQF_ORT_FIELD_CNT_SHIFT) |
 			  (layer_idx * I40E_MAX_FLXPLD_FIED);
 		I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(33 + layer_idx), flx_ort);
-		i40e_global_cfg_warning(I40E_WARNING_ENA_FLX_PLD);
 	}
 
 	for (i = 0; i < num; i++) {
diff --git a/drivers/net/i40e/i40e_flow.c b/drivers/net/i40e/i40e_flow.c
index 89de6a59..c67b264d 100644
--- a/drivers/net/i40e/i40e_flow.c
+++ b/drivers/net/i40e/i40e_flow.c
@@ -2263,7 +2263,6 @@ i40e_flow_set_fdir_flex_pit(struct i40e_pf *pf,
 			  (raw_id << I40E_GLQF_ORT_FIELD_CNT_SHIFT) |
 			  (layer_idx * I40E_MAX_FLXPLD_FIED);
 		I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(33 + layer_idx), flx_ort);
-		i40e_global_cfg_warning(I40E_WARNING_ENA_FLX_PLD);
 	}
 
 	/* Set flex pit */
diff --git a/drivers/net/i40e/i40e_rxtx.c b/drivers/net/i40e/i40e_rxtx.c
index 6032d554..2a28ee34 100644
--- a/drivers/net/i40e/i40e_rxtx.c
+++ b/drivers/net/i40e/i40e_rxtx.c
@@ -1439,13 +1439,15 @@ i40e_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
 
 		/* Check for m->nb_segs to not exceed the limits. */
 		if (!(ol_flags & PKT_TX_TCP_SEG)) {
-			if (m->nb_segs > I40E_TX_MAX_SEG ||
-			    m->nb_segs > I40E_TX_MAX_MTU_SEG) {
+			if (m->nb_segs > I40E_TX_MAX_MTU_SEG ||
+			    m->pkt_len > I40E_FRAME_SIZE_MAX) {
 				rte_errno = -EINVAL;
 				return i;
 			}
-		} else if ((m->tso_segsz < I40E_MIN_TSO_MSS) ||
-				(m->tso_segsz > I40E_MAX_TSO_MSS)) {
+		} else if (m->nb_segs > I40E_TX_MAX_SEG ||
+			   m->tso_segsz < I40E_MIN_TSO_MSS ||
+			   m->tso_segsz > I40E_MAX_TSO_MSS ||
+			   m->pkt_len > I40E_TSO_FRAME_SIZE_MAX) {
 			/* MSS outside the range (256B - 9674B) are considered
 			 * malicious
 			 */
@@ -1458,6 +1460,12 @@ i40e_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
 			return i;
 		}
 
+		/* check the size of packet */
+		if (m->pkt_len < I40E_TX_MIN_PKT_LEN) {
+			rte_errno = -EINVAL;
+			return i;
+		}
+
 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
 		ret = rte_validate_tx_offload(m);
 		if (ret != 0) {
@@ -1524,38 +1532,36 @@ int
 i40e_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 {
 	struct i40e_rx_queue *rxq;
-	int err = -1;
+	int err;
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
 	PMD_INIT_FUNC_TRACE();
 
-	if (rx_queue_id < dev->data->nb_rx_queues) {
-		rxq = dev->data->rx_queues[rx_queue_id];
-
-		err = i40e_alloc_rx_queue_mbufs(rxq);
-		if (err) {
-			PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
-			return err;
-		}
+	rxq = dev->data->rx_queues[rx_queue_id];
 
-		rte_wmb();
+	err = i40e_alloc_rx_queue_mbufs(rxq);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
+		return err;
+	}
 
-		/* Init the RX tail regieter. */
-		I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
+	rte_wmb();
 
-		err = i40e_switch_rx_queue(hw, rxq->reg_idx, TRUE);
+	/* Init the RX tail regieter. */
+	I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 
-		if (err) {
-			PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
-				    rx_queue_id);
+	err = i40e_switch_rx_queue(hw, rxq->reg_idx, TRUE);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
+			    rx_queue_id);
 
-			i40e_rx_queue_release_mbufs(rxq);
-			i40e_reset_rx_queue(rxq);
-		} else
-			dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+		i40e_rx_queue_release_mbufs(rxq);
+		i40e_reset_rx_queue(rxq);
+		return err;
 	}
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 
-	return err;
+	return 0;
 }
 
 int
@@ -1565,24 +1571,21 @@ i40e_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	int err;
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
-	if (rx_queue_id < dev->data->nb_rx_queues) {
-		rxq = dev->data->rx_queues[rx_queue_id];
+	rxq = dev->data->rx_queues[rx_queue_id];
 
-		/*
-		* rx_queue_id is queue id application refers to, while
-		* rxq->reg_idx is the real queue index.
-		*/
-		err = i40e_switch_rx_queue(hw, rxq->reg_idx, FALSE);
-
-		if (err) {
-			PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
-				    rx_queue_id);
-			return err;
-		}
-		i40e_rx_queue_release_mbufs(rxq);
-		i40e_reset_rx_queue(rxq);
-		dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+	/*
+	 * rx_queue_id is queue id application refers to, while
+	 * rxq->reg_idx is the real queue index.
+	 */
+	err = i40e_switch_rx_queue(hw, rxq->reg_idx, FALSE);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
+			    rx_queue_id);
+		return err;
 	}
+	i40e_rx_queue_release_mbufs(rxq);
+	i40e_reset_rx_queue(rxq);
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 
 	return 0;
 }
@@ -1590,28 +1593,27 @@ i40e_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 int
 i40e_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 {
-	int err = -1;
+	int err;
 	struct i40e_tx_queue *txq;
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
 	PMD_INIT_FUNC_TRACE();
 
-	if (tx_queue_id < dev->data->nb_tx_queues) {
-		txq = dev->data->tx_queues[tx_queue_id];
+	txq = dev->data->tx_queues[tx_queue_id];
 
-		/*
-		* tx_queue_id is queue id application refers to, while
-		* rxq->reg_idx is the real queue index.
-		*/
-		err = i40e_switch_tx_queue(hw, txq->reg_idx, TRUE);
-		if (err)
-			PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
-				    tx_queue_id);
-		else
-			dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+	/*
+	 * tx_queue_id is queue id application refers to, while
+	 * rxq->reg_idx is the real queue index.
+	 */
+	err = i40e_switch_tx_queue(hw, txq->reg_idx, TRUE);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
+			    tx_queue_id);
+		return err;
 	}
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 
-	return err;
+	return 0;
 }
 
 int
@@ -1621,26 +1623,23 @@ i40e_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	int err;
 	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
-	if (tx_queue_id < dev->data->nb_tx_queues) {
-		txq = dev->data->tx_queues[tx_queue_id];
+	txq = dev->data->tx_queues[tx_queue_id];
 
-		/*
-		* tx_queue_id is queue id application refers to, while
-		* txq->reg_idx is the real queue index.
-		*/
-		err = i40e_switch_tx_queue(hw, txq->reg_idx, FALSE);
-
-		if (err) {
-			PMD_DRV_LOG(ERR, "Failed to switch TX queue %u of",
-				    tx_queue_id);
-			return err;
-		}
-
-		i40e_tx_queue_release_mbufs(txq);
-		i40e_reset_tx_queue(txq);
-		dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+	/*
+	 * tx_queue_id is queue id application refers to, while
+	 * txq->reg_idx is the real queue index.
+	 */
+	err = i40e_switch_tx_queue(hw, txq->reg_idx, FALSE);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u of",
+			    tx_queue_id);
+		return err;
 	}
 
+	i40e_tx_queue_release_mbufs(txq);
+	i40e_reset_tx_queue(txq);
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+
 	return 0;
 }
 
@@ -1829,8 +1828,10 @@ i40e_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->queue_id = queue_idx;
 	rxq->reg_idx = reg_idx;
 	rxq->port_id = dev->data->port_id;
-	rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.offloads &
-			DEV_RX_OFFLOAD_CRC_STRIP) ? 0 : ETHER_CRC_LEN);
+	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+		rxq->crc_len = ETHER_CRC_LEN;
+	else
+		rxq->crc_len = 0;
 	rxq->drop_en = rx_conf->rx_drop_en;
 	rxq->vsi = vsi;
 	rxq->rx_deferred_start = rx_conf->rx_deferred_start;
@@ -2087,7 +2088,7 @@ i40e_dev_tx_queue_setup_runtime(struct rte_eth_dev *dev,
 	}
 	/* check simple tx conflict */
 	if (ad->tx_simple_allowed) {
-		if (txq->offloads != 0 ||
+		if ((txq->offloads & ~DEV_TX_OFFLOAD_MBUF_FAST_FREE) != 0 ||
 				txq->tx_rs_thresh < RTE_PMD_I40E_TX_MAX_BURST) {
 			PMD_DRV_LOG(ERR, "No-simple tx is required.");
 			return -EINVAL;
diff --git a/drivers/net/i40e/i40e_rxtx.h b/drivers/net/i40e/i40e_rxtx.h
index ea73a8a1..3fc619af 100644
--- a/drivers/net/i40e/i40e_rxtx.h
+++ b/drivers/net/i40e/i40e_rxtx.h
@@ -30,6 +30,8 @@
 #define I40E_TX_MAX_SEG     UINT8_MAX
 #define I40E_TX_MAX_MTU_SEG 8
 
+#define I40E_TX_MIN_PKT_LEN 17
+
 #undef container_of
 #define container_of(ptr, type, member) ({ \
 		typeof(((type *)0)->member)(*__mptr) = (ptr); \
diff --git a/drivers/net/i40e/i40e_rxtx_vec_avx2.c b/drivers/net/i40e/i40e_rxtx_vec_avx2.c
index dbcb61f3..23179b3b 100644
--- a/drivers/net/i40e/i40e_rxtx_vec_avx2.c
+++ b/drivers/net/i40e/i40e_rxtx_vec_avx2.c
@@ -188,7 +188,7 @@ _recv_raw_pkts_vec_avx2(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
 	/* See if we need to rearm the RX queue - gives the prefetch a bit
 	 * of time to act
 	 */
-	while (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH)
+	if (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH)
 		i40e_rxq_rearm(rxq);
 
 	/* Before we start moving massive data around, check to see if
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 7aa1a751..bba62b1c 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -1709,6 +1709,7 @@ rte_pmd_i40e_process_ddp_package(uint16_t port, uint8_t *buff,
 				PMD_DRV_LOG(ERR, "Profile of group 0 already exists.");
 			else if (is_exist == 3)
 				PMD_DRV_LOG(ERR, "Profile of different group already exists");
+			i40e_update_customized_info(dev, buff, size, op);
 			rte_free(profile_info_sec);
 			return -EEXIST;
 		}
@@ -3162,8 +3163,6 @@ rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 			i40e_check_write_global_reg(hw,
 						  I40E_GLQF_HASH_MSK(i, pctype),
 						  mask_reg[i]);
-		i40e_global_cfg_warning(I40E_WARNING_HASH_INSET);
-		i40e_global_cfg_warning(I40E_WARNING_HASH_MSK);
 		break;
 	case INSET_FDIR:
 		i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0),
@@ -3175,7 +3174,6 @@ rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 			i40e_check_write_global_reg(hw,
 						    I40E_GLQF_FD_MSK(i, pctype),
 						    mask_reg[i]);
-		i40e_global_cfg_warning(I40E_WARNING_FD_MSK);
 		break;
 	case INSET_FDIR_FLX:
 		i40e_check_write_reg(hw, I40E_PRTQF_FD_FLXINSET(pctype),
diff --git a/drivers/net/ifc/Makefile b/drivers/net/ifc/Makefile
index 1011995b..39b36ae5 100644
--- a/drivers/net/ifc/Makefile
+++ b/drivers/net/ifc/Makefile
@@ -6,7 +6,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 #
 # library name
 #
-LIB = librte_ifcvf_vdpa.a
+LIB = librte_pmd_ifc.a
 
 LDLIBS += -lpthread
 LDLIBS += -lrte_eal -lrte_pci -lrte_vhost -lrte_bus_pci
@@ -22,14 +22,14 @@ BASE_DRIVER_OBJS=$(sort $(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.
 
 VPATH += $(SRCDIR)/base
 
-EXPORT_MAP := rte_ifcvf_version.map
+EXPORT_MAP := rte_pmd_ifc_version.map
 
 LIBABIVER := 1
 
 #
 # all source are stored in SRCS-y
 #
-SRCS-$(CONFIG_RTE_LIBRTE_IFCVF_VDPA_PMD) += ifcvf_vdpa.c
-SRCS-$(CONFIG_RTE_LIBRTE_IFCVF_VDPA_PMD) += ifcvf.c
+SRCS-$(CONFIG_RTE_LIBRTE_IFC_PMD) += ifcvf_vdpa.c
+SRCS-$(CONFIG_RTE_LIBRTE_IFC_PMD) += ifcvf.c
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/ifc/ifcvf_vdpa.c b/drivers/net/ifc/ifcvf_vdpa.c
index c6627c23..88d81403 100644
--- a/drivers/net/ifc/ifcvf_vdpa.c
+++ b/drivers/net/ifc/ifcvf_vdpa.c
@@ -646,6 +646,9 @@ ifcvf_get_vdpa_features(int did, uint64_t *features)
 
 #define VDPA_SUPPORTED_PROTOCOL_FEATURES \
 		(1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK | \
+		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ | \
+		 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD | \
+		 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER | \
 		 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD)
 static int
 ifcvf_get_protocol_features(int did __rte_unused, uint64_t *features)
@@ -782,9 +785,7 @@ RTE_PMD_REGISTER_PCI(net_ifcvf, rte_ifcvf_vdpa);
 RTE_PMD_REGISTER_PCI_TABLE(net_ifcvf, pci_id_ifcvf_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_ifcvf, "* vfio-pci");
 
-RTE_INIT(ifcvf_vdpa_init_log);
-static void
-ifcvf_vdpa_init_log(void)
+RTE_INIT(ifcvf_vdpa_init_log)
 {
 	ifcvf_vdpa_logtype = rte_log_register("pmd.net.ifcvf_vdpa");
 	if (ifcvf_vdpa_logtype >= 0)
diff --git a/drivers/net/ifc/meson.build b/drivers/net/ifc/meson.build
new file mode 100644
index 00000000..72df070a
--- /dev/null
+++ b/drivers/net/ifc/meson.build
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+build = dpdk_conf.has('RTE_LIBRTE_VHOST')
+allow_experimental_apis = true
+sources = files('ifcvf_vdpa.c', 'base/ifcvf.c')
+includes += include_directories('base')
+deps += 'vhost'
diff --git a/drivers/net/ifc/rte_ifcvf_version.map b/drivers/net/ifc/rte_pmd_ifc_version.map
index 9b9ab1a4..9b9ab1a4 100644
--- a/drivers/net/ifc/rte_ifcvf_version.map
+++ b/drivers/net/ifc/rte_pmd_ifc_version.map
diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c b/drivers/net/ixgbe/ixgbe_ethdev.c
index 87d2ad09..26b19273 100644
--- a/drivers/net/ixgbe/ixgbe_ethdev.c
+++ b/drivers/net/ixgbe/ixgbe_ethdev.c
@@ -3120,9 +3120,18 @@ ixgbe_read_stats_registers(struct ixgbe_hw *hw,
 	}
 
 	/* Flow Director Stats registers */
-	hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
-	hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
-
+	if (hw->mac.type != ixgbe_mac_82598EB) {
+		hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
+		hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
+		hw_stats->fdirustat_add += IXGBE_READ_REG(hw,
+					IXGBE_FDIRUSTAT) & 0xFFFF;
+		hw_stats->fdirustat_remove += (IXGBE_READ_REG(hw,
+					IXGBE_FDIRUSTAT) >> 16) & 0xFFFF;
+		hw_stats->fdirfstat_fadd += IXGBE_READ_REG(hw,
+					IXGBE_FDIRFSTAT) & 0xFFFF;
+		hw_stats->fdirfstat_fremove += (IXGBE_READ_REG(hw,
+					IXGBE_FDIRFSTAT) >> 16) & 0xFFFF;
+	}
 	/* MACsec Stats registers */
 	macsec_stats->out_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECTXUT);
 	macsec_stats->out_pkts_encrypted +=
@@ -3755,6 +3764,14 @@ ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 		dev_info->speed_capa |= ETH_LINK_SPEED_2_5G;
 		dev_info->speed_capa |= ETH_LINK_SPEED_5G;
 	}
+
+	/* Driver-preferred Rx/Tx parameters */
+	dev_info->default_rxportconf.burst_size = 32;
+	dev_info->default_txportconf.burst_size = 32;
+	dev_info->default_rxportconf.nb_queues = 1;
+	dev_info->default_txportconf.nb_queues = 1;
+	dev_info->default_rxportconf.ring_size = 256;
+	dev_info->default_txportconf.ring_size = 256;
 }
 
 static const uint32_t *
@@ -4991,12 +5008,12 @@ ixgbevf_dev_configure(struct rte_eth_dev *dev)
 	 * Keep the persistent behavior the same as Host PF
 	 */
 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
-	if (!(conf->rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP)) {
+	if (rte_eth_dev_must_keep_crc(conf->rxmode.offloads)) {
 		PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
 		conf->rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
 	}
 #else
-	if (conf->rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP) {
+	if (!rte_eth_dev_must_keep_crc(conf->rxmode.offloads)) {
 		PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
 		conf->rxmode.offloads &= ~DEV_RX_OFFLOAD_CRC_STRIP;
 	}
@@ -8567,9 +8584,7 @@ RTE_PMD_REGISTER_PCI(net_ixgbe_vf, rte_ixgbevf_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_ixgbe_vf, pci_id_ixgbevf_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_ixgbe_vf, "* igb_uio | vfio-pci");
 
-RTE_INIT(ixgbe_init_log);
-static void
-ixgbe_init_log(void)
+RTE_INIT(ixgbe_init_log)
 {
 	ixgbe_logtype_init = rte_log_register("pmd.net.ixgbe.init");
 	if (ixgbe_logtype_init >= 0)
diff --git a/drivers/net/ixgbe/ixgbe_ethdev.h b/drivers/net/ixgbe/ixgbe_ethdev.h
index e42ec30d..d0b93968 100644
--- a/drivers/net/ixgbe/ixgbe_ethdev.h
+++ b/drivers/net/ixgbe/ixgbe_ethdev.h
@@ -100,6 +100,11 @@
 #define IXGBE_5TUPLE_MAX_PRI            7
 #define IXGBE_5TUPLE_MIN_PRI            1
 
+/* bit of VXLAN tunnel type | 7 bits of zeros  | 8 bits of zeros*/
+#define IXGBE_FDIR_VXLAN_TUNNEL_TYPE    0x8000
+/* bit of NVGRE tunnel type | 7 bits of zeros  | 8 bits of zeros*/
+#define IXGBE_FDIR_NVGRE_TUNNEL_TYPE    0x0
+
 #define IXGBE_RSS_OFFLOAD_ALL ( \
 	ETH_RSS_IPV4 | \
 	ETH_RSS_NONFRAG_IPV4_TCP | \
diff --git a/drivers/net/ixgbe/ixgbe_fdir.c b/drivers/net/ixgbe/ixgbe_fdir.c
index d5e51797..e559f0fa 100644
--- a/drivers/net/ixgbe/ixgbe_fdir.c
+++ b/drivers/net/ixgbe/ixgbe_fdir.c
@@ -394,9 +394,12 @@ fdir_set_input_mask_x550(struct rte_eth_dev *dev)
 				IXGBE_FDIRIP6M_TNI_VNI;
 
 	if (mode == RTE_FDIR_MODE_PERFECT_TUNNEL) {
-		mac_mask = info->mask.mac_addr_byte_mask;
-		fdiripv6m |= (mac_mask << IXGBE_FDIRIP6M_INNER_MAC_SHIFT)
-				& IXGBE_FDIRIP6M_INNER_MAC;
+		fdiripv6m |= IXGBE_FDIRIP6M_INNER_MAC;
+		mac_mask = info->mask.mac_addr_byte_mask &
+			(IXGBE_FDIRIP6M_INNER_MAC >>
+			IXGBE_FDIRIP6M_INNER_MAC_SHIFT);
+		fdiripv6m &= ~((mac_mask << IXGBE_FDIRIP6M_INNER_MAC_SHIFT) &
+				IXGBE_FDIRIP6M_INNER_MAC);
 
 		switch (info->mask.tunnel_type_mask) {
 		case 0:
@@ -771,10 +774,19 @@ ixgbe_fdir_filter_to_atr_input(const struct rte_eth_fdir_filter *fdir_filter,
 			input->formatted.inner_mac,
 			fdir_filter->input.flow.tunnel_flow.mac_addr.addr_bytes,
 			sizeof(input->formatted.inner_mac));
-		input->formatted.tunnel_type =
-			fdir_filter->input.flow.tunnel_flow.tunnel_type;
+		if (fdir_filter->input.flow.tunnel_flow.tunnel_type ==
+				RTE_FDIR_TUNNEL_TYPE_VXLAN)
+			input->formatted.tunnel_type =
+					IXGBE_FDIR_VXLAN_TUNNEL_TYPE;
+		else if (fdir_filter->input.flow.tunnel_flow.tunnel_type ==
+				RTE_FDIR_TUNNEL_TYPE_NVGRE)
+			input->formatted.tunnel_type =
+					IXGBE_FDIR_NVGRE_TUNNEL_TYPE;
+		else
+			PMD_DRV_LOG(ERR, " invalid tunnel type arguments.");
+
 		input->formatted.tni_vni =
-			fdir_filter->input.flow.tunnel_flow.tunnel_id;
+			fdir_filter->input.flow.tunnel_flow.tunnel_id >> 8;
 	}
 
 	return 0;
@@ -1001,8 +1013,7 @@ fdir_write_perfect_filter_82599(struct ixgbe_hw *hw,
 			IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(2), 0);
 		} else {
 			/* tunnel mode */
-			if (input->formatted.tunnel_type !=
-				RTE_FDIR_TUNNEL_TYPE_NVGRE)
+			if (input->formatted.tunnel_type)
 				tunnel_type = 0x80000000;
 			tunnel_type |= addr_high;
 			IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(0), addr_low);
@@ -1010,6 +1021,9 @@ fdir_write_perfect_filter_82599(struct ixgbe_hw *hw,
 			IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(2),
 					input->formatted.tni_vni);
 		}
+		IXGBE_WRITE_REG(hw, IXGBE_FDIRIPSA, 0);
+		IXGBE_WRITE_REG(hw, IXGBE_FDIRIPDA, 0);
+		IXGBE_WRITE_REG(hw, IXGBE_FDIRPORT, 0);
 	}
 
 	/* record vlan (little-endian) and flex_bytes(big-endian) */
diff --git a/drivers/net/ixgbe/ixgbe_flow.c b/drivers/net/ixgbe/ixgbe_flow.c
index eb0644c8..1adf1b80 100644
--- a/drivers/net/ixgbe/ixgbe_flow.c
+++ b/drivers/net/ixgbe/ixgbe_flow.c
@@ -1739,7 +1739,8 @@ ixgbe_parse_fdir_filter_normal(struct rte_eth_dev *dev,
 				return -rte_errno;
 			}
 		} else {
-			if (item->type != RTE_FLOW_ITEM_TYPE_IPV4) {
+			if (item->type != RTE_FLOW_ITEM_TYPE_IPV4 &&
+					item->type != RTE_FLOW_ITEM_TYPE_VLAN) {
 				memset(rule, 0, sizeof(struct ixgbe_fdir_rule));
 				rte_flow_error_set(error, EINVAL,
 					RTE_FLOW_ERROR_TYPE_ITEM,
@@ -2437,7 +2438,7 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr,
 	/* Get the VxLAN info */
 	if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
 		rule->ixgbe_fdir.formatted.tunnel_type =
-			RTE_FDIR_TUNNEL_TYPE_VXLAN;
+				IXGBE_FDIR_VXLAN_TUNNEL_TYPE;
 
 		/* Only care about VNI, others should be masked. */
 		if (!item->mask) {
@@ -2487,17 +2488,15 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr,
 			rule->b_spec = TRUE;
 			vxlan_spec = item->spec;
 			rte_memcpy(((uint8_t *)
-				&rule->ixgbe_fdir.formatted.tni_vni + 1),
+				&rule->ixgbe_fdir.formatted.tni_vni),
 				vxlan_spec->vni, RTE_DIM(vxlan_spec->vni));
-			rule->ixgbe_fdir.formatted.tni_vni = rte_be_to_cpu_32(
-				rule->ixgbe_fdir.formatted.tni_vni);
 		}
 	}
 
 	/* Get the NVGRE info */
 	if (item->type == RTE_FLOW_ITEM_TYPE_NVGRE) {
 		rule->ixgbe_fdir.formatted.tunnel_type =
-			RTE_FDIR_TUNNEL_TYPE_NVGRE;
+				IXGBE_FDIR_NVGRE_TUNNEL_TYPE;
 
 		/**
 		 * Only care about flags0, flags1, protocol and TNI,
@@ -2587,7 +2586,6 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr,
 			/* tni is a 24-bits bit field */
 			rte_memcpy(&rule->ixgbe_fdir.formatted.tni_vni,
 			nvgre_spec->tni, RTE_DIM(nvgre_spec->tni));
-			rule->ixgbe_fdir.formatted.tni_vni <<= 8;
 		}
 	}
 
diff --git a/drivers/net/ixgbe/ixgbe_ipsec.c b/drivers/net/ixgbe/ixgbe_ipsec.c
index de7ed367..08405f1e 100644
--- a/drivers/net/ixgbe/ixgbe_ipsec.c
+++ b/drivers/net/ixgbe/ixgbe_ipsec.c
@@ -609,7 +609,7 @@ ixgbe_crypto_enable_ipsec(struct rte_eth_dev *dev)
 		PMD_DRV_LOG(ERR, "RSC and IPsec not supported");
 		return -1;
 	}
-	if (!(rx_offloads & DEV_RX_OFFLOAD_CRC_STRIP)) {
+	if (rte_eth_dev_must_keep_crc(rx_offloads)) {
 		PMD_DRV_LOG(ERR, "HW CRC strip needs to be enabled for IPsec");
 		return -1;
 	}
diff --git a/drivers/net/ixgbe/ixgbe_pf.c b/drivers/net/ixgbe/ixgbe_pf.c
index 4d199c80..4b833ffa 100644
--- a/drivers/net/ixgbe/ixgbe_pf.c
+++ b/drivers/net/ixgbe/ixgbe_pf.c
@@ -128,21 +128,23 @@ void ixgbe_pf_host_uninit(struct rte_eth_dev *eth_dev)
 
 	PMD_INIT_FUNC_TRACE();
 
-	vfinfo = IXGBE_DEV_PRIVATE_TO_P_VFDATA(eth_dev->data->dev_private);
-
 	RTE_ETH_DEV_SRIOV(eth_dev).active = 0;
 	RTE_ETH_DEV_SRIOV(eth_dev).nb_q_per_pool = 0;
 	RTE_ETH_DEV_SRIOV(eth_dev).def_vmdq_idx = 0;
 	RTE_ETH_DEV_SRIOV(eth_dev).def_pool_q_idx = 0;
 
-	ret = rte_eth_switch_domain_free((*vfinfo)->switch_domain_id);
-	if (ret)
-		PMD_INIT_LOG(WARNING, "failed to free switch domain: %d", ret);
-
 	vf_num = dev_num_vf(eth_dev);
 	if (vf_num == 0)
 		return;
 
+	vfinfo = IXGBE_DEV_PRIVATE_TO_P_VFDATA(eth_dev->data->dev_private);
+	if (*vfinfo == NULL)
+		return;
+
+	ret = rte_eth_switch_domain_free((*vfinfo)->switch_domain_id);
+	if (ret)
+		PMD_INIT_LOG(WARNING, "failed to free switch domain: %d", ret);
+
 	rte_free(*vfinfo);
 	*vfinfo = NULL;
 }
diff --git a/drivers/net/ixgbe/ixgbe_rxtx.c b/drivers/net/ixgbe/ixgbe_rxtx.c
index 3e13d26a..f82b74a9 100644
--- a/drivers/net/ixgbe/ixgbe_rxtx.c
+++ b/drivers/net/ixgbe/ixgbe_rxtx.c
@@ -1420,7 +1420,7 @@ rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
 	/*
 	 * Check if VLAN present only.
 	 * Do not check whether L3/L4 rx checksum done by NIC or not,
-	 * That can be found from rte_eth_rxmode.hw_ip_checksum flag
+	 * That can be found from rte_eth_rxmode.offloads flag
 	 */
 	pkt_flags = (rx_status & IXGBE_RXD_STAT_VP) ?  vlan_flags : 0;
 
@@ -2849,6 +2849,7 @@ ixgbe_get_rx_port_offloads(struct rte_eth_dev *dev)
 		   DEV_RX_OFFLOAD_UDP_CKSUM   |
 		   DEV_RX_OFFLOAD_TCP_CKSUM   |
 		   DEV_RX_OFFLOAD_CRC_STRIP   |
+		   DEV_RX_OFFLOAD_KEEP_CRC    |
 		   DEV_RX_OFFLOAD_JUMBO_FRAME |
 		   DEV_RX_OFFLOAD_SCATTER;
 
@@ -2935,8 +2936,10 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
 		queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
 	rxq->port_id = dev->data->port_id;
-	rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.offloads &
-		DEV_RX_OFFLOAD_CRC_STRIP) ? 0 : ETHER_CRC_LEN);
+	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
+		rxq->crc_len = ETHER_CRC_LEN;
+	else
+		rxq->crc_len = 0;
 	rxq->drop_en = rx_conf->rx_drop_en;
 	rxq->rx_deferred_start = rx_conf->rx_deferred_start;
 	rxq->offloads = offloads;
@@ -4702,7 +4705,7 @@ ixgbe_set_rsc(struct rte_eth_dev *dev)
 
 	/* RSC global configuration (chapter 4.6.7.2.1 of 82599 Spec) */
 
-	if (!(rx_conf->offloads & DEV_RX_OFFLOAD_CRC_STRIP) &&
+	if (rte_eth_dev_must_keep_crc(rx_conf->offloads) &&
 	     (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
 		/*
 		 * According to chapter of 4.6.7.2.1 of the Spec Rev.
@@ -4851,10 +4854,10 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
 	 * Configure CRC stripping, if any.
 	 */
 	hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
-	if (rx_conf->offloads & DEV_RX_OFFLOAD_CRC_STRIP)
-		hlreg0 |= IXGBE_HLREG0_RXCRCSTRP;
-	else
+	if (rte_eth_dev_must_keep_crc(rx_conf->offloads))
 		hlreg0 &= ~IXGBE_HLREG0_RXCRCSTRP;
+	else
+		hlreg0 |= IXGBE_HLREG0_RXCRCSTRP;
 
 	/*
 	 * Configure jumbo frame support, if any.
@@ -4892,8 +4895,8 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
 		 * Reset crc_len in case it was changed after queue setup by a
 		 * call to configure.
 		 */
-		rxq->crc_len = (rx_conf->offloads & DEV_RX_OFFLOAD_CRC_STRIP) ?
-				0 : ETHER_CRC_LEN;
+		rxq->crc_len = rte_eth_dev_must_keep_crc(rx_conf->offloads) ?
+				ETHER_CRC_LEN : 0;
 
 		/* Setup the Base and Length of the Rx Descriptor Rings */
 		bus_addr = rxq->rx_ring_phys_addr;
@@ -4962,10 +4965,10 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
 	if (hw->mac.type == ixgbe_mac_82599EB ||
 	    hw->mac.type == ixgbe_mac_X540) {
 		rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
-		if (rx_conf->offloads & DEV_RX_OFFLOAD_CRC_STRIP)
-			rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
-		else
+		if (rte_eth_dev_must_keep_crc(rx_conf->offloads))
 			rdrxctl &= ~IXGBE_RDRXCTL_CRCSTRIP;
+		else
+			rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
 		rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
 		IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
 	}
@@ -5173,34 +5176,30 @@ ixgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	PMD_INIT_FUNC_TRACE();
 	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
-	if (rx_queue_id < dev->data->nb_rx_queues) {
-		rxq = dev->data->rx_queues[rx_queue_id];
-
-		/* Allocate buffers for descriptor rings */
-		if (ixgbe_alloc_rx_queue_mbufs(rxq) != 0) {
-			PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
-				     rx_queue_id);
-			return -1;
-		}
-		rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
-		rxdctl |= IXGBE_RXDCTL_ENABLE;
-		IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
+	rxq = dev->data->rx_queues[rx_queue_id];
 
-		/* Wait until RX Enable ready */
-		poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
-		do {
-			rte_delay_ms(1);
-			rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
-		} while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
-		if (!poll_ms)
-			PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d",
-				     rx_queue_id);
-		rte_wmb();
-		IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
-		IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), rxq->nb_rx_desc - 1);
-		dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
-	} else
+	/* Allocate buffers for descriptor rings */
+	if (ixgbe_alloc_rx_queue_mbufs(rxq) != 0) {
+		PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
+			     rx_queue_id);
 		return -1;
+	}
+	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
+	rxdctl |= IXGBE_RXDCTL_ENABLE;
+	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
+
+	/* Wait until RX Enable ready */
+	poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
+	do {
+		rte_delay_ms(1);
+		rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
+	} while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
+	if (!poll_ms)
+		PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
+	rte_wmb();
+	IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
+	IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), rxq->nb_rx_desc - 1);
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 
 	return 0;
 }
@@ -5221,30 +5220,26 @@ ixgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	PMD_INIT_FUNC_TRACE();
 	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
-	if (rx_queue_id < dev->data->nb_rx_queues) {
-		rxq = dev->data->rx_queues[rx_queue_id];
+	rxq = dev->data->rx_queues[rx_queue_id];
 
-		rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
-		rxdctl &= ~IXGBE_RXDCTL_ENABLE;
-		IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
+	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
+	rxdctl &= ~IXGBE_RXDCTL_ENABLE;
+	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
 
-		/* Wait until RX Enable bit clear */
-		poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
-		do {
-			rte_delay_ms(1);
-			rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
-		} while (--poll_ms && (rxdctl & IXGBE_RXDCTL_ENABLE));
-		if (!poll_ms)
-			PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d",
-				     rx_queue_id);
+	/* Wait until RX Enable bit clear */
+	poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
+	do {
+		rte_delay_ms(1);
+		rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
+	} while (--poll_ms && (rxdctl & IXGBE_RXDCTL_ENABLE));
+	if (!poll_ms)
+		PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
 
-		rte_delay_us(RTE_IXGBE_WAIT_100_US);
+	rte_delay_us(RTE_IXGBE_WAIT_100_US);
 
-		ixgbe_rx_queue_release_mbufs(rxq);
-		ixgbe_reset_rx_queue(adapter, rxq);
-		dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
-	} else
-		return -1;
+	ixgbe_rx_queue_release_mbufs(rxq);
+	ixgbe_reset_rx_queue(adapter, rxq);
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 
 	return 0;
 }
@@ -5264,30 +5259,27 @@ ixgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	PMD_INIT_FUNC_TRACE();
 	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
-	if (tx_queue_id < dev->data->nb_tx_queues) {
-		txq = dev->data->tx_queues[tx_queue_id];
-		txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
-		txdctl |= IXGBE_TXDCTL_ENABLE;
-		IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
+	txq = dev->data->tx_queues[tx_queue_id];
+	txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
+	txdctl |= IXGBE_TXDCTL_ENABLE;
+	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
 
-		/* Wait until TX Enable ready */
-		if (hw->mac.type == ixgbe_mac_82599EB) {
-			poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
-			do {
-				rte_delay_ms(1);
-				txdctl = IXGBE_READ_REG(hw,
-					IXGBE_TXDCTL(txq->reg_idx));
-			} while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
-			if (!poll_ms)
-				PMD_INIT_LOG(ERR, "Could not enable "
-					     "Tx Queue %d", tx_queue_id);
-		}
-		rte_wmb();
-		IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
-		IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
-		dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
-	} else
-		return -1;
+	/* Wait until TX Enable ready */
+	if (hw->mac.type == ixgbe_mac_82599EB) {
+		poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
+		do {
+			rte_delay_ms(1);
+			txdctl = IXGBE_READ_REG(hw,
+				IXGBE_TXDCTL(txq->reg_idx));
+		} while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
+		if (!poll_ms)
+			PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d",
+				tx_queue_id);
+	}
+	rte_wmb();
+	IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
+	IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 
 	return 0;
 }
@@ -5307,9 +5299,6 @@ ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	PMD_INIT_FUNC_TRACE();
 	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
-	if (tx_queue_id >= dev->data->nb_tx_queues)
-		return -1;
-
 	txq = dev->data->tx_queues[tx_queue_id];
 
 	/* Wait until TX queue is empty */
@@ -5323,8 +5312,9 @@ ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 					       IXGBE_TDT(txq->reg_idx));
 		} while (--poll_ms && (txtdh != txtdt));
 		if (!poll_ms)
-			PMD_INIT_LOG(ERR, "Tx Queue %d is not empty "
-				     "when stopping.", tx_queue_id);
+			PMD_INIT_LOG(ERR,
+				"Tx Queue %d is not empty when stopping.",
+				tx_queue_id);
 	}
 
 	txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
@@ -5340,8 +5330,8 @@ ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 						IXGBE_TXDCTL(txq->reg_idx));
 		} while (--poll_ms && (txdctl & IXGBE_TXDCTL_ENABLE));
 		if (!poll_ms)
-			PMD_INIT_LOG(ERR, "Could not disable "
-				     "Tx Queue %d", tx_queue_id);
+			PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
+				tx_queue_id);
 	}
 
 	if (txq->ops != NULL) {
diff --git a/drivers/net/kni/meson.build b/drivers/net/kni/meson.build
new file mode 100644
index 00000000..0f784c6d
--- /dev/null
+++ b/drivers/net/kni/meson.build
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+# this driver can be built if-and-only-if KNI library is buildable
+build = dpdk_conf.has('RTE_LIBRTE_KNI')
+allow_experimental_apis = true
+sources = files('rte_eth_kni.c')
+deps += 'kni'
diff --git a/drivers/net/kni/rte_eth_kni.c b/drivers/net/kni/rte_eth_kni.c
index ab63ea42..085bb845 100644
--- a/drivers/net/kni/rte_eth_kni.c
+++ b/drivers/net/kni/rte_eth_kni.c
@@ -207,6 +207,7 @@ eth_kni_dev_info(struct rte_eth_dev *dev __rte_unused,
 	dev_info->max_rx_queues = KNI_MAX_QUEUE_PER_PORT;
 	dev_info->max_tx_queues = KNI_MAX_QUEUE_PER_PORT;
 	dev_info->min_rx_bufsize = 0;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -419,6 +420,7 @@ eth_kni_probe(struct rte_vdev_device *vdev)
 		}
 		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &eth_kni_ops;
+		eth_dev->device = &vdev->device;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -487,9 +489,7 @@ static struct rte_vdev_driver eth_kni_drv = {
 RTE_PMD_REGISTER_VDEV(net_kni, eth_kni_drv);
 RTE_PMD_REGISTER_PARAM_STRING(net_kni, ETH_KNI_NO_REQUEST_THREAD_ARG "=<int>");
 
-RTE_INIT(eth_kni_init_log);
-static void
-eth_kni_init_log(void)
+RTE_INIT(eth_kni_init_log)
 {
 	eth_kni_logtype = rte_log_register("pmd.net.kni");
 	if (eth_kni_logtype >= 0)
diff --git a/drivers/net/liquidio/Makefile b/drivers/net/liquidio/Makefile
index fc5f18ad..f1092851 100644
--- a/drivers/net/liquidio/Makefile
+++ b/drivers/net/liquidio/Makefile
@@ -15,7 +15,7 @@ LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
 
-EXPORT_MAP := rte_pmd_lio_version.map
+EXPORT_MAP := rte_pmd_liquidio_version.map
 
 LIBABIVER := 1
 
diff --git a/drivers/net/liquidio/lio_ethdev.c b/drivers/net/liquidio/lio_ethdev.c
index 64b1b86c..93e89007 100644
--- a/drivers/net/liquidio/lio_ethdev.c
+++ b/drivers/net/liquidio/lio_ethdev.c
@@ -2143,9 +2143,7 @@ RTE_PMD_REGISTER_PCI(net_liovf, rte_liovf_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_liovf, pci_id_liovf_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_liovf, "* igb_uio | vfio-pci");
 
-RTE_INIT(lio_init_log);
-static void
-lio_init_log(void)
+RTE_INIT(lio_init_log)
 {
 	lio_logtype_init = rte_log_register("pmd.net.liquidio.init");
 	if (lio_logtype_init >= 0)
diff --git a/drivers/net/liquidio/meson.build b/drivers/net/liquidio/meson.build
new file mode 100644
index 00000000..9ae48e21
--- /dev/null
+++ b/drivers/net/liquidio/meson.build
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+sources = files('base/lio_23xx_vf.c',
+	'base/lio_mbox.c',
+	'lio_ethdev.c',
+	'lio_rxtx.c')
+includes += include_directories('base')
diff --git a/drivers/net/liquidio/rte_pmd_lio_version.map b/drivers/net/liquidio/rte_pmd_liquidio_version.map
index 8591cc0b..8591cc0b 100644
--- a/drivers/net/liquidio/rte_pmd_lio_version.map
+++ b/drivers/net/liquidio/rte_pmd_liquidio_version.map
diff --git a/drivers/net/meson.build b/drivers/net/meson.build
index b7d00a04..9c28ed4d 100644
--- a/drivers/net/meson.build
+++ b/drivers/net/meson.build
@@ -1,10 +1,33 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['af_packet', 'axgbe', 'bonding', 'dpaa', 'dpaa2',
-	'e1000', 'enic', 'fm10k', 'i40e', 'ixgbe',
-	'mvpp2', 'null', 'octeontx', 'pcap', 'ring',
-	'sfc', 'thunderx', 'virtio']
+drivers = ['af_packet',
+	'ark',
+	'avp',
+	'axgbe', 'bonding',
+	'bnx2x',
+	'bnxt',
+	'cxgbe',
+	'dpaa', 'dpaa2',
+	'e1000',
+	'ena',
+	'enic',
+	'failsafe',
+	'fm10k', 'i40e',
+	'ifc',
+	'ixgbe',
+	'kni',
+	'liquidio',
+	'mvpp2',
+	'netvsc',
+	'nfp',
+	'null', 'octeontx', 'pcap', 'ring',
+	'sfc',
+	'softnic',
+	'szedata2',
+	'thunderx',
+	'vhost',
+	'virtio']
 std_deps = ['ethdev', 'kvargs'] # 'ethdev' also pulls in mbuf, net, eal etc
 std_deps += ['bus_pci']         # very many PMDs depend on PCI, so make std
 std_deps += ['bus_vdev']        # same with vdev bus
diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index 73f9d405..92e93225 100644
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -85,6 +85,11 @@ mlx4_autoconf.h.new: FORCE
 mlx4_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 	$Q $(RM) -f -- '$@'
 	$Q : > '$@'
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_MLX4_WQE_LSO_SEG \
+		infiniband/mlx4dv.h \
+		type 'struct mlx4_wqe_lso_seg' \
+		$(AUTOCONF_OUTPUT)
 
 # Create mlx4_autoconf.h or update it in case it differs from the new one.
 
@@ -110,7 +115,7 @@ endif
 $(LIB_GLUE): mlx4_glue.o
 	$Q $(LD) $(GLUE_LDFLAGS) $(EXTRA_LDFLAGS) \
 		-Wl,-h,$(LIB_GLUE) \
-		-s -shared -o $@ $< -libverbs -lmlx4
+		-shared -o $@ $< -libverbs -lmlx4
 
 mlx4_glue.o: mlx4_autoconf.h
 
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index a29814b3..defc0d4b 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -575,14 +575,14 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 	ibv_dev = list[i];
 	DEBUG("device opened");
 	if (mlx4_glue->query_device(attr_ctx, &device_attr)) {
-		rte_errno = ENODEV;
+		err = ENODEV;
 		goto error;
 	}
 	INFO("%u port(s) detected", device_attr.phys_port_cnt);
 	conf.ports.present |= (UINT64_C(1) << device_attr.phys_port_cnt) - 1;
 	if (mlx4_args(pci_dev->device.devargs, &conf)) {
 		ERROR("failed to process device arguments");
-		rte_errno = EINVAL;
+		err = EINVAL;
 		goto error;
 	}
 	/* Use all ports when none are defined */
@@ -590,7 +590,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		conf.ports.enabled = conf.ports.present;
 	/* Retrieve extended device attributes. */
 	if (mlx4_glue->query_device_ex(attr_ctx, NULL, &device_attr_ex)) {
-		rte_errno = ENODEV;
+		err = ENODEV;
 		goto error;
 	}
 	assert(device_attr.max_sge >= MLX4_MAX_SGE);
@@ -609,18 +609,18 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		DEBUG("using port %u", port);
 		ctx = mlx4_glue->open_device(ibv_dev);
 		if (ctx == NULL) {
-			rte_errno = ENODEV;
+			err = ENODEV;
 			goto port_error;
 		}
 		/* Check port status. */
 		err = mlx4_glue->query_port(ctx, port, &port_attr);
 		if (err) {
-			rte_errno = err;
-			ERROR("port query failed: %s", strerror(rte_errno));
+			err = ENODEV;
+			ERROR("port query failed: %s", strerror(err));
 			goto port_error;
 		}
 		if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
-			rte_errno = ENOTSUP;
+			err = ENOTSUP;
 			ERROR("port %d is not configured in Ethernet mode",
 			      port);
 			goto port_error;
@@ -630,15 +630,16 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 			      port, mlx4_glue->port_state_str(port_attr.state),
 			      port_attr.state);
 		/* Make asynchronous FD non-blocking to handle interrupts. */
-		if (mlx4_fd_set_non_blocking(ctx->async_fd) < 0) {
+		err = mlx4_fd_set_non_blocking(ctx->async_fd);
+		if (err) {
 			ERROR("cannot make asynchronous FD non-blocking: %s",
-			      strerror(rte_errno));
+			      strerror(err));
 			goto port_error;
 		}
 		/* Allocate protection domain. */
 		pd = mlx4_glue->alloc_pd(ctx);
 		if (pd == NULL) {
-			rte_errno = ENOMEM;
+			err = ENOMEM;
 			ERROR("PD allocation failure");
 			goto port_error;
 		}
@@ -647,7 +648,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 				   sizeof(*priv),
 				   RTE_CACHE_LINE_SIZE);
 		if (priv == NULL) {
-			rte_errno = ENOMEM;
+			err = ENOMEM;
 			ERROR("priv allocation failure");
 			goto port_error;
 		}
@@ -672,14 +673,27 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 						   &device_attr_ex);
 		DEBUG("supported RSS hash fields mask: %016" PRIx64,
 		      priv->hw_rss_sup);
+		priv->hw_rss_max_qps =
+			device_attr_ex.rss_caps.max_rwq_indirection_table_size;
+		DEBUG("MAX RSS queues %d", priv->hw_rss_max_qps);
 		priv->hw_fcs_strip = !!(device_attr_ex.raw_packet_caps &
 					IBV_RAW_PACKET_CAP_SCATTER_FCS);
 		DEBUG("FCS stripping toggling is %ssupported",
 		      priv->hw_fcs_strip ? "" : "not ");
+		priv->tso =
+			((device_attr_ex.tso_caps.max_tso > 0) &&
+			 (device_attr_ex.tso_caps.supported_qpts &
+			  (1 << IBV_QPT_RAW_PACKET)));
+		if (priv->tso)
+			priv->tso_max_payload_sz =
+					device_attr_ex.tso_caps.max_tso;
+		DEBUG("TSO is %ssupported",
+		      priv->tso ? "" : "not ");
 		/* Configure the first MAC address by default. */
-		if (mlx4_get_mac(priv, &mac.addr_bytes)) {
+		err = mlx4_get_mac(priv, &mac.addr_bytes);
+		if (err) {
 			ERROR("cannot get MAC address, is mlx4_en loaded?"
-			      " (rte_errno: %s)", strerror(rte_errno));
+			      " (error: %s)", strerror(err));
 			goto port_error;
 		}
 		INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
@@ -712,8 +726,8 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 			eth_dev = rte_eth_dev_allocate(name);
 		}
 		if (eth_dev == NULL) {
+			err = ENOMEM;
 			ERROR("can not allocate rte ethdev");
-			rte_errno = ENOMEM;
 			goto port_error;
 		}
 		eth_dev->data->dev_private = priv;
@@ -776,8 +790,6 @@ port_error:
 			rte_eth_dev_release_port(eth_dev);
 		break;
 	}
-	if (i == device_attr.phys_port_cnt)
-		return 0;
 	/*
 	 * XXX if something went wrong in the loop above, there is a resource
 	 * leak (ctx, pd, priv, dpdk ethdev) but we can do nothing about it as
@@ -789,8 +801,9 @@ error:
 		claim_zero(mlx4_glue->close_device(attr_ctx));
 	if (list)
 		mlx4_glue->free_device_list(list);
-	assert(rte_errno >= 0);
-	return -rte_errno;
+	if (err)
+		rte_errno = err;
+	return -err;
 }
 
 static const struct rte_pci_id mlx4_pci_id_map[] = {
@@ -954,9 +967,7 @@ glue_error:
 /**
  * Driver initialization routine.
  */
-RTE_INIT(rte_mlx4_pmd_init);
-static void
-rte_mlx4_pmd_init(void)
+RTE_INIT(rte_mlx4_pmd_init)
 {
 	/*
 	 * MLX4_DEVICE_FATAL_CLEANUP tells ibv_destroy functions we
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 300cb4d7..e6fb934f 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -47,6 +47,9 @@
 /** Interrupt alarm timeout value in microseconds. */
 #define MLX4_INTR_ALARM_TIMEOUT 100000
 
+/* Maximum packet headers size (L2+L3+L4) for TSO. */
+#define MLX4_MAX_TSO_HEADER 192
+
 /** Port parameter. */
 #define MLX4_PMD_PORT_KVARG "port"
 
@@ -90,6 +93,9 @@ struct priv {
 	uint32_t hw_csum:1; /**< Checksum offload is supported. */
 	uint32_t hw_csum_l2tun:1; /**< Checksum support for L2 tunnels. */
 	uint32_t hw_fcs_strip:1; /**< FCS stripping toggling is supported. */
+	uint32_t tso:1; /**< Transmit segmentation offload is supported. */
+	uint32_t tso_max_payload_sz; /**< Max supported TSO payload size. */
+	uint32_t hw_rss_max_qps; /**< Max Rx Queues supported by RSS. */
 	uint64_t hw_rss_sup; /**< Supported RSS hash fields (Verbs format). */
 	struct rte_intr_handle intr_handle; /**< Port interrupt handle. */
 	struct mlx4_drop *drop; /**< Shared resources for drop flow rules. */
diff --git a/drivers/net/mlx4/mlx4_prm.h b/drivers/net/mlx4/mlx4_prm.h
index e15a3c14..aef77ba0 100644
--- a/drivers/net/mlx4/mlx4_prm.h
+++ b/drivers/net/mlx4/mlx4_prm.h
@@ -19,6 +19,7 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
+#include "mlx4_autoconf.h"
 
 /* ConnectX-3 Tx queue basic block. */
 #define MLX4_TXBB_SHIFT 6
@@ -40,6 +41,7 @@
 /* Work queue element (WQE) flags. */
 #define MLX4_WQE_CTRL_IIP_HDR_CSUM (1 << 28)
 #define MLX4_WQE_CTRL_IL4_HDR_CSUM (1 << 27)
+#define MLX4_WQE_CTRL_RR (1 << 6)
 
 /* CQE checksum flags. */
 enum {
@@ -51,6 +53,7 @@ enum {
 };
 
 /* CQE status flags. */
+#define MLX4_CQE_STATUS_IPV6F (1 << 12)
 #define MLX4_CQE_STATUS_IPV4 (1 << 22)
 #define MLX4_CQE_STATUS_IPV4F (1 << 23)
 #define MLX4_CQE_STATUS_IPV6 (1 << 24)
@@ -97,6 +100,19 @@ struct mlx4_cq {
 	int arm_sn; /**< Rx event counter. */
 };
 
+#ifndef HAVE_IBV_MLX4_WQE_LSO_SEG
+/*
+ * WQE LSO segment structure.
+ * Defined here as backward compatibility for rdma-core v17 and below.
+ * Similar definition is found in infiniband/mlx4dv.h in rdma-core v18
+ * and above.
+ */
+struct mlx4_wqe_lso_seg {
+	rte_be32_t mss_hdr_size;
+	rte_be32_t header[];
+};
+#endif
+
 /**
  * Retrieve a CQE entry from a CQ.
  *
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 87688c1c..9737da2e 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -338,6 +338,12 @@ mlx4_rss_init(struct priv *priv)
 
 	if (priv->rss_init)
 		return 0;
+	if (priv->dev->data->nb_rx_queues > priv->hw_rss_max_qps) {
+		ERROR("RSS does not support more than %d queues",
+		      priv->hw_rss_max_qps);
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
 	/* Prepare range for RSS contexts before creating the first WQ. */
 	ret = mlx4_glue->dv_set_context_attr
 		(priv->ctx,
@@ -672,7 +678,9 @@ uint64_t
 mlx4_get_rx_queue_offloads(struct priv *priv)
 {
 	uint64_t offloads = DEV_RX_OFFLOAD_SCATTER |
-			    DEV_RX_OFFLOAD_CRC_STRIP;
+			    DEV_RX_OFFLOAD_CRC_STRIP |
+			    DEV_RX_OFFLOAD_KEEP_CRC |
+			    DEV_RX_OFFLOAD_JUMBO_FRAME;
 
 	if (priv->hw_csum)
 		offloads |= DEV_RX_OFFLOAD_CHECKSUM;
@@ -771,16 +779,16 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		     (void *)dev, idx, desc);
 	}
 	/* By default, FCS (CRC) is stripped by hardware. */
-	if (offloads & DEV_RX_OFFLOAD_CRC_STRIP) {
-		crc_present = 0;
-	} else if (priv->hw_fcs_strip) {
-		crc_present = 1;
-	} else {
-		WARN("%p: CRC stripping has been disabled but will still"
-		     " be performed by hardware, make sure MLNX_OFED and"
-		     " firmware are up to date",
-		     (void *)dev);
-		crc_present = 0;
+	crc_present = 0;
+	if (rte_eth_dev_must_keep_crc(offloads)) {
+		if (priv->hw_fcs_strip) {
+			crc_present = 1;
+		} else {
+			WARN("%p: CRC stripping has been disabled but will still"
+			     " be performed by hardware, make sure MLNX_OFED and"
+			     " firmware are up to date",
+			     (void *)dev);
+		}
 	}
 	DEBUG("%p: CRC stripping is %s, %u bytes will be subtracted from"
 	      " incoming frames to hide it",
diff --git a/drivers/net/mlx4/mlx4_rxtx.c b/drivers/net/mlx4/mlx4_rxtx.c
index a92da66b..8c88effc 100644
--- a/drivers/net/mlx4/mlx4_rxtx.c
+++ b/drivers/net/mlx4/mlx4_rxtx.c
@@ -38,10 +38,29 @@
  * DWORD (32 byte) of a TXBB.
  */
 struct pv {
-	volatile struct mlx4_wqe_data_seg *dseg;
+	union {
+		volatile struct mlx4_wqe_data_seg *dseg;
+		volatile uint32_t *dst;
+	};
 	uint32_t val;
 };
 
+/** A helper structure for TSO packet handling. */
+struct tso_info {
+	/** Pointer to the array of saved first DWORD (32 byte) of a TXBB. */
+	struct pv *pv;
+	/** Current entry in the pv array. */
+	int pv_counter;
+	/** Total size of the WQE including padding. */
+	uint32_t wqe_size;
+	/** Size of TSO header to prepend to each packet to send. */
+	uint16_t tso_header_size;
+	/** Total size of the TSO segment in the WQE. */
+	uint16_t wqe_tso_seg_size;
+	/** Raw WQE size in units of 16 Bytes and without padding. */
+	uint8_t fence_size;
+};
+
 /** A table to translate Rx completion flags to packet type. */
 uint32_t mlx4_ptype_table[0x100] __rte_cache_aligned = {
 	/*
@@ -52,49 +71,58 @@ uint32_t mlx4_ptype_table[0x100] __rte_cache_aligned = {
 	 *  bit[4] - MLX4_CQE_STATUS_TCP
 	 *  bit[3] - MLX4_CQE_STATUS_IPV4OPT
 	 *  bit[2] - MLX4_CQE_STATUS_IPV6
-	 *  bit[1] - MLX4_CQE_STATUS_IPV4F
+	 *  bit[1] - MLX4_CQE_STATUS_IPF
 	 *  bit[0] - MLX4_CQE_STATUS_IPV4
 	 * giving a total of up to 256 entries.
 	 */
+	/* L2 */
 	[0x00] = RTE_PTYPE_L2_ETHER,
+	/* L3 */
 	[0x01] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_L4_NONFRAG,
 	[0x02] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_L4_FRAG,
 	[0x03] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_L4_FRAG,
-	[0x04] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
-	[0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT,
+	[0x04] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_NONFRAG,
+	[0x06] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_FRAG,
+	[0x08] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
+		     RTE_PTYPE_L4_NONFRAG,
+	[0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
+		     RTE_PTYPE_L4_NONFRAG,
 	[0x0a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
 		     RTE_PTYPE_L4_FRAG,
+	[0x0b] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
+		     RTE_PTYPE_L4_FRAG,
+	/* TCP */
 	[0x11] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_L4_TCP,
-	[0x12] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_L4_TCP,
 	[0x14] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_L4_TCP,
+	[0x16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_FRAG,
 	[0x18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
 		     RTE_PTYPE_L4_TCP,
 	[0x19] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
 		     RTE_PTYPE_L4_TCP,
-	[0x1a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
-		     RTE_PTYPE_L4_TCP,
+	/* UDP */
 	[0x21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_L4_UDP,
-	[0x22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_L4_UDP,
 	[0x24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_L4_UDP,
+	[0x26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_L4_FRAG,
 	[0x28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
 		     RTE_PTYPE_L4_UDP,
 	[0x29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
 		     RTE_PTYPE_L4_UDP,
-	[0x2a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
-		     RTE_PTYPE_L4_UDP,
 	/* Tunneled - L3 IPV6 */
 	[0x80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
 	[0x81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_NONFRAG,
 	[0x82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_FRAG,
@@ -102,65 +130,58 @@ uint32_t mlx4_ptype_table[0x100] __rte_cache_aligned = {
 		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_FRAG,
 	[0x84] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_NONFRAG,
+	[0x86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_FRAG,
 	[0x88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT,
+		     RTE_PTYPE_INNER_L3_IPV4_EXT |
+		     RTE_PTYPE_INNER_L4_NONFRAG,
 	[0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT,
+		     RTE_PTYPE_INNER_L3_IPV4_EXT |
+		     RTE_PTYPE_INNER_L4_NONFRAG,
 	[0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG,
+		     RTE_PTYPE_INNER_L3_IPV4_EXT |
+		     RTE_PTYPE_INNER_L4_FRAG,
+	[0x8b] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT |
+		     RTE_PTYPE_INNER_L4_FRAG,
 	/* Tunneled - L3 IPV6, TCP */
 	[0x91] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_TCP,
-	[0x92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L4_FRAG |
-		     RTE_PTYPE_INNER_L4_TCP,
-	[0x93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L4_FRAG |
-		     RTE_PTYPE_INNER_L4_TCP,
 	[0x94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_TCP,
+	[0x96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_FRAG,
 	[0x98] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT |
-		     RTE_PTYPE_INNER_L4_TCP,
+		     RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP,
 	[0x99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT |
-		     RTE_PTYPE_INNER_L4_TCP,
-	[0x9a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG |
-		     RTE_PTYPE_INNER_L4_TCP,
+		     RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP,
 	/* Tunneled - L3 IPV6, UDP */
-	[0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+	[0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_UDP,
-	[0xa2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L4_FRAG |
-		     RTE_PTYPE_INNER_L4_UDP,
-	[0xa3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L4_FRAG |
-		     RTE_PTYPE_INNER_L4_UDP,
-	[0xa4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+	[0xa4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_UDP,
-	[0xa8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+	[0xa6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_FRAG,
+	[0xa8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT |
 		     RTE_PTYPE_INNER_L4_UDP,
-	[0xa9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+	[0xa9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT |
 		     RTE_PTYPE_INNER_L4_UDP,
-	[0xaa] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG |
-		     RTE_PTYPE_INNER_L4_UDP,
 	/* Tunneled - L3 IPV4 */
 	[0xc0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 	[0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_NONFRAG,
 	[0xc2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_FRAG,
@@ -168,65 +189,54 @@ uint32_t mlx4_ptype_table[0x100] __rte_cache_aligned = {
 		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_FRAG,
 	[0xc4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_NONFRAG,
+	[0xc6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_FRAG,
 	[0xc8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT,
+		     RTE_PTYPE_INNER_L3_IPV4_EXT |
+		     RTE_PTYPE_INNER_L4_NONFRAG,
 	[0xc9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT,
+		     RTE_PTYPE_INNER_L3_IPV4_EXT |
+		     RTE_PTYPE_INNER_L4_NONFRAG,
 	[0xca] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT |
 		     RTE_PTYPE_INNER_L4_FRAG,
+	[0xcb] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT |
+		     RTE_PTYPE_INNER_L4_FRAG,
 	/* Tunneled - L3 IPV4, TCP */
-	[0xd0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L4_TCP,
 	[0xd1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_TCP,
-	[0xd2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L4_FRAG |
-		     RTE_PTYPE_INNER_L4_TCP,
-	[0xd3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L4_FRAG |
-		     RTE_PTYPE_INNER_L4_TCP,
 	[0xd4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_TCP,
+	[0xd6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_FRAG,
 	[0xd8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT |
 		     RTE_PTYPE_INNER_L4_TCP,
 	[0xd9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT |
 		     RTE_PTYPE_INNER_L4_TCP,
-	[0xda] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG |
-		     RTE_PTYPE_INNER_L4_TCP,
 	/* Tunneled - L3 IPV4, UDP */
-	[0xe0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L4_UDP,
 	[0xe1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_UDP,
-	[0xe2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L4_FRAG |
-		     RTE_PTYPE_INNER_L4_UDP,
-	[0xe3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L4_FRAG |
-		     RTE_PTYPE_INNER_L4_UDP,
 	[0xe4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_UDP,
+	[0xe6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+		     RTE_PTYPE_INNER_L4_FRAG,
 	[0xe8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP,
+		     RTE_PTYPE_INNER_L3_IPV4_EXT |
+		     RTE_PTYPE_INNER_L4_UDP,
 	[0xe9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP,
-	[0xea] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
-		     RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG |
+		     RTE_PTYPE_INNER_L3_IPV4_EXT |
 		     RTE_PTYPE_INNER_L4_UDP,
 };
 
@@ -377,6 +387,342 @@ mlx4_fill_tx_data_seg(volatile struct mlx4_wqe_data_seg *dseg,
 }
 
 /**
+ * Obtain and calculate TSO information needed for assembling a TSO WQE.
+ *
+ * @param buf
+ *   Pointer to the first packet mbuf.
+ * @param txq
+ *   Pointer to Tx queue structure.
+ * @param tinfo
+ *   Pointer to a structure to fill the info with.
+ *
+ * @return
+ *   0 on success, negative value upon error.
+ */
+static inline int
+mlx4_tx_burst_tso_get_params(struct rte_mbuf *buf,
+			     struct txq *txq,
+			     struct tso_info *tinfo)
+{
+	struct mlx4_sq *sq = &txq->msq;
+	const uint8_t tunneled = txq->priv->hw_csum_l2tun &&
+				 (buf->ol_flags & PKT_TX_TUNNEL_MASK);
+
+	tinfo->tso_header_size = buf->l2_len + buf->l3_len + buf->l4_len;
+	if (tunneled)
+		tinfo->tso_header_size +=
+				buf->outer_l2_len + buf->outer_l3_len;
+	if (unlikely(buf->tso_segsz == 0 ||
+		     tinfo->tso_header_size == 0 ||
+		     tinfo->tso_header_size > MLX4_MAX_TSO_HEADER ||
+		     tinfo->tso_header_size > buf->data_len))
+		return -EINVAL;
+	/*
+	 * Calculate the WQE TSO segment size
+	 * Note:
+	 * 1. An LSO segment must be padded such that the subsequent data
+	 *    segment is 16-byte aligned.
+	 * 2. The start address of the TSO segment is always 16 Bytes aligned.
+	 */
+	tinfo->wqe_tso_seg_size = RTE_ALIGN(sizeof(struct mlx4_wqe_lso_seg) +
+					    tinfo->tso_header_size,
+					    sizeof(struct mlx4_wqe_data_seg));
+	tinfo->fence_size = ((sizeof(struct mlx4_wqe_ctrl_seg) +
+			     tinfo->wqe_tso_seg_size) >> MLX4_SEG_SHIFT) +
+			     buf->nb_segs;
+	tinfo->wqe_size =
+		RTE_ALIGN((uint32_t)(tinfo->fence_size << MLX4_SEG_SHIFT),
+			  MLX4_TXBB_SIZE);
+	/* Validate WQE size and WQE space in the send queue. */
+	if (sq->remain_size < tinfo->wqe_size ||
+	    tinfo->wqe_size > MLX4_MAX_WQE_SIZE)
+		return -ENOMEM;
+	/* Init pv. */
+	tinfo->pv = (struct pv *)txq->bounce_buf;
+	tinfo->pv_counter = 0;
+	return 0;
+}
+
+/**
+ * Fill the TSO WQE data segments with info on buffers to transmit .
+ *
+ * @param buf
+ *   Pointer to the first packet mbuf.
+ * @param txq
+ *   Pointer to Tx queue structure.
+ * @param tinfo
+ *   Pointer to TSO info to use.
+ * @param dseg
+ *   Pointer to the first data segment in the TSO WQE.
+ * @param ctrl
+ *   Pointer to the control segment in the TSO WQE.
+ *
+ * @return
+ *   0 on success, negative value upon error.
+ */
+static inline volatile struct mlx4_wqe_ctrl_seg *
+mlx4_tx_burst_fill_tso_dsegs(struct rte_mbuf *buf,
+			     struct txq *txq,
+			     struct tso_info *tinfo,
+			     volatile struct mlx4_wqe_data_seg *dseg,
+			     volatile struct mlx4_wqe_ctrl_seg *ctrl)
+{
+	uint32_t lkey;
+	int nb_segs = buf->nb_segs;
+	int nb_segs_txbb;
+	struct mlx4_sq *sq = &txq->msq;
+	struct rte_mbuf *sbuf = buf;
+	struct pv *pv = tinfo->pv;
+	int *pv_counter = &tinfo->pv_counter;
+	volatile struct mlx4_wqe_ctrl_seg *ctrl_next =
+			(volatile struct mlx4_wqe_ctrl_seg *)
+				((volatile uint8_t *)ctrl + tinfo->wqe_size);
+	uint16_t data_len = sbuf->data_len - tinfo->tso_header_size;
+	uintptr_t data_addr = rte_pktmbuf_mtod_offset(sbuf, uintptr_t,
+						      tinfo->tso_header_size);
+
+	do {
+		/* how many dseg entries do we have in the current TXBB ? */
+		nb_segs_txbb = (MLX4_TXBB_SIZE -
+				((uintptr_t)dseg & (MLX4_TXBB_SIZE - 1))) >>
+			       MLX4_SEG_SHIFT;
+		switch (nb_segs_txbb) {
+#ifndef NDEBUG
+		default:
+			/* Should never happen. */
+			rte_panic("%p: Invalid number of SGEs(%d) for a TXBB",
+			(void *)txq, nb_segs_txbb);
+			/* rte_panic never returns. */
+			break;
+#endif /* NDEBUG */
+		case 4:
+			/* Memory region key for this memory pool. */
+			lkey = mlx4_tx_mb2mr(txq, sbuf);
+			if (unlikely(lkey == (uint32_t)-1))
+				goto err;
+			dseg->addr = rte_cpu_to_be_64(data_addr);
+			dseg->lkey = lkey;
+			/*
+			 * This data segment starts at the beginning of a new
+			 * TXBB, so we need to postpone its byte_count writing
+			 * for later.
+			 */
+			pv[*pv_counter].dseg = dseg;
+			/*
+			 * Zero length segment is treated as inline segment
+			 * with zero data.
+			 */
+			pv[(*pv_counter)++].val =
+				rte_cpu_to_be_32(data_len ?
+						 data_len :
+						 0x80000000);
+			if (--nb_segs == 0)
+				return ctrl_next;
+			/* Prepare next buf info */
+			sbuf = sbuf->next;
+			dseg++;
+			data_len = sbuf->data_len;
+			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
+			/* fallthrough */
+		case 3:
+			lkey = mlx4_tx_mb2mr(txq, sbuf);
+			if (unlikely(lkey == (uint32_t)-1))
+				goto err;
+			mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
+					rte_cpu_to_be_32(data_len ?
+							 data_len :
+							 0x80000000));
+			if (--nb_segs == 0)
+				return ctrl_next;
+			/* Prepare next buf info */
+			sbuf = sbuf->next;
+			dseg++;
+			data_len = sbuf->data_len;
+			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
+			/* fallthrough */
+		case 2:
+			lkey = mlx4_tx_mb2mr(txq, sbuf);
+			if (unlikely(lkey == (uint32_t)-1))
+				goto err;
+			mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
+					rte_cpu_to_be_32(data_len ?
+							 data_len :
+							 0x80000000));
+			if (--nb_segs == 0)
+				return ctrl_next;
+			/* Prepare next buf info */
+			sbuf = sbuf->next;
+			dseg++;
+			data_len = sbuf->data_len;
+			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
+			/* fallthrough */
+		case 1:
+			lkey = mlx4_tx_mb2mr(txq, sbuf);
+			if (unlikely(lkey == (uint32_t)-1))
+				goto err;
+			mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
+					rte_cpu_to_be_32(data_len ?
+							 data_len :
+							 0x80000000));
+			if (--nb_segs == 0)
+				return ctrl_next;
+			/* Prepare next buf info */
+			sbuf = sbuf->next;
+			dseg++;
+			data_len = sbuf->data_len;
+			data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
+			/* fallthrough */
+		}
+		/* Wrap dseg if it points at the end of the queue. */
+		if ((volatile uint8_t *)dseg >= sq->eob)
+			dseg = (volatile struct mlx4_wqe_data_seg *)
+					((volatile uint8_t *)dseg - sq->size);
+	} while (true);
+err:
+	return NULL;
+}
+
+/**
+ * Fill the packet's l2, l3 and l4 headers to the WQE.
+ *
+ * This will be used as the header for each TSO segment that is transmitted.
+ *
+ * @param buf
+ *   Pointer to the first packet mbuf.
+ * @param txq
+ *   Pointer to Tx queue structure.
+ * @param tinfo
+ *   Pointer to TSO info to use.
+ * @param ctrl
+ *   Pointer to the control segment in the TSO WQE.
+ *
+ * @return
+ *   0 on success, negative value upon error.
+ */
+static inline volatile struct mlx4_wqe_data_seg *
+mlx4_tx_burst_fill_tso_hdr(struct rte_mbuf *buf,
+			   struct txq *txq,
+			   struct tso_info *tinfo,
+			   volatile struct mlx4_wqe_ctrl_seg *ctrl)
+{
+	volatile struct mlx4_wqe_lso_seg *tseg =
+		(volatile struct mlx4_wqe_lso_seg *)(ctrl + 1);
+	struct mlx4_sq *sq = &txq->msq;
+	struct pv *pv = tinfo->pv;
+	int *pv_counter = &tinfo->pv_counter;
+	int remain_size = tinfo->tso_header_size;
+	char *from = rte_pktmbuf_mtod(buf, char *);
+	uint16_t txbb_avail_space;
+	/* Union to overcome volatile constraints when copying TSO header. */
+	union {
+		volatile uint8_t *vto;
+		uint8_t *to;
+	} thdr = { .vto = (volatile uint8_t *)tseg->header, };
+
+	/*
+	 * TSO data always starts at offset 20 from the beginning of the TXBB
+	 * (16 byte ctrl + 4byte TSO desc). Since each TXBB is 64Byte aligned
+	 * we can write the first 44 TSO header bytes without worry for TxQ
+	 * wrapping or overwriting the first TXBB 32bit word.
+	 */
+	txbb_avail_space = MLX4_TXBB_SIZE -
+			   (sizeof(struct mlx4_wqe_ctrl_seg) +
+			    sizeof(struct mlx4_wqe_lso_seg));
+	while (remain_size >= (int)(txbb_avail_space + sizeof(uint32_t))) {
+		/* Copy to end of txbb. */
+		rte_memcpy(thdr.to, from, txbb_avail_space);
+		from += txbb_avail_space;
+		thdr.to += txbb_avail_space;
+		/* New TXBB, Check for TxQ wrap. */
+		if (thdr.to >= sq->eob)
+			thdr.vto = sq->buf;
+		/* New TXBB, stash the first 32bits for later use. */
+		pv[*pv_counter].dst = (volatile uint32_t *)thdr.to;
+		pv[(*pv_counter)++].val = *(uint32_t *)from,
+		from += sizeof(uint32_t);
+		thdr.to += sizeof(uint32_t);
+		remain_size -= txbb_avail_space + sizeof(uint32_t);
+		/* Avail space in new TXBB is TXBB size - 4 */
+		txbb_avail_space = MLX4_TXBB_SIZE - sizeof(uint32_t);
+	}
+	if (remain_size > txbb_avail_space) {
+		rte_memcpy(thdr.to, from, txbb_avail_space);
+		from += txbb_avail_space;
+		thdr.to += txbb_avail_space;
+		remain_size -= txbb_avail_space;
+		/* New TXBB, Check for TxQ wrap. */
+		if (thdr.to >= sq->eob)
+			thdr.vto = sq->buf;
+		pv[*pv_counter].dst = (volatile uint32_t *)thdr.to;
+		rte_memcpy(&pv[*pv_counter].val, from, remain_size);
+		(*pv_counter)++;
+	} else if (remain_size) {
+		rte_memcpy(thdr.to, from, remain_size);
+	}
+	tseg->mss_hdr_size = rte_cpu_to_be_32((buf->tso_segsz << 16) |
+					      tinfo->tso_header_size);
+	/* Calculate data segment location */
+	return (volatile struct mlx4_wqe_data_seg *)
+				((uintptr_t)tseg + tinfo->wqe_tso_seg_size);
+}
+
+/**
+ * Write data segments and header for TSO uni/multi segment packet.
+ *
+ * @param buf
+ *   Pointer to the first packet mbuf.
+ * @param txq
+ *   Pointer to Tx queue structure.
+ * @param ctrl
+ *   Pointer to the WQE control segment.
+ *
+ * @return
+ *   Pointer to the next WQE control segment on success, NULL otherwise.
+ */
+static volatile struct mlx4_wqe_ctrl_seg *
+mlx4_tx_burst_tso(struct rte_mbuf *buf, struct txq *txq,
+		  volatile struct mlx4_wqe_ctrl_seg *ctrl)
+{
+	volatile struct mlx4_wqe_data_seg *dseg;
+	volatile struct mlx4_wqe_ctrl_seg *ctrl_next;
+	struct mlx4_sq *sq = &txq->msq;
+	struct tso_info tinfo;
+	struct pv *pv;
+	int pv_counter;
+	int ret;
+
+	ret = mlx4_tx_burst_tso_get_params(buf, txq, &tinfo);
+	if (unlikely(ret))
+		goto error;
+	dseg = mlx4_tx_burst_fill_tso_hdr(buf, txq, &tinfo, ctrl);
+	if (unlikely(dseg == NULL))
+		goto error;
+	if ((uintptr_t)dseg >= (uintptr_t)sq->eob)
+		dseg = (volatile struct mlx4_wqe_data_seg *)
+					((uintptr_t)dseg - sq->size);
+	ctrl_next = mlx4_tx_burst_fill_tso_dsegs(buf, txq, &tinfo, dseg, ctrl);
+	if (unlikely(ctrl_next == NULL))
+		goto error;
+	/* Write the first DWORD of each TXBB save earlier. */
+	if (likely(tinfo.pv_counter)) {
+		pv = tinfo.pv;
+		pv_counter = tinfo.pv_counter;
+		/* Need a barrier here before writing the first TXBB word. */
+		rte_io_wmb();
+		do {
+			--pv_counter;
+			*pv[pv_counter].dst = pv[pv_counter].val;
+		} while (pv_counter > 0);
+	}
+	ctrl->fence_size = tinfo.fence_size;
+	sq->remain_size -= tinfo.wqe_size;
+	return ctrl_next;
+error:
+	txq->stats.odropped++;
+	return NULL;
+}
+
+/**
  * Write data segments of multi-segment packet.
  *
  * @param buf
@@ -569,6 +915,7 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			uint16_t flags16[2];
 		} srcrb;
 		uint32_t lkey;
+		bool tso = txq->priv->tso && (buf->ol_flags & PKT_TX_TCP_SEG);
 
 		/* Clean up old buffer. */
 		if (likely(elt->buf != NULL)) {
@@ -587,7 +934,16 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			} while (tmp != NULL);
 		}
 		RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf);
-		if (buf->nb_segs == 1) {
+		if (tso) {
+			/* Change opcode to TSO */
+			owner_opcode &= ~MLX4_OPCODE_CONFIG_CMD;
+			owner_opcode |= MLX4_OPCODE_LSO | MLX4_WQE_CTRL_RR;
+			ctrl_next = mlx4_tx_burst_tso(buf, txq, ctrl);
+			if (!ctrl_next) {
+				elt->buf = NULL;
+				break;
+			}
+		} else if (buf->nb_segs == 1) {
 			/* Validate WQE space in the send queue. */
 			if (sq->remain_size < MLX4_TXBB_SIZE) {
 				elt->buf = NULL;
@@ -728,11 +1084,13 @@ rxq_cq_to_pkt_type(volatile struct mlx4_cqe *cqe,
 	 *  bit[4] - MLX4_CQE_STATUS_TCP
 	 *  bit[3] - MLX4_CQE_STATUS_IPV4OPT
 	 *  bit[2] - MLX4_CQE_STATUS_IPV6
-	 *  bit[1] - MLX4_CQE_STATUS_IPV4F
+	 *  bit[1] - MLX4_CQE_STATUS_IPF
 	 *  bit[0] - MLX4_CQE_STATUS_IPV4
 	 * giving a total of up to 256 entries.
 	 */
 	idx |= ((status & MLX4_CQE_STATUS_PTYPE_MASK) >> 22);
+	if (status & MLX4_CQE_STATUS_IPV6)
+		idx |= ((status & MLX4_CQE_STATUS_IPV6F) >> 11);
 	return mlx4_ptype_table[idx];
 }
 
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index 4c025e3a..ffa8abfc 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -90,7 +90,7 @@ struct mlx4_txq_stats {
 	unsigned int idx; /**< Mapping index. */
 	uint64_t opackets; /**< Total of successfully sent packets. */
 	uint64_t obytes; /**< Total of successfully sent bytes. */
-	uint64_t odropped; /**< Total of packets not sent when Tx ring full. */
+	uint64_t odropped; /**< Total number of packets failed to transmit. */
 };
 
 /** Tx queue descriptor. */
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index 6edaadbb..9aa7440d 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -116,8 +116,14 @@ mlx4_get_tx_port_offloads(struct priv *priv)
 			     DEV_TX_OFFLOAD_UDP_CKSUM |
 			     DEV_TX_OFFLOAD_TCP_CKSUM);
 	}
-	if (priv->hw_csum_l2tun)
+	if (priv->tso)
+		offloads |= DEV_TX_OFFLOAD_TCP_TSO;
+	if (priv->hw_csum_l2tun) {
 		offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
+		if (priv->tso)
+			offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
+				     DEV_TX_OFFLOAD_GRE_TNL_TSO);
+	}
 	return offloads;
 }
 
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 8a5229e6..2e70dec5 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -33,6 +33,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_nl.c
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_nl_flow.c
 
 ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
 INSTALL-$(CONFIG_RTE_LIBRTE_MLX5_PMD)-lib += $(LIB_GLUE)
@@ -56,6 +57,7 @@ LDLIBS += -ldl
 else
 LDLIBS += -libverbs -lmlx5
 endif
+LDLIBS += -lmnl
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
@@ -100,7 +102,7 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT \
 		infiniband/mlx5dv.h \
-		enum MLX5DV_CONTEXT_MASK_STRIDING_RQ \
+		enum MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_TUNNEL_SUPPORT \
@@ -150,7 +152,237 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
 		infiniband/verbs.h \
-		enum IBV_FLOW_SPEC_ACTION_COUNT \
+		type 'struct ibv_counter_set_init_attr' \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NL_NLDEV \
+		rdma/rdma_netlink.h \
+		enum RDMA_NL_NLDEV \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_CMD_GET \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_CMD_GET \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_CMD_PORT_GET \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_CMD_PORT_GET \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_ATTR_DEV_INDEX \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_ATTR_DEV_INDEX \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_ATTR_DEV_NAME \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_ATTR_DEV_NAME \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_ATTR_PORT_INDEX \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_ATTR_PORT_INDEX \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_RDMA_NLDEV_ATTR_NDEV_INDEX \
+		rdma/rdma_netlink.h \
+		enum RDMA_NLDEV_ATTR_NDEV_INDEX \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IFLA_PHYS_SWITCH_ID \
+		linux/if_link.h \
+		enum IFLA_PHYS_SWITCH_ID \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IFLA_PHYS_PORT_NAME \
+		linux/if_link.h \
+		enum IFLA_PHYS_PORT_NAME \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_ACT \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_ACT \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_FLAGS \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_FLAGS \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_ETH_TYPE \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_ETH_TYPE \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_ETH_DST \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_ETH_DST \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_ETH_DST_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_ETH_DST_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_ETH_SRC \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_ETH_SRC \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_ETH_SRC_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IP_PROTO \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IP_PROTO \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV4_SRC \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV4_SRC \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV4_SRC_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV4_DST \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV4_DST \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV4_DST_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV6_SRC \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV6_SRC \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV6_SRC_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV6_DST \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV6_DST \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_IPV6_DST_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_TCP_SRC \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_TCP_SRC \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_TCP_SRC_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_TCP_DST \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_TCP_DST \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_TCP_DST_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_TCP_DST_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_UDP_SRC \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_UDP_SRC \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_UDP_SRC_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_UDP_DST \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_UDP_DST \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_UDP_DST_MASK \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_UDP_DST_MASK \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_VLAN_ID \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_VLAN_ID \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_VLAN_PRIO \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_VLAN_PRIO \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE \
+		linux/pkt_cls.h \
+		enum TCA_FLOWER_KEY_VLAN_ETH_TYPE \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_TC_ACT_VLAN \
+		linux/tc_act/tc_vlan.h \
+		enum TCA_VLAN_PUSH_VLAN_PRIORITY \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_40000baseKR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_40000baseKR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_40000baseCR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_40000baseCR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_40000baseSR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_40000baseSR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_40000baseLR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_40000baseLR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_56000baseKR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_56000baseKR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_56000baseCR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_56000baseCR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_56000baseSR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_56000baseSR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_SUPPORTED_56000baseLR4_Full \
+		/usr/include/linux/ethtool.h \
+		define SUPPORTED_56000baseLR4_Full \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_STATIC_ASSERT \
+		/usr/include/assert.h \
+		define static_assert \
 		$(AUTOCONF_OUTPUT)
 
 # Create mlx5_autoconf.h or update it in case it differs from the new one.
@@ -177,7 +409,7 @@ endif
 $(LIB_GLUE): mlx5_glue.o
 	$Q $(LD) $(GLUE_LDFLAGS) $(EXTRA_LDFLAGS) \
 		-Wl,-h,$(LIB_GLUE) \
-		-s -shared -o $@ $< -libverbs -lmlx5
+		-shared -o $@ $< -libverbs -lmlx5
 
 mlx5_glue.o: mlx5_autoconf.h
 
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index c933e274..ec63bc6e 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -13,6 +13,7 @@
 #include <errno.h>
 #include <net/if.h>
 #include <sys/mman.h>
+#include <linux/netlink.h>
 #include <linux/rtnetlink.h>
 
 /* Verbs header. */
@@ -36,6 +37,7 @@
 #include <rte_kvargs.h>
 #include <rte_rwlock.h>
 #include <rte_spinlock.h>
+#include <rte_string_fns.h>
 
 #include "mlx5.h"
 #include "mlx5_utils.h"
@@ -90,6 +92,9 @@
 /* Activate Netlink support in VF mode. */
 #define MLX5_VF_NL_EN "vf_nl_en"
 
+/* Select port representors to instantiate. */
+#define MLX5_REPRESENTOR "representor"
+
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -237,6 +242,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	/* In case mlx5_dev_stop() has not been called. */
 	mlx5_dev_interrupt_handler_uninstall(dev);
 	mlx5_traffic_disable(dev);
+	mlx5_flow_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
 	dev->rx_pkt_burst = removed_rx_burst;
 	dev->tx_pkt_burst = removed_tx_burst;
@@ -256,7 +262,6 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 		priv->txqs_n = 0;
 		priv->txqs = NULL;
 	}
-	mlx5_flow_delete_drop_queue(dev);
 	mlx5_mprq_free_mp(dev);
 	mlx5_mr_release(dev);
 	if (priv->pd != NULL) {
@@ -273,8 +278,12 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 		mlx5_socket_uninit(dev);
 	if (priv->config.vf)
 		mlx5_nl_mac_addr_flush(dev);
-	if (priv->nl_socket >= 0)
-		close(priv->nl_socket);
+	if (priv->nl_socket_route >= 0)
+		close(priv->nl_socket_route);
+	if (priv->nl_socket_rdma >= 0)
+		close(priv->nl_socket_rdma);
+	if (priv->mnl_socket)
+		mlx5_nl_flow_socket_destroy(priv->mnl_socket);
 	ret = mlx5_hrxq_ibv_verify(dev);
 	if (ret)
 		DRV_LOG(WARNING, "port %u some hash Rx queue still remain",
@@ -303,7 +312,27 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
+	if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
+		unsigned int c = 0;
+		unsigned int i = mlx5_dev_to_port_id(dev->device, NULL, 0);
+		uint16_t port_id[i];
+
+		i = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, i), i);
+		while (i--) {
+			struct priv *opriv =
+				rte_eth_devices[port_id[i]].data->dev_private;
+
+			if (!opriv ||
+			    opriv->domain_id != priv->domain_id ||
+			    &rte_eth_devices[port_id[i]] == dev)
+				continue;
+			++c;
+		}
+		if (!c)
+			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
+	}
 	memset(priv, 0, sizeof(*priv));
+	priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
 }
 
 const struct eth_dev_ops mlx5_dev_ops = {
@@ -370,6 +399,10 @@ const struct eth_dev_ops mlx5_dev_ops_isolate = {
 	.dev_set_link_down = mlx5_set_link_down,
 	.dev_set_link_up = mlx5_set_link_up,
 	.dev_close = mlx5_dev_close,
+	.promiscuous_enable = mlx5_promiscuous_enable,
+	.promiscuous_disable = mlx5_promiscuous_disable,
+	.allmulticast_enable = mlx5_allmulticast_enable,
+	.allmulticast_disable = mlx5_allmulticast_disable,
 	.link_update = mlx5_link_update,
 	.stats_get = mlx5_stats_get,
 	.stats_reset = mlx5_stats_reset,
@@ -400,39 +433,6 @@ const struct eth_dev_ops mlx5_dev_ops_isolate = {
 	.is_removed = mlx5_is_removed,
 };
 
-static struct {
-	struct rte_pci_addr pci_addr; /* associated PCI address */
-	uint32_t ports; /* physical ports bitfield. */
-} mlx5_dev[32];
-
-/**
- * Get device index in mlx5_dev[] from PCI bus address.
- *
- * @param[in] pci_addr
- *   PCI bus address to look for.
- *
- * @return
- *   mlx5_dev[] index on success, -1 on failure.
- */
-static int
-mlx5_dev_idx(struct rte_pci_addr *pci_addr)
-{
-	unsigned int i;
-	int ret = -1;
-
-	assert(pci_addr != NULL);
-	for (i = 0; (i != RTE_DIM(mlx5_dev)); ++i) {
-		if ((mlx5_dev[i].pci_addr.domain == pci_addr->domain) &&
-		    (mlx5_dev[i].pci_addr.bus == pci_addr->bus) &&
-		    (mlx5_dev[i].pci_addr.devid == pci_addr->devid) &&
-		    (mlx5_dev[i].pci_addr.function == pci_addr->function))
-			return i;
-		if ((mlx5_dev[i].ports == 0) && (ret == -1))
-			ret = i;
-	}
-	return ret;
-}
-
 /**
  * Verify and store value for device argument.
  *
@@ -452,6 +452,9 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 	struct mlx5_dev_config *config = opaque;
 	unsigned long tmp;
 
+	/* No-op, port representors are processed in mlx5_dev_spawn(). */
+	if (!strcmp(MLX5_REPRESENTOR, key))
+		return 0;
 	errno = 0;
 	tmp = strtoul(val, NULL, 0);
 	if (errno) {
@@ -524,6 +527,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_RX_VEC_EN,
 		MLX5_L3_VXLAN_EN,
 		MLX5_VF_NL_EN,
+		MLX5_REPRESENTOR,
 		NULL,
 	};
 	struct rte_kvargs *kvlist;
@@ -600,7 +604,7 @@ mlx5_uar_init_primary(struct rte_eth_dev *dev)
 	rte_memseg_walk(find_lower_va_bound, &addr);
 
 	/* keep distance to hugepages to minimize potential conflicts. */
-	addr = RTE_PTR_SUB(addr, MLX5_UAR_OFFSET + MLX5_UAR_SIZE);
+	addr = RTE_PTR_SUB(addr, (uintptr_t)(MLX5_UAR_OFFSET + MLX5_UAR_SIZE));
 	/* anonymous mmap, no real memory consumption. */
 	addr = mmap(addr, MLX5_UAR_SIZE,
 		    PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
@@ -668,133 +672,115 @@ mlx5_uar_init_secondary(struct rte_eth_dev *dev)
 }
 
 /**
- * DPDK callback to register a PCI device.
+ * Spawn an Ethernet device from Verbs information.
  *
- * This function creates an Ethernet device for each port of a given
- * PCI device.
- *
- * @param[in] pci_drv
- *   PCI driver structure (mlx5_driver).
- * @param[in] pci_dev
- *   PCI device information.
+ * @param dpdk_dev
+ *   Backing DPDK device.
+ * @param ibv_dev
+ *   Verbs device.
+ * @param vf
+ *   If nonzero, enable VF-specific features.
+ * @param[in] switch_info
+ *   Switch properties of Ethernet device.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   A valid Ethernet device object on success, NULL otherwise and rte_errno
+ *   is set. The following error is defined:
+ *
+ *   EBUSY: device is not supposed to be spawned.
  */
-static int
-mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
-	       struct rte_pci_device *pci_dev)
+static struct rte_eth_dev *
+mlx5_dev_spawn(struct rte_device *dpdk_dev,
+	       struct ibv_device *ibv_dev,
+	       int vf,
+	       const struct mlx5_switch_info *switch_info)
 {
-	struct ibv_device **list = NULL;
-	struct ibv_device *ibv_dev;
+	struct ibv_context *ctx;
+	struct ibv_device_attr_ex attr;
+	struct ibv_port_attr port_attr;
+	struct ibv_pd *pd = NULL;
+	struct mlx5dv_context dv_attr = { .comp_mask = 0 };
+	struct mlx5_dev_config config = {
+		.vf = !!vf,
+		.tx_vec_en = 1,
+		.rx_vec_en = 1,
+		.mpw_hdr_dseg = 0,
+		.txq_inline = MLX5_ARG_UNSET,
+		.txqs_inline = MLX5_ARG_UNSET,
+		.inline_max_packet_sz = MLX5_ARG_UNSET,
+		.vf_nl_en = 1,
+		.mprq = {
+			.enabled = 0,
+			.stride_num_n = MLX5_MPRQ_STRIDE_NUM_N,
+			.max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN,
+			.min_rxqs_num = MLX5_MPRQ_MIN_RXQS,
+		},
+	};
+	struct rte_eth_dev *eth_dev = NULL;
+	struct priv *priv = NULL;
 	int err = 0;
-	struct ibv_context *attr_ctx = NULL;
-	struct ibv_device_attr_ex device_attr;
-	unsigned int vf = 0;
 	unsigned int mps;
 	unsigned int cqe_comp;
 	unsigned int tunnel_en = 0;
 	unsigned int mpls_en = 0;
 	unsigned int swp = 0;
-	unsigned int verb_priorities = 0;
 	unsigned int mprq = 0;
 	unsigned int mprq_min_stride_size_n = 0;
 	unsigned int mprq_max_stride_size_n = 0;
 	unsigned int mprq_min_stride_num_n = 0;
 	unsigned int mprq_max_stride_num_n = 0;
-	int idx;
-	int i;
-	struct mlx5dv_context attrs_out = {0};
 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	struct ibv_counter_set_description cs_desc;
+	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
 #endif
+	struct ether_addr mac;
+	char name[RTE_ETH_NAME_MAX_LEN];
+	int own_domain_id = 0;
+	unsigned int i;
 
-	/* Prepare shared data between primary and secondary process. */
-	mlx5_prepare_shared_data();
-	assert(pci_drv == &mlx5_driver);
-	/* Get mlx5_dev[] index. */
-	idx = mlx5_dev_idx(&pci_dev->addr);
-	if (idx == -1) {
-		DRV_LOG(ERR, "this driver cannot support any more adapters");
-		err = ENOMEM;
-		goto error;
-	}
-	DRV_LOG(DEBUG, "using driver device index %d", idx);
-	/* Save PCI address. */
-	mlx5_dev[idx].pci_addr = pci_dev->addr;
-	list = mlx5_glue->get_device_list(&i);
-	if (list == NULL) {
-		assert(errno);
-		err = errno;
-		if (errno == ENOSYS)
-			DRV_LOG(ERR,
-				"cannot list devices, is ib_uverbs loaded?");
-		goto error;
-	}
-	assert(i >= 0);
-	/*
-	 * For each listed device, check related sysfs entry against
-	 * the provided PCI ID.
-	 */
-	while (i != 0) {
-		struct rte_pci_addr pci_addr;
+	/* Determine if this port representor is supposed to be spawned. */
+	if (switch_info->representor && dpdk_dev->devargs) {
+		struct rte_eth_devargs eth_da;
 
-		--i;
-		DRV_LOG(DEBUG, "checking device \"%s\"", list[i]->name);
-		if (mlx5_ibv_device_to_pci_addr(list[i], &pci_addr))
-			continue;
-		if ((pci_dev->addr.domain != pci_addr.domain) ||
-		    (pci_dev->addr.bus != pci_addr.bus) ||
-		    (pci_dev->addr.devid != pci_addr.devid) ||
-		    (pci_dev->addr.function != pci_addr.function))
-			continue;
-		DRV_LOG(INFO, "PCI information matches, using device \"%s\"",
-			list[i]->name);
-		vf = ((pci_dev->id.device_id ==
-		       PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) ||
-		      (pci_dev->id.device_id ==
-		       PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) ||
-		      (pci_dev->id.device_id ==
-		       PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) ||
-		      (pci_dev->id.device_id ==
-		       PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF));
-		attr_ctx = mlx5_glue->open_device(list[i]);
-		rte_errno = errno;
-		err = rte_errno;
-		break;
-	}
-	if (attr_ctx == NULL) {
-		switch (err) {
-		case 0:
-			DRV_LOG(ERR,
-				"cannot access device, is mlx5_ib loaded?");
-			err = ENODEV;
-			break;
-		case EINVAL:
-			DRV_LOG(ERR,
-				"cannot use device, are drivers up to date?");
-			break;
+		err = rte_eth_devargs_parse(dpdk_dev->devargs->args, &eth_da);
+		if (err) {
+			rte_errno = -err;
+			DRV_LOG(ERR, "failed to process device arguments: %s",
+				strerror(rte_errno));
+			return NULL;
 		}
-		goto error;
+		for (i = 0; i < eth_da.nb_representor_ports; ++i)
+			if (eth_da.representor_ports[i] ==
+			    (uint16_t)switch_info->port_name)
+				break;
+		if (i == eth_da.nb_representor_ports) {
+			rte_errno = EBUSY;
+			return NULL;
+		}
+	}
+	/* Prepare shared data between primary and secondary process. */
+	mlx5_prepare_shared_data();
+	errno = 0;
+	ctx = mlx5_glue->open_device(ibv_dev);
+	if (!ctx) {
+		rte_errno = errno ? errno : ENODEV;
+		return NULL;
 	}
-	ibv_dev = list[i];
-	DRV_LOG(DEBUG, "device opened");
 #ifdef HAVE_IBV_MLX5_MOD_SWP
-	attrs_out.comp_mask |= MLX5DV_CONTEXT_MASK_SWP;
+	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP;
 #endif
 	/*
 	 * Multi-packet send is supported by ConnectX-4 Lx PF as well
 	 * as all ConnectX-5 devices.
 	 */
 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	attrs_out.comp_mask |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS;
+	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS;
 #endif
 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
-	attrs_out.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
+	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
 #endif
-	mlx5_glue->dv_query_device(attr_ctx, &attrs_out);
-	if (attrs_out.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) {
-		if (attrs_out.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) {
+	mlx5_glue->dv_query_device(ctx, &dv_attr);
+	if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) {
+		if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) {
 			DRV_LOG(DEBUG, "enhanced MPW is supported");
 			mps = MLX5_MPW_ENHANCED;
 		} else {
@@ -805,15 +791,17 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 		DRV_LOG(DEBUG, "MPW isn't supported");
 		mps = MLX5_MPW_DISABLED;
 	}
+	config.mps = mps;
 #ifdef HAVE_IBV_MLX5_MOD_SWP
-	if (attrs_out.comp_mask & MLX5DV_CONTEXT_MASK_SWP)
-		swp = attrs_out.sw_parsing_caps.sw_parsing_offloads;
+	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_SWP)
+		swp = dv_attr.sw_parsing_caps.sw_parsing_offloads;
 	DRV_LOG(DEBUG, "SWP support: %u", swp);
 #endif
+	config.swp = !!swp;
 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
-	if (attrs_out.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) {
+	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) {
 		struct mlx5dv_striding_rq_caps mprq_caps =
-			attrs_out.striding_rq_caps;
+			dv_attr.striding_rq_caps;
 
 		DRV_LOG(DEBUG, "\tmin_single_stride_log_num_of_bytes: %d",
 			mprq_caps.min_single_stride_log_num_of_bytes);
@@ -835,18 +823,21 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 			mprq_caps.min_single_wqe_log_num_of_strides;
 		mprq_max_stride_num_n =
 			mprq_caps.max_single_wqe_log_num_of_strides;
+		config.mprq.stride_num_n = RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
+						   mprq_min_stride_num_n);
 	}
 #endif
 	if (RTE_CACHE_LINE_SIZE == 128 &&
-	    !(attrs_out.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP))
+	    !(dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP))
 		cqe_comp = 0;
 	else
 		cqe_comp = 1;
+	config.cqe_comp = cqe_comp;
 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	if (attrs_out.comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) {
-		tunnel_en = ((attrs_out.tunnel_offloads_caps &
+	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) {
+		tunnel_en = ((dv_attr.tunnel_offloads_caps &
 			      MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN) &&
-			     (attrs_out.tunnel_offloads_caps &
+			     (dv_attr.tunnel_offloads_caps &
 			      MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE));
 	}
 	DRV_LOG(DEBUG, "tunnel offloading is %ssupported",
@@ -855,10 +846,11 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	DRV_LOG(WARNING,
 		"tunnel offloading disabled due to old OFED/rdma-core version");
 #endif
+	config.tunnel_en = tunnel_en;
 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	mpls_en = ((attrs_out.tunnel_offloads_caps &
+	mpls_en = ((dv_attr.tunnel_offloads_caps &
 		    MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) &&
-		   (attrs_out.tunnel_offloads_caps &
+		   (dv_attr.tunnel_offloads_caps &
 		    MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP));
 	DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is %ssupported",
 		mpls_en ? "" : "not ");
@@ -866,387 +858,595 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	DRV_LOG(WARNING, "MPLS over GRE/UDP tunnel offloading disabled due to"
 		" old OFED/rdma-core version or firmware configuration");
 #endif
-	err = mlx5_glue->query_device_ex(attr_ctx, NULL, &device_attr);
+	config.mpls_en = mpls_en;
+	err = mlx5_glue->query_device_ex(ctx, NULL, &attr);
 	if (err) {
 		DEBUG("ibv_query_device_ex() failed");
 		goto error;
 	}
-	DRV_LOG(INFO, "%u port(s) detected",
-		device_attr.orig_attr.phys_port_cnt);
-	for (i = 0; i < device_attr.orig_attr.phys_port_cnt; i++) {
-		char name[RTE_ETH_NAME_MAX_LEN];
-		int len;
-		uint32_t port = i + 1; /* ports are indexed from one */
-		uint32_t test = (1 << i);
-		struct ibv_context *ctx = NULL;
-		struct ibv_port_attr port_attr;
-		struct ibv_pd *pd = NULL;
-		struct priv *priv = NULL;
-		struct rte_eth_dev *eth_dev = NULL;
-		struct ibv_device_attr_ex device_attr_ex;
-		struct ether_addr mac;
-		struct mlx5_dev_config config = {
-			.cqe_comp = cqe_comp,
-			.mps = mps,
-			.tunnel_en = tunnel_en,
-			.mpls_en = mpls_en,
-			.tx_vec_en = 1,
-			.rx_vec_en = 1,
-			.mpw_hdr_dseg = 0,
-			.txq_inline = MLX5_ARG_UNSET,
-			.txqs_inline = MLX5_ARG_UNSET,
-			.inline_max_packet_sz = MLX5_ARG_UNSET,
-			.vf_nl_en = 1,
-			.swp = !!swp,
-			.mprq = {
-				.enabled = 0, /* Disabled by default. */
-				.stride_num_n = RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
-							mprq_min_stride_num_n),
-				.max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN,
-				.min_rxqs_num = MLX5_MPRQ_MIN_RXQS,
-			},
-		};
-
-		len = snprintf(name, sizeof(name), PCI_PRI_FMT,
-			 pci_dev->addr.domain, pci_dev->addr.bus,
-			 pci_dev->addr.devid, pci_dev->addr.function);
-		if (device_attr.orig_attr.phys_port_cnt > 1)
-			snprintf(name + len, sizeof(name), " port %u", i);
-		mlx5_dev[idx].ports |= test;
-		if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
-			eth_dev = rte_eth_dev_attach_secondary(name);
-			if (eth_dev == NULL) {
-				DRV_LOG(ERR, "can not attach rte ethdev");
-				rte_errno = ENOMEM;
-				err = rte_errno;
-				goto error;
-			}
-			eth_dev->device = &pci_dev->device;
-			eth_dev->dev_ops = &mlx5_dev_sec_ops;
-			err = mlx5_uar_init_secondary(eth_dev);
-			if (err) {
-				err = rte_errno;
-				goto error;
-			}
-			/* Receive command fd from primary process */
-			err = mlx5_socket_connect(eth_dev);
-			if (err < 0) {
-				err = rte_errno;
-				goto error;
-			}
-			/* Remap UAR for Tx queues. */
-			err = mlx5_tx_uar_remap(eth_dev, err);
-			if (err) {
-				err = rte_errno;
-				goto error;
-			}
-			/*
-			 * Ethdev pointer is still required as input since
-			 * the primary device is not accessible from the
-			 * secondary process.
-			 */
-			eth_dev->rx_pkt_burst =
-				mlx5_select_rx_function(eth_dev);
-			eth_dev->tx_pkt_burst =
-				mlx5_select_tx_function(eth_dev);
-			rte_eth_dev_probing_finish(eth_dev);
-			continue;
-		}
-		DRV_LOG(DEBUG, "using port %u (%08" PRIx32 ")", port, test);
-		ctx = mlx5_glue->open_device(ibv_dev);
-		if (ctx == NULL) {
-			err = ENODEV;
-			goto port_error;
+	if (!switch_info->representor)
+		rte_strlcpy(name, dpdk_dev->name, sizeof(name));
+	else
+		snprintf(name, sizeof(name), "%s_representor_%u",
+			 dpdk_dev->name, switch_info->port_name);
+	DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name);
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
+		eth_dev = rte_eth_dev_attach_secondary(name);
+		if (eth_dev == NULL) {
+			DRV_LOG(ERR, "can not attach rte ethdev");
+			rte_errno = ENOMEM;
+			err = rte_errno;
+			goto error;
 		}
-		/* Check port status. */
-		err = mlx5_glue->query_port(ctx, port, &port_attr);
+		eth_dev->device = dpdk_dev;
+		eth_dev->dev_ops = &mlx5_dev_sec_ops;
+		err = mlx5_uar_init_secondary(eth_dev);
 		if (err) {
-			DRV_LOG(ERR, "port query failed: %s", strerror(err));
-			goto port_error;
-		}
-		if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
-			DRV_LOG(ERR,
-				"port %d is not configured in Ethernet mode",
-				port);
-			err = EINVAL;
-			goto port_error;
-		}
-		if (port_attr.state != IBV_PORT_ACTIVE)
-			DRV_LOG(DEBUG, "port %d is not active: \"%s\" (%d)",
-				port,
-				mlx5_glue->port_state_str(port_attr.state),
-				port_attr.state);
-		/* Allocate protection domain. */
-		pd = mlx5_glue->alloc_pd(ctx);
-		if (pd == NULL) {
-			DRV_LOG(ERR, "PD allocation failure");
-			err = ENOMEM;
-			goto port_error;
+			err = rte_errno;
+			goto error;
 		}
-		mlx5_dev[idx].ports |= test;
-		/* from rte_ethdev.c */
-		priv = rte_zmalloc("ethdev private structure",
-				   sizeof(*priv),
-				   RTE_CACHE_LINE_SIZE);
-		if (priv == NULL) {
-			DRV_LOG(ERR, "priv allocation failure");
-			err = ENOMEM;
-			goto port_error;
+		/* Receive command fd from primary process */
+		err = mlx5_socket_connect(eth_dev);
+		if (err < 0) {
+			err = rte_errno;
+			goto error;
 		}
-		priv->ctx = ctx;
-		strncpy(priv->ibdev_path, priv->ctx->device->ibdev_path,
-			sizeof(priv->ibdev_path));
-		priv->device_attr = device_attr;
-		priv->port = port;
-		priv->pd = pd;
-		priv->mtu = ETHER_MTU;
-		err = mlx5_args(&config, pci_dev->device.devargs);
+		/* Remap UAR for Tx queues. */
+		err = mlx5_tx_uar_remap(eth_dev, err);
 		if (err) {
-			DRV_LOG(ERR, "failed to process device arguments: %s",
-				strerror(err));
 			err = rte_errno;
-			goto port_error;
+			goto error;
+		}
+		/*
+		 * Ethdev pointer is still required as input since
+		 * the primary device is not accessible from the
+		 * secondary process.
+		 */
+		eth_dev->rx_pkt_burst = mlx5_select_rx_function(eth_dev);
+		eth_dev->tx_pkt_burst = mlx5_select_tx_function(eth_dev);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return eth_dev;
+	}
+	/* Check port status. */
+	err = mlx5_glue->query_port(ctx, 1, &port_attr);
+	if (err) {
+		DRV_LOG(ERR, "port query failed: %s", strerror(err));
+		goto error;
+	}
+	if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
+		DRV_LOG(ERR, "port is not configured in Ethernet mode");
+		err = EINVAL;
+		goto error;
+	}
+	if (port_attr.state != IBV_PORT_ACTIVE)
+		DRV_LOG(DEBUG, "port is not active: \"%s\" (%d)",
+			mlx5_glue->port_state_str(port_attr.state),
+			port_attr.state);
+	/* Allocate protection domain. */
+	pd = mlx5_glue->alloc_pd(ctx);
+	if (pd == NULL) {
+		DRV_LOG(ERR, "PD allocation failure");
+		err = ENOMEM;
+		goto error;
+	}
+	priv = rte_zmalloc("ethdev private structure",
+			   sizeof(*priv),
+			   RTE_CACHE_LINE_SIZE);
+	if (priv == NULL) {
+		DRV_LOG(ERR, "priv allocation failure");
+		err = ENOMEM;
+		goto error;
+	}
+	priv->ctx = ctx;
+	strncpy(priv->ibdev_name, priv->ctx->device->name,
+		sizeof(priv->ibdev_name));
+	strncpy(priv->ibdev_path, priv->ctx->device->ibdev_path,
+		sizeof(priv->ibdev_path));
+	priv->device_attr = attr;
+	priv->pd = pd;
+	priv->mtu = ETHER_MTU;
+#ifndef RTE_ARCH_64
+	/* Initialize UAR access locks for 32bit implementations. */
+	rte_spinlock_init(&priv->uar_lock_cq);
+	for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++)
+		rte_spinlock_init(&priv->uar_lock[i]);
+#endif
+	/* Some internal functions rely on Netlink sockets, open them now. */
+	priv->nl_socket_rdma = mlx5_nl_init(NETLINK_RDMA);
+	priv->nl_socket_route =	mlx5_nl_init(NETLINK_ROUTE);
+	priv->nl_sn = 0;
+	priv->representor = !!switch_info->representor;
+	priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
+	priv->representor_id =
+		switch_info->representor ? switch_info->port_name : -1;
+	/*
+	 * Look for sibling devices in order to reuse their switch domain
+	 * if any, otherwise allocate one.
+	 */
+	i = mlx5_dev_to_port_id(dpdk_dev, NULL, 0);
+	if (i > 0) {
+		uint16_t port_id[i];
+
+		i = RTE_MIN(mlx5_dev_to_port_id(dpdk_dev, port_id, i), i);
+		while (i--) {
+			const struct priv *opriv =
+				rte_eth_devices[port_id[i]].data->dev_private;
+
+			if (!opriv ||
+			    opriv->domain_id ==
+			    RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
+				continue;
+			priv->domain_id = opriv->domain_id;
+			break;
 		}
-		err = mlx5_glue->query_device_ex(ctx, NULL, &device_attr_ex);
+	}
+	if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
+		err = rte_eth_switch_domain_alloc(&priv->domain_id);
 		if (err) {
-			DRV_LOG(ERR, "ibv_query_device_ex() failed");
-			goto port_error;
+			err = rte_errno;
+			DRV_LOG(ERR, "unable to allocate switch domain: %s",
+				strerror(rte_errno));
+			goto error;
 		}
-		config.hw_csum = !!(device_attr_ex.device_cap_flags_ex &
-				    IBV_DEVICE_RAW_IP_CSUM);
-		DRV_LOG(DEBUG, "checksum offloading is %ssupported",
-			(config.hw_csum ? "" : "not "));
+		own_domain_id = 1;
+	}
+	err = mlx5_args(&config, dpdk_dev->devargs);
+	if (err) {
+		err = rte_errno;
+		DRV_LOG(ERR, "failed to process device arguments: %s",
+			strerror(rte_errno));
+		goto error;
+	}
+	config.hw_csum = !!(attr.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM);
+	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
+		(config.hw_csum ? "" : "not "));
 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-		config.flow_counter_en = !!(device_attr.max_counter_sets);
-		mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
-		DRV_LOG(DEBUG,
-			"counter type = %d, num of cs = %ld, attributes = %d",
-			cs_desc.counter_type, cs_desc.num_of_cs,
-			cs_desc.attributes);
+	config.flow_counter_en = !!attr.max_counter_sets;
+	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
+	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
+		cs_desc.counter_type, cs_desc.num_of_cs,
+		cs_desc.attributes);
 #endif
-		config.ind_table_max_size =
-			device_attr_ex.rss_caps.max_rwq_indirection_table_size;
-		/* Remove this check once DPDK supports larger/variable
-		 * indirection tables. */
-		if (config.ind_table_max_size >
-				(unsigned int)ETH_RSS_RETA_SIZE_512)
-			config.ind_table_max_size = ETH_RSS_RETA_SIZE_512;
-		DRV_LOG(DEBUG, "maximum Rx indirection table size is %u",
-			config.ind_table_max_size);
-		config.hw_vlan_strip = !!(device_attr_ex.raw_packet_caps &
-					 IBV_RAW_PACKET_CAP_CVLAN_STRIPPING);
-		DRV_LOG(DEBUG, "VLAN stripping is %ssupported",
-			(config.hw_vlan_strip ? "" : "not "));
-
-		config.hw_fcs_strip = !!(device_attr_ex.raw_packet_caps &
-					 IBV_RAW_PACKET_CAP_SCATTER_FCS);
-		DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported",
-			(config.hw_fcs_strip ? "" : "not "));
-
+	config.ind_table_max_size =
+		attr.rss_caps.max_rwq_indirection_table_size;
+	/*
+	 * Remove this check once DPDK supports larger/variable
+	 * indirection tables.
+	 */
+	if (config.ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512)
+		config.ind_table_max_size = ETH_RSS_RETA_SIZE_512;
+	DRV_LOG(DEBUG, "maximum Rx indirection table size is %u",
+		config.ind_table_max_size);
+	config.hw_vlan_strip = !!(attr.raw_packet_caps &
+				  IBV_RAW_PACKET_CAP_CVLAN_STRIPPING);
+	DRV_LOG(DEBUG, "VLAN stripping is %ssupported",
+		(config.hw_vlan_strip ? "" : "not "));
+	config.hw_fcs_strip = !!(attr.raw_packet_caps &
+				 IBV_RAW_PACKET_CAP_SCATTER_FCS);
+	DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported",
+		(config.hw_fcs_strip ? "" : "not "));
 #ifdef HAVE_IBV_WQ_FLAG_RX_END_PADDING
-		config.hw_padding = !!device_attr_ex.rx_pad_end_addr_align;
+	config.hw_padding = !!attr.rx_pad_end_addr_align;
 #endif
-		DRV_LOG(DEBUG,
-			"hardware Rx end alignment padding is %ssupported",
-			(config.hw_padding ? "" : "not "));
-		config.vf = vf;
-		config.tso = ((device_attr_ex.tso_caps.max_tso > 0) &&
-			      (device_attr_ex.tso_caps.supported_qpts &
-			      (1 << IBV_QPT_RAW_PACKET)));
-		if (config.tso)
-			config.tso_max_payload_sz =
-					device_attr_ex.tso_caps.max_tso;
-		if (config.mps && !mps) {
-			DRV_LOG(ERR,
-				"multi-packet send not supported on this device"
-				" (" MLX5_TXQ_MPW_EN ")");
-			err = ENOTSUP;
-			goto port_error;
-		}
-		DRV_LOG(INFO, "%s MPS is %s",
-			config.mps == MLX5_MPW_ENHANCED ? "enhanced " : "",
-			config.mps != MLX5_MPW_DISABLED ? "enabled" :
-			"disabled");
-		if (config.cqe_comp && !cqe_comp) {
-			DRV_LOG(WARNING, "Rx CQE compression isn't supported");
-			config.cqe_comp = 0;
-		}
-		config.mprq.enabled = config.mprq.enabled && mprq;
-		if (config.mprq.enabled) {
-			if (config.mprq.stride_num_n > mprq_max_stride_num_n ||
-			    config.mprq.stride_num_n < mprq_min_stride_num_n) {
-				config.mprq.stride_num_n =
-					RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
-						mprq_min_stride_num_n);
-				DRV_LOG(WARNING,
-					"the number of strides"
-					" for Multi-Packet RQ is out of range,"
-					" setting default value (%u)",
-					1 << config.mprq.stride_num_n);
-			}
-			config.mprq.min_stride_size_n = mprq_min_stride_size_n;
-			config.mprq.max_stride_size_n = mprq_max_stride_size_n;
-		}
-		eth_dev = rte_eth_dev_allocate(name);
-		if (eth_dev == NULL) {
-			DRV_LOG(ERR, "can not allocate rte ethdev");
-			err = ENOMEM;
-			goto port_error;
-		}
-		eth_dev->data->dev_private = priv;
-		priv->dev_data = eth_dev->data;
-		eth_dev->data->mac_addrs = priv->mac;
-		eth_dev->device = &pci_dev->device;
-		rte_eth_copy_pci_info(eth_dev, pci_dev);
-		eth_dev->device->driver = &mlx5_driver.driver;
-		err = mlx5_uar_init_primary(eth_dev);
-		if (err) {
-			err = rte_errno;
-			goto port_error;
-		}
-		/* Configure the first MAC address by default. */
-		if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
-			DRV_LOG(ERR,
-				"port %u cannot get MAC address, is mlx5_en"
-				" loaded? (errno: %s)",
-				eth_dev->data->port_id, strerror(errno));
-			err = ENODEV;
-			goto port_error;
+	DRV_LOG(DEBUG, "hardware Rx end alignment padding is %ssupported",
+		(config.hw_padding ? "" : "not "));
+	config.tso = (attr.tso_caps.max_tso > 0 &&
+		      (attr.tso_caps.supported_qpts &
+		       (1 << IBV_QPT_RAW_PACKET)));
+	if (config.tso)
+		config.tso_max_payload_sz = attr.tso_caps.max_tso;
+	if (config.mps && !mps) {
+		DRV_LOG(ERR,
+			"multi-packet send not supported on this device"
+			" (" MLX5_TXQ_MPW_EN ")");
+		err = ENOTSUP;
+		goto error;
+	}
+	DRV_LOG(INFO, "%sMPS is %s",
+		config.mps == MLX5_MPW_ENHANCED ? "enhanced " : "",
+		config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled");
+	if (config.cqe_comp && !cqe_comp) {
+		DRV_LOG(WARNING, "Rx CQE compression isn't supported");
+		config.cqe_comp = 0;
+	}
+	if (config.mprq.enabled && mprq) {
+		if (config.mprq.stride_num_n > mprq_max_stride_num_n ||
+		    config.mprq.stride_num_n < mprq_min_stride_num_n) {
+			config.mprq.stride_num_n =
+				RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
+					mprq_min_stride_num_n);
+			DRV_LOG(WARNING,
+				"the number of strides"
+				" for Multi-Packet RQ is out of range,"
+				" setting default value (%u)",
+				1 << config.mprq.stride_num_n);
 		}
-		DRV_LOG(INFO,
-			"port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
-			eth_dev->data->port_id,
-			mac.addr_bytes[0], mac.addr_bytes[1],
-			mac.addr_bytes[2], mac.addr_bytes[3],
-			mac.addr_bytes[4], mac.addr_bytes[5]);
+		config.mprq.min_stride_size_n = mprq_min_stride_size_n;
+		config.mprq.max_stride_size_n = mprq_max_stride_size_n;
+	} else if (config.mprq.enabled && !mprq) {
+		DRV_LOG(WARNING, "Multi-Packet RQ isn't supported");
+		config.mprq.enabled = 0;
+	}
+	eth_dev = rte_eth_dev_allocate(name);
+	if (eth_dev == NULL) {
+		DRV_LOG(ERR, "can not allocate rte ethdev");
+		err = ENOMEM;
+		goto error;
+	}
+	if (priv->representor)
+		eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
+	eth_dev->data->dev_private = priv;
+	priv->dev_data = eth_dev->data;
+	eth_dev->data->mac_addrs = priv->mac;
+	eth_dev->device = dpdk_dev;
+	eth_dev->device->driver = &mlx5_driver.driver;
+	err = mlx5_uar_init_primary(eth_dev);
+	if (err) {
+		err = rte_errno;
+		goto error;
+	}
+	/* Configure the first MAC address by default. */
+	if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
+		DRV_LOG(ERR,
+			"port %u cannot get MAC address, is mlx5_en"
+			" loaded? (errno: %s)",
+			eth_dev->data->port_id, strerror(rte_errno));
+		err = ENODEV;
+		goto error;
+	}
+	DRV_LOG(INFO,
+		"port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
+		eth_dev->data->port_id,
+		mac.addr_bytes[0], mac.addr_bytes[1],
+		mac.addr_bytes[2], mac.addr_bytes[3],
+		mac.addr_bytes[4], mac.addr_bytes[5]);
 #ifndef NDEBUG
-		{
-			char ifname[IF_NAMESIZE];
-
-			if (mlx5_get_ifname(eth_dev, &ifname) == 0)
-				DRV_LOG(DEBUG, "port %u ifname is \"%s\"",
-					eth_dev->data->port_id, ifname);
-			else
-				DRV_LOG(DEBUG, "port %u ifname is unknown",
-					eth_dev->data->port_id);
-		}
+	{
+		char ifname[IF_NAMESIZE];
+
+		if (mlx5_get_ifname(eth_dev, &ifname) == 0)
+			DRV_LOG(DEBUG, "port %u ifname is \"%s\"",
+				eth_dev->data->port_id, ifname);
+		else
+			DRV_LOG(DEBUG, "port %u ifname is unknown",
+				eth_dev->data->port_id);
+	}
 #endif
-		/* Get actual MTU if possible. */
-		err = mlx5_get_mtu(eth_dev, &priv->mtu);
-		if (err) {
-			err = rte_errno;
-			goto port_error;
-		}
-		DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id,
-			priv->mtu);
-		/*
-		 * Initialize burst functions to prevent crashes before link-up.
-		 */
-		eth_dev->rx_pkt_burst = removed_rx_burst;
-		eth_dev->tx_pkt_burst = removed_tx_burst;
-		eth_dev->dev_ops = &mlx5_dev_ops;
-		/* Register MAC address. */
-		claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
-		priv->nl_socket = -1;
-		priv->nl_sn = 0;
-		if (vf && config.vf_nl_en) {
-			priv->nl_socket = mlx5_nl_init(RTMGRP_LINK);
-			if (priv->nl_socket < 0)
-				priv->nl_socket = -1;
-			mlx5_nl_mac_addr_sync(eth_dev);
-		}
-		TAILQ_INIT(&priv->flows);
-		TAILQ_INIT(&priv->ctrl_flows);
-		/* Hint libmlx5 to use PMD allocator for data plane resources */
-		struct mlx5dv_ctx_allocators alctr = {
-			.alloc = &mlx5_alloc_verbs_buf,
-			.free = &mlx5_free_verbs_buf,
-			.data = priv,
-		};
-		mlx5_glue->dv_set_context_attr(ctx,
-					       MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
-					       (void *)((uintptr_t)&alctr));
-		/* Bring Ethernet device up. */
-		DRV_LOG(DEBUG, "port %u forcing Ethernet interface up",
-			eth_dev->data->port_id);
-		mlx5_set_link_up(eth_dev);
-		/*
-		 * Even though the interrupt handler is not installed yet,
-		 * interrupts will still trigger on the asyn_fd from
-		 * Verbs context returned by ibv_open_device().
-		 */
-		mlx5_link_update(eth_dev, 0);
-		/* Store device configuration on private structure. */
-		priv->config = config;
-		/* Create drop queue. */
-		err = mlx5_flow_create_drop_queue(eth_dev);
-		if (err) {
-			DRV_LOG(ERR, "port %u drop queue allocation failed: %s",
-				eth_dev->data->port_id, strerror(rte_errno));
-			err = rte_errno;
-			goto port_error;
-		}
-		/* Supported Verbs flow priority number detection. */
-		if (verb_priorities == 0)
-			verb_priorities = mlx5_get_max_verbs_prio(eth_dev);
-		if (verb_priorities < MLX5_VERBS_FLOW_PRIO_8) {
-			DRV_LOG(ERR, "port %u wrong Verbs flow priorities: %u",
-				eth_dev->data->port_id, verb_priorities);
-			goto port_error;
+	/* Get actual MTU if possible. */
+	err = mlx5_get_mtu(eth_dev, &priv->mtu);
+	if (err) {
+		err = rte_errno;
+		goto error;
+	}
+	DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id,
+		priv->mtu);
+	/* Initialize burst functions to prevent crashes before link-up. */
+	eth_dev->rx_pkt_burst = removed_rx_burst;
+	eth_dev->tx_pkt_burst = removed_tx_burst;
+	eth_dev->dev_ops = &mlx5_dev_ops;
+	/* Register MAC address. */
+	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
+	if (vf && config.vf_nl_en)
+		mlx5_nl_mac_addr_sync(eth_dev);
+	priv->mnl_socket = mlx5_nl_flow_socket_create();
+	if (!priv->mnl_socket) {
+		err = -rte_errno;
+		DRV_LOG(WARNING,
+			"flow rules relying on switch offloads will not be"
+			" supported: cannot open libmnl socket: %s",
+			strerror(rte_errno));
+	} else {
+		struct rte_flow_error error;
+		unsigned int ifindex = mlx5_ifindex(eth_dev);
+
+		if (!ifindex) {
+			err = -rte_errno;
+			error.message =
+				"cannot retrieve network interface index";
+		} else {
+			err = mlx5_nl_flow_init(priv->mnl_socket, ifindex,
+						&error);
 		}
-		priv->config.max_verbs_prio = verb_priorities;
-		/*
-		 * Once the device is added to the list of memory event
-		 * callback, its global MR cache table cannot be expanded
-		 * on the fly because of deadlock. If it overflows, lookup
-		 * should be done by searching MR list linearly, which is slow.
-		 */
-		err = mlx5_mr_btree_init(&priv->mr.cache,
-					 MLX5_MR_BTREE_CACHE_N * 2,
-					 eth_dev->device->numa_node);
 		if (err) {
-			err = rte_errno;
-			goto port_error;
+			DRV_LOG(WARNING,
+				"flow rules relying on switch offloads will"
+				" not be supported: %s: %s",
+				error.message, strerror(rte_errno));
+			mlx5_nl_flow_socket_destroy(priv->mnl_socket);
+			priv->mnl_socket = NULL;
 		}
-		/* Add device to memory callback list. */
-		rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
-		LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
-				 priv, mem_event_cb);
-		rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
-		rte_eth_dev_probing_finish(eth_dev);
-		continue;
-port_error:
-		if (priv)
-			rte_free(priv);
-		if (pd)
-			claim_zero(mlx5_glue->dealloc_pd(pd));
-		if (ctx)
-			claim_zero(mlx5_glue->close_device(ctx));
-		if (eth_dev && rte_eal_process_type() == RTE_PROC_PRIMARY)
-			rte_eth_dev_release_port(eth_dev);
-		break;
 	}
+	TAILQ_INIT(&priv->flows);
+	TAILQ_INIT(&priv->ctrl_flows);
+	/* Hint libmlx5 to use PMD allocator for data plane resources */
+	struct mlx5dv_ctx_allocators alctr = {
+		.alloc = &mlx5_alloc_verbs_buf,
+		.free = &mlx5_free_verbs_buf,
+		.data = priv,
+	};
+	mlx5_glue->dv_set_context_attr(ctx, MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
+				       (void *)((uintptr_t)&alctr));
+	/* Bring Ethernet device up. */
+	DRV_LOG(DEBUG, "port %u forcing Ethernet interface up",
+		eth_dev->data->port_id);
+	mlx5_set_link_up(eth_dev);
 	/*
-	 * XXX if something went wrong in the loop above, there is a resource
-	 * leak (ctx, pd, priv, dpdk ethdev) but we can do nothing about it as
-	 * long as the dpdk does not provide a way to deallocate a ethdev and a
-	 * way to enumerate the registered ethdevs to free the previous ones.
+	 * Even though the interrupt handler is not installed yet,
+	 * interrupts will still trigger on the asyn_fd from
+	 * Verbs context returned by ibv_open_device().
 	 */
-	/* no port found, complain */
-	if (!mlx5_dev[idx].ports) {
-		rte_errno = ENODEV;
+	mlx5_link_update(eth_dev, 0);
+	/* Store device configuration on private structure. */
+	priv->config = config;
+	/* Supported Verbs flow priority number detection. */
+	err = mlx5_flow_discover_priorities(eth_dev);
+	if (err < 0)
+		goto error;
+	priv->config.flow_prio = err;
+	/*
+	 * Once the device is added to the list of memory event
+	 * callback, its global MR cache table cannot be expanded
+	 * on the fly because of deadlock. If it overflows, lookup
+	 * should be done by searching MR list linearly, which is slow.
+	 */
+	err = mlx5_mr_btree_init(&priv->mr.cache,
+				 MLX5_MR_BTREE_CACHE_N * 2,
+				 eth_dev->device->numa_node);
+	if (err) {
 		err = rte_errno;
+		goto error;
 	}
+	/* Add device to memory callback list. */
+	rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
+	LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
+			 priv, mem_event_cb);
+	rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
+	return eth_dev;
 error:
-	if (attr_ctx)
-		claim_zero(mlx5_glue->close_device(attr_ctx));
-	if (list)
-		mlx5_glue->free_device_list(list);
-	if (err) {
-		rte_errno = err;
+	if (priv) {
+		if (priv->nl_socket_route >= 0)
+			close(priv->nl_socket_route);
+		if (priv->nl_socket_rdma >= 0)
+			close(priv->nl_socket_rdma);
+		if (priv->mnl_socket)
+			mlx5_nl_flow_socket_destroy(priv->mnl_socket);
+		if (own_domain_id)
+			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
+		rte_free(priv);
+	}
+	if (pd)
+		claim_zero(mlx5_glue->dealloc_pd(pd));
+	if (eth_dev)
+		rte_eth_dev_release_port(eth_dev);
+	if (ctx)
+		claim_zero(mlx5_glue->close_device(ctx));
+	assert(err > 0);
+	rte_errno = err;
+	return NULL;
+}
+
+/** Data associated with devices to spawn. */
+struct mlx5_dev_spawn_data {
+	unsigned int ifindex; /**< Network interface index. */
+	struct mlx5_switch_info info; /**< Switch information. */
+	struct ibv_device *ibv_dev; /**< Associated IB device. */
+	struct rte_eth_dev *eth_dev; /**< Associated Ethernet device. */
+};
+
+/**
+ * Comparison callback to sort device data.
+ *
+ * This is meant to be used with qsort().
+ *
+ * @param a[in]
+ *   Pointer to pointer to first data object.
+ * @param b[in]
+ *   Pointer to pointer to second data object.
+ *
+ * @return
+ *   0 if both objects are equal, less than 0 if the first argument is less
+ *   than the second, greater than 0 otherwise.
+ */
+static int
+mlx5_dev_spawn_data_cmp(const void *a, const void *b)
+{
+	const struct mlx5_switch_info *si_a =
+		&((const struct mlx5_dev_spawn_data *)a)->info;
+	const struct mlx5_switch_info *si_b =
+		&((const struct mlx5_dev_spawn_data *)b)->info;
+	int ret;
+
+	/* Master device first. */
+	ret = si_b->master - si_a->master;
+	if (ret)
+		return ret;
+	/* Then representor devices. */
+	ret = si_b->representor - si_a->representor;
+	if (ret)
+		return ret;
+	/* Unidentified devices come last in no specific order. */
+	if (!si_a->representor)
+		return 0;
+	/* Order representors by name. */
+	return si_a->port_name - si_b->port_name;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns Ethernet devices out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+	       struct rte_pci_device *pci_dev)
+{
+	struct ibv_device **ibv_list;
+	unsigned int n = 0;
+	int vf;
+	int ret;
+
+	assert(pci_drv == &mlx5_driver);
+	errno = 0;
+	ibv_list = mlx5_glue->get_device_list(&ret);
+	if (!ibv_list) {
+		rte_errno = errno ? errno : ENOSYS;
+		DRV_LOG(ERR, "cannot list devices, is ib_uverbs loaded?");
 		return -rte_errno;
 	}
-	return 0;
+
+	struct ibv_device *ibv_match[ret + 1];
+
+	while (ret-- > 0) {
+		struct rte_pci_addr pci_addr;
+
+		DRV_LOG(DEBUG, "checking device \"%s\"", ibv_list[ret]->name);
+		if (mlx5_ibv_device_to_pci_addr(ibv_list[ret], &pci_addr))
+			continue;
+		if (pci_dev->addr.domain != pci_addr.domain ||
+		    pci_dev->addr.bus != pci_addr.bus ||
+		    pci_dev->addr.devid != pci_addr.devid ||
+		    pci_dev->addr.function != pci_addr.function)
+			continue;
+		DRV_LOG(INFO, "PCI information matches for device \"%s\"",
+			ibv_list[ret]->name);
+		ibv_match[n++] = ibv_list[ret];
+	}
+	ibv_match[n] = NULL;
+
+	struct mlx5_dev_spawn_data list[n];
+	int nl_route = n ? mlx5_nl_init(NETLINK_ROUTE) : -1;
+	int nl_rdma = n ? mlx5_nl_init(NETLINK_RDMA) : -1;
+	unsigned int i;
+	unsigned int u;
+
+	/*
+	 * The existence of several matching entries (n > 1) means port
+	 * representors have been instantiated. No existing Verbs call nor
+	 * /sys entries can tell them apart, this can only be done through
+	 * Netlink calls assuming kernel drivers are recent enough to
+	 * support them.
+	 *
+	 * In the event of identification failure through Netlink, try again
+	 * through sysfs, then either:
+	 *
+	 * 1. No device matches (n == 0), complain and bail out.
+	 * 2. A single IB device matches (n == 1) and is not a representor,
+	 *    assume no switch support.
+	 * 3. Otherwise no safe assumptions can be made; complain louder and
+	 *    bail out.
+	 */
+	for (i = 0; i != n; ++i) {
+		list[i].ibv_dev = ibv_match[i];
+		list[i].eth_dev = NULL;
+		if (nl_rdma < 0)
+			list[i].ifindex = 0;
+		else
+			list[i].ifindex = mlx5_nl_ifindex
+				(nl_rdma, list[i].ibv_dev->name);
+		if (nl_route < 0 ||
+		    !list[i].ifindex ||
+		    mlx5_nl_switch_info(nl_route, list[i].ifindex,
+					&list[i].info) ||
+		    ((!list[i].info.representor && !list[i].info.master) &&
+		     mlx5_sysfs_switch_info(list[i].ifindex, &list[i].info))) {
+			list[i].ifindex = 0;
+			memset(&list[i].info, 0, sizeof(list[i].info));
+			continue;
+		}
+	}
+	if (nl_rdma >= 0)
+		close(nl_rdma);
+	if (nl_route >= 0)
+		close(nl_route);
+	/* Count unidentified devices. */
+	for (u = 0, i = 0; i != n; ++i)
+		if (!list[i].info.master && !list[i].info.representor)
+			++u;
+	if (u) {
+		if (n == 1 && u == 1) {
+			/* Case #2. */
+			DRV_LOG(INFO, "no switch support detected");
+		} else {
+			/* Case #3. */
+			DRV_LOG(ERR,
+				"unable to tell which of the matching devices"
+				" is the master (lack of kernel support?)");
+			n = 0;
+		}
+	}
+	/*
+	 * Sort list to probe devices in natural order for users convenience
+	 * (i.e. master first, then representors from lowest to highest ID).
+	 */
+	if (n)
+		qsort(list, n, sizeof(*list), mlx5_dev_spawn_data_cmp);
+	switch (pci_dev->id.device_id) {
+	case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
+	case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
+	case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
+	case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
+		vf = 1;
+		break;
+	default:
+		vf = 0;
+	}
+	for (i = 0; i != n; ++i) {
+		uint32_t restore;
+
+		list[i].eth_dev = mlx5_dev_spawn
+			(&pci_dev->device, list[i].ibv_dev, vf, &list[i].info);
+		if (!list[i].eth_dev) {
+			if (rte_errno != EBUSY)
+				break;
+			/* Device is disabled, ignore it. */
+			continue;
+		}
+		restore = list[i].eth_dev->data->dev_flags;
+		rte_eth_copy_pci_info(list[i].eth_dev, pci_dev);
+		/* Restore non-PCI flags cleared by the above call. */
+		list[i].eth_dev->data->dev_flags |= restore;
+		rte_eth_dev_probing_finish(list[i].eth_dev);
+	}
+	mlx5_glue->free_device_list(ibv_list);
+	if (!n) {
+		DRV_LOG(WARNING,
+			"no Verbs device matches PCI device " PCI_PRI_FMT ","
+			" are kernel drivers loaded?",
+			pci_dev->addr.domain, pci_dev->addr.bus,
+			pci_dev->addr.devid, pci_dev->addr.function);
+		rte_errno = ENOENT;
+		ret = -rte_errno;
+	} else if (i != n) {
+		DRV_LOG(ERR,
+			"probe of PCI device " PCI_PRI_FMT " aborted after"
+			" encountering an error: %s",
+			pci_dev->addr.domain, pci_dev->addr.bus,
+			pci_dev->addr.devid, pci_dev->addr.function,
+			strerror(rte_errno));
+		ret = -rte_errno;
+		/* Roll back. */
+		while (i--) {
+			if (!list[i].eth_dev)
+				continue;
+			mlx5_dev_close(list[i].eth_dev);
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+				rte_free(list[i].eth_dev->data->dev_private);
+			claim_zero(rte_eth_dev_release_port(list[i].eth_dev));
+		}
+		/* Restore original error. */
+		rte_errno = -ret;
+	} else {
+		ret = 0;
+	}
+	return ret;
 }
 
 static const struct rte_pci_id mlx5_pci_id_map[] = {
@@ -1435,10 +1635,13 @@ glue_error:
 /**
  * Driver initialization routine.
  */
-RTE_INIT(rte_mlx5_pmd_init);
-static void
-rte_mlx5_pmd_init(void)
+RTE_INIT(rte_mlx5_pmd_init)
 {
+	/* Initialize driver log type. */
+	mlx5_logtype = rte_log_register("pmd.net.mlx5");
+	if (mlx5_logtype >= 0)
+		rte_log_set_level(mlx5_logtype, RTE_LOG_NOTICE);
+
 	/* Build the static tables for Verbs conversion. */
 	mlx5_set_ptype_table();
 	mlx5_set_cksum_table();
@@ -1453,6 +1656,11 @@ rte_mlx5_pmd_init(void)
 	/* Match the size of Rx completion entry to the size of a cacheline. */
 	if (RTE_CACHE_LINE_SIZE == 128)
 		setenv("MLX5_CQE_SIZE", "128", 0);
+	/*
+	 * MLX5_DEVICE_FATAL_CLEANUP tells ibv_destroy functions to
+	 * cleanup all the Verbs resources even when the device was removed.
+	 */
+	setenv("MLX5_DEVICE_FATAL_CLEANUP", "1", 1);
 #ifdef RTE_LIBRTE_MLX5_DLOPEN_DEPS
 	if (mlx5_glue_init())
 		return;
@@ -1480,11 +1688,3 @@ rte_mlx5_pmd_init(void)
 RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__);
 RTE_PMD_REGISTER_PCI_TABLE(net_mlx5, mlx5_pci_id_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_mlx5, "* ib_uverbs & mlx5_core & mlx5_ib");
-
-/** Initialize driver log type. */
-RTE_INIT(vdev_netvsc_init_log)
-{
-	mlx5_logtype = rte_log_register("pmd.net.mlx5");
-	if (mlx5_logtype >= 0)
-		rte_log_set_level(mlx5_logtype, RTE_LOG_NOTICE);
-}
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 997b04a3..a3a34cff 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -53,6 +53,14 @@ enum {
 	PCI_DEVICE_ID_MELLANOX_CONNECTX5BF = 0xa2d2,
 };
 
+/** Switch information returned by mlx5_nl_switch_info(). */
+struct mlx5_switch_info {
+	uint32_t master:1; /**< Master device. */
+	uint32_t representor:1; /**< Representor device. */
+	int32_t port_name; /**< Representor port name. */
+	uint64_t switch_id; /**< Switch identifier. */
+};
+
 LIST_HEAD(mlx5_dev_list, priv);
 
 /* Shared memory between primary and secondary processes. */
@@ -114,7 +122,7 @@ struct mlx5_dev_config {
 		unsigned int min_rxqs_num;
 		/* Rx queue count threshold to enable MPRQ. */
 	} mprq; /* Configurations for Multi-Packet RQ. */
-	unsigned int max_verbs_prio; /* Number of Verb flow priorities. */
+	unsigned int flow_prio; /* Number of flow priorities. */
 	unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
 	unsigned int ind_table_max_size; /* Maximum indirection table size. */
 	int txq_inline; /* Maximum packet size for inlining. */
@@ -131,9 +139,6 @@ enum mlx5_verbs_alloc_type {
 	MLX5_VERBS_ALLOC_TYPE_RX_QUEUE,
 };
 
-/* 8 Verbs priorities. */
-#define MLX5_VERBS_FLOW_PRIO_8 8
-
 /**
  * Verbs allocator needs a context to know in the callback which kind of
  * resources it is allocating.
@@ -145,12 +150,27 @@ struct mlx5_verbs_alloc_ctx {
 
 LIST_HEAD(mlx5_mr_list, mlx5_mr);
 
+/* Flow drop context necessary due to Verbs API. */
+struct mlx5_drop {
+	struct mlx5_hrxq *hrxq; /* Hash Rx queue queue. */
+	struct mlx5_rxq_ibv *rxq; /* Verbs Rx queue. */
+};
+
+/** DPDK port to network interface index (ifindex) conversion. */
+struct mlx5_nl_flow_ptoi {
+	uint16_t port_id; /**< DPDK port ID. */
+	unsigned int ifindex; /**< Network interface index. */
+};
+
+struct mnl_socket;
+
 struct priv {
 	LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
 	struct rte_eth_dev_data *dev_data;  /* Pointer to device data. */
 	struct ibv_context *ctx; /* Verbs context. */
 	struct ibv_device_attr_ex device_attr; /* Device properties. */
 	struct ibv_pd *pd; /* Protection Domain. */
+	char ibdev_name[IBV_SYSFS_NAME_MAX]; /* IB device name. */
 	char ibdev_path[IBV_SYSFS_PATH_MAX]; /* IB device path for secondary */
 	struct ether_addr mac[MLX5_MAX_MAC_ADDRESSES]; /* MAC addresses. */
 	BITFIELD_DECLARE(mac_own, uint64_t, MLX5_MAX_MAC_ADDRESSES);
@@ -159,8 +179,10 @@ struct priv {
 	unsigned int vlan_filter_n; /* Number of configured VLAN filters. */
 	/* Device properties. */
 	uint16_t mtu; /* Configured MTU. */
-	uint8_t port; /* Physical port number. */
 	unsigned int isolated:1; /* Whether isolated mode is enabled. */
+	unsigned int representor:1; /* Device is a port representor. */
+	uint16_t domain_id; /* Switch domain identifier. */
+	int32_t representor_id; /* Port representor identifier. */
 	/* RX/TX queues. */
 	unsigned int rxqs_n; /* RX queues array size. */
 	unsigned int txqs_n; /* TX queues array size. */
@@ -171,9 +193,11 @@ struct priv {
 	struct rte_intr_handle intr_handle; /* Interrupt handler. */
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
-	struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
+	struct mlx5_drop drop_queue; /* Flow drop queues. */
 	struct mlx5_flows flows; /* RTE Flow rules. */
 	struct mlx5_flows ctrl_flows; /* Control flow rules. */
+	LIST_HEAD(counters, mlx5_flow_counter) flow_counters;
+	/* Flow counters. */
 	struct {
 		uint32_t dev_gen; /* Generation number to flush local caches. */
 		rte_rwlock_t rwlock; /* MR Lock. */
@@ -196,8 +220,15 @@ struct priv {
 	struct mlx5_dev_config config; /* Device configuration. */
 	struct mlx5_verbs_alloc_ctx verbs_alloc_ctx;
 	/* Context for Verbs allocator. */
-	int nl_socket; /* Netlink socket. */
+	int nl_socket_rdma; /* Netlink socket (NETLINK_RDMA). */
+	int nl_socket_route; /* Netlink socket (NETLINK_ROUTE). */
 	uint32_t nl_sn; /* Netlink message sequence number. */
+#ifndef RTE_ARCH_64
+	rte_spinlock_t uar_lock_cq; /* CQs share a common distinct UAR */
+	rte_spinlock_t uar_lock[MLX5_UAR_PAGE_NUM_MAX];
+	/* UAR same-page access control required in 32bit implementations. */
+#endif
+	struct mnl_socket *mnl_socket; /* Libmnl socket. */
 };
 
 #define PORT_ID(priv) ((priv)->dev_data->port_id)
@@ -209,9 +240,12 @@ int mlx5_getenv_int(const char *);
 
 /* mlx5_ethdev.c */
 
+int mlx5_get_master_ifname(const struct rte_eth_dev *dev,
+			   char (*ifname)[IF_NAMESIZE]);
 int mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]);
-int mlx5_ifindex(const struct rte_eth_dev *dev);
-int mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr);
+unsigned int mlx5_ifindex(const struct rte_eth_dev *dev);
+int mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr,
+	       int master);
 int mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu);
 int mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep,
 		   unsigned int flags);
@@ -236,6 +270,11 @@ int mlx5_set_link_up(struct rte_eth_dev *dev);
 int mlx5_is_removed(struct rte_eth_dev *dev);
 eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev);
 eth_rx_burst_t mlx5_select_rx_function(struct rte_eth_dev *dev);
+unsigned int mlx5_dev_to_port_id(const struct rte_device *dev,
+				 uint16_t *port_list,
+				 unsigned int port_list_n);
+int mlx5_sysfs_switch_info(unsigned int ifindex,
+			   struct mlx5_switch_info *info);
 
 /* mlx5_mac.c */
 
@@ -296,7 +335,8 @@ int mlx5_traffic_restart(struct rte_eth_dev *dev);
 
 /* mlx5_flow.c */
 
-unsigned int mlx5_get_max_verbs_prio(struct rte_eth_dev *dev);
+int mlx5_flow_discover_priorities(struct rte_eth_dev *dev);
+void mlx5_flow_print(struct rte_flow *flow);
 int mlx5_flow_validate(struct rte_eth_dev *dev,
 		       const struct rte_flow_attr *attr,
 		       const struct rte_flow_item items[],
@@ -343,7 +383,7 @@ int mlx5_socket_connect(struct rte_eth_dev *priv);
 
 /* mlx5_nl.c */
 
-int mlx5_nl_init(uint32_t nlgroups);
+int mlx5_nl_init(int protocol);
 int mlx5_nl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac,
 			 uint32_t index);
 int mlx5_nl_mac_addr_remove(struct rte_eth_dev *dev, struct ether_addr *mac,
@@ -352,5 +392,27 @@ void mlx5_nl_mac_addr_sync(struct rte_eth_dev *dev);
 void mlx5_nl_mac_addr_flush(struct rte_eth_dev *dev);
 int mlx5_nl_promisc(struct rte_eth_dev *dev, int enable);
 int mlx5_nl_allmulti(struct rte_eth_dev *dev, int enable);
+unsigned int mlx5_nl_ifindex(int nl, const char *name);
+int mlx5_nl_switch_info(int nl, unsigned int ifindex,
+			struct mlx5_switch_info *info);
+
+/* mlx5_nl_flow.c */
+
+int mlx5_nl_flow_transpose(void *buf,
+			   size_t size,
+			   const struct mlx5_nl_flow_ptoi *ptoi,
+			   const struct rte_flow_attr *attr,
+			   const struct rte_flow_item *pattern,
+			   const struct rte_flow_action *actions,
+			   struct rte_flow_error *error);
+void mlx5_nl_flow_brand(void *buf, uint32_t handle);
+int mlx5_nl_flow_create(struct mnl_socket *nl, void *buf,
+			struct rte_flow_error *error);
+int mlx5_nl_flow_destroy(struct mnl_socket *nl, void *buf,
+			 struct rte_flow_error *error);
+int mlx5_nl_flow_init(struct mnl_socket *nl, unsigned int ifindex,
+		      struct rte_flow_error *error);
+struct mnl_socket *mlx5_nl_flow_socket_create(void);
+void mlx5_nl_flow_socket_destroy(struct mnl_socket *nl);
 
 #endif /* RTE_PMD_MLX5_H_ */
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 51124cdc..f2a16795 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -64,10 +64,11 @@
 #define MLX5_VPMD_MIN_TXQS 4
 
 /* Threshold of buffer replenishment for vectorized Rx. */
-#define MLX5_VPMD_RXQ_RPLNSH_THRESH   64U
+#define MLX5_VPMD_RXQ_RPLNSH_THRESH(n) \
+	(RTE_MIN(MLX5_VPMD_RX_MAX_BURST, (unsigned int)(n) >> 2))
 
 /* Maximum size of burst for vectorized Rx. */
-#define MLX5_VPMD_RX_MAX_BURST        MLX5_VPMD_RXQ_RPLNSH_THRESH
+#define MLX5_VPMD_RX_MAX_BURST 64U
 
 /*
  * Maximum size of burst for vectorized Tx. This is related to the maximum size
@@ -86,17 +87,31 @@
 #define MLX5_LINK_STATUS_TIMEOUT 10
 
 /* Reserved address space for UAR mapping. */
-#define MLX5_UAR_SIZE (1ULL << 32)
+#define MLX5_UAR_SIZE (1ULL << (sizeof(uintptr_t) * 4))
 
 /* Offset of reserved UAR address space to hugepage memory. Offset is used here
  * to minimize possibility of address next to hugepage being used by other code
  * in either primary or secondary process, failing to map TX UAR would make TX
  * packets invisible to HW.
  */
-#define MLX5_UAR_OFFSET (1ULL << 32)
+#define MLX5_UAR_OFFSET (1ULL << (sizeof(uintptr_t) * 4))
+
+/* Maximum number of UAR pages used by a port,
+ * These are the size and mask for an array of mutexes used to synchronize
+ * the access to port's UARs on platforms that do not support 64 bit writes.
+ * In such systems it is possible to issue the 64 bits DoorBells through two
+ * consecutive writes, each write 32 bits. The access to a UAR page (which can
+ * be accessible by all threads in the process) must be synchronized
+ * (for example, using a semaphore). Such a synchronization is not required
+ * when ringing DoorBells on different UAR pages.
+ * A port with 512 Tx queues uses 8, 4kBytes, UAR pages which are shared
+ * among the ports.
+ */
+#define MLX5_UAR_PAGE_NUM_MAX 64
+#define MLX5_UAR_PAGE_NUM_MASK ((MLX5_UAR_PAGE_NUM_MAX) - 1)
 
 /* Log 2 of the default number of strides per WQE for Multi-Packet RQ. */
-#define MLX5_MPRQ_STRIDE_NUM_N 4U
+#define MLX5_MPRQ_STRIDE_NUM_N 6U
 
 /* Two-byte shift is disabled for Multi-Packet RQ. */
 #define MLX5_MPRQ_TWO_BYTE_SHIFT 0
@@ -111,6 +126,11 @@
 #define MLX5_MPRQ_MIN_RXQS 12
 
 /* Cache size of mempool for Multi-Packet RQ. */
-#define MLX5_MPRQ_MP_CACHE_SZ 32
+#define MLX5_MPRQ_MP_CACHE_SZ 32U
+
+/* Definition of static_assert found in /usr/include/assert.h */
+#ifndef HAVE_STATIC_ASSERT
+#define static_assert _Static_assert
+#endif
 
 #endif /* RTE_PMD_MLX5_DEFS_H_ */
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 90488af3..34c5b95e 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -41,6 +41,32 @@
 #include "mlx5_rxtx.h"
 #include "mlx5_utils.h"
 
+/* Supported speed values found in /usr/include/linux/ethtool.h */
+#ifndef HAVE_SUPPORTED_40000baseKR4_Full
+#define SUPPORTED_40000baseKR4_Full (1 << 23)
+#endif
+#ifndef HAVE_SUPPORTED_40000baseCR4_Full
+#define SUPPORTED_40000baseCR4_Full (1 << 24)
+#endif
+#ifndef HAVE_SUPPORTED_40000baseSR4_Full
+#define SUPPORTED_40000baseSR4_Full (1 << 25)
+#endif
+#ifndef HAVE_SUPPORTED_40000baseLR4_Full
+#define SUPPORTED_40000baseLR4_Full (1 << 26)
+#endif
+#ifndef HAVE_SUPPORTED_56000baseKR4_Full
+#define SUPPORTED_56000baseKR4_Full (1 << 27)
+#endif
+#ifndef HAVE_SUPPORTED_56000baseCR4_Full
+#define SUPPORTED_56000baseCR4_Full (1 << 28)
+#endif
+#ifndef HAVE_SUPPORTED_56000baseSR4_Full
+#define SUPPORTED_56000baseSR4_Full (1 << 29)
+#endif
+#ifndef HAVE_SUPPORTED_56000baseLR4_Full
+#define SUPPORTED_56000baseLR4_Full (1 << 30)
+#endif
+
 /* Add defines in case the running kernel is not the same as user headers. */
 #ifndef ETHTOOL_GLINKSETTINGS
 struct ethtool_link_settings {
@@ -93,7 +119,7 @@ struct ethtool_link_settings {
 #endif
 
 /**
- * Get interface name from private structure.
+ * Get master interface name from private structure.
  *
  * @param[in] dev
  *   Pointer to Ethernet device.
@@ -104,7 +130,8 @@ struct ethtool_link_settings {
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE])
+mlx5_get_master_ifname(const struct rte_eth_dev *dev,
+		       char (*ifname)[IF_NAMESIZE])
 {
 	struct priv *priv = dev->data->dev_private;
 	DIR *dir;
@@ -166,7 +193,7 @@ try_dev_id:
 		if (dev_port == dev_port_prev)
 			goto try_dev_id;
 		dev_port_prev = dev_port;
-		if (dev_port == (priv->port - 1u))
+		if (dev_port == 0)
 			strlcpy(match, name, sizeof(match));
 	}
 	closedir(dir);
@@ -179,30 +206,59 @@ try_dev_id:
 }
 
 /**
- * Get the interface index from device name.
+ * Get interface name from private structure.
+ *
+ * This is a port representor-aware version of mlx5_get_master_ifname().
  *
  * @param[in] dev
  *   Pointer to Ethernet device.
+ * @param[out] ifname
+ *   Interface name output buffer.
  *
  * @return
- *   Interface index on success, a negative errno value otherwise and
- *   rte_errno is set.
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
+mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE])
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int ifindex =
+		priv->nl_socket_rdma >= 0 ?
+		mlx5_nl_ifindex(priv->nl_socket_rdma, priv->ibdev_name) : 0;
+
+	if (!ifindex) {
+		if (!priv->representor)
+			return mlx5_get_master_ifname(dev, ifname);
+		rte_errno = ENXIO;
+		return -rte_errno;
+	}
+	if (if_indextoname(ifindex, &(*ifname)[0]))
+		return 0;
+	rte_errno = errno;
+	return -rte_errno;
+}
+
+/**
+ * Get the interface index from device name.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   Nonzero interface index on success, zero otherwise and rte_errno is set.
+ */
+unsigned int
 mlx5_ifindex(const struct rte_eth_dev *dev)
 {
 	char ifname[IF_NAMESIZE];
-	int ret;
+	unsigned int ifindex;
 
-	ret = mlx5_get_ifname(dev, &ifname);
-	if (ret)
-		return ret;
-	ret = if_nametoindex(ifname);
-	if (ret == -1) {
+	if (mlx5_get_ifname(dev, &ifname))
+		return 0;
+	ifindex = if_nametoindex(ifname);
+	if (!ifindex)
 		rte_errno = errno;
-		return -rte_errno;
-	}
-	return ret;
+	return ifindex;
 }
 
 /**
@@ -214,12 +270,16 @@ mlx5_ifindex(const struct rte_eth_dev *dev)
  *   Request number to pass to ioctl().
  * @param[out] ifr
  *   Interface request structure output buffer.
+ * @param master
+ *   When device is a port representor, perform request on master device
+ *   instead.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
+mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr,
+	   int master)
 {
 	int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 	int ret = 0;
@@ -228,7 +288,10 @@ mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
 		rte_errno = errno;
 		return -rte_errno;
 	}
-	ret = mlx5_get_ifname(dev, &ifr->ifr_name);
+	if (master)
+		ret = mlx5_get_master_ifname(dev, &ifr->ifr_name);
+	else
+		ret = mlx5_get_ifname(dev, &ifr->ifr_name);
 	if (ret)
 		goto error;
 	ret = ioctl(sock, req, ifr);
@@ -258,7 +321,7 @@ int
 mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
 {
 	struct ifreq request;
-	int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
+	int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request, 0);
 
 	if (ret)
 		return ret;
@@ -282,7 +345,7 @@ mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 {
 	struct ifreq request = { .ifr_mtu = mtu, };
 
-	return mlx5_ifreq(dev, SIOCSIFMTU, &request);
+	return mlx5_ifreq(dev, SIOCSIFMTU, &request, 0);
 }
 
 /**
@@ -302,13 +365,13 @@ int
 mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
 {
 	struct ifreq request;
-	int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
+	int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request, 0);
 
 	if (ret)
 		return ret;
 	request.ifr_flags &= keep;
 	request.ifr_flags |= flags & ~keep;
-	return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
+	return mlx5_ifreq(dev, SIOCSIFFLAGS, &request, 0);
 }
 
 /**
@@ -335,15 +398,15 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
 
 	if (use_app_rss_key &&
 	    (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len !=
-	     rss_hash_default_key_len)) {
-		DRV_LOG(ERR, "port %u RSS key len must be %zu Bytes long",
-			dev->data->port_id, rss_hash_default_key_len);
+	     MLX5_RSS_HASH_KEY_LEN)) {
+		DRV_LOG(ERR, "port %u RSS key len must be %s Bytes long",
+			dev->data->port_id, RTE_STR(MLX5_RSS_HASH_KEY_LEN));
 		rte_errno = EINVAL;
 		return -rte_errno;
 	}
 	priv->rss_conf.rss_key =
 		rte_realloc(priv->rss_conf.rss_key,
-			    rss_hash_default_key_len, 0);
+			    MLX5_RSS_HASH_KEY_LEN, 0);
 	if (!priv->rss_conf.rss_key) {
 		DRV_LOG(ERR, "port %u cannot allocate RSS hash key memory (%u)",
 			dev->data->port_id, rxqs_n);
@@ -354,8 +417,8 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
 	       use_app_rss_key ?
 	       dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key :
 	       rss_hash_default_key,
-	       rss_hash_default_key_len);
-	priv->rss_conf.rss_key_len = rss_hash_default_key_len;
+	       MLX5_RSS_HASH_KEY_LEN);
+	priv->rss_conf.rss_key_len = MLX5_RSS_HASH_KEY_LEN;
 	priv->rss_conf.rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
 	priv->rxqs = (void *)dev->data->rx_queues;
 	priv->txqs = (void *)dev->data->tx_queues;
@@ -473,10 +536,34 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 		info->if_index = if_nametoindex(ifname);
 	info->reta_size = priv->reta_idx_n ?
 		priv->reta_idx_n : config->ind_table_max_size;
-	info->hash_key_size = rss_hash_default_key_len;
+	info->hash_key_size = MLX5_RSS_HASH_KEY_LEN;
 	info->speed_capa = priv->link_speed_capa;
 	info->flow_type_rss_offloads = ~MLX5_RSS_HF_MASK;
 	mlx5_set_default_params(dev, info);
+	info->switch_info.name = dev->data->name;
+	info->switch_info.domain_id = priv->domain_id;
+	info->switch_info.port_id = priv->representor_id;
+	if (priv->representor) {
+		unsigned int i = mlx5_dev_to_port_id(dev->device, NULL, 0);
+		uint16_t port_id[i];
+
+		i = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, i), i);
+		while (i--) {
+			struct priv *opriv =
+				rte_eth_devices[port_id[i]].data->dev_private;
+
+			if (!opriv ||
+			    opriv->representor ||
+			    opriv->domain_id != priv->domain_id)
+				continue;
+			/*
+			 * Override switch name with that of the master
+			 * device.
+			 */
+			info->switch_info.name = opriv->dev_data->name;
+			break;
+		}
+	}
 }
 
 /**
@@ -540,7 +627,7 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 	int link_speed = 0;
 	int ret;
 
-	ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
+	ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr, 1);
 	if (ret) {
 		DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 			dev->data->port_id, strerror(rte_errno));
@@ -550,7 +637,7 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 	dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
 				(ifr.ifr_flags & IFF_RUNNING));
 	ifr.ifr_data = (void *)&edata;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
 	if (ret) {
 		DRV_LOG(WARNING,
 			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s",
@@ -611,7 +698,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 	uint64_t sc;
 	int ret;
 
-	ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
+	ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr, 1);
 	if (ret) {
 		DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 			dev->data->port_id, strerror(rte_errno));
@@ -621,7 +708,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 	dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
 				(ifr.ifr_flags & IFF_RUNNING));
 	ifr.ifr_data = (void *)&gcmd;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
 	if (ret) {
 		DRV_LOG(DEBUG,
 			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
@@ -638,7 +725,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 
 	*ecmd = gcmd;
 	ifr.ifr_data = (void *)ecmd;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
 	if (ret) {
 		DRV_LOG(DEBUG,
 			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
@@ -801,7 +888,7 @@ mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 	int ret;
 
 	ifr.ifr_data = (void *)&ethpause;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
 	if (ret) {
 		DRV_LOG(WARNING,
 			"port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
@@ -854,7 +941,7 @@ mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 		ethpause.tx_pause = 1;
 	else
 		ethpause.tx_pause = 0;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 0);
 	if (ret) {
 		DRV_LOG(WARNING,
 			"port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
@@ -1193,3 +1280,93 @@ mlx5_is_removed(struct rte_eth_dev *dev)
 		return 1;
 	return 0;
 }
+
+/**
+ * Get port ID list of mlx5 instances sharing a common device.
+ *
+ * @param[in] dev
+ *   Device to look for.
+ * @param[out] port_list
+ *   Result buffer for collected port IDs.
+ * @param port_list_n
+ *   Maximum number of entries in result buffer. If 0, @p port_list can be
+ *   NULL.
+ *
+ * @return
+ *   Number of matching instances regardless of the @p port_list_n
+ *   parameter, 0 if none were found.
+ */
+unsigned int
+mlx5_dev_to_port_id(const struct rte_device *dev, uint16_t *port_list,
+		    unsigned int port_list_n)
+{
+	uint16_t id;
+	unsigned int n = 0;
+
+	RTE_ETH_FOREACH_DEV(id) {
+		struct rte_eth_dev *ldev = &rte_eth_devices[id];
+
+		if (!ldev->device ||
+		    !ldev->device->driver ||
+		    strcmp(ldev->device->driver->name, MLX5_DRIVER_NAME) ||
+		    ldev->device != dev)
+			continue;
+		if (n < port_list_n)
+			port_list[n] = id;
+		n++;
+	}
+	return n;
+}
+
+/**
+ * Get switch information associated with network interface.
+ *
+ * @param ifindex
+ *   Network interface index.
+ * @param[out] info
+ *   Switch information object, populated in case of success.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info)
+{
+	char ifname[IF_NAMESIZE];
+	FILE *file;
+	struct mlx5_switch_info data = { .master = 0, };
+	bool port_name_set = false;
+	bool port_switch_id_set = false;
+	char c;
+
+	if (!if_indextoname(ifindex, ifname)) {
+		rte_errno = errno;
+		return -rte_errno;
+	}
+
+	MKSTR(phys_port_name, "/sys/class/net/%s/phys_port_name",
+	      ifname);
+	MKSTR(phys_switch_id, "/sys/class/net/%s/phys_switch_id",
+	      ifname);
+
+	file = fopen(phys_port_name, "rb");
+	if (file != NULL) {
+		port_name_set =
+			fscanf(file, "%d%c", &data.port_name, &c) == 2 &&
+			c == '\n';
+		fclose(file);
+	}
+	file = fopen(phys_switch_id, "rb");
+	if (file == NULL) {
+		rte_errno = errno;
+		return -rte_errno;
+	}
+	port_switch_id_set =
+		fscanf(file, "%" SCNx64 "%c", &data.switch_id, &c) == 2 &&
+		c == '\n';
+	fclose(file);
+	data.master = port_switch_id_set && !port_name_set;
+	data.representor = port_switch_id_set && port_name_set;
+	*info = data;
+	return 0;
+}
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 994be05b..ca4625b6 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -4,6 +4,7 @@
  */
 
 #include <sys/queue.h>
+#include <stdalign.h>
 #include <stdint.h>
 #include <string.h>
 
@@ -31,497 +32,289 @@
 #include "mlx5_prm.h"
 #include "mlx5_glue.h"
 
-/* Flow priority for control plane flows. */
-#define MLX5_CTRL_FLOW_PRIORITY 1
-
-/* Internet Protocol versions. */
-#define MLX5_IPV4 4
-#define MLX5_IPV6 6
-#define MLX5_GRE 47
-
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-struct ibv_flow_spec_counter_action {
-	int dummy;
-};
-#endif
-
 /* Dev ops structure defined in mlx5.c */
 extern const struct eth_dev_ops mlx5_dev_ops;
 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
-/** Structure give to the conversion functions. */
-struct mlx5_flow_data {
-	struct rte_eth_dev *dev; /** Ethernet device. */
-	struct mlx5_flow_parse *parser; /** Parser context. */
-	struct rte_flow_error *error; /** Error context. */
-};
-
-static int
-mlx5_flow_create_eth(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vlan(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_ipv4(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_ipv6(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_udp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_tcp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vxlan(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
-			   const void *default_mask,
-			   struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_gre(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_mpls(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-struct mlx5_flow_parse;
-
-static void
-mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
-		      unsigned int size);
-
-static int
-mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
-
-static int
-mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser);
-
-/* Hash RX queue types. */
-enum hash_rxq_type {
-	HASH_RXQ_TCPV4,
-	HASH_RXQ_UDPV4,
-	HASH_RXQ_IPV4,
-	HASH_RXQ_TCPV6,
-	HASH_RXQ_UDPV6,
-	HASH_RXQ_IPV6,
-	HASH_RXQ_ETH,
-	HASH_RXQ_TUNNEL,
+/* Pattern outer Layer bits. */
+#define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
+#define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
+#define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
+#define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
+
+/* Pattern inner Layer bits. */
+#define MLX5_FLOW_LAYER_INNER_L2 (1u << 6)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV4 (1u << 7)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV6 (1u << 8)
+#define MLX5_FLOW_LAYER_INNER_L4_UDP (1u << 9)
+#define MLX5_FLOW_LAYER_INNER_L4_TCP (1u << 10)
+#define MLX5_FLOW_LAYER_INNER_VLAN (1u << 11)
+
+/* Pattern tunnel Layer bits. */
+#define MLX5_FLOW_LAYER_VXLAN (1u << 12)
+#define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
+#define MLX5_FLOW_LAYER_GRE (1u << 14)
+#define MLX5_FLOW_LAYER_MPLS (1u << 15)
+
+/* Outer Masks. */
+#define MLX5_FLOW_LAYER_OUTER_L3 \
+	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
+#define MLX5_FLOW_LAYER_OUTER_L4 \
+	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
+#define MLX5_FLOW_LAYER_OUTER \
+	(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_L3 | \
+	 MLX5_FLOW_LAYER_OUTER_L4)
+
+/* Tunnel Masks. */
+#define MLX5_FLOW_LAYER_TUNNEL \
+	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
+	 MLX5_FLOW_LAYER_GRE | MLX5_FLOW_LAYER_MPLS)
+
+/* Inner Masks. */
+#define MLX5_FLOW_LAYER_INNER_L3 \
+	(MLX5_FLOW_LAYER_INNER_L3_IPV4 | MLX5_FLOW_LAYER_INNER_L3_IPV6)
+#define MLX5_FLOW_LAYER_INNER_L4 \
+	(MLX5_FLOW_LAYER_INNER_L4_UDP | MLX5_FLOW_LAYER_INNER_L4_TCP)
+#define MLX5_FLOW_LAYER_INNER \
+	(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3 | \
+	 MLX5_FLOW_LAYER_INNER_L4)
+
+/* Actions that modify the fate of matching traffic. */
+#define MLX5_FLOW_FATE_DROP (1u << 0)
+#define MLX5_FLOW_FATE_QUEUE (1u << 1)
+#define MLX5_FLOW_FATE_RSS (1u << 2)
+
+/* Modify a packet. */
+#define MLX5_FLOW_MOD_FLAG (1u << 0)
+#define MLX5_FLOW_MOD_MARK (1u << 1)
+#define MLX5_FLOW_MOD_COUNT (1u << 2)
+
+/* possible L3 layers protocols filtering. */
+#define MLX5_IP_PROTOCOL_TCP 6
+#define MLX5_IP_PROTOCOL_UDP 17
+#define MLX5_IP_PROTOCOL_GRE 47
+#define MLX5_IP_PROTOCOL_MPLS 147
+
+/* Priority reserved for default flows. */
+#define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
+
+enum mlx5_expansion {
+	MLX5_EXPANSION_ROOT,
+	MLX5_EXPANSION_ROOT_OUTER,
+	MLX5_EXPANSION_ROOT_ETH_VLAN,
+	MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
+	MLX5_EXPANSION_OUTER_ETH,
+	MLX5_EXPANSION_OUTER_ETH_VLAN,
+	MLX5_EXPANSION_OUTER_VLAN,
+	MLX5_EXPANSION_OUTER_IPV4,
+	MLX5_EXPANSION_OUTER_IPV4_UDP,
+	MLX5_EXPANSION_OUTER_IPV4_TCP,
+	MLX5_EXPANSION_OUTER_IPV6,
+	MLX5_EXPANSION_OUTER_IPV6_UDP,
+	MLX5_EXPANSION_OUTER_IPV6_TCP,
+	MLX5_EXPANSION_VXLAN,
+	MLX5_EXPANSION_VXLAN_GPE,
+	MLX5_EXPANSION_GRE,
+	MLX5_EXPANSION_MPLS,
+	MLX5_EXPANSION_ETH,
+	MLX5_EXPANSION_ETH_VLAN,
+	MLX5_EXPANSION_VLAN,
+	MLX5_EXPANSION_IPV4,
+	MLX5_EXPANSION_IPV4_UDP,
+	MLX5_EXPANSION_IPV4_TCP,
+	MLX5_EXPANSION_IPV6,
+	MLX5_EXPANSION_IPV6_UDP,
+	MLX5_EXPANSION_IPV6_TCP,
 };
 
-/* Initialization data for hash RX queue. */
-struct hash_rxq_init {
-	uint64_t hash_fields; /* Fields that participate in the hash. */
-	uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
-	unsigned int flow_priority; /* Flow priority to use. */
-	unsigned int ip_version; /* Internet protocol. */
-};
-
-/* Initialization data for hash RX queues. */
-const struct hash_rxq_init hash_rxq_init[] = {
-	[HASH_RXQ_TCPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4 |
-				IBV_RX_HASH_SRC_PORT_TCP |
-				IBV_RX_HASH_DST_PORT_TCP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV4,
+/** Supported expansion of items. */
+static const struct rte_flow_expand_node mlx5_support_expansion[] = {
+	[MLX5_EXPANSION_ROOT] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
+						 MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_END,
 	},
-	[HASH_RXQ_UDPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4 |
-				IBV_RX_HASH_SRC_PORT_UDP |
-				IBV_RX_HASH_DST_PORT_UDP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV4,
+	[MLX5_EXPANSION_ROOT_OUTER] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
+						 MLX5_EXPANSION_OUTER_IPV4,
+						 MLX5_EXPANSION_OUTER_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_END,
 	},
-	[HASH_RXQ_IPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4),
-		.dpdk_rss_hf = (ETH_RSS_IPV4 |
-				ETH_RSS_FRAG_IPV4),
-		.flow_priority = 1,
-		.ip_version = MLX5_IPV4,
+	[MLX5_EXPANSION_ROOT_ETH_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
+		.type = RTE_FLOW_ITEM_TYPE_END,
 	},
-	[HASH_RXQ_TCPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6 |
-				IBV_RX_HASH_SRC_PORT_TCP |
-				IBV_RX_HASH_DST_PORT_TCP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV6,
+	[MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH_VLAN),
+		.type = RTE_FLOW_ITEM_TYPE_END,
 	},
-	[HASH_RXQ_UDPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6 |
-				IBV_RX_HASH_SRC_PORT_UDP |
-				IBV_RX_HASH_DST_PORT_UDP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV6,
+	[MLX5_EXPANSION_OUTER_ETH] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
+						 MLX5_EXPANSION_OUTER_IPV6,
+						 MLX5_EXPANSION_MPLS),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+		.rss_types = 0,
 	},
-	[HASH_RXQ_IPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6),
-		.dpdk_rss_hf = (ETH_RSS_IPV6 |
-				ETH_RSS_FRAG_IPV6),
-		.flow_priority = 1,
-		.ip_version = MLX5_IPV6,
+	[MLX5_EXPANSION_OUTER_ETH_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+		.rss_types = 0,
 	},
-	[HASH_RXQ_ETH] = {
-		.hash_fields = 0,
-		.dpdk_rss_hf = 0,
-		.flow_priority = 2,
+	[MLX5_EXPANSION_OUTER_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
+						 MLX5_EXPANSION_OUTER_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_VLAN,
 	},
-};
-
-/* Number of entries in hash_rxq_init[]. */
-const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
-
-/** Structure for holding counter stats. */
-struct mlx5_flow_counter_stats {
-	uint64_t hits; /**< Number of packets matched by the rule. */
-	uint64_t bytes; /**< Number of bytes matched by the rule. */
-};
-
-/** Structure for Drop queue. */
-struct mlx5_hrxq_drop {
-	struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
-	struct ibv_qp *qp; /**< Verbs queue pair. */
-	struct ibv_wq *wq; /**< Verbs work queue. */
-	struct ibv_cq *cq; /**< Verbs completion queue. */
-};
-
-/* Flows structures. */
-struct mlx5_flow {
-	uint64_t hash_fields; /**< Fields that participate in the hash. */
-	struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
-	struct ibv_flow *ibv_flow; /**< Verbs flow. */
-	struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
-};
-
-/* Drop flows structures. */
-struct mlx5_flow_drop {
-	struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
-	struct ibv_flow *ibv_flow; /**< Verbs flow. */
-};
-
-struct rte_flow {
-	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
-	uint32_t mark:1; /**< Set if the flow is marked. */
-	uint32_t drop:1; /**< Drop queue. */
-	struct rte_flow_action_rss rss_conf; /**< RSS configuration */
-	uint16_t (*queues)[]; /**< Queues indexes to use. */
-	uint8_t rss_key[40]; /**< copy of the RSS key. */
-	uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
-	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
-	struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
-	struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
-	/**< Flow with Rx queue. */
-};
-
-/** Static initializer for items. */
-#define ITEMS(...) \
-	(const enum rte_flow_item_type []){ \
-		__VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
-	}
-
-#define IS_TUNNEL(type) ( \
-	(type) == RTE_FLOW_ITEM_TYPE_VXLAN || \
-	(type) == RTE_FLOW_ITEM_TYPE_VXLAN_GPE || \
-	(type) == RTE_FLOW_ITEM_TYPE_GRE || \
-	(type) == RTE_FLOW_ITEM_TYPE_MPLS)
-
-const uint32_t flow_ptype[] = {
-	[RTE_FLOW_ITEM_TYPE_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN,
-	[RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = RTE_PTYPE_TUNNEL_VXLAN_GPE,
-	[RTE_FLOW_ITEM_TYPE_GRE] = RTE_PTYPE_TUNNEL_GRE,
-	[RTE_FLOW_ITEM_TYPE_MPLS] = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
-};
-
-#define PTYPE_IDX(t) ((RTE_PTYPE_TUNNEL_MASK & (t)) >> 12)
-
-const uint32_t ptype_ext[] = {
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] = RTE_PTYPE_TUNNEL_VXLAN |
-					      RTE_PTYPE_L4_UDP,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)]	= RTE_PTYPE_TUNNEL_VXLAN_GPE |
-						  RTE_PTYPE_L4_UDP,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)] = RTE_PTYPE_TUNNEL_GRE,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_GRE)] =
-		RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_UDP)] =
-		RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP,
-};
-
-/** Structure to generate a simple graph of layers supported by the NIC. */
-struct mlx5_flow_items {
-	/** List of possible actions for these items. */
-	const enum rte_flow_action_type *const actions;
-	/** Bit-masks corresponding to the possibilities for the item. */
-	const void *mask;
-	/**
-	 * Default bit-masks to use when item->mask is not provided. When
-	 * \default_mask is also NULL, the full supported bit-mask (\mask) is
-	 * used instead.
-	 */
-	const void *default_mask;
-	/** Bit-masks size in bytes. */
-	const unsigned int mask_sz;
-	/**
-	 * Conversion function from rte_flow to NIC specific flow.
-	 *
-	 * @param item
-	 *   rte_flow item to convert.
-	 * @param default_mask
-	 *   Default bit-masks to use when item->mask is not provided.
-	 * @param data
-	 *   Internal structure to store the conversion.
-	 *
-	 * @return
-	 *   0 on success, a negative errno value otherwise and rte_errno is
-	 *   set.
-	 */
-	int (*convert)(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data);
-	/** Size in bytes of the destination structure. */
-	const unsigned int dst_sz;
-	/** List of possible following items.  */
-	const enum rte_flow_item_type *const items;
-};
-
-/** Valid action for this PMD. */
-static const enum rte_flow_action_type valid_actions[] = {
-	RTE_FLOW_ACTION_TYPE_DROP,
-	RTE_FLOW_ACTION_TYPE_QUEUE,
-	RTE_FLOW_ACTION_TYPE_MARK,
-	RTE_FLOW_ACTION_TYPE_FLAG,
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	RTE_FLOW_ACTION_TYPE_COUNT,
-#endif
-	RTE_FLOW_ACTION_TYPE_END,
-};
-
-/** Graph of supported items and associated actions. */
-static const struct mlx5_flow_items mlx5_flow_items[] = {
-	[RTE_FLOW_ITEM_TYPE_END] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_VXLAN,
-			       RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
-			       RTE_FLOW_ITEM_TYPE_GRE),
+	[MLX5_EXPANSION_OUTER_IPV4] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT
+			(MLX5_EXPANSION_OUTER_IPV4_UDP,
+			 MLX5_EXPANSION_OUTER_IPV4_TCP,
+			 MLX5_EXPANSION_GRE),
+		.type = RTE_FLOW_ITEM_TYPE_IPV4,
+		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			ETH_RSS_NONFRAG_IPV4_OTHER,
 	},
-	[RTE_FLOW_ITEM_TYPE_ETH] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_eth){
-			.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-			.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-			.type = -1,
-		},
-		.default_mask = &rte_flow_item_eth_mask,
-		.mask_sz = sizeof(struct rte_flow_item_eth),
-		.convert = mlx5_flow_create_eth,
-		.dst_sz = sizeof(struct ibv_flow_spec_eth),
+	[MLX5_EXPANSION_OUTER_IPV4_UDP] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
+						 MLX5_EXPANSION_VXLAN_GPE),
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 	},
-	[RTE_FLOW_ITEM_TYPE_VLAN] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vlan){
-			.tci = -1,
-			.inner_type = -1,
-		},
-		.default_mask = &rte_flow_item_vlan_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vlan),
-		.convert = mlx5_flow_create_vlan,
-		.dst_sz = 0,
+	[MLX5_EXPANSION_OUTER_IPV4_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
 	},
-	[RTE_FLOW_ITEM_TYPE_IPV4] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
-			       RTE_FLOW_ITEM_TYPE_TCP,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_ipv4){
-			.hdr = {
-				.src_addr = -1,
-				.dst_addr = -1,
-				.type_of_service = -1,
-				.next_proto_id = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_ipv4_mask,
-		.mask_sz = sizeof(struct rte_flow_item_ipv4),
-		.convert = mlx5_flow_create_ipv4,
-		.dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
+	[MLX5_EXPANSION_OUTER_IPV6] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT
+			(MLX5_EXPANSION_OUTER_IPV6_UDP,
+			 MLX5_EXPANSION_OUTER_IPV6_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV6,
+		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
+			ETH_RSS_NONFRAG_IPV6_OTHER,
 	},
-	[RTE_FLOW_ITEM_TYPE_IPV6] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
-			       RTE_FLOW_ITEM_TYPE_TCP,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_ipv6){
-			.hdr = {
-				.src_addr = {
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-				},
-				.dst_addr = {
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-				},
-				.vtc_flow = -1,
-				.proto = -1,
-				.hop_limits = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_ipv6_mask,
-		.mask_sz = sizeof(struct rte_flow_item_ipv6),
-		.convert = mlx5_flow_create_ipv6,
-		.dst_sz = sizeof(struct ibv_flow_spec_ipv6),
+	[MLX5_EXPANSION_OUTER_IPV6_UDP] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
+						 MLX5_EXPANSION_VXLAN_GPE),
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 	},
-	[RTE_FLOW_ITEM_TYPE_UDP] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN,
-			       RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
-			       RTE_FLOW_ITEM_TYPE_MPLS),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_udp){
-			.hdr = {
-				.src_port = -1,
-				.dst_port = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_udp_mask,
-		.mask_sz = sizeof(struct rte_flow_item_udp),
-		.convert = mlx5_flow_create_udp,
-		.dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
+	[MLX5_EXPANSION_OUTER_IPV6_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 	},
-	[RTE_FLOW_ITEM_TYPE_TCP] = {
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_tcp){
-			.hdr = {
-				.src_port = -1,
-				.dst_port = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_tcp_mask,
-		.mask_sz = sizeof(struct rte_flow_item_tcp),
-		.convert = mlx5_flow_create_tcp,
-		.dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
+	[MLX5_EXPANSION_VXLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
+		.type = RTE_FLOW_ITEM_TYPE_VXLAN,
 	},
-	[RTE_FLOW_ITEM_TYPE_GRE] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6,
-			       RTE_FLOW_ITEM_TYPE_MPLS),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_gre){
-			.protocol = -1,
-		},
-		.default_mask = &rte_flow_item_gre_mask,
-		.mask_sz = sizeof(struct rte_flow_item_gre),
-		.convert = mlx5_flow_create_gre,
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-		.dst_sz = sizeof(struct ibv_flow_spec_gre),
-#else
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-#endif
+	[MLX5_EXPANSION_VXLAN_GPE] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
+						 MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 	},
-	[RTE_FLOW_ITEM_TYPE_MPLS] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_mpls){
-			.label_tc_s = "\xff\xff\xf0",
-		},
-		.default_mask = &rte_flow_item_mpls_mask,
-		.mask_sz = sizeof(struct rte_flow_item_mpls),
-		.convert = mlx5_flow_create_mpls,
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-		.dst_sz = sizeof(struct ibv_flow_spec_mpls),
-#endif
+	[MLX5_EXPANSION_GRE] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
+		.type = RTE_FLOW_ITEM_TYPE_GRE,
 	},
-	[RTE_FLOW_ITEM_TYPE_VXLAN] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4, /* For L3 VXLAN. */
-			       RTE_FLOW_ITEM_TYPE_IPV6), /* For L3 VXLAN. */
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vxlan){
-			.vni = "\xff\xff\xff",
-		},
-		.default_mask = &rte_flow_item_vxlan_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vxlan),
-		.convert = mlx5_flow_create_vxlan,
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
+	[MLX5_EXPANSION_MPLS] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_MPLS,
 	},
-	[RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vxlan_gpe){
-			.vni = "\xff\xff\xff",
-		},
-		.default_mask = &rte_flow_item_vxlan_gpe_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vxlan_gpe),
-		.convert = mlx5_flow_create_vxlan_gpe,
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
+	[MLX5_EXPANSION_ETH] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+	},
+	[MLX5_EXPANSION_ETH_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+	},
+	[MLX5_EXPANSION_VLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_VLAN,
+	},
+	[MLX5_EXPANSION_IPV4] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
+						 MLX5_EXPANSION_IPV4_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV4,
+		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			ETH_RSS_NONFRAG_IPV4_OTHER,
+	},
+	[MLX5_EXPANSION_IPV4_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
+	},
+	[MLX5_EXPANSION_IPV4_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
+	},
+	[MLX5_EXPANSION_IPV6] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
+						 MLX5_EXPANSION_IPV6_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV6,
+		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
+			ETH_RSS_NONFRAG_IPV6_OTHER,
+	},
+	[MLX5_EXPANSION_IPV6_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
+	},
+	[MLX5_EXPANSION_IPV6_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 	},
 };
 
-/** Structure to pass to the conversion function. */
-struct mlx5_flow_parse {
-	uint32_t inner; /**< Verbs value, set once tunnel is encountered. */
-	uint32_t create:1;
-	/**< Whether resources should remain after a validate. */
-	uint32_t drop:1; /**< Target is a drop queue. */
-	uint32_t mark:1; /**< Mark is present in the flow. */
-	uint32_t count:1; /**< Count is present in the flow. */
-	uint32_t mark_id; /**< Mark identifier. */
-	struct rte_flow_action_rss rss_conf; /**< RSS configuration */
-	uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
-	uint8_t rss_key[40]; /**< copy of the RSS key. */
-	enum hash_rxq_type layer; /**< Last pattern layer detected. */
-	enum hash_rxq_type out_layer; /**< Last outer pattern layer detected. */
-	uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
-	struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
+/** Handles information leading to a drop fate. */
+struct mlx5_flow_verbs {
+	LIST_ENTRY(mlx5_flow_verbs) next;
+	unsigned int size; /**< Size of the attribute. */
 	struct {
-		struct ibv_flow_attr *ibv_attr;
-		/**< Pointer to Verbs attributes. */
-		unsigned int offset;
-		/**< Current position or total size of the attribute. */
-		uint64_t hash_fields; /**< Verbs hash fields. */
-	} queue[RTE_DIM(hash_rxq_init)];
+		struct ibv_flow_attr *attr;
+		/**< Pointer to the Specification buffer. */
+		uint8_t *specs; /**< Pointer to the specifications. */
+	};
+	struct ibv_flow *flow; /**< Verbs flow pointer. */
+	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
+	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
+};
+
+/* Counters information. */
+struct mlx5_flow_counter {
+	LIST_ENTRY(mlx5_flow_counter) next; /**< Pointer to the next counter. */
+	uint32_t shared:1; /**< Share counter ID with other flow rules. */
+	uint32_t ref_cnt:31; /**< Reference counter. */
+	uint32_t id; /**< Counter ID. */
+	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+	uint64_t hits; /**< Number of packets matched by the rule. */
+	uint64_t bytes; /**< Number of bytes matched by the rule. */
+};
+
+/* Flow structure. */
+struct rte_flow {
+	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
+	struct rte_flow_attr attributes; /**< User flow attribute. */
+	uint32_t l3_protocol_en:1; /**< Protocol filtering requested. */
+	uint32_t layers;
+	/**< Bit-fields of present layers see MLX5_FLOW_LAYER_*. */
+	uint32_t modifier;
+	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
+	uint32_t fate;
+	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
+	uint8_t l3_protocol; /**< valid when l3_protocol_en is set. */
+	LIST_HEAD(verbs, mlx5_flow_verbs) verbs; /**< Verbs flows list. */
+	struct mlx5_flow_verbs *cur_verbs;
+	/**< Current Verbs flow structure being filled. */
+	struct mlx5_flow_counter *counter; /**< Holds Verbs flow counter. */
+	struct rte_flow_action_rss rss;/**< RSS context. */
+	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
+	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
+	void *nl_flow; /**< Netlink flow buffer if relevant. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
@@ -529,12 +322,8 @@ static const struct rte_flow_ops mlx5_flow_ops = {
 	.create = mlx5_flow_create,
 	.destroy = mlx5_flow_destroy,
 	.flush = mlx5_flow_flush,
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	.query = mlx5_flow_query,
-#else
-	.query = NULL,
-#endif
 	.isolate = mlx5_flow_isolate,
+	.query = mlx5_flow_query,
 };
 
 /* Convert FDIR request to Generic flow. */
@@ -569,883 +358,459 @@ struct ibv_spec_header {
 	uint16_t size;
 };
 
-/**
- * Check item is fully supported by the NIC matching capability.
- *
- * @param item[in]
- *   Item specification.
- * @param mask[in]
- *   Bit-masks covering supported fields to compare with spec, last and mask in
- *   \item.
- * @param size
- *   Bit-Mask size in bytes.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+/*
+ * Number of sub priorities.
+ * For each kind of pattern matching i.e. L2, L3, L4 to have a correct
+ * matching on the NIC (firmware dependent) L4 most have the higher priority
+ * followed by L3 and ending with L2.
  */
-static int
-mlx5_flow_item_validate(const struct rte_flow_item *item,
-			const uint8_t *mask, unsigned int size)
-{
-	unsigned int i;
-	const uint8_t *spec = item->spec;
-	const uint8_t *last = item->last;
-	const uint8_t *m = item->mask ? item->mask : mask;
+#define MLX5_PRIORITY_MAP_L2 2
+#define MLX5_PRIORITY_MAP_L3 1
+#define MLX5_PRIORITY_MAP_L4 0
+#define MLX5_PRIORITY_MAP_MAX 3
+
+/* Map of Verbs to Flow priority with 8 Verbs priorities. */
+static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
+	{ 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
+};
 
-	if (!spec && (item->mask || last))
-		goto error;
-	if (!spec)
-		return 0;
-	/*
-	 * Single-pass check to make sure that:
-	 * - item->mask is supported, no bits are set outside mask.
-	 * - Both masked item->spec and item->last are equal (no range
-	 *   supported).
-	 */
-	for (i = 0; i < size; i++) {
-		if (!m[i])
-			continue;
-		if ((m[i] | mask[i]) != mask[i])
-			goto error;
-		if (last && ((spec[i] & m[i]) != (last[i] & m[i])))
-			goto error;
-	}
-	return 0;
-error:
-	rte_errno = ENOTSUP;
-	return -rte_errno;
-}
+/* Map of Verbs to Flow priority with 16 Verbs priorities. */
+static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
+	{ 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
+	{ 9, 10, 11 }, { 12, 13, 14 },
+};
+
+/* Tunnel information. */
+struct mlx5_flow_tunnel_info {
+	uint32_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
+	uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
+};
+
+static struct mlx5_flow_tunnel_info tunnels_info[] = {
+	{
+		.tunnel = MLX5_FLOW_LAYER_VXLAN,
+		.ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
+		.ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_GRE,
+		.ptype = RTE_PTYPE_TUNNEL_GRE,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
+		.ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_MPLS,
+		.ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
+	},
+};
 
 /**
- * Extract attribute to the parser.
+ * Discover the maximum number of priority available.
  *
- * @param[in] attr
- *   Flow rule attributes.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
+ * @param[in] dev
+ *   Pointer to Ethernet device.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   number of supported flow priority on success, a negative errno
+ *   value otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_convert_attributes(const struct rte_flow_attr *attr,
-			     struct rte_flow_error *error)
+int
+mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 {
-	if (attr->group) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
-				   NULL,
-				   "groups are not supported");
-		return -rte_errno;
-	}
-	if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
-				   NULL,
-				   "priorities are not supported");
-		return -rte_errno;
-	}
-	if (attr->egress) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
-				   NULL,
-				   "egress is not supported");
+	struct {
+		struct ibv_flow_attr attr;
+		struct ibv_flow_spec_eth eth;
+		struct ibv_flow_spec_action_drop drop;
+	} flow_attr = {
+		.attr = {
+			.num_of_specs = 2,
+		},
+		.eth = {
+			.type = IBV_FLOW_SPEC_ETH,
+			.size = sizeof(struct ibv_flow_spec_eth),
+		},
+		.drop = {
+			.size = sizeof(struct ibv_flow_spec_action_drop),
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+		},
+	};
+	struct ibv_flow *flow;
+	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
+	uint16_t vprio[] = { 8, 16 };
+	int i;
+	int priority = 0;
+
+	if (!drop) {
+		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	if (attr->transfer) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
-				   NULL,
-				   "transfer is not supported");
-		return -rte_errno;
+	for (i = 0; i != RTE_DIM(vprio); i++) {
+		flow_attr.attr.priority = vprio[i] - 1;
+		flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
+		if (!flow)
+			break;
+		claim_zero(mlx5_glue->destroy_flow(flow));
+		priority = vprio[i];
 	}
-	if (!attr->ingress) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
-				   NULL,
-				   "only ingress is supported");
+	switch (priority) {
+	case 8:
+		priority = RTE_DIM(priority_map_3);
+		break;
+	case 16:
+		priority = RTE_DIM(priority_map_5);
+		break;
+	default:
+		rte_errno = ENOTSUP;
+		DRV_LOG(ERR,
+			"port %u verbs maximum priority: %d expected 8/16",
+			dev->data->port_id, vprio[i]);
 		return -rte_errno;
 	}
-	return 0;
+	mlx5_hrxq_drop_release(dev);
+	DRV_LOG(INFO, "port %u flow maximum priority: %d",
+		dev->data->port_id, priority);
+	return priority;
 }
 
 /**
- * Extract actions request to the parser.
+ * Adjust flow priority.
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param[in] actions
- *   Associated actions (list terminated by the END action).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ * @param flow
+ *   Pointer to an rte flow.
  */
-static int
-mlx5_flow_convert_actions(struct rte_eth_dev *dev,
-			  const struct rte_flow_action actions[],
-			  struct rte_flow_error *error,
-			  struct mlx5_flow_parse *parser)
+static void
+mlx5_flow_adjust_priority(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
-	enum { FATE = 1, MARK = 2, COUNT = 4, };
-	uint32_t overlap = 0;
 	struct priv *priv = dev->data->dev_private;
+	uint32_t priority = flow->attributes.priority;
+	uint32_t subpriority = flow->cur_verbs->attr->priority;
 
-	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
-		if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
-			continue;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			parser->drop = 1;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
-			const struct rte_flow_action_queue *queue =
-				(const struct rte_flow_action_queue *)
-				actions->conf;
-
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			if (!queue || (queue->index > (priv->rxqs_n - 1)))
-				goto exit_action_not_supported;
-			parser->queues[0] = queue->index;
-			parser->rss_conf = (struct rte_flow_action_rss){
-				.queue_num = 1,
-				.queue = parser->queues,
-			};
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
-			const struct rte_flow_action_rss *rss =
-				(const struct rte_flow_action_rss *)
-				actions->conf;
-			const uint8_t *rss_key;
-			uint32_t rss_key_len;
-			uint16_t n;
-
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			if (rss->func &&
-			    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "the only supported RSS hash"
-						   " function is Toeplitz");
-				return -rte_errno;
-			}
-#ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-			if (parser->rss_conf.level > 1) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "a nonzero RSS encapsulation"
-						   " level is not supported");
-				return -rte_errno;
-			}
-#endif
-			if (parser->rss_conf.level > 2) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "RSS encapsulation level"
-						   " > 1 is not supported");
-				return -rte_errno;
-			}
-			if (rss->types & MLX5_RSS_HF_MASK) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "unsupported RSS type"
-						   " requested");
-				return -rte_errno;
-			}
-			if (rss->key_len) {
-				rss_key_len = rss->key_len;
-				rss_key = rss->key;
-			} else {
-				rss_key_len = rss_hash_default_key_len;
-				rss_key = rss_hash_default_key;
-			}
-			if (rss_key_len != RTE_DIM(parser->rss_key)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "RSS hash key must be"
-						   " exactly 40 bytes long");
-				return -rte_errno;
-			}
-			if (!rss->queue_num) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "no valid queues");
-				return -rte_errno;
-			}
-			if (rss->queue_num > RTE_DIM(parser->queues)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "too many queues for RSS"
-						   " context");
-				return -rte_errno;
-			}
-			for (n = 0; n < rss->queue_num; ++n) {
-				if (rss->queue[n] >= priv->rxqs_n) {
-					rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "queue id > number of"
-						   " queues");
-					return -rte_errno;
-				}
-			}
-			parser->rss_conf = (struct rte_flow_action_rss){
-				.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
-				.level = rss->level ? rss->level : 1,
-				.types = rss->types,
-				.key_len = rss_key_len,
-				.queue_num = rss->queue_num,
-				.key = memcpy(parser->rss_key, rss_key,
-					      sizeof(*rss_key) * rss_key_len),
-				.queue = memcpy(parser->queues, rss->queue,
-						sizeof(*rss->queue) *
-						rss->queue_num),
-			};
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
-			const struct rte_flow_action_mark *mark =
-				(const struct rte_flow_action_mark *)
-				actions->conf;
-
-			if (overlap & MARK)
-				goto exit_action_overlap;
-			overlap |= MARK;
-			if (!mark) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "mark must be defined");
-				return -rte_errno;
-			} else if (mark->id >= MLX5_FLOW_MARK_MAX) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "mark must be between 0"
-						   " and 16777199");
-				return -rte_errno;
-			}
-			parser->mark = 1;
-			parser->mark_id = mark->id;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
-			if (overlap & MARK)
-				goto exit_action_overlap;
-			overlap |= MARK;
-			parser->mark = 1;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
-			   priv->config.flow_counter_en) {
-			if (overlap & COUNT)
-				goto exit_action_overlap;
-			overlap |= COUNT;
-			parser->count = 1;
-		} else {
-			goto exit_action_not_supported;
-		}
-	}
-	/* When fate is unknown, drop traffic. */
-	if (!(overlap & FATE))
-		parser->drop = 1;
-	if (parser->drop && parser->mark)
-		parser->mark = 0;
-	if (!parser->rss_conf.queue_num && !parser->drop) {
-		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "no valid action");
-		return -rte_errno;
+	switch (priv->config.flow_prio) {
+	case RTE_DIM(priority_map_3):
+		priority = priority_map_3[priority][subpriority];
+		break;
+	case RTE_DIM(priority_map_5):
+		priority = priority_map_5[priority][subpriority];
+		break;
 	}
-	return 0;
-exit_action_not_supported:
-	rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
-			   actions, "action not supported");
-	return -rte_errno;
-exit_action_overlap:
-	rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
-			   actions, "overlapping actions are not supported");
-	return -rte_errno;
+	flow->cur_verbs->attr->priority = priority;
 }
 
 /**
- * Validate items.
+ * Get a flow counter.
  *
- * @param[in] items
- *   Pattern specification (list terminated by the END pattern item).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] shared
+ *   Indicate if this counter is shared with other flows.
+ * @param[in] id
+ *   Counter identifier.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   A pointer to the counter, NULL otherwise and rte_errno is set.
  */
-static int
-mlx5_flow_convert_items_validate(struct rte_eth_dev *dev,
-				 const struct rte_flow_item items[],
-				 struct rte_flow_error *error,
-				 struct mlx5_flow_parse *parser)
+static struct mlx5_flow_counter *
+mlx5_flow_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 {
 	struct priv *priv = dev->data->dev_private;
-	const struct mlx5_flow_items *cur_item = mlx5_flow_items;
-	unsigned int i;
-	unsigned int last_voids = 0;
-	int ret = 0;
-
-	/* Initialise the offsets to start after verbs attribute. */
-	for (i = 0; i != hash_rxq_init_n; ++i)
-		parser->queue[i].offset = sizeof(struct ibv_flow_attr);
-	for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
-		const struct mlx5_flow_items *token = NULL;
-		unsigned int n;
+	struct mlx5_flow_counter *cnt;
 
-		if (items->type == RTE_FLOW_ITEM_TYPE_VOID) {
-			last_voids++;
+	LIST_FOREACH(cnt, &priv->flow_counters, next) {
+		if (!cnt->shared || cnt->shared != shared)
 			continue;
-		}
-		for (i = 0;
-		     cur_item->items &&
-		     cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
-		     ++i) {
-			if (cur_item->items[i] == items->type) {
-				token = &mlx5_flow_items[items->type];
-				break;
-			}
-		}
-		if (!token) {
-			ret = -ENOTSUP;
-			goto exit_item_not_supported;
-		}
-		cur_item = token;
-		ret = mlx5_flow_item_validate(items,
-					      (const uint8_t *)cur_item->mask,
-					      cur_item->mask_sz);
-		if (ret)
-			goto exit_item_not_supported;
-		if (IS_TUNNEL(items->type)) {
-			if (parser->tunnel &&
-			    !((items - last_voids - 1)->type ==
-			      RTE_FLOW_ITEM_TYPE_GRE && items->type ==
-			      RTE_FLOW_ITEM_TYPE_MPLS)) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   items,
-						   "Cannot recognize multiple"
-						   " tunnel encapsulations.");
-				return -rte_errno;
-			}
-			if (items->type == RTE_FLOW_ITEM_TYPE_MPLS &&
-			    !priv->config.mpls_en) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   items,
-						   "MPLS not supported or"
-						   " disabled in firmware"
-						   " configuration.");
-				return -rte_errno;
-			}
-			if (!priv->config.tunnel_en &&
-			    parser->rss_conf.level > 1) {
-				rte_flow_error_set(error, ENOTSUP,
-					RTE_FLOW_ERROR_TYPE_ITEM,
-					items,
-					"RSS on tunnel is not supported");
-				return -rte_errno;
-			}
-			parser->inner = IBV_FLOW_SPEC_INNER;
-			parser->tunnel = flow_ptype[items->type];
-		}
-		if (parser->drop) {
-			parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
-		} else {
-			for (n = 0; n != hash_rxq_init_n; ++n)
-				parser->queue[n].offset += cur_item->dst_sz;
-		}
-		last_voids = 0;
-	}
-	if (parser->drop) {
-		parser->queue[HASH_RXQ_ETH].offset +=
-			sizeof(struct ibv_flow_spec_action_drop);
-	}
-	if (parser->mark) {
-		for (i = 0; i != hash_rxq_init_n; ++i)
-			parser->queue[i].offset +=
-				sizeof(struct ibv_flow_spec_action_tag);
-	}
-	if (parser->count) {
-		unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-
-		for (i = 0; i != hash_rxq_init_n; ++i)
-			parser->queue[i].offset += size;
+		if (cnt->id != id)
+			continue;
+		cnt->ref_cnt++;
+		return cnt;
 	}
-	return 0;
-exit_item_not_supported:
-	return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
-				  items, "item not supported");
-}
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 
-/**
- * Allocate memory space to store verbs flow attributes.
- *
- * @param[in] size
- *   Amount of byte to allocate.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   A verbs flow attribute on success, NULL otherwise and rte_errno is set.
- */
-static struct ibv_flow_attr *
-mlx5_flow_convert_allocate(unsigned int size, struct rte_flow_error *error)
-{
-	struct ibv_flow_attr *ibv_attr;
+	struct mlx5_flow_counter tmpl = {
+		.shared = shared,
+		.id = id,
+		.cs = mlx5_glue->create_counter_set
+			(priv->ctx,
+			 &(struct ibv_counter_set_init_attr){
+				 .counter_set_id = id,
+			 }),
+		.hits = 0,
+		.bytes = 0,
+	};
 
-	ibv_attr = rte_calloc(__func__, 1, size, 0);
-	if (!ibv_attr) {
-		rte_flow_error_set(error, ENOMEM,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "cannot allocate verbs spec attributes");
+	if (!tmpl.cs) {
+		rte_errno = errno;
 		return NULL;
 	}
-	return ibv_attr;
-}
-
-/**
- * Make inner packet matching with an higher priority from the non Inner
- * matching.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in, out] parser
- *   Internal parser structure.
- * @param attr
- *   User flow attribute.
- */
-static void
-mlx5_flow_update_priority(struct rte_eth_dev *dev,
-			  struct mlx5_flow_parse *parser,
-			  const struct rte_flow_attr *attr)
-{
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-	uint16_t priority;
-
-	/*			8 priorities	>= 16 priorities
-	 * Control flow:	4-7		8-15
-	 * User normal flow:	1-3		4-7
-	 * User tunnel flow:	0-2		0-3
-	 */
-	priority = attr->priority * MLX5_VERBS_FLOW_PRIO_8;
-	if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
-		priority /= 2;
-	/*
-	 * Lower non-tunnel flow Verbs priority 1 if only support 8 Verbs
-	 * priorities, lower 4 otherwise.
-	 */
-	if (!parser->inner) {
-		if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
-			priority += 1;
-		else
-			priority += MLX5_VERBS_FLOW_PRIO_8 / 2;
-	}
-	if (parser->drop) {
-		parser->queue[HASH_RXQ_ETH].ibv_attr->priority = priority +
-				hash_rxq_init[HASH_RXQ_ETH].flow_priority;
-		return;
-	}
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		parser->queue[i].ibv_attr->priority = priority +
-				hash_rxq_init[i].flow_priority;
+	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
+	if (!cnt) {
+		rte_errno = ENOMEM;
+		return NULL;
 	}
+	*cnt = tmpl;
+	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
+	return cnt;
+#endif
+	rte_errno = ENOTSUP;
+	return NULL;
 }
 
 /**
- * Finalise verbs flow attributes.
+ * Release a flow counter.
  *
- * @param[in, out] parser
- *   Internal parser structure.
+ * @param[in] counter
+ *   Pointer to the counter handler.
  */
 static void
-mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser)
+mlx5_flow_counter_release(struct mlx5_flow_counter *counter)
 {
-	unsigned int i;
-	uint32_t inner = parser->inner;
-
-	/* Don't create extra flows for outer RSS. */
-	if (parser->tunnel && parser->rss_conf.level < 2)
-		return;
-	/*
-	 * Fill missing layers in verbs specifications, or compute the correct
-	 * offset to allocate the memory space for the attributes and
-	 * specifications.
-	 */
-	for (i = 0; i != hash_rxq_init_n - 1; ++i) {
-		union {
-			struct ibv_flow_spec_ipv4_ext ipv4;
-			struct ibv_flow_spec_ipv6 ipv6;
-			struct ibv_flow_spec_tcp_udp udp_tcp;
-			struct ibv_flow_spec_eth eth;
-		} specs;
-		void *dst;
-		uint16_t size;
-
-		if (i == parser->layer)
-			continue;
-		if (parser->layer == HASH_RXQ_ETH ||
-		    parser->layer == HASH_RXQ_TUNNEL) {
-			if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
-				size = sizeof(struct ibv_flow_spec_ipv4_ext);
-				specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
-					.type = inner | IBV_FLOW_SPEC_IPV4_EXT,
-					.size = size,
-				};
-			} else {
-				size = sizeof(struct ibv_flow_spec_ipv6);
-				specs.ipv6 = (struct ibv_flow_spec_ipv6){
-					.type = inner | IBV_FLOW_SPEC_IPV6,
-					.size = size,
-				};
-			}
-			if (parser->queue[i].ibv_attr) {
-				dst = (void *)((uintptr_t)
-					       parser->queue[i].ibv_attr +
-					       parser->queue[i].offset);
-				memcpy(dst, &specs, size);
-				++parser->queue[i].ibv_attr->num_of_specs;
-			}
-			parser->queue[i].offset += size;
-		}
-		if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
-		    (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
-			size = sizeof(struct ibv_flow_spec_tcp_udp);
-			specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
-				.type = inner | ((i == HASH_RXQ_UDPV4 ||
-					  i == HASH_RXQ_UDPV6) ?
-					 IBV_FLOW_SPEC_UDP :
-					 IBV_FLOW_SPEC_TCP),
-				.size = size,
-			};
-			if (parser->queue[i].ibv_attr) {
-				dst = (void *)((uintptr_t)
-					       parser->queue[i].ibv_attr +
-					       parser->queue[i].offset);
-				memcpy(dst, &specs, size);
-				++parser->queue[i].ibv_attr->num_of_specs;
-			}
-			parser->queue[i].offset += size;
-		}
+	if (--counter->ref_cnt == 0) {
+		claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
+		LIST_REMOVE(counter, next);
+		rte_free(counter);
 	}
 }
 
 /**
- * Update flows according to pattern and RSS hash fields.
+ * Verify the @p attributes will be correctly understood by the NIC and store
+ * them in the @p flow if everything is correct.
  *
- * @param[in, out] parser
- *   Internal parser structure.
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] attributes
+ *   Pointer to flow attributes
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_flow_convert_rss(struct mlx5_flow_parse *parser)
+mlx5_flow_attributes(struct rte_eth_dev *dev,
+		     const struct rte_flow_attr *attributes,
+		     struct rte_flow *flow,
+		     struct rte_flow_error *error)
 {
-	unsigned int i;
-	enum hash_rxq_type start;
-	enum hash_rxq_type layer;
-	int outer = parser->tunnel && parser->rss_conf.level < 2;
-	uint64_t rss = parser->rss_conf.types;
-
-	layer = outer ? parser->out_layer : parser->layer;
-	if (layer == HASH_RXQ_TUNNEL)
-		layer = HASH_RXQ_ETH;
-	if (outer) {
-		/* Only one hash type for outer RSS. */
-		if (rss && layer == HASH_RXQ_ETH) {
-			start = HASH_RXQ_TCPV4;
-		} else if (rss && layer != HASH_RXQ_ETH &&
-			   !(rss & hash_rxq_init[layer].dpdk_rss_hf)) {
-			/* If RSS not match L4 pattern, try L3 RSS. */
-			if (layer < HASH_RXQ_IPV4)
-				layer = HASH_RXQ_IPV4;
-			else if (layer > HASH_RXQ_IPV4 && layer < HASH_RXQ_IPV6)
-				layer = HASH_RXQ_IPV6;
-			start = layer;
-		} else {
-			start = layer;
-		}
-		/* Scan first valid hash type. */
-		for (i = start; rss && i <= layer; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-			if (hash_rxq_init[i].dpdk_rss_hf & rss)
-				break;
-		}
-		if (rss && i <= layer)
-			parser->queue[layer].hash_fields =
-					hash_rxq_init[i].hash_fields;
-		/* Trim unused hash types. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (parser->queue[i].ibv_attr && i != layer) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			}
-		}
-	} else {
-		/* Expand for inner or normal RSS. */
-		if (rss && (layer == HASH_RXQ_ETH || layer == HASH_RXQ_IPV4))
-			start = HASH_RXQ_TCPV4;
-		else if (rss && layer == HASH_RXQ_IPV6)
-			start = HASH_RXQ_TCPV6;
-		else
-			start = layer;
-		/* For L4 pattern, try L3 RSS if no L4 RSS. */
-		/* Trim unused hash types. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-			if (i < start || i > layer) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-				continue;
-			}
-			if (!rss)
-				continue;
-			if (hash_rxq_init[i].dpdk_rss_hf & rss) {
-				parser->queue[i].hash_fields =
-						hash_rxq_init[i].hash_fields;
-			} else if (i != layer) {
-				/* Remove unused RSS expansion. */
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			} else if (layer < HASH_RXQ_IPV4 &&
-				   (hash_rxq_init[HASH_RXQ_IPV4].dpdk_rss_hf &
-				    rss)) {
-				/* Allow IPv4 RSS on L4 pattern. */
-				parser->queue[i].hash_fields =
-					hash_rxq_init[HASH_RXQ_IPV4]
-						.hash_fields;
-			} else if (i > HASH_RXQ_IPV4 && i < HASH_RXQ_IPV6 &&
-				   (hash_rxq_init[HASH_RXQ_IPV6].dpdk_rss_hf &
-				    rss)) {
-				/* Allow IPv4 RSS on L4 pattern. */
-				parser->queue[i].hash_fields =
-					hash_rxq_init[HASH_RXQ_IPV6]
-						.hash_fields;
-			}
-		}
-	}
+	uint32_t priority_max =
+		((struct priv *)dev->data->dev_private)->config.flow_prio - 1;
+
+	if (attributes->group)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+					  NULL,
+					  "groups is not supported");
+	if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
+	    attributes->priority >= priority_max)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+					  NULL,
+					  "priority out of range");
+	if (attributes->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+					  NULL,
+					  "egress is not supported");
+	if (attributes->transfer)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
+					  NULL,
+					  "transfer is not supported");
+	if (!attributes->ingress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+					  NULL,
+					  "ingress attribute is mandatory");
+	flow->attributes = *attributes;
+	if (attributes->priority == MLX5_FLOW_PRIO_RSVD)
+		flow->attributes.priority = priority_max;
 	return 0;
 }
 
 /**
- * Validate and convert a flow supported by the NIC.
+ * Verify the @p item specifications (spec, last, mask) are compatible with the
+ * NIC capabilities.
  *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in] attr
- *   Flow rule attributes.
- * @param[in] pattern
- *   Pattern specification (list terminated by the END pattern item).
- * @param[in] actions
- *   Associated actions (list terminated by the END action).
+ * @param[in] item
+ *   Item specification.
+ * @param[in] mask
+ *   @p item->mask or flow default bit-masks.
+ * @param[in] nic_mask
+ *   Bit-masks covering supported fields by the NIC to compare with user mask.
+ * @param[in] size
+ *   Bit-masks size in bytes.
  * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
+ *   Pointer to error structure.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_flow_convert(struct rte_eth_dev *dev,
-		  const struct rte_flow_attr *attr,
-		  const struct rte_flow_item items[],
-		  const struct rte_flow_action actions[],
-		  struct rte_flow_error *error,
-		  struct mlx5_flow_parse *parser)
+mlx5_flow_item_acceptable(const struct rte_flow_item *item,
+			  const uint8_t *mask,
+			  const uint8_t *nic_mask,
+			  unsigned int size,
+			  struct rte_flow_error *error)
 {
-	const struct mlx5_flow_items *cur_item = mlx5_flow_items;
 	unsigned int i;
-	int ret;
 
-	/* First step. Validate the attributes, items and actions. */
-	*parser = (struct mlx5_flow_parse){
-		.create = parser->create,
-		.layer = HASH_RXQ_ETH,
-		.mark_id = MLX5_FLOW_MARK_DEFAULT,
-	};
-	ret = mlx5_flow_convert_attributes(attr, error);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert_actions(dev, actions, error, parser);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert_items_validate(dev, items, error, parser);
-	if (ret)
-		return ret;
-	mlx5_flow_convert_finalise(parser);
-	/*
-	 * Second step.
-	 * Allocate the memory space to store verbs specifications.
-	 */
-	if (parser->drop) {
-		unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
-
-		parser->queue[HASH_RXQ_ETH].ibv_attr =
-			mlx5_flow_convert_allocate(offset, error);
-		if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
-			goto exit_enomem;
-		parser->queue[HASH_RXQ_ETH].offset =
-			sizeof(struct ibv_flow_attr);
-	} else {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			unsigned int offset;
-
-			offset = parser->queue[i].offset;
-			parser->queue[i].ibv_attr =
-				mlx5_flow_convert_allocate(offset, error);
-			if (!parser->queue[i].ibv_attr)
-				goto exit_enomem;
-			parser->queue[i].offset = sizeof(struct ibv_flow_attr);
-		}
-	}
-	/* Third step. Conversion parse, fill the specifications. */
-	parser->inner = 0;
-	parser->tunnel = 0;
-	parser->layer = HASH_RXQ_ETH;
-	for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
-		struct mlx5_flow_data data = {
-			.dev = dev,
-			.parser = parser,
-			.error = error,
-		};
+	assert(nic_mask);
+	for (i = 0; i < size; ++i)
+		if ((nic_mask[i] | mask[i]) != nic_mask[i])
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "mask enables non supported"
+						  " bits");
+	if (!item->spec && (item->mask || item->last))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "mask/last without a spec is not"
+					  " supported");
+	if (item->spec && item->last) {
+		uint8_t spec[size];
+		uint8_t last[size];
+		unsigned int i;
+		int ret;
 
-		if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
-			continue;
-		cur_item = &mlx5_flow_items[items->type];
-		ret = cur_item->convert(items,
-					(cur_item->default_mask ?
-					 cur_item->default_mask :
-					 cur_item->mask),
-					 &data);
-		if (ret)
-			goto exit_free;
-	}
-	if (!parser->drop) {
-		/* RSS check, remove unused hash types. */
-		ret = mlx5_flow_convert_rss(parser);
-		if (ret)
-			goto exit_free;
-		/* Complete missing specification. */
-		mlx5_flow_convert_finalise(parser);
-	}
-	mlx5_flow_update_priority(dev, parser, attr);
-	if (parser->mark)
-		mlx5_flow_create_flag_mark(parser, parser->mark_id);
-	if (parser->count && parser->create) {
-		mlx5_flow_create_count(dev, parser);
-		if (!parser->cs)
-			goto exit_count_error;
-	}
-exit_free:
-	/* Only verification is expected, all resources should be released. */
-	if (!parser->create) {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (parser->queue[i].ibv_attr) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			}
-		}
-	}
-	return ret;
-exit_enomem:
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser->queue[i].ibv_attr) {
-			rte_free(parser->queue[i].ibv_attr);
-			parser->queue[i].ibv_attr = NULL;
+		for (i = 0; i < size; ++i) {
+			spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
+			last[i] = ((const uint8_t *)item->last)[i] & mask[i];
 		}
+		ret = memcmp(spec, last, size);
+		if (ret != 0)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "range is not supported");
 	}
-	rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL, "cannot allocate verbs spec attributes");
-	return -rte_errno;
-exit_count_error:
-	rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL, "cannot create counter");
-	return -rte_errno;
+	return 0;
 }
 
 /**
- * Copy the specification created into the flow.
+ * Add a verbs item specification into @p flow.
  *
- * @param parser
- *   Internal parser structure.
- * @param src
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] src
  *   Create specification.
- * @param size
+ * @param[in] size
  *   Size in bytes of the specification to copy.
  */
 static void
-mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
-		      unsigned int size)
+mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
 {
-	unsigned int i;
-	void *dst;
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset);
+	if (verbs->specs) {
+		void *dst;
+
+		dst = (void *)(verbs->specs + verbs->size);
 		memcpy(dst, src, size);
-		++parser->queue[i].ibv_attr->num_of_specs;
-		parser->queue[i].offset += size;
+		++verbs->attr->num_of_specs;
 	}
+	verbs->size += size;
+}
+
+/**
+ * Adjust verbs hash fields according to the @p flow information.
+ *
+ * @param[in, out] flow.
+ *   Pointer to flow structure.
+ * @param[in] tunnel
+ *   1 when the hash field is for a tunnel item.
+ * @param[in] layer_types
+ *   ETH_RSS_* types.
+ * @param[in] hash_fields
+ *   Item hash fields.
+ */
+static void
+mlx5_flow_verbs_hashfields_adjust(struct rte_flow *flow,
+				  int tunnel __rte_unused,
+				  uint32_t layer_types, uint64_t hash_fields)
+{
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+	hash_fields |= (tunnel ? IBV_RX_HASH_INNER : 0);
+	if (flow->rss.level == 2 && !tunnel)
+		hash_fields = 0;
+	else if (flow->rss.level < 2 && tunnel)
+		hash_fields = 0;
+#endif
+	if (!(flow->rss.types & layer_types))
+		hash_fields = 0;
+	flow->cur_verbs->hash_fields |= hash_fields;
 }
 
 /**
- * Convert Ethernet item to Verbs specification.
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
  *
- * @param item[in]
+ * @param[in] item
  *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_eth(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
+mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
 {
 	const struct rte_flow_item_eth *spec = item->spec;
 	const struct rte_flow_item_eth *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
+	const struct rte_flow_item_eth nic_mask = {
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.type = RTE_BE16(0xffff),
+	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 	struct ibv_flow_spec_eth eth = {
-		.type = parser->inner | IBV_FLOW_SPEC_ETH,
-		.size = eth_size,
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
 	};
+	int ret;
 
-	parser->layer = HASH_RXQ_ETH;
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+			    MLX5_FLOW_LAYER_OUTER_L2))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layers already configured");
+	if (!mask)
+		mask = &rte_flow_item_eth_mask;
+	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
+					(const uint8_t *)&nic_mask,
+					sizeof(struct rte_flow_item_eth),
+					error);
+	if (ret)
+		return ret;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+		MLX5_FLOW_LAYER_OUTER_L2;
+	if (size > flow_size)
+		return size;
 	if (spec) {
 		unsigned int i;
 
-		if (!mask)
-			mask = default_mask;
 		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 		eth.val.ether_type = spec->type;
@@ -1459,112 +824,211 @@ mlx5_flow_create_eth(const struct rte_flow_item *item,
 		}
 		eth.val.ether_type &= eth.mask.ether_type;
 	}
-	mlx5_flow_create_copy(parser, &eth, eth_size);
-	return 0;
+	flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	mlx5_flow_spec_verbs_add(flow, &eth, size);
+	return size;
+}
+
+/**
+ * Update the VLAN tag in the Verbs Ethernet specification.
+ *
+ * @param[in, out] attr
+ *   Pointer to Verbs attributes structure.
+ * @param[in] eth
+ *   Verbs structure containing the VLAN information to copy.
+ */
+static void
+mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
+			   struct ibv_flow_spec_eth *eth)
+{
+	unsigned int i;
+	const enum ibv_flow_spec_type search = eth->type;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			struct ibv_flow_spec_eth *e =
+				(struct ibv_flow_spec_eth *)hdr;
+
+			e->val.vlan_tag = eth->val.vlan_tag;
+			e->mask.vlan_tag = eth->mask.vlan_tag;
+			e->val.ether_type = eth->val.ether_type;
+			e->mask.ether_type = eth->mask.ether_type;
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
 }
 
 /**
- * Convert VLAN item to Verbs specification.
+ * Convert the @p item into @p flow (or by updating the already present
+ * Ethernet Verbs) specification after ensuring the NIC will understand and
+ * process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
  *
- * @param item[in]
+ * @param[in] item
  *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_vlan(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
+mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
 {
 	const struct rte_flow_item_vlan *spec = item->spec;
 	const struct rte_flow_item_vlan *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	struct ibv_flow_spec_eth *eth;
-	const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
-	const char *msg = "VLAN cannot be empty";
-
+	const struct rte_flow_item_vlan nic_mask = {
+		.tci = RTE_BE16(0x0fff),
+		.inner_type = RTE_BE16(0xffff),
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
+	};
+	int ret;
+	const uint32_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
+					MLX5_FLOW_LAYER_INNER_L4) :
+		(MLX5_FLOW_LAYER_OUTER_L3 | MLX5_FLOW_LAYER_OUTER_L4);
+	const uint32_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
+		MLX5_FLOW_LAYER_OUTER_VLAN;
+	const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+		MLX5_FLOW_LAYER_OUTER_L2;
+
+	if (flow->layers & vlanm)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VLAN layer already configured");
+	else if ((flow->layers & l34m) != 0)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layer cannot follow L3/L4 layer");
+	if (!mask)
+		mask = &rte_flow_item_vlan_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_vlan), error);
+	if (ret)
+		return ret;
 	if (spec) {
-		unsigned int i;
-		if (!mask)
-			mask = default_mask;
-
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-
-			eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				       parser->queue[i].offset - eth_size);
-			eth->val.vlan_tag = spec->tci;
-			eth->mask.vlan_tag = mask->tci;
-			eth->val.vlan_tag &= eth->mask.vlan_tag;
-			/*
-			 * From verbs perspective an empty VLAN is equivalent
-			 * to a packet without VLAN layer.
-			 */
-			if (!eth->mask.vlan_tag)
-				goto error;
-			/* Outer TPID cannot be matched. */
-			if (eth->mask.ether_type) {
-				msg = "VLAN TPID matching is not supported";
-				goto error;
-			}
-			eth->val.ether_type = spec->inner_type;
-			eth->mask.ether_type = mask->inner_type;
-			eth->val.ether_type &= eth->mask.ether_type;
-		}
-		return 0;
+		eth.val.vlan_tag = spec->tci;
+		eth.mask.vlan_tag = mask->tci;
+		eth.val.vlan_tag &= eth.mask.vlan_tag;
+		eth.val.ether_type = spec->inner_type;
+		eth.mask.ether_type = mask->inner_type;
+		eth.val.ether_type &= eth.mask.ether_type;
 	}
-error:
-	return rte_flow_error_set(data->error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
-				  item, msg);
+	/*
+	 * From verbs perspective an empty VLAN is equivalent
+	 * to a packet without VLAN layer.
+	 */
+	if (!eth.mask.vlan_tag)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					  item->spec,
+					  "VLAN cannot be empty");
+	if (!(flow->layers & l2m)) {
+		if (size <= flow_size) {
+			flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+			mlx5_flow_spec_verbs_add(flow, &eth, size);
+		}
+	} else {
+		if (flow->cur_verbs)
+			mlx5_flow_item_vlan_update(flow->cur_verbs->attr,
+						   &eth);
+		size = 0; /* Only an update is done in eth specification. */
+	}
+	flow->layers |= tunnel ?
+		(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_VLAN) :
+		(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN);
+	return size;
 }
 
 /**
- * Convert IPv4 item to Verbs specification.
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
  *
- * @param item[in]
+ * @param[in] item
  *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_ipv4(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
+mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
 {
-	struct priv *priv = data->dev->data->dev_private;
 	const struct rte_flow_item_ipv4 *spec = item->spec;
 	const struct rte_flow_item_ipv4 *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
+	const struct rte_flow_item_ipv4 nic_mask = {
+		.hdr = {
+			.src_addr = RTE_BE32(0xffffffff),
+			.dst_addr = RTE_BE32(0xffffffff),
+			.type_of_service = 0xff,
+			.next_proto_id = 0xff,
+		},
+	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 	struct ibv_flow_spec_ipv4_ext ipv4 = {
-		.type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
-		.size = ipv4_size,
+		.type = IBV_FLOW_SPEC_IPV4_EXT |
+			(tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
 	};
+	int ret;
 
-	if (parser->layer == HASH_RXQ_TUNNEL &&
-	    parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
-	    !priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			    MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				 MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	parser->layer = HASH_RXQ_IPV4;
+					  "L3 cannot follow an L4 layer.");
+	if (!mask)
+		mask = &rte_flow_item_ipv4_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_ipv4), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
+		MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 	if (spec) {
-		if (!mask)
-			mask = default_mask;
 		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 			.src_ip = spec->hdr.src_addr,
 			.dst_ip = spec->hdr.dst_addr,
@@ -1583,55 +1047,108 @@ mlx5_flow_create_ipv4(const struct rte_flow_item *item,
 		ipv4.val.proto &= ipv4.mask.proto;
 		ipv4.val.tos &= ipv4.mask.tos;
 	}
-	mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
-	return 0;
+	flow->l3_protocol_en = !!ipv4.mask.proto;
+	flow->l3_protocol = ipv4.val.proto;
+	if (size <= flow_size) {
+		mlx5_flow_verbs_hashfields_adjust
+			(flow, tunnel,
+			 (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			  ETH_RSS_NONFRAG_IPV4_OTHER),
+			 (IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4));
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
+		mlx5_flow_spec_verbs_add(flow, &ipv4, size);
+	}
+	return size;
 }
 
 /**
- * Convert IPv6 item to Verbs specification.
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
  *
- * @param item[in]
+ * @param[in] item
  *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_ipv6(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
+mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
 {
-	struct priv *priv = data->dev->data->dev_private;
 	const struct rte_flow_item_ipv6 *spec = item->spec;
 	const struct rte_flow_item_ipv6 *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
+	const struct rte_flow_item_ipv6 nic_mask = {
+		.hdr = {
+			.src_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.dst_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.vtc_flow = RTE_BE32(0xffffffff),
+			.proto = 0xff,
+			.hop_limits = 0xff,
+		},
+	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 	struct ibv_flow_spec_ipv6 ipv6 = {
-		.type = parser->inner | IBV_FLOW_SPEC_IPV6,
-		.size = ipv6_size,
+		.type = IBV_FLOW_SPEC_IPV6 | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
 	};
+	int ret;
 
-	if (parser->layer == HASH_RXQ_TUNNEL &&
-	    parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
-	    !priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			    MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				 MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 cannot follow an L4 layer.");
+	/*
+	 * IPv6 is not recognised by the NIC inside a GRE tunnel.
+	 * Such support has to be disabled as the rule will be
+	 * accepted.  Issue reproduced with Mellanox OFED 4.3-3.0.2.1 and
+	 * Mellanox OFED 4.4-1.0.0.0.
+	 */
+	if (tunnel && flow->layers & MLX5_FLOW_LAYER_GRE)
+		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	parser->layer = HASH_RXQ_IPV6;
+					  "IPv6 inside a GRE tunnel is"
+					  " not recognised.");
+	if (!mask)
+		mask = &rte_flow_item_ipv6_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_ipv6), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
+		MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 	if (spec) {
 		unsigned int i;
 		uint32_t vtc_flow_val;
 		uint32_t vtc_flow_mask;
 
-		if (!mask)
-			mask = default_mask;
 		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 		       RTE_DIM(ipv6.val.src_ip));
 		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
@@ -1666,44 +1183,86 @@ mlx5_flow_create_ipv6(const struct rte_flow_item *item,
 		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
 	}
-	mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
-	return 0;
+	flow->l3_protocol_en = !!ipv6.mask.next_hdr;
+	flow->l3_protocol = ipv6.val.next_hdr;
+	if (size <= flow_size) {
+		mlx5_flow_verbs_hashfields_adjust
+			(flow, tunnel,
+			 (ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER),
+			 (IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6));
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
+		mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	}
+	return size;
 }
 
 /**
- * Convert UDP item to Verbs specification.
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
  *
- * @param item[in]
+ * @param[in] item
  *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_udp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
+mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
 {
 	const struct rte_flow_item_udp *spec = item->spec;
 	const struct rte_flow_item_udp *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 	struct ibv_flow_spec_tcp_udp udp = {
-		.type = parser->inner | IBV_FLOW_SPEC_UDP,
-		.size = udp_size,
+		.type = IBV_FLOW_SPEC_UDP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
 	};
+	int ret;
 
-	if (parser->layer == HASH_RXQ_IPV4)
-		parser->layer = HASH_RXQ_UDPV4;
-	else
-		parser->layer = HASH_RXQ_UDPV6;
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_UDP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with UDP layer");
+	if (!(flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			      MLX5_FLOW_LAYER_OUTER_L3)))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 is mandatory to filter"
+					  " on L4");
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+			    MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already"
+					  " present");
+	if (!mask)
+		mask = &rte_flow_item_udp_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_udp_mask,
+		 sizeof(struct rte_flow_item_udp), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
+		MLX5_FLOW_LAYER_OUTER_L4_UDP;
 	if (spec) {
-		if (!mask)
-			mask = default_mask;
 		udp.val.dst_port = spec->hdr.dst_port;
 		udp.val.src_port = spec->hdr.src_port;
 		udp.mask.dst_port = mask->hdr.dst_port;
@@ -1712,44 +1271,81 @@ mlx5_flow_create_udp(const struct rte_flow_item *item,
 		udp.val.src_port &= udp.mask.src_port;
 		udp.val.dst_port &= udp.mask.dst_port;
 	}
-	mlx5_flow_create_copy(parser, &udp, udp_size);
-	return 0;
+	if (size <= flow_size) {
+		mlx5_flow_verbs_hashfields_adjust(flow, tunnel, ETH_RSS_UDP,
+						  (IBV_RX_HASH_SRC_PORT_UDP |
+						   IBV_RX_HASH_DST_PORT_UDP));
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
+		mlx5_flow_spec_verbs_add(flow, &udp, size);
+	}
+	return size;
 }
 
 /**
- * Convert TCP item to Verbs specification.
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
  *
- * @param item[in]
+ * @param[in] item
  *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_tcp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
+mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
 {
 	const struct rte_flow_item_tcp *spec = item->spec;
 	const struct rte_flow_item_tcp *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 	struct ibv_flow_spec_tcp_udp tcp = {
-		.type = parser->inner | IBV_FLOW_SPEC_TCP,
-		.size = tcp_size,
+		.type = IBV_FLOW_SPEC_TCP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
+		.size = size,
 	};
+	int ret;
 
-	if (parser->layer == HASH_RXQ_IPV4)
-		parser->layer = HASH_RXQ_TCPV4;
-	else
-		parser->layer = HASH_RXQ_TCPV6;
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_TCP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with TCP layer");
+	if (!(flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			      MLX5_FLOW_LAYER_OUTER_L3)))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 is mandatory to filter on L4");
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+			    MLX5_FLOW_LAYER_OUTER_L4))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already present");
+	if (!mask)
+		mask = &rte_flow_item_tcp_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_tcp_mask,
+		 sizeof(struct rte_flow_item_tcp), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |=  tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
+		MLX5_FLOW_LAYER_OUTER_L4_TCP;
 	if (spec) {
-		if (!mask)
-			mask = default_mask;
 		tcp.val.dst_port = spec->hdr.dst_port;
 		tcp.val.src_port = spec->hdr.src_port;
 		tcp.mask.dst_port = mask->hdr.dst_port;
@@ -1758,49 +1354,78 @@ mlx5_flow_create_tcp(const struct rte_flow_item *item,
 		tcp.val.src_port &= tcp.mask.src_port;
 		tcp.val.dst_port &= tcp.mask.dst_port;
 	}
-	mlx5_flow_create_copy(parser, &tcp, tcp_size);
-	return 0;
+	if (size <= flow_size) {
+		mlx5_flow_verbs_hashfields_adjust(flow, tunnel, ETH_RSS_TCP,
+						  (IBV_RX_HASH_SRC_PORT_TCP |
+						   IBV_RX_HASH_DST_PORT_TCP));
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
+		mlx5_flow_spec_verbs_add(flow, &tcp, size);
+	}
+	return size;
 }
 
 /**
- * Convert VXLAN item to Verbs specification.
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
  *
- * @param item[in]
+ * @param[in] item
  *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_vxlan(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data)
+mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
+		     const size_t flow_size, struct rte_flow_error *error)
 {
 	const struct rte_flow_item_vxlan *spec = item->spec;
 	const struct rte_flow_item_vxlan *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
 	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 	struct ibv_flow_spec_tunnel vxlan = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 		.size = size,
 	};
+	int ret;
 	union vni {
 		uint32_t vlan_id;
 		uint8_t vni[4];
-	} id;
+	} id = { .vlan_id = 0, };
 
-	id.vni[0] = 0;
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	/*
+	 * Verify only UDPv4 is present as defined in
+	 * https://tools.ietf.org/html/rfc7348
+	 */
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "no outer UDP layer found");
+	if (!mask)
+		mask = &rte_flow_item_vxlan_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_vxlan_mask,
+		 sizeof(struct rte_flow_item_vxlan), error);
+	if (ret < 0)
+		return ret;
 	if (spec) {
-		if (!mask)
-			mask = default_mask;
 		memcpy(&id.vni[1], spec->vni, 3);
 		vxlan.val.tunnel_id = id.vlan_id;
 		memcpy(&id.vni[1], mask->vni, 3);
@@ -1809,148 +1434,272 @@ mlx5_flow_create_vxlan(const struct rte_flow_item *item,
 		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 	}
 	/*
-	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
-	 * layer is defined in the Verbs specification it is interpreted as
-	 * wildcard and all packets will match this rule, if it follows a full
-	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
-	 * before will also match this rule.
-	 * To avoid such situation, VNI 0 is currently refused.
+	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if
+	 * only this layer is defined in the Verbs specification it is
+	 * interpreted as wildcard and all packets will match this
+	 * rule, if it follows a full stack layer (ex: eth / ipv4 /
+	 * udp), all packets matching the layers before will also
+	 * match this rule.  To avoid such situation, VNI 0 is
+	 * currently refused.
 	 */
-	/* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
-	if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
-		return rte_flow_error_set(data->error, EINVAL,
+	if (!vxlan.val.tunnel_id)
+		return rte_flow_error_set(error, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "VxLAN vni cannot be 0");
-	mlx5_flow_create_copy(parser, &vxlan, size);
-	return 0;
+					  "VXLAN vni cannot be 0");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN tunnel must be fully defined");
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &vxlan, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_VXLAN;
+	return size;
 }
 
 /**
- * Convert VXLAN-GPE item to Verbs specification.
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
  *
- * @param item[in]
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] item
  *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
-			   const void *default_mask,
-			   struct mlx5_flow_data *data)
+mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
+			 const struct rte_flow_item *item,
+			 struct rte_flow *flow, const size_t flow_size,
+			 struct rte_flow_error *error)
 {
-	struct priv *priv = data->dev->data->dev_private;
 	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
 	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel vxlan = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
+	struct ibv_flow_spec_tunnel vxlan_gpe = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 		.size = size,
 	};
+	int ret;
 	union vni {
 		uint32_t vlan_id;
 		uint8_t vni[4];
-	} id;
+	} id = { .vlan_id = 0, };
 
-	if (!priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
+	if (!((struct priv *)dev->data->dev_private)->config.l3_vxlan_en)
+		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	id.vni[0] = 0;
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
+					  "L3 VXLAN is not enabled by device"
+					  " parameter and/or not configured in"
+					  " firmware");
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	/*
+	 * Verify only UDPv4 is present as defined in
+	 * https://tools.ietf.org/html/rfc7348
+	 */
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "no outer UDP layer found");
+	if (!mask)
+		mask = &rte_flow_item_vxlan_gpe_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
+		 sizeof(struct rte_flow_item_vxlan_gpe), error);
+	if (ret < 0)
+		return ret;
 	if (spec) {
-		if (!mask)
-			mask = default_mask;
 		memcpy(&id.vni[1], spec->vni, 3);
-		vxlan.val.tunnel_id = id.vlan_id;
+		vxlan_gpe.val.tunnel_id = id.vlan_id;
 		memcpy(&id.vni[1], mask->vni, 3);
-		vxlan.mask.tunnel_id = id.vlan_id;
+		vxlan_gpe.mask.tunnel_id = id.vlan_id;
 		if (spec->protocol)
-			return rte_flow_error_set(data->error, EINVAL,
-						  RTE_FLOW_ERROR_TYPE_ITEM,
-						  item,
-						  "VxLAN-GPE protocol not"
-						  " supported");
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ITEM,
+				 item,
+				 "VxLAN-GPE protocol not supported");
 		/* Remove unwanted bits from values. */
-		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+		vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 	}
 	/*
 	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
 	 * layer is defined in the Verbs specification it is interpreted as
 	 * wildcard and all packets will match this rule, if it follows a full
 	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
-	 * before will also match this rule.
-	 * To avoid such situation, VNI 0 is currently refused.
+	 * before will also match this rule.  To avoid such situation, VNI 0
+	 * is currently refused.
 	 */
-	/* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
-	if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
-		return rte_flow_error_set(data->error, EINVAL,
+	if (!vxlan_gpe.val.tunnel_id)
+		return rte_flow_error_set(error, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "VxLAN-GPE vni cannot be 0");
-	mlx5_flow_create_copy(parser, &vxlan, size);
-	return 0;
+					  "VXLAN-GPE vni cannot be 0");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN-GPE tunnel must be fully"
+					  " defined");
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &vxlan_gpe, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_VXLAN_GPE;
+	return size;
+}
+
+/**
+ * Update the protocol in Verbs IPv4/IPv6 spec.
+ *
+ * @param[in, out] attr
+ *   Pointer to Verbs attributes structure.
+ * @param[in] search
+ *   Specification type to search in order to update the IP protocol.
+ * @param[in] protocol
+ *   Protocol value to set if none is present in the specification.
+ */
+static void
+mlx5_flow_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
+				      enum ibv_flow_spec_type search,
+				      uint8_t protocol)
+{
+	unsigned int i;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	if (!attr)
+		return;
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			union {
+				struct ibv_flow_spec_ipv4_ext *ipv4;
+				struct ibv_flow_spec_ipv6 *ipv6;
+			} ip;
+
+			switch (search) {
+			case IBV_FLOW_SPEC_IPV4_EXT:
+				ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
+				if (!ip.ipv4->val.proto) {
+					ip.ipv4->val.proto = protocol;
+					ip.ipv4->mask.proto = 0xff;
+				}
+				break;
+			case IBV_FLOW_SPEC_IPV6:
+				ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
+				if (!ip.ipv6->val.next_hdr) {
+					ip.ipv6->val.next_hdr = protocol;
+					ip.ipv6->mask.next_hdr = 0xff;
+				}
+				break;
+			default:
+				break;
+			}
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
 }
 
 /**
- * Convert GRE item to Verbs specification.
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * It will also update the previous L3 layer with the protocol value matching
+ * the GRE.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
  *
- * @param item[in]
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] item
  *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_gre(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
+mlx5_flow_item_gre(const struct rte_flow_item *item,
+		   struct rte_flow *flow, const size_t flow_size,
+		   struct rte_flow_error *error)
 {
-	struct mlx5_flow_parse *parser = data->parser;
-#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	(void)default_mask;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel tunnel = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-#else
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 	const struct rte_flow_item_gre *spec = item->spec;
 	const struct rte_flow_item_gre *mask = item->mask;
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 	unsigned int size = sizeof(struct ibv_flow_spec_gre);
 	struct ibv_flow_spec_gre tunnel = {
-		.type = parser->inner | IBV_FLOW_SPEC_GRE,
+		.type = IBV_FLOW_SPEC_GRE,
+		.size = size,
+	};
+#else
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel tunnel = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 		.size = size,
 	};
 #endif
-	struct ibv_flow_spec_ipv4_ext *ipv4;
-	struct ibv_flow_spec_ipv6 *ipv6;
-	unsigned int i;
+	int ret;
 
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_GRE)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with this GRE layer");
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 Layer is missing");
+	if (!mask)
+		mask = &rte_flow_item_gre_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_gre_mask,
+		 sizeof(struct rte_flow_item_gre), error);
+	if (ret < 0)
+		return ret;
 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 	if (spec) {
-		if (!mask)
-			mask = default_mask;
 		tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 		tunnel.val.protocol = spec->protocol;
 		tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
@@ -1960,480 +1709,1288 @@ mlx5_flow_create_gre(const struct rte_flow_item *item,
 		tunnel.val.protocol &= tunnel.mask.protocol;
 		tunnel.val.key &= tunnel.mask.key;
 	}
-#endif
-	/* Update encapsulation IP layer protocol. */
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		if (parser->out_layer == HASH_RXQ_IPV4) {
-			ipv4 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset -
-				sizeof(struct ibv_flow_spec_ipv4_ext));
-			if (ipv4->mask.proto && ipv4->val.proto != MLX5_GRE)
-				break;
-			ipv4->val.proto = MLX5_GRE;
-			ipv4->mask.proto = 0xff;
-		} else if (parser->out_layer == HASH_RXQ_IPV6) {
-			ipv6 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset -
-				sizeof(struct ibv_flow_spec_ipv6));
-			if (ipv6->mask.next_hdr &&
-			    ipv6->val.next_hdr != MLX5_GRE)
-				break;
-			ipv6->val.next_hdr = MLX5_GRE;
-			ipv6->mask.next_hdr = 0xff;
-		}
-	}
-	if (i != hash_rxq_init_n)
-		return rte_flow_error_set(data->error, EINVAL,
+#else
+	if (spec && (spec->protocol & mask->protocol))
+		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "IP protocol of GRE must be 47");
-	mlx5_flow_create_copy(parser, &tunnel, size);
-	return 0;
+					  "without MPLS support the"
+					  " specification cannot be used for"
+					  " filtering");
+#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
+	if (size <= flow_size) {
+		if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
+			mlx5_flow_item_gre_ip_protocol_update
+				(verbs->attr, IBV_FLOW_SPEC_IPV4_EXT,
+				 MLX5_IP_PROTOCOL_GRE);
+		else
+			mlx5_flow_item_gre_ip_protocol_update
+				(verbs->attr, IBV_FLOW_SPEC_IPV6,
+				 MLX5_IP_PROTOCOL_GRE);
+		mlx5_flow_spec_verbs_add(flow, &tunnel, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_GRE;
+	return size;
 }
 
 /**
- * Convert MPLS item to Verbs specification.
- * MPLS tunnel types currently supported are MPLS-in-GRE and MPLS-in-UDP.
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
  *
- * @param item[in]
+ * @param[in] item
  *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_mpls(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
+mlx5_flow_item_mpls(const struct rte_flow_item *item __rte_unused,
+		    struct rte_flow *flow __rte_unused,
+		    const size_t flow_size __rte_unused,
+		    struct rte_flow_error *error)
 {
-#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	(void)default_mask;
-	return rte_flow_error_set(data->error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_ITEM,
-				  item,
-				  "MPLS is not supported by driver");
-#else
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 	const struct rte_flow_item_mpls *spec = item->spec;
 	const struct rte_flow_item_mpls *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
 	unsigned int size = sizeof(struct ibv_flow_spec_mpls);
 	struct ibv_flow_spec_mpls mpls = {
 		.type = IBV_FLOW_SPEC_MPLS,
 		.size = size,
 	};
+	int ret;
 
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	if (parser->layer == HASH_RXQ_UDPV4 ||
-	    parser->layer == HASH_RXQ_UDPV6) {
-		parser->tunnel =
-			ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_UDP)];
-		parser->out_layer = parser->layer;
-	} else {
-		parser->tunnel =
-			ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_GRE)];
-		/* parser->out_layer stays as in GRE out_layer. */
-	}
-	parser->layer = HASH_RXQ_TUNNEL;
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_MPLS)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with MPLS layer");
+	/* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL &&
+	    (flow->layers & MLX5_FLOW_LAYER_GRE) != MLX5_FLOW_LAYER_GRE)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already"
+					  " present");
+	if (!mask)
+		mask = &rte_flow_item_mpls_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_mpls_mask,
+		 sizeof(struct rte_flow_item_mpls), error);
+	if (ret < 0)
+		return ret;
 	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		/*
-		 * The verbs label field includes the entire MPLS header:
-		 * bits 0:19 - label value field.
-		 * bits 20:22 - traffic class field.
-		 * bits 23 - bottom of stack bit.
-		 * bits 24:31 - ttl field.
-		 */
-		mpls.val.label = *(const uint32_t *)spec;
-		mpls.mask.label = *(const uint32_t *)mask;
-		/* Remove unwanted bits from values. */
+		memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
+		memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
+		/* Remove unwanted bits from values.  */
 		mpls.val.label &= mpls.mask.label;
 	}
-	mlx5_flow_create_copy(parser, &mpls, size);
-	return 0;
-#endif
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &mpls, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_MPLS;
+	return size;
+#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_ITEM,
+				  item,
+				  "MPLS is not supported by Verbs, please"
+				  " update.");
 }
 
 /**
- * Convert mark/flag action to Verbs specification.
+ * Convert the @p pattern into a Verbs specifications after ensuring the NIC
+ * will understand and process it correctly.
+ * The conversion is performed item per item, each of them is written into
+ * the @p flow if its size is lesser or equal to @p flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end of @p pattern, unless an error is encountered.
  *
- * @param parser
- *   Internal parser structure.
- * @param mark_id
- *   Mark identifier.
+ * @param[in] pattern
+ *   Flow pattern.
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @pattern  has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_items(struct rte_eth_dev *dev,
+		const struct rte_flow_item pattern[],
+		struct rte_flow *flow, const size_t flow_size,
+		struct rte_flow_error *error)
+{
+	int remain = flow_size;
+	size_t size = 0;
+
+	for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
+		int ret = 0;
+
+		switch (pattern->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			ret = mlx5_flow_item_eth(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			ret = mlx5_flow_item_vlan(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			ret = mlx5_flow_item_ipv4(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			ret = mlx5_flow_item_ipv6(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			ret = mlx5_flow_item_udp(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			ret = mlx5_flow_item_tcp(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN:
+			ret = mlx5_flow_item_vxlan(pattern, flow, remain,
+						   error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+			ret = mlx5_flow_item_vxlan_gpe(dev, pattern, flow,
+						       remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_GRE:
+			ret = mlx5_flow_item_gre(pattern, flow, remain, error);
+			break;
+		case RTE_FLOW_ITEM_TYPE_MPLS:
+			ret = mlx5_flow_item_mpls(pattern, flow, remain, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  pattern,
+						  "item not supported");
+		}
+		if (ret < 0)
+			return ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		size += ret;
+	}
+	if (!flow->layers) {
+		const struct rte_flow_item item = {
+			.type = RTE_FLOW_ITEM_TYPE_ETH,
+		};
+
+		return mlx5_flow_item_eth(&item, flow, flow_size, error);
+	}
+	return size;
+}
+
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_drop(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
+	struct ibv_flow_spec_action_drop drop = {
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+			.size = size,
+	};
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "drop is not compatible with"
+					  " flag/mark action");
+	if (size < flow_size)
+		mlx5_flow_spec_verbs_add(flow, &drop, size);
+	flow->fate |= MLX5_FLOW_FATE_DROP;
+	return size;
+}
+
+/**
+ * Convert the @p action into @p flow after ensuring the NIC will understand
+ * and process it correctly.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
+mlx5_flow_action_queue(struct rte_eth_dev *dev,
+		       const struct rte_flow_action *action,
+		       struct rte_flow *flow,
+		       struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_queue *queue = action->conf;
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (queue->index >= priv->rxqs_n)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue index out of range");
+	if (!(*priv->rxqs)[queue->index])
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue is not configured");
+	if (flow->queue)
+		(*flow->queue)[0] = queue->index;
+	flow->rss.queue_num = 1;
+	flow->fate |= MLX5_FLOW_FATE_QUEUE;
+	return 0;
+}
+
+/**
+ * Ensure the @p action will be understood and used correctly by the  NIC.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param action[in]
+ *   Pointer to flow actions array.
+ * @param flow[in, out]
+ *   Pointer to the rte_flow structure.
+ * @param error[in, out]
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success @p flow->queue array and @p flow->rss are filled and valid.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_rss(struct rte_eth_dev *dev,
+		     const struct rte_flow_action *action,
+		     struct rte_flow *flow,
+		     struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_rss *rss = action->conf;
+	unsigned int i;
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
+	    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->func,
+					  "RSS hash function not supported");
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+	if (rss->level > 2)
+#else
+	if (rss->level > 1)
+#endif
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->level,
+					  "tunnel RSS is not supported");
+	if (rss->key_len < MLX5_RSS_HASH_KEY_LEN)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too small");
+	if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too large");
+	if (!rss->queue_num)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  rss,
+					  "no queues were provided for RSS");
+	if (rss->queue_num > priv->config.ind_table_max_size)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->queue_num,
+					  "number of queues too large");
+	if (rss->types & MLX5_RSS_HF_MASK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->types,
+					  "some RSS protocols are not"
+					  " supported");
+	for (i = 0; i != rss->queue_num; ++i) {
+		if (rss->queue[i] >= priv->rxqs_n)
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 rss,
+				 "queue index out of range");
+		if (!(*priv->rxqs)[rss->queue[i]])
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 &rss->queue[i],
+				 "queue is not configured");
+	}
+	if (flow->queue)
+		memcpy((*flow->queue), rss->queue,
+		       rss->queue_num * sizeof(uint16_t));
+	flow->rss.queue_num = rss->queue_num;
+	memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
+	flow->rss.types = rss->types;
+	flow->rss.level = rss->level;
+	flow->fate |= MLX5_FLOW_FATE_RSS;
+	return 0;
+}
+
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_flag(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
 {
 	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 	struct ibv_flow_spec_action_tag tag = {
 		.type = IBV_FLOW_SPEC_ACTION_TAG,
 		.size = size,
-		.tag_id = mlx5_flow_mark_set(mark_id),
+		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+
+	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flag action already present");
+	if (flow->fate & MLX5_FLOW_FATE_DROP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flag is not compatible with drop"
+					  " action");
+	if (flow->modifier & MLX5_FLOW_MOD_MARK)
+		size = 0;
+	else if (size <= flow_size && verbs)
+		mlx5_flow_spec_verbs_add(flow, &tag, size);
+	flow->modifier |= MLX5_FLOW_MOD_FLAG;
+	return size;
+}
 
-	assert(parser->mark);
-	mlx5_flow_create_copy(parser, &tag, size);
-	return 0;
+/**
+ * Update verbs specification to modify the flag to mark.
+ *
+ * @param[in, out] verbs
+ *   Pointer to the mlx5_flow_verbs structure.
+ * @param[in] mark_id
+ *   Mark identifier to replace the flag.
+ */
+static void
+mlx5_flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
+{
+	struct ibv_spec_header *hdr;
+	int i;
+
+	if (!verbs)
+		return;
+	/* Update Verbs specification. */
+	hdr = (struct ibv_spec_header *)verbs->specs;
+	if (!hdr)
+		return;
+	for (i = 0; i != verbs->attr->num_of_specs; ++i) {
+		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
+			struct ibv_flow_spec_action_tag *t =
+				(struct ibv_flow_spec_action_tag *)hdr;
+
+			t->tag_id = mlx5_flow_mark_set(mark_id);
+		}
+		hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
+	}
 }
 
 /**
- * Convert count action to Verbs specification.
+ * Convert the @p action into @p flow (or by updating the already present
+ * Flag Verbs specification) after ensuring the NIC will understand and
+ * process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
  *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Pointer to MLX5 flow parser structure.
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_mark(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	const struct rte_flow_action_mark *mark = action->conf;
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+
+	if (!mark)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "configuration cannot be null");
+	if (mark->id >= MLX5_FLOW_MARK_MAX)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &mark->id,
+					  "mark id must in 0 <= id < "
+					  RTE_STR(MLX5_FLOW_MARK_MAX));
+	if (flow->modifier & MLX5_FLOW_MOD_MARK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "mark action already present");
+	if (flow->fate & MLX5_FLOW_FATE_DROP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "mark is not compatible with drop"
+					  " action");
+	if (flow->modifier & MLX5_FLOW_MOD_FLAG) {
+		mlx5_flow_verbs_mark_update(verbs, mark->id);
+		size = 0;
+	} else if (size <= flow_size) {
+		tag.tag_id = mlx5_flow_mark_set(mark->id);
+		mlx5_flow_spec_verbs_add(flow, &tag, size);
+	}
+	flow->modifier |= MLX5_FLOW_MOD_MARK;
+	return size;
+}
+
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param action[in]
+ *   Action configuration.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param error[int, out]
+ *   Pointer to error structure.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused,
-		       struct mlx5_flow_parse *parser __rte_unused)
+mlx5_flow_action_count(struct rte_eth_dev *dev,
+		       const struct rte_flow_action *action,
+		       struct rte_flow *flow,
+		       const size_t flow_size __rte_unused,
+		       struct rte_flow_error *error)
 {
+	const struct rte_flow_action_count *count = action->conf;
 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	struct priv *priv = dev->data->dev_private;
 	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-	struct ibv_counter_set_init_attr init_attr = {0};
 	struct ibv_flow_spec_counter_action counter = {
 		.type = IBV_FLOW_SPEC_ACTION_COUNT,
 		.size = size,
-		.counter_set_handle = 0,
 	};
+#endif
 
-	init_attr.counter_set_id = 0;
-	parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr);
-	if (!parser->cs) {
-		rte_errno = EINVAL;
-		return -rte_errno;
-	}
-	counter.counter_set_handle = parser->cs->handle;
-	mlx5_flow_create_copy(parser, &counter, size);
+	if (!flow->counter) {
+		flow->counter = mlx5_flow_counter_new(dev, count->shared,
+						      count->id);
+		if (!flow->counter)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  action,
+						  "cannot get counter"
+						  " context.");
+	}
+	if (!((struct priv *)dev->data->dev_private)->config.flow_counter_en)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flow counters are not supported.");
+	flow->modifier |= MLX5_FLOW_MOD_COUNT;
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	counter.counter_set_handle = flow->counter->cs->handle;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &counter, size);
+	return size;
 #endif
 	return 0;
 }
 
 /**
- * Complete flow rule creation with a drop queue.
+ * Convert the @p action into @p flow after ensuring the NIC will understand
+ * and process it correctly.
+ * The conversion is performed action per action, each of them is written into
+ * the @p flow if its size is lesser or equal to @p flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end of @p action, unless an error is encountered.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in] actions
+ *   Pointer to flow actions array.
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p actions has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_actions(struct rte_eth_dev *dev,
+		  const struct rte_flow_action actions[],
+		  struct rte_flow *flow, const size_t flow_size,
+		  struct rte_flow_error *error)
+{
+	size_t size = 0;
+	int remain = flow_size;
+	int ret = 0;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_FLAG:
+			ret = mlx5_flow_action_flag(actions, flow, remain,
+						    error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_MARK:
+			ret = mlx5_flow_action_mark(actions, flow, remain,
+						    error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			ret = mlx5_flow_action_drop(actions, flow, remain,
+						    error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			ret = mlx5_flow_action_queue(dev, actions, flow, error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_RSS:
+			ret = mlx5_flow_action_rss(dev, actions, flow, error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_action_count(dev, actions, flow, remain,
+						     error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
+			return ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		size += ret;
+	}
+	if (!flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "no fate action found");
+	return size;
+}
+
+/**
+ * Validate flow rule and fill flow structure accordingly.
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
+ * @param[out] flow
+ *   Pointer to flow structure.
+ * @param flow_size
+ *   Size of allocated space for @p flow.
+ * @param[in] attr
+ *   Flow rule attributes.
+ * @param[in] pattern
+ *   Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ *   Associated actions (list terminated by the END action).
  * @param[out] error
  *   Perform verbose error reporting if not NULL.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   A positive value representing the size of the flow object in bytes
+ *   regardless of @p flow_size on success, a negative errno value otherwise
+ *   and rte_errno is set.
  */
 static int
-mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev,
-				   struct mlx5_flow_parse *parser,
-				   struct rte_flow *flow,
-				   struct rte_flow_error *error)
+mlx5_flow_merge_switch(struct rte_eth_dev *dev,
+		       struct rte_flow *flow,
+		       size_t flow_size,
+		       const struct rte_flow_attr *attr,
+		       const struct rte_flow_item pattern[],
+		       const struct rte_flow_action actions[],
+		       struct rte_flow_error *error)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct ibv_flow_spec_action_drop *drop;
-	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
+	unsigned int n = mlx5_dev_to_port_id(dev->device, NULL, 0);
+	uint16_t port_id[!n + n];
+	struct mlx5_nl_flow_ptoi ptoi[!n + n + 1];
+	size_t off = RTE_ALIGN_CEIL(sizeof(*flow), alignof(max_align_t));
+	unsigned int i;
+	unsigned int own = 0;
+	int ret;
 
-	assert(priv->pd);
-	assert(priv->ctx);
-	flow->drop = 1;
-	drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
-			parser->queue[HASH_RXQ_ETH].offset);
-	*drop = (struct ibv_flow_spec_action_drop){
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-			.size = size,
-	};
-	++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
-	parser->queue[HASH_RXQ_ETH].offset += size;
-	flow->frxq[HASH_RXQ_ETH].ibv_attr =
-		parser->queue[HASH_RXQ_ETH].ibv_attr;
-	if (parser->count)
-		flow->cs = parser->cs;
-	if (!dev->data->dev_started)
-		return 0;
-	parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
-	flow->frxq[HASH_RXQ_ETH].ibv_flow =
-		mlx5_glue->create_flow(priv->flow_drop_queue->qp,
-				       flow->frxq[HASH_RXQ_ETH].ibv_attr);
-	if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-		rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "flow rule creation failure");
-		goto error;
-	}
-	return 0;
-error:
-	assert(flow);
-	if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-		claim_zero(mlx5_glue->destroy_flow
-			   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-		flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
-	}
-	if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
-		rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
-		flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
+	/* At least one port is needed when no switch domain is present. */
+	if (!n) {
+		n = 1;
+		port_id[0] = dev->data->port_id;
+	} else {
+		n = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, n), n);
+	}
+	for (i = 0; i != n; ++i) {
+		struct rte_eth_dev_info dev_info;
+
+		rte_eth_dev_info_get(port_id[i], &dev_info);
+		if (port_id[i] == dev->data->port_id)
+			own = i;
+		ptoi[i].port_id = port_id[i];
+		ptoi[i].ifindex = dev_info.if_index;
+	}
+	/* Ensure first entry of ptoi[] is the current device. */
+	if (own) {
+		ptoi[n] = ptoi[0];
+		ptoi[0] = ptoi[own];
+		ptoi[own] = ptoi[n];
+	}
+	/* An entry with zero ifindex terminates ptoi[]. */
+	ptoi[n].port_id = 0;
+	ptoi[n].ifindex = 0;
+	if (flow_size < off)
+		flow_size = 0;
+	ret = mlx5_nl_flow_transpose((uint8_t *)flow + off,
+				     flow_size ? flow_size - off : 0,
+				     ptoi, attr, pattern, actions, error);
+	if (ret < 0)
+		return ret;
+	if (flow_size) {
+		*flow = (struct rte_flow){
+			.attributes = *attr,
+			.nl_flow = (uint8_t *)flow + off,
+		};
+		/*
+		 * Generate a reasonably unique handle based on the address
+		 * of the target buffer.
+		 *
+		 * This is straightforward on 32-bit systems where the flow
+		 * pointer can be used directly. Otherwise, its least
+		 * significant part is taken after shifting it by the
+		 * previous power of two of the pointed buffer size.
+		 */
+		if (sizeof(flow) <= 4)
+			mlx5_nl_flow_brand(flow->nl_flow, (uintptr_t)flow);
+		else
+			mlx5_nl_flow_brand
+				(flow->nl_flow,
+				 (uintptr_t)flow >>
+				 rte_log2_u32(rte_align32prevpow2(flow_size)));
 	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-		parser->cs = NULL;
+	return off + ret;
+}
+
+static unsigned int
+mlx5_find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
+{
+	const struct rte_flow_item *item;
+	unsigned int has_vlan = 0;
+
+	for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
+		if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
+			has_vlan = 1;
+			break;
+		}
 	}
-	return -rte_errno;
+	if (has_vlan)
+		return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
+				       MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
+	return rss_level < 2 ? MLX5_EXPANSION_ROOT :
+			       MLX5_EXPANSION_ROOT_OUTER;
 }
 
 /**
- * Create hash Rx queues when RSS is enabled.
- *
- * @param dev
+ * Convert the @p attributes, @p pattern, @p action, into an flow for the NIC
+ * after ensuring the NIC will understand and process it correctly.
+ * The conversion is only performed item/action per item/action, each of
+ * them is written into the @p flow if its size is lesser or equal to @p
+ * flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end, unless an error is encountered.
+ *
+ * @param[in] dev
  *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[in] attributes
+ *   Flow rule attributes.
+ * @param[in] pattern
+ *   Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ *   Associated actions (list terminated by the END action).
  * @param[out] error
  *   Perform verbose error reporting if not NULL.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the flow has fully been converted and
+ *   can be applied, otherwise another call with this returned memory size
+ *   should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev,
-				  struct mlx5_flow_parse *parser,
-				  struct rte_flow *flow,
-				  struct rte_flow_error *error)
+mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
+		const size_t flow_size,
+		const struct rte_flow_attr *attributes,
+		const struct rte_flow_item pattern[],
+		const struct rte_flow_action actions[],
+		struct rte_flow_error *error)
+{
+	struct rte_flow local_flow = { .layers = 0, };
+	size_t size = sizeof(*flow);
+	union {
+		struct rte_flow_expand_rss buf;
+		uint8_t buffer[2048];
+	} expand_buffer;
+	struct rte_flow_expand_rss *buf = &expand_buffer.buf;
+	struct mlx5_flow_verbs *original_verbs = NULL;
+	size_t original_verbs_size = 0;
+	uint32_t original_layers = 0;
+	int expanded_pattern_idx = 0;
+	int ret;
+	uint32_t i;
+
+	if (attributes->transfer)
+		return mlx5_flow_merge_switch(dev, flow, flow_size,
+					      attributes, pattern,
+					      actions, error);
+	if (size > flow_size)
+		flow = &local_flow;
+	ret = mlx5_flow_attributes(dev, attributes, flow, error);
+	if (ret < 0)
+		return ret;
+	ret = mlx5_flow_actions(dev, actions, &local_flow, 0, error);
+	if (ret < 0)
+		return ret;
+	if (local_flow.rss.types) {
+		unsigned int graph_root;
+
+		graph_root = mlx5_find_graph_root(pattern,
+						  local_flow.rss.level);
+		ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
+					  pattern, local_flow.rss.types,
+					  mlx5_support_expansion,
+					  graph_root);
+		assert(ret > 0 &&
+		       (unsigned int)ret < sizeof(expand_buffer.buffer));
+	} else {
+		buf->entries = 1;
+		buf->entry[0].pattern = (void *)(uintptr_t)pattern;
+	}
+	size += RTE_ALIGN_CEIL(local_flow.rss.queue_num * sizeof(uint16_t),
+			       sizeof(void *));
+	if (size <= flow_size)
+		flow->queue = (void *)(flow + 1);
+	LIST_INIT(&flow->verbs);
+	flow->layers = 0;
+	flow->modifier = 0;
+	flow->fate = 0;
+	for (i = 0; i != buf->entries; ++i) {
+		size_t off = size;
+		size_t off2;
+
+		flow->layers = original_layers;
+		size += sizeof(struct ibv_flow_attr) +
+			sizeof(struct mlx5_flow_verbs);
+		off2 = size;
+		if (size < flow_size) {
+			flow->cur_verbs = (void *)((uintptr_t)flow + off);
+			flow->cur_verbs->attr = (void *)(flow->cur_verbs + 1);
+			flow->cur_verbs->specs =
+				(void *)(flow->cur_verbs->attr + 1);
+		}
+		/* First iteration convert the pattern into Verbs. */
+		if (i == 0) {
+			/* Actions don't need to be converted several time. */
+			ret = mlx5_flow_actions(dev, actions, flow,
+						(size < flow_size) ?
+						flow_size - size : 0,
+						error);
+			if (ret < 0)
+				return ret;
+			size += ret;
+		} else {
+			/*
+			 * Next iteration means the pattern has already been
+			 * converted and an expansion is necessary to match
+			 * the user RSS request.  For that only the expanded
+			 * items will be converted, the common part with the
+			 * user pattern are just copied into the next buffer
+			 * zone.
+			 */
+			size += original_verbs_size;
+			if (size < flow_size) {
+				rte_memcpy(flow->cur_verbs->attr,
+					   original_verbs->attr,
+					   original_verbs_size +
+					   sizeof(struct ibv_flow_attr));
+				flow->cur_verbs->size = original_verbs_size;
+			}
+		}
+		ret = mlx5_flow_items
+			(dev,
+			 (const struct rte_flow_item *)
+			 &buf->entry[i].pattern[expanded_pattern_idx],
+			 flow,
+			 (size < flow_size) ? flow_size - size : 0, error);
+		if (ret < 0)
+			return ret;
+		size += ret;
+		if (size <= flow_size) {
+			mlx5_flow_adjust_priority(dev, flow);
+			LIST_INSERT_HEAD(&flow->verbs, flow->cur_verbs, next);
+		}
+		/*
+		 * Keep a pointer of the first verbs conversion and the layers
+		 * it has encountered.
+		 */
+		if (i == 0) {
+			original_verbs = flow->cur_verbs;
+			original_verbs_size = size - off2;
+			original_layers = flow->layers;
+			/*
+			 * move the index of the expanded pattern to the
+			 * first item not addressed yet.
+			 */
+			if (pattern->type == RTE_FLOW_ITEM_TYPE_END) {
+				expanded_pattern_idx++;
+			} else {
+				const struct rte_flow_item *item = pattern;
+
+				for (item = pattern;
+				     item->type != RTE_FLOW_ITEM_TYPE_END;
+				     ++item)
+					expanded_pattern_idx++;
+			}
+		}
+	}
+	/* Restore the origin layers in the flow. */
+	flow->layers = original_layers;
+	return size;
+}
+
+/**
+ * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
+ * if several tunnel rules are used on this queue, the tunnel ptype will be
+ * cleared.
+ *
+ * @param rxq_ctrl
+ *   Rx queue to update.
+ */
+static void
+mlx5_flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
 {
 	unsigned int i;
+	uint32_t tunnel_ptype = 0;
 
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
+	/* Look up for the ptype to use. */
+	for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
+		if (!rxq_ctrl->flow_tunnels_n[i])
 			continue;
-		flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
-		parser->queue[i].ibv_attr = NULL;
-		flow->frxq[i].hash_fields = parser->queue[i].hash_fields;
-		if (!dev->data->dev_started)
-			continue;
-		flow->frxq[i].hrxq =
-			mlx5_hrxq_get(dev,
-				      parser->rss_conf.key,
-				      parser->rss_conf.key_len,
-				      flow->frxq[i].hash_fields,
-				      parser->rss_conf.queue,
-				      parser->rss_conf.queue_num,
-				      parser->tunnel,
-				      parser->rss_conf.level);
-		if (flow->frxq[i].hrxq)
-			continue;
-		flow->frxq[i].hrxq =
-			mlx5_hrxq_new(dev,
-				      parser->rss_conf.key,
-				      parser->rss_conf.key_len,
-				      flow->frxq[i].hash_fields,
-				      parser->rss_conf.queue,
-				      parser->rss_conf.queue_num,
-				      parser->tunnel,
-				      parser->rss_conf.level);
-		if (!flow->frxq[i].hrxq) {
-			return rte_flow_error_set(error, ENOMEM,
-						  RTE_FLOW_ERROR_TYPE_HANDLE,
-						  NULL,
-						  "cannot create hash rxq");
+		if (!tunnel_ptype) {
+			tunnel_ptype = tunnels_info[i].ptype;
+		} else {
+			tunnel_ptype = 0;
+			break;
 		}
 	}
-	return 0;
+	rxq_ctrl->rxq.tunnel = tunnel_ptype;
 }
 
 /**
- * RXQ update after flow rule creation.
+ * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the flow.
  *
- * @param dev
+ * @param[in] dev
  *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to the flow rule.
+ * @param[in] flow
+ *   Pointer to flow structure.
  */
 static void
-mlx5_flow_create_update_rxqs(struct rte_eth_dev *dev, struct rte_flow *flow)
+mlx5_flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
 	struct priv *priv = dev->data->dev_private;
+	const int mark = !!(flow->modifier &
+			    (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK));
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int i;
-	unsigned int j;
 
-	if (!dev->data->dev_started)
-		return;
-	for (i = 0; i != flow->rss_conf.queue_num; ++i) {
-		struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
-						 [(*flow->queues)[i]];
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int idx = (*flow->queue)[i];
 		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
-		uint8_t tunnel = PTYPE_IDX(flow->tunnel);
+			container_of((*priv->rxqs)[idx],
+				     struct mlx5_rxq_ctrl, rxq);
 
-		rxq_data->mark |= flow->mark;
-		if (!tunnel)
-			continue;
-		rxq_ctrl->tunnel_types[tunnel] += 1;
-		/* Clear tunnel type if more than one tunnel types set. */
-		for (j = 0; j != RTE_DIM(rxq_ctrl->tunnel_types); ++j) {
-			if (j == tunnel)
-				continue;
-			if (rxq_ctrl->tunnel_types[j] > 0) {
-				rxq_data->tunnel = 0;
-				break;
+		if (mark) {
+			rxq_ctrl->rxq.mark = 1;
+			rxq_ctrl->flow_mark_n++;
+		}
+		if (tunnel) {
+			unsigned int j;
+
+			/* Increase the counter matching the flow. */
+			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
+				if ((tunnels_info[j].tunnel & flow->layers) ==
+				    tunnels_info[j].tunnel) {
+					rxq_ctrl->flow_tunnels_n[j]++;
+					break;
+				}
 			}
+			mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
 		}
-		if (j == RTE_DIM(rxq_ctrl->tunnel_types))
-			rxq_data->tunnel = flow->tunnel;
 	}
 }
 
 /**
- * Dump flow hash RX queue detail.
+ * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
+ * @p flow if no other flow uses it with the same kind of request.
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to the rte_flow.
- * @param hrxq_idx
- *   Hash RX queue index.
+ * @param[in] flow
+ *   Pointer to the flow.
  */
 static void
-mlx5_flow_dump(struct rte_eth_dev *dev __rte_unused,
-	       struct rte_flow *flow __rte_unused,
-	       unsigned int hrxq_idx __rte_unused)
+mlx5_flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
-#ifndef NDEBUG
-	uintptr_t spec_ptr;
-	uint16_t j;
-	char buf[256];
-	uint8_t off;
-	uint64_t extra_hash_fields = 0;
+	struct priv *priv = dev->data->dev_private;
+	const int mark = !!(flow->modifier &
+			    (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK));
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int i;
 
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	if (flow->tunnel && flow->rss_conf.level > 1)
-		extra_hash_fields = (uint32_t)IBV_RX_HASH_INNER;
-#endif
-	spec_ptr = (uintptr_t)(flow->frxq[hrxq_idx].ibv_attr + 1);
-	for (j = 0, off = 0; j < flow->frxq[hrxq_idx].ibv_attr->num_of_specs;
-	     j++) {
-		struct ibv_flow_spec *spec = (void *)spec_ptr;
-		off += sprintf(buf + off, " %x(%hu)", spec->hdr.type,
-			       spec->hdr.size);
-		spec_ptr += spec->hdr.size;
-	}
-	DRV_LOG(DEBUG,
-		"port %u Verbs flow %p type %u: hrxq:%p qp:%p ind:%p,"
-		" hash:%" PRIx64 "/%u specs:%hhu(%hu), priority:%hu, type:%d,"
-		" flags:%x, comp_mask:%x specs:%s",
-		dev->data->port_id, (void *)flow, hrxq_idx,
-		(void *)flow->frxq[hrxq_idx].hrxq,
-		(void *)flow->frxq[hrxq_idx].hrxq->qp,
-		(void *)flow->frxq[hrxq_idx].hrxq->ind_table,
-		(flow->frxq[hrxq_idx].hash_fields | extra_hash_fields),
-		flow->rss_conf.queue_num,
-		flow->frxq[hrxq_idx].ibv_attr->num_of_specs,
-		flow->frxq[hrxq_idx].ibv_attr->size,
-		flow->frxq[hrxq_idx].ibv_attr->priority,
-		flow->frxq[hrxq_idx].ibv_attr->type,
-		flow->frxq[hrxq_idx].ibv_attr->flags,
-		flow->frxq[hrxq_idx].ibv_attr->comp_mask,
-		buf);
-#endif
+	assert(dev->data->dev_started);
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int idx = (*flow->queue)[i];
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[idx],
+				     struct mlx5_rxq_ctrl, rxq);
+
+		if (mark) {
+			rxq_ctrl->flow_mark_n--;
+			rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
+		}
+		if (tunnel) {
+			unsigned int j;
+
+			/* Decrease the counter matching the flow. */
+			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
+				if ((tunnels_info[j].tunnel & flow->layers) ==
+				    tunnels_info[j].tunnel) {
+					rxq_ctrl->flow_tunnels_n[j]--;
+					break;
+				}
+			}
+			mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
+		}
+	}
 }
 
 /**
- * Complete flow rule creation.
+ * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
+ */
+static void
+mlx5_flow_rxq_flags_clear(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+
+	for (i = 0; i != priv->rxqs_n; ++i) {
+		struct mlx5_rxq_ctrl *rxq_ctrl;
+		unsigned int j;
+
+		if (!(*priv->rxqs)[i])
+			continue;
+		rxq_ctrl = container_of((*priv->rxqs)[i],
+					struct mlx5_rxq_ctrl, rxq);
+		rxq_ctrl->flow_mark_n = 0;
+		rxq_ctrl->rxq.mark = 0;
+		for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
+			rxq_ctrl->flow_tunnels_n[j] = 0;
+		rxq_ctrl->rxq.tunnel = 0;
+	}
+}
+
+/**
+ * Validate a flow supported by the NIC.
+ *
+ * @see rte_flow_validate()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_validate(struct rte_eth_dev *dev,
+		   const struct rte_flow_attr *attr,
+		   const struct rte_flow_item items[],
+		   const struct rte_flow_action actions[],
+		   struct rte_flow_error *error)
+{
+	int ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+
+	if (ret < 0)
+		return ret;
+	return 0;
+}
+
+/**
+ * Remove the flow.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ */
+static void
+mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_verbs *verbs;
+
+	if (flow->nl_flow && priv->mnl_socket)
+		mlx5_nl_flow_destroy(priv->mnl_socket, flow->nl_flow, NULL);
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->flow) {
+			claim_zero(mlx5_glue->destroy_flow(verbs->flow));
+			verbs->flow = NULL;
+		}
+		if (verbs->hrxq) {
+			if (flow->fate & MLX5_FLOW_FATE_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
+		}
+	}
+	if (flow->counter) {
+		mlx5_flow_counter_release(flow->counter);
+		flow->counter = NULL;
+	}
+}
+
+/**
+ * Apply the flow.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
  * @param[out] error
- *   Perform verbose error reporting if not NULL.
+ *   Pointer to error structure.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
-			      struct mlx5_flow_parse *parser,
-			      struct rte_flow *flow,
-			      struct rte_flow_error *error)
+mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
+		struct rte_flow_error *error)
 {
-	struct priv *priv __rte_unused = dev->data->dev_private;
-	int ret;
-	unsigned int i;
-	unsigned int flows_n = 0;
-
-	assert(priv->pd);
-	assert(priv->ctx);
-	assert(!parser->drop);
-	ret = mlx5_flow_create_action_queue_rss(dev, parser, flow, error);
-	if (ret)
-		goto error;
-	if (parser->count)
-		flow->cs = parser->cs;
-	if (!dev->data->dev_started)
-		return 0;
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!flow->frxq[i].hrxq)
-			continue;
-		flow->frxq[i].ibv_flow =
-			mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
-					       flow->frxq[i].ibv_attr);
-		mlx5_flow_dump(dev, flow, i);
-		if (!flow->frxq[i].ibv_flow) {
-			rte_flow_error_set(error, ENOMEM,
-					   RTE_FLOW_ERROR_TYPE_HANDLE,
-					   NULL, "flow rule creation failure");
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_verbs *verbs;
+	int err;
+
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (flow->fate & MLX5_FLOW_FATE_DROP) {
+			verbs->hrxq = mlx5_hrxq_drop_new(dev);
+			if (!verbs->hrxq) {
+				rte_flow_error_set
+					(error, errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get drop hash queue");
+				goto error;
+			}
+		} else {
+			struct mlx5_hrxq *hrxq;
+
+			hrxq = mlx5_hrxq_get(dev, flow->key,
+					     MLX5_RSS_HASH_KEY_LEN,
+					     verbs->hash_fields,
+					     (*flow->queue),
+					     flow->rss.queue_num);
+			if (!hrxq)
+				hrxq = mlx5_hrxq_new(dev, flow->key,
+						     MLX5_RSS_HASH_KEY_LEN,
+						     verbs->hash_fields,
+						     (*flow->queue),
+						     flow->rss.queue_num,
+						     !!(flow->layers &
+						      MLX5_FLOW_LAYER_TUNNEL));
+			if (!hrxq) {
+				rte_flow_error_set
+					(error, rte_errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get hash queue");
+				goto error;
+			}
+			verbs->hrxq = hrxq;
+		}
+		verbs->flow =
+			mlx5_glue->create_flow(verbs->hrxq->qp, verbs->attr);
+		if (!verbs->flow) {
+			rte_flow_error_set(error, errno,
+					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					   NULL,
+					   "hardware refuses to create flow");
 			goto error;
 		}
-		++flows_n;
 	}
-	if (!flows_n) {
-		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "internal error in flow creation");
+	if (flow->nl_flow &&
+	    priv->mnl_socket &&
+	    mlx5_nl_flow_create(priv->mnl_socket, flow->nl_flow, error))
 		goto error;
-	}
-	mlx5_flow_create_update_rxqs(dev, flow);
 	return 0;
 error:
-	ret = rte_errno; /* Save rte_errno before cleanup. */
-	assert(flow);
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (flow->frxq[i].ibv_flow) {
-			struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
-
-			claim_zero(mlx5_glue->destroy_flow(ibv_flow));
+	err = rte_errno; /* Save rte_errno before cleanup. */
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->hrxq) {
+			if (flow->fate & MLX5_FLOW_FATE_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
 		}
-		if (flow->frxq[i].hrxq)
-			mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
-		if (flow->frxq[i].ibv_attr)
-			rte_free(flow->frxq[i].ibv_attr);
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-		parser->cs = NULL;
 	}
-	rte_errno = ret; /* Restore rte_errno. */
+	rte_errno = err; /* Restore rte_errno. */
 	return -rte_errno;
 }
 
 /**
- * Convert a flow.
+ * Create a flow and add it to @p list.
  *
  * @param dev
  *   Pointer to Ethernet device.
@@ -2441,7 +2998,7 @@ error:
  *   Pointer to a TAILQ flow list.
  * @param[in] attr
  *   Flow rule attributes.
- * @param[in] pattern
+ * @param[in] items
  *   Pattern specification (list terminated by the END pattern item).
  * @param[in] actions
  *   Associated actions (list terminated by the END action).
@@ -2459,81 +3016,43 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 		      const struct rte_flow_action actions[],
 		      struct rte_flow_error *error)
 {
-	struct mlx5_flow_parse parser = { .create = 1, };
 	struct rte_flow *flow = NULL;
-	unsigned int i;
+	size_t size = 0;
 	int ret;
 
-	ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
-	if (ret)
-		goto exit;
-	flow = rte_calloc(__func__, 1,
-			  sizeof(*flow) +
-			  parser.rss_conf.queue_num * sizeof(uint16_t),
-			  0);
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
+	if (ret < 0)
+		return NULL;
+	size = ret;
+	flow = rte_calloc(__func__, 1, size, 0);
 	if (!flow) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL,
-				   "cannot allocate flow memory");
+				   "not enough memory to create flow");
 		return NULL;
 	}
-	/* Copy configuration. */
-	flow->queues = (uint16_t (*)[])(flow + 1);
-	flow->tunnel = parser.tunnel;
-	flow->rss_conf = (struct rte_flow_action_rss){
-		.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
-		.level = parser.rss_conf.level,
-		.types = parser.rss_conf.types,
-		.key_len = parser.rss_conf.key_len,
-		.queue_num = parser.rss_conf.queue_num,
-		.key = memcpy(flow->rss_key, parser.rss_conf.key,
-			      sizeof(*parser.rss_conf.key) *
-			      parser.rss_conf.key_len),
-		.queue = memcpy(flow->queues, parser.rss_conf.queue,
-				sizeof(*parser.rss_conf.queue) *
-				parser.rss_conf.queue_num),
-	};
-	flow->mark = parser.mark;
-	/* finalise the flow. */
-	if (parser.drop)
-		ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow,
-							 error);
-	else
-		ret = mlx5_flow_create_action_queue(dev, &parser, flow, error);
-	if (ret)
-		goto exit;
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
+	if (ret < 0) {
+		rte_free(flow);
+		return NULL;
+	}
+	assert((size_t)ret == size);
+	if (dev->data->dev_started) {
+		ret = mlx5_flow_apply(dev, flow, error);
+		if (ret < 0) {
+			ret = rte_errno; /* Save rte_errno before cleanup. */
+			if (flow) {
+				mlx5_flow_remove(dev, flow);
+				rte_free(flow);
+			}
+			rte_errno = ret; /* Restore rte_errno. */
+			return NULL;
+		}
+	}
 	TAILQ_INSERT_TAIL(list, flow, next);
-	DRV_LOG(DEBUG, "port %u flow created %p", dev->data->port_id,
-		(void *)flow);
+	mlx5_flow_rxq_flags_set(dev, flow);
 	return flow;
-exit:
-	DRV_LOG(ERR, "port %u flow creation error: %s", dev->data->port_id,
-		error->message);
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser.queue[i].ibv_attr)
-			rte_free(parser.queue[i].ibv_attr);
-	}
-	rte_free(flow);
-	return NULL;
-}
-
-/**
- * Validate a flow supported by the NIC.
- *
- * @see rte_flow_validate()
- * @see rte_flow_ops
- */
-int
-mlx5_flow_validate(struct rte_eth_dev *dev,
-		   const struct rte_flow_attr *attr,
-		   const struct rte_flow_item items[],
-		   const struct rte_flow_action actions[],
-		   struct rte_flow_error *error)
-{
-	struct mlx5_flow_parse parser = { .create = 0, };
-
-	return mlx5_flow_convert(dev, attr, items, actions, error, &parser);
 }
 
 /**
@@ -2549,10 +3068,9 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	struct priv *priv = dev->data->dev_private;
-
-	return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions,
-				     error);
+	return mlx5_flow_list_create
+		(dev, &((struct priv *)dev->data->dev_private)->flows,
+		 attr, items, actions, error);
 }
 
 /**
@@ -2569,95 +3087,14 @@ static void
 mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
 		       struct rte_flow *flow)
 {
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-
-	if (flow->drop || !dev->data->dev_started)
-		goto free;
-	for (i = 0; flow->tunnel && i != flow->rss_conf.queue_num; ++i) {
-		/* Update queue tunnel type. */
-		struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
-						 [(*flow->queues)[i]];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
-		uint8_t tunnel = PTYPE_IDX(flow->tunnel);
-
-		assert(rxq_ctrl->tunnel_types[tunnel] > 0);
-		rxq_ctrl->tunnel_types[tunnel] -= 1;
-		if (!rxq_ctrl->tunnel_types[tunnel]) {
-			/* Update tunnel type. */
-			uint8_t j;
-			uint8_t types = 0;
-			uint8_t last;
-
-			for (j = 0; j < RTE_DIM(rxq_ctrl->tunnel_types); j++)
-				if (rxq_ctrl->tunnel_types[j]) {
-					types += 1;
-					last = j;
-				}
-			/* Keep same if more than one tunnel types left. */
-			if (types == 1)
-				rxq_data->tunnel = ptype_ext[last];
-			else if (types == 0)
-				/* No tunnel type left. */
-				rxq_data->tunnel = 0;
-		}
-	}
-	for (i = 0; flow->mark && i != flow->rss_conf.queue_num; ++i) {
-		struct rte_flow *tmp;
-		int mark = 0;
-
-		/*
-		 * To remove the mark from the queue, the queue must not be
-		 * present in any other marked flow (RSS or not).
-		 */
-		TAILQ_FOREACH(tmp, list, next) {
-			unsigned int j;
-			uint16_t *tqs = NULL;
-			uint16_t tq_n = 0;
-
-			if (!tmp->mark)
-				continue;
-			for (j = 0; j != hash_rxq_init_n; ++j) {
-				if (!tmp->frxq[j].hrxq)
-					continue;
-				tqs = tmp->frxq[j].hrxq->ind_table->queues;
-				tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
-			}
-			if (!tq_n)
-				continue;
-			for (j = 0; (j != tq_n) && !mark; j++)
-				if (tqs[j] == (*flow->queues)[i])
-					mark = 1;
-		}
-		(*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
-	}
-free:
-	if (flow->drop) {
-		if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-		rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
-	} else {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			struct mlx5_flow *frxq = &flow->frxq[i];
-
-			if (frxq->ibv_flow)
-				claim_zero(mlx5_glue->destroy_flow
-					   (frxq->ibv_flow));
-			if (frxq->hrxq)
-				mlx5_hrxq_release(dev, frxq->hrxq);
-			if (frxq->ibv_attr)
-				rte_free(frxq->ibv_attr);
-		}
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-	}
+	mlx5_flow_remove(dev, flow);
 	TAILQ_REMOVE(list, flow, next);
-	DRV_LOG(DEBUG, "port %u flow destroyed %p", dev->data->port_id,
-		(void *)flow);
+	/*
+	 * Update RX queue flags only if port is started, otherwise it is
+	 * already clean.
+	 */
+	if (dev->data->dev_started)
+		mlx5_flow_rxq_flags_trim(dev, flow);
 	rte_free(flow);
 }
 
@@ -2681,135 +3118,6 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
 }
 
 /**
- * Create drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-int
-mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
-{
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_hrxq_drop *fdq = NULL;
-
-	assert(priv->pd);
-	assert(priv->ctx);
-	fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
-	if (!fdq) {
-		DRV_LOG(WARNING,
-			"port %u cannot allocate memory for drop queue",
-			dev->data->port_id);
-		rte_errno = ENOMEM;
-		return -rte_errno;
-	}
-	fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
-	if (!fdq->cq) {
-		DRV_LOG(WARNING, "port %u cannot allocate CQ for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->wq = mlx5_glue->create_wq
-		(priv->ctx,
-		 &(struct ibv_wq_init_attr){
-			.wq_type = IBV_WQT_RQ,
-			.max_wr = 1,
-			.max_sge = 1,
-			.pd = priv->pd,
-			.cq = fdq->cq,
-		 });
-	if (!fdq->wq) {
-		DRV_LOG(WARNING, "port %u cannot allocate WQ for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->ind_table = mlx5_glue->create_rwq_ind_table
-		(priv->ctx,
-		 &(struct ibv_rwq_ind_table_init_attr){
-			.log_ind_tbl_size = 0,
-			.ind_tbl = &fdq->wq,
-			.comp_mask = 0,
-		 });
-	if (!fdq->ind_table) {
-		DRV_LOG(WARNING,
-			"port %u cannot allocate indirection table for drop"
-			" queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->qp = mlx5_glue->create_qp_ex
-		(priv->ctx,
-		 &(struct ibv_qp_init_attr_ex){
-			.qp_type = IBV_QPT_RAW_PACKET,
-			.comp_mask =
-				IBV_QP_INIT_ATTR_PD |
-				IBV_QP_INIT_ATTR_IND_TABLE |
-				IBV_QP_INIT_ATTR_RX_HASH,
-			.rx_hash_conf = (struct ibv_rx_hash_conf){
-				.rx_hash_function =
-					IBV_RX_HASH_FUNC_TOEPLITZ,
-				.rx_hash_key_len = rss_hash_default_key_len,
-				.rx_hash_key = rss_hash_default_key,
-				.rx_hash_fields_mask = 0,
-				},
-			.rwq_ind_tbl = fdq->ind_table,
-			.pd = priv->pd
-		 });
-	if (!fdq->qp) {
-		DRV_LOG(WARNING, "port %u cannot allocate QP for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	priv->flow_drop_queue = fdq;
-	return 0;
-error:
-	if (fdq->qp)
-		claim_zero(mlx5_glue->destroy_qp(fdq->qp));
-	if (fdq->ind_table)
-		claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
-	if (fdq->wq)
-		claim_zero(mlx5_glue->destroy_wq(fdq->wq));
-	if (fdq->cq)
-		claim_zero(mlx5_glue->destroy_cq(fdq->cq));
-	if (fdq)
-		rte_free(fdq);
-	priv->flow_drop_queue = NULL;
-	return -rte_errno;
-}
-
-/**
- * Delete drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- */
-void
-mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
-{
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
-
-	if (!fdq)
-		return;
-	if (fdq->qp)
-		claim_zero(mlx5_glue->destroy_qp(fdq->qp));
-	if (fdq->ind_table)
-		claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
-	if (fdq->wq)
-		claim_zero(mlx5_glue->destroy_wq(fdq->wq));
-	if (fdq->cq)
-		claim_zero(mlx5_glue->destroy_cq(fdq->cq));
-	rte_free(fdq);
-	priv->flow_drop_queue = NULL;
-}
-
-/**
  * Remove all flows.
  *
  * @param dev
@@ -2820,68 +3128,11 @@ mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
 void
 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
-	struct priv *priv = dev->data->dev_private;
 	struct rte_flow *flow;
-	unsigned int i;
 
-	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
-		struct mlx5_ind_table_ibv *ind_tbl = NULL;
-
-		if (flow->drop) {
-			if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
-				continue;
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-			flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
-			DRV_LOG(DEBUG, "port %u flow %p removed",
-				dev->data->port_id, (void *)flow);
-			/* Next flow. */
-			continue;
-		}
-		/* Verify the flow has not already been cleaned. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_flow)
-				continue;
-			/*
-			 * Indirection table may be necessary to remove the
-			 * flags in the Rx queues.
-			 * This helps to speed-up the process by avoiding
-			 * another loop.
-			 */
-			ind_tbl = flow->frxq[i].hrxq->ind_table;
-			break;
-		}
-		if (i == hash_rxq_init_n)
-			return;
-		if (flow->mark) {
-			assert(ind_tbl);
-			for (i = 0; i != ind_tbl->queues_n; ++i)
-				(*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
-		}
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_flow)
-				continue;
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[i].ibv_flow));
-			flow->frxq[i].ibv_flow = NULL;
-			mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
-			flow->frxq[i].hrxq = NULL;
-		}
-		DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id,
-			(void *)flow);
-	}
-	/* Cleanup Rx queue tunnel info. */
-	for (i = 0; i != priv->rxqs_n; ++i) {
-		struct mlx5_rxq_data *q = (*priv->rxqs)[i];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(q, struct mlx5_rxq_ctrl, rxq);
-
-		if (!q)
-			continue;
-		memset((void *)rxq_ctrl->tunnel_types, 0,
-		       sizeof(rxq_ctrl->tunnel_types));
-		q->tunnel = 0;
-	}
+	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
+		mlx5_flow_remove(dev, flow);
+	mlx5_flow_rxq_flags_clear(dev);
 }
 
 /**
@@ -2898,75 +3149,22 @@ mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 int
 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
-	struct priv *priv = dev->data->dev_private;
 	struct rte_flow *flow;
+	struct rte_flow_error error;
+	int ret = 0;
 
 	TAILQ_FOREACH(flow, list, next) {
-		unsigned int i;
-
-		if (flow->drop) {
-			flow->frxq[HASH_RXQ_ETH].ibv_flow =
-				mlx5_glue->create_flow
-				(priv->flow_drop_queue->qp,
-				 flow->frxq[HASH_RXQ_ETH].ibv_attr);
-			if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p cannot be applied",
-					dev->data->port_id, (void *)flow);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-			DRV_LOG(DEBUG, "port %u flow %p applied",
-				dev->data->port_id, (void *)flow);
-			/* Next flow. */
-			continue;
-		}
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_attr)
-				continue;
-			flow->frxq[i].hrxq =
-				mlx5_hrxq_get(dev, flow->rss_conf.key,
-					      flow->rss_conf.key_len,
-					      flow->frxq[i].hash_fields,
-					      flow->rss_conf.queue,
-					      flow->rss_conf.queue_num,
-					      flow->tunnel,
-					      flow->rss_conf.level);
-			if (flow->frxq[i].hrxq)
-				goto flow_create;
-			flow->frxq[i].hrxq =
-				mlx5_hrxq_new(dev, flow->rss_conf.key,
-					      flow->rss_conf.key_len,
-					      flow->frxq[i].hash_fields,
-					      flow->rss_conf.queue,
-					      flow->rss_conf.queue_num,
-					      flow->tunnel,
-					      flow->rss_conf.level);
-			if (!flow->frxq[i].hrxq) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p cannot create hash"
-					" rxq",
-					dev->data->port_id, (void *)flow);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-flow_create:
-			mlx5_flow_dump(dev, flow, i);
-			flow->frxq[i].ibv_flow =
-				mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
-						       flow->frxq[i].ibv_attr);
-			if (!flow->frxq[i].ibv_flow) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p type %u cannot be"
-					" applied",
-					dev->data->port_id, (void *)flow, i);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-		}
-		mlx5_flow_create_update_rxqs(dev, flow);
+		ret = mlx5_flow_apply(dev, flow, &error);
+		if (ret < 0)
+			goto error;
+		mlx5_flow_rxq_flags_set(dev, flow);
 	}
 	return 0;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_flow_stop(dev, list);
+	rte_errno = ret; /* Restore rte_errno. */
+	return -rte_errno;
 }
 
 /**
@@ -3019,7 +3217,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
-		.priority = MLX5_CTRL_FLOW_PRIORITY,
+		.priority = MLX5_FLOW_PRIO_RSVD,
 	};
 	struct rte_flow_item items[] = {
 		{
@@ -3129,49 +3327,88 @@ mlx5_flow_flush(struct rte_eth_dev *dev,
 	return 0;
 }
 
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+/**
+ * Isolated mode.
+ *
+ * @see rte_flow_isolate()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_isolate(struct rte_eth_dev *dev,
+		  int enable,
+		  struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	if (dev->data->dev_started) {
+		rte_flow_error_set(error, EBUSY,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "port must be stopped first");
+		return -rte_errno;
+	}
+	priv->isolated = !!enable;
+	if (enable)
+		dev->dev_ops = &mlx5_dev_ops_isolate;
+	else
+		dev->dev_ops = &mlx5_dev_ops;
+	return 0;
+}
+
 /**
  * Query flow counter.
  *
- * @param cs
- *   the counter set.
- * @param counter_value
- *   returned data from the counter.
+ * @param flow
+ *   Pointer to the flow.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_flow_query_count(struct ibv_counter_set *cs,
-		      struct mlx5_flow_counter_stats *counter_stats,
-		      struct rte_flow_query_count *query_count,
+mlx5_flow_query_count(struct rte_flow *flow __rte_unused,
+		      void *data __rte_unused,
 		      struct rte_flow_error *error)
 {
-	uint64_t counters[2];
-	struct ibv_query_counter_set_attr query_cs_attr = {
-		.cs = cs,
-		.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
-	};
-	struct ibv_counter_set_data query_out = {
-		.out = counters,
-		.outlen = 2 * sizeof(uint64_t),
-	};
-	int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
-
-	if (err)
-		return rte_flow_error_set(error, err,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "cannot read counter");
-	query_count->hits_set = 1;
-	query_count->bytes_set = 1;
-	query_count->hits = counters[0] - counter_stats->hits;
-	query_count->bytes = counters[1] - counter_stats->bytes;
-	if (query_count->reset) {
-		counter_stats->hits = counters[0];
-		counter_stats->bytes = counters[1];
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	if (flow->modifier & MLX5_FLOW_MOD_COUNT) {
+		struct rte_flow_query_count *qc = data;
+		uint64_t counters[2] = {0, 0};
+		struct ibv_query_counter_set_attr query_cs_attr = {
+			.cs = flow->counter->cs,
+			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
+		};
+		struct ibv_counter_set_data query_out = {
+			.out = counters,
+			.outlen = 2 * sizeof(uint64_t),
+		};
+		int err = mlx5_glue->query_counter_set(&query_cs_attr,
+						       &query_out);
+
+		if (err)
+			return rte_flow_error_set
+				(error, err,
+				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				 NULL,
+				 "cannot read counter");
+		qc->hits_set = 1;
+		qc->bytes_set = 1;
+		qc->hits = counters[0] - flow->counter->hits;
+		qc->bytes = counters[1] - flow->counter->bytes;
+		if (qc->reset) {
+			flow->counter->hits = counters[0];
+			flow->counter->bytes = counters[1];
+		}
+		return 0;
 	}
-	return 0;
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "flow does not have counter");
+#endif
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "counters are not available");
 }
 
 /**
@@ -3183,54 +3420,28 @@ mlx5_flow_query_count(struct ibv_counter_set *cs,
 int
 mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
 		struct rte_flow *flow,
-		const struct rte_flow_action *action __rte_unused,
+		const struct rte_flow_action *actions,
 		void *data,
 		struct rte_flow_error *error)
 {
-	if (flow->cs) {
-		int ret;
+	int ret = 0;
 
-		ret = mlx5_flow_query_count(flow->cs,
-					    &flow->counter_stats,
-					    (struct rte_flow_query_count *)data,
-					    error);
-		if (ret)
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_query_count(flow, data, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
 			return ret;
-	} else {
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "no counter found for flow");
-	}
-	return 0;
-}
-#endif
-
-/**
- * Isolated mode.
- *
- * @see rte_flow_isolate()
- * @see rte_flow_ops
- */
-int
-mlx5_flow_isolate(struct rte_eth_dev *dev,
-		  int enable,
-		  struct rte_flow_error *error)
-{
-	struct priv *priv = dev->data->dev_private;
-
-	if (dev->data->dev_started) {
-		rte_flow_error_set(error, EBUSY,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "port must be stopped first");
-		return -rte_errno;
 	}
-	priv->isolated = !!enable;
-	if (enable)
-		dev->dev_ops = &mlx5_dev_ops_isolate;
-	else
-		dev->dev_ops = &mlx5_dev_ops;
 	return 0;
 }
 
@@ -3445,9 +3656,6 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
 			.type = 0,
 		},
 	};
-	struct mlx5_flow_parse parser = {
-		.layer = HASH_RXQ_ETH,
-	};
 	struct rte_flow_error error;
 	struct rte_flow *flow;
 	int ret;
@@ -3455,10 +3663,6 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
 	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
 	if (ret)
 		return ret;
-	ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
-				attributes.actions, &error, &parser);
-	if (ret)
-		return ret;
 	flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
 				     attributes.items, attributes.actions,
 				     &error);
@@ -3482,94 +3686,11 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_fdir_filter_delete(struct rte_eth_dev *dev,
-			const struct rte_eth_fdir_filter *fdir_filter)
+mlx5_fdir_filter_delete(struct rte_eth_dev *dev __rte_unused,
+			const struct rte_eth_fdir_filter *fdir_filter
+			__rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_fdir attributes = {
-		.attr.group = 0,
-	};
-	struct mlx5_flow_parse parser = {
-		.create = 1,
-		.layer = HASH_RXQ_ETH,
-	};
-	struct rte_flow_error error;
-	struct rte_flow *flow;
-	unsigned int i;
-	int ret;
-
-	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
-				attributes.actions, &error, &parser);
-	if (ret)
-		goto exit;
-	/*
-	 * Special case for drop action which is only set in the
-	 * specifications when the flow is created.  In this situation the
-	 * drop specification is missing.
-	 */
-	if (parser.drop) {
-		struct ibv_flow_spec_action_drop *drop;
-
-		drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr +
-				parser.queue[HASH_RXQ_ETH].offset);
-		*drop = (struct ibv_flow_spec_action_drop){
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-			.size = sizeof(struct ibv_flow_spec_action_drop),
-		};
-		parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++;
-	}
-	TAILQ_FOREACH(flow, &priv->flows, next) {
-		struct ibv_flow_attr *attr;
-		struct ibv_spec_header *attr_h;
-		void *spec;
-		struct ibv_flow_attr *flow_attr;
-		struct ibv_spec_header *flow_h;
-		void *flow_spec;
-		unsigned int specs_n;
-		unsigned int queue_id = parser.drop ? HASH_RXQ_ETH :
-						      parser.layer;
-
-		attr = parser.queue[queue_id].ibv_attr;
-		flow_attr = flow->frxq[queue_id].ibv_attr;
-		/* Compare first the attributes. */
-		if (!flow_attr ||
-		    memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
-			continue;
-		if (attr->num_of_specs == 0)
-			continue;
-		spec = (void *)((uintptr_t)attr +
-				sizeof(struct ibv_flow_attr));
-		flow_spec = (void *)((uintptr_t)flow_attr +
-				     sizeof(struct ibv_flow_attr));
-		specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
-		for (i = 0; i != specs_n; ++i) {
-			attr_h = spec;
-			flow_h = flow_spec;
-			if (memcmp(spec, flow_spec,
-				   RTE_MIN(attr_h->size, flow_h->size)))
-				goto wrong_flow;
-			spec = (void *)((uintptr_t)spec + attr_h->size);
-			flow_spec = (void *)((uintptr_t)flow_spec +
-					     flow_h->size);
-		}
-		/* At this point, the flow match. */
-		break;
-wrong_flow:
-		/* The flow does not match. */
-		continue;
-	}
-	ret = rte_errno; /* Save rte_errno before cleanup. */
-	if (flow)
-		mlx5_flow_list_destroy(dev, &priv->flows, flow);
-exit:
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser.queue[i].ibv_attr)
-			rte_free(parser.queue[i].ibv_attr);
-	}
-	rte_errno = ret; /* Restore rte_errno. */
+	rte_errno = ENOTSUP;
 	return -rte_errno;
 }
 
@@ -3725,56 +3846,3 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
 	}
 	return 0;
 }
-
-/**
- * Detect number of Verbs flow priorities supported.
- *
- * @param dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   number of supported Verbs flow priority.
- */
-unsigned int
-mlx5_get_max_verbs_prio(struct rte_eth_dev *dev)
-{
-	struct priv *priv = dev->data->dev_private;
-	unsigned int verb_priorities = MLX5_VERBS_FLOW_PRIO_8;
-	struct {
-		struct ibv_flow_attr attr;
-		struct ibv_flow_spec_eth eth;
-		struct ibv_flow_spec_action_drop drop;
-	} flow_attr = {
-		.attr = {
-			.num_of_specs = 2,
-		},
-		.eth = {
-			.type = IBV_FLOW_SPEC_ETH,
-			.size = sizeof(struct ibv_flow_spec_eth),
-		},
-		.drop = {
-			.size = sizeof(struct ibv_flow_spec_action_drop),
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-		},
-	};
-	struct ibv_flow *flow;
-
-	do {
-		flow_attr.attr.priority = verb_priorities - 1;
-		flow = mlx5_glue->create_flow(priv->flow_drop_queue->qp,
-					      &flow_attr.attr);
-		if (flow) {
-			claim_zero(mlx5_glue->destroy_flow(flow));
-			/* Try more priorities. */
-			verb_priorities *= 2;
-		} else {
-			/* Failed, restore last right number. */
-			verb_priorities /= 2;
-			break;
-		}
-	} while (1);
-	DRV_LOG(DEBUG, "port %u Verbs flow priorities: %d,"
-		" user flow priorities: %d",
-		dev->data->port_id, verb_priorities, MLX5_CTRL_FLOW_PRIORITY);
-	return verb_priorities;
-}
diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
index c7965e51..84f9492a 100644
--- a/drivers/net/mlx5/mlx5_glue.c
+++ b/drivers/net/mlx5/mlx5_glue.c
@@ -4,6 +4,7 @@
  */
 
 #include <errno.h>
+#include <stdalign.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -23,6 +24,8 @@
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
 
+#include <rte_config.h>
+
 #include "mlx5_autoconf.h"
 #include "mlx5_glue.h"
 
@@ -343,6 +346,7 @@ mlx5_glue_dv_create_qp(struct ibv_context *context,
 #endif
 }
 
+alignas(RTE_CACHE_LINE_SIZE)
 const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
 	.version = MLX5_GLUE_VERSION,
 	.fork_init = mlx5_glue_fork_init,
diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c
index 672a4761..12ee37f5 100644
--- a/drivers/net/mlx5/mlx5_mac.c
+++ b/drivers/net/mlx5/mlx5_mac.c
@@ -49,7 +49,7 @@ mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN])
 	struct ifreq request;
 	int ret;
 
-	ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request);
+	ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request, 0);
 	if (ret)
 		return ret;
 	memcpy(mac, request.ifr_hwaddr.sa_data, ETHER_ADDR_LEN);
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 08105a44..1d1bcb5f 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -198,9 +198,8 @@ mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket)
 				      0, socket);
 	if (bt->table == NULL) {
 		rte_errno = ENOMEM;
-		DRV_LOG(ERR,
-			"failed to allocate memory for btree cache on socket %d",
-			socket);
+		DEBUG("failed to allocate memory for btree cache on socket %d",
+		      socket);
 		return -rte_errno;
 	}
 	bt->size = n;
@@ -208,8 +207,8 @@ mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket)
 	(*bt->table)[bt->len++] = (struct mlx5_mr_cache) {
 		.lkey = UINT32_MAX,
 	};
-	DRV_LOG(DEBUG, "initialized B-tree %p with table %p",
-		(void *)bt, (void *)bt->table);
+	DEBUG("initialized B-tree %p with table %p",
+	      (void *)bt, (void *)bt->table);
 	return 0;
 }
 
@@ -224,8 +223,8 @@ mlx5_mr_btree_free(struct mlx5_mr_btree *bt)
 {
 	if (bt == NULL)
 		return;
-	DRV_LOG(DEBUG, "freeing B-tree %p with table %p",
-		(void *)bt, (void *)bt->table);
+	DEBUG("freeing B-tree %p with table %p",
+	      (void *)bt, (void *)bt->table);
 	rte_free(bt->table);
 	memset(bt, 0, sizeof(*bt));
 }
@@ -236,9 +235,10 @@ mlx5_mr_btree_free(struct mlx5_mr_btree *bt)
  * @param bt
  *   Pointer to B-tree structure.
  */
-static void
-mlx5_mr_btree_dump(struct mlx5_mr_btree *bt)
+void
+mlx5_mr_btree_dump(struct mlx5_mr_btree *bt __rte_unused)
 {
+#ifndef NDEBUG
 	int idx;
 	struct mlx5_mr_cache *lkp_tbl;
 
@@ -248,11 +248,11 @@ mlx5_mr_btree_dump(struct mlx5_mr_btree *bt)
 	for (idx = 0; idx < bt->len; ++idx) {
 		struct mlx5_mr_cache *entry = &lkp_tbl[idx];
 
-		DRV_LOG(DEBUG,
-			"B-tree(%p)[%u],"
-			" [0x%" PRIxPTR ", 0x%" PRIxPTR ") lkey=0x%x",
-			(void *)bt, idx, entry->start, entry->end, entry->lkey);
+		DEBUG("B-tree(%p)[%u],"
+		      " [0x%" PRIxPTR ", 0x%" PRIxPTR ") lkey=0x%x",
+		      (void *)bt, idx, entry->start, entry->end, entry->lkey);
 	}
+#endif
 }
 
 /**
@@ -576,11 +576,10 @@ alloc_resources:
 	assert(msl->page_sz == ms->hugepage_sz);
 	/* Number of memsegs in the range. */
 	ms_n = len / msl->page_sz;
-	DRV_LOG(DEBUG,
-		"port %u extending %p to [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
-		" page_sz=0x%" PRIx64 ", ms_n=%u",
-		dev->data->port_id, (void *)addr,
-		data.start, data.end, msl->page_sz, ms_n);
+	DEBUG("port %u extending %p to [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
+	      " page_sz=0x%" PRIx64 ", ms_n=%u",
+	      dev->data->port_id, (void *)addr,
+	      data.start, data.end, msl->page_sz, ms_n);
 	/* Size of memory for bitmap. */
 	bmp_size = rte_bitmap_get_memory_footprint(ms_n);
 	mr = rte_zmalloc_socket(NULL,
@@ -589,10 +588,9 @@ alloc_resources:
 				bmp_size,
 				RTE_CACHE_LINE_SIZE, msl->socket_id);
 	if (mr == NULL) {
-		DRV_LOG(WARNING,
-			"port %u unable to allocate memory for a new MR of"
-			" address (%p).",
-			dev->data->port_id, (void *)addr);
+		DEBUG("port %u unable to allocate memory for a new MR of"
+		      " address (%p).",
+		      dev->data->port_id, (void *)addr);
 		rte_errno = ENOMEM;
 		goto err_nolock;
 	}
@@ -606,10 +604,9 @@ alloc_resources:
 	bmp_mem = RTE_PTR_ALIGN_CEIL(mr + 1, RTE_CACHE_LINE_SIZE);
 	mr->ms_bmp = rte_bitmap_init(ms_n, bmp_mem, bmp_size);
 	if (mr->ms_bmp == NULL) {
-		DRV_LOG(WARNING,
-			"port %u unable to initialize bitamp for a new MR of"
-			" address (%p).",
-			dev->data->port_id, (void *)addr);
+		DEBUG("port %u unable to initialize bitamp for a new MR of"
+		      " address (%p).",
+		      dev->data->port_id, (void *)addr);
 		rte_errno = EINVAL;
 		goto err_nolock;
 	}
@@ -625,11 +622,10 @@ alloc_resources:
 	data_re = data;
 	if (len > msl->page_sz &&
 	    !rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data_re)) {
-		DRV_LOG(WARNING,
-			"port %u unable to find virtually contiguous"
-			" chunk for address (%p)."
-			" rte_memseg_contig_walk() failed.",
-			dev->data->port_id, (void *)addr);
+		DEBUG("port %u unable to find virtually contiguous"
+		      " chunk for address (%p)."
+		      " rte_memseg_contig_walk() failed.",
+		      dev->data->port_id, (void *)addr);
 		rte_errno = ENXIO;
 		goto err_memlock;
 	}
@@ -657,9 +653,8 @@ alloc_resources:
 		 * here again.
 		 */
 		mr_btree_insert(&priv->mr.cache, entry);
-		DRV_LOG(DEBUG,
-			"port %u found MR for %p on final lookup, abort",
-			dev->data->port_id, (void *)addr);
+		DEBUG("port %u found MR for %p on final lookup, abort",
+		      dev->data->port_id, (void *)addr);
 		rte_rwlock_write_unlock(&priv->mr.rwlock);
 		rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
 		/*
@@ -707,22 +702,20 @@ alloc_resources:
 	mr->ibv_mr = mlx5_glue->reg_mr(priv->pd, (void *)data.start, len,
 				       IBV_ACCESS_LOCAL_WRITE);
 	if (mr->ibv_mr == NULL) {
-		DRV_LOG(WARNING,
-			"port %u fail to create a verbs MR for address (%p)",
-			dev->data->port_id, (void *)addr);
+		DEBUG("port %u fail to create a verbs MR for address (%p)",
+		      dev->data->port_id, (void *)addr);
 		rte_errno = EINVAL;
 		goto err_mrlock;
 	}
 	assert((uintptr_t)mr->ibv_mr->addr == data.start);
 	assert(mr->ibv_mr->length == len);
 	LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
-	DRV_LOG(DEBUG,
-		"port %u MR CREATED (%p) for %p:\n"
-		"  [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
-		" lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
-		dev->data->port_id, (void *)mr, (void *)addr,
-		data.start, data.end, rte_cpu_to_be_32(mr->ibv_mr->lkey),
-		mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
+	DEBUG("port %u MR CREATED (%p) for %p:\n"
+	      "  [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
+	      " lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
+	      dev->data->port_id, (void *)mr, (void *)addr,
+	      data.start, data.end, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+	      mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
 	/* Insert to the global cache table. */
 	mr_insert_dev_cache(dev, mr);
 	/* Fill in output data. */
@@ -797,8 +790,8 @@ mlx5_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
 	int i;
 	int rebuild = 0;
 
-	DRV_LOG(DEBUG, "port %u free callback: addr=%p, len=%zu",
-		dev->data->port_id, addr, len);
+	DEBUG("port %u free callback: addr=%p, len=%zu",
+	      dev->data->port_id, addr, len);
 	msl = rte_mem_virt2memseg_list(addr);
 	/* addr and len must be page-aligned. */
 	assert((uintptr_t)addr == RTE_ALIGN((uintptr_t)addr, msl->page_sz));
@@ -825,14 +818,14 @@ mlx5_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
 		pos = ms_idx - mr->ms_base_idx;
 		assert(rte_bitmap_get(mr->ms_bmp, pos));
 		assert(pos < mr->ms_bmp_n);
-		DRV_LOG(DEBUG, "port %u MR(%p): clear bitmap[%u] for addr %p",
-			dev->data->port_id, (void *)mr, pos, (void *)start);
+		DEBUG("port %u MR(%p): clear bitmap[%u] for addr %p",
+		      dev->data->port_id, (void *)mr, pos, (void *)start);
 		rte_bitmap_clear(mr->ms_bmp, pos);
 		if (--mr->ms_n == 0) {
 			LIST_REMOVE(mr, mr);
 			LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
-			DRV_LOG(DEBUG, "port %u remove MR(%p) from list",
-				dev->data->port_id, (void *)mr);
+			DEBUG("port %u remove MR(%p) from list",
+			      dev->data->port_id, (void *)mr);
 		}
 		/*
 		 * MR is fragmented or will be freed. the global cache must be
@@ -852,13 +845,11 @@ mlx5_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
 		 * before the core sees the newly allocated memory.
 		 */
 		++priv->mr.dev_gen;
-		DRV_LOG(DEBUG, "broadcasting local cache flush, gen=%d",
-			priv->mr.dev_gen);
+		DEBUG("broadcasting local cache flush, gen=%d",
+		      priv->mr.dev_gen);
 		rte_smp_wmb();
 	}
 	rte_rwlock_write_unlock(&priv->mr.rwlock);
-	if (rebuild && rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
-		mlx5_mr_dump_dev(dev);
 }
 
 /**
@@ -1123,8 +1114,9 @@ mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
  *   Pointer to Ethernet device.
  */
 void
-mlx5_mr_dump_dev(struct rte_eth_dev *dev)
+mlx5_mr_dump_dev(struct rte_eth_dev *dev __rte_unused)
 {
+#ifndef NDEBUG
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_mr *mr;
 	int mr_n = 0;
@@ -1135,11 +1127,10 @@ mlx5_mr_dump_dev(struct rte_eth_dev *dev)
 	LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
 		unsigned int n;
 
-		DRV_LOG(DEBUG,
-			"port %u MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
-			dev->data->port_id, mr_n++,
-			rte_cpu_to_be_32(mr->ibv_mr->lkey),
-			mr->ms_n, mr->ms_bmp_n);
+		DEBUG("port %u MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
+		      dev->data->port_id, mr_n++,
+		      rte_cpu_to_be_32(mr->ibv_mr->lkey),
+		      mr->ms_n, mr->ms_bmp_n);
 		if (mr->ms_n == 0)
 			continue;
 		for (n = 0; n < mr->ms_bmp_n; ) {
@@ -1148,14 +1139,14 @@ mlx5_mr_dump_dev(struct rte_eth_dev *dev)
 			n = mr_find_next_chunk(mr, &ret, n);
 			if (!ret.end)
 				break;
-			DRV_LOG(DEBUG,
-				"  chunk[%u], [0x%" PRIxPTR ", 0x%" PRIxPTR ")",
-				chunk_n++, ret.start, ret.end);
+			DEBUG("  chunk[%u], [0x%" PRIxPTR ", 0x%" PRIxPTR ")",
+			      chunk_n++, ret.start, ret.end);
 		}
 	}
-	DRV_LOG(DEBUG, "port %u dumping global cache", dev->data->port_id);
+	DEBUG("port %u dumping global cache", dev->data->port_id);
 	mlx5_mr_btree_dump(&priv->mr.cache);
 	rte_rwlock_read_unlock(&priv->mr.rwlock);
+#endif
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_mr.h b/drivers/net/mlx5/mlx5_mr.h
index e0b28215..a57003fe 100644
--- a/drivers/net/mlx5/mlx5_mr.h
+++ b/drivers/net/mlx5/mlx5_mr.h
@@ -74,9 +74,12 @@ void mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
 			  size_t len, void *arg);
 int mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
 		      struct rte_mempool *mp);
-void mlx5_mr_dump_dev(struct rte_eth_dev *dev);
 void mlx5_mr_release(struct rte_eth_dev *dev);
 
+/* Debug purpose functions. */
+void mlx5_mr_btree_dump(struct mlx5_mr_btree *bt);
+void mlx5_mr_dump_dev(struct rte_eth_dev *dev);
+
 /**
  * Look up LKey from given lookup table by linear search. Firstly look up the
  * last-hit entry. If miss, the entire array is searched. If found, update the
diff --git a/drivers/net/mlx5/mlx5_nl.c b/drivers/net/mlx5/mlx5_nl.c
index dca85835..d61826ae 100644
--- a/drivers/net/mlx5/mlx5_nl.c
+++ b/drivers/net/mlx5/mlx5_nl.c
@@ -3,10 +3,21 @@
  * Copyright 2018 Mellanox Technologies, Ltd
  */
 
+#include <errno.h>
+#include <linux/if_link.h>
 #include <linux/netlink.h>
 #include <linux/rtnetlink.h>
+#include <net/if.h>
+#include <rdma/rdma_netlink.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
 #include <unistd.h>
 
+#include <rte_errno.h>
+
 #include "mlx5.h"
 #include "mlx5_utils.h"
 
@@ -27,6 +38,40 @@
 	((struct rtattr *)(((char *)(r)) + NLMSG_ALIGN(sizeof(struct ndmsg))))
 #endif
 
+/*
+ * The following definitions are normally found in rdma/rdma_netlink.h,
+ * however they are so recent that most systems do not expose them yet.
+ */
+#ifndef HAVE_RDMA_NL_NLDEV
+#define RDMA_NL_NLDEV 5
+#endif
+#ifndef HAVE_RDMA_NLDEV_CMD_GET
+#define RDMA_NLDEV_CMD_GET 1
+#endif
+#ifndef HAVE_RDMA_NLDEV_CMD_PORT_GET
+#define RDMA_NLDEV_CMD_PORT_GET 5
+#endif
+#ifndef HAVE_RDMA_NLDEV_ATTR_DEV_INDEX
+#define RDMA_NLDEV_ATTR_DEV_INDEX 1
+#endif
+#ifndef HAVE_RDMA_NLDEV_ATTR_DEV_NAME
+#define RDMA_NLDEV_ATTR_DEV_NAME 2
+#endif
+#ifndef HAVE_RDMA_NLDEV_ATTR_PORT_INDEX
+#define RDMA_NLDEV_ATTR_PORT_INDEX 3
+#endif
+#ifndef HAVE_RDMA_NLDEV_ATTR_NDEV_INDEX
+#define RDMA_NLDEV_ATTR_NDEV_INDEX 50
+#endif
+
+/* These are normally found in linux/if_link.h. */
+#ifndef HAVE_IFLA_PHYS_SWITCH_ID
+#define IFLA_PHYS_SWITCH_ID 36
+#endif
+#ifndef HAVE_IFLA_PHYS_PORT_NAME
+#define IFLA_PHYS_PORT_NAME 38
+#endif
+
 /* Add/remove MAC address through Netlink */
 struct mlx5_nl_mac_addr {
 	struct ether_addr (*mac)[];
@@ -34,29 +79,35 @@ struct mlx5_nl_mac_addr {
 	int mac_n; /**< Number of addresses in the array. */
 };
 
+/** Data structure used by mlx5_nl_ifindex_cb(). */
+struct mlx5_nl_ifindex_data {
+	const char *name; /**< IB device name (in). */
+	uint32_t ibindex; /**< IB device index (out). */
+	uint32_t ifindex; /**< Network interface index (out). */
+};
+
 /**
  * Opens a Netlink socket.
  *
- * @param nl_groups
- *   Netlink group value (e.g. RTMGRP_LINK).
+ * @param protocol
+ *   Netlink protocol (e.g. NETLINK_ROUTE, NETLINK_RDMA).
  *
  * @return
  *   A file descriptor on success, a negative errno value otherwise and
  *   rte_errno is set.
  */
 int
-mlx5_nl_init(uint32_t nl_groups)
+mlx5_nl_init(int protocol)
 {
 	int fd;
 	int sndbuf_size = MLX5_SEND_BUF_SIZE;
 	int rcvbuf_size = MLX5_RECV_BUF_SIZE;
 	struct sockaddr_nl local = {
 		.nl_family = AF_NETLINK,
-		.nl_groups = nl_groups,
 	};
 	int ret;
 
-	fd = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, NETLINK_ROUTE);
+	fd = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, protocol);
 	if (fd == -1) {
 		rte_errno = errno;
 		return -rte_errno;
@@ -311,7 +362,7 @@ mlx5_nl_mac_addr_list(struct rte_eth_dev *dev, struct ether_addr (*mac)[],
 		      int *mac_n)
 {
 	struct priv *priv = dev->data->dev_private;
-	int iface_idx = mlx5_ifindex(dev);
+	unsigned int iface_idx = mlx5_ifindex(dev);
 	struct {
 		struct nlmsghdr	hdr;
 		struct ifinfomsg ifm;
@@ -334,9 +385,9 @@ mlx5_nl_mac_addr_list(struct rte_eth_dev *dev, struct ether_addr (*mac)[],
 	int ret;
 	uint32_t sn = priv->nl_sn++;
 
-	if (priv->nl_socket == -1)
+	if (priv->nl_socket_route == -1)
 		return 0;
-	fd = priv->nl_socket;
+	fd = priv->nl_socket_route;
 	ret = mlx5_nl_request(fd, &req.hdr, sn, &req.ifm,
 			      sizeof(struct ifinfomsg));
 	if (ret < 0)
@@ -370,7 +421,7 @@ mlx5_nl_mac_addr_modify(struct rte_eth_dev *dev, struct ether_addr *mac,
 			int add)
 {
 	struct priv *priv = dev->data->dev_private;
-	int iface_idx = mlx5_ifindex(dev);
+	unsigned int iface_idx = mlx5_ifindex(dev);
 	struct {
 		struct nlmsghdr hdr;
 		struct ndmsg ndm;
@@ -398,9 +449,9 @@ mlx5_nl_mac_addr_modify(struct rte_eth_dev *dev, struct ether_addr *mac,
 	int ret;
 	uint32_t sn = priv->nl_sn++;
 
-	if (priv->nl_socket == -1)
+	if (priv->nl_socket_route == -1)
 		return 0;
-	fd = priv->nl_socket;
+	fd = priv->nl_socket_route;
 	memcpy(RTA_DATA(&req.rta), mac, ETHER_ADDR_LEN);
 	req.hdr.nlmsg_len = NLMSG_ALIGN(req.hdr.nlmsg_len) +
 		RTA_ALIGN(req.rta.rta_len);
@@ -549,7 +600,7 @@ static int
 mlx5_nl_device_flags(struct rte_eth_dev *dev, uint32_t flags, int enable)
 {
 	struct priv *priv = dev->data->dev_private;
-	int iface_idx = mlx5_ifindex(dev);
+	unsigned int iface_idx = mlx5_ifindex(dev);
 	struct {
 		struct nlmsghdr hdr;
 		struct ifinfomsg ifi;
@@ -569,9 +620,9 @@ mlx5_nl_device_flags(struct rte_eth_dev *dev, uint32_t flags, int enable)
 	int ret;
 
 	assert(!(flags & ~(IFF_PROMISC | IFF_ALLMULTI)));
-	if (priv->nl_socket < 0)
+	if (priv->nl_socket_route < 0)
 		return 0;
-	fd = priv->nl_socket;
+	fd = priv->nl_socket_route;
 	ret = mlx5_nl_send(fd, &req.hdr, priv->nl_sn++);
 	if (ret < 0)
 		return ret;
@@ -625,3 +676,241 @@ mlx5_nl_allmulti(struct rte_eth_dev *dev, int enable)
 			strerror(rte_errno));
 	return ret;
 }
+
+/**
+ * Process network interface information from Netlink message.
+ *
+ * @param nh
+ *   Pointer to Netlink message header.
+ * @param arg
+ *   Opaque data pointer for this callback.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_nl_ifindex_cb(struct nlmsghdr *nh, void *arg)
+{
+	struct mlx5_nl_ifindex_data *data = arg;
+	size_t off = NLMSG_HDRLEN;
+	uint32_t ibindex = 0;
+	uint32_t ifindex = 0;
+	int found = 0;
+
+	if (nh->nlmsg_type !=
+	    RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET) &&
+	    nh->nlmsg_type !=
+	    RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_PORT_GET))
+		goto error;
+	while (off < nh->nlmsg_len) {
+		struct nlattr *na = (void *)((uintptr_t)nh + off);
+		void *payload = (void *)((uintptr_t)na + NLA_HDRLEN);
+
+		if (na->nla_len > nh->nlmsg_len - off)
+			goto error;
+		switch (na->nla_type) {
+		case RDMA_NLDEV_ATTR_DEV_INDEX:
+			ibindex = *(uint32_t *)payload;
+			break;
+		case RDMA_NLDEV_ATTR_DEV_NAME:
+			if (!strcmp(payload, data->name))
+				found = 1;
+			break;
+		case RDMA_NLDEV_ATTR_NDEV_INDEX:
+			ifindex = *(uint32_t *)payload;
+			break;
+		default:
+			break;
+		}
+		off += NLA_ALIGN(na->nla_len);
+	}
+	if (found) {
+		data->ibindex = ibindex;
+		data->ifindex = ifindex;
+	}
+	return 0;
+error:
+	rte_errno = EINVAL;
+	return -rte_errno;
+}
+
+/**
+ * Get index of network interface associated with some IB device.
+ *
+ * This is the only somewhat safe method to avoid resorting to heuristics
+ * when faced with port representors. Unfortunately it requires at least
+ * Linux 4.17.
+ *
+ * @param nl
+ *   Netlink socket of the RDMA kind (NETLINK_RDMA).
+ * @param[in] name
+ *   IB device name.
+ *
+ * @return
+ *   A valid (nonzero) interface index on success, 0 otherwise and rte_errno
+ *   is set.
+ */
+unsigned int
+mlx5_nl_ifindex(int nl, const char *name)
+{
+	static const uint32_t pindex = 1;
+	uint32_t seq = random();
+	struct mlx5_nl_ifindex_data data = {
+		.name = name,
+		.ibindex = 0, /* Determined during first pass. */
+		.ifindex = 0, /* Determined during second pass. */
+	};
+	union {
+		struct nlmsghdr nh;
+		uint8_t buf[NLMSG_HDRLEN +
+			    NLA_HDRLEN + NLA_ALIGN(sizeof(data.ibindex)) +
+			    NLA_HDRLEN + NLA_ALIGN(sizeof(pindex))];
+	} req = {
+		.nh = {
+			.nlmsg_len = NLMSG_LENGTH(0),
+			.nlmsg_type = RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+						       RDMA_NLDEV_CMD_GET),
+			.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_DUMP,
+		},
+	};
+	struct nlattr *na;
+	int ret;
+
+	ret = mlx5_nl_send(nl, &req.nh, seq);
+	if (ret < 0)
+		return 0;
+	ret = mlx5_nl_recv(nl, seq, mlx5_nl_ifindex_cb, &data);
+	if (ret < 0)
+		return 0;
+	if (!data.ibindex)
+		goto error;
+	++seq;
+	req.nh.nlmsg_type = RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+					     RDMA_NLDEV_CMD_PORT_GET);
+	req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
+	req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.buf) - NLMSG_HDRLEN);
+	na = (void *)((uintptr_t)req.buf + NLMSG_HDRLEN);
+	na->nla_len = NLA_HDRLEN + sizeof(data.ibindex);
+	na->nla_type = RDMA_NLDEV_ATTR_DEV_INDEX;
+	memcpy((void *)((uintptr_t)na + NLA_HDRLEN),
+	       &data.ibindex, sizeof(data.ibindex));
+	na = (void *)((uintptr_t)na + NLA_ALIGN(na->nla_len));
+	na->nla_len = NLA_HDRLEN + sizeof(pindex);
+	na->nla_type = RDMA_NLDEV_ATTR_PORT_INDEX;
+	memcpy((void *)((uintptr_t)na + NLA_HDRLEN),
+	       &pindex, sizeof(pindex));
+	ret = mlx5_nl_send(nl, &req.nh, seq);
+	if (ret < 0)
+		return 0;
+	ret = mlx5_nl_recv(nl, seq, mlx5_nl_ifindex_cb, &data);
+	if (ret < 0)
+		return 0;
+	if (!data.ifindex)
+		goto error;
+	return data.ifindex;
+error:
+	rte_errno = ENODEV;
+	return 0;
+}
+
+/**
+ * Process switch information from Netlink message.
+ *
+ * @param nh
+ *   Pointer to Netlink message header.
+ * @param arg
+ *   Opaque data pointer for this callback.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_nl_switch_info_cb(struct nlmsghdr *nh, void *arg)
+{
+	struct mlx5_switch_info info = {
+		.master = 0,
+		.representor = 0,
+		.port_name = 0,
+		.switch_id = 0,
+	};
+	size_t off = NLMSG_LENGTH(sizeof(struct ifinfomsg));
+	bool port_name_set = false;
+	bool switch_id_set = false;
+
+	if (nh->nlmsg_type != RTM_NEWLINK)
+		goto error;
+	while (off < nh->nlmsg_len) {
+		struct rtattr *ra = (void *)((uintptr_t)nh + off);
+		void *payload = RTA_DATA(ra);
+		char *end;
+		unsigned int i;
+
+		if (ra->rta_len > nh->nlmsg_len - off)
+			goto error;
+		switch (ra->rta_type) {
+		case IFLA_PHYS_PORT_NAME:
+			errno = 0;
+			info.port_name = strtol(payload, &end, 0);
+			if (errno ||
+			    (size_t)(end - (char *)payload) != strlen(payload))
+				goto error;
+			port_name_set = true;
+			break;
+		case IFLA_PHYS_SWITCH_ID:
+			info.switch_id = 0;
+			for (i = 0; i < RTA_PAYLOAD(ra); ++i) {
+				info.switch_id <<= 8;
+				info.switch_id |= ((uint8_t *)payload)[i];
+			}
+			switch_id_set = true;
+			break;
+		}
+		off += RTA_ALIGN(ra->rta_len);
+	}
+	info.master = switch_id_set && !port_name_set;
+	info.representor = switch_id_set && port_name_set;
+	memcpy(arg, &info, sizeof(info));
+	return 0;
+error:
+	rte_errno = EINVAL;
+	return -rte_errno;
+}
+
+/**
+ * Get switch information associated with network interface.
+ *
+ * @param nl
+ *   Netlink socket of the ROUTE kind (NETLINK_ROUTE).
+ * @param ifindex
+ *   Network interface index.
+ * @param[out] info
+ *   Switch information object, populated in case of success.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_switch_info(int nl, unsigned int ifindex, struct mlx5_switch_info *info)
+{
+	uint32_t seq = random();
+	struct {
+		struct nlmsghdr nh;
+		struct ifinfomsg info;
+	} req = {
+		.nh = {
+			.nlmsg_len = NLMSG_LENGTH(sizeof(req.info)),
+			.nlmsg_type = RTM_GETLINK,
+			.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK,
+		},
+		.info = {
+			.ifi_family = AF_UNSPEC,
+			.ifi_index = ifindex,
+		},
+	};
+	int ret;
+
+	ret = mlx5_nl_send(nl, &req.nh, seq);
+	if (ret >= 0)
+		ret = mlx5_nl_recv(nl, seq, mlx5_nl_switch_info_cb, info);
+	return ret;
+}
diff --git a/drivers/net/mlx5/mlx5_nl_flow.c b/drivers/net/mlx5/mlx5_nl_flow.c
new file mode 100644
index 00000000..a1c8c340
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_nl_flow.c
@@ -0,0 +1,1248 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <libmnl/libmnl.h>
+#include <linux/if_ether.h>
+#include <linux/netlink.h>
+#include <linux/pkt_cls.h>
+#include <linux/pkt_sched.h>
+#include <linux/rtnetlink.h>
+#include <linux/tc_act/tc_gact.h>
+#include <linux/tc_act/tc_mirred.h>
+#include <netinet/in.h>
+#include <stdalign.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/socket.h>
+
+#include <rte_byteorder.h>
+#include <rte_errno.h>
+#include <rte_ether.h>
+#include <rte_flow.h>
+
+#include "mlx5.h"
+#include "mlx5_autoconf.h"
+
+#ifdef HAVE_TC_ACT_VLAN
+
+#include <linux/tc_act/tc_vlan.h>
+
+#else /* HAVE_TC_ACT_VLAN */
+
+#define TCA_VLAN_ACT_POP 1
+#define TCA_VLAN_ACT_PUSH 2
+#define TCA_VLAN_ACT_MODIFY 3
+#define TCA_VLAN_PARMS 2
+#define TCA_VLAN_PUSH_VLAN_ID 3
+#define TCA_VLAN_PUSH_VLAN_PROTOCOL 4
+#define TCA_VLAN_PAD 5
+#define TCA_VLAN_PUSH_VLAN_PRIORITY 6
+
+struct tc_vlan {
+	tc_gen;
+	int v_action;
+};
+
+#endif /* HAVE_TC_ACT_VLAN */
+
+/* Normally found in linux/netlink.h. */
+#ifndef NETLINK_CAP_ACK
+#define NETLINK_CAP_ACK 10
+#endif
+
+/* Normally found in linux/pkt_sched.h. */
+#ifndef TC_H_MIN_INGRESS
+#define TC_H_MIN_INGRESS 0xfff2u
+#endif
+
+/* Normally found in linux/pkt_cls.h. */
+#ifndef TCA_CLS_FLAGS_SKIP_SW
+#define TCA_CLS_FLAGS_SKIP_SW (1 << 1)
+#endif
+#ifndef HAVE_TCA_FLOWER_ACT
+#define TCA_FLOWER_ACT 3
+#endif
+#ifndef HAVE_TCA_FLOWER_FLAGS
+#define TCA_FLOWER_FLAGS 22
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_TYPE
+#define TCA_FLOWER_KEY_ETH_TYPE 8
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_DST
+#define TCA_FLOWER_KEY_ETH_DST 4
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_DST_MASK
+#define TCA_FLOWER_KEY_ETH_DST_MASK 5
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC
+#define TCA_FLOWER_KEY_ETH_SRC 6
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_ETH_SRC_MASK
+#define TCA_FLOWER_KEY_ETH_SRC_MASK 7
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IP_PROTO
+#define TCA_FLOWER_KEY_IP_PROTO 9
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC
+#define TCA_FLOWER_KEY_IPV4_SRC 10
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_SRC_MASK
+#define TCA_FLOWER_KEY_IPV4_SRC_MASK 11
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST
+#define TCA_FLOWER_KEY_IPV4_DST 12
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV4_DST_MASK
+#define TCA_FLOWER_KEY_IPV4_DST_MASK 13
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC
+#define TCA_FLOWER_KEY_IPV6_SRC 14
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_SRC_MASK
+#define TCA_FLOWER_KEY_IPV6_SRC_MASK 15
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST
+#define TCA_FLOWER_KEY_IPV6_DST 16
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_IPV6_DST_MASK
+#define TCA_FLOWER_KEY_IPV6_DST_MASK 17
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC
+#define TCA_FLOWER_KEY_TCP_SRC 18
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_SRC_MASK
+#define TCA_FLOWER_KEY_TCP_SRC_MASK 35
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_DST
+#define TCA_FLOWER_KEY_TCP_DST 19
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_TCP_DST_MASK
+#define TCA_FLOWER_KEY_TCP_DST_MASK 36
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC
+#define TCA_FLOWER_KEY_UDP_SRC 20
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_SRC_MASK
+#define TCA_FLOWER_KEY_UDP_SRC_MASK 37
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_DST
+#define TCA_FLOWER_KEY_UDP_DST 21
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_UDP_DST_MASK
+#define TCA_FLOWER_KEY_UDP_DST_MASK 38
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_VLAN_ID
+#define TCA_FLOWER_KEY_VLAN_ID 23
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_VLAN_PRIO
+#define TCA_FLOWER_KEY_VLAN_PRIO 24
+#endif
+#ifndef HAVE_TCA_FLOWER_KEY_VLAN_ETH_TYPE
+#define TCA_FLOWER_KEY_VLAN_ETH_TYPE 25
+#endif
+
+/** Parser state definitions for mlx5_nl_flow_trans[]. */
+enum mlx5_nl_flow_trans {
+	INVALID,
+	BACK,
+	ATTR,
+	PATTERN,
+	ITEM_VOID,
+	ITEM_PORT_ID,
+	ITEM_ETH,
+	ITEM_VLAN,
+	ITEM_IPV4,
+	ITEM_IPV6,
+	ITEM_TCP,
+	ITEM_UDP,
+	ACTIONS,
+	ACTION_VOID,
+	ACTION_PORT_ID,
+	ACTION_DROP,
+	ACTION_OF_POP_VLAN,
+	ACTION_OF_PUSH_VLAN,
+	ACTION_OF_SET_VLAN_VID,
+	ACTION_OF_SET_VLAN_PCP,
+	END,
+};
+
+#define TRANS(...) (const enum mlx5_nl_flow_trans []){ __VA_ARGS__, INVALID, }
+
+#define PATTERN_COMMON \
+	ITEM_VOID, ITEM_PORT_ID, ACTIONS
+#define ACTIONS_COMMON \
+	ACTION_VOID, ACTION_OF_POP_VLAN, ACTION_OF_PUSH_VLAN, \
+	ACTION_OF_SET_VLAN_VID, ACTION_OF_SET_VLAN_PCP
+#define ACTIONS_FATE \
+	ACTION_PORT_ID, ACTION_DROP
+
+/** Parser state transitions used by mlx5_nl_flow_transpose(). */
+static const enum mlx5_nl_flow_trans *const mlx5_nl_flow_trans[] = {
+	[INVALID] = NULL,
+	[BACK] = NULL,
+	[ATTR] = TRANS(PATTERN),
+	[PATTERN] = TRANS(ITEM_ETH, PATTERN_COMMON),
+	[ITEM_VOID] = TRANS(BACK),
+	[ITEM_PORT_ID] = TRANS(BACK),
+	[ITEM_ETH] = TRANS(ITEM_IPV4, ITEM_IPV6, ITEM_VLAN, PATTERN_COMMON),
+	[ITEM_VLAN] = TRANS(ITEM_IPV4, ITEM_IPV6, PATTERN_COMMON),
+	[ITEM_IPV4] = TRANS(ITEM_TCP, ITEM_UDP, PATTERN_COMMON),
+	[ITEM_IPV6] = TRANS(ITEM_TCP, ITEM_UDP, PATTERN_COMMON),
+	[ITEM_TCP] = TRANS(PATTERN_COMMON),
+	[ITEM_UDP] = TRANS(PATTERN_COMMON),
+	[ACTIONS] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
+	[ACTION_VOID] = TRANS(BACK),
+	[ACTION_PORT_ID] = TRANS(ACTION_VOID, END),
+	[ACTION_DROP] = TRANS(ACTION_VOID, END),
+	[ACTION_OF_POP_VLAN] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
+	[ACTION_OF_PUSH_VLAN] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
+	[ACTION_OF_SET_VLAN_VID] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
+	[ACTION_OF_SET_VLAN_PCP] = TRANS(ACTIONS_FATE, ACTIONS_COMMON),
+	[END] = NULL,
+};
+
+/** Empty masks for known item types. */
+static const union {
+	struct rte_flow_item_port_id port_id;
+	struct rte_flow_item_eth eth;
+	struct rte_flow_item_vlan vlan;
+	struct rte_flow_item_ipv4 ipv4;
+	struct rte_flow_item_ipv6 ipv6;
+	struct rte_flow_item_tcp tcp;
+	struct rte_flow_item_udp udp;
+} mlx5_nl_flow_mask_empty;
+
+/** Supported masks for known item types. */
+static const struct {
+	struct rte_flow_item_port_id port_id;
+	struct rte_flow_item_eth eth;
+	struct rte_flow_item_vlan vlan;
+	struct rte_flow_item_ipv4 ipv4;
+	struct rte_flow_item_ipv6 ipv6;
+	struct rte_flow_item_tcp tcp;
+	struct rte_flow_item_udp udp;
+} mlx5_nl_flow_mask_supported = {
+	.port_id = {
+		.id = 0xffffffff,
+	},
+	.eth = {
+		.type = RTE_BE16(0xffff),
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+	},
+	.vlan = {
+		/* PCP and VID only, no DEI. */
+		.tci = RTE_BE16(0xefff),
+		.inner_type = RTE_BE16(0xffff),
+	},
+	.ipv4.hdr = {
+		.next_proto_id = 0xff,
+		.src_addr = RTE_BE32(0xffffffff),
+		.dst_addr = RTE_BE32(0xffffffff),
+	},
+	.ipv6.hdr = {
+		.proto = 0xff,
+		.src_addr =
+			"\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff",
+		.dst_addr =
+			"\xff\xff\xff\xff\xff\xff\xff\xff"
+			"\xff\xff\xff\xff\xff\xff\xff\xff",
+	},
+	.tcp.hdr = {
+		.src_port = RTE_BE16(0xffff),
+		.dst_port = RTE_BE16(0xffff),
+	},
+	.udp.hdr = {
+		.src_port = RTE_BE16(0xffff),
+		.dst_port = RTE_BE16(0xffff),
+	},
+};
+
+/**
+ * Retrieve mask for pattern item.
+ *
+ * This function does basic sanity checks on a pattern item in order to
+ * return the most appropriate mask for it.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in] mask_default
+ *   Default mask for pattern item as specified by the flow API.
+ * @param[in] mask_supported
+ *   Mask fields supported by the implementation.
+ * @param[in] mask_empty
+ *   Empty mask to return when there is no specification.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   Either @p item->mask or one of the mask parameters on success, NULL
+ *   otherwise and rte_errno is set.
+ */
+static const void *
+mlx5_nl_flow_item_mask(const struct rte_flow_item *item,
+		       const void *mask_default,
+		       const void *mask_supported,
+		       const void *mask_empty,
+		       size_t mask_size,
+		       struct rte_flow_error *error)
+{
+	const uint8_t *mask;
+	size_t i;
+
+	/* item->last and item->mask cannot exist without item->spec. */
+	if (!item->spec && (item->mask || item->last)) {
+		rte_flow_error_set
+			(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
+			 "\"mask\" or \"last\" field provided without a"
+			 " corresponding \"spec\"");
+		return NULL;
+	}
+	/* No spec, no mask, no problem. */
+	if (!item->spec)
+		return mask_empty;
+	mask = item->mask ? item->mask : mask_default;
+	assert(mask);
+	/*
+	 * Single-pass check to make sure that:
+	 * - Mask is supported, no bits are set outside mask_supported.
+	 * - Both item->spec and item->last are included in mask.
+	 */
+	for (i = 0; i != mask_size; ++i) {
+		if (!mask[i])
+			continue;
+		if ((mask[i] | ((const uint8_t *)mask_supported)[i]) !=
+		    ((const uint8_t *)mask_supported)[i]) {
+			rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask, "unsupported field found in \"mask\"");
+			return NULL;
+		}
+		if (item->last &&
+		    (((const uint8_t *)item->spec)[i] & mask[i]) !=
+		    (((const uint8_t *)item->last)[i] & mask[i])) {
+			rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_LAST,
+				 item->last,
+				 "range between \"spec\" and \"last\" not"
+				 " comprised in \"mask\"");
+			return NULL;
+		}
+	}
+	return mask;
+}
+
+/**
+ * Transpose flow rule description to rtnetlink message.
+ *
+ * This function transposes a flow rule description to a traffic control
+ * (TC) filter creation message ready to be sent over Netlink.
+ *
+ * Target interface is specified as the first entry of the @p ptoi table.
+ * Subsequent entries enable this function to resolve other DPDK port IDs
+ * found in the flow rule.
+ *
+ * @param[out] buf
+ *   Output message buffer. May be NULL when @p size is 0.
+ * @param size
+ *   Size of @p buf. Message may be truncated if not large enough.
+ * @param[in] ptoi
+ *   DPDK port ID to network interface index translation table. This table
+ *   is terminated by an entry with a zero ifindex value.
+ * @param[in] attr
+ *   Flow rule attributes.
+ * @param[in] pattern
+ *   Pattern specification.
+ * @param[in] actions
+ *   Associated actions.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   A positive value representing the exact size of the message in bytes
+ *   regardless of the @p size parameter on success, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_flow_transpose(void *buf,
+		       size_t size,
+		       const struct mlx5_nl_flow_ptoi *ptoi,
+		       const struct rte_flow_attr *attr,
+		       const struct rte_flow_item *pattern,
+		       const struct rte_flow_action *actions,
+		       struct rte_flow_error *error)
+{
+	alignas(struct nlmsghdr)
+	uint8_t buf_tmp[mnl_nlmsg_size(sizeof(struct tcmsg) + 1024)];
+	const struct rte_flow_item *item;
+	const struct rte_flow_action *action;
+	unsigned int n;
+	uint32_t act_index_cur;
+	bool in_port_id_set;
+	bool eth_type_set;
+	bool vlan_present;
+	bool vlan_eth_type_set;
+	bool ip_proto_set;
+	struct nlattr *na_flower;
+	struct nlattr *na_flower_act;
+	struct nlattr *na_vlan_id;
+	struct nlattr *na_vlan_priority;
+	const enum mlx5_nl_flow_trans *trans;
+	const enum mlx5_nl_flow_trans *back;
+
+	if (!size)
+		goto error_nobufs;
+init:
+	item = pattern;
+	action = actions;
+	n = 0;
+	act_index_cur = 0;
+	in_port_id_set = false;
+	eth_type_set = false;
+	vlan_present = false;
+	vlan_eth_type_set = false;
+	ip_proto_set = false;
+	na_flower = NULL;
+	na_flower_act = NULL;
+	na_vlan_id = NULL;
+	na_vlan_priority = NULL;
+	trans = TRANS(ATTR);
+	back = trans;
+trans:
+	switch (trans[n++]) {
+		union {
+			const struct rte_flow_item_port_id *port_id;
+			const struct rte_flow_item_eth *eth;
+			const struct rte_flow_item_vlan *vlan;
+			const struct rte_flow_item_ipv4 *ipv4;
+			const struct rte_flow_item_ipv6 *ipv6;
+			const struct rte_flow_item_tcp *tcp;
+			const struct rte_flow_item_udp *udp;
+		} spec, mask;
+		union {
+			const struct rte_flow_action_port_id *port_id;
+			const struct rte_flow_action_of_push_vlan *of_push_vlan;
+			const struct rte_flow_action_of_set_vlan_vid *
+				of_set_vlan_vid;
+			const struct rte_flow_action_of_set_vlan_pcp *
+				of_set_vlan_pcp;
+		} conf;
+		struct nlmsghdr *nlh;
+		struct tcmsg *tcm;
+		struct nlattr *act_index;
+		struct nlattr *act;
+		unsigned int i;
+
+	case INVALID:
+		if (item->type)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
+				 item, "unsupported pattern item combination");
+		else if (action->type)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
+				 action, "unsupported action combination");
+		return rte_flow_error_set
+			(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+			 "flow rule lacks some kind of fate action");
+	case BACK:
+		trans = back;
+		n = 0;
+		goto trans;
+	case ATTR:
+		/*
+		 * Supported attributes: no groups, some priorities and
+		 * ingress only. Don't care about transfer as it is the
+		 * caller's problem.
+		 */
+		if (attr->group)
+			return rte_flow_error_set
+				(error, ENOTSUP,
+				 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+				 attr, "groups are not supported");
+		if (attr->priority > 0xfffe)
+			return rte_flow_error_set
+				(error, ENOTSUP,
+				 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+				 attr, "lowest priority level is 0xfffe");
+		if (!attr->ingress)
+			return rte_flow_error_set
+				(error, ENOTSUP,
+				 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+				 attr, "only ingress is supported");
+		if (attr->egress)
+			return rte_flow_error_set
+				(error, ENOTSUP,
+				 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+				 attr, "egress is not supported");
+		if (size < mnl_nlmsg_size(sizeof(*tcm)))
+			goto error_nobufs;
+		nlh = mnl_nlmsg_put_header(buf);
+		nlh->nlmsg_type = 0;
+		nlh->nlmsg_flags = 0;
+		nlh->nlmsg_seq = 0;
+		tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
+		tcm->tcm_family = AF_UNSPEC;
+		tcm->tcm_ifindex = ptoi[0].ifindex;
+		/*
+		 * Let kernel pick a handle by default. A predictable handle
+		 * can be set by the caller on the resulting buffer through
+		 * mlx5_nl_flow_brand().
+		 */
+		tcm->tcm_handle = 0;
+		tcm->tcm_parent = TC_H_MAKE(TC_H_INGRESS, TC_H_MIN_INGRESS);
+		/*
+		 * Priority cannot be zero to prevent the kernel from
+		 * picking one automatically.
+		 */
+		tcm->tcm_info = TC_H_MAKE((attr->priority + 1) << 16,
+					  RTE_BE16(ETH_P_ALL));
+		break;
+	case PATTERN:
+		if (!mnl_attr_put_strz_check(buf, size, TCA_KIND, "flower"))
+			goto error_nobufs;
+		na_flower = mnl_attr_nest_start_check(buf, size, TCA_OPTIONS);
+		if (!na_flower)
+			goto error_nobufs;
+		if (!mnl_attr_put_u32_check(buf, size, TCA_FLOWER_FLAGS,
+					    TCA_CLS_FLAGS_SKIP_SW))
+			goto error_nobufs;
+		break;
+	case ITEM_VOID:
+		if (item->type != RTE_FLOW_ITEM_TYPE_VOID)
+			goto trans;
+		++item;
+		break;
+	case ITEM_PORT_ID:
+		if (item->type != RTE_FLOW_ITEM_TYPE_PORT_ID)
+			goto trans;
+		mask.port_id = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_port_id_mask,
+			 &mlx5_nl_flow_mask_supported.port_id,
+			 &mlx5_nl_flow_mask_empty.port_id,
+			 sizeof(mlx5_nl_flow_mask_supported.port_id), error);
+		if (!mask.port_id)
+			return -rte_errno;
+		if (mask.port_id == &mlx5_nl_flow_mask_empty.port_id) {
+			in_port_id_set = 1;
+			++item;
+			break;
+		}
+		spec.port_id = item->spec;
+		if (mask.port_id->id && mask.port_id->id != 0xffffffff)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.port_id,
+				 "no support for partial mask on"
+				 " \"id\" field");
+		if (!mask.port_id->id)
+			i = 0;
+		else
+			for (i = 0; ptoi[i].ifindex; ++i)
+				if (ptoi[i].port_id == spec.port_id->id)
+					break;
+		if (!ptoi[i].ifindex)
+			return rte_flow_error_set
+				(error, ENODEV, RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+				 spec.port_id,
+				 "missing data to convert port ID to ifindex");
+		tcm = mnl_nlmsg_get_payload(buf);
+		if (in_port_id_set &&
+		    ptoi[i].ifindex != (unsigned int)tcm->tcm_ifindex)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+				 spec.port_id,
+				 "cannot match traffic for several port IDs"
+				 " through a single flow rule");
+		tcm->tcm_ifindex = ptoi[i].ifindex;
+		in_port_id_set = 1;
+		++item;
+		break;
+	case ITEM_ETH:
+		if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
+			goto trans;
+		mask.eth = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_eth_mask,
+			 &mlx5_nl_flow_mask_supported.eth,
+			 &mlx5_nl_flow_mask_empty.eth,
+			 sizeof(mlx5_nl_flow_mask_supported.eth), error);
+		if (!mask.eth)
+			return -rte_errno;
+		if (mask.eth == &mlx5_nl_flow_mask_empty.eth) {
+			++item;
+			break;
+		}
+		spec.eth = item->spec;
+		if (mask.eth->type && mask.eth->type != RTE_BE16(0xffff))
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.eth,
+				 "no support for partial mask on"
+				 " \"type\" field");
+		if (mask.eth->type) {
+			if (!mnl_attr_put_u16_check(buf, size,
+						    TCA_FLOWER_KEY_ETH_TYPE,
+						    spec.eth->type))
+				goto error_nobufs;
+			eth_type_set = 1;
+		}
+		if ((!is_zero_ether_addr(&mask.eth->dst) &&
+		     (!mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_ETH_DST,
+					  ETHER_ADDR_LEN,
+					  spec.eth->dst.addr_bytes) ||
+		      !mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_ETH_DST_MASK,
+					  ETHER_ADDR_LEN,
+					  mask.eth->dst.addr_bytes))) ||
+		    (!is_zero_ether_addr(&mask.eth->src) &&
+		     (!mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_ETH_SRC,
+					  ETHER_ADDR_LEN,
+					  spec.eth->src.addr_bytes) ||
+		      !mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_ETH_SRC_MASK,
+					  ETHER_ADDR_LEN,
+					  mask.eth->src.addr_bytes))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ITEM_VLAN:
+		if (item->type != RTE_FLOW_ITEM_TYPE_VLAN)
+			goto trans;
+		mask.vlan = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_vlan_mask,
+			 &mlx5_nl_flow_mask_supported.vlan,
+			 &mlx5_nl_flow_mask_empty.vlan,
+			 sizeof(mlx5_nl_flow_mask_supported.vlan), error);
+		if (!mask.vlan)
+			return -rte_errno;
+		if (!eth_type_set &&
+		    !mnl_attr_put_u16_check(buf, size,
+					    TCA_FLOWER_KEY_ETH_TYPE,
+					    RTE_BE16(ETH_P_8021Q)))
+			goto error_nobufs;
+		eth_type_set = 1;
+		vlan_present = 1;
+		if (mask.vlan == &mlx5_nl_flow_mask_empty.vlan) {
+			++item;
+			break;
+		}
+		spec.vlan = item->spec;
+		if ((mask.vlan->tci & RTE_BE16(0xe000) &&
+		     (mask.vlan->tci & RTE_BE16(0xe000)) != RTE_BE16(0xe000)) ||
+		    (mask.vlan->tci & RTE_BE16(0x0fff) &&
+		     (mask.vlan->tci & RTE_BE16(0x0fff)) != RTE_BE16(0x0fff)) ||
+		    (mask.vlan->inner_type &&
+		     mask.vlan->inner_type != RTE_BE16(0xffff)))
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.vlan,
+				 "no support for partial masks on"
+				 " \"tci\" (PCP and VID parts) and"
+				 " \"inner_type\" fields");
+		if (mask.vlan->inner_type) {
+			if (!mnl_attr_put_u16_check
+			    (buf, size, TCA_FLOWER_KEY_VLAN_ETH_TYPE,
+			     spec.vlan->inner_type))
+				goto error_nobufs;
+			vlan_eth_type_set = 1;
+		}
+		if ((mask.vlan->tci & RTE_BE16(0xe000) &&
+		     !mnl_attr_put_u8_check
+		     (buf, size, TCA_FLOWER_KEY_VLAN_PRIO,
+		      (rte_be_to_cpu_16(spec.vlan->tci) >> 13) & 0x7)) ||
+		    (mask.vlan->tci & RTE_BE16(0x0fff) &&
+		     !mnl_attr_put_u16_check
+		     (buf, size, TCA_FLOWER_KEY_VLAN_ID,
+		      rte_be_to_cpu_16(spec.vlan->tci & RTE_BE16(0x0fff)))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ITEM_IPV4:
+		if (item->type != RTE_FLOW_ITEM_TYPE_IPV4)
+			goto trans;
+		mask.ipv4 = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_ipv4_mask,
+			 &mlx5_nl_flow_mask_supported.ipv4,
+			 &mlx5_nl_flow_mask_empty.ipv4,
+			 sizeof(mlx5_nl_flow_mask_supported.ipv4), error);
+		if (!mask.ipv4)
+			return -rte_errno;
+		if ((!eth_type_set || !vlan_eth_type_set) &&
+		    !mnl_attr_put_u16_check(buf, size,
+					    vlan_present ?
+					    TCA_FLOWER_KEY_VLAN_ETH_TYPE :
+					    TCA_FLOWER_KEY_ETH_TYPE,
+					    RTE_BE16(ETH_P_IP)))
+			goto error_nobufs;
+		eth_type_set = 1;
+		vlan_eth_type_set = 1;
+		if (mask.ipv4 == &mlx5_nl_flow_mask_empty.ipv4) {
+			++item;
+			break;
+		}
+		spec.ipv4 = item->spec;
+		if (mask.ipv4->hdr.next_proto_id &&
+		    mask.ipv4->hdr.next_proto_id != 0xff)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.ipv4,
+				 "no support for partial mask on"
+				 " \"hdr.next_proto_id\" field");
+		if (mask.ipv4->hdr.next_proto_id) {
+			if (!mnl_attr_put_u8_check
+			    (buf, size, TCA_FLOWER_KEY_IP_PROTO,
+			     spec.ipv4->hdr.next_proto_id))
+				goto error_nobufs;
+			ip_proto_set = 1;
+		}
+		if ((mask.ipv4->hdr.src_addr &&
+		     (!mnl_attr_put_u32_check(buf, size,
+					      TCA_FLOWER_KEY_IPV4_SRC,
+					      spec.ipv4->hdr.src_addr) ||
+		      !mnl_attr_put_u32_check(buf, size,
+					      TCA_FLOWER_KEY_IPV4_SRC_MASK,
+					      mask.ipv4->hdr.src_addr))) ||
+		    (mask.ipv4->hdr.dst_addr &&
+		     (!mnl_attr_put_u32_check(buf, size,
+					      TCA_FLOWER_KEY_IPV4_DST,
+					      spec.ipv4->hdr.dst_addr) ||
+		      !mnl_attr_put_u32_check(buf, size,
+					      TCA_FLOWER_KEY_IPV4_DST_MASK,
+					      mask.ipv4->hdr.dst_addr))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ITEM_IPV6:
+		if (item->type != RTE_FLOW_ITEM_TYPE_IPV6)
+			goto trans;
+		mask.ipv6 = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_ipv6_mask,
+			 &mlx5_nl_flow_mask_supported.ipv6,
+			 &mlx5_nl_flow_mask_empty.ipv6,
+			 sizeof(mlx5_nl_flow_mask_supported.ipv6), error);
+		if (!mask.ipv6)
+			return -rte_errno;
+		if ((!eth_type_set || !vlan_eth_type_set) &&
+		    !mnl_attr_put_u16_check(buf, size,
+					    vlan_present ?
+					    TCA_FLOWER_KEY_VLAN_ETH_TYPE :
+					    TCA_FLOWER_KEY_ETH_TYPE,
+					    RTE_BE16(ETH_P_IPV6)))
+			goto error_nobufs;
+		eth_type_set = 1;
+		vlan_eth_type_set = 1;
+		if (mask.ipv6 == &mlx5_nl_flow_mask_empty.ipv6) {
+			++item;
+			break;
+		}
+		spec.ipv6 = item->spec;
+		if (mask.ipv6->hdr.proto && mask.ipv6->hdr.proto != 0xff)
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.ipv6,
+				 "no support for partial mask on"
+				 " \"hdr.proto\" field");
+		if (mask.ipv6->hdr.proto) {
+			if (!mnl_attr_put_u8_check
+			    (buf, size, TCA_FLOWER_KEY_IP_PROTO,
+			     spec.ipv6->hdr.proto))
+				goto error_nobufs;
+			ip_proto_set = 1;
+		}
+		if ((!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.src_addr) &&
+		     (!mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_IPV6_SRC,
+					  sizeof(spec.ipv6->hdr.src_addr),
+					  spec.ipv6->hdr.src_addr) ||
+		      !mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_IPV6_SRC_MASK,
+					  sizeof(mask.ipv6->hdr.src_addr),
+					  mask.ipv6->hdr.src_addr))) ||
+		    (!IN6_IS_ADDR_UNSPECIFIED(mask.ipv6->hdr.dst_addr) &&
+		     (!mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_IPV6_DST,
+					  sizeof(spec.ipv6->hdr.dst_addr),
+					  spec.ipv6->hdr.dst_addr) ||
+		      !mnl_attr_put_check(buf, size,
+					  TCA_FLOWER_KEY_IPV6_DST_MASK,
+					  sizeof(mask.ipv6->hdr.dst_addr),
+					  mask.ipv6->hdr.dst_addr))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ITEM_TCP:
+		if (item->type != RTE_FLOW_ITEM_TYPE_TCP)
+			goto trans;
+		mask.tcp = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_tcp_mask,
+			 &mlx5_nl_flow_mask_supported.tcp,
+			 &mlx5_nl_flow_mask_empty.tcp,
+			 sizeof(mlx5_nl_flow_mask_supported.tcp), error);
+		if (!mask.tcp)
+			return -rte_errno;
+		if (!ip_proto_set &&
+		    !mnl_attr_put_u8_check(buf, size,
+					   TCA_FLOWER_KEY_IP_PROTO,
+					   IPPROTO_TCP))
+			goto error_nobufs;
+		if (mask.tcp == &mlx5_nl_flow_mask_empty.tcp) {
+			++item;
+			break;
+		}
+		spec.tcp = item->spec;
+		if ((mask.tcp->hdr.src_port &&
+		     mask.tcp->hdr.src_port != RTE_BE16(0xffff)) ||
+		    (mask.tcp->hdr.dst_port &&
+		     mask.tcp->hdr.dst_port != RTE_BE16(0xffff)))
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.tcp,
+				 "no support for partial masks on"
+				 " \"hdr.src_port\" and \"hdr.dst_port\""
+				 " fields");
+		if ((mask.tcp->hdr.src_port &&
+		     (!mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_TCP_SRC,
+					      spec.tcp->hdr.src_port) ||
+		      !mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_TCP_SRC_MASK,
+					      mask.tcp->hdr.src_port))) ||
+		    (mask.tcp->hdr.dst_port &&
+		     (!mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_TCP_DST,
+					      spec.tcp->hdr.dst_port) ||
+		      !mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_TCP_DST_MASK,
+					      mask.tcp->hdr.dst_port))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ITEM_UDP:
+		if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
+			goto trans;
+		mask.udp = mlx5_nl_flow_item_mask
+			(item, &rte_flow_item_udp_mask,
+			 &mlx5_nl_flow_mask_supported.udp,
+			 &mlx5_nl_flow_mask_empty.udp,
+			 sizeof(mlx5_nl_flow_mask_supported.udp), error);
+		if (!mask.udp)
+			return -rte_errno;
+		if (!ip_proto_set &&
+		    !mnl_attr_put_u8_check(buf, size,
+					   TCA_FLOWER_KEY_IP_PROTO,
+					   IPPROTO_UDP))
+			goto error_nobufs;
+		if (mask.udp == &mlx5_nl_flow_mask_empty.udp) {
+			++item;
+			break;
+		}
+		spec.udp = item->spec;
+		if ((mask.udp->hdr.src_port &&
+		     mask.udp->hdr.src_port != RTE_BE16(0xffff)) ||
+		    (mask.udp->hdr.dst_port &&
+		     mask.udp->hdr.dst_port != RTE_BE16(0xffff)))
+			return rte_flow_error_set
+				(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM_MASK,
+				 mask.udp,
+				 "no support for partial masks on"
+				 " \"hdr.src_port\" and \"hdr.dst_port\""
+				 " fields");
+		if ((mask.udp->hdr.src_port &&
+		     (!mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_UDP_SRC,
+					      spec.udp->hdr.src_port) ||
+		      !mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_UDP_SRC_MASK,
+					      mask.udp->hdr.src_port))) ||
+		    (mask.udp->hdr.dst_port &&
+		     (!mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_UDP_DST,
+					      spec.udp->hdr.dst_port) ||
+		      !mnl_attr_put_u16_check(buf, size,
+					      TCA_FLOWER_KEY_UDP_DST_MASK,
+					      mask.udp->hdr.dst_port))))
+			goto error_nobufs;
+		++item;
+		break;
+	case ACTIONS:
+		if (item->type != RTE_FLOW_ITEM_TYPE_END)
+			goto trans;
+		assert(na_flower);
+		assert(!na_flower_act);
+		na_flower_act =
+			mnl_attr_nest_start_check(buf, size, TCA_FLOWER_ACT);
+		if (!na_flower_act)
+			goto error_nobufs;
+		act_index_cur = 1;
+		break;
+	case ACTION_VOID:
+		if (action->type != RTE_FLOW_ACTION_TYPE_VOID)
+			goto trans;
+		++action;
+		break;
+	case ACTION_PORT_ID:
+		if (action->type != RTE_FLOW_ACTION_TYPE_PORT_ID)
+			goto trans;
+		conf.port_id = action->conf;
+		if (conf.port_id->original)
+			i = 0;
+		else
+			for (i = 0; ptoi[i].ifindex; ++i)
+				if (ptoi[i].port_id == conf.port_id->id)
+					break;
+		if (!ptoi[i].ifindex)
+			return rte_flow_error_set
+				(error, ENODEV, RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 conf.port_id,
+				 "missing data to convert port ID to ifindex");
+		act_index =
+			mnl_attr_nest_start_check(buf, size, act_index_cur++);
+		if (!act_index ||
+		    !mnl_attr_put_strz_check(buf, size, TCA_ACT_KIND, "mirred"))
+			goto error_nobufs;
+		act = mnl_attr_nest_start_check(buf, size, TCA_ACT_OPTIONS);
+		if (!act)
+			goto error_nobufs;
+		if (!mnl_attr_put_check(buf, size, TCA_MIRRED_PARMS,
+					sizeof(struct tc_mirred),
+					&(struct tc_mirred){
+						.action = TC_ACT_STOLEN,
+						.eaction = TCA_EGRESS_REDIR,
+						.ifindex = ptoi[i].ifindex,
+					}))
+			goto error_nobufs;
+		mnl_attr_nest_end(buf, act);
+		mnl_attr_nest_end(buf, act_index);
+		++action;
+		break;
+	case ACTION_DROP:
+		if (action->type != RTE_FLOW_ACTION_TYPE_DROP)
+			goto trans;
+		act_index =
+			mnl_attr_nest_start_check(buf, size, act_index_cur++);
+		if (!act_index ||
+		    !mnl_attr_put_strz_check(buf, size, TCA_ACT_KIND, "gact"))
+			goto error_nobufs;
+		act = mnl_attr_nest_start_check(buf, size, TCA_ACT_OPTIONS);
+		if (!act)
+			goto error_nobufs;
+		if (!mnl_attr_put_check(buf, size, TCA_GACT_PARMS,
+					sizeof(struct tc_gact),
+					&(struct tc_gact){
+						.action = TC_ACT_SHOT,
+					}))
+			goto error_nobufs;
+		mnl_attr_nest_end(buf, act);
+		mnl_attr_nest_end(buf, act_index);
+		++action;
+		break;
+	case ACTION_OF_POP_VLAN:
+		if (action->type != RTE_FLOW_ACTION_TYPE_OF_POP_VLAN)
+			goto trans;
+		conf.of_push_vlan = NULL;
+		i = TCA_VLAN_ACT_POP;
+		goto action_of_vlan;
+	case ACTION_OF_PUSH_VLAN:
+		if (action->type != RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN)
+			goto trans;
+		conf.of_push_vlan = action->conf;
+		i = TCA_VLAN_ACT_PUSH;
+		goto action_of_vlan;
+	case ACTION_OF_SET_VLAN_VID:
+		if (action->type != RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
+			goto trans;
+		conf.of_set_vlan_vid = action->conf;
+		if (na_vlan_id)
+			goto override_na_vlan_id;
+		i = TCA_VLAN_ACT_MODIFY;
+		goto action_of_vlan;
+	case ACTION_OF_SET_VLAN_PCP:
+		if (action->type != RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)
+			goto trans;
+		conf.of_set_vlan_pcp = action->conf;
+		if (na_vlan_priority)
+			goto override_na_vlan_priority;
+		i = TCA_VLAN_ACT_MODIFY;
+		goto action_of_vlan;
+action_of_vlan:
+		act_index =
+			mnl_attr_nest_start_check(buf, size, act_index_cur++);
+		if (!act_index ||
+		    !mnl_attr_put_strz_check(buf, size, TCA_ACT_KIND, "vlan"))
+			goto error_nobufs;
+		act = mnl_attr_nest_start_check(buf, size, TCA_ACT_OPTIONS);
+		if (!act)
+			goto error_nobufs;
+		if (!mnl_attr_put_check(buf, size, TCA_VLAN_PARMS,
+					sizeof(struct tc_vlan),
+					&(struct tc_vlan){
+						.action = TC_ACT_PIPE,
+						.v_action = i,
+					}))
+			goto error_nobufs;
+		if (i == TCA_VLAN_ACT_POP) {
+			mnl_attr_nest_end(buf, act);
+			mnl_attr_nest_end(buf, act_index);
+			++action;
+			break;
+		}
+		if (i == TCA_VLAN_ACT_PUSH &&
+		    !mnl_attr_put_u16_check(buf, size,
+					    TCA_VLAN_PUSH_VLAN_PROTOCOL,
+					    conf.of_push_vlan->ethertype))
+			goto error_nobufs;
+		na_vlan_id = mnl_nlmsg_get_payload_tail(buf);
+		if (!mnl_attr_put_u16_check(buf, size, TCA_VLAN_PAD, 0))
+			goto error_nobufs;
+		na_vlan_priority = mnl_nlmsg_get_payload_tail(buf);
+		if (!mnl_attr_put_u8_check(buf, size, TCA_VLAN_PAD, 0))
+			goto error_nobufs;
+		mnl_attr_nest_end(buf, act);
+		mnl_attr_nest_end(buf, act_index);
+		if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
+override_na_vlan_id:
+			na_vlan_id->nla_type = TCA_VLAN_PUSH_VLAN_ID;
+			*(uint16_t *)mnl_attr_get_payload(na_vlan_id) =
+				rte_be_to_cpu_16
+				(conf.of_set_vlan_vid->vlan_vid);
+		} else if (action->type ==
+			   RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
+override_na_vlan_priority:
+			na_vlan_priority->nla_type =
+				TCA_VLAN_PUSH_VLAN_PRIORITY;
+			*(uint8_t *)mnl_attr_get_payload(na_vlan_priority) =
+				conf.of_set_vlan_pcp->vlan_pcp;
+		}
+		++action;
+		break;
+	case END:
+		if (item->type != RTE_FLOW_ITEM_TYPE_END ||
+		    action->type != RTE_FLOW_ACTION_TYPE_END)
+			goto trans;
+		if (na_flower_act)
+			mnl_attr_nest_end(buf, na_flower_act);
+		if (na_flower)
+			mnl_attr_nest_end(buf, na_flower);
+		nlh = buf;
+		return nlh->nlmsg_len;
+	}
+	back = trans;
+	trans = mlx5_nl_flow_trans[trans[n - 1]];
+	n = 0;
+	goto trans;
+error_nobufs:
+	if (buf != buf_tmp) {
+		buf = buf_tmp;
+		size = sizeof(buf_tmp);
+		goto init;
+	}
+	return rte_flow_error_set
+		(error, ENOBUFS, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+		 "generated TC message is too large");
+}
+
+/**
+ * Brand rtnetlink buffer with unique handle.
+ *
+ * This handle should be unique for a given network interface to avoid
+ * collisions.
+ *
+ * @param buf
+ *   Flow rule buffer previously initialized by mlx5_nl_flow_transpose().
+ * @param handle
+ *   Unique 32-bit handle to use.
+ */
+void
+mlx5_nl_flow_brand(void *buf, uint32_t handle)
+{
+	struct tcmsg *tcm = mnl_nlmsg_get_payload(buf);
+
+	tcm->tcm_handle = handle;
+}
+
+/**
+ * Send Netlink message with acknowledgment.
+ *
+ * @param nl
+ *   Libmnl socket to use.
+ * @param nlh
+ *   Message to send. This function always raises the NLM_F_ACK flag before
+ *   sending.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_nl_flow_nl_ack(struct mnl_socket *nl, struct nlmsghdr *nlh)
+{
+	alignas(struct nlmsghdr)
+	uint8_t ans[mnl_nlmsg_size(sizeof(struct nlmsgerr)) +
+		    nlh->nlmsg_len - sizeof(*nlh)];
+	uint32_t seq = random();
+	int ret;
+
+	nlh->nlmsg_flags |= NLM_F_ACK;
+	nlh->nlmsg_seq = seq;
+	ret = mnl_socket_sendto(nl, nlh, nlh->nlmsg_len);
+	if (ret != -1)
+		ret = mnl_socket_recvfrom(nl, ans, sizeof(ans));
+	if (ret != -1)
+		ret = mnl_cb_run
+			(ans, ret, seq, mnl_socket_get_portid(nl), NULL, NULL);
+	if (!ret)
+		return 0;
+	rte_errno = errno;
+	return -rte_errno;
+}
+
+/**
+ * Create a Netlink flow rule.
+ *
+ * @param nl
+ *   Libmnl socket to use.
+ * @param buf
+ *   Flow rule buffer previously initialized by mlx5_nl_flow_transpose().
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_flow_create(struct mnl_socket *nl, void *buf,
+		    struct rte_flow_error *error)
+{
+	struct nlmsghdr *nlh = buf;
+
+	nlh->nlmsg_type = RTM_NEWTFILTER;
+	nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
+	if (!mlx5_nl_flow_nl_ack(nl, nlh))
+		return 0;
+	return rte_flow_error_set
+		(error, rte_errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+		 "netlink: failed to create TC flow rule");
+}
+
+/**
+ * Destroy a Netlink flow rule.
+ *
+ * @param nl
+ *   Libmnl socket to use.
+ * @param buf
+ *   Flow rule buffer previously initialized by mlx5_nl_flow_transpose().
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_flow_destroy(struct mnl_socket *nl, void *buf,
+		     struct rte_flow_error *error)
+{
+	struct nlmsghdr *nlh = buf;
+
+	nlh->nlmsg_type = RTM_DELTFILTER;
+	nlh->nlmsg_flags = NLM_F_REQUEST;
+	if (!mlx5_nl_flow_nl_ack(nl, nlh))
+		return 0;
+	return rte_flow_error_set
+		(error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+		 "netlink: failed to destroy TC flow rule");
+}
+
+/**
+ * Initialize ingress qdisc of a given network interface.
+ *
+ * @param nl
+ *   Libmnl socket of the @p NETLINK_ROUTE kind.
+ * @param ifindex
+ *   Index of network interface to initialize.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_nl_flow_init(struct mnl_socket *nl, unsigned int ifindex,
+		  struct rte_flow_error *error)
+{
+	struct nlmsghdr *nlh;
+	struct tcmsg *tcm;
+	alignas(struct nlmsghdr)
+	uint8_t buf[mnl_nlmsg_size(sizeof(*tcm) + 128)];
+
+	/* Destroy existing ingress qdisc and everything attached to it. */
+	nlh = mnl_nlmsg_put_header(buf);
+	nlh->nlmsg_type = RTM_DELQDISC;
+	nlh->nlmsg_flags = NLM_F_REQUEST;
+	tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
+	tcm->tcm_family = AF_UNSPEC;
+	tcm->tcm_ifindex = ifindex;
+	tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
+	tcm->tcm_parent = TC_H_INGRESS;
+	/* Ignore errors when qdisc is already absent. */
+	if (mlx5_nl_flow_nl_ack(nl, nlh) &&
+	    rte_errno != EINVAL && rte_errno != ENOENT)
+		return rte_flow_error_set
+			(error, rte_errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			 NULL, "netlink: failed to remove ingress qdisc");
+	/* Create fresh ingress qdisc. */
+	nlh = mnl_nlmsg_put_header(buf);
+	nlh->nlmsg_type = RTM_NEWQDISC;
+	nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
+	tcm = mnl_nlmsg_put_extra_header(nlh, sizeof(*tcm));
+	tcm->tcm_family = AF_UNSPEC;
+	tcm->tcm_ifindex = ifindex;
+	tcm->tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0);
+	tcm->tcm_parent = TC_H_INGRESS;
+	mnl_attr_put_strz_check(nlh, sizeof(buf), TCA_KIND, "ingress");
+	if (mlx5_nl_flow_nl_ack(nl, nlh))
+		return rte_flow_error_set
+			(error, rte_errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			 NULL, "netlink: failed to create ingress qdisc");
+	return 0;
+}
+
+/**
+ * Create and configure a libmnl socket for Netlink flow rules.
+ *
+ * @return
+ *   A valid libmnl socket object pointer on success, NULL otherwise and
+ *   rte_errno is set.
+ */
+struct mnl_socket *
+mlx5_nl_flow_socket_create(void)
+{
+	struct mnl_socket *nl = mnl_socket_open(NETLINK_ROUTE);
+
+	if (nl) {
+		mnl_socket_setsockopt(nl, NETLINK_CAP_ACK, &(int){ 1 },
+				      sizeof(int));
+		if (!mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID))
+			return nl;
+	}
+	rte_errno = errno;
+	if (nl)
+		mnl_socket_close(nl);
+	return NULL;
+}
+
+/**
+ * Destroy a libmnl socket.
+ */
+void
+mlx5_nl_flow_socket_destroy(struct mnl_socket *nl)
+{
+	mnl_socket_close(nl);
+}
diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h
index 0cf370cd..0870d32f 100644
--- a/drivers/net/mlx5/mlx5_prm.h
+++ b/drivers/net/mlx5/mlx5_prm.h
@@ -21,6 +21,9 @@
 #include <rte_vect.h>
 #include "mlx5_autoconf.h"
 
+/* RSS hash key size. */
+#define MLX5_RSS_HASH_KEY_LEN 40
+
 /* Get CQE owner bit. */
 #define MLX5_CQE_OWNER(op_own) ((op_own) & MLX5_CQE_OWNER_MASK)
 
@@ -240,7 +243,9 @@ struct mlx5_cqe {
 	uint8_t padding[64];
 #endif
 	uint8_t pkt_info;
-	uint8_t rsvd0[11];
+	uint8_t rsvd0;
+	uint16_t wqe_id;
+	uint8_t rsvd3[8];
 	uint32_t rx_hash_res;
 	uint8_t rx_hash_type;
 	uint8_t rsvd1[11];
@@ -285,7 +290,10 @@ struct mlx5_cqe {
 struct mlx5_mini_cqe8 {
 	union {
 		uint32_t rx_hash_result;
-		uint32_t checksum;
+		struct {
+			uint16_t checksum;
+			uint16_t stride_idx;
+		};
 		struct {
 			uint16_t wqe_counter;
 			uint8_t  s_wqe_opcode;
diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c
index d69b4c09..b95778a8 100644
--- a/drivers/net/mlx5/mlx5_rss.c
+++ b/drivers/net/mlx5/mlx5_rss.c
@@ -50,10 +50,11 @@ mlx5_rss_hash_update(struct rte_eth_dev *dev,
 		return -rte_errno;
 	}
 	if (rss_conf->rss_key && rss_conf->rss_key_len) {
-		if (rss_conf->rss_key_len != rss_hash_default_key_len) {
+		if (rss_conf->rss_key_len != MLX5_RSS_HASH_KEY_LEN) {
 			DRV_LOG(ERR,
-				"port %u RSS key len must be %zu Bytes long",
-				dev->data->port_id, rss_hash_default_key_len);
+				"port %u RSS key len must be %s Bytes long",
+				dev->data->port_id,
+				RTE_STR(MLX5_RSS_HASH_KEY_LEN));
 			rte_errno = EINVAL;
 			return -rte_errno;
 		}
diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c
index 80824bc4..e74fdef8 100644
--- a/drivers/net/mlx5/mlx5_rxmode.c
+++ b/drivers/net/mlx5/mlx5_rxmode.c
@@ -32,10 +32,18 @@
 void
 mlx5_promiscuous_enable(struct rte_eth_dev *dev)
 {
+	struct priv *priv = dev->data->dev_private;
 	int ret;
 
 	dev->data->promiscuous = 1;
-	if (((struct priv *)dev->data->dev_private)->config.vf)
+	if (priv->isolated) {
+		DRV_LOG(WARNING,
+			"port %u cannot enable promiscuous mode"
+			" in flow isolation mode",
+			dev->data->port_id);
+		return;
+	}
+	if (priv->config.vf)
 		mlx5_nl_promisc(dev, 1);
 	ret = mlx5_traffic_restart(dev);
 	if (ret)
@@ -52,10 +60,11 @@ mlx5_promiscuous_enable(struct rte_eth_dev *dev)
 void
 mlx5_promiscuous_disable(struct rte_eth_dev *dev)
 {
+	struct priv *priv = dev->data->dev_private;
 	int ret;
 
 	dev->data->promiscuous = 0;
-	if (((struct priv *)dev->data->dev_private)->config.vf)
+	if (priv->config.vf)
 		mlx5_nl_promisc(dev, 0);
 	ret = mlx5_traffic_restart(dev);
 	if (ret)
@@ -72,10 +81,18 @@ mlx5_promiscuous_disable(struct rte_eth_dev *dev)
 void
 mlx5_allmulticast_enable(struct rte_eth_dev *dev)
 {
+	struct priv *priv = dev->data->dev_private;
 	int ret;
 
 	dev->data->all_multicast = 1;
-	if (((struct priv *)dev->data->dev_private)->config.vf)
+	if (priv->isolated) {
+		DRV_LOG(WARNING,
+			"port %u cannot enable allmulticast mode"
+			" in flow isolation mode",
+			dev->data->port_id);
+		return;
+	}
+	if (priv->config.vf)
 		mlx5_nl_allmulti(dev, 1);
 	ret = mlx5_traffic_restart(dev);
 	if (ret)
@@ -92,10 +109,11 @@ mlx5_allmulticast_enable(struct rte_eth_dev *dev)
 void
 mlx5_allmulticast_disable(struct rte_eth_dev *dev)
 {
+	struct priv *priv = dev->data->dev_private;
 	int ret;
 
 	dev->data->all_multicast = 0;
-	if (((struct priv *)dev->data->dev_private)->config.vf)
+	if (priv->config.vf)
 		mlx5_nl_allmulti(dev, 0);
 	ret = mlx5_traffic_restart(dev);
 	if (ret)
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index de3f869e..1f7bfd44 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -52,7 +52,9 @@ uint8_t rss_hash_default_key[] = {
 };
 
 /* Length of the default RSS hash key. */
-const size_t rss_hash_default_key_len = sizeof(rss_hash_default_key);
+static_assert(MLX5_RSS_HASH_KEY_LEN ==
+	      (unsigned int)sizeof(rss_hash_default_key),
+	      "wrong RSS default key size.");
 
 /**
  * Check whether Multi-Packet RQ can be enabled for the device.
@@ -386,8 +388,10 @@ mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev)
 			     DEV_RX_OFFLOAD_TIMESTAMP |
 			     DEV_RX_OFFLOAD_JUMBO_FRAME);
 
+	offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
 	if (config->hw_fcs_strip)
-		offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
+		offloads |= DEV_RX_OFFLOAD_KEEP_CRC;
+
 	if (config->hw_csum)
 		offloads |= (DEV_RX_OFFLOAD_IPV4_CKSUM |
 			     DEV_RX_OFFLOAD_UDP_CKSUM |
@@ -643,7 +647,8 @@ mlx5_arm_cq(struct mlx5_rxq_data *rxq, int sq_n_rxq)
 	doorbell = (uint64_t)doorbell_hi << 32;
 	doorbell |=  rxq->cqn;
 	rxq->cq_db[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi);
-	rte_write64(rte_cpu_to_be_64(doorbell), cq_db_reg);
+	mlx5_uar_write64(rte_cpu_to_be_64(doorbell),
+			 cq_db_reg, rxq->uar_lock_cq);
 }
 
 /**
@@ -818,7 +823,13 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
 	if (config->cqe_comp && !rxq_data->hw_timestamp) {
 		attr.cq.mlx5.comp_mask |=
 			MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;
+#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
+		attr.cq.mlx5.cqe_comp_res_format =
+			mprq_en ? MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX :
+				  MLX5DV_CQE_RES_FORMAT_HASH;
+#else
 		attr.cq.mlx5.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH;
+#endif
 		/*
 		 * For vectorized Rx, it must not be doubled in order to
 		 * make cq_ci and rq_ci aligned.
@@ -976,7 +987,7 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
 	rxq_data->rq_db = rwq.dbrec;
 	rxq_data->cqe_n = log2above(cq_info.cqe_cnt);
 	rxq_data->cq_ci = 0;
-	rxq_data->strd_ci = 0;
+	rxq_data->consumed_strd = 0;
 	rxq_data->rq_pi = 0;
 	rxq_data->zip = (struct rxq_zip){
 		.ai = 0,
@@ -993,8 +1004,6 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
 	DRV_LOG(DEBUG, "port %u rxq %u updated with %p", dev->data->port_id,
 		idx, (void *)&tmpl);
 	rte_atomic32_inc(&tmpl->refcnt);
-	DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
-		dev->data->port_id, idx, rte_atomic32_read(&tmpl->refcnt));
 	LIST_INSERT_HEAD(&priv->rxqsibv, tmpl, next);
 	priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
 	return tmpl;
@@ -1036,9 +1045,6 @@ mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
 	rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
 	if (rxq_ctrl->ibv) {
 		rte_atomic32_inc(&rxq_ctrl->ibv->refcnt);
-		DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
-			dev->data->port_id, rxq_ctrl->idx,
-			rte_atomic32_read(&rxq_ctrl->ibv->refcnt));
 	}
 	return rxq_ctrl->ibv;
 }
@@ -1058,9 +1064,6 @@ mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv)
 	assert(rxq_ibv);
 	assert(rxq_ibv->wq);
 	assert(rxq_ibv->cq);
-	DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
-		PORT_ID(rxq_ibv->rxq_ctrl->priv),
-		rxq_ibv->rxq_ctrl->idx, rte_atomic32_read(&rxq_ibv->refcnt));
 	if (rte_atomic32_dec_and_test(&rxq_ibv->refcnt)) {
 		rxq_free_elts(rxq_ibv->rxq_ctrl);
 		claim_zero(mlx5_glue->destroy_wq(rxq_ibv->wq));
@@ -1231,6 +1234,13 @@ mlx5_mprq_alloc_mp(struct rte_eth_dev *dev)
 	 */
 	desc *= 4;
 	obj_num = desc + MLX5_MPRQ_MP_CACHE_SZ * priv->rxqs_n;
+	/*
+	 * rte_mempool_create_empty() has sanity check to refuse large cache
+	 * size compared to the number of elements.
+	 * CACHE_FLUSHTHRESH_MULTIPLIER is defined in a C file, so using a
+	 * constant number 2 instead.
+	 */
+	obj_num = RTE_MAX(obj_num, MLX5_MPRQ_MP_CACHE_SZ * 2);
 	/* Check a mempool is already allocated and if it can be resued. */
 	if (mp != NULL && mp->elt_size >= obj_size && mp->size >= obj_num) {
 		DRV_LOG(DEBUG, "port %u mempool %s is being reused",
@@ -1346,7 +1356,7 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		sizeof(struct rte_mbuf_ext_shared_info) +
 		RTE_PKTMBUF_HEADROOM;
 	if (mprq_en &&
-	    desc >= (1U << config->mprq.stride_num_n) &&
+	    desc > (1U << config->mprq.stride_num_n) &&
 	    mprq_stride_size <= (1U << config->mprq.max_stride_size_n)) {
 		/* TODO: Rx scatter isn't supported yet. */
 		tmpl->rxq.sges_n = 0;
@@ -1401,6 +1411,14 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 			dev->data->dev_conf.rxmode.max_rx_pkt_len,
 			mb_len - RTE_PKTMBUF_HEADROOM);
 	}
+	if (mprq_en && !mlx5_rxq_mprq_enabled(&tmpl->rxq))
+		DRV_LOG(WARNING,
+			"port %u MPRQ is requested but cannot be enabled"
+			" (requested: desc = %u, stride_sz = %u,"
+			" supported: min_stride_num = %u, max_stride_sz = %u).",
+			dev->data->port_id, desc, mprq_stride_size,
+			(1 << config->mprq.stride_num_n),
+			(1 << config->mprq.max_stride_size_n));
 	DRV_LOG(DEBUG, "port %u maximum number of segments per packet: %u",
 		dev->data->port_id, 1 << tmpl->rxq.sges_n);
 	if (desc % (1 << tmpl->rxq.sges_n)) {
@@ -1419,17 +1437,17 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	/* Configure VLAN stripping. */
 	tmpl->rxq.vlan_strip = !!(offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
 	/* By default, FCS (CRC) is stripped by hardware. */
-	if (offloads & DEV_RX_OFFLOAD_CRC_STRIP) {
-		tmpl->rxq.crc_present = 0;
-	} else if (config->hw_fcs_strip) {
-		tmpl->rxq.crc_present = 1;
-	} else {
-		DRV_LOG(WARNING,
-			"port %u CRC stripping has been disabled but will"
-			" still be performed by hardware, make sure MLNX_OFED"
-			" and firmware are up to date",
-			dev->data->port_id);
-		tmpl->rxq.crc_present = 0;
+	tmpl->rxq.crc_present = 0;
+	if (rte_eth_dev_must_keep_crc(offloads)) {
+		if (config->hw_fcs_strip) {
+			tmpl->rxq.crc_present = 1;
+		} else {
+			DRV_LOG(WARNING,
+				"port %u CRC stripping has been disabled but will"
+				" still be performed by hardware, make sure MLNX_OFED"
+				" and firmware are up to date",
+				dev->data->port_id);
+		}
 	}
 	DRV_LOG(DEBUG,
 		"port %u CRC stripping is %s, %u bytes will be subtracted from"
@@ -1447,10 +1465,11 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	tmpl->rxq.elts_n = log2above(desc);
 	tmpl->rxq.elts =
 		(struct rte_mbuf *(*)[1 << tmpl->rxq.elts_n])(tmpl + 1);
+#ifndef RTE_ARCH_64
+	tmpl->rxq.uar_lock_cq = &priv->uar_lock_cq;
+#endif
 	tmpl->idx = idx;
 	rte_atomic32_inc(&tmpl->refcnt);
-	DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d", dev->data->port_id,
-		idx, rte_atomic32_read(&tmpl->refcnt));
 	LIST_INSERT_HEAD(&priv->rxqsctrl, tmpl, next);
 	return tmpl;
 error:
@@ -1481,9 +1500,6 @@ mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx)
 					rxq);
 		mlx5_rxq_ibv_get(dev, idx);
 		rte_atomic32_inc(&rxq_ctrl->refcnt);
-		DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d",
-			dev->data->port_id, rxq_ctrl->idx,
-			rte_atomic32_read(&rxq_ctrl->refcnt));
 	}
 	return rxq_ctrl;
 }
@@ -1511,8 +1527,6 @@ mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx)
 	assert(rxq_ctrl->priv);
 	if (rxq_ctrl->ibv && !mlx5_rxq_ibv_release(rxq_ctrl->ibv))
 		rxq_ctrl->ibv = NULL;
-	DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d", dev->data->port_id,
-		rxq_ctrl->idx, rte_atomic32_read(&rxq_ctrl->refcnt));
 	if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) {
 		mlx5_mr_btree_free(&rxq_ctrl->rxq.mr_ctrl.cache_bh);
 		LIST_REMOVE(rxq_ctrl, next);
@@ -1630,14 +1644,10 @@ mlx5_ind_table_ibv_new(struct rte_eth_dev *dev, const uint16_t *queues,
 	}
 	rte_atomic32_inc(&ind_tbl->refcnt);
 	LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next);
-	DEBUG("port %u new indirection table %p: queues:%u refcnt:%d",
-	      dev->data->port_id, (void *)ind_tbl, 1 << wq_n,
-	      rte_atomic32_read(&ind_tbl->refcnt));
 	return ind_tbl;
 error:
 	rte_free(ind_tbl);
-	DRV_LOG(DEBUG, "port %u cannot create indirection table",
-		dev->data->port_id);
+	DEBUG("port %u cannot create indirection table", dev->data->port_id);
 	return NULL;
 }
 
@@ -1672,9 +1682,6 @@ mlx5_ind_table_ibv_get(struct rte_eth_dev *dev, const uint16_t *queues,
 		unsigned int i;
 
 		rte_atomic32_inc(&ind_tbl->refcnt);
-		DRV_LOG(DEBUG, "port %u indirection table %p: refcnt %d",
-			dev->data->port_id, (void *)ind_tbl,
-			rte_atomic32_read(&ind_tbl->refcnt));
 		for (i = 0; i != ind_tbl->queues_n; ++i)
 			mlx5_rxq_get(dev, ind_tbl->queues[i]);
 	}
@@ -1698,15 +1705,9 @@ mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
 {
 	unsigned int i;
 
-	DRV_LOG(DEBUG, "port %u indirection table %p: refcnt %d",
-		dev->data->port_id, (void *)ind_tbl,
-		rte_atomic32_read(&ind_tbl->refcnt));
-	if (rte_atomic32_dec_and_test(&ind_tbl->refcnt)) {
+	if (rte_atomic32_dec_and_test(&ind_tbl->refcnt))
 		claim_zero(mlx5_glue->destroy_rwq_ind_table
 			   (ind_tbl->ind_table));
-		DEBUG("port %u delete indirection table %p: queues: %u",
-		      dev->data->port_id, (void *)ind_tbl, ind_tbl->queues_n);
-	}
 	for (i = 0; i != ind_tbl->queues_n; ++i)
 		claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
 	if (!rte_atomic32_read(&ind_tbl->refcnt)) {
@@ -1758,10 +1759,6 @@ mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev)
  *   first queue index will be taken for the indirection table.
  * @param queues_n
  *   Number of queues.
- * @param tunnel
- *   Tunnel type, implies tunnel offloading like inner checksum if available.
- * @param rss_level
- *   RSS hash on tunnel level.
  *
  * @return
  *   The Verbs object initialised, NULL otherwise and rte_errno is set.
@@ -1771,16 +1768,13 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 	      const uint8_t *rss_key, uint32_t rss_key_len,
 	      uint64_t hash_fields,
 	      const uint16_t *queues, uint32_t queues_n,
-	      uint32_t tunnel, uint32_t rss_level)
+	      int tunnel __rte_unused)
 {
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
 	struct mlx5_ind_table_ibv *ind_tbl;
 	struct ibv_qp *qp;
 	int err;
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	struct mlx5dv_qp_init_attr qp_init_attr = {0};
-#endif
 
 	queues_n = hash_fields ? queues_n : 1;
 	ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
@@ -1791,15 +1785,10 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 		return NULL;
 	}
 	if (!rss_key_len) {
-		rss_key_len = rss_hash_default_key_len;
+		rss_key_len = MLX5_RSS_HASH_KEY_LEN;
 		rss_key = rss_hash_default_key;
 	}
 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	if (tunnel) {
-		qp_init_attr.comp_mask =
-				MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
-		qp_init_attr.create_flags = MLX5DV_QP_CREATE_TUNNEL_OFFLOADS;
-	}
 	qp = mlx5_glue->dv_create_qp
 		(priv->ctx,
 		 &(struct ibv_qp_init_attr_ex){
@@ -1811,25 +1800,20 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 			.rx_hash_conf = (struct ibv_rx_hash_conf){
 				.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
 				.rx_hash_key_len = rss_key_len ? rss_key_len :
-						   rss_hash_default_key_len,
+						   MLX5_RSS_HASH_KEY_LEN,
 				.rx_hash_key = rss_key ?
 					       (void *)(uintptr_t)rss_key :
 					       rss_hash_default_key,
-				.rx_hash_fields_mask = hash_fields |
-					(tunnel && rss_level > 1 ?
-					(uint32_t)IBV_RX_HASH_INNER : 0),
+				.rx_hash_fields_mask = hash_fields,
 			},
 			.rwq_ind_tbl = ind_tbl->ind_table,
 			.pd = priv->pd,
 		 },
-		 &qp_init_attr);
-	DEBUG("port %u new QP:%p ind_tbl:%p hash_fields:0x%" PRIx64
-	      " tunnel:0x%x level:%u dv_attr:comp_mask:0x%" PRIx64
-	      " create_flags:0x%x",
-	      dev->data->port_id, (void *)qp, (void *)ind_tbl,
-	      (tunnel && rss_level == 2 ? (uint32_t)IBV_RX_HASH_INNER : 0) |
-	      hash_fields, tunnel, rss_level,
-	      qp_init_attr.comp_mask, qp_init_attr.create_flags);
+		 &(struct mlx5dv_qp_init_attr){
+			.comp_mask = tunnel ?
+				MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS : 0,
+			.create_flags = MLX5DV_QP_CREATE_TUNNEL_OFFLOADS,
+		 });
 #else
 	qp = mlx5_glue->create_qp_ex
 		(priv->ctx,
@@ -1842,7 +1826,7 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 			.rx_hash_conf = (struct ibv_rx_hash_conf){
 				.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
 				.rx_hash_key_len = rss_key_len ? rss_key_len :
-						   rss_hash_default_key_len,
+						   MLX5_RSS_HASH_KEY_LEN,
 				.rx_hash_key = rss_key ?
 					       (void *)(uintptr_t)rss_key :
 					       rss_hash_default_key,
@@ -1851,10 +1835,6 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 			.rwq_ind_tbl = ind_tbl->ind_table,
 			.pd = priv->pd,
 		 });
-	DEBUG("port %u new QP:%p ind_tbl:%p hash_fields:0x%" PRIx64
-	      " tunnel:0x%x level:%hhu",
-	      dev->data->port_id, (void *)qp, (void *)ind_tbl,
-	      hash_fields, tunnel, rss_level);
 #endif
 	if (!qp) {
 		rte_errno = errno;
@@ -1867,14 +1847,9 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 	hrxq->qp = qp;
 	hrxq->rss_key_len = rss_key_len;
 	hrxq->hash_fields = hash_fields;
-	hrxq->tunnel = tunnel;
-	hrxq->rss_level = rss_level;
 	memcpy(hrxq->rss_key, rss_key, rss_key_len);
 	rte_atomic32_inc(&hrxq->refcnt);
 	LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next);
-	DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d",
-		dev->data->port_id, (void *)hrxq,
-		rte_atomic32_read(&hrxq->refcnt));
 	return hrxq;
 error:
 	err = rte_errno; /* Save rte_errno before cleanup. */
@@ -1897,10 +1872,6 @@ error:
  *   first queue index will be taken for the indirection table.
  * @param queues_n
  *   Number of queues.
- * @param tunnel
- *   Tunnel type, implies tunnel offloading like inner checksum if available.
- * @param rss_level
- *   RSS hash on tunnel level
  *
  * @return
  *   An hash Rx queue on success.
@@ -1909,8 +1880,7 @@ struct mlx5_hrxq *
 mlx5_hrxq_get(struct rte_eth_dev *dev,
 	      const uint8_t *rss_key, uint32_t rss_key_len,
 	      uint64_t hash_fields,
-	      const uint16_t *queues, uint32_t queues_n,
-	      uint32_t tunnel, uint32_t rss_level)
+	      const uint16_t *queues, uint32_t queues_n)
 {
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
@@ -1925,10 +1895,6 @@ mlx5_hrxq_get(struct rte_eth_dev *dev,
 			continue;
 		if (hrxq->hash_fields != hash_fields)
 			continue;
-		if (hrxq->tunnel != tunnel)
-			continue;
-		if (hrxq->rss_level != rss_level)
-			continue;
 		ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
 		if (!ind_tbl)
 			continue;
@@ -1937,9 +1903,6 @@ mlx5_hrxq_get(struct rte_eth_dev *dev,
 			continue;
 		}
 		rte_atomic32_inc(&hrxq->refcnt);
-		DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d",
-			dev->data->port_id, (void *)hrxq,
-			rte_atomic32_read(&hrxq->refcnt));
 		return hrxq;
 	}
 	return NULL;
@@ -1959,15 +1922,8 @@ mlx5_hrxq_get(struct rte_eth_dev *dev,
 int
 mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
 {
-	DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d",
-		dev->data->port_id, (void *)hrxq,
-		rte_atomic32_read(&hrxq->refcnt));
 	if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
 		claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
-		DEBUG("port %u delete QP %p: hash: 0x%" PRIx64 ", tunnel:"
-		      " 0x%x, level: %u",
-		      dev->data->port_id, (void *)hrxq, hrxq->hash_fields,
-		      hrxq->tunnel, hrxq->rss_level);
 		mlx5_ind_table_ibv_release(dev, hrxq->ind_table);
 		LIST_REMOVE(hrxq, next);
 		rte_free(hrxq);
@@ -2001,3 +1957,235 @@ mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev)
 	}
 	return ret;
 }
+
+/**
+ * Create a drop Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_rxq_ibv *
+mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_cq *cq;
+	struct ibv_wq *wq = NULL;
+	struct mlx5_rxq_ibv *rxq;
+
+	if (priv->drop_queue.rxq)
+		return priv->drop_queue.rxq;
+	cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
+	if (!cq) {
+		DEBUG("port %u cannot allocate CQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	wq = mlx5_glue->create_wq(priv->ctx,
+		 &(struct ibv_wq_init_attr){
+			.wq_type = IBV_WQT_RQ,
+			.max_wr = 1,
+			.max_sge = 1,
+			.pd = priv->pd,
+			.cq = cq,
+		 });
+	if (!wq) {
+		DEBUG("port %u cannot allocate WQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	rxq = rte_calloc(__func__, 1, sizeof(*rxq), 0);
+	if (!rxq) {
+		DEBUG("port %u cannot allocate drop Rx queue memory",
+		      dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	rxq->cq = cq;
+	rxq->wq = wq;
+	priv->drop_queue.rxq = rxq;
+	return rxq;
+error:
+	if (wq)
+		claim_zero(mlx5_glue->destroy_wq(wq));
+	if (cq)
+		claim_zero(mlx5_glue->destroy_cq(cq));
+	return NULL;
+}
+
+/**
+ * Release a drop Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+void
+mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_ibv *rxq = priv->drop_queue.rxq;
+
+	if (rxq->wq)
+		claim_zero(mlx5_glue->destroy_wq(rxq->wq));
+	if (rxq->cq)
+		claim_zero(mlx5_glue->destroy_cq(rxq->cq));
+	rte_free(rxq);
+	priv->drop_queue.rxq = NULL;
+}
+
+/**
+ * Create a drop indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_ind_table_ibv *
+mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct mlx5_rxq_ibv *rxq;
+	struct mlx5_ind_table_ibv tmpl;
+
+	rxq = mlx5_rxq_ibv_drop_new(dev);
+	if (!rxq)
+		return NULL;
+	tmpl.ind_table = mlx5_glue->create_rwq_ind_table
+		(priv->ctx,
+		 &(struct ibv_rwq_ind_table_init_attr){
+			.log_ind_tbl_size = 0,
+			.ind_tbl = &rxq->wq,
+			.comp_mask = 0,
+		 });
+	if (!tmpl.ind_table) {
+		DEBUG("port %u cannot allocate indirection table for drop"
+		      " queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	ind_tbl = rte_calloc(__func__, 1, sizeof(*ind_tbl), 0);
+	if (!ind_tbl) {
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	ind_tbl->ind_table = tmpl.ind_table;
+	return ind_tbl;
+error:
+	mlx5_rxq_ibv_drop_release(dev);
+	return NULL;
+}
+
+/**
+ * Release a drop indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl = priv->drop_queue.hrxq->ind_table;
+
+	claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl->ind_table));
+	mlx5_rxq_ibv_drop_release(dev);
+	rte_free(ind_tbl);
+	priv->drop_queue.hrxq->ind_table = NULL;
+}
+
+/**
+ * Create a drop Rx Hash queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_hrxq *
+mlx5_hrxq_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct ibv_qp *qp;
+	struct mlx5_hrxq *hrxq;
+
+	if (priv->drop_queue.hrxq) {
+		rte_atomic32_inc(&priv->drop_queue.hrxq->refcnt);
+		return priv->drop_queue.hrxq;
+	}
+	ind_tbl = mlx5_ind_table_ibv_drop_new(dev);
+	if (!ind_tbl)
+		return NULL;
+	qp = mlx5_glue->create_qp_ex(priv->ctx,
+		 &(struct ibv_qp_init_attr_ex){
+			.qp_type = IBV_QPT_RAW_PACKET,
+			.comp_mask =
+				IBV_QP_INIT_ATTR_PD |
+				IBV_QP_INIT_ATTR_IND_TABLE |
+				IBV_QP_INIT_ATTR_RX_HASH,
+			.rx_hash_conf = (struct ibv_rx_hash_conf){
+				.rx_hash_function =
+					IBV_RX_HASH_FUNC_TOEPLITZ,
+				.rx_hash_key_len = MLX5_RSS_HASH_KEY_LEN,
+				.rx_hash_key = rss_hash_default_key,
+				.rx_hash_fields_mask = 0,
+				},
+			.rwq_ind_tbl = ind_tbl->ind_table,
+			.pd = priv->pd
+		 });
+	if (!qp) {
+		DEBUG("port %u cannot allocate QP for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	hrxq = rte_calloc(__func__, 1, sizeof(*hrxq), 0);
+	if (!hrxq) {
+		DRV_LOG(WARNING,
+			"port %u cannot allocate memory for drop queue",
+			dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	hrxq->ind_table = ind_tbl;
+	hrxq->qp = qp;
+	priv->drop_queue.hrxq = hrxq;
+	rte_atomic32_set(&hrxq->refcnt, 1);
+	return hrxq;
+error:
+	if (ind_tbl)
+		mlx5_ind_table_ibv_drop_release(dev);
+	return NULL;
+}
+
+/**
+ * Release a drop hash Rx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_hrxq_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq;
+
+	if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
+		claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
+		mlx5_ind_table_ibv_drop_release(dev);
+		rte_free(hrxq);
+		priv->drop_queue.hrxq = NULL;
+	}
+}
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 52785946..2d14f8a6 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -38,7 +38,7 @@ rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe);
 
 static __rte_always_inline int
 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
-		 uint16_t cqe_cnt, uint32_t *rss_hash);
+		 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe);
 
 static __rte_always_inline uint32_t
 rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe);
@@ -495,6 +495,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	volatile struct mlx5_wqe_ctrl *last_wqe = NULL;
 	unsigned int segs_n = 0;
 	const unsigned int max_inline = txq->max_inline;
+	uint64_t addr_64;
 
 	if (unlikely(!pkts_n))
 		return 0;
@@ -503,8 +504,6 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Start processing. */
 	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
-	/* A CQE slot must always be available. */
-	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
 		return 0;
@@ -711,12 +710,12 @@ pkt_inline:
 			ds = 3;
 use_dseg:
 			/* Add the remaining packet as a simple ds. */
-			addr = rte_cpu_to_be_64(addr);
+			addr_64 = rte_cpu_to_be_64(addr);
 			*dseg = (rte_v128u32_t){
 				rte_cpu_to_be_32(length),
 				mlx5_tx_mb2mr(txq, buf),
-				addr,
-				addr >> 32,
+				addr_64,
+				addr_64 >> 32,
 			};
 			++ds;
 			if (!segs_n)
@@ -750,12 +749,12 @@ next_seg:
 		total_length += length;
 #endif
 		/* Store segment information. */
-		addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, uintptr_t));
+		addr_64 = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, uintptr_t));
 		*dseg = (rte_v128u32_t){
 			rte_cpu_to_be_32(length),
 			mlx5_tx_mb2mr(txq, buf),
-			addr,
-			addr >> 32,
+			addr_64,
+			addr_64 >> 32,
 		};
 		(*txq->elts)[++elts_head & elts_m] = buf;
 		if (--segs_n)
@@ -816,14 +815,13 @@ next_wqe:
 	/* Check whether completion threshold has been reached. */
 	comp = txq->elts_comp + i + j + k;
 	if (comp >= MLX5_TX_COMP_THRESH) {
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
 		/* Request completion on last WQE. */
 		last_wqe->ctrl2 = rte_cpu_to_be_32(8);
 		/* Save elts_head in unused "immediate" field of WQE. */
 		last_wqe->ctrl3 = txq->elts_head;
 		txq->elts_comp = 0;
-#ifndef NDEBUG
-		++txq->cq_pi;
-#endif
 	} else {
 		txq->elts_comp = comp;
 	}
@@ -942,8 +940,6 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Start processing. */
 	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
-	/* A CQE slot must always be available. */
-	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
 		return 0;
@@ -1032,14 +1028,13 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	if (comp >= MLX5_TX_COMP_THRESH) {
 		volatile struct mlx5_wqe *wqe = mpw.wqe;
 
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
 		/* Request completion on last WQE. */
 		wqe->ctrl[2] = rte_cpu_to_be_32(8);
 		/* Save elts_head in unused "immediate" field of WQE. */
 		wqe->ctrl[3] = elts_head;
 		txq->elts_comp = 0;
-#ifndef NDEBUG
-		++txq->cq_pi;
-#endif
 	} else {
 		txq->elts_comp = comp;
 	}
@@ -1171,8 +1166,6 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 	/* Start processing. */
 	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
-	/* A CQE slot must always be available. */
-	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	do {
 		struct rte_mbuf *buf = *(pkts++);
 		uintptr_t addr;
@@ -1329,14 +1322,13 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 	if (comp >= MLX5_TX_COMP_THRESH) {
 		volatile struct mlx5_wqe *wqe = mpw.wqe;
 
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
 		/* Request completion on last WQE. */
 		wqe->ctrl[2] = rte_cpu_to_be_32(8);
 		/* Save elts_head in unused "immediate" field of WQE. */
 		wqe->ctrl[3] = elts_head;
 		txq->elts_comp = 0;
-#ifndef NDEBUG
-		++txq->cq_pi;
-#endif
 	} else {
 		txq->elts_comp = comp;
 	}
@@ -1450,6 +1442,7 @@ txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 	unsigned int mpw_room = 0;
 	unsigned int inl_pad = 0;
 	uint32_t inl_hdr;
+	uint64_t addr_64;
 	struct mlx5_mpw mpw = {
 		.state = MLX5_MPW_STATE_CLOSED,
 	};
@@ -1459,8 +1452,6 @@ txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 	/* Start processing. */
 	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
-	/* A CQE slot must always be available. */
-	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
 		return 0;
@@ -1586,13 +1577,13 @@ txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 					((uintptr_t)mpw.data.raw +
 					 inl_pad);
 			(*txq->elts)[elts_head++ & elts_m] = buf;
-			addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
-								 uintptr_t));
+			addr_64 = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
+								    uintptr_t));
 			*dseg = (rte_v128u32_t) {
 				rte_cpu_to_be_32(length),
 				mlx5_tx_mb2mr(txq, buf),
-				addr,
-				addr >> 32,
+				addr_64,
+				addr_64 >> 32,
 			};
 			mpw.data.raw = (volatile void *)(dseg + 1);
 			mpw.total_len += (inl_pad + sizeof(*dseg));
@@ -1616,15 +1607,14 @@ txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 			 (1 << txq->wqe_n) / MLX5_TX_COMP_THRESH_INLINE_DIV) {
 		volatile struct mlx5_wqe *wqe = mpw.wqe;
 
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
 		/* Request completion on last WQE. */
 		wqe->ctrl[2] = rte_cpu_to_be_32(8);
 		/* Save elts_head in unused "immediate" field of WQE. */
 		wqe->ctrl[3] = elts_head;
 		txq->elts_comp = 0;
 		txq->mpw_comp = txq->wqe_ci;
-#ifndef NDEBUG
-		++txq->cq_pi;
-#endif
 	} else {
 		txq->elts_comp += j;
 	}
@@ -1722,8 +1712,9 @@ rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe)
  *   Pointer to RX queue.
  * @param cqe
  *   CQE to process.
- * @param[out] rss_hash
- *   Packet RSS Hash result.
+ * @param[out] mcqe
+ *   Store pointer to mini-CQE if compressed. Otherwise, the pointer is not
+ *   written.
  *
  * @return
  *   Packet size in bytes (0 if there is none), -1 in case of completion
@@ -1731,7 +1722,7 @@ rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe)
  */
 static inline int
 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
-		 uint16_t cqe_cnt, uint32_t *rss_hash)
+		 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe)
 {
 	struct rxq_zip *zip = &rxq->zip;
 	uint16_t cqe_n = cqe_cnt + 1;
@@ -1745,7 +1736,7 @@ mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
 			(uintptr_t)(&(*rxq->cqes)[zip->ca & cqe_cnt].pkt_info);
 
 		len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt);
-		*rss_hash = rte_be_to_cpu_32((*mc)[zip->ai & 7].rx_hash_result);
+		*mcqe = &(*mc)[zip->ai & 7];
 		if ((++zip->ai & 7) == 0) {
 			/* Invalidate consumed CQEs */
 			idx = zip->ca;
@@ -1810,7 +1801,7 @@ mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
 			zip->cq_ci = rxq->cq_ci + zip->cqe_cnt;
 			/* Get packet size to return. */
 			len = rte_be_to_cpu_32((*mc)[0].byte_cnt);
-			*rss_hash = rte_be_to_cpu_32((*mc)[0].rx_hash_result);
+			*mcqe = &(*mc)[0];
 			zip->ai = 1;
 			/* Prefetch all the entries to be invalidated */
 			idx = zip->ca;
@@ -1821,7 +1812,6 @@ mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
 			}
 		} else {
 			len = rte_be_to_cpu_32(cqe->byte_cnt);
-			*rss_hash = rte_be_to_cpu_32(cqe->rx_hash_res);
 		}
 		/* Error while receiving packet. */
 		if (unlikely(MLX5_CQE_OPCODE(op_own) == MLX5_CQE_RESP_ERR))
@@ -1934,7 +1924,8 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		volatile struct mlx5_wqe_data_seg *wqe =
 			&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx];
 		struct rte_mbuf *rep = (*rxq->elts)[idx];
-		uint32_t rss_hash_res = 0;
+		volatile struct mlx5_mini_cqe8 *mcqe = NULL;
+		uint32_t rss_hash_res;
 
 		if (pkt)
 			NEXT(seg) = rep;
@@ -1964,8 +1955,7 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		}
 		if (!pkt) {
 			cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
-			len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt,
-					       &rss_hash_res);
+			len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, &mcqe);
 			if (!len) {
 				rte_mbuf_raw_free(rep);
 				break;
@@ -1979,6 +1969,10 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			pkt = seg;
 			assert(len >= (rxq->crc_present << 2));
 			pkt->ol_flags = 0;
+			/* If compressed, take hash result from mini-CQE. */
+			rss_hash_res = rte_be_to_cpu_32(mcqe == NULL ?
+							cqe->rx_hash_res :
+							mcqe->rx_hash_result);
 			rxq_cq_to_mbuf(rxq, pkt, cqe, rss_hash_res);
 			if (rxq->crc_present)
 				len -= ETHER_CRC_LEN;
@@ -2104,7 +2098,7 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
 	unsigned int i = 0;
 	uint16_t rq_ci = rxq->rq_ci;
-	uint16_t strd_idx = rxq->strd_ci;
+	uint16_t consumed_strd = rxq->consumed_strd;
 	struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
 
 	while (i < pkts_n) {
@@ -2112,12 +2106,14 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		void *addr;
 		int ret;
 		unsigned int len;
-		uint16_t consumed_strd;
+		uint16_t strd_cnt;
+		uint16_t strd_idx;
 		uint32_t offset;
 		uint32_t byte_cnt;
+		volatile struct mlx5_mini_cqe8 *mcqe = NULL;
 		uint32_t rss_hash_res = 0;
 
-		if (strd_idx == strd_n) {
+		if (consumed_strd == strd_n) {
 			/* Replace WQE only if the buffer is still in use. */
 			if (rte_atomic16_read(&buf->refcnt) > 1) {
 				mprq_buf_replace(rxq, rq_ci & wq_mask);
@@ -2137,12 +2133,12 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 					rxq->mprq_repl = rep;
 			}
 			/* Advance to the next WQE. */
-			strd_idx = 0;
+			consumed_strd = 0;
 			++rq_ci;
 			buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
 		}
 		cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
-		ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &rss_hash_res);
+		ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &mcqe);
 		if (!ret)
 			break;
 		if (unlikely(ret == -1)) {
@@ -2151,14 +2147,21 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			continue;
 		}
 		byte_cnt = ret;
-		consumed_strd = (byte_cnt & MLX5_MPRQ_STRIDE_NUM_MASK) >>
-				MLX5_MPRQ_STRIDE_NUM_SHIFT;
-		assert(consumed_strd);
-		/* Calculate offset before adding up stride index. */
-		offset = strd_idx * strd_sz + strd_shift;
-		strd_idx += consumed_strd;
+		strd_cnt = (byte_cnt & MLX5_MPRQ_STRIDE_NUM_MASK) >>
+			   MLX5_MPRQ_STRIDE_NUM_SHIFT;
+		assert(strd_cnt);
+		consumed_strd += strd_cnt;
 		if (byte_cnt & MLX5_MPRQ_FILLER_MASK)
 			continue;
+		if (mcqe == NULL) {
+			rss_hash_res = rte_be_to_cpu_32(cqe->rx_hash_res);
+			strd_idx = rte_be_to_cpu_16(cqe->wqe_counter);
+		} else {
+			/* mini-CQE for MPRQ doesn't have hash result. */
+			strd_idx = rte_be_to_cpu_16(mcqe->stride_idx);
+		}
+		assert(strd_idx < strd_n);
+		assert(!((rte_be_to_cpu_16(cqe->wqe_id) ^ rq_ci) & wq_mask));
 		/*
 		 * Currently configured to receive a packet per a stride. But if
 		 * MTU is adjusted through kernel interface, device could
@@ -2166,7 +2169,7 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * case, the packet should be dropped because it is bigger than
 		 * the max_rx_pkt_len.
 		 */
-		if (unlikely(consumed_strd > 1)) {
+		if (unlikely(strd_cnt > 1)) {
 			++rxq->stats.idropped;
 			continue;
 		}
@@ -2179,6 +2182,7 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		assert((int)len >= (rxq->crc_present << 2));
 		if (rxq->crc_present)
 			len -= ETHER_CRC_LEN;
+		offset = strd_idx * strd_sz + strd_shift;
 		addr = RTE_PTR_ADD(mlx5_mprq_buf_addr(buf), offset);
 		/* Initialize the offload flag. */
 		pkt->ol_flags = 0;
@@ -2201,7 +2205,7 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		} else {
 			rte_iova_t buf_iova;
 			struct rte_mbuf_ext_shared_info *shinfo;
-			uint16_t buf_len = consumed_strd * strd_sz;
+			uint16_t buf_len = strd_cnt * strd_sz;
 
 			/* Increment the refcnt of the whole chunk. */
 			rte_atomic16_add_return(&buf->refcnt, 1);
@@ -2250,7 +2254,7 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		++i;
 	}
 	/* Update the consumer indexes. */
-	rxq->strd_ci = strd_idx;
+	rxq->consumed_strd = consumed_strd;
 	rte_cio_wmb();
 	*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
 	if (rq_ci != rxq->rq_ci) {
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index f53bb43c..48ed2b20 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -26,6 +26,8 @@
 #include <rte_common.h>
 #include <rte_hexdump.h>
 #include <rte_atomic.h>
+#include <rte_spinlock.h>
+#include <rte_io.h>
 
 #include "mlx5_utils.h"
 #include "mlx5.h"
@@ -34,6 +36,9 @@
 #include "mlx5_defs.h"
 #include "mlx5_prm.h"
 
+/* Support tunnel matching. */
+#define MLX5_FLOW_TUNNEL 5
+
 struct mlx5_rxq_stats {
 	unsigned int idx; /**< Mapping index. */
 #ifdef MLX5_PMD_SOFT_COUNTERS
@@ -93,7 +98,7 @@ struct mlx5_rxq_data {
 	volatile uint32_t *cq_db;
 	uint16_t port_id;
 	uint16_t rq_ci;
-	uint16_t strd_ci; /* Stride index in a WQE for Multi-Packet RQ. */
+	uint16_t consumed_strd; /* Number of consumed strides in WQE. */
 	uint16_t rq_pi;
 	uint16_t cq_ci;
 	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
@@ -115,6 +120,10 @@ struct mlx5_rxq_data {
 	void *cq_uar; /* CQ user access region. */
 	uint32_t cqn; /* CQ number. */
 	uint8_t cq_arm_sn; /* CQ arm seq number. */
+#ifndef RTE_ARCH_64
+	rte_spinlock_t *uar_lock_cq;
+	/* CQ (UAR) access lock required for 32bit implementations */
+#endif
 	uint32_t tunnel; /* Tunnel information. */
 } __rte_cache_aligned;
 
@@ -136,9 +145,10 @@ struct mlx5_rxq_ctrl {
 	struct priv *priv; /* Back pointer to private data. */
 	struct mlx5_rxq_data rxq; /* Data path structure. */
 	unsigned int socket; /* CPU socket ID for allocations. */
-	uint32_t tunnel_types[16]; /* Tunnel type counter. */
 	unsigned int irq:1; /* Whether IRQ is enabled. */
 	uint16_t idx; /* Queue index. */
+	uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */
+	uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */
 };
 
 /* Indirection table. */
@@ -157,8 +167,6 @@ struct mlx5_hrxq {
 	struct mlx5_ind_table_ibv *ind_table; /* Indirection table. */
 	struct ibv_qp *qp; /* Verbs queue pair. */
 	uint64_t hash_fields; /* Verbs Hash fields. */
-	uint32_t tunnel; /* Tunnel type. */
-	uint32_t rss_level; /* RSS on tunnel level. */
 	uint32_t rss_key_len; /* Hash key length in bytes. */
 	uint8_t rss_key[]; /* Hash key. */
 };
@@ -196,6 +204,10 @@ struct mlx5_txq_data {
 	volatile void *bf_reg; /* Blueflame register remapped. */
 	struct rte_mbuf *(*elts)[]; /* TX elements. */
 	struct mlx5_txq_stats stats; /* TX queue counters. */
+#ifndef RTE_ARCH_64
+	rte_spinlock_t *uar_lock;
+	/* UAR access lock required for 32bit implementations */
+#endif
 } __rte_cache_aligned;
 
 /* Verbs Rx queue elements. */
@@ -225,7 +237,6 @@ struct mlx5_txq_ctrl {
 /* mlx5_rxq.c */
 
 extern uint8_t rss_hash_default_key[];
-extern const size_t rss_hash_default_key_len;
 
 int mlx5_check_mprq_support(struct rte_eth_dev *dev);
 int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq);
@@ -245,6 +256,8 @@ struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
 int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv);
 int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv);
+struct mlx5_rxq_ibv *mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev);
 int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev);
 struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
 				   uint16_t desc, unsigned int socket,
@@ -265,18 +278,21 @@ struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev,
 int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
 			       struct mlx5_ind_table_ibv *ind_tbl);
 int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev);
 struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
 				const uint16_t *queues, uint32_t queues_n,
-				uint32_t tunnel, uint32_t rss_level);
+				int tunnel __rte_unused);
 struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
-				const uint16_t *queues, uint32_t queues_n,
-				uint32_t tunnel, uint32_t rss_level);
+				const uint16_t *queues, uint32_t queues_n);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
 int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
+void mlx5_hrxq_drop_release(struct rte_eth_dev *dev);
 uint64_t mlx5_get_rx_port_offloads(void);
 uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
 
@@ -348,6 +364,63 @@ void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
 uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr);
 uint32_t mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr);
 
+/**
+ * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
+ * 64bit architectures.
+ *
+ * @param val
+ *   value to write in CPU endian format.
+ * @param addr
+ *   Address to write to.
+ * @param lock
+ *   Address of the lock to use for that UAR access.
+ */
+static __rte_always_inline void
+__mlx5_uar_write64_relaxed(uint64_t val, volatile void *addr,
+			   rte_spinlock_t *lock __rte_unused)
+{
+#ifdef RTE_ARCH_64
+	rte_write64_relaxed(val, addr);
+#else /* !RTE_ARCH_64 */
+	rte_spinlock_lock(lock);
+	rte_write32_relaxed(val, addr);
+	rte_io_wmb();
+	rte_write32_relaxed(val >> 32,
+			    (volatile void *)((volatile char *)addr + 4));
+	rte_spinlock_unlock(lock);
+#endif
+}
+
+/**
+ * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
+ * 64bit architectures while guaranteeing the order of execution with the
+ * code being executed.
+ *
+ * @param val
+ *   value to write in CPU endian format.
+ * @param addr
+ *   Address to write to.
+ * @param lock
+ *   Address of the lock to use for that UAR access.
+ */
+static __rte_always_inline void
+__mlx5_uar_write64(uint64_t val, volatile void *addr, rte_spinlock_t *lock)
+{
+	rte_io_wmb();
+	__mlx5_uar_write64_relaxed(val, addr, lock);
+}
+
+/* Assist macros, used instead of directly calling the functions they wrap. */
+#ifdef RTE_ARCH_64
+#define mlx5_uar_write64_relaxed(val, dst, lock) \
+		__mlx5_uar_write64_relaxed(val, dst, NULL)
+#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, NULL)
+#else
+#define mlx5_uar_write64_relaxed(val, dst, lock) \
+		__mlx5_uar_write64_relaxed(val, dst, lock)
+#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
+#endif
+
 #ifndef NDEBUG
 /**
  * Verify or set magic value in CQE.
@@ -362,7 +435,7 @@ static inline int
 check_cqe_seen(volatile struct mlx5_cqe *cqe)
 {
 	static const uint8_t magic[] = "seen";
-	volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0;
+	volatile uint8_t (*buf)[sizeof(cqe->rsvd1)] = &cqe->rsvd1;
 	int ret = 1;
 	unsigned int i;
 
@@ -614,7 +687,7 @@ mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe,
 	*txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
 	/* Ensure ordering between DB record and BF copy. */
 	rte_wmb();
-	*dst = *src;
+	mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock);
 	if (cond)
 		rte_wmb();
 }
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h
index 598dc751..fb884f92 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.h
@@ -91,9 +91,9 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n)
 		&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[elts_idx];
 	unsigned int i;
 
-	assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH);
+	assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n));
 	assert(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi)));
-	assert(MLX5_VPMD_RXQ_RPLNSH_THRESH > MLX5_VPMD_DESCS_PER_LOOP);
+	assert(MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n) > MLX5_VPMD_DESCS_PER_LOOP);
 	/* Not to cross queue end. */
 	n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx);
 	if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) {
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
index 71a5eaf2..b37b7381 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
@@ -107,8 +107,6 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 
 	assert(elts_n > pkts_n);
 	mlx5_tx_complete(txq);
-	/* A CQE slot must always be available. */
-	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	if (unlikely(!pkts_n))
 		return 0;
 	for (n = 0; n < pkts_n; ++n) {
@@ -176,12 +174,11 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 	txq->elts_comp += (uint16_t)(elts_head - txq->elts_head);
 	txq->elts_head = elts_head;
 	if (txq->elts_comp >= MLX5_TX_COMP_THRESH) {
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
 		wqe->ctrl[2] = rte_cpu_to_be_32(8);
 		wqe->ctrl[3] = txq->elts_head;
 		txq->elts_comp = 0;
-#ifndef NDEBUG
-		++txq->cq_pi;
-#endif
 	}
 #ifdef MLX5_PMD_SOFT_COUNTERS
 	txq->stats.opackets += n;
@@ -245,8 +242,6 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
 	assert(elts_n > pkts_n);
 	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
-	/* A CQE slot must always be available. */
-	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
 	if (unlikely(!pkts_n))
@@ -282,11 +277,10 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
 	if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) {
 		txq->elts_comp += pkts_n;
 	} else {
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
 		/* Request a completion. */
 		txq->elts_comp = 0;
-#ifndef NDEBUG
-		++txq->cq_pi;
-#endif
 		comp_req = 8;
 	}
 	/* Fill CTRL in the header. */
@@ -739,7 +733,7 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
 	 *   N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished).
 	 */
 	repl_n = q_n - (rxq->rq_ci - rxq->rq_pi);
-	if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH)
+	if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n))
 		mlx5_rx_replenish_bulk_mbuf(rxq, repl_n);
 	/* See if there're unreturned mbufs from compressed CQE. */
 	rcvd_pkt = rxq->cq_ci - rxq->rq_pi;
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
index 3e985d61..54b3783c 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
@@ -107,8 +107,6 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 
 	assert(elts_n > pkts_n);
 	mlx5_tx_complete(txq);
-	/* A CQE slot must always be available. */
-	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	if (unlikely(!pkts_n))
 		return 0;
 	for (n = 0; n < pkts_n; ++n) {
@@ -177,12 +175,11 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 	txq->elts_comp += (uint16_t)(elts_head - txq->elts_head);
 	txq->elts_head = elts_head;
 	if (txq->elts_comp >= MLX5_TX_COMP_THRESH) {
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
 		wqe->ctrl[2] = rte_cpu_to_be_32(8);
 		wqe->ctrl[3] = txq->elts_head;
 		txq->elts_comp = 0;
-#ifndef NDEBUG
-		++txq->cq_pi;
-#endif
 	}
 #ifdef MLX5_PMD_SOFT_COUNTERS
 	txq->stats.opackets += n;
@@ -244,8 +241,6 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
 	assert(elts_n > pkts_n);
 	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
-	/* A CQE slot must always be available. */
-	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
 	assert(pkts_n <= MLX5_DSEG_MAX - nb_dword_in_hdr);
@@ -283,11 +278,10 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
 	if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) {
 		txq->elts_comp += pkts_n;
 	} else {
+		/* A CQE slot must always be available. */
+		assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci));
 		/* Request a completion. */
 		txq->elts_comp = 0;
-#ifndef NDEBUG
-		++txq->cq_pi;
-#endif
 		comp_req = 8;
 	}
 	/* Fill CTRL in the header. */
@@ -724,7 +718,7 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
 	 *   N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished).
 	 */
 	repl_n = q_n - (rxq->rq_ci - rxq->rq_pi);
-	if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH)
+	if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n))
 		mlx5_rx_replenish_bulk_mbuf(rxq, repl_n);
 	/* See if there're unreturned mbufs from compressed CQE. */
 	rcvd_pkt = rxq->cq_ci - rxq->rq_pi;
diff --git a/drivers/net/mlx5/mlx5_socket.c b/drivers/net/mlx5/mlx5_socket.c
index 99297d5c..a3a52291 100644
--- a/drivers/net/mlx5/mlx5_socket.c
+++ b/drivers/net/mlx5/mlx5_socket.c
@@ -36,6 +36,12 @@ mlx5_socket_init(struct rte_eth_dev *dev)
 	int flags;
 
 	/*
+	 * Close the last socket that was used to communicate
+	 * with the secondary process
+	 */
+	if (priv->primary_socket)
+		mlx5_socket_uninit(dev);
+	/*
 	 * Initialise the socket to communicate with the secondary
 	 * process.
 	 */
diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c
index 875dd102..91f3d474 100644
--- a/drivers/net/mlx5/mlx5_stats.c
+++ b/drivers/net/mlx5/mlx5_stats.c
@@ -146,7 +146,7 @@ mlx5_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
 	et_stats->cmd = ETHTOOL_GSTATS;
 	et_stats->n_stats = xstats_ctrl->stats_n;
 	ifr.ifr_data = (caddr_t)et_stats;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
 	if (ret) {
 		DRV_LOG(WARNING,
 			"port %u unable to read statistic values from device",
@@ -194,7 +194,7 @@ mlx5_ethtool_get_stats_n(struct rte_eth_dev *dev) {
 
 	drvinfo.cmd = ETHTOOL_GDRVINFO;
 	ifr.ifr_data = (caddr_t)&drvinfo;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
 	if (ret) {
 		DRV_LOG(WARNING, "port %u unable to query number of statistics",
 			dev->data->port_id);
@@ -244,7 +244,7 @@ mlx5_xstats_init(struct rte_eth_dev *dev)
 	strings->string_set = ETH_SS_STATS;
 	strings->len = dev_stats_n;
 	ifr.ifr_data = (caddr_t)strings;
-	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
+	ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
 	if (ret) {
 		DRV_LOG(WARNING, "port %u unable to get statistic names",
 			dev->data->port_id);
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 3e7c0a90..e2a9bb70 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -46,7 +46,6 @@ mlx5_txq_start(struct rte_eth_dev *dev)
 	unsigned int i;
 	int ret;
 
-	/* Add memory regions to Tx queues. */
 	for (i = 0; i != priv->txqs_n; ++i) {
 		struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i);
 
@@ -60,12 +59,17 @@ mlx5_txq_start(struct rte_eth_dev *dev)
 		}
 	}
 	ret = mlx5_tx_uar_remap(dev, priv->ctx->cmd_fd);
-	if (ret)
+	if (ret) {
+		/* Adjust index for rollback. */
+		i = priv->txqs_n - 1;
 		goto error;
+	}
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_txq_stop(dev);
+	do {
+		mlx5_txq_release(dev, i);
+	} while (i-- != 0);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -103,8 +107,10 @@ mlx5_rxq_start(struct rte_eth_dev *dev)
 	int ret = 0;
 
 	/* Allocate/reuse/resize mempool for Multi-Packet RQ. */
-	if (mlx5_mprq_alloc_mp(dev))
-		goto error;
+	if (mlx5_mprq_alloc_mp(dev)) {
+		/* Should not release Rx queues but return immediately. */
+		return -rte_errno;
+	}
 	for (i = 0; i != priv->rxqs_n; ++i) {
 		struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i);
 		struct rte_mempool *mp;
@@ -130,7 +136,9 @@ mlx5_rxq_start(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_rxq_stop(dev);
+	do {
+		mlx5_rxq_release(dev, i);
+	} while (i-- != 0);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -152,23 +160,21 @@ mlx5_dev_start(struct rte_eth_dev *dev)
 	struct priv *priv = dev->data->dev_private;
 	int ret;
 
-	dev->data->dev_started = 1;
-	DRV_LOG(DEBUG, "port %u allocating and configuring hash Rx queues",
-		dev->data->port_id);
+	DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id);
 	ret = mlx5_txq_start(dev);
 	if (ret) {
 		DRV_LOG(ERR, "port %u Tx queue allocation failed: %s",
 			dev->data->port_id, strerror(rte_errno));
-		goto error;
+		return -rte_errno;
 	}
 	ret = mlx5_rxq_start(dev);
 	if (ret) {
 		DRV_LOG(ERR, "port %u Rx queue allocation failed: %s",
 			dev->data->port_id, strerror(rte_errno));
-		goto error;
+		mlx5_txq_stop(dev);
+		return -rte_errno;
 	}
-	if (rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
-		mlx5_mr_dump_dev(dev);
+	dev->data->dev_started = 1;
 	ret = mlx5_rx_intr_vec_enable(dev);
 	if (ret) {
 		DRV_LOG(ERR, "port %u Rx interrupt vector creation failed",
@@ -223,8 +229,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	dev->tx_pkt_burst = removed_tx_burst;
 	rte_wmb();
 	usleep(1000 * priv->rxqs_n);
-	DRV_LOG(DEBUG, "port %u cleaning up and destroying hash Rx queues",
-		dev->data->port_id);
+	DRV_LOG(DEBUG, "port %u stopping device", dev->data->port_id);
 	mlx5_flow_stop(dev, &priv->flows);
 	mlx5_traffic_disable(dev);
 	mlx5_rx_intr_vec_disable(dev);
@@ -302,9 +307,8 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 			struct rte_flow_item_vlan vlan_spec = {
 				.tci = rte_cpu_to_be_16(vlan),
 			};
-			struct rte_flow_item_vlan vlan_mask = {
-				.tci = 0xffff,
-			};
+			struct rte_flow_item_vlan vlan_mask =
+				rte_flow_item_vlan_mask;
 
 			ret = mlx5_ctrl_flow_vlan(dev, &bcast, &bcast,
 						  &vlan_spec, &vlan_mask);
@@ -341,9 +345,8 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 			struct rte_flow_item_vlan vlan_spec = {
 				.tci = rte_cpu_to_be_16(vlan),
 			};
-			struct rte_flow_item_vlan vlan_mask = {
-				.tci = 0xffff,
-			};
+			struct rte_flow_item_vlan vlan_mask =
+				rte_flow_item_vlan_mask;
 
 			ret = mlx5_ctrl_flow_vlan(dev, &unicast,
 						  &unicast_mask,
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 691ea071..f9bc4739 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -113,15 +113,20 @@ mlx5_get_tx_port_offloads(struct rte_eth_dev *dev)
 			     DEV_TX_OFFLOAD_TCP_CKSUM);
 	if (config->tso)
 		offloads |= DEV_TX_OFFLOAD_TCP_TSO;
+	if (config->swp) {
+		if (config->hw_csum)
+			offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
+		if (config->tso)
+			offloads |= (DEV_TX_OFFLOAD_IP_TNL_TSO |
+				     DEV_TX_OFFLOAD_UDP_TNL_TSO);
+	}
+
 	if (config->tunnel_en) {
 		if (config->hw_csum)
 			offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
 		if (config->tso)
 			offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
 				     DEV_TX_OFFLOAD_GRE_TNL_TSO);
-		if (config->swp)
-			offloads |= (DEV_TX_OFFLOAD_IP_TNL_TSO |
-				     DEV_TX_OFFLOAD_UDP_TNL_TSO);
 	}
 	return offloads;
 }
@@ -250,6 +255,9 @@ mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd)
 	struct mlx5_txq_ctrl *txq_ctrl;
 	int already_mapped;
 	size_t page_size = sysconf(_SC_PAGESIZE);
+#ifndef RTE_ARCH_64
+	unsigned int lock_idx;
+#endif
 
 	memset(pages, 0, priv->txqs_n * sizeof(uintptr_t));
 	/*
@@ -276,7 +284,7 @@ mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd)
 		}
 		/* new address in reserved UAR address space. */
 		addr = RTE_PTR_ADD(priv->uar_base,
-				   uar_va & (MLX5_UAR_SIZE - 1));
+				   uar_va & (uintptr_t)(MLX5_UAR_SIZE - 1));
 		if (!already_mapped) {
 			pages[pages_n++] = uar_va;
 			/* fixed mmap to specified address in reserved
@@ -300,6 +308,12 @@ mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd)
 		else
 			assert(txq_ctrl->txq.bf_reg ==
 			       RTE_PTR_ADD((void *)addr, off));
+#ifndef RTE_ARCH_64
+		/* Assign a UAR lock according to UAR page number */
+		lock_idx = (txq_ctrl->uar_mmap_offset / page_size) &
+			   MLX5_UAR_PAGE_NUM_MASK;
+		txq->uar_lock = &priv->uar_lock[lock_idx];
+#endif
 	}
 	return 0;
 }
@@ -429,7 +443,7 @@ mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
 		/* Move the QP to this state. */
 		.qp_state = IBV_QPS_INIT,
 		/* Primary port number. */
-		.port_num = priv->port
+		.port_num = 1,
 	};
 	ret = mlx5_glue->modify_qp(tmpl.qp, &attr.mod,
 				   (IBV_QP_STATE | IBV_QP_PORT));
@@ -506,6 +520,8 @@ mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
 	rte_atomic32_inc(&txq_ibv->refcnt);
 	if (qp.comp_mask & MLX5DV_QP_MASK_UAR_MMAP_OFFSET) {
 		txq_ctrl->uar_mmap_offset = qp.uar_mmap_offset;
+		DRV_LOG(DEBUG, "port %u: uar_mmap_offset 0x%lx",
+			dev->data->port_id, txq_ctrl->uar_mmap_offset);
 	} else {
 		DRV_LOG(ERR,
 			"port %u failed to retrieve UAR info, invalid"
@@ -514,8 +530,6 @@ mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
 		rte_errno = EINVAL;
 		goto error;
 	}
-	DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d",
-		dev->data->port_id, idx, rte_atomic32_read(&txq_ibv->refcnt));
 	LIST_INSERT_HEAD(&priv->txqsibv, txq_ibv, next);
 	txq_ibv->txq_ctrl = txq_ctrl;
 	priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
@@ -553,12 +567,8 @@ mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
 	if (!(*priv->txqs)[idx])
 		return NULL;
 	txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
-	if (txq_ctrl->ibv) {
+	if (txq_ctrl->ibv)
 		rte_atomic32_inc(&txq_ctrl->ibv->refcnt);
-		DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d",
-			dev->data->port_id, txq_ctrl->idx,
-		      rte_atomic32_read(&txq_ctrl->ibv->refcnt));
-	}
 	return txq_ctrl->ibv;
 }
 
@@ -575,9 +585,6 @@ int
 mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv)
 {
 	assert(txq_ibv);
-	DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d",
-		PORT_ID(txq_ibv->txq_ctrl->priv),
-		txq_ibv->txq_ctrl->idx, rte_atomic32_read(&txq_ibv->refcnt));
 	if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) {
 		claim_zero(mlx5_glue->destroy_qp(txq_ibv->qp));
 		claim_zero(mlx5_glue->destroy_cq(txq_ibv->cq));
@@ -716,7 +723,7 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
 						   max_tso_inline);
 		txq_ctrl->txq.tso_en = 1;
 	}
-	txq_ctrl->txq.tunnel_en = config->tunnel_en;
+	txq_ctrl->txq.tunnel_en = config->tunnel_en | config->swp;
 	txq_ctrl->txq.swp_en = ((DEV_TX_OFFLOAD_IP_TNL_TSO |
 				 DEV_TX_OFFLOAD_UDP_TNL_TSO |
 				 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) &
@@ -778,8 +785,6 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		(struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])(tmpl + 1);
 	tmpl->txq.stats.idx = idx;
 	rte_atomic32_inc(&tmpl->refcnt);
-	DRV_LOG(DEBUG, "port %u Tx queue %u: refcnt %d", dev->data->port_id,
-		idx, rte_atomic32_read(&tmpl->refcnt));
 	LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next);
 	return tmpl;
 error:
@@ -809,9 +814,6 @@ mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx)
 				    txq);
 		mlx5_txq_ibv_get(dev, idx);
 		rte_atomic32_inc(&ctrl->refcnt);
-		DRV_LOG(DEBUG, "port %u Tx queue %u refcnt %d",
-			dev->data->port_id,
-			ctrl->idx, rte_atomic32_read(&ctrl->refcnt));
 	}
 	return ctrl;
 }
@@ -837,8 +839,6 @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
 	if (!(*priv->txqs)[idx])
 		return 0;
 	txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
-	DRV_LOG(DEBUG, "port %u Tx queue %u: refcnt %d", dev->data->port_id,
-		txq->idx, rte_atomic32_read(&txq->refcnt));
 	if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv))
 		txq->ibv = NULL;
 	if (priv->uar_base)
diff --git a/drivers/net/mvpp2/mrvl_ethdev.c b/drivers/net/mvpp2/mrvl_ethdev.c
index ea6a7864..a2d0576e 100644
--- a/drivers/net/mvpp2/mrvl_ethdev.c
+++ b/drivers/net/mvpp2/mrvl_ethdev.c
@@ -94,6 +94,8 @@ struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 
+int mrvl_logtype;
+
 struct mrvl_ifnames {
 	const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
 	int idx;
@@ -206,7 +208,7 @@ mrvl_init_hif(int core_id)
 
 	ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 	if (ret < 0) {
-		RTE_LOG(ERR, PMD, "Failed to allocate hif %d\n", core_id);
+		MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 		return ret;
 	}
 
@@ -216,7 +218,7 @@ mrvl_init_hif(int core_id)
 	params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 	ret = pp2_hif_init(&params, &hifs[core_id]);
 	if (ret) {
-		RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", core_id);
+		MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 		return ret;
 	}
 
@@ -235,7 +237,7 @@ mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 
 	ret = mrvl_init_hif(core_id);
 	if (ret < 0) {
-		RTE_LOG(ERR, PMD, "Failed to allocate hif %d\n", core_id);
+		MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 		goto out;
 	}
 
@@ -265,7 +267,7 @@ static int
 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 {
 	if (rss_conf->rss_key)
-		RTE_LOG(WARNING, PMD, "Changing hash key is not supported\n");
+		MRVL_LOG(WARNING, "Changing hash key is not supported");
 
 	if (rss_conf->rss_hf == 0) {
 		priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
@@ -307,19 +309,21 @@ mrvl_dev_configure(struct rte_eth_dev *dev)
 
 	if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 	    dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
-		RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n",
+		MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 			dev->data->dev_conf.rxmode.mq_mode);
 		return -EINVAL;
 	}
 
-	if (!(dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP)) {
-		RTE_LOG(INFO, PMD,
-			"L2 CRC stripping is always enabled in hw\n");
+	/* KEEP_CRC offload flag is not supported by PMD
+	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
+	 */
+	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads)) {
+		MRVL_LOG(INFO, "L2 CRC stripping is always enabled in hw");
 		dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
 	}
 
 	if (dev->data->dev_conf.rxmode.split_hdr_size) {
-		RTE_LOG(INFO, PMD, "Split headers not supported\n");
+		MRVL_LOG(INFO, "Split headers not supported");
 		return -EINVAL;
 	}
 
@@ -343,7 +347,7 @@ mrvl_dev_configure(struct rte_eth_dev *dev)
 
 	if (dev->data->nb_rx_queues == 1 &&
 	    dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
-		RTE_LOG(WARNING, PMD, "Disabling hash for 1 rx queue\n");
+		MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 		priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 
 		return 0;
@@ -467,7 +471,7 @@ mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 	/* passing 1 enables given tx queue */
 	ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 	if (ret) {
-		RTE_LOG(ERR, PMD, "Failed to start txq %d\n", queue_id);
+		MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 		return ret;
 	}
 
@@ -499,7 +503,7 @@ mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 	/* passing 0 disables given tx queue */
 	ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 	if (ret) {
-		RTE_LOG(ERR, PMD, "Failed to stop txq %d\n", queue_id);
+		MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 		return ret;
 	}
 
@@ -546,7 +550,7 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 		priv->bpool_init_size += buffs_to_add;
 		ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 		if (ret)
-			RTE_LOG(ERR, PMD, "Failed to add buffers to bpool\n");
+			MRVL_LOG(ERR, "Failed to add buffers to bpool");
 	}
 
 	/*
@@ -561,7 +565,7 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 
 	ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 	if (ret) {
-		RTE_LOG(ERR, PMD, "Failed to init ppio\n");
+		MRVL_LOG(ERR, "Failed to init ppio");
 		return ret;
 	}
 
@@ -574,8 +578,8 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 	if (!priv->uc_mc_flushed) {
 		ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 		if (ret) {
-			RTE_LOG(ERR, PMD,
-				"Failed to flush uc/mc filter list\n");
+			MRVL_LOG(ERR,
+				"Failed to flush uc/mc filter list");
 			goto out;
 		}
 		priv->uc_mc_flushed = 1;
@@ -584,7 +588,7 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 	if (!priv->vlan_flushed) {
 		ret = pp2_ppio_flush_vlan(priv->ppio);
 		if (ret) {
-			RTE_LOG(ERR, PMD, "Failed to flush vlan list\n");
+			MRVL_LOG(ERR, "Failed to flush vlan list");
 			/*
 			 * TODO
 			 * once pp2_ppio_flush_vlan() is supported jump to out
@@ -598,14 +602,14 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 	if (mrvl_qos_cfg) {
 		ret = mrvl_start_qos_mapping(priv);
 		if (ret) {
-			RTE_LOG(ERR, PMD, "Failed to setup QoS mapping\n");
+			MRVL_LOG(ERR, "Failed to setup QoS mapping");
 			goto out;
 		}
 	}
 
 	ret = mrvl_dev_set_link_up(dev);
 	if (ret) {
-		RTE_LOG(ERR, PMD, "Failed to set link up\n");
+		MRVL_LOG(ERR, "Failed to set link up");
 		goto out;
 	}
 
@@ -629,7 +633,7 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 
 	return 0;
 out:
-	RTE_LOG(ERR, PMD, "Failed to start device\n");
+	MRVL_LOG(ERR, "Failed to start device");
 	pp2_ppio_deinit(priv->ppio);
 	return ret;
 }
@@ -645,7 +649,7 @@ mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 {
 	int i;
 
-	RTE_LOG(INFO, PMD, "Flushing rx queues\n");
+	MRVL_LOG(INFO, "Flushing rx queues");
 	for (i = 0; i < dev->data->nb_rx_queues; i++) {
 		int ret, num;
 
@@ -674,7 +678,7 @@ mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 	int i, j;
 	struct mrvl_txq *txq;
 
-	RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n");
+	MRVL_LOG(INFO, "Flushing tx shadow queues");
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		txq = (struct mrvl_txq *)dev->data->tx_queues[i];
 
@@ -722,7 +726,7 @@ mrvl_flush_bpool(struct rte_eth_dev *dev)
 
 	ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 	if (ret) {
-		RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n");
+		MRVL_LOG(ERR, "Failed to get bpool buffers number");
 		return;
 	}
 
@@ -887,7 +891,7 @@ mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 
 	ret = pp2_ppio_set_promisc(priv->ppio, 1);
 	if (ret)
-		RTE_LOG(ERR, PMD, "Failed to enable promiscuous mode\n");
+		MRVL_LOG(ERR, "Failed to enable promiscuous mode");
 }
 
 /**
@@ -910,7 +914,7 @@ mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 
 	ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 	if (ret)
-		RTE_LOG(ERR, PMD, "Failed enable all-multicast mode\n");
+		MRVL_LOG(ERR, "Failed enable all-multicast mode");
 }
 
 /**
@@ -930,7 +934,7 @@ mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 
 	ret = pp2_ppio_set_promisc(priv->ppio, 0);
 	if (ret)
-		RTE_LOG(ERR, PMD, "Failed to disable promiscuous mode\n");
+		MRVL_LOG(ERR, "Failed to disable promiscuous mode");
 }
 
 /**
@@ -950,7 +954,7 @@ mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 
 	ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 	if (ret)
-		RTE_LOG(ERR, PMD, "Failed to disable all-multicast mode\n");
+		MRVL_LOG(ERR, "Failed to disable all-multicast mode");
 }
 
 /**
@@ -979,7 +983,7 @@ mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 	if (ret) {
 		ether_format_addr(buf, sizeof(buf),
 				  &dev->data->mac_addrs[index]);
-		RTE_LOG(ERR, PMD, "Failed to remove mac %s\n", buf);
+		MRVL_LOG(ERR, "Failed to remove mac %s", buf);
 	}
 }
 
@@ -1032,7 +1036,7 @@ mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 	ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
 	if (ret) {
 		ether_format_addr(buf, sizeof(buf), mac_addr);
-		RTE_LOG(ERR, PMD, "Failed to add mac %s\n", buf);
+		MRVL_LOG(ERR, "Failed to add mac %s", buf);
 		return -1;
 	}
 
@@ -1066,7 +1070,7 @@ mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 	if (ret) {
 		char buf[ETHER_ADDR_FMT_SIZE];
 		ether_format_addr(buf, sizeof(buf), mac_addr);
-		RTE_LOG(ERR, PMD, "Failed to set mac to %s\n", buf);
+		MRVL_LOG(ERR, "Failed to set mac to %s", buf);
 	}
 
 	return ret;
@@ -1103,8 +1107,8 @@ mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 
 		idx = rxq->queue_id;
 		if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
-			RTE_LOG(ERR, PMD,
-				"rx queue %d stats out of range (0 - %d)\n",
+			MRVL_LOG(ERR,
+				"rx queue %d stats out of range (0 - %d)",
 				idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
 			continue;
 		}
@@ -1114,8 +1118,8 @@ mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 						  priv->rxq_map[idx].inq,
 						  &rx_stats, 0);
 		if (unlikely(ret)) {
-			RTE_LOG(ERR, PMD,
-				"Failed to update rx queue %d stats\n", idx);
+			MRVL_LOG(ERR,
+				"Failed to update rx queue %d stats", idx);
 			break;
 		}
 
@@ -1138,16 +1142,16 @@ mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 
 		idx = txq->queue_id;
 		if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
-			RTE_LOG(ERR, PMD,
-				"tx queue %d stats out of range (0 - %d)\n",
+			MRVL_LOG(ERR,
+				"tx queue %d stats out of range (0 - %d)",
 				idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
 		}
 
 		ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
 						   &tx_stats, 0);
 		if (unlikely(ret)) {
-			RTE_LOG(ERR, PMD,
-				"Failed to update tx queue %d stats\n", idx);
+			MRVL_LOG(ERR,
+				"Failed to update tx queue %d stats", idx);
 			break;
 		}
 
@@ -1158,7 +1162,7 @@ mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 
 	ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
 	if (unlikely(ret)) {
-		RTE_LOG(ERR, PMD, "Failed to update port statistics\n");
+		MRVL_LOG(ERR, "Failed to update port statistics");
 		return ret;
 	}
 
@@ -1480,8 +1484,8 @@ mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
 	for (i = 0; i < num; i++) {
 		if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
 			!= cookie_addr_high) {
-			RTE_LOG(ERR, PMD,
-				"mbuf virtual addr high 0x%lx out of range\n",
+			MRVL_LOG(ERR,
+				"mbuf virtual addr high 0x%lx out of range",
 				(uint64_t)mbufs[i] >> 32);
 			goto out;
 		}
@@ -1544,7 +1548,7 @@ mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		/*
 		 * Unknown TC mapping, mapping will not have a correct queue.
 		 */
-		RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n",
+		MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
 			idx, priv->ppio_id);
 		return -EFAULT;
 	}
@@ -1552,8 +1556,8 @@ mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
 		   MRVL_PKT_EFFEC_OFFS;
 	if (min_size < max_rx_pkt_len) {
-		RTE_LOG(ERR, PMD,
-			"Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n",
+		MRVL_LOG(ERR,
+			"Mbuf size must be increased to %u bytes to hold up to %u bytes of data.",
 			max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
 			MRVL_PKT_EFFEC_OFFS,
 			max_rx_pkt_len);
@@ -1612,9 +1616,12 @@ mrvl_rx_queue_release(void *rxq)
 	if (core_id == LCORE_ID_ANY)
 		core_id = 0;
 
+	if (!q)
+		return;
+
 	hif = mrvl_get_hif(q->priv, core_id);
 
-	if (!q || !hif)
+	if (!hif)
 		return;
 
 	tc = q->priv->rxq_map[q->queue_id].tc;
@@ -1717,7 +1724,7 @@ mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 
 	ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
 	if (ret) {
-		RTE_LOG(ERR, PMD, "Failed to read rx pause state\n");
+		MRVL_LOG(ERR, "Failed to read rx pause state");
 		return ret;
 	}
 
@@ -1750,7 +1757,7 @@ mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 	    fc_conf->pause_time ||
 	    fc_conf->mac_ctrl_frame_fwd ||
 	    fc_conf->autoneg) {
-		RTE_LOG(ERR, PMD, "Flowctrl parameter is not supported\n");
+		MRVL_LOG(ERR, "Flowctrl parameter is not supported");
 
 		return -EINVAL;
 	}
@@ -1762,8 +1769,8 @@ mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 		en = fc_conf->mode == RTE_FC_NONE ? 0 : 1;
 		ret = pp2_ppio_set_rx_pause(priv->ppio, en);
 		if (ret)
-			RTE_LOG(ERR, PMD,
-				"Failed to change flowctrl on RX side\n");
+			MRVL_LOG(ERR,
+				"Failed to change flowctrl on RX side");
 
 		return ret;
 	}
@@ -1854,7 +1861,7 @@ mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
 		*(const void **)arg = &mrvl_flow_ops;
 		return 0;
 	default:
-		RTE_LOG(WARNING, PMD, "Filter type (%d) not supported",
+		MRVL_LOG(WARNING, "Filter type (%d) not supported",
 				filter_type);
 		return -EINVAL;
 	}
@@ -1951,7 +1958,7 @@ mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
 		*l4_offset = *l3_offset + MRVL_ARP_LENGTH;
 		break;
 	default:
-		RTE_LOG(DEBUG, PMD, "Failed to recognise l3 packet type\n");
+		MRVL_LOG(DEBUG, "Failed to recognise l3 packet type");
 		break;
 	}
 
@@ -1963,7 +1970,7 @@ mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
 		packet_type |= RTE_PTYPE_L4_UDP;
 		break;
 	default:
-		RTE_LOG(DEBUG, PMD, "Failed to recognise l4 packet type\n");
+		MRVL_LOG(DEBUG, "Failed to recognise l4 packet type");
 		break;
 	}
 
@@ -2034,7 +2041,7 @@ mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
 			    q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
 	if (unlikely(ret < 0)) {
-		RTE_LOG(ERR, PMD, "Failed to receive packets\n");
+		MRVL_LOG(ERR, "Failed to receive packets");
 		return 0;
 	}
 	mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
@@ -2101,15 +2108,15 @@ mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 			     (!rx_done && num < q->priv->bpool_init_size))) {
 			ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
 			if (ret)
-				RTE_LOG(ERR, PMD, "Failed to fill bpool\n");
+				MRVL_LOG(ERR, "Failed to fill bpool");
 		} else if (unlikely(num > q->priv->bpool_max_size)) {
 			int i;
 			int pkt_to_remove = num - q->priv->bpool_init_size;
 			struct rte_mbuf *mbuf;
 			struct pp2_buff_inf buff;
 
-			RTE_LOG(DEBUG, PMD,
-				"\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n",
+			MRVL_LOG(DEBUG,
+				"port-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)",
 				bpool->pp2_id, q->priv->ppio->port_id,
 				bpool->id, pkt_to_remove, num,
 				q->priv->bpool_init_size);
@@ -2229,8 +2236,8 @@ mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
 	for (i = 0; i < nb_done; i++) {
 		entry = &sq->ent[sq->tail + num];
 		if (unlikely(!entry->buff.addr)) {
-			RTE_LOG(ERR, PMD,
-				"Shadow memory @%d: cookie(%lx), pa(%lx)!\n",
+			MRVL_LOG(ERR,
+				"Shadow memory @%d: cookie(%lx), pa(%lx)!",
 				sq->tail, (u64)entry->buff.cookie,
 				(u64)entry->buff.addr);
 			skip_bufs = 1;
@@ -2307,8 +2314,8 @@ mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 
 	sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
 	if (unlikely(nb_pkts > sq_free_size)) {
-		RTE_LOG(DEBUG, PMD,
-			"No room in shadow queue for %d packets! %d packets will be sent.\n",
+		MRVL_LOG(DEBUG,
+			"No room in shadow queue for %d packets! %d packets will be sent.",
 			nb_pkts, sq_free_size);
 		nb_pkts = sq_free_size;
 	}
@@ -2494,7 +2501,7 @@ mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
 		rte_zmalloc("mac_addrs",
 			    ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
 	if (!eth_dev->data->mac_addrs) {
-		RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n");
+		MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
 		ret = -ENOMEM;
 		goto out_free_priv;
 	}
@@ -2633,9 +2640,9 @@ rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
 	 */
 	if (!mrvl_qos_cfg) {
 		cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
-		RTE_LOG(INFO, PMD, "Parsing config file!\n");
+		MRVL_LOG(INFO, "Parsing config file!");
 		if (cfgnum > 1) {
-			RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n");
+			MRVL_LOG(ERR, "Cannot handle more than one config file!");
 			goto out_free_kvlist;
 		} else if (cfgnum == 1) {
 			rte_kvargs_process(kvlist, MRVL_CFG_ARG,
@@ -2646,7 +2653,7 @@ rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
 	if (mrvl_dev_num)
 		goto init_devices;
 
-	RTE_LOG(INFO, PMD, "Perform MUSDK initializations\n");
+	MRVL_LOG(INFO, "Perform MUSDK initializations");
 	/*
 	 * ret == -EEXIST is correct, it means DMA
 	 * has been already initialized (by another PMD).
@@ -2656,13 +2663,13 @@ rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
 		if (ret != -EEXIST)
 			goto out_free_kvlist;
 		else
-			RTE_LOG(INFO, PMD,
-				"DMA memory has been already initialized by a different driver.\n");
+			MRVL_LOG(INFO,
+				"DMA memory has been already initialized by a different driver.");
 	}
 
 	ret = mrvl_init_pp2();
 	if (ret) {
-		RTE_LOG(ERR, PMD, "Failed to init PP!\n");
+		MRVL_LOG(ERR, "Failed to init PP!");
 		goto out_deinit_dma;
 	}
 
@@ -2674,7 +2681,7 @@ rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
 
 init_devices:
 	for (i = 0; i < ifnum; i++) {
-		RTE_LOG(INFO, PMD, "Creating %s\n", ifnames.names[i]);
+		MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
 		ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
 		if (ret)
 			goto out_cleanup;
@@ -2718,7 +2725,7 @@ rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
 	if (!name)
 		return -EINVAL;
 
-	RTE_LOG(INFO, PMD, "Removing %s\n", name);
+	MRVL_LOG(INFO, "Removing %s", name);
 
 	RTE_ETH_FOREACH_DEV(i) { /* FIXME: removing all devices! */
 		char ifname[RTE_ETH_NAME_MAX_LEN];
@@ -2729,7 +2736,7 @@ rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
 	}
 
 	if (mrvl_dev_num == 0) {
-		RTE_LOG(INFO, PMD, "Perform MUSDK deinit\n");
+		MRVL_LOG(INFO, "Perform MUSDK deinit");
 		mrvl_deinit_hifs();
 		mrvl_deinit_pp2();
 		mv_sys_dma_mem_destroy();
@@ -2745,3 +2752,10 @@ static struct rte_vdev_driver pmd_mrvl_drv = {
 
 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
+
+RTE_INIT(mrvl_init_log)
+{
+	mrvl_logtype = rte_log_register("pmd.net.mvpp2");
+	if (mrvl_logtype >= 0)
+		rte_log_set_level(mrvl_logtype, RTE_LOG_NOTICE);
+}
diff --git a/drivers/net/mvpp2/mrvl_ethdev.h b/drivers/net/mvpp2/mrvl_ethdev.h
index 3a428092..3726f788 100644
--- a/drivers/net/mvpp2/mrvl_ethdev.h
+++ b/drivers/net/mvpp2/mrvl_ethdev.h
@@ -98,4 +98,12 @@ struct mrvl_priv {
 
 /** Flow operations forward declaration. */
 extern const struct rte_flow_ops mrvl_flow_ops;
+
+/** Current log type. */
+extern int mrvl_logtype;
+
+#define MRVL_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, mrvl_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
 #endif /* _MRVL_ETHDEV_H_ */
diff --git a/drivers/net/mvpp2/mrvl_flow.c b/drivers/net/mvpp2/mrvl_flow.c
index 437c987c..ecc34192 100644
--- a/drivers/net/mvpp2/mrvl_flow.c
+++ b/drivers/net/mvpp2/mrvl_flow.c
@@ -1054,7 +1054,7 @@ mrvl_parse_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 	}
 
 	if (mask->type) {
-		RTE_LOG(WARNING, PMD, "eth type mask is ignored\n");
+		MRVL_LOG(WARNING, "eth type mask is ignored");
 		ret = mrvl_parse_type(spec, mask, flow);
 		if (ret)
 			goto out;
@@ -1093,14 +1093,14 @@ mrvl_parse_vlan(const struct rte_flow_item *item,
 
 	m = rte_be_to_cpu_16(mask->tci);
 	if (m & MRVL_VLAN_ID_MASK) {
-		RTE_LOG(WARNING, PMD, "vlan id mask is ignored\n");
+		MRVL_LOG(WARNING, "vlan id mask is ignored");
 		ret = mrvl_parse_vlan_id(spec, mask, flow);
 		if (ret)
 			goto out;
 	}
 
 	if (m & MRVL_VLAN_PRI_MASK) {
-		RTE_LOG(WARNING, PMD, "vlan pri mask is ignored\n");
+		MRVL_LOG(WARNING, "vlan pri mask is ignored");
 		ret = mrvl_parse_vlan_pri(spec, mask, flow);
 		if (ret)
 			goto out;
@@ -1109,7 +1109,7 @@ mrvl_parse_vlan(const struct rte_flow_item *item,
 	if (flow->pattern & F_TYPE) {
 		rte_flow_error_set(error, ENOTSUP,
 				   RTE_FLOW_ERROR_TYPE_ITEM, item,
-				   "VLAN TPID matching is not supported\n");
+				   "VLAN TPID matching is not supported");
 		return -rte_errno;
 	}
 	if (mask->inner_type) {
@@ -1120,7 +1120,7 @@ mrvl_parse_vlan(const struct rte_flow_item *item,
 			.type = mask->inner_type,
 		};
 
-		RTE_LOG(WARNING, PMD, "inner eth type mask is ignored\n");
+		MRVL_LOG(WARNING, "inner eth type mask is ignored");
 		ret = mrvl_parse_type(&spec_eth, &mask_eth, flow);
 		if (ret)
 			goto out;
@@ -1186,7 +1186,7 @@ mrvl_parse_ip4(const struct rte_flow_item *item,
 	}
 
 	if (mask->hdr.next_proto_id) {
-		RTE_LOG(WARNING, PMD, "next proto id mask is ignored\n");
+		MRVL_LOG(WARNING, "next proto id mask is ignored");
 		ret = mrvl_parse_ip4_proto(spec, mask, flow);
 		if (ret)
 			goto out;
@@ -1257,7 +1257,7 @@ mrvl_parse_ip6(const struct rte_flow_item *item,
 	}
 
 	if (mask->hdr.proto) {
-		RTE_LOG(WARNING, PMD, "next header mask is ignored\n");
+		MRVL_LOG(WARNING, "next header mask is ignored");
 		ret = mrvl_parse_ip6_next_hdr(spec, mask, flow);
 		if (ret)
 			goto out;
@@ -1306,14 +1306,14 @@ mrvl_parse_tcp(const struct rte_flow_item *item,
 	}
 
 	if (mask->hdr.src_port) {
-		RTE_LOG(WARNING, PMD, "tcp sport mask is ignored\n");
+		MRVL_LOG(WARNING, "tcp sport mask is ignored");
 		ret = mrvl_parse_tcp_sport(spec, mask, flow);
 		if (ret)
 			goto out;
 	}
 
 	if (mask->hdr.dst_port) {
-		RTE_LOG(WARNING, PMD, "tcp dport mask is ignored\n");
+		MRVL_LOG(WARNING, "tcp dport mask is ignored");
 		ret = mrvl_parse_tcp_dport(spec, mask, flow);
 		if (ret)
 			goto out;
@@ -1357,14 +1357,14 @@ mrvl_parse_udp(const struct rte_flow_item *item,
 	}
 
 	if (mask->hdr.src_port) {
-		RTE_LOG(WARNING, PMD, "udp sport mask is ignored\n");
+		MRVL_LOG(WARNING, "udp sport mask is ignored");
 		ret = mrvl_parse_udp_sport(spec, mask, flow);
 		if (ret)
 			goto out;
 	}
 
 	if (mask->hdr.dst_port) {
-		RTE_LOG(WARNING, PMD, "udp dport mask is ignored\n");
+		MRVL_LOG(WARNING, "udp dport mask is ignored");
 		ret = mrvl_parse_udp_dport(spec, mask, flow);
 		if (ret)
 			goto out;
@@ -2280,8 +2280,8 @@ mrvl_flow_parse_actions(struct mrvl_priv *priv,
 				 * Unknown TC mapping, mapping will not have
 				 * a correct queue.
 				 */
-				RTE_LOG(ERR, PMD,
-					"Unknown TC mapping for queue %hu eth%hhu\n",
+				MRVL_LOG(ERR,
+					"Unknown TC mapping for queue %hu eth%hhu",
 					q->index, priv->ppio_id);
 
 				rte_flow_error_set(error, EFAULT,
@@ -2290,8 +2290,8 @@ mrvl_flow_parse_actions(struct mrvl_priv *priv,
 				return -rte_errno;
 			}
 
-			RTE_LOG(DEBUG, PMD,
-				"Action: Assign packets to queue %d, tc:%d, q:%d\n",
+			MRVL_LOG(DEBUG,
+				"Action: Assign packets to queue %d, tc:%d, q:%d",
 				q->index, priv->rxq_map[q->index].tc,
 				priv->rxq_map[q->index].inq);
 
@@ -2384,7 +2384,7 @@ mrvl_create_cls_table(struct rte_eth_dev *dev, struct rte_flow *first_flow)
 	memset(&priv->cls_tbl_params, 0, sizeof(priv->cls_tbl_params));
 
 	priv->cls_tbl_params.type = mrvl_engine_type(first_flow);
-	RTE_LOG(INFO, PMD, "Setting cls search engine type to %s\n",
+	MRVL_LOG(INFO, "Setting cls search engine type to %s",
 			priv->cls_tbl_params.type == PP2_CLS_TBL_EXACT_MATCH ?
 			"exact" : "maskable");
 	priv->cls_tbl_params.max_num_rules = MRVL_CLS_MAX_NUM_RULES;
diff --git a/drivers/net/mvpp2/mrvl_qos.c b/drivers/net/mvpp2/mrvl_qos.c
index 70d000ca..71856c1a 100644
--- a/drivers/net/mvpp2/mrvl_qos.c
+++ b/drivers/net/mvpp2/mrvl_qos.c
@@ -138,7 +138,7 @@ get_outq_cfg(struct rte_cfgfile *file, int port, int outq,
 			cfg->port[port].outq[outq].sched_mode =
 				PP2_PPIO_SCHED_M_WRR;
 		} else {
-			RTE_LOG(ERR, PMD, "Unknown token: %s\n", entry);
+			MRVL_LOG(ERR, "Unknown token: %s", entry);
 			return -1;
 		}
 	}
@@ -159,7 +159,7 @@ get_outq_cfg(struct rte_cfgfile *file, int port, int outq,
 	 * global port rate limiting has priority.
 	 */
 	if (cfg->port[port].rate_limit_enable) {
-		RTE_LOG(WARNING, PMD, "Port %d rate limiting already enabled\n",
+		MRVL_LOG(WARNING, "Port %d rate limiting already enabled",
 			port);
 		return 0;
 	}
@@ -340,7 +340,7 @@ parse_tc_cfg(struct rte_cfgfile *file, int port, int tc,
 			RTE_DIM(cfg->port[port].tc[tc].inq),
 			MRVL_PP2_RXQ_MAX);
 		if (n < 0) {
-			RTE_LOG(ERR, PMD, "Error %d while parsing: %s\n",
+			MRVL_LOG(ERR, "Error %d while parsing: %s",
 				n, entry);
 			return n;
 		}
@@ -355,7 +355,7 @@ parse_tc_cfg(struct rte_cfgfile *file, int port, int tc,
 			RTE_DIM(cfg->port[port].tc[tc].pcp),
 			MAX_PCP);
 		if (n < 0) {
-			RTE_LOG(ERR, PMD, "Error %d while parsing: %s\n",
+			MRVL_LOG(ERR, "Error %d while parsing: %s",
 				n, entry);
 			return n;
 		}
@@ -370,7 +370,7 @@ parse_tc_cfg(struct rte_cfgfile *file, int port, int tc,
 			RTE_DIM(cfg->port[port].tc[tc].dscp),
 			MAX_DSCP);
 		if (n < 0) {
-			RTE_LOG(ERR, PMD, "Error %d while parsing: %s\n",
+			MRVL_LOG(ERR, "Error %d while parsing: %s",
 				n, entry);
 			return n;
 		}
@@ -390,7 +390,7 @@ parse_tc_cfg(struct rte_cfgfile *file, int port, int tc,
 				sizeof(MRVL_TOK_PLCR_DEFAULT_COLOR_RED))) {
 			cfg->port[port].tc[tc].color = PP2_PPIO_COLOR_RED;
 		} else {
-			RTE_LOG(ERR, PMD, "Error while parsing: %s\n", entry);
+			MRVL_LOG(ERR, "Error while parsing: %s", entry);
 			return -1;
 		}
 	}
@@ -435,7 +435,7 @@ mrvl_get_qoscfg(const char *key __rte_unused, const char *path,
 
 	if (n == 0) {
 		/* This is weird, but not bad. */
-		RTE_LOG(WARNING, PMD, "Empty configuration file?\n");
+		MRVL_LOG(WARNING, "Empty configuration file?");
 		return 0;
 	}
 
@@ -461,8 +461,8 @@ mrvl_get_qoscfg(const char *key __rte_unused, const char *path,
 				return -1;
 			(*cfg)->port[n].default_tc = (uint8_t)val;
 		} else {
-			RTE_LOG(ERR, PMD,
-				"Default Traffic Class required in custom configuration!\n");
+			MRVL_LOG(ERR,
+				"Default Traffic Class required in custom configuration!");
 			return -1;
 		}
 
@@ -489,7 +489,7 @@ mrvl_get_qoscfg(const char *key __rte_unused, const char *path,
 					sizeof(MRVL_TOK_PLCR_UNIT_PACKETS))) {
 					unit = PP2_CLS_PLCR_PACKETS_TOKEN_UNIT;
 				} else {
-					RTE_LOG(ERR, PMD, "Unknown token: %s\n",
+					MRVL_LOG(ERR, "Unknown token: %s",
 						entry);
 					return -1;
 				}
@@ -511,8 +511,8 @@ mrvl_get_qoscfg(const char *key __rte_unused, const char *path,
 					sizeof(MRVL_TOK_PLCR_COLOR_AWARE))) {
 					mode = PP2_CLS_PLCR_COLOR_AWARE_MODE;
 				} else {
-					RTE_LOG(ERR, PMD,
-						"Error in parsing: %s\n",
+					MRVL_LOG(ERR,
+						"Error in parsing: %s",
 						entry);
 					return -1;
 				}
@@ -682,7 +682,7 @@ setup_policer(struct mrvl_priv *priv, struct pp2_cls_plcr_params *params)
 
 	ret = pp2_cls_plcr_init(params, &priv->policer);
 	if (ret) {
-		RTE_LOG(ERR, PMD, "Failed to setup %s\n", match);
+		MRVL_LOG(ERR, "Failed to setup %s", match);
 		return -1;
 	}
 
@@ -742,8 +742,8 @@ mrvl_configure_rxqs(struct mrvl_priv *priv, uint16_t portid,
 	for (tc = 0; tc < RTE_DIM(port_cfg->tc); ++tc) {
 		if (port_cfg->tc[tc].pcps > RTE_DIM(port_cfg->tc[0].pcp)) {
 			/* Better safe than sorry. */
-			RTE_LOG(ERR, PMD,
-				"Too many PCPs configured in TC %zu!\n", tc);
+			MRVL_LOG(ERR,
+				"Too many PCPs configured in TC %zu!", tc);
 			return -1;
 		}
 		for (i = 0; i < port_cfg->tc[tc].pcps; ++i) {
@@ -764,8 +764,8 @@ mrvl_configure_rxqs(struct mrvl_priv *priv, uint16_t portid,
 	for (tc = 0; tc < RTE_DIM(port_cfg->tc); ++tc) {
 		if (port_cfg->tc[tc].dscps > RTE_DIM(port_cfg->tc[0].dscp)) {
 			/* Better safe than sorry. */
-			RTE_LOG(ERR, PMD,
-				"Too many DSCPs configured in TC %zu!\n", tc);
+			MRVL_LOG(ERR,
+				"Too many DSCPs configured in TC %zu!", tc);
 			return -1;
 		}
 		for (i = 0; i < port_cfg->tc[tc].dscps; ++i) {
@@ -786,8 +786,8 @@ mrvl_configure_rxqs(struct mrvl_priv *priv, uint16_t portid,
 	for (tc = 0; tc < RTE_DIM(port_cfg->tc); ++tc) {
 		if (port_cfg->tc[tc].inqs > RTE_DIM(port_cfg->tc[0].inq)) {
 			/* Overflow. */
-			RTE_LOG(ERR, PMD,
-				"Too many RX queues configured per TC %zu!\n",
+			MRVL_LOG(ERR,
+				"Too many RX queues configured per TC %zu!",
 				tc);
 			return -1;
 		}
@@ -795,7 +795,7 @@ mrvl_configure_rxqs(struct mrvl_priv *priv, uint16_t portid,
 			uint8_t idx = port_cfg->tc[tc].inq[i];
 
 			if (idx > RTE_DIM(priv->rxq_map)) {
-				RTE_LOG(ERR, PMD, "Bad queue index %d!\n", idx);
+				MRVL_LOG(ERR, "Bad queue index %d!", idx);
 				return -1;
 			}
 
@@ -878,7 +878,7 @@ mrvl_start_qos_mapping(struct mrvl_priv *priv)
 	size_t i;
 
 	if (priv->ppio == NULL) {
-		RTE_LOG(ERR, PMD, "ppio must not be NULL here!\n");
+		MRVL_LOG(ERR, "ppio must not be NULL here!");
 		return -1;
 	}
 
diff --git a/drivers/net/netvsc/Makefile b/drivers/net/netvsc/Makefile
new file mode 100644
index 00000000..3c713af3
--- /dev/null
+++ b/drivers/net/netvsc/Makefile
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+LIB = librte_pmd_netvsc.a
+
+CFLAGS += -O3 $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+EXPORT_MAP := rte_pmd_netvsc_version.map
+
+LIBABIVER := 1
+
+SRCS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += hn_ethdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += hn_rxtx.c
+SRCS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += hn_rndis.c
+SRCS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += hn_nvs.c
+
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
+LDLIBS += -lrte_bus_vmbus
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c
new file mode 100644
index 00000000..78b842ba
--- /dev/null
+++ b/drivers/net/netvsc/hn_ethdev.c
@@ -0,0 +1,761 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2018 Microsoft Corporation
+ * Copyright(c) 2013-2016 Brocade Communications Systems, Inc.
+ * All rights reserved.
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <unistd.h>
+
+#include <rte_ethdev.h>
+#include <rte_memcpy.h>
+#include <rte_string_fns.h>
+#include <rte_memzone.h>
+#include <rte_malloc.h>
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_ether.h>
+#include <rte_ethdev_driver.h>
+#include <rte_cycles.h>
+#include <rte_errno.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_dev.h>
+#include <rte_bus_vmbus.h>
+
+#include "hn_logs.h"
+#include "hn_var.h"
+#include "hn_rndis.h"
+#include "hn_nvs.h"
+#include "ndis.h"
+
+#define HN_TX_OFFLOAD_CAPS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
+			    DEV_TX_OFFLOAD_TCP_CKSUM  | \
+			    DEV_TX_OFFLOAD_UDP_CKSUM  | \
+			    DEV_TX_OFFLOAD_TCP_TSO    | \
+			    DEV_TX_OFFLOAD_MULTI_SEGS | \
+			    DEV_TX_OFFLOAD_VLAN_INSERT)
+
+#define HN_RX_OFFLOAD_CAPS (DEV_RX_OFFLOAD_CHECKSUM | \
+			    DEV_RX_OFFLOAD_VLAN_STRIP | \
+			    DEV_RX_OFFLOAD_CRC_STRIP)
+
+int hn_logtype_init;
+int hn_logtype_driver;
+
+struct hn_xstats_name_off {
+	char name[RTE_ETH_XSTATS_NAME_SIZE];
+	unsigned int offset;
+};
+
+static const struct hn_xstats_name_off hn_stat_strings[] = {
+	{ "good_packets",           offsetof(struct hn_stats, packets) },
+	{ "good_bytes",             offsetof(struct hn_stats, bytes) },
+	{ "errors",                 offsetof(struct hn_stats, errors) },
+	{ "allocation_failed",      offsetof(struct hn_stats, nomemory) },
+	{ "multicast_packets",      offsetof(struct hn_stats, multicast) },
+	{ "broadcast_packets",      offsetof(struct hn_stats, broadcast) },
+	{ "undersize_packets",      offsetof(struct hn_stats, size_bins[0]) },
+	{ "size_64_packets",        offsetof(struct hn_stats, size_bins[1]) },
+	{ "size_65_127_packets",    offsetof(struct hn_stats, size_bins[2]) },
+	{ "size_128_255_packets",   offsetof(struct hn_stats, size_bins[3]) },
+	{ "size_256_511_packets",   offsetof(struct hn_stats, size_bins[4]) },
+	{ "size_512_1023_packets",  offsetof(struct hn_stats, size_bins[5]) },
+	{ "size_1024_1518_packets", offsetof(struct hn_stats, size_bins[6]) },
+	{ "size_1519_max_packets",  offsetof(struct hn_stats, size_bins[7]) },
+};
+
+static struct rte_eth_dev *
+eth_dev_vmbus_allocate(struct rte_vmbus_device *dev, size_t private_data_size)
+{
+	struct rte_eth_dev *eth_dev;
+	const char *name;
+
+	if (!dev)
+		return NULL;
+
+	name = dev->device.name;
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		eth_dev = rte_eth_dev_allocate(name);
+		if (!eth_dev) {
+			PMD_DRV_LOG(NOTICE, "can not allocate rte ethdev");
+			return NULL;
+		}
+
+		if (private_data_size) {
+			eth_dev->data->dev_private =
+				rte_zmalloc_socket(name, private_data_size,
+						     RTE_CACHE_LINE_SIZE, dev->device.numa_node);
+			if (!eth_dev->data->dev_private) {
+				PMD_DRV_LOG(NOTICE, "can not allocate driver data");
+				rte_eth_dev_release_port(eth_dev);
+				return NULL;
+			}
+		}
+	} else {
+		eth_dev = rte_eth_dev_attach_secondary(name);
+		if (!eth_dev) {
+			PMD_DRV_LOG(NOTICE, "can not attach secondary");
+			return NULL;
+		}
+	}
+
+	eth_dev->device = &dev->device;
+	eth_dev->intr_handle = &dev->intr_handle;
+
+	return eth_dev;
+}
+
+static void
+eth_dev_vmbus_release(struct rte_eth_dev *eth_dev)
+{
+	/* free ether device */
+	rte_eth_dev_release_port(eth_dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+		rte_free(eth_dev->data->dev_private);
+
+	eth_dev->data->dev_private = NULL;
+
+	/*
+	 * Secondary process will check the name to attach.
+	 * Clear this field to avoid attaching a released ports.
+	 */
+	eth_dev->data->name[0] = '\0';
+
+	eth_dev->device = NULL;
+	eth_dev->intr_handle = NULL;
+}
+
+/* Update link status.
+ * Note: the DPDK definition of "wait_to_complete"
+ *   means block this call until link is up.
+ *   which is not worth supporting.
+ */
+static int
+hn_dev_link_update(struct rte_eth_dev *dev,
+		   __rte_unused int wait_to_complete)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct rte_eth_link link, old;
+	int error;
+
+	old = dev->data->dev_link;
+
+	error = hn_rndis_get_linkstatus(hv);
+	if (error)
+		return error;
+
+	hn_rndis_get_linkspeed(hv);
+
+	link = (struct rte_eth_link) {
+		.link_duplex = ETH_LINK_FULL_DUPLEX,
+		.link_autoneg = ETH_LINK_SPEED_FIXED,
+		.link_speed = hv->link_speed / 10000,
+	};
+
+	if (hv->link_status == NDIS_MEDIA_STATE_CONNECTED)
+		link.link_status = ETH_LINK_UP;
+	else
+		link.link_status = ETH_LINK_DOWN;
+
+	if (old.link_status == link.link_status)
+		return 0;
+
+	PMD_INIT_LOG(DEBUG, "Port %d is %s", dev->data->port_id,
+		     (link.link_status == ETH_LINK_UP) ? "up" : "down");
+
+	return rte_eth_linkstatus_set(dev, &link);
+}
+
+static void hn_dev_info_get(struct rte_eth_dev *dev,
+			    struct rte_eth_dev_info *dev_info)
+{
+	struct hn_data *hv = dev->data->dev_private;
+
+	dev_info->speed_capa = ETH_LINK_SPEED_10G;
+	dev_info->min_rx_bufsize = HN_MIN_RX_BUF_SIZE;
+	dev_info->max_rx_pktlen  = HN_MAX_XFER_LEN;
+	dev_info->max_mac_addrs  = 1;
+
+	dev_info->hash_key_size = NDIS_HASH_KEYSIZE_TOEPLITZ;
+	dev_info->flow_type_rss_offloads =
+		ETH_RSS_IPV4 | ETH_RSS_IPV6 | ETH_RSS_TCP | ETH_RSS_UDP;
+
+	dev_info->max_rx_queues = hv->max_queues;
+	dev_info->max_tx_queues = hv->max_queues;
+
+	hn_rndis_get_offload(hv, dev_info);
+}
+
+static void
+hn_dev_promiscuous_enable(struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+
+	hn_rndis_set_rxfilter(hv, NDIS_PACKET_TYPE_PROMISCUOUS);
+}
+
+static void
+hn_dev_promiscuous_disable(struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	uint32_t filter;
+
+	filter = NDIS_PACKET_TYPE_DIRECTED | NDIS_PACKET_TYPE_BROADCAST;
+	if (dev->data->all_multicast)
+		filter |= NDIS_PACKET_TYPE_ALL_MULTICAST;
+	hn_rndis_set_rxfilter(hv, filter);
+}
+
+static void
+hn_dev_allmulticast_enable(struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+
+	hn_rndis_set_rxfilter(hv, NDIS_PACKET_TYPE_DIRECTED |
+			      NDIS_PACKET_TYPE_ALL_MULTICAST |
+			NDIS_PACKET_TYPE_BROADCAST);
+}
+
+static void
+hn_dev_allmulticast_disable(struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+
+	hn_rndis_set_rxfilter(hv, NDIS_PACKET_TYPE_DIRECTED |
+			     NDIS_PACKET_TYPE_BROADCAST);
+}
+
+/* Setup shared rx/tx queue data */
+static int hn_subchan_configure(struct hn_data *hv,
+				uint32_t subchan)
+{
+	struct vmbus_channel *primary = hn_primary_chan(hv);
+	int err;
+	unsigned int retry = 0;
+
+	PMD_DRV_LOG(DEBUG,
+		    "open %u subchannels", subchan);
+
+	/* Send create sub channels command */
+	err = hn_nvs_alloc_subchans(hv, &subchan);
+	if (err)
+		return  err;
+
+	while (subchan > 0) {
+		struct vmbus_channel *new_sc;
+		uint16_t chn_index;
+
+		err = rte_vmbus_subchan_open(primary, &new_sc);
+		if (err == -ENOENT && ++retry < 1000) {
+			/* This can happen if not ready yet */
+			rte_delay_ms(10);
+			continue;
+		}
+
+		if (err) {
+			PMD_DRV_LOG(ERR,
+				    "open subchannel failed: %d", err);
+			return err;
+		}
+
+		retry = 0;
+		chn_index = rte_vmbus_sub_channel_index(new_sc);
+		if (chn_index == 0 || chn_index > hv->max_queues) {
+			PMD_DRV_LOG(ERR,
+				    "Invalid subchannel offermsg channel %u",
+				    chn_index);
+			return -EIO;
+		}
+
+		PMD_DRV_LOG(DEBUG, "new sub channel %u", chn_index);
+		hv->channels[chn_index] = new_sc;
+		--subchan;
+	}
+
+	return err;
+}
+
+static int hn_dev_configure(struct rte_eth_dev *dev)
+{
+	const struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
+	const struct rte_eth_rxmode *rxmode = &dev_conf->rxmode;
+	const struct rte_eth_txmode *txmode = &dev_conf->txmode;
+
+	const struct rte_eth_rss_conf *rss_conf =
+		&dev_conf->rx_adv_conf.rss_conf;
+	struct hn_data *hv = dev->data->dev_private;
+	uint64_t unsupported;
+	int err, subchan;
+
+	PMD_INIT_FUNC_TRACE();
+
+	unsupported = txmode->offloads & ~HN_TX_OFFLOAD_CAPS;
+	if (unsupported) {
+		PMD_DRV_LOG(NOTICE,
+			    "unsupported TX offload: %#" PRIx64,
+			    unsupported);
+		return -EINVAL;
+	}
+
+	unsupported = rxmode->offloads & ~HN_RX_OFFLOAD_CAPS;
+	if (unsupported) {
+		PMD_DRV_LOG(NOTICE,
+			    "unsupported RX offload: %#" PRIx64,
+			    rxmode->offloads);
+		return -EINVAL;
+	}
+
+	err = hn_rndis_conf_offload(hv, txmode->offloads,
+				    rxmode->offloads);
+	if (err) {
+		PMD_DRV_LOG(NOTICE,
+			    "offload configure failed");
+		return err;
+	}
+
+	hv->num_queues = RTE_MAX(dev->data->nb_rx_queues,
+				 dev->data->nb_tx_queues);
+	subchan = hv->num_queues - 1;
+	if (subchan > 0) {
+		err = hn_subchan_configure(hv, subchan);
+		if (err) {
+			PMD_DRV_LOG(NOTICE,
+				    "subchannel configuration failed");
+			return err;
+		}
+
+		err = hn_rndis_conf_rss(hv, rss_conf);
+		if (err) {
+			PMD_DRV_LOG(NOTICE,
+				    "rss configuration failed");
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static int hn_dev_stats_get(struct rte_eth_dev *dev,
+			    struct rte_eth_stats *stats)
+{
+	unsigned int i;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		const struct hn_tx_queue *txq = dev->data->tx_queues[i];
+
+		if (!txq)
+			continue;
+
+		stats->opackets += txq->stats.packets;
+		stats->obytes += txq->stats.bytes;
+		stats->oerrors += txq->stats.errors + txq->stats.nomemory;
+
+		if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
+			stats->q_opackets[i] = txq->stats.packets;
+			stats->q_obytes[i] = txq->stats.bytes;
+		}
+	}
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		const struct hn_rx_queue *rxq = dev->data->rx_queues[i];
+
+		if (!rxq)
+			continue;
+
+		stats->ipackets += rxq->stats.packets;
+		stats->ibytes += rxq->stats.bytes;
+		stats->ierrors += rxq->stats.errors;
+		stats->imissed += rxq->ring_full;
+
+		if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
+			stats->q_ipackets[i] = rxq->stats.packets;
+			stats->q_ibytes[i] = rxq->stats.bytes;
+		}
+	}
+
+	stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
+	return 0;
+}
+
+static void
+hn_dev_stats_reset(struct rte_eth_dev *dev)
+{
+	unsigned int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		struct hn_tx_queue *txq = dev->data->tx_queues[i];
+
+		if (!txq)
+			continue;
+		memset(&txq->stats, 0, sizeof(struct hn_stats));
+	}
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		struct hn_rx_queue *rxq = dev->data->rx_queues[i];
+
+		if (!rxq)
+			continue;
+
+		memset(&rxq->stats, 0, sizeof(struct hn_stats));
+		rxq->ring_full = 0;
+	}
+}
+
+static int
+hn_dev_xstats_get_names(struct rte_eth_dev *dev,
+			struct rte_eth_xstat_name *xstats_names,
+			__rte_unused unsigned int limit)
+{
+	unsigned int i, t, count = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (!xstats_names)
+		return dev->data->nb_tx_queues * RTE_DIM(hn_stat_strings)
+			+ dev->data->nb_rx_queues * RTE_DIM(hn_stat_strings);
+
+	/* Note: limit checked in rte_eth_xstats_names() */
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		const struct hn_tx_queue *txq = dev->data->tx_queues[i];
+
+		if (!txq)
+			continue;
+
+		for (t = 0; t < RTE_DIM(hn_stat_strings); t++)
+			snprintf(xstats_names[count++].name,
+				 RTE_ETH_XSTATS_NAME_SIZE,
+				 "tx_q%u_%s", i, hn_stat_strings[t].name);
+	}
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++)  {
+		const struct hn_rx_queue *rxq = dev->data->rx_queues[i];
+
+		if (!rxq)
+			continue;
+
+		for (t = 0; t < RTE_DIM(hn_stat_strings); t++)
+			snprintf(xstats_names[count++].name,
+				 RTE_ETH_XSTATS_NAME_SIZE,
+				 "rx_q%u_%s", i,
+				 hn_stat_strings[t].name);
+	}
+
+	return count;
+}
+
+static int
+hn_dev_xstats_get(struct rte_eth_dev *dev,
+		  struct rte_eth_xstat *xstats,
+		  unsigned int n)
+{
+	unsigned int i, t, count = 0;
+
+	const unsigned int nstats =
+		dev->data->nb_tx_queues * RTE_DIM(hn_stat_strings)
+		+ dev->data->nb_rx_queues * RTE_DIM(hn_stat_strings);
+	const char *stats;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (n < nstats)
+		return nstats;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		const struct hn_tx_queue *txq = dev->data->tx_queues[i];
+
+		if (!txq)
+			continue;
+
+		stats = (const char *)&txq->stats;
+		for (t = 0; t < RTE_DIM(hn_stat_strings); t++)
+			xstats[count++].value = *(const uint64_t *)
+				(stats + hn_stat_strings[t].offset);
+	}
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		const struct hn_rx_queue *rxq = dev->data->rx_queues[i];
+
+		if (!rxq)
+			continue;
+
+		stats = (const char *)&rxq->stats;
+		for (t = 0; t < RTE_DIM(hn_stat_strings); t++)
+			xstats[count++].value = *(const uint64_t *)
+				(stats + hn_stat_strings[t].offset);
+	}
+
+	return count;
+}
+
+static int
+hn_dev_start(struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* check if lsc interrupt feature is enabled */
+	if (dev->data->dev_conf.intr_conf.lsc) {
+		PMD_DRV_LOG(ERR, "link status not supported yet");
+		return -ENOTSUP;
+	}
+
+	return hn_rndis_set_rxfilter(hv,
+				     NDIS_PACKET_TYPE_BROADCAST |
+				     NDIS_PACKET_TYPE_ALL_MULTICAST |
+				     NDIS_PACKET_TYPE_DIRECTED);
+}
+
+static void
+hn_dev_stop(struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+
+	PMD_INIT_FUNC_TRACE();
+
+	hn_rndis_set_rxfilter(hv, 0);
+}
+
+static void
+hn_dev_close(struct rte_eth_dev *dev __rte_unused)
+{
+	PMD_INIT_LOG(DEBUG, "close");
+}
+
+static const struct eth_dev_ops hn_eth_dev_ops = {
+	.dev_configure		= hn_dev_configure,
+	.dev_start		= hn_dev_start,
+	.dev_stop		= hn_dev_stop,
+	.dev_close		= hn_dev_close,
+	.dev_infos_get		= hn_dev_info_get,
+	.txq_info_get		= hn_dev_tx_queue_info,
+	.rxq_info_get		= hn_dev_rx_queue_info,
+	.promiscuous_enable     = hn_dev_promiscuous_enable,
+	.promiscuous_disable    = hn_dev_promiscuous_disable,
+	.allmulticast_enable    = hn_dev_allmulticast_enable,
+	.allmulticast_disable   = hn_dev_allmulticast_disable,
+	.tx_queue_setup		= hn_dev_tx_queue_setup,
+	.tx_queue_release	= hn_dev_tx_queue_release,
+	.rx_queue_setup		= hn_dev_rx_queue_setup,
+	.rx_queue_release	= hn_dev_rx_queue_release,
+	.link_update		= hn_dev_link_update,
+	.stats_get		= hn_dev_stats_get,
+	.xstats_get		= hn_dev_xstats_get,
+	.xstats_get_names	= hn_dev_xstats_get_names,
+	.stats_reset            = hn_dev_stats_reset,
+	.xstats_reset		= hn_dev_stats_reset,
+};
+
+/*
+ * Setup connection between PMD and kernel.
+ */
+static int
+hn_attach(struct hn_data *hv, unsigned int mtu)
+{
+	int error;
+
+	/* Attach NVS */
+	error = hn_nvs_attach(hv, mtu);
+	if (error)
+		goto failed_nvs;
+
+	/* Attach RNDIS */
+	error = hn_rndis_attach(hv);
+	if (error)
+		goto failed_rndis;
+
+	/*
+	 * NOTE:
+	 * Under certain conditions on certain versions of Hyper-V,
+	 * the RNDIS rxfilter is _not_ zero on the hypervisor side
+	 * after the successful RNDIS initialization.
+	 */
+	hn_rndis_set_rxfilter(hv, NDIS_PACKET_TYPE_NONE);
+	return 0;
+failed_rndis:
+	hn_nvs_detach(hv);
+failed_nvs:
+	return error;
+}
+
+static void
+hn_detach(struct hn_data *hv)
+{
+	hn_nvs_detach(hv);
+	hn_rndis_detach(hv);
+}
+
+static int
+eth_hn_dev_init(struct rte_eth_dev *eth_dev)
+{
+	struct hn_data *hv = eth_dev->data->dev_private;
+	struct rte_device *device = eth_dev->device;
+	struct rte_vmbus_device *vmbus;
+	unsigned int rxr_cnt;
+	int err, max_chan;
+
+	PMD_INIT_FUNC_TRACE();
+
+	vmbus = container_of(device, struct rte_vmbus_device, device);
+	eth_dev->dev_ops = &hn_eth_dev_ops;
+	eth_dev->tx_pkt_burst = &hn_xmit_pkts;
+	eth_dev->rx_pkt_burst = &hn_recv_pkts;
+
+	/*
+	 * for secondary processes, we don't initialize any further as primary
+	 * has already done this work.
+	 */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	/* Since Hyper-V only supports one MAC address, just use local data */
+	eth_dev->data->mac_addrs = &hv->mac_addr;
+
+	hv->vmbus = vmbus;
+	hv->rxbuf_res = &vmbus->resource[HV_RECV_BUF_MAP];
+	hv->chim_res  = &vmbus->resource[HV_SEND_BUF_MAP];
+	hv->port_id = eth_dev->data->port_id;
+
+	/* Initialize primary channel input for control operations */
+	err = rte_vmbus_chan_open(vmbus, &hv->channels[0]);
+	if (err)
+		return err;
+
+	hv->primary = hn_rx_queue_alloc(hv, 0,
+					eth_dev->device->numa_node);
+
+	if (!hv->primary)
+		return -ENOMEM;
+
+	err = hn_attach(hv, ETHER_MTU);
+	if  (err)
+		goto failed;
+
+	err = hn_tx_pool_init(eth_dev);
+	if (err)
+		goto failed;
+
+	err = hn_rndis_get_eaddr(hv, hv->mac_addr.addr_bytes);
+	if (err)
+		goto failed;
+
+	max_chan = rte_vmbus_max_channels(vmbus);
+	PMD_INIT_LOG(DEBUG, "VMBus max channels %d", max_chan);
+	if (max_chan <= 0)
+		goto failed;
+
+	if (hn_rndis_query_rsscaps(hv, &rxr_cnt) != 0)
+		rxr_cnt = 1;
+
+	hv->max_queues = RTE_MIN(rxr_cnt, (unsigned int)max_chan);
+
+	return 0;
+
+failed:
+	PMD_INIT_LOG(NOTICE, "device init failed");
+
+	hn_detach(hv);
+	return err;
+}
+
+static int
+eth_hn_dev_uninit(struct rte_eth_dev *eth_dev)
+{
+	struct hn_data *hv = eth_dev->data->dev_private;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	hn_dev_stop(eth_dev);
+	hn_dev_close(eth_dev);
+
+	eth_dev->dev_ops = NULL;
+	eth_dev->tx_pkt_burst = NULL;
+	eth_dev->rx_pkt_burst = NULL;
+
+	hn_detach(hv);
+	rte_vmbus_chan_close(hv->primary->chan);
+	rte_free(hv->primary);
+
+	eth_dev->data->mac_addrs = NULL;
+
+	return 0;
+}
+
+static int eth_hn_probe(struct rte_vmbus_driver *drv __rte_unused,
+			struct rte_vmbus_device *dev)
+{
+	struct rte_eth_dev *eth_dev;
+	int ret;
+
+	PMD_INIT_FUNC_TRACE();
+
+	eth_dev = eth_dev_vmbus_allocate(dev, sizeof(struct hn_data));
+	if (!eth_dev)
+		return -ENOMEM;
+
+	ret = eth_hn_dev_init(eth_dev);
+	if (ret)
+		eth_dev_vmbus_release(eth_dev);
+	else
+		rte_eth_dev_probing_finish(eth_dev);
+
+	return ret;
+}
+
+static int eth_hn_remove(struct rte_vmbus_device *dev)
+{
+	struct rte_eth_dev *eth_dev;
+	int ret;
+
+	PMD_INIT_FUNC_TRACE();
+
+	eth_dev = rte_eth_dev_allocated(dev->device.name);
+	if (!eth_dev)
+		return -ENODEV;
+
+	ret = eth_hn_dev_uninit(eth_dev);
+	if (ret)
+		return ret;
+
+	eth_dev_vmbus_release(eth_dev);
+	return 0;
+}
+
+/* Network device GUID */
+static const rte_uuid_t hn_net_ids[] = {
+	/*  f8615163-df3e-46c5-913f-f2d2f965ed0e */
+	RTE_UUID_INIT(0xf8615163, 0xdf3e, 0x46c5, 0x913f, 0xf2d2f965ed0eULL),
+	{ 0 }
+};
+
+static struct rte_vmbus_driver rte_netvsc_pmd = {
+	.id_table = hn_net_ids,
+	.probe = eth_hn_probe,
+	.remove = eth_hn_remove,
+};
+
+RTE_PMD_REGISTER_VMBUS(net_netvsc, rte_netvsc_pmd);
+RTE_PMD_REGISTER_KMOD_DEP(net_netvsc, "* uio_hv_generic");
+
+RTE_INIT(hn_init_log);
+static void
+hn_init_log(void)
+{
+	hn_logtype_init = rte_log_register("pmd.net.netvsc.init");
+	if (hn_logtype_init >= 0)
+		rte_log_set_level(hn_logtype_init, RTE_LOG_NOTICE);
+	hn_logtype_driver = rte_log_register("pmd.net.netvsc.driver");
+	if (hn_logtype_driver >= 0)
+		rte_log_set_level(hn_logtype_driver, RTE_LOG_NOTICE);
+}
diff --git a/drivers/net/netvsc/hn_logs.h b/drivers/net/netvsc/hn_logs.h
new file mode 100644
index 00000000..cddadef0
--- /dev/null
+++ b/drivers/net/netvsc/hn_logs.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+
+#ifndef _HN_LOGS_H_
+#define _HN_LOGS_H_
+
+#include <rte_log.h>
+
+extern int hn_logtype_init;
+extern int hn_logtype_driver;
+
+#define PMD_INIT_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, hn_logtype_init, "%s(): " fmt "\n",\
+		__func__, ## args)
+#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
+
+#ifdef RTE_LIBRTE_NETVSC_DEBUG_RX
+#define PMD_RX_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, hn_logtype_driver, \
+		"%s() rx: " fmt "\n", __func__, ## args)
+#else
+#define PMD_RX_LOG(level, fmt, args...) do { } while (0)
+#endif
+
+#ifdef RTE_LIBRTE_NETVSC_DEBUG_TX
+#define PMD_TX_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, hn_logtype_driver, \
+		"%s() tx: " fmt "\n", __func__, ## args)
+#else
+#define PMD_TX_LOG(level, fmt, args...) do { } while (0)
+#endif
+
+#define PMD_DRV_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, hn_logtype_driver, "%s(): " fmt "\n", \
+		__func__, ## args)
+
+#endif /* _HN_LOGS_H_ */
diff --git a/drivers/net/netvsc/hn_nvs.c b/drivers/net/netvsc/hn_nvs.c
new file mode 100644
index 00000000..77d3b839
--- /dev/null
+++ b/drivers/net/netvsc/hn_nvs.c
@@ -0,0 +1,546 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018 Microsoft Corp.
+ * Copyright (c) 2010-2012 Citrix Inc.
+ * Copyright (c) 2012 NetApp Inc.
+ * All rights reserved.
+ */
+
+/*
+ * Network Virtualization Service.
+ */
+
+
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <unistd.h>
+
+#include <rte_ethdev.h>
+#include <rte_string_fns.h>
+#include <rte_memzone.h>
+#include <rte_malloc.h>
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_ether.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_cycles.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_dev.h>
+#include <rte_bus_vmbus.h>
+
+#include "hn_logs.h"
+#include "hn_var.h"
+#include "hn_nvs.h"
+
+static const uint32_t hn_nvs_version[] = {
+	NVS_VERSION_61,
+	NVS_VERSION_6,
+	NVS_VERSION_5,
+	NVS_VERSION_4,
+	NVS_VERSION_2,
+	NVS_VERSION_1
+};
+
+static int hn_nvs_req_send(struct hn_data *hv,
+			   void *req, uint32_t reqlen)
+{
+	return rte_vmbus_chan_send(hn_primary_chan(hv),
+				   VMBUS_CHANPKT_TYPE_INBAND,
+				   req, reqlen, 0,
+				   VMBUS_CHANPKT_FLAG_NONE, NULL);
+}
+
+static int
+hn_nvs_execute(struct hn_data *hv,
+	       void *req, uint32_t reqlen,
+	       void *resp, uint32_t resplen,
+	       uint32_t type)
+{
+	struct vmbus_channel *chan = hn_primary_chan(hv);
+	char buffer[NVS_RESPSIZE_MAX];
+	const struct hn_nvs_hdr *hdr;
+	uint32_t len;
+	int ret;
+
+	/* Send request to ring buffer */
+	ret = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_INBAND,
+				  req, reqlen, 0,
+				  VMBUS_CHANPKT_FLAG_RC, NULL);
+
+	if (ret) {
+		PMD_DRV_LOG(ERR, "send request failed: %d", ret);
+		return ret;
+	}
+
+ retry:
+	len = sizeof(buffer);
+	ret = rte_vmbus_chan_recv(chan, buffer, &len, NULL);
+	if (ret == -EAGAIN) {
+		rte_delay_us(HN_CHAN_INTERVAL_US);
+		goto retry;
+	}
+
+	if (ret < 0) {
+		PMD_DRV_LOG(ERR, "recv response failed: %d", ret);
+		return ret;
+	}
+
+	hdr = (struct hn_nvs_hdr *)buffer;
+	if (hdr->type != type) {
+		PMD_DRV_LOG(ERR, "unexpected NVS resp %#x, expect %#x",
+			    hdr->type, type);
+		return -EINVAL;
+	}
+
+	if (len < resplen) {
+		PMD_DRV_LOG(ERR,
+			    "invalid NVS resp len %u (expect %u)",
+			    len, resplen);
+		return -EINVAL;
+	}
+
+	memcpy(resp, buffer, resplen);
+
+	/* All pass! */
+	return 0;
+}
+
+static int
+hn_nvs_doinit(struct hn_data *hv, uint32_t nvs_ver)
+{
+	struct hn_nvs_init init;
+	struct hn_nvs_init_resp resp;
+	uint32_t status;
+	int error;
+
+	memset(&init, 0, sizeof(init));
+	init.type = NVS_TYPE_INIT;
+	init.ver_min = nvs_ver;
+	init.ver_max = nvs_ver;
+
+	error = hn_nvs_execute(hv, &init, sizeof(init),
+			       &resp, sizeof(resp),
+			       NVS_TYPE_INIT_RESP);
+	if (error)
+		return error;
+
+	status = resp.status;
+	if (status != NVS_STATUS_OK) {
+		/* Not fatal, try other versions */
+		PMD_INIT_LOG(DEBUG, "nvs init failed for ver 0x%x",
+			     nvs_ver);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+hn_nvs_conn_rxbuf(struct hn_data *hv)
+{
+	struct hn_nvs_rxbuf_conn conn;
+	struct hn_nvs_rxbuf_connresp resp;
+	uint32_t status;
+	int error;
+
+	/* Kernel has already setup RXBUF on primary channel. */
+
+	/*
+	 * Connect RXBUF to NVS.
+	 */
+	conn.type = NVS_TYPE_RXBUF_CONN;
+	conn.gpadl = hv->rxbuf_res->phys_addr;
+	conn.sig = NVS_RXBUF_SIG;
+	PMD_DRV_LOG(DEBUG, "connect rxbuff va=%p gpad=%#" PRIx64,
+		    hv->rxbuf_res->addr,
+		    hv->rxbuf_res->phys_addr);
+
+	error = hn_nvs_execute(hv, &conn, sizeof(conn),
+			       &resp, sizeof(resp),
+			       NVS_TYPE_RXBUF_CONNRESP);
+	if (error) {
+		PMD_DRV_LOG(ERR,
+			    "exec nvs rxbuf conn failed: %d",
+			    error);
+		return error;
+	}
+
+	status = resp.status;
+	if (status != NVS_STATUS_OK) {
+		PMD_DRV_LOG(ERR,
+			    "nvs rxbuf conn failed: %x", status);
+		return -EIO;
+	}
+	if (resp.nsect != 1) {
+		PMD_DRV_LOG(ERR,
+			    "nvs rxbuf response num sections %u != 1",
+			    resp.nsect);
+		return -EIO;
+	}
+
+	PMD_DRV_LOG(INFO,
+		    "receive buffer size %u count %u",
+		    resp.nvs_sect[0].slotsz,
+		    resp.nvs_sect[0].slotcnt);
+	hv->rxbuf_section_cnt = resp.nvs_sect[0].slotcnt;
+
+	hv->rxbuf_info = rte_calloc("HN_RXBUF_INFO", hv->rxbuf_section_cnt,
+				    sizeof(*hv->rxbuf_info), RTE_CACHE_LINE_SIZE);
+	if (!hv->rxbuf_info) {
+		PMD_DRV_LOG(ERR,
+			    "could not allocate rxbuf info");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void
+hn_nvs_disconn_rxbuf(struct hn_data *hv)
+{
+	struct hn_nvs_rxbuf_disconn disconn;
+	int error;
+
+	/*
+	 * Disconnect RXBUF from NVS.
+	 */
+	memset(&disconn, 0, sizeof(disconn));
+	disconn.type = NVS_TYPE_RXBUF_DISCONN;
+	disconn.sig = NVS_RXBUF_SIG;
+
+	/* NOTE: No response. */
+	error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
+	if (error) {
+		PMD_DRV_LOG(ERR,
+			    "send nvs rxbuf disconn failed: %d",
+			    error);
+	}
+
+	rte_free(hv->rxbuf_info);
+	/*
+	 * Linger long enough for NVS to disconnect RXBUF.
+	 */
+	rte_delay_ms(200);
+}
+
+static void
+hn_nvs_disconn_chim(struct hn_data *hv)
+{
+	int error;
+
+	if (hv->chim_cnt != 0) {
+		struct hn_nvs_chim_disconn disconn;
+
+		/* Disconnect chimney sending buffer from NVS. */
+		memset(&disconn, 0, sizeof(disconn));
+		disconn.type = NVS_TYPE_CHIM_DISCONN;
+		disconn.sig = NVS_CHIM_SIG;
+
+		/* NOTE: No response. */
+		error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
+
+		if (error) {
+			PMD_DRV_LOG(ERR,
+				    "send nvs chim disconn failed: %d", error);
+		}
+
+		hv->chim_cnt = 0;
+		/*
+		 * Linger long enough for NVS to disconnect chimney
+		 * sending buffer.
+		 */
+		rte_delay_ms(200);
+	}
+}
+
+static int
+hn_nvs_conn_chim(struct hn_data *hv)
+{
+	struct hn_nvs_chim_conn chim;
+	struct hn_nvs_chim_connresp resp;
+	uint32_t sectsz;
+	unsigned long len = hv->chim_res->len;
+	int error;
+
+	/* Connect chimney sending buffer to NVS */
+	memset(&chim, 0, sizeof(chim));
+	chim.type = NVS_TYPE_CHIM_CONN;
+	chim.gpadl = hv->chim_res->phys_addr;
+	chim.sig = NVS_CHIM_SIG;
+	PMD_DRV_LOG(DEBUG, "connect send buf va=%p gpad=%#" PRIx64,
+		    hv->chim_res->addr,
+		    hv->chim_res->phys_addr);
+
+	error = hn_nvs_execute(hv, &chim, sizeof(chim),
+			       &resp, sizeof(resp),
+			       NVS_TYPE_CHIM_CONNRESP);
+	if (error) {
+		PMD_DRV_LOG(ERR, "exec nvs chim conn failed");
+		goto cleanup;
+	}
+
+	if (resp.status != NVS_STATUS_OK) {
+		PMD_DRV_LOG(ERR, "nvs chim conn failed: %x",
+			    resp.status);
+		error = -EIO;
+		goto cleanup;
+	}
+
+	sectsz = resp.sectsz;
+	if (sectsz == 0 || sectsz & (sizeof(uint32_t) - 1)) {
+		/* Can't use chimney sending buffer; done! */
+		PMD_DRV_LOG(NOTICE,
+			    "invalid chimney sending buffer section size: %u",
+			    sectsz);
+		return 0;
+	}
+
+	hv->chim_szmax = sectsz;
+	hv->chim_cnt = len / sectsz;
+
+	PMD_DRV_LOG(INFO, "send buffer %lu section size:%u, count:%u",
+		    len, hv->chim_szmax, hv->chim_cnt);
+
+	if (len % hv->chim_szmax != 0) {
+		PMD_DRV_LOG(NOTICE,
+			    "chimney sending sections are not properly aligned");
+	}
+
+	/* Done! */
+	return 0;
+
+cleanup:
+	hn_nvs_disconn_chim(hv);
+	return error;
+}
+
+/*
+ * Configure MTU and enable VLAN.
+ */
+static int
+hn_nvs_conf_ndis(struct hn_data *hv, unsigned int mtu)
+{
+	struct hn_nvs_ndis_conf conf;
+	int error;
+
+	memset(&conf, 0, sizeof(conf));
+	conf.type = NVS_TYPE_NDIS_CONF;
+	conf.mtu = mtu + ETHER_HDR_LEN;
+	conf.caps = NVS_NDIS_CONF_VLAN;
+
+	/* TODO enable SRIOV */
+	//if (hv->nvs_ver >= NVS_VERSION_5)
+	//	conf.caps |= NVS_NDIS_CONF_SRIOV;
+
+	/* NOTE: No response. */
+	error = hn_nvs_req_send(hv, &conf, sizeof(conf));
+	if (error) {
+		PMD_DRV_LOG(ERR,
+			    "send nvs ndis conf failed: %d", error);
+		return error;
+	}
+
+	return 0;
+}
+
+static int
+hn_nvs_init_ndis(struct hn_data *hv)
+{
+	struct hn_nvs_ndis_init ndis;
+	int error;
+
+	memset(&ndis, 0, sizeof(ndis));
+	ndis.type = NVS_TYPE_NDIS_INIT;
+	ndis.ndis_major = NDIS_VERSION_MAJOR(hv->ndis_ver);
+	ndis.ndis_minor = NDIS_VERSION_MINOR(hv->ndis_ver);
+
+	/* NOTE: No response. */
+	error = hn_nvs_req_send(hv, &ndis, sizeof(ndis));
+	if (error)
+		PMD_DRV_LOG(ERR,
+			    "send nvs ndis init failed: %d", error);
+
+	return error;
+}
+
+static int
+hn_nvs_init(struct hn_data *hv)
+{
+	unsigned int i;
+	int error;
+
+	/*
+	 * Find the supported NVS version and set NDIS version accordingly.
+	 */
+	for (i = 0; i < RTE_DIM(hn_nvs_version); ++i) {
+		error = hn_nvs_doinit(hv, hn_nvs_version[i]);
+		if (error) {
+			PMD_INIT_LOG(DEBUG, "version %#x error %d",
+				     hn_nvs_version[i], error);
+			continue;
+		}
+
+		hv->nvs_ver = hn_nvs_version[i];
+
+		/* Set NDIS version according to NVS version. */
+		hv->ndis_ver = NDIS_VERSION_6_30;
+		if (hv->nvs_ver <= NVS_VERSION_4)
+			hv->ndis_ver = NDIS_VERSION_6_1;
+
+		PMD_INIT_LOG(DEBUG,
+			     "NVS version %#x, NDIS version %u.%u",
+			     hv->nvs_ver, NDIS_VERSION_MAJOR(hv->ndis_ver),
+			     NDIS_VERSION_MINOR(hv->ndis_ver));
+		return 0;
+	}
+
+	PMD_DRV_LOG(ERR,
+		    "no NVS compatible version available");
+	return -ENXIO;
+}
+
+int
+hn_nvs_attach(struct hn_data *hv, unsigned int mtu)
+{
+	int error;
+
+	/*
+	 * Initialize NVS.
+	 */
+	error = hn_nvs_init(hv);
+	if (error)
+		return error;
+
+	/** Configure NDIS before initializing it. */
+	if (hv->nvs_ver >= NVS_VERSION_2) {
+		error = hn_nvs_conf_ndis(hv, mtu);
+		if (error)
+			return error;
+	}
+
+	/*
+	 * Initialize NDIS.
+	 */
+	error = hn_nvs_init_ndis(hv);
+	if (error)
+		return error;
+
+	/*
+	 * Connect RXBUF.
+	 */
+	error = hn_nvs_conn_rxbuf(hv);
+	if (error)
+		return error;
+
+	/*
+	 * Connect chimney sending buffer.
+	 */
+	error = hn_nvs_conn_chim(hv);
+	if (error) {
+		hn_nvs_disconn_rxbuf(hv);
+		return error;
+	}
+
+	return 0;
+}
+
+void
+hn_nvs_detach(struct hn_data *hv __rte_unused)
+{
+	PMD_INIT_FUNC_TRACE();
+
+	/* NOTE: there are no requests to stop the NVS. */
+	hn_nvs_disconn_rxbuf(hv);
+	hn_nvs_disconn_chim(hv);
+}
+
+/*
+ * Ack the consumed RXBUF associated w/ this channel packet,
+ * so that this RXBUF can be recycled by the hypervisor.
+ */
+void
+hn_nvs_ack_rxbuf(struct vmbus_channel *chan, uint64_t tid)
+{
+	unsigned int retries = 0;
+	struct hn_nvs_rndis_ack ack = {
+		.type = NVS_TYPE_RNDIS_ACK,
+		.status = NVS_STATUS_OK,
+	};
+	int error;
+
+	PMD_RX_LOG(DEBUG, "ack RX id %" PRIu64, tid);
+
+ again:
+	error = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP,
+				    &ack, sizeof(ack), tid,
+				    VMBUS_CHANPKT_FLAG_NONE, NULL);
+
+	if (error == 0)
+		return;
+
+	if (error == -EAGAIN) {
+		/*
+		 * NOTE:
+		 * This should _not_ happen in real world, since the
+		 * consumption of the TX bufring from the TX path is
+		 * controlled.
+		 */
+		PMD_RX_LOG(NOTICE, "RXBUF ack retry");
+		if (++retries < 10) {
+			rte_delay_ms(1);
+			goto again;
+		}
+	}
+	/* RXBUF leaks! */
+	PMD_DRV_LOG(ERR, "RXBUF ack failed");
+}
+
+int
+hn_nvs_alloc_subchans(struct hn_data *hv, uint32_t *nsubch)
+{
+	struct hn_nvs_subch_req req;
+	struct hn_nvs_subch_resp resp;
+	int error;
+
+	memset(&req, 0, sizeof(req));
+	req.type = NVS_TYPE_SUBCH_REQ;
+	req.op = NVS_SUBCH_OP_ALLOC;
+	req.nsubch = *nsubch;
+
+	error = hn_nvs_execute(hv, &req, sizeof(req),
+			       &resp, sizeof(resp),
+			       NVS_TYPE_SUBCH_RESP);
+	if (error)
+		return error;
+
+	if (resp.status != NVS_STATUS_OK) {
+		PMD_INIT_LOG(ERR,
+			     "nvs subch alloc failed: %#x",
+			     resp.status);
+		return -EIO;
+	}
+
+	if (resp.nsubch > *nsubch) {
+		PMD_INIT_LOG(NOTICE,
+			     "%u subchans are allocated, requested %u",
+			     resp.nsubch, *nsubch);
+	}
+	*nsubch = resp.nsubch;
+
+	return 0;
+}
+
+void
+hn_nvs_set_datapath(struct hn_data *hv, uint32_t path)
+{
+	struct hn_nvs_datapath dp;
+
+	memset(&dp, 0, sizeof(dp));
+	dp.type = NVS_TYPE_SET_DATAPATH;
+	dp.active_path = path;
+
+	hn_nvs_req_send(hv, &dp, sizeof(dp));
+}
diff --git a/drivers/net/netvsc/hn_nvs.h b/drivers/net/netvsc/hn_nvs.h
new file mode 100644
index 00000000..984a9c11
--- /dev/null
+++ b/drivers/net/netvsc/hn_nvs.h
@@ -0,0 +1,229 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018 Microsoft Corp.
+ * All rights reserved.
+ */
+
+/*
+ * The indirection table message is the largest message
+ * received from host, and that is 112 bytes.
+ */
+#define NVS_RESPSIZE_MAX	256
+
+/*
+ * NDIS protocol version numbers
+ */
+#define NDIS_VERSION_6_1		0x00060001
+#define NDIS_VERSION_6_20		0x00060014
+#define NDIS_VERSION_6_30		0x0006001e
+#define NDIS_VERSION_MAJOR(ver)	(((ver) & 0xffff0000) >> 16)
+#define NDIS_VERSION_MINOR(ver)	((ver) & 0xffff)
+
+/*
+ * NVS versions.
+ */
+#define NVS_VERSION_1		0x00002
+#define NVS_VERSION_2		0x30002
+#define NVS_VERSION_4		0x40000
+#define NVS_VERSION_5		0x50000
+#define NVS_VERSION_6		0x60000
+#define NVS_VERSION_61		0x60001
+
+#define NVS_RXBUF_SIG		0xcafe
+#define NVS_CHIM_SIG			0xface
+
+#define NVS_CHIM_IDX_INVALID		0xffffffff
+
+#define NVS_RNDIS_MTYPE_DATA		0
+#define NVS_RNDIS_MTYPE_CTRL		1
+
+/*
+ * NVS message transacion status codes.
+ */
+#define NVS_STATUS_OK		1
+#define NVS_STATUS_FAILED		2
+
+/*
+ * NVS request/response message types.
+ */
+#define NVS_TYPE_INIT		1
+#define NVS_TYPE_INIT_RESP	2
+
+#define NVS_TYPE_NDIS_INIT	100
+#define NVS_TYPE_RXBUF_CONN	101
+#define NVS_TYPE_RXBUF_CONNRESP	102
+#define NVS_TYPE_RXBUF_DISCONN	103
+#define NVS_TYPE_CHIM_CONN	104
+#define NVS_TYPE_CHIM_CONNRESP	105
+#define NVS_TYPE_CHIM_DISCONN	106
+#define NVS_TYPE_RNDIS		107
+#define NVS_TYPE_RNDIS_ACK	108
+
+#define NVS_TYPE_NDIS_CONF	125
+#define NVS_TYPE_VFASSOC_NOTE	128	/* notification */
+#define NVS_TYPE_SET_DATAPATH	129
+#define NVS_TYPE_SUBCH_REQ	133
+#define NVS_TYPE_SUBCH_RESP	133	/* same as SUBCH_REQ */
+#define NVS_TYPE_TXTBL_NOTE	134	/* notification */
+
+
+/* NVS message common header */
+struct hn_nvs_hdr {
+	uint32_t	type;
+} __rte_packed;
+
+struct hn_nvs_init {
+	uint32_t	type;	/* NVS_TYPE_INIT */
+	uint32_t	ver_min;
+	uint32_t	ver_max;
+	uint8_t		rsvd[28];
+} __rte_packed;
+
+struct hn_nvs_init_resp {
+	uint32_t	type;	/* NVS_TYPE_INIT_RESP */
+	uint32_t	ver;	/* deprecated */
+	uint32_t	rsvd;
+	uint32_t	status;	/* NVS_STATUS_ */
+} __rte_packed;
+
+/* No response */
+struct hn_nvs_ndis_conf {
+	uint32_t	type;	/* NVS_TYPE_NDIS_CONF */
+	uint32_t	mtu;
+	uint32_t	rsvd;
+	uint64_t	caps;	/* NVS_NDIS_CONF_ */
+	uint8_t		rsvd1[20];
+} __rte_packed;
+
+#define NVS_NDIS_CONF_SRIOV		0x0004
+#define NVS_NDIS_CONF_VLAN		0x0008
+
+/* No response */
+struct hn_nvs_ndis_init {
+	uint32_t	type;	/* NVS_TYPE_NDIS_INIT */
+	uint32_t	ndis_major;	/* NDIS_VERSION_MAJOR_ */
+	uint32_t	ndis_minor;	/* NDIS_VERSION_MINOR_ */
+	uint8_t		rsvd[28];
+} __rte_packed;
+
+#define NVS_DATAPATH_SYNTHETIC	0
+#define NVS_DATAPATH_VF		1
+
+/* No response */
+struct hn_nvs_datapath {
+	uint32_t	type;	/* NVS_TYPE_SET_DATAPATH */
+	uint32_t	active_path;/* NVS_DATAPATH_* */
+	uint8_t		rsvd[32];
+} __rte_packed;
+
+struct hn_nvs_rxbuf_conn {
+	uint32_t	type;	/* NVS_TYPE_RXBUF_CONN */
+	uint32_t	gpadl;	/* RXBUF vmbus GPADL */
+	uint16_t	sig;	/* NVS_RXBUF_SIG */
+	uint8_t		rsvd[30];
+} __rte_packed;
+
+struct hn_nvs_rxbuf_sect {
+	uint32_t	start;
+	uint32_t	slotsz;
+	uint32_t	slotcnt;
+	uint32_t	end;
+} __rte_packed;
+
+struct hn_nvs_rxbuf_connresp {
+	uint32_t	type;	/* NVS_TYPE_RXBUF_CONNRESP */
+	uint32_t	status;	/* NVS_STATUS_ */
+	uint32_t	nsect;	/* # of elem in nvs_sect */
+	struct hn_nvs_rxbuf_sect nvs_sect[1];
+} __rte_packed;
+
+/* No response */
+struct hn_nvs_rxbuf_disconn {
+	uint32_t	type;	/* NVS_TYPE_RXBUF_DISCONN */
+	uint16_t	sig;	/* NVS_RXBUF_SIG */
+	uint8_t		rsvd[34];
+} __rte_packed;
+
+struct hn_nvs_chim_conn {
+	uint32_t	type;	/* NVS_TYPE_CHIM_CONN */
+	uint32_t	gpadl;	/* chimney buf vmbus GPADL */
+	uint16_t	sig;	/* NDIS_NVS_CHIM_SIG */
+	uint8_t		rsvd[30];
+} __rte_packed;
+
+struct hn_nvs_chim_connresp {
+	uint32_t	type;	/* NVS_TYPE_CHIM_CONNRESP */
+	uint32_t	status;	/* NVS_STATUS_ */
+	uint32_t	sectsz;	/* section size */
+} __rte_packed;
+
+/* No response */
+struct hn_nvs_chim_disconn {
+	uint32_t	type;	/* NVS_TYPE_CHIM_DISCONN */
+	uint16_t	sig;	/* NVS_CHIM_SIG */
+	uint8_t		rsvd[34];
+} __rte_packed;
+
+#define NVS_SUBCH_OP_ALLOC		1
+
+struct hn_nvs_subch_req {
+	uint32_t	type;	/* NVS_TYPE_SUBCH_REQ */
+	uint32_t	op;	/* NVS_SUBCH_OP_ */
+	uint32_t	nsubch;
+	uint8_t		rsvd[28];
+} __rte_packed;
+
+struct hn_nvs_subch_resp {
+	uint32_t	type;	/* NVS_TYPE_SUBCH_RESP */
+	uint32_t	status;	/* NVS_STATUS_ */
+	uint32_t	nsubch;
+	uint8_t		rsvd[28];
+} __rte_packed;
+
+struct hn_nvs_rndis {
+	uint32_t	type;	/* NVS_TYPE_RNDIS */
+	uint32_t	rndis_mtype;/* NVS_RNDIS_MTYPE_ */
+	/*
+	 * Chimney sending buffer index and size.
+	 *
+	 * NOTE:
+	 * If nvs_chim_idx is set to NVS_CHIM_IDX_INVALID
+	 * and nvs_chim_sz is set to 0, then chimney sending
+	 * buffer is _not_ used by this RNDIS message.
+	 */
+	uint32_t	chim_idx;
+	uint32_t	chim_sz;
+	uint8_t		rsvd[24];
+} __rte_packed;
+
+struct hn_nvs_rndis_ack {
+	uint32_t	type;	/* NVS_TYPE_RNDIS_ACK */
+	uint32_t	status;	/* NVS_STATUS_ */
+	uint8_t		rsvd[32];
+} __rte_packed;
+
+
+int	hn_nvs_attach(struct hn_data *hv, unsigned int mtu);
+void	hn_nvs_detach(struct hn_data *hv);
+void	hn_nvs_ack_rxbuf(struct vmbus_channel *chan, uint64_t tid);
+int	hn_nvs_alloc_subchans(struct hn_data *hv, uint32_t *nsubch);
+void	hn_nvs_set_datapath(struct hn_data *hv, uint32_t path);
+
+static inline int
+hn_nvs_send(struct vmbus_channel *chan, uint16_t flags,
+	    void *nvs_msg, int nvs_msglen, uintptr_t sndc,
+	    bool *need_sig)
+{
+	return rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_INBAND,
+				   nvs_msg, nvs_msglen, (uint64_t)sndc,
+				   flags, need_sig);
+}
+
+static inline int
+hn_nvs_send_sglist(struct vmbus_channel *chan,
+		   struct vmbus_gpa sg[], unsigned int sglen,
+		   void *nvs_msg, int nvs_msglen,
+		   uintptr_t sndc, bool *need_sig)
+{
+	return rte_vmbus_chan_send_sglist(chan, sg, sglen, nvs_msg, nvs_msglen,
+					  (uint64_t)sndc, need_sig);
+}
diff --git a/drivers/net/netvsc/hn_rndis.c b/drivers/net/netvsc/hn_rndis.c
new file mode 100644
index 00000000..bde33969
--- /dev/null
+++ b/drivers/net/netvsc/hn_rndis.c
@@ -0,0 +1,1099 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2009-2018 Microsoft Corp.
+ * Copyright (c) 2010-2012 Citrix Inc.
+ * Copyright (c) 2012 NetApp Inc.
+ * All rights reserved.
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <unistd.h>
+
+#include <rte_ethdev.h>
+#include <rte_string_fns.h>
+#include <rte_memzone.h>
+#include <rte_malloc.h>
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_ether.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_cycles.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_dev.h>
+#include <rte_bus_vmbus.h>
+
+#include "hn_logs.h"
+#include "hn_var.h"
+#include "hn_nvs.h"
+#include "hn_rndis.h"
+#include "ndis.h"
+
+#define HN_RNDIS_XFER_SIZE		0x4000
+
+#define HN_NDIS_TXCSUM_CAP_IP4		\
+	(NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT)
+#define HN_NDIS_TXCSUM_CAP_TCP4		\
+	(NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT)
+#define HN_NDIS_TXCSUM_CAP_TCP6		\
+	(NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \
+	 NDIS_TXCSUM_CAP_IP6EXT)
+#define HN_NDIS_TXCSUM_CAP_UDP6		\
+	(NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT)
+#define HN_NDIS_LSOV2_CAP_IP6		\
+	(NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT)
+
+/* Get unique request id */
+static inline uint32_t
+hn_rndis_rid(struct hn_data *hv)
+{
+	uint32_t rid;
+
+	do {
+		rid = rte_atomic32_add_return(&hv->rndis_req_id, 1);
+	} while (rid == 0);
+
+	return rid;
+}
+
+static void *hn_rndis_alloc(struct hn_data *hv, size_t size)
+{
+	return rte_zmalloc_socket("RNDIS", size, PAGE_SIZE,
+				 hv->vmbus->device.numa_node);
+}
+
+#ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP
+void hn_rndis_dump(const void *buf)
+{
+	const union {
+		struct rndis_msghdr hdr;
+		struct rndis_packet_msg pkt;
+		struct rndis_init_req init_request;
+		struct rndis_init_comp init_complete;
+		struct rndis_halt_req halt;
+		struct rndis_query_req query_request;
+		struct rndis_query_comp query_complete;
+		struct rndis_set_req set_request;
+		struct rndis_set_comp set_complete;
+		struct rndis_reset_req reset_request;
+		struct rndis_reset_comp reset_complete;
+		struct rndis_keepalive_req keepalive_request;
+		struct rndis_keepalive_comp keepalive_complete;
+		struct rndis_status_msg indicate_status;
+	} *rndis_msg = buf;
+
+	switch (rndis_msg->hdr.type) {
+	case RNDIS_PACKET_MSG: {
+		const struct rndis_pktinfo *ppi;
+		unsigned int ppi_len;
+
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_MSG_PACKET (len %u, data %u:%u, # oob %u %u:%u, pkt %u:%u)\n",
+			    rndis_msg->pkt.len,
+			    rndis_msg->pkt.dataoffset,
+			    rndis_msg->pkt.datalen,
+			    rndis_msg->pkt.oobdataelements,
+			    rndis_msg->pkt.oobdataoffset,
+			    rndis_msg->pkt.oobdatalen,
+			    rndis_msg->pkt.pktinfooffset,
+			    rndis_msg->pkt.pktinfolen);
+
+		ppi = (const struct rndis_pktinfo *)
+			((const char *)buf
+			 + RNDIS_PACKET_MSG_OFFSET_ABS(rndis_msg->pkt.pktinfooffset));
+
+		ppi_len = rndis_msg->pkt.pktinfolen;
+		while (ppi_len > 0) {
+			const void *ppi_data;
+
+			ppi_data = ppi->data;
+
+			rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+				"    PPI (size %u, type %u, offs %u data %#x)\n",
+				ppi->size, ppi->type, ppi->offset,
+				*(const uint32_t *)ppi_data);
+			if (ppi->size == 0)
+				break;
+			ppi_len -= ppi->size;
+			ppi = (const struct rndis_pktinfo *)
+				((const char *)ppi + ppi->size);
+		}
+		break;
+	}
+	case RNDIS_INITIALIZE_MSG:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_MSG_INIT (len %u id %#x, ver %u.%u max xfer %u)\n",
+			    rndis_msg->init_request.len,
+			    rndis_msg->init_request.rid,
+			    rndis_msg->init_request.ver_major,
+			    rndis_msg->init_request.ver_minor,
+			    rndis_msg->init_request.max_xfersz);
+		break;
+
+	case RNDIS_INITIALIZE_CMPLT:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_MSG_INIT_C (len %u, id %#x, status 0x%x, vers %u.%u, "
+			    "flags %d, max xfer %u, max pkts %u, aligned %u)\n",
+			    rndis_msg->init_complete.len,
+			    rndis_msg->init_complete.rid,
+			    rndis_msg->init_complete.status,
+			    rndis_msg->init_complete.ver_major,
+			    rndis_msg->init_complete.ver_minor,
+			    rndis_msg->init_complete.devflags,
+			    rndis_msg->init_complete.pktmaxsz,
+			    rndis_msg->init_complete.pktmaxcnt,
+			    rndis_msg->init_complete.align);
+		break;
+
+	case RNDIS_HALT_MSG:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_HALT (len %u id %#x)\n",
+			    rndis_msg->halt.len, rndis_msg->halt.rid);
+		break;
+
+	case RNDIS_QUERY_MSG:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_QUERY (len %u, id %#x, oid %#x, info %u:%u)\n",
+			    rndis_msg->query_request.len,
+			    rndis_msg->query_request.rid,
+			    rndis_msg->query_request.oid,
+			    rndis_msg->query_request.infobuflen,
+			    rndis_msg->query_request.infobufoffset);
+		break;
+
+	case RNDIS_QUERY_CMPLT:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_MSG_QUERY_C (len %u, id %#x, status 0x%x, buf %u:%u)\n",
+			    rndis_msg->query_complete.len,
+			    rndis_msg->query_complete.rid,
+			    rndis_msg->query_complete.status,
+			    rndis_msg->query_complete.infobuflen,
+			    rndis_msg->query_complete.infobufoffset);
+		break;
+
+	case RNDIS_SET_MSG:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_SET (len %u, id %#x, oid %#x, info %u:%u)\n",
+			    rndis_msg->set_request.len,
+			    rndis_msg->set_request.rid,
+			    rndis_msg->set_request.oid,
+			    rndis_msg->set_request.infobuflen,
+			    rndis_msg->set_request.infobufoffset);
+		break;
+
+	case RNDIS_SET_CMPLT:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)\n",
+			    rndis_msg->set_complete.len,
+			    rndis_msg->set_complete.rid,
+			    rndis_msg->set_complete.status);
+		break;
+
+	case RNDIS_INDICATE_STATUS_MSG:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_MSG_INDICATE (len %u, status %#x, buf len %u, buf offset %u)\n",
+			    rndis_msg->indicate_status.len,
+			    rndis_msg->indicate_status.status,
+			    rndis_msg->indicate_status.stbuflen,
+			    rndis_msg->indicate_status.stbufoffset);
+		break;
+
+	case RNDIS_RESET_MSG:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_RESET (len %u, id %#x)\n",
+			    rndis_msg->reset_request.len,
+			    rndis_msg->reset_request.rid);
+		break;
+
+	case RNDIS_RESET_CMPLT:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_RESET_C (len %u, status %#x address %#x)\n",
+			    rndis_msg->reset_complete.len,
+			    rndis_msg->reset_complete.status,
+			    rndis_msg->reset_complete.adrreset);
+		break;
+
+	case RNDIS_KEEPALIVE_MSG:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_KEEPALIVE (len %u, id %#x)\n",
+			    rndis_msg->keepalive_request.len,
+			    rndis_msg->keepalive_request.rid);
+		break;
+
+	case RNDIS_KEEPALIVE_CMPLT:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS_KEEPALIVE_C (len %u, id %#x address %#x)\n",
+			    rndis_msg->keepalive_complete.len,
+			    rndis_msg->keepalive_complete.rid,
+			    rndis_msg->keepalive_complete.status);
+		break;
+
+	default:
+		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
+			    "RNDIS type %#x len %u\n",
+			    rndis_msg->hdr.type,
+			    rndis_msg->hdr.len);
+		break;
+	}
+}
+#endif
+
+static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan,
+				  const void *req, uint32_t reqlen)
+
+{
+	struct hn_nvs_rndis nvs_rndis = {
+		.type = NVS_TYPE_RNDIS,
+		.rndis_mtype = NVS_RNDIS_MTYPE_CTRL,
+		.chim_idx = NVS_CHIM_IDX_INVALID,
+		.chim_sz = 0
+	};
+	struct vmbus_gpa sg;
+	rte_iova_t addr;
+
+	addr = rte_malloc_virt2iova(req);
+	if (unlikely(addr == RTE_BAD_IOVA)) {
+		PMD_DRV_LOG(ERR, "RNDIS send request can not get iova");
+		return -EINVAL;
+	}
+
+	if (unlikely(reqlen > PAGE_SIZE)) {
+		PMD_DRV_LOG(ERR, "RNDIS request %u greater than page size",
+			    reqlen);
+		return -EINVAL;
+	}
+
+	sg.page = addr / PAGE_SIZE;
+	sg.ofs  = addr & PAGE_MASK;
+	sg.len  = reqlen;
+
+	if (sg.ofs + reqlen >  PAGE_SIZE) {
+		PMD_DRV_LOG(ERR, "RNDIS request crosses page bounary");
+		return -EINVAL;
+	}
+
+	hn_rndis_dump(req);
+
+	return hn_nvs_send_sglist(chan, &sg, 1,
+				  &nvs_rndis, sizeof(nvs_rndis), 0U, NULL);
+}
+
+void hn_rndis_link_status(struct hn_data *hv __rte_unused, const void *msg)
+{
+	const struct rndis_status_msg *indicate = msg;
+
+	hn_rndis_dump(msg);
+
+	PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status);
+
+	switch (indicate->status) {
+	case RNDIS_STATUS_LINK_SPEED_CHANGE:
+	case RNDIS_STATUS_NETWORK_CHANGE:
+	case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG:
+		/* ignore not in DPDK API */
+		break;
+
+	case RNDIS_STATUS_MEDIA_CONNECT:
+	case RNDIS_STATUS_MEDIA_DISCONNECT:
+		/* TODO handle as LSC interrupt  */
+		break;
+	default:
+		PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x",
+			    indicate->status);
+	}
+}
+
+/* Callback from hn_process_events when response is visible */
+void hn_rndis_receive_response(struct hn_data *hv,
+			       const void *data, uint32_t len)
+{
+	const struct rndis_init_comp *hdr = data;
+
+	hn_rndis_dump(data);
+
+	if (len < sizeof(3 * sizeof(uint32_t))) {
+		PMD_DRV_LOG(ERR,
+			    "missing RNDIS header %u", len);
+		return;
+	}
+
+	if (len < hdr->len) {
+		PMD_DRV_LOG(ERR,
+			    "truncated RNDIS response %u", len);
+		return;
+	}
+
+	if  (len > sizeof(hv->rndis_resp)) {
+		PMD_DRV_LOG(NOTICE,
+			    "RNDIS response exceeds buffer");
+		len = sizeof(hv->rndis_resp);
+	}
+
+	if (hdr->rid == 0) {
+		PMD_DRV_LOG(NOTICE,
+			    "RNDIS response id zero!");
+	}
+
+	memcpy(hv->rndis_resp, data, len);
+
+	/* make sure response copied before update */
+	rte_smp_wmb();
+
+	if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) {
+		PMD_DRV_LOG(ERR,
+			    "received id %#x pending id %#x",
+			    hdr->rid, (uint32_t)hv->rndis_pending);
+	}
+}
+
+/* Do request/response transaction */
+static int hn_rndis_exec1(struct hn_data *hv,
+			  const void *req, uint32_t reqlen,
+			  void *comp, uint32_t comp_len)
+{
+	const struct rndis_halt_req *hdr = req;
+	uint32_t rid = hdr->rid;
+	struct vmbus_channel *chan = hn_primary_chan(hv);
+	int error;
+
+	if (comp_len > sizeof(hv->rndis_resp)) {
+		PMD_DRV_LOG(ERR,
+			    "Expected completion size %u exceeds buffer %zu",
+			    comp_len, sizeof(hv->rndis_resp));
+		return -EIO;
+	}
+
+	if (comp != NULL &&
+	    rte_atomic32_cmpset(&hv->rndis_pending, 0, rid) == 0) {
+		PMD_DRV_LOG(ERR,
+			    "Request already pending");
+		return -EBUSY;
+	}
+
+	error = hn_nvs_send_rndis_ctrl(chan, req, reqlen);
+	if (error) {
+		PMD_DRV_LOG(ERR, "RNDIS ctrl send failed: %d", error);
+		return error;
+	}
+
+	if (comp) {
+		/* Poll primary channel until response received */
+		while (hv->rndis_pending == rid)
+			hn_process_events(hv, 0);
+
+		memcpy(comp, hv->rndis_resp, comp_len);
+	}
+
+	return 0;
+}
+
+/* Do transaction and validate response */
+static int hn_rndis_execute(struct hn_data *hv, uint32_t rid,
+			    const void *req, uint32_t reqlen,
+			    void *comp, uint32_t comp_len, uint32_t comp_type)
+{
+	const struct rndis_comp_hdr *hdr = comp;
+	int ret;
+
+	memset(comp, 0, comp_len);
+
+	ret = hn_rndis_exec1(hv, req, reqlen, comp, comp_len);
+	if (ret < 0)
+		return ret;
+	/*
+	 * Check this RNDIS complete message.
+	 */
+	if (unlikely(hdr->type != comp_type)) {
+		PMD_DRV_LOG(ERR,
+			    "unexpected RNDIS response complete %#x expect %#x",
+			    hdr->type, comp_type);
+
+		return -ENXIO;
+	}
+	if (unlikely(hdr->rid != rid)) {
+		PMD_DRV_LOG(ERR,
+			    "RNDIS comp rid mismatch %#x, expect %#x",
+			    hdr->rid, rid);
+		return -EINVAL;
+	}
+
+	/* All pass! */
+	return 0;
+}
+
+static int
+hn_rndis_query(struct hn_data *hv, uint32_t oid,
+	       const void *idata, uint32_t idlen,
+	       void *odata, uint32_t odlen)
+{
+	struct rndis_query_req *req;
+	struct rndis_query_comp *comp;
+	uint32_t reqlen, comp_len;
+	int error = -EIO;
+	unsigned int ofs;
+	uint32_t rid;
+
+	reqlen = sizeof(*req) + idlen;
+	req = hn_rndis_alloc(hv, reqlen);
+	if (req == NULL)
+		return -ENOMEM;
+
+	comp_len = sizeof(*comp) + odlen;
+	comp = rte_zmalloc("QUERY", comp_len, PAGE_SIZE);
+	if (!comp) {
+		error = -ENOMEM;
+		goto done;
+	}
+	comp->status = RNDIS_STATUS_PENDING;
+
+	rid = hn_rndis_rid(hv);
+
+	req->type = RNDIS_QUERY_MSG;
+	req->len = reqlen;
+	req->rid = rid;
+	req->oid = oid;
+	req->infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET;
+	req->infobuflen = idlen;
+
+	/* Input data immediately follows RNDIS query. */
+	memcpy(req + 1, idata, idlen);
+
+	error = hn_rndis_execute(hv, rid, req, reqlen,
+				 comp, comp_len, RNDIS_QUERY_CMPLT);
+
+	if (error)
+		goto done;
+
+	if (comp->status != RNDIS_STATUS_SUCCESS) {
+		PMD_DRV_LOG(ERR, "RNDIS query 0x%08x failed: status 0x%08x",
+			    oid, comp->status);
+		error = -EINVAL;
+		goto done;
+	}
+
+	if (comp->infobuflen == 0 || comp->infobufoffset == 0) {
+		/* No output data! */
+		PMD_DRV_LOG(ERR, "RNDIS query 0x%08x, no data", oid);
+		error = 0;
+		goto done;
+	}
+
+	/*
+	 * Check output data length and offset.
+	 */
+	/* ofs is the offset from the beginning of comp. */
+	ofs = RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(comp->infobufoffset);
+	if (ofs < sizeof(*comp) || ofs + comp->infobuflen > comp_len) {
+		PMD_DRV_LOG(ERR, "RNDIS query invalid comp ib off/len, %u/%u",
+			    comp->infobufoffset, comp->infobuflen);
+		error = -EINVAL;
+		goto done;
+	}
+
+	/* Save output data. */
+	if (comp->infobuflen < odlen)
+		odlen = comp->infobuflen;
+
+	/* ofs is the offset from the beginning of comp. */
+	memcpy(odata, (const char *)comp + ofs, odlen);
+
+	error = 0;
+done:
+	rte_free(comp);
+	rte_free(req);
+	return error;
+}
+
+static int
+hn_rndis_halt(struct hn_data *hv)
+{
+	struct rndis_halt_req *halt;
+
+	halt = hn_rndis_alloc(hv, sizeof(*halt));
+	if (halt == NULL)
+		return -ENOMEM;
+
+	halt->type = RNDIS_HALT_MSG;
+	halt->len = sizeof(*halt);
+	halt->rid = hn_rndis_rid(hv);
+
+	/* No RNDIS completion; rely on NVS message send completion */
+	hn_rndis_exec1(hv, halt, sizeof(*halt), NULL, 0);
+
+	rte_free(halt);
+
+	PMD_INIT_LOG(DEBUG, "RNDIS halt done");
+	return 0;
+}
+
+static int
+hn_rndis_query_hwcaps(struct hn_data *hv, struct ndis_offload *caps)
+{
+	struct ndis_offload in;
+	uint32_t caps_len, size;
+	int error;
+
+	memset(caps, 0, sizeof(*caps));
+	memset(&in, 0, sizeof(in));
+	in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD;
+
+	if (hv->ndis_ver >= NDIS_VERSION_6_30) {
+		in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3;
+		size = NDIS_OFFLOAD_SIZE;
+	} else if (hv->ndis_ver >= NDIS_VERSION_6_1) {
+		in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2;
+		size = NDIS_OFFLOAD_SIZE_6_1;
+	} else {
+		in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1;
+		size = NDIS_OFFLOAD_SIZE_6_0;
+	}
+	in.ndis_hdr.ndis_size = size;
+
+	caps_len = NDIS_OFFLOAD_SIZE;
+	error = hn_rndis_query(hv, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES,
+			       &in, size, caps, caps_len);
+	if (error)
+		return error;
+
+	/* Preliminary verification. */
+	if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) {
+		PMD_DRV_LOG(NOTICE, "invalid NDIS objtype 0x%02x",
+			    caps->ndis_hdr.ndis_type);
+		return -EINVAL;
+	}
+	if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) {
+		PMD_DRV_LOG(NOTICE, "invalid NDIS objrev 0x%02x",
+			    caps->ndis_hdr.ndis_rev);
+		return -EINVAL;
+	}
+	if (caps->ndis_hdr.ndis_size > caps_len) {
+		PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u, data size %u",
+			    caps->ndis_hdr.ndis_size, caps_len);
+		return -EINVAL;
+	} else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) {
+		PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u",
+			    caps->ndis_hdr.ndis_size);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int
+hn_rndis_query_rsscaps(struct hn_data *hv,
+		       unsigned int *rxr_cnt0)
+{
+	struct ndis_rss_caps in, caps;
+	unsigned int indsz, rxr_cnt;
+	uint32_t caps_len;
+	int error;
+
+	*rxr_cnt0 = 0;
+
+	if (hv->ndis_ver < NDIS_VERSION_6_20) {
+		PMD_DRV_LOG(DEBUG, "RSS not supported on this host");
+		return -EOPNOTSUPP;
+	}
+
+	memset(&in, 0, sizeof(in));
+	in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS;
+	in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2;
+	in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE;
+
+	caps_len = NDIS_RSS_CAPS_SIZE;
+	error = hn_rndis_query(hv, OID_GEN_RECEIVE_SCALE_CAPABILITIES,
+			       &in, NDIS_RSS_CAPS_SIZE,
+			       &caps, caps_len);
+	if (error)
+		return error;
+
+	PMD_INIT_LOG(DEBUG, "RX rings %u indirect %u caps %#x",
+		     caps.ndis_nrxr, caps.ndis_nind, caps.ndis_caps);
+	/*
+	 * Preliminary verification.
+	 */
+	if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) {
+		PMD_DRV_LOG(ERR, "invalid NDIS objtype 0x%02x",
+			    caps.ndis_hdr.ndis_type);
+		return -EINVAL;
+	}
+	if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) {
+		PMD_DRV_LOG(ERR, "invalid NDIS objrev 0x%02x",
+			    caps.ndis_hdr.ndis_rev);
+		return -EINVAL;
+	}
+	if (caps.ndis_hdr.ndis_size > caps_len) {
+		PMD_DRV_LOG(ERR,
+			    "invalid NDIS objsize %u, data size %u",
+			    caps.ndis_hdr.ndis_size, caps_len);
+		return -EINVAL;
+	} else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) {
+		PMD_DRV_LOG(ERR, "invalid NDIS objsize %u",
+			    caps.ndis_hdr.ndis_size);
+		return -EINVAL;
+	}
+
+	/*
+	 * Save information for later RSS configuration.
+	 */
+	if (caps.ndis_nrxr == 0) {
+		PMD_DRV_LOG(ERR, "0 RX rings!?");
+		return -EINVAL;
+	}
+	rxr_cnt = caps.ndis_nrxr;
+
+	if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE &&
+	    caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) {
+		if (caps.ndis_nind > NDIS_HASH_INDCNT) {
+			PMD_DRV_LOG(ERR,
+				    "too many RSS indirect table entries %u",
+				    caps.ndis_nind);
+			return -EOPNOTSUPP;
+		}
+		if (!rte_is_power_of_2(caps.ndis_nind)) {
+			PMD_DRV_LOG(ERR,
+				    "RSS indirect table size is not power-of-2 %u",
+				    caps.ndis_nind);
+		}
+
+		indsz = caps.ndis_nind;
+	} else {
+		indsz = NDIS_HASH_INDCNT;
+	}
+
+	if (indsz < rxr_cnt) {
+		PMD_DRV_LOG(NOTICE,
+			    "# of RX rings (%d) > RSS indirect table size %d",
+			    rxr_cnt, indsz);
+		rxr_cnt = indsz;
+	}
+
+	hv->rss_offloads = 0;
+	if (caps.ndis_caps & NDIS_RSS_CAP_IPV4)
+		hv->rss_offloads |= ETH_RSS_IPV4
+			| ETH_RSS_NONFRAG_IPV4_TCP
+			| ETH_RSS_NONFRAG_IPV4_UDP;
+	if (caps.ndis_caps & NDIS_RSS_CAP_IPV6)
+		hv->rss_offloads |= ETH_RSS_IPV6
+			| ETH_RSS_NONFRAG_IPV6_TCP;
+	if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX)
+		hv->rss_offloads |= ETH_RSS_IPV6_EX
+			| ETH_RSS_IPV6_TCP_EX;
+
+	/* Commit! */
+	*rxr_cnt0 = rxr_cnt;
+
+	return 0;
+}
+
+static int
+hn_rndis_set(struct hn_data *hv, uint32_t oid, const void *data, uint32_t dlen)
+{
+	struct rndis_set_req *req;
+	struct rndis_set_comp comp;
+	uint32_t reqlen, comp_len;
+	uint32_t rid;
+	int error;
+
+	reqlen = sizeof(*req) + dlen;
+	req = rte_zmalloc("RNDIS_SET", reqlen, PAGE_SIZE);
+	if (!req)
+		return -ENOMEM;
+
+	rid = hn_rndis_rid(hv);
+	req->type = RNDIS_SET_MSG;
+	req->len = reqlen;
+	req->rid = rid;
+	req->oid = oid;
+	req->infobuflen = dlen;
+	req->infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET;
+
+	/* Data immediately follows RNDIS set. */
+	memcpy(req + 1, data, dlen);
+
+	comp_len = sizeof(comp);
+	error = hn_rndis_execute(hv, rid, req, reqlen,
+				 &comp, comp_len,
+				 RNDIS_SET_CMPLT);
+	if (error) {
+		PMD_DRV_LOG(ERR, "exec RNDIS set %#" PRIx32 " failed",
+			    oid);
+		error = EIO;
+		goto done;
+	}
+
+	if (comp.status != RNDIS_STATUS_SUCCESS) {
+		PMD_DRV_LOG(ERR,
+			    "RNDIS set %#" PRIx32 " failed: status %#" PRIx32,
+			    oid, comp.status);
+		error = EIO;
+		goto done;
+	}
+
+done:
+	rte_free(req);
+	return error;
+}
+
+int hn_rndis_conf_offload(struct hn_data *hv,
+			  uint64_t tx_offloads, uint64_t rx_offloads)
+{
+	struct ndis_offload_params params;
+	struct ndis_offload hwcaps;
+	int error;
+
+	error = hn_rndis_query_hwcaps(hv, &hwcaps);
+	if (error) {
+		PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
+		return error;
+	}
+
+	/* NOTE: 0 means "no change" */
+	memset(&params, 0, sizeof(params));
+
+	params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT;
+	if (hv->ndis_ver < NDIS_VERSION_6_30) {
+		params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2;
+		params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE_6_1;
+	} else {
+		params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3;
+		params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE;
+	}
+
+	if (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) {
+		if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_TCP4)
+			params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX;
+		else
+			goto unsupported;
+
+		if (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_TCP6)
+			params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX;
+		else
+			goto unsupported;
+	}
+
+	if (rx_offloads & DEV_RX_OFFLOAD_TCP_CKSUM) {
+		if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4)
+		    == NDIS_RXCSUM_CAP_TCP4)
+			params.ndis_tcp4csum |= NDIS_OFFLOAD_PARAM_RX;
+		else
+			goto unsupported;
+
+		if ((hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)
+		    == NDIS_RXCSUM_CAP_TCP6)
+			params.ndis_tcp6csum |= NDIS_OFFLOAD_PARAM_RX;
+		else
+			goto unsupported;
+	}
+
+	if (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) {
+		if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4)
+			params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX;
+		else
+			goto unsupported;
+
+		if ((hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6)
+		    == NDIS_TXCSUM_CAP_UDP6)
+			params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX;
+		else
+			goto unsupported;
+	}
+
+	if (rx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) {
+		if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4)
+			params.ndis_udp4csum |= NDIS_OFFLOAD_PARAM_RX;
+		else
+			goto unsupported;
+
+		if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)
+			params.ndis_udp6csum |= NDIS_OFFLOAD_PARAM_RX;
+		else
+			goto unsupported;
+	}
+
+	if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) {
+		if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_IP4)
+		    == NDIS_TXCSUM_CAP_IP4)
+			params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX;
+		else
+			goto unsupported;
+	}
+	if (rx_offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) {
+		if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
+			params.ndis_ip4csum |= NDIS_OFFLOAD_PARAM_RX;
+		else
+			goto unsupported;
+	}
+
+	if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) {
+		if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023)
+			params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON;
+		else
+			goto unsupported;
+
+		if ((hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6)
+		    == HN_NDIS_LSOV2_CAP_IP6)
+			params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON;
+		else
+			goto unsupported;
+	}
+
+	error = hn_rndis_set(hv, OID_TCP_OFFLOAD_PARAMETERS, &params,
+			     params.ndis_hdr.ndis_size);
+	if (error) {
+		PMD_DRV_LOG(ERR, "offload config failed");
+		return error;
+	}
+
+	return 0;
+ unsupported:
+	PMD_DRV_LOG(NOTICE,
+		    "offload tx:%" PRIx64 " rx:%" PRIx64 " not supported by this version",
+		    tx_offloads, rx_offloads);
+	return -EINVAL;
+}
+
+int hn_rndis_get_offload(struct hn_data *hv,
+			 struct rte_eth_dev_info *dev_info)
+{
+	struct ndis_offload hwcaps;
+	int error;
+
+	memset(&hwcaps, 0, sizeof(hwcaps));
+
+	error = hn_rndis_query_hwcaps(hv, &hwcaps);
+	if (error) {
+		PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
+		return error;
+	}
+
+	dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
+				    DEV_TX_OFFLOAD_VLAN_INSERT;
+
+	if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_IP4)
+	    == HN_NDIS_TXCSUM_CAP_IP4)
+		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM;
+
+	if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_TCP4)
+	    == HN_NDIS_TXCSUM_CAP_TCP4 &&
+	    (hwcaps.ndis_csum.ndis_ip6_txcsum & HN_NDIS_TXCSUM_CAP_TCP6)
+	    == HN_NDIS_TXCSUM_CAP_TCP6)
+		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_CKSUM;
+
+	if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) &&
+	    (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6))
+		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_UDP_CKSUM;
+
+	if ((hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) &&
+	    (hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6)
+	    == HN_NDIS_LSOV2_CAP_IP6)
+		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
+
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
+				    DEV_RX_OFFLOAD_CRC_STRIP;
+
+	if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
+		dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM;
+
+	if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) &&
+	    (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6))
+		dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_CKSUM;
+
+	if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) &&
+	    (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6))
+		dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_UDP_CKSUM;
+
+	return 0;
+}
+
+int
+hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter)
+{
+	int error;
+
+	error = hn_rndis_set(hv, OID_GEN_CURRENT_PACKET_FILTER,
+			     &filter, sizeof(filter));
+	if (error) {
+		PMD_DRV_LOG(ERR, "set RX filter %#" PRIx32 " failed: %d",
+			    filter, error);
+	} else {
+		PMD_DRV_LOG(DEBUG, "set RX filter %#" PRIx32 " done", filter);
+	}
+
+	return error;
+}
+
+/* The default RSS key.
+ * This value is the same as MLX5 so that flows will be
+ * received on same path for both VF ans synthetic NIC.
+ */
+static const uint8_t rss_default_key[NDIS_HASH_KEYSIZE_TOEPLITZ] = {
+	0x2c, 0xc6, 0x81, 0xd1,	0x5b, 0xdb, 0xf4, 0xf7,
+	0xfc, 0xa2, 0x83, 0x19,	0xdb, 0x1a, 0x3e, 0x94,
+	0x6b, 0x9e, 0x38, 0xd9,	0x2c, 0x9c, 0x03, 0xd1,
+	0xad, 0x99, 0x44, 0xa7,	0xd9, 0x56, 0x3d, 0x59,
+	0x06, 0x3c, 0x25, 0xf3,	0xfc, 0x1f, 0xdc, 0x2a,
+};
+
+int hn_rndis_conf_rss(struct hn_data *hv,
+		      const struct rte_eth_rss_conf *rss_conf)
+{
+	struct ndis_rssprm_toeplitz rssp;
+	struct ndis_rss_params *prm = &rssp.rss_params;
+	const uint8_t *rss_key = rss_conf->rss_key ? : rss_default_key;
+	uint32_t rss_hash;
+	unsigned int i;
+	int error;
+
+	PMD_INIT_FUNC_TRACE();
+
+	memset(&rssp, 0, sizeof(rssp));
+
+	prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS;
+	prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2;
+	prm->ndis_hdr.ndis_size = sizeof(*prm);
+	prm->ndis_flags = 0;
+
+	rss_hash = NDIS_HASH_FUNCTION_TOEPLITZ;
+	if (rss_conf->rss_hf & ETH_RSS_IPV4)
+		rss_hash |= NDIS_HASH_IPV4;
+	if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
+		rss_hash |= NDIS_HASH_TCP_IPV4;
+	if (rss_conf->rss_hf & ETH_RSS_IPV6)
+		rss_hash |=  NDIS_HASH_IPV6;
+	if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
+		rss_hash |= NDIS_HASH_TCP_IPV6;
+
+	prm->ndis_hash = rss_hash;
+	prm->ndis_indsize = sizeof(rssp.rss_ind[0]) * NDIS_HASH_INDCNT;
+	prm->ndis_indoffset = offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]);
+	prm->ndis_keysize = NDIS_HASH_KEYSIZE_TOEPLITZ;
+	prm->ndis_keyoffset = offsetof(struct ndis_rssprm_toeplitz, rss_key[0]);
+
+	for (i = 0; i < NDIS_HASH_INDCNT; i++)
+		rssp.rss_ind[i] = i % hv->num_queues;
+
+	/* Set hask key values */
+	memcpy(&rssp.rss_key, rss_key, NDIS_HASH_KEYSIZE_TOEPLITZ);
+
+	error = hn_rndis_set(hv, OID_GEN_RECEIVE_SCALE_PARAMETERS,
+			     &rssp, sizeof(rssp));
+	if (error) {
+		PMD_DRV_LOG(ERR,
+			    "RSS config num queues=%u failed: %d",
+			    hv->num_queues, error);
+	}
+	return error;
+}
+
+static int hn_rndis_init(struct hn_data *hv)
+{
+	struct rndis_init_req *req;
+	struct rndis_init_comp comp;
+	uint32_t comp_len, rid;
+	int error;
+
+	req = hn_rndis_alloc(hv, sizeof(*req));
+	if (!req) {
+		PMD_DRV_LOG(ERR, "no memory for RNDIS init");
+		return -ENXIO;
+	}
+
+	rid = hn_rndis_rid(hv);
+	req->type = RNDIS_INITIALIZE_MSG;
+	req->len = sizeof(*req);
+	req->rid = rid;
+	req->ver_major = RNDIS_VERSION_MAJOR;
+	req->ver_minor = RNDIS_VERSION_MINOR;
+	req->max_xfersz = HN_RNDIS_XFER_SIZE;
+
+	comp_len = RNDIS_INIT_COMP_SIZE_MIN;
+	error = hn_rndis_execute(hv, rid, req, sizeof(*req),
+				 &comp, comp_len,
+				 RNDIS_INITIALIZE_CMPLT);
+	if (error)
+		goto done;
+
+	if (comp.status != RNDIS_STATUS_SUCCESS) {
+		PMD_DRV_LOG(ERR, "RNDIS init failed: status 0x%08x",
+			    comp.status);
+		error = -EIO;
+		goto done;
+	}
+
+	hv->rndis_agg_size = comp.pktmaxsz;
+	hv->rndis_agg_pkts = comp.pktmaxcnt;
+	hv->rndis_agg_align = 1U << comp.align;
+
+	if (hv->rndis_agg_align < sizeof(uint32_t)) {
+		/*
+		 * The RNDIS packet message encap assumes that the RNDIS
+		 * packet message is at least 4 bytes aligned.  Fix up the
+		 * alignment here, if the remote side sets the alignment
+		 * too low.
+		 */
+		PMD_DRV_LOG(NOTICE,
+			    "fixup RNDIS aggpkt align: %u -> %zu",
+			    hv->rndis_agg_align, sizeof(uint32_t));
+		hv->rndis_agg_align = sizeof(uint32_t);
+	}
+
+	PMD_INIT_LOG(INFO,
+		     "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u",
+		     comp.ver_major, comp.ver_minor,
+		     hv->rndis_agg_size, hv->rndis_agg_pkts,
+		     hv->rndis_agg_align);
+	error = 0;
+done:
+	rte_free(req);
+	return error;
+}
+
+int
+hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr)
+{
+	uint32_t eaddr_len;
+	int error;
+
+	eaddr_len = ETHER_ADDR_LEN;
+	error = hn_rndis_query(hv, OID_802_3_PERMANENT_ADDRESS, NULL, 0,
+			       eaddr, eaddr_len);
+	if (error)
+		return error;
+
+	PMD_DRV_LOG(INFO, "MAC address %02x:%02x:%02x:%02x:%02x:%02x",
+		    eaddr[0], eaddr[1], eaddr[2],
+		    eaddr[3], eaddr[4], eaddr[5]);
+	return 0;
+}
+
+int
+hn_rndis_get_linkstatus(struct hn_data *hv)
+{
+	return hn_rndis_query(hv, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0,
+			      &hv->link_status, sizeof(uint32_t));
+}
+
+int
+hn_rndis_get_linkspeed(struct hn_data *hv)
+{
+	return hn_rndis_query(hv, OID_GEN_LINK_SPEED, NULL, 0,
+			      &hv->link_speed, sizeof(uint32_t));
+}
+
+int
+hn_rndis_attach(struct hn_data *hv)
+{
+	/* Initialize RNDIS. */
+	return hn_rndis_init(hv);
+}
+
+void
+hn_rndis_detach(struct hn_data *hv)
+{
+	/* Halt the RNDIS. */
+	hn_rndis_halt(hv);
+}
diff --git a/drivers/net/netvsc/hn_rndis.h b/drivers/net/netvsc/hn_rndis.h
new file mode 100644
index 00000000..89e2e6ba
--- /dev/null
+++ b/drivers/net/netvsc/hn_rndis.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+
+#include "rndis.h"
+
+struct hn_data;
+
+void hn_rndis_receive_response(struct hn_data *hv,
+			      const void *data, uint32_t len);
+void	hn_rndis_link_status(struct hn_data *hv, const void *data);
+int	hn_rndis_attach(struct hn_data *hv);
+void	hn_rndis_detach(struct hn_data *hv);
+int	hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr);
+int	hn_rndis_get_linkstatus(struct hn_data *hv);
+int	hn_rndis_get_linkspeed(struct hn_data *hv);
+int	hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter);
+void	hn_rndis_rx_ctrl(struct hn_data *hv, const void *data,
+			 int dlen);
+int	hn_rndis_get_offload(struct hn_data *hv,
+			     struct rte_eth_dev_info *dev_info);
+int	hn_rndis_conf_offload(struct hn_data *hv,
+			      uint64_t tx_offloads,
+			      uint64_t rx_offloads);
+int	hn_rndis_query_rsscaps(struct hn_data *hv,
+			       unsigned int *rxr_cnt0);
+int	hn_rndis_conf_rss(struct hn_data *hv,
+			  const struct rte_eth_rss_conf *rss_conf);
+
+#ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP
+void hn_rndis_dump(const void *buf);
+#else
+#define hn_rndis_dump(buf)
+#endif
diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
new file mode 100644
index 00000000..02ef27e3
--- /dev/null
+++ b/drivers/net/netvsc/hn_rxtx.c
@@ -0,0 +1,1334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2018 Microsoft Corporation
+ * Copyright(c) 2013-2016 Brocade Communications Systems, Inc.
+ * All rights reserved.
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <unistd.h>
+#include <strings.h>
+
+#include <rte_ethdev.h>
+#include <rte_memcpy.h>
+#include <rte_string_fns.h>
+#include <rte_memzone.h>
+#include <rte_malloc.h>
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_ether.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_dev.h>
+#include <rte_net.h>
+#include <rte_bus_vmbus.h>
+#include <rte_spinlock.h>
+
+#include "hn_logs.h"
+#include "hn_var.h"
+#include "hn_rndis.h"
+#include "hn_nvs.h"
+#include "ndis.h"
+
+#define HN_NVS_SEND_MSG_SIZE \
+	(sizeof(struct vmbus_chanpkt_hdr) + sizeof(struct hn_nvs_rndis))
+
+#define HN_TXD_CACHE_SIZE	32 /* per cpu tx_descriptor pool cache */
+#define HN_TXCOPY_THRESHOLD	512
+
+#define HN_RXCOPY_THRESHOLD	256
+#define HN_RXQ_EVENT_DEFAULT	2048
+
+struct hn_rxinfo {
+	uint32_t	vlan_info;
+	uint32_t	csum_info;
+	uint32_t	hash_info;
+	uint32_t	hash_value;
+};
+
+#define HN_RXINFO_VLAN			0x0001
+#define HN_RXINFO_CSUM			0x0002
+#define HN_RXINFO_HASHINF		0x0004
+#define HN_RXINFO_HASHVAL		0x0008
+#define HN_RXINFO_ALL			\
+	(HN_RXINFO_VLAN |		\
+	 HN_RXINFO_CSUM |		\
+	 HN_RXINFO_HASHINF |		\
+	 HN_RXINFO_HASHVAL)
+
+#define HN_NDIS_VLAN_INFO_INVALID	0xffffffff
+#define HN_NDIS_RXCSUM_INFO_INVALID	0
+#define HN_NDIS_HASH_INFO_INVALID	0
+
+/*
+ * Per-transmit book keeping.
+ * A slot in transmit ring (chim_index) is reserved for each transmit.
+ *
+ * There are two types of transmit:
+ *   - buffered transmit where chimney buffer is used and RNDIS header
+ *     is in the buffer. mbuf == NULL for this case.
+ *
+ *   - direct transmit where RNDIS header is in the in  rndis_pkt
+ *     mbuf is freed after transmit.
+ *
+ * Descriptors come from per-port pool which is used
+ * to limit number of outstanding requests per device.
+ */
+struct hn_txdesc {
+	struct rte_mbuf *m;
+
+	uint16_t	queue_id;
+	uint16_t	chim_index;
+	uint32_t	chim_size;
+	uint32_t	data_size;
+	uint32_t	packets;
+
+	struct rndis_packet_msg *rndis_pkt;
+};
+
+#define HN_RNDIS_PKT_LEN				\
+	(sizeof(struct rndis_packet_msg) +		\
+	 RNDIS_PKTINFO_SIZE(NDIS_HASH_VALUE_SIZE) +	\
+	 RNDIS_PKTINFO_SIZE(NDIS_VLAN_INFO_SIZE) +	\
+	 RNDIS_PKTINFO_SIZE(NDIS_LSO2_INFO_SIZE) +	\
+	 RNDIS_PKTINFO_SIZE(NDIS_TXCSUM_INFO_SIZE))
+
+/* Minimum space required for a packet */
+#define HN_PKTSIZE_MIN(align) \
+	RTE_ALIGN(ETHER_MIN_LEN + HN_RNDIS_PKT_LEN, align)
+
+#define DEFAULT_TX_FREE_THRESH 32U
+
+static void
+hn_update_packet_stats(struct hn_stats *stats, const struct rte_mbuf *m)
+{
+	uint32_t s = m->pkt_len;
+	const struct ether_addr *ea;
+
+	if (s == 64) {
+		stats->size_bins[1]++;
+	} else if (s > 64 && s < 1024) {
+		uint32_t bin;
+
+		/* count zeros, and offset into correct bin */
+		bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
+		stats->size_bins[bin]++;
+	} else {
+		if (s < 64)
+			stats->size_bins[0]++;
+		else if (s < 1519)
+			stats->size_bins[6]++;
+		else if (s >= 1519)
+			stats->size_bins[7]++;
+	}
+
+	ea = rte_pktmbuf_mtod(m, const struct ether_addr *);
+	if (is_multicast_ether_addr(ea)) {
+		if (is_broadcast_ether_addr(ea))
+			stats->broadcast++;
+		else
+			stats->multicast++;
+	}
+}
+
+static inline unsigned int hn_rndis_pktlen(const struct rndis_packet_msg *pkt)
+{
+	return pkt->pktinfooffset + pkt->pktinfolen;
+}
+
+static inline uint32_t
+hn_rndis_pktmsg_offset(uint32_t ofs)
+{
+	return ofs - offsetof(struct rndis_packet_msg, dataoffset);
+}
+
+static void hn_txd_init(struct rte_mempool *mp __rte_unused,
+			void *opaque, void *obj, unsigned int idx)
+{
+	struct hn_txdesc *txd = obj;
+	struct rte_eth_dev *dev = opaque;
+	struct rndis_packet_msg *pkt;
+
+	memset(txd, 0, sizeof(*txd));
+	txd->chim_index = idx;
+
+	pkt = rte_malloc_socket("RNDIS_TX", HN_RNDIS_PKT_LEN,
+				rte_align32pow2(HN_RNDIS_PKT_LEN),
+				dev->device->numa_node);
+	if (!pkt)
+		rte_exit(EXIT_FAILURE, "can not allocate RNDIS header");
+
+	txd->rndis_pkt = pkt;
+}
+
+/*
+ * Unlike Linux and FreeBSD, this driver uses a mempool
+ * to limit outstanding transmits and reserve buffers
+ */
+int
+hn_tx_pool_init(struct rte_eth_dev *dev)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	char name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *mp;
+
+	snprintf(name, sizeof(name),
+		 "hn_txd_%u", dev->data->port_id);
+
+	PMD_INIT_LOG(DEBUG, "create a TX send pool %s n=%u size=%zu socket=%d",
+		     name, hv->chim_cnt, sizeof(struct hn_txdesc),
+		     dev->device->numa_node);
+
+	mp = rte_mempool_create(name, hv->chim_cnt, sizeof(struct hn_txdesc),
+				HN_TXD_CACHE_SIZE, 0,
+				NULL, NULL,
+				hn_txd_init, dev,
+				dev->device->numa_node, 0);
+	if (!mp) {
+		PMD_DRV_LOG(ERR,
+			    "mempool %s create failed: %d", name, rte_errno);
+		return -rte_errno;
+	}
+
+	hv->tx_pool = mp;
+	return 0;
+}
+
+static void hn_reset_txagg(struct hn_tx_queue *txq)
+{
+	txq->agg_szleft = txq->agg_szmax;
+	txq->agg_pktleft = txq->agg_pktmax;
+	txq->agg_txd = NULL;
+	txq->agg_prevpkt = NULL;
+}
+
+int
+hn_dev_tx_queue_setup(struct rte_eth_dev *dev,
+		      uint16_t queue_idx, uint16_t nb_desc __rte_unused,
+		      unsigned int socket_id,
+		      const struct rte_eth_txconf *tx_conf)
+
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct hn_tx_queue *txq;
+	uint32_t tx_free_thresh;
+
+	PMD_INIT_FUNC_TRACE();
+
+	txq = rte_zmalloc_socket("HN_TXQ", sizeof(*txq), RTE_CACHE_LINE_SIZE,
+				 socket_id);
+	if (!txq)
+		return -ENOMEM;
+
+	txq->hv = hv;
+	txq->chan = hv->channels[queue_idx];
+	txq->port_id = dev->data->port_id;
+	txq->queue_id = queue_idx;
+
+	tx_free_thresh = tx_conf->tx_free_thresh;
+	if (tx_free_thresh == 0)
+		tx_free_thresh = RTE_MIN(hv->chim_cnt / 4,
+					 DEFAULT_TX_FREE_THRESH);
+
+	if (tx_free_thresh >= hv->chim_cnt - 3)
+		tx_free_thresh = hv->chim_cnt - 3;
+
+	txq->free_thresh = tx_free_thresh;
+
+	txq->agg_szmax  = RTE_MIN(hv->chim_szmax, hv->rndis_agg_size);
+	txq->agg_pktmax = hv->rndis_agg_pkts;
+	txq->agg_align  = hv->rndis_agg_align;
+
+	hn_reset_txagg(txq);
+
+	dev->data->tx_queues[queue_idx] = txq;
+
+	return 0;
+}
+
+void
+hn_dev_tx_queue_release(void *arg)
+{
+	struct hn_tx_queue *txq = arg;
+	struct hn_txdesc *txd;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (!txq)
+		return;
+
+	/* If any pending data is still present just drop it */
+	txd = txq->agg_txd;
+	if (txd)
+		rte_mempool_put(txq->hv->tx_pool, txd);
+
+	rte_free(txq);
+}
+
+void
+hn_dev_tx_queue_info(struct rte_eth_dev *dev, uint16_t queue_idx,
+		     struct rte_eth_txq_info *qinfo)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	struct hn_tx_queue *txq = dev->data->rx_queues[queue_idx];
+
+	qinfo->conf.tx_free_thresh = txq->free_thresh;
+	qinfo->nb_desc = hv->tx_pool->size;
+}
+
+static void
+hn_nvs_send_completed(struct rte_eth_dev *dev, uint16_t queue_id,
+		      unsigned long xactid, const struct hn_nvs_rndis_ack *ack)
+{
+	struct hn_txdesc *txd = (struct hn_txdesc *)xactid;
+	struct hn_tx_queue *txq;
+
+	/* Control packets are sent with xacid == 0 */
+	if (!txd)
+		return;
+
+	txq = dev->data->tx_queues[queue_id];
+	if (likely(ack->status == NVS_STATUS_OK)) {
+		PMD_TX_LOG(DEBUG, "port %u:%u complete tx %u packets %u bytes %u",
+			   txq->port_id, txq->queue_id, txd->chim_index,
+			   txd->packets, txd->data_size);
+		txq->stats.bytes += txd->data_size;
+		txq->stats.packets += txd->packets;
+	} else {
+		PMD_TX_LOG(NOTICE, "port %u:%u complete tx %u failed status %u",
+			   txq->port_id, txq->queue_id, txd->chim_index, ack->status);
+		++txq->stats.errors;
+	}
+
+	rte_pktmbuf_free(txd->m);
+
+	rte_mempool_put(txq->hv->tx_pool, txd);
+}
+
+/* Handle transmit completion events */
+static void
+hn_nvs_handle_comp(struct rte_eth_dev *dev, uint16_t queue_id,
+		   const struct vmbus_chanpkt_hdr *pkt,
+		   const void *data)
+{
+	const struct hn_nvs_hdr *hdr = data;
+
+	switch (hdr->type) {
+	case NVS_TYPE_RNDIS_ACK:
+		hn_nvs_send_completed(dev, queue_id, pkt->xactid, data);
+		break;
+
+	default:
+		PMD_TX_LOG(NOTICE,
+			   "unexpected send completion type %u",
+			   hdr->type);
+	}
+}
+
+/* Parse per-packet info (meta data) */
+static int
+hn_rndis_rxinfo(const void *info_data, unsigned int info_dlen,
+		struct hn_rxinfo *info)
+{
+	const struct rndis_pktinfo *pi = info_data;
+	uint32_t mask = 0;
+
+	while (info_dlen != 0) {
+		const void *data;
+		uint32_t dlen;
+
+		if (unlikely(info_dlen < sizeof(*pi)))
+			return -EINVAL;
+
+		if (unlikely(info_dlen < pi->size))
+			return -EINVAL;
+		info_dlen -= pi->size;
+
+		if (unlikely(pi->size & RNDIS_PKTINFO_SIZE_ALIGNMASK))
+			return -EINVAL;
+		if (unlikely(pi->size < pi->offset))
+			return -EINVAL;
+
+		dlen = pi->size - pi->offset;
+		data = pi->data;
+
+		switch (pi->type) {
+		case NDIS_PKTINFO_TYPE_VLAN:
+			if (unlikely(dlen < NDIS_VLAN_INFO_SIZE))
+				return -EINVAL;
+			info->vlan_info = *((const uint32_t *)data);
+			mask |= HN_RXINFO_VLAN;
+			break;
+
+		case NDIS_PKTINFO_TYPE_CSUM:
+			if (unlikely(dlen < NDIS_RXCSUM_INFO_SIZE))
+				return -EINVAL;
+			info->csum_info = *((const uint32_t *)data);
+			mask |= HN_RXINFO_CSUM;
+			break;
+
+		case NDIS_PKTINFO_TYPE_HASHVAL:
+			if (unlikely(dlen < NDIS_HASH_VALUE_SIZE))
+				return -EINVAL;
+			info->hash_value = *((const uint32_t *)data);
+			mask |= HN_RXINFO_HASHVAL;
+			break;
+
+		case NDIS_PKTINFO_TYPE_HASHINF:
+			if (unlikely(dlen < NDIS_HASH_INFO_SIZE))
+				return -EINVAL;
+			info->hash_info = *((const uint32_t *)data);
+			mask |= HN_RXINFO_HASHINF;
+			break;
+
+		default:
+			goto next;
+		}
+
+		if (mask == HN_RXINFO_ALL)
+			break; /* All found; done */
+next:
+		pi = (const struct rndis_pktinfo *)
+		    ((const uint8_t *)pi + pi->size);
+	}
+
+	/*
+	 * Final fixup.
+	 * - If there is no hash value, invalidate the hash info.
+	 */
+	if (!(mask & HN_RXINFO_HASHVAL))
+		info->hash_info = HN_NDIS_HASH_INFO_INVALID;
+	return 0;
+}
+
+/*
+ * Ack the consumed RXBUF associated w/ this channel packet,
+ * so that this RXBUF can be recycled by the hypervisor.
+ */
+static void hn_rx_buf_release(struct hn_rx_bufinfo *rxb)
+{
+	struct rte_mbuf_ext_shared_info *shinfo = &rxb->shinfo;
+	struct hn_data *hv = rxb->hv;
+
+	if (rte_mbuf_ext_refcnt_update(shinfo, -1) == 0) {
+		hn_nvs_ack_rxbuf(rxb->chan, rxb->xactid);
+		--hv->rxbuf_outstanding;
+	}
+}
+
+static void hn_rx_buf_free_cb(void *buf __rte_unused, void *opaque)
+{
+	hn_rx_buf_release(opaque);
+}
+
+static struct hn_rx_bufinfo *hn_rx_buf_init(const struct hn_rx_queue *rxq,
+					    const struct vmbus_chanpkt_rxbuf *pkt)
+{
+	struct hn_rx_bufinfo *rxb;
+
+	rxb = rxq->hv->rxbuf_info + pkt->hdr.xactid;
+	rxb->chan = rxq->chan;
+	rxb->xactid = pkt->hdr.xactid;
+	rxb->hv = rxq->hv;
+
+	rxb->shinfo.free_cb = hn_rx_buf_free_cb;
+	rxb->shinfo.fcb_opaque = rxb;
+	rte_mbuf_ext_refcnt_set(&rxb->shinfo, 1);
+	return rxb;
+}
+
+static void hn_rxpkt(struct hn_rx_queue *rxq, struct hn_rx_bufinfo *rxb,
+		     uint8_t *data, unsigned int headroom, unsigned int dlen,
+		     const struct hn_rxinfo *info)
+{
+	struct hn_data *hv = rxq->hv;
+	struct rte_mbuf *m;
+
+	m = rte_pktmbuf_alloc(rxq->mb_pool);
+	if (unlikely(!m)) {
+		struct rte_eth_dev *dev =
+			&rte_eth_devices[rxq->port_id];
+
+		dev->data->rx_mbuf_alloc_failed++;
+		return;
+	}
+
+	/*
+	 * For large packets, avoid copy if possible but need to keep
+	 * some space available in receive area for later packets.
+	 */
+	if (dlen >= HN_RXCOPY_THRESHOLD &&
+	    hv->rxbuf_outstanding < hv->rxbuf_section_cnt / 2) {
+		struct rte_mbuf_ext_shared_info *shinfo;
+		const void *rxbuf;
+		rte_iova_t iova;
+
+		/*
+		 * Build an external mbuf that points to recveive area.
+		 * Use refcount to handle multiple packets in same
+		 * receive buffer section.
+		 */
+		rxbuf = hv->rxbuf_res->addr;
+		iova = rte_mem_virt2iova(rxbuf) + RTE_PTR_DIFF(data, rxbuf);
+		shinfo = &rxb->shinfo;
+
+		if (rte_mbuf_ext_refcnt_update(shinfo, 1) == 1)
+			++hv->rxbuf_outstanding;
+
+		rte_pktmbuf_attach_extbuf(m, data, iova,
+					  dlen + headroom, shinfo);
+		m->data_off = headroom;
+	} else {
+		/* Mbuf's in pool must be large enough to hold small packets */
+		if (unlikely(rte_pktmbuf_tailroom(m) < dlen)) {
+			rte_pktmbuf_free_seg(m);
+			++rxq->stats.errors;
+			return;
+		}
+		rte_memcpy(rte_pktmbuf_mtod(m, void *),
+			   data + headroom, dlen);
+	}
+
+	m->port = rxq->port_id;
+	m->pkt_len = dlen;
+	m->data_len = dlen;
+	m->packet_type = rte_net_get_ptype(m, NULL,
+					   RTE_PTYPE_L2_MASK |
+					   RTE_PTYPE_L3_MASK |
+					   RTE_PTYPE_L4_MASK);
+
+	if (info->vlan_info != HN_NDIS_VLAN_INFO_INVALID) {
+		m->vlan_tci = info->vlan_info;
+		m->ol_flags |= PKT_RX_VLAN_STRIPPED | PKT_RX_VLAN;
+	}
+
+	if (info->csum_info != HN_NDIS_RXCSUM_INFO_INVALID) {
+		if (info->csum_info & NDIS_RXCSUM_INFO_IPCS_OK)
+			m->ol_flags |= PKT_RX_IP_CKSUM_GOOD;
+
+		if (info->csum_info & (NDIS_RXCSUM_INFO_UDPCS_OK
+				       | NDIS_RXCSUM_INFO_TCPCS_OK))
+			m->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
+		else if (info->csum_info & (NDIS_RXCSUM_INFO_TCPCS_FAILED
+					    | NDIS_RXCSUM_INFO_UDPCS_FAILED))
+			m->ol_flags |= PKT_RX_L4_CKSUM_BAD;
+	}
+
+	if (info->hash_info != HN_NDIS_HASH_INFO_INVALID) {
+		m->ol_flags |= PKT_RX_RSS_HASH;
+		m->hash.rss = info->hash_value;
+	}
+
+	PMD_RX_LOG(DEBUG,
+		   "port %u:%u RX id %"PRIu64" size %u type %#x ol_flags %#"PRIx64,
+		   rxq->port_id, rxq->queue_id, rxb->xactid,
+		   m->pkt_len, m->packet_type, m->ol_flags);
+
+	++rxq->stats.packets;
+	rxq->stats.bytes += m->pkt_len;
+	hn_update_packet_stats(&rxq->stats, m);
+
+	if (unlikely(rte_ring_sp_enqueue(rxq->rx_ring, m) != 0)) {
+		++rxq->ring_full;
+		rte_pktmbuf_free(m);
+	}
+}
+
+static void hn_rndis_rx_data(struct hn_rx_queue *rxq,
+			     struct hn_rx_bufinfo *rxb,
+			     void *data, uint32_t dlen)
+{
+	unsigned int data_off, data_len, pktinfo_off, pktinfo_len;
+	const struct rndis_packet_msg *pkt = data;
+	struct hn_rxinfo info = {
+		.vlan_info = HN_NDIS_VLAN_INFO_INVALID,
+		.csum_info = HN_NDIS_RXCSUM_INFO_INVALID,
+		.hash_info = HN_NDIS_HASH_INFO_INVALID,
+	};
+	int err;
+
+	hn_rndis_dump(pkt);
+
+	if (unlikely(dlen < sizeof(*pkt)))
+		goto error;
+
+	if (unlikely(dlen < pkt->len))
+		goto error; /* truncated RNDIS from host */
+
+	if (unlikely(pkt->len < pkt->datalen
+		     + pkt->oobdatalen + pkt->pktinfolen))
+		goto error;
+
+	if (unlikely(pkt->datalen == 0))
+		goto error;
+
+	/* Check offsets. */
+	if (unlikely(pkt->dataoffset < RNDIS_PACKET_MSG_OFFSET_MIN))
+		goto error;
+
+	if (likely(pkt->pktinfooffset > 0) &&
+	    unlikely(pkt->pktinfooffset < RNDIS_PACKET_MSG_OFFSET_MIN ||
+		     (pkt->pktinfooffset & RNDIS_PACKET_MSG_OFFSET_ALIGNMASK)))
+		goto error;
+
+	data_off = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->dataoffset);
+	data_len = pkt->datalen;
+	pktinfo_off = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->pktinfooffset);
+	pktinfo_len = pkt->pktinfolen;
+
+	if (likely(pktinfo_len > 0)) {
+		err = hn_rndis_rxinfo((const uint8_t *)pkt + pktinfo_off,
+				      pktinfo_len, &info);
+		if (err)
+			goto error;
+	}
+
+	if (unlikely(data_off + data_len > pkt->len))
+		goto error;
+
+	if (unlikely(data_len < ETHER_HDR_LEN))
+		goto error;
+
+	hn_rxpkt(rxq, rxb, data, data_off, data_len, &info);
+	return;
+error:
+	++rxq->stats.errors;
+}
+
+static void
+hn_rndis_receive(const struct rte_eth_dev *dev, struct hn_rx_queue *rxq,
+		 struct hn_rx_bufinfo *rxb, void *buf, uint32_t len)
+{
+	const struct rndis_msghdr *hdr = buf;
+
+	switch (hdr->type) {
+	case RNDIS_PACKET_MSG:
+		if (dev->data->dev_started)
+			hn_rndis_rx_data(rxq, rxb, buf, len);
+		break;
+
+	case RNDIS_INDICATE_STATUS_MSG:
+		hn_rndis_link_status(rxq->hv, buf);
+		break;
+
+	case RNDIS_INITIALIZE_CMPLT:
+	case RNDIS_QUERY_CMPLT:
+	case RNDIS_SET_CMPLT:
+		hn_rndis_receive_response(rxq->hv, buf, len);
+		break;
+
+	default:
+		PMD_DRV_LOG(NOTICE,
+			    "unexpected RNDIS message (type %#x len %u)",
+			    hdr->type, len);
+		break;
+	}
+}
+
+static void
+hn_nvs_handle_rxbuf(struct rte_eth_dev *dev,
+		    struct hn_data *hv,
+		    struct hn_rx_queue *rxq,
+		    const struct vmbus_chanpkt_hdr *hdr,
+		    const void *buf)
+{
+	const struct vmbus_chanpkt_rxbuf *pkt;
+	const struct hn_nvs_hdr *nvs_hdr = buf;
+	uint32_t rxbuf_sz = hv->rxbuf_res->len;
+	char *rxbuf = hv->rxbuf_res->addr;
+	unsigned int i, hlen, count;
+	struct hn_rx_bufinfo *rxb;
+
+	/* At minimum we need type header */
+	if (unlikely(vmbus_chanpkt_datalen(hdr) < sizeof(*nvs_hdr))) {
+		PMD_RX_LOG(ERR, "invalid receive nvs RNDIS");
+		return;
+	}
+
+	/* Make sure that this is a RNDIS message. */
+	if (unlikely(nvs_hdr->type != NVS_TYPE_RNDIS)) {
+		PMD_RX_LOG(ERR, "nvs type %u, not RNDIS",
+			   nvs_hdr->type);
+		return;
+	}
+
+	hlen = vmbus_chanpkt_getlen(hdr->hlen);
+	if (unlikely(hlen < sizeof(*pkt))) {
+		PMD_RX_LOG(ERR, "invalid rxbuf chanpkt");
+		return;
+	}
+
+	pkt = container_of(hdr, const struct vmbus_chanpkt_rxbuf, hdr);
+	if (unlikely(pkt->rxbuf_id != NVS_RXBUF_SIG)) {
+		PMD_RX_LOG(ERR, "invalid rxbuf_id 0x%08x",
+			   pkt->rxbuf_id);
+		return;
+	}
+
+	count = pkt->rxbuf_cnt;
+	if (unlikely(hlen < offsetof(struct vmbus_chanpkt_rxbuf,
+				     rxbuf[count]))) {
+		PMD_RX_LOG(ERR, "invalid rxbuf_cnt %u", count);
+		return;
+	}
+
+	if (pkt->hdr.xactid > hv->rxbuf_section_cnt) {
+		PMD_RX_LOG(ERR, "invalid rxbuf section id %" PRIx64,
+			   pkt->hdr.xactid);
+		return;
+	}
+
+	/* Setup receive buffer info to allow for callback */
+	rxb = hn_rx_buf_init(rxq, pkt);
+
+	/* Each range represents 1 RNDIS pkt that contains 1 Ethernet frame */
+	for (i = 0; i < count; ++i) {
+		unsigned int ofs, len;
+
+		ofs = pkt->rxbuf[i].ofs;
+		len = pkt->rxbuf[i].len;
+
+		if (unlikely(ofs + len > rxbuf_sz)) {
+			PMD_RX_LOG(ERR,
+				   "%uth RNDIS msg overflow ofs %u, len %u",
+				   i, ofs, len);
+			continue;
+		}
+
+		if (unlikely(len == 0)) {
+			PMD_RX_LOG(ERR, "%uth RNDIS msg len %u", i, len);
+			continue;
+		}
+
+		hn_rndis_receive(dev, rxq, rxb,
+				 rxbuf + ofs, len);
+	}
+
+	/* Send ACK now if external mbuf not used */
+	hn_rx_buf_release(rxb);
+}
+
+struct hn_rx_queue *hn_rx_queue_alloc(struct hn_data *hv,
+				      uint16_t queue_id,
+				      unsigned int socket_id)
+{
+	struct hn_rx_queue *rxq;
+
+	rxq = rte_zmalloc_socket("HN_RXQ",
+				 sizeof(*rxq) + HN_RXQ_EVENT_DEFAULT,
+				 RTE_CACHE_LINE_SIZE, socket_id);
+	if (rxq) {
+		rxq->hv = hv;
+		rxq->chan = hv->channels[queue_id];
+		rte_spinlock_init(&rxq->ring_lock);
+		rxq->port_id = hv->port_id;
+		rxq->queue_id = queue_id;
+	}
+	return rxq;
+}
+
+int
+hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
+		      uint16_t queue_idx, uint16_t nb_desc,
+		      unsigned int socket_id,
+		      const struct rte_eth_rxconf *rx_conf __rte_unused,
+		      struct rte_mempool *mp)
+{
+	struct hn_data *hv = dev->data->dev_private;
+	char ring_name[RTE_RING_NAMESIZE];
+	struct hn_rx_queue *rxq;
+	unsigned int count;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (queue_idx == 0) {
+		rxq = hv->primary;
+	} else {
+		rxq = hn_rx_queue_alloc(hv, queue_idx, socket_id);
+		if (!rxq)
+			return -ENOMEM;
+	}
+
+	rxq->mb_pool = mp;
+	count = rte_mempool_avail_count(mp) / dev->data->nb_rx_queues;
+	if (nb_desc == 0 || nb_desc > count)
+		nb_desc = count;
+
+	/*
+	 * Staging ring from receive event logic to rx_pkts.
+	 * rx_pkts assumes caller is handling multi-thread issue.
+	 * event logic has locking.
+	 */
+	snprintf(ring_name, sizeof(ring_name),
+		 "hn_rx_%u_%u", dev->data->port_id, queue_idx);
+	rxq->rx_ring = rte_ring_create(ring_name,
+				       rte_align32pow2(nb_desc),
+				       socket_id, 0);
+	if (!rxq->rx_ring)
+		goto fail;
+
+	dev->data->rx_queues[queue_idx] = rxq;
+	return 0;
+
+fail:
+	rte_ring_free(rxq->rx_ring);
+	rte_free(rxq->event_buf);
+	rte_free(rxq);
+	return -ENOMEM;
+}
+
+void
+hn_dev_rx_queue_release(void *arg)
+{
+	struct hn_rx_queue *rxq = arg;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (!rxq)
+		return;
+
+	rte_ring_free(rxq->rx_ring);
+	rxq->rx_ring = NULL;
+	rxq->mb_pool = NULL;
+
+	if (rxq != rxq->hv->primary) {
+		rte_free(rxq->event_buf);
+		rte_free(rxq);
+	}
+}
+
+void
+hn_dev_rx_queue_info(struct rte_eth_dev *dev, uint16_t queue_idx,
+		     struct rte_eth_rxq_info *qinfo)
+{
+	struct hn_rx_queue *rxq = dev->data->rx_queues[queue_idx];
+
+	qinfo->mp = rxq->mb_pool;
+	qinfo->scattered_rx = 1;
+	qinfo->nb_desc = rte_ring_get_capacity(rxq->rx_ring);
+}
+
+static void
+hn_nvs_handle_notify(const struct vmbus_chanpkt_hdr *pkthdr,
+		     const void *data)
+{
+	const struct hn_nvs_hdr *hdr = data;
+
+	if (unlikely(vmbus_chanpkt_datalen(pkthdr) < sizeof(*hdr))) {
+		PMD_DRV_LOG(ERR, "invalid nvs notify");
+		return;
+	}
+
+	PMD_DRV_LOG(INFO,
+		    "got notify, nvs type %u", hdr->type);
+}
+
+/*
+ * Process pending events on the channel.
+ * Called from both Rx queue poll and Tx cleanup
+ */
+void hn_process_events(struct hn_data *hv, uint16_t queue_id)
+{
+	struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id];
+	struct hn_rx_queue *rxq;
+	uint32_t bytes_read = 0;
+	int ret = 0;
+
+	rxq = queue_id == 0 ? hv->primary : dev->data->rx_queues[queue_id];
+
+	/* If no pending data then nothing to do */
+	if (rte_vmbus_chan_rx_empty(rxq->chan))
+		return;
+
+	/*
+	 * Since channel is shared between Rx and TX queue need to have a lock
+	 * since DPDK does not force same CPU to be used for Rx/Tx.
+	 */
+	if (unlikely(!rte_spinlock_trylock(&rxq->ring_lock)))
+		return;
+
+	for (;;) {
+		const struct vmbus_chanpkt_hdr *pkt;
+		uint32_t len = HN_RXQ_EVENT_DEFAULT;
+		const void *data;
+
+		ret = rte_vmbus_chan_recv_raw(rxq->chan, rxq->event_buf, &len);
+		if (ret == -EAGAIN)
+			break;	/* ring is empty */
+
+		else if (ret == -ENOBUFS)
+			rte_exit(EXIT_FAILURE, "event buffer not big enough (%u < %u)",
+				 HN_RXQ_EVENT_DEFAULT, len);
+		else if (ret <= 0)
+			rte_exit(EXIT_FAILURE,
+				 "vmbus ring buffer error: %d", ret);
+
+		bytes_read += ret;
+		pkt = (const struct vmbus_chanpkt_hdr *)rxq->event_buf;
+		data = (char *)rxq->event_buf + vmbus_chanpkt_getlen(pkt->hlen);
+
+		switch (pkt->type) {
+		case VMBUS_CHANPKT_TYPE_COMP:
+			hn_nvs_handle_comp(dev, queue_id, pkt, data);
+			break;
+
+		case VMBUS_CHANPKT_TYPE_RXBUF:
+			hn_nvs_handle_rxbuf(dev, hv, rxq, pkt, data);
+			break;
+
+		case VMBUS_CHANPKT_TYPE_INBAND:
+			hn_nvs_handle_notify(pkt, data);
+			break;
+
+		default:
+			PMD_DRV_LOG(ERR, "unknown chan pkt %u", pkt->type);
+			break;
+		}
+
+		if (rxq->rx_ring && rte_ring_full(rxq->rx_ring))
+			break;
+	}
+
+	if (bytes_read > 0)
+		rte_vmbus_chan_signal_read(rxq->chan, bytes_read);
+
+	rte_spinlock_unlock(&rxq->ring_lock);
+}
+
+static void hn_append_to_chim(struct hn_tx_queue *txq,
+			      struct rndis_packet_msg *pkt,
+			      const struct rte_mbuf *m)
+{
+	struct hn_txdesc *txd = txq->agg_txd;
+	uint8_t *buf = (uint8_t *)pkt;
+	unsigned int data_offs;
+
+	hn_rndis_dump(pkt);
+
+	data_offs = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->dataoffset);
+	txd->chim_size += pkt->len;
+	txd->data_size += m->pkt_len;
+	++txd->packets;
+	hn_update_packet_stats(&txq->stats, m);
+
+	for (; m; m = m->next) {
+		uint16_t len = rte_pktmbuf_data_len(m);
+
+		rte_memcpy(buf + data_offs,
+			   rte_pktmbuf_mtod(m, const char *), len);
+		data_offs += len;
+	}
+}
+
+/*
+ * Send pending aggregated data in chimney buffer (if any).
+ * Returns error if send was unsuccessful because channel ring buffer
+ * was full.
+ */
+static int hn_flush_txagg(struct hn_tx_queue *txq, bool *need_sig)
+
+{
+	struct hn_txdesc *txd = txq->agg_txd;
+	struct hn_nvs_rndis rndis;
+	int ret;
+
+	if (!txd)
+		return 0;
+
+	rndis = (struct hn_nvs_rndis) {
+		.type = NVS_TYPE_RNDIS,
+		.rndis_mtype = NVS_RNDIS_MTYPE_DATA,
+		.chim_idx = txd->chim_index,
+		.chim_sz = txd->chim_size,
+	};
+
+	PMD_TX_LOG(DEBUG, "port %u:%u tx %u size %u",
+		   txq->port_id, txq->queue_id, txd->chim_index, txd->chim_size);
+
+	ret = hn_nvs_send(txq->chan, VMBUS_CHANPKT_FLAG_RC,
+			  &rndis, sizeof(rndis), (uintptr_t)txd, need_sig);
+
+	if (likely(ret == 0))
+		hn_reset_txagg(txq);
+	else
+		PMD_TX_LOG(NOTICE, "port %u:%u send failed: %d",
+			   txq->port_id, txq->queue_id, ret);
+
+	return ret;
+}
+
+static struct hn_txdesc *hn_new_txd(struct hn_data *hv,
+				    struct hn_tx_queue *txq)
+{
+	struct hn_txdesc *txd;
+
+	if (rte_mempool_get(hv->tx_pool, (void **)&txd)) {
+		++txq->stats.nomemory;
+		PMD_TX_LOG(DEBUG, "tx pool exhausted!");
+		return NULL;
+	}
+
+	txd->m = NULL;
+	txd->queue_id = txq->queue_id;
+	txd->packets = 0;
+	txd->data_size = 0;
+	txd->chim_size = 0;
+
+	return txd;
+}
+
+static void *
+hn_try_txagg(struct hn_data *hv, struct hn_tx_queue *txq, uint32_t pktsize)
+{
+	struct hn_txdesc *agg_txd = txq->agg_txd;
+	struct rndis_packet_msg *pkt;
+	void *chim;
+
+	if (agg_txd) {
+		unsigned int padding, olen;
+
+		/*
+		 * Update the previous RNDIS packet's total length,
+		 * it can be increased due to the mandatory alignment
+		 * padding for this RNDIS packet.  And update the
+		 * aggregating txdesc's chimney sending buffer size
+		 * accordingly.
+		 *
+		 * Zero-out the padding, as required by the RNDIS spec.
+		 */
+		pkt = txq->agg_prevpkt;
+		olen = pkt->len;
+		padding = RTE_ALIGN(olen, txq->agg_align) - olen;
+		if (padding > 0) {
+			agg_txd->chim_size += padding;
+			pkt->len += padding;
+			memset((uint8_t *)pkt + olen, 0, padding);
+		}
+
+		chim = (uint8_t *)pkt + pkt->len;
+
+		txq->agg_pktleft--;
+		txq->agg_szleft -= pktsize;
+		if (txq->agg_szleft < HN_PKTSIZE_MIN(txq->agg_align)) {
+			/*
+			 * Probably can't aggregate more packets,
+			 * flush this aggregating txdesc proactively.
+			 */
+			txq->agg_pktleft = 0;
+		}
+	} else {
+		agg_txd = hn_new_txd(hv, txq);
+		if (!agg_txd)
+			return NULL;
+
+		chim = (uint8_t *)hv->chim_res->addr
+			+ agg_txd->chim_index * hv->chim_szmax;
+
+		txq->agg_txd = agg_txd;
+		txq->agg_pktleft = txq->agg_pktmax - 1;
+		txq->agg_szleft = txq->agg_szmax - pktsize;
+	}
+	txq->agg_prevpkt = chim;
+
+	return chim;
+}
+
+static inline void *
+hn_rndis_pktinfo_append(struct rndis_packet_msg *pkt,
+			uint32_t pi_dlen, uint32_t pi_type)
+{
+	const uint32_t pi_size = RNDIS_PKTINFO_SIZE(pi_dlen);
+	struct rndis_pktinfo *pi;
+
+	/*
+	 * Per-packet-info does not move; it only grows.
+	 *
+	 * NOTE:
+	 * pktinfooffset in this phase counts from the beginning
+	 * of rndis_packet_msg.
+	 */
+	pi = (struct rndis_pktinfo *)((uint8_t *)pkt + hn_rndis_pktlen(pkt));
+
+	pkt->pktinfolen += pi_size;
+
+	pi->size = pi_size;
+	pi->type = pi_type;
+	pi->offset = RNDIS_PKTINFO_OFFSET;
+
+	return pi->data;
+}
+
+/* Put RNDIS header and packet info on packet */
+static void hn_encap(struct rndis_packet_msg *pkt,
+		     uint16_t queue_id,
+		     const struct rte_mbuf *m)
+{
+	unsigned int hlen = m->l2_len + m->l3_len;
+	uint32_t *pi_data;
+	uint32_t pkt_hlen;
+
+	pkt->type = RNDIS_PACKET_MSG;
+	pkt->len = m->pkt_len;
+	pkt->dataoffset = 0;
+	pkt->datalen = m->pkt_len;
+	pkt->oobdataoffset = 0;
+	pkt->oobdatalen = 0;
+	pkt->oobdataelements = 0;
+	pkt->pktinfooffset = sizeof(*pkt);
+	pkt->pktinfolen = 0;
+	pkt->vchandle = 0;
+	pkt->reserved = 0;
+
+	/*
+	 * Set the hash value for this packet, to the queue_id to cause
+	 * TX done event for this packet on the right channel.
+	 */
+	pi_data = hn_rndis_pktinfo_append(pkt, NDIS_HASH_VALUE_SIZE,
+					  NDIS_PKTINFO_TYPE_HASHVAL);
+	*pi_data = queue_id;
+
+	if (m->ol_flags & PKT_TX_VLAN_PKT) {
+		pi_data = hn_rndis_pktinfo_append(pkt, NDIS_VLAN_INFO_SIZE,
+						  NDIS_PKTINFO_TYPE_VLAN);
+		*pi_data = m->vlan_tci;
+	}
+
+	if (m->ol_flags & PKT_TX_TCP_SEG) {
+		pi_data = hn_rndis_pktinfo_append(pkt, NDIS_LSO2_INFO_SIZE,
+						  NDIS_PKTINFO_TYPE_LSO);
+
+		if (m->ol_flags & PKT_TX_IPV6) {
+			*pi_data = NDIS_LSO2_INFO_MAKEIPV6(hlen,
+							   m->tso_segsz);
+		} else {
+			*pi_data = NDIS_LSO2_INFO_MAKEIPV4(hlen,
+							   m->tso_segsz);
+		}
+	} else if (m->ol_flags &
+		   (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | PKT_TX_IP_CKSUM)) {
+		pi_data = hn_rndis_pktinfo_append(pkt, NDIS_TXCSUM_INFO_SIZE,
+						  NDIS_PKTINFO_TYPE_CSUM);
+		*pi_data = 0;
+
+		if (m->ol_flags & PKT_TX_IPV6)
+			*pi_data |= NDIS_TXCSUM_INFO_IPV6;
+		if (m->ol_flags & PKT_TX_IPV4) {
+			*pi_data |= NDIS_TXCSUM_INFO_IPV4;
+
+			if (m->ol_flags & PKT_TX_IP_CKSUM)
+				*pi_data |= NDIS_TXCSUM_INFO_IPCS;
+		}
+
+		if (m->ol_flags & PKT_TX_TCP_CKSUM)
+			*pi_data |= NDIS_TXCSUM_INFO_MKTCPCS(hlen);
+		else if (m->ol_flags & PKT_TX_UDP_CKSUM)
+			*pi_data |= NDIS_TXCSUM_INFO_MKUDPCS(hlen);
+	}
+
+	pkt_hlen = pkt->pktinfooffset + pkt->pktinfolen;
+	/* Fixup RNDIS packet message total length */
+	pkt->len += pkt_hlen;
+
+	/* Convert RNDIS packet message offsets */
+	pkt->dataoffset = hn_rndis_pktmsg_offset(pkt_hlen);
+	pkt->pktinfooffset = hn_rndis_pktmsg_offset(pkt->pktinfooffset);
+}
+
+/* How many scatter gather list elements ar needed */
+static unsigned int hn_get_slots(const struct rte_mbuf *m)
+{
+	unsigned int slots = 1; /* for RNDIS header */
+
+	while (m) {
+		unsigned int size = rte_pktmbuf_data_len(m);
+		unsigned int offs = rte_mbuf_data_iova(m) & PAGE_MASK;
+
+		slots += (offs + size + PAGE_SIZE - 1) / PAGE_SIZE;
+		m = m->next;
+	}
+
+	return slots;
+}
+
+/* Build scatter gather list from chained mbuf */
+static unsigned int hn_fill_sg(struct vmbus_gpa *sg,
+			       const struct rte_mbuf *m)
+{
+	unsigned int segs = 0;
+
+	while (m) {
+		rte_iova_t addr = rte_mbuf_data_iova(m);
+		unsigned int page = addr / PAGE_SIZE;
+		unsigned int offset = addr & PAGE_MASK;
+		unsigned int len = rte_pktmbuf_data_len(m);
+
+		while (len > 0) {
+			unsigned int bytes = RTE_MIN(len, PAGE_SIZE - offset);
+
+			sg[segs].page = page;
+			sg[segs].ofs = offset;
+			sg[segs].len = bytes;
+			segs++;
+
+			++page;
+			offset = 0;
+			len -= bytes;
+		}
+		m = m->next;
+	}
+
+	return segs;
+}
+
+/* Transmit directly from mbuf */
+static int hn_xmit_sg(struct hn_tx_queue *txq,
+		      const struct hn_txdesc *txd, const struct rte_mbuf *m,
+		      bool *need_sig)
+{
+	struct vmbus_gpa sg[hn_get_slots(m)];
+	struct hn_nvs_rndis nvs_rndis = {
+		.type = NVS_TYPE_RNDIS,
+		.rndis_mtype = NVS_RNDIS_MTYPE_DATA,
+		.chim_sz = txd->chim_size,
+	};
+	rte_iova_t addr;
+	unsigned int segs;
+
+	/* attach aggregation data if present */
+	if (txd->chim_size > 0)
+		nvs_rndis.chim_idx = txd->chim_index;
+	else
+		nvs_rndis.chim_idx = NVS_CHIM_IDX_INVALID;
+
+	hn_rndis_dump(txd->rndis_pkt);
+
+	/* pass IOVA of rndis header in first segment */
+	addr = rte_malloc_virt2iova(txd->rndis_pkt);
+	if (unlikely(addr == RTE_BAD_IOVA)) {
+		PMD_DRV_LOG(ERR, "RNDIS transmit can not get iova");
+		return -EINVAL;
+	}
+
+	sg[0].page = addr / PAGE_SIZE;
+	sg[0].ofs = addr & PAGE_MASK;
+	sg[0].len = RNDIS_PACKET_MSG_OFFSET_ABS(hn_rndis_pktlen(txd->rndis_pkt));
+	segs = 1;
+
+	hn_update_packet_stats(&txq->stats, m);
+
+	segs += hn_fill_sg(sg + 1, m);
+
+	PMD_TX_LOG(DEBUG, "port %u:%u tx %u segs %u size %u",
+		   txq->port_id, txq->queue_id, txd->chim_index,
+		   segs, nvs_rndis.chim_sz);
+
+	return hn_nvs_send_sglist(txq->chan, sg, segs,
+				  &nvs_rndis, sizeof(nvs_rndis),
+				  (uintptr_t)txd, need_sig);
+}
+
+uint16_t
+hn_xmit_pkts(void *ptxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+	struct hn_tx_queue *txq = ptxq;
+	struct hn_data *hv = txq->hv;
+	bool need_sig = false;
+	uint16_t nb_tx;
+	int ret;
+
+	if (unlikely(hv->closed))
+		return 0;
+
+	if (rte_mempool_avail_count(hv->tx_pool) <= txq->free_thresh)
+		hn_process_events(hv, txq->queue_id);
+
+	for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
+		struct rte_mbuf *m = tx_pkts[nb_tx];
+		uint32_t pkt_size = m->pkt_len + HN_RNDIS_PKT_LEN;
+		struct rndis_packet_msg *pkt;
+
+		/* For small packets aggregate them in chimney buffer */
+		if (m->pkt_len < HN_TXCOPY_THRESHOLD && pkt_size <= txq->agg_szmax) {
+			/* If this packet will not fit, then flush  */
+			if (txq->agg_pktleft == 0 ||
+			    RTE_ALIGN(pkt_size, txq->agg_align) > txq->agg_szleft) {
+				if (hn_flush_txagg(txq, &need_sig))
+					goto fail;
+			}
+
+			pkt = hn_try_txagg(hv, txq, pkt_size);
+			if (unlikely(!pkt))
+				break;
+
+			hn_encap(pkt, txq->queue_id, m);
+			hn_append_to_chim(txq, pkt, m);
+
+			rte_pktmbuf_free(m);
+
+			/* if buffer is full, flush */
+			if (txq->agg_pktleft == 0 &&
+			    hn_flush_txagg(txq, &need_sig))
+				goto fail;
+		} else {
+			struct hn_txdesc *txd;
+
+			/* can send chimney data and large packet at once */
+			txd = txq->agg_txd;
+			if (txd) {
+				hn_reset_txagg(txq);
+			} else {
+				txd = hn_new_txd(hv, txq);
+				if (unlikely(!txd))
+					break;
+			}
+
+			pkt = txd->rndis_pkt;
+			txd->m = m;
+			txd->data_size += m->pkt_len;
+			++txd->packets;
+
+			hn_encap(pkt, txq->queue_id, m);
+
+			ret = hn_xmit_sg(txq, txd, m, &need_sig);
+			if (unlikely(ret != 0)) {
+				PMD_TX_LOG(NOTICE, "sg send failed: %d", ret);
+				++txq->stats.errors;
+				rte_mempool_put(hv->tx_pool, txd);
+				goto fail;
+			}
+		}
+	}
+
+	/* If partial buffer left, then try and send it.
+	 * if that fails, then reuse it on next send.
+	 */
+	hn_flush_txagg(txq, &need_sig);
+
+fail:
+	if (need_sig)
+		rte_vmbus_chan_signal_tx(txq->chan);
+
+	return nb_tx;
+}
+
+uint16_t
+hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	struct hn_rx_queue *rxq = prxq;
+	struct hn_data *hv = rxq->hv;
+
+	if (unlikely(hv->closed))
+		return 0;
+
+	/* If ring is empty then process more */
+	if (rte_ring_count(rxq->rx_ring) < nb_pkts)
+		hn_process_events(hv, rxq->queue_id);
+
+	/* Get mbufs off staging ring */
+	return rte_ring_sc_dequeue_burst(rxq->rx_ring, (void **)rx_pkts,
+					 nb_pkts, NULL);
+}
diff --git a/drivers/net/netvsc/hn_var.h b/drivers/net/netvsc/hn_var.h
new file mode 100644
index 00000000..f7ff8585
--- /dev/null
+++ b/drivers/net/netvsc/hn_var.h
@@ -0,0 +1,158 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2009-2018 Microsoft Corp.
+ * Copyright (c) 2016 Brocade Communications Systems, Inc.
+ * Copyright (c) 2012 NetApp Inc.
+ * Copyright (c) 2012 Citrix Inc.
+ * All rights reserved.
+ */
+
+/*
+ * Tunable ethdev params
+ */
+#define HN_MIN_RX_BUF_SIZE	1024
+#define HN_MAX_XFER_LEN		2048
+#define	HN_MAX_MAC_ADDRS	1
+#define HN_MAX_CHANNELS		64
+
+/* Claimed to be 12232B */
+#define HN_MTU_MAX		(9 * 1024)
+
+/* Retry interval */
+#define HN_CHAN_INTERVAL_US	100
+
+/* Buffers need to be aligned */
+#ifndef PAGE_SIZE
+#define PAGE_SIZE 4096
+#endif
+
+#ifndef PAGE_MASK
+#define PAGE_MASK (PAGE_SIZE - 1)
+#endif
+
+struct hn_data;
+struct hn_txdesc;
+
+struct hn_stats {
+	uint64_t	packets;
+	uint64_t	bytes;
+	uint64_t	errors;
+	uint64_t	nomemory;
+	uint64_t	multicast;
+	uint64_t	broadcast;
+	/* Size bins in array as RFC 2819, undersized [0], 64 [1], etc */
+	uint64_t	size_bins[8];
+};
+
+struct hn_tx_queue {
+	struct hn_data  *hv;
+	struct vmbus_channel *chan;
+	uint16_t	port_id;
+	uint16_t	queue_id;
+	uint32_t	free_thresh;
+
+	/* Applied packet transmission aggregation limits. */
+	uint32_t	agg_szmax;
+	uint32_t	agg_pktmax;
+	uint32_t	agg_align;
+
+	/* Packet transmission aggregation states */
+	struct hn_txdesc *agg_txd;
+	uint32_t	agg_pktleft;
+	uint32_t	agg_szleft;
+	struct rndis_packet_msg *agg_prevpkt;
+
+	struct hn_stats stats;
+};
+
+struct hn_rx_queue {
+	struct hn_data  *hv;
+	struct vmbus_channel *chan;
+	struct rte_mempool *mb_pool;
+	struct rte_ring *rx_ring;
+
+	rte_spinlock_t ring_lock;
+	uint32_t event_sz;
+	uint16_t port_id;
+	uint16_t queue_id;
+	struct hn_stats stats;
+	uint64_t ring_full;
+
+	uint8_t	event_buf[];
+};
+
+
+/* multi-packet data from host */
+struct hn_rx_bufinfo {
+	struct vmbus_channel *chan;
+	struct hn_data *hv;
+	uint64_t	xactid;
+	struct rte_mbuf_ext_shared_info shinfo;
+} __rte_cache_aligned;
+
+struct hn_data {
+	struct rte_vmbus_device *vmbus;
+	struct hn_rx_queue *primary;
+	uint16_t	port_id;
+	bool		closed;
+	uint32_t	link_status;
+	uint32_t	link_speed;
+
+	struct rte_mem_resource *rxbuf_res;	/* UIO resource for Rx */
+	struct hn_rx_bufinfo *rxbuf_info;
+	uint32_t	rxbuf_section_cnt;	/* # of Rx sections */
+	volatile uint32_t rxbuf_outstanding;
+	uint16_t	max_queues;		/* Max available queues */
+	uint16_t	num_queues;
+	uint64_t	rss_offloads;
+
+	struct rte_mem_resource *chim_res;	/* UIO resource for Tx */
+	struct rte_mempool *tx_pool;		/* Tx descriptors */
+	uint32_t	chim_szmax;		/* Max size per buffer */
+	uint32_t	chim_cnt;		/* Max packets per buffer */
+
+	uint32_t	nvs_ver;
+	uint32_t	ndis_ver;
+	uint32_t	rndis_agg_size;
+	uint32_t	rndis_agg_pkts;
+	uint32_t	rndis_agg_align;
+
+	volatile uint32_t  rndis_pending;
+	rte_atomic32_t	rndis_req_id;
+	uint8_t		rndis_resp[256];
+
+	struct ether_addr mac_addr;
+	struct vmbus_channel *channels[HN_MAX_CHANNELS];
+};
+
+static inline struct vmbus_channel *
+hn_primary_chan(const struct hn_data *hv)
+{
+	return hv->channels[0];
+}
+
+void hn_process_events(struct hn_data *hv, uint16_t queue_id);
+
+uint16_t hn_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+		      uint16_t nb_pkts);
+uint16_t hn_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+		      uint16_t nb_pkts);
+
+int	hn_tx_pool_init(struct rte_eth_dev *dev);
+int	hn_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
+			      uint16_t nb_desc, unsigned int socket_id,
+			      const struct rte_eth_txconf *tx_conf);
+void	hn_dev_tx_queue_release(void *arg);
+void	hn_dev_tx_queue_info(struct rte_eth_dev *dev, uint16_t queue_idx,
+			     struct rte_eth_txq_info *qinfo);
+
+struct hn_rx_queue *hn_rx_queue_alloc(struct hn_data *hv,
+				      uint16_t queue_id,
+				      unsigned int socket_id);
+int	hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
+			      uint16_t queue_idx, uint16_t nb_desc,
+			      unsigned int socket_id,
+			      const struct rte_eth_rxconf *rx_conf,
+			      struct rte_mempool *mp);
+void	hn_dev_rx_queue_release(void *arg);
+void	hn_dev_rx_queue_info(struct rte_eth_dev *dev, uint16_t queue_idx,
+			     struct rte_eth_rxq_info *qinfo);
diff --git a/drivers/net/netvsc/meson.build b/drivers/net/netvsc/meson.build
new file mode 100644
index 00000000..a717cdd4
--- /dev/null
+++ b/drivers/net/netvsc/meson.build
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Microsoft Corporation
+
+build = dpdk_conf.has('RTE_LIBRTE_VMBUS_BUS')
+version = 2
+sources = files('hn_ethdev.c', 'hn_rxtx.c', 'hn_rndis.c', 'hn_nvs.c')
+
+deps += ['bus_vmbus' ]
+
+allow_experimental_apis = true
diff --git a/drivers/net/netvsc/ndis.h b/drivers/net/netvsc/ndis.h
new file mode 100644
index 00000000..2e7ca99b
--- /dev/null
+++ b/drivers/net/netvsc/ndis.h
@@ -0,0 +1,378 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018 Microsoft Corp.
+ * All rights reserved.
+ */
+
+#ifndef _NET_NDIS_H_
+#define _NET_NDIS_H_
+
+#define	NDIS_MEDIA_STATE_CONNECTED	0
+#define	NDIS_MEDIA_STATE_DISCONNECTED	1
+
+#define	NDIS_NETCHANGE_TYPE_POSSIBLE	1
+#define	NDIS_NETCHANGE_TYPE_DEFINITE	2
+#define	NDIS_NETCHANGE_TYPE_FROMMEDIA	3
+
+#define	NDIS_OFFLOAD_SET_NOCHG		0
+#define	NDIS_OFFLOAD_SET_ON		1
+#define	NDIS_OFFLOAD_SET_OFF		2
+
+/* a.k.a GRE MAC */
+#define	NDIS_ENCAP_TYPE_NVGRE		0x00000001
+
+#define	NDIS_HASH_FUNCTION_MASK		0x000000FF	/* see hash function */
+#define	NDIS_HASH_TYPE_MASK		0x00FFFF00	/* see hash type */
+
+/* hash function */
+#define	NDIS_HASH_FUNCTION_TOEPLITZ	0x00000001
+
+/* hash type */
+#define	NDIS_HASH_IPV4			0x00000100
+#define	NDIS_HASH_TCP_IPV4		0x00000200
+#define	NDIS_HASH_IPV6			0x00000400
+#define	NDIS_HASH_IPV6_EX		0x00000800
+#define	NDIS_HASH_TCP_IPV6		0x00001000
+#define	NDIS_HASH_TCP_IPV6_EX		0x00002000
+
+#define	NDIS_HASH_KEYSIZE_TOEPLITZ	40
+#define	NDIS_HASH_INDCNT		128
+
+#define	NDIS_OBJTYPE_DEFAULT		0x80
+#define	NDIS_OBJTYPE_RSS_CAPS		0x88
+#define	NDIS_OBJTYPE_RSS_PARAMS		0x89
+#define	NDIS_OBJTYPE_OFFLOAD		0xa7
+
+struct ndis_object_hdr {
+	uint8_t			ndis_type;	/* NDIS_OBJTYPE_ */
+	uint8_t			ndis_rev;	/* type specific */
+	uint16_t		ndis_size;	/* incl. this hdr */
+} __rte_packed;
+
+/*
+ * OID_TCP_OFFLOAD_PARAMETERS
+ * ndis_type: NDIS_OBJTYPE_DEFAULT
+ */
+struct ndis_offload_params {
+	struct ndis_object_hdr	ndis_hdr;
+	uint8_t			ndis_ip4csum;	/* NDIS_OFFLOAD_PARAM_ */
+	uint8_t			ndis_tcp4csum;	/* NDIS_OFFLOAD_PARAM_ */
+	uint8_t			ndis_udp4csum;	/* NDIS_OFFLOAD_PARAM_ */
+	uint8_t			ndis_tcp6csum;	/* NDIS_OFFLOAD_PARAM_ */
+	uint8_t			ndis_udp6csum;	/* NDIS_OFFLOAD_PARAM_ */
+	uint8_t			ndis_lsov1;	/* NDIS_OFFLOAD_PARAM_ */
+	uint8_t			ndis_ipsecv1;	/* NDIS_OFFLOAD_IPSECV1_ */
+	uint8_t			ndis_lsov2_ip4;	/* NDIS_OFFLOAD_LSOV2_ */
+	uint8_t			ndis_lsov2_ip6;	/* NDIS_OFFLOAD_LSOV2_ */
+	uint8_t			ndis_tcp4conn;	/* 0 */
+	uint8_t			ndis_tcp6conn;	/* 0 */
+	uint32_t		ndis_flags;	/* 0 */
+	/* NDIS >= 6.1 */
+	uint8_t			ndis_ipsecv2;	/* NDIS_OFFLOAD_IPSECV2_ */
+	uint8_t			ndis_ipsecv2_ip4;/* NDIS_OFFLOAD_IPSECV2_ */
+	/* NDIS >= 6.30 */
+	uint8_t			ndis_rsc_ip4;	/* NDIS_OFFLOAD_RSC_ */
+	uint8_t			ndis_rsc_ip6;	/* NDIS_OFFLOAD_RSC_ */
+	uint8_t			ndis_encap;	/* NDIS_OFFLOAD_SET_ */
+	uint8_t			ndis_encap_types;/* NDIS_ENCAP_TYPE_ */
+};
+
+#define	NDIS_OFFLOAD_PARAMS_SIZE	sizeof(struct ndis_offload_params)
+#define	NDIS_OFFLOAD_PARAMS_SIZE_6_1	\
+	offsetof(struct ndis_offload_params, ndis_rsc_ip4)
+
+#define	NDIS_OFFLOAD_PARAMS_REV_2	2	/* NDIS 6.1 */
+#define	NDIS_OFFLOAD_PARAMS_REV_3	3	/* NDIS 6.30 */
+
+#define	NDIS_OFFLOAD_PARAM_NOCHG	0	/* common */
+#define	NDIS_OFFLOAD_PARAM_OFF		1
+#define	NDIS_OFFLOAD_PARAM_TX		2
+#define	NDIS_OFFLOAD_PARAM_RX		3
+#define	NDIS_OFFLOAD_PARAM_TXRX		4
+
+/* NDIS_OFFLOAD_PARAM_NOCHG */
+#define	NDIS_OFFLOAD_LSOV1_OFF		1
+#define	NDIS_OFFLOAD_LSOV1_ON		2
+
+/* NDIS_OFFLOAD_PARAM_NOCHG */
+#define	NDIS_OFFLOAD_IPSECV1_OFF	1
+#define	NDIS_OFFLOAD_IPSECV1_AH		2
+#define	NDIS_OFFLOAD_IPSECV1_ESP	3
+#define	NDIS_OFFLOAD_IPSECV1_AH_ESP	4
+
+/* NDIS_OFFLOAD_PARAM_NOCHG */
+#define	NDIS_OFFLOAD_LSOV2_OFF		1
+#define	NDIS_OFFLOAD_LSOV2_ON		2
+
+/* NDIS_OFFLOAD_PARAM_NOCHG */
+#define	NDIS_OFFLOAD_IPSECV2_OFF	1
+#define	NDIS_OFFLOAD_IPSECV2_AH		2
+#define	NDIS_OFFLOAD_IPSECV2_ESP	3
+#define	NDIS_OFFLOAD_IPSECV2_AH_ESP	4
+
+/* NDIS_OFFLOAD_PARAM_NOCHG */
+#define	NDIS_OFFLOAD_RSC_OFF		1
+#define	NDIS_OFFLOAD_RSC_ON		2
+
+/*
+ * OID_GEN_RECEIVE_SCALE_CAPABILITIES
+ * ndis_type: NDIS_OBJTYPE_RSS_CAPS
+ */
+struct ndis_rss_caps {
+	struct ndis_object_hdr		ndis_hdr;
+	uint32_t			ndis_caps;	/* NDIS_RSS_CAP_ */
+	uint32_t			ndis_nmsi;	/* # of MSIs */
+	uint32_t			ndis_nrxr;	/* # of RX rings */
+	/* NDIS >= 6.30 */
+	uint16_t			ndis_nind;	/* # of indtbl ent. */
+	uint16_t			ndis_pad;
+} __rte_packed;
+
+#define	NDIS_RSS_CAPS_SIZE		\
+	offsetof(struct ndis_rss_caps, ndis_pad)
+#define	NDIS_RSS_CAPS_SIZE_6_0		\
+	offsetof(struct ndis_rss_caps, ndis_nind)
+
+#define	NDIS_RSS_CAPS_REV_1		1	/* NDIS 6.{0,1,20} */
+#define	NDIS_RSS_CAPS_REV_2		2	/* NDIS 6.30 */
+
+#define	NDIS_RSS_CAP_MSI		0x01000000
+#define	NDIS_RSS_CAP_CLASSIFY_ISR	0x02000000
+#define	NDIS_RSS_CAP_CLASSIFY_DPC	0x04000000
+#define	NDIS_RSS_CAP_MSIX		0x08000000
+#define	NDIS_RSS_CAP_IPV4		0x00000100
+#define	NDIS_RSS_CAP_IPV6		0x00000200
+#define	NDIS_RSS_CAP_IPV6_EX		0x00000400
+#define	NDIS_RSS_CAP_HASH_TOEPLITZ	NDIS_HASH_FUNCTION_TOEPLITZ
+#define	NDIS_RSS_CAP_HASHFUNC_MASK	NDIS_HASH_FUNCTION_MASK
+
+/*
+ * OID_GEN_RECEIVE_SCALE_PARAMETERS
+ * ndis_type: NDIS_OBJTYPE_RSS_PARAMS
+ */
+struct ndis_rss_params {
+	struct ndis_object_hdr		ndis_hdr;
+	uint16_t			ndis_flags;	/* NDIS_RSS_FLAG_ */
+	uint16_t			ndis_bcpu;	/* base cpu 0 */
+	uint32_t			ndis_hash;	/* NDIS_HASH_ */
+	uint16_t			ndis_indsize;	/* indirect table */
+	uint32_t			ndis_indoffset;
+	uint16_t			ndis_keysize;	/* hash key */
+	uint32_t			ndis_keyoffset;
+	/* NDIS >= 6.20 */
+	uint32_t			ndis_cpumaskoffset;
+	uint32_t			ndis_cpumaskcnt;
+	uint32_t			ndis_cpumaskentsz;
+};
+
+#define	NDIS_RSS_PARAMS_SIZE		sizeof(struct ndis_rss_params)
+#define	NDIS_RSS_PARAMS_SIZE_6_0	\
+	offsetof(struct ndis_rss_params, ndis_cpumaskoffset)
+
+#define	NDIS_RSS_PARAMS_REV_1		1	/* NDIS 6.0 */
+#define	NDIS_RSS_PARAMS_REV_2		2	/* NDIS 6.20 */
+
+#define	NDIS_RSS_FLAG_NONE		0x0000
+#define	NDIS_RSS_FLAG_BCPU_UNCHG	0x0001
+#define	NDIS_RSS_FLAG_HASH_UNCHG	0x0002
+#define	NDIS_RSS_FLAG_IND_UNCHG		0x0004
+#define	NDIS_RSS_FLAG_KEY_UNCHG		0x0008
+#define	NDIS_RSS_FLAG_DISABLE		0x0010
+
+/* non-standard convenient struct */
+struct ndis_rssprm_toeplitz {
+	struct ndis_rss_params		rss_params;
+	/* Indirect table */
+	uint32_t			rss_ind[NDIS_HASH_INDCNT];
+	/* Toeplitz hash key */
+	uint8_t				rss_key[NDIS_HASH_KEYSIZE_TOEPLITZ];
+};
+
+#define	NDIS_RSSPRM_TOEPLITZ_SIZE(nind)	\
+	offsetof(struct ndis_rssprm_toeplitz, rss_ind[nind])
+
+/*
+ * OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES
+ * ndis_type: NDIS_OBJTYPE_OFFLOAD
+ */
+
+#define	NDIS_OFFLOAD_ENCAP_NONE		0x0000
+#define	NDIS_OFFLOAD_ENCAP_NULL		0x0001
+#define	NDIS_OFFLOAD_ENCAP_8023		0x0002
+#define	NDIS_OFFLOAD_ENCAP_8023PQ	0x0004
+#define	NDIS_OFFLOAD_ENCAP_8023PQ_OOB	0x0008
+#define	NDIS_OFFLOAD_ENCAP_RFC1483	0x0010
+
+struct ndis_csum_offload {
+	uint32_t			ndis_ip4_txenc;	/*NDIS_OFFLOAD_ENCAP_*/
+	uint32_t			ndis_ip4_txcsum;
+#define	NDIS_TXCSUM_CAP_IP4OPT		0x001
+#define	NDIS_TXCSUM_CAP_TCP4OPT		0x004
+#define	NDIS_TXCSUM_CAP_TCP4		0x010
+#define	NDIS_TXCSUM_CAP_UDP4		0x040
+#define	NDIS_TXCSUM_CAP_IP4		0x100
+	uint32_t			ndis_ip4_rxenc;	/*NDIS_OFFLOAD_ENCAP_*/
+	uint32_t			ndis_ip4_rxcsum;
+#define	NDIS_RXCSUM_CAP_IP4OPT		0x001
+#define	NDIS_RXCSUM_CAP_TCP4OPT		0x004
+#define	NDIS_RXCSUM_CAP_TCP4		0x010
+#define	NDIS_RXCSUM_CAP_UDP4		0x040
+#define	NDIS_RXCSUM_CAP_IP4		0x100
+	uint32_t			ndis_ip6_txenc;	/*NDIS_OFFLOAD_ENCAP_*/
+	uint32_t			ndis_ip6_txcsum;
+#define	NDIS_TXCSUM_CAP_IP6EXT		0x001
+#define	NDIS_TXCSUM_CAP_TCP6OPT		0x004
+#define	NDIS_TXCSUM_CAP_TCP6		0x010
+#define	NDIS_TXCSUM_CAP_UDP6		0x040
+	uint32_t			ndis_ip6_rxenc;	/*NDIS_OFFLOAD_ENCAP_*/
+	uint32_t			ndis_ip6_rxcsum;
+#define	NDIS_RXCSUM_CAP_IP6EXT		0x001
+#define	NDIS_RXCSUM_CAP_TCP6OPT		0x004
+#define	NDIS_RXCSUM_CAP_TCP6		0x010
+#define	NDIS_RXCSUM_CAP_UDP6		0x040
+};
+
+struct ndis_lsov1_offload {
+	uint32_t			ndis_encap;	/*NDIS_OFFLOAD_ENCAP_*/
+	uint32_t			ndis_maxsize;
+	uint32_t			ndis_minsegs;
+	uint32_t			ndis_opts;
+};
+
+struct ndis_ipsecv1_offload {
+	uint32_t			ndis_encap;	/*NDIS_OFFLOAD_ENCAP_*/
+	uint32_t			ndis_ah_esp;
+	uint32_t			ndis_xport_tun;
+	uint32_t			ndis_ip4_opts;
+	uint32_t			ndis_flags;
+	uint32_t			ndis_ip4_ah;
+	uint32_t			ndis_ip4_esp;
+};
+
+struct ndis_lsov2_offload {
+	uint32_t			ndis_ip4_encap;	/*NDIS_OFFLOAD_ENCAP_*/
+	uint32_t			ndis_ip4_maxsz;
+	uint32_t			ndis_ip4_minsg;
+	uint32_t			ndis_ip6_encap;	/*NDIS_OFFLOAD_ENCAP_*/
+	uint32_t			ndis_ip6_maxsz;
+	uint32_t			ndis_ip6_minsg;
+	uint32_t			ndis_ip6_opts;
+#define	NDIS_LSOV2_CAP_IP6EXT		0x001
+#define	NDIS_LSOV2_CAP_TCP6OPT		0x004
+};
+
+struct ndis_ipsecv2_offload {
+	uint32_t			ndis_encap;	/*NDIS_OFFLOAD_ENCAP_*/
+	uint16_t			ndis_ip6;
+	uint16_t			ndis_ip4opt;
+	uint16_t			ndis_ip6ext;
+	uint16_t			ndis_ah;
+	uint16_t			ndis_esp;
+	uint16_t			ndis_ah_esp;
+	uint16_t			ndis_xport;
+	uint16_t			ndis_tun;
+	uint16_t			ndis_xport_tun;
+	uint16_t			ndis_lso;
+	uint16_t			ndis_extseq;
+	uint32_t			ndis_udp_esp;
+	uint32_t			ndis_auth;
+	uint32_t			ndis_crypto;
+	uint32_t			ndis_sa_caps;
+};
+
+struct ndis_rsc_offload {
+	uint16_t			ndis_ip4;
+	uint16_t			ndis_ip6;
+};
+
+struct ndis_encap_offload {
+	uint32_t			ndis_flags;
+	uint32_t			ndis_maxhdr;
+};
+
+struct ndis_offload {
+	struct ndis_object_hdr		ndis_hdr;
+	struct ndis_csum_offload	ndis_csum;
+	struct ndis_lsov1_offload	ndis_lsov1;
+	struct ndis_ipsecv1_offload	ndis_ipsecv1;
+	struct ndis_lsov2_offload	ndis_lsov2;
+	uint32_t			ndis_flags;
+	/* NDIS >= 6.1 */
+	struct ndis_ipsecv2_offload	ndis_ipsecv2;
+	/* NDIS >= 6.30 */
+	struct ndis_rsc_offload		ndis_rsc;
+	struct ndis_encap_offload	ndis_encap_gre;
+};
+
+#define	NDIS_OFFLOAD_SIZE		sizeof(struct ndis_offload)
+#define	NDIS_OFFLOAD_SIZE_6_0		offsetof(struct ndis_offload, ndis_ipsecv2)
+#define	NDIS_OFFLOAD_SIZE_6_1		offsetof(struct ndis_offload, ndis_rsc)
+
+#define	NDIS_OFFLOAD_REV_1		1	/* NDIS 6.0 */
+#define	NDIS_OFFLOAD_REV_2		2	/* NDIS 6.1 */
+#define	NDIS_OFFLOAD_REV_3		3	/* NDIS 6.30 */
+
+/*
+ * Per-packet-info
+ */
+
+/* VLAN */
+#define	NDIS_VLAN_INFO_SIZE		sizeof(uint32_t)
+#define	NDIS_VLAN_INFO_PRI_MASK		0x0007
+#define	NDIS_VLAN_INFO_CFI_MASK		0x0008
+#define	NDIS_VLAN_INFO_ID_MASK		0xfff0
+#define	NDIS_VLAN_INFO_MAKE(id, pri, cfi)	\
+	(((pri) & NDIS_VLAN_INFO_PRI_MASK) |	\
+	 (((cfi) & 0x1) << 3) | (((id) & 0xfff) << 4))
+#define	NDIS_VLAN_INFO_ID(inf)		(((inf) & NDIS_VLAN_INFO_ID_MASK) >> 4)
+#define	NDIS_VLAN_INFO_CFI(inf)		(((inf) & NDIS_VLAN_INFO_CFI_MASK) >> 3)
+#define	NDIS_VLAN_INFO_PRI(inf)		((inf) & NDIS_VLAN_INFO_PRI_MASK)
+
+/* Reception checksum */
+#define	NDIS_RXCSUM_INFO_SIZE		sizeof(uint32_t)
+#define	NDIS_RXCSUM_INFO_TCPCS_FAILED	0x0001
+#define	NDIS_RXCSUM_INFO_UDPCS_FAILED	0x0002
+#define	NDIS_RXCSUM_INFO_IPCS_FAILED	0x0004
+#define	NDIS_RXCSUM_INFO_TCPCS_OK	0x0008
+#define	NDIS_RXCSUM_INFO_UDPCS_OK	0x0010
+#define	NDIS_RXCSUM_INFO_IPCS_OK	0x0020
+#define	NDIS_RXCSUM_INFO_LOOPBACK	0x0040
+#define	NDIS_RXCSUM_INFO_TCPCS_INVAL	0x0080
+#define	NDIS_RXCSUM_INFO_IPCS_INVAL	0x0100
+
+/* LSOv2 */
+#define	NDIS_LSO2_INFO_SIZE		sizeof(uint32_t)
+#define	NDIS_LSO2_INFO_MSS_MASK		0x000fffff
+#define	NDIS_LSO2_INFO_THOFF_MASK	0x3ff00000
+#define	NDIS_LSO2_INFO_ISLSO2		0x40000000
+#define	NDIS_LSO2_INFO_ISIPV6		0x80000000
+
+#define	NDIS_LSO2_INFO_MAKE(thoff, mss)				\
+	((((uint32_t)(mss)) & NDIS_LSO2_INFO_MSS_MASK) |	\
+	 ((((uint32_t)(thoff)) & 0x3ff) << 20) |		\
+	 NDIS_LSO2_INFO_ISLSO2)
+
+#define	NDIS_LSO2_INFO_MAKEIPV4(thoff, mss)			\
+	NDIS_LSO2_INFO_MAKE((thoff), (mss))
+
+#define	NDIS_LSO2_INFO_MAKEIPV6(thoff, mss)			\
+	(NDIS_LSO2_INFO_MAKE((thoff), (mss)) | NDIS_LSO2_INFO_ISIPV6)
+
+/* Transmission checksum */
+#define	NDIS_TXCSUM_INFO_SIZE		sizeof(uint32_t)
+#define	NDIS_TXCSUM_INFO_IPV4		0x00000001
+#define	NDIS_TXCSUM_INFO_IPV6		0x00000002
+#define	NDIS_TXCSUM_INFO_TCPCS		0x00000004
+#define	NDIS_TXCSUM_INFO_UDPCS		0x00000008
+#define	NDIS_TXCSUM_INFO_IPCS		0x00000010
+#define	NDIS_TXCSUM_INFO_THOFF		0x03ff0000
+
+#define	NDIS_TXCSUM_INFO_MKL4CS(thoff, flag)			\
+	((((uint32_t)(thoff)) << 16) | (flag))
+
+#define	NDIS_TXCSUM_INFO_MKTCPCS(thoff)				\
+	NDIS_TXCSUM_INFO_MKL4CS((thoff), NDIS_TXCSUM_INFO_TCPCS)
+
+#define	NDIS_TXCSUM_INFO_MKUDPCS(thoff)				\
+	NDIS_TXCSUM_INFO_MKL4CS((thoff), NDIS_TXCSUM_INFO_UDPCS)
+
+#endif	/* !_NET_NDIS_H_ */
diff --git a/drivers/net/netvsc/rndis.h b/drivers/net/netvsc/rndis.h
new file mode 100644
index 00000000..eac9a99f
--- /dev/null
+++ b/drivers/net/netvsc/rndis.h
@@ -0,0 +1,414 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018 Microsoft Corp.
+ * Copyright (c) 2010 Jonathan Armani <armani@openbsd.org>
+ * Copyright (c) 2010 Fabien Romano <fabien@openbsd.org>
+ * Copyright (c) 2010 Michael Knudsen <mk@openbsd.org>
+ * All rights reserved.
+ */
+
+#ifndef	_NET_RNDIS_H_
+#define	_NET_RNDIS_H_
+
+/* Canonical major/minor version as of 22th Aug. 2016. */
+#define	RNDIS_VERSION_MAJOR		0x00000001
+#define	RNDIS_VERSION_MINOR		0x00000000
+
+#define	RNDIS_STATUS_SUCCESS		0x00000000
+#define	RNDIS_STATUS_PENDING		0x00000103
+
+#define RNDIS_STATUS_ONLINE		0x40010003
+#define RNDIS_STATUS_RESET_START	0x40010004
+#define RNDIS_STATUS_RESET_END		0x40010005
+#define RNDIS_STATUS_RING_STATUS	0x40010006
+#define RNDIS_STATUS_CLOSED		0x40010007
+#define RNDIS_STATUS_WAN_LINE_UP	0x40010008
+#define RNDIS_STATUS_WAN_LINE_DOWN	0x40010009
+#define RNDIS_STATUS_WAN_FRAGMENT	0x4001000A
+#define	RNDIS_STATUS_MEDIA_CONNECT	0x4001000B
+#define	RNDIS_STATUS_MEDIA_DISCONNECT	0x4001000C
+#define RNDIS_STATUS_HARDWARE_LINE_UP	0x4001000D
+#define RNDIS_STATUS_HARDWARE_LINE_DOWN	0x4001000E
+#define RNDIS_STATUS_INTERFACE_UP	0x4001000F
+#define RNDIS_STATUS_INTERFACE_DOWN	0x40010010
+#define RNDIS_STATUS_MEDIA_BUSY		0x40010011
+#define	RNDIS_STATUS_MEDIA_SPECIFIC_INDICATION	0x40010012
+#define RNDIS_STATUS_WW_INDICATION	RDIA_SPECIFIC_INDICATION
+#define RNDIS_STATUS_LINK_SPEED_CHANGE	0x40010013
+#define RNDIS_STATUS_NETWORK_CHANGE	0x40010018
+#define	RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG 0x40020006
+
+#define	RNDIS_STATUS_FAILURE		0xC0000001
+#define RNDIS_STATUS_RESOURCES		0xC000009A
+#define	RNDIS_STATUS_NOT_SUPPORTED	0xC00000BB
+#define RNDIS_STATUS_CLOSING		0xC0010002
+#define RNDIS_STATUS_BAD_VERSION	0xC0010004
+#define RNDIS_STATUS_BAD_CHARACTERISTICS 0xC0010005
+#define RNDIS_STATUS_ADAPTER_NOT_FOUND	0xC0010006
+#define RNDIS_STATUS_OPEN_FAILED	0xC0010007
+#define RNDIS_STATUS_DEVICE_FAILED	0xC0010008
+#define RNDIS_STATUS_MULTICAST_FULL	0xC0010009
+#define RNDIS_STATUS_MULTICAST_EXISTS	0xC001000A
+#define RNDIS_STATUS_MULTICAST_NOT_FOUND 0xC001000B
+#define RNDIS_STATUS_REQUEST_ABORTED	0xC001000C
+#define RNDIS_STATUS_RESET_IN_PROGRESS	0xC001000D
+#define RNDIS_STATUS_CLOSING_INDICATING	0xC001000E
+#define RNDIS_STATUS_INVALID_PACKET	0xC001000F
+#define RNDIS_STATUS_OPEN_LIST_FULL	0xC0010010
+#define RNDIS_STATUS_ADAPTER_NOT_READY	0xC0010011
+#define RNDIS_STATUS_ADAPTER_NOT_OPEN	0xC0010012
+#define RNDIS_STATUS_NOT_INDICATING	0xC0010013
+#define RNDIS_STATUS_INVALID_LENGTH	0xC0010014
+#define	RNDIS_STATUS_INVALID_DATA	0xC0010015
+#define RNDIS_STATUS_BUFFER_TOO_SHORT	0xC0010016
+#define RNDIS_STATUS_INVALID_OID	0xC0010017
+#define RNDIS_STATUS_ADAPTER_REMOVED	0xC0010018
+#define RNDIS_STATUS_UNSUPPORTED_MEDIA	0xC0010019
+#define RNDIS_STATUS_GROUP_ADDRESS_IN_US 0xC001001A
+#define RNDIS_STATUS_FILE_NOT_FOUND	0xC001001B
+#define RNDIS_STATUS_ERROR_READING_FILE	0xC001001C
+#define RNDIS_STATUS_ALREADY_MAPPED	0xC001001D
+#define RNDIS_STATUS_RESOURCE_CONFLICT	0xC001001E
+#define RNDIS_STATUS_NO_CABLE		0xC001001F
+
+#define	OID_GEN_SUPPORTED_LIST		0x00010101
+#define	OID_GEN_HARDWARE_STATUS		0x00010102
+#define	OID_GEN_MEDIA_SUPPORTED		0x00010103
+#define	OID_GEN_MEDIA_IN_USE		0x00010104
+#define	OID_GEN_MAXIMUM_LOOKAHEAD	0x00010105
+#define	OID_GEN_MAXIMUM_FRAME_SIZE	0x00010106
+#define	OID_GEN_LINK_SPEED		0x00010107
+#define	OID_GEN_TRANSMIT_BUFFER_SPACE	0x00010108
+#define	OID_GEN_RECEIVE_BUFFER_SPACE	0x00010109
+#define	OID_GEN_TRANSMIT_BLOCK_SIZE	0x0001010A
+#define	OID_GEN_RECEIVE_BLOCK_SIZE	0x0001010B
+#define	OID_GEN_VENDOR_ID		0x0001010C
+#define	OID_GEN_VENDOR_DESCRIPTION	0x0001010D
+#define	OID_GEN_CURRENT_PACKET_FILTER	0x0001010E
+#define	OID_GEN_CURRENT_LOOKAHEAD	0x0001010F
+#define	OID_GEN_DRIVER_VERSION		0x00010110
+#define	OID_GEN_MAXIMUM_TOTAL_SIZE	0x00010111
+#define	OID_GEN_PROTOCOL_OPTIONS	0x00010112
+#define	OID_GEN_MAC_OPTIONS		0x00010113
+#define	OID_GEN_MEDIA_CONNECT_STATUS	0x00010114
+#define	OID_GEN_MAXIMUM_SEND_PACKETS	0x00010115
+#define	OID_GEN_VENDOR_DRIVER_VERSION	0x00010116
+#define	OID_GEN_SUPPORTED_GUIDS		0x00010117
+#define	OID_GEN_NETWORK_LAYER_ADDRESSES	0x00010118
+#define	OID_GEN_TRANSPORT_HEADER_OFFSET	0x00010119
+#define	OID_GEN_RECEIVE_SCALE_CAPABILITIES	0x00010203
+#define	OID_GEN_RECEIVE_SCALE_PARAMETERS	0x00010204
+#define	OID_GEN_MACHINE_NAME		0x0001021A
+#define	OID_GEN_RNDIS_CONFIG_PARAMETER	0x0001021B
+#define	OID_GEN_VLAN_ID			0x0001021C
+
+#define	OID_802_3_PERMANENT_ADDRESS	0x01010101
+#define	OID_802_3_CURRENT_ADDRESS	0x01010102
+#define	OID_802_3_MULTICAST_LIST	0x01010103
+#define	OID_802_3_MAXIMUM_LIST_SIZE	0x01010104
+#define	OID_802_3_MAC_OPTIONS		0x01010105
+#define	OID_802_3_RCV_ERROR_ALIGNMENT	0x01020101
+#define	OID_802_3_XMIT_ONE_COLLISION	0x01020102
+#define	OID_802_3_XMIT_MORE_COLLISIONS	0x01020103
+#define	OID_802_3_XMIT_DEFERRED		0x01020201
+#define	OID_802_3_XMIT_MAX_COLLISIONS	0x01020202
+#define	OID_802_3_RCV_OVERRUN		0x01020203
+#define	OID_802_3_XMIT_UNDERRUN		0x01020204
+#define	OID_802_3_XMIT_HEARTBEAT_FAILURE	0x01020205
+#define	OID_802_3_XMIT_TIMES_CRS_LOST	0x01020206
+#define	OID_802_3_XMIT_LATE_COLLISIONS	0x01020207
+
+#define	OID_TCP_OFFLOAD_PARAMETERS	0xFC01020C
+#define	OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES	0xFC01020D
+
+#define	RNDIS_MEDIUM_802_3		0x00000000
+
+/* Device flags */
+#define	RNDIS_DF_CONNECTIONLESS		0x00000001
+#define	RNDIS_DF_CONNECTION_ORIENTED	0x00000002
+
+/*
+ * Common RNDIS message header.
+ */
+struct rndis_msghdr {
+	uint32_t type;
+	uint32_t len;
+};
+
+/*
+ * RNDIS data message
+ */
+#define	RNDIS_PACKET_MSG		0x00000001
+
+struct rndis_packet_msg {
+	uint32_t type;
+	uint32_t len;
+	uint32_t dataoffset;
+	uint32_t datalen;
+	uint32_t oobdataoffset;
+	uint32_t oobdatalen;
+	uint32_t oobdataelements;
+	uint32_t pktinfooffset;
+	uint32_t pktinfolen;
+	uint32_t vchandle;
+	uint32_t reserved;
+};
+
+/*
+ * Minimum value for dataoffset, oobdataoffset, and
+ * pktinfooffset.
+ */
+#define	RNDIS_PACKET_MSG_OFFSET_MIN		\
+	(sizeof(struct rndis_packet_msg) -	\
+	 offsetof(struct rndis_packet_msg, dataoffset))
+
+/* Offset from the beginning of rndis_packet_msg. */
+#define	RNDIS_PACKET_MSG_OFFSET_ABS(ofs)	\
+	((ofs) + offsetof(struct rndis_packet_msg, dataoffset))
+
+#define	RNDIS_PACKET_MSG_OFFSET_ALIGN		4
+#define	RNDIS_PACKET_MSG_OFFSET_ALIGNMASK	\
+	(RNDIS_PACKET_MSG_OFFSET_ALIGN - 1)
+
+/* Per-packet-info for RNDIS data message */
+struct rndis_pktinfo {
+	uint32_t size;
+	uint32_t type;		/* NDIS_PKTINFO_TYPE_ */
+	uint32_t offset;
+	uint8_t data[];
+};
+
+#define	RNDIS_PKTINFO_OFFSET		\
+	offsetof(struct rndis_pktinfo, data[0])
+#define	RNDIS_PKTINFO_SIZE_ALIGN	4
+#define	RNDIS_PKTINFO_SIZE_ALIGNMASK	(RNDIS_PKTINFO_SIZE_ALIGN - 1)
+
+#define	NDIS_PKTINFO_TYPE_CSUM		0
+#define	NDIS_PKTINFO_TYPE_IPSEC		1
+#define	NDIS_PKTINFO_TYPE_LSO		2
+#define	NDIS_PKTINFO_TYPE_CLASSIFY	3
+/* reserved 4 */
+#define	NDIS_PKTINFO_TYPE_SGLIST	5
+#define	NDIS_PKTINFO_TYPE_VLAN		6
+#define	NDIS_PKTINFO_TYPE_ORIG		7
+#define	NDIS_PKTINFO_TYPE_PKT_CANCELID	8
+#define	NDIS_PKTINFO_TYPE_ORIG_NBLIST	9
+#define	NDIS_PKTINFO_TYPE_CACHE_NBLIST	10
+#define	NDIS_PKTINFO_TYPE_PKT_PAD	11
+
+/* RNDIS extension */
+
+/* Per-packet hash info */
+#define NDIS_HASH_INFO_SIZE		sizeof(uint32_t)
+#define NDIS_PKTINFO_TYPE_HASHINF	NDIS_PKTINFO_TYPE_ORIG_NBLIST
+/* NDIS_HASH_ */
+
+/* Per-packet hash value */
+#define NDIS_HASH_VALUE_SIZE		sizeof(uint32_t)
+#define NDIS_PKTINFO_TYPE_HASHVAL	NDIS_PKTINFO_TYPE_PKT_CANCELID
+
+/* Per-packet-info size */
+#define RNDIS_PKTINFO_SIZE(dlen)	offsetof(struct rndis_pktinfo, data[dlen])
+
+/*
+ * RNDIS control messages
+ */
+
+/*
+ * Common header for RNDIS completion messages.
+ *
+ * NOTE: It does not apply to RNDIS_RESET_CMPLT.
+ */
+struct rndis_comp_hdr {
+	uint32_t type;
+	uint32_t len;
+	uint32_t rid;
+	uint32_t status;
+};
+
+/* Initialize the device. */
+#define	RNDIS_INITIALIZE_MSG	0x00000002
+#define	RNDIS_INITIALIZE_CMPLT	0x80000002
+
+struct rndis_init_req {
+	uint32_t type;
+	uint32_t len;
+	uint32_t rid;
+	uint32_t ver_major;
+	uint32_t ver_minor;
+	uint32_t max_xfersz;
+};
+
+struct rndis_init_comp {
+	uint32_t type;
+	uint32_t len;
+	uint32_t rid;
+	uint32_t status;
+	uint32_t ver_major;
+	uint32_t ver_minor;
+	uint32_t devflags;
+	uint32_t medium;
+	uint32_t pktmaxcnt;
+	uint32_t pktmaxsz;
+	uint32_t align;
+	uint32_t aflistoffset;
+	uint32_t aflistsz;
+};
+
+#define	RNDIS_INIT_COMP_SIZE_MIN	\
+	offsetof(struct rndis_init_comp, aflistsz)
+
+/* Halt the device.  No response sent. */
+#define	RNDIS_HALT_MSG		0x00000003
+
+struct rndis_halt_req {
+	uint32_t type;
+	uint32_t len;
+	uint32_t rid;
+};
+
+/* Send a query object. */
+#define	RNDIS_QUERY_MSG		0x00000004
+#define	RNDIS_QUERY_CMPLT	0x80000004
+
+struct rndis_query_req {
+	uint32_t type;
+	uint32_t len;
+	uint32_t rid;
+	uint32_t oid;
+	uint32_t infobuflen;
+	uint32_t infobufoffset;
+	uint32_t devicevchdl;
+};
+
+#define	RNDIS_QUERY_REQ_INFOBUFOFFSET		\
+	(sizeof(struct rndis_query_req) -	\
+	 offsetof(struct rndis_query_req, rid))
+
+struct rndis_query_comp {
+	uint32_t type;
+	uint32_t len;
+	uint32_t rid;
+	uint32_t status;
+	uint32_t infobuflen;
+	uint32_t infobufoffset;
+};
+
+/* infobuf offset from the beginning of rndis_query_comp. */
+#define	RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(ofs)	\
+	((ofs) + offsetof(struct rndis_query_comp, rid))
+
+/* Send a set object request. */
+#define	RNDIS_SET_MSG		0x00000005
+#define	RNDIS_SET_CMPLT		0x80000005
+
+struct rndis_set_req {
+	uint32_t type;
+	uint32_t len;
+	uint32_t rid;
+	uint32_t oid;
+	uint32_t infobuflen;
+	uint32_t infobufoffset;
+	uint32_t devicevchdl;
+};
+
+#define	RNDIS_SET_REQ_INFOBUFOFFSET		\
+	(sizeof(struct rndis_set_req) -		\
+	 offsetof(struct rndis_set_req, rid))
+
+struct rndis_set_comp {
+	uint32_t type;
+	uint32_t len;
+	uint32_t rid;
+	uint32_t status;
+};
+
+/*
+ * Parameter used by OID_GEN_RNDIS_CONFIG_PARAMETER.
+ */
+#define	RNDIS_SET_PARAM_NUMERIC	0x00000000
+#define	RNDIS_SET_PARAM_STRING	0x00000002
+
+struct rndis_set_parameter {
+	uint32_t nameoffset;
+	uint32_t namelen;
+	uint32_t type;
+	uint32_t valueoffset;
+	uint32_t valuelen;
+};
+
+/* Perform a soft reset on the device. */
+#define	RNDIS_RESET_MSG		0x00000006
+#define	RNDIS_RESET_CMPLT		0x80000006
+
+struct rndis_reset_req {
+	uint32_t type;
+	uint32_t len;
+	uint32_t rid;
+};
+
+struct rndis_reset_comp {
+	uint32_t type;
+	uint32_t len;
+	uint32_t status;
+	uint32_t adrreset;
+};
+
+/* 802.3 link-state or undefined message error.  Sent by device. */
+#define	RNDIS_INDICATE_STATUS_MSG	0x00000007
+
+struct rndis_status_msg {
+	uint32_t type;
+	uint32_t len;
+	uint32_t status;
+	uint32_t stbuflen;
+	uint32_t stbufoffset;
+	/* rndis_diag_info */
+};
+
+/* stbuf offset from the beginning of rndis_status_msg. */
+#define	RNDIS_STBUFOFFSET_ABS(ofs)	\
+	((ofs) + offsetof(struct rndis_status_msg, status))
+
+/*
+ * Immediately after rndis_status_msg.stbufoffset, if a control
+ * message is malformatted, or a packet message contains inappropriate
+ * content.
+ */
+struct rndis_diag_info {
+	uint32_t diagstatus;
+	uint32_t erroffset;
+};
+
+/* Keepalive message.  May be sent by device. */
+#define	RNDIS_KEEPALIVE_MSG	0x00000008
+#define	RNDIS_KEEPALIVE_CMPLT	0x80000008
+
+struct rndis_keepalive_req {
+	uint32_t type;
+	uint32_t len;
+	uint32_t rid;
+};
+
+struct rndis_keepalive_comp {
+	uint32_t type;
+	uint32_t len;
+	uint32_t rid;
+	uint32_t status;
+};
+
+/* Packet filter bits used by OID_GEN_CURRENT_PACKET_FILTER */
+#define	NDIS_PACKET_TYPE_NONE			0x00000000
+#define	NDIS_PACKET_TYPE_DIRECTED		0x00000001
+#define	NDIS_PACKET_TYPE_MULTICAST		0x00000002
+#define	NDIS_PACKET_TYPE_ALL_MULTICAST		0x00000004
+#define	NDIS_PACKET_TYPE_BROADCAST		0x00000008
+#define	NDIS_PACKET_TYPE_SOURCE_ROUTING		0x00000010
+#define	NDIS_PACKET_TYPE_PROMISCUOUS		0x00000020
+#define	NDIS_PACKET_TYPE_SMT			0x00000040
+#define	NDIS_PACKET_TYPE_ALL_LOCAL		0x00000080
+#define	NDIS_PACKET_TYPE_GROUP			0x00001000
+#define	NDIS_PACKET_TYPE_ALL_FUNCTIONAL		0x00002000
+#define	NDIS_PACKET_TYPE_FUNCTIONAL		0x00004000
+#define	NDIS_PACKET_TYPE_MAC_FRAME		0x00008000
+
+#endif	/* !_NET_RNDIS_H_ */
diff --git a/drivers/net/netvsc/rte_pmd_netvsc_version.map b/drivers/net/netvsc/rte_pmd_netvsc_version.map
new file mode 100644
index 00000000..d534019a
--- /dev/null
+++ b/drivers/net/netvsc/rte_pmd_netvsc_version.map
@@ -0,0 +1,5 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+
+DPDK_18.08 {
+	local: *;
+};
diff --git a/drivers/net/nfp/meson.build b/drivers/net/nfp/meson.build
new file mode 100644
index 00000000..3ba37e27
--- /dev/null
+++ b/drivers/net/nfp/meson.build
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+sources = files('nfpcore/nfp_cpp_pcie_ops.c',
+	'nfpcore/nfp_nsp.c',
+	'nfpcore/nfp_cppcore.c',
+	'nfpcore/nfp_resource.c',
+	'nfpcore/nfp_mip.c',
+	'nfpcore/nfp_nffw.c',
+	'nfpcore/nfp_rtsym.c',
+	'nfpcore/nfp_nsp_cmds.c',
+	'nfpcore/nfp_crc.c',
+	'nfpcore/nfp_mutex.c',
+	'nfpcore/nfp_nsp_eth.c',
+	'nfpcore/nfp_hwinfo.c',
+	'nfp_net.c')
diff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c
index faad1ee9..6e5e305f 100644
--- a/drivers/net/nfp/nfp_net.c
+++ b/drivers/net/nfp/nfp_net.c
@@ -411,8 +411,10 @@ nfp_net_configure(struct rte_eth_dev *dev)
 		return -EINVAL;
 	}
 
-	/* Checking RX offloads */
-	if (!(rxmode->offloads & DEV_RX_OFFLOAD_CRC_STRIP))
+	/* KEEP_CRC offload flag is not supported by PMD
+	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
+	 */
+	if (rte_eth_dev_must_keep_crc(rxmode->offloads))
 		PMD_INIT_LOG(INFO, "HW does strip CRC. No configurable!");
 
 	return 0;
@@ -1166,7 +1168,8 @@ nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 					     DEV_RX_OFFLOAD_UDP_CKSUM |
 					     DEV_RX_OFFLOAD_TCP_CKSUM;
 
-	dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME;
+	dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME |
+				     DEV_RX_OFFLOAD_KEEP_CRC;
 
 	if (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)
 		dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT;
@@ -1436,9 +1439,9 @@ nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 
 	/* switch to jumbo mode if needed */
 	if ((uint32_t)mtu > ETHER_MAX_LEN)
-		dev->data->dev_conf.rxmode.jumbo_frame = 1;
+		dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
 	else
-		dev->data->dev_conf.rxmode.jumbo_frame = 0;
+		dev->data->dev_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
 
 	/* update max frame size */
 	dev->data->dev_conf.rxmode.max_rx_pkt_len = (uint32_t)mtu;
@@ -2253,11 +2256,15 @@ nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 				txq->wr_p = 0;
 
 			pkt_size -= dma_size;
-			if (!pkt_size)
-				/* End of packet */
-				txds->offset_eop |= PCIE_DESC_TX_EOP;
+
+			/*
+			 * Making the EOP, packets with just one segment
+			 * the priority
+			 */
+			if (likely(!pkt_size))
+				txds->offset_eop = PCIE_DESC_TX_EOP;
 			else
-				txds->offset_eop &= PCIE_DESC_TX_OFFSET_MASK;
+				txds->offset_eop = 0;
 
 			pkt = pkt->next;
 			/* Referencing next free TX descriptor */
@@ -3126,9 +3133,9 @@ static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	 * use a lock file if UIO is being used.
 	 */
 	if (dev->kdrv == RTE_KDRV_VFIO)
-		cpp = nfp_cpp_from_device_name(dev->device.name, 0);
+		cpp = nfp_cpp_from_device_name(dev, 0);
 	else
-		cpp = nfp_cpp_from_device_name(dev->device.name, 1);
+		cpp = nfp_cpp_from_device_name(dev, 1);
 
 	if (!cpp) {
 		PMD_DRV_LOG(ERR, "A CPP handle can not be obtained");
@@ -3277,9 +3284,7 @@ RTE_PMD_REGISTER_PCI_TABLE(net_nfp_vf, pci_id_nfp_vf_net_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio");
 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_vf, "* igb_uio | uio_pci_generic | vfio");
 
-RTE_INIT(nfp_init_log);
-static void
-nfp_init_log(void)
+RTE_INIT(nfp_init_log)
 {
 	nfp_logtype_init = rte_log_register("pmd.net.nfp.init");
 	if (nfp_logtype_init >= 0)
diff --git a/drivers/net/nfp/nfpcore/nfp-common/nfp_platform.h b/drivers/net/nfp/nfpcore/nfp-common/nfp_platform.h
index b8541c59..d46574b1 100644
--- a/drivers/net/nfp/nfpcore/nfp-common/nfp_platform.h
+++ b/drivers/net/nfp/nfpcore/nfp-common/nfp_platform.h
@@ -13,7 +13,6 @@
 #include <stdlib.h>
 #include <ctype.h>
 #include <inttypes.h>
-#include <sys/cdefs.h>
 #include <sys/stat.h>
 #include <limits.h>
 #include <errno.h>
diff --git a/drivers/net/nfp/nfpcore/nfp_cpp.h b/drivers/net/nfp/nfpcore/nfp_cpp.h
index de2ff84e..1427954c 100644
--- a/drivers/net/nfp/nfpcore/nfp_cpp.h
+++ b/drivers/net/nfp/nfpcore/nfp_cpp.h
@@ -6,6 +6,8 @@
 #ifndef __NFP_CPP_H__
 #define __NFP_CPP_H__
 
+#include <rte_ethdev_pci.h>
+
 #include "nfp-common/nfp_platform.h"
 #include "nfp-common/nfp_resid.h"
 
@@ -54,7 +56,7 @@ struct nfp_cpp_operations {
 	size_t area_priv_size;
 
 	/* Instance an NFP CPP */
-	int (*init)(struct nfp_cpp *cpp, const char *devname);
+	int (*init)(struct nfp_cpp *cpp, struct rte_pci_device *dev);
 
 	/*
 	 * Free the bus.
@@ -181,7 +183,7 @@ uint32_t __nfp_cpp_model_autodetect(struct nfp_cpp *cpp);
  *
  * @return NFP CPP handle, or NULL on failure (and set errno accordingly).
  */
-struct nfp_cpp *nfp_cpp_from_device_name(const char *devname,
+struct nfp_cpp *nfp_cpp_from_device_name(struct rte_pci_device *dev,
 					 int driver_lock_needed);
 
 /*
diff --git a/drivers/net/nfp/nfpcore/nfp_cpp_pcie_ops.c b/drivers/net/nfp/nfpcore/nfp_cpp_pcie_ops.c
index 2f5e7f6d..c68d9400 100644
--- a/drivers/net/nfp/nfpcore/nfp_cpp_pcie_ops.c
+++ b/drivers/net/nfp/nfpcore/nfp_cpp_pcie_ops.c
@@ -31,6 +31,7 @@
 #include <sys/file.h>
 #include <sys/stat.h>
 
+#include <rte_ethdev_pci.h>
 #include <rte_string_fns.h>
 
 #include "nfp_cpp.h"
@@ -309,13 +310,8 @@ nfp_enable_bars(struct nfp_pcie_user *nfp)
 		bar->csr = nfp->cfg +
 			   NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(bar->index >> 3,
 							   bar->index & 7);
-		bar->iomem =
-		    (char *)mmap(0, 1 << bar->bitsize, PROT_READ | PROT_WRITE,
-				 MAP_SHARED, nfp->device,
-				 bar->index << bar->bitsize);
 
-		if (bar->iomem == MAP_FAILED)
-			return (-ENOMEM);
+		bar->iomem = nfp->cfg + (bar->index << bar->bitsize);
 	}
 	return 0;
 }
@@ -345,7 +341,6 @@ nfp_disable_bars(struct nfp_pcie_user *nfp)
 	for (x = ARRAY_SIZE(nfp->bar); x > 0; x--) {
 		bar = &nfp->bar[x - 1];
 		if (bar->iomem) {
-			munmap(bar->iomem, 1 << (nfp->barsz - 3));
 			bar->iomem = NULL;
 			bar->lock = 0;
 		}
@@ -639,61 +634,32 @@ nfp_acquire_process_lock(struct nfp_pcie_user *desc)
 }
 
 static int
-nfp6000_set_model(struct nfp_pcie_user *desc, struct nfp_cpp *cpp)
+nfp6000_set_model(struct rte_pci_device *dev, struct nfp_cpp *cpp)
 {
-	char tmp_str[80];
-	uint32_t tmp;
-	int fp;
-
-	snprintf(tmp_str, sizeof(tmp_str), "%s/%s/config", PCI_DEVICES,
-		 desc->busdev);
-
-	fp = open(tmp_str, O_RDONLY);
-	if (!fp)
-		return -1;
-
-	lseek(fp, 0x2e, SEEK_SET);
+	uint32_t model;
 
-	if (read(fp, &tmp, sizeof(tmp)) != sizeof(tmp)) {
-		printf("Error reading config file for model\n");
+	if (rte_pci_read_config(dev, &model, 4, 0x2e) < 0) {
+		printf("nfp set model failed\n");
 		return -1;
 	}
 
-	tmp = tmp << 16;
-
-	if (close(fp) == -1)
-		return -1;
-
-	nfp_cpp_model_set(cpp, tmp);
+	model  = model << 16;
+	nfp_cpp_model_set(cpp, model);
 
 	return 0;
 }
 
 static int
-nfp6000_set_interface(struct nfp_pcie_user *desc, struct nfp_cpp *cpp)
+nfp6000_set_interface(struct rte_pci_device *dev, struct nfp_cpp *cpp)
 {
-	char tmp_str[80];
-	uint16_t tmp;
-	int fp;
-
-	snprintf(tmp_str, sizeof(tmp_str), "%s/%s/config", PCI_DEVICES,
-		 desc->busdev);
+	uint16_t interface;
 
-	fp = open(tmp_str, O_RDONLY);
-	if (!fp)
-		return -1;
-
-	lseek(fp, 0x154, SEEK_SET);
-
-	if (read(fp, &tmp, sizeof(tmp)) != sizeof(tmp)) {
-		printf("error reading config file for interface\n");
+	if (rte_pci_read_config(dev, &interface, 2, 0x154) < 0) {
+		printf("nfp set interface failed\n");
 		return -1;
 	}
 
-	if (close(fp) == -1)
-		return -1;
-
-	nfp_cpp_interface_set(cpp, tmp);
+	nfp_cpp_interface_set(cpp, interface);
 
 	return 0;
 }
@@ -704,7 +670,7 @@ nfp6000_set_interface(struct nfp_pcie_user *desc, struct nfp_cpp *cpp)
 #define PCI_EXT_CAP_NEXT(header)	((header >> 20) & 0xffc)
 #define PCI_EXT_CAP_ID_DSN	0x03
 static int
-nfp_pci_find_next_ext_capability(int fp, int cap)
+nfp_pci_find_next_ext_capability(struct rte_pci_device *dev, int cap)
 {
 	uint32_t header;
 	int ttl;
@@ -713,9 +679,8 @@ nfp_pci_find_next_ext_capability(int fp, int cap)
 	/* minimum 8 bytes per capability */
 	ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
 
-	lseek(fp, pos, SEEK_SET);
-	if (read(fp, &header, sizeof(header)) != sizeof(header)) {
-		printf("error reading config file for serial\n");
+	if (rte_pci_read_config(dev, &header, 4, pos) < 0) {
+		printf("nfp error reading extended capabilities\n");
 		return -1;
 	}
 
@@ -734,9 +699,8 @@ nfp_pci_find_next_ext_capability(int fp, int cap)
 		if (pos < PCI_CFG_SPACE_SIZE)
 			break;
 
-		lseek(fp, pos, SEEK_SET);
-		if (read(fp, &header, sizeof(header)) != sizeof(header)) {
-			printf("error reading config file for serial\n");
+		if (rte_pci_read_config(dev, &header, 4, pos) < 0) {
+			printf("nfp error reading extended capabilities\n");
 			return -1;
 		}
 	}
@@ -745,99 +709,70 @@ nfp_pci_find_next_ext_capability(int fp, int cap)
 }
 
 static int
-nfp6000_set_serial(struct nfp_pcie_user *desc, struct nfp_cpp *cpp)
+nfp6000_set_serial(struct rte_pci_device *dev, struct nfp_cpp *cpp)
 {
-	char tmp_str[80];
 	uint16_t tmp;
 	uint8_t serial[6];
 	int serial_len = 6;
-	int fp, pos;
+	int pos;
 
-	snprintf(tmp_str, sizeof(tmp_str), "%s/%s/config", PCI_DEVICES,
-		 desc->busdev);
-
-	fp = open(tmp_str, O_RDONLY);
-	if (!fp)
-		return -1;
-
-	pos = nfp_pci_find_next_ext_capability(fp, PCI_EXT_CAP_ID_DSN);
+	pos = nfp_pci_find_next_ext_capability(dev, PCI_EXT_CAP_ID_DSN);
 	if (pos <= 0) {
-		printf("PCI_EXT_CAP_ID_DSN not found. Using default offset\n");
-		lseek(fp, 0x156, SEEK_SET);
+		printf("PCI_EXT_CAP_ID_DSN not found. nfp set serial failed\n");
+		return -1;
 	} else {
-		lseek(fp, pos + 6, SEEK_SET);
+		pos += 6;
 	}
 
-	if (read(fp, &tmp, sizeof(tmp)) != sizeof(tmp)) {
-		printf("error reading config file for serial\n");
+	if (rte_pci_read_config(dev, &tmp, 2, pos) < 0) {
+		printf("nfp set serial failed\n");
 		return -1;
 	}
 
 	serial[4] = (uint8_t)((tmp >> 8) & 0xff);
 	serial[5] = (uint8_t)(tmp & 0xff);
 
-	if (read(fp, &tmp, sizeof(tmp)) != sizeof(tmp)) {
-		printf("error reading config file for serial\n");
+	pos += 2;
+	if (rte_pci_read_config(dev, &tmp, 2, pos) < 0) {
+		printf("nfp set serial failed\n");
 		return -1;
 	}
 
 	serial[2] = (uint8_t)((tmp >> 8) & 0xff);
 	serial[3] = (uint8_t)(tmp & 0xff);
 
-	if (read(fp, &tmp, sizeof(tmp)) != sizeof(tmp)) {
-		printf("error reading config file for serial\n");
+	pos += 2;
+	if (rte_pci_read_config(dev, &tmp, 2, pos) < 0) {
+		printf("nfp set serial failed\n");
 		return -1;
 	}
 
 	serial[0] = (uint8_t)((tmp >> 8) & 0xff);
 	serial[1] = (uint8_t)(tmp & 0xff);
 
-	if (close(fp) == -1)
-		return -1;
-
 	nfp_cpp_serial_set(cpp, serial, serial_len);
 
 	return 0;
 }
 
 static int
-nfp6000_set_barsz(struct nfp_pcie_user *desc)
+nfp6000_set_barsz(struct rte_pci_device *dev, struct nfp_pcie_user *desc)
 {
-	char tmp_str[80];
-	unsigned long start, end, flags, tmp;
-	int i;
-	FILE *fp;
-
-	snprintf(tmp_str, sizeof(tmp_str), "%s/%s/resource", PCI_DEVICES,
-		 desc->busdev);
-
-	fp = fopen(tmp_str, "r");
-	if (!fp)
-		return -1;
+	unsigned long tmp;
+	int i = 0;
 
-	if (fscanf(fp, "0x%lx 0x%lx 0x%lx", &start, &end, &flags) == 0) {
-		printf("error reading resource file for bar size\n");
-		fclose(fp);
-		return -1;
-	}
+	tmp = dev->mem_resource[0].len;
 
-	if (fclose(fp) == -1)
-		return -1;
-
-	tmp = (end - start) + 1;
-	i = 0;
 	while (tmp >>= 1)
 		i++;
+
 	desc->barsz = i;
 	return 0;
 }
 
 static int
-nfp6000_init(struct nfp_cpp *cpp, const char *devname)
+nfp6000_init(struct nfp_cpp *cpp, struct rte_pci_device *dev)
 {
-	char link[120];
-	char tmp_str[80];
-	ssize_t size;
 	int ret = 0;
 	uint32_t model;
 	struct nfp_pcie_user *desc;
@@ -848,7 +783,7 @@ nfp6000_init(struct nfp_cpp *cpp, const char *devname)
 
 
 	memset(desc->busdev, 0, BUSDEV_SZ);
-	strlcpy(desc->busdev, devname, sizeof(desc->busdev));
+	strlcpy(desc->busdev, dev->device.name, sizeof(desc->busdev));
 
 	if (cpp->driver_lock_needed) {
 		ret = nfp_acquire_process_lock(desc);
@@ -856,39 +791,16 @@ nfp6000_init(struct nfp_cpp *cpp, const char *devname)
 			return -1;
 	}
 
-	snprintf(tmp_str, sizeof(tmp_str), "%s/%s/driver", PCI_DEVICES,
-		 desc->busdev);
-
-	size = readlink(tmp_str, link, sizeof(link));
-
-	if (size == -1)
-		tmp_str[0] = '\0';
-
-	if (size == sizeof(link))
-		tmp_str[0] = '\0';
-
-	snprintf(tmp_str, sizeof(tmp_str), "%s/%s/resource0", PCI_DEVICES,
-		 desc->busdev);
-
-	desc->device = open(tmp_str, O_RDWR);
-	if (desc->device == -1)
-		return -1;
-
-	if (nfp6000_set_model(desc, cpp) < 0)
+	if (nfp6000_set_model(dev, cpp) < 0)
 		return -1;
-	if (nfp6000_set_interface(desc, cpp) < 0)
+	if (nfp6000_set_interface(dev, cpp) < 0)
 		return -1;
-	if (nfp6000_set_serial(desc, cpp) < 0)
+	if (nfp6000_set_serial(dev, cpp) < 0)
 		return -1;
-	if (nfp6000_set_barsz(desc) < 0)
+	if (nfp6000_set_barsz(dev, desc) < 0)
 		return -1;
 
-	desc->cfg = (char *)mmap(0, 1 << (desc->barsz - 3),
-				 PROT_READ | PROT_WRITE,
-				 MAP_SHARED, desc->device, 0);
-
-	if (desc->cfg == MAP_FAILED)
-		return -1;
+	desc->cfg = (char *)dev->mem_resource[0].addr;
 
 	nfp_enable_bars(desc);
 
@@ -904,16 +816,8 @@ static void
 nfp6000_free(struct nfp_cpp *cpp)
 {
 	struct nfp_pcie_user *desc = nfp_cpp_priv(cpp);
-	int x;
 
-	/* Unmap may cause if there are any pending transaxctions */
 	nfp_disable_bars(desc);
-	munmap(desc->cfg, 1 << (desc->barsz - 3));
-
-	for (x = ARRAY_SIZE(desc->bar); x > 0; x--) {
-		if (desc->bar[x - 1].iomem)
-			munmap(desc->bar[x - 1].iomem, 1 << (desc->barsz - 3));
-	}
 	if (cpp->driver_lock_needed)
 		close(desc->lock);
 	close(desc->device);
diff --git a/drivers/net/nfp/nfpcore/nfp_cppcore.c b/drivers/net/nfp/nfpcore/nfp_cppcore.c
index f61143f7..75d3c974 100644
--- a/drivers/net/nfp/nfpcore/nfp_cppcore.c
+++ b/drivers/net/nfp/nfpcore/nfp_cppcore.c
@@ -12,6 +12,7 @@
 #include <sys/types.h>
 
 #include <rte_byteorder.h>
+#include <rte_ethdev_pci.h>
 
 #include "nfp_cpp.h"
 #include "nfp_target.h"
@@ -542,7 +543,7 @@ nfp_xpb_readl(struct nfp_cpp *cpp, uint32_t xpb_addr, uint32_t *value)
 }
 
 static struct nfp_cpp *
-nfp_cpp_alloc(const char *devname, int driver_lock_needed)
+nfp_cpp_alloc(struct rte_pci_device *dev, int driver_lock_needed)
 {
 	const struct nfp_cpp_operations *ops;
 	struct nfp_cpp *cpp;
@@ -561,7 +562,7 @@ nfp_cpp_alloc(const char *devname, int driver_lock_needed)
 	cpp->driver_lock_needed = driver_lock_needed;
 
 	if (cpp->op->init) {
-		err = cpp->op->init(cpp, devname);
+		err = cpp->op->init(cpp, dev);
 		if (err < 0) {
 			free(cpp);
 			return NULL;
@@ -604,9 +605,9 @@ nfp_cpp_free(struct nfp_cpp *cpp)
 }
 
 struct nfp_cpp *
-nfp_cpp_from_device_name(const char *devname, int driver_lock_needed)
+nfp_cpp_from_device_name(struct rte_pci_device *dev, int driver_lock_needed)
 {
-	return nfp_cpp_alloc(devname, driver_lock_needed);
+	return nfp_cpp_alloc(dev, driver_lock_needed);
 }
 
 /*
diff --git a/drivers/net/null/rte_eth_null.c b/drivers/net/null/rte_eth_null.c
index 1d2e6b9e..244f8654 100644
--- a/drivers/net/null/rte_eth_null.c
+++ b/drivers/net/null/rte_eth_null.c
@@ -305,6 +305,7 @@ eth_dev_info(struct rte_eth_dev *dev,
 	dev_info->min_rx_bufsize = 0;
 	dev_info->reta_size = internals->reta_size;
 	dev_info->flow_type_rss_offloads = internals->flow_type_rss_offloads;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -623,6 +624,7 @@ rte_pmd_null_probe(struct rte_vdev_device *dev)
 		}
 		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &ops;
+		eth_dev->device = &dev->device;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -697,9 +699,7 @@ RTE_PMD_REGISTER_PARAM_STRING(net_null,
 	"size=<int> "
 	"copy=<int>");
 
-RTE_INIT(eth_null_init_log);
-static void
-eth_null_init_log(void)
+RTE_INIT(eth_null_init_log)
 {
 	eth_null_logtype = rte_log_register("pmd.net.null");
 	if (eth_null_logtype >= 0)
diff --git a/drivers/net/octeontx/octeontx_ethdev.c b/drivers/net/octeontx/octeontx_ethdev.c
index 1eb453b2..0f3d5d67 100644
--- a/drivers/net/octeontx/octeontx_ethdev.c
+++ b/drivers/net/octeontx/octeontx_ethdev.c
@@ -46,9 +46,7 @@ int otx_net_logtype_mbox;
 int otx_net_logtype_init;
 int otx_net_logtype_driver;
 
-RTE_INIT(otx_net_init_log);
-static void
-otx_net_init_log(void)
+RTE_INIT(otx_net_init_log)
 {
 	otx_net_logtype_mbox = rte_log_register("pmd.net.octeontx.mbox");
 	if (otx_net_logtype_mbox >= 0)
@@ -283,7 +281,10 @@ octeontx_dev_configure(struct rte_eth_dev *dev)
 		return -EINVAL;
 	}
 
-	if (!(rxmode->offloads & DEV_RX_OFFLOAD_CRC_STRIP)) {
+	/* KEEP_CRC offload flag is not supported by PMD
+	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
+	 */
+	if (rte_eth_dev_must_keep_crc(rxmode->offloads)) {
 		PMD_INIT_LOG(NOTICE, "can't disable hw crc strip");
 		rxmode->offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
 	}
@@ -352,6 +353,9 @@ octeontx_dev_close(struct rte_eth_dev *dev)
 
 		rte_free(txq);
 	}
+
+	dev->tx_pkt_burst = NULL;
+	dev->rx_pkt_burst = NULL;
 }
 
 static int
@@ -445,9 +449,6 @@ octeontx_dev_stop(struct rte_eth_dev *dev)
 			     ret);
 		return;
 	}
-
-	dev->tx_pkt_burst = NULL;
-	dev->rx_pkt_burst = NULL;
 }
 
 static void
@@ -787,7 +788,7 @@ octeontx_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
 	pki_qos_cfg_t pki_qos;
 	uintptr_t pool;
 	int ret, port;
-	uint8_t gaura;
+	uint16_t gaura;
 	unsigned int ev_queues = (nic->ev_queues * nic->port_id) + qidx;
 	unsigned int ev_ports = (nic->ev_ports * nic->port_id) + qidx;
 
@@ -898,8 +899,8 @@ octeontx_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
 
 		pool = (uintptr_t)mb_pool->pool_id;
 
-		/* Get the gpool Id */
-		gaura = octeontx_fpa_bufpool_gpool(pool);
+		/* Get the gaura Id */
+		gaura = octeontx_fpa_bufpool_gaura(pool);
 
 		pki_qos.qpg_qos = PKI_QPG_QOS_NONE;
 		pki_qos.num_entry = 1;
@@ -1014,6 +1015,8 @@ octeontx_create(struct rte_vdev_device *dev, int port, uint8_t evdev,
 		if (eth_dev == NULL)
 			return -ENODEV;
 
+		eth_dev->dev_ops = &octeontx_dev_ops;
+		eth_dev->device = &dev->device;
 		eth_dev->tx_pkt_burst = octeontx_xmit_pkts;
 		eth_dev->rx_pkt_burst = octeontx_recv_pkts;
 		rte_eth_dev_probing_finish(eth_dev);
@@ -1182,6 +1185,7 @@ octeontx_probe(struct rte_vdev_device *dev)
 		}
 		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &octeontx_dev_ops;
+		eth_dev->device = &dev->device;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
diff --git a/drivers/net/octeontx/octeontx_rxtx.c b/drivers/net/octeontx/octeontx_rxtx.c
index 2502d90e..a9149b4e 100644
--- a/drivers/net/octeontx/octeontx_rxtx.c
+++ b/drivers/net/octeontx/octeontx_rxtx.c
@@ -31,7 +31,7 @@ __octeontx_xmit_pkts(void *lmtline_va, void *ioreg_va, int64_t *fc_status_va,
 		return -ENOSPC;
 
 	/* Get the gaura Id */
-	gaura_id = octeontx_fpa_bufpool_gpool((uintptr_t)tx_pkt->pool->pool_id);
+	gaura_id = octeontx_fpa_bufpool_gaura((uintptr_t)tx_pkt->pool->pool_id);
 
 	/* Setup PKO_SEND_HDR_S */
 	cmd_buf[0] = tx_pkt->data_len & 0xffff;
diff --git a/drivers/net/pcap/rte_eth_pcap.c b/drivers/net/pcap/rte_eth_pcap.c
index 6bd4a7d7..e8810a17 100644
--- a/drivers/net/pcap/rte_eth_pcap.c
+++ b/drivers/net/pcap/rte_eth_pcap.c
@@ -26,6 +26,7 @@
 #define ETH_PCAP_RX_PCAP_ARG  "rx_pcap"
 #define ETH_PCAP_TX_PCAP_ARG  "tx_pcap"
 #define ETH_PCAP_RX_IFACE_ARG "rx_iface"
+#define ETH_PCAP_RX_IFACE_IN_ARG "rx_iface_in"
 #define ETH_PCAP_TX_IFACE_ARG "tx_iface"
 #define ETH_PCAP_IFACE_ARG    "iface"
 
@@ -83,6 +84,7 @@ static const char *valid_arguments[] = {
 	ETH_PCAP_RX_PCAP_ARG,
 	ETH_PCAP_TX_PCAP_ARG,
 	ETH_PCAP_RX_IFACE_ARG,
+	ETH_PCAP_RX_IFACE_IN_ARG,
 	ETH_PCAP_TX_IFACE_ARG,
 	ETH_PCAP_IFACE_ARG,
 	NULL
@@ -430,6 +432,7 @@ eth_dev_start(struct rte_eth_dev *dev)
 				return -1;
 			rx->pcap = tx->pcap;
 		}
+
 		goto status_up;
 	}
 
@@ -465,6 +468,12 @@ eth_dev_start(struct rte_eth_dev *dev)
 	}
 
 status_up:
+	for (i = 0; i < dev->data->nb_rx_queues; i++)
+		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++)
+		dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
+
 	dev->data->dev_link.link_status = ETH_LINK_UP;
 
 	return 0;
@@ -517,6 +526,12 @@ eth_dev_stop(struct rte_eth_dev *dev)
 	}
 
 status_down:
+	for (i = 0; i < dev->data->nb_rx_queues; i++)
+		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++)
+		dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
+
 	dev->data->dev_link.link_status = ETH_LINK_DOWN;
 }
 
@@ -538,6 +553,7 @@ eth_dev_info(struct rte_eth_dev *dev,
 	dev_info->max_rx_queues = dev->data->nb_rx_queues;
 	dev_info->max_tx_queues = dev->data->nb_tx_queues;
 	dev_info->min_rx_bufsize = 0;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -643,6 +659,38 @@ eth_tx_queue_setup(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static int
+eth_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+
+	return 0;
+}
+
+static int
+eth_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+
+	return 0;
+}
+
+static int
+eth_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+
+	return 0;
+}
+
+static int
+eth_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+
+	return 0;
+}
+
 static const struct eth_dev_ops ops = {
 	.dev_start = eth_dev_start,
 	.dev_stop = eth_dev_stop,
@@ -651,6 +699,10 @@ static const struct eth_dev_ops ops = {
 	.dev_infos_get = eth_dev_info,
 	.rx_queue_setup = eth_rx_queue_setup,
 	.tx_queue_setup = eth_tx_queue_setup,
+	.rx_queue_start = eth_rx_queue_start,
+	.tx_queue_start = eth_tx_queue_start,
+	.rx_queue_stop = eth_rx_queue_stop,
+	.tx_queue_stop = eth_tx_queue_stop,
 	.rx_queue_release = eth_queue_release,
 	.tx_queue_release = eth_queue_release,
 	.link_update = eth_link_update,
@@ -658,6 +710,22 @@ static const struct eth_dev_ops ops = {
 	.stats_reset = eth_stats_reset,
 };
 
+static int
+add_queue(struct pmd_devargs *pmd, const char *name, const char *type,
+		pcap_t *pcap, pcap_dumper_t *dumper)
+{
+	if (pmd->num_of_queue >= RTE_PMD_PCAP_MAX_QUEUES)
+		return -1;
+	if (pcap)
+		pmd->queue[pmd->num_of_queue].pcap = pcap;
+	if (dumper)
+		pmd->queue[pmd->num_of_queue].dumper = dumper;
+	pmd->queue[pmd->num_of_queue].name = name;
+	pmd->queue[pmd->num_of_queue].type = type;
+	pmd->num_of_queue++;
+	return 0;
+}
+
 /*
  * Function handler that opens the pcap file for reading a stores a
  * reference of it for use it later on.
@@ -665,18 +733,16 @@ static const struct eth_dev_ops ops = {
 static int
 open_rx_pcap(const char *key, const char *value, void *extra_args)
 {
-	unsigned int i;
 	const char *pcap_filename = value;
 	struct pmd_devargs *rx = extra_args;
 	pcap_t *pcap = NULL;
 
-	for (i = 0; i < rx->num_of_queue; i++) {
-		if (open_single_rx_pcap(pcap_filename, &pcap) < 0)
-			return -1;
+	if (open_single_rx_pcap(pcap_filename, &pcap) < 0)
+		return -1;
 
-		rx->queue[i].pcap = pcap;
-		rx->queue[i].name = pcap_filename;
-		rx->queue[i].type = key;
+	if (add_queue(rx, pcap_filename, key, pcap, NULL) < 0) {
+		pcap_close(pcap);
+		return -1;
 	}
 
 	return 0;
@@ -689,18 +755,16 @@ open_rx_pcap(const char *key, const char *value, void *extra_args)
 static int
 open_tx_pcap(const char *key, const char *value, void *extra_args)
 {
-	unsigned int i;
 	const char *pcap_filename = value;
 	struct pmd_devargs *dumpers = extra_args;
 	pcap_dumper_t *dumper;
 
-	for (i = 0; i < dumpers->num_of_queue; i++) {
-		if (open_single_tx_pcap(pcap_filename, &dumper) < 0)
-			return -1;
+	if (open_single_tx_pcap(pcap_filename, &dumper) < 0)
+		return -1;
 
-		dumpers->queue[i].dumper = dumper;
-		dumpers->queue[i].name = pcap_filename;
-		dumpers->queue[i].type = key;
+	if (add_queue(dumpers, pcap_filename, key, NULL, dumper) < 0) {
+		pcap_dump_close(dumper);
+		return -1;
 	}
 
 	return 0;
@@ -726,48 +790,76 @@ open_rx_tx_iface(const char *key, const char *value, void *extra_args)
 	return 0;
 }
 
+static inline int
+set_iface_direction(const char *iface, pcap_t *pcap,
+		pcap_direction_t direction)
+{
+	const char *direction_str = (direction == PCAP_D_IN) ? "IN" : "OUT";
+	if (pcap_setdirection(pcap, direction) < 0) {
+		PMD_LOG(ERR, "Setting %s pcap direction %s failed - %s\n",
+				iface, direction_str, pcap_geterr(pcap));
+		return -1;
+	}
+	PMD_LOG(INFO, "Setting %s pcap direction %s\n",
+			iface, direction_str);
+	return 0;
+}
+
+static inline int
+open_iface(const char *key, const char *value, void *extra_args)
+{
+	const char *iface = value;
+	struct pmd_devargs *pmd = extra_args;
+	pcap_t *pcap = NULL;
+
+	if (open_single_iface(iface, &pcap) < 0)
+		return -1;
+	if (add_queue(pmd, iface, key, pcap, NULL) < 0) {
+		pcap_close(pcap);
+		return -1;
+	}
+
+	return 0;
+}
+
 /*
  * Opens a NIC for reading packets from it
  */
 static inline int
 open_rx_iface(const char *key, const char *value, void *extra_args)
 {
-	unsigned int i;
-	const char *iface = value;
-	struct pmd_devargs *rx = extra_args;
-	pcap_t *pcap = NULL;
+	int ret = open_iface(key, value, extra_args);
+	if (ret < 0)
+		return ret;
+	if (strcmp(key, ETH_PCAP_RX_IFACE_IN_ARG) == 0) {
+		struct pmd_devargs *pmd = extra_args;
+		unsigned int qid = pmd->num_of_queue - 1;
 
-	for (i = 0; i < rx->num_of_queue; i++) {
-		if (open_single_iface(iface, &pcap) < 0)
-			return -1;
-		rx->queue[i].pcap = pcap;
-		rx->queue[i].name = iface;
-		rx->queue[i].type = key;
+		set_iface_direction(pmd->queue[qid].name,
+				pmd->queue[qid].pcap,
+				PCAP_D_IN);
 	}
 
 	return 0;
 }
 
+static inline int
+rx_iface_args_process(const char *key, const char *value, void *extra_args)
+{
+	if (strcmp(key, ETH_PCAP_RX_IFACE_ARG) == 0 ||
+			strcmp(key, ETH_PCAP_RX_IFACE_IN_ARG) == 0)
+		return open_rx_iface(key, value, extra_args);
+
+	return 0;
+}
+
 /*
  * Opens a NIC for writing packets to it
  */
 static int
 open_tx_iface(const char *key, const char *value, void *extra_args)
 {
-	unsigned int i;
-	const char *iface = value;
-	struct pmd_devargs *tx = extra_args;
-	pcap_t *pcap;
-
-	for (i = 0; i < tx->num_of_queue; i++) {
-		if (open_single_iface(iface, &pcap) < 0)
-			return -1;
-		tx->queue[i].pcap = pcap;
-		tx->queue[i].name = iface;
-		tx->queue[i].type = key;
-	}
-
-	return 0;
+	return open_iface(key, value, extra_args);
 }
 
 static struct rte_vdev_driver pmd_pcap_drv;
@@ -925,6 +1017,7 @@ pmd_pcap_probe(struct rte_vdev_device *dev)
 		}
 		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &ops;
+		eth_dev->device = &dev->device;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -958,22 +1051,16 @@ pmd_pcap_probe(struct rte_vdev_device *dev)
 	 * We check whether we want to open a RX stream from a real NIC or a
 	 * pcap file
 	 */
-	pcaps.num_of_queue = rte_kvargs_count(kvlist, ETH_PCAP_RX_PCAP_ARG);
-	if (pcaps.num_of_queue)
-		is_rx_pcap = 1;
-	else
-		pcaps.num_of_queue = rte_kvargs_count(kvlist,
-				ETH_PCAP_RX_IFACE_ARG);
-
-	if (pcaps.num_of_queue > RTE_PMD_PCAP_MAX_QUEUES)
-		pcaps.num_of_queue = RTE_PMD_PCAP_MAX_QUEUES;
+	is_rx_pcap = rte_kvargs_count(kvlist, ETH_PCAP_RX_PCAP_ARG) ? 1 : 0;
+	pcaps.num_of_queue = 0;
 
-	if (is_rx_pcap)
+	if (is_rx_pcap) {
 		ret = rte_kvargs_process(kvlist, ETH_PCAP_RX_PCAP_ARG,
 				&open_rx_pcap, &pcaps);
-	else
-		ret = rte_kvargs_process(kvlist, ETH_PCAP_RX_IFACE_ARG,
-				&open_rx_iface, &pcaps);
+	} else {
+		ret = rte_kvargs_process(kvlist, NULL,
+				&rx_iface_args_process, &pcaps);
+	}
 
 	if (ret < 0)
 		goto free_kvlist;
@@ -982,15 +1069,8 @@ pmd_pcap_probe(struct rte_vdev_device *dev)
 	 * We check whether we want to open a TX stream to a real NIC or a
 	 * pcap file
 	 */
-	dumpers.num_of_queue = rte_kvargs_count(kvlist, ETH_PCAP_TX_PCAP_ARG);
-	if (dumpers.num_of_queue)
-		is_tx_pcap = 1;
-	else
-		dumpers.num_of_queue = rte_kvargs_count(kvlist,
-				ETH_PCAP_TX_IFACE_ARG);
-
-	if (dumpers.num_of_queue > RTE_PMD_PCAP_MAX_QUEUES)
-		dumpers.num_of_queue = RTE_PMD_PCAP_MAX_QUEUES;
+	is_tx_pcap = rte_kvargs_count(kvlist, ETH_PCAP_TX_PCAP_ARG) ? 1 : 0;
+	dumpers.num_of_queue = 0;
 
 	if (is_tx_pcap)
 		ret = rte_kvargs_process(kvlist, ETH_PCAP_TX_PCAP_ARG,
@@ -1046,12 +1126,11 @@ RTE_PMD_REGISTER_PARAM_STRING(net_pcap,
 	ETH_PCAP_RX_PCAP_ARG "=<string> "
 	ETH_PCAP_TX_PCAP_ARG "=<string> "
 	ETH_PCAP_RX_IFACE_ARG "=<ifc> "
+	ETH_PCAP_RX_IFACE_IN_ARG "=<ifc> "
 	ETH_PCAP_TX_IFACE_ARG "=<ifc> "
 	ETH_PCAP_IFACE_ARG "=<ifc>");
 
-RTE_INIT(eth_pcap_init_log);
-static void
-eth_pcap_init_log(void)
+RTE_INIT(eth_pcap_init_log)
 {
 	eth_pcap_logtype = rte_log_register("pmd.net.pcap");
 	if (eth_pcap_logtype >= 0)
diff --git a/drivers/net/qede/LICENSE.qede_pmd b/drivers/net/qede/LICENSE.qede_pmd
deleted file mode 100644
index 022085a1..00000000
--- a/drivers/net/qede/LICENSE.qede_pmd
+++ /dev/null
@@ -1,3 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2016 - 2018 Cavium Inc.
- */
diff --git a/drivers/net/qede/Makefile b/drivers/net/qede/Makefile
index c30a867e..488ca1d9 100644
--- a/drivers/net/qede/Makefile
+++ b/drivers/net/qede/Makefile
@@ -1,8 +1,7 @@
+#    SPDX-License-Identifier: BSD-3-Clause
 #    Copyright (c) 2016 - 2018 Cavium Inc.
 #    All rights reserved.
 #    www.cavium.com
-#
-#    See LICENSE.qede_pmd for copyright and licensing details.
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
diff --git a/drivers/net/qede/base/bcm_osal.c b/drivers/net/qede/base/bcm_osal.c
index ca1c2b11..d5d6f8e2 100644
--- a/drivers/net/qede/base/bcm_osal.c
+++ b/drivers/net/qede/base/bcm_osal.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include <rte_memzone.h>
@@ -201,6 +199,11 @@ void osal_dma_free_mem(struct ecore_dev *p_dev, dma_addr_t phys)
 			DP_VERBOSE(p_dev, ECORE_MSG_SP,
 				"Free memzone %s\n", ecore_mz_mapping[j]->name);
 			rte_memzone_free(ecore_mz_mapping[j]);
+			while (j < ecore_mz_count - 1) {
+				ecore_mz_mapping[j] = ecore_mz_mapping[j + 1];
+				j++;
+			}
+			ecore_mz_count--;
 			return;
 		}
 	}
diff --git a/drivers/net/qede/base/bcm_osal.h b/drivers/net/qede/base/bcm_osal.h
index 27090c79..630867fa 100644
--- a/drivers/net/qede/base/bcm_osal.h
+++ b/drivers/net/qede/base/bcm_osal.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __BCM_OSAL_H
diff --git a/drivers/net/qede/base/common_hsi.h b/drivers/net/qede/base/common_hsi.h
index 52507844..ca8e59db 100644
--- a/drivers/net/qede/base/common_hsi.h
+++ b/drivers/net/qede/base/common_hsi.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __COMMON_HSI__
diff --git a/drivers/net/qede/base/ecore.h b/drivers/net/qede/base/ecore.h
index 57d6aa95..5d79fdf0 100644
--- a/drivers/net/qede/base/ecore.h
+++ b/drivers/net/qede/base/ecore.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_H
diff --git a/drivers/net/qede/base/ecore_attn_values.h b/drivers/net/qede/base/ecore_attn_values.h
index d893e0a6..ec773fbd 100644
--- a/drivers/net/qede/base/ecore_attn_values.h
+++ b/drivers/net/qede/base/ecore_attn_values.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ATTN_VALUES_H__
diff --git a/drivers/net/qede/base/ecore_chain.h b/drivers/net/qede/base/ecore_chain.h
index 0b797460..6d0382d3 100644
--- a/drivers/net/qede/base/ecore_chain.h
+++ b/drivers/net/qede/base/ecore_chain.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_CHAIN_H__
diff --git a/drivers/net/qede/base/ecore_cxt.c b/drivers/net/qede/base/ecore_cxt.c
index a91b2ff3..bf36ce58 100644
--- a/drivers/net/qede/base/ecore_cxt.c
+++ b/drivers/net/qede/base/ecore_cxt.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
diff --git a/drivers/net/qede/base/ecore_cxt.h b/drivers/net/qede/base/ecore_cxt.h
index 3bcbe8f1..f8c955ca 100644
--- a/drivers/net/qede/base/ecore_cxt.h
+++ b/drivers/net/qede/base/ecore_cxt.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef _ECORE_CID_
diff --git a/drivers/net/qede/base/ecore_cxt_api.h b/drivers/net/qede/base/ecore_cxt_api.h
index 65509add..6c8b2831 100644
--- a/drivers/net/qede/base/ecore_cxt_api.h
+++ b/drivers/net/qede/base/ecore_cxt_api.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_CXT_API_H__
diff --git a/drivers/net/qede/base/ecore_dcbx.c b/drivers/net/qede/base/ecore_dcbx.c
index 9b8d39f6..96678745 100644
--- a/drivers/net/qede/base/ecore_dcbx.c
+++ b/drivers/net/qede/base/ecore_dcbx.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
diff --git a/drivers/net/qede/base/ecore_dcbx.h b/drivers/net/qede/base/ecore_dcbx.h
index 49df62ce..519e6cea 100644
--- a/drivers/net/qede/base/ecore_dcbx.h
+++ b/drivers/net/qede/base/ecore_dcbx.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_DCBX_H__
diff --git a/drivers/net/qede/base/ecore_dcbx_api.h b/drivers/net/qede/base/ecore_dcbx_api.h
index 2ad1def4..eaf8e082 100644
--- a/drivers/net/qede/base/ecore_dcbx_api.h
+++ b/drivers/net/qede/base/ecore_dcbx_api.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_DCBX_API_H__
diff --git a/drivers/net/qede/base/ecore_dev.c b/drivers/net/qede/base/ecore_dev.c
index 4ebbedd6..31f1f3ee 100644
--- a/drivers/net/qede/base/ecore_dev.c
+++ b/drivers/net/qede/base/ecore_dev.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
@@ -2513,9 +2511,8 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev,
 			}
 		}
 
-		/* Log and clean previous pglue_b errors if such exist */
+		/* Log and clear previous pglue_b errors if such exist */
 		ecore_pglueb_rbc_attn_handler(p_hwfn, p_hwfn->p_main_ptt, true);
-		ecore_pglueb_clear_err(p_hwfn, p_hwfn->p_main_ptt);
 
 		/* Enable the PF's internal FID_enable in the PXP */
 		rc = ecore_pglueb_set_pfid_enable(p_hwfn, p_hwfn->p_main_ptt,
@@ -2523,6 +2520,13 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev,
 		if (rc != ECORE_SUCCESS)
 			goto load_err;
 
+		/* Clear the pglue_b was_error indication.
+		 * In E4 it must be done after the BME and the internal
+		 * FID_enable for the PF are set, since VDMs may cause the
+		 * indication to be set again.
+		 */
+		ecore_pglueb_clear_err(p_hwfn, p_hwfn->p_main_ptt);
+
 		switch (load_code) {
 		case FW_MSG_CODE_DRV_LOAD_ENGINE:
 			rc = ecore_hw_init_common(p_hwfn, p_hwfn->p_main_ptt,
diff --git a/drivers/net/qede/base/ecore_dev_api.h b/drivers/net/qede/base/ecore_dev_api.h
index 0dd78d60..02bacc22 100644
--- a/drivers/net/qede/base/ecore_dev_api.h
+++ b/drivers/net/qede/base/ecore_dev_api.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_DEV_API_H__
diff --git a/drivers/net/qede/base/ecore_gtt_reg_addr.h b/drivers/net/qede/base/ecore_gtt_reg_addr.h
index ac29dc49..8c8fed4e 100644
--- a/drivers/net/qede/base/ecore_gtt_reg_addr.h
+++ b/drivers/net/qede/base/ecore_gtt_reg_addr.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef GTT_REG_ADDR_H
diff --git a/drivers/net/qede/base/ecore_gtt_values.h b/drivers/net/qede/base/ecore_gtt_values.h
index d9af94df..adc20c0c 100644
--- a/drivers/net/qede/base/ecore_gtt_values.h
+++ b/drivers/net/qede/base/ecore_gtt_values.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __PREVENT_PXP_GLOBAL_WIN__
diff --git a/drivers/net/qede/base/ecore_hsi_common.h b/drivers/net/qede/base/ecore_hsi_common.h
index d400fa91..2d761b97 100644
--- a/drivers/net/qede/base/ecore_hsi_common.h
+++ b/drivers/net/qede/base/ecore_hsi_common.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_HSI_COMMON__
diff --git a/drivers/net/qede/base/ecore_hsi_debug_tools.h b/drivers/net/qede/base/ecore_hsi_debug_tools.h
index 262834ed..bf548722 100644
--- a/drivers/net/qede/base/ecore_hsi_debug_tools.h
+++ b/drivers/net/qede/base/ecore_hsi_debug_tools.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_HSI_DEBUG_TOOLS__
diff --git a/drivers/net/qede/base/ecore_hsi_eth.h b/drivers/net/qede/base/ecore_hsi_eth.h
index efe3bb24..6b512305 100644
--- a/drivers/net/qede/base/ecore_hsi_eth.h
+++ b/drivers/net/qede/base/ecore_hsi_eth.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_HSI_ETH__
diff --git a/drivers/net/qede/base/ecore_hsi_init_func.h b/drivers/net/qede/base/ecore_hsi_init_func.h
index c318514b..d77edaa1 100644
--- a/drivers/net/qede/base/ecore_hsi_init_func.h
+++ b/drivers/net/qede/base/ecore_hsi_init_func.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_HSI_INIT_FUNC__
diff --git a/drivers/net/qede/base/ecore_hsi_init_tool.h b/drivers/net/qede/base/ecore_hsi_init_tool.h
index 2e338a98..0e157f9b 100644
--- a/drivers/net/qede/base/ecore_hsi_init_tool.h
+++ b/drivers/net/qede/base/ecore_hsi_init_tool.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_HSI_INIT_TOOL__
diff --git a/drivers/net/qede/base/ecore_hw.c b/drivers/net/qede/base/ecore_hw.c
index b00c33d4..51bba27e 100644
--- a/drivers/net/qede/base/ecore_hw.c
+++ b/drivers/net/qede/base/ecore_hw.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
diff --git a/drivers/net/qede/base/ecore_hw.h b/drivers/net/qede/base/ecore_hw.h
index f3f513e8..394207eb 100644
--- a/drivers/net/qede/base/ecore_hw.h
+++ b/drivers/net/qede/base/ecore_hw.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_HW_H__
diff --git a/drivers/net/qede/base/ecore_hw_defs.h b/drivers/net/qede/base/ecore_hw_defs.h
index 2f4bd536..b8c2686f 100644
--- a/drivers/net/qede/base/ecore_hw_defs.h
+++ b/drivers/net/qede/base/ecore_hw_defs.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef _ECORE_IGU_DEF_H_
diff --git a/drivers/net/qede/base/ecore_init_fw_funcs.c b/drivers/net/qede/base/ecore_init_fw_funcs.c
index 3f986629..b8496cb2 100644
--- a/drivers/net/qede/base/ecore_init_fw_funcs.c
+++ b/drivers/net/qede/base/ecore_init_fw_funcs.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
diff --git a/drivers/net/qede/base/ecore_init_fw_funcs.h b/drivers/net/qede/base/ecore_init_fw_funcs.h
index 310c9ed7..1024bb26 100644
--- a/drivers/net/qede/base/ecore_init_fw_funcs.h
+++ b/drivers/net/qede/base/ecore_init_fw_funcs.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef _INIT_FW_FUNCS_H
diff --git a/drivers/net/qede/base/ecore_init_ops.c b/drivers/net/qede/base/ecore_init_ops.c
index eadccf40..b7636f36 100644
--- a/drivers/net/qede/base/ecore_init_ops.c
+++ b/drivers/net/qede/base/ecore_init_ops.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 /* include the precompiled configuration values - only once */
diff --git a/drivers/net/qede/base/ecore_init_ops.h b/drivers/net/qede/base/ecore_init_ops.h
index 7ca00e4e..de7846d4 100644
--- a/drivers/net/qede/base/ecore_init_ops.h
+++ b/drivers/net/qede/base/ecore_init_ops.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_INIT_OPS__
diff --git a/drivers/net/qede/base/ecore_int.c b/drivers/net/qede/base/ecore_int.c
index 7272f059..4c271d35 100644
--- a/drivers/net/qede/base/ecore_int.c
+++ b/drivers/net/qede/base/ecore_int.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include <rte_string_fns.h>
@@ -233,15 +231,19 @@ static const char *grc_timeout_attn_master_to_str(u8 master)
 
 static enum _ecore_status_t ecore_grc_attn_cb(struct ecore_hwfn *p_hwfn)
 {
+	enum _ecore_status_t rc = ECORE_SUCCESS;
 	u32 tmp, tmp2;
 
 	/* We've already cleared the timeout interrupt register, so we learn
-	 * of interrupts via the validity register
+	 * of interrupts via the validity register.
+	 * Any attention which is not for a timeout event is treated as fatal.
 	 */
 	tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt,
 		       GRC_REG_TIMEOUT_ATTN_ACCESS_VALID);
-	if (!(tmp & ECORE_GRC_ATTENTION_VALID_BIT))
+	if (!(tmp & ECORE_GRC_ATTENTION_VALID_BIT)) {
+		rc = ECORE_INVAL;
 		goto out;
+	}
 
 	/* Read the GRC timeout information */
 	tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt,
@@ -265,11 +267,11 @@ static enum _ecore_status_t ecore_grc_attn_cb(struct ecore_hwfn *p_hwfn)
 		  (tmp2 & ECORE_GRC_ATTENTION_VF_MASK) >>
 		  ECORE_GRC_ATTENTION_VF_SHIFT);
 
-out:
-	/* Regardles of anything else, clean the validity bit */
+	/* Clean the validity bit */
 	ecore_wr(p_hwfn, p_hwfn->p_dpc_ptt,
 		 GRC_REG_TIMEOUT_ATTN_ACCESS_VALID, 0);
-	return ECORE_SUCCESS;
+out:
+	return rc;
 }
 
 #define ECORE_PGLUE_ATTENTION_VALID (1 << 29)
diff --git a/drivers/net/qede/base/ecore_int.h b/drivers/net/qede/base/ecore_int.h
index bb22fdd8..041240d7 100644
--- a/drivers/net/qede/base/ecore_int.h
+++ b/drivers/net/qede/base/ecore_int.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_INT_H__
diff --git a/drivers/net/qede/base/ecore_int_api.h b/drivers/net/qede/base/ecore_int_api.h
index dff53776..aeaf469e 100644
--- a/drivers/net/qede/base/ecore_int_api.h
+++ b/drivers/net/qede/base/ecore_int_api.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_INT_API_H__
diff --git a/drivers/net/qede/base/ecore_iov_api.h b/drivers/net/qede/base/ecore_iov_api.h
index ee7cad74..29001d71 100644
--- a/drivers/net/qede/base/ecore_iov_api.h
+++ b/drivers/net/qede/base/ecore_iov_api.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_SRIOV_API_H__
diff --git a/drivers/net/qede/base/ecore_iro.h b/drivers/net/qede/base/ecore_iro.h
index e4dc1c92..05693029 100644
--- a/drivers/net/qede/base/ecore_iro.h
+++ b/drivers/net/qede/base/ecore_iro.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __IRO_H__
diff --git a/drivers/net/qede/base/ecore_iro_values.h b/drivers/net/qede/base/ecore_iro_values.h
index b47abeb9..685fa2e8 100644
--- a/drivers/net/qede/base/ecore_iro_values.h
+++ b/drivers/net/qede/base/ecore_iro_values.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __IRO_VALUES_H__
diff --git a/drivers/net/qede/base/ecore_l2.c b/drivers/net/qede/base/ecore_l2.c
index 91d89e56..d71f4616 100644
--- a/drivers/net/qede/base/ecore_l2.c
+++ b/drivers/net/qede/base/ecore_l2.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
diff --git a/drivers/net/qede/base/ecore_l2.h b/drivers/net/qede/base/ecore_l2.h
index bea6a6df..8fa40302 100644
--- a/drivers/net/qede/base/ecore_l2.h
+++ b/drivers/net/qede/base/ecore_l2.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_L2_H__
diff --git a/drivers/net/qede/base/ecore_l2_api.h b/drivers/net/qede/base/ecore_l2_api.h
index 43ebbd12..575b9e3a 100644
--- a/drivers/net/qede/base/ecore_l2_api.h
+++ b/drivers/net/qede/base/ecore_l2_api.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_L2_API_H__
diff --git a/drivers/net/qede/base/ecore_mcp.c b/drivers/net/qede/base/ecore_mcp.c
index 784d28c5..ea14c172 100644
--- a/drivers/net/qede/base/ecore_mcp.c
+++ b/drivers/net/qede/base/ecore_mcp.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
diff --git a/drivers/net/qede/base/ecore_mcp.h b/drivers/net/qede/base/ecore_mcp.h
index c422736f..8e125310 100644
--- a/drivers/net/qede/base/ecore_mcp.h
+++ b/drivers/net/qede/base/ecore_mcp.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_MCP_H__
diff --git a/drivers/net/qede/base/ecore_mcp_api.h b/drivers/net/qede/base/ecore_mcp_api.h
index 06b33bb8..cfb9f99d 100644
--- a/drivers/net/qede/base/ecore_mcp_api.h
+++ b/drivers/net/qede/base/ecore_mcp_api.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_MCP_API_H__
diff --git a/drivers/net/qede/base/ecore_mng_tlv.c b/drivers/net/qede/base/ecore_mng_tlv.c
index b48076a3..f7666472 100644
--- a/drivers/net/qede/base/ecore_mng_tlv.c
+++ b/drivers/net/qede/base/ecore_mng_tlv.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
diff --git a/drivers/net/qede/base/ecore_proto_if.h b/drivers/net/qede/base/ecore_proto_if.h
index f049d821..f91b25e2 100644
--- a/drivers/net/qede/base/ecore_proto_if.h
+++ b/drivers/net/qede/base/ecore_proto_if.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_PROTO_IF_H__
diff --git a/drivers/net/qede/base/ecore_rt_defs.h b/drivers/net/qede/base/ecore_rt_defs.h
index 7dec2dd5..721b8c15 100644
--- a/drivers/net/qede/base/ecore_rt_defs.h
+++ b/drivers/net/qede/base/ecore_rt_defs.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __RT_DEFS_H__
diff --git a/drivers/net/qede/base/ecore_sp_api.h b/drivers/net/qede/base/ecore_sp_api.h
index 469bf1d6..4633dbeb 100644
--- a/drivers/net/qede/base/ecore_sp_api.h
+++ b/drivers/net/qede/base/ecore_sp_api.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_SP_API_H__
diff --git a/drivers/net/qede/base/ecore_sp_commands.c b/drivers/net/qede/base/ecore_sp_commands.c
index 47c1febf..b43baf9d 100644
--- a/drivers/net/qede/base/ecore_sp_commands.c
+++ b/drivers/net/qede/base/ecore_sp_commands.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
diff --git a/drivers/net/qede/base/ecore_sp_commands.h b/drivers/net/qede/base/ecore_sp_commands.h
index d160a76e..e57414cf 100644
--- a/drivers/net/qede/base/ecore_sp_commands.h
+++ b/drivers/net/qede/base/ecore_sp_commands.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_SP_COMMANDS_H__
diff --git a/drivers/net/qede/base/ecore_spq.c b/drivers/net/qede/base/ecore_spq.c
index db169e6e..776c86f7 100644
--- a/drivers/net/qede/base/ecore_spq.c
+++ b/drivers/net/qede/base/ecore_spq.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
diff --git a/drivers/net/qede/base/ecore_spq.h b/drivers/net/qede/base/ecore_spq.h
index 7d9be3e9..6142c399 100644
--- a/drivers/net/qede/base/ecore_spq.h
+++ b/drivers/net/qede/base/ecore_spq.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_SPQ_H__
diff --git a/drivers/net/qede/base/ecore_sriov.c b/drivers/net/qede/base/ecore_sriov.c
index 451aabb1..f7ebf7ad 100644
--- a/drivers/net/qede/base/ecore_sriov.c
+++ b/drivers/net/qede/base/ecore_sriov.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
@@ -61,6 +59,8 @@ const char *ecore_channel_tlvs_string[] = {
 	"CHANNEL_TLV_COALESCE_UPDATE",
 	"CHANNEL_TLV_QID",
 	"CHANNEL_TLV_COALESCE_READ",
+	"CHANNEL_TLV_BULLETIN_UPDATE_MAC",
+	"CHANNEL_TLV_UPDATE_MTU",
 	"CHANNEL_TLV_MAX"
 };
 
@@ -2858,6 +2858,45 @@ out:
 			       length, status);
 }
 
+static enum _ecore_status_t
+ecore_iov_vf_pf_update_mtu(struct ecore_hwfn *p_hwfn,
+				    struct ecore_ptt *p_ptt,
+				    struct ecore_vf_info *p_vf)
+{
+	struct ecore_iov_vf_mbx *mbx = &p_vf->vf_mbx;
+	struct ecore_sp_vport_update_params params;
+	enum _ecore_status_t rc = ECORE_SUCCESS;
+	struct vfpf_update_mtu_tlv *p_req;
+	u8 status = PFVF_STATUS_SUCCESS;
+
+	/* Valiate PF can send such a request */
+	if (!p_vf->vport_instance) {
+		DP_VERBOSE(p_hwfn, ECORE_MSG_IOV,
+			   "No VPORT instance available for VF[%d], failing MTU update\n",
+			   p_vf->abs_vf_id);
+		status = PFVF_STATUS_FAILURE;
+		goto send_status;
+	}
+
+	p_req = &mbx->req_virt->update_mtu;
+
+	OSAL_MEMSET(&params, 0, sizeof(params));
+	params.opaque_fid =  p_vf->opaque_fid;
+	params.vport_id = p_vf->vport_id;
+	params.mtu = p_req->mtu;
+	rc = ecore_sp_vport_update(p_hwfn, &params, ECORE_SPQ_MODE_EBLOCK,
+				   OSAL_NULL);
+
+	if (rc)
+		status = PFVF_STATUS_FAILURE;
+send_status:
+	ecore_iov_prepare_resp(p_hwfn, p_ptt, p_vf,
+			       CHANNEL_TLV_UPDATE_MTU,
+			       sizeof(struct pfvf_def_resp_tlv),
+			       status);
+	return rc;
+}
+
 void *ecore_iov_search_list_tlvs(struct ecore_hwfn *p_hwfn,
 				 void *p_tlvs_list, u16 req_type)
 {
@@ -4140,6 +4179,9 @@ void ecore_iov_process_mbx_req(struct ecore_hwfn *p_hwfn,
 		case CHANNEL_TLV_COALESCE_READ:
 			ecore_iov_vf_pf_get_coalesce(p_hwfn, p_ptt, p_vf);
 			break;
+		case CHANNEL_TLV_UPDATE_MTU:
+			ecore_iov_vf_pf_update_mtu(p_hwfn, p_ptt, p_vf);
+			break;
 		}
 	} else if (ecore_iov_tlv_supported(mbx->first_tlv.tl.type)) {
 		/* If we've received a message from a VF we consider malicious
diff --git a/drivers/net/qede/base/ecore_sriov.h b/drivers/net/qede/base/ecore_sriov.h
index 8d846d3e..50c7d2c9 100644
--- a/drivers/net/qede/base/ecore_sriov.h
+++ b/drivers/net/qede/base/ecore_sriov.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_SRIOV_H__
diff --git a/drivers/net/qede/base/ecore_status.h b/drivers/net/qede/base/ecore_status.h
index 3af2b57d..b893f1d4 100644
--- a/drivers/net/qede/base/ecore_status.h
+++ b/drivers/net/qede/base/ecore_status.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_STATUS_H__
diff --git a/drivers/net/qede/base/ecore_utils.h b/drivers/net/qede/base/ecore_utils.h
index f6459c35..249136b0 100644
--- a/drivers/net/qede/base/ecore_utils.h
+++ b/drivers/net/qede/base/ecore_utils.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_UTILS_H__
diff --git a/drivers/net/qede/base/ecore_vf.c b/drivers/net/qede/base/ecore_vf.c
index b7b3b872..d2213f79 100644
--- a/drivers/net/qede/base/ecore_vf.c
+++ b/drivers/net/qede/base/ecore_vf.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "bcm_osal.h"
@@ -1628,6 +1626,39 @@ exit:
 	return rc;
 }
 
+enum _ecore_status_t
+ecore_vf_pf_update_mtu(struct ecore_hwfn *p_hwfn, u16 mtu)
+{
+	struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info;
+	struct vfpf_update_mtu_tlv *p_req;
+	struct pfvf_def_resp_tlv *p_resp;
+	enum _ecore_status_t rc;
+
+	if (!mtu)
+		return ECORE_INVAL;
+
+	/* clear mailbox and prep header tlv */
+	p_req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_UPDATE_MTU,
+				 sizeof(*p_req));
+	p_req->mtu = mtu;
+	DP_VERBOSE(p_hwfn, ECORE_MSG_IOV,
+		   "Requesting MTU update to %d\n", mtu);
+
+	/* add list termination tlv */
+	ecore_add_tlv(&p_iov->offset,
+		      CHANNEL_TLV_LIST_END,
+		      sizeof(struct channel_list_end_tlv));
+
+	p_resp = &p_iov->pf2vf_reply->default_resp;
+	rc = ecore_send_msg2pf(p_hwfn, &p_resp->hdr.status, sizeof(*p_resp));
+	if (p_resp->hdr.status == PFVF_STATUS_NOT_SUPPORTED)
+		rc = ECORE_INVAL;
+
+	ecore_vf_pf_req_end(p_hwfn, rc);
+
+	return rc;
+}
+
 u16 ecore_vf_get_igu_sb_id(struct ecore_hwfn *p_hwfn,
 			   u16               sb_id)
 {
diff --git a/drivers/net/qede/base/ecore_vf.h b/drivers/net/qede/base/ecore_vf.h
index e26b30bf..a07f82eb 100644
--- a/drivers/net/qede/base/ecore_vf.h
+++ b/drivers/net/qede/base/ecore_vf.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_VF_H__
@@ -319,5 +317,14 @@ void ecore_vf_set_vf_start_tunn_update_param(struct ecore_tunnel_info *p_tun);
 
 u32 ecore_vf_hw_bar_size(struct ecore_hwfn *p_hwfn,
 		     enum BAR_ID bar_id);
+
+/**
+ * @brief - ecore_vf_pf_update_mtu Update MTU for VF.
+ *
+ * @param p_hwfn
+ * @param - mtu
+ */
+enum _ecore_status_t
+ecore_vf_pf_update_mtu(struct ecore_hwfn *p_hwfn, u16 mtu);
 #endif
 #endif /* __ECORE_VF_H__ */
diff --git a/drivers/net/qede/base/ecore_vf_api.h b/drivers/net/qede/base/ecore_vf_api.h
index af7bc36b..1a9fb3b1 100644
--- a/drivers/net/qede/base/ecore_vf_api.h
+++ b/drivers/net/qede/base/ecore_vf_api.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_VF_API_H__
diff --git a/drivers/net/qede/base/ecore_vfpf_if.h b/drivers/net/qede/base/ecore_vfpf_if.h
index dce937e0..c30677ab 100644
--- a/drivers/net/qede/base/ecore_vfpf_if.h
+++ b/drivers/net/qede/base/ecore_vfpf_if.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ECORE_VF_PF_IF_H__
@@ -531,6 +529,18 @@ struct pfvf_read_coal_resp_tlv {
 	u8 padding[6];
 };
 
+struct vfpf_bulletin_update_mac_tlv {
+	struct vfpf_first_tlv first_tlv;
+	u8 mac[ETH_ALEN];
+	u8 padding[2];
+};
+
+struct vfpf_update_mtu_tlv {
+	struct vfpf_first_tlv first_tlv;
+	u16 mtu;
+	u8 padding[6];
+};
+
 union vfpf_tlvs {
 	struct vfpf_first_tlv			first_tlv;
 	struct vfpf_acquire_tlv			acquire;
@@ -545,6 +555,8 @@ union vfpf_tlvs {
 	struct vfpf_update_tunn_param_tlv	tunn_param_update;
 	struct vfpf_update_coalesce		update_coalesce;
 	struct vfpf_read_coal_req_tlv		read_coal_req;
+	struct vfpf_bulletin_update_mac_tlv	bulletin_update_mac;
+	struct vfpf_update_mtu_tlv		update_mtu;
 	struct tlv_buffer_size			tlv_buf_size;
 };
 
@@ -675,6 +687,8 @@ enum {
 	CHANNEL_TLV_COALESCE_UPDATE,
 	CHANNEL_TLV_QID,
 	CHANNEL_TLV_COALESCE_READ,
+	CHANNEL_TLV_BULLETIN_UPDATE_MAC,
+	CHANNEL_TLV_UPDATE_MTU,
 	CHANNEL_TLV_MAX,
 
 	/* Required for iterating over vport-update tlvs.
diff --git a/drivers/net/qede/base/eth_common.h b/drivers/net/qede/base/eth_common.h
index 9de49b64..abfa6854 100644
--- a/drivers/net/qede/base/eth_common.h
+++ b/drivers/net/qede/base/eth_common.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef __ETH_COMMON__
diff --git a/drivers/net/qede/base/mcp_public.h b/drivers/net/qede/base/mcp_public.h
index 752473e1..81aa88e7 100644
--- a/drivers/net/qede/base/mcp_public.h
+++ b/drivers/net/qede/base/mcp_public.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 /****************************************************************************
diff --git a/drivers/net/qede/base/nvm_cfg.h b/drivers/net/qede/base/nvm_cfg.h
index a20d0674..ab86260e 100644
--- a/drivers/net/qede/base/nvm_cfg.h
+++ b/drivers/net/qede/base/nvm_cfg.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 /****************************************************************************
diff --git a/drivers/net/qede/base/reg_addr.h b/drivers/net/qede/base/reg_addr.h
index 71f0ca13..402f6204 100644
--- a/drivers/net/qede/base/reg_addr.h
+++ b/drivers/net/qede/base/reg_addr.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #define  CDU_REG_CID_ADDR_PARAMS_CONTEXT_SIZE_SHIFT \
diff --git a/drivers/net/qede/qede_ethdev.c b/drivers/net/qede/qede_ethdev.c
index 3206cc6c..df52ea92 100644
--- a/drivers/net/qede/qede_ethdev.c
+++ b/drivers/net/qede/qede_ethdev.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include "qede_ethdev.h"
@@ -339,6 +337,24 @@ static void qede_interrupt_action(struct ecore_hwfn *p_hwfn)
 }
 
 static void
+qede_interrupt_handler_intx(void *param)
+{
+	struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)param;
+	struct qede_dev *qdev = eth_dev->data->dev_private;
+	struct ecore_dev *edev = &qdev->edev;
+	u64 status;
+
+	/* Check if our device actually raised an interrupt */
+	status = ecore_int_igu_read_sisr_reg(ECORE_LEADING_HWFN(edev));
+	if (status & 0x1) {
+		qede_interrupt_action(ECORE_LEADING_HWFN(edev));
+
+		if (rte_intr_enable(eth_dev->intr_handle))
+			DP_ERR(edev, "rte_intr_enable failed\n");
+	}
+}
+
+static void
 qede_interrupt_handler(void *param)
 {
 	struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)param;
@@ -518,14 +534,9 @@ int qede_activate_vport(struct rte_eth_dev *eth_dev, bool flg)
 	params.update_vport_active_tx_flg = 1;
 	params.vport_active_rx_flg = flg;
 	params.vport_active_tx_flg = flg;
-	if (!qdev->enable_tx_switching) {
-		if ((QEDE_NPAR_TX_SWITCHING != NULL) ||
-		    ((QEDE_VF_TX_SWITCHING != NULL) && IS_VF(edev))) {
-			params.update_tx_switching_flg = 1;
-			params.tx_switching_flg = !flg;
-			DP_INFO(edev, "%s tx-switching is disabled\n",
-				QEDE_NPAR_TX_SWITCHING ? "NPAR" : "VF");
-		}
+	if (~qdev->enable_tx_switching & flg) {
+		params.update_tx_switching_flg = 1;
+		params.tx_switching_flg = !flg;
 	}
 	for_each_hwfn(edev, i) {
 		p_hwfn = &edev->hwfns[i];
@@ -603,37 +614,6 @@ int qede_enable_tpa(struct rte_eth_dev *eth_dev, bool flg)
 	return 0;
 }
 
-/* Update MTU via vport-update without doing port restart.
- * The vport must be deactivated before calling this API.
- */
-int qede_update_mtu(struct rte_eth_dev *eth_dev, uint16_t mtu)
-{
-	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
-	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
-	struct ecore_sp_vport_update_params params;
-	struct ecore_hwfn *p_hwfn;
-	int rc;
-	int i;
-
-	memset(&params, 0, sizeof(struct ecore_sp_vport_update_params));
-	params.vport_id = 0;
-	params.mtu = mtu;
-	params.vport_id = 0;
-	for_each_hwfn(edev, i) {
-		p_hwfn = &edev->hwfns[i];
-		params.opaque_fid = p_hwfn->hw_info.opaque_fid;
-		rc = ecore_sp_vport_update(p_hwfn, &params,
-				ECORE_SPQ_MODE_EBLOCK, NULL);
-		if (rc != ECORE_SUCCESS) {
-			DP_ERR(edev, "Failed to update MTU\n");
-			return -1;
-		}
-	}
-	DP_INFO(edev, "MTU updated to %u\n", mtu);
-
-	return 0;
-}
-
 static void qede_set_ucast_cmn_params(struct ecore_filter_ucast *ucast)
 {
 	memset(ucast, 0, sizeof(struct ecore_filter_ucast));
@@ -979,7 +959,10 @@ qede_mac_int_ops(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *ucast,
 	if (rc == 0)
 		rc = ecore_filter_ucast_cmd(edev, ucast,
 					    ECORE_SPQ_MODE_CB, NULL);
-	if (rc != ECORE_SUCCESS)
+	/* Indicate error only for add filter operation.
+	 * Delete filter operations are not severe.
+	 */
+	if ((rc != ECORE_SUCCESS) && add)
 		DP_ERR(edev, "MAC filter failed, rc = %d, op = %d\n",
 		       rc, add);
 
@@ -993,7 +976,11 @@ qede_mac_addr_add(struct rte_eth_dev *eth_dev, struct ether_addr *mac_addr,
 	struct ecore_filter_ucast ucast;
 	int re;
 
+	if (!is_valid_assigned_ether_addr(mac_addr))
+		return -EINVAL;
+
 	qede_set_ucast_cmn_params(&ucast);
+	ucast.opcode = ECORE_FILTER_ADD;
 	ucast.type = ECORE_FILTER_MAC;
 	ether_addr_copy(mac_addr, (struct ether_addr *)&ucast.mac);
 	re = (int)qede_mac_int_ops(eth_dev, &ucast, 1);
@@ -1015,6 +1002,9 @@ qede_mac_addr_remove(struct rte_eth_dev *eth_dev, uint32_t index)
 		return;
 	}
 
+	if (!is_valid_assigned_ether_addr(&eth_dev->data->mac_addrs[index]))
+		return;
+
 	qede_set_ucast_cmn_params(&ucast);
 	ucast.opcode = ECORE_FILTER_REMOVE;
 	ucast.type = ECORE_FILTER_MAC;
@@ -1038,8 +1028,9 @@ qede_mac_addr_set(struct rte_eth_dev *eth_dev, struct ether_addr *mac_addr)
 		return -EPERM;
 	}
 
-	qede_mac_addr_add(eth_dev, mac_addr, 0, 0);
-	return 0;
+	qede_mac_addr_remove(eth_dev, 0);
+
+	return qede_mac_addr_add(eth_dev, mac_addr, 0, 0);
 }
 
 static void qede_config_accept_any_vlan(struct qede_dev *qdev, bool flg)
@@ -1296,6 +1287,12 @@ static int qede_dev_start(struct rte_eth_dev *eth_dev)
 
 	PMD_INIT_FUNC_TRACE(edev);
 
+	/* Update MTU only if it has changed */
+	if (eth_dev->data->mtu != qdev->mtu) {
+		if (qede_update_mtu(eth_dev, qdev->mtu))
+			goto err;
+	}
+
 	/* Configure TPA parameters */
 	if (rxmode->offloads & DEV_RX_OFFLOAD_TCP_LRO) {
 		if (qede_enable_tpa(eth_dev, true))
@@ -1359,9 +1356,6 @@ static void qede_dev_stop(struct rte_eth_dev *eth_dev)
 	/* Disable traffic */
 	ecore_hw_stop_fastpath(edev); /* TBD - loop */
 
-	if (IS_PF(edev))
-		qede_mac_addr_remove(eth_dev, 0);
-
 	DP_INFO(edev, "Device is stopped\n");
 }
 
@@ -1387,8 +1381,12 @@ static int qede_args_check(const char *key, const char *val, void *opaque)
 	}
 
 	if ((strcmp(QEDE_NPAR_TX_SWITCHING, key) == 0) ||
-	    (strcmp(QEDE_VF_TX_SWITCHING, key) == 0))
+	    ((strcmp(QEDE_VF_TX_SWITCHING, key) == 0) && IS_VF(edev))) {
 		qdev->enable_tx_switching = !!tmp;
+		DP_INFO(edev, "Disabling %s tx-switching\n",
+			strcmp(QEDE_NPAR_TX_SWITCHING, key) ?
+			"VF" : "NPAR");
+	}
 
 	return ret;
 }
@@ -1463,7 +1461,8 @@ static int qede_dev_configure(struct rte_eth_dev *eth_dev)
 
 	/* Parse devargs and fix up rxmode */
 	if (qede_args(eth_dev))
-		return -ENOTSUP;
+		DP_NOTICE(edev, false,
+			  "Invalid devargs supplied, requested change will not take effect\n");
 
 	if (!(rxmode->mq_mode == ETH_MQ_RX_NONE ||
 	      rxmode->mq_mode == ETH_MQ_RX_RSS)) {
@@ -1496,8 +1495,7 @@ static int qede_dev_configure(struct rte_eth_dev *eth_dev)
 
 	/* Enable VLAN offloads by default */
 	ret = qede_vlan_offload_set(eth_dev, ETH_VLAN_STRIP_MASK  |
-					     ETH_VLAN_FILTER_MASK |
-					     ETH_VLAN_EXTEND_MASK);
+					     ETH_VLAN_FILTER_MASK);
 	if (ret)
 		return ret;
 
@@ -1557,6 +1555,7 @@ qede_dev_info_get(struct rte_eth_dev *eth_dev,
 				     DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
 				     DEV_RX_OFFLOAD_TCP_LRO	|
 				     DEV_RX_OFFLOAD_CRC_STRIP	|
+				     DEV_RX_OFFLOAD_KEEP_CRC    |
 				     DEV_RX_OFFLOAD_SCATTER	|
 				     DEV_RX_OFFLOAD_JUMBO_FRAME |
 				     DEV_RX_OFFLOAD_VLAN_FILTER |
@@ -1571,7 +1570,6 @@ qede_dev_info_get(struct rte_eth_dev *eth_dev,
 				     DEV_TX_OFFLOAD_UDP_CKSUM	|
 				     DEV_TX_OFFLOAD_TCP_CKSUM	|
 				     DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
-				     DEV_TX_OFFLOAD_QINQ_INSERT |
 				     DEV_TX_OFFLOAD_MULTI_SEGS  |
 				     DEV_TX_OFFLOAD_TCP_TSO	|
 				     DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
@@ -1585,11 +1583,7 @@ qede_dev_info_get(struct rte_eth_dev *eth_dev,
 	dev_info->default_rxconf = (struct rte_eth_rxconf) {
 		/* Packets are always dropped if no descriptors are available */
 		.rx_drop_en = 1,
-		/* The below RX offloads are always enabled */
-		.offloads = (DEV_RX_OFFLOAD_CRC_STRIP  |
-			     DEV_RX_OFFLOAD_IPV4_CKSUM |
-			     DEV_RX_OFFLOAD_TCP_CKSUM  |
-			     DEV_RX_OFFLOAD_UDP_CKSUM),
+		.offloads = 0,
 	};
 
 	memset(&link, 0, sizeof(struct qed_link_output));
@@ -1615,18 +1609,20 @@ qede_link_update(struct rte_eth_dev *eth_dev, __rte_unused int wait_to_complete)
 {
 	struct qede_dev *qdev = eth_dev->data->dev_private;
 	struct ecore_dev *edev = &qdev->edev;
+	struct qed_link_output q_link;
+	struct rte_eth_link link;
 	uint16_t link_duplex;
-	struct qed_link_output link;
-	struct rte_eth_link *curr = &eth_dev->data->dev_link;
 
-	memset(&link, 0, sizeof(struct qed_link_output));
-	qdev->ops->common->get_link(edev, &link);
+	memset(&q_link, 0, sizeof(q_link));
+	memset(&link, 0, sizeof(link));
+
+	qdev->ops->common->get_link(edev, &q_link);
 
 	/* Link Speed */
-	curr->link_speed = link.speed;
+	link.link_speed = q_link.speed;
 
 	/* Link Mode */
-	switch (link.duplex) {
+	switch (q_link.duplex) {
 	case QEDE_DUPLEX_HALF:
 		link_duplex = ETH_LINK_HALF_DUPLEX;
 		break;
@@ -1637,21 +1633,20 @@ qede_link_update(struct rte_eth_dev *eth_dev, __rte_unused int wait_to_complete)
 	default:
 		link_duplex = -1;
 	}
-	curr->link_duplex = link_duplex;
+	link.link_duplex = link_duplex;
 
 	/* Link Status */
-	curr->link_status = (link.link_up) ? ETH_LINK_UP : ETH_LINK_DOWN;
+	link.link_status = q_link.link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
 
 	/* AN */
-	curr->link_autoneg = (link.supported_caps & QEDE_SUPPORTED_AUTONEG) ?
+	link.link_autoneg = (q_link.supported_caps & QEDE_SUPPORTED_AUTONEG) ?
 			     ETH_LINK_AUTONEG : ETH_LINK_FIXED;
 
 	DP_INFO(edev, "Link - Speed %u Mode %u AN %u Status %u\n",
-		curr->link_speed, curr->link_duplex,
-		curr->link_autoneg, curr->link_status);
+		link.link_speed, link.link_duplex,
+		link.link_autoneg, link.link_status);
 
-	/* return 0 means link status changed, -1 means not changed */
-	return ((curr->link_status == link.link_up) ? -1 : 0);
+	return rte_eth_linkstatus_set(eth_dev, &link);
 }
 
 static void qede_promiscuous_enable(struct rte_eth_dev *eth_dev)
@@ -1737,8 +1732,20 @@ static void qede_dev_close(struct rte_eth_dev *eth_dev)
 	qdev->ops->common->slowpath_stop(edev);
 	qdev->ops->common->remove(edev);
 	rte_intr_disable(&pci_dev->intr_handle);
-	rte_intr_callback_unregister(&pci_dev->intr_handle,
-				     qede_interrupt_handler, (void *)eth_dev);
+
+	switch (pci_dev->intr_handle.type) {
+	case RTE_INTR_HANDLE_UIO_INTX:
+	case RTE_INTR_HANDLE_VFIO_LEGACY:
+		rte_intr_callback_unregister(&pci_dev->intr_handle,
+					     qede_interrupt_handler_intx,
+					     (void *)eth_dev);
+		break;
+	default:
+		rte_intr_callback_unregister(&pci_dev->intr_handle,
+					   qede_interrupt_handler,
+					   (void *)eth_dev);
+	}
+
 	if (ECORE_IS_CMT(edev))
 		rte_eal_alarm_cancel(qede_poll_sp_sb_cb, (void *)eth_dev);
 }
@@ -2056,6 +2063,70 @@ qede_set_mc_addr_list(struct rte_eth_dev *eth_dev, struct ether_addr *mc_addrs,
 	return qede_add_mcast_filters(eth_dev, mc_addrs, mc_addrs_num);
 }
 
+/* Update MTU via vport-update without doing port restart.
+ * The vport must be deactivated before calling this API.
+ */
+int qede_update_mtu(struct rte_eth_dev *eth_dev, uint16_t mtu)
+{
+	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+	struct ecore_hwfn *p_hwfn;
+	int rc;
+	int i;
+
+	if (IS_PF(edev)) {
+		struct ecore_sp_vport_update_params params;
+
+		memset(&params, 0, sizeof(struct ecore_sp_vport_update_params));
+		params.vport_id = 0;
+		params.mtu = mtu;
+		params.vport_id = 0;
+		for_each_hwfn(edev, i) {
+			p_hwfn = &edev->hwfns[i];
+			params.opaque_fid = p_hwfn->hw_info.opaque_fid;
+			rc = ecore_sp_vport_update(p_hwfn, &params,
+					ECORE_SPQ_MODE_EBLOCK, NULL);
+			if (rc != ECORE_SUCCESS)
+				goto err;
+		}
+	} else {
+		for_each_hwfn(edev, i) {
+			p_hwfn = &edev->hwfns[i];
+			rc = ecore_vf_pf_update_mtu(p_hwfn, mtu);
+			if (rc == ECORE_INVAL) {
+				DP_INFO(edev, "VF MTU Update TLV not supported\n");
+				/* Recreate vport */
+				rc = qede_start_vport(qdev, mtu);
+				if (rc != ECORE_SUCCESS)
+					goto err;
+
+				/* Restore config lost due to vport stop */
+				if (eth_dev->data->promiscuous)
+					qede_promiscuous_enable(eth_dev);
+				else
+					qede_promiscuous_disable(eth_dev);
+
+				if (eth_dev->data->all_multicast)
+					qede_allmulticast_enable(eth_dev);
+				else
+					qede_allmulticast_disable(eth_dev);
+
+				qede_vlan_offload_set(eth_dev,
+						      qdev->vlan_offload_mask);
+			} else if (rc != ECORE_SUCCESS) {
+				goto err;
+			}
+		}
+	}
+	DP_INFO(edev, "%s MTU updated to %u\n", IS_PF(edev) ? "PF" : "VF", mtu);
+
+	return 0;
+
+err:
+	DP_ERR(edev, "Failed to update MTU\n");
+	return -1;
+}
+
 static int qede_flow_ctrl_set(struct rte_eth_dev *eth_dev,
 			      struct rte_eth_fc_conf *fc_conf)
 {
@@ -2210,7 +2281,7 @@ int qede_rss_hash_update(struct rte_eth_dev *eth_dev,
 	vport_update_params.vport_id = 0;
 	/* pass the L2 handles instead of qids */
 	for (i = 0 ; i < ECORE_RSS_IND_TABLE_SIZE ; i++) {
-		idx = qdev->rss_ind_table[i];
+		idx = i % QEDE_RSS_COUNT(qdev);
 		rss_params.rss_ind_table[i] = qdev->fp_array[idx].rxq->handle;
 	}
 	vport_update_params.rss_params = &rss_params;
@@ -2458,12 +2529,8 @@ static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 		dev->data->dev_started = 0;
 		qede_dev_stop(dev);
 		restart = true;
-	} else {
-		if (IS_PF(edev))
-			qede_mac_addr_remove(dev, 0);
 	}
 	rte_delay_ms(1000);
-	qede_start_vport(qdev, mtu); /* Recreate vport */
 	qdev->mtu = mtu;
 
 	/* Fix up RX buf size for all queues of the port */
@@ -2483,25 +2550,9 @@ static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 		}
 	}
 	if (max_rx_pkt_len > ETHER_MAX_LEN)
-		dev->data->dev_conf.rxmode.jumbo_frame = 1;
-	else
-		dev->data->dev_conf.rxmode.jumbo_frame = 0;
-
-	/* Restore config lost due to vport stop */
-	if (IS_PF(edev))
-		qede_mac_addr_set(dev, &qdev->primary_mac);
-
-	if (dev->data->promiscuous)
-		qede_promiscuous_enable(dev);
+		dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
 	else
-		qede_promiscuous_disable(dev);
-
-	if (dev->data->all_multicast)
-		qede_allmulticast_enable(dev);
-	else
-		qede_allmulticast_disable(dev);
-
-	qede_vlan_offload_set(dev, qdev->vlan_offload_mask);
+		dev->data->dev_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
 
 	if (!dev->data->dev_started && restart) {
 		qede_dev_start(dev);
@@ -2999,6 +3050,9 @@ static const struct eth_dev_ops qede_eth_vf_dev_ops = {
 	.mtu_set = qede_set_mtu,
 	.udp_tunnel_port_add = qede_udp_dst_port_add,
 	.udp_tunnel_port_del = qede_udp_dst_port_del,
+	.mac_addr_add = qede_mac_addr_add,
+	.mac_addr_remove = qede_mac_addr_remove,
+	.mac_addr_set = qede_mac_addr_set,
 };
 
 static void qede_update_pf_params(struct ecore_dev *edev)
@@ -3027,6 +3081,7 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf)
 	/* Fix up ecore debug level */
 	uint32_t dp_module = ~0 & ~ECORE_MSG_HW;
 	uint8_t dp_level = ECORE_LEVEL_VERBOSE;
+	uint32_t int_mode;
 	int rc;
 
 	/* Extract key data structures */
@@ -3071,8 +3126,22 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf)
 		return -ENODEV;
 	}
 	qede_update_pf_params(edev);
-	rte_intr_callback_register(&pci_dev->intr_handle,
-				   qede_interrupt_handler, (void *)eth_dev);
+
+	switch (pci_dev->intr_handle.type) {
+	case RTE_INTR_HANDLE_UIO_INTX:
+	case RTE_INTR_HANDLE_VFIO_LEGACY:
+		int_mode = ECORE_INT_MODE_INTA;
+		rte_intr_callback_register(&pci_dev->intr_handle,
+					   qede_interrupt_handler_intx,
+					   (void *)eth_dev);
+		break;
+	default:
+		int_mode = ECORE_INT_MODE_MSIX;
+		rte_intr_callback_register(&pci_dev->intr_handle,
+					   qede_interrupt_handler,
+					   (void *)eth_dev);
+	}
+
 	if (rte_intr_enable(&pci_dev->intr_handle)) {
 		DP_ERR(edev, "rte_intr_enable() failed\n");
 		return -ENODEV;
@@ -3080,7 +3149,8 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf)
 
 	/* Start the Slowpath-process */
 	memset(&params, 0, sizeof(struct qed_slowpath_params));
-	params.int_mode = ECORE_INT_MODE_MSIX;
+
+	params.int_mode = int_mode;
 	params.drv_major = QEDE_PMD_VERSION_MAJOR;
 	params.drv_minor = QEDE_PMD_VERSION_MINOR;
 	params.drv_rev = QEDE_PMD_VERSION_REVISION;
@@ -3163,7 +3233,7 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf)
 						ECORE_LEADING_HWFN(edev),
 						vf_mac,
 						&is_mac_forced);
-			if (is_mac_exist && is_mac_forced) {
+			if (is_mac_exist) {
 				DP_INFO(edev, "VF macaddr received from PF\n");
 				ether_addr_copy((struct ether_addr *)&vf_mac,
 						&eth_dev->data->mac_addrs[0]);
@@ -3371,9 +3441,7 @@ RTE_PMD_REGISTER_PCI(net_qede_vf, rte_qedevf_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_qede_vf, pci_id_qedevf_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_qede_vf, "* igb_uio | vfio-pci");
 
-RTE_INIT(qede_init_log);
-static void
-qede_init_log(void)
+RTE_INIT(qede_init_log)
 {
 	qede_logtype_init = rte_log_register("pmd.net.qede.init");
 	if (qede_logtype_init >= 0)
diff --git a/drivers/net/qede/qede_ethdev.h b/drivers/net/qede/qede_ethdev.h
index a335d4da..6e9a5b4b 100644
--- a/drivers/net/qede/qede_ethdev.h
+++ b/drivers/net/qede/qede_ethdev.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 
@@ -34,6 +32,7 @@
 #include "base/nvm_cfg.h"
 #include "base/ecore_sp_commands.h"
 #include "base/ecore_l2.h"
+#include "base/ecore_vf.h"
 
 #include "qede_logs.h"
 #include "qede_if.h"
@@ -45,7 +44,7 @@
 /* Driver versions */
 #define QEDE_PMD_VER_PREFIX		"QEDE PMD"
 #define QEDE_PMD_VERSION_MAJOR		2
-#define QEDE_PMD_VERSION_MINOR	        8
+#define QEDE_PMD_VERSION_MINOR	        9
 #define QEDE_PMD_VERSION_REVISION       0
 #define QEDE_PMD_VERSION_PATCH	        1
 
diff --git a/drivers/net/qede/qede_fdir.c b/drivers/net/qede/qede_fdir.c
index 9d0b0526..83580d04 100644
--- a/drivers/net/qede/qede_fdir.c
+++ b/drivers/net/qede/qede_fdir.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2017 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include <rte_udp.h>
@@ -465,5 +463,8 @@ int qede_ntuple_filter_conf(struct rte_eth_dev *eth_dev,
 		udpv4_flow->src_port = ntuple->src_port;
 		udpv4_flow->dst_port = ntuple->dst_port;
 	}
+
+	fdir_entry.action.rx_queue = ntuple->queue;
+
 	return qede_config_cmn_fdir_filter(eth_dev, &fdir_entry, add);
 }
diff --git a/drivers/net/qede/qede_if.h b/drivers/net/qede/qede_if.h
index 01f17c9a..ee5e54c1 100644
--- a/drivers/net/qede/qede_if.h
+++ b/drivers/net/qede/qede_if.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef _QEDE_IF_H
diff --git a/drivers/net/qede/qede_logs.h b/drivers/net/qede/qede_logs.h
index e7f714f2..3187d97b 100644
--- a/drivers/net/qede/qede_logs.h
+++ b/drivers/net/qede/qede_logs.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #ifndef _QEDE_LOGS_H_
diff --git a/drivers/net/qede/qede_main.c b/drivers/net/qede/qede_main.c
index c3407fe9..46fa8371 100644
--- a/drivers/net/qede/qede_main.c
+++ b/drivers/net/qede/qede_main.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include <limits.h>
@@ -281,7 +279,7 @@ static int qed_slowpath_start(struct ecore_dev *edev,
 	/* Start the slowpath */
 	memset(&hw_init_params, 0, sizeof(hw_init_params));
 	hw_init_params.b_hw_start = true;
-	hw_init_params.int_mode = ECORE_INT_MODE_MSIX;
+	hw_init_params.int_mode = params->int_mode;
 	hw_init_params.allow_npar_tx_switch = true;
 	hw_init_params.bin_fw_data = data;
 
@@ -634,8 +632,11 @@ void qed_link_update(struct ecore_hwfn *hwfn)
 {
 	struct ecore_dev *edev = hwfn->p_dev;
 	struct qede_dev *qdev = (struct qede_dev *)edev;
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)qdev->ethdev;
 
-	qede_link_update((struct rte_eth_dev *)qdev->ethdev, 0);
+	if (!qede_link_update(dev, 0))
+		_rte_eth_dev_callback_process(dev,
+					      RTE_ETH_EVENT_INTR_LSC, NULL);
 }
 
 static int qed_drain(struct ecore_dev *edev)
diff --git a/drivers/net/qede/qede_rxtx.c b/drivers/net/qede/qede_rxtx.c
index bdb5d6f1..0f157ded 100644
--- a/drivers/net/qede/qede_rxtx.c
+++ b/drivers/net/qede/qede_rxtx.c
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 #include <rte_net.h>
@@ -192,9 +190,15 @@ static void qede_rx_queue_release_mbufs(struct qede_rx_queue *rxq)
 void qede_rx_queue_release(void *rx_queue)
 {
 	struct qede_rx_queue *rxq = rx_queue;
+	struct qede_dev *qdev = rxq->qdev;
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+
+	PMD_INIT_FUNC_TRACE(edev);
 
 	if (rxq) {
 		qede_rx_queue_release_mbufs(rxq);
+		qdev->ops->common->chain_free(edev, &rxq->rx_bd_ring);
+		qdev->ops->common->chain_free(edev, &rxq->rx_comp_ring);
 		rte_free(rxq->sw_rx_ring);
 		rte_free(rxq);
 	}
@@ -350,9 +354,14 @@ static void qede_tx_queue_release_mbufs(struct qede_tx_queue *txq)
 void qede_tx_queue_release(void *tx_queue)
 {
 	struct qede_tx_queue *txq = tx_queue;
+	struct qede_dev *qdev = txq->qdev;
+	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
+
+	PMD_INIT_FUNC_TRACE(edev);
 
 	if (txq) {
 		qede_tx_queue_release_mbufs(txq);
+		qdev->ops->common->chain_free(edev, &txq->tx_pbl);
 		rte_free(txq->sw_tx_ring);
 		rte_free(txq);
 	}
@@ -441,8 +450,6 @@ void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev)
 	struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
 	struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 	struct qede_fastpath *fp;
-	struct qede_rx_queue *rxq;
-	struct qede_tx_queue *txq;
 	uint16_t sb_idx;
 	uint8_t i;
 
@@ -467,21 +474,13 @@ void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev)
 	for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 		if (eth_dev->data->rx_queues[i]) {
 			qede_rx_queue_release(eth_dev->data->rx_queues[i]);
-			rxq = eth_dev->data->rx_queues[i];
-			qdev->ops->common->chain_free(edev,
-						      &rxq->rx_bd_ring);
-			qdev->ops->common->chain_free(edev,
-						      &rxq->rx_comp_ring);
 			eth_dev->data->rx_queues[i] = NULL;
 		}
 	}
 
 	for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
 		if (eth_dev->data->tx_queues[i]) {
-			txq = eth_dev->data->tx_queues[i];
 			qede_tx_queue_release(eth_dev->data->tx_queues[i]);
-			qdev->ops->common->chain_free(edev,
-						      &txq->tx_pbl);
 			eth_dev->data->tx_queues[i] = NULL;
 		}
 	}
@@ -1970,7 +1969,7 @@ qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		}
 
 		/* Descriptor based VLAN insertion */
-		if (tx_ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) {
+		if (tx_ol_flags & PKT_TX_VLAN_PKT) {
 			vlan = rte_cpu_to_le_16(mbuf->vlan_tci);
 			bd1_bd_flags_bf |=
 			    1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT;
diff --git a/drivers/net/qede/qede_rxtx.h b/drivers/net/qede/qede_rxtx.h
index 84a834d2..e710fbae 100644
--- a/drivers/net/qede/qede_rxtx.h
+++ b/drivers/net/qede/qede_rxtx.h
@@ -1,9 +1,7 @@
-/*
+/* SPDX-License-Identifier: BSD-3-Clause
  * Copyright (c) 2016 - 2018 Cavium Inc.
  * All rights reserved.
  * www.cavium.com
- *
- * See LICENSE.qede_pmd for copyright and licensing details.
  */
 
 
@@ -147,7 +145,6 @@
 				   PKT_TX_TCP_SEG)
 
 #define QEDE_TX_OFFLOAD_MASK (QEDE_TX_CSUM_OFFLOAD_MASK | \
-			      PKT_TX_QINQ_PKT           | \
 			      PKT_TX_VLAN_PKT		| \
 			      PKT_TX_TUNNEL_VXLAN	| \
 			      PKT_TX_TUNNEL_GENEVE	| \
diff --git a/drivers/net/ring/rte_eth_ring.c b/drivers/net/ring/rte_eth_ring.c
index 35b837c3..791deb0b 100644
--- a/drivers/net/ring/rte_eth_ring.c
+++ b/drivers/net/ring/rte_eth_ring.c
@@ -164,6 +164,7 @@ eth_dev_info(struct rte_eth_dev *dev,
 	dev_info->max_rx_queues = (uint16_t)internals->max_rx_queues;
 	dev_info->max_tx_queues = (uint16_t)internals->max_tx_queues;
 	dev_info->min_rx_bufsize = 0;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -684,9 +685,7 @@ RTE_PMD_REGISTER_ALIAS(net_ring, eth_ring);
 RTE_PMD_REGISTER_PARAM_STRING(net_ring,
 	ETH_RING_NUMA_NODE_ACTION_ARG "=name:node:action(ATTACH|CREATE)");
 
-RTE_INIT(eth_ring_init_log);
-static void
-eth_ring_init_log(void)
+RTE_INIT(eth_ring_init_log)
 {
 	eth_ring_logtype = rte_log_register("pmd.net.ring");
 	if (eth_ring_logtype >= 0)
diff --git a/drivers/net/sfc/meson.build b/drivers/net/sfc/meson.build
index 3aa14c7b..2d34e869 100644
--- a/drivers/net/sfc/meson.build
+++ b/drivers/net/sfc/meson.build
@@ -6,7 +6,7 @@
 # This software was jointly developed between OKTET Labs (under contract
 # for Solarflare) and Solarflare Communications, Inc.
 
-if arch_subdir != 'x86'
+if arch_subdir != 'x86' or cc.sizeof('void *') == 4
 	build = false
 endif
 
diff --git a/drivers/net/sfc/sfc_dp_rx.h b/drivers/net/sfc/sfc_dp_rx.h
index 83faad16..ce96e83f 100644
--- a/drivers/net/sfc/sfc_dp_rx.h
+++ b/drivers/net/sfc/sfc_dp_rx.h
@@ -195,6 +195,7 @@ struct sfc_dp_rx {
 #define SFC_DP_RX_FEAT_TUNNELS			0x4
 #define SFC_DP_RX_FEAT_FLOW_FLAG		0x8
 #define SFC_DP_RX_FEAT_FLOW_MARK		0x10
+#define SFC_DP_RX_FEAT_CHECKSUM			0x20
 	sfc_dp_rx_get_dev_info_t		*get_dev_info;
 	sfc_dp_rx_pool_ops_supported_t		*pool_ops_supported;
 	sfc_dp_rx_qsize_up_rings_t		*qsize_up_rings;
diff --git a/drivers/net/sfc/sfc_dp_tx.h b/drivers/net/sfc/sfc_dp_tx.h
index a075612c..eda9676c 100644
--- a/drivers/net/sfc/sfc_dp_tx.h
+++ b/drivers/net/sfc/sfc_dp_tx.h
@@ -39,8 +39,6 @@ struct sfc_dp_tx_qcreate_info {
 	unsigned int		max_fill_level;
 	/** Minimum number of unused Tx descriptors to do reap */
 	unsigned int		free_thresh;
-	/** Transmit queue configuration flags */
-	unsigned int		flags;
 	/** Offloads enabled on the transmit queue */
 	uint64_t		offloads;
 	/** Tx queue size */
diff --git a/drivers/net/sfc/sfc_ef10_essb_rx.c b/drivers/net/sfc/sfc_ef10_essb_rx.c
index 5f5af602..81c8f7fb 100644
--- a/drivers/net/sfc/sfc_ef10_essb_rx.c
+++ b/drivers/net/sfc/sfc_ef10_essb_rx.c
@@ -322,6 +322,12 @@ sfc_ef10_essb_rx_get_pending(struct sfc_ef10_essb_rxq *rxq,
 			const efx_qword_t *qwordp;
 			uint16_t pkt_len;
 
+			/* Buffers to be discarded have 0 in packet type */
+			if (unlikely(m->packet_type == 0)) {
+				rte_mempool_put(rxq->refill_mb_pool, m);
+				goto next_buf;
+			}
+
 			rx_pkts[n_rx_pkts++] = m;
 
 			/* Parse pseudo-header */
@@ -357,6 +363,7 @@ sfc_ef10_essb_rx_get_pending(struct sfc_ef10_essb_rxq *rxq,
 				EFX_QWORD_FIELD(*qwordp,
 						ES_EZ_ESSB_RX_PREFIX_MARK);
 
+next_buf:
 			m = sfc_ef10_essb_next_mbuf(rxq, m);
 		} while (todo_bufs-- > 0);
 	}
@@ -404,21 +411,45 @@ sfc_ef10_essb_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 
 static sfc_dp_rx_qdesc_npending_t sfc_ef10_essb_rx_qdesc_npending;
 static unsigned int
-sfc_ef10_essb_rx_qdesc_npending(__rte_unused struct sfc_dp_rxq *dp_rxq)
+sfc_ef10_essb_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq)
 {
-	/*
-	 * Correct implementation requires EvQ polling and events
-	 * processing.
-	 */
-	return -ENOTSUP;
+	struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
+	const unsigned int evq_old_read_ptr = rxq->evq_read_ptr;
+	efx_qword_t rx_ev;
+
+	if (unlikely(rxq->flags & (SFC_EF10_ESSB_RXQ_NOT_RUNNING |
+				   SFC_EF10_ESSB_RXQ_EXCEPTION)))
+		return rxq->bufs_pending;
+
+	while (sfc_ef10_essb_rx_event_get(rxq, &rx_ev)) {
+		/*
+		 * DROP_EVENT is an internal to the NIC, software should
+		 * never see it and, therefore, may ignore it.
+		 */
+		sfc_ef10_essb_rx_process_ev(rxq, rx_ev);
+	}
+
+	sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->evq_ptr_mask,
+			   evq_old_read_ptr, rxq->evq_read_ptr);
+
+	return rxq->bufs_pending;
 }
 
 static sfc_dp_rx_qdesc_status_t sfc_ef10_essb_rx_qdesc_status;
 static int
-sfc_ef10_essb_rx_qdesc_status(__rte_unused struct sfc_dp_rxq *dp_rxq,
-			      __rte_unused uint16_t offset)
+sfc_ef10_essb_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset)
 {
-	return -ENOTSUP;
+	struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
+	unsigned int pending = sfc_ef10_essb_rx_qdesc_npending(dp_rxq);
+
+	if (offset < pending)
+		return RTE_ETH_RX_DESC_DONE;
+
+	if (offset < (rxq->added - rxq->completed) * rxq->block_size +
+		     rxq->left_in_completed - rxq->block_size)
+		return RTE_ETH_RX_DESC_AVAIL;
+
+	return RTE_ETH_RX_DESC_UNAVAIL;
 }
 
 static sfc_dp_rx_get_dev_info_t sfc_ef10_essb_rx_get_dev_info;
@@ -647,29 +678,20 @@ static void
 sfc_ef10_essb_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 {
 	struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
-	unsigned int i, j;
+	unsigned int i;
 	const struct sfc_ef10_essb_rx_sw_desc *rxd;
 	struct rte_mbuf *m;
 
-	if (rxq->completed != rxq->added && rxq->left_in_completed > 0) {
-		rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
-		m = sfc_ef10_essb_mbuf_by_index(rxq, rxd->first_mbuf,
-				rxq->block_size - rxq->left_in_completed);
-		do {
-			rxq->left_in_completed--;
-			rte_mempool_put(rxq->refill_mb_pool, m);
-			m = sfc_ef10_essb_next_mbuf(rxq, m);
-		} while (rxq->left_in_completed > 0);
-		rxq->completed++;
-	}
-
 	for (i = rxq->completed; i != rxq->added; ++i) {
 		rxd = &rxq->sw_ring[i & rxq->rxq_ptr_mask];
-		m = rxd->first_mbuf;
-		for (j = 0; j < rxq->block_size; ++j) {
+		m = sfc_ef10_essb_mbuf_by_index(rxq, rxd->first_mbuf,
+				rxq->block_size - rxq->left_in_completed);
+		while (rxq->left_in_completed > 0) {
 			rte_mempool_put(rxq->refill_mb_pool, m);
 			m = sfc_ef10_essb_next_mbuf(rxq, m);
+			rxq->left_in_completed--;
 		}
+		rxq->left_in_completed = rxq->block_size;
 	}
 
 	rxq->flags &= ~SFC_EF10_ESSB_RXQ_STARTED;
@@ -683,7 +705,8 @@ struct sfc_dp_rx sfc_ef10_essb_rx = {
 				  SFC_DP_HW_FW_CAP_RX_ES_SUPER_BUFFER,
 	},
 	.features		= SFC_DP_RX_FEAT_FLOW_FLAG |
-				  SFC_DP_RX_FEAT_FLOW_MARK,
+				  SFC_DP_RX_FEAT_FLOW_MARK |
+				  SFC_DP_RX_FEAT_CHECKSUM,
 	.get_dev_info		= sfc_ef10_essb_rx_get_dev_info,
 	.pool_ops_supported	= sfc_ef10_essb_rx_pool_ops_supported,
 	.qsize_up_rings		= sfc_ef10_essb_rx_qsize_up_rings,
diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
index 42b35b9b..6a5052b9 100644
--- a/drivers/net/sfc/sfc_ef10_rx.c
+++ b/drivers/net/sfc/sfc_ef10_rx.c
@@ -658,7 +658,8 @@ struct sfc_dp_rx sfc_ef10_rx = {
 		.hw_fw_caps	= SFC_DP_HW_FW_CAP_EF10,
 	},
 	.features		= SFC_DP_RX_FEAT_MULTI_PROCESS |
-				  SFC_DP_RX_FEAT_TUNNELS,
+				  SFC_DP_RX_FEAT_TUNNELS |
+				  SFC_DP_RX_FEAT_CHECKSUM,
 	.get_dev_info		= sfc_ef10_rx_get_dev_info,
 	.qsize_up_rings		= sfc_ef10_rx_qsize_up_rings,
 	.qcreate		= sfc_ef10_rx_qcreate,
diff --git a/drivers/net/sfc/sfc_ef10_rx_ev.h b/drivers/net/sfc/sfc_ef10_rx_ev.h
index 615bd29b..868c755f 100644
--- a/drivers/net/sfc/sfc_ef10_rx_ev.h
+++ b/drivers/net/sfc/sfc_ef10_rx_ev.h
@@ -37,8 +37,10 @@ sfc_ef10_rx_ev_to_offloads(const efx_qword_t rx_ev, struct rte_mbuf *m,
 	if (unlikely(rx_ev.eq_u64[0] &
 		rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) |
 				 (1ull << ESF_DZ_RX_ECRC_ERR_LBN) |
-				 (1ull << ESF_DZ_RX_PARSE_INCOMPLETE_LBN))))
+				 (1ull << ESF_DZ_RX_PARSE_INCOMPLETE_LBN)))) {
+		/* Zero packet type is used as a marker to dicard bad packets */
 		goto done;
+	}
 
 #if SFC_EF10_RX_EV_ENCAP_SUPPORT
 	switch (EFX_QWORD_FIELD(rx_ev, ESF_EZ_RX_ENCAP_HDR)) {
@@ -120,6 +122,8 @@ sfc_ef10_rx_ev_to_offloads(const efx_qword_t rx_ev, struct rte_mbuf *m,
 		if (tun_ptype == 0)
 			l2_ptype = RTE_PTYPE_L2_ETHER_ARP;
 		break;
+	case ESE_DZ_L3_CLASS_UNKNOWN:
+		break;
 	default:
 		/* Unexpected Layer 3 class */
 		SFC_ASSERT(false);
@@ -157,6 +161,8 @@ sfc_ef10_rx_ev_to_offloads(const efx_qword_t rx_ev, struct rte_mbuf *m,
 		SFC_ASSERT(false);
 	}
 
+	SFC_ASSERT(l2_ptype != 0);
+
 done:
 	m->ol_flags = ol_flags & ol_mask;
 	m->packet_type = tun_ptype | l2_ptype | l3_ptype | l4_ptype;
diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c
index 1b6499f8..9decbf5a 100644
--- a/drivers/net/sfc/sfc_ethdev.c
+++ b/drivers/net/sfc/sfc_ethdev.c
@@ -874,14 +874,12 @@ sfc_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	}
 
 	/*
-	 * The driver does not use it, but other PMDs update jumbo_frame
+	 * The driver does not use it, but other PMDs update jumbo frame
 	 * flag and max_rx_pkt_len when MTU is set.
 	 */
 	if (mtu > ETHER_MAX_LEN) {
 		struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
-
 		rxmode->offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
-		rxmode->jumbo_frame = 1;
 	}
 
 	dev->data->dev_conf.rxmode.max_rx_pkt_len = sa->port.pdu;
@@ -1029,7 +1027,7 @@ sfc_set_mc_addr_list(struct rte_eth_dev *dev, struct ether_addr *mc_addr_set,
 	if (rc != 0)
 		sfc_err(sa, "cannot set multicast address list (rc = %u)", rc);
 
-	SFC_ASSERT(rc > 0);
+	SFC_ASSERT(rc >= 0);
 	return -rc;
 }
 
@@ -1057,9 +1055,7 @@ sfc_rx_queue_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
 	qinfo->conf.rx_free_thresh = rxq->refill_threshold;
 	qinfo->conf.rx_drop_en = 1;
 	qinfo->conf.rx_deferred_start = rxq_info->deferred_start;
-	qinfo->conf.offloads = DEV_RX_OFFLOAD_IPV4_CKSUM |
-			       DEV_RX_OFFLOAD_UDP_CKSUM |
-			       DEV_RX_OFFLOAD_TCP_CKSUM;
+	qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
 	if (rxq_info->type_flags & EFX_RXQ_FLAG_SCATTER) {
 		qinfo->conf.offloads |= DEV_RX_OFFLOAD_SCATTER;
 		qinfo->scattered_rx = 1;
@@ -1089,7 +1085,6 @@ sfc_tx_queue_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 
 	memset(qinfo, 0, sizeof(*qinfo));
 
-	qinfo->conf.txq_flags = txq_info->txq->flags;
 	qinfo->conf.offloads = txq_info->txq->offloads;
 	qinfo->conf.tx_free_thresh = txq_info->txq->free_thresh;
 	qinfo->conf.tx_deferred_start = txq_info->deferred_start;
@@ -2097,9 +2092,7 @@ RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx,
 	SFC_KVARG_RXD_WAIT_TIMEOUT_NS "=<long> "
 	SFC_KVARG_STATS_UPDATE_PERIOD_MS "=<long>");
 
-RTE_INIT(sfc_driver_register_logtype);
-static void
-sfc_driver_register_logtype(void)
+RTE_INIT(sfc_driver_register_logtype)
 {
 	int ret;
 
diff --git a/drivers/net/sfc/sfc_filter.c b/drivers/net/sfc/sfc_filter.c
index 77e2ea56..6ff380a3 100644
--- a/drivers/net/sfc/sfc_filter.c
+++ b/drivers/net/sfc/sfc_filter.c
@@ -75,6 +75,7 @@ int
 sfc_filter_attach(struct sfc_adapter *sa)
 {
 	int rc;
+	unsigned int i;
 
 	sfc_log_init(sa, "entry");
 
@@ -88,6 +89,19 @@ sfc_filter_attach(struct sfc_adapter *sa)
 
 	efx_filter_fini(sa->nic);
 
+	sa->filter.supports_ip_proto_or_addr_filter = B_FALSE;
+	sa->filter.supports_rem_or_local_port_filter = B_FALSE;
+	for (i = 0; i < sa->filter.supported_match_num; ++i) {
+		if (sa->filter.supported_match[i] &
+		    (EFX_FILTER_MATCH_IP_PROTO | EFX_FILTER_MATCH_LOC_HOST |
+		     EFX_FILTER_MATCH_REM_HOST))
+			sa->filter.supports_ip_proto_or_addr_filter = B_TRUE;
+
+		if (sa->filter.supported_match[i] &
+		    (EFX_FILTER_MATCH_LOC_PORT | EFX_FILTER_MATCH_REM_PORT))
+			sa->filter.supports_rem_or_local_port_filter = B_TRUE;
+	}
+
 	sfc_log_init(sa, "done");
 
 	return 0;
diff --git a/drivers/net/sfc/sfc_filter.h b/drivers/net/sfc/sfc_filter.h
index d3e1c2f9..64ab114e 100644
--- a/drivers/net/sfc/sfc_filter.h
+++ b/drivers/net/sfc/sfc_filter.h
@@ -25,6 +25,16 @@ struct sfc_filter {
 	uint32_t			*supported_match;
 	/** List of flow rules */
 	struct sfc_flow_list		flow_list;
+	/**
+	 * Supports any of ip_proto, remote host or local host
+	 * filters. This flag is used for filter match exceptions
+	 */
+	boolean_t			supports_ip_proto_or_addr_filter;
+	/**
+	 * Supports any of remote port or local port filters.
+	 * This flag is used for filter match exceptions
+	 */
+	boolean_t			supports_rem_or_local_port_filter;
 };
 
 struct sfc_adapter;
diff --git a/drivers/net/sfc/sfc_flow.c b/drivers/net/sfc/sfc_flow.c
index 5613d59a..371648b0 100644
--- a/drivers/net/sfc/sfc_flow.c
+++ b/drivers/net/sfc/sfc_flow.c
@@ -93,6 +93,8 @@ static sfc_flow_spec_check sfc_flow_check_unknown_dst_flags;
 static sfc_flow_spec_set_vals sfc_flow_set_ethertypes;
 static sfc_flow_spec_set_vals sfc_flow_set_ifrm_unknown_dst_flags;
 static sfc_flow_spec_check sfc_flow_check_ifrm_unknown_dst_flags;
+static sfc_flow_spec_set_vals sfc_flow_set_outer_vid_flag;
+static sfc_flow_spec_check sfc_flow_check_outer_vid_flag;
 
 static boolean_t
 sfc_flow_is_zero(const uint8_t *buf, unsigned int size)
@@ -371,7 +373,8 @@ sfc_flow_parse_vlan(const struct rte_flow_item *item,
 	 * the outer tag and the next matches the inner tag.
 	 */
 	if (mask->tci == supp_mask.tci) {
-		vid = rte_bswap16(spec->tci);
+		/* Apply mask to keep VID only */
+		vid = rte_bswap16(spec->tci & mask->tci);
 
 		if (!(efx_spec->efs_match_flags &
 		      EFX_FILTER_MATCH_OUTER_VID)) {
@@ -1780,6 +1783,43 @@ sfc_flow_set_ethertypes(struct sfc_flow_spec *spec,
 }
 
 /**
+ * Set the EFX_FILTER_MATCH_OUTER_VID match flag with value 0
+ * in the same specifications after copying.
+ *
+ * @param spec[in, out]
+ *   SFC flow specification to update.
+ * @param filters_count_for_one_val[in]
+ *   How many specifications should have the same match flag, what is the
+ *   number of specifications before copying.
+ * @param error[out]
+ *   Perform verbose error reporting if not NULL.
+ */
+static int
+sfc_flow_set_outer_vid_flag(struct sfc_flow_spec *spec,
+			    unsigned int filters_count_for_one_val,
+			    struct rte_flow_error *error)
+{
+	unsigned int i;
+
+	if (filters_count_for_one_val != spec->count) {
+		rte_flow_error_set(error, EINVAL,
+			RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+			"Number of specifications is incorrect "
+			"while copying by outer VLAN ID");
+		return -rte_errno;
+	}
+
+	for (i = 0; i < spec->count; i++) {
+		spec->filters[i].efs_match_flags |=
+			EFX_FILTER_MATCH_OUTER_VID;
+
+		spec->filters[i].efs_outer_vid = 0;
+	}
+
+	return 0;
+}
+
+/**
  * Set the EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST and
  * EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST match flags in the same
  * specifications after copying.
@@ -1858,6 +1898,36 @@ sfc_flow_check_ifrm_unknown_dst_flags(efx_filter_match_flags_t match,
 	return B_FALSE;
 }
 
+/**
+ * Check that the list of supported filters has a filter that differs
+ * from @p match in that it has no flag EFX_FILTER_MATCH_OUTER_VID
+ * in this case that filter will be used and the flag
+ * EFX_FILTER_MATCH_OUTER_VID is not needed.
+ *
+ * @param match[in]
+ *   The match flags of filter.
+ * @param spec[in]
+ *   Specification to be supplemented.
+ * @param filter[in]
+ *   SFC filter with list of supported filters.
+ */
+static boolean_t
+sfc_flow_check_outer_vid_flag(efx_filter_match_flags_t match,
+			      __rte_unused efx_filter_spec_t *spec,
+			      struct sfc_filter *filter)
+{
+	unsigned int i;
+	efx_filter_match_flags_t match_without_vid =
+		match & ~EFX_FILTER_MATCH_OUTER_VID;
+
+	for (i = 0; i < filter->supported_match_num; i++) {
+		if (match_without_vid == filter->supported_match[i])
+			return B_FALSE;
+	}
+
+	return B_TRUE;
+}
+
 /*
  * Match flags that can be automatically added to filters.
  * Selecting the last minimum when searching for the copy flag ensures that the
@@ -1885,6 +1955,12 @@ static const struct sfc_flow_copy_flag sfc_flow_copy_flags[] = {
 		.set_vals = sfc_flow_set_ifrm_unknown_dst_flags,
 		.spec_check = sfc_flow_check_ifrm_unknown_dst_flags,
 	},
+	{
+		.flag = EFX_FILTER_MATCH_OUTER_VID,
+		.vals_count = 1,
+		.set_vals = sfc_flow_set_outer_vid_flag,
+		.spec_check = sfc_flow_check_outer_vid_flag,
+	},
 };
 
 /* Get item from array sfc_flow_copy_flags */
@@ -2094,11 +2170,14 @@ sfc_flow_is_match_with_vids(efx_filter_match_flags_t match_flags,
  * Check whether the spec maps to a hardware filter which is known to be
  * ineffective despite being valid.
  *
+ * @param filter[in]
+ *   SFC filter with list of supported filters.
  * @param spec[in]
  *   SFC flow specification.
  */
 static boolean_t
-sfc_flow_is_match_flags_exception(struct sfc_flow_spec *spec)
+sfc_flow_is_match_flags_exception(struct sfc_filter *filter,
+				  struct sfc_flow_spec *spec)
 {
 	unsigned int i;
 	uint16_t ether_type;
@@ -2114,8 +2193,9 @@ sfc_flow_is_match_flags_exception(struct sfc_flow_spec *spec)
 						EFX_FILTER_MATCH_ETHER_TYPE |
 						EFX_FILTER_MATCH_LOC_MAC)) {
 			ether_type = spec->filters[i].efs_ether_type;
-			if (ether_type == EFX_ETHER_TYPE_IPV4 ||
-			    ether_type == EFX_ETHER_TYPE_IPV6)
+			if (filter->supports_ip_proto_or_addr_filter &&
+			    (ether_type == EFX_ETHER_TYPE_IPV4 ||
+			     ether_type == EFX_ETHER_TYPE_IPV6))
 				return B_TRUE;
 		} else if (sfc_flow_is_match_with_vids(match_flags,
 				EFX_FILTER_MATCH_ETHER_TYPE |
@@ -2125,8 +2205,9 @@ sfc_flow_is_match_flags_exception(struct sfc_flow_spec *spec)
 				EFX_FILTER_MATCH_IP_PROTO |
 				EFX_FILTER_MATCH_LOC_MAC)) {
 			ip_proto = spec->filters[i].efs_ip_proto;
-			if (ip_proto == EFX_IPPROTO_TCP ||
-			    ip_proto == EFX_IPPROTO_UDP)
+			if (filter->supports_rem_or_local_port_filter &&
+			    (ip_proto == EFX_IPPROTO_TCP ||
+			     ip_proto == EFX_IPPROTO_UDP))
 				return B_TRUE;
 		}
 	}
@@ -2153,7 +2234,7 @@ sfc_flow_validate_match_flags(struct sfc_adapter *sa,
 			return rc;
 	}
 
-	if (sfc_flow_is_match_flags_exception(&flow->spec)) {
+	if (sfc_flow_is_match_flags_exception(&sa->filter, &flow->spec)) {
 		rte_flow_error_set(error, ENOTSUP,
 			RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 			"The flow rule pattern is unsupported");
diff --git a/drivers/net/sfc/sfc_rx.c b/drivers/net/sfc/sfc_rx.c
index cc76a5b1..d8503e20 100644
--- a/drivers/net/sfc/sfc_rx.c
+++ b/drivers/net/sfc/sfc_rx.c
@@ -517,7 +517,8 @@ struct sfc_dp_rx sfc_efx_rx = {
 		.type		= SFC_DP_RX,
 		.hw_fw_caps	= 0,
 	},
-	.features		= SFC_DP_RX_FEAT_SCATTER,
+	.features		= SFC_DP_RX_FEAT_SCATTER |
+				  SFC_DP_RX_FEAT_CHECKSUM,
 	.qsize_up_rings		= sfc_efx_rx_qsize_up_rings,
 	.qcreate		= sfc_efx_rx_qcreate,
 	.qdestroy		= sfc_efx_rx_qdestroy,
@@ -792,9 +793,12 @@ sfc_rx_get_dev_offload_caps(struct sfc_adapter *sa)
 
 	caps |= DEV_RX_OFFLOAD_JUMBO_FRAME;
 	caps |= DEV_RX_OFFLOAD_CRC_STRIP;
-	caps |= DEV_RX_OFFLOAD_IPV4_CKSUM;
-	caps |= DEV_RX_OFFLOAD_UDP_CKSUM;
-	caps |= DEV_RX_OFFLOAD_TCP_CKSUM;
+
+	if (sa->dp_rx->features & SFC_DP_RX_FEAT_CHECKSUM) {
+		caps |= DEV_RX_OFFLOAD_IPV4_CKSUM;
+		caps |= DEV_RX_OFFLOAD_UDP_CKSUM;
+		caps |= DEV_RX_OFFLOAD_TCP_CKSUM;
+	}
 
 	if (encp->enc_tunnel_encapsulations_supported &&
 	    (sa->dp_rx->features & SFC_DP_RX_FEAT_TUNNELS))
@@ -817,10 +821,8 @@ sfc_rx_get_queue_offload_caps(struct sfc_adapter *sa)
 static int
 sfc_rx_qcheck_conf(struct sfc_adapter *sa, unsigned int rxq_max_fill_level,
 		   const struct rte_eth_rxconf *rx_conf,
-		   uint64_t offloads)
+		   __rte_unused uint64_t offloads)
 {
-	uint64_t offloads_supported = sfc_rx_get_dev_offload_caps(sa) |
-				      sfc_rx_get_queue_offload_caps(sa);
 	int rc = 0;
 
 	if (rx_conf->rx_thresh.pthresh != 0 ||
@@ -842,14 +844,6 @@ sfc_rx_qcheck_conf(struct sfc_adapter *sa, unsigned int rxq_max_fill_level,
 		rc = EINVAL;
 	}
 
-	if ((offloads & DEV_RX_OFFLOAD_CHECKSUM) !=
-	    DEV_RX_OFFLOAD_CHECKSUM)
-		sfc_warn(sa, "Rx checksum offloads cannot be disabled - always on (IPv4/TCP/UDP)");
-
-	if ((offloads_supported & DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM) &&
-	    (~offloads & DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM))
-		sfc_warn(sa, "Rx outer IPv4 checksum offload cannot be disabled - always on");
-
 	return rc;
 }
 
@@ -1424,6 +1418,8 @@ sfc_rx_qinit_info(struct sfc_adapter *sa, unsigned int sw_index)
 static int
 sfc_rx_check_mode(struct sfc_adapter *sa, struct rte_eth_rxmode *rxmode)
 {
+	uint64_t offloads_supported = sfc_rx_get_dev_offload_caps(sa) |
+				      sfc_rx_get_queue_offload_caps(sa);
 	struct sfc_rss *rss = &sa->rss;
 	int rc = 0;
 
@@ -1443,10 +1439,29 @@ sfc_rx_check_mode(struct sfc_adapter *sa, struct rte_eth_rxmode *rxmode)
 		rc = EINVAL;
 	}
 
-	if (~rxmode->offloads & DEV_RX_OFFLOAD_CRC_STRIP) {
+	/* KEEP_CRC offload flag is not supported by PMD
+	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
+	 */
+	if (rte_eth_dev_must_keep_crc(rxmode->offloads)) {
 		sfc_warn(sa, "FCS stripping cannot be disabled - always on");
 		rxmode->offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
-		rxmode->hw_strip_crc = 1;
+	}
+
+	/*
+	 * Requested offloads are validated against supported by ethdev,
+	 * so unsupported offloads cannot be added as the result of
+	 * below check.
+	 */
+	if ((rxmode->offloads & DEV_RX_OFFLOAD_CHECKSUM) !=
+	    (offloads_supported & DEV_RX_OFFLOAD_CHECKSUM)) {
+		sfc_warn(sa, "Rx checksum offloads cannot be disabled - always on (IPv4/TCP/UDP)");
+		rxmode->offloads |= DEV_RX_OFFLOAD_CHECKSUM;
+	}
+
+	if ((offloads_supported & DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM) &&
+	    (~rxmode->offloads & DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM)) {
+		sfc_warn(sa, "Rx outer IPv4 checksum offload cannot be disabled - always on");
+		rxmode->offloads |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM;
 	}
 
 	return rc;
diff --git a/drivers/net/sfc/sfc_tx.c b/drivers/net/sfc/sfc_tx.c
index 1bcc2c69..6d42a1a6 100644
--- a/drivers/net/sfc/sfc_tx.c
+++ b/drivers/net/sfc/sfc_tx.c
@@ -171,7 +171,6 @@ sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
 	txq->free_thresh =
 		(tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh :
 		SFC_TX_DEFAULT_FREE_THRESH;
-	txq->flags = tx_conf->txq_flags;
 	txq->offloads = offloads;
 
 	rc = sfc_dma_alloc(sa, "txq", sw_index, EFX_TXQ_SIZE(txq_info->entries),
@@ -182,7 +181,6 @@ sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
 	memset(&info, 0, sizeof(info));
 	info.max_fill_level = txq_max_fill_level;
 	info.free_thresh = txq->free_thresh;
-	info.flags = tx_conf->txq_flags;
 	info.offloads = offloads;
 	info.txq_entries = txq_info->entries;
 	info.dma_desc_size_max = encp->enc_tx_dma_desc_size_max;
@@ -431,18 +429,10 @@ sfc_tx_qstart(struct sfc_adapter *sa, unsigned int sw_index)
 	if (rc != 0)
 		goto fail_ev_qstart;
 
-	/*
-	 * The absence of ETH_TXQ_FLAGS_IGNORE is associated with a legacy
-	 * application which expects that IPv4 checksum offload is enabled
-	 * all the time as there is no legacy flag to turn off the offload.
-	 */
-	if ((txq->offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) ||
-	    (~txq->flags & ETH_TXQ_FLAGS_IGNORE))
+	if (txq->offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)
 		flags |= EFX_TXQ_CKSUM_IPV4;
 
-	if ((txq->offloads & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) ||
-	    ((~txq->flags & ETH_TXQ_FLAGS_IGNORE) &&
-	     (offloads_supported & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)))
+	if (txq->offloads & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)
 		flags |= EFX_TXQ_CKSUM_INNER_IPV4;
 
 	if ((txq->offloads & DEV_TX_OFFLOAD_TCP_CKSUM) ||
@@ -453,16 +443,7 @@ sfc_tx_qstart(struct sfc_adapter *sa, unsigned int sw_index)
 			flags |= EFX_TXQ_CKSUM_INNER_TCPUDP;
 	}
 
-	/*
-	 * The absence of ETH_TXQ_FLAGS_IGNORE is associated with a legacy
-	 * application. In turn, the absence of ETH_TXQ_FLAGS_NOXSUMTCP is
-	 * associated specifically with a legacy application which expects
-	 * both TCP checksum offload and TSO to be enabled because the legacy
-	 * API does not provide a dedicated mechanism to control TSO.
-	 */
-	if ((txq->offloads & DEV_TX_OFFLOAD_TCP_TSO) ||
-	    ((~txq->flags & ETH_TXQ_FLAGS_IGNORE) &&
-	     (~txq->flags & ETH_TXQ_FLAGS_NOXSUMTCP)))
+	if (txq->offloads & DEV_TX_OFFLOAD_TCP_TSO)
 		flags |= EFX_TXQ_FATSOV2;
 
 	rc = efx_tx_qcreate(sa->nic, sw_index, 0, &txq->mem,
diff --git a/drivers/net/sfc/sfc_tx.h b/drivers/net/sfc/sfc_tx.h
index c2e5f13e..146b805c 100644
--- a/drivers/net/sfc/sfc_tx.h
+++ b/drivers/net/sfc/sfc_tx.h
@@ -58,7 +58,6 @@ struct sfc_txq {
 	struct sfc_dp_txq		*dp;
 	efx_txq_t			*common;
 	unsigned int			free_thresh;
-	unsigned int			flags;
 	uint64_t			offloads;
 };
 
diff --git a/drivers/net/softnic/Makefile b/drivers/net/softnic/Makefile
index d56fecd6..ea9b65f4 100644
--- a/drivers/net/softnic/Makefile
+++ b/drivers/net/softnic/Makefile
@@ -8,13 +8,15 @@ include $(RTE_SDK)/mk/rte.vars.mk
 #
 LIB = librte_pmd_softnic.a
 
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
+LDLIBS += -lrte_pipeline -lrte_port -lrte_table
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_sched
 LDLIBS += -lrte_bus_vdev
 
-EXPORT_MAP := rte_pmd_eth_softnic_version.map
+EXPORT_MAP := rte_pmd_softnic_version.map
 
 LIBABIVER := 1
 
@@ -22,11 +24,30 @@ LIBABIVER := 1
 # all source are stored in SRCS-y
 #
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_mempool.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_swq.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_link.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_tm.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_tap.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_action.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_pipeline.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_thread.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_cli.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += parser.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += conn.c
 
 #
 # Export include files
 #
 SYMLINK-y-include += rte_eth_softnic.h
 
+ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+$(info Softnic PMD can only operate in a linuxapp environment, \
+please change the definition of the RTE_TARGET environment variable)
+all:
+clean:
+else
+
 include $(RTE_SDK)/mk/rte.lib.mk
+
+endif
diff --git a/drivers/net/softnic/conn.c b/drivers/net/softnic/conn.c
new file mode 100644
index 00000000..990cf40f
--- /dev/null
+++ b/drivers/net/softnic/conn.c
@@ -0,0 +1,332 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+
+#define __USE_GNU
+#include <sys/socket.h>
+
+#include <sys/epoll.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <errno.h>
+
+#include "conn.h"
+
+#define MSG_CMD_TOO_LONG "Command too long."
+
+struct softnic_conn {
+	char *welcome;
+	char *prompt;
+	char *buf;
+	char *msg_in;
+	char *msg_out;
+	size_t buf_size;
+	size_t msg_in_len_max;
+	size_t msg_out_len_max;
+	size_t msg_in_len;
+	int fd_server;
+	int fd_client_group;
+	softnic_conn_msg_handle_t msg_handle;
+	void *msg_handle_arg;
+};
+
+struct softnic_conn *
+softnic_conn_init(struct softnic_conn_params *p)
+{
+	struct sockaddr_in server_address;
+	struct softnic_conn *conn;
+	int fd_server, fd_client_group, status;
+
+	memset(&server_address, 0, sizeof(server_address));
+
+	/* Check input arguments */
+	if (p == NULL ||
+		p->welcome == NULL ||
+		p->prompt == NULL ||
+		p->addr == NULL ||
+		p->buf_size == 0 ||
+		p->msg_in_len_max == 0 ||
+		p->msg_out_len_max == 0 ||
+		p->msg_handle == NULL)
+		return NULL;
+
+	status = inet_aton(p->addr, &server_address.sin_addr);
+	if (status == 0)
+		return NULL;
+
+	/* Memory allocation */
+	conn = calloc(1, sizeof(struct softnic_conn));
+	if (conn == NULL)
+		return NULL;
+
+	conn->welcome = calloc(1, CONN_WELCOME_LEN_MAX + 1);
+	conn->prompt = calloc(1, CONN_PROMPT_LEN_MAX + 1);
+	conn->buf = calloc(1, p->buf_size);
+	conn->msg_in = calloc(1, p->msg_in_len_max + 1);
+	conn->msg_out = calloc(1, p->msg_out_len_max + 1);
+
+	if (conn->welcome == NULL ||
+		conn->prompt == NULL ||
+		conn->buf == NULL ||
+		conn->msg_in == NULL ||
+		conn->msg_out == NULL) {
+		softnic_conn_free(conn);
+		return NULL;
+	}
+
+	/* Server socket */
+	server_address.sin_family = AF_INET;
+	server_address.sin_port = htons(p->port);
+
+	fd_server = socket(AF_INET,
+		SOCK_STREAM | SOCK_NONBLOCK,
+		0);
+	if (fd_server == -1) {
+		softnic_conn_free(conn);
+		return NULL;
+	}
+
+	status = bind(fd_server,
+		(struct sockaddr *)&server_address,
+		sizeof(server_address));
+	if (status == -1) {
+		softnic_conn_free(conn);
+		close(fd_server);
+		return NULL;
+	}
+
+	status = listen(fd_server, 16);
+	if (status == -1) {
+		softnic_conn_free(conn);
+		close(fd_server);
+		return NULL;
+	}
+
+	/* Client group */
+	fd_client_group = epoll_create(1);
+	if (fd_client_group == -1) {
+		softnic_conn_free(conn);
+		close(fd_server);
+		return NULL;
+	}
+
+	/* Fill in */
+	strncpy(conn->welcome, p->welcome, CONN_WELCOME_LEN_MAX);
+	strncpy(conn->prompt, p->prompt, CONN_PROMPT_LEN_MAX);
+	conn->buf_size = p->buf_size;
+	conn->msg_in_len_max = p->msg_in_len_max;
+	conn->msg_out_len_max = p->msg_out_len_max;
+	conn->msg_in_len = 0;
+	conn->fd_server = fd_server;
+	conn->fd_client_group = fd_client_group;
+	conn->msg_handle = p->msg_handle;
+	conn->msg_handle_arg = p->msg_handle_arg;
+
+	return conn;
+}
+
+void
+softnic_conn_free(struct softnic_conn *conn)
+{
+	if (conn == NULL)
+		return;
+
+	if (conn->fd_client_group)
+		close(conn->fd_client_group);
+
+	if (conn->fd_server)
+		close(conn->fd_server);
+
+	free(conn->msg_out);
+	free(conn->msg_in);
+	free(conn->prompt);
+	free(conn->welcome);
+	free(conn);
+}
+
+int
+softnic_conn_poll_for_conn(struct softnic_conn *conn)
+{
+	struct sockaddr_in client_address;
+	struct epoll_event event;
+	socklen_t client_address_length;
+	int fd_client, status;
+
+	/* Check input arguments */
+	if (conn == NULL)
+		return -1;
+
+	/* Server socket */
+	client_address_length = sizeof(client_address);
+	fd_client = accept4(conn->fd_server,
+		(struct sockaddr *)&client_address,
+		&client_address_length,
+		SOCK_NONBLOCK);
+	if (fd_client == -1) {
+		if (errno == EAGAIN || errno == EWOULDBLOCK)
+			return 0;
+
+		return -1;
+	}
+
+	/* Client group */
+	event.events = EPOLLIN | EPOLLRDHUP | EPOLLHUP;
+	event.data.fd = fd_client;
+
+	status = epoll_ctl(conn->fd_client_group,
+		EPOLL_CTL_ADD,
+		fd_client,
+		&event);
+	if (status == -1) {
+		close(fd_client);
+		return -1;
+	}
+
+	/* Client */
+	status = write(fd_client,
+		conn->welcome,
+		strlen(conn->welcome));
+	if (status == -1) {
+		close(fd_client);
+		return -1;
+	}
+
+	status = write(fd_client,
+		conn->prompt,
+		strlen(conn->prompt));
+	if (status == -1) {
+		close(fd_client);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+data_event_handle(struct softnic_conn *conn,
+	int fd_client)
+{
+	ssize_t len, i, status;
+
+	/* Read input message */
+
+	len = read(fd_client,
+		conn->buf,
+		conn->buf_size);
+	if (len == -1) {
+		if (errno == EAGAIN || errno == EWOULDBLOCK)
+			return 0;
+
+		return -1;
+	}
+	if (len == 0)
+		return 0;
+
+	/* Handle input messages */
+	for (i = 0; i < len; i++) {
+		if (conn->buf[i] == '\n') {
+			size_t n;
+
+			conn->msg_in[conn->msg_in_len] = 0;
+			conn->msg_out[0] = 0;
+
+			conn->msg_handle(conn->msg_in,
+				conn->msg_out,
+				conn->msg_out_len_max,
+				conn->msg_handle_arg);
+
+			n = strlen(conn->msg_out);
+			if (n) {
+				status = write(fd_client,
+					conn->msg_out,
+					n);
+				if (status == -1)
+					return status;
+			}
+
+			conn->msg_in_len = 0;
+		} else if (conn->msg_in_len < conn->msg_in_len_max) {
+			conn->msg_in[conn->msg_in_len] = conn->buf[i];
+			conn->msg_in_len++;
+		} else {
+			status = write(fd_client,
+				MSG_CMD_TOO_LONG,
+				strlen(MSG_CMD_TOO_LONG));
+			if (status == -1)
+				return status;
+
+			conn->msg_in_len = 0;
+		}
+	}
+
+	/* Write prompt */
+	status = write(fd_client,
+		conn->prompt,
+		strlen(conn->prompt));
+	if (status == -1)
+		return status;
+
+	return 0;
+}
+
+static int
+control_event_handle(struct softnic_conn *conn,
+	int fd_client)
+{
+	int status;
+
+	status = epoll_ctl(conn->fd_client_group,
+		EPOLL_CTL_DEL,
+		fd_client,
+		NULL);
+	if (status == -1)
+		return -1;
+
+	status = close(fd_client);
+	if (status == -1)
+		return -1;
+
+	return 0;
+}
+
+int
+softnic_conn_poll_for_msg(struct softnic_conn *conn)
+{
+	struct epoll_event event;
+	int fd_client, status, status_data = 0, status_control = 0;
+
+	/* Check input arguments */
+	if (conn == NULL)
+		return -1;
+
+	/* Client group */
+	status = epoll_wait(conn->fd_client_group,
+		&event,
+		1,
+		0);
+	if (status == -1)
+		return -1;
+	if (status == 0)
+		return 0;
+
+	fd_client = event.data.fd;
+
+	/* Data available */
+	if (event.events & EPOLLIN)
+		status_data = data_event_handle(conn, fd_client);
+
+	/* Control events */
+	if (event.events & (EPOLLRDHUP | EPOLLERR | EPOLLHUP))
+		status_control = control_event_handle(conn, fd_client);
+
+	if (status_data || status_control)
+		return -1;
+
+	return 0;
+}
diff --git a/drivers/net/softnic/conn.h b/drivers/net/softnic/conn.h
new file mode 100644
index 00000000..631edeef
--- /dev/null
+++ b/drivers/net/softnic/conn.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#ifndef __INCLUDE_CONN_H__
+#define __INCLUDE_CONN_H__
+
+#include <stdint.h>
+
+struct softnic_conn;
+
+#ifndef CONN_WELCOME_LEN_MAX
+#define CONN_WELCOME_LEN_MAX                               1024
+#endif
+
+#ifndef CONN_PROMPT_LEN_MAX
+#define CONN_PROMPT_LEN_MAX                                16
+#endif
+
+typedef void (*softnic_conn_msg_handle_t)(char *msg_in,
+	char *msg_out,
+	size_t msg_out_len_max,
+	void *arg);
+
+struct softnic_conn_params {
+	const char *welcome;
+	const char *prompt;
+	const char *addr;
+	uint16_t port;
+	size_t buf_size;
+	size_t msg_in_len_max;
+	size_t msg_out_len_max;
+	softnic_conn_msg_handle_t msg_handle;
+	void *msg_handle_arg;
+};
+
+struct softnic_conn *
+softnic_conn_init(struct softnic_conn_params *p);
+
+void
+softnic_conn_free(struct softnic_conn *conn);
+
+int
+softnic_conn_poll_for_conn(struct softnic_conn *conn);
+
+int
+softnic_conn_poll_for_msg(struct softnic_conn *conn);
+
+#endif
diff --git a/drivers/net/softnic/firmware.cli b/drivers/net/softnic/firmware.cli
new file mode 100644
index 00000000..300cf6e3
--- /dev/null
+++ b/drivers/net/softnic/firmware.cli
@@ -0,0 +1,21 @@
+; SPDX-License-Identifier: BSD-3-Clause
+; Copyright(c) 2018 Intel Corporation
+
+link LINK dev 0000:02:00.0
+
+pipeline RX period 10 offset_port_id 0
+pipeline RX port in bsz 32 link LINK rxq 0
+pipeline RX port out bsz 32 swq RXQ0
+pipeline RX table match stub
+pipeline RX port in 0 table 0
+pipeline RX table 0 rule add match default action fwd port 0
+
+pipeline TX period 10 offset_port_id 0
+pipeline TX port in bsz 32 swq TXQ0
+pipeline TX port out bsz 32 link LINK txq 0
+pipeline TX table match stub
+pipeline TX port in 0 table 0
+pipeline TX table 0 rule add match default action fwd port 0
+
+thread 1 pipeline RX enable
+thread 1 pipeline TX enable
diff --git a/drivers/net/softnic/hash_func.h b/drivers/net/softnic/hash_func.h
new file mode 100644
index 00000000..198d2b20
--- /dev/null
+++ b/drivers/net/softnic/hash_func.h
@@ -0,0 +1,359 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#ifndef __INCLUDE_HASH_FUNC_H__
+#define __INCLUDE_HASH_FUNC_H__
+
+#include <rte_common.h>
+
+static inline uint64_t
+hash_xor_key8(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t xor0;
+
+	xor0 = seed ^ (k[0] & m[0]);
+
+	return (xor0 >> 32) ^ xor0;
+}
+
+static inline uint64_t
+hash_xor_key16(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t xor0;
+
+	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+
+	return (xor0 >> 32) ^ xor0;
+}
+
+static inline uint64_t
+hash_xor_key24(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t xor0;
+
+	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+
+	xor0 ^= k[2] & m[2];
+
+	return (xor0 >> 32) ^ xor0;
+}
+
+static inline uint64_t
+hash_xor_key32(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t xor0, xor1;
+
+	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
+
+	xor0 ^= xor1;
+
+	return (xor0 >> 32) ^ xor0;
+}
+
+static inline uint64_t
+hash_xor_key40(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t xor0, xor1;
+
+	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
+
+	xor0 ^= xor1;
+
+	xor0 ^= k[4] & m[4];
+
+	return (xor0 >> 32) ^ xor0;
+}
+
+static inline uint64_t
+hash_xor_key48(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t xor0, xor1, xor2;
+
+	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
+	xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
+
+	xor0 ^= xor1;
+
+	xor0 ^= xor2;
+
+	return (xor0 >> 32) ^ xor0;
+}
+
+static inline uint64_t
+hash_xor_key56(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t xor0, xor1, xor2;
+
+	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
+	xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
+
+	xor0 ^= xor1;
+	xor2 ^= k[6] & m[6];
+
+	xor0 ^= xor2;
+
+	return (xor0 >> 32) ^ xor0;
+}
+
+static inline uint64_t
+hash_xor_key64(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t xor0, xor1, xor2, xor3;
+
+	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
+	xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
+	xor3 = (k[6] & m[6]) ^ (k[7] & m[7]);
+
+	xor0 ^= xor1;
+	xor2 ^= xor3;
+
+	xor0 ^= xor2;
+
+	return (xor0 >> 32) ^ xor0;
+}
+
+#if defined(RTE_ARCH_X86_64)
+
+#include <x86intrin.h>
+
+static inline uint64_t
+hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t crc0;
+
+	crc0 = _mm_crc32_u64(seed, k[0] & m[0]);
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t k0, crc0, crc1;
+
+	k0 = k[0] & m[0];
+
+	crc0 = _mm_crc32_u64(k0, seed);
+	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
+
+	crc0 ^= crc1;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t k0, k2, crc0, crc1;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+
+	crc0 = _mm_crc32_u64(k0, seed);
+	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
+
+	crc0 = _mm_crc32_u64(crc0, k2);
+
+	crc0 ^= crc1;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t k0, k2, crc0, crc1, crc2, crc3;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+
+	crc0 = _mm_crc32_u64(k0, seed);
+	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
+
+	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
+	crc3 = k2 >> 32;
+
+	crc0 = _mm_crc32_u64(crc0, crc1);
+	crc1 = _mm_crc32_u64(crc2, crc3);
+
+	crc0 ^= crc1;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t k0, k2, crc0, crc1, crc2, crc3;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+
+	crc0 = _mm_crc32_u64(k0, seed);
+	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
+
+	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
+	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
+
+	crc0 = _mm_crc32_u64(crc0, crc1);
+	crc1 = _mm_crc32_u64(crc2, crc3);
+
+	crc0 ^= crc1;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t k0, k2, k5, crc0, crc1, crc2, crc3;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+	k5 = k[5] & m[5];
+
+	crc0 = _mm_crc32_u64(k0, seed);
+	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
+
+	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
+	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
+
+	crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
+	crc1 = _mm_crc32_u64(crc3, k5);
+
+	crc0 ^= crc1;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+	k5 = k[5] & m[5];
+
+	crc0 = _mm_crc32_u64(k0, seed);
+	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
+
+	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
+	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
+
+	crc4 = _mm_crc32_u64(k5, k[6] & m[6]);
+	crc5 = k5 >> 32;
+
+	crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
+	crc1 = _mm_crc32_u64(crc3, (crc4 << 32) ^ crc5);
+
+	crc0 ^= crc1;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+	k5 = k[5] & m[5];
+
+	crc0 = _mm_crc32_u64(k0, seed);
+	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
+
+	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
+	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
+
+	crc4 = _mm_crc32_u64(k5, k[6] & m[6]);
+	crc5 = _mm_crc32_u64(k5 >> 32, k[7] & m[7]);
+
+	crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
+	crc1 = _mm_crc32_u64(crc3, (crc4 << 32) ^ crc5);
+
+	crc0 ^= crc1;
+
+	return crc0;
+}
+
+#define hash_default_key8			hash_crc_key8
+#define hash_default_key16			hash_crc_key16
+#define hash_default_key24			hash_crc_key24
+#define hash_default_key32			hash_crc_key32
+#define hash_default_key40			hash_crc_key40
+#define hash_default_key48			hash_crc_key48
+#define hash_default_key56			hash_crc_key56
+#define hash_default_key64			hash_crc_key64
+
+#elif defined(RTE_ARCH_ARM64)
+#include "hash_func_arm64.h"
+#else
+
+#define hash_default_key8			hash_xor_key8
+#define hash_default_key16			hash_xor_key16
+#define hash_default_key24			hash_xor_key24
+#define hash_default_key32			hash_xor_key32
+#define hash_default_key40			hash_xor_key40
+#define hash_default_key48			hash_xor_key48
+#define hash_default_key56			hash_xor_key56
+#define hash_default_key64			hash_xor_key64
+
+#endif
+
+#endif
diff --git a/drivers/net/softnic/hash_func_arm64.h b/drivers/net/softnic/hash_func_arm64.h
new file mode 100644
index 00000000..ae6c0f41
--- /dev/null
+++ b/drivers/net/softnic/hash_func_arm64.h
@@ -0,0 +1,261 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Linaro Limited. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef __HASH_FUNC_ARM64_H__
+#define __HASH_FUNC_ARM64_H__
+
+#define _CRC32CX(crc, val)	\
+	__asm__("crc32cx %w[c], %w[c], %x[v]":[c] "+r" (crc):[v] "r" (val))
+
+static inline uint64_t
+hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key;
+	uint64_t *m = mask;
+	uint32_t crc0;
+
+	crc0 = seed;
+	_CRC32CX(crc0, k[0] & m[0]);
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key, k0;
+	uint64_t *m = mask;
+	uint32_t crc0, crc1;
+
+	k0 = k[0] & m[0];
+
+	crc0 = k0;
+	_CRC32CX(crc0, seed);
+	crc1 = k0 >> 32;
+	_CRC32CX(crc1, k[1] & m[1]);
+
+	crc0 ^= crc1;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key, k0, k2;
+	uint64_t *m = mask;
+	uint32_t crc0, crc1;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+
+	crc0 = k0;
+	_CRC32CX(crc0, seed);
+	crc1 = k0 >> 32;
+	_CRC32CX(crc1, k[1] & m[1]);
+
+	_CRC32CX(crc0, k2);
+
+	crc0 ^= crc1;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key, k0, k2;
+	uint64_t *m = mask;
+	uint32_t crc0, crc1, crc2, crc3;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+
+	crc0 = k0;
+	_CRC32CX(crc0, seed);
+	crc1 = k0 >> 32;
+	_CRC32CX(crc1, k[1] & m[1]);
+
+	crc2 = k2;
+	_CRC32CX(crc2, k[3] & m[3]);
+	crc3 = k2 >> 32;
+
+	_CRC32CX(crc0, crc1);
+	_CRC32CX(crc2, crc3);
+
+	crc0 ^= crc2;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key, k0, k2;
+	uint64_t *m = mask;
+	uint32_t crc0, crc1, crc2, crc3;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+
+	crc0 = k0;
+	_CRC32CX(crc0, seed);
+	crc1 = k0 >> 32;
+	_CRC32CX(crc1, k[1] & m[1]);
+
+	crc2 = k2;
+	_CRC32CX(crc2, k[3] & m[3]);
+	crc3 = k2 >> 32;
+	_CRC32CX(crc3, k[4] & m[4]);
+
+	_CRC32CX(crc0, crc1);
+	_CRC32CX(crc2, crc3);
+
+	crc0 ^= crc2;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key, k0, k2, k5;
+	uint64_t *m = mask;
+	uint32_t crc0, crc1, crc2, crc3;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+	k5 = k[5] & m[5];
+
+	crc0 = k0;
+	_CRC32CX(crc0, seed);
+	crc1 = k0 >> 32;
+	_CRC32CX(crc1, k[1] & m[1]);
+
+	crc2 = k2;
+	_CRC32CX(crc2, k[3] & m[3]);
+	crc3 = k2 >> 32;
+	_CRC32CX(crc3, k[4] & m[4]);
+
+	_CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2);
+	_CRC32CX(crc3, k5);
+
+	crc0 ^= crc3;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key, k0, k2, k5;
+	uint64_t *m = mask;
+	uint32_t crc0, crc1, crc2, crc3, crc4, crc5;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+	k5 = k[5] & m[5];
+
+	crc0 = k0;
+	_CRC32CX(crc0, seed);
+	crc1 = k0 >> 32;
+	_CRC32CX(crc1, k[1] & m[1]);
+
+	crc2 = k2;
+	_CRC32CX(crc2, k[3] & m[3]);
+	crc3 = k2 >> 32;
+	_CRC32CX(crc3, k[4] & m[4]);
+
+	crc4 = k5;
+	 _CRC32CX(crc4, k[6] & m[6]);
+	crc5 = k5 >> 32;
+
+	_CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2);
+	_CRC32CX(crc3, ((uint64_t)crc4 << 32) ^ crc5);
+
+	crc0 ^= crc3;
+
+	return crc0;
+}
+
+static inline uint64_t
+hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size,
+	uint64_t seed)
+{
+	uint64_t *k = key, k0, k2, k5;
+	uint64_t *m = mask;
+	uint32_t crc0, crc1, crc2, crc3, crc4, crc5;
+
+	k0 = k[0] & m[0];
+	k2 = k[2] & m[2];
+	k5 = k[5] & m[5];
+
+	crc0 = k0;
+	_CRC32CX(crc0, seed);
+	crc1 = k0 >> 32;
+	_CRC32CX(crc1, k[1] & m[1]);
+
+	crc2 = k2;
+	_CRC32CX(crc2, k[3] & m[3]);
+	crc3 = k2 >> 32;
+	_CRC32CX(crc3, k[4] & m[4]);
+
+	crc4 = k5;
+	 _CRC32CX(crc4, k[6] & m[6]);
+	crc5 = k5 >> 32;
+	_CRC32CX(crc5, k[7] & m[7]);
+
+	_CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2);
+	_CRC32CX(crc3, ((uint64_t)crc4 << 32) ^ crc5);
+
+	crc0 ^= crc3;
+
+	return crc0;
+}
+
+#define hash_default_key8			hash_crc_key8
+#define hash_default_key16			hash_crc_key16
+#define hash_default_key24			hash_crc_key24
+#define hash_default_key32			hash_crc_key32
+#define hash_default_key40			hash_crc_key40
+#define hash_default_key48			hash_crc_key48
+#define hash_default_key56			hash_crc_key56
+#define hash_default_key64			hash_crc_key64
+
+#endif
diff --git a/drivers/net/softnic/meson.build b/drivers/net/softnic/meson.build
new file mode 100644
index 00000000..ff982274
--- /dev/null
+++ b/drivers/net/softnic/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+allow_experimental_apis = true
+install_headers('rte_eth_softnic.h')
+sources = files('rte_eth_softnic_tm.c',
+	'rte_eth_softnic.c',
+	'rte_eth_softnic_mempool.c',
+	'rte_eth_softnic_swq.c',
+	'rte_eth_softnic_link.c',
+	'rte_eth_softnic_tap.c',
+	'rte_eth_softnic_action.c',
+	'rte_eth_softnic_pipeline.c',
+	'rte_eth_softnic_thread.c',
+	'rte_eth_softnic_cli.c',
+	'parser.c',
+	'conn.c')
+deps += ['pipeline', 'port', 'table', 'sched']
diff --git a/drivers/net/softnic/parser.c b/drivers/net/softnic/parser.c
new file mode 100644
index 00000000..a8688a21
--- /dev/null
+++ b/drivers/net/softnic/parser.c
@@ -0,0 +1,703 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016 Intel Corporation.
+ * Copyright (c) 2009, Olivier MATZ <zer0@droids-corp.org>
+ * All rights reserved.
+ */
+
+/* For inet_pton4() and inet_pton6() functions:
+ *
+ * Copyright (c) 1996 by Internet Software Consortium.
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SOFTWARE CONSORTIUM DISCLAIMS
+ * ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL INTERNET SOFTWARE
+ * CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
+ * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
+ * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+ * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <getopt.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <string.h>
+#include <libgen.h>
+#include <unistd.h>
+#include <sys/wait.h>
+
+#include <rte_errno.h>
+
+#include "parser.h"
+
+static uint32_t
+get_hex_val(char c)
+{
+	switch (c) {
+	case '0': case '1': case '2': case '3': case '4': case '5':
+	case '6': case '7': case '8': case '9':
+		return c - '0';
+	case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
+		return c - 'A' + 10;
+	case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
+		return c - 'a' + 10;
+	default:
+		return 0;
+	}
+}
+
+int
+softnic_parser_read_arg_bool(const char *p)
+{
+	p = skip_white_spaces(p);
+	int result = -EINVAL;
+
+	if (((p[0] == 'y') && (p[1] == 'e') && (p[2] == 's')) ||
+		((p[0] == 'Y') && (p[1] == 'E') && (p[2] == 'S'))) {
+		p += 3;
+		result = 1;
+	}
+
+	if (((p[0] == 'o') && (p[1] == 'n')) ||
+		((p[0] == 'O') && (p[1] == 'N'))) {
+		p += 2;
+		result = 1;
+	}
+
+	if (((p[0] == 'n') && (p[1] == 'o')) ||
+		((p[0] == 'N') && (p[1] == 'O'))) {
+		p += 2;
+		result = 0;
+	}
+
+	if (((p[0] == 'o') && (p[1] == 'f') && (p[2] == 'f')) ||
+		((p[0] == 'O') && (p[1] == 'F') && (p[2] == 'F'))) {
+		p += 3;
+		result = 0;
+	}
+
+	p = skip_white_spaces(p);
+
+	if (p[0] != '\0')
+		return -EINVAL;
+
+	return result;
+}
+
+int
+softnic_parser_read_int32(int32_t *value, const char *p)
+{
+	char *next;
+	int32_t val;
+
+	p = skip_white_spaces(p);
+	if (!isdigit(*p))
+		return -EINVAL;
+
+	val = strtol(p, &next, 10);
+	if (p == next)
+		return -EINVAL;
+
+	*value = val;
+	return 0;
+}
+
+int
+softnic_parser_read_uint64(uint64_t *value, const char *p)
+{
+	char *next;
+	uint64_t val;
+
+	p = skip_white_spaces(p);
+	if (!isdigit(*p))
+		return -EINVAL;
+
+	val = strtoul(p, &next, 10);
+	if (p == next)
+		return -EINVAL;
+
+	p = next;
+	switch (*p) {
+	case 'T':
+		val *= 1024ULL;
+		/* fall through */
+	case 'G':
+		val *= 1024ULL;
+		/* fall through */
+	case 'M':
+		val *= 1024ULL;
+		/* fall through */
+	case 'k':
+	case 'K':
+		val *= 1024ULL;
+		p++;
+		break;
+	}
+
+	p = skip_white_spaces(p);
+	if (*p != '\0')
+		return -EINVAL;
+
+	*value = val;
+	return 0;
+}
+
+int
+softnic_parser_read_uint64_hex(uint64_t *value, const char *p)
+{
+	char *next;
+	uint64_t val;
+
+	p = skip_white_spaces(p);
+
+	val = strtoul(p, &next, 16);
+	if (p == next)
+		return -EINVAL;
+
+	p = skip_white_spaces(next);
+	if (*p != '\0')
+		return -EINVAL;
+
+	*value = val;
+	return 0;
+}
+
+int
+softnic_parser_read_uint32(uint32_t *value, const char *p)
+{
+	uint64_t val = 0;
+	int ret = softnic_parser_read_uint64(&val, p);
+
+	if (ret < 0)
+		return ret;
+
+	if (val > UINT32_MAX)
+		return -ERANGE;
+
+	*value = val;
+	return 0;
+}
+
+int
+softnic_parser_read_uint32_hex(uint32_t *value, const char *p)
+{
+	uint64_t val = 0;
+	int ret = softnic_parser_read_uint64_hex(&val, p);
+
+	if (ret < 0)
+		return ret;
+
+	if (val > UINT32_MAX)
+		return -ERANGE;
+
+	*value = val;
+	return 0;
+}
+
+int
+softnic_parser_read_uint16(uint16_t *value, const char *p)
+{
+	uint64_t val = 0;
+	int ret = softnic_parser_read_uint64(&val, p);
+
+	if (ret < 0)
+		return ret;
+
+	if (val > UINT16_MAX)
+		return -ERANGE;
+
+	*value = val;
+	return 0;
+}
+
+int
+softnic_parser_read_uint16_hex(uint16_t *value, const char *p)
+{
+	uint64_t val = 0;
+	int ret = softnic_parser_read_uint64_hex(&val, p);
+
+	if (ret < 0)
+		return ret;
+
+	if (val > UINT16_MAX)
+		return -ERANGE;
+
+	*value = val;
+	return 0;
+}
+
+int
+softnic_parser_read_uint8(uint8_t *value, const char *p)
+{
+	uint64_t val = 0;
+	int ret = softnic_parser_read_uint64(&val, p);
+
+	if (ret < 0)
+		return ret;
+
+	if (val > UINT8_MAX)
+		return -ERANGE;
+
+	*value = val;
+	return 0;
+}
+
+int
+softnic_parser_read_uint8_hex(uint8_t *value, const char *p)
+{
+	uint64_t val = 0;
+	int ret = softnic_parser_read_uint64_hex(&val, p);
+
+	if (ret < 0)
+		return ret;
+
+	if (val > UINT8_MAX)
+		return -ERANGE;
+
+	*value = val;
+	return 0;
+}
+
+int
+softnic_parse_tokenize_string(char *string, char *tokens[], uint32_t *n_tokens)
+{
+	uint32_t i;
+
+	if (string == NULL ||
+		tokens == NULL ||
+		(*n_tokens < 1))
+		return -EINVAL;
+
+	for (i = 0; i < *n_tokens; i++) {
+		tokens[i] = strtok_r(string, PARSE_DELIMITER, &string);
+		if (tokens[i] == NULL)
+			break;
+	}
+
+	if (i == *n_tokens &&
+		strtok_r(string, PARSE_DELIMITER, &string) != NULL)
+		return -E2BIG;
+
+	*n_tokens = i;
+	return 0;
+}
+
+int
+softnic_parse_hex_string(char *src, uint8_t *dst, uint32_t *size)
+{
+	char *c;
+	uint32_t len, i;
+
+	/* Check input parameters */
+	if (src == NULL ||
+		dst == NULL ||
+		size == NULL ||
+		(*size == 0))
+		return -1;
+
+	len = strlen(src);
+	if (((len & 3) != 0) ||
+		(len > (*size) * 2))
+		return -1;
+	*size = len / 2;
+
+	for (c = src; *c != 0; c++) {
+		if ((((*c) >= '0') && ((*c) <= '9')) ||
+			(((*c) >= 'A') && ((*c) <= 'F')) ||
+			(((*c) >= 'a') && ((*c) <= 'f')))
+			continue;
+
+		return -1;
+	}
+
+	/* Convert chars to bytes */
+	for (i = 0; i < *size; i++)
+		dst[i] = get_hex_val(src[2 * i]) * 16 +
+			get_hex_val(src[2 * i + 1]);
+
+	return 0;
+}
+
+int
+softnic_parse_mpls_labels(char *string, uint32_t *labels, uint32_t *n_labels)
+{
+	uint32_t n_max_labels = *n_labels, count = 0;
+
+	/* Check for void list of labels */
+	if (strcmp(string, "<void>") == 0) {
+		*n_labels = 0;
+		return 0;
+	}
+
+	/* At least one label should be present */
+	for ( ; (*string != '\0'); ) {
+		char *next;
+		int value;
+
+		if (count >= n_max_labels)
+			return -1;
+
+		if (count > 0) {
+			if (string[0] != ':')
+				return -1;
+
+			string++;
+		}
+
+		value = strtol(string, &next, 10);
+		if (next == string)
+			return -1;
+		string = next;
+
+		labels[count++] = (uint32_t)value;
+	}
+
+	*n_labels = count;
+	return 0;
+}
+
+#define INADDRSZ 4
+#define IN6ADDRSZ 16
+
+/* int
+ * inet_pton4(src, dst)
+ *      like inet_aton() but without all the hexadecimal and shorthand.
+ * return:
+ *      1 if `src' is a valid dotted quad, else 0.
+ * notice:
+ *      does not touch `dst' unless it's returning 1.
+ * author:
+ *      Paul Vixie, 1996.
+ */
+static int
+inet_pton4(const char *src, unsigned char *dst)
+{
+	static const char digits[] = "0123456789";
+	int saw_digit, octets, ch;
+	unsigned char tmp[INADDRSZ], *tp;
+
+	saw_digit = 0;
+	octets = 0;
+	*(tp = tmp) = 0;
+	while ((ch = *src++) != '\0') {
+		const char *pch;
+
+		pch = strchr(digits, ch);
+		if (pch != NULL) {
+			unsigned int new = *tp * 10 + (pch - digits);
+
+			if (new > 255)
+				return 0;
+			if (!saw_digit) {
+				if (++octets > 4)
+					return 0;
+				saw_digit = 1;
+			}
+			*tp = (unsigned char)new;
+		} else if (ch == '.' && saw_digit) {
+			if (octets == 4)
+				return 0;
+			*++tp = 0;
+			saw_digit = 0;
+		} else
+			return 0;
+	}
+	if (octets < 4)
+		return 0;
+
+	memcpy(dst, tmp, INADDRSZ);
+	return 1;
+}
+
+/* int
+ * inet_pton6(src, dst)
+ *      convert presentation level address to network order binary form.
+ * return:
+ *      1 if `src' is a valid [RFC1884 2.2] address, else 0.
+ * notice:
+ *      (1) does not touch `dst' unless it's returning 1.
+ *      (2) :: in a full address is silently ignored.
+ * credit:
+ *      inspired by Mark Andrews.
+ * author:
+ *      Paul Vixie, 1996.
+ */
+static int
+inet_pton6(const char *src, unsigned char *dst)
+{
+	static const char xdigits_l[] = "0123456789abcdef",
+		xdigits_u[] = "0123456789ABCDEF";
+	unsigned char tmp[IN6ADDRSZ], *tp = 0, *endp = 0, *colonp = 0;
+	const char *xdigits = 0, *curtok = 0;
+	int ch = 0, saw_xdigit = 0, count_xdigit = 0;
+	unsigned int val = 0;
+	unsigned int dbloct_count = 0;
+
+	memset((tp = tmp), '\0', IN6ADDRSZ);
+	endp = tp + IN6ADDRSZ;
+	colonp = NULL;
+	/* Leading :: requires some special handling. */
+	if (*src == ':')
+		if (*++src != ':')
+			return 0;
+	curtok = src;
+	saw_xdigit = count_xdigit = 0;
+	val = 0;
+
+	while ((ch = *src++) != '\0') {
+		const char *pch;
+
+		pch = strchr((xdigits = xdigits_l), ch);
+		if (pch == NULL)
+			pch = strchr((xdigits = xdigits_u), ch);
+		if (pch != NULL) {
+			if (count_xdigit >= 4)
+				return 0;
+			val <<= 4;
+			val |= (pch - xdigits);
+			if (val > 0xffff)
+				return 0;
+			saw_xdigit = 1;
+			count_xdigit++;
+			continue;
+		}
+		if (ch == ':') {
+			curtok = src;
+			if (!saw_xdigit) {
+				if (colonp)
+					return 0;
+				colonp = tp;
+				continue;
+			} else if (*src == '\0') {
+				return 0;
+			}
+			if (tp + sizeof(int16_t) > endp)
+				return 0;
+			*tp++ = (unsigned char)((val >> 8) & 0xff);
+			*tp++ = (unsigned char)(val & 0xff);
+			saw_xdigit = 0;
+			count_xdigit = 0;
+			val = 0;
+			dbloct_count++;
+			continue;
+		}
+		if (ch == '.' && ((tp + INADDRSZ) <= endp) &&
+		    inet_pton4(curtok, tp) > 0) {
+			tp += INADDRSZ;
+			saw_xdigit = 0;
+			dbloct_count += 2;
+			break;  /* '\0' was seen by inet_pton4(). */
+		}
+		return 0;
+	}
+	if (saw_xdigit) {
+		if (tp + sizeof(int16_t) > endp)
+			return 0;
+		*tp++ = (unsigned char)((val >> 8) & 0xff);
+		*tp++ = (unsigned char)(val & 0xff);
+		dbloct_count++;
+	}
+	if (colonp != NULL) {
+		/* if we already have 8 double octets, having a colon means error */
+		if (dbloct_count == 8)
+			return 0;
+
+		/* Since some memmove()'s erroneously fail to handle
+		 * overlapping regions, we'll do the shift by hand.
+		 */
+		const int n = tp - colonp;
+		int i;
+
+		for (i = 1; i <= n; i++) {
+			endp[-i] = colonp[n - i];
+			colonp[n - i] = 0;
+		}
+		tp = endp;
+	}
+	if (tp != endp)
+		return 0;
+	memcpy(dst, tmp, IN6ADDRSZ);
+	return 1;
+}
+
+static struct ether_addr *
+my_ether_aton(const char *a)
+{
+	int i;
+	char *end;
+	unsigned long o[ETHER_ADDR_LEN];
+	static struct ether_addr ether_addr;
+
+	i = 0;
+	do {
+		errno = 0;
+		o[i] = strtoul(a, &end, 16);
+		if (errno != 0 || end == a || (end[0] != ':' && end[0] != 0))
+			return NULL;
+		a = end + 1;
+	} while (++i != sizeof(o) / sizeof(o[0]) && end[0] != 0);
+
+	/* Junk at the end of line */
+	if (end[0] != 0)
+		return NULL;
+
+	/* Support the format XX:XX:XX:XX:XX:XX */
+	if (i == ETHER_ADDR_LEN) {
+		while (i-- != 0) {
+			if (o[i] > UINT8_MAX)
+				return NULL;
+			ether_addr.addr_bytes[i] = (uint8_t)o[i];
+		}
+	/* Support the format XXXX:XXXX:XXXX */
+	} else if (i == ETHER_ADDR_LEN / 2) {
+		while (i-- != 0) {
+			if (o[i] > UINT16_MAX)
+				return NULL;
+			ether_addr.addr_bytes[i * 2] = (uint8_t)(o[i] >> 8);
+			ether_addr.addr_bytes[i * 2 + 1] = (uint8_t)(o[i] & 0xff);
+		}
+	/* unknown format */
+	} else
+		return NULL;
+
+	return (struct ether_addr *)&ether_addr;
+}
+
+int
+softnic_parse_ipv4_addr(const char *token, struct in_addr *ipv4)
+{
+	if (strlen(token) >= INET_ADDRSTRLEN)
+		return -EINVAL;
+
+	if (inet_pton4(token, (unsigned char *)ipv4) != 1)
+		return -EINVAL;
+
+	return 0;
+}
+
+int
+softnic_parse_ipv6_addr(const char *token, struct in6_addr *ipv6)
+{
+	if (strlen(token) >= INET6_ADDRSTRLEN)
+		return -EINVAL;
+
+	if (inet_pton6(token, (unsigned char *)ipv6) != 1)
+		return -EINVAL;
+
+	return 0;
+}
+
+int
+softnic_parse_mac_addr(const char *token, struct ether_addr *addr)
+{
+	struct ether_addr *tmp;
+
+	tmp = my_ether_aton(token);
+	if (tmp == NULL)
+		return -1;
+
+	memcpy(addr, tmp, sizeof(struct ether_addr));
+	return 0;
+}
+
+int
+softnic_parse_cpu_core(const char *entry,
+	struct softnic_cpu_core_params *p)
+{
+	size_t num_len;
+	char num[8];
+
+	uint32_t s = 0, c = 0, h = 0, val;
+	uint8_t s_parsed = 0, c_parsed = 0, h_parsed = 0;
+	const char *next = skip_white_spaces(entry);
+	char type;
+
+	if (p == NULL)
+		return -EINVAL;
+
+	/* Expect <CORE> or [sX][cY][h]. At least one parameter is required. */
+	while (*next != '\0') {
+		/* If everything parsed nothing should left */
+		if (s_parsed && c_parsed && h_parsed)
+			return -EINVAL;
+
+		type = *next;
+		switch (type) {
+		case 's':
+		case 'S':
+			if (s_parsed || c_parsed || h_parsed)
+				return -EINVAL;
+			s_parsed = 1;
+			next++;
+			break;
+		case 'c':
+		case 'C':
+			if (c_parsed || h_parsed)
+				return -EINVAL;
+			c_parsed = 1;
+			next++;
+			break;
+		case 'h':
+		case 'H':
+			if (h_parsed)
+				return -EINVAL;
+			h_parsed = 1;
+			next++;
+			break;
+		default:
+			/* If it start from digit it must be only core id. */
+			if (!isdigit(*next) || s_parsed || c_parsed || h_parsed)
+				return -EINVAL;
+
+			type = 'C';
+		}
+
+		for (num_len = 0; *next != '\0'; next++, num_len++) {
+			if (num_len == RTE_DIM(num))
+				return -EINVAL;
+
+			if (!isdigit(*next))
+				break;
+
+			num[num_len] = *next;
+		}
+
+		if (num_len == 0 && type != 'h' && type != 'H')
+			return -EINVAL;
+
+		if (num_len != 0 && (type == 'h' || type == 'H'))
+			return -EINVAL;
+
+		num[num_len] = '\0';
+		val = strtol(num, NULL, 10);
+
+		h = 0;
+		switch (type) {
+		case 's':
+		case 'S':
+			s = val;
+			break;
+		case 'c':
+		case 'C':
+			c = val;
+			break;
+		case 'h':
+		case 'H':
+			h = 1;
+			break;
+		}
+	}
+
+	p->socket_id = s;
+	p->core_id = c;
+	p->thread_id = h;
+	return 0;
+}
diff --git a/drivers/net/softnic/parser.h b/drivers/net/softnic/parser.h
new file mode 100644
index 00000000..1ee3f82a
--- /dev/null
+++ b/drivers/net/softnic/parser.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2016 Intel Corporation
+ */
+
+#ifndef __INCLUDE_SOFTNIC_PARSER_H__
+#define __INCLUDE_SOFTNIC_PARSER_H__
+
+#include <stdint.h>
+
+#include <rte_ip.h>
+#include <rte_ether.h>
+
+#define PARSE_DELIMITER				" \f\n\r\t\v"
+
+#define skip_white_spaces(pos)			\
+({						\
+	__typeof__(pos) _p = (pos);		\
+	for ( ; isspace(*_p); _p++)		\
+		;				\
+	_p;					\
+})
+
+static inline size_t
+skip_digits(const char *src)
+{
+	size_t i;
+
+	for (i = 0; isdigit(src[i]); i++)
+		;
+
+	return i;
+}
+
+int softnic_parser_read_arg_bool(const char *p);
+
+int softnic_parser_read_int32(int32_t *value, const char *p);
+
+int softnic_parser_read_uint64(uint64_t *value, const char *p);
+int softnic_parser_read_uint32(uint32_t *value, const char *p);
+int softnic_parser_read_uint16(uint16_t *value, const char *p);
+int softnic_parser_read_uint8(uint8_t *value, const char *p);
+
+int softnic_parser_read_uint64_hex(uint64_t *value, const char *p);
+int softnic_parser_read_uint32_hex(uint32_t *value, const char *p);
+int softnic_parser_read_uint16_hex(uint16_t *value, const char *p);
+int softnic_parser_read_uint8_hex(uint8_t *value, const char *p);
+
+int softnic_parse_hex_string(char *src, uint8_t *dst, uint32_t *size);
+
+int softnic_parse_ipv4_addr(const char *token, struct in_addr *ipv4);
+int softnic_parse_ipv6_addr(const char *token, struct in6_addr *ipv6);
+int softnic_parse_mac_addr(const char *token, struct ether_addr *addr);
+int softnic_parse_mpls_labels(char *string,
+		uint32_t *labels, uint32_t *n_labels);
+
+struct softnic_cpu_core_params {
+	uint32_t socket_id;
+	uint32_t core_id;
+	uint32_t thread_id;
+};
+
+int softnic_parse_cpu_core(const char *entry,
+		struct softnic_cpu_core_params *p);
+
+int softnic_parse_tokenize_string(char *string,
+		char *tokens[], uint32_t *n_tokens);
+
+#endif
diff --git a/drivers/net/softnic/rte_eth_softnic.c b/drivers/net/softnic/rte_eth_softnic.c
index 6b3c13e5..30fb3952 100644
--- a/drivers/net/softnic/rte_eth_softnic.c
+++ b/drivers/net/softnic/rte_eth_softnic.c
@@ -13,43 +13,51 @@
 #include <rte_kvargs.h>
 #include <rte_errno.h>
 #include <rte_ring.h>
-#include <rte_sched.h>
 #include <rte_tm_driver.h>
 
 #include "rte_eth_softnic.h"
 #include "rte_eth_softnic_internals.h"
 
-#define DEV_HARD(p)					\
-	(&rte_eth_devices[p->hard.port_id])
-
-#define PMD_PARAM_SOFT_TM					"soft_tm"
-#define PMD_PARAM_SOFT_TM_RATE				"soft_tm_rate"
-#define PMD_PARAM_SOFT_TM_NB_QUEUES			"soft_tm_nb_queues"
-#define PMD_PARAM_SOFT_TM_QSIZE0			"soft_tm_qsize0"
-#define PMD_PARAM_SOFT_TM_QSIZE1			"soft_tm_qsize1"
-#define PMD_PARAM_SOFT_TM_QSIZE2			"soft_tm_qsize2"
-#define PMD_PARAM_SOFT_TM_QSIZE3			"soft_tm_qsize3"
-#define PMD_PARAM_SOFT_TM_ENQ_BSZ			"soft_tm_enq_bsz"
-#define PMD_PARAM_SOFT_TM_DEQ_BSZ			"soft_tm_deq_bsz"
-
-#define PMD_PARAM_HARD_NAME					"hard_name"
-#define PMD_PARAM_HARD_TX_QUEUE_ID			"hard_tx_queue_id"
+#define PMD_PARAM_FIRMWARE                                 "firmware"
+#define PMD_PARAM_CONN_PORT                                "conn_port"
+#define PMD_PARAM_CPU_ID                                   "cpu_id"
+#define PMD_PARAM_TM_N_QUEUES                              "tm_n_queues"
+#define PMD_PARAM_TM_QSIZE0                                "tm_qsize0"
+#define PMD_PARAM_TM_QSIZE1                                "tm_qsize1"
+#define PMD_PARAM_TM_QSIZE2                                "tm_qsize2"
+#define PMD_PARAM_TM_QSIZE3                                "tm_qsize3"
 
 static const char *pmd_valid_args[] = {
-	PMD_PARAM_SOFT_TM,
-	PMD_PARAM_SOFT_TM_RATE,
-	PMD_PARAM_SOFT_TM_NB_QUEUES,
-	PMD_PARAM_SOFT_TM_QSIZE0,
-	PMD_PARAM_SOFT_TM_QSIZE1,
-	PMD_PARAM_SOFT_TM_QSIZE2,
-	PMD_PARAM_SOFT_TM_QSIZE3,
-	PMD_PARAM_SOFT_TM_ENQ_BSZ,
-	PMD_PARAM_SOFT_TM_DEQ_BSZ,
-	PMD_PARAM_HARD_NAME,
-	PMD_PARAM_HARD_TX_QUEUE_ID,
+	PMD_PARAM_FIRMWARE,
+	PMD_PARAM_CONN_PORT,
+	PMD_PARAM_CPU_ID,
+	PMD_PARAM_TM_N_QUEUES,
+	PMD_PARAM_TM_QSIZE0,
+	PMD_PARAM_TM_QSIZE1,
+	PMD_PARAM_TM_QSIZE2,
+	PMD_PARAM_TM_QSIZE3,
 	NULL
 };
 
+static const char welcome[] =
+	"\n"
+	"Welcome to Soft NIC!\n"
+	"\n";
+
+static const char prompt[] = "softnic> ";
+
+struct softnic_conn_params conn_params_default = {
+	.welcome = welcome,
+	.prompt = prompt,
+	.addr = "0.0.0.0",
+	.port = 0,
+	.buf_size = 1024 * 1024,
+	.msg_in_len_max = 1024,
+	.msg_out_len_max = 1024 * 1024,
+	.msg_handle = softnic_cli_process,
+	.msg_handle_arg = NULL,
+};
+
 static const struct rte_eth_dev_info pmd_dev_info = {
 	.min_rx_bufsize = 0,
 	.max_rx_pktlen = UINT32_MAX,
@@ -65,6 +73,7 @@ static const struct rte_eth_dev_info pmd_dev_info = {
 		.nb_min = 0,
 		.nb_align = 1,
 	},
+	.rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP,
 };
 
 static int pmd_softnic_logtype;
@@ -81,50 +90,36 @@ pmd_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
 }
 
 static int
-pmd_dev_configure(struct rte_eth_dev *dev)
+pmd_dev_configure(struct rte_eth_dev *dev __rte_unused)
 {
-	struct pmd_internals *p = dev->data->dev_private;
-	struct rte_eth_dev *hard_dev = DEV_HARD(p);
-
-	if (dev->data->nb_rx_queues > hard_dev->data->nb_rx_queues)
-		return -1;
-
-	if (p->params.hard.tx_queue_id >= hard_dev->data->nb_tx_queues)
-		return -1;
-
 	return 0;
 }
 
 static int
 pmd_rx_queue_setup(struct rte_eth_dev *dev,
 	uint16_t rx_queue_id,
-	uint16_t nb_rx_desc __rte_unused,
-	unsigned int socket_id,
+	uint16_t nb_rx_desc,
+	unsigned int socket_id __rte_unused,
 	const struct rte_eth_rxconf *rx_conf __rte_unused,
 	struct rte_mempool *mb_pool __rte_unused)
 {
+	char name[NAME_SIZE];
 	struct pmd_internals *p = dev->data->dev_private;
+	struct softnic_swq *swq;
 
-	if (p->params.soft.intrusive == 0) {
-		struct pmd_rx_queue *rxq;
-
-		rxq = rte_zmalloc_socket(p->params.soft.name,
-			sizeof(struct pmd_rx_queue), 0, socket_id);
-		if (rxq == NULL)
-			return -ENOMEM;
+	struct softnic_swq_params params = {
+		.size = nb_rx_desc,
+	};
 
-		rxq->hard.port_id = p->hard.port_id;
-		rxq->hard.rx_queue_id = rx_queue_id;
-		dev->data->rx_queues[rx_queue_id] = rxq;
-	} else {
-		struct rte_eth_dev *hard_dev = DEV_HARD(p);
-		void *rxq = hard_dev->data->rx_queues[rx_queue_id];
+	snprintf(name, sizeof(name), "RXQ%u", rx_queue_id);
 
-		if (rxq == NULL)
-			return -1;
+	swq = softnic_swq_create(p,
+		name,
+		&params);
+	if (swq == NULL)
+		return -1;
 
-		dev->data->rx_queues[rx_queue_id] = rxq;
-	}
+	dev->data->rx_queues[rx_queue_id] = swq->r;
 	return 0;
 }
 
@@ -132,21 +127,26 @@ static int
 pmd_tx_queue_setup(struct rte_eth_dev *dev,
 	uint16_t tx_queue_id,
 	uint16_t nb_tx_desc,
-	unsigned int socket_id,
+	unsigned int socket_id __rte_unused,
 	const struct rte_eth_txconf *tx_conf __rte_unused)
 {
-	uint32_t size = RTE_ETH_NAME_MAX_LEN + strlen("_txq") + 4;
-	char name[size];
-	struct rte_ring *r;
-
-	snprintf(name, sizeof(name), "%s_txq%04x",
-		dev->data->name, tx_queue_id);
-	r = rte_ring_create(name, nb_tx_desc, socket_id,
-		RING_F_SP_ENQ | RING_F_SC_DEQ);
-	if (r == NULL)
+	char name[NAME_SIZE];
+	struct pmd_internals *p = dev->data->dev_private;
+	struct softnic_swq *swq;
+
+	struct softnic_swq_params params = {
+		.size = nb_tx_desc,
+	};
+
+	snprintf(name, sizeof(name), "TXQ%u", tx_queue_id);
+
+	swq = softnic_swq_create(p,
+		name,
+		&params);
+	if (swq == NULL)
 		return -1;
 
-	dev->data->tx_queues[tx_queue_id] = r;
+	dev->data->tx_queues[tx_queue_id] = swq->r;
 	return 0;
 }
 
@@ -154,23 +154,19 @@ static int
 pmd_dev_start(struct rte_eth_dev *dev)
 {
 	struct pmd_internals *p = dev->data->dev_private;
+	int status;
 
-	if (tm_used(dev)) {
-		int status = tm_start(p);
-
-		if (status)
-			return status;
-	}
+	/* Firmware */
+	status = softnic_cli_script_process(p,
+		p->params.firmware,
+		conn_params_default.msg_in_len_max,
+		conn_params_default.msg_out_len_max);
+	if (status)
+		return status;
 
+	/* Link UP */
 	dev->data->dev_link.link_status = ETH_LINK_UP;
 
-	if (p->params.soft.intrusive) {
-		struct rte_eth_dev *hard_dev = DEV_HARD(p);
-
-		/* The hard_dev->rx_pkt_burst should be stable by now */
-		dev->rx_pkt_burst = hard_dev->rx_pkt_burst;
-	}
-
 	return 0;
 }
 
@@ -179,20 +175,27 @@ pmd_dev_stop(struct rte_eth_dev *dev)
 {
 	struct pmd_internals *p = dev->data->dev_private;
 
+	/* Link DOWN */
 	dev->data->dev_link.link_status = ETH_LINK_DOWN;
 
-	if (tm_used(dev))
-		tm_stop(p);
+	/* Firmware */
+	softnic_pipeline_disable_all(p);
+	softnic_pipeline_free(p);
+	softnic_table_action_profile_free(p);
+	softnic_port_in_action_profile_free(p);
+	softnic_tap_free(p);
+	softnic_tmgr_free(p);
+	softnic_link_free(p);
+	softnic_softnic_swq_free_keep_rxq_txq(p);
+	softnic_mempool_free(p);
+
+	tm_hierarchy_free(p);
 }
 
 static void
-pmd_dev_close(struct rte_eth_dev *dev)
+pmd_dev_close(struct rte_eth_dev *dev __rte_unused)
 {
-	uint32_t i;
-
-	/* TX queues */
-	for (i = 0; i < dev->data->nb_tx_queues; i++)
-		rte_ring_free((struct rte_ring *)dev->data->tx_queues[i]);
+	return;
 }
 
 static int
@@ -203,10 +206,9 @@ pmd_link_update(struct rte_eth_dev *dev __rte_unused,
 }
 
 static int
-pmd_tm_ops_get(struct rte_eth_dev *dev, void *arg)
+pmd_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
 {
-	*(const struct rte_tm_ops **)arg =
-		(tm_enabled(dev)) ? &pmd_tm_ops : NULL;
+	*(const struct rte_tm_ops **)arg = &pmd_tm_ops;
 
 	return 0;
 }
@@ -228,12 +230,10 @@ pmd_rx_pkt_burst(void *rxq,
 	struct rte_mbuf **rx_pkts,
 	uint16_t nb_pkts)
 {
-	struct pmd_rx_queue *rx_queue = rxq;
-
-	return rte_eth_rx_burst(rx_queue->hard.port_id,
-		rx_queue->hard.rx_queue_id,
-		rx_pkts,
-		nb_pkts);
+	return (uint16_t)rte_ring_sc_dequeue_burst(rxq,
+		(void **)rx_pkts,
+		nb_pkts,
+		NULL);
 }
 
 static uint16_t
@@ -241,239 +241,56 @@ pmd_tx_pkt_burst(void *txq,
 	struct rte_mbuf **tx_pkts,
 	uint16_t nb_pkts)
 {
-	return (uint16_t)rte_ring_enqueue_burst(txq,
+	return (uint16_t)rte_ring_sp_enqueue_burst(txq,
 		(void **)tx_pkts,
 		nb_pkts,
 		NULL);
 }
 
-static __rte_always_inline int
-run_default(struct rte_eth_dev *dev)
-{
-	struct pmd_internals *p = dev->data->dev_private;
-
-	/* Persistent context: Read Only (update not required) */
-	struct rte_mbuf **pkts = p->soft.def.pkts;
-	uint16_t nb_tx_queues = dev->data->nb_tx_queues;
-
-	/* Persistent context: Read - Write (update required) */
-	uint32_t txq_pos = p->soft.def.txq_pos;
-	uint32_t pkts_len = p->soft.def.pkts_len;
-	uint32_t flush_count = p->soft.def.flush_count;
-
-	/* Not part of the persistent context */
-	uint32_t pos;
-	uint16_t i;
-
-	/* Soft device TXQ read, Hard device TXQ write */
-	for (i = 0; i < nb_tx_queues; i++) {
-		struct rte_ring *txq = dev->data->tx_queues[txq_pos];
-
-		/* Read soft device TXQ burst to packet enqueue buffer */
-		pkts_len += rte_ring_sc_dequeue_burst(txq,
-			(void **)&pkts[pkts_len],
-			DEFAULT_BURST_SIZE,
-			NULL);
-
-		/* Increment soft device TXQ */
-		txq_pos++;
-		if (txq_pos >= nb_tx_queues)
-			txq_pos = 0;
-
-		/* Hard device TXQ write when complete burst is available */
-		if (pkts_len >= DEFAULT_BURST_SIZE) {
-			for (pos = 0; pos < pkts_len; )
-				pos += rte_eth_tx_burst(p->hard.port_id,
-					p->params.hard.tx_queue_id,
-					&pkts[pos],
-					(uint16_t)(pkts_len - pos));
-
-			pkts_len = 0;
-			flush_count = 0;
-			break;
-		}
-	}
-
-	if (flush_count >= FLUSH_COUNT_THRESHOLD) {
-		for (pos = 0; pos < pkts_len; )
-			pos += rte_eth_tx_burst(p->hard.port_id,
-				p->params.hard.tx_queue_id,
-				&pkts[pos],
-				(uint16_t)(pkts_len - pos));
-
-		pkts_len = 0;
-		flush_count = 0;
-	}
-
-	p->soft.def.txq_pos = txq_pos;
-	p->soft.def.pkts_len = pkts_len;
-	p->soft.def.flush_count = flush_count + 1;
-
-	return 0;
-}
-
-static __rte_always_inline int
-run_tm(struct rte_eth_dev *dev)
-{
-	struct pmd_internals *p = dev->data->dev_private;
-
-	/* Persistent context: Read Only (update not required) */
-	struct rte_sched_port *sched = p->soft.tm.sched;
-	struct rte_mbuf **pkts_enq = p->soft.tm.pkts_enq;
-	struct rte_mbuf **pkts_deq = p->soft.tm.pkts_deq;
-	uint32_t enq_bsz = p->params.soft.tm.enq_bsz;
-	uint32_t deq_bsz = p->params.soft.tm.deq_bsz;
-	uint16_t nb_tx_queues = dev->data->nb_tx_queues;
-
-	/* Persistent context: Read - Write (update required) */
-	uint32_t txq_pos = p->soft.tm.txq_pos;
-	uint32_t pkts_enq_len = p->soft.tm.pkts_enq_len;
-	uint32_t flush_count = p->soft.tm.flush_count;
-
-	/* Not part of the persistent context */
-	uint32_t pkts_deq_len, pos;
-	uint16_t i;
-
-	/* Soft device TXQ read, TM enqueue */
-	for (i = 0; i < nb_tx_queues; i++) {
-		struct rte_ring *txq = dev->data->tx_queues[txq_pos];
-
-		/* Read TXQ burst to packet enqueue buffer */
-		pkts_enq_len += rte_ring_sc_dequeue_burst(txq,
-			(void **)&pkts_enq[pkts_enq_len],
-			enq_bsz,
-			NULL);
-
-		/* Increment TXQ */
-		txq_pos++;
-		if (txq_pos >= nb_tx_queues)
-			txq_pos = 0;
-
-		/* TM enqueue when complete burst is available */
-		if (pkts_enq_len >= enq_bsz) {
-			rte_sched_port_enqueue(sched, pkts_enq, pkts_enq_len);
-
-			pkts_enq_len = 0;
-			flush_count = 0;
-			break;
-		}
-	}
-
-	if (flush_count >= FLUSH_COUNT_THRESHOLD) {
-		if (pkts_enq_len)
-			rte_sched_port_enqueue(sched, pkts_enq, pkts_enq_len);
-
-		pkts_enq_len = 0;
-		flush_count = 0;
-	}
-
-	p->soft.tm.txq_pos = txq_pos;
-	p->soft.tm.pkts_enq_len = pkts_enq_len;
-	p->soft.tm.flush_count = flush_count + 1;
-
-	/* TM dequeue, Hard device TXQ write */
-	pkts_deq_len = rte_sched_port_dequeue(sched, pkts_deq, deq_bsz);
-
-	for (pos = 0; pos < pkts_deq_len; )
-		pos += rte_eth_tx_burst(p->hard.port_id,
-			p->params.hard.tx_queue_id,
-			&pkts_deq[pos],
-			(uint16_t)(pkts_deq_len - pos));
-
-	return 0;
-}
-
-int
-rte_pmd_softnic_run(uint16_t port_id)
-{
-	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
-
-#ifdef RTE_LIBRTE_ETHDEV_DEBUG
-	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
-#endif
-
-	return (tm_used(dev)) ? run_tm(dev) : run_default(dev);
-}
-
-static struct ether_addr eth_addr = { .addr_bytes = {0} };
-
-static uint32_t
-eth_dev_speed_max_mbps(uint32_t speed_capa)
-{
-	uint32_t rate_mbps[32] = {
-		ETH_SPEED_NUM_NONE,
-		ETH_SPEED_NUM_10M,
-		ETH_SPEED_NUM_10M,
-		ETH_SPEED_NUM_100M,
-		ETH_SPEED_NUM_100M,
-		ETH_SPEED_NUM_1G,
-		ETH_SPEED_NUM_2_5G,
-		ETH_SPEED_NUM_5G,
-		ETH_SPEED_NUM_10G,
-		ETH_SPEED_NUM_20G,
-		ETH_SPEED_NUM_25G,
-		ETH_SPEED_NUM_40G,
-		ETH_SPEED_NUM_50G,
-		ETH_SPEED_NUM_56G,
-		ETH_SPEED_NUM_100G,
-	};
-
-	uint32_t pos = (speed_capa) ? (31 - __builtin_clz(speed_capa)) : 0;
-	return rate_mbps[pos];
-}
-
-static int
-default_init(struct pmd_internals *p,
-	struct pmd_params *params,
-	int numa_node)
-{
-	p->soft.def.pkts = rte_zmalloc_socket(params->soft.name,
-		2 * DEFAULT_BURST_SIZE * sizeof(struct rte_mbuf *),
-		0,
-		numa_node);
-
-	if (p->soft.def.pkts == NULL)
-		return -ENOMEM;
-
-	return 0;
-}
-
-static void
-default_free(struct pmd_internals *p)
-{
-	rte_free(p->soft.def.pkts);
-}
-
 static void *
-pmd_init(struct pmd_params *params, int numa_node)
+pmd_init(struct pmd_params *params)
 {
 	struct pmd_internals *p;
 	int status;
 
-	p = rte_zmalloc_socket(params->soft.name,
+	p = rte_zmalloc_socket(params->name,
 		sizeof(struct pmd_internals),
 		0,
-		numa_node);
+		params->cpu_id);
 	if (p == NULL)
 		return NULL;
 
+	/* Params */
 	memcpy(&p->params, params, sizeof(p->params));
-	rte_eth_dev_get_port_by_name(params->hard.name, &p->hard.port_id);
 
-	/* Default */
-	status = default_init(p, params, numa_node);
+	/* Resources */
+	tm_hierarchy_init(p);
+
+	softnic_mempool_init(p);
+	softnic_swq_init(p);
+	softnic_link_init(p);
+	softnic_tmgr_init(p);
+	softnic_tap_init(p);
+	softnic_port_in_action_profile_init(p);
+	softnic_table_action_profile_init(p);
+	softnic_pipeline_init(p);
+
+	status = softnic_thread_init(p);
 	if (status) {
-		free(p->params.hard.name);
 		rte_free(p);
 		return NULL;
 	}
 
-	/* Traffic Management (TM)*/
-	if (params->soft.flags & PMD_FEATURE_TM) {
-		status = tm_init(p, params, numa_node);
-		if (status) {
-			default_free(p);
-			free(p->params.hard.name);
+	if (params->conn_port) {
+		struct softnic_conn_params conn_params;
+
+		memcpy(&conn_params, &conn_params_default, sizeof(conn_params));
+		conn_params.port = p->params.conn_port;
+		conn_params.msg_handle_arg = p;
+
+		p->conn = softnic_conn_init(&conn_params);
+		if (p->conn == NULL) {
+			softnic_thread_free(p);
 			rte_free(p);
 			return NULL;
 		}
@@ -485,57 +302,62 @@ pmd_init(struct pmd_params *params, int numa_node)
 static void
 pmd_free(struct pmd_internals *p)
 {
-	if (p->params.soft.flags & PMD_FEATURE_TM)
-		tm_free(p);
+	if (p == NULL)
+		return;
 
-	default_free(p);
+	if (p->params.conn_port)
+		softnic_conn_free(p->conn);
+
+	softnic_thread_free(p);
+	softnic_pipeline_free(p);
+	softnic_table_action_profile_free(p);
+	softnic_port_in_action_profile_free(p);
+	softnic_tap_free(p);
+	softnic_tmgr_free(p);
+	softnic_link_free(p);
+	softnic_swq_free(p);
+	softnic_mempool_free(p);
+
+	tm_hierarchy_free(p);
 
-	free(p->params.hard.name);
 	rte_free(p);
 }
 
+static struct ether_addr eth_addr = {
+	.addr_bytes = {0},
+};
+
 static int
 pmd_ethdev_register(struct rte_vdev_device *vdev,
 	struct pmd_params *params,
 	void *dev_private)
 {
-	struct rte_eth_dev_info hard_info;
-	struct rte_eth_dev *soft_dev;
-	uint32_t hard_speed;
-	int numa_node;
-	uint16_t hard_port_id;
-
-	rte_eth_dev_get_port_by_name(params->hard.name, &hard_port_id);
-	rte_eth_dev_info_get(hard_port_id, &hard_info);
-	hard_speed = eth_dev_speed_max_mbps(hard_info.speed_capa);
-	numa_node = rte_eth_dev_socket_id(hard_port_id);
+	struct rte_eth_dev *dev;
 
 	/* Ethdev entry allocation */
-	soft_dev = rte_eth_dev_allocate(params->soft.name);
-	if (!soft_dev)
+	dev = rte_eth_dev_allocate(params->name);
+	if (!dev)
 		return -ENOMEM;
 
 	/* dev */
-	soft_dev->rx_pkt_burst = (params->soft.intrusive) ?
-		NULL : /* set up later */
-		pmd_rx_pkt_burst;
-	soft_dev->tx_pkt_burst = pmd_tx_pkt_burst;
-	soft_dev->tx_pkt_prepare = NULL;
-	soft_dev->dev_ops = &pmd_ops;
-	soft_dev->device = &vdev->device;
+	dev->rx_pkt_burst = pmd_rx_pkt_burst;
+	dev->tx_pkt_burst = pmd_tx_pkt_burst;
+	dev->tx_pkt_prepare = NULL;
+	dev->dev_ops = &pmd_ops;
+	dev->device = &vdev->device;
 
 	/* dev->data */
-	soft_dev->data->dev_private = dev_private;
-	soft_dev->data->dev_link.link_speed = hard_speed;
-	soft_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
-	soft_dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
-	soft_dev->data->dev_link.link_status = ETH_LINK_DOWN;
-	soft_dev->data->mac_addrs = &eth_addr;
-	soft_dev->data->promiscuous = 1;
-	soft_dev->data->kdrv = RTE_KDRV_NONE;
-	soft_dev->data->numa_node = numa_node;
-
-	rte_eth_dev_probing_finish(soft_dev);
+	dev->data->dev_private = dev_private;
+	dev->data->dev_link.link_speed = ETH_SPEED_NUM_100G;
+	dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
+	dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
+	dev->data->dev_link.link_status = ETH_LINK_DOWN;
+	dev->data->mac_addrs = &eth_addr;
+	dev->data->promiscuous = 1;
+	dev->data->kdrv = RTE_KDRV_NONE;
+	dev->data->numa_node = params->cpu_id;
+
+	rte_eth_dev_probing_finish(dev);
 
 	return 0;
 }
@@ -566,10 +388,21 @@ get_uint32(const char *key __rte_unused, const char *value, void *extra_args)
 }
 
 static int
-pmd_parse_args(struct pmd_params *p, const char *name, const char *params)
+get_uint16(const char *key __rte_unused, const char *value, void *extra_args)
+{
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	*(uint16_t *)extra_args = strtoull(value, NULL, 0);
+
+	return 0;
+}
+
+static int
+pmd_parse_args(struct pmd_params *p, const char *params)
 {
 	struct rte_kvargs *kvlist;
-	int i, ret;
+	int ret = 0;
 
 	kvlist = rte_kvargs_parse(params, pmd_valid_args);
 	if (kvlist == NULL)
@@ -577,141 +410,71 @@ pmd_parse_args(struct pmd_params *p, const char *name, const char *params)
 
 	/* Set default values */
 	memset(p, 0, sizeof(*p));
-	p->soft.name = name;
-	p->soft.intrusive = INTRUSIVE;
-	p->soft.tm.rate = 0;
-	p->soft.tm.nb_queues = SOFTNIC_SOFT_TM_NB_QUEUES;
-	for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
-		p->soft.tm.qsize[i] = SOFTNIC_SOFT_TM_QUEUE_SIZE;
-	p->soft.tm.enq_bsz = SOFTNIC_SOFT_TM_ENQ_BSZ;
-	p->soft.tm.deq_bsz = SOFTNIC_SOFT_TM_DEQ_BSZ;
-	p->hard.tx_queue_id = SOFTNIC_HARD_TX_QUEUE_ID;
-
-	/* SOFT: TM (optional) */
-	if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM) == 1) {
-		char *s;
-
-		ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM,
-			&get_string, &s);
+	p->firmware = SOFTNIC_FIRMWARE;
+	p->cpu_id = SOFTNIC_CPU_ID;
+	p->tm.n_queues = SOFTNIC_TM_N_QUEUES;
+	p->tm.qsize[0] = SOFTNIC_TM_QUEUE_SIZE;
+	p->tm.qsize[1] = SOFTNIC_TM_QUEUE_SIZE;
+	p->tm.qsize[2] = SOFTNIC_TM_QUEUE_SIZE;
+	p->tm.qsize[3] = SOFTNIC_TM_QUEUE_SIZE;
+
+	/* Firmware script (optional) */
+	if (rte_kvargs_count(kvlist, PMD_PARAM_FIRMWARE) == 1) {
+		ret = rte_kvargs_process(kvlist, PMD_PARAM_FIRMWARE,
+			&get_string, &p->firmware);
 		if (ret < 0)
 			goto out_free;
-
-		if (strcmp(s, "on") == 0)
-			p->soft.flags |= PMD_FEATURE_TM;
-		else if (strcmp(s, "off") == 0)
-			p->soft.flags &= ~PMD_FEATURE_TM;
-		else
-			ret = -EINVAL;
-
-		free(s);
-		if (ret)
-			goto out_free;
 	}
 
-	/* SOFT: TM rate (measured in bytes/second) (optional) */
-	if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_RATE) == 1) {
-		ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_RATE,
-			&get_uint32, &p->soft.tm.rate);
+	/* Connection listening port (optional) */
+	if (rte_kvargs_count(kvlist, PMD_PARAM_CONN_PORT) == 1) {
+		ret = rte_kvargs_process(kvlist, PMD_PARAM_CONN_PORT,
+			&get_uint16, &p->conn_port);
 		if (ret < 0)
 			goto out_free;
-
-		p->soft.flags |= PMD_FEATURE_TM;
 	}
 
-	/* SOFT: TM number of queues (optional) */
-	if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_NB_QUEUES) == 1) {
-		ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_NB_QUEUES,
-			&get_uint32, &p->soft.tm.nb_queues);
+	/* CPU ID (optional) */
+	if (rte_kvargs_count(kvlist, PMD_PARAM_CPU_ID) == 1) {
+		ret = rte_kvargs_process(kvlist, PMD_PARAM_CPU_ID,
+			&get_uint32, &p->cpu_id);
 		if (ret < 0)
 			goto out_free;
-
-		p->soft.flags |= PMD_FEATURE_TM;
 	}
 
-	/* SOFT: TM queue size 0 .. 3 (optional) */
-	if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_QSIZE0) == 1) {
-		uint32_t qsize;
-
-		ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_QSIZE0,
-			&get_uint32, &qsize);
+	/* TM number of queues (optional) */
+	if (rte_kvargs_count(kvlist, PMD_PARAM_TM_N_QUEUES) == 1) {
+		ret = rte_kvargs_process(kvlist, PMD_PARAM_TM_N_QUEUES,
+			&get_uint32, &p->tm.n_queues);
 		if (ret < 0)
 			goto out_free;
-
-		p->soft.tm.qsize[0] = (uint16_t)qsize;
-		p->soft.flags |= PMD_FEATURE_TM;
 	}
 
-	if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_QSIZE1) == 1) {
-		uint32_t qsize;
-
-		ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_QSIZE1,
-			&get_uint32, &qsize);
+	/* TM queue size 0 .. 3 (optional) */
+	if (rte_kvargs_count(kvlist, PMD_PARAM_TM_QSIZE0) == 1) {
+		ret = rte_kvargs_process(kvlist, PMD_PARAM_TM_QSIZE0,
+			&get_uint32, &p->tm.qsize[0]);
 		if (ret < 0)
 			goto out_free;
-
-		p->soft.tm.qsize[1] = (uint16_t)qsize;
-		p->soft.flags |= PMD_FEATURE_TM;
 	}
 
-	if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_QSIZE2) == 1) {
-		uint32_t qsize;
-
-		ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_QSIZE2,
-			&get_uint32, &qsize);
+	if (rte_kvargs_count(kvlist, PMD_PARAM_TM_QSIZE1) == 1) {
+		ret = rte_kvargs_process(kvlist, PMD_PARAM_TM_QSIZE1,
+			&get_uint32, &p->tm.qsize[1]);
 		if (ret < 0)
 			goto out_free;
-
-		p->soft.tm.qsize[2] = (uint16_t)qsize;
-		p->soft.flags |= PMD_FEATURE_TM;
 	}
 
-	if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_QSIZE3) == 1) {
-		uint32_t qsize;
-
-		ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_QSIZE3,
-			&get_uint32, &qsize);
+	if (rte_kvargs_count(kvlist, PMD_PARAM_TM_QSIZE2) == 1) {
+		ret = rte_kvargs_process(kvlist, PMD_PARAM_TM_QSIZE2,
+			&get_uint32, &p->tm.qsize[2]);
 		if (ret < 0)
 			goto out_free;
-
-		p->soft.tm.qsize[3] = (uint16_t)qsize;
-		p->soft.flags |= PMD_FEATURE_TM;
 	}
 
-	/* SOFT: TM enqueue burst size (optional) */
-	if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_ENQ_BSZ) == 1) {
-		ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_ENQ_BSZ,
-			&get_uint32, &p->soft.tm.enq_bsz);
-		if (ret < 0)
-			goto out_free;
-
-		p->soft.flags |= PMD_FEATURE_TM;
-	}
-
-	/* SOFT: TM dequeue burst size (optional) */
-	if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_DEQ_BSZ) == 1) {
-		ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_DEQ_BSZ,
-			&get_uint32, &p->soft.tm.deq_bsz);
-		if (ret < 0)
-			goto out_free;
-
-		p->soft.flags |= PMD_FEATURE_TM;
-	}
-
-	/* HARD: name (mandatory) */
-	if (rte_kvargs_count(kvlist, PMD_PARAM_HARD_NAME) == 1) {
-		ret = rte_kvargs_process(kvlist, PMD_PARAM_HARD_NAME,
-			&get_string, &p->hard.name);
-		if (ret < 0)
-			goto out_free;
-	} else {
-		ret = -EINVAL;
-		goto out_free;
-	}
-
-	/* HARD: tx_queue_id (optional) */
-	if (rte_kvargs_count(kvlist, PMD_PARAM_HARD_TX_QUEUE_ID) == 1) {
-		ret = rte_kvargs_process(kvlist, PMD_PARAM_HARD_TX_QUEUE_ID,
-			&get_uint32, &p->hard.tx_queue_id);
+	if (rte_kvargs_count(kvlist, PMD_PARAM_TM_QSIZE3) == 1) {
+		ret = rte_kvargs_process(kvlist, PMD_PARAM_TM_QSIZE3,
+			&get_uint32, &p->tm.qsize[3]);
 		if (ret < 0)
 			goto out_free;
 	}
@@ -726,68 +489,31 @@ pmd_probe(struct rte_vdev_device *vdev)
 {
 	struct pmd_params p;
 	const char *params;
-	int status;
+	int status = 0;
 
-	struct rte_eth_dev_info hard_info;
-	uint32_t hard_speed;
-	uint16_t hard_port_id;
-	int numa_node;
 	void *dev_private;
-	struct rte_eth_dev *eth_dev;
 	const char *name = rte_vdev_device_name(vdev);
 
 	PMD_LOG(INFO, "Probing device \"%s\"", name);
 
 	/* Parse input arguments */
 	params = rte_vdev_device_args(vdev);
-
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
-	    strlen(params) == 0) {
-		eth_dev = rte_eth_dev_attach_secondary(name);
-		if (!eth_dev) {
-			PMD_LOG(ERR, "Failed to probe %s", name);
-			return -1;
-		}
-		/* TODO: request info from primary to set up Rx and Tx */
-		eth_dev->dev_ops = &pmd_ops;
-		rte_eth_dev_probing_finish(eth_dev);
-		return 0;
-	}
-
 	if (!params)
 		return -EINVAL;
 
-	status = pmd_parse_args(&p, rte_vdev_device_name(vdev), params);
+	status = pmd_parse_args(&p, params);
 	if (status)
 		return status;
 
-	/* Check input arguments */
-	if (rte_eth_dev_get_port_by_name(p.hard.name, &hard_port_id))
-		return -EINVAL;
-
-	rte_eth_dev_info_get(hard_port_id, &hard_info);
-	hard_speed = eth_dev_speed_max_mbps(hard_info.speed_capa);
-	numa_node = rte_eth_dev_socket_id(hard_port_id);
-
-	if (p.hard.tx_queue_id >= hard_info.max_tx_queues)
-		return -EINVAL;
-
-	if (p.soft.flags & PMD_FEATURE_TM) {
-		status = tm_params_check(&p, hard_speed);
-
-		if (status)
-			return status;
-	}
+	p.name = name;
 
 	/* Allocate and initialize soft ethdev private data */
-	dev_private = pmd_init(&p, numa_node);
+	dev_private = pmd_init(&p);
 	if (dev_private == NULL)
 		return -ENOMEM;
 
 	/* Register soft ethdev */
-	PMD_LOG(INFO,
-		"Creating soft ethdev \"%s\" for hard ethdev \"%s\"",
-		p.soft.name, p.hard.name);
+	PMD_LOG(INFO, "Creating soft ethdev \"%s\"", p.name);
 
 	status = pmd_ethdev_register(vdev, &p, dev_private);
 	if (status) {
@@ -807,8 +533,7 @@ pmd_remove(struct rte_vdev_device *vdev)
 	if (!vdev)
 		return -EINVAL;
 
-	PMD_LOG(INFO, "Removing device \"%s\"",
-		rte_vdev_device_name(vdev));
+	PMD_LOG(INFO, "Removing device \"%s\"", rte_vdev_device_name(vdev));
 
 	/* Find the ethdev entry */
 	dev = rte_eth_dev_allocated(rte_vdev_device_name(vdev));
@@ -817,9 +542,9 @@ pmd_remove(struct rte_vdev_device *vdev)
 	p = dev->data->dev_private;
 
 	/* Free device data structures*/
-	pmd_free(p);
 	rte_free(dev->data);
 	rte_eth_dev_release_port(dev);
+	pmd_free(p);
 
 	return 0;
 }
@@ -831,23 +556,39 @@ static struct rte_vdev_driver pmd_softnic_drv = {
 
 RTE_PMD_REGISTER_VDEV(net_softnic, pmd_softnic_drv);
 RTE_PMD_REGISTER_PARAM_STRING(net_softnic,
-	PMD_PARAM_SOFT_TM	 "=on|off "
-	PMD_PARAM_SOFT_TM_RATE "=<int> "
-	PMD_PARAM_SOFT_TM_NB_QUEUES "=<int> "
-	PMD_PARAM_SOFT_TM_QSIZE0 "=<int> "
-	PMD_PARAM_SOFT_TM_QSIZE1 "=<int> "
-	PMD_PARAM_SOFT_TM_QSIZE2 "=<int> "
-	PMD_PARAM_SOFT_TM_QSIZE3 "=<int> "
-	PMD_PARAM_SOFT_TM_ENQ_BSZ "=<int> "
-	PMD_PARAM_SOFT_TM_DEQ_BSZ "=<int> "
-	PMD_PARAM_HARD_NAME "=<string> "
-	PMD_PARAM_HARD_TX_QUEUE_ID "=<int>");
-
-RTE_INIT(pmd_softnic_init_log);
-static void
-pmd_softnic_init_log(void)
+	PMD_PARAM_FIRMWARE "=<string> "
+	PMD_PARAM_CONN_PORT "=<uint16> "
+	PMD_PARAM_CPU_ID "=<uint32> "
+	PMD_PARAM_TM_N_QUEUES "=<uint32> "
+	PMD_PARAM_TM_QSIZE0 "=<uint32> "
+	PMD_PARAM_TM_QSIZE1 "=<uint32> "
+	PMD_PARAM_TM_QSIZE2 "=<uint32> "
+	PMD_PARAM_TM_QSIZE3 "=<uint32>"
+);
+
+
+RTE_INIT(pmd_softnic_init_log)
 {
 	pmd_softnic_logtype = rte_log_register("pmd.net.softnic");
 	if (pmd_softnic_logtype >= 0)
 		rte_log_set_level(pmd_softnic_logtype, RTE_LOG_NOTICE);
 }
+
+int
+rte_pmd_softnic_manage(uint16_t port_id)
+{
+	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
+	struct pmd_internals *softnic;
+
+#ifdef RTE_LIBRTE_ETHDEV_DEBUG
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
+#endif
+
+	softnic = dev->data->dev_private;
+
+	softnic_conn_poll_for_conn(softnic->conn);
+
+	softnic_conn_poll_for_msg(softnic->conn);
+
+	return 0;
+}
diff --git a/drivers/net/softnic/rte_eth_softnic.h b/drivers/net/softnic/rte_eth_softnic.h
index 9a2c7ba9..048dfe6b 100644
--- a/drivers/net/softnic/rte_eth_softnic.h
+++ b/drivers/net/softnic/rte_eth_softnic.h
@@ -11,42 +11,53 @@
 extern "C" {
 #endif
 
-#ifndef SOFTNIC_SOFT_TM_NB_QUEUES
-#define SOFTNIC_SOFT_TM_NB_QUEUES			65536
+/** Firmware. */
+#ifndef SOFTNIC_FIRMWARE
+#define SOFTNIC_FIRMWARE                                   "firmware.cli"
 #endif
 
-#ifndef SOFTNIC_SOFT_TM_QUEUE_SIZE
-#define SOFTNIC_SOFT_TM_QUEUE_SIZE			64
+/** TCP connection port (0 = no connectivity). */
+#ifndef SOFTNIC_CONN_PORT
+#define SOFTNIC_CONN_PORT                                  0
 #endif
 
-#ifndef SOFTNIC_SOFT_TM_ENQ_BSZ
-#define SOFTNIC_SOFT_TM_ENQ_BSZ				32
+/** NUMA node ID. */
+#ifndef SOFTNIC_CPU_ID
+#define SOFTNIC_CPU_ID                                     0
 #endif
 
-#ifndef SOFTNIC_SOFT_TM_DEQ_BSZ
-#define SOFTNIC_SOFT_TM_DEQ_BSZ				24
+/** Traffic Manager: Number of scheduler queues. */
+#ifndef SOFTNIC_TM_N_QUEUES
+#define SOFTNIC_TM_N_QUEUES                                (64 * 1024)
 #endif
 
-#ifndef SOFTNIC_HARD_TX_QUEUE_ID
-#define SOFTNIC_HARD_TX_QUEUE_ID			0
+/** Traffic Manager: Scheduler queue size (per traffic class). */
+#ifndef SOFTNIC_TM_QUEUE_SIZE
+#define SOFTNIC_TM_QUEUE_SIZE                              64
 #endif
 
 /**
- * Run the traffic management function on the softnic device
+ * Soft NIC run.
  *
- * This function read the packets from the softnic input queues, insert into
- * QoS scheduler queues based on mbuf sched field value and transmit the
- * scheduled packets out through the hard device interface.
- *
- * @param portid
- *    port id of the soft device.
+ * @param port_id
+ *    Port ID of the Soft NIC device.
  * @return
- *    zero.
+ *    Zero on success, error code otherwise.
  */
-
 int
 rte_pmd_softnic_run(uint16_t port_id);
 
+/**
+ * Soft NIC manage.
+ *
+ * @param port_id
+ *    Port ID of the Soft NIC device.
+ * @return
+ *    Zero on success, error code otherwise.
+ */
+int __rte_experimental
+rte_pmd_softnic_manage(uint16_t port_id);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/drivers/net/softnic/rte_eth_softnic_action.c b/drivers/net/softnic/rte_eth_softnic_action.c
new file mode 100644
index 00000000..c25f4dd9
--- /dev/null
+++ b/drivers/net/softnic/rte_eth_softnic_action.c
@@ -0,0 +1,389 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_string_fns.h>
+
+#include "hash_func.h"
+#include "rte_eth_softnic_internals.h"
+
+/**
+ * Input port
+ */
+int
+softnic_port_in_action_profile_init(struct pmd_internals *p)
+{
+	TAILQ_INIT(&p->port_in_action_profile_list);
+
+	return 0;
+}
+
+void
+softnic_port_in_action_profile_free(struct pmd_internals *p)
+{
+	for ( ; ; ) {
+		struct softnic_port_in_action_profile *profile;
+
+		profile = TAILQ_FIRST(&p->port_in_action_profile_list);
+		if (profile == NULL)
+			break;
+
+		TAILQ_REMOVE(&p->port_in_action_profile_list, profile, node);
+		free(profile);
+	}
+}
+
+struct softnic_port_in_action_profile *
+softnic_port_in_action_profile_find(struct pmd_internals *p,
+	const char *name)
+{
+	struct softnic_port_in_action_profile *profile;
+
+	if (name == NULL)
+		return NULL;
+
+	TAILQ_FOREACH(profile, &p->port_in_action_profile_list, node)
+		if (strcmp(profile->name, name) == 0)
+			return profile;
+
+	return NULL;
+}
+
+struct softnic_port_in_action_profile *
+softnic_port_in_action_profile_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_port_in_action_profile_params *params)
+{
+	struct softnic_port_in_action_profile *profile;
+	struct rte_port_in_action_profile *ap;
+	int status;
+
+	/* Check input params */
+	if (name == NULL ||
+		softnic_port_in_action_profile_find(p, name) ||
+		params == NULL)
+		return NULL;
+
+	if ((params->action_mask & (1LLU << RTE_PORT_IN_ACTION_LB)) &&
+		params->lb.f_hash == NULL) {
+		switch (params->lb.key_size) {
+		case  8:
+			params->lb.f_hash = hash_default_key8;
+			break;
+
+		case 16:
+			params->lb.f_hash = hash_default_key16;
+			break;
+
+		case 24:
+			params->lb.f_hash = hash_default_key24;
+			break;
+
+		case 32:
+			params->lb.f_hash = hash_default_key32;
+			break;
+
+		case 40:
+			params->lb.f_hash = hash_default_key40;
+			break;
+
+		case 48:
+			params->lb.f_hash = hash_default_key48;
+			break;
+
+		case 56:
+			params->lb.f_hash = hash_default_key56;
+			break;
+
+		case 64:
+			params->lb.f_hash = hash_default_key64;
+			break;
+
+		default:
+			return NULL;
+		}
+
+		params->lb.seed = 0;
+	}
+
+	/* Resource */
+	ap = rte_port_in_action_profile_create(0);
+	if (ap == NULL)
+		return NULL;
+
+	if (params->action_mask & (1LLU << RTE_PORT_IN_ACTION_FLTR)) {
+		status = rte_port_in_action_profile_action_register(ap,
+			RTE_PORT_IN_ACTION_FLTR,
+			&params->fltr);
+
+		if (status) {
+			rte_port_in_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_PORT_IN_ACTION_LB)) {
+		status = rte_port_in_action_profile_action_register(ap,
+			RTE_PORT_IN_ACTION_LB,
+			&params->lb);
+
+		if (status) {
+			rte_port_in_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	status = rte_port_in_action_profile_freeze(ap);
+	if (status) {
+		rte_port_in_action_profile_free(ap);
+		return NULL;
+	}
+
+	/* Node allocation */
+	profile = calloc(1, sizeof(struct softnic_port_in_action_profile));
+	if (profile == NULL) {
+		rte_port_in_action_profile_free(ap);
+		return NULL;
+	}
+
+	/* Node fill in */
+	strlcpy(profile->name, name, sizeof(profile->name));
+	memcpy(&profile->params, params, sizeof(*params));
+	profile->ap = ap;
+
+	/* Node add to list */
+	TAILQ_INSERT_TAIL(&p->port_in_action_profile_list, profile, node);
+
+	return profile;
+}
+
+/**
+ * Table
+ */
+int
+softnic_table_action_profile_init(struct pmd_internals *p)
+{
+	TAILQ_INIT(&p->table_action_profile_list);
+
+	return 0;
+}
+
+void
+softnic_table_action_profile_free(struct pmd_internals *p)
+{
+	for ( ; ; ) {
+		struct softnic_table_action_profile *profile;
+
+		profile = TAILQ_FIRST(&p->table_action_profile_list);
+		if (profile == NULL)
+			break;
+
+		TAILQ_REMOVE(&p->table_action_profile_list, profile, node);
+		free(profile);
+	}
+}
+
+struct softnic_table_action_profile *
+softnic_table_action_profile_find(struct pmd_internals *p,
+	const char *name)
+{
+	struct softnic_table_action_profile *profile;
+
+	if (name == NULL)
+		return NULL;
+
+	TAILQ_FOREACH(profile, &p->table_action_profile_list, node)
+		if (strcmp(profile->name, name) == 0)
+			return profile;
+
+	return NULL;
+}
+
+struct softnic_table_action_profile *
+softnic_table_action_profile_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_table_action_profile_params *params)
+{
+	struct softnic_table_action_profile *profile;
+	struct rte_table_action_profile *ap;
+	int status;
+
+	/* Check input params */
+	if (name == NULL ||
+		softnic_table_action_profile_find(p, name) ||
+		params == NULL ||
+		((params->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) == 0))
+		return NULL;
+
+	if ((params->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) &&
+		params->lb.f_hash == NULL) {
+		switch (params->lb.key_size) {
+		case 8:
+			params->lb.f_hash = hash_default_key8;
+			break;
+
+		case 16:
+			params->lb.f_hash = hash_default_key16;
+			break;
+
+		case 24:
+			params->lb.f_hash = hash_default_key24;
+			break;
+
+		case 32:
+			params->lb.f_hash = hash_default_key32;
+			break;
+
+		case 40:
+			params->lb.f_hash = hash_default_key40;
+			break;
+
+		case 48:
+			params->lb.f_hash = hash_default_key48;
+			break;
+
+		case 56:
+			params->lb.f_hash = hash_default_key56;
+			break;
+
+		case 64:
+			params->lb.f_hash = hash_default_key64;
+			break;
+
+		default:
+			return NULL;
+		}
+
+		params->lb.seed = 0;
+	}
+
+	/* Resource */
+	ap = rte_table_action_profile_create(&params->common);
+	if (ap == NULL)
+		return NULL;
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_FWD,
+			NULL);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_LB,
+			&params->lb);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_MTR,
+			&params->mtr);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_TM,
+			&params->tm);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_ENCAP,
+			&params->encap);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_NAT,
+			&params->nat);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_TTL,
+			&params->ttl);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_STATS,
+			&params->stats);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_TIME,
+			NULL);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	status = rte_table_action_profile_freeze(ap);
+	if (status) {
+		rte_table_action_profile_free(ap);
+		return NULL;
+	}
+
+	/* Node allocation */
+	profile = calloc(1, sizeof(struct softnic_table_action_profile));
+	if (profile == NULL) {
+		rte_table_action_profile_free(ap);
+		return NULL;
+	}
+
+	/* Node fill in */
+	strlcpy(profile->name, name, sizeof(profile->name));
+	memcpy(&profile->params, params, sizeof(*params));
+	profile->ap = ap;
+
+	/* Node add to list */
+	TAILQ_INSERT_TAIL(&p->table_action_profile_list, profile, node);
+
+	return profile;
+}
diff --git a/drivers/net/softnic/rte_eth_softnic_cli.c b/drivers/net/softnic/rte_eth_softnic_cli.c
new file mode 100644
index 00000000..0c7448cc
--- /dev/null
+++ b/drivers/net/softnic/rte_eth_softnic_cli.c
@@ -0,0 +1,5259 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_cycles.h>
+
+#include "rte_eth_softnic_internals.h"
+#include "parser.h"
+
+#ifndef CMD_MAX_TOKENS
+#define CMD_MAX_TOKENS     256
+#endif
+
+#define MSG_OUT_OF_MEMORY   "Not enough memory.\n"
+#define MSG_CMD_UNKNOWN     "Unknown command \"%s\".\n"
+#define MSG_CMD_UNIMPLEM    "Command \"%s\" not implemented.\n"
+#define MSG_ARG_NOT_ENOUGH  "Not enough arguments for command \"%s\".\n"
+#define MSG_ARG_TOO_MANY    "Too many arguments for command \"%s\".\n"
+#define MSG_ARG_MISMATCH    "Wrong number of arguments for command \"%s\".\n"
+#define MSG_ARG_NOT_FOUND   "Argument \"%s\" not found.\n"
+#define MSG_ARG_INVALID     "Invalid value for argument \"%s\".\n"
+#define MSG_FILE_ERR        "Error in file \"%s\" at line %u.\n"
+#define MSG_FILE_NOT_ENOUGH "Not enough rules in file \"%s\".\n"
+#define MSG_CMD_FAIL        "Command \"%s\" failed.\n"
+
+static int
+is_comment(char *in)
+{
+	if ((strlen(in) && index("!#%;", in[0])) ||
+		(strncmp(in, "//", 2) == 0) ||
+		(strncmp(in, "--", 2) == 0))
+		return 1;
+
+	return 0;
+}
+
+/**
+ * mempool <mempool_name>
+ *  buffer <buffer_size>
+ *  pool <pool_size>
+ *  cache <cache_size>
+ */
+static void
+cmd_mempool(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct softnic_mempool_params p;
+	char *name;
+	struct softnic_mempool *mempool;
+
+	if (n_tokens != 8) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	name = tokens[1];
+
+	if (strcmp(tokens[2], "buffer") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "buffer");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.buffer_size, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "buffer_size");
+		return;
+	}
+
+	if (strcmp(tokens[4], "pool") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pool");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.pool_size, tokens[5]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "pool_size");
+		return;
+	}
+
+	if (strcmp(tokens[6], "cache") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "cache");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.cache_size, tokens[7]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "cache_size");
+		return;
+	}
+
+	mempool = softnic_mempool_create(softnic, name, &p);
+	if (mempool == NULL) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * link <link_name>
+ *    dev <device_name> | port <port_id>
+ */
+static void
+cmd_link(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct softnic_link_params p;
+	struct softnic_link *link;
+	char *name;
+
+	memset(&p, 0, sizeof(p));
+
+	if (n_tokens != 4) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+	name = tokens[1];
+
+	if (strcmp(tokens[2], "dev") == 0) {
+		p.dev_name = tokens[3];
+	} else if (strcmp(tokens[2], "port") == 0) {
+		p.dev_name = NULL;
+
+		if (softnic_parser_read_uint16(&p.port_id, tokens[3]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "port_id");
+			return;
+		}
+	} else {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "dev or port");
+		return;
+	}
+
+	link = softnic_link_create(softnic, name, &p);
+	if (link == NULL) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * swq <swq_name>
+ *  size <size>
+ */
+static void
+cmd_swq(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct softnic_swq_params p;
+	char *name;
+	struct softnic_swq *swq;
+
+	if (n_tokens != 4) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	name = tokens[1];
+
+	if (strcmp(tokens[2], "size") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "size");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.size, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "size");
+		return;
+	}
+
+	swq = softnic_swq_create(softnic, name, &p);
+	if (swq == NULL) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * tmgr shaper profile
+ *  id <profile_id>
+ *  rate <tb_rate> size <tb_size>
+ *  adj <packet_length_adjust>
+ */
+static void
+cmd_tmgr_shaper_profile(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct rte_tm_shaper_params sp;
+	struct rte_tm_error error;
+	uint32_t shaper_profile_id;
+	uint16_t port_id;
+	int status;
+
+	memset(&sp, 0, sizeof(struct rte_tm_shaper_params));
+
+	if (n_tokens != 11) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	if (strcmp(tokens[1], "shaper") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "shaper");
+		return;
+	}
+
+	if (strcmp(tokens[2], "profile") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile");
+		return;
+	}
+
+	if (strcmp(tokens[3], "id") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "id");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&shaper_profile_id, tokens[4]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "profile_id");
+		return;
+	}
+
+	if (strcmp(tokens[5], "rate") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rate");
+		return;
+	}
+
+	if (softnic_parser_read_uint64(&sp.peak.rate, tokens[6]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "tb_rate");
+		return;
+	}
+
+	if (strcmp(tokens[7], "size") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "size");
+		return;
+	}
+
+	if (softnic_parser_read_uint64(&sp.peak.size, tokens[8]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "tb_size");
+		return;
+	}
+
+	if (strcmp(tokens[9], "adj") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "adj");
+		return;
+	}
+
+	if (softnic_parser_read_int32(&sp.pkt_length_adjust, tokens[10]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "packet_length_adjust");
+		return;
+	}
+
+	status = rte_eth_dev_get_port_by_name(softnic->params.name, &port_id);
+	if (status)
+		return;
+
+	status = rte_tm_shaper_profile_add(port_id, shaper_profile_id, &sp, &error);
+	if (status != 0) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * tmgr shared shaper
+ *  id <shared_shaper_id>
+ *  profile <shaper_profile_id>
+ */
+static void
+cmd_tmgr_shared_shaper(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct rte_tm_error error;
+	uint32_t shared_shaper_id, shaper_profile_id;
+	uint16_t port_id;
+	int status;
+
+	if (n_tokens != 7) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	if (strcmp(tokens[1], "shared") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "shared");
+		return;
+	}
+
+	if (strcmp(tokens[2], "shaper") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "shaper");
+		return;
+	}
+
+	if (strcmp(tokens[3], "id") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "id");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&shared_shaper_id, tokens[4]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "shared_shaper_id");
+		return;
+	}
+
+	if (strcmp(tokens[5], "profile") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&shaper_profile_id, tokens[6]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "shaper_profile_id");
+		return;
+	}
+
+	status = rte_eth_dev_get_port_by_name(softnic->params.name, &port_id);
+	if (status)
+		return;
+
+	status = rte_tm_shared_shaper_add_update(port_id,
+		shared_shaper_id,
+		shaper_profile_id,
+		&error);
+	if (status != 0) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * tmgr node
+ *   id <node_id>
+ *   parent <parent_node_id | none>
+ *   priority <priority>
+ *   weight <weight>
+ *   [shaper profile <shaper_profile_id>]
+ *   [shared shaper <shared_shaper_id>]
+ *   [nonleaf sp <n_sp_priorities>]
+ */
+static void
+cmd_tmgr_node(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct rte_tm_error error;
+	struct rte_tm_node_params np;
+	uint32_t node_id, parent_node_id, priority, weight, shared_shaper_id;
+	uint16_t port_id;
+	int status;
+
+	memset(&np, 0, sizeof(struct rte_tm_node_params));
+	np.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
+	np.nonleaf.n_sp_priorities = 1;
+
+	if (n_tokens < 10) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	if (strcmp(tokens[1], "node") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "node");
+		return;
+	}
+
+	if (strcmp(tokens[2], "id") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "id");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&node_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "node_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "parent") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "parent");
+		return;
+	}
+
+	if (strcmp(tokens[5], "none") == 0)
+		parent_node_id = RTE_TM_NODE_ID_NULL;
+	else {
+		if (softnic_parser_read_uint32(&parent_node_id, tokens[5]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "parent_node_id");
+			return;
+		}
+	}
+
+	if (strcmp(tokens[6], "priority") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "priority");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&priority, tokens[7]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "priority");
+		return;
+	}
+
+	if (strcmp(tokens[8], "weight") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "weight");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&weight, tokens[9]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "weight");
+		return;
+	}
+
+	tokens += 10;
+	n_tokens -= 10;
+
+	if (n_tokens >= 2 &&
+		(strcmp(tokens[0], "shaper") == 0) &&
+		(strcmp(tokens[1], "profile") == 0)) {
+		if (n_tokens < 3) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, "tmgr node");
+			return;
+		}
+
+		if (strcmp(tokens[2], "none") == 0) {
+			np.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
+		} else {
+			if (softnic_parser_read_uint32(&np.shaper_profile_id, tokens[2]) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "shaper_profile_id");
+				return;
+			}
+		}
+
+		tokens += 3;
+		n_tokens -= 3;
+	} /* shaper profile */
+
+	if (n_tokens >= 2 &&
+		(strcmp(tokens[0], "shared") == 0) &&
+		(strcmp(tokens[1], "shaper") == 0)) {
+		if (n_tokens < 3) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, "tmgr node");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&shared_shaper_id, tokens[2]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "shared_shaper_id");
+			return;
+		}
+
+		np.shared_shaper_id = &shared_shaper_id;
+		np.n_shared_shapers = 1;
+
+		tokens += 3;
+		n_tokens -= 3;
+	} /* shared shaper */
+
+	if (n_tokens >= 2 &&
+		(strcmp(tokens[0], "nonleaf") == 0) &&
+		(strcmp(tokens[1], "sp") == 0)) {
+		if (n_tokens < 3) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, "tmgr node");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&np.nonleaf.n_sp_priorities, tokens[2]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "n_sp_priorities");
+			return;
+		}
+
+		tokens += 3;
+		n_tokens -= 3;
+	} /* nonleaf sp <n_sp_priorities> */
+
+	if (n_tokens) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	status = rte_eth_dev_get_port_by_name(softnic->params.name, &port_id);
+	if (status != 0)
+		return;
+
+	status = rte_tm_node_add(port_id,
+		node_id,
+		parent_node_id,
+		priority,
+		weight,
+		RTE_TM_NODE_LEVEL_ID_ANY,
+		&np,
+		&error);
+	if (status != 0) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+static uint32_t
+root_node_id(uint32_t n_spp,
+	uint32_t n_pps)
+{
+	uint32_t n_queues = n_spp * n_pps * RTE_SCHED_QUEUES_PER_PIPE;
+	uint32_t n_tc = n_spp * n_pps * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE;
+	uint32_t n_pipes = n_spp * n_pps;
+
+	return n_queues + n_tc + n_pipes + n_spp;
+}
+
+static uint32_t
+subport_node_id(uint32_t n_spp,
+	uint32_t n_pps,
+	uint32_t subport_id)
+{
+	uint32_t n_pipes = n_spp * n_pps;
+	uint32_t n_tc = n_pipes * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE;
+	uint32_t n_queues = n_pipes * RTE_SCHED_QUEUES_PER_PIPE;
+
+	return n_queues + n_tc + n_pipes + subport_id;
+}
+
+static uint32_t
+pipe_node_id(uint32_t n_spp,
+	uint32_t n_pps,
+	uint32_t subport_id,
+	uint32_t pipe_id)
+{
+	uint32_t n_pipes = n_spp * n_pps;
+	uint32_t n_tc = n_pipes * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE;
+	uint32_t n_queues = n_pipes * RTE_SCHED_QUEUES_PER_PIPE;
+
+	return n_queues +
+		n_tc +
+		pipe_id +
+		subport_id * n_pps;
+}
+
+static uint32_t
+tc_node_id(uint32_t n_spp,
+	uint32_t n_pps,
+	uint32_t subport_id,
+	uint32_t pipe_id,
+	uint32_t tc_id)
+{
+	uint32_t n_pipes = n_spp * n_pps;
+	uint32_t n_queues = n_pipes * RTE_SCHED_QUEUES_PER_PIPE;
+
+	return n_queues +
+		tc_id +
+		(pipe_id + subport_id * n_pps) * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE;
+}
+
+static uint32_t
+queue_node_id(uint32_t n_spp __rte_unused,
+	uint32_t n_pps,
+	uint32_t subport_id,
+	uint32_t pipe_id,
+	uint32_t tc_id,
+	uint32_t queue_id)
+{
+	return queue_id +
+		tc_id * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE +
+		(pipe_id + subport_id * n_pps) * RTE_SCHED_QUEUES_PER_PIPE;
+}
+
+struct tmgr_hierarchy_default_params {
+	uint32_t n_spp; /**< Number of subports per port. */
+	uint32_t n_pps; /**< Number of pipes per subport. */
+
+	struct {
+		uint32_t port;
+		uint32_t subport;
+		uint32_t pipe;
+		uint32_t tc[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
+	} shaper_profile_id;
+
+	struct {
+		uint32_t tc[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
+		uint32_t tc_valid[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
+	} shared_shaper_id;
+
+	struct {
+		uint32_t queue[RTE_SCHED_QUEUES_PER_PIPE];
+	} weight;
+};
+
+static int
+tmgr_hierarchy_default(struct pmd_internals *softnic,
+	struct tmgr_hierarchy_default_params *params)
+{
+	struct rte_tm_node_params root_node_params = {
+		.shaper_profile_id = params->shaper_profile_id.port,
+		.nonleaf = {
+			.n_sp_priorities = 1,
+		},
+	};
+
+	struct rte_tm_node_params subport_node_params = {
+		.shaper_profile_id = params->shaper_profile_id.subport,
+		.nonleaf = {
+			.n_sp_priorities = 1,
+		},
+	};
+
+	struct rte_tm_node_params pipe_node_params = {
+		.shaper_profile_id = params->shaper_profile_id.pipe,
+		.nonleaf = {
+			.n_sp_priorities = RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE,
+		},
+	};
+
+	struct rte_tm_node_params tc_node_params[] = {
+		[0] = {
+			.shaper_profile_id = params->shaper_profile_id.tc[0],
+			.shared_shaper_id = &params->shared_shaper_id.tc[0],
+			.n_shared_shapers =
+				(&params->shared_shaper_id.tc_valid[0]) ? 1 : 0,
+			.nonleaf = {
+				.n_sp_priorities = 1,
+			},
+		},
+
+		[1] = {
+			.shaper_profile_id = params->shaper_profile_id.tc[1],
+			.shared_shaper_id = &params->shared_shaper_id.tc[1],
+			.n_shared_shapers =
+				(&params->shared_shaper_id.tc_valid[1]) ? 1 : 0,
+			.nonleaf = {
+				.n_sp_priorities = 1,
+			},
+		},
+
+		[2] = {
+			.shaper_profile_id = params->shaper_profile_id.tc[2],
+			.shared_shaper_id = &params->shared_shaper_id.tc[2],
+			.n_shared_shapers =
+				(&params->shared_shaper_id.tc_valid[2]) ? 1 : 0,
+			.nonleaf = {
+				.n_sp_priorities = 1,
+			},
+		},
+
+		[3] = {
+			.shaper_profile_id = params->shaper_profile_id.tc[3],
+			.shared_shaper_id = &params->shared_shaper_id.tc[3],
+			.n_shared_shapers =
+				(&params->shared_shaper_id.tc_valid[3]) ? 1 : 0,
+			.nonleaf = {
+				.n_sp_priorities = 1,
+			},
+		},
+	};
+
+	struct rte_tm_node_params queue_node_params = {
+		.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE,
+	};
+
+	struct rte_tm_error error;
+	uint32_t n_spp = params->n_spp, n_pps = params->n_pps, s;
+	int status;
+	uint16_t port_id;
+
+	status = rte_eth_dev_get_port_by_name(softnic->params.name, &port_id);
+	if (status)
+		return -1;
+
+	/* Hierarchy level 0: Root node */
+	status = rte_tm_node_add(port_id,
+		root_node_id(n_spp, n_pps),
+		RTE_TM_NODE_ID_NULL,
+		0,
+		1,
+		RTE_TM_NODE_LEVEL_ID_ANY,
+		&root_node_params,
+		&error);
+	if (status)
+		return -1;
+
+	/* Hierarchy level 1: Subport nodes */
+	for (s = 0; s < params->n_spp; s++) {
+		uint32_t p;
+
+		status = rte_tm_node_add(port_id,
+			subport_node_id(n_spp, n_pps, s),
+			root_node_id(n_spp, n_pps),
+			0,
+			1,
+			RTE_TM_NODE_LEVEL_ID_ANY,
+			&subport_node_params,
+			&error);
+		if (status)
+			return -1;
+
+		/* Hierarchy level 2: Pipe nodes */
+		for (p = 0; p < params->n_pps; p++) {
+			uint32_t t;
+
+			status = rte_tm_node_add(port_id,
+				pipe_node_id(n_spp, n_pps, s, p),
+				subport_node_id(n_spp, n_pps, s),
+				0,
+				1,
+				RTE_TM_NODE_LEVEL_ID_ANY,
+				&pipe_node_params,
+				&error);
+			if (status)
+				return -1;
+
+			/* Hierarchy level 3: Traffic class nodes */
+			for (t = 0; t < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; t++) {
+				uint32_t q;
+
+				status = rte_tm_node_add(port_id,
+					tc_node_id(n_spp, n_pps, s, p, t),
+					pipe_node_id(n_spp, n_pps, s, p),
+					t,
+					1,
+					RTE_TM_NODE_LEVEL_ID_ANY,
+					&tc_node_params[t],
+					&error);
+				if (status)
+					return -1;
+
+				/* Hierarchy level 4: Queue nodes */
+				for (q = 0; q < RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS; q++) {
+					status = rte_tm_node_add(port_id,
+						queue_node_id(n_spp, n_pps, s, p, t, q),
+						tc_node_id(n_spp, n_pps, s, p, t),
+						0,
+						params->weight.queue[q],
+						RTE_TM_NODE_LEVEL_ID_ANY,
+						&queue_node_params,
+						&error);
+					if (status)
+						return -1;
+				} /* Queue */
+			} /* TC */
+		} /* Pipe */
+	} /* Subport */
+
+	return 0;
+}
+
+
+/**
+ * tmgr hierarchy-default
+ *  spp <n_subports_per_port>
+ *  pps <n_pipes_per_subport>
+ *  shaper profile
+ *   port <profile_id>
+ *   subport <profile_id>
+ *   pipe <profile_id>
+ *   tc0 <profile_id>
+ *   tc1 <profile_id>
+ *   tc2 <profile_id>
+ *   tc3 <profile_id>
+ *  shared shaper
+ *   tc0 <id | none>
+ *   tc1 <id | none>
+ *   tc2 <id | none>
+ *   tc3 <id | none>
+ *  weight
+ *   queue  <q0> ... <q15>
+ */
+static void
+cmd_tmgr_hierarchy_default(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct tmgr_hierarchy_default_params p;
+	int i, status;
+
+	memset(&p, 0, sizeof(p));
+
+	if (n_tokens != 50) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	if (strcmp(tokens[1], "hierarchy-default") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "hierarchy-default");
+		return;
+	}
+
+	if (strcmp(tokens[2], "spp") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "spp");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.n_spp, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "n_subports_per_port");
+		return;
+	}
+
+	if (strcmp(tokens[4], "pps") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pps");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.n_pps, tokens[5]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "n_pipes_per_subport");
+		return;
+	}
+
+	/* Shaper profile */
+
+	if (strcmp(tokens[6], "shaper") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "shaper");
+		return;
+	}
+
+	if (strcmp(tokens[7], "profile") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile");
+		return;
+	}
+
+	if (strcmp(tokens[8], "port") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.shaper_profile_id.port, tokens[9]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "port profile id");
+		return;
+	}
+
+	if (strcmp(tokens[10], "subport") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "subport");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.shaper_profile_id.subport, tokens[11]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "subport profile id");
+		return;
+	}
+
+	if (strcmp(tokens[12], "pipe") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pipe");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.shaper_profile_id.pipe, tokens[13]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "pipe_profile_id");
+		return;
+	}
+
+	if (strcmp(tokens[14], "tc0") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "tc0");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.shaper_profile_id.tc[0], tokens[15]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "tc0 profile id");
+		return;
+	}
+
+	if (strcmp(tokens[16], "tc1") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "tc1");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.shaper_profile_id.tc[1], tokens[17]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "tc1 profile id");
+		return;
+	}
+
+	if (strcmp(tokens[18], "tc2") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "tc2");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.shaper_profile_id.tc[2], tokens[19]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "tc2 profile id");
+		return;
+	}
+
+	if (strcmp(tokens[20], "tc3") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "tc3");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.shaper_profile_id.tc[3], tokens[21]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "tc3 profile id");
+		return;
+	}
+
+	/* Shared shaper */
+
+	if (strcmp(tokens[22], "shared") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "shared");
+		return;
+	}
+
+	if (strcmp(tokens[23], "shaper") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "shaper");
+		return;
+	}
+
+	if (strcmp(tokens[24], "tc0") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "tc0");
+		return;
+	}
+
+	if (strcmp(tokens[25], "none") == 0)
+		p.shared_shaper_id.tc_valid[0] = 0;
+	else {
+		if (softnic_parser_read_uint32(&p.shared_shaper_id.tc[0], tokens[25]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "shared shaper tc0");
+			return;
+		}
+
+		p.shared_shaper_id.tc_valid[0] = 1;
+	}
+
+	if (strcmp(tokens[26], "tc1") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "tc1");
+		return;
+	}
+
+	if (strcmp(tokens[27], "none") == 0)
+		p.shared_shaper_id.tc_valid[1] = 0;
+	else {
+		if (softnic_parser_read_uint32(&p.shared_shaper_id.tc[1], tokens[27]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "shared shaper tc1");
+			return;
+		}
+
+		p.shared_shaper_id.tc_valid[1] = 1;
+	}
+
+	if (strcmp(tokens[28], "tc2") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "tc2");
+		return;
+	}
+
+	if (strcmp(tokens[29], "none") == 0)
+		p.shared_shaper_id.tc_valid[2] = 0;
+	else {
+		if (softnic_parser_read_uint32(&p.shared_shaper_id.tc[2], tokens[29]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "shared shaper tc2");
+			return;
+		}
+
+		p.shared_shaper_id.tc_valid[2] = 1;
+	}
+
+	if (strcmp(tokens[30], "tc3") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "tc3");
+		return;
+	}
+
+	if (strcmp(tokens[31], "none") == 0)
+		p.shared_shaper_id.tc_valid[3] = 0;
+	else {
+		if (softnic_parser_read_uint32(&p.shared_shaper_id.tc[3], tokens[31]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "shared shaper tc3");
+			return;
+		}
+
+		p.shared_shaper_id.tc_valid[3] = 1;
+	}
+
+	/* Weight */
+
+	if (strcmp(tokens[32], "weight") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "weight");
+		return;
+	}
+
+	if (strcmp(tokens[33], "queue") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "queue");
+		return;
+	}
+
+	for (i = 0; i < 16; i++) {
+		if (softnic_parser_read_uint32(&p.weight.queue[i], tokens[34 + i]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "weight queue");
+			return;
+		}
+	}
+
+	status = tmgr_hierarchy_default(softnic, &p);
+	if (status != 0) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * tmgr hierarchy commit
+ */
+static void
+cmd_tmgr_hierarchy_commit(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct rte_tm_error error;
+	uint16_t port_id;
+	int status;
+
+	if (n_tokens != 3) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	if (strcmp(tokens[1], "hierarchy") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "hierarchy");
+		return;
+	}
+
+	if (strcmp(tokens[2], "commit") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "commit");
+		return;
+	}
+
+	status = rte_eth_dev_get_port_by_name(softnic->params.name, &port_id);
+	if (status != 0)
+		return;
+
+	status = rte_tm_hierarchy_commit(port_id, 1, &error);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * tmgr <tmgr_name>
+ */
+static void
+cmd_tmgr(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	char *name;
+	struct softnic_tmgr_port *tmgr_port;
+
+	if (n_tokens != 2) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	name = tokens[1];
+
+	tmgr_port = softnic_tmgr_port_create(softnic, name);
+	if (tmgr_port == NULL) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * tap <tap_name>
+ */
+static void
+cmd_tap(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	char *name;
+	struct softnic_tap *tap;
+
+	if (n_tokens != 2) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	name = tokens[1];
+
+	tap = softnic_tap_create(softnic, name);
+	if (tap == NULL) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * port in action profile <profile_name>
+ *  [filter match | mismatch offset <key_offset> mask <key_mask> key <key_value> port <port_id>]
+ *  [balance offset <key_offset> mask <key_mask> port <port_id0> ... <port_id15>]
+ */
+static void
+cmd_port_in_action_profile(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct softnic_port_in_action_profile_params p;
+	struct softnic_port_in_action_profile *ap;
+	char *name;
+	uint32_t t0;
+
+	memset(&p, 0, sizeof(p));
+
+	if (n_tokens < 5) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	if (strcmp(tokens[1], "in") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "in");
+		return;
+	}
+
+	if (strcmp(tokens[2], "action") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "action");
+		return;
+	}
+
+	if (strcmp(tokens[3], "profile") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile");
+		return;
+	}
+
+	name = tokens[4];
+
+	t0 = 5;
+
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "filter") == 0)) {
+		uint32_t size;
+
+		if (n_tokens < t0 + 10) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, "port in action profile filter");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 1], "match") == 0) {
+			p.fltr.filter_on_match = 1;
+		} else if (strcmp(tokens[t0 + 1], "mismatch") == 0) {
+			p.fltr.filter_on_match = 0;
+		} else {
+			snprintf(out, out_size, MSG_ARG_INVALID, "match or mismatch");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 2], "offset") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "offset");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.fltr.key_offset,
+			tokens[t0 + 3]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_offset");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 4], "mask") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "mask");
+			return;
+		}
+
+		size = RTE_PORT_IN_ACTION_FLTR_KEY_SIZE;
+		if ((softnic_parse_hex_string(tokens[t0 + 5],
+			p.fltr.key_mask, &size) != 0) ||
+			size != RTE_PORT_IN_ACTION_FLTR_KEY_SIZE) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_mask");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 6], "key") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "key");
+			return;
+		}
+
+		size = RTE_PORT_IN_ACTION_FLTR_KEY_SIZE;
+		if ((softnic_parse_hex_string(tokens[t0 + 7],
+			p.fltr.key, &size) != 0) ||
+			size != RTE_PORT_IN_ACTION_FLTR_KEY_SIZE) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_value");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 8], "port") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.fltr.port_id,
+			tokens[t0 + 9]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "port_id");
+			return;
+		}
+
+		p.action_mask |= 1LLU << RTE_PORT_IN_ACTION_FLTR;
+		t0 += 10;
+	} /* filter */
+
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "balance") == 0)) {
+		uint32_t i;
+
+		if (n_tokens < t0 + 22) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"port in action profile balance");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 1], "offset") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "offset");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.lb.key_offset,
+			tokens[t0 + 2]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_offset");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 3], "mask") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "mask");
+			return;
+		}
+
+		p.lb.key_size = RTE_PORT_IN_ACTION_LB_KEY_SIZE_MAX;
+		if (softnic_parse_hex_string(tokens[t0 + 4],
+			p.lb.key_mask, &p.lb.key_size) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_mask");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 5], "port") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+			return;
+		}
+
+		for (i = 0; i < 16; i++)
+			if (softnic_parser_read_uint32(&p.lb.port_id[i],
+			tokens[t0 + 6 + i]) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "port_id");
+				return;
+			}
+
+		p.action_mask |= 1LLU << RTE_PORT_IN_ACTION_LB;
+		t0 += 22;
+	} /* balance */
+
+	if (t0 < n_tokens) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	ap = softnic_port_in_action_profile_create(softnic, name, &p);
+	if (ap == NULL) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * table action profile <profile_name>
+ *  ipv4 | ipv6
+ *  offset <ip_offset>
+ *  fwd
+ *  [balance offset <key_offset> mask <key_mask> outoffset <out_offset>]
+ *  [meter srtcm | trtcm
+ *      tc <n_tc>
+ *      stats none | pkts | bytes | both]
+ *  [tm spp <n_subports_per_port> pps <n_pipes_per_subport>]
+ *  [encap ether | vlan | qinq | mpls | pppoe]
+ *  [nat src | dst
+ *      proto udp | tcp]
+ *  [ttl drop | fwd
+ *      stats none | pkts]
+ *  [stats pkts | bytes | both]
+ *  [time]
+ */
+static void
+cmd_table_action_profile(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct softnic_table_action_profile_params p;
+	struct softnic_table_action_profile *ap;
+	char *name;
+	uint32_t t0;
+
+	memset(&p, 0, sizeof(p));
+
+	if (n_tokens < 8) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	if (strcmp(tokens[1], "action") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "action");
+		return;
+	}
+
+	if (strcmp(tokens[2], "profile") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile");
+		return;
+	}
+
+	name = tokens[3];
+
+	if (strcmp(tokens[4], "ipv4") == 0) {
+		p.common.ip_version = 1;
+	} else if (strcmp(tokens[4], "ipv6") == 0) {
+		p.common.ip_version = 0;
+	} else {
+		snprintf(out, out_size, MSG_ARG_INVALID, "ipv4 or ipv6");
+		return;
+	}
+
+	if (strcmp(tokens[5], "offset") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "offset");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.common.ip_offset,
+		tokens[6]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "ip_offset");
+		return;
+	}
+
+	if (strcmp(tokens[7], "fwd") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "fwd");
+		return;
+	}
+
+	p.action_mask |= 1LLU << RTE_TABLE_ACTION_FWD;
+
+	t0 = 8;
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "balance") == 0)) {
+		if (n_tokens < t0 + 7) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, "table action profile balance");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 1], "offset") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "offset");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.lb.key_offset,
+			tokens[t0 + 2]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_offset");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 3], "mask") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "mask");
+			return;
+		}
+
+		p.lb.key_size = RTE_PORT_IN_ACTION_LB_KEY_SIZE_MAX;
+		if (softnic_parse_hex_string(tokens[t0 + 4],
+			p.lb.key_mask, &p.lb.key_size) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_mask");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 5], "outoffset") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "outoffset");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.lb.out_offset,
+			tokens[t0 + 6]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "out_offset");
+			return;
+		}
+
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_LB;
+		t0 += 7;
+	} /* balance */
+
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "meter") == 0)) {
+		if (n_tokens < t0 + 6) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"table action profile meter");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 1], "srtcm") == 0) {
+			p.mtr.alg = RTE_TABLE_ACTION_METER_SRTCM;
+		} else if (strcmp(tokens[t0 + 1], "trtcm") == 0) {
+			p.mtr.alg = RTE_TABLE_ACTION_METER_TRTCM;
+		} else {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"srtcm or trtcm");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 2], "tc") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "tc");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.mtr.n_tc,
+			tokens[t0 + 3]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "n_tc");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 4], "stats") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "stats");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 5], "none") == 0) {
+			p.mtr.n_packets_enabled = 0;
+			p.mtr.n_bytes_enabled = 0;
+		} else if (strcmp(tokens[t0 + 5], "pkts") == 0) {
+			p.mtr.n_packets_enabled = 1;
+			p.mtr.n_bytes_enabled = 0;
+		} else if (strcmp(tokens[t0 + 5], "bytes") == 0) {
+			p.mtr.n_packets_enabled = 0;
+			p.mtr.n_bytes_enabled = 1;
+		} else if (strcmp(tokens[t0 + 5], "both") == 0) {
+			p.mtr.n_packets_enabled = 1;
+			p.mtr.n_bytes_enabled = 1;
+		} else {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"none or pkts or bytes or both");
+			return;
+		}
+
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_MTR;
+		t0 += 6;
+	} /* meter */
+
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "tm") == 0)) {
+		if (n_tokens < t0 + 5) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"table action profile tm");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 1], "spp") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "spp");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.tm.n_subports_per_port,
+			tokens[t0 + 2]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"n_subports_per_port");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 3], "pps") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pps");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.tm.n_pipes_per_subport,
+			tokens[t0 + 4]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"n_pipes_per_subport");
+			return;
+		}
+
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_TM;
+		t0 += 5;
+	} /* tm */
+
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "encap") == 0)) {
+		if (n_tokens < t0 + 2) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"action profile encap");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 1], "ether") == 0) {
+			p.encap.encap_mask = 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER;
+		} else if (strcmp(tokens[t0 + 1], "vlan") == 0) {
+			p.encap.encap_mask = 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN;
+		} else if (strcmp(tokens[t0 + 1], "qinq") == 0) {
+			p.encap.encap_mask = 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ;
+		} else if (strcmp(tokens[t0 + 1], "mpls") == 0) {
+			p.encap.encap_mask = 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS;
+		} else if (strcmp(tokens[t0 + 1], "pppoe") == 0) {
+			p.encap.encap_mask = 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE;
+		} else {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, "encap");
+			return;
+		}
+
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_ENCAP;
+		t0 += 2;
+	} /* encap */
+
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "nat") == 0)) {
+		if (n_tokens < t0 + 4) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"table action profile nat");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 1], "src") == 0) {
+			p.nat.source_nat = 1;
+		} else if (strcmp(tokens[t0 + 1], "dst") == 0) {
+			p.nat.source_nat = 0;
+		} else {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"src or dst");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 2], "proto") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "proto");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 3], "tcp") == 0) {
+			p.nat.proto = 0x06;
+		} else if (strcmp(tokens[t0 + 3], "udp") == 0) {
+			p.nat.proto = 0x11;
+		} else {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"tcp or udp");
+			return;
+		}
+
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_NAT;
+		t0 += 4;
+	} /* nat */
+
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "ttl") == 0)) {
+		if (n_tokens < t0 + 4) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"table action profile ttl");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 1], "drop") == 0) {
+			p.ttl.drop = 1;
+		} else if (strcmp(tokens[t0 + 1], "fwd") == 0) {
+			p.ttl.drop = 0;
+		} else {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"drop or fwd");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 2], "stats") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "stats");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 3], "none") == 0) {
+			p.ttl.n_packets_enabled = 0;
+		} else if (strcmp(tokens[t0 + 3], "pkts") == 0) {
+			p.ttl.n_packets_enabled = 1;
+		} else {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"none or pkts");
+			return;
+		}
+
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_TTL;
+		t0 += 4;
+	} /* ttl */
+
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "stats") == 0)) {
+		if (n_tokens < t0 + 2) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"table action profile stats");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 1], "pkts") == 0) {
+			p.stats.n_packets_enabled = 1;
+			p.stats.n_bytes_enabled = 0;
+		} else if (strcmp(tokens[t0 + 1], "bytes") == 0) {
+			p.stats.n_packets_enabled = 0;
+			p.stats.n_bytes_enabled = 1;
+		} else if (strcmp(tokens[t0 + 1], "both") == 0) {
+			p.stats.n_packets_enabled = 1;
+			p.stats.n_bytes_enabled = 1;
+		} else {
+			snprintf(out, out_size,	MSG_ARG_NOT_FOUND,
+				"pkts or bytes or both");
+			return;
+		}
+
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_STATS;
+		t0 += 2;
+	} /* stats */
+
+	if (t0 < n_tokens &&
+		(strcmp(tokens[t0], "time") == 0)) {
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_TIME;
+		t0 += 1;
+	} /* time */
+
+	if (t0 < n_tokens) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	ap = softnic_table_action_profile_create(softnic, name, &p);
+	if (ap == NULL) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name>
+ *  period <timer_period_ms>
+ *  offset_port_id <offset_port_id>
+ */
+static void
+cmd_pipeline(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct pipeline_params p;
+	char *name;
+	struct pipeline *pipeline;
+
+	if (n_tokens != 6) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	name = tokens[1];
+
+	if (strcmp(tokens[2], "period") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "period");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.timer_period_ms,
+		tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "timer_period_ms");
+		return;
+	}
+
+	if (strcmp(tokens[4], "offset_port_id") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "offset_port_id");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.offset_port_id,
+		tokens[5]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "offset_port_id");
+		return;
+	}
+
+	pipeline = softnic_pipeline_create(softnic, name, &p);
+	if (pipeline == NULL) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> port in
+ *  bsz <burst_size>
+ *  link <link_name> rxq <queue_id>
+ *  | swq <swq_name>
+ *  | tmgr <tmgr_name>
+ *  | tap <tap_name> mempool <mempool_name> mtu <mtu>
+ *  | source mempool <mempool_name> file <file_name> bpp <n_bytes_per_pkt>
+ *  [action <port_in_action_profile_name>]
+ *  [disabled]
+ */
+static void
+cmd_pipeline_port_in(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct softnic_port_in_params p;
+	char *pipeline_name;
+	uint32_t t0;
+	int enabled, status;
+
+	if (n_tokens < 7) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "port") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (strcmp(tokens[3], "in") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "in");
+		return;
+	}
+
+	if (strcmp(tokens[4], "bsz") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "bsz");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.burst_size, tokens[5]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "burst_size");
+		return;
+	}
+
+	t0 = 6;
+
+	if (strcmp(tokens[t0], "link") == 0) {
+		if (n_tokens < t0 + 4) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port in link");
+			return;
+		}
+
+		p.type = PORT_IN_RXQ;
+
+		p.dev_name = tokens[t0 + 1];
+
+		if (strcmp(tokens[t0 + 2], "rxq") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rxq");
+			return;
+		}
+
+		if (softnic_parser_read_uint16(&p.rxq.queue_id,
+			tokens[t0 + 3]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"queue_id");
+			return;
+		}
+		t0 += 4;
+	} else if (strcmp(tokens[t0], "swq") == 0) {
+		if (n_tokens < t0 + 2) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port in swq");
+			return;
+		}
+
+		p.type = PORT_IN_SWQ;
+
+		p.dev_name = tokens[t0 + 1];
+
+		t0 += 2;
+	} else if (strcmp(tokens[t0], "tmgr") == 0) {
+		if (n_tokens < t0 + 2) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port in tmgr");
+			return;
+		}
+
+		p.type = PORT_IN_TMGR;
+
+		p.dev_name = tokens[t0 + 1];
+
+		t0 += 2;
+	} else if (strcmp(tokens[t0], "tap") == 0) {
+		if (n_tokens < t0 + 6) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port in tap");
+			return;
+		}
+
+		p.type = PORT_IN_TAP;
+
+		p.dev_name = tokens[t0 + 1];
+
+		if (strcmp(tokens[t0 + 2], "mempool") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"mempool");
+			return;
+		}
+
+		p.tap.mempool_name = tokens[t0 + 3];
+
+		if (strcmp(tokens[t0 + 4], "mtu") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"mtu");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.tap.mtu,
+			tokens[t0 + 5]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "mtu");
+			return;
+		}
+
+		t0 += 6;
+	} else if (strcmp(tokens[t0], "source") == 0) {
+		if (n_tokens < t0 + 6) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port in source");
+			return;
+		}
+
+		p.type = PORT_IN_SOURCE;
+
+		p.dev_name = NULL;
+
+		if (strcmp(tokens[t0 + 1], "mempool") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"mempool");
+			return;
+		}
+
+		p.source.mempool_name = tokens[t0 + 2];
+
+		if (strcmp(tokens[t0 + 3], "file") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"file");
+			return;
+		}
+
+		p.source.file_name = tokens[t0 + 4];
+
+		if (strcmp(tokens[t0 + 5], "bpp") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"bpp");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.source.n_bytes_per_pkt,
+			tokens[t0 + 6]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"n_bytes_per_pkt");
+			return;
+		}
+
+		t0 += 7;
+	} else {
+		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
+		return;
+	}
+
+	p.action_profile_name = NULL;
+	if (n_tokens > t0 &&
+		(strcmp(tokens[t0], "action") == 0)) {
+		if (n_tokens < t0 + 2) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, "action");
+			return;
+		}
+
+		p.action_profile_name = tokens[t0 + 1];
+
+		t0 += 2;
+	}
+
+	enabled = 1;
+	if (n_tokens > t0 &&
+		(strcmp(tokens[t0], "disabled") == 0)) {
+		enabled = 0;
+
+		t0 += 1;
+	}
+
+	if (n_tokens != t0) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	status = softnic_pipeline_port_in_create(softnic,
+		pipeline_name,
+		&p,
+		enabled);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> port out
+ *  bsz <burst_size>
+ *  link <link_name> txq <txq_id>
+ *  | swq <swq_name>
+ *  | tmgr <tmgr_name>
+ *  | tap <tap_name>
+ *  | sink [file <file_name> pkts <max_n_pkts>]
+ */
+static void
+cmd_pipeline_port_out(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct softnic_port_out_params p;
+	char *pipeline_name;
+	int status;
+
+	memset(&p, 0, sizeof(p));
+
+	if (n_tokens < 7) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "port") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (strcmp(tokens[3], "out") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "out");
+		return;
+	}
+
+	if (strcmp(tokens[4], "bsz") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "bsz");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&p.burst_size, tokens[5]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "burst_size");
+		return;
+	}
+
+	if (strcmp(tokens[6], "link") == 0) {
+		if (n_tokens != 10) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port out link");
+			return;
+		}
+
+		p.type = PORT_OUT_TXQ;
+
+		p.dev_name = tokens[7];
+
+		if (strcmp(tokens[8], "txq") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "txq");
+			return;
+		}
+
+		if (softnic_parser_read_uint16(&p.txq.queue_id,
+			tokens[9]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "queue_id");
+			return;
+		}
+	} else if (strcmp(tokens[6], "swq") == 0) {
+		if (n_tokens != 8) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port out swq");
+			return;
+		}
+
+		p.type = PORT_OUT_SWQ;
+
+		p.dev_name = tokens[7];
+	} else if (strcmp(tokens[6], "tmgr") == 0) {
+		if (n_tokens != 8) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port out tmgr");
+			return;
+		}
+
+		p.type = PORT_OUT_TMGR;
+
+		p.dev_name = tokens[7];
+	} else if (strcmp(tokens[6], "tap") == 0) {
+		if (n_tokens != 8) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port out tap");
+			return;
+		}
+
+		p.type = PORT_OUT_TAP;
+
+		p.dev_name = tokens[7];
+	} else if (strcmp(tokens[6], "sink") == 0) {
+		if ((n_tokens != 7) && (n_tokens != 11)) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port out sink");
+			return;
+		}
+
+		p.type = PORT_OUT_SINK;
+
+		p.dev_name = NULL;
+
+		if (n_tokens == 7) {
+			p.sink.file_name = NULL;
+			p.sink.max_n_pkts = 0;
+		} else {
+			if (strcmp(tokens[7], "file") != 0) {
+				snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+					"file");
+				return;
+			}
+
+			p.sink.file_name = tokens[8];
+
+			if (strcmp(tokens[9], "pkts") != 0) {
+				snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pkts");
+				return;
+			}
+
+			if (softnic_parser_read_uint32(&p.sink.max_n_pkts,
+				tokens[10]) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "max_n_pkts");
+				return;
+			}
+		}
+	} else {
+		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
+		return;
+	}
+
+	status = softnic_pipeline_port_out_create(softnic, pipeline_name, &p);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> table
+ *      match
+ *      acl
+ *          ipv4 | ipv6
+ *          offset <ip_header_offset>
+ *          size <n_rules>
+ *      | array
+ *          offset <key_offset>
+ *          size <n_keys>
+ *      | hash
+ *          ext | lru
+ *          key <key_size>
+ *          mask <key_mask>
+ *          offset <key_offset>
+ *          buckets <n_buckets>
+ *          size <n_keys>
+ *      | lpm
+ *          ipv4 | ipv6
+ *          offset <ip_header_offset>
+ *          size <n_rules>
+ *      | stub
+ *  [action <table_action_profile_name>]
+ */
+static void
+cmd_pipeline_table(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	uint8_t key_mask[TABLE_RULE_MATCH_SIZE_MAX];
+	struct softnic_table_params p;
+	char *pipeline_name;
+	uint32_t t0;
+	int status;
+
+	if (n_tokens < 5) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (strcmp(tokens[3], "match") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "match");
+		return;
+	}
+
+	t0 = 4;
+	if (strcmp(tokens[t0], "acl") == 0) {
+		if (n_tokens < t0 + 6) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline table acl");
+			return;
+		}
+
+		p.match_type = TABLE_ACL;
+
+		if (strcmp(tokens[t0 + 1], "ipv4") == 0) {
+			p.match.acl.ip_version = 1;
+		} else if (strcmp(tokens[t0 + 1], "ipv6") == 0) {
+			p.match.acl.ip_version = 0;
+		} else {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"ipv4 or ipv6");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 2], "offset") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "offset");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.match.acl.ip_header_offset,
+			tokens[t0 + 3]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"ip_header_offset");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 4], "size") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "size");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.match.acl.n_rules,
+			tokens[t0 + 5]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "n_rules");
+			return;
+		}
+
+		t0 += 6;
+	} else if (strcmp(tokens[t0], "array") == 0) {
+		if (n_tokens < t0 + 5) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline table array");
+			return;
+		}
+
+		p.match_type = TABLE_ARRAY;
+
+		if (strcmp(tokens[t0 + 1], "offset") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "offset");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.match.array.key_offset,
+			tokens[t0 + 2]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_offset");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 3], "size") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "size");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.match.array.n_keys,
+			tokens[t0 + 4]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "n_keys");
+			return;
+		}
+
+		t0 += 5;
+	} else if (strcmp(tokens[t0], "hash") == 0) {
+		uint32_t key_mask_size = TABLE_RULE_MATCH_SIZE_MAX;
+
+		if (n_tokens < t0 + 12) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline table hash");
+			return;
+		}
+
+		p.match_type = TABLE_HASH;
+
+		if (strcmp(tokens[t0 + 1], "ext") == 0) {
+			p.match.hash.extendable_bucket = 1;
+		} else if (strcmp(tokens[t0 + 1], "lru") == 0) {
+			p.match.hash.extendable_bucket = 0;
+		} else {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"ext or lru");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 2], "key") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "key");
+			return;
+		}
+
+		if ((softnic_parser_read_uint32(&p.match.hash.key_size,
+			tokens[t0 + 3]) != 0) ||
+			p.match.hash.key_size == 0 ||
+			p.match.hash.key_size > TABLE_RULE_MATCH_SIZE_MAX) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_size");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 4], "mask") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "mask");
+			return;
+		}
+
+		if ((softnic_parse_hex_string(tokens[t0 + 5],
+			key_mask, &key_mask_size) != 0) ||
+			key_mask_size != p.match.hash.key_size) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_mask");
+			return;
+		}
+		p.match.hash.key_mask = key_mask;
+
+		if (strcmp(tokens[t0 + 6], "offset") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "offset");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.match.hash.key_offset,
+			tokens[t0 + 7]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_offset");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 8], "buckets") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "buckets");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.match.hash.n_buckets,
+			tokens[t0 + 9]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "n_buckets");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 10], "size") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "size");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.match.hash.n_keys,
+			tokens[t0 + 11]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "n_keys");
+			return;
+		}
+
+		t0 += 12;
+	} else if (strcmp(tokens[t0], "lpm") == 0) {
+		if (n_tokens < t0 + 6) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline table lpm");
+			return;
+		}
+
+		p.match_type = TABLE_LPM;
+
+		if (strcmp(tokens[t0 + 1], "ipv4") == 0) {
+			p.match.lpm.key_size = 4;
+		} else if (strcmp(tokens[t0 + 1], "ipv6") == 0) {
+			p.match.lpm.key_size = 16;
+		} else {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"ipv4 or ipv6");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 2], "offset") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "offset");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.match.lpm.key_offset,
+			tokens[t0 + 3]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "key_offset");
+			return;
+		}
+
+		if (strcmp(tokens[t0 + 4], "size") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "size");
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&p.match.lpm.n_rules,
+			tokens[t0 + 5]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "n_rules");
+			return;
+		}
+
+		t0 += 6;
+	} else if (strcmp(tokens[t0], "stub") == 0) {
+		p.match_type = TABLE_STUB;
+
+		t0 += 1;
+	} else {
+		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
+		return;
+	}
+
+	p.action_profile_name = NULL;
+	if (n_tokens > t0 &&
+		(strcmp(tokens[t0], "action") == 0)) {
+		if (n_tokens < t0 + 2) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, "action");
+			return;
+		}
+
+		p.action_profile_name = tokens[t0 + 1];
+
+		t0 += 2;
+	}
+
+	if (n_tokens > t0) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	status = softnic_pipeline_table_create(softnic, pipeline_name, &p);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> port in <port_id> table <table_id>
+ */
+static void
+cmd_pipeline_port_in_table(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	char *pipeline_name;
+	uint32_t port_id, table_id;
+	int status;
+
+	if (n_tokens != 7) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "port") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (strcmp(tokens[3], "in") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "in");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&port_id, tokens[4]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "port_id");
+		return;
+	}
+
+	if (strcmp(tokens[5], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&table_id, tokens[6]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	status = softnic_pipeline_port_in_connect_to_table(softnic,
+		pipeline_name,
+		port_id,
+		table_id);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> port in <port_id> stats read [clear]
+ */
+
+#define MSG_PIPELINE_PORT_IN_STATS                         \
+	"Pkts in: %" PRIu64 "\n"                           \
+	"Pkts dropped by AH: %" PRIu64 "\n"                \
+	"Pkts dropped by other: %" PRIu64 "\n"
+
+static void
+cmd_pipeline_port_in_stats(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct rte_pipeline_port_in_stats stats;
+	char *pipeline_name;
+	uint32_t port_id;
+	int clear, status;
+
+	if (n_tokens != 7 &&
+		n_tokens != 8) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "port") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (strcmp(tokens[3], "in") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "in");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&port_id, tokens[4]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "port_id");
+		return;
+	}
+
+	if (strcmp(tokens[5], "stats") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "stats");
+		return;
+	}
+
+	if (strcmp(tokens[6], "read") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "read");
+		return;
+	}
+
+	clear = 0;
+	if (n_tokens == 8) {
+		if (strcmp(tokens[7], "clear") != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "clear");
+			return;
+		}
+
+		clear = 1;
+	}
+
+	status = softnic_pipeline_port_in_stats_read(softnic,
+		pipeline_name,
+		port_id,
+		&stats,
+		clear);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+
+	snprintf(out, out_size, MSG_PIPELINE_PORT_IN_STATS,
+		stats.stats.n_pkts_in,
+		stats.n_pkts_dropped_by_ah,
+		stats.stats.n_pkts_drop);
+}
+
+/**
+ * pipeline <pipeline_name> port in <port_id> enable
+ */
+static void
+cmd_softnic_pipeline_port_in_enable(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	char *pipeline_name;
+	uint32_t port_id;
+	int status;
+
+	if (n_tokens != 6) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "port") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (strcmp(tokens[3], "in") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "in");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&port_id, tokens[4]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "port_id");
+		return;
+	}
+
+	if (strcmp(tokens[5], "enable") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "enable");
+		return;
+	}
+
+	status = softnic_pipeline_port_in_enable(softnic, pipeline_name, port_id);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> port in <port_id> disable
+ */
+static void
+cmd_softnic_pipeline_port_in_disable(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	char *pipeline_name;
+	uint32_t port_id;
+	int status;
+
+	if (n_tokens != 6) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "port") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (strcmp(tokens[3], "in") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "in");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&port_id, tokens[4]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "port_id");
+		return;
+	}
+
+	if (strcmp(tokens[5], "disable") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "disable");
+		return;
+	}
+
+	status = softnic_pipeline_port_in_disable(softnic, pipeline_name, port_id);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> port out <port_id> stats read [clear]
+ */
+#define MSG_PIPELINE_PORT_OUT_STATS                        \
+	"Pkts in: %" PRIu64 "\n"                           \
+	"Pkts dropped by AH: %" PRIu64 "\n"                \
+	"Pkts dropped by other: %" PRIu64 "\n"
+
+static void
+cmd_pipeline_port_out_stats(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct rte_pipeline_port_out_stats stats;
+	char *pipeline_name;
+	uint32_t port_id;
+	int clear, status;
+
+	if (n_tokens != 7 &&
+		n_tokens != 8) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "port") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (strcmp(tokens[3], "out") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "out");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&port_id, tokens[4]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "port_id");
+		return;
+	}
+
+	if (strcmp(tokens[5], "stats") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "stats");
+		return;
+	}
+
+	if (strcmp(tokens[6], "read") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "read");
+		return;
+	}
+
+	clear = 0;
+	if (n_tokens == 8) {
+		if (strcmp(tokens[7], "clear") != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "clear");
+			return;
+		}
+
+		clear = 1;
+	}
+
+	status = softnic_pipeline_port_out_stats_read(softnic,
+		pipeline_name,
+		port_id,
+		&stats,
+		clear);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+
+	snprintf(out, out_size, MSG_PIPELINE_PORT_OUT_STATS,
+		stats.stats.n_pkts_in,
+		stats.n_pkts_dropped_by_ah,
+		stats.stats.n_pkts_drop);
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id> stats read [clear]
+ */
+#define MSG_PIPELINE_TABLE_STATS                                     \
+	"Pkts in: %" PRIu64 "\n"                                     \
+	"Pkts in with lookup miss: %" PRIu64 "\n"                    \
+	"Pkts in with lookup hit dropped by AH: %" PRIu64 "\n"       \
+	"Pkts in with lookup hit dropped by others: %" PRIu64 "\n"   \
+	"Pkts in with lookup miss dropped by AH: %" PRIu64 "\n"      \
+	"Pkts in with lookup miss dropped by others: %" PRIu64 "\n"
+
+static void
+cmd_pipeline_table_stats(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct rte_pipeline_table_stats stats;
+	char *pipeline_name;
+	uint32_t table_id;
+	int clear, status;
+
+	if (n_tokens != 6 &&
+		n_tokens != 7) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "stats") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "stats");
+		return;
+	}
+
+	if (strcmp(tokens[5], "read") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "read");
+		return;
+	}
+
+	clear = 0;
+	if (n_tokens == 7) {
+		if (strcmp(tokens[6], "clear") != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "clear");
+			return;
+		}
+
+		clear = 1;
+	}
+
+	status = softnic_pipeline_table_stats_read(softnic,
+		pipeline_name,
+		table_id,
+		&stats,
+		clear);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+
+	snprintf(out, out_size, MSG_PIPELINE_TABLE_STATS,
+		stats.stats.n_pkts_in,
+		stats.stats.n_pkts_lookup_miss,
+		stats.n_pkts_dropped_by_lkp_hit_ah,
+		stats.n_pkts_dropped_lkp_hit,
+		stats.n_pkts_dropped_by_lkp_miss_ah,
+		stats.n_pkts_dropped_lkp_miss);
+}
+
+/**
+ * <match> ::=
+ *
+ * match
+ *    acl
+ *       priority <priority>
+ *       ipv4 | ipv6 <sa> <sa_depth> <da> <da_depth>
+ *       <sp0> <sp1> <dp0> <dp1> <proto>
+ *    | array <pos>
+ *    | hash
+ *       raw <key>
+ *       | ipv4_5tuple <sa> <da> <sp> <dp> <proto>
+ *       | ipv6_5tuple <sa> <da> <sp> <dp> <proto>
+ *       | ipv4_addr <addr>
+ *       | ipv6_addr <addr>
+ *       | qinq <svlan> <cvlan>
+ *    | lpm
+ *       ipv4 | ipv6 <addr> <depth>
+ */
+struct pkt_key_qinq {
+	uint16_t ethertype_svlan;
+	uint16_t svlan;
+	uint16_t ethertype_cvlan;
+	uint16_t cvlan;
+} __attribute__((__packed__));
+
+struct pkt_key_ipv4_5tuple {
+	uint8_t time_to_live;
+	uint8_t proto;
+	uint16_t hdr_checksum;
+	uint32_t sa;
+	uint32_t da;
+	uint16_t sp;
+	uint16_t dp;
+} __attribute__((__packed__));
+
+struct pkt_key_ipv6_5tuple {
+	uint16_t payload_length;
+	uint8_t proto;
+	uint8_t hop_limit;
+	uint8_t sa[16];
+	uint8_t da[16];
+	uint16_t sp;
+	uint16_t dp;
+} __attribute__((__packed__));
+
+struct pkt_key_ipv4_addr {
+	uint32_t addr;
+} __attribute__((__packed__));
+
+struct pkt_key_ipv6_addr {
+	uint8_t addr[16];
+} __attribute__((__packed__));
+
+static uint32_t
+parse_match(char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size,
+	struct softnic_table_rule_match *m)
+{
+	memset(m, 0, sizeof(*m));
+
+	if (n_tokens < 2)
+		return 0;
+
+	if (strcmp(tokens[0], "match") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "match");
+		return 0;
+	}
+
+	if (strcmp(tokens[1], "acl") == 0) {
+		if (n_tokens < 14) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+			return 0;
+		}
+
+		m->match_type = TABLE_ACL;
+
+		if (strcmp(tokens[2], "priority") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "priority");
+			return 0;
+		}
+
+		if (softnic_parser_read_uint32(&m->match.acl.priority,
+			tokens[3]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "priority");
+			return 0;
+		}
+
+		if (strcmp(tokens[4], "ipv4") == 0) {
+			struct in_addr saddr, daddr;
+
+			m->match.acl.ip_version = 1;
+
+			if (softnic_parse_ipv4_addr(tokens[5], &saddr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "sa");
+				return 0;
+			}
+			m->match.acl.ipv4.sa = rte_be_to_cpu_32(saddr.s_addr);
+
+			if (softnic_parse_ipv4_addr(tokens[7], &daddr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "da");
+				return 0;
+			}
+			m->match.acl.ipv4.da = rte_be_to_cpu_32(daddr.s_addr);
+		} else if (strcmp(tokens[4], "ipv6") == 0) {
+			struct in6_addr saddr, daddr;
+
+			m->match.acl.ip_version = 0;
+
+			if (softnic_parse_ipv6_addr(tokens[5], &saddr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "sa");
+				return 0;
+			}
+			memcpy(m->match.acl.ipv6.sa, saddr.s6_addr, 16);
+
+			if (softnic_parse_ipv6_addr(tokens[7], &daddr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "da");
+				return 0;
+			}
+			memcpy(m->match.acl.ipv6.da, daddr.s6_addr, 16);
+		} else {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+				"ipv4 or ipv6");
+			return 0;
+		}
+
+		if (softnic_parser_read_uint32(&m->match.acl.sa_depth,
+			tokens[6]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "sa_depth");
+			return 0;
+		}
+
+		if (softnic_parser_read_uint32(&m->match.acl.da_depth,
+			tokens[8]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "da_depth");
+			return 0;
+		}
+
+		if (softnic_parser_read_uint16(&m->match.acl.sp0, tokens[9]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "sp0");
+			return 0;
+		}
+
+		if (softnic_parser_read_uint16(&m->match.acl.sp1, tokens[10]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "sp1");
+			return 0;
+		}
+
+		if (softnic_parser_read_uint16(&m->match.acl.dp0, tokens[11]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "dp0");
+			return 0;
+		}
+
+		if (softnic_parser_read_uint16(&m->match.acl.dp1, tokens[12]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "dp1");
+			return 0;
+		}
+
+		if (softnic_parser_read_uint8(&m->match.acl.proto, tokens[13]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "proto");
+			return 0;
+		}
+
+		m->match.acl.proto_mask = 0xff;
+
+		return 14;
+	} /* acl */
+
+	if (strcmp(tokens[1], "array") == 0) {
+		if (n_tokens < 3) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+			return 0;
+		}
+
+		m->match_type = TABLE_ARRAY;
+
+		if (softnic_parser_read_uint32(&m->match.array.pos, tokens[2]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "pos");
+			return 0;
+		}
+
+		return 3;
+	} /* array */
+
+	if (strcmp(tokens[1], "hash") == 0) {
+		if (n_tokens < 3) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+			return 0;
+		}
+
+		m->match_type = TABLE_HASH;
+
+		if (strcmp(tokens[2], "raw") == 0) {
+			uint32_t key_size = TABLE_RULE_MATCH_SIZE_MAX;
+
+			if (n_tokens < 4) {
+				snprintf(out, out_size, MSG_ARG_MISMATCH,
+					tokens[0]);
+				return 0;
+			}
+
+			if (softnic_parse_hex_string(tokens[3],
+				m->match.hash.key, &key_size) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "key");
+				return 0;
+			}
+
+			return 4;
+		} /* hash raw */
+
+		if (strcmp(tokens[2], "ipv4_5tuple") == 0) {
+			struct pkt_key_ipv4_5tuple *ipv4 =
+				(struct pkt_key_ipv4_5tuple *)m->match.hash.key;
+			struct in_addr saddr, daddr;
+			uint16_t sp, dp;
+			uint8_t proto;
+
+			if (n_tokens < 8) {
+				snprintf(out, out_size, MSG_ARG_MISMATCH,
+					tokens[0]);
+				return 0;
+			}
+
+			if (softnic_parse_ipv4_addr(tokens[3], &saddr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "sa");
+				return 0;
+			}
+
+			if (softnic_parse_ipv4_addr(tokens[4], &daddr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "da");
+				return 0;
+			}
+
+			if (softnic_parser_read_uint16(&sp, tokens[5]) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "sp");
+				return 0;
+			}
+
+			if (softnic_parser_read_uint16(&dp, tokens[6]) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "dp");
+				return 0;
+			}
+
+			if (softnic_parser_read_uint8(&proto, tokens[7]) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"proto");
+				return 0;
+			}
+
+			ipv4->sa = saddr.s_addr;
+			ipv4->da = daddr.s_addr;
+			ipv4->sp = rte_cpu_to_be_16(sp);
+			ipv4->dp = rte_cpu_to_be_16(dp);
+			ipv4->proto = proto;
+
+			return 8;
+		} /* hash ipv4_5tuple */
+
+		if (strcmp(tokens[2], "ipv6_5tuple") == 0) {
+			struct pkt_key_ipv6_5tuple *ipv6 =
+				(struct pkt_key_ipv6_5tuple *)m->match.hash.key;
+			struct in6_addr saddr, daddr;
+			uint16_t sp, dp;
+			uint8_t proto;
+
+			if (n_tokens < 8) {
+				snprintf(out, out_size, MSG_ARG_MISMATCH,
+					tokens[0]);
+				return 0;
+			}
+
+			if (softnic_parse_ipv6_addr(tokens[3], &saddr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "sa");
+				return 0;
+			}
+
+			if (softnic_parse_ipv6_addr(tokens[4], &daddr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "da");
+				return 0;
+			}
+
+			if (softnic_parser_read_uint16(&sp, tokens[5]) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "sp");
+				return 0;
+			}
+
+			if (softnic_parser_read_uint16(&dp, tokens[6]) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID, "dp");
+				return 0;
+			}
+
+			if (softnic_parser_read_uint8(&proto, tokens[7]) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"proto");
+				return 0;
+			}
+
+			memcpy(ipv6->sa, saddr.s6_addr, 16);
+			memcpy(ipv6->da, daddr.s6_addr, 16);
+			ipv6->sp = rte_cpu_to_be_16(sp);
+			ipv6->dp = rte_cpu_to_be_16(dp);
+			ipv6->proto = proto;
+
+			return 8;
+		} /* hash ipv6_5tuple */
+
+		if (strcmp(tokens[2], "ipv4_addr") == 0) {
+			struct pkt_key_ipv4_addr *ipv4_addr =
+				(struct pkt_key_ipv4_addr *)m->match.hash.key;
+			struct in_addr addr;
+
+			if (n_tokens < 4) {
+				snprintf(out, out_size, MSG_ARG_MISMATCH,
+					tokens[0]);
+				return 0;
+			}
+
+			if (softnic_parse_ipv4_addr(tokens[3], &addr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"addr");
+				return 0;
+			}
+
+			ipv4_addr->addr = addr.s_addr;
+
+			return 4;
+		} /* hash ipv4_addr */
+
+		if (strcmp(tokens[2], "ipv6_addr") == 0) {
+			struct pkt_key_ipv6_addr *ipv6_addr =
+				(struct pkt_key_ipv6_addr *)m->match.hash.key;
+			struct in6_addr addr;
+
+			if (n_tokens < 4) {
+				snprintf(out, out_size, MSG_ARG_MISMATCH,
+					tokens[0]);
+				return 0;
+			}
+
+			if (softnic_parse_ipv6_addr(tokens[3], &addr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"addr");
+				return 0;
+			}
+
+			memcpy(ipv6_addr->addr, addr.s6_addr, 16);
+
+			return 4;
+		} /* hash ipv6_5tuple */
+
+		if (strcmp(tokens[2], "qinq") == 0) {
+			struct pkt_key_qinq *qinq =
+				(struct pkt_key_qinq *)m->match.hash.key;
+			uint16_t svlan, cvlan;
+
+			if (n_tokens < 5) {
+				snprintf(out, out_size, MSG_ARG_MISMATCH,
+					tokens[0]);
+				return 0;
+			}
+
+			if ((softnic_parser_read_uint16(&svlan, tokens[3]) != 0) ||
+				svlan > 0xFFF) {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"svlan");
+				return 0;
+			}
+
+			if ((softnic_parser_read_uint16(&cvlan, tokens[4]) != 0) ||
+				cvlan > 0xFFF) {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"cvlan");
+				return 0;
+			}
+
+			qinq->svlan = rte_cpu_to_be_16(svlan);
+			qinq->cvlan = rte_cpu_to_be_16(cvlan);
+
+			return 5;
+		} /* hash qinq */
+
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return 0;
+	} /* hash */
+
+	if (strcmp(tokens[1], "lpm") == 0) {
+		if (n_tokens < 5) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+			return 0;
+		}
+
+		m->match_type = TABLE_LPM;
+
+		if (strcmp(tokens[2], "ipv4") == 0) {
+			struct in_addr addr;
+
+			m->match.lpm.ip_version = 1;
+
+			if (softnic_parse_ipv4_addr(tokens[3], &addr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"addr");
+				return 0;
+			}
+
+			m->match.lpm.ipv4 = rte_be_to_cpu_32(addr.s_addr);
+		} else if (strcmp(tokens[2], "ipv6") == 0) {
+			struct in6_addr addr;
+
+			m->match.lpm.ip_version = 0;
+
+			if (softnic_parse_ipv6_addr(tokens[3], &addr) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"addr");
+				return 0;
+			}
+
+			memcpy(m->match.lpm.ipv6, addr.s6_addr, 16);
+		} else {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"ipv4 or ipv6");
+			return 0;
+		}
+
+		if (softnic_parser_read_uint8(&m->match.lpm.depth, tokens[4]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "depth");
+			return 0;
+		}
+
+		return 5;
+	} /* lpm */
+
+	snprintf(out, out_size, MSG_ARG_MISMATCH,
+		"acl or array or hash or lpm");
+	return 0;
+}
+
+/**
+ * table_action ::=
+ *
+ * action
+ *    fwd
+ *       drop
+ *       | port <port_id>
+ *       | meta
+ *       | table <table_id>
+ *    [balance <out0> ... <out7>]
+ *    [meter
+ *       tc0 meter <meter_profile_id> policer g <pa> y <pa> r <pa>
+ *       [tc1 meter <meter_profile_id> policer g <pa> y <pa> r <pa>
+ *       tc2 meter <meter_profile_id> policer g <pa> y <pa> r <pa>
+ *       tc3 meter <meter_profile_id> policer g <pa> y <pa> r <pa>]]
+ *    [tm subport <subport_id> pipe <pipe_id>]
+ *    [encap
+ *       ether <da> <sa>
+ *       | vlan <da> <sa> <pcp> <dei> <vid>
+ *       | qinq <da> <sa> <pcp> <dei> <vid> <pcp> <dei> <vid>
+ *       | mpls unicast | multicast
+ *          <da> <sa>
+ *          label0 <label> <tc> <ttl>
+ *          [label1 <label> <tc> <ttl>
+ *          [label2 <label> <tc> <ttl>
+ *          [label3 <label> <tc> <ttl>]]]
+ *       | pppoe <da> <sa> <session_id>]
+ *    [nat ipv4 | ipv6 <addr> <port>]
+ *    [ttl dec | keep]
+ *    [stats]
+ *    [time]
+ *
+ * where:
+ *    <pa> ::= g | y | r | drop
+ */
+static uint32_t
+parse_table_action_fwd(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	if (n_tokens == 0 ||
+		(strcmp(tokens[0], "fwd") != 0))
+		return 0;
+
+	tokens++;
+	n_tokens--;
+
+	if (n_tokens && (strcmp(tokens[0], "drop") == 0)) {
+		a->fwd.action = RTE_PIPELINE_ACTION_DROP;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_FWD;
+		return 1 + 1;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "port") == 0)) {
+		uint32_t id;
+
+		if (n_tokens < 2 ||
+			softnic_parser_read_uint32(&id, tokens[1]))
+			return 0;
+
+		a->fwd.action = RTE_PIPELINE_ACTION_PORT;
+		a->fwd.id = id;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_FWD;
+		return 1 + 2;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "meta") == 0)) {
+		a->fwd.action = RTE_PIPELINE_ACTION_PORT_META;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_FWD;
+		return 1 + 1;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "table") == 0)) {
+		uint32_t id;
+
+		if (n_tokens < 2 ||
+			softnic_parser_read_uint32(&id, tokens[1]))
+			return 0;
+
+		a->fwd.action = RTE_PIPELINE_ACTION_TABLE;
+		a->fwd.id = id;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_FWD;
+		return 1 + 2;
+	}
+
+	return 0;
+}
+
+static uint32_t
+parse_table_action_balance(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	uint32_t i;
+
+	if (n_tokens == 0 ||
+		(strcmp(tokens[0], "balance") != 0))
+		return 0;
+
+	tokens++;
+	n_tokens--;
+
+	if (n_tokens < RTE_TABLE_ACTION_LB_TABLE_SIZE)
+		return 0;
+
+	for (i = 0; i < RTE_TABLE_ACTION_LB_TABLE_SIZE; i++)
+		if (softnic_parser_read_uint32(&a->lb.out[i], tokens[i]) != 0)
+			return 0;
+
+	a->action_mask |= 1 << RTE_TABLE_ACTION_LB;
+	return 1 + RTE_TABLE_ACTION_LB_TABLE_SIZE;
+}
+
+static int
+parse_policer_action(char *token, enum rte_table_action_policer *a)
+{
+	if (strcmp(token, "g") == 0) {
+		*a = RTE_TABLE_ACTION_POLICER_COLOR_GREEN;
+		return 0;
+	}
+
+	if (strcmp(token, "y") == 0) {
+		*a = RTE_TABLE_ACTION_POLICER_COLOR_YELLOW;
+		return 0;
+	}
+
+	if (strcmp(token, "r") == 0) {
+		*a = RTE_TABLE_ACTION_POLICER_COLOR_RED;
+		return 0;
+	}
+
+	if (strcmp(token, "drop") == 0) {
+		*a = RTE_TABLE_ACTION_POLICER_DROP;
+		return 0;
+	}
+
+	return -1;
+}
+
+static uint32_t
+parse_table_action_meter_tc(char **tokens,
+	uint32_t n_tokens,
+	struct rte_table_action_mtr_tc_params *mtr)
+{
+	if (n_tokens < 9 ||
+		strcmp(tokens[0], "meter") ||
+		softnic_parser_read_uint32(&mtr->meter_profile_id, tokens[1]) ||
+		strcmp(tokens[2], "policer") ||
+		strcmp(tokens[3], "g") ||
+		parse_policer_action(tokens[4], &mtr->policer[e_RTE_METER_GREEN]) ||
+		strcmp(tokens[5], "y") ||
+		parse_policer_action(tokens[6], &mtr->policer[e_RTE_METER_YELLOW]) ||
+		strcmp(tokens[7], "r") ||
+		parse_policer_action(tokens[8], &mtr->policer[e_RTE_METER_RED]))
+		return 0;
+
+	return 9;
+}
+
+static uint32_t
+parse_table_action_meter(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	if (n_tokens == 0 ||
+		strcmp(tokens[0], "meter"))
+		return 0;
+
+	tokens++;
+	n_tokens--;
+
+	if (n_tokens < 10 ||
+		strcmp(tokens[0], "tc0") ||
+		(parse_table_action_meter_tc(tokens + 1,
+			n_tokens - 1,
+			&a->mtr.mtr[0]) == 0))
+		return 0;
+
+	tokens += 10;
+	n_tokens -= 10;
+
+	if (n_tokens == 0 ||
+		strcmp(tokens[0], "tc1")) {
+		a->mtr.tc_mask = 1;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_MTR;
+		return 1 + 10;
+	}
+
+	if (n_tokens < 30 ||
+		(parse_table_action_meter_tc(tokens + 1,
+			n_tokens - 1, &a->mtr.mtr[1]) == 0) ||
+		strcmp(tokens[10], "tc2") ||
+		(parse_table_action_meter_tc(tokens + 11,
+			n_tokens - 11, &a->mtr.mtr[2]) == 0) ||
+		strcmp(tokens[20], "tc3") ||
+		(parse_table_action_meter_tc(tokens + 21,
+			n_tokens - 21, &a->mtr.mtr[3]) == 0))
+		return 0;
+
+	a->mtr.tc_mask = 0xF;
+	a->action_mask |= 1 << RTE_TABLE_ACTION_MTR;
+	return 1 + 10 + 3 * 10;
+}
+
+static uint32_t
+parse_table_action_tm(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	uint32_t subport_id, pipe_id;
+
+	if (n_tokens < 5 ||
+		strcmp(tokens[0], "tm") ||
+		strcmp(tokens[1], "subport") ||
+		softnic_parser_read_uint32(&subport_id, tokens[2]) ||
+		strcmp(tokens[3], "pipe") ||
+		softnic_parser_read_uint32(&pipe_id, tokens[4]))
+		return 0;
+
+	a->tm.subport_id = subport_id;
+	a->tm.pipe_id = pipe_id;
+	a->action_mask |= 1 << RTE_TABLE_ACTION_TM;
+	return 5;
+}
+
+static uint32_t
+parse_table_action_encap(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	if (n_tokens == 0 ||
+		strcmp(tokens[0], "encap"))
+		return 0;
+
+	tokens++;
+	n_tokens--;
+
+	/* ether */
+	if (n_tokens && (strcmp(tokens[0], "ether") == 0)) {
+		if (n_tokens < 3 ||
+			softnic_parse_mac_addr(tokens[1], &a->encap.ether.ether.da) ||
+			softnic_parse_mac_addr(tokens[2], &a->encap.ether.ether.sa))
+			return 0;
+
+		a->encap.type = RTE_TABLE_ACTION_ENCAP_ETHER;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_ENCAP;
+		return 1 + 3;
+	}
+
+	/* vlan */
+	if (n_tokens && (strcmp(tokens[0], "vlan") == 0)) {
+		uint32_t pcp, dei, vid;
+
+		if (n_tokens < 6 ||
+			softnic_parse_mac_addr(tokens[1], &a->encap.vlan.ether.da) ||
+			softnic_parse_mac_addr(tokens[2], &a->encap.vlan.ether.sa) ||
+			softnic_parser_read_uint32(&pcp, tokens[3]) ||
+			pcp > 0x7 ||
+			softnic_parser_read_uint32(&dei, tokens[4]) ||
+			dei > 0x1 ||
+			softnic_parser_read_uint32(&vid, tokens[5]) ||
+			vid > 0xFFF)
+			return 0;
+
+		a->encap.vlan.vlan.pcp = pcp & 0x7;
+		a->encap.vlan.vlan.dei = dei & 0x1;
+		a->encap.vlan.vlan.vid = vid & 0xFFF;
+		a->encap.type = RTE_TABLE_ACTION_ENCAP_VLAN;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_ENCAP;
+		return 1 + 6;
+	}
+
+	/* qinq */
+	if (n_tokens && (strcmp(tokens[0], "qinq") == 0)) {
+		uint32_t svlan_pcp, svlan_dei, svlan_vid;
+		uint32_t cvlan_pcp, cvlan_dei, cvlan_vid;
+
+		if (n_tokens < 9 ||
+			softnic_parse_mac_addr(tokens[1], &a->encap.qinq.ether.da) ||
+			softnic_parse_mac_addr(tokens[2], &a->encap.qinq.ether.sa) ||
+			softnic_parser_read_uint32(&svlan_pcp, tokens[3]) ||
+			svlan_pcp > 0x7 ||
+			softnic_parser_read_uint32(&svlan_dei, tokens[4]) ||
+			svlan_dei > 0x1 ||
+			softnic_parser_read_uint32(&svlan_vid, tokens[5]) ||
+			svlan_vid > 0xFFF ||
+			softnic_parser_read_uint32(&cvlan_pcp, tokens[6]) ||
+			cvlan_pcp > 0x7 ||
+			softnic_parser_read_uint32(&cvlan_dei, tokens[7]) ||
+			cvlan_dei > 0x1 ||
+			softnic_parser_read_uint32(&cvlan_vid, tokens[8]) ||
+			cvlan_vid > 0xFFF)
+			return 0;
+
+		a->encap.qinq.svlan.pcp = svlan_pcp & 0x7;
+		a->encap.qinq.svlan.dei = svlan_dei & 0x1;
+		a->encap.qinq.svlan.vid = svlan_vid & 0xFFF;
+		a->encap.qinq.cvlan.pcp = cvlan_pcp & 0x7;
+		a->encap.qinq.cvlan.dei = cvlan_dei & 0x1;
+		a->encap.qinq.cvlan.vid = cvlan_vid & 0xFFF;
+		a->encap.type = RTE_TABLE_ACTION_ENCAP_QINQ;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_ENCAP;
+		return 1 + 9;
+	}
+
+	/* mpls */
+	if (n_tokens && (strcmp(tokens[0], "mpls") == 0)) {
+		uint32_t label, tc, ttl;
+
+		if (n_tokens < 8)
+			return 0;
+
+		if (strcmp(tokens[1], "unicast") == 0)
+			a->encap.mpls.unicast = 1;
+		else if (strcmp(tokens[1], "multicast") == 0)
+			a->encap.mpls.unicast = 0;
+		else
+			return 0;
+
+		if (softnic_parse_mac_addr(tokens[2], &a->encap.mpls.ether.da) ||
+			softnic_parse_mac_addr(tokens[3], &a->encap.mpls.ether.sa) ||
+			strcmp(tokens[4], "label0") ||
+			softnic_parser_read_uint32(&label, tokens[5]) ||
+			label > 0xFFFFF ||
+			softnic_parser_read_uint32(&tc, tokens[6]) ||
+			tc > 0x7 ||
+			softnic_parser_read_uint32(&ttl, tokens[7]) ||
+			ttl > 0x3F)
+			return 0;
+
+		a->encap.mpls.mpls[0].label = label;
+		a->encap.mpls.mpls[0].tc = tc;
+		a->encap.mpls.mpls[0].ttl = ttl;
+
+		tokens += 8;
+		n_tokens -= 8;
+
+		if (n_tokens == 0 ||
+			strcmp(tokens[0], "label1")) {
+			a->encap.mpls.mpls_count = 1;
+			a->encap.type = RTE_TABLE_ACTION_ENCAP_MPLS;
+			a->action_mask |= 1 << RTE_TABLE_ACTION_ENCAP;
+			return 1 + 8;
+		}
+
+		if (n_tokens < 4 ||
+			softnic_parser_read_uint32(&label, tokens[1]) ||
+			label > 0xFFFFF ||
+			softnic_parser_read_uint32(&tc, tokens[2]) ||
+			tc > 0x7 ||
+			softnic_parser_read_uint32(&ttl, tokens[3]) ||
+			ttl > 0x3F)
+			return 0;
+
+		a->encap.mpls.mpls[1].label = label;
+		a->encap.mpls.mpls[1].tc = tc;
+		a->encap.mpls.mpls[1].ttl = ttl;
+
+		tokens += 4;
+		n_tokens -= 4;
+
+		if (n_tokens == 0 ||
+			strcmp(tokens[0], "label2")) {
+			a->encap.mpls.mpls_count = 2;
+			a->encap.type = RTE_TABLE_ACTION_ENCAP_MPLS;
+			a->action_mask |= 1 << RTE_TABLE_ACTION_ENCAP;
+			return 1 + 8 + 4;
+		}
+
+		if (n_tokens < 4 ||
+			softnic_parser_read_uint32(&label, tokens[1]) ||
+			label > 0xFFFFF ||
+			softnic_parser_read_uint32(&tc, tokens[2]) ||
+			tc > 0x7 ||
+			softnic_parser_read_uint32(&ttl, tokens[3]) ||
+			ttl > 0x3F)
+			return 0;
+
+		a->encap.mpls.mpls[2].label = label;
+		a->encap.mpls.mpls[2].tc = tc;
+		a->encap.mpls.mpls[2].ttl = ttl;
+
+		tokens += 4;
+		n_tokens -= 4;
+
+		if (n_tokens == 0 ||
+			strcmp(tokens[0], "label3")) {
+			a->encap.mpls.mpls_count = 3;
+			a->encap.type = RTE_TABLE_ACTION_ENCAP_MPLS;
+			a->action_mask |= 1 << RTE_TABLE_ACTION_ENCAP;
+			return 1 + 8 + 4 + 4;
+		}
+
+		if (n_tokens < 4 ||
+			softnic_parser_read_uint32(&label, tokens[1]) ||
+			label > 0xFFFFF ||
+			softnic_parser_read_uint32(&tc, tokens[2]) ||
+			tc > 0x7 ||
+			softnic_parser_read_uint32(&ttl, tokens[3]) ||
+			ttl > 0x3F)
+			return 0;
+
+		a->encap.mpls.mpls[3].label = label;
+		a->encap.mpls.mpls[3].tc = tc;
+		a->encap.mpls.mpls[3].ttl = ttl;
+
+		a->encap.mpls.mpls_count = 4;
+		a->encap.type = RTE_TABLE_ACTION_ENCAP_MPLS;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_ENCAP;
+		return 1 + 8 + 4 + 4 + 4;
+	}
+
+	/* pppoe */
+	if (n_tokens && (strcmp(tokens[0], "pppoe") == 0)) {
+		if (n_tokens < 4 ||
+			softnic_parse_mac_addr(tokens[1], &a->encap.pppoe.ether.da) ||
+			softnic_parse_mac_addr(tokens[2], &a->encap.pppoe.ether.sa) ||
+			softnic_parser_read_uint16(&a->encap.pppoe.pppoe.session_id,
+				tokens[3]))
+			return 0;
+
+		a->encap.type = RTE_TABLE_ACTION_ENCAP_PPPOE;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_ENCAP;
+		return 1 + 4;
+	}
+
+	return 0;
+}
+
+static uint32_t
+parse_table_action_nat(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	if (n_tokens < 4 ||
+		strcmp(tokens[0], "nat"))
+		return 0;
+
+	if (strcmp(tokens[1], "ipv4") == 0) {
+		struct in_addr addr;
+		uint16_t port;
+
+		if (softnic_parse_ipv4_addr(tokens[2], &addr) ||
+			softnic_parser_read_uint16(&port, tokens[3]))
+			return 0;
+
+		a->nat.ip_version = 1;
+		a->nat.addr.ipv4 = rte_be_to_cpu_32(addr.s_addr);
+		a->nat.port = port;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_NAT;
+		return 4;
+	}
+
+	if (strcmp(tokens[1], "ipv6") == 0) {
+		struct in6_addr addr;
+		uint16_t port;
+
+		if (softnic_parse_ipv6_addr(tokens[2], &addr) ||
+			softnic_parser_read_uint16(&port, tokens[3]))
+			return 0;
+
+		a->nat.ip_version = 0;
+		memcpy(a->nat.addr.ipv6, addr.s6_addr, 16);
+		a->nat.port = port;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_NAT;
+		return 4;
+	}
+
+	return 0;
+}
+
+static uint32_t
+parse_table_action_ttl(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	if (n_tokens < 2 ||
+		strcmp(tokens[0], "ttl"))
+		return 0;
+
+	if (strcmp(tokens[1], "dec") == 0)
+		a->ttl.decrement = 1;
+	else if (strcmp(tokens[1], "keep") == 0)
+		a->ttl.decrement = 0;
+	else
+		return 0;
+
+	a->action_mask |= 1 << RTE_TABLE_ACTION_TTL;
+	return 2;
+}
+
+static uint32_t
+parse_table_action_stats(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	if (n_tokens < 1 ||
+		strcmp(tokens[0], "stats"))
+		return 0;
+
+	a->stats.n_packets = 0;
+	a->stats.n_bytes = 0;
+	a->action_mask |= 1 << RTE_TABLE_ACTION_STATS;
+	return 1;
+}
+
+static uint32_t
+parse_table_action_time(char **tokens,
+	uint32_t n_tokens,
+	struct softnic_table_rule_action *a)
+{
+	if (n_tokens < 1 ||
+		strcmp(tokens[0], "time"))
+		return 0;
+
+	a->time.time = rte_rdtsc();
+	a->action_mask |= 1 << RTE_TABLE_ACTION_TIME;
+	return 1;
+}
+
+static uint32_t
+parse_table_action(char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size,
+	struct softnic_table_rule_action *a)
+{
+	uint32_t n_tokens0 = n_tokens;
+
+	memset(a, 0, sizeof(*a));
+
+	if (n_tokens < 2 ||
+		strcmp(tokens[0], "action"))
+		return 0;
+
+	tokens++;
+	n_tokens--;
+
+	if (n_tokens && (strcmp(tokens[0], "fwd") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_fwd(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action fwd");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "balance") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_balance(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action balance");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "meter") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_meter(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action meter");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "tm") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_tm(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action tm");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "encap") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_encap(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action encap");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "nat") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_nat(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action nat");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "ttl") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_ttl(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action ttl");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "stats") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_stats(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action stats");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "time") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_time(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action time");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens0 - n_tokens == 1) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "action");
+		return 0;
+	}
+
+	return n_tokens0 - n_tokens;
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id> rule add
+ *    match <match>
+ *    action <table_action>
+ */
+static void
+cmd_softnic_pipeline_table_rule_add(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct softnic_table_rule_match m;
+	struct softnic_table_rule_action a;
+	char *pipeline_name;
+	void *data;
+	uint32_t table_id, t0, n_tokens_parsed;
+	int status;
+
+	if (n_tokens < 8) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "rule") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rule");
+		return;
+	}
+
+	if (strcmp(tokens[5], "add") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "add");
+		return;
+	}
+
+	t0 = 6;
+
+	/* match */
+	n_tokens_parsed = parse_match(tokens + t0,
+		n_tokens - t0,
+		out,
+		out_size,
+		&m);
+	if (n_tokens_parsed == 0)
+		return;
+	t0 += n_tokens_parsed;
+
+	/* action */
+	n_tokens_parsed = parse_table_action(tokens + t0,
+		n_tokens - t0,
+		out,
+		out_size,
+		&a);
+	if (n_tokens_parsed == 0)
+		return;
+	t0 += n_tokens_parsed;
+
+	if (t0 != n_tokens) {
+		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
+		return;
+	}
+
+	status = softnic_pipeline_table_rule_add(softnic,
+		pipeline_name,
+		table_id,
+		&m,
+		&a,
+		&data);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id> rule add
+ *    match
+ *       default
+ *    action
+ *       fwd
+ *          drop
+ *          | port <port_id>
+ *          | meta
+ *          | table <table_id>
+ */
+static void
+cmd_softnic_pipeline_table_rule_add_default(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct softnic_table_rule_action action;
+	void *data;
+	char *pipeline_name;
+	uint32_t table_id;
+	int status;
+
+	if (n_tokens != 11 &&
+		n_tokens != 12) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "rule") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rule");
+		return;
+	}
+
+	if (strcmp(tokens[5], "add") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "add");
+		return;
+	}
+
+	if (strcmp(tokens[6], "match") != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "match");
+		return;
+	}
+
+	if (strcmp(tokens[7], "default") != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "default");
+		return;
+	}
+
+	if (strcmp(tokens[8], "action") != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "action");
+		return;
+	}
+
+	if (strcmp(tokens[9], "fwd") != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "fwd");
+		return;
+	}
+
+	action.action_mask = 1 << RTE_TABLE_ACTION_FWD;
+
+	if (strcmp(tokens[10], "drop") == 0) {
+		if (n_tokens != 11) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+			return;
+		}
+
+		action.fwd.action = RTE_PIPELINE_ACTION_DROP;
+	} else if (strcmp(tokens[10], "port") == 0) {
+		uint32_t id;
+
+		if (n_tokens != 12) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&id, tokens[11]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "port_id");
+			return;
+		}
+
+		action.fwd.action = RTE_PIPELINE_ACTION_PORT;
+		action.fwd.id = id;
+	} else if (strcmp(tokens[10], "meta") == 0) {
+		if (n_tokens != 11) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+			return;
+		}
+
+		action.fwd.action = RTE_PIPELINE_ACTION_PORT_META;
+	} else if (strcmp(tokens[10], "table") == 0) {
+		uint32_t id;
+
+		if (n_tokens != 12) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+			return;
+		}
+
+		if (softnic_parser_read_uint32(&id, tokens[11]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+			return;
+		}
+
+		action.fwd.action = RTE_PIPELINE_ACTION_TABLE;
+		action.fwd.id = id;
+	} else {
+		snprintf(out, out_size, MSG_ARG_INVALID,
+			"drop or port or meta or table");
+		return;
+	}
+
+	status = softnic_pipeline_table_rule_add_default(softnic,
+		pipeline_name,
+		table_id,
+		&action,
+		&data);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id> rule add bulk <file_name> <n_rules>
+ *
+ * File <file_name>:
+ * - line format: match <match> action <action>
+ */
+static int
+cli_rule_file_process(const char *file_name,
+	size_t line_len_max,
+	struct softnic_table_rule_match *m,
+	struct softnic_table_rule_action *a,
+	uint32_t *n_rules,
+	uint32_t *line_number,
+	char *out,
+	size_t out_size);
+
+static void
+cmd_softnic_pipeline_table_rule_add_bulk(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct softnic_table_rule_match *match;
+	struct softnic_table_rule_action *action;
+	void **data;
+	char *pipeline_name, *file_name;
+	uint32_t table_id, n_rules, n_rules_parsed, line_number;
+	int status;
+
+	if (n_tokens != 9) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "rule") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rule");
+		return;
+	}
+
+	if (strcmp(tokens[5], "add") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "add");
+		return;
+	}
+
+	if (strcmp(tokens[6], "bulk") != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "bulk");
+		return;
+	}
+
+	file_name = tokens[7];
+
+	if ((softnic_parser_read_uint32(&n_rules, tokens[8]) != 0) ||
+		n_rules == 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "n_rules");
+		return;
+	}
+
+	/* Memory allocation. */
+	match = calloc(n_rules, sizeof(struct softnic_table_rule_match));
+	action = calloc(n_rules, sizeof(struct softnic_table_rule_action));
+	data = calloc(n_rules, sizeof(void *));
+	if (match == NULL ||
+		action == NULL ||
+		data == NULL) {
+		snprintf(out, out_size, MSG_OUT_OF_MEMORY);
+		free(data);
+		free(action);
+		free(match);
+		return;
+	}
+
+	/* Load rule file */
+	n_rules_parsed = n_rules;
+	status = cli_rule_file_process(file_name,
+		1024,
+		match,
+		action,
+		&n_rules_parsed,
+		&line_number,
+		out,
+		out_size);
+	if (status) {
+		snprintf(out, out_size, MSG_FILE_ERR, file_name, line_number);
+		free(data);
+		free(action);
+		free(match);
+		return;
+	}
+	if (n_rules_parsed != n_rules) {
+		snprintf(out, out_size, MSG_FILE_NOT_ENOUGH, file_name);
+		free(data);
+		free(action);
+		free(match);
+		return;
+	}
+
+	/* Rule bulk add */
+	status = softnic_pipeline_table_rule_add_bulk(softnic,
+		pipeline_name,
+		table_id,
+		match,
+		action,
+		data,
+		&n_rules);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		free(data);
+		free(action);
+		free(match);
+		return;
+	}
+
+	/* Memory free */
+	free(data);
+	free(action);
+	free(match);
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id> rule delete
+ *    match <match>
+ */
+static void
+cmd_softnic_pipeline_table_rule_delete(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct softnic_table_rule_match m;
+	char *pipeline_name;
+	uint32_t table_id, n_tokens_parsed, t0;
+	int status;
+
+	if (n_tokens < 8) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "rule") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rule");
+		return;
+	}
+
+	if (strcmp(tokens[5], "delete") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "delete");
+		return;
+	}
+
+	t0 = 6;
+
+	/* match */
+	n_tokens_parsed = parse_match(tokens + t0,
+		n_tokens - t0,
+		out,
+		out_size,
+		&m);
+	if (n_tokens_parsed == 0)
+		return;
+	t0 += n_tokens_parsed;
+
+	if (n_tokens != t0) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	status = softnic_pipeline_table_rule_delete(softnic,
+		pipeline_name,
+		table_id,
+		&m);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id> rule delete
+ *    match
+ *       default
+ */
+static void
+cmd_softnic_pipeline_table_rule_delete_default(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	char *pipeline_name;
+	uint32_t table_id;
+	int status;
+
+	if (n_tokens != 8) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "rule") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rule");
+		return;
+	}
+
+	if (strcmp(tokens[5], "delete") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "delete");
+		return;
+	}
+
+	if (strcmp(tokens[6], "match") != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "match");
+		return;
+	}
+
+	if (strcmp(tokens[7], "default") != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "default");
+		return;
+	}
+
+	status = softnic_pipeline_table_rule_delete_default(softnic,
+		pipeline_name,
+		table_id);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id> rule read stats [clear]
+ */
+static void
+cmd_softnic_pipeline_table_rule_stats_read(struct pmd_internals *softnic __rte_unused,
+	char **tokens,
+	uint32_t n_tokens __rte_unused,
+	char *out,
+	size_t out_size)
+{
+	snprintf(out, out_size, MSG_CMD_UNIMPLEM, tokens[0]);
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id> meter profile <meter_profile_id>
+ *  add srtcm cir <cir> cbs <cbs> ebs <ebs>
+ *  | trtcm cir <cir> pir <pir> cbs <cbs> pbs <pbs>
+ */
+static void
+cmd_pipeline_table_meter_profile_add(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct rte_table_action_meter_profile p;
+	char *pipeline_name;
+	uint32_t table_id, meter_profile_id;
+	int status;
+
+	if (n_tokens < 9) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "meter") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter");
+		return;
+	}
+
+	if (strcmp(tokens[5], "profile") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&meter_profile_id, tokens[6]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "meter_profile_id");
+		return;
+	}
+
+	if (strcmp(tokens[7], "add") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "add");
+		return;
+	}
+
+	if (strcmp(tokens[8], "srtcm") == 0) {
+		if (n_tokens != 15) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				tokens[0]);
+			return;
+		}
+
+		p.alg = RTE_TABLE_ACTION_METER_SRTCM;
+
+		if (strcmp(tokens[9], "cir") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "cir");
+			return;
+		}
+
+		if (softnic_parser_read_uint64(&p.srtcm.cir, tokens[10]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "cir");
+			return;
+		}
+
+		if (strcmp(tokens[11], "cbs") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "cbs");
+			return;
+		}
+
+		if (softnic_parser_read_uint64(&p.srtcm.cbs, tokens[12]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "cbs");
+			return;
+		}
+
+		if (strcmp(tokens[13], "ebs") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "ebs");
+			return;
+		}
+
+		if (softnic_parser_read_uint64(&p.srtcm.ebs, tokens[14]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "ebs");
+			return;
+		}
+	} else if (strcmp(tokens[8], "trtcm") == 0) {
+		if (n_tokens != 17) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+			return;
+		}
+
+		p.alg = RTE_TABLE_ACTION_METER_TRTCM;
+
+		if (strcmp(tokens[9], "cir") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "cir");
+			return;
+		}
+
+		if (softnic_parser_read_uint64(&p.trtcm.cir, tokens[10]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "cir");
+			return;
+		}
+
+		if (strcmp(tokens[11], "pir") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pir");
+			return;
+		}
+
+		if (softnic_parser_read_uint64(&p.trtcm.pir, tokens[12]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "pir");
+			return;
+		}
+		if (strcmp(tokens[13], "cbs") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "cbs");
+			return;
+		}
+
+		if (softnic_parser_read_uint64(&p.trtcm.cbs, tokens[14]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "cbs");
+			return;
+		}
+
+		if (strcmp(tokens[15], "pbs") != 0) {
+			snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pbs");
+			return;
+		}
+
+		if (softnic_parser_read_uint64(&p.trtcm.pbs, tokens[16]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "pbs");
+			return;
+		}
+	} else {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	status = softnic_pipeline_table_mtr_profile_add(softnic,
+		pipeline_name,
+		table_id,
+		meter_profile_id,
+		&p);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id>
+ *  meter profile <meter_profile_id> delete
+ */
+static void
+cmd_pipeline_table_meter_profile_delete(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	char *pipeline_name;
+	uint32_t table_id, meter_profile_id;
+	int status;
+
+	if (n_tokens != 8) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "meter") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter");
+		return;
+	}
+
+	if (strcmp(tokens[5], "profile") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "profile");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&meter_profile_id, tokens[6]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "meter_profile_id");
+		return;
+	}
+
+	if (strcmp(tokens[7], "delete") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "delete");
+		return;
+	}
+
+	status = softnic_pipeline_table_mtr_profile_delete(softnic,
+		pipeline_name,
+		table_id,
+		meter_profile_id);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id> rule read meter [clear]
+ */
+static void
+cmd_pipeline_table_rule_meter_read(struct pmd_internals *softnic __rte_unused,
+	char **tokens,
+	uint32_t n_tokens __rte_unused,
+	char *out,
+	size_t out_size)
+{
+	snprintf(out, out_size, MSG_CMD_UNIMPLEM, tokens[0]);
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id> dscp <file_name>
+ *
+ * File <file_name>:
+ *  - exactly 64 lines
+ *  - line format: <tc_id> <tc_queue_id> <color>, with <color> as: g | y | r
+ */
+static int
+load_dscp_table(struct rte_table_action_dscp_table *dscp_table,
+	const char *file_name,
+	uint32_t *line_number)
+{
+	FILE *f = NULL;
+	uint32_t dscp, l;
+
+	/* Check input arguments */
+	if (dscp_table == NULL ||
+		file_name == NULL ||
+		line_number == NULL) {
+		if (line_number)
+			*line_number = 0;
+		return -EINVAL;
+	}
+
+	/* Open input file */
+	f = fopen(file_name, "r");
+	if (f == NULL) {
+		*line_number = 0;
+		return -EINVAL;
+	}
+
+	/* Read file */
+	for (dscp = 0, l = 1; ; l++) {
+		char line[64];
+		char *tokens[3];
+		enum rte_meter_color color;
+		uint32_t tc_id, tc_queue_id, n_tokens = RTE_DIM(tokens);
+
+		if (fgets(line, sizeof(line), f) == NULL)
+			break;
+
+		if (is_comment(line))
+			continue;
+
+		if (softnic_parse_tokenize_string(line, tokens, &n_tokens)) {
+			*line_number = l;
+			fclose(f);
+			return -EINVAL;
+		}
+
+		if (n_tokens == 0)
+			continue;
+
+		if (dscp >= RTE_DIM(dscp_table->entry) ||
+			n_tokens != RTE_DIM(tokens) ||
+			softnic_parser_read_uint32(&tc_id, tokens[0]) ||
+			tc_id >= RTE_TABLE_ACTION_TC_MAX ||
+			softnic_parser_read_uint32(&tc_queue_id, tokens[1]) ||
+			tc_queue_id >= RTE_TABLE_ACTION_TC_QUEUE_MAX ||
+			(strlen(tokens[2]) != 1)) {
+			*line_number = l;
+			fclose(f);
+			return -EINVAL;
+		}
+
+		switch (tokens[2][0]) {
+		case 'g':
+		case 'G':
+			color = e_RTE_METER_GREEN;
+			break;
+
+		case 'y':
+		case 'Y':
+			color = e_RTE_METER_YELLOW;
+			break;
+
+		case 'r':
+		case 'R':
+			color = e_RTE_METER_RED;
+			break;
+
+		default:
+			*line_number = l;
+			fclose(f);
+			return -EINVAL;
+		}
+
+		dscp_table->entry[dscp].tc_id = tc_id;
+		dscp_table->entry[dscp].tc_queue_id = tc_queue_id;
+		dscp_table->entry[dscp].color = color;
+		dscp++;
+	}
+
+	/* Close file */
+	fclose(f);
+	return 0;
+}
+
+static void
+cmd_pipeline_table_dscp(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct rte_table_action_dscp_table dscp_table;
+	char *pipeline_name, *file_name;
+	uint32_t table_id, line_number;
+	int status;
+
+	if (n_tokens != 6) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "port");
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "dscp") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "dscp");
+		return;
+	}
+
+	file_name = tokens[5];
+
+	status = load_dscp_table(&dscp_table, file_name, &line_number);
+	if (status) {
+		snprintf(out, out_size, MSG_FILE_ERR, file_name, line_number);
+		return;
+	}
+
+	status = softnic_pipeline_table_dscp_table_update(softnic,
+		pipeline_name,
+		table_id,
+		UINT64_MAX,
+		&dscp_table);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
+
+/**
+ * pipeline <pipeline_name> table <table_id> rule read ttl [clear]
+ */
+static void
+cmd_softnic_pipeline_table_rule_ttl_read(struct pmd_internals *softnic __rte_unused,
+	char **tokens,
+	uint32_t n_tokens __rte_unused,
+	char *out,
+	size_t out_size)
+{
+	snprintf(out, out_size, MSG_CMD_UNIMPLEM, tokens[0]);
+}
+
+/**
+ * thread <thread_id> pipeline <pipeline_name> enable
+ */
+static void
+cmd_softnic_thread_pipeline_enable(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	char *pipeline_name;
+	uint32_t thread_id;
+	int status;
+
+	if (n_tokens != 5) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&thread_id, tokens[1]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "thread_id");
+		return;
+	}
+
+	if (strcmp(tokens[2], "pipeline") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pipeline");
+		return;
+	}
+
+	pipeline_name = tokens[3];
+
+	if (strcmp(tokens[4], "enable") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "enable");
+		return;
+	}
+
+	status = softnic_thread_pipeline_enable(softnic, thread_id, pipeline_name);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, "thread pipeline enable");
+		return;
+	}
+}
+
+/**
+ * thread <thread_id> pipeline <pipeline_name> disable
+ */
+static void
+cmd_softnic_thread_pipeline_disable(struct pmd_internals *softnic,
+	char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	char *pipeline_name;
+	uint32_t thread_id;
+	int status;
+
+	if (n_tokens != 5) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	if (softnic_parser_read_uint32(&thread_id, tokens[1]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "thread_id");
+		return;
+	}
+
+	if (strcmp(tokens[2], "pipeline") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "pipeline");
+		return;
+	}
+
+	pipeline_name = tokens[3];
+
+	if (strcmp(tokens[4], "disable") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "disable");
+		return;
+	}
+
+	status = softnic_thread_pipeline_disable(softnic, thread_id, pipeline_name);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL,
+			"thread pipeline disable");
+		return;
+	}
+}
+
+void
+softnic_cli_process(char *in, char *out, size_t out_size, void *arg)
+{
+	char *tokens[CMD_MAX_TOKENS];
+	uint32_t n_tokens = RTE_DIM(tokens);
+	struct pmd_internals *softnic = arg;
+	int status;
+
+	if (is_comment(in))
+		return;
+
+	status = softnic_parse_tokenize_string(in, tokens, &n_tokens);
+	if (status) {
+		snprintf(out, out_size, MSG_ARG_TOO_MANY, "");
+		return;
+	}
+
+	if (n_tokens == 0)
+		return;
+
+	if (strcmp(tokens[0], "mempool") == 0) {
+		cmd_mempool(softnic, tokens, n_tokens, out, out_size);
+		return;
+	}
+
+	if (strcmp(tokens[0], "link") == 0) {
+		cmd_link(softnic, tokens, n_tokens, out, out_size);
+		return;
+	}
+
+	if (strcmp(tokens[0], "swq") == 0) {
+		cmd_swq(softnic, tokens, n_tokens, out, out_size);
+		return;
+	}
+
+	if (strcmp(tokens[0], "tmgr") == 0) {
+		if (n_tokens == 2) {
+			cmd_tmgr(softnic, tokens, n_tokens, out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 3 &&
+			(strcmp(tokens[1], "shaper") == 0) &&
+			(strcmp(tokens[2], "profile") == 0)) {
+			cmd_tmgr_shaper_profile(softnic, tokens, n_tokens, out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 3 &&
+			(strcmp(tokens[1], "shared") == 0) &&
+			(strcmp(tokens[2], "shaper") == 0)) {
+			cmd_tmgr_shared_shaper(softnic, tokens, n_tokens, out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 2 &&
+			(strcmp(tokens[1], "node") == 0)) {
+			cmd_tmgr_node(softnic, tokens, n_tokens, out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 2 &&
+			(strcmp(tokens[1], "hierarchy-default") == 0)) {
+			cmd_tmgr_hierarchy_default(softnic, tokens, n_tokens, out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 3 &&
+			(strcmp(tokens[1], "hierarchy") == 0) &&
+			(strcmp(tokens[2], "commit") == 0)) {
+			cmd_tmgr_hierarchy_commit(softnic, tokens, n_tokens, out, out_size);
+			return;
+		}
+	}
+
+	if (strcmp(tokens[0], "tap") == 0) {
+		cmd_tap(softnic, tokens, n_tokens, out, out_size);
+		return;
+	}
+
+	if (strcmp(tokens[0], "port") == 0) {
+		cmd_port_in_action_profile(softnic, tokens, n_tokens, out, out_size);
+		return;
+	}
+
+	if (strcmp(tokens[0], "table") == 0) {
+		cmd_table_action_profile(softnic, tokens, n_tokens, out, out_size);
+		return;
+	}
+
+	if (strcmp(tokens[0], "pipeline") == 0) {
+		if (n_tokens >= 3 &&
+			(strcmp(tokens[2], "period") == 0)) {
+			cmd_pipeline(softnic, tokens, n_tokens, out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 5 &&
+			(strcmp(tokens[2], "port") == 0) &&
+			(strcmp(tokens[3], "in") == 0) &&
+			(strcmp(tokens[4], "bsz") == 0)) {
+			cmd_pipeline_port_in(softnic, tokens, n_tokens, out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 5 &&
+			(strcmp(tokens[2], "port") == 0) &&
+			(strcmp(tokens[3], "out") == 0) &&
+			(strcmp(tokens[4], "bsz") == 0)) {
+			cmd_pipeline_port_out(softnic, tokens, n_tokens, out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 4 &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[3], "match") == 0)) {
+			cmd_pipeline_table(softnic, tokens, n_tokens, out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 6 &&
+			(strcmp(tokens[2], "port") == 0) &&
+			(strcmp(tokens[3], "in") == 0) &&
+			(strcmp(tokens[5], "table") == 0)) {
+			cmd_pipeline_port_in_table(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 6 &&
+			(strcmp(tokens[2], "port") == 0) &&
+			(strcmp(tokens[3], "in") == 0) &&
+			(strcmp(tokens[5], "stats") == 0)) {
+			cmd_pipeline_port_in_stats(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 6 &&
+			(strcmp(tokens[2], "port") == 0) &&
+			(strcmp(tokens[3], "in") == 0) &&
+			(strcmp(tokens[5], "enable") == 0)) {
+			cmd_softnic_pipeline_port_in_enable(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 6 &&
+			(strcmp(tokens[2], "port") == 0) &&
+			(strcmp(tokens[3], "in") == 0) &&
+			(strcmp(tokens[5], "disable") == 0)) {
+			cmd_softnic_pipeline_port_in_disable(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 6 &&
+			(strcmp(tokens[2], "port") == 0) &&
+			(strcmp(tokens[3], "out") == 0) &&
+			(strcmp(tokens[5], "stats") == 0)) {
+			cmd_pipeline_port_out_stats(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 5 &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "stats") == 0)) {
+			cmd_pipeline_table_stats(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 7 &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "rule") == 0) &&
+			(strcmp(tokens[5], "add") == 0) &&
+			(strcmp(tokens[6], "match") == 0)) {
+			if (n_tokens >= 8 &&
+				(strcmp(tokens[7], "default") == 0)) {
+				cmd_softnic_pipeline_table_rule_add_default(softnic, tokens,
+					n_tokens, out, out_size);
+				return;
+			}
+
+			cmd_softnic_pipeline_table_rule_add(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 7 &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "rule") == 0) &&
+			(strcmp(tokens[5], "add") == 0) &&
+			(strcmp(tokens[6], "bulk") == 0)) {
+			cmd_softnic_pipeline_table_rule_add_bulk(softnic, tokens,
+				n_tokens, out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 7 &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "rule") == 0) &&
+			(strcmp(tokens[5], "delete") == 0) &&
+			(strcmp(tokens[6], "match") == 0)) {
+			if (n_tokens >= 8 &&
+				(strcmp(tokens[7], "default") == 0)) {
+				cmd_softnic_pipeline_table_rule_delete_default(softnic, tokens,
+					n_tokens, out, out_size);
+				return;
+				}
+
+			cmd_softnic_pipeline_table_rule_delete(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 7 &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "rule") == 0) &&
+			(strcmp(tokens[5], "read") == 0) &&
+			(strcmp(tokens[6], "stats") == 0)) {
+			cmd_softnic_pipeline_table_rule_stats_read(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 8 &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "meter") == 0) &&
+			(strcmp(tokens[5], "profile") == 0) &&
+			(strcmp(tokens[7], "add") == 0)) {
+			cmd_pipeline_table_meter_profile_add(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 8 &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "meter") == 0) &&
+			(strcmp(tokens[5], "profile") == 0) &&
+			(strcmp(tokens[7], "delete") == 0)) {
+			cmd_pipeline_table_meter_profile_delete(softnic, tokens,
+				n_tokens, out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 7 &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "rule") == 0) &&
+			(strcmp(tokens[5], "read") == 0) &&
+			(strcmp(tokens[6], "meter") == 0)) {
+			cmd_pipeline_table_rule_meter_read(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 5 &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "dscp") == 0)) {
+			cmd_pipeline_table_dscp(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 7 &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "rule") == 0) &&
+			(strcmp(tokens[5], "read") == 0) &&
+			(strcmp(tokens[6], "ttl") == 0)) {
+			cmd_softnic_pipeline_table_rule_ttl_read(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+	}
+
+	if (strcmp(tokens[0], "thread") == 0) {
+		if (n_tokens >= 5 &&
+			(strcmp(tokens[4], "enable") == 0)) {
+			cmd_softnic_thread_pipeline_enable(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
+		if (n_tokens >= 5 &&
+			(strcmp(tokens[4], "disable") == 0)) {
+			cmd_softnic_thread_pipeline_disable(softnic, tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+	}
+
+	snprintf(out, out_size, MSG_CMD_UNKNOWN, tokens[0]);
+}
+
+int
+softnic_cli_script_process(struct pmd_internals *softnic,
+	const char *file_name,
+	size_t msg_in_len_max,
+	size_t msg_out_len_max)
+{
+	char *msg_in = NULL, *msg_out = NULL;
+	FILE *f = NULL;
+
+	/* Check input arguments */
+	if (file_name == NULL ||
+		(strlen(file_name) == 0) ||
+		msg_in_len_max == 0 ||
+		msg_out_len_max == 0)
+		return -EINVAL;
+
+	msg_in = malloc(msg_in_len_max + 1);
+	msg_out = malloc(msg_out_len_max + 1);
+	if (msg_in == NULL ||
+		msg_out == NULL) {
+		free(msg_out);
+		free(msg_in);
+		return -ENOMEM;
+	}
+
+	/* Open input file */
+	f = fopen(file_name, "r");
+	if (f == NULL) {
+		free(msg_out);
+		free(msg_in);
+		return -EIO;
+	}
+
+	/* Read file */
+	for ( ; ; ) {
+		if (fgets(msg_in, msg_in_len_max + 1, f) == NULL)
+			break;
+
+		printf("%s", msg_in);
+		msg_out[0] = 0;
+
+		softnic_cli_process(msg_in,
+			msg_out,
+			msg_out_len_max,
+			softnic);
+
+		if (strlen(msg_out))
+			printf("%s", msg_out);
+	}
+
+	/* Close file */
+	fclose(f);
+	free(msg_out);
+	free(msg_in);
+	return 0;
+}
+
+static int
+cli_rule_file_process(const char *file_name,
+	size_t line_len_max,
+	struct softnic_table_rule_match *m,
+	struct softnic_table_rule_action *a,
+	uint32_t *n_rules,
+	uint32_t *line_number,
+	char *out,
+	size_t out_size)
+{
+	FILE *f = NULL;
+	char *line = NULL;
+	uint32_t rule_id, line_id;
+	int status = 0;
+
+	/* Check input arguments */
+	if (file_name == NULL ||
+		(strlen(file_name) == 0) ||
+		line_len_max == 0) {
+		*line_number = 0;
+		return -EINVAL;
+	}
+
+	/* Memory allocation */
+	line = malloc(line_len_max + 1);
+	if (line == NULL) {
+		*line_number = 0;
+		return -ENOMEM;
+	}
+
+	/* Open file */
+	f = fopen(file_name, "r");
+	if (f == NULL) {
+		*line_number = 0;
+		free(line);
+		return -EIO;
+	}
+
+	/* Read file */
+	for (line_id = 1, rule_id = 0; rule_id < *n_rules; line_id++) {
+		char *tokens[CMD_MAX_TOKENS];
+		uint32_t n_tokens, n_tokens_parsed, t0;
+
+		/* Read next line from file. */
+		if (fgets(line, line_len_max + 1, f) == NULL)
+			break;
+
+		/* Comment. */
+		if (is_comment(line))
+			continue;
+
+		/* Parse line. */
+		n_tokens = RTE_DIM(tokens);
+		status = softnic_parse_tokenize_string(line, tokens, &n_tokens);
+		if (status) {
+			status = -EINVAL;
+			break;
+		}
+
+		/* Empty line. */
+		if (n_tokens == 0)
+			continue;
+		t0 = 0;
+
+		/* Rule match. */
+		n_tokens_parsed = parse_match(tokens + t0,
+			n_tokens - t0,
+			out,
+			out_size,
+			&m[rule_id]);
+		if (n_tokens_parsed == 0) {
+			status = -EINVAL;
+			break;
+		}
+		t0 += n_tokens_parsed;
+
+		/* Rule action. */
+		n_tokens_parsed = parse_table_action(tokens + t0,
+			n_tokens - t0,
+			out,
+			out_size,
+			&a[rule_id]);
+		if (n_tokens_parsed == 0) {
+			status = -EINVAL;
+			break;
+		}
+		t0 += n_tokens_parsed;
+
+		/* Line completed. */
+		if (t0 < n_tokens) {
+			status = -EINVAL;
+			break;
+		}
+
+		/* Increment rule count */
+		rule_id++;
+	}
+
+	/* Close file */
+	fclose(f);
+
+	/* Memory free */
+	free(line);
+
+	*n_rules = rule_id;
+	*line_number = line_id;
+	return status;
+}
diff --git a/drivers/net/softnic/rte_eth_softnic_internals.h b/drivers/net/softnic/rte_eth_softnic_internals.h
index 050e3e7e..a25eb874 100644
--- a/drivers/net/softnic/rte_eth_softnic_internals.h
+++ b/drivers/net/softnic/rte_eth_softnic_internals.h
@@ -5,79 +5,97 @@
 #ifndef __INCLUDE_RTE_ETH_SOFTNIC_INTERNALS_H__
 #define __INCLUDE_RTE_ETH_SOFTNIC_INTERNALS_H__
 
+#include <stddef.h>
 #include <stdint.h>
+#include <sys/queue.h>
 
+#include <rte_mempool.h>
 #include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_ethdev.h>
 #include <rte_sched.h>
+#include <rte_port_in_action.h>
+#include <rte_table_action.h>
+#include <rte_pipeline.h>
+
 #include <rte_ethdev_driver.h>
 #include <rte_tm_driver.h>
 
 #include "rte_eth_softnic.h"
+#include "conn.h"
+
+#define NAME_SIZE                                            64
 
 /**
  * PMD Parameters
  */
 
-enum pmd_feature {
-	PMD_FEATURE_TM = 1, /**< Traffic Management (TM) */
-};
-
-#ifndef INTRUSIVE
-#define INTRUSIVE					0
-#endif
-
 struct pmd_params {
-	/** Parameters for the soft device (to be created) */
-	struct {
-		const char *name; /**< Name */
-		uint32_t flags; /**< Flags */
+	const char *name;
+	const char *firmware;
+	uint16_t conn_port;
+	uint32_t cpu_id;
 
-		/** 0 = Access hard device though API only (potentially slower,
-		 *      but safer);
-		 *  1 = Access hard device private data structures is allowed
-		 *      (potentially faster).
-		 */
-		int intrusive;
+	/** Traffic Management (TM) */
+	struct {
+		uint32_t n_queues; /**< Number of queues */
+		uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
+	} tm;
+};
 
-		/** Traffic Management (TM) */
-		struct {
-			uint32_t rate; /**< Rate (bytes/second) */
-			uint32_t nb_queues; /**< Number of queues */
-			uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
-			/**< Queue size per traffic class */
-			uint32_t enq_bsz; /**< Enqueue burst size */
-			uint32_t deq_bsz; /**< Dequeue burst size */
-		} tm;
-	} soft;
+/**
+ * MEMPOOL
+ */
+struct softnic_mempool_params {
+	uint32_t buffer_size;
+	uint32_t pool_size;
+	uint32_t cache_size;
+};
 
-	/** Parameters for the hard device (existing) */
-	struct {
-		char *name; /**< Name */
-		uint16_t tx_queue_id; /**< TX queue ID */
-	} hard;
+struct softnic_mempool {
+	TAILQ_ENTRY(softnic_mempool) node;
+	char name[NAME_SIZE];
+	struct rte_mempool *m;
+	uint32_t buffer_size;
 };
 
+TAILQ_HEAD(softnic_mempool_list, softnic_mempool);
+
 /**
- * Default Internals
+ * SWQ
  */
+struct softnic_swq_params {
+	uint32_t size;
+};
 
-#ifndef DEFAULT_BURST_SIZE
-#define DEFAULT_BURST_SIZE				32
-#endif
+struct softnic_swq {
+	TAILQ_ENTRY(softnic_swq) node;
+	char name[NAME_SIZE];
+	struct rte_ring *r;
+};
 
-#ifndef FLUSH_COUNT_THRESHOLD
-#define FLUSH_COUNT_THRESHOLD			(1 << 17)
-#endif
+TAILQ_HEAD(softnic_swq_list, softnic_swq);
+
+/**
+ * LINK
+ */
+struct softnic_link_params {
+	const char *dev_name;
+	uint16_t port_id; /**< Valid only when *dev_name* is NULL. */
+};
 
-struct default_internals {
-	struct rte_mbuf **pkts;
-	uint32_t pkts_len;
-	uint32_t txq_pos;
-	uint32_t flush_count;
+struct softnic_link {
+	TAILQ_ENTRY(softnic_link) node;
+	char name[NAME_SIZE];
+	uint16_t port_id;
+	uint32_t n_rxq;
+	uint32_t n_txq;
 };
 
+TAILQ_HEAD(softnic_link_list, softnic_link);
+
 /**
- * Traffic Management (TM) Internals
+ * TMGR
  */
 
 #ifndef TM_MAX_SUBPORTS
@@ -185,16 +203,281 @@ struct tm_internals {
 
 	/** Blueprints */
 	struct tm_params params;
+};
+
+struct softnic_tmgr_port {
+	TAILQ_ENTRY(softnic_tmgr_port) node;
+	char name[NAME_SIZE];
+	struct rte_sched_port *s;
+};
+
+TAILQ_HEAD(softnic_tmgr_port_list, softnic_tmgr_port);
+
+/**
+ * TAP
+ */
+struct softnic_tap {
+	TAILQ_ENTRY(softnic_tap) node;
+	char name[NAME_SIZE];
+	int fd;
+};
+
+TAILQ_HEAD(softnic_tap_list, softnic_tap);
+
+/**
+ * Input port action
+ */
+struct softnic_port_in_action_profile_params {
+	uint64_t action_mask;
+	struct rte_port_in_action_fltr_config fltr;
+	struct rte_port_in_action_lb_config lb;
+};
+
+struct softnic_port_in_action_profile {
+	TAILQ_ENTRY(softnic_port_in_action_profile) node;
+	char name[NAME_SIZE];
+	struct softnic_port_in_action_profile_params params;
+	struct rte_port_in_action_profile *ap;
+};
+
+TAILQ_HEAD(softnic_port_in_action_profile_list, softnic_port_in_action_profile);
+
+/**
+ * Table action
+ */
+struct softnic_table_action_profile_params {
+	uint64_t action_mask;
+	struct rte_table_action_common_config common;
+	struct rte_table_action_lb_config lb;
+	struct rte_table_action_mtr_config mtr;
+	struct rte_table_action_tm_config tm;
+	struct rte_table_action_encap_config encap;
+	struct rte_table_action_nat_config nat;
+	struct rte_table_action_ttl_config ttl;
+	struct rte_table_action_stats_config stats;
+};
+
+struct softnic_table_action_profile {
+	TAILQ_ENTRY(softnic_table_action_profile) node;
+	char name[NAME_SIZE];
+	struct softnic_table_action_profile_params params;
+	struct rte_table_action_profile *ap;
+};
+
+TAILQ_HEAD(softnic_table_action_profile_list, softnic_table_action_profile);
+
+/**
+ * Pipeline
+ */
+struct pipeline_params {
+	uint32_t timer_period_ms;
+	uint32_t offset_port_id;
+};
+
+enum softnic_port_in_type {
+	PORT_IN_RXQ,
+	PORT_IN_SWQ,
+	PORT_IN_TMGR,
+	PORT_IN_TAP,
+	PORT_IN_SOURCE,
+};
+
+struct softnic_port_in_params {
+	/* Read */
+	enum softnic_port_in_type type;
+	const char *dev_name;
+	union {
+		struct {
+			uint16_t queue_id;
+		} rxq;
+
+		struct {
+			const char *mempool_name;
+			uint32_t mtu;
+		} tap;
+
+		struct {
+			const char *mempool_name;
+			const char *file_name;
+			uint32_t n_bytes_per_pkt;
+		} source;
+	};
+	uint32_t burst_size;
+
+	/* Action */
+	const char *action_profile_name;
+};
+
+enum softnic_port_out_type {
+	PORT_OUT_TXQ,
+	PORT_OUT_SWQ,
+	PORT_OUT_TMGR,
+	PORT_OUT_TAP,
+	PORT_OUT_SINK,
+};
+
+struct softnic_port_out_params {
+	enum softnic_port_out_type type;
+	const char *dev_name;
+	union {
+		struct {
+			uint16_t queue_id;
+		} txq;
 
-	/** Run-time */
-	struct rte_sched_port *sched;
-	struct rte_mbuf **pkts_enq;
-	struct rte_mbuf **pkts_deq;
-	uint32_t pkts_enq_len;
-	uint32_t txq_pos;
-	uint32_t flush_count;
+		struct {
+			const char *file_name;
+			uint32_t max_n_pkts;
+		} sink;
+	};
+	uint32_t burst_size;
+	int retry;
+	uint32_t n_retries;
+};
+
+enum softnic_table_type {
+	TABLE_ACL,
+	TABLE_ARRAY,
+	TABLE_HASH,
+	TABLE_LPM,
+	TABLE_STUB,
+};
+
+struct softnic_table_acl_params {
+	uint32_t n_rules;
+	uint32_t ip_header_offset;
+	int ip_version;
+};
+
+struct softnic_table_array_params {
+	uint32_t n_keys;
+	uint32_t key_offset;
+};
+
+struct softnic_table_hash_params {
+	uint32_t n_keys;
+	uint32_t key_offset;
+	uint32_t key_size;
+	uint8_t *key_mask;
+	uint32_t n_buckets;
+	int extendable_bucket;
+};
+
+struct softnic_table_lpm_params {
+	uint32_t n_rules;
+	uint32_t key_offset;
+	uint32_t key_size;
+};
+
+struct softnic_table_params {
+	/* Match */
+	enum softnic_table_type match_type;
+	union {
+		struct softnic_table_acl_params acl;
+		struct softnic_table_array_params array;
+		struct softnic_table_hash_params hash;
+		struct softnic_table_lpm_params lpm;
+	} match;
+
+	/* Action */
+	const char *action_profile_name;
+};
+
+struct softnic_port_in {
+	struct softnic_port_in_params params;
+	struct softnic_port_in_action_profile *ap;
+	struct rte_port_in_action *a;
+};
+
+struct softnic_table {
+	struct softnic_table_params params;
+	struct softnic_table_action_profile *ap;
+	struct rte_table_action *a;
 };
 
+struct pipeline {
+	TAILQ_ENTRY(pipeline) node;
+	char name[NAME_SIZE];
+
+	struct rte_pipeline *p;
+	struct softnic_port_in port_in[RTE_PIPELINE_PORT_IN_MAX];
+	struct softnic_table table[RTE_PIPELINE_TABLE_MAX];
+	uint32_t n_ports_in;
+	uint32_t n_ports_out;
+	uint32_t n_tables;
+
+	struct rte_ring *msgq_req;
+	struct rte_ring *msgq_rsp;
+	uint32_t timer_period_ms;
+
+	int enabled;
+	uint32_t thread_id;
+	uint32_t cpu_id;
+};
+
+TAILQ_HEAD(pipeline_list, pipeline);
+
+/**
+ * Thread
+ */
+#ifndef THREAD_PIPELINES_MAX
+#define THREAD_PIPELINES_MAX                               256
+#endif
+
+#ifndef THREAD_MSGQ_SIZE
+#define THREAD_MSGQ_SIZE                                   64
+#endif
+
+#ifndef THREAD_TIMER_PERIOD_MS
+#define THREAD_TIMER_PERIOD_MS                             100
+#endif
+
+/**
+ * Master thead: data plane thread context
+ */
+struct softnic_thread {
+	struct rte_ring *msgq_req;
+	struct rte_ring *msgq_rsp;
+
+	uint32_t enabled;
+};
+
+/**
+ * Data plane threads: context
+ */
+#ifndef TABLE_RULE_ACTION_SIZE_MAX
+#define TABLE_RULE_ACTION_SIZE_MAX                         2048
+#endif
+
+struct softnic_table_data {
+	struct rte_table_action *a;
+};
+
+struct pipeline_data {
+	struct rte_pipeline *p;
+	struct softnic_table_data table_data[RTE_PIPELINE_TABLE_MAX];
+	uint32_t n_tables;
+
+	struct rte_ring *msgq_req;
+	struct rte_ring *msgq_rsp;
+	uint64_t timer_period; /* Measured in CPU cycles. */
+	uint64_t time_next;
+
+	uint8_t buffer[TABLE_RULE_ACTION_SIZE_MAX];
+};
+
+struct softnic_thread_data {
+	struct rte_pipeline *p[THREAD_PIPELINES_MAX];
+	uint32_t n_pipelines;
+
+	struct pipeline_data pipeline_data[THREAD_PIPELINES_MAX];
+	struct rte_ring *msgq_req;
+	struct rte_ring *msgq_rsp;
+	uint64_t timer_period; /* Measured in CPU cycles. */
+	uint64_t time_next;
+	uint64_t time_next_min;
+	uint64_t iter;
+} __rte_cache_aligned;
+
 /**
  * PMD Internals
  */
@@ -202,61 +485,426 @@ struct pmd_internals {
 	/** Params */
 	struct pmd_params params;
 
-	/** Soft device */
 	struct {
-		struct default_internals def; /**< Default */
 		struct tm_internals tm; /**< Traffic Management */
 	} soft;
 
-	/** Hard device */
-	struct {
-		uint16_t port_id;
-	} hard;
-};
-
-struct pmd_rx_queue {
-	/** Hard device */
-	struct {
-		uint16_t port_id;
-		uint16_t rx_queue_id;
-	} hard;
+	struct softnic_conn *conn;
+	struct softnic_mempool_list mempool_list;
+	struct softnic_swq_list swq_list;
+	struct softnic_link_list link_list;
+	struct softnic_tmgr_port_list tmgr_port_list;
+	struct softnic_tap_list tap_list;
+	struct softnic_port_in_action_profile_list port_in_action_profile_list;
+	struct softnic_table_action_profile_list table_action_profile_list;
+	struct pipeline_list pipeline_list;
+	struct softnic_thread thread[RTE_MAX_LCORE];
+	struct softnic_thread_data thread_data[RTE_MAX_LCORE];
 };
 
 /**
- * Traffic Management (TM) Operation
+ * MEMPOOL
  */
-extern const struct rte_tm_ops pmd_tm_ops;
+int
+softnic_mempool_init(struct pmd_internals *p);
+
+void
+softnic_mempool_free(struct pmd_internals *p);
+
+struct softnic_mempool *
+softnic_mempool_find(struct pmd_internals *p,
+	const char *name);
 
+struct softnic_mempool *
+softnic_mempool_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_mempool_params *params);
+
+/**
+ * SWQ
+ */
 int
-tm_params_check(struct pmd_params *params, uint32_t hard_rate);
+softnic_swq_init(struct pmd_internals *p);
+
+void
+softnic_swq_free(struct pmd_internals *p);
+
+void
+softnic_softnic_swq_free_keep_rxq_txq(struct pmd_internals *p);
+
+struct softnic_swq *
+softnic_swq_find(struct pmd_internals *p,
+	const char *name);
 
+struct softnic_swq *
+softnic_swq_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_swq_params *params);
+
+/**
+ * LINK
+ */
 int
-tm_init(struct pmd_internals *p, struct pmd_params *params, int numa_node);
+softnic_link_init(struct pmd_internals *p);
 
 void
-tm_free(struct pmd_internals *p);
+softnic_link_free(struct pmd_internals *p);
+
+struct softnic_link *
+softnic_link_find(struct pmd_internals *p,
+	const char *name);
+
+struct softnic_link *
+softnic_link_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_link_params *params);
 
+/**
+ * TMGR
+ */
 int
-tm_start(struct pmd_internals *p);
+softnic_tmgr_init(struct pmd_internals *p);
 
 void
-tm_stop(struct pmd_internals *p);
+softnic_tmgr_free(struct pmd_internals *p);
 
-static inline int
-tm_enabled(struct rte_eth_dev *dev)
-{
-	struct pmd_internals *p = dev->data->dev_private;
+struct softnic_tmgr_port *
+softnic_tmgr_port_find(struct pmd_internals *p,
+	const char *name);
 
-	return (p->params.soft.flags & PMD_FEATURE_TM);
-}
+struct softnic_tmgr_port *
+softnic_tmgr_port_create(struct pmd_internals *p,
+	const char *name);
+
+void
+tm_hierarchy_init(struct pmd_internals *p);
+
+void
+tm_hierarchy_free(struct pmd_internals *p);
 
 static inline int
 tm_used(struct rte_eth_dev *dev)
 {
 	struct pmd_internals *p = dev->data->dev_private;
 
-	return (p->params.soft.flags & PMD_FEATURE_TM) &&
-		p->soft.tm.h.n_tm_nodes[TM_NODE_LEVEL_PORT];
+	return p->soft.tm.h.n_tm_nodes[TM_NODE_LEVEL_PORT];
 }
 
+extern const struct rte_tm_ops pmd_tm_ops;
+
+/**
+ * TAP
+ */
+int
+softnic_tap_init(struct pmd_internals *p);
+
+void
+softnic_tap_free(struct pmd_internals *p);
+
+struct softnic_tap *
+softnic_tap_find(struct pmd_internals *p,
+	const char *name);
+
+struct softnic_tap *
+softnic_tap_create(struct pmd_internals *p,
+	const char *name);
+
+/**
+ * Input port action
+ */
+int
+softnic_port_in_action_profile_init(struct pmd_internals *p);
+
+void
+softnic_port_in_action_profile_free(struct pmd_internals *p);
+
+struct softnic_port_in_action_profile *
+softnic_port_in_action_profile_find(struct pmd_internals *p,
+	const char *name);
+
+struct softnic_port_in_action_profile *
+softnic_port_in_action_profile_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_port_in_action_profile_params *params);
+
+/**
+ * Table action
+ */
+int
+softnic_table_action_profile_init(struct pmd_internals *p);
+
+void
+softnic_table_action_profile_free(struct pmd_internals *p);
+
+struct softnic_table_action_profile *
+softnic_table_action_profile_find(struct pmd_internals *p,
+	const char *name);
+
+struct softnic_table_action_profile *
+softnic_table_action_profile_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_table_action_profile_params *params);
+
+/**
+ * Pipeline
+ */
+int
+softnic_pipeline_init(struct pmd_internals *p);
+
+void
+softnic_pipeline_free(struct pmd_internals *p);
+
+void
+softnic_pipeline_disable_all(struct pmd_internals *p);
+
+struct pipeline *
+softnic_pipeline_find(struct pmd_internals *p, const char *name);
+
+struct pipeline *
+softnic_pipeline_create(struct pmd_internals *p,
+	const char *name,
+	struct pipeline_params *params);
+
+int
+softnic_pipeline_port_in_create(struct pmd_internals *p,
+	const char *pipeline_name,
+	struct softnic_port_in_params *params,
+	int enabled);
+
+int
+softnic_pipeline_port_in_connect_to_table(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t port_id,
+	uint32_t table_id);
+
+int
+softnic_pipeline_port_out_create(struct pmd_internals *p,
+	const char *pipeline_name,
+	struct softnic_port_out_params *params);
+
+int
+softnic_pipeline_table_create(struct pmd_internals *p,
+	const char *pipeline_name,
+	struct softnic_table_params *params);
+
+struct softnic_table_rule_match_acl {
+	int ip_version;
+
+	RTE_STD_C11
+	union {
+		struct {
+			uint32_t sa;
+			uint32_t da;
+		} ipv4;
+
+		struct {
+			uint8_t sa[16];
+			uint8_t da[16];
+		} ipv6;
+	};
+
+	uint32_t sa_depth;
+	uint32_t da_depth;
+	uint16_t sp0;
+	uint16_t sp1;
+	uint16_t dp0;
+	uint16_t dp1;
+	uint8_t proto;
+	uint8_t proto_mask;
+	uint32_t priority;
+};
+
+struct softnic_table_rule_match_array {
+	uint32_t pos;
+};
+
+#ifndef TABLE_RULE_MATCH_SIZE_MAX
+#define TABLE_RULE_MATCH_SIZE_MAX                          256
+#endif
+
+struct softnic_table_rule_match_hash {
+	uint8_t key[TABLE_RULE_MATCH_SIZE_MAX];
+};
+
+struct softnic_table_rule_match_lpm {
+	int ip_version;
+
+	RTE_STD_C11
+	union {
+		uint32_t ipv4;
+		uint8_t ipv6[16];
+	};
+
+	uint8_t depth;
+};
+
+struct softnic_table_rule_match {
+	enum softnic_table_type match_type;
+
+	union {
+		struct softnic_table_rule_match_acl acl;
+		struct softnic_table_rule_match_array array;
+		struct softnic_table_rule_match_hash hash;
+		struct softnic_table_rule_match_lpm lpm;
+	} match;
+};
+
+struct softnic_table_rule_action {
+	uint64_t action_mask;
+	struct rte_table_action_fwd_params fwd;
+	struct rte_table_action_lb_params lb;
+	struct rte_table_action_mtr_params mtr;
+	struct rte_table_action_tm_params tm;
+	struct rte_table_action_encap_params encap;
+	struct rte_table_action_nat_params nat;
+	struct rte_table_action_ttl_params ttl;
+	struct rte_table_action_stats_params stats;
+	struct rte_table_action_time_params time;
+};
+
+int
+softnic_pipeline_port_in_stats_read(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t port_id,
+	struct rte_pipeline_port_in_stats *stats,
+	int clear);
+
+int
+softnic_pipeline_port_in_enable(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t port_id);
+
+int
+softnic_pipeline_port_in_disable(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t port_id);
+
+int
+softnic_pipeline_port_out_stats_read(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t port_id,
+	struct rte_pipeline_port_out_stats *stats,
+	int clear);
+
+int
+softnic_pipeline_table_stats_read(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id,
+	struct rte_pipeline_table_stats *stats,
+	int clear);
+
+int
+softnic_pipeline_table_rule_add(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id,
+	struct softnic_table_rule_match *match,
+	struct softnic_table_rule_action *action,
+	void **data);
+
+int
+softnic_pipeline_table_rule_add_bulk(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id,
+	struct softnic_table_rule_match *match,
+	struct softnic_table_rule_action *action,
+	void **data,
+	uint32_t *n_rules);
+
+int
+softnic_pipeline_table_rule_add_default(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id,
+	struct softnic_table_rule_action *action,
+	void **data);
+
+int
+softnic_pipeline_table_rule_delete(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id,
+	struct softnic_table_rule_match *match);
+
+int
+softnic_pipeline_table_rule_delete_default(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id);
+
+int
+softnic_pipeline_table_rule_stats_read(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id,
+	void *data,
+	struct rte_table_action_stats_counters *stats,
+	int clear);
+
+int
+softnic_pipeline_table_mtr_profile_add(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id,
+	uint32_t meter_profile_id,
+	struct rte_table_action_meter_profile *profile);
+
+int
+softnic_pipeline_table_mtr_profile_delete(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id,
+	uint32_t meter_profile_id);
+
+int
+softnic_pipeline_table_rule_mtr_read(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id,
+	void *data,
+	uint32_t tc_mask,
+	struct rte_table_action_mtr_counters *stats,
+	int clear);
+
+int
+softnic_pipeline_table_dscp_table_update(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id,
+	uint64_t dscp_mask,
+	struct rte_table_action_dscp_table *dscp_table);
+
+int
+softnic_pipeline_table_rule_ttl_read(struct pmd_internals *p,
+	const char *pipeline_name,
+	uint32_t table_id,
+	void *data,
+	struct rte_table_action_ttl_counters *stats,
+	int clear);
+
+/**
+ * Thread
+ */
+int
+softnic_thread_init(struct pmd_internals *p);
+
+void
+softnic_thread_free(struct pmd_internals *p);
+
+int
+softnic_thread_pipeline_enable(struct pmd_internals *p,
+	uint32_t thread_id,
+	const char *pipeline_name);
+
+int
+softnic_thread_pipeline_disable(struct pmd_internals *p,
+	uint32_t thread_id,
+	const char *pipeline_name);
+
+/**
+ * CLI
+ */
+void
+softnic_cli_process(char *in,
+	char *out,
+	size_t out_size,
+	void *arg);
+
+int
+softnic_cli_script_process(struct pmd_internals *softnic,
+	const char *file_name,
+	size_t msg_in_len_max,
+	size_t msg_out_len_max);
+
 #endif /* __INCLUDE_RTE_ETH_SOFTNIC_INTERNALS_H__ */
diff --git a/drivers/net/softnic/rte_eth_softnic_link.c b/drivers/net/softnic/rte_eth_softnic_link.c
new file mode 100644
index 00000000..d669913a
--- /dev/null
+++ b/drivers/net/softnic/rte_eth_softnic_link.c
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_ethdev.h>
+#include <rte_string_fns.h>
+
+#include "rte_eth_softnic_internals.h"
+
+int
+softnic_link_init(struct pmd_internals *p)
+{
+	TAILQ_INIT(&p->link_list);
+
+	return 0;
+}
+
+void
+softnic_link_free(struct pmd_internals *p)
+{
+	for ( ; ; ) {
+		struct softnic_link *link;
+
+		link = TAILQ_FIRST(&p->link_list);
+		if (link == NULL)
+			break;
+
+		TAILQ_REMOVE(&p->link_list, link, node);
+		free(link);
+	}
+}
+
+struct softnic_link *
+softnic_link_find(struct pmd_internals *p,
+	const char *name)
+{
+	struct softnic_link *link;
+
+	if (name == NULL)
+		return NULL;
+
+	TAILQ_FOREACH(link, &p->link_list, node)
+		if (strcmp(link->name, name) == 0)
+			return link;
+
+	return NULL;
+}
+
+struct softnic_link *
+softnic_link_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_link_params *params)
+{
+	struct rte_eth_dev_info port_info;
+	struct softnic_link *link;
+	uint16_t port_id;
+
+	/* Check input params */
+	if (name == NULL ||
+		softnic_link_find(p, name) ||
+		params == NULL)
+		return NULL;
+
+	port_id = params->port_id;
+	if (params->dev_name) {
+		int status;
+
+		status = rte_eth_dev_get_port_by_name(params->dev_name,
+			&port_id);
+
+		if (status)
+			return NULL;
+	} else {
+		if (!rte_eth_dev_is_valid_port(port_id))
+			return NULL;
+	}
+
+	rte_eth_dev_info_get(port_id, &port_info);
+
+	/* Node allocation */
+	link = calloc(1, sizeof(struct softnic_link));
+	if (link == NULL)
+		return NULL;
+
+	/* Node fill in */
+	strlcpy(link->name, name, sizeof(link->name));
+	link->port_id = port_id;
+	link->n_rxq = port_info.nb_rx_queues;
+	link->n_txq = port_info.nb_tx_queues;
+
+	/* Node add to list */
+	TAILQ_INSERT_TAIL(&p->link_list, link, node);
+
+	return link;
+}
diff --git a/drivers/net/softnic/rte_eth_softnic_mempool.c b/drivers/net/softnic/rte_eth_softnic_mempool.c
new file mode 100644
index 00000000..d5c569f9
--- /dev/null
+++ b/drivers/net/softnic/rte_eth_softnic_mempool.c
@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_mbuf.h>
+#include <rte_string_fns.h>
+
+#include "rte_eth_softnic_internals.h"
+
+#define BUFFER_SIZE_MIN        (sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
+
+int
+softnic_mempool_init(struct pmd_internals *p)
+{
+	TAILQ_INIT(&p->mempool_list);
+
+	return 0;
+}
+
+void
+softnic_mempool_free(struct pmd_internals *p)
+{
+	for ( ; ; ) {
+		struct softnic_mempool *mempool;
+
+		mempool = TAILQ_FIRST(&p->mempool_list);
+		if (mempool == NULL)
+			break;
+
+		TAILQ_REMOVE(&p->mempool_list, mempool, node);
+		rte_mempool_free(mempool->m);
+		free(mempool);
+	}
+}
+
+struct softnic_mempool *
+softnic_mempool_find(struct pmd_internals *p,
+	const char *name)
+{
+	struct softnic_mempool *mempool;
+
+	if (name == NULL)
+		return NULL;
+
+	TAILQ_FOREACH(mempool, &p->mempool_list, node)
+		if (strcmp(mempool->name, name) == 0)
+			return mempool;
+
+	return NULL;
+}
+
+struct softnic_mempool *
+softnic_mempool_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_mempool_params *params)
+{
+	char mempool_name[NAME_SIZE];
+	struct softnic_mempool *mempool;
+	struct rte_mempool *m;
+
+	/* Check input params */
+	if (name == NULL ||
+		softnic_mempool_find(p, name) ||
+		params == NULL ||
+		params->buffer_size < BUFFER_SIZE_MIN ||
+		params->pool_size == 0)
+		return NULL;
+
+	/* Resource create */
+	snprintf(mempool_name, sizeof(mempool_name), "%s_%s",
+		p->params.name,
+		name);
+
+	m = rte_pktmbuf_pool_create(mempool_name,
+		params->pool_size,
+		params->cache_size,
+		0,
+		params->buffer_size - sizeof(struct rte_mbuf),
+		p->params.cpu_id);
+
+	if (m == NULL)
+		return NULL;
+
+	/* Node allocation */
+	mempool = calloc(1, sizeof(struct softnic_mempool));
+	if (mempool == NULL) {
+		rte_mempool_free(m);
+		return NULL;
+	}
+
+	/* Node fill in */
+	strlcpy(mempool->name, name, sizeof(mempool->name));
+	mempool->m = m;
+	mempool->buffer_size = params->buffer_size;
+
+	/* Node add to list */
+	TAILQ_INSERT_TAIL(&p->mempool_list, mempool, node);
+
+	return mempool;
+}
diff --git a/drivers/net/softnic/rte_eth_softnic_pipeline.c b/drivers/net/softnic/rte_eth_softnic_pipeline.c
new file mode 100644
index 00000000..45136a4a
--- /dev/null
+++ b/drivers/net/softnic/rte_eth_softnic_pipeline.c
@@ -0,0 +1,966 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_ip.h>
+#include <rte_tcp.h>
+
+#include <rte_string_fns.h>
+#include <rte_port_ethdev.h>
+#include <rte_port_ring.h>
+#include <rte_port_source_sink.h>
+#include <rte_port_fd.h>
+#include <rte_port_sched.h>
+
+#include <rte_table_acl.h>
+#include <rte_table_array.h>
+#include <rte_table_hash.h>
+#include <rte_table_lpm.h>
+#include <rte_table_lpm_ipv6.h>
+#include <rte_table_stub.h>
+
+#include "rte_eth_softnic_internals.h"
+
+#include "hash_func.h"
+
+#ifndef PIPELINE_MSGQ_SIZE
+#define PIPELINE_MSGQ_SIZE                                 64
+#endif
+
+#ifndef TABLE_LPM_NUMBER_TBL8
+#define TABLE_LPM_NUMBER_TBL8                              256
+#endif
+
+int
+softnic_pipeline_init(struct pmd_internals *p)
+{
+	TAILQ_INIT(&p->pipeline_list);
+
+	return 0;
+}
+
+void
+softnic_pipeline_free(struct pmd_internals *p)
+{
+	for ( ; ; ) {
+		struct pipeline *pipeline;
+
+		pipeline = TAILQ_FIRST(&p->pipeline_list);
+		if (pipeline == NULL)
+			break;
+
+		TAILQ_REMOVE(&p->pipeline_list, pipeline, node);
+		rte_ring_free(pipeline->msgq_req);
+		rte_ring_free(pipeline->msgq_rsp);
+		rte_pipeline_free(pipeline->p);
+		free(pipeline);
+	}
+}
+
+void
+softnic_pipeline_disable_all(struct pmd_internals *p)
+{
+	struct pipeline *pipeline;
+
+	TAILQ_FOREACH(pipeline, &p->pipeline_list, node)
+		if (pipeline->enabled)
+			softnic_thread_pipeline_disable(p,
+				pipeline->thread_id,
+				pipeline->name);
+}
+
+struct pipeline *
+softnic_pipeline_find(struct pmd_internals *p,
+	const char *name)
+{
+	struct pipeline *pipeline;
+
+	if (name == NULL)
+		return NULL;
+
+	TAILQ_FOREACH(pipeline, &p->pipeline_list, node)
+		if (strcmp(name, pipeline->name) == 0)
+			return pipeline;
+
+	return NULL;
+}
+
+struct pipeline *
+softnic_pipeline_create(struct pmd_internals *softnic,
+	const char *name,
+	struct pipeline_params *params)
+{
+	char resource_name[NAME_MAX];
+	struct rte_pipeline_params pp;
+	struct pipeline *pipeline;
+	struct rte_pipeline *p;
+	struct rte_ring *msgq_req;
+	struct rte_ring *msgq_rsp;
+
+	/* Check input params */
+	if (name == NULL ||
+		softnic_pipeline_find(softnic, name) ||
+		params == NULL ||
+		params->timer_period_ms == 0)
+		return NULL;
+
+	/* Resource create */
+	snprintf(resource_name, sizeof(resource_name), "%s-%s-REQ",
+		softnic->params.name,
+		name);
+
+	msgq_req = rte_ring_create(resource_name,
+		PIPELINE_MSGQ_SIZE,
+		softnic->params.cpu_id,
+		RING_F_SP_ENQ | RING_F_SC_DEQ);
+	if (msgq_req == NULL)
+		return NULL;
+
+	snprintf(resource_name, sizeof(resource_name), "%s-%s-RSP",
+		softnic->params.name,
+		name);
+
+	msgq_rsp = rte_ring_create(resource_name,
+		PIPELINE_MSGQ_SIZE,
+		softnic->params.cpu_id,
+		RING_F_SP_ENQ | RING_F_SC_DEQ);
+	if (msgq_rsp == NULL) {
+		rte_ring_free(msgq_req);
+		return NULL;
+	}
+
+	snprintf(resource_name, sizeof(resource_name), "%s_%s",
+		softnic->params.name,
+		name);
+
+	pp.name = resource_name;
+	pp.socket_id = (int)softnic->params.cpu_id;
+	pp.offset_port_id = params->offset_port_id;
+
+	p = rte_pipeline_create(&pp);
+	if (p == NULL) {
+		rte_ring_free(msgq_rsp);
+		rte_ring_free(msgq_req);
+		return NULL;
+	}
+
+	/* Node allocation */
+	pipeline = calloc(1, sizeof(struct pipeline));
+	if (pipeline == NULL) {
+		rte_pipeline_free(p);
+		rte_ring_free(msgq_rsp);
+		rte_ring_free(msgq_req);
+		return NULL;
+	}
+
+	/* Node fill in */
+	strlcpy(pipeline->name, name, sizeof(pipeline->name));
+	pipeline->p = p;
+	pipeline->n_ports_in = 0;
+	pipeline->n_ports_out = 0;
+	pipeline->n_tables = 0;
+	pipeline->msgq_req = msgq_req;
+	pipeline->msgq_rsp = msgq_rsp;
+	pipeline->timer_period_ms = params->timer_period_ms;
+	pipeline->enabled = 0;
+	pipeline->cpu_id = softnic->params.cpu_id;
+
+	/* Node add to list */
+	TAILQ_INSERT_TAIL(&softnic->pipeline_list, pipeline, node);
+
+	return pipeline;
+}
+
+int
+softnic_pipeline_port_in_create(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	struct softnic_port_in_params *params,
+	int enabled)
+{
+	struct rte_pipeline_port_in_params p;
+
+	union {
+		struct rte_port_ethdev_reader_params ethdev;
+		struct rte_port_ring_reader_params ring;
+		struct rte_port_sched_reader_params sched;
+		struct rte_port_fd_reader_params fd;
+		struct rte_port_source_params source;
+	} pp;
+
+	struct pipeline *pipeline;
+	struct softnic_port_in *port_in;
+	struct softnic_port_in_action_profile *ap;
+	struct rte_port_in_action *action;
+	uint32_t port_id;
+	int status;
+
+	memset(&p, 0, sizeof(p));
+	memset(&pp, 0, sizeof(pp));
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		params == NULL ||
+		params->burst_size == 0 ||
+		params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)
+		return -1;
+
+	pipeline = softnic_pipeline_find(softnic, pipeline_name);
+	if (pipeline == NULL)
+		return -1;
+
+	ap = NULL;
+	if (params->action_profile_name) {
+		ap = softnic_port_in_action_profile_find(softnic,
+			params->action_profile_name);
+		if (ap == NULL)
+			return -1;
+	}
+
+	switch (params->type) {
+	case PORT_IN_RXQ:
+	{
+		struct softnic_link *link;
+
+		link = softnic_link_find(softnic, params->dev_name);
+		if (link == NULL)
+			return -1;
+
+		if (params->rxq.queue_id >= link->n_rxq)
+			return -1;
+
+		pp.ethdev.port_id = link->port_id;
+		pp.ethdev.queue_id = params->rxq.queue_id;
+
+		p.ops = &rte_port_ethdev_reader_ops;
+		p.arg_create = &pp.ethdev;
+		break;
+	}
+
+	case PORT_IN_SWQ:
+	{
+		struct softnic_swq *swq;
+
+		swq = softnic_swq_find(softnic, params->dev_name);
+		if (swq == NULL)
+			return -1;
+
+		pp.ring.ring = swq->r;
+
+		p.ops = &rte_port_ring_reader_ops;
+		p.arg_create = &pp.ring;
+		break;
+	}
+
+	case PORT_IN_TMGR:
+	{
+		struct softnic_tmgr_port *tmgr_port;
+
+		tmgr_port = softnic_tmgr_port_find(softnic, params->dev_name);
+		if (tmgr_port == NULL)
+			return -1;
+
+		pp.sched.sched = tmgr_port->s;
+
+		p.ops = &rte_port_sched_reader_ops;
+		p.arg_create = &pp.sched;
+		break;
+	}
+
+	case PORT_IN_TAP:
+	{
+		struct softnic_tap *tap;
+		struct softnic_mempool *mempool;
+
+		tap = softnic_tap_find(softnic, params->dev_name);
+		mempool = softnic_mempool_find(softnic, params->tap.mempool_name);
+		if (tap == NULL || mempool == NULL)
+			return -1;
+
+		pp.fd.fd = tap->fd;
+		pp.fd.mempool = mempool->m;
+		pp.fd.mtu = params->tap.mtu;
+
+		p.ops = &rte_port_fd_reader_ops;
+		p.arg_create = &pp.fd;
+		break;
+	}
+
+	case PORT_IN_SOURCE:
+	{
+		struct softnic_mempool *mempool;
+
+		mempool = softnic_mempool_find(softnic, params->source.mempool_name);
+		if (mempool == NULL)
+			return -1;
+
+		pp.source.mempool = mempool->m;
+		pp.source.file_name = params->source.file_name;
+		pp.source.n_bytes_per_pkt = params->source.n_bytes_per_pkt;
+
+		p.ops = &rte_port_source_ops;
+		p.arg_create = &pp.source;
+		break;
+	}
+
+	default:
+		return -1;
+	}
+
+	p.burst_size = params->burst_size;
+
+	/* Resource create */
+	action = NULL;
+	p.f_action = NULL;
+	p.arg_ah = NULL;
+
+	if (ap) {
+		action = rte_port_in_action_create(ap->ap,
+			softnic->params.cpu_id);
+		if (action == NULL)
+			return -1;
+
+		status = rte_port_in_action_params_get(action,
+			&p);
+		if (status) {
+			rte_port_in_action_free(action);
+			return -1;
+		}
+	}
+
+	status = rte_pipeline_port_in_create(pipeline->p,
+		&p,
+		&port_id);
+	if (status) {
+		rte_port_in_action_free(action);
+		return -1;
+	}
+
+	if (enabled)
+		rte_pipeline_port_in_enable(pipeline->p, port_id);
+
+	/* Pipeline */
+	port_in = &pipeline->port_in[pipeline->n_ports_in];
+	memcpy(&port_in->params, params, sizeof(*params));
+	port_in->ap = ap;
+	port_in->a = action;
+	pipeline->n_ports_in++;
+
+	return 0;
+}
+
+int
+softnic_pipeline_port_in_connect_to_table(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t port_id,
+	uint32_t table_id)
+{
+	struct pipeline *pipeline;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL)
+		return -1;
+
+	pipeline = softnic_pipeline_find(softnic, pipeline_name);
+	if (pipeline == NULL ||
+		port_id >= pipeline->n_ports_in ||
+		table_id >= pipeline->n_tables)
+		return -1;
+
+	/* Resource */
+	status = rte_pipeline_port_in_connect_to_table(pipeline->p,
+		port_id,
+		table_id);
+
+	return status;
+}
+
+int
+softnic_pipeline_port_out_create(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	struct softnic_port_out_params *params)
+{
+	struct rte_pipeline_port_out_params p;
+
+	union {
+		struct rte_port_ethdev_writer_params ethdev;
+		struct rte_port_ring_writer_params ring;
+		struct rte_port_sched_writer_params sched;
+		struct rte_port_fd_writer_params fd;
+		struct rte_port_sink_params sink;
+	} pp;
+
+	union {
+		struct rte_port_ethdev_writer_nodrop_params ethdev;
+		struct rte_port_ring_writer_nodrop_params ring;
+		struct rte_port_fd_writer_nodrop_params fd;
+	} pp_nodrop;
+
+	struct pipeline *pipeline;
+	uint32_t port_id;
+	int status;
+
+	memset(&p, 0, sizeof(p));
+	memset(&pp, 0, sizeof(pp));
+	memset(&pp_nodrop, 0, sizeof(pp_nodrop));
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		params == NULL ||
+		params->burst_size == 0 ||
+		params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)
+		return -1;
+
+	pipeline = softnic_pipeline_find(softnic, pipeline_name);
+	if (pipeline == NULL)
+		return -1;
+
+	switch (params->type) {
+	case PORT_OUT_TXQ:
+	{
+		struct softnic_link *link;
+
+		link = softnic_link_find(softnic, params->dev_name);
+		if (link == NULL)
+			return -1;
+
+		if (params->txq.queue_id >= link->n_txq)
+			return -1;
+
+		pp.ethdev.port_id = link->port_id;
+		pp.ethdev.queue_id = params->txq.queue_id;
+		pp.ethdev.tx_burst_sz = params->burst_size;
+
+		pp_nodrop.ethdev.port_id = link->port_id;
+		pp_nodrop.ethdev.queue_id = params->txq.queue_id;
+		pp_nodrop.ethdev.tx_burst_sz = params->burst_size;
+		pp_nodrop.ethdev.n_retries = params->n_retries;
+
+		if (params->retry == 0) {
+			p.ops = &rte_port_ethdev_writer_ops;
+			p.arg_create = &pp.ethdev;
+		} else {
+			p.ops = &rte_port_ethdev_writer_nodrop_ops;
+			p.arg_create = &pp_nodrop.ethdev;
+		}
+		break;
+	}
+
+	case PORT_OUT_SWQ:
+	{
+		struct softnic_swq *swq;
+
+		swq = softnic_swq_find(softnic, params->dev_name);
+		if (swq == NULL)
+			return -1;
+
+		pp.ring.ring = swq->r;
+		pp.ring.tx_burst_sz = params->burst_size;
+
+		pp_nodrop.ring.ring = swq->r;
+		pp_nodrop.ring.tx_burst_sz = params->burst_size;
+		pp_nodrop.ring.n_retries = params->n_retries;
+
+		if (params->retry == 0) {
+			p.ops = &rte_port_ring_writer_ops;
+			p.arg_create = &pp.ring;
+		} else {
+			p.ops = &rte_port_ring_writer_nodrop_ops;
+			p.arg_create = &pp_nodrop.ring;
+		}
+		break;
+	}
+
+	case PORT_OUT_TMGR:
+	{
+		struct softnic_tmgr_port *tmgr_port;
+
+		tmgr_port = softnic_tmgr_port_find(softnic, params->dev_name);
+		if (tmgr_port == NULL)
+			return -1;
+
+		pp.sched.sched = tmgr_port->s;
+		pp.sched.tx_burst_sz = params->burst_size;
+
+		p.ops = &rte_port_sched_writer_ops;
+		p.arg_create = &pp.sched;
+		break;
+	}
+
+	case PORT_OUT_TAP:
+	{
+		struct softnic_tap *tap;
+
+		tap = softnic_tap_find(softnic, params->dev_name);
+		if (tap == NULL)
+			return -1;
+
+		pp.fd.fd = tap->fd;
+		pp.fd.tx_burst_sz = params->burst_size;
+
+		pp_nodrop.fd.fd = tap->fd;
+		pp_nodrop.fd.tx_burst_sz = params->burst_size;
+		pp_nodrop.fd.n_retries = params->n_retries;
+
+		if (params->retry == 0) {
+			p.ops = &rte_port_fd_writer_ops;
+			p.arg_create = &pp.fd;
+		} else {
+			p.ops = &rte_port_fd_writer_nodrop_ops;
+			p.arg_create = &pp_nodrop.fd;
+		}
+		break;
+	}
+
+	case PORT_OUT_SINK:
+	{
+		pp.sink.file_name = params->sink.file_name;
+		pp.sink.max_n_pkts = params->sink.max_n_pkts;
+
+		p.ops = &rte_port_sink_ops;
+		p.arg_create = &pp.sink;
+		break;
+	}
+
+	default:
+		return -1;
+	}
+
+	p.f_action = NULL;
+	p.arg_ah = NULL;
+
+	/* Resource create */
+	status = rte_pipeline_port_out_create(pipeline->p,
+		&p,
+		&port_id);
+
+	if (status)
+		return -1;
+
+	/* Pipeline */
+	pipeline->n_ports_out++;
+
+	return 0;
+}
+
+static const struct rte_acl_field_def table_acl_field_format_ipv4[] = {
+	/* Protocol */
+	[0] = {
+		.type = RTE_ACL_FIELD_TYPE_BITMASK,
+		.size = sizeof(uint8_t),
+		.field_index = 0,
+		.input_index = 0,
+		.offset = offsetof(struct ipv4_hdr, next_proto_id),
+	},
+
+	/* Source IP address (IPv4) */
+	[1] = {
+		.type = RTE_ACL_FIELD_TYPE_MASK,
+		.size = sizeof(uint32_t),
+		.field_index = 1,
+		.input_index = 1,
+		.offset = offsetof(struct ipv4_hdr, src_addr),
+	},
+
+	/* Destination IP address (IPv4) */
+	[2] = {
+		.type = RTE_ACL_FIELD_TYPE_MASK,
+		.size = sizeof(uint32_t),
+		.field_index = 2,
+		.input_index = 2,
+		.offset = offsetof(struct ipv4_hdr, dst_addr),
+	},
+
+	/* Source Port */
+	[3] = {
+		.type = RTE_ACL_FIELD_TYPE_RANGE,
+		.size = sizeof(uint16_t),
+		.field_index = 3,
+		.input_index = 3,
+		.offset = sizeof(struct ipv4_hdr) +
+			offsetof(struct tcp_hdr, src_port),
+	},
+
+	/* Destination Port */
+	[4] = {
+		.type = RTE_ACL_FIELD_TYPE_RANGE,
+		.size = sizeof(uint16_t),
+		.field_index = 4,
+		.input_index = 3,
+		.offset = sizeof(struct ipv4_hdr) +
+			offsetof(struct tcp_hdr, dst_port),
+	},
+};
+
+static const struct rte_acl_field_def table_acl_field_format_ipv6[] = {
+	/* Protocol */
+	[0] = {
+		.type = RTE_ACL_FIELD_TYPE_BITMASK,
+		.size = sizeof(uint8_t),
+		.field_index = 0,
+		.input_index = 0,
+		.offset = offsetof(struct ipv6_hdr, proto),
+	},
+
+	/* Source IP address (IPv6) */
+	[1] = {
+		.type = RTE_ACL_FIELD_TYPE_MASK,
+		.size = sizeof(uint32_t),
+		.field_index = 1,
+		.input_index = 1,
+		.offset = offsetof(struct ipv6_hdr, src_addr[0]),
+	},
+
+	[2] = {
+		.type = RTE_ACL_FIELD_TYPE_MASK,
+		.size = sizeof(uint32_t),
+		.field_index = 2,
+		.input_index = 2,
+		.offset = offsetof(struct ipv6_hdr, src_addr[4]),
+	},
+
+	[3] = {
+		.type = RTE_ACL_FIELD_TYPE_MASK,
+		.size = sizeof(uint32_t),
+		.field_index = 3,
+		.input_index = 3,
+		.offset = offsetof(struct ipv6_hdr, src_addr[8]),
+	},
+
+	[4] = {
+		.type = RTE_ACL_FIELD_TYPE_MASK,
+		.size = sizeof(uint32_t),
+		.field_index = 4,
+		.input_index = 4,
+		.offset = offsetof(struct ipv6_hdr, src_addr[12]),
+	},
+
+	/* Destination IP address (IPv6) */
+	[5] = {
+		.type = RTE_ACL_FIELD_TYPE_MASK,
+		.size = sizeof(uint32_t),
+		.field_index = 5,
+		.input_index = 5,
+		.offset = offsetof(struct ipv6_hdr, dst_addr[0]),
+	},
+
+	[6] = {
+		.type = RTE_ACL_FIELD_TYPE_MASK,
+		.size = sizeof(uint32_t),
+		.field_index = 6,
+		.input_index = 6,
+		.offset = offsetof(struct ipv6_hdr, dst_addr[4]),
+	},
+
+	[7] = {
+		.type = RTE_ACL_FIELD_TYPE_MASK,
+		.size = sizeof(uint32_t),
+		.field_index = 7,
+		.input_index = 7,
+		.offset = offsetof(struct ipv6_hdr, dst_addr[8]),
+	},
+
+	[8] = {
+		.type = RTE_ACL_FIELD_TYPE_MASK,
+		.size = sizeof(uint32_t),
+		.field_index = 8,
+		.input_index = 8,
+		.offset = offsetof(struct ipv6_hdr, dst_addr[12]),
+	},
+
+	/* Source Port */
+	[9] = {
+		.type = RTE_ACL_FIELD_TYPE_RANGE,
+		.size = sizeof(uint16_t),
+		.field_index = 9,
+		.input_index = 9,
+		.offset = sizeof(struct ipv6_hdr) +
+			offsetof(struct tcp_hdr, src_port),
+	},
+
+	/* Destination Port */
+	[10] = {
+		.type = RTE_ACL_FIELD_TYPE_RANGE,
+		.size = sizeof(uint16_t),
+		.field_index = 10,
+		.input_index = 9,
+		.offset = sizeof(struct ipv6_hdr) +
+			offsetof(struct tcp_hdr, dst_port),
+	},
+};
+
+int
+softnic_pipeline_table_create(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	struct softnic_table_params *params)
+{
+	char name[NAME_MAX];
+	struct rte_pipeline_table_params p;
+
+	union {
+		struct rte_table_acl_params acl;
+		struct rte_table_array_params array;
+		struct rte_table_hash_params hash;
+		struct rte_table_lpm_params lpm;
+		struct rte_table_lpm_ipv6_params lpm_ipv6;
+	} pp;
+
+	struct pipeline *pipeline;
+	struct softnic_table *table;
+	struct softnic_table_action_profile *ap;
+	struct rte_table_action *action;
+	uint32_t table_id;
+	int status;
+
+	memset(&p, 0, sizeof(p));
+	memset(&pp, 0, sizeof(pp));
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		params == NULL)
+		return -1;
+
+	pipeline = softnic_pipeline_find(softnic, pipeline_name);
+	if (pipeline == NULL ||
+		pipeline->n_tables >= RTE_PIPELINE_TABLE_MAX)
+		return -1;
+
+	ap = NULL;
+	if (params->action_profile_name) {
+		ap = softnic_table_action_profile_find(softnic,
+			params->action_profile_name);
+		if (ap == NULL)
+			return -1;
+	}
+
+	snprintf(name, NAME_MAX, "%s_%s_table%u",
+		softnic->params.name, pipeline_name, pipeline->n_tables);
+
+	switch (params->match_type) {
+	case TABLE_ACL:
+	{
+		uint32_t ip_header_offset = params->match.acl.ip_header_offset -
+			(sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM);
+		uint32_t i;
+
+		if (params->match.acl.n_rules == 0)
+			return -1;
+
+		pp.acl.name = name;
+		pp.acl.n_rules = params->match.acl.n_rules;
+		if (params->match.acl.ip_version) {
+			memcpy(&pp.acl.field_format,
+				&table_acl_field_format_ipv4,
+				sizeof(table_acl_field_format_ipv4));
+			pp.acl.n_rule_fields =
+				RTE_DIM(table_acl_field_format_ipv4);
+		} else {
+			memcpy(&pp.acl.field_format,
+				&table_acl_field_format_ipv6,
+				sizeof(table_acl_field_format_ipv6));
+			pp.acl.n_rule_fields =
+				RTE_DIM(table_acl_field_format_ipv6);
+		}
+
+		for (i = 0; i < pp.acl.n_rule_fields; i++)
+			pp.acl.field_format[i].offset += ip_header_offset;
+
+		p.ops = &rte_table_acl_ops;
+		p.arg_create = &pp.acl;
+		break;
+	}
+
+	case TABLE_ARRAY:
+	{
+		if (params->match.array.n_keys == 0)
+			return -1;
+
+		pp.array.n_entries = params->match.array.n_keys;
+		pp.array.offset = params->match.array.key_offset;
+
+		p.ops = &rte_table_array_ops;
+		p.arg_create = &pp.array;
+		break;
+	}
+
+	case TABLE_HASH:
+	{
+		struct rte_table_ops *ops;
+		rte_table_hash_op_hash f_hash;
+
+		if (params->match.hash.n_keys == 0)
+			return -1;
+
+		switch (params->match.hash.key_size) {
+		case  8:
+			f_hash = hash_default_key8;
+			break;
+		case 16:
+			f_hash = hash_default_key16;
+			break;
+		case 24:
+			f_hash = hash_default_key24;
+			break;
+		case 32:
+			f_hash = hash_default_key32;
+			break;
+		case 40:
+			f_hash = hash_default_key40;
+			break;
+		case 48:
+			f_hash = hash_default_key48;
+			break;
+		case 56:
+			f_hash = hash_default_key56;
+			break;
+		case 64:
+			f_hash = hash_default_key64;
+			break;
+		default:
+			return -1;
+		}
+
+		pp.hash.name = name;
+		pp.hash.key_size = params->match.hash.key_size;
+		pp.hash.key_offset = params->match.hash.key_offset;
+		pp.hash.key_mask = params->match.hash.key_mask;
+		pp.hash.n_keys = params->match.hash.n_keys;
+		pp.hash.n_buckets = params->match.hash.n_buckets;
+		pp.hash.f_hash = f_hash;
+		pp.hash.seed = 0;
+
+		if (params->match.hash.extendable_bucket)
+			switch (params->match.hash.key_size) {
+			case  8:
+				ops = &rte_table_hash_key8_ext_ops;
+				break;
+			case 16:
+				ops = &rte_table_hash_key16_ext_ops;
+				break;
+			default:
+				ops = &rte_table_hash_ext_ops;
+			}
+		else
+			switch (params->match.hash.key_size) {
+			case  8:
+				ops = &rte_table_hash_key8_lru_ops;
+				break;
+			case 16:
+				ops = &rte_table_hash_key16_lru_ops;
+				break;
+			default:
+				ops = &rte_table_hash_lru_ops;
+			}
+
+		p.ops = ops;
+		p.arg_create = &pp.hash;
+		break;
+	}
+
+	case TABLE_LPM:
+	{
+		if (params->match.lpm.n_rules == 0)
+			return -1;
+
+		switch (params->match.lpm.key_size) {
+		case 4:
+		{
+			pp.lpm.name = name;
+			pp.lpm.n_rules = params->match.lpm.n_rules;
+			pp.lpm.number_tbl8s = TABLE_LPM_NUMBER_TBL8;
+			pp.lpm.flags = 0;
+			pp.lpm.entry_unique_size = p.action_data_size +
+				sizeof(struct rte_pipeline_table_entry);
+			pp.lpm.offset = params->match.lpm.key_offset;
+
+			p.ops = &rte_table_lpm_ops;
+			p.arg_create = &pp.lpm;
+			break;
+		}
+
+		case 16:
+		{
+			pp.lpm_ipv6.name = name;
+			pp.lpm_ipv6.n_rules = params->match.lpm.n_rules;
+			pp.lpm_ipv6.number_tbl8s = TABLE_LPM_NUMBER_TBL8;
+			pp.lpm_ipv6.entry_unique_size = p.action_data_size +
+				sizeof(struct rte_pipeline_table_entry);
+			pp.lpm_ipv6.offset = params->match.lpm.key_offset;
+
+			p.ops = &rte_table_lpm_ipv6_ops;
+			p.arg_create = &pp.lpm_ipv6;
+			break;
+		}
+
+		default:
+			return -1;
+		}
+
+		break;
+	}
+
+	case TABLE_STUB:
+	{
+		p.ops = &rte_table_stub_ops;
+		p.arg_create = NULL;
+		break;
+	}
+
+	default:
+		return -1;
+	}
+
+	/* Resource create */
+	action = NULL;
+	p.f_action_hit = NULL;
+	p.f_action_miss = NULL;
+	p.arg_ah = NULL;
+
+	if (ap) {
+		action = rte_table_action_create(ap->ap,
+			softnic->params.cpu_id);
+		if (action == NULL)
+			return -1;
+
+		status = rte_table_action_table_params_get(action,
+			&p);
+		if (status ||
+			((p.action_data_size +
+			sizeof(struct rte_pipeline_table_entry)) >
+			TABLE_RULE_ACTION_SIZE_MAX)) {
+			rte_table_action_free(action);
+			return -1;
+		}
+	}
+
+	if (params->match_type == TABLE_LPM) {
+		if (params->match.lpm.key_size == 4)
+			pp.lpm.entry_unique_size = p.action_data_size +
+				sizeof(struct rte_pipeline_table_entry);
+
+		if (params->match.lpm.key_size == 16)
+			pp.lpm_ipv6.entry_unique_size = p.action_data_size +
+				sizeof(struct rte_pipeline_table_entry);
+	}
+
+	status = rte_pipeline_table_create(pipeline->p,
+		&p,
+		&table_id);
+	if (status) {
+		rte_table_action_free(action);
+		return -1;
+	}
+
+	/* Pipeline */
+	table = &pipeline->table[pipeline->n_tables];
+	memcpy(&table->params, params, sizeof(*params));
+	table->ap = ap;
+	table->a = action;
+	pipeline->n_tables++;
+
+	return 0;
+}
diff --git a/drivers/net/softnic/rte_eth_softnic_swq.c b/drivers/net/softnic/rte_eth_softnic_swq.c
new file mode 100644
index 00000000..2083d0a9
--- /dev/null
+++ b/drivers/net/softnic/rte_eth_softnic_swq.c
@@ -0,0 +1,114 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_string_fns.h>
+#include <rte_tailq.h>
+
+#include "rte_eth_softnic_internals.h"
+
+int
+softnic_swq_init(struct pmd_internals *p)
+{
+	TAILQ_INIT(&p->swq_list);
+
+	return 0;
+}
+
+void
+softnic_swq_free(struct pmd_internals *p)
+{
+	for ( ; ; ) {
+		struct softnic_swq *swq;
+
+		swq = TAILQ_FIRST(&p->swq_list);
+		if (swq == NULL)
+			break;
+
+		TAILQ_REMOVE(&p->swq_list, swq, node);
+		rte_ring_free(swq->r);
+		free(swq);
+	}
+}
+
+void
+softnic_softnic_swq_free_keep_rxq_txq(struct pmd_internals *p)
+{
+	struct softnic_swq *swq, *tswq;
+
+	TAILQ_FOREACH_SAFE(swq, &p->swq_list, node, tswq) {
+		if ((strncmp(swq->name, "RXQ", strlen("RXQ")) == 0) ||
+			(strncmp(swq->name, "TXQ", strlen("TXQ")) == 0))
+			continue;
+
+		TAILQ_REMOVE(&p->swq_list, swq, node);
+		rte_ring_free(swq->r);
+		free(swq);
+	}
+}
+
+struct softnic_swq *
+softnic_swq_find(struct pmd_internals *p,
+	const char *name)
+{
+	struct softnic_swq *swq;
+
+	if (name == NULL)
+		return NULL;
+
+	TAILQ_FOREACH(swq, &p->swq_list, node)
+		if (strcmp(swq->name, name) == 0)
+			return swq;
+
+	return NULL;
+}
+
+struct softnic_swq *
+softnic_swq_create(struct pmd_internals *p,
+	const char *name,
+	struct softnic_swq_params *params)
+{
+	char ring_name[NAME_SIZE];
+	struct softnic_swq *swq;
+	struct rte_ring *r;
+	unsigned int flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
+
+	/* Check input params */
+	if (name == NULL ||
+		softnic_swq_find(p, name) ||
+		params == NULL ||
+		params->size == 0)
+		return NULL;
+
+	/* Resource create */
+	snprintf(ring_name, sizeof(ring_name), "%s_%s",
+		p->params.name,
+		name);
+
+	r = rte_ring_create(ring_name,
+		params->size,
+		p->params.cpu_id,
+		flags);
+
+	if (r == NULL)
+		return NULL;
+
+	/* Node allocation */
+	swq = calloc(1, sizeof(struct softnic_swq));
+	if (swq == NULL) {
+		rte_ring_free(r);
+		return NULL;
+	}
+
+	/* Node fill in */
+	strlcpy(swq->name, name, sizeof(swq->name));
+	swq->r = r;
+
+	/* Node add to list */
+	TAILQ_INSERT_TAIL(&p->swq_list, swq, node);
+
+	return swq;
+}
diff --git a/drivers/net/softnic/rte_eth_softnic_tap.c b/drivers/net/softnic/rte_eth_softnic_tap.c
new file mode 100644
index 00000000..bcc23a9f
--- /dev/null
+++ b/drivers/net/softnic/rte_eth_softnic_tap.c
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <netinet/in.h>
+#ifdef RTE_EXEC_ENV_LINUXAPP
+#include <linux/if.h>
+#include <linux/if_tun.h>
+#endif
+#include <sys/ioctl.h>
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <rte_string_fns.h>
+
+#include "rte_eth_softnic_internals.h"
+
+#define TAP_DEV                                            "/dev/net/tun"
+
+int
+softnic_tap_init(struct pmd_internals *p)
+{
+	TAILQ_INIT(&p->tap_list);
+
+	return 0;
+}
+
+void
+softnic_tap_free(struct pmd_internals *p)
+{
+	for ( ; ; ) {
+		struct softnic_tap *tap;
+
+		tap = TAILQ_FIRST(&p->tap_list);
+		if (tap == NULL)
+			break;
+
+		TAILQ_REMOVE(&p->tap_list, tap, node);
+		free(tap);
+	}
+}
+
+struct softnic_tap *
+softnic_tap_find(struct pmd_internals *p,
+	const char *name)
+{
+	struct softnic_tap *tap;
+
+	if (name == NULL)
+		return NULL;
+
+	TAILQ_FOREACH(tap, &p->tap_list, node)
+		if (strcmp(tap->name, name) == 0)
+			return tap;
+
+	return NULL;
+}
+
+#ifndef RTE_EXEC_ENV_LINUXAPP
+
+struct softnic_tap *
+softnic_tap_create(struct pmd_internals *p __rte_unused,
+	const char *name __rte_unused)
+{
+	return NULL;
+}
+
+#else
+
+struct softnic_tap *
+softnic_tap_create(struct pmd_internals *p,
+	const char *name)
+{
+	struct softnic_tap *tap;
+	struct ifreq ifr;
+	int fd, status;
+
+	/* Check input params */
+	if (name == NULL ||
+		softnic_tap_find(p, name))
+		return NULL;
+
+	/* Resource create */
+	fd = open(TAP_DEV, O_RDWR | O_NONBLOCK);
+	if (fd < 0)
+		return NULL;
+
+	memset(&ifr, 0, sizeof(ifr));
+	ifr.ifr_flags = IFF_TAP | IFF_NO_PI; /* No packet information */
+	snprintf(ifr.ifr_name, IFNAMSIZ, "%s", name);
+
+	status = ioctl(fd, TUNSETIFF, (void *)&ifr);
+	if (status < 0) {
+		close(fd);
+		return NULL;
+	}
+
+	/* Node allocation */
+	tap = calloc(1, sizeof(struct softnic_tap));
+	if (tap == NULL) {
+		close(fd);
+		return NULL;
+	}
+	/* Node fill in */
+	strlcpy(tap->name, name, sizeof(tap->name));
+	tap->fd = fd;
+
+	/* Node add to list */
+	TAILQ_INSERT_TAIL(&p->tap_list, tap, node);
+
+	return tap;
+}
+
+#endif
diff --git a/drivers/net/softnic/rte_eth_softnic_thread.c b/drivers/net/softnic/rte_eth_softnic_thread.c
new file mode 100644
index 00000000..8a150903
--- /dev/null
+++ b/drivers/net/softnic/rte_eth_softnic_thread.c
@@ -0,0 +1,2929 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <stdlib.h>
+
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_lcore.h>
+#include <rte_ring.h>
+
+#include <rte_table_acl.h>
+#include <rte_table_array.h>
+#include <rte_table_hash.h>
+#include <rte_table_lpm.h>
+#include <rte_table_lpm_ipv6.h>
+#include "rte_eth_softnic_internals.h"
+
+/**
+ * Master thread: data plane thread init
+ */
+void
+softnic_thread_free(struct pmd_internals *softnic)
+{
+	uint32_t i;
+
+	RTE_LCORE_FOREACH_SLAVE(i) {
+		struct softnic_thread *t = &softnic->thread[i];
+
+		/* MSGQs */
+		if (t->msgq_req)
+			rte_ring_free(t->msgq_req);
+
+		if (t->msgq_rsp)
+			rte_ring_free(t->msgq_rsp);
+	}
+}
+
+int
+softnic_thread_init(struct pmd_internals *softnic)
+{
+	uint32_t i;
+
+	RTE_LCORE_FOREACH_SLAVE(i) {
+		char ring_name[NAME_MAX];
+		struct rte_ring *msgq_req, *msgq_rsp;
+		struct softnic_thread *t = &softnic->thread[i];
+		struct softnic_thread_data *t_data = &softnic->thread_data[i];
+		uint32_t cpu_id = rte_lcore_to_socket_id(i);
+
+		/* MSGQs */
+		snprintf(ring_name, sizeof(ring_name), "%s-TH%u-REQ",
+			softnic->params.name,
+			i);
+
+		msgq_req = rte_ring_create(ring_name,
+			THREAD_MSGQ_SIZE,
+			cpu_id,
+			RING_F_SP_ENQ | RING_F_SC_DEQ);
+
+		if (msgq_req == NULL) {
+			softnic_thread_free(softnic);
+			return -1;
+		}
+
+		snprintf(ring_name, sizeof(ring_name), "%s-TH%u-RSP",
+			softnic->params.name,
+			i);
+
+		msgq_rsp = rte_ring_create(ring_name,
+			THREAD_MSGQ_SIZE,
+			cpu_id,
+			RING_F_SP_ENQ | RING_F_SC_DEQ);
+
+		if (msgq_rsp == NULL) {
+			softnic_thread_free(softnic);
+			return -1;
+		}
+
+		/* Master thread records */
+		t->msgq_req = msgq_req;
+		t->msgq_rsp = msgq_rsp;
+		t->enabled = 1;
+
+		/* Data plane thread records */
+		t_data->n_pipelines = 0;
+		t_data->msgq_req = msgq_req;
+		t_data->msgq_rsp = msgq_rsp;
+		t_data->timer_period =
+			(rte_get_tsc_hz() * THREAD_TIMER_PERIOD_MS) / 1000;
+		t_data->time_next = rte_get_tsc_cycles() + t_data->timer_period;
+		t_data->time_next_min = t_data->time_next;
+	}
+
+	return 0;
+}
+
+static inline int
+thread_is_running(uint32_t thread_id)
+{
+	enum rte_lcore_state_t thread_state;
+
+	thread_state = rte_eal_get_lcore_state(thread_id);
+	return (thread_state == RUNNING)? 1 : 0;
+}
+
+/**
+ * Pipeline is running when:
+ *    (A) Pipeline is mapped to a data plane thread AND
+ *    (B) Its data plane thread is in RUNNING state.
+ */
+static inline int
+pipeline_is_running(struct pipeline *p)
+{
+	if (p->enabled == 0)
+		return 0;
+
+	return thread_is_running(p->thread_id);
+}
+
+/**
+ * Master thread & data plane threads: message passing
+ */
+enum thread_req_type {
+	THREAD_REQ_PIPELINE_ENABLE = 0,
+	THREAD_REQ_PIPELINE_DISABLE,
+	THREAD_REQ_MAX
+};
+
+struct thread_msg_req {
+	enum thread_req_type type;
+
+	union {
+		struct {
+			struct rte_pipeline *p;
+			struct {
+				struct rte_table_action *a;
+			} table[RTE_PIPELINE_TABLE_MAX];
+			struct rte_ring *msgq_req;
+			struct rte_ring *msgq_rsp;
+			uint32_t timer_period_ms;
+			uint32_t n_tables;
+		} pipeline_enable;
+
+		struct {
+			struct rte_pipeline *p;
+		} pipeline_disable;
+	};
+};
+
+struct thread_msg_rsp {
+	int status;
+};
+
+/**
+ * Master thread
+ */
+static struct thread_msg_req *
+thread_msg_alloc(void)
+{
+	size_t size = RTE_MAX(sizeof(struct thread_msg_req),
+		sizeof(struct thread_msg_rsp));
+
+	return calloc(1, size);
+}
+
+static void
+thread_msg_free(struct thread_msg_rsp *rsp)
+{
+	free(rsp);
+}
+
+static struct thread_msg_rsp *
+thread_msg_send_recv(struct pmd_internals *softnic,
+	uint32_t thread_id,
+	struct thread_msg_req *req)
+{
+	struct softnic_thread *t = &softnic->thread[thread_id];
+	struct rte_ring *msgq_req = t->msgq_req;
+	struct rte_ring *msgq_rsp = t->msgq_rsp;
+	struct thread_msg_rsp *rsp;
+	int status;
+
+	/* send */
+	do {
+		status = rte_ring_sp_enqueue(msgq_req, req);
+	} while (status == -ENOBUFS);
+
+	/* recv */
+	do {
+		status = rte_ring_sc_dequeue(msgq_rsp, (void **)&rsp);
+	} while (status != 0);
+
+	return rsp;
+}
+
+int
+softnic_thread_pipeline_enable(struct pmd_internals *softnic,
+	uint32_t thread_id,
+	const char *pipeline_name)
+{
+	struct pipeline *p = softnic_pipeline_find(softnic, pipeline_name);
+	struct softnic_thread *t;
+	struct thread_msg_req *req;
+	struct thread_msg_rsp *rsp;
+	uint32_t i;
+	int status;
+
+	/* Check input params */
+	if ((thread_id >= RTE_MAX_LCORE) ||
+		(p == NULL) ||
+		(p->n_ports_in == 0) ||
+		(p->n_ports_out == 0) ||
+		(p->n_tables == 0))
+		return -1;
+
+	t = &softnic->thread[thread_id];
+	if ((t->enabled == 0) ||
+		p->enabled)
+		return -1;
+
+	if (!thread_is_running(thread_id)) {
+		struct softnic_thread_data *td = &softnic->thread_data[thread_id];
+		struct pipeline_data *tdp = &td->pipeline_data[td->n_pipelines];
+
+		if (td->n_pipelines >= THREAD_PIPELINES_MAX)
+			return -1;
+
+		/* Data plane thread */
+		td->p[td->n_pipelines] = p->p;
+
+		tdp->p = p->p;
+		for (i = 0; i < p->n_tables; i++)
+			tdp->table_data[i].a =
+				p->table[i].a;
+		tdp->n_tables = p->n_tables;
+
+		tdp->msgq_req = p->msgq_req;
+		tdp->msgq_rsp = p->msgq_rsp;
+		tdp->timer_period = (rte_get_tsc_hz() * p->timer_period_ms) / 1000;
+		tdp->time_next = rte_get_tsc_cycles() + tdp->timer_period;
+
+		td->n_pipelines++;
+
+		/* Pipeline */
+		p->thread_id = thread_id;
+		p->enabled = 1;
+
+		return 0;
+	}
+
+	/* Allocate request */
+	req = thread_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = THREAD_REQ_PIPELINE_ENABLE;
+	req->pipeline_enable.p = p->p;
+	for (i = 0; i < p->n_tables; i++)
+		req->pipeline_enable.table[i].a =
+			p->table[i].a;
+	req->pipeline_enable.msgq_req = p->msgq_req;
+	req->pipeline_enable.msgq_rsp = p->msgq_rsp;
+	req->pipeline_enable.timer_period_ms = p->timer_period_ms;
+	req->pipeline_enable.n_tables = p->n_tables;
+
+	/* Send request and wait for response */
+	rsp = thread_msg_send_recv(softnic, thread_id, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+
+	/* Free response */
+	thread_msg_free(rsp);
+
+	/* Request completion */
+	if (status)
+		return status;
+
+	p->thread_id = thread_id;
+	p->enabled = 1;
+
+	return 0;
+}
+
+int
+softnic_thread_pipeline_disable(struct pmd_internals *softnic,
+	uint32_t thread_id,
+	const char *pipeline_name)
+{
+	struct pipeline *p = softnic_pipeline_find(softnic, pipeline_name);
+	struct softnic_thread *t;
+	struct thread_msg_req *req;
+	struct thread_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if ((thread_id >= RTE_MAX_LCORE) ||
+		(p == NULL))
+		return -1;
+
+	t = &softnic->thread[thread_id];
+	if (t->enabled == 0)
+		return -1;
+
+	if (p->enabled == 0)
+		return 0;
+
+	if (p->thread_id != thread_id)
+		return -1;
+
+	if (!thread_is_running(thread_id)) {
+		struct softnic_thread_data *td = &softnic->thread_data[thread_id];
+		uint32_t i;
+
+		for (i = 0; i < td->n_pipelines; i++) {
+			struct pipeline_data *tdp = &td->pipeline_data[i];
+
+			if (tdp->p != p->p)
+				continue;
+
+			/* Data plane thread */
+			if (i < td->n_pipelines - 1) {
+				struct rte_pipeline *pipeline_last =
+					td->p[td->n_pipelines - 1];
+				struct pipeline_data *tdp_last =
+					&td->pipeline_data[td->n_pipelines - 1];
+
+				td->p[i] = pipeline_last;
+				memcpy(tdp, tdp_last, sizeof(*tdp));
+			}
+
+			td->n_pipelines--;
+
+			/* Pipeline */
+			p->enabled = 0;
+
+			break;
+		}
+
+		return 0;
+	}
+
+	/* Allocate request */
+	req = thread_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = THREAD_REQ_PIPELINE_DISABLE;
+	req->pipeline_disable.p = p->p;
+
+	/* Send request and wait for response */
+	rsp = thread_msg_send_recv(softnic, thread_id, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+
+	/* Free response */
+	thread_msg_free(rsp);
+
+	/* Request completion */
+	if (status)
+		return status;
+
+	p->enabled = 0;
+
+	return 0;
+}
+
+/**
+ * Data plane threads: message handling
+ */
+static inline struct thread_msg_req *
+thread_msg_recv(struct rte_ring *msgq_req)
+{
+	struct thread_msg_req *req;
+
+	int status = rte_ring_sc_dequeue(msgq_req, (void **)&req);
+
+	if (status != 0)
+		return NULL;
+
+	return req;
+}
+
+static inline void
+thread_msg_send(struct rte_ring *msgq_rsp,
+	struct thread_msg_rsp *rsp)
+{
+	int status;
+
+	do {
+		status = rte_ring_sp_enqueue(msgq_rsp, rsp);
+	} while (status == -ENOBUFS);
+}
+
+static struct thread_msg_rsp *
+thread_msg_handle_pipeline_enable(struct softnic_thread_data *t,
+	struct thread_msg_req *req)
+{
+	struct thread_msg_rsp *rsp = (struct thread_msg_rsp *)req;
+	struct pipeline_data *p = &t->pipeline_data[t->n_pipelines];
+	uint32_t i;
+
+	/* Request */
+	if (t->n_pipelines >= THREAD_PIPELINES_MAX) {
+		rsp->status = -1;
+		return rsp;
+	}
+
+	t->p[t->n_pipelines] = req->pipeline_enable.p;
+
+	p->p = req->pipeline_enable.p;
+	for (i = 0; i < req->pipeline_enable.n_tables; i++)
+		p->table_data[i].a =
+			req->pipeline_enable.table[i].a;
+
+	p->n_tables = req->pipeline_enable.n_tables;
+
+	p->msgq_req = req->pipeline_enable.msgq_req;
+	p->msgq_rsp = req->pipeline_enable.msgq_rsp;
+	p->timer_period =
+		(rte_get_tsc_hz() * req->pipeline_enable.timer_period_ms) / 1000;
+	p->time_next = rte_get_tsc_cycles() + p->timer_period;
+
+	t->n_pipelines++;
+
+	/* Response */
+	rsp->status = 0;
+	return rsp;
+}
+
+static struct thread_msg_rsp *
+thread_msg_handle_pipeline_disable(struct softnic_thread_data *t,
+	struct thread_msg_req *req)
+{
+	struct thread_msg_rsp *rsp = (struct thread_msg_rsp *)req;
+	uint32_t n_pipelines = t->n_pipelines;
+	struct rte_pipeline *pipeline = req->pipeline_disable.p;
+	uint32_t i;
+
+	/* find pipeline */
+	for (i = 0; i < n_pipelines; i++) {
+		struct pipeline_data *p = &t->pipeline_data[i];
+
+		if (p->p != pipeline)
+			continue;
+
+		if (i < n_pipelines - 1) {
+			struct rte_pipeline *pipeline_last =
+				t->p[n_pipelines - 1];
+			struct pipeline_data *p_last =
+				&t->pipeline_data[n_pipelines - 1];
+
+			t->p[i] = pipeline_last;
+			memcpy(p, p_last, sizeof(*p));
+		}
+
+		t->n_pipelines--;
+
+		rsp->status = 0;
+		return rsp;
+	}
+
+	/* should not get here */
+	rsp->status = 0;
+	return rsp;
+}
+
+static void
+thread_msg_handle(struct softnic_thread_data *t)
+{
+	for ( ; ; ) {
+		struct thread_msg_req *req;
+		struct thread_msg_rsp *rsp;
+
+		req = thread_msg_recv(t->msgq_req);
+		if (req == NULL)
+			break;
+
+		switch (req->type) {
+		case THREAD_REQ_PIPELINE_ENABLE:
+			rsp = thread_msg_handle_pipeline_enable(t, req);
+			break;
+
+		case THREAD_REQ_PIPELINE_DISABLE:
+			rsp = thread_msg_handle_pipeline_disable(t, req);
+			break;
+
+		default:
+			rsp = (struct thread_msg_rsp *)req;
+			rsp->status = -1;
+		}
+
+		thread_msg_send(t->msgq_rsp, rsp);
+	}
+}
+
+/**
+ * Master thread & data plane threads: message passing
+ */
+enum pipeline_req_type {
+	/* Port IN */
+	PIPELINE_REQ_PORT_IN_STATS_READ,
+	PIPELINE_REQ_PORT_IN_ENABLE,
+	PIPELINE_REQ_PORT_IN_DISABLE,
+
+	/* Port OUT */
+	PIPELINE_REQ_PORT_OUT_STATS_READ,
+
+	/* Table */
+	PIPELINE_REQ_TABLE_STATS_READ,
+	PIPELINE_REQ_TABLE_RULE_ADD,
+	PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT,
+	PIPELINE_REQ_TABLE_RULE_ADD_BULK,
+	PIPELINE_REQ_TABLE_RULE_DELETE,
+	PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT,
+	PIPELINE_REQ_TABLE_RULE_STATS_READ,
+	PIPELINE_REQ_TABLE_MTR_PROFILE_ADD,
+	PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE,
+	PIPELINE_REQ_TABLE_RULE_MTR_READ,
+	PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE,
+	PIPELINE_REQ_TABLE_RULE_TTL_READ,
+	PIPELINE_REQ_MAX
+};
+
+struct pipeline_msg_req_port_in_stats_read {
+	int clear;
+};
+
+struct pipeline_msg_req_port_out_stats_read {
+	int clear;
+};
+
+struct pipeline_msg_req_table_stats_read {
+	int clear;
+};
+
+struct pipeline_msg_req_table_rule_add {
+	struct softnic_table_rule_match match;
+	struct softnic_table_rule_action action;
+};
+
+struct pipeline_msg_req_table_rule_add_default {
+	struct softnic_table_rule_action action;
+};
+
+struct pipeline_msg_req_table_rule_add_bulk {
+	struct softnic_table_rule_match *match;
+	struct softnic_table_rule_action *action;
+	void **data;
+	uint32_t n_rules;
+	int bulk;
+};
+
+struct pipeline_msg_req_table_rule_delete {
+	struct softnic_table_rule_match match;
+};
+
+struct pipeline_msg_req_table_rule_stats_read {
+	void *data;
+	int clear;
+};
+
+struct pipeline_msg_req_table_mtr_profile_add {
+	uint32_t meter_profile_id;
+	struct rte_table_action_meter_profile profile;
+};
+
+struct pipeline_msg_req_table_mtr_profile_delete {
+	uint32_t meter_profile_id;
+};
+
+struct pipeline_msg_req_table_rule_mtr_read {
+	void *data;
+	uint32_t tc_mask;
+	int clear;
+};
+
+struct pipeline_msg_req_table_dscp_table_update {
+	uint64_t dscp_mask;
+	struct rte_table_action_dscp_table dscp_table;
+};
+
+struct pipeline_msg_req_table_rule_ttl_read {
+	void *data;
+	int clear;
+};
+
+struct pipeline_msg_req {
+	enum pipeline_req_type type;
+	uint32_t id; /* Port IN, port OUT or table ID */
+
+	RTE_STD_C11
+	union {
+		struct pipeline_msg_req_port_in_stats_read port_in_stats_read;
+		struct pipeline_msg_req_port_out_stats_read port_out_stats_read;
+		struct pipeline_msg_req_table_stats_read table_stats_read;
+		struct pipeline_msg_req_table_rule_add table_rule_add;
+		struct pipeline_msg_req_table_rule_add_default table_rule_add_default;
+		struct pipeline_msg_req_table_rule_add_bulk table_rule_add_bulk;
+		struct pipeline_msg_req_table_rule_delete table_rule_delete;
+		struct pipeline_msg_req_table_rule_stats_read table_rule_stats_read;
+		struct pipeline_msg_req_table_mtr_profile_add table_mtr_profile_add;
+		struct pipeline_msg_req_table_mtr_profile_delete table_mtr_profile_delete;
+		struct pipeline_msg_req_table_rule_mtr_read table_rule_mtr_read;
+		struct pipeline_msg_req_table_dscp_table_update table_dscp_table_update;
+		struct pipeline_msg_req_table_rule_ttl_read table_rule_ttl_read;
+	};
+};
+
+struct pipeline_msg_rsp_port_in_stats_read {
+	struct rte_pipeline_port_in_stats stats;
+};
+
+struct pipeline_msg_rsp_port_out_stats_read {
+	struct rte_pipeline_port_out_stats stats;
+};
+
+struct pipeline_msg_rsp_table_stats_read {
+	struct rte_pipeline_table_stats stats;
+};
+
+struct pipeline_msg_rsp_table_rule_add {
+	void *data;
+};
+
+struct pipeline_msg_rsp_table_rule_add_default {
+	void *data;
+};
+
+struct pipeline_msg_rsp_table_rule_add_bulk {
+	uint32_t n_rules;
+};
+
+struct pipeline_msg_rsp_table_rule_stats_read {
+	struct rte_table_action_stats_counters stats;
+};
+
+struct pipeline_msg_rsp_table_rule_mtr_read {
+	struct rte_table_action_mtr_counters stats;
+};
+
+struct pipeline_msg_rsp_table_rule_ttl_read {
+	struct rte_table_action_ttl_counters stats;
+};
+
+struct pipeline_msg_rsp {
+	int status;
+
+	RTE_STD_C11
+	union {
+		struct pipeline_msg_rsp_port_in_stats_read port_in_stats_read;
+		struct pipeline_msg_rsp_port_out_stats_read port_out_stats_read;
+		struct pipeline_msg_rsp_table_stats_read table_stats_read;
+		struct pipeline_msg_rsp_table_rule_add table_rule_add;
+		struct pipeline_msg_rsp_table_rule_add_default table_rule_add_default;
+		struct pipeline_msg_rsp_table_rule_add_bulk table_rule_add_bulk;
+		struct pipeline_msg_rsp_table_rule_stats_read table_rule_stats_read;
+		struct pipeline_msg_rsp_table_rule_mtr_read table_rule_mtr_read;
+		struct pipeline_msg_rsp_table_rule_ttl_read table_rule_ttl_read;
+	};
+};
+
+/**
+ * Master thread
+ */
+static struct pipeline_msg_req *
+pipeline_msg_alloc(void)
+{
+	size_t size = RTE_MAX(sizeof(struct pipeline_msg_req),
+		sizeof(struct pipeline_msg_rsp));
+
+	return calloc(1, size);
+}
+
+static void
+pipeline_msg_free(struct pipeline_msg_rsp *rsp)
+{
+	free(rsp);
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_send_recv(struct pipeline *p,
+	struct pipeline_msg_req *req)
+{
+	struct rte_ring *msgq_req = p->msgq_req;
+	struct rte_ring *msgq_rsp = p->msgq_rsp;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* send */
+	do {
+		status = rte_ring_sp_enqueue(msgq_req, req);
+	} while (status == -ENOBUFS);
+
+	/* recv */
+	do {
+		status = rte_ring_sc_dequeue(msgq_rsp, (void **)&rsp);
+	} while (status != 0);
+
+	return rsp;
+}
+
+int
+softnic_pipeline_port_in_stats_read(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t port_id,
+	struct rte_pipeline_port_in_stats *stats,
+	int clear)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		stats == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		port_id >= p->n_ports_in)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_port_in_stats_read(p->p,
+			port_id,
+			stats,
+			clear);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_PORT_IN_STATS_READ;
+	req->id = port_id;
+	req->port_in_stats_read.clear = clear;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+	if (status)
+		memcpy(stats, &rsp->port_in_stats_read.stats, sizeof(*stats));
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_port_in_enable(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t port_id)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		port_id >= p->n_ports_in)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_port_in_enable(p->p, port_id);
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_PORT_IN_ENABLE;
+	req->id = port_id;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_port_in_disable(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t port_id)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		port_id >= p->n_ports_in)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_port_in_disable(p->p, port_id);
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_PORT_IN_DISABLE;
+	req->id = port_id;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_port_out_stats_read(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t port_id,
+	struct rte_pipeline_port_out_stats *stats,
+	int clear)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		stats == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		port_id >= p->n_ports_out)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_port_out_stats_read(p->p,
+			port_id,
+			stats,
+			clear);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_PORT_OUT_STATS_READ;
+	req->id = port_id;
+	req->port_out_stats_read.clear = clear;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+	if (status)
+		memcpy(stats, &rsp->port_out_stats_read.stats, sizeof(*stats));
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_table_stats_read(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id,
+	struct rte_pipeline_table_stats *stats,
+	int clear)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		stats == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_table_stats_read(p->p,
+			table_id,
+			stats,
+			clear);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_STATS_READ;
+	req->id = table_id;
+	req->table_stats_read.clear = clear;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+	if (status)
+		memcpy(stats, &rsp->table_stats_read.stats, sizeof(*stats));
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+static int
+match_check(struct softnic_table_rule_match *match,
+	struct pipeline *p,
+	uint32_t table_id)
+{
+	struct softnic_table *table;
+
+	if (match == NULL ||
+		p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	table = &p->table[table_id];
+	if (match->match_type != table->params.match_type)
+		return -1;
+
+	switch (match->match_type) {
+	case TABLE_ACL:
+	{
+		struct softnic_table_acl_params *t = &table->params.match.acl;
+		struct softnic_table_rule_match_acl *r = &match->match.acl;
+
+		if ((r->ip_version && (t->ip_version == 0)) ||
+			((r->ip_version == 0) && t->ip_version))
+			return -1;
+
+		if (r->ip_version) {
+			if (r->sa_depth > 32 ||
+				r->da_depth > 32)
+				return -1;
+		} else {
+			if (r->sa_depth > 128 ||
+				r->da_depth > 128)
+				return -1;
+		}
+		return 0;
+	}
+
+	case TABLE_ARRAY:
+		return 0;
+
+	case TABLE_HASH:
+		return 0;
+
+	case TABLE_LPM:
+	{
+		struct softnic_table_lpm_params *t = &table->params.match.lpm;
+		struct softnic_table_rule_match_lpm *r = &match->match.lpm;
+
+		if ((r->ip_version && (t->key_size != 4)) ||
+			((r->ip_version == 0) && (t->key_size != 16)))
+			return -1;
+
+		if (r->ip_version) {
+			if (r->depth > 32)
+				return -1;
+		} else {
+			if (r->depth > 128)
+				return -1;
+		}
+		return 0;
+	}
+
+	case TABLE_STUB:
+		return -1;
+
+	default:
+		return -1;
+	}
+}
+
+static int
+action_check(struct softnic_table_rule_action *action,
+	struct pipeline *p,
+	uint32_t table_id)
+{
+	struct softnic_table_action_profile *ap;
+
+	if (action == NULL ||
+		p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	ap = p->table[table_id].ap;
+	if (action->action_mask != ap->params.action_mask)
+		return -1;
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
+		if (action->fwd.action == RTE_PIPELINE_ACTION_PORT &&
+			action->fwd.id >= p->n_ports_out)
+			return -1;
+
+		if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE &&
+			action->fwd.id >= p->n_tables)
+			return -1;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
+		uint32_t tc_mask0 = (1 << ap->params.mtr.n_tc) - 1;
+		uint32_t tc_mask1 = action->mtr.tc_mask;
+
+		if (tc_mask1 != tc_mask0)
+			return -1;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
+		uint32_t n_subports_per_port =
+			ap->params.tm.n_subports_per_port;
+		uint32_t n_pipes_per_subport =
+			ap->params.tm.n_pipes_per_subport;
+		uint32_t subport_id = action->tm.subport_id;
+		uint32_t pipe_id = action->tm.pipe_id;
+
+		if (subport_id >= n_subports_per_port ||
+			pipe_id >= n_pipes_per_subport)
+			return -1;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
+		uint64_t encap_mask = ap->params.encap.encap_mask;
+		enum rte_table_action_encap_type type = action->encap.type;
+
+		if ((encap_mask & (1LLU << type)) == 0)
+			return -1;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
+		int ip_version0 = ap->params.common.ip_version;
+		int ip_version1 = action->nat.ip_version;
+
+		if ((ip_version1 && (ip_version0 == 0)) ||
+			((ip_version1 == 0) && ip_version0))
+			return -1;
+	}
+
+	return 0;
+}
+
+static int
+action_default_check(struct softnic_table_rule_action *action,
+	struct pipeline *p,
+	uint32_t table_id)
+{
+	if (action == NULL ||
+		action->action_mask != (1LLU << RTE_TABLE_ACTION_FWD) ||
+		p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
+		if (action->fwd.action == RTE_PIPELINE_ACTION_PORT &&
+			action->fwd.id >= p->n_ports_out)
+			return -1;
+
+		if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE &&
+			action->fwd.id >= p->n_tables)
+			return -1;
+	}
+
+	return 0;
+}
+
+union table_rule_match_low_level {
+	struct rte_table_acl_rule_add_params acl_add;
+	struct rte_table_acl_rule_delete_params acl_delete;
+	struct rte_table_array_key array;
+	uint8_t hash[TABLE_RULE_MATCH_SIZE_MAX];
+	struct rte_table_lpm_key lpm_ipv4;
+	struct rte_table_lpm_ipv6_key lpm_ipv6;
+};
+
+static int
+match_convert(struct softnic_table_rule_match *mh,
+	union table_rule_match_low_level *ml,
+	int add);
+
+static int
+action_convert(struct rte_table_action *a,
+	struct softnic_table_rule_action *action,
+	struct rte_pipeline_table_entry *data);
+
+int
+softnic_pipeline_table_rule_add(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id,
+	struct softnic_table_rule_match *match,
+	struct softnic_table_rule_action *action,
+	void **data)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		match == NULL ||
+		action == NULL ||
+		data == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables ||
+		match_check(match, p, table_id) ||
+		action_check(action, p, table_id))
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+		union table_rule_match_low_level match_ll;
+		struct rte_pipeline_table_entry *data_in, *data_out;
+		int key_found;
+		uint8_t *buffer;
+
+		buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
+		if (buffer == NULL)
+			return -1;
+
+		/* Table match-action rule conversion */
+		data_in = (struct rte_pipeline_table_entry *)buffer;
+
+		status = match_convert(match, &match_ll, 1);
+		if (status) {
+			free(buffer);
+			return -1;
+		}
+
+		status = action_convert(a, action, data_in);
+		if (status) {
+			free(buffer);
+			return -1;
+		}
+
+		/* Add rule (match, action) to table */
+		status = rte_pipeline_table_entry_add(p->p,
+				table_id,
+				&match_ll,
+				data_in,
+				&key_found,
+				&data_out);
+		if (status) {
+			free(buffer);
+			return -1;
+		}
+
+		/* Write Response */
+		*data = data_out;
+
+		free(buffer);
+		return 0;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_RULE_ADD;
+	req->id = table_id;
+	memcpy(&req->table_rule_add.match, match, sizeof(*match));
+	memcpy(&req->table_rule_add.action, action, sizeof(*action));
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+	if (status == 0)
+		*data = rsp->table_rule_add.data;
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_table_rule_add_default(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id,
+	struct softnic_table_rule_action *action,
+	void **data)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		action == NULL ||
+		data == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables ||
+		action_default_check(action, p, table_id))
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		struct rte_pipeline_table_entry *data_in, *data_out;
+		uint8_t *buffer;
+
+		buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
+		if (buffer == NULL)
+			return -1;
+
+		/* Apply actions */
+		data_in = (struct rte_pipeline_table_entry *)buffer;
+
+		data_in->action = action->fwd.action;
+		if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
+			data_in->port_id = action->fwd.id;
+		if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
+			data_in->table_id = action->fwd.id;
+
+		/* Add default rule to table */
+		status = rte_pipeline_table_default_entry_add(p->p,
+				table_id,
+				data_in,
+				&data_out);
+		if (status) {
+			free(buffer);
+			return -1;
+		}
+
+		/* Write Response */
+		*data = data_out;
+
+		free(buffer);
+		return 0;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT;
+	req->id = table_id;
+	memcpy(&req->table_rule_add_default.action, action, sizeof(*action));
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+	if (status == 0)
+		*data = rsp->table_rule_add_default.data;
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_table_rule_add_bulk(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id,
+	struct softnic_table_rule_match *match,
+	struct softnic_table_rule_action *action,
+	void **data,
+	uint32_t *n_rules)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	uint32_t i;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		match == NULL ||
+		action == NULL ||
+		data == NULL ||
+		n_rules == NULL ||
+		(*n_rules == 0))
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	for (i = 0; i < *n_rules; i++)
+		if (match_check(match, p, table_id) ||
+			action_check(action, p, table_id))
+			return -1;
+
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+		union table_rule_match_low_level *match_ll;
+		uint8_t *action_ll;
+		void **match_ll_ptr;
+		struct rte_pipeline_table_entry **action_ll_ptr;
+		struct rte_pipeline_table_entry **entries_ptr =
+			(struct rte_pipeline_table_entry **)data;
+		uint32_t bulk =
+			(p->table[table_id].params.match_type == TABLE_ACL) ? 1 : 0;
+		int *found;
+
+		/* Memory allocation */
+		match_ll = calloc(*n_rules, sizeof(union table_rule_match_low_level));
+		action_ll = calloc(*n_rules, TABLE_RULE_ACTION_SIZE_MAX);
+		match_ll_ptr = calloc(*n_rules, sizeof(void *));
+		action_ll_ptr =
+			calloc(*n_rules, sizeof(struct rte_pipeline_table_entry *));
+		found = calloc(*n_rules, sizeof(int));
+
+		if (match_ll == NULL ||
+			action_ll == NULL ||
+			match_ll_ptr == NULL ||
+			action_ll_ptr == NULL ||
+			found == NULL)
+			goto fail;
+
+		for (i = 0; i < *n_rules; i++) {
+			match_ll_ptr[i] = (void *)&match_ll[i];
+			action_ll_ptr[i] =
+				(struct rte_pipeline_table_entry *)&action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
+		}
+
+		/* Rule match conversion */
+		for (i = 0; i < *n_rules; i++) {
+			status = match_convert(&match[i], match_ll_ptr[i], 1);
+			if (status)
+				goto fail;
+		}
+
+		/* Rule action conversion */
+		for (i = 0; i < *n_rules; i++) {
+			status = action_convert(a, &action[i], action_ll_ptr[i]);
+			if (status)
+				goto fail;
+		}
+
+		/* Add rule (match, action) to table */
+		if (bulk) {
+			status = rte_pipeline_table_entry_add_bulk(p->p,
+				table_id,
+				match_ll_ptr,
+				action_ll_ptr,
+				*n_rules,
+				found,
+				entries_ptr);
+			if (status)
+				*n_rules = 0;
+		} else {
+			for (i = 0; i < *n_rules; i++) {
+				status = rte_pipeline_table_entry_add(p->p,
+					table_id,
+					match_ll_ptr[i],
+					action_ll_ptr[i],
+					&found[i],
+					&entries_ptr[i]);
+				if (status) {
+					*n_rules = i;
+					break;
+				}
+			}
+		}
+
+		/* Free */
+		free(found);
+		free(action_ll_ptr);
+		free(match_ll_ptr);
+		free(action_ll);
+		free(match_ll);
+
+		return status;
+
+fail:
+		free(found);
+		free(action_ll_ptr);
+		free(match_ll_ptr);
+		free(action_ll);
+		free(match_ll);
+
+		*n_rules = 0;
+		return -1;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_RULE_ADD_BULK;
+	req->id = table_id;
+	req->table_rule_add_bulk.match = match;
+	req->table_rule_add_bulk.action = action;
+	req->table_rule_add_bulk.data = data;
+	req->table_rule_add_bulk.n_rules = *n_rules;
+	req->table_rule_add_bulk.bulk =
+		(p->table[table_id].params.match_type == TABLE_ACL) ? 1 : 0;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+	if (status == 0)
+		*n_rules = rsp->table_rule_add_bulk.n_rules;
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_table_rule_delete(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id,
+	struct softnic_table_rule_match *match)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		match == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables ||
+		match_check(match, p, table_id))
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		union table_rule_match_low_level match_ll;
+		int key_found;
+
+		status = match_convert(match, &match_ll, 0);
+		if (status)
+			return -1;
+
+		status = rte_pipeline_table_entry_delete(p->p,
+				table_id,
+				&match_ll,
+				&key_found,
+				NULL);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_RULE_DELETE;
+	req->id = table_id;
+	memcpy(&req->table_rule_delete.match, match, sizeof(*match));
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_table_rule_delete_default(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_table_default_entry_delete(p->p,
+			table_id,
+			NULL);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT;
+	req->id = table_id;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_table_rule_stats_read(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id,
+	void *data,
+	struct rte_table_action_stats_counters *stats,
+	int clear)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		data == NULL ||
+		stats == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_stats_read(a,
+			data,
+			stats,
+			clear);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_RULE_STATS_READ;
+	req->id = table_id;
+	req->table_rule_stats_read.data = data;
+	req->table_rule_stats_read.clear = clear;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+	if (status)
+		memcpy(stats, &rsp->table_rule_stats_read.stats, sizeof(*stats));
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_table_mtr_profile_add(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id,
+	uint32_t meter_profile_id,
+	struct rte_table_action_meter_profile *profile)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		profile == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_meter_profile_add(a,
+			meter_profile_id,
+			profile);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_MTR_PROFILE_ADD;
+	req->id = table_id;
+	req->table_mtr_profile_add.meter_profile_id = meter_profile_id;
+	memcpy(&req->table_mtr_profile_add.profile, profile, sizeof(*profile));
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_table_mtr_profile_delete(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id,
+	uint32_t meter_profile_id)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_meter_profile_delete(a,
+				meter_profile_id);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE;
+	req->id = table_id;
+	req->table_mtr_profile_delete.meter_profile_id = meter_profile_id;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_table_rule_mtr_read(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id,
+	void *data,
+	uint32_t tc_mask,
+	struct rte_table_action_mtr_counters *stats,
+	int clear)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		data == NULL ||
+		stats == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_meter_read(a,
+				data,
+				tc_mask,
+				stats,
+				clear);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_RULE_MTR_READ;
+	req->id = table_id;
+	req->table_rule_mtr_read.data = data;
+	req->table_rule_mtr_read.tc_mask = tc_mask;
+	req->table_rule_mtr_read.clear = clear;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+	if (status)
+		memcpy(stats, &rsp->table_rule_mtr_read.stats, sizeof(*stats));
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_table_dscp_table_update(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id,
+	uint64_t dscp_mask,
+	struct rte_table_action_dscp_table *dscp_table)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		dscp_table == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_dscp_table_update(a,
+				dscp_mask,
+				dscp_table);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE;
+	req->id = table_id;
+	req->table_dscp_table_update.dscp_mask = dscp_mask;
+	memcpy(&req->table_dscp_table_update.dscp_table,
+		dscp_table, sizeof(*dscp_table));
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+int
+softnic_pipeline_table_rule_ttl_read(struct pmd_internals *softnic,
+	const char *pipeline_name,
+	uint32_t table_id,
+	void *data,
+	struct rte_table_action_ttl_counters *stats,
+	int clear)
+{
+	struct pipeline *p;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	int status;
+
+	/* Check input params */
+	if (pipeline_name == NULL ||
+		data == NULL ||
+		stats == NULL)
+		return -1;
+
+	p = softnic_pipeline_find(softnic, pipeline_name);
+	if (p == NULL ||
+		table_id >= p->n_tables)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_ttl_read(a,
+				data,
+				stats,
+				clear);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_RULE_TTL_READ;
+	req->id = table_id;
+	req->table_rule_ttl_read.data = data;
+	req->table_rule_ttl_read.clear = clear;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+	if (status)
+		memcpy(stats, &rsp->table_rule_ttl_read.stats, sizeof(*stats));
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
+/**
+ * Data plane threads: message handling
+ */
+static inline struct pipeline_msg_req *
+pipeline_msg_recv(struct rte_ring *msgq_req)
+{
+	struct pipeline_msg_req *req;
+
+	int status = rte_ring_sc_dequeue(msgq_req, (void **)&req);
+
+	if (status != 0)
+		return NULL;
+
+	return req;
+}
+
+static inline void
+pipeline_msg_send(struct rte_ring *msgq_rsp,
+	struct pipeline_msg_rsp *rsp)
+{
+	int status;
+
+	do {
+		status = rte_ring_sp_enqueue(msgq_rsp, rsp);
+	} while (status == -ENOBUFS);
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_port_in_stats_read(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t port_id = req->id;
+	int clear = req->port_in_stats_read.clear;
+
+	rsp->status = rte_pipeline_port_in_stats_read(p->p,
+		port_id,
+		&rsp->port_in_stats_read.stats,
+		clear);
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_port_in_enable(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t port_id = req->id;
+
+	rsp->status = rte_pipeline_port_in_enable(p->p,
+		port_id);
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_port_in_disable(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t port_id = req->id;
+
+	rsp->status = rte_pipeline_port_in_disable(p->p,
+		port_id);
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_port_out_stats_read(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t port_id = req->id;
+	int clear = req->port_out_stats_read.clear;
+
+	rsp->status = rte_pipeline_port_out_stats_read(p->p,
+		port_id,
+		&rsp->port_out_stats_read.stats,
+		clear);
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_stats_read(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t port_id = req->id;
+	int clear = req->table_stats_read.clear;
+
+	rsp->status = rte_pipeline_table_stats_read(p->p,
+		port_id,
+		&rsp->table_stats_read.stats,
+		clear);
+
+	return rsp;
+}
+
+static int
+match_convert_ipv6_depth(uint32_t depth, uint32_t *depth32)
+{
+	if (depth > 128)
+		return -1;
+
+	switch (depth / 32) {
+	case 0:
+		depth32[0] = depth;
+		depth32[1] = 0;
+		depth32[2] = 0;
+		depth32[3] = 0;
+		return 0;
+
+	case 1:
+		depth32[0] = 32;
+		depth32[1] = depth - 32;
+		depth32[2] = 0;
+		depth32[3] = 0;
+		return 0;
+
+	case 2:
+		depth32[0] = 32;
+		depth32[1] = 32;
+		depth32[2] = depth - 64;
+		depth32[3] = 0;
+		return 0;
+
+	case 3:
+		depth32[0] = 32;
+		depth32[1] = 32;
+		depth32[2] = 32;
+		depth32[3] = depth - 96;
+		return 0;
+
+	case 4:
+		depth32[0] = 32;
+		depth32[1] = 32;
+		depth32[2] = 32;
+		depth32[3] = 32;
+		return 0;
+
+	default:
+		return -1;
+	}
+}
+
+static int
+match_convert(struct softnic_table_rule_match *mh,
+	union table_rule_match_low_level *ml,
+	int add)
+{
+	memset(ml, 0, sizeof(*ml));
+
+	switch (mh->match_type) {
+	case TABLE_ACL:
+		if (mh->match.acl.ip_version)
+			if (add) {
+				ml->acl_add.field_value[0].value.u8 =
+					mh->match.acl.proto;
+				ml->acl_add.field_value[0].mask_range.u8 =
+					mh->match.acl.proto_mask;
+
+				ml->acl_add.field_value[1].value.u32 =
+					mh->match.acl.ipv4.sa;
+				ml->acl_add.field_value[1].mask_range.u32 =
+					mh->match.acl.sa_depth;
+
+				ml->acl_add.field_value[2].value.u32 =
+					mh->match.acl.ipv4.da;
+				ml->acl_add.field_value[2].mask_range.u32 =
+					mh->match.acl.da_depth;
+
+				ml->acl_add.field_value[3].value.u16 =
+					mh->match.acl.sp0;
+				ml->acl_add.field_value[3].mask_range.u16 =
+					mh->match.acl.sp1;
+
+				ml->acl_add.field_value[4].value.u16 =
+					mh->match.acl.dp0;
+				ml->acl_add.field_value[4].mask_range.u16 =
+					mh->match.acl.dp1;
+
+				ml->acl_add.priority =
+					(int32_t)mh->match.acl.priority;
+			} else {
+				ml->acl_delete.field_value[0].value.u8 =
+					mh->match.acl.proto;
+				ml->acl_delete.field_value[0].mask_range.u8 =
+					mh->match.acl.proto_mask;
+
+				ml->acl_delete.field_value[1].value.u32 =
+					mh->match.acl.ipv4.sa;
+				ml->acl_delete.field_value[1].mask_range.u32 =
+					mh->match.acl.sa_depth;
+
+				ml->acl_delete.field_value[2].value.u32 =
+					mh->match.acl.ipv4.da;
+				ml->acl_delete.field_value[2].mask_range.u32 =
+					mh->match.acl.da_depth;
+
+				ml->acl_delete.field_value[3].value.u16 =
+					mh->match.acl.sp0;
+				ml->acl_delete.field_value[3].mask_range.u16 =
+					mh->match.acl.sp1;
+
+				ml->acl_delete.field_value[4].value.u16 =
+					mh->match.acl.dp0;
+				ml->acl_delete.field_value[4].mask_range.u16 =
+					mh->match.acl.dp1;
+			}
+		else
+			if (add) {
+				uint32_t *sa32 =
+					(uint32_t *)mh->match.acl.ipv6.sa;
+				uint32_t *da32 =
+					(uint32_t *)mh->match.acl.ipv6.da;
+				uint32_t sa32_depth[4], da32_depth[4];
+				int status;
+
+				status = match_convert_ipv6_depth(mh->match.acl.sa_depth,
+					sa32_depth);
+				if (status)
+					return status;
+
+				status = match_convert_ipv6_depth(
+					mh->match.acl.da_depth,
+					da32_depth);
+				if (status)
+					return status;
+
+				ml->acl_add.field_value[0].value.u8 =
+					mh->match.acl.proto;
+				ml->acl_add.field_value[0].mask_range.u8 =
+					mh->match.acl.proto_mask;
+
+				ml->acl_add.field_value[1].value.u32 = sa32[0];
+				ml->acl_add.field_value[1].mask_range.u32 =
+					sa32_depth[0];
+				ml->acl_add.field_value[2].value.u32 = sa32[1];
+				ml->acl_add.field_value[2].mask_range.u32 =
+					sa32_depth[1];
+				ml->acl_add.field_value[3].value.u32 = sa32[2];
+				ml->acl_add.field_value[3].mask_range.u32 =
+					sa32_depth[2];
+				ml->acl_add.field_value[4].value.u32 = sa32[3];
+				ml->acl_add.field_value[4].mask_range.u32 =
+					sa32_depth[3];
+
+				ml->acl_add.field_value[5].value.u32 = da32[0];
+				ml->acl_add.field_value[5].mask_range.u32 =
+					da32_depth[0];
+				ml->acl_add.field_value[6].value.u32 = da32[1];
+				ml->acl_add.field_value[6].mask_range.u32 =
+					da32_depth[1];
+				ml->acl_add.field_value[7].value.u32 = da32[2];
+				ml->acl_add.field_value[7].mask_range.u32 =
+					da32_depth[2];
+				ml->acl_add.field_value[8].value.u32 = da32[3];
+				ml->acl_add.field_value[8].mask_range.u32 =
+					da32_depth[3];
+
+				ml->acl_add.field_value[9].value.u16 =
+					mh->match.acl.sp0;
+				ml->acl_add.field_value[9].mask_range.u16 =
+					mh->match.acl.sp1;
+
+				ml->acl_add.field_value[10].value.u16 =
+					mh->match.acl.dp0;
+				ml->acl_add.field_value[10].mask_range.u16 =
+					mh->match.acl.dp1;
+
+				ml->acl_add.priority =
+					(int32_t)mh->match.acl.priority;
+			} else {
+				uint32_t *sa32 =
+					(uint32_t *)mh->match.acl.ipv6.sa;
+				uint32_t *da32 =
+					(uint32_t *)mh->match.acl.ipv6.da;
+				uint32_t sa32_depth[4], da32_depth[4];
+				int status;
+
+				status = match_convert_ipv6_depth(mh->match.acl.sa_depth,
+					sa32_depth);
+				if (status)
+					return status;
+
+				status = match_convert_ipv6_depth(mh->match.acl.da_depth,
+					da32_depth);
+				if (status)
+					return status;
+
+				ml->acl_delete.field_value[0].value.u8 =
+					mh->match.acl.proto;
+				ml->acl_delete.field_value[0].mask_range.u8 =
+					mh->match.acl.proto_mask;
+
+				ml->acl_delete.field_value[1].value.u32 =
+					sa32[0];
+				ml->acl_delete.field_value[1].mask_range.u32 =
+					sa32_depth[0];
+				ml->acl_delete.field_value[2].value.u32 =
+					sa32[1];
+				ml->acl_delete.field_value[2].mask_range.u32 =
+					sa32_depth[1];
+				ml->acl_delete.field_value[3].value.u32 =
+					sa32[2];
+				ml->acl_delete.field_value[3].mask_range.u32 =
+					sa32_depth[2];
+				ml->acl_delete.field_value[4].value.u32 =
+					sa32[3];
+				ml->acl_delete.field_value[4].mask_range.u32 =
+					sa32_depth[3];
+
+				ml->acl_delete.field_value[5].value.u32 =
+					da32[0];
+				ml->acl_delete.field_value[5].mask_range.u32 =
+					da32_depth[0];
+				ml->acl_delete.field_value[6].value.u32 =
+					da32[1];
+				ml->acl_delete.field_value[6].mask_range.u32 =
+					da32_depth[1];
+				ml->acl_delete.field_value[7].value.u32 =
+					da32[2];
+				ml->acl_delete.field_value[7].mask_range.u32 =
+					da32_depth[2];
+				ml->acl_delete.field_value[8].value.u32 =
+					da32[3];
+				ml->acl_delete.field_value[8].mask_range.u32 =
+					da32_depth[3];
+
+				ml->acl_delete.field_value[9].value.u16 =
+					mh->match.acl.sp0;
+				ml->acl_delete.field_value[9].mask_range.u16 =
+					mh->match.acl.sp1;
+
+				ml->acl_delete.field_value[10].value.u16 =
+					mh->match.acl.dp0;
+				ml->acl_delete.field_value[10].mask_range.u16 =
+					mh->match.acl.dp1;
+			}
+		return 0;
+
+	case TABLE_ARRAY:
+		ml->array.pos = mh->match.array.pos;
+		return 0;
+
+	case TABLE_HASH:
+		memcpy(ml->hash, mh->match.hash.key, sizeof(ml->hash));
+		return 0;
+
+	case TABLE_LPM:
+		if (mh->match.lpm.ip_version) {
+			ml->lpm_ipv4.ip = mh->match.lpm.ipv4;
+			ml->lpm_ipv4.depth = mh->match.lpm.depth;
+		} else {
+			memcpy(ml->lpm_ipv6.ip,
+				mh->match.lpm.ipv6, sizeof(ml->lpm_ipv6.ip));
+			ml->lpm_ipv6.depth = mh->match.lpm.depth;
+		}
+
+		return 0;
+
+	default:
+		return -1;
+	}
+}
+
+static int
+action_convert(struct rte_table_action *a,
+	struct softnic_table_rule_action *action,
+	struct rte_pipeline_table_entry *data)
+{
+	int status;
+
+	/* Apply actions */
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_FWD,
+			&action->fwd);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_LB,
+			&action->lb);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_MTR,
+			&action->mtr);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_TM,
+			&action->tm);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_ENCAP,
+			&action->encap);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_NAT,
+			&action->nat);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_TTL,
+			&action->ttl);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_STATS,
+			&action->stats);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_TIME,
+			&action->time);
+
+		if (status)
+			return status;
+	}
+
+	return 0;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_rule_add(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	union table_rule_match_low_level match_ll;
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	struct softnic_table_rule_match *match = &req->table_rule_add.match;
+	struct softnic_table_rule_action *action = &req->table_rule_add.action;
+	struct rte_pipeline_table_entry *data_in, *data_out;
+	uint32_t table_id = req->id;
+	int key_found, status;
+	struct rte_table_action *a = p->table_data[table_id].a;
+
+	/* Apply actions */
+	memset(p->buffer, 0, sizeof(p->buffer));
+	data_in = (struct rte_pipeline_table_entry *)p->buffer;
+
+	status = match_convert(match, &match_ll, 1);
+	if (status) {
+		rsp->status = -1;
+		return rsp;
+	}
+
+	status = action_convert(a, action, data_in);
+	if (status) {
+		rsp->status = -1;
+		return rsp;
+	}
+
+	status = rte_pipeline_table_entry_add(p->p,
+		table_id,
+		&match_ll,
+		data_in,
+		&key_found,
+		&data_out);
+	if (status) {
+		rsp->status = -1;
+		return rsp;
+	}
+
+	/* Write response */
+	rsp->status = 0;
+	rsp->table_rule_add.data = data_out;
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_rule_add_default(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	struct softnic_table_rule_action *action = &req->table_rule_add_default.action;
+	struct rte_pipeline_table_entry *data_in, *data_out;
+	uint32_t table_id = req->id;
+	int status;
+
+	/* Apply actions */
+	memset(p->buffer, 0, sizeof(p->buffer));
+	data_in = (struct rte_pipeline_table_entry *)p->buffer;
+
+	data_in->action = action->fwd.action;
+	if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
+		data_in->port_id = action->fwd.id;
+	if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
+		data_in->table_id = action->fwd.id;
+
+	/* Add default rule to table */
+	status = rte_pipeline_table_default_entry_add(p->p,
+		table_id,
+		data_in,
+		&data_out);
+	if (status) {
+		rsp->status = -1;
+		return rsp;
+	}
+
+	/* Write response */
+	rsp->status = 0;
+	rsp->table_rule_add_default.data = data_out;
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_rule_add_bulk(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+
+	uint32_t table_id = req->id;
+	struct softnic_table_rule_match *match = req->table_rule_add_bulk.match;
+	struct softnic_table_rule_action *action = req->table_rule_add_bulk.action;
+	struct rte_pipeline_table_entry **data =
+		(struct rte_pipeline_table_entry **)req->table_rule_add_bulk.data;
+	uint32_t n_rules = req->table_rule_add_bulk.n_rules;
+	uint32_t bulk = req->table_rule_add_bulk.bulk;
+
+	struct rte_table_action *a = p->table_data[table_id].a;
+	union table_rule_match_low_level *match_ll;
+	uint8_t *action_ll;
+	void **match_ll_ptr;
+	struct rte_pipeline_table_entry **action_ll_ptr;
+	int *found, status;
+	uint32_t i;
+
+	/* Memory allocation */
+	match_ll = calloc(n_rules, sizeof(union table_rule_match_low_level));
+	action_ll = calloc(n_rules, TABLE_RULE_ACTION_SIZE_MAX);
+	match_ll_ptr = calloc(n_rules, sizeof(void *));
+	action_ll_ptr =
+		calloc(n_rules, sizeof(struct rte_pipeline_table_entry *));
+	found = calloc(n_rules, sizeof(int));
+
+	if (match_ll == NULL ||
+		action_ll == NULL ||
+		match_ll_ptr == NULL ||
+		action_ll_ptr == NULL ||
+		found == NULL)
+		goto fail;
+
+	for (i = 0; i < n_rules; i++) {
+		match_ll_ptr[i] = (void *)&match_ll[i];
+		action_ll_ptr[i] =
+			(struct rte_pipeline_table_entry *)&action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
+	}
+
+	/* Rule match conversion */
+	for (i = 0; i < n_rules; i++) {
+		status = match_convert(&match[i], match_ll_ptr[i], 1);
+		if (status)
+			goto fail;
+	}
+
+	/* Rule action conversion */
+	for (i = 0; i < n_rules; i++) {
+		status = action_convert(a, &action[i], action_ll_ptr[i]);
+		if (status)
+			goto fail;
+	}
+
+	/* Add rule (match, action) to table */
+	if (bulk) {
+		status = rte_pipeline_table_entry_add_bulk(p->p,
+			table_id,
+			match_ll_ptr,
+			action_ll_ptr,
+			n_rules,
+			found,
+			data);
+		if (status)
+			n_rules = 0;
+	} else {
+		for (i = 0; i < n_rules; i++) {
+			status = rte_pipeline_table_entry_add(p->p,
+				table_id,
+				match_ll_ptr[i],
+				action_ll_ptr[i],
+				&found[i],
+				&data[i]);
+			if (status) {
+				n_rules = i;
+				break;
+			}
+		}
+	}
+
+	/* Write response */
+	rsp->status = 0;
+	rsp->table_rule_add_bulk.n_rules = n_rules;
+
+	/* Free */
+	free(found);
+	free(action_ll_ptr);
+	free(match_ll_ptr);
+	free(action_ll);
+	free(match_ll);
+
+	return rsp;
+
+fail:
+	free(found);
+	free(action_ll_ptr);
+	free(match_ll_ptr);
+	free(action_ll);
+	free(match_ll);
+
+	rsp->status = -1;
+	rsp->table_rule_add_bulk.n_rules = 0;
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_rule_delete(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	union table_rule_match_low_level match_ll;
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	struct softnic_table_rule_match *match = &req->table_rule_delete.match;
+	uint32_t table_id = req->id;
+	int key_found, status;
+
+	status = match_convert(match, &match_ll, 0);
+	if (status) {
+		rsp->status = -1;
+		return rsp;
+	}
+
+	rsp->status = rte_pipeline_table_entry_delete(p->p,
+		table_id,
+		&match_ll,
+		&key_found,
+		NULL);
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_rule_delete_default(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t table_id = req->id;
+
+	rsp->status = rte_pipeline_table_default_entry_delete(p->p,
+		table_id,
+		NULL);
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_rule_stats_read(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t table_id = req->id;
+	void *data = req->table_rule_stats_read.data;
+	int clear = req->table_rule_stats_read.clear;
+	struct rte_table_action *a = p->table_data[table_id].a;
+
+	rsp->status = rte_table_action_stats_read(a,
+		data,
+		&rsp->table_rule_stats_read.stats,
+		clear);
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_mtr_profile_add(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t table_id = req->id;
+	uint32_t meter_profile_id = req->table_mtr_profile_add.meter_profile_id;
+	struct rte_table_action_meter_profile *profile =
+		&req->table_mtr_profile_add.profile;
+	struct rte_table_action *a = p->table_data[table_id].a;
+
+	rsp->status = rte_table_action_meter_profile_add(a,
+		meter_profile_id,
+		profile);
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_mtr_profile_delete(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t table_id = req->id;
+	uint32_t meter_profile_id =
+		req->table_mtr_profile_delete.meter_profile_id;
+	struct rte_table_action *a = p->table_data[table_id].a;
+
+	rsp->status = rte_table_action_meter_profile_delete(a,
+		meter_profile_id);
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_rule_mtr_read(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t table_id = req->id;
+	void *data = req->table_rule_mtr_read.data;
+	uint32_t tc_mask = req->table_rule_mtr_read.tc_mask;
+	int clear = req->table_rule_mtr_read.clear;
+	struct rte_table_action *a = p->table_data[table_id].a;
+
+	rsp->status = rte_table_action_meter_read(a,
+		data,
+		tc_mask,
+		&rsp->table_rule_mtr_read.stats,
+		clear);
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_dscp_table_update(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t table_id = req->id;
+	uint64_t dscp_mask = req->table_dscp_table_update.dscp_mask;
+	struct rte_table_action_dscp_table *dscp_table =
+		&req->table_dscp_table_update.dscp_table;
+	struct rte_table_action *a = p->table_data[table_id].a;
+
+	rsp->status = rte_table_action_dscp_table_update(a,
+		dscp_mask,
+		dscp_table);
+
+	return rsp;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_rule_ttl_read(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *)req;
+	uint32_t table_id = req->id;
+	void *data = req->table_rule_ttl_read.data;
+	int clear = req->table_rule_ttl_read.clear;
+	struct rte_table_action *a = p->table_data[table_id].a;
+
+	rsp->status = rte_table_action_ttl_read(a,
+		data,
+		&rsp->table_rule_ttl_read.stats,
+		clear);
+
+	return rsp;
+}
+
+static void
+pipeline_msg_handle(struct pipeline_data *p)
+{
+	for ( ; ; ) {
+		struct pipeline_msg_req *req;
+		struct pipeline_msg_rsp *rsp;
+
+		req = pipeline_msg_recv(p->msgq_req);
+		if (req == NULL)
+			break;
+
+		switch (req->type) {
+		case PIPELINE_REQ_PORT_IN_STATS_READ:
+			rsp = pipeline_msg_handle_port_in_stats_read(p, req);
+			break;
+
+		case PIPELINE_REQ_PORT_IN_ENABLE:
+			rsp = pipeline_msg_handle_port_in_enable(p, req);
+			break;
+
+		case PIPELINE_REQ_PORT_IN_DISABLE:
+			rsp = pipeline_msg_handle_port_in_disable(p, req);
+			break;
+
+		case PIPELINE_REQ_PORT_OUT_STATS_READ:
+			rsp = pipeline_msg_handle_port_out_stats_read(p, req);
+			break;
+
+		case PIPELINE_REQ_TABLE_STATS_READ:
+			rsp = pipeline_msg_handle_table_stats_read(p, req);
+			break;
+
+		case PIPELINE_REQ_TABLE_RULE_ADD:
+			rsp = pipeline_msg_handle_table_rule_add(p, req);
+			break;
+
+		case PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT:
+			rsp = pipeline_msg_handle_table_rule_add_default(p,	req);
+			break;
+
+		case PIPELINE_REQ_TABLE_RULE_ADD_BULK:
+			rsp = pipeline_msg_handle_table_rule_add_bulk(p, req);
+			break;
+
+		case PIPELINE_REQ_TABLE_RULE_DELETE:
+			rsp = pipeline_msg_handle_table_rule_delete(p, req);
+			break;
+
+		case PIPELINE_REQ_TABLE_RULE_DELETE_DEFAULT:
+			rsp = pipeline_msg_handle_table_rule_delete_default(p, req);
+			break;
+
+		case PIPELINE_REQ_TABLE_RULE_STATS_READ:
+			rsp = pipeline_msg_handle_table_rule_stats_read(p, req);
+			break;
+
+		case PIPELINE_REQ_TABLE_MTR_PROFILE_ADD:
+			rsp = pipeline_msg_handle_table_mtr_profile_add(p, req);
+			break;
+
+		case PIPELINE_REQ_TABLE_MTR_PROFILE_DELETE:
+			rsp = pipeline_msg_handle_table_mtr_profile_delete(p, req);
+			break;
+
+		case PIPELINE_REQ_TABLE_RULE_MTR_READ:
+			rsp = pipeline_msg_handle_table_rule_mtr_read(p, req);
+			break;
+
+		case PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE:
+			rsp = pipeline_msg_handle_table_dscp_table_update(p, req);
+			break;
+
+		case PIPELINE_REQ_TABLE_RULE_TTL_READ:
+			rsp = pipeline_msg_handle_table_rule_ttl_read(p, req);
+			break;
+
+		default:
+			rsp = (struct pipeline_msg_rsp *)req;
+			rsp->status = -1;
+		}
+
+		pipeline_msg_send(p->msgq_rsp, rsp);
+	}
+}
+
+/**
+ * Data plane threads: main
+ */
+int
+rte_pmd_softnic_run(uint16_t port_id)
+{
+	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
+	struct pmd_internals *softnic;
+	struct softnic_thread_data *t;
+	uint32_t thread_id, j;
+
+#ifdef RTE_LIBRTE_ETHDEV_DEBUG
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
+#endif
+
+	softnic = dev->data->dev_private;
+	thread_id = rte_lcore_id();
+	t = &softnic->thread_data[thread_id];
+	t->iter++;
+
+	/* Data Plane */
+	for (j = 0; j < t->n_pipelines; j++)
+		rte_pipeline_run(t->p[j]);
+
+	/* Control Plane */
+	if ((t->iter & 0xFLLU) == 0) {
+		uint64_t time = rte_get_tsc_cycles();
+		uint64_t time_next_min = UINT64_MAX;
+
+		if (time < t->time_next_min)
+			return 0;
+
+		/* Pipeline message queues */
+		for (j = 0; j < t->n_pipelines; j++) {
+			struct pipeline_data *p =
+				&t->pipeline_data[j];
+			uint64_t time_next = p->time_next;
+
+			if (time_next <= time) {
+				pipeline_msg_handle(p);
+				rte_pipeline_flush(p->p);
+				time_next = time + p->timer_period;
+				p->time_next = time_next;
+			}
+
+			if (time_next < time_next_min)
+				time_next_min = time_next;
+		}
+
+		/* Thread message queues */
+		{
+			uint64_t time_next = t->time_next;
+
+			if (time_next <= time) {
+				thread_msg_handle(t);
+				time_next = time + t->timer_period;
+				t->time_next = time_next;
+			}
+
+			if (time_next < time_next_min)
+				time_next_min = time_next;
+		}
+
+		t->time_next_min = time_next_min;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/softnic/rte_eth_softnic_tm.c b/drivers/net/softnic/rte_eth_softnic_tm.c
index 11d638a9..baaafbe2 100644
--- a/drivers/net/softnic/rte_eth_softnic_tm.c
+++ b/drivers/net/softnic/rte_eth_softnic_tm.c
@@ -7,62 +7,148 @@
 #include <string.h>
 
 #include <rte_malloc.h>
+#include <rte_string_fns.h>
 
 #include "rte_eth_softnic_internals.h"
 #include "rte_eth_softnic.h"
 
-#define BYTES_IN_MBPS		(1000 * 1000 / 8)
 #define SUBPORT_TC_PERIOD	10
 #define PIPE_TC_PERIOD		40
 
 int
-tm_params_check(struct pmd_params *params, uint32_t hard_rate)
+softnic_tmgr_init(struct pmd_internals *p)
 {
-	uint64_t hard_rate_bytes_per_sec = (uint64_t)hard_rate * BYTES_IN_MBPS;
-	uint32_t i;
+	TAILQ_INIT(&p->tmgr_port_list);
 
-	/* rate */
-	if (params->soft.tm.rate) {
-		if (params->soft.tm.rate > hard_rate_bytes_per_sec)
-			return -EINVAL;
-	} else {
-		params->soft.tm.rate =
-			(hard_rate_bytes_per_sec > UINT32_MAX) ?
-				UINT32_MAX : hard_rate_bytes_per_sec;
+	return 0;
+}
+
+void
+softnic_tmgr_free(struct pmd_internals *p)
+{
+	for ( ; ; ) {
+		struct softnic_tmgr_port *tmgr_port;
+
+		tmgr_port = TAILQ_FIRST(&p->tmgr_port_list);
+		if (tmgr_port == NULL)
+			break;
+
+		TAILQ_REMOVE(&p->tmgr_port_list, tmgr_port, node);
+		rte_sched_port_free(tmgr_port->s);
+		free(tmgr_port);
 	}
+}
+
+struct softnic_tmgr_port *
+softnic_tmgr_port_find(struct pmd_internals *p,
+	const char *name)
+{
+	struct softnic_tmgr_port *tmgr_port;
 
-	/* nb_queues */
-	if (params->soft.tm.nb_queues == 0)
-		return -EINVAL;
+	if (name == NULL)
+		return NULL;
 
-	if (params->soft.tm.nb_queues < RTE_SCHED_QUEUES_PER_PIPE)
-		params->soft.tm.nb_queues = RTE_SCHED_QUEUES_PER_PIPE;
+	TAILQ_FOREACH(tmgr_port, &p->tmgr_port_list, node)
+		if (strcmp(tmgr_port->name, name) == 0)
+			return tmgr_port;
 
-	params->soft.tm.nb_queues =
-		rte_align32pow2(params->soft.tm.nb_queues);
+	return NULL;
+}
 
-	/* qsize */
-	for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
-		if (params->soft.tm.qsize[i] == 0)
-			return -EINVAL;
+struct softnic_tmgr_port *
+softnic_tmgr_port_create(struct pmd_internals *p,
+	const char *name)
+{
+	struct softnic_tmgr_port *tmgr_port;
+	struct tm_params *t = &p->soft.tm.params;
+	struct rte_sched_port *sched;
+	uint32_t n_subports, subport_id;
+
+	/* Check input params */
+	if (name == NULL ||
+		softnic_tmgr_port_find(p, name))
+		return NULL;
+
+	/*
+	 * Resource
+	 */
+
+	/* Is hierarchy frozen? */
+	if (p->soft.tm.hierarchy_frozen == 0)
+		return NULL;
+
+	/* Port */
+	sched = rte_sched_port_config(&t->port_params);
+	if (sched == NULL)
+		return NULL;
+
+	/* Subport */
+	n_subports = t->port_params.n_subports_per_port;
+	for (subport_id = 0; subport_id < n_subports; subport_id++) {
+		uint32_t n_pipes_per_subport = t->port_params.n_pipes_per_subport;
+		uint32_t pipe_id;
+		int status;
 
-		params->soft.tm.qsize[i] =
-			rte_align32pow2(params->soft.tm.qsize[i]);
+		status = rte_sched_subport_config(sched,
+			subport_id,
+			&t->subport_params[subport_id]);
+		if (status) {
+			rte_sched_port_free(sched);
+			return NULL;
+		}
+
+		/* Pipe */
+		for (pipe_id = 0; pipe_id < n_pipes_per_subport; pipe_id++) {
+			int pos = subport_id * TM_MAX_PIPES_PER_SUBPORT + pipe_id;
+			int profile_id = t->pipe_to_profile[pos];
+
+			if (profile_id < 0)
+				continue;
+
+			status = rte_sched_pipe_config(sched,
+				subport_id,
+				pipe_id,
+				profile_id);
+			if (status) {
+				rte_sched_port_free(sched);
+				return NULL;
+			}
+		}
 	}
 
-	/* enq_bsz, deq_bsz */
-	if (params->soft.tm.enq_bsz == 0 ||
-		params->soft.tm.deq_bsz == 0 ||
-		params->soft.tm.deq_bsz >= params->soft.tm.enq_bsz)
-		return -EINVAL;
+	/* Node allocation */
+	tmgr_port = calloc(1, sizeof(struct softnic_tmgr_port));
+	if (tmgr_port == NULL) {
+		rte_sched_port_free(sched);
+		return NULL;
+	}
 
-	return 0;
+	/* Node fill in */
+	strlcpy(tmgr_port->name, name, sizeof(tmgr_port->name));
+	tmgr_port->s = sched;
+
+	/* Node add to list */
+	TAILQ_INSERT_TAIL(&p->tmgr_port_list, tmgr_port, node);
+
+	return tmgr_port;
 }
 
-static void
+static struct rte_sched_port *
+SCHED(struct pmd_internals *p)
+{
+	struct softnic_tmgr_port *tmgr_port;
+
+	tmgr_port = softnic_tmgr_port_find(p, "TMGR");
+	if (tmgr_port == NULL)
+		return NULL;
+
+	return tmgr_port->s;
+}
+
+void
 tm_hierarchy_init(struct pmd_internals *p)
 {
-	memset(&p->soft.tm.h, 0, sizeof(p->soft.tm.h));
+	memset(&p->soft.tm, 0, sizeof(p->soft.tm));
 
 	/* Initialize shaper profile list */
 	TAILQ_INIT(&p->soft.tm.h.shaper_profiles);
@@ -77,8 +163,8 @@ tm_hierarchy_init(struct pmd_internals *p)
 	TAILQ_INIT(&p->soft.tm.h.nodes);
 }
 
-static void
-tm_hierarchy_uninit(struct pmd_internals *p)
+void
+tm_hierarchy_free(struct pmd_internals *p)
 {
 	/* Remove all nodes*/
 	for ( ; ; ) {
@@ -129,111 +215,7 @@ tm_hierarchy_uninit(struct pmd_internals *p)
 		free(shaper_profile);
 	}
 
-	memset(&p->soft.tm.h, 0, sizeof(p->soft.tm.h));
-}
-
-int
-tm_init(struct pmd_internals *p,
-	struct pmd_params *params,
-	int numa_node)
-{
-	uint32_t enq_bsz = params->soft.tm.enq_bsz;
-	uint32_t deq_bsz = params->soft.tm.deq_bsz;
-
-	p->soft.tm.pkts_enq = rte_zmalloc_socket(params->soft.name,
-		2 * enq_bsz * sizeof(struct rte_mbuf *),
-		0,
-		numa_node);
-
-	if (p->soft.tm.pkts_enq == NULL)
-		return -ENOMEM;
-
-	p->soft.tm.pkts_deq = rte_zmalloc_socket(params->soft.name,
-		deq_bsz * sizeof(struct rte_mbuf *),
-		0,
-		numa_node);
-
-	if (p->soft.tm.pkts_deq == NULL) {
-		rte_free(p->soft.tm.pkts_enq);
-		return -ENOMEM;
-	}
-
 	tm_hierarchy_init(p);
-
-	return 0;
-}
-
-void
-tm_free(struct pmd_internals *p)
-{
-	tm_hierarchy_uninit(p);
-	rte_free(p->soft.tm.pkts_enq);
-	rte_free(p->soft.tm.pkts_deq);
-}
-
-int
-tm_start(struct pmd_internals *p)
-{
-	struct tm_params *t = &p->soft.tm.params;
-	uint32_t n_subports, subport_id;
-	int status;
-
-	/* Is hierarchy frozen? */
-	if (p->soft.tm.hierarchy_frozen == 0)
-		return -1;
-
-	/* Port */
-	p->soft.tm.sched = rte_sched_port_config(&t->port_params);
-	if (p->soft.tm.sched == NULL)
-		return -1;
-
-	/* Subport */
-	n_subports = t->port_params.n_subports_per_port;
-	for (subport_id = 0; subport_id < n_subports; subport_id++) {
-		uint32_t n_pipes_per_subport =
-			t->port_params.n_pipes_per_subport;
-		uint32_t pipe_id;
-
-		status = rte_sched_subport_config(p->soft.tm.sched,
-			subport_id,
-			&t->subport_params[subport_id]);
-		if (status) {
-			rte_sched_port_free(p->soft.tm.sched);
-			return -1;
-		}
-
-		/* Pipe */
-		n_pipes_per_subport = t->port_params.n_pipes_per_subport;
-		for (pipe_id = 0; pipe_id < n_pipes_per_subport; pipe_id++) {
-			int pos = subport_id * TM_MAX_PIPES_PER_SUBPORT +
-				pipe_id;
-			int profile_id = t->pipe_to_profile[pos];
-
-			if (profile_id < 0)
-				continue;
-
-			status = rte_sched_pipe_config(p->soft.tm.sched,
-				subport_id,
-				pipe_id,
-				profile_id);
-			if (status) {
-				rte_sched_port_free(p->soft.tm.sched);
-				return -1;
-			}
-		}
-	}
-
-	return 0;
-}
-
-void
-tm_stop(struct pmd_internals *p)
-{
-	if (p->soft.tm.sched)
-		rte_sched_port_free(p->soft.tm.sched);
-
-	/* Unfreeze hierarchy */
-	p->soft.tm.hierarchy_frozen = 0;
 }
 
 static struct tm_shaper_profile *
@@ -384,7 +366,7 @@ static uint32_t
 tm_level_get_max_nodes(struct rte_eth_dev *dev, enum tm_node_level level)
 {
 	struct pmd_internals *p = dev->data->dev_private;
-	uint32_t n_queues_max = p->params.soft.tm.nb_queues;
+	uint32_t n_queues_max = p->params.tm.n_queues;
 	uint32_t n_tc_max = n_queues_max / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS;
 	uint32_t n_pipes_max = n_tc_max / RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE;
 	uint32_t n_subports_max = n_pipes_max;
@@ -429,7 +411,7 @@ pmd_tm_node_type_get(struct rte_eth_dev *dev,
 		   NULL,
 		   rte_strerror(EINVAL));
 
-	*is_leaf = node_id < p->params.soft.tm.nb_queues;
+	*is_leaf = node_id < p->params.tm.n_queues;
 
 	return 0;
 }
@@ -1083,7 +1065,7 @@ update_subport_tc_rate(struct rte_eth_dev *dev,
 	subport_params.tc_rate[tc_id] = sp_new->params.peak.rate;
 
 	/* Update the subport configuration. */
-	if (rte_sched_subport_config(p->soft.tm.sched,
+	if (rte_sched_subport_config(SCHED(p),
 		subport_id, &subport_params))
 		return -1;
 
@@ -1362,7 +1344,7 @@ node_add_check_port(struct rte_eth_dev *dev,
 		params->shaper_profile_id);
 
 	/* node type: non-leaf */
-	if (node_id < p->params.soft.tm.nb_queues)
+	if (node_id < p->params.tm.n_queues)
 		return -rte_tm_error_set(error,
 			EINVAL,
 			RTE_TM_ERROR_TYPE_NODE_ID,
@@ -1385,12 +1367,9 @@ node_add_check_port(struct rte_eth_dev *dev,
 			NULL,
 			rte_strerror(EINVAL));
 
-	/* Shaper must be valid.
-	 * Shaper profile peak rate must fit the configured port rate.
-	 */
+	/* Shaper must be valid */
 	if (params->shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE ||
-		sp == NULL ||
-		sp->params.peak.rate > p->params.soft.tm.rate)
+		sp == NULL)
 		return -rte_tm_error_set(error,
 			EINVAL,
 			RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID,
@@ -1437,7 +1416,7 @@ node_add_check_subport(struct rte_eth_dev *dev,
 	struct pmd_internals *p = dev->data->dev_private;
 
 	/* node type: non-leaf */
-	if (node_id < p->params.soft.tm.nb_queues)
+	if (node_id < p->params.tm.n_queues)
 		return -rte_tm_error_set(error,
 			EINVAL,
 			RTE_TM_ERROR_TYPE_NODE_ID,
@@ -1509,7 +1488,7 @@ node_add_check_pipe(struct rte_eth_dev *dev,
 	struct pmd_internals *p = dev->data->dev_private;
 
 	/* node type: non-leaf */
-	if (node_id < p->params.soft.tm.nb_queues)
+	if (node_id < p->params.tm.n_queues)
 		return -rte_tm_error_set(error,
 			EINVAL,
 			RTE_TM_ERROR_TYPE_NODE_ID,
@@ -1586,7 +1565,7 @@ node_add_check_tc(struct rte_eth_dev *dev,
 	struct pmd_internals *p = dev->data->dev_private;
 
 	/* node type: non-leaf */
-	if (node_id < p->params.soft.tm.nb_queues)
+	if (node_id < p->params.tm.n_queues)
 		return -rte_tm_error_set(error,
 			EINVAL,
 			RTE_TM_ERROR_TYPE_NODE_ID,
@@ -1659,7 +1638,7 @@ node_add_check_queue(struct rte_eth_dev *dev,
 	struct pmd_internals *p = dev->data->dev_private;
 
 	/* node type: leaf */
-	if (node_id >= p->params.soft.tm.nb_queues)
+	if (node_id >= p->params.tm.n_queues)
 		return -rte_tm_error_set(error,
 			EINVAL,
 			RTE_TM_ERROR_TYPE_NODE_ID,
@@ -2548,10 +2527,10 @@ hierarchy_blueprints_create(struct rte_eth_dev *dev)
 		.n_subports_per_port = root->n_children,
 		.n_pipes_per_subport = h->n_tm_nodes[TM_NODE_LEVEL_PIPE] /
 			h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT],
-		.qsize = {p->params.soft.tm.qsize[0],
-			p->params.soft.tm.qsize[1],
-			p->params.soft.tm.qsize[2],
-			p->params.soft.tm.qsize[3],
+		.qsize = {p->params.tm.qsize[0],
+			p->params.tm.qsize[1],
+			p->params.tm.qsize[2],
+			p->params.tm.qsize[3],
 		},
 		.pipe_profiles = t->pipe_profiles,
 		.n_pipe_profiles = t->n_pipe_profiles,
@@ -2614,10 +2593,8 @@ pmd_tm_hierarchy_commit(struct rte_eth_dev *dev,
 
 	status = hierarchy_commit_check(dev, error);
 	if (status) {
-		if (clear_on_fail) {
-			tm_hierarchy_uninit(p);
-			tm_hierarchy_init(p);
-		}
+		if (clear_on_fail)
+			tm_hierarchy_free(p);
 
 		return status;
 	}
@@ -2659,7 +2636,7 @@ update_pipe_weight(struct rte_eth_dev *dev, struct tm_node *np, uint32_t weight)
 		return -1;
 
 	/* Update the pipe profile used by the current pipe. */
-	if (rte_sched_pipe_config(p->soft.tm.sched, subport_id, pipe_id,
+	if (rte_sched_pipe_config(SCHED(p), subport_id, pipe_id,
 		(int32_t)pipe_profile_id))
 		return -1;
 
@@ -2708,7 +2685,7 @@ update_queue_weight(struct rte_eth_dev *dev,
 		return -1;
 
 	/* Update the pipe profile used by the current pipe. */
-	if (rte_sched_pipe_config(p->soft.tm.sched, subport_id, pipe_id,
+	if (rte_sched_pipe_config(SCHED(p), subport_id, pipe_id,
 		(int32_t)pipe_profile_id))
 		return -1;
 
@@ -2841,7 +2818,7 @@ update_subport_rate(struct rte_eth_dev *dev,
 	subport_params.tb_size = sp->params.peak.size;
 
 	/* Update the subport configuration. */
-	if (rte_sched_subport_config(p->soft.tm.sched, subport_id,
+	if (rte_sched_subport_config(SCHED(p), subport_id,
 		&subport_params))
 		return -1;
 
@@ -2888,7 +2865,7 @@ update_pipe_rate(struct rte_eth_dev *dev,
 		return -1;
 
 	/* Update the pipe profile used by the current pipe. */
-	if (rte_sched_pipe_config(p->soft.tm.sched, subport_id, pipe_id,
+	if (rte_sched_pipe_config(SCHED(p), subport_id, pipe_id,
 		(int32_t)pipe_profile_id))
 		return -1;
 
@@ -2933,7 +2910,7 @@ update_tc_rate(struct rte_eth_dev *dev,
 		return -1;
 
 	/* Update the pipe profile used by the current pipe. */
-	if (rte_sched_pipe_config(p->soft.tm.sched, subport_id, pipe_id,
+	if (rte_sched_pipe_config(SCHED(p), subport_id, pipe_id,
 		(int32_t)pipe_profile_id))
 		return -1;
 
@@ -3068,8 +3045,7 @@ read_port_stats(struct rte_eth_dev *dev,
 		uint32_t tc_ov, id;
 
 		/* Stats read */
-		int status = rte_sched_subport_read_stats(
-			p->soft.tm.sched,
+		int status = rte_sched_subport_read_stats(SCHED(p),
 			subport_id,
 			&s,
 			&tc_ov);
@@ -3116,8 +3092,7 @@ read_subport_stats(struct rte_eth_dev *dev,
 	uint32_t tc_ov, tc_id;
 
 	/* Stats read */
-	int status = rte_sched_subport_read_stats(
-		p->soft.tm.sched,
+	int status = rte_sched_subport_read_stats(SCHED(p),
 		subport_id,
 		&s,
 		&tc_ov);
@@ -3177,8 +3152,7 @@ read_pipe_stats(struct rte_eth_dev *dev,
 			i / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS,
 			i % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS);
 
-		int status = rte_sched_queue_read_stats(
-			p->soft.tm.sched,
+		int status = rte_sched_queue_read_stats(SCHED(p),
 			qid,
 			&s,
 			&qlen);
@@ -3238,8 +3212,7 @@ read_tc_stats(struct rte_eth_dev *dev,
 			tc_id,
 			i);
 
-		int status = rte_sched_queue_read_stats(
-			p->soft.tm.sched,
+		int status = rte_sched_queue_read_stats(SCHED(p),
 			qid,
 			&s,
 			&qlen);
@@ -3298,8 +3271,7 @@ read_queue_stats(struct rte_eth_dev *dev,
 		tc_id,
 		queue_id);
 
-	int status = rte_sched_queue_read_stats(
-		p->soft.tm.sched,
+	int status = rte_sched_queue_read_stats(SCHED(p),
 		qid,
 		&s,
 		&qlen);
diff --git a/drivers/net/softnic/rte_pmd_eth_softnic_version.map b/drivers/net/softnic/rte_pmd_softnic_version.map
index fb2cb68c..bc44b06f 100644
--- a/drivers/net/softnic/rte_pmd_eth_softnic_version.map
+++ b/drivers/net/softnic/rte_pmd_softnic_version.map
@@ -5,3 +5,9 @@ DPDK_17.11 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_softnic_manage;
+};
diff --git a/drivers/net/szedata2/meson.build b/drivers/net/szedata2/meson.build
new file mode 100644
index 00000000..da373374
--- /dev/null
+++ b/drivers/net/szedata2/meson.build
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+dep = cc.find_library('sze2', required: false)
+build = dep.found()
+ext_deps += dep
+sources = files('rte_eth_szedata2.c')
diff --git a/drivers/net/szedata2/rte_eth_szedata2.c b/drivers/net/szedata2/rte_eth_szedata2.c
index 910c64d0..1d20cb51 100644
--- a/drivers/net/szedata2/rte_eth_szedata2.c
+++ b/drivers/net/szedata2/rte_eth_szedata2.c
@@ -1056,7 +1056,8 @@ eth_dev_info(struct rte_eth_dev *dev,
 	dev_info->max_rx_queues = internals->max_rx_queues;
 	dev_info->max_tx_queues = internals->max_tx_queues;
 	dev_info->min_rx_bufsize = 0;
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_SCATTER;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_SCATTER |
+				    DEV_RX_OFFLOAD_CRC_STRIP;
 	dev_info->tx_offload_capa = 0;
 	dev_info->rx_queue_offload_capa = 0;
 	dev_info->tx_queue_offload_capa = 0;
@@ -1922,9 +1923,7 @@ RTE_PMD_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table);
 RTE_PMD_REGISTER_KMOD_DEP(RTE_SZEDATA2_DRIVER_NAME,
 	"* combo6core & combov3 & szedata2 & ( szedata2_cv3 | szedata2_cv3_fdt )");
 
-RTE_INIT(szedata2_init_log);
-static void
-szedata2_init_log(void)
+RTE_INIT(szedata2_init_log)
 {
 	szedata2_logtype_init = rte_log_register("pmd.net.szedata2.init");
 	if (szedata2_logtype_init >= 0)
diff --git a/drivers/net/tap/Makefile b/drivers/net/tap/Makefile
index ccc5c5fc..32433653 100644
--- a/drivers/net/tap/Makefile
+++ b/drivers/net/tap/Makefile
@@ -24,7 +24,7 @@ CFLAGS += -I.
 CFLAGS += $(WERROR_FLAGS)
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash
-LDLIBS += -lrte_bus_vdev
+LDLIBS += -lrte_bus_vdev -lrte_gso
 
 CFLAGS += -DTAP_MAX_QUEUES=$(TAP_MAX_QUEUES)
 
diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index 5531fe9d..feb92b48 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -17,6 +17,7 @@
 #include <rte_ip.h>
 #include <rte_string_fns.h>
 
+#include <assert.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/socket.h>
@@ -55,6 +56,12 @@
 #define ETH_TAP_CMP_MAC_FMT     "0123456789ABCDEFabcdef"
 #define ETH_TAP_MAC_ARG_FMT     ETH_TAP_MAC_FIXED "|" ETH_TAP_USR_MAC_FMT
 
+#define TAP_GSO_MBUFS_PER_CORE	128
+#define TAP_GSO_MBUF_SEG_SIZE	128
+#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUFS_NUM \
+	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
+
 static struct rte_vdev_driver pmd_tap_drv;
 static struct rte_vdev_driver pmd_tun_drv;
 
@@ -65,7 +72,7 @@ static const char *valid_arguments[] = {
 	NULL
 };
 
-static int tap_unit;
+static unsigned int tap_unit;
 static unsigned int tun_unit;
 
 static char tuntap_name[8];
@@ -412,12 +419,45 @@ tap_tx_offload_get_queue_capa(void)
 	return DEV_TX_OFFLOAD_MULTI_SEGS |
 	       DEV_TX_OFFLOAD_IPV4_CKSUM |
 	       DEV_TX_OFFLOAD_UDP_CKSUM |
-	       DEV_TX_OFFLOAD_TCP_CKSUM;
+	       DEV_TX_OFFLOAD_TCP_CKSUM |
+	       DEV_TX_OFFLOAD_TCP_TSO;
 }
 
+/* Finalize l4 checksum calculation */
 static void
-tap_tx_offload(char *packet, uint64_t ol_flags, unsigned int l2_len,
-	       unsigned int l3_len)
+tap_tx_l4_cksum(uint16_t *l4_cksum, uint16_t l4_phdr_cksum,
+		uint32_t l4_raw_cksum)
+{
+	if (l4_cksum) {
+		uint32_t cksum;
+
+		cksum = __rte_raw_cksum_reduce(l4_raw_cksum);
+		cksum += l4_phdr_cksum;
+
+		cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff);
+		cksum = (~cksum) & 0xffff;
+		if (cksum == 0)
+			cksum = 0xffff;
+		*l4_cksum = cksum;
+	}
+}
+
+/* Accumaulate L4 raw checksums */
+static void
+tap_tx_l4_add_rcksum(char *l4_data, unsigned int l4_len, uint16_t *l4_cksum,
+			uint32_t *l4_raw_cksum)
+{
+	if (l4_cksum == NULL)
+		return;
+
+	*l4_raw_cksum = __rte_raw_cksum(l4_data, l4_len, *l4_raw_cksum);
+}
+
+/* L3 and L4 pseudo headers checksum offloads */
+static void
+tap_tx_l3_cksum(char *packet, uint64_t ol_flags, unsigned int l2_len,
+		unsigned int l3_len, unsigned int l4_len, uint16_t **l4_cksum,
+		uint16_t *l4_phdr_cksum, uint32_t *l4_raw_cksum)
 {
 	void *l3_hdr = packet + l2_len;
 
@@ -430,69 +470,50 @@ tap_tx_offload(char *packet, uint64_t ol_flags, unsigned int l2_len,
 		iph->hdr_checksum = (cksum == 0xffff) ? cksum : ~cksum;
 	}
 	if (ol_flags & PKT_TX_L4_MASK) {
-		uint16_t l4_len;
-		uint32_t cksum;
-		uint16_t *l4_cksum;
 		void *l4_hdr;
 
 		l4_hdr = packet + l2_len + l3_len;
 		if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM)
-			l4_cksum = &((struct udp_hdr *)l4_hdr)->dgram_cksum;
+			*l4_cksum = &((struct udp_hdr *)l4_hdr)->dgram_cksum;
 		else if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM)
-			l4_cksum = &((struct tcp_hdr *)l4_hdr)->cksum;
+			*l4_cksum = &((struct tcp_hdr *)l4_hdr)->cksum;
 		else
 			return;
-		*l4_cksum = 0;
-		if (ol_flags & PKT_TX_IPV4) {
-			struct ipv4_hdr *iph = l3_hdr;
-
-			l4_len = rte_be_to_cpu_16(iph->total_length) - l3_len;
-			cksum = rte_ipv4_phdr_cksum(l3_hdr, 0);
-		} else {
-			struct ipv6_hdr *ip6h = l3_hdr;
-
-			/* payload_len does not include ext headers */
-			l4_len = rte_be_to_cpu_16(ip6h->payload_len) -
-				l3_len + sizeof(struct ipv6_hdr);
-			cksum = rte_ipv6_phdr_cksum(l3_hdr, 0);
-		}
-		cksum += rte_raw_cksum(l4_hdr, l4_len);
-		cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff);
-		cksum = (~cksum) & 0xffff;
-		if (cksum == 0)
-			cksum = 0xffff;
-		*l4_cksum = cksum;
+		**l4_cksum = 0;
+		if (ol_flags & PKT_TX_IPV4)
+			*l4_phdr_cksum = rte_ipv4_phdr_cksum(l3_hdr, 0);
+		else
+			*l4_phdr_cksum = rte_ipv6_phdr_cksum(l3_hdr, 0);
+		*l4_raw_cksum = __rte_raw_cksum(l4_hdr, l4_len, 0);
 	}
 }
 
-/* Callback to handle sending packets from the tap interface
- */
-static uint16_t
-pmd_tx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
+static inline void
+tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
+			struct rte_mbuf **pmbufs,
+			uint16_t *num_packets, unsigned long *num_tx_bytes)
 {
-	struct tx_queue *txq = queue;
-	uint16_t num_tx = 0;
-	unsigned long num_tx_bytes = 0;
-	uint32_t max_size;
 	int i;
+	uint16_t l234_hlen;
 
-	if (unlikely(nb_pkts == 0))
-		return 0;
-
-	max_size = *txq->mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN + 4);
-	for (i = 0; i < nb_pkts; i++) {
-		struct rte_mbuf *mbuf = bufs[num_tx];
-		struct iovec iovecs[mbuf->nb_segs + 1];
+	for (i = 0; i < num_mbufs; i++) {
+		struct rte_mbuf *mbuf = pmbufs[i];
+		struct iovec iovecs[mbuf->nb_segs + 2];
 		struct tun_pi pi = { .flags = 0, .proto = 0x00 };
 		struct rte_mbuf *seg = mbuf;
 		char m_copy[mbuf->data_len];
+		int proto;
 		int n;
 		int j;
-
-		/* stats.errs will be incremented */
-		if (rte_pktmbuf_pkt_len(mbuf) > max_size)
-			break;
-
+		int k; /* current index in iovecs for copying segments */
+		uint16_t seg_len; /* length of first segment */
+		uint16_t nb_segs;
+		uint16_t *l4_cksum; /* l4 checksum (pseudo header + payload) */
+		uint32_t l4_raw_cksum = 0; /* TCP/UDP payload raw checksum */
+		uint16_t l4_phdr_cksum = 0; /* TCP/UDP pseudo header checksum */
+		uint16_t is_cksum = 0; /* in case cksum should be offloaded */
+
+		l4_cksum = NULL;
 		if (txq->type == ETH_TUNTAP_TYPE_TUN) {
 			/*
 			 * TUN and TAP are created with IFF_NO_PI disabled.
@@ -505,44 +526,163 @@ pmd_tx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 			 * is 4 or 6, then protocol field is updated.
 			 */
 			char *buff_data = rte_pktmbuf_mtod(seg, void *);
-			j = (*buff_data & 0xf0);
-			pi.proto = (j == 0x40) ? rte_cpu_to_be_16(ETHER_TYPE_IPv4) :
-				(j == 0x60) ? rte_cpu_to_be_16(ETHER_TYPE_IPv6) : 0x00;
+			proto = (*buff_data & 0xf0);
+			pi.proto = (proto == 0x40) ?
+				rte_cpu_to_be_16(ETHER_TYPE_IPv4) :
+				((proto == 0x60) ?
+					rte_cpu_to_be_16(ETHER_TYPE_IPv6) :
+					0x00);
 		}
 
-		iovecs[0].iov_base = &pi;
-		iovecs[0].iov_len = sizeof(pi);
-		for (j = 1; j <= mbuf->nb_segs; j++) {
-			iovecs[j].iov_len = rte_pktmbuf_data_len(seg);
-			iovecs[j].iov_base =
-				rte_pktmbuf_mtod(seg, void *);
-			seg = seg->next;
-		}
+		k = 0;
+		iovecs[k].iov_base = &pi;
+		iovecs[k].iov_len = sizeof(pi);
+		k++;
+
+		nb_segs = mbuf->nb_segs;
 		if (txq->csum &&
 		    ((mbuf->ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_IPV4) ||
 		     (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM ||
 		     (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM))) {
-			/* Support only packets with all data in the same seg */
-			if (mbuf->nb_segs > 1)
+			is_cksum = 1;
+
+			/* Support only packets with at least layer 4
+			 * header included in the first segment
+			 */
+			seg_len = rte_pktmbuf_data_len(mbuf);
+			l234_hlen = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len;
+			if (seg_len < l234_hlen)
 				break;
-			/* To change checksums, work on a copy of data. */
+
+			/* To change checksums, work on a * copy of l2, l3
+			 * headers + l4 pseudo header
+			 */
 			rte_memcpy(m_copy, rte_pktmbuf_mtod(mbuf, void *),
-				   rte_pktmbuf_data_len(mbuf));
-			tap_tx_offload(m_copy, mbuf->ol_flags,
-				       mbuf->l2_len, mbuf->l3_len);
-			iovecs[1].iov_base = m_copy;
+					l234_hlen);
+			tap_tx_l3_cksum(m_copy, mbuf->ol_flags,
+				       mbuf->l2_len, mbuf->l3_len, mbuf->l4_len,
+				       &l4_cksum, &l4_phdr_cksum,
+				       &l4_raw_cksum);
+			iovecs[k].iov_base = m_copy;
+			iovecs[k].iov_len = l234_hlen;
+			k++;
+
+			/* Update next iovecs[] beyond l2, l3, l4 headers */
+			if (seg_len > l234_hlen) {
+				iovecs[k].iov_len = seg_len - l234_hlen;
+				iovecs[k].iov_base =
+					rte_pktmbuf_mtod(seg, char *) +
+						l234_hlen;
+				tap_tx_l4_add_rcksum(iovecs[k].iov_base,
+					iovecs[k].iov_len, l4_cksum,
+					&l4_raw_cksum);
+				k++;
+				nb_segs++;
+			}
+			seg = seg->next;
+		}
+
+		for (j = k; j <= nb_segs; j++) {
+			iovecs[j].iov_len = rte_pktmbuf_data_len(seg);
+			iovecs[j].iov_base = rte_pktmbuf_mtod(seg, void *);
+			if (is_cksum)
+				tap_tx_l4_add_rcksum(iovecs[j].iov_base,
+					iovecs[j].iov_len, l4_cksum,
+					&l4_raw_cksum);
+			seg = seg->next;
 		}
+
+		if (is_cksum)
+			tap_tx_l4_cksum(l4_cksum, l4_phdr_cksum, l4_raw_cksum);
+
 		/* copy the tx frame data */
-		n = writev(txq->fd, iovecs, mbuf->nb_segs + 1);
+		n = writev(txq->fd, iovecs, j);
 		if (n <= 0)
 			break;
+		(*num_packets)++;
+		(*num_tx_bytes) += rte_pktmbuf_pkt_len(mbuf);
+	}
+}
 
+/* Callback to handle sending packets from the tap interface
+ */
+static uint16_t
+pmd_tx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
+{
+	struct tx_queue *txq = queue;
+	uint16_t num_tx = 0;
+	uint16_t num_packets = 0;
+	unsigned long num_tx_bytes = 0;
+	uint32_t max_size;
+	int i;
+
+	if (unlikely(nb_pkts == 0))
+		return 0;
+
+	struct rte_mbuf *gso_mbufs[MAX_GSO_MBUFS];
+	max_size = *txq->mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN + 4);
+	for (i = 0; i < nb_pkts; i++) {
+		struct rte_mbuf *mbuf_in = bufs[num_tx];
+		struct rte_mbuf **mbuf;
+		uint16_t num_mbufs = 0;
+		uint16_t tso_segsz = 0;
+		int ret;
+		uint16_t hdrs_len;
+		int j;
+		uint64_t tso;
+
+		tso = mbuf_in->ol_flags & PKT_TX_TCP_SEG;
+		if (tso) {
+			struct rte_gso_ctx *gso_ctx = &txq->gso_ctx;
+
+			assert(gso_ctx != NULL);
+
+			/* TCP segmentation implies TCP checksum offload */
+			mbuf_in->ol_flags |= PKT_TX_TCP_CKSUM;
+
+			/* gso size is calculated without ETHER_CRC_LEN */
+			hdrs_len = mbuf_in->l2_len + mbuf_in->l3_len +
+					mbuf_in->l4_len;
+			tso_segsz = mbuf_in->tso_segsz + hdrs_len;
+			if (unlikely(tso_segsz == hdrs_len) ||
+				tso_segsz > *txq->mtu) {
+				txq->stats.errs++;
+				break;
+			}
+			gso_ctx->gso_size = tso_segsz;
+			ret = rte_gso_segment(mbuf_in, /* packet to segment */
+				gso_ctx, /* gso control block */
+				(struct rte_mbuf **)&gso_mbufs, /* out mbufs */
+				RTE_DIM(gso_mbufs)); /* max tso mbufs */
+
+			/* ret contains the number of new created mbufs */
+			if (ret < 0)
+				break;
+
+			mbuf = gso_mbufs;
+			num_mbufs = ret;
+		} else {
+			/* stats.errs will be incremented */
+			if (rte_pktmbuf_pkt_len(mbuf_in) > max_size)
+				break;
+
+			/* ret 0 indicates no new mbufs were created */
+			ret = 0;
+			mbuf = &mbuf_in;
+			num_mbufs = 1;
+		}
+
+		tap_write_mbufs(txq, num_mbufs, mbuf,
+				&num_packets, &num_tx_bytes);
 		num_tx++;
-		num_tx_bytes += mbuf->pkt_len;
-		rte_pktmbuf_free(mbuf);
+		/* free original mbuf */
+		rte_pktmbuf_free(mbuf_in);
+		/* free tso mbufs */
+		for (j = 0; j < ret; j++)
+			rte_pktmbuf_free(mbuf[j]);
 	}
 
-	txq->stats.opackets += num_tx;
+	txq->stats.opackets += num_packets;
 	txq->stats.errs += nb_pkts - num_tx;
 	txq->stats.obytes += num_tx_bytes;
 
@@ -641,12 +781,22 @@ tap_link_set_up(struct rte_eth_dev *dev)
 static int
 tap_dev_start(struct rte_eth_dev *dev)
 {
-	int err;
+	int err, i;
 
 	err = tap_intr_handle_set(dev, 1);
 	if (err)
 		return err;
-	return tap_link_set_up(dev);
+
+	err = tap_link_set_up(dev);
+	if (err)
+		return err;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++)
+		dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
+	for (i = 0; i < dev->data->nb_rx_queues; i++)
+		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
+
+	return err;
 }
 
 /* This function gets called when the current port gets stopped.
@@ -654,6 +804,13 @@ tap_dev_start(struct rte_eth_dev *dev)
 static void
 tap_dev_stop(struct rte_eth_dev *dev)
 {
+	int i;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++)
+		dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
+	for (i = 0; i < dev->data->nb_rx_queues; i++)
+		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
+
 	tap_intr_handle_set(dev, 0);
 	tap_link_set_down(dev);
 }
@@ -1004,31 +1161,75 @@ tap_mac_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 }
 
 static int
+tap_gso_ctx_setup(struct rte_gso_ctx *gso_ctx, struct rte_eth_dev *dev)
+{
+	uint32_t gso_types;
+	char pool_name[64];
+
+	/*
+	 * Create private mbuf pool with TAP_GSO_MBUF_SEG_SIZE bytes
+	 * size per mbuf use this pool for both direct and indirect mbufs
+	 */
+
+	struct rte_mempool *mp;      /* Mempool for GSO packets */
+
+	/* initialize GSO context */
+	gso_types = DEV_TX_OFFLOAD_TCP_TSO;
+	snprintf(pool_name, sizeof(pool_name), "mp_%s", dev->device->name);
+	mp = rte_mempool_lookup((const char *)pool_name);
+	if (!mp) {
+		mp = rte_pktmbuf_pool_create(pool_name, TAP_GSO_MBUFS_NUM,
+			TAP_GSO_MBUF_CACHE_SIZE, 0,
+			RTE_PKTMBUF_HEADROOM + TAP_GSO_MBUF_SEG_SIZE,
+			SOCKET_ID_ANY);
+		if (!mp) {
+			struct pmd_internals *pmd = dev->data->dev_private;
+			RTE_LOG(DEBUG, PMD, "%s: failed to create mbuf pool for device %s\n",
+				pmd->name, dev->device->name);
+			return -1;
+		}
+	}
+
+	gso_ctx->direct_pool = mp;
+	gso_ctx->indirect_pool = mp;
+	gso_ctx->gso_types = gso_types;
+	gso_ctx->gso_size = 0; /* gso_size is set in tx_burst() per packet */
+	gso_ctx->flag = 0;
+
+	return 0;
+}
+
+static int
 tap_setup_queue(struct rte_eth_dev *dev,
 		struct pmd_internals *internals,
 		uint16_t qid,
 		int is_rx)
 {
+	int ret;
 	int *fd;
 	int *other_fd;
 	const char *dir;
 	struct pmd_internals *pmd = dev->data->dev_private;
 	struct rx_queue *rx = &internals->rxq[qid];
 	struct tx_queue *tx = &internals->txq[qid];
+	struct rte_gso_ctx *gso_ctx;
 
 	if (is_rx) {
 		fd = &rx->fd;
 		other_fd = &tx->fd;
 		dir = "rx";
+		gso_ctx = NULL;
 	} else {
 		fd = &tx->fd;
 		other_fd = &rx->fd;
 		dir = "tx";
+		gso_ctx = &tx->gso_ctx;
 	}
 	if (*fd != -1) {
 		/* fd for this queue already exists */
 		TAP_LOG(DEBUG, "%s: fd %d for %s queue qid %d exists",
 			pmd->name, *fd, dir, qid);
+		gso_ctx = NULL;
 	} else if (*other_fd != -1) {
 		/* Only other_fd exists. dup it */
 		*fd = dup(*other_fd);
@@ -1053,6 +1254,11 @@ tap_setup_queue(struct rte_eth_dev *dev,
 
 	tx->mtu = &dev->data->mtu;
 	rx->rxmode = &dev->data->dev_conf.rxmode;
+	if (gso_ctx) {
+		ret = tap_gso_ctx_setup(gso_ctx, dev);
+		if (ret)
+			return -1;
+	}
 
 	tx->type = pmd->type;
 
@@ -1342,6 +1548,37 @@ tap_rss_hash_update(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static int
+tap_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+
+	return 0;
+}
+
+static int
+tap_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+
+	return 0;
+}
+
+static int
+tap_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+
+	return 0;
+}
+
+static int
+tap_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
+{
+	dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+
+	return 0;
+}
 static const struct eth_dev_ops ops = {
 	.dev_start              = tap_dev_start,
 	.dev_stop               = tap_dev_stop,
@@ -1350,6 +1587,10 @@ static const struct eth_dev_ops ops = {
 	.dev_infos_get          = tap_dev_info,
 	.rx_queue_setup         = tap_rx_queue_setup,
 	.tx_queue_setup         = tap_tx_queue_setup,
+	.rx_queue_start         = tap_rx_queue_start,
+	.tx_queue_start         = tap_tx_queue_start,
+	.rx_queue_stop          = tap_rx_queue_stop,
+	.tx_queue_stop          = tap_tx_queue_stop,
 	.rx_queue_release       = tap_rx_queue_release,
 	.tx_queue_release       = tap_tx_queue_release,
 	.flow_ctrl_get          = tap_flow_ctrl_get,
@@ -1690,6 +1931,8 @@ rte_pmd_tun_probe(struct rte_vdev_device *dev)
 			return -1;
 		}
 		eth_dev->dev_ops = &ops;
+		eth_dev->device = &dev->device;
+		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
 
@@ -1759,12 +2002,13 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 		}
 		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &ops;
+		eth_dev->device = &dev->device;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
 
 	speed = ETH_SPEED_NUM_10G;
-	snprintf(tap_name, sizeof(tap_name), "%s%d",
+	snprintf(tap_name, sizeof(tap_name), "%s%u",
 		 DEFAULT_TAP_NAME, tap_unit++);
 	memset(remote_iface, 0, RTE_ETH_NAME_MAX_LEN);
 
@@ -1888,9 +2132,7 @@ RTE_PMD_REGISTER_PARAM_STRING(net_tap,
 			      ETH_TAP_REMOTE_ARG "=<string>");
 int tap_logtype;
 
-RTE_INIT(tap_init_log);
-static void
-tap_init_log(void)
+RTE_INIT(tap_init_log)
 {
 	tap_logtype = rte_log_register("pmd.net.tap");
 	if (tap_logtype >= 0)
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 7b21d0d8..44e2773f 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -15,6 +15,7 @@
 
 #include <rte_ethdev_driver.h>
 #include <rte_ether.h>
+#include <rte_gso.h>
 #include "tap_log.h"
 
 #ifdef IFF_MULTI_QUEUE
@@ -22,6 +23,7 @@
 #else
 #define RTE_PMD_TAP_MAX_QUEUES	1
 #endif
+#define MAX_GSO_MBUFS 64
 
 enum rte_tuntap_type {
 	ETH_TUNTAP_TYPE_UNKNOWN,
@@ -59,6 +61,7 @@ struct tx_queue {
 	uint16_t *mtu;                  /* Pointer to MTU from dev_data */
 	uint16_t csum:1;                /* Enable checksum offloading */
 	struct pkt_stats stats;         /* Stats for this TX queue */
+	struct rte_gso_ctx gso_ctx;     /* GSO context */
 };
 
 struct pmd_internals {
diff --git a/drivers/net/tap/tap_flow.c b/drivers/net/tap/tap_flow.c
index 6b60e6dc..0e01af62 100644
--- a/drivers/net/tap/tap_flow.c
+++ b/drivers/net/tap/tap_flow.c
@@ -537,7 +537,7 @@ tap_flow_create_eth(const struct rte_flow_item *item, void *data)
 	if (!flow)
 		return 0;
 	msg = &flow->msg;
-	if (!is_zero_ether_addr(&spec->dst)) {
+	if (!is_zero_ether_addr(&mask->dst)) {
 		tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST, ETHER_ADDR_LEN,
 			   &spec->dst.addr_bytes);
 		tap_nlattr_add(&msg->nh,
@@ -651,13 +651,13 @@ tap_flow_create_ipv4(const struct rte_flow_item *item, void *data)
 		info->eth_type = htons(ETH_P_IP);
 	if (!spec)
 		return 0;
-	if (spec->hdr.dst_addr) {
+	if (mask->hdr.dst_addr) {
 		tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST,
 			     spec->hdr.dst_addr);
 		tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST_MASK,
 			     mask->hdr.dst_addr);
 	}
-	if (spec->hdr.src_addr) {
+	if (mask->hdr.src_addr) {
 		tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC,
 			     spec->hdr.src_addr);
 		tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC_MASK,
@@ -707,13 +707,13 @@ tap_flow_create_ipv6(const struct rte_flow_item *item, void *data)
 		info->eth_type = htons(ETH_P_IPV6);
 	if (!spec)
 		return 0;
-	if (memcmp(spec->hdr.dst_addr, empty_addr, 16)) {
+	if (memcmp(mask->hdr.dst_addr, empty_addr, 16)) {
 		tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST,
 			   sizeof(spec->hdr.dst_addr), &spec->hdr.dst_addr);
 		tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST_MASK,
 			   sizeof(mask->hdr.dst_addr), &mask->hdr.dst_addr);
 	}
-	if (memcmp(spec->hdr.src_addr, empty_addr, 16)) {
+	if (memcmp(mask->hdr.src_addr, empty_addr, 16)) {
 		tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC,
 			   sizeof(spec->hdr.src_addr), &spec->hdr.src_addr);
 		tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC_MASK,
@@ -762,10 +762,10 @@ tap_flow_create_udp(const struct rte_flow_item *item, void *data)
 	tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_UDP);
 	if (!spec)
 		return 0;
-	if (spec->hdr.dst_port & mask->hdr.dst_port)
+	if (mask->hdr.dst_port)
 		tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_DST,
 			     spec->hdr.dst_port);
-	if (spec->hdr.src_port & mask->hdr.src_port)
+	if (mask->hdr.src_port)
 		tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_SRC,
 			     spec->hdr.src_port);
 	return 0;
@@ -808,10 +808,10 @@ tap_flow_create_tcp(const struct rte_flow_item *item, void *data)
 	tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_TCP);
 	if (!spec)
 		return 0;
-	if (spec->hdr.dst_port & mask->hdr.dst_port)
+	if (mask->hdr.dst_port)
 		tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_DST,
 			     spec->hdr.dst_port);
-	if (spec->hdr.src_port & mask->hdr.src_port)
+	if (mask->hdr.src_port)
 		tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_SRC,
 			     spec->hdr.src_port);
 	return 0;
diff --git a/drivers/net/thunderx/base/nicvf_hw.c b/drivers/net/thunderx/base/nicvf_hw.c
index ea8092ca..5b1abe20 100644
--- a/drivers/net/thunderx/base/nicvf_hw.c
+++ b/drivers/net/thunderx/base/nicvf_hw.c
@@ -699,6 +699,19 @@ nicvf_vlan_hw_strip(struct nicvf *nic, bool enable)
 	else
 		val &= ~((STRIP_SECOND_VLAN | STRIP_FIRST_VLAN) << 25);
 
+	nic->vlan_strip = enable;
+	nicvf_reg_write(nic, NIC_VNIC_RQ_GEN_CFG, val);
+}
+
+void
+nicvf_first_skip_config(struct nicvf *nic, uint8_t num_dwords)
+{
+	uint64_t val;
+
+	val = nicvf_reg_read(nic, NIC_VNIC_RQ_GEN_CFG);
+	val &= ~(0xfULL);
+	val |= (num_dwords & 0xf);
+
 	nicvf_reg_write(nic, NIC_VNIC_RQ_GEN_CFG, val);
 }
 
diff --git a/drivers/net/thunderx/base/nicvf_hw.h b/drivers/net/thunderx/base/nicvf_hw.h
index 284d0bdf..fd13ea84 100644
--- a/drivers/net/thunderx/base/nicvf_hw.h
+++ b/drivers/net/thunderx/base/nicvf_hw.h
@@ -193,6 +193,7 @@ uint32_t nicvf_qsize_sq_roundup(uint32_t val);
 void nicvf_vlan_hw_strip(struct nicvf *nic, bool enable);
 
 void nicvf_apad_config(struct nicvf *nic, bool enable);
+void nicvf_first_skip_config(struct nicvf *nic, uint8_t dwords);
 
 int nicvf_rss_config(struct nicvf *nic, uint32_t  qcnt, uint64_t cfg);
 int nicvf_rss_term(struct nicvf *nic);
diff --git a/drivers/net/thunderx/nicvf_ethdev.c b/drivers/net/thunderx/nicvf_ethdev.c
index 99fcd516..a55c3ca6 100644
--- a/drivers/net/thunderx/nicvf_ethdev.c
+++ b/drivers/net/thunderx/nicvf_ethdev.c
@@ -34,6 +34,8 @@
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
 #include <rte_tailq.h>
+#include <rte_devargs.h>
+#include <rte_kvargs.h>
 
 #include "base/nicvf_plat.h"
 
@@ -50,10 +52,10 @@ static void nicvf_dev_stop(struct rte_eth_dev *dev);
 static void nicvf_dev_stop_cleanup(struct rte_eth_dev *dev, bool cleanup);
 static void nicvf_vf_stop(struct rte_eth_dev *dev, struct nicvf *nic,
 			  bool cleanup);
+static int nicvf_vlan_offload_config(struct rte_eth_dev *dev, int mask);
+static int nicvf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 
-RTE_INIT(nicvf_init_log);
-static void
-nicvf_init_log(void)
+RTE_INIT(nicvf_init_log)
 {
 	nicvf_logtype_mbox = rte_log_register("pmd.net.thunderx.mbox");
 	if (nicvf_logtype_mbox >= 0)
@@ -355,11 +357,9 @@ nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 	}
 
 	memcpy((char *)ptypes + copied, &ptypes_end, sizeof(ptypes_end));
-	if (dev->rx_pkt_burst == nicvf_recv_pkts ||
-		dev->rx_pkt_burst == nicvf_recv_pkts_multiseg)
-		return ptypes;
 
-	return NULL;
+	/* All Ptypes are supported in all Rx functions. */
+	return ptypes;
 }
 
 static void
@@ -883,7 +883,7 @@ nicvf_dev_tx_queue_release(void *sq)
 static void
 nicvf_set_tx_function(struct rte_eth_dev *dev)
 {
-	struct nicvf_txq *txq;
+	struct nicvf_txq *txq = NULL;
 	size_t i;
 	bool multiseg = false;
 
@@ -904,6 +904,9 @@ nicvf_set_tx_function(struct rte_eth_dev *dev)
 		dev->tx_pkt_burst = nicvf_xmit_pkts;
 	}
 
+	if (!txq)
+		return;
+
 	if (txq->pool_free == nicvf_single_pool_free_xmited_buffers)
 		PMD_DRV_LOG(DEBUG, "Using single-mempool tx free method");
 	else
@@ -913,13 +916,23 @@ nicvf_set_tx_function(struct rte_eth_dev *dev)
 static void
 nicvf_set_rx_function(struct rte_eth_dev *dev)
 {
-	if (dev->data->scattered_rx) {
-		PMD_DRV_LOG(DEBUG, "Using multi-segment rx callback");
-		dev->rx_pkt_burst = nicvf_recv_pkts_multiseg;
-	} else {
-		PMD_DRV_LOG(DEBUG, "Using single-segment rx callback");
-		dev->rx_pkt_burst = nicvf_recv_pkts;
-	}
+	struct nicvf *nic = nicvf_pmd_priv(dev);
+
+	const eth_rx_burst_t rx_burst_func[2][2][2] = {
+	/* [NORMAL/SCATTER] [CKSUM/NO_CKSUM] [VLAN_STRIP/NO_VLAN_STRIP] */
+		[0][0][0] = nicvf_recv_pkts_no_offload,
+		[0][0][1] = nicvf_recv_pkts_vlan_strip,
+		[0][1][0] = nicvf_recv_pkts_cksum,
+		[0][1][1] = nicvf_recv_pkts_cksum_vlan_strip,
+		[1][0][0] = nicvf_recv_pkts_multiseg_no_offload,
+		[1][0][1] = nicvf_recv_pkts_multiseg_vlan_strip,
+		[1][1][0] = nicvf_recv_pkts_multiseg_cksum,
+		[1][1][1] = nicvf_recv_pkts_multiseg_cksum_vlan_strip,
+	};
+
+	dev->rx_pkt_burst =
+		rx_burst_func[dev->data->scattered_rx]
+			[nic->offload_cksum][nic->vlan_strip];
 }
 
 static int
@@ -1230,6 +1243,7 @@ nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq)
 {
 	uintptr_t p;
 	struct rte_mbuf mb_def;
+	struct nicvf *nic = rxq->nic;
 
 	RTE_BUILD_BUG_ON(sizeof(union mbuf_initializer) != 8);
 	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0);
@@ -1239,8 +1253,11 @@ nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq)
 				offsetof(struct rte_mbuf, data_off) != 4);
 	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) -
 				offsetof(struct rte_mbuf, data_off) != 6);
+	RTE_BUILD_BUG_ON(offsetof(struct nicvf_rxq, rxq_fastpath_data_end) -
+				offsetof(struct nicvf_rxq,
+					rxq_fastpath_data_start) > 128);
 	mb_def.nb_segs = 1;
-	mb_def.data_off = RTE_PKTMBUF_HEADROOM;
+	mb_def.data_off = RTE_PKTMBUF_HEADROOM + (nic->skip_bytes);
 	mb_def.port = rxq->port_id;
 	rte_mbuf_refcnt_set(&mb_def, 1);
 
@@ -1260,9 +1277,19 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
 	struct nicvf_rxq *rxq;
 	struct nicvf *nic = nicvf_pmd_priv(dev);
 	uint64_t offloads;
+	uint32_t buffsz;
+	struct rte_pktmbuf_pool_private *mbp_priv;
 
 	PMD_INIT_FUNC_TRACE();
 
+	/* First skip check */
+	mbp_priv = rte_mempool_get_priv(mp);
+	buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
+	if (buffsz < (uint32_t)(nic->skip_bytes)) {
+		PMD_INIT_LOG(ERR, "First skip is more than configured buffer size");
+		return -EINVAL;
+	}
+
 	if (qidx >= MAX_RCV_QUEUES_PER_QS)
 		nic = nic->snicvf[qidx / MAX_RCV_QUEUES_PER_QS - 1];
 
@@ -1298,6 +1325,7 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
 		return -EINVAL;
 	}
 
+
 	/* Check rx_free_thresh upper bound */
 	rx_free_thresh = (uint16_t)((rx_conf->rx_free_thresh) ?
 				rx_conf->rx_free_thresh :
@@ -1452,7 +1480,7 @@ nicvf_vf_start(struct rte_eth_dev *dev, struct nicvf *nic, uint32_t rbdrsz)
 	struct rte_mbuf *mbuf;
 	uint16_t rx_start, rx_end;
 	uint16_t tx_start, tx_end;
-	bool vlan_strip;
+	int mask;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -1498,6 +1526,7 @@ nicvf_vf_start(struct rte_eth_dev *dev, struct nicvf *nic, uint32_t rbdrsz)
 			return -EINVAL;
 		}
 		rxq->mbuf_phys_off -= data_off;
+		rxq->mbuf_phys_off -= nic->skip_bytes;
 
 		if (mbuf_phys_off == 0)
 			mbuf_phys_off = rxq->mbuf_phys_off;
@@ -1572,9 +1601,9 @@ nicvf_vf_start(struct rte_eth_dev *dev, struct nicvf *nic, uint32_t rbdrsz)
 		     nic->rbdr->tail, nb_rbdr_desc, nic->vf_id);
 
 	/* Configure VLAN Strip */
-	vlan_strip = !!(dev->data->dev_conf.rxmode.offloads &
-			DEV_RX_OFFLOAD_VLAN_STRIP);
-	nicvf_vlan_hw_strip(nic, vlan_strip);
+	mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
+		ETH_VLAN_EXTEND_MASK;
+	ret = nicvf_vlan_offload_config(dev, mask);
 
 	/* Based on the packet type(IPv4 or IPv6), the nicvf HW aligns L3 data
 	 * to the 64bit memory address.
@@ -1727,7 +1756,7 @@ nicvf_dev_start(struct rte_eth_dev *dev)
 			return ret;
 	}
 
-	/* Configure callbacks based on scatter mode */
+	/* Configure callbacks based on offloads */
 	nicvf_set_tx_function(dev);
 	nicvf_set_rx_function(dev);
 
@@ -1887,7 +1916,10 @@ nicvf_dev_configure(struct rte_eth_dev *dev)
 		return -EINVAL;
 	}
 
-	if ((rxmode->offloads & DEV_RX_OFFLOAD_CRC_STRIP) == 0) {
+	/* KEEP_CRC offload flag is not supported by PMD
+	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
+	 */
+	if (rte_eth_dev_must_keep_crc(rxmode->offloads)) {
 		PMD_INIT_LOG(NOTICE, "Can't disable hw crc strip");
 		rxmode->offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
 	}
@@ -1943,6 +1975,9 @@ nicvf_dev_configure(struct rte_eth_dev *dev)
 		}
 	}
 
+	if (rxmode->offloads & DEV_RX_OFFLOAD_CHECKSUM)
+		nic->offload_cksum = 1;
+
 	PMD_INIT_LOG(DEBUG, "Configured ethdev port%d hwcap=0x%" PRIx64,
 		dev->data->port_id, nicvf_hw_cap(nic));
 
@@ -1962,6 +1997,7 @@ static const struct eth_dev_ops nicvf_eth_dev_ops = {
 	.dev_infos_get            = nicvf_dev_info_get,
 	.dev_supported_ptypes_get = nicvf_dev_supported_ptypes_get,
 	.mtu_set                  = nicvf_dev_set_mtu,
+	.vlan_offload_set         = nicvf_vlan_offload_set,
 	.reta_update              = nicvf_dev_reta_update,
 	.reta_query               = nicvf_dev_reta_query,
 	.rss_hash_update          = nicvf_dev_rss_hash_update,
@@ -1979,6 +2015,83 @@ static const struct eth_dev_ops nicvf_eth_dev_ops = {
 };
 
 static int
+nicvf_vlan_offload_config(struct rte_eth_dev *dev, int mask)
+{
+	struct rte_eth_rxmode *rxmode;
+	struct nicvf *nic = nicvf_pmd_priv(dev);
+	rxmode = &dev->data->dev_conf.rxmode;
+	if (mask & ETH_VLAN_STRIP_MASK) {
+		if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
+			nicvf_vlan_hw_strip(nic, true);
+		else
+			nicvf_vlan_hw_strip(nic, false);
+	}
+
+	return 0;
+}
+
+static int
+nicvf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
+{
+	nicvf_vlan_offload_config(dev, mask);
+
+	return 0;
+}
+
+static inline int
+nicvf_set_first_skip(struct rte_eth_dev *dev)
+{
+	int bytes_to_skip = 0;
+	int ret = 0;
+	unsigned int i;
+	struct rte_kvargs *kvlist;
+	static const char *const skip[] = {
+		SKIP_DATA_BYTES,
+		NULL};
+	struct nicvf *nic = nicvf_pmd_priv(dev);
+
+	if (!dev->device->devargs) {
+		nicvf_first_skip_config(nic, 0);
+		return ret;
+	}
+
+	kvlist = rte_kvargs_parse(dev->device->devargs->args, skip);
+	if (!kvlist)
+		return -EINVAL;
+
+	if (kvlist->count == 0)
+		goto exit;
+
+	for (i = 0; i != kvlist->count; ++i) {
+		const struct rte_kvargs_pair *pair = &kvlist->pairs[i];
+
+		if (!strcmp(pair->key, SKIP_DATA_BYTES))
+			bytes_to_skip = atoi(pair->value);
+	}
+
+	/*128 bytes amounts to one cache line*/
+	if (bytes_to_skip >= 0 && bytes_to_skip < 128) {
+		if (!(bytes_to_skip % 8)) {
+			nicvf_first_skip_config(nic, (bytes_to_skip / 8));
+			nic->skip_bytes = bytes_to_skip;
+			goto kvlist_free;
+		} else {
+			PMD_INIT_LOG(ERR, "skip_data_bytes should be multiple of 8");
+			ret = -EINVAL;
+			goto exit;
+		}
+	} else {
+		PMD_INIT_LOG(ERR, "skip_data_bytes should be less than 128");
+		ret = -EINVAL;
+		goto exit;
+	}
+exit:
+	nicvf_first_skip_config(nic, 0);
+kvlist_free:
+	rte_kvargs_free(kvlist);
+	return ret;
+}
+static int
 nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
 {
 	int ret;
@@ -2087,6 +2200,11 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
 		goto malloc_fail;
 	}
 
+	ret = nicvf_set_first_skip(eth_dev);
+	if (ret) {
+		PMD_INIT_LOG(ERR, "Failed to configure first skip");
+		goto malloc_fail;
+	}
 	PMD_INIT_LOG(INFO, "Port %d (%x:%x) mac=%02x:%02x:%02x:%02x:%02x:%02x",
 		eth_dev->data->port_id, nic->vendor_id, nic->device_id,
 		nic->mac_addr[0], nic->mac_addr[1], nic->mac_addr[2],
@@ -2159,3 +2277,4 @@ static struct rte_pci_driver rte_nicvf_pmd = {
 RTE_PMD_REGISTER_PCI(net_thunderx, rte_nicvf_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_thunderx, pci_id_nicvf_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_thunderx, "* igb_uio | uio_pci_generic | vfio-pci");
+RTE_PMD_REGISTER_PARAM_STRING(net_thunderx, SKIP_DATA_BYTES "=<int>");
diff --git a/drivers/net/thunderx/nicvf_ethdev.h b/drivers/net/thunderx/nicvf_ethdev.h
index ea8dccd1..ae440fef 100644
--- a/drivers/net/thunderx/nicvf_ethdev.h
+++ b/drivers/net/thunderx/nicvf_ethdev.h
@@ -38,6 +38,7 @@
 	DEV_TX_OFFLOAD_MULTI_SEGS)
 
 #define NICVF_RX_OFFLOAD_CAPA ( \
+	DEV_RX_OFFLOAD_CHECKSUM    | \
 	DEV_RX_OFFLOAD_VLAN_STRIP  | \
 	DEV_RX_OFFLOAD_CRC_STRIP   | \
 	DEV_RX_OFFLOAD_JUMBO_FRAME | \
@@ -51,6 +52,7 @@
 
 #define VLAN_TAG_SIZE                   4	/* 802.3ac tag */
 
+#define SKIP_DATA_BYTES "skip_data_bytes"
 static inline struct nicvf *
 nicvf_pmd_priv(struct rte_eth_dev *eth_dev)
 {
diff --git a/drivers/net/thunderx/nicvf_rxtx.c b/drivers/net/thunderx/nicvf_rxtx.c
index 72305d9d..247c3568 100644
--- a/drivers/net/thunderx/nicvf_rxtx.c
+++ b/drivers/net/thunderx/nicvf_rxtx.c
@@ -162,12 +162,14 @@ nicvf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		free_desc -= TX_DESC_PER_PKT;
 	}
 
-	sq->tail = tail;
-	sq->xmit_bufs += i;
-	rte_wmb();
+	if (likely(i)) {
+		sq->tail = tail;
+		sq->xmit_bufs += i;
+		rte_wmb();
 
-	/* Inform HW to xmit the packets */
-	nicvf_addr_write(sq->sq_door, i * TX_DESC_PER_PKT);
+		/* Inform HW to xmit the packets */
+		nicvf_addr_write(sq->sq_door, i * TX_DESC_PER_PKT);
+	}
 	return i;
 }
 
@@ -218,12 +220,14 @@ nicvf_xmit_pkts_multiseg(void *tx_queue, struct rte_mbuf **tx_pkts,
 		}
 	}
 
-	sq->tail = tail;
-	sq->xmit_bufs += used_bufs;
-	rte_wmb();
+	if (likely(used_desc)) {
+		sq->tail = tail;
+		sq->xmit_bufs += used_bufs;
+		rte_wmb();
 
-	/* Inform HW to xmit the packets */
-	nicvf_addr_write(sq->sq_door, used_desc);
+		/* Inform HW to xmit the packets */
+		nicvf_addr_write(sq->sq_door, used_desc);
+	}
 	return i;
 }
 
@@ -327,6 +331,20 @@ nicvf_rx_classify_pkt(cqe_rx_word0_t cqe_rx_w0)
 	return ptype_table[cqe_rx_w0.l3_type][cqe_rx_w0.l4_type];
 }
 
+static inline uint64_t __hot
+nicvf_set_olflags(const cqe_rx_word0_t cqe_rx_w0)
+{
+	static const uint64_t flag_table[3] __rte_cache_aligned = {
+		PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD,
+		PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_UNKNOWN,
+		PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD,
+	};
+
+	const uint8_t idx = (cqe_rx_w0.err_opcode == CQE_RX_ERR_L4_CHK) << 1 |
+		(cqe_rx_w0.err_opcode == CQE_RX_ERR_IP_CHK);
+	return flag_table[idx];
+}
+
 static inline int __hot
 nicvf_fill_rbdr(struct nicvf_rxq *rxq, int to_fill)
 {
@@ -385,11 +403,13 @@ nicvf_rx_offload(cqe_rx_word0_t cqe_rx_w0, cqe_rx_word2_t cqe_rx_w2,
 	if (likely(cqe_rx_w0.rss_alg)) {
 		pkt->hash.rss = cqe_rx_w2.rss_tag;
 		pkt->ol_flags |= PKT_RX_RSS_HASH;
+
 	}
 }
 
-uint16_t __hot
-nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+static __rte_always_inline uint16_t
+nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
+		const uint32_t flag)
 {
 	uint32_t i, to_process;
 	struct cqe_rx_t *cqe_rx;
@@ -420,7 +440,19 @@ nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		rb0_ptr = *((uint64_t *)cqe_rx + rbptr_offset);
 		pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt
 				(rb0_ptr - cqe_rx_w1.align_pad, mbuf_phys_off);
-		pkt->ol_flags = 0;
+
+		if (flag & NICVF_RX_OFFLOAD_NONE)
+			pkt->ol_flags = 0;
+		if (flag & NICVF_RX_OFFLOAD_CKSUM)
+			pkt->ol_flags = nicvf_set_olflags(cqe_rx_w0);
+		if (flag & NICVF_RX_OFFLOAD_VLAN_STRIP) {
+			if (unlikely(cqe_rx_w0.vlan_stripped)) {
+				pkt->ol_flags |= PKT_RX_VLAN
+							| PKT_RX_VLAN_STRIPPED;
+				pkt->vlan_tci =
+					rte_cpu_to_be_16(cqe_rx_w2.vlan_tci);
+			}
+		}
 		pkt->data_len = cqe_rx_w3.rb0_sz;
 		pkt->pkt_len = cqe_rx_w3.rb0_sz;
 		pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0);
@@ -445,11 +477,43 @@ nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	return to_process;
 }
 
-static inline uint16_t __hot
+uint16_t __hot
+nicvf_recv_pkts_no_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts)
+{
+	return nicvf_recv_pkts(rx_queue, rx_pkts, nb_pkts,
+			NICVF_RX_OFFLOAD_NONE);
+}
+
+uint16_t __hot
+nicvf_recv_pkts_cksum(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts)
+{
+	return nicvf_recv_pkts(rx_queue, rx_pkts, nb_pkts,
+			NICVF_RX_OFFLOAD_CKSUM);
+}
+
+uint16_t __hot
+nicvf_recv_pkts_vlan_strip(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts)
+{
+	return nicvf_recv_pkts(rx_queue, rx_pkts, nb_pkts,
+			NICVF_RX_OFFLOAD_NONE | NICVF_RX_OFFLOAD_VLAN_STRIP);
+}
+
+uint16_t __hot
+nicvf_recv_pkts_cksum_vlan_strip(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts)
+{
+	return nicvf_recv_pkts(rx_queue, rx_pkts, nb_pkts,
+			NICVF_RX_OFFLOAD_CKSUM | NICVF_RX_OFFLOAD_VLAN_STRIP);
+}
+
+static __rte_always_inline uint16_t __hot
 nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx,
 			uint64_t mbuf_phys_off,
 			struct rte_mbuf **rx_pkt, uint8_t rbptr_offset,
-			uint64_t mbuf_init)
+			uint64_t mbuf_init, const uint32_t flag)
 {
 	struct rte_mbuf *pkt, *seg, *prev;
 	cqe_rx_word0_t cqe_rx_w0;
@@ -467,12 +531,22 @@ nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx,
 	pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt
 			(rb_ptr[0] - cqe_rx_w1.align_pad, mbuf_phys_off);
 
-	pkt->ol_flags = 0;
 	pkt->pkt_len = cqe_rx_w1.pkt_len;
 	pkt->data_len = rb_sz[nicvf_frag_num(0)];
 	nicvf_mbuff_init_mseg_update(
 				pkt, mbuf_init, cqe_rx_w1.align_pad, nb_segs);
 	pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0);
+	if (flag & NICVF_RX_OFFLOAD_NONE)
+		pkt->ol_flags = 0;
+	if (flag & NICVF_RX_OFFLOAD_CKSUM)
+		pkt->ol_flags = nicvf_set_olflags(cqe_rx_w0);
+	if (flag & NICVF_RX_OFFLOAD_VLAN_STRIP) {
+		if (unlikely(cqe_rx_w0.vlan_stripped)) {
+			pkt->ol_flags |= PKT_RX_VLAN
+				| PKT_RX_VLAN_STRIPPED;
+			pkt->vlan_tci = rte_cpu_to_be_16(cqe_rx_w2.vlan_tci);
+		}
+	}
 	nicvf_rx_offload(cqe_rx_w0, cqe_rx_w2, pkt);
 
 	*rx_pkt = pkt;
@@ -491,9 +565,9 @@ nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx,
 	return nb_segs;
 }
 
-uint16_t __hot
+static __rte_always_inline uint16_t __hot
 nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts,
-			 uint16_t nb_pkts)
+			 uint16_t nb_pkts, const uint32_t flag)
 {
 	union cq_entry_t *cq_entry;
 	struct cqe_rx_t *cqe_rx;
@@ -515,7 +589,7 @@ nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts,
 		cq_entry = &desc[cqe_head];
 		cqe_rx = (struct cqe_rx_t *)cq_entry;
 		nb_segs = nicvf_process_cq_mseg_entry(cqe_rx, mbuf_phys_off,
-			rx_pkts + i, rbptr_offset, mbuf_init);
+			rx_pkts + i, rbptr_offset, mbuf_init, flag);
 		buffers_consumed += nb_segs;
 		cqe_head = (cqe_head + 1) & cqe_mask;
 		nicvf_prefetch_store_keep(rx_pkts[i]);
@@ -535,6 +609,38 @@ nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts,
 	return to_process;
 }
 
+uint16_t __hot
+nicvf_recv_pkts_multiseg_no_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts)
+{
+	return nicvf_recv_pkts_multiseg(rx_queue, rx_pkts, nb_pkts,
+			NICVF_RX_OFFLOAD_NONE);
+}
+
+uint16_t __hot
+nicvf_recv_pkts_multiseg_cksum(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts)
+{
+	return nicvf_recv_pkts_multiseg(rx_queue, rx_pkts, nb_pkts,
+			NICVF_RX_OFFLOAD_CKSUM);
+}
+
+uint16_t __hot
+nicvf_recv_pkts_multiseg_vlan_strip(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts)
+{
+	return nicvf_recv_pkts_multiseg(rx_queue, rx_pkts, nb_pkts,
+			NICVF_RX_OFFLOAD_NONE | NICVF_RX_OFFLOAD_VLAN_STRIP);
+}
+
+uint16_t __hot
+nicvf_recv_pkts_multiseg_cksum_vlan_strip(void *rx_queue,
+		struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	return nicvf_recv_pkts_multiseg(rx_queue, rx_pkts, nb_pkts,
+			NICVF_RX_OFFLOAD_CKSUM | NICVF_RX_OFFLOAD_VLAN_STRIP);
+}
+
 uint32_t
 nicvf_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx)
 {
diff --git a/drivers/net/thunderx/nicvf_rxtx.h b/drivers/net/thunderx/nicvf_rxtx.h
index 8bdd582e..a39808cb 100644
--- a/drivers/net/thunderx/nicvf_rxtx.h
+++ b/drivers/net/thunderx/nicvf_rxtx.h
@@ -8,6 +8,10 @@
 #include <rte_byteorder.h>
 #include <rte_ethdev_driver.h>
 
+#define NICVF_RX_OFFLOAD_NONE           0x1
+#define NICVF_RX_OFFLOAD_CKSUM          0x2
+#define NICVF_RX_OFFLOAD_VLAN_STRIP     0x4
+
 #define NICVF_TX_OFFLOAD_MASK (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK)
 
 #ifndef __hot
@@ -86,9 +90,23 @@ nicvf_mbuff_init_mseg_update(struct rte_mbuf *pkt, const uint64_t mbuf_init,
 uint32_t nicvf_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx);
 uint32_t nicvf_dev_rbdr_refill(struct rte_eth_dev *dev, uint16_t queue_idx);
 
-uint16_t nicvf_recv_pkts(void *rxq, struct rte_mbuf **rx_pkts, uint16_t pkts);
-uint16_t nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts,
-				  uint16_t nb_pkts);
+uint16_t nicvf_recv_pkts_no_offload(void *rxq, struct rte_mbuf **rx_pkts,
+		uint16_t pkts);
+uint16_t nicvf_recv_pkts_cksum(void *rxq, struct rte_mbuf **rx_pkts,
+		uint16_t pkts);
+uint16_t nicvf_recv_pkts_vlan_strip(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts);
+uint16_t nicvf_recv_pkts_cksum_vlan_strip(void *rx_queue,
+		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+
+uint16_t nicvf_recv_pkts_multiseg_no_offload(void *rx_queue,
+		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t nicvf_recv_pkts_multiseg_cksum(void *rx_queue,
+		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t nicvf_recv_pkts_multiseg_vlan_strip(void *rx_queue,
+		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t nicvf_recv_pkts_multiseg_cksum_vlan_strip(void *rx_queue,
+		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
 
 uint16_t nicvf_xmit_pkts(void *txq, struct rte_mbuf **tx_pkts, uint16_t pkts);
 uint16_t nicvf_xmit_pkts_multiseg(void *txq, struct rte_mbuf **tx_pkts,
diff --git a/drivers/net/thunderx/nicvf_struct.h b/drivers/net/thunderx/nicvf_struct.h
index d4a83c37..dd52f38e 100644
--- a/drivers/net/thunderx/nicvf_struct.h
+++ b/drivers/net/thunderx/nicvf_struct.h
@@ -55,25 +55,27 @@ union mbuf_initializer {
 };
 
 struct nicvf_rxq {
+	MARKER rxq_fastpath_data_start;
+	uint8_t  rbptr_offset;
+	uint16_t rx_free_thresh;
+	uint32_t head;
+	uint32_t qlen_mask;
+	int32_t recv_buffers;
+	int32_t available_space;
 	uint64_t mbuf_phys_off;
 	uintptr_t cq_status;
 	uintptr_t cq_door;
-	union mbuf_initializer mbuf_initializer;
-	nicvf_iova_addr_t phys;
-	union cq_entry_t *desc;
 	struct nicvf_rbdr *shared_rbdr;
-	struct nicvf *nic;
 	struct rte_mempool *pool;
-	uint32_t head;
-	uint32_t qlen_mask;
-	int32_t available_space;
-	int32_t recv_buffers;
-	uint16_t rx_free_thresh;
-	uint16_t queue_id;
-	uint16_t precharge_cnt;
+	union cq_entry_t *desc;
+	union mbuf_initializer mbuf_initializer;
+	MARKER rxq_fastpath_data_end;
 	uint8_t rx_drop_en;
+	uint16_t precharge_cnt;
 	uint16_t port_id;
-	uint8_t  rbptr_offset;
+	uint16_t queue_id;
+	struct nicvf *nic;
+	nicvf_iova_addr_t phys;
 } __rte_cache_aligned;
 
 struct nicvf {
@@ -85,6 +87,8 @@ struct nicvf {
 	bool loopback_supported;
 	bool pf_acked:1;
 	bool pf_nacked:1;
+	bool offload_cksum:1;
+	bool vlan_strip:1;
 	uint64_t hwcap;
 	uint8_t link_up;
 	uint8_t	duplex;
@@ -99,6 +103,7 @@ struct nicvf {
 	struct rte_intr_handle intr_handle;
 	uint8_t cpi_alg;
 	uint16_t mtu;
+	int skip_bytes;
 	bool vlan_filter_en;
 	uint8_t mac_addr[ETHER_ADDR_LEN];
 	/* secondary queue set support */
diff --git a/drivers/net/vhost/meson.build b/drivers/net/vhost/meson.build
new file mode 100644
index 00000000..9b067c35
--- /dev/null
+++ b/drivers/net/vhost/meson.build
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+build = dpdk_conf.has('RTE_LIBRTE_VHOST')
+version = 2
+sources = files('rte_eth_vhost.c')
+install_headers('rte_eth_vhost.h')
+deps += 'vhost'
diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index ba9d768a..e58f3221 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -1070,7 +1070,8 @@ eth_dev_info(struct rte_eth_dev *dev,
 
 	dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
 				DEV_TX_OFFLOAD_VLAN_INSERT;
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
+				    DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
@@ -1353,6 +1354,7 @@ rte_pmd_vhost_probe(struct rte_vdev_device *dev)
 		}
 		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &ops;
+		eth_dev->device = &dev->device;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -1456,9 +1458,7 @@ RTE_PMD_REGISTER_PARAM_STRING(net_vhost,
 	"iface=<ifc> "
 	"queues=<int>");
 
-RTE_INIT(vhost_init_log);
-static void
-vhost_init_log(void)
+RTE_INIT(vhost_init_log)
 {
 	vhost_logtype = rte_log_register("pmd.net.vhost");
 	if (vhost_logtype >= 0)
diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
index df50a571..614357da 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1320,6 +1320,11 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
 		PMD_INIT_LOG(INFO, "virtio: using simple Rx path on port %u",
 			eth_dev->data->port_id);
 		eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
+	} else if (hw->use_inorder_rx) {
+		PMD_INIT_LOG(INFO,
+			"virtio: using inorder mergeable buffer Rx path on port %u",
+			eth_dev->data->port_id);
+		eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts_inorder;
 	} else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
 		PMD_INIT_LOG(INFO,
 			"virtio: using mergeable buffer Rx path on port %u",
@@ -1331,10 +1336,10 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
 		eth_dev->rx_pkt_burst = &virtio_recv_pkts;
 	}
 
-	if (hw->use_simple_tx) {
-		PMD_INIT_LOG(INFO, "virtio: using simple Tx path on port %u",
+	if (hw->use_inorder_tx) {
+		PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on port %u",
 			eth_dev->data->port_id);
-		eth_dev->tx_pkt_burst = virtio_xmit_pkts_simple;
+		eth_dev->tx_pkt_burst = virtio_xmit_pkts_inorder;
 	} else {
 		PMD_INIT_LOG(INFO, "virtio: using standard Tx path on port %u",
 			eth_dev->data->port_id);
@@ -1781,9 +1786,7 @@ static struct rte_pci_driver rte_virtio_pmd = {
 	.remove = eth_virtio_pci_remove,
 };
 
-RTE_INIT(rte_virtio_pmd_init);
-static void
-rte_virtio_pmd_init(void)
+RTE_INIT(rte_virtio_pmd_init)
 {
 	if (rte_eal_iopl_init() != 0) {
 		PMD_INIT_LOG(ERR, "IOPL call failed - cannot use virtio PMD");
@@ -1793,6 +1796,22 @@ rte_virtio_pmd_init(void)
 	rte_pci_register(&rte_virtio_pmd);
 }
 
+static bool
+rx_offload_enabled(struct virtio_hw *hw)
+{
+	return vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) ||
+		vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
+		vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6);
+}
+
+static bool
+tx_offload_enabled(struct virtio_hw *hw)
+{
+	return vtpci_with_feature(hw, VIRTIO_NET_F_CSUM) ||
+		vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) ||
+		vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO6);
+}
+
 /*
  * Configure virtio device
  * It returns 0 on success.
@@ -1801,8 +1820,10 @@ static int
 virtio_dev_configure(struct rte_eth_dev *dev)
 {
 	const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
+	const struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode;
 	struct virtio_hw *hw = dev->data->dev_private;
 	uint64_t rx_offloads = rxmode->offloads;
+	uint64_t tx_offloads = txmode->offloads;
 	uint64_t req_features;
 	int ret;
 
@@ -1824,6 +1845,15 @@ virtio_dev_configure(struct rte_eth_dev *dev)
 			(1ULL << VIRTIO_NET_F_GUEST_TSO4) |
 			(1ULL << VIRTIO_NET_F_GUEST_TSO6);
 
+	if (tx_offloads & (DEV_TX_OFFLOAD_UDP_CKSUM |
+			   DEV_TX_OFFLOAD_TCP_CKSUM))
+		req_features |= (1ULL << VIRTIO_NET_F_CSUM);
+
+	if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO)
+		req_features |=
+			(1ULL << VIRTIO_NET_F_HOST_TSO4) |
+			(1ULL << VIRTIO_NET_F_HOST_TSO6);
+
 	/* if request features changed, reinit the device */
 	if (req_features != hw->req_guest_features) {
 		ret = virtio_init_device(dev, req_features);
@@ -1861,6 +1891,9 @@ virtio_dev_configure(struct rte_eth_dev *dev)
 		return -ENOTSUP;
 	}
 
+	hw->has_tx_offload = tx_offload_enabled(hw);
+	hw->has_rx_offload = rx_offload_enabled(hw);
+
 	if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
 		/* Enable vector (0) for Link State Intrerrupt */
 		if (VTPCI_OPS(hw)->set_config_irq(hw, 0) ==
@@ -1872,21 +1905,30 @@ virtio_dev_configure(struct rte_eth_dev *dev)
 	rte_spinlock_init(&hw->state_lock);
 
 	hw->use_simple_rx = 1;
-	hw->use_simple_tx = 1;
+
+	if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) {
+		hw->use_inorder_tx = 1;
+		if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+			hw->use_inorder_rx = 1;
+			hw->use_simple_rx = 0;
+		} else {
+			hw->use_inorder_rx = 0;
+		}
+	}
 
 #if defined RTE_ARCH_ARM64 || defined RTE_ARCH_ARM
 	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
 		hw->use_simple_rx = 0;
-		hw->use_simple_tx = 0;
 	}
 #endif
 	if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
-		hw->use_simple_rx = 0;
-		hw->use_simple_tx = 0;
+		 hw->use_simple_rx = 0;
 	}
 
 	if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM |
-			   DEV_RX_OFFLOAD_TCP_CKSUM))
+			   DEV_RX_OFFLOAD_TCP_CKSUM |
+			   DEV_RX_OFFLOAD_TCP_LRO |
+			   DEV_RX_OFFLOAD_VLAN_STRIP))
 		hw->use_simple_rx = 0;
 
 	return 0;
@@ -2122,12 +2164,10 @@ virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE;
 	dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN;
 	dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS;
-	dev_info->default_txconf = (struct rte_eth_txconf) {
-		.txq_flags = ETH_TXQ_FLAGS_NOOFFLOADS
-	};
 
 	host_features = VTPCI_OPS(hw)->get_features(hw);
-	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
+				    DEV_RX_OFFLOAD_CRC_STRIP;
 	if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
 		dev_info->rx_offload_capa |=
 			DEV_RX_OFFLOAD_TCP_CKSUM |
@@ -2142,14 +2182,14 @@ virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 
 	dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
 				    DEV_TX_OFFLOAD_VLAN_INSERT;
-	if (hw->guest_features & (1ULL << VIRTIO_NET_F_CSUM)) {
+	if (host_features & (1ULL << VIRTIO_NET_F_CSUM)) {
 		dev_info->tx_offload_capa |=
 			DEV_TX_OFFLOAD_UDP_CKSUM |
 			DEV_TX_OFFLOAD_TCP_CKSUM;
 	}
 	tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) |
 		(1ULL << VIRTIO_NET_F_HOST_TSO6);
-	if ((hw->guest_features & tso_mask) == tso_mask)
+	if ((host_features & tso_mask) == tso_mask)
 		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
 }
 
@@ -2168,9 +2208,7 @@ RTE_PMD_EXPORT_NAME(net_virtio, __COUNTER__);
 RTE_PMD_REGISTER_PCI_TABLE(net_virtio, pci_id_virtio_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_virtio, "* igb_uio | uio_pci_generic | vfio-pci");
 
-RTE_INIT(virtio_init_log);
-static void
-virtio_init_log(void)
+RTE_INIT(virtio_init_log)
 {
 	virtio_logtype_init = rte_log_register("pmd.net.virtio.init");
 	if (virtio_logtype_init >= 0)
diff --git a/drivers/net/virtio/virtio_ethdev.h b/drivers/net/virtio/virtio_ethdev.h
index bb40064e..b726ad10 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -28,14 +28,12 @@
 	 1u << VIRTIO_NET_F_CTRL_VQ	  |	\
 	 1u << VIRTIO_NET_F_CTRL_RX	  |	\
 	 1u << VIRTIO_NET_F_CTRL_VLAN	  |	\
-	 1u << VIRTIO_NET_F_CSUM	  |	\
-	 1u << VIRTIO_NET_F_HOST_TSO4	  |	\
-	 1u << VIRTIO_NET_F_HOST_TSO6	  |	\
 	 1u << VIRTIO_NET_F_MRG_RXBUF	  |	\
 	 1u << VIRTIO_NET_F_MTU	| \
 	 1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE |	\
 	 1u << VIRTIO_RING_F_INDIRECT_DESC |    \
 	 1ULL << VIRTIO_F_VERSION_1       |	\
+	 1ULL << VIRTIO_F_IN_ORDER        |	\
 	 1ULL << VIRTIO_F_IOMMU_PLATFORM)
 
 #define VIRTIO_PMD_SUPPORTED_GUEST_FEATURES	\
@@ -44,13 +42,6 @@
 	 1u << VIRTIO_NET_F_GUEST_TSO4     |	\
 	 1u << VIRTIO_NET_F_GUEST_TSO6)
 
-#define VIRTIO_PMD_PER_DEVICE_RX_OFFLOADS	\
-	(DEV_RX_OFFLOAD_TCP_CKSUM |		\
-	 DEV_RX_OFFLOAD_UDP_CKSUM |		\
-	 DEV_RX_OFFLOAD_TCP_LRO |		\
-	 DEV_RX_OFFLOAD_VLAN_FILTER |		\
-	 DEV_RX_OFFLOAD_VLAN_STRIP)
-
 /*
  * CQ function prototype
  */
@@ -83,9 +74,15 @@ uint16_t virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 uint16_t virtio_recv_mergeable_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		uint16_t nb_pkts);
 
+uint16_t virtio_recv_mergeable_pkts_inorder(void *rx_queue,
+		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+
 uint16_t virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		uint16_t nb_pkts);
 
+uint16_t virtio_xmit_pkts_inorder(void *tx_queue, struct rte_mbuf **tx_pkts,
+		uint16_t nb_pkts);
+
 uint16_t virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
 		uint16_t nb_pkts);
 
diff --git a/drivers/net/virtio/virtio_pci.h b/drivers/net/virtio/virtio_pci.h
index a28ba833..58fdd3d4 100644
--- a/drivers/net/virtio/virtio_pci.h
+++ b/drivers/net/virtio/virtio_pci.h
@@ -6,6 +6,7 @@
 #define _VIRTIO_PCI_H_
 
 #include <stdint.h>
+#include <stdbool.h>
 
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -121,6 +122,12 @@ struct virtnet_ctl;
 #define VIRTIO_TRANSPORT_F_START 28
 #define VIRTIO_TRANSPORT_F_END   34
 
+/*
+ * Inorder feature indicates that all buffers are used by the device
+ * in the same order in which they have been made available.
+ */
+#define VIRTIO_F_IN_ORDER 35
+
 /* The Guest publishes the used index for which it expects an interrupt
  * at the end of the avail ring. Host should ignore the avail->flags field. */
 /* The Host publishes the avail index for which it expects a kick
@@ -232,7 +239,10 @@ struct virtio_hw {
 	uint8_t	    use_msix;
 	uint8_t     modern;
 	uint8_t     use_simple_rx;
-	uint8_t     use_simple_tx;
+	uint8_t     use_inorder_rx;
+	uint8_t     use_inorder_tx;
+	bool        has_tx_offload;
+	bool        has_rx_offload;
 	uint16_t    port_id;
 	uint8_t     mac_addr[ETHER_ADDR_LEN];
 	uint32_t    notify_off_multiplier;
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 92fab217..eb891433 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -48,6 +48,13 @@ virtio_dev_rx_queue_done(void *rxq, uint16_t offset)
 }
 
 void
+vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num)
+{
+	vq->vq_free_cnt += num;
+	vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1);
+}
+
+void
 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
 {
 	struct vring_desc *dp, *dp_tail;
@@ -115,6 +122,44 @@ virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 	return i;
 }
 
+static uint16_t
+virtqueue_dequeue_rx_inorder(struct virtqueue *vq,
+			struct rte_mbuf **rx_pkts,
+			uint32_t *len,
+			uint16_t num)
+{
+	struct vring_used_elem *uep;
+	struct rte_mbuf *cookie;
+	uint16_t used_idx = 0;
+	uint16_t i;
+
+	if (unlikely(num == 0))
+		return 0;
+
+	for (i = 0; i < num; i++) {
+		used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1);
+		/* Desc idx same as used idx */
+		uep = &vq->vq_ring.used->ring[used_idx];
+		len[i] = uep->len;
+		cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie;
+
+		if (unlikely(cookie == NULL)) {
+			PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
+				vq->vq_used_cons_idx);
+			break;
+		}
+
+		rte_prefetch0(cookie);
+		rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
+		rx_pkts[i]  = cookie;
+		vq->vq_used_cons_idx++;
+		vq->vq_descx[used_idx].cookie = NULL;
+	}
+
+	vq_ring_free_inorder(vq, used_idx, i);
+	return i;
+}
+
 #ifndef DEFAULT_TX_FREE_THRESH
 #define DEFAULT_TX_FREE_THRESH 32
 #endif
@@ -143,6 +188,83 @@ virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
 	}
 }
 
+/* Cleanup from completed inorder transmits. */
+static void
+virtio_xmit_cleanup_inorder(struct virtqueue *vq, uint16_t num)
+{
+	uint16_t i, used_idx, desc_idx = 0, last_idx;
+	int16_t free_cnt = 0;
+	struct vq_desc_extra *dxp = NULL;
+
+	if (unlikely(num == 0))
+		return;
+
+	for (i = 0; i < num; i++) {
+		struct vring_used_elem *uep;
+
+		used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1);
+		uep = &vq->vq_ring.used->ring[used_idx];
+		desc_idx = (uint16_t)uep->id;
+
+		dxp = &vq->vq_descx[desc_idx];
+		vq->vq_used_cons_idx++;
+
+		if (dxp->cookie != NULL) {
+			rte_pktmbuf_free(dxp->cookie);
+			dxp->cookie = NULL;
+		}
+	}
+
+	last_idx = desc_idx + dxp->ndescs - 1;
+	free_cnt = last_idx - vq->vq_desc_tail_idx;
+	if (free_cnt <= 0)
+		free_cnt += vq->vq_nentries;
+
+	vq_ring_free_inorder(vq, last_idx, free_cnt);
+}
+
+static inline int
+virtqueue_enqueue_refill_inorder(struct virtqueue *vq,
+			struct rte_mbuf **cookies,
+			uint16_t num)
+{
+	struct vq_desc_extra *dxp;
+	struct virtio_hw *hw = vq->hw;
+	struct vring_desc *start_dp;
+	uint16_t head_idx, idx, i = 0;
+
+	if (unlikely(vq->vq_free_cnt == 0))
+		return -ENOSPC;
+	if (unlikely(vq->vq_free_cnt < num))
+		return -EMSGSIZE;
+
+	head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1);
+	start_dp = vq->vq_ring.desc;
+
+	while (i < num) {
+		idx = head_idx & (vq->vq_nentries - 1);
+		dxp = &vq->vq_descx[idx];
+		dxp->cookie = (void *)cookies[i];
+		dxp->ndescs = 1;
+
+		start_dp[idx].addr =
+				VIRTIO_MBUF_ADDR(cookies[i], vq) +
+				RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
+		start_dp[idx].len =
+				cookies[i]->buf_len -
+				RTE_PKTMBUF_HEADROOM +
+				hw->vtnet_hdr_size;
+		start_dp[idx].flags =  VRING_DESC_F_WRITE;
+
+		vq_update_avail_ring(vq, idx);
+		head_idx++;
+		i++;
+	}
+
+	vq->vq_desc_head_idx += num;
+	vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
+	return 0;
+}
 
 static inline int
 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
@@ -225,13 +347,6 @@ virtio_tso_fix_cksum(struct rte_mbuf *m)
 	}
 }
 
-static inline int
-tx_offload_enabled(struct virtio_hw *hw)
-{
-	return vtpci_with_feature(hw, VIRTIO_NET_F_CSUM) ||
-		vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) ||
-		vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO6);
-}
 
 /* avoid write operation when necessary, to lessen cache issues */
 #define ASSIGN_UNLESS_EQUAL(var, val) do {	\
@@ -240,8 +355,111 @@ tx_offload_enabled(struct virtio_hw *hw)
 } while (0)
 
 static inline void
+virtqueue_xmit_offload(struct virtio_net_hdr *hdr,
+			struct rte_mbuf *cookie,
+			bool offload)
+{
+	if (offload) {
+		if (cookie->ol_flags & PKT_TX_TCP_SEG)
+			cookie->ol_flags |= PKT_TX_TCP_CKSUM;
+
+		switch (cookie->ol_flags & PKT_TX_L4_MASK) {
+		case PKT_TX_UDP_CKSUM:
+			hdr->csum_start = cookie->l2_len + cookie->l3_len;
+			hdr->csum_offset = offsetof(struct udp_hdr,
+				dgram_cksum);
+			hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
+			break;
+
+		case PKT_TX_TCP_CKSUM:
+			hdr->csum_start = cookie->l2_len + cookie->l3_len;
+			hdr->csum_offset = offsetof(struct tcp_hdr, cksum);
+			hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
+			break;
+
+		default:
+			ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0);
+			ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0);
+			ASSIGN_UNLESS_EQUAL(hdr->flags, 0);
+			break;
+		}
+
+		/* TCP Segmentation Offload */
+		if (cookie->ol_flags & PKT_TX_TCP_SEG) {
+			virtio_tso_fix_cksum(cookie);
+			hdr->gso_type = (cookie->ol_flags & PKT_TX_IPV6) ?
+				VIRTIO_NET_HDR_GSO_TCPV6 :
+				VIRTIO_NET_HDR_GSO_TCPV4;
+			hdr->gso_size = cookie->tso_segsz;
+			hdr->hdr_len =
+				cookie->l2_len +
+				cookie->l3_len +
+				cookie->l4_len;
+		} else {
+			ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0);
+			ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0);
+			ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0);
+		}
+	}
+}
+
+static inline void
+virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq,
+			struct rte_mbuf **cookies,
+			uint16_t num)
+{
+	struct vq_desc_extra *dxp;
+	struct virtqueue *vq = txvq->vq;
+	struct vring_desc *start_dp;
+	struct virtio_net_hdr *hdr;
+	uint16_t idx;
+	uint16_t head_size = vq->hw->vtnet_hdr_size;
+	uint16_t i = 0;
+
+	idx = vq->vq_desc_head_idx;
+	start_dp = vq->vq_ring.desc;
+
+	while (i < num) {
+		idx = idx & (vq->vq_nentries - 1);
+		dxp = &vq->vq_descx[idx];
+		dxp->cookie = (void *)cookies[i];
+		dxp->ndescs = 1;
+
+		hdr = (struct virtio_net_hdr *)
+			rte_pktmbuf_prepend(cookies[i], head_size);
+		cookies[i]->pkt_len -= head_size;
+
+		/* if offload disabled, it is not zeroed below, do it now */
+		if (!vq->hw->has_tx_offload) {
+			ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0);
+			ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0);
+			ASSIGN_UNLESS_EQUAL(hdr->flags, 0);
+			ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0);
+			ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0);
+			ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0);
+		}
+
+		virtqueue_xmit_offload(hdr, cookies[i],
+				vq->hw->has_tx_offload);
+
+		start_dp[idx].addr  = VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq);
+		start_dp[idx].len   = cookies[i]->data_len;
+		start_dp[idx].flags = 0;
+
+		vq_update_avail_ring(vq, idx);
+
+		idx++;
+		i++;
+	};
+
+	vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
+	vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1);
+}
+
+static inline void
 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
-		       uint16_t needed, int use_indirect, int can_push)
+			uint16_t needed, int use_indirect, int can_push,
+			int in_order)
 {
 	struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
 	struct vq_desc_extra *dxp;
@@ -251,9 +469,7 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 	uint16_t head_idx, idx;
 	uint16_t head_size = vq->hw->vtnet_hdr_size;
 	struct virtio_net_hdr *hdr;
-	int offload;
 
-	offload = tx_offload_enabled(vq->hw);
 	head_idx = vq->vq_desc_head_idx;
 	idx = head_idx;
 	dxp = &vq->vq_descx[idx];
@@ -270,8 +486,9 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 		 * which is wrong. Below subtract restores correct pkt size.
 		 */
 		cookie->pkt_len -= head_size;
+
 		/* if offload disabled, it is not zeroed below, do it now */
-		if (offload == 0) {
+		if (!vq->hw->has_tx_offload) {
 			ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0);
 			ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0);
 			ASSIGN_UNLESS_EQUAL(hdr->flags, 0);
@@ -308,49 +525,7 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 		idx = start_dp[idx].next;
 	}
 
-	/* Checksum Offload / TSO */
-	if (offload) {
-		if (cookie->ol_flags & PKT_TX_TCP_SEG)
-			cookie->ol_flags |= PKT_TX_TCP_CKSUM;
-
-		switch (cookie->ol_flags & PKT_TX_L4_MASK) {
-		case PKT_TX_UDP_CKSUM:
-			hdr->csum_start = cookie->l2_len + cookie->l3_len;
-			hdr->csum_offset = offsetof(struct udp_hdr,
-				dgram_cksum);
-			hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
-			break;
-
-		case PKT_TX_TCP_CKSUM:
-			hdr->csum_start = cookie->l2_len + cookie->l3_len;
-			hdr->csum_offset = offsetof(struct tcp_hdr, cksum);
-			hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
-			break;
-
-		default:
-			ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0);
-			ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0);
-			ASSIGN_UNLESS_EQUAL(hdr->flags, 0);
-			break;
-		}
-
-		/* TCP Segmentation Offload */
-		if (cookie->ol_flags & PKT_TX_TCP_SEG) {
-			virtio_tso_fix_cksum(cookie);
-			hdr->gso_type = (cookie->ol_flags & PKT_TX_IPV6) ?
-				VIRTIO_NET_HDR_GSO_TCPV6 :
-				VIRTIO_NET_HDR_GSO_TCPV4;
-			hdr->gso_size = cookie->tso_segsz;
-			hdr->hdr_len =
-				cookie->l2_len +
-				cookie->l3_len +
-				cookie->l4_len;
-		} else {
-			ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0);
-			ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0);
-			ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0);
-		}
-	}
+	virtqueue_xmit_offload(hdr, cookie, vq->hw->has_tx_offload);
 
 	do {
 		start_dp[idx].addr  = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq);
@@ -362,11 +537,15 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 	if (use_indirect)
 		idx = vq->vq_ring.desc[head_idx].next;
 
-	vq->vq_desc_head_idx = idx;
-	if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
-		vq->vq_desc_tail_idx = idx;
 	vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
+
+	vq->vq_desc_head_idx = idx;
 	vq_update_avail_ring(vq, head_idx);
+
+	if (!in_order) {
+		if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
+			vq->vq_desc_tail_idx = idx;
+	}
 }
 
 void
@@ -421,7 +600,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx)
 	struct virtnet_rx *rxvq = &vq->rxq;
 	struct rte_mbuf *m;
 	uint16_t desc_idx;
-	int error, nbufs;
+	int error, nbufs, i;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -451,6 +630,25 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx)
 			virtio_rxq_rearm_vec(rxvq);
 			nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
 		}
+	} else if (hw->use_inorder_rx) {
+		if ((!virtqueue_full(vq))) {
+			uint16_t free_cnt = vq->vq_free_cnt;
+			struct rte_mbuf *pkts[free_cnt];
+
+			if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts,
+				free_cnt)) {
+				error = virtqueue_enqueue_refill_inorder(vq,
+						pkts,
+						free_cnt);
+				if (unlikely(error)) {
+					for (i = 0; i < free_cnt; i++)
+						rte_pktmbuf_free(pkts[i]);
+				}
+			}
+
+			nbufs += free_cnt;
+			vq_update_avail_idx(vq);
+		}
 	} else {
 		while (!virtqueue_full(vq)) {
 			m = rte_mbuf_raw_alloc(rxvq->mpool);
@@ -498,10 +696,6 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
 
 	PMD_INIT_FUNC_TRACE();
 
-	/* cannot use simple rxtx funcs with multisegs or offloads */
-	if (dev->data->dev_conf.txmode.offloads)
-		hw->use_simple_tx = 0;
-
 	if (nb_desc == 0 || nb_desc > vq->vq_nentries)
 		nb_desc = vq->vq_nentries;
 	vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
@@ -536,31 +730,11 @@ virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
 	uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
 	struct virtio_hw *hw = dev->data->dev_private;
 	struct virtqueue *vq = hw->vqs[vtpci_queue_idx];
-	uint16_t mid_idx = vq->vq_nentries >> 1;
-	struct virtnet_tx *txvq = &vq->txq;
-	uint16_t desc_idx;
 
 	PMD_INIT_FUNC_TRACE();
 
-	if (hw->use_simple_tx) {
-		for (desc_idx = 0; desc_idx < mid_idx; desc_idx++) {
-			vq->vq_ring.avail->ring[desc_idx] =
-				desc_idx + mid_idx;
-			vq->vq_ring.desc[desc_idx + mid_idx].next =
-				desc_idx;
-			vq->vq_ring.desc[desc_idx + mid_idx].addr =
-				txvq->virtio_net_hdr_mem +
-				offsetof(struct virtio_tx_region, tx_hdr);
-			vq->vq_ring.desc[desc_idx + mid_idx].len =
-				vq->hw->vtnet_hdr_size;
-			vq->vq_ring.desc[desc_idx + mid_idx].flags =
-				VRING_DESC_F_NEXT;
-			vq->vq_ring.desc[desc_idx].flags = 0;
-		}
-		for (desc_idx = mid_idx; desc_idx < vq->vq_nentries;
-		     desc_idx++)
-			vq->vq_ring.avail->ring[desc_idx] = desc_idx;
-	}
+	if (hw->use_inorder_tx)
+		vq->vq_ring.desc[vq->vq_nentries - 1].next = 0;
 
 	VIRTQUEUE_DUMP(vq);
 
@@ -576,6 +750,19 @@ virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
 	 * successful since it was just dequeued.
 	 */
 	error = virtqueue_enqueue_recv_refill(vq, m);
+
+	if (unlikely(error)) {
+		RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
+		rte_pktmbuf_free(m);
+	}
+}
+
+static void
+virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m)
+{
+	int error;
+
+	error = virtqueue_enqueue_refill_inorder(vq, &m, 1);
 	if (unlikely(error)) {
 		RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf");
 		rte_pktmbuf_free(m);
@@ -614,6 +801,15 @@ virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
 	}
 }
 
+static inline void
+virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m)
+{
+	VIRTIO_DUMP_PACKET(m, m->data_len);
+
+	rxvq->stats.bytes += m->pkt_len;
+	virtio_update_packet_stats(&rxvq->stats, m);
+}
+
 /* Optionally fill offload information in structure */
 static int
 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)
@@ -686,14 +882,6 @@ virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)
 	return 0;
 }
 
-static inline int
-rx_offload_enabled(struct virtio_hw *hw)
-{
-	return vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) ||
-		vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
-		vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6);
-}
-
 #define VIRTIO_MBUF_BURST_SZ 64
 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
 uint16_t
@@ -709,7 +897,6 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	int error;
 	uint32_t i, nb_enqueued;
 	uint32_t hdr_size;
-	int offload;
 	struct virtio_net_hdr *hdr;
 
 	nb_rx = 0;
@@ -731,7 +918,6 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	nb_enqueued = 0;
 	hdr_size = hw->vtnet_hdr_size;
-	offload = rx_offload_enabled(hw);
 
 	for (i = 0; i < num ; i++) {
 		rxm = rcv_pkts[i];
@@ -760,24 +946,20 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		if (hw->vlan_strip)
 			rte_vlan_strip(rxm);
 
-		if (offload && virtio_rx_offload(rxm, hdr) < 0) {
+		if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) {
 			virtio_discard_rxbuf(vq, rxm);
 			rxvq->stats.errors++;
 			continue;
 		}
 
-		VIRTIO_DUMP_PACKET(rxm, rxm->data_len);
+		virtio_rx_stats_updated(rxvq, rxm);
 
 		rx_pkts[nb_rx++] = rxm;
-
-		rxvq->stats.bytes += rxm->pkt_len;
-		virtio_update_packet_stats(&rxvq->stats, rxm);
 	}
 
 	rxvq->stats.packets += nb_rx;
 
 	/* Allocate new mbuf for the used descriptor */
-	error = ENOSPC;
 	while (likely(!virtqueue_full(vq))) {
 		new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool);
 		if (unlikely(new_mbuf == NULL)) {
@@ -807,6 +989,193 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 }
 
 uint16_t
+virtio_recv_mergeable_pkts_inorder(void *rx_queue,
+			struct rte_mbuf **rx_pkts,
+			uint16_t nb_pkts)
+{
+	struct virtnet_rx *rxvq = rx_queue;
+	struct virtqueue *vq = rxvq->vq;
+	struct virtio_hw *hw = vq->hw;
+	struct rte_mbuf *rxm;
+	struct rte_mbuf *prev;
+	uint16_t nb_used, num, nb_rx;
+	uint32_t len[VIRTIO_MBUF_BURST_SZ];
+	struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
+	int error;
+	uint32_t nb_enqueued;
+	uint32_t seg_num;
+	uint32_t seg_res;
+	uint32_t hdr_size;
+	int32_t i;
+
+	nb_rx = 0;
+	if (unlikely(hw->started == 0))
+		return nb_rx;
+
+	nb_used = VIRTQUEUE_NUSED(vq);
+	nb_used = RTE_MIN(nb_used, nb_pkts);
+	nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ);
+
+	virtio_rmb();
+
+	PMD_RX_LOG(DEBUG, "used:%d", nb_used);
+
+	nb_enqueued = 0;
+	seg_num = 1;
+	seg_res = 0;
+	hdr_size = hw->vtnet_hdr_size;
+
+	num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used);
+
+	for (i = 0; i < num; i++) {
+		struct virtio_net_hdr_mrg_rxbuf *header;
+
+		PMD_RX_LOG(DEBUG, "dequeue:%d", num);
+		PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
+
+		rxm = rcv_pkts[i];
+
+		if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
+			PMD_RX_LOG(ERR, "Packet drop");
+			nb_enqueued++;
+			virtio_discard_rxbuf_inorder(vq, rxm);
+			rxvq->stats.errors++;
+			continue;
+		}
+
+		header = (struct virtio_net_hdr_mrg_rxbuf *)
+			 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM
+			 - hdr_size);
+		seg_num = header->num_buffers;
+
+		if (seg_num == 0)
+			seg_num = 1;
+
+		rxm->data_off = RTE_PKTMBUF_HEADROOM;
+		rxm->nb_segs = seg_num;
+		rxm->ol_flags = 0;
+		rxm->vlan_tci = 0;
+		rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
+		rxm->data_len = (uint16_t)(len[i] - hdr_size);
+
+		rxm->port = rxvq->port_id;
+
+		rx_pkts[nb_rx] = rxm;
+		prev = rxm;
+
+		if (vq->hw->has_rx_offload &&
+				virtio_rx_offload(rxm, &header->hdr) < 0) {
+			virtio_discard_rxbuf_inorder(vq, rxm);
+			rxvq->stats.errors++;
+			continue;
+		}
+
+		if (hw->vlan_strip)
+			rte_vlan_strip(rx_pkts[nb_rx]);
+
+		seg_res = seg_num - 1;
+
+		/* Merge remaining segments */
+		while (seg_res != 0 && i < (num - 1)) {
+			i++;
+
+			rxm = rcv_pkts[i];
+			rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
+			rxm->pkt_len = (uint32_t)(len[i]);
+			rxm->data_len = (uint16_t)(len[i]);
+
+			rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
+			rx_pkts[nb_rx]->data_len += (uint16_t)(len[i]);
+
+			if (prev)
+				prev->next = rxm;
+
+			prev = rxm;
+			seg_res -= 1;
+		}
+
+		if (!seg_res) {
+			virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
+			nb_rx++;
+		}
+	}
+
+	/* Last packet still need merge segments */
+	while (seg_res != 0) {
+		uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
+					VIRTIO_MBUF_BURST_SZ);
+
+		prev = rcv_pkts[nb_rx];
+		if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) {
+			num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len,
+							   rcv_cnt);
+			uint16_t extra_idx = 0;
+
+			rcv_cnt = num;
+			while (extra_idx < rcv_cnt) {
+				rxm = rcv_pkts[extra_idx];
+				rxm->data_off =
+					RTE_PKTMBUF_HEADROOM - hdr_size;
+				rxm->pkt_len = (uint32_t)(len[extra_idx]);
+				rxm->data_len = (uint16_t)(len[extra_idx]);
+				prev->next = rxm;
+				prev = rxm;
+				rx_pkts[nb_rx]->pkt_len += len[extra_idx];
+				rx_pkts[nb_rx]->data_len += len[extra_idx];
+				extra_idx += 1;
+			};
+			seg_res -= rcv_cnt;
+
+			if (!seg_res) {
+				virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
+				nb_rx++;
+			}
+		} else {
+			PMD_RX_LOG(ERR,
+					"No enough segments for packet.");
+			virtio_discard_rxbuf_inorder(vq, prev);
+			rxvq->stats.errors++;
+			break;
+		}
+	}
+
+	rxvq->stats.packets += nb_rx;
+
+	/* Allocate new mbuf for the used descriptor */
+
+	if (likely(!virtqueue_full(vq))) {
+		/* free_cnt may include mrg descs */
+		uint16_t free_cnt = vq->vq_free_cnt;
+		struct rte_mbuf *new_pkts[free_cnt];
+
+		if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
+			error = virtqueue_enqueue_refill_inorder(vq, new_pkts,
+					free_cnt);
+			if (unlikely(error)) {
+				for (i = 0; i < free_cnt; i++)
+					rte_pktmbuf_free(new_pkts[i]);
+			}
+			nb_enqueued += free_cnt;
+		} else {
+			struct rte_eth_dev *dev =
+				&rte_eth_devices[rxvq->port_id];
+			dev->data->rx_mbuf_alloc_failed += free_cnt;
+		}
+	}
+
+	if (likely(nb_enqueued)) {
+		vq_update_avail_idx(vq);
+
+		if (unlikely(virtqueue_kick_prepare(vq))) {
+			virtqueue_notify(vq);
+			PMD_RX_LOG(DEBUG, "Notified");
+		}
+	}
+
+	return nb_rx;
+}
+
+uint16_t
 virtio_recv_mergeable_pkts(void *rx_queue,
 			struct rte_mbuf **rx_pkts,
 			uint16_t nb_pkts)
@@ -825,7 +1194,6 @@ virtio_recv_mergeable_pkts(void *rx_queue,
 	uint16_t extra_idx;
 	uint32_t seg_res;
 	uint32_t hdr_size;
-	int offload;
 
 	nb_rx = 0;
 	if (unlikely(hw->started == 0))
@@ -843,7 +1211,6 @@ virtio_recv_mergeable_pkts(void *rx_queue,
 	extra_idx = 0;
 	seg_res = 0;
 	hdr_size = hw->vtnet_hdr_size;
-	offload = rx_offload_enabled(hw);
 
 	while (i < nb_used) {
 		struct virtio_net_hdr_mrg_rxbuf *header;
@@ -888,7 +1255,8 @@ virtio_recv_mergeable_pkts(void *rx_queue,
 		rx_pkts[nb_rx] = rxm;
 		prev = rxm;
 
-		if (offload && virtio_rx_offload(rxm, &header->hdr) < 0) {
+		if (hw->has_rx_offload &&
+				virtio_rx_offload(rxm, &header->hdr) < 0) {
 			virtio_discard_rxbuf(vq, rxm);
 			rxvq->stats.errors++;
 			continue;
@@ -950,7 +1318,6 @@ virtio_recv_mergeable_pkts(void *rx_queue,
 	rxvq->stats.packets += nb_rx;
 
 	/* Allocate new mbuf for the used descriptor */
-	error = ENOSPC;
 	while (likely(!virtqueue_full(vq))) {
 		new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool);
 		if (unlikely(new_mbuf == NULL)) {
@@ -1053,12 +1420,124 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		}
 
 		/* Enqueue Packet buffers */
-		virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, can_push);
+		virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect,
+			can_push, 0);
+
+		txvq->stats.bytes += txm->pkt_len;
+		virtio_update_packet_stats(&txvq->stats, txm);
+	}
+
+	txvq->stats.packets += nb_tx;
+
+	if (likely(nb_tx)) {
+		vq_update_avail_idx(vq);
+
+		if (unlikely(virtqueue_kick_prepare(vq))) {
+			virtqueue_notify(vq);
+			PMD_TX_LOG(DEBUG, "Notified backend after xmit");
+		}
+	}
+
+	return nb_tx;
+}
+
+uint16_t
+virtio_xmit_pkts_inorder(void *tx_queue,
+			struct rte_mbuf **tx_pkts,
+			uint16_t nb_pkts)
+{
+	struct virtnet_tx *txvq = tx_queue;
+	struct virtqueue *vq = txvq->vq;
+	struct virtio_hw *hw = vq->hw;
+	uint16_t hdr_size = hw->vtnet_hdr_size;
+	uint16_t nb_used, nb_avail, nb_tx = 0, nb_inorder_pkts = 0;
+	struct rte_mbuf *inorder_pkts[nb_pkts];
+	int error;
+
+	if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
+		return nb_tx;
+
+	if (unlikely(nb_pkts < 1))
+		return nb_pkts;
+
+	VIRTQUEUE_DUMP(vq);
+	PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
+	nb_used = VIRTQUEUE_NUSED(vq);
+
+	virtio_rmb();
+	if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
+		virtio_xmit_cleanup_inorder(vq, nb_used);
+
+	if (unlikely(!vq->vq_free_cnt))
+		virtio_xmit_cleanup_inorder(vq, nb_used);
+
+	nb_avail = RTE_MIN(vq->vq_free_cnt, nb_pkts);
+
+	for (nb_tx = 0; nb_tx < nb_avail; nb_tx++) {
+		struct rte_mbuf *txm = tx_pkts[nb_tx];
+		int slots, need;
+
+		/* Do VLAN tag insertion */
+		if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) {
+			error = rte_vlan_insert(&txm);
+			if (unlikely(error)) {
+				rte_pktmbuf_free(txm);
+				continue;
+			}
+		}
+
+		/* optimize ring usage */
+		if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
+		     vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) &&
+		     rte_mbuf_refcnt_read(txm) == 1 &&
+		     RTE_MBUF_DIRECT(txm) &&
+		     txm->nb_segs == 1 &&
+		     rte_pktmbuf_headroom(txm) >= hdr_size &&
+		     rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
+				__alignof__(struct virtio_net_hdr_mrg_rxbuf))) {
+			inorder_pkts[nb_inorder_pkts] = txm;
+			nb_inorder_pkts++;
+
+			txvq->stats.bytes += txm->pkt_len;
+			virtio_update_packet_stats(&txvq->stats, txm);
+			continue;
+		}
+
+		if (nb_inorder_pkts) {
+			virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts,
+							nb_inorder_pkts);
+			nb_inorder_pkts = 0;
+		}
+
+		slots = txm->nb_segs + 1;
+		need = slots - vq->vq_free_cnt;
+		if (unlikely(need > 0)) {
+			nb_used = VIRTQUEUE_NUSED(vq);
+			virtio_rmb();
+			need = RTE_MIN(need, (int)nb_used);
+
+			virtio_xmit_cleanup_inorder(vq, need);
+
+			need = slots - vq->vq_free_cnt;
+
+			if (unlikely(need > 0)) {
+				PMD_TX_LOG(ERR,
+					"No free tx descriptors to transmit");
+				break;
+			}
+		}
+		/* Enqueue Packet buffers */
+		virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1);
 
 		txvq->stats.bytes += txm->pkt_len;
 		virtio_update_packet_stats(&txvq->stats, txm);
 	}
 
+	/* Transmit all inorder packets */
+	if (nb_inorder_pkts)
+		virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts,
+						nb_inorder_pkts);
+
 	txvq->stats.packets += nb_tx;
 
 	if (likely(nb_tx)) {
@@ -1070,5 +1549,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		}
 	}
 
+	VIRTQUEUE_DUMP(vq);
+
 	return nb_tx;
 }
diff --git a/drivers/net/virtio/virtio_rxtx_simple.c b/drivers/net/virtio/virtio_rxtx_simple.c
index 51520758..31e565b4 100644
--- a/drivers/net/virtio/virtio_rxtx_simple.c
+++ b/drivers/net/virtio/virtio_rxtx_simple.c
@@ -27,73 +27,6 @@
 #pragma GCC diagnostic ignored "-Wcast-qual"
 #endif
 
-uint16_t
-virtio_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
-	uint16_t nb_pkts)
-{
-	struct virtnet_tx *txvq = tx_queue;
-	struct virtqueue *vq = txvq->vq;
-	struct virtio_hw *hw = vq->hw;
-	uint16_t nb_used;
-	uint16_t desc_idx;
-	struct vring_desc *start_dp;
-	uint16_t nb_tail, nb_commit;
-	int i;
-	uint16_t desc_idx_max = (vq->vq_nentries >> 1) - 1;
-	uint16_t nb_tx = 0;
-
-	if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
-		return nb_tx;
-
-	nb_used = VIRTQUEUE_NUSED(vq);
-	rte_compiler_barrier();
-
-	if (nb_used >= VIRTIO_TX_FREE_THRESH)
-		virtio_xmit_cleanup_simple(vq);
-
-	nb_commit = nb_pkts = RTE_MIN((vq->vq_free_cnt >> 1), nb_pkts);
-	desc_idx = (uint16_t)(vq->vq_avail_idx & desc_idx_max);
-	start_dp = vq->vq_ring.desc;
-	nb_tail = (uint16_t) (desc_idx_max + 1 - desc_idx);
-
-	if (nb_commit >= nb_tail) {
-		for (i = 0; i < nb_tail; i++)
-			vq->vq_descx[desc_idx + i].cookie = tx_pkts[i];
-		for (i = 0; i < nb_tail; i++) {
-			start_dp[desc_idx].addr =
-				VIRTIO_MBUF_DATA_DMA_ADDR(*tx_pkts, vq);
-			start_dp[desc_idx].len = (*tx_pkts)->pkt_len;
-			tx_pkts++;
-			desc_idx++;
-		}
-		nb_commit -= nb_tail;
-		desc_idx = 0;
-	}
-	for (i = 0; i < nb_commit; i++)
-		vq->vq_descx[desc_idx + i].cookie = tx_pkts[i];
-	for (i = 0; i < nb_commit; i++) {
-		start_dp[desc_idx].addr =
-			VIRTIO_MBUF_DATA_DMA_ADDR(*tx_pkts, vq);
-		start_dp[desc_idx].len = (*tx_pkts)->pkt_len;
-		tx_pkts++;
-		desc_idx++;
-	}
-
-	rte_compiler_barrier();
-
-	vq->vq_free_cnt -= (uint16_t)(nb_pkts << 1);
-	vq->vq_avail_idx += nb_pkts;
-	vq->vq_ring.avail->idx = vq->vq_avail_idx;
-	txvq->stats.packets += nb_pkts;
-
-	if (likely(nb_pkts)) {
-		if (unlikely(virtqueue_kick_prepare(vq)))
-			virtqueue_notify(vq);
-	}
-
-	return nb_pkts;
-}
-
 int __attribute__((cold))
 virtio_rxq_vec_setup(struct virtnet_rx *rxq)
 {
diff --git a/drivers/net/virtio/virtio_rxtx_simple.h b/drivers/net/virtio/virtio_rxtx_simple.h
index 303904d6..dc97e4cc 100644
--- a/drivers/net/virtio/virtio_rxtx_simple.h
+++ b/drivers/net/virtio/virtio_rxtx_simple.h
@@ -55,53 +55,4 @@ virtio_rxq_rearm_vec(struct virtnet_rx *rxvq)
 	vq_update_avail_idx(vq);
 }
 
-#define VIRTIO_TX_FREE_THRESH 32
-#define VIRTIO_TX_MAX_FREE_BUF_SZ 32
-#define VIRTIO_TX_FREE_NR 32
-/* TODO: vq->tx_free_cnt could mean num of free slots so we could avoid shift */
-static inline void
-virtio_xmit_cleanup_simple(struct virtqueue *vq)
-{
-	uint16_t i, desc_idx;
-	uint32_t nb_free = 0;
-	struct rte_mbuf *m, *free[VIRTIO_TX_MAX_FREE_BUF_SZ];
-
-	desc_idx = (uint16_t)(vq->vq_used_cons_idx &
-		   ((vq->vq_nentries >> 1) - 1));
-	m = (struct rte_mbuf *)vq->vq_descx[desc_idx++].cookie;
-	m = rte_pktmbuf_prefree_seg(m);
-	if (likely(m != NULL)) {
-		free[0] = m;
-		nb_free = 1;
-		for (i = 1; i < VIRTIO_TX_FREE_NR; i++) {
-			m = (struct rte_mbuf *)vq->vq_descx[desc_idx++].cookie;
-			m = rte_pktmbuf_prefree_seg(m);
-			if (likely(m != NULL)) {
-				if (likely(m->pool == free[0]->pool))
-					free[nb_free++] = m;
-				else {
-					rte_mempool_put_bulk(free[0]->pool,
-						(void **)free,
-						RTE_MIN(RTE_DIM(free),
-							nb_free));
-					free[0] = m;
-					nb_free = 1;
-				}
-			}
-		}
-		rte_mempool_put_bulk(free[0]->pool, (void **)free,
-			RTE_MIN(RTE_DIM(free), nb_free));
-	} else {
-		for (i = 1; i < VIRTIO_TX_FREE_NR; i++) {
-			m = (struct rte_mbuf *)vq->vq_descx[desc_idx++].cookie;
-			m = rte_pktmbuf_prefree_seg(m);
-			if (m != NULL)
-				rte_mempool_put(m->pool, m);
-		}
-	}
-
-	vq->vq_used_cons_idx += VIRTIO_TX_FREE_NR;
-	vq->vq_free_cnt += (VIRTIO_TX_FREE_NR << 1);
-}
-
 #endif /* _VIRTIO_RXTX_SIMPLE_H_ */
diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c b/drivers/net/virtio/virtio_user/virtio_user_dev.c
index 4322527f..7df600b0 100644
--- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
+++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
@@ -371,11 +371,13 @@ virtio_user_dev_setup(struct virtio_user_dev *dev)
 	 1ULL << VIRTIO_NET_F_GUEST_CSUM	|	\
 	 1ULL << VIRTIO_NET_F_GUEST_TSO4	|	\
 	 1ULL << VIRTIO_NET_F_GUEST_TSO6	|	\
+	 1ULL << VIRTIO_F_IN_ORDER		|	\
 	 1ULL << VIRTIO_F_VERSION_1)
 
 int
 virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues,
-		     int cq, int queue_size, const char *mac, char **ifname)
+		     int cq, int queue_size, const char *mac, char **ifname,
+		     int mrg_rxbuf, int in_order)
 {
 	pthread_mutex_init(&dev->mutex, NULL);
 	snprintf(dev->path, PATH_MAX, "%s", path);
@@ -384,6 +386,7 @@ virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues,
 	dev->queue_pairs = 1; /* mq disabled by default */
 	dev->queue_size = queue_size;
 	dev->mac_specified = 0;
+	dev->unsupported_features = 0;
 	parse_mac(dev, mac);
 
 	if (*ifname) {
@@ -419,10 +422,22 @@ virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues,
 		dev->device_features = VIRTIO_USER_SUPPORTED_FEATURES;
 	}
 
-	if (dev->mac_specified)
+	if (!mrg_rxbuf) {
+		dev->device_features &= ~(1ull << VIRTIO_NET_F_MRG_RXBUF);
+		dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF);
+	}
+
+	if (!in_order) {
+		dev->device_features &= ~(1ull << VIRTIO_F_IN_ORDER);
+		dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER);
+	}
+
+	if (dev->mac_specified) {
 		dev->device_features |= (1ull << VIRTIO_NET_F_MAC);
-	else
+	} else {
 		dev->device_features &= ~(1ull << VIRTIO_NET_F_MAC);
+		dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC);
+	}
 
 	if (cq) {
 		/* device does not really need to know anything about CQ,
@@ -437,6 +452,14 @@ virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues,
 		dev->device_features &= ~(1ull << VIRTIO_NET_F_GUEST_ANNOUNCE);
 		dev->device_features &= ~(1ull << VIRTIO_NET_F_MQ);
 		dev->device_features &= ~(1ull << VIRTIO_NET_F_CTRL_MAC_ADDR);
+		dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ);
+		dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX);
+		dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN);
+		dev->unsupported_features |=
+			(1ull << VIRTIO_NET_F_GUEST_ANNOUNCE);
+		dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ);
+		dev->unsupported_features |=
+			(1ull << VIRTIO_NET_F_CTRL_MAC_ADDR);
 	}
 
 	/* The backend will not report this feature, we add it explicitly */
@@ -444,6 +467,7 @@ virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues,
 		dev->device_features |= (1ull << VIRTIO_NET_F_STATUS);
 
 	dev->device_features &= VIRTIO_USER_SUPPORTED_FEATURES;
+	dev->unsupported_features |= ~VIRTIO_USER_SUPPORTED_FEATURES;
 
 	if (rte_mem_event_callback_register(VIRTIO_USER_MEM_EVENT_CLB_NAME,
 				virtio_user_mem_event_cb, dev)) {
diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.h b/drivers/net/virtio/virtio_user/virtio_user_dev.h
index d2d4cb82..d6e0e137 100644
--- a/drivers/net/virtio/virtio_user/virtio_user_dev.h
+++ b/drivers/net/virtio/virtio_user/virtio_user_dev.h
@@ -33,6 +33,7 @@ struct virtio_user_dev {
 				   * and will be sync with device
 				   */
 	uint64_t	device_features; /* supported features by device */
+	uint64_t	unsupported_features; /* unsupported features mask */
 	uint8_t		status;
 	uint16_t	port_id;
 	uint8_t		mac_addr[ETHER_ADDR_LEN];
@@ -47,7 +48,8 @@ int is_vhost_user_by_type(const char *path);
 int virtio_user_start_device(struct virtio_user_dev *dev);
 int virtio_user_stop_device(struct virtio_user_dev *dev);
 int virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues,
-			 int cq, int queue_size, const char *mac, char **ifname);
+			 int cq, int queue_size, const char *mac, char **ifname,
+			 int mrg_rxbuf, int in_order);
 void virtio_user_dev_uninit(struct virtio_user_dev *dev);
 void virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx);
 uint8_t virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs);
diff --git a/drivers/net/virtio/virtio_user_ethdev.c b/drivers/net/virtio/virtio_user_ethdev.c
index 1c102ca7..525d16ca 100644
--- a/drivers/net/virtio/virtio_user_ethdev.c
+++ b/drivers/net/virtio/virtio_user_ethdev.c
@@ -30,7 +30,6 @@ virtio_user_server_reconnect(struct virtio_user_dev *dev)
 	int ret;
 	int flag;
 	int connectfd;
-	uint64_t features = dev->device_features;
 	struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id];
 
 	connectfd = accept(dev->listenfd, NULL, NULL);
@@ -45,15 +44,8 @@ virtio_user_server_reconnect(struct virtio_user_dev *dev)
 		return -1;
 	}
 
-	features &= ~dev->device_features;
-	/* For following bits, vhost-user doesn't really need to know */
-	features &= ~(1ull << VIRTIO_NET_F_MAC);
-	features &= ~(1ull << VIRTIO_NET_F_CTRL_VLAN);
-	features &= ~(1ull << VIRTIO_NET_F_CTRL_MAC_ADDR);
-	features &= ~(1ull << VIRTIO_NET_F_STATUS);
-	if (features)
-		PMD_INIT_LOG(ERR, "WARNING: Some features 0x%" PRIx64 " are not supported by vhost-user!",
-			     features);
+	/* umask vhost-user unsupported features */
+	dev->device_features &= ~(dev->unsupported_features);
 
 	dev->features &= dev->device_features;
 
@@ -366,8 +358,12 @@ static const char *valid_args[] = {
 	VIRTIO_USER_ARG_QUEUE_SIZE,
 #define VIRTIO_USER_ARG_INTERFACE_NAME "iface"
 	VIRTIO_USER_ARG_INTERFACE_NAME,
-#define VIRTIO_USER_ARG_SERVER_MODE "server"
+#define VIRTIO_USER_ARG_SERVER_MODE    "server"
 	VIRTIO_USER_ARG_SERVER_MODE,
+#define VIRTIO_USER_ARG_MRG_RXBUF      "mrg_rxbuf"
+	VIRTIO_USER_ARG_MRG_RXBUF,
+#define VIRTIO_USER_ARG_IN_ORDER       "in_order"
+	VIRTIO_USER_ARG_IN_ORDER,
 	NULL
 };
 
@@ -440,7 +436,8 @@ virtio_user_eth_dev_alloc(struct rte_vdev_device *vdev)
 	hw->use_msix = 1;
 	hw->modern   = 0;
 	hw->use_simple_rx = 0;
-	hw->use_simple_tx = 0;
+	hw->use_inorder_rx = 0;
+	hw->use_inorder_tx = 0;
 	hw->virtio_user_dev = dev;
 	return eth_dev;
 }
@@ -470,6 +467,8 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
 	uint64_t cq = VIRTIO_USER_DEF_CQ_EN;
 	uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ;
 	uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE;
+	uint64_t mrg_rxbuf = 1;
+	uint64_t in_order = 1;
 	char *path = NULL;
 	char *ifname = NULL;
 	char *mac_addr = NULL;
@@ -569,6 +568,24 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
 		goto end;
 	}
 
+	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MRG_RXBUF) == 1) {
+		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MRG_RXBUF,
+				       &get_integer_arg, &mrg_rxbuf) < 0) {
+			PMD_INIT_LOG(ERR, "error to parse %s",
+				     VIRTIO_USER_ARG_MRG_RXBUF);
+			goto end;
+		}
+	}
+
+	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_IN_ORDER) == 1) {
+		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_IN_ORDER,
+				       &get_integer_arg, &in_order) < 0) {
+			PMD_INIT_LOG(ERR, "error to parse %s",
+				     VIRTIO_USER_ARG_IN_ORDER);
+			goto end;
+		}
+	}
+
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 		struct virtio_user_dev *vu_dev;
 
@@ -585,7 +602,8 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
 		else
 			vu_dev->is_server = false;
 		if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq,
-				 queue_size, mac_addr, &ifname) < 0) {
+				 queue_size, mac_addr, &ifname, mrg_rxbuf,
+				 in_order) < 0) {
 			PMD_INIT_LOG(ERR, "virtio_user_dev_init fails");
 			virtio_user_eth_dev_free(eth_dev);
 			goto end;
@@ -663,4 +681,7 @@ RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user,
 	"cq=<int> "
 	"queue_size=<int> "
 	"queues=<int> "
-	"iface=<string>");
+	"iface=<string> "
+	"server=<0|1> "
+	"mrg_rxbuf=<0|1> "
+	"in_order=<0|1>");
diff --git a/drivers/net/virtio/virtqueue.c b/drivers/net/virtio/virtqueue.c
index a7d0a9cb..56a77cc7 100644
--- a/drivers/net/virtio/virtqueue.c
+++ b/drivers/net/virtio/virtqueue.c
@@ -74,6 +74,14 @@ virtqueue_rxvq_flush(struct virtqueue *vq)
 			desc_idx = used_idx;
 			rte_pktmbuf_free(vq->sw_ring[desc_idx]);
 			vq->vq_free_cnt++;
+		} else if (hw->use_inorder_rx) {
+			desc_idx = (uint16_t)uep->id;
+			dxp = &vq->vq_descx[desc_idx];
+			if (dxp->cookie != NULL) {
+				rte_pktmbuf_free(dxp->cookie);
+				dxp->cookie = NULL;
+			}
+			vq_ring_free_inorder(vq, desc_idx, 1);
 		} else {
 			desc_idx = (uint16_t)uep->id;
 			dxp = &vq->vq_descx[desc_idx];
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 14364f35..26518ed9 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -306,6 +306,8 @@ virtio_get_queue_type(struct virtio_hw *hw, uint16_t vtpci_queue_idx)
 #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx))
 
 void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx);
+void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx,
+			  uint16_t num);
 
 static inline void
 vq_update_avail_idx(struct virtqueue *vq)
diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.c b/drivers/net/vmxnet3/vmxnet3_ethdev.c
index ba932ff2..2613cd13 100644
--- a/drivers/net/vmxnet3/vmxnet3_ethdev.c
+++ b/drivers/net/vmxnet3/vmxnet3_ethdev.c
@@ -57,7 +57,8 @@
 	 DEV_RX_OFFLOAD_UDP_CKSUM |	\
 	 DEV_RX_OFFLOAD_TCP_CKSUM |	\
 	 DEV_RX_OFFLOAD_TCP_LRO |	\
-	 DEV_RX_OFFLOAD_JUMBO_FRAME)
+	 DEV_RX_OFFLOAD_JUMBO_FRAME |   \
+	 DEV_RX_OFFLOAD_CRC_STRIP)
 
 static int eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev);
 static int eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev);
@@ -1053,7 +1054,6 @@ vmxnet3_dev_info_get(struct rte_eth_dev *dev __rte_unused,
 	dev_info->speed_capa = ETH_LINK_SPEED_10G;
 	dev_info->max_mac_addrs = VMXNET3_MAX_MAC_ADDRS;
 
-	dev_info->default_txconf.txq_flags = ETH_TXQ_FLAGS_NOXSUMSCTP;
 	dev_info->flow_type_rss_offloads = VMXNET3_RSS_OFFLOAD_ALL;
 
 	dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
@@ -1320,9 +1320,7 @@ RTE_PMD_REGISTER_PCI(net_vmxnet3, rte_vmxnet3_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_vmxnet3, pci_id_vmxnet3_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_vmxnet3, "* igb_uio | uio_pci_generic | vfio-pci");
 
-RTE_INIT(vmxnet3_init_log);
-static void
-vmxnet3_init_log(void)
+RTE_INIT(vmxnet3_init_log)
 {
 	vmxnet3_logtype_init = rte_log_register("pmd.net.vmxnet3.init");
 	if (vmxnet3_logtype_init >= 0)
diff --git a/drivers/raw/dpaa2_cmdif/dpaa2_cmdif.c b/drivers/raw/dpaa2_cmdif/dpaa2_cmdif.c
index 095a34b2..469960a3 100644
--- a/drivers/raw/dpaa2_cmdif/dpaa2_cmdif.c
+++ b/drivers/raw/dpaa2_cmdif/dpaa2_cmdif.c
@@ -289,10 +289,7 @@ static struct rte_vdev_driver dpaa2_cmdif_drv = {
 
 RTE_PMD_REGISTER_VDEV(DPAA2_CMDIF_PMD_NAME, dpaa2_cmdif_drv);
 
-RTE_INIT(dpaa2_cmdif_init_log);
-
-static void
-dpaa2_cmdif_init_log(void)
+RTE_INIT(dpaa2_cmdif_init_log)
 {
 	dpaa2_cmdif_logtype = rte_log_register("pmd.raw.dpaa2.cmdif");
 	if (dpaa2_cmdif_logtype >= 0)
diff --git a/drivers/raw/dpaa2_cmdif/dpaa2_cmdif_logs.h b/drivers/raw/dpaa2_cmdif/dpaa2_cmdif_logs.h
index 598a621c..8991e832 100644
--- a/drivers/raw/dpaa2_cmdif/dpaa2_cmdif_logs.h
+++ b/drivers/raw/dpaa2_cmdif/dpaa2_cmdif_logs.h
@@ -19,7 +19,7 @@ extern int dpaa2_cmdif_logtype;
 	rte_log(RTE_LOG_DEBUG, dpaa2_cmdif_logtype, "dpaa2_cmdif: %s(): " \
 		fmt "\n", __func__, ## args)
 
-#define DPAA2_CMDIF_FUNC_TRACE() DPAA2_CMDIF_LOG(DEBUG, ">>")
+#define DPAA2_CMDIF_FUNC_TRACE() DPAA2_CMDIF_DEBUG(">>")
 
 #define DPAA2_CMDIF_INFO(fmt, args...) \
 	DPAA2_CMDIF_LOG(INFO, fmt, ## args)
diff --git a/drivers/raw/dpaa2_cmdif/meson.build b/drivers/raw/dpaa2_cmdif/meson.build
index 8c909438..1d146872 100644
--- a/drivers/raw/dpaa2_cmdif/meson.build
+++ b/drivers/raw/dpaa2_cmdif/meson.build
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright 2018 NXP
 
+build = dpdk_conf.has('RTE_LIBRTE_DPAA2_MEMPOOL')
 deps += ['rawdev', 'mempool_dpaa2', 'bus_vdev']
 sources = files('dpaa2_cmdif.c')
 
diff --git a/drivers/raw/dpaa2_qdma/dpaa2_qdma.c b/drivers/raw/dpaa2_qdma/dpaa2_qdma.c
index 1d15c302..2787d302 100644
--- a/drivers/raw/dpaa2_qdma/dpaa2_qdma.c
+++ b/drivers/raw/dpaa2_qdma/dpaa2_qdma.c
@@ -985,6 +985,7 @@ rte_dpaa2_qdma_remove(struct rte_dpaa2_device *dpaa2_dev)
 }
 
 static struct rte_dpaa2_driver rte_dpaa2_qdma_pmd = {
+	.drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
 	.drv_type = DPAA2_QDMA,
 	.probe = rte_dpaa2_qdma_probe,
 	.remove = rte_dpaa2_qdma_remove,
@@ -992,9 +993,7 @@ static struct rte_dpaa2_driver rte_dpaa2_qdma_pmd = {
 
 RTE_PMD_REGISTER_DPAA2(dpaa2_qdma, rte_dpaa2_qdma_pmd);
 
-RTE_INIT(dpaa2_qdma_init_log);
-static void
-dpaa2_qdma_init_log(void)
+RTE_INIT(dpaa2_qdma_init_log)
 {
 	dpaa2_qdma_logtype = rte_log_register("pmd.raw.dpaa2.qdma");
 	if (dpaa2_qdma_logtype >= 0)
diff --git a/drivers/raw/dpaa2_qdma/dpaa2_qdma_logs.h b/drivers/raw/dpaa2_qdma/dpaa2_qdma_logs.h
index fafe352b..4779e4ce 100644
--- a/drivers/raw/dpaa2_qdma/dpaa2_qdma_logs.h
+++ b/drivers/raw/dpaa2_qdma/dpaa2_qdma_logs.h
@@ -19,7 +19,7 @@ extern int dpaa2_qdma_logtype;
 	rte_log(RTE_LOG_DEBUG, dpaa2_qdma_logtype, "dpaa2_qdma: %s(): " \
 		fmt "\n", __func__, ## args)
 
-#define DPAA2_QDMA_FUNC_TRACE() DPAA2_QDMA_LOG(DEBUG, ">>")
+#define DPAA2_QDMA_FUNC_TRACE() DPAA2_QDMA_DEBUG(">>")
 
 #define DPAA2_QDMA_INFO(fmt, args...) \
 	DPAA2_QDMA_LOG(INFO, fmt, ## args)
diff --git a/drivers/raw/dpaa2_qdma/meson.build b/drivers/raw/dpaa2_qdma/meson.build
index a2eb1d2f..b6a081f1 100644
--- a/drivers/raw/dpaa2_qdma/meson.build
+++ b/drivers/raw/dpaa2_qdma/meson.build
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright 2018 NXP
 
+build = dpdk_conf.has('RTE_LIBRTE_DPAA2_MEMPOOL')
 deps += ['rawdev', 'mempool_dpaa2', 'ring']
 sources = files('dpaa2_qdma.c')
 
diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
index 030ed1b6..3fed0578 100644
--- a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
@@ -517,9 +517,7 @@ RTE_PMD_REGISTER_PCI(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd);
 RTE_PMD_REGISTER_KMOD_DEP(ifpga_rawdev_pci_driver, "* igb_uio | uio_pci_generic | vfio-pci");
 
-RTE_INIT(ifpga_rawdev_init_log);
-static void
-ifpga_rawdev_init_log(void)
+RTE_INIT(ifpga_rawdev_init_log)
 {
 	ifpga_rawdev_logtype = rte_log_register("driver.raw.init");
 	if (ifpga_rawdev_logtype >= 0)
diff --git a/drivers/raw/skeleton_rawdev/Makefile b/drivers/raw/skeleton_rawdev/Makefile
index bacc66dd..3f97c2ee 100644
--- a/drivers/raw/skeleton_rawdev/Makefile
+++ b/drivers/raw/skeleton_rawdev/Makefile
@@ -8,7 +8,6 @@ include $(RTE_SDK)/mk/rte.vars.mk
 #
 LIB = librte_pmd_skeleton_rawdev.a
 
-CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 LDLIBS += -lrte_eal
diff --git a/drivers/raw/skeleton_rawdev/meson.build b/drivers/raw/skeleton_rawdev/meson.build
index 7cb2d3fb..b4a6ed08 100644
--- a/drivers/raw/skeleton_rawdev/meson.build
+++ b/drivers/raw/skeleton_rawdev/meson.build
@@ -4,5 +4,3 @@
 deps += ['rawdev', 'kvargs', 'mbuf', 'bus_vdev']
 sources = files('skeleton_rawdev.c',
                'skeleton_rawdev_test.c')
-
-allow_experimental_apis = true
diff --git a/drivers/raw/skeleton_rawdev/skeleton_rawdev.c b/drivers/raw/skeleton_rawdev/skeleton_rawdev.c
index 6bdbbb50..6518a2d9 100644
--- a/drivers/raw/skeleton_rawdev/skeleton_rawdev.c
+++ b/drivers/raw/skeleton_rawdev/skeleton_rawdev.c
@@ -305,6 +305,18 @@ static int skeleton_rawdev_queue_release(struct rte_rawdev *dev,
 	return ret;
 }
 
+static uint16_t skeleton_rawdev_queue_count(struct rte_rawdev *dev)
+{
+	struct skeleton_rawdev *skeldev;
+
+	SKELETON_PMD_FUNC_TRACE();
+
+	RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
+
+	skeldev = skeleton_rawdev_get_priv(dev);
+	return skeldev->num_queues;
+}
+
 static int skeleton_rawdev_get_attr(struct rte_rawdev *dev,
 				    const char *attr_name,
 				    uint64_t *attr_value)
@@ -524,6 +536,7 @@ static const struct rte_rawdev_ops skeleton_rawdev_ops = {
 	.queue_def_conf = skeleton_rawdev_queue_def_conf,
 	.queue_setup = skeleton_rawdev_queue_setup,
 	.queue_release = skeleton_rawdev_queue_release,
+	.queue_count = skeleton_rawdev_queue_count,
 
 	.attr_get = skeleton_rawdev_get_attr,
 	.attr_set = skeleton_rawdev_set_attr,
@@ -744,10 +757,7 @@ static struct rte_vdev_driver skeleton_pmd_drv = {
 
 RTE_PMD_REGISTER_VDEV(SKELETON_PMD_RAWDEV_NAME, skeleton_pmd_drv);
 
-RTE_INIT(skeleton_pmd_init_log);
-
-static void
-skeleton_pmd_init_log(void)
+RTE_INIT(skeleton_pmd_init_log)
 {
 	skeleton_pmd_logtype = rte_log_register("rawdev.skeleton");
 	if (skeleton_pmd_logtype >= 0)
diff --git a/drivers/raw/skeleton_rawdev/skeleton_rawdev_test.c b/drivers/raw/skeleton_rawdev/skeleton_rawdev_test.c
index 3eb5c3a7..3405b898 100644
--- a/drivers/raw/skeleton_rawdev/skeleton_rawdev_test.c
+++ b/drivers/raw/skeleton_rawdev/skeleton_rawdev_test.c
@@ -194,6 +194,18 @@ test_rawdev_queue_default_conf_get(void)
 }
 
 static int
+test_rawdev_queue_count(void)
+{
+	unsigned int q_count;
+
+	/* Get the current configuration */
+	q_count = rte_rawdev_queue_count(TEST_DEV_ID);
+	RTE_TEST_ASSERT_EQUAL(q_count, 1, "Invalid queue count (%d)", q_count);
+
+	return TEST_SUCCESS;
+}
+
+static int
 test_rawdev_queue_setup(void)
 {
 	int ret;
@@ -429,6 +441,7 @@ test_rawdev_skeldev(void)
 	SKELDEV_TEST_RUN(test_rawdev_configure, NULL,
 			 test_rawdev_queue_default_conf_get);
 	SKELDEV_TEST_RUN(test_rawdev_configure, NULL, test_rawdev_queue_setup);
+	SKELDEV_TEST_RUN(NULL, NULL, test_rawdev_queue_count);
 	SKELDEV_TEST_RUN(test_rawdev_queue_setup, NULL,
 			 test_rawdev_queue_release);
 	SKELDEV_TEST_RUN(NULL, NULL, test_rawdev_attr_set_get);
diff --git a/examples/bbdev_app/main.c b/examples/bbdev_app/main.c
index 254cc067..045a190b 100644
--- a/examples/bbdev_app/main.c
+++ b/examples/bbdev_app/main.c
@@ -64,11 +64,7 @@ static const struct rte_eth_conf port_conf = {
 		.mq_mode = ETH_MQ_RX_NONE,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.header_split = 0, /**< Header Split disabled */
-		.hw_ip_checksum = 0, /**< IP checksum offload disabled */
-		.hw_vlan_filter = 0, /**< VLAN filtering disabled */
-		.jumbo_frame = 0, /**< Jumbo Frame Support disabled */
-		.hw_strip_crc = 0, /**< CRC stripped by hardware */
+		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/examples/bond/main.c b/examples/bond/main.c
index 65e0edd2..23d0981a 100644
--- a/examples/bond/main.c
+++ b/examples/bond/main.c
@@ -122,7 +122,6 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode = ETH_MQ_RX_NONE,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.rx_adv_conf = {
@@ -154,6 +153,18 @@ slave_port_init(uint16_t portid, struct rte_mempool *mbuf_pool)
 	if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 		local_port_conf.txmode.offloads |=
 			DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+	local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+		dev_info.flow_type_rss_offloads;
+	if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+			port_conf.rx_adv_conf.rss_conf.rss_hf) {
+		printf("Port %u modified RSS hash function based on hardware support,"
+			"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+			portid,
+			port_conf.rx_adv_conf.rss_conf.rss_hf,
+			local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+	}
+
 	retval = rte_eth_dev_configure(portid, 1, 1, &local_port_conf);
 	if (retval != 0)
 		rte_exit(EXIT_FAILURE, "port %u: configuration failed (res=%d)\n",
@@ -177,7 +188,6 @@ slave_port_init(uint16_t portid, struct rte_mempool *mbuf_pool)
 
 	/* TX setup */
 	txq_conf = dev_info.default_txconf;
-	txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txq_conf.offloads = local_port_conf.txmode.offloads;
 	retval = rte_eth_tx_queue_setup(portid, 0, nb_txd,
 				rte_eth_dev_socket_id(portid), &txq_conf);
@@ -246,7 +256,6 @@ bond_port_init(struct rte_mempool *mbuf_pool)
 
 	/* TX setup */
 	txq_conf = dev_info.default_txconf;
-	txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txq_conf.offloads = local_port_conf.txmode.offloads;
 	retval = rte_eth_tx_queue_setup(BOND_PORT, 0, nb_txd,
 				rte_eth_dev_socket_id(BOND_PORT), &txq_conf);
diff --git a/examples/distributor/main.c b/examples/distributor/main.c
index 2c593648..03a05e3d 100644
--- a/examples/distributor/main.c
+++ b/examples/distributor/main.c
@@ -80,7 +80,6 @@ static const struct rte_eth_conf port_conf_default = {
 	.rxmode = {
 		.mq_mode = ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
-		.ignore_offload_bitfield = 1,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
@@ -124,6 +123,17 @@ port_init(uint16_t port, struct rte_mempool *mbuf_pool)
 		port_conf.txmode.offloads |=
 			DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 
+	port_conf.rx_adv_conf.rss_conf.rss_hf &=
+		dev_info.flow_type_rss_offloads;
+	if (port_conf.rx_adv_conf.rss_conf.rss_hf !=
+			port_conf_default.rx_adv_conf.rss_conf.rss_hf) {
+		printf("Port %u modified RSS hash function based on hardware support,"
+			"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+			port,
+			port_conf_default.rx_adv_conf.rss_conf.rss_hf,
+			port_conf.rx_adv_conf.rss_conf.rss_hf);
+	}
+
 	retval = rte_eth_dev_configure(port, rxRings, txRings, &port_conf);
 	if (retval != 0)
 		return retval;
@@ -141,7 +151,6 @@ port_init(uint16_t port, struct rte_mempool *mbuf_pool)
 	}
 
 	txconf = dev_info.default_txconf;
-	txconf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txconf.offloads = port_conf.txmode.offloads;
 	for (q = 0; q < txRings; q++) {
 		retval = rte_eth_tx_queue_setup(port, q, nb_txd,
diff --git a/examples/ethtool/Makefile b/examples/ethtool/Makefile
index 2b40b4b6..3d9d4f06 100644
--- a/examples/ethtool/Makefile
+++ b/examples/ethtool/Makefile
@@ -10,7 +10,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(info This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 else
diff --git a/examples/ethtool/ethtool-app/main.c b/examples/ethtool/ethtool-app/main.c
index c1815bb9..dc93adfe 100644
--- a/examples/ethtool/ethtool-app/main.c
+++ b/examples/ethtool/ethtool-app/main.c
@@ -99,7 +99,6 @@ static void setup_ports(struct app_config *app_cfg, int cnt_ports)
 
 	memset(&cfg_port, 0, sizeof(cfg_port));
 	cfg_port.txmode.mq_mode = ETH_MQ_TX_NONE;
-	cfg_port.rxmode.ignore_offload_bitfield = 1;
 
 	for (idx_port = 0; idx_port < cnt_ports; idx_port++) {
 		struct app_port *ptr_port = &app_cfg->ports[idx_port];
@@ -142,7 +141,6 @@ static void setup_ports(struct app_config *app_cfg, int cnt_ports)
 				 "rte_eth_rx_queue_setup failed"
 				);
 		txconf = dev_info.default_txconf;
-		txconf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 		if (rte_eth_tx_queue_setup(
 			    idx_port, 0, nb_txd,
 			    rte_eth_dev_socket_id(idx_port), &txconf) < 0)
diff --git a/examples/ethtool/lib/Makefile b/examples/ethtool/lib/Makefile
index 2576910f..6eaa640b 100644
--- a/examples/ethtool/lib/Makefile
+++ b/examples/ethtool/lib/Makefile
@@ -10,7 +10,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(error This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 endif
diff --git a/examples/eventdev_pipeline/main.c b/examples/eventdev_pipeline/main.c
index b698e4ca..700bc696 100644
--- a/examples/eventdev_pipeline/main.c
+++ b/examples/eventdev_pipeline/main.c
@@ -267,7 +267,6 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool)
 		.rxmode = {
 			.mq_mode = ETH_MQ_RX_RSS,
 			.max_rx_pkt_len = ETHER_MAX_LEN,
-			.ignore_offload_bitfield = 1,
 		},
 		.rx_adv_conf = {
 			.rss_conf = {
@@ -293,6 +292,17 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool)
 		port_conf.txmode.offloads |=
 			DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 
+	port_conf.rx_adv_conf.rss_conf.rss_hf &=
+		dev_info.flow_type_rss_offloads;
+	if (port_conf.rx_adv_conf.rss_conf.rss_hf !=
+			port_conf_default.rx_adv_conf.rss_conf.rss_hf) {
+		printf("Port %u modified RSS hash function based on hardware support,"
+			"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+			port,
+			port_conf_default.rx_adv_conf.rss_conf.rss_hf,
+			port_conf.rx_adv_conf.rss_conf.rss_hf);
+	}
+
 	/* Configure the Ethernet device. */
 	retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
 	if (retval != 0)
@@ -307,7 +317,6 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool)
 	}
 
 	txconf = dev_info.default_txconf;
-	txconf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txconf.offloads = port_conf_default.txmode.offloads;
 	/* Allocate and set up 1 TX queue per Ethernet port. */
 	for (q = 0; q < tx_rings; q++) {
diff --git a/examples/exception_path/main.c b/examples/exception_path/main.c
index 2b381a5d..440422bc 100644
--- a/examples/exception_path/main.c
+++ b/examples/exception_path/main.c
@@ -88,7 +88,6 @@
 /* Options for configuring ethernet port */
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
-		.ignore_offload_bitfield = 1,
 		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
@@ -132,6 +131,9 @@ print_stats(void)
 	       " Lcore    Port            RX            TX    Dropped on TX\n"
 	       "-------  ------  ------------  ------------  ---------------\n");
 	RTE_LCORE_FOREACH(i) {
+		/* limit ourselves to application supported cores only */
+		if (i >= APP_MAX_LCORE)
+			break;
 		printf("%6u %7u %13"PRIu64" %13"PRIu64" %16"PRIu64"\n",
 		       i, (unsigned)port_ids[i],
 		       lcore_stats[i].rx, lcore_stats[i].tx,
@@ -457,7 +459,6 @@ init_port(uint16_t port)
 				port, ret);
 
 	txq_conf = dev_info.default_txconf;
-	txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txq_conf.offloads = local_port_conf.txmode.offloads;
 	ret = rte_eth_tx_queue_setup(port, 0, nb_txd,
 				rte_eth_dev_socket_id(port),
diff --git a/examples/flow_classify/flow_classify.c b/examples/flow_classify/flow_classify.c
index 6412fe44..19961292 100644
--- a/examples/flow_classify/flow_classify.c
+++ b/examples/flow_classify/flow_classify.c
@@ -62,7 +62,6 @@ const char cb_port_delim[] = ":";
 static const struct rte_eth_conf port_conf_default = {
 	.rxmode = {
 		.max_rx_pkt_len = ETHER_MAX_LEN,
-		.ignore_offload_bitfield = 1,
 	},
 };
 
@@ -222,7 +221,6 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool)
 	}
 
 	txconf = dev_info.default_txconf;
-	txconf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txconf.offloads = port_conf.txmode.offloads;
 	/* Allocate and set up 1 TX queue per Ethernet port. */
 	for (q = 0; q < tx_rings; q++) {
diff --git a/examples/flow_filtering/main.c b/examples/flow_filtering/main.c
index e0ee5167..ce91e8a6 100644
--- a/examples/flow_filtering/main.c
+++ b/examples/flow_filtering/main.c
@@ -121,7 +121,6 @@ init_port(void)
 	struct rte_eth_conf port_conf = {
 		.rxmode = {
 			.split_hdr_size = 0,
-			.ignore_offload_bitfield = 1,
 			.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 		},
 		.txmode = {
@@ -133,6 +132,22 @@ init_port(void)
 				DEV_TX_OFFLOAD_SCTP_CKSUM  |
 				DEV_TX_OFFLOAD_TCP_TSO,
 		},
+		/*
+		 * Initialize fdir_conf of rte_eth_conf.
+		 * Fdir is used in flow filtering for I40e,
+		 * so rte_flow rules involve some fdir
+		 * configurations. In long term it's better
+		 * that drivers don't require any fdir
+		 * configuration for rte_flow, but we need to
+		 * get this workaround so that sample app can
+		 * run on I40e.
+		 */
+		.fdir_conf = {
+			.mode = RTE_FDIR_MODE_PERFECT,
+			.pballoc = RTE_FDIR_PBALLOC_64K,
+			.status = RTE_FDIR_REPORT_STATUS,
+			.drop_queue = 127,
+		},
 	};
 	struct rte_eth_txconf txq_conf;
 	struct rte_eth_rxconf rxq_conf;
diff --git a/examples/ip_fragmentation/main.c b/examples/ip_fragmentation/main.c
index 8952ea45..5306d767 100644
--- a/examples/ip_fragmentation/main.c
+++ b/examples/ip_fragmentation/main.c
@@ -140,7 +140,6 @@ static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.max_rx_pkt_len = JUMBO_FRAME_MAX_SIZE,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
 			     DEV_RX_OFFLOAD_JUMBO_FRAME |
 			     DEV_RX_OFFLOAD_CRC_STRIP),
@@ -973,7 +972,6 @@ main(int argc, char **argv)
 			fflush(stdout);
 
 			txconf = &dev_info.default_txconf;
-			txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 			txconf->offloads = local_port_conf.txmode.offloads;
 			ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
 						     socket, txconf);
diff --git a/examples/ip_pipeline/Makefile b/examples/ip_pipeline/Makefile
index 11d2b35d..3fb98ce3 100644
--- a/examples/ip_pipeline/Makefile
+++ b/examples/ip_pipeline/Makefile
@@ -67,7 +67,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(info This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 all:
diff --git a/examples/ip_pipeline/cli.c b/examples/ip_pipeline/cli.c
index c9587f56..102a1d6b 100644
--- a/examples/ip_pipeline/cli.c
+++ b/examples/ip_pipeline/cli.c
@@ -9,6 +9,7 @@
 
 #include <rte_common.h>
 #include <rte_cycles.h>
+#include <rte_ethdev.h>
 
 #include "cli.h"
 #include "kni.h"
@@ -48,13 +49,13 @@ is_comment(char *in)
 	return 0;
 }
 
-/**
- * mempool <mempool_name>
- *  buffer <buffer_size>
- *  pool <pool_size>
- *  cache <cache_size>
- *  cpu <cpu_id>
- */
+static const char cmd_mempool_help[] =
+"mempool <mempool_name>\n"
+"   buffer <buffer_size>\n"
+"   pool <pool_size>\n"
+"   cache <cache_size>\n"
+"   cpu <cpu_id>\n";
+
 static void
 cmd_mempool(char **tokens,
 	uint32_t n_tokens,
@@ -119,14 +120,14 @@ cmd_mempool(char **tokens,
 	}
 }
 
-/**
- * link <link_name>
- *  dev <device_name> | port <port_id>
- *  rxq <n_queues> <queue_size> <mempool_name>
- *  txq <n_queues> <queue_size>
- *  promiscuous on | off
- *  [rss <qid_0> ... <qid_n>]
- */
+static const char cmd_link_help[] =
+"link <link_name>\n"
+"   dev <device_name> | port <port_id>\n"
+"   rxq <n_queues> <queue_size> <mempool_name>\n"
+"   txq <n_queues> <queue_size>\n"
+"   promiscuous on | off\n"
+"   [rss <qid_0> ... <qid_n>]\n";
+
 static void
 cmd_link(char **tokens,
 	uint32_t n_tokens,
@@ -237,12 +238,97 @@ cmd_link(char **tokens,
 	}
 }
 
-/**
- * swq <swq_name>
- *  size <size>
- *  cpu <cpu_id>
+/* Print the link stats and info */
+static void
+print_link_info(struct link *link, char *out, size_t out_size)
+{
+	struct rte_eth_stats stats;
+	struct ether_addr mac_addr;
+	struct rte_eth_link eth_link;
+	uint16_t mtu;
+
+	memset(&stats, 0, sizeof(stats));
+	rte_eth_stats_get(link->port_id, &stats);
+
+	rte_eth_macaddr_get(link->port_id, &mac_addr);
+	rte_eth_link_get(link->port_id, &eth_link);
+	rte_eth_dev_get_mtu(link->port_id, &mtu);
+
+	snprintf(out, out_size,
+		"\n"
+		"%s: flags=<%s> mtu %u\n"
+		"\tether %02X:%02X:%02X:%02X:%02X:%02X rxqueues %u txqueues %u\n"
+		"\tport# %u  speed %u Mbps\n"
+		"\tRX packets %" PRIu64"  bytes %" PRIu64"\n"
+		"\tRX errors %" PRIu64"  missed %" PRIu64"  no-mbuf %" PRIu64"\n"
+		"\tTX packets %" PRIu64"  bytes %" PRIu64"\n"
+		"\tTX errors %" PRIu64"\n",
+		link->name,
+		eth_link.link_status == 0 ? "DOWN" : "UP",
+		mtu,
+		mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
+		mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
+		mac_addr.addr_bytes[4], mac_addr.addr_bytes[5],
+		link->n_rxq,
+		link->n_txq,
+		link->port_id,
+		eth_link.link_speed,
+		stats.ipackets,
+		stats.ibytes,
+		stats.ierrors,
+		stats.imissed,
+		stats.rx_nombuf,
+		stats.opackets,
+		stats.obytes,
+		stats.oerrors);
+}
+
+/*
+ * link show [<link_name>]
  */
 static void
+cmd_link_show(char **tokens, uint32_t n_tokens, char *out, size_t out_size)
+{
+	struct link *link;
+	char *link_name;
+
+	if (n_tokens != 2 && n_tokens != 3) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	if (n_tokens == 2) {
+		link = link_next(NULL);
+
+		while (link != NULL) {
+			out_size = out_size - strlen(out);
+			out = &out[strlen(out)];
+
+			print_link_info(link, out, out_size);
+			link = link_next(link);
+		}
+	} else {
+		out_size = out_size - strlen(out);
+		out = &out[strlen(out)];
+
+		link_name = tokens[2];
+		link = link_find(link_name);
+
+		if (link == NULL) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+					"Link does not exist");
+			return;
+		}
+		print_link_info(link, out, out_size);
+	}
+}
+
+static const char cmd_swq_help[] =
+"swq <swq_name>\n"
+"   size <size>\n"
+"   cpu <cpu_id>\n";
+
+static void
 cmd_swq(char **tokens,
 	uint32_t n_tokens,
 	char *out,
@@ -286,12 +372,12 @@ cmd_swq(char **tokens,
 	}
 }
 
-/**
- * tmgr subport profile
- *  <tb_rate> <tb_size>
- *  <tc0_rate> <tc1_rate> <tc2_rate> <tc3_rate>
- *  <tc_period>
- */
+static const char cmd_tmgr_subport_profile_help[] =
+"tmgr subport profile\n"
+"   <tb_rate> <tb_size>\n"
+"   <tc0_rate> <tc1_rate> <tc2_rate> <tc3_rate>\n"
+"   <tc_period>\n";
+
 static void
 cmd_tmgr_subport_profile(char **tokens,
 	uint32_t n_tokens,
@@ -334,14 +420,14 @@ cmd_tmgr_subport_profile(char **tokens,
 	}
 }
 
-/**
- * tmgr pipe profile
- *  <tb_rate> <tb_size>
- *  <tc0_rate> <tc1_rate> <tc2_rate> <tc3_rate>
- *  <tc_period>
- *  <tc_ov_weight>
- *  <wrr_weight0..15>
- */
+static const char cmd_tmgr_pipe_profile_help[] =
+"tmgr pipe profile\n"
+"   <tb_rate> <tb_size>\n"
+"   <tc0_rate> <tc1_rate> <tc2_rate> <tc3_rate>\n"
+"   <tc_period>\n"
+"   <tc_ov_weight>\n"
+"   <wrr_weight0..15>\n";
+
 static void
 cmd_tmgr_pipe_profile(char **tokens,
 	uint32_t n_tokens,
@@ -397,16 +483,16 @@ cmd_tmgr_pipe_profile(char **tokens,
 	}
 }
 
-/**
- * tmgr <tmgr_name>
- *  rate <rate>
- *  spp <n_subports_per_port>
- *  pps <n_pipes_per_subport>
- *  qsize <qsize_tc0> <qsize_tc1> <qsize_tc2> <qsize_tc3>
- *  fo <frame_overhead>
- *  mtu <mtu>
- *  cpu <cpu_id>
- */
+static const char cmd_tmgr_help[] =
+"tmgr <tmgr_name>\n"
+"   rate <rate>\n"
+"   spp <n_subports_per_port>\n"
+"   pps <n_pipes_per_subport>\n"
+"   qsize <qsize_tc0> <qsize_tc1> <qsize_tc2> <qsize_tc3>\n"
+"   fo <frame_overhead>\n"
+"   mtu <mtu>\n"
+"   cpu <cpu_id>\n";
+
 static void
 cmd_tmgr(char **tokens,
 	uint32_t n_tokens,
@@ -503,10 +589,10 @@ cmd_tmgr(char **tokens,
 	}
 }
 
-/**
- * tmgr <tmgr_name> subport <subport_id>
- *  profile <subport_profile_id>
- */
+static const char cmd_tmgr_subport_help[] =
+"tmgr <tmgr_name> subport <subport_id>\n"
+"   profile <subport_profile_id>\n";
+
 static void
 cmd_tmgr_subport(char **tokens,
 	uint32_t n_tokens,
@@ -541,11 +627,12 @@ cmd_tmgr_subport(char **tokens,
 	}
 }
 
-/**
- * tmgr <tmgr_name> subport <subport_id> pipe
- *  from <pipe_id_first> to <pipe_id_last>
- *  profile <pipe_profile_id>
- */
+
+static const char cmd_tmgr_subport_pipe_help[] =
+"tmgr <tmgr_name> subport <subport_id> pipe\n"
+"   from <pipe_id_first> to <pipe_id_last>\n"
+"   profile <pipe_profile_id>\n";
+
 static void
 cmd_tmgr_subport_pipe(char **tokens,
 	uint32_t n_tokens,
@@ -611,9 +698,10 @@ cmd_tmgr_subport_pipe(char **tokens,
 	}
 }
 
-/**
- * tap <tap_name>
- */
+
+static const char cmd_tap_help[] =
+"tap <tap_name>\n";
+
 static void
 cmd_tap(char **tokens,
 	uint32_t n_tokens,
@@ -637,12 +725,12 @@ cmd_tap(char **tokens,
 	}
 }
 
-/**
- * kni <kni_name>
- *  link <link_name>
- *  mempool <mempool_name>
- *  [thread <thread_id>]
- */
+static const char cmd_kni_help[] =
+"kni <kni_name>\n"
+"   link <link_name>\n"
+"   mempool <mempool_name>\n"
+"   [thread <thread_id>]\n";
+
 static void
 cmd_kni(char **tokens,
 	uint32_t n_tokens,
@@ -697,11 +785,12 @@ cmd_kni(char **tokens,
 	}
 }
 
-/**
- * port in action profile <profile_name>
- *  [filter match | mismatch offset <key_offset> mask <key_mask> key <key_value> port <port_id>]
- *  [balance offset <key_offset> mask <key_mask> port <port_id0> ... <port_id15>]
- */
+
+static const char cmd_port_in_action_profile_help[] =
+"port in action profile <profile_name>\n"
+"   [filter match | mismatch offset <key_offset> mask <key_mask> key <key_value> port <port_id>]\n"
+"   [balance offset <key_offset> mask <key_mask> port <port_id0> ... <port_id15>]\n";
+
 static void
 cmd_port_in_action_profile(char **tokens,
 	uint32_t n_tokens,
@@ -861,24 +950,25 @@ cmd_port_in_action_profile(char **tokens,
 	}
 }
 
-/**
- * table action profile <profile_name>
- *  ipv4 | ipv6
- *  offset <ip_offset>
- *  fwd
- *  [balance offset <key_offset> mask <key_mask> outoffset <out_offset>]
- *  [meter srtcm | trtcm
- *      tc <n_tc>
- *      stats none | pkts | bytes | both]
- *  [tm spp <n_subports_per_port> pps <n_pipes_per_subport>]
- *  [encap ether | vlan | qinq | mpls | pppoe]
- *  [nat src | dst
- *      proto udp | tcp]
- *  [ttl drop | fwd
- *      stats none | pkts]
- *  [stats pkts | bytes | both]
- *  [time]
- */
+
+static const char cmd_table_action_profile_help[] =
+"table action profile <profile_name>\n"
+"   ipv4 | ipv6\n"
+"   offset <ip_offset>\n"
+"   fwd\n"
+"   [balance offset <key_offset> mask <key_mask> outoffset <out_offset>]\n"
+"   [meter srtcm | trtcm\n"
+"       tc <n_tc>\n"
+"       stats none | pkts | bytes | both]\n"
+"   [tm spp <n_subports_per_port> pps <n_pipes_per_subport>]\n"
+"   [encap ether | vlan | qinq | mpls | pppoe]\n"
+"   [nat src | dst\n"
+"       proto udp | tcp]\n"
+"   [ttl drop | fwd\n"
+"       stats none | pkts]\n"
+"   [stats pkts | bytes | both]\n"
+"   [time]\n";
+
 static void
 cmd_table_action_profile(char **tokens,
 	uint32_t n_tokens,
@@ -1207,12 +1297,12 @@ cmd_table_action_profile(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name>
- *  period <timer_period_ms>
- *  offset_port_id <offset_port_id>
- *  cpu <cpu_id>
- */
+static const char cmd_pipeline_help[] =
+"pipeline <pipeline_name>\n"
+"   period <timer_period_ms>\n"
+"   offset_port_id <offset_port_id>\n"
+"   cpu <cpu_id>\n";
+
 static void
 cmd_pipeline(char **tokens,
 	uint32_t n_tokens,
@@ -1267,18 +1357,18 @@ cmd_pipeline(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> port in
- *  bsz <burst_size>
- *  link <link_name> rxq <queue_id>
- *  | swq <swq_name>
- *  | tmgr <tmgr_name>
- *  | tap <tap_name> mempool <mempool_name> mtu <mtu>
- *  | kni <kni_name>
- *  | source mempool <mempool_name> file <file_name> bpp <n_bytes_per_pkt>
- *  [action <port_in_action_profile_name>]
- *  [disabled]
- */
+static const char cmd_pipeline_port_in_help[] =
+"pipeline <pipeline_name> port in\n"
+"   bsz <burst_size>\n"
+"   link <link_name> rxq <queue_id>\n"
+"   | swq <swq_name>\n"
+"   | tmgr <tmgr_name>\n"
+"   | tap <tap_name> mempool <mempool_name> mtu <mtu>\n"
+"   | kni <kni_name>\n"
+"   | source mempool <mempool_name> file <file_name> bpp <n_bytes_per_pkt>\n"
+"   [action <port_in_action_profile_name>]\n"
+"   [disabled]\n";
+
 static void
 cmd_pipeline_port_in(char **tokens,
 	uint32_t n_tokens,
@@ -1486,16 +1576,16 @@ cmd_pipeline_port_in(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> port out
- *  bsz <burst_size>
- *  link <link_name> txq <txq_id>
- *  | swq <swq_name>
- *  | tmgr <tmgr_name>
- *  | tap <tap_name>
- *  | kni <kni_name>
- *  | sink [file <file_name> pkts <max_n_pkts>]
- */
+static const char cmd_pipeline_port_out_help[] =
+"pipeline <pipeline_name> port out\n"
+"   bsz <burst_size>\n"
+"   link <link_name> txq <txq_id>\n"
+"   | swq <swq_name>\n"
+"   | tmgr <tmgr_name>\n"
+"   | tap <tap_name>\n"
+"   | kni <kni_name>\n"
+"   | sink [file <file_name> pkts <max_n_pkts>]\n";
+
 static void
 cmd_pipeline_port_out(char **tokens,
 	uint32_t n_tokens,
@@ -1640,30 +1730,30 @@ cmd_pipeline_port_out(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> table
- *      match
- *      acl
- *          ipv4 | ipv6
- *          offset <ip_header_offset>
- *          size <n_rules>
- *      | array
- *          offset <key_offset>
- *          size <n_keys>
- *      | hash
- *          ext | lru
- *          key <key_size>
- *          mask <key_mask>
- *          offset <key_offset>
- *          buckets <n_buckets>
- *          size <n_keys>
- *      | lpm
- *          ipv4 | ipv6
- *          offset <ip_header_offset>
- *          size <n_rules>
- *      | stub
- *  [action <table_action_profile_name>]
- */
+static const char cmd_pipeline_table_help[] =
+"pipeline <pipeline_name> table\n"
+"       match\n"
+"       acl\n"
+"           ipv4 | ipv6\n"
+"           offset <ip_header_offset>\n"
+"           size <n_rules>\n"
+"       | array\n"
+"           offset <key_offset>\n"
+"           size <n_keys>\n"
+"       | hash\n"
+"           ext | lru\n"
+"           key <key_size>\n"
+"           mask <key_mask>\n"
+"           offset <key_offset>\n"
+"           buckets <n_buckets>\n"
+"           size <n_keys>\n"
+"       | lpm\n"
+"           ipv4 | ipv6\n"
+"           offset <ip_header_offset>\n"
+"           size <n_rules>\n"
+"       | stub\n"
+"   [action <table_action_profile_name>]\n";
+
 static void
 cmd_pipeline_table(char **tokens,
 	uint32_t n_tokens,
@@ -1925,9 +2015,9 @@ cmd_pipeline_table(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> port in <port_id> table <table_id>
- */
+static const char cmd_pipeline_port_in_table_help[] =
+"pipeline <pipeline_name> port in <port_id> table <table_id>\n";
+
 static void
 cmd_pipeline_port_in_table(char **tokens,
 	uint32_t n_tokens,
@@ -1979,9 +2069,9 @@ cmd_pipeline_port_in_table(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> port in <port_id> stats read [clear]
- */
+
+static const char cmd_pipeline_port_in_stats_help[] =
+"pipeline <pipeline_name> port in <port_id> stats read [clear]\n";
 
 #define MSG_PIPELINE_PORT_IN_STATS                         \
 	"Pkts in: %" PRIu64 "\n"                           \
@@ -2056,9 +2146,10 @@ cmd_pipeline_port_in_stats(char **tokens,
 		stats.stats.n_pkts_drop);
 }
 
-/**
- * pipeline <pipeline_name> port in <port_id> enable
- */
+
+static const char cmd_pipeline_port_in_enable_help[] =
+"pipeline <pipeline_name> port in <port_id> enable\n";
+
 static void
 cmd_pipeline_port_in_enable(char **tokens,
 	uint32_t n_tokens,
@@ -2103,9 +2194,10 @@ cmd_pipeline_port_in_enable(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> port in <port_id> disable
- */
+
+static const char cmd_pipeline_port_in_disable_help[] =
+"pipeline <pipeline_name> port in <port_id> disable\n";
+
 static void
 cmd_pipeline_port_in_disable(char **tokens,
 	uint32_t n_tokens,
@@ -2150,9 +2242,10 @@ cmd_pipeline_port_in_disable(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> port out <port_id> stats read [clear]
- */
+
+static const char cmd_pipeline_port_out_stats_help[] =
+"pipeline <pipeline_name> port out <port_id> stats read [clear]\n";
+
 #define MSG_PIPELINE_PORT_OUT_STATS                        \
 	"Pkts in: %" PRIu64 "\n"                           \
 	"Pkts dropped by AH: %" PRIu64 "\n"                \
@@ -2226,9 +2319,10 @@ cmd_pipeline_port_out_stats(char **tokens,
 		stats.stats.n_pkts_drop);
 }
 
-/**
- * pipeline <pipeline_name> table <table_id> stats read [clear]
- */
+
+static const char cmd_pipeline_table_stats_help[] =
+"pipeline <pipeline_name> table <table_id> stats read [clear]\n";
+
 #define MSG_PIPELINE_TABLE_STATS                                     \
 	"Pkts in: %" PRIu64 "\n"                                     \
 	"Pkts in with lookup miss: %" PRIu64 "\n"                    \
@@ -3406,11 +3500,12 @@ parse_table_action(char **tokens,
 	return n_tokens0 - n_tokens;
 }
 
-/**
- * pipeline <pipeline_name> table <table_id> rule add
- *    match <match>
- *    action <table_action>
- */
+
+static const char cmd_pipeline_table_rule_add_help[] =
+"pipeline <pipeline_name> table <table_id> rule add\n"
+"     match <match>\n"
+"     action <table_action>\n";
+
 static void
 cmd_pipeline_table_rule_add(char **tokens,
 	uint32_t n_tokens,
@@ -3486,17 +3581,18 @@ cmd_pipeline_table_rule_add(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> table <table_id> rule add
- *    match
- *       default
- *    action
- *       fwd
- *          drop
- *          | port <port_id>
- *          | meta
- *          | table <table_id>
- */
+
+static const char cmd_pipeline_table_rule_add_default_help[] =
+"pipeline <pipeline_name> table <table_id> rule add\n"
+"     match\n"
+"        default\n"
+"     action\n"
+"        fwd\n"
+"           drop\n"
+"           | port <port_id>\n"
+"           | meta\n"
+"           | table <table_id>\n";
+
 static void
 cmd_pipeline_table_rule_add_default(char **tokens,
 	uint32_t n_tokens,
@@ -3618,12 +3714,13 @@ cmd_pipeline_table_rule_add_default(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> table <table_id> rule add bulk <file_name> <n_rules>
- *
- * File <file_name>:
- * - line format: match <match> action <action>
- */
+
+static const char cmd_pipeline_table_rule_add_bulk_help[] =
+"pipeline <pipeline_name> table <table_id> rule add bulk <file_name> <n_rules>\n"
+"\n"
+"  File <file_name>:\n"
+"  - line format: match <match> action <action>\n";
+
 static int
 cli_rule_file_process(const char *file_name,
 	size_t line_len_max,
@@ -3745,10 +3842,11 @@ cmd_pipeline_table_rule_add_bulk(char **tokens,
 	free(match);
 }
 
-/**
- * pipeline <pipeline_name> table <table_id> rule delete
- *    match <match>
- */
+
+static const char cmd_pipeline_table_rule_delete_help[] =
+"pipeline <pipeline_name> table <table_id> rule delete\n"
+"     match <match>\n";
+
 static void
 cmd_pipeline_table_rule_delete(char **tokens,
 	uint32_t n_tokens,
@@ -3813,11 +3911,12 @@ cmd_pipeline_table_rule_delete(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> table <table_id> rule delete
- *    match
- *       default
- */
+
+static const char cmd_pipeline_table_rule_delete_default_help[] =
+"pipeline <pipeline_name> table <table_id> rule delete\n"
+"     match\n"
+"        default\n";
+
 static void
 cmd_pipeline_table_rule_delete_default(char **tokens,
 	uint32_t n_tokens,
@@ -3873,9 +3972,10 @@ cmd_pipeline_table_rule_delete_default(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> table <table_id> rule read stats [clear]
- */
+
+static const char cmd_pipeline_table_rule_stats_read_help[] =
+"pipeline <pipeline_name> table <table_id> rule read stats [clear]\n";
+
 static void
 cmd_pipeline_table_rule_stats_read(char **tokens,
 	uint32_t n_tokens __rte_unused,
@@ -3885,11 +3985,12 @@ cmd_pipeline_table_rule_stats_read(char **tokens,
 	snprintf(out, out_size, MSG_CMD_UNIMPLEM, tokens[0]);
 }
 
-/**
- * pipeline <pipeline_name> table <table_id> meter profile <meter_profile_id>
- *  add srtcm cir <cir> cbs <cbs> ebs <ebs>
- *  | trtcm cir <cir> pir <pir> cbs <cbs> pbs <pbs>
- */
+
+static const char cmd_pipeline_table_meter_profile_add_help[] =
+"pipeline <pipeline_name> table <table_id> meter profile <meter_profile_id>\n"
+"   add srtcm cir <cir> cbs <cbs> ebs <ebs>\n"
+"   | trtcm cir <cir> pir <pir> cbs <cbs> pbs <pbs>\n";
+
 static void
 cmd_pipeline_table_meter_profile_add(char **tokens,
 	uint32_t n_tokens,
@@ -4037,10 +4138,11 @@ cmd_pipeline_table_meter_profile_add(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> table <table_id>
- *  meter profile <meter_profile_id> delete
- */
+
+static const char cmd_pipeline_table_meter_profile_delete_help[] =
+"pipeline <pipeline_name> table <table_id>\n"
+"   meter profile <meter_profile_id> delete\n";
+
 static void
 cmd_pipeline_table_meter_profile_delete(char **tokens,
 	uint32_t n_tokens,
@@ -4097,9 +4199,10 @@ cmd_pipeline_table_meter_profile_delete(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> table <table_id> rule read meter [clear]
- */
+
+static const char cmd_pipeline_table_rule_meter_read_help[] =
+"pipeline <pipeline_name> table <table_id> rule read meter [clear]\n";
+
 static void
 cmd_pipeline_table_rule_meter_read(char **tokens,
 	uint32_t n_tokens __rte_unused,
@@ -4109,13 +4212,14 @@ cmd_pipeline_table_rule_meter_read(char **tokens,
 	snprintf(out, out_size, MSG_CMD_UNIMPLEM, tokens[0]);
 }
 
-/**
- * pipeline <pipeline_name> table <table_id> dscp <file_name>
- *
- * File <file_name>:
- *  - exactly 64 lines
- *  - line format: <tc_id> <tc_queue_id> <color>, with <color> as: g | y | r
- */
+
+static const char cmd_pipeline_table_dscp_help[] =
+"pipeline <pipeline_name> table <table_id> dscp <file_name>\n"
+"\n"
+" File <file_name>:\n"
+"   - exactly 64 lines\n"
+"   - line format: <tc_id> <tc_queue_id> <color>, with <color> as: g | y | r\n";
+
 static int
 load_dscp_table(struct rte_table_action_dscp_table *dscp_table,
 	const char *file_name,
@@ -4258,9 +4362,10 @@ cmd_pipeline_table_dscp(char **tokens,
 	}
 }
 
-/**
- * pipeline <pipeline_name> table <table_id> rule read ttl [clear]
- */
+
+static const char cmd_pipeline_table_rule_ttl_read_help[] =
+"pipeline <pipeline_name> table <table_id> rule read ttl [clear]\n";
+
 static void
 cmd_pipeline_table_rule_ttl_read(char **tokens,
 	uint32_t n_tokens __rte_unused,
@@ -4270,9 +4375,10 @@ cmd_pipeline_table_rule_ttl_read(char **tokens,
 	snprintf(out, out_size, MSG_CMD_UNIMPLEM, tokens[0]);
 }
 
-/**
- * thread <thread_id> pipeline <pipeline_name> enable
- */
+
+static const char cmd_thread_pipeline_enable_help[] =
+"thread <thread_id> pipeline <pipeline_name> enable\n";
+
 static void
 cmd_thread_pipeline_enable(char **tokens,
 	uint32_t n_tokens,
@@ -4312,9 +4418,10 @@ cmd_thread_pipeline_enable(char **tokens,
 	}
 }
 
-/**
- * thread <thread_id> pipeline <pipeline_name> disable
- */
+
+static const char cmd_thread_pipeline_disable_help[] =
+"thread <thread_id> pipeline <pipeline_name> disable\n";
+
 static void
 cmd_thread_pipeline_disable(char **tokens,
 	uint32_t n_tokens,
@@ -4355,6 +4462,316 @@ cmd_thread_pipeline_disable(char **tokens,
 	}
 }
 
+static void
+cmd_help(char **tokens, uint32_t n_tokens, char *out, size_t out_size)
+{
+	tokens++;
+	n_tokens--;
+
+	if (n_tokens == 0) {
+		snprintf(out, out_size,
+			"Type 'help <command>' for details on each command.\n\n"
+			"List of commands:\n"
+			"\tmempool\n"
+			"\tlink\n"
+			"\tswq\n"
+			"\ttmgr subport profile\n"
+			"\ttmgr pipe profile\n"
+			"\ttmgr\n"
+			"\ttmgr subport\n"
+			"\ttmgr subport pipe\n"
+			"\ttap\n"
+			"\tkni\n"
+			"\tport in action profile\n"
+			"\ttable action profile\n"
+			"\tpipeline\n"
+			"\tpipeline port in\n"
+			"\tpipeline port out\n"
+			"\tpipeline table\n"
+			"\tpipeline port in table\n"
+			"\tpipeline port in stats\n"
+			"\tpipeline port in enable\n"
+			"\tpipeline port in disable\n"
+			"\tpipeline port out stats\n"
+			"\tpipeline table stats\n"
+			"\tpipeline table rule add\n"
+			"\tpipeline table rule add default\n"
+			"\tpipeline table rule add bulk\n"
+			"\tpipeline table rule delete\n"
+			"\tpipeline table rule delete default\n"
+			"\tpipeline table rule stats read\n"
+			"\tpipeline table meter profile add\n"
+			"\tpipeline table meter profile delete\n"
+			"\tpipeline table rule meter read\n"
+			"\tpipeline table dscp\n"
+			"\tpipeline table rule ttl read\n"
+			"\tthread pipeline enable\n"
+			"\tthread pipeline disable\n\n");
+		return;
+	}
+
+	if (strcmp(tokens[0], "mempool") == 0) {
+		snprintf(out, out_size, "\n%s\n", cmd_mempool_help);
+		return;
+	}
+
+	if (strcmp(tokens[0], "link") == 0) {
+		snprintf(out, out_size, "\n%s\n", cmd_link_help);
+		return;
+	}
+
+	if (strcmp(tokens[0], "swq") == 0) {
+		snprintf(out, out_size, "\n%s\n", cmd_swq_help);
+		return;
+	}
+
+	if (strcmp(tokens[0], "tmgr") == 0) {
+		if (n_tokens == 1) {
+			snprintf(out, out_size, "\n%s\n", cmd_tmgr_help);
+			return;
+		}
+
+		if ((n_tokens == 2) &&
+			(strcmp(tokens[1], "subport")) == 0) {
+			snprintf(out, out_size, "\n%s\n", cmd_tmgr_subport_help);
+			return;
+		}
+
+		if ((n_tokens == 3) &&
+			(strcmp(tokens[1], "subport") == 0) &&
+			(strcmp(tokens[2], "profile") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_tmgr_subport_profile_help);
+			return;
+		}
+
+		if ((n_tokens == 3) &&
+			(strcmp(tokens[1], "subport") == 0) &&
+			(strcmp(tokens[2], "pipe") == 0)) {
+			snprintf(out, out_size, "\n%s\n", cmd_tmgr_subport_pipe_help);
+			return;
+		}
+
+		if ((n_tokens == 3) &&
+			(strcmp(tokens[1], "pipe") == 0) &&
+			(strcmp(tokens[2], "profile") == 0)) {
+			snprintf(out, out_size, "\n%s\n", cmd_tmgr_pipe_profile_help);
+			return;
+		}
+	}
+
+	if (strcmp(tokens[0], "tap") == 0) {
+		snprintf(out, out_size, "\n%s\n", cmd_tap_help);
+		return;
+	}
+
+	if (strcmp(tokens[0], "kni") == 0) {
+		snprintf(out, out_size, "\n%s\n", cmd_kni_help);
+		return;
+	}
+
+	if ((n_tokens == 4) &&
+		(strcmp(tokens[0], "port") == 0) &&
+		(strcmp(tokens[1], "in") == 0) &&
+		(strcmp(tokens[2], "action") == 0) &&
+		(strcmp(tokens[3], "profile") == 0)) {
+		snprintf(out, out_size, "\n%s\n", cmd_port_in_action_profile_help);
+		return;
+	}
+
+	if ((n_tokens == 3) &&
+		(strcmp(tokens[0], "table") == 0) &&
+		(strcmp(tokens[1], "action") == 0) &&
+		(strcmp(tokens[2], "profile") == 0)) {
+		snprintf(out, out_size, "\n%s\n", cmd_table_action_profile_help);
+		return;
+	}
+
+	if ((strcmp(tokens[0], "pipeline") == 0) && (n_tokens == 1)) {
+		snprintf(out, out_size, "\n%s\n", cmd_pipeline_help);
+		return;
+	}
+
+	if ((strcmp(tokens[0], "pipeline") == 0) &&
+		(strcmp(tokens[1], "port") == 0)) {
+		if ((n_tokens == 3) && (strcmp(tokens[2], "in")) == 0) {
+			snprintf(out, out_size, "\n%s\n", cmd_pipeline_port_in_help);
+			return;
+		}
+
+		if ((n_tokens == 3) && (strcmp(tokens[2], "out")) == 0) {
+			snprintf(out, out_size, "\n%s\n", cmd_pipeline_port_out_help);
+			return;
+		}
+
+		if ((n_tokens == 4) &&
+			(strcmp(tokens[2], "in") == 0) &&
+			(strcmp(tokens[3], "table") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_port_in_table_help);
+			return;
+		}
+
+		if ((n_tokens == 4) &&
+			(strcmp(tokens[2], "in") == 0) &&
+			(strcmp(tokens[3], "stats") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_port_in_stats_help);
+			return;
+		}
+
+		if ((n_tokens == 4) &&
+			(strcmp(tokens[2], "in") == 0) &&
+			(strcmp(tokens[3], "enable") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_port_in_enable_help);
+			return;
+		}
+
+		if ((n_tokens == 4) &&
+			(strcmp(tokens[2], "in") == 0) &&
+			(strcmp(tokens[3], "disable") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_port_in_disable_help);
+			return;
+		}
+
+		if ((n_tokens == 4) &&
+			(strcmp(tokens[2], "out") == 0) &&
+			(strcmp(tokens[3], "stats") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_port_out_stats_help);
+			return;
+		}
+	}
+
+	if ((strcmp(tokens[0], "pipeline") == 0) &&
+		(strcmp(tokens[1], "table") == 0)) {
+		if (n_tokens == 2) {
+			snprintf(out, out_size, "\n%s\n", cmd_pipeline_table_help);
+			return;
+		}
+
+		if ((n_tokens == 3) && strcmp(tokens[2], "stats") == 0) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_stats_help);
+			return;
+		}
+
+		if ((n_tokens == 3) && strcmp(tokens[2], "dscp") == 0) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_dscp_help);
+			return;
+		}
+
+		if ((n_tokens == 4) &&
+			(strcmp(tokens[2], "rule") == 0) &&
+			(strcmp(tokens[3], "add") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_rule_add_help);
+			return;
+		}
+
+		if ((n_tokens == 5) &&
+			(strcmp(tokens[2], "rule") == 0) &&
+			(strcmp(tokens[3], "add") == 0) &&
+			(strcmp(tokens[4], "default") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_rule_add_default_help);
+			return;
+		}
+
+		if ((n_tokens == 5) &&
+			(strcmp(tokens[2], "rule") == 0) &&
+			(strcmp(tokens[3], "add") == 0) &&
+			(strcmp(tokens[4], "bulk") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_rule_add_bulk_help);
+			return;
+		}
+
+		if ((n_tokens == 4) &&
+			(strcmp(tokens[2], "rule") == 0) &&
+			(strcmp(tokens[3], "delete") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_rule_delete_help);
+			return;
+		}
+
+		if ((n_tokens == 5) &&
+			(strcmp(tokens[2], "rule") == 0) &&
+			(strcmp(tokens[3], "delete") == 0) &&
+			(strcmp(tokens[4], "default") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_rule_delete_default_help);
+			return;
+		}
+
+		if ((n_tokens == 5) &&
+			(strcmp(tokens[2], "rule") == 0) &&
+			(strcmp(tokens[3], "stats") == 0) &&
+			(strcmp(tokens[4], "read") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_rule_stats_read_help);
+			return;
+		}
+
+		if ((n_tokens == 5) &&
+			(strcmp(tokens[2], "meter") == 0) &&
+			(strcmp(tokens[3], "profile") == 0) &&
+			(strcmp(tokens[4], "add") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_meter_profile_add_help);
+			return;
+		}
+
+		if ((n_tokens == 5) &&
+			(strcmp(tokens[2], "meter") == 0) &&
+			(strcmp(tokens[3], "profile") == 0) &&
+			(strcmp(tokens[4], "delete") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_meter_profile_delete_help);
+			return;
+		}
+
+		if ((n_tokens == 5) &&
+			(strcmp(tokens[2], "rule") == 0) &&
+			(strcmp(tokens[3], "meter") == 0) &&
+			(strcmp(tokens[4], "read") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_rule_meter_read_help);
+			return;
+		}
+
+		if ((n_tokens == 5) &&
+			(strcmp(tokens[2], "rule") == 0) &&
+			(strcmp(tokens[3], "ttl") == 0) &&
+			(strcmp(tokens[4], "read") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_rule_ttl_read_help);
+			return;
+		}
+	}
+
+	if ((n_tokens == 3) &&
+		(strcmp(tokens[0], "thread") == 0) &&
+		(strcmp(tokens[1], "pipeline") == 0)) {
+		if (strcmp(tokens[2], "enable") == 0) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_thread_pipeline_enable_help);
+			return;
+		}
+
+		if (strcmp(tokens[2], "disable") == 0) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_thread_pipeline_disable_help);
+			return;
+		}
+	}
+
+	snprintf(out, out_size, "Invalid command\n");
+}
+
 void
 cli_process(char *in, char *out, size_t out_size)
 {
@@ -4374,12 +4791,22 @@ cli_process(char *in, char *out, size_t out_size)
 	if (n_tokens == 0)
 		return;
 
+	if (strcmp(tokens[0], "help") == 0) {
+		cmd_help(tokens, n_tokens, out, out_size);
+		return;
+	}
+
 	if (strcmp(tokens[0], "mempool") == 0) {
 		cmd_mempool(tokens, n_tokens, out, out_size);
 		return;
 	}
 
 	if (strcmp(tokens[0], "link") == 0) {
+		if (strcmp(tokens[1], "show") == 0) {
+			cmd_link_show(tokens, n_tokens, out, out_size);
+			return;
+		}
+
 		cmd_link(tokens, n_tokens, out, out_size);
 		return;
 	}
diff --git a/examples/ip_pipeline/examples/rss.cli b/examples/ip_pipeline/examples/rss.cli
new file mode 100644
index 00000000..29c0a913
--- /dev/null
+++ b/examples/ip_pipeline/examples/rss.cli
@@ -0,0 +1,112 @@
+; SPDX-License-Identifier: BSD-3-Clause
+; Copyright(c) 2010-2018 Intel Corporation
+
+; This setup demonstrates the usage of NIC Receive Side Scaling (RSS) feature.
+; Each NIC splits the input traffic into 4 RX queues, with each of its RX queues
+; being handled by a different pipeline:
+;
+;                                                    +-----------+    +----------+
+;                       +--------------------------->|           |    |          |
+;                       |       +------------------->| PIPELINE0 +--->|  LINK 0  |--->
+;                       |       |     +------------->| (CORE A)  |    |    TX    |
+;                       |       |     |     +------->|           |    |          |
+;                       |       |     |     |        +-----------+    +----------+
+;    +----------+       |       |     |     |
+;    |          |-------+       |     |     |
+;--->|  LINK 0  |-----------+   |     |     |
+;    |    RX    |---------+ |   |     |     |
+;    |          |-------+ | |   |     |     |
+;    +----------+       | | |   |     |     |        +-----------+    +----------+
+;                       | | +---|-----|-----|------->|           |    |          |
+;    +----------+       | |     | +---|-----|------->| PIPELINE1 +--->|  LINK 1  |--->
+;    |          |-------|-|-----+ |   | +---|------->| (CORE B)  |    |    TX    |
+;--->|  LINK 1  |-------|-|-------+   | |   | +----->|           |    |          |
+;    |    RX    |-------|-|-------+   | |   | |      +-----------+    +----------+
+;    |          |-------|-|-----+ |   | |   | |
+;    +----------+       | |     | |   | |   | |
+;                       | |     | |   | |   | |
+;    +----------+       | |     | |   | |   | |
+;    |          |-------|-|-----|-|---+ |   | |
+;--->|  LINK 2  |-------|-|-----|-|-----+   | |      +-----------+    +----------+
+;    |    RX    |-----+ | +-----|-|---------|-|----->|           |    |          |
+;    |          |---+ | |       | +---------|-|----->| PIPELINE2 +--->|  LINK 2  |--->
+;    +----------+   | +-|-------|-----------|-|----->| (CORE C)  |    |    TX    |
+;                   |   |       |           | | +--->|           |    |          |
+;    +----------+   |   |       |           | | |    +-----------+    +----------+
+;    |          |---|---|-------|-----------+ | |
+;--->|  LINK 3  |---|---|-------|-------------+ |
+;    |    RX    |---|---|-------|---------------+
+;    |          |---|---|-------|-----------+
+;    +----------+   |   |       |           |
+;                   |   |       |           |        +-----------+    +----------+
+;                   |   +-------|-----------|------->|           |    |          |
+;                   |           +-----------|------->| PIPELINE3 +--->|  LINK 3  |--->
+;                   +-----------------------|------->| (CORE D)  |    |    TX    |
+;                                           +------->|           |    |          |
+;                                                    +-----------+    +----------+
+;
+;
+
+mempool MEMPOOL0 buffer 2304 pool 32K cache 256 cpu 0
+
+link LINK0 dev 0000:02:00.0 rxq 4 128 MEMPOOL0 txq 1 512 promiscuous on rss 0 1 2 3
+link LINK1 dev 0000:02:00.1 rxq 4 128 MEMPOOL0 txq 1 512 promiscuous on rss 0 1 2 3
+link LINK2 dev 0000:06:00.0 rxq 4 128 MEMPOOL0 txq 1 512 promiscuous on rss 0 1 2 3
+link LINK3 dev 0000:06:00.1 rxq 4 128 MEMPOOL0 txq 1 512 promiscuous on rss 0 1 2 3
+
+pipeline PIPELINE0 period 10 offset_port_id 0 cpu 0
+pipeline PIPELINE0 port in bsz 32 link LINK0 rxq 0
+pipeline PIPELINE0 port in bsz 32 link LINK1 rxq 0
+pipeline PIPELINE0 port in bsz 32 link LINK2 rxq 0
+pipeline PIPELINE0 port in bsz 32 link LINK3 rxq 0
+pipeline PIPELINE0 port out bsz 32 link LINK0 txq 0
+pipeline PIPELINE0 table match stub
+pipeline PIPELINE0 port in 0 table 0
+pipeline PIPELINE0 port in 1 table 0
+pipeline PIPELINE0 port in 2 table 0
+pipeline PIPELINE0 port in 3 table 0
+pipeline PIPELINE0 table 0 rule add match default action fwd port 0
+
+pipeline PIPELINE1 period 10 offset_port_id 0 cpu 0
+pipeline PIPELINE1 port in bsz 32 link LINK0 rxq 1
+pipeline PIPELINE1 port in bsz 32 link LINK1 rxq 1
+pipeline PIPELINE1 port in bsz 32 link LINK2 rxq 1
+pipeline PIPELINE1 port in bsz 32 link LINK3 rxq 1
+pipeline PIPELINE1 port out bsz 32 link LINK1 txq 0
+pipeline PIPELINE1 table match stub
+pipeline PIPELINE1 port in 0 table 0
+pipeline PIPELINE1 port in 1 table 0
+pipeline PIPELINE1 port in 2 table 0
+pipeline PIPELINE1 port in 3 table 0
+pipeline PIPELINE1 table 0 rule add match default action fwd port 0
+
+pipeline PIPELINE2 period 10 offset_port_id 0 cpu 0
+pipeline PIPELINE2 port in bsz 32 link LINK0 rxq 2
+pipeline PIPELINE2 port in bsz 32 link LINK1 rxq 2
+pipeline PIPELINE2 port in bsz 32 link LINK2 rxq 2
+pipeline PIPELINE2 port in bsz 32 link LINK3 rxq 2
+pipeline PIPELINE2 port out bsz 32 link LINK2 txq 0
+pipeline PIPELINE2 table match stub
+pipeline PIPELINE2 port in 0 table 0
+pipeline PIPELINE2 port in 1 table 0
+pipeline PIPELINE2 port in 2 table 0
+pipeline PIPELINE2 port in 3 table 0
+pipeline PIPELINE2 table 0 rule add match default action fwd port 0
+
+pipeline PIPELINE3 period 10 offset_port_id 0 cpu 0
+pipeline PIPELINE3 port in bsz 32 link LINK0 rxq 3
+pipeline PIPELINE3 port in bsz 32 link LINK1 rxq 3
+pipeline PIPELINE3 port in bsz 32 link LINK2 rxq 3
+pipeline PIPELINE3 port in bsz 32 link LINK3 rxq 3
+pipeline PIPELINE3 port out bsz 32 link LINK3 txq 0
+pipeline PIPELINE3 table match stub
+pipeline PIPELINE3 port in 0 table 0
+pipeline PIPELINE3 port in 1 table 0
+pipeline PIPELINE3 port in 2 table 0
+pipeline PIPELINE3 port in 3 table 0
+pipeline PIPELINE3 table 0 rule add match default action fwd port 0
+
+thread 1 pipeline PIPELINE0 enable
+thread 2 pipeline PIPELINE1 enable
+thread 3 pipeline PIPELINE2 enable
+thread 4 pipeline PIPELINE3 enable
diff --git a/examples/ip_pipeline/link.c b/examples/ip_pipeline/link.c
index b8a431f3..392a890f 100644
--- a/examples/ip_pipeline/link.c
+++ b/examples/ip_pipeline/link.c
@@ -36,22 +36,19 @@ link_find(const char *name)
 	return NULL;
 }
 
+struct link *
+link_next(struct link *link)
+{
+	return (link == NULL) ? TAILQ_FIRST(&link_list) : TAILQ_NEXT(link, node);
+}
+
 static struct rte_eth_conf port_conf_default = {
 	.link_speeds = 0,
 	.rxmode = {
 		.mq_mode = ETH_MQ_RX_NONE,
-
-		.header_split   = 0, /* Header split */
-		.hw_ip_checksum = 0, /* IP checksum offload */
-		.hw_vlan_filter = 0, /* VLAN filtering */
-		.hw_vlan_strip  = 0, /* VLAN strip */
-		.hw_vlan_extend = 0, /* Extended VLAN */
-		.jumbo_frame    = 0, /* Jumbo frame support */
-		.hw_strip_crc   = 1, /* CRC strip by HW */
-		.enable_scatter = 0, /* Scattered packets RX handler */
-
 		.max_rx_pkt_len = 9000, /* Jumbo frame max packet len */
 		.split_hdr_size = 0, /* Header split buffer size */
+		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
@@ -162,7 +159,8 @@ link_create(const char *name, struct link_params *params)
 	if (rss) {
 		port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
 		port_conf.rx_adv_conf.rss_conf.rss_hf =
-			ETH_RSS_IPV4 | ETH_RSS_IPV6;
+			(ETH_RSS_IP | ETH_RSS_TCP | ETH_RSS_UDP) &
+			port_info.flow_type_rss_offloads;
 	}
 
 	cpu_id = (uint32_t) rte_eth_dev_socket_id(port_id);
diff --git a/examples/ip_pipeline/link.h b/examples/ip_pipeline/link.h
index 37d3dc43..34ff1149 100644
--- a/examples/ip_pipeline/link.h
+++ b/examples/ip_pipeline/link.h
@@ -30,6 +30,9 @@ link_init(void);
 struct link *
 link_find(const char *name);
 
+struct link *
+link_next(struct link *link);
+
 struct link_params_rss {
 	uint32_t queue_id[LINK_RXQ_RSS_MAX];
 	uint32_t n_queues;
diff --git a/examples/ip_pipeline/thread.c b/examples/ip_pipeline/thread.c
index fa85cf63..7fc03332 100644
--- a/examples/ip_pipeline/thread.c
+++ b/examples/ip_pipeline/thread.c
@@ -155,6 +155,29 @@ thread_init(void)
 	return 0;
 }
 
+static inline int
+thread_is_running(uint32_t thread_id)
+{
+	enum rte_lcore_state_t thread_state;
+
+	thread_state = rte_eal_get_lcore_state(thread_id);
+	return (thread_state == RUNNING) ? 1 : 0;
+}
+
+/**
+ * Pipeline is running when:
+ *    (A) Pipeline is mapped to a data plane thread AND
+ *    (B) Its data plane thread is in RUNNING state.
+ */
+static inline int
+pipeline_is_running(struct pipeline *p)
+{
+	if (p->enabled == 0)
+		return 0;
+
+	return thread_is_running(p->thread_id);
+}
+
 /**
  * Master thread & data plane threads: message passing
  */
@@ -254,6 +277,36 @@ thread_pipeline_enable(uint32_t thread_id,
 		p->enabled)
 		return -1;
 
+	if (!thread_is_running(thread_id)) {
+		struct thread_data *td = &thread_data[thread_id];
+		struct pipeline_data *tdp = &td->pipeline_data[td->n_pipelines];
+
+		if (td->n_pipelines >= THREAD_PIPELINES_MAX)
+			return -1;
+
+		/* Data plane thread */
+		td->p[td->n_pipelines] = p->p;
+
+		tdp->p = p->p;
+		for (i = 0; i < p->n_tables; i++)
+			tdp->table_data[i].a = p->table[i].a;
+
+		tdp->n_tables = p->n_tables;
+
+		tdp->msgq_req = p->msgq_req;
+		tdp->msgq_rsp = p->msgq_rsp;
+		tdp->timer_period = (rte_get_tsc_hz() * p->timer_period_ms) / 1000;
+		tdp->time_next = rte_get_tsc_cycles() + tdp->timer_period;
+
+		td->n_pipelines++;
+
+		/* Pipeline */
+		p->thread_id = thread_id;
+		p->enabled = 1;
+
+		return 0;
+	}
+
 	/* Allocate request */
 	req = thread_msg_alloc();
 	if (req == NULL)
@@ -316,6 +369,38 @@ thread_pipeline_disable(uint32_t thread_id,
 	if (p->thread_id != thread_id)
 		return -1;
 
+	if (!thread_is_running(thread_id)) {
+		struct thread_data *td = &thread_data[thread_id];
+		uint32_t i;
+
+		for (i = 0; i < td->n_pipelines; i++) {
+			struct pipeline_data *tdp = &td->pipeline_data[i];
+
+			if (tdp->p != p->p)
+				continue;
+
+			/* Data plane thread */
+			if (i < td->n_pipelines - 1) {
+				struct rte_pipeline *pipeline_last =
+					td->p[td->n_pipelines - 1];
+				struct pipeline_data *tdp_last =
+					&td->pipeline_data[td->n_pipelines - 1];
+
+				td->p[i] = pipeline_last;
+				memcpy(tdp, tdp_last, sizeof(*tdp));
+			}
+
+			td->n_pipelines--;
+
+			/* Pipeline */
+			p->enabled = 0;
+
+			break;
+		}
+
+		return 0;
+	}
+
 	/* Allocate request */
 	req = thread_msg_alloc();
 	if (req == NULL)
@@ -698,10 +783,18 @@ pipeline_port_in_stats_read(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(port_id >= p->n_ports_in))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_port_in_stats_read(p->p,
+			port_id,
+			stats,
+			clear);
+
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -743,10 +836,14 @@ pipeline_port_in_enable(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(port_id >= p->n_ports_in))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_port_in_enable(p->p, port_id);
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -785,10 +882,14 @@ pipeline_port_in_disable(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(port_id >= p->n_ports_in))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_port_in_disable(p->p, port_id);
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -830,10 +931,18 @@ pipeline_port_out_stats_read(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(port_id >= p->n_ports_out))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_port_out_stats_read(p->p,
+			port_id,
+			stats,
+			clear);
+
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -878,10 +987,18 @@ pipeline_table_stats_read(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_table_stats_read(p->p,
+			table_id,
+			stats,
+			clear);
+
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1070,6 +1187,25 @@ action_default_check(struct table_rule_action *action,
 	return 0;
 }
 
+union table_rule_match_low_level {
+	struct rte_table_acl_rule_add_params acl_add;
+	struct rte_table_acl_rule_delete_params acl_delete;
+	struct rte_table_array_key array;
+	uint8_t hash[TABLE_RULE_MATCH_SIZE_MAX];
+	struct rte_table_lpm_key lpm_ipv4;
+	struct rte_table_lpm_ipv6_key lpm_ipv6;
+};
+
+static int
+match_convert(struct table_rule_match *mh,
+	union table_rule_match_low_level *ml,
+	int add);
+
+static int
+action_convert(struct rte_table_action *a,
+	struct table_rule_action *action,
+	struct rte_pipeline_table_entry *data);
+
 int
 pipeline_table_rule_add(const char *pipeline_name,
 	uint32_t table_id,
@@ -1091,12 +1227,56 @@ pipeline_table_rule_add(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables) ||
 		match_check(match, p, table_id) ||
 		action_check(action, p, table_id))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+		union table_rule_match_low_level match_ll;
+		struct rte_pipeline_table_entry *data_in, *data_out;
+		int key_found;
+		uint8_t *buffer;
+
+		buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
+		if (buffer == NULL)
+			return -1;
+
+		/* Table match-action rule conversion */
+		data_in = (struct rte_pipeline_table_entry *)buffer;
+
+		status = match_convert(match, &match_ll, 1);
+		if (status) {
+			free(buffer);
+			return -1;
+		}
+
+		status = action_convert(a, action, data_in);
+		if (status) {
+			free(buffer);
+			return -1;
+		}
+
+		/* Add rule (match, action) to table */
+		status = rte_pipeline_table_entry_add(p->p,
+				table_id,
+				&match_ll,
+				data_in,
+				&key_found,
+				&data_out);
+		if (status) {
+			free(buffer);
+			return -1;
+		}
+
+		/* Write Response */
+		*data = data_out;
+
+		free(buffer);
+		return 0;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1143,11 +1323,44 @@ pipeline_table_rule_add_default(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables) ||
 		action_default_check(action, p, table_id))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		struct rte_pipeline_table_entry *data_in, *data_out;
+		uint8_t *buffer;
+
+		buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
+		if (buffer == NULL)
+			return -1;
+
+		/* Apply actions */
+		data_in = (struct rte_pipeline_table_entry *)buffer;
+
+		data_in->action = action->fwd.action;
+		if (action->fwd.action == RTE_PIPELINE_ACTION_PORT)
+			data_in->port_id = action->fwd.id;
+		if (action->fwd.action == RTE_PIPELINE_ACTION_TABLE)
+			data_in->table_id = action->fwd.id;
+
+		/* Add default rule to table */
+		status = rte_pipeline_table_default_entry_add(p->p,
+				table_id,
+				data_in,
+				&data_out);
+		if (status) {
+			free(buffer);
+			return -1;
+		}
+
+		/* Write Response */
+		*data = data_out;
+
+		free(buffer);
+		return 0;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1199,7 +1412,6 @@ pipeline_table_rule_add_bulk(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables))
 		return -1;
 
@@ -1208,6 +1420,99 @@ pipeline_table_rule_add_bulk(const char *pipeline_name,
 			action_check(action, p, table_id))
 			return -1;
 
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+		union table_rule_match_low_level *match_ll;
+		uint8_t *action_ll;
+		void **match_ll_ptr;
+		struct rte_pipeline_table_entry **action_ll_ptr;
+		struct rte_pipeline_table_entry **entries_ptr =
+			(struct rte_pipeline_table_entry **)data;
+		uint32_t bulk =
+			(p->table[table_id].params.match_type == TABLE_ACL) ? 1 : 0;
+		int *found;
+
+		/* Memory allocation */
+		match_ll = calloc(*n_rules, sizeof(union table_rule_match_low_level));
+		action_ll = calloc(*n_rules, TABLE_RULE_ACTION_SIZE_MAX);
+		match_ll_ptr = calloc(*n_rules, sizeof(void *));
+		action_ll_ptr =
+			calloc(*n_rules, sizeof(struct rte_pipeline_table_entry *));
+		found = calloc(*n_rules, sizeof(int));
+
+		if (match_ll == NULL ||
+			action_ll == NULL ||
+			match_ll_ptr == NULL ||
+			action_ll_ptr == NULL ||
+			found == NULL)
+			goto fail;
+
+		for (i = 0; i < *n_rules; i++) {
+			match_ll_ptr[i] = (void *)&match_ll[i];
+			action_ll_ptr[i] =
+				(struct rte_pipeline_table_entry *)&action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
+		}
+
+		/* Rule match conversion */
+		for (i = 0; i < *n_rules; i++) {
+			status = match_convert(&match[i], match_ll_ptr[i], 1);
+			if (status)
+				goto fail;
+		}
+
+		/* Rule action conversion */
+		for (i = 0; i < *n_rules; i++) {
+			status = action_convert(a, &action[i], action_ll_ptr[i]);
+			if (status)
+				goto fail;
+		}
+
+		/* Add rule (match, action) to table */
+		if (bulk) {
+			status = rte_pipeline_table_entry_add_bulk(p->p,
+				table_id,
+				match_ll_ptr,
+				action_ll_ptr,
+				*n_rules,
+				found,
+				entries_ptr);
+			if (status)
+				*n_rules = 0;
+		} else {
+			for (i = 0; i < *n_rules; i++) {
+				status = rte_pipeline_table_entry_add(p->p,
+					table_id,
+					match_ll_ptr[i],
+					action_ll_ptr[i],
+					&found[i],
+					&entries_ptr[i]);
+				if (status) {
+					*n_rules = i;
+					break;
+				}
+			}
+		}
+
+		/* Free */
+		free(found);
+		free(action_ll_ptr);
+		free(match_ll_ptr);
+		free(action_ll);
+		free(match_ll);
+
+		return status;
+
+fail:
+		free(found);
+		free(action_ll_ptr);
+		free(match_ll_ptr);
+		free(action_ll);
+		free(match_ll);
+
+		*n_rules = 0;
+		return -1;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1256,11 +1561,27 @@ pipeline_table_rule_delete(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables) ||
 		match_check(match, p, table_id))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		union table_rule_match_low_level match_ll;
+		int key_found;
+
+		status = match_convert(match, &match_ll, 0);
+		if (status)
+			return -1;
+
+		status = rte_pipeline_table_entry_delete(p->p,
+				table_id,
+				&match_ll,
+				&key_found,
+				NULL);
+
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1300,10 +1621,17 @@ pipeline_table_rule_delete_default(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		status = rte_pipeline_table_default_entry_delete(p->p,
+			table_id,
+			NULL);
+
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1347,10 +1675,20 @@ pipeline_table_rule_stats_read(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_stats_read(a,
+			data,
+			stats,
+			clear);
+
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1396,10 +1734,19 @@ pipeline_table_mtr_profile_add(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_meter_profile_add(a,
+			meter_profile_id,
+			profile);
+
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1441,10 +1788,18 @@ pipeline_table_mtr_profile_delete(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_meter_profile_delete(a,
+				meter_profile_id);
+
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1490,10 +1845,21 @@ pipeline_table_rule_mtr_read(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_meter_read(a,
+				data,
+				tc_mask,
+				stats,
+				clear);
+
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1540,10 +1906,19 @@ pipeline_table_dscp_table_update(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_dscp_table_update(a,
+				dscp_mask,
+				dscp_table);
+
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1590,10 +1965,20 @@ pipeline_table_rule_ttl_read(const char *pipeline_name,
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(p->enabled == 0) ||
 		(table_id >= p->n_tables))
 		return -1;
 
+	if (!pipeline_is_running(p)) {
+		struct rte_table_action *a = p->table[table_id].a;
+
+		status = rte_table_action_ttl_read(a,
+				data,
+				stats,
+				clear);
+
+		return status;
+	}
+
 	/* Allocate request */
 	req = pipeline_msg_alloc();
 	if (req == NULL)
@@ -1722,15 +2107,6 @@ pipeline_msg_handle_table_stats_read(struct pipeline_data *p,
 	return rsp;
 }
 
-union table_rule_match_low_level {
-	struct rte_table_acl_rule_add_params acl_add;
-	struct rte_table_acl_rule_delete_params acl_delete;
-	struct rte_table_array_key array;
-	uint8_t hash[TABLE_RULE_MATCH_SIZE_MAX];
-	struct rte_table_lpm_key lpm_ipv4;
-	struct rte_table_lpm_ipv6_key lpm_ipv6;
-};
-
 static int
 match_convert_ipv6_depth(uint32_t depth, uint32_t *depth32)
 {
@@ -2002,138 +2378,136 @@ match_convert(struct table_rule_match *mh,
 	}
 }
 
-static struct pipeline_msg_rsp *
-pipeline_msg_handle_table_rule_add(struct pipeline_data *p,
-	struct pipeline_msg_req *req)
+static int
+action_convert(struct rte_table_action *a,
+	struct table_rule_action *action,
+	struct rte_pipeline_table_entry *data)
 {
-	union table_rule_match_low_level match_ll;
-	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
-	struct table_rule_match *match = &req->table_rule_add.match;
-	struct table_rule_action *action = &req->table_rule_add.action;
-	struct rte_pipeline_table_entry *data_in, *data_out;
-	uint32_t table_id = req->id;
-	int key_found, status;
-	struct rte_table_action *a = p->table_data[table_id].a;
+	int status;
 
 	/* Apply actions */
-	memset(p->buffer, 0, sizeof(p->buffer));
-	data_in = (struct rte_pipeline_table_entry *) p->buffer;
-
 	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
 		status = rte_table_action_apply(a,
-			data_in,
+			data,
 			RTE_TABLE_ACTION_FWD,
 			&action->fwd);
 
-		if (status) {
-			rsp->status = -1;
-			return rsp;
-		}
+		if (status)
+			return status;
 	}
 
 	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) {
 		status = rte_table_action_apply(a,
-			data_in,
+			data,
 			RTE_TABLE_ACTION_LB,
 			&action->lb);
 
-		if (status) {
-			rsp->status = -1;
-			return rsp;
-		}
+		if (status)
+			return status;
 	}
 
 	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
 		status = rte_table_action_apply(a,
-			data_in,
+			data,
 			RTE_TABLE_ACTION_MTR,
 			&action->mtr);
 
-		if (status) {
-			rsp->status = -1;
-			return rsp;
-		}
+		if (status)
+			return status;
 	}
 
 	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
 		status = rte_table_action_apply(a,
-			data_in,
+			data,
 			RTE_TABLE_ACTION_TM,
 			&action->tm);
 
-		if (status) {
-			rsp->status = -1;
-			return rsp;
-		}
+		if (status)
+			return status;
 	}
 
 	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
 		status = rte_table_action_apply(a,
-			data_in,
+			data,
 			RTE_TABLE_ACTION_ENCAP,
 			&action->encap);
 
-		if (status) {
-			rsp->status = -1;
-			return rsp;
-		}
+		if (status)
+			return status;
 	}
 
 	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
 		status = rte_table_action_apply(a,
-			data_in,
+			data,
 			RTE_TABLE_ACTION_NAT,
 			&action->nat);
 
-		if (status) {
-			rsp->status = -1;
-			return rsp;
-		}
+		if (status)
+			return status;
 	}
 
 	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
 		status = rte_table_action_apply(a,
-			data_in,
+			data,
 			RTE_TABLE_ACTION_TTL,
 			&action->ttl);
 
-		if (status) {
-			rsp->status = -1;
-			return rsp;
-		}
+		if (status)
+			return status;
 	}
 
 	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) {
 		status = rte_table_action_apply(a,
-			data_in,
+			data,
 			RTE_TABLE_ACTION_STATS,
 			&action->stats);
 
-		if (status) {
-			rsp->status = -1;
-			return rsp;
-		}
+		if (status)
+			return status;
 	}
 
 	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) {
 		status = rte_table_action_apply(a,
-			data_in,
+			data,
 			RTE_TABLE_ACTION_TIME,
 			&action->time);
 
-		if (status) {
-			rsp->status = -1;
-			return rsp;
-		}
+		if (status)
+			return status;
 	}
 
-	/* Add rule (match, action) to table */
+	return 0;
+}
+
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_rule_add(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	union table_rule_match_low_level match_ll;
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
+	struct table_rule_match *match = &req->table_rule_add.match;
+	struct table_rule_action *action = &req->table_rule_add.action;
+	struct rte_pipeline_table_entry *data_in, *data_out;
+	uint32_t table_id = req->id;
+	int key_found, status;
+	struct rte_table_action *a = p->table_data[table_id].a;
+
+	/* Apply actions */
+	memset(p->buffer, 0, sizeof(p->buffer));
+	data_in = (struct rte_pipeline_table_entry *) p->buffer;
+
 	status = match_convert(match, &match_ll, 1);
 	if (status) {
 		rsp->status = -1;
 		return rsp;
 	}
 
+	status = action_convert(a, action, data_in);
+	if (status) {
+		rsp->status = -1;
+		return rsp;
+	}
+
 	status = rte_pipeline_table_entry_add(p->p,
 		table_id,
 		&match_ll,
@@ -2242,98 +2616,9 @@ pipeline_msg_handle_table_rule_add_bulk(struct pipeline_data *p,
 
 	/* Rule action conversion */
 	for (i = 0; i < n_rules; i++) {
-		void *data_in = action_ll_ptr[i];
-		struct table_rule_action *act = &action[i];
-
-		if (act->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
-			status = rte_table_action_apply(a,
-				data_in,
-				RTE_TABLE_ACTION_FWD,
-				&act->fwd);
-
-			if (status)
-				goto fail;
-		}
-
-		if (act->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) {
-			status = rte_table_action_apply(a,
-				data_in,
-				RTE_TABLE_ACTION_LB,
-				&act->lb);
-
-			if (status)
-				goto fail;
-		}
-
-		if (act->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
-			status = rte_table_action_apply(a,
-				data_in,
-				RTE_TABLE_ACTION_MTR,
-				&act->mtr);
-
-			if (status)
-				goto fail;
-		}
-
-		if (act->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
-			status = rte_table_action_apply(a,
-				data_in,
-				RTE_TABLE_ACTION_TM,
-				&act->tm);
-
-			if (status)
-				goto fail;
-		}
-
-		if (act->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
-			status = rte_table_action_apply(a,
-				data_in,
-				RTE_TABLE_ACTION_ENCAP,
-				&act->encap);
-
-			if (status)
-				goto fail;
-		}
-
-		if (act->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
-			status = rte_table_action_apply(a,
-				data_in,
-				RTE_TABLE_ACTION_NAT,
-				&act->nat);
-
-			if (status)
-				goto fail;
-		}
-
-		if (act->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
-			status = rte_table_action_apply(a,
-				data_in,
-				RTE_TABLE_ACTION_TTL,
-				&act->ttl);
-
-			if (status)
-				goto fail;
-		}
-
-		if (act->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) {
-			status = rte_table_action_apply(a,
-				data_in,
-				RTE_TABLE_ACTION_STATS,
-				&act->stats);
-
-			if (status)
-				goto fail;
-		}
-
-		if (act->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) {
-			status = rte_table_action_apply(a,
-				data_in,
-				RTE_TABLE_ACTION_TIME,
-				&act->time);
-
-			if (status)
-				goto fail;
-		}
+		status = action_convert(a, &action[i], action_ll_ptr[i]);
+		if (status)
+			goto fail;
 	}
 
 	/* Add rule (match, action) to table */
diff --git a/examples/ip_reassembly/main.c b/examples/ip_reassembly/main.c
index 3e8e79c2..b830f67a 100644
--- a/examples/ip_reassembly/main.c
+++ b/examples/ip_reassembly/main.c
@@ -164,7 +164,6 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode        = ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = JUMBO_FRAME_MAX_SIZE,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
 			     DEV_RX_OFFLOAD_JUMBO_FRAME |
 			     DEV_RX_OFFLOAD_CRC_STRIP),
@@ -1083,6 +1082,18 @@ main(int argc, char **argv)
 		if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 			local_port_conf.txmode.offloads |=
 				DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+			dev_info.flow_type_rss_offloads;
+		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+				port_conf.rx_adv_conf.rss_conf.rss_hf) {
+			printf("Port %u modified RSS hash function based on hardware support,"
+				"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+				portid,
+				port_conf.rx_adv_conf.rss_conf.rss_hf,
+				local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+		}
+
 		ret = rte_eth_dev_configure(portid, 1, (uint16_t)n_tx_queue,
 					    &local_port_conf);
 		if (ret < 0) {
@@ -1121,7 +1132,6 @@ main(int argc, char **argv)
 			fflush(stdout);
 
 			txconf = &dev_info.default_txconf;
-			txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 			txconf->offloads = local_port_conf.txmode.offloads;
 
 			ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index a5da8b28..b45b87bd 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -199,7 +199,6 @@ static struct rte_eth_conf port_conf = {
 		.split_hdr_size = 0,
 		.offloads = DEV_RX_OFFLOAD_CHECKSUM |
 			    DEV_RX_OFFLOAD_CRC_STRIP,
-		.ignore_offload_bitfield = 1,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
@@ -331,6 +330,7 @@ prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port)
 		pkt->l3_len = sizeof(struct ip);
 		pkt->l2_len = ETHER_HDR_LEN;
 
+		ip->ip_sum = 0;
 		ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4);
 	} else {
 		pkt->ol_flags |= PKT_TX_IPV6;
@@ -510,11 +510,13 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,
 		sa_idx = ip->res[i] & PROTECT_MASK;
 		if (ip->res[i] & DISCARD)
 			rte_pktmbuf_free(m);
+		else if (ip->res[i] & BYPASS)
+			ip->pkts[j++] = m;
 		else if (sa_idx < IPSEC_SA_MAX_ENTRIES) {
 			ipsec->res[ipsec->num] = sa_idx;
 			ipsec->pkts[ipsec->num++] = m;
-		} else /* BYPASS */
-			ip->pkts[j++] = m;
+		} else /* invalid SA idx */
+			rte_pktmbuf_free(m);
 	}
 	ip->num = j;
 }
@@ -1440,6 +1442,12 @@ cryptodevs_init(void)
 		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
 		dev_conf.nb_queue_pairs = qp;
 
+		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
+		if (dev_max_sess != 0 && dev_max_sess < (CDEV_MP_NB_OBJS / 2))
+			rte_exit(EXIT_FAILURE,
+				"Device does not support at least %u "
+				"sessions", CDEV_MP_NB_OBJS / 2);
+
 		if (!socket_ctx[dev_conf.socket_id].session_pool) {
 			char mp_name[RTE_MEMPOOL_NAMESIZE];
 			struct rte_mempool *sess_mp;
@@ -1566,6 +1574,18 @@ port_init(uint16_t portid)
 	if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 		local_port_conf.txmode.offloads |=
 			DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+	local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+		dev_info.flow_type_rss_offloads;
+	if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+			port_conf.rx_adv_conf.rss_conf.rss_hf) {
+		printf("Port %u modified RSS hash function based on hardware support,"
+			"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+			portid,
+			port_conf.rx_adv_conf.rss_conf.rss_hf,
+			local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+	}
+
 	ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue,
 			&local_port_conf);
 	if (ret < 0)
@@ -1592,7 +1612,6 @@ port_init(uint16_t portid)
 		printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id);
 
 		txconf = &dev_info.default_txconf;
-		txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 		txconf->offloads = local_port_conf.txmode.offloads;
 
 		ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd,
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 9b87278c..c998c807 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -191,7 +191,7 @@ ipsec_metadata_size(void)
 static inline struct ipsec_mbuf_metadata *
 get_priv(struct rte_mbuf *m)
 {
-	return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
+	return rte_mbuf_to_priv(m);
 }
 
 static inline void *
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index d9dcc0e0..4ab8e098 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -939,7 +939,7 @@ inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx)
 {
 	struct ipsec_mbuf_metadata *priv;
 
-	priv = RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
+	priv = get_priv(m);
 
 	return (sa_ctx->sa[sa_idx].spi == priv->sa->spi);
 }
diff --git a/examples/ipv4_multicast/main.c b/examples/ipv4_multicast/main.c
index ad2072f4..331c32e7 100644
--- a/examples/ipv4_multicast/main.c
+++ b/examples/ipv4_multicast/main.c
@@ -109,7 +109,6 @@ static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.max_rx_pkt_len = JUMBO_FRAME_MAX_SIZE,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = (DEV_RX_OFFLOAD_JUMBO_FRAME |
 			     DEV_RX_OFFLOAD_CRC_STRIP),
 	},
@@ -764,7 +763,6 @@ main(int argc, char **argv)
 			fflush(stdout);
 
 			txconf = &dev_info.default_txconf;
-			txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 			txconf->offloads = local_port_conf.txmode.offloads;
 			ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
 						     rte_lcore_to_socket_id(lcore_id), txconf);
diff --git a/examples/kni/Makefile b/examples/kni/Makefile
index 562dc274..7e19d2e2 100644
--- a/examples/kni/Makefile
+++ b/examples/kni/Makefile
@@ -48,7 +48,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(error This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 endif
diff --git a/examples/kni/main.c b/examples/kni/main.c
index 4b162deb..81336087 100644
--- a/examples/kni/main.c
+++ b/examples/kni/main.c
@@ -95,7 +95,6 @@ static struct kni_port_params *kni_port_params_array[RTE_MAX_ETHPORTS];
 /* Options for configuring ethernet port */
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
-		.ignore_offload_bitfield = 1,
 		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
@@ -607,7 +606,6 @@ init_port(uint16_t port)
 				"port%u (%d)\n", (unsigned)port, ret);
 
 	txq_conf = dev_info.default_txconf;
-	txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txq_conf.offloads = local_port_conf.txmode.offloads;
 	ret = rte_eth_tx_queue_setup(port, 0, nb_txd,
 		rte_eth_dev_socket_id(port), &txq_conf);
diff --git a/examples/kni/meson.build b/examples/kni/meson.build
index 0443ab99..79131639 100644
--- a/examples/kni/meson.build
+++ b/examples/kni/meson.build
@@ -6,9 +6,8 @@
 # To build this example as a standalone application with an already-installed
 # DPDK instance, use 'make'
 
-if host_machine.system() != 'linux'
-	build = false
-endif
+# this app can be built if-and-only-if KNI library is buildable
+build = dpdk_conf.has('RTE_LIBRTE_KNI')
 deps += ['kni', 'bus_pci']
 sources = files(
 	'main.c'
diff --git a/examples/l2fwd-crypto/Makefile b/examples/l2fwd-crypto/Makefile
index a67f087b..6658fd0d 100644
--- a/examples/l2fwd-crypto/Makefile
+++ b/examples/l2fwd-crypto/Makefile
@@ -51,5 +51,11 @@ include $(RTE_SDK)/mk/rte.vars.mk
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 
+ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y)
+ifeq ($(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER),y)
+LDLIBS += -lrte_pmd_crypto_scheduler
+endif
+endif
+
 include $(RTE_SDK)/mk/rte.extapp.mk
 endif
diff --git a/examples/l2fwd-crypto/main.c b/examples/l2fwd-crypto/main.c
index 4bca87b1..6061b751 100644
--- a/examples/l2fwd-crypto/main.c
+++ b/examples/l2fwd-crypto/main.c
@@ -42,6 +42,9 @@
 #include <rte_prefetch.h>
 #include <rte_random.h>
 #include <rte_hexdump.h>
+#ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER
+#include <rte_cryptodev_scheduler.h>
+#endif
 
 enum cdev_type {
 	CDEV_TYPE_ANY,
@@ -59,7 +62,6 @@ enum cdev_type {
 #define MAX_AAD_SIZE 65535
 #define MAX_PKT_BURST 32
 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
-#define MAX_SESSIONS 32
 #define SESSION_POOL_CACHE_SIZE 0
 
 #define MAXIMUM_IV_LENGTH	16
@@ -211,7 +213,6 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode = ETH_MQ_RX_NONE,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
@@ -407,7 +408,7 @@ l2fwd_simple_crypto_enqueue(struct rte_mbuf *m,
 	/* Zero pad data to be crypto'd so it is block aligned */
 	data_len  = rte_pktmbuf_data_len(m) - ipdata_offset;
 
-	if (cparams->do_hash && cparams->hash_verify)
+	if ((cparams->do_hash || cparams->do_aead) && cparams->hash_verify)
 		data_len -= cparams->digest_length;
 
 	if (cparams->do_cipher) {
@@ -1930,21 +1931,19 @@ check_supported_size(uint16_t length, uint16_t min, uint16_t max,
 static int
 check_iv_param(const struct rte_crypto_param_range *iv_range_size,
 		unsigned int iv_param, int iv_random_size,
-		uint16_t *iv_length)
+		uint16_t iv_length)
 {
 	/*
 	 * Check if length of provided IV is supported
 	 * by the algorithm chosen.
 	 */
 	if (iv_param) {
-		if (check_supported_size(*iv_length,
+		if (check_supported_size(iv_length,
 				iv_range_size->min,
 				iv_range_size->max,
 				iv_range_size->increment)
-					!= 0) {
-			printf("Unsupported IV length\n");
+					!= 0)
 			return -1;
-		}
 	/*
 	 * Check if length of IV to be randomly generated
 	 * is supported by the algorithm chosen.
@@ -1954,14 +1953,250 @@ check_iv_param(const struct rte_crypto_param_range *iv_range_size,
 				iv_range_size->min,
 				iv_range_size->max,
 				iv_range_size->increment)
-					!= 0) {
-			printf("Unsupported IV length\n");
+					!= 0)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_capabilities(struct l2fwd_crypto_options *options, uint8_t cdev_id)
+{
+	struct rte_cryptodev_info dev_info;
+	const struct rte_cryptodev_capabilities *cap;
+
+	rte_cryptodev_info_get(cdev_id, &dev_info);
+
+	/* Set AEAD parameters */
+	if (options->xform_chain == L2FWD_CRYPTO_AEAD) {
+		/* Check if device supports AEAD algo */
+		cap = check_device_support_aead_algo(options, &dev_info,
+						cdev_id);
+		if (cap == NULL)
+			return -1;
+
+		if (check_iv_param(&cap->sym.aead.iv_size,
+				options->aead_iv_param,
+				options->aead_iv_random_size,
+				options->aead_iv.length) != 0) {
+			RTE_LOG(DEBUG, USER1,
+				"Device %u does not support IV length\n",
+				cdev_id);
+			return -1;
+		}
+
+		/*
+		 * Check if length of provided AEAD key is supported
+		 * by the algorithm chosen.
+		 */
+		if (options->aead_key_param) {
+			if (check_supported_size(
+					options->aead_xform.aead.key.length,
+					cap->sym.aead.key_size.min,
+					cap->sym.aead.key_size.max,
+					cap->sym.aead.key_size.increment)
+						!= 0) {
+				RTE_LOG(DEBUG, USER1,
+					"Device %u does not support "
+					"AEAD key length\n",
+					cdev_id);
+				return -1;
+			}
+		/*
+		 * Check if length of the aead key to be randomly generated
+		 * is supported by the algorithm chosen.
+		 */
+		} else if (options->aead_key_random_size != -1) {
+			if (check_supported_size(options->aead_key_random_size,
+					cap->sym.aead.key_size.min,
+					cap->sym.aead.key_size.max,
+					cap->sym.aead.key_size.increment)
+						!= 0) {
+				RTE_LOG(DEBUG, USER1,
+					"Device %u does not support "
+					"AEAD key length\n",
+					cdev_id);
+				return -1;
+			}
+		}
+
+
+		/*
+		 * Check if length of provided AAD is supported
+		 * by the algorithm chosen.
+		 */
+		if (options->aad_param) {
+			if (check_supported_size(options->aad.length,
+					cap->sym.aead.aad_size.min,
+					cap->sym.aead.aad_size.max,
+					cap->sym.aead.aad_size.increment)
+						!= 0) {
+				RTE_LOG(DEBUG, USER1,
+					"Device %u does not support "
+					"AAD length\n",
+					cdev_id);
+				return -1;
+			}
+		/*
+		 * Check if length of AAD to be randomly generated
+		 * is supported by the algorithm chosen.
+		 */
+		} else if (options->aad_random_size != -1) {
+			if (check_supported_size(options->aad_random_size,
+					cap->sym.aead.aad_size.min,
+					cap->sym.aead.aad_size.max,
+					cap->sym.aead.aad_size.increment)
+						!= 0) {
+				RTE_LOG(DEBUG, USER1,
+					"Device %u does not support "
+					"AAD length\n",
+					cdev_id);
+				return -1;
+			}
+		}
+
+		/* Check if digest size is supported by the algorithm. */
+		if (options->digest_size != -1) {
+			if (check_supported_size(options->digest_size,
+					cap->sym.aead.digest_size.min,
+					cap->sym.aead.digest_size.max,
+					cap->sym.aead.digest_size.increment)
+						!= 0) {
+				RTE_LOG(DEBUG, USER1,
+					"Device %u does not support "
+					"digest length\n",
+					cdev_id);
+				return -1;
+			}
+		}
+	}
+
+	/* Set cipher parameters */
+	if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH ||
+			options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER ||
+			options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) {
+		/* Check if device supports cipher algo */
+		cap = check_device_support_cipher_algo(options, &dev_info,
+						cdev_id);
+		if (cap == NULL)
+			return -1;
+
+		if (check_iv_param(&cap->sym.cipher.iv_size,
+				options->cipher_iv_param,
+				options->cipher_iv_random_size,
+				options->cipher_iv.length) != 0) {
+			RTE_LOG(DEBUG, USER1,
+				"Device %u does not support IV length\n",
+				cdev_id);
+			return -1;
+		}
+
+		/*
+		 * Check if length of provided cipher key is supported
+		 * by the algorithm chosen.
+		 */
+		if (options->ckey_param) {
+			if (check_supported_size(
+					options->cipher_xform.cipher.key.length,
+					cap->sym.cipher.key_size.min,
+					cap->sym.cipher.key_size.max,
+					cap->sym.cipher.key_size.increment)
+						!= 0) {
+				RTE_LOG(DEBUG, USER1,
+					"Device %u does not support cipher "
+					"key length\n",
+					cdev_id);
+				return -1;
+			}
+		/*
+		 * Check if length of the cipher key to be randomly generated
+		 * is supported by the algorithm chosen.
+		 */
+		} else if (options->ckey_random_size != -1) {
+			if (check_supported_size(options->ckey_random_size,
+					cap->sym.cipher.key_size.min,
+					cap->sym.cipher.key_size.max,
+					cap->sym.cipher.key_size.increment)
+						!= 0) {
+				RTE_LOG(DEBUG, USER1,
+					"Device %u does not support cipher "
+					"key length\n",
+					cdev_id);
+				return -1;
+			}
+		}
+	}
+
+	/* Set auth parameters */
+	if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH ||
+			options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER ||
+			options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) {
+		/* Check if device supports auth algo */
+		cap = check_device_support_auth_algo(options, &dev_info,
+						cdev_id);
+		if (cap == NULL)
 			return -1;
+
+		if (check_iv_param(&cap->sym.auth.iv_size,
+				options->auth_iv_param,
+				options->auth_iv_random_size,
+				options->auth_iv.length) != 0) {
+			RTE_LOG(DEBUG, USER1,
+				"Device %u does not support IV length\n",
+				cdev_id);
+			return -1;
+		}
+		/*
+		 * Check if length of provided auth key is supported
+		 * by the algorithm chosen.
+		 */
+		if (options->akey_param) {
+			if (check_supported_size(
+					options->auth_xform.auth.key.length,
+					cap->sym.auth.key_size.min,
+					cap->sym.auth.key_size.max,
+					cap->sym.auth.key_size.increment)
+						!= 0) {
+				RTE_LOG(DEBUG, USER1,
+					"Device %u does not support auth "
+					"key length\n",
+					cdev_id);
+				return -1;
+			}
+		/*
+		 * Check if length of the auth key to be randomly generated
+		 * is supported by the algorithm chosen.
+		 */
+		} else if (options->akey_random_size != -1) {
+			if (check_supported_size(options->akey_random_size,
+					cap->sym.auth.key_size.min,
+					cap->sym.auth.key_size.max,
+					cap->sym.auth.key_size.increment)
+						!= 0) {
+				RTE_LOG(DEBUG, USER1,
+					"Device %u does not support auth "
+					"key length\n",
+					cdev_id);
+				return -1;
+			}
+		}
+
+		/* Check if digest size is supported by the algorithm. */
+		if (options->digest_size != -1) {
+			if (check_supported_size(options->digest_size,
+					cap->sym.auth.digest_size.min,
+					cap->sym.auth.digest_size.max,
+					cap->sym.auth.digest_size.increment)
+						!= 0) {
+				RTE_LOG(DEBUG, USER1,
+					"Device %u does not support "
+					"digest length\n",
+					cdev_id);
+				return -1;
+			}
 		}
-		*iv_length = iv_random_size;
-	/* No size provided, use minimum size. */
-	} else
-		*iv_length = iv_range_size->min;
+	}
 
 	return 0;
 }
@@ -1970,9 +2205,10 @@ static int
 initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports,
 		uint8_t *enabled_cdevs)
 {
-	unsigned int cdev_id, cdev_count, enabled_cdev_count = 0;
+	uint8_t cdev_id, cdev_count, enabled_cdev_count = 0;
 	const struct rte_cryptodev_capabilities *cap;
 	unsigned int sess_sz, max_sess_sz = 0;
+	uint32_t sessions_needed = 0;
 	int retval;
 
 	cdev_count = rte_cryptodev_count();
@@ -1981,16 +2217,31 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports,
 		return -1;
 	}
 
-	for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) {
+	for (cdev_id = 0; cdev_id < cdev_count && enabled_cdev_count < nb_ports;
+			cdev_id++) {
+		if (check_cryptodev_mask(options, cdev_id) < 0)
+			continue;
+
+		if (check_capabilities(options, cdev_id) < 0)
+			continue;
+
 		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
 		if (sess_sz > max_sess_sz)
 			max_sess_sz = sess_sz;
+
+		l2fwd_enabled_crypto_mask |= (((uint64_t)1) << cdev_id);
+
+		enabled_cdevs[cdev_id] = 1;
+		enabled_cdev_count++;
 	}
 
-	for (cdev_id = 0; cdev_id < cdev_count && enabled_cdev_count < nb_ports;
-			cdev_id++) {
+	for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) {
 		struct rte_cryptodev_qp_conf qp_conf;
 		struct rte_cryptodev_info dev_info;
+
+		if (enabled_cdevs[cdev_id] == 0)
+			continue;
+
 		retval = rte_cryptodev_socket_id(cdev_id);
 
 		if (retval < 0) {
@@ -2005,11 +2256,23 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports,
 			.socket_id = socket_id,
 		};
 
-		if (check_cryptodev_mask(options, (uint8_t)cdev_id))
-			continue;
-
 		rte_cryptodev_info_get(cdev_id, &dev_info);
 
+		/*
+		 * Two sessions objects are required for each session
+		 * (one for the header, one for the private data)
+		 */
+		if (!strcmp(dev_info.driver_name, "crypto_scheduler")) {
+#ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER
+			uint32_t nb_slaves =
+				rte_cryptodev_scheduler_slaves_get(cdev_id,
+								NULL);
+
+			sessions_needed = 2 * enabled_cdev_count * nb_slaves;
+#endif
+		} else
+			sessions_needed = 2 * enabled_cdev_count;
+
 		if (session_pool_socket[socket_id] == NULL) {
 			char mp_name[RTE_MEMPOOL_NAMESIZE];
 			struct rte_mempool *sess_mp;
@@ -2022,7 +2285,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports,
 			 * device private data
 			 */
 			sess_mp = rte_mempool_create(mp_name,
-						MAX_SESSIONS * 2,
+						sessions_needed,
 						max_sess_sz,
 						SESSION_POOL_CACHE_SIZE,
 						0, NULL, NULL, NULL,
@@ -2041,106 +2304,57 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports,
 
 		/* Set AEAD parameters */
 		if (options->xform_chain == L2FWD_CRYPTO_AEAD) {
-			/* Check if device supports AEAD algo */
 			cap = check_device_support_aead_algo(options, &dev_info,
 							cdev_id);
-			if (cap == NULL)
-				continue;
 
 			options->block_size = cap->sym.aead.block_size;
 
-			check_iv_param(&cap->sym.aead.iv_size,
-					options->aead_iv_param,
-					options->aead_iv_random_size,
-					&options->aead_iv.length);
+			/* Set IV if not provided from command line */
+			if (options->aead_iv_param == 0) {
+				if (options->aead_iv_random_size != -1)
+					options->aead_iv.length =
+						options->aead_iv_random_size;
+				/* No size provided, use minimum size. */
+				else
+					options->aead_iv.length =
+						cap->sym.aead.iv_size.min;
+			}
 
-			/*
-			 * Check if length of provided AEAD key is supported
-			 * by the algorithm chosen.
-			 */
-			if (options->aead_key_param) {
-				if (check_supported_size(
-						options->aead_xform.aead.key.length,
-						cap->sym.aead.key_size.min,
-						cap->sym.aead.key_size.max,
-						cap->sym.aead.key_size.increment)
-							!= 0) {
-					printf("Unsupported aead key length\n");
-					return -1;
-				}
-			/*
-			 * Check if length of the aead key to be randomly generated
-			 * is supported by the algorithm chosen.
-			 */
-			} else if (options->aead_key_random_size != -1) {
-				if (check_supported_size(options->aead_key_random_size,
-						cap->sym.aead.key_size.min,
-						cap->sym.aead.key_size.max,
-						cap->sym.aead.key_size.increment)
-							!= 0) {
-					printf("Unsupported aead key length\n");
-					return -1;
-				}
-				options->aead_xform.aead.key.length =
-							options->aead_key_random_size;
-			/* No size provided, use minimum size. */
-			} else
-				options->aead_xform.aead.key.length =
+			/* Set key if not provided from command line */
+			if (options->aead_key_param == 0) {
+				if (options->aead_key_random_size != -1)
+					options->aead_xform.aead.key.length =
+						options->aead_key_random_size;
+				/* No size provided, use minimum size. */
+				else
+					options->aead_xform.aead.key.length =
 						cap->sym.aead.key_size.min;
 
-			if (!options->aead_key_param)
 				generate_random_key(
 					options->aead_xform.aead.key.data,
 					options->aead_xform.aead.key.length);
+			}
 
-			/*
-			 * Check if length of provided AAD is supported
-			 * by the algorithm chosen.
-			 */
-			if (options->aad_param) {
-				if (check_supported_size(options->aad.length,
-						cap->sym.aead.aad_size.min,
-						cap->sym.aead.aad_size.max,
-						cap->sym.aead.aad_size.increment)
-							!= 0) {
-					printf("Unsupported AAD length\n");
-					return -1;
-				}
-			/*
-			 * Check if length of AAD to be randomly generated
-			 * is supported by the algorithm chosen.
-			 */
-			} else if (options->aad_random_size != -1) {
-				if (check_supported_size(options->aad_random_size,
-						cap->sym.aead.aad_size.min,
-						cap->sym.aead.aad_size.max,
-						cap->sym.aead.aad_size.increment)
-							!= 0) {
-					printf("Unsupported AAD length\n");
-					return -1;
-				}
-				options->aad.length = options->aad_random_size;
-			/* No size provided, use minimum size. */
-			} else
-				options->aad.length = cap->sym.auth.aad_size.min;
+			/* Set AAD if not provided from command line */
+			if (options->aad_param == 0) {
+				if (options->aad_random_size != -1)
+					options->aad.length =
+						options->aad_random_size;
+				/* No size provided, use minimum size. */
+				else
+					options->aad.length =
+						cap->sym.auth.aad_size.min;
+			}
 
 			options->aead_xform.aead.aad_length =
 						options->aad.length;
 
-			/* Check if digest size is supported by the algorithm. */
-			if (options->digest_size != -1) {
-				if (check_supported_size(options->digest_size,
-						cap->sym.aead.digest_size.min,
-						cap->sym.aead.digest_size.max,
-						cap->sym.aead.digest_size.increment)
-							!= 0) {
-					printf("Unsupported digest length\n");
-					return -1;
-				}
+			/* Set digest size if not provided from command line */
+			if (options->digest_size != -1)
 				options->aead_xform.aead.digest_length =
 							options->digest_size;
-			/* No size provided, use minimum size. */
-			} else
+				/* No size provided, use minimum size. */
+			else
 				options->aead_xform.aead.digest_length =
 						cap->sym.aead.digest_size.min;
 		}
@@ -2149,127 +2363,76 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports,
 		if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH ||
 				options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER ||
 				options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) {
-			/* Check if device supports cipher algo */
 			cap = check_device_support_cipher_algo(options, &dev_info,
 							cdev_id);
-			if (cap == NULL)
-				continue;
-
 			options->block_size = cap->sym.cipher.block_size;
 
-			check_iv_param(&cap->sym.cipher.iv_size,
-					options->cipher_iv_param,
-					options->cipher_iv_random_size,
-					&options->cipher_iv.length);
+			/* Set IV if not provided from command line */
+			if (options->cipher_iv_param == 0) {
+				if (options->cipher_iv_random_size != -1)
+					options->cipher_iv.length =
+						options->cipher_iv_random_size;
+				/* No size provided, use minimum size. */
+				else
+					options->cipher_iv.length =
+						cap->sym.cipher.iv_size.min;
+			}
 
-			/*
-			 * Check if length of provided cipher key is supported
-			 * by the algorithm chosen.
-			 */
-			if (options->ckey_param) {
-				if (check_supported_size(
-						options->cipher_xform.cipher.key.length,
-						cap->sym.cipher.key_size.min,
-						cap->sym.cipher.key_size.max,
-						cap->sym.cipher.key_size.increment)
-							!= 0) {
-					printf("Unsupported cipher key length\n");
-					return -1;
-				}
-			/*
-			 * Check if length of the cipher key to be randomly generated
-			 * is supported by the algorithm chosen.
-			 */
-			} else if (options->ckey_random_size != -1) {
-				if (check_supported_size(options->ckey_random_size,
-						cap->sym.cipher.key_size.min,
-						cap->sym.cipher.key_size.max,
-						cap->sym.cipher.key_size.increment)
-							!= 0) {
-					printf("Unsupported cipher key length\n");
-					return -1;
-				}
-				options->cipher_xform.cipher.key.length =
-							options->ckey_random_size;
-			/* No size provided, use minimum size. */
-			} else
-				options->cipher_xform.cipher.key.length =
+			/* Set key if not provided from command line */
+			if (options->ckey_param == 0) {
+				if (options->ckey_random_size != -1)
+					options->cipher_xform.cipher.key.length =
+						options->ckey_random_size;
+				/* No size provided, use minimum size. */
+				else
+					options->cipher_xform.cipher.key.length =
 						cap->sym.cipher.key_size.min;
 
-			if (!options->ckey_param)
 				generate_random_key(
 					options->cipher_xform.cipher.key.data,
 					options->cipher_xform.cipher.key.length);
-
+			}
 		}
 
 		/* Set auth parameters */
 		if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH ||
 				options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER ||
 				options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) {
-			/* Check if device supports auth algo */
 			cap = check_device_support_auth_algo(options, &dev_info,
 							cdev_id);
-			if (cap == NULL)
-				continue;
 
-			check_iv_param(&cap->sym.auth.iv_size,
-					options->auth_iv_param,
-					options->auth_iv_random_size,
-					&options->auth_iv.length);
-			/*
-			 * Check if length of provided auth key is supported
-			 * by the algorithm chosen.
-			 */
-			if (options->akey_param) {
-				if (check_supported_size(
-						options->auth_xform.auth.key.length,
-						cap->sym.auth.key_size.min,
-						cap->sym.auth.key_size.max,
-						cap->sym.auth.key_size.increment)
-							!= 0) {
-					printf("Unsupported auth key length\n");
-					return -1;
-				}
-			/*
-			 * Check if length of the auth key to be randomly generated
-			 * is supported by the algorithm chosen.
-			 */
-			} else if (options->akey_random_size != -1) {
-				if (check_supported_size(options->akey_random_size,
-						cap->sym.auth.key_size.min,
-						cap->sym.auth.key_size.max,
-						cap->sym.auth.key_size.increment)
-							!= 0) {
-					printf("Unsupported auth key length\n");
-					return -1;
-				}
-				options->auth_xform.auth.key.length =
-							options->akey_random_size;
-			/* No size provided, use minimum size. */
-			} else
-				options->auth_xform.auth.key.length =
+			/* Set IV if not provided from command line */
+			if (options->auth_iv_param == 0) {
+				if (options->auth_iv_random_size != -1)
+					options->auth_iv.length =
+						options->auth_iv_random_size;
+				/* No size provided, use minimum size. */
+				else
+					options->auth_iv.length =
+						cap->sym.auth.iv_size.min;
+			}
+
+			/* Set key if not provided from command line */
+			if (options->akey_param == 0) {
+				if (options->akey_random_size != -1)
+					options->auth_xform.auth.key.length =
+						options->akey_random_size;
+				/* No size provided, use minimum size. */
+				else
+					options->auth_xform.auth.key.length =
 						cap->sym.auth.key_size.min;
 
-			if (!options->akey_param)
 				generate_random_key(
 					options->auth_xform.auth.key.data,
 					options->auth_xform.auth.key.length);
+			}
 
-			/* Check if digest size is supported by the algorithm. */
-			if (options->digest_size != -1) {
-				if (check_supported_size(options->digest_size,
-						cap->sym.auth.digest_size.min,
-						cap->sym.auth.digest_size.max,
-						cap->sym.auth.digest_size.increment)
-							!= 0) {
-					printf("Unsupported digest length\n");
-					return -1;
-				}
+			/* Set digest size if not provided from command line */
+			if (options->digest_size != -1)
 				options->auth_xform.auth.digest_length =
 							options->digest_size;
-			/* No size provided, use minimum size. */
-			} else
+				/* No size provided, use minimum size. */
+			else
 				options->auth_xform.auth.digest_length =
 						cap->sym.auth.digest_size.min;
 		}
@@ -2296,11 +2459,6 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports,
 					cdev_id, retval);
 			return -1;
 		}
-
-		l2fwd_enabled_crypto_mask |= (((uint64_t)1) << cdev_id);
-
-		enabled_cdevs[cdev_id] = 1;
-		enabled_cdev_count++;
 	}
 
 	return enabled_cdev_count;
@@ -2371,7 +2529,6 @@ initialize_ports(struct l2fwd_crypto_options *options)
 		/* init one TX queue on each port */
 		fflush(stdout);
 		txq_conf = dev_info.default_txconf;
-		txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 		txq_conf.offloads = local_port_conf.txmode.offloads;
 		retval = rte_eth_tx_queue_setup(portid, 0, nb_txd,
 				rte_eth_dev_socket_id(portid),
diff --git a/examples/l2fwd-jobstats/main.c b/examples/l2fwd-jobstats/main.c
index 34553faa..af542338 100644
--- a/examples/l2fwd-jobstats/main.c
+++ b/examples/l2fwd-jobstats/main.c
@@ -90,7 +90,6 @@ struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
@@ -876,7 +875,6 @@ main(int argc, char **argv)
 
 		/* init one TX queue on each port */
 		txq_conf = dev_info.default_txconf;
-		txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 		txq_conf.offloads = local_port_conf.txmode.offloads;
 		fflush(stdout);
 		ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
diff --git a/examples/l2fwd-keepalive/main.c b/examples/l2fwd-keepalive/main.c
index a18b707c..2d8b4d1c 100644
--- a/examples/l2fwd-keepalive/main.c
+++ b/examples/l2fwd-keepalive/main.c
@@ -81,7 +81,6 @@ struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
@@ -671,7 +670,6 @@ main(int argc, char **argv)
 		/* init one TX queue on each port */
 		fflush(stdout);
 		txq_conf = dev_info.default_txconf;
-		txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 		txq_conf.offloads = local_port_conf.txmode.offloads;
 		ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
 				rte_eth_dev_socket_id(portid),
diff --git a/examples/l2fwd/main.c b/examples/l2fwd/main.c
index 69084357..9bb4c5bc 100644
--- a/examples/l2fwd/main.c
+++ b/examples/l2fwd/main.c
@@ -82,7 +82,6 @@ static struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
@@ -670,7 +669,6 @@ main(int argc, char **argv)
 		/* init one TX queue on each port */
 		fflush(stdout);
 		txq_conf = dev_info.default_txconf;
-		txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 		txq_conf.offloads = local_port_conf.txmode.offloads;
 		ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
 				rte_eth_dev_socket_id(portid),
diff --git a/examples/l3fwd-acl/main.c b/examples/l3fwd-acl/main.c
index 33ad467d..7c063a8d 100644
--- a/examples/l3fwd-acl/main.c
+++ b/examples/l3fwd-acl/main.c
@@ -127,7 +127,6 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode	= ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = (DEV_RX_OFFLOAD_CRC_STRIP |
 			     DEV_RX_OFFLOAD_CHECKSUM),
 	},
@@ -1926,6 +1925,18 @@ main(int argc, char **argv)
 		if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 			local_port_conf.txmode.offloads |=
 				DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+			dev_info.flow_type_rss_offloads;
+		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+				port_conf.rx_adv_conf.rss_conf.rss_hf) {
+			printf("Port %u modified RSS hash function based on hardware support,"
+				"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+				portid,
+				port_conf.rx_adv_conf.rss_conf.rss_hf,
+				local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+		}
+
 		ret = rte_eth_dev_configure(portid, nb_rx_queue,
 					(uint16_t)n_tx_queue, &local_port_conf);
 		if (ret < 0)
@@ -1982,7 +1993,6 @@ main(int argc, char **argv)
 
 			rte_eth_dev_info_get(portid, &dev_info);
 			txconf = &dev_info.default_txconf;
-			txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 			txconf->offloads = local_port_conf.txmode.offloads;
 			ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
 						     socketid, txconf);
diff --git a/examples/l3fwd-power/Makefile b/examples/l3fwd-power/Makefile
index 390b7d6b..d7e39a34 100644
--- a/examples/l3fwd-power/Makefile
+++ b/examples/l3fwd-power/Makefile
@@ -1,11 +1,11 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2010-2014 Intel Corporation
+# Copyright(c) 2010-2018 Intel Corporation
 
 # binary name
 APP = l3fwd-power
 
 # all source are stored in SRCS-y
-SRCS-y := main.c
+SRCS-y := main.c perf_core.c
 
 # Build using pkg-config variables if possible
 $(shell pkg-config --exists libdpdk)
@@ -48,7 +48,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(info This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 all:
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 596d6454..d15cd520 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2016 Intel Corporation
+ * Copyright(c) 2010-2018 Intel Corporation
  */
 
 #include <stdio.h>
@@ -44,6 +44,9 @@
 #include <rte_power.h>
 #include <rte_spinlock.h>
 
+#include "perf_core.h"
+#include "main.h"
+
 #define RTE_LOGTYPE_L3FWD_POWER RTE_LOGTYPE_USER1
 
 #define MAX_PKT_BURST 32
@@ -155,14 +158,7 @@ struct lcore_rx_queue {
 #define MAX_RX_QUEUE_INTERRUPT_PER_PORT 16
 
 
-#define MAX_LCORE_PARAMS 1024
-struct lcore_params {
-	uint16_t port_id;
-	uint8_t queue_id;
-	uint8_t lcore_id;
-} __rte_cache_aligned;
-
-static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
+struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 static struct lcore_params lcore_params_array_default[] = {
 	{0, 0, 2},
 	{0, 1, 2},
@@ -175,8 +171,8 @@ static struct lcore_params lcore_params_array_default[] = {
 	{3, 1, 3},
 };
 
-static struct lcore_params * lcore_params = lcore_params_array_default;
-static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) /
+struct lcore_params *lcore_params = lcore_params_array_default;
+uint16_t nb_lcore_params = sizeof(lcore_params_array_default) /
 				sizeof(lcore_params_array_default[0]);
 
 static struct rte_eth_conf port_conf = {
@@ -184,7 +180,6 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode        = ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = (DEV_RX_OFFLOAD_CRC_STRIP |
 			     DEV_RX_OFFLOAD_CHECKSUM),
 	},
@@ -1121,10 +1116,15 @@ print_usage(const char *prgname)
 {
 	printf ("%s [EAL options] -- -p PORTMASK -P"
 		"  [--config (port,queue,lcore)[,(port,queue,lcore]]"
+		"  [--high-perf-cores CORELIST"
+		"  [--perf-config (port,queue,hi_perf,lcore_index)[,(port,queue,hi_perf,lcore_index]]"
 		"  [--enable-jumbo [--max-pkt-len PKTLEN]]\n"
 		"  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
 		"  -P : enable promiscuous mode\n"
 		"  --config (port,queue,lcore): rx queues configuration\n"
+		"  --high-perf-cores CORELIST: list of high performance cores\n"
+		"  --perf-config: similar as config, cores specified as indices"
+		" for bins containing high or regular performance cores\n"
 		"  --no-numa: optional, disable numa awareness\n"
 		"  --enable-jumbo: enable jumbo frame"
 		" which max packet len is PKTLEN in decimal (64-9600)\n"
@@ -1234,6 +1234,8 @@ parse_args(int argc, char **argv)
 	char *prgname = argv[0];
 	static struct option lgopts[] = {
 		{"config", 1, 0, 0},
+		{"perf-config", 1, 0, 0},
+		{"high-perf-cores", 1, 0, 0},
 		{"no-numa", 0, 0, 0},
 		{"enable-jumbo", 0, 0, 0},
 		{CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0},
@@ -1272,6 +1274,26 @@ parse_args(int argc, char **argv)
 			}
 
 			if (!strncmp(lgopts[option_index].name,
+					"perf-config", 11)) {
+				ret = parse_perf_config(optarg);
+				if (ret) {
+					printf("invalid perf-config\n");
+					print_usage(prgname);
+					return -1;
+				}
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					"high-perf-cores", 15)) {
+				ret = parse_perf_core_list(optarg);
+				if (ret) {
+					printf("invalid high-perf-cores\n");
+					print_usage(prgname);
+					return -1;
+				}
+			}
+
+			if (!strncmp(lgopts[option_index].name,
 						"no-numa", 7)) {
 				printf("numa is disabled \n");
 				numa_on = 0;
@@ -1609,6 +1631,23 @@ static int check_ptype(uint16_t portid)
 
 }
 
+static int
+init_power_library(void)
+{
+	int ret = 0, lcore_id;
+	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+		if (rte_lcore_is_enabled(lcore_id)) {
+			/* init power management library */
+			ret = rte_power_init(lcore_id);
+			if (ret)
+				RTE_LOG(ERR, POWER,
+				"Library initialization failed on core %u\n",
+				lcore_id);
+		}
+	}
+	return ret;
+}
+
 int
 main(int argc, char **argv)
 {
@@ -1643,6 +1682,12 @@ main(int argc, char **argv)
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
 
+	if (init_power_library())
+		rte_exit(EXIT_FAILURE, "init_power_library failed\n");
+
+	if (update_lcore_params() < 0)
+		rte_exit(EXIT_FAILURE, "update_lcore_params failed\n");
+
 	if (check_lcore_params() < 0)
 		rte_exit(EXIT_FAILURE, "check_lcore_params failed\n");
 
@@ -1693,6 +1738,18 @@ main(int argc, char **argv)
 		if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 			local_port_conf.txmode.offloads |=
 				DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+			dev_info.flow_type_rss_offloads;
+		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+				port_conf.rx_adv_conf.rss_conf.rss_hf) {
+			printf("Port %u modified RSS hash function based on hardware support,"
+				"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+				portid,
+				port_conf.rx_adv_conf.rss_conf.rss_hf,
+				local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+		}
+
 		ret = rte_eth_dev_configure(portid, nb_rx_queue,
 					(uint16_t)n_tx_queue, &local_port_conf);
 		if (ret < 0)
@@ -1750,7 +1807,6 @@ main(int argc, char **argv)
 			fflush(stdout);
 
 			txconf = &dev_info.default_txconf;
-			txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 			txconf->offloads = local_port_conf.txmode.offloads;
 			ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
 						     socketid, txconf);
@@ -1773,12 +1829,6 @@ main(int argc, char **argv)
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
 
-		/* init power management library */
-		ret = rte_power_init(lcore_id);
-		if (ret)
-			RTE_LOG(ERR, POWER,
-				"Library initialization failed on core %u\n", lcore_id);
-
 		/* init timer structures for each enabled lcore */
 		rte_timer_init(&power_timers[lcore_id]);
 		hz = rte_get_timer_hz();
diff --git a/examples/l3fwd-power/main.h b/examples/l3fwd-power/main.h
new file mode 100644
index 00000000..258de98f
--- /dev/null
+++ b/examples/l3fwd-power/main.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#ifndef _MAIN_H_
+#define _MAIN_H_
+
+
+#define MAX_LCORE_PARAMS 1024
+struct lcore_params {
+	uint16_t port_id;
+	uint8_t queue_id;
+	uint8_t lcore_id;
+} __rte_cache_aligned;
+
+extern struct lcore_params *lcore_params;
+extern uint16_t nb_lcore_params;
+extern struct lcore_params lcore_params_array[];
+
+#endif /* _MAIN_H_ */
diff --git a/examples/l3fwd-power/meson.build b/examples/l3fwd-power/meson.build
index f633a0f1..20c80543 100644
--- a/examples/l3fwd-power/meson.build
+++ b/examples/l3fwd-power/meson.build
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2017 Intel Corporation
+# Copyright(c) 2018 Intel Corporation
 
 # meson file, for building this example as part of a main DPDK build.
 #
@@ -11,5 +11,5 @@ if host_machine.system() != 'linux'
 endif
 deps += ['power', 'timer', 'lpm', 'hash']
 sources = files(
-	'main.c'
+	'main.c', 'perf_core.c'
 )
diff --git a/examples/l3fwd-power/perf_core.c b/examples/l3fwd-power/perf_core.c
new file mode 100644
index 00000000..83948ea2
--- /dev/null
+++ b/examples/l3fwd-power/perf_core.c
@@ -0,0 +1,230 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_memory.h>
+#include <rte_lcore.h>
+#include <rte_power.h>
+#include <rte_string_fns.h>
+
+#include "perf_core.h"
+#include "main.h"
+
+
+static uint16_t hp_lcores[RTE_MAX_LCORE];
+static uint16_t nb_hp_lcores;
+
+struct perf_lcore_params {
+	uint16_t port_id;
+	uint8_t queue_id;
+	uint8_t high_perf;
+	uint8_t lcore_idx;
+} __rte_cache_aligned;
+
+static struct perf_lcore_params prf_lc_prms[MAX_LCORE_PARAMS];
+static uint16_t nb_prf_lc_prms;
+
+int
+update_lcore_params(void)
+{
+	uint8_t non_perf_lcores[RTE_MAX_LCORE];
+	uint16_t nb_non_perf_lcores = 0;
+	int i, j, ret;
+
+	/* if perf-config option was not used do nothing */
+	if (nb_prf_lc_prms == 0)
+		return 0;
+
+	/* if high-perf-cores option was not used query every available core */
+	if (nb_hp_lcores == 0) {
+		for (i = 0; i < RTE_MAX_LCORE; i++) {
+			if (rte_lcore_is_enabled(i)) {
+				struct rte_power_core_capabilities caps;
+				ret = rte_power_get_capabilities(i, &caps);
+				if (ret == 0 && caps.turbo) {
+					hp_lcores[nb_hp_lcores] = i;
+					nb_hp_lcores++;
+				}
+			}
+		}
+	}
+
+	/* create a list on non high performance cores*/
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		if (rte_lcore_is_enabled(i)) {
+			int hp = 0;
+			for (j = 0; j < nb_hp_lcores; j++) {
+				if (hp_lcores[j] == i) {
+					hp = 1;
+					break;
+				}
+			}
+			if (!hp)
+				non_perf_lcores[nb_non_perf_lcores++] = i;
+		}
+	}
+
+	/* update the lcore config */
+	for (i = 0; i < nb_prf_lc_prms; i++) {
+		int lcore = -1;
+		if (prf_lc_prms[i].high_perf) {
+			if (prf_lc_prms[i].lcore_idx < nb_hp_lcores)
+				lcore = hp_lcores[prf_lc_prms[i].lcore_idx];
+		} else {
+			if (prf_lc_prms[i].lcore_idx < nb_non_perf_lcores)
+				lcore =
+				non_perf_lcores[prf_lc_prms[i].lcore_idx];
+		}
+
+		if (lcore < 0) {
+			printf("Performance cores configuration error\n");
+			return -1;
+		}
+
+		lcore_params_array[i].lcore_id = lcore;
+		lcore_params_array[i].queue_id = prf_lc_prms[i].queue_id;
+		lcore_params_array[i].port_id = prf_lc_prms[i].port_id;
+	}
+
+	lcore_params = lcore_params_array;
+	nb_lcore_params = nb_prf_lc_prms;
+
+	printf("Updated performance core configuration\n");
+	for (i = 0; i < nb_prf_lc_prms; i++)
+		printf("\t(%d,%d,%d)\n", lcore_params[i].port_id,
+				lcore_params[i].queue_id,
+				lcore_params[i].lcore_id);
+
+	return 0;
+}
+
+int
+parse_perf_config(const char *q_arg)
+{
+	char s[256];
+	const char *p, *p0 = q_arg;
+	char *end;
+	enum fieldnames {
+		FLD_PORT = 0,
+		FLD_QUEUE,
+		FLD_LCORE_HP,
+		FLD_LCORE_IDX,
+		_NUM_FLD
+	};
+	unsigned long int_fld[_NUM_FLD];
+	char *str_fld[_NUM_FLD];
+	int i;
+	unsigned int size;
+
+	nb_prf_lc_prms = 0;
+
+	while ((p = strchr(p0, '(')) != NULL) {
+		++p;
+		p0 = strchr(p, ')');
+		if (p0 == NULL)
+			return -1;
+
+		size = p0 - p;
+		if (size >= sizeof(s))
+			return -1;
+
+		snprintf(s, sizeof(s), "%.*s", size, p);
+		if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
+								_NUM_FLD)
+			return -1;
+		for (i = 0; i < _NUM_FLD; i++) {
+			errno = 0;
+			int_fld[i] = strtoul(str_fld[i], &end, 0);
+			if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
+				return -1;
+		}
+		if (nb_prf_lc_prms >= MAX_LCORE_PARAMS) {
+			printf("exceeded max number of lcore params: %hu\n",
+					nb_prf_lc_prms);
+			return -1;
+		}
+		prf_lc_prms[nb_prf_lc_prms].port_id =
+				(uint8_t)int_fld[FLD_PORT];
+		prf_lc_prms[nb_prf_lc_prms].queue_id =
+				(uint8_t)int_fld[FLD_QUEUE];
+		prf_lc_prms[nb_prf_lc_prms].high_perf =
+				!!(uint8_t)int_fld[FLD_LCORE_HP];
+		prf_lc_prms[nb_prf_lc_prms].lcore_idx =
+				(uint8_t)int_fld[FLD_LCORE_IDX];
+		++nb_prf_lc_prms;
+	}
+
+	return 0;
+}
+
+int
+parse_perf_core_list(const char *corelist)
+{
+	int i, idx = 0;
+	unsigned int count = 0;
+	char *end = NULL;
+	int min, max;
+
+	if (corelist == NULL) {
+		printf("invalid core list\n");
+		return -1;
+	}
+
+
+	/* Remove all blank characters ahead and after */
+	while (isblank(*corelist))
+		corelist++;
+	i = strlen(corelist);
+	while ((i > 0) && isblank(corelist[i - 1]))
+		i--;
+
+	/* Get list of cores */
+	min = RTE_MAX_LCORE;
+	do {
+		while (isblank(*corelist))
+			corelist++;
+		if (*corelist == '\0')
+			return -1;
+		errno = 0;
+		idx = strtoul(corelist, &end, 10);
+		if (errno || end == NULL)
+			return -1;
+		while (isblank(*end))
+			end++;
+		if (*end == '-') {
+			min = idx;
+		} else if ((*end == ',') || (*end == '\0')) {
+			max = idx;
+			if (min == RTE_MAX_LCORE)
+				min = idx;
+			for (idx = min; idx <= max; idx++) {
+				hp_lcores[count] = idx;
+				count++;
+			}
+			min = RTE_MAX_LCORE;
+		} else {
+			printf("invalid core list\n");
+			return -1;
+		}
+		corelist = end + 1;
+	} while (*end != '\0');
+
+	if (count == 0) {
+		printf("invalid core list\n");
+		return -1;
+	}
+
+	nb_hp_lcores = count;
+
+	printf("Configured %d high performance cores\n", nb_hp_lcores);
+	for (i = 0; i < nb_hp_lcores; i++)
+		printf("\tHigh performance core %d %d\n",
+				i, hp_lcores[i]);
+
+	return 0;
+}
+
diff --git a/examples/l3fwd-power/perf_core.h b/examples/l3fwd-power/perf_core.h
new file mode 100644
index 00000000..7b7b747b
--- /dev/null
+++ b/examples/l3fwd-power/perf_core.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#ifndef _PERF_CORE_H_
+#define _PERF_CORE_H_
+
+int parse_perf_config(const char *q_arg);
+int parse_perf_core_list(const char *corelist);
+int update_lcore_params(void);
+
+#endif /* _PERF_CORE_H_ */
diff --git a/examples/l3fwd-vf/main.c b/examples/l3fwd-vf/main.c
index aaafb7bc..5edd91a7 100644
--- a/examples/l3fwd-vf/main.c
+++ b/examples/l3fwd-vf/main.c
@@ -161,7 +161,6 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode	= ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = (DEV_RX_OFFLOAD_CRC_STRIP |
 			     DEV_RX_OFFLOAD_CHECKSUM),
 	},
@@ -981,6 +980,18 @@ main(int argc, char **argv)
 		if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 			local_port_conf.txmode.offloads |=
 				DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+			dev_info.flow_type_rss_offloads;
+		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+				port_conf.rx_adv_conf.rss_conf.rss_hf) {
+			printf("Port %u modified RSS hash function based on hardware support,"
+				"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+				portid,
+				port_conf.rx_adv_conf.rss_conf.rss_hf,
+				local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+		}
+
 		ret = rte_eth_dev_configure(portid, nb_rx_queue,
 					    n_tx_queue, &local_port_conf);
 		if (ret < 0)
@@ -1009,7 +1020,6 @@ main(int argc, char **argv)
 		fflush(stdout);
 
 		txconf = &dev_info.default_txconf;
-		txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 		txconf->offloads = local_port_conf.txmode.offloads;
 		ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
 						 socketid, txconf);
diff --git a/examples/l3fwd/l3fwd_em.c b/examples/l3fwd/l3fwd_em.c
index 9dc3b8c4..fa8f82be 100644
--- a/examples/l3fwd/l3fwd_em.c
+++ b/examples/l3fwd/l3fwd_em.c
@@ -18,7 +18,6 @@
 #include <rte_debug.h>
 #include <rte_ether.h>
 #include <rte_ethdev.h>
-#include <rte_mempool.h>
 #include <rte_cycles.h>
 #include <rte_mbuf.h>
 #include <rte_ip.h>
diff --git a/examples/l3fwd/l3fwd_lpm.c b/examples/l3fwd/l3fwd_lpm.c
index a747126a..b1dc195a 100644
--- a/examples/l3fwd/l3fwd_lpm.c
+++ b/examples/l3fwd/l3fwd_lpm.c
@@ -17,7 +17,6 @@
 #include <rte_debug.h>
 #include <rte_ether.h>
 #include <rte_ethdev.h>
-#include <rte_mempool.h>
 #include <rte_cycles.h>
 #include <rte_mbuf.h>
 #include <rte_ip.h>
diff --git a/examples/l3fwd/main.c b/examples/l3fwd/main.c
index bf7dbd81..ab019b9e 100644
--- a/examples/l3fwd/main.c
+++ b/examples/l3fwd/main.c
@@ -120,7 +120,6 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode = ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = (DEV_RX_OFFLOAD_CRC_STRIP |
 			     DEV_RX_OFFLOAD_CHECKSUM),
 	},
@@ -861,6 +860,18 @@ main(int argc, char **argv)
 		if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 			local_port_conf.txmode.offloads |=
 				DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+			dev_info.flow_type_rss_offloads;
+		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+				port_conf.rx_adv_conf.rss_conf.rss_hf) {
+			printf("Port %u modified RSS hash function based on hardware support,"
+				"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+				portid,
+				port_conf.rx_adv_conf.rss_conf.rss_hf,
+				local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+		}
+
 		ret = rte_eth_dev_configure(portid, nb_rx_queue,
 					(uint16_t)n_tx_queue, &local_port_conf);
 		if (ret < 0)
@@ -909,7 +920,6 @@ main(int argc, char **argv)
 			fflush(stdout);
 
 			txconf = &dev_info.default_txconf;
-			txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 			txconf->offloads = local_port_conf.txmode.offloads;
 			ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
 						     socketid, txconf);
diff --git a/examples/link_status_interrupt/main.c b/examples/link_status_interrupt/main.c
index f5689568..3b732076 100644
--- a/examples/link_status_interrupt/main.c
+++ b/examples/link_status_interrupt/main.c
@@ -79,7 +79,6 @@ struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
@@ -653,7 +652,6 @@ main(int argc, char **argv)
 		/* init one TX queue logical core on each port */
 		fflush(stdout);
 		txq_conf = dev_info.default_txconf;
-		txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 		txq_conf.offloads = local_port_conf.txmode.offloads;
 		ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
 				rte_eth_dev_socket_id(portid),
diff --git a/examples/load_balancer/init.c b/examples/load_balancer/init.c
index 8d8dbe61..f2045f23 100644
--- a/examples/load_balancer/init.c
+++ b/examples/load_balancer/init.c
@@ -45,7 +45,6 @@ static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.mq_mode	= ETH_MQ_RX_RSS,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
 			     DEV_RX_OFFLOAD_CRC_STRIP),
 	},
@@ -417,6 +416,18 @@ app_init_nics(void)
 		if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 			local_port_conf.txmode.offloads |=
 				DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+			dev_info.flow_type_rss_offloads;
+		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+				port_conf.rx_adv_conf.rss_conf.rss_hf) {
+			printf("Port %u modified RSS hash function based on hardware support,"
+				"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+				port,
+				port_conf.rx_adv_conf.rss_conf.rss_hf,
+				local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+		}
+
 		ret = rte_eth_dev_configure(
 			port,
 			(uint8_t) n_rx_queues,
@@ -466,7 +477,6 @@ app_init_nics(void)
 		}
 
 		txq_conf = dev_info.default_txconf;
-		txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 		txq_conf.offloads = local_port_conf.txmode.offloads;
 		/* Init TX queues */
 		if (app.nic_tx_port_mask[port] == 1) {
diff --git a/examples/meson.build b/examples/meson.build
index 3d156849..4ee7a111 100644
--- a/examples/meson.build
+++ b/examples/meson.build
@@ -18,12 +18,17 @@ else
 	examples = get_option('examples').split(',')
 	allow_skips = false # error out if we can't build a requested app
 endif
+default_cflags = machine_args
+if cc.has_argument('-Wno-format-truncation')
+	default_cflags += '-Wno-format-truncation'
+endif
 foreach example: examples
 	name = example
 	build = true
 	sources = []
 	allow_experimental_apis = false
-	cflags = machine_args
+	cflags = default_cflags
+
 	ext_deps = [execinfo]
 	includes = [include_directories(example)]
 	deps = ['eal', 'mempool', 'net', 'mbuf', 'ethdev', 'cmdline']
diff --git a/examples/multi_process/client_server_mp/mp_server/Makefile b/examples/multi_process/client_server_mp/mp_server/Makefile
index af7246e6..9c1caae7 100644
--- a/examples/multi_process/client_server_mp/mp_server/Makefile
+++ b/examples/multi_process/client_server_mp/mp_server/Makefile
@@ -10,7 +10,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(error This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 endif
diff --git a/examples/multi_process/l2fwd_fork/Makefile b/examples/multi_process/l2fwd_fork/Makefile
deleted file mode 100644
index b65582ef..00000000
--- a/examples/multi_process/l2fwd_fork/Makefile
+++ /dev/null
@@ -1,22 +0,0 @@
-# SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2010-2014 Intel Corporation
-
-# binary name
-APP = l2fwd-fork
-
-# all source are stored in SRCS-y
-SRCS-y := main.c flib.c
-
-ifeq ($(RTE_SDK),)
-$(error "Please define RTE_SDK environment variable")
-endif
-
-# Default target, can be overridden by command line or environment
-RTE_TARGET ?= x86_64-native-linuxapp-gcc
-
-include $(RTE_SDK)/mk/rte.vars.mk
-
-CFLAGS += -O3
-CFLAGS += $(WERROR_FLAGS)
-
-include $(RTE_SDK)/mk/rte.extapp.mk
diff --git a/examples/multi_process/l2fwd_fork/flib.c b/examples/multi_process/l2fwd_fork/flib.c
deleted file mode 100644
index 52c6152b..00000000
--- a/examples/multi_process/l2fwd_fork/flib.c
+++ /dev/null
@@ -1,280 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2014 Intel Corporation
- */
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <inttypes.h>
-#include <sys/types.h>
-#include <sys/queue.h>
-#include <sys/wait.h>
-#include <sys/prctl.h>
-#include <netinet/in.h>
-#include <setjmp.h>
-#include <stdarg.h>
-#include <ctype.h>
-#include <errno.h>
-#include <getopt.h>
-#include <dirent.h>
-#include <signal.h>
-
-#include <rte_common.h>
-#include <rte_log.h>
-#include <rte_malloc.h>
-#include <rte_memory.h>
-#include <rte_memcpy.h>
-#include <rte_eal.h>
-#include <rte_launch.h>
-#include <rte_atomic.h>
-#include <rte_cycles.h>
-#include <rte_prefetch.h>
-#include <rte_lcore.h>
-#include <rte_per_lcore.h>
-#include <rte_branch_prediction.h>
-#include <rte_interrupts.h>
-#include <rte_random.h>
-#include <rte_debug.h>
-#include <rte_ether.h>
-#include <rte_ethdev.h>
-#include <rte_mempool.h>
-#include <rte_mbuf.h>
-#include <rte_string_fns.h>
-
-#include "flib.h"
-
-#define SIG_PARENT_EXIT SIGUSR1
-
-struct lcore_stat {
-	pid_t pid;           /**< pthread identifier */
-	lcore_function_t *f; /**< function to call */
-	void *arg;           /**< argument of function */
-	slave_exit_notify *cb_fn;
-} __rte_cache_aligned;
-
-
-static struct lcore_stat *core_cfg;
-static uint16_t *lcore_cfg = NULL;
-
-/* signal handler to be notified after parent leaves */
-static void
-sighand_parent_exit(int sig)
-{
-	printf("lcore = %u : Find parent leaves, sig=%d\n", rte_lcore_id(),
-			sig);
-	printf("Child leaving\n");
-	exit(0);
-
-	return;
-}
-
-/**
- * Real function entrance ran in slave process
- **/
-static int
-slave_proc_func(void)
-{
-	struct rte_config *config;
-	unsigned slave_id = rte_lcore_id();
-	struct lcore_stat *cfg = &core_cfg[slave_id];
-
-	if (prctl(PR_SET_PDEATHSIG, SIG_PARENT_EXIT, 0, 0, 0, 0) != 0)
-		printf("Warning: Slave can't register for being notified in"
-               "case master process exited\n");
-	else {
-		struct sigaction act;
-		memset(&act, 0 , sizeof(act));
-		act.sa_handler = sighand_parent_exit;
-		if (sigaction(SIG_PARENT_EXIT, &act, NULL) != 0)
-			printf("Fail to register signal handler:%d\n", SIG_PARENT_EXIT);
-	}
-
-	/* Set slave process to SECONDARY to avoid operation like dev_start/stop etc */
-	config = rte_eal_get_configuration();
-	if (NULL == config)
-		printf("Warning:Can't get rte_config\n");
-	else
-		config->process_type = RTE_PROC_SECONDARY;
-
-	printf("Core %u is ready (pid=%d)\n", slave_id, (int)cfg->pid);
-
-	exit(cfg->f(cfg->arg));
-}
-
-/**
- * function entrance ran in master thread, which will spawn slave process and wait until
- * specific slave exited.
- **/
-static int
-lcore_func(void *arg __attribute__((unused)))
-{
-	unsigned slave_id = rte_lcore_id();
-	struct lcore_stat *cfg = &core_cfg[slave_id];
-	int pid, stat;
-
-	if (rte_get_master_lcore() == slave_id)
-		return cfg->f(cfg->arg);
-
-	/* fork a slave process */
-	pid = fork();
-
-	if (pid == -1) {
-		printf("Failed to fork\n");
-		return -1;
-	} else if (pid == 0) /* child */
-		return slave_proc_func();
-	else { /* parent */
-		cfg->pid = pid;
-
-		waitpid(pid, &stat, 0);
-
-		cfg->pid = 0;
-		cfg->f = NULL;
-		cfg->arg = NULL;
-		/* Notify slave's exit if applicable */
-		if(cfg->cb_fn)
-			cfg->cb_fn(slave_id, stat);
-		return stat;
-	}
-}
-
-static int
-lcore_id_init(void)
-{
-	int i;
-	/* Setup lcore ID allocation map */
-	lcore_cfg = rte_zmalloc("LCORE_ID_MAP",
-						sizeof(uint16_t) * RTE_MAX_LCORE,
-						RTE_CACHE_LINE_SIZE);
-
-	if(lcore_cfg == NULL)
-		rte_panic("Failed to malloc\n");
-
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		if (rte_lcore_is_enabled(i))
-			lcore_cfg[i] = 1;
-	}
-	return 0;
-}
-
-int
-flib_assign_lcore_id(void)
-{
-	unsigned i;
-	int ret;
-
-	/**
-	 * thread assigned a lcore id previously, or a  slave thread. But still have
-	 * a bug here: If the core mask includes core 0, and that core call this
-	 * function, it still can get a new lcore id.
-	 **/
-	if (rte_lcore_id() != 0)
-		return -1;
-
-	do {
-		/* Find a lcore id not used yet, avoid to use lcore ID 0 */
-		for (i = 1; i < RTE_MAX_LCORE; i++) {
-			if (lcore_cfg[i] == 0)
-				break;
-		}
-		if (i == RTE_MAX_LCORE)
-			return -1;
-
-		/* Assign new lcore id to this thread */
-
-		ret = rte_atomic16_cmpset(&lcore_cfg[i], 0, 1);
-	} while (unlikely(ret == 0));
-
-	RTE_PER_LCORE(_lcore_id) = i;
-	return i;
-}
-
-void
-flib_free_lcore_id(unsigned lcore_id)
-{
-	/* id is not valid or belongs to pinned core id */
-	if (lcore_id >= RTE_MAX_LCORE || lcore_id == 0 ||
-		rte_lcore_is_enabled(lcore_id))
-		return;
-
-	lcore_cfg[lcore_id] = 0;
-}
-
-int
-flib_register_slave_exit_notify(unsigned slave_id,
-	slave_exit_notify *cb)
-{
-	if (cb == NULL)
-		return -EFAULT;
-
-	if (!rte_lcore_is_enabled(slave_id))
-		return -ENOENT;
-
-	core_cfg[slave_id].cb_fn = cb;
-
-	return 0;
-}
-
-enum slave_stat
-flib_query_slave_status(unsigned slave_id)
-{
-	if (!rte_lcore_is_enabled(slave_id))
-		return ST_FREEZE;
-	/* pid only be set when slave process spawned */
-	if (core_cfg[slave_id].pid != 0)
-		return ST_RUN;
-	else
-		return ST_IDLE;
-}
-
-int
-flib_remote_launch(lcore_function_t *f,
-					void *arg, unsigned slave_id)
-{
-	if (f == NULL)
-		return -1;
-
-	if (!rte_lcore_is_enabled(slave_id))
-		return -1;
-
-	/* Wait until specific lcore state change to WAIT */
-	rte_eal_wait_lcore(slave_id);
-
-	core_cfg[slave_id].f = f;
-	core_cfg[slave_id].arg = arg;
-
-	return rte_eal_remote_launch(lcore_func, NULL, slave_id);
-}
-
-int
-flib_mp_remote_launch(lcore_function_t *f, void *arg,
-			enum rte_rmt_call_master_t call_master)
-{
-	int i;
-
-	RTE_LCORE_FOREACH_SLAVE(i) {
-		core_cfg[i].arg = arg;
-		core_cfg[i].f = f;
-	}
-
-	return rte_eal_mp_remote_launch(lcore_func, NULL, call_master);
-}
-
-int
-flib_init(void)
-{
-	if ((core_cfg = rte_zmalloc("core_cfg",
-		sizeof(struct lcore_stat) * RTE_MAX_LCORE,
-		RTE_CACHE_LINE_SIZE)) == NULL ) {
-		printf("rte_zmalloc failed\n");
-		return -1;
-	}
-
-	if (lcore_id_init() != 0) {
-		printf("lcore_id_init failed\n");
-		return -1;
-	}
-
-	return 0;
-}
diff --git a/examples/multi_process/l2fwd_fork/flib.h b/examples/multi_process/l2fwd_fork/flib.h
deleted file mode 100644
index 8bc13a1f..00000000
--- a/examples/multi_process/l2fwd_fork/flib.h
+++ /dev/null
@@ -1,120 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2014 Intel Corporation
- */
-
-#ifndef __FLIB_H
-#define __FLIB_H
-
-/* callback function pointer when specific slave leaves */
-typedef void (slave_exit_notify)(unsigned slaveid, int stat);
-
-enum slave_stat{
-	ST_FREEZE = 1,
-	ST_IDLE,
-	ST_RUN,
-	ST_ZOMBIE,	/* Not implemented yet */
-};
-
-/**
- * Initialize the fork lib.
- *
- * @return
- *    - 0 : fork lib initialized successfully
- *    - -1 : fork lib initialized failed
- */
-int flib_init(void);
-
-/**
- * Check that every SLAVE lcores are in WAIT state, then call
- * flib_remote_launch() for all of them. If call_master is true
- * (set to CALL_MASTER), also call the function on the master lcore.
- *
- * @param f:
- *	function pointer need to run
- * @param arg:
- *	argument for f to carry
- * @param call_master
- *	- SKIP_MASTER : only launch function on slave lcores
- *	- CALL_MASTER : launch function on master and slave lcores
- * @return
- *    - 0 : function  execute successfully
- *    - -1 :  function  execute  failed
- */
-int flib_mp_remote_launch(lcore_function_t *f,
-		void *arg, enum rte_rmt_call_master_t call_master);
-
-/**
- * Send a message to a slave lcore identified by slave_id to call a
- * function f with argument arg.
- *
- * @param f:
- *	function pointer need to run
- * @param arg:
- *	argument for f to carry
- * @param slave_id
- *	slave lcore id to run on
- * @return
- *    - 0 : function  execute successfully
- *    - -1 :  function  execute  failed
- */
-int flib_remote_launch(lcore_function_t *f,
-					void *arg, unsigned slave_id);
-
-/**
- * Query the running stat for specific slave, wont' work in with master id
- *
- * @param slave_id:
- *	lcore id which should not be master id
- * @return
- *    - ST_FREEZE : lcore is not in enabled core mask
- *	 - ST_IDLE     : lcore is idle
- *    -  ST_RUN     : lcore is running something
- */
-enum slave_stat
-flib_query_slave_status(unsigned slave_id);
-
-/**
- * Register a callback function to be notified in case specific slave exit.
- *
- * @param slave_id:
- *	lcore id which should not be master id
- * @param cb:
- *	callback pointer to register
- * @return
- *    - 0            :  function  execute successfully
- *    - -EFAULT  :  argument error
- *    - -ENOENT :  slave_id not correct
- */
-int flib_register_slave_exit_notify(unsigned slave_id,
-	slave_exit_notify *cb);
-
-/**
- * Assign a lcore ID to non-slave thread.  Non-slave thread refers to thread that
- * not created by function rte_eal_remote_launch or rte_eal_mp_remote_launch.
- * These threads can either bind lcore or float among different lcores.
- * This lcore ID will be unique in multi-thread or multi-process DPDK running
- * environment, then it can benefit from using the cache mechanism provided in
- * mempool library.
- * After calling successfully, use rte_lcore_id() to get the assigned lcore ID, but
- * other lcore funtions can't guarantee to work correctly.
- *
- * @return
- *   -    -1  : can't assign a lcore id with 3 possibilities.
- *                 - it's not non-slave thread.
- *                 - it had assign a lcore id previously
- *                 - the lcore id is running out.
- *   -  > 0 :  the assigned lcore id.
- */
-int flib_assign_lcore_id(void);
-
-/**
- * Free the lcore_id that assigned in flib_assign_lcore_id().
- * call it in case non-slave thread is leaving or left.
- *
- * @param lcore_id
- * The identifier of the lcore, which MUST be between 1 and
- *   RTE_MAX_LCORE-1.
- */
-void flib_free_lcore_id(unsigned lcore_id);
-
-#endif /* __FLIB_H  */
diff --git a/examples/multi_process/l2fwd_fork/main.c b/examples/multi_process/l2fwd_fork/main.c
deleted file mode 100644
index 94318ab6..00000000
--- a/examples/multi_process/l2fwd_fork/main.c
+++ /dev/null
@@ -1,1259 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2016 Intel Corporation
- */
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
-#include <stdint.h>
-#include <sched.h>
-#include <inttypes.h>
-#include <sys/types.h>
-#include <sys/queue.h>
-#include <netinet/in.h>
-#include <setjmp.h>
-#include <stdarg.h>
-#include <ctype.h>
-#include <errno.h>
-#include <getopt.h>
-
-#include <rte_common.h>
-#include <rte_log.h>
-#include <rte_memory.h>
-#include <rte_memcpy.h>
-#include <rte_eal.h>
-#include <rte_launch.h>
-#include <rte_atomic.h>
-#include <rte_spinlock.h>
-#include <rte_cycles.h>
-#include <rte_prefetch.h>
-#include <rte_lcore.h>
-#include <rte_per_lcore.h>
-#include <rte_branch_prediction.h>
-#include <rte_interrupts.h>
-#include <rte_random.h>
-#include <rte_debug.h>
-#include <rte_ether.h>
-#include <rte_ethdev.h>
-#include <rte_ring.h>
-#include <rte_mempool.h>
-#include <rte_mbuf.h>
-#include <rte_malloc.h>
-
-#include "flib.h"
-
-#define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
-#define MBUF_NAME	"mbuf_pool_%d"
-#define MBUF_DATA_SIZE	RTE_MBUF_DEFAULT_BUF_SIZE
-#define NB_MBUF   8192
-#define RING_MASTER_NAME	"l2fwd_ring_m2s_"
-#define RING_SLAVE_NAME		"l2fwd_ring_s2m_"
-#define MAX_NAME_LEN	32
-/* RECREATE flag indicate needs initialize resource and launch slave_core again */
-#define SLAVE_RECREATE_FLAG 0x1
-/* RESTART flag indicate needs restart port and send START command again */
-#define SLAVE_RESTART_FLAG 0x2
-#define INVALID_MAPPING_ID	((unsigned)LCORE_ID_ANY)
-/* Maximum message buffer per slave */
-#define NB_CORE_MSGBUF	32
-enum l2fwd_cmd{
-	CMD_START,
-	CMD_STOP,
-};
-
-#define MAX_PKT_BURST 32
-#define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
-
-/*
- * Configurable number of RX/TX ring descriptors
- */
-#define RTE_TEST_RX_DESC_DEFAULT 1024
-#define RTE_TEST_TX_DESC_DEFAULT 1024
-static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
-static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
-
-/* ethernet addresses of ports */
-static struct ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS];
-
-/* mask of enabled ports */
-static uint32_t l2fwd_enabled_port_mask = 0;
-
-/* list of enabled ports */
-static uint32_t l2fwd_dst_ports[RTE_MAX_ETHPORTS];
-
-static unsigned int l2fwd_rx_queue_per_lcore = 1;
-
-struct mbuf_table {
-	unsigned len;
-	struct rte_mbuf *m_table[MAX_PKT_BURST];
-};
-
-#define MAX_RX_QUEUE_PER_LCORE 16
-#define MAX_TX_QUEUE_PER_PORT 16
-struct lcore_queue_conf {
-	unsigned n_rx_port;
-	unsigned rx_port_list[MAX_RX_QUEUE_PER_LCORE];
-} __rte_cache_aligned;
-struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE];
-
-struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
-
-struct lcore_resource_struct {
-	int enabled;	/* Only set in case this lcore involved into packet forwarding */
-	int flags; 	    /* Set only slave need to restart or recreate */
-	unsigned lcore_id;  /*  lcore ID */
-	unsigned pair_id; 	/* dependency lcore ID on port */
-	char ring_name[2][MAX_NAME_LEN];
-	/* ring[0] for master send cmd, slave read */
-	/* ring[1] for slave send ack, master read */
-	struct rte_ring *ring[2];
-	int port_num;					/* Total port numbers */
-	/* Port id for that lcore to receive packets */
-	uint16_t port[RTE_MAX_ETHPORTS];
-}__attribute__((packed)) __rte_cache_aligned;
-
-static struct lcore_resource_struct lcore_resource[RTE_MAX_LCORE];
-static struct rte_mempool *message_pool;
-static rte_spinlock_t res_lock = RTE_SPINLOCK_INITIALIZER;
-/* use floating processes */
-static int float_proc = 0;
-/* Save original cpu affinity */
-struct cpu_aff_arg{
-	cpu_set_t set;
-	size_t size;
-}cpu_aff;
-
-static const struct rte_eth_conf port_conf = {
-	.rxmode = {
-		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
-	},
-	.txmode = {
-		.mq_mode = ETH_MQ_TX_NONE,
-	},
-};
-
-static struct rte_mempool * l2fwd_pktmbuf_pool[RTE_MAX_ETHPORTS];
-
-/* Per-port statistics struct */
-struct l2fwd_port_statistics {
-	uint64_t tx;
-	uint64_t rx;
-	uint64_t dropped;
-} __rte_cache_aligned;
-struct l2fwd_port_statistics *port_statistics;
-/**
- * pointer to lcore ID mapping array, used to return lcore id in case slave
- * process exited unexpectedly, use only floating process option applied
- **/
-unsigned *mapping_id;
-
-/* A tsc-based timer responsible for triggering statistics printout */
-#define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */
-#define MAX_TIMER_PERIOD 86400 /* 1 day max */
-static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000; /* default period is 10 seconds */
-
-static int l2fwd_launch_one_lcore(void *dummy);
-
-/* Print out statistics on packets dropped */
-static void
-print_stats(void)
-{
-	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
-	unsigned portid;
-
-	total_packets_dropped = 0;
-	total_packets_tx = 0;
-	total_packets_rx = 0;
-
-	const char clr[] = { 27, '[', '2', 'J', '\0' };
-	const char topLeft[] = { 27, '[', '1', ';', '1', 'H','\0' };
-
-		/* Clear screen and move to top left */
-	printf("%s%s", clr, topLeft);
-
-	printf("\nPort statistics ====================================");
-
-	for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
-		/* skip disabled ports */
-		if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
-			continue;
-		printf("\nStatistics for port %u ------------------------------"
-			   "\nPackets sent: %24"PRIu64
-			   "\nPackets received: %20"PRIu64
-			   "\nPackets dropped: %21"PRIu64,
-			   portid,
-			   port_statistics[portid].tx,
-			   port_statistics[portid].rx,
-			   port_statistics[portid].dropped);
-
-		total_packets_dropped += port_statistics[portid].dropped;
-		total_packets_tx += port_statistics[portid].tx;
-		total_packets_rx += port_statistics[portid].rx;
-	}
-	printf("\nAggregate statistics ==============================="
-		   "\nTotal packets sent: %18"PRIu64
-		   "\nTotal packets received: %14"PRIu64
-		   "\nTotal packets dropped: %15"PRIu64,
-		   total_packets_tx,
-		   total_packets_rx,
-		   total_packets_dropped);
-	printf("\n====================================================\n");
-}
-
-static int
-clear_cpu_affinity(void)
-{
-	int s;
-
-	s = sched_setaffinity(0, cpu_aff.size, &cpu_aff.set);
-	if (s != 0) {
-		printf("sched_setaffinity failed:%s\n", strerror(errno));
-		return -1;
-	}
-
-	return 0;
-}
-
-static int
-get_cpu_affinity(void)
-{
-	int s;
-
-	cpu_aff.size = sizeof(cpu_set_t);
-	CPU_ZERO(&cpu_aff.set);
-
-	s = sched_getaffinity(0, cpu_aff.size, &cpu_aff.set);
-	if (s != 0) {
-		printf("sched_getaffinity failed:%s\n", strerror(errno));
-		return -1;
-	}
-
-	return 0;
-}
-
-/**
- * This fnciton demonstrates the approach to create ring in first instance
- * or re-attach an existed ring in later instance.
- **/
-static struct rte_ring *
-create_ring(const char *name, unsigned count,
-					int socket_id,unsigned flags)
-{
-	struct rte_ring *ring;
-
-	if (name == NULL)
-		return NULL;
-
-	/* If already create, just attached it */
-	if (likely((ring = rte_ring_lookup(name)) != NULL))
-		return ring;
-
-	/* First call it, create one */
-	return rte_ring_create(name, count, socket_id, flags);
-}
-
-/* Malloc with rte_malloc on structures that shared by master and slave */
-static int
-l2fwd_malloc_shared_struct(void)
-{
-	port_statistics = rte_zmalloc("port_stat",
-						sizeof(struct l2fwd_port_statistics) * RTE_MAX_ETHPORTS,
-						0);
-	if (port_statistics == NULL)
-		return -1;
-
-	/* allocate  mapping_id array */
-	if (float_proc) {
-		int i;
-		mapping_id = rte_malloc("mapping_id", sizeof(unsigned) * RTE_MAX_LCORE,
-								0);
-
-		if (mapping_id == NULL)
-			return -1;
-
-		for (i = 0 ;i < RTE_MAX_LCORE; i++)
-			mapping_id[i] = INVALID_MAPPING_ID;
-	}
-	return 0;
-}
-
-/* Create ring which used for communicate among master and slave */
-static int
-create_ms_ring(unsigned slaveid)
-{
-	unsigned flag = RING_F_SP_ENQ | RING_F_SC_DEQ;
-	struct lcore_resource_struct *res = &lcore_resource[slaveid];
-	unsigned socketid = rte_socket_id();
-
-	/* Always assume create ring on master socket_id */
-	/* Default only create a ring size 32 */
-	snprintf(res->ring_name[0], MAX_NAME_LEN, "%s%u",
-			RING_MASTER_NAME, slaveid);
-	if ((res->ring[0] = create_ring(res->ring_name[0], NB_CORE_MSGBUF,
-				socketid, flag)) == NULL) {
-		printf("Create m2s ring %s failed\n", res->ring_name[0]);
-		return -1;
-	}
-
-	snprintf(res->ring_name[1], MAX_NAME_LEN, "%s%u",
-			RING_SLAVE_NAME, slaveid);
-	if ((res->ring[1] = create_ring(res->ring_name[1], NB_CORE_MSGBUF,
-		socketid, flag)) == NULL) {
-		printf("Create s2m ring %s failed\n", res->ring_name[1]);
-		return -1;
-	}
-
-	return 0;
-}
-
-/* send command to pair in paired master and slave ring */
-static inline int
-sendcmd(unsigned slaveid, enum l2fwd_cmd cmd, int is_master)
-{
-	struct lcore_resource_struct *res = &lcore_resource[slaveid];
-	void *msg;
-	int fd = !is_master;
-
-	/* Only check master, it must be enabled and running if it is slave */
-	if (is_master && !res->enabled)
-		return -1;
-
-	if (res->ring[fd] == NULL)
-		return -1;
-
-	if (rte_mempool_get(message_pool, &msg) < 0) {
-		printf("Error to get message buffer\n");
-		return -1;
-	}
-
-	*(enum l2fwd_cmd *)msg = cmd;
-
-	if (rte_ring_enqueue(res->ring[fd], msg) != 0) {
-		printf("Enqueue error\n");
-		rte_mempool_put(message_pool, msg);
-		return -1;
-	}
-
-	return 0;
-}
-
-/* Get command from pair in paired master and slave ring */
-static inline int
-getcmd(unsigned slaveid, enum l2fwd_cmd *cmd, int is_master)
-{
-	struct lcore_resource_struct *res = &lcore_resource[slaveid];
-	void *msg;
-	int fd = !!is_master;
-	int ret;
-	/* Only check master, it must be enabled and running if it is slave */
-	if (is_master && (!res->enabled))
-		return -1;
-
-	if (res->ring[fd] == NULL)
-		return -1;
-
-	ret = rte_ring_dequeue(res->ring[fd], &msg);
-
-	if (ret == 0) {
-		*cmd = *(enum l2fwd_cmd *)msg;
-		rte_mempool_put(message_pool, msg);
-	}
-	return ret;
-}
-
-/* Master send command to slave and wait until ack received or error met */
-static int
-master_sendcmd_with_ack(unsigned slaveid, enum l2fwd_cmd cmd)
-{
-	enum l2fwd_cmd ack_cmd;
-	int ret = -1;
-
-	if (sendcmd(slaveid, cmd, 1) != 0)
-		rte_exit(EXIT_FAILURE, "Failed to send message\n");
-
-	/* Get ack */
-	while (1) {
-		ret = getcmd(slaveid, &ack_cmd, 1);
-		if (ret == 0 && cmd == ack_cmd)
-			break;
-
-		/* If slave not running yet, return an error */
-		if (flib_query_slave_status(slaveid) != ST_RUN) {
-			ret = -ENOENT;
-			break;
-		}
-	}
-
-	return ret;
-}
-
-/* restart all port that assigned to that slave lcore */
-static int
-reset_slave_all_ports(unsigned slaveid)
-{
-	struct lcore_resource_struct *slave = &lcore_resource[slaveid];
-	int i, ret = 0;
-
-	/* stop/start port */
-	for (i = 0; i < slave->port_num; i++) {
-		char buf_name[RTE_MEMPOOL_NAMESIZE];
-		struct rte_mempool *pool;
-		printf("Stop port :%d\n", slave->port[i]);
-		rte_eth_dev_stop(slave->port[i]);
-		snprintf(buf_name, RTE_MEMPOOL_NAMESIZE, MBUF_NAME, slave->port[i]);
-		pool = rte_mempool_lookup(buf_name);
-		if (pool)
-			printf("Port %d mempool free object is %u(%u)\n", slave->port[i],
-				rte_mempool_avail_count(pool),
-				(unsigned int)NB_MBUF);
-		else
-			printf("Can't find mempool %s\n", buf_name);
-
-		printf("Start port :%d\n", slave->port[i]);
-		ret = rte_eth_dev_start(slave->port[i]);
-		if (ret != 0)
-			break;
-	}
-	return ret;
-}
-
-static int
-reset_shared_structures(unsigned slaveid)
-{
-	int ret;
-	/* Only port are shared resource here */
-	ret = reset_slave_all_ports(slaveid);
-
-	return ret;
-}
-
-/**
- * Call this function to re-create resource that needed for slave process that
- * exited in last instance
- **/
-static int
-init_slave_res(unsigned slaveid)
-{
-	struct lcore_resource_struct *slave = &lcore_resource[slaveid];
-	enum l2fwd_cmd cmd;
-
-	if (!slave->enabled) {
-		printf("Something wrong with lcore=%u enabled=%d\n",slaveid,
-			slave->enabled);
-		return -1;
-	}
-
-	/* Initialize ring */
-	if (create_ms_ring(slaveid) != 0)
-		rte_exit(EXIT_FAILURE, "failed to create ring for slave %u\n",
-				slaveid);
-
-	/* drain un-read buffer if have */
-	while (getcmd(slaveid, &cmd, 1) == 0);
-	while (getcmd(slaveid, &cmd, 0) == 0);
-
-	return 0;
-}
-
-static int
-recreate_one_slave(unsigned slaveid)
-{
-	int ret = 0;
-	/* Re-initialize resource for stalled slave */
-	if ((ret = init_slave_res(slaveid)) != 0) {
-		printf("Init slave=%u failed\n", slaveid);
-		return ret;
-	}
-
-	if ((ret = flib_remote_launch(l2fwd_launch_one_lcore, NULL, slaveid))
-		!= 0)
-		printf("Launch slave %u failed\n", slaveid);
-
-	return ret;
-}
-
-/**
- * remapping resource belong to slave_id to new lcore that gets from flib_assign_lcore_id(),
- * used only floating process option applied.
- *
- * @param slaveid
- *   original lcore_id that apply for remapping
- */
-static void
-remapping_slave_resource(unsigned slaveid, unsigned map_id)
-{
-
-	/* remapping lcore_resource */
-	memcpy(&lcore_resource[map_id], &lcore_resource[slaveid],
-			sizeof(struct lcore_resource_struct));
-
-	/* remapping lcore_queue_conf */
-	memcpy(&lcore_queue_conf[map_id], &lcore_queue_conf[slaveid],
-			sizeof(struct lcore_queue_conf));
-}
-
-static int
-reset_pair(unsigned slaveid, unsigned pairid)
-{
-	int ret;
-	if ((ret = reset_shared_structures(slaveid)) != 0)
-		goto back;
-
-	if((ret = reset_shared_structures(pairid)) != 0)
-		goto back;
-
-	if (float_proc) {
-		unsigned map_id = mapping_id[slaveid];
-
-		if (map_id != INVALID_MAPPING_ID) {
-			printf("%u return mapping id %u\n", slaveid, map_id);
-			flib_free_lcore_id(map_id);
-			mapping_id[slaveid] = INVALID_MAPPING_ID;
-		}
-
-		map_id = mapping_id[pairid];
-		if (map_id != INVALID_MAPPING_ID) {
-			printf("%u return mapping id %u\n", pairid, map_id);
-			flib_free_lcore_id(map_id);
-			mapping_id[pairid] = INVALID_MAPPING_ID;
-		}
-	}
-
-	if((ret = recreate_one_slave(slaveid)) != 0)
-		goto back;
-
-	ret = recreate_one_slave(pairid);
-
-back:
-	return ret;
-}
-
-static void
-slave_exit_cb(unsigned slaveid, __attribute__((unused))int stat)
-{
-	struct lcore_resource_struct *slave = &lcore_resource[slaveid];
-
-	printf("Get slave %u leave info\n", slaveid);
-	if (!slave->enabled) {
-		printf("Lcore=%u not registered for it's exit\n", slaveid);
-		return;
-	}
-	rte_spinlock_lock(&res_lock);
-
-	/* Change the state and wait master to start them */
-	slave->flags = SLAVE_RECREATE_FLAG;
-
-	rte_spinlock_unlock(&res_lock);
-}
-
-static void
-l2fwd_simple_forward(struct rte_mbuf *m, unsigned portid)
-{
-	struct ether_hdr *eth;
-	void *tmp;
-	unsigned dst_port;
-	int sent;
-	struct rte_eth_dev_tx_buffer *buffer;
-
-	dst_port = l2fwd_dst_ports[portid];
-	eth = rte_pktmbuf_mtod(m, struct ether_hdr *);
-
-	/* 02:00:00:00:00:xx */
-	tmp = &eth->d_addr.addr_bytes[0];
-	*((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dst_port << 40);
-
-	/* src addr */
-	ether_addr_copy(&l2fwd_ports_eth_addr[dst_port], &eth->s_addr);
-
-	buffer = tx_buffer[dst_port];
-	sent = rte_eth_tx_buffer(dst_port, 0, buffer, m);
-	if (sent)
-		port_statistics[dst_port].tx += sent;
-}
-
-/* main processing loop */
-static void
-l2fwd_main_loop(void)
-{
-	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
-	struct rte_mbuf *m;
-	int sent;
-	unsigned lcore_id;
-	uint64_t prev_tsc, diff_tsc, cur_tsc;
-	unsigned i, j, portid, nb_rx;
-	struct lcore_queue_conf *qconf;
-	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S *
-			BURST_TX_DRAIN_US;
-	struct rte_eth_dev_tx_buffer *buffer;
-
-	prev_tsc = 0;
-
-	lcore_id = rte_lcore_id();
-
-	qconf = &lcore_queue_conf[lcore_id];
-
-	if (qconf->n_rx_port == 0) {
-		RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id);
-		return;
-	}
-
-	RTE_LOG(INFO, L2FWD, "entering main loop on lcore %u\n", lcore_id);
-
-	for (i = 0; i < qconf->n_rx_port; i++) {
-		portid = qconf->rx_port_list[i];
-		RTE_LOG(INFO, L2FWD, " -- lcoreid=%u portid=%u\n", lcore_id,
-			portid);
-	}
-
-	while (1) {
-		enum l2fwd_cmd cmd;
-		cur_tsc = rte_rdtsc();
-
-		if (unlikely(getcmd(lcore_id, &cmd, 0) == 0)) {
-			sendcmd(lcore_id, cmd, 0);
-
-			/* If get stop command, stop forwarding and exit */
-			if (cmd == CMD_STOP) {
-				return;
-			}
-		}
-
-		/*
-		 * TX burst queue drain
-		 */
-		diff_tsc = cur_tsc - prev_tsc;
-		if (unlikely(diff_tsc > drain_tsc)) {
-
-			for (i = 0; i < qconf->n_rx_port; i++) {
-
-				portid = l2fwd_dst_ports[qconf->rx_port_list[i]];
-				buffer = tx_buffer[portid];
-
-				sent = rte_eth_tx_buffer_flush(portid, 0, buffer);
-				if (sent)
-					port_statistics[portid].tx += sent;
-
-			}
-
-			prev_tsc = cur_tsc;
-		}
-
-		/*
-		 * Read packet from RX queues
-		 */
-		for (i = 0; i < qconf->n_rx_port; i++) {
-
-			portid = qconf->rx_port_list[i];
-			nb_rx = rte_eth_rx_burst((uint8_t) portid, 0,
-						 pkts_burst, MAX_PKT_BURST);
-
-			port_statistics[portid].rx += nb_rx;
-
-			for (j = 0; j < nb_rx; j++) {
-				m = pkts_burst[j];
-				rte_prefetch0(rte_pktmbuf_mtod(m, void *));
-				l2fwd_simple_forward(m, portid);
-			}
-		}
-	}
-}
-
-static int
-l2fwd_launch_one_lcore(__attribute__((unused)) void *dummy)
-{
-	unsigned lcore_id = rte_lcore_id();
-
-	if (float_proc) {
-		unsigned flcore_id;
-
-		/* Change it to floating process, also change it's lcore_id */
-		clear_cpu_affinity();
-		RTE_PER_LCORE(_lcore_id) = 0;
-		/* Get a lcore_id */
-		if (flib_assign_lcore_id() < 0 ) {
-			printf("flib_assign_lcore_id failed\n");
-			return -1;
-		}
-		flcore_id = rte_lcore_id();
-		/* Set mapping id, so master can return it after slave exited */
-		mapping_id[lcore_id] = flcore_id;
-		printf("Org lcore_id = %u, cur lcore_id = %u\n",
-				lcore_id, flcore_id);
-		remapping_slave_resource(lcore_id, flcore_id);
-	}
-
-	l2fwd_main_loop();
-
-	/* return lcore_id before return */
-	if (float_proc) {
-		flib_free_lcore_id(rte_lcore_id());
-		mapping_id[lcore_id] = INVALID_MAPPING_ID;
-	}
-	return 0;
-}
-
-/* display usage */
-static void
-l2fwd_usage(const char *prgname)
-{
-	printf("%s [EAL options] -- -p PORTMASK -s COREMASK [-q NQ] -f\n"
-	       "  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
-	       "  -q NQ: number of queue (=ports) per lcore (default is 1)\n"
-	       "  -f use floating process which won't bind to any core to run\n"
-		   "  -T PERIOD: statistics will be refreshed each PERIOD seconds (0 to disable, 10 default, 86400 maximum)\n",
-	       prgname);
-}
-
-static int
-l2fwd_parse_portmask(const char *portmask)
-{
-	char *end = NULL;
-	unsigned long pm;
-
-	/* parse hexadecimal string */
-	pm = strtoul(portmask, &end, 16);
-	if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
-		return -1;
-
-	if (pm == 0)
-		return -1;
-
-	return pm;
-}
-
-static unsigned int
-l2fwd_parse_nqueue(const char *q_arg)
-{
-	char *end = NULL;
-	unsigned long n;
-
-	/* parse hexadecimal string */
-	n = strtoul(q_arg, &end, 10);
-	if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
-		return 0;
-	if (n == 0)
-		return 0;
-	if (n >= MAX_RX_QUEUE_PER_LCORE)
-		return 0;
-
-	return n;
-}
-
-static int
-l2fwd_parse_timer_period(const char *q_arg)
-{
-	char *end = NULL;
-	int n;
-
-	/* parse number string */
-	n = strtol(q_arg, &end, 10);
-	if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
-		return -1;
-	if (n >= MAX_TIMER_PERIOD)
-		return -1;
-
-	return n;
-}
-
-/* Parse the argument given in the command line of the application */
-static int
-l2fwd_parse_args(int argc, char **argv)
-{
-	int opt, ret;
-	char **argvopt;
-	int option_index;
-	char *prgname = argv[0];
-	static struct option lgopts[] = {
-		{NULL, 0, 0, 0}
-	};
-	int has_pmask = 0;
-
-	argvopt = argv;
-
-	while ((opt = getopt_long(argc, argvopt, "p:q:T:f",
-				  lgopts, &option_index)) != EOF) {
-
-		switch (opt) {
-		/* portmask */
-		case 'p':
-			l2fwd_enabled_port_mask = l2fwd_parse_portmask(optarg);
-			if (l2fwd_enabled_port_mask == 0) {
-				printf("invalid portmask\n");
-				l2fwd_usage(prgname);
-				return -1;
-			}
-			has_pmask = 1;
-			break;
-
-		/* nqueue */
-		case 'q':
-			l2fwd_rx_queue_per_lcore = l2fwd_parse_nqueue(optarg);
-			if (l2fwd_rx_queue_per_lcore == 0) {
-				printf("invalid queue number\n");
-				l2fwd_usage(prgname);
-				return -1;
-			}
-			break;
-
-		/* timer period */
-		case 'T':
-			timer_period = l2fwd_parse_timer_period(optarg) * 1000 * TIMER_MILLISECOND;
-			if (timer_period < 0) {
-				printf("invalid timer period\n");
-				l2fwd_usage(prgname);
-				return -1;
-			}
-			break;
-
-		/* use floating process */
-		case 'f':
-			float_proc = 1;
-			break;
-
-		/* long options */
-		case 0:
-			l2fwd_usage(prgname);
-			return -1;
-
-		default:
-			l2fwd_usage(prgname);
-			return -1;
-		}
-	}
-
-	if (optind >= 0)
-		argv[optind-1] = prgname;
-
-	if (!has_pmask) {
-		l2fwd_usage(prgname);
-		return -1;
-	}
-	ret = optind-1;
-	optind = 1; /* reset getopt lib */
-	return ret;
-}
-
-/* Check the link status of all ports in up to 9s, and print them finally */
-static void
-check_all_ports_link_status(uint32_t port_mask)
-{
-#define CHECK_INTERVAL 100 /* 100ms */
-#define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
-	uint16_t portid;
-	uint8_t count, all_ports_up, print_flag = 0;
-	struct rte_eth_link link;
-
-	printf("\nChecking link status");
-	fflush(stdout);
-	for (count = 0; count <= MAX_CHECK_TIME; count++) {
-		all_ports_up = 1;
-		RTE_ETH_FOREACH_DEV(portid) {
-			if ((port_mask & (1 << portid)) == 0)
-				continue;
-			memset(&link, 0, sizeof(link));
-			rte_eth_link_get_nowait(portid, &link);
-			/* print link status if flag set */
-			if (print_flag == 1) {
-				if (link.link_status)
-					printf(
-					"Port%d Link Up- speed %u Mbps- %s\n",
-					portid, link.link_speed,
-				(link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
-					("full-duplex") : ("half-duplex\n"));
-				else
-					printf("Port %d Link Down\n", portid);
-				continue;
-			}
-			/* clear all_ports_up flag if any link down */
-			if (link.link_status == ETH_LINK_DOWN) {
-				all_ports_up = 0;
-				break;
-			}
-		}
-		/* after finally printing all link status, get out */
-		if (print_flag == 1)
-			break;
-
-		if (all_ports_up == 0) {
-			printf(".");
-			fflush(stdout);
-			rte_delay_ms(CHECK_INTERVAL);
-		}
-
-		/* set the print_flag if all ports up or timeout */
-		if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
-			print_flag = 1;
-			printf("done\n");
-		}
-	}
-}
-
-int
-main(int argc, char **argv)
-{
-	struct lcore_queue_conf *qconf;
-	int ret;
-	uint16_t nb_ports;
-	uint16_t nb_ports_available = 0;
-	uint16_t portid, last_port;
-	unsigned rx_lcore_id;
-	unsigned nb_ports_in_mask = 0;
-	unsigned i;
-	uint64_t prev_tsc, diff_tsc, cur_tsc, timer_tsc;
-
-	/* Save cpu_affinity first, restore it in case it's floating process option */
-	if (get_cpu_affinity() != 0)
-		rte_exit(EXIT_FAILURE, "get_cpu_affinity error\n");
-
-	/* Also tries to set cpu affinity to detect whether  it will fail in child process */
-	if(clear_cpu_affinity() != 0)
-		rte_exit(EXIT_FAILURE, "clear_cpu_affinity error\n");
-
-	/* init EAL */
-	ret = rte_eal_init(argc, argv);
-	if (ret < 0)
-		rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
-	argc -= ret;
-	argv += ret;
-
-	/* parse application arguments (after the EAL ones) */
-	ret = l2fwd_parse_args(argc, argv);
-	if (ret < 0)
-		rte_exit(EXIT_FAILURE, "Invalid L2FWD arguments\n");
-
-	/*flib init */
-	if (flib_init() != 0)
-		rte_exit(EXIT_FAILURE, "flib init error");
-
-	/**
-	  * Allocated structures that slave lcore would change. For those that slaves are
-	  * read only, needn't use malloc to share and global or static variables is ok since
-	  * slave inherit all the knowledge that master initialized.
-	  **/
-	if (l2fwd_malloc_shared_struct() != 0)
-		rte_exit(EXIT_FAILURE, "malloc mem failed\n");
-
-	/* Initialize lcore_resource structures */
-	memset(lcore_resource, 0, sizeof(lcore_resource));
-	for (i = 0; i < RTE_MAX_LCORE; i++)
-		lcore_resource[i].lcore_id = i;
-
-	nb_ports = rte_eth_dev_count_avail();
-	if (nb_ports == 0)
-		rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n");
-
-	/* create the mbuf pool */
-	RTE_ETH_FOREACH_DEV(portid) {
-		/* skip ports that are not enabled */
-		if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
-			continue;
-		char buf_name[RTE_MEMPOOL_NAMESIZE];
-		snprintf(buf_name, RTE_MEMPOOL_NAMESIZE, MBUF_NAME, portid);
-		l2fwd_pktmbuf_pool[portid] =
-			rte_pktmbuf_pool_create(buf_name, NB_MBUF, 32,
-				0, MBUF_DATA_SIZE, rte_socket_id());
-		if (l2fwd_pktmbuf_pool[portid] == NULL)
-			rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
-
-		printf("Create mbuf %s\n", buf_name);
-	}
-
-	/* reset l2fwd_dst_ports */
-	for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++)
-		l2fwd_dst_ports[portid] = 0;
-	last_port = 0;
-
-	/*
-	 * Each logical core is assigned a dedicated TX queue on each port.
-	 */
-	RTE_ETH_FOREACH_DEV(portid) {
-		/* skip ports that are not enabled */
-		if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
-			continue;
-
-		if (nb_ports_in_mask % 2) {
-			l2fwd_dst_ports[portid] = last_port;
-			l2fwd_dst_ports[last_port] = portid;
-		}
-		else
-			last_port = portid;
-
-		nb_ports_in_mask++;
-	}
-	if (nb_ports_in_mask % 2) {
-		printf("Notice: odd number of ports in portmask.\n");
-		l2fwd_dst_ports[last_port] = last_port;
-	}
-
-	rx_lcore_id = 0;
-	qconf = NULL;
-
-	/* Initialize the port/queue configuration of each logical core */
-	RTE_ETH_FOREACH_DEV(portid) {
-		struct lcore_resource_struct *res;
-		/* skip ports that are not enabled */
-		if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
-			continue;
-
-		/* get the lcore_id for this port */
-		/* skip master lcore */
-		while (rte_lcore_is_enabled(rx_lcore_id) == 0 ||
-			   rte_get_master_lcore() == rx_lcore_id ||
-		       lcore_queue_conf[rx_lcore_id].n_rx_port ==
-		       l2fwd_rx_queue_per_lcore) {
-
-			rx_lcore_id++;
-			if (rx_lcore_id >= RTE_MAX_LCORE)
-				rte_exit(EXIT_FAILURE, "Not enough cores\n");
-		}
-
-		if (qconf != &lcore_queue_conf[rx_lcore_id])
-			/* Assigned a new logical core in the loop above. */
-			qconf = &lcore_queue_conf[rx_lcore_id];
-
-		qconf->rx_port_list[qconf->n_rx_port] = portid;
-		qconf->n_rx_port++;
-
-		/* Save the port resource info into lcore_resource strucutres */
-		res = &lcore_resource[rx_lcore_id];
-		res->enabled = 1;
-		res->port[res->port_num++] = portid;
-
-		printf("Lcore %u: RX port %u\n", rx_lcore_id, (unsigned) portid);
-	}
-
-	/* Initialise each port */
-	RTE_ETH_FOREACH_DEV(portid) {
-		struct rte_eth_rxconf rxq_conf;
-		struct rte_eth_txconf txq_conf;
-		struct rte_eth_conf local_port_conf = port_conf;
-
-		/* skip ports that are not enabled */
-		if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) {
-			printf("Skipping disabled port %u\n", (unsigned) portid);
-			continue;
-		}
-		nb_ports_available++;
-		/* init port */
-		printf("Initializing port %u... ", (unsigned) portid);
-		fflush(stdout);
-		rte_eth_dev_info_get(portid, &dev_info);
-		if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
-			local_port_conf.txmode.offloads |=
-				DEV_TX_OFFLOAD_MBUF_FAST_FREE;
-		ret = rte_eth_dev_configure(portid, 1, 1, &local_port_conf);
-		if (ret < 0)
-			rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n",
-				  ret, (unsigned) portid);
-
-		ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
-						       &nb_txd);
-		if (ret < 0)
-			rte_exit(EXIT_FAILURE,
-				 "rte_eth_dev_adjust_nb_rx_tx_desc: err=%d, port=%u\n",
-				 ret, (unsigned) portid);
-
-		rte_eth_macaddr_get(portid,&l2fwd_ports_eth_addr[portid]);
-
-		/* init one RX queue */
-		fflush(stdout);
-		rxq_conf = dev_info.default_rxconf;
-		rxq_conf.offloads = local_port_conf.rxmode.offloads;
-		ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
-					     rte_eth_dev_socket_id(portid),
-					     &rxq_conf,
-					     l2fwd_pktmbuf_pool[portid]);
-		if (ret < 0)
-			rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u\n",
-				  ret, (unsigned) portid);
-
-		/* init one TX queue on each port */
-		fflush(stdout);
-		txq_conf = dev_info.default_txconf;
-		txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
-		txq_conf.tx_offloads = local_port_conf.txmode.offloads;
-		ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
-				rte_eth_dev_socket_id(portid),
-				&txq_conf);
-		if (ret < 0)
-			rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup:err=%d, port=%u\n",
-				ret, (unsigned) portid);
-
-		/* Initialize TX buffers */
-		tx_buffer[portid] = rte_zmalloc_socket("tx_buffer",
-				RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST), 0,
-				rte_eth_dev_socket_id(portid));
-		if (tx_buffer[portid] == NULL)
-			rte_exit(EXIT_FAILURE, "Cannot allocate buffer for tx on port %u\n",
-					(unsigned) portid);
-
-		rte_eth_tx_buffer_init(tx_buffer[portid], MAX_PKT_BURST);
-
-		ret = rte_eth_tx_buffer_set_err_callback(tx_buffer[portid],
-				rte_eth_tx_buffer_count_callback,
-				&port_statistics[portid].dropped);
-		if (ret < 0)
-				rte_exit(EXIT_FAILURE, "Cannot set error callback for "
-						"tx buffer on port %u\n", (unsigned) portid);
-
-		/* Start device */
-		ret = rte_eth_dev_start(portid);
-		if (ret < 0)
-			rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
-				  ret, (unsigned) portid);
-
-		printf("done: \n");
-
-		rte_eth_promiscuous_enable(portid);
-
-		printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
-				(unsigned) portid,
-				l2fwd_ports_eth_addr[portid].addr_bytes[0],
-				l2fwd_ports_eth_addr[portid].addr_bytes[1],
-				l2fwd_ports_eth_addr[portid].addr_bytes[2],
-				l2fwd_ports_eth_addr[portid].addr_bytes[3],
-				l2fwd_ports_eth_addr[portid].addr_bytes[4],
-				l2fwd_ports_eth_addr[portid].addr_bytes[5]);
-
-		/* initialize port stats */
-		//memset(&port_statistics, 0, sizeof(port_statistics));
-	}
-
-	if (!nb_ports_available) {
-		rte_exit(EXIT_FAILURE,
-			"All available ports are disabled. Please set portmask.\n");
-	}
-
-	check_all_ports_link_status(l2fwd_enabled_port_mask);
-
-	/* Record pair lcore */
-	/**
-	 * Since l2fwd example would create pair between different neighbour port, that's
-	 * port 0 receive and forward to port 1, the same to port 1, these 2 ports will have
-	 * dependency. If one port stopped working (killed, for example), the port need to
-	 * be stopped/started again. During the time, another port need to wait until stop/start
-	 * procedure completed. So, record the pair relationship for those lcores working
-	 * on ports.
-	 **/
-	RTE_ETH_FOREACH_DEV(portid) {
-		uint32_t pair_port;
-		unsigned lcore = 0, pair_lcore = 0;
-		unsigned j, find_lcore, find_pair_lcore;
-		/* skip ports that are not enabled */
-		if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
-			continue;
-
-		/* Find pair ports' lcores */
-		find_lcore = find_pair_lcore = 0;
-		pair_port = l2fwd_dst_ports[portid];
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
-			if (!rte_lcore_is_enabled(i))
-				continue;
-			for (j = 0; j < lcore_queue_conf[i].n_rx_port;j++) {
-				if (lcore_queue_conf[i].rx_port_list[j] == portid) {
-					lcore = i;
-					find_lcore = 1;
-					break;
-				}
-				if (lcore_queue_conf[i].rx_port_list[j] == pair_port) {
-					pair_lcore = i;
-					find_pair_lcore = 1;
-					break;
-				}
-			}
-			if (find_lcore && find_pair_lcore)
-				break;
-		}
-		if (!find_lcore || !find_pair_lcore)
-			rte_exit(EXIT_FAILURE, "Not find port=%d pair\n", portid);
-
-		printf("lcore %u and %u paired\n", lcore, pair_lcore);
-		lcore_resource[lcore].pair_id = pair_lcore;
-		lcore_resource[pair_lcore].pair_id = lcore;
-	}
-
-	/* Create message buffer for all master and slave */
-	message_pool = rte_mempool_create("ms_msg_pool",
-			   NB_CORE_MSGBUF * RTE_MAX_LCORE,
-			   sizeof(enum l2fwd_cmd), NB_CORE_MSGBUF / 2,
-			   0, NULL, NULL, NULL, NULL, rte_socket_id(), 0);
-
-	if (message_pool == NULL)
-		rte_exit(EXIT_FAILURE, "Create msg mempool failed\n");
-
-	/* Create ring for each master and slave pair, also register cb when slave leaves */
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		/**
-		 * Only create ring and register slave_exit cb in case that core involved into
-		 * packet forwarding
-		 **/
-		if (lcore_resource[i].enabled) {
-			/* Create ring for master and slave communication */
-			ret = create_ms_ring(i);
-			if (ret != 0)
-				rte_exit(EXIT_FAILURE, "Create ring for lcore=%u failed",
-				i);
-
-			if (flib_register_slave_exit_notify(i,
-				slave_exit_cb) != 0)
-				rte_exit(EXIT_FAILURE,
-						"Register master_trace_slave_exit failed");
-		}
-	}
-
-	/* launch per-lcore init on every lcore except master */
-	flib_mp_remote_launch(l2fwd_launch_one_lcore, NULL, SKIP_MASTER);
-
-	/* print statistics 10 second */
-	prev_tsc = cur_tsc = rte_rdtsc();
-	timer_tsc = 0;
-	while (1) {
-		sleep(1);
-		cur_tsc = rte_rdtsc();
-		diff_tsc = cur_tsc - prev_tsc;
-		/* if timer is enabled */
-		if (timer_period > 0) {
-
-			/* advance the timer */
-			timer_tsc += diff_tsc;
-
-			/* if timer has reached its timeout */
-			if (unlikely(timer_tsc >= (uint64_t) timer_period)) {
-
-				print_stats();
-				/* reset the timer */
-				timer_tsc = 0;
-			}
-		}
-
-		prev_tsc = cur_tsc;
-
-		/* Check any slave need restart or recreate */
-		rte_spinlock_lock(&res_lock);
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
-			struct lcore_resource_struct *res  = &lcore_resource[i];
-			struct lcore_resource_struct *pair = &lcore_resource[res->pair_id];
-
-			/* If find slave exited, try to reset pair */
-			if (res->enabled && res->flags && pair->enabled) {
-				if (!pair->flags) {
-					master_sendcmd_with_ack(pair->lcore_id, CMD_STOP);
-					rte_spinlock_unlock(&res_lock);
-					sleep(1);
-					rte_spinlock_lock(&res_lock);
-					if (pair->flags)
-						continue;
-				}
-				if (reset_pair(res->lcore_id, pair->lcore_id) != 0)
-					rte_exit(EXIT_FAILURE, "failed to reset slave");
-				res->flags  = 0;
-				pair->flags = 0;
-			}
-		}
-		rte_spinlock_unlock(&res_lock);
-	}
-
-}
diff --git a/examples/multi_process/symmetric_mp/main.c b/examples/multi_process/symmetric_mp/main.c
index 16f21a18..c6c6a537 100644
--- a/examples/multi_process/symmetric_mp/main.c
+++ b/examples/multi_process/symmetric_mp/main.c
@@ -178,7 +178,6 @@ smp_port_init(uint16_t port, struct rte_mempool *mbuf_pool,
 			.rxmode = {
 				.mq_mode	= ETH_MQ_RX_RSS,
 				.split_hdr_size = 0,
-				.ignore_offload_bitfield = 1,
 				.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
 					     DEV_RX_OFFLOAD_CRC_STRIP),
 			},
@@ -200,6 +199,7 @@ smp_port_init(uint16_t port, struct rte_mempool *mbuf_pool,
 	uint16_t q;
 	uint16_t nb_rxd = RX_RING_SIZE;
 	uint16_t nb_txd = TX_RING_SIZE;
+	uint64_t rss_hf_tmp;
 
 	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
 		return 0;
@@ -216,6 +216,17 @@ smp_port_init(uint16_t port, struct rte_mempool *mbuf_pool,
 	if (info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 		port_conf.txmode.offloads |=
 			DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+	rss_hf_tmp = port_conf.rx_adv_conf.rss_conf.rss_hf;
+	port_conf.rx_adv_conf.rss_conf.rss_hf &= info.flow_type_rss_offloads;
+	if (port_conf.rx_adv_conf.rss_conf.rss_hf != rss_hf_tmp) {
+		printf("Port %u modified RSS hash function based on hardware support,"
+			"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+			port,
+			rss_hf_tmp,
+			port_conf.rx_adv_conf.rss_conf.rss_hf);
+	}
+
 	retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
 	if (retval < 0)
 		return retval;
@@ -236,7 +247,6 @@ smp_port_init(uint16_t port, struct rte_mempool *mbuf_pool,
 	}
 
 	txq_conf = info.default_txconf;
-	txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txq_conf.offloads = port_conf.txmode.offloads;
 	for (q = 0; q < tx_rings; q ++) {
 		retval = rte_eth_tx_queue_setup(port, q, nb_txd,
diff --git a/examples/netmap_compat/bridge/Makefile b/examples/netmap_compat/bridge/Makefile
index a7c9c14a..856c847b 100644
--- a/examples/netmap_compat/bridge/Makefile
+++ b/examples/netmap_compat/bridge/Makefile
@@ -10,7 +10,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(info This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 all:
diff --git a/examples/netmap_compat/bridge/bridge.c b/examples/netmap_compat/bridge/bridge.c
index cb1882e6..7afca28c 100644
--- a/examples/netmap_compat/bridge/bridge.c
+++ b/examples/netmap_compat/bridge/bridge.c
@@ -26,7 +26,6 @@
 struct rte_eth_conf eth_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
diff --git a/examples/netmap_compat/lib/compat_netmap.c b/examples/netmap_compat/lib/compat_netmap.c
index af3dd223..0be0663e 100644
--- a/examples/netmap_compat/lib/compat_netmap.c
+++ b/examples/netmap_compat/lib/compat_netmap.c
@@ -708,7 +708,6 @@ rte_netmap_init_port(uint16_t portid, const struct rte_netmap_port_conf *conf)
 	rxq_conf = dev_info.default_rxconf;
 	rxq_conf.offloads = conf->eth_conf->rxmode.offloads;
 	txq_conf = dev_info.default_txconf;
-	txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txq_conf.offloads = conf->eth_conf->txmode.offloads;
 	for (i = 0; i < conf->nr_tx_rings; i++) {
 		ret = rte_eth_tx_queue_setup(portid, i, tx_slots,
diff --git a/examples/packet_ordering/main.c b/examples/packet_ordering/main.c
index 7ace7d10..149bfdd0 100644
--- a/examples/packet_ordering/main.c
+++ b/examples/packet_ordering/main.c
@@ -35,11 +35,7 @@ volatile uint8_t quit_signal;
 
 static struct rte_mempool *mbuf_pool;
 
-static struct rte_eth_conf port_conf_default = {
-	.rxmode = {
-		.ignore_offload_bitfield = 1,
-	},
-};
+static struct rte_eth_conf port_conf_default;
 
 struct worker_thread_args {
 	struct rte_ring *ring_in;
@@ -293,7 +289,6 @@ configure_eth_port(uint16_t port_id)
 	}
 
 	txconf = dev_info.default_txconf;
-	txconf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txconf.offloads = port_conf.txmode.offloads;
 	for (q = 0; q < txRings; q++) {
 		ret = rte_eth_tx_queue_setup(port_id, q, nb_txd,
diff --git a/examples/performance-thread/common/lthread_pool.h b/examples/performance-thread/common/lthread_pool.h
index 315a2e21..6f93775f 100644
--- a/examples/performance-thread/common/lthread_pool.h
+++ b/examples/performance-thread/common/lthread_pool.h
@@ -1,69 +1,7 @@
 /*
- *-
- *   BSD LICENSE
- *
- *   Copyright(c) 2015 Intel Corporation. All rights reserved.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/*
- * Some portions of this software is derived from the producer
- * consumer queues described by Dmitry Vyukov and published  here
- * http://www.1024cores.net
- *
- * Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
- * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DMITRY VYUKOV OR CONTRIBUTORS
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
- * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * The views and conclusions contained in the software and documentation are
- * those of the authors and should not be interpreted as representing official
- * policies, either expressed or implied, of Dmitry Vyukov.
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 Intel Corporation.
+ * Copyright 2010-2011 Dmitry Vyukov
  */
 
 #ifndef LTHREAD_POOL_H_
diff --git a/examples/performance-thread/common/lthread_queue.h b/examples/performance-thread/common/lthread_queue.h
index 833ed92b..5b63ba22 100644
--- a/examples/performance-thread/common/lthread_queue.h
+++ b/examples/performance-thread/common/lthread_queue.h
@@ -1,69 +1,7 @@
 /*
- *-
- *   BSD LICENSE
- *
- *   Copyright(c) 2015 Intel Corporation. All rights reserved.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/*
- * Some portions of this software is derived from the producer
- * consumer queues described by Dmitry Vyukov and published  here
- * http://www.1024cores.net
- *
- * Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
- * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DMITRY VYUKOV OR CONTRIBUTORS
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
- * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * The views and conclusions contained in the software and documentation are
- * those of the authors and should not be interpreted as representing official
- * policies, either expressed or implied, of Dmitry Vyukov.
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2015 Intel Corporation.
+ * Copyright 2010-2011 Dmitry Vyukov
  */
 
 #ifndef LTHREAD_QUEUE_H_
diff --git a/examples/performance-thread/l3fwd-thread/main.c b/examples/performance-thread/l3fwd-thread/main.c
index 40d80723..5392fcea 100644
--- a/examples/performance-thread/l3fwd-thread/main.c
+++ b/examples/performance-thread/l3fwd-thread/main.c
@@ -306,7 +306,6 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode = ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
 			     DEV_RX_OFFLOAD_CRC_STRIP),
 	},
@@ -3551,6 +3550,18 @@ main(int argc, char **argv)
 		if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 			local_port_conf.txmode.offloads |=
 				DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+			dev_info.flow_type_rss_offloads;
+		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+				port_conf.rx_adv_conf.rss_conf.rss_hf) {
+			printf("Port %u modified RSS hash function based on hardware support,"
+				"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+				portid,
+				port_conf.rx_adv_conf.rss_conf.rss_hf,
+				local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+		}
+
 		ret = rte_eth_dev_configure(portid, nb_rx_queue,
 					(uint16_t)n_tx_queue, &local_port_conf);
 		if (ret < 0)
@@ -3597,7 +3608,6 @@ main(int argc, char **argv)
 			fflush(stdout);
 
 			txconf = &dev_info.default_txconf;
-			txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 			txconf->offloads = local_port_conf.txmode.offloads;
 			ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
 						     socketid, txconf);
diff --git a/examples/ptpclient/ptpclient.c b/examples/ptpclient/ptpclient.c
index c44013bc..82ae71c1 100644
--- a/examples/ptpclient/ptpclient.c
+++ b/examples/ptpclient/ptpclient.c
@@ -50,7 +50,6 @@ static uint8_t ptp_enabled_ports[RTE_MAX_ETHPORTS];
 static const struct rte_eth_conf port_conf_default = {
 	.rxmode = {
 		.max_rx_pkt_len = ETHER_MAX_LEN,
-		.ignore_offload_bitfield = 1,
 	},
 };
 
@@ -217,11 +216,9 @@ port_init(uint16_t port, struct rte_mempool *mbuf_pool)
 
 	/* Allocate and set up 1 TX queue per Ethernet port. */
 	for (q = 0; q < tx_rings; q++) {
-		/* Setup txq_flags */
 		struct rte_eth_txconf *txconf;
 
 		txconf = &dev_info.default_txconf;
-		txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 		txconf->offloads = port_conf.txmode.offloads;
 
 		retval = rte_eth_tx_queue_setup(port, q, nb_txd,
diff --git a/examples/qos_meter/Makefile b/examples/qos_meter/Makefile
index 6da24076..46341b1a 100644
--- a/examples/qos_meter/Makefile
+++ b/examples/qos_meter/Makefile
@@ -50,7 +50,6 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 
diff --git a/examples/qos_meter/main.c b/examples/qos_meter/main.c
index 42cf4b29..5cf4e9df 100644
--- a/examples/qos_meter/main.c
+++ b/examples/qos_meter/main.c
@@ -56,7 +56,6 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode	= ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
 			     DEV_RX_OFFLOAD_CRC_STRIP),
 	},
@@ -333,6 +332,17 @@ main(int argc, char **argv)
 	rte_eth_dev_info_get(port_rx, &dev_info);
 	if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 		conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+	conf.rx_adv_conf.rss_conf.rss_hf &= dev_info.flow_type_rss_offloads;
+	if (conf.rx_adv_conf.rss_conf.rss_hf !=
+			port_conf.rx_adv_conf.rss_conf.rss_hf) {
+		printf("Port %u modified RSS hash function based on hardware support,"
+			"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+			port_rx,
+			port_conf.rx_adv_conf.rss_conf.rss_hf,
+			conf.rx_adv_conf.rss_conf.rss_hf);
+	}
+
 	ret = rte_eth_dev_configure(port_rx, 1, 1, &conf);
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Port %d configuration error (%d)\n", port_rx, ret);
@@ -351,7 +361,6 @@ main(int argc, char **argv)
 		rte_exit(EXIT_FAILURE, "Port %d RX queue setup error (%d)\n", port_rx, ret);
 
 	txq_conf = dev_info.default_txconf;
-	txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txq_conf.offloads = conf.txmode.offloads;
 	ret = rte_eth_tx_queue_setup(port_rx, NIC_TX_QUEUE, nb_txd,
 				rte_eth_dev_socket_id(port_rx),
@@ -363,6 +372,17 @@ main(int argc, char **argv)
 	rte_eth_dev_info_get(port_tx, &dev_info);
 	if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 		conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+	conf.rx_adv_conf.rss_conf.rss_hf &= dev_info.flow_type_rss_offloads;
+	if (conf.rx_adv_conf.rss_conf.rss_hf !=
+			port_conf.rx_adv_conf.rss_conf.rss_hf) {
+		printf("Port %u modified RSS hash function based on hardware support,"
+			"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+			port_tx,
+			port_conf.rx_adv_conf.rss_conf.rss_hf,
+			conf.rx_adv_conf.rss_conf.rss_hf);
+	}
+
 	ret = rte_eth_dev_configure(port_tx, 1, 1, &conf);
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Port %d configuration error (%d)\n", port_tx, ret);
@@ -383,7 +403,6 @@ main(int argc, char **argv)
 		rte_exit(EXIT_FAILURE, "Port %d RX queue setup error (%d)\n", port_tx, ret);
 
 	txq_conf = dev_info.default_txconf;
-	txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txq_conf.offloads = conf.txmode.offloads;
 	ret = rte_eth_tx_queue_setup(port_tx, NIC_TX_QUEUE, nb_txd,
 				rte_eth_dev_socket_id(port_tx),
diff --git a/examples/qos_meter/meson.build b/examples/qos_meter/meson.build
index 10cd4bc7..ef7779f2 100644
--- a/examples/qos_meter/meson.build
+++ b/examples/qos_meter/meson.build
@@ -7,7 +7,6 @@
 # DPDK instance, use 'make'
 
 deps += 'meter'
-allow_experimental_apis = true
 sources = files(
 	'main.c', 'rte_policer.c'
 )
diff --git a/examples/qos_sched/Makefile b/examples/qos_sched/Makefile
index 0f0a31ff..a7ecf978 100644
--- a/examples/qos_sched/Makefile
+++ b/examples/qos_sched/Makefile
@@ -48,7 +48,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(info This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 all:
diff --git a/examples/qos_sched/init.c b/examples/qos_sched/init.c
index c9e48797..94cbb26f 100644
--- a/examples/qos_sched/init.c
+++ b/examples/qos_sched/init.c
@@ -59,7 +59,6 @@ static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
@@ -96,7 +95,6 @@ app_init_port(uint16_t portid, struct rte_mempool *mp)
 	tx_conf.tx_free_thresh = 0;
 	tx_conf.tx_rs_thresh = 0;
 	tx_conf.tx_deferred_start = 0;
-	tx_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 
 	/* init port */
 	RTE_LOG(INFO, APP, "Initializing port %"PRIu16"... ", portid);
diff --git a/examples/quota_watermark/qw/init.c b/examples/quota_watermark/qw/init.c
index 00725bc9..19164385 100644
--- a/examples/quota_watermark/qw/init.c
+++ b/examples/quota_watermark/qw/init.c
@@ -24,7 +24,6 @@
 static struct rte_eth_conf port_conf = {
 		.rxmode = {
 			.split_hdr_size = 0,
-			.ignore_offload_bitfield = 1,
 			.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 		},
 		.txmode = {
@@ -82,7 +81,6 @@ void configure_eth_port(uint16_t port_id)
 
 	/* Initialize the port's TX queue */
 	txq_conf = dev_info.default_txconf;
-	txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txq_conf.offloads = local_port_conf.txmode.offloads;
 	ret = rte_eth_tx_queue_setup(port_id, 0, nb_txd,
 			rte_eth_dev_socket_id(port_id),
diff --git a/examples/rxtx_callbacks/main.c b/examples/rxtx_callbacks/main.c
index e63ea288..2058be62 100644
--- a/examples/rxtx_callbacks/main.c
+++ b/examples/rxtx_callbacks/main.c
@@ -20,7 +20,6 @@
 static const struct rte_eth_conf port_conf_default = {
 	.rxmode = {
 		.max_rx_pkt_len = ETHER_MAX_LEN,
-		.ignore_offload_bitfield = 1,
 	},
 };
 
@@ -104,7 +103,6 @@ port_init(uint16_t port, struct rte_mempool *mbuf_pool)
 	}
 
 	txconf = dev_info.default_txconf;
-	txconf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txconf.offloads = port_conf.txmode.offloads;
 	for (q = 0; q < tx_rings; q++) {
 		retval = rte_eth_tx_queue_setup(port, q, nb_txd,
diff --git a/examples/server_node_efd/server/Makefile b/examples/server_node_efd/server/Makefile
index cbb91ebe..df6614c6 100644
--- a/examples/server_node_efd/server/Makefile
+++ b/examples/server_node_efd/server/Makefile
@@ -10,7 +10,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV), "linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(error This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 endif
diff --git a/examples/server_node_efd/server/init.c b/examples/server_node_efd/server/init.c
index 7dfe2fa2..af5a18e2 100644
--- a/examples/server_node_efd/server/init.c
+++ b/examples/server_node_efd/server/init.c
@@ -97,7 +97,6 @@ init_port(uint16_t port_num)
 	struct rte_eth_conf port_conf = {
 		.rxmode = {
 			.mq_mode = ETH_MQ_RX_RSS,
-			.ignore_offload_bitfield = 1,
 		},
 	};
 	const uint16_t rx_rings = 1, tx_rings = num_nodes;
@@ -139,7 +138,6 @@ init_port(uint16_t port_num)
 	}
 
 	txconf = dev_info.default_txconf;
-	txconf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txconf.offloads = port_conf.txmode.offloads;
 	for (q = 0; q < tx_rings; q++) {
 		retval = rte_eth_tx_queue_setup(port_num, q, tx_ring_size,
diff --git a/examples/skeleton/basicfwd.c b/examples/skeleton/basicfwd.c
index 03bc3585..4aba1dc3 100644
--- a/examples/skeleton/basicfwd.c
+++ b/examples/skeleton/basicfwd.c
@@ -20,7 +20,6 @@
 static const struct rte_eth_conf port_conf_default = {
 	.rxmode = {
 		.max_rx_pkt_len = ETHER_MAX_LEN,
-		.ignore_offload_bitfield = 1,
 	},
 };
 
@@ -68,7 +67,6 @@ port_init(uint16_t port, struct rte_mempool *mbuf_pool)
 	}
 
 	txconf = dev_info.default_txconf;
-	txconf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txconf.offloads = port_conf.txmode.offloads;
 	/* Allocate and set up 1 TX queue per Ethernet port. */
 	for (q = 0; q < tx_rings; q++) {
diff --git a/examples/tep_termination/Makefile b/examples/tep_termination/Makefile
index 2b93446c..8ec1a38e 100644
--- a/examples/tep_termination/Makefile
+++ b/examples/tep_termination/Makefile
@@ -52,7 +52,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(error This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 endif
diff --git a/examples/tep_termination/vxlan_setup.c b/examples/tep_termination/vxlan_setup.c
index 299c29d2..b99ab97d 100644
--- a/examples/tep_termination/vxlan_setup.c
+++ b/examples/tep_termination/vxlan_setup.c
@@ -69,7 +69,6 @@ uint8_t tep_filter_type[] = {RTE_TUNNEL_FILTER_IMAC_TENID,
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
@@ -131,7 +130,6 @@ vxlan_port_init(uint16_t port, struct rte_mempool *mbuf_pool)
 
 	rxconf = &dev_info.default_rxconf;
 	txconf = &dev_info.default_txconf;
-	txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 
 	if (!rte_eth_dev_is_valid_port(port))
 		return -1;
diff --git a/examples/vhost/Makefile b/examples/vhost/Makefile
index 67cc55b1..a2ea97a0 100644
--- a/examples/vhost/Makefile
+++ b/examples/vhost/Makefile
@@ -52,7 +52,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(info This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 all:
diff --git a/examples/vhost/main.c b/examples/vhost/main.c
index 1659ef31..2175c118 100644
--- a/examples/vhost/main.c
+++ b/examples/vhost/main.c
@@ -116,7 +116,6 @@ static struct rte_eth_conf vmdq_conf_default = {
 	.rxmode = {
 		.mq_mode        = ETH_MQ_RX_VMDQ_ONLY,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 		/*
 		 * VLAN strip is necessary for 1G NIC such as I350,
 		 * this fixes bug of ipv4 forwarding in guest can't
@@ -256,7 +255,6 @@ port_init(uint16_t port)
 	rxconf = &dev_info.default_rxconf;
 	txconf = &dev_info.default_txconf;
 	rxconf->rx_drop_en = 1;
-	txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 
 	/*configure the number of supported virtio devices based on VMDQ limits */
 	num_devices = dev_info.max_vmdq_pools;
diff --git a/examples/vhost_crypto/Makefile b/examples/vhost_crypto/Makefile
index 1bb65e8f..83d33101 100644
--- a/examples/vhost_crypto/Makefile
+++ b/examples/vhost_crypto/Makefile
@@ -10,7 +10,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(info This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 all:
diff --git a/examples/vhost_scsi/Makefile b/examples/vhost_scsi/Makefile
index 31bd2563..fa0cf727 100644
--- a/examples/vhost_scsi/Makefile
+++ b/examples/vhost_scsi/Makefile
@@ -51,7 +51,7 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
-ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 $(info This application can only operate in a linuxapp environment, \
 please change the definition of the RTE_TARGET environment variable)
 all:
diff --git a/examples/vm_power_manager/Makefile b/examples/vm_power_manager/Makefile
index ef2a9f95..13a5205b 100644
--- a/examples/vm_power_manager/Makefile
+++ b/examples/vm_power_manager/Makefile
@@ -19,7 +19,12 @@ APP = vm_power_mgr
 
 # all source are stored in SRCS-y
 SRCS-y := main.c vm_power_cli.c power_manager.c channel_manager.c
-SRCS-y += channel_monitor.c
+SRCS-y += channel_monitor.c parse.c
+ifeq ($(CONFIG_RTE_ARCH_X86_64),y)
+SRCS-y += oob_monitor_x86.c
+else
+SRCS-y += oob_monitor_nop.c
+endif
 
 CFLAGS += -O3 -I$(RTE_SDK)/lib/librte_power/
 CFLAGS += $(WERROR_FLAGS)
diff --git a/examples/vm_power_manager/channel_monitor.c b/examples/vm_power_manager/channel_monitor.c
index 73bddd99..7fa47ba9 100644
--- a/examples/vm_power_manager/channel_monitor.c
+++ b/examples/vm_power_manager/channel_monitor.c
@@ -27,6 +27,7 @@
 #include "channel_commands.h"
 #include "channel_manager.h"
 #include "power_manager.h"
+#include "oob_monitor.h"
 
 #define RTE_LOGTYPE_CHANNEL_MONITOR RTE_LOGTYPE_USER1
 
@@ -92,6 +93,10 @@ get_pcpu_to_control(struct policy *pol)
 	struct vm_info info;
 	int pcpu, count;
 	uint64_t mask_u64b;
+	struct core_info *ci;
+	int ret;
+
+	ci = get_core_info();
 
 	RTE_LOG(INFO, CHANNEL_MONITOR, "Looking for pcpu for %s\n",
 			pol->pkt.vm_name);
@@ -100,8 +105,22 @@ get_pcpu_to_control(struct policy *pol)
 	for (count = 0; count < pol->pkt.num_vcpu; count++) {
 		mask_u64b = info.pcpu_mask[pol->pkt.vcpu_to_control[count]];
 		for (pcpu = 0; mask_u64b; mask_u64b &= ~(1ULL << pcpu++)) {
-			if ((mask_u64b >> pcpu) & 1)
-				pol->core_share[count].pcpu = pcpu;
+			if ((mask_u64b >> pcpu) & 1) {
+				if (pol->pkt.policy_to_use == BRANCH_RATIO) {
+					ci->cd[pcpu].oob_enabled = 1;
+					ret = add_core_to_monitor(pcpu);
+					if (ret == 0)
+						printf("Monitoring pcpu %d via Branch Ratio\n",
+								pcpu);
+					else
+						printf("Failed to start OOB Monitoring pcpu %d\n",
+								pcpu);
+
+				} else {
+					pol->core_share[count].pcpu = pcpu;
+					printf("Monitoring pcpu %d\n", pcpu);
+				}
+			}
 		}
 	}
 }
diff --git a/examples/vm_power_manager/guest_cli/Makefile b/examples/vm_power_manager/guest_cli/Makefile
index d710e22d..8b1db861 100644
--- a/examples/vm_power_manager/guest_cli/Makefile
+++ b/examples/vm_power_manager/guest_cli/Makefile
@@ -14,7 +14,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 APP = guest_vm_power_mgr
 
 # all source are stored in SRCS-y
-SRCS-y := main.c vm_power_cli_guest.c
+SRCS-y := main.c vm_power_cli_guest.c parse.c
 
 CFLAGS += -O3 -I$(RTE_SDK)/lib/librte_power/
 CFLAGS += $(WERROR_FLAGS)
diff --git a/examples/vm_power_manager/guest_cli/main.c b/examples/vm_power_manager/guest_cli/main.c
index b17936d6..36365b12 100644
--- a/examples/vm_power_manager/guest_cli/main.c
+++ b/examples/vm_power_manager/guest_cli/main.c
@@ -2,23 +2,20 @@
  * Copyright(c) 2010-2014 Intel Corporation
  */
 
-/*
 #include <stdio.h>
-#include <string.h>
-#include <stdint.h>
-#include <sys/epoll.h>
-#include <fcntl.h>
-#include <unistd.h>
 #include <stdlib.h>
-#include <errno.h>
-*/
 #include <signal.h>
+#include <getopt.h>
+#include <string.h>
 
 #include <rte_lcore.h>
 #include <rte_power.h>
 #include <rte_debug.h>
+#include <rte_eal.h>
+#include <rte_log.h>
 
 #include "vm_power_cli_guest.h"
+#include "parse.h"
 
 static void
 sig_handler(int signo)
@@ -32,6 +29,136 @@ sig_handler(int signo)
 
 }
 
+#define MAX_HOURS 24
+
+/* Parse the argument given in the command line of the application */
+static int
+parse_args(int argc, char **argv)
+{
+	int opt, ret;
+	char **argvopt;
+	int option_index;
+	char *prgname = argv[0];
+	const struct option lgopts[] = {
+		{ "vm-name", required_argument, 0, 'n'},
+		{ "busy-hours", required_argument, 0, 'b'},
+		{ "quiet-hours", required_argument, 0, 'q'},
+		{ "port-list", required_argument, 0, 'p'},
+		{ "vcpu-list", required_argument, 0, 'l'},
+		{ "policy", required_argument, 0, 'o'},
+		{NULL, 0, 0, 0}
+	};
+	struct channel_packet *policy;
+	unsigned short int hours[MAX_HOURS];
+	unsigned short int cores[MAX_VCPU_PER_VM];
+	unsigned short int ports[MAX_VCPU_PER_VM];
+	int i, cnt, idx;
+
+	policy = get_policy();
+	set_policy_defaults(policy);
+
+	argvopt = argv;
+
+	while ((opt = getopt_long(argc, argvopt, "n:b:q:p:",
+				  lgopts, &option_index)) != EOF) {
+
+		switch (opt) {
+		/* portmask */
+		case 'n':
+			strcpy(policy->vm_name, optarg);
+			printf("Setting VM Name to [%s]\n", policy->vm_name);
+			break;
+		case 'b':
+		case 'q':
+			//printf("***Processing set using [%s]\n", optarg);
+			cnt = parse_set(optarg, hours, MAX_HOURS);
+			if (cnt < 0) {
+				printf("Invalid value passed to quiet/busy hours - [%s]\n",
+						optarg);
+				break;
+			}
+			idx = 0;
+			for (i = 0; i < MAX_HOURS; i++) {
+				if (hours[i]) {
+					if (opt == 'b') {
+						printf("***Busy Hour %d\n", i);
+						policy->timer_policy.busy_hours
+							[idx++] = i;
+					} else {
+						printf("***Quiet Hour %d\n", i);
+						policy->timer_policy.quiet_hours
+							[idx++] = i;
+					}
+				}
+			}
+			break;
+		case 'l':
+			cnt = parse_set(optarg, cores, MAX_VCPU_PER_VM);
+			if (cnt < 0) {
+				printf("Invalid value passed to vcpu-list - [%s]\n",
+						optarg);
+				break;
+			}
+			idx = 0;
+			for (i = 0; i < MAX_VCPU_PER_VM; i++) {
+				if (cores[i]) {
+					printf("***Using core %d\n", i);
+					policy->vcpu_to_control[idx++] = i;
+				}
+			}
+			policy->num_vcpu = idx;
+			printf("Total cores: %d\n", idx);
+			break;
+		case 'p':
+			cnt = parse_set(optarg, ports, MAX_VCPU_PER_VM);
+			if (cnt < 0) {
+				printf("Invalid value passed to port-list - [%s]\n",
+						optarg);
+				break;
+			}
+			idx = 0;
+			for (i = 0; i < MAX_VCPU_PER_VM; i++) {
+				if (ports[i]) {
+					printf("***Using port %d\n", i);
+					set_policy_mac(i, idx++);
+				}
+			}
+			policy->nb_mac_to_monitor = idx;
+			printf("Total Ports: %d\n", idx);
+			break;
+		case 'o':
+			if (!strcmp(optarg, "TRAFFIC"))
+				policy->policy_to_use = TRAFFIC;
+			else if (!strcmp(optarg, "TIME"))
+				policy->policy_to_use = TIME;
+			else if (!strcmp(optarg, "WORKLOAD"))
+				policy->policy_to_use = WORKLOAD;
+			else if (!strcmp(optarg, "BRANCH_RATIO"))
+				policy->policy_to_use = BRANCH_RATIO;
+			else {
+				printf("Invalid policy specified: %s\n",
+						optarg);
+				return -1;
+			}
+			break;
+		/* long options */
+
+		case 0:
+			break;
+
+		default:
+			return -1;
+		}
+	}
+
+	if (optind >= 0)
+		argv[optind-1] = prgname;
+
+	ret = optind-1;
+	optind = 0; /* reset getopt lib */
+	return ret;
+}
+
 int
 main(int argc, char **argv)
 {
@@ -45,6 +172,14 @@ main(int argc, char **argv)
 	signal(SIGINT, sig_handler);
 	signal(SIGTERM, sig_handler);
 
+	argc -= ret;
+	argv += ret;
+
+	/* parse application arguments (after the EAL ones) */
+	ret = parse_args(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "Invalid arguments\n");
+
 	rte_power_set_env(PM_ENV_KVM_VM);
 	RTE_LCORE_FOREACH(lcore_id) {
 		rte_power_init(lcore_id);
diff --git a/examples/vm_power_manager/guest_cli/parse.c b/examples/vm_power_manager/guest_cli/parse.c
new file mode 100644
index 00000000..528df6d6
--- /dev/null
+++ b/examples/vm_power_manager/guest_cli/parse.c
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation.
+ * Copyright(c) 2014 6WIND S.A.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <rte_log.h>
+#include "parse.h"
+
+/*
+ * Parse elem, the elem could be single number/range or group
+ * 1) A single number elem, it's just a simple digit. e.g. 9
+ * 2) A single range elem, two digits with a '-' between. e.g. 2-6
+ * 3) A group elem, combines multiple 1) or 2) e.g 0,2-4,6
+ *    Within group, '-' used for a range separator;
+ *                       ',' used for a single number.
+ */
+int
+parse_set(const char *input, uint16_t set[], unsigned int num)
+{
+	unsigned int idx;
+	const char *str = input;
+	char *end = NULL;
+	unsigned int min, max;
+
+	memset(set, 0, num * sizeof(uint16_t));
+
+	while (isblank(*str))
+		str++;
+
+	/* only digit or left bracket is qualify for start point */
+	if (!isdigit(*str) || *str == '\0')
+		return -1;
+
+	while (isblank(*str))
+		str++;
+	if (*str == '\0')
+		return -1;
+
+	min = num;
+	do {
+
+		/* go ahead to the first digit */
+		while (isblank(*str))
+			str++;
+		if (!isdigit(*str))
+			return -1;
+
+		/* get the digit value */
+		errno = 0;
+		idx = strtoul(str, &end, 10);
+		if (errno || end == NULL || idx >= num)
+			return -1;
+
+		/* go ahead to separator '-' and ',' */
+		while (isblank(*end))
+			end++;
+		if (*end == '-') {
+			if (min == num)
+				min = idx;
+			else /* avoid continuous '-' */
+				return -1;
+		} else if ((*end == ',') || (*end == '\0')) {
+			max = idx;
+
+			if (min == num)
+				min = idx;
+
+			for (idx = RTE_MIN(min, max);
+					idx <= RTE_MAX(min, max); idx++) {
+				set[idx] = 1;
+			}
+			min = num;
+		} else
+			return -1;
+
+		str = end + 1;
+	} while (*end != '\0');
+
+	return str - input;
+}
diff --git a/examples/vm_power_manager/guest_cli/parse.h b/examples/vm_power_manager/guest_cli/parse.h
new file mode 100644
index 00000000..c8aa0ea5
--- /dev/null
+++ b/examples/vm_power_manager/guest_cli/parse.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef PARSE_H_
+#define PARSE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int
+parse_set(const char *, uint16_t [], unsigned int);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* PARSE_H_ */
diff --git a/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c b/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c
index 43bdeace..0db1b804 100644
--- a/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c
+++ b/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c
@@ -33,6 +33,71 @@ struct cmd_quit_result {
 	cmdline_fixed_string_t quit;
 };
 
+union PFID {
+	struct ether_addr addr;
+	uint64_t pfid;
+};
+
+static struct channel_packet policy;
+
+struct channel_packet *
+get_policy(void)
+{
+	return &policy;
+}
+
+int
+set_policy_mac(int port, int idx)
+{
+	struct channel_packet *policy;
+	union PFID pfid;
+
+	/* Use port MAC address as the vfid */
+	rte_eth_macaddr_get(port, &pfid.addr);
+
+	printf("Port %u MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 ":"
+			"%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n",
+			port,
+			pfid.addr.addr_bytes[0], pfid.addr.addr_bytes[1],
+			pfid.addr.addr_bytes[2], pfid.addr.addr_bytes[3],
+			pfid.addr.addr_bytes[4], pfid.addr.addr_bytes[5]);
+	policy = get_policy();
+	policy->vfid[idx] = pfid.pfid;
+	return 0;
+}
+
+void
+set_policy_defaults(struct channel_packet *pkt)
+{
+	set_policy_mac(0, 0);
+	pkt->nb_mac_to_monitor = 1;
+
+	pkt->t_boost_status.tbEnabled = false;
+
+	pkt->vcpu_to_control[0] = 0;
+	pkt->vcpu_to_control[1] = 1;
+	pkt->num_vcpu = 2;
+	/* Dummy Population. */
+	pkt->traffic_policy.min_packet_thresh = 96000;
+	pkt->traffic_policy.avg_max_packet_thresh = 1800000;
+	pkt->traffic_policy.max_max_packet_thresh = 2000000;
+
+	pkt->timer_policy.busy_hours[0] = 3;
+	pkt->timer_policy.busy_hours[1] = 4;
+	pkt->timer_policy.busy_hours[2] = 5;
+	pkt->timer_policy.quiet_hours[0] = 11;
+	pkt->timer_policy.quiet_hours[1] = 12;
+	pkt->timer_policy.quiet_hours[2] = 13;
+
+	pkt->timer_policy.hours_to_use_traffic_profile[0] = 8;
+	pkt->timer_policy.hours_to_use_traffic_profile[1] = 10;
+
+	pkt->workload = LOW;
+	pkt->policy_to_use = TIME;
+	pkt->command = PKT_POLICY;
+	strcpy(pkt->vm_name, "ubuntu2");
+}
+
 static void cmd_quit_parsed(__attribute__((unused)) void *parsed_result,
 				__attribute__((unused)) struct cmdline *cl,
 			    __attribute__((unused)) void *data)
@@ -118,54 +183,12 @@ struct cmd_send_policy_result {
 	cmdline_fixed_string_t cmd;
 };
 
-union PFID {
-	struct ether_addr addr;
-	uint64_t pfid;
-};
-
 static inline int
-send_policy(void)
+send_policy(struct channel_packet *pkt)
 {
-	struct channel_packet pkt;
 	int ret;
 
-	union PFID pfid;
-	/* Use port MAC address as the vfid */
-	rte_eth_macaddr_get(0, &pfid.addr);
-	printf("Port %u MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 ":"
-			"%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n",
-			1,
-			pfid.addr.addr_bytes[0], pfid.addr.addr_bytes[1],
-			pfid.addr.addr_bytes[2], pfid.addr.addr_bytes[3],
-			pfid.addr.addr_bytes[4], pfid.addr.addr_bytes[5]);
-	pkt.vfid[0] = pfid.pfid;
-
-	pkt.nb_mac_to_monitor = 1;
-	pkt.t_boost_status.tbEnabled = false;
-
-	pkt.vcpu_to_control[0] = 0;
-	pkt.vcpu_to_control[1] = 1;
-	pkt.num_vcpu = 2;
-	/* Dummy Population. */
-	pkt.traffic_policy.min_packet_thresh = 96000;
-	pkt.traffic_policy.avg_max_packet_thresh = 1800000;
-	pkt.traffic_policy.max_max_packet_thresh = 2000000;
-
-	pkt.timer_policy.busy_hours[0] = 3;
-	pkt.timer_policy.busy_hours[1] = 4;
-	pkt.timer_policy.busy_hours[2] = 5;
-	pkt.timer_policy.quiet_hours[0] = 11;
-	pkt.timer_policy.quiet_hours[1] = 12;
-	pkt.timer_policy.quiet_hours[2] = 13;
-
-	pkt.timer_policy.hours_to_use_traffic_profile[0] = 8;
-	pkt.timer_policy.hours_to_use_traffic_profile[1] = 10;
-
-	pkt.workload = LOW;
-	pkt.policy_to_use = TIME;
-	pkt.command = PKT_POLICY;
-	strcpy(pkt.vm_name, "ubuntu2");
-	ret = rte_power_guest_channel_send_msg(&pkt, 1);
+	ret = rte_power_guest_channel_send_msg(pkt, 1);
 	if (ret == 0)
 		return 1;
 	RTE_LOG(DEBUG, POWER, "Error sending message: %s\n",
@@ -182,7 +205,7 @@ cmd_send_policy_parsed(void *parsed_result, struct cmdline *cl,
 
 	if (!strcmp(res->cmd, "now")) {
 		printf("Sending Policy down now!\n");
-		ret = send_policy();
+		ret = send_policy(&policy);
 	}
 	if (ret != 1)
 		cmdline_printf(cl, "Error sending message: %s\n",
diff --git a/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h b/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h
index 75a26296..fd77f6a6 100644
--- a/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h
+++ b/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h
@@ -11,6 +11,12 @@ extern "C" {
 
 #include "channel_commands.h"
 
+struct channel_packet *get_policy(void);
+
+int set_policy_mac(int port, int idx);
+
+void set_policy_defaults(struct channel_packet *pkt);
+
 void run_cli(__attribute__((unused)) void *arg);
 
 #ifdef __cplusplus
diff --git a/examples/vm_power_manager/main.c b/examples/vm_power_manager/main.c
index c9805a46..58c5fa45 100644
--- a/examples/vm_power_manager/main.c
+++ b/examples/vm_power_manager/main.c
@@ -29,6 +29,8 @@
 #include "channel_monitor.h"
 #include "power_manager.h"
 #include "vm_power_cli.h"
+#include "oob_monitor.h"
+#include "parse.h"
 #include <rte_pmd_ixgbe.h>
 #include <rte_pmd_i40e.h>
 #include <rte_pmd_bnxt.h>
@@ -47,7 +49,6 @@ static volatile bool force_quit;
 static const struct rte_eth_conf port_conf_default = {
 	.rxmode = {
 		.max_rx_pkt_len = ETHER_MAX_LEN,
-		.ignore_offload_bitfield = 1,
 	},
 };
 
@@ -83,7 +84,6 @@ port_init(uint16_t port, struct rte_mempool *mbuf_pool)
 	}
 
 	txq_conf = dev_info.default_txconf;
-	txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txq_conf.offloads = port_conf.txmode.offloads;
 	/* Allocate and set up 1 TX queue per Ethernet port. */
 	for (q = 0; q < tx_rings; q++) {
@@ -135,18 +135,25 @@ parse_portmask(const char *portmask)
 static int
 parse_args(int argc, char **argv)
 {
-	int opt, ret;
+	int opt, ret, cnt, i;
 	char **argvopt;
+	uint16_t *oob_enable;
 	int option_index;
 	char *prgname = argv[0];
+	struct core_info *ci;
+	float branch_ratio;
 	static struct option lgopts[] = {
 		{ "mac-updating", no_argument, 0, 1},
 		{ "no-mac-updating", no_argument, 0, 0},
+		{ "core-list", optional_argument, 0, 'l'},
+		{ "port-list", optional_argument, 0, 'p'},
+		{ "branch-ratio", optional_argument, 0, 'b'},
 		{NULL, 0, 0, 0}
 	};
 	argvopt = argv;
+	ci = get_core_info();
 
-	while ((opt = getopt_long(argc, argvopt, "p:q:T:",
+	while ((opt = getopt_long(argc, argvopt, "l:p:q:T:b:",
 				  lgopts, &option_index)) != EOF) {
 
 		switch (opt) {
@@ -158,6 +165,39 @@ parse_args(int argc, char **argv)
 				return -1;
 			}
 			break;
+		case 'l':
+			oob_enable = malloc(ci->core_count * sizeof(uint16_t));
+			if (oob_enable == NULL) {
+				printf("Error - Unable to allocate memory\n");
+				return -1;
+			}
+			cnt = parse_set(optarg, oob_enable, ci->core_count);
+			if (cnt < 0) {
+				printf("Invalid core-list - [%s]\n",
+						optarg);
+				break;
+			}
+			for (i = 0; i < ci->core_count; i++) {
+				if (oob_enable[i]) {
+					printf("***Using core %d\n", i);
+					ci->cd[i].oob_enabled = 1;
+					ci->cd[i].global_enabled_cpus = 1;
+				}
+			}
+			free(oob_enable);
+			break;
+		case 'b':
+			branch_ratio = 0.0;
+			if (strlen(optarg))
+				branch_ratio = atof(optarg);
+			if (branch_ratio <= 0.0) {
+				printf("invalid branch ratio specified\n");
+				return -1;
+			}
+			ci->branch_ratio_threshold = branch_ratio;
+			printf("***Setting branch ratio to %f\n",
+					branch_ratio);
+			break;
 		/* long options */
 		case 0:
 			break;
@@ -243,6 +283,17 @@ run_monitor(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+static int
+run_core_monitor(__attribute__((unused)) void *arg)
+{
+	if (branch_monitor_init() < 0) {
+		printf("Unable to initialize core monitor\n");
+		return -1;
+	}
+	run_branch_monitor();
+	return 0;
+}
+
 static void
 sig_handler(int signo)
 {
@@ -261,8 +312,15 @@ main(int argc, char **argv)
 	unsigned int nb_ports;
 	struct rte_mempool *mbuf_pool;
 	uint16_t portid;
+	struct core_info *ci;
 
 
+	ret = core_info_init();
+	if (ret < 0)
+		rte_panic("Cannot allocate core info\n");
+
+	ci = get_core_info();
+
 	ret = rte_eal_init(argc, argv);
 	if (ret < 0)
 		rte_panic("Cannot init EAL\n");
@@ -280,51 +338,56 @@ main(int argc, char **argv)
 
 	nb_ports = rte_eth_dev_count_avail();
 
-	mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports,
-		MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
+	if (nb_ports > 0) {
+		mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL",
+				NUM_MBUFS * nb_ports, MBUF_CACHE_SIZE, 0,
+				RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
 
-	if (mbuf_pool == NULL)
-		rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
+		if (mbuf_pool == NULL)
+			rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
 
-	/* Initialize ports. */
-	RTE_ETH_FOREACH_DEV(portid) {
-		struct ether_addr eth;
-		int w, j;
-		int ret;
+		/* Initialize ports. */
+		RTE_ETH_FOREACH_DEV(portid) {
+			struct ether_addr eth;
+			int w, j;
+			int ret;
 
-		if ((enabled_port_mask & (1 << portid)) == 0)
-			continue;
+			if ((enabled_port_mask & (1 << portid)) == 0)
+				continue;
 
-		eth.addr_bytes[0] = 0xe0;
-		eth.addr_bytes[1] = 0xe0;
-		eth.addr_bytes[2] = 0xe0;
-		eth.addr_bytes[3] = 0xe0;
-		eth.addr_bytes[4] = portid + 0xf0;
+			eth.addr_bytes[0] = 0xe0;
+			eth.addr_bytes[1] = 0xe0;
+			eth.addr_bytes[2] = 0xe0;
+			eth.addr_bytes[3] = 0xe0;
+			eth.addr_bytes[4] = portid + 0xf0;
 
-		if (port_init(portid, mbuf_pool) != 0)
-			rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu8 "\n",
+			if (port_init(portid, mbuf_pool) != 0)
+				rte_exit(EXIT_FAILURE,
+					"Cannot init port %"PRIu8 "\n",
 					portid);
 
-		for (w = 0; w < MAX_VFS; w++) {
-			eth.addr_bytes[5] = w + 0xf0;
-
-			ret = rte_pmd_ixgbe_set_vf_mac_addr(portid,
-						w, &eth);
-			if (ret == -ENOTSUP)
-				ret = rte_pmd_i40e_set_vf_mac_addr(portid,
-						w, &eth);
-			if (ret == -ENOTSUP)
-				ret = rte_pmd_bnxt_set_vf_mac_addr(portid,
-						w, &eth);
-
-			switch (ret) {
-			case 0:
-				printf("Port %d VF %d MAC: ",
-						portid, w);
-				for (j = 0; j < 6; j++) {
-					printf("%02x", eth.addr_bytes[j]);
-					if (j < 5)
-						printf(":");
+			for (w = 0; w < MAX_VFS; w++) {
+				eth.addr_bytes[5] = w + 0xf0;
+
+				ret = rte_pmd_ixgbe_set_vf_mac_addr(portid,
+							w, &eth);
+				if (ret == -ENOTSUP)
+					ret = rte_pmd_i40e_set_vf_mac_addr(
+							portid, w, &eth);
+				if (ret == -ENOTSUP)
+					ret = rte_pmd_bnxt_set_vf_mac_addr(
+							portid, w, &eth);
+
+				switch (ret) {
+				case 0:
+					printf("Port %d VF %d MAC: ",
+							portid, w);
+					for (j = 0; j < 5; j++) {
+						printf("%02x:",
+							eth.addr_bytes[j]);
+					}
+					printf("%02x\n", eth.addr_bytes[5]);
+					break;
 				}
 				printf("\n");
 				break;
@@ -332,16 +395,23 @@ main(int argc, char **argv)
 		}
 	}
 
+	check_all_ports_link_status(enabled_port_mask);
+
 	lcore_id = rte_get_next_lcore(-1, 1, 0);
 	if (lcore_id == RTE_MAX_LCORE) {
-		RTE_LOG(ERR, EAL, "A minimum of two cores are required to run "
+		RTE_LOG(ERR, EAL, "A minimum of three cores are required to run "
 				"application\n");
 		return 0;
 	}
-
-	check_all_ports_link_status(enabled_port_mask);
+	printf("Running channel monitor on lcore id %d\n", lcore_id);
 	rte_eal_remote_launch(run_monitor, NULL, lcore_id);
 
+	lcore_id = rte_get_next_lcore(lcore_id, 1, 0);
+	if (lcore_id == RTE_MAX_LCORE) {
+		RTE_LOG(ERR, EAL, "A minimum of three cores are required to run "
+				"application\n");
+		return 0;
+	}
 	if (power_manager_init() < 0) {
 		printf("Unable to initialize power manager\n");
 		return -1;
@@ -350,8 +420,17 @@ main(int argc, char **argv)
 		printf("Unable to initialize channel manager\n");
 		return -1;
 	}
+
+	printf("Running core monitor on lcore id %d\n", lcore_id);
+	rte_eal_remote_launch(run_core_monitor, NULL, lcore_id);
+
 	run_cli(NULL);
 
+	branch_monitor_exit();
+
 	rte_eal_mp_wait_lcore();
+
+	free(ci->cd);
+
 	return 0;
 }
diff --git a/examples/vm_power_manager/oob_monitor.h b/examples/vm_power_manager/oob_monitor.h
new file mode 100644
index 00000000..b96e08df
--- /dev/null
+++ b/examples/vm_power_manager/oob_monitor.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef OOB_MONITOR_H_
+#define OOB_MONITOR_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Setup the Branch Monitor resources required to initialize epoll.
+ * Must be called first before calling other functions.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int branch_monitor_init(void);
+
+/**
+ * Run the OOB branch monitor, loops forever on on epoll_wait.
+ *
+ *
+ * @return
+ *  None
+ */
+void run_branch_monitor(void);
+
+/**
+ * Exit the OOB Branch Monitor.
+ *
+ * @return
+ *  None
+ */
+void branch_monitor_exit(void);
+
+/**
+ * Add a core to the list of cores to monitor.
+ *
+ * @param core
+ *  Core Number
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int add_core_to_monitor(int core);
+
+/**
+ * Remove a previously added core from core list.
+ *
+ * @param core
+ *  Core Number
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int remove_core_from_monitor(int core);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* OOB_MONITOR_H_ */
diff --git a/examples/vm_power_manager/oob_monitor_nop.c b/examples/vm_power_manager/oob_monitor_nop.c
new file mode 100644
index 00000000..7e7b8bc1
--- /dev/null
+++ b/examples/vm_power_manager/oob_monitor_nop.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation
+ */
+
+#include "oob_monitor.h"
+
+void branch_monitor_exit(void)
+{
+}
+
+__attribute__((unused)) static float
+apply_policy(__attribute__((unused)) int core)
+{
+	return 0.0;
+}
+
+int
+add_core_to_monitor(__attribute__((unused)) int core)
+{
+	return 0;
+}
+
+int
+remove_core_from_monitor(__attribute__((unused)) int core)
+{
+	return 0;
+}
+
+int
+branch_monitor_init(void)
+{
+	return 0;
+}
+
+void
+run_branch_monitor(void)
+{
+}
diff --git a/examples/vm_power_manager/oob_monitor_x86.c b/examples/vm_power_manager/oob_monitor_x86.c
new file mode 100644
index 00000000..589c604e
--- /dev/null
+++ b/examples/vm_power_manager/oob_monitor_x86.c
@@ -0,0 +1,258 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <rte_log.h>
+
+#include "oob_monitor.h"
+#include "power_manager.h"
+#include "channel_manager.h"
+
+static volatile unsigned run_loop = 1;
+static uint64_t g_branches, g_branch_misses;
+static int g_active;
+
+void branch_monitor_exit(void)
+{
+	run_loop = 0;
+}
+
+/* Number of microseconds between each poll */
+#define INTERVAL 100
+#define PRINT_LOOP_COUNT (1000000/INTERVAL)
+#define IA32_PERFEVTSEL0 0x186
+#define IA32_PERFEVTSEL1 0x187
+#define IA32_PERFCTR0 0xc1
+#define IA32_PERFCTR1 0xc2
+#define IA32_PERFEVT_BRANCH_HITS 0x05300c4
+#define IA32_PERFEVT_BRANCH_MISS 0x05300c5
+
+static float
+apply_policy(int core)
+{
+	struct core_info *ci;
+	uint64_t counter;
+	uint64_t branches, branch_misses;
+	uint32_t last_branches, last_branch_misses;
+	int hits_diff, miss_diff;
+	float ratio;
+	int ret;
+
+	g_active = 0;
+	ci = get_core_info();
+
+	last_branches = ci->cd[core].last_branches;
+	last_branch_misses = ci->cd[core].last_branch_misses;
+
+	ret = pread(ci->cd[core].msr_fd, &counter,
+			sizeof(counter), IA32_PERFCTR0);
+	if (ret < 0)
+		RTE_LOG(ERR, POWER_MANAGER,
+				"unable to read counter for core %u\n",
+				core);
+	branches = counter;
+
+	ret = pread(ci->cd[core].msr_fd, &counter,
+			sizeof(counter), IA32_PERFCTR1);
+	if (ret < 0)
+		RTE_LOG(ERR, POWER_MANAGER,
+				"unable to read counter for core %u\n",
+				core);
+	branch_misses = counter;
+
+
+	ci->cd[core].last_branches = branches;
+	ci->cd[core].last_branch_misses = branch_misses;
+
+	hits_diff = (int)branches - (int)last_branches;
+	if (hits_diff <= 0) {
+		/* Likely a counter overflow condition, skip this round */
+		return -1.0;
+	}
+
+	miss_diff = (int)branch_misses - (int)last_branch_misses;
+	if (miss_diff <= 0) {
+		/* Likely a counter overflow condition, skip this round */
+		return -1.0;
+	}
+
+	g_branches = hits_diff;
+	g_branch_misses = miss_diff;
+
+	if (hits_diff < (INTERVAL*100)) {
+		/* Likely no workload running on this core. Skip. */
+		return -1.0;
+	}
+
+	ratio = (float)miss_diff * (float)100 / (float)hits_diff;
+
+	if (ratio < ci->branch_ratio_threshold)
+		power_manager_scale_core_min(core);
+	else
+		power_manager_scale_core_max(core);
+
+	g_active = 1;
+	return ratio;
+}
+
+int
+add_core_to_monitor(int core)
+{
+	struct core_info *ci;
+	char proc_file[UNIX_PATH_MAX];
+	int ret;
+
+	ci = get_core_info();
+
+	if (core < ci->core_count) {
+		long setup;
+
+		snprintf(proc_file, UNIX_PATH_MAX, "/dev/cpu/%d/msr", core);
+		ci->cd[core].msr_fd = open(proc_file, O_RDWR | O_SYNC);
+		if (ci->cd[core].msr_fd < 0) {
+			RTE_LOG(ERR, POWER_MANAGER,
+					"Error opening MSR file for core %d "
+					"(is msr kernel module loaded?)\n",
+					core);
+			return -1;
+		}
+		/*
+		 * Set up branch counters
+		 */
+		setup = IA32_PERFEVT_BRANCH_HITS;
+		ret = pwrite(ci->cd[core].msr_fd, &setup,
+				sizeof(setup), IA32_PERFEVTSEL0);
+		if (ret < 0) {
+			RTE_LOG(ERR, POWER_MANAGER,
+					"unable to set counter for core %u\n",
+					core);
+			return ret;
+		}
+		setup = IA32_PERFEVT_BRANCH_MISS;
+		ret = pwrite(ci->cd[core].msr_fd, &setup,
+				sizeof(setup), IA32_PERFEVTSEL1);
+		if (ret < 0) {
+			RTE_LOG(ERR, POWER_MANAGER,
+					"unable to set counter for core %u\n",
+					core);
+			return ret;
+		}
+		/*
+		 * Close the file and re-open as read only so
+		 * as not to hog the resource
+		 */
+		close(ci->cd[core].msr_fd);
+		ci->cd[core].msr_fd = open(proc_file, O_RDONLY);
+		if (ci->cd[core].msr_fd < 0) {
+			RTE_LOG(ERR, POWER_MANAGER,
+					"Error opening MSR file for core %d "
+					"(is msr kernel module loaded?)\n",
+					core);
+			return -1;
+		}
+		ci->cd[core].oob_enabled = 1;
+	}
+	return 0;
+}
+
+int
+remove_core_from_monitor(int core)
+{
+	struct core_info *ci;
+	char proc_file[UNIX_PATH_MAX];
+	int ret;
+
+	ci = get_core_info();
+
+	if (ci->cd[core].oob_enabled) {
+		long setup;
+
+		/*
+		 * close the msr file, then reopen rw so we can
+		 * disable the counters
+		 */
+		if (ci->cd[core].msr_fd != 0)
+			close(ci->cd[core].msr_fd);
+		snprintf(proc_file, UNIX_PATH_MAX, "/dev/cpu/%d/msr", core);
+		ci->cd[core].msr_fd = open(proc_file, O_RDWR | O_SYNC);
+		if (ci->cd[core].msr_fd < 0) {
+			RTE_LOG(ERR, POWER_MANAGER,
+					"Error opening MSR file for core %d "
+					"(is msr kernel module loaded?)\n",
+					core);
+			return -1;
+		}
+		setup = 0x0; /* clear event */
+		ret = pwrite(ci->cd[core].msr_fd, &setup,
+				sizeof(setup), IA32_PERFEVTSEL0);
+		if (ret < 0) {
+			RTE_LOG(ERR, POWER_MANAGER,
+					"unable to set counter for core %u\n",
+					core);
+			return ret;
+		}
+		setup = 0x0; /* clear event */
+		ret = pwrite(ci->cd[core].msr_fd, &setup,
+				sizeof(setup), IA32_PERFEVTSEL1);
+		if (ret < 0) {
+			RTE_LOG(ERR, POWER_MANAGER,
+					"unable to set counter for core %u\n",
+					core);
+			return ret;
+		}
+
+		close(ci->cd[core].msr_fd);
+		ci->cd[core].msr_fd = 0;
+		ci->cd[core].oob_enabled = 0;
+	}
+	return 0;
+}
+
+int
+branch_monitor_init(void)
+{
+	return 0;
+}
+
+void
+run_branch_monitor(void)
+{
+	struct core_info *ci;
+	int print = 0;
+	float ratio;
+	int printed;
+	int reads = 0;
+
+	ci = get_core_info();
+
+	while (run_loop) {
+
+		if (!run_loop)
+			break;
+		usleep(INTERVAL);
+		int j;
+		print++;
+		printed = 0;
+		for (j = 0; j < ci->core_count; j++) {
+			if (ci->cd[j].oob_enabled) {
+				ratio = apply_policy(j);
+				if ((print > PRINT_LOOP_COUNT) && (g_active)) {
+					printf("  %d: %.4f {%lu} {%d}", j,
+							ratio, g_branches,
+							reads);
+					printed = 1;
+					reads = 0;
+				} else {
+					reads++;
+				}
+			}
+		}
+		if (print > PRINT_LOOP_COUNT) {
+			if (printed)
+				printf("\n");
+			print = 0;
+		}
+	}
+}
diff --git a/examples/vm_power_manager/parse.c b/examples/vm_power_manager/parse.c
new file mode 100644
index 00000000..8231533b
--- /dev/null
+++ b/examples/vm_power_manager/parse.c
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation.
+ * Copyright(c) 2014 6WIND S.A.
+ */
+
+#include <string.h>
+#include <rte_log.h>
+#include "parse.h"
+
+/*
+ * Parse elem, the elem could be single number/range or group
+ * 1) A single number elem, it's just a simple digit. e.g. 9
+ * 2) A single range elem, two digits with a '-' between. e.g. 2-6
+ * 3) A group elem, combines multiple 1) or 2) e.g 0,2-4,6
+ *    Within group, '-' used for a range separator;
+ *                       ',' used for a single number.
+ */
+int
+parse_set(const char *input, uint16_t set[], unsigned int num)
+{
+	unsigned int idx;
+	const char *str = input;
+	char *end = NULL;
+	unsigned int min, max;
+
+	memset(set, 0, num * sizeof(uint16_t));
+
+	while (isblank(*str))
+		str++;
+
+	/* only digit or left bracket is qualify for start point */
+	if (!isdigit(*str) || *str == '\0')
+		return -1;
+
+	while (isblank(*str))
+		str++;
+	if (*str == '\0')
+		return -1;
+
+	min = num;
+	do {
+
+		/* go ahead to the first digit */
+		while (isblank(*str))
+			str++;
+		if (!isdigit(*str))
+			return -1;
+
+		/* get the digit value */
+		errno = 0;
+		idx = strtoul(str, &end, 10);
+		if (errno || end == NULL || idx >= num)
+			return -1;
+
+		/* go ahead to separator '-' and ',' */
+		while (isblank(*end))
+			end++;
+		if (*end == '-') {
+			if (min == num)
+				min = idx;
+			else /* avoid continuous '-' */
+				return -1;
+		} else if ((*end == ',') || (*end == '\0')) {
+			max = idx;
+
+			if (min == num)
+				min = idx;
+
+			for (idx = RTE_MIN(min, max);
+					idx <= RTE_MAX(min, max); idx++) {
+				set[idx] = 1;
+			}
+			min = num;
+		} else
+			return -1;
+
+		str = end + 1;
+	} while (*end != '\0');
+
+	return str - input;
+}
diff --git a/examples/vm_power_manager/parse.h b/examples/vm_power_manager/parse.h
new file mode 100644
index 00000000..a5971e9a
--- /dev/null
+++ b/examples/vm_power_manager/parse.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef PARSE_H_
+#define PARSE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int
+parse_set(const char *, uint16_t [], unsigned int);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* PARSE_H_ */
diff --git a/examples/vm_power_manager/power_manager.c b/examples/vm_power_manager/power_manager.c
index 35db2559..b7769c3c 100644
--- a/examples/vm_power_manager/power_manager.c
+++ b/examples/vm_power_manager/power_manager.c
@@ -12,20 +12,21 @@
 #include <dirent.h>
 #include <errno.h>
 
+#include <sys/sysinfo.h>
 #include <sys/types.h>
 
 #include <rte_log.h>
 #include <rte_power.h>
 #include <rte_spinlock.h>
 
+#include "channel_manager.h"
 #include "power_manager.h"
-
-#define RTE_LOGTYPE_POWER_MANAGER RTE_LOGTYPE_USER1
+#include "oob_monitor.h"
 
 #define POWER_SCALE_CORE(DIRECTION, core_num , ret) do { \
-	if (core_num >= POWER_MGR_MAX_CPUS) \
+	if (core_num >= ci.core_count) \
 		return -1; \
-	if (!(global_enabled_cpus & (1ULL << core_num))) \
+	if (!(ci.cd[core_num].global_enabled_cpus)) \
 		return -1; \
 	rte_spinlock_lock(&global_core_freq_info[core_num].power_sl); \
 	ret = rte_power_freq_##DIRECTION(core_num); \
@@ -36,7 +37,7 @@
 	int i; \
 	for (i = 0; core_mask; core_mask &= ~(1 << i++)) { \
 		if ((core_mask >> i) & 1) { \
-			if (!(global_enabled_cpus & (1ULL << i))) \
+			if (!(ci.cd[i].global_enabled_cpus)) \
 				continue; \
 			rte_spinlock_lock(&global_core_freq_info[i].power_sl); \
 			if (rte_power_freq_##DIRECTION(i) != 1) \
@@ -54,63 +55,82 @@ struct freq_info {
 
 static struct freq_info global_core_freq_info[POWER_MGR_MAX_CPUS];
 
-static uint64_t global_enabled_cpus;
+struct core_info ci;
 
 #define SYSFS_CPU_PATH "/sys/devices/system/cpu/cpu%u/topology/core_id"
 
-static unsigned
-set_host_cpus_mask(void)
+struct core_info *
+get_core_info(void)
 {
-	char path[PATH_MAX];
-	unsigned i;
-	unsigned num_cpus = 0;
-
-	for (i = 0; i < POWER_MGR_MAX_CPUS; i++) {
-		snprintf(path, sizeof(path), SYSFS_CPU_PATH, i);
-		if (access(path, F_OK) == 0) {
-			global_enabled_cpus |= 1ULL << i;
-			num_cpus++;
-		} else
-			return num_cpus;
+	return &ci;
+}
+
+int
+core_info_init(void)
+{
+	struct core_info *ci;
+	int i;
+
+	ci = get_core_info();
+
+	ci->core_count = get_nprocs_conf();
+	ci->branch_ratio_threshold = BRANCH_RATIO_THRESHOLD;
+	ci->cd = malloc(ci->core_count * sizeof(struct core_details));
+	if (!ci->cd) {
+		RTE_LOG(ERR, POWER_MANAGER, "Failed to allocate memory for core info.");
+		return -1;
 	}
-	return num_cpus;
+	for (i = 0; i < ci->core_count; i++) {
+		ci->cd[i].global_enabled_cpus = 1;
+		ci->cd[i].oob_enabled = 0;
+		ci->cd[i].msr_fd = 0;
+	}
+	printf("%d cores in system\n", ci->core_count);
+	return 0;
 }
 
 int
 power_manager_init(void)
 {
-	unsigned int i, num_cpus, num_freqs;
-	uint64_t cpu_mask;
+	unsigned int i, num_cpus = 0, num_freqs = 0;
 	int ret = 0;
+	struct core_info *ci;
+
+	rte_power_set_env(PM_ENV_ACPI_CPUFREQ);
 
-	num_cpus = set_host_cpus_mask();
-	if (num_cpus == 0) {
-		RTE_LOG(ERR, POWER_MANAGER, "Unable to detected host CPUs, please "
-			"ensure that sufficient privileges exist to inspect sysfs\n");
+	ci = get_core_info();
+	if (!ci) {
+		RTE_LOG(ERR, POWER_MANAGER,
+				"Failed to get core info!\n");
 		return -1;
 	}
-	rte_power_set_env(PM_ENV_ACPI_CPUFREQ);
-	cpu_mask = global_enabled_cpus;
-	for (i = 0; cpu_mask; cpu_mask &= ~(1 << i++)) {
-		if (rte_power_init(i) < 0)
-			RTE_LOG(ERR, POWER_MANAGER,
-					"Unable to initialize power manager "
-					"for core %u\n", i);
-		num_freqs = rte_power_freqs(i, global_core_freq_info[i].freqs,
+
+	for (i = 0; i < ci->core_count; i++) {
+		if (ci->cd[i].global_enabled_cpus) {
+			if (rte_power_init(i) < 0)
+				RTE_LOG(ERR, POWER_MANAGER,
+						"Unable to initialize power manager "
+						"for core %u\n", i);
+			num_cpus++;
+			num_freqs = rte_power_freqs(i,
+					global_core_freq_info[i].freqs,
 					RTE_MAX_LCORE_FREQS);
-		if (num_freqs == 0) {
-			RTE_LOG(ERR, POWER_MANAGER,
-				"Unable to get frequency list for core %u\n",
-				i);
-			global_enabled_cpus &= ~(1 << i);
-			num_cpus--;
-			ret = -1;
+			if (num_freqs == 0) {
+				RTE_LOG(ERR, POWER_MANAGER,
+					"Unable to get frequency list for core %u\n",
+					i);
+				ci->cd[i].oob_enabled = 0;
+				ret = -1;
+			}
+			global_core_freq_info[i].num_freqs = num_freqs;
+
+			rte_spinlock_init(&global_core_freq_info[i].power_sl);
 		}
-		global_core_freq_info[i].num_freqs = num_freqs;
-		rte_spinlock_init(&global_core_freq_info[i].power_sl);
+		if (ci->cd[i].oob_enabled)
+			add_core_to_monitor(i);
 	}
-	RTE_LOG(INFO, POWER_MANAGER, "Detected %u host CPUs , enabled core mask:"
-					" 0x%"PRIx64"\n", num_cpus, global_enabled_cpus);
+	RTE_LOG(INFO, POWER_MANAGER, "Managing %u cores out of %u available host cores\n",
+			num_cpus, ci->core_count);
 	return ret;
 
 }
@@ -125,7 +145,7 @@ power_manager_get_current_frequency(unsigned core_num)
 				core_num, POWER_MGR_MAX_CPUS-1);
 		return -1;
 	}
-	if (!(global_enabled_cpus & (1ULL << core_num)))
+	if (!(ci.cd[core_num].global_enabled_cpus))
 		return 0;
 
 	rte_spinlock_lock(&global_core_freq_info[core_num].power_sl);
@@ -144,15 +164,26 @@ power_manager_exit(void)
 {
 	unsigned int i;
 	int ret = 0;
+	struct core_info *ci;
+
+	ci = get_core_info();
+	if (!ci) {
+		RTE_LOG(ERR, POWER_MANAGER,
+				"Failed to get core info!\n");
+		return -1;
+	}
 
-	for (i = 0; global_enabled_cpus; global_enabled_cpus &= ~(1 << i++)) {
-		if (rte_power_exit(i) < 0) {
-			RTE_LOG(ERR, POWER_MANAGER, "Unable to shutdown power manager "
-					"for core %u\n", i);
-			ret = -1;
+	for (i = 0; i < ci->core_count; i++) {
+		if (ci->cd[i].global_enabled_cpus) {
+			if (rte_power_exit(i) < 0) {
+				RTE_LOG(ERR, POWER_MANAGER, "Unable to shutdown power manager "
+						"for core %u\n", i);
+				ret = -1;
+			}
+			ci->cd[i].global_enabled_cpus = 0;
 		}
+		remove_core_from_monitor(i);
 	}
-	global_enabled_cpus = 0;
 	return ret;
 }
 
@@ -268,10 +299,12 @@ int
 power_manager_scale_core_med(unsigned int core_num)
 {
 	int ret = 0;
+	struct core_info *ci;
 
+	ci = get_core_info();
 	if (core_num >= POWER_MGR_MAX_CPUS)
 		return -1;
-	if (!(global_enabled_cpus & (1ULL << core_num)))
+	if (!(ci->cd[core_num].global_enabled_cpus))
 		return -1;
 	rte_spinlock_lock(&global_core_freq_info[core_num].power_sl);
 	ret = rte_power_set_freq(core_num,
diff --git a/examples/vm_power_manager/power_manager.h b/examples/vm_power_manager/power_manager.h
index 8a8a84aa..605b3c8f 100644
--- a/examples/vm_power_manager/power_manager.h
+++ b/examples/vm_power_manager/power_manager.h
@@ -8,6 +8,29 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+struct core_details {
+	uint64_t last_branches;
+	uint64_t last_branch_misses;
+	uint16_t global_enabled_cpus;
+	uint16_t oob_enabled;
+	int msr_fd;
+};
+
+struct core_info {
+	uint16_t core_count;
+	struct core_details *cd;
+	float branch_ratio_threshold;
+};
+
+#define BRANCH_RATIO_THRESHOLD 0.1
+
+struct core_info *
+get_core_info(void);
+
+int
+core_info_init(void);
+
+#define RTE_LOGTYPE_POWER_MANAGER RTE_LOGTYPE_USER1
 
 /* Maximum number of CPUS to manage */
 #define POWER_MGR_MAX_CPUS 64
diff --git a/examples/vmdq/main.c b/examples/vmdq/main.c
index 52596dd5..627a5da4 100644
--- a/examples/vmdq/main.c
+++ b/examples/vmdq/main.c
@@ -65,7 +65,6 @@ static const struct rte_eth_conf vmdq_conf_default = {
 	.rxmode = {
 		.mq_mode        = ETH_MQ_RX_VMDQ_ONLY,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 	},
 
 	.txmode = {
@@ -237,7 +236,6 @@ port_init(uint16_t port, struct rte_mempool *mbuf_pool)
 	rxconf = &dev_info.default_rxconf;
 	rxconf->rx_drop_en = 1;
 	txconf = &dev_info.default_txconf;
-	txconf->txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txconf->offloads = port_conf.txmode.offloads;
 	for (q = 0; q < rxRings; q++) {
 		retval = rte_eth_rx_queue_setup(port, q, rxRingSize,
diff --git a/examples/vmdq_dcb/main.c b/examples/vmdq_dcb/main.c
index 2626a2f1..64636839 100644
--- a/examples/vmdq_dcb/main.c
+++ b/examples/vmdq_dcb/main.c
@@ -71,7 +71,6 @@ static const struct rte_eth_conf vmdq_dcb_conf_default = {
 	.rxmode = {
 		.mq_mode        = ETH_MQ_RX_VMDQ_DCB,
 		.split_hdr_size = 0,
-		.ignore_offload_bitfield = 1,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_VMDQ_DCB,
@@ -197,6 +196,7 @@ port_init(uint16_t port, struct rte_mempool *mbuf_pool)
 	uint16_t queues_per_pool;
 	uint32_t max_nb_pools;
 	struct rte_eth_txconf txq_conf;
+	uint64_t rss_hf_tmp;
 
 	/*
 	 * The max pool number from dev_info will be used to validate the pool
@@ -257,6 +257,18 @@ port_init(uint16_t port, struct rte_mempool *mbuf_pool)
 	if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 		port_conf.txmode.offloads |=
 			DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+
+	rss_hf_tmp = port_conf.rx_adv_conf.rss_conf.rss_hf;
+	port_conf.rx_adv_conf.rss_conf.rss_hf &=
+		dev_info.flow_type_rss_offloads;
+	if (port_conf.rx_adv_conf.rss_conf.rss_hf != rss_hf_tmp) {
+		printf("Port %u modified RSS hash function based on hardware support,"
+			"requested:%#"PRIx64" configured:%#"PRIx64"\n",
+			port,
+			rss_hf_tmp,
+			port_conf.rx_adv_conf.rss_conf.rss_hf);
+	}
+
 	/*
 	 * Though in this example, all queues including pf queues are setup.
 	 * This is because VMDQ queues doesn't always start from zero, and the
@@ -290,7 +302,6 @@ port_init(uint16_t port, struct rte_mempool *mbuf_pool)
 	}
 
 	txq_conf = dev_info.default_txconf;
-	txq_conf.txq_flags = ETH_TXQ_FLAGS_IGNORE;
 	txq_conf.offloads = port_conf.txmode.offloads;
 	for (q = 0; q < num_queues; q++) {
 		retval = rte_eth_tx_queue_setup(port, q, txRingSize,
diff --git a/kernel/linux/igb_uio/compat.h b/kernel/linux/igb_uio/compat.h
index b4131a5c..8dbb896a 100644
--- a/kernel/linux/igb_uio/compat.h
+++ b/kernel/linux/igb_uio/compat.h
@@ -126,10 +126,6 @@ static bool pci_check_and_mask_intx(struct pci_dev *pdev)
 #define HAVE_PCI_IS_BRIDGE_API 1
 #endif
 
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)
-#define HAVE_ALLOC_IRQ_VECTORS 1
-#endif
-
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0)
 #define HAVE_MSI_LIST_IN_GENERIC_DEVICE 1
 #endif
@@ -137,3 +133,22 @@ static bool pci_check_and_mask_intx(struct pci_dev *pdev)
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)
 #define HAVE_PCI_MSI_MASK_IRQ 1
 #endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)
+#define HAVE_ALLOC_IRQ_VECTORS 1
+#endif
+
+static inline bool igbuio_kernel_is_locked_down(void)
+{
+#ifdef CONFIG_LOCK_DOWN_KERNEL
+#ifdef CONFIG_LOCK_DOWN_IN_EFI_SECURE_BOOT
+	return kernel_is_locked_down(NULL);
+#elif defined(CONFIG_EFI_SECURE_BOOT_LOCK_DOWN)
+	return kernel_is_locked_down();
+#else
+	return false;
+#endif
+#else
+	return false;
+#endif
+}
diff --git a/kernel/linux/igb_uio/igb_uio.c b/kernel/linux/igb_uio/igb_uio.c
index cd9b7e72..3398eacd 100644
--- a/kernel/linux/igb_uio/igb_uio.c
+++ b/kernel/linux/igb_uio/igb_uio.c
@@ -30,6 +30,7 @@ struct rte_uio_pci_dev {
 	int refcnt;
 };
 
+static int wc_activate;
 static char *intr_mode;
 static enum rte_intr_mode igbuio_intr_mode_preferred = RTE_INTR_MODE_MSIX;
 /* sriov sysfs */
@@ -375,9 +376,13 @@ igbuio_pci_setup_iomem(struct pci_dev *dev, struct uio_info *info,
 	len = pci_resource_len(dev, pci_bar);
 	if (addr == 0 || len == 0)
 		return -1;
-	internal_addr = ioremap(addr, len);
-	if (internal_addr == NULL)
-		return -1;
+	if (wc_activate == 0) {
+		internal_addr = ioremap(addr, len);
+		if (internal_addr == NULL)
+			return -1;
+	} else {
+		internal_addr = NULL;
+	}
 	info->mem[n].name = name;
 	info->mem[n].addr = addr;
 	info->mem[n].internal_addr = internal_addr;
@@ -621,6 +626,14 @@ igbuio_pci_init_module(void)
 {
 	int ret;
 
+	if (igbuio_kernel_is_locked_down()) {
+		pr_err("Not able to use module, kernel lock down is enabled\n");
+		return -EINVAL;
+	}
+
+	if (wc_activate != 0)
+		pr_info("wc_activate is set\n");
+
 	ret = igbuio_config_intr_mode(intr_mode);
 	if (ret < 0)
 		return ret;
@@ -645,6 +658,12 @@ MODULE_PARM_DESC(intr_mode,
 "    " RTE_INTR_MODE_LEGACY_NAME "     Use Legacy interrupt\n"
 "\n");
 
+module_param(wc_activate, int, 0);
+MODULE_PARM_DESC(wc_activate,
+"Activate support for write combining (WC) (default=0)\n"
+"    0 - disable\n"
+"    other - enable\n");
+
 MODULE_DESCRIPTION("UIO driver for Intel IGB PCI cards");
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Intel Corporation");
diff --git a/kernel/linux/kni/ethtool/igb/igb_ethtool.c b/kernel/linux/kni/ethtool/igb/igb_ethtool.c
index 064528bc..002f75c4 100644
--- a/kernel/linux/kni/ethtool/igb/igb_ethtool.c
+++ b/kernel/linux/kni/ethtool/igb/igb_ethtool.c
@@ -811,9 +811,10 @@ static void igb_get_drvinfo(struct net_device *netdev,
 	strncpy(drvinfo->driver,  igb_driver_name, sizeof(drvinfo->driver) - 1);
 	strncpy(drvinfo->version, igb_driver_version, sizeof(drvinfo->version) - 1);
 
-	strncpy(drvinfo->fw_version, adapter->fw_version,
-		sizeof(drvinfo->fw_version) - 1);
-	strncpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info) -1);
+	strlcpy(drvinfo->fw_version, adapter->fw_version,
+		sizeof(drvinfo->fw_version));
+	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+		sizeof(drvinfo->bus_info));
 	drvinfo->n_stats = IGB_STATS_LEN;
 	drvinfo->testinfo_len = IGB_TEST_LEN;
 	drvinfo->regdump_len = igb_get_regs_len(netdev);
diff --git a/kernel/linux/kni/ethtool/igb/kcompat.h b/kernel/linux/kni/ethtool/igb/kcompat.h
index 40a8d99c..ae1b5309 100644
--- a/kernel/linux/kni/ethtool/igb/kcompat.h
+++ b/kernel/linux/kni/ethtool/igb/kcompat.h
@@ -3929,6 +3929,11 @@ skb_set_hash(struct sk_buff *skb, __u32 hash, __always_unused int type)
 #endif
 #endif
 
+#if (defined(RHEL_RELEASE_CODE) && \
+	(RHEL_RELEASE_VERSION(7, 5) <= RHEL_RELEASE_CODE))
+#define ndo_change_mtu ndo_change_mtu_rh74
+#endif
+
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)
 #define HAVE_PCI_ENABLE_MSIX
 #endif
diff --git a/lib/Makefile b/lib/Makefile
index d82462ba..afa604e2 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -4,7 +4,10 @@
 include $(RTE_SDK)/mk/rte.vars.mk
 
 DIRS-y += librte_compat
+DIRS-$(CONFIG_RTE_LIBRTE_KVARGS) += librte_kvargs
+DEPDIRS-librte_kvargs := librte_compat
 DIRS-$(CONFIG_RTE_LIBRTE_EAL) += librte_eal
+DEPDIRS-librte_eal := librte_kvargs
 DIRS-$(CONFIG_RTE_LIBRTE_PCI) += librte_pci
 DEPDIRS-librte_pci := librte_eal
 DIRS-$(CONFIG_RTE_LIBRTE_RING) += librte_ring
@@ -76,8 +79,6 @@ DEPDIRS-librte_flow_classify :=  librte_net librte_table librte_acl
 DIRS-$(CONFIG_RTE_LIBRTE_SCHED) += librte_sched
 DEPDIRS-librte_sched := librte_eal librte_mempool librte_mbuf librte_net
 DEPDIRS-librte_sched += librte_timer
-DIRS-$(CONFIG_RTE_LIBRTE_KVARGS) += librte_kvargs
-DEPDIRS-librte_kvargs := librte_eal
 DIRS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += librte_distributor
 DEPDIRS-librte_distributor := librte_eal librte_mbuf librte_ethdev
 DIRS-$(CONFIG_RTE_LIBRTE_PORT) += librte_port
diff --git a/lib/librte_bbdev/rte_bbdev.c b/lib/librte_bbdev/rte_bbdev.c
index 28434e08..c4cc18d9 100644
--- a/lib/librte_bbdev/rte_bbdev.c
+++ b/lib/librte_bbdev/rte_bbdev.c
@@ -1125,9 +1125,7 @@ rte_bbdev_op_type_str(enum rte_bbdev_op_type op_type)
 	return NULL;
 }
 
-RTE_INIT(rte_bbdev_init_log);
-static void
-rte_bbdev_init_log(void)
+RTE_INIT(rte_bbdev_init_log)
 {
 	bbdev_logtype = rte_log_register("lib.bbdev");
 	if (bbdev_logtype >= 0)
diff --git a/lib/librte_bitratestats/rte_bitrate.c b/lib/librte_bitratestats/rte_bitrate.c
index 964e3c39..c4b28f62 100644
--- a/lib/librte_bitratestats/rte_bitrate.c
+++ b/lib/librte_bitratestats/rte_bitrate.c
@@ -47,6 +47,9 @@ rte_stats_bitrate_reg(struct rte_stats_bitrates *bitrate_data)
 	};
 	int return_value;
 
+	if (bitrate_data == NULL)
+		return -EINVAL;
+
 	return_value = rte_metrics_reg_names(&names[0], ARRAY_SIZE(names));
 	if (return_value >= 0)
 		bitrate_data->id_stats_set = return_value;
@@ -65,6 +68,9 @@ rte_stats_bitrate_calc(struct rte_stats_bitrates *bitrate_data,
 	const int64_t alpha_percent = 20;
 	uint64_t values[6];
 
+	if (bitrate_data == NULL)
+		return -EINVAL;
+
 	ret_code = rte_eth_stats_get(port_id, &eth_stats);
 	if (ret_code != 0)
 		return ret_code;
diff --git a/lib/librte_bpf/bpf.c b/lib/librte_bpf/bpf.c
index dc6d1099..f590c8c3 100644
--- a/lib/librte_bpf/bpf.c
+++ b/lib/librte_bpf/bpf.c
@@ -53,10 +53,7 @@ bpf_jit(struct rte_bpf *bpf)
 	return rc;
 }
 
-RTE_INIT(rte_bpf_init_log);
-
-static void
-rte_bpf_init_log(void)
+RTE_INIT(rte_bpf_init_log)
 {
 	rte_bpf_logtype = rte_log_register("lib.bpf");
 	if (rte_bpf_logtype >= 0)
diff --git a/lib/librte_bpf/bpf_def.h b/lib/librte_bpf/bpf_def.h
index 6b69de34..c10f3aec 100644
--- a/lib/librte_bpf/bpf_def.h
+++ b/lib/librte_bpf/bpf_def.h
@@ -131,6 +131,11 @@ struct ebpf_insn {
 	int32_t imm;
 };
 
+/*
+ * eBPF allows functions with R1-R5 as arguments.
+ */
+#define	EBPF_FUNC_MAX_ARGS	(EBPF_REG_6 - EBPF_REG_1)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_bpf/bpf_exec.c b/lib/librte_bpf/bpf_exec.c
index e373b1f3..6a79139c 100644
--- a/lib/librte_bpf/bpf_exec.c
+++ b/lib/librte_bpf/bpf_exec.c
@@ -402,7 +402,7 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
 			break;
 		/* call instructions */
 		case (BPF_JMP | EBPF_CALL):
-			reg[EBPF_REG_0] = bpf->prm.xsym[ins->imm].func(
+			reg[EBPF_REG_0] = bpf->prm.xsym[ins->imm].func.val(
 				reg[EBPF_REG_1], reg[EBPF_REG_2],
 				reg[EBPF_REG_3], reg[EBPF_REG_4],
 				reg[EBPF_REG_5]);
diff --git a/lib/librte_bpf/bpf_impl.h b/lib/librte_bpf/bpf_impl.h
index 5d7e65c3..b577e2cb 100644
--- a/lib/librte_bpf/bpf_impl.h
+++ b/lib/librte_bpf/bpf_impl.h
@@ -34,6 +34,20 @@ extern int rte_bpf_logtype;
 #define	RTE_BPF_LOG(lvl, fmt, args...) \
 	rte_log(RTE_LOG_## lvl, rte_bpf_logtype, fmt, ##args)
 
+static inline size_t
+bpf_size(uint32_t bpf_op_sz)
+{
+	if (bpf_op_sz == BPF_B)
+		return sizeof(uint8_t);
+	else if (bpf_op_sz == BPF_H)
+		return sizeof(uint16_t);
+	else if (bpf_op_sz == BPF_W)
+		return sizeof(uint32_t);
+	else if (bpf_op_sz == EBPF_DW)
+		return sizeof(uint64_t);
+	return 0;
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_bpf/bpf_jit_x86.c b/lib/librte_bpf/bpf_jit_x86.c
index 111e028d..68ea389f 100644
--- a/lib/librte_bpf/bpf_jit_x86.c
+++ b/lib/librte_bpf/bpf_jit_x86.c
@@ -113,20 +113,6 @@ union bpf_jit_imm {
 	uint8_t u8[4];
 };
 
-static size_t
-bpf_size(uint32_t bpf_op_sz)
-{
-	if (bpf_op_sz == BPF_B)
-		return sizeof(uint8_t);
-	else if (bpf_op_sz == BPF_H)
-		return sizeof(uint16_t);
-	else if (bpf_op_sz == BPF_W)
-		return sizeof(uint32_t);
-	else if (bpf_op_sz == EBPF_DW)
-		return sizeof(uint64_t);
-	return 0;
-}
-
 /*
  * In many cases for imm8 we can produce shorter code.
  */
@@ -1294,7 +1280,8 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
 			break;
 		/* call instructions */
 		case (BPF_JMP | EBPF_CALL):
-			emit_call(st, (uintptr_t)bpf->prm.xsym[ins->imm].func);
+			emit_call(st,
+				(uintptr_t)bpf->prm.xsym[ins->imm].func.val);
 			break;
 		/* return instruction */
 		case (BPF_JMP | EBPF_EXIT):
diff --git a/lib/librte_bpf/bpf_load.c b/lib/librte_bpf/bpf_load.c
index d1c9abd7..2b84fe72 100644
--- a/lib/librte_bpf/bpf_load.c
+++ b/lib/librte_bpf/bpf_load.c
@@ -51,17 +51,64 @@ bpf_load(const struct rte_bpf_prm *prm)
 	return bpf;
 }
 
+/*
+ * Check that user provided external symbol.
+ */
+static int
+bpf_check_xsym(const struct rte_bpf_xsym *xsym)
+{
+	uint32_t i;
+
+	if (xsym->name == NULL)
+		return -EINVAL;
+
+	if (xsym->type == RTE_BPF_XTYPE_VAR) {
+		if (xsym->var.desc.type == RTE_BPF_ARG_UNDEF)
+			return -EINVAL;
+	} else if (xsym->type == RTE_BPF_XTYPE_FUNC) {
+
+		if (xsym->func.nb_args > EBPF_FUNC_MAX_ARGS)
+			return -EINVAL;
+
+		/* check function arguments */
+		for (i = 0; i != xsym->func.nb_args; i++) {
+			if (xsym->func.args[i].type == RTE_BPF_ARG_UNDEF)
+				return -EINVAL;
+		}
+
+		/* check return value info */
+		if (xsym->func.ret.type != RTE_BPF_ARG_UNDEF &&
+				xsym->func.ret.size == 0)
+			return -EINVAL;
+	} else
+		return -EINVAL;
+
+	return 0;
+}
+
 __rte_experimental struct rte_bpf *
 rte_bpf_load(const struct rte_bpf_prm *prm)
 {
 	struct rte_bpf *bpf;
 	int32_t rc;
+	uint32_t i;
 
-	if (prm == NULL || prm->ins == NULL) {
+	if (prm == NULL || prm->ins == NULL ||
+			(prm->nb_xsym != 0 && prm->xsym == NULL)) {
 		rte_errno = EINVAL;
 		return NULL;
 	}
 
+	rc = 0;
+	for (i = 0; i != prm->nb_xsym && rc == 0; i++)
+		rc = bpf_check_xsym(prm->xsym + i);
+
+	if (rc != 0) {
+		rte_errno = -rc;
+		RTE_BPF_LOG(ERR, "%s: %d-th xsym is invalid\n", __func__, i);
+		return NULL;
+	}
+
 	bpf = bpf_load(prm);
 	if (bpf == NULL) {
 		rte_errno = ENOMEM;
diff --git a/lib/librte_bpf/bpf_load_elf.c b/lib/librte_bpf/bpf_load_elf.c
index 6ab03d86..96d3630f 100644
--- a/lib/librte_bpf/bpf_load_elf.c
+++ b/lib/librte_bpf/bpf_load_elf.c
@@ -81,9 +81,9 @@ resolve_xsym(const char *sn, size_t ofs, struct ebpf_insn *ins, size_t ins_sz,
 		ins[idx].imm = fidx;
 	/* for variable we need to store its absolute address */
 	else {
-		ins[idx].imm = (uintptr_t)prm->xsym[fidx].var;
+		ins[idx].imm = (uintptr_t)prm->xsym[fidx].var.val;
 		ins[idx + 1].imm =
-			(uint64_t)(uintptr_t)prm->xsym[fidx].var >> 32;
+			(uint64_t)(uintptr_t)prm->xsym[fidx].var.val >> 32;
 	}
 
 	return 0;
diff --git a/lib/librte_bpf/bpf_validate.c b/lib/librte_bpf/bpf_validate.c
index b7081c85..83983efc 100644
--- a/lib/librte_bpf/bpf_validate.c
+++ b/lib/librte_bpf/bpf_validate.c
@@ -11,9 +11,28 @@
 
 #include <rte_common.h>
 #include <rte_eal.h>
+#include <rte_byteorder.h>
 
 #include "bpf_impl.h"
 
+struct bpf_reg_val {
+	struct rte_bpf_arg v;
+	uint64_t mask;
+	struct {
+		int64_t min;
+		int64_t max;
+	} s;
+	struct {
+		uint64_t min;
+		uint64_t max;
+	} u;
+};
+
+struct bpf_eval_state {
+	struct bpf_reg_val rv[EBPF_REG_NUM];
+	struct bpf_reg_val sv[MAX_BPF_STACK_SIZE / sizeof(uint64_t)];
+};
+
 /* possible instruction node colour */
 enum {
 	WHITE,
@@ -31,14 +50,6 @@ enum {
 	MAX_EDGE_TYPE
 };
 
-struct bpf_reg_state {
-	uint64_t val;
-};
-
-struct bpf_eval_state {
-	struct bpf_reg_state rs[EBPF_REG_NUM];
-};
-
 #define	MAX_EDGES	2
 
 struct inst_node {
@@ -54,12 +65,13 @@ struct inst_node {
 struct bpf_verifier {
 	const struct rte_bpf_prm *prm;
 	struct inst_node *in;
-	int32_t stack_sz;
+	uint64_t stack_sz;
 	uint32_t nb_nodes;
 	uint32_t nb_jcc_nodes;
 	uint32_t node_colour[MAX_NODE_COLOUR];
 	uint32_t edge_type[MAX_EDGE_TYPE];
 	struct bpf_eval_state *evst;
+	struct inst_node *evin;
 	struct {
 		uint32_t num;
 		uint32_t cur;
@@ -101,40 +113,823 @@ check_alu_bele(const struct ebpf_insn *ins)
 }
 
 static const char *
-eval_stack(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+eval_exit(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+	RTE_SET_USED(ins);
+	if (bvf->evst->rv[EBPF_REG_0].v.type == RTE_BPF_ARG_UNDEF)
+		return "undefined return value";
+	return NULL;
+}
+
+/* setup max possible with this mask bounds */
+static void
+eval_umax_bound(struct bpf_reg_val *rv, uint64_t mask)
+{
+	rv->u.max = mask;
+	rv->u.min = 0;
+}
+
+static void
+eval_smax_bound(struct bpf_reg_val *rv, uint64_t mask)
+{
+	rv->s.max = mask >> 1;
+	rv->s.min = rv->s.max ^ UINT64_MAX;
+}
+
+static void
+eval_max_bound(struct bpf_reg_val *rv, uint64_t mask)
+{
+	eval_umax_bound(rv, mask);
+	eval_smax_bound(rv, mask);
+}
+
+static void
+eval_fill_max_bound(struct bpf_reg_val *rv, uint64_t mask)
+{
+	eval_max_bound(rv, mask);
+	rv->v.type = RTE_BPF_ARG_RAW;
+	rv->mask = mask;
+}
+
+static void
+eval_fill_imm64(struct bpf_reg_val *rv, uint64_t mask, uint64_t val)
+{
+	rv->mask = mask;
+	rv->s.min = val;
+	rv->s.max = val;
+	rv->u.min = val;
+	rv->u.max = val;
+}
+
+static void
+eval_fill_imm(struct bpf_reg_val *rv, uint64_t mask, int32_t imm)
+{
+	uint64_t v;
+
+	v = (uint64_t)imm & mask;
+
+	rv->v.type = RTE_BPF_ARG_RAW;
+	eval_fill_imm64(rv, mask, v);
+}
+
+static const char *
+eval_ld_imm64(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
 {
-	int32_t ofs;
+	uint32_t i;
+	uint64_t val;
+	struct bpf_reg_val *rd;
+
+	val = (uint32_t)ins[0].imm | (uint64_t)(uint32_t)ins[1].imm << 32;
 
-	ofs = ins->off;
+	rd = bvf->evst->rv + ins->dst_reg;
+	rd->v.type = RTE_BPF_ARG_RAW;
+	eval_fill_imm64(rd, UINT64_MAX, val);
 
-	if (ofs >= 0 || ofs < -MAX_BPF_STACK_SIZE)
-		return "stack boundary violation";
+	for (i = 0; i != bvf->prm->nb_xsym; i++) {
+
+		/* load of external variable */
+		if (bvf->prm->xsym[i].type == RTE_BPF_XTYPE_VAR &&
+				(uintptr_t)bvf->prm->xsym[i].var.val == val) {
+			rd->v = bvf->prm->xsym[i].var.desc;
+			eval_fill_imm64(rd, UINT64_MAX, 0);
+			break;
+		}
+	}
 
-	ofs = -ofs;
-	bvf->stack_sz = RTE_MAX(bvf->stack_sz, ofs);
 	return NULL;
 }
 
+static void
+eval_apply_mask(struct bpf_reg_val *rv, uint64_t mask)
+{
+	struct bpf_reg_val rt;
+
+	rt.u.min = rv->u.min & mask;
+	rt.u.max = rv->u.max & mask;
+	if (rt.u.min != rv->u.min || rt.u.max != rv->u.max) {
+		rv->u.max = RTE_MAX(rt.u.max, mask);
+		rv->u.min = 0;
+	}
+
+	eval_smax_bound(&rt, mask);
+	rv->s.max = RTE_MIN(rt.s.max, rv->s.max);
+	rv->s.min = RTE_MAX(rt.s.min, rv->s.min);
+
+	rv->mask = mask;
+}
+
+static void
+eval_add(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)
+{
+	struct bpf_reg_val rv;
+
+	rv.u.min = (rd->u.min + rs->u.min) & msk;
+	rv.u.max = (rd->u.min + rs->u.max) & msk;
+	rv.s.min = (rd->s.min + rs->s.min) & msk;
+	rv.s.max = (rd->s.max + rs->s.max) & msk;
+
+	/*
+	 * if at least one of the operands is not constant,
+	 * then check for overflow
+	 */
+	if ((rd->u.min != rd->u.max || rs->u.min != rs->u.max) &&
+			(rv.u.min < rd->u.min || rv.u.max < rd->u.max))
+		eval_umax_bound(&rv, msk);
+
+	if ((rd->s.min != rd->s.max || rs->s.min != rs->s.max) &&
+			(((rs->s.min < 0 && rv.s.min > rd->s.min) ||
+			rv.s.min < rd->s.min) ||
+			((rs->s.max < 0 && rv.s.max > rd->s.max) ||
+				rv.s.max < rd->s.max)))
+		eval_smax_bound(&rv, msk);
+
+	rd->s = rv.s;
+	rd->u = rv.u;
+}
+
+static void
+eval_sub(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)
+{
+	struct bpf_reg_val rv;
+
+	rv.u.min = (rd->u.min - rs->u.min) & msk;
+	rv.u.max = (rd->u.min - rs->u.max) & msk;
+	rv.s.min = (rd->s.min - rs->s.min) & msk;
+	rv.s.max = (rd->s.max - rs->s.max) & msk;
+
+	/*
+	 * if at least one of the operands is not constant,
+	 * then check for overflow
+	 */
+	if ((rd->u.min != rd->u.max || rs->u.min != rs->u.max) &&
+			(rv.u.min > rd->u.min || rv.u.max > rd->u.max))
+		eval_umax_bound(&rv, msk);
+
+	if ((rd->s.min != rd->s.max || rs->s.min != rs->s.max) &&
+			(((rs->s.min < 0 && rv.s.min < rd->s.min) ||
+			rv.s.min > rd->s.min) ||
+			((rs->s.max < 0 && rv.s.max < rd->s.max) ||
+			rv.s.max > rd->s.max)))
+		eval_smax_bound(&rv, msk);
+
+	rd->s = rv.s;
+	rd->u = rv.u;
+}
+
+static void
+eval_lsh(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+	uint64_t msk)
+{
+	/* check if shift value is less then max result bits */
+	if (rs->u.max >= opsz) {
+		eval_max_bound(rd, msk);
+		return;
+	}
+
+	/* check for overflow */
+	if (rd->u.max > RTE_LEN2MASK(opsz - rs->u.max, uint64_t))
+		eval_umax_bound(rd, msk);
+	else {
+		rd->u.max <<= rs->u.max;
+		rd->u.min <<= rs->u.min;
+	}
+
+	/* check that dreg values are and would remain always positive */
+	if ((uint64_t)rd->s.min >> (opsz - 1) != 0 || rd->s.max >=
+			RTE_LEN2MASK(opsz - rs->u.max - 1, int64_t))
+		eval_smax_bound(rd, msk);
+	else {
+		rd->s.max <<= rs->u.max;
+		rd->s.min <<= rs->u.min;
+	}
+}
+
+static void
+eval_rsh(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+	uint64_t msk)
+{
+	/* check if shift value is less then max result bits */
+	if (rs->u.max >= opsz) {
+		eval_max_bound(rd, msk);
+		return;
+	}
+
+	rd->u.max >>= rs->u.min;
+	rd->u.min >>= rs->u.max;
+
+	/* check that dreg values are always positive */
+	if ((uint64_t)rd->s.min >> (opsz - 1) != 0)
+		eval_smax_bound(rd, msk);
+	else {
+		rd->s.max >>= rs->u.min;
+		rd->s.min >>= rs->u.max;
+	}
+}
+
+static void
+eval_arsh(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+	uint64_t msk)
+{
+	uint32_t shv;
+
+	/* check if shift value is less then max result bits */
+	if (rs->u.max >= opsz) {
+		eval_max_bound(rd, msk);
+		return;
+	}
+
+	rd->u.max = (int64_t)rd->u.max >> rs->u.min;
+	rd->u.min = (int64_t)rd->u.min >> rs->u.max;
+
+	/* if we have 32-bit values - extend them to 64-bit */
+	if (opsz == sizeof(uint32_t) * CHAR_BIT) {
+		rd->s.min <<= opsz;
+		rd->s.max <<= opsz;
+		shv = opsz;
+	} else
+		shv = 0;
+
+	if (rd->s.min < 0)
+		rd->s.min = (rd->s.min >> (rs->u.min + shv)) & msk;
+	else
+		rd->s.min = (rd->s.min >> (rs->u.max + shv)) & msk;
+
+	if (rd->s.max < 0)
+		rd->s.max = (rd->s.max >> (rs->u.max + shv)) & msk;
+	else
+		rd->s.max = (rd->s.max >> (rs->u.min + shv)) & msk;
+}
+
+static uint64_t
+eval_umax_bits(uint64_t v, size_t opsz)
+{
+	if (v == 0)
+		return 0;
+
+	v = __builtin_clzll(v);
+	return RTE_LEN2MASK(opsz - v, uint64_t);
+}
+
+/* estimate max possible value for (v1 & v2) */
+static uint64_t
+eval_uand_max(uint64_t v1, uint64_t v2, size_t opsz)
+{
+	v1 = eval_umax_bits(v1, opsz);
+	v2 = eval_umax_bits(v2, opsz);
+	return (v1 & v2);
+}
+
+/* estimate max possible value for (v1 | v2) */
+static uint64_t
+eval_uor_max(uint64_t v1, uint64_t v2, size_t opsz)
+{
+	v1 = eval_umax_bits(v1, opsz);
+	v2 = eval_umax_bits(v2, opsz);
+	return (v1 | v2);
+}
+
+static void
+eval_and(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+	uint64_t msk)
+{
+	/* both operands are constants */
+	if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
+		rd->u.min &= rs->u.min;
+		rd->u.max &= rs->u.max;
+	} else {
+		rd->u.max = eval_uand_max(rd->u.max, rs->u.max, opsz);
+		rd->u.min &= rs->u.min;
+	}
+
+	/* both operands are constants */
+	if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
+		rd->s.min &= rs->s.min;
+		rd->s.max &= rs->s.max;
+	/* at least one of operand is non-negative */
+	} else if (rd->s.min >= 0 || rs->s.min >= 0) {
+		rd->s.max = eval_uand_max(rd->s.max & (msk >> 1),
+			rs->s.max & (msk >> 1), opsz);
+		rd->s.min &= rs->s.min;
+	} else
+		eval_smax_bound(rd, msk);
+}
+
+static void
+eval_or(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+	uint64_t msk)
+{
+	/* both operands are constants */
+	if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
+		rd->u.min |= rs->u.min;
+		rd->u.max |= rs->u.max;
+	} else {
+		rd->u.max = eval_uor_max(rd->u.max, rs->u.max, opsz);
+		rd->u.min |= rs->u.min;
+	}
+
+	/* both operands are constants */
+	if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
+		rd->s.min |= rs->s.min;
+		rd->s.max |= rs->s.max;
+
+	/* both operands are non-negative */
+	} else if (rd->s.min >= 0 || rs->s.min >= 0) {
+		rd->s.max = eval_uor_max(rd->s.max, rs->s.max, opsz);
+		rd->s.min |= rs->s.min;
+	} else
+		eval_smax_bound(rd, msk);
+}
+
+static void
+eval_xor(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+	uint64_t msk)
+{
+	/* both operands are constants */
+	if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
+		rd->u.min ^= rs->u.min;
+		rd->u.max ^= rs->u.max;
+	} else {
+		rd->u.max = eval_uor_max(rd->u.max, rs->u.max, opsz);
+		rd->u.min = 0;
+	}
+
+	/* both operands are constants */
+	if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
+		rd->s.min ^= rs->s.min;
+		rd->s.max ^= rs->s.max;
+
+	/* both operands are non-negative */
+	} else if (rd->s.min >= 0 || rs->s.min >= 0) {
+		rd->s.max = eval_uor_max(rd->s.max, rs->s.max, opsz);
+		rd->s.min = 0;
+	} else
+		eval_smax_bound(rd, msk);
+}
+
+static void
+eval_mul(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
+	uint64_t msk)
+{
+	/* both operands are constants */
+	if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
+		rd->u.min = (rd->u.min * rs->u.min) & msk;
+		rd->u.max = (rd->u.max * rs->u.max) & msk;
+	/* check for overflow */
+	} else if (rd->u.max <= msk >> opsz / 2 && rs->u.max <= msk >> opsz) {
+		rd->u.max *= rs->u.max;
+		rd->u.min *= rd->u.min;
+	} else
+		eval_umax_bound(rd, msk);
+
+	/* both operands are constants */
+	if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
+		rd->s.min = (rd->s.min * rs->s.min) & msk;
+		rd->s.max = (rd->s.max * rs->s.max) & msk;
+	/* check that both operands are positive and no overflow */
+	} else if (rd->s.min >= 0 && rs->s.min >= 0) {
+		rd->s.max *= rs->s.max;
+		rd->s.min *= rd->s.min;
+	} else
+		eval_smax_bound(rd, msk);
+}
+
 static const char *
-eval_store(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+eval_divmod(uint32_t op, struct bpf_reg_val *rd, struct bpf_reg_val *rs,
+	size_t opsz, uint64_t msk)
 {
-	if (ins->dst_reg == EBPF_REG_10)
-		return eval_stack(bvf, ins);
+	/* both operands are constants */
+	if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
+		if (rs->u.max == 0)
+			return "division by 0";
+		if (op == BPF_DIV) {
+			rd->u.min /= rs->u.min;
+			rd->u.max /= rs->u.max;
+		} else {
+			rd->u.min %= rs->u.min;
+			rd->u.max %= rs->u.max;
+		}
+	} else {
+		if (op == BPF_MOD)
+			rd->u.max = RTE_MIN(rd->u.max, rs->u.max - 1);
+		else
+			rd->u.max = rd->u.max;
+		rd->u.min = 0;
+	}
+
+	/* if we have 32-bit values - extend them to 64-bit */
+	if (opsz == sizeof(uint32_t) * CHAR_BIT) {
+		rd->s.min = (int32_t)rd->s.min;
+		rd->s.max = (int32_t)rd->s.max;
+		rs->s.min = (int32_t)rs->s.min;
+		rs->s.max = (int32_t)rs->s.max;
+	}
+
+	/* both operands are constants */
+	if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
+		if (rs->s.max == 0)
+			return "division by 0";
+		if (op == BPF_DIV) {
+			rd->s.min /= rs->s.min;
+			rd->s.max /= rs->s.max;
+		} else {
+			rd->s.min %= rs->s.min;
+			rd->s.max %= rs->s.max;
+		}
+	} else if (op == BPF_MOD) {
+		rd->s.min = RTE_MAX(rd->s.max, 0);
+		rd->s.min = RTE_MIN(rd->s.min, 0);
+	} else
+		eval_smax_bound(rd, msk);
+
+	rd->s.max &= msk;
+	rd->s.min &= msk;
+
 	return NULL;
 }
 
+static void
+eval_neg(struct bpf_reg_val *rd, size_t opsz, uint64_t msk)
+{
+	uint64_t ux, uy;
+	int64_t sx, sy;
+
+	/* if we have 32-bit values - extend them to 64-bit */
+	if (opsz == sizeof(uint32_t) * CHAR_BIT) {
+		rd->u.min = (int32_t)rd->u.min;
+		rd->u.max = (int32_t)rd->u.max;
+	}
+
+	ux = -(int64_t)rd->u.min & msk;
+	uy = -(int64_t)rd->u.max & msk;
+
+	rd->u.max = RTE_MAX(ux, uy);
+	rd->u.min = RTE_MIN(ux, uy);
+
+	/* if we have 32-bit values - extend them to 64-bit */
+	if (opsz == sizeof(uint32_t) * CHAR_BIT) {
+		rd->s.min = (int32_t)rd->s.min;
+		rd->s.max = (int32_t)rd->s.max;
+	}
+
+	sx = -rd->s.min & msk;
+	sy = -rd->s.max & msk;
+
+	rd->s.max = RTE_MAX(sx, sy);
+	rd->s.min = RTE_MIN(sx, sy);
+}
+
+/*
+ * check that destination and source operand are in defined state.
+ */
+static const char *
+eval_defined(const struct bpf_reg_val *dst, const struct bpf_reg_val *src)
+{
+	if (dst != NULL && dst->v.type == RTE_BPF_ARG_UNDEF)
+		return "dest reg value is undefined";
+	if (src != NULL && src->v.type == RTE_BPF_ARG_UNDEF)
+		return "src reg value is undefined";
+	return NULL;
+}
+
+static const char *
+eval_alu(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+	uint64_t msk;
+	uint32_t op;
+	size_t opsz;
+	const char *err;
+	struct bpf_eval_state *st;
+	struct bpf_reg_val *rd, rs;
+
+	opsz = (BPF_CLASS(ins->code) == BPF_ALU) ?
+		sizeof(uint32_t) : sizeof(uint64_t);
+	opsz = opsz * CHAR_BIT;
+	msk = RTE_LEN2MASK(opsz, uint64_t);
+
+	st = bvf->evst;
+	rd = st->rv + ins->dst_reg;
+
+	if (BPF_SRC(ins->code) == BPF_X) {
+		rs = st->rv[ins->src_reg];
+		eval_apply_mask(&rs, msk);
+	} else
+		eval_fill_imm(&rs, msk, ins->imm);
+
+	eval_apply_mask(rd, msk);
+
+	op = BPF_OP(ins->code);
+
+	err = eval_defined((op != EBPF_MOV) ? rd : NULL,
+			(op != BPF_NEG) ? &rs : NULL);
+	if (err != NULL)
+		return err;
+
+	if (op == BPF_ADD)
+		eval_add(rd, &rs, msk);
+	else if (op == BPF_SUB)
+		eval_sub(rd, &rs, msk);
+	else if (op == BPF_LSH)
+		eval_lsh(rd, &rs, opsz, msk);
+	else if (op == BPF_RSH)
+		eval_rsh(rd, &rs, opsz, msk);
+	else if (op == EBPF_ARSH)
+		eval_arsh(rd, &rs, opsz, msk);
+	else if (op == BPF_AND)
+		eval_and(rd, &rs, opsz, msk);
+	else if (op == BPF_OR)
+		eval_or(rd, &rs, opsz, msk);
+	else if (op == BPF_XOR)
+		eval_xor(rd, &rs, opsz, msk);
+	else if (op == BPF_MUL)
+		eval_mul(rd, &rs, opsz, msk);
+	else if (op == BPF_DIV || op == BPF_MOD)
+		err = eval_divmod(op, rd, &rs, opsz, msk);
+	else if (op == BPF_NEG)
+		eval_neg(rd, opsz, msk);
+	else if (op == EBPF_MOV)
+		*rd = rs;
+	else
+		eval_max_bound(rd, msk);
+
+	return err;
+}
+
+static const char *
+eval_bele(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+	uint64_t msk;
+	struct bpf_eval_state *st;
+	struct bpf_reg_val *rd;
+	const char *err;
+
+	msk = RTE_LEN2MASK(ins->imm, uint64_t);
+
+	st = bvf->evst;
+	rd = st->rv + ins->dst_reg;
+
+	err = eval_defined(rd, NULL);
+	if (err != NULL)
+		return err;
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+	if (ins->code == (BPF_ALU | EBPF_END | EBPF_TO_BE))
+		eval_max_bound(rd, msk);
+	else
+		eval_apply_mask(rd, msk);
+#else
+	if (ins->code == (BPF_ALU | EBPF_END | EBPF_TO_LE))
+		eval_max_bound(rd, msk);
+	else
+		eval_apply_mask(rd, msk);
+#endif
+
+	return NULL;
+}
+
+static const char *
+eval_ptr(struct bpf_verifier *bvf, struct bpf_reg_val *rm, uint32_t opsz,
+	uint32_t align, int16_t off)
+{
+	struct bpf_reg_val rv;
+
+	/* calculate reg + offset */
+	eval_fill_imm(&rv, rm->mask, off);
+	eval_add(rm, &rv, rm->mask);
+
+	if (RTE_BPF_ARG_PTR_TYPE(rm->v.type) == 0)
+		return "destination is not a pointer";
+
+	if (rm->mask != UINT64_MAX)
+		return "pointer truncation";
+
+	if (rm->u.max + opsz > rm->v.size ||
+			(uint64_t)rm->s.max + opsz > rm->v.size ||
+			rm->s.min < 0)
+		return "memory boundary violation";
+
+	if (rm->u.max % align !=  0)
+		return "unaligned memory access";
+
+	if (rm->v.type == RTE_BPF_ARG_PTR_STACK) {
+
+		if (rm->u.max != rm->u.min || rm->s.max != rm->s.min ||
+				rm->u.max != (uint64_t)rm->s.max)
+			return "stack access with variable offset";
+
+		bvf->stack_sz = RTE_MAX(bvf->stack_sz, rm->v.size - rm->u.max);
+
+	/* pointer to mbuf */
+	} else if (rm->v.type == RTE_BPF_ARG_PTR_MBUF) {
+
+		if (rm->u.max != rm->u.min || rm->s.max != rm->s.min ||
+				rm->u.max != (uint64_t)rm->s.max)
+			return "mbuf access with variable offset";
+	}
+
+	return NULL;
+}
+
+static void
+eval_max_load(struct bpf_reg_val *rv, uint64_t mask)
+{
+	eval_umax_bound(rv, mask);
+
+	/* full 64-bit load */
+	if (mask == UINT64_MAX)
+		eval_smax_bound(rv, mask);
+
+	/* zero-extend load */
+	rv->s.min = rv->u.min;
+	rv->s.max = rv->u.max;
+}
+
+
 static const char *
 eval_load(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
 {
-	if (ins->src_reg == EBPF_REG_10)
-		return eval_stack(bvf, ins);
+	uint32_t opsz;
+	uint64_t msk;
+	const char *err;
+	struct bpf_eval_state *st;
+	struct bpf_reg_val *rd, rs;
+	const struct bpf_reg_val *sv;
+
+	st = bvf->evst;
+	rd = st->rv + ins->dst_reg;
+	rs = st->rv[ins->src_reg];
+	opsz = bpf_size(BPF_SIZE(ins->code));
+	msk = RTE_LEN2MASK(opsz * CHAR_BIT, uint64_t);
+
+	err = eval_ptr(bvf, &rs, opsz, 1, ins->off);
+	if (err != NULL)
+		return err;
+
+	if (rs.v.type == RTE_BPF_ARG_PTR_STACK) {
+
+		sv = st->sv + rs.u.max / sizeof(uint64_t);
+		if (sv->v.type == RTE_BPF_ARG_UNDEF || sv->mask < msk)
+			return "undefined value on the stack";
+
+		*rd = *sv;
+
+	/* pointer to mbuf */
+	} else if (rs.v.type == RTE_BPF_ARG_PTR_MBUF) {
+
+		if (rs.u.max == offsetof(struct rte_mbuf, next)) {
+			eval_fill_imm(rd, msk, 0);
+			rd->v = rs.v;
+		} else if (rs.u.max == offsetof(struct rte_mbuf, buf_addr)) {
+			eval_fill_imm(rd, msk, 0);
+			rd->v.type = RTE_BPF_ARG_PTR;
+			rd->v.size = rs.v.buf_size;
+		} else if (rs.u.max == offsetof(struct rte_mbuf, data_off)) {
+			eval_fill_imm(rd, msk, RTE_PKTMBUF_HEADROOM);
+			rd->v.type = RTE_BPF_ARG_RAW;
+		} else {
+			eval_max_load(rd, msk);
+			rd->v.type = RTE_BPF_ARG_RAW;
+		}
+
+	/* pointer to raw data */
+	} else {
+		eval_max_load(rd, msk);
+		rd->v.type = RTE_BPF_ARG_RAW;
+	}
+
 	return NULL;
 }
 
 static const char *
+eval_mbuf_store(const struct bpf_reg_val *rv, uint32_t opsz)
+{
+	uint32_t i;
+
+	static const struct {
+		size_t off;
+		size_t sz;
+	} mbuf_ro_fileds[] = {
+		{ .off = offsetof(struct rte_mbuf, buf_addr), },
+		{ .off = offsetof(struct rte_mbuf, refcnt), },
+		{ .off = offsetof(struct rte_mbuf, nb_segs), },
+		{ .off = offsetof(struct rte_mbuf, buf_len), },
+		{ .off = offsetof(struct rte_mbuf, pool), },
+		{ .off = offsetof(struct rte_mbuf, next), },
+		{ .off = offsetof(struct rte_mbuf, priv_size), },
+	};
+
+	for (i = 0; i != RTE_DIM(mbuf_ro_fileds) &&
+			(mbuf_ro_fileds[i].off + mbuf_ro_fileds[i].sz <=
+			rv->u.max || rv->u.max + opsz <= mbuf_ro_fileds[i].off);
+			i++)
+		;
+
+	if (i != RTE_DIM(mbuf_ro_fileds))
+		return "store to the read-only mbuf field";
+
+	return NULL;
+
+}
+
+static const char *
+eval_store(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+	uint32_t opsz;
+	uint64_t msk;
+	const char *err;
+	struct bpf_eval_state *st;
+	struct bpf_reg_val rd, rs, *sv;
+
+	opsz = bpf_size(BPF_SIZE(ins->code));
+	msk = RTE_LEN2MASK(opsz * CHAR_BIT, uint64_t);
+
+	st = bvf->evst;
+	rd = st->rv[ins->dst_reg];
+
+	if (BPF_CLASS(ins->code) == BPF_STX) {
+		rs = st->rv[ins->src_reg];
+		eval_apply_mask(&rs, msk);
+	} else
+		eval_fill_imm(&rs, msk, ins->imm);
+
+	err = eval_defined(NULL, &rs);
+	if (err != NULL)
+		return err;
+
+	err = eval_ptr(bvf, &rd, opsz, 1, ins->off);
+	if (err != NULL)
+		return err;
+
+	if (rd.v.type == RTE_BPF_ARG_PTR_STACK) {
+
+		sv = st->sv + rd.u.max / sizeof(uint64_t);
+		if (BPF_CLASS(ins->code) == BPF_STX &&
+				BPF_MODE(ins->code) == EBPF_XADD)
+			eval_max_bound(sv, msk);
+		else
+			*sv = rs;
+
+	/* pointer to mbuf */
+	} else if (rd.v.type == RTE_BPF_ARG_PTR_MBUF) {
+		err = eval_mbuf_store(&rd, opsz);
+		if (err != NULL)
+			return err;
+	}
+
+	return NULL;
+}
+
+static const char *
+eval_func_arg(struct bpf_verifier *bvf, const struct rte_bpf_arg *arg,
+	struct bpf_reg_val *rv)
+{
+	uint32_t i, n;
+	struct bpf_eval_state *st;
+	const char *err;
+
+	st = bvf->evst;
+
+	if (rv->v.type == RTE_BPF_ARG_UNDEF)
+		return "Undefined argument type";
+
+	if (arg->type != rv->v.type &&
+			arg->type != RTE_BPF_ARG_RAW &&
+			(arg->type != RTE_BPF_ARG_PTR ||
+			RTE_BPF_ARG_PTR_TYPE(rv->v.type) == 0))
+		return "Invalid argument type";
+
+	err = NULL;
+
+	/* argument is a pointer */
+	if (RTE_BPF_ARG_PTR_TYPE(arg->type) != 0) {
+
+		err = eval_ptr(bvf, rv, arg->size, 1, 0);
+
+		/*
+		 * pointer to the variable on the stack is passed
+		 * as an argument, mark stack space it occupies as initialized.
+		 */
+		if (err == NULL && rv->v.type == RTE_BPF_ARG_PTR_STACK) {
+
+			i = rv->u.max / sizeof(uint64_t);
+			n = i + arg->size / sizeof(uint64_t);
+			while (i != n) {
+				eval_fill_max_bound(st->sv + i, UINT64_MAX);
+				i++;
+			};
+		}
+	}
+
+	return err;
+}
+
+static const char *
 eval_call(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
 {
-	uint32_t idx;
+	uint64_t msk;
+	uint32_t i, idx;
+	struct bpf_reg_val *rv;
+	const struct rte_bpf_xsym *xsym;
+	const char *err;
 
 	idx = ins->imm;
 
@@ -145,6 +940,144 @@ eval_call(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
 	/* for now don't support function calls on 32 bit platform */
 	if (sizeof(uint64_t) != sizeof(uintptr_t))
 		return "function calls are supported only for 64 bit apps";
+
+	xsym = bvf->prm->xsym + idx;
+
+	/* evaluate function arguments */
+	err = NULL;
+	for (i = 0; i != xsym->func.nb_args && err == NULL; i++) {
+		err = eval_func_arg(bvf, xsym->func.args + i,
+			bvf->evst->rv + EBPF_REG_1 + i);
+	}
+
+	/* R1-R5 argument/scratch registers */
+	for (i = EBPF_REG_1; i != EBPF_REG_6; i++)
+		bvf->evst->rv[i].v.type = RTE_BPF_ARG_UNDEF;
+
+	/* update return value */
+
+	rv = bvf->evst->rv + EBPF_REG_0;
+	rv->v = xsym->func.ret;
+	msk = (rv->v.type == RTE_BPF_ARG_RAW) ?
+		RTE_LEN2MASK(rv->v.size * CHAR_BIT, uint64_t) : UINTPTR_MAX;
+	eval_max_bound(rv, msk);
+	rv->mask = msk;
+
+	return err;
+}
+
+static void
+eval_jeq_jne(struct bpf_reg_val *trd, struct bpf_reg_val *trs)
+{
+	/* sreg is constant */
+	if (trs->u.min == trs->u.max) {
+		trd->u = trs->u;
+	/* dreg is constant */
+	} else if (trd->u.min == trd->u.max) {
+		trs->u = trd->u;
+	} else {
+		trd->u.max = RTE_MIN(trd->u.max, trs->u.max);
+		trd->u.min = RTE_MAX(trd->u.min, trs->u.min);
+		trs->u = trd->u;
+	}
+
+	/* sreg is constant */
+	if (trs->s.min == trs->s.max) {
+		trd->s = trs->s;
+	/* dreg is constant */
+	} else if (trd->s.min == trd->s.max) {
+		trs->s = trd->s;
+	} else {
+		trd->s.max = RTE_MIN(trd->s.max, trs->s.max);
+		trd->s.min = RTE_MAX(trd->s.min, trs->s.min);
+		trs->s = trd->s;
+	}
+}
+
+static void
+eval_jgt_jle(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
+	struct bpf_reg_val *frd, struct bpf_reg_val *frs)
+{
+	frd->u.max = RTE_MIN(frd->u.max, frs->u.min);
+	trd->u.min = RTE_MAX(trd->u.min, trs->u.min + 1);
+}
+
+static void
+eval_jlt_jge(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
+	struct bpf_reg_val *frd, struct bpf_reg_val *frs)
+{
+	frd->u.min = RTE_MAX(frd->u.min, frs->u.min);
+	trd->u.max = RTE_MIN(trd->u.max, trs->u.max - 1);
+}
+
+static void
+eval_jsgt_jsle(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
+	struct bpf_reg_val *frd, struct bpf_reg_val *frs)
+{
+	frd->s.max = RTE_MIN(frd->s.max, frs->s.min);
+	trd->s.min = RTE_MAX(trd->s.min, trs->s.min + 1);
+}
+
+static void
+eval_jslt_jsge(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
+	struct bpf_reg_val *frd, struct bpf_reg_val *frs)
+{
+	frd->s.min = RTE_MAX(frd->s.min, frs->s.min);
+	trd->s.max = RTE_MIN(trd->s.max, trs->s.max - 1);
+}
+
+static const char *
+eval_jcc(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+	uint32_t op;
+	const char *err;
+	struct bpf_eval_state *fst, *tst;
+	struct bpf_reg_val *frd, *frs, *trd, *trs;
+	struct bpf_reg_val rvf, rvt;
+
+	tst = bvf->evst;
+	fst = bvf->evin->evst;
+
+	frd = fst->rv + ins->dst_reg;
+	trd = tst->rv + ins->dst_reg;
+
+	if (BPF_SRC(ins->code) == BPF_X) {
+		frs = fst->rv + ins->src_reg;
+		trs = tst->rv + ins->src_reg;
+	} else {
+		frs = &rvf;
+		trs = &rvt;
+		eval_fill_imm(frs, UINT64_MAX, ins->imm);
+		eval_fill_imm(trs, UINT64_MAX, ins->imm);
+	}
+
+	err = eval_defined(trd, trs);
+	if (err != NULL)
+		return err;
+
+	op = BPF_OP(ins->code);
+
+	if (op == BPF_JEQ)
+		eval_jeq_jne(trd, trs);
+	else if (op == EBPF_JNE)
+		eval_jeq_jne(frd, frs);
+	else if (op == BPF_JGT)
+		eval_jgt_jle(trd, trs, frd, frs);
+	else if (op == EBPF_JLE)
+		eval_jgt_jle(frd, frs, trd, trs);
+	else if (op == EBPF_JLT)
+		eval_jlt_jge(trd, trs, frd, frs);
+	else if (op == BPF_JGE)
+		eval_jlt_jge(frd, frs, trd, trs);
+	else if (op == EBPF_JSGT)
+		eval_jsgt_jsle(trd, trs, frd, frs);
+	else if (op == EBPF_JSLE)
+		eval_jsgt_jsle(frd, frs, trd, trs);
+	else if (op == EBPF_JLT)
+		eval_jslt_jsge(trd, trs, frd, frs);
+	else if (op == EBPF_JSGE)
+		eval_jslt_jsge(frd, frs, trd, trs);
+
 	return NULL;
 }
 
@@ -157,256 +1090,306 @@ static const struct bpf_ins_check ins_chk[UINT8_MAX] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_SUB | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_AND | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_OR | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_LSH | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_RSH | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_XOR | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_MUL | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | EBPF_MOV | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_DIV | BPF_K)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 1, .max = UINT32_MAX},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_MOD | BPF_K)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 1, .max = UINT32_MAX},
+		.eval = eval_alu,
 	},
 	/* ALU IMM 64-bit instructions */
 	[(EBPF_ALU64 | BPF_ADD | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_SUB | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_AND | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_OR | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_LSH | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_RSH | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | EBPF_ARSH | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_XOR | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_MUL | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | EBPF_MOV | BPF_K)] = {
 		.mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX,},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_DIV | BPF_K)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 1, .max = UINT32_MAX},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_MOD | BPF_K)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 1, .max = UINT32_MAX},
+		.eval = eval_alu,
 	},
 	/* ALU REG 32-bit instructions */
 	[(BPF_ALU | BPF_ADD | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_SUB | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_AND | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_OR | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_LSH | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_RSH | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_XOR | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_MUL | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_DIV | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_MOD | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | EBPF_MOV | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | BPF_NEG)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(BPF_ALU | EBPF_END | EBPF_TO_BE)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 16, .max = 64},
 		.check = check_alu_bele,
+		.eval = eval_bele,
 	},
 	[(BPF_ALU | EBPF_END | EBPF_TO_LE)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 16, .max = 64},
 		.check = check_alu_bele,
+		.eval = eval_bele,
 	},
 	/* ALU REG 64-bit instructions */
 	[(EBPF_ALU64 | BPF_ADD | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_SUB | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_AND | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_OR | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_LSH | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_RSH | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | EBPF_ARSH | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_XOR | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_MUL | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_DIV | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_MOD | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | EBPF_MOV | BPF_X)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	[(EBPF_ALU64 | BPF_NEG)] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_alu,
 	},
 	/* load instructions */
 	[(BPF_LDX | BPF_MEM | BPF_B)] = {
@@ -438,6 +1421,7 @@ static const struct bpf_ins_check ins_chk[UINT8_MAX] = {
 		.mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_ld_imm64,
 	},
 	/* store REG instructions */
 	[(BPF_STX | BPF_MEM | BPF_B)] = {
@@ -513,92 +1497,110 @@ static const struct bpf_ins_check ins_chk[UINT8_MAX] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JNE | BPF_K)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | BPF_JGT | BPF_K)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JLT | BPF_K)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | BPF_JGE | BPF_K)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JLE | BPF_K)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JSGT | BPF_K)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JSLT | BPF_K)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JSGE | BPF_K)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JSLE | BPF_K)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | BPF_JSET | BPF_K)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_jcc,
 	},
 	/* jcc REG instructions */
 	[(BPF_JMP | BPF_JEQ | BPF_X)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JNE | BPF_X)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | BPF_JGT | BPF_X)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JLT | BPF_X)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | BPF_JGE | BPF_X)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JLE | BPF_X)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JSGT | BPF_X)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JSLT | BPF_X)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
@@ -609,16 +1611,19 @@ static const struct bpf_ins_check ins_chk[UINT8_MAX] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | EBPF_JSLE | BPF_X)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_jcc,
 	},
 	[(BPF_JMP | BPF_JSET | BPF_X)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
 		.off = { .min = 0, .max = UINT16_MAX},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_jcc,
 	},
 	/* call instruction */
 	[(BPF_JMP | EBPF_CALL)] = {
@@ -632,6 +1637,7 @@ static const struct bpf_ins_check ins_chk[UINT8_MAX] = {
 		.mask = { .dreg = ZERO_REG, .sreg = ZERO_REG},
 		.off = { .min = 0, .max = 0},
 		.imm = { .min = 0, .max = 0},
+		.eval = eval_exit,
 	},
 };
 
@@ -1046,7 +2052,7 @@ save_eval_state(struct bpf_verifier *bvf, struct inst_node *node)
 	st = pull_eval_state(bvf);
 	if (st == NULL) {
 		RTE_BPF_LOG(ERR,
-			"%s: internal error (out of space) at pc: %u",
+			"%s: internal error (out of space) at pc: %u\n",
 			__func__, get_node_idx(bvf, node));
 		return -ENOMEM;
 	}
@@ -1078,6 +2084,32 @@ restore_eval_state(struct bpf_verifier *bvf, struct inst_node *node)
 	push_eval_state(bvf);
 }
 
+static void
+log_eval_state(const struct bpf_verifier *bvf, const struct ebpf_insn *ins,
+	uint32_t pc, int32_t loglvl)
+{
+	const struct bpf_eval_state *st;
+	const struct bpf_reg_val *rv;
+
+	rte_log(loglvl, rte_bpf_logtype, "%s(pc=%u):\n", __func__, pc);
+
+	st = bvf->evst;
+	rv = st->rv + ins->dst_reg;
+
+	rte_log(loglvl, rte_bpf_logtype,
+		"r%u={\n"
+		"\tv={type=%u, size=%zu},\n"
+		"\tmask=0x%" PRIx64 ",\n"
+		"\tu={min=0x%" PRIx64 ", max=0x%" PRIx64 "},\n"
+		"\ts={min=%" PRId64 ", max=%" PRId64 "},\n"
+		"};\n",
+		ins->dst_reg,
+		rv->v.type, rv->v.size,
+		rv->mask,
+		rv->u.min, rv->u.max,
+		rv->s.min, rv->s.max);
+}
+
 /*
  * Do second pass through CFG and try to evaluate instructions
  * via each possible path.
@@ -1096,23 +2128,56 @@ evaluate(struct bpf_verifier *bvf)
 	const struct ebpf_insn *ins;
 	struct inst_node *next, *node;
 
-	node = bvf->in;
+	/* initial state of frame pointer */
+	static const struct bpf_reg_val rvfp = {
+		.v = {
+			.type = RTE_BPF_ARG_PTR_STACK,
+			.size = MAX_BPF_STACK_SIZE,
+		},
+		.mask = UINT64_MAX,
+		.u = {.min = MAX_BPF_STACK_SIZE, .max = MAX_BPF_STACK_SIZE},
+		.s = {.min = MAX_BPF_STACK_SIZE, .max = MAX_BPF_STACK_SIZE},
+	};
+
+	bvf->evst->rv[EBPF_REG_1].v = bvf->prm->prog_arg;
+	bvf->evst->rv[EBPF_REG_1].mask = UINT64_MAX;
+	if (bvf->prm->prog_arg.type == RTE_BPF_ARG_RAW)
+		eval_max_bound(bvf->evst->rv + EBPF_REG_1, UINT64_MAX);
+
+	bvf->evst->rv[EBPF_REG_10] = rvfp;
+
 	ins = bvf->prm->ins;
+	node = bvf->in;
+	next = node;
 	rc = 0;
 
 	while (node != NULL && rc == 0) {
 
-		/* current node evaluation */
-		idx = get_node_idx(bvf, node);
-		op = ins[idx].code;
+		/*
+		 * current node evaluation, make sure we evaluate
+		 * each node only once.
+		 */
+		if (next != NULL) {
+
+			bvf->evin = node;
+			idx = get_node_idx(bvf, node);
+			op = ins[idx].code;
 
-		if (ins_chk[op].eval != NULL) {
-			err = ins_chk[op].eval(bvf, ins + idx);
-			if (err != NULL) {
-				RTE_BPF_LOG(ERR, "%s: %s at pc: %u\n",
-					__func__, err, idx);
-				rc = -EINVAL;
+			/* for jcc node make a copy of evaluatoion state */
+			if (node->nb_edge > 1)
+				rc |= save_eval_state(bvf, node);
+
+			if (ins_chk[op].eval != NULL && rc == 0) {
+				err = ins_chk[op].eval(bvf, ins + idx);
+				if (err != NULL) {
+					RTE_BPF_LOG(ERR, "%s: %s at pc: %u\n",
+						__func__, err, idx);
+					rc = -EINVAL;
+				}
 			}
+
+			log_eval_state(bvf, ins + idx, idx, RTE_LOG_DEBUG);
+			bvf->evin = NULL;
 		}
 
 		/* proceed through CFG */
@@ -1120,9 +2185,8 @@ evaluate(struct bpf_verifier *bvf)
 		if (next != NULL) {
 
 			/* proceed with next child */
-			if (node->cur_edge != node->nb_edge)
-				rc |= save_eval_state(bvf, node);
-			else if (node->evst != NULL)
+			if (node->cur_edge == node->nb_edge &&
+					node->evst != NULL)
 				restore_eval_state(bvf, node);
 
 			next->prev_node = get_node_idx(bvf, node);
diff --git a/lib/librte_bpf/meson.build b/lib/librte_bpf/meson.build
index de9de009..bc0cd78f 100644
--- a/lib/librte_bpf/meson.build
+++ b/lib/librte_bpf/meson.build
@@ -8,7 +8,7 @@ sources = files('bpf.c',
 		'bpf_pkt.c',
 		'bpf_validate.c')
 
-if arch_subdir == 'x86'
+if arch_subdir == 'x86' and cc.sizeof('void *') == 8
 	sources += files('bpf_jit_x86.c')
 endif
 
diff --git a/lib/librte_bpf/rte_bpf.h b/lib/librte_bpf/rte_bpf.h
index 1249a992..ad62ef2c 100644
--- a/lib/librte_bpf/rte_bpf.h
+++ b/lib/librte_bpf/rte_bpf.h
@@ -40,7 +40,11 @@ enum rte_bpf_arg_type {
  */
 struct rte_bpf_arg {
 	enum rte_bpf_arg_type type;
-	size_t size;     /**< for pointer types, size of data it points to */
+	/**
+	 * for ptr type - max size of data buffer it points to
+	 * for raw type - the size (in bytes) of the value
+	 */
+	size_t size;
 	size_t buf_size;
 	/**< for mbuf ptr type, max size of rte_mbuf data buffer */
 };
@@ -66,10 +70,19 @@ struct rte_bpf_xsym {
 	const char *name;        /**< name */
 	enum rte_bpf_xtype type; /**< type */
 	union {
-		uint64_t (*func)(uint64_t, uint64_t, uint64_t,
+		struct {
+			uint64_t (*val)(uint64_t, uint64_t, uint64_t,
 				uint64_t, uint64_t);
-		void *var;
-	}; /**< value */
+			uint32_t nb_args;
+			struct rte_bpf_arg args[EBPF_FUNC_MAX_ARGS];
+			/**< Function arguments descriptions. */
+			struct rte_bpf_arg ret; /**< function return value. */
+		} func;
+		struct {
+			void *val; /**< actual memory location */
+			struct rte_bpf_arg desc; /**< type, size, etc. */
+		} var; /**< external variable */
+	};
 };
 
 /**
diff --git a/lib/librte_cmdline/cmdline_parse.c b/lib/librte_cmdline/cmdline_parse.c
index 961f9bef..9666e90c 100644
--- a/lib/librte_cmdline/cmdline_parse.c
+++ b/lib/librte_cmdline/cmdline_parse.c
@@ -208,9 +208,6 @@ cmdline_parse(struct cmdline *cl, const char * buf)
 	int err = CMDLINE_PARSE_NOMATCH;
 	int tok;
 	cmdline_parse_ctx_t *ctx;
-#ifdef RTE_LIBRTE_CMDLINE_DEBUG
-	char debug_buf[BUFSIZ];
-#endif
 	char *result_buf = result.buf;
 
 	if (!cl || !buf)
@@ -250,10 +247,8 @@ cmdline_parse(struct cmdline *cl, const char * buf)
 		return linelen;
 	}
 
-#ifdef RTE_LIBRTE_CMDLINE_DEBUG
-	strlcpy(debug_buf, buf, (linelen > 64 ? 64 : linelen));
-	debug_printf("Parse line : len=%d, <%s>\n", linelen, debug_buf);
-#endif
+	debug_printf("Parse line : len=%d, <%.*s>\n",
+		     linelen, linelen > 64 ? 64 : linelen, buf);
 
 	/* parse it !! */
 	inst = ctx[inst_num];
diff --git a/lib/librte_compat/Makefile b/lib/librte_compat/Makefile
index 0c57533c..61089fe7 100644
--- a/lib/librte_compat/Makefile
+++ b/lib/librte_compat/Makefile
@@ -1,33 +1,6 @@
-#   BSD LICENSE
-#
-#   Copyright(c) 2013 Neil Horman <nhorman@tuxdriver.com>
-#   All rights reserved.
-#
-#   Redistribution and use in source and binary forms, with or without
-#   modification, are permitted provided that the following conditions
-#   are met:
-#
-#     * Redistributions of source code must retain the above copyright
-#       notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright
-#       notice, this list of conditions and the following disclaimer in
-#       the documentation and/or other materials provided with the
-#       distribution.
-#     * Neither the name of Intel Corporation nor the names of its
-#       contributors may be used to endorse or promote products derived
-#       from this software without specific prior written permission.
-#
-#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2013 Neil Horman <nhorman@tuxdriver.com>
+# All rights reserved.
 
 include $(RTE_SDK)/mk/rte.vars.mk
 
diff --git a/lib/librte_compressdev/rte_comp.c b/lib/librte_compressdev/rte_comp.c
index d596ba87..98ad0cfd 100644
--- a/lib/librte_compressdev/rte_comp.c
+++ b/lib/librte_compressdev/rte_comp.c
@@ -14,8 +14,12 @@ rte_comp_get_feature_name(uint64_t flag)
 		return "STATEFUL_COMPRESSION";
 	case RTE_COMP_FF_STATEFUL_DECOMPRESSION:
 		return "STATEFUL_DECOMPRESSION";
-	case RTE_COMP_FF_MBUF_SCATTER_GATHER:
-		return "MBUF_SCATTER_GATHER";
+	case RTE_COMP_FF_OOP_SGL_IN_SGL_OUT:
+		return "OOP_SGL_IN_SGL_OUT";
+	case RTE_COMP_FF_OOP_SGL_IN_LB_OUT:
+		return "OOP_SGL_IN_LB_OUT";
+	case RTE_COMP_FF_OOP_LB_IN_SGL_OUT:
+		return "OOP_LB_IN_SGL_OUT";
 	case RTE_COMP_FF_MULTI_PKT_CHECKSUM:
 		return "MULTI_PKT_CHECKSUM";
 	case RTE_COMP_FF_ADLER32_CHECKSUM:
@@ -32,6 +36,10 @@ rte_comp_get_feature_name(uint64_t flag)
 		return "SHA2_SHA256_HASH";
 	case RTE_COMP_FF_SHAREABLE_PRIV_XFORM:
 		return "SHAREABLE_PRIV_XFORM";
+	case RTE_COMP_FF_HUFFMAN_FIXED:
+		return "HUFFMAN_FIXED";
+	case RTE_COMP_FF_HUFFMAN_DYNAMIC:
+		return "HUFFMAN_DYNAMIC";
 	default:
 		return NULL;
 	}
diff --git a/lib/librte_compressdev/rte_comp.h b/lib/librte_compressdev/rte_comp.h
index 5b513c77..ee9056ea 100644
--- a/lib/librte_compressdev/rte_comp.h
+++ b/lib/librte_compressdev/rte_comp.h
@@ -30,28 +30,43 @@ extern "C" {
 /**< Stateful compression is supported */
 #define RTE_COMP_FF_STATEFUL_DECOMPRESSION	(1ULL << 1)
 /**< Stateful decompression is supported */
-#define	RTE_COMP_FF_MBUF_SCATTER_GATHER		(1ULL << 2)
-/**< Scatter-gather mbufs are supported */
-#define RTE_COMP_FF_ADLER32_CHECKSUM		(1ULL << 3)
+#define RTE_COMP_FF_OOP_SGL_IN_SGL_OUT		(1ULL << 2)
+/**< Out-of-place Scatter-gather (SGL) buffers,
+ * with multiple segments, are supported in input and output
+ */
+#define RTE_COMP_FF_OOP_SGL_IN_LB_OUT		(1ULL << 3)
+/**< Out-of-place Scatter-gather (SGL) buffers are supported
+ * in input, combined with linear buffers (LB), with a
+ * single segment, in output
+ */
+#define RTE_COMP_FF_OOP_LB_IN_SGL_OUT		(1ULL << 4)
+/**< Out-of-place Scatter-gather (SGL) buffers are supported
+ * in output, combined with linear buffers (LB) in input
+ */
+#define RTE_COMP_FF_ADLER32_CHECKSUM		(1ULL << 5)
 /**< Adler-32 Checksum is supported */
-#define RTE_COMP_FF_CRC32_CHECKSUM		(1ULL << 4)
+#define RTE_COMP_FF_CRC32_CHECKSUM		(1ULL << 6)
 /**< CRC32 Checksum is supported */
-#define RTE_COMP_FF_CRC32_ADLER32_CHECKSUM	(1ULL << 5)
+#define RTE_COMP_FF_CRC32_ADLER32_CHECKSUM	(1ULL << 7)
 /**< Adler-32/CRC32 Checksum is supported */
-#define RTE_COMP_FF_MULTI_PKT_CHECKSUM		(1ULL << 6)
+#define RTE_COMP_FF_MULTI_PKT_CHECKSUM		(1ULL << 8)
 /**< Generation of checksum across multiple stateless packets is supported */
-#define RTE_COMP_FF_SHA1_HASH			(1ULL << 7)
+#define RTE_COMP_FF_SHA1_HASH			(1ULL << 9)
 /**< SHA1 Hash is supported */
-#define RTE_COMP_FF_SHA2_SHA256_HASH		(1ULL << 8)
+#define RTE_COMP_FF_SHA2_SHA256_HASH		(1ULL << 10)
 /**< SHA256 Hash of SHA2 family is supported */
-#define RTE_COMP_FF_NONCOMPRESSED_BLOCKS	(1ULL << 9)
+#define RTE_COMP_FF_NONCOMPRESSED_BLOCKS	(1ULL << 11)
 /**< Creation of non-compressed blocks using RTE_COMP_LEVEL_NONE is supported */
-#define RTE_COMP_FF_SHAREABLE_PRIV_XFORM	(1ULL << 10)
+#define RTE_COMP_FF_SHAREABLE_PRIV_XFORM	(1ULL << 12)
 /**< Private xforms created by the PMD can be shared
  * across multiple stateless operations. If not set, then app needs
  * to create as many priv_xforms as it expects to have stateless
  * operations in-flight.
  */
+#define RTE_COMP_FF_HUFFMAN_FIXED		(1ULL << 13)
+/**< Fixed huffman encoding is supported */
+#define RTE_COMP_FF_HUFFMAN_DYNAMIC		(1ULL << 14)
+/**< Dynamic huffman encoding is supported */
 
 /** Status of comp operation */
 enum rte_comp_op_status {
diff --git a/lib/librte_compressdev/rte_compressdev.c b/lib/librte_compressdev/rte_compressdev.c
index 6a38917d..9091dd6e 100644
--- a/lib/librte_compressdev/rte_compressdev.c
+++ b/lib/librte_compressdev/rte_compressdev.c
@@ -764,10 +764,7 @@ rte_compressdev_name_get(uint8_t dev_id)
 	return dev->data->name;
 }
 
-RTE_INIT(rte_compressdev_log);
-
-static void
-rte_compressdev_log(void)
+RTE_INIT(rte_compressdev_log)
 {
 	compressdev_logtype = rte_log_register("lib.compressdev");
 	if (compressdev_logtype >= 0)
diff --git a/lib/librte_cryptodev/Makefile b/lib/librte_cryptodev/Makefile
index bba8dee9..c1148887 100644
--- a/lib/librte_cryptodev/Makefile
+++ b/lib/librte_cryptodev/Makefile
@@ -23,6 +23,7 @@ SYMLINK-y-include += rte_crypto.h
 SYMLINK-y-include += rte_crypto_sym.h
 SYMLINK-y-include += rte_cryptodev.h
 SYMLINK-y-include += rte_cryptodev_pmd.h
+SYMLINK-y-include += rte_crypto_asym.h
 
 # versioning export map
 EXPORT_MAP := rte_cryptodev_version.map
diff --git a/lib/librte_cryptodev/meson.build b/lib/librte_cryptodev/meson.build
index bd5fed89..295f509e 100644
--- a/lib/librte_cryptodev/meson.build
+++ b/lib/librte_cryptodev/meson.build
@@ -6,5 +6,6 @@ sources = files('rte_cryptodev.c', 'rte_cryptodev_pmd.c')
 headers = files('rte_cryptodev.h',
 	'rte_cryptodev_pmd.h',
 	'rte_crypto.h',
-	'rte_crypto_sym.h')
+	'rte_crypto_sym.h',
+	'rte_crypto_asym.h')
 deps += ['kvargs', 'mbuf']
diff --git a/lib/librte_cryptodev/rte_crypto.h b/lib/librte_cryptodev/rte_crypto.h
index 25404264..fd5ef3a8 100644
--- a/lib/librte_cryptodev/rte_crypto.h
+++ b/lib/librte_cryptodev/rte_crypto.h
@@ -23,6 +23,7 @@ extern "C" {
 #include <rte_common.h>
 
 #include "rte_crypto_sym.h"
+#include "rte_crypto_asym.h"
 
 /** Crypto operation types */
 enum rte_crypto_op_type {
@@ -30,6 +31,8 @@ enum rte_crypto_op_type {
 	/**< Undefined operation type */
 	RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 	/**< Symmetric operation */
+	RTE_CRYPTO_OP_TYPE_ASYMMETRIC
+	/**< Asymmetric operation */
 };
 
 /** Status of crypto operation */
@@ -73,26 +76,37 @@ enum rte_crypto_op_sess_type {
  * rte_cryptodev_enqueue_burst() / rte_cryptodev_dequeue_burst() .
  */
 struct rte_crypto_op {
-	uint8_t type;
-	/**< operation type */
-	uint8_t status;
-	/**<
-	 * operation status - this is reset to
-	 * RTE_CRYPTO_OP_STATUS_NOT_PROCESSED on allocation from mempool and
-	 * will be set to RTE_CRYPTO_OP_STATUS_SUCCESS after crypto operation
-	 * is successfully processed by a crypto PMD
-	 */
-	uint8_t sess_type;
-	/**< operation session type */
-	uint16_t private_data_offset;
-	/**< Offset to indicate start of private data (if any). The offset
-	 * is counted from the start of the rte_crypto_op including IV.
-	 * The private data may be used by the application to store
-	 * information which should remain untouched in the library/driver
-	 */
-
-	uint8_t reserved[3];
-	/**< Reserved bytes to fill 64 bits for future additions */
+	__extension__
+	union {
+		uint64_t raw;
+		__extension__
+		struct {
+			uint8_t type;
+			/**< operation type */
+			uint8_t status;
+			/**<
+			 * operation status - this is reset to
+			 * RTE_CRYPTO_OP_STATUS_NOT_PROCESSED on allocation
+			 * from mempool and will be set to
+			 * RTE_CRYPTO_OP_STATUS_SUCCESS after crypto operation
+			 * is successfully processed by a crypto PMD
+			 */
+			uint8_t sess_type;
+			/**< operation session type */
+			uint8_t reserved[3];
+			/**< Reserved bytes to fill 64 bits for
+			 * future additions
+			 */
+			uint16_t private_data_offset;
+			/**< Offset to indicate start of private data (if any).
+			 * The offset is counted from the start of the
+			 * rte_crypto_op including IV.
+			 * The private data may be used by the application
+			 * to store information which should remain untouched
+			 * in the library/driver
+			 */
+		};
+	};
 	struct rte_mempool *mempool;
 	/**< crypto operation mempool which operation is allocated from */
 
@@ -103,6 +117,10 @@ struct rte_crypto_op {
 	union {
 		struct rte_crypto_sym_op sym[0];
 		/**< Symmetric operation parameters */
+
+		struct rte_crypto_asym_op asym[0];
+		/**< Asymmetric operation parameters */
+
 	}; /**< operation specific parameters */
 };
 
@@ -123,6 +141,9 @@ __rte_crypto_op_reset(struct rte_crypto_op *op, enum rte_crypto_op_type type)
 	case RTE_CRYPTO_OP_TYPE_SYMMETRIC:
 		__rte_crypto_sym_op_reset(op->sym);
 		break;
+	case RTE_CRYPTO_OP_TYPE_ASYMMETRIC:
+		memset(op->asym, 0, sizeof(struct rte_crypto_asym_op));
+	break;
 	case RTE_CRYPTO_OP_TYPE_UNDEFINED:
 	default:
 		break;
@@ -289,9 +310,14 @@ __rte_crypto_op_get_priv_data(struct rte_crypto_op *op, uint32_t size)
 	if (likely(op->mempool != NULL)) {
 		priv_size = __rte_crypto_op_get_priv_data_size(op->mempool);
 
-		if (likely(priv_size >= size))
-			return (void *)((uint8_t *)(op + 1) +
+		if (likely(priv_size >= size)) {
+			if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
+				return (void *)((uint8_t *)(op + 1) +
 					sizeof(struct rte_crypto_sym_op));
+			if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
+				return (void *)((uint8_t *)(op + 1) +
+					sizeof(struct rte_crypto_asym_op));
+		}
 	}
 
 	return NULL;
@@ -394,6 +420,24 @@ rte_crypto_op_attach_sym_session(struct rte_crypto_op *op,
 	return __rte_crypto_sym_op_attach_sym_session(op->sym, sess);
 }
 
+/**
+ * Attach a asymmetric session to a crypto operation
+ *
+ * @param	op	crypto operation, must be of type asymmetric
+ * @param	sess	cryptodev session
+ */
+static inline int
+rte_crypto_op_attach_asym_session(struct rte_crypto_op *op,
+		struct rte_cryptodev_asym_session *sess)
+{
+	if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_ASYMMETRIC))
+		return -1;
+
+	op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
+	op->asym->session = sess;
+	return 0;
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_cryptodev/rte_crypto_asym.h b/lib/librte_cryptodev/rte_crypto_asym.h
new file mode 100644
index 00000000..5e185b2d
--- /dev/null
+++ b/lib/librte_cryptodev/rte_crypto_asym.h
@@ -0,0 +1,496 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium Networks
+ */
+
+#ifndef _RTE_CRYPTO_ASYM_H_
+#define _RTE_CRYPTO_ASYM_H_
+
+/**
+ * @file rte_crypto_asym.h
+ *
+ * RTE Definitions for Asymmetric Cryptography
+ *
+ * Defines asymmetric algorithms and modes, as well as supported
+ * asymmetric crypto operations.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <string.h>
+#include <stdint.h>
+
+#include <rte_memory.h>
+#include <rte_mempool.h>
+#include <rte_common.h>
+
+typedef struct rte_crypto_param_t {
+	uint8_t *data;
+	/**< pointer to buffer holding data */
+	rte_iova_t iova;
+	/**< IO address of data buffer */
+	size_t length;
+	/**< length of data in bytes */
+} rte_crypto_param;
+
+/** asym xform type name strings */
+extern const char *
+rte_crypto_asym_xform_strings[];
+
+/** asym operations type name strings */
+extern const char *
+rte_crypto_asym_op_strings[];
+
+/**
+ * Asymmetric crypto transformation types.
+ * Each xform type maps to one asymmetric algorithm
+ * performing specific operation
+ *
+ */
+enum rte_crypto_asym_xform_type {
+	RTE_CRYPTO_ASYM_XFORM_UNSPECIFIED = 0,
+	/**< Invalid xform. */
+	RTE_CRYPTO_ASYM_XFORM_NONE,
+	/**< Xform type None.
+	 * May be supported by PMD to support
+	 * passthrough op for debugging purpose.
+	 * if xform_type none , op_type is disregarded.
+	 */
+	RTE_CRYPTO_ASYM_XFORM_RSA,
+	/**< RSA. Performs Encrypt, Decrypt, Sign and Verify.
+	 * Refer to rte_crypto_asym_op_type
+	 */
+	RTE_CRYPTO_ASYM_XFORM_DH,
+	/**< Diffie-Hellman.
+	 * Performs Key Generate and Shared Secret Compute.
+	 * Refer to rte_crypto_asym_op_type
+	 */
+	RTE_CRYPTO_ASYM_XFORM_DSA,
+	/**< Digital Signature Algorithm
+	 * Performs Signature Generation and Verification.
+	 * Refer to rte_crypto_asym_op_type
+	 */
+	RTE_CRYPTO_ASYM_XFORM_MODINV,
+	/**< Modular Inverse
+	 * Perform Modulus inverse b^(-1) mod n
+	 */
+	RTE_CRYPTO_ASYM_XFORM_MODEX,
+	/**< Modular Exponentiation
+	 * Perform Modular Exponentiation b^e mod n
+	 */
+	RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
+	/**< End of list */
+};
+
+/**
+ * Asymmetric crypto operation type variants
+ */
+enum rte_crypto_asym_op_type {
+	RTE_CRYPTO_ASYM_OP_ENCRYPT,
+	/**< Asymmetric Encrypt operation */
+	RTE_CRYPTO_ASYM_OP_DECRYPT,
+	/**< Asymmetric Decrypt operation */
+	RTE_CRYPTO_ASYM_OP_SIGN,
+	/**< Signature Generation operation */
+	RTE_CRYPTO_ASYM_OP_VERIFY,
+	/**< Signature Verification operation */
+	RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE,
+	/**< DH Private Key generation operation */
+	RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE,
+	/**< DH Public Key generation operation */
+	RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE,
+	/**< DH Shared Secret compute operation */
+	RTE_CRYPTO_ASYM_OP_LIST_END
+};
+
+/**
+ * Padding types for RSA signature.
+ */
+enum rte_crypto_rsa_padding_type {
+	RTE_CRYPTO_RSA_PADDING_NONE = 0,
+	/**< RSA no padding scheme */
+	RTE_CRYPTO_RSA_PKCS1_V1_5_BT0,
+	/**< RSA PKCS#1 V1.5 Block Type 0 padding scheme
+	 * as descibed in rfc2313
+	 */
+	RTE_CRYPTO_RSA_PKCS1_V1_5_BT1,
+	/**< RSA PKCS#1 V1.5 Block Type 01 padding scheme
+	 * as descibed in rfc2313
+	 */
+	RTE_CRYPTO_RSA_PKCS1_V1_5_BT2,
+	/**< RSA PKCS#1 V1.5 Block Type 02 padding scheme
+	 * as descibed in rfc2313
+	 */
+	RTE_CRYPTO_RSA_PADDING_OAEP,
+	/**< RSA PKCS#1 OAEP padding scheme */
+	RTE_CRYPTO_RSA_PADDING_PSS,
+	/**< RSA PKCS#1 PSS padding scheme */
+	RTE_CRYPTO_RSA_PADDING_TYPE_LIST_END
+};
+
+/**
+ * RSA private key type enumeration
+ *
+ * enumerates private key format required to perform RSA crypto
+ * transform.
+ *
+ */
+enum rte_crypto_rsa_priv_key_type {
+	RTE_RSA_KEY_TYPE_EXP,
+	/**< RSA private key is an exponent */
+	RTE_RSA_KET_TYPE_QT,
+	/**< RSA private key is in quintuple format
+	 * See rte_crypto_rsa_priv_key_qt
+	 */
+};
+
+/**
+ * Structure describing RSA private key in quintuple format.
+ * See PKCS V1.5 RSA Cryptography Standard.
+ */
+struct rte_crypto_rsa_priv_key_qt {
+	rte_crypto_param p;
+	/**< p - Private key component P
+	 * Private key component of RSA parameter  required for CRT method
+	 * of private key operations in Octet-string network byte order
+	 * format.
+	 */
+
+	rte_crypto_param q;
+	/**< q - Private key component Q
+	 * Private key component of RSA parameter  required for CRT method
+	 * of private key operations in Octet-string network byte order
+	 * format.
+	 */
+
+	rte_crypto_param dP;
+	/**< dP - Private CRT component
+	 * Private CRT component of RSA parameter  required for CRT method
+	 * RSA private key operations in Octet-string network byte order
+	 * format.
+	 * dP = d mod ( p - 1 )
+	 */
+
+	rte_crypto_param dQ;
+	/**< dQ - Private CRT component
+	 * Private CRT component of RSA parameter  required for CRT method
+	 * RSA private key operations in Octet-string network byte order
+	 * format.
+	 * dQ = d mod ( q - 1 )
+	 */
+
+	rte_crypto_param qInv;
+	/**< qInv - Private CRT component
+	 * Private CRT component of RSA parameter  required for CRT method
+	 * RSA private key operations in Octet-string network byte order
+	 * format.
+	 * qInv = inv q mod p
+	 */
+};
+
+/**
+ * Asymmetric RSA transform data
+ *
+ * Structure describing RSA xform params
+ *
+ */
+struct rte_crypto_rsa_xform {
+	rte_crypto_param n;
+	/**< n - Prime modulus
+	 * Prime modulus data of RSA operation in Octet-string network
+	 * byte order format.
+	 */
+
+	rte_crypto_param e;
+	/**< e - Public key exponent
+	 * Public key exponent used for RSA public key operations in Octet-
+	 * string network byte order format.
+	 */
+
+	enum rte_crypto_rsa_priv_key_type key_type;
+
+	__extension__
+	union {
+		rte_crypto_param d;
+		/**< d - Private key exponent
+		 * Private key exponent used for RSA
+		 * private key operations in
+		 * Octet-string  network byte order format.
+		 */
+
+		struct rte_crypto_rsa_priv_key_qt qt;
+		/**< qt - Private key in quintuple format */
+	};
+};
+
+/**
+ * Asymmetric Modular exponentiation transform data
+ *
+ * Structure describing modular exponentation xform param
+ *
+ */
+struct rte_crypto_modex_xform {
+	rte_crypto_param modulus;
+	/**< modulus
+	 * Prime modulus of the modexp transform operation in octet-string
+	 * network byte order format.
+	 */
+
+	rte_crypto_param exponent;
+	/**< exponent
+	 * Private exponent of the modexp transform operation in
+	 * octet-string network byte order format.
+	 */
+};
+
+/**
+ * Asymmetric modular inverse transform operation
+ *
+ * Structure describing modulus inverse xform params
+ *
+ */
+struct rte_crypto_modinv_xform {
+	rte_crypto_param modulus;
+	/**<
+	 * Pointer to the prime modulus data for modular
+	 * inverse operation in octet-string network byte
+	 * order format.
+	 */
+};
+
+/**
+ * Asymmetric DH transform data
+ *
+ * Structure describing deffie-hellman xform params
+ *
+ */
+struct rte_crypto_dh_xform {
+	enum rte_crypto_asym_op_type type;
+	/**< Setup xform for key generate or shared secret compute */
+
+	rte_crypto_param p;
+	/**< p : Prime modulus data
+	 * DH prime modulous data in octet-string network byte order format.
+	 *
+	 */
+
+	rte_crypto_param g;
+	/**< g : Generator
+	 * DH group generator data in octet-string network byte order
+	 * format.
+	 *
+	 */
+};
+
+/**
+ * Asymmetric Digital Signature transform operation
+ *
+ * Structure describing DSA xform params
+ *
+ */
+struct rte_crypto_dsa_xform {
+	rte_crypto_param p;
+	/**< p - Prime modulus
+	 * Prime modulus data for DSA operation in Octet-string network byte
+	 * order format.
+	 */
+	rte_crypto_param q;
+	/**< q : Order of the subgroup.
+	 * Order of the subgroup data in Octet-string network byte order
+	 * format.
+	 * (p-1) % q = 0
+	 */
+	rte_crypto_param g;
+	/**< g: Generator of the subgroup
+	 * Generator  data in Octet-string network byte order format.
+	 */
+	rte_crypto_param x;
+	/**< x: Private key of the signer in octet-string network
+	 * byte order format.
+	 * Used when app has pre-defined private key.
+	 * Valid only when xform chain is DSA ONLY.
+	 * if xform chain is DH private key generate + DSA, then DSA sign
+	 * compute will use internally generated key.
+	 */
+};
+
+/**
+ * Operations params for modular operations:
+ * exponentiation and invert
+ *
+ */
+struct rte_crypto_mod_op_param {
+	rte_crypto_param base;
+	/**<
+	 * Pointer to base of modular exponentiation/inversion data in
+	 * Octet-string network byte order format.
+	 */
+};
+
+/**
+ * Asymmetric crypto transform data
+ *
+ * Structure describing asym xforms.
+ */
+struct rte_crypto_asym_xform {
+	struct rte_crypto_asym_xform *next;
+	/**< Pointer to next xform to set up xform chain.*/
+	enum rte_crypto_asym_xform_type xform_type;
+	/**< Asymmetric crypto transform */
+
+	__extension__
+	union {
+		struct rte_crypto_rsa_xform rsa;
+		/**< RSA xform parameters */
+
+		struct rte_crypto_modex_xform modex;
+		/**< Modular Exponentiation xform parameters */
+
+		struct rte_crypto_modinv_xform modinv;
+		/**< Modulus Inverse xform parameters */
+
+		struct rte_crypto_dh_xform dh;
+		/**< DH xform parameters */
+
+		struct rte_crypto_dsa_xform dsa;
+		/**< DSA xform parameters */
+	};
+};
+
+struct rte_cryptodev_asym_session;
+
+/**
+ * RSA operation params
+ *
+ */
+struct rte_crypto_rsa_op_param {
+	enum rte_crypto_asym_op_type op_type;
+	/**< Type of RSA operation for transform */;
+
+	rte_crypto_param message;
+	/**<
+	 * Pointer to data
+	 * - to be encrypted for RSA public encrypt.
+	 * - to be decrypted for RSA private decrypt.
+	 * - to be signed for RSA sign generation.
+	 * - to be authenticated for RSA sign verification.
+	 */
+
+	rte_crypto_param sign;
+	/**<
+	 * Pointer to RSA signature data. If operation is RSA
+	 * sign @ref RTE_CRYPTO_ASYM_OP_SIGN, buffer will be
+	 * over-written with generated signature.
+	 *
+	 * Length of the signature data will be equal to the
+	 * RSA prime modulus length.
+	 */
+
+	enum rte_crypto_rsa_padding_type pad;
+	/**< RSA padding scheme to be used for transform */
+
+	enum rte_crypto_auth_algorithm md;
+	/**< Hash algorithm to be used for data hash if padding
+	 * scheme is either OAEP or PSS. Valid hash algorithms
+	 * are:
+	 * MD5, SHA1, SHA224, SHA256, SHA384, SHA512
+	 */
+
+	enum rte_crypto_auth_algorithm mgf1md;
+	/**<
+	 * Hash algorithm to be used for mask generation if
+	 * padding scheme is either OAEP or PSS. If padding
+	 * scheme is unspecified data hash algorithm is used
+	 * for mask generation. Valid hash algorithms are:
+	 * MD5, SHA1, SHA224, SHA256, SHA384, SHA512
+	 */
+};
+
+/**
+ * Diffie-Hellman Operations params.
+ * @note:
+ */
+struct rte_crypto_dh_op_param {
+	rte_crypto_param pub_key;
+	/**<
+	 * Output generated public key when xform type is
+	 * DH PUB_KEY_GENERATION.
+	 * Input peer public key when xform type is DH
+	 * SHARED_SECRET_COMPUTATION
+	 * pub_key is in octet-string network byte order format.
+	 *
+	 */
+
+	rte_crypto_param priv_key;
+	/**<
+	 * Output generated private key if xform type is
+	 * DH PRIVATE_KEY_GENERATION
+	 * Input when xform type is DH SHARED_SECRET_COMPUTATION.
+	 * priv_key is in octet-string network byte order format.
+	 *
+	 */
+
+	rte_crypto_param shared_secret;
+	/**<
+	 * Output with calculated shared secret
+	 * when dh xform set up with op type = SHARED_SECRET_COMPUTATION.
+	 * shared_secret is an octet-string network byte order format.
+	 *
+	 */
+};
+
+/**
+ * DSA Operations params
+ *
+ */
+struct rte_crypto_dsa_op_param {
+	enum rte_crypto_asym_op_type op_type;
+	/**< Signature Generation or Verification */
+	rte_crypto_param message;
+	/**< input message to be signed or verified */
+	rte_crypto_param r;
+	/**< dsa sign component 'r' value
+	 *
+	 * output if op_type = sign generate,
+	 * input if op_type = sign verify
+	 */
+	rte_crypto_param s;
+	/**< dsa sign component 's' value
+	 *
+	 * output if op_type = sign generate,
+	 * input if op_type = sign verify
+	 */
+	rte_crypto_param y;
+	/**< y : Public key of the signer.
+	 * Public key data of the signer in Octet-string network byte order
+	 * format.
+	 * y = g^x mod p
+	 */
+};
+
+/**
+ * Asymmetric Cryptographic Operation.
+ *
+ * Structure describing asymmetric crypto operation params.
+ *
+ */
+struct rte_crypto_asym_op {
+	struct rte_cryptodev_asym_session *session;
+	/**< Handle for the initialised session context */
+
+	__extension__
+	union {
+		struct rte_crypto_rsa_op_param rsa;
+		struct rte_crypto_mod_op_param modex;
+		struct rte_crypto_mod_op_param modinv;
+		struct rte_crypto_dh_op_param dh;
+		struct rte_crypto_dsa_op_param dsa;
+	};
+} __rte_cache_aligned;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_CRYPTO_ASYM_H_ */
diff --git a/lib/librte_cryptodev/rte_cryptodev.c b/lib/librte_cryptodev/rte_cryptodev.c
index 7e582124..63ae23f0 100644
--- a/lib/librte_cryptodev/rte_cryptodev.c
+++ b/lib/librte_cryptodev/rte_cryptodev.c
@@ -166,6 +166,31 @@ rte_crypto_aead_operation_strings[] = {
 	[RTE_CRYPTO_AEAD_OP_DECRYPT]	= "decrypt"
 };
 
+/**
+ * Asymmetric crypto transform operation strings identifiers.
+ */
+const char *rte_crypto_asym_xform_strings[] = {
+	[RTE_CRYPTO_ASYM_XFORM_NONE]	= "none",
+	[RTE_CRYPTO_ASYM_XFORM_RSA]	= "rsa",
+	[RTE_CRYPTO_ASYM_XFORM_MODEX]	= "modexp",
+	[RTE_CRYPTO_ASYM_XFORM_MODINV]	= "modinv",
+	[RTE_CRYPTO_ASYM_XFORM_DH]	= "dh",
+	[RTE_CRYPTO_ASYM_XFORM_DSA]	= "dsa",
+};
+
+/**
+ * Asymmetric crypto operation strings identifiers.
+ */
+const char *rte_crypto_asym_op_strings[] = {
+	[RTE_CRYPTO_ASYM_OP_ENCRYPT]	= "encrypt",
+	[RTE_CRYPTO_ASYM_OP_DECRYPT]	= "decrypt",
+	[RTE_CRYPTO_ASYM_OP_SIGN]	= "sign",
+	[RTE_CRYPTO_ASYM_OP_VERIFY]	= "verify",
+	[RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE]	= "priv_key_generate",
+	[RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE] = "pub_key_generate",
+	[RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE] = "sharedsecret_compute",
+};
+
 int
 rte_cryptodev_get_cipher_algo_enum(enum rte_crypto_cipher_algorithm *algo_enum,
 		const char *algo_string)
@@ -217,6 +242,24 @@ rte_cryptodev_get_aead_algo_enum(enum rte_crypto_aead_algorithm *algo_enum,
 	return -1;
 }
 
+int __rte_experimental
+rte_cryptodev_asym_get_xform_enum(enum rte_crypto_asym_xform_type *xform_enum,
+		const char *xform_string)
+{
+	unsigned int i;
+
+	for (i = 1; i < RTE_DIM(rte_crypto_asym_xform_strings); i++) {
+		if (strcmp(xform_string,
+			rte_crypto_asym_xform_strings[i]) == 0) {
+			*xform_enum = (enum rte_crypto_asym_xform_type) i;
+			return 0;
+		}
+	}
+
+	/* Invalid string */
+	return -1;
+}
+
 /**
  * The crypto auth operation strings identifiers.
  * It could be used in application command line.
@@ -287,6 +330,28 @@ param_range_check(uint16_t size, const struct rte_crypto_param_range *range)
 	return -1;
 }
 
+const struct rte_cryptodev_asymmetric_xform_capability * __rte_experimental
+rte_cryptodev_asym_capability_get(uint8_t dev_id,
+		const struct rte_cryptodev_asym_capability_idx *idx)
+{
+	const struct rte_cryptodev_capabilities *capability;
+	struct rte_cryptodev_info dev_info;
+	unsigned int i = 0;
+
+	memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
+	rte_cryptodev_info_get(dev_id, &dev_info);
+
+	while ((capability = &dev_info.capabilities[i++])->op !=
+			RTE_CRYPTO_OP_TYPE_UNDEFINED) {
+		if (capability->op != RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
+			continue;
+
+		if (capability->asym.xform_capa.xform_type == idx->type)
+			return &capability->asym.xform_capa;
+	}
+	return NULL;
+};
+
 int
 rte_cryptodev_sym_capability_check_cipher(
 		const struct rte_cryptodev_symmetric_capability *capability,
@@ -338,6 +403,42 @@ rte_cryptodev_sym_capability_check_aead(
 
 	return 0;
 }
+int __rte_experimental
+rte_cryptodev_asym_xform_capability_check_optype(
+	const struct rte_cryptodev_asymmetric_xform_capability *capability,
+	enum rte_crypto_asym_op_type op_type)
+{
+	if (capability->op_types & (1 << op_type))
+		return 1;
+
+	return 0;
+}
+
+int __rte_experimental
+rte_cryptodev_asym_xform_capability_check_modlen(
+	const struct rte_cryptodev_asymmetric_xform_capability *capability,
+	uint16_t modlen)
+{
+	/* no need to check for limits, if min or max = 0 */
+	if (capability->modlen.min != 0) {
+		if (modlen < capability->modlen.min)
+			return -1;
+	}
+
+	if (capability->modlen.max != 0) {
+		if (modlen > capability->modlen.max)
+			return -1;
+	}
+
+	/* in any case, check if given modlen is module increment */
+	if (capability->modlen.increment != 0) {
+		if (modlen % (capability->modlen.increment))
+			return -1;
+	}
+
+	return 0;
+}
+
 
 const char *
 rte_cryptodev_get_feature_name(uint64_t flag)
@@ -361,8 +462,16 @@ rte_cryptodev_get_feature_name(uint64_t flag)
 		return "CPU_AESNI";
 	case RTE_CRYPTODEV_FF_HW_ACCELERATED:
 		return "HW_ACCELERATED";
-	case RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER:
-		return "MBUF_SCATTER_GATHER";
+	case RTE_CRYPTODEV_FF_IN_PLACE_SGL:
+		return "IN_PLACE_SGL";
+	case RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT:
+		return "OOP_SGL_IN_SGL_OUT";
+	case RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT:
+		return "OOP_SGL_IN_LB_OUT";
+	case RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT:
+		return "OOP_LB_IN_SGL_OUT";
+	case RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT:
+		return "OOP_LB_IN_LB_OUT";
 	case RTE_CRYPTODEV_FF_CPU_NEON:
 		return "CPU_NEON";
 	case RTE_CRYPTODEV_FF_CPU_ARM_CE:
@@ -703,50 +812,6 @@ rte_cryptodev_queue_pairs_config(struct rte_cryptodev *dev, uint16_t nb_qpairs,
 }
 
 int
-rte_cryptodev_queue_pair_start(uint8_t dev_id, uint16_t queue_pair_id)
-{
-	struct rte_cryptodev *dev;
-
-	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
-		CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
-		return -EINVAL;
-	}
-
-	dev = &rte_crypto_devices[dev_id];
-	if (queue_pair_id >= dev->data->nb_queue_pairs) {
-		CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
-		return -EINVAL;
-	}
-
-	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_start, -ENOTSUP);
-
-	return dev->dev_ops->queue_pair_start(dev, queue_pair_id);
-
-}
-
-int
-rte_cryptodev_queue_pair_stop(uint8_t dev_id, uint16_t queue_pair_id)
-{
-	struct rte_cryptodev *dev;
-
-	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
-		CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
-		return -EINVAL;
-	}
-
-	dev = &rte_crypto_devices[dev_id];
-	if (queue_pair_id >= dev->data->nb_queue_pairs) {
-		CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
-		return -EINVAL;
-	}
-
-	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_stop, -ENOTSUP);
-
-	return dev->dev_ops->queue_pair_stop(dev, queue_pair_id);
-
-}
-
-int
 rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config)
 {
 	struct rte_cryptodev *dev;
@@ -966,6 +1031,7 @@ rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info)
 	(*dev->dev_ops->dev_infos_get)(dev, dev_info);
 
 	dev_info->driver_name = dev->device->driver->name;
+	dev_info->device = dev->device;
 }
 
 
@@ -1098,8 +1164,46 @@ rte_cryptodev_sym_session_init(uint8_t dev_id,
 
 	index = dev->driver_id;
 
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->sym_session_configure, -ENOTSUP);
+
 	if (sess->sess_private_data[index] == NULL) {
-		ret = dev->dev_ops->session_configure(dev, xforms, sess, mp);
+		ret = dev->dev_ops->sym_session_configure(dev, xforms,
+							sess, mp);
+		if (ret < 0) {
+			CDEV_LOG_ERR(
+				"dev_id %d failed to configure session details",
+				dev_id);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+int __rte_experimental
+rte_cryptodev_asym_session_init(uint8_t dev_id,
+		struct rte_cryptodev_asym_session *sess,
+		struct rte_crypto_asym_xform *xforms,
+		struct rte_mempool *mp)
+{
+	struct rte_cryptodev *dev;
+	uint8_t index;
+	int ret;
+
+	dev = rte_cryptodev_pmd_get_dev(dev_id);
+
+	if (sess == NULL || xforms == NULL || dev == NULL)
+		return -EINVAL;
+
+	index = dev->driver_id;
+
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_configure,
+				-ENOTSUP);
+
+	if (sess->sess_private_data[index] == NULL) {
+		ret = dev->dev_ops->asym_session_configure(dev,
+							xforms,
+							sess, mp);
 		if (ret < 0) {
 			CDEV_LOG_ERR(
 				"dev_id %d failed to configure session details",
@@ -1123,70 +1227,53 @@ rte_cryptodev_sym_session_create(struct rte_mempool *mp)
 	}
 
 	/* Clear device session pointer.
-	 * Include the flag indicating presence of private data
+	 * Include the flag indicating presence of user data
 	 */
 	memset(sess, 0, (sizeof(void *) * nb_drivers) + sizeof(uint8_t));
 
 	return sess;
 }
 
-int
-rte_cryptodev_queue_pair_attach_sym_session(uint8_t dev_id, uint16_t qp_id,
-		struct rte_cryptodev_sym_session *sess)
+struct rte_cryptodev_asym_session * __rte_experimental
+rte_cryptodev_asym_session_create(struct rte_mempool *mp)
 {
-	struct rte_cryptodev *dev;
+	struct rte_cryptodev_asym_session *sess;
 
-	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
-		CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
-		return -EINVAL;
+	/* Allocate a session structure from the session pool */
+	if (rte_mempool_get(mp, (void **)&sess)) {
+		CDEV_LOG_ERR("couldn't get object from session mempool");
+		return NULL;
 	}
 
-	dev = &rte_crypto_devices[dev_id];
-
-	/* The API is optional, not returning error if driver do not suuport */
-	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->qp_attach_session, 0);
-
-	void *sess_priv = get_session_private_data(sess, dev->driver_id);
-
-	if (dev->dev_ops->qp_attach_session(dev, qp_id, sess_priv)) {
-		CDEV_LOG_ERR("dev_id %d failed to attach qp: %d with session",
-				dev_id, qp_id);
-		return -EPERM;
-	}
+	/* Clear device session pointer.
+	 * Include the flag indicating presence of private data
+	 */
+	memset(sess, 0, (sizeof(void *) * nb_drivers) + sizeof(uint8_t));
 
-	return 0;
+	return sess;
 }
 
 int
-rte_cryptodev_queue_pair_detach_sym_session(uint8_t dev_id, uint16_t qp_id,
+rte_cryptodev_sym_session_clear(uint8_t dev_id,
 		struct rte_cryptodev_sym_session *sess)
 {
 	struct rte_cryptodev *dev;
 
-	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
-		CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
-		return -EINVAL;
-	}
-
-	dev = &rte_crypto_devices[dev_id];
+	dev = rte_cryptodev_pmd_get_dev(dev_id);
 
-	/* The API is optional, not returning error if driver do not suuport */
-	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->qp_detach_session, 0);
+	if (dev == NULL || sess == NULL)
+		return -EINVAL;
 
-	void *sess_priv = get_session_private_data(sess, dev->driver_id);
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->sym_session_clear, -ENOTSUP);
 
-	if (dev->dev_ops->qp_detach_session(dev, qp_id, sess_priv)) {
-		CDEV_LOG_ERR("dev_id %d failed to detach qp: %d from session",
-				dev_id, qp_id);
-		return -EPERM;
-	}
+	dev->dev_ops->sym_session_clear(dev, sess);
 
 	return 0;
 }
 
-int
-rte_cryptodev_sym_session_clear(uint8_t dev_id,
-		struct rte_cryptodev_sym_session *sess)
+int __rte_experimental
+rte_cryptodev_asym_session_clear(uint8_t dev_id,
+		struct rte_cryptodev_asym_session *sess)
 {
 	struct rte_cryptodev *dev;
 
@@ -1195,7 +1282,9 @@ rte_cryptodev_sym_session_clear(uint8_t dev_id,
 	if (dev == NULL || sess == NULL)
 		return -EINVAL;
 
-	dev->dev_ops->session_clear(dev, sess);
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_clear, -ENOTSUP);
+
+	dev->dev_ops->asym_session_clear(dev, sess);
 
 	return 0;
 }
@@ -1212,7 +1301,7 @@ rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess)
 
 	/* Check that all device private data has been freed */
 	for (i = 0; i < nb_drivers; i++) {
-		sess_priv = get_session_private_data(sess, i);
+		sess_priv = get_sym_session_private_data(sess, i);
 		if (sess_priv != NULL)
 			return -EBUSY;
 	}
@@ -1224,27 +1313,51 @@ rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess)
 	return 0;
 }
 
-unsigned int
-rte_cryptodev_get_header_session_size(void)
+int __rte_experimental
+rte_cryptodev_asym_session_free(struct rte_cryptodev_asym_session *sess)
 {
-	return rte_cryptodev_sym_get_header_session_size();
+	uint8_t i;
+	void *sess_priv;
+	struct rte_mempool *sess_mp;
+
+	if (sess == NULL)
+		return -EINVAL;
+
+	/* Check that all device private data has been freed */
+	for (i = 0; i < nb_drivers; i++) {
+		sess_priv = get_asym_session_private_data(sess, i);
+		if (sess_priv != NULL)
+			return -EBUSY;
+	}
+
+	/* Return session to mempool */
+	sess_mp = rte_mempool_from_obj(sess);
+	rte_mempool_put(sess_mp, sess);
+
+	return 0;
 }
 
+
 unsigned int
 rte_cryptodev_sym_get_header_session_size(void)
 {
 	/*
 	 * Header contains pointers to the private data
 	 * of all registered drivers, and a flag which
-	 * indicates presence of private data
+	 * indicates presence of user data
 	 */
 	return ((sizeof(void *) * nb_drivers) + sizeof(uint8_t));
 }
 
-unsigned int
-rte_cryptodev_get_private_session_size(uint8_t dev_id)
+unsigned int __rte_experimental
+rte_cryptodev_asym_get_header_session_size(void)
 {
-	return rte_cryptodev_sym_get_private_session_size(dev_id);
+	/*
+	 * Header contains pointers to the private data
+	 * of all registered drivers, and a flag which
+	 * indicates presence of private data
+	 */
+	return ((sizeof(void *) * nb_drivers) + sizeof(uint8_t));
 }
 
 unsigned int
@@ -1259,10 +1372,10 @@ rte_cryptodev_sym_get_private_session_size(uint8_t dev_id)
 
 	dev = rte_cryptodev_pmd_get_dev(dev_id);
 
-	if (*dev->dev_ops->session_get_size == NULL)
+	if (*dev->dev_ops->sym_session_get_size == NULL)
 		return 0;
 
-	priv_sess_size = (*dev->dev_ops->session_get_size)(dev);
+	priv_sess_size = (*dev->dev_ops->sym_session_get_size)(dev);
 
 	/*
 	 * If size is less than session header size,
@@ -1276,32 +1389,55 @@ rte_cryptodev_sym_get_private_session_size(uint8_t dev_id)
 
 }
 
+unsigned int __rte_experimental
+rte_cryptodev_asym_get_private_session_size(uint8_t dev_id)
+{
+	struct rte_cryptodev *dev;
+	unsigned int header_size = sizeof(void *) * nb_drivers;
+	unsigned int priv_sess_size;
+
+	if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
+		return 0;
+
+	dev = rte_cryptodev_pmd_get_dev(dev_id);
+
+	if (*dev->dev_ops->asym_session_get_size == NULL)
+		return 0;
+
+	priv_sess_size = (*dev->dev_ops->asym_session_get_size)(dev);
+	if (priv_sess_size < header_size)
+		return header_size;
+
+	return priv_sess_size;
+
+}
+
 int __rte_experimental
-rte_cryptodev_sym_session_set_private_data(
+rte_cryptodev_sym_session_set_user_data(
 					struct rte_cryptodev_sym_session *sess,
 					void *data,
 					uint16_t size)
 {
 	uint16_t off_set = sizeof(void *) * nb_drivers;
-	uint8_t *private_data_present = (uint8_t *)sess + off_set;
+	uint8_t *user_data_present = (uint8_t *)sess + off_set;
 
 	if (sess == NULL)
 		return -EINVAL;
 
-	*private_data_present = 1;
+	*user_data_present = 1;
 	off_set += sizeof(uint8_t);
 	rte_memcpy((uint8_t *)sess + off_set, data, size);
 	return 0;
 }
 
 void * __rte_experimental
-rte_cryptodev_sym_session_get_private_data(
+rte_cryptodev_sym_session_get_user_data(
 					struct rte_cryptodev_sym_session *sess)
 {
 	uint16_t off_set = sizeof(void *) * nb_drivers;
-	uint8_t *private_data_present = (uint8_t *)sess + off_set;
+	uint8_t *user_data_present = (uint8_t *)sess + off_set;
 
-	if (sess == NULL || !*private_data_present)
+	if (sess == NULL || !*user_data_present)
 		return NULL;
 
 	off_set += sizeof(uint8_t);
@@ -1335,9 +1471,17 @@ rte_crypto_op_pool_create(const char *name, enum rte_crypto_op_type type,
 	struct rte_crypto_op_pool_private *priv;
 
 	unsigned elt_size = sizeof(struct rte_crypto_op) +
-			sizeof(struct rte_crypto_sym_op) +
 			priv_size;
 
+	if (type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
+		elt_size += sizeof(struct rte_crypto_sym_op);
+	} else if (type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
+		elt_size += sizeof(struct rte_crypto_asym_op);
+	} else {
+		CDEV_LOG_ERR("Invalid op_type\n");
+		return NULL;
+	}
+
 	/* lookup mempool in case already allocated */
 	struct rte_mempool *mp = rte_mempool_lookup(name);
 
diff --git a/lib/librte_cryptodev/rte_cryptodev.h b/lib/librte_cryptodev/rte_cryptodev.h
index 92ce6d49..4099823f 100644
--- a/lib/librte_cryptodev/rte_cryptodev.h
+++ b/lib/librte_cryptodev/rte_cryptodev.h
@@ -1,32 +1,5 @@
-/*-
- *
- *   Copyright(c) 2015-2017 Intel Corporation. All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2017 Intel Corporation.
  */
 
 #ifndef _RTE_CRYPTODEV_H_
@@ -65,7 +38,6 @@ extern const char **rte_cyptodev_names;
 		RTE_FMT(RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
 			RTE_FMT_TAIL(__VA_ARGS__,)))
 
-#ifdef RTE_LIBRTE_CRYPTODEV_DEBUG
 #define CDEV_LOG_DEBUG(...) \
 	RTE_LOG(DEBUG, CRYPTODEV, \
 		RTE_FMT("%s() line %u: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
@@ -76,13 +48,6 @@ extern const char **rte_cyptodev_names;
 		RTE_FMT("[%s] %s: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
 			dev, __func__, RTE_FMT_TAIL(__VA_ARGS__,)))
 
-#else
-#define CDEV_LOG_DEBUG(...) (void)0
-#define CDEV_PMD_TRACE(...) (void)0
-#endif
-
-
-
 /**
  * A macro that points to an offset from the start
  * of the crypto operation structure (rte_crypto_op)
@@ -178,6 +143,35 @@ struct rte_cryptodev_symmetric_capability {
 	};
 };
 
+/**
+ * Asymmetric Xform Crypto Capability
+ *
+ */
+struct rte_cryptodev_asymmetric_xform_capability {
+	enum rte_crypto_asym_xform_type xform_type;
+	/**< Transform type: RSA/MODEXP/DH/DSA/MODINV */
+
+	uint32_t op_types;
+	/**< bitmask for supported rte_crypto_asym_op_type */
+
+	__extension__
+	union {
+		struct rte_crypto_param_range modlen;
+		/**< Range of modulus length supported by modulus based xform.
+		 * Value 0 mean implementation default
+		 */
+	};
+};
+
+/**
+ * Asymmetric Crypto Capability
+ *
+ */
+struct rte_cryptodev_asymmetric_capability {
+	struct rte_cryptodev_asymmetric_xform_capability xform_capa;
+};
+
+
 /** Structure used to capture a capability of a crypto device */
 struct rte_cryptodev_capabilities {
 	enum rte_crypto_op_type op;
@@ -187,6 +181,8 @@ struct rte_cryptodev_capabilities {
 	union {
 		struct rte_cryptodev_symmetric_capability sym;
 		/**< Symmetric operation capability parameters */
+		struct rte_cryptodev_asymmetric_capability asym;
+		/**< Asymmetric operation capability parameters */
 	};
 };
 
@@ -201,7 +197,17 @@ struct rte_cryptodev_sym_capability_idx {
 };
 
 /**
- *  Provide capabilities available for defined device and algorithm
+ * Structure used to describe asymmetric crypto xforms
+ * Each xform maps to one asym algorithm.
+ *
+ */
+struct rte_cryptodev_asym_capability_idx {
+	enum rte_crypto_asym_xform_type type;
+	/**< Asymmetric xform (algo) type */
+};
+
+/**
+ * Provide capabilities available for defined device and algorithm
  *
  * @param	dev_id		The identifier of the device.
  * @param	idx		Description of crypto algorithms.
@@ -215,6 +221,20 @@ rte_cryptodev_sym_capability_get(uint8_t dev_id,
 		const struct rte_cryptodev_sym_capability_idx *idx);
 
 /**
+ *  Provide capabilities available for defined device and xform
+ *
+ * @param	dev_id		The identifier of the device.
+ * @param	idx		Description of asym crypto xform.
+ *
+ * @return
+ *   - Return description of the asymmetric crypto capability if exist.
+ *   - Return NULL if the capability not exist.
+ */
+const struct rte_cryptodev_asymmetric_xform_capability * __rte_experimental
+rte_cryptodev_asym_capability_get(uint8_t dev_id,
+		const struct rte_cryptodev_asym_capability_idx *idx);
+
+/**
  * Check if key size and initial vector are supported
  * in crypto cipher capability
  *
@@ -270,6 +290,36 @@ rte_cryptodev_sym_capability_check_aead(
 		uint16_t iv_size);
 
 /**
+ * Check if op type is supported
+ *
+ * @param	capability	Description of the asymmetric crypto capability.
+ * @param	op_type		op type
+ *
+ * @return
+ *   - Return 1 if the op type is supported
+ *   - Return 0 if unsupported
+ */
+int __rte_experimental
+rte_cryptodev_asym_xform_capability_check_optype(
+	const struct rte_cryptodev_asymmetric_xform_capability *capability,
+		enum rte_crypto_asym_op_type op_type);
+
+/**
+ * Check if modulus length is in supported range
+ *
+ * @param	capability	Description of the asymmetric crypto capability.
+ * @param	modlen		modulus length.
+ *
+ * @return
+ *   - Return 0 if the parameters are in range of the capability.
+ *   - Return -1 if the parameters are out of range of the capability.
+ */
+int __rte_experimental
+rte_cryptodev_asym_xform_capability_check_modlen(
+	const struct rte_cryptodev_asymmetric_xform_capability *capability,
+		uint16_t modlen);
+
+/**
  * Provide the cipher algorithm enum, given an algorithm string
  *
  * @param	algo_enum	A pointer to the cipher algorithm
@@ -314,6 +364,22 @@ int
 rte_cryptodev_get_aead_algo_enum(enum rte_crypto_aead_algorithm *algo_enum,
 		const char *algo_string);
 
+/**
+ * Provide the Asymmetric xform enum, given an xform string
+ *
+ * @param	xform_enum	A pointer to the xform type
+ *				enum to be filled
+ * @param	xform_string	xform string
+ *
+ * @return
+ * - Return -1 if string is not valid
+ * - Return 0 if the string is valid
+ */
+int __rte_experimental
+rte_cryptodev_asym_get_xform_enum(enum rte_crypto_asym_xform_type *xform_enum,
+		const char *xform_string);
+
+
 /** Macro used at end of crypto PMD list */
 #define RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() \
 	{ RTE_CRYPTO_OP_TYPE_UNDEFINED }
@@ -327,31 +393,50 @@ rte_cryptodev_get_aead_algo_enum(enum rte_crypto_aead_algorithm *algo_enum,
  *
  * Keep these flags synchronised with rte_cryptodev_get_feature_name()
  */
-#define	RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO	(1ULL << 0)
+#define	RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO		(1ULL << 0)
 /**< Symmetric crypto operations are supported */
-#define	RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO	(1ULL << 1)
+#define	RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO		(1ULL << 1)
 /**< Asymmetric crypto operations are supported */
-#define	RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING	(1ULL << 2)
+#define	RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING		(1ULL << 2)
 /**< Chaining symmetric crypto operations are supported */
-#define	RTE_CRYPTODEV_FF_CPU_SSE		(1ULL << 3)
+#define	RTE_CRYPTODEV_FF_CPU_SSE			(1ULL << 3)
 /**< Utilises CPU SIMD SSE instructions */
-#define	RTE_CRYPTODEV_FF_CPU_AVX		(1ULL << 4)
+#define	RTE_CRYPTODEV_FF_CPU_AVX			(1ULL << 4)
 /**< Utilises CPU SIMD AVX instructions */
-#define	RTE_CRYPTODEV_FF_CPU_AVX2		(1ULL << 5)
+#define	RTE_CRYPTODEV_FF_CPU_AVX2			(1ULL << 5)
 /**< Utilises CPU SIMD AVX2 instructions */
-#define	RTE_CRYPTODEV_FF_CPU_AESNI		(1ULL << 6)
+#define	RTE_CRYPTODEV_FF_CPU_AESNI			(1ULL << 6)
 /**< Utilises CPU AES-NI instructions */
-#define	RTE_CRYPTODEV_FF_HW_ACCELERATED		(1ULL << 7)
-/**< Operations are off-loaded to an external hardware accelerator */
-#define	RTE_CRYPTODEV_FF_CPU_AVX512		(1ULL << 8)
+#define	RTE_CRYPTODEV_FF_HW_ACCELERATED			(1ULL << 7)
+/**< Operations are off-loaded to an
+ * external hardware accelerator
+ */
+#define	RTE_CRYPTODEV_FF_CPU_AVX512			(1ULL << 8)
 /**< Utilises CPU SIMD AVX512 instructions */
-#define	RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER	(1ULL << 9)
-/**< Scatter-gather mbufs are supported */
-#define	RTE_CRYPTODEV_FF_CPU_NEON		(1ULL << 10)
+#define	RTE_CRYPTODEV_FF_IN_PLACE_SGL			(1ULL << 9)
+/**< In-place Scatter-gather (SGL) buffers, with multiple segments,
+ * are supported
+ */
+#define RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT		(1ULL << 10)
+/**< Out-of-place Scatter-gather (SGL) buffers are
+ * supported in input and output
+ */
+#define RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT		(1ULL << 11)
+/**< Out-of-place Scatter-gather (SGL) buffers are supported
+ * in input, combined with linear buffers (LB), with a
+ * single segment in output
+ */
+#define RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT		(1ULL << 12)
+/**< Out-of-place Scatter-gather (SGL) buffers are supported
+ * in output, combined with linear buffers (LB) in input
+ */
+#define RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT		(1ULL << 13)
+/**< Out-of-place linear buffers (LB) are supported in input and output */
+#define	RTE_CRYPTODEV_FF_CPU_NEON			(1ULL << 14)
 /**< Utilises CPU NEON instructions */
-#define	RTE_CRYPTODEV_FF_CPU_ARM_CE		(1ULL << 11)
+#define	RTE_CRYPTODEV_FF_CPU_ARM_CE			(1ULL << 15)
 /**< Utilises ARM CPU Cryptographic Extensions */
-#define	RTE_CRYPTODEV_FF_SECURITY		(1ULL << 12)
+#define	RTE_CRYPTODEV_FF_SECURITY			(1ULL << 16)
 /**< Support Security Protocol Processing */
 
 
@@ -369,9 +454,9 @@ rte_cryptodev_get_feature_name(uint64_t flag);
 
 /**  Crypto device information */
 struct rte_cryptodev_info {
-	const char *driver_name;		/**< Driver name. */
-	uint8_t driver_id;			/**< Driver identifier */
-	struct rte_pci_device *pci_dev;		/**< PCI information. */
+	const char *driver_name;	/**< Driver name. */
+	uint8_t driver_id;		/**< Driver identifier */
+	struct rte_device *device;	/**< Generic device information. */
 
 	uint64_t feature_flags;
 	/**< Feature flags exposes HW/SW features for the given device */
@@ -382,12 +467,17 @@ struct rte_cryptodev_info {
 	unsigned max_nb_queue_pairs;
 	/**< Maximum number of queues pairs supported by device. */
 
+	uint16_t min_mbuf_headroom_req;
+	/**< Minimum mbuf headroom required by device */
+
+	uint16_t min_mbuf_tailroom_req;
+	/**< Minimum mbuf tailroom required by device */
+
 	struct {
 		unsigned max_nb_sessions;
-		/**< Maximum number of sessions supported by device. */
-		unsigned int max_nb_sessions_per_qp;
-		/**< Maximum number of sessions per queue pair.
-		 * Default 0 for infinite sessions
+		/**< Maximum number of sessions supported by device.
+		 * If 0, the device does not have any limitation in
+		 * number of sessions that can be used.
 		 */
 	} sym;
 };
@@ -603,43 +693,6 @@ rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
 		struct rte_mempool *session_pool);
 
 /**
- * @deprecated
- * Start a specified queue pair of a device. It is used
- * when deferred_start flag of the specified queue is true.
- *
- * @param	dev_id		The identifier of the device
- * @param	queue_pair_id	The index of the queue pair to start. The value
- *				must be in the range [0, nb_queue_pair - 1]
- *				previously supplied to
- *				rte_crypto_dev_configure().
- * @return
- *   - 0: Success, the transmit queue is correctly set up.
- *   - -EINVAL: The dev_id or the queue_id out of range.
- *   - -ENOTSUP: The function not supported in PMD driver.
- */
-__rte_deprecated
-extern int
-rte_cryptodev_queue_pair_start(uint8_t dev_id, uint16_t queue_pair_id);
-
-/**
- * @deprecated
- * Stop specified queue pair of a device
- *
- * @param	dev_id		The identifier of the device
- * @param	queue_pair_id	The index of the queue pair to stop. The value
- *				must be in the range [0, nb_queue_pair - 1]
- *				previously supplied to
- *				rte_cryptodev_configure().
- * @return
- *   - 0: Success, the transmit queue is correctly set up.
- *   - -EINVAL: The dev_id or the queue_id out of range.
- *   - -ENOTSUP: The function not supported in PMD driver.
- */
-__rte_deprecated
-extern int
-rte_cryptodev_queue_pair_stop(uint8_t dev_id, uint16_t queue_pair_id);
-
-/**
  * Get the number of queue pairs on a specific crypto device
  *
  * @param	dev_id		Crypto device identifier.
@@ -902,9 +955,14 @@ rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id,
  */
 struct rte_cryptodev_sym_session {
 	__extension__ void *sess_private_data[0];
-	/**< Private session material */
+	/**< Private symmetric session material */
 };
 
+/** Cryptodev asymmetric crypto session */
+struct rte_cryptodev_asym_session {
+	__extension__ void *sess_private_data[0];
+	/**< Private asymmetric session material */
+};
 
 /**
  * Create symmetric crypto session header (generic with no private data)
@@ -919,6 +977,18 @@ struct rte_cryptodev_sym_session *
 rte_cryptodev_sym_session_create(struct rte_mempool *mempool);
 
 /**
+ * Create asymmetric crypto session header (generic with no private data)
+ *
+ * @param   mempool    mempool to allocate asymmetric session
+ *                     objects from
+ * @return
+ *  - On success return pointer to asym-session
+ *  - On failure returns NULL
+ */
+struct rte_cryptodev_asym_session * __rte_experimental
+rte_cryptodev_asym_session_create(struct rte_mempool *mempool);
+
+/**
  * Frees symmetric crypto session header, after checking that all
  * the device private data has been freed, returning it
  * to its original mempool.
@@ -934,6 +1004,21 @@ int
 rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess);
 
 /**
+ * Frees asymmetric crypto session header, after checking that all
+ * the device private data has been freed, returning it
+ * to its original mempool.
+ *
+ * @param   sess     Session header to be freed.
+ *
+ * @return
+ *  - 0 if successful.
+ *  - -EINVAL if session is NULL.
+ *  - -EBUSY if not all device private data has been freed.
+ */
+int __rte_experimental
+rte_cryptodev_asym_session_free(struct rte_cryptodev_asym_session *sess);
+
+/**
  * Fill out private data for the device id, based on its device type.
  *
  * @param   dev_id   ID of device that we want the session to be used on
@@ -945,7 +1030,8 @@ rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess);
  * @return
  *  - On success, zero.
  *  - -EINVAL if input parameters are invalid.
- *  - -ENOTSUP if crypto device does not support the crypto transform.
+ *  - -ENOTSUP if crypto device does not support the crypto transform or
+ *    does not support symmetric operations.
  *  - -ENOMEM if the private session could not be allocated.
  */
 int
@@ -955,6 +1041,27 @@ rte_cryptodev_sym_session_init(uint8_t dev_id,
 			struct rte_mempool *mempool);
 
 /**
+ * Initialize asymmetric session on a device with specific asymmetric xform
+ *
+ * @param   dev_id   ID of device that we want the session to be used on
+ * @param   sess     Session to be set up on a device
+ * @param   xforms   Asymmetric crypto transform operations to apply on flow
+ *                   processed with this session
+ * @param   mempool  Mempool to be used for internal allocation.
+ *
+ * @return
+ *  - On success, zero.
+ *  - -EINVAL if input parameters are invalid.
+ *  - -ENOTSUP if crypto device does not support the crypto transform.
+ *  - -ENOMEM if the private session could not be allocated.
+ */
+int __rte_experimental
+rte_cryptodev_asym_session_init(uint8_t dev_id,
+			struct rte_cryptodev_asym_session *sess,
+			struct rte_crypto_asym_xform *xforms,
+			struct rte_mempool *mempool);
+
+/**
  * Frees private data for the device id, based on its device type,
  * returning it to its mempool. It is the application's responsibility
  * to ensure that private session data is not cleared while there are
@@ -966,44 +1073,43 @@ rte_cryptodev_sym_session_init(uint8_t dev_id,
  * @return
  *  - 0 if successful.
  *  - -EINVAL if device is invalid or session is NULL.
+ *  - -ENOTSUP if crypto device does not support symmetric operations.
  */
 int
 rte_cryptodev_sym_session_clear(uint8_t dev_id,
 			struct rte_cryptodev_sym_session *sess);
 
 /**
- * @deprecated
- * Get the size of the header session, for all registered drivers.
+ * Frees resources held by asymmetric session during rte_cryptodev_session_init
  *
+ * @param   dev_id   ID of device that uses the asymmetric session.
+ * @param   sess     Asymmetric session setup on device using
+ *					 rte_cryptodev_session_init
  * @return
- *   Size of the header session.
+ *  - 0 if successful.
+ *  - -EINVAL if device is invalid or session is NULL.
  */
-__rte_deprecated
-unsigned int
-rte_cryptodev_get_header_session_size(void);
+int __rte_experimental
+rte_cryptodev_asym_session_clear(uint8_t dev_id,
+			struct rte_cryptodev_asym_session *sess);
 
 /**
- * @deprecated
- * Get the size of the private session data for a device.
- *
- * @param	dev_id		The device identifier.
+ * Get the size of the header session, for all registered drivers.
  *
  * @return
- *   - Size of the private data, if successful
- *   - 0 if device is invalid or does not have private session
+ *   Size of the symmetric eader session.
  */
-__rte_deprecated
 unsigned int
-rte_cryptodev_get_private_session_size(uint8_t dev_id);
+rte_cryptodev_sym_get_header_session_size(void);
 
 /**
- * Get the size of the header session, for all registered drivers.
+ * Get the size of the asymmetric session header, for all registered drivers.
  *
  * @return
- *   Size of the symmetric eader session.
+ *   Size of the asymmetric header session.
  */
-unsigned int
-rte_cryptodev_sym_get_header_session_size(void);
+unsigned int __rte_experimental
+rte_cryptodev_asym_get_header_session_size(void);
 
 /**
  * Get the size of the private symmetric session data
@@ -1020,40 +1126,17 @@ unsigned int
 rte_cryptodev_sym_get_private_session_size(uint8_t dev_id);
 
 /**
- * @deprecated
- * Attach queue pair with sym session.
- *
- * @param	dev_id		Device to which the session will be attached.
- * @param	qp_id		Queue pair to which the session will be attached.
- * @param	session		Session pointer previously allocated by
- *				*rte_cryptodev_sym_session_create*.
- *
- * @return
- *  - On success, zero.
- *  - On failure, a negative value.
- */
-__rte_deprecated
-int
-rte_cryptodev_queue_pair_attach_sym_session(uint8_t dev_id, uint16_t qp_id,
-		struct rte_cryptodev_sym_session *session);
-
-/**
- * @deprecated
- * Detach queue pair with sym session.
+ * Get the size of the private data for asymmetric session
+ * on device
  *
- * @param	dev_id		Device to which the session is attached.
- * @param	qp_id		Queue pair to which the session is attached.
- * @param	session		Session pointer previously allocated by
- *				*rte_cryptodev_sym_session_create*.
+ * @param	dev_id		The device identifier.
  *
  * @return
- *  - On success, zero.
- *  - On failure, a negative value.
+ *   - Size of the asymmetric private data, if successful
+ *   - 0 if device is invalid or does not have private session
  */
-__rte_deprecated
-int
-rte_cryptodev_queue_pair_detach_sym_session(uint8_t dev_id, uint16_t qp_id,
-		struct rte_cryptodev_sym_session *session);
+unsigned int __rte_experimental
+rte_cryptodev_asym_get_private_session_size(uint8_t dev_id);
 
 /**
  * Provide driver identifier.
@@ -1076,35 +1159,35 @@ int rte_cryptodev_driver_id_get(const char *name);
 const char *rte_cryptodev_driver_name_get(uint8_t driver_id);
 
 /**
- * Set private data for a session.
+ * Store user data in a session.
  *
  * @param	sess		Session pointer allocated by
  *				*rte_cryptodev_sym_session_create*.
- * @param	data		Pointer to the private data.
- * @param	size		Size of the private data.
+ * @param	data		Pointer to the user data.
+ * @param	size		Size of the user data.
  *
  * @return
  *  - On success, zero.
  *  - On failure, a negative value.
  */
 int __rte_experimental
-rte_cryptodev_sym_session_set_private_data(
+rte_cryptodev_sym_session_set_user_data(
 					struct rte_cryptodev_sym_session *sess,
 					void *data,
 					uint16_t size);
 
 /**
- * Get private data of a session.
+ * Get user data stored in a session.
  *
  * @param	sess		Session pointer allocated by
  *				*rte_cryptodev_sym_session_create*.
  *
  * @return
- *  - On success return pointer to private data.
+ *  - On success return pointer to user data.
  *  - On failure returns NULL.
  */
 void * __rte_experimental
-rte_cryptodev_sym_session_get_private_data(
+rte_cryptodev_sym_session_get_user_data(
 					struct rte_cryptodev_sym_session *sess);
 
 #ifdef __cplusplus
diff --git a/lib/librte_cryptodev/rte_cryptodev_pmd.c b/lib/librte_cryptodev/rte_cryptodev_pmd.c
index f2aac24b..2088ac3f 100644
--- a/lib/librte_cryptodev/rte_cryptodev_pmd.c
+++ b/lib/librte_cryptodev/rte_cryptodev_pmd.c
@@ -66,13 +66,6 @@ rte_cryptodev_pmd_parse_input_args(
 			goto free_kvlist;
 
 		ret = rte_kvargs_process(kvlist,
-				RTE_CRYPTODEV_PMD_MAX_NB_SESS_ARG,
-				&rte_cryptodev_pmd_parse_uint_arg,
-				&params->max_nb_sessions);
-		if (ret < 0)
-			goto free_kvlist;
-
-		ret = rte_kvargs_process(kvlist,
 				RTE_CRYPTODEV_PMD_SOCKET_ID_ARG,
 				&rte_cryptodev_pmd_parse_uint_arg,
 				&params->socket_id);
@@ -109,10 +102,9 @@ rte_cryptodev_pmd_create(const char *name,
 			device->driver->name, name);
 
 	CDEV_LOG_INFO("[%s] - Initialisation parameters - name: %s,"
-			"socket id: %d, max queue pairs: %u, max sessions: %u",
+			"socket id: %d, max queue pairs: %u",
 			device->driver->name, name,
-			params->socket_id, params->max_nb_queue_pairs,
-			params->max_nb_sessions);
+			params->socket_id, params->max_nb_queue_pairs);
 
 	/* allocate device structure */
 	cryptodev = rte_cryptodev_pmd_allocate(name, params->socket_id);
diff --git a/lib/librte_cryptodev/rte_cryptodev_pmd.h b/lib/librte_cryptodev/rte_cryptodev_pmd.h
index 69d77693..6ff49d64 100644
--- a/lib/librte_cryptodev/rte_cryptodev_pmd.h
+++ b/lib/librte_cryptodev/rte_cryptodev_pmd.h
@@ -1,32 +1,5 @@
-/*-
- *
- *   Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2015-2016 Intel Corporation.
  */
 
 #ifndef _RTE_CRYPTODEV_PMD_H_
@@ -59,18 +32,15 @@ extern "C" {
 
 
 #define RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS	8
-#define RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS	2048
 
 #define RTE_CRYPTODEV_PMD_NAME_ARG			("name")
 #define RTE_CRYPTODEV_PMD_MAX_NB_QP_ARG			("max_nb_queue_pairs")
-#define RTE_CRYPTODEV_PMD_MAX_NB_SESS_ARG		("max_nb_sessions")
 #define RTE_CRYPTODEV_PMD_SOCKET_ID_ARG			("socket_id")
 
 
 static const char * const cryptodev_pmd_valid_params[] = {
 	RTE_CRYPTODEV_PMD_NAME_ARG,
 	RTE_CRYPTODEV_PMD_MAX_NB_QP_ARG,
-	RTE_CRYPTODEV_PMD_MAX_NB_SESS_ARG,
 	RTE_CRYPTODEV_PMD_SOCKET_ID_ARG
 };
 
@@ -83,7 +53,6 @@ struct rte_cryptodev_pmd_init_params {
 	size_t private_data_size;
 	int socket_id;
 	unsigned int max_nb_queue_pairs;
-	unsigned int max_nb_sessions;
 };
 
 /** Global structure used for maintaining state of allocated crypto devices */
@@ -216,28 +185,6 @@ typedef void (*cryptodev_info_get_t)(struct rte_cryptodev *dev,
 				struct rte_cryptodev_info *dev_info);
 
 /**
- * Start queue pair of a device.
- *
- * @param	dev	Crypto device pointer
- * @param	qp_id	Queue Pair Index
- *
- * @return	Returns 0 on success.
- */
-typedef int (*cryptodev_queue_pair_start_t)(struct rte_cryptodev *dev,
-				uint16_t qp_id);
-
-/**
- * Stop queue pair of a device.
- *
- * @param	dev	Crypto device pointer
- * @param	qp_id	Queue Pair Index
- *
- * @return	Returns 0 on success.
- */
-typedef int (*cryptodev_queue_pair_stop_t)(struct rte_cryptodev *dev,
-				uint16_t qp_id);
-
-/**
  * Setup a queue pair for a device.
  *
  * @param	dev		Crypto device pointer
@@ -302,6 +249,17 @@ typedef int (*cryptodev_sym_create_session_pool_t)(
  */
 typedef unsigned (*cryptodev_sym_get_session_private_size_t)(
 		struct rte_cryptodev *dev);
+/**
+ * Get the size of a asymmetric cryptodev session
+ *
+ * @param	dev		Crypto device pointer
+ *
+ * @return
+ *  - On success returns the size of the session structure for device
+ *  - On failure returns 0
+ */
+typedef unsigned int (*cryptodev_asym_get_session_private_size_t)(
+		struct rte_cryptodev *dev);
 
 /**
  * Configure a Crypto session on a device.
@@ -321,7 +279,24 @@ typedef int (*cryptodev_sym_configure_session_t)(struct rte_cryptodev *dev,
 		struct rte_crypto_sym_xform *xform,
 		struct rte_cryptodev_sym_session *session,
 		struct rte_mempool *mp);
-
+/**
+ * Configure a Crypto asymmetric session on a device.
+ *
+ * @param	dev		Crypto device pointer
+ * @param	xform		Single or chain of crypto xforms
+ * @param	priv_sess	Pointer to cryptodev's private session structure
+ * @param	mp		Mempool where the private session is allocated
+ *
+ * @return
+ *  - Returns 0 if private session structure have been created successfully.
+ *  - Returns -EINVAL if input parameters are invalid.
+ *  - Returns -ENOTSUP if crypto device does not support the crypto transform.
+ *  - Returns -ENOMEM if the private session could not be allocated.
+ */
+typedef int (*cryptodev_asym_configure_session_t)(struct rte_cryptodev *dev,
+		struct rte_crypto_asym_xform *xform,
+		struct rte_cryptodev_asym_session *session,
+		struct rte_mempool *mp);
 /**
  * Free driver private session data.
  *
@@ -330,32 +305,14 @@ typedef int (*cryptodev_sym_configure_session_t)(struct rte_cryptodev *dev,
  */
 typedef void (*cryptodev_sym_free_session_t)(struct rte_cryptodev *dev,
 		struct rte_cryptodev_sym_session *sess);
-
-/**
- * Optional API for drivers to attach sessions with queue pair.
- * @param	dev		Crypto device pointer
- * @param	qp_id		queue pair id for attaching session
- * @param	priv_sess       Pointer to cryptodev's private session structure
- * @return
- *  - Return 0 on success
- */
-typedef int (*cryptodev_sym_queue_pair_attach_session_t)(
-		  struct rte_cryptodev *dev,
-		  uint16_t qp_id,
-		  void *session_private);
-
 /**
- * Optional API for drivers to detach sessions from queue pair.
+ * Free asymmetric session private data.
+ *
  * @param	dev		Crypto device pointer
- * @param	qp_id		queue pair id for detaching session
- * @param	priv_sess       Pointer to cryptodev's private session structure
- * @return
- *  - Return 0 on success
+ * @param	sess		Cryptodev session structure
  */
-typedef int (*cryptodev_sym_queue_pair_detach_session_t)(
-		  struct rte_cryptodev *dev,
-		  uint16_t qp_id,
-		  void *session_private);
+typedef void (*cryptodev_asym_free_session_t)(struct rte_cryptodev *dev,
+		struct rte_cryptodev_asym_session *sess);
 
 /** Crypto device operations function pointer table */
 struct rte_cryptodev_ops {
@@ -375,23 +332,21 @@ struct rte_cryptodev_ops {
 	/**< Set up a device queue pair. */
 	cryptodev_queue_pair_release_t queue_pair_release;
 	/**< Release a queue pair. */
-	cryptodev_queue_pair_start_t queue_pair_start;
-	/**< Start a queue pair. */
-	cryptodev_queue_pair_stop_t queue_pair_stop;
-	/**< Stop a queue pair. */
 	cryptodev_queue_pair_count_t queue_pair_count;
 	/**< Get count of the queue pairs. */
 
-	cryptodev_sym_get_session_private_size_t session_get_size;
+	cryptodev_sym_get_session_private_size_t sym_session_get_size;
 	/**< Return private session. */
-	cryptodev_sym_configure_session_t session_configure;
+	cryptodev_asym_get_session_private_size_t asym_session_get_size;
+	/**< Return asym session private size. */
+	cryptodev_sym_configure_session_t sym_session_configure;
 	/**< Configure a Crypto session. */
-	cryptodev_sym_free_session_t session_clear;
+	cryptodev_asym_configure_session_t asym_session_configure;
+	/**< Configure asymmetric Crypto session. */
+	cryptodev_sym_free_session_t sym_session_clear;
+	/**< Clear a Crypto sessions private data. */
+	cryptodev_asym_free_session_t asym_session_clear;
 	/**< Clear a Crypto sessions private data. */
-	cryptodev_sym_queue_pair_attach_session_t qp_attach_session;
-	/**< Attach session to queue pair. */
-	cryptodev_sym_queue_pair_detach_session_t qp_detach_session;
-	/**< Detach session from queue pair. */
 };
 
 
@@ -516,20 +471,32 @@ uint8_t rte_cryptodev_allocate_driver(struct cryptodev_driver *crypto_drv,
 
 
 #define RTE_PMD_REGISTER_CRYPTO_DRIVER(crypto_drv, drv, driver_id)\
-RTE_INIT(init_ ##driver_id);\
-static void init_ ##driver_id(void)\
+RTE_INIT(init_ ##driver_id)\
 {\
 	driver_id = rte_cryptodev_allocate_driver(&crypto_drv, &(drv));\
 }
 
 static inline void *
-get_session_private_data(const struct rte_cryptodev_sym_session *sess,
+get_sym_session_private_data(const struct rte_cryptodev_sym_session *sess,
+		uint8_t driver_id) {
+	return sess->sess_private_data[driver_id];
+}
+
+static inline void
+set_sym_session_private_data(struct rte_cryptodev_sym_session *sess,
+		uint8_t driver_id, void *private_data)
+{
+	sess->sess_private_data[driver_id] = private_data;
+}
+
+static inline void *
+get_asym_session_private_data(const struct rte_cryptodev_asym_session *sess,
 		uint8_t driver_id) {
 	return sess->sess_private_data[driver_id];
 }
 
 static inline void
-set_session_private_data(struct rte_cryptodev_sym_session *sess,
+set_asym_session_private_data(struct rte_cryptodev_asym_session *sess,
 		uint8_t driver_id, void *private_data)
 {
 	sess->sess_private_data[driver_id] = private_data;
diff --git a/lib/librte_cryptodev/rte_cryptodev_version.map b/lib/librte_cryptodev/rte_cryptodev_version.map
index be8f4c1a..7ca00735 100644
--- a/lib/librte_cryptodev/rte_cryptodev_version.map
+++ b/lib/librte_cryptodev/rte_cryptodev_version.map
@@ -22,8 +22,6 @@ DPDK_16.04 {
 	rte_cryptodev_stop;
 	rte_cryptodev_queue_pair_count;
 	rte_cryptodev_queue_pair_setup;
-	rte_cryptodev_queue_pair_start;
-	rte_cryptodev_queue_pair_stop;
 	rte_crypto_op_pool_create;
 
 	local: *;
@@ -52,8 +50,6 @@ DPDK_17.05 {
 
 	rte_cryptodev_get_auth_algo_enum;
 	rte_cryptodev_get_cipher_algo_enum;
-	rte_cryptodev_queue_pair_attach_sym_session;
-	rte_cryptodev_queue_pair_detach_sym_session;
 
 } DPDK_17.02;
 
@@ -65,8 +61,6 @@ DPDK_17.08 {
 	rte_cryptodev_driver_id_get;
 	rte_cryptodev_driver_name_get;
 	rte_cryptodev_get_aead_algo_enum;
-	rte_cryptodev_get_header_session_size;
-	rte_cryptodev_get_private_session_size;
 	rte_cryptodev_sym_capability_check_aead;
 	rte_cryptodev_sym_session_init;
 	rte_cryptodev_sym_session_clear;
@@ -97,6 +91,18 @@ DPDK_18.05 {
 EXPERIMENTAL {
         global:
 
-	rte_cryptodev_sym_session_get_private_data;
-	rte_cryptodev_sym_session_set_private_data;
+	rte_cryptodev_asym_capability_get;
+	rte_cryptodev_asym_get_header_session_size;
+	rte_cryptodev_asym_get_private_session_size;
+	rte_cryptodev_asym_get_xform_enum;
+	rte_cryptodev_asym_session_clear;
+	rte_cryptodev_asym_session_create;
+	rte_cryptodev_asym_session_free;
+	rte_cryptodev_asym_session_init;
+	rte_cryptodev_asym_xform_capability_check_modlen;
+	rte_cryptodev_asym_xform_capability_check_optype;
+	rte_cryptodev_sym_session_get_user_data;
+	rte_cryptodev_sym_session_set_user_data;
+	rte_crypto_asym_op_strings;
+	rte_crypto_asym_xform_strings;
 };
diff --git a/lib/librte_eal/bsdapp/eal/Makefile b/lib/librte_eal/bsdapp/eal/Makefile
index 3fd33f1e..d27da3d1 100644
--- a/lib/librte_eal/bsdapp/eal/Makefile
+++ b/lib/librte_eal/bsdapp/eal/Makefile
@@ -18,10 +18,11 @@ CFLAGS += $(WERROR_FLAGS) -O3
 LDLIBS += -lexecinfo
 LDLIBS += -lpthread
 LDLIBS += -lgcc_s
+LDLIBS += -lrte_kvargs
 
 EXPORT_MAP := ../../rte_eal_version.map
 
-LIBABIVER := 7
+LIBABIVER := 8
 
 # specific to bsdapp exec-env
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) := eal.c
@@ -52,12 +53,14 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_hypervisor.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_string_fns.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_hexdump.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_devargs.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_class.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_bus.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_dev.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_options.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_thread.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_proc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_fbarray.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_uuid.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_heap.c
diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c
index dc279542..d7ae9d68 100644
--- a/lib/librte_eal/bsdapp/eal/eal.c
+++ b/lib/librte_eal/bsdapp/eal/eal.c
@@ -147,19 +147,9 @@ eal_get_runtime_dir(void)
 }
 
 /* Return user provided mbuf pool ops name */
-const char * __rte_experimental
-rte_eal_mbuf_user_pool_ops(void)
-{
-	return internal_config.user_mbuf_pool_ops_name;
-}
-
-/* Return mbuf pool ops name */
 const char *
-rte_eal_mbuf_default_mempool_ops(void)
+rte_eal_mbuf_user_pool_ops(void)
 {
-	if (internal_config.user_mbuf_pool_ops_name == NULL)
-		return RTE_MBUF_DEFAULT_MEMPOOL_OPS;
-
 	return internal_config.user_mbuf_pool_ops_name;
 }
 
@@ -286,12 +276,17 @@ eal_proc_type_detect(void)
 	enum rte_proc_type_t ptype = RTE_PROC_PRIMARY;
 	const char *pathname = eal_runtime_config_path();
 
-	/* if we can open the file but not get a write-lock we are a secondary
-	 * process. NOTE: if we get a file handle back, we keep that open
-	 * and don't close it to prevent a race condition between multiple opens */
-	if (((mem_cfg_fd = open(pathname, O_RDWR)) >= 0) &&
-			(fcntl(mem_cfg_fd, F_SETLK, &wr_lock) < 0))
-		ptype = RTE_PROC_SECONDARY;
+	/* if there no shared config, there can be no secondary processes */
+	if (!internal_config.no_shconf) {
+		/* if we can open the file but not get a write-lock we are a
+		 * secondary process. NOTE: if we get a file handle back, we
+		 * keep that open and don't close it to prevent a race condition
+		 * between multiple opens.
+		 */
+		if (((mem_cfg_fd = open(pathname, O_RDWR)) >= 0) &&
+				(fcntl(mem_cfg_fd, F_SETLK, &wr_lock) < 0))
+			ptype = RTE_PROC_SECONDARY;
+	}
 
 	RTE_LOG(INFO, EAL, "Auto-detected process type: %s\n",
 			ptype == RTE_PROC_PRIMARY ? "PRIMARY" : "SECONDARY");
@@ -468,6 +463,14 @@ eal_parse_args(int argc, char **argv)
 		}
 	}
 
+	/* create runtime data directory */
+	if (internal_config.no_shconf == 0 &&
+			eal_create_runtime_dir() < 0) {
+		RTE_LOG(ERR, EAL, "Cannot create runtime directory\n");
+		ret = -1;
+		goto out;
+	}
+
 	if (eal_adjust_config(&internal_config) != 0) {
 		ret = -1;
 		goto out;
@@ -600,13 +603,6 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
-	/* create runtime data directory */
-	if (eal_create_runtime_dir() < 0) {
-		rte_eal_init_alert("Cannot create runtime directory\n");
-		rte_errno = EACCES;
-		return -1;
-	}
-
 	/* FreeBSD always uses legacy memory model */
 	internal_config.legacy_mem = true;
 
@@ -625,6 +621,11 @@ rte_eal_init(int argc, char **argv)
 
 	rte_config_init();
 
+	if (rte_eal_intr_init() < 0) {
+		rte_eal_init_alert("Cannot init interrupt-handling thread\n");
+		return -1;
+	}
+
 	/* Put mp channel init before bus scan so that we can init the vdev
 	 * bus through mp channel in the secondary process before the bus scan.
 	 */
@@ -713,11 +714,6 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
-	if (rte_eal_intr_init() < 0) {
-		rte_eal_init_alert("Cannot init interrupt-handling thread\n");
-		return -1;
-	}
-
 	if (rte_eal_timer_init() < 0) {
 		rte_eal_init_alert("Cannot init HPET or TSC timers\n");
 		rte_errno = ENOTSUP;
@@ -866,21 +862,21 @@ int rte_vfio_clear_group(__rte_unused int vfio_group_fd)
 	return 0;
 }
 
-int __rte_experimental
+int
 rte_vfio_dma_map(uint64_t __rte_unused vaddr, __rte_unused uint64_t iova,
 		  __rte_unused uint64_t len)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_dma_unmap(uint64_t __rte_unused vaddr, uint64_t __rte_unused iova,
 		    __rte_unused uint64_t len)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_get_group_num(__rte_unused const char *sysfs_base,
 		       __rte_unused const char *dev_addr,
 		       __rte_unused int *iommu_group_num)
@@ -888,45 +884,45 @@ rte_vfio_get_group_num(__rte_unused const char *sysfs_base,
 	return -1;
 }
 
-int  __rte_experimental
+int
 rte_vfio_get_container_fd(void)
 {
 	return -1;
 }
 
-int  __rte_experimental
+int
 rte_vfio_get_group_fd(__rte_unused int iommu_group_num)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_create(void)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_destroy(__rte_unused int container_fd)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_group_bind(__rte_unused int container_fd,
 		__rte_unused int iommu_group_num)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_group_unbind(__rte_unused int container_fd,
 		__rte_unused int iommu_group_num)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_dma_map(__rte_unused int container_fd,
 			__rte_unused uint64_t vaddr,
 			__rte_unused uint64_t iova,
@@ -935,7 +931,7 @@ rte_vfio_container_dma_map(__rte_unused int container_fd,
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_dma_unmap(__rte_unused int container_fd,
 			__rte_unused uint64_t vaddr,
 			__rte_unused uint64_t iova,
diff --git a/lib/librte_eal/bsdapp/eal/eal_alarm.c b/lib/librte_eal/bsdapp/eal/eal_alarm.c
index eb3913c9..51ea4b8c 100644
--- a/lib/librte_eal/bsdapp/eal/eal_alarm.c
+++ b/lib/librte_eal/bsdapp/eal/eal_alarm.c
@@ -1,31 +1,314 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2010-2018 Intel Corporation
  */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
+#include <time.h>
 #include <errno.h>
 
 #include <rte_alarm.h>
+#include <rte_cycles.h>
 #include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_spinlock.h>
+
 #include "eal_private.h"
+#include "eal_alarm_private.h"
+
+#define NS_PER_US 1000
+
+#ifdef CLOCK_MONOTONIC_RAW /* Defined in glibc bits/time.h */
+#define CLOCK_TYPE_ID CLOCK_MONOTONIC_RAW
+#else
+#define CLOCK_TYPE_ID CLOCK_MONOTONIC
+#endif
+
+struct alarm_entry {
+	LIST_ENTRY(alarm_entry) next;
+	struct rte_intr_handle handle;
+	struct timespec time;
+	rte_eal_alarm_callback cb_fn;
+	void *cb_arg;
+	volatile uint8_t executing;
+	volatile pthread_t executing_id;
+};
+
+static LIST_HEAD(alarm_list, alarm_entry) alarm_list = LIST_HEAD_INITIALIZER();
+static rte_spinlock_t alarm_list_lk = RTE_SPINLOCK_INITIALIZER;
+
+static struct rte_intr_handle intr_handle = {.fd = -1 };
+static void eal_alarm_callback(void *arg);
 
 int
 rte_eal_alarm_init(void)
 {
+	intr_handle.type = RTE_INTR_HANDLE_ALARM;
+
+	/* on FreeBSD, timers don't use fd's, and their identifiers are stored
+	 * in separate namespace from fd's, so using any value is OK. however,
+	 * EAL interrupts handler expects fd's to be unique, so use an actual fd
+	 * to guarantee unique timer identifier.
+	 */
+	intr_handle.fd = open("/dev/zero", O_RDONLY);
+
+	return 0;
+}
+
+static inline int
+timespec_cmp(const struct timespec *now, const struct timespec *at)
+{
+	if (now->tv_sec < at->tv_sec)
+		return -1;
+	if (now->tv_sec > at->tv_sec)
+		return 1;
+	if (now->tv_nsec < at->tv_nsec)
+		return -1;
+	if (now->tv_nsec > at->tv_nsec)
+		return 1;
 	return 0;
 }
 
+static inline uint64_t
+diff_ns(struct timespec *now, struct timespec *at)
+{
+	uint64_t now_ns, at_ns;
+
+	if (timespec_cmp(now, at) >= 0)
+		return 0;
+
+	now_ns = now->tv_sec * NS_PER_S + now->tv_nsec;
+	at_ns = at->tv_sec * NS_PER_S + at->tv_nsec;
+
+	return at_ns - now_ns;
+}
 
 int
-rte_eal_alarm_set(uint64_t us __rte_unused,
-		rte_eal_alarm_callback cb_fn __rte_unused,
-		void *cb_arg __rte_unused)
+eal_alarm_get_timeout_ns(uint64_t *val)
 {
-	return -ENOTSUP;
+	struct alarm_entry *ap;
+	struct timespec now;
+
+	if (clock_gettime(CLOCK_TYPE_ID, &now) < 0)
+		return -1;
+
+	if (LIST_EMPTY(&alarm_list))
+		return -1;
+
+	ap = LIST_FIRST(&alarm_list);
+
+	*val = diff_ns(&now, &ap->time);
+
+	return 0;
+}
+
+static int
+unregister_current_callback(void)
+{
+	struct alarm_entry *ap;
+	int ret = 0;
+
+	if (!LIST_EMPTY(&alarm_list)) {
+		ap = LIST_FIRST(&alarm_list);
+
+		do {
+			ret = rte_intr_callback_unregister(&intr_handle,
+				eal_alarm_callback, &ap->time);
+		} while (ret == -EAGAIN);
+	}
+
+	return ret;
 }
 
+static int
+register_first_callback(void)
+{
+	struct alarm_entry *ap;
+	int ret = 0;
+
+	if (!LIST_EMPTY(&alarm_list)) {
+		ap = LIST_FIRST(&alarm_list);
+
+		/* register a new callback */
+		ret = rte_intr_callback_register(&intr_handle,
+				eal_alarm_callback, &ap->time);
+	}
+	return ret;
+}
+
+static void
+eal_alarm_callback(void *arg __rte_unused)
+{
+	struct timespec now;
+	struct alarm_entry *ap;
+
+	rte_spinlock_lock(&alarm_list_lk);
+	ap = LIST_FIRST(&alarm_list);
+
+	if (clock_gettime(CLOCK_TYPE_ID, &now) < 0)
+		return;
+
+	while (ap != NULL && timespec_cmp(&now, &ap->time) >= 0) {
+		ap->executing = 1;
+		ap->executing_id = pthread_self();
+		rte_spinlock_unlock(&alarm_list_lk);
+
+		ap->cb_fn(ap->cb_arg);
+
+		rte_spinlock_lock(&alarm_list_lk);
+
+		LIST_REMOVE(ap, next);
+		free(ap);
+
+		ap = LIST_FIRST(&alarm_list);
+	}
+
+	/* timer has been deleted from the kqueue, so recreate it if needed */
+	register_first_callback();
+
+	rte_spinlock_unlock(&alarm_list_lk);
+}
+
+
 int
-rte_eal_alarm_cancel(rte_eal_alarm_callback cb_fn __rte_unused,
-		void *cb_arg __rte_unused)
+rte_eal_alarm_set(uint64_t us, rte_eal_alarm_callback cb_fn, void *cb_arg)
 {
-	return -ENOTSUP;
+	struct alarm_entry *ap, *new_alarm;
+	struct timespec now;
+	uint64_t ns;
+	int ret = 0;
+
+	/* check parameters, also ensure us won't cause a uint64_t overflow */
+	if (us < 1 || us > (UINT64_MAX - US_PER_S) || cb_fn == NULL)
+		return -EINVAL;
+
+	new_alarm = calloc(1, sizeof(*new_alarm));
+	if (new_alarm == NULL)
+		return -ENOMEM;
+
+	/* use current time to calculate absolute time of alarm */
+	clock_gettime(CLOCK_TYPE_ID, &now);
+
+	ns = us * NS_PER_US;
+
+	new_alarm->cb_fn = cb_fn;
+	new_alarm->cb_arg = cb_arg;
+	new_alarm->time.tv_nsec = (now.tv_nsec + ns) % NS_PER_S;
+	new_alarm->time.tv_sec = now.tv_sec + ((now.tv_nsec + ns) / NS_PER_S);
+
+	rte_spinlock_lock(&alarm_list_lk);
+
+	if (LIST_EMPTY(&alarm_list))
+		LIST_INSERT_HEAD(&alarm_list, new_alarm, next);
+	else {
+		LIST_FOREACH(ap, &alarm_list, next) {
+			if (timespec_cmp(&new_alarm->time, &ap->time) < 0) {
+				LIST_INSERT_BEFORE(ap, new_alarm, next);
+				break;
+			}
+			if (LIST_NEXT(ap, next) == NULL) {
+				LIST_INSERT_AFTER(ap, new_alarm, next);
+				break;
+			}
+		}
+	}
+
+	/* re-register first callback just in case */
+	register_first_callback();
+
+	rte_spinlock_unlock(&alarm_list_lk);
+
+	return ret;
+}
+
+int
+rte_eal_alarm_cancel(rte_eal_alarm_callback cb_fn, void *cb_arg)
+{
+	struct alarm_entry *ap, *ap_prev;
+	int count = 0;
+	int err = 0;
+	int executing;
+
+	if (!cb_fn) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+
+	do {
+		executing = 0;
+		rte_spinlock_lock(&alarm_list_lk);
+		/* remove any matches at the start of the list */
+		while (1) {
+			ap = LIST_FIRST(&alarm_list);
+			if (ap == NULL)
+				break;
+			if (cb_fn != ap->cb_fn)
+				break;
+			if (cb_arg != ap->cb_arg && cb_arg != (void *) -1)
+				break;
+			if (ap->executing == 0) {
+				LIST_REMOVE(ap, next);
+				free(ap);
+				count++;
+			} else {
+				/* If calling from other context, mark that
+				 * alarm is executing so loop can spin till it
+				 * finish. Otherwise we are trying to cancel
+				 * ourselves - mark it by EINPROGRESS.
+				 */
+				if (pthread_equal(ap->executing_id,
+						pthread_self()) == 0)
+					executing++;
+				else
+					err = EINPROGRESS;
+
+				break;
+			}
+		}
+		ap_prev = ap;
+
+		/* now go through list, removing entries not at start */
+		LIST_FOREACH(ap, &alarm_list, next) {
+			/* this won't be true first time through */
+			if (cb_fn == ap->cb_fn &&
+					(cb_arg == (void *)-1 ||
+					 cb_arg == ap->cb_arg)) {
+				if (ap->executing == 0) {
+					LIST_REMOVE(ap, next);
+					free(ap);
+					count++;
+					ap = ap_prev;
+				} else if (pthread_equal(ap->executing_id,
+							 pthread_self()) == 0) {
+					executing++;
+				} else {
+					err = EINPROGRESS;
+				}
+			}
+			ap_prev = ap;
+		}
+		rte_spinlock_unlock(&alarm_list_lk);
+	} while (executing != 0);
+
+	if (count == 0 && err == 0)
+		rte_errno = ENOENT;
+	else if (err)
+		rte_errno = err;
+
+	rte_spinlock_lock(&alarm_list_lk);
+
+	/* unregister if no alarms left, otherwise re-register first */
+	if (LIST_EMPTY(&alarm_list))
+		unregister_current_callback();
+	else
+		register_first_callback();
+
+	rte_spinlock_unlock(&alarm_list_lk);
+
+	return count;
 }
diff --git a/lib/librte_eal/bsdapp/eal/eal_alarm_private.h b/lib/librte_eal/bsdapp/eal/eal_alarm_private.h
new file mode 100644
index 00000000..65c71151
--- /dev/null
+++ b/lib/librte_eal/bsdapp/eal/eal_alarm_private.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef EAL_ALARM_PRIVATE_H
+#define EAL_ALARM_PRIVATE_H
+
+#include <inttypes.h>
+
+/*
+ * FreeBSD needs a back-channel communication mechanism between interrupt and
+ * alarm thread, because on FreeBSD, timer period is set up inside the interrupt
+ * API and not inside alarm API like on Linux.
+ */
+
+int
+eal_alarm_get_timeout_ns(uint64_t *val);
+
+#endif // EAL_ALARM_PRIVATE_H
diff --git a/lib/librte_eal/bsdapp/eal/eal_hugepage_info.c b/lib/librte_eal/bsdapp/eal/eal_hugepage_info.c
index 836feb67..1e8f5df2 100644
--- a/lib/librte_eal/bsdapp/eal/eal_hugepage_info.c
+++ b/lib/librte_eal/bsdapp/eal/eal_hugepage_info.c
@@ -101,6 +101,10 @@ eal_hugepage_info_init(void)
 	hpi->num_pages[0] = num_buffers;
 	hpi->lock_descriptor = fd;
 
+	/* for no shared files mode, do not create shared memory config */
+	if (internal_config.no_shconf)
+		return 0;
+
 	tmp_hpi = create_shared_memory(eal_hugepage_info_path(),
 			sizeof(internal_config.hugepage_info));
 	if (tmp_hpi == NULL ) {
diff --git a/lib/librte_eal/bsdapp/eal/eal_interrupts.c b/lib/librte_eal/bsdapp/eal/eal_interrupts.c
index 290d53ab..2feee2d5 100644
--- a/lib/librte_eal/bsdapp/eal/eal_interrupts.c
+++ b/lib/librte_eal/bsdapp/eal/eal_interrupts.c
@@ -1,51 +1,479 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2010-2018 Intel Corporation
  */
 
+#include <string.h>
+#include <sys/types.h>
+#include <sys/event.h>
+#include <sys/queue.h>
+#include <unistd.h>
+
+#include <rte_errno.h>
+#include <rte_lcore.h>
+#include <rte_spinlock.h>
 #include <rte_common.h>
 #include <rte_interrupts.h>
+
 #include "eal_private.h"
+#include "eal_alarm_private.h"
+
+#define MAX_INTR_EVENTS 16
+
+/**
+ * union buffer for reading on different devices
+ */
+union rte_intr_read_buffer {
+	char charbuf[16];                /* for others */
+};
+
+TAILQ_HEAD(rte_intr_cb_list, rte_intr_callback);
+TAILQ_HEAD(rte_intr_source_list, rte_intr_source);
+
+struct rte_intr_callback {
+	TAILQ_ENTRY(rte_intr_callback) next;
+	rte_intr_callback_fn cb_fn;  /**< callback address */
+	void *cb_arg;                /**< parameter for callback */
+};
+
+struct rte_intr_source {
+	TAILQ_ENTRY(rte_intr_source) next;
+	struct rte_intr_handle intr_handle; /**< interrupt handle */
+	struct rte_intr_cb_list callbacks;  /**< user callbacks */
+	uint32_t active;
+};
+
+/* global spinlock for interrupt data operation */
+static rte_spinlock_t intr_lock = RTE_SPINLOCK_INITIALIZER;
+
+/* interrupt sources list */
+static struct rte_intr_source_list intr_sources;
+
+/* interrupt handling thread */
+static pthread_t intr_thread;
+
+static volatile int kq = -1;
+
+static int
+intr_source_to_kevent(const struct rte_intr_handle *ih, struct kevent *ke)
+{
+	/* alarm callbacks are special case */
+	if (ih->type == RTE_INTR_HANDLE_ALARM) {
+		uint64_t timeout_ns;
+
+		/* get soonest alarm timeout */
+		if (eal_alarm_get_timeout_ns(&timeout_ns) < 0)
+			return -1;
+
+		ke->filter = EVFILT_TIMER;
+		/* timers are one shot */
+		ke->flags |= EV_ONESHOT;
+		ke->fflags = NOTE_NSECONDS;
+		ke->data = timeout_ns;
+	} else {
+		ke->filter = EVFILT_READ;
+	}
+	ke->ident = ih->fd;
+
+	return 0;
+}
 
 int
 rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
-			rte_intr_callback_fn cb,
-			void *cb_arg)
+		rte_intr_callback_fn cb, void *cb_arg)
 {
-	RTE_SET_USED(intr_handle);
-	RTE_SET_USED(cb);
-	RTE_SET_USED(cb_arg);
+	struct rte_intr_callback *callback = NULL;
+	struct rte_intr_source *src = NULL;
+	int ret, add_event;
 
-	return -ENOTSUP;
+	/* first do parameter checking */
+	if (intr_handle == NULL || intr_handle->fd < 0 || cb == NULL) {
+		RTE_LOG(ERR, EAL,
+			"Registering with invalid input parameter\n");
+		return -EINVAL;
+	}
+	if (kq < 0) {
+		RTE_LOG(ERR, EAL, "Kqueue is not active: %d\n", kq);
+		return -ENODEV;
+	}
+
+	/* allocate a new interrupt callback entity */
+	callback = calloc(1, sizeof(*callback));
+	if (callback == NULL) {
+		RTE_LOG(ERR, EAL, "Can not allocate memory\n");
+		return -ENOMEM;
+	}
+	callback->cb_fn = cb;
+	callback->cb_arg = cb_arg;
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if there is at least one callback registered for the fd */
+	TAILQ_FOREACH(src, &intr_sources, next) {
+		if (src->intr_handle.fd == intr_handle->fd) {
+			/* we had no interrupts for this */
+			if (TAILQ_EMPTY(&src->callbacks))
+				add_event = 1;
+
+			TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
+			ret = 0;
+			break;
+		}
+	}
+
+	/* no existing callbacks for this - add new source */
+	if (src == NULL) {
+		src = calloc(1, sizeof(*src));
+		if (src == NULL) {
+			RTE_LOG(ERR, EAL, "Can not allocate memory\n");
+			ret = -ENOMEM;
+			goto fail;
+		} else {
+			src->intr_handle = *intr_handle;
+			TAILQ_INIT(&src->callbacks);
+			TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
+			TAILQ_INSERT_TAIL(&intr_sources, src, next);
+			add_event = 1;
+			ret = 0;
+		}
+	}
+
+	/* add events to the queue. timer events are special as we need to
+	 * re-set the timer.
+	 */
+	if (add_event || src->intr_handle.type == RTE_INTR_HANDLE_ALARM) {
+		struct kevent ke;
+
+		memset(&ke, 0, sizeof(ke));
+		ke.flags = EV_ADD; /* mark for addition to the queue */
+
+		if (intr_source_to_kevent(intr_handle, &ke) < 0) {
+			RTE_LOG(ERR, EAL, "Cannot convert interrupt handle to kevent\n");
+			ret = -ENODEV;
+			goto fail;
+		}
+
+		/**
+		 * add the intr file descriptor into wait list.
+		 */
+		if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
+			/* currently, nic_uio does not support interrupts, so
+			 * this error will always be triggered and output to the
+			 * user. so, don't output it unless debug log level set.
+			 */
+			if (errno == ENODEV)
+				RTE_LOG(DEBUG, EAL, "Interrupt handle %d not supported\n",
+					src->intr_handle.fd);
+			else
+				RTE_LOG(ERR, EAL, "Error adding fd %d "
+						"kevent, %s\n",
+						src->intr_handle.fd,
+						strerror(errno));
+			ret = -errno;
+			goto fail;
+		}
+	}
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+fail:
+	/* clean up */
+	if (src != NULL) {
+		TAILQ_REMOVE(&(src->callbacks), callback, next);
+		if (TAILQ_EMPTY(&(src->callbacks))) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+	}
+	free(callback);
+	rte_spinlock_unlock(&intr_lock);
+	return ret;
 }
 
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
-			rte_intr_callback_fn cb,
-			void *cb_arg)
+		rte_intr_callback_fn cb_fn, void *cb_arg)
 {
-	RTE_SET_USED(intr_handle);
-	RTE_SET_USED(cb);
-	RTE_SET_USED(cb_arg);
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
 
-	return -ENOTSUP;
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+	if (kq < 0) {
+		RTE_LOG(ERR, EAL, "Kqueue is not active\n");
+		return -ENODEV;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* interrupt source has some active callbacks right now. */
+	} else if (src->active != 0) {
+		ret = -EAGAIN;
+
+	/* ok to remove. */
+	} else {
+		struct kevent ke;
+
+		ret = 0;
+
+		/* remove it from the kqueue */
+		memset(&ke, 0, sizeof(ke));
+		ke.flags = EV_DELETE; /* mark for deletion from the queue */
+
+		if (intr_source_to_kevent(intr_handle, &ke) < 0) {
+			RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
+			ret = -ENODEV;
+			goto out;
+		}
+
+		/**
+		 * remove intr file descriptor from wait list.
+		 */
+		if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
+			RTE_LOG(ERR, EAL, "Error removing fd %d kevent, %s\n",
+				src->intr_handle.fd, strerror(errno));
+			/* removing non-existent even is an expected condition
+			 * in some circumstances (e.g. oneshot events).
+			 */
+		}
+
+		/*walk through the callbacks and remove all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				free(cb);
+				ret++;
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+	}
+out:
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
 }
 
 int
-rte_intr_enable(const struct rte_intr_handle *intr_handle __rte_unused)
+rte_intr_enable(const struct rte_intr_handle *intr_handle)
 {
-	return -ENOTSUP;
+	if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV)
+		return 0;
+
+	if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0)
+		return -1;
+
+	switch (intr_handle->type) {
+	/* not used at this moment */
+	case RTE_INTR_HANDLE_ALARM:
+		return -1;
+	/* not used at this moment */
+	case RTE_INTR_HANDLE_DEV_EVENT:
+		return -1;
+	/* unknown handle type */
+	default:
+		RTE_LOG(ERR, EAL,
+			"Unknown handle type of fd %d\n",
+					intr_handle->fd);
+		return -1;
+	}
+
+	return 0;
 }
 
 int
-rte_intr_disable(const struct rte_intr_handle *intr_handle __rte_unused)
+rte_intr_disable(const struct rte_intr_handle *intr_handle)
 {
-	return -ENOTSUP;
+	if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV)
+		return 0;
+
+	if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0)
+		return -1;
+
+	switch (intr_handle->type) {
+	/* not used at this moment */
+	case RTE_INTR_HANDLE_ALARM:
+		return -1;
+	/* not used at this moment */
+	case RTE_INTR_HANDLE_DEV_EVENT:
+		return -1;
+	/* unknown handle type */
+	default:
+		RTE_LOG(ERR, EAL,
+			"Unknown handle type of fd %d\n",
+					intr_handle->fd);
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+eal_intr_process_interrupts(struct kevent *events, int nfds)
+{
+	struct rte_intr_callback active_cb;
+	union rte_intr_read_buffer buf;
+	struct rte_intr_callback *cb;
+	struct rte_intr_source *src;
+	bool call = false;
+	int n, bytes_read;
+
+	for (n = 0; n < nfds; n++) {
+		int event_fd = events[n].ident;
+
+		rte_spinlock_lock(&intr_lock);
+		TAILQ_FOREACH(src, &intr_sources, next)
+			if (src->intr_handle.fd == event_fd)
+				break;
+		if (src == NULL) {
+			rte_spinlock_unlock(&intr_lock);
+			continue;
+		}
+
+		/* mark this interrupt source as active and release the lock. */
+		src->active = 1;
+		rte_spinlock_unlock(&intr_lock);
+
+		/* set the length to be read dor different handle type */
+		switch (src->intr_handle.type) {
+		case RTE_INTR_HANDLE_ALARM:
+			bytes_read = 0;
+			call = true;
+			break;
+		case RTE_INTR_HANDLE_VDEV:
+		case RTE_INTR_HANDLE_EXT:
+			bytes_read = 0;
+			call = true;
+			break;
+		case RTE_INTR_HANDLE_DEV_EVENT:
+			bytes_read = 0;
+			call = true;
+			break;
+		default:
+			bytes_read = 1;
+			break;
+		}
+
+		if (bytes_read > 0) {
+			/**
+			 * read out to clear the ready-to-be-read flag
+			 * for epoll_wait.
+			 */
+			bytes_read = read(event_fd, &buf, bytes_read);
+			if (bytes_read < 0) {
+				if (errno == EINTR || errno == EWOULDBLOCK)
+					continue;
+
+				RTE_LOG(ERR, EAL, "Error reading from file "
+					"descriptor %d: %s\n",
+					event_fd,
+					strerror(errno));
+			} else if (bytes_read == 0)
+				RTE_LOG(ERR, EAL, "Read nothing from file "
+					"descriptor %d\n", event_fd);
+			else
+				call = true;
+		}
+
+		/* grab a lock, again to call callbacks and update status. */
+		rte_spinlock_lock(&intr_lock);
+
+		if (call) {
+			/* Finally, call all callbacks. */
+			TAILQ_FOREACH(cb, &src->callbacks, next) {
+
+				/* make a copy and unlock. */
+				active_cb = *cb;
+				rte_spinlock_unlock(&intr_lock);
+
+				/* call the actual callback */
+				active_cb.cb_fn(active_cb.cb_arg);
+
+				/*get the lock back. */
+				rte_spinlock_lock(&intr_lock);
+			}
+		}
+
+		/* we done with that interrupt source, release it. */
+		src->active = 0;
+		rte_spinlock_unlock(&intr_lock);
+	}
+}
+
+static void *
+eal_intr_thread_main(void *arg __rte_unused)
+{
+	struct kevent events[MAX_INTR_EVENTS];
+	int nfds;
+
+	/* host thread, never break out */
+	for (;;) {
+		/* do not change anything, just wait */
+		nfds = kevent(kq, NULL, 0, events, MAX_INTR_EVENTS, NULL);
+
+		/* kevent fail */
+		if (nfds < 0) {
+			if (errno == EINTR)
+				continue;
+			RTE_LOG(ERR, EAL,
+				"kevent returns with fail\n");
+			break;
+		}
+		/* kevent timeout, will never happen here */
+		else if (nfds == 0)
+			continue;
+
+		/* kevent has at least one fd ready to read */
+		eal_intr_process_interrupts(events, nfds);
+	}
+	close(kq);
+	kq = -1;
+	return NULL;
 }
 
 int
 rte_eal_intr_init(void)
 {
-	return 0;
+	int ret = 0;
+
+	/* init the global interrupt source head */
+	TAILQ_INIT(&intr_sources);
+
+	kq = kqueue();
+	if (kq < 0) {
+		RTE_LOG(ERR, EAL, "Cannot create kqueue instance\n");
+		return -1;
+	}
+
+	/* create the host thread to wait/handle the interrupt */
+	ret = rte_ctrl_thread_create(&intr_thread, "eal-intr-thread", NULL,
+			eal_intr_thread_main, NULL);
+	if (ret != 0) {
+		rte_errno = -ret;
+		RTE_LOG(ERR, EAL,
+			"Failed to create thread for interrupt handling\n");
+	}
+
+	return ret;
 }
 
 int
diff --git a/lib/librte_eal/bsdapp/eal/eal_memory.c b/lib/librte_eal/bsdapp/eal/eal_memory.c
index a5e03478..16d2bc7c 100644
--- a/lib/librte_eal/bsdapp/eal/eal_memory.c
+++ b/lib/librte_eal/bsdapp/eal/eal_memory.c
@@ -12,6 +12,7 @@
 
 #include <rte_eal.h>
 #include <rte_eal_memconfig.h>
+#include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_string_fns.h>
 #include "eal_private.h"
@@ -104,6 +105,8 @@ rte_eal_hugepage_init(void)
 	/* map all hugepages and sort them */
 	for (i = 0; i < internal_config.num_hugepage_sizes; i ++){
 		struct hugepage_info *hpi;
+		rte_iova_t prev_end = 0;
+		int prev_ms_idx = -1;
 		uint64_t page_sz, mem_needed;
 		unsigned int n_pages, max_pages;
 
@@ -124,10 +127,27 @@ rte_eal_hugepage_init(void)
 			int error;
 			size_t sysctl_size = sizeof(physaddr);
 			char physaddr_str[64];
+			bool is_adjacent;
+
+			/* first, check if this segment is IOVA-adjacent to
+			 * the previous one.
+			 */
+			snprintf(physaddr_str, sizeof(physaddr_str),
+					"hw.contigmem.physaddr.%d", j);
+			error = sysctlbyname(physaddr_str, &physaddr,
+					&sysctl_size, NULL, 0);
+			if (error < 0) {
+				RTE_LOG(ERR, EAL, "Failed to get physical addr for buffer %u "
+						"from %s\n", j, hpi->hugedir);
+				return -1;
+			}
+
+			is_adjacent = prev_end != 0 && physaddr == prev_end;
+			prev_end = physaddr + hpi->hugepage_sz;
 
 			for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS;
 					msl_idx++) {
-				bool empty;
+				bool empty, need_hole;
 				msl = &mcfg->memsegs[msl_idx];
 				arr = &msl->memseg_arr;
 
@@ -136,20 +156,23 @@ rte_eal_hugepage_init(void)
 
 				empty = arr->count == 0;
 
-				/* we need 1, plus hole if not empty */
+				/* we need a hole if this isn't an empty memseg
+				 * list, and if previous segment was not
+				 * adjacent to current one.
+				 */
+				need_hole = !empty && !is_adjacent;
+
+				/* we need 1, plus hole if not adjacent */
 				ms_idx = rte_fbarray_find_next_n_free(arr,
-						0, 1 + (empty ? 1 : 0));
+						0, 1 + (need_hole ? 1 : 0));
 
 				/* memseg list is full? */
 				if (ms_idx < 0)
 					continue;
 
-				/* leave some space between memsegs, they are
-				 * not IOVA contiguous, so they shouldn't be VA
-				 * contiguous either.
-				 */
-				if (!empty)
+				if (need_hole && prev_ms_idx == ms_idx - 1)
 					ms_idx++;
+				prev_ms_idx = ms_idx;
 
 				break;
 			}
@@ -178,16 +201,6 @@ rte_eal_hugepage_init(void)
 				return -1;
 			}
 
-			snprintf(physaddr_str, sizeof(physaddr_str), "hw.contigmem"
-					".physaddr.%d", j);
-			error = sysctlbyname(physaddr_str, &physaddr, &sysctl_size,
-					NULL, 0);
-			if (error < 0) {
-				RTE_LOG(ERR, EAL, "Failed to get physical addr for buffer %u "
-						"from %s\n", j, hpi->hugedir);
-				return -1;
-			}
-
 			seg->addr = addr;
 			seg->iova = physaddr;
 			seg->hugepage_sz = page_sz;
@@ -200,7 +213,7 @@ rte_eal_hugepage_init(void)
 
 			RTE_LOG(INFO, EAL, "Mapped memory segment %u @ %p: physaddr:0x%"
 					PRIx64", len %zu\n",
-					seg_idx, addr, physaddr, page_sz);
+					seg_idx++, addr, physaddr, page_sz);
 
 			total_mem += seg->len;
 		}
@@ -288,3 +301,217 @@ rte_eal_using_phys_addrs(void)
 {
 	return 0;
 }
+
+static uint64_t
+get_mem_amount(uint64_t page_sz, uint64_t max_mem)
+{
+	uint64_t area_sz, max_pages;
+
+	/* limit to RTE_MAX_MEMSEG_PER_LIST pages or RTE_MAX_MEM_MB_PER_LIST */
+	max_pages = RTE_MAX_MEMSEG_PER_LIST;
+	max_mem = RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_LIST << 20, max_mem);
+
+	area_sz = RTE_MIN(page_sz * max_pages, max_mem);
+
+	/* make sure the list isn't smaller than the page size */
+	area_sz = RTE_MAX(area_sz, page_sz);
+
+	return RTE_ALIGN(area_sz, page_sz);
+}
+
+#define MEMSEG_LIST_FMT "memseg-%" PRIu64 "k-%i-%i"
+static int
+alloc_memseg_list(struct rte_memseg_list *msl, uint64_t page_sz,
+		int n_segs, int socket_id, int type_msl_idx)
+{
+	char name[RTE_FBARRAY_NAME_LEN];
+
+	snprintf(name, sizeof(name), MEMSEG_LIST_FMT, page_sz >> 10, socket_id,
+		 type_msl_idx);
+	if (rte_fbarray_init(&msl->memseg_arr, name, n_segs,
+			sizeof(struct rte_memseg))) {
+		RTE_LOG(ERR, EAL, "Cannot allocate memseg list: %s\n",
+			rte_strerror(rte_errno));
+		return -1;
+	}
+
+	msl->page_sz = page_sz;
+	msl->socket_id = socket_id;
+	msl->base_va = NULL;
+
+	RTE_LOG(DEBUG, EAL, "Memseg list allocated: 0x%zxkB at socket %i\n",
+			(size_t)page_sz >> 10, socket_id);
+
+	return 0;
+}
+
+static int
+alloc_va_space(struct rte_memseg_list *msl)
+{
+	uint64_t page_sz;
+	size_t mem_sz;
+	void *addr;
+	int flags = 0;
+
+#ifdef RTE_ARCH_PPC_64
+	flags |= MAP_HUGETLB;
+#endif
+
+	page_sz = msl->page_sz;
+	mem_sz = page_sz * msl->memseg_arr.len;
+
+	addr = eal_get_virtual_area(msl->base_va, &mem_sz, page_sz, 0, flags);
+	if (addr == NULL) {
+		if (rte_errno == EADDRNOTAVAIL)
+			RTE_LOG(ERR, EAL, "Could not mmap %llu bytes at [%p] - please use '--base-virtaddr' option\n",
+				(unsigned long long)mem_sz, msl->base_va);
+		else
+			RTE_LOG(ERR, EAL, "Cannot reserve memory\n");
+		return -1;
+	}
+	msl->base_va = addr;
+
+	return 0;
+}
+
+
+static int
+memseg_primary_init(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int hpi_idx, msl_idx = 0;
+	struct rte_memseg_list *msl;
+	uint64_t max_mem, total_mem;
+
+	/* no-huge does not need this at all */
+	if (internal_config.no_hugetlbfs)
+		return 0;
+
+	/* FreeBSD has an issue where core dump will dump the entire memory
+	 * contents, including anonymous zero-page memory. Therefore, while we
+	 * will be limiting total amount of memory to RTE_MAX_MEM_MB, we will
+	 * also be further limiting total memory amount to whatever memory is
+	 * available to us through contigmem driver (plus spacing blocks).
+	 *
+	 * so, at each stage, we will be checking how much memory we are
+	 * preallocating, and adjust all the values accordingly.
+	 */
+
+	max_mem = (uint64_t)RTE_MAX_MEM_MB << 20;
+	total_mem = 0;
+
+	/* create memseg lists */
+	for (hpi_idx = 0; hpi_idx < (int) internal_config.num_hugepage_sizes;
+			hpi_idx++) {
+		uint64_t max_type_mem, total_type_mem = 0;
+		uint64_t avail_mem;
+		int type_msl_idx, max_segs, avail_segs, total_segs = 0;
+		struct hugepage_info *hpi;
+		uint64_t hugepage_sz;
+
+		hpi = &internal_config.hugepage_info[hpi_idx];
+		hugepage_sz = hpi->hugepage_sz;
+
+		/* no NUMA support on FreeBSD */
+
+		/* check if we've already exceeded total memory amount */
+		if (total_mem >= max_mem)
+			break;
+
+		/* first, calculate theoretical limits according to config */
+		max_type_mem = RTE_MIN(max_mem - total_mem,
+			(uint64_t)RTE_MAX_MEM_MB_PER_TYPE << 20);
+		max_segs = RTE_MAX_MEMSEG_PER_TYPE;
+
+		/* now, limit all of that to whatever will actually be
+		 * available to us, because without dynamic allocation support,
+		 * all of that extra memory will be sitting there being useless
+		 * and slowing down core dumps in case of a crash.
+		 *
+		 * we need (N*2)-1 segments because we cannot guarantee that
+		 * each segment will be IOVA-contiguous with the previous one,
+		 * so we will allocate more and put spaces inbetween segments
+		 * that are non-contiguous.
+		 */
+		avail_segs = (hpi->num_pages[0] * 2) - 1;
+		avail_mem = avail_segs * hugepage_sz;
+
+		max_type_mem = RTE_MIN(avail_mem, max_type_mem);
+		max_segs = RTE_MIN(avail_segs, max_segs);
+
+		type_msl_idx = 0;
+		while (total_type_mem < max_type_mem &&
+				total_segs < max_segs) {
+			uint64_t cur_max_mem, cur_mem;
+			unsigned int n_segs;
+
+			if (msl_idx >= RTE_MAX_MEMSEG_LISTS) {
+				RTE_LOG(ERR, EAL,
+					"No more space in memseg lists, please increase %s\n",
+					RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS));
+				return -1;
+			}
+
+			msl = &mcfg->memsegs[msl_idx++];
+
+			cur_max_mem = max_type_mem - total_type_mem;
+
+			cur_mem = get_mem_amount(hugepage_sz,
+					cur_max_mem);
+			n_segs = cur_mem / hugepage_sz;
+
+			if (alloc_memseg_list(msl, hugepage_sz, n_segs,
+					0, type_msl_idx))
+				return -1;
+
+			total_segs += msl->memseg_arr.len;
+			total_type_mem = total_segs * hugepage_sz;
+			type_msl_idx++;
+
+			if (alloc_va_space(msl)) {
+				RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list\n");
+				return -1;
+			}
+		}
+		total_mem += total_type_mem;
+	}
+	return 0;
+}
+
+static int
+memseg_secondary_init(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int msl_idx = 0;
+	struct rte_memseg_list *msl;
+
+	for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS; msl_idx++) {
+
+		msl = &mcfg->memsegs[msl_idx];
+
+		/* skip empty memseg lists */
+		if (msl->memseg_arr.len == 0)
+			continue;
+
+		if (rte_fbarray_attach(&msl->memseg_arr)) {
+			RTE_LOG(ERR, EAL, "Cannot attach to primary process memseg lists\n");
+			return -1;
+		}
+
+		/* preallocate VA space */
+		if (alloc_va_space(msl)) {
+			RTE_LOG(ERR, EAL, "Cannot preallocate VA space for hugepage memory\n");
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+rte_eal_memseg_init(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY ?
+			memseg_primary_init() :
+			memseg_secondary_init();
+}
diff --git a/lib/librte_eal/bsdapp/eal/meson.build b/lib/librte_eal/bsdapp/eal/meson.build
index 47e16a64..3945b529 100644
--- a/lib/librte_eal/bsdapp/eal/meson.build
+++ b/lib/librte_eal/bsdapp/eal/meson.build
@@ -16,3 +16,5 @@ env_sources = files('eal_alarm.c',
 		'eal_memory.c',
 		'eal_dev.c'
 )
+
+deps += ['kvargs']
diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index 48f870f2..cca68826 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -11,12 +11,12 @@ INC += rte_per_lcore.h rte_random.h
 INC += rte_tailq.h rte_interrupts.h rte_alarm.h
 INC += rte_string_fns.h rte_version.h
 INC += rte_eal_memconfig.h rte_malloc_heap.h
-INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h
+INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h rte_class.h
 INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h
 INC += rte_malloc.h rte_keepalive.h rte_time.h
 INC += rte_service.h rte_service_component.h
 INC += rte_bitmap.h rte_vfio.h rte_hypervisor.h rte_test.h
-INC += rte_reciprocal.h rte_fbarray.h
+INC += rte_reciprocal.h rte_fbarray.h rte_uuid.h
 
 GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
 GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
diff --git a/lib/librte_eal/common/eal_common_class.c b/lib/librte_eal/common/eal_common_class.c
new file mode 100644
index 00000000..404a9065
--- /dev/null
+++ b/lib/librte_eal/common/eal_common_class.c
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Gaëtan Rivet
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/queue.h>
+
+#include <rte_class.h>
+#include <rte_debug.h>
+
+struct rte_class_list rte_class_list =
+	TAILQ_HEAD_INITIALIZER(rte_class_list);
+
+__rte_experimental void
+rte_class_register(struct rte_class *class)
+{
+	RTE_VERIFY(class);
+	RTE_VERIFY(class->name && strlen(class->name));
+
+	TAILQ_INSERT_TAIL(&rte_class_list, class, next);
+	RTE_LOG(DEBUG, EAL, "Registered [%s] device class.\n", class->name);
+}
+
+__rte_experimental void
+rte_class_unregister(struct rte_class *class)
+{
+	TAILQ_REMOVE(&rte_class_list, class, next);
+	RTE_LOG(DEBUG, EAL, "Unregistered [%s] device class.\n", class->name);
+}
+
+__rte_experimental
+struct rte_class *
+rte_class_find(const struct rte_class *start, rte_class_cmp_t cmp,
+	       const void *data)
+{
+	struct rte_class *cls;
+
+	if (start != NULL)
+		cls = TAILQ_NEXT(start, next);
+	else
+		cls = TAILQ_FIRST(&rte_class_list);
+	while (cls != NULL) {
+		if (cmp(cls, data) == 0)
+			break;
+		cls = TAILQ_NEXT(cls, next);
+	}
+	return cls;
+}
+
+static int
+cmp_class_name(const struct rte_class *class, const void *_name)
+{
+	const char *name = _name;
+
+	return strcmp(class->name, name);
+}
+
+__rte_experimental
+struct rte_class *
+rte_class_find_by_name(const char *name)
+{
+	return rte_class_find(NULL, cmp_class_name, (const void *)name);
+}
diff --git a/lib/librte_eal/common/eal_common_dev.c b/lib/librte_eal/common/eal_common_dev.c
index 61cb3b16..678dbcac 100644
--- a/lib/librte_eal/common/eal_common_dev.c
+++ b/lib/librte_eal/common/eal_common_dev.c
@@ -10,9 +10,12 @@
 
 #include <rte_compat.h>
 #include <rte_bus.h>
+#include <rte_class.h>
 #include <rte_dev.h>
 #include <rte_devargs.h>
 #include <rte_debug.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
 #include <rte_spinlock.h>
 #include <rte_malloc.h>
@@ -42,17 +45,27 @@ static struct dev_event_cb_list dev_event_cbs;
 /* spinlock for device callbacks */
 static rte_spinlock_t dev_event_lock = RTE_SPINLOCK_INITIALIZER;
 
-static int cmp_detached_dev_name(const struct rte_device *dev,
-	const void *_name)
-{
-	const char *name = _name;
+struct dev_next_ctx {
+	struct rte_dev_iterator *it;
+	const char *bus_str;
+	const char *cls_str;
+};
 
-	/* skip attached devices */
-	if (dev->driver != NULL)
-		return 1;
+#define CTX(it, bus_str, cls_str) \
+	(&(const struct dev_next_ctx){ \
+		.it = it, \
+		.bus_str = bus_str, \
+		.cls_str = cls_str, \
+	})
 
-	return strcmp(dev->name, name);
-}
+#define ITCTX(ptr) \
+	(((struct dev_next_ctx *)(intptr_t)ptr)->it)
+
+#define BUSCTX(ptr) \
+	(((struct dev_next_ctx *)(intptr_t)ptr)->bus_str)
+
+#define CLSCTX(ptr) \
+	(((struct dev_next_ctx *)(intptr_t)ptr)->cls_str)
 
 static int cmp_dev_name(const struct rte_device *dev, const void *_name)
 {
@@ -138,8 +151,8 @@ int __rte_experimental rte_eal_hotplug_add(const char *busname, const char *devn
 	if (da == NULL)
 		return -ENOMEM;
 
-	ret = rte_devargs_parse(da, "%s:%s,%s",
-				    busname, devname, devargs);
+	ret = rte_devargs_parsef(da, "%s:%s,%s",
+				 busname, devname, devargs);
 	if (ret)
 		goto err_devarg;
 
@@ -151,14 +164,19 @@ int __rte_experimental rte_eal_hotplug_add(const char *busname, const char *devn
 	if (ret)
 		goto err_devarg;
 
-	dev = bus->find_device(NULL, cmp_detached_dev_name, devname);
+	dev = bus->find_device(NULL, cmp_dev_name, devname);
 	if (dev == NULL) {
-		RTE_LOG(ERR, EAL, "Cannot find unplugged device (%s)\n",
+		RTE_LOG(ERR, EAL, "Cannot find device (%s)\n",
 			devname);
 		ret = -ENODEV;
 		goto err_devarg;
 	}
 
+	if (dev->driver != NULL) {
+		RTE_LOG(ERR, EAL, "Device is already plugged\n");
+		return -EEXIST;
+	}
+
 	ret = bus->plug(dev);
 	if (ret) {
 		RTE_LOG(ERR, EAL, "Driver cannot attach the device (%s)\n",
@@ -200,6 +218,11 @@ rte_eal_hotplug_remove(const char *busname, const char *devname)
 		return -EINVAL;
 	}
 
+	if (dev->driver == NULL) {
+		RTE_LOG(ERR, EAL, "Device is already unplugged\n");
+		return -ENOENT;
+	}
+
 	ret = bus->unplug(dev);
 	if (ret)
 		RTE_LOG(ERR, EAL, "Driver cannot detach the device (%s)\n",
@@ -343,3 +366,201 @@ dev_callback_process(char *device_name, enum rte_dev_event_type event)
 	}
 	rte_spinlock_unlock(&dev_event_lock);
 }
+
+__rte_experimental
+int
+rte_dev_iterator_init(struct rte_dev_iterator *it,
+		      const char *dev_str)
+{
+	struct rte_devargs devargs;
+	struct rte_class *cls = NULL;
+	struct rte_bus *bus = NULL;
+
+	/* Having both bus_str and cls_str NULL is illegal,
+	 * marking this iterator as invalid unless
+	 * everything goes well.
+	 */
+	it->bus_str = NULL;
+	it->cls_str = NULL;
+
+	devargs.data = dev_str;
+	if (rte_devargs_layers_parse(&devargs, dev_str))
+		goto get_out;
+
+	bus = devargs.bus;
+	cls = devargs.cls;
+	/* The string should have at least
+	 * one layer specified.
+	 */
+	if (bus == NULL && cls == NULL) {
+		RTE_LOG(ERR, EAL,
+			"Either bus or class must be specified.\n");
+		rte_errno = EINVAL;
+		goto get_out;
+	}
+	if (bus != NULL && bus->dev_iterate == NULL) {
+		RTE_LOG(ERR, EAL, "Bus %s not supported\n", bus->name);
+		rte_errno = ENOTSUP;
+		goto get_out;
+	}
+	if (cls != NULL && cls->dev_iterate == NULL) {
+		RTE_LOG(ERR, EAL, "Class %s not supported\n", cls->name);
+		rte_errno = ENOTSUP;
+		goto get_out;
+	}
+	it->bus_str = devargs.bus_str;
+	it->cls_str = devargs.cls_str;
+	it->dev_str = dev_str;
+	it->bus = bus;
+	it->cls = cls;
+	it->device = NULL;
+	it->class_device = NULL;
+get_out:
+	return -rte_errno;
+}
+
+static char *
+dev_str_sane_copy(const char *str)
+{
+	size_t end;
+	char *copy;
+
+	end = strcspn(str, ",/");
+	if (str[end] == ',') {
+		copy = strdup(&str[end + 1]);
+	} else {
+		/* '/' or '\0' */
+		copy = strdup("");
+	}
+	if (copy == NULL) {
+		rte_errno = ENOMEM;
+	} else {
+		char *slash;
+
+		slash = strchr(copy, '/');
+		if (slash != NULL)
+			slash[0] = '\0';
+	}
+	return copy;
+}
+
+static int
+class_next_dev_cmp(const struct rte_class *cls,
+		   const void *ctx)
+{
+	struct rte_dev_iterator *it;
+	const char *cls_str = NULL;
+	void *dev;
+
+	if (cls->dev_iterate == NULL)
+		return 1;
+	it = ITCTX(ctx);
+	cls_str = CLSCTX(ctx);
+	dev = it->class_device;
+	/* it->cls_str != NULL means a class
+	 * was specified in the devstr.
+	 */
+	if (it->cls_str != NULL && cls != it->cls)
+		return 1;
+	/* If an error occurred previously,
+	 * no need to test further.
+	 */
+	if (rte_errno != 0)
+		return -1;
+	dev = cls->dev_iterate(dev, cls_str, it);
+	it->class_device = dev;
+	return dev == NULL;
+}
+
+static int
+bus_next_dev_cmp(const struct rte_bus *bus,
+		 const void *ctx)
+{
+	struct rte_device *dev = NULL;
+	struct rte_class *cls = NULL;
+	struct rte_dev_iterator *it;
+	const char *bus_str = NULL;
+
+	if (bus->dev_iterate == NULL)
+		return 1;
+	it = ITCTX(ctx);
+	bus_str = BUSCTX(ctx);
+	dev = it->device;
+	/* it->bus_str != NULL means a bus
+	 * was specified in the devstr.
+	 */
+	if (it->bus_str != NULL && bus != it->bus)
+		return 1;
+	/* If an error occurred previously,
+	 * no need to test further.
+	 */
+	if (rte_errno != 0)
+		return -1;
+	if (it->cls_str == NULL) {
+		dev = bus->dev_iterate(dev, bus_str, it);
+		goto end;
+	}
+	/* cls_str != NULL */
+	if (dev == NULL) {
+next_dev_on_bus:
+		dev = bus->dev_iterate(dev, bus_str, it);
+		it->device = dev;
+	}
+	if (dev == NULL)
+		return 1;
+	if (it->cls != NULL)
+		cls = TAILQ_PREV(it->cls, rte_class_list, next);
+	cls = rte_class_find(cls, class_next_dev_cmp, ctx);
+	if (cls != NULL) {
+		it->cls = cls;
+		goto end;
+	}
+	goto next_dev_on_bus;
+end:
+	it->device = dev;
+	return dev == NULL;
+}
+__rte_experimental
+struct rte_device *
+rte_dev_iterator_next(struct rte_dev_iterator *it)
+{
+	struct rte_bus *bus = NULL;
+	int old_errno = rte_errno;
+	char *bus_str = NULL;
+	char *cls_str = NULL;
+
+	rte_errno = 0;
+	if (it->bus_str == NULL && it->cls_str == NULL) {
+		/* Invalid iterator. */
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	if (it->bus != NULL)
+		bus = TAILQ_PREV(it->bus, rte_bus_list, next);
+	if (it->bus_str != NULL) {
+		bus_str = dev_str_sane_copy(it->bus_str);
+		if (bus_str == NULL)
+			goto out;
+	}
+	if (it->cls_str != NULL) {
+		cls_str = dev_str_sane_copy(it->cls_str);
+		if (cls_str == NULL)
+			goto out;
+	}
+	while ((bus = rte_bus_find(bus, bus_next_dev_cmp,
+				   CTX(it, bus_str, cls_str)))) {
+		if (it->device != NULL) {
+			it->bus = bus;
+			goto out;
+		}
+		if (it->bus_str != NULL ||
+		    rte_errno != 0)
+			break;
+	}
+	if (rte_errno == 0)
+		rte_errno = old_errno;
+out:
+	free(bus_str);
+	free(cls_str);
+	return it->device;
+}
diff --git a/lib/librte_eal/common/eal_common_devargs.c b/lib/librte_eal/common/eal_common_devargs.c
index b0434158..dac2402a 100644
--- a/lib/librte_eal/common/eal_common_devargs.c
+++ b/lib/librte_eal/common/eal_common_devargs.c
@@ -13,9 +13,14 @@
 #include <string.h>
 #include <stdarg.h>
 
+#include <rte_bus.h>
+#include <rte_class.h>
 #include <rte_compat.h>
 #include <rte_dev.h>
 #include <rte_devargs.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_log.h>
 #include <rte_tailq.h>
 #include "eal_private.h"
 
@@ -56,30 +61,164 @@ rte_eal_parse_devargs_str(const char *devargs_str,
 	return 0;
 }
 
+static size_t
+devargs_layer_count(const char *s)
+{
+	size_t i = s ? 1 : 0;
+
+	while (s != NULL && s[0] != '\0') {
+		i += s[0] == '/';
+		s++;
+	}
+	return i;
+}
+
+int
+rte_devargs_layers_parse(struct rte_devargs *devargs,
+			 const char *devstr)
+{
+	struct {
+		const char *key;
+		const char *str;
+		struct rte_kvargs *kvlist;
+	} layers[] = {
+		{ "bus=",    NULL, NULL, },
+		{ "class=",  NULL, NULL, },
+		{ "driver=", NULL, NULL, },
+	};
+	struct rte_kvargs_pair *kv = NULL;
+	struct rte_class *cls = NULL;
+	struct rte_bus *bus = NULL;
+	const char *s = devstr;
+	size_t nblayer;
+	size_t i = 0;
+	int ret = 0;
+
+	/* Split each sub-lists. */
+	nblayer = devargs_layer_count(devstr);
+	if (nblayer > RTE_DIM(layers)) {
+		RTE_LOG(ERR, EAL, "Invalid format: too many layers (%zu)\n",
+			nblayer);
+		ret = -E2BIG;
+		goto get_out;
+	}
+
+	/* If the devargs points the devstr
+	 * as source data, then it should not allocate
+	 * anything and keep referring only to it.
+	 */
+	if (devargs->data != devstr) {
+		devargs->data = strdup(devstr);
+		if (devargs->data == NULL) {
+			RTE_LOG(ERR, EAL, "OOM\n");
+			ret = -ENOMEM;
+			goto get_out;
+		}
+		s = devargs->data;
+	}
+
+	while (s != NULL) {
+		if (i >= RTE_DIM(layers)) {
+			RTE_LOG(ERR, EAL, "Unrecognized layer %s\n", s);
+			ret = -EINVAL;
+			goto get_out;
+		}
+		/*
+		 * The last layer is free-form.
+		 * The "driver" key is not required (but accepted).
+		 */
+		if (strncmp(layers[i].key, s, strlen(layers[i].key)) &&
+				i != RTE_DIM(layers) - 1)
+			goto next_layer;
+		layers[i].str = s;
+		layers[i].kvlist = rte_kvargs_parse_delim(s, NULL, "/");
+		if (layers[i].kvlist == NULL) {
+			RTE_LOG(ERR, EAL, "Could not parse %s\n", s);
+			ret = -EINVAL;
+			goto get_out;
+		}
+		s = strchr(s, '/');
+		if (s != NULL)
+			s++;
+next_layer:
+		i++;
+	}
+
+	/* Parse each sub-list. */
+	for (i = 0; i < RTE_DIM(layers); i++) {
+		if (layers[i].kvlist == NULL)
+			continue;
+		kv = &layers[i].kvlist->pairs[0];
+		if (strcmp(kv->key, "bus") == 0) {
+			bus = rte_bus_find_by_name(kv->value);
+			if (bus == NULL) {
+				RTE_LOG(ERR, EAL, "Could not find bus \"%s\"\n",
+					kv->value);
+				ret = -EFAULT;
+				goto get_out;
+			}
+		} else if (strcmp(kv->key, "class") == 0) {
+			cls = rte_class_find_by_name(kv->value);
+			if (cls == NULL) {
+				RTE_LOG(ERR, EAL, "Could not find class \"%s\"\n",
+					kv->value);
+				ret = -EFAULT;
+				goto get_out;
+			}
+		} else if (strcmp(kv->key, "driver") == 0) {
+			/* Ignore */
+			continue;
+		}
+	}
+
+	/* Fill devargs fields. */
+	devargs->bus_str = layers[0].str;
+	devargs->cls_str = layers[1].str;
+	devargs->drv_str = layers[2].str;
+	devargs->bus = bus;
+	devargs->cls = cls;
+
+	/* If we own the data, clean up a bit
+	 * the several layers string, to ease
+	 * their parsing afterward.
+	 */
+	if (devargs->data != devstr) {
+		char *s = (void *)(intptr_t)(devargs->data);
+
+		while ((s = strchr(s, '/'))) {
+			*s = '\0';
+			s++;
+		}
+	}
+
+get_out:
+	for (i = 0; i < RTE_DIM(layers); i++) {
+		if (layers[i].kvlist)
+			rte_kvargs_free(layers[i].kvlist);
+	}
+	if (ret != 0)
+		rte_errno = -ret;
+	return ret;
+}
+
 static int
 bus_name_cmp(const struct rte_bus *bus, const void *name)
 {
 	return strncmp(bus->name, name, strlen(bus->name));
 }
 
-int __rte_experimental
-rte_devargs_parse(struct rte_devargs *da, const char *format, ...)
+__rte_experimental
+int
+rte_devargs_parse(struct rte_devargs *da, const char *dev)
 {
 	struct rte_bus *bus = NULL;
-	va_list ap;
-	va_start(ap, format);
-	char dev[vsnprintf(NULL, 0, format, ap) + 1];
 	const char *devname;
 	const size_t maxlen = sizeof(da->name);
 	size_t i;
 
-	va_end(ap);
 	if (da == NULL)
 		return -EINVAL;
 
-	va_start(ap, format);
-	vsnprintf(dev, sizeof(dev), format, ap);
-	va_end(ap);
 	/* Retrieve eventual bus info */
 	do {
 		devname = dev;
@@ -96,7 +235,7 @@ rte_devargs_parse(struct rte_devargs *da, const char *format, ...)
 		da->name[i] = devname[i];
 		i++;
 		if (i == maxlen) {
-			fprintf(stderr, "WARNING: Parsing \"%s\": device name should be shorter than %zu\n",
+			RTE_LOG(WARNING, EAL, "Parsing \"%s\": device name should be shorter than %zu\n",
 				dev, maxlen);
 			da->name[i - 1] = '\0';
 			return -EINVAL;
@@ -106,7 +245,7 @@ rte_devargs_parse(struct rte_devargs *da, const char *format, ...)
 	if (bus == NULL) {
 		bus = rte_bus_find_by_device_name(da->name);
 		if (bus == NULL) {
-			fprintf(stderr, "ERROR: failed to parse device \"%s\"\n",
+			RTE_LOG(ERR, EAL, "failed to parse device \"%s\"\n",
 				da->name);
 			return -EFAULT;
 		}
@@ -118,12 +257,40 @@ rte_devargs_parse(struct rte_devargs *da, const char *format, ...)
 	else
 		da->args = strdup("");
 	if (da->args == NULL) {
-		fprintf(stderr, "ERROR: not enough memory to parse arguments\n");
+		RTE_LOG(ERR, EAL, "not enough memory to parse arguments\n");
 		return -ENOMEM;
 	}
 	return 0;
 }
 
+__rte_experimental
+int
+rte_devargs_parsef(struct rte_devargs *da, const char *format, ...)
+{
+	va_list ap;
+	size_t len;
+	char *dev;
+
+	if (da == NULL)
+		return -EINVAL;
+
+	va_start(ap, format);
+	len = vsnprintf(NULL, 0, format, ap);
+	va_end(ap);
+
+	dev = calloc(1, len + 1);
+	if (dev == NULL) {
+		RTE_LOG(ERR, EAL, "not enough memory to parse device\n");
+		return -ENOMEM;
+	}
+
+	va_start(ap, format);
+	vsnprintf(dev, len + 1, format, ap);
+	va_end(ap);
+
+	return rte_devargs_parse(da, dev);
+}
+
 int __rte_experimental
 rte_devargs_insert(struct rte_devargs *da)
 {
@@ -150,7 +317,7 @@ rte_devargs_add(enum rte_devtype devtype, const char *devargs_str)
 	if (devargs == NULL)
 		goto fail;
 
-	if (rte_devargs_parse(devargs, "%s", dev))
+	if (rte_devargs_parse(devargs, dev))
 		goto fail;
 	devargs->type = devtype;
 	bus = devargs->bus;
diff --git a/lib/librte_eal/common/eal_common_fbarray.c b/lib/librte_eal/common/eal_common_fbarray.c
index 019f84c1..43caf3ce 100644
--- a/lib/librte_eal/common/eal_common_fbarray.c
+++ b/lib/librte_eal/common/eal_common_fbarray.c
@@ -231,7 +231,7 @@ find_next_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n,
 		return MASK_GET_IDX(msk_idx, run_start);
 	}
 	/* we didn't find anything */
-	rte_errno = used ? -ENOENT : -ENOSPC;
+	rte_errno = used ? ENOENT : ENOSPC;
 	return -1;
 }
 
@@ -287,7 +287,7 @@ find_next(const struct rte_fbarray *arr, unsigned int start, bool used)
 		return MASK_GET_IDX(idx, found);
 	}
 	/* we didn't find anything */
-	rte_errno = used ? -ENOENT : -ENOSPC;
+	rte_errno = used ? ENOENT : ENOSPC;
 	return -1;
 }
 
@@ -353,6 +353,277 @@ find_contig(const struct rte_fbarray *arr, unsigned int start, bool used)
 }
 
 static int
+find_prev_n(const struct rte_fbarray *arr, unsigned int start, unsigned int n,
+		bool used)
+{
+	const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
+			arr->len);
+	unsigned int msk_idx, lookbehind_idx, first, first_mod;
+	uint64_t ignore_msk;
+
+	/*
+	 * mask only has granularity of MASK_ALIGN, but start may not be aligned
+	 * on that boundary, so construct a special mask to exclude anything we
+	 * don't want to see to avoid confusing ctz.
+	 */
+	first = MASK_LEN_TO_IDX(start);
+	first_mod = MASK_LEN_TO_MOD(start);
+	/* we're going backwards, so mask must start from the top */
+	ignore_msk = first_mod == MASK_ALIGN - 1 ?
+				-1ULL : /* prevent overflow */
+				~(-1ULL << (first_mod + 1));
+
+	/* go backwards, include zero */
+	msk_idx = first;
+	do {
+		uint64_t cur_msk, lookbehind_msk;
+		unsigned int run_start, run_end, ctz, left;
+		bool found = false;
+		/*
+		 * The process of getting n consecutive bits from the top for
+		 * arbitrary n is a bit involved, but here it is in a nutshell:
+		 *
+		 *  1. let n be the number of consecutive bits we're looking for
+		 *  2. check if n can fit in one mask, and if so, do n-1
+		 *     lshift-ands to see if there is an appropriate run inside
+		 *     our current mask
+		 *    2a. if we found a run, bail out early
+		 *    2b. if we didn't find a run, proceed
+		 *  3. invert the mask and count trailing zeroes (that is, count
+		 *     how many consecutive set bits we had starting from the
+		 *     start of current mask) as k
+		 *    3a. if k is 0, continue to next mask
+		 *    3b. if k is not 0, we have a potential run
+		 *  4. to satisfy our requirements, next mask must have n-k
+		 *     consecutive set bits at the end, so we will do (n-k-1)
+		 *     lshift-ands and check if last bit is set.
+		 *
+		 * Step 4 will need to be repeated if (n-k) > MASK_ALIGN until
+		 * we either run out of masks, lose the run, or find what we
+		 * were looking for.
+		 */
+		cur_msk = msk->data[msk_idx];
+		left = n;
+
+		/* if we're looking for free spaces, invert the mask */
+		if (!used)
+			cur_msk = ~cur_msk;
+
+		/* if we have an ignore mask, ignore once */
+		if (ignore_msk) {
+			cur_msk &= ignore_msk;
+			ignore_msk = 0;
+		}
+
+		/* if n can fit in within a single mask, do a search */
+		if (n <= MASK_ALIGN) {
+			uint64_t tmp_msk = cur_msk;
+			unsigned int s_idx;
+			for (s_idx = 0; s_idx < n - 1; s_idx++)
+				tmp_msk &= tmp_msk << 1ULL;
+			/* we found what we were looking for */
+			if (tmp_msk != 0) {
+				/* clz will give us offset from end of mask, and
+				 * we only get the end of our run, not start,
+				 * so adjust result to point to where start
+				 * would have been.
+				 */
+				run_start = MASK_ALIGN -
+						__builtin_clzll(tmp_msk) - n;
+				return MASK_GET_IDX(msk_idx, run_start);
+			}
+		}
+
+		/*
+		 * we didn't find our run within the mask, or n > MASK_ALIGN,
+		 * so we're going for plan B.
+		 */
+
+		/* count trailing zeroes on inverted mask */
+		if (~cur_msk == 0)
+			ctz = sizeof(cur_msk) * 8;
+		else
+			ctz = __builtin_ctzll(~cur_msk);
+
+		/* if there aren't any runs at the start either, just
+		 * continue
+		 */
+		if (ctz == 0)
+			continue;
+
+		/* we have a partial run at the start, so try looking behind */
+		run_end = MASK_GET_IDX(msk_idx, ctz);
+		left -= ctz;
+
+		/* go backwards, include zero */
+		lookbehind_idx = msk_idx - 1;
+
+		/* we can't lookbehind as we've run out of masks, so stop */
+		if (msk_idx == 0)
+			break;
+
+		do {
+			const uint64_t last_bit = 1ULL << (MASK_ALIGN - 1);
+			unsigned int s_idx, need;
+
+			lookbehind_msk = msk->data[lookbehind_idx];
+
+			/* if we're looking for free space, invert the mask */
+			if (!used)
+				lookbehind_msk = ~lookbehind_msk;
+
+			/* figure out how many consecutive bits we need here */
+			need = RTE_MIN(left, MASK_ALIGN);
+
+			for (s_idx = 0; s_idx < need - 1; s_idx++)
+				lookbehind_msk &= lookbehind_msk << 1ULL;
+
+			/* if last bit is not set, we've lost the run */
+			if ((lookbehind_msk & last_bit) == 0) {
+				/*
+				 * we've scanned this far, so we know there are
+				 * no runs in the space we've lookbehind-scanned
+				 * as well, so skip that on next iteration.
+				 */
+				ignore_msk = -1ULL << need;
+				msk_idx = lookbehind_idx;
+				break;
+			}
+
+			left -= need;
+
+			/* check if we've found what we were looking for */
+			if (left == 0) {
+				found = true;
+				break;
+			}
+		} while ((lookbehind_idx--) != 0); /* decrement after check to
+						    * include zero
+						    */
+
+		/* we didn't find anything, so continue */
+		if (!found)
+			continue;
+
+		/* we've found what we were looking for, but we only know where
+		 * the run ended, so calculate start position.
+		 */
+		return run_end - n;
+	} while (msk_idx-- != 0); /* decrement after check to include zero */
+	/* we didn't find anything */
+	rte_errno = used ? ENOENT : ENOSPC;
+	return -1;
+}
+
+static int
+find_prev(const struct rte_fbarray *arr, unsigned int start, bool used)
+{
+	const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
+			arr->len);
+	unsigned int idx, first, first_mod;
+	uint64_t ignore_msk;
+
+	/*
+	 * mask only has granularity of MASK_ALIGN, but start may not be aligned
+	 * on that boundary, so construct a special mask to exclude anything we
+	 * don't want to see to avoid confusing clz.
+	 */
+	first = MASK_LEN_TO_IDX(start);
+	first_mod = MASK_LEN_TO_MOD(start);
+	/* we're going backwards, so mask must start from the top */
+	ignore_msk = first_mod == MASK_ALIGN - 1 ?
+				-1ULL : /* prevent overflow */
+				~(-1ULL << (first_mod + 1));
+
+	/* go backwards, include zero */
+	idx = first;
+	do {
+		uint64_t cur = msk->data[idx];
+		int found;
+
+		/* if we're looking for free entries, invert mask */
+		if (!used)
+			cur = ~cur;
+
+		/* ignore everything before start on first iteration */
+		if (idx == first)
+			cur &= ignore_msk;
+
+		/* check if we have any entries */
+		if (cur == 0)
+			continue;
+
+		/*
+		 * find last set bit - that will correspond to whatever it is
+		 * that we're looking for. we're counting trailing zeroes, thus
+		 * the value we get is counted from end of mask, so calculate
+		 * position from start of mask.
+		 */
+		found = MASK_ALIGN - __builtin_clzll(cur) - 1;
+
+		return MASK_GET_IDX(idx, found);
+	} while (idx-- != 0); /* decrement after check  to include zero*/
+
+	/* we didn't find anything */
+	rte_errno = used ? ENOENT : ENOSPC;
+	return -1;
+}
+
+static int
+find_rev_contig(const struct rte_fbarray *arr, unsigned int start, bool used)
+{
+	const struct used_mask *msk = get_used_mask(arr->data, arr->elt_sz,
+			arr->len);
+	unsigned int idx, first, first_mod;
+	unsigned int need_len, result = 0;
+
+	first = MASK_LEN_TO_IDX(start);
+	first_mod = MASK_LEN_TO_MOD(start);
+
+	/* go backwards, include zero */
+	idx = first;
+	do {
+		uint64_t cur = msk->data[idx];
+		unsigned int run_len;
+
+		need_len = MASK_ALIGN;
+
+		/* if we're looking for free entries, invert mask */
+		if (!used)
+			cur = ~cur;
+
+		/* ignore everything after start on first iteration */
+		if (idx == first) {
+			unsigned int end_len = MASK_ALIGN - first_mod - 1;
+			cur <<= end_len;
+			/* at the start, we don't need the full mask len */
+			need_len -= end_len;
+		}
+
+		/* we will be looking for zeroes, so invert the mask */
+		cur = ~cur;
+
+		/* if mask is zero, we have a complete run */
+		if (cur == 0)
+			goto endloop;
+
+		/*
+		 * see where run ends, starting from the end.
+		 */
+		run_len = __builtin_clzll(cur);
+
+		/* add however many zeroes we've had in the last run and quit */
+		if (run_len < need_len) {
+			result += run_len;
+			break;
+		}
+endloop:
+		result += need_len;
+	} while (idx-- != 0); /* decrement after check to include zero */
+	return result;
+}
+
+static int
 set_used(struct rte_fbarray *arr, unsigned int idx, bool used)
 {
 	struct used_mask *msk;
@@ -434,39 +705,52 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,
 	if (data == NULL)
 		goto fail;
 
-	eal_get_fbarray_path(path, sizeof(path), name);
+	if (internal_config.no_shconf) {
+		/* remap virtual area as writable */
+		void *new_data = mmap(data, mmap_len, PROT_READ | PROT_WRITE,
+				MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+		if (new_data == MAP_FAILED) {
+			RTE_LOG(DEBUG, EAL, "%s(): couldn't remap anonymous memory: %s\n",
+					__func__, strerror(errno));
+			goto fail;
+		}
+	} else {
+		eal_get_fbarray_path(path, sizeof(path), name);
 
-	/*
-	 * Each fbarray is unique to process namespace, i.e. the filename
-	 * depends on process prefix. Try to take out a lock and see if we
-	 * succeed. If we don't, someone else is using it already.
-	 */
-	fd = open(path, O_CREAT | O_RDWR, 0600);
-	if (fd < 0) {
-		RTE_LOG(DEBUG, EAL, "%s(): couldn't open %s: %s\n", __func__,
-				path, strerror(errno));
-		rte_errno = errno;
-		goto fail;
-	} else if (flock(fd, LOCK_EX | LOCK_NB)) {
-		RTE_LOG(DEBUG, EAL, "%s(): couldn't lock %s: %s\n", __func__,
-				path, strerror(errno));
-		rte_errno = EBUSY;
-		goto fail;
-	}
+		/*
+		 * Each fbarray is unique to process namespace, i.e. the
+		 * filename depends on process prefix. Try to take out a lock
+		 * and see if we succeed. If we don't, someone else is using it
+		 * already.
+		 */
+		fd = open(path, O_CREAT | O_RDWR, 0600);
+		if (fd < 0) {
+			RTE_LOG(DEBUG, EAL, "%s(): couldn't open %s: %s\n",
+					__func__, path, strerror(errno));
+			rte_errno = errno;
+			goto fail;
+		} else if (flock(fd, LOCK_EX | LOCK_NB)) {
+			RTE_LOG(DEBUG, EAL, "%s(): couldn't lock %s: %s\n",
+					__func__, path, strerror(errno));
+			rte_errno = EBUSY;
+			goto fail;
+		}
 
-	/* take out a non-exclusive lock, so that other processes could still
-	 * attach to it, but no other process could reinitialize it.
-	 */
-	if (flock(fd, LOCK_SH | LOCK_NB)) {
-		rte_errno = errno;
-		goto fail;
-	}
+		/* take out a non-exclusive lock, so that other processes could
+		 * still attach to it, but no other process could reinitialize
+		 * it.
+		 */
+		if (flock(fd, LOCK_SH | LOCK_NB)) {
+			rte_errno = errno;
+			goto fail;
+		}
 
-	if (resize_and_map(fd, data, mmap_len))
-		goto fail;
+		if (resize_and_map(fd, data, mmap_len))
+			goto fail;
 
-	/* we've mmap'ed the file, we can now close the fd */
-	close(fd);
+		/* we've mmap'ed the file, we can now close the fd */
+		close(fd);
+	}
 
 	/* initialize the data */
 	memset(data, 0, mmap_len);
@@ -675,8 +959,8 @@ rte_fbarray_is_used(struct rte_fbarray *arr, unsigned int idx)
 	return ret;
 }
 
-int __rte_experimental
-rte_fbarray_find_next_free(struct rte_fbarray *arr, unsigned int start)
+static int
+fbarray_find(struct rte_fbarray *arr, unsigned int start, bool next, bool used)
 {
 	int ret = -1;
 
@@ -688,36 +972,106 @@ rte_fbarray_find_next_free(struct rte_fbarray *arr, unsigned int start)
 	/* prevent array from changing under us */
 	rte_rwlock_read_lock(&arr->rwlock);
 
-	if (arr->len == arr->count) {
-		rte_errno = ENOSPC;
-		goto out;
+	/* cheap checks to prevent doing useless work */
+	if (!used) {
+		if (arr->len == arr->count) {
+			rte_errno = ENOSPC;
+			goto out;
+		}
+		if (arr->count == 0) {
+			ret = start;
+			goto out;
+		}
+	} else {
+		if (arr->count == 0) {
+			rte_errno = ENOENT;
+			goto out;
+		}
+		if (arr->len == arr->count) {
+			ret = start;
+			goto out;
+		}
 	}
-
-	ret = find_next(arr, start, false);
+	if (next)
+		ret = find_next(arr, start, used);
+	else
+		ret = find_prev(arr, start, used);
 out:
 	rte_rwlock_read_unlock(&arr->rwlock);
 	return ret;
 }
 
 int __rte_experimental
+rte_fbarray_find_next_free(struct rte_fbarray *arr, unsigned int start)
+{
+	return fbarray_find(arr, start, true, false);
+}
+
+int __rte_experimental
 rte_fbarray_find_next_used(struct rte_fbarray *arr, unsigned int start)
 {
+	return fbarray_find(arr, start, true, true);
+}
+
+int __rte_experimental
+rte_fbarray_find_prev_free(struct rte_fbarray *arr, unsigned int start)
+{
+	return fbarray_find(arr, start, false, false);
+}
+
+int __rte_experimental
+rte_fbarray_find_prev_used(struct rte_fbarray *arr, unsigned int start)
+{
+	return fbarray_find(arr, start, false, true);
+}
+
+static int
+fbarray_find_n(struct rte_fbarray *arr, unsigned int start, unsigned int n,
+		bool next, bool used)
+{
 	int ret = -1;
 
-	if (arr == NULL || start >= arr->len) {
+	if (arr == NULL || start >= arr->len || n > arr->len || n == 0) {
 		rte_errno = EINVAL;
 		return -1;
 	}
+	if (next && (arr->len - start) < n) {
+		rte_errno = used ? ENOENT : ENOSPC;
+		return -1;
+	}
+	if (!next && start < (n - 1)) {
+		rte_errno = used ? ENOENT : ENOSPC;
+		return -1;
+	}
 
 	/* prevent array from changing under us */
 	rte_rwlock_read_lock(&arr->rwlock);
 
-	if (arr->count == 0) {
-		rte_errno = ENOENT;
-		goto out;
+	/* cheap checks to prevent doing useless work */
+	if (!used) {
+		if (arr->len == arr->count || arr->len - arr->count < n) {
+			rte_errno = ENOSPC;
+			goto out;
+		}
+		if (arr->count == 0) {
+			ret = next ? start : start - n + 1;
+			goto out;
+		}
+	} else {
+		if (arr->count < n) {
+			rte_errno = ENOENT;
+			goto out;
+		}
+		if (arr->count == arr->len) {
+			ret = next ? start : start - n + 1;
+			goto out;
+		}
 	}
 
-	ret = find_next(arr, start, true);
+	if (next)
+		ret = find_next_n(arr, start, n, used);
+	else
+		ret = find_prev_n(arr, start, n, used);
 out:
 	rte_rwlock_read_unlock(&arr->rwlock);
 	return ret;
@@ -727,54 +1081,33 @@ int __rte_experimental
 rte_fbarray_find_next_n_free(struct rte_fbarray *arr, unsigned int start,
 		unsigned int n)
 {
-	int ret = -1;
-
-	if (arr == NULL || start >= arr->len || n > arr->len) {
-		rte_errno = EINVAL;
-		return -1;
-	}
-
-	/* prevent array from changing under us */
-	rte_rwlock_read_lock(&arr->rwlock);
-
-	if (arr->len == arr->count || arr->len - arr->count < n) {
-		rte_errno = ENOSPC;
-		goto out;
-	}
-
-	ret = find_next_n(arr, start, n, false);
-out:
-	rte_rwlock_read_unlock(&arr->rwlock);
-	return ret;
+	return fbarray_find_n(arr, start, n, true, false);
 }
 
 int __rte_experimental
 rte_fbarray_find_next_n_used(struct rte_fbarray *arr, unsigned int start,
 		unsigned int n)
 {
-	int ret = -1;
-
-	if (arr == NULL || start >= arr->len || n > arr->len) {
-		rte_errno = EINVAL;
-		return -1;
-	}
-
-	/* prevent array from changing under us */
-	rte_rwlock_read_lock(&arr->rwlock);
-
-	if (arr->count < n) {
-		rte_errno = ENOENT;
-		goto out;
-	}
+	return fbarray_find_n(arr, start, n, true, true);
+}
 
-	ret = find_next_n(arr, start, n, true);
-out:
-	rte_rwlock_read_unlock(&arr->rwlock);
-	return ret;
+int __rte_experimental
+rte_fbarray_find_prev_n_free(struct rte_fbarray *arr, unsigned int start,
+		unsigned int n)
+{
+	return fbarray_find_n(arr, start, n, false, false);
 }
 
 int __rte_experimental
-rte_fbarray_find_contig_free(struct rte_fbarray *arr, unsigned int start)
+rte_fbarray_find_prev_n_used(struct rte_fbarray *arr, unsigned int start,
+		unsigned int n)
+{
+	return fbarray_find_n(arr, start, n, false, true);
+}
+
+static int
+fbarray_find_contig(struct rte_fbarray *arr, unsigned int start, bool next,
+		bool used)
 {
 	int ret = -1;
 
@@ -786,39 +1119,66 @@ rte_fbarray_find_contig_free(struct rte_fbarray *arr, unsigned int start)
 	/* prevent array from changing under us */
 	rte_rwlock_read_lock(&arr->rwlock);
 
-	if (arr->len == arr->count) {
-		rte_errno = ENOSPC;
-		goto out;
-	}
-
-	if (arr->count == 0) {
-		ret = arr->len - start;
-		goto out;
+	/* cheap checks to prevent doing useless work */
+	if (used) {
+		if (arr->count == 0) {
+			ret = 0;
+			goto out;
+		}
+		if (next && arr->count == arr->len) {
+			ret = arr->len - start;
+			goto out;
+		}
+		if (!next && arr->count == arr->len) {
+			ret = start + 1;
+			goto out;
+		}
+	} else {
+		if (arr->len == arr->count) {
+			ret = 0;
+			goto out;
+		}
+		if (next && arr->count == 0) {
+			ret = arr->len - start;
+			goto out;
+		}
+		if (!next && arr->count == 0) {
+			ret = start + 1;
+			goto out;
+		}
 	}
 
-	ret = find_contig(arr, start, false);
+	if (next)
+		ret = find_contig(arr, start, used);
+	else
+		ret = find_rev_contig(arr, start, used);
 out:
 	rte_rwlock_read_unlock(&arr->rwlock);
 	return ret;
 }
 
 int __rte_experimental
-rte_fbarray_find_contig_used(struct rte_fbarray *arr, unsigned int start)
+rte_fbarray_find_contig_free(struct rte_fbarray *arr, unsigned int start)
 {
-	int ret = -1;
-
-	if (arr == NULL || start >= arr->len) {
-		rte_errno = EINVAL;
-		return -1;
-	}
+	return fbarray_find_contig(arr, start, true, false);
+}
 
-	/* prevent array from changing under us */
-	rte_rwlock_read_lock(&arr->rwlock);
+int __rte_experimental
+rte_fbarray_find_contig_used(struct rte_fbarray *arr, unsigned int start)
+{
+	return fbarray_find_contig(arr, start, true, true);
+}
 
-	ret = find_contig(arr, start, true);
+int __rte_experimental
+rte_fbarray_find_rev_contig_free(struct rte_fbarray *arr, unsigned int start)
+{
+	return fbarray_find_contig(arr, start, false, false);
+}
 
-	rte_rwlock_read_unlock(&arr->rwlock);
-	return ret;
+int __rte_experimental
+rte_fbarray_find_rev_contig_used(struct rte_fbarray *arr, unsigned int start)
+{
+	return fbarray_find_contig(arr, start, false, true);
 }
 
 int __rte_experimental
diff --git a/lib/librte_eal/common/eal_common_log.c b/lib/librte_eal/common/eal_common_log.c
index 81811894..c714a4bd 100644
--- a/lib/librte_eal/common/eal_common_log.c
+++ b/lib/librte_eal/common/eal_common_log.c
@@ -335,9 +335,7 @@ static const struct logtype logtype_strings[] = {
 };
 
 /* Logging should be first initializer (before drivers and bus) */
-RTE_INIT_PRIO(rte_log_init, LOG);
-static void
-rte_log_init(void)
+RTE_INIT_PRIO(rte_log_init, LOG)
 {
 	uint32_t i;
 
diff --git a/lib/librte_eal/common/eal_common_memory.c b/lib/librte_eal/common/eal_common_memory.c
index 4f0688f9..fbfb1b05 100644
--- a/lib/librte_eal/common/eal_common_memory.c
+++ b/lib/librte_eal/common/eal_common_memory.c
@@ -34,7 +34,7 @@
 
 #define MEMSEG_LIST_FMT "memseg-%" PRIu64 "k-%i-%i"
 
-static uint64_t baseaddr_offset;
+static void *next_baseaddr;
 static uint64_t system_page_sz;
 
 void *
@@ -56,21 +56,27 @@ eal_get_virtual_area(void *requested_addr, size_t *size,
 	allow_shrink = (flags & EAL_VIRTUAL_AREA_ALLOW_SHRINK) > 0;
 	unmap = (flags & EAL_VIRTUAL_AREA_UNMAP) > 0;
 
-	if (requested_addr == NULL && internal_config.base_virtaddr != 0) {
-		requested_addr = (void *) (internal_config.base_virtaddr +
-				(size_t)baseaddr_offset);
+	if (next_baseaddr == NULL && internal_config.base_virtaddr != 0 &&
+			rte_eal_process_type() == RTE_PROC_PRIMARY)
+		next_baseaddr = (void *) internal_config.base_virtaddr;
+
+	if (requested_addr == NULL && next_baseaddr != NULL) {
+		requested_addr = next_baseaddr;
 		requested_addr = RTE_PTR_ALIGN(requested_addr, page_sz);
 		addr_is_hint = true;
 	}
 
-	/* if requested address is not aligned by page size, or if requested
-	 * address is NULL, add page size to requested length as we may get an
-	 * address that's aligned by system page size, which can be smaller than
-	 * our requested page size. additionally, we shouldn't try to align if
-	 * system page size is the same as requested page size.
+	/* we don't need alignment of resulting pointer in the following cases:
+	 *
+	 * 1. page size is equal to system size
+	 * 2. we have a requested address, and it is page-aligned, and we will
+	 *    be discarding the address if we get a different one.
+	 *
+	 * for all other cases, alignment is potentially necessary.
 	 */
 	no_align = (requested_addr != NULL &&
-		((uintptr_t)requested_addr & (page_sz - 1)) == 0) ||
+		requested_addr == RTE_PTR_ALIGN(requested_addr, page_sz) &&
+		!addr_is_hint) ||
 		page_sz == system_page_sz;
 
 	do {
@@ -116,6 +122,8 @@ eal_get_virtual_area(void *requested_addr, size_t *size,
 		RTE_LOG(WARNING, EAL, "WARNING! Base virtual address hint (%p != %p) not respected!\n",
 			requested_addr, aligned_addr);
 		RTE_LOG(WARNING, EAL, "   This may cause issues with mapping memory into secondary processes\n");
+	} else if (next_baseaddr != NULL) {
+		next_baseaddr = RTE_PTR_ADD(aligned_addr, *size);
 	}
 
 	RTE_LOG(DEBUG, EAL, "Virtual area found at %p (size = 0x%zx)\n",
@@ -148,387 +156,9 @@ eal_get_virtual_area(void *requested_addr, size_t *size,
 			munmap(aligned_end, after_len);
 	}
 
-	baseaddr_offset += *size;
-
 	return aligned_addr;
 }
 
-static uint64_t
-get_mem_amount(uint64_t page_sz, uint64_t max_mem)
-{
-	uint64_t area_sz, max_pages;
-
-	/* limit to RTE_MAX_MEMSEG_PER_LIST pages or RTE_MAX_MEM_MB_PER_LIST */
-	max_pages = RTE_MAX_MEMSEG_PER_LIST;
-	max_mem = RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_LIST << 20, max_mem);
-
-	area_sz = RTE_MIN(page_sz * max_pages, max_mem);
-
-	/* make sure the list isn't smaller than the page size */
-	area_sz = RTE_MAX(area_sz, page_sz);
-
-	return RTE_ALIGN(area_sz, page_sz);
-}
-
-static int
-free_memseg_list(struct rte_memseg_list *msl)
-{
-	if (rte_fbarray_destroy(&msl->memseg_arr)) {
-		RTE_LOG(ERR, EAL, "Cannot destroy memseg list\n");
-		return -1;
-	}
-	memset(msl, 0, sizeof(*msl));
-	return 0;
-}
-
-static int
-alloc_memseg_list(struct rte_memseg_list *msl, uint64_t page_sz,
-		uint64_t max_mem, int socket_id, int type_msl_idx)
-{
-	char name[RTE_FBARRAY_NAME_LEN];
-	uint64_t mem_amount;
-	int max_segs;
-
-	mem_amount = get_mem_amount(page_sz, max_mem);
-	max_segs = mem_amount / page_sz;
-
-	snprintf(name, sizeof(name), MEMSEG_LIST_FMT, page_sz >> 10, socket_id,
-		 type_msl_idx);
-	if (rte_fbarray_init(&msl->memseg_arr, name, max_segs,
-			sizeof(struct rte_memseg))) {
-		RTE_LOG(ERR, EAL, "Cannot allocate memseg list: %s\n",
-			rte_strerror(rte_errno));
-		return -1;
-	}
-
-	msl->page_sz = page_sz;
-	msl->socket_id = socket_id;
-	msl->base_va = NULL;
-
-	RTE_LOG(DEBUG, EAL, "Memseg list allocated: 0x%zxkB at socket %i\n",
-			(size_t)page_sz >> 10, socket_id);
-
-	return 0;
-}
-
-static int
-alloc_va_space(struct rte_memseg_list *msl)
-{
-	uint64_t page_sz;
-	size_t mem_sz;
-	void *addr;
-	int flags = 0;
-
-#ifdef RTE_ARCH_PPC_64
-	flags |= MAP_HUGETLB;
-#endif
-
-	page_sz = msl->page_sz;
-	mem_sz = page_sz * msl->memseg_arr.len;
-
-	addr = eal_get_virtual_area(msl->base_va, &mem_sz, page_sz, 0, flags);
-	if (addr == NULL) {
-		if (rte_errno == EADDRNOTAVAIL)
-			RTE_LOG(ERR, EAL, "Could not mmap %llu bytes at [%p] - please use '--base-virtaddr' option\n",
-				(unsigned long long)mem_sz, msl->base_va);
-		else
-			RTE_LOG(ERR, EAL, "Cannot reserve memory\n");
-		return -1;
-	}
-	msl->base_va = addr;
-
-	return 0;
-}
-
-static int __rte_unused
-memseg_primary_init_32(void)
-{
-	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	int active_sockets, hpi_idx, msl_idx = 0;
-	unsigned int socket_id, i;
-	struct rte_memseg_list *msl;
-	uint64_t extra_mem_per_socket, total_extra_mem, total_requested_mem;
-	uint64_t max_mem;
-
-	/* no-huge does not need this at all */
-	if (internal_config.no_hugetlbfs)
-		return 0;
-
-	/* this is a giant hack, but desperate times call for desperate
-	 * measures. in legacy 32-bit mode, we cannot preallocate VA space,
-	 * because having upwards of 2 gigabytes of VA space already mapped will
-	 * interfere with our ability to map and sort hugepages.
-	 *
-	 * therefore, in legacy 32-bit mode, we will be initializing memseg
-	 * lists much later - in eal_memory.c, right after we unmap all the
-	 * unneeded pages. this will not affect secondary processes, as those
-	 * should be able to mmap the space without (too many) problems.
-	 */
-	if (internal_config.legacy_mem)
-		return 0;
-
-	/* 32-bit mode is a very special case. we cannot know in advance where
-	 * the user will want to allocate their memory, so we have to do some
-	 * heuristics.
-	 */
-	active_sockets = 0;
-	total_requested_mem = 0;
-	if (internal_config.force_sockets)
-		for (i = 0; i < rte_socket_count(); i++) {
-			uint64_t mem;
-
-			socket_id = rte_socket_id_by_idx(i);
-			mem = internal_config.socket_mem[socket_id];
-
-			if (mem == 0)
-				continue;
-
-			active_sockets++;
-			total_requested_mem += mem;
-		}
-	else
-		total_requested_mem = internal_config.memory;
-
-	max_mem = (uint64_t)RTE_MAX_MEM_MB << 20;
-	if (total_requested_mem > max_mem) {
-		RTE_LOG(ERR, EAL, "Invalid parameters: 32-bit process can at most use %uM of memory\n",
-				(unsigned int)(max_mem >> 20));
-		return -1;
-	}
-	total_extra_mem = max_mem - total_requested_mem;
-	extra_mem_per_socket = active_sockets == 0 ? total_extra_mem :
-			total_extra_mem / active_sockets;
-
-	/* the allocation logic is a little bit convoluted, but here's how it
-	 * works, in a nutshell:
-	 *  - if user hasn't specified on which sockets to allocate memory via
-	 *    --socket-mem, we allocate all of our memory on master core socket.
-	 *  - if user has specified sockets to allocate memory on, there may be
-	 *    some "unused" memory left (e.g. if user has specified --socket-mem
-	 *    such that not all memory adds up to 2 gigabytes), so add it to all
-	 *    sockets that are in use equally.
-	 *
-	 * page sizes are sorted by size in descending order, so we can safely
-	 * assume that we dispense with bigger page sizes first.
-	 */
-
-	/* create memseg lists */
-	for (i = 0; i < rte_socket_count(); i++) {
-		int hp_sizes = (int) internal_config.num_hugepage_sizes;
-		uint64_t max_socket_mem, cur_socket_mem;
-		unsigned int master_lcore_socket;
-		struct rte_config *cfg = rte_eal_get_configuration();
-		bool skip;
-
-		socket_id = rte_socket_id_by_idx(i);
-
-#ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
-		if (socket_id > 0)
-			break;
-#endif
-
-		/* if we didn't specifically request memory on this socket */
-		skip = active_sockets != 0 &&
-				internal_config.socket_mem[socket_id] == 0;
-		/* ...or if we didn't specifically request memory on *any*
-		 * socket, and this is not master lcore
-		 */
-		master_lcore_socket = rte_lcore_to_socket_id(cfg->master_lcore);
-		skip |= active_sockets == 0 && socket_id != master_lcore_socket;
-
-		if (skip) {
-			RTE_LOG(DEBUG, EAL, "Will not preallocate memory on socket %u\n",
-					socket_id);
-			continue;
-		}
-
-		/* max amount of memory on this socket */
-		max_socket_mem = (active_sockets != 0 ?
-					internal_config.socket_mem[socket_id] :
-					internal_config.memory) +
-					extra_mem_per_socket;
-		cur_socket_mem = 0;
-
-		for (hpi_idx = 0; hpi_idx < hp_sizes; hpi_idx++) {
-			uint64_t max_pagesz_mem, cur_pagesz_mem = 0;
-			uint64_t hugepage_sz;
-			struct hugepage_info *hpi;
-			int type_msl_idx, max_segs, total_segs = 0;
-
-			hpi = &internal_config.hugepage_info[hpi_idx];
-			hugepage_sz = hpi->hugepage_sz;
-
-			/* check if pages are actually available */
-			if (hpi->num_pages[socket_id] == 0)
-				continue;
-
-			max_segs = RTE_MAX_MEMSEG_PER_TYPE;
-			max_pagesz_mem = max_socket_mem - cur_socket_mem;
-
-			/* make it multiple of page size */
-			max_pagesz_mem = RTE_ALIGN_FLOOR(max_pagesz_mem,
-					hugepage_sz);
-
-			RTE_LOG(DEBUG, EAL, "Attempting to preallocate "
-					"%" PRIu64 "M on socket %i\n",
-					max_pagesz_mem >> 20, socket_id);
-
-			type_msl_idx = 0;
-			while (cur_pagesz_mem < max_pagesz_mem &&
-					total_segs < max_segs) {
-				if (msl_idx >= RTE_MAX_MEMSEG_LISTS) {
-					RTE_LOG(ERR, EAL,
-						"No more space in memseg lists, please increase %s\n",
-						RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS));
-					return -1;
-				}
-
-				msl = &mcfg->memsegs[msl_idx];
-
-				if (alloc_memseg_list(msl, hugepage_sz,
-						max_pagesz_mem, socket_id,
-						type_msl_idx)) {
-					/* failing to allocate a memseg list is
-					 * a serious error.
-					 */
-					RTE_LOG(ERR, EAL, "Cannot allocate memseg list\n");
-					return -1;
-				}
-
-				if (alloc_va_space(msl)) {
-					/* if we couldn't allocate VA space, we
-					 * can try with smaller page sizes.
-					 */
-					RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list, retrying with different page size\n");
-					/* deallocate memseg list */
-					if (free_memseg_list(msl))
-						return -1;
-					break;
-				}
-
-				total_segs += msl->memseg_arr.len;
-				cur_pagesz_mem = total_segs * hugepage_sz;
-				type_msl_idx++;
-				msl_idx++;
-			}
-			cur_socket_mem += cur_pagesz_mem;
-		}
-		if (cur_socket_mem == 0) {
-			RTE_LOG(ERR, EAL, "Cannot allocate VA space on socket %u\n",
-				socket_id);
-			return -1;
-		}
-	}
-
-	return 0;
-}
-
-static int __rte_unused
-memseg_primary_init(void)
-{
-	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	int i, socket_id, hpi_idx, msl_idx = 0;
-	struct rte_memseg_list *msl;
-	uint64_t max_mem, total_mem;
-
-	/* no-huge does not need this at all */
-	if (internal_config.no_hugetlbfs)
-		return 0;
-
-	max_mem = (uint64_t)RTE_MAX_MEM_MB << 20;
-	total_mem = 0;
-
-	/* create memseg lists */
-	for (hpi_idx = 0; hpi_idx < (int) internal_config.num_hugepage_sizes;
-			hpi_idx++) {
-		struct hugepage_info *hpi;
-		uint64_t hugepage_sz;
-
-		hpi = &internal_config.hugepage_info[hpi_idx];
-		hugepage_sz = hpi->hugepage_sz;
-
-		for (i = 0; i < (int) rte_socket_count(); i++) {
-			uint64_t max_type_mem, total_type_mem = 0;
-			int type_msl_idx, max_segs, total_segs = 0;
-
-			socket_id = rte_socket_id_by_idx(i);
-
-#ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
-			if (socket_id > 0)
-				break;
-#endif
-
-			if (total_mem >= max_mem)
-				break;
-
-			max_type_mem = RTE_MIN(max_mem - total_mem,
-				(uint64_t)RTE_MAX_MEM_MB_PER_TYPE << 20);
-			max_segs = RTE_MAX_MEMSEG_PER_TYPE;
-
-			type_msl_idx = 0;
-			while (total_type_mem < max_type_mem &&
-					total_segs < max_segs) {
-				uint64_t cur_max_mem;
-				if (msl_idx >= RTE_MAX_MEMSEG_LISTS) {
-					RTE_LOG(ERR, EAL,
-						"No more space in memseg lists, please increase %s\n",
-						RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS));
-					return -1;
-				}
-
-				msl = &mcfg->memsegs[msl_idx++];
-
-				cur_max_mem = max_type_mem - total_type_mem;
-				if (alloc_memseg_list(msl, hugepage_sz,
-						cur_max_mem, socket_id,
-						type_msl_idx))
-					return -1;
-
-				total_segs += msl->memseg_arr.len;
-				total_type_mem = total_segs * hugepage_sz;
-				type_msl_idx++;
-
-				if (alloc_va_space(msl)) {
-					RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list\n");
-					return -1;
-				}
-			}
-			total_mem += total_type_mem;
-		}
-	}
-	return 0;
-}
-
-static int
-memseg_secondary_init(void)
-{
-	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	int msl_idx = 0;
-	struct rte_memseg_list *msl;
-
-	for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS; msl_idx++) {
-
-		msl = &mcfg->memsegs[msl_idx];
-
-		/* skip empty memseg lists */
-		if (msl->memseg_arr.len == 0)
-			continue;
-
-		if (rte_fbarray_attach(&msl->memseg_arr)) {
-			RTE_LOG(ERR, EAL, "Cannot attach to primary process memseg lists\n");
-			return -1;
-		}
-
-		/* preallocate VA space */
-		if (alloc_va_space(msl)) {
-			RTE_LOG(ERR, EAL, "Cannot preallocate VA space for hugepage memory\n");
-			return -1;
-		}
-	}
-
-	return 0;
-}
-
 static struct rte_memseg *
 virt2memseg(const void *addr, const struct rte_memseg_list *msl)
 {
@@ -536,6 +166,9 @@ virt2memseg(const void *addr, const struct rte_memseg_list *msl)
 	void *start, *end;
 	int ms_idx;
 
+	if (msl == NULL)
+		return NULL;
+
 	/* a memseg list was specified, check if it's the right one */
 	start = msl->base_va;
 	end = RTE_PTR_ADD(start, (size_t)msl->page_sz * msl->memseg_arr.len);
@@ -788,14 +421,11 @@ rte_mem_lock_page(const void *virt)
 }
 
 int __rte_experimental
-rte_memseg_contig_walk(rte_memseg_contig_walk_t func, void *arg)
+rte_memseg_contig_walk_thread_unsafe(rte_memseg_contig_walk_t func, void *arg)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 	int i, ms_idx, ret = 0;
 
-	/* do not allow allocations/frees/init while we iterate */
-	rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
-
 	for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
 		struct rte_memseg_list *msl = &mcfg->memsegs[i];
 		const struct rte_memseg *ms;
@@ -820,30 +450,34 @@ rte_memseg_contig_walk(rte_memseg_contig_walk_t func, void *arg)
 			len = n_segs * msl->page_sz;
 
 			ret = func(msl, ms, len, arg);
-			if (ret < 0) {
-				ret = -1;
-				goto out;
-			} else if (ret > 0) {
-				ret = 1;
-				goto out;
-			}
+			if (ret)
+				return ret;
 			ms_idx = rte_fbarray_find_next_used(arr,
 					ms_idx + n_segs);
 		}
 	}
-out:
-	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
-	return ret;
+	return 0;
 }
 
 int __rte_experimental
-rte_memseg_walk(rte_memseg_walk_t func, void *arg)
+rte_memseg_contig_walk(rte_memseg_contig_walk_t func, void *arg)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	int i, ms_idx, ret = 0;
+	int ret = 0;
 
 	/* do not allow allocations/frees/init while we iterate */
 	rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
+	ret = rte_memseg_contig_walk_thread_unsafe(func, arg);
+	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+
+	return ret;
+}
+
+int __rte_experimental
+rte_memseg_walk_thread_unsafe(rte_memseg_walk_t func, void *arg)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int i, ms_idx, ret = 0;
 
 	for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
 		struct rte_memseg_list *msl = &mcfg->memsegs[i];
@@ -859,29 +493,33 @@ rte_memseg_walk(rte_memseg_walk_t func, void *arg)
 		while (ms_idx >= 0) {
 			ms = rte_fbarray_get(arr, ms_idx);
 			ret = func(msl, ms, arg);
-			if (ret < 0) {
-				ret = -1;
-				goto out;
-			} else if (ret > 0) {
-				ret = 1;
-				goto out;
-			}
+			if (ret)
+				return ret;
 			ms_idx = rte_fbarray_find_next_used(arr, ms_idx + 1);
 		}
 	}
-out:
-	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
-	return ret;
+	return 0;
 }
 
 int __rte_experimental
-rte_memseg_list_walk(rte_memseg_list_walk_t func, void *arg)
+rte_memseg_walk(rte_memseg_walk_t func, void *arg)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	int i, ret = 0;
+	int ret = 0;
 
 	/* do not allow allocations/frees/init while we iterate */
 	rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
+	ret = rte_memseg_walk_thread_unsafe(func, arg);
+	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+
+	return ret;
+}
+
+int __rte_experimental
+rte_memseg_list_walk_thread_unsafe(rte_memseg_list_walk_t func, void *arg)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int i, ret = 0;
 
 	for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
 		struct rte_memseg_list *msl = &mcfg->memsegs[i];
@@ -890,17 +528,23 @@ rte_memseg_list_walk(rte_memseg_list_walk_t func, void *arg)
 			continue;
 
 		ret = func(msl, arg);
-		if (ret < 0) {
-			ret = -1;
-			goto out;
-		}
-		if (ret > 0) {
-			ret = 1;
-			goto out;
-		}
+		if (ret)
+			return ret;
 	}
-out:
+	return 0;
+}
+
+int __rte_experimental
+rte_memseg_list_walk(rte_memseg_list_walk_t func, void *arg)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int ret = 0;
+
+	/* do not allow allocations/frees/init while we iterate */
+	rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
+	ret = rte_memseg_list_walk_thread_unsafe(func, arg);
 	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+
 	return ret;
 }
 
@@ -918,15 +562,7 @@ rte_eal_memory_init(void)
 	/* lock mem hotplug here, to prevent races while we init */
 	rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
 
-	retval = rte_eal_process_type() == RTE_PROC_PRIMARY ?
-#ifndef RTE_ARCH_64
-			memseg_primary_init_32() :
-#else
-			memseg_primary_init() :
-#endif
-			memseg_secondary_init();
-
-	if (retval < 0)
+	if (rte_eal_memseg_init() < 0)
 		goto fail;
 
 	if (eal_memalloc_init() < 0)
diff --git a/lib/librte_eal/common/eal_common_memzone.c b/lib/librte_eal/common/eal_common_memzone.c
index faa3b061..7300fe05 100644
--- a/lib/librte_eal/common/eal_common_memzone.c
+++ b/lib/librte_eal/common/eal_common_memzone.c
@@ -52,38 +52,6 @@ memzone_lookup_thread_unsafe(const char *name)
 	return NULL;
 }
 
-
-/* This function will return the greatest free block if a heap has been
- * specified. If no heap has been specified, it will return the heap and
- * length of the greatest free block available in all heaps */
-static size_t
-find_heap_max_free_elem(int *s, unsigned align)
-{
-	struct rte_mem_config *mcfg;
-	struct rte_malloc_socket_stats stats;
-	int i, socket = *s;
-	size_t len = 0;
-
-	/* get pointer to global configuration */
-	mcfg = rte_eal_get_configuration()->mem_config;
-
-	for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
-		if ((socket != SOCKET_ID_ANY) && (socket != i))
-			continue;
-
-		malloc_heap_get_stats(&mcfg->malloc_heaps[i], &stats);
-		if (stats.greatest_free_size > len) {
-			len = stats.greatest_free_size;
-			*s = i;
-		}
-	}
-
-	if (len < MALLOC_ELEM_OVERHEAD + align)
-		return 0;
-
-	return len - MALLOC_ELEM_OVERHEAD - align;
-}
-
 static const struct rte_memzone *
 memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 		int socket_id, unsigned int flags, unsigned int align,
@@ -92,6 +60,7 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 	struct rte_memzone *mz;
 	struct rte_mem_config *mcfg;
 	struct rte_fbarray *arr;
+	void *mz_addr;
 	size_t requested_len;
 	int mz_idx;
 	bool contig;
@@ -140,8 +109,7 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 		return NULL;
 	}
 
-	len += RTE_CACHE_LINE_MASK;
-	len &= ~((size_t) RTE_CACHE_LINE_MASK);
+	len = RTE_ALIGN_CEIL(len, RTE_CACHE_LINE_SIZE);
 
 	/* save minimal requested  length */
 	requested_len = RTE_MAX((size_t)RTE_CACHE_LINE_SIZE,  len);
@@ -165,27 +133,18 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 	/* malloc only cares about size flags, remove contig flag from flags */
 	flags &= ~RTE_MEMZONE_IOVA_CONTIG;
 
-	if (len == 0) {
-		/* len == 0 is only allowed for non-contiguous zones */
-		if (contig) {
-			RTE_LOG(DEBUG, EAL, "Reserving zero-length contiguous memzones is not supported\n");
-			rte_errno = EINVAL;
-			return NULL;
-		}
-		if (bound != 0)
+	if (len == 0 && bound == 0) {
+		/* no size constraints were placed, so use malloc elem len */
+		requested_len = 0;
+		mz_addr = malloc_heap_alloc_biggest(NULL, socket_id, flags,
+				align, contig);
+	} else {
+		if (len == 0)
 			requested_len = bound;
-		else {
-			requested_len = find_heap_max_free_elem(&socket_id, align);
-			if (requested_len == 0) {
-				rte_errno = ENOMEM;
-				return NULL;
-			}
-		}
+		/* allocate memory on heap */
+		mz_addr = malloc_heap_alloc(NULL, requested_len, socket_id,
+				flags, align, bound, contig);
 	}
-
-	/* allocate memory on heap */
-	void *mz_addr = malloc_heap_alloc(NULL, requested_len, socket_id, flags,
-			align, bound, contig);
 	if (mz_addr == NULL) {
 		rte_errno = ENOMEM;
 		return NULL;
@@ -213,8 +172,9 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 	snprintf(mz->name, sizeof(mz->name), "%s", name);
 	mz->iova = rte_malloc_virt2iova(mz_addr);
 	mz->addr = mz_addr;
-	mz->len = (requested_len == 0 ?
-			(elem->size - MALLOC_ELEM_OVERHEAD) : requested_len);
+	mz->len = requested_len == 0 ?
+			elem->size - elem->pad - MALLOC_ELEM_OVERHEAD :
+			requested_len;
 	mz->hugepage_sz = elem->msl->page_sz;
 	mz->socket_id = elem->msl->socket_id;
 	mz->flags = 0;
diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index ecebb292..dd5f9740 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -66,10 +66,12 @@ eal_long_options[] = {
 	{OPT_NO_HUGE,           0, NULL, OPT_NO_HUGE_NUM          },
 	{OPT_NO_PCI,            0, NULL, OPT_NO_PCI_NUM           },
 	{OPT_NO_SHCONF,         0, NULL, OPT_NO_SHCONF_NUM        },
+	{OPT_IN_MEMORY,         0, NULL, OPT_IN_MEMORY_NUM        },
 	{OPT_PCI_BLACKLIST,     1, NULL, OPT_PCI_BLACKLIST_NUM    },
 	{OPT_PCI_WHITELIST,     1, NULL, OPT_PCI_WHITELIST_NUM    },
 	{OPT_PROC_TYPE,         1, NULL, OPT_PROC_TYPE_NUM        },
 	{OPT_SOCKET_MEM,        1, NULL, OPT_SOCKET_MEM_NUM       },
+	{OPT_SOCKET_LIMIT,      1, NULL, OPT_SOCKET_LIMIT_NUM     },
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
@@ -179,6 +181,10 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 	/* zero out the NUMA config */
 	for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
 		internal_cfg->socket_mem[i] = 0;
+	internal_cfg->force_socket_limits = 0;
+	/* zero out the NUMA limits config */
+	for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
+		internal_cfg->socket_limit[i] = 0;
 	/* zero out hugedir descriptors */
 	for (i = 0; i < MAX_HUGEPAGE_SIZES; i++) {
 		memset(&internal_cfg->hugepage_info[i], 0,
@@ -315,6 +321,7 @@ eal_parse_service_coremask(const char *coremask)
 	unsigned int count = 0;
 	char c;
 	int val;
+	uint32_t taken_lcore_count = 0;
 
 	if (coremask == NULL)
 		return -1;
@@ -348,7 +355,7 @@ eal_parse_service_coremask(const char *coremask)
 				if (master_lcore_parsed &&
 						cfg->master_lcore == lcore) {
 					RTE_LOG(ERR, EAL,
-						"Error: lcore %u is master lcore, cannot use as service core\n",
+						"lcore %u is master lcore, cannot use as service core\n",
 						idx);
 					return -1;
 				}
@@ -358,6 +365,10 @@ eal_parse_service_coremask(const char *coremask)
 						"lcore %u unavailable\n", idx);
 					return -1;
 				}
+
+				if (cfg->lcore_role[idx] == ROLE_RTE)
+					taken_lcore_count++;
+
 				lcore_config[idx].core_role = ROLE_SERVICE;
 				count++;
 			}
@@ -374,11 +385,28 @@ eal_parse_service_coremask(const char *coremask)
 	if (count == 0)
 		return -1;
 
+	if (core_parsed && taken_lcore_count != count) {
+		RTE_LOG(WARNING, EAL,
+			"Not all service cores are in the coremask. "
+			"Please ensure -c or -l includes service cores\n");
+	}
+
 	cfg->service_lcore_count = count;
 	return 0;
 }
 
 static int
+eal_service_cores_parsed(void)
+{
+	int idx;
+	for (idx = 0; idx < RTE_MAX_LCORE; idx++) {
+		if (lcore_config[idx].core_role == ROLE_SERVICE)
+			return 1;
+	}
+	return 0;
+}
+
+static int
 eal_parse_coremask(const char *coremask)
 {
 	struct rte_config *cfg = rte_eal_get_configuration();
@@ -387,6 +415,11 @@ eal_parse_coremask(const char *coremask)
 	char c;
 	int val;
 
+	if (eal_service_cores_parsed())
+		RTE_LOG(WARNING, EAL,
+			"Service cores parsed before dataplane cores. "
+			"Please ensure -c is before -s or -S\n");
+
 	if (coremask == NULL)
 		return -1;
 	/* Remove all blank characters ahead and after .
@@ -418,6 +451,7 @@ eal_parse_coremask(const char *coremask)
 					        "unavailable\n", idx);
 					return -1;
 				}
+
 				cfg->lcore_role[idx] = ROLE_RTE;
 				lcore_config[idx].core_index = count;
 				count++;
@@ -449,6 +483,7 @@ eal_parse_service_corelist(const char *corelist)
 	unsigned count = 0;
 	char *end = NULL;
 	int min, max;
+	uint32_t taken_lcore_count = 0;
 
 	if (corelist == NULL)
 		return -1;
@@ -490,6 +525,9 @@ eal_parse_service_corelist(const char *corelist)
 							idx);
 						return -1;
 					}
+					if (cfg->lcore_role[idx] == ROLE_RTE)
+						taken_lcore_count++;
+
 					lcore_config[idx].core_role =
 							ROLE_SERVICE;
 					count++;
@@ -504,6 +542,12 @@ eal_parse_service_corelist(const char *corelist)
 	if (count == 0)
 		return -1;
 
+	if (core_parsed && taken_lcore_count != count) {
+		RTE_LOG(WARNING, EAL,
+			"Not all service cores were in the coremask. "
+			"Please ensure -c or -l includes service cores\n");
+	}
+
 	return 0;
 }
 
@@ -516,6 +560,11 @@ eal_parse_corelist(const char *corelist)
 	char *end = NULL;
 	int min, max;
 
+	if (eal_service_cores_parsed())
+		RTE_LOG(WARNING, EAL,
+			"Service cores parsed before dataplane cores. "
+			"Please ensure -l is before -s or -S\n");
+
 	if (corelist == NULL)
 		return -1;
 
@@ -590,7 +639,8 @@ eal_parse_master_lcore(const char *arg)
 
 	/* ensure master core is not used as service core */
 	if (lcore_config[cfg->master_lcore].core_role == ROLE_SERVICE) {
-		RTE_LOG(ERR, EAL, "Error: Master lcore is used as a service core.\n");
+		RTE_LOG(ERR, EAL,
+			"Error: Master lcore is used as a service core\n");
 		return -1;
 	}
 
@@ -1165,6 +1215,13 @@ eal_parse_common_option(int opt, const char *optarg,
 		conf->no_shconf = 1;
 		break;
 
+	case OPT_IN_MEMORY_NUM:
+		conf->in_memory = 1;
+		/* in-memory is a superset of noshconf and huge-unlink */
+		conf->no_shconf = 1;
+		conf->hugepage_unlink = 1;
+		break;
+
 	case OPT_PROC_TYPE_NUM:
 		conf->process_type = eal_parse_proc_type(optarg);
 		break;
@@ -1316,12 +1373,23 @@ eal_check_common_options(struct internal_config *internal_cfg)
 			"be specified together with --"OPT_NO_HUGE"\n");
 		return -1;
 	}
-
-	if (internal_cfg->no_hugetlbfs && internal_cfg->hugepage_unlink) {
+	if (internal_cfg->no_hugetlbfs && internal_cfg->hugepage_unlink &&
+			!internal_cfg->in_memory) {
 		RTE_LOG(ERR, EAL, "Option --"OPT_HUGE_UNLINK" cannot "
 			"be specified together with --"OPT_NO_HUGE"\n");
 		return -1;
 	}
+	if (internal_config.force_socket_limits && internal_config.legacy_mem) {
+		RTE_LOG(ERR, EAL, "Option --"OPT_SOCKET_LIMIT
+			" is only supported in non-legacy memory mode\n");
+	}
+	if (internal_cfg->single_file_segments &&
+			internal_cfg->hugepage_unlink) {
+		RTE_LOG(ERR, EAL, "Option --"OPT_SINGLE_FILE_SEGMENTS" is "
+			"not compatible with neither --"OPT_IN_MEMORY" nor "
+			"--"OPT_HUGE_UNLINK"\n");
+		return -1;
+	}
 
 	return 0;
 }
@@ -1370,6 +1438,8 @@ eal_common_usage(void)
 	       "                      Set specific log level\n"
 	       "  -v                  Display version information on startup\n"
 	       "  -h, --help          This help\n"
+	       "  --"OPT_IN_MEMORY"   Operate entirely in memory. This will\n"
+	       "                      disable secondary process support\n"
 	       "\nEAL options for DEBUG use only:\n"
 	       "  --"OPT_HUGE_UNLINK"       Unlink hugepage files after init\n"
 	       "  --"OPT_NO_HUGE"           Use malloc instead of hugetlbfs\n"
diff --git a/lib/librte_eal/common/eal_common_proc.c b/lib/librte_eal/common/eal_common_proc.c
index 707d8ab3..9fcb9121 100644
--- a/lib/librte_eal/common/eal_common_proc.c
+++ b/lib/librte_eal/common/eal_common_proc.c
@@ -20,6 +20,7 @@
 #include <sys/un.h>
 #include <unistd.h>
 
+#include <rte_alarm.h>
 #include <rte_common.h>
 #include <rte_cycles.h>
 #include <rte_eal.h>
@@ -94,11 +95,9 @@ TAILQ_HEAD(pending_request_list, pending_request);
 static struct {
 	struct pending_request_list requests;
 	pthread_mutex_t lock;
-	pthread_cond_t async_cond;
 } pending_requests = {
 	.requests = TAILQ_HEAD_INITIALIZER(pending_requests.requests),
 	.lock = PTHREAD_MUTEX_INITIALIZER,
-	.async_cond = PTHREAD_COND_INITIALIZER
 	/**< used in async requests only */
 };
 
@@ -106,6 +105,16 @@ static struct {
 static int
 mp_send(struct rte_mp_msg *msg, const char *peer, int type);
 
+/* for use with alarm callback */
+static void
+async_reply_handle(void *arg);
+
+/* for use with process_msg */
+static struct pending_request *
+async_reply_handle_thread_unsafe(void *arg);
+
+static void
+trigger_async_action(struct pending_request *req);
 
 static struct pending_request *
 find_pending_request(const char *dst, const char *act_name)
@@ -290,6 +299,8 @@ process_msg(struct mp_msg_internal *m, struct sockaddr_un *s)
 	RTE_LOG(DEBUG, EAL, "msg: %s\n", msg->name);
 
 	if (m->type == MP_REP || m->type == MP_IGN) {
+		struct pending_request *req = NULL;
+
 		pthread_mutex_lock(&pending_requests.lock);
 		pending_req = find_pending_request(s->sun_path, msg->name);
 		if (pending_req) {
@@ -301,11 +312,14 @@ process_msg(struct mp_msg_internal *m, struct sockaddr_un *s)
 			if (pending_req->type == REQUEST_TYPE_SYNC)
 				pthread_cond_signal(&pending_req->sync.cond);
 			else if (pending_req->type == REQUEST_TYPE_ASYNC)
-				pthread_cond_signal(
-					&pending_requests.async_cond);
+				req = async_reply_handle_thread_unsafe(
+						pending_req);
 		} else
 			RTE_LOG(ERR, EAL, "Drop mp reply: %s\n", msg->name);
 		pthread_mutex_unlock(&pending_requests.lock);
+
+		if (req != NULL)
+			trigger_async_action(req);
 		return;
 	}
 
@@ -365,7 +379,6 @@ timespec_cmp(const struct timespec *a, const struct timespec *b)
 }
 
 enum async_action {
-	ACTION_NONE, /**< don't do anything */
 	ACTION_FREE, /**< free the action entry, but don't trigger callback */
 	ACTION_TRIGGER /**< trigger callback, then free action entry */
 };
@@ -375,7 +388,7 @@ process_async_request(struct pending_request *sr, const struct timespec *now)
 {
 	struct async_request_param *param;
 	struct rte_mp_reply *reply;
-	bool timeout, received, last_msg;
+	bool timeout, last_msg;
 
 	param = sr->async.param;
 	reply = &param->user_reply;
@@ -383,13 +396,6 @@ process_async_request(struct pending_request *sr, const struct timespec *now)
 	/* did we timeout? */
 	timeout = timespec_cmp(&param->end, now) <= 0;
 
-	/* did we receive a response? */
-	received = sr->reply_received != 0;
-
-	/* if we didn't time out, and we didn't receive a response, ignore */
-	if (!timeout && !received)
-		return ACTION_NONE;
-
 	/* if we received a response, adjust relevant data and copy mesasge. */
 	if (sr->reply_received == 1 && sr->reply) {
 		struct rte_mp_msg *msg, *user_msgs, *tmp;
@@ -448,118 +454,58 @@ trigger_async_action(struct pending_request *sr)
 	free(sr->async.param->user_reply.msgs);
 	free(sr->async.param);
 	free(sr->request);
+	free(sr);
 }
 
 static struct pending_request *
-check_trigger(struct timespec *ts)
+async_reply_handle_thread_unsafe(void *arg)
 {
-	struct pending_request *next, *cur, *trigger = NULL;
-
-	TAILQ_FOREACH_SAFE(cur, &pending_requests.requests, next, next) {
-		enum async_action action;
-		if (cur->type != REQUEST_TYPE_ASYNC)
-			continue;
+	struct pending_request *req = (struct pending_request *)arg;
+	enum async_action action;
+	struct timespec ts_now;
+	struct timeval now;
 
-		action = process_async_request(cur, ts);
-		if (action == ACTION_FREE) {
-			TAILQ_REMOVE(&pending_requests.requests, cur, next);
-			free(cur);
-		} else if (action == ACTION_TRIGGER) {
-			TAILQ_REMOVE(&pending_requests.requests, cur, next);
-			trigger = cur;
-			break;
-		}
+	if (gettimeofday(&now, NULL) < 0) {
+		RTE_LOG(ERR, EAL, "Cannot get current time\n");
+		goto no_trigger;
 	}
-	return trigger;
-}
+	ts_now.tv_nsec = now.tv_usec * 1000;
+	ts_now.tv_sec = now.tv_sec;
 
-static void
-wait_for_async_messages(void)
-{
-	struct pending_request *sr;
-	struct timespec timeout;
-	bool timedwait = false;
-	bool nowait = false;
-	int ret;
+	action = process_async_request(req, &ts_now);
 
-	/* scan through the list and see if there are any timeouts that
-	 * are earlier than our current timeout.
-	 */
-	TAILQ_FOREACH(sr, &pending_requests.requests, next) {
-		if (sr->type != REQUEST_TYPE_ASYNC)
-			continue;
-		if (!timedwait || timespec_cmp(&sr->async.param->end,
-				&timeout) < 0) {
-			memcpy(&timeout, &sr->async.param->end,
-				sizeof(timeout));
-			timedwait = true;
-		}
+	TAILQ_REMOVE(&pending_requests.requests, req, next);
 
-		/* sometimes, we don't even wait */
-		if (sr->reply_received) {
-			nowait = true;
-			break;
+	if (rte_eal_alarm_cancel(async_reply_handle, req) < 0) {
+		/* if we failed to cancel the alarm because it's already in
+		 * progress, don't proceed because otherwise we will end up
+		 * handling the same message twice.
+		 */
+		if (rte_errno == EINPROGRESS) {
+			RTE_LOG(DEBUG, EAL, "Request handling is already in progress\n");
+			goto no_trigger;
 		}
+		RTE_LOG(ERR, EAL, "Failed to cancel alarm\n");
 	}
 
-	if (nowait)
-		return;
-
-	do {
-		ret = timedwait ?
-			pthread_cond_timedwait(
-				&pending_requests.async_cond,
-				&pending_requests.lock,
-				&timeout) :
-			pthread_cond_wait(
-				&pending_requests.async_cond,
-				&pending_requests.lock);
-	} while (ret != 0 && ret != ETIMEDOUT);
-
-	/* we've been woken up or timed out */
+	if (action == ACTION_TRIGGER)
+		return req;
+no_trigger:
+	free(req);
+	return NULL;
 }
 
-static void *
-async_reply_handle(void *arg __rte_unused)
+static void
+async_reply_handle(void *arg)
 {
-	struct timeval now;
-	struct timespec ts_now;
-	while (1) {
-		struct pending_request *trigger = NULL;
-
-		pthread_mutex_lock(&pending_requests.lock);
+	struct pending_request *req;
 
-		/* we exit this function holding the lock */
-		wait_for_async_messages();
-
-		if (gettimeofday(&now, NULL) < 0) {
-			pthread_mutex_unlock(&pending_requests.lock);
-			RTE_LOG(ERR, EAL, "Cannot get current time\n");
-			break;
-		}
-		ts_now.tv_nsec = now.tv_usec * 1000;
-		ts_now.tv_sec = now.tv_sec;
-
-		do {
-			trigger = check_trigger(&ts_now);
-			/* unlock request list */
-			pthread_mutex_unlock(&pending_requests.lock);
-
-			if (trigger) {
-				trigger_async_action(trigger);
-				free(trigger);
-
-				/* we've triggered a callback, but there may be
-				 * more, so lock the list and check again.
-				 */
-				pthread_mutex_lock(&pending_requests.lock);
-			}
-		} while (trigger);
-	}
-
-	RTE_LOG(ERR, EAL, "ERROR: asynchronous requests disabled\n");
+	pthread_mutex_lock(&pending_requests.lock);
+	req = async_reply_handle_thread_unsafe(arg);
+	pthread_mutex_unlock(&pending_requests.lock);
 
-	return NULL;
+	if (req != NULL)
+		trigger_async_action(req);
 }
 
 static int
@@ -624,7 +570,15 @@ rte_mp_channel_init(void)
 {
 	char path[PATH_MAX];
 	int dir_fd;
-	pthread_t mp_handle_tid, async_reply_handle_tid;
+	pthread_t mp_handle_tid;
+
+	/* in no shared files mode, we do not have secondary processes support,
+	 * so no need to initialize IPC.
+	 */
+	if (internal_config.no_shconf) {
+		RTE_LOG(DEBUG, EAL, "No shared files mode enabled, IPC will be disabled\n");
+		return 0;
+	}
 
 	/* create filter path */
 	create_socket_path("*", path, sizeof(path));
@@ -671,17 +625,6 @@ rte_mp_channel_init(void)
 		return -1;
 	}
 
-	if (rte_ctrl_thread_create(&async_reply_handle_tid,
-			"rte_mp_async", NULL,
-			async_reply_handle, NULL) < 0) {
-		RTE_LOG(ERR, EAL, "failed to create mp thead: %s\n",
-			strerror(errno));
-		close(mp_fd);
-		close(dir_fd);
-		mp_fd = -1;
-		return -1;
-	}
-
 	/* unlock the directory */
 	flock(dir_fd, LOCK_UN);
 	close(dir_fd);
@@ -786,7 +729,7 @@ mp_send(struct rte_mp_msg *msg, const char *peer, int type)
 
 	dir_fd = dirfd(mp_dir);
 	/* lock the directory to prevent processes spinning up while we send */
-	if (flock(dir_fd, LOCK_EX)) {
+	if (flock(dir_fd, LOCK_SH)) {
 		RTE_LOG(ERR, EAL, "Unable to lock directory %s\n",
 			mp_dir_path);
 		rte_errno = errno;
@@ -853,7 +796,7 @@ rte_mp_sendmsg(struct rte_mp_msg *msg)
 
 static int
 mp_request_async(const char *dst, struct rte_mp_msg *req,
-		struct async_request_param *param)
+		struct async_request_param *param, const struct timespec *ts)
 {
 	struct rte_mp_msg *reply_msg;
 	struct pending_request *pending_req, *exist;
@@ -898,6 +841,13 @@ mp_request_async(const char *dst, struct rte_mp_msg *req,
 
 	param->user_reply.nb_sent++;
 
+	if (rte_eal_alarm_set(ts->tv_sec * 1000000 + ts->tv_nsec / 1000,
+			      async_reply_handle, pending_req) < 0) {
+		RTE_LOG(ERR, EAL, "Fail to set alarm for request %s:%s\n",
+			dst, req->name);
+		rte_panic("Fix the above shit to properly free all memory\n");
+	}
+
 	return 0;
 fail:
 	free(pending_req);
@@ -988,6 +938,12 @@ rte_mp_request_sync(struct rte_mp_msg *req, struct rte_mp_reply *reply,
 
 	if (check_input(req) == false)
 		return -1;
+
+	if (internal_config.no_shconf) {
+		RTE_LOG(DEBUG, EAL, "No shared files mode enabled, IPC is disabled\n");
+		return 0;
+	}
+
 	if (gettimeofday(&now, NULL) < 0) {
 		RTE_LOG(ERR, EAL, "Faile to get current time\n");
 		rte_errno = errno;
@@ -1020,7 +976,7 @@ rte_mp_request_sync(struct rte_mp_msg *req, struct rte_mp_reply *reply,
 
 	dir_fd = dirfd(mp_dir);
 	/* lock the directory to prevent processes spinning up while we send */
-	if (flock(dir_fd, LOCK_EX)) {
+	if (flock(dir_fd, LOCK_SH)) {
 		RTE_LOG(ERR, EAL, "Unable to lock directory %s\n",
 			mp_dir_path);
 		closedir(mp_dir);
@@ -1072,6 +1028,12 @@ rte_mp_request_async(struct rte_mp_msg *req, const struct timespec *ts,
 
 	if (check_input(req) == false)
 		return -1;
+
+	if (internal_config.no_shconf) {
+		RTE_LOG(DEBUG, EAL, "No shared files mode enabled, IPC is disabled\n");
+		return 0;
+	}
+
 	if (gettimeofday(&now, NULL) < 0) {
 		RTE_LOG(ERR, EAL, "Faile to get current time\n");
 		rte_errno = errno;
@@ -1119,7 +1081,7 @@ rte_mp_request_async(struct rte_mp_msg *req, const struct timespec *ts,
 
 	/* for secondary process, send request to the primary process only */
 	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
-		ret = mp_request_async(eal_mp_socket_path(), copy, param);
+		ret = mp_request_async(eal_mp_socket_path(), copy, param, ts);
 
 		/* if we didn't send anything, put dummy request on the queue */
 		if (ret == 0 && reply->nb_sent == 0) {
@@ -1146,7 +1108,7 @@ rte_mp_request_async(struct rte_mp_msg *req, const struct timespec *ts,
 	dir_fd = dirfd(mp_dir);
 
 	/* lock the directory to prevent processes spinning up while we send */
-	if (flock(dir_fd, LOCK_EX)) {
+	if (flock(dir_fd, LOCK_SH)) {
 		RTE_LOG(ERR, EAL, "Unable to lock directory %s\n",
 			mp_dir_path);
 		rte_errno = errno;
@@ -1162,7 +1124,7 @@ rte_mp_request_async(struct rte_mp_msg *req, const struct timespec *ts,
 		snprintf(path, sizeof(path), "%s/%s", mp_dir_path,
 			 ent->d_name);
 
-		if (mp_request_async(path, copy, param))
+		if (mp_request_async(path, copy, param, ts))
 			ret = -1;
 	}
 	/* if we didn't send anything, put dummy request on the queue */
@@ -1171,9 +1133,6 @@ rte_mp_request_async(struct rte_mp_msg *req, const struct timespec *ts,
 		dummy_used = true;
 	}
 
-	/* trigger async request thread wake up */
-	pthread_cond_signal(&pending_requests.async_cond);
-
 	/* finally, unlock the queue */
 	pthread_mutex_unlock(&pending_requests.lock);
 
@@ -1213,5 +1172,10 @@ rte_mp_reply(struct rte_mp_msg *msg, const char *peer)
 		return -1;
 	}
 
+	if (internal_config.no_shconf) {
+		RTE_LOG(DEBUG, EAL, "No shared files mode enabled, IPC is disabled\n");
+		return 0;
+	}
+
 	return mp_send(msg, peer, MP_REP);
 }
diff --git a/lib/librte_eal/common/eal_common_thread.c b/lib/librte_eal/common/eal_common_thread.c
index 42398630..48ef4d6d 100644
--- a/lib/librte_eal/common/eal_common_thread.c
+++ b/lib/librte_eal/common/eal_common_thread.c
@@ -175,7 +175,7 @@ rte_ctrl_thread_create(pthread_t *thread, const char *name,
 
 	params = malloc(sizeof(*params));
 	if (!params)
-		return -1;
+		return -ENOMEM;
 
 	params->start_routine = start_routine;
 	params->arg = arg;
@@ -185,13 +185,14 @@ rte_ctrl_thread_create(pthread_t *thread, const char *name,
 	ret = pthread_create(thread, attr, rte_thread_init, (void *)params);
 	if (ret != 0) {
 		free(params);
-		return ret;
+		return -ret;
 	}
 
 	if (name != NULL) {
 		ret = rte_thread_setname(*thread, name);
 		if (ret < 0)
-			goto fail;
+			RTE_LOG(DEBUG, EAL,
+				"Cannot set name for ctrl thread\n");
 	}
 
 	cpu_found = 0;
@@ -227,5 +228,5 @@ fail:
 	}
 	pthread_cancel(*thread);
 	pthread_join(*thread, NULL);
-	return ret;
+	return -ret;
 }
diff --git a/lib/librte_eal/common/eal_common_uuid.c b/lib/librte_eal/common/eal_common_uuid.c
new file mode 100644
index 00000000..1b93c5b3
--- /dev/null
+++ b/lib/librte_eal/common/eal_common_uuid.c
@@ -0,0 +1,193 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C) 1996, 1997 Theodore Ts'o.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, and the entire permission notice in its entirety,
+ *    including the disclaimer of warranties.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote
+ *    products derived from this software without specific prior
+ *    written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
+ * WHICH ARE HEREBY DISCLAIMED.	 IN NO EVENT SHALL THE AUTHOR BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <ctype.h>
+
+#include <rte_uuid.h>
+
+/* UUID packed form */
+struct uuid {
+	uint32_t	time_low;
+	uint16_t	time_mid;
+	uint16_t	time_hi_and_version;
+	uint16_t	clock_seq;
+	uint8_t		node[6];
+};
+
+static void uuid_pack(const struct uuid *uu, rte_uuid_t ptr)
+{
+	uint32_t tmp;
+	uint8_t	*out = ptr;
+
+	tmp = uu->time_low;
+	out[3] = (uint8_t) tmp;
+	tmp >>= 8;
+	out[2] = (uint8_t) tmp;
+	tmp >>= 8;
+	out[1] = (uint8_t) tmp;
+	tmp >>= 8;
+	out[0] = (uint8_t) tmp;
+
+	tmp = uu->time_mid;
+	out[5] = (uint8_t) tmp;
+	tmp >>= 8;
+	out[4] = (uint8_t) tmp;
+
+	tmp = uu->time_hi_and_version;
+	out[7] = (uint8_t) tmp;
+	tmp >>= 8;
+	out[6] = (uint8_t) tmp;
+
+	tmp = uu->clock_seq;
+	out[9] = (uint8_t) tmp;
+	tmp >>= 8;
+	out[8] = (uint8_t) tmp;
+
+	memcpy(out+10, uu->node, 6);
+}
+
+static void uuid_unpack(const rte_uuid_t in, struct uuid *uu)
+{
+	const uint8_t *ptr = in;
+	uint32_t tmp;
+
+	tmp = *ptr++;
+	tmp = (tmp << 8) | *ptr++;
+	tmp = (tmp << 8) | *ptr++;
+	tmp = (tmp << 8) | *ptr++;
+	uu->time_low = tmp;
+
+	tmp = *ptr++;
+	tmp = (tmp << 8) | *ptr++;
+	uu->time_mid = tmp;
+
+	tmp = *ptr++;
+	tmp = (tmp << 8) | *ptr++;
+	uu->time_hi_and_version = tmp;
+
+	tmp = *ptr++;
+	tmp = (tmp << 8) | *ptr++;
+	uu->clock_seq = tmp;
+
+	memcpy(uu->node, ptr, 6);
+}
+
+bool rte_uuid_is_null(const rte_uuid_t uu)
+{
+	const uint8_t *cp = uu;
+	int i;
+
+	for (i = 0; i < 16; i++)
+		if (*cp++)
+			return false;
+	return true;
+}
+
+/*
+ * rte_uuid_compare() - compare two UUIDs.
+ */
+int rte_uuid_compare(const rte_uuid_t uu1, const rte_uuid_t uu2)
+{
+	struct uuid	uuid1, uuid2;
+
+	uuid_unpack(uu1, &uuid1);
+	uuid_unpack(uu2, &uuid2);
+
+#define UUCMP(u1, u2) \
+	do { if (u1 != u2) return (u1 < u2) ? -1 : 1; } while (0)
+
+	UUCMP(uuid1.time_low, uuid2.time_low);
+	UUCMP(uuid1.time_mid, uuid2.time_mid);
+	UUCMP(uuid1.time_hi_and_version, uuid2.time_hi_and_version);
+	UUCMP(uuid1.clock_seq, uuid2.clock_seq);
+#undef UUCMP
+
+	return memcmp(uuid1.node, uuid2.node, 6);
+}
+
+int rte_uuid_parse(const char *in, rte_uuid_t uu)
+{
+	struct uuid	uuid;
+	int		i;
+	const char	*cp;
+	char		buf[3];
+
+	if (strlen(in) != 36)
+		return -1;
+
+	for (i = 0, cp = in; i <= 36; i++, cp++) {
+		if ((i == 8) || (i == 13) || (i == 18) ||
+		    (i == 23)) {
+			if (*cp == '-')
+				continue;
+			else
+				return -1;
+		}
+		if (i == 36)
+			if (*cp == 0)
+				continue;
+		if (!isxdigit(*cp))
+			return -1;
+	}
+
+	uuid.time_low = strtoul(in, NULL, 16);
+	uuid.time_mid = strtoul(in+9, NULL, 16);
+	uuid.time_hi_and_version = strtoul(in+14, NULL, 16);
+	uuid.clock_seq = strtoul(in+19, NULL, 16);
+	cp = in+24;
+	buf[2] = 0;
+
+	for (i = 0; i < 6; i++) {
+		buf[0] = *cp++;
+		buf[1] = *cp++;
+		uuid.node[i] = strtoul(buf, NULL, 16);
+	}
+
+	uuid_pack(&uuid, uu);
+	return 0;
+}
+
+void rte_uuid_unparse(const rte_uuid_t uu, char *out, size_t len)
+{
+	struct uuid uuid;
+
+	uuid_unpack(uu, &uuid);
+
+	snprintf(out, len,
+		 "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x",
+		uuid.time_low, uuid.time_mid, uuid.time_hi_and_version,
+		uuid.clock_seq >> 8, uuid.clock_seq & 0xFF,
+		uuid.node[0], uuid.node[1], uuid.node[2],
+		uuid.node[3], uuid.node[4], uuid.node[5]);
+}
diff --git a/lib/librte_eal/common/eal_filesystem.h b/lib/librte_eal/common/eal_filesystem.h
index 364f38d1..de05febf 100644
--- a/lib/librte_eal/common/eal_filesystem.h
+++ b/lib/librte_eal/common/eal_filesystem.h
@@ -12,7 +12,6 @@
 #define EAL_FILESYSTEM_H
 
 /** Path of rte config file. */
-#define RUNTIME_CONFIG_FMT "%s/.%s_config"
 
 #include <stdint.h>
 #include <limits.h>
@@ -30,17 +29,14 @@ eal_create_runtime_dir(void);
 const char *
 eal_get_runtime_dir(void);
 
+#define RUNTIME_CONFIG_FNAME "config"
 static inline const char *
 eal_runtime_config_path(void)
 {
 	static char buffer[PATH_MAX]; /* static so auto-zeroed */
-	const char *directory = "/var/run";
-	const char *home_dir = getenv("HOME");
 
-	if (getuid() != 0 && home_dir != NULL)
-		directory = home_dir;
-	snprintf(buffer, sizeof(buffer) - 1, RUNTIME_CONFIG_FMT, directory,
-			internal_config.hugefile_prefix);
+	snprintf(buffer, sizeof(buffer) - 1, "%s/%s", eal_get_runtime_dir(),
+			RUNTIME_CONFIG_FNAME);
 	return buffer;
 }
 
diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
index c4cbf3ac..00ee6e06 100644
--- a/lib/librte_eal/common/eal_internal_cfg.h
+++ b/lib/librte_eal/common/eal_internal_cfg.h
@@ -41,11 +41,17 @@ struct internal_config {
 	volatile unsigned vmware_tsc_map; /**< true to use VMware TSC mapping
 										* instead of native TSC */
 	volatile unsigned no_shconf;      /**< true if there is no shared config */
+	volatile unsigned in_memory;
+	/**< true if DPDK should operate entirely in-memory and not create any
+	 * shared files or runtime data.
+	 */
 	volatile unsigned create_uio_dev; /**< true to create /dev/uioX devices */
 	volatile enum rte_proc_type_t process_type; /**< multi-process proc type */
 	/** true to try allocating memory on specific sockets */
 	volatile unsigned force_sockets;
 	volatile uint64_t socket_mem[RTE_MAX_NUMA_NODES]; /**< amount of memory per socket */
+	volatile unsigned force_socket_limits;
+	volatile uint64_t socket_limit[RTE_MAX_NUMA_NODES]; /**< limit amount of memory per socket */
 	uintptr_t base_virtaddr;          /**< base address to try and reserve memory from */
 	volatile unsigned legacy_mem;
 	/**< true to enable legacy memory behavior (no dynamic allocation,
diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
index 211ae06a..96e16678 100644
--- a/lib/librte_eal/common/eal_options.h
+++ b/lib/librte_eal/common/eal_options.h
@@ -45,8 +45,12 @@ enum {
 	OPT_NO_PCI_NUM,
 #define OPT_NO_SHCONF         "no-shconf"
 	OPT_NO_SHCONF_NUM,
+#define OPT_IN_MEMORY         "in-memory"
+	OPT_IN_MEMORY_NUM,
 #define OPT_SOCKET_MEM        "socket-mem"
 	OPT_SOCKET_MEM_NUM,
+#define OPT_SOCKET_LIMIT        "socket-limit"
+	OPT_SOCKET_LIMIT_NUM,
 #define OPT_SYSLOG            "syslog"
 	OPT_SYSLOG_NUM,
 #define OPT_VDEV              "vdev"
diff --git a/lib/librte_eal/common/eal_private.h b/lib/librte_eal/common/eal_private.h
index bdadc4d5..4f809a83 100644
--- a/lib/librte_eal/common/eal_private.h
+++ b/lib/librte_eal/common/eal_private.h
@@ -47,6 +47,18 @@ void eal_log_set_default(FILE *default_log);
 int rte_eal_cpu_init(void);
 
 /**
+ * Create memseg lists
+ *
+ * This function is private to EAL.
+ *
+ * Preallocate virtual memory.
+ *
+ * @return
+ *   0 on success, negative on error
+ */
+int rte_eal_memseg_init(void);
+
+/**
  * Map memory
  *
  * This function is private to EAL.
@@ -258,4 +270,38 @@ int rte_mp_channel_init(void);
  */
 void dev_callback_process(char *device_name, enum rte_dev_event_type event);
 
+/**
+ * @internal
+ * Parse a device string and store its information in an
+ * rte_devargs structure.
+ *
+ * A device description is split by layers of abstraction of the device:
+ * bus, class and driver. Each layer will offer a set of properties that
+ * can be applied either to configure or recognize a device.
+ *
+ * This function will parse those properties and prepare the rte_devargs
+ * to be given to each layers for processing.
+ *
+ * Note: if the "data" field of the devargs points to devstr,
+ * then no dynamic allocation is performed and the rte_devargs
+ * can be safely discarded.
+ *
+ * Otherwise ``data`` will hold a workable copy of devstr, that will be
+ * used by layers descriptors within rte_devargs. In this case,
+ * any rte_devargs should be cleaned-up before being freed.
+ *
+ * @param da
+ *   rte_devargs structure to fill.
+ *
+ * @param devstr
+ *   Device string.
+ *
+ * @return
+ *   0 on success.
+ *   Negative errno values on error (rte_errno is set).
+ */
+int
+rte_devargs_layers_parse(struct rte_devargs *devargs,
+			 const char *devstr);
+
 #endif /* _EAL_PRIVATE_H_ */
diff --git a/lib/librte_eal/common/include/rte_bitmap.h b/lib/librte_eal/common/include/rte_bitmap.h
index 7d4935fc..d9facc64 100644
--- a/lib/librte_eal/common/include/rte_bitmap.h
+++ b/lib/librte_eal/common/include/rte_bitmap.h
@@ -198,12 +198,12 @@ rte_bitmap_get_memory_footprint(uint32_t n_bits) {
 /**
  * Bitmap initialization
  *
- * @param mem_size
- *   Minimum expected size of bitmap.
+ * @param n_bits
+ *   Number of pre-allocated bits in array2.
  * @param mem
  *   Base address of array1 and array2.
- * @param n_bits
- *   Number of pre-allocated bits in array2. Must be non-zero and multiple of 512.
+ * @param mem_size
+ *   Minimum expected size of bitmap.
  * @return
  *   Handle to bitmap instance.
  */
diff --git a/lib/librte_eal/common/include/rte_bus.h b/lib/librte_eal/common/include/rte_bus.h
index eb9eded4..b7b5b084 100644
--- a/lib/librte_eal/common/include/rte_bus.h
+++ b/lib/librte_eal/common/include/rte_bus.h
@@ -211,6 +211,7 @@ struct rte_bus {
 	rte_bus_parse_t parse;       /**< Parse a device name */
 	struct rte_bus_conf conf;    /**< Bus configuration */
 	rte_bus_get_iommu_class_t get_iommu_class; /**< Get iommu class */
+	rte_dev_iterate_t dev_iterate; /**< Device iterator. */
 };
 
 /**
@@ -325,8 +326,7 @@ enum rte_iova_mode rte_bus_get_iommu_class(void);
  * The constructor has higher priority than PMD constructors.
  */
 #define RTE_REGISTER_BUS(nm, bus) \
-RTE_INIT_PRIO(businitfn_ ##nm, BUS); \
-static void businitfn_ ##nm(void) \
+RTE_INIT_PRIO(businitfn_ ##nm, BUS) \
 {\
 	(bus).name = RTE_STR(nm);\
 	rte_bus_register(&bus); \
diff --git a/lib/librte_eal/common/include/rte_class.h b/lib/librte_eal/common/include/rte_class.h
new file mode 100644
index 00000000..276c91e9
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_class.h
@@ -0,0 +1,134 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Gaëtan Rivet
+ */
+
+#ifndef _RTE_CLASS_H_
+#define _RTE_CLASS_H_
+
+/**
+ * @file
+ *
+ * DPDK device class interface.
+ *
+ * This file describes the interface of the device class
+ * abstraction layer.
+ *
+ * A device class defines the type of function a device
+ * will be used for e.g.: Ethernet adapter (eth),
+ * cryptographic coprocessor (crypto), etc.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sys/queue.h>
+
+#include <rte_dev.h>
+
+/** Double linked list of classes */
+TAILQ_HEAD(rte_class_list, rte_class);
+
+/**
+ * A structure describing a generic device class.
+ */
+struct rte_class {
+	TAILQ_ENTRY(rte_class) next; /**< Next device class in linked list */
+	const char *name; /**< Name of the class */
+	rte_dev_iterate_t dev_iterate; /**< Device iterator. */
+};
+
+/**
+ * Class comparison function.
+ *
+ * @param cls
+ *	Class under test.
+ *
+ * @param data
+ *	Data to compare against.
+ *
+ * @return
+ *	0 if the class matches the data.
+ *	!0 if the class does not match.
+ *	<0 if ordering is possible and the class is lower than the data.
+ *	>0 if ordering is possible and the class is greater than the data.
+ */
+typedef int (*rte_class_cmp_t)(const struct rte_class *cls, const void *data);
+
+/**
+ * Class iterator to find a particular class.
+ *
+ * This function compares each registered class to find one that matches
+ * the data passed as parameter.
+ *
+ * If the comparison function returns zero this function will stop iterating
+ * over any more classes. To continue a search the class of a previous search
+ * can be passed via the start parameter.
+ *
+ * @param start
+ *	Starting point for the iteration.
+ *
+ * @param cmp
+ *	Comparison function.
+ *
+ * @param data
+ *	 Data to pass to comparison function.
+ *
+ * @return
+ *	 A pointer to a rte_class structure or NULL in case no class matches
+ */
+__rte_experimental
+struct rte_class *
+rte_class_find(const struct rte_class *start, rte_class_cmp_t cmp,
+	       const void *data);
+
+/**
+ * Find the registered class for a given name.
+ */
+__rte_experimental
+struct rte_class *
+rte_class_find_by_name(const char *name);
+
+/**
+ * Register a Class handle.
+ *
+ * @param cls
+ *   A pointer to a rte_class structure describing the class
+ *   to be registered.
+ */
+__rte_experimental
+void rte_class_register(struct rte_class *cls);
+
+/**
+ * Unregister a Class handle.
+ *
+ * @param cls
+ *   A pointer to a rte_class structure describing the class
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_class_unregister(struct rte_class *cls);
+
+/**
+ * Helper for Class registration.
+ * The constructor has lower priority than Bus constructors.
+ * The constructor has higher priority than PMD constructors.
+ */
+#define RTE_REGISTER_CLASS(nm, cls) \
+RTE_INIT_PRIO(classinitfn_ ##nm, CLASS) \
+{\
+	(cls).name = RTE_STR(nm); \
+	rte_class_register(&cls); \
+}
+
+#define RTE_UNREGISTER_CLASS(nm, cls) \
+RTE_FINI_PRIO(classfinifn_ ##nm, CLASS) \
+{ \
+	rte_class_unregister(&cls); \
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_CLASS_H_ */
diff --git a/lib/librte_eal/common/include/rte_common.h b/lib/librte_eal/common/include/rte_common.h
index 434adfd4..069c13ec 100644
--- a/lib/librte_eal/common/include/rte_common.h
+++ b/lib/librte_eal/common/include/rte_common.h
@@ -83,6 +83,7 @@ typedef uint16_t unaligned_uint16_t;
 
 #define RTE_PRIORITY_LOG 101
 #define RTE_PRIORITY_BUS 110
+#define RTE_PRIORITY_CLASS 120
 #define RTE_PRIORITY_LAST 65535
 
 #define RTE_PRIO(prio) \
@@ -112,6 +113,29 @@ static void __attribute__((constructor(RTE_PRIO(prio)), used)) func(void)
 	RTE_INIT_PRIO(func, LAST)
 
 /**
+ * Run after main() with low priority.
+ *
+ * @param func
+ *   Destructor function name.
+ * @param prio
+ *   Priority number must be above 100.
+ *   Lowest number is the last to run.
+ */
+#define RTE_FINI_PRIO(func, prio) \
+static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
+
+/**
+ * Run after main() with high priority.
+ *
+ * The destructor will be run *before* prioritized destructors.
+ *
+ * @param func
+ *   Destructor function name.
+ */
+#define RTE_FINI(func) \
+	RTE_FINI_PRIO(func, LAST)
+
+/**
  * Force a function to be inlined
  */
 #define __rte_always_inline inline __attribute__((always_inline))
@@ -294,6 +318,11 @@ rte_combine64ms1b(register uint64_t v)
 /*********** Macros to work with powers of 2 ********/
 
 /**
+ * Macro to return 1 if n is a power of 2, 0 otherwise
+ */
+#define RTE_IS_POWER_OF_2(n) ((n) && !(((n) - 1) & (n)))
+
+/**
  * Returns true if n is a power of 2
  * @param n
  *     Number to check
diff --git a/lib/librte_eal/common/include/rte_dev.h b/lib/librte_eal/common/include/rte_dev.h
index 3879ff3c..b80a8059 100644
--- a/lib/librte_eal/common/include/rte_dev.h
+++ b/lib/librte_eal/common/include/rte_dev.h
@@ -69,9 +69,7 @@ rte_pmd_debug_trace(const char *func_name, const char *fmt, ...)
  * Enable RTE_PMD_DEBUG_TRACE() when at least one component relying on the
  * RTE_*_RET() macros defined below is compiled in debug mode.
  */
-#if defined(RTE_LIBRTE_ETHDEV_DEBUG) || \
-	defined(RTE_LIBRTE_CRYPTODEV_DEBUG) || \
-	defined(RTE_LIBRTE_EVENTDEV_DEBUG)
+#if defined(RTE_LIBRTE_EVENTDEV_DEBUG)
 #define RTE_PMD_DEBUG_TRACE(...) \
 	rte_pmd_debug_trace(__func__, __VA_ARGS__)
 #else
@@ -176,6 +174,7 @@ struct rte_device {
  * @return
  *   0 on success, negative on error.
  */
+__rte_deprecated
 int rte_eal_dev_attach(const char *name, const char *devargs);
 
 /**
@@ -186,6 +185,7 @@ int rte_eal_dev_attach(const char *name, const char *devargs);
  * @return
  *   0 on success, negative on error.
  */
+__rte_deprecated
 int rte_eal_dev_detach(struct rte_device *dev);
 
 /**
@@ -285,6 +285,103 @@ __attribute__((used)) = str
 static const char DRV_EXP_TAG(name, kmod_dep_export)[] \
 __attribute__((used)) = str
 
+/**
+ * Iteration context.
+ *
+ * This context carries over the current iteration state.
+ */
+struct rte_dev_iterator {
+	const char *dev_str; /**< device string. */
+	const char *bus_str; /**< bus-related part of device string. */
+	const char *cls_str; /**< class-related part of device string. */
+	struct rte_bus *bus; /**< bus handle. */
+	struct rte_class *cls; /**< class handle. */
+	struct rte_device *device; /**< current position. */
+	void *class_device; /**< additional specialized context. */
+};
+
+/**
+ * Device iteration function.
+ *
+ * Find the next device matching properties passed in parameters.
+ * The function takes an additional ``start`` parameter, that is
+ * used as starting context when relevant.
+ *
+ * The function returns the current element in the iteration.
+ * This return value will potentially be used as a start parameter
+ * in subsequent calls to the function.
+ *
+ * The additional iterator parameter is only there if a specific
+ * implementation needs additional context. It must not be modified by
+ * the iteration function itself.
+ *
+ * @param start
+ *   Starting iteration context.
+ *
+ * @param devstr
+ *   Device description string.
+ *
+ * @param it
+ *   Device iterator.
+ *
+ * @return
+ *   The address of the current element matching the device description
+ *   string.
+ */
+typedef void *(*rte_dev_iterate_t)(const void *start,
+				   const char *devstr,
+				   const struct rte_dev_iterator *it);
+
+/**
+ * Initializes a device iterator.
+ *
+ * This iterator allows accessing a list of devices matching a criteria.
+ * The device matching is made among all buses and classes currently registered,
+ * filtered by the device description given as parameter.
+ *
+ * This function will not allocate any memory. It is safe to stop the
+ * iteration at any moment and let the iterator go out of context.
+ *
+ * @param it
+ *   Device iterator handle.
+ *
+ * @param str
+ *   Device description string.
+ *
+ * @return
+ *   0 on successful initialization.
+ *   <0 on error.
+ */
+__rte_experimental
+int
+rte_dev_iterator_init(struct rte_dev_iterator *it, const char *str);
+
+/**
+ * Iterates on a device iterator.
+ *
+ * Generates a new rte_device handle corresponding to the next element
+ * in the list described in comprehension by the iterator.
+ *
+ * The next object is returned, and the iterator is updated.
+ *
+ * @param it
+ *   Device iterator handle.
+ *
+ * @return
+ *   An rte_device handle if found.
+ *   NULL if an error occurred (rte_errno is set).
+ *   NULL if no device could be found (rte_errno is not set).
+ */
+__rte_experimental
+struct rte_device *
+rte_dev_iterator_next(struct rte_dev_iterator *it);
+
+#define RTE_DEV_FOREACH(dev, devstr, it) \
+	for (rte_dev_iterator_init(it, devstr), \
+	     dev = rte_dev_iterator_next(it); \
+	     dev != NULL; \
+	     dev = rte_dev_iterator_next(it))
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_eal/common/include/rte_devargs.h b/lib/librte_eal/common/include/rte_devargs.h
index 58fbd90a..097a4ce7 100644
--- a/lib/librte_eal/common/include/rte_devargs.h
+++ b/lib/librte_eal/common/include/rte_devargs.h
@@ -51,12 +51,19 @@ struct rte_devargs {
 	enum rte_devtype type;
 	/** Device policy. */
 	enum rte_dev_policy policy;
-	/** Bus handle for the device. */
-	struct rte_bus *bus;
 	/** Name of the device. */
 	char name[RTE_DEV_NAME_MAX_LEN];
+	RTE_STD_C11
+	union {
 	/** Arguments string as given by user or "" for no argument. */
-	char *args;
+		char *args;
+		const char *drv_str;
+	};
+	struct rte_bus *bus; /**< bus handle. */
+	struct rte_class *cls; /**< class handle. */
+	const char *bus_str; /**< bus-related part of device string. */
+	const char *cls_str; /**< class-related part of device string. */
+	const char *data; /**< Device string storage. */
 };
 
 /**
@@ -96,6 +103,42 @@ int rte_eal_parse_devargs_str(const char *devargs_str,
  * in argument. Store which bus will handle the device, its name
  * and the eventual device parameters.
  *
+ * The syntax is:
+ *
+ *     bus:device_identifier,arg1=val1,arg2=val2
+ *
+ * where "bus:" is the bus name followed by any character separator.
+ * The bus name is optional. If no bus name is specified, each bus
+ * will attempt to recognize the device identifier. The first one
+ * to succeed will be used.
+ *
+ * Examples:
+ *
+ *     pci:0000:05.00.0,arg=val
+ *     05.00.0,arg=val
+ *     vdev:net_ring0
+ *
+ * @param da
+ *   The devargs structure holding the device information.
+ *
+ * @param dev
+ *   String describing a device.
+ *
+ * @return
+ *   - 0 on success.
+ *   - Negative errno on error.
+ */
+__rte_experimental
+int
+rte_devargs_parse(struct rte_devargs *da, const char *dev);
+
+/**
+ * Parse a device string.
+ *
+ * Verify that a bus is capable of handling the device passed
+ * in argument. Store which bus will handle the device, its name
+ * and the eventual device parameters.
+ *
  * The device string is built with a printf-like syntax.
  *
  * The syntax is:
@@ -124,8 +167,8 @@ int rte_eal_parse_devargs_str(const char *devargs_str,
  */
 __rte_experimental
 int
-rte_devargs_parse(struct rte_devargs *da,
-		  const char *format, ...)
+rte_devargs_parsef(struct rte_devargs *da,
+		   const char *format, ...)
 __attribute__((format(printf, 2, 0)));
 
 /**
diff --git a/lib/librte_eal/common/include/rte_eal.h b/lib/librte_eal/common/include/rte_eal.h
index 8de5d69e..e114dcbd 100644
--- a/lib/librte_eal/common/include/rte_eal.h
+++ b/lib/librte_eal/common/include/rte_eal.h
@@ -490,27 +490,13 @@ static inline int rte_gettid(void)
 enum rte_iova_mode rte_eal_iova_mode(void);
 
 /**
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice
- *
  * Get user provided pool ops name for mbuf
  *
  * @return
  *   returns user provided pool ops name.
  */
-const char * __rte_experimental
-rte_eal_mbuf_user_pool_ops(void);
-
-/**
- * @deprecated
- * Get default pool ops name for mbuf
- *
- * @return
- *   returns default pool ops name.
- */
-__rte_deprecated
 const char *
-rte_eal_mbuf_default_mempool_ops(void);
+rte_eal_mbuf_user_pool_ops(void);
 
 #ifdef __cplusplus
 }
diff --git a/lib/librte_eal/common/include/rte_fbarray.h b/lib/librte_eal/common/include/rte_fbarray.h
index 3e61fffe..5d880551 100644
--- a/lib/librte_eal/common/include/rte_fbarray.h
+++ b/lib/librte_eal/common/include/rte_fbarray.h
@@ -336,6 +336,120 @@ rte_fbarray_find_contig_free(struct rte_fbarray *arr,
 int __rte_experimental
 rte_fbarray_find_contig_used(struct rte_fbarray *arr, unsigned int start);
 
+/**
+ * Find index of previous free element, starting at specified index.
+ *
+ * @param arr
+ *   Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
+ *
+ * @param start
+ *   Element index to start search from.
+ *
+ * @return
+ *  - non-negative integer on success.
+ *  - -1 on failure, with ``rte_errno`` indicating reason for failure.
+ */
+int __rte_experimental
+rte_fbarray_find_prev_free(struct rte_fbarray *arr, unsigned int start);
+
+
+/**
+ * Find index of previous used element, starting at specified index.
+ *
+ * @param arr
+ *   Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
+ *
+ * @param start
+ *   Element index to start search from.
+ *
+ * @return
+ *  - non-negative integer on success.
+ *  - -1 on failure, with ``rte_errno`` indicating reason for failure.
+ */
+int __rte_experimental
+rte_fbarray_find_prev_used(struct rte_fbarray *arr, unsigned int start);
+
+
+/**
+ * Find lowest start index of chunk of ``n`` free elements, down from specified
+ * index.
+ *
+ * @param arr
+ *   Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
+ *
+ * @param start
+ *   Element index to start search from.
+ *
+ * @param n
+ *   Number of free elements to look for.
+ *
+ * @return
+ *  - non-negative integer on success.
+ *  - -1 on failure, with ``rte_errno`` indicating reason for failure.
+ */
+int __rte_experimental
+rte_fbarray_find_prev_n_free(struct rte_fbarray *arr, unsigned int start,
+		unsigned int n);
+
+
+/**
+ * Find lowest start index of chunk of ``n`` used elements, down from specified
+ * index.
+ *
+ * @param arr
+ *   Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
+ *
+ * @param start
+ *   Element index to start search from.
+ *
+ * @param n
+ *   Number of used elements to look for.
+ *
+ * @return
+ *  - non-negative integer on success.
+ *  - -1 on failure, with ``rte_errno`` indicating reason for failure.
+ */
+int __rte_experimental
+rte_fbarray_find_prev_n_used(struct rte_fbarray *arr, unsigned int start,
+		unsigned int n);
+
+
+/**
+ * Find how many more free entries there are before specified index (like
+ * ``rte_fbarray_find_contig_free`` but going in reverse).
+ *
+ * @param arr
+ *   Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
+ *
+ * @param start
+ *   Element index to start search from.
+ *
+ * @return
+ *  - non-negative integer on success.
+ *  - -1 on failure, with ``rte_errno`` indicating reason for failure.
+ */
+int __rte_experimental
+rte_fbarray_find_rev_contig_free(struct rte_fbarray *arr,
+		unsigned int start);
+
+
+/**
+ * Find how many more used entries there are before specified index (like
+ * ``rte_fbarray_find_contig_used`` but going in reverse).
+ *
+ * @param arr
+ *   Valid pointer to allocated and correctly set up ``rte_fbarray`` structure.
+ *
+ * @param start
+ *   Element index to start search from.
+ *
+ * @return
+ *  - non-negative integer on success.
+ *  - -1 on failure, with ``rte_errno`` indicating reason for failure.
+ */
+int __rte_experimental
+rte_fbarray_find_rev_contig_used(struct rte_fbarray *arr, unsigned int start);
+
 
 /**
  * Dump ``rte_fbarray`` metadata.
diff --git a/lib/librte_eal/common/include/rte_memory.h b/lib/librte_eal/common/include/rte_memory.h
index aab9f6fe..c4b7f4cf 100644
--- a/lib/librte_eal/common/include/rte_memory.h
+++ b/lib/librte_eal/common/include/rte_memory.h
@@ -264,6 +264,60 @@ int __rte_experimental
 rte_memseg_list_walk(rte_memseg_list_walk_t func, void *arg);
 
 /**
+ * Walk list of all memsegs without performing any locking.
+ *
+ * @note This function does not perform any locking, and is only safe to call
+ *       from within memory-related callback functions.
+ *
+ * @param func
+ *   Iterator function
+ * @param arg
+ *   Argument passed to iterator
+ * @return
+ *   0 if walked over the entire list
+ *   1 if stopped by the user
+ *   -1 if user function reported error
+ */
+int __rte_experimental
+rte_memseg_walk_thread_unsafe(rte_memseg_walk_t func, void *arg);
+
+/**
+ * Walk each VA-contiguous area without performing any locking.
+ *
+ * @note This function does not perform any locking, and is only safe to call
+ *       from within memory-related callback functions.
+ *
+ * @param func
+ *   Iterator function
+ * @param arg
+ *   Argument passed to iterator
+ * @return
+ *   0 if walked over the entire list
+ *   1 if stopped by the user
+ *   -1 if user function reported error
+ */
+int __rte_experimental
+rte_memseg_contig_walk_thread_unsafe(rte_memseg_contig_walk_t func, void *arg);
+
+/**
+ * Walk each allocated memseg list without performing any locking.
+ *
+ * @note This function does not perform any locking, and is only safe to call
+ *       from within memory-related callback functions.
+ *
+ * @param func
+ *   Iterator function
+ * @param arg
+ *   Argument passed to iterator
+ * @return
+ *   0 if walked over the entire list
+ *   1 if stopped by the user
+ *   -1 if user function reported error
+ */
+int __rte_experimental
+rte_memseg_list_walk_thread_unsafe(rte_memseg_list_walk_t func, void *arg);
+
+/**
  * Dump the physical memory layout to a file.
  *
  * @note This function read-locks the memory hotplug subsystem, and thus cannot
diff --git a/lib/librte_eal/common/include/rte_memzone.h b/lib/librte_eal/common/include/rte_memzone.h
index ef370fa6..f478fa9e 100644
--- a/lib/librte_eal/common/include/rte_memzone.h
+++ b/lib/librte_eal/common/include/rte_memzone.h
@@ -81,8 +81,12 @@ struct rte_memzone {
  *   memzones from memory that is already available. It will not trigger any
  *   new allocations.
  *
- * @note Reserving IOVA-contiguous memzones with len set to 0 is not currently
- *   supported.
+ * @note: When reserving memzones with len set to 0, it is preferable to also
+ *   set a valid socket_id. Setting socket_id to SOCKET_ID_ANY is supported, but
+ *   will likely not yield expected results. Specifically, the resulting memzone
+ *   may not necessarily be the biggest memzone available, but rather biggest
+ *   memzone available on socket id corresponding to an lcore from which
+ *   reservation was called.
  *
  * @param name
  *   The name of the memzone. If it already exists, the function will
@@ -141,8 +145,12 @@ const struct rte_memzone *rte_memzone_reserve(const char *name,
  *   memzones from memory that is already available. It will not trigger any
  *   new allocations.
  *
- * @note Reserving IOVA-contiguous memzones with len set to 0 is not currently
- *   supported.
+ * @note: When reserving memzones with len set to 0, it is preferable to also
+ *   set a valid socket_id. Setting socket_id to SOCKET_ID_ANY is supported, but
+ *   will likely not yield expected results. Specifically, the resulting memzone
+ *   may not necessarily be the biggest memzone available, but rather biggest
+ *   memzone available on socket id corresponding to an lcore from which
+ *   reservation was called.
  *
  * @param name
  *   The name of the memzone. If it already exists, the function will
@@ -206,8 +214,12 @@ const struct rte_memzone *rte_memzone_reserve_aligned(const char *name,
  *   memzones from memory that is already available. It will not trigger any
  *   new allocations.
  *
- * @note Reserving IOVA-contiguous memzones with len set to 0 is not currently
- *   supported.
+ * @note: When reserving memzones with len set to 0, it is preferable to also
+ *   set a valid socket_id. Setting socket_id to SOCKET_ID_ANY is supported, but
+ *   will likely not yield expected results. Specifically, the resulting memzone
+ *   may not necessarily be the biggest memzone available, but rather biggest
+ *   memzone available on socket id corresponding to an lcore from which
+ *   reservation was called.
  *
  * @param name
  *   The name of the memzone. If it already exists, the function will
diff --git a/lib/librte_eal/common/include/rte_service.h b/lib/librte_eal/common/include/rte_service.h
index aea4d91b..34b41aff 100644
--- a/lib/librte_eal/common/include/rte_service.h
+++ b/lib/librte_eal/common/include/rte_service.h
@@ -162,6 +162,26 @@ int32_t rte_service_runstate_set(uint32_t id, uint32_t runstate);
 int32_t rte_service_runstate_get(uint32_t id);
 
 /**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * This function returns whether the service may be currently executing on
+ * at least one lcore, or definitely is not. This function can be used to
+ * determine if, after setting the service runstate to stopped, the service
+ * is still executing a service lcore.
+ *
+ * Care must be taken if calling this function when the service runstate is
+ * running, since the result of this function may be incorrect by the time the
+ * function returns due to service cores running in parallel.
+ *
+ * @retval 1 Service may be running on one or more lcores
+ * @retval 0 Service is not running on any lcore
+ * @retval -EINVAL Invalid service id
+ */
+int32_t __rte_experimental
+rte_service_may_be_active(uint32_t id);
+
+/**
  * Enable or disable the check for a service-core being mapped to the service.
  * An application can disable the check when takes the responsibility to run a
  * service itself using *rte_service_run_iter_on_app_lcore*.
@@ -363,6 +383,42 @@ int32_t rte_service_attr_get(uint32_t id, uint32_t attr_id,
  */
 int32_t rte_service_attr_reset_all(uint32_t id);
 
+/**
+ * Returns the number of times the service runner has looped.
+ */
+#define RTE_SERVICE_LCORE_ATTR_LOOPS 0
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Get an attribute from a service core.
+ *
+ * @param lcore Id of the service core.
+ * @param attr_id Id of the attribute to be retrieved.
+ * @param [out] attr_value Pointer to storage in which to write retrieved value.
+ * @retval 0 Success, the attribute value has been written to *attr_value*.
+ *         -EINVAL Invalid lcore, attr_id or attr_value was NULL.
+ *         -ENOTSUP lcore is not a service core.
+ */
+int32_t __rte_experimental
+rte_service_lcore_attr_get(uint32_t lcore, uint32_t attr_id,
+			   uint64_t *attr_value);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Reset all attribute values of a service core.
+ *
+ * @param lcore The service core to reset all the statistics of
+ * @retval 0 Successfully reset attributes
+ *         -EINVAL Invalid service id provided
+ *         -ENOTSUP lcore is not a service core.
+ */
+int32_t __rte_experimental
+rte_service_lcore_attr_reset_all(uint32_t lcore);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_eal/common/include/rte_tailq.h b/lib/librte_eal/common/include/rte_tailq.h
index 8dccaefc..9b01abb2 100644
--- a/lib/librte_eal/common/include/rte_tailq.h
+++ b/lib/librte_eal/common/include/rte_tailq.h
@@ -119,8 +119,7 @@ struct rte_tailq_head *rte_eal_tailq_lookup(const char *name);
 int rte_eal_tailq_register(struct rte_tailq_elem *t);
 
 #define EAL_REGISTER_TAILQ(t) \
-RTE_INIT(tailqinitfn_ ##t); \
-static void tailqinitfn_ ##t(void) \
+RTE_INIT(tailqinitfn_ ##t) \
 { \
 	if (rte_eal_tailq_register(&t) < 0) \
 		rte_panic("Cannot initialize tailq: %s\n", t.name); \
diff --git a/lib/librte_eal/common/include/rte_uuid.h b/lib/librte_eal/common/include/rte_uuid.h
new file mode 100644
index 00000000..2c846b5f
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_uuid.h
@@ -0,0 +1,129 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C) 1996, 1997, 1998 Theodore Ts'o.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, and the entire permission notice in its entirety,
+ *    including the disclaimer of warranties.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote
+ *    products derived from this software without specific prior
+ *    written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
+ * WHICH ARE HEREBY DISCLAIMED.	 IN NO EVENT SHALL THE AUTHOR BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+/**
+ * @file
+ *
+ * UUID related functions originally from libuuid
+ */
+
+#ifndef _RTE_UUID_H_
+#define _RTE_UUID_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+
+/**
+ * Struct describing a Universal Unique Identifer
+ */
+typedef unsigned char rte_uuid_t[16];
+
+/**
+ * Helper for defining UUID values for id tables.
+ */
+#define RTE_UUID_INIT(a, b, c, d, e) {		\
+	((a) >> 24) & 0xff, ((a) >> 16) & 0xff,	\
+	((a) >> 8) & 0xff, (a) & 0xff,		\
+	((b) >> 8) & 0xff, (b) & 0xff,		\
+	((c) >> 8) & 0xff, (c) & 0xff,		\
+	((d) >> 8) & 0xff, (d) & 0xff,		\
+	((e) >> 40) & 0xff, ((e) >> 32) & 0xff, \
+	((e) >> 24) & 0xff, ((e) >> 16) & 0xff, \
+	((e) >> 8) & 0xff, (e) & 0xff		\
+}
+
+/**
+ * Test if UUID is all zeros.
+ *
+ * @param uu
+ *    The uuid to check.
+ * @return
+ *    true if uuid is NULL value, false otherwise
+ */
+bool rte_uuid_is_null(const rte_uuid_t uu);
+
+/**
+ * Copy uuid.
+ *
+ * @param dst
+ *    Destination uuid
+ * @param src
+ *    Source uuid
+ */
+static inline void rte_uuid_copy(rte_uuid_t dst, const rte_uuid_t src)
+{
+	memcpy(dst, src, sizeof(rte_uuid_t));
+}
+
+/**
+ * Compare two UUID's
+ *
+ * @param a
+ *    A UUID to compare
+ * @param b
+ *    A UUID to compare
+ * @return
+ *   returns an integer less than, equal to, or greater than zero if UUID a is
+ *   is less than, equal, or greater than UUID b.
+ */
+int	rte_uuid_compare(const rte_uuid_t a, const rte_uuid_t b);
+
+/**
+ * Extract UUID from string
+ *
+ * @param in
+ *    Pointer to string of characters to convert
+ * @param uu
+ *    Destination UUID
+ * @return
+ *    Returns 0 on succes, and -1 if string is not a valid UUID.
+ */
+int	rte_uuid_parse(const char *in, rte_uuid_t uu);
+
+/**
+ * Convert UUID to string
+ *
+ * @param uu
+ *    UUID to format
+ * @param out
+ *    Resulting string buffer
+ * @param len
+ *    Sizeof the available string buffer
+ */
+#define RTE_UUID_STRLEN	(36 + 1)
+void	rte_uuid_unparse(const rte_uuid_t uu, char *out, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RTE_UUID_H */
diff --git a/lib/librte_eal/common/include/rte_version.h b/lib/librte_eal/common/include/rte_version.h
index 6e2a2362..7c6714a2 100644
--- a/lib/librte_eal/common/include/rte_version.h
+++ b/lib/librte_eal/common/include/rte_version.h
@@ -32,7 +32,7 @@ extern "C" {
 /**
  * Minor version/month number i.e. the mm in yy.mm.z
  */
-#define RTE_VER_MONTH 05
+#define RTE_VER_MONTH 8
 
 /**
  * Patch level number i.e. the z in yy.mm.z
diff --git a/lib/librte_eal/common/include/rte_vfio.h b/lib/librte_eal/common/include/rte_vfio.h
index f90972fa..5ca13fcc 100644
--- a/lib/librte_eal/common/include/rte_vfio.h
+++ b/lib/librte_eal/common/include/rte_vfio.h
@@ -179,7 +179,7 @@ rte_vfio_clear_group(int vfio_group_fd);
  *   0 if success.
  *   -1 on error.
  */
-int  __rte_experimental
+int
 rte_vfio_dma_map(uint64_t vaddr, uint64_t iova, uint64_t len);
 
 
@@ -200,7 +200,7 @@ rte_vfio_dma_map(uint64_t vaddr, uint64_t iova, uint64_t len);
  *   -1 on error.
  */
 
-int __rte_experimental
+int
 rte_vfio_dma_unmap(uint64_t vaddr, uint64_t iova, uint64_t len);
 /**
  * Parse IOMMU group number for a device
@@ -222,7 +222,7 @@ rte_vfio_dma_unmap(uint64_t vaddr, uint64_t iova, uint64_t len);
  *   0 for non-existent group or VFIO
  *  <0 for errors
  */
-int __rte_experimental
+int
 rte_vfio_get_group_num(const char *sysfs_base,
 		      const char *dev_addr, int *iommu_group_num);
 
@@ -236,7 +236,7 @@ rte_vfio_get_group_num(const char *sysfs_base,
  *  > 0 container fd
  *  < 0 for errors
  */
-int __rte_experimental
+int
 rte_vfio_get_container_fd(void);
 
 /**
@@ -252,13 +252,10 @@ rte_vfio_get_container_fd(void);
  *  > 0 group fd
  *  < 0 for errors
  */
-int __rte_experimental
+int
 rte_vfio_get_group_fd(int iommu_group_num);
 
 /**
- * @warning
- * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
- *
  * Create a new container for device binding.
  *
  * @note Any newly allocated DPDK memory will not be mapped into these
@@ -269,13 +266,10 @@ rte_vfio_get_group_fd(int iommu_group_num);
  *   the container fd if successful
  *   <0 if failed
  */
-int __rte_experimental
+int
 rte_vfio_container_create(void);
 
 /**
- * @warning
- * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
- *
  * Destroy the container, unbind all vfio groups within it.
  *
  * @param container_fd
@@ -285,13 +279,10 @@ rte_vfio_container_create(void);
  *    0 if successful
  *   <0 if failed
  */
-int __rte_experimental
+int
 rte_vfio_container_destroy(int container_fd);
 
 /**
- * @warning
- * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
- *
  * Bind a IOMMU group to a container.
  *
  * @param container_fd
@@ -304,13 +295,10 @@ rte_vfio_container_destroy(int container_fd);
  *   group fd if successful
  *   <0 if failed
  */
-int __rte_experimental
+int
 rte_vfio_container_group_bind(int container_fd, int iommu_group_num);
 
 /**
- * @warning
- * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
- *
  * Unbind a IOMMU group from a container.
  *
  * @param container_fd
@@ -323,13 +311,10 @@ rte_vfio_container_group_bind(int container_fd, int iommu_group_num);
  *    0 if successful
  *   <0 if failed
  */
-int __rte_experimental
+int
 rte_vfio_container_group_unbind(int container_fd, int iommu_group_num);
 
 /**
- * @warning
- * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
- *
  * Perform DMA mapping for devices in a container.
  *
  * @param container_fd
@@ -348,14 +333,11 @@ rte_vfio_container_group_unbind(int container_fd, int iommu_group_num);
  *    0 if successful
  *   <0 if failed
  */
-int __rte_experimental
+int
 rte_vfio_container_dma_map(int container_fd, uint64_t vaddr,
 		uint64_t iova, uint64_t len);
 
 /**
- * @warning
- * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
- *
  * Perform DMA unmapping for devices in a container.
  *
  * @param container_fd
@@ -374,7 +356,7 @@ rte_vfio_container_dma_map(int container_fd, uint64_t vaddr,
  *    0 if successful
  *   <0 if failed
  */
-int __rte_experimental
+int
 rte_vfio_container_dma_unmap(int container_fd, uint64_t vaddr,
 		uint64_t iova, uint64_t len);
 
diff --git a/lib/librte_eal/common/malloc_elem.c b/lib/librte_eal/common/malloc_elem.c
index 9bfe9b9b..e0a8ed15 100644
--- a/lib/librte_eal/common/malloc_elem.c
+++ b/lib/librte_eal/common/malloc_elem.c
@@ -18,10 +18,89 @@
 #include <rte_common.h>
 #include <rte_spinlock.h>
 
+#include "eal_internal_cfg.h"
 #include "eal_memalloc.h"
 #include "malloc_elem.h"
 #include "malloc_heap.h"
 
+size_t
+malloc_elem_find_max_iova_contig(struct malloc_elem *elem, size_t align)
+{
+	void *cur_page, *contig_seg_start, *page_end, *cur_seg_end;
+	void *data_start, *data_end;
+	rte_iova_t expected_iova;
+	struct rte_memseg *ms;
+	size_t page_sz, cur, max;
+
+	page_sz = (size_t)elem->msl->page_sz;
+	data_start = RTE_PTR_ADD(elem, MALLOC_ELEM_HEADER_LEN);
+	data_end = RTE_PTR_ADD(elem, elem->size - MALLOC_ELEM_TRAILER_LEN);
+	/* segment must start after header and with specified alignment */
+	contig_seg_start = RTE_PTR_ALIGN_CEIL(data_start, align);
+
+	/* if we're in IOVA as VA mode, or if we're in legacy mode with
+	 * hugepages, all elements are IOVA-contiguous.
+	 */
+	if (rte_eal_iova_mode() == RTE_IOVA_VA ||
+			(internal_config.legacy_mem && rte_eal_has_hugepages()))
+		return RTE_PTR_DIFF(data_end, contig_seg_start);
+
+	cur_page = RTE_PTR_ALIGN_FLOOR(contig_seg_start, page_sz);
+	ms = rte_mem_virt2memseg(cur_page, elem->msl);
+
+	/* do first iteration outside the loop */
+	page_end = RTE_PTR_ADD(cur_page, page_sz);
+	cur_seg_end = RTE_MIN(page_end, data_end);
+	cur = RTE_PTR_DIFF(cur_seg_end, contig_seg_start) -
+			MALLOC_ELEM_TRAILER_LEN;
+	max = cur;
+	expected_iova = ms->iova + page_sz;
+	/* memsegs are contiguous in memory */
+	ms++;
+
+	cur_page = RTE_PTR_ADD(cur_page, page_sz);
+
+	while (cur_page < data_end) {
+		page_end = RTE_PTR_ADD(cur_page, page_sz);
+		cur_seg_end = RTE_MIN(page_end, data_end);
+
+		/* reset start of contiguous segment if unexpected iova */
+		if (ms->iova != expected_iova) {
+			/* next contiguous segment must start at specified
+			 * alignment.
+			 */
+			contig_seg_start = RTE_PTR_ALIGN(cur_page, align);
+			/* new segment start may be on a different page, so find
+			 * the page and skip to next iteration to make sure
+			 * we're not blowing past data end.
+			 */
+			ms = rte_mem_virt2memseg(contig_seg_start, elem->msl);
+			cur_page = ms->addr;
+			/* don't trigger another recalculation */
+			expected_iova = ms->iova;
+			continue;
+		}
+		/* cur_seg_end ends on a page boundary or on data end. if we're
+		 * looking at data end, then malloc trailer is already included
+		 * in the calculations. if we're looking at page end, then we
+		 * know there's more data past this page and thus there's space
+		 * for malloc element trailer, so don't count it here.
+		 */
+		cur = RTE_PTR_DIFF(cur_seg_end, contig_seg_start);
+		/* update max if cur value is bigger */
+		if (cur > max)
+			max = cur;
+
+		/* move to next page */
+		cur_page = page_end;
+		expected_iova = ms->iova + page_sz;
+		/* memsegs are contiguous in memory */
+		ms++;
+	}
+
+	return max;
+}
+
 /*
  * Initialize a general malloc_elem header structure
  */
@@ -386,16 +465,18 @@ malloc_elem_join_adjacent_free(struct malloc_elem *elem)
 	if (elem->next != NULL && elem->next->state == ELEM_FREE &&
 			next_elem_is_adjacent(elem)) {
 		void *erase;
+		size_t erase_len;
 
 		/* we will want to erase the trailer and header */
 		erase = RTE_PTR_SUB(elem->next, MALLOC_ELEM_TRAILER_LEN);
+		erase_len = MALLOC_ELEM_OVERHEAD + elem->next->pad;
 
 		/* remove from free list, join to this one */
 		malloc_elem_free_list_remove(elem->next);
 		join_elem(elem, elem->next);
 
-		/* erase header and trailer */
-		memset(erase, 0, MALLOC_ELEM_OVERHEAD);
+		/* erase header, trailer and pad */
+		memset(erase, 0, erase_len);
 	}
 
 	/*
@@ -406,9 +487,11 @@ malloc_elem_join_adjacent_free(struct malloc_elem *elem)
 			prev_elem_is_adjacent(elem)) {
 		struct malloc_elem *new_elem;
 		void *erase;
+		size_t erase_len;
 
 		/* we will want to erase trailer and header */
 		erase = RTE_PTR_SUB(elem, MALLOC_ELEM_TRAILER_LEN);
+		erase_len = MALLOC_ELEM_OVERHEAD + elem->pad;
 
 		/* remove from free list, join to this one */
 		malloc_elem_free_list_remove(elem->prev);
@@ -416,8 +499,8 @@ malloc_elem_join_adjacent_free(struct malloc_elem *elem)
 		new_elem = elem->prev;
 		join_elem(new_elem, elem);
 
-		/* erase header and trailer */
-		memset(erase, 0, MALLOC_ELEM_OVERHEAD);
+		/* erase header, trailer and pad */
+		memset(erase, 0, erase_len);
 
 		elem = new_elem;
 	}
@@ -436,7 +519,7 @@ malloc_elem_free(struct malloc_elem *elem)
 	void *ptr;
 	size_t data_len;
 
-	ptr = RTE_PTR_ADD(elem, sizeof(*elem));
+	ptr = RTE_PTR_ADD(elem, MALLOC_ELEM_HEADER_LEN);
 	data_len = elem->size - MALLOC_ELEM_OVERHEAD;
 
 	elem = malloc_elem_join_adjacent_free(elem);
diff --git a/lib/librte_eal/common/malloc_elem.h b/lib/librte_eal/common/malloc_elem.h
index 7331af9c..e2bda4c0 100644
--- a/lib/librte_eal/common/malloc_elem.h
+++ b/lib/librte_eal/common/malloc_elem.h
@@ -179,4 +179,10 @@ malloc_elem_free_list_index(size_t size);
 void
 malloc_elem_free_list_insert(struct malloc_elem *elem);
 
+/*
+ * Find biggest IOVA-contiguous zone within an element with specified alignment.
+ */
+size_t
+malloc_elem_find_max_iova_contig(struct malloc_elem *elem, size_t align);
+
 #endif /* MALLOC_ELEM_H_ */
diff --git a/lib/librte_eal/common/malloc_heap.c b/lib/librte_eal/common/malloc_heap.c
index d6cf3af8..12aaf2d7 100644
--- a/lib/librte_eal/common/malloc_heap.c
+++ b/lib/librte_eal/common/malloc_heap.c
@@ -149,6 +149,52 @@ find_suitable_element(struct malloc_heap *heap, size_t size,
 }
 
 /*
+ * Iterates through the freelist for a heap to find a free element with the
+ * biggest size and requested alignment. Will also set size to whatever element
+ * size that was found.
+ * Returns null on failure, or pointer to element on success.
+ */
+static struct malloc_elem *
+find_biggest_element(struct malloc_heap *heap, size_t *size,
+		unsigned int flags, size_t align, bool contig)
+{
+	struct malloc_elem *elem, *max_elem = NULL;
+	size_t idx, max_size = 0;
+
+	for (idx = 0; idx < RTE_HEAP_NUM_FREELISTS; idx++) {
+		for (elem = LIST_FIRST(&heap->free_head[idx]);
+				!!elem; elem = LIST_NEXT(elem, free_list)) {
+			size_t cur_size;
+			if (!check_hugepage_sz(flags, elem->msl->page_sz))
+				continue;
+			if (contig) {
+				cur_size =
+					malloc_elem_find_max_iova_contig(elem,
+							align);
+			} else {
+				void *data_start = RTE_PTR_ADD(elem,
+						MALLOC_ELEM_HEADER_LEN);
+				void *data_end = RTE_PTR_ADD(elem, elem->size -
+						MALLOC_ELEM_TRAILER_LEN);
+				void *aligned = RTE_PTR_ALIGN_CEIL(data_start,
+						align);
+				/* check if aligned data start is beyond end */
+				if (aligned >= data_end)
+					continue;
+				cur_size = RTE_PTR_DIFF(data_end, aligned);
+			}
+			if (cur_size > max_size) {
+				max_size = cur_size;
+				max_elem = elem;
+			}
+		}
+	}
+
+	*size = max_size;
+	return max_elem;
+}
+
+/*
  * Main function to allocate a block of memory from the heap.
  * It locks the free list, scans it, and adds a new memseg if the
  * scan fails. Once the new memseg is added, it re-scans and should return
@@ -174,6 +220,26 @@ heap_alloc(struct malloc_heap *heap, const char *type __rte_unused, size_t size,
 	return elem == NULL ? NULL : (void *)(&elem[1]);
 }
 
+static void *
+heap_alloc_biggest(struct malloc_heap *heap, const char *type __rte_unused,
+		unsigned int flags, size_t align, bool contig)
+{
+	struct malloc_elem *elem;
+	size_t size;
+
+	align = RTE_CACHE_LINE_ROUNDUP(align);
+
+	elem = find_biggest_element(heap, &size, flags, align, contig);
+	if (elem != NULL) {
+		elem = malloc_elem_alloc(elem, size, align, 0, contig);
+
+		/* increase heap's count of allocated elements */
+		heap->alloc_count++;
+	}
+
+	return elem == NULL ? NULL : (void *)(&elem[1]);
+}
+
 /* this function is exposed in malloc_mp.h */
 void
 rollback_expand_heap(struct rte_memseg **ms, int n_segs,
@@ -575,6 +641,66 @@ malloc_heap_alloc(const char *type, size_t size, int socket_arg,
 	return NULL;
 }
 
+static void *
+heap_alloc_biggest_on_socket(const char *type, int socket, unsigned int flags,
+		size_t align, bool contig)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	struct malloc_heap *heap = &mcfg->malloc_heaps[socket];
+	void *ret;
+
+	rte_spinlock_lock(&(heap->lock));
+
+	align = align == 0 ? 1 : align;
+
+	ret = heap_alloc_biggest(heap, type, flags, align, contig);
+
+	rte_spinlock_unlock(&(heap->lock));
+
+	return ret;
+}
+
+void *
+malloc_heap_alloc_biggest(const char *type, int socket_arg, unsigned int flags,
+		size_t align, bool contig)
+{
+	int socket, i, cur_socket;
+	void *ret;
+
+	/* return NULL if align is not power-of-2 */
+	if ((align && !rte_is_power_of_2(align)))
+		return NULL;
+
+	if (!rte_eal_has_hugepages())
+		socket_arg = SOCKET_ID_ANY;
+
+	if (socket_arg == SOCKET_ID_ANY)
+		socket = malloc_get_numa_socket();
+	else
+		socket = socket_arg;
+
+	/* Check socket parameter */
+	if (socket >= RTE_MAX_NUMA_NODES)
+		return NULL;
+
+	ret = heap_alloc_biggest_on_socket(type, socket, flags, align,
+			contig);
+	if (ret != NULL || socket_arg != SOCKET_ID_ANY)
+		return ret;
+
+	/* try other heaps */
+	for (i = 0; i < (int) rte_socket_count(); i++) {
+		cur_socket = rte_socket_id_by_idx(i);
+		if (cur_socket == socket)
+			continue;
+		ret = heap_alloc_biggest_on_socket(type, cur_socket, flags,
+				align, contig);
+		if (ret != NULL)
+			return ret;
+	}
+	return NULL;
+}
+
 /* this function is exposed in malloc_mp.h */
 int
 malloc_heap_free_pages(void *aligned_start, size_t aligned_len)
diff --git a/lib/librte_eal/common/malloc_heap.h b/lib/librte_eal/common/malloc_heap.h
index 03b80141..f52cb555 100644
--- a/lib/librte_eal/common/malloc_heap.h
+++ b/lib/librte_eal/common/malloc_heap.h
@@ -29,6 +29,10 @@ void *
 malloc_heap_alloc(const char *type, size_t size, int socket, unsigned int flags,
 		size_t align, size_t bound, bool contig);
 
+void *
+malloc_heap_alloc_biggest(const char *type, int socket, unsigned int flags,
+		size_t align, bool contig);
+
 int
 malloc_heap_free(struct malloc_elem *elem);
 
diff --git a/lib/librte_eal/common/meson.build b/lib/librte_eal/common/meson.build
index 8a3dcfee..56005bea 100644
--- a/lib/librte_eal/common/meson.build
+++ b/lib/librte_eal/common/meson.build
@@ -8,6 +8,7 @@ common_objs = []
 common_sources = files(
 	'eal_common_bus.c',
 	'eal_common_cpuflags.c',
+	'eal_common_class.c',
 	'eal_common_devargs.c',
 	'eal_common_dev.c',
 	'eal_common_errno.c',
@@ -25,6 +26,7 @@ common_sources = files(
 	'eal_common_tailqs.c',
 	'eal_common_thread.c',
 	'eal_common_timer.c',
+	'eal_common_uuid.c',
 	'malloc_elem.c',
 	'malloc_heap.c',
 	'malloc_mp.c',
@@ -46,6 +48,7 @@ common_headers = files(
 	'include/rte_branch_prediction.h',
 	'include/rte_bus.h',
 	'include/rte_bitmap.h',
+	'include/rte_class.h',
 	'include/rte_common.h',
 	'include/rte_debug.h',
 	'include/rte_devargs.h',
@@ -75,6 +78,7 @@ common_headers = files(
 	'include/rte_string_fns.h',
 	'include/rte_tailq.h',
 	'include/rte_time.h',
+	'include/rte_uuid.h',
 	'include/rte_version.h')
 
 # special case install the generic headers, since they go in a subdir
diff --git a/lib/librte_eal/common/rte_service.c b/lib/librte_eal/common/rte_service.c
index 73507aac..8767c722 100644
--- a/lib/librte_eal/common/rte_service.c
+++ b/lib/librte_eal/common/rte_service.c
@@ -52,6 +52,7 @@ struct rte_service_spec_impl {
 	rte_atomic32_t num_mapped_cores;
 	uint64_t calls;
 	uint64_t cycles_spent;
+	uint8_t active_on_lcore[RTE_MAX_LCORE];
 } __rte_cache_aligned;
 
 /* the internal values of a service core */
@@ -61,7 +62,7 @@ struct core_state {
 	uint8_t runstate; /* running or stopped */
 	uint8_t is_service_core; /* set if core is currently a service core */
 
-	/* extreme statistics */
+	uint64_t loops;
 	uint64_t calls_per_service[RTE_SERVICE_NUM_MAX];
 } __rte_cache_aligned;
 
@@ -347,15 +348,19 @@ rte_service_runner_do_callback(struct rte_service_spec_impl *s,
 
 
 static inline int32_t
-service_run(uint32_t i, struct core_state *cs, uint64_t service_mask)
+service_run(uint32_t i, int lcore, struct core_state *cs, uint64_t service_mask)
 {
 	if (!service_valid(i))
 		return -EINVAL;
 	struct rte_service_spec_impl *s = &rte_services[i];
 	if (s->comp_runstate != RUNSTATE_RUNNING ||
 			s->app_runstate != RUNSTATE_RUNNING ||
-			!(service_mask & (UINT64_C(1) << i)))
+			!(service_mask & (UINT64_C(1) << i))) {
+		s->active_on_lcore[lcore] = 0;
 		return -ENOEXEC;
+	}
+
+	s->active_on_lcore[lcore] = 1;
 
 	/* check do we need cmpset, if MT safe or <= 1 core
 	 * mapped, atomic ops are not required.
@@ -374,6 +379,25 @@ service_run(uint32_t i, struct core_state *cs, uint64_t service_mask)
 	return 0;
 }
 
+int32_t __rte_experimental
+rte_service_may_be_active(uint32_t id)
+{
+	uint32_t ids[RTE_MAX_LCORE] = {0};
+	struct rte_service_spec_impl *s = &rte_services[id];
+	int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE);
+	int i;
+
+	if (!service_valid(id))
+		return -EINVAL;
+
+	for (i = 0; i < lcore_count; i++) {
+		if (s->active_on_lcore[ids[i]])
+			return 1;
+	}
+
+	return 0;
+}
+
 int32_t rte_service_run_iter_on_app_lcore(uint32_t id,
 		uint32_t serialize_mt_unsafe)
 {
@@ -398,7 +422,7 @@ int32_t rte_service_run_iter_on_app_lcore(uint32_t id,
 		return -EBUSY;
 	}
 
-	int ret = service_run(id, cs, UINT64_MAX);
+	int ret = service_run(id, rte_lcore_id(), cs, UINT64_MAX);
 
 	if (serialize_mt_unsafe)
 		rte_atomic32_dec(&s->num_mapped_cores);
@@ -419,9 +443,11 @@ rte_service_runner_func(void *arg)
 
 		for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
 			/* return value ignored as no change to code flow */
-			service_run(i, cs, service_mask);
+			service_run(i, lcore, cs, service_mask);
 		}
 
+		cs->loops++;
+
 		rte_smp_rmb();
 	}
 
@@ -729,6 +755,28 @@ rte_service_attr_get(uint32_t id, uint32_t attr_id, uint32_t *attr_value)
 	}
 }
 
+int32_t __rte_experimental
+rte_service_lcore_attr_get(uint32_t lcore, uint32_t attr_id,
+			   uint64_t *attr_value)
+{
+	struct core_state *cs;
+
+	if (lcore >= RTE_MAX_LCORE || !attr_value)
+		return -EINVAL;
+
+	cs = &lcore_states[lcore];
+	if (!cs->is_service_core)
+		return -ENOTSUP;
+
+	switch (attr_id) {
+	case RTE_SERVICE_LCORE_ATTR_LOOPS:
+		*attr_value = cs->loops;
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
 static void
 rte_service_dump_one(FILE *f, struct rte_service_spec_impl *s,
 		     uint64_t all_cycles, uint32_t reset)
@@ -764,6 +812,23 @@ rte_service_attr_reset_all(uint32_t id)
 	return 0;
 }
 
+int32_t __rte_experimental
+rte_service_lcore_attr_reset_all(uint32_t lcore)
+{
+	struct core_state *cs;
+
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	cs = &lcore_states[lcore];
+	if (!cs->is_service_core)
+		return -ENOTSUP;
+
+	cs->loops = 0;
+
+	return 0;
+}
+
 static void
 service_dump_calls_per_lcore(FILE *f, uint32_t lcore, uint32_t reset)
 {
diff --git a/lib/librte_eal/linuxapp/eal/Makefile b/lib/librte_eal/linuxapp/eal/Makefile
index 3719ec9d..fd92c75c 100644
--- a/lib/librte_eal/linuxapp/eal/Makefile
+++ b/lib/librte_eal/linuxapp/eal/Makefile
@@ -10,7 +10,7 @@ ARCH_DIR ?= $(RTE_ARCH)
 EXPORT_MAP := ../../rte_eal_version.map
 VPATH += $(RTE_SDK)/lib/librte_eal/common/arch/$(ARCH_DIR)
 
-LIBABIVER := 7
+LIBABIVER := 8
 
 VPATH += $(RTE_SDK)/lib/librte_eal/common
 
@@ -24,6 +24,7 @@ LDLIBS += -ldl
 LDLIBS += -lpthread
 LDLIBS += -lgcc_s
 LDLIBS += -lrt
+LDLIBS += -lrte_kvargs
 ifeq ($(CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES),y)
 LDLIBS += -lnuma
 endif
@@ -60,12 +61,14 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_hypervisor.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_string_fns.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_hexdump.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_devargs.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_class.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_bus.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_dev.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_options.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_thread.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_proc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_fbarray.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_uuid.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_heap.c
diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 8655b869..e59ac657 100644
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -155,22 +155,12 @@ eal_get_runtime_dir(void)
 }
 
 /* Return user provided mbuf pool ops name */
-const char * __rte_experimental
+const char *
 rte_eal_mbuf_user_pool_ops(void)
 {
 	return internal_config.user_mbuf_pool_ops_name;
 }
 
-/* Return mbuf pool ops name */
-const char *
-rte_eal_mbuf_default_mempool_ops(void)
-{
-	if (internal_config.user_mbuf_pool_ops_name == NULL)
-		return RTE_MBUF_DEFAULT_MEMPOOL_OPS;
-
-	return internal_config.user_mbuf_pool_ops_name;
-}
-
 /* Return a pointer to the configuration structure */
 struct rte_config *
 rte_eal_get_configuration(void)
@@ -344,12 +334,17 @@ eal_proc_type_detect(void)
 	enum rte_proc_type_t ptype = RTE_PROC_PRIMARY;
 	const char *pathname = eal_runtime_config_path();
 
-	/* if we can open the file but not get a write-lock we are a secondary
-	 * process. NOTE: if we get a file handle back, we keep that open
-	 * and don't close it to prevent a race condition between multiple opens */
-	if (((mem_cfg_fd = open(pathname, O_RDWR)) >= 0) &&
-			(fcntl(mem_cfg_fd, F_SETLK, &wr_lock) < 0))
-		ptype = RTE_PROC_SECONDARY;
+	/* if there no shared config, there can be no secondary processes */
+	if (!internal_config.no_shconf) {
+		/* if we can open the file but not get a write-lock we are a
+		 * secondary process. NOTE: if we get a file handle back, we
+		 * keep that open and don't close it to prevent a race condition
+		 * between multiple opens.
+		 */
+		if (((mem_cfg_fd = open(pathname, O_RDWR)) >= 0) &&
+				(fcntl(mem_cfg_fd, F_SETLK, &wr_lock) < 0))
+			ptype = RTE_PROC_SECONDARY;
+	}
 
 	RTE_LOG(INFO, EAL, "Auto-detected process type: %s\n",
 			ptype == RTE_PROC_PRIMARY ? "PRIMARY" : "SECONDARY");
@@ -405,6 +400,7 @@ eal_usage(const char *prgname)
 	eal_common_usage();
 	printf("EAL Linux options:\n"
 	       "  --"OPT_SOCKET_MEM"        Memory to allocate on sockets (comma separated values)\n"
+	       "  --"OPT_SOCKET_LIMIT"      Limit memory allocation on sockets (comma separated values)\n"
 	       "  --"OPT_HUGE_DIR"          Directory where hugetlbfs is mounted\n"
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_BASE_VIRTADDR"     Base virtual address\n"
@@ -434,46 +430,45 @@ rte_set_application_usage_hook( rte_usage_hook_t usage_func )
 }
 
 static int
-eal_parse_socket_mem(char *socket_mem)
+eal_parse_socket_arg(char *strval, volatile uint64_t *socket_arg)
 {
 	char * arg[RTE_MAX_NUMA_NODES];
 	char *end;
 	int arg_num, i, len;
 	uint64_t total_mem = 0;
 
-	len = strnlen(socket_mem, SOCKET_MEM_STRLEN);
+	len = strnlen(strval, SOCKET_MEM_STRLEN);
 	if (len == SOCKET_MEM_STRLEN) {
 		RTE_LOG(ERR, EAL, "--socket-mem is too long\n");
 		return -1;
 	}
 
 	/* all other error cases will be caught later */
-	if (!isdigit(socket_mem[len-1]))
+	if (!isdigit(strval[len-1]))
 		return -1;
 
 	/* split the optarg into separate socket values */
-	arg_num = rte_strsplit(socket_mem, len,
+	arg_num = rte_strsplit(strval, len,
 			arg, RTE_MAX_NUMA_NODES, ',');
 
 	/* if split failed, or 0 arguments */
 	if (arg_num <= 0)
 		return -1;
 
-	internal_config.force_sockets = 1;
-
 	/* parse each defined socket option */
 	errno = 0;
 	for (i = 0; i < arg_num; i++) {
+		uint64_t val;
 		end = NULL;
-		internal_config.socket_mem[i] = strtoull(arg[i], &end, 10);
+		val = strtoull(arg[i], &end, 10);
 
 		/* check for invalid input */
 		if ((errno != 0)  ||
 				(arg[i][0] == '\0') || (end == NULL) || (*end != '\0'))
 			return -1;
-		internal_config.socket_mem[i] *= 1024ULL;
-		internal_config.socket_mem[i] *= 1024ULL;
-		total_mem += internal_config.socket_mem[i];
+		val <<= 20;
+		total_mem += val;
+		socket_arg[i] = val;
 	}
 
 	/* check if we have a positive amount of total memory */
@@ -621,13 +616,27 @@ eal_parse_args(int argc, char **argv)
 			break;
 
 		case OPT_SOCKET_MEM_NUM:
-			if (eal_parse_socket_mem(optarg) < 0) {
+			if (eal_parse_socket_arg(optarg,
+					internal_config.socket_mem) < 0) {
 				RTE_LOG(ERR, EAL, "invalid parameters for --"
 						OPT_SOCKET_MEM "\n");
 				eal_usage(prgname);
 				ret = -1;
 				goto out;
 			}
+			internal_config.force_sockets = 1;
+			break;
+
+		case OPT_SOCKET_LIMIT_NUM:
+			if (eal_parse_socket_arg(optarg,
+					internal_config.socket_limit) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_SOCKET_LIMIT "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			internal_config.force_socket_limits = 1;
 			break;
 
 		case OPT_BASE_VIRTADDR_NUM:
@@ -678,6 +687,14 @@ eal_parse_args(int argc, char **argv)
 		}
 	}
 
+	/* create runtime data directory */
+	if (internal_config.no_shconf == 0 &&
+			eal_create_runtime_dir() < 0) {
+		RTE_LOG(ERR, EAL, "Cannot create runtime directory\n");
+		ret = -1;
+		goto out;
+	}
+
 	if (eal_adjust_config(&internal_config) != 0) {
 		ret = -1;
 		goto out;
@@ -817,13 +834,6 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
-	/* create runtime data directory */
-	if (eal_create_runtime_dir() < 0) {
-		rte_eal_init_alert("Cannot create runtime directory\n");
-		rte_errno = EACCES;
-		return -1;
-	}
-
 	if (eal_plugins_init() < 0) {
 		rte_eal_init_alert("Cannot init plugins\n");
 		rte_errno = EINVAL;
@@ -839,6 +849,11 @@ rte_eal_init(int argc, char **argv)
 
 	rte_config_init();
 
+	if (rte_eal_intr_init() < 0) {
+		rte_eal_init_alert("Cannot init interrupt-handling thread\n");
+		return -1;
+	}
+
 	/* Put mp channel init before bus scan so that we can init the vdev
 	 * bus through mp channel in the secondary process before the bus scan.
 	 */
@@ -968,11 +983,6 @@ rte_eal_init(int argc, char **argv)
 		rte_config.master_lcore, (int)thread_id, cpuset,
 		ret == 0 ? "" : "...");
 
-	if (rte_eal_intr_init() < 0) {
-		rte_eal_init_alert("Cannot init interrupt-handling thread\n");
-		return -1;
-	}
-
 	RTE_LCORE_FOREACH_SLAVE(i) {
 
 		/*
@@ -1044,9 +1054,26 @@ rte_eal_init(int argc, char **argv)
 	return fctret;
 }
 
+static int
+mark_freeable(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
+		void *arg __rte_unused)
+{
+	/* ms is const, so find this memseg */
+	struct rte_memseg *found = rte_mem_virt2memseg(ms->addr, msl);
+
+	found->flags &= ~RTE_MEMSEG_FLAG_DO_NOT_FREE;
+
+	return 0;
+}
+
 int __rte_experimental
 rte_eal_cleanup(void)
 {
+	/* if we're in a primary process, we need to mark hugepages as freeable
+	 * so that finalization can release them back to the system.
+	 */
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+		rte_memseg_walk(mark_freeable, NULL);
 	rte_service_finalize();
 	return 0;
 }
diff --git a/lib/librte_eal/linuxapp/eal/eal_alarm.c b/lib/librte_eal/linuxapp/eal/eal_alarm.c
index c115e823..391d2a65 100644
--- a/lib/librte_eal/linuxapp/eal/eal_alarm.c
+++ b/lib/librte_eal/linuxapp/eal/eal_alarm.c
@@ -19,7 +19,6 @@
 #include <rte_launch.h>
 #include <rte_lcore.h>
 #include <rte_errno.h>
-#include <rte_malloc.h>
 #include <rte_spinlock.h>
 #include <eal_private.h>
 
@@ -91,7 +90,7 @@ eal_alarm_callback(void *arg __rte_unused)
 		rte_spinlock_lock(&alarm_list_lk);
 
 		LIST_REMOVE(ap, next);
-		rte_free(ap);
+		free(ap);
 	}
 
 	if (!LIST_EMPTY(&alarm_list)) {
@@ -122,7 +121,7 @@ rte_eal_alarm_set(uint64_t us, rte_eal_alarm_callback cb_fn, void *cb_arg)
 	if (us < 1 || us > (UINT64_MAX - US_PER_S) || cb_fn == NULL)
 		return -EINVAL;
 
-	new_alarm = rte_zmalloc(NULL, sizeof(*new_alarm), 0);
+	new_alarm = calloc(1, sizeof(*new_alarm));
 	if (new_alarm == NULL)
 		return -ENOMEM;
 
@@ -196,7 +195,7 @@ rte_eal_alarm_cancel(rte_eal_alarm_callback cb_fn, void *cb_arg)
 
 			if (ap->executing == 0) {
 				LIST_REMOVE(ap, next);
-				rte_free(ap);
+				free(ap);
 				count++;
 			} else {
 				/* If calling from other context, mark that alarm is executing
@@ -220,7 +219,7 @@ rte_eal_alarm_cancel(rte_eal_alarm_callback cb_fn, void *cb_arg)
 
 				if (ap->executing == 0) {
 					LIST_REMOVE(ap, next);
-					rte_free(ap);
+					free(ap);
 					count++;
 					ap = ap_prev;
 				} else if (pthread_equal(ap->executing_id, pthread_self()) == 0)
diff --git a/lib/librte_eal/linuxapp/eal/eal_hugepage_info.c b/lib/librte_eal/linuxapp/eal/eal_hugepage_info.c
index 7eca711b..3a7d4b22 100644
--- a/lib/librte_eal/linuxapp/eal/eal_hugepage_info.c
+++ b/lib/librte_eal/linuxapp/eal/eal_hugepage_info.c
@@ -18,6 +18,8 @@
 #include <sys/queue.h>
 #include <sys/stat.h>
 
+#include <linux/mman.h> /* for hugetlb-related flags */
+
 #include <rte_memory.h>
 #include <rte_eal.h>
 #include <rte_launch.h>
@@ -313,11 +315,49 @@ compare_hpi(const void *a, const void *b)
 	return hpi_b->hugepage_sz - hpi_a->hugepage_sz;
 }
 
+static void
+calc_num_pages(struct hugepage_info *hpi, struct dirent *dirent)
+{
+	uint64_t total_pages = 0;
+	unsigned int i;
+
+	/*
+	 * first, try to put all hugepages into relevant sockets, but
+	 * if first attempts fails, fall back to collecting all pages
+	 * in one socket and sorting them later
+	 */
+	total_pages = 0;
+	/* we also don't want to do this for legacy init */
+	if (!internal_config.legacy_mem)
+		for (i = 0; i < rte_socket_count(); i++) {
+			int socket = rte_socket_id_by_idx(i);
+			unsigned int num_pages =
+					get_num_hugepages_on_node(
+						dirent->d_name, socket);
+			hpi->num_pages[socket] = num_pages;
+			total_pages += num_pages;
+		}
+	/*
+	 * we failed to sort memory from the get go, so fall
+	 * back to old way
+	 */
+	if (total_pages == 0) {
+		hpi->num_pages[0] = get_num_hugepages(dirent->d_name);
+
+#ifndef RTE_ARCH_64
+		/* for 32-bit systems, limit number of hugepages to
+		 * 1GB per page size */
+		hpi->num_pages[0] = RTE_MIN(hpi->num_pages[0],
+				RTE_PGSIZE_1G / hpi->hugepage_sz);
+#endif
+	}
+}
+
 static int
 hugepage_info_init(void)
 {	const char dirent_start_text[] = "hugepages-";
 	const size_t dirent_start_len = sizeof(dirent_start_text) - 1;
-	unsigned int i, total_pages, num_sizes = 0;
+	unsigned int i, num_sizes = 0;
 	DIR *dir;
 	struct dirent *dirent;
 
@@ -355,6 +395,22 @@ hugepage_info_init(void)
 					"%" PRIu64 " reserved, but no mounted "
 					"hugetlbfs found for that size\n",
 					num_pages, hpi->hugepage_sz);
+			/* if we have kernel support for reserving hugepages
+			 * through mmap, and we're in in-memory mode, treat this
+			 * page size as valid. we cannot be in legacy mode at
+			 * this point because we've checked this earlier in the
+			 * init process.
+			 */
+#ifdef MAP_HUGE_SHIFT
+			if (internal_config.in_memory) {
+				RTE_LOG(DEBUG, EAL, "In-memory mode enabled, "
+					"hugepages of size %" PRIu64 " bytes "
+					"will be allocated anonymously\n",
+					hpi->hugepage_sz);
+				calc_num_pages(hpi, dirent);
+				num_sizes++;
+			}
+#endif
 			continue;
 		}
 
@@ -371,35 +427,7 @@ hugepage_info_init(void)
 		if (clear_hugedir(hpi->hugedir) == -1)
 			break;
 
-		/*
-		 * first, try to put all hugepages into relevant sockets, but
-		 * if first attempts fails, fall back to collecting all pages
-		 * in one socket and sorting them later
-		 */
-		total_pages = 0;
-		/* we also don't want to do this for legacy init */
-		if (!internal_config.legacy_mem)
-			for (i = 0; i < rte_socket_count(); i++) {
-				int socket = rte_socket_id_by_idx(i);
-				unsigned int num_pages =
-						get_num_hugepages_on_node(
-							dirent->d_name, socket);
-				hpi->num_pages[socket] = num_pages;
-				total_pages += num_pages;
-			}
-		/*
-		 * we failed to sort memory from the get go, so fall
-		 * back to old way
-		 */
-		if (total_pages == 0)
-			hpi->num_pages[0] = get_num_hugepages(dirent->d_name);
-
-#ifndef RTE_ARCH_64
-		/* for 32-bit systems, limit number of hugepages to
-		 * 1GB per page size */
-		hpi->num_pages[0] = RTE_MIN(hpi->num_pages[0],
-					    RTE_PGSIZE_1G / hpi->hugepage_sz);
-#endif
+		calc_num_pages(hpi, dirent);
 
 		num_sizes++;
 	}
@@ -423,8 +451,7 @@ hugepage_info_init(void)
 
 		for (j = 0; j < RTE_MAX_NUMA_NODES; j++)
 			num_pages += hpi->num_pages[j];
-		if (strnlen(hpi->hugedir, sizeof(hpi->hugedir)) != 0 &&
-				num_pages > 0)
+		if (num_pages > 0)
 			return 0;
 	}
 
@@ -446,6 +473,10 @@ eal_hugepage_info_init(void)
 	if (hugepage_info_init() < 0)
 		return -1;
 
+	/* for no shared files mode, we're done */
+	if (internal_config.no_shconf)
+		return 0;
+
 	hpi = &internal_config.hugepage_info[0];
 
 	tmp_hpi = create_shared_memory(eal_hugepage_info_path(),
diff --git a/lib/librte_eal/linuxapp/eal/eal_interrupts.c b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
index 056d41c1..4076c6d6 100644
--- a/lib/librte_eal/linuxapp/eal/eal_interrupts.c
+++ b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
@@ -30,7 +30,6 @@
 #include <rte_branch_prediction.h>
 #include <rte_debug.h>
 #include <rte_log.h>
-#include <rte_malloc.h>
 #include <rte_errno.h>
 #include <rte_spinlock.h>
 #include <rte_pause.h>
@@ -405,8 +404,7 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 
 	/* allocate a new interrupt callback entity */
-	callback = rte_zmalloc("interrupt callback list",
-				sizeof(*callback), 0);
+	callback = calloc(1, sizeof(*callback));
 	if (callback == NULL) {
 		RTE_LOG(ERR, EAL, "Can not allocate memory\n");
 		return -ENOMEM;
@@ -420,7 +418,7 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	TAILQ_FOREACH(src, &intr_sources, next) {
 		if (src->intr_handle.fd == intr_handle->fd) {
 			/* we had no interrupts for this */
-			if TAILQ_EMPTY(&src->callbacks)
+			if (TAILQ_EMPTY(&src->callbacks))
 				wake_thread = 1;
 
 			TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
@@ -431,10 +429,10 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 
 	/* no existing callbacks for this - add new source */
 	if (src == NULL) {
-		if ((src = rte_zmalloc("interrupt source list",
-				sizeof(*src), 0)) == NULL) {
+		src = calloc(1, sizeof(*src));
+		if (src == NULL) {
 			RTE_LOG(ERR, EAL, "Can not allocate memory\n");
-			rte_free(callback);
+			free(callback);
 			ret = -ENOMEM;
 		} else {
 			src->intr_handle = *intr_handle;
@@ -501,7 +499,7 @@ rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
 					cb->cb_arg == cb_arg)) {
 				TAILQ_REMOVE(&src->callbacks, cb, next);
-				rte_free(cb);
+				free(cb);
 				ret++;
 			}
 		}
@@ -509,7 +507,7 @@ rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 		/* all callbacks for that source are removed. */
 		if (TAILQ_EMPTY(&src->callbacks)) {
 			TAILQ_REMOVE(&intr_sources, src, next);
-			rte_free(src);
+			free(src);
 		}
 	}
 
diff --git a/lib/librte_eal/linuxapp/eal/eal_log.c b/lib/librte_eal/linuxapp/eal/eal_log.c
index ff145884..9d02dddb 100644
--- a/lib/librte_eal/linuxapp/eal/eal_log.c
+++ b/lib/librte_eal/linuxapp/eal/eal_log.c
@@ -25,25 +25,14 @@
 static ssize_t
 console_log_write(__attribute__((unused)) void *c, const char *buf, size_t size)
 {
-	char copybuf[BUFSIZ + 1];
 	ssize_t ret;
-	uint32_t loglevel;
 
 	/* write on stdout */
 	ret = fwrite(buf, 1, size, stdout);
 	fflush(stdout);
 
-	/* truncate message if too big (should not happen) */
-	if (size > BUFSIZ)
-		size = BUFSIZ;
-
 	/* Syslog error levels are from 0 to 7, so subtract 1 to convert */
-	loglevel = rte_log_cur_msg_loglevel() - 1;
-	memcpy(copybuf, buf, size);
-	copybuf[size] = '\0';
-
-	/* write on syslog too */
-	syslog(loglevel, "%s", copybuf);
+	syslog(rte_log_cur_msg_loglevel() - 1, "%.*s", (int)size, buf);
 
 	return ret;
 }
diff --git a/lib/librte_eal/linuxapp/eal/eal_memalloc.c b/lib/librte_eal/linuxapp/eal/eal_memalloc.c
index 8c11f98c..aa95551a 100644
--- a/lib/librte_eal/linuxapp/eal/eal_memalloc.c
+++ b/lib/librte_eal/linuxapp/eal/eal_memalloc.c
@@ -28,6 +28,7 @@
 #include <numaif.h>
 #endif
 #include <linux/falloc.h>
+#include <linux/mman.h> /* for hugetlb-related mmap flags */
 
 #include <rte_common.h>
 #include <rte_log.h>
@@ -39,6 +40,16 @@
 #include "eal_filesystem.h"
 #include "eal_internal_cfg.h"
 #include "eal_memalloc.h"
+#include "eal_private.h"
+
+const int anonymous_hugepages_supported =
+#ifdef MAP_HUGE_SHIFT
+		1;
+#define RTE_MAP_HUGE_SHIFT MAP_HUGE_SHIFT
+#else
+		0;
+#define RTE_MAP_HUGE_SHIFT 26
+#endif
 
 /*
  * not all kernel version support fallocate on hugetlbfs, so fall back to
@@ -171,32 +182,6 @@ get_file_size(int fd)
 	return st.st_size;
 }
 
-/* we cannot use rte_memseg_list_walk() here because we will be holding a
- * write lock whenever we enter every function in this file, however copying
- * the same iteration code everywhere is not ideal as well. so, use a lockless
- * copy of memseg list walk here.
- */
-static int
-memseg_list_walk_thread_unsafe(rte_memseg_list_walk_t func, void *arg)
-{
-	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	int i, ret = 0;
-
-	for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
-		struct rte_memseg_list *msl = &mcfg->memsegs[i];
-
-		if (msl->base_va == NULL)
-			continue;
-
-		ret = func(msl, arg);
-		if (ret < 0)
-			return -1;
-		if (ret > 0)
-			return 1;
-	}
-	return 0;
-}
-
 /* returns 1 on successful lock, 0 on unsuccessful lock, -1 on error */
 static int lock(int fd, int type)
 {
@@ -486,45 +471,84 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 	int cur_socket_id = 0;
 #endif
 	uint64_t map_offset;
+	rte_iova_t iova;
+	void *va;
 	char path[PATH_MAX];
 	int ret = 0;
 	int fd;
 	size_t alloc_sz;
-
-	/* takes out a read lock on segment or segment list */
-	fd = get_seg_fd(path, sizeof(path), hi, list_idx, seg_idx);
-	if (fd < 0) {
-		RTE_LOG(ERR, EAL, "Couldn't get fd on hugepage file\n");
-		return -1;
-	}
+	int flags;
+	void *new_addr;
 
 	alloc_sz = hi->hugepage_sz;
-	if (internal_config.single_file_segments) {
-		map_offset = seg_idx * alloc_sz;
-		ret = resize_hugefile(fd, path, list_idx, seg_idx, map_offset,
-				alloc_sz, true);
-		if (ret < 0)
-			goto resized;
-	} else {
+	if (!internal_config.single_file_segments &&
+			internal_config.in_memory &&
+			anonymous_hugepages_supported) {
+		int log2, flags;
+
+		log2 = rte_log2_u32(alloc_sz);
+		/* as per mmap() manpage, all page sizes are log2 of page size
+		 * shifted by MAP_HUGE_SHIFT
+		 */
+		flags = (log2 << RTE_MAP_HUGE_SHIFT) | MAP_HUGETLB | MAP_FIXED |
+				MAP_PRIVATE | MAP_ANONYMOUS;
+		fd = -1;
+		va = mmap(addr, alloc_sz, PROT_READ | PROT_WRITE, flags, -1, 0);
+
+		/* single-file segments codepath will never be active because
+		 * in-memory mode is incompatible with it and it's stopped at
+		 * EAL initialization stage, however the compiler doesn't know
+		 * that and complains about map_offset being used uninitialized
+		 * on failure codepaths while having in-memory mode enabled. so,
+		 * assign a value here.
+		 */
 		map_offset = 0;
-		if (ftruncate(fd, alloc_sz) < 0) {
-			RTE_LOG(DEBUG, EAL, "%s(): ftruncate() failed: %s\n",
-				__func__, strerror(errno));
-			goto resized;
+	} else {
+		/* takes out a read lock on segment or segment list */
+		fd = get_seg_fd(path, sizeof(path), hi, list_idx, seg_idx);
+		if (fd < 0) {
+			RTE_LOG(ERR, EAL, "Couldn't get fd on hugepage file\n");
+			return -1;
 		}
-	}
 
-	/*
-	 * map the segment, and populate page tables, the kernel fills this
-	 * segment with zeros if it's a new page.
-	 */
-	void *va = mmap(addr, alloc_sz, PROT_READ | PROT_WRITE,
-			MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd, map_offset);
+		if (internal_config.single_file_segments) {
+			map_offset = seg_idx * alloc_sz;
+			ret = resize_hugefile(fd, path, list_idx, seg_idx,
+					map_offset, alloc_sz, true);
+			if (ret < 0)
+				goto resized;
+		} else {
+			map_offset = 0;
+			if (ftruncate(fd, alloc_sz) < 0) {
+				RTE_LOG(DEBUG, EAL, "%s(): ftruncate() failed: %s\n",
+					__func__, strerror(errno));
+				goto resized;
+			}
+			if (internal_config.hugepage_unlink) {
+				if (unlink(path)) {
+					RTE_LOG(DEBUG, EAL, "%s(): unlink() failed: %s\n",
+						__func__, strerror(errno));
+					goto resized;
+				}
+			}
+		}
+
+		/*
+		 * map the segment, and populate page tables, the kernel fills
+		 * this segment with zeros if it's a new page.
+		 */
+		va = mmap(addr, alloc_sz, PROT_READ | PROT_WRITE,
+				MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd,
+				map_offset);
+	}
 
 	if (va == MAP_FAILED) {
 		RTE_LOG(DEBUG, EAL, "%s(): mmap() failed: %s\n", __func__,
 			strerror(errno));
-		goto resized;
+		/* mmap failed, but the previous region might have been
+		 * unmapped anyway. try to remap it
+		 */
+		goto unmapped;
 	}
 	if (va != addr) {
 		RTE_LOG(DEBUG, EAL, "%s(): wrong mmap() address\n", __func__);
@@ -532,24 +556,6 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 		goto resized;
 	}
 
-	rte_iova_t iova = rte_mem_virt2iova(addr);
-	if (iova == RTE_BAD_PHYS_ADDR) {
-		RTE_LOG(DEBUG, EAL, "%s(): can't get IOVA addr\n",
-			__func__);
-		goto mapped;
-	}
-
-#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
-	move_pages(getpid(), 1, &addr, NULL, &cur_socket_id, 0);
-
-	if (cur_socket_id != socket_id) {
-		RTE_LOG(DEBUG, EAL,
-				"%s(): allocation happened on wrong socket (wanted %d, got %d)\n",
-			__func__, socket_id, cur_socket_id);
-		goto mapped;
-	}
-#endif
-
 	/* In linux, hugetlb limitations, like cgroup, are
 	 * enforced at fault time instead of mmap(), even
 	 * with the option of MAP_POPULATE. Kernel will send
@@ -562,9 +568,6 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 			(unsigned int)(alloc_sz >> 20));
 		goto mapped;
 	}
-	/* for non-single file segments, we can close fd here */
-	if (!internal_config.single_file_segments)
-		close(fd);
 
 	/* we need to trigger a write to the page to enforce page fault and
 	 * ensure that page is accessible to us, but we can't overwrite value
@@ -573,6 +576,28 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 	 */
 	*(volatile int *)addr = *(volatile int *)addr;
 
+	iova = rte_mem_virt2iova(addr);
+	if (iova == RTE_BAD_PHYS_ADDR) {
+		RTE_LOG(DEBUG, EAL, "%s(): can't get IOVA addr\n",
+			__func__);
+		goto mapped;
+	}
+
+#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
+	move_pages(getpid(), 1, &addr, NULL, &cur_socket_id, 0);
+
+	if (cur_socket_id != socket_id) {
+		RTE_LOG(DEBUG, EAL,
+				"%s(): allocation happened on wrong socket (wanted %d, got %d)\n",
+			__func__, socket_id, cur_socket_id);
+		goto mapped;
+	}
+#endif
+	/* for non-single file segments that aren't in-memory, we can close fd
+	 * here */
+	if (!internal_config.single_file_segments && !internal_config.in_memory)
+		close(fd);
+
 	ms->addr = addr;
 	ms->hugepage_sz = alloc_sz;
 	ms->len = alloc_sz;
@@ -585,14 +610,32 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 
 mapped:
 	munmap(addr, alloc_sz);
+unmapped:
+	flags = MAP_FIXED;
+#ifdef RTE_ARCH_PPC_64
+	flags |= MAP_HUGETLB;
+#endif
+	new_addr = eal_get_virtual_area(addr, &alloc_sz, alloc_sz, 0, flags);
+	if (new_addr != addr) {
+		if (new_addr != NULL)
+			munmap(new_addr, alloc_sz);
+		/* we're leaving a hole in our virtual address space. if
+		 * somebody else maps this hole now, we could accidentally
+		 * override it in the future.
+		 */
+		RTE_LOG(CRIT, EAL, "Can't mmap holes in our virtual address space\n");
+	}
 resized:
+	/* in-memory mode will never be single-file-segments mode */
 	if (internal_config.single_file_segments) {
 		resize_hugefile(fd, path, list_idx, seg_idx, map_offset,
 				alloc_sz, false);
 		/* ignore failure, can't make it any worse */
 	} else {
 		/* only remove file if we can take out a write lock */
-		if (lock(fd, LOCK_EX) == 1)
+		if (internal_config.hugepage_unlink == 0 &&
+				internal_config.in_memory == 0 &&
+				lock(fd, LOCK_EX) == 1)
 			unlink(path);
 		close(fd);
 	}
@@ -617,6 +660,12 @@ free_seg(struct rte_memseg *ms, struct hugepage_info *hi,
 		return -1;
 	}
 
+	/* if we've already unlinked the page, nothing needs to be done */
+	if (internal_config.hugepage_unlink) {
+		memset(ms, 0, sizeof(*ms));
+		return 0;
+	}
+
 	/* if we are not in single file segments mode, we're going to unmap the
 	 * segment and thus drop the lock on original fd, but hugepage dir is
 	 * now locked so we can take out another one without races.
@@ -695,7 +744,7 @@ alloc_seg_walk(const struct rte_memseg_list *msl, void *arg)
 	 * during init, we already hold a write lock, so don't try to take out
 	 * another one.
 	 */
-	if (wa->hi->lock_descriptor == -1) {
+	if (wa->hi->lock_descriptor == -1 && !internal_config.in_memory) {
 		dir_fd = open(wa->hi->hugedir, O_RDONLY);
 		if (dir_fd < 0) {
 			RTE_LOG(ERR, EAL, "%s(): Cannot open '%s': %s\n",
@@ -799,7 +848,7 @@ free_seg_walk(const struct rte_memseg_list *msl, void *arg)
 	 * during init, we already hold a write lock, so don't try to take out
 	 * another one.
 	 */
-	if (wa->hi->lock_descriptor == -1) {
+	if (wa->hi->lock_descriptor == -1 && !internal_config.in_memory) {
 		dir_fd = open(wa->hi->hugedir, O_RDONLY);
 		if (dir_fd < 0) {
 			RTE_LOG(ERR, EAL, "%s(): Cannot open '%s': %s\n",
@@ -878,7 +927,8 @@ eal_memalloc_alloc_seg_bulk(struct rte_memseg **ms, int n_segs, size_t page_sz,
 	wa.socket = socket;
 	wa.segs_allocated = 0;
 
-	ret = memseg_list_walk_thread_unsafe(alloc_seg_walk, &wa);
+	/* memalloc is locked, so it's safe to use thread-unsafe version */
+	ret = rte_memseg_list_walk_thread_unsafe(alloc_seg_walk, &wa);
 	if (ret == 0) {
 		RTE_LOG(ERR, EAL, "%s(): couldn't find suitable memseg_list\n",
 			__func__);
@@ -943,7 +993,10 @@ eal_memalloc_free_seg_bulk(struct rte_memseg **ms, int n_segs)
 		wa.ms = cur;
 		wa.hi = hi;
 
-		walk_res = memseg_list_walk_thread_unsafe(free_seg_walk, &wa);
+		/* memalloc is locked, so it's safe to use thread-unsafe version
+		 */
+		walk_res = rte_memseg_list_walk_thread_unsafe(free_seg_walk,
+				&wa);
 		if (walk_res == 1)
 			continue;
 		if (walk_res == 0)
@@ -1230,7 +1283,8 @@ eal_memalloc_sync_with_primary(void)
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 		return 0;
 
-	if (memseg_list_walk_thread_unsafe(sync_walk, NULL))
+	/* memalloc is locked, so it's safe to call thread-unsafe version */
+	if (rte_memseg_list_walk_thread_unsafe(sync_walk, NULL))
 		return -1;
 	return 0;
 }
diff --git a/lib/librte_eal/linuxapp/eal/eal_memory.c b/lib/librte_eal/linuxapp/eal/eal_memory.c
index c917de1c..dbf19499 100644
--- a/lib/librte_eal/linuxapp/eal/eal_memory.c
+++ b/lib/librte_eal/linuxapp/eal/eal_memory.c
@@ -66,7 +66,7 @@ static bool phys_addrs_available = true;
 static void
 test_phys_addrs_available(void)
 {
-	uint64_t tmp;
+	uint64_t tmp = 0;
 	phys_addr_t physaddr;
 
 	if (!rte_eal_has_hugepages()) {
@@ -521,7 +521,18 @@ static void *
 create_shared_memory(const char *filename, const size_t mem_size)
 {
 	void *retval;
-	int fd = open(filename, O_CREAT | O_RDWR, 0666);
+	int fd;
+
+	/* if no shared files mode is used, create anonymous memory instead */
+	if (internal_config.no_shconf) {
+		retval = mmap(NULL, mem_size, PROT_READ | PROT_WRITE,
+				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+		if (retval == MAP_FAILED)
+			return NULL;
+		return retval;
+	}
+
+	fd = open(filename, O_CREAT | O_RDWR, 0666);
 	if (fd < 0)
 		return NULL;
 	if (ftruncate(fd, mem_size) < 0) {
@@ -767,6 +778,34 @@ remap_segment(struct hugepage_file *hugepages, int seg_start, int seg_end)
 	return 0;
 }
 
+static uint64_t
+get_mem_amount(uint64_t page_sz, uint64_t max_mem)
+{
+	uint64_t area_sz, max_pages;
+
+	/* limit to RTE_MAX_MEMSEG_PER_LIST pages or RTE_MAX_MEM_MB_PER_LIST */
+	max_pages = RTE_MAX_MEMSEG_PER_LIST;
+	max_mem = RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_LIST << 20, max_mem);
+
+	area_sz = RTE_MIN(page_sz * max_pages, max_mem);
+
+	/* make sure the list isn't smaller than the page size */
+	area_sz = RTE_MAX(area_sz, page_sz);
+
+	return RTE_ALIGN(area_sz, page_sz);
+}
+
+static int
+free_memseg_list(struct rte_memseg_list *msl)
+{
+	if (rte_fbarray_destroy(&msl->memseg_arr)) {
+		RTE_LOG(ERR, EAL, "Cannot destroy memseg list\n");
+		return -1;
+	}
+	memset(msl, 0, sizeof(*msl));
+	return 0;
+}
+
 #define MEMSEG_LIST_FMT "memseg-%" PRIu64 "k-%i-%i"
 static int
 alloc_memseg_list(struct rte_memseg_list *msl, uint64_t page_sz,
@@ -1049,8 +1088,7 @@ get_socket_mem_size(int socket)
 
 	for (i = 0; i < internal_config.num_hugepage_sizes; i++){
 		struct hugepage_info *hpi = &internal_config.hugepage_info[i];
-		if (strnlen(hpi->hugedir, sizeof(hpi->hugedir)) != 0)
-			size += hpi->hugepage_sz * hpi->num_pages[socket];
+		size += hpi->hugepage_sz * hpi->num_pages[socket];
 	}
 
 	return size;
@@ -1605,6 +1643,15 @@ hugepage_count_walk(const struct rte_memseg_list *msl, void *arg)
 }
 
 static int
+limits_callback(int socket_id, size_t cur_limit, size_t new_len)
+{
+	RTE_SET_USED(socket_id);
+	RTE_SET_USED(cur_limit);
+	RTE_SET_USED(new_len);
+	return -1;
+}
+
+static int
 eal_hugepage_init(void)
 {
 	struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES];
@@ -1687,6 +1734,18 @@ eal_hugepage_init(void)
 			free(pages);
 		}
 	}
+	/* if socket limits were specified, set them */
+	if (internal_config.force_socket_limits) {
+		unsigned int i;
+		for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
+			uint64_t limit = internal_config.socket_limit[i];
+			if (limit == 0)
+				continue;
+			if (rte_mem_alloc_validator_register("socket-limit",
+					limits_callback, i, limit))
+				RTE_LOG(ERR, EAL, "Failed to register socket limits validator callback\n");
+		}
+	}
 	return 0;
 }
 
@@ -1840,3 +1899,316 @@ rte_eal_using_phys_addrs(void)
 {
 	return phys_addrs_available;
 }
+
+static int __rte_unused
+memseg_primary_init_32(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int active_sockets, hpi_idx, msl_idx = 0;
+	unsigned int socket_id, i;
+	struct rte_memseg_list *msl;
+	uint64_t extra_mem_per_socket, total_extra_mem, total_requested_mem;
+	uint64_t max_mem;
+
+	/* no-huge does not need this at all */
+	if (internal_config.no_hugetlbfs)
+		return 0;
+
+	/* this is a giant hack, but desperate times call for desperate
+	 * measures. in legacy 32-bit mode, we cannot preallocate VA space,
+	 * because having upwards of 2 gigabytes of VA space already mapped will
+	 * interfere with our ability to map and sort hugepages.
+	 *
+	 * therefore, in legacy 32-bit mode, we will be initializing memseg
+	 * lists much later - in eal_memory.c, right after we unmap all the
+	 * unneeded pages. this will not affect secondary processes, as those
+	 * should be able to mmap the space without (too many) problems.
+	 */
+	if (internal_config.legacy_mem)
+		return 0;
+
+	/* 32-bit mode is a very special case. we cannot know in advance where
+	 * the user will want to allocate their memory, so we have to do some
+	 * heuristics.
+	 */
+	active_sockets = 0;
+	total_requested_mem = 0;
+	if (internal_config.force_sockets)
+		for (i = 0; i < rte_socket_count(); i++) {
+			uint64_t mem;
+
+			socket_id = rte_socket_id_by_idx(i);
+			mem = internal_config.socket_mem[socket_id];
+
+			if (mem == 0)
+				continue;
+
+			active_sockets++;
+			total_requested_mem += mem;
+		}
+	else
+		total_requested_mem = internal_config.memory;
+
+	max_mem = (uint64_t)RTE_MAX_MEM_MB << 20;
+	if (total_requested_mem > max_mem) {
+		RTE_LOG(ERR, EAL, "Invalid parameters: 32-bit process can at most use %uM of memory\n",
+				(unsigned int)(max_mem >> 20));
+		return -1;
+	}
+	total_extra_mem = max_mem - total_requested_mem;
+	extra_mem_per_socket = active_sockets == 0 ? total_extra_mem :
+			total_extra_mem / active_sockets;
+
+	/* the allocation logic is a little bit convoluted, but here's how it
+	 * works, in a nutshell:
+	 *  - if user hasn't specified on which sockets to allocate memory via
+	 *    --socket-mem, we allocate all of our memory on master core socket.
+	 *  - if user has specified sockets to allocate memory on, there may be
+	 *    some "unused" memory left (e.g. if user has specified --socket-mem
+	 *    such that not all memory adds up to 2 gigabytes), so add it to all
+	 *    sockets that are in use equally.
+	 *
+	 * page sizes are sorted by size in descending order, so we can safely
+	 * assume that we dispense with bigger page sizes first.
+	 */
+
+	/* create memseg lists */
+	for (i = 0; i < rte_socket_count(); i++) {
+		int hp_sizes = (int) internal_config.num_hugepage_sizes;
+		uint64_t max_socket_mem, cur_socket_mem;
+		unsigned int master_lcore_socket;
+		struct rte_config *cfg = rte_eal_get_configuration();
+		bool skip;
+
+		socket_id = rte_socket_id_by_idx(i);
+
+#ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
+		if (socket_id > 0)
+			break;
+#endif
+
+		/* if we didn't specifically request memory on this socket */
+		skip = active_sockets != 0 &&
+				internal_config.socket_mem[socket_id] == 0;
+		/* ...or if we didn't specifically request memory on *any*
+		 * socket, and this is not master lcore
+		 */
+		master_lcore_socket = rte_lcore_to_socket_id(cfg->master_lcore);
+		skip |= active_sockets == 0 && socket_id != master_lcore_socket;
+
+		if (skip) {
+			RTE_LOG(DEBUG, EAL, "Will not preallocate memory on socket %u\n",
+					socket_id);
+			continue;
+		}
+
+		/* max amount of memory on this socket */
+		max_socket_mem = (active_sockets != 0 ?
+					internal_config.socket_mem[socket_id] :
+					internal_config.memory) +
+					extra_mem_per_socket;
+		cur_socket_mem = 0;
+
+		for (hpi_idx = 0; hpi_idx < hp_sizes; hpi_idx++) {
+			uint64_t max_pagesz_mem, cur_pagesz_mem = 0;
+			uint64_t hugepage_sz;
+			struct hugepage_info *hpi;
+			int type_msl_idx, max_segs, total_segs = 0;
+
+			hpi = &internal_config.hugepage_info[hpi_idx];
+			hugepage_sz = hpi->hugepage_sz;
+
+			/* check if pages are actually available */
+			if (hpi->num_pages[socket_id] == 0)
+				continue;
+
+			max_segs = RTE_MAX_MEMSEG_PER_TYPE;
+			max_pagesz_mem = max_socket_mem - cur_socket_mem;
+
+			/* make it multiple of page size */
+			max_pagesz_mem = RTE_ALIGN_FLOOR(max_pagesz_mem,
+					hugepage_sz);
+
+			RTE_LOG(DEBUG, EAL, "Attempting to preallocate "
+					"%" PRIu64 "M on socket %i\n",
+					max_pagesz_mem >> 20, socket_id);
+
+			type_msl_idx = 0;
+			while (cur_pagesz_mem < max_pagesz_mem &&
+					total_segs < max_segs) {
+				uint64_t cur_mem;
+				unsigned int n_segs;
+
+				if (msl_idx >= RTE_MAX_MEMSEG_LISTS) {
+					RTE_LOG(ERR, EAL,
+						"No more space in memseg lists, please increase %s\n",
+						RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS));
+					return -1;
+				}
+
+				msl = &mcfg->memsegs[msl_idx];
+
+				cur_mem = get_mem_amount(hugepage_sz,
+						max_pagesz_mem);
+				n_segs = cur_mem / hugepage_sz;
+
+				if (alloc_memseg_list(msl, hugepage_sz, n_segs,
+						socket_id, type_msl_idx)) {
+					/* failing to allocate a memseg list is
+					 * a serious error.
+					 */
+					RTE_LOG(ERR, EAL, "Cannot allocate memseg list\n");
+					return -1;
+				}
+
+				if (alloc_va_space(msl)) {
+					/* if we couldn't allocate VA space, we
+					 * can try with smaller page sizes.
+					 */
+					RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list, retrying with different page size\n");
+					/* deallocate memseg list */
+					if (free_memseg_list(msl))
+						return -1;
+					break;
+				}
+
+				total_segs += msl->memseg_arr.len;
+				cur_pagesz_mem = total_segs * hugepage_sz;
+				type_msl_idx++;
+				msl_idx++;
+			}
+			cur_socket_mem += cur_pagesz_mem;
+		}
+		if (cur_socket_mem == 0) {
+			RTE_LOG(ERR, EAL, "Cannot allocate VA space on socket %u\n",
+				socket_id);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static int __rte_unused
+memseg_primary_init(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int i, socket_id, hpi_idx, msl_idx = 0;
+	struct rte_memseg_list *msl;
+	uint64_t max_mem, total_mem;
+
+	/* no-huge does not need this at all */
+	if (internal_config.no_hugetlbfs)
+		return 0;
+
+	max_mem = (uint64_t)RTE_MAX_MEM_MB << 20;
+	total_mem = 0;
+
+	/* create memseg lists */
+	for (hpi_idx = 0; hpi_idx < (int) internal_config.num_hugepage_sizes;
+			hpi_idx++) {
+		struct hugepage_info *hpi;
+		uint64_t hugepage_sz;
+
+		hpi = &internal_config.hugepage_info[hpi_idx];
+		hugepage_sz = hpi->hugepage_sz;
+
+		for (i = 0; i < (int) rte_socket_count(); i++) {
+			uint64_t max_type_mem, total_type_mem = 0;
+			int type_msl_idx, max_segs, total_segs = 0;
+
+			socket_id = rte_socket_id_by_idx(i);
+
+#ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
+			if (socket_id > 0)
+				break;
+#endif
+
+			if (total_mem >= max_mem)
+				break;
+
+			max_type_mem = RTE_MIN(max_mem - total_mem,
+				(uint64_t)RTE_MAX_MEM_MB_PER_TYPE << 20);
+			max_segs = RTE_MAX_MEMSEG_PER_TYPE;
+
+			type_msl_idx = 0;
+			while (total_type_mem < max_type_mem &&
+					total_segs < max_segs) {
+				uint64_t cur_max_mem, cur_mem;
+				unsigned int n_segs;
+
+				if (msl_idx >= RTE_MAX_MEMSEG_LISTS) {
+					RTE_LOG(ERR, EAL,
+						"No more space in memseg lists, please increase %s\n",
+						RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS));
+					return -1;
+				}
+
+				msl = &mcfg->memsegs[msl_idx++];
+
+				cur_max_mem = max_type_mem - total_type_mem;
+
+				cur_mem = get_mem_amount(hugepage_sz,
+						cur_max_mem);
+				n_segs = cur_mem / hugepage_sz;
+
+				if (alloc_memseg_list(msl, hugepage_sz, n_segs,
+						socket_id, type_msl_idx))
+					return -1;
+
+				total_segs += msl->memseg_arr.len;
+				total_type_mem = total_segs * hugepage_sz;
+				type_msl_idx++;
+
+				if (alloc_va_space(msl)) {
+					RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list\n");
+					return -1;
+				}
+			}
+			total_mem += total_type_mem;
+		}
+	}
+	return 0;
+}
+
+static int
+memseg_secondary_init(void)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int msl_idx = 0;
+	struct rte_memseg_list *msl;
+
+	for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS; msl_idx++) {
+
+		msl = &mcfg->memsegs[msl_idx];
+
+		/* skip empty memseg lists */
+		if (msl->memseg_arr.len == 0)
+			continue;
+
+		if (rte_fbarray_attach(&msl->memseg_arr)) {
+			RTE_LOG(ERR, EAL, "Cannot attach to primary process memseg lists\n");
+			return -1;
+		}
+
+		/* preallocate VA space */
+		if (alloc_va_space(msl)) {
+			RTE_LOG(ERR, EAL, "Cannot preallocate VA space for hugepage memory\n");
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+rte_eal_memseg_init(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY ?
+#ifndef RTE_ARCH_64
+			memseg_primary_init_32() :
+#else
+			memseg_primary_init() :
+#endif
+			memseg_secondary_init();
+}
diff --git a/lib/librte_eal/linuxapp/eal/eal_thread.c b/lib/librte_eal/linuxapp/eal/eal_thread.c
index f652ff98..b496fc71 100644
--- a/lib/librte_eal/linuxapp/eal/eal_thread.c
+++ b/lib/librte_eal/linuxapp/eal/eal_thread.c
@@ -176,7 +176,7 @@ int rte_sys_gettid(void)
 
 int rte_thread_setname(pthread_t id, const char *name)
 {
-	int ret = -1;
+	int ret = ENOSYS;
 #if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
 #if __GLIBC_PREREQ(2, 12)
 	ret = pthread_setname_np(id, name);
@@ -184,5 +184,5 @@ int rte_thread_setname(pthread_t id, const char *name)
 #endif
 	RTE_SET_USED(id);
 	RTE_SET_USED(name);
-	return ret;
+	return -ret;
 }
diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.c b/lib/librte_eal/linuxapp/eal/eal_vfio.c
index a2bbdfbf..c68dc38e 100644
--- a/lib/librte_eal/linuxapp/eal/eal_vfio.c
+++ b/lib/librte_eal/linuxapp/eal/eal_vfio.c
@@ -87,42 +87,6 @@ static const struct vfio_iommu_type iommu_types[] = {
 	},
 };
 
-/* for sPAPR IOMMU, we will need to walk memseg list, but we cannot use
- * rte_memseg_walk() because by the time we enter callback we will be holding a
- * write lock, so regular rte-memseg_walk will deadlock. copying the same
- * iteration code everywhere is not ideal as well. so, use a lockless copy of
- * memseg walk here.
- */
-static int
-memseg_walk_thread_unsafe(rte_memseg_walk_t func, void *arg)
-{
-	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	int i, ms_idx, ret = 0;
-
-	for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
-		struct rte_memseg_list *msl = &mcfg->memsegs[i];
-		const struct rte_memseg *ms;
-		struct rte_fbarray *arr;
-
-		if (msl->memseg_arr.count == 0)
-			continue;
-
-		arr = &msl->memseg_arr;
-
-		ms_idx = rte_fbarray_find_next_used(arr, 0);
-		while (ms_idx >= 0) {
-			ms = rte_fbarray_get(arr, ms_idx);
-			ret = func(msl, ms, arg);
-			if (ret < 0)
-				return -1;
-			if (ret > 0)
-				return 1;
-			ms_idx = rte_fbarray_find_next_used(arr, ms_idx + 1);
-		}
-	}
-	return 0;
-}
-
 static int
 is_null_map(const struct user_mem_map *map)
 {
@@ -575,10 +539,6 @@ int
 rte_vfio_clear_group(int vfio_group_fd)
 {
 	int i;
-	struct rte_mp_msg mp_req, *mp_rep;
-	struct rte_mp_reply mp_reply;
-	struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
-	struct vfio_mp_param *p = (struct vfio_mp_param *)mp_req.param;
 	struct vfio_config *vfio_cfg;
 
 	vfio_cfg = get_vfio_cfg_by_group_fd(vfio_group_fd);
@@ -587,40 +547,15 @@ rte_vfio_clear_group(int vfio_group_fd)
 		return -1;
 	}
 
-	if (internal_config.process_type == RTE_PROC_PRIMARY) {
-
-		i = get_vfio_group_idx(vfio_group_fd);
-		if (i < 0)
-			return -1;
-		vfio_cfg->vfio_groups[i].group_num = -1;
-		vfio_cfg->vfio_groups[i].fd = -1;
-		vfio_cfg->vfio_groups[i].devices = 0;
-		vfio_cfg->vfio_active_groups--;
-		return 0;
-	}
-
-	p->req = SOCKET_CLR_GROUP;
-	p->group_num = vfio_group_fd;
-	strcpy(mp_req.name, EAL_VFIO_MP);
-	mp_req.len_param = sizeof(*p);
-	mp_req.num_fds = 0;
-
-	if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 &&
-	    mp_reply.nb_received == 1) {
-		mp_rep = &mp_reply.msgs[0];
-		p = (struct vfio_mp_param *)mp_rep->param;
-		if (p->result == SOCKET_OK) {
-			free(mp_reply.msgs);
-			return 0;
-		} else if (p->result == SOCKET_NO_FD)
-			RTE_LOG(ERR, EAL, "  BAD VFIO group fd!\n");
-		else
-			RTE_LOG(ERR, EAL, "  no such VFIO group fd!\n");
-
-		free(mp_reply.msgs);
-	}
+	i = get_vfio_group_idx(vfio_group_fd);
+	if (i < 0)
+		return -1;
+	vfio_cfg->vfio_groups[i].group_num = -1;
+	vfio_cfg->vfio_groups[i].fd = -1;
+	vfio_cfg->vfio_groups[i].devices = 0;
+	vfio_cfg->vfio_active_groups--;
 
-	return -1;
+	return 0;
 }
 
 int
@@ -1357,7 +1292,8 @@ vfio_spapr_dma_mem_map(int vfio_container_fd, uint64_t vaddr, uint64_t iova,
 	/* check if window size needs to be adjusted */
 	memset(&param, 0, sizeof(param));
 
-	if (memseg_walk_thread_unsafe(vfio_spapr_window_size_walk,
+	/* we're inside a callback so use thread-unsafe version */
+	if (rte_memseg_walk_thread_unsafe(vfio_spapr_window_size_walk,
 				&param) < 0) {
 		RTE_LOG(ERR, EAL, "Could not get window size\n");
 		ret = -1;
@@ -1386,7 +1322,9 @@ vfio_spapr_dma_mem_map(int vfio_container_fd, uint64_t vaddr, uint64_t iova,
 				ret = -1;
 				goto out;
 			}
-			if (memseg_walk_thread_unsafe(vfio_spapr_map_walk,
+			/* we're inside a callback, so use thread-unsafe version
+			 */
+			if (rte_memseg_walk_thread_unsafe(vfio_spapr_map_walk,
 					&vfio_container_fd) < 0) {
 				RTE_LOG(ERR, EAL, "Could not recreate DMA maps\n");
 				ret = -1;
@@ -1624,7 +1562,7 @@ out:
 	return ret;
 }
 
-int __rte_experimental
+int
 rte_vfio_dma_map(uint64_t vaddr, uint64_t iova, uint64_t len)
 {
 	if (len == 0) {
@@ -1635,7 +1573,7 @@ rte_vfio_dma_map(uint64_t vaddr, uint64_t iova, uint64_t len)
 	return container_dma_map(default_vfio_cfg, vaddr, iova, len);
 }
 
-int __rte_experimental
+int
 rte_vfio_dma_unmap(uint64_t vaddr, uint64_t iova, uint64_t len)
 {
 	if (len == 0) {
@@ -1678,7 +1616,7 @@ rte_vfio_noiommu_is_enabled(void)
 	return c == 'Y';
 }
 
-int __rte_experimental
+int
 rte_vfio_container_create(void)
 {
 	int i;
@@ -1728,7 +1666,7 @@ rte_vfio_container_destroy(int container_fd)
 	return 0;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_group_bind(int container_fd, int iommu_group_num)
 {
 	struct vfio_config *vfio_cfg;
@@ -1774,11 +1712,11 @@ rte_vfio_container_group_bind(int container_fd, int iommu_group_num)
 	return vfio_group_fd;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_group_unbind(int container_fd, int iommu_group_num)
 {
 	struct vfio_config *vfio_cfg;
-	struct vfio_group *cur_grp;
+	struct vfio_group *cur_grp = NULL;
 	int i;
 
 	vfio_cfg = get_vfio_cfg_by_container_fd(container_fd);
@@ -1795,7 +1733,7 @@ rte_vfio_container_group_unbind(int container_fd, int iommu_group_num)
 	}
 
 	/* This should not happen */
-	if (i == VFIO_MAX_GROUPS) {
+	if (i == VFIO_MAX_GROUPS || cur_grp == NULL) {
 		RTE_LOG(ERR, EAL, "Specified group number not found\n");
 		return -1;
 	}
@@ -1813,7 +1751,7 @@ rte_vfio_container_group_unbind(int container_fd, int iommu_group_num)
 	return 0;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_dma_map(int container_fd, uint64_t vaddr, uint64_t iova,
 		uint64_t len)
 {
@@ -1833,7 +1771,7 @@ rte_vfio_container_dma_map(int container_fd, uint64_t vaddr, uint64_t iova,
 	return container_dma_map(vfio_cfg, vaddr, iova, len);
 }
 
-int __rte_experimental
+int
 rte_vfio_container_dma_unmap(int container_fd, uint64_t vaddr, uint64_t iova,
 		uint64_t len)
 {
@@ -1855,14 +1793,14 @@ rte_vfio_container_dma_unmap(int container_fd, uint64_t vaddr, uint64_t iova,
 
 #else
 
-int __rte_experimental
+int
 rte_vfio_dma_map(uint64_t __rte_unused vaddr, __rte_unused uint64_t iova,
 		  __rte_unused uint64_t len)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_dma_unmap(uint64_t __rte_unused vaddr, uint64_t __rte_unused iova,
 		    __rte_unused uint64_t len)
 {
@@ -1909,7 +1847,7 @@ rte_vfio_clear_group(__rte_unused int vfio_group_fd)
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_get_group_num(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *iommu_group_num)
@@ -1917,45 +1855,45 @@ rte_vfio_get_group_num(__rte_unused const char *sysfs_base,
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_get_container_fd(void)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_get_group_fd(__rte_unused int iommu_group_num)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_create(void)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_destroy(__rte_unused int container_fd)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_group_bind(__rte_unused int container_fd,
 		__rte_unused int iommu_group_num)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_group_unbind(__rte_unused int container_fd,
 		__rte_unused int iommu_group_num)
 {
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_dma_map(__rte_unused int container_fd,
 		__rte_unused uint64_t vaddr,
 		__rte_unused uint64_t iova,
@@ -1964,7 +1902,7 @@ rte_vfio_container_dma_map(__rte_unused int container_fd,
 	return -1;
 }
 
-int __rte_experimental
+int
 rte_vfio_container_dma_unmap(__rte_unused int container_fd,
 		__rte_unused uint64_t vaddr,
 		__rte_unused uint64_t iova,
diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.h b/lib/librte_eal/linuxapp/eal/eal_vfio.h
index e65b1037..68d4750a 100644
--- a/lib/librte_eal/linuxapp/eal/eal_vfio.h
+++ b/lib/librte_eal/linuxapp/eal/eal_vfio.h
@@ -129,7 +129,6 @@ int vfio_mp_sync_setup(void);
 
 #define SOCKET_REQ_CONTAINER 0x100
 #define SOCKET_REQ_GROUP 0x200
-#define SOCKET_CLR_GROUP 0x300
 #define SOCKET_OK 0x0
 #define SOCKET_NO_FD 0x1
 #define SOCKET_ERR 0xFF
diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c b/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c
index 9c202bb0..680a24aa 100644
--- a/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c
+++ b/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c
@@ -55,14 +55,6 @@ vfio_mp_primary(const struct rte_mp_msg *msg, const void *peer)
 			reply.fds[0] = fd;
 		}
 		break;
-	case SOCKET_CLR_GROUP:
-		r->req = SOCKET_CLR_GROUP;
-		r->group_num = m->group_num;
-		if (rte_vfio_clear_group(m->group_num) < 0)
-			r->result = SOCKET_NO_FD;
-		else
-			r->result = SOCKET_OK;
-		break;
 	case SOCKET_REQ_CONTAINER:
 		r->req = SOCKET_REQ_CONTAINER;
 		fd = rte_vfio_get_container_fd();
diff --git a/lib/librte_eal/linuxapp/eal/meson.build b/lib/librte_eal/linuxapp/eal/meson.build
index cce37712..6e31c2aa 100644
--- a/lib/librte_eal/linuxapp/eal/meson.build
+++ b/lib/librte_eal/linuxapp/eal/meson.build
@@ -23,6 +23,7 @@ env_sources = files('eal_alarm.c',
 		'eal_dev.c',
 )
 
+deps += ['kvargs']
 if has_libnuma == 1
 	dpdk_conf.set10('RTE_EAL_NUMA_AWARE_HUGEPAGES', true)
 endif
diff --git a/lib/librte_eal/meson.build b/lib/librte_eal/meson.build
index 4aa63e3d..e1fde15d 100644
--- a/lib/librte_eal/meson.build
+++ b/lib/librte_eal/meson.build
@@ -18,12 +18,13 @@ elif host_machine.system() == 'freebsd'
 	subdir('bsdapp/eal')
 
 else
-	error('unsupported system type @0@'.format(hostmachine.system()))
+	error('unsupported system type "@0@"'.format(host_machine.system()))
 endif
 
-version = 7  # the version of the EAL API
+version = 8  # the version of the EAL API
 allow_experimental_apis = true
 deps += 'compat'
+deps += 'kvargs'
 cflags += '-D_GNU_SOURCE'
 sources = common_sources + env_sources
 objs = common_objs + env_objs
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index f7dd0e7b..344a43d3 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -181,7 +181,6 @@ DPDK_17.11 {
 	rte_bus_get_iommu_class;
 	rte_eal_has_pci;
 	rte_eal_iova_mode;
-	rte_eal_mbuf_default_mempool_ops;
 	rte_eal_using_phys_addrs;
 	rte_eal_vfio_intr_mode;
 	rte_lcore_has_role;
@@ -241,25 +240,53 @@ DPDK_18.05 {
 
 } DPDK_18.02;
 
+DPDK_18.08 {
+	global:
+
+	rte_eal_mbuf_user_pool_ops;
+	rte_uuid_compare;
+	rte_uuid_is_null;
+	rte_uuid_parse;
+	rte_uuid_unparse;
+	rte_vfio_container_create;
+	rte_vfio_container_destroy;
+	rte_vfio_container_dma_map;
+	rte_vfio_container_dma_unmap;
+	rte_vfio_container_group_bind;
+	rte_vfio_container_group_unbind;
+	rte_vfio_dma_map;
+	rte_vfio_dma_unmap;
+	rte_vfio_get_container_fd;
+	rte_vfio_get_group_fd;
+	rte_vfio_get_group_num;
+
+} DPDK_18.05;
+
 EXPERIMENTAL {
 	global:
 
+	rte_class_find;
+	rte_class_find_by_name;
+	rte_class_register;
+	rte_class_unregister;
 	rte_ctrl_thread_create;
 	rte_dev_event_callback_register;
 	rte_dev_event_callback_unregister;
 	rte_dev_event_monitor_start;
 	rte_dev_event_monitor_stop;
+	rte_dev_iterator_init;
+	rte_dev_iterator_next;
 	rte_devargs_add;
 	rte_devargs_dump;
 	rte_devargs_insert;
 	rte_devargs_next;
 	rte_devargs_parse;
+	rte_devargs_parsef;
 	rte_devargs_remove;
 	rte_devargs_type_count;
 	rte_eal_cleanup;
 	rte_eal_hotplug_add;
 	rte_eal_hotplug_remove;
-	rte_eal_mbuf_user_pool_ops;
 	rte_fbarray_attach;
 	rte_fbarray_destroy;
 	rte_fbarray_detach;
@@ -269,8 +296,14 @@ EXPERIMENTAL {
 	rte_fbarray_find_next_used;
 	rte_fbarray_find_next_n_free;
 	rte_fbarray_find_next_n_used;
+	rte_fbarray_find_prev_free;
+	rte_fbarray_find_prev_used;
+	rte_fbarray_find_prev_n_free;
+	rte_fbarray_find_prev_n_used;
 	rte_fbarray_find_contig_free;
 	rte_fbarray_find_contig_used;
+	rte_fbarray_find_rev_contig_free;
+	rte_fbarray_find_rev_contig_used;
 	rte_fbarray_get;
 	rte_fbarray_init;
 	rte_fbarray_is_used;
@@ -286,25 +319,20 @@ EXPERIMENTAL {
 	rte_mem_virt2memseg;
 	rte_mem_virt2memseg_list;
 	rte_memseg_contig_walk;
+	rte_memseg_contig_walk_thread_unsafe;
 	rte_memseg_list_walk;
+	rte_memseg_list_walk_thread_unsafe;
 	rte_memseg_walk;
+	rte_memseg_walk_thread_unsafe;
 	rte_mp_action_register;
 	rte_mp_action_unregister;
 	rte_mp_reply;
 	rte_mp_request_sync;
 	rte_mp_request_async;
 	rte_mp_sendmsg;
+	rte_service_lcore_attr_get;
+	rte_service_lcore_attr_reset_all;
+	rte_service_may_be_active;
 	rte_socket_count;
 	rte_socket_id_by_idx;
-	rte_vfio_dma_map;
-	rte_vfio_dma_unmap;
-	rte_vfio_get_container_fd;
-	rte_vfio_get_group_fd;
-	rte_vfio_get_group_num;
-	rte_vfio_container_create;
-	rte_vfio_container_destroy;
-	rte_vfio_container_dma_map;
-	rte_vfio_container_dma_unmap;
-	rte_vfio_container_group_bind;
-	rte_vfio_container_group_unbind;
 };
diff --git a/lib/librte_ethdev/Makefile b/lib/librte_ethdev/Makefile
index c2f2f7d8..0935a275 100644
--- a/lib/librte_ethdev/Makefile
+++ b/lib/librte_ethdev/Makefile
@@ -16,7 +16,7 @@ LDLIBS += -lrte_mbuf
 
 EXPORT_MAP := rte_ethdev_version.map
 
-LIBABIVER := 9
+LIBABIVER := 10
 
 SRCS-y += rte_ethdev.c
 SRCS-y += rte_flow.c
diff --git a/lib/librte_ethdev/meson.build b/lib/librte_ethdev/meson.build
index aed5d226..596cd0f3 100644
--- a/lib/librte_ethdev/meson.build
+++ b/lib/librte_ethdev/meson.build
@@ -2,7 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 name = 'ethdev'
-version = 9
+version = 10
 allow_experimental_apis = true
 sources = files('ethdev_profile.c',
 	'rte_ethdev.c',
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index cd4bfd3c..4c320250 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -42,10 +42,7 @@
 #include "rte_ethdev_driver.h"
 #include "ethdev_profile.h"
 
-static int ethdev_logtype;
-
-#define ethdev_log(level, fmt, ...) \
-	rte_log(RTE_LOG_ ## level, ethdev_logtype, fmt "\n", ## __VA_ARGS__)
+int rte_eth_dev_logtype;
 
 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
@@ -129,6 +126,7 @@ static const struct {
 	RTE_RX_OFFLOAD_BIT2STR(SCATTER),
 	RTE_RX_OFFLOAD_BIT2STR(TIMESTAMP),
 	RTE_RX_OFFLOAD_BIT2STR(SECURITY),
+	RTE_RX_OFFLOAD_BIT2STR(KEEP_CRC),
 };
 
 #undef RTE_RX_OFFLOAD_BIT2STR
@@ -303,14 +301,16 @@ rte_eth_dev_allocate(const char *name)
 	rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 
 	if (_rte_eth_dev_allocated(name) != NULL) {
-		ethdev_log(ERR, "Ethernet device with name %s already allocated",
-				name);
+		RTE_ETHDEV_LOG(ERR,
+			"Ethernet device with name %s already allocated\n",
+			name);
 		goto unlock;
 	}
 
 	port_id = rte_eth_dev_find_free_port();
 	if (port_id == RTE_MAX_ETHPORTS) {
-		ethdev_log(ERR, "Reached maximum number of Ethernet ports");
+		RTE_ETHDEV_LOG(ERR,
+			"Reached maximum number of Ethernet ports\n");
 		goto unlock;
 	}
 
@@ -346,8 +346,8 @@ rte_eth_dev_attach_secondary(const char *name)
 			break;
 	}
 	if (i == RTE_MAX_ETHPORTS) {
-		RTE_PMD_DEBUG_TRACE(
-			"device %s is not driven by the primary process\n",
+		RTE_ETHDEV_LOG(ERR,
+			"Device %s is not driven by the primary process\n",
 			name);
 	} else {
 		eth_dev = eth_dev_get(i);
@@ -394,7 +394,8 @@ rte_eth_is_valid_owner_id(uint64_t owner_id)
 {
 	if (owner_id == RTE_ETH_DEV_NO_OWNER ||
 	    rte_eth_dev_shared_data->next_owner_id <= owner_id) {
-		RTE_PMD_DEBUG_TRACE("Invalid owner_id=%016"PRIX64".\n", owner_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid owner_id=%016"PRIx64"\n",
+			owner_id);
 		return 0;
 	}
 	return 1;
@@ -437,7 +438,8 @@ _rte_eth_dev_owner_set(const uint16_t port_id, const uint64_t old_owner_id,
 	int sret;
 
 	if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
-		RTE_PMD_DEBUG_TRACE("Port id %"PRIu16" is not allocated.\n", port_id);
+		RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
+			port_id);
 		return -ENODEV;
 	}
 
@@ -447,22 +449,22 @@ _rte_eth_dev_owner_set(const uint16_t port_id, const uint64_t old_owner_id,
 
 	port_owner = &rte_eth_devices[port_id].data->owner;
 	if (port_owner->id != old_owner_id) {
-		RTE_PMD_DEBUG_TRACE("Cannot set owner to port %d already owned"
-				    " by %s_%016"PRIX64".\n", port_id,
-				    port_owner->name, port_owner->id);
+		RTE_ETHDEV_LOG(ERR,
+			"Cannot set owner to port %u already owned by %s_%016"PRIX64"\n",
+			port_id, port_owner->name, port_owner->id);
 		return -EPERM;
 	}
 
 	sret = snprintf(port_owner->name, RTE_ETH_MAX_OWNER_NAME_LEN, "%s",
 			new_owner->name);
 	if (sret < 0 || sret >= RTE_ETH_MAX_OWNER_NAME_LEN)
-		RTE_PMD_DEBUG_TRACE("Port %d owner name was truncated.\n",
-				    port_id);
+		RTE_ETHDEV_LOG(ERR, "Port %u owner name was truncated\n",
+			port_id);
 
 	port_owner->id = new_owner->id;
 
-	RTE_PMD_DEBUG_TRACE("Port %d owner is %s_%016"PRIX64".\n", port_id,
-			    new_owner->name, new_owner->id);
+	RTE_ETHDEV_LOG(DEBUG, "Port %u owner is %s_%016"PRIx64"\n",
+		port_id, new_owner->name, new_owner->id);
 
 	return 0;
 }
@@ -514,8 +516,9 @@ rte_eth_dev_owner_delete(const uint64_t owner_id)
 			if (rte_eth_devices[port_id].data->owner.id == owner_id)
 				memset(&rte_eth_devices[port_id].data->owner, 0,
 				       sizeof(struct rte_eth_dev_owner));
-		RTE_PMD_DEBUG_TRACE("All port owners owned by %016"PRIX64
-				" identifier have removed.\n", owner_id);
+		RTE_ETHDEV_LOG(ERR,
+			"All port owners owned by %016"PRIx64" identifier have removed\n",
+			owner_id);
 	}
 
 	rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
@@ -532,7 +535,8 @@ rte_eth_dev_owner_get(const uint16_t port_id, struct rte_eth_dev_owner *owner)
 	rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 
 	if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
-		RTE_PMD_DEBUG_TRACE("Port id %"PRIu16" is not allocated.\n", port_id);
+		RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
+			port_id);
 		ret = -ENODEV;
 	} else {
 		rte_memcpy(owner, &ethdev->data->owner, sizeof(*owner));
@@ -596,7 +600,7 @@ rte_eth_dev_get_name_by_port(uint16_t port_id, char *name)
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 
 	if (name == NULL) {
-		RTE_PMD_DEBUG_TRACE("Null pointer is specified\n");
+		RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 		return -EINVAL;
 	}
 
@@ -613,7 +617,7 @@ rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id)
 	uint32_t pid;
 
 	if (name == NULL) {
-		RTE_PMD_DEBUG_TRACE("Null pointer is specified\n");
+		RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 		return -EINVAL;
 	}
 
@@ -654,7 +658,7 @@ rte_eth_dev_attach(const char *devargs, uint16_t *port_id)
 	}
 
 	/* parse devargs */
-	if (rte_devargs_parse(&da, "%s", devargs))
+	if (rte_devargs_parse(&da, devargs))
 		goto err;
 
 	ret = rte_eal_hotplug_add(da.bus->name, da.name, da.args);
@@ -663,7 +667,7 @@ rte_eth_dev_attach(const char *devargs, uint16_t *port_id)
 
 	/* no point looking at the port count if no port exists */
 	if (!rte_eth_dev_count_total()) {
-		ethdev_log(ERR, "No port found for device (%s)", da.name);
+		RTE_ETHDEV_LOG(ERR, "No port found for device (%s)\n", da.name);
 		ret = -1;
 		goto err;
 	}
@@ -698,8 +702,8 @@ rte_eth_dev_detach(uint16_t port_id, char *name __rte_unused)
 
 	dev_flags = rte_eth_devices[port_id].data->dev_flags;
 	if (dev_flags & RTE_ETH_DEV_BONDED_SLAVE) {
-		ethdev_log(ERR,
-			"Port %" PRIu16 " is bonded, cannot detach", port_id);
+		RTE_ETHDEV_LOG(ERR,
+			"Port %"PRIu16" is bonded, cannot detach\n", port_id);
 		return -ENOTSUP;
 	}
 
@@ -778,21 +782,22 @@ rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id)
 
 	dev = &rte_eth_devices[port_id];
 	if (!dev->data->dev_started) {
-		RTE_PMD_DEBUG_TRACE(
-		    "port %d must be started before start any queue\n", port_id);
+		RTE_ETHDEV_LOG(ERR,
+			"Port %u must be started before start any queue\n",
+			port_id);
 		return -EINVAL;
 	}
 
 	if (rx_queue_id >= dev->data->nb_rx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 		return -EINVAL;
 	}
 
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
 
 	if (dev->data->rx_queue_state[rx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
-		RTE_PMD_DEBUG_TRACE("Queue %" PRIu16" of device with port_id=%" PRIu8
-			" already started\n",
+		RTE_ETHDEV_LOG(INFO,
+			"Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 			rx_queue_id, port_id);
 		return 0;
 	}
@@ -811,15 +816,15 @@ rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id)
 
 	dev = &rte_eth_devices[port_id];
 	if (rx_queue_id >= dev->data->nb_rx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 		return -EINVAL;
 	}
 
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
 
 	if (dev->data->rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
-		RTE_PMD_DEBUG_TRACE("Queue %" PRIu16" of device with port_id=%" PRIu8
-			" already stopped\n",
+		RTE_ETHDEV_LOG(INFO,
+			"Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
 			rx_queue_id, port_id);
 		return 0;
 	}
@@ -837,28 +842,27 @@ rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id)
 
 	dev = &rte_eth_devices[port_id];
 	if (!dev->data->dev_started) {
-		RTE_PMD_DEBUG_TRACE(
-		    "port %d must be started before start any queue\n", port_id);
+		RTE_ETHDEV_LOG(ERR,
+			"Port %u must be started before start any queue\n",
+			port_id);
 		return -EINVAL;
 	}
 
 	if (tx_queue_id >= dev->data->nb_tx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
 		return -EINVAL;
 	}
 
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
 
 	if (dev->data->tx_queue_state[tx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
-		RTE_PMD_DEBUG_TRACE("Queue %" PRIu16" of device with port_id=%" PRIu8
-			" already started\n",
+		RTE_ETHDEV_LOG(INFO,
+			"Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 			tx_queue_id, port_id);
 		return 0;
 	}
 
-	return eth_err(port_id, dev->dev_ops->tx_queue_start(dev,
-							     tx_queue_id));
-
+	return eth_err(port_id, dev->dev_ops->tx_queue_start(dev, tx_queue_id));
 }
 
 int
@@ -870,15 +874,15 @@ rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id)
 
 	dev = &rte_eth_devices[port_id];
 	if (tx_queue_id >= dev->data->nb_tx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
 		return -EINVAL;
 	}
 
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
 
 	if (dev->data->tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
-		RTE_PMD_DEBUG_TRACE("Queue %" PRIu16" of device with port_id=%" PRIu8
-			" already stopped\n",
+		RTE_ETHDEV_LOG(INFO,
+			"Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
 			tx_queue_id, port_id);
 		return 0;
 	}
@@ -970,41 +974,6 @@ rte_eth_speed_bitflag(uint32_t speed, int duplex)
 	}
 }
 
-/**
- * A conversion function from rxmode bitfield API.
- */
-static void
-rte_eth_convert_rx_offload_bitfield(const struct rte_eth_rxmode *rxmode,
-				    uint64_t *rx_offloads)
-{
-	uint64_t offloads = 0;
-
-	if (rxmode->header_split == 1)
-		offloads |= DEV_RX_OFFLOAD_HEADER_SPLIT;
-	if (rxmode->hw_ip_checksum == 1)
-		offloads |= DEV_RX_OFFLOAD_CHECKSUM;
-	if (rxmode->hw_vlan_filter == 1)
-		offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
-	if (rxmode->hw_vlan_strip == 1)
-		offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
-	if (rxmode->hw_vlan_extend == 1)
-		offloads |= DEV_RX_OFFLOAD_VLAN_EXTEND;
-	if (rxmode->jumbo_frame == 1)
-		offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
-	if (rxmode->hw_strip_crc == 1)
-		offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
-	if (rxmode->enable_scatter == 1)
-		offloads |= DEV_RX_OFFLOAD_SCATTER;
-	if (rxmode->enable_lro == 1)
-		offloads |= DEV_RX_OFFLOAD_TCP_LRO;
-	if (rxmode->hw_timestamp == 1)
-		offloads |= DEV_RX_OFFLOAD_TIMESTAMP;
-	if (rxmode->security == 1)
-		offloads |= DEV_RX_OFFLOAD_SECURITY;
-
-	*rx_offloads = offloads;
-}
-
 const char * __rte_experimental
 rte_eth_dev_rx_offload_name(uint64_t offload)
 {
@@ -1071,33 +1040,26 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 	}
 
 	if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
-		RTE_PMD_DEBUG_TRACE(
+		RTE_ETHDEV_LOG(ERR,
 			"Number of RX queues requested (%u) is greater than max supported(%d)\n",
 			nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
 		return -EINVAL;
 	}
 
 	if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
-		RTE_PMD_DEBUG_TRACE(
+		RTE_ETHDEV_LOG(ERR,
 			"Number of TX queues requested (%u) is greater than max supported(%d)\n",
 			nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
 		return -EINVAL;
 	}
 
 	if (dev->data->dev_started) {
-		RTE_PMD_DEBUG_TRACE(
-		    "port %d must be stopped to allow configuration\n", port_id);
+		RTE_ETHDEV_LOG(ERR,
+			"Port %u must be stopped to allow configuration\n",
+			port_id);
 		return -EBUSY;
 	}
 
-	/*
-	 * Convert between the offloads API to enable PMDs to support
-	 * only one of them.
-	 */
-	if (dev_conf->rxmode.ignore_offload_bitfield == 0)
-		rte_eth_convert_rx_offload_bitfield(
-				&dev_conf->rxmode, &local_conf.rxmode.offloads);
-
 	/* Copy the dev_conf parameter into the dev structure */
 	memcpy(&dev->data->dev_conf, &local_conf, sizeof(dev->data->dev_conf));
 
@@ -1107,28 +1069,28 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 	 * configured device.
 	 */
 	if (nb_rx_q > dev_info.max_rx_queues) {
-		RTE_PMD_DEBUG_TRACE("ethdev port_id=%d nb_rx_queues=%d > %d\n",
-				port_id, nb_rx_q, dev_info.max_rx_queues);
+		RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_rx_queues=%u > %u\n",
+			port_id, nb_rx_q, dev_info.max_rx_queues);
 		return -EINVAL;
 	}
 
 	if (nb_tx_q > dev_info.max_tx_queues) {
-		RTE_PMD_DEBUG_TRACE("ethdev port_id=%d nb_tx_queues=%d > %d\n",
-				port_id, nb_tx_q, dev_info.max_tx_queues);
+		RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_tx_queues=%u > %u\n",
+			port_id, nb_tx_q, dev_info.max_tx_queues);
 		return -EINVAL;
 	}
 
 	/* Check that the device supports requested interrupts */
 	if ((dev_conf->intr_conf.lsc == 1) &&
-		(!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
-			RTE_PMD_DEBUG_TRACE("driver %s does not support lsc\n",
-					dev->device->driver->name);
-			return -EINVAL;
+			(!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
+		RTE_ETHDEV_LOG(ERR, "Driver %s does not support lsc\n",
+			dev->device->driver->name);
+		return -EINVAL;
 	}
 	if ((dev_conf->intr_conf.rmv == 1) &&
-	    (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
-		RTE_PMD_DEBUG_TRACE("driver %s does not support rmv\n",
-				    dev->device->driver->name);
+			(!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
+		RTE_ETHDEV_LOG(ERR, "Driver %s does not support rmv\n",
+			dev->device->driver->name);
 		return -EINVAL;
 	}
 
@@ -1137,19 +1099,16 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 	 * length is supported by the configured device.
 	 */
 	if (local_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
-		if (dev_conf->rxmode.max_rx_pkt_len >
-		    dev_info.max_rx_pktlen) {
-			RTE_PMD_DEBUG_TRACE("ethdev port_id=%d max_rx_pkt_len %u"
-				" > max valid value %u\n",
-				port_id,
-				(unsigned)dev_conf->rxmode.max_rx_pkt_len,
-				(unsigned)dev_info.max_rx_pktlen);
+		if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen) {
+			RTE_ETHDEV_LOG(ERR,
+				"Ethdev port_id=%u max_rx_pkt_len %u > max valid value %u\n",
+				port_id, dev_conf->rxmode.max_rx_pkt_len,
+				dev_info.max_rx_pktlen);
 			return -EINVAL;
 		} else if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN) {
-			RTE_PMD_DEBUG_TRACE("ethdev port_id=%d max_rx_pkt_len %u"
-				" < min valid value %u\n",
-				port_id,
-				(unsigned)dev_conf->rxmode.max_rx_pkt_len,
+			RTE_ETHDEV_LOG(ERR,
+				"Ethdev port_id=%u max_rx_pkt_len %u < min valid value %u\n",
+				port_id, dev_conf->rxmode.max_rx_pkt_len,
 				(unsigned)ETHER_MIN_LEN);
 			return -EINVAL;
 		}
@@ -1164,36 +1123,42 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 	/* Any requested offloading must be within its device capabilities */
 	if ((local_conf.rxmode.offloads & dev_info.rx_offload_capa) !=
 	     local_conf.rxmode.offloads) {
-		ethdev_log(ERR, "ethdev port_id=%d requested Rx offloads "
-				"0x%" PRIx64 " doesn't match Rx offloads "
-				"capabilities 0x%" PRIx64 " in %s()\n",
-				port_id,
-				local_conf.rxmode.offloads,
-				dev_info.rx_offload_capa,
-				__func__);
-		/* Will return -EINVAL in the next release */
+		RTE_ETHDEV_LOG(ERR,
+			"Ethdev port_id=%u requested Rx offloads 0x%"PRIx64" doesn't match Rx offloads "
+			"capabilities 0x%"PRIx64" in %s()\n",
+			port_id, local_conf.rxmode.offloads,
+			dev_info.rx_offload_capa,
+			__func__);
+		return -EINVAL;
 	}
 	if ((local_conf.txmode.offloads & dev_info.tx_offload_capa) !=
 	     local_conf.txmode.offloads) {
-		ethdev_log(ERR, "ethdev port_id=%d requested Tx offloads "
-				"0x%" PRIx64 " doesn't match Tx offloads "
-				"capabilities 0x%" PRIx64 " in %s()\n",
-				port_id,
-				local_conf.txmode.offloads,
-				dev_info.tx_offload_capa,
-				__func__);
-		/* Will return -EINVAL in the next release */
+		RTE_ETHDEV_LOG(ERR,
+			"Ethdev port_id=%u requested Tx offloads 0x%"PRIx64" doesn't match Tx offloads "
+			"capabilities 0x%"PRIx64" in %s()\n",
+			port_id, local_conf.txmode.offloads,
+			dev_info.tx_offload_capa,
+			__func__);
+		return -EINVAL;
+	}
+
+	if ((local_conf.rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP) &&
+			(local_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)) {
+		RTE_ETHDEV_LOG(ERR,
+			"Port id=%u not allowed to set both CRC STRIP and KEEP CRC offload flags\n",
+			port_id);
+		return -EINVAL;
 	}
 
 	/* Check that device supports requested rss hash functions. */
 	if ((dev_info.flow_type_rss_offloads |
 	     dev_conf->rx_adv_conf.rss_conf.rss_hf) !=
 	    dev_info.flow_type_rss_offloads) {
-		RTE_PMD_DEBUG_TRACE("ethdev port_id=%d invalid rss_hf: "
-				    "0x%"PRIx64", valid value: 0x%"PRIx64"\n",
-				    port_id,
-				    dev_conf->rx_adv_conf.rss_conf.rss_hf,
-				    dev_info.flow_type_rss_offloads);
+		RTE_ETHDEV_LOG(ERR,
+			"Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
+			port_id, dev_conf->rx_adv_conf.rss_conf.rss_hf,
+			dev_info.flow_type_rss_offloads);
+		return -EINVAL;
 	}
 
 	/*
@@ -1201,23 +1166,25 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 	 */
 	diag = rte_eth_dev_rx_queue_config(dev, nb_rx_q);
 	if (diag != 0) {
-		RTE_PMD_DEBUG_TRACE("port%d rte_eth_dev_rx_queue_config = %d\n",
-				port_id, diag);
+		RTE_ETHDEV_LOG(ERR,
+			"Port%u rte_eth_dev_rx_queue_config = %d\n",
+			port_id, diag);
 		return diag;
 	}
 
 	diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
 	if (diag != 0) {
-		RTE_PMD_DEBUG_TRACE("port%d rte_eth_dev_tx_queue_config = %d\n",
-				port_id, diag);
+		RTE_ETHDEV_LOG(ERR,
+			"Port%u rte_eth_dev_tx_queue_config = %d\n",
+			port_id, diag);
 		rte_eth_dev_rx_queue_config(dev, 0);
 		return diag;
 	}
 
 	diag = (*dev->dev_ops->dev_configure)(dev);
 	if (diag != 0) {
-		RTE_PMD_DEBUG_TRACE("port%d dev_configure = %d\n",
-				port_id, diag);
+		RTE_ETHDEV_LOG(ERR, "Port%u dev_configure = %d\n",
+			port_id, diag);
 		rte_eth_dev_rx_queue_config(dev, 0);
 		rte_eth_dev_tx_queue_config(dev, 0);
 		return eth_err(port_id, diag);
@@ -1226,8 +1193,8 @@ rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 	/* Initialize Rx profiling if enabled at compilation time. */
 	diag = __rte_eth_profile_rx_init(port_id, dev);
 	if (diag != 0) {
-		RTE_PMD_DEBUG_TRACE("port%d __rte_eth_profile_rx_init = %d\n",
-				port_id, diag);
+		RTE_ETHDEV_LOG(ERR, "Port%u __rte_eth_profile_rx_init = %d\n",
+			port_id, diag);
 		rte_eth_dev_rx_queue_config(dev, 0);
 		rte_eth_dev_tx_queue_config(dev, 0);
 		return eth_err(port_id, diag);
@@ -1240,8 +1207,7 @@ void
 _rte_eth_dev_reset(struct rte_eth_dev *dev)
 {
 	if (dev->data->dev_started) {
-		RTE_PMD_DEBUG_TRACE(
-			"port %d must be stopped to allow reset\n",
+		RTE_ETHDEV_LOG(ERR, "Port %u must be stopped to allow reset\n",
 			dev->data->port_id);
 		return;
 	}
@@ -1320,8 +1286,8 @@ rte_eth_dev_start(uint16_t port_id)
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
 
 	if (dev->data->dev_started != 0) {
-		RTE_PMD_DEBUG_TRACE("Device with port_id=%" PRIu16
-			" already started\n",
+		RTE_ETHDEV_LOG(INFO,
+			"Device with port_id=%"PRIu16" already started\n",
 			port_id);
 		return 0;
 	}
@@ -1352,8 +1318,8 @@ rte_eth_dev_stop(uint16_t port_id)
 	RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
 
 	if (dev->data->dev_started == 0) {
-		RTE_PMD_DEBUG_TRACE("Device with port_id=%" PRIu16
-			" already stopped\n",
+		RTE_ETHDEV_LOG(INFO,
+			"Device with port_id=%"PRIu16" already stopped\n",
 			port_id);
 		return;
 	}
@@ -1465,7 +1431,7 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
 
 	dev = &rte_eth_devices[port_id];
 	if (rx_queue_id >= dev->data->nb_rx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 		return -EINVAL;
 	}
 
@@ -1479,23 +1445,20 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
 	 */
 	rte_eth_dev_info_get(port_id, &dev_info);
 	if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
-		RTE_PMD_DEBUG_TRACE("%s private_data_size %d < %d\n",
-				mp->name, (int) mp->private_data_size,
-				(int) sizeof(struct rte_pktmbuf_pool_private));
+		RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
+			mp->name, (int)mp->private_data_size,
+			(int)sizeof(struct rte_pktmbuf_pool_private));
 		return -ENOSPC;
 	}
 	mbp_buf_size = rte_pktmbuf_data_room_size(mp);
 
 	if ((mbp_buf_size - RTE_PKTMBUF_HEADROOM) < dev_info.min_rx_bufsize) {
-		RTE_PMD_DEBUG_TRACE("%s mbuf_data_room_size %d < %d "
-				"(RTE_PKTMBUF_HEADROOM=%d + min_rx_bufsize(dev)"
-				"=%d)\n",
-				mp->name,
-				(int)mbp_buf_size,
-				(int)(RTE_PKTMBUF_HEADROOM +
-				      dev_info.min_rx_bufsize),
-				(int)RTE_PKTMBUF_HEADROOM,
-				(int)dev_info.min_rx_bufsize);
+		RTE_ETHDEV_LOG(ERR,
+			"%s mbuf_data_room_size %d < %d (RTE_PKTMBUF_HEADROOM=%d + min_rx_bufsize(dev)=%d)\n",
+			mp->name, (int)mbp_buf_size,
+			(int)(RTE_PKTMBUF_HEADROOM + dev_info.min_rx_bufsize),
+			(int)RTE_PKTMBUF_HEADROOM,
+			(int)dev_info.min_rx_bufsize);
 		return -EINVAL;
 	}
 
@@ -1511,10 +1474,9 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
 			nb_rx_desc < dev_info.rx_desc_lim.nb_min ||
 			nb_rx_desc % dev_info.rx_desc_lim.nb_align != 0) {
 
-		RTE_PMD_DEBUG_TRACE("Invalid value for nb_rx_desc(=%hu), "
-			"should be: <= %hu, = %hu, and a product of %hu\n",
-			nb_rx_desc,
-			dev_info.rx_desc_lim.nb_max,
+		RTE_ETHDEV_LOG(ERR,
+			"Invalid value for nb_rx_desc(=%hu), should be: <= %hu, = %hu, and a product of %hu\n",
+			nb_rx_desc, dev_info.rx_desc_lim.nb_max,
 			dev_info.rx_desc_lim.nb_min,
 			dev_info.rx_desc_lim.nb_align);
 		return -EINVAL;
@@ -1542,14 +1504,6 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
 		rx_conf = &dev_info.default_rxconf;
 
 	local_conf = *rx_conf;
-	if (dev->data->dev_conf.rxmode.ignore_offload_bitfield == 0) {
-		/**
-		 * Reflect port offloads to queue offloads in order for
-		 * offloads to not be discarded.
-		 */
-		rte_eth_convert_rx_offload_bitfield(&dev->data->dev_conf.rxmode,
-						    &local_conf.offloads);
-	}
 
 	/*
 	 * If an offloading has already been enabled in
@@ -1571,16 +1525,13 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
 	 */
 	if ((local_conf.offloads & dev_info.rx_queue_offload_capa) !=
 	     local_conf.offloads) {
-		ethdev_log(ERR, "Ethdev port_id=%d rx_queue_id=%d, new "
-				"added offloads 0x%" PRIx64 " must be "
-				"within pre-queue offload capabilities 0x%"
-				PRIx64 " in %s()\n",
-				port_id,
-				rx_queue_id,
-				local_conf.offloads,
-				dev_info.rx_queue_offload_capa,
-				__func__);
-		/* Will return -EINVAL in the next release */
+		RTE_ETHDEV_LOG(ERR,
+			"Ethdev port_id=%d rx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
+			"within pre-queue offload capabilities 0x%"PRIx64" in %s()\n",
+			port_id, rx_queue_id, local_conf.offloads,
+			dev_info.rx_queue_offload_capa,
+			__func__);
+		return -EINVAL;
 	}
 
 	ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
@@ -1594,55 +1545,6 @@ rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
 	return eth_err(port_id, ret);
 }
 
-/**
- * Convert from tx offloads to txq_flags.
- */
-static void
-rte_eth_convert_tx_offload(const uint64_t tx_offloads, uint32_t *txq_flags)
-{
-	uint32_t flags = 0;
-
-	if (!(tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS))
-		flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
-	if (!(tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT))
-		flags |= ETH_TXQ_FLAGS_NOVLANOFFL;
-	if (!(tx_offloads & DEV_TX_OFFLOAD_SCTP_CKSUM))
-		flags |= ETH_TXQ_FLAGS_NOXSUMSCTP;
-	if (!(tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM))
-		flags |= ETH_TXQ_FLAGS_NOXSUMUDP;
-	if (!(tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM))
-		flags |= ETH_TXQ_FLAGS_NOXSUMTCP;
-	if (tx_offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
-		flags |= ETH_TXQ_FLAGS_NOREFCOUNT | ETH_TXQ_FLAGS_NOMULTMEMP;
-
-	*txq_flags = flags;
-}
-
-/**
- * A conversion function from txq_flags API.
- */
-static void
-rte_eth_convert_txq_flags(const uint32_t txq_flags, uint64_t *tx_offloads)
-{
-	uint64_t offloads = 0;
-
-	if (!(txq_flags & ETH_TXQ_FLAGS_NOMULTSEGS))
-		offloads |= DEV_TX_OFFLOAD_MULTI_SEGS;
-	if (!(txq_flags & ETH_TXQ_FLAGS_NOVLANOFFL))
-		offloads |= DEV_TX_OFFLOAD_VLAN_INSERT;
-	if (!(txq_flags & ETH_TXQ_FLAGS_NOXSUMSCTP))
-		offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM;
-	if (!(txq_flags & ETH_TXQ_FLAGS_NOXSUMUDP))
-		offloads |= DEV_TX_OFFLOAD_UDP_CKSUM;
-	if (!(txq_flags & ETH_TXQ_FLAGS_NOXSUMTCP))
-		offloads |= DEV_TX_OFFLOAD_TCP_CKSUM;
-	if ((txq_flags & ETH_TXQ_FLAGS_NOREFCOUNT) &&
-	    (txq_flags & ETH_TXQ_FLAGS_NOMULTMEMP))
-		offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE;
-
-	*tx_offloads = offloads;
-}
-
 int
 rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
 		       uint16_t nb_tx_desc, unsigned int socket_id,
@@ -1657,7 +1559,7 @@ rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
 
 	dev = &rte_eth_devices[port_id];
 	if (tx_queue_id >= dev->data->nb_tx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
 		return -EINVAL;
 	}
 
@@ -1676,12 +1578,11 @@ rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
 	if (nb_tx_desc > dev_info.tx_desc_lim.nb_max ||
 	    nb_tx_desc < dev_info.tx_desc_lim.nb_min ||
 	    nb_tx_desc % dev_info.tx_desc_lim.nb_align != 0) {
-		RTE_PMD_DEBUG_TRACE("Invalid value for nb_tx_desc(=%hu), "
-				"should be: <= %hu, = %hu, and a product of %hu\n",
-				nb_tx_desc,
-				dev_info.tx_desc_lim.nb_max,
-				dev_info.tx_desc_lim.nb_min,
-				dev_info.tx_desc_lim.nb_align);
+		RTE_ETHDEV_LOG(ERR,
+			"Invalid value for nb_tx_desc(=%hu), should be: <= %hu, = %hu, and a product of %hu\n",
+			nb_tx_desc, dev_info.tx_desc_lim.nb_max,
+			dev_info.tx_desc_lim.nb_min,
+			dev_info.tx_desc_lim.nb_align);
 		return -EINVAL;
 	}
 
@@ -1706,15 +1607,7 @@ rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
 	if (tx_conf == NULL)
 		tx_conf = &dev_info.default_txconf;
 
-	/*
-	 * Convert between the offloads API to enable PMDs to support
-	 * only one of them.
-	 */
 	local_conf = *tx_conf;
-	if (!(tx_conf->txq_flags & ETH_TXQ_FLAGS_IGNORE)) {
-		rte_eth_convert_txq_flags(tx_conf->txq_flags,
-					  &local_conf.offloads);
-	}
 
 	/*
 	 * If an offloading has already been enabled in
@@ -1736,16 +1629,13 @@ rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
 	 */
 	if ((local_conf.offloads & dev_info.tx_queue_offload_capa) !=
 	     local_conf.offloads) {
-		ethdev_log(ERR, "Ethdev port_id=%d tx_queue_id=%d, new "
-				"added offloads 0x%" PRIx64 " must be "
-				"within pre-queue offload capabilities 0x%"
-				PRIx64 " in %s()\n",
-				port_id,
-				tx_queue_id,
-				local_conf.offloads,
-				dev_info.tx_queue_offload_capa,
-				__func__);
-		/* Will return -EINVAL in the next release */
+		RTE_ETHDEV_LOG(ERR,
+			"Ethdev port_id=%d tx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
+			"within pre-queue offload capabilities 0x%"PRIx64" in %s()\n",
+			port_id, tx_queue_id, local_conf.offloads,
+			dev_info.tx_queue_offload_capa,
+			__func__);
+		return -EINVAL;
 	}
 
 	return eth_err(port_id, (*dev->dev_ops->tx_queue_setup)(dev,
@@ -2009,19 +1899,19 @@ rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
 
 	if (!id) {
-		RTE_PMD_DEBUG_TRACE("Error: id pointer is NULL\n");
+		RTE_ETHDEV_LOG(ERR, "Id pointer is NULL\n");
 		return -ENOMEM;
 	}
 
 	if (!xstat_name) {
-		RTE_PMD_DEBUG_TRACE("Error: xstat_name pointer is NULL\n");
+		RTE_ETHDEV_LOG(ERR, "xstat_name pointer is NULL\n");
 		return -ENOMEM;
 	}
 
 	/* Get count */
 	cnt_xstats = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL);
 	if (cnt_xstats  < 0) {
-		RTE_PMD_DEBUG_TRACE("Error: Cannot get count of xstats\n");
+		RTE_ETHDEV_LOG(ERR, "Cannot get count of xstats\n");
 		return -ENODEV;
 	}
 
@@ -2030,7 +1920,7 @@ rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
 
 	if (cnt_xstats != rte_eth_xstats_get_names_by_id(
 			port_id, xstats_names, cnt_xstats, NULL)) {
-		RTE_PMD_DEBUG_TRACE("Error: Cannot get xstats lookup\n");
+		RTE_ETHDEV_LOG(ERR, "Cannot get xstats lookup\n");
 		return -1;
 	}
 
@@ -2153,7 +2043,7 @@ rte_eth_xstats_get_names_by_id(uint16_t port_id,
 		sizeof(struct rte_eth_xstat_name));
 
 	if (!xstats_names_copy) {
-		RTE_PMD_DEBUG_TRACE("ERROR: can't allocate memory");
+		RTE_ETHDEV_LOG(ERR, "Can't allocate memory\n");
 		return -ENOMEM;
 	}
 
@@ -2181,7 +2071,7 @@ rte_eth_xstats_get_names_by_id(uint16_t port_id,
 	/* Filter stats */
 	for (i = 0; i < size; i++) {
 		if (ids[i] >= expected_entries) {
-			RTE_PMD_DEBUG_TRACE("ERROR: id value isn't valid\n");
+			RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
 			free(xstats_names_copy);
 			return -1;
 		}
@@ -2366,7 +2256,7 @@ rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
 	/* Filter stats */
 	for (i = 0; i < size; i++) {
 		if (ids[i] >= expected_entries) {
-			RTE_PMD_DEBUG_TRACE("ERROR: id value isn't valid\n");
+			RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
 			return -1;
 		}
 		values[i] = xstats[ids[i]].value;
@@ -2456,6 +2346,16 @@ set_queue_stats_mapping(uint16_t port_id, uint16_t queue_id, uint8_t stat_idx,
 	dev = &rte_eth_devices[port_id];
 
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
+
+	if (is_rx && (queue_id >= dev->data->nb_rx_queues))
+		return -EINVAL;
+
+	if (!is_rx && (queue_id >= dev->data->nb_tx_queues))
+		return -EINVAL;
+
+	if (stat_idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
+		return -EINVAL;
+
 	return (*dev->dev_ops->queue_stats_mapping_set)
 			(dev, queue_id, stat_idx, is_rx);
 }
@@ -2495,7 +2395,6 @@ void
 rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
 {
 	struct rte_eth_dev *dev;
-	struct rte_eth_txconf *txconf;
 	const struct rte_eth_desc_lim lim = {
 		.nb_max = UINT16_MAX,
 		.nb_min = 0,
@@ -2517,9 +2416,6 @@ rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
 	dev_info->nb_tx_queues = dev->data->nb_tx_queues;
 
 	dev_info->dev_flags = &dev->data->dev_flags;
-	txconf = &dev_info->default_txconf;
-	/* convert offload to txq_flags to support legacy app */
-	rte_eth_convert_tx_offload(txconf->offloads, &txconf->txq_flags);
 }
 
 int
@@ -2598,13 +2494,14 @@ rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on)
 	dev = &rte_eth_devices[port_id];
 	if (!(dev->data->dev_conf.rxmode.offloads &
 	      DEV_RX_OFFLOAD_VLAN_FILTER)) {
-		RTE_PMD_DEBUG_TRACE("port %d: vlan-filtering disabled\n", port_id);
+		RTE_ETHDEV_LOG(ERR, "Port %u: vlan-filtering disabled\n",
+			port_id);
 		return -ENOSYS;
 	}
 
 	if (vlan_id > 4095) {
-		RTE_PMD_DEBUG_TRACE("(port_id=%d) invalid vlan_id=%u > 4095\n",
-				port_id, (unsigned) vlan_id);
+		RTE_ETHDEV_LOG(ERR, "Port_id=%u invalid vlan_id=%u > 4095\n",
+			port_id, vlan_id);
 		return -EINVAL;
 	}
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
@@ -2637,7 +2534,7 @@ rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id,
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
 	dev = &rte_eth_devices[port_id];
 	if (rx_queue_id >= dev->data->nb_rx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid rx_queue_id=%d\n", port_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid rx_queue_id=%u\n", rx_queue_id);
 		return -EINVAL;
 	}
 
@@ -2786,7 +2683,7 @@ rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
 
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
 	if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
-		RTE_PMD_DEBUG_TRACE("Invalid send_xon, only 0/1 allowed\n");
+		RTE_ETHDEV_LOG(ERR, "Invalid send_xon, only 0/1 allowed\n");
 		return -EINVAL;
 	}
 
@@ -2803,7 +2700,7 @@ rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id,
 
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
 	if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
-		RTE_PMD_DEBUG_TRACE("Invalid priority, only 0-7 allowed\n");
+		RTE_ETHDEV_LOG(ERR, "Invalid priority, only 0-7 allowed\n");
 		return -EINVAL;
 	}
 
@@ -2844,7 +2741,7 @@ rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
 		return -EINVAL;
 
 	if (max_rxq == 0) {
-		RTE_PMD_DEBUG_TRACE("No receive queue is available\n");
+		RTE_ETHDEV_LOG(ERR, "No receive queue is available\n");
 		return -EINVAL;
 	}
 
@@ -2853,8 +2750,9 @@ rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
 		shift = i % RTE_RETA_GROUP_SIZE;
 		if ((reta_conf[idx].mask & (1ULL << shift)) &&
 			(reta_conf[idx].reta[shift] >= max_rxq)) {
-			RTE_PMD_DEBUG_TRACE("reta_conf[%u]->reta[%u]: %u exceeds "
-				"the maximum rxq index: %u\n", idx, shift,
+			RTE_ETHDEV_LOG(ERR,
+				"reta_conf[%u]->reta[%u]: %u exceeds the maximum rxq index: %u\n",
+				idx, shift,
 				reta_conf[idx].reta[shift], max_rxq);
 			return -EINVAL;
 		}
@@ -2923,11 +2821,11 @@ rte_eth_dev_rss_hash_update(uint16_t port_id,
 	rte_eth_dev_info_get(port_id, &dev_info);
 	if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
 	    dev_info.flow_type_rss_offloads) {
-		RTE_PMD_DEBUG_TRACE("ethdev port_id=%d invalid rss_hf: "
-				    "0x%"PRIx64", valid value: 0x%"PRIx64"\n",
-				    port_id,
-				    rss_conf->rss_hf,
-				    dev_info.flow_type_rss_offloads);
+		RTE_ETHDEV_LOG(ERR,
+			"Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
+			port_id, rss_conf->rss_hf,
+			dev_info.flow_type_rss_offloads);
+		return -EINVAL;
 	}
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
 	return eth_err(port_id, (*dev->dev_ops->rss_hash_update)(dev,
@@ -2955,12 +2853,12 @@ rte_eth_dev_udp_tunnel_port_add(uint16_t port_id,
 
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
 	if (udp_tunnel == NULL) {
-		RTE_PMD_DEBUG_TRACE("Invalid udp_tunnel parameter\n");
+		RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
 		return -EINVAL;
 	}
 
 	if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
-		RTE_PMD_DEBUG_TRACE("Invalid tunnel type\n");
+		RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
 		return -EINVAL;
 	}
 
@@ -2980,12 +2878,12 @@ rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id,
 	dev = &rte_eth_devices[port_id];
 
 	if (udp_tunnel == NULL) {
-		RTE_PMD_DEBUG_TRACE("Invalid udp_tunnel parameter\n");
+		RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
 		return -EINVAL;
 	}
 
 	if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
-		RTE_PMD_DEBUG_TRACE("Invalid tunnel type\n");
+		RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
 		return -EINVAL;
 	}
 
@@ -3053,12 +2951,12 @@ rte_eth_dev_mac_addr_add(uint16_t port_id, struct ether_addr *addr,
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
 
 	if (is_zero_ether_addr(addr)) {
-		RTE_PMD_DEBUG_TRACE("port %d: Cannot add NULL MAC address\n",
+		RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
 			port_id);
 		return -EINVAL;
 	}
 	if (pool >= ETH_64_POOLS) {
-		RTE_PMD_DEBUG_TRACE("pool id must be 0-%d\n", ETH_64_POOLS - 1);
+		RTE_ETHDEV_LOG(ERR, "Pool id must be 0-%d\n", ETH_64_POOLS - 1);
 		return -EINVAL;
 	}
 
@@ -3066,7 +2964,7 @@ rte_eth_dev_mac_addr_add(uint16_t port_id, struct ether_addr *addr,
 	if (index < 0) {
 		index = get_mac_addr_index(port_id, &null_mac_addr);
 		if (index < 0) {
-			RTE_PMD_DEBUG_TRACE("port %d: MAC address array full\n",
+			RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
 				port_id);
 			return -ENOSPC;
 		}
@@ -3104,7 +3002,9 @@ rte_eth_dev_mac_addr_remove(uint16_t port_id, struct ether_addr *addr)
 
 	index = get_mac_addr_index(port_id, addr);
 	if (index == 0) {
-		RTE_PMD_DEBUG_TRACE("port %d: Cannot remove default MAC address\n", port_id);
+		RTE_ETHDEV_LOG(ERR,
+			"Port %u: Cannot remove default MAC address\n",
+			port_id);
 		return -EADDRINUSE;
 	} else if (index < 0)
 		return 0;  /* Do nothing if address wasn't found */
@@ -3181,7 +3081,7 @@ rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct ether_addr *addr,
 
 	dev = &rte_eth_devices[port_id];
 	if (is_zero_ether_addr(addr)) {
-		RTE_PMD_DEBUG_TRACE("port %d: Cannot add NULL MAC address\n",
+		RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
 			port_id);
 		return -EINVAL;
 	}
@@ -3193,15 +3093,16 @@ rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct ether_addr *addr,
 
 	if (index < 0) {
 		if (!on) {
-			RTE_PMD_DEBUG_TRACE("port %d: the MAC address was not "
-				"set in UTA\n", port_id);
+			RTE_ETHDEV_LOG(ERR,
+				"Port %u: the MAC address was not set in UTA\n",
+				port_id);
 			return -EINVAL;
 		}
 
 		index = get_hash_mac_addr_index(port_id, &null_mac_addr);
 		if (index < 0) {
-			RTE_PMD_DEBUG_TRACE("port %d: MAC address array full\n",
-					port_id);
+			RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
+				port_id);
 			return -ENOSPC;
 		}
 	}
@@ -3249,14 +3150,15 @@ int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
 	link = dev->data->dev_link;
 
 	if (queue_idx > dev_info.max_tx_queues) {
-		RTE_PMD_DEBUG_TRACE("set queue rate limit:port %d: "
-				"invalid queue id=%d\n", port_id, queue_idx);
+		RTE_ETHDEV_LOG(ERR,
+			"Set queue rate limit:port %u: invalid queue id=%u\n",
+			port_id, queue_idx);
 		return -EINVAL;
 	}
 
 	if (tx_rate > link.link_speed) {
-		RTE_PMD_DEBUG_TRACE("set queue rate limit:invalid tx_rate=%d, "
-				"bigger than link speed= %d\n",
+		RTE_ETHDEV_LOG(ERR,
+			"Set queue rate limit:invalid tx_rate=%u, bigger than link speed= %d\n",
 			tx_rate, link.link_speed);
 		return -EINVAL;
 	}
@@ -3275,26 +3177,28 @@ rte_eth_mirror_rule_set(uint16_t port_id,
 
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
 	if (mirror_conf->rule_type == 0) {
-		RTE_PMD_DEBUG_TRACE("mirror rule type can not be 0.\n");
+		RTE_ETHDEV_LOG(ERR, "Mirror rule type can not be 0\n");
 		return -EINVAL;
 	}
 
 	if (mirror_conf->dst_pool >= ETH_64_POOLS) {
-		RTE_PMD_DEBUG_TRACE("Invalid dst pool, pool id must be 0-%d\n",
-				ETH_64_POOLS - 1);
+		RTE_ETHDEV_LOG(ERR, "Invalid dst pool, pool id must be 0-%d\n",
+			ETH_64_POOLS - 1);
 		return -EINVAL;
 	}
 
 	if ((mirror_conf->rule_type & (ETH_MIRROR_VIRTUAL_POOL_UP |
 	     ETH_MIRROR_VIRTUAL_POOL_DOWN)) &&
 	    (mirror_conf->pool_mask == 0)) {
-		RTE_PMD_DEBUG_TRACE("Invalid mirror pool, pool mask can not be 0.\n");
+		RTE_ETHDEV_LOG(ERR,
+			"Invalid mirror pool, pool mask can not be 0\n");
 		return -EINVAL;
 	}
 
 	if ((mirror_conf->rule_type & ETH_MIRROR_VLAN) &&
 	    mirror_conf->vlan.vlan_mask == 0) {
-		RTE_PMD_DEBUG_TRACE("Invalid vlan mask, vlan mask can not be 0.\n");
+		RTE_ETHDEV_LOG(ERR,
+			"Invalid vlan mask, vlan mask can not be 0\n");
 		return -EINVAL;
 	}
 
@@ -3341,7 +3245,7 @@ rte_eth_dev_callback_register(uint16_t port_id,
 		return -EINVAL;
 
 	if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
-		ethdev_log(ERR, "Invalid port_id=%d", port_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
 		return -EINVAL;
 	}
 
@@ -3404,7 +3308,7 @@ rte_eth_dev_callback_unregister(uint16_t port_id,
 		return -EINVAL;
 
 	if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
-		ethdev_log(ERR, "Invalid port_id=%d", port_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
 		return -EINVAL;
 	}
 
@@ -3498,13 +3402,13 @@ rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data)
 	dev = &rte_eth_devices[port_id];
 
 	if (!dev->intr_handle) {
-		RTE_PMD_DEBUG_TRACE("RX Intr handle unset\n");
+		RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
 		return -ENOTSUP;
 	}
 
 	intr_handle = dev->intr_handle;
 	if (!intr_handle->intr_vec) {
-		RTE_PMD_DEBUG_TRACE("RX Intr vector unset\n");
+		RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
 		return -EPERM;
 	}
 
@@ -3512,9 +3416,9 @@ rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data)
 		vec = intr_handle->intr_vec[qid];
 		rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
 		if (rc && rc != -EEXIST) {
-			RTE_PMD_DEBUG_TRACE("p %u q %u rx ctl error"
-					" op %d epfd %d vec %u\n",
-					port_id, qid, op, epfd, vec);
+			RTE_ETHDEV_LOG(ERR,
+				"p %u q %u rx ctl error op %d epfd %d vec %u\n",
+				port_id, qid, op, epfd, vec);
 		}
 	}
 
@@ -3649,27 +3553,27 @@ rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id,
 
 	dev = &rte_eth_devices[port_id];
 	if (queue_id >= dev->data->nb_rx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%u\n", queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
 		return -EINVAL;
 	}
 
 	if (!dev->intr_handle) {
-		RTE_PMD_DEBUG_TRACE("RX Intr handle unset\n");
+		RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
 		return -ENOTSUP;
 	}
 
 	intr_handle = dev->intr_handle;
 	if (!intr_handle->intr_vec) {
-		RTE_PMD_DEBUG_TRACE("RX Intr vector unset\n");
+		RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
 		return -EPERM;
 	}
 
 	vec = intr_handle->intr_vec[queue_id];
 	rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
 	if (rc && rc != -EEXIST) {
-		RTE_PMD_DEBUG_TRACE("p %u q %u rx ctl error"
-				" op %d epfd %d vec %u\n",
-				port_id, queue_id, op, epfd, vec);
+		RTE_ETHDEV_LOG(ERR,
+			"p %u q %u rx ctl error op %d epfd %d vec %u\n",
+			port_id, queue_id, op, epfd, vec);
 		return rc;
 	}
 
@@ -3936,7 +3840,7 @@ rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id,
 
 	dev = &rte_eth_devices[port_id];
 	if (queue_id >= dev->data->nb_rx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
 		return -EINVAL;
 	}
 
@@ -3952,7 +3856,6 @@ rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
 	struct rte_eth_txq_info *qinfo)
 {
 	struct rte_eth_dev *dev;
-	struct rte_eth_txconf *txconf = &qinfo->conf;
 
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
 
@@ -3961,7 +3864,7 @@ rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
 
 	dev = &rte_eth_devices[port_id];
 	if (queue_id >= dev->data->nb_tx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
 		return -EINVAL;
 	}
 
@@ -3969,8 +3872,6 @@ rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
 
 	memset(qinfo, 0, sizeof(*qinfo));
 	dev->dev_ops->txq_info_get(dev, queue_id, qinfo);
-	/* convert offload to txq_flags to support legacy app */
-	rte_eth_convert_tx_offload(txconf->offloads, &txconf->txq_flags);
 
 	return 0;
 }
@@ -4178,12 +4079,12 @@ rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id,
 
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
 	if (l2_tunnel == NULL) {
-		RTE_PMD_DEBUG_TRACE("Invalid l2_tunnel parameter\n");
+		RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
 		return -EINVAL;
 	}
 
 	if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
-		RTE_PMD_DEBUG_TRACE("Invalid tunnel type\n");
+		RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
 		return -EINVAL;
 	}
 
@@ -4205,17 +4106,17 @@ rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id,
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
 
 	if (l2_tunnel == NULL) {
-		RTE_PMD_DEBUG_TRACE("Invalid l2_tunnel parameter\n");
+		RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
 		return -EINVAL;
 	}
 
 	if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
-		RTE_PMD_DEBUG_TRACE("Invalid tunnel type.\n");
+		RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
 		return -EINVAL;
 	}
 
 	if (mask == 0) {
-		RTE_PMD_DEBUG_TRACE("Mask should have a value.\n");
+		RTE_ETHDEV_LOG(ERR, "Mask should have a value\n");
 		return -EINVAL;
 	}
 
@@ -4516,11 +4417,9 @@ parse_cleanup:
 	return result;
 }
 
-RTE_INIT(ethdev_init_log);
-static void
-ethdev_init_log(void)
+RTE_INIT(ethdev_init_log)
 {
-	ethdev_logtype = rte_log_register("lib.ethdev");
-	if (ethdev_logtype >= 0)
-		rte_log_set_level(ethdev_logtype, RTE_LOG_INFO);
+	rte_eth_dev_logtype = rte_log_register("lib.ethdev");
+	if (rte_eth_dev_logtype >= 0)
+		rte_log_set_level(rte_eth_dev_logtype, RTE_LOG_INFO);
 }
diff --git a/lib/librte_ethdev/rte_ethdev.h b/lib/librte_ethdev/rte_ethdev.h
index 36e3984e..7070e9ab 100644
--- a/lib/librte_ethdev/rte_ethdev.h
+++ b/lib/librte_ethdev/rte_ethdev.h
@@ -159,6 +159,11 @@ extern "C" {
 #include "rte_eth_ctrl.h"
 #include "rte_dev_info.h"
 
+extern int rte_eth_dev_logtype;
+
+#define RTE_ETHDEV_LOG(level, ...) \
+	rte_log(RTE_LOG_ ## level, rte_eth_dev_logtype, "" __VA_ARGS__)
+
 struct rte_mbuf;
 
 /**
@@ -321,7 +326,7 @@ enum rte_eth_tx_mq_mode {
 struct rte_eth_rxmode {
 	/** The multi-queue packet distribution mode to be used, e.g. RSS. */
 	enum rte_eth_rx_mq_mode mq_mode;
-	uint32_t max_rx_pkt_len;  /**< Only used if jumbo_frame enabled. */
+	uint32_t max_rx_pkt_len;  /**< Only used if JUMBO_FRAME enabled. */
 	uint16_t split_hdr_size;  /**< hdr buf size (header_split enabled).*/
 	/**
 	 * Per-port Rx offloads to be set using DEV_RX_OFFLOAD_* flags.
@@ -329,33 +334,6 @@ struct rte_eth_rxmode {
 	 * structure are allowed to be set.
 	 */
 	uint64_t offloads;
-	__extension__
-	/**
-	 * Below bitfield API is obsolete. Application should
-	 * enable per-port offloads using the offload field
-	 * above.
-	 */
-	uint16_t header_split : 1, /**< Header Split enable. */
-		hw_ip_checksum   : 1, /**< IP/UDP/TCP checksum offload enable. */
-		hw_vlan_filter   : 1, /**< VLAN filter enable. */
-		hw_vlan_strip    : 1, /**< VLAN strip enable. */
-		hw_vlan_extend   : 1, /**< Extended VLAN enable. */
-		jumbo_frame      : 1, /**< Jumbo Frame Receipt enable. */
-		hw_strip_crc     : 1, /**< Enable CRC stripping by hardware. */
-		enable_scatter   : 1, /**< Enable scatter packets rx handler */
-		enable_lro       : 1, /**< Enable LRO */
-		hw_timestamp     : 1, /**< Enable HW timestamp */
-		security	 : 1, /**< Enable rte_security offloads */
-		/**
-		 * When set the offload bitfield should be ignored.
-		 * Instead per-port Rx offloads should be set on offloads
-		 * field above.
-		 * Per-queue offloads shuold be set on rte_eth_rxq_conf
-		 * structure.
-		 * This bit is temporary till rxmode bitfield offloads API will
-		 * be deprecated.
-		 */
-		ignore_offload_bitfield : 1;
 };
 
 /**
@@ -702,28 +680,6 @@ struct rte_eth_rxconf {
 	uint64_t offloads;
 };
 
-#define ETH_TXQ_FLAGS_NOMULTSEGS 0x0001 /**< nb_segs=1 for all mbufs */
-#define ETH_TXQ_FLAGS_NOREFCOUNT 0x0002 /**< refcnt can be ignored */
-#define ETH_TXQ_FLAGS_NOMULTMEMP 0x0004 /**< all bufs come from same mempool */
-#define ETH_TXQ_FLAGS_NOVLANOFFL 0x0100 /**< disable VLAN offload */
-#define ETH_TXQ_FLAGS_NOXSUMSCTP 0x0200 /**< disable SCTP checksum offload */
-#define ETH_TXQ_FLAGS_NOXSUMUDP  0x0400 /**< disable UDP checksum offload */
-#define ETH_TXQ_FLAGS_NOXSUMTCP  0x0800 /**< disable TCP checksum offload */
-#define ETH_TXQ_FLAGS_NOOFFLOADS \
-		(ETH_TXQ_FLAGS_NOVLANOFFL | ETH_TXQ_FLAGS_NOXSUMSCTP | \
-		 ETH_TXQ_FLAGS_NOXSUMUDP  | ETH_TXQ_FLAGS_NOXSUMTCP)
-#define ETH_TXQ_FLAGS_NOXSUMS \
-		(ETH_TXQ_FLAGS_NOXSUMSCTP | ETH_TXQ_FLAGS_NOXSUMUDP | \
-		 ETH_TXQ_FLAGS_NOXSUMTCP)
-/**
- * When set the txq_flags should be ignored,
- * instead per-queue Tx offloads will be set on offloads field
- * located on rte_eth_txq_conf struct.
- * This flag is temporary till the rte_eth_txq_conf.txq_flags
- * API will be deprecated.
- */
-#define ETH_TXQ_FLAGS_IGNORE	0x8000
-
 /**
  * A structure used to configure a TX ring of an Ethernet port.
  */
@@ -733,7 +689,6 @@ struct rte_eth_txconf {
 	uint16_t tx_free_thresh; /**< Start freeing TX buffers if there are
 				      less free descriptors than this value. */
 
-	uint32_t txq_flags; /**< Set flags for the Tx queue */
 	uint8_t tx_deferred_start; /**< Do not start queue with rte_eth_dev_start(). */
 	/**
 	 * Per-queue Tx offloads to be set  using DEV_TX_OFFLOAD_* flags.
@@ -939,6 +894,12 @@ struct rte_eth_conf {
 #define DEV_RX_OFFLOAD_SCATTER		0x00002000
 #define DEV_RX_OFFLOAD_TIMESTAMP	0x00004000
 #define DEV_RX_OFFLOAD_SECURITY         0x00008000
+
+/**
+ * Invalid to set both DEV_RX_OFFLOAD_CRC_STRIP and DEV_RX_OFFLOAD_KEEP_CRC
+ * No DEV_RX_OFFLOAD_CRC_STRIP flag means keep CRC
+ */
+#define DEV_RX_OFFLOAD_KEEP_CRC		0x00010000
 #define DEV_RX_OFFLOAD_CHECKSUM (DEV_RX_OFFLOAD_IPV4_CKSUM | \
 				 DEV_RX_OFFLOAD_UDP_CKSUM | \
 				 DEV_RX_OFFLOAD_TCP_CKSUM)
@@ -1003,8 +964,6 @@ struct rte_eth_conf {
  * mentioned in rte_tx_offload_names in rte_ethdev.c file.
  */
 
-struct rte_pci_device;
-
 /*
  * Fallback default preferred Rx/Tx port parameters.
  * These are used if an application requests default parameters
@@ -1194,14 +1153,14 @@ struct rte_eth_dcb_info {
 /* Macros to check for valid port */
 #define RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, retval) do { \
 	if (!rte_eth_dev_is_valid_port(port_id)) { \
-		RTE_PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id); \
+		RTE_ETHDEV_LOG(ERR, "Invalid port_id=%u\n", port_id); \
 		return retval; \
 	} \
 } while (0)
 
 #define RTE_ETH_VALID_PORTID_OR_RET(port_id) do { \
 	if (!rte_eth_dev_is_valid_port(port_id)) { \
-		RTE_PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id); \
+		RTE_ETHDEV_LOG(ERR, "Invalid port_id=%u\n", port_id); \
 		return; \
 	} \
 } while (0)
@@ -1472,6 +1431,7 @@ uint16_t __rte_experimental rte_eth_dev_count_total(void);
  * @return
  *  0 on success and port_id is filled, negative on error
  */
+__rte_deprecated
 int rte_eth_dev_attach(const char *devargs, uint16_t *port_id);
 
 /**
@@ -1487,6 +1447,7 @@ int rte_eth_dev_attach(const char *devargs, uint16_t *port_id);
  * @return
  *  0 on success and devname is filled, negative on error
  */
+__rte_deprecated
 int rte_eth_dev_detach(uint16_t port_id, char *devname);
 
 /**
@@ -1554,9 +1515,11 @@ const char * __rte_experimental rte_eth_dev_tx_offload_name(uint64_t offload);
  *        the [rt]x_offload_capa returned from rte_eth_dev_infos_get().
  *        Any type of device supported offloading set in the input argument
  *        eth_conf->[rt]xmode.offloads to rte_eth_dev_configure() is enabled
- *        on all queues and it can't be disabled in rte_eth_[rt]x_queue_setup().
- *     - the Receive Side Scaling (RSS) configuration when using multiple RX
- *         queues per port.
+ *        on all queues and it can't be disabled in rte_eth_[rt]x_queue_setup()
+ *     -  the Receive Side Scaling (RSS) configuration when using multiple RX
+ *        queues per port. Any RSS hash function set in eth_conf->rss_conf.rss_hf
+ *        must be within the flow_type_rss_offloads provided by drivers via
+ *        rte_eth_dev_infos_get() API.
  *
  *   Embedding all configuration information in a single data structure
  *   is the more flexible method that allows the addition of new features
@@ -1669,12 +1632,6 @@ int rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
  *     The *tx_rs_thresh* value should be less or equal then
  *     *tx_free_thresh* value, and both of them should be less then
  *     *nb_tx_desc* - 3.
- *   - The *txq_flags* member contains flags to pass to the TX queue setup
- *     function to configure the behavior of the TX queue. This should be set
- *     to 0 if no special configuration is required.
- *     This API is obsolete and will be deprecated. Applications
- *     should set it to ETH_TXQ_FLAGS_IGNORE and use
- *     the offloads field below.
  *   - The *offloads* member contains Tx offloads to be enabled.
  *     If an offloading set in tx_conf->offloads
  *     hasn't been set in the input argument eth_conf->txmode.offloads
@@ -2003,6 +1960,15 @@ int rte_eth_stats_reset(uint16_t port_id);
 /**
  * Retrieve names of extended statistics of an Ethernet device.
  *
+ * There is an assumption that 'xstat_names' and 'xstats' arrays are matched
+ * by array index:
+ *  xstats_names[i].name => xstats[i].value
+ *
+ * And the array index is same with id field of 'struct rte_eth_xstat':
+ *  xstats[i].id == i
+ *
+ * This assumption makes key-value pair matching less flexible but simpler.
+ *
  * @param port_id
  *   The port identifier of the Ethernet device.
  * @param xstats_names
@@ -2027,13 +1993,20 @@ int rte_eth_xstats_get_names(uint16_t port_id,
 /**
  * Retrieve extended statistics of an Ethernet device.
  *
+ * There is an assumption that 'xstat_names' and 'xstats' arrays are matched
+ * by array index:
+ *  xstats_names[i].name => xstats[i].value
+ *
+ * And the array index is same with id field of 'struct rte_eth_xstat':
+ *  xstats[i].id == i
+ *
+ * This assumption makes key-value pair matching less flexible but simpler.
+ *
  * @param port_id
  *   The port identifier of the Ethernet device.
  * @param xstats
  *   A pointer to a table of structure of type *rte_eth_xstat*
- *   to be filled with device statistics ids and values: id is the
- *   index of the name string in xstats_names (see rte_eth_xstats_get_names()),
- *   and value is the statistic counter.
+ *   to be filled with device statistics ids and values.
  *   This parameter can be set to NULL if n is 0.
  * @param n
  *   The size of the xstats array (number of elements).
@@ -2144,7 +2117,7 @@ void rte_eth_xstats_reset(uint16_t port_id);
  * @param stat_idx
  *   The per-queue packet statistics functionality number that the transmit
  *   queue is to be assigned.
- *   The value must be in the range [0, RTE_MAX_ETHPORT_QUEUE_STATS_MAPS - 1].
+ *   The value must be in the range [0, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1].
  * @return
  *   Zero if successful. Non-zero otherwise.
  */
@@ -2164,7 +2137,7 @@ int rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id,
  * @param stat_idx
  *   The per-queue packet statistics functionality number that the receive
  *   queue is to be assigned.
- *   The value must be in the range [0, RTE_MAX_ETHPORT_QUEUE_STATS_MAPS - 1].
+ *   The value must be in the range [0, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1].
  * @return
  *   Zero if successful. Non-zero otherwise.
  */
@@ -3656,11 +3629,11 @@ rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id,
 				  uint8_t en);
 
 /**
-* Get the port id from pci address or device name
-* Example:
-* - PCIe, 0000:2:00.0
-* - SoC, fsl-gmac0
-* - vdev, net_pcap0
+* Get the port id from device name. The device name should be specified
+* as below:
+* - PCIe address (Domain:Bus:Device.Function), for example- 0000:2:00.0
+* - SoC device name, for example- fsl-gmac0
+* - vdev dpdk name, for example- net_[pcap0|null0|tap0]
 *
 * @param name
 *  pci address or name of the device
@@ -3674,11 +3647,10 @@ int
 rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id);
 
 /**
-* Get the device name from port id
-* Example:
-* - PCIe Bus:Domain:Function, 0000:02:00.0
-* - SoC device name, fsl-gmac0
-* - vdev dpdk name, net_[pcap0|null0|tun0|tap0]
+* Get the device name from port id. The device name is specified as below:
+* - PCIe address (Domain:Bus:Device.Function), for example- 0000:02:00.0
+* - SoC device name, for example- fsl-gmac0
+* - vdev dpdk name, for example- net_[pcap0|null0|tun0|tap0]
 *
 * @param port_id
 *   Port identifier of the device.
@@ -3837,7 +3809,7 @@ rte_eth_rx_burst(uint16_t port_id, uint16_t queue_id,
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->rx_pkt_burst, 0);
 
 	if (queue_id >= dev->data->nb_rx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
 		return 0;
 	}
 #endif
@@ -4070,7 +4042,7 @@ static inline int rte_eth_tx_descriptor_status(uint16_t port_id,
  *
  * If the PMD is DEV_TX_OFFLOAD_MT_LOCKFREE capable, multiple threads can
  * invoke this function concurrently on the same tx queue without SW lock.
- * @see rte_eth_dev_info_get, struct rte_eth_txconf::txq_flags
+ * @see rte_eth_dev_info_get, struct rte_eth_txconf::offloads
  *
  * @see rte_eth_tx_prepare to perform some prior checks or adjustments
  * for offloads.
@@ -4103,7 +4075,7 @@ rte_eth_tx_burst(uint16_t port_id, uint16_t queue_id,
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->tx_pkt_burst, 0);
 
 	if (queue_id >= dev->data->nb_tx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
 		return 0;
 	}
 #endif
@@ -4189,7 +4161,7 @@ rte_eth_tx_prepare(uint16_t port_id, uint16_t queue_id,
 
 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
 	if (!rte_eth_dev_is_valid_port(port_id)) {
-		RTE_PMD_DEBUG_TRACE("Invalid TX port_id=%d\n", port_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid TX port_id=%u\n", port_id);
 		rte_errno = -EINVAL;
 		return 0;
 	}
@@ -4199,7 +4171,7 @@ rte_eth_tx_prepare(uint16_t port_id, uint16_t queue_id,
 
 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
 	if (queue_id >= dev->data->nb_tx_queues) {
-		RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", queue_id);
+		RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
 		rte_errno = -EINVAL;
 		return 0;
 	}
diff --git a/lib/librte_ethdev/rte_ethdev_driver.h b/lib/librte_ethdev/rte_ethdev_driver.h
index c9c825e3..c6d9bc1a 100644
--- a/lib/librte_ethdev/rte_ethdev_driver.h
+++ b/lib/librte_ethdev/rte_ethdev_driver.h
@@ -38,7 +38,6 @@ struct rte_eth_dev *rte_eth_dev_allocated(const char *name);
  * to that slot for the driver to use.
  *
  * @param	name	Unique identifier name for each Ethernet device
- * @param	type	Device type of this Ethernet device
  * @return
  *   - Slot in the rte_dev_devices array for a new device;
  */
@@ -325,6 +324,32 @@ typedef int (*ethdev_uninit_t)(struct rte_eth_dev *ethdev);
 int __rte_experimental
 rte_eth_dev_destroy(struct rte_eth_dev *ethdev, ethdev_uninit_t ethdev_uninit);
 
+/**
+ * PMD helper function to check if keeping CRC is requested
+ *
+ * @note
+ * When CRC_STRIP offload flag is removed and default behavior switch to
+ * strip CRC, as planned, this helper function is not that useful and will be
+ * removed. In PMDs this function will be replaced with check:
+ *   if (offloads & DEV_RX_OFFLOAD_KEEP_CRC)
+ *
+ * @param rx_offloads
+ *   offload bits to be applied
+ *
+ * @return
+ *   Return positive if keeping CRC is requested,
+ *   zero if stripping CRC is requested
+ */
+static inline int
+rte_eth_dev_must_keep_crc(uint64_t rx_offloads)
+{
+	if (rx_offloads & DEV_RX_OFFLOAD_CRC_STRIP)
+		return 0;
+
+	/* no KEEP_CRC or CRC_STRIP offload flags means keep CRC */
+	return 1;
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_ethdev/rte_ethdev_pci.h b/lib/librte_ethdev/rte_ethdev_pci.h
index 2cfd3727..f652596f 100644
--- a/lib/librte_ethdev/rte_ethdev_pci.h
+++ b/lib/librte_ethdev/rte_ethdev_pci.h
@@ -53,8 +53,8 @@ rte_eth_copy_pci_info(struct rte_eth_dev *eth_dev,
 	struct rte_pci_device *pci_dev)
 {
 	if ((eth_dev == NULL) || (pci_dev == NULL)) {
-		RTE_PMD_DEBUG_TRACE("NULL pointer eth_dev=%p pci_dev=%p\n",
-				eth_dev, pci_dev);
+		RTE_ETHDEV_LOG(ERR, "NULL pointer eth_dev=%p pci_dev=%p",
+			(void *)eth_dev, (void *)pci_dev);
 		return;
 	}
 
diff --git a/lib/librte_ethdev/rte_ethdev_version.map b/lib/librte_ethdev/rte_ethdev_version.map
index 40cf42b8..38f117f0 100644
--- a/lib/librte_ethdev/rte_ethdev_version.map
+++ b/lib/librte_ethdev/rte_ethdev_version.map
@@ -213,6 +213,13 @@ DPDK_18.05 {
 
 } DPDK_18.02;
 
+DPDK_18.08 {
+	global:
+
+	rte_eth_dev_logtype;
+
+} DPDK_18.05;
+
 EXPERIMENTAL {
 	global:
 
@@ -232,6 +239,7 @@ EXPERIMENTAL {
 	rte_eth_dev_tx_offload_name;
 	rte_eth_switch_domain_alloc;
 	rte_eth_switch_domain_free;
+	rte_flow_expand_rss;
 	rte_mtr_capabilities_get;
 	rte_mtr_create;
 	rte_mtr_destroy;
diff --git a/lib/librte_ethdev/rte_flow.c b/lib/librte_ethdev/rte_flow.c
index b2afba08..cff4b520 100644
--- a/lib/librte_ethdev/rte_flow.c
+++ b/lib/librte_ethdev/rte_flow.c
@@ -84,7 +84,7 @@ static const struct rte_flow_desc_data rte_flow_desc_action[] = {
 	MK_FLOW_ACTION(FLAG, 0),
 	MK_FLOW_ACTION(QUEUE, sizeof(struct rte_flow_action_queue)),
 	MK_FLOW_ACTION(DROP, 0),
-	MK_FLOW_ACTION(COUNT, 0),
+	MK_FLOW_ACTION(COUNT, sizeof(struct rte_flow_action_count)),
 	MK_FLOW_ACTION(RSS, sizeof(struct rte_flow_action_rss)),
 	MK_FLOW_ACTION(PF, 0),
 	MK_FLOW_ACTION(VF, sizeof(struct rte_flow_action_vf)),
@@ -526,3 +526,110 @@ store:
 	}
 	return 0;
 }
+
+/**
+ * Expand RSS flows into several possible flows according to the RSS hash
+ * fields requested and the driver capabilities.
+ */
+int __rte_experimental
+rte_flow_expand_rss(struct rte_flow_expand_rss *buf, size_t size,
+		    const struct rte_flow_item *pattern, uint64_t types,
+		    const struct rte_flow_expand_node graph[],
+		    int graph_root_index)
+{
+	const int elt_n = 8;
+	const struct rte_flow_item *item;
+	const struct rte_flow_expand_node *node = &graph[graph_root_index];
+	const int *next_node;
+	const int *stack[elt_n];
+	int stack_pos = 0;
+	struct rte_flow_item flow_items[elt_n];
+	unsigned int i;
+	size_t lsize;
+	size_t user_pattern_size = 0;
+	void *addr = NULL;
+
+	lsize = offsetof(struct rte_flow_expand_rss, entry) +
+		elt_n * sizeof(buf->entry[0]);
+	if (lsize <= size) {
+		buf->entry[0].priority = 0;
+		buf->entry[0].pattern = (void *)&buf->entry[elt_n];
+		buf->entries = 0;
+		addr = buf->entry[0].pattern;
+	}
+	for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
+		const struct rte_flow_expand_node *next = NULL;
+
+		for (i = 0; node->next && node->next[i]; ++i) {
+			next = &graph[node->next[i]];
+			if (next->type == item->type)
+				break;
+		}
+		if (next)
+			node = next;
+		user_pattern_size += sizeof(*item);
+	}
+	user_pattern_size += sizeof(*item); /* Handle END item. */
+	lsize += user_pattern_size;
+	/* Copy the user pattern in the first entry of the buffer. */
+	if (lsize <= size) {
+		rte_memcpy(addr, pattern, user_pattern_size);
+		addr = (void *)(((uintptr_t)addr) + user_pattern_size);
+		buf->entries = 1;
+	}
+	/* Start expanding. */
+	memset(flow_items, 0, sizeof(flow_items));
+	user_pattern_size -= sizeof(*item);
+	next_node = node->next;
+	stack[stack_pos] = next_node;
+	node = next_node ? &graph[*next_node] : NULL;
+	while (node) {
+		flow_items[stack_pos].type = node->type;
+		if (node->rss_types & types) {
+			/*
+			 * compute the number of items to copy from the
+			 * expansion and copy it.
+			 * When the stack_pos is 0, there are 1 element in it,
+			 * plus the addition END item.
+			 */
+			int elt = stack_pos + 2;
+
+			flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END;
+			lsize += elt * sizeof(*item) + user_pattern_size;
+			if (lsize <= size) {
+				size_t n = elt * sizeof(*item);
+
+				buf->entry[buf->entries].priority =
+					stack_pos + 1;
+				buf->entry[buf->entries].pattern = addr;
+				buf->entries++;
+				rte_memcpy(addr, buf->entry[0].pattern,
+					   user_pattern_size);
+				addr = (void *)(((uintptr_t)addr) +
+						user_pattern_size);
+				rte_memcpy(addr, flow_items, n);
+				addr = (void *)(((uintptr_t)addr) + n);
+			}
+		}
+		/* Go deeper. */
+		if (node->next) {
+			next_node = node->next;
+			if (stack_pos++ == elt_n) {
+				rte_errno = E2BIG;
+				return -rte_errno;
+			}
+			stack[stack_pos] = next_node;
+		} else if (*(next_node + 1)) {
+			/* Follow up with the next possibility. */
+			++next_node;
+		} else {
+			/* Move to the next path. */
+			if (stack_pos)
+				next_node = stack[--stack_pos];
+			next_node++;
+			stack[stack_pos] = next_node;
+		}
+		node = *next_node ? &graph[*next_node] : NULL;
+	};
+	return lsize;
+}
diff --git a/lib/librte_ethdev/rte_flow_driver.h b/lib/librte_ethdev/rte_flow_driver.h
index 1c90c600..688f7230 100644
--- a/lib/librte_ethdev/rte_flow_driver.h
+++ b/lib/librte_ethdev/rte_flow_driver.h
@@ -114,6 +114,69 @@ struct rte_flow_ops {
 const struct rte_flow_ops *
 rte_flow_ops_get(uint16_t port_id, struct rte_flow_error *error);
 
+/** Helper macro to build input graph for rte_flow_expand_rss(). */
+#define RTE_FLOW_EXPAND_RSS_NEXT(...) \
+	(const int []){ \
+		__VA_ARGS__, 0, \
+	}
+
+/** Node object of input graph for rte_flow_expand_rss(). */
+struct rte_flow_expand_node {
+	const int *const next;
+	/**<
+	 * List of next node indexes. Index 0 is interpreted as a terminator.
+	 */
+	const enum rte_flow_item_type type;
+	/**< Pattern item type of current node. */
+	uint64_t rss_types;
+	/**<
+	 * RSS types bit-field associated with this node
+	 * (see ETH_RSS_* definitions).
+	 */
+};
+
+/** Object returned by rte_flow_expand_rss(). */
+struct rte_flow_expand_rss {
+	uint32_t entries;
+	/**< Number of entries @p patterns and @p priorities. */
+	struct {
+		struct rte_flow_item *pattern; /**< Expanded pattern array. */
+		uint32_t priority; /**< Priority offset for each expansion. */
+	} entry[];
+};
+
+/**
+ * Expand RSS flows into several possible flows according to the RSS hash
+ * fields requested and the driver capabilities.
+ *
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * @param[out] buf
+ *   Buffer to store the result expansion.
+ * @param[in] size
+ *   Buffer size in bytes. If 0, @p buf can be NULL.
+ * @param[in] pattern
+ *   User flow pattern.
+ * @param[in] types
+ *   RSS types to expand (see ETH_RSS_* definitions).
+ * @param[in] graph
+ *   Input graph to expand @p pattern according to @p types.
+ * @param[in] graph_root_index
+ *   Index of root node in @p graph, typically 0.
+ *
+ * @return
+ *   A positive value representing the size of @p buf in bytes regardless of
+ *   @p size on success, a negative errno value otherwise and rte_errno is
+ *   set, the following errors are defined:
+ *
+ *   -E2BIG: graph-depth @p graph is too deep.
+ */
+int __rte_experimental
+rte_flow_expand_rss(struct rte_flow_expand_rss *buf, size_t size,
+		    const struct rte_flow_item *pattern, uint64_t types,
+		    const struct rte_flow_expand_node graph[],
+		    int graph_root_index);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_ethdev/rte_tm.h b/lib/librte_ethdev/rte_tm.h
index 72554038..955f02ff 100644
--- a/lib/librte_ethdev/rte_tm.h
+++ b/lib/librte_ethdev/rte_tm.h
@@ -1569,6 +1569,10 @@ rte_tm_hierarchy_commit(uint16_t port_id,
 /**
  * Traffic manager node parent update
  *
+ * This function may be used to move a node and its children to a different
+ * parent.  Additionally, if the new parent is the same as the current parent,
+ * this function will update the priority/weight of an existing node.
+ *
  * Restriction for root node: its parent cannot be changed.
  *
  * This function can only be called after the rte_tm_hierarchy_commit()
diff --git a/lib/librte_eventdev/Makefile b/lib/librte_eventdev/Makefile
index b3e25464..47f599a6 100644
--- a/lib/librte_eventdev/Makefile
+++ b/lib/librte_eventdev/Makefile
@@ -8,14 +8,19 @@ include $(RTE_SDK)/mk/rte.vars.mk
 LIB = librte_eventdev.a
 
 # library version
-LIBABIVER := 4
+LIBABIVER := 5
 
 # build flags
 CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
+ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
+CFLAGS += -DLINUX
+else
+CFLAGS += -DBSD
+endif
 LDLIBS += -lrte_eal -lrte_ring -lrte_ethdev -lrte_hash -lrte_mempool -lrte_timer
-LDLIBS += -lrte_mbuf -lrte_cryptodev
+LDLIBS += -lrte_mbuf -lrte_cryptodev -lpthread
 
 # library source files
 SRCS-y += rte_eventdev.c
diff --git a/lib/librte_eventdev/meson.build b/lib/librte_eventdev/meson.build
index bd138bd5..3cbaf298 100644
--- a/lib/librte_eventdev/meson.build
+++ b/lib/librte_eventdev/meson.build
@@ -1,8 +1,15 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-version = 4
+version = 5
 allow_experimental_apis = true
+
+if host_machine.system() == 'linux'
+	cflags += '-DLINUX'
+else
+	cflags += '-DBSD'
+endif
+
 sources = files('rte_eventdev.c',
 		'rte_event_ring.c',
 		'rte_event_eth_rx_adapter.c',
diff --git a/lib/librte_eventdev/rte_event_crypto_adapter.c b/lib/librte_eventdev/rte_event_crypto_adapter.c
index ba63a87b..11b28ca9 100644
--- a/lib/librte_eventdev/rte_event_crypto_adapter.c
+++ b/lib/librte_eventdev/rte_event_crypto_adapter.c
@@ -342,7 +342,7 @@ eca_enq_to_cryptodev(struct rte_event_crypto_adapter *adapter,
 		if (crypto_op == NULL)
 			continue;
 		if (crypto_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
-			m_data = rte_cryptodev_sym_session_get_private_data(
+			m_data = rte_cryptodev_sym_session_get_user_data(
 					crypto_op->sym->session);
 			if (m_data == NULL) {
 				rte_pktmbuf_free(crypto_op->sym->m_src);
@@ -512,7 +512,7 @@ eca_ops_enqueue_burst(struct rte_event_crypto_adapter *adapter,
 	for (i = 0; i < num; i++) {
 		struct rte_event *ev = &events[nb_ev++];
 		if (ops[i]->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
-			m_data = rte_cryptodev_sym_session_get_private_data(
+			m_data = rte_cryptodev_sym_session_get_user_data(
 					ops[i]->sym->session);
 		} else if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS &&
 				ops[i]->private_data_offset) {
diff --git a/lib/librte_eventdev/rte_event_eth_rx_adapter.c b/lib/librte_eventdev/rte_event_eth_rx_adapter.c
index 6f705095..f5e5a0b5 100644
--- a/lib/librte_eventdev/rte_event_eth_rx_adapter.c
+++ b/lib/librte_eventdev/rte_event_eth_rx_adapter.c
@@ -2,6 +2,11 @@
  * Copyright(c) 2017 Intel Corporation.
  * All rights reserved.
  */
+#if defined(LINUX)
+#include <sys/epoll.h>
+#endif
+#include <unistd.h>
+
 #include <rte_cycles.h>
 #include <rte_common.h>
 #include <rte_dev.h>
@@ -11,6 +16,7 @@
 #include <rte_malloc.h>
 #include <rte_service_component.h>
 #include <rte_thash.h>
+#include <rte_interrupts.h>
 
 #include "rte_eventdev.h"
 #include "rte_eventdev_pmd.h"
@@ -24,6 +30,22 @@
 #define ETH_RX_ADAPTER_MEM_NAME_LEN	32
 
 #define RSS_KEY_SIZE	40
+/* value written to intr thread pipe to signal thread exit */
+#define ETH_BRIDGE_INTR_THREAD_EXIT	1
+/* Sentinel value to detect initialized file handle */
+#define INIT_FD		-1
+
+/*
+ * Used to store port and queue ID of interrupting Rx queue
+ */
+union queue_data {
+	RTE_STD_C11
+	void *ptr;
+	struct {
+		uint16_t port;
+		uint16_t queue;
+	};
+};
 
 /*
  * There is an instance of this struct per polled Rx queue added to the
@@ -75,6 +97,30 @@ struct rte_event_eth_rx_adapter {
 	uint16_t enq_block_count;
 	/* Block start ts */
 	uint64_t rx_enq_block_start_ts;
+	/* epoll fd used to wait for Rx interrupts */
+	int epd;
+	/* Num of interrupt driven interrupt queues */
+	uint32_t num_rx_intr;
+	/* Used to send <dev id, queue id> of interrupting Rx queues from
+	 * the interrupt thread to the Rx thread
+	 */
+	struct rte_ring *intr_ring;
+	/* Rx Queue data (dev id, queue id) for the last non-empty
+	 * queue polled
+	 */
+	union queue_data qd;
+	/* queue_data is valid */
+	int qd_valid;
+	/* Interrupt ring lock, synchronizes Rx thread
+	 * and interrupt thread
+	 */
+	rte_spinlock_t intr_ring_lock;
+	/* event array passed to rte_poll_wait */
+	struct rte_epoll_event *epoll_events;
+	/* Count of interrupt vectors in use */
+	uint32_t num_intr_vec;
+	/* Thread blocked on Rx interrupts */
+	pthread_t rx_intr_thread;
 	/* Configuration callback for rte_service configuration */
 	rte_event_eth_rx_adapter_conf_cb conf_cb;
 	/* Configuration callback argument */
@@ -91,12 +137,20 @@ struct rte_event_eth_rx_adapter {
 	int socket_id;
 	/* Per adapter EAL service */
 	uint32_t service_id;
+	/* Adapter started flag */
+	uint8_t rxa_started;
+	/* Adapter ID */
+	uint8_t id;
 } __rte_cache_aligned;
 
 /* Per eth device */
 struct eth_device_info {
 	struct rte_eth_dev *dev;
 	struct eth_rx_queue_info *rx_queue;
+	/* Rx callback */
+	rte_event_eth_rx_adapter_cb_fn cb_fn;
+	/* Rx callback argument */
+	void *cb_arg;
 	/* Set if ethdev->eventdev packet transfer uses a
 	 * hardware mechanism
 	 */
@@ -107,15 +161,42 @@ struct eth_device_info {
 	 * rx_adapter_stop callback needs to be invoked
 	 */
 	uint8_t dev_rx_started;
-	/* If nb_dev_queues > 0, the start callback will
+	/* Number of queues added for this device */
+	uint16_t nb_dev_queues;
+	/* Number of poll based queues
+	 * If nb_rx_poll > 0, the start callback will
 	 * be invoked if not already invoked
 	 */
-	uint16_t nb_dev_queues;
+	uint16_t nb_rx_poll;
+	/* Number of interrupt based queues
+	 * If nb_rx_intr > 0, the start callback will
+	 * be invoked if not already invoked.
+	 */
+	uint16_t nb_rx_intr;
+	/* Number of queues that use the shared interrupt */
+	uint16_t nb_shared_intr;
+	/* sum(wrr(q)) for all queues within the device
+	 * useful when deleting all device queues
+	 */
+	uint32_t wrr_len;
+	/* Intr based queue index to start polling from, this is used
+	 * if the number of shared interrupts is non-zero
+	 */
+	uint16_t next_q_idx;
+	/* Intr based queue indices */
+	uint16_t *intr_queue;
+	/* device generates per Rx queue interrupt for queue index
+	 * for queue indices < RTE_MAX_RXTX_INTR_VEC_ID - 1
+	 */
+	int multi_intr_cap;
+	/* shared interrupt enabled */
+	int shared_intr_enabled;
 };
 
 /* Per Rx queue */
 struct eth_rx_queue_info {
 	int queue_enabled;	/* True if added */
+	int intr_enabled;
 	uint16_t wt;		/* Polling weight */
 	uint8_t event_queue_id;	/* Event queue to enqueue packets to */
 	uint8_t sched_type;	/* Sched type for events */
@@ -127,30 +208,30 @@ struct eth_rx_queue_info {
 static struct rte_event_eth_rx_adapter **event_eth_rx_adapter;
 
 static inline int
-valid_id(uint8_t id)
+rxa_validate_id(uint8_t id)
 {
 	return id < RTE_EVENT_ETH_RX_ADAPTER_MAX_INSTANCE;
 }
 
 #define RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) do { \
-	if (!valid_id(id)) { \
+	if (!rxa_validate_id(id)) { \
 		RTE_EDEV_LOG_ERR("Invalid eth Rx adapter id = %d\n", id); \
 		return retval; \
 	} \
 } while (0)
 
 static inline int
-sw_rx_adapter_queue_count(struct rte_event_eth_rx_adapter *rx_adapter)
+rxa_sw_adapter_queue_count(struct rte_event_eth_rx_adapter *rx_adapter)
 {
-	return rx_adapter->num_rx_polled;
+	return rx_adapter->num_rx_polled + rx_adapter->num_rx_intr;
 }
 
 /* Greatest common divisor */
-static uint16_t gcd_u16(uint16_t a, uint16_t b)
+static uint16_t rxa_gcd_u16(uint16_t a, uint16_t b)
 {
 	uint16_t r = a % b;
 
-	return r ? gcd_u16(b, r) : b;
+	return r ? rxa_gcd_u16(b, r) : b;
 }
 
 /* Returns the next queue in the polling sequence
@@ -158,7 +239,7 @@ static uint16_t gcd_u16(uint16_t a, uint16_t b)
  * http://kb.linuxvirtualserver.org/wiki/Weighted_Round-Robin_Scheduling
  */
 static int
-wrr_next(struct rte_event_eth_rx_adapter *rx_adapter,
+rxa_wrr_next(struct rte_event_eth_rx_adapter *rx_adapter,
 	 unsigned int n, int *cw,
 	 struct eth_rx_poll_entry *eth_rx_poll, uint16_t max_wt,
 	 uint16_t gcd, int prev)
@@ -186,13 +267,298 @@ wrr_next(struct rte_event_eth_rx_adapter *rx_adapter,
 	}
 }
 
-/* Precalculate WRR polling sequence for all queues in rx_adapter */
+static inline int
+rxa_shared_intr(struct eth_device_info *dev_info,
+	int rx_queue_id)
+{
+	int multi_intr_cap;
+
+	if (dev_info->dev->intr_handle == NULL)
+		return 0;
+
+	multi_intr_cap = rte_intr_cap_multiple(dev_info->dev->intr_handle);
+	return !multi_intr_cap ||
+		rx_queue_id >= RTE_MAX_RXTX_INTR_VEC_ID - 1;
+}
+
+static inline int
+rxa_intr_queue(struct eth_device_info *dev_info,
+	int rx_queue_id)
+{
+	struct eth_rx_queue_info *queue_info;
+
+	queue_info = &dev_info->rx_queue[rx_queue_id];
+	return dev_info->rx_queue &&
+		!dev_info->internal_event_port &&
+		queue_info->queue_enabled && queue_info->wt == 0;
+}
+
+static inline int
+rxa_polled_queue(struct eth_device_info *dev_info,
+	int rx_queue_id)
+{
+	struct eth_rx_queue_info *queue_info;
+
+	queue_info = &dev_info->rx_queue[rx_queue_id];
+	return !dev_info->internal_event_port &&
+		dev_info->rx_queue &&
+		queue_info->queue_enabled && queue_info->wt != 0;
+}
+
+/* Calculate change in number of vectors after Rx queue ID is add/deleted */
 static int
-eth_poll_wrr_calc(struct rte_event_eth_rx_adapter *rx_adapter)
+rxa_nb_intr_vect(struct eth_device_info *dev_info, int rx_queue_id, int add)
+{
+	uint16_t i;
+	int n, s;
+	uint16_t nbq;
+
+	nbq = dev_info->dev->data->nb_rx_queues;
+	n = 0; /* non shared count */
+	s = 0; /* shared count */
+
+	if (rx_queue_id == -1) {
+		for (i = 0; i < nbq; i++) {
+			if (!rxa_shared_intr(dev_info, i))
+				n += add ? !rxa_intr_queue(dev_info, i) :
+					rxa_intr_queue(dev_info, i);
+			else
+				s += add ? !rxa_intr_queue(dev_info, i) :
+					rxa_intr_queue(dev_info, i);
+		}
+
+		if (s > 0) {
+			if ((add && dev_info->nb_shared_intr == 0) ||
+				(!add && dev_info->nb_shared_intr))
+				n += 1;
+		}
+	} else {
+		if (!rxa_shared_intr(dev_info, rx_queue_id))
+			n = add ? !rxa_intr_queue(dev_info, rx_queue_id) :
+				rxa_intr_queue(dev_info, rx_queue_id);
+		else
+			n = add ? !dev_info->nb_shared_intr :
+				dev_info->nb_shared_intr == 1;
+	}
+
+	return add ? n : -n;
+}
+
+/* Calculate nb_rx_intr after deleting interrupt mode rx queues
+ */
+static void
+rxa_calc_nb_post_intr_del(struct rte_event_eth_rx_adapter *rx_adapter,
+			struct eth_device_info *dev_info,
+			int rx_queue_id,
+			uint32_t *nb_rx_intr)
+{
+	uint32_t intr_diff;
+
+	if (rx_queue_id == -1)
+		intr_diff = dev_info->nb_rx_intr;
+	else
+		intr_diff = rxa_intr_queue(dev_info, rx_queue_id);
+
+	*nb_rx_intr = rx_adapter->num_rx_intr - intr_diff;
+}
+
+/* Calculate nb_rx_* after adding interrupt mode rx queues, newly added
+ * interrupt queues could currently be poll mode Rx queues
+ */
+static void
+rxa_calc_nb_post_add_intr(struct rte_event_eth_rx_adapter *rx_adapter,
+			struct eth_device_info *dev_info,
+			int rx_queue_id,
+			uint32_t *nb_rx_poll,
+			uint32_t *nb_rx_intr,
+			uint32_t *nb_wrr)
+{
+	uint32_t intr_diff;
+	uint32_t poll_diff;
+	uint32_t wrr_len_diff;
+
+	if (rx_queue_id == -1) {
+		intr_diff = dev_info->dev->data->nb_rx_queues -
+						dev_info->nb_rx_intr;
+		poll_diff = dev_info->nb_rx_poll;
+		wrr_len_diff = dev_info->wrr_len;
+	} else {
+		intr_diff = !rxa_intr_queue(dev_info, rx_queue_id);
+		poll_diff = rxa_polled_queue(dev_info, rx_queue_id);
+		wrr_len_diff = poll_diff ? dev_info->rx_queue[rx_queue_id].wt :
+					0;
+	}
+
+	*nb_rx_intr = rx_adapter->num_rx_intr + intr_diff;
+	*nb_rx_poll = rx_adapter->num_rx_polled - poll_diff;
+	*nb_wrr = rx_adapter->wrr_len - wrr_len_diff;
+}
+
+/* Calculate size of the eth_rx_poll and wrr_sched arrays
+ * after deleting poll mode rx queues
+ */
+static void
+rxa_calc_nb_post_poll_del(struct rte_event_eth_rx_adapter *rx_adapter,
+			struct eth_device_info *dev_info,
+			int rx_queue_id,
+			uint32_t *nb_rx_poll,
+			uint32_t *nb_wrr)
+{
+	uint32_t poll_diff;
+	uint32_t wrr_len_diff;
+
+	if (rx_queue_id == -1) {
+		poll_diff = dev_info->nb_rx_poll;
+		wrr_len_diff = dev_info->wrr_len;
+	} else {
+		poll_diff = rxa_polled_queue(dev_info, rx_queue_id);
+		wrr_len_diff = poll_diff ? dev_info->rx_queue[rx_queue_id].wt :
+					0;
+	}
+
+	*nb_rx_poll = rx_adapter->num_rx_polled - poll_diff;
+	*nb_wrr = rx_adapter->wrr_len - wrr_len_diff;
+}
+
+/* Calculate nb_rx_* after adding poll mode rx queues
+ */
+static void
+rxa_calc_nb_post_add_poll(struct rte_event_eth_rx_adapter *rx_adapter,
+			struct eth_device_info *dev_info,
+			int rx_queue_id,
+			uint16_t wt,
+			uint32_t *nb_rx_poll,
+			uint32_t *nb_rx_intr,
+			uint32_t *nb_wrr)
+{
+	uint32_t intr_diff;
+	uint32_t poll_diff;
+	uint32_t wrr_len_diff;
+
+	if (rx_queue_id == -1) {
+		intr_diff = dev_info->nb_rx_intr;
+		poll_diff = dev_info->dev->data->nb_rx_queues -
+						dev_info->nb_rx_poll;
+		wrr_len_diff = wt*dev_info->dev->data->nb_rx_queues
+				- dev_info->wrr_len;
+	} else {
+		intr_diff = rxa_intr_queue(dev_info, rx_queue_id);
+		poll_diff = !rxa_polled_queue(dev_info, rx_queue_id);
+		wrr_len_diff = rxa_polled_queue(dev_info, rx_queue_id) ?
+				wt - dev_info->rx_queue[rx_queue_id].wt :
+				wt;
+	}
+
+	*nb_rx_poll = rx_adapter->num_rx_polled + poll_diff;
+	*nb_rx_intr = rx_adapter->num_rx_intr - intr_diff;
+	*nb_wrr = rx_adapter->wrr_len + wrr_len_diff;
+}
+
+/* Calculate nb_rx_* after adding rx_queue_id */
+static void
+rxa_calc_nb_post_add(struct rte_event_eth_rx_adapter *rx_adapter,
+		struct eth_device_info *dev_info,
+		int rx_queue_id,
+		uint16_t wt,
+		uint32_t *nb_rx_poll,
+		uint32_t *nb_rx_intr,
+		uint32_t *nb_wrr)
+{
+	if (wt != 0)
+		rxa_calc_nb_post_add_poll(rx_adapter, dev_info, rx_queue_id,
+					wt, nb_rx_poll, nb_rx_intr, nb_wrr);
+	else
+		rxa_calc_nb_post_add_intr(rx_adapter, dev_info, rx_queue_id,
+					nb_rx_poll, nb_rx_intr, nb_wrr);
+}
+
+/* Calculate nb_rx_* after deleting rx_queue_id */
+static void
+rxa_calc_nb_post_del(struct rte_event_eth_rx_adapter *rx_adapter,
+		struct eth_device_info *dev_info,
+		int rx_queue_id,
+		uint32_t *nb_rx_poll,
+		uint32_t *nb_rx_intr,
+		uint32_t *nb_wrr)
+{
+	rxa_calc_nb_post_poll_del(rx_adapter, dev_info, rx_queue_id, nb_rx_poll,
+				nb_wrr);
+	rxa_calc_nb_post_intr_del(rx_adapter, dev_info, rx_queue_id,
+				nb_rx_intr);
+}
+
+/*
+ * Allocate the rx_poll array
+ */
+static struct eth_rx_poll_entry *
+rxa_alloc_poll(struct rte_event_eth_rx_adapter *rx_adapter,
+	uint32_t num_rx_polled)
+{
+	size_t len;
+
+	len  = RTE_ALIGN(num_rx_polled * sizeof(*rx_adapter->eth_rx_poll),
+							RTE_CACHE_LINE_SIZE);
+	return  rte_zmalloc_socket(rx_adapter->mem_name,
+				len,
+				RTE_CACHE_LINE_SIZE,
+				rx_adapter->socket_id);
+}
+
+/*
+ * Allocate the WRR array
+ */
+static uint32_t *
+rxa_alloc_wrr(struct rte_event_eth_rx_adapter *rx_adapter, int nb_wrr)
+{
+	size_t len;
+
+	len = RTE_ALIGN(nb_wrr * sizeof(*rx_adapter->wrr_sched),
+			RTE_CACHE_LINE_SIZE);
+	return  rte_zmalloc_socket(rx_adapter->mem_name,
+				len,
+				RTE_CACHE_LINE_SIZE,
+				rx_adapter->socket_id);
+}
+
+static int
+rxa_alloc_poll_arrays(struct rte_event_eth_rx_adapter *rx_adapter,
+		uint32_t nb_poll,
+		uint32_t nb_wrr,
+		struct eth_rx_poll_entry **rx_poll,
+		uint32_t **wrr_sched)
+{
+
+	if (nb_poll == 0) {
+		*rx_poll = NULL;
+		*wrr_sched = NULL;
+		return 0;
+	}
+
+	*rx_poll = rxa_alloc_poll(rx_adapter, nb_poll);
+	if (*rx_poll == NULL) {
+		*wrr_sched = NULL;
+		return -ENOMEM;
+	}
+
+	*wrr_sched = rxa_alloc_wrr(rx_adapter, nb_wrr);
+	if (*wrr_sched == NULL) {
+		rte_free(*rx_poll);
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+/* Precalculate WRR polling sequence for all queues in rx_adapter */
+static void
+rxa_calc_wrr_sequence(struct rte_event_eth_rx_adapter *rx_adapter,
+		struct eth_rx_poll_entry *rx_poll,
+		uint32_t *rx_wrr)
 {
 	uint16_t d;
 	uint16_t q;
 	unsigned int i;
+	int prev = -1;
+	int cw = -1;
 
 	/* Initialize variables for calculation of wrr schedule */
 	uint16_t max_wrr_pos = 0;
@@ -200,79 +566,52 @@ eth_poll_wrr_calc(struct rte_event_eth_rx_adapter *rx_adapter)
 	uint16_t max_wt = 0;
 	uint16_t gcd = 0;
 
-	struct eth_rx_poll_entry *rx_poll = NULL;
-	uint32_t *rx_wrr = NULL;
+	if (rx_poll == NULL)
+		return;
 
-	if (rx_adapter->num_rx_polled) {
-		size_t len = RTE_ALIGN(rx_adapter->num_rx_polled *
-				sizeof(*rx_adapter->eth_rx_poll),
-				RTE_CACHE_LINE_SIZE);
-		rx_poll = rte_zmalloc_socket(rx_adapter->mem_name,
-					     len,
-					     RTE_CACHE_LINE_SIZE,
-					     rx_adapter->socket_id);
-		if (rx_poll == NULL)
-			return -ENOMEM;
+	/* Generate array of all queues to poll, the size of this
+	 * array is poll_q
+	 */
+	RTE_ETH_FOREACH_DEV(d) {
+		uint16_t nb_rx_queues;
+		struct eth_device_info *dev_info =
+				&rx_adapter->eth_devices[d];
+		nb_rx_queues = dev_info->dev->data->nb_rx_queues;
+		if (dev_info->rx_queue == NULL)
+			continue;
+		if (dev_info->internal_event_port)
+			continue;
+		dev_info->wrr_len = 0;
+		for (q = 0; q < nb_rx_queues; q++) {
+			struct eth_rx_queue_info *queue_info =
+				&dev_info->rx_queue[q];
+			uint16_t wt;
 
-		/* Generate array of all queues to poll, the size of this
-		 * array is poll_q
-		 */
-		RTE_ETH_FOREACH_DEV(d) {
-			uint16_t nb_rx_queues;
-			struct eth_device_info *dev_info =
-					&rx_adapter->eth_devices[d];
-			nb_rx_queues = dev_info->dev->data->nb_rx_queues;
-			if (dev_info->rx_queue == NULL)
+			if (!rxa_polled_queue(dev_info, q))
 				continue;
-			for (q = 0; q < nb_rx_queues; q++) {
-				struct eth_rx_queue_info *queue_info =
-					&dev_info->rx_queue[q];
-				if (queue_info->queue_enabled == 0)
-					continue;
-
-				uint16_t wt = queue_info->wt;
-				rx_poll[poll_q].eth_dev_id = d;
-				rx_poll[poll_q].eth_rx_qid = q;
-				max_wrr_pos += wt;
-				max_wt = RTE_MAX(max_wt, wt);
-				gcd = (gcd) ? gcd_u16(gcd, wt) : wt;
-				poll_q++;
-			}
-		}
-
-		len = RTE_ALIGN(max_wrr_pos * sizeof(*rx_wrr),
-				RTE_CACHE_LINE_SIZE);
-		rx_wrr = rte_zmalloc_socket(rx_adapter->mem_name,
-					    len,
-					    RTE_CACHE_LINE_SIZE,
-					    rx_adapter->socket_id);
-		if (rx_wrr == NULL) {
-			rte_free(rx_poll);
-			return -ENOMEM;
-		}
-
-		/* Generate polling sequence based on weights */
-		int prev = -1;
-		int cw = -1;
-		for (i = 0; i < max_wrr_pos; i++) {
-			rx_wrr[i] = wrr_next(rx_adapter, poll_q, &cw,
-					     rx_poll, max_wt, gcd, prev);
-			prev = rx_wrr[i];
+			wt = queue_info->wt;
+			rx_poll[poll_q].eth_dev_id = d;
+			rx_poll[poll_q].eth_rx_qid = q;
+			max_wrr_pos += wt;
+			dev_info->wrr_len += wt;
+			max_wt = RTE_MAX(max_wt, wt);
+			gcd = (gcd) ? rxa_gcd_u16(gcd, wt) : wt;
+			poll_q++;
 		}
 	}
 
-	rte_free(rx_adapter->eth_rx_poll);
-	rte_free(rx_adapter->wrr_sched);
-
-	rx_adapter->eth_rx_poll = rx_poll;
-	rx_adapter->wrr_sched = rx_wrr;
-	rx_adapter->wrr_len = max_wrr_pos;
-
-	return 0;
+	/* Generate polling sequence based on weights */
+	prev = -1;
+	cw = -1;
+	for (i = 0; i < max_wrr_pos; i++) {
+		rx_wrr[i] = rxa_wrr_next(rx_adapter, poll_q, &cw,
+				     rx_poll, max_wt, gcd, prev);
+		prev = rx_wrr[i];
+	}
 }
 
 static inline void
-mtoip(struct rte_mbuf *m, struct ipv4_hdr **ipv4_hdr,
+rxa_mtoip(struct rte_mbuf *m, struct ipv4_hdr **ipv4_hdr,
 	struct ipv6_hdr **ipv6_hdr)
 {
 	struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
@@ -311,7 +650,7 @@ mtoip(struct rte_mbuf *m, struct ipv4_hdr **ipv4_hdr,
 
 /* Calculate RSS hash for IPv4/6 */
 static inline uint32_t
-do_softrss(struct rte_mbuf *m, const uint8_t *rss_key_be)
+rxa_do_softrss(struct rte_mbuf *m, const uint8_t *rss_key_be)
 {
 	uint32_t input_len;
 	void *tuple;
@@ -320,7 +659,7 @@ do_softrss(struct rte_mbuf *m, const uint8_t *rss_key_be)
 	struct ipv4_hdr *ipv4_hdr;
 	struct ipv6_hdr *ipv6_hdr;
 
-	mtoip(m, &ipv4_hdr, &ipv6_hdr);
+	rxa_mtoip(m, &ipv4_hdr, &ipv6_hdr);
 
 	if (ipv4_hdr) {
 		ipv4_tuple.src_addr = rte_be_to_cpu_32(ipv4_hdr->src_addr);
@@ -339,13 +678,13 @@ do_softrss(struct rte_mbuf *m, const uint8_t *rss_key_be)
 }
 
 static inline int
-rx_enq_blocked(struct rte_event_eth_rx_adapter *rx_adapter)
+rxa_enq_blocked(struct rte_event_eth_rx_adapter *rx_adapter)
 {
 	return !!rx_adapter->enq_block_count;
 }
 
 static inline void
-rx_enq_block_start_ts(struct rte_event_eth_rx_adapter *rx_adapter)
+rxa_enq_block_start_ts(struct rte_event_eth_rx_adapter *rx_adapter)
 {
 	if (rx_adapter->rx_enq_block_start_ts)
 		return;
@@ -358,13 +697,13 @@ rx_enq_block_start_ts(struct rte_event_eth_rx_adapter *rx_adapter)
 }
 
 static inline void
-rx_enq_block_end_ts(struct rte_event_eth_rx_adapter *rx_adapter,
+rxa_enq_block_end_ts(struct rte_event_eth_rx_adapter *rx_adapter,
 		    struct rte_event_eth_rx_adapter_stats *stats)
 {
 	if (unlikely(!stats->rx_enq_start_ts))
 		stats->rx_enq_start_ts = rte_get_tsc_cycles();
 
-	if (likely(!rx_enq_blocked(rx_adapter)))
+	if (likely(!rxa_enq_blocked(rx_adapter)))
 		return;
 
 	rx_adapter->enq_block_count = 0;
@@ -380,8 +719,8 @@ rx_enq_block_end_ts(struct rte_event_eth_rx_adapter *rx_adapter,
  * this function
  */
 static inline void
-buf_event_enqueue(struct rte_event_eth_rx_adapter *rx_adapter,
-		  struct rte_event *ev)
+rxa_buffer_event(struct rte_event_eth_rx_adapter *rx_adapter,
+		struct rte_event *ev)
 {
 	struct rte_eth_event_enqueue_buffer *buf =
 	    &rx_adapter->event_enqueue_buffer;
@@ -390,7 +729,7 @@ buf_event_enqueue(struct rte_event_eth_rx_adapter *rx_adapter,
 
 /* Enqueue buffered events to event device */
 static inline uint16_t
-flush_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter)
+rxa_flush_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter)
 {
 	struct rte_eth_event_enqueue_buffer *buf =
 	    &rx_adapter->event_enqueue_buffer;
@@ -407,8 +746,8 @@ flush_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter)
 		stats->rx_enq_retry++;
 	}
 
-	n ? rx_enq_block_end_ts(rx_adapter, stats) :
-		rx_enq_block_start_ts(rx_adapter);
+	n ? rxa_enq_block_end_ts(rx_adapter, stats) :
+		rxa_enq_block_start_ts(rx_adapter);
 
 	buf->count -= n;
 	stats->rx_enq_count += n;
@@ -417,18 +756,19 @@ flush_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter)
 }
 
 static inline void
-fill_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter,
-	uint8_t dev_id,
-	uint16_t rx_queue_id,
-	struct rte_mbuf **mbufs,
-	uint16_t num)
+rxa_buffer_mbufs(struct rte_event_eth_rx_adapter *rx_adapter,
+		uint16_t eth_dev_id,
+		uint16_t rx_queue_id,
+		struct rte_mbuf **mbufs,
+		uint16_t num)
 {
 	uint32_t i;
-	struct eth_device_info *eth_device_info =
-					&rx_adapter->eth_devices[dev_id];
+	struct eth_device_info *dev_info =
+					&rx_adapter->eth_devices[eth_dev_id];
 	struct eth_rx_queue_info *eth_rx_queue_info =
-					&eth_device_info->rx_queue[rx_queue_id];
-
+					&dev_info->rx_queue[rx_queue_id];
+	struct rte_eth_event_enqueue_buffer *buf =
+					&rx_adapter->event_enqueue_buffer;
 	int32_t qid = eth_rx_queue_info->event_queue_id;
 	uint8_t sched_type = eth_rx_queue_info->sched_type;
 	uint8_t priority = eth_rx_queue_info->priority;
@@ -439,6 +779,8 @@ fill_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter,
 	uint32_t rss;
 	int do_rss;
 	uint64_t ts;
+	struct rte_mbuf *cb_mbufs[BATCH_SIZE];
+	uint16_t nb_cb;
 
 	/* 0xffff ffff if PKT_RX_RSS_HASH is set, otherwise 0 */
 	rss_mask = ~(((m->ol_flags & PKT_RX_RSS_HASH) != 0) - 1);
@@ -454,12 +796,26 @@ fill_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter,
 		}
 	}
 
+
+	nb_cb = dev_info->cb_fn ? dev_info->cb_fn(eth_dev_id, rx_queue_id,
+						ETH_EVENT_BUFFER_SIZE,
+						buf->count, mbufs,
+						num,
+						dev_info->cb_arg,
+						cb_mbufs) :
+						num;
+	if (nb_cb < num) {
+		mbufs = cb_mbufs;
+		num = nb_cb;
+	}
+
 	for (i = 0; i < num; i++) {
 		m = mbufs[i];
 		struct rte_event *ev = &events[i];
 
 		rss = do_rss ?
-			do_softrss(m, rx_adapter->rss_key_be) : m->hash.rss;
+			rxa_do_softrss(m, rx_adapter->rss_key_be) :
+			m->hash.rss;
 		flow_id =
 		    eth_rx_queue_info->flow_id &
 				eth_rx_queue_info->flow_id_mask;
@@ -473,8 +829,275 @@ fill_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter,
 		ev->priority = priority;
 		ev->mbuf = m;
 
-		buf_event_enqueue(rx_adapter, ev);
+		rxa_buffer_event(rx_adapter, ev);
+	}
+}
+
+/* Enqueue packets from  <port, q>  to event buffer */
+static inline uint32_t
+rxa_eth_rx(struct rte_event_eth_rx_adapter *rx_adapter,
+	uint16_t port_id,
+	uint16_t queue_id,
+	uint32_t rx_count,
+	uint32_t max_rx,
+	int *rxq_empty)
+{
+	struct rte_mbuf *mbufs[BATCH_SIZE];
+	struct rte_eth_event_enqueue_buffer *buf =
+					&rx_adapter->event_enqueue_buffer;
+	struct rte_event_eth_rx_adapter_stats *stats =
+					&rx_adapter->stats;
+	uint16_t n;
+	uint32_t nb_rx = 0;
+
+	if (rxq_empty)
+		*rxq_empty = 0;
+	/* Don't do a batch dequeue from the rx queue if there isn't
+	 * enough space in the enqueue buffer.
+	 */
+	while (BATCH_SIZE <= (RTE_DIM(buf->events) - buf->count)) {
+		if (buf->count >= BATCH_SIZE)
+			rxa_flush_event_buffer(rx_adapter);
+
+		stats->rx_poll_count++;
+		n = rte_eth_rx_burst(port_id, queue_id, mbufs, BATCH_SIZE);
+		if (unlikely(!n)) {
+			if (rxq_empty)
+				*rxq_empty = 1;
+			break;
+		}
+		rxa_buffer_mbufs(rx_adapter, port_id, queue_id, mbufs, n);
+		nb_rx += n;
+		if (rx_count + nb_rx > max_rx)
+			break;
 	}
+
+	if (buf->count >= BATCH_SIZE)
+		rxa_flush_event_buffer(rx_adapter);
+
+	return nb_rx;
+}
+
+static inline void
+rxa_intr_ring_enqueue(struct rte_event_eth_rx_adapter *rx_adapter,
+		void *data)
+{
+	uint16_t port_id;
+	uint16_t queue;
+	int err;
+	union queue_data qd;
+	struct eth_device_info *dev_info;
+	struct eth_rx_queue_info *queue_info;
+	int *intr_enabled;
+
+	qd.ptr = data;
+	port_id = qd.port;
+	queue = qd.queue;
+
+	dev_info = &rx_adapter->eth_devices[port_id];
+	queue_info = &dev_info->rx_queue[queue];
+	rte_spinlock_lock(&rx_adapter->intr_ring_lock);
+	if (rxa_shared_intr(dev_info, queue))
+		intr_enabled = &dev_info->shared_intr_enabled;
+	else
+		intr_enabled = &queue_info->intr_enabled;
+
+	if (*intr_enabled) {
+		*intr_enabled = 0;
+		err = rte_ring_enqueue(rx_adapter->intr_ring, data);
+		/* Entry should always be available.
+		 * The ring size equals the maximum number of interrupt
+		 * vectors supported (an interrupt vector is shared in
+		 * case of shared interrupts)
+		 */
+		if (err)
+			RTE_EDEV_LOG_ERR("Failed to enqueue interrupt"
+				" to ring: %s", strerror(err));
+		else
+			rte_eth_dev_rx_intr_disable(port_id, queue);
+	}
+	rte_spinlock_unlock(&rx_adapter->intr_ring_lock);
+}
+
+static int
+rxa_intr_ring_check_avail(struct rte_event_eth_rx_adapter *rx_adapter,
+			uint32_t num_intr_vec)
+{
+	if (rx_adapter->num_intr_vec + num_intr_vec >
+				RTE_EVENT_ETH_INTR_RING_SIZE) {
+		RTE_EDEV_LOG_ERR("Exceeded intr ring slots current"
+		" %d needed %d limit %d", rx_adapter->num_intr_vec,
+		num_intr_vec, RTE_EVENT_ETH_INTR_RING_SIZE);
+		return -ENOSPC;
+	}
+
+	return 0;
+}
+
+/* Delete entries for (dev, queue) from the interrupt ring */
+static void
+rxa_intr_ring_del_entries(struct rte_event_eth_rx_adapter *rx_adapter,
+			struct eth_device_info *dev_info,
+			uint16_t rx_queue_id)
+{
+	int i, n;
+	union queue_data qd;
+
+	rte_spinlock_lock(&rx_adapter->intr_ring_lock);
+
+	n = rte_ring_count(rx_adapter->intr_ring);
+	for (i = 0; i < n; i++) {
+		rte_ring_dequeue(rx_adapter->intr_ring, &qd.ptr);
+		if (!rxa_shared_intr(dev_info, rx_queue_id)) {
+			if (qd.port == dev_info->dev->data->port_id &&
+				qd.queue == rx_queue_id)
+				continue;
+		} else {
+			if (qd.port == dev_info->dev->data->port_id)
+				continue;
+		}
+		rte_ring_enqueue(rx_adapter->intr_ring, qd.ptr);
+	}
+
+	rte_spinlock_unlock(&rx_adapter->intr_ring_lock);
+}
+
+/* pthread callback handling interrupt mode receive queues
+ * After receiving an Rx interrupt, it enqueues the port id and queue id of the
+ * interrupting queue to the adapter's ring buffer for interrupt events.
+ * These events are picked up by rxa_intr_ring_dequeue() which is invoked from
+ * the adapter service function.
+ */
+static void *
+rxa_intr_thread(void *arg)
+{
+	struct rte_event_eth_rx_adapter *rx_adapter = arg;
+	struct rte_epoll_event *epoll_events = rx_adapter->epoll_events;
+	int n, i;
+
+	while (1) {
+		n = rte_epoll_wait(rx_adapter->epd, epoll_events,
+				RTE_EVENT_ETH_INTR_RING_SIZE, -1);
+		if (unlikely(n < 0))
+			RTE_EDEV_LOG_ERR("rte_epoll_wait returned error %d",
+					n);
+		for (i = 0; i < n; i++) {
+			rxa_intr_ring_enqueue(rx_adapter,
+					epoll_events[i].epdata.data);
+		}
+	}
+
+	return NULL;
+}
+
+/* Dequeue <port, q> from interrupt ring and enqueue received
+ * mbufs to eventdev
+ */
+static inline uint32_t
+rxa_intr_ring_dequeue(struct rte_event_eth_rx_adapter *rx_adapter)
+{
+	uint32_t n;
+	uint32_t nb_rx = 0;
+	int rxq_empty;
+	struct rte_eth_event_enqueue_buffer *buf;
+	rte_spinlock_t *ring_lock;
+	uint8_t max_done = 0;
+
+	if (rx_adapter->num_rx_intr == 0)
+		return 0;
+
+	if (rte_ring_count(rx_adapter->intr_ring) == 0
+		&& !rx_adapter->qd_valid)
+		return 0;
+
+	buf = &rx_adapter->event_enqueue_buffer;
+	ring_lock = &rx_adapter->intr_ring_lock;
+
+	if (buf->count >= BATCH_SIZE)
+		rxa_flush_event_buffer(rx_adapter);
+
+	while (BATCH_SIZE <= (RTE_DIM(buf->events) - buf->count)) {
+		struct eth_device_info *dev_info;
+		uint16_t port;
+		uint16_t queue;
+		union queue_data qd  = rx_adapter->qd;
+		int err;
+
+		if (!rx_adapter->qd_valid) {
+			struct eth_rx_queue_info *queue_info;
+
+			rte_spinlock_lock(ring_lock);
+			err = rte_ring_dequeue(rx_adapter->intr_ring, &qd.ptr);
+			if (err) {
+				rte_spinlock_unlock(ring_lock);
+				break;
+			}
+
+			port = qd.port;
+			queue = qd.queue;
+			rx_adapter->qd = qd;
+			rx_adapter->qd_valid = 1;
+			dev_info = &rx_adapter->eth_devices[port];
+			if (rxa_shared_intr(dev_info, queue))
+				dev_info->shared_intr_enabled = 1;
+			else {
+				queue_info = &dev_info->rx_queue[queue];
+				queue_info->intr_enabled = 1;
+			}
+			rte_eth_dev_rx_intr_enable(port, queue);
+			rte_spinlock_unlock(ring_lock);
+		} else {
+			port = qd.port;
+			queue = qd.queue;
+
+			dev_info = &rx_adapter->eth_devices[port];
+		}
+
+		if (rxa_shared_intr(dev_info, queue)) {
+			uint16_t i;
+			uint16_t nb_queues;
+
+			nb_queues = dev_info->dev->data->nb_rx_queues;
+			n = 0;
+			for (i = dev_info->next_q_idx; i < nb_queues; i++) {
+				uint8_t enq_buffer_full;
+
+				if (!rxa_intr_queue(dev_info, i))
+					continue;
+				n = rxa_eth_rx(rx_adapter, port, i, nb_rx,
+					rx_adapter->max_nb_rx,
+					&rxq_empty);
+				nb_rx += n;
+
+				enq_buffer_full = !rxq_empty && n == 0;
+				max_done = nb_rx > rx_adapter->max_nb_rx;
+
+				if (enq_buffer_full || max_done) {
+					dev_info->next_q_idx = i;
+					goto done;
+				}
+			}
+
+			rx_adapter->qd_valid = 0;
+
+			/* Reinitialize for next interrupt */
+			dev_info->next_q_idx = dev_info->multi_intr_cap ?
+						RTE_MAX_RXTX_INTR_VEC_ID - 1 :
+						0;
+		} else {
+			n = rxa_eth_rx(rx_adapter, port, queue, nb_rx,
+				rx_adapter->max_nb_rx,
+				&rxq_empty);
+			rx_adapter->qd_valid = !rxq_empty;
+			nb_rx += n;
+			if (nb_rx > rx_adapter->max_nb_rx)
+				break;
+		}
+	}
+
+done:
+	rx_adapter->stats.rx_intr_packets += nb_rx;
+	return nb_rx;
 }
 
 /*
@@ -491,12 +1114,10 @@ fill_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter,
  * it.
  */
 static inline uint32_t
-eth_rx_poll(struct rte_event_eth_rx_adapter *rx_adapter)
+rxa_poll(struct rte_event_eth_rx_adapter *rx_adapter)
 {
 	uint32_t num_queue;
-	uint16_t n;
 	uint32_t nb_rx = 0;
-	struct rte_mbuf *mbufs[BATCH_SIZE];
 	struct rte_eth_event_enqueue_buffer *buf;
 	uint32_t wrr_pos;
 	uint32_t max_nb_rx;
@@ -504,7 +1125,7 @@ eth_rx_poll(struct rte_event_eth_rx_adapter *rx_adapter)
 	wrr_pos = rx_adapter->wrr_pos;
 	max_nb_rx = rx_adapter->max_nb_rx;
 	buf = &rx_adapter->event_enqueue_buffer;
-	struct rte_event_eth_rx_adapter_stats *stats = &rx_adapter->stats;
+	stats = &rx_adapter->stats;
 
 	/* Iterate through a WRR sequence */
 	for (num_queue = 0; num_queue < rx_adapter->wrr_len; num_queue++) {
@@ -516,45 +1137,42 @@ eth_rx_poll(struct rte_event_eth_rx_adapter *rx_adapter)
 		 * enough space in the enqueue buffer.
 		 */
 		if (buf->count >= BATCH_SIZE)
-			flush_event_buffer(rx_adapter);
-		if (BATCH_SIZE > (ETH_EVENT_BUFFER_SIZE - buf->count))
-			break;
-
-		stats->rx_poll_count++;
-		n = rte_eth_rx_burst(d, qid, mbufs, BATCH_SIZE);
+			rxa_flush_event_buffer(rx_adapter);
+		if (BATCH_SIZE > (ETH_EVENT_BUFFER_SIZE - buf->count)) {
+			rx_adapter->wrr_pos = wrr_pos;
+			return nb_rx;
+		}
 
-		if (n) {
-			stats->rx_packets += n;
-			/* The check before rte_eth_rx_burst() ensures that
-			 * all n mbufs can be buffered
-			 */
-			fill_event_buffer(rx_adapter, d, qid, mbufs, n);
-			nb_rx += n;
-			if (nb_rx > max_nb_rx) {
-				rx_adapter->wrr_pos =
+		nb_rx += rxa_eth_rx(rx_adapter, d, qid, nb_rx, max_nb_rx,
+				NULL);
+		if (nb_rx > max_nb_rx) {
+			rx_adapter->wrr_pos =
 				    (wrr_pos + 1) % rx_adapter->wrr_len;
-				return nb_rx;
-			}
+			break;
 		}
 
 		if (++wrr_pos == rx_adapter->wrr_len)
 			wrr_pos = 0;
 	}
-
 	return nb_rx;
 }
 
 static int
-event_eth_rx_adapter_service_func(void *args)
+rxa_service_func(void *args)
 {
 	struct rte_event_eth_rx_adapter *rx_adapter = args;
-	struct rte_eth_event_enqueue_buffer *buf;
+	struct rte_event_eth_rx_adapter_stats *stats;
 
-	buf = &rx_adapter->event_enqueue_buffer;
 	if (rte_spinlock_trylock(&rx_adapter->rx_lock) == 0)
 		return 0;
-	if (eth_rx_poll(rx_adapter) == 0 && buf->count)
-		flush_event_buffer(rx_adapter);
+	if (!rx_adapter->rxa_started) {
+		return 0;
+		rte_spinlock_unlock(&rx_adapter->rx_lock);
+	}
+
+	stats = &rx_adapter->stats;
+	stats->rx_packets += rxa_intr_ring_dequeue(rx_adapter);
+	stats->rx_packets += rxa_poll(rx_adapter);
 	rte_spinlock_unlock(&rx_adapter->rx_lock);
 	return 0;
 }
@@ -586,14 +1204,14 @@ rte_event_eth_rx_adapter_init(void)
 }
 
 static inline struct rte_event_eth_rx_adapter *
-id_to_rx_adapter(uint8_t id)
+rxa_id_to_adapter(uint8_t id)
 {
 	return event_eth_rx_adapter ?
 		event_eth_rx_adapter[id] : NULL;
 }
 
 static int
-default_conf_cb(uint8_t id, uint8_t dev_id,
+rxa_default_conf_cb(uint8_t id, uint8_t dev_id,
 		struct rte_event_eth_rx_adapter_conf *conf, void *arg)
 {
 	int ret;
@@ -602,7 +1220,7 @@ default_conf_cb(uint8_t id, uint8_t dev_id,
 	int started;
 	uint8_t port_id;
 	struct rte_event_port_conf *port_conf = arg;
-	struct rte_event_eth_rx_adapter *rx_adapter = id_to_rx_adapter(id);
+	struct rte_event_eth_rx_adapter *rx_adapter = rxa_id_to_adapter(id);
 
 	dev = &rte_eventdevs[rx_adapter->eventdev_id];
 	dev_conf = dev->data->dev_conf;
@@ -639,7 +1257,351 @@ default_conf_cb(uint8_t id, uint8_t dev_id,
 }
 
 static int
-init_service(struct rte_event_eth_rx_adapter *rx_adapter, uint8_t id)
+rxa_epoll_create1(void)
+{
+#if defined(LINUX)
+	int fd;
+	fd = epoll_create1(EPOLL_CLOEXEC);
+	return fd < 0 ? -errno : fd;
+#elif defined(BSD)
+	return -ENOTSUP;
+#endif
+}
+
+static int
+rxa_init_epd(struct rte_event_eth_rx_adapter *rx_adapter)
+{
+	if (rx_adapter->epd != INIT_FD)
+		return 0;
+
+	rx_adapter->epd = rxa_epoll_create1();
+	if (rx_adapter->epd < 0) {
+		int err = rx_adapter->epd;
+		rx_adapter->epd = INIT_FD;
+		RTE_EDEV_LOG_ERR("epoll_create1() failed, err %d", err);
+		return err;
+	}
+
+	return 0;
+}
+
+static int
+rxa_create_intr_thread(struct rte_event_eth_rx_adapter *rx_adapter)
+{
+	int err;
+	char thread_name[RTE_MAX_THREAD_NAME_LEN];
+
+	if (rx_adapter->intr_ring)
+		return 0;
+
+	rx_adapter->intr_ring = rte_ring_create("intr_ring",
+					RTE_EVENT_ETH_INTR_RING_SIZE,
+					rte_socket_id(), 0);
+	if (!rx_adapter->intr_ring)
+		return -ENOMEM;
+
+	rx_adapter->epoll_events = rte_zmalloc_socket(rx_adapter->mem_name,
+					RTE_EVENT_ETH_INTR_RING_SIZE *
+					sizeof(struct rte_epoll_event),
+					RTE_CACHE_LINE_SIZE,
+					rx_adapter->socket_id);
+	if (!rx_adapter->epoll_events) {
+		err = -ENOMEM;
+		goto error;
+	}
+
+	rte_spinlock_init(&rx_adapter->intr_ring_lock);
+
+	snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN,
+			"rx-intr-thread-%d", rx_adapter->id);
+
+	err = rte_ctrl_thread_create(&rx_adapter->rx_intr_thread, thread_name,
+				NULL, rxa_intr_thread, rx_adapter);
+	if (!err) {
+		rte_thread_setname(rx_adapter->rx_intr_thread, thread_name);
+		return 0;
+	}
+
+	RTE_EDEV_LOG_ERR("Failed to create interrupt thread err = %d\n", err);
+error:
+	rte_ring_free(rx_adapter->intr_ring);
+	rx_adapter->intr_ring = NULL;
+	rx_adapter->epoll_events = NULL;
+	return err;
+}
+
+static int
+rxa_destroy_intr_thread(struct rte_event_eth_rx_adapter *rx_adapter)
+{
+	int err;
+
+	err = pthread_cancel(rx_adapter->rx_intr_thread);
+	if (err)
+		RTE_EDEV_LOG_ERR("Can't cancel interrupt thread err = %d\n",
+				err);
+
+	err = pthread_join(rx_adapter->rx_intr_thread, NULL);
+	if (err)
+		RTE_EDEV_LOG_ERR("Can't join interrupt thread err = %d\n", err);
+
+	rte_free(rx_adapter->epoll_events);
+	rte_ring_free(rx_adapter->intr_ring);
+	rx_adapter->intr_ring = NULL;
+	rx_adapter->epoll_events = NULL;
+	return 0;
+}
+
+static int
+rxa_free_intr_resources(struct rte_event_eth_rx_adapter *rx_adapter)
+{
+	int ret;
+
+	if (rx_adapter->num_rx_intr == 0)
+		return 0;
+
+	ret = rxa_destroy_intr_thread(rx_adapter);
+	if (ret)
+		return ret;
+
+	close(rx_adapter->epd);
+	rx_adapter->epd = INIT_FD;
+
+	return ret;
+}
+
+static int
+rxa_disable_intr(struct rte_event_eth_rx_adapter *rx_adapter,
+	struct eth_device_info *dev_info,
+	uint16_t rx_queue_id)
+{
+	int err;
+	uint16_t eth_dev_id = dev_info->dev->data->port_id;
+	int sintr = rxa_shared_intr(dev_info, rx_queue_id);
+
+	err = rte_eth_dev_rx_intr_disable(eth_dev_id, rx_queue_id);
+	if (err) {
+		RTE_EDEV_LOG_ERR("Could not disable interrupt for Rx queue %u",
+			rx_queue_id);
+		return err;
+	}
+
+	err = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
+					rx_adapter->epd,
+					RTE_INTR_EVENT_DEL,
+					0);
+	if (err)
+		RTE_EDEV_LOG_ERR("Interrupt event deletion failed %d", err);
+
+	if (sintr)
+		dev_info->rx_queue[rx_queue_id].intr_enabled = 0;
+	else
+		dev_info->shared_intr_enabled = 0;
+	return err;
+}
+
+static int
+rxa_del_intr_queue(struct rte_event_eth_rx_adapter *rx_adapter,
+		struct eth_device_info *dev_info,
+		int rx_queue_id)
+{
+	int err;
+	int i;
+	int s;
+
+	if (dev_info->nb_rx_intr == 0)
+		return 0;
+
+	err = 0;
+	if (rx_queue_id == -1) {
+		s = dev_info->nb_shared_intr;
+		for (i = 0; i < dev_info->nb_rx_intr; i++) {
+			int sintr;
+			uint16_t q;
+
+			q = dev_info->intr_queue[i];
+			sintr = rxa_shared_intr(dev_info, q);
+			s -= sintr;
+
+			if (!sintr || s == 0) {
+
+				err = rxa_disable_intr(rx_adapter, dev_info,
+						q);
+				if (err)
+					return err;
+				rxa_intr_ring_del_entries(rx_adapter, dev_info,
+							q);
+			}
+		}
+	} else {
+		if (!rxa_intr_queue(dev_info, rx_queue_id))
+			return 0;
+		if (!rxa_shared_intr(dev_info, rx_queue_id) ||
+				dev_info->nb_shared_intr == 1) {
+			err = rxa_disable_intr(rx_adapter, dev_info,
+					rx_queue_id);
+			if (err)
+				return err;
+			rxa_intr_ring_del_entries(rx_adapter, dev_info,
+						rx_queue_id);
+		}
+
+		for (i = 0; i < dev_info->nb_rx_intr; i++) {
+			if (dev_info->intr_queue[i] == rx_queue_id) {
+				for (; i < dev_info->nb_rx_intr - 1; i++)
+					dev_info->intr_queue[i] =
+						dev_info->intr_queue[i + 1];
+				break;
+			}
+		}
+	}
+
+	return err;
+}
+
+static int
+rxa_config_intr(struct rte_event_eth_rx_adapter *rx_adapter,
+	struct eth_device_info *dev_info,
+	uint16_t rx_queue_id)
+{
+	int err, err1;
+	uint16_t eth_dev_id = dev_info->dev->data->port_id;
+	union queue_data qd;
+	int init_fd;
+	uint16_t *intr_queue;
+	int sintr = rxa_shared_intr(dev_info, rx_queue_id);
+
+	if (rxa_intr_queue(dev_info, rx_queue_id))
+		return 0;
+
+	intr_queue = dev_info->intr_queue;
+	if (dev_info->intr_queue == NULL) {
+		size_t len =
+			dev_info->dev->data->nb_rx_queues * sizeof(uint16_t);
+		dev_info->intr_queue =
+			rte_zmalloc_socket(
+				rx_adapter->mem_name,
+				len,
+				0,
+				rx_adapter->socket_id);
+		if (dev_info->intr_queue == NULL)
+			return -ENOMEM;
+	}
+
+	init_fd = rx_adapter->epd;
+	err = rxa_init_epd(rx_adapter);
+	if (err)
+		goto err_free_queue;
+
+	qd.port = eth_dev_id;
+	qd.queue = rx_queue_id;
+
+	err = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
+					rx_adapter->epd,
+					RTE_INTR_EVENT_ADD,
+					qd.ptr);
+	if (err) {
+		RTE_EDEV_LOG_ERR("Failed to add interrupt event for"
+			" Rx Queue %u err %d", rx_queue_id, err);
+		goto err_del_fd;
+	}
+
+	err = rte_eth_dev_rx_intr_enable(eth_dev_id, rx_queue_id);
+	if (err) {
+		RTE_EDEV_LOG_ERR("Could not enable interrupt for"
+				" Rx Queue %u err %d", rx_queue_id, err);
+
+		goto err_del_event;
+	}
+
+	err = rxa_create_intr_thread(rx_adapter);
+	if (!err)  {
+		if (sintr)
+			dev_info->shared_intr_enabled = 1;
+		else
+			dev_info->rx_queue[rx_queue_id].intr_enabled = 1;
+		return 0;
+	}
+
+
+	err = rte_eth_dev_rx_intr_disable(eth_dev_id, rx_queue_id);
+	if (err)
+		RTE_EDEV_LOG_ERR("Could not disable interrupt for"
+				" Rx Queue %u err %d", rx_queue_id, err);
+err_del_event:
+	err1 = rte_eth_dev_rx_intr_ctl_q(eth_dev_id, rx_queue_id,
+					rx_adapter->epd,
+					RTE_INTR_EVENT_DEL,
+					0);
+	if (err1) {
+		RTE_EDEV_LOG_ERR("Could not delete event for"
+				" Rx Queue %u err %d", rx_queue_id, err1);
+	}
+err_del_fd:
+	if (init_fd == INIT_FD) {
+		close(rx_adapter->epd);
+		rx_adapter->epd = -1;
+	}
+err_free_queue:
+	if (intr_queue == NULL)
+		rte_free(dev_info->intr_queue);
+
+	return err;
+}
+
+static int
+rxa_add_intr_queue(struct rte_event_eth_rx_adapter *rx_adapter,
+	struct eth_device_info *dev_info,
+	int rx_queue_id)
+
+{
+	int i, j, err;
+	int si = -1;
+	int shared_done = (dev_info->nb_shared_intr > 0);
+
+	if (rx_queue_id != -1) {
+		if (rxa_shared_intr(dev_info, rx_queue_id) && shared_done)
+			return 0;
+		return rxa_config_intr(rx_adapter, dev_info, rx_queue_id);
+	}
+
+	err = 0;
+	for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++) {
+
+		if (rxa_shared_intr(dev_info, i) && shared_done)
+			continue;
+
+		err = rxa_config_intr(rx_adapter, dev_info, i);
+
+		shared_done = err == 0 && rxa_shared_intr(dev_info, i);
+		if (shared_done) {
+			si = i;
+			dev_info->shared_intr_enabled = 1;
+		}
+		if (err)
+			break;
+	}
+
+	if (err == 0)
+		return 0;
+
+	shared_done = (dev_info->nb_shared_intr > 0);
+	for (j = 0; j < i; j++) {
+		if (rxa_intr_queue(dev_info, j))
+			continue;
+		if (rxa_shared_intr(dev_info, j) && si != j)
+			continue;
+		err = rxa_disable_intr(rx_adapter, dev_info, j);
+		if (err)
+			break;
+
+	}
+
+	return err;
+}
+
+
+static int
+rxa_init_service(struct rte_event_eth_rx_adapter *rx_adapter, uint8_t id)
 {
 	int ret;
 	struct rte_service_spec service;
@@ -652,7 +1614,7 @@ init_service(struct rte_event_eth_rx_adapter *rx_adapter, uint8_t id)
 	snprintf(service.name, ETH_RX_ADAPTER_SERVICE_NAME_LEN,
 		"rte_event_eth_rx_adapter_%d", id);
 	service.socket_id = rx_adapter->socket_id;
-	service.callback = event_eth_rx_adapter_service_func;
+	service.callback = rxa_service_func;
 	service.callback_userdata = rx_adapter;
 	/* Service function handles locking for queue add/del updates */
 	service.capabilities = RTE_SERVICE_CAP_MT_SAFE;
@@ -673,6 +1635,7 @@ init_service(struct rte_event_eth_rx_adapter *rx_adapter, uint8_t id)
 	rx_adapter->event_port_id = rx_adapter_conf.event_port_id;
 	rx_adapter->max_nb_rx = rx_adapter_conf.max_nb_rx;
 	rx_adapter->service_inited = 1;
+	rx_adapter->epd = INIT_FD;
 	return 0;
 
 err_done:
@@ -680,9 +1643,8 @@ err_done:
 	return ret;
 }
 
-
 static void
-update_queue_info(struct rte_event_eth_rx_adapter *rx_adapter,
+rxa_update_queue(struct rte_event_eth_rx_adapter *rx_adapter,
 		struct eth_device_info *dev_info,
 		int32_t rx_queue_id,
 		uint8_t add)
@@ -696,7 +1658,7 @@ update_queue_info(struct rte_event_eth_rx_adapter *rx_adapter,
 
 	if (rx_queue_id == -1) {
 		for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++)
-			update_queue_info(rx_adapter, dev_info, i, add);
+			rxa_update_queue(rx_adapter, dev_info, i, add);
 	} else {
 		queue_info = &dev_info->rx_queue[rx_queue_id];
 		enabled = queue_info->queue_enabled;
@@ -711,31 +1673,65 @@ update_queue_info(struct rte_event_eth_rx_adapter *rx_adapter,
 	}
 }
 
-static int
-event_eth_rx_adapter_queue_del(struct rte_event_eth_rx_adapter *rx_adapter,
-			    struct eth_device_info *dev_info,
-			    uint16_t rx_queue_id)
+static void
+rxa_sw_del(struct rte_event_eth_rx_adapter *rx_adapter,
+	struct eth_device_info *dev_info,
+	int32_t rx_queue_id)
 {
-	struct eth_rx_queue_info *queue_info;
+	int pollq;
+	int intrq;
+	int sintrq;
+
 
 	if (rx_adapter->nb_queues == 0)
-		return 0;
+		return;
 
-	queue_info = &dev_info->rx_queue[rx_queue_id];
-	rx_adapter->num_rx_polled -= queue_info->queue_enabled;
-	update_queue_info(rx_adapter, dev_info, rx_queue_id, 0);
-	return 0;
+	if (rx_queue_id == -1) {
+		uint16_t nb_rx_queues;
+		uint16_t i;
+
+		nb_rx_queues = dev_info->dev->data->nb_rx_queues;
+		for (i = 0; i <	nb_rx_queues; i++)
+			rxa_sw_del(rx_adapter, dev_info, i);
+		return;
+	}
+
+	pollq = rxa_polled_queue(dev_info, rx_queue_id);
+	intrq = rxa_intr_queue(dev_info, rx_queue_id);
+	sintrq = rxa_shared_intr(dev_info, rx_queue_id);
+	rxa_update_queue(rx_adapter, dev_info, rx_queue_id, 0);
+	rx_adapter->num_rx_polled -= pollq;
+	dev_info->nb_rx_poll -= pollq;
+	rx_adapter->num_rx_intr -= intrq;
+	dev_info->nb_rx_intr -= intrq;
+	dev_info->nb_shared_intr -= intrq && sintrq;
 }
 
 static void
-event_eth_rx_adapter_queue_add(struct rte_event_eth_rx_adapter *rx_adapter,
-		struct eth_device_info *dev_info,
-		uint16_t rx_queue_id,
-		const struct rte_event_eth_rx_adapter_queue_conf *conf)
-
+rxa_add_queue(struct rte_event_eth_rx_adapter *rx_adapter,
+	struct eth_device_info *dev_info,
+	int32_t rx_queue_id,
+	const struct rte_event_eth_rx_adapter_queue_conf *conf)
 {
 	struct eth_rx_queue_info *queue_info;
 	const struct rte_event *ev = &conf->ev;
+	int pollq;
+	int intrq;
+	int sintrq;
+
+	if (rx_queue_id == -1) {
+		uint16_t nb_rx_queues;
+		uint16_t i;
+
+		nb_rx_queues = dev_info->dev->data->nb_rx_queues;
+		for (i = 0; i <	nb_rx_queues; i++)
+			rxa_add_queue(rx_adapter, dev_info, i, conf);
+		return;
+	}
+
+	pollq = rxa_polled_queue(dev_info, rx_queue_id);
+	intrq = rxa_intr_queue(dev_info, rx_queue_id);
+	sintrq = rxa_shared_intr(dev_info, rx_queue_id);
 
 	queue_info = &dev_info->rx_queue[rx_queue_id];
 	queue_info->event_queue_id = ev->queue_id;
@@ -749,69 +1745,162 @@ event_eth_rx_adapter_queue_add(struct rte_event_eth_rx_adapter *rx_adapter,
 		queue_info->flow_id_mask = ~0;
 	}
 
-	/* The same queue can be added more than once */
-	rx_adapter->num_rx_polled += !queue_info->queue_enabled;
-	update_queue_info(rx_adapter, dev_info, rx_queue_id, 1);
+	rxa_update_queue(rx_adapter, dev_info, rx_queue_id, 1);
+	if (rxa_polled_queue(dev_info, rx_queue_id)) {
+		rx_adapter->num_rx_polled += !pollq;
+		dev_info->nb_rx_poll += !pollq;
+		rx_adapter->num_rx_intr -= intrq;
+		dev_info->nb_rx_intr -= intrq;
+		dev_info->nb_shared_intr -= intrq && sintrq;
+	}
+
+	if (rxa_intr_queue(dev_info, rx_queue_id)) {
+		rx_adapter->num_rx_polled -= pollq;
+		dev_info->nb_rx_poll -= pollq;
+		rx_adapter->num_rx_intr += !intrq;
+		dev_info->nb_rx_intr += !intrq;
+		dev_info->nb_shared_intr += !intrq && sintrq;
+		if (dev_info->nb_shared_intr == 1) {
+			if (dev_info->multi_intr_cap)
+				dev_info->next_q_idx =
+					RTE_MAX_RXTX_INTR_VEC_ID - 1;
+			else
+				dev_info->next_q_idx = 0;
+		}
+	}
 }
 
-static int add_rx_queue(struct rte_event_eth_rx_adapter *rx_adapter,
+static int rxa_sw_add(struct rte_event_eth_rx_adapter *rx_adapter,
 		uint16_t eth_dev_id,
 		int rx_queue_id,
 		const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
 {
 	struct eth_device_info *dev_info = &rx_adapter->eth_devices[eth_dev_id];
 	struct rte_event_eth_rx_adapter_queue_conf temp_conf;
-	uint32_t i;
 	int ret;
+	struct eth_rx_poll_entry *rx_poll;
+	struct eth_rx_queue_info *rx_queue;
+	uint32_t *rx_wrr;
+	uint16_t nb_rx_queues;
+	uint32_t nb_rx_poll, nb_wrr;
+	uint32_t nb_rx_intr;
+	int num_intr_vec;
+	uint16_t wt;
 
 	if (queue_conf->servicing_weight == 0) {
-
 		struct rte_eth_dev_data *data = dev_info->dev->data;
-		if (data->dev_conf.intr_conf.rxq) {
-			RTE_EDEV_LOG_ERR("Interrupt driven queues"
-					" not supported");
-			return -ENOTSUP;
-		}
-		temp_conf = *queue_conf;
 
-		/* If Rx interrupts are disabled set wt = 1 */
-		temp_conf.servicing_weight = 1;
+		temp_conf = *queue_conf;
+		if (!data->dev_conf.intr_conf.rxq) {
+			/* If Rx interrupts are disabled set wt = 1 */
+			temp_conf.servicing_weight = 1;
+		}
 		queue_conf = &temp_conf;
 	}
 
+	nb_rx_queues = dev_info->dev->data->nb_rx_queues;
+	rx_queue = dev_info->rx_queue;
+	wt = queue_conf->servicing_weight;
+
 	if (dev_info->rx_queue == NULL) {
 		dev_info->rx_queue =
 		    rte_zmalloc_socket(rx_adapter->mem_name,
-				       dev_info->dev->data->nb_rx_queues *
+				       nb_rx_queues *
 				       sizeof(struct eth_rx_queue_info), 0,
 				       rx_adapter->socket_id);
 		if (dev_info->rx_queue == NULL)
 			return -ENOMEM;
 	}
+	rx_wrr = NULL;
+	rx_poll = NULL;
 
-	if (rx_queue_id == -1) {
-		for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++)
-			event_eth_rx_adapter_queue_add(rx_adapter,
-						dev_info, i,
-						queue_conf);
+	rxa_calc_nb_post_add(rx_adapter, dev_info, rx_queue_id,
+			queue_conf->servicing_weight,
+			&nb_rx_poll, &nb_rx_intr, &nb_wrr);
+
+	if (dev_info->dev->intr_handle)
+		dev_info->multi_intr_cap =
+			rte_intr_cap_multiple(dev_info->dev->intr_handle);
+
+	ret = rxa_alloc_poll_arrays(rx_adapter, nb_rx_poll, nb_wrr,
+				&rx_poll, &rx_wrr);
+	if (ret)
+		goto err_free_rxqueue;
+
+	if (wt == 0) {
+		num_intr_vec = rxa_nb_intr_vect(dev_info, rx_queue_id, 1);
+
+		ret = rxa_intr_ring_check_avail(rx_adapter, num_intr_vec);
+		if (ret)
+			goto err_free_rxqueue;
+
+		ret = rxa_add_intr_queue(rx_adapter, dev_info, rx_queue_id);
+		if (ret)
+			goto err_free_rxqueue;
 	} else {
-		event_eth_rx_adapter_queue_add(rx_adapter, dev_info,
-					  (uint16_t)rx_queue_id,
-					  queue_conf);
+
+		num_intr_vec = 0;
+		if (rx_adapter->num_rx_intr > nb_rx_intr) {
+			num_intr_vec = rxa_nb_intr_vect(dev_info,
+						rx_queue_id, 0);
+			/* interrupt based queues are being converted to
+			 * poll mode queues, delete the interrupt configuration
+			 * for those.
+			 */
+			ret = rxa_del_intr_queue(rx_adapter,
+						dev_info, rx_queue_id);
+			if (ret)
+				goto err_free_rxqueue;
+		}
 	}
 
-	ret = eth_poll_wrr_calc(rx_adapter);
-	if (ret) {
-		event_eth_rx_adapter_queue_del(rx_adapter,
-					dev_info, rx_queue_id);
-		return ret;
+	if (nb_rx_intr == 0) {
+		ret = rxa_free_intr_resources(rx_adapter);
+		if (ret)
+			goto err_free_rxqueue;
 	}
 
-	return ret;
+	if (wt == 0) {
+		uint16_t i;
+
+		if (rx_queue_id  == -1) {
+			for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++)
+				dev_info->intr_queue[i] = i;
+		} else {
+			if (!rxa_intr_queue(dev_info, rx_queue_id))
+				dev_info->intr_queue[nb_rx_intr - 1] =
+					rx_queue_id;
+		}
+	}
+
+
+
+	rxa_add_queue(rx_adapter, dev_info, rx_queue_id, queue_conf);
+	rxa_calc_wrr_sequence(rx_adapter, rx_poll, rx_wrr);
+
+	rte_free(rx_adapter->eth_rx_poll);
+	rte_free(rx_adapter->wrr_sched);
+
+	rx_adapter->eth_rx_poll = rx_poll;
+	rx_adapter->wrr_sched = rx_wrr;
+	rx_adapter->wrr_len = nb_wrr;
+	rx_adapter->num_intr_vec += num_intr_vec;
+	return 0;
+
+err_free_rxqueue:
+	if (rx_queue == NULL) {
+		rte_free(dev_info->rx_queue);
+		dev_info->rx_queue = NULL;
+	}
+
+	rte_free(rx_poll);
+	rte_free(rx_wrr);
+
+	return 0;
 }
 
 static int
-rx_adapter_ctrl(uint8_t id, int start)
+rxa_ctrl(uint8_t id, int start)
 {
 	struct rte_event_eth_rx_adapter *rx_adapter;
 	struct rte_eventdev *dev;
@@ -821,7 +1910,7 @@ rx_adapter_ctrl(uint8_t id, int start)
 	int stop = !start;
 
 	RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
-	rx_adapter = id_to_rx_adapter(id);
+	rx_adapter = rxa_id_to_adapter(id);
 	if (rx_adapter == NULL)
 		return -EINVAL;
 
@@ -845,8 +1934,12 @@ rx_adapter_ctrl(uint8_t id, int start)
 						&rte_eth_devices[i]);
 	}
 
-	if (use_service)
+	if (use_service) {
+		rte_spinlock_lock(&rx_adapter->rx_lock);
+		rx_adapter->rxa_started = start;
 		rte_service_runstate_set(rx_adapter->service_id, start);
+		rte_spinlock_unlock(&rx_adapter->rx_lock);
+	}
 
 	return 0;
 }
@@ -880,7 +1973,7 @@ rte_event_eth_rx_adapter_create_ext(uint8_t id, uint8_t dev_id,
 			return ret;
 	}
 
-	rx_adapter = id_to_rx_adapter(id);
+	rx_adapter = rxa_id_to_adapter(id);
 	if (rx_adapter != NULL) {
 		RTE_EDEV_LOG_ERR("Eth Rx adapter exists id = %" PRIu8, id);
 		return -EEXIST;
@@ -902,6 +1995,7 @@ rte_event_eth_rx_adapter_create_ext(uint8_t id, uint8_t dev_id,
 	rx_adapter->socket_id = socket_id;
 	rx_adapter->conf_cb = conf_cb;
 	rx_adapter->conf_arg = conf_arg;
+	rx_adapter->id = id;
 	strcpy(rx_adapter->mem_name, mem_name);
 	rx_adapter->eth_devices = rte_zmalloc_socket(rx_adapter->mem_name,
 					/* FIXME: incompatible with hotplug */
@@ -922,7 +2016,7 @@ rte_event_eth_rx_adapter_create_ext(uint8_t id, uint8_t dev_id,
 		rx_adapter->eth_devices[i].dev = &rte_eth_devices[i];
 
 	event_eth_rx_adapter[id] = rx_adapter;
-	if (conf_cb == default_conf_cb)
+	if (conf_cb == rxa_default_conf_cb)
 		rx_adapter->default_cb_arg = 1;
 	return 0;
 }
@@ -943,7 +2037,7 @@ rte_event_eth_rx_adapter_create(uint8_t id, uint8_t dev_id,
 		return -ENOMEM;
 	*pc = *port_config;
 	ret = rte_event_eth_rx_adapter_create_ext(id, dev_id,
-					default_conf_cb,
+					rxa_default_conf_cb,
 					pc);
 	if (ret)
 		rte_free(pc);
@@ -957,7 +2051,7 @@ rte_event_eth_rx_adapter_free(uint8_t id)
 
 	RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 
-	rx_adapter = id_to_rx_adapter(id);
+	rx_adapter = rxa_id_to_adapter(id);
 	if (rx_adapter == NULL)
 		return -EINVAL;
 
@@ -987,12 +2081,11 @@ rte_event_eth_rx_adapter_queue_add(uint8_t id,
 	struct rte_event_eth_rx_adapter *rx_adapter;
 	struct rte_eventdev *dev;
 	struct eth_device_info *dev_info;
-	int start_service;
 
 	RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
 
-	rx_adapter = id_to_rx_adapter(id);
+	rx_adapter = rxa_id_to_adapter(id);
 	if ((rx_adapter == NULL) || (queue_conf == NULL))
 		return -EINVAL;
 
@@ -1030,7 +2123,6 @@ rte_event_eth_rx_adapter_queue_add(uint8_t id,
 		return -EINVAL;
 	}
 
-	start_service = 0;
 	dev_info = &rx_adapter->eth_devices[eth_dev_id];
 
 	if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
@@ -1050,28 +2142,29 @@ rte_event_eth_rx_adapter_queue_add(uint8_t id,
 				&rte_eth_devices[eth_dev_id],
 				rx_queue_id, queue_conf);
 		if (ret == 0) {
-			update_queue_info(rx_adapter,
+			dev_info->internal_event_port = 1;
+			rxa_update_queue(rx_adapter,
 					&rx_adapter->eth_devices[eth_dev_id],
 					rx_queue_id,
 					1);
 		}
 	} else {
 		rte_spinlock_lock(&rx_adapter->rx_lock);
-		ret = init_service(rx_adapter, id);
-		if (ret == 0)
-			ret = add_rx_queue(rx_adapter, eth_dev_id, rx_queue_id,
+		dev_info->internal_event_port = 0;
+		ret = rxa_init_service(rx_adapter, id);
+		if (ret == 0) {
+			uint32_t service_id = rx_adapter->service_id;
+			ret = rxa_sw_add(rx_adapter, eth_dev_id, rx_queue_id,
 					queue_conf);
+			rte_service_component_runstate_set(service_id,
+				rxa_sw_adapter_queue_count(rx_adapter));
+		}
 		rte_spinlock_unlock(&rx_adapter->rx_lock);
-		if (ret == 0)
-			start_service = !!sw_rx_adapter_queue_count(rx_adapter);
 	}
 
 	if (ret)
 		return ret;
 
-	if (start_service)
-		rte_service_component_runstate_set(rx_adapter->service_id, 1);
-
 	return 0;
 }
 
@@ -1084,12 +2177,17 @@ rte_event_eth_rx_adapter_queue_del(uint8_t id, uint16_t eth_dev_id,
 	struct rte_event_eth_rx_adapter *rx_adapter;
 	struct eth_device_info *dev_info;
 	uint32_t cap;
-	uint16_t i;
+	uint32_t nb_rx_poll = 0;
+	uint32_t nb_wrr = 0;
+	uint32_t nb_rx_intr;
+	struct eth_rx_poll_entry *rx_poll = NULL;
+	uint32_t *rx_wrr = NULL;
+	int num_intr_vec;
 
 	RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 	RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
 
-	rx_adapter = id_to_rx_adapter(id);
+	rx_adapter = rxa_id_to_adapter(id);
 	if (rx_adapter == NULL)
 		return -EINVAL;
 
@@ -1116,7 +2214,7 @@ rte_event_eth_rx_adapter_queue_del(uint8_t id, uint16_t eth_dev_id,
 						&rte_eth_devices[eth_dev_id],
 						rx_queue_id);
 		if (ret == 0) {
-			update_queue_info(rx_adapter,
+			rxa_update_queue(rx_adapter,
 					&rx_adapter->eth_devices[eth_dev_id],
 					rx_queue_id,
 					0);
@@ -1126,48 +2224,78 @@ rte_event_eth_rx_adapter_queue_del(uint8_t id, uint16_t eth_dev_id,
 			}
 		}
 	} else {
-		int rc;
+		rxa_calc_nb_post_del(rx_adapter, dev_info, rx_queue_id,
+			&nb_rx_poll, &nb_rx_intr, &nb_wrr);
+
+		ret = rxa_alloc_poll_arrays(rx_adapter, nb_rx_poll, nb_wrr,
+			&rx_poll, &rx_wrr);
+		if (ret)
+			return ret;
+
 		rte_spinlock_lock(&rx_adapter->rx_lock);
-		if (rx_queue_id == -1) {
-			for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++)
-				event_eth_rx_adapter_queue_del(rx_adapter,
-							dev_info,
-							i);
-		} else {
-			event_eth_rx_adapter_queue_del(rx_adapter,
-						dev_info,
-						(uint16_t)rx_queue_id);
+
+		num_intr_vec = 0;
+		if (rx_adapter->num_rx_intr > nb_rx_intr) {
+
+			num_intr_vec = rxa_nb_intr_vect(dev_info,
+						rx_queue_id, 0);
+			ret = rxa_del_intr_queue(rx_adapter, dev_info,
+					rx_queue_id);
+			if (ret)
+				goto unlock_ret;
+		}
+
+		if (nb_rx_intr == 0) {
+			ret = rxa_free_intr_resources(rx_adapter);
+			if (ret)
+				goto unlock_ret;
+		}
+
+		rxa_sw_del(rx_adapter, dev_info, rx_queue_id);
+		rxa_calc_wrr_sequence(rx_adapter, rx_poll, rx_wrr);
+
+		rte_free(rx_adapter->eth_rx_poll);
+		rte_free(rx_adapter->wrr_sched);
+
+		if (nb_rx_intr == 0) {
+			rte_free(dev_info->intr_queue);
+			dev_info->intr_queue = NULL;
 		}
 
-		rc = eth_poll_wrr_calc(rx_adapter);
-		if (rc)
-			RTE_EDEV_LOG_ERR("WRR recalculation failed %" PRId32,
-					rc);
+		rx_adapter->eth_rx_poll = rx_poll;
+		rx_adapter->wrr_sched = rx_wrr;
+		rx_adapter->wrr_len = nb_wrr;
+		rx_adapter->num_intr_vec += num_intr_vec;
 
 		if (dev_info->nb_dev_queues == 0) {
 			rte_free(dev_info->rx_queue);
 			dev_info->rx_queue = NULL;
 		}
-
+unlock_ret:
 		rte_spinlock_unlock(&rx_adapter->rx_lock);
+		if (ret) {
+			rte_free(rx_poll);
+			rte_free(rx_wrr);
+			return ret;
+		}
+
 		rte_service_component_runstate_set(rx_adapter->service_id,
-				sw_rx_adapter_queue_count(rx_adapter));
+				rxa_sw_adapter_queue_count(rx_adapter));
 	}
 
 	return ret;
 }
 
-
 int
 rte_event_eth_rx_adapter_start(uint8_t id)
 {
-	return rx_adapter_ctrl(id, 1);
+	return rxa_ctrl(id, 1);
 }
 
 int
 rte_event_eth_rx_adapter_stop(uint8_t id)
 {
-	return rx_adapter_ctrl(id, 0);
+	return rxa_ctrl(id, 0);
 }
 
 int
@@ -1184,7 +2312,7 @@ rte_event_eth_rx_adapter_stats_get(uint8_t id,
 
 	RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 
-	rx_adapter = id_to_rx_adapter(id);
+	rx_adapter = rxa_id_to_adapter(id);
 	if (rx_adapter  == NULL || stats == NULL)
 		return -EINVAL;
 
@@ -1222,7 +2350,7 @@ rte_event_eth_rx_adapter_stats_reset(uint8_t id)
 
 	RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 
-	rx_adapter = id_to_rx_adapter(id);
+	rx_adapter = rxa_id_to_adapter(id);
 	if (rx_adapter == NULL)
 		return -EINVAL;
 
@@ -1247,7 +2375,7 @@ rte_event_eth_rx_adapter_service_id_get(uint8_t id, uint32_t *service_id)
 
 	RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 
-	rx_adapter = id_to_rx_adapter(id);
+	rx_adapter = rxa_id_to_adapter(id);
 	if (rx_adapter == NULL || service_id == NULL)
 		return -EINVAL;
 
@@ -1256,3 +2384,47 @@ rte_event_eth_rx_adapter_service_id_get(uint8_t id, uint32_t *service_id)
 
 	return rx_adapter->service_inited ? 0 : -ESRCH;
 }
+
+int rte_event_eth_rx_adapter_cb_register(uint8_t id,
+					uint16_t eth_dev_id,
+					rte_event_eth_rx_adapter_cb_fn cb_fn,
+					void *cb_arg)
+{
+	struct rte_event_eth_rx_adapter *rx_adapter;
+	struct eth_device_info *dev_info;
+	uint32_t cap;
+	int ret;
+
+	RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL);
+
+	rx_adapter = rxa_id_to_adapter(id);
+	if (rx_adapter == NULL)
+		return -EINVAL;
+
+	dev_info = &rx_adapter->eth_devices[eth_dev_id];
+	if (dev_info->rx_queue == NULL)
+		return -EINVAL;
+
+	ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id,
+						eth_dev_id,
+						&cap);
+	if (ret) {
+		RTE_EDEV_LOG_ERR("Failed to get adapter caps edev %" PRIu8
+			"eth port %" PRIu16, id, eth_dev_id);
+		return ret;
+	}
+
+	if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) {
+		RTE_EDEV_LOG_ERR("Rx callback not supported for eth port %"
+				PRIu16, eth_dev_id);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&rx_adapter->rx_lock);
+	dev_info->cb_fn = cb_fn;
+	dev_info->cb_arg = cb_arg;
+	rte_spinlock_unlock(&rx_adapter->rx_lock);
+
+	return 0;
+}
diff --git a/lib/librte_eventdev/rte_event_eth_rx_adapter.h b/lib/librte_eventdev/rte_event_eth_rx_adapter.h
index 307b2b50..332ee216 100644
--- a/lib/librte_eventdev/rte_event_eth_rx_adapter.h
+++ b/lib/librte_eventdev/rte_event_eth_rx_adapter.h
@@ -63,9 +63,22 @@
  * rte_event_eth_rx_adapter_service_id_get() function can be used to retrieve
  * the service function ID of the adapter in this case.
  *
+ * For SW based packet transfers, i.e., when the
+ * RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT is not set in the adapter's
+ * capabilities flags for a particular ethernet device, the service function
+ * temporarily enqueues mbufs to an event buffer before batch enqueueing these
+ * to the event device. If the buffer fills up, the service function stops
+ * dequeueing packets from the ethernet device. The application may want to
+ * monitor the buffer fill level and instruct the service function to
+ * selectively buffer packets. The application may also use some other
+ * criteria to decide which packets should enter the event device even when
+ * the event buffer fill level is low. The
+ * rte_event_eth_rx_adapter_cb_register() function allows the
+ * application to register a callback that selects which packets to enqueue
+ * to the event device.
+ *
  * Note:
- * 1) Interrupt driven receive queues are currently unimplemented.
- * 2) Devices created after an instance of rte_event_eth_rx_adapter_create
+ * 1) Devices created after an instance of rte_event_eth_rx_adapter_create
  *  should be added to a new instance of the rx adapter.
  */
 
@@ -199,12 +212,55 @@ struct rte_event_eth_rx_adapter_stats {
 	 * block cycles can be used to compute the percentage of
 	 * cycles the service is blocked by the event device.
 	 */
+	uint64_t rx_intr_packets;
+	/**< Received packet count for interrupt mode Rx queues */
 };
 
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice
  *
+ * Callback function invoked by the SW adapter before it continues
+ * to process packets. The callback is passed the size of the enqueue
+ * buffer in the SW adapter and the occupancy of the buffer. The
+ * callback can use these values to decide which mbufs should be
+ * enqueued to the event device. If the return value of the callback
+ * is less than nb_mbuf then the SW adapter uses the return value to
+ * enqueue enq_mbuf[] to the event device.
+ *
+ * @param eth_dev_id
+ *  Port identifier of the Ethernet device.
+ * @param queue_id
+ *  Receive queue index.
+ * @param enqueue_buf_size
+ *  Total enqueue buffer size.
+ * @param enqueue_buf_count
+ *  mbuf count in enqueue buffer.
+ * @param mbuf
+ *  mbuf array.
+ * @param nb_mbuf
+ *  mbuf count.
+ * @param cb_arg
+ *  Callback argument.
+ * @param[out] enq_mbuf
+ *  The adapter enqueues enq_mbuf[] if the return value of the
+ *  callback is less than nb_mbuf
+ * @return
+ *  Returns the number of mbufs should be enqueued to eventdev
+ */
+typedef uint16_t (*rte_event_eth_rx_adapter_cb_fn)(uint16_t eth_dev_id,
+						uint16_t queue_id,
+						uint32_t enqueue_buf_size,
+						uint32_t enqueue_buf_count,
+						struct rte_mbuf **mbuf,
+						uint16_t nb_mbuf,
+						void *cb_arg,
+						struct rte_mbuf **enq_buf);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
  * Create a new ethernet Rx event adapter with the specified identifier.
  *
  * @param id
@@ -425,6 +481,32 @@ int rte_event_eth_rx_adapter_stats_reset(uint8_t id);
  */
 int rte_event_eth_rx_adapter_service_id_get(uint8_t id, uint32_t *service_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Register callback to process Rx packets, this is supported for
+ * SW based packet transfers.
+ * @see rte_event_eth_rx_cb_fn
+ *
+ * @param id
+ *  Adapter identifier.
+ * @param eth_dev_id
+ *  Port identifier of Ethernet device.
+ * @param cb_fn
+ *  Callback function.
+ * @param cb_arg
+ *  Callback arg.
+ * @return
+ *  - 0: Success
+ *  - <0: Error code on failure.
+ */
+int __rte_experimental
+rte_event_eth_rx_adapter_cb_register(uint8_t id,
+				uint16_t eth_dev_id,
+				rte_event_eth_rx_adapter_cb_fn cb_fn,
+				void *cb_arg);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_eventdev/rte_event_ring.c b/lib/librte_eventdev/rte_event_ring.c
index eb67751d..16d02a95 100644
--- a/lib/librte_eventdev/rte_event_ring.c
+++ b/lib/librte_eventdev/rte_event_ring.c
@@ -82,11 +82,16 @@ rte_event_ring_create(const char *name, unsigned int count, int socket_id,
 	mz = rte_memzone_reserve(mz_name, ring_size, socket_id, mz_flags);
 	if (mz != NULL) {
 		r = mz->addr;
-		/*
-		 * no need to check return value here, we already checked the
-		 * arguments above
-		 */
-		rte_event_ring_init(r, name, requested_count, flags);
+		/* Check return value in case rte_ring_init() fails on size */
+		int err = rte_event_ring_init(r, name, requested_count, flags);
+		if (err) {
+			RTE_LOG(ERR, RING, "Ring init failed\n");
+			if (rte_memzone_free(mz) != 0)
+				RTE_LOG(ERR, RING, "Cannot free memzone\n");
+			rte_free(te);
+			rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+			return NULL;
+		}
 
 		te->data = (void *) r;
 		r->r.memzone = mz;
diff --git a/lib/librte_eventdev/rte_event_timer_adapter.c b/lib/librte_eventdev/rte_event_timer_adapter.c
index 6f1d672c..79070d48 100644
--- a/lib/librte_eventdev/rte_event_timer_adapter.c
+++ b/lib/librte_eventdev/rte_event_timer_adapter.c
@@ -1282,9 +1282,7 @@ static const struct rte_event_timer_adapter_ops sw_event_adapter_timer_ops = {
 	.cancel_burst = sw_event_timer_cancel_burst,
 };
 
-RTE_INIT(event_timer_adapter_init_log);
-static void
-event_timer_adapter_init_log(void)
+RTE_INIT(event_timer_adapter_init_log)
 {
 	evtim_logtype = rte_log_register("lib.eventdev.adapter.timer");
 	if (evtim_logtype >= 0)
diff --git a/lib/librte_eventdev/rte_eventdev.c b/lib/librte_eventdev/rte_eventdev.c
index 7ca9fd14..801810ed 100644
--- a/lib/librte_eventdev/rte_eventdev.c
+++ b/lib/librte_eventdev/rte_eventdev.c
@@ -57,16 +57,21 @@ int
 rte_event_dev_get_dev_id(const char *name)
 {
 	int i;
+	uint8_t cmp;
 
 	if (!name)
 		return -EINVAL;
 
-	for (i = 0; i < rte_eventdev_globals->nb_devs; i++)
-		if ((strcmp(rte_event_devices[i].data->name, name)
-				== 0) &&
-				(rte_event_devices[i].attached ==
-						RTE_EVENTDEV_ATTACHED))
+	for (i = 0; i < rte_eventdev_globals->nb_devs; i++) {
+		cmp = (strncmp(rte_event_devices[i].data->name, name,
+				RTE_EVENTDEV_NAME_MAX_LEN) == 0) ||
+			(rte_event_devices[i].dev ? (strncmp(
+				rte_event_devices[i].dev->driver->name, name,
+					 RTE_EVENTDEV_NAME_MAX_LEN) == 0) : 0);
+		if (cmp && (rte_event_devices[i].attached ==
+					RTE_EVENTDEV_ATTACHED))
 			return i;
+	}
 	return -ENODEV;
 }
 
diff --git a/lib/librte_eventdev/rte_eventdev_version.map b/lib/librte_eventdev/rte_eventdev_version.map
index c3f18d6d..12835e9f 100644
--- a/lib/librte_eventdev/rte_eventdev_version.map
+++ b/lib/librte_eventdev/rte_eventdev_version.map
@@ -83,6 +83,19 @@ DPDK_18.05 {
 EXPERIMENTAL {
 	global:
 
+	rte_event_crypto_adapter_caps_get;
+	rte_event_crypto_adapter_create;
+	rte_event_crypto_adapter_create_ext;
+	rte_event_crypto_adapter_event_port_get;
+	rte_event_crypto_adapter_free;
+	rte_event_crypto_adapter_queue_pair_add;
+	rte_event_crypto_adapter_queue_pair_del;
+	rte_event_crypto_adapter_service_id_get;
+	rte_event_crypto_adapter_start;
+	rte_event_crypto_adapter_stats_get;
+	rte_event_crypto_adapter_stats_reset;
+	rte_event_crypto_adapter_stop;
+	rte_event_eth_rx_adapter_cb_register;
 	rte_event_timer_adapter_caps_get;
 	rte_event_timer_adapter_create;
 	rte_event_timer_adapter_create_ext;
@@ -97,16 +110,4 @@ EXPERIMENTAL {
 	rte_event_timer_arm_burst;
 	rte_event_timer_arm_tmo_tick_burst;
 	rte_event_timer_cancel_burst;
-	rte_event_crypto_adapter_caps_get;
-	rte_event_crypto_adapter_create;
-	rte_event_crypto_adapter_create_ext;
-	rte_event_crypto_adapter_event_port_get;
-	rte_event_crypto_adapter_free;
-	rte_event_crypto_adapter_queue_pair_add;
-	rte_event_crypto_adapter_queue_pair_del;
-	rte_event_crypto_adapter_service_id_get;
-	rte_event_crypto_adapter_start;
-	rte_event_crypto_adapter_stats_get;
-	rte_event_crypto_adapter_stats_reset;
-	rte_event_crypto_adapter_stop;
 };
diff --git a/lib/librte_flow_classify/rte_flow_classify.c b/lib/librte_flow_classify/rte_flow_classify.c
index 591d98e2..4c3469da 100644
--- a/lib/librte_flow_classify/rte_flow_classify.c
+++ b/lib/librte_flow_classify/rte_flow_classify.c
@@ -673,10 +673,7 @@ rte_flow_classifier_query(struct rte_flow_classifier *cls,
 	return ret;
 }
 
-RTE_INIT(librte_flow_classify_init_log);
-
-static void
-librte_flow_classify_init_log(void)
+RTE_INIT(librte_flow_classify_init_log)
 {
 	librte_flow_classify_logtype =
 		rte_log_register("lib.flow_classify");
diff --git a/lib/librte_gso/Makefile b/lib/librte_gso/Makefile
index 3648ec09..1fac53a8 100644
--- a/lib/librte_gso/Makefile
+++ b/lib/librte_gso/Makefile
@@ -19,6 +19,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_GSO) += rte_gso.c
 SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_common.c
 SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_tcp4.c
 SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_tunnel_tcp4.c
+SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_udp4.c
 
 # install this header file
 SYMLINK-$(CONFIG_RTE_LIBRTE_GSO)-include += rte_gso.h
diff --git a/lib/librte_gso/gso_common.h b/lib/librte_gso/gso_common.h
index 5ca59745..6cd764ff 100644
--- a/lib/librte_gso/gso_common.h
+++ b/lib/librte_gso/gso_common.h
@@ -31,6 +31,9 @@
 		(PKT_TX_TCP_SEG | PKT_TX_IPV4 | PKT_TX_OUTER_IPV4 | \
 		 PKT_TX_TUNNEL_GRE))
 
+#define IS_IPV4_UDP(flag) (((flag) & (PKT_TX_UDP_SEG | PKT_TX_IPV4)) == \
+		(PKT_TX_UDP_SEG | PKT_TX_IPV4))
+
 /**
  * Internal function which updates the UDP header of a packet, following
  * segmentation. This is required to update the header's datagram length field.
diff --git a/lib/librte_gso/gso_udp4.c b/lib/librte_gso/gso_udp4.c
new file mode 100644
index 00000000..927dee12
--- /dev/null
+++ b/lib/librte_gso/gso_udp4.c
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include "gso_common.h"
+#include "gso_udp4.h"
+
+#define IPV4_HDR_MF_BIT (1U << 13)
+
+static inline void
+update_ipv4_udp_headers(struct rte_mbuf *pkt, struct rte_mbuf **segs,
+		uint16_t nb_segs)
+{
+	struct ipv4_hdr *ipv4_hdr;
+	uint16_t frag_offset = 0, is_mf;
+	uint16_t l2_hdrlen = pkt->l2_len, l3_hdrlen = pkt->l3_len;
+	uint16_t tail_idx = nb_segs - 1, length, i;
+
+	/*
+	 * Update IP header fields for output segments. Specifically,
+	 * keep the same IP id, update fragment offset and total
+	 * length.
+	 */
+	for (i = 0; i < nb_segs; i++) {
+		ipv4_hdr = rte_pktmbuf_mtod_offset(segs[i], struct ipv4_hdr *,
+			l2_hdrlen);
+		length = segs[i]->pkt_len - l2_hdrlen;
+		ipv4_hdr->total_length = rte_cpu_to_be_16(length);
+
+		is_mf = i < tail_idx ? IPV4_HDR_MF_BIT : 0;
+		ipv4_hdr->fragment_offset =
+			rte_cpu_to_be_16(frag_offset | is_mf);
+		frag_offset += ((length - l3_hdrlen) >> 3);
+	}
+}
+
+int
+gso_udp4_segment(struct rte_mbuf *pkt,
+		uint16_t gso_size,
+		struct rte_mempool *direct_pool,
+		struct rte_mempool *indirect_pool,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out)
+{
+	struct ipv4_hdr *ipv4_hdr;
+	uint16_t pyld_unit_size, hdr_offset;
+	uint16_t frag_off;
+	int ret;
+
+	/* Don't process the fragmented packet */
+	ipv4_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv4_hdr *,
+			pkt->l2_len);
+	frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
+	if (unlikely(IS_FRAGMENTED(frag_off))) {
+		pkts_out[0] = pkt;
+		return 1;
+	}
+
+	/*
+	 * UDP fragmentation is the same as IP fragmentation.
+	 * Except the first one, other output packets just have l2
+	 * and l3 headers.
+	 */
+	hdr_offset = pkt->l2_len + pkt->l3_len;
+
+	/* Don't process the packet without data. */
+	if (unlikely(hdr_offset + pkt->l4_len >= pkt->pkt_len)) {
+		pkts_out[0] = pkt;
+		return 1;
+	}
+
+	pyld_unit_size = gso_size - hdr_offset;
+
+	/* Segment the payload */
+	ret = gso_do_segment(pkt, hdr_offset, pyld_unit_size, direct_pool,
+			indirect_pool, pkts_out, nb_pkts_out);
+	if (ret > 1)
+		update_ipv4_udp_headers(pkt, pkts_out, ret);
+
+	return ret;
+}
diff --git a/lib/librte_gso/gso_udp4.h b/lib/librte_gso/gso_udp4.h
new file mode 100644
index 00000000..b2a2908e
--- /dev/null
+++ b/lib/librte_gso/gso_udp4.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef _GSO_UDP4_H_
+#define _GSO_UDP4_H_
+
+#include <stdint.h>
+#include <rte_mbuf.h>
+
+/**
+ * Segment an UDP/IPv4 packet. This function doesn't check if the input
+ * packet has correct checksums, and doesn't update checksums for output
+ * GSO segments. Furthermore, it doesn't process IP fragment packets.
+ *
+ * @param pkt
+ *  The packet mbuf to segment.
+ * @param gso_size
+ *  The max length of a GSO segment, measured in bytes.
+ * @param direct_pool
+ *  MBUF pool used for allocating direct buffers for output segments.
+ * @param indirect_pool
+ *  MBUF pool used for allocating indirect buffers for output segments.
+ * @param pkts_out
+ *  Pointer array used to store the MBUF addresses of output GSO
+ *  segments, when the function succeeds. If the memory space in
+ *  pkts_out is insufficient, it fails and returns -EINVAL.
+ * @param nb_pkts_out
+ *  The max number of items that 'pkts_out' can keep.
+ *
+ * @return
+ *   - The number of GSO segments filled in pkts_out on success.
+ *   - Return -ENOMEM if run out of memory in MBUF pools.
+ *   - Return -EINVAL for invalid parameters.
+ */
+int gso_udp4_segment(struct rte_mbuf *pkt,
+		uint16_t gso_size,
+		struct rte_mempool *direct_pool,
+		struct rte_mempool *indirect_pool,
+		struct rte_mbuf **pkts_out,
+		uint16_t nb_pkts_out);
+#endif
diff --git a/lib/librte_gso/meson.build b/lib/librte_gso/meson.build
index 056534fb..ad8dd858 100644
--- a/lib/librte_gso/meson.build
+++ b/lib/librte_gso/meson.build
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-sources = files('gso_common.c', 'gso_tcp4.c',
+sources = files('gso_common.c', 'gso_tcp4.c', 'gso_udp4.c',
  		'gso_tunnel_tcp4.c', 'rte_gso.c')
 headers = files('rte_gso.h')
 deps += ['ethdev']
diff --git a/lib/librte_gso/rte_gso.c b/lib/librte_gso/rte_gso.c
index a44e3d43..751b5b62 100644
--- a/lib/librte_gso/rte_gso.c
+++ b/lib/librte_gso/rte_gso.c
@@ -11,6 +11,17 @@
 #include "gso_common.h"
 #include "gso_tcp4.h"
 #include "gso_tunnel_tcp4.h"
+#include "gso_udp4.h"
+
+#define ILLEGAL_UDP_GSO_CTX(ctx) \
+	((((ctx)->gso_types & DEV_TX_OFFLOAD_UDP_TSO) == 0) || \
+	 (ctx)->gso_size < RTE_GSO_UDP_SEG_SIZE_MIN)
+
+#define ILLEGAL_TCP_GSO_CTX(ctx) \
+	((((ctx)->gso_types & (DEV_TX_OFFLOAD_TCP_TSO | \
+		DEV_TX_OFFLOAD_VXLAN_TNL_TSO | \
+		DEV_TX_OFFLOAD_GRE_TNL_TSO)) == 0) || \
+		(ctx)->gso_size < RTE_GSO_SEG_SIZE_MIN)
 
 int
 rte_gso_segment(struct rte_mbuf *pkt,
@@ -27,14 +38,12 @@ rte_gso_segment(struct rte_mbuf *pkt,
 
 	if (pkt == NULL || pkts_out == NULL || gso_ctx == NULL ||
 			nb_pkts_out < 1 ||
-			gso_ctx->gso_size < RTE_GSO_SEG_SIZE_MIN ||
-			((gso_ctx->gso_types & (DEV_TX_OFFLOAD_TCP_TSO |
-			DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
-			DEV_TX_OFFLOAD_GRE_TNL_TSO)) == 0))
+			(ILLEGAL_UDP_GSO_CTX(gso_ctx) &&
+			 ILLEGAL_TCP_GSO_CTX(gso_ctx)))
 		return -EINVAL;
 
 	if (gso_ctx->gso_size >= pkt->pkt_len) {
-		pkt->ol_flags &= (~PKT_TX_TCP_SEG);
+		pkt->ol_flags &= (~(PKT_TX_TCP_SEG | PKT_TX_UDP_SEG));
 		pkts_out[0] = pkt;
 		return 1;
 	}
@@ -59,6 +68,11 @@ rte_gso_segment(struct rte_mbuf *pkt,
 		ret = gso_tcp4_segment(pkt, gso_size, ipid_delta,
 				direct_pool, indirect_pool,
 				pkts_out, nb_pkts_out);
+	} else if (IS_IPV4_UDP(pkt->ol_flags) &&
+			(gso_ctx->gso_types & DEV_TX_OFFLOAD_UDP_TSO)) {
+		pkt->ol_flags &= (~PKT_TX_UDP_SEG);
+		ret = gso_udp4_segment(pkt, gso_size, direct_pool,
+				indirect_pool, pkts_out, nb_pkts_out);
 	} else {
 		/* unsupported packet, skip */
 		pkts_out[0] = pkt;
diff --git a/lib/librte_gso/rte_gso.h b/lib/librte_gso/rte_gso.h
index f4abd61c..a626a11e 100644
--- a/lib/librte_gso/rte_gso.h
+++ b/lib/librte_gso/rte_gso.h
@@ -17,10 +17,14 @@ extern "C" {
 #include <stdint.h>
 #include <rte_mbuf.h>
 
-/* Minimum GSO segment size. */
+/* Minimum GSO segment size for TCP based packets. */
 #define RTE_GSO_SEG_SIZE_MIN (sizeof(struct ether_hdr) + \
 		sizeof(struct ipv4_hdr) + sizeof(struct tcp_hdr) + 1)
 
+/* Minimum GSO segment size for UDP based packets. */
+#define RTE_GSO_UDP_SEG_SIZE_MIN (sizeof(struct ether_hdr) + \
+		sizeof(struct ipv4_hdr) + sizeof(struct udp_hdr) + 1)
+
 /* GSO flags for rte_gso_ctx. */
 #define RTE_GSO_FLAG_IPID_FIXED (1ULL << 0)
 /**< Use fixed IP ids for output GSO segments. Setting
diff --git a/lib/librte_hash/meson.build b/lib/librte_hash/meson.build
index e139e1d7..efc06ede 100644
--- a/lib/librte_hash/meson.build
+++ b/lib/librte_hash/meson.build
@@ -6,7 +6,6 @@ headers = files('rte_cmp_arm64.h',
 	'rte_cmp_x86.h',
 	'rte_crc_arm64.h',
 	'rte_cuckoo_hash.h',
-	'rte_cuckoo_hash_x86.h',
 	'rte_fbk_hash.h',
 	'rte_hash_crc.h',
 	'rte_hash.h',
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index a07543a2..f7b86c8c 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -31,9 +31,6 @@
 #include "rte_hash.h"
 #include "rte_cuckoo_hash.h"
 
-#if defined(RTE_ARCH_X86)
-#include "rte_cuckoo_hash_x86.h"
-#endif
 
 TAILQ_HEAD(rte_hash_list, rte_tailq_entry);
 
@@ -93,8 +90,10 @@ rte_hash_create(const struct rte_hash_parameters *params)
 	void *buckets = NULL;
 	char ring_name[RTE_RING_NAMESIZE];
 	unsigned num_key_slots;
-	unsigned hw_trans_mem_support = 0;
 	unsigned i;
+	unsigned int hw_trans_mem_support = 0, multi_writer_support = 0;
+	unsigned int readwrite_concur_support = 0;
+
 	rte_hash_function default_hash_func = (rte_hash_function)rte_jhash;
 
 	hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
@@ -107,7 +106,6 @@ rte_hash_create(const struct rte_hash_parameters *params)
 	/* Check for valid parameters */
 	if ((params->entries > RTE_HASH_ENTRIES_MAX) ||
 			(params->entries < RTE_HASH_BUCKET_ENTRIES) ||
-			!rte_is_power_of_2(RTE_HASH_BUCKET_ENTRIES) ||
 			(params->key_len == 0)) {
 		rte_errno = EINVAL;
 		RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
@@ -118,21 +116,29 @@ rte_hash_create(const struct rte_hash_parameters *params)
 	if (params->extra_flag & RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT)
 		hw_trans_mem_support = 1;
 
+	if (params->extra_flag & RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD)
+		multi_writer_support = 1;
+
+	if (params->extra_flag & RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY) {
+		readwrite_concur_support = 1;
+		multi_writer_support = 1;
+	}
+
 	/* Store all keys and leave the first entry as a dummy entry for lookup_bulk */
-	if (hw_trans_mem_support)
+	if (multi_writer_support)
 		/*
 		 * Increase number of slots by total number of indices
 		 * that can be stored in the lcore caches
 		 * except for the first cache
 		 */
 		num_key_slots = params->entries + (RTE_MAX_LCORE - 1) *
-					LCORE_CACHE_SIZE + 1;
+					(LCORE_CACHE_SIZE - 1) + 1;
 	else
 		num_key_slots = params->entries + 1;
 
 	snprintf(ring_name, sizeof(ring_name), "HT_%s", params->name);
 	/* Create ring (Dummy slot index is not enqueued) */
-	r = rte_ring_create(ring_name, rte_align32pow2(num_key_slots - 1),
+	r = rte_ring_create(ring_name, rte_align32pow2(num_key_slots),
 			params->socket_id, 0);
 	if (r == NULL) {
 		RTE_LOG(ERR, HASH, "memory allocation failed\n");
@@ -233,7 +239,7 @@ rte_hash_create(const struct rte_hash_parameters *params)
 	h->cmp_jump_table_idx = KEY_OTHER_BYTES;
 #endif
 
-	if (hw_trans_mem_support) {
+	if (multi_writer_support) {
 		h->local_free_slots = rte_zmalloc_socket(NULL,
 				sizeof(struct lcore_cache) * RTE_MAX_LCORE,
 				RTE_CACHE_LINE_SIZE, params->socket_id);
@@ -261,6 +267,8 @@ rte_hash_create(const struct rte_hash_parameters *params)
 	h->key_store = k;
 	h->free_slots = r;
 	h->hw_trans_mem_support = hw_trans_mem_support;
+	h->multi_writer_support = multi_writer_support;
+	h->readwrite_concur_support = readwrite_concur_support;
 
 #if defined(RTE_ARCH_X86)
 	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
@@ -271,24 +279,20 @@ rte_hash_create(const struct rte_hash_parameters *params)
 #endif
 		h->sig_cmp_fn = RTE_HASH_COMPARE_SCALAR;
 
-	/* Turn on multi-writer only with explicit flat from user and TM
+	/* Turn on multi-writer only with explicit flag from user and TM
 	 * support.
 	 */
-	if (params->extra_flag & RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD) {
-		if (h->hw_trans_mem_support) {
-			h->add_key = ADD_KEY_MULTIWRITER_TM;
-		} else {
-			h->add_key = ADD_KEY_MULTIWRITER;
-			h->multiwriter_lock = rte_malloc(NULL,
-							sizeof(rte_spinlock_t),
-							LCORE_CACHE_SIZE);
-			rte_spinlock_init(h->multiwriter_lock);
-		}
-	} else
-		h->add_key = ADD_KEY_SINGLEWRITER;
+	if (h->multi_writer_support) {
+		h->readwrite_lock = rte_malloc(NULL, sizeof(rte_rwlock_t),
+						RTE_CACHE_LINE_SIZE);
+		if (h->readwrite_lock == NULL)
+			goto err_unlock;
+
+		rte_rwlock_init(h->readwrite_lock);
+	}
 
 	/* Populate free slots ring. Entry zero is reserved for key misses. */
-	for (i = 1; i < params->entries + 1; i++)
+	for (i = 1; i < num_key_slots; i++)
 		rte_ring_sp_enqueue(r, (void *)((uintptr_t) i));
 
 	te->data = (void *) h;
@@ -335,11 +339,10 @@ rte_hash_free(struct rte_hash *h)
 
 	rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
 
-	if (h->hw_trans_mem_support)
+	if (h->multi_writer_support) {
 		rte_free(h->local_free_slots);
-
-	if (h->add_key == ADD_KEY_MULTIWRITER)
-		rte_free(h->multiwriter_lock);
+		rte_free(h->readwrite_lock);
+	}
 	rte_ring_free(h->free_slots);
 	rte_free(h->key_store);
 	rte_free(h->buckets);
@@ -366,15 +369,78 @@ rte_hash_secondary_hash(const hash_sig_t primary_hash)
 	return primary_hash ^ ((tag + 1) * alt_bits_xor);
 }
 
+int32_t
+rte_hash_count(const struct rte_hash *h)
+{
+	uint32_t tot_ring_cnt, cached_cnt = 0;
+	uint32_t i, ret;
+
+	if (h == NULL)
+		return -EINVAL;
+
+	if (h->multi_writer_support) {
+		tot_ring_cnt = h->entries + (RTE_MAX_LCORE - 1) *
+					(LCORE_CACHE_SIZE - 1);
+		for (i = 0; i < RTE_MAX_LCORE; i++)
+			cached_cnt += h->local_free_slots[i].len;
+
+		ret = tot_ring_cnt - rte_ring_count(h->free_slots) -
+								cached_cnt;
+	} else {
+		tot_ring_cnt = h->entries;
+		ret = tot_ring_cnt - rte_ring_count(h->free_slots);
+	}
+	return ret;
+}
+
+/* Read write locks implemented using rte_rwlock */
+static inline void
+__hash_rw_writer_lock(const struct rte_hash *h)
+{
+	if (h->multi_writer_support && h->hw_trans_mem_support)
+		rte_rwlock_write_lock_tm(h->readwrite_lock);
+	else if (h->multi_writer_support)
+		rte_rwlock_write_lock(h->readwrite_lock);
+}
+
+
+static inline void
+__hash_rw_reader_lock(const struct rte_hash *h)
+{
+	if (h->readwrite_concur_support && h->hw_trans_mem_support)
+		rte_rwlock_read_lock_tm(h->readwrite_lock);
+	else if (h->readwrite_concur_support)
+		rte_rwlock_read_lock(h->readwrite_lock);
+}
+
+static inline void
+__hash_rw_writer_unlock(const struct rte_hash *h)
+{
+	if (h->multi_writer_support && h->hw_trans_mem_support)
+		rte_rwlock_write_unlock_tm(h->readwrite_lock);
+	else if (h->multi_writer_support)
+		rte_rwlock_write_unlock(h->readwrite_lock);
+}
+
+static inline void
+__hash_rw_reader_unlock(const struct rte_hash *h)
+{
+	if (h->readwrite_concur_support && h->hw_trans_mem_support)
+		rte_rwlock_read_unlock_tm(h->readwrite_lock);
+	else if (h->readwrite_concur_support)
+		rte_rwlock_read_unlock(h->readwrite_lock);
+}
+
 void
 rte_hash_reset(struct rte_hash *h)
 {
 	void *ptr;
-	unsigned i;
+	uint32_t tot_ring_cnt, i;
 
 	if (h == NULL)
 		return;
 
+	__hash_rw_writer_lock(h);
 	memset(h->buckets, 0, h->num_buckets * sizeof(struct rte_hash_bucket));
 	memset(h->key_store, 0, h->key_entry_size * (h->entries + 1));
 
@@ -383,97 +449,260 @@ rte_hash_reset(struct rte_hash *h)
 		rte_pause();
 
 	/* Repopulate the free slots ring. Entry zero is reserved for key misses */
-	for (i = 1; i < h->entries + 1; i++)
+	if (h->multi_writer_support)
+		tot_ring_cnt = h->entries + (RTE_MAX_LCORE - 1) *
+					(LCORE_CACHE_SIZE - 1);
+	else
+		tot_ring_cnt = h->entries;
+
+	for (i = 1; i < tot_ring_cnt + 1; i++)
 		rte_ring_sp_enqueue(h->free_slots, (void *)((uintptr_t) i));
 
-	if (h->hw_trans_mem_support) {
+	if (h->multi_writer_support) {
 		/* Reset local caches per lcore */
 		for (i = 0; i < RTE_MAX_LCORE; i++)
 			h->local_free_slots[i].len = 0;
 	}
+	__hash_rw_writer_unlock(h);
 }
 
-/* Search for an entry that can be pushed to its alternative location */
-static inline int
-make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt,
-		unsigned int *nr_pushes)
+/*
+ * Function called to enqueue back an index in the cache/ring,
+ * as slot has not being used and it can be used in the
+ * next addition attempt.
+ */
+static inline void
+enqueue_slot_back(const struct rte_hash *h,
+		struct lcore_cache *cached_free_slots,
+		void *slot_id)
 {
-	unsigned i, j;
-	int ret;
-	uint32_t next_bucket_idx;
-	struct rte_hash_bucket *next_bkt[RTE_HASH_BUCKET_ENTRIES];
+	if (h->multi_writer_support) {
+		cached_free_slots->objs[cached_free_slots->len] = slot_id;
+		cached_free_slots->len++;
+	} else
+		rte_ring_sp_enqueue(h->free_slots, slot_id);
+}
+
+/* Search a key from bucket and update its data */
+static inline int32_t
+search_and_update(const struct rte_hash *h, void *data, const void *key,
+	struct rte_hash_bucket *bkt, hash_sig_t sig, hash_sig_t alt_hash)
+{
+	int i;
+	struct rte_hash_key *k, *keys = h->key_store;
 
-	/*
-	 * Push existing item (search for bucket with space in
-	 * alternative locations) to its alternative location
-	 */
 	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
-		/* Search for space in alternative locations */
-		next_bucket_idx = bkt->sig_alt[i] & h->bucket_bitmask;
-		next_bkt[i] = &h->buckets[next_bucket_idx];
-		for (j = 0; j < RTE_HASH_BUCKET_ENTRIES; j++) {
-			if (next_bkt[i]->key_idx[j] == EMPTY_SLOT)
-				break;
+		if (bkt->sig_current[i] == sig &&
+				bkt->sig_alt[i] == alt_hash) {
+			k = (struct rte_hash_key *) ((char *)keys +
+					bkt->key_idx[i] * h->key_entry_size);
+			if (rte_hash_cmp_eq(key, k->key, h) == 0) {
+				/* Update data */
+				k->pdata = data;
+				/*
+				 * Return index where key is stored,
+				 * subtracting the first dummy index
+				 */
+				return bkt->key_idx[i] - 1;
+			}
 		}
-
-		if (j != RTE_HASH_BUCKET_ENTRIES)
-			break;
 	}
+	return -1;
+}
 
-	/* Alternative location has spare room (end of recursive function) */
-	if (i != RTE_HASH_BUCKET_ENTRIES) {
-		next_bkt[i]->sig_alt[j] = bkt->sig_current[i];
-		next_bkt[i]->sig_current[j] = bkt->sig_alt[i];
-		next_bkt[i]->key_idx[j] = bkt->key_idx[i];
-		return i;
+/* Only tries to insert at one bucket (@prim_bkt) without trying to push
+ * buckets around.
+ * return 1 if matching existing key, return 0 if succeeds, return -1 for no
+ * empty entry.
+ */
+static inline int32_t
+rte_hash_cuckoo_insert_mw(const struct rte_hash *h,
+		struct rte_hash_bucket *prim_bkt,
+		struct rte_hash_bucket *sec_bkt,
+		const struct rte_hash_key *key, void *data,
+		hash_sig_t sig, hash_sig_t alt_hash, uint32_t new_idx,
+		int32_t *ret_val)
+{
+	unsigned int i;
+	struct rte_hash_bucket *cur_bkt = prim_bkt;
+	int32_t ret;
+
+	__hash_rw_writer_lock(h);
+	/* Check if key was inserted after last check but before this
+	 * protected region in case of inserting duplicated keys.
+	 */
+	ret = search_and_update(h, data, key, cur_bkt, sig, alt_hash);
+	if (ret != -1) {
+		__hash_rw_writer_unlock(h);
+		*ret_val = ret;
+		return 1;
+	}
+	ret = search_and_update(h, data, key, sec_bkt, alt_hash, sig);
+	if (ret != -1) {
+		__hash_rw_writer_unlock(h);
+		*ret_val = ret;
+		return 1;
 	}
 
-	/* Pick entry that has not been pushed yet */
-	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++)
-		if (bkt->flag[i] == 0)
+	/* Insert new entry if there is room in the primary
+	 * bucket.
+	 */
+	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+		/* Check if slot is available */
+		if (likely(prim_bkt->key_idx[i] == EMPTY_SLOT)) {
+			prim_bkt->sig_current[i] = sig;
+			prim_bkt->sig_alt[i] = alt_hash;
+			prim_bkt->key_idx[i] = new_idx;
 			break;
+		}
+	}
+	__hash_rw_writer_unlock(h);
 
-	/* All entries have been pushed, so entry cannot be added */
-	if (i == RTE_HASH_BUCKET_ENTRIES || ++(*nr_pushes) > RTE_HASH_MAX_PUSHES)
-		return -ENOSPC;
+	if (i != RTE_HASH_BUCKET_ENTRIES)
+		return 0;
 
-	/* Set flag to indicate that this entry is going to be pushed */
-	bkt->flag[i] = 1;
+	/* no empty entry */
+	return -1;
+}
 
-	/* Need room in alternative bucket to insert the pushed entry */
-	ret = make_space_bucket(h, next_bkt[i], nr_pushes);
-	/*
-	 * After recursive function.
-	 * Clear flags and insert the pushed entry
-	 * in its alternative location if successful,
-	 * or return error
+/* Shift buckets along provided cuckoo_path (@leaf and @leaf_slot) and fill
+ * the path head with new entry (sig, alt_hash, new_idx)
+ * return 1 if matched key found, return -1 if cuckoo path invalided and fail,
+ * return 0 if succeeds.
+ */
+static inline int
+rte_hash_cuckoo_move_insert_mw(const struct rte_hash *h,
+			struct rte_hash_bucket *bkt,
+			struct rte_hash_bucket *alt_bkt,
+			const struct rte_hash_key *key, void *data,
+			struct queue_node *leaf, uint32_t leaf_slot,
+			hash_sig_t sig, hash_sig_t alt_hash, uint32_t new_idx,
+			int32_t *ret_val)
+{
+	uint32_t prev_alt_bkt_idx;
+	struct rte_hash_bucket *cur_bkt = bkt;
+	struct queue_node *prev_node, *curr_node = leaf;
+	struct rte_hash_bucket *prev_bkt, *curr_bkt = leaf->bkt;
+	uint32_t prev_slot, curr_slot = leaf_slot;
+	int32_t ret;
+
+	__hash_rw_writer_lock(h);
+
+	/* In case empty slot was gone before entering protected region */
+	if (curr_bkt->key_idx[curr_slot] != EMPTY_SLOT) {
+		__hash_rw_writer_unlock(h);
+		return -1;
+	}
+
+	/* Check if key was inserted after last check but before this
+	 * protected region.
 	 */
-	bkt->flag[i] = 0;
-	if (ret >= 0) {
-		next_bkt[i]->sig_alt[ret] = bkt->sig_current[i];
-		next_bkt[i]->sig_current[ret] = bkt->sig_alt[i];
-		next_bkt[i]->key_idx[ret] = bkt->key_idx[i];
-		return i;
-	} else
-		return ret;
+	ret = search_and_update(h, data, key, cur_bkt, sig, alt_hash);
+	if (ret != -1) {
+		__hash_rw_writer_unlock(h);
+		*ret_val = ret;
+		return 1;
+	}
+
+	ret = search_and_update(h, data, key, alt_bkt, alt_hash, sig);
+	if (ret != -1) {
+		__hash_rw_writer_unlock(h);
+		*ret_val = ret;
+		return 1;
+	}
+
+	while (likely(curr_node->prev != NULL)) {
+		prev_node = curr_node->prev;
+		prev_bkt = prev_node->bkt;
+		prev_slot = curr_node->prev_slot;
+
+		prev_alt_bkt_idx =
+			prev_bkt->sig_alt[prev_slot] & h->bucket_bitmask;
+
+		if (unlikely(&h->buckets[prev_alt_bkt_idx]
+				!= curr_bkt)) {
+			/* revert it to empty, otherwise duplicated keys */
+			curr_bkt->key_idx[curr_slot] = EMPTY_SLOT;
+			__hash_rw_writer_unlock(h);
+			return -1;
+		}
+
+		/* Need to swap current/alt sig to allow later
+		 * Cuckoo insert to move elements back to its
+		 * primary bucket if available
+		 */
+		curr_bkt->sig_alt[curr_slot] =
+			 prev_bkt->sig_current[prev_slot];
+		curr_bkt->sig_current[curr_slot] =
+			prev_bkt->sig_alt[prev_slot];
+		curr_bkt->key_idx[curr_slot] =
+			prev_bkt->key_idx[prev_slot];
+
+		curr_slot = prev_slot;
+		curr_node = prev_node;
+		curr_bkt = curr_node->bkt;
+	}
+
+	curr_bkt->sig_current[curr_slot] = sig;
+	curr_bkt->sig_alt[curr_slot] = alt_hash;
+	curr_bkt->key_idx[curr_slot] = new_idx;
+
+	__hash_rw_writer_unlock(h);
+
+	return 0;
 
 }
 
 /*
- * Function called to enqueue back an index in the cache/ring,
- * as slot has not being used and it can be used in the
- * next addition attempt.
+ * Make space for new key, using bfs Cuckoo Search and Multi-Writer safe
+ * Cuckoo
  */
-static inline void
-enqueue_slot_back(const struct rte_hash *h,
-		struct lcore_cache *cached_free_slots,
-		void *slot_id)
+static inline int
+rte_hash_cuckoo_make_space_mw(const struct rte_hash *h,
+			struct rte_hash_bucket *bkt,
+			struct rte_hash_bucket *sec_bkt,
+			const struct rte_hash_key *key, void *data,
+			hash_sig_t sig, hash_sig_t alt_hash,
+			uint32_t new_idx, int32_t *ret_val)
 {
-	if (h->hw_trans_mem_support) {
-		cached_free_slots->objs[cached_free_slots->len] = slot_id;
-		cached_free_slots->len++;
-	} else
-		rte_ring_sp_enqueue(h->free_slots, slot_id);
+	unsigned int i;
+	struct queue_node queue[RTE_HASH_BFS_QUEUE_MAX_LEN];
+	struct queue_node *tail, *head;
+	struct rte_hash_bucket *curr_bkt, *alt_bkt;
+
+	tail = queue;
+	head = queue + 1;
+	tail->bkt = bkt;
+	tail->prev = NULL;
+	tail->prev_slot = -1;
+
+	/* Cuckoo bfs Search */
+	while (likely(tail != head && head <
+					queue + RTE_HASH_BFS_QUEUE_MAX_LEN -
+					RTE_HASH_BUCKET_ENTRIES)) {
+		curr_bkt = tail->bkt;
+		for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+			if (curr_bkt->key_idx[i] == EMPTY_SLOT) {
+				int32_t ret = rte_hash_cuckoo_move_insert_mw(h,
+						bkt, sec_bkt, key, data,
+						tail, i, sig, alt_hash,
+						new_idx, ret_val);
+				if (likely(ret != -1))
+					return ret;
+			}
+
+			/* Enqueue new node and keep prev node info */
+			alt_bkt = &(h->buckets[curr_bkt->sig_alt[i]
+						    & h->bucket_bitmask]);
+			head->bkt = alt_bkt;
+			head->prev = tail;
+			head->prev_slot = i;
+			head++;
+		}
+		tail++;
+	}
+
+	return -ENOSPC;
 }
 
 static inline int32_t
@@ -482,19 +711,15 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
 {
 	hash_sig_t alt_hash;
 	uint32_t prim_bucket_idx, sec_bucket_idx;
-	unsigned i;
 	struct rte_hash_bucket *prim_bkt, *sec_bkt;
-	struct rte_hash_key *new_k, *k, *keys = h->key_store;
+	struct rte_hash_key *new_k, *keys = h->key_store;
 	void *slot_id = NULL;
 	uint32_t new_idx;
 	int ret;
 	unsigned n_slots;
 	unsigned lcore_id;
 	struct lcore_cache *cached_free_slots = NULL;
-	unsigned int nr_pushes = 0;
-
-	if (h->add_key == ADD_KEY_MULTIWRITER)
-		rte_spinlock_lock(h->multiwriter_lock);
+	int32_t ret_val;
 
 	prim_bucket_idx = sig & h->bucket_bitmask;
 	prim_bkt = &h->buckets[prim_bucket_idx];
@@ -505,8 +730,24 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
 	sec_bkt = &h->buckets[sec_bucket_idx];
 	rte_prefetch0(sec_bkt);
 
-	/* Get a new slot for storing the new key */
-	if (h->hw_trans_mem_support) {
+	/* Check if key is already inserted in primary location */
+	__hash_rw_writer_lock(h);
+	ret = search_and_update(h, data, key, prim_bkt, sig, alt_hash);
+	if (ret != -1) {
+		__hash_rw_writer_unlock(h);
+		return ret;
+	}
+
+	/* Check if key is already inserted in secondary location */
+	ret = search_and_update(h, data, key, sec_bkt, alt_hash, sig);
+	if (ret != -1) {
+		__hash_rw_writer_unlock(h);
+		return ret;
+	}
+	__hash_rw_writer_unlock(h);
+
+	/* Did not find a match, so get a new slot for storing the new key */
+	if (h->multi_writer_support) {
 		lcore_id = rte_lcore_id();
 		cached_free_slots = &h->local_free_slots[lcore_id];
 		/* Try to get a free slot from the local cache */
@@ -516,8 +757,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
 					cached_free_slots->objs,
 					LCORE_CACHE_SIZE, NULL);
 			if (n_slots == 0) {
-				ret = -ENOSPC;
-				goto failure;
+				return -ENOSPC;
 			}
 
 			cached_free_slots->len += n_slots;
@@ -528,124 +768,50 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
 		slot_id = cached_free_slots->objs[cached_free_slots->len];
 	} else {
 		if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0) {
-			ret = -ENOSPC;
-			goto failure;
+			return -ENOSPC;
 		}
 	}
 
 	new_k = RTE_PTR_ADD(keys, (uintptr_t)slot_id * h->key_entry_size);
-	rte_prefetch0(new_k);
 	new_idx = (uint32_t)((uintptr_t) slot_id);
-
-	/* Check if key is already inserted in primary location */
-	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
-		if (prim_bkt->sig_current[i] == sig &&
-				prim_bkt->sig_alt[i] == alt_hash) {
-			k = (struct rte_hash_key *) ((char *)keys +
-					prim_bkt->key_idx[i] * h->key_entry_size);
-			if (rte_hash_cmp_eq(key, k->key, h) == 0) {
-				/* Enqueue index of free slot back in the ring. */
-				enqueue_slot_back(h, cached_free_slots, slot_id);
-				/* Update data */
-				k->pdata = data;
-				/*
-				 * Return index where key is stored,
-				 * subtracting the first dummy index
-				 */
-				ret = prim_bkt->key_idx[i] - 1;
-				goto failure;
-			}
-		}
-	}
-
-	/* Check if key is already inserted in secondary location */
-	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
-		if (sec_bkt->sig_alt[i] == sig &&
-				sec_bkt->sig_current[i] == alt_hash) {
-			k = (struct rte_hash_key *) ((char *)keys +
-					sec_bkt->key_idx[i] * h->key_entry_size);
-			if (rte_hash_cmp_eq(key, k->key, h) == 0) {
-				/* Enqueue index of free slot back in the ring. */
-				enqueue_slot_back(h, cached_free_slots, slot_id);
-				/* Update data */
-				k->pdata = data;
-				/*
-				 * Return index where key is stored,
-				 * subtracting the first dummy index
-				 */
-				ret = sec_bkt->key_idx[i] - 1;
-				goto failure;
-			}
-		}
-	}
-
 	/* Copy key */
 	rte_memcpy(new_k->key, key, h->key_len);
 	new_k->pdata = data;
 
-#if defined(RTE_ARCH_X86) /* currently only x86 support HTM */
-	if (h->add_key == ADD_KEY_MULTIWRITER_TM) {
-		ret = rte_hash_cuckoo_insert_mw_tm(prim_bkt,
-				sig, alt_hash, new_idx);
-		if (ret >= 0)
-			return new_idx - 1;
 
-		/* Primary bucket full, need to make space for new entry */
-		ret = rte_hash_cuckoo_make_space_mw_tm(h, prim_bkt, sig,
-							alt_hash, new_idx);
+	/* Find an empty slot and insert */
+	ret = rte_hash_cuckoo_insert_mw(h, prim_bkt, sec_bkt, key, data,
+					sig, alt_hash, new_idx, &ret_val);
+	if (ret == 0)
+		return new_idx - 1;
+	else if (ret == 1) {
+		enqueue_slot_back(h, cached_free_slots, slot_id);
+		return ret_val;
+	}
 
-		if (ret >= 0)
-			return new_idx - 1;
+	/* Primary bucket full, need to make space for new entry */
+	ret = rte_hash_cuckoo_make_space_mw(h, prim_bkt, sec_bkt, key, data,
+					sig, alt_hash, new_idx, &ret_val);
+	if (ret == 0)
+		return new_idx - 1;
+	else if (ret == 1) {
+		enqueue_slot_back(h, cached_free_slots, slot_id);
+		return ret_val;
+	}
 
-		/* Also search secondary bucket to get better occupancy */
-		ret = rte_hash_cuckoo_make_space_mw_tm(h, sec_bkt, sig,
-							alt_hash, new_idx);
+	/* Also search secondary bucket to get better occupancy */
+	ret = rte_hash_cuckoo_make_space_mw(h, sec_bkt, prim_bkt, key, data,
+					alt_hash, sig, new_idx, &ret_val);
 
-		if (ret >= 0)
-			return new_idx - 1;
+	if (ret == 0)
+		return new_idx - 1;
+	else if (ret == 1) {
+		enqueue_slot_back(h, cached_free_slots, slot_id);
+		return ret_val;
 	} else {
-#endif
-		for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
-			/* Check if slot is available */
-			if (likely(prim_bkt->key_idx[i] == EMPTY_SLOT)) {
-				prim_bkt->sig_current[i] = sig;
-				prim_bkt->sig_alt[i] = alt_hash;
-				prim_bkt->key_idx[i] = new_idx;
-				break;
-			}
-		}
-
-		if (i != RTE_HASH_BUCKET_ENTRIES) {
-			if (h->add_key == ADD_KEY_MULTIWRITER)
-				rte_spinlock_unlock(h->multiwriter_lock);
-			return new_idx - 1;
-		}
-
-		/* Primary bucket full, need to make space for new entry
-		 * After recursive function.
-		 * Insert the new entry in the position of the pushed entry
-		 * if successful or return error and
-		 * store the new slot back in the ring
-		 */
-		ret = make_space_bucket(h, prim_bkt, &nr_pushes);
-		if (ret >= 0) {
-			prim_bkt->sig_current[ret] = sig;
-			prim_bkt->sig_alt[ret] = alt_hash;
-			prim_bkt->key_idx[ret] = new_idx;
-			if (h->add_key == ADD_KEY_MULTIWRITER)
-				rte_spinlock_unlock(h->multiwriter_lock);
-			return new_idx - 1;
-		}
-#if defined(RTE_ARCH_X86)
+		enqueue_slot_back(h, cached_free_slots, slot_id);
+		return ret;
 	}
-#endif
-	/* Error in addition, store new slot back in the ring and return error */
-	enqueue_slot_back(h, cached_free_slots, (void *)((uintptr_t) new_idx));
-
-failure:
-	if (h->add_key == ADD_KEY_MULTIWRITER)
-		rte_spinlock_unlock(h->multiwriter_lock);
-	return ret;
 }
 
 int32_t
@@ -690,20 +856,15 @@ rte_hash_add_key_data(const struct rte_hash *h, const void *key, void *data)
 	else
 		return ret;
 }
+
+/* Search one bucket to find the match key */
 static inline int32_t
-__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
-					hash_sig_t sig, void **data)
+search_one_bucket(const struct rte_hash *h, const void *key, hash_sig_t sig,
+			void **data, const struct rte_hash_bucket *bkt)
 {
-	uint32_t bucket_idx;
-	hash_sig_t alt_hash;
-	unsigned i;
-	struct rte_hash_bucket *bkt;
+	int i;
 	struct rte_hash_key *k, *keys = h->key_store;
 
-	bucket_idx = sig & h->bucket_bitmask;
-	bkt = &h->buckets[bucket_idx];
-
-	/* Check if key is in primary location */
 	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
 		if (bkt->sig_current[i] == sig &&
 				bkt->key_idx[i] != EMPTY_SLOT) {
@@ -720,30 +881,41 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
 			}
 		}
 	}
+	return -1;
+}
+
+static inline int32_t
+__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
+					hash_sig_t sig, void **data)
+{
+	uint32_t bucket_idx;
+	hash_sig_t alt_hash;
+	struct rte_hash_bucket *bkt;
+	int ret;
+
+	bucket_idx = sig & h->bucket_bitmask;
+	bkt = &h->buckets[bucket_idx];
 
+	__hash_rw_reader_lock(h);
+
+	/* Check if key is in primary location */
+	ret = search_one_bucket(h, key, sig, data, bkt);
+	if (ret != -1) {
+		__hash_rw_reader_unlock(h);
+		return ret;
+	}
 	/* Calculate secondary hash */
 	alt_hash = rte_hash_secondary_hash(sig);
 	bucket_idx = alt_hash & h->bucket_bitmask;
 	bkt = &h->buckets[bucket_idx];
 
 	/* Check if key is in secondary location */
-	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
-		if (bkt->sig_current[i] == alt_hash &&
-				bkt->sig_alt[i] == sig) {
-			k = (struct rte_hash_key *) ((char *)keys +
-					bkt->key_idx[i] * h->key_entry_size);
-			if (rte_hash_cmp_eq(key, k->key, h) == 0) {
-				if (data != NULL)
-					*data = k->pdata;
-				/*
-				 * Return index where key is stored,
-				 * subtracting the first dummy index
-				 */
-				return bkt->key_idx[i] - 1;
-			}
-		}
+	ret = search_one_bucket(h, key, alt_hash, data, bkt);
+	if (ret != -1) {
+		__hash_rw_reader_unlock(h);
+		return ret;
 	}
-
+	__hash_rw_reader_unlock(h);
 	return -ENOENT;
 }
 
@@ -785,7 +957,7 @@ remove_entry(const struct rte_hash *h, struct rte_hash_bucket *bkt, unsigned i)
 
 	bkt->sig_current[i] = NULL_SIGNATURE;
 	bkt->sig_alt[i] = NULL_SIGNATURE;
-	if (h->hw_trans_mem_support) {
+	if (h->multi_writer_support) {
 		lcore_id = rte_lcore_id();
 		cached_free_slots = &h->local_free_slots[lcore_id];
 		/* Cache full, need to free it. */
@@ -806,20 +978,15 @@ remove_entry(const struct rte_hash *h, struct rte_hash_bucket *bkt, unsigned i)
 	}
 }
 
+/* Search one bucket and remove the matched key */
 static inline int32_t
-__rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
-						hash_sig_t sig)
+search_and_remove(const struct rte_hash *h, const void *key,
+			struct rte_hash_bucket *bkt, hash_sig_t sig)
 {
-	uint32_t bucket_idx;
-	hash_sig_t alt_hash;
-	unsigned i;
-	struct rte_hash_bucket *bkt;
 	struct rte_hash_key *k, *keys = h->key_store;
+	unsigned int i;
 	int32_t ret;
 
-	bucket_idx = sig & h->bucket_bitmask;
-	bkt = &h->buckets[bucket_idx];
-
 	/* Check if key is in primary location */
 	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
 		if (bkt->sig_current[i] == sig &&
@@ -839,32 +1006,42 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
 			}
 		}
 	}
+	return -1;
+}
+
+static inline int32_t
+__rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
+						hash_sig_t sig)
+{
+	uint32_t bucket_idx;
+	hash_sig_t alt_hash;
+	struct rte_hash_bucket *bkt;
+	int32_t ret;
+
+	bucket_idx = sig & h->bucket_bitmask;
+	bkt = &h->buckets[bucket_idx];
+
+	__hash_rw_writer_lock(h);
+	/* look for key in primary bucket */
+	ret = search_and_remove(h, key, bkt, sig);
+	if (ret != -1) {
+		__hash_rw_writer_unlock(h);
+		return ret;
+	}
 
 	/* Calculate secondary hash */
 	alt_hash = rte_hash_secondary_hash(sig);
 	bucket_idx = alt_hash & h->bucket_bitmask;
 	bkt = &h->buckets[bucket_idx];
 
-	/* Check if key is in secondary location */
-	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
-		if (bkt->sig_current[i] == alt_hash &&
-				bkt->key_idx[i] != EMPTY_SLOT) {
-			k = (struct rte_hash_key *) ((char *)keys +
-					bkt->key_idx[i] * h->key_entry_size);
-			if (rte_hash_cmp_eq(key, k->key, h) == 0) {
-				remove_entry(h, bkt, i);
-
-				/*
-				 * Return index where key is stored,
-				 * subtracting the first dummy index
-				 */
-				ret = bkt->key_idx[i] - 1;
-				bkt->key_idx[i] = EMPTY_SLOT;
-				return ret;
-			}
-		}
+	/* look for key in secondary bucket */
+	ret = search_and_remove(h, key, bkt, alt_hash);
+	if (ret != -1) {
+		__hash_rw_writer_unlock(h);
+		return ret;
 	}
 
+	__hash_rw_writer_unlock(h);
 	return -ENOENT;
 }
 
@@ -1006,6 +1183,7 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
 		rte_prefetch0(secondary_bkt[i]);
 	}
 
+	__hash_rw_reader_lock(h);
 	/* Compare signatures and prefetch key slot of first hit */
 	for (i = 0; i < num_keys; i++) {
 		compare_signatures(&prim_hitmask[i], &sec_hitmask[i],
@@ -1088,6 +1266,8 @@ next_key:
 		continue;
 	}
 
+	__hash_rw_reader_unlock(h);
+
 	if (hit_mask != NULL)
 		*hit_mask = hits;
 }
@@ -1146,7 +1326,7 @@ rte_hash_iterate(const struct rte_hash *h, const void **key, void **data, uint32
 		bucket_idx = *next / RTE_HASH_BUCKET_ENTRIES;
 		idx = *next % RTE_HASH_BUCKET_ENTRIES;
 	}
-
+	__hash_rw_reader_lock(h);
 	/* Get position of entry in key table */
 	position = h->buckets[bucket_idx].key_idx[idx];
 	next_key = (struct rte_hash_key *) ((char *)h->key_store +
@@ -1155,6 +1335,8 @@ rte_hash_iterate(const struct rte_hash *h, const void **key, void **data, uint32
 	*key = next_key->key;
 	*data = next_key->pdata;
 
+	__hash_rw_reader_unlock(h);
+
 	/* Increment iterator */
 	(*next)++;
 
diff --git a/lib/librte_hash/rte_cuckoo_hash.h b/lib/librte_hash/rte_cuckoo_hash.h
index 7a54e555..b43f467d 100644
--- a/lib/librte_hash/rte_cuckoo_hash.h
+++ b/lib/librte_hash/rte_cuckoo_hash.h
@@ -88,15 +88,14 @@ const rte_hash_cmp_eq_t cmp_jump_table[NUM_KEY_CMP_CASES] = {
 
 #endif
 
-enum add_key_case {
-	ADD_KEY_SINGLEWRITER = 0,
-	ADD_KEY_MULTIWRITER,
-	ADD_KEY_MULTIWRITER_TM,
-};
 
 /** Number of items per bucket. */
 #define RTE_HASH_BUCKET_ENTRIES		8
 
+#if !RTE_IS_POWER_OF_2(RTE_HASH_BUCKET_ENTRIES)
+#error RTE_HASH_BUCKET_ENTRIES must be a power of 2
+#endif
+
 #define NULL_SIGNATURE			0
 
 #define EMPTY_SLOT			0
@@ -155,18 +154,20 @@ struct rte_hash {
 
 	struct rte_ring *free_slots;
 	/**< Ring that stores all indexes of the free slots in the key table */
-	uint8_t hw_trans_mem_support;
-	/**< Hardware transactional memory support */
+
 	struct lcore_cache *local_free_slots;
 	/**< Local cache per lcore, storing some indexes of the free slots */
-	enum add_key_case add_key; /**< Multi-writer hash add behavior */
-
-	rte_spinlock_t *multiwriter_lock; /**< Multi-writer spinlock for w/o TM */
 
 	/* Fields used in lookup */
 
 	uint32_t key_len __rte_cache_aligned;
 	/**< Length of hash key. */
+	uint8_t hw_trans_mem_support;
+	/**< If hardware transactional memory is used. */
+	uint8_t multi_writer_support;
+	/**< If multi-writer support is enabled. */
+	uint8_t readwrite_concur_support;
+	/**< If read-write concurrency support is enabled */
 	rte_hash_function hash_func;    /**< Function used to calculate hash. */
 	uint32_t hash_func_init_val;    /**< Init value used by hash_func. */
 	rte_hash_cmp_eq_t rte_hash_custom_cmp_eq;
@@ -184,6 +185,7 @@ struct rte_hash {
 	/**< Table with buckets storing all the	hash values and key indexes
 	 * to the key table.
 	 */
+	rte_rwlock_t *readwrite_lock; /**< Read-write lock thread-safety. */
 } __rte_cache_aligned;
 
 struct queue_node {
diff --git a/lib/librte_hash/rte_cuckoo_hash_x86.h b/lib/librte_hash/rte_cuckoo_hash_x86.h
deleted file mode 100644
index 2c5b017e..00000000
--- a/lib/librte_hash/rte_cuckoo_hash_x86.h
+++ /dev/null
@@ -1,164 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2016 Intel Corporation
- */
-
-/* rte_cuckoo_hash_x86.h
- * This file holds all x86 specific Cuckoo Hash functions
- */
-
-/* Only tries to insert at one bucket (@prim_bkt) without trying to push
- * buckets around
- */
-static inline unsigned
-rte_hash_cuckoo_insert_mw_tm(struct rte_hash_bucket *prim_bkt,
-		hash_sig_t sig, hash_sig_t alt_hash, uint32_t new_idx)
-{
-	unsigned i, status;
-	unsigned try = 0;
-
-	while (try < RTE_HASH_TSX_MAX_RETRY) {
-		status = rte_xbegin();
-		if (likely(status == RTE_XBEGIN_STARTED)) {
-			/* Insert new entry if there is room in the primary
-			* bucket.
-			*/
-			for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
-				/* Check if slot is available */
-				if (likely(prim_bkt->key_idx[i] == EMPTY_SLOT)) {
-					prim_bkt->sig_current[i] = sig;
-					prim_bkt->sig_alt[i] = alt_hash;
-					prim_bkt->key_idx[i] = new_idx;
-					break;
-				}
-			}
-			rte_xend();
-
-			if (i != RTE_HASH_BUCKET_ENTRIES)
-				return 0;
-
-			break; /* break off try loop if transaction commits */
-		} else {
-			/* If we abort we give up this cuckoo path. */
-			try++;
-			rte_pause();
-		}
-	}
-
-	return -1;
-}
-
-/* Shift buckets along provided cuckoo_path (@leaf and @leaf_slot) and fill
- * the path head with new entry (sig, alt_hash, new_idx)
- */
-static inline int
-rte_hash_cuckoo_move_insert_mw_tm(const struct rte_hash *h,
-			struct queue_node *leaf, uint32_t leaf_slot,
-			hash_sig_t sig, hash_sig_t alt_hash, uint32_t new_idx)
-{
-	unsigned try = 0;
-	unsigned status;
-	uint32_t prev_alt_bkt_idx;
-
-	struct queue_node *prev_node, *curr_node = leaf;
-	struct rte_hash_bucket *prev_bkt, *curr_bkt = leaf->bkt;
-	uint32_t prev_slot, curr_slot = leaf_slot;
-
-	while (try < RTE_HASH_TSX_MAX_RETRY) {
-		status = rte_xbegin();
-		if (likely(status == RTE_XBEGIN_STARTED)) {
-			while (likely(curr_node->prev != NULL)) {
-				prev_node = curr_node->prev;
-				prev_bkt = prev_node->bkt;
-				prev_slot = curr_node->prev_slot;
-
-				prev_alt_bkt_idx
-					= prev_bkt->sig_alt[prev_slot]
-					    & h->bucket_bitmask;
-
-				if (unlikely(&h->buckets[prev_alt_bkt_idx]
-					     != curr_bkt)) {
-					rte_xabort(RTE_XABORT_CUCKOO_PATH_INVALIDED);
-				}
-
-				/* Need to swap current/alt sig to allow later
-				 * Cuckoo insert to move elements back to its
-				 * primary bucket if available
-				 */
-				curr_bkt->sig_alt[curr_slot] =
-				    prev_bkt->sig_current[prev_slot];
-				curr_bkt->sig_current[curr_slot] =
-				    prev_bkt->sig_alt[prev_slot];
-				curr_bkt->key_idx[curr_slot]
-				    = prev_bkt->key_idx[prev_slot];
-
-				curr_slot = prev_slot;
-				curr_node = prev_node;
-				curr_bkt = curr_node->bkt;
-			}
-
-			curr_bkt->sig_current[curr_slot] = sig;
-			curr_bkt->sig_alt[curr_slot] = alt_hash;
-			curr_bkt->key_idx[curr_slot] = new_idx;
-
-			rte_xend();
-
-			return 0;
-		}
-
-		/* If we abort we give up this cuckoo path, since most likely it's
-		 * no longer valid as TSX detected data conflict
-		 */
-		try++;
-		rte_pause();
-	}
-
-	return -1;
-}
-
-/*
- * Make space for new key, using bfs Cuckoo Search and Multi-Writer safe
- * Cuckoo
- */
-static inline int
-rte_hash_cuckoo_make_space_mw_tm(const struct rte_hash *h,
-			struct rte_hash_bucket *bkt,
-			hash_sig_t sig, hash_sig_t alt_hash,
-			uint32_t new_idx)
-{
-	unsigned i;
-	struct queue_node queue[RTE_HASH_BFS_QUEUE_MAX_LEN];
-	struct queue_node *tail, *head;
-	struct rte_hash_bucket *curr_bkt, *alt_bkt;
-
-	tail = queue;
-	head = queue + 1;
-	tail->bkt = bkt;
-	tail->prev = NULL;
-	tail->prev_slot = -1;
-
-	/* Cuckoo bfs Search */
-	while (likely(tail != head && head <
-					queue + RTE_HASH_BFS_QUEUE_MAX_LEN -
-					RTE_HASH_BUCKET_ENTRIES)) {
-		curr_bkt = tail->bkt;
-		for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
-			if (curr_bkt->key_idx[i] == EMPTY_SLOT) {
-				if (likely(rte_hash_cuckoo_move_insert_mw_tm(h,
-						tail, i, sig,
-						alt_hash, new_idx) == 0))
-					return 0;
-			}
-
-			/* Enqueue new node and keep prev node info */
-			alt_bkt = &(h->buckets[curr_bkt->sig_alt[i]
-						    & h->bucket_bitmask]);
-			head->bkt = alt_bkt;
-			head->prev = tail;
-			head->prev_slot = i;
-			head++;
-		}
-		tail++;
-	}
-
-	return -ENOSPC;
-}
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index f71ca9fb..9e7d9315 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -34,6 +34,9 @@ extern "C" {
 /** Default behavior of insertion, single writer/multi writer */
 #define RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD 0x02
 
+/** Flag to support reader writer concurrency */
+#define RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY 0x04
+
 /** Signature of key that is stored internally. */
 typedef uint32_t hash_sig_t;
 
@@ -124,9 +127,22 @@ void
 rte_hash_reset(struct rte_hash *h);
 
 /**
+ * Return the number of keys in the hash table
+ * @param h
+ *  Hash table to query from
+ * @return
+ *   - -EINVAL if parameters are invalid
+ *   - A value indicating how many keys were inserted in the table.
+ */
+int32_t
+rte_hash_count(const struct rte_hash *h);
+
+/**
  * Add a key-value pair to an existing hash table.
  * This operation is not multi-thread safe
- * and should only be called from one thread.
+ * and should only be called from one thread by default.
+ * Thread safety can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to add the key to.
@@ -146,7 +162,9 @@ rte_hash_add_key_data(const struct rte_hash *h, const void *key, void *data);
  * Add a key-value pair with a pre-computed hash value
  * to an existing hash table.
  * This operation is not multi-thread safe
- * and should only be called from one thread.
+ * and should only be called from one thread by default.
+ * Thread safety can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to add the key to.
@@ -167,7 +185,9 @@ rte_hash_add_key_with_hash_data(const struct rte_hash *h, const void *key,
 
 /**
  * Add a key to an existing hash table. This operation is not multi-thread safe
- * and should only be called from one thread.
+ * and should only be called from one thread by default.
+ * Thread safety can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to add the key to.
@@ -185,7 +205,9 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);
 /**
  * Add a key to an existing hash table.
  * This operation is not multi-thread safe
- * and should only be called from one thread.
+ * and should only be called from one thread by default.
+ * Thread safety can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to add the key to.
@@ -205,7 +227,9 @@ rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t
 /**
  * Remove a key from an existing hash table.
  * This operation is not multi-thread safe
- * and should only be called from one thread.
+ * and should only be called from one thread by default.
+ * Thread safety can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to remove the key from.
@@ -224,7 +248,9 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
 /**
  * Remove a key from an existing hash table.
  * This operation is not multi-thread safe
- * and should only be called from one thread.
+ * and should only be called from one thread by default.
+ * Thread safety can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to remove the key from.
@@ -244,7 +270,9 @@ rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t
 
 /**
  * Find a key in the hash table given the position.
- * This operation is multi-thread safe.
+ * This operation is multi-thread safe with regarding to other lookup threads.
+ * Read-write concurrency can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to get the key from.
@@ -254,8 +282,8 @@ rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t
  *   Output containing a pointer to the key
  * @return
  *   - 0 if retrieved successfully
- *   - EINVAL if the parameters are invalid.
- *   - ENOENT if no valid key is found in the given position.
+ *   - -EINVAL if the parameters are invalid.
+ *   - -ENOENT if no valid key is found in the given position.
  */
 int
 rte_hash_get_key_with_position(const struct rte_hash *h, const int32_t position,
@@ -263,7 +291,9 @@ rte_hash_get_key_with_position(const struct rte_hash *h, const int32_t position,
 
 /**
  * Find a key-value pair in the hash table.
- * This operation is multi-thread safe.
+ * This operation is multi-thread safe with regarding to other lookup threads.
+ * Read-write concurrency can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to look in.
@@ -272,9 +302,11 @@ rte_hash_get_key_with_position(const struct rte_hash *h, const int32_t position,
  * @param data
  *   Output with pointer to data returned from the hash table.
  * @return
- *   0 if successful lookup
- *   - EINVAL if the parameters are invalid.
- *   - ENOENT if the key is not found.
+ *   - A positive value that can be used by the caller as an offset into an
+ *     array of user data. This value is unique for this key, and is the same
+ *     value that was returned when the key was added.
+ *   - -EINVAL if the parameters are invalid.
+ *   - -ENOENT if the key is not found.
  */
 int
 rte_hash_lookup_data(const struct rte_hash *h, const void *key, void **data);
@@ -282,7 +314,9 @@ rte_hash_lookup_data(const struct rte_hash *h, const void *key, void **data);
 /**
  * Find a key-value pair with a pre-computed hash value
  * to an existing hash table.
- * This operation is multi-thread safe.
+ * This operation is multi-thread safe with regarding to other lookup threads.
+ * Read-write concurrency can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to look in.
@@ -293,9 +327,11 @@ rte_hash_lookup_data(const struct rte_hash *h, const void *key, void **data);
  * @param data
  *   Output with pointer to data returned from the hash table.
  * @return
- *   0 if successful lookup
- *   - EINVAL if the parameters are invalid.
- *   - ENOENT if the key is not found.
+ *   - A positive value that can be used by the caller as an offset into an
+ *     array of user data. This value is unique for this key, and is the same
+ *     value that was returned when the key was added.
+ *   - -EINVAL if the parameters are invalid.
+ *   - -ENOENT if the key is not found.
  */
 int
 rte_hash_lookup_with_hash_data(const struct rte_hash *h, const void *key,
@@ -303,7 +339,9 @@ rte_hash_lookup_with_hash_data(const struct rte_hash *h, const void *key,
 
 /**
  * Find a key in the hash table.
- * This operation is multi-thread safe.
+ * This operation is multi-thread safe with regarding to other lookup threads.
+ * Read-write concurrency can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to look in.
@@ -321,7 +359,9 @@ rte_hash_lookup(const struct rte_hash *h, const void *key);
 
 /**
  * Find a key in the hash table.
- * This operation is multi-thread safe.
+ * This operation is multi-thread safe with regarding to other lookup threads.
+ * Read-write concurrency can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to look in.
@@ -356,7 +396,9 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
 
 /**
  * Find multiple keys in the hash table.
- * This operation is multi-thread safe.
+ * This operation is multi-thread safe with regarding to other lookup threads.
+ * Read-write concurrency can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to look in.
@@ -377,7 +419,9 @@ rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
 
 /**
  * Find multiple keys in the hash table.
- * This operation is multi-thread safe.
+ * This operation is multi-thread safe with regarding to other lookup threads.
+ * Read-write concurrency can be enabled by setting flag during
+ * table creation.
  *
  * @param h
  *   Hash table to look in.
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index 52a2576f..e216ac8e 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -45,3 +45,11 @@ DPDK_16.07 {
 	rte_hash_get_key_with_position;
 
 } DPDK_2.2;
+
+
+DPDK_18.08 {
+	global:
+
+	rte_hash_count;
+
+} DPDK_16.07;
diff --git a/lib/librte_kni/meson.build b/lib/librte_kni/meson.build
index c4b21961..a738a033 100644
--- a/lib/librte_kni/meson.build
+++ b/lib/librte_kni/meson.build
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-if host_machine.system() != 'linux'
+if host_machine.system() != 'linux' or cc.sizeof('void *') == 4
 	build = false
 endif
 version = 2
diff --git a/lib/librte_kni/rte_kni.c b/lib/librte_kni/rte_kni.c
index 8a8f6c1c..65f6a2b0 100644
--- a/lib/librte_kni/rte_kni.c
+++ b/lib/librte_kni/rte_kni.c
@@ -715,6 +715,9 @@ rte_kni_get(const char *name)
 	struct rte_kni_memzone_slot *it;
 	struct rte_kni *kni;
 
+	if (name == NULL || name[0] == '\0')
+		return NULL;
+
 	/* Note: could be improved perf-wise if necessary */
 	for (i = 0; i < kni_memzone_pool.max_ifaces; i++) {
 		it = &kni_memzone_pool.slots[i];
diff --git a/lib/librte_kvargs/Makefile b/lib/librte_kvargs/Makefile
index 39d5ac33..87593954 100644
--- a/lib/librte_kvargs/Makefile
+++ b/lib/librte_kvargs/Makefile
@@ -7,7 +7,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 LIB = librte_kvargs.a
 
 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3
-LDLIBS += -lrte_eal
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include
 
 EXPORT_MAP := rte_kvargs_version.map
 
diff --git a/lib/librte_kvargs/meson.build b/lib/librte_kvargs/meson.build
index 0c5b9cb2..acd3e543 100644
--- a/lib/librte_kvargs/meson.build
+++ b/lib/librte_kvargs/meson.build
@@ -1,6 +1,11 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
+includes = [global_inc]
+includes += include_directories('../librte_eal/common/include')
+
 version = 1
 sources = files('rte_kvargs.c')
 headers = files('rte_kvargs.h')
+
+deps += 'compat'
diff --git a/lib/librte_kvargs/rte_kvargs.c b/lib/librte_kvargs/rte_kvargs.c
index d92a5f9d..a28f7694 100644
--- a/lib/librte_kvargs/rte_kvargs.c
+++ b/lib/librte_kvargs/rte_kvargs.c
@@ -6,7 +6,6 @@
 #include <string.h>
 #include <stdlib.h>
 
-#include <rte_log.h>
 #include <rte_string_fns.h>
 
 #include "rte_kvargs.h"
@@ -28,29 +27,22 @@ rte_kvargs_tokenize(struct rte_kvargs *kvlist, const char *params)
 	 * to pass to rte_strsplit
 	 */
 	kvlist->str = strdup(params);
-	if (kvlist->str == NULL) {
-		RTE_LOG(ERR, PMD, "Cannot parse arguments: not enough memory\n");
+	if (kvlist->str == NULL)
 		return -1;
-	}
 
 	/* browse each key/value pair and add it in kvlist */
 	str = kvlist->str;
 	while ((str = strtok_r(str, RTE_KVARGS_PAIRS_DELIM, &ctx1)) != NULL) {
 
 		i = kvlist->count;
-		if (i >= RTE_KVARGS_MAX) {
-			RTE_LOG(ERR, PMD, "Cannot parse arguments: list full\n");
+		if (i >= RTE_KVARGS_MAX)
 			return -1;
-		}
 
 		kvlist->pairs[i].key = strtok_r(str, RTE_KVARGS_KV_DELIM, &ctx2);
 		kvlist->pairs[i].value = strtok_r(NULL, RTE_KVARGS_KV_DELIM, &ctx2);
-		if (kvlist->pairs[i].key == NULL || kvlist->pairs[i].value == NULL) {
-			RTE_LOG(ERR, PMD,
-				"Cannot parse arguments: wrong key or value\n"
-				"params=<%s>\n", params);
+		if (kvlist->pairs[i].key == NULL ||
+		    kvlist->pairs[i].value == NULL)
 			return -1;
-		}
 
 		kvlist->count++;
 		str = NULL;
@@ -89,12 +81,8 @@ check_for_valid_keys(struct rte_kvargs *kvlist,
 	for (i = 0; i < kvlist->count; i++) {
 		pair = &kvlist->pairs[i];
 		ret = is_valid_key(valid, pair->key);
-		if (!ret) {
-			RTE_LOG(ERR, PMD,
-				"Error parsing device, invalid key <%s>\n",
-				pair->key);
+		if (!ret)
 			return -1;
-		}
 	}
 	return 0;
 }
@@ -180,3 +168,38 @@ rte_kvargs_parse(const char *args, const char * const valid_keys[])
 
 	return kvlist;
 }
+
+__rte_experimental
+struct rte_kvargs *
+rte_kvargs_parse_delim(const char *args, const char * const valid_keys[],
+		       const char *valid_ends)
+{
+	struct rte_kvargs *kvlist = NULL;
+	char *copy;
+	size_t len;
+
+	if (valid_ends == NULL)
+		return rte_kvargs_parse(args, valid_keys);
+
+	copy = strdup(args);
+	if (copy == NULL)
+		return NULL;
+
+	len = strcspn(copy, valid_ends);
+	copy[len] = '\0';
+
+	kvlist = rte_kvargs_parse(copy, valid_keys);
+
+	free(copy);
+	return kvlist;
+}
+
+__rte_experimental
+int
+rte_kvargs_strcmp(const char *key __rte_unused,
+		  const char *value, void *opaque)
+{
+	const char *str = opaque;
+
+	return -abs(strcmp(str, value));
+}
diff --git a/lib/librte_kvargs/rte_kvargs.h b/lib/librte_kvargs/rte_kvargs.h
index 51b8120b..fc041956 100644
--- a/lib/librte_kvargs/rte_kvargs.h
+++ b/lib/librte_kvargs/rte_kvargs.h
@@ -25,6 +25,8 @@
 extern "C" {
 #endif
 
+#include <rte_compat.h>
+
 /** Maximum number of key/value associations */
 #define RTE_KVARGS_MAX 32
 
@@ -72,6 +74,36 @@ struct rte_kvargs *rte_kvargs_parse(const char *args,
 		const char *const valid_keys[]);
 
 /**
+ * Allocate a rte_kvargs and store key/value associations from a string.
+ * This version will consider any byte from valid_ends as a possible
+ * terminating character, and will not parse beyond any of their occurrence.
+ *
+ * The function allocates and fills an rte_kvargs structure from a given
+ * string whose format is key1=value1,key2=value2,...
+ *
+ * The structure can be freed with rte_kvargs_free().
+ *
+ * @param args
+ *   The input string containing the key/value associations
+ *
+ * @param valid_keys
+ *   A list of valid keys (table of const char *, the last must be NULL).
+ *   This argument is ignored if NULL
+ *
+ * @param valid_ends
+ *   Acceptable terminating characters.
+ *   If NULL, the behavior is the same as ``rte_kvargs_parse``.
+ *
+ * @return
+ *   - A pointer to an allocated rte_kvargs structure on success
+ *   - NULL on error
+ */
+__rte_experimental
+struct rte_kvargs *rte_kvargs_parse_delim(const char *args,
+		const char *const valid_keys[],
+		const char *valid_ends);
+
+/**
  * Free a rte_kvargs structure
  *
  * Free a rte_kvargs structure previously allocated with
@@ -121,6 +153,32 @@ int rte_kvargs_process(const struct rte_kvargs *kvlist,
 unsigned rte_kvargs_count(const struct rte_kvargs *kvlist,
 	const char *key_match);
 
+/**
+ * Generic kvarg handler for string comparison.
+ *
+ * This function can be used for a generic string comparison processing
+ * on a list of kvargs.
+ *
+ * @param key
+ *   kvarg pair key.
+ *
+ * @param value
+ *   kvarg pair value.
+ *
+ * @param opaque
+ *   Opaque pointer to a string.
+ *
+ * @return
+ *   0 if the strings match.
+ *   !0 otherwise or on error.
+ *
+ *   Unless strcmp, comparison ordering is not kept.
+ *   In order for rte_kvargs_process to stop processing on match error,
+ *   a negative value is returned even if strcmp had returned a positive one.
+ */
+__rte_experimental
+int rte_kvargs_strcmp(const char *key, const char *value, void *opaque);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_kvargs/rte_kvargs_version.map b/lib/librte_kvargs/rte_kvargs_version.map
index 2030ec46..8f4b4e3f 100644
--- a/lib/librte_kvargs/rte_kvargs_version.map
+++ b/lib/librte_kvargs/rte_kvargs_version.map
@@ -8,3 +8,11 @@ DPDK_2.0 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_kvargs_parse_delim;
+	rte_kvargs_strcmp;
+
+} DPDK_2.0;
diff --git a/lib/librte_latencystats/rte_latencystats.c b/lib/librte_latencystats/rte_latencystats.c
index 46c69bf0..1fdec68e 100644
--- a/lib/librte_latencystats/rte_latencystats.c
+++ b/lib/librte_latencystats/rte_latencystats.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017 Intel Corporation
+ * Copyright(c) 2018 Intel Corporation
  */
 
 #include <unistd.h>
@@ -265,6 +265,7 @@ rte_latencystats_uninit(void)
 	uint16_t qid;
 	int ret = 0;
 	struct rxtx_cbs *cbs = NULL;
+	const struct rte_memzone *mz = NULL;
 
 	/** De register Rx/Tx callbacks */
 	RTE_ETH_FOREACH_DEV(pid) {
@@ -288,6 +289,11 @@ rte_latencystats_uninit(void)
 		}
 	}
 
+	/* free up the memzone */
+	mz = rte_memzone_lookup(MZ_RTE_LATENCY_STATS);
+	if (mz)
+		rte_memzone_free(mz);
+
 	return 0;
 }
 
diff --git a/lib/librte_mbuf/Makefile b/lib/librte_mbuf/Makefile
index 8749a00f..e2b98a25 100644
--- a/lib/librte_mbuf/Makefile
+++ b/lib/librte_mbuf/Makefile
@@ -6,7 +6,6 @@ include $(RTE_SDK)/mk/rte.vars.mk
 # library name
 LIB = librte_mbuf.a
 
-CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3
 LDLIBS += -lrte_eal -lrte_mempool
 
diff --git a/lib/librte_mbuf/meson.build b/lib/librte_mbuf/meson.build
index 869c17c1..45ffb0db 100644
--- a/lib/librte_mbuf/meson.build
+++ b/lib/librte_mbuf/meson.build
@@ -2,7 +2,6 @@
 # Copyright(c) 2017 Intel Corporation
 
 version = 3
-allow_experimental_apis = true
 sources = files('rte_mbuf.c', 'rte_mbuf_ptype.c', 'rte_mbuf_pool_ops.c')
 headers = files('rte_mbuf.h', 'rte_mbuf_ptype.h', 'rte_mbuf_pool_ops.h')
 deps += ['mempool']
diff --git a/lib/librte_mbuf/rte_mbuf.c b/lib/librte_mbuf/rte_mbuf.c
index fca580ef..e714c5a5 100644
--- a/lib/librte_mbuf/rte_mbuf.c
+++ b/lib/librte_mbuf/rte_mbuf.c
@@ -107,7 +107,7 @@ rte_pktmbuf_init(struct rte_mempool *mp,
 }
 
 /* Helper to create a mbuf pool with given mempool ops name*/
-struct rte_mempool * __rte_experimental
+struct rte_mempool *
 rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
 	unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
 	int socket_id, const char *ops_name)
diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h
index 8e6b4d29..9ce5d76d 100644
--- a/lib/librte_mbuf/rte_mbuf.h
+++ b/lib/librte_mbuf/rte_mbuf.h
@@ -729,6 +729,24 @@ rte_mbuf_to_baddr(struct rte_mbuf *md)
 }
 
 /**
+ * Return the starting address of the private data area embedded in
+ * the given mbuf.
+ *
+ * Note that no check is made to ensure that a private data area
+ * actually exists in the supplied mbuf.
+ *
+ * @param m
+ *   The pointer to the mbuf.
+ * @return
+ *   The starting address of the private data area of the given mbuf.
+ */
+static inline void * __rte_experimental
+rte_mbuf_to_priv(struct rte_mbuf *m)
+{
+	return RTE_PTR_ADD(m, sizeof(struct rte_mbuf));
+}
+
+/**
  * Returns TRUE if given mbuf is cloned by mbuf indirection, or FALSE
  * otherwise.
  *
@@ -1146,7 +1164,7 @@ rte_pktmbuf_pool_create(const char *name, unsigned n,
  *    - EEXIST - a memzone with the same name already exists
  *    - ENOMEM - no appropriate memory area found in which to create memzone
  */
-struct rte_mempool * __rte_experimental
+struct rte_mempool *
 rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n,
 	unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
 	int socket_id, const char *ops_name);
diff --git a/lib/librte_mbuf/rte_mbuf_pool_ops.c b/lib/librte_mbuf/rte_mbuf_pool_ops.c
index a1d4699f..5722976f 100644
--- a/lib/librte_mbuf/rte_mbuf_pool_ops.c
+++ b/lib/librte_mbuf/rte_mbuf_pool_ops.c
@@ -9,7 +9,7 @@
 #include <rte_errno.h>
 #include <rte_mbuf_pool_ops.h>
 
-int __rte_experimental
+int
 rte_mbuf_set_platform_mempool_ops(const char *ops_name)
 {
 	const struct rte_memzone *mz;
@@ -35,7 +35,7 @@ rte_mbuf_set_platform_mempool_ops(const char *ops_name)
 	return -EEXIST;
 }
 
-const char * __rte_experimental
+const char *
 rte_mbuf_platform_mempool_ops(void)
 {
 	const struct rte_memzone *mz;
@@ -46,7 +46,7 @@ rte_mbuf_platform_mempool_ops(void)
 	return mz->addr;
 }
 
-int __rte_experimental
+int
 rte_mbuf_set_user_mempool_ops(const char *ops_name)
 {
 	const struct rte_memzone *mz;
@@ -67,7 +67,7 @@ rte_mbuf_set_user_mempool_ops(const char *ops_name)
 
 }
 
-const char * __rte_experimental
+const char *
 rte_mbuf_user_mempool_ops(void)
 {
 	const struct rte_memzone *mz;
@@ -79,7 +79,7 @@ rte_mbuf_user_mempool_ops(void)
 }
 
 /* Return mbuf pool ops name */
-const char * __rte_experimental
+const char *
 rte_mbuf_best_mempool_ops(void)
 {
 	/* User defined mempool ops takes the priority */
diff --git a/lib/librte_mbuf/rte_mbuf_pool_ops.h b/lib/librte_mbuf/rte_mbuf_pool_ops.h
index ebf5bf0f..7ed95a49 100644
--- a/lib/librte_mbuf/rte_mbuf_pool_ops.h
+++ b/lib/librte_mbuf/rte_mbuf_pool_ops.h
@@ -12,9 +12,6 @@
  * These APIs are for configuring the mbuf pool ops names to be largely used by
  * rte_pktmbuf_pool_create(). However, this can also be used to set and inquire
  * the best mempool ops available.
- *
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice
  */
 
 #include <rte_compat.h>
@@ -34,7 +31,7 @@ extern "C" {
  *   - On success, zero.
  *   - On failure, a negative value.
  */
-int __rte_experimental
+int
 rte_mbuf_set_platform_mempool_ops(const char *ops_name);
 
 /**
@@ -46,7 +43,7 @@ rte_mbuf_set_platform_mempool_ops(const char *ops_name);
  *   - On success, platform pool ops name.
  *   - On failure, NULL.
  */
-const char * __rte_experimental
+const char *
 rte_mbuf_platform_mempool_ops(void);
 
 /**
@@ -60,7 +57,7 @@ rte_mbuf_platform_mempool_ops(void);
  *   - On success, zero.
  *   - On failure, a negative value.
  */
-int __rte_experimental
+int
 rte_mbuf_set_user_mempool_ops(const char *ops_name);
 
 /**
@@ -72,7 +69,7 @@ rte_mbuf_set_user_mempool_ops(const char *ops_name);
  *   - On success, user pool ops name..
  *   - On failure, NULL.
  */
-const char * __rte_experimental
+const char *
 rte_mbuf_user_mempool_ops(void);
 
 /**
@@ -87,7 +84,7 @@ rte_mbuf_user_mempool_ops(void);
  * @return
  *   returns preferred mbuf pool ops name
  */
-const char * __rte_experimental
+const char *
 rte_mbuf_best_mempool_ops(void);
 
 
diff --git a/lib/librte_mbuf/rte_mbuf_ptype.h b/lib/librte_mbuf/rte_mbuf_ptype.h
index 79ea3142..01acc66e 100644
--- a/lib/librte_mbuf/rte_mbuf_ptype.h
+++ b/lib/librte_mbuf/rte_mbuf_ptype.h
@@ -653,9 +653,9 @@ extern "C" {
 #define  RTE_ETH_IS_IPV4_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV4)
 
 /**
- * Check if the (outer) L3 header is IPv4. To avoid comparing IPv4 types one by
- * one, bit 6 is selected to be used for IPv4 only. Then checking bit 6 can
- * determine if it is an IPV4 packet.
+ * Check if the (outer) L3 header is IPv6. To avoid comparing IPv6 types one by
+ * one, bit 6 is selected to be used for IPv6 only. Then checking bit 6 can
+ * determine if it is an IPV6 packet.
  */
 #define  RTE_ETH_IS_IPV6_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV6)
 
diff --git a/lib/librte_mbuf/rte_mbuf_version.map b/lib/librte_mbuf/rte_mbuf_version.map
index 1bb9538d..cae68db8 100644
--- a/lib/librte_mbuf/rte_mbuf_version.map
+++ b/lib/librte_mbuf/rte_mbuf_version.map
@@ -35,7 +35,7 @@ DPDK_16.11 {
 
 } DPDK_2.1;
 
-EXPERIMENTAL {
+DPDK_18.08 {
 	global:
 
 	rte_mbuf_best_mempool_ops;
@@ -44,4 +44,4 @@ EXPERIMENTAL {
 	rte_mbuf_set_user_mempool_ops;
 	rte_mbuf_user_mempool_ops;
 	rte_pktmbuf_pool_create_by_ops;
-};
+} DPDK_16.11;
diff --git a/lib/librte_member/rte_member.c b/lib/librte_member/rte_member.c
index e147dd1f..702c01d3 100644
--- a/lib/librte_member/rte_member.c
+++ b/lib/librte_member/rte_member.c
@@ -297,10 +297,7 @@ rte_member_reset(const struct rte_member_setsum *setsum)
 	}
 }
 
-RTE_INIT(librte_member_init_log);
-
-static void
-librte_member_init_log(void)
+RTE_INIT(librte_member_init_log)
 {
 	librte_member_logtype = rte_log_register("lib.member");
 	if (librte_member_logtype >= 0)
diff --git a/lib/librte_mempool/Makefile b/lib/librte_mempool/Makefile
index e3c32b14..20bf63fb 100644
--- a/lib/librte_mempool/Makefile
+++ b/lib/librte_mempool/Makefile
@@ -7,15 +7,12 @@ include $(RTE_SDK)/mk/rte.vars.mk
 LIB = librte_mempool.a
 
 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3
-# Allow deprecated symbol to use deprecated rte_mempool_populate_iova_tab()
-# from earlier deprecated rte_mempool_populate_phys_tab()
-CFLAGS += -Wno-deprecated-declarations
 CFLAGS += -DALLOW_EXPERIMENTAL_API
 LDLIBS += -lrte_eal -lrte_ring
 
 EXPORT_MAP := rte_mempool_version.map
 
-LIBABIVER := 4
+LIBABIVER := 5
 
 # memseg walk is not yet part of stable API
 CFLAGS += -DALLOW_EXPERIMENTAL_API
diff --git a/lib/librte_mempool/meson.build b/lib/librte_mempool/meson.build
index d507e551..38d7ae89 100644
--- a/lib/librte_mempool/meson.build
+++ b/lib/librte_mempool/meson.build
@@ -5,17 +5,13 @@ allow_experimental_apis = true
 
 extra_flags = []
 
-# Allow deprecated symbol to use deprecated rte_mempool_populate_iova_tab()
-# from earlier deprecated rte_mempool_populate_phys_tab()
-extra_flags += '-Wno-deprecated-declarations'
-
 foreach flag: extra_flags
 	if cc.has_argument(flag)
 		cflags += flag
 	endif
 endforeach
 
-version = 4
+version = 5
 sources = files('rte_mempool.c', 'rte_mempool_ops.c',
 		'rte_mempool_ops_default.c')
 headers = files('rte_mempool.h')
diff --git a/lib/librte_mempool/rte_mempool.c b/lib/librte_mempool/rte_mempool.c
index 8c8b9f80..03e6b5f7 100644
--- a/lib/librte_mempool/rte_mempool.c
+++ b/lib/librte_mempool/rte_mempool.c
@@ -225,93 +225,6 @@ rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
 	return sz->total_size;
 }
 
-
-/*
- * Internal function to calculate required memory chunk size shared
- * by default implementation of the corresponding callback and
- * deprecated external function.
- */
-size_t
-rte_mempool_calc_mem_size_helper(uint32_t elt_num, size_t total_elt_sz,
-				 uint32_t pg_shift)
-{
-	size_t obj_per_page, pg_num, pg_sz;
-
-	if (total_elt_sz == 0)
-		return 0;
-
-	if (pg_shift == 0)
-		return total_elt_sz * elt_num;
-
-	pg_sz = (size_t)1 << pg_shift;
-	obj_per_page = pg_sz / total_elt_sz;
-	if (obj_per_page == 0)
-		return RTE_ALIGN_CEIL(total_elt_sz, pg_sz) * elt_num;
-
-	pg_num = (elt_num + obj_per_page - 1) / obj_per_page;
-	return pg_num << pg_shift;
-}
-
-/*
- * Calculate maximum amount of memory required to store given number of objects.
- */
-size_t
-rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift,
-		      __rte_unused unsigned int flags)
-{
-	return rte_mempool_calc_mem_size_helper(elt_num, total_elt_sz,
-						pg_shift);
-}
-
-/*
- * Calculate how much memory would be actually required with the
- * given memory footprint to store required number of elements.
- */
-ssize_t
-rte_mempool_xmem_usage(__rte_unused void *vaddr, uint32_t elt_num,
-	size_t total_elt_sz, const rte_iova_t iova[], uint32_t pg_num,
-	uint32_t pg_shift, __rte_unused unsigned int flags)
-{
-	uint32_t elt_cnt = 0;
-	rte_iova_t start, end;
-	uint32_t iova_idx;
-	size_t pg_sz = (size_t)1 << pg_shift;
-
-	/* if iova is NULL, assume contiguous memory */
-	if (iova == NULL) {
-		start = 0;
-		end = pg_sz * pg_num;
-		iova_idx = pg_num;
-	} else {
-		start = iova[0];
-		end = iova[0] + pg_sz;
-		iova_idx = 1;
-	}
-	while (elt_cnt < elt_num) {
-
-		if (end - start >= total_elt_sz) {
-			/* enough contiguous memory, add an object */
-			start += total_elt_sz;
-			elt_cnt++;
-		} else if (iova_idx < pg_num) {
-			/* no room to store one obj, add a page */
-			if (end == iova[iova_idx]) {
-				end += pg_sz;
-			} else {
-				start = iova[iova_idx];
-				end = iova[iova_idx] + pg_sz;
-			}
-			iova_idx++;
-
-		} else {
-			/* no more page, return how many elements fit */
-			return -(size_t)elt_cnt;
-		}
-	}
-
-	return (size_t)iova_idx << pg_shift;
-}
-
 /* free a memchunk allocated with rte_memzone_reserve() */
 static void
 rte_mempool_memchunk_mz_free(__rte_unused struct rte_mempool_memhdr *memhdr,
@@ -423,63 +336,6 @@ fail:
 	return ret;
 }
 
-int
-rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr,
-	phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
-	void *opaque)
-{
-	return rte_mempool_populate_iova(mp, vaddr, paddr, len, free_cb, opaque);
-}
-
-/* Add objects in the pool, using a table of physical pages. Return the
- * number of objects added, or a negative value on error.
- */
-int
-rte_mempool_populate_iova_tab(struct rte_mempool *mp, char *vaddr,
-	const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift,
-	rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
-{
-	uint32_t i, n;
-	int ret, cnt = 0;
-	size_t pg_sz = (size_t)1 << pg_shift;
-
-	/* mempool must not be populated */
-	if (mp->nb_mem_chunks != 0)
-		return -EEXIST;
-
-	if (mp->flags & MEMPOOL_F_NO_IOVA_CONTIG)
-		return rte_mempool_populate_iova(mp, vaddr, RTE_BAD_IOVA,
-			pg_num * pg_sz, free_cb, opaque);
-
-	for (i = 0; i < pg_num && mp->populated_size < mp->size; i += n) {
-
-		/* populate with the largest group of contiguous pages */
-		for (n = 1; (i + n) < pg_num &&
-			     iova[i + n - 1] + pg_sz == iova[i + n]; n++)
-			;
-
-		ret = rte_mempool_populate_iova(mp, vaddr + i * pg_sz,
-			iova[i], n * pg_sz, free_cb, opaque);
-		if (ret < 0) {
-			rte_mempool_free_memchunks(mp);
-			return ret;
-		}
-		/* no need to call the free callback for next chunks */
-		free_cb = NULL;
-		cnt += ret;
-	}
-	return cnt;
-}
-
-int
-rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr,
-	const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift,
-	rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
-{
-	return rte_mempool_populate_iova_tab(mp, vaddr, paddr, pg_num, pg_shift,
-			free_cb, opaque);
-}
-
 /* Populate the mempool with a virtual area. Return the number of
  * objects added, or a negative value on error.
  */
@@ -916,6 +772,12 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
 
 	mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list);
 
+	/* asked for zero items */
+	if (n == 0) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+
 	/* asked cache too big */
 	if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE ||
 	    CALC_CACHE_FLUSHTHRESH(cache_size) > n) {
@@ -1065,66 +927,6 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
 	return NULL;
 }
 
-/*
- * Create the mempool over already allocated chunk of memory.
- * That external memory buffer can consists of physically disjoint pages.
- * Setting vaddr to NULL, makes mempool to fallback to rte_mempool_create()
- * behavior.
- */
-struct rte_mempool *
-rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size,
-		unsigned cache_size, unsigned private_data_size,
-		rte_mempool_ctor_t *mp_init, void *mp_init_arg,
-		rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
-		int socket_id, unsigned flags, void *vaddr,
-		const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift)
-{
-	struct rte_mempool *mp = NULL;
-	int ret;
-
-	/* no virtual address supplied, use rte_mempool_create() */
-	if (vaddr == NULL)
-		return rte_mempool_create(name, n, elt_size, cache_size,
-			private_data_size, mp_init, mp_init_arg,
-			obj_init, obj_init_arg, socket_id, flags);
-
-	/* check that we have both VA and PA */
-	if (iova == NULL) {
-		rte_errno = EINVAL;
-		return NULL;
-	}
-
-	/* Check that pg_shift parameter is valid. */
-	if (pg_shift > MEMPOOL_PG_SHIFT_MAX) {
-		rte_errno = EINVAL;
-		return NULL;
-	}
-
-	mp = rte_mempool_create_empty(name, n, elt_size, cache_size,
-		private_data_size, socket_id, flags);
-	if (mp == NULL)
-		return NULL;
-
-	/* call the mempool priv initializer */
-	if (mp_init)
-		mp_init(mp, mp_init_arg);
-
-	ret = rte_mempool_populate_iova_tab(mp, vaddr, iova, pg_num, pg_shift,
-		NULL, NULL);
-	if (ret < 0 || ret != (int)mp->size)
-		goto fail;
-
-	/* call the object initializers */
-	if (obj_init)
-		rte_mempool_obj_iter(mp, obj_init, obj_init_arg);
-
-	return mp;
-
- fail:
-	rte_mempool_free(mp);
-	return NULL;
-}
-
 /* Return the number of entries in the mempool */
 unsigned int
 rte_mempool_avail_count(const struct rte_mempool *mp)
diff --git a/lib/librte_mempool/rte_mempool.h b/lib/librte_mempool/rte_mempool.h
index 1f59553b..7c9cd9a2 100644
--- a/lib/librte_mempool/rte_mempool.h
+++ b/lib/librte_mempool/rte_mempool.h
@@ -488,28 +488,6 @@ ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
 		size_t *min_chunk_size, size_t *align);
 
 /**
- * @internal Helper function to calculate memory size required to store
- * specified number of objects in assumption that the memory buffer will
- * be aligned at page boundary.
- *
- * Note that if object size is bigger than page size, then it assumes
- * that pages are grouped in subsets of physically continuous pages big
- * enough to store at least one object.
- *
- * @param elt_num
- *   Number of elements.
- * @param total_elt_sz
- *   The size of each element, including header and trailer, as returned
- *   by rte_mempool_calc_obj_size().
- * @param pg_shift
- *   LOG2 of the physical pages size. If set to 0, ignore page boundaries.
- * @return
- *   Required memory size aligned at page boundary.
- */
-size_t rte_mempool_calc_mem_size_helper(uint32_t elt_num, size_t total_elt_sz,
-		uint32_t pg_shift);
-
-/**
  * Function to be called for each populated object.
  *
  * @param[in] mp
@@ -974,74 +952,6 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
 		   int socket_id, unsigned flags);
 
 /**
- * @deprecated
- * Create a new mempool named *name* in memory.
- *
- * The pool contains n elements of elt_size. Its size is set to n.
- * This function uses ``memzone_reserve()`` to allocate the mempool header
- * (and the objects if vaddr is NULL).
- * Depending on the input parameters, mempool elements can be either allocated
- * together with the mempool header, or an externally provided memory buffer
- * could be used to store mempool objects. In later case, that external
- * memory buffer can consist of set of disjoint physical pages.
- *
- * @param name
- *   The name of the mempool.
- * @param n
- *   The number of elements in the mempool. The optimum size (in terms of
- *   memory usage) for a mempool is when n is a power of two minus one:
- *   n = (2^q - 1).
- * @param elt_size
- *   The size of each element.
- * @param cache_size
- *   Size of the cache. See rte_mempool_create() for details.
- * @param private_data_size
- *   The size of the private data appended after the mempool
- *   structure. This is useful for storing some private data after the
- *   mempool structure, as is done for rte_mbuf_pool for example.
- * @param mp_init
- *   A function pointer that is called for initialization of the pool,
- *   before object initialization. The user can initialize the private
- *   data in this function if needed. This parameter can be NULL if
- *   not needed.
- * @param mp_init_arg
- *   An opaque pointer to data that can be used in the mempool
- *   constructor function.
- * @param obj_init
- *   A function called for each object at initialization of the pool.
- *   See rte_mempool_create() for details.
- * @param obj_init_arg
- *   An opaque pointer passed to the object constructor function.
- * @param socket_id
- *   The *socket_id* argument is the socket identifier in the case of
- *   NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
- *   constraint for the reserved zone.
- * @param flags
- *   Flags controlling the behavior of the mempool. See
- *   rte_mempool_create() for details.
- * @param vaddr
- *   Virtual address of the externally allocated memory buffer.
- *   Will be used to store mempool objects.
- * @param iova
- *   Array of IO addresses of the pages that comprises given memory buffer.
- * @param pg_num
- *   Number of elements in the iova array.
- * @param pg_shift
- *   LOG2 of the physical pages size.
- * @return
- *   The pointer to the new allocated mempool, on success. NULL on error
- *   with rte_errno set appropriately. See rte_mempool_create() for details.
- */
-__rte_deprecated
-struct rte_mempool *
-rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size,
-		unsigned cache_size, unsigned private_data_size,
-		rte_mempool_ctor_t *mp_init, void *mp_init_arg,
-		rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
-		int socket_id, unsigned flags, void *vaddr,
-		const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift);
-
-/**
  * Create an empty mempool
  *
  * The mempool is allocated and initialized, but it is not populated: no
@@ -1123,48 +1033,6 @@ int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
 	rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
 	void *opaque);
 
-__rte_deprecated
-int rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr,
-	phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
-	void *opaque);
-
-/**
- * @deprecated
- * Add physical memory for objects in the pool at init
- *
- * Add a virtually contiguous memory chunk in the pool where objects can
- * be instantiated. The IO addresses corresponding to the virtual
- * area are described in iova[], pg_num, pg_shift.
- *
- * @param mp
- *   A pointer to the mempool structure.
- * @param vaddr
- *   The virtual address of memory that should be used to store objects.
- * @param iova
- *   An array of IO addresses of each page composing the virtual area.
- * @param pg_num
- *   Number of elements in the iova array.
- * @param pg_shift
- *   LOG2 of the physical pages size.
- * @param free_cb
- *   The callback used to free this chunk when destroying the mempool.
- * @param opaque
- *   An opaque argument passed to free_cb.
- * @return
- *   The number of objects added on success.
- *   On error, the chunks are not added in the memory list of the
- *   mempool and a negative errno is returned.
- */
-__rte_deprecated
-int rte_mempool_populate_iova_tab(struct rte_mempool *mp, char *vaddr,
-	const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift,
-	rte_mempool_memchunk_free_cb_t *free_cb, void *opaque);
-
-__rte_deprecated
-int rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr,
-	const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift,
-	rte_mempool_memchunk_free_cb_t *free_cb, void *opaque);
-
 /**
  * Add virtually contiguous memory for objects in the pool at init
  *
@@ -1746,13 +1614,6 @@ rte_mempool_virt2iova(const void *elt)
 	return hdr->iova;
 }
 
-__rte_deprecated
-static inline phys_addr_t
-rte_mempool_virt2phy(__rte_unused const struct rte_mempool *mp, const void *elt)
-{
-	return rte_mempool_virt2iova(elt);
-}
-
 /**
  * Check the consistency of mempool objects.
  *
@@ -1822,68 +1683,6 @@ uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
 	struct rte_mempool_objsz *sz);
 
 /**
- * @deprecated
- * Get the size of memory required to store mempool elements.
- *
- * Calculate the maximum amount of memory required to store given number
- * of objects. Assume that the memory buffer will be aligned at page
- * boundary.
- *
- * Note that if object size is bigger than page size, then it assumes
- * that pages are grouped in subsets of physically continuous pages big
- * enough to store at least one object.
- *
- * @param elt_num
- *   Number of elements.
- * @param total_elt_sz
- *   The size of each element, including header and trailer, as returned
- *   by rte_mempool_calc_obj_size().
- * @param pg_shift
- *   LOG2 of the physical pages size. If set to 0, ignore page boundaries.
- * @param flags
- *  The mempool flags.
- * @return
- *   Required memory size aligned at page boundary.
- */
-__rte_deprecated
-size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz,
-	uint32_t pg_shift, unsigned int flags);
-
-/**
- * @deprecated
- * Get the size of memory required to store mempool elements.
- *
- * Calculate how much memory would be actually required with the given
- * memory footprint to store required number of objects.
- *
- * @param vaddr
- *   Virtual address of the externally allocated memory buffer.
- *   Will be used to store mempool objects.
- * @param elt_num
- *   Number of elements.
- * @param total_elt_sz
- *   The size of each element, including header and trailer, as returned
- *   by rte_mempool_calc_obj_size().
- * @param iova
- *   Array of IO addresses of the pages that comprises given memory buffer.
- * @param pg_num
- *   Number of elements in the iova array.
- * @param pg_shift
- *   LOG2 of the physical pages size.
- * @param flags
- *  The mempool flags.
- * @return
- *   On success, the number of bytes needed to store given number of
- *   objects, aligned to the given page size. If the provided memory
- *   buffer is too small, return a negative value whose absolute value
- *   is the actual number of elements that can be stored in that buffer.
- */
-__rte_deprecated
-ssize_t rte_mempool_xmem_usage(void *vaddr, uint32_t elt_num,
-	size_t total_elt_sz, const rte_iova_t iova[], uint32_t pg_num,
-	uint32_t pg_shift, unsigned int flags);
-
-/**
  * Walk list of all memory pools
  *
  * @param func
diff --git a/lib/librte_mempool/rte_mempool_ops_default.c b/lib/librte_mempool/rte_mempool_ops_default.c
index fd63ca13..4e2bfc82 100644
--- a/lib/librte_mempool/rte_mempool_ops_default.c
+++ b/lib/librte_mempool/rte_mempool_ops_default.c
@@ -12,12 +12,31 @@ rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
 				     size_t *min_chunk_size, size_t *align)
 {
 	size_t total_elt_sz;
+	size_t obj_per_page, pg_num, pg_sz;
 	size_t mem_size;
 
 	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
-
-	mem_size = rte_mempool_calc_mem_size_helper(obj_num, total_elt_sz,
-						    pg_shift);
+	if (total_elt_sz == 0) {
+		mem_size = 0;
+	} else if (pg_shift == 0) {
+		mem_size = total_elt_sz * obj_num;
+	} else {
+		pg_sz = (size_t)1 << pg_shift;
+		obj_per_page = pg_sz / total_elt_sz;
+		if (obj_per_page == 0) {
+			/*
+			 * Note that if object size is bigger than page size,
+			 * then it is assumed that pages are grouped in subsets
+			 * of physically continuous pages big enough to store
+			 * at least one object.
+			 */
+			mem_size =
+				RTE_ALIGN_CEIL(total_elt_sz, pg_sz) * obj_num;
+		} else {
+			pg_num = (obj_num + obj_per_page - 1) / obj_per_page;
+			mem_size = pg_num << pg_shift;
+		}
+	}
 
 	*min_chunk_size = RTE_MAX((size_t)1 << pg_shift, total_elt_sz);
 
diff --git a/lib/librte_mempool/rte_mempool_version.map b/lib/librte_mempool/rte_mempool_version.map
index 7091b954..17cbca46 100644
--- a/lib/librte_mempool/rte_mempool_version.map
+++ b/lib/librte_mempool/rte_mempool_version.map
@@ -8,9 +8,6 @@ DPDK_2.0 {
 	rte_mempool_list_dump;
 	rte_mempool_lookup;
 	rte_mempool_walk;
-	rte_mempool_xmem_create;
-	rte_mempool_xmem_size;
-	rte_mempool_xmem_usage;
 
 	local: *;
 };
@@ -34,8 +31,6 @@ DPDK_16.07 {
 	rte_mempool_ops_table;
 	rte_mempool_populate_anon;
 	rte_mempool_populate_default;
-	rte_mempool_populate_phys;
-	rte_mempool_populate_phys_tab;
 	rte_mempool_populate_virt;
 	rte_mempool_register_ops;
 	rte_mempool_set_ops_byname;
@@ -46,7 +41,6 @@ DPDK_17.11 {
 	global:
 
 	rte_mempool_populate_iova;
-	rte_mempool_populate_iova_tab;
 
 } DPDK_16.07;
 
diff --git a/lib/librte_meter/rte_meter.c b/lib/librte_meter/rte_meter.c
index 59af5ef2..473f69ab 100644
--- a/lib/librte_meter/rte_meter.c
+++ b/lib/librte_meter/rte_meter.c
@@ -30,7 +30,7 @@ rte_meter_get_tb_params(uint64_t hz, uint64_t rate, uint64_t *tb_period, uint64_
 	}
 }
 
-int __rte_experimental
+int
 rte_meter_srtcm_profile_config(struct rte_meter_srtcm_profile *p,
 	struct rte_meter_srtcm_params *params)
 {
@@ -68,7 +68,7 @@ rte_meter_srtcm_config(struct rte_meter_srtcm *m,
 	return 0;
 }
 
-int __rte_experimental
+int
 rte_meter_trtcm_profile_config(struct rte_meter_trtcm_profile *p,
 	struct rte_meter_trtcm_params *params)
 {
diff --git a/lib/librte_meter/rte_meter.h b/lib/librte_meter/rte_meter.h
index 03d80566..58a05158 100644
--- a/lib/librte_meter/rte_meter.h
+++ b/lib/librte_meter/rte_meter.h
@@ -20,7 +20,6 @@ extern "C" {
  ***/
 
 #include <stdint.h>
-#include <rte_compat.h>
 
 /*
  * Application Programmer's Interface (API)
@@ -82,7 +81,7 @@ struct rte_meter_trtcm;
  * @return
  *    0 upon success, error code otherwise
  */
-int __rte_experimental
+int
 rte_meter_srtcm_profile_config(struct rte_meter_srtcm_profile *p,
 	struct rte_meter_srtcm_params *params);
 
@@ -96,7 +95,7 @@ rte_meter_srtcm_profile_config(struct rte_meter_srtcm_profile *p,
  * @return
  *    0 upon success, error code otherwise
  */
-int __rte_experimental
+int
 rte_meter_trtcm_profile_config(struct rte_meter_trtcm_profile *p,
 	struct rte_meter_trtcm_params *params);
 
diff --git a/lib/librte_meter/rte_meter_version.map b/lib/librte_meter/rte_meter_version.map
index 9215d4cb..cb79f0c2 100644
--- a/lib/librte_meter/rte_meter_version.map
+++ b/lib/librte_meter/rte_meter_version.map
@@ -11,7 +11,7 @@ DPDK_2.0 {
 	local: *;
 };
 
-EXPERIMENTAL {
+DPDK_18.08 {
 	global:
 
 	rte_meter_srtcm_profile_config;
diff --git a/lib/librte_metrics/rte_metrics.c b/lib/librte_metrics/rte_metrics.c
index 258f0582..99a96b65 100644
--- a/lib/librte_metrics/rte_metrics.c
+++ b/lib/librte_metrics/rte_metrics.c
@@ -96,6 +96,9 @@ rte_metrics_reg_names(const char * const *names, uint16_t cnt_names)
 	/* Some sanity checks */
 	if (cnt_names < 1 || names == NULL)
 		return -EINVAL;
+	for (idx_name = 0; idx_name < cnt_names; idx_name++)
+		if (names[idx_name] == NULL)
+			return -EINVAL;
 
 	memzone = rte_memzone_lookup(RTE_METRICS_MEMZONE_NAME);
 	if (memzone == NULL)
@@ -159,6 +162,11 @@ rte_metrics_update_values(int port_id,
 	stats = memzone->addr;
 
 	rte_spinlock_lock(&stats->lock);
+
+	if (key >= stats->cnt_stats) {
+		rte_spinlock_unlock(&stats->lock);
+		return -EINVAL;
+	}
 	idx_metric = key;
 	cnt_setsize = 1;
 	while (idx_metric < stats->cnt_stats) {
@@ -200,9 +208,8 @@ rte_metrics_get_names(struct rte_metric_name *names,
 	int return_value;
 
 	memzone = rte_memzone_lookup(RTE_METRICS_MEMZONE_NAME);
-	/* If not allocated, fail silently */
 	if (memzone == NULL)
-		return 0;
+		return -EIO;
 
 	stats = memzone->addr;
 	rte_spinlock_lock(&stats->lock);
@@ -238,9 +245,9 @@ rte_metrics_get_values(int port_id,
 		return -EINVAL;
 
 	memzone = rte_memzone_lookup(RTE_METRICS_MEMZONE_NAME);
-	/* If not allocated, fail silently */
 	if (memzone == NULL)
-		return 0;
+		return -EIO;
+
 	stats = memzone->addr;
 	rte_spinlock_lock(&stats->lock);
 
diff --git a/lib/librte_net/rte_ip.h b/lib/librte_net/rte_ip.h
index 72dc2456..f2a8904a 100644
--- a/lib/librte_net/rte_ip.h
+++ b/lib/librte_net/rte_ip.h
@@ -108,25 +108,25 @@ __rte_raw_cksum(const void *buf, size_t len, uint32_t sum)
 	/* workaround gcc strict-aliasing warning */
 	uintptr_t ptr = (uintptr_t)buf;
 	typedef uint16_t __attribute__((__may_alias__)) u16_p;
-	const u16_p *u16 = (const u16_p *)ptr;
-
-	while (len >= (sizeof(*u16) * 4)) {
-		sum += u16[0];
-		sum += u16[1];
-		sum += u16[2];
-		sum += u16[3];
-		len -= sizeof(*u16) * 4;
-		u16 += 4;
+	const u16_p *u16_buf = (const u16_p *)ptr;
+
+	while (len >= (sizeof(*u16_buf) * 4)) {
+		sum += u16_buf[0];
+		sum += u16_buf[1];
+		sum += u16_buf[2];
+		sum += u16_buf[3];
+		len -= sizeof(*u16_buf) * 4;
+		u16_buf += 4;
 	}
-	while (len >= sizeof(*u16)) {
-		sum += *u16;
-		len -= sizeof(*u16);
-		u16 += 1;
+	while (len >= sizeof(*u16_buf)) {
+		sum += *u16_buf;
+		len -= sizeof(*u16_buf);
+		u16_buf += 1;
 	}
 
 	/* if length is in odd bytes */
 	if (len == 1)
-		sum += *((const uint8_t *)u16);
+		sum += *((const uint8_t *)u16_buf);
 
 	return sum;
 }
diff --git a/lib/librte_power/channel_commands.h b/lib/librte_power/channel_commands.h
index 5e8b4ab5..ee638eef 100644
--- a/lib/librte_power/channel_commands.h
+++ b/lib/librte_power/channel_commands.h
@@ -48,7 +48,8 @@ enum workload {HIGH, MEDIUM, LOW};
 enum policy_to_use {
 	TRAFFIC,
 	TIME,
-	WORKLOAD
+	WORKLOAD,
+	BRANCH_RATIO
 };
 
 struct traffic {
diff --git a/lib/librte_power/power_acpi_cpufreq.c b/lib/librte_power/power_acpi_cpufreq.c
index bce933e9..cd5978d5 100644
--- a/lib/librte_power/power_acpi_cpufreq.c
+++ b/lib/librte_power/power_acpi_cpufreq.c
@@ -623,3 +623,24 @@ power_acpi_disable_turbo(unsigned int lcore_id)
 
 	return 0;
 }
+
+int power_acpi_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps)
+{
+	struct rte_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+	if (caps == NULL) {
+		RTE_LOG(ERR, POWER, "Invalid argument\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	caps->capabilities = 0;
+	caps->turbo = !!(pi->turbo_available);
+
+	return 0;
+}
diff --git a/lib/librte_power/power_acpi_cpufreq.h b/lib/librte_power/power_acpi_cpufreq.h
index edeeb27a..1af74160 100644
--- a/lib/librte_power/power_acpi_cpufreq.h
+++ b/lib/librte_power/power_acpi_cpufreq.h
@@ -14,6 +14,7 @@
 #include <rte_byteorder.h>
 #include <rte_log.h>
 #include <rte_string_fns.h>
+#include "rte_power.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -196,6 +197,21 @@ int power_acpi_enable_turbo(unsigned int lcore_id);
  */
 int power_acpi_disable_turbo(unsigned int lcore_id);
 
+/**
+ * Returns power capabilities for a specific lcore.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param caps
+ *  pointer to rte_power_core_capabilities object.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_acpi_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_power/power_kvm_vm.c b/lib/librte_power/power_kvm_vm.c
index 38e9066f..20659b72 100644
--- a/lib/librte_power/power_kvm_vm.c
+++ b/lib/librte_power/power_kvm_vm.c
@@ -124,3 +124,11 @@ power_kvm_vm_disable_turbo(unsigned int lcore_id)
 {
 	return send_msg(lcore_id, CPU_POWER_DISABLE_TURBO);
 }
+
+struct rte_power_core_capabilities;
+int power_kvm_vm_get_capabilities(__rte_unused unsigned int lcore_id,
+		__rte_unused struct rte_power_core_capabilities *caps)
+{
+	RTE_LOG(ERR, POWER, "rte_power_get_capabilities is not implemented for Virtual Machine Power Management\n");
+	return -ENOTSUP;
+}
diff --git a/lib/librte_power/power_kvm_vm.h b/lib/librte_power/power_kvm_vm.h
index 446d6997..94d4aa12 100644
--- a/lib/librte_power/power_kvm_vm.h
+++ b/lib/librte_power/power_kvm_vm.h
@@ -14,6 +14,7 @@
 #include <rte_byteorder.h>
 #include <rte_log.h>
 #include <rte_string_fns.h>
+#include "rte_power.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -177,6 +178,22 @@ int power_kvm_vm_enable_turbo(unsigned int lcore_id);
  *  - Negative on error.
  */
 int power_kvm_vm_disable_turbo(unsigned int lcore_id);
+
+/**
+ * Returns power capabilities for a specific lcore.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param caps
+ *  pointer to rte_power_core_capabilities object.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_kvm_vm_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_power/rte_power.c b/lib/librte_power/rte_power.c
index 6c8fb403..208b7919 100644
--- a/lib/librte_power/rte_power.c
+++ b/lib/librte_power/rte_power.c
@@ -24,6 +24,7 @@ rte_power_freq_change_t rte_power_freq_min = NULL;
 rte_power_freq_change_t rte_power_turbo_status;
 rte_power_freq_change_t rte_power_freq_enable_turbo;
 rte_power_freq_change_t rte_power_freq_disable_turbo;
+rte_power_get_capabilities_t rte_power_get_capabilities;
 
 int
 rte_power_set_env(enum power_management_env env)
@@ -42,6 +43,7 @@ rte_power_set_env(enum power_management_env env)
 		rte_power_turbo_status = power_acpi_turbo_status;
 		rte_power_freq_enable_turbo = power_acpi_enable_turbo;
 		rte_power_freq_disable_turbo = power_acpi_disable_turbo;
+		rte_power_get_capabilities = power_acpi_get_capabilities;
 	} else if (env == PM_ENV_KVM_VM) {
 		rte_power_freqs = power_kvm_vm_freqs;
 		rte_power_get_freq = power_kvm_vm_get_freq;
@@ -53,6 +55,7 @@ rte_power_set_env(enum power_management_env env)
 		rte_power_turbo_status = power_kvm_vm_turbo_status;
 		rte_power_freq_enable_turbo = power_kvm_vm_enable_turbo;
 		rte_power_freq_disable_turbo = power_kvm_vm_disable_turbo;
+		rte_power_get_capabilities = power_kvm_vm_get_capabilities;
 	} else {
 		RTE_LOG(ERR, POWER, "Invalid Power Management Environment(%d) set\n",
 				env);
diff --git a/lib/librte_power/rte_power.h b/lib/librte_power/rte_power.h
index b4b7357b..d70bc0b3 100644
--- a/lib/librte_power/rte_power.h
+++ b/lib/librte_power/rte_power.h
@@ -247,6 +247,38 @@ extern rte_power_freq_change_t rte_power_freq_enable_turbo;
  */
 extern rte_power_freq_change_t rte_power_freq_disable_turbo;
 
+/**
+ * Power capabilities summary.
+ */
+struct rte_power_core_capabilities {
+	RTE_STD_C11
+	union {
+		uint64_t capabilities;
+		RTE_STD_C11
+		struct {
+			uint64_t turbo:1;	/**< Turbo can be enabled. */
+		};
+	};
+};
+
+/**
+ * Returns power capabilities for a specific lcore.
+ * Function pointer definition. Review each environments
+ * specific documentation for usage.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param caps
+ *  pointer to rte_power_core_capabilities object.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+typedef int (*rte_power_get_capabilities_t)(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps);
+
+extern rte_power_get_capabilities_t rte_power_get_capabilities;
 
 #ifdef __cplusplus
 }
diff --git a/lib/librte_power/rte_power_version.map b/lib/librte_power/rte_power_version.map
index 96dc42ec..dd587dfb 100644
--- a/lib/librte_power/rte_power_version.map
+++ b/lib/librte_power/rte_power_version.map
@@ -25,4 +25,11 @@ DPDK_17.11 {
 	rte_power_freq_enable_turbo;
 	rte_power_turbo_status;
 
-} DPDK_2.0;
\ No newline at end of file
+} DPDK_2.0;
+
+DPDK_18.08 {
+	global:
+
+	rte_power_get_capabilities;
+
+} DPDK_17.11;
diff --git a/lib/librte_rawdev/Makefile b/lib/librte_rawdev/Makefile
index b9105b06..addb288d 100644
--- a/lib/librte_rawdev/Makefile
+++ b/lib/librte_rawdev/Makefile
@@ -10,7 +10,6 @@ LIB = librte_rawdev.a
 LIBABIVER := 1
 
 # build flags
-CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 LDLIBS += -lrte_eal
diff --git a/lib/librte_rawdev/meson.build b/lib/librte_rawdev/meson.build
index dcd37ad4..a20fbdc0 100644
--- a/lib/librte_rawdev/meson.build
+++ b/lib/librte_rawdev/meson.build
@@ -1,6 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2018 Intel Corporation
 
-allow_experimental_apis = true
 sources = files('rte_rawdev.c')
 headers = files('rte_rawdev.h', 'rte_rawdev_pmd.h')
diff --git a/lib/librte_rawdev/rte_rawdev.c b/lib/librte_rawdev/rte_rawdev.c
index 284e6aec..62b6b97e 100644
--- a/lib/librte_rawdev/rte_rawdev.c
+++ b/lib/librte_rawdev/rte_rawdev.c
@@ -46,13 +46,13 @@ static struct rte_rawdev_global rawdev_globals = {
 struct rte_rawdev_global *rte_rawdev_globals = &rawdev_globals;
 
 /* Raw device, northbound API implementation */
-uint8_t __rte_experimental
+uint8_t
 rte_rawdev_count(void)
 {
 	return rte_rawdev_globals->nb_devs;
 }
 
-uint16_t __rte_experimental
+uint16_t
 rte_rawdev_get_dev_id(const char *name)
 {
 	uint16_t i;
@@ -69,7 +69,7 @@ rte_rawdev_get_dev_id(const char *name)
 	return -ENODEV;
 }
 
-int __rte_experimental
+int
 rte_rawdev_socket_id(uint16_t dev_id)
 {
 	struct rte_rawdev *dev;
@@ -80,7 +80,7 @@ rte_rawdev_socket_id(uint16_t dev_id)
 	return dev->socket_id;
 }
 
-int __rte_experimental
+int
 rte_rawdev_info_get(uint16_t dev_id, struct rte_rawdev_info *dev_info)
 {
 	struct rte_rawdev *rawdev;
@@ -102,7 +102,7 @@ rte_rawdev_info_get(uint16_t dev_id, struct rte_rawdev_info *dev_info)
 	return 0;
 }
 
-int __rte_experimental
+int
 rte_rawdev_configure(uint16_t dev_id, struct rte_rawdev_info *dev_conf)
 {
 	struct rte_rawdev *dev;
@@ -131,7 +131,7 @@ rte_rawdev_configure(uint16_t dev_id, struct rte_rawdev_info *dev_conf)
 	return diag;
 }
 
-int __rte_experimental
+int
 rte_rawdev_queue_conf_get(uint16_t dev_id,
 			  uint16_t queue_id,
 			  rte_rawdev_obj_t queue_conf)
@@ -146,7 +146,7 @@ rte_rawdev_queue_conf_get(uint16_t dev_id,
 	return 0;
 }
 
-int __rte_experimental
+int
 rte_rawdev_queue_setup(uint16_t dev_id,
 		       uint16_t queue_id,
 		       rte_rawdev_obj_t queue_conf)
@@ -160,7 +160,7 @@ rte_rawdev_queue_setup(uint16_t dev_id,
 	return (*dev->dev_ops->queue_setup)(dev, queue_id, queue_conf);
 }
 
-int __rte_experimental
+int
 rte_rawdev_queue_release(uint16_t dev_id, uint16_t queue_id)
 {
 	struct rte_rawdev *dev;
@@ -172,7 +172,19 @@ rte_rawdev_queue_release(uint16_t dev_id, uint16_t queue_id)
 	return (*dev->dev_ops->queue_release)(dev, queue_id);
 }
 
-int __rte_experimental
+uint16_t
+rte_rawdev_queue_count(uint16_t dev_id)
+{
+	struct rte_rawdev *dev;
+
+	RTE_RAWDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_rawdevs[dev_id];
+
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_count, -ENOTSUP);
+	return (*dev->dev_ops->queue_count)(dev);
+}
+
+int
 rte_rawdev_get_attr(uint16_t dev_id,
 		    const char *attr_name,
 		    uint64_t *attr_value)
@@ -186,7 +198,7 @@ rte_rawdev_get_attr(uint16_t dev_id,
 	return (*dev->dev_ops->attr_get)(dev, attr_name, attr_value);
 }
 
-int __rte_experimental
+int
 rte_rawdev_set_attr(uint16_t dev_id,
 		    const char *attr_name,
 		    const uint64_t attr_value)
@@ -200,7 +212,7 @@ rte_rawdev_set_attr(uint16_t dev_id,
 	return (*dev->dev_ops->attr_set)(dev, attr_name, attr_value);
 }
 
-int __rte_experimental
+int
 rte_rawdev_enqueue_buffers(uint16_t dev_id,
 			   struct rte_rawdev_buf **buffers,
 			   unsigned int count,
@@ -215,7 +227,7 @@ rte_rawdev_enqueue_buffers(uint16_t dev_id,
 	return (*dev->dev_ops->enqueue_bufs)(dev, buffers, count, context);
 }
 
-int __rte_experimental
+int
 rte_rawdev_dequeue_buffers(uint16_t dev_id,
 			   struct rte_rawdev_buf **buffers,
 			   unsigned int count,
@@ -230,7 +242,7 @@ rte_rawdev_dequeue_buffers(uint16_t dev_id,
 	return (*dev->dev_ops->dequeue_bufs)(dev, buffers, count, context);
 }
 
-int __rte_experimental
+int
 rte_rawdev_dump(uint16_t dev_id, FILE *f)
 {
 	struct rte_rawdev *dev;
@@ -251,7 +263,7 @@ xstats_get_count(uint16_t dev_id)
 	return (*dev->dev_ops->xstats_get_names)(dev, NULL, 0);
 }
 
-int __rte_experimental
+int
 rte_rawdev_xstats_names_get(uint16_t dev_id,
 		struct rte_rawdev_xstats_name *xstats_names,
 		unsigned int size)
@@ -274,7 +286,7 @@ rte_rawdev_xstats_names_get(uint16_t dev_id,
 }
 
 /* retrieve rawdev extended statistics */
-int __rte_experimental
+int
 rte_rawdev_xstats_get(uint16_t dev_id,
 		      const unsigned int ids[],
 		      uint64_t values[],
@@ -287,7 +299,7 @@ rte_rawdev_xstats_get(uint16_t dev_id,
 	return (*dev->dev_ops->xstats_get)(dev, ids, values, n);
 }
 
-uint64_t __rte_experimental
+uint64_t
 rte_rawdev_xstats_by_name_get(uint16_t dev_id,
 			      const char *name,
 			      unsigned int *id)
@@ -306,7 +318,7 @@ rte_rawdev_xstats_by_name_get(uint16_t dev_id,
 	return (*dev->dev_ops->xstats_get_by_name)(dev, name, id);
 }
 
-int __rte_experimental
+int
 rte_rawdev_xstats_reset(uint16_t dev_id,
 			const uint32_t ids[], uint32_t nb_ids)
 {
@@ -317,7 +329,7 @@ rte_rawdev_xstats_reset(uint16_t dev_id,
 	return (*dev->dev_ops->xstats_reset)(dev, ids, nb_ids);
 }
 
-int __rte_experimental
+int
 rte_rawdev_firmware_status_get(uint16_t dev_id, rte_rawdev_obj_t status_info)
 {
 	RTE_RAWDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
@@ -327,7 +339,7 @@ rte_rawdev_firmware_status_get(uint16_t dev_id, rte_rawdev_obj_t status_info)
 	return (*dev->dev_ops->firmware_status_get)(dev, status_info);
 }
 
-int __rte_experimental
+int
 rte_rawdev_firmware_version_get(uint16_t dev_id, rte_rawdev_obj_t version_info)
 {
 	RTE_RAWDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
@@ -337,7 +349,7 @@ rte_rawdev_firmware_version_get(uint16_t dev_id, rte_rawdev_obj_t version_info)
 	return (*dev->dev_ops->firmware_version_get)(dev, version_info);
 }
 
-int __rte_experimental
+int
 rte_rawdev_firmware_load(uint16_t dev_id, rte_rawdev_obj_t firmware_image)
 {
 	RTE_RAWDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
@@ -350,7 +362,7 @@ rte_rawdev_firmware_load(uint16_t dev_id, rte_rawdev_obj_t firmware_image)
 	return (*dev->dev_ops->firmware_load)(dev, firmware_image);
 }
 
-int __rte_experimental
+int
 rte_rawdev_firmware_unload(uint16_t dev_id)
 {
 	RTE_RAWDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
@@ -360,7 +372,7 @@ rte_rawdev_firmware_unload(uint16_t dev_id)
 	return (*dev->dev_ops->firmware_unload)(dev);
 }
 
-int __rte_experimental
+int
 rte_rawdev_selftest(uint16_t dev_id)
 {
 	RTE_RAWDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
@@ -370,7 +382,7 @@ rte_rawdev_selftest(uint16_t dev_id)
 	return (*dev->dev_ops->dev_selftest)();
 }
 
-int __rte_experimental
+int
 rte_rawdev_start(uint16_t dev_id)
 {
 	struct rte_rawdev *dev;
@@ -397,7 +409,7 @@ rte_rawdev_start(uint16_t dev_id)
 	return 0;
 }
 
-void __rte_experimental
+void
 rte_rawdev_stop(uint16_t dev_id)
 {
 	struct rte_rawdev *dev;
@@ -419,7 +431,7 @@ rte_rawdev_stop(uint16_t dev_id)
 	dev->started = 0;
 }
 
-int __rte_experimental
+int
 rte_rawdev_close(uint16_t dev_id)
 {
 	struct rte_rawdev *dev;
@@ -438,7 +450,7 @@ rte_rawdev_close(uint16_t dev_id)
 	return (*dev->dev_ops->dev_close)(dev);
 }
 
-int __rte_experimental
+int
 rte_rawdev_reset(uint16_t dev_id)
 {
 	struct rte_rawdev *dev;
@@ -465,7 +477,7 @@ rte_rawdev_find_free_device_index(void)
 	return RTE_RAWDEV_MAX_DEVS;
 }
 
-struct rte_rawdev * __rte_experimental
+struct rte_rawdev *
 rte_rawdev_pmd_allocate(const char *name, size_t dev_priv_size, int socket_id)
 {
 	struct rte_rawdev *rawdev;
@@ -506,7 +518,7 @@ rte_rawdev_pmd_allocate(const char *name, size_t dev_priv_size, int socket_id)
 	return rawdev;
 }
 
-int __rte_experimental
+int
 rte_rawdev_pmd_release(struct rte_rawdev *rawdev)
 {
 	int ret;
@@ -532,10 +544,7 @@ rte_rawdev_pmd_release(struct rte_rawdev *rawdev)
 	return 0;
 }
 
-RTE_INIT(librawdev_init_log);
-
-static void
-librawdev_init_log(void)
+RTE_INIT(librawdev_init_log)
 {
 	librawdev_logtype = rte_log_register("lib.rawdev");
 	if (librawdev_logtype >= 0)
diff --git a/lib/librte_rawdev/rte_rawdev.h b/lib/librte_rawdev/rte_rawdev.h
index 2e14919b..684bfdb8 100644
--- a/lib/librte_rawdev/rte_rawdev.h
+++ b/lib/librte_rawdev/rte_rawdev.h
@@ -35,7 +35,7 @@ typedef void *rte_rawdev_obj_t;
  * @return
  *   The total number of usable raw devices.
  */
-uint8_t __rte_experimental
+uint8_t
 rte_rawdev_count(void);
 
 /**
@@ -48,7 +48,7 @@ rte_rawdev_count(void);
  *   Returns raw device identifier on success.
  *   - <0: Failure to find named raw device.
  */
-uint16_t __rte_experimental
+uint16_t
 rte_rawdev_get_dev_id(const char *name);
 
 /**
@@ -61,7 +61,7 @@ rte_rawdev_get_dev_id(const char *name);
  *   a default of zero if the socket could not be determined.
  *   -(-EINVAL)  dev_id value is out of range.
  */
-int __rte_experimental
+int
 rte_rawdev_socket_id(uint16_t dev_id);
 
 /**
@@ -84,7 +84,7 @@ struct rte_rawdev_info;
  *   - <0: Error code returned by the driver info get function.
  *
  */
-int __rte_experimental
+int
 rte_rawdev_info_get(uint16_t dev_id, struct rte_rawdev_info *dev_info);
 
 /**
@@ -111,7 +111,7 @@ rte_rawdev_info_get(uint16_t dev_id, struct rte_rawdev_info *dev_info);
  *   - 0: Success, device configured.
  *   - <0: Error code returned by the driver configuration function.
  */
-int __rte_experimental
+int
 rte_rawdev_configure(uint16_t dev_id, struct rte_rawdev_info *dev_conf);
 
 
@@ -137,7 +137,7 @@ rte_rawdev_configure(uint16_t dev_id, struct rte_rawdev_info *dev_conf);
  * @see rte_raw_queue_setup()
  *
  */
-int __rte_experimental
+int
 rte_rawdev_queue_conf_get(uint16_t dev_id,
 			  uint16_t queue_id,
 			  rte_rawdev_obj_t queue_conf);
@@ -160,7 +160,7 @@ rte_rawdev_queue_conf_get(uint16_t dev_id,
  *   - 0: Success, raw queue correctly set up.
  *   - <0: raw queue configuration failed
  */
-int __rte_experimental
+int
 rte_rawdev_queue_setup(uint16_t dev_id,
 		       uint16_t queue_id,
 		       rte_rawdev_obj_t queue_conf);
@@ -180,8 +180,9 @@ rte_rawdev_queue_setup(uint16_t dev_id,
  *   - 0: Success, raw queue released.
  *   - <0: raw queue configuration failed
  */
-int __rte_experimental
+int
 rte_rawdev_queue_release(uint16_t dev_id, uint16_t queue_id);
+
 /**
  * Get the number of raw queues on a specific raw device
  *
@@ -190,7 +191,7 @@ rte_rawdev_queue_release(uint16_t dev_id, uint16_t queue_id);
  * @return
  *   - The number of configured raw queues
  */
-uint16_t __rte_experimental
+uint16_t
 rte_rawdev_queue_count(uint16_t dev_id);
 
 /**
@@ -208,7 +209,7 @@ rte_rawdev_queue_count(uint16_t dev_id);
  *   - 0: Success, device started.
  *   < 0: Failure
  */
-int __rte_experimental
+int
 rte_rawdev_start(uint16_t dev_id);
 
 /**
@@ -218,7 +219,7 @@ rte_rawdev_start(uint16_t dev_id);
  * @param dev_id
  *   Raw device identifier.
  */
-void __rte_experimental
+void
 rte_rawdev_stop(uint16_t dev_id);
 
 /**
@@ -232,7 +233,7 @@ rte_rawdev_stop(uint16_t dev_id);
  *  - <0 on failure to close device
  *  - (-EAGAIN) if device is busy
  */
-int __rte_experimental
+int
 rte_rawdev_close(uint16_t dev_id);
 
 /**
@@ -246,7 +247,7 @@ rte_rawdev_close(uint16_t dev_id);
  *   0 for sucessful reset,
  *  !0 for failure in resetting
  */
-int __rte_experimental
+int
 rte_rawdev_reset(uint16_t dev_id);
 
 #define RTE_RAWDEV_NAME_MAX_LEN	(64)
@@ -316,7 +317,7 @@ struct rte_rawdev_buf {
  *   - 0: on success
  *   - <0: on failure.
  */
-int __rte_experimental
+int
 rte_rawdev_dump(uint16_t dev_id, FILE *f);
 
 /**
@@ -338,7 +339,7 @@ rte_rawdev_dump(uint16_t dev_id, FILE *f);
  *   0 for success
  *  !0 Error; attr_value remains untouched in case of error.
  */
-int __rte_experimental
+int
 rte_rawdev_get_attr(uint16_t dev_id,
 		    const char *attr_name,
 		    uint64_t *attr_value);
@@ -357,7 +358,7 @@ rte_rawdev_get_attr(uint16_t dev_id,
  *   0 for success
  *  !0 Error
  */
-int __rte_experimental
+int
 rte_rawdev_set_attr(uint16_t dev_id,
 		    const char *attr_name,
 		    const uint64_t attr_value);
@@ -383,7 +384,7 @@ rte_rawdev_set_attr(uint16_t dev_id,
  *  Whether partial enqueue is failure or success is defined between app
  *  and driver implementation.
  */
-int __rte_experimental
+int
 rte_rawdev_enqueue_buffers(uint16_t dev_id,
 			   struct rte_rawdev_buf **buffers,
 			   unsigned int count,
@@ -414,7 +415,7 @@ rte_rawdev_enqueue_buffers(uint16_t dev_id,
  *  Whether partial enqueue is failure or success is defined between app
  *  and driver implementation.
  */
-int __rte_experimental
+int
 rte_rawdev_dequeue_buffers(uint16_t dev_id,
 			   struct rte_rawdev_buf **buffers,
 			   unsigned int count,
@@ -454,7 +455,7 @@ struct rte_rawdev_xstats_name {
  *        -ENODEV for invalid *dev_id*
  *        -ENOTSUP if the device doesn't support this function.
  */
-int __rte_experimental
+int
 rte_rawdev_xstats_names_get(uint16_t dev_id,
 			    struct rte_rawdev_xstats_name *xstats_names,
 			    unsigned int size);
@@ -478,7 +479,7 @@ rte_rawdev_xstats_names_get(uint16_t dev_id,
  *        -ENODEV for invalid *dev_id*
  *        -ENOTSUP if the device doesn't support this function.
  */
-int __rte_experimental
+int
 rte_rawdev_xstats_get(uint16_t dev_id,
 		      const unsigned int ids[],
 		      uint64_t values[],
@@ -500,7 +501,7 @@ rte_rawdev_xstats_get(uint16_t dev_id,
  *   - positive value or zero: the stat value
  *   - negative value: -EINVAL if stat not found, -ENOTSUP if not supported.
  */
-uint64_t __rte_experimental
+uint64_t
 rte_rawdev_xstats_by_name_get(uint16_t dev_id,
 			      const char *name,
 			      unsigned int *id);
@@ -520,7 +521,7 @@ rte_rawdev_xstats_by_name_get(uint16_t dev_id,
  *   - zero: successfully reset the statistics to zero
  *   - negative value: -EINVAL invalid parameters, -ENOTSUP if not supported.
  */
-int __rte_experimental
+int
 rte_rawdev_xstats_reset(uint16_t dev_id,
 			const uint32_t ids[],
 			uint32_t nb_ids);
@@ -539,7 +540,7 @@ rte_rawdev_xstats_reset(uint16_t dev_id,
  *   0 for success,
  *  !0 for failure, `status_info` argument state is undefined
  */
-int __rte_experimental
+int
 rte_rawdev_firmware_status_get(uint16_t dev_id,
 			       rte_rawdev_obj_t status_info);
 
@@ -557,7 +558,7 @@ rte_rawdev_firmware_status_get(uint16_t dev_id,
  *   0 for success,
  *  !0 for failure, `version_info` argument state is undefined
  */
-int __rte_experimental
+int
 rte_rawdev_firmware_version_get(uint16_t dev_id,
 				rte_rawdev_obj_t version_info);
 
@@ -574,7 +575,7 @@ rte_rawdev_firmware_version_get(uint16_t dev_id,
  *   0 for successful load
  *  !0 for failure to load the provided image, or image incorrect.
  */
-int __rte_experimental
+int
 rte_rawdev_firmware_load(uint16_t dev_id, rte_rawdev_obj_t firmware_image);
 
 /**
@@ -586,7 +587,7 @@ rte_rawdev_firmware_load(uint16_t dev_id, rte_rawdev_obj_t firmware_image);
  *   0 for successful Unload
  *  !0 for failure in unloading
  */
-int __rte_experimental
+int
 rte_rawdev_firmware_unload(uint16_t dev_id);
 
 /**
@@ -599,7 +600,7 @@ rte_rawdev_firmware_unload(uint16_t dev_id);
  *   - -ENOTSUP if the device doesn't support selftest
  *   - other values < 0 on failure.
  */
-int __rte_experimental
+int
 rte_rawdev_selftest(uint16_t dev_id);
 
 #ifdef __cplusplus
diff --git a/lib/librte_rawdev/rte_rawdev_pmd.h b/lib/librte_rawdev/rte_rawdev_pmd.h
index 408adf0f..bb9bbc35 100644
--- a/lib/librte_rawdev/rte_rawdev_pmd.h
+++ b/lib/librte_rawdev/rte_rawdev_pmd.h
@@ -251,6 +251,24 @@ typedef int (*rawdev_queue_release_t)(struct rte_rawdev *dev,
 				      uint16_t queue_id);
 
 /**
+ * Get the count of number of queues configured on this device.
+ *
+ * Another way to fetch this information is to fetch the device configuration.
+ * But, that assumes that the device configuration managed by the driver has
+ * that kind of information.
+ *
+ * This function helps in getting queue count supported, independently. It
+ * can help in cases where iterator needs to be implemented.
+ *
+ * @param
+ *   Raw device pointer
+ * @return
+ *   Number of queues; 0 is assumed to be a valid response.
+ *
+ */
+typedef uint16_t (*rawdev_queue_count_t)(struct rte_rawdev *dev);
+
+/**
  * Enqueue an array of raw buffers to the device.
  *
  * Buffer being used is opaque - it can be obtained from mempool or from
@@ -506,6 +524,8 @@ struct rte_rawdev_ops {
 	rawdev_queue_setup_t queue_setup;
 	/**< Release an raw queue. */
 	rawdev_queue_release_t queue_release;
+	/**< Get the number of queues attached to the device */
+	rawdev_queue_count_t queue_count;
 
 	/**< Enqueue an array of raw buffers to device. */
 	rawdev_enqueue_bufs_t enqueue_bufs;
@@ -556,7 +576,7 @@ struct rte_rawdev_ops {
  * @return
  *   - Slot in the rte_dev_devices array for a new device;
  */
-struct rte_rawdev * __rte_experimental
+struct rte_rawdev *
 rte_rawdev_pmd_allocate(const char *name, size_t dev_private_size,
 			int socket_id);
 
@@ -568,7 +588,7 @@ rte_rawdev_pmd_allocate(const char *name, size_t dev_private_size,
  * @return
  *   - 0 on success, negative on error
  */
-int __rte_experimental
+int
 rte_rawdev_pmd_release(struct rte_rawdev *rawdev);
 
 /**
@@ -585,7 +605,7 @@ rte_rawdev_pmd_release(struct rte_rawdev *rawdev);
  *   - Raw device pointer if device is successfully created.
  *   - NULL if device cannot be created.
  */
-struct rte_rawdev * __rte_experimental
+struct rte_rawdev *
 rte_rawdev_pmd_init(const char *name, size_t dev_private_size,
 		    int socket_id);
 
@@ -597,7 +617,7 @@ rte_rawdev_pmd_init(const char *name, size_t dev_private_size,
  * @return
  *   - 0 on success, negative on error
  */
-int __rte_experimental
+int
 rte_rawdev_pmd_uninit(const char *name);
 
 #ifdef __cplusplus
diff --git a/lib/librte_rawdev/rte_rawdev_version.map b/lib/librte_rawdev/rte_rawdev_version.map
index af4465e2..b61dbff1 100644
--- a/lib/librte_rawdev/rte_rawdev_version.map
+++ b/lib/librte_rawdev/rte_rawdev_version.map
@@ -1,4 +1,4 @@
-EXPERIMENTAL {
+DPDK_18.08 {
 	global:
 
 	rte_rawdev_close;
@@ -16,6 +16,7 @@ EXPERIMENTAL {
 	rte_rawdev_pmd_allocate;
 	rte_rawdev_pmd_release;
 	rte_rawdev_queue_conf_get;
+	rte_rawdev_queue_count;
 	rte_rawdev_queue_setup;
 	rte_rawdev_queue_release;
 	rte_rawdev_reset;
diff --git a/lib/librte_ring/rte_ring.h b/lib/librte_ring/rte_ring.h
index 12458225..7a731d07 100644
--- a/lib/librte_ring/rte_ring.h
+++ b/lib/librte_ring/rte_ring.h
@@ -26,8 +26,9 @@
  * - Bulk dequeue.
  * - Bulk enqueue.
  *
- * Note: the ring implementation is not preemptable. A lcore must not
- * be interrupted by another task that uses the same ring.
+ * Note: the ring implementation is not preemptible. Refer to Programmer's
+ * guide/Environment Abstraction Layer/Multiple pthread/Known Issues/rte_ring
+ * for more information.
  *
  */
 
@@ -382,7 +383,7 @@ __rte_ring_do_dequeue(struct rte_ring *r, void **obj_table,
 	uint32_t cons_head, cons_next;
 	uint32_t entries;
 
-	n = __rte_ring_move_cons_head(r, is_sc, n, behavior,
+	n = __rte_ring_move_cons_head(r, (int)is_sc, n, behavior,
 			&cons_head, &cons_next, &entries);
 	if (n == 0)
 		goto end;
diff --git a/lib/librte_ring/rte_ring_c11_mem.h b/lib/librte_ring/rte_ring_c11_mem.h
index cb3f82b1..94df3c4a 100644
--- a/lib/librte_ring/rte_ring_c11_mem.h
+++ b/lib/librte_ring/rte_ring_c11_mem.h
@@ -66,14 +66,14 @@ __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp,
 
 		*old_head = __atomic_load_n(&r->prod.head,
 					__ATOMIC_ACQUIRE);
-		const uint32_t cons_tail = r->cons.tail;
+
 		/*
 		 *  The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
 		 * *old_head > cons_tail). So 'free_entries' is always between 0
 		 * and capacity (which is < size).
 		 */
-		*free_entries = (capacity + cons_tail - *old_head);
+		*free_entries = (capacity + r->cons.tail - *old_head);
 
 		/* check that we have enough room in ring */
 		if (unlikely(n > *free_entries))
@@ -133,13 +133,13 @@ __rte_ring_move_cons_head(struct rte_ring *r, int is_sc,
 		n = max;
 		*old_head = __atomic_load_n(&r->cons.head,
 					__ATOMIC_ACQUIRE);
-		const uint32_t prod_tail = r->prod.tail;
+
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
 		 * cons_head > prod_tail). So 'entries' is always between 0
 		 * and size(ring)-1.
 		 */
-		*entries = (prod_tail - *old_head);
+		*entries = (r->prod.tail - *old_head);
 
 		/* Set the actual entries for dequeue */
 		if (n > *entries)
diff --git a/lib/librte_security/rte_security.c b/lib/librte_security/rte_security.c
index 1e559c99..1954960a 100644
--- a/lib/librte_security/rte_security.c
+++ b/lib/librte_security/rte_security.c
@@ -1,34 +1,6 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright 2017 NXP.
- *   Copyright(c) 2017 Intel Corporation. All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of NXP nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 NXP.
+ * Copyright(c) 2017 Intel Corporation.
  */
 
 #include <rte_malloc.h>
@@ -91,7 +63,6 @@ rte_security_session_destroy(struct rte_security_ctx *instance,
 			     struct rte_security_session *sess)
 {
 	int ret;
-	struct rte_mempool *mp = rte_mempool_from_obj(sess);
 
 	RTE_FUNC_PTR_OR_ERR_RET(*instance->ops->session_destroy, -ENOTSUP);
 
@@ -100,7 +71,7 @@ rte_security_session_destroy(struct rte_security_ctx *instance,
 
 	ret = instance->ops->session_destroy(instance->device, sess);
 	if (!ret)
-		rte_mempool_put(mp, (void *)sess);
+		rte_mempool_put(rte_mempool_from_obj(sess), (void *)sess);
 
 	return ret;
 }
diff --git a/lib/librte_security/rte_security.h b/lib/librte_security/rte_security.h
index afa2861f..b0d1b97e 100644
--- a/lib/librte_security/rte_security.h
+++ b/lib/librte_security/rte_security.h
@@ -1,34 +1,6 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright 2017 NXP.
- *   Copyright(c) 2017 Intel Corporation. All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of NXP nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 NXP.
+ * Copyright(c) 2017 Intel Corporation.
  */
 
 #ifndef _RTE_SECURITY_H_
diff --git a/lib/librte_security/rte_security_driver.h b/lib/librte_security/rte_security_driver.h
index 0583f889..42f42ffe 100644
--- a/lib/librte_security/rte_security_driver.h
+++ b/lib/librte_security/rte_security_driver.h
@@ -1,34 +1,6 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2017 Intel Corporation. All rights reserved.
- *   Copyright 2017 NXP.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 NXP.
+ * Copyright(c) 2017 Intel Corporation.
  */
 
 #ifndef _RTE_SECURITY_DRIVER_H_
diff --git a/lib/librte_vhost/iotlb.c b/lib/librte_vhost/iotlb.c
index c11ebcaa..c6354fef 100644
--- a/lib/librte_vhost/iotlb.c
+++ b/lib/librte_vhost/iotlb.c
@@ -303,6 +303,13 @@ out:
 	return vva;
 }
 
+void
+vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq)
+{
+	vhost_user_iotlb_cache_remove_all(vq);
+	vhost_user_iotlb_pending_remove_all(vq);
+}
+
 int
 vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
 {
@@ -315,8 +322,7 @@ vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
 		 * The cache has already been initialized,
 		 * just drop all cached and pending entries.
 		 */
-		vhost_user_iotlb_cache_remove_all(vq);
-		vhost_user_iotlb_pending_remove_all(vq);
+		vhost_user_iotlb_flush_all(vq);
 	}
 
 #ifdef RTE_LIBRTE_VHOST_NUMA
diff --git a/lib/librte_vhost/iotlb.h b/lib/librte_vhost/iotlb.h
index e7083e37..60b9e4c5 100644
--- a/lib/librte_vhost/iotlb.h
+++ b/lib/librte_vhost/iotlb.h
@@ -73,7 +73,7 @@ void vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq, uint64_t iova,
 						uint8_t perm);
 void vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq, uint64_t iova,
 						uint64_t size, uint8_t perm);
-
+void vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq);
 int vhost_user_iotlb_init(struct virtio_net *dev, int vq_index);
 
 #endif /* _VHOST_IOTLB_H_ */
diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
index 7f0cb9bc..b02673d4 100644
--- a/lib/librte_vhost/rte_vhost.h
+++ b/lib/librte_vhost/rte_vhost.h
@@ -58,6 +58,14 @@ extern "C" {
 #define VHOST_USER_PROTOCOL_F_CRYPTO_SESSION 7
 #endif
 
+#ifndef VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD
+#define VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD 10
+#endif
+
+#ifndef VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
+#define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER 11
+#endif
+
 /** Indicate whether protocol features negotiation is supported. */
 #ifndef VHOST_USER_F_PROTOCOL_FEATURES
 #define VHOST_USER_F_PROTOCOL_FEATURES	30
diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
index 0399c37b..d6303174 100644
--- a/lib/librte_vhost/socket.c
+++ b/lib/librte_vhost/socket.c
@@ -853,6 +853,12 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
 	vsocket->supported_features = VIRTIO_NET_SUPPORTED_FEATURES;
 	vsocket->features           = VIRTIO_NET_SUPPORTED_FEATURES;
 
+	/* Dequeue zero copy can't assure descriptors returned in order */
+	if (vsocket->dequeue_zero_copy) {
+		vsocket->supported_features &= ~(1ULL << VIRTIO_F_IN_ORDER);
+		vsocket->features &= ~(1ULL << VIRTIO_F_IN_ORDER);
+	}
+
 	if (!(flags & RTE_VHOST_USER_IOMMU_SUPPORT)) {
 		vsocket->supported_features &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 		vsocket->features &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index afded495..3c9be10a 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -93,9 +93,12 @@ cleanup_device(struct virtio_net *dev, int destroy)
 }
 
 void
-free_vq(struct vhost_virtqueue *vq)
+free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)
 {
-	rte_free(vq->shadow_used_ring);
+	if (vq_is_packed(dev))
+		rte_free(vq->shadow_used_packed);
+	else
+		rte_free(vq->shadow_used_split);
 	rte_free(vq->batch_copy_elems);
 	rte_mempool_free(vq->iotlb_pool);
 	rte_free(vq);
@@ -110,19 +113,16 @@ free_device(struct virtio_net *dev)
 	uint32_t i;
 
 	for (i = 0; i < dev->nr_vring; i++)
-		free_vq(dev->virtqueue[i]);
+		free_vq(dev, dev->virtqueue[i]);
 
 	rte_free(dev);
 }
 
-int
-vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
+static int
+vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
 {
 	uint64_t req_size, size;
 
-	if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
-		goto out;
-
 	req_size = sizeof(struct vring_desc) * vq->size;
 	size = req_size;
 	vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq,
@@ -153,6 +153,55 @@ vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 	if (!vq->used || size != req_size)
 		return -1;
 
+	return 0;
+}
+
+static int
+vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq)
+{
+	uint64_t req_size, size;
+
+	req_size = sizeof(struct vring_packed_desc) * vq->size;
+	size = req_size;
+	vq->desc_packed = (struct vring_packed_desc *)(uintptr_t)
+		vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr,
+				&size, VHOST_ACCESS_RW);
+	if (!vq->desc_packed || size != req_size)
+		return -1;
+
+	req_size = sizeof(struct vring_packed_desc_event);
+	size = req_size;
+	vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t)
+		vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr,
+				&size, VHOST_ACCESS_RW);
+	if (!vq->driver_event || size != req_size)
+		return -1;
+
+	req_size = sizeof(struct vring_packed_desc_event);
+	size = req_size;
+	vq->device_event = (struct vring_packed_desc_event *)(uintptr_t)
+		vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr,
+				&size, VHOST_ACCESS_RW);
+	if (!vq->device_event || size != req_size)
+		return -1;
+
+	return 0;
+}
+
+int
+vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
+{
+
+	if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
+		goto out;
+
+	if (vq_is_packed(dev)) {
+		if (vring_translate_packed(dev, vq) < 0)
+			return -1;
+	} else {
+		if (vring_translate_split(dev, vq) < 0)
+			return -1;
+	}
 out:
 	vq->access_ok = 1;
 
@@ -234,6 +283,9 @@ alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 	dev->virtqueue[vring_idx] = vq;
 	init_vring_queue(dev, vring_idx);
 	rte_spinlock_init(&vq->access_lock);
+	vq->avail_wrap_counter = 1;
+	vq->used_wrap_counter = 1;
+	vq->signalled_used_valid = false;
 
 	dev->nr_vring += 1;
 
@@ -268,21 +320,21 @@ vhost_new_device(void)
 	struct virtio_net *dev;
 	int i;
 
-	dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0);
-	if (dev == NULL) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"Failed to allocate memory for new dev.\n");
-		return -1;
-	}
-
 	for (i = 0; i < MAX_VHOST_DEVICE; i++) {
 		if (vhost_devices[i] == NULL)
 			break;
 	}
+
 	if (i == MAX_VHOST_DEVICE) {
 		RTE_LOG(ERR, VHOST_CONFIG,
 			"Failed to find a free slot for new device.\n");
-		rte_free(dev);
+		return -1;
+	}
+
+	dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0);
+	if (dev == NULL) {
+		RTE_LOG(ERR, VHOST_CONFIG,
+			"Failed to allocate memory for new dev.\n");
 		return -1;
 	}
 
@@ -291,10 +343,27 @@ vhost_new_device(void)
 	dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET;
 	dev->slave_req_fd = -1;
 	dev->vdpa_dev_id = -1;
+	rte_spinlock_init(&dev->slave_req_lock);
 
 	return i;
 }
 
+void
+vhost_destroy_device_notify(struct virtio_net *dev)
+{
+	struct rte_vdpa_device *vdpa_dev;
+	int did;
+
+	if (dev->flags & VIRTIO_DEV_RUNNING) {
+		did = dev->vdpa_dev_id;
+		vdpa_dev = rte_vdpa_get_device(did);
+		if (vdpa_dev && vdpa_dev->ops->dev_close)
+			vdpa_dev->ops->dev_close(dev->vid);
+		dev->flags &= ~VIRTIO_DEV_RUNNING;
+		dev->notify_ops->destroy_device(dev->vid);
+	}
+}
+
 /*
  * Invoked when there is the vhost-user connection is broken (when
  * the virtio device is being detached).
@@ -303,20 +372,11 @@ void
 vhost_destroy_device(int vid)
 {
 	struct virtio_net *dev = get_device(vid);
-	struct rte_vdpa_device *vdpa_dev;
-	int did = -1;
 
 	if (dev == NULL)
 		return;
 
-	if (dev->flags & VIRTIO_DEV_RUNNING) {
-		did = dev->vdpa_dev_id;
-		vdpa_dev = rte_vdpa_get_device(did);
-		if (vdpa_dev && vdpa_dev->ops->dev_close)
-			vdpa_dev->ops->dev_close(dev->vid);
-		dev->flags &= ~VIRTIO_DEV_RUNNING;
-		dev->notify_ops->destroy_device(vid);
-	}
+	vhost_destroy_device_notify(dev);
 
 	cleanup_device(dev, 1);
 	free_device(dev);
@@ -346,6 +406,8 @@ vhost_detach_vdpa_device(int vid)
 	if (dev == NULL)
 		return;
 
+	vhost_user_host_notifier_ctrl(vid, false);
+
 	dev->vdpa_dev_id = -1;
 }
 
@@ -558,7 +620,11 @@ rte_vhost_vring_call(int vid, uint16_t vring_idx)
 	if (!vq)
 		return -1;
 
-	vhost_vring_call(dev, vq);
+	if (vq_is_packed(dev))
+		vhost_vring_call_packed(dev, vq);
+	else
+		vhost_vring_call_split(dev, vq);
+
 	return 0;
 }
 
@@ -579,19 +645,52 @@ rte_vhost_avail_entries(int vid, uint16_t queue_id)
 	return *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx;
 }
 
+static inline void
+vhost_enable_notify_split(struct vhost_virtqueue *vq, int enable)
+{
+	if (enable)
+		vq->used->flags &= ~VRING_USED_F_NO_NOTIFY;
+	else
+		vq->used->flags |= VRING_USED_F_NO_NOTIFY;
+}
+
+static inline void
+vhost_enable_notify_packed(struct virtio_net *dev,
+		struct vhost_virtqueue *vq, int enable)
+{
+	uint16_t flags;
+
+	if (!enable)
+		vq->device_event->flags = VRING_EVENT_F_DISABLE;
+
+	flags = VRING_EVENT_F_ENABLE;
+	if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
+		flags = VRING_EVENT_F_DESC;
+		vq->device_event->off_wrap = vq->last_avail_idx |
+			vq->avail_wrap_counter << 15;
+	}
+
+	rte_smp_wmb();
+
+	vq->device_event->flags = flags;
+}
+
 int
 rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
 {
 	struct virtio_net *dev = get_device(vid);
+	struct vhost_virtqueue *vq;
 
 	if (!dev)
 		return -1;
 
-	if (enable)
-		dev->virtqueue[queue_id]->used->flags &=
-			~VRING_USED_F_NO_NOTIFY;
+	vq = dev->virtqueue[queue_id];
+
+	if (vq_is_packed(dev))
+		vhost_enable_notify_packed(dev, vq, enable);
 	else
-		dev->virtqueue[queue_id]->used->flags |= VRING_USED_F_NO_NOTIFY;
+		vhost_enable_notify_split(vq, enable);
+
 	return 0;
 }
 
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 58c425a5..760a09c0 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -43,6 +43,7 @@
  * from vring to do scatter RX.
  */
 struct buf_vector {
+	uint64_t buf_iova;
 	uint64_t buf_addr;
 	uint32_t buf_len;
 	uint32_t desc_idx;
@@ -55,6 +56,7 @@ struct buf_vector {
 struct zcopy_mbuf {
 	struct rte_mbuf *mbuf;
 	uint32_t desc_idx;
+	uint16_t desc_count;
 	uint16_t in_use;
 
 	TAILQ_ENTRY(zcopy_mbuf) next;
@@ -79,19 +81,35 @@ struct log_cache_entry {
 	unsigned long val;
 };
 
+struct vring_used_elem_packed {
+	uint16_t id;
+	uint32_t len;
+	uint32_t count;
+};
+
 /**
  * Structure contains variables relevant to RX/TX virtqueues.
  */
 struct vhost_virtqueue {
-	struct vring_desc	*desc;
-	struct vring_avail	*avail;
-	struct vring_used	*used;
+	union {
+		struct vring_desc	*desc;
+		struct vring_packed_desc   *desc_packed;
+	};
+	union {
+		struct vring_avail	*avail;
+		struct vring_packed_desc_event *driver_event;
+	};
+	union {
+		struct vring_used	*used;
+		struct vring_packed_desc_event *device_event;
+	};
 	uint32_t		size;
 
 	uint16_t		last_avail_idx;
 	uint16_t		last_used_idx;
 	/* Last used index we notify to front end. */
 	uint16_t		signalled_used;
+	bool			signalled_used_valid;
 #define VIRTIO_INVALID_EVENTFD		(-1)
 #define VIRTIO_UNINITIALIZED_EVENTFD	(-2)
 
@@ -115,12 +133,17 @@ struct vhost_virtqueue {
 	struct zcopy_mbuf	*zmbufs;
 	struct zcopy_mbuf_list	zmbuf_list;
 
-	struct vring_used_elem  *shadow_used_ring;
+	union {
+		struct vring_used_elem  *shadow_used_split;
+		struct vring_used_elem_packed *shadow_used_packed;
+	};
 	uint16_t                shadow_used_idx;
 	struct vhost_vring_addr ring_addrs;
 
 	struct batch_copy_elem	*batch_copy_elems;
 	uint16_t		batch_copy_nb_elems;
+	bool			used_wrap_counter;
+	bool			avail_wrap_counter;
 
 	struct log_cache_entry log_cache[VHOST_LOG_CACHE_NR];
 	uint16_t log_cache_nb_elem;
@@ -191,6 +214,42 @@ struct vhost_msg {
  #define VIRTIO_F_VERSION_1 32
 #endif
 
+/* Declare packed ring related bits for older kernels */
+#ifndef VIRTIO_F_RING_PACKED
+
+#define VIRTIO_F_RING_PACKED 34
+
+#define VRING_DESC_F_NEXT	1
+#define VRING_DESC_F_WRITE	2
+#define VRING_DESC_F_INDIRECT	4
+
+#define VRING_DESC_F_AVAIL	(1ULL << 7)
+#define VRING_DESC_F_USED	(1ULL << 15)
+
+struct vring_packed_desc {
+	uint64_t addr;
+	uint32_t len;
+	uint16_t id;
+	uint16_t flags;
+};
+
+#define VRING_EVENT_F_ENABLE 0x0
+#define VRING_EVENT_F_DISABLE 0x1
+#define VRING_EVENT_F_DESC 0x2
+
+struct vring_packed_desc_event {
+	uint16_t off_wrap;
+	uint16_t flags;
+};
+#endif
+
+/*
+ * Available and used descs are in same order
+ */
+#ifndef VIRTIO_F_IN_ORDER
+#define VIRTIO_F_IN_ORDER      35
+#endif
+
 /* Features supported by this builtin vhost-user net driver. */
 #define VIRTIO_NET_SUPPORTED_FEATURES ((1ULL << VIRTIO_NET_F_MRG_RXBUF) | \
 				(1ULL << VIRTIO_F_ANY_LAYOUT) | \
@@ -214,7 +273,8 @@ struct vhost_msg {
 				(1ULL << VIRTIO_NET_F_GUEST_ECN) | \
 				(1ULL << VIRTIO_RING_F_INDIRECT_DESC) | \
 				(1ULL << VIRTIO_RING_F_EVENT_IDX) | \
-				(1ULL << VIRTIO_NET_F_MTU) | \
+				(1ULL << VIRTIO_NET_F_MTU)  | \
+				(1ULL << VIRTIO_F_IN_ORDER) | \
 				(1ULL << VIRTIO_F_IOMMU_PLATFORM))
 
 
@@ -301,6 +361,7 @@ struct virtio_net {
 	struct guest_page       *guest_pages;
 
 	int			slave_req_fd;
+	rte_spinlock_t		slave_req_lock;
 
 	/*
 	 * Device id to identify a specific backend device.
@@ -314,6 +375,19 @@ struct virtio_net {
 	struct vhost_user_extern_ops extern_ops;
 } __rte_cache_aligned;
 
+static __rte_always_inline bool
+vq_is_packed(struct virtio_net *dev)
+{
+	return dev->features & (1ull << VIRTIO_F_RING_PACKED);
+}
+
+static inline bool
+desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
+{
+	return wrap_counter == !!(desc->flags & VRING_DESC_F_AVAIL) &&
+		wrap_counter != !!(desc->flags & VRING_DESC_F_USED);
+}
+
 #define VHOST_LOG_PAGE	4096
 
 /*
@@ -428,6 +502,7 @@ vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
 	vq->log_cache[i].offset = offset;
 	vq->log_cache[i].val = (1UL << bit_nr);
+	vq->log_cache_nb_elem++;
 }
 
 static __rte_always_inline void
@@ -535,9 +610,10 @@ int vhost_new_device(void);
 void cleanup_device(struct virtio_net *dev, int destroy);
 void reset_device(struct virtio_net *dev);
 void vhost_destroy_device(int);
+void vhost_destroy_device_notify(struct virtio_net *dev);
 
 void cleanup_vq(struct vhost_virtqueue *vq, int destroy);
-void free_vq(struct vhost_virtqueue *vq);
+void free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq);
 
 int alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx);
 
@@ -588,10 +664,10 @@ vhost_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old)
 }
 
 static __rte_always_inline void
-vhost_vring_call(struct virtio_net *dev, struct vhost_virtqueue *vq)
+vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
 {
 	/* Flush used->idx update before we read avail->flags. */
-	rte_mb();
+	rte_smp_mb();
 
 	/* Don't kick guest if we don't reach index specified by guest. */
 	if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
@@ -615,4 +691,55 @@ vhost_vring_call(struct virtio_net *dev, struct vhost_virtqueue *vq)
 	}
 }
 
+static __rte_always_inline void
+vhost_vring_call_packed(struct virtio_net *dev, struct vhost_virtqueue *vq)
+{
+	uint16_t old, new, off, off_wrap;
+	bool signalled_used_valid, kick = false;
+
+	/* Flush used desc update. */
+	rte_smp_mb();
+
+	if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) {
+		if (vq->driver_event->flags !=
+				VRING_EVENT_F_DISABLE)
+			kick = true;
+		goto kick;
+	}
+
+	old = vq->signalled_used;
+	new = vq->last_used_idx;
+	vq->signalled_used = new;
+	signalled_used_valid = vq->signalled_used_valid;
+	vq->signalled_used_valid = true;
+
+	if (vq->driver_event->flags != VRING_EVENT_F_DESC) {
+		if (vq->driver_event->flags != VRING_EVENT_F_DISABLE)
+			kick = true;
+		goto kick;
+	}
+
+	if (unlikely(!signalled_used_valid)) {
+		kick = true;
+		goto kick;
+	}
+
+	rte_smp_rmb();
+
+	off_wrap = vq->driver_event->off_wrap;
+	off = off_wrap & ~(1 << 15);
+
+	if (new <= old)
+		old -= vq->size;
+
+	if (vq->used_wrap_counter != off_wrap >> 15)
+		off -= vq->size;
+
+	if (vhost_need_event(off, new, old))
+		kick = true;
+kick:
+	if (kick)
+		eventfd_write(vq->callfd, (eventfd_t)1);
+}
+
 #endif /* _VHOST_NET_CDEV_H_ */
diff --git a/lib/librte_vhost/vhost_crypto.c b/lib/librte_vhost/vhost_crypto.c
index f1650738..57341ef8 100644
--- a/lib/librte_vhost/vhost_crypto.c
+++ b/lib/librte_vhost/vhost_crypto.c
@@ -940,8 +940,7 @@ vhost_crypto_process_one_req(struct vhost_crypto *vcrypto,
 		struct vhost_virtqueue *vq, struct rte_crypto_op *op,
 		struct vring_desc *head, uint16_t desc_idx)
 {
-	struct vhost_crypto_data_req *vc_req = RTE_PTR_ADD(op->sym->m_src,
-			sizeof(struct rte_mbuf));
+	struct vhost_crypto_data_req *vc_req = rte_mbuf_to_priv(op->sym->m_src);
 	struct rte_cryptodev_sym_session *session;
 	struct virtio_crypto_op_data_req *req, tmp_req;
 	struct virtio_crypto_inhdr *inhdr;
@@ -1062,8 +1061,7 @@ vhost_crypto_finalize_one_request(struct rte_crypto_op *op,
 {
 	struct rte_mbuf *m_src = op->sym->m_src;
 	struct rte_mbuf *m_dst = op->sym->m_dst;
-	struct vhost_crypto_data_req *vc_req = RTE_PTR_ADD(m_src,
-			sizeof(struct rte_mbuf));
+	struct vhost_crypto_data_req *vc_req = rte_mbuf_to_priv(m_src);
 	uint16_t desc_idx;
 	int ret = 0;
 
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index 947290fc..a2d4c9ff 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -135,17 +135,7 @@ vhost_user_set_owner(void)
 static int
 vhost_user_reset_owner(struct virtio_net *dev)
 {
-	struct rte_vdpa_device *vdpa_dev;
-	int did = -1;
-
-	if (dev->flags & VIRTIO_DEV_RUNNING) {
-		did = dev->vdpa_dev_id;
-		vdpa_dev = rte_vdpa_get_device(did);
-		if (vdpa_dev && vdpa_dev->ops->dev_close)
-			vdpa_dev->ops->dev_close(dev->vid);
-		dev->flags &= ~VIRTIO_DEV_RUNNING;
-		dev->notify_ops->destroy_device(dev->vid);
-	}
+	vhost_destroy_device_notify(dev);
 
 	cleanup_device(dev, 0);
 	reset_device(dev);
@@ -243,7 +233,7 @@ vhost_user_set_features(struct virtio_net *dev, uint64_t features)
 
 			dev->virtqueue[dev->nr_vring] = NULL;
 			cleanup_vq(vq, 1);
-			free_vq(vq);
+			free_vq(dev, vq);
 		}
 	}
 
@@ -292,13 +282,26 @@ vhost_user_set_vring_num(struct virtio_net *dev,
 		TAILQ_INIT(&vq->zmbuf_list);
 	}
 
-	vq->shadow_used_ring = rte_malloc(NULL,
+	if (vq_is_packed(dev)) {
+		vq->shadow_used_packed = rte_malloc(NULL,
+				vq->size *
+				sizeof(struct vring_used_elem_packed),
+				RTE_CACHE_LINE_SIZE);
+		if (!vq->shadow_used_packed) {
+			RTE_LOG(ERR, VHOST_CONFIG,
+					"failed to allocate memory for shadow used ring.\n");
+			return -1;
+		}
+
+	} else {
+		vq->shadow_used_split = rte_malloc(NULL,
 				vq->size * sizeof(struct vring_used_elem),
 				RTE_CACHE_LINE_SIZE);
-	if (!vq->shadow_used_ring) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"failed to allocate memory for shadow used ring.\n");
-		return -1;
+		if (!vq->shadow_used_split) {
+			RTE_LOG(ERR, VHOST_CONFIG,
+					"failed to allocate memory for shadow used ring.\n");
+			return -1;
+		}
 	}
 
 	vq->batch_copy_elems = rte_malloc(NULL,
@@ -325,7 +328,8 @@ numa_realloc(struct virtio_net *dev, int index)
 	struct virtio_net *old_dev;
 	struct vhost_virtqueue *old_vq, *vq;
 	struct zcopy_mbuf *new_zmbuf;
-	struct vring_used_elem *new_shadow_used_ring;
+	struct vring_used_elem *new_shadow_used_split;
+	struct vring_used_elem_packed *new_shadow_used_packed;
 	struct batch_copy_elem *new_batch_copy_elems;
 	int ret;
 
@@ -360,13 +364,26 @@ numa_realloc(struct virtio_net *dev, int index)
 			vq->zmbufs = new_zmbuf;
 		}
 
-		new_shadow_used_ring = rte_malloc_socket(NULL,
-			vq->size * sizeof(struct vring_used_elem),
-			RTE_CACHE_LINE_SIZE,
-			newnode);
-		if (new_shadow_used_ring) {
-			rte_free(vq->shadow_used_ring);
-			vq->shadow_used_ring = new_shadow_used_ring;
+		if (vq_is_packed(dev)) {
+			new_shadow_used_packed = rte_malloc_socket(NULL,
+					vq->size *
+					sizeof(struct vring_used_elem_packed),
+					RTE_CACHE_LINE_SIZE,
+					newnode);
+			if (new_shadow_used_packed) {
+				rte_free(vq->shadow_used_packed);
+				vq->shadow_used_packed = new_shadow_used_packed;
+			}
+		} else {
+			new_shadow_used_split = rte_malloc_socket(NULL,
+					vq->size *
+					sizeof(struct vring_used_elem),
+					RTE_CACHE_LINE_SIZE,
+					newnode);
+			if (new_shadow_used_split) {
+				rte_free(vq->shadow_used_split);
+				vq->shadow_used_split = new_shadow_used_split;
+			}
 		}
 
 		new_batch_copy_elems = rte_malloc_socket(NULL,
@@ -477,6 +494,51 @@ translate_ring_addresses(struct virtio_net *dev, int vq_index)
 	struct vhost_vring_addr *addr = &vq->ring_addrs;
 	uint64_t len;
 
+	if (vq_is_packed(dev)) {
+		len = sizeof(struct vring_packed_desc) * vq->size;
+		vq->desc_packed = (struct vring_packed_desc *)(uintptr_t)
+			ring_addr_to_vva(dev, vq, addr->desc_user_addr, &len);
+		vq->log_guest_addr = 0;
+		if (vq->desc_packed == NULL ||
+				len != sizeof(struct vring_packed_desc) *
+				vq->size) {
+			RTE_LOG(DEBUG, VHOST_CONFIG,
+				"(%d) failed to map desc_packed ring.\n",
+				dev->vid);
+			return dev;
+		}
+
+		dev = numa_realloc(dev, vq_index);
+		vq = dev->virtqueue[vq_index];
+		addr = &vq->ring_addrs;
+
+		len = sizeof(struct vring_packed_desc_event);
+		vq->driver_event = (struct vring_packed_desc_event *)
+					(uintptr_t)ring_addr_to_vva(dev,
+					vq, addr->avail_user_addr, &len);
+		if (vq->driver_event == NULL ||
+				len != sizeof(struct vring_packed_desc_event)) {
+			RTE_LOG(DEBUG, VHOST_CONFIG,
+				"(%d) failed to find driver area address.\n",
+				dev->vid);
+			return dev;
+		}
+
+		len = sizeof(struct vring_packed_desc_event);
+		vq->device_event = (struct vring_packed_desc_event *)
+					(uintptr_t)ring_addr_to_vva(dev,
+					vq, addr->used_user_addr, &len);
+		if (vq->device_event == NULL ||
+				len != sizeof(struct vring_packed_desc_event)) {
+			RTE_LOG(DEBUG, VHOST_CONFIG,
+				"(%d) failed to find device area address.\n",
+				dev->vid);
+			return dev;
+		}
+
+		return dev;
+	}
+
 	/* The addresses are converted from QEMU virtual to Vhost virtual. */
 	if (vq->desc && vq->avail && vq->used)
 		return dev;
@@ -751,6 +813,11 @@ vhost_user_set_mem_table(struct virtio_net **pdev, struct VhostUserMsg *pmsg)
 		dev->mem = NULL;
 	}
 
+	/* Flush IOTLB cache as previous HVAs are now invalid */
+	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+		for (i = 0; i < dev->nr_vring; i++)
+			vhost_user_iotlb_flush_all(dev->virtqueue[i]);
+
 	dev->nr_guest_pages = 0;
 	if (!dev->guest_pages) {
 		dev->max_guest_pages = 8;
@@ -885,10 +952,20 @@ err_mmap:
 	return -1;
 }
 
-static int
-vq_is_ready(struct vhost_virtqueue *vq)
+static bool
+vq_is_ready(struct virtio_net *dev, struct vhost_virtqueue *vq)
 {
-	return vq && vq->desc && vq->avail && vq->used &&
+	bool rings_ok;
+
+	if (!vq)
+		return false;
+
+	if (vq_is_packed(dev))
+		rings_ok = !!vq->desc_packed;
+	else
+		rings_ok = vq->desc && vq->avail && vq->used;
+
+	return rings_ok &&
 	       vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD &&
 	       vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD;
 }
@@ -905,7 +982,7 @@ virtio_is_ready(struct virtio_net *dev)
 	for (i = 0; i < dev->nr_vring; i++) {
 		vq = dev->virtqueue[i];
 
-		if (!vq_is_ready(vq))
+		if (!vq_is_ready(dev, vq))
 			return 0;
 	}
 
@@ -996,18 +1073,9 @@ vhost_user_get_vring_base(struct virtio_net *dev,
 			  VhostUserMsg *msg)
 {
 	struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
-	struct rte_vdpa_device *vdpa_dev;
-	int did = -1;
 
 	/* We have to stop the queue (virtio) if it is running. */
-	if (dev->flags & VIRTIO_DEV_RUNNING) {
-		did = dev->vdpa_dev_id;
-		vdpa_dev = rte_vdpa_get_device(did);
-		if (vdpa_dev && vdpa_dev->ops->dev_close)
-			vdpa_dev->ops->dev_close(dev->vid);
-		dev->flags &= ~VIRTIO_DEV_RUNNING;
-		dev->notify_ops->destroy_device(dev->vid);
-	}
+	vhost_destroy_device_notify(dev);
 
 	dev->flags &= ~VIRTIO_DEV_READY;
 	dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;
@@ -1035,8 +1103,13 @@ vhost_user_get_vring_base(struct virtio_net *dev,
 
 	if (dev->dequeue_zero_copy)
 		free_zmbufs(vq);
-	rte_free(vq->shadow_used_ring);
-	vq->shadow_used_ring = NULL;
+	if (vq_is_packed(dev)) {
+		rte_free(vq->shadow_used_packed);
+		vq->shadow_used_packed = NULL;
+	} else {
+		rte_free(vq->shadow_used_split);
+		vq->shadow_used_split = NULL;
+	}
 
 	rte_free(vq->batch_copy_elems);
 	vq->batch_copy_elems = NULL;
@@ -1384,6 +1457,22 @@ send_vhost_reply(int sockfd, struct VhostUserMsg *msg)
 	return send_vhost_message(sockfd, msg, NULL, 0);
 }
 
+static int
+send_vhost_slave_message(struct virtio_net *dev, struct VhostUserMsg *msg,
+			 int *fds, int fd_num)
+{
+	int ret;
+
+	if (msg->flags & VHOST_USER_NEED_REPLY)
+		rte_spinlock_lock(&dev->slave_req_lock);
+
+	ret = send_vhost_message(dev->slave_req_fd, msg, fds, fd_num);
+	if (ret < 0 && (msg->flags & VHOST_USER_NEED_REPLY))
+		rte_spinlock_unlock(&dev->slave_req_lock);
+
+	return ret;
+}
+
 /*
  * Allocate a queue pair if it hasn't been allocated yet
  */
@@ -1705,11 +1794,45 @@ skip_to_reply:
 		if (vdpa_dev->ops->dev_conf)
 			vdpa_dev->ops->dev_conf(dev->vid);
 		dev->flags |= VIRTIO_DEV_VDPA_CONFIGURED;
+		if (vhost_user_host_notifier_ctrl(dev->vid, true) != 0) {
+			RTE_LOG(INFO, VHOST_CONFIG,
+				"(%d) software relay is used for vDPA, performance may be low.\n",
+				dev->vid);
+		}
 	}
 
 	return 0;
 }
 
+static int process_slave_message_reply(struct virtio_net *dev,
+				       const VhostUserMsg *msg)
+{
+	VhostUserMsg msg_reply;
+	int ret;
+
+	if ((msg->flags & VHOST_USER_NEED_REPLY) == 0)
+		return 0;
+
+	if (read_vhost_message(dev->slave_req_fd, &msg_reply) < 0) {
+		ret = -1;
+		goto out;
+	}
+
+	if (msg_reply.request.slave != msg->request.slave) {
+		RTE_LOG(ERR, VHOST_CONFIG,
+			"Received unexpected msg type (%u), expected %u\n",
+			msg_reply.request.slave, msg->request.slave);
+		ret = -1;
+		goto out;
+	}
+
+	ret = msg_reply.payload.u64 ? -1 : 0;
+
+out:
+	rte_spinlock_unlock(&dev->slave_req_lock);
+	return ret;
+}
+
 int
 vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm)
 {
@@ -1735,3 +1858,101 @@ vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm)
 
 	return 0;
 }
+
+static int vhost_user_slave_set_vring_host_notifier(struct virtio_net *dev,
+						    int index, int fd,
+						    uint64_t offset,
+						    uint64_t size)
+{
+	int *fdp = NULL;
+	size_t fd_num = 0;
+	int ret;
+	struct VhostUserMsg msg = {
+		.request.slave = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG,
+		.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY,
+		.size = sizeof(msg.payload.area),
+		.payload.area = {
+			.u64 = index & VHOST_USER_VRING_IDX_MASK,
+			.size = size,
+			.offset = offset,
+		},
+	};
+
+	if (fd < 0)
+		msg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
+	else {
+		fdp = &fd;
+		fd_num = 1;
+	}
+
+	ret = send_vhost_slave_message(dev, &msg, fdp, fd_num);
+	if (ret < 0) {
+		RTE_LOG(ERR, VHOST_CONFIG,
+			"Failed to set host notifier (%d)\n", ret);
+		return ret;
+	}
+
+	return process_slave_message_reply(dev, &msg);
+}
+
+int vhost_user_host_notifier_ctrl(int vid, bool enable)
+{
+	struct virtio_net *dev;
+	struct rte_vdpa_device *vdpa_dev;
+	int vfio_device_fd, did, ret = 0;
+	uint64_t offset, size;
+	unsigned int i;
+
+	dev = get_device(vid);
+	if (!dev)
+		return -ENODEV;
+
+	did = dev->vdpa_dev_id;
+	if (did < 0)
+		return -EINVAL;
+
+	if (!(dev->features & (1ULL << VIRTIO_F_VERSION_1)) ||
+	    !(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)) ||
+	    !(dev->protocol_features &
+			(1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ)) ||
+	    !(dev->protocol_features &
+			(1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD)) ||
+	    !(dev->protocol_features &
+			(1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER)))
+		return -ENOTSUP;
+
+	vdpa_dev = rte_vdpa_get_device(did);
+	if (!vdpa_dev)
+		return -ENODEV;
+
+	RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_vfio_device_fd, -ENOTSUP);
+	RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_notify_area, -ENOTSUP);
+
+	vfio_device_fd = vdpa_dev->ops->get_vfio_device_fd(vid);
+	if (vfio_device_fd < 0)
+		return -ENOTSUP;
+
+	if (enable) {
+		for (i = 0; i < dev->nr_vring; i++) {
+			if (vdpa_dev->ops->get_notify_area(vid, i, &offset,
+					&size) < 0) {
+				ret = -ENOTSUP;
+				goto disable;
+			}
+
+			if (vhost_user_slave_set_vring_host_notifier(dev, i,
+					vfio_device_fd, offset, size) < 0) {
+				ret = -EFAULT;
+				goto disable;
+			}
+		}
+	} else {
+disable:
+		for (i = 0; i < dev->nr_vring; i++) {
+			vhost_user_slave_set_vring_host_notifier(dev, i, -1,
+					0, 0);
+		}
+	}
+
+	return ret;
+}
diff --git a/lib/librte_vhost/vhost_user.h b/lib/librte_vhost/vhost_user.h
index 1ad5cf46..42166adf 100644
--- a/lib/librte_vhost/vhost_user.h
+++ b/lib/librte_vhost/vhost_user.h
@@ -20,7 +20,9 @@
 					 (1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK) | \
 					 (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU) | \
 					 (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
-					 (1ULL << VHOST_USER_PROTOCOL_F_CRYPTO_SESSION))
+					 (1ULL << VHOST_USER_PROTOCOL_F_CRYPTO_SESSION) | \
+					 (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
+					 (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER))
 
 typedef enum VhostUserRequest {
 	VHOST_USER_NONE = 0,
@@ -54,6 +56,7 @@ typedef enum VhostUserRequest {
 typedef enum VhostUserSlaveRequest {
 	VHOST_USER_SLAVE_NONE = 0,
 	VHOST_USER_SLAVE_IOTLB_MSG = 1,
+	VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
 	VHOST_USER_SLAVE_MAX
 } VhostUserSlaveRequest;
 
@@ -99,6 +102,12 @@ typedef struct VhostUserCryptoSessionParam {
 	uint8_t auth_key_buf[VHOST_USER_CRYPTO_MAX_HMAC_KEY_LENGTH];
 } VhostUserCryptoSessionParam;
 
+typedef struct VhostUserVringArea {
+	uint64_t u64;
+	uint64_t size;
+	uint64_t offset;
+} VhostUserVringArea;
+
 typedef struct VhostUserMsg {
 	union {
 		uint32_t master; /* a VhostUserRequest value */
@@ -120,6 +129,7 @@ typedef struct VhostUserMsg {
 		VhostUserLog    log;
 		struct vhost_iotlb_msg iotlb;
 		VhostUserCryptoSessionParam crypto_session;
+		VhostUserVringArea area;
 	} payload;
 	int fds[VHOST_MEMORY_MAX_NREGIONS];
 } __attribute((packed)) VhostUserMsg;
@@ -133,6 +143,7 @@ typedef struct VhostUserMsg {
 /* vhost_user.c */
 int vhost_user_msg_handler(int vid, int fd);
 int vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm);
+int vhost_user_host_notifier_ctrl(int vid, bool enable);
 
 /* socket.c */
 int read_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num);
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 76ec5f08..99c7afc8 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -25,22 +25,27 @@
 
 #define MAX_BATCH_LEN 256
 
+static  __rte_always_inline bool
+rxvq_is_mergeable(struct virtio_net *dev)
+{
+	return dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF);
+}
+
 static bool
 is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t nr_vring)
 {
 	return (is_tx ^ (idx & 1)) == 0 && idx < nr_vring;
 }
 
-static __rte_always_inline struct vring_desc *
+static __rte_always_inline void *
 alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq,
-					 struct vring_desc *desc)
+		uint64_t desc_addr, uint64_t desc_len)
 {
-	struct vring_desc *idesc;
+	void *idesc;
 	uint64_t src, dst;
-	uint64_t len, remain = desc->len;
-	uint64_t desc_addr = desc->addr;
+	uint64_t len, remain = desc_len;
 
-	idesc = rte_malloc(__func__, desc->len, 0);
+	idesc = rte_malloc(__func__, desc_len, 0);
 	if (unlikely(!idesc))
 		return 0;
 
@@ -66,17 +71,18 @@ alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq,
 }
 
 static __rte_always_inline void
-free_ind_table(struct vring_desc *idesc)
+free_ind_table(void *idesc)
 {
 	rte_free(idesc);
 }
 
 static __rte_always_inline void
-do_flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
-			  uint16_t to, uint16_t from, uint16_t size)
+do_flush_shadow_used_ring_split(struct virtio_net *dev,
+			struct vhost_virtqueue *vq,
+			uint16_t to, uint16_t from, uint16_t size)
 {
 	rte_memcpy(&vq->used->ring[to],
-			&vq->shadow_used_ring[from],
+			&vq->shadow_used_split[from],
 			size * sizeof(struct vring_used_elem));
 	vhost_log_cache_used_vring(dev, vq,
 			offsetof(struct vring_used, ring[to]),
@@ -84,22 +90,22 @@ do_flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
 }
 
 static __rte_always_inline void
-flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq)
+flush_shadow_used_ring_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
 {
 	uint16_t used_idx = vq->last_used_idx & (vq->size - 1);
 
 	if (used_idx + vq->shadow_used_idx <= vq->size) {
-		do_flush_shadow_used_ring(dev, vq, used_idx, 0,
+		do_flush_shadow_used_ring_split(dev, vq, used_idx, 0,
 					  vq->shadow_used_idx);
 	} else {
 		uint16_t size;
 
 		/* update used ring interval [used_idx, vq->size] */
 		size = vq->size - used_idx;
-		do_flush_shadow_used_ring(dev, vq, used_idx, 0, size);
+		do_flush_shadow_used_ring_split(dev, vq, used_idx, 0, size);
 
 		/* update the left half used ring interval [0, left_size] */
-		do_flush_shadow_used_ring(dev, vq, 0, size,
+		do_flush_shadow_used_ring_split(dev, vq, 0, size,
 					  vq->shadow_used_idx - size);
 	}
 	vq->last_used_idx += vq->shadow_used_idx;
@@ -109,18 +115,84 @@ flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq)
 	vhost_log_cache_sync(dev, vq);
 
 	*(volatile uint16_t *)&vq->used->idx += vq->shadow_used_idx;
+	vq->shadow_used_idx = 0;
 	vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
 		sizeof(vq->used->idx));
 }
 
 static __rte_always_inline void
-update_shadow_used_ring(struct vhost_virtqueue *vq,
+update_shadow_used_ring_split(struct vhost_virtqueue *vq,
 			 uint16_t desc_idx, uint16_t len)
 {
 	uint16_t i = vq->shadow_used_idx++;
 
-	vq->shadow_used_ring[i].id  = desc_idx;
-	vq->shadow_used_ring[i].len = len;
+	vq->shadow_used_split[i].id  = desc_idx;
+	vq->shadow_used_split[i].len = len;
+}
+
+static __rte_always_inline void
+flush_shadow_used_ring_packed(struct virtio_net *dev,
+			struct vhost_virtqueue *vq)
+{
+	int i;
+	uint16_t used_idx = vq->last_used_idx;
+
+	/* Split loop in two to save memory barriers */
+	for (i = 0; i < vq->shadow_used_idx; i++) {
+		vq->desc_packed[used_idx].id = vq->shadow_used_packed[i].id;
+		vq->desc_packed[used_idx].len = vq->shadow_used_packed[i].len;
+
+		used_idx += vq->shadow_used_packed[i].count;
+		if (used_idx >= vq->size)
+			used_idx -= vq->size;
+	}
+
+	rte_smp_wmb();
+
+	for (i = 0; i < vq->shadow_used_idx; i++) {
+		uint16_t flags;
+
+		if (vq->shadow_used_packed[i].len)
+			flags = VRING_DESC_F_WRITE;
+		else
+			flags = 0;
+
+		if (vq->used_wrap_counter) {
+			flags |= VRING_DESC_F_USED;
+			flags |= VRING_DESC_F_AVAIL;
+		} else {
+			flags &= ~VRING_DESC_F_USED;
+			flags &= ~VRING_DESC_F_AVAIL;
+		}
+
+		vq->desc_packed[vq->last_used_idx].flags = flags;
+
+		vhost_log_cache_used_vring(dev, vq,
+					vq->last_used_idx *
+					sizeof(struct vring_packed_desc),
+					sizeof(struct vring_packed_desc));
+
+		vq->last_used_idx += vq->shadow_used_packed[i].count;
+		if (vq->last_used_idx >= vq->size) {
+			vq->used_wrap_counter ^= 1;
+			vq->last_used_idx -= vq->size;
+		}
+	}
+
+	rte_smp_wmb();
+	vq->shadow_used_idx = 0;
+	vhost_log_cache_sync(dev, vq);
+}
+
+static __rte_always_inline void
+update_shadow_used_ring_packed(struct vhost_virtqueue *vq,
+			 uint16_t desc_idx, uint16_t len, uint16_t count)
+{
+	uint16_t i = vq->shadow_used_idx++;
+
+	vq->shadow_used_packed[i].id  = desc_idx;
+	vq->shadow_used_packed[i].len = len;
+	vq->shadow_used_packed[i].count = count;
 }
 
 static inline void
@@ -135,6 +207,8 @@ do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq)
 		vhost_log_cache_write(dev, vq, elem[i].log_addr, elem[i].len);
 		PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0);
 	}
+
+	vq->batch_copy_nb_elems = 0;
 }
 
 static inline void
@@ -146,6 +220,8 @@ do_data_copy_dequeue(struct vhost_virtqueue *vq)
 
 	for (i = 0; i < count; i++)
 		rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
+
+	vq->batch_copy_nb_elems = 0;
 }
 
 /* avoid write operation when necessary, to lessen cache issues */
@@ -154,7 +230,7 @@ do_data_copy_dequeue(struct vhost_virtqueue *vq)
 		(var) = (val);			\
 } while (0)
 
-static void
+static __rte_always_inline void
 virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)
 {
 	uint64_t csum_l4 = m_buf->ol_flags & PKT_TX_L4_MASK;
@@ -216,324 +292,47 @@ virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)
 }
 
 static __rte_always_inline int
-copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
-		  struct vring_desc *descs, struct rte_mbuf *m,
-		  uint16_t desc_idx, uint32_t size)
+map_one_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
+		struct buf_vector *buf_vec, uint16_t *vec_idx,
+		uint64_t desc_iova, uint64_t desc_len, uint8_t perm)
 {
-	uint32_t desc_avail, desc_offset;
-	uint32_t mbuf_avail, mbuf_offset;
-	uint32_t cpy_len;
-	uint64_t desc_chunck_len;
-	struct vring_desc *desc;
-	uint64_t desc_addr, desc_gaddr;
-	/* A counter to avoid desc dead loop chain */
-	uint16_t nr_desc = 1;
-	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
-	uint16_t copy_nb = vq->batch_copy_nb_elems;
-	int error = 0;
+	uint16_t vec_id = *vec_idx;
 
-	desc = &descs[desc_idx];
-	desc_chunck_len = desc->len;
-	desc_gaddr = desc->addr;
-	desc_addr = vhost_iova_to_vva(dev, vq, desc_gaddr,
-					&desc_chunck_len, VHOST_ACCESS_RW);
-	/*
-	 * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid
-	 * performance issue with some versions of gcc (4.8.4 and 5.3.0) which
-	 * otherwise stores offset on the stack instead of in a register.
-	 */
-	if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr) {
-		error = -1;
-		goto out;
-	}
+	while (desc_len) {
+		uint64_t desc_addr;
+		uint64_t desc_chunck_len = desc_len;
 
-	rte_prefetch0((void *)(uintptr_t)desc_addr);
-
-	if (likely(desc_chunck_len >= dev->vhost_hlen)) {
-		virtio_enqueue_offload(m,
-				(struct virtio_net_hdr *)(uintptr_t)desc_addr);
-		PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0);
-		vhost_log_cache_write(dev, vq, desc_gaddr, dev->vhost_hlen);
-	} else {
-		struct virtio_net_hdr vnet_hdr;
-		uint64_t remain = dev->vhost_hlen;
-		uint64_t len;
-		uint64_t src = (uint64_t)(uintptr_t)&vnet_hdr, dst;
-		uint64_t guest_addr = desc_gaddr;
-
-		virtio_enqueue_offload(m, &vnet_hdr);
-
-		while (remain) {
-			len = remain;
-			dst = vhost_iova_to_vva(dev, vq, guest_addr,
-					&len, VHOST_ACCESS_RW);
-			if (unlikely(!dst || !len)) {
-				error = -1;
-				goto out;
-			}
-
-			rte_memcpy((void *)(uintptr_t)dst,
-					(void *)(uintptr_t)src, len);
-
-			PRINT_PACKET(dev, (uintptr_t)dst, (uint32_t)len, 0);
-			vhost_log_cache_write(dev, vq, guest_addr, len);
-			remain -= len;
-			guest_addr += len;
-			src += len;
-		}
-	}
+		if (unlikely(vec_id >= BUF_VECTOR_MAX))
+			return -1;
 
-	desc_avail  = desc->len - dev->vhost_hlen;
-	if (unlikely(desc_chunck_len < dev->vhost_hlen)) {
-		desc_chunck_len = desc_avail;
-		desc_gaddr = desc->addr + dev->vhost_hlen;
-		desc_addr = vhost_iova_to_vva(dev,
-				vq, desc_gaddr,
+		desc_addr = vhost_iova_to_vva(dev, vq,
+				desc_iova,
 				&desc_chunck_len,
-				VHOST_ACCESS_RW);
-		if (unlikely(!desc_addr)) {
-			error = -1;
-			goto out;
-		}
-
-		desc_offset = 0;
-	} else {
-		desc_offset = dev->vhost_hlen;
-		desc_chunck_len -= dev->vhost_hlen;
-	}
-
-	mbuf_avail  = rte_pktmbuf_data_len(m);
-	mbuf_offset = 0;
-	while (mbuf_avail != 0 || m->next != NULL) {
-		/* done with current mbuf, fetch next */
-		if (mbuf_avail == 0) {
-			m = m->next;
-
-			mbuf_offset = 0;
-			mbuf_avail  = rte_pktmbuf_data_len(m);
-		}
-
-		/* done with current desc buf, fetch next */
-		if (desc_avail == 0) {
-			if ((desc->flags & VRING_DESC_F_NEXT) == 0) {
-				/* Room in vring buffer is not enough */
-				error = -1;
-				goto out;
-			}
-			if (unlikely(desc->next >= size || ++nr_desc > size)) {
-				error = -1;
-				goto out;
-			}
-
-			desc = &descs[desc->next];
-			desc_chunck_len = desc->len;
-			desc_gaddr = desc->addr;
-			desc_addr = vhost_iova_to_vva(dev, vq, desc_gaddr,
-							&desc_chunck_len,
-							VHOST_ACCESS_RW);
-			if (unlikely(!desc_addr)) {
-				error = -1;
-				goto out;
-			}
-
-			desc_offset = 0;
-			desc_avail  = desc->len;
-		} else if (unlikely(desc_chunck_len == 0)) {
-			desc_chunck_len = desc_avail;
-			desc_gaddr += desc_offset;
-			desc_addr = vhost_iova_to_vva(dev,
-					vq, desc_gaddr,
-					&desc_chunck_len, VHOST_ACCESS_RW);
-			if (unlikely(!desc_addr)) {
-				error = -1;
-				goto out;
-			}
-			desc_offset = 0;
-		}
-
-		cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail);
-		if (likely(cpy_len > MAX_BATCH_LEN || copy_nb >= vq->size)) {
-			rte_memcpy((void *)((uintptr_t)(desc_addr +
-							desc_offset)),
-				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
-				cpy_len);
-			vhost_log_cache_write(dev, vq, desc_gaddr + desc_offset,
-					cpy_len);
-			PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
-				     cpy_len, 0);
-		} else {
-			batch_copy[copy_nb].dst =
-				(void *)((uintptr_t)(desc_addr + desc_offset));
-			batch_copy[copy_nb].src =
-				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
-			batch_copy[copy_nb].log_addr = desc_gaddr + desc_offset;
-			batch_copy[copy_nb].len = cpy_len;
-			copy_nb++;
-		}
-
-		mbuf_avail  -= cpy_len;
-		mbuf_offset += cpy_len;
-		desc_avail  -= cpy_len;
-		desc_offset += cpy_len;
-		desc_chunck_len -= cpy_len;
-	}
-
-out:
-	vq->batch_copy_nb_elems = copy_nb;
-
-	return error;
-}
-
-/**
- * This function adds buffers to the virtio devices RX virtqueue. Buffers can
- * be received from the physical port or from another virtio device. A packet
- * count is returned to indicate the number of packets that are successfully
- * added to the RX queue. This function works when the mbuf is scattered, but
- * it doesn't support the mergeable feature.
- */
-static __rte_always_inline uint32_t
-virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
-	      struct rte_mbuf **pkts, uint32_t count)
-{
-	struct vhost_virtqueue *vq;
-	uint16_t avail_idx, free_entries, start_idx;
-	uint16_t desc_indexes[MAX_PKT_BURST];
-	struct vring_desc *descs;
-	uint16_t used_idx;
-	uint32_t i, sz;
-
-	VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
-	if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
-		RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
-			dev->vid, __func__, queue_id);
-		return 0;
-	}
-
-	vq = dev->virtqueue[queue_id];
-
-	rte_spinlock_lock(&vq->access_lock);
-
-	if (unlikely(vq->enabled == 0))
-		goto out_access_unlock;
-
-	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
-		vhost_user_iotlb_rd_lock(vq);
-
-	if (unlikely(vq->access_ok == 0)) {
-		if (unlikely(vring_translate(dev, vq) < 0)) {
-			count = 0;
-			goto out;
-		}
-	}
-
-	avail_idx = *((volatile uint16_t *)&vq->avail->idx);
-	start_idx = vq->last_used_idx;
-	free_entries = avail_idx - start_idx;
-	count = RTE_MIN(count, free_entries);
-	count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST);
-	if (count == 0)
-		goto out;
-
-	VHOST_LOG_DEBUG(VHOST_DATA, "(%d) start_idx %d | end_idx %d\n",
-		dev->vid, start_idx, start_idx + count);
-
-	vq->batch_copy_nb_elems = 0;
-
-	/* Retrieve all of the desc indexes first to avoid caching issues. */
-	rte_prefetch0(&vq->avail->ring[start_idx & (vq->size - 1)]);
-	for (i = 0; i < count; i++) {
-		used_idx = (start_idx + i) & (vq->size - 1);
-		desc_indexes[i] = vq->avail->ring[used_idx];
-		vq->used->ring[used_idx].id = desc_indexes[i];
-		vq->used->ring[used_idx].len = pkts[i]->pkt_len +
-					       dev->vhost_hlen;
-		vhost_log_cache_used_vring(dev, vq,
-			offsetof(struct vring_used, ring[used_idx]),
-			sizeof(vq->used->ring[used_idx]));
-	}
-
-	rte_prefetch0(&vq->desc[desc_indexes[0]]);
-	for (i = 0; i < count; i++) {
-		struct vring_desc *idesc = NULL;
-		uint16_t desc_idx = desc_indexes[i];
-		int err;
-
-		if (vq->desc[desc_idx].flags & VRING_DESC_F_INDIRECT) {
-			uint64_t dlen = vq->desc[desc_idx].len;
-			descs = (struct vring_desc *)(uintptr_t)
-				vhost_iova_to_vva(dev,
-						vq, vq->desc[desc_idx].addr,
-						&dlen, VHOST_ACCESS_RO);
-			if (unlikely(!descs)) {
-				count = i;
-				break;
-			}
-
-			if (unlikely(dlen < vq->desc[desc_idx].len)) {
-				/*
-				 * The indirect desc table is not contiguous
-				 * in process VA space, we have to copy it.
-				 */
-				idesc = alloc_copy_ind_table(dev, vq,
-							&vq->desc[desc_idx]);
-				if (unlikely(!idesc))
-					break;
-
-				descs = idesc;
-			}
-
-			desc_idx = 0;
-			sz = vq->desc[desc_idx].len / sizeof(*descs);
-		} else {
-			descs = vq->desc;
-			sz = vq->size;
-		}
-
-		err = copy_mbuf_to_desc(dev, vq, descs, pkts[i], desc_idx, sz);
-		if (unlikely(err)) {
-			count = i;
-			free_ind_table(idesc);
-			break;
-		}
+				perm);
+		if (unlikely(!desc_addr))
+			return -1;
 
-		if (i + 1 < count)
-			rte_prefetch0(&vq->desc[desc_indexes[i+1]]);
+		buf_vec[vec_id].buf_iova = desc_iova;
+		buf_vec[vec_id].buf_addr = desc_addr;
+		buf_vec[vec_id].buf_len  = desc_chunck_len;
 
-		if (unlikely(!!idesc))
-			free_ind_table(idesc);
+		desc_len -= desc_chunck_len;
+		desc_iova += desc_chunck_len;
+		vec_id++;
 	}
+	*vec_idx = vec_id;
 
-	do_data_copy_enqueue(dev, vq);
-
-	rte_smp_wmb();
-
-	vhost_log_cache_sync(dev, vq);
-
-	*(volatile uint16_t *)&vq->used->idx += count;
-	vq->last_used_idx += count;
-	vhost_log_used_vring(dev, vq,
-		offsetof(struct vring_used, idx),
-		sizeof(vq->used->idx));
-
-	vhost_vring_call(dev, vq);
-out:
-	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
-		vhost_user_iotlb_rd_unlock(vq);
-
-out_access_unlock:
-	rte_spinlock_unlock(&vq->access_lock);
-
-	return count;
+	return 0;
 }
 
 static __rte_always_inline int
-fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
-			 uint32_t avail_idx, uint32_t *vec_idx,
+fill_vec_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
+			 uint32_t avail_idx, uint16_t *vec_idx,
 			 struct buf_vector *buf_vec, uint16_t *desc_chain_head,
-			 uint16_t *desc_chain_len)
+			 uint16_t *desc_chain_len, uint8_t perm)
 {
 	uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)];
-	uint32_t vec_id = *vec_idx;
+	uint16_t vec_id = *vec_idx;
 	uint32_t len    = 0;
 	uint64_t dlen;
 	struct vring_desc *descs = vq->desc;
@@ -555,7 +354,8 @@ fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			 * The indirect desc table is not contiguous
 			 * in process VA space, we have to copy it.
 			 */
-			idesc = alloc_copy_ind_table(dev, vq, &vq->desc[idx]);
+			idesc = alloc_copy_ind_table(dev, vq,
+					vq->desc[idx].addr, vq->desc[idx].len);
 			if (unlikely(!idesc))
 				return -1;
 
@@ -566,16 +366,19 @@ fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	}
 
 	while (1) {
-		if (unlikely(vec_id >= BUF_VECTOR_MAX || idx >= vq->size)) {
+		if (unlikely(idx >= vq->size)) {
 			free_ind_table(idesc);
 			return -1;
 		}
 
 		len += descs[idx].len;
-		buf_vec[vec_id].buf_addr = descs[idx].addr;
-		buf_vec[vec_id].buf_len  = descs[idx].len;
-		buf_vec[vec_id].desc_idx = idx;
-		vec_id++;
+
+		if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
+						descs[idx].addr, descs[idx].len,
+						perm))) {
+			free_ind_table(idesc);
+			return -1;
+		}
 
 		if ((descs[idx].flags & VRING_DESC_F_NEXT) == 0)
 			break;
@@ -596,13 +399,14 @@ fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
  * Returns -1 on fail, 0 on success
  */
 static inline int
-reserve_avail_buf_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
+reserve_avail_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 				uint32_t size, struct buf_vector *buf_vec,
-				uint16_t *num_buffers, uint16_t avail_head)
+				uint16_t *num_buffers, uint16_t avail_head,
+				uint16_t *nr_vec)
 {
 	uint16_t cur_idx;
-	uint32_t vec_idx = 0;
-	uint16_t tries = 0;
+	uint16_t vec_idx = 0;
+	uint16_t max_tries, tries = 0;
 
 	uint16_t head_idx = 0;
 	uint16_t len = 0;
@@ -610,49 +414,223 @@ reserve_avail_buf_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	*num_buffers = 0;
 	cur_idx  = vq->last_avail_idx;
 
+	if (rxvq_is_mergeable(dev))
+		max_tries = vq->size - 1;
+	else
+		max_tries = 1;
+
 	while (size > 0) {
 		if (unlikely(cur_idx == avail_head))
 			return -1;
+		/*
+		 * if we tried all available ring items, and still
+		 * can't get enough buf, it means something abnormal
+		 * happened.
+		 */
+		if (unlikely(++tries > max_tries))
+			return -1;
 
-		if (unlikely(fill_vec_buf(dev, vq, cur_idx, &vec_idx, buf_vec,
-						&head_idx, &len) < 0))
+		if (unlikely(fill_vec_buf_split(dev, vq, cur_idx,
+						&vec_idx, buf_vec,
+						&head_idx, &len,
+						VHOST_ACCESS_RW) < 0))
 			return -1;
 		len = RTE_MIN(len, size);
-		update_shadow_used_ring(vq, head_idx, len);
+		update_shadow_used_ring_split(vq, head_idx, len);
 		size -= len;
 
 		cur_idx++;
-		tries++;
 		*num_buffers += 1;
+	}
+
+	*nr_vec = vec_idx;
+
+	return 0;
+}
+
+static __rte_always_inline int
+fill_vec_buf_packed_indirect(struct virtio_net *dev,
+			struct vhost_virtqueue *vq,
+			struct vring_packed_desc *desc, uint16_t *vec_idx,
+			struct buf_vector *buf_vec, uint16_t *len, uint8_t perm)
+{
+	uint16_t i;
+	uint32_t nr_descs;
+	uint16_t vec_id = *vec_idx;
+	uint64_t dlen;
+	struct vring_packed_desc *descs, *idescs = NULL;
+
+	dlen = desc->len;
+	descs = (struct vring_packed_desc *)(uintptr_t)
+		vhost_iova_to_vva(dev, vq, desc->addr, &dlen, VHOST_ACCESS_RO);
+	if (unlikely(!descs))
+		return -1;
+
+	if (unlikely(dlen < desc->len)) {
+		/*
+		 * The indirect desc table is not contiguous
+		 * in process VA space, we have to copy it.
+		 */
+		idescs = alloc_copy_ind_table(dev, vq, desc->addr, desc->len);
+		if (unlikely(!idescs))
+			return -1;
+
+		descs = idescs;
+	}
+
+	nr_descs =  desc->len / sizeof(struct vring_packed_desc);
+	if (unlikely(nr_descs >= vq->size)) {
+		free_ind_table(idescs);
+		return -1;
+	}
+
+	for (i = 0; i < nr_descs; i++) {
+		if (unlikely(vec_id >= BUF_VECTOR_MAX)) {
+			free_ind_table(idescs);
+			return -1;
+		}
+
+		*len += descs[i].len;
+		if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
+						descs[i].addr, descs[i].len,
+						perm)))
+			return -1;
+	}
+	*vec_idx = vec_id;
+
+	if (unlikely(!!idescs))
+		free_ind_table(idescs);
+
+	return 0;
+}
+
+static __rte_always_inline int
+fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
+				uint16_t avail_idx, uint16_t *desc_count,
+				struct buf_vector *buf_vec, uint16_t *vec_idx,
+				uint16_t *buf_id, uint16_t *len, uint8_t perm)
+{
+	bool wrap_counter = vq->avail_wrap_counter;
+	struct vring_packed_desc *descs = vq->desc_packed;
+	uint16_t vec_id = *vec_idx;
+
+	if (avail_idx < vq->last_avail_idx)
+		wrap_counter ^= 1;
+
+	if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
+		return -1;
+
+	*desc_count = 0;
+
+	while (1) {
+		if (unlikely(vec_id >= BUF_VECTOR_MAX))
+			return -1;
+
+		*desc_count += 1;
+		*buf_id = descs[avail_idx].id;
+
+		if (descs[avail_idx].flags & VRING_DESC_F_INDIRECT) {
+			if (unlikely(fill_vec_buf_packed_indirect(dev, vq,
+							&descs[avail_idx],
+							&vec_id, buf_vec,
+							len, perm) < 0))
+				return -1;
+		} else {
+			*len += descs[avail_idx].len;
+
+			if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
+							descs[avail_idx].addr,
+							descs[avail_idx].len,
+							perm)))
+				return -1;
+		}
+
+		if ((descs[avail_idx].flags & VRING_DESC_F_NEXT) == 0)
+			break;
+
+		if (++avail_idx >= vq->size) {
+			avail_idx -= vq->size;
+			wrap_counter ^= 1;
+		}
+	}
+
+	*vec_idx = vec_id;
+
+	return 0;
+}
+
+/*
+ * Returns -1 on fail, 0 on success
+ */
+static inline int
+reserve_avail_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
+				uint32_t size, struct buf_vector *buf_vec,
+				uint16_t *nr_vec, uint16_t *num_buffers,
+				uint16_t *nr_descs)
+{
+	uint16_t avail_idx;
+	uint16_t vec_idx = 0;
+	uint16_t max_tries, tries = 0;
+
+	uint16_t buf_id = 0;
+	uint16_t len = 0;
+	uint16_t desc_count;
+
+	*num_buffers = 0;
+	avail_idx = vq->last_avail_idx;
 
+	if (rxvq_is_mergeable(dev))
+		max_tries = vq->size - 1;
+	else
+		max_tries = 1;
+
+	while (size > 0) {
 		/*
 		 * if we tried all available ring items, and still
 		 * can't get enough buf, it means something abnormal
 		 * happened.
 		 */
-		if (unlikely(tries >= vq->size))
+		if (unlikely(++tries > max_tries))
 			return -1;
+
+		if (unlikely(fill_vec_buf_packed(dev, vq,
+						avail_idx, &desc_count,
+						buf_vec, &vec_idx,
+						&buf_id, &len,
+						VHOST_ACCESS_RO) < 0))
+			return -1;
+
+		len = RTE_MIN(len, size);
+		update_shadow_used_ring_packed(vq, buf_id, len, desc_count);
+		size -= len;
+
+		avail_idx += desc_count;
+		if (avail_idx >= vq->size)
+			avail_idx -= vq->size;
+
+		*nr_descs += desc_count;
+		*num_buffers += 1;
 	}
 
+	*nr_vec = vec_idx;
+
 	return 0;
 }
 
 static __rte_always_inline int
-copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
+copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			    struct rte_mbuf *m, struct buf_vector *buf_vec,
-			    uint16_t num_buffers)
+			    uint16_t nr_vec, uint16_t num_buffers)
 {
 	uint32_t vec_idx = 0;
-	uint64_t desc_addr, desc_gaddr;
 	uint32_t mbuf_offset, mbuf_avail;
-	uint32_t desc_offset, desc_avail;
+	uint32_t buf_offset, buf_avail;
+	uint64_t buf_addr, buf_iova, buf_len;
 	uint32_t cpy_len;
-	uint64_t desc_chunck_len;
-	uint64_t hdr_addr, hdr_phys_addr;
+	uint64_t hdr_addr;
 	struct rte_mbuf *hdr_mbuf;
 	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 	struct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL;
-	uint16_t copy_nb = vq->batch_copy_nb_elems;
 	int error = 0;
 
 	if (unlikely(m == NULL)) {
@@ -660,82 +638,61 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		goto out;
 	}
 
-	desc_chunck_len = buf_vec[vec_idx].buf_len;
-	desc_gaddr = buf_vec[vec_idx].buf_addr;
-	desc_addr = vhost_iova_to_vva(dev, vq,
-					desc_gaddr,
-					&desc_chunck_len,
-					VHOST_ACCESS_RW);
-	if (buf_vec[vec_idx].buf_len < dev->vhost_hlen || !desc_addr) {
+	buf_addr = buf_vec[vec_idx].buf_addr;
+	buf_iova = buf_vec[vec_idx].buf_iova;
+	buf_len = buf_vec[vec_idx].buf_len;
+
+	if (nr_vec > 1)
+		rte_prefetch0((void *)(uintptr_t)buf_vec[1].buf_addr);
+
+	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
 		error = -1;
 		goto out;
 	}
 
 	hdr_mbuf = m;
-	hdr_addr = desc_addr;
-	if (unlikely(desc_chunck_len < dev->vhost_hlen))
+	hdr_addr = buf_addr;
+	if (unlikely(buf_len < dev->vhost_hlen))
 		hdr = &tmp_hdr;
 	else
 		hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
-	hdr_phys_addr = desc_gaddr;
-	rte_prefetch0((void *)(uintptr_t)hdr_addr);
 
 	VHOST_LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n",
 		dev->vid, num_buffers);
 
-	desc_avail  = buf_vec[vec_idx].buf_len - dev->vhost_hlen;
-	if (unlikely(desc_chunck_len < dev->vhost_hlen)) {
-		desc_chunck_len = desc_avail;
-		desc_gaddr += dev->vhost_hlen;
-		desc_addr = vhost_iova_to_vva(dev, vq,
-				desc_gaddr,
-				&desc_chunck_len,
-				VHOST_ACCESS_RW);
-		if (unlikely(!desc_addr)) {
-			error = -1;
-			goto out;
-		}
-
-		desc_offset = 0;
+	if (unlikely(buf_len < dev->vhost_hlen)) {
+		buf_offset = dev->vhost_hlen - buf_len;
+		vec_idx++;
+		buf_addr = buf_vec[vec_idx].buf_addr;
+		buf_iova = buf_vec[vec_idx].buf_iova;
+		buf_len = buf_vec[vec_idx].buf_len;
+		buf_avail = buf_len - buf_offset;
 	} else {
-		desc_offset = dev->vhost_hlen;
-		desc_chunck_len -= dev->vhost_hlen;
+		buf_offset = dev->vhost_hlen;
+		buf_avail = buf_len - dev->vhost_hlen;
 	}
 
-
 	mbuf_avail  = rte_pktmbuf_data_len(m);
 	mbuf_offset = 0;
 	while (mbuf_avail != 0 || m->next != NULL) {
-		/* done with current desc buf, get the next one */
-		if (desc_avail == 0) {
+		/* done with current buf, get the next one */
+		if (buf_avail == 0) {
 			vec_idx++;
-			desc_chunck_len = buf_vec[vec_idx].buf_len;
-			desc_gaddr = buf_vec[vec_idx].buf_addr;
-			desc_addr =
-				vhost_iova_to_vva(dev, vq,
-					desc_gaddr,
-					&desc_chunck_len,
-					VHOST_ACCESS_RW);
-			if (unlikely(!desc_addr)) {
+			if (unlikely(vec_idx >= nr_vec)) {
 				error = -1;
 				goto out;
 			}
 
-			/* Prefetch buffer address. */
-			rte_prefetch0((void *)(uintptr_t)desc_addr);
-			desc_offset = 0;
-			desc_avail  = buf_vec[vec_idx].buf_len;
-		} else if (unlikely(desc_chunck_len == 0)) {
-			desc_chunck_len = desc_avail;
-			desc_gaddr += desc_offset;
-			desc_addr = vhost_iova_to_vva(dev, vq,
-					desc_gaddr,
-					&desc_chunck_len, VHOST_ACCESS_RW);
-			if (unlikely(!desc_addr)) {
-				error = -1;
-				goto out;
-			}
-			desc_offset = 0;
+			buf_addr = buf_vec[vec_idx].buf_addr;
+			buf_iova = buf_vec[vec_idx].buf_iova;
+			buf_len = buf_vec[vec_idx].buf_len;
+
+			/* Prefetch next buffer address. */
+			if (vec_idx + 1 < nr_vec)
+				rte_prefetch0((void *)(uintptr_t)
+						buf_vec[vec_idx + 1].buf_addr);
+			buf_offset = 0;
+			buf_avail  = buf_len;
 		}
 
 		/* done with current mbuf, get the next one */
@@ -748,24 +705,21 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
 		if (hdr_addr) {
 			virtio_enqueue_offload(hdr_mbuf, &hdr->hdr);
-			ASSIGN_UNLESS_EQUAL(hdr->num_buffers, num_buffers);
+			if (rxvq_is_mergeable(dev))
+				ASSIGN_UNLESS_EQUAL(hdr->num_buffers,
+						num_buffers);
 
 			if (unlikely(hdr == &tmp_hdr)) {
 				uint64_t len;
 				uint64_t remain = dev->vhost_hlen;
 				uint64_t src = (uint64_t)(uintptr_t)hdr, dst;
-				uint64_t guest_addr = hdr_phys_addr;
+				uint64_t iova = buf_vec[0].buf_iova;
+				uint16_t hdr_vec_idx = 0;
 
 				while (remain) {
-					len = remain;
-					dst = vhost_iova_to_vva(dev, vq,
-							guest_addr, &len,
-							VHOST_ACCESS_RW);
-					if (unlikely(!dst || !len)) {
-						error = -1;
-						goto out;
-					}
-
+					len = RTE_MIN(remain,
+						buf_vec[hdr_vec_idx].buf_len);
+					dst = buf_vec[hdr_vec_idx].buf_addr;
 					rte_memcpy((void *)(uintptr_t)dst,
 							(void *)(uintptr_t)src,
 							len);
@@ -773,103 +727,125 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
 					PRINT_PACKET(dev, (uintptr_t)dst,
 							(uint32_t)len, 0);
 					vhost_log_cache_write(dev, vq,
-							guest_addr, len);
+							iova, len);
 
 					remain -= len;
-					guest_addr += len;
+					iova += len;
 					src += len;
+					hdr_vec_idx++;
 				}
 			} else {
 				PRINT_PACKET(dev, (uintptr_t)hdr_addr,
 						dev->vhost_hlen, 0);
-				vhost_log_cache_write(dev, vq, hdr_phys_addr,
+				vhost_log_cache_write(dev, vq,
+						buf_vec[0].buf_iova,
 						dev->vhost_hlen);
 			}
 
 			hdr_addr = 0;
 		}
 
-		cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail);
+		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
-		if (likely(cpy_len > MAX_BATCH_LEN || copy_nb >= vq->size)) {
-			rte_memcpy((void *)((uintptr_t)(desc_addr +
-							desc_offset)),
+		if (likely(cpy_len > MAX_BATCH_LEN ||
+					vq->batch_copy_nb_elems >= vq->size)) {
+			rte_memcpy((void *)((uintptr_t)(buf_addr + buf_offset)),
 				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 				cpy_len);
-			vhost_log_cache_write(dev, vq, desc_gaddr + desc_offset,
+			vhost_log_cache_write(dev, vq, buf_iova + buf_offset,
 					cpy_len);
-			PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
+			PRINT_PACKET(dev, (uintptr_t)(buf_addr + buf_offset),
 				cpy_len, 0);
 		} else {
-			batch_copy[copy_nb].dst =
-				(void *)((uintptr_t)(desc_addr + desc_offset));
-			batch_copy[copy_nb].src =
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				(void *)((uintptr_t)(buf_addr + buf_offset));
+			batch_copy[vq->batch_copy_nb_elems].src =
 				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
-			batch_copy[copy_nb].log_addr = desc_gaddr + desc_offset;
-			batch_copy[copy_nb].len = cpy_len;
-			copy_nb++;
+			batch_copy[vq->batch_copy_nb_elems].log_addr =
+				buf_iova + buf_offset;
+			batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
+			vq->batch_copy_nb_elems++;
 		}
 
 		mbuf_avail  -= cpy_len;
 		mbuf_offset += cpy_len;
-		desc_avail  -= cpy_len;
-		desc_offset += cpy_len;
-		desc_chunck_len -= cpy_len;
+		buf_avail  -= cpy_len;
+		buf_offset += cpy_len;
 	}
 
 out:
-	vq->batch_copy_nb_elems = copy_nb;
 
 	return error;
 }
 
 static __rte_always_inline uint32_t
-virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
+virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	struct rte_mbuf **pkts, uint32_t count)
 {
-	struct vhost_virtqueue *vq;
 	uint32_t pkt_idx = 0;
 	uint16_t num_buffers;
 	struct buf_vector buf_vec[BUF_VECTOR_MAX];
 	uint16_t avail_head;
 
-	VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
-	if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
-		RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
-			dev->vid, __func__, queue_id);
-		return 0;
-	}
+	rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
+	avail_head = *((volatile uint16_t *)&vq->avail->idx);
 
-	vq = dev->virtqueue[queue_id];
+	for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
+		uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
+		uint16_t nr_vec = 0;
 
-	rte_spinlock_lock(&vq->access_lock);
+		if (unlikely(reserve_avail_buf_split(dev, vq,
+						pkt_len, buf_vec, &num_buffers,
+						avail_head, &nr_vec) < 0)) {
+			VHOST_LOG_DEBUG(VHOST_DATA,
+				"(%d) failed to get enough desc from vring\n",
+				dev->vid);
+			vq->shadow_used_idx -= num_buffers;
+			break;
+		}
 
-	if (unlikely(vq->enabled == 0))
-		goto out_access_unlock;
+		rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr);
 
-	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
-		vhost_user_iotlb_rd_lock(vq);
+		VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
+			dev->vid, vq->last_avail_idx,
+			vq->last_avail_idx + num_buffers);
 
-	if (unlikely(vq->access_ok == 0))
-		if (unlikely(vring_translate(dev, vq) < 0))
-			goto out;
+		if (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx],
+						buf_vec, nr_vec,
+						num_buffers) < 0) {
+			vq->shadow_used_idx -= num_buffers;
+			break;
+		}
 
-	count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
-	if (count == 0)
-		goto out;
+		vq->last_avail_idx += num_buffers;
+	}
 
-	vq->batch_copy_nb_elems = 0;
+	do_data_copy_enqueue(dev, vq);
 
-	rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
+	if (likely(vq->shadow_used_idx)) {
+		flush_shadow_used_ring_split(dev, vq);
+		vhost_vring_call_split(dev, vq);
+	}
+
+	return pkt_idx;
+}
+
+static __rte_always_inline uint32_t
+virtio_dev_rx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
+	struct rte_mbuf **pkts, uint32_t count)
+{
+	uint32_t pkt_idx = 0;
+	uint16_t num_buffers;
+	struct buf_vector buf_vec[BUF_VECTOR_MAX];
 
-	vq->shadow_used_idx = 0;
-	avail_head = *((volatile uint16_t *)&vq->avail->idx);
 	for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 		uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
+		uint16_t nr_vec = 0;
+		uint16_t nr_descs = 0;
 
-		if (unlikely(reserve_avail_buf_mergeable(dev, vq,
-						pkt_len, buf_vec, &num_buffers,
-						avail_head) < 0)) {
+		if (unlikely(reserve_avail_buf_packed(dev, vq,
+						pkt_len, buf_vec, &nr_vec,
+						&num_buffers, &nr_descs) < 0)) {
 			VHOST_LOG_DEBUG(VHOST_DATA,
 				"(%d) failed to get enough desc from vring\n",
 				dev->vid);
@@ -877,26 +853,72 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
 			break;
 		}
 
+		rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr);
+
 		VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
 			dev->vid, vq->last_avail_idx,
 			vq->last_avail_idx + num_buffers);
 
-		if (copy_mbuf_to_desc_mergeable(dev, vq, pkts[pkt_idx],
-						buf_vec, num_buffers) < 0) {
+		if (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx],
+						buf_vec, nr_vec,
+						num_buffers) < 0) {
 			vq->shadow_used_idx -= num_buffers;
 			break;
 		}
 
-		vq->last_avail_idx += num_buffers;
+		vq->last_avail_idx += nr_descs;
+		if (vq->last_avail_idx >= vq->size) {
+			vq->last_avail_idx -= vq->size;
+			vq->avail_wrap_counter ^= 1;
+		}
 	}
 
 	do_data_copy_enqueue(dev, vq);
 
 	if (likely(vq->shadow_used_idx)) {
-		flush_shadow_used_ring(dev, vq);
-		vhost_vring_call(dev, vq);
+		flush_shadow_used_ring_packed(dev, vq);
+		vhost_vring_call_packed(dev, vq);
 	}
 
+	return pkt_idx;
+}
+
+static __rte_always_inline uint32_t
+virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
+	struct rte_mbuf **pkts, uint32_t count)
+{
+	struct vhost_virtqueue *vq;
+
+	VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
+	if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
+		RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
+			dev->vid, __func__, queue_id);
+		return 0;
+	}
+
+	vq = dev->virtqueue[queue_id];
+
+	rte_spinlock_lock(&vq->access_lock);
+
+	if (unlikely(vq->enabled == 0))
+		goto out_access_unlock;
+
+	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+		vhost_user_iotlb_rd_lock(vq);
+
+	if (unlikely(vq->access_ok == 0))
+		if (unlikely(vring_translate(dev, vq) < 0))
+			goto out;
+
+	count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
+	if (count == 0)
+		goto out;
+
+	if (vq_is_packed(dev))
+		count = virtio_dev_rx_packed(dev, vq, pkts, count);
+	else
+		count = virtio_dev_rx_split(dev, vq, pkts, count);
+
 out:
 	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 		vhost_user_iotlb_rd_unlock(vq);
@@ -904,7 +926,7 @@ out:
 out_access_unlock:
 	rte_spinlock_unlock(&vq->access_lock);
 
-	return pkt_idx;
+	return count;
 }
 
 uint16_t
@@ -923,10 +945,7 @@ rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
 		return 0;
 	}
 
-	if (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF))
-		return virtio_dev_merge_rx(dev, queue_id, pkts, count);
-	else
-		return virtio_dev_rx(dev, queue_id, pkts, count);
+	return virtio_dev_rx(dev, queue_id, pkts, count);
 }
 
 static inline bool
@@ -1051,76 +1070,60 @@ put_zmbuf(struct zcopy_mbuf *zmbuf)
 
 static __rte_always_inline int
 copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
-		  struct vring_desc *descs, uint16_t max_desc,
-		  struct rte_mbuf *m, uint16_t desc_idx,
-		  struct rte_mempool *mbuf_pool)
+		  struct buf_vector *buf_vec, uint16_t nr_vec,
+		  struct rte_mbuf *m, struct rte_mempool *mbuf_pool)
 {
-	struct vring_desc *desc;
-	uint64_t desc_addr, desc_gaddr;
-	uint32_t desc_avail, desc_offset;
+	uint32_t buf_avail, buf_offset;
+	uint64_t buf_addr, buf_iova, buf_len;
 	uint32_t mbuf_avail, mbuf_offset;
 	uint32_t cpy_len;
-	uint64_t desc_chunck_len;
 	struct rte_mbuf *cur = m, *prev = m;
 	struct virtio_net_hdr tmp_hdr;
 	struct virtio_net_hdr *hdr = NULL;
 	/* A counter to avoid desc dead loop chain */
-	uint32_t nr_desc = 1;
+	uint16_t vec_idx = 0;
 	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
-	uint16_t copy_nb = vq->batch_copy_nb_elems;
 	int error = 0;
 
-	desc = &descs[desc_idx];
-	if (unlikely((desc->len < dev->vhost_hlen)) ||
-			(desc->flags & VRING_DESC_F_INDIRECT)) {
-		error = -1;
-		goto out;
-	}
+	buf_addr = buf_vec[vec_idx].buf_addr;
+	buf_iova = buf_vec[vec_idx].buf_iova;
+	buf_len = buf_vec[vec_idx].buf_len;
 
-	desc_chunck_len = desc->len;
-	desc_gaddr = desc->addr;
-	desc_addr = vhost_iova_to_vva(dev,
-					vq, desc_gaddr,
-					&desc_chunck_len,
-					VHOST_ACCESS_RO);
-	if (unlikely(!desc_addr)) {
+	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
 		error = -1;
 		goto out;
 	}
 
+	if (likely(nr_vec > 1))
+		rte_prefetch0((void *)(uintptr_t)buf_vec[1].buf_addr);
+
 	if (virtio_net_with_host_offload(dev)) {
-		if (unlikely(desc_chunck_len < sizeof(struct virtio_net_hdr))) {
-			uint64_t len = desc_chunck_len;
+		if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
+			uint64_t len;
 			uint64_t remain = sizeof(struct virtio_net_hdr);
-			uint64_t src = desc_addr;
+			uint64_t src;
 			uint64_t dst = (uint64_t)(uintptr_t)&tmp_hdr;
-			uint64_t guest_addr = desc_gaddr;
+			uint16_t hdr_vec_idx = 0;
 
 			/*
 			 * No luck, the virtio-net header doesn't fit
 			 * in a contiguous virtual area.
 			 */
 			while (remain) {
-				len = remain;
-				src = vhost_iova_to_vva(dev, vq,
-						guest_addr, &len,
-						VHOST_ACCESS_RO);
-				if (unlikely(!src || !len)) {
-					error = -1;
-					goto out;
-				}
-
+				len = RTE_MIN(remain,
+					buf_vec[hdr_vec_idx].buf_len);
+				src = buf_vec[hdr_vec_idx].buf_addr;
 				rte_memcpy((void *)(uintptr_t)dst,
 						   (void *)(uintptr_t)src, len);
 
-				guest_addr += len;
 				remain -= len;
 				dst += len;
+				hdr_vec_idx++;
 			}
 
 			hdr = &tmp_hdr;
 		} else {
-			hdr = (struct virtio_net_hdr *)((uintptr_t)desc_addr);
+			hdr = (struct virtio_net_hdr *)((uintptr_t)buf_addr);
 			rte_prefetch0(hdr);
 		}
 	}
@@ -1130,61 +1133,40 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	 * for Tx: the first for storing the header, and others
 	 * for storing the data.
 	 */
-	if (likely((desc->len == dev->vhost_hlen) &&
-		   (desc->flags & VRING_DESC_F_NEXT) != 0)) {
-		desc = &descs[desc->next];
-		if (unlikely(desc->flags & VRING_DESC_F_INDIRECT)) {
-			error = -1;
+	if (unlikely(buf_len < dev->vhost_hlen)) {
+		buf_offset = dev->vhost_hlen - buf_len;
+		vec_idx++;
+		buf_addr = buf_vec[vec_idx].buf_addr;
+		buf_iova = buf_vec[vec_idx].buf_iova;
+		buf_len = buf_vec[vec_idx].buf_len;
+		buf_avail  = buf_len - buf_offset;
+	} else if (buf_len == dev->vhost_hlen) {
+		if (unlikely(++vec_idx >= nr_vec))
 			goto out;
-		}
-
-		desc_chunck_len = desc->len;
-		desc_gaddr = desc->addr;
-		desc_addr = vhost_iova_to_vva(dev,
-							vq, desc_gaddr,
-							&desc_chunck_len,
-							VHOST_ACCESS_RO);
-		if (unlikely(!desc_addr)) {
-			error = -1;
-			goto out;
-		}
+		buf_addr = buf_vec[vec_idx].buf_addr;
+		buf_iova = buf_vec[vec_idx].buf_iova;
+		buf_len = buf_vec[vec_idx].buf_len;
 
-		desc_offset = 0;
-		desc_avail  = desc->len;
-		nr_desc    += 1;
+		buf_offset = 0;
+		buf_avail = buf_len;
 	} else {
-		desc_avail  = desc->len - dev->vhost_hlen;
-
-		if (unlikely(desc_chunck_len < dev->vhost_hlen)) {
-			desc_chunck_len = desc_avail;
-			desc_gaddr += dev->vhost_hlen;
-			desc_addr = vhost_iova_to_vva(dev,
-					vq, desc_gaddr,
-					&desc_chunck_len,
-					VHOST_ACCESS_RO);
-			if (unlikely(!desc_addr)) {
-				error = -1;
-				goto out;
-			}
-
-			desc_offset = 0;
-		} else {
-			desc_offset = dev->vhost_hlen;
-			desc_chunck_len -= dev->vhost_hlen;
-		}
+		buf_offset = dev->vhost_hlen;
+		buf_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen;
 	}
 
-	rte_prefetch0((void *)(uintptr_t)(desc_addr + desc_offset));
+	rte_prefetch0((void *)(uintptr_t)
+			(buf_addr + buf_offset));
 
-	PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
-			(uint32_t)desc_chunck_len, 0);
+	PRINT_PACKET(dev,
+			(uintptr_t)(buf_addr + buf_offset),
+			(uint32_t)buf_avail, 0);
 
 	mbuf_offset = 0;
 	mbuf_avail  = m->buf_len - RTE_PKTMBUF_HEADROOM;
 	while (1) {
 		uint64_t hpa;
 
-		cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail);
+		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
 		/*
 		 * A desc buf might across two host physical pages that are
@@ -1192,11 +1174,11 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		 * will be copied even though zero copy is enabled.
 		 */
 		if (unlikely(dev->dequeue_zero_copy && (hpa = gpa_to_hpa(dev,
-					desc_gaddr + desc_offset, cpy_len)))) {
+					buf_iova + buf_offset, cpy_len)))) {
 			cur->data_len = cpy_len;
 			cur->data_off = 0;
-			cur->buf_addr = (void *)(uintptr_t)(desc_addr
-				+ desc_offset);
+			cur->buf_addr =
+				(void *)(uintptr_t)(buf_addr + buf_offset);
 			cur->buf_iova = hpa;
 
 			/*
@@ -1206,81 +1188,53 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			mbuf_avail = cpy_len;
 		} else {
 			if (likely(cpy_len > MAX_BATCH_LEN ||
-				   copy_nb >= vq->size ||
-				   (hdr && cur == m) ||
-				   desc->len != desc_chunck_len)) {
+				   vq->batch_copy_nb_elems >= vq->size ||
+				   (hdr && cur == m))) {
 				rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
 								   mbuf_offset),
-					   (void *)((uintptr_t)(desc_addr +
-								desc_offset)),
+					   (void *)((uintptr_t)(buf_addr +
+							   buf_offset)),
 					   cpy_len);
 			} else {
-				batch_copy[copy_nb].dst =
+				batch_copy[vq->batch_copy_nb_elems].dst =
 					rte_pktmbuf_mtod_offset(cur, void *,
 								mbuf_offset);
-				batch_copy[copy_nb].src =
-					(void *)((uintptr_t)(desc_addr +
-							     desc_offset));
-				batch_copy[copy_nb].len = cpy_len;
-				copy_nb++;
+				batch_copy[vq->batch_copy_nb_elems].src =
+					(void *)((uintptr_t)(buf_addr +
+								buf_offset));
+				batch_copy[vq->batch_copy_nb_elems].len =
+					cpy_len;
+				vq->batch_copy_nb_elems++;
 			}
 		}
 
 		mbuf_avail  -= cpy_len;
 		mbuf_offset += cpy_len;
-		desc_avail  -= cpy_len;
-		desc_chunck_len -= cpy_len;
-		desc_offset += cpy_len;
+		buf_avail -= cpy_len;
+		buf_offset += cpy_len;
 
-		/* This desc reaches to its end, get the next one */
-		if (desc_avail == 0) {
-			if ((desc->flags & VRING_DESC_F_NEXT) == 0)
+		/* This buf reaches to its end, get the next one */
+		if (buf_avail == 0) {
+			if (++vec_idx >= nr_vec)
 				break;
 
-			if (unlikely(desc->next >= max_desc ||
-				     ++nr_desc > max_desc)) {
-				error = -1;
-				goto out;
-			}
-			desc = &descs[desc->next];
-			if (unlikely(desc->flags & VRING_DESC_F_INDIRECT)) {
-				error = -1;
-				goto out;
-			}
+			buf_addr = buf_vec[vec_idx].buf_addr;
+			buf_iova = buf_vec[vec_idx].buf_iova;
+			buf_len = buf_vec[vec_idx].buf_len;
 
-			desc_chunck_len = desc->len;
-			desc_gaddr = desc->addr;
-			desc_addr = vhost_iova_to_vva(dev,
-							vq, desc_gaddr,
-							&desc_chunck_len,
-							VHOST_ACCESS_RO);
-			if (unlikely(!desc_addr)) {
-				error = -1;
-				goto out;
-			}
+			/*
+			 * Prefecth desc n + 1 buffer while
+			 * desc n buffer is processed.
+			 */
+			if (vec_idx + 1 < nr_vec)
+				rte_prefetch0((void *)(uintptr_t)
+						buf_vec[vec_idx + 1].buf_addr);
 
-			rte_prefetch0((void *)(uintptr_t)desc_addr);
-
-			desc_offset = 0;
-			desc_avail  = desc->len;
-
-			PRINT_PACKET(dev, (uintptr_t)desc_addr,
-					(uint32_t)desc_chunck_len, 0);
-		} else if (unlikely(desc_chunck_len == 0)) {
-			desc_chunck_len = desc_avail;
-			desc_gaddr += desc_offset;
-			desc_addr = vhost_iova_to_vva(dev, vq,
-					desc_gaddr,
-					&desc_chunck_len,
-					VHOST_ACCESS_RO);
-			if (unlikely(!desc_addr)) {
-				error = -1;
-				goto out;
-			}
-			desc_offset = 0;
+			buf_offset = 0;
+			buf_avail  = buf_len;
 
-			PRINT_PACKET(dev, (uintptr_t)desc_addr,
-					(uint32_t)desc_chunck_len, 0);
+			PRINT_PACKET(dev, (uintptr_t)buf_addr,
+					(uint32_t)buf_avail, 0);
 		}
 
 		/*
@@ -1316,40 +1270,10 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		vhost_dequeue_offload(hdr, m);
 
 out:
-	vq->batch_copy_nb_elems = copy_nb;
 
 	return error;
 }
 
-static __rte_always_inline void
-update_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
-		 uint32_t used_idx, uint32_t desc_idx)
-{
-	vq->used->ring[used_idx].id  = desc_idx;
-	vq->used->ring[used_idx].len = 0;
-	vhost_log_cache_used_vring(dev, vq,
-			offsetof(struct vring_used, ring[used_idx]),
-			sizeof(vq->used->ring[used_idx]));
-}
-
-static __rte_always_inline void
-update_used_idx(struct virtio_net *dev, struct vhost_virtqueue *vq,
-		uint32_t count)
-{
-	if (unlikely(count == 0))
-		return;
-
-	rte_smp_wmb();
-	rte_smp_rmb();
-
-	vhost_log_cache_sync(dev, vq);
-
-	vq->used->idx += count;
-	vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
-			sizeof(vq->used->idx));
-	vhost_vring_call(dev, vq);
-}
-
 static __rte_always_inline struct zcopy_mbuf *
 get_zmbuf(struct vhost_virtqueue *vq)
 {
@@ -1409,66 +1333,137 @@ restore_mbuf(struct rte_mbuf *m)
 	}
 }
 
-uint16_t
-rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
+static __rte_always_inline uint16_t
+virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 {
-	struct virtio_net *dev;
-	struct rte_mbuf *rarp_mbuf = NULL;
-	struct vhost_virtqueue *vq;
-	uint32_t desc_indexes[MAX_PKT_BURST];
-	uint32_t used_idx;
-	uint32_t i = 0;
+	uint16_t i;
 	uint16_t free_entries;
-	uint16_t avail_idx;
 
-	dev = get_device(vid);
-	if (!dev)
-		return 0;
+	if (unlikely(dev->dequeue_zero_copy)) {
+		struct zcopy_mbuf *zmbuf, *next;
 
-	if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
-		RTE_LOG(ERR, VHOST_DATA,
-			"(%d) %s: built-in vhost net backend is disabled.\n",
-			dev->vid, __func__);
-		return 0;
-	}
+		for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
+		     zmbuf != NULL; zmbuf = next) {
+			next = TAILQ_NEXT(zmbuf, next);
 
-	if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) {
-		RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
-			dev->vid, __func__, queue_id);
-		return 0;
+			if (mbuf_is_consumed(zmbuf->mbuf)) {
+				update_shadow_used_ring_split(vq,
+						zmbuf->desc_idx, 0);
+				TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
+				restore_mbuf(zmbuf->mbuf);
+				rte_pktmbuf_free(zmbuf->mbuf);
+				put_zmbuf(zmbuf);
+				vq->nr_zmbuf -= 1;
+			}
+		}
+
+		flush_shadow_used_ring_split(dev, vq);
+		vhost_vring_call_split(dev, vq);
 	}
 
-	vq = dev->virtqueue[queue_id];
+	rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
 
-	if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0))
+	free_entries = *((volatile uint16_t *)&vq->avail->idx) -
+			vq->last_avail_idx;
+	if (free_entries == 0)
 		return 0;
 
-	if (unlikely(vq->enabled == 0))
-		goto out_access_unlock;
+	VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
 
-	vq->batch_copy_nb_elems = 0;
+	count = RTE_MIN(count, MAX_PKT_BURST);
+	count = RTE_MIN(count, free_entries);
+	VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
+			dev->vid, count);
 
-	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
-		vhost_user_iotlb_rd_lock(vq);
+	for (i = 0; i < count; i++) {
+		struct buf_vector buf_vec[BUF_VECTOR_MAX];
+		uint16_t head_idx, dummy_len;
+		uint16_t nr_vec = 0;
+		int err;
 
-	if (unlikely(vq->access_ok == 0))
-		if (unlikely(vring_translate(dev, vq) < 0))
-			goto out;
+		if (unlikely(fill_vec_buf_split(dev, vq,
+						vq->last_avail_idx + i,
+						&nr_vec, buf_vec,
+						&head_idx, &dummy_len,
+						VHOST_ACCESS_RO) < 0))
+			break;
+
+		if (likely(dev->dequeue_zero_copy == 0))
+			update_shadow_used_ring_split(vq, head_idx, 0);
+
+		rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr);
+
+		pkts[i] = rte_pktmbuf_alloc(mbuf_pool);
+		if (unlikely(pkts[i] == NULL)) {
+			RTE_LOG(ERR, VHOST_DATA,
+				"Failed to allocate memory for mbuf.\n");
+			break;
+		}
+
+		err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],
+				mbuf_pool);
+		if (unlikely(err)) {
+			rte_pktmbuf_free(pkts[i]);
+			break;
+		}
+
+		if (unlikely(dev->dequeue_zero_copy)) {
+			struct zcopy_mbuf *zmbuf;
+
+			zmbuf = get_zmbuf(vq);
+			if (!zmbuf) {
+				rte_pktmbuf_free(pkts[i]);
+				break;
+			}
+			zmbuf->mbuf = pkts[i];
+			zmbuf->desc_idx = head_idx;
+
+			/*
+			 * Pin lock the mbuf; we will check later to see
+			 * whether the mbuf is freed (when we are the last
+			 * user) or not. If that's the case, we then could
+			 * update the used ring safely.
+			 */
+			rte_mbuf_refcnt_update(pkts[i], 1);
+
+			vq->nr_zmbuf += 1;
+			TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
+		}
+	}
+	vq->last_avail_idx += i;
+
+	if (likely(dev->dequeue_zero_copy == 0)) {
+		do_data_copy_dequeue(vq);
+		if (unlikely(i < count))
+			vq->shadow_used_idx = i;
+		flush_shadow_used_ring_split(dev, vq);
+		vhost_vring_call_split(dev, vq);
+	}
+
+	return i;
+}
+
+static __rte_always_inline uint16_t
+virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
+{
+	uint16_t i;
+
+	rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
 
 	if (unlikely(dev->dequeue_zero_copy)) {
 		struct zcopy_mbuf *zmbuf, *next;
-		int nr_updated = 0;
 
 		for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
 		     zmbuf != NULL; zmbuf = next) {
 			next = TAILQ_NEXT(zmbuf, next);
 
 			if (mbuf_is_consumed(zmbuf->mbuf)) {
-				used_idx = vq->last_used_idx++ & (vq->size - 1);
-				update_used_ring(dev, vq, used_idx,
-						 zmbuf->desc_idx);
-				nr_updated += 1;
+				update_shadow_used_ring_packed(vq,
+						zmbuf->desc_idx,
+						0,
+						zmbuf->desc_count);
 
 				TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
 				restore_mbuf(zmbuf->mbuf);
@@ -1478,122 +1473,46 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
 			}
 		}
 
-		update_used_idx(dev, vq, nr_updated);
+		flush_shadow_used_ring_packed(dev, vq);
+		vhost_vring_call_packed(dev, vq);
 	}
 
-	/*
-	 * Construct a RARP broadcast packet, and inject it to the "pkts"
-	 * array, to looks like that guest actually send such packet.
-	 *
-	 * Check user_send_rarp() for more information.
-	 *
-	 * broadcast_rarp shares a cacheline in the virtio_net structure
-	 * with some fields that are accessed during enqueue and
-	 * rte_atomic16_cmpset() causes a write if using cmpxchg. This could
-	 * result in false sharing between enqueue and dequeue.
-	 *
-	 * Prevent unnecessary false sharing by reading broadcast_rarp first
-	 * and only performing cmpset if the read indicates it is likely to
-	 * be set.
-	 */
-
-	if (unlikely(rte_atomic16_read(&dev->broadcast_rarp) &&
-			rte_atomic16_cmpset((volatile uint16_t *)
-				&dev->broadcast_rarp.cnt, 1, 0))) {
-
-		rarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac);
-		if (rarp_mbuf == NULL) {
-			RTE_LOG(ERR, VHOST_DATA,
-				"Failed to make RARP packet.\n");
-			return 0;
-		}
-		count -= 1;
-	}
-
-	free_entries = *((volatile uint16_t *)&vq->avail->idx) -
-			vq->last_avail_idx;
-	if (free_entries == 0)
-		goto out;
-
 	VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
 
-	/* Prefetch available and used ring */
-	avail_idx = vq->last_avail_idx & (vq->size - 1);
-	used_idx  = vq->last_used_idx  & (vq->size - 1);
-	rte_prefetch0(&vq->avail->ring[avail_idx]);
-	rte_prefetch0(&vq->used->ring[used_idx]);
-
 	count = RTE_MIN(count, MAX_PKT_BURST);
-	count = RTE_MIN(count, free_entries);
 	VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
 			dev->vid, count);
 
-	/* Retrieve all of the head indexes first to avoid caching issues. */
 	for (i = 0; i < count; i++) {
-		avail_idx = (vq->last_avail_idx + i) & (vq->size - 1);
-		used_idx  = (vq->last_used_idx  + i) & (vq->size - 1);
-		desc_indexes[i] = vq->avail->ring[avail_idx];
-
-		if (likely(dev->dequeue_zero_copy == 0))
-			update_used_ring(dev, vq, used_idx, desc_indexes[i]);
-	}
-
-	/* Prefetch descriptor index. */
-	rte_prefetch0(&vq->desc[desc_indexes[0]]);
-	for (i = 0; i < count; i++) {
-		struct vring_desc *desc, *idesc = NULL;
-		uint16_t sz, idx;
-		uint64_t dlen;
+		struct buf_vector buf_vec[BUF_VECTOR_MAX];
+		uint16_t buf_id, dummy_len;
+		uint16_t desc_count, nr_vec = 0;
 		int err;
 
-		if (likely(i + 1 < count))
-			rte_prefetch0(&vq->desc[desc_indexes[i + 1]]);
-
-		if (vq->desc[desc_indexes[i]].flags & VRING_DESC_F_INDIRECT) {
-			dlen = vq->desc[desc_indexes[i]].len;
-			desc = (struct vring_desc *)(uintptr_t)
-				vhost_iova_to_vva(dev, vq,
-						vq->desc[desc_indexes[i]].addr,
-						&dlen,
-						VHOST_ACCESS_RO);
-			if (unlikely(!desc))
-				break;
+		if (unlikely(fill_vec_buf_packed(dev, vq,
+						vq->last_avail_idx, &desc_count,
+						buf_vec, &nr_vec,
+						&buf_id, &dummy_len,
+						VHOST_ACCESS_RW) < 0))
+			break;
 
-			if (unlikely(dlen < vq->desc[desc_indexes[i]].len)) {
-				/*
-				 * The indirect desc table is not contiguous
-				 * in process VA space, we have to copy it.
-				 */
-				idesc = alloc_copy_ind_table(dev, vq,
-						&vq->desc[desc_indexes[i]]);
-				if (unlikely(!idesc))
-					break;
-
-				desc = idesc;
-			}
+		if (likely(dev->dequeue_zero_copy == 0))
+			update_shadow_used_ring_packed(vq, buf_id, 0,
+					desc_count);
 
-			rte_prefetch0(desc);
-			sz = vq->desc[desc_indexes[i]].len / sizeof(*desc);
-			idx = 0;
-		} else {
-			desc = vq->desc;
-			sz = vq->size;
-			idx = desc_indexes[i];
-		}
+		rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr);
 
 		pkts[i] = rte_pktmbuf_alloc(mbuf_pool);
 		if (unlikely(pkts[i] == NULL)) {
 			RTE_LOG(ERR, VHOST_DATA,
 				"Failed to allocate memory for mbuf.\n");
-			free_ind_table(idesc);
 			break;
 		}
 
-		err = copy_desc_to_mbuf(dev, vq, desc, sz, pkts[i], idx,
-					mbuf_pool);
+		err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],
+				mbuf_pool);
 		if (unlikely(err)) {
 			rte_pktmbuf_free(pkts[i]);
-			free_ind_table(idesc);
 			break;
 		}
 
@@ -1603,11 +1522,11 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
 			zmbuf = get_zmbuf(vq);
 			if (!zmbuf) {
 				rte_pktmbuf_free(pkts[i]);
-				free_ind_table(idesc);
 				break;
 			}
 			zmbuf->mbuf = pkts[i];
-			zmbuf->desc_idx = desc_indexes[i];
+			zmbuf->desc_idx = buf_id;
+			zmbuf->desc_count = desc_count;
 
 			/*
 			 * Pin lock the mbuf; we will check later to see
@@ -1621,17 +1540,102 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
 			TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
 		}
 
-		if (unlikely(!!idesc))
-			free_ind_table(idesc);
+		vq->last_avail_idx += desc_count;
+		if (vq->last_avail_idx >= vq->size) {
+			vq->last_avail_idx -= vq->size;
+			vq->avail_wrap_counter ^= 1;
+		}
 	}
-	vq->last_avail_idx += i;
 
 	if (likely(dev->dequeue_zero_copy == 0)) {
 		do_data_copy_dequeue(vq);
-		vq->last_used_idx += i;
-		update_used_idx(dev, vq, i);
+		if (unlikely(i < count))
+			vq->shadow_used_idx = i;
+		flush_shadow_used_ring_packed(dev, vq);
+		vhost_vring_call_packed(dev, vq);
+	}
+
+	return i;
+}
+
+uint16_t
+rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
+{
+	struct virtio_net *dev;
+	struct rte_mbuf *rarp_mbuf = NULL;
+	struct vhost_virtqueue *vq;
+
+	dev = get_device(vid);
+	if (!dev)
+		return 0;
+
+	if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
+		RTE_LOG(ERR, VHOST_DATA,
+			"(%d) %s: built-in vhost net backend is disabled.\n",
+			dev->vid, __func__);
+		return 0;
+	}
+
+	if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) {
+		RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
+			dev->vid, __func__, queue_id);
+		return 0;
+	}
+
+	vq = dev->virtqueue[queue_id];
+
+	if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0))
+		return 0;
+
+	if (unlikely(vq->enabled == 0)) {
+		count = 0;
+		goto out_access_unlock;
 	}
 
+	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+		vhost_user_iotlb_rd_lock(vq);
+
+	if (unlikely(vq->access_ok == 0))
+		if (unlikely(vring_translate(dev, vq) < 0)) {
+			count = 0;
+			goto out;
+		}
+
+	/*
+	 * Construct a RARP broadcast packet, and inject it to the "pkts"
+	 * array, to looks like that guest actually send such packet.
+	 *
+	 * Check user_send_rarp() for more information.
+	 *
+	 * broadcast_rarp shares a cacheline in the virtio_net structure
+	 * with some fields that are accessed during enqueue and
+	 * rte_atomic16_cmpset() causes a write if using cmpxchg. This could
+	 * result in false sharing between enqueue and dequeue.
+	 *
+	 * Prevent unnecessary false sharing by reading broadcast_rarp first
+	 * and only performing cmpset if the read indicates it is likely to
+	 * be set.
+	 */
+	if (unlikely(rte_atomic16_read(&dev->broadcast_rarp) &&
+			rte_atomic16_cmpset((volatile uint16_t *)
+				&dev->broadcast_rarp.cnt, 1, 0))) {
+
+		rarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac);
+		if (rarp_mbuf == NULL) {
+			RTE_LOG(ERR, VHOST_DATA,
+				"Failed to make RARP packet.\n");
+			count = 0;
+			goto out;
+		}
+		count -= 1;
+	}
+
+	if (vq_is_packed(dev))
+		count = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count);
+	else
+		count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count);
+
 out:
 	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 		vhost_user_iotlb_rd_unlock(vq);
@@ -1644,10 +1648,10 @@ out_access_unlock:
 		 * Inject it to the head of "pkts" array, so that switch's mac
 		 * learning table will get updated first.
 		 */
-		memmove(&pkts[1], pkts, i * sizeof(struct rte_mbuf *));
+		memmove(&pkts[1], pkts, count * sizeof(struct rte_mbuf *));
 		pkts[0] = rarp_mbuf;
-		i += 1;
+		count += 1;
 	}
 
-	return i;
+	return count;
 }
diff --git a/lib/meson.build b/lib/meson.build
index 9d11571f..eb91f100 100644
--- a/lib/meson.build
+++ b/lib/meson.build
@@ -9,7 +9,8 @@
 # given as a dep, no need to mention ring. This is especially true for the
 # core libs which are widely reused, so their deps are kept to a minimum.
 libraries = [ 'compat', # just a header, used for versioning
-	'eal', 'ring', 'mempool', 'mbuf', 'net', 'kvargs', 'ethdev', 'pci', # core
+	'kvargs',
+	'eal', 'ring', 'mempool', 'mbuf', 'net', 'ethdev', 'pci', # core
 	'metrics', # bitrate/latency stats depends on this
 	'hash',    # efd depends on this
 	'timer',   # eventdev depends on this
@@ -25,6 +26,10 @@ libraries = [ 'compat', # just a header, used for versioning
 	# flow_classify lib depends on pkt framework table lib
 	'flow_classify', 'bpf']
 
+default_cflags = machine_args
+if cc.has_argument('-Wno-format-truncation')
+	default_cflags += '-Wno-format-truncation'
+endif
 foreach l:libraries
 	build = true
 	name = l
@@ -33,7 +38,7 @@ foreach l:libraries
 	sources = []
 	headers = []
 	includes = []
-	cflags = machine_args
+	cflags = default_cflags
 	objs = [] # other object files to link against, used e.g. for
 	          # instruction-set optimized versions of code
 
@@ -41,9 +46,12 @@ foreach l:libraries
 	# external package/library requirements
 	ext_deps = []
 	deps = ['eal']   # eal is standard dependency except for itself
-	if l == 'eal'
+	if l == 'kvargs'
 		deps = []
 	endif
+	if l == 'eal'
+		deps = ['kvargs']
+	endif
 
 	dir_name = 'librte_' + l
 	subdir(dir_name)
@@ -63,6 +71,10 @@ foreach l:libraries
 			shared_deps = ext_deps
 			static_deps = ext_deps
 			foreach d:deps
+				if not is_variable('shared_rte_' + d)
+					error('Missing dependency ' + d +
+						' for library ' + lib_name)
+				endif
 				shared_deps += [get_variable('shared_rte_' + d)]
 				static_deps += [get_variable('static_rte_' + d)]
 			endforeach
diff --git a/meson.build b/meson.build
index 5b2234f0..e718972f 100644
--- a/meson.build
+++ b/meson.build
@@ -2,7 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 project('DPDK', 'C',
-	version: '18.05.0',
+	version: '18.08.0',
 	license: 'BSD',
 	default_options: ['buildtype=release', 'default_library=static'],
 	meson_version: '>= 0.41'
diff --git a/mk/machine/dpaa/rte.vars.mk b/mk/machine/dpaa/rte.vars.mk
index bddcb800..75df626f 100644
--- a/mk/machine/dpaa/rte.vars.mk
+++ b/mk/machine/dpaa/rte.vars.mk
@@ -32,3 +32,6 @@ MACHINE_CFLAGS += -march=armv8-a+crc
 ifdef CONFIG_RTE_ARCH_ARM_TUNE
 MACHINE_CFLAGS += -mtune=$(CONFIG_RTE_ARCH_ARM_TUNE:"%"=%)
 endif
+
+# To avoid TLS corruption issue.
+MACHINE_CFLAGS += -mtls-dialect=trad
diff --git a/mk/machine/dpaa2/rte.vars.mk b/mk/machine/dpaa2/rte.vars.mk
index 2fd2eac8..aaa03c42 100644
--- a/mk/machine/dpaa2/rte.vars.mk
+++ b/mk/machine/dpaa2/rte.vars.mk
@@ -32,3 +32,6 @@ MACHINE_CFLAGS += -march=armv8-a+crc
 ifdef CONFIG_RTE_ARCH_ARM_TUNE
 MACHINE_CFLAGS += -mtune=$(CONFIG_RTE_ARCH_ARM_TUNE:"%"=%)
 endif
+
+# To avoid TLS corruption issue.
+MACHINE_CFLAGS += -mtls-dialect=trad
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 1e32c83e..de33883b 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -31,15 +31,19 @@ _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
 # Order is important: from higher level to lower level
 #
 _LDLIBS-$(CONFIG_RTE_LIBRTE_FLOW_CLASSIFY)  += -lrte_flow_classify
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PIPELINE)       += --whole-archive
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PIPELINE)       += -lrte_pipeline
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PIPELINE)       += --no-whole-archive
+_LDLIBS-$(CONFIG_RTE_LIBRTE_TABLE)          += --whole-archive
 _LDLIBS-$(CONFIG_RTE_LIBRTE_TABLE)          += -lrte_table
+_LDLIBS-$(CONFIG_RTE_LIBRTE_TABLE)          += --no-whole-archive
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PORT)           += --whole-archive
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PORT)           += -lrte_port
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PORT)           += --no-whole-archive
 
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PDUMP)          += -lrte_pdump
 _LDLIBS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR)    += -lrte_distributor
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IP_FRAG)        += -lrte_ip_frag
-_LDLIBS-$(CONFIG_RTE_LIBRTE_GRO)            += -lrte_gro
-_LDLIBS-$(CONFIG_RTE_LIBRTE_GSO)            += -lrte_gso
 _LDLIBS-$(CONFIG_RTE_LIBRTE_METER)          += -lrte_meter
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LPM)            += -lrte_lpm
 # librte_acl needs --whole-archive because of weak functions
@@ -61,6 +65,8 @@ endif
 _LDLIBS-y += --whole-archive
 
 _LDLIBS-$(CONFIG_RTE_LIBRTE_CFGFILE)        += -lrte_cfgfile
+_LDLIBS-$(CONFIG_RTE_LIBRTE_GRO)            += -lrte_gro
+_LDLIBS-$(CONFIG_RTE_LIBRTE_GSO)            += -lrte_gso
 _LDLIBS-$(CONFIG_RTE_LIBRTE_HASH)           += -lrte_hash
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MEMBER)         += -lrte_member
 _LDLIBS-$(CONFIG_RTE_LIBRTE_VHOST)          += -lrte_vhost
@@ -143,9 +149,9 @@ else
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
 endif
 ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -ldl
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -ldl -lmnl
 else
-_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs -lmlx5
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs -lmlx5 -lmnl
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MVPP2_PMD)      += -lrte_pmd_mvpp2 -L$(LIBMUSDK_PATH)/lib -lmusdk
 _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lrte_pmd_nfp
@@ -165,11 +171,14 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD)     += -lrte_pmd_virtio
 ifeq ($(CONFIG_RTE_LIBRTE_VHOST),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_VHOST)      += -lrte_pmd_vhost
 ifeq ($(CONFIG_RTE_EAL_VFIO),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_IFCVF_VDPA_PMD) += -lrte_ifcvf_vdpa
+_LDLIBS-$(CONFIG_RTE_LIBRTE_IFC_PMD) += -lrte_pmd_ifc
 endif # $(CONFIG_RTE_EAL_VFIO)
 endif # $(CONFIG_RTE_LIBRTE_VHOST)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD)    += -lrte_pmd_vmxnet3_uio
 
+_LDLIBS-$(CONFIG_RTE_LIBRTE_VMBUS)          += -lrte_bus_vmbus
+_LDLIBS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD)     += -lrte_pmd_netvsc
+
 ifeq ($(CONFIG_RTE_LIBRTE_BBDEV),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_BBDEV_NULL)     += -lrte_pmd_bbdev_null
 
@@ -190,7 +199,9 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM)   += -lIPSec_MB
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_CCP)         += -lrte_pmd_ccp -lcrypto
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_OPENSSL)     += -lrte_pmd_openssl -lcrypto
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += -lrte_pmd_null_crypto
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_QAT)         += -lrte_pmd_qat -lcrypto
+ifeq ($(CONFIG_RTE_LIBRTE_PMD_QAT),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_QAT_SYM)     += -lrte_pmd_qat -lcrypto
+endif # CONFIG_RTE_LIBRTE_PMD_QAT
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G)      += -lrte_pmd_snow3g
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G)      += -L$(LIBSSO_SNOW3G_PATH)/build -lsso_snow3g
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI)      += -lrte_pmd_kasumi
@@ -213,6 +224,13 @@ endif # CONFIG_RTE_LIBRTE_CRYPTODEV
 ifeq ($(CONFIG_RTE_LIBRTE_COMPRESSDEV),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ISAL) += -lrte_pmd_isal_comp
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ISAL) += -lisal
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_ZIPVF) += -lrte_pmd_octeontx_zip
+# Link QAT driver if it has not been linked yet
+ifeq ($(CONFIG_RTE_LIBRTE_PMD_QAT_SYM),n)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_QAT)  += -lrte_pmd_qat
+endif # CONFIG_RTE_LIBRTE_PMD_QAT_SYM
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ZLIB) += -lrte_pmd_zlib
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ZLIB) += -lz
 endif # CONFIG_RTE_LIBRTE_COMPRESSDEV
 
 ifeq ($(CONFIG_RTE_LIBRTE_EVENTDEV),y)
diff --git a/mk/rte.sdkinstall.mk b/mk/rte.sdkinstall.mk
index e756ec62..8296e6db 100644
--- a/mk/rte.sdkinstall.mk
+++ b/mk/rte.sdkinstall.mk
@@ -85,18 +85,22 @@ else
 	@echo Installation in $(DESTDIR)$(prefix)/ complete
 endif
 
+# when installing we want recursive copies preserving timestamps only, no
+# preservation of user/group ids or permissions
+CP_FLAGS=-dR --preserve=timestamps
+TAR_X_FLAGS=--strip-components=1 --keep-newer-files --no-same-owner --no-same-permissions
+
 install-runtime:
 	$(Q)$(call rte_mkdir, $(DESTDIR)$(libdir))
-	$(Q)cp -a    $O/lib/* $(DESTDIR)$(libdir)
+	$(Q)cp $(CP_FLAGS)    $O/lib/* $(DESTDIR)$(libdir)
 	$(Q)$(call rte_mkdir, $(DESTDIR)$(bindir))
 	$(Q)tar -cf -      -C $O --exclude 'app/*.map' \
 		--exclude app/dpdk-pmdinfogen \
 		--exclude 'app/cmdline*' --exclude app/test \
 		--exclude app/testacl --exclude app/testpipeline app | \
-	    tar -xf -      -C $(DESTDIR)$(bindir) --strip-components=1 \
-		--keep-newer-files
+	    tar -xf -      -C $(DESTDIR)$(bindir) $(TAR_X_FLAGS)
 	$(Q)$(call rte_mkdir,      $(DESTDIR)$(datadir))
-	$(Q)cp -a $(RTE_SDK)/usertools $(DESTDIR)$(datadir)
+	$(Q)cp $(CP_FLAGS) $(RTE_SDK)/usertools $(DESTDIR)$(datadir)
 	$(Q)$(call rte_mkdir,      $(DESTDIR)$(sbindir))
 	$(Q)$(call rte_symlink,    $(DESTDIR)$(datadir)/usertools/dpdk-devbind.py, \
 	                           $(DESTDIR)$(sbindir)/dpdk-devbind)
@@ -104,30 +108,29 @@ install-runtime:
 	                           $(DESTDIR)$(bindir)/dpdk-pmdinfo)
 ifneq ($(wildcard $O/doc/man/*/*.1),)
 	$(Q)$(call rte_mkdir,      $(DESTDIR)$(mandir)/man1)
-	$(Q)cp -a $O/doc/man/*/*.1 $(DESTDIR)$(mandir)/man1
+	$(Q)cp $(CP_FLAGS) $O/doc/man/*/*.1 $(DESTDIR)$(mandir)/man1
 endif
 ifneq ($(wildcard $O/doc/man/*/*.8),)
 	$(Q)$(call rte_mkdir,      $(DESTDIR)$(mandir)/man8)
-	$(Q)cp -a $O/doc/man/*/*.8 $(DESTDIR)$(mandir)/man8
+	$(Q)cp $(CP_FLAGS) $O/doc/man/*/*.8 $(DESTDIR)$(mandir)/man8
 endif
 
 install-kmod:
 ifneq ($(wildcard $O/kmod/*),)
 	$(Q)$(call rte_mkdir, $(DESTDIR)$(kerneldir))
-	$(Q)cp -a   $O/kmod/* $(DESTDIR)$(kerneldir)
+	$(Q)cp $(CP_FLAGS)   $O/kmod/* $(DESTDIR)$(kerneldir)
 endif
 
 install-sdk:
 	$(Q)$(call rte_mkdir, $(DESTDIR)$(includedir))
 	$(Q)tar -chf -     -C $O include | \
-	    tar -xf -      -C $(DESTDIR)$(includedir) --strip-components=1 \
-		--keep-newer-files
+	    tar -xf -      -C $(DESTDIR)$(includedir) $(TAR_X_FLAGS)
 	$(Q)$(call rte_mkdir,                            $(DESTDIR)$(sdkdir))
-	$(Q)cp -a               $(RTE_SDK)/mk            $(DESTDIR)$(sdkdir)
-	$(Q)cp -a               $(RTE_SDK)/buildtools    $(DESTDIR)$(sdkdir)
+	$(Q)cp $(CP_FLAGS)      $(RTE_SDK)/mk            $(DESTDIR)$(sdkdir)
+	$(Q)cp $(CP_FLAGS)      $(RTE_SDK)/buildtools    $(DESTDIR)$(sdkdir)
 	$(Q)$(call rte_mkdir,                            $(DESTDIR)$(targetdir)/app)
-	$(Q)cp -a               $O/.config               $(DESTDIR)$(targetdir)
-	$(Q)cp -a               $O/app/dpdk-pmdinfogen   $(DESTDIR)$(targetdir)/app
+	$(Q)cp $(CP_FLAGS)      $O/.config               $(DESTDIR)$(targetdir)
+	$(Q)cp $(CP_FLAGS)      $O/app/dpdk-pmdinfogen   $(DESTDIR)$(targetdir)/app
 	$(Q)$(call rte_symlink, $(DESTDIR)$(includedir), $(DESTDIR)$(targetdir)/include)
 	$(Q)$(call rte_symlink, $(DESTDIR)$(libdir),     $(DESTDIR)$(targetdir)/lib)
 
@@ -135,12 +138,11 @@ install-doc:
 ifneq ($(wildcard $O/doc/html),)
 	$(Q)$(call rte_mkdir, $(DESTDIR)$(docdir))
 	$(Q)tar -cf -      -C $O/doc --exclude 'html/guides/.*' html | \
-	    tar -xf -      -C $(DESTDIR)$(docdir) --strip-components=1 \
-		--keep-newer-files
+	    tar -xf -      -C $(DESTDIR)$(docdir) $(TAR_X_FLAGS)
 endif
 ifneq ($(wildcard $O/doc/*/*/*pdf),)
 	$(Q)$(call rte_mkdir,     $(DESTDIR)$(docdir)/guides)
-	$(Q)cp -a $O/doc/*/*/*pdf $(DESTDIR)$(docdir)/guides
+	$(Q)cp $(CP_FLAGS) $O/doc/*/*/*pdf $(DESTDIR)$(docdir)/guides
 endif
 	$(Q)$(call rte_mkdir,         $(DESTDIR)$(datadir))
-	$(Q)cp -a $(RTE_SDK)/examples $(DESTDIR)$(datadir)
+	$(Q)cp $(CP_FLAGS) $(RTE_SDK)/examples $(DESTDIR)$(datadir)
diff --git a/mk/rte.sdkroot.mk b/mk/rte.sdkroot.mk
index f43cc782..18c88017 100644
--- a/mk/rte.sdkroot.mk
+++ b/mk/rte.sdkroot.mk
@@ -68,8 +68,8 @@ config defconfig showconfigs showversion showversionum:
 cscope gtags tags etags:
 	$(Q)$(RTE_SDK)/devtools/build-tags.sh $@ $T
 
-.PHONY: test test-basic test-fast test-ring test-mempool test-perf coverage
-test test-basic test-fast test-ring test-mempool test-perf coverage:
+.PHONY: test test-fast test-perf coverage test-drivers test-dump
+test test-fast test-perf coverage test-drivers test-dump:
 	$(Q)$(MAKE) -f $(RTE_SDK)/mk/rte.sdktest.mk $@
 
 test: test-build
diff --git a/mk/rte.sdktest.mk b/mk/rte.sdktest.mk
index ee1fe0c7..29559280 100644
--- a/mk/rte.sdktest.mk
+++ b/mk/rte.sdktest.mk
@@ -18,14 +18,34 @@ DIR := $(shell basename $(RTE_OUTPUT))
 #
 # test: launch auto-tests, very simple for now.
 #
-.PHONY: test test-basic test-fast test-perf coverage
+.PHONY: test test-fast test-perf test-drivers test-dump coverage
 
-PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf
-coverage: BLACKLIST=-$(PERFLIST)
-test-fast: BLACKLIST=-$(PERFLIST)
-test-perf: WHITELIST=$(PERFLIST)
+PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf,\
+         reciprocal_division,reciprocal_division_perf,lpm_perf,red_all,\
+         barrier,hash_multiwriter,timer_racecond,efd,hash_functions,\
+         eventdev_selftest_sw,member_perf,efd_perf,lpm6_perf,red_perf,\
+         distributor_perf,ring_pmd_perf,pmd_perf,ring_perf
+DRIVERSLIST=link_bonding,link_bonding_mode4,link_bonding_rssconf,\
+            cryptodev_sw_mrvl,cryptodev_dpaa2_sec,cryptodev_dpaa_sec,\
+            cryptodev_qat,cryptodev_aesni_mb,cryptodev_openssl,\
+            cryptodev_scheduler,cryptodev_aesni_gcm,cryptodev_null,\
+            cryptodev_sw_snow3g,cryptodev_sw_kasumi,cryptodev_sw_zuc
+DUMPLIST=dump_struct_sizes,dump_mempool,dump_malloc_stats,dump_devargs,\
+         dump_log_types,dump_ring,dump_physmem,dump_memzone
 
-test test-basic test-fast test-perf:
+SPACESTR:=
+SPACESTR+=
+STRIPPED_PERFLIST=$(subst $(SPACESTR),,$(PERFLIST))
+STRIPPED_DRIVERSLIST=$(subst $(SPACESTR),,$(DRIVERSLIST))
+STRIPPED_DUMPLIST=$(subst $(SPACESTR),,$(DUMPLIST))
+
+coverage: BLACKLIST=-$(STRIPPED_PERFLIST)
+test-fast: BLACKLIST=-$(STRIPPED_PERFLIST),$(STRIPPED_DRIVERSLIST),$(STRIPPED_DUMPLIST)
+test-perf: WHITELIST=$(STRIPPED_PERFLIST)
+test-drivers: WHITELIST=$(STRIPPED_DRIVERSLIST)
+test-dump: WHITELIST=$(STRIPPED_DUMPLIST)
+
+test test-fast test-perf test-drivers test-dump:
 	@mkdir -p $(AUTOTEST_DIR) ; \
 	cd $(AUTOTEST_DIR) ; \
 	if [ -f $(RTE_OUTPUT)/app/test ]; then \
diff --git a/mk/toolchain/gcc/rte.toolchain-compat.mk b/mk/toolchain/gcc/rte.toolchain-compat.mk
index 255c8967..1e4434fa 100644
--- a/mk/toolchain/gcc/rte.toolchain-compat.mk
+++ b/mk/toolchain/gcc/rte.toolchain-compat.mk
@@ -15,6 +15,11 @@ GCC_MINOR = $(shell echo __GNUC_MINOR__ | $(CC) -E -x c - | tail -n 1)
 GCC_PATCHLEVEL = $(shell echo __GNUC_PATCHLEVEL__ | $(CC) -E -x c - | tail -n 1)
 GCC_VERSION = $(GCC_MAJOR)$(GCC_MINOR)
 
+HOST_GCC_MAJOR = $(shell echo __GNUC__ | $(HOSTCC) -E -x c - | tail -n 1)
+HOST_GCC_MINOR = $(shell echo __GNUC_MINOR__ | $(HOSTCC) -E -x c - | tail -n 1)
+HOST_GCC_PATCHLEVEL = $(shell echo __GNUC_PATCHLEVEL__ | $(HOSTCC) -E -x c - | tail -n 1)
+HOST_GCC_VERSION = $(HOST_GCC_MAJOR)$(HOST_GCC_MINOR)
+
 # if GCC is older than 4.x
 ifeq ($(shell test $(GCC_VERSION) -lt 40 && echo 1), 1)
 	MACHINE_CFLAGS =
diff --git a/mk/toolchain/gcc/rte.vars.mk b/mk/toolchain/gcc/rte.vars.mk
index 7e4531ba..d8b99faf 100644
--- a/mk/toolchain/gcc/rte.vars.mk
+++ b/mk/toolchain/gcc/rte.vars.mk
@@ -71,6 +71,15 @@ ifeq ($(shell test $(GCC_VERSION) -lt 47 && echo 1), 1)
 WERROR_FLAGS += -Wno-uninitialized
 endif
 
+HOST_WERROR_FLAGS := $(WERROR_FLAGS)
+
+ifeq ($(shell test $(HOST_GCC_VERSION) -gt 70 && echo 1), 1)
+# Tell GCC only to error for switch fallthroughs without a suitable comment
+HOST_WERROR_FLAGS += -Wimplicit-fallthrough=2
+# Ignore errors for snprintf truncation
+HOST_WERROR_FLAGS += -Wno-format-truncation
+endif
+
 ifeq ($(shell test $(GCC_VERSION) -gt 70 && echo 1), 1)
 # Tell GCC only to error for switch fallthroughs without a suitable comment
 WERROR_FLAGS += -Wimplicit-fallthrough=2
diff --git a/pkg/dpdk.spec b/pkg/dpdk.spec
index bba69dd1..4fd927b3 100644
--- a/pkg/dpdk.spec
+++ b/pkg/dpdk.spec
@@ -2,7 +2,7 @@
 # Copyright 2014 6WIND S.A.
 
 Name: dpdk
-Version: 18.05
+Version: 18.08
 Release: 1
 Packager: packaging@6wind.com
 URL: http://dpdk.org
diff --git a/test/test-pipeline/init.c b/test/test-pipeline/init.c
index 19cf04a6..f33216c9 100644
--- a/test/test-pipeline/init.c
+++ b/test/test-pipeline/init.c
@@ -70,11 +70,7 @@ struct app_params app = {
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.header_split   = 0, /* Header Split disabled */
-		.hw_ip_checksum = 1, /* IP checksum offload enabled */
-		.hw_vlan_filter = 0, /* VLAN filtering disabled */
-		.jumbo_frame    = 0, /* Jumbo Frame Support disabled */
-		.hw_strip_crc   = 1, /* CRC stripped by hardware */
+		.offloads = DEV_RX_OFFLOAD_CHECKSUM | DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
diff --git a/test/test/Makefile b/test/test/Makefile
index eccc8efc..e6967bab 100644
--- a/test/test/Makefile
+++ b/test/test/Makefile
@@ -70,6 +70,7 @@ SRCS-y += test_memzone.c
 SRCS-y += test_bitmap.c
 SRCS-y += test_reciprocal_division.c
 SRCS-y += test_reciprocal_division_perf.c
+SRCS-y += test_fbarray.c
 
 SRCS-y += test_ring.c
 SRCS-y += test_ring_perf.c
@@ -113,6 +114,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf.c
 SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_functions.c
 SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_scaling.c
 SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_multiwriter.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_readwrite.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm.c
 SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm_perf.c
@@ -179,6 +181,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_RING) += test_pmd_ring_perf.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += test_cryptodev_blockcipher.c
 SRCS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += test_cryptodev.c
+SRCS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += test_cryptodev_asym.c
 
 ifeq ($(CONFIG_RTE_COMPRESSDEV_TEST),y)
 SRCS-$(CONFIG_RTE_LIBRTE_COMPRESSDEV) += test_compressdev.c
diff --git a/test/test/autotest.py b/test/test/autotest.py
index 1cfd8cf2..12997fdf 100644
--- a/test/test/autotest.py
+++ b/test/test/autotest.py
@@ -36,14 +36,15 @@ cmdline = "%s -c f -n 4" % (sys.argv[1])
 
 print(cmdline)
 
-runner = autotest_runner.AutotestRunner(cmdline, target, test_blacklist,
-                                        test_whitelist)
+# how many workers to run tests with. FreeBSD doesn't support multiple primary
+# processes, so make it 1, otherwise make it 4. ignored for non-parallel tests
+n_processes = 1 if "bsdapp" in target else 4
 
-for test_group in autotest_data.parallel_test_group_list:
-    runner.add_parallel_test_group(test_group)
+runner = autotest_runner.AutotestRunner(cmdline, target, test_blacklist,
+                                        test_whitelist, n_processes)
 
-for test_group in autotest_data.non_parallel_test_group_list:
-    runner.add_non_parallel_test_group(test_group)
+runner.parallel_tests = autotest_data.parallel_test_list[:]
+runner.non_parallel_tests = autotest_data.non_parallel_test_list[:]
 
 num_fails = runner.run_all_tests()
 
diff --git a/test/test/autotest_data.py b/test/test/autotest_data.py
index aacfe0a6..f68d9b11 100644
--- a/test/test/autotest_data.py
+++ b/test/test/autotest_data.py
@@ -3,465 +3,662 @@
 
 # Test data for autotests
 
-from glob import glob
 from autotest_test_funcs import *
 
-
-# quick and dirty function to find out number of sockets
-def num_sockets():
-    result = len(glob("/sys/devices/system/node/node*"))
-    if result == 0:
-        return 1
-    return result
-
-
-# Assign given number to each socket
-# e.g. 32 becomes 32,32 or 32,32,32,32
-def per_sockets(num):
-    return ",".join([str(num)] * num_sockets())
-
 # groups of tests that can be run in parallel
 # the grouping has been found largely empirically
-parallel_test_group_list = [
-    {
-        "Prefix":    "group_1",
-        "Memory":    per_sockets(8),
-        "Tests":
-        [
-            {
-                "Name":    "Cycles autotest",
-                "Command": "cycles_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Timer autotest",
-                "Command": "timer_autotest",
-                "Func":    timer_autotest,
-                "Report":   None,
-            },
-            {
-                "Name":    "Debug autotest",
-                "Command": "debug_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Errno autotest",
-                "Command": "errno_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Meter autotest",
-                "Command": "meter_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Common autotest",
-                "Command": "common_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Resource autotest",
-                "Command": "resource_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
-    },
-    {
-        "Prefix":    "group_2",
-        "Memory":    "16",
-        "Tests":
-        [
-            {
-                "Name":    "Memory autotest",
-                "Command": "memory_autotest",
-                "Func":    memory_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Read/write lock autotest",
-                "Command": "rwlock_autotest",
-                "Func":    rwlock_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Logs autotest",
-                "Command": "logs_autotest",
-                "Func":    logs_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "CPU flags autotest",
-                "Command": "cpuflags_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Version autotest",
-                "Command": "version_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "EAL filesystem autotest",
-                "Command": "eal_fs_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "EAL flags autotest",
-                "Command": "eal_flags_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Hash autotest",
-                "Command": "hash_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ],
-    },
-    {
-        "Prefix":    "group_3",
-        "Memory":    per_sockets(512),
-        "Tests":
-        [
-            {
-                "Name":    "LPM autotest",
-                "Command": "lpm_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "LPM6 autotest",
-                "Command": "lpm6_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Memcpy autotest",
-                "Command": "memcpy_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Memzone autotest",
-                "Command": "memzone_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "String autotest",
-                "Command": "string_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Alarm autotest",
-                "Command": "alarm_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
-    },
-    {
-        "Prefix":    "group_4",
-        "Memory":    per_sockets(128),
-        "Tests":
-        [
-            {
-                "Name":    "PCI autotest",
-                "Command": "pci_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Malloc autotest",
-                "Command": "malloc_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Multi-process autotest",
-                "Command": "multiprocess_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Mbuf autotest",
-                "Command": "mbuf_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Per-lcore autotest",
-                "Command": "per_lcore_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Ring autotest",
-                "Command": "ring_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
-    },
-    {
-        "Prefix":    "group_5",
-        "Memory":    "32",
-        "Tests":
-        [
-            {
-                "Name":    "Spinlock autotest",
-                "Command": "spinlock_autotest",
-                "Func":    spinlock_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Byte order autotest",
-                "Command": "byteorder_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "TAILQ autotest",
-                "Command": "tailq_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Command-line autotest",
-                "Command": "cmdline_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Interrupts autotest",
-                "Command": "interrupt_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
-    },
-    {
-        "Prefix":    "group_6",
-        "Memory":    per_sockets(512),
-        "Tests":
-        [
-            {
-                "Name":    "Function reentrancy autotest",
-                "Command": "func_reentrancy_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Mempool autotest",
-                "Command": "mempool_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Atomics autotest",
-                "Command": "atomic_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Prefetch autotest",
-                "Command": "prefetch_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Red autotest",
-                "Command": "red_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
-    },
-    {
-        "Prefix":    "group_7",
-        "Memory":    "64",
-        "Tests":
-        [
-            {
-                "Name":    "PMD ring autotest",
-                "Command": "ring_pmd_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Access list control autotest",
-                "Command": "acl_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-            {
-                "Name":    "Sched autotest",
-                "Command": "sched_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
+parallel_test_list = [
+    {
+        "Name":    "Cycles autotest",
+        "Command": "cycles_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Timer autotest",
+        "Command": "timer_autotest",
+        "Func":    timer_autotest,
+        "Report":   None,
+    },
+    {
+        "Name":    "Debug autotest",
+        "Command": "debug_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Errno autotest",
+        "Command": "errno_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Meter autotest",
+        "Command": "meter_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Common autotest",
+        "Command": "common_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Resource autotest",
+        "Command": "resource_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Memory autotest",
+        "Command": "memory_autotest",
+        "Func":    memory_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Read/write lock autotest",
+        "Command": "rwlock_autotest",
+        "Func":    rwlock_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Logs autotest",
+        "Command": "logs_autotest",
+        "Func":    logs_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "CPU flags autotest",
+        "Command": "cpuflags_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Version autotest",
+        "Command": "version_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "EAL filesystem autotest",
+        "Command": "eal_fs_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "EAL flags autotest",
+        "Command": "eal_flags_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Hash autotest",
+        "Command": "hash_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "LPM autotest",
+        "Command": "lpm_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "LPM6 autotest",
+        "Command": "lpm6_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Memcpy autotest",
+        "Command": "memcpy_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Memzone autotest",
+        "Command": "memzone_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "String autotest",
+        "Command": "string_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Alarm autotest",
+        "Command": "alarm_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Malloc autotest",
+        "Command": "malloc_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Multi-process autotest",
+        "Command": "multiprocess_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Mbuf autotest",
+        "Command": "mbuf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Per-lcore autotest",
+        "Command": "per_lcore_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Ring autotest",
+        "Command": "ring_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Spinlock autotest",
+        "Command": "spinlock_autotest",
+        "Func":    spinlock_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Byte order autotest",
+        "Command": "byteorder_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "TAILQ autotest",
+        "Command": "tailq_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Command-line autotest",
+        "Command": "cmdline_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Interrupts autotest",
+        "Command": "interrupt_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Function reentrancy autotest",
+        "Command": "func_reentrancy_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Mempool autotest",
+        "Command": "mempool_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Atomics autotest",
+        "Command": "atomic_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Prefetch autotest",
+        "Command": "prefetch_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Red autotest",
+        "Command": "red_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "PMD ring autotest",
+        "Command": "ring_pmd_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Access list control autotest",
+        "Command": "acl_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Sched autotest",
+        "Command": "sched_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Eventdev selftest octeontx",
+        "Command": "eventdev_selftest_octeontx",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Event ring autotest",
+        "Command": "event_ring_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Table autotest",
+        "Command": "table_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Flow classify autotest",
+        "Command": "flow_classify_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Event eth rx adapter autotest",
+        "Command": "event_eth_rx_adapter_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "User delay",
+        "Command": "user_delay_us",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Rawdev autotest",
+        "Command": "rawdev_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Kvargs autotest",
+        "Command": "kvargs_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Devargs autotest",
+        "Command": "devargs_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Link bonding autotest",
+        "Command": "link_bonding_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Link bonding mode4 autotest",
+        "Command": "link_bonding_mode4_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Link bonding rssconf autotest",
+        "Command": "link_bonding_rssconf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Crc autotest",
+        "Command": "crc_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Distributor autotest",
+        "Command": "distributor_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Reorder autotest",
+        "Command": "reorder_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Barrier autotest",
+        "Command": "barrier_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Bitmap test",
+        "Command": "bitmap_test",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Hash scaling autotest",
+        "Command": "hash_scaling_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Hash multiwriter autotest",
+        "Command": "hash_multiwriter_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Service autotest",
+        "Command": "service_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Timer racecond autotest",
+        "Command": "timer_racecond_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Member autotest",
+        "Command": "member_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":   "Efd_autotest",
+        "Command": "efd_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Thash autotest",
+        "Command": "thash_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Hash function autotest",
+        "Command": "hash_functions_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev sw mrvl autotest",
+        "Command": "cryptodev_sw_mrvl_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev dpaa2 sec autotest",
+        "Command": "cryptodev_dpaa2_sec_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev dpaa sec autotest",
+        "Command": "cryptodev_dpaa_sec_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev qat autotest",
+        "Command": "cryptodev_qat_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev aesni mb autotest",
+        "Command": "cryptodev_aesni_mb_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev openssl autotest",
+        "Command": "cryptodev_openssl_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev scheduler autotest",
+        "Command": "cryptodev_scheduler_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev aesni gcm autotest",
+        "Command": "cryptodev_aesni_gcm_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev null autotest",
+        "Command": "cryptodev_null_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev sw snow3g autotest",
+        "Command": "cryptodev_sw_snow3g_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev sw kasumi autotest",
+        "Command": "cryptodev_sw_kasumi_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Cryptodev_sw_zuc_autotest",
+        "Command": "cryptodev_sw_zuc_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Reciprocal division",
+        "Command": "reciprocal_division",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Red all",
+        "Command": "red_all",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Fbarray autotest",
+        "Command": "fbarray_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    #
+    #Please always keep all dump tests at the end and together!
+    #
+    {
+        "Name":    "Dump physmem",
+        "Command": "dump_physmem",
+        "Func":    dump_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Dump memzone",
+        "Command": "dump_memzone",
+        "Func":    dump_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Dump struct sizes",
+        "Command": "dump_struct_sizes",
+        "Func":    dump_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Dump mempool",
+        "Command": "dump_mempool",
+        "Func":    dump_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Dump malloc stats",
+        "Command": "dump_malloc_stats",
+        "Func":    dump_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Dump devargs",
+        "Command": "dump_devargs",
+        "Func":    dump_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Dump log types",
+        "Command": "dump_log_types",
+        "Func":    dump_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Dump_ring",
+        "Command": "dump_ring",
+        "Func":    dump_autotest,
+        "Report":  None,
     },
 ]
 
 # tests that should not be run when any other tests are running
-non_parallel_test_group_list = [
-
+non_parallel_test_list = [
+    {
+        "Name":    "Eventdev common autotest",
+        "Command": "eventdev_common_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Eventdev selftest sw",
+        "Command": "eventdev_selftest_sw",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "KNI autotest",
+        "Command": "kni_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Mempool performance autotest",
+        "Command": "mempool_perf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Memcpy performance autotest",
+        "Command": "memcpy_perf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Hash performance autotest",
+        "Command": "hash_perf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":       "Power autotest",
+        "Command":    "power_autotest",
+        "Func":       default_autotest,
+        "Report":      None,
+    },
+    {
+        "Name":       "Power ACPI cpufreq autotest",
+        "Command":    "power_acpi_cpufreq_autotest",
+        "Func":       default_autotest,
+        "Report":     None,
+    },
     {
-        "Prefix":    "eventdev",
-        "Memory":    "512",
-        "Tests":
-        [
-            {
-                "Name":    "Eventdev common autotest",
-                "Command": "eventdev_common_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
-    },
-    {
-        "Prefix":    "eventdev_sw",
-        "Memory":    "512",
-        "Tests":
-        [
-            {
-                "Name":    "Eventdev sw autotest",
-                "Command": "eventdev_sw_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
-    },
-    {
-        "Prefix":    "kni",
-        "Memory":    "512",
-        "Tests":
-        [
-            {
-                "Name":    "KNI autotest",
-                "Command": "kni_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
-    },
-    {
-        "Prefix":    "mempool_perf",
-        "Memory":    per_sockets(256),
-        "Tests":
-        [
-            {
-                "Name":    "Mempool performance autotest",
-                "Command": "mempool_perf_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
-    },
-    {
-        "Prefix":    "memcpy_perf",
-        "Memory":    per_sockets(512),
-        "Tests":
-        [
-            {
-                "Name":    "Memcpy performance autotest",
-                "Command": "memcpy_perf_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
-    },
-    {
-        "Prefix":    "hash_perf",
-        "Memory":    per_sockets(512),
-        "Tests":
-        [
-            {
-                "Name":    "Hash performance autotest",
-                "Command": "hash_perf_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
-    },
-    {
-        "Prefix":      "power",
-        "Memory":      "16",
-        "Tests":
-        [
-            {
-                "Name":       "Power autotest",
-                "Command":    "power_autotest",
-                "Func":       default_autotest,
-                "Report":      None,
-            },
-        ]
-    },
-    {
-        "Prefix":      "power_acpi_cpufreq",
-        "Memory":      "16",
-        "Tests":
-        [
-            {
-                "Name":       "Power ACPI cpufreq autotest",
-                "Command":    "power_acpi_cpufreq_autotest",
-                "Func":       default_autotest,
-                "Report":     None,
-            },
-        ]
-    },
-    {
-        "Prefix":      "power_kvm_vm",
-        "Memory":      "16",
-        "Tests":
-        [
-            {
-                "Name":       "Power KVM VM  autotest",
-                "Command":    "power_kvm_vm_autotest",
-                "Func":       default_autotest,
-                "Report":     None,
-            },
-        ]
-    },
-    {
-        "Prefix":    "timer_perf",
-        "Memory":    per_sockets(512),
-        "Tests":
-        [
-            {
-                "Name":    "Timer performance autotest",
-                "Command": "timer_perf_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
+        "Name":       "Power KVM VM  autotest",
+        "Command":    "power_kvm_vm_autotest",
+        "Func":       default_autotest,
+        "Report":     None,
     },
+    {
+        "Name":    "Timer performance autotest",
+        "Command": "timer_perf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
 
+        "Name":    "Pmd perf autotest",
+        "Command": "pmd_perf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Ring pmd perf autotest",
+        "Command": "ring_pmd_perf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Distributor perf autotest",
+        "Command": "distributor_perf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Red_perf",
+        "Command": "red_perf",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Lpm6 perf autotest",
+        "Command": "lpm6_perf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Lpm perf autotest",
+        "Command": "lpm_perf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+         "Name":    "Efd perf autotest",
+         "Command": "efd_perf_autotest",
+         "Func":    default_autotest,
+         "Report":  None,
+    },
+    {
+        "Name":    "Member perf autotest",
+        "Command": "member_perf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
+    {
+        "Name":    "Reciprocal division perf",
+        "Command": "reciprocal_division_perf",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
     #
     # Please always make sure that ring_perf is the last test!
     #
     {
-        "Prefix":    "ring_perf",
-        "Memory":    per_sockets(512),
-        "Tests":
-        [
-            {
-                "Name":    "Ring performance autotest",
-                "Command": "ring_perf_autotest",
-                "Func":    default_autotest,
-                "Report":  None,
-            },
-        ]
+        "Name":    "Ring performance autotest",
+        "Command": "ring_perf_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
     },
 ]
diff --git a/test/test/autotest_runner.py b/test/test/autotest_runner.py
index a692f069..36941a40 100644
--- a/test/test/autotest_runner.py
+++ b/test/test/autotest_runner.py
@@ -3,18 +3,19 @@
 
 # The main logic behind running autotests in parallel
 
+from __future__ import print_function
 import StringIO
 import csv
-import multiprocessing
+from multiprocessing import Pool, Queue
 import pexpect
 import re
 import subprocess
 import sys
 import time
+import glob
+import os
 
 # wait for prompt
-
-
 def wait_prompt(child):
     try:
         child.sendline()
@@ -27,143 +28,150 @@ def wait_prompt(child):
     else:
         return False
 
-# run a test group
-# each result tuple in results list consists of:
-#   result value (0 or -1)
-#   result string
-#   test name
-#   total test run time (double)
-#   raw test log
-#   test report (if not available, should be None)
-#
-# this function needs to be outside AutotestRunner class
-# because otherwise Pool won't work (or rather it will require
-# quite a bit of effort to make it work).
 
+# get all valid NUMA nodes
+def get_numa_nodes():
+    return [
+        int(
+            re.match(r"node(\d+)", os.path.basename(node))
+            .group(1)
+        )
+        for node in glob.glob("/sys/devices/system/node/node*")
+    ]
 
-def run_test_group(cmdline, test_group):
-    results = []
-    child = None
+
+# find first (or any, really) CPU on a particular node, will be used to spread
+# processes around NUMA nodes to avoid exhausting memory on particular node
+def first_cpu_on_node(node_nr):
+    cpu_path = glob.glob("/sys/devices/system/node/node%d/cpu*" % node_nr)[0]
+    cpu_name = os.path.basename(cpu_path)
+    m = re.match(r"cpu(\d+)", cpu_name)
+    return int(m.group(1))
+
+
+pool_child = None  # per-process child
+
+
+# we initialize each worker with a queue because we need per-pool unique
+# command-line arguments, but we cannot do different arguments in an initializer
+# because the API doesn't allow per-worker initializer arguments. so, instead,
+# we will initialize with a shared queue, and dequeue command-line arguments
+# from this queue
+def pool_init(queue, result_queue):
+    global pool_child
+
+    cmdline, prefix = queue.get()
     start_time = time.time()
-    startuplog = None
+    name = ("Start %s" % prefix) if prefix != "" else "Start"
+
+    # use default prefix if no prefix was specified
+    prefix_cmdline = "--file-prefix=%s" % prefix if prefix != "" else ""
+
+    # append prefix to cmdline
+    cmdline = "%s %s" % (cmdline, prefix_cmdline)
+
+    # prepare logging of init
+    startuplog = StringIO.StringIO()
 
     # run test app
     try:
-        # prepare logging of init
-        startuplog = StringIO.StringIO()
 
-        print >>startuplog, "\n%s %s\n" % ("=" * 20, test_group["Prefix"])
-        print >>startuplog, "\ncmdline=%s" % cmdline
+        print("\n%s %s\n" % ("=" * 20, prefix), file=startuplog)
+        print("\ncmdline=%s" % cmdline, file=startuplog)
 
-        child = pexpect.spawn(cmdline, logfile=startuplog)
+        pool_child = pexpect.spawn(cmdline, logfile=startuplog)
 
         # wait for target to boot
-        if not wait_prompt(child):
-            child.close()
+        if not wait_prompt(pool_child):
+            pool_child.close()
 
-            results.append((-1,
+            result = tuple((-1,
                             "Fail [No prompt]",
-                            "Start %s" % test_group["Prefix"],
+                            name,
+                            time.time() - start_time,
+                            startuplog.getvalue(),
+                            None))
+            pool_child = None
+        else:
+            result = tuple((0,
+                            "Success",
+                            name,
                             time.time() - start_time,
                             startuplog.getvalue(),
                             None))
-
-            # mark all tests as failed
-            for test in test_group["Tests"]:
-                results.append((-1, "Fail [No prompt]", test["Name"],
-                                time.time() - start_time, "", None))
-            # exit test
-            return results
-
     except:
-        results.append((-1,
+        result = tuple((-1,
                         "Fail [Can't run]",
-                        "Start %s" % test_group["Prefix"],
+                        name,
                         time.time() - start_time,
                         startuplog.getvalue(),
                         None))
+        pool_child = None
 
-        # mark all tests as failed
-        for t in test_group["Tests"]:
-            results.append((-1, "Fail [Can't run]", t["Name"],
-                            time.time() - start_time, "", None))
-        # exit test
-        return results
-
-    # startup was successful
-    results.append((0, "Success", "Start %s" % test_group["Prefix"],
-                    time.time() - start_time, startuplog.getvalue(), None))
+    result_queue.put(result)
 
-    # parse the binary for available test commands
-    binary = cmdline.split()[0]
-    stripped = 'not stripped' not in subprocess.check_output(['file', binary])
-    if not stripped:
-        symbols = subprocess.check_output(['nm', binary]).decode('utf-8')
-        avail_cmds = re.findall('test_register_(\w+)', symbols)
 
-    # run all tests in test group
-    for test in test_group["Tests"]:
+# run a test
+# each result tuple in results list consists of:
+#   result value (0 or -1)
+#   result string
+#   test name
+#   total test run time (double)
+#   raw test log
+#   test report (if not available, should be None)
+#
+# this function needs to be outside AutotestRunner class because otherwise Pool
+# won't work (or rather it will require quite a bit of effort to make it work).
+def run_test(target, test):
+    global pool_child
 
-        # create log buffer for each test
-        # in multiprocessing environment, the logging would be
-        # interleaved and will create a mess, hence the buffering
-        logfile = StringIO.StringIO()
-        child.logfile = logfile
+    if pool_child is None:
+        return -1, "Fail [No test process]", test["Name"], 0, "", None
 
-        result = ()
+    # create log buffer for each test
+    # in multiprocessing environment, the logging would be
+    # interleaved and will create a mess, hence the buffering
+    logfile = StringIO.StringIO()
+    pool_child.logfile = logfile
 
-        # make a note when the test started
-        start_time = time.time()
+    # make a note when the test started
+    start_time = time.time()
 
-        try:
-            # print test name to log buffer
-            print >>logfile, "\n%s %s\n" % ("-" * 20, test["Name"])
+    try:
+        # print test name to log buffer
+        print("\n%s %s\n" % ("-" * 20, test["Name"]), file=logfile)
 
-            # run test function associated with the test
-            if stripped or test["Command"] in avail_cmds:
-                result = test["Func"](child, test["Command"])
-            else:
-                result = (0, "Skipped [Not Available]")
+        # run test function associated with the test
+        result = test["Func"](pool_child, test["Command"])
 
-            # make a note when the test was finished
-            end_time = time.time()
+        # make a note when the test was finished
+        end_time = time.time()
 
-            # append test data to the result tuple
-            result += (test["Name"], end_time - start_time,
-                       logfile.getvalue())
+        log = logfile.getvalue()
 
-            # call report function, if any defined, and supply it with
-            # target and complete log for test run
-            if test["Report"]:
-                report = test["Report"](self.target, log)
+        # append test data to the result tuple
+        result += (test["Name"], end_time - start_time, log)
 
-                # append report to results tuple
-                result += (report,)
-            else:
-                # report is None
-                result += (None,)
-        except:
-            # make a note when the test crashed
-            end_time = time.time()
-
-            # mark test as failed
-            result = (-1, "Fail [Crash]", test["Name"],
-                      end_time - start_time, logfile.getvalue(), None)
-        finally:
-            # append the results to the results list
-            results.append(result)
+        # call report function, if any defined, and supply it with
+        # target and complete log for test run
+        if test["Report"]:
+            report = test["Report"](target, log)
 
-    # regardless of whether test has crashed, try quitting it
-    try:
-        child.sendline("quit")
-        child.close()
-    # if the test crashed, just do nothing instead
+            # append report to results tuple
+            result += (report,)
+        else:
+            # report is None
+            result += (None,)
     except:
-        # nop
-        pass
+        # make a note when the test crashed
+        end_time = time.time()
+
+        # mark test as failed
+        result = (-1, "Fail [Crash]", test["Name"],
+                  end_time - start_time, logfile.getvalue(), None)
 
     # return test results
-    return results
+    return result
 
 
 # class representing an instance of autotests run
@@ -181,11 +189,17 @@ class AutotestRunner:
     blacklist = []
     whitelist = []
 
-    def __init__(self, cmdline, target, blacklist, whitelist):
+    def __init__(self, cmdline, target, blacklist, whitelist, n_processes):
         self.cmdline = cmdline
         self.target = target
+        self.binary = cmdline.split()[0]
         self.blacklist = blacklist
         self.whitelist = whitelist
+        self.skipped = []
+        self.parallel_tests = []
+        self.non_parallel_tests = []
+        self.n_processes = n_processes
+        self.active_processes = 0
 
         # log file filename
         logfile = "%s.log" % target
@@ -199,177 +213,196 @@ class AutotestRunner:
         self.csvwriter.writerow(["test_name", "test_result", "result_str"])
 
     # set up cmdline string
-    def __get_cmdline(self, test):
-        cmdline = self.cmdline
-
-        # append memory limitations for each test
-        # otherwise tests won't run in parallel
-        if "i686" not in self.target:
-            cmdline += " --socket-mem=%s" % test["Memory"]
-        else:
-            # affinitize startup so that tests don't fail on i686
-            cmdline = "taskset 1 " + cmdline
-            cmdline += " -m " + str(sum(map(int, test["Memory"].split(","))))
-
-        # set group prefix for autotest group
-        # otherwise they won't run in parallel
-        cmdline += " --file-prefix=%s" % test["Prefix"]
+    def __get_cmdline(self, cpu_nr):
+        cmdline = ("taskset -c %i " % cpu_nr) + self.cmdline
 
         return cmdline
 
-    def add_parallel_test_group(self, test_group):
-        self.parallel_test_groups.append(test_group)
+    def __process_result(self, result):
 
-    def add_non_parallel_test_group(self, test_group):
-        self.non_parallel_test_groups.append(test_group)
+        # unpack result tuple
+        test_result, result_str, test_name, \
+            test_time, log, report = result
 
-    def __process_results(self, results):
-        # this iterates over individual test results
-        for i, result in enumerate(results):
+        # get total run time
+        cur_time = time.time()
+        total_time = int(cur_time - self.start)
 
-            # increase total number of tests that were run
-            # do not include "start" test
-            if i > 0:
-                self.n_tests += 1
+        # print results, test run time and total time since start
+        result = ("%s:" % test_name).ljust(30)
+        result += result_str.ljust(29)
+        result += "[%02dm %02ds]" % (test_time / 60, test_time % 60)
 
-            # unpack result tuple
-            test_result, result_str, test_name, \
-                test_time, log, report = result
+        # don't print out total time every line, it's the same anyway
+        print(result + "[%02dm %02ds]" % (total_time / 60, total_time % 60))
 
-            # get total run time
-            cur_time = time.time()
-            total_time = int(cur_time - self.start)
+        # if test failed and it wasn't a "start" test
+        if test_result < 0:
+            self.fails += 1
 
-            # print results, test run time and total time since start
-            result = ("%s:" % test_name).ljust(30)
-            result += result_str.ljust(29)
-            result += "[%02dm %02ds]" % (test_time / 60, test_time % 60)
+        # collect logs
+        self.log_buffers.append(log)
 
-            # don't print out total time every line, it's the same anyway
-            if i == len(results) - 1:
-                print(result,
-                      "[%02dm %02ds]" % (total_time / 60, total_time % 60))
+        # create report if it exists
+        if report:
+            try:
+                f = open("%s_%s_report.rst" %
+                         (self.target, test_name), "w")
+            except IOError:
+                print("Report for %s could not be created!" % test_name)
             else:
-                print(result)
-
-            # if test failed and it wasn't a "start" test
-            if test_result < 0 and not i == 0:
-                self.fails += 1
-
-            # collect logs
-            self.log_buffers.append(log)
-
-            # create report if it exists
-            if report:
-                try:
-                    f = open("%s_%s_report.rst" %
-                             (self.target, test_name), "w")
-                except IOError:
-                    print("Report for %s could not be created!" % test_name)
-                else:
-                    with f:
-                        f.write(report)
-
-            # write test result to CSV file
-            if i != 0:
-                self.csvwriter.writerow([test_name, test_result, result_str])
+                with f:
+                    f.write(report)
+
+        # write test result to CSV file
+        self.csvwriter.writerow([test_name, test_result, result_str])
+
+    # this function checks individual test and decides if this test should be in
+    # the group by comparing it against  whitelist/blacklist. it also checks if
+    # the test is compiled into the binary, and marks it as skipped if necessary
+    def __filter_test(self, test):
+        test_cmd = test["Command"]
+        test_id = test_cmd
+
+        # dump tests are specified in full e.g. "Dump_mempool"
+        if "_autotest" in test_id:
+            test_id = test_id[:-len("_autotest")]
+
+        # filter out blacklisted/whitelisted tests
+        if self.blacklist and test_id in self.blacklist:
+            return False
+        if self.whitelist and test_id not in self.whitelist:
+            return False
+
+        # if test wasn't compiled in, remove it as well
+
+        # parse the binary for available test commands
+        stripped = 'not stripped' not in \
+                   subprocess.check_output(['file', self.binary])
+        if not stripped:
+            symbols = subprocess.check_output(['nm',
+                                               self.binary]).decode('utf-8')
+            avail_cmds = re.findall('test_register_(\w+)', symbols)
+
+            if test_cmd not in avail_cmds:
+                # notify user
+                result = 0, "Skipped [Not compiled]", test_id, 0, "", None
+                self.skipped.append(tuple(result))
+                return False
 
-    # this function iterates over test groups and removes each
-    # test that is not in whitelist/blacklist
-    def __filter_groups(self, test_groups):
-        groups_to_remove = []
+        return True
 
-        # filter out tests from parallel test groups
-        for i, test_group in enumerate(test_groups):
+    def __run_test_group(self, test_group, worker_cmdlines):
+        group_queue = Queue()
+        init_result_queue = Queue()
+        for proc, cmdline in enumerate(worker_cmdlines):
+            prefix = "test%i" % proc if len(worker_cmdlines) > 1 else ""
+            group_queue.put(tuple((cmdline, prefix)))
 
-            # iterate over a copy so that we could safely delete individual
-            # tests
-            for test in test_group["Tests"][:]:
-                test_id = test["Command"]
+        # create a pool of worker threads
+        # we will initialize child in the initializer, and we don't need to
+        # close the child because when the pool worker gets destroyed, child
+        # closes the process
+        pool = Pool(processes=len(worker_cmdlines),
+                    initializer=pool_init,
+                    initargs=(group_queue, init_result_queue))
+
+        results = []
 
-                # dump tests are specified in full e.g. "Dump_mempool"
-                if "_autotest" in test_id:
-                    test_id = test_id[:-len("_autotest")]
+        # process all initialization results
+        for _ in range(len(worker_cmdlines)):
+            self.__process_result(init_result_queue.get())
 
-                # filter out blacklisted/whitelisted tests
-                if self.blacklist and test_id in self.blacklist:
-                    test_group["Tests"].remove(test)
-                    continue
-                if self.whitelist and test_id not in self.whitelist:
-                    test_group["Tests"].remove(test)
+        # run all tests asynchronously
+        for test in test_group:
+            result = pool.apply_async(run_test, (self.target, test))
+            results.append(result)
+
+        # tell the pool to stop all processes once done
+        pool.close()
+
+        # iterate while we have group execution results to get
+        while len(results) > 0:
+            # iterate over a copy to be able to safely delete results
+            # this iterates over a list of group results
+            for async_result in results[:]:
+                # if the thread hasn't finished yet, continue
+                if not async_result.ready():
                     continue
 
-            # modify or remove original group
-            if len(test_group["Tests"]) > 0:
-                test_groups[i] = test_group
-            else:
-                # remember which groups should be deleted
-                # put the numbers backwards so that we start
-                # deleting from the end, not from the beginning
-                groups_to_remove.insert(0, i)
+                res = async_result.get()
 
-        # remove test groups that need to be removed
-        for i in groups_to_remove:
-            del test_groups[i]
+                self.__process_result(res)
 
-        return test_groups
+                # remove result from results list once we're done with it
+                results.remove(async_result)
 
     # iterate over test groups and run tests associated with them
     def run_all_tests(self):
         # filter groups
-        self.parallel_test_groups = \
-            self.__filter_groups(self.parallel_test_groups)
-        self.non_parallel_test_groups = \
-            self.__filter_groups(self.non_parallel_test_groups)
-
-        # create a pool of worker threads
-        pool = multiprocessing.Pool(processes=1)
-
-        results = []
-
-        # whatever happens, try to save as much logs as possible
-        try:
-
-            # create table header
-            print("")
-            print("Test name".ljust(30), "Test result".ljust(29),
-                  "Test".center(9), "Total".center(9))
-            print("=" * 80)
-
-            # make a note of tests start time
-            self.start = time.time()
+        self.parallel_tests = list(
+            filter(self.__filter_test,
+                   self.parallel_tests)
+        )
+        self.non_parallel_tests = list(
+            filter(self.__filter_test,
+                   self.non_parallel_tests)
+        )
+
+        parallel_cmdlines = []
+        # FreeBSD doesn't have NUMA support
+        numa_nodes = get_numa_nodes()
+        if len(numa_nodes) > 0:
+            for proc in range(self.n_processes):
+                # spread cpu affinity between NUMA nodes to have less chance of
+                # running out of memory while running multiple test apps in
+                # parallel. to do that, alternate between NUMA nodes in a round
+                # robin fashion, and pick an arbitrary CPU from that node to
+                # taskset our execution to
+                numa_node = numa_nodes[self.active_processes % len(numa_nodes)]
+                cpu_nr = first_cpu_on_node(numa_node)
+                parallel_cmdlines += [self.__get_cmdline(cpu_nr)]
+                # increase number of active processes so that the next cmdline
+                # gets a different NUMA node
+                self.active_processes += 1
+        else:
+            parallel_cmdlines = [self.cmdline] * self.n_processes
 
-            # assign worker threads to run test groups
-            for test_group in self.parallel_test_groups:
-                result = pool.apply_async(run_test_group,
-                                          [self.__get_cmdline(test_group),
-                                           test_group])
-                results.append(result)
+        print("Running tests with %d workers" % self.n_processes)
 
-            # iterate while we have group execution results to get
-            while len(results) > 0:
+        # create table header
+        print("")
+        print("Test name".ljust(30) + "Test result".ljust(29) +
+              "Test".center(9) + "Total".center(9))
+        print("=" * 80)
 
-                # iterate over a copy to be able to safely delete results
-                # this iterates over a list of group results
-                for group_result in results[:]:
+        if len(self.skipped):
+            print("Skipped autotests:")
 
-                    # if the thread hasn't finished yet, continue
-                    if not group_result.ready():
-                        continue
+            # print out any skipped tests
+            for result in self.skipped:
+                # unpack result tuple
+                test_result, result_str, test_name, _, _, _ = result
+                self.csvwriter.writerow([test_name, test_result, result_str])
 
-                    res = group_result.get()
+                t = ("%s:" % test_name).ljust(30)
+                t += result_str.ljust(29)
+                t += "[00m 00s]"
 
-                    self.__process_results(res)
+                print(t)
 
-                    # remove result from results list once we're done with it
-                    results.remove(group_result)
+        # make a note of tests start time
+        self.start = time.time()
 
-            # run non_parallel tests. they are run one by one, synchronously
-            for test_group in self.non_parallel_test_groups:
-                group_result = run_test_group(
-                    self.__get_cmdline(test_group), test_group)
+        # whatever happens, try to save as much logs as possible
+        try:
+            if len(self.parallel_tests) > 0:
+                print("Parallel autotests:")
+                self.__run_test_group(self.parallel_tests, parallel_cmdlines)
 
-                self.__process_results(group_result)
+            if len(self.non_parallel_tests) > 0:
+                print("Non-parallel autotests:")
+                self.__run_test_group(self.non_parallel_tests, [self.cmdline])
 
             # get total run time
             cur_time = time.time()
diff --git a/test/test/meson.build b/test/test/meson.build
index a907fd25..b1dd6eca 100644
--- a/test/test/meson.build
+++ b/test/test/meson.build
@@ -22,6 +22,7 @@ test_sources = files('commands.c',
 	'test_cpuflags.c',
 	'test_crc.c',
 	'test_cryptodev.c',
+	'test_cryptodev_asym.c',
 	'test_cryptodev_blockcipher.c',
 	'test_cycles.c',
 	'test_debug.c',
@@ -130,6 +131,7 @@ test_names = [
 	'cryptodev_qat_autotest',
 	'cryptodev_aesni_mb_autotest',
 	'cryptodev_openssl_autotest',
+	'cryptodev_openssl_asym_autotest',
 	'cryptodev_aesni_gcm_autotest',
 	'cryptodev_null_autotest',
 	'cryptodev_sw_snow3g_autotest',
@@ -235,6 +237,11 @@ if dpdk_conf.has('RTE_LIBRTE_KNI')
 	test_deps += 'kni'
 endif
 
+cflags = machine_args
+if cc.has_argument('-Wno-format-truncation')
+    cflags += '-Wno-format-truncation'
+endif
+
 test_dep_objs = []
 compress_test_dep = dependency('zlib', required: false)
 if compress_test_dep.found()
@@ -260,7 +267,7 @@ if get_option('tests')
 		test_sources,
 		link_whole: link_libs,
 		dependencies: test_dep_objs,
-		c_args: [machine_args, '-DALLOW_EXPERIMENTAL_API'],
+		c_args: [cflags, '-DALLOW_EXPERIMENTAL_API'],
 		install_rpath: driver_install_path,
 		install: false)
 
diff --git a/test/test/process.h b/test/test/process.h
index 11986d5c..ba3a1850 100644
--- a/test/test/process.h
+++ b/test/test/process.h
@@ -5,6 +5,11 @@
 #ifndef _PROCESS_H_
 #define _PROCESS_H_
 
+#include <limits.h> /* PATH_MAX */
+#include <libgen.h> /* basename et al */
+#include <stdlib.h> /* NULL */
+#include <unistd.h> /* readlink */
+
 #ifdef RTE_EXEC_ENV_BSDAPP
 #define self "curproc"
 #define exe "file"
@@ -61,4 +66,28 @@ process_dup(const char *const argv[], int numargs, const char *env_value)
 	return status;
 }
 
+/* FreeBSD doesn't support file prefixes, so force compile failures for any
+ * tests attempting to use this function on FreeBSD.
+ */
+#ifdef RTE_EXEC_ENV_LINUXAPP
+static char *
+get_current_prefix(char *prefix, int size)
+{
+	char path[PATH_MAX] = {0};
+	char buf[PATH_MAX] = {0};
+
+	/* get file for config (fd is always 3) */
+	snprintf(path, sizeof(path), "/proc/self/fd/%d", 3);
+
+	/* return NULL on error */
+	if (readlink(path, buf, sizeof(buf)) == -1)
+		return NULL;
+
+	/* get the prefix */
+	snprintf(prefix, size, "%s", basename(dirname(buf)));
+
+	return prefix;
+}
+#endif
+
 #endif /* _PROCESS_H_ */
diff --git a/test/test/test_bpf.c b/test/test/test_bpf.c
index cbd6be63..fa17c4f7 100644
--- a/test/test/test_bpf.c
+++ b/test/test/test_bpf.c
@@ -1530,7 +1530,156 @@ static const struct rte_bpf_xsym test_call1_xsym[] = {
 	{
 		.name = RTE_STR(dummy_func1),
 		.type = RTE_BPF_XTYPE_FUNC,
-		.func = (void *)dummy_func1,
+		.func = {
+			.val = (void *)dummy_func1,
+			.nb_args = 3,
+			.args = {
+				[0] = {
+					.type = RTE_BPF_ARG_PTR,
+					.size = sizeof(struct dummy_offset),
+				},
+				[1] = {
+					.type = RTE_BPF_ARG_PTR,
+					.size = sizeof(uint32_t),
+				},
+				[2] = {
+					.type = RTE_BPF_ARG_PTR,
+					.size = sizeof(uint64_t),
+				},
+			},
+		},
+	},
+};
+
+static const struct ebpf_insn test_call2_prog[] = {
+
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_1,
+		.src_reg = EBPF_REG_10,
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_K),
+		.dst_reg = EBPF_REG_1,
+		.imm = -(int32_t)sizeof(struct dummy_offset),
+	},
+	{
+		.code = (EBPF_ALU64 | EBPF_MOV | BPF_X),
+		.dst_reg = EBPF_REG_2,
+		.src_reg = EBPF_REG_10,
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_K),
+		.dst_reg = EBPF_REG_2,
+		.imm = -2 * (int32_t)sizeof(struct dummy_offset),
+	},
+	{
+		.code = (BPF_JMP | EBPF_CALL),
+		.imm = 0,
+	},
+	{
+		.code = (BPF_LDX | BPF_MEM | EBPF_DW),
+		.dst_reg = EBPF_REG_1,
+		.src_reg = EBPF_REG_10,
+		.off = -(int32_t)(sizeof(struct dummy_offset) -
+			offsetof(struct dummy_offset, u64)),
+	},
+	{
+		.code = (BPF_LDX | BPF_MEM | BPF_W),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_10,
+		.off = -(int32_t)(sizeof(struct dummy_offset) -
+			offsetof(struct dummy_offset, u32)),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_1,
+	},
+	{
+		.code = (BPF_LDX | BPF_MEM | BPF_H),
+		.dst_reg = EBPF_REG_1,
+		.src_reg = EBPF_REG_10,
+		.off = -(int32_t)(2 * sizeof(struct dummy_offset) -
+			offsetof(struct dummy_offset, u16)),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_1,
+	},
+	{
+		.code = (BPF_LDX | BPF_MEM | BPF_B),
+		.dst_reg = EBPF_REG_1,
+		.src_reg = EBPF_REG_10,
+		.off = -(int32_t)(2 * sizeof(struct dummy_offset) -
+			offsetof(struct dummy_offset, u8)),
+	},
+	{
+		.code = (EBPF_ALU64 | BPF_ADD | BPF_X),
+		.dst_reg = EBPF_REG_0,
+		.src_reg = EBPF_REG_1,
+	},
+	{
+		.code = (BPF_JMP | EBPF_EXIT),
+	},
+
+};
+
+static void
+dummy_func2(struct dummy_offset *a, struct dummy_offset *b)
+{
+	uint64_t v;
+
+	v = 0;
+	a->u64 = v++;
+	a->u32 = v++;
+	a->u16 = v++;
+	a->u8 = v++;
+	b->u64 = v++;
+	b->u32 = v++;
+	b->u16 = v++;
+	b->u8 = v++;
+}
+
+static int
+test_call2_check(uint64_t rc, const void *arg)
+{
+	uint64_t v;
+	struct dummy_offset a, b;
+
+	RTE_SET_USED(arg);
+
+	dummy_func2(&a, &b);
+	v = a.u64 + a.u32 + b.u16 + b.u8;
+
+	if (v != rc) {
+		printf("%s@%d: invalid return value "
+			"expected=0x%" PRIx64 ", actual=0x%" PRIx64 "\n",
+			__func__, __LINE__, v, rc);
+		return -1;
+	}
+	return 0;
+}
+
+static const struct rte_bpf_xsym test_call2_xsym[] = {
+	{
+		.name = RTE_STR(dummy_func2),
+		.type = RTE_BPF_XTYPE_FUNC,
+		.func = {
+			.val = (void *)dummy_func2,
+			.nb_args = 2,
+			.args = {
+				[0] = {
+					.type = RTE_BPF_ARG_PTR,
+					.size = sizeof(struct dummy_offset),
+				},
+				[1] = {
+					.type = RTE_BPF_ARG_PTR,
+					.size = sizeof(struct dummy_offset),
+				},
+			},
+		},
 	},
 };
 
@@ -1693,6 +1842,24 @@ static const struct bpf_test tests[] = {
 		/* for now don't support function calls on 32 bit platform */
 		.allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)),
 	},
+	{
+		.name = "test_call2",
+		.arg_sz = sizeof(struct dummy_offset),
+		.prm = {
+			.ins = test_call2_prog,
+			.nb_ins = RTE_DIM(test_call2_prog),
+			.prog_arg = {
+				.type = RTE_BPF_ARG_PTR,
+				.size = sizeof(struct dummy_offset),
+			},
+			.xsym = test_call2_xsym,
+			.nb_xsym = RTE_DIM(test_call2_xsym),
+		},
+		.prepare = test_store1_prepare,
+		.check_result = test_call2_check,
+		/* for now don't support function calls on 32 bit platform */
+		.allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)),
+	},
 };
 
 static int
diff --git a/test/test/test_compressdev.c b/test/test/test_compressdev.c
index 640942ba..86453882 100644
--- a/test/test/test_compressdev.c
+++ b/test/test/test_compressdev.c
@@ -14,6 +14,8 @@
 #include "test_compressdev_test_buffer.h"
 #include "test.h"
 
+#define DIV_CEIL(a, b)  ((a) / (b) + ((a) % (b) != 0))
+
 #define DEFAULT_WINDOW_SIZE 15
 #define DEFAULT_MEM_LEVEL 8
 #define MAX_DEQD_RETRIES 10
@@ -25,7 +27,9 @@
  * due to the compress block headers
  */
 #define COMPRESS_BUF_SIZE_RATIO 1.3
-#define NUM_MBUFS 16
+#define NUM_LARGE_MBUFS 16
+#define SMALL_SEG_SIZE 256
+#define MAX_SEGS 16
 #define NUM_OPS 16
 #define NUM_MAX_XFORMS 16
 #define NUM_MAX_INFLIGHT_OPS 128
@@ -50,7 +54,8 @@ struct priv_op_data {
 };
 
 struct comp_testsuite_params {
-	struct rte_mempool *mbuf_pool;
+	struct rte_mempool *large_mbuf_pool;
+	struct rte_mempool *small_mbuf_pool;
 	struct rte_mempool *op_pool;
 	struct rte_comp_xform *def_comp_xform;
 	struct rte_comp_xform *def_decomp_xform;
@@ -63,7 +68,8 @@ testsuite_teardown(void)
 {
 	struct comp_testsuite_params *ts_params = &testsuite_params;
 
-	rte_mempool_free(ts_params->mbuf_pool);
+	rte_mempool_free(ts_params->large_mbuf_pool);
+	rte_mempool_free(ts_params->small_mbuf_pool);
 	rte_mempool_free(ts_params->op_pool);
 	rte_free(ts_params->def_comp_xform);
 	rte_free(ts_params->def_decomp_xform);
@@ -73,6 +79,7 @@ static int
 testsuite_setup(void)
 {
 	struct comp_testsuite_params *ts_params = &testsuite_params;
+	uint32_t max_buf_size = 0;
 	unsigned int i;
 
 	if (rte_compressdev_count() == 0) {
@@ -80,28 +87,41 @@ testsuite_setup(void)
 		return TEST_FAILED;
 	}
 
-	uint32_t max_buf_size = 0;
+	RTE_LOG(NOTICE, USER1, "Running tests on device %s\n",
+				rte_compressdev_name_get(0));
+
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++)
 		max_buf_size = RTE_MAX(max_buf_size,
 				strlen(compress_test_bufs[i]) + 1);
 
-	max_buf_size *= COMPRESS_BUF_SIZE_RATIO;
 	/*
 	 * Buffers to be used in compression and decompression.
 	 * Since decompressed data might be larger than
 	 * compressed data (due to block header),
 	 * buffers should be big enough for both cases.
 	 */
-	ts_params->mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool",
-			NUM_MBUFS,
+	max_buf_size *= COMPRESS_BUF_SIZE_RATIO;
+	ts_params->large_mbuf_pool = rte_pktmbuf_pool_create("large_mbuf_pool",
+			NUM_LARGE_MBUFS,
 			CACHE_SIZE, 0,
 			max_buf_size + RTE_PKTMBUF_HEADROOM,
 			rte_socket_id());
-	if (ts_params->mbuf_pool == NULL) {
+	if (ts_params->large_mbuf_pool == NULL) {
 		RTE_LOG(ERR, USER1, "Large mbuf pool could not be created\n");
 		return TEST_FAILED;
 	}
 
+	/* Create mempool with smaller buffers for SGL testing */
+	ts_params->small_mbuf_pool = rte_pktmbuf_pool_create("small_mbuf_pool",
+			NUM_LARGE_MBUFS * MAX_SEGS,
+			CACHE_SIZE, 0,
+			SMALL_SEG_SIZE + RTE_PKTMBUF_HEADROOM,
+			rte_socket_id());
+	if (ts_params->small_mbuf_pool == NULL) {
+		RTE_LOG(ERR, USER1, "Small mbuf pool could not be created\n");
+		goto exit;
+	}
+
 	ts_params->op_pool = rte_comp_op_pool_create("op_pool", NUM_OPS,
 				0, sizeof(struct priv_op_data),
 				rte_socket_id());
@@ -278,7 +298,9 @@ compress_zlib(struct rte_comp_op *op,
 	z_stream stream;
 	int zlib_flush;
 	int strategy, window_bits, comp_level;
-	int ret = -1;
+	int ret = TEST_FAILED;
+	uint8_t *single_src_buf = NULL;
+	uint8_t *single_dst_buf = NULL;
 
 	/* initialize zlib stream */
 	stream.zalloc = Z_NULL;
@@ -310,11 +332,39 @@ compress_zlib(struct rte_comp_op *op,
 	}
 
 	/* Assuming stateless operation */
-	stream.avail_in = op->src.length;
-	stream.next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
-	stream.avail_out = op->m_dst->data_len;
-	stream.next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
+	/* SGL */
+	if (op->m_src->nb_segs > 1) {
+		single_src_buf = rte_malloc(NULL,
+				rte_pktmbuf_pkt_len(op->m_src), 0);
+		if (single_src_buf == NULL) {
+			RTE_LOG(ERR, USER1, "Buffer could not be allocated\n");
+			goto exit;
+		}
+		single_dst_buf = rte_malloc(NULL,
+				rte_pktmbuf_pkt_len(op->m_dst), 0);
+		if (single_dst_buf == NULL) {
+			RTE_LOG(ERR, USER1, "Buffer could not be allocated\n");
+			goto exit;
+		}
+		if (rte_pktmbuf_read(op->m_src, 0,
+					rte_pktmbuf_pkt_len(op->m_src),
+					single_src_buf) == NULL) {
+			RTE_LOG(ERR, USER1,
+				"Buffer could not be read entirely\n");
+			goto exit;
+		}
+
+		stream.avail_in = op->src.length;
+		stream.next_in = single_src_buf;
+		stream.avail_out = rte_pktmbuf_pkt_len(op->m_dst);
+		stream.next_out = single_dst_buf;
 
+	} else {
+		stream.avail_in = op->src.length;
+		stream.next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
+		stream.avail_out = op->m_dst->data_len;
+		stream.next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
+	}
 	/* Stateless operation, all buffer will be compressed in one go */
 	zlib_flush = map_zlib_flush_flag(op->flush_flag);
 	ret = deflate(&stream, zlib_flush);
@@ -327,8 +377,30 @@ compress_zlib(struct rte_comp_op *op,
 	if (ret != Z_STREAM_END)
 		goto exit;
 
-	op->consumed = op->src.length - stream.avail_in;
-	op->produced = op->m_dst->data_len - stream.avail_out;
+	/* Copy data to destination SGL */
+	if (op->m_src->nb_segs > 1) {
+		uint32_t remaining_data = stream.total_out;
+		uint8_t *src_data = single_dst_buf;
+		struct rte_mbuf *dst_buf = op->m_dst;
+
+		while (remaining_data > 0) {
+			uint8_t *dst_data = rte_pktmbuf_mtod(dst_buf,
+					uint8_t *);
+			/* Last segment */
+			if (remaining_data < dst_buf->data_len) {
+				memcpy(dst_data, src_data, remaining_data);
+				remaining_data = 0;
+			} else {
+				memcpy(dst_data, src_data, dst_buf->data_len);
+				remaining_data -= dst_buf->data_len;
+				src_data += dst_buf->data_len;
+				dst_buf = dst_buf->next;
+			}
+		}
+	}
+
+	op->consumed = stream.total_in;
+	op->produced = stream.total_out;
 	op->status = RTE_COMP_OP_STATUS_SUCCESS;
 
 	deflateReset(&stream);
@@ -336,6 +408,8 @@ compress_zlib(struct rte_comp_op *op,
 	ret = 0;
 exit:
 	deflateEnd(&stream);
+	rte_free(single_src_buf);
+	rte_free(single_dst_buf);
 
 	return ret;
 }
@@ -348,6 +422,8 @@ decompress_zlib(struct rte_comp_op *op,
 	int window_bits;
 	int zlib_flush;
 	int ret = TEST_FAILED;
+	uint8_t *single_src_buf = NULL;
+	uint8_t *single_dst_buf = NULL;
 
 	/* initialize zlib stream */
 	stream.zalloc = Z_NULL;
@@ -368,10 +444,39 @@ decompress_zlib(struct rte_comp_op *op,
 	}
 
 	/* Assuming stateless operation */
-	stream.avail_in = op->src.length;
-	stream.next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
-	stream.avail_out = op->m_dst->data_len;
-	stream.next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
+	/* SGL */
+	if (op->m_src->nb_segs > 1) {
+		single_src_buf = rte_malloc(NULL,
+				rte_pktmbuf_pkt_len(op->m_src), 0);
+		if (single_src_buf == NULL) {
+			RTE_LOG(ERR, USER1, "Buffer could not be allocated\n");
+			goto exit;
+		}
+		single_dst_buf = rte_malloc(NULL,
+				rte_pktmbuf_pkt_len(op->m_dst), 0);
+		if (single_dst_buf == NULL) {
+			RTE_LOG(ERR, USER1, "Buffer could not be allocated\n");
+			goto exit;
+		}
+		if (rte_pktmbuf_read(op->m_src, 0,
+					rte_pktmbuf_pkt_len(op->m_src),
+					single_src_buf) == NULL) {
+			RTE_LOG(ERR, USER1,
+				"Buffer could not be read entirely\n");
+			goto exit;
+		}
+
+		stream.avail_in = op->src.length;
+		stream.next_in = single_src_buf;
+		stream.avail_out = rte_pktmbuf_pkt_len(op->m_dst);
+		stream.next_out = single_dst_buf;
+
+	} else {
+		stream.avail_in = op->src.length;
+		stream.next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *);
+		stream.avail_out = op->m_dst->data_len;
+		stream.next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *);
+	}
 
 	/* Stateless operation, all buffer will be compressed in one go */
 	zlib_flush = map_zlib_flush_flag(op->flush_flag);
@@ -385,8 +490,29 @@ decompress_zlib(struct rte_comp_op *op,
 	if (ret != Z_STREAM_END)
 		goto exit;
 
-	op->consumed = op->src.length - stream.avail_in;
-	op->produced = op->m_dst->data_len - stream.avail_out;
+	if (op->m_src->nb_segs > 1) {
+		uint32_t remaining_data = stream.total_out;
+		uint8_t *src_data = single_dst_buf;
+		struct rte_mbuf *dst_buf = op->m_dst;
+
+		while (remaining_data > 0) {
+			uint8_t *dst_data = rte_pktmbuf_mtod(dst_buf,
+					uint8_t *);
+			/* Last segment */
+			if (remaining_data < dst_buf->data_len) {
+				memcpy(dst_data, src_data, remaining_data);
+				remaining_data = 0;
+			} else {
+				memcpy(dst_data, src_data, dst_buf->data_len);
+				remaining_data -= dst_buf->data_len;
+				src_data += dst_buf->data_len;
+				dst_buf = dst_buf->next;
+			}
+		}
+	}
+
+	op->consumed = stream.total_in;
+	op->produced = stream.total_out;
 	op->status = RTE_COMP_OP_STATUS_SUCCESS;
 
 	inflateReset(&stream);
@@ -398,6 +524,100 @@ exit:
 	return ret;
 }
 
+static int
+prepare_sgl_bufs(const char *test_buf, struct rte_mbuf *head_buf,
+		uint32_t total_data_size,
+		struct rte_mempool *small_mbuf_pool,
+		struct rte_mempool *large_mbuf_pool,
+		uint8_t limit_segs_in_sgl)
+{
+	uint32_t remaining_data = total_data_size;
+	uint16_t num_remaining_segs = DIV_CEIL(remaining_data, SMALL_SEG_SIZE);
+	struct rte_mempool *pool;
+	struct rte_mbuf *next_seg;
+	uint32_t data_size;
+	char *buf_ptr;
+	const char *data_ptr = test_buf;
+	uint16_t i;
+	int ret;
+
+	if (limit_segs_in_sgl != 0 && num_remaining_segs > limit_segs_in_sgl)
+		num_remaining_segs = limit_segs_in_sgl - 1;
+
+	/*
+	 * Allocate data in the first segment (header) and
+	 * copy data if test buffer is provided
+	 */
+	if (remaining_data < SMALL_SEG_SIZE)
+		data_size = remaining_data;
+	else
+		data_size = SMALL_SEG_SIZE;
+	buf_ptr = rte_pktmbuf_append(head_buf, data_size);
+	if (buf_ptr == NULL) {
+		RTE_LOG(ERR, USER1,
+			"Not enough space in the 1st buffer\n");
+		return -1;
+	}
+
+	if (data_ptr != NULL) {
+		/* Copy characters without NULL terminator */
+		strncpy(buf_ptr, data_ptr, data_size);
+		data_ptr += data_size;
+	}
+	remaining_data -= data_size;
+	num_remaining_segs--;
+
+	/*
+	 * Allocate the rest of the segments,
+	 * copy the rest of the data and chain the segments.
+	 */
+	for (i = 0; i < num_remaining_segs; i++) {
+
+		if (i == (num_remaining_segs - 1)) {
+			/* last segment */
+			if (remaining_data > SMALL_SEG_SIZE)
+				pool = large_mbuf_pool;
+			else
+				pool = small_mbuf_pool;
+			data_size = remaining_data;
+		} else {
+			data_size = SMALL_SEG_SIZE;
+			pool = small_mbuf_pool;
+		}
+
+		next_seg = rte_pktmbuf_alloc(pool);
+		if (next_seg == NULL) {
+			RTE_LOG(ERR, USER1,
+				"New segment could not be allocated "
+				"from the mempool\n");
+			return -1;
+		}
+		buf_ptr = rte_pktmbuf_append(next_seg, data_size);
+		if (buf_ptr == NULL) {
+			RTE_LOG(ERR, USER1,
+				"Not enough space in the buffer\n");
+			rte_pktmbuf_free(next_seg);
+			return -1;
+		}
+		if (data_ptr != NULL) {
+			/* Copy characters without NULL terminator */
+			strncpy(buf_ptr, data_ptr, data_size);
+			data_ptr += data_size;
+		}
+		remaining_data -= data_size;
+
+		ret = rte_pktmbuf_chain(head_buf, next_seg);
+		if (ret != 0) {
+			rte_pktmbuf_free(next_seg);
+			RTE_LOG(ERR, USER1,
+				"Segment could not chained\n");
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
 /*
  * Compresses and decompresses buffer with compressdev API and Zlib API
  */
@@ -409,6 +629,7 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 		struct rte_comp_xform *decompress_xforms[],
 		unsigned int num_xforms,
 		enum rte_comp_op_type state,
+		unsigned int sgl,
 		enum zlib_direction zlib_dir)
 {
 	struct comp_testsuite_params *ts_params = &testsuite_params;
@@ -423,10 +644,13 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 	uint16_t num_priv_xforms = 0;
 	unsigned int deqd_retries = 0;
 	struct priv_op_data *priv_data;
-	char *data_ptr;
+	char *buf_ptr;
 	unsigned int i;
+	struct rte_mempool *buf_pool;
+	uint32_t data_size;
 	const struct rte_compressdev_capabilities *capa =
 		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	char *contig_buf = NULL;
 
 	/* Initialize all arrays to NULL */
 	memset(uncomp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
@@ -435,8 +659,14 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 	memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
 	memset(priv_xforms, 0, sizeof(void *) * num_bufs);
 
+	if (sgl)
+		buf_pool = ts_params->small_mbuf_pool;
+	else
+		buf_pool = ts_params->large_mbuf_pool;
+
 	/* Prepare the source mbufs with the data */
-	ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, uncomp_bufs, num_bufs);
+	ret = rte_pktmbuf_alloc_bulk(buf_pool,
+				uncomp_bufs, num_bufs);
 	if (ret < 0) {
 		RTE_LOG(ERR, USER1,
 			"Source mbufs could not be allocated "
@@ -444,15 +674,26 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 		goto exit;
 	}
 
-	for (i = 0; i < num_bufs; i++) {
-		data_ptr = rte_pktmbuf_append(uncomp_bufs[i],
-					strlen(test_bufs[i]) + 1);
-		snprintf(data_ptr, strlen(test_bufs[i]) + 1, "%s",
-					test_bufs[i]);
+	if (sgl) {
+		for (i = 0; i < num_bufs; i++) {
+			data_size = strlen(test_bufs[i]) + 1;
+			if (prepare_sgl_bufs(test_bufs[i], uncomp_bufs[i],
+					data_size,
+					ts_params->small_mbuf_pool,
+					ts_params->large_mbuf_pool,
+					MAX_SEGS) < 0)
+				goto exit;
+		}
+	} else {
+		for (i = 0; i < num_bufs; i++) {
+			data_size = strlen(test_bufs[i]) + 1;
+			buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			snprintf(buf_ptr, data_size, "%s", test_bufs[i]);
+		}
 	}
 
 	/* Prepare the destination mbufs */
-	ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, comp_bufs, num_bufs);
+	ret = rte_pktmbuf_alloc_bulk(buf_pool, comp_bufs, num_bufs);
 	if (ret < 0) {
 		RTE_LOG(ERR, USER1,
 			"Destination mbufs could not be allocated "
@@ -460,9 +701,25 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 		goto exit;
 	}
 
-	for (i = 0; i < num_bufs; i++)
-		rte_pktmbuf_append(comp_bufs[i],
-			strlen(test_bufs[i]) * COMPRESS_BUF_SIZE_RATIO);
+	if (sgl) {
+		for (i = 0; i < num_bufs; i++) {
+			data_size = strlen(test_bufs[i]) *
+				COMPRESS_BUF_SIZE_RATIO;
+			if (prepare_sgl_bufs(NULL, comp_bufs[i],
+					data_size,
+					ts_params->small_mbuf_pool,
+					ts_params->large_mbuf_pool,
+					MAX_SEGS) < 0)
+				goto exit;
+		}
+
+	} else {
+		for (i = 0; i < num_bufs; i++) {
+			data_size = strlen(test_bufs[i]) *
+				COMPRESS_BUF_SIZE_RATIO;
+			rte_pktmbuf_append(comp_bufs[i], data_size);
+		}
+	}
 
 	/* Build the compression operations */
 	ret = rte_comp_op_bulk_alloc(ts_params->op_pool, ops, num_bufs);
@@ -627,7 +884,7 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 	}
 
 	/* Allocate buffers for decompressed data */
-	ret = rte_pktmbuf_alloc_bulk(ts_params->mbuf_pool, uncomp_bufs, num_bufs);
+	ret = rte_pktmbuf_alloc_bulk(buf_pool, uncomp_bufs, num_bufs);
 	if (ret < 0) {
 		RTE_LOG(ERR, USER1,
 			"Destination mbufs could not be allocated "
@@ -635,10 +892,26 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 		goto exit;
 	}
 
-	for (i = 0; i < num_bufs; i++) {
-		priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
-		rte_pktmbuf_append(uncomp_bufs[i],
-				strlen(test_bufs[priv_data->orig_idx]) + 1);
+	if (sgl) {
+		for (i = 0; i < num_bufs; i++) {
+			priv_data = (struct priv_op_data *)
+					(ops_processed[i] + 1);
+			data_size = strlen(test_bufs[priv_data->orig_idx]) + 1;
+			if (prepare_sgl_bufs(NULL, uncomp_bufs[i],
+					data_size,
+					ts_params->small_mbuf_pool,
+					ts_params->large_mbuf_pool,
+					MAX_SEGS) < 0)
+				goto exit;
+		}
+
+	} else {
+		for (i = 0; i < num_bufs; i++) {
+			priv_data = (struct priv_op_data *)
+					(ops_processed[i] + 1);
+			data_size = strlen(test_bufs[priv_data->orig_idx]) + 1;
+			rte_pktmbuf_append(uncomp_bufs[i], data_size);
+		}
 	}
 
 	/* Build the decompression operations */
@@ -813,12 +1086,23 @@ test_deflate_comp_decomp(const char * const test_bufs[],
 	for (i = 0; i < num_bufs; i++) {
 		priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 		const char *buf1 = test_bufs[priv_data->orig_idx];
-		const char *buf2 = rte_pktmbuf_mtod(ops_processed[i]->m_dst,
-				const char *);
+		const char *buf2;
+		contig_buf = rte_malloc(NULL, ops_processed[i]->produced, 0);
+		if (contig_buf == NULL) {
+			RTE_LOG(ERR, USER1, "Contiguous buffer could not "
+					"be allocated\n");
+			goto exit;
+		}
+
+		buf2 = rte_pktmbuf_read(ops_processed[i]->m_dst, 0,
+				ops_processed[i]->produced, contig_buf);
 
 		if (compare_buffers(buf1, strlen(buf1) + 1,
 				buf2, ops_processed[i]->produced) < 0)
 			goto exit;
+
+		rte_free(contig_buf);
+		contig_buf = NULL;
 	}
 
 	ret_status = 0;
@@ -835,6 +1119,7 @@ exit:
 		if (priv_xforms[i] != NULL)
 			rte_compressdev_private_xform_free(0, priv_xforms[i]);
 	}
+	rte_free(contig_buf);
 
 	return ret_status;
 }
@@ -846,6 +1131,14 @@ test_compressdev_deflate_stateless_fixed(void)
 	const char *test_buffer;
 	uint16_t i;
 	int ret;
+	const struct rte_compressdev_capabilities *capab;
+
+	capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+	if ((capab->comp_feature_flags & RTE_COMP_FF_HUFFMAN_FIXED) == 0)
+		return -ENOTSUP;
+
 	struct rte_comp_xform *compress_xform =
 			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 
@@ -870,6 +1163,7 @@ test_compressdev_deflate_stateless_fixed(void)
 				&ts_params->def_decomp_xform,
 				1,
 				RTE_COMP_OP_STATELESS,
+				0,
 				ZLIB_DECOMPRESS) < 0) {
 			ret = TEST_FAILED;
 			goto exit;
@@ -882,6 +1176,7 @@ test_compressdev_deflate_stateless_fixed(void)
 				&ts_params->def_decomp_xform,
 				1,
 				RTE_COMP_OP_STATELESS,
+				0,
 				ZLIB_COMPRESS) < 0) {
 			ret = TEST_FAILED;
 			goto exit;
@@ -905,6 +1200,14 @@ test_compressdev_deflate_stateless_dynamic(void)
 	struct rte_comp_xform *compress_xform =
 			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
 
+	const struct rte_compressdev_capabilities *capab;
+
+	capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+	if ((capab->comp_feature_flags & RTE_COMP_FF_HUFFMAN_DYNAMIC) == 0)
+		return -ENOTSUP;
+
 	if (compress_xform == NULL) {
 		RTE_LOG(ERR, USER1,
 			"Compress xform could not be created\n");
@@ -926,6 +1229,7 @@ test_compressdev_deflate_stateless_dynamic(void)
 				&ts_params->def_decomp_xform,
 				1,
 				RTE_COMP_OP_STATELESS,
+				0,
 				ZLIB_DECOMPRESS) < 0) {
 			ret = TEST_FAILED;
 			goto exit;
@@ -938,6 +1242,7 @@ test_compressdev_deflate_stateless_dynamic(void)
 				&ts_params->def_decomp_xform,
 				1,
 				RTE_COMP_OP_STATELESS,
+				0,
 				ZLIB_COMPRESS) < 0) {
 			ret = TEST_FAILED;
 			goto exit;
@@ -969,6 +1274,7 @@ test_compressdev_deflate_stateless_multi_op(void)
 			&ts_params->def_decomp_xform,
 			1,
 			RTE_COMP_OP_STATELESS,
+			0,
 			ZLIB_DECOMPRESS) < 0)
 		return TEST_FAILED;
 
@@ -979,6 +1285,7 @@ test_compressdev_deflate_stateless_multi_op(void)
 			&ts_params->def_decomp_xform,
 			1,
 			RTE_COMP_OP_STATELESS,
+			0,
 			ZLIB_COMPRESS) < 0)
 		return TEST_FAILED;
 
@@ -1018,6 +1325,7 @@ test_compressdev_deflate_stateless_multi_level(void)
 					&ts_params->def_decomp_xform,
 					1,
 					RTE_COMP_OP_STATELESS,
+					0,
 					ZLIB_DECOMPRESS) < 0) {
 				ret = TEST_FAILED;
 				goto exit;
@@ -1088,6 +1396,7 @@ test_compressdev_deflate_stateless_multi_xform(void)
 			decompress_xforms,
 			NUM_XFORMS,
 			RTE_COMP_OP_STATELESS,
+			0,
 			ZLIB_DECOMPRESS) < 0) {
 		ret = TEST_FAILED;
 		goto exit;
@@ -1103,6 +1412,48 @@ exit:
 	return ret;
 }
 
+static int
+test_compressdev_deflate_stateless_sgl(void)
+{
+	struct comp_testsuite_params *ts_params = &testsuite_params;
+	uint16_t i;
+	const char *test_buffer;
+	const struct rte_compressdev_capabilities *capab;
+
+	capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+	if ((capab->comp_feature_flags & RTE_COMP_FF_OOP_SGL_IN_SGL_OUT) == 0)
+		return -ENOTSUP;
+
+	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+		test_buffer = compress_test_bufs[i];
+		/* Compress with compressdev, decompress with Zlib */
+		if (test_deflate_comp_decomp(&test_buffer, 1,
+				&i,
+				&ts_params->def_comp_xform,
+				&ts_params->def_decomp_xform,
+				1,
+				RTE_COMP_OP_STATELESS,
+				1,
+				ZLIB_DECOMPRESS) < 0)
+			return TEST_FAILED;
+
+		/* Compress with Zlib, decompress with compressdev */
+		if (test_deflate_comp_decomp(&test_buffer, 1,
+				&i,
+				&ts_params->def_comp_xform,
+				&ts_params->def_decomp_xform,
+				1,
+				RTE_COMP_OP_STATELESS,
+				1,
+				ZLIB_COMPRESS) < 0)
+			return TEST_FAILED;
+	}
+
+	return TEST_SUCCESS;
+}
+
 static struct unit_test_suite compressdev_testsuite  = {
 	.suite_name = "compressdev unit test suite",
 	.setup = testsuite_setup,
@@ -1120,6 +1471,8 @@ static struct unit_test_suite compressdev_testsuite  = {
 			test_compressdev_deflate_stateless_multi_level),
 		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
 			test_compressdev_deflate_stateless_multi_xform),
+		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+			test_compressdev_deflate_stateless_sgl),
 		TEST_CASES_END() /**< NULL terminate unit test array */
 	}
 };
diff --git a/test/test/test_cryptodev.c b/test/test/test_cryptodev.c
index 389f7967..a6044b26 100644
--- a/test/test/test_cryptodev.c
+++ b/test/test/test_cryptodev.c
@@ -39,6 +39,7 @@
 #include "test_cryptodev_hmac_test_vectors.h"
 
 #define VDEV_ARGS_SIZE 100
+#define MAX_NB_SESSIONS            4
 
 static int gbl_driver_id;
 
@@ -435,9 +436,17 @@ testsuite_setup(void)
 	 * Create mempool with maximum number of sessions * 2,
 	 * to include the session headers
 	 */
+	if (info.sym.max_nb_sessions != 0 &&
+			info.sym.max_nb_sessions < MAX_NB_SESSIONS) {
+		RTE_LOG(ERR, USER1, "Device does not support "
+				"at least %u sessions\n",
+				MAX_NB_SESSIONS);
+		return TEST_FAILED;
+	}
+
 	ts_params->session_mpool = rte_mempool_create(
 				"test_sess_mp",
-				info.sym.max_nb_sessions * 2,
+				MAX_NB_SESSIONS * 2,
 				session_size,
 				0, 0, NULL, NULL, NULL,
 				NULL, SOCKET_ID_ANY,
@@ -585,7 +594,7 @@ test_device_configure_invalid_dev_id(void)
 	dev_id = ts_params->valid_devs[ts_params->valid_dev_count - 1];
 
 	/* Stop the device in case it's started so it can be configured */
-	rte_cryptodev_stop(ts_params->valid_devs[dev_id]);
+	rte_cryptodev_stop(dev_id);
 
 	TEST_ASSERT_SUCCESS(rte_cryptodev_configure(dev_id, &ts_params->conf),
 			"Failed test for rte_cryptodev_configure: "
@@ -3151,8 +3160,11 @@ test_kasumi_encryption_sgl(const struct kasumi_test_data *tdata)
 	struct rte_cryptodev_info dev_info;
 
 	rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info);
-	if (!(dev_info.feature_flags & RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER)) {
-		printf("Device doesn't support scatter-gather. "
+
+	uint64_t feat_flags = dev_info.feature_flags;
+
+	if (!(feat_flags & RTE_CRYPTODEV_FF_IN_PLACE_SGL)) {
+		printf("Device doesn't support in-place scatter-gather. "
 				"Test Skipped.\n");
 		return 0;
 	}
@@ -3299,8 +3311,11 @@ test_kasumi_encryption_oop_sgl(const struct kasumi_test_data *tdata)
 	struct rte_cryptodev_info dev_info;
 
 	rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info);
-	if (!(dev_info.feature_flags & RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER)) {
-		printf("Device doesn't support scatter-gather. "
+
+	uint64_t feat_flags = dev_info.feature_flags;
+	if (!(feat_flags & RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT)) {
+		printf("Device doesn't support out-of-place scatter-gather "
+				"in both input and output mbufs. "
 				"Test Skipped.\n");
 		return 0;
 	}
@@ -3650,8 +3665,12 @@ test_snow3g_encryption_oop_sgl(const struct snow3g_test_data *tdata)
 	struct rte_cryptodev_info dev_info;
 
 	rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info);
-	if (!(dev_info.feature_flags & RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER)) {
-		printf("Device doesn't support scatter-gather. "
+
+	uint64_t feat_flags = dev_info.feature_flags;
+
+	if (!(feat_flags & RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT)) {
+		printf("Device doesn't support out-of-place scatter-gather "
+				"in both input and output mbufs. "
 				"Test Skipped.\n");
 		return 0;
 	}
@@ -4484,10 +4503,13 @@ test_zuc_encryption_sgl(const struct wireless_test_data *tdata)
 		return -ENOTSUP;
 
 	rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info);
-	if (!(dev_info.feature_flags & RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER)) {
-		printf("Device doesn't support scatter-gather. "
+
+	uint64_t feat_flags = dev_info.feature_flags;
+
+	if (!(feat_flags & RTE_CRYPTODEV_FF_IN_PLACE_SGL)) {
+		printf("Device doesn't support in-place scatter-gather. "
 				"Test Skipped.\n");
-		return -ENOTSUP;
+		return 0;
 	}
 
 	plaintext_len = ceil_byte_length(tdata->plaintext.len);
@@ -4998,6 +5020,24 @@ test_DES_cipheronly_mb_all(void)
 
 	return TEST_SUCCESS;
 }
+static int
+test_3DES_cipheronly_mb_all(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	int status;
+
+	status = test_blockcipher_all_tests(ts_params->mbuf_pool,
+		ts_params->op_mpool,
+		ts_params->session_mpool,
+		ts_params->valid_devs[0],
+		rte_cryptodev_driver_id_get(
+		RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)),
+		BLKCIPHER_3DES_CIPHERONLY_TYPE);
+
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	return TEST_SUCCESS;
+}
 
 static int
 test_DES_docsis_mb_all(void)
@@ -6499,10 +6539,10 @@ test_multi_session(void)
 
 	sessions = rte_malloc(NULL,
 			(sizeof(struct rte_cryptodev_sym_session *) *
-			dev_info.sym.max_nb_sessions) + 1, 0);
+			MAX_NB_SESSIONS) + 1, 0);
 
 	/* Create multiple crypto sessions*/
-	for (i = 0; i < dev_info.sym.max_nb_sessions; i++) {
+	for (i = 0; i < MAX_NB_SESSIONS; i++) {
 
 		sessions[i] = rte_cryptodev_sym_session_create(
 				ts_params->session_mpool);
@@ -6551,7 +6591,7 @@ test_multi_session(void)
 	TEST_ASSERT_NULL(sessions[i],
 			"Session creation succeeded unexpectedly!");
 
-	for (i = 0; i < dev_info.sym.max_nb_sessions; i++) {
+	for (i = 0; i < MAX_NB_SESSIONS; i++) {
 		rte_cryptodev_sym_session_clear(ts_params->valid_devs[0],
 				sessions[i]);
 		rte_cryptodev_sym_session_free(sessions[i]);
@@ -6610,7 +6650,7 @@ test_multi_session_random_usage(void)
 
 	sessions = rte_malloc(NULL,
 			(sizeof(struct rte_cryptodev_sym_session *)
-					* dev_info.sym.max_nb_sessions) + 1, 0);
+					* MAX_NB_SESSIONS) + 1, 0);
 
 	for (i = 0; i < MB_SESSION_NUMBER; i++) {
 		sessions[i] = rte_cryptodev_sym_session_create(
@@ -8538,6 +8578,14 @@ test_scheduler_attach_slave_op(void)
 		unsigned int session_size =
 			rte_cryptodev_sym_get_private_session_size(i);
 
+		if (info.sym.max_nb_sessions != 0 &&
+				info.sym.max_nb_sessions < MAX_NB_SESSIONS) {
+			RTE_LOG(ERR, USER1,
+					"Device does not support "
+					"at least %u sessions\n",
+					MAX_NB_SESSIONS);
+			return TEST_FAILED;
+		}
 		/*
 		 * Create mempool with maximum number of sessions * 2,
 		 * to include the session headers
@@ -8545,7 +8593,7 @@ test_scheduler_attach_slave_op(void)
 		if (ts_params->session_mpool == NULL) {
 			ts_params->session_mpool = rte_mempool_create(
 					"test_sess_mp",
-					info.sym.max_nb_sessions * 2,
+					MAX_NB_SESSIONS * 2,
 					session_size,
 					0, 0, NULL, NULL, NULL,
 					NULL, SOCKET_ID_ANY,
@@ -9003,6 +9051,8 @@ static struct unit_test_suite cryptodev_aesni_mb_testsuite  = {
 		TEST_CASE_ST(ut_setup, ut_teardown,
 						test_DES_docsis_mb_all),
 		TEST_CASE_ST(ut_setup, ut_teardown,
+						test_3DES_cipheronly_mb_all),
+		TEST_CASE_ST(ut_setup, ut_teardown,
 			test_AES_CCM_authenticated_encryption_test_case_128_1),
 		TEST_CASE_ST(ut_setup, ut_teardown,
 			test_AES_CCM_authenticated_decryption_test_case_128_1),
@@ -9908,9 +9958,9 @@ test_cryptodev_qat(void /*argv __rte_unused, int argc __rte_unused*/)
 			RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD));
 
 	if (gbl_driver_id == -1) {
-		RTE_LOG(ERR, USER1, "QAT PMD must be loaded. Check if "
-				"CONFIG_RTE_LIBRTE_PMD_QAT is enabled "
-				"in config file to run this testsuite.\n");
+		RTE_LOG(ERR, USER1, "QAT PMD must be loaded. Check that both "
+		"CONFIG_RTE_LIBRTE_PMD_QAT and CONFIG_RTE_LIBRTE_PMD_QAT_SYM "
+		"are enabled in config file to run this testsuite.\n");
 		return TEST_SKIPPED;
 	}
 
diff --git a/test/test/test_cryptodev_asym.c b/test/test/test_cryptodev_asym.c
new file mode 100644
index 00000000..2fdfc1df
--- /dev/null
+++ b/test/test/test_cryptodev_asym.c
@@ -0,0 +1,1369 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium Networks
+ */
+
+#include <rte_bus_vdev.h>
+#include <rte_common.h>
+#include <rte_hexdump.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_memcpy.h>
+#include <rte_pause.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_crypto.h>
+
+#include "test_cryptodev.h"
+#include "test_cryptodev_dh_test_vectors.h"
+#include "test_cryptodev_dsa_test_vectors.h"
+#include "test_cryptodev_mod_test_vectors.h"
+#include "test_cryptodev_rsa_test_vectors.h"
+#include "test_cryptodev_asym_util.h"
+#include "test.h"
+
+#define TEST_NUM_BUFS 10
+#define TEST_NUM_SESSIONS 4
+
+static int gbl_driver_id;
+struct crypto_testsuite_params {
+	struct rte_mempool *op_mpool;
+	struct rte_mempool *session_mpool;
+	struct rte_cryptodev_config conf;
+	struct rte_cryptodev_qp_conf qp_conf;
+	uint8_t valid_devs[RTE_CRYPTO_MAX_DEVS];
+	uint8_t valid_dev_count;
+};
+
+struct crypto_unittest_params {
+	struct rte_cryptodev_asym_session *sess;
+	struct rte_crypto_op *op;
+};
+
+static struct crypto_testsuite_params testsuite_params = { NULL };
+
+static int
+test_rsa_sign_verify(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct rte_mempool *op_mpool = ts_params->op_mpool;
+	struct rte_mempool *sess_mpool = ts_params->session_mpool;
+	uint8_t dev_id = ts_params->valid_devs[0];
+	struct rte_crypto_asym_op *asym_op = NULL;
+	struct rte_crypto_op *op = NULL, *result_op = NULL;
+	struct rte_cryptodev_asym_session *sess = NULL;
+	int status = TEST_SUCCESS;
+	uint8_t output_buf[TEST_DATA_SIZE] = {0};
+	uint8_t input_buf[TEST_DATA_SIZE] = {0};
+
+	sess = rte_cryptodev_asym_session_create(sess_mpool);
+
+	if (!sess) {
+		RTE_LOG(ERR, USER1, "line %u "
+				"FAILED: %s", __LINE__,
+				"Session creation failed");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	if (rte_cryptodev_asym_session_init(dev_id, sess, &rsa_xform,
+				sess_mpool) < 0) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "unabled to config sym session");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	/* set up crypto op data structure */
+	op = rte_crypto_op_alloc(op_mpool, RTE_CRYPTO_OP_TYPE_ASYMMETRIC);
+	if (!op) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__,
+				"Failed to allocate asymmetric crypto "
+				"operation struct");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	asym_op = op->asym;
+	/* Compute sign on the test vector */
+	asym_op->rsa.op_type = RTE_CRYPTO_ASYM_OP_SIGN;
+
+	memcpy(input_buf, &rsaplaintext.data,
+			rsaplaintext.len);
+	asym_op->rsa.message.data = input_buf;
+	asym_op->rsa.message.length = rsaplaintext.len;
+	asym_op->rsa.sign.data = output_buf;
+	asym_op->rsa.pad = RTE_CRYPTO_RSA_PKCS1_V1_5_BT1;
+
+	debug_hexdump(stdout, "message", asym_op->rsa.message.data,
+			asym_op->rsa.message.length);
+
+	/* attach asymmetric crypto session to crypto operations */
+	rte_crypto_op_attach_asym_session(op, sess);
+
+	RTE_LOG(DEBUG, USER1, "Process ASYM operation");
+
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	debug_hexdump(stdout, "signed message", asym_op->rsa.sign.data,
+			asym_op->rsa.sign.length);
+	asym_op = result_op->asym;
+
+	/* Verify sign */
+	asym_op->rsa.op_type = RTE_CRYPTO_ASYM_OP_VERIFY;
+	asym_op->rsa.pad = RTE_CRYPTO_RSA_PKCS1_V1_5_BT2;
+
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	status = TEST_SUCCESS;
+	int ret = 0;
+	ret = rsa_verify(&rsaplaintext, result_op);
+	if (ret)
+		status = TEST_FAILED;
+
+error_exit:
+
+	if (sess) {
+		rte_cryptodev_asym_session_clear(dev_id, sess);
+		rte_cryptodev_asym_session_free(sess);
+	}
+
+	if (op)
+		rte_crypto_op_free(op);
+
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	return status;
+}
+
+static int
+test_rsa_enc_dec(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct rte_mempool *op_mpool = ts_params->op_mpool;
+	struct rte_mempool *sess_mpool = ts_params->session_mpool;
+	uint8_t dev_id = ts_params->valid_devs[0];
+	struct rte_crypto_asym_op *asym_op = NULL;
+	struct rte_crypto_op *op = NULL, *result_op = NULL;
+	struct rte_cryptodev_asym_session *sess = NULL;
+	int status = TEST_SUCCESS;
+	uint8_t input_buf[TEST_DATA_SIZE] = {0};
+
+	sess = rte_cryptodev_asym_session_create(sess_mpool);
+
+	if (!sess) {
+		RTE_LOG(ERR, USER1, "line %u "
+				"FAILED: %s", __LINE__,
+				"Session creation failed");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	if (rte_cryptodev_asym_session_init(dev_id, sess, &rsa_xform,
+				sess_mpool) < 0) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "unabled to config sym session");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	/* set up crypto op data structure */
+	op = rte_crypto_op_alloc(op_mpool, RTE_CRYPTO_OP_TYPE_ASYMMETRIC);
+	if (!op) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__,
+				"Failed to allocate asymmetric crypto "
+				"operation struct");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	asym_op = op->asym;
+	/*Compute encryption on the test vector */
+	asym_op->rsa.op_type = RTE_CRYPTO_ASYM_OP_ENCRYPT;
+
+	memcpy(input_buf, rsaplaintext.data,
+			rsaplaintext.len);
+	asym_op->rsa.message.data = input_buf;
+	asym_op->rsa.message.length = rsaplaintext.len;
+	asym_op->rsa.pad = RTE_CRYPTO_RSA_PKCS1_V1_5_BT2;
+
+	debug_hexdump(stdout, "message", asym_op->rsa.message.data,
+			asym_op->rsa.message.length);
+
+	/* attach asymmetric crypto session to crypto operations */
+	rte_crypto_op_attach_asym_session(op, sess);
+
+	RTE_LOG(DEBUG, USER1, "Process ASYM operation");
+
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	debug_hexdump(stdout, "encrypted message", asym_op->rsa.message.data,
+			asym_op->rsa.message.length);
+	/* Use the resulted output as decryption Input vector*/
+	asym_op = result_op->asym;
+	asym_op->rsa.op_type = RTE_CRYPTO_ASYM_OP_DECRYPT;
+	asym_op->rsa.pad = RTE_CRYPTO_RSA_PKCS1_V1_5_BT1;
+
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	status = TEST_SUCCESS;
+	int ret = 0;
+	ret = rsa_verify(&rsaplaintext, result_op);
+	if (ret)
+		status = TEST_FAILED;
+
+error_exit:
+
+	if (sess) {
+		rte_cryptodev_asym_session_clear(dev_id, sess);
+		rte_cryptodev_asym_session_free(sess);
+	}
+
+	if (op)
+		rte_crypto_op_free(op);
+
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	return status;
+}
+
+static int
+testsuite_setup(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct rte_cryptodev_info info;
+	uint32_t i = 0, nb_devs, dev_id;
+	int ret;
+	uint16_t qp_id;
+
+	memset(ts_params, 0, sizeof(*ts_params));
+
+	ts_params->op_mpool = rte_crypto_op_pool_create(
+			"CRYPTO_ASYM_OP_POOL",
+			RTE_CRYPTO_OP_TYPE_ASYMMETRIC,
+			TEST_NUM_BUFS, 0,
+			0,
+			rte_socket_id());
+	if (ts_params->op_mpool == NULL) {
+		RTE_LOG(ERR, USER1, "Can't create ASYM_CRYPTO_OP_POOL\n");
+		return TEST_FAILED;
+	}
+
+	/* Create an OPENSSL device if required */
+	if (gbl_driver_id == rte_cryptodev_driver_id_get(
+			RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD))) {
+		nb_devs = rte_cryptodev_device_count_by_driver(
+				rte_cryptodev_driver_id_get(
+				RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)));
+		if (nb_devs < 1) {
+			ret = rte_vdev_init(
+				RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD),
+				NULL);
+
+			TEST_ASSERT(ret == 0, "Failed to create "
+				"instance of pmd : %s",
+				RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD));
+		}
+	}
+
+	nb_devs = rte_cryptodev_count();
+	if (nb_devs < 1) {
+		RTE_LOG(ERR, USER1, "No crypto devices found?\n");
+		return TEST_FAILED;
+	}
+
+	/* Create list of valid crypto devs */
+	for (i = 0; i < nb_devs; i++) {
+		rte_cryptodev_info_get(i, &info);
+		if (info.driver_id == gbl_driver_id)
+			ts_params->valid_devs[ts_params->valid_dev_count++] = i;
+	}
+
+	if (ts_params->valid_dev_count < 1)
+		return TEST_FAILED;
+
+	/* Set up all the qps on the first of the valid devices found */
+
+	dev_id = ts_params->valid_devs[0];
+
+	rte_cryptodev_info_get(dev_id, &info);
+
+	/* check if device support asymmetric, skip if not */
+	if (!(info.feature_flags &
+				RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO)) {
+		RTE_LOG(ERR, USER1, "Device doesn't support asymmetric. "
+				"Test Skipped.\n");
+		return TEST_FAILED;
+	}
+
+	/* configure device with num qp */
+	ts_params->conf.nb_queue_pairs = info.max_nb_queue_pairs;
+	ts_params->conf.socket_id = SOCKET_ID_ANY;
+	TEST_ASSERT_SUCCESS(rte_cryptodev_configure(dev_id,
+			&ts_params->conf),
+			"Failed to configure cryptodev %u with %u qps",
+			dev_id, ts_params->conf.nb_queue_pairs);
+
+	/* configure qp */
+	ts_params->qp_conf.nb_descriptors = DEFAULT_NUM_OPS_INFLIGHT;
+	for (qp_id = 0; qp_id < info.max_nb_queue_pairs; qp_id++) {
+		TEST_ASSERT_SUCCESS(rte_cryptodev_queue_pair_setup(
+			dev_id, qp_id, &ts_params->qp_conf,
+			rte_cryptodev_socket_id(dev_id),
+			ts_params->session_mpool),
+			"Failed to setup queue pair %u on cryptodev %u ASYM",
+			qp_id, dev_id);
+	}
+
+	/* setup asym session pool */
+	unsigned int session_size =
+		rte_cryptodev_asym_get_private_session_size(dev_id);
+	/*
+	 * Create mempool with TEST_NUM_SESSIONS * 2,
+	 * to include the session headers
+	 */
+	ts_params->session_mpool = rte_mempool_create(
+				"test_asym_sess_mp",
+				TEST_NUM_SESSIONS * 2,
+				session_size,
+				0, 0, NULL, NULL, NULL,
+				NULL, SOCKET_ID_ANY,
+				0);
+
+	TEST_ASSERT_NOT_NULL(ts_params->session_mpool,
+			"session mempool allocation failed");
+
+	return TEST_SUCCESS;
+}
+
+static void
+testsuite_teardown(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+
+	if (ts_params->op_mpool != NULL) {
+		RTE_LOG(DEBUG, USER1, "CRYPTO_OP_POOL count %u\n",
+		rte_mempool_avail_count(ts_params->op_mpool));
+	}
+
+	/* Free session mempools */
+	if (ts_params->session_mpool != NULL) {
+		rte_mempool_free(ts_params->session_mpool);
+		ts_params->session_mpool = NULL;
+	}
+}
+
+static int
+ut_setup(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+
+	uint16_t qp_id;
+
+	/* Reconfigure device to default parameters */
+	ts_params->conf.socket_id = SOCKET_ID_ANY;
+
+	TEST_ASSERT_SUCCESS(rte_cryptodev_configure(ts_params->valid_devs[0],
+			&ts_params->conf),
+			"Failed to configure cryptodev %u",
+			ts_params->valid_devs[0]);
+
+	for (qp_id = 0; qp_id < ts_params->conf.nb_queue_pairs ; qp_id++) {
+		TEST_ASSERT_SUCCESS(rte_cryptodev_queue_pair_setup(
+			ts_params->valid_devs[0], qp_id,
+			&ts_params->qp_conf,
+			rte_cryptodev_socket_id(ts_params->valid_devs[0]),
+			ts_params->session_mpool),
+			"Failed to setup queue pair %u on cryptodev %u",
+			qp_id, ts_params->valid_devs[0]);
+	}
+
+	rte_cryptodev_stats_reset(ts_params->valid_devs[0]);
+
+	/* Start the device */
+	TEST_ASSERT_SUCCESS(rte_cryptodev_start(ts_params->valid_devs[0]),
+						"Failed to start cryptodev %u",
+						ts_params->valid_devs[0]);
+
+	return TEST_SUCCESS;
+}
+
+static void
+ut_teardown(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct rte_cryptodev_stats stats;
+
+	rte_cryptodev_stats_get(ts_params->valid_devs[0], &stats);
+
+	/* Stop the device */
+	rte_cryptodev_stop(ts_params->valid_devs[0]);
+}
+
+static inline void print_asym_capa(
+		const struct rte_cryptodev_asymmetric_xform_capability *capa)
+{
+	int i = 0;
+
+	printf("\nxform type: %s\n===================\n",
+			rte_crypto_asym_xform_strings[capa->xform_type]);
+	printf("operation supported -");
+
+	for (i = 0; i < RTE_CRYPTO_ASYM_OP_LIST_END; i++) {
+		/* check supported operations */
+		if (rte_cryptodev_asym_xform_capability_check_optype(capa, i))
+			printf(" %s",
+					rte_crypto_asym_op_strings[i]);
+		}
+		switch (capa->xform_type) {
+		case RTE_CRYPTO_ASYM_XFORM_RSA:
+		case RTE_CRYPTO_ASYM_XFORM_MODINV:
+		case RTE_CRYPTO_ASYM_XFORM_MODEX:
+		case RTE_CRYPTO_ASYM_XFORM_DH:
+		case RTE_CRYPTO_ASYM_XFORM_DSA:
+			printf(" modlen: min %d max %d increment %d\n",
+					capa->modlen.min,
+					capa->modlen.max,
+					capa->modlen.increment);
+		break;
+		default:
+			break;
+		}
+}
+
+static int
+test_capability(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	uint8_t dev_id = ts_params->valid_devs[0];
+	struct rte_cryptodev_info dev_info;
+	const struct rte_cryptodev_capabilities *dev_capa;
+	int i = 0;
+	struct rte_cryptodev_asym_capability_idx idx;
+	const struct rte_cryptodev_asymmetric_xform_capability *capa;
+
+	rte_cryptodev_info_get(dev_id, &dev_info);
+	if (!(dev_info.feature_flags &
+				RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO)) {
+		RTE_LOG(INFO, USER1,
+				"Device doesn't support asymmetric. Test Skipped\n");
+		return TEST_SUCCESS;
+	}
+
+	/* print xform capability */
+	for (i = 0;
+		dev_info.capabilities[i].op != RTE_CRYPTO_OP_TYPE_UNDEFINED;
+		i++) {
+		dev_capa = &(dev_info.capabilities[i]);
+		if (dev_info.capabilities[i].op ==
+				RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
+			idx.type = dev_capa->asym.xform_capa.xform_type;
+
+			capa = rte_cryptodev_asym_capability_get(dev_id,
+				(const struct
+				rte_cryptodev_asym_capability_idx *) &idx);
+			print_asym_capa(capa);
+			}
+	}
+	return TEST_SUCCESS;
+}
+
+static int
+test_dh_gen_shared_sec(struct rte_crypto_asym_xform *xfrm)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct rte_mempool *op_mpool = ts_params->op_mpool;
+	struct rte_mempool *sess_mpool = ts_params->session_mpool;
+	uint8_t dev_id = ts_params->valid_devs[0];
+	struct rte_crypto_asym_op *asym_op = NULL;
+	struct rte_crypto_op *op = NULL, *result_op = NULL;
+	struct rte_cryptodev_asym_session *sess = NULL;
+	int status = TEST_SUCCESS;
+	uint8_t output[TEST_DH_MOD_LEN];
+	struct rte_crypto_asym_xform xform = *xfrm;
+	uint8_t peer[] = "01234567890123456789012345678901234567890123456789";
+
+	sess = rte_cryptodev_asym_session_create(sess_mpool);
+	if (sess == NULL) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s", __LINE__,
+				"Session creation failed");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	/* set up crypto op data structure */
+	op = rte_crypto_op_alloc(op_mpool, RTE_CRYPTO_OP_TYPE_ASYMMETRIC);
+	if (!op) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to allocate asymmetric crypto "
+			"operation struct");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	asym_op = op->asym;
+
+	/* Setup a xform and op to generate private key only */
+	xform.dh.type = RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE;
+	xform.next = NULL;
+	asym_op->dh.priv_key.data = dh_test_params.priv_key.data;
+	asym_op->dh.priv_key.length = dh_test_params.priv_key.length;
+	asym_op->dh.pub_key.data = (uint8_t *)peer;
+	asym_op->dh.pub_key.length = sizeof(peer);
+	asym_op->dh.shared_secret.data = output;
+	asym_op->dh.shared_secret.length = sizeof(output);
+
+	if (rte_cryptodev_asym_session_init(dev_id, sess, &xform,
+			sess_mpool) < 0) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "unabled to config sym session");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	/* attach asymmetric crypto session to crypto operations */
+	rte_crypto_op_attach_asym_session(op, sess);
+
+	RTE_LOG(DEBUG, USER1, "Process ASYM operation");
+
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	debug_hexdump(stdout, "shared secret:",
+			asym_op->dh.shared_secret.data,
+			asym_op->dh.shared_secret.length);
+
+error_exit:
+	if (sess != NULL) {
+		rte_cryptodev_asym_session_clear(dev_id, sess);
+		rte_cryptodev_asym_session_free(sess);
+	}
+	if (op != NULL)
+		rte_crypto_op_free(op);
+	return status;
+}
+
+static int
+test_dh_gen_priv_key(struct rte_crypto_asym_xform *xfrm)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct rte_mempool *op_mpool = ts_params->op_mpool;
+	struct rte_mempool *sess_mpool = ts_params->session_mpool;
+	uint8_t dev_id = ts_params->valid_devs[0];
+	struct rte_crypto_asym_op *asym_op = NULL;
+	struct rte_crypto_op *op = NULL, *result_op = NULL;
+	struct rte_cryptodev_asym_session *sess = NULL;
+	int status = TEST_SUCCESS;
+	uint8_t output[TEST_DH_MOD_LEN];
+	struct rte_crypto_asym_xform xform = *xfrm;
+
+	sess = rte_cryptodev_asym_session_create(sess_mpool);
+	if (sess == NULL) {
+		RTE_LOG(ERR, USER1,
+				 "line %u FAILED: %s", __LINE__,
+				"Session creation failed");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	/* set up crypto op data structure */
+	op = rte_crypto_op_alloc(op_mpool, RTE_CRYPTO_OP_TYPE_ASYMMETRIC);
+	if (!op) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to allocate asymmetric crypto "
+			"operation struct");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	asym_op = op->asym;
+
+	/* Setup a xform and op to generate private key only */
+	xform.dh.type = RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE;
+	xform.next = NULL;
+	asym_op->dh.priv_key.data = output;
+	asym_op->dh.priv_key.length = sizeof(output);
+
+	if (rte_cryptodev_asym_session_init(dev_id, sess, &xform,
+			sess_mpool) < 0) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "unabled to config sym session");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	/* attach asymmetric crypto session to crypto operations */
+	rte_crypto_op_attach_asym_session(op, sess);
+
+	RTE_LOG(DEBUG, USER1, "Process ASYM operation");
+
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	debug_hexdump(stdout, "private key:",
+			asym_op->dh.priv_key.data,
+			asym_op->dh.priv_key.length);
+
+
+error_exit:
+	if (sess != NULL) {
+		rte_cryptodev_asym_session_clear(dev_id, sess);
+		rte_cryptodev_asym_session_free(sess);
+	}
+	if (op != NULL)
+		rte_crypto_op_free(op);
+
+	return status;
+}
+
+
+static int
+test_dh_gen_pub_key(struct rte_crypto_asym_xform *xfrm)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct rte_mempool *op_mpool = ts_params->op_mpool;
+	struct rte_mempool *sess_mpool = ts_params->session_mpool;
+	uint8_t dev_id = ts_params->valid_devs[0];
+	struct rte_crypto_asym_op *asym_op = NULL;
+	struct rte_crypto_op *op = NULL, *result_op = NULL;
+	struct rte_cryptodev_asym_session *sess = NULL;
+	int status = TEST_SUCCESS;
+	uint8_t output[TEST_DH_MOD_LEN];
+	struct rte_crypto_asym_xform xform = *xfrm;
+
+	sess = rte_cryptodev_asym_session_create(sess_mpool);
+	if (sess == NULL) {
+		RTE_LOG(ERR, USER1,
+				 "line %u FAILED: %s", __LINE__,
+				"Session creation failed");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	/* set up crypto op data structure */
+	op = rte_crypto_op_alloc(op_mpool, RTE_CRYPTO_OP_TYPE_ASYMMETRIC);
+	if (!op) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to allocate asymmetric crypto "
+			"operation struct");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	asym_op = op->asym;
+	/* Setup a xform chain to generate public key
+	 * using test private key
+	 *
+	 */
+	xform.dh.type = RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE;
+	xform.next = NULL;
+
+	asym_op->dh.pub_key.data = output;
+	asym_op->dh.pub_key.length = sizeof(output);
+	/* load pre-defined private key */
+	asym_op->dh.priv_key.data = rte_malloc(NULL,
+					dh_test_params.priv_key.length,
+					0);
+	asym_op->dh.priv_key = dh_test_params.priv_key;
+
+	if (rte_cryptodev_asym_session_init(dev_id, sess, &xform,
+			sess_mpool) < 0) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "unabled to config sym session");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	/* attach asymmetric crypto session to crypto operations */
+	rte_crypto_op_attach_asym_session(op, sess);
+
+	RTE_LOG(DEBUG, USER1, "Process ASYM operation");
+
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	debug_hexdump(stdout, "pub key:",
+			asym_op->dh.pub_key.data, asym_op->dh.pub_key.length);
+
+	debug_hexdump(stdout, "priv key:",
+			asym_op->dh.priv_key.data, asym_op->dh.priv_key.length);
+
+error_exit:
+	if (sess != NULL) {
+		rte_cryptodev_asym_session_clear(dev_id, sess);
+		rte_cryptodev_asym_session_free(sess);
+	}
+	if (op != NULL)
+		rte_crypto_op_free(op);
+
+	return status;
+}
+
+static int
+test_dh_gen_kp(struct rte_crypto_asym_xform *xfrm)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct rte_mempool *op_mpool = ts_params->op_mpool;
+	struct rte_mempool *sess_mpool = ts_params->session_mpool;
+	uint8_t dev_id = ts_params->valid_devs[0];
+	struct rte_crypto_asym_op *asym_op = NULL;
+	struct rte_crypto_op *op = NULL, *result_op = NULL;
+	struct rte_cryptodev_asym_session *sess = NULL;
+	int status = TEST_SUCCESS;
+	uint8_t out_pub_key[TEST_DH_MOD_LEN];
+	uint8_t out_prv_key[TEST_DH_MOD_LEN];
+	struct rte_crypto_asym_xform pub_key_xform;
+	struct rte_crypto_asym_xform xform = *xfrm;
+
+	sess = rte_cryptodev_asym_session_create(sess_mpool);
+	if (sess == NULL) {
+		RTE_LOG(ERR, USER1,
+				 "line %u FAILED: %s", __LINE__,
+				"Session creation failed");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	/* set up crypto op data structure */
+	op = rte_crypto_op_alloc(op_mpool, RTE_CRYPTO_OP_TYPE_ASYMMETRIC);
+	if (!op) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to allocate asymmetric crypto "
+			"operation struct");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	asym_op = op->asym;
+	/* Setup a xform chain to generate
+	 * private key first followed by
+	 * public key
+	 */xform.dh.type = RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE;
+	pub_key_xform.xform_type = RTE_CRYPTO_ASYM_XFORM_DH;
+	pub_key_xform.dh.type = RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE;
+	xform.next = &pub_key_xform;
+
+	asym_op->dh.pub_key.data = out_pub_key;
+	asym_op->dh.pub_key.length = sizeof(out_pub_key);
+	asym_op->dh.priv_key.data = out_prv_key;
+	asym_op->dh.priv_key.length = sizeof(out_prv_key);
+	if (rte_cryptodev_asym_session_init(dev_id, sess, &xform,
+			sess_mpool) < 0) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "unabled to config sym session");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	/* attach asymmetric crypto session to crypto operations */
+	rte_crypto_op_attach_asym_session(op, sess);
+
+	RTE_LOG(DEBUG, USER1, "Process ASYM operation");
+
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	debug_hexdump(stdout, "priv key:",
+			out_prv_key, asym_op->dh.priv_key.length);
+	debug_hexdump(stdout, "pub key:",
+			out_pub_key, asym_op->dh.pub_key.length);
+
+error_exit:
+	if (sess != NULL) {
+		rte_cryptodev_asym_session_clear(dev_id, sess);
+		rte_cryptodev_asym_session_free(sess);
+	}
+	if (op != NULL)
+		rte_crypto_op_free(op);
+
+	return status;
+}
+
+static int
+test_mod_inv(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct rte_mempool *op_mpool = ts_params->op_mpool;
+	struct rte_mempool *sess_mpool = ts_params->session_mpool;
+	uint8_t dev_id = ts_params->valid_devs[0];
+	struct rte_crypto_asym_op *asym_op = NULL;
+	struct rte_crypto_op *op = NULL, *result_op = NULL;
+	struct rte_cryptodev_asym_session *sess = NULL;
+	int status = TEST_SUCCESS;
+	struct rte_cryptodev_asym_capability_idx cap_idx;
+	const struct rte_cryptodev_asymmetric_xform_capability *capability;
+	uint8_t input[TEST_DATA_SIZE] = {0};
+	int ret = 0;
+
+	if (rte_cryptodev_asym_get_xform_enum(
+		&modinv_xform.xform_type, "modinv") < 0) {
+		RTE_LOG(ERR, USER1,
+				 "Invalid ASYNC algorithm specified\n");
+		return -1;
+	}
+
+	cap_idx.type = modinv_xform.xform_type;
+	capability = rte_cryptodev_asym_capability_get(dev_id,
+					&cap_idx);
+
+	if (rte_cryptodev_asym_xform_capability_check_modlen(
+		capability,
+		modinv_xform.modinv.modulus.length)) {
+		RTE_LOG(ERR, USER1,
+				 "Invalid MODULOUS length specified\n");
+				return -1;
+		}
+
+	sess = rte_cryptodev_asym_session_create(sess_mpool);
+	if (!sess) {
+		RTE_LOG(ERR, USER1, "line %u "
+				"FAILED: %s", __LINE__,
+				"Session creation failed");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	if (rte_cryptodev_asym_session_init(dev_id, sess, &modinv_xform,
+			sess_mpool) < 0) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "unabled to config sym session");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	/* generate crypto op data structure */
+	op = rte_crypto_op_alloc(op_mpool, RTE_CRYPTO_OP_TYPE_ASYMMETRIC);
+	if (!op) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to allocate asymmetric crypto "
+			"operation struct");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	asym_op = op->asym;
+	memcpy(input, base, sizeof(base));
+	asym_op->modinv.base.data = input;
+	asym_op->modinv.base.length = sizeof(base);
+
+	/* attach asymmetric crypto session to crypto operations */
+	rte_crypto_op_attach_asym_session(op, sess);
+
+	RTE_LOG(DEBUG, USER1, "Process ASYM operation");
+
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	ret = verify_modinv(mod_inv, result_op);
+	if (ret) {
+		RTE_LOG(ERR, USER1,
+			 "operation verification failed\n");
+		status = TEST_FAILED;
+	}
+
+error_exit:
+	if (sess) {
+		rte_cryptodev_asym_session_clear(dev_id, sess);
+		rte_cryptodev_asym_session_free(sess);
+	}
+
+	if (op)
+		rte_crypto_op_free(op);
+
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	return status;
+}
+
+static int
+test_mod_exp(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct rte_mempool *op_mpool = ts_params->op_mpool;
+	struct rte_mempool *sess_mpool = ts_params->session_mpool;
+	uint8_t dev_id = ts_params->valid_devs[0];
+	struct rte_crypto_asym_op *asym_op = NULL;
+	struct rte_crypto_op *op = NULL, *result_op = NULL;
+	struct rte_cryptodev_asym_session *sess = NULL;
+	int status = TEST_SUCCESS;
+	struct rte_cryptodev_asym_capability_idx cap_idx;
+	const struct rte_cryptodev_asymmetric_xform_capability *capability;
+	uint8_t input[TEST_DATA_SIZE] = {0};
+	int ret = 0;
+
+	if (rte_cryptodev_asym_get_xform_enum(&modex_xform.xform_type,
+		"modexp")
+		< 0) {
+		RTE_LOG(ERR, USER1,
+				"Invalid ASYNC algorithm specified\n");
+		return -1;
+	}
+
+	/* check for modlen capability */
+	cap_idx.type = modex_xform.xform_type;
+	capability = rte_cryptodev_asym_capability_get(dev_id, &cap_idx);
+
+	if (rte_cryptodev_asym_xform_capability_check_modlen(
+			capability, modex_xform.modex.modulus.length)) {
+		RTE_LOG(ERR, USER1,
+				"Invalid MODULOUS length specified\n");
+				return -1;
+		}
+
+	/* generate crypto op data structure */
+	op = rte_crypto_op_alloc(op_mpool, RTE_CRYPTO_OP_TYPE_ASYMMETRIC);
+	if (!op) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to allocate asymmetric crypto "
+			"operation struct");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	sess = rte_cryptodev_asym_session_create(sess_mpool);
+	if (!sess) {
+		RTE_LOG(ERR, USER1,
+				 "line %u "
+				"FAILED: %s", __LINE__,
+				"Session creation failed");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	if (rte_cryptodev_asym_session_init(dev_id, sess, &modex_xform,
+			sess_mpool) < 0) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "unabled to config sym session");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	asym_op = op->asym;
+	memcpy(input, base, sizeof(base));
+	asym_op->modex.base.data = input;
+	asym_op->modex.base.length = sizeof(base);
+	/* attach asymmetric crypto session to crypto operations */
+	rte_crypto_op_attach_asym_session(op, sess);
+
+	RTE_LOG(DEBUG, USER1, "Process ASYM operation");
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	ret = verify_modexp(mod_exp, result_op);
+	if (ret) {
+		RTE_LOG(ERR, USER1,
+			 "operation verification failed\n");
+		status = TEST_FAILED;
+	}
+
+error_exit:
+	if (sess != NULL) {
+		rte_cryptodev_asym_session_clear(dev_id, sess);
+		rte_cryptodev_asym_session_free(sess);
+	}
+
+	if (op != NULL)
+		rte_crypto_op_free(op);
+
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	return status;
+}
+
+static int
+test_dh_keygenration(void)
+{
+	int status;
+
+	debug_hexdump(stdout, "p:", dh_xform.dh.p.data, dh_xform.dh.p.length);
+	debug_hexdump(stdout, "g:", dh_xform.dh.g.data, dh_xform.dh.g.length);
+	debug_hexdump(stdout, "priv_key:", dh_test_params.priv_key.data,
+			dh_test_params.priv_key.length);
+
+	RTE_LOG(INFO, USER1,
+		"Test Public and Private key pair generation\n");
+
+	status = test_dh_gen_kp(&dh_xform);
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	RTE_LOG(INFO, USER1,
+		"Test Public Key Generation using pre-defined priv key\n");
+
+	status = test_dh_gen_pub_key(&dh_xform);
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	RTE_LOG(INFO, USER1,
+		"Test Private Key Generation only\n");
+
+	status = test_dh_gen_priv_key(&dh_xform);
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	RTE_LOG(INFO, USER1,
+		"Test shared secret compute\n");
+
+	status = test_dh_gen_shared_sec(&dh_xform);
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+
+	return status;
+}
+
+static int
+test_dsa_sign(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct rte_mempool *op_mpool = ts_params->op_mpool;
+	struct rte_mempool *sess_mpool = ts_params->session_mpool;
+	uint8_t dev_id = ts_params->valid_devs[0];
+	struct rte_crypto_asym_op *asym_op = NULL;
+	struct rte_crypto_op *op = NULL, *result_op = NULL;
+	struct rte_cryptodev_asym_session *sess = NULL;
+	int status = TEST_SUCCESS;
+	uint8_t r[TEST_DH_MOD_LEN];
+	uint8_t s[TEST_DH_MOD_LEN];
+	uint8_t dgst[] = "35d81554afaad2cf18f3a1770d5fedc4ea5be344";
+
+	sess = rte_cryptodev_asym_session_create(sess_mpool);
+	if (sess == NULL) {
+		RTE_LOG(ERR, USER1,
+				 "line %u FAILED: %s", __LINE__,
+				"Session creation failed");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	/* set up crypto op data structure */
+	op = rte_crypto_op_alloc(op_mpool, RTE_CRYPTO_OP_TYPE_ASYMMETRIC);
+	if (!op) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to allocate asymmetric crypto "
+			"operation struct");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+	asym_op = op->asym;
+
+	debug_hexdump(stdout, "p: ", dsa_xform.dsa.p.data,
+			dsa_xform.dsa.p.length);
+	debug_hexdump(stdout, "q: ", dsa_xform.dsa.q.data,
+			dsa_xform.dsa.q.length);
+	debug_hexdump(stdout, "g: ", dsa_xform.dsa.g.data,
+			dsa_xform.dsa.g.length);
+	debug_hexdump(stdout, "priv_key: ", dsa_xform.dsa.x.data,
+			dsa_xform.dsa.x.length);
+
+	if (rte_cryptodev_asym_session_init(dev_id, sess, &dsa_xform,
+				sess_mpool) < 0) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "unabled to config sym session");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	/* attach asymmetric crypto session to crypto operations */
+	rte_crypto_op_attach_asym_session(op, sess);
+	asym_op->dsa.op_type = RTE_CRYPTO_ASYM_OP_SIGN;
+	asym_op->dsa.message.data = dgst;
+	asym_op->dsa.message.length = sizeof(dgst);
+	asym_op->dsa.r.length = sizeof(r);
+	asym_op->dsa.r.data = r;
+	asym_op->dsa.s.length = sizeof(s);
+	asym_op->dsa.s.data = s;
+
+	RTE_LOG(DEBUG, USER1, "Process ASYM operation");
+
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	asym_op = result_op->asym;
+
+	debug_hexdump(stdout, "r:",
+			asym_op->dsa.r.data, asym_op->dsa.r.length);
+	debug_hexdump(stdout, "s:",
+			asym_op->dsa.s.data, asym_op->dsa.s.length);
+
+	/* Test PMD DSA sign verification using signer public key */
+	asym_op->dsa.op_type = RTE_CRYPTO_ASYM_OP_VERIFY;
+
+	/* copy signer public key */
+	asym_op->dsa.y.data = dsa_test_params.y.data;
+	asym_op->dsa.y.length = dsa_test_params.y.length;
+
+	/* Process crypto operation */
+	if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Error sending packet for operation");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	while (rte_cryptodev_dequeue_burst(dev_id, 0, &result_op, 1) == 0)
+		rte_pause();
+
+	if (result_op == NULL) {
+		RTE_LOG(ERR, USER1,
+			"line %u FAILED: %s",
+			__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+		goto error_exit;
+	}
+
+	if (result_op->status != RTE_CRYPTO_OP_STATUS_SUCCESS) {
+		RTE_LOG(ERR, USER1,
+				"line %u FAILED: %s",
+				__LINE__, "Failed to process asym crypto op");
+		status = TEST_FAILED;
+	}
+error_exit:
+	if (sess != NULL) {
+		rte_cryptodev_asym_session_clear(dev_id, sess);
+		rte_cryptodev_asym_session_free(sess);
+	}
+	if (op != NULL)
+		rte_crypto_op_free(op);
+	return status;
+}
+
+static int
+test_dsa(void)
+{
+	int status;
+	status = test_dsa_sign();
+	TEST_ASSERT_EQUAL(status, 0, "Test failed");
+	return status;
+}
+
+
+static struct unit_test_suite cryptodev_openssl_asym_testsuite  = {
+	.suite_name = "Crypto Device OPENSSL ASYM Unit Test Suite",
+	.setup = testsuite_setup,
+	.teardown = testsuite_teardown,
+	.unit_test_cases = {
+		TEST_CASE_ST(ut_setup, ut_teardown, test_capability),
+		TEST_CASE_ST(ut_setup, ut_teardown, test_dsa),
+		TEST_CASE_ST(ut_setup, ut_teardown, test_dh_keygenration),
+		TEST_CASE_ST(ut_setup, ut_teardown, test_rsa_enc_dec),
+		TEST_CASE_ST(ut_setup, ut_teardown, test_rsa_sign_verify),
+		TEST_CASE_ST(ut_setup, ut_teardown, test_mod_inv),
+		TEST_CASE_ST(ut_setup, ut_teardown, test_mod_exp),
+		TEST_CASES_END() /**< NULL terminate unit test array */
+	}
+};
+
+static int
+test_cryptodev_openssl_asym(void)
+{
+	gbl_driver_id = rte_cryptodev_driver_id_get(
+			RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD));
+
+	if (gbl_driver_id == -1) {
+		RTE_LOG(ERR, USER1, "OPENSSL PMD must be loaded. Check if "
+				"CONFIG_RTE_LIBRTE_PMD_OPENSSL is enabled "
+				"in config file to run this testsuite.\n");
+		return TEST_FAILED;
+	}
+
+	return unit_test_suite_runner(&cryptodev_openssl_asym_testsuite);
+}
+
+REGISTER_TEST_COMMAND(cryptodev_openssl_asym_autotest,
+					  test_cryptodev_openssl_asym);
diff --git a/test/test/test_cryptodev_asym_util.h b/test/test/test_cryptodev_asym_util.h
new file mode 100644
index 00000000..dff0c2ad
--- /dev/null
+++ b/test/test/test_cryptodev_asym_util.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium Networks
+ */
+
+#ifndef TEST_CRYPTODEV_ASYM_TEST_UTIL_H__
+#define TEST_CRYPTODEV_ASYM_TEST_UTIL_H__
+
+/* Below Apis compare resulted buffer to original test vector */
+
+static inline int rsa_verify(struct rsa_test_data *rsa_param,
+		struct rte_crypto_op *result_op)
+{
+	if (memcmp(rsa_param->data,
+				result_op->asym->rsa.message.data,
+				result_op->asym->rsa.message.length))
+		return -1;
+	return 0;
+}
+
+static inline int verify_modinv(uint8_t *mod_inv,
+		struct rte_crypto_op *result_op)
+{
+	if (memcmp(mod_inv, result_op->asym->modinv.base.data,
+				result_op->asym->modinv.base.length))
+		return -1;
+	return 0;
+}
+
+static inline int verify_modexp(uint8_t *mod_exp,
+		struct rte_crypto_op *result_op)
+{
+	if (memcmp(mod_exp, result_op->asym->modex.base.data,
+				result_op->asym->modex.base.length))
+		return -1;
+	return 0;
+}
+
+#endif /* TEST_CRYPTODEV_ASYM_TEST_UTIL_H__ */
+
+
+
+
diff --git a/test/test/test_cryptodev_blockcipher.c b/test/test/test_cryptodev_blockcipher.c
index 256a7daa..f2701f8f 100644
--- a/test/test/test_cryptodev_blockcipher.c
+++ b/test/test/test_cryptodev_blockcipher.c
@@ -75,14 +75,28 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 
 	int nb_segs = 1;
 
+	rte_cryptodev_info_get(dev_id, &dev_info);
+
 	if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SG) {
-		rte_cryptodev_info_get(dev_id, &dev_info);
-		if (!(dev_info.feature_flags &
-				RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER)) {
-			printf("Device doesn't support scatter-gather. "
+		uint64_t feat_flags = dev_info.feature_flags;
+		uint64_t oop_flag = RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT;
+
+		if (t->feature_mask && BLOCKCIPHER_TEST_FEATURE_OOP) {
+			if (!(feat_flags & oop_flag)) {
+				printf("Device doesn't support out-of-place "
+					"scatter-gather in input mbuf. "
+					"Test Skipped.\n");
+				return 0;
+			}
+		} else {
+			if (!(feat_flags & RTE_CRYPTODEV_FF_IN_PLACE_SGL)) {
+				printf("Device doesn't support in-place "
+					"scatter-gather mbufs. "
 					"Test Skipped.\n");
-			return 0;
+				return 0;
+			}
 		}
+
 		nb_segs = 3;
 	}
 
@@ -438,11 +452,34 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 		uint8_t value;
 		uint32_t head_unchanged_len, changed_len = 0;
 		uint32_t i;
+		uint32_t hdroom_used = 0, tlroom_used = 0;
+		uint32_t hdroom = 0;
 
 		mbuf = sym_op->m_src;
+		/*
+		 * Crypto PMDs specify the headroom & tailroom it would use
+		 * when processing the crypto operation. PMD is free to modify
+		 * this space, and so the verification check should skip that
+		 * block.
+		 */
+		hdroom_used = dev_info.min_mbuf_headroom_req;
+		tlroom_used = dev_info.min_mbuf_tailroom_req;
+
+		/* Get headroom */
+		hdroom = rte_pktmbuf_headroom(mbuf);
+
 		head_unchanged_len = mbuf->buf_len;
 
 		for (i = 0; i < mbuf->buf_len; i++) {
+
+			/* Skip headroom used by PMD */
+			if (i == hdroom - hdroom_used)
+				i += hdroom_used;
+
+			/* Skip tailroom used by PMD */
+			if (i == (hdroom + mbuf->data_len))
+				i += tlroom_used;
+
 			value = *((uint8_t *)(mbuf->buf_addr)+i);
 			if (value != tmp_src_buf[i]) {
 				snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
@@ -455,14 +492,13 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 
 		mbuf = sym_op->m_dst;
 		if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH) {
-			head_unchanged_len = rte_pktmbuf_headroom(mbuf) +
-						sym_op->auth.data.offset;
+			head_unchanged_len = hdroom + sym_op->auth.data.offset;
 			changed_len = sym_op->auth.data.length;
 			if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_GEN)
 				changed_len += digest_len;
 		} else {
 			/* cipher-only */
-			head_unchanged_len = rte_pktmbuf_headroom(mbuf) +
+			head_unchanged_len = hdroom +
 					sym_op->cipher.data.offset;
 			changed_len = sym_op->cipher.data.length;
 		}
@@ -486,15 +522,30 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 		uint8_t value;
 		uint32_t head_unchanged_len = 0, changed_len = 0;
 		uint32_t i;
+		uint32_t hdroom_used = 0, tlroom_used = 0;
+		uint32_t hdroom = 0;
+
+		/*
+		 * Crypto PMDs specify the headroom & tailroom it would use
+		 * when processing the crypto operation. PMD is free to modify
+		 * this space, and so the verification check should skip that
+		 * block.
+		 */
+		hdroom_used = dev_info.min_mbuf_headroom_req;
+		tlroom_used = dev_info.min_mbuf_tailroom_req;
 
 		mbuf = sym_op->m_src;
+
+		/* Get headroom */
+		hdroom = rte_pktmbuf_headroom(mbuf);
+
 		if (t->op_mask & BLOCKCIPHER_TEST_OP_CIPHER) {
-			head_unchanged_len = rte_pktmbuf_headroom(mbuf) +
+			head_unchanged_len = hdroom +
 					sym_op->cipher.data.offset;
 			changed_len = sym_op->cipher.data.length;
 		} else {
 			/* auth-only */
-			head_unchanged_len = rte_pktmbuf_headroom(mbuf) +
+			head_unchanged_len = hdroom +
 					sym_op->auth.data.offset +
 					sym_op->auth.data.length;
 			changed_len = 0;
@@ -504,8 +555,18 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 			changed_len += digest_len;
 
 		for (i = 0; i < mbuf->buf_len; i++) {
+
+			/* Skip headroom used by PMD */
+			if (i == hdroom - hdroom_used)
+				i += hdroom_used;
+
 			if (i == head_unchanged_len)
 				i += changed_len;
+
+			/* Skip tailroom used by PMD */
+			if (i == (hdroom + mbuf->data_len))
+				i += tlroom_used;
+
 			value = *((uint8_t *)(mbuf->buf_addr)+i);
 			if (value != tmp_src_buf[i]) {
 				snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
diff --git a/test/test/test_cryptodev_des_test_vectors.h b/test/test/test_cryptodev_des_test_vectors.h
index 43be83d6..10334560 100644
--- a/test/test/test_cryptodev_des_test_vectors.h
+++ b/test/test/test_cryptodev_des_test_vectors.h
@@ -792,6 +792,30 @@ triple_des192cbc_hmac_sha1_test_vector = {
 		.len = 20
 	}
 };
+static const struct blockcipher_test_data
+triple_des64cbc_test_vector = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_3DES_CBC,
+	.cipher_key = {
+		.data = {
+			0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2
+		},
+		.len = 8
+	},
+	.iv = {
+		.data = {
+			0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+		},
+		.len = 8
+	},
+	.plaintext = {
+		.data = plaintext_des,
+		.len = 512
+	},
+	.ciphertext = {
+		.data = ciphertext512_des,
+		.len = 512
+	},
+};
 
 static const struct blockcipher_test_data
 des_cbc_test_vector = {
@@ -1205,6 +1229,22 @@ static const struct blockcipher_test_case triple_des_chain_test_cases[] = {
 
 static const struct blockcipher_test_case triple_des_cipheronly_test_cases[] = {
 	{
+		.test_descr = "3DES-64-CBC Encryption",
+		.test_data = &triple_des64cbc_test_vector,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB |
+			BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL |
+			BLOCKCIPHER_TEST_TARGET_PMD_QAT
+	},
+	{
+		.test_descr = "3DES-64-CBC Decryption",
+		.test_data = &triple_des64cbc_test_vector,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB |
+			BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL |
+			BLOCKCIPHER_TEST_TARGET_PMD_QAT
+	},
+	{
 		.test_descr = "3DES-128-CBC Encryption",
 		.test_data = &triple_des128cbc_test_vector,
 		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
@@ -1212,7 +1252,8 @@ static const struct blockcipher_test_case triple_des_cipheronly_test_cases[] = {
 			BLOCKCIPHER_TEST_TARGET_PMD_QAT |
 			BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC |
 			BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC |
-			BLOCKCIPHER_TEST_TARGET_PMD_CCP
+			BLOCKCIPHER_TEST_TARGET_PMD_CCP |
+			BLOCKCIPHER_TEST_TARGET_PMD_MB
 	},
 	{
 		.test_descr = "3DES-128-CBC Decryption",
@@ -1222,7 +1263,8 @@ static const struct blockcipher_test_case triple_des_cipheronly_test_cases[] = {
 			BLOCKCIPHER_TEST_TARGET_PMD_QAT |
 			BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC |
 			BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC |
-			BLOCKCIPHER_TEST_TARGET_PMD_CCP
+			BLOCKCIPHER_TEST_TARGET_PMD_CCP |
+			BLOCKCIPHER_TEST_TARGET_PMD_MB
 	},
 	{
 		.test_descr = "3DES-192-CBC Encryption",
@@ -1233,7 +1275,8 @@ static const struct blockcipher_test_case triple_des_cipheronly_test_cases[] = {
 			BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC |
 			BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC |
 			BLOCKCIPHER_TEST_TARGET_PMD_MVSAM |
-			BLOCKCIPHER_TEST_TARGET_PMD_CCP
+			BLOCKCIPHER_TEST_TARGET_PMD_CCP |
+			BLOCKCIPHER_TEST_TARGET_PMD_MB
 	},
 	{
 		.test_descr = "3DES-192-CBC Decryption",
@@ -1244,7 +1287,8 @@ static const struct blockcipher_test_case triple_des_cipheronly_test_cases[] = {
 			BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC |
 			BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC |
 			BLOCKCIPHER_TEST_TARGET_PMD_MVSAM |
-			BLOCKCIPHER_TEST_TARGET_PMD_CCP
+			BLOCKCIPHER_TEST_TARGET_PMD_CCP |
+			BLOCKCIPHER_TEST_TARGET_PMD_MB
 	},
 	{
 		.test_descr = "3DES-128-CTR Encryption",
diff --git a/test/test/test_cryptodev_dh_test_vectors.h b/test/test/test_cryptodev_dh_test_vectors.h
new file mode 100644
index 00000000..fe7510dc
--- /dev/null
+++ b/test/test/test_cryptodev_dh_test_vectors.h
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium Networks
+ */
+
+#ifndef TEST_CRYPTODEV_DH_TEST_VECTORS_H_
+#define TEST_CRYPTODEV_DH_TEST_VECTORS_H_
+
+#include "rte_crypto_asym.h"
+
+#define TEST_DATA_SIZE 4096
+#define TEST_DH_MOD_LEN 1024
+
+
+struct dh_test_param {
+	rte_crypto_param priv_key;
+};
+
+uint8_t dh_priv[] = {
+	0x46, 0x3c, 0x7b, 0x43, 0xd1, 0xb8, 0xd4, 0x7a,
+	0x56, 0x28, 0x85, 0x79, 0xcc, 0xd8, 0x90, 0x03,
+	0x0c, 0x4b, 0xc6, 0xd3, 0x7f, 0xb3, 0x19, 0x84,
+	0x8a, 0xc6, 0x0d, 0x24, 0x5e, 0xaa, 0x7e, 0x7a,
+	0x73, 0x88, 0xa6, 0x47, 0x7c, 0x42, 0x78, 0x63,
+	0x11, 0x12, 0xd3, 0xa0, 0xc5, 0xfe, 0xfd, 0xf2,
+	0x9e, 0x17, 0x90, 0xe5, 0x6d, 0xcc, 0x20, 0x6f,
+	0xe8, 0x82, 0x28, 0xbf, 0x5c, 0xe6, 0xd4, 0x86,
+	0x5c, 0x35, 0x32, 0x97, 0xc2, 0x86, 0x1b, 0xc5,
+	0x59, 0x1c, 0x0b, 0x1b, 0xec, 0x60, 0x3c, 0x1d,
+	0x8d, 0x7f, 0xf0, 0xc7, 0x48, 0x3a, 0x51, 0x09,
+	0xf2, 0x3e, 0x9e, 0x35, 0x74, 0x98, 0x4d, 0xad,
+	0x39, 0xa7, 0xf2, 0xd2, 0xb4, 0x32, 0xd3, 0xc8,
+	0xe9, 0x45, 0xbe, 0x56, 0xe7, 0x87, 0xe0, 0xa0,
+	0x97, 0x6b, 0x5f, 0x99, 0x5e, 0x41, 0x59, 0x33,
+	0x95, 0x64, 0x0d, 0xe9, 0x58, 0x5b, 0xa6, 0x38
+};
+
+uint8_t dh_p[] = {
+	0xef, 0xee, 0x8c, 0x8b, 0x3f, 0x85, 0x95, 0xcd,
+	0x4d, 0x68, 0x5d, 0x4a, 0x5d, 0x1f, 0x2a, 0x2e,
+	0xdd, 0xcf, 0xef, 0x1b, 0x3b, 0xe9, 0x7b, 0x0c,
+	0x13, 0xee, 0x76, 0xd5, 0x93, 0xca, 0x8b, 0xc8,
+	0x0b, 0x97, 0x00, 0xec, 0x1f, 0x34, 0xa2, 0xce,
+	0x83, 0x8d, 0x80, 0xea, 0xfe, 0x11, 0xed, 0x28,
+	0xdd, 0x32, 0x22, 0x77, 0x96, 0x4e, 0xc5, 0xed,
+	0xc8, 0x7a, 0x52, 0x10, 0x22, 0xcc, 0xb2, 0x4d,
+	0xd3, 0xda, 0x03, 0xf5, 0x1e, 0xa8, 0x79, 0x23,
+	0x8b, 0xe1, 0x78, 0x47, 0x07, 0x5b, 0x26, 0xbb,
+	0x53, 0x46, 0x0b, 0x18, 0x5c, 0x07, 0x4e, 0xb6,
+	0x76, 0xc9, 0xa8, 0xd5, 0x30, 0xa3, 0xbe, 0x8d,
+	0xae, 0xcd, 0x34, 0x68, 0x62, 0x5f, 0xb9, 0x5c,
+	0x34, 0x90, 0xf0, 0xda, 0x47, 0x86, 0x36, 0x04,
+	0x28, 0xbc, 0x7d, 0xae, 0x9d, 0x4e, 0x61, 0x28,
+	0x70, 0xdb, 0xa6, 0x55, 0x04, 0x46, 0x27, 0xe3
+};
+
+uint8_t dh_g[] = {0x02};
+
+struct dh_test_param dh_test_params = {
+	.priv_key = {
+		.data = dh_priv,
+		.length = sizeof(dh_priv)
+	}
+};
+
+struct rte_crypto_asym_xform dh_xform = {
+	.next = NULL,
+	.xform_type = RTE_CRYPTO_ASYM_XFORM_DH,
+	.dh = {
+		.p = {
+			.data = dh_p,
+			.length = sizeof(dh_p)
+		},
+		.g = {
+			.data = dh_g,
+			.length = sizeof(dh_g)
+		},
+	}
+};
+
+#endif /* TEST_CRYPTODEV_DH_TEST_VECTORS_H__ */
diff --git a/test/test/test_cryptodev_dsa_test_vectors.h b/test/test/test_cryptodev_dsa_test_vectors.h
new file mode 100644
index 00000000..bbcb0d72
--- /dev/null
+++ b/test/test/test_cryptodev_dsa_test_vectors.h
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium Networks
+ */
+
+#ifndef TEST_CRYPTODEV_DSA_TEST_VECTORS_H_
+#define TEST_CRYPTODEV_DSA_TEST_VECTORS_H_
+
+#include "rte_crypto_asym.h"
+
+#define TEST_DATA_SIZE 4096
+
+
+struct dsa_test_param {
+	rte_crypto_param y;
+};
+
+static unsigned char dsa_x[] = {
+	0xc5, 0x3e, 0xae, 0x6d, 0x45, 0x32, 0x31, 0x64,
+	0xc7, 0xd0, 0x7a, 0xf5, 0x71, 0x57, 0x03, 0x74,
+	0x4a, 0x63, 0xfc, 0x3a
+};
+
+uint8_t dsa_y[] = {
+	0x31, 0x3f, 0xd9, 0xeb, 0xca, 0x91, 0x57, 0x4e,
+	0x1c, 0x2e, 0xeb, 0xe1, 0x51, 0x7c, 0x57, 0xe0,
+	0xc2, 0x1b, 0x02, 0x09, 0x87, 0x21, 0x40, 0xc5,
+	0x32, 0x87, 0x61, 0xbb, 0xb2, 0x45, 0x0b, 0x33,
+	0xf1, 0xb1, 0x8b, 0x40, 0x9c, 0xe9, 0xab, 0x7c,
+	0x4c, 0xd8, 0xfd, 0xa3, 0x39, 0x1e, 0x8e, 0x34,
+	0x86, 0x83, 0x57, 0xc1, 0x99, 0xe1, 0x6a, 0x6b,
+	0x2e, 0xba, 0x06, 0xd6, 0x74, 0x9d, 0xef, 0x79,
+	0x1d, 0x79, 0xe9, 0x5d, 0x3a, 0x4d, 0x09, 0xb2,
+	0x4c, 0x39, 0x2a, 0xd8, 0x9d, 0xbf, 0x10, 0x09,
+	0x95, 0xae, 0x19, 0xc0, 0x10, 0x62, 0x05, 0x6b,
+	0xb1, 0x4b, 0xce, 0x00, 0x5e, 0x87, 0x31, 0xef,
+	0xde, 0x17, 0x5f, 0x95, 0xb9, 0x75, 0x08, 0x9b,
+	0xdc, 0xda, 0xea, 0x56, 0x2b, 0x32, 0x78, 0x6d,
+	0x96, 0xf5, 0xa3, 0x1a, 0xed, 0xf7, 0x53, 0x64,
+	0x00, 0x8a, 0xd4, 0xff, 0xfe, 0xbb, 0x97, 0x0b
+};
+
+static unsigned char dsa_p[] = {
+	0xa8,  0xf9,  0xcd,  0x20,  0x1e,  0x5e,  0x35,  0xd8,
+	0x92,  0xf8,  0x5f,  0x80,  0xe4,  0xdb,  0x25,  0x99,
+	0xa5,  0x67,  0x6a,  0x3b,  0x1d,  0x4f,  0x19,  0x03,
+	0x30,  0xed,  0x32,  0x56,  0xb2,  0x6d,  0x0e,  0x80,
+	0xa0,  0xe4,  0x9a,  0x8f,  0xff,  0xaa,  0xad,  0x2a,
+	0x24,  0xf4,  0x72,  0xd2,  0x57,  0x32,  0x41,  0xd4,
+	0xd6,  0xd6,  0xc7,  0x48,  0x0c,  0x80,  0xb4,  0xc6,
+	0x7b,  0xb4,  0x47,  0x9c,  0x15,  0xad,  0xa7,  0xea,
+	0x84,  0x24,  0xd2,  0x50,  0x2f,  0xa0,  0x14,  0x72,
+	0xe7,  0x60,  0x24,  0x17,  0x13,  0xda,  0xb0,  0x25,
+	0xae,  0x1b,  0x02,  0xe1,  0x70,  0x3a,  0x14,  0x35,
+	0xf6,  0x2d,  0xdf,  0x4e,  0xe4,  0xc1,  0xb6,  0x64,
+	0x06,  0x6e,  0xb2,  0x2f,  0x2e,  0x3b,  0xf2,  0x8b,
+	0xb7,  0x0a,  0x2a,  0x76,  0xe4,  0xfd,  0x5e,  0xbe,
+	0x2d,  0x12,  0x29,  0x68,  0x1b,  0x5b,  0x06,  0x43,
+	0x9a,  0xc9,  0xc7,  0xe9,  0xd8,  0xbd,  0xe2,  0x83
+};
+
+static unsigned char dsa_q[] = {
+	0xf8,  0x5f,  0x0f,  0x83,  0xac,  0x4d,  0xf7,  0xea,
+	0x0c,  0xdf,  0x8f,  0x46,  0x9b,  0xfe,  0xea,  0xea,
+	0x14,  0x15,  0x64,  0x95
+};
+
+static unsigned char dsa_g[] = {
+	0x2b,  0x31,  0x52,  0xff,  0x6c,  0x62,  0xf1,  0x46,
+	0x22,  0xb8,  0xf4,  0x8e,  0x59,  0xf8,  0xaf,  0x46,
+	0x88,  0x3b,  0x38,  0xe7,  0x9b,  0x8c,  0x74,  0xde,
+	0xea,  0xe9,  0xdf,  0x13,  0x1f,  0x8b,  0x85,  0x6e,
+	0x3a,  0xd6,  0xc8,  0x45,  0x5d,  0xab,  0x87,  0xcc,
+	0x0d,  0xa8,  0xac,  0x97,  0x34,  0x17,  0xce,  0x4f,
+	0x78,  0x78,  0x55,  0x7d,  0x6c,  0xdf,  0x40,  0xb3,
+	0x5b,  0x4a,  0x0c,  0xa3,  0xeb,  0x31,  0x0c,  0x6a,
+	0x95,  0xd6,  0x8c,  0xe2,  0x84,  0xad,  0x4e,  0x25,
+	0xea,  0x28,  0x59,  0x16,  0x11,  0xee,  0x08,  0xb8,
+	0x44,  0x4b,  0xd6,  0x4b,  0x25,  0xf3,  0xf7,  0xc5,
+	0x72,  0x41,  0x0d,  0xdf,  0xb3,  0x9c,  0xc7,  0x28,
+	0xb9,  0xc9,  0x36,  0xf8,  0x5f,  0x41,  0x91,  0x29,
+	0x86,  0x99,  0x29,  0xcd,  0xb9,  0x09,  0xa6,  0xa3,
+	0xa9,  0x9b,  0xbe,  0x08,  0x92,  0x16,  0x36,  0x81,
+	0x71,  0xbd,  0x0b,  0xa8,  0x1d,  0xe4,  0xfe,  0x33
+};
+
+struct dsa_test_param dsa_test_params = {
+	.y = {
+		.data = dsa_y,
+		.length = sizeof(dsa_y)
+	}
+};
+
+struct rte_crypto_asym_xform dsa_xform = {
+	.next = NULL,
+	.xform_type = RTE_CRYPTO_ASYM_XFORM_DSA,
+	.dsa = {
+		.p = {
+			.data = dsa_p,
+			.length = sizeof(dsa_p)
+		},
+		.q = {
+			.data = dsa_q,
+			.length = sizeof(dsa_q)
+		},
+		.g = {
+			.data = dsa_g,
+			.length = sizeof(dsa_g)
+		},
+		.x = {
+			.data = dsa_x,
+			.length = sizeof(dsa_x)
+		}
+
+	}
+};
+
+#endif /* TEST_CRYPTODEV_DSA_TEST_VECTORS_H__ */
diff --git a/test/test/test_cryptodev_mod_test_vectors.h b/test/test/test_cryptodev_mod_test_vectors.h
new file mode 100644
index 00000000..a25c676a
--- /dev/null
+++ b/test/test/test_cryptodev_mod_test_vectors.h
@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium Networks
+ */
+
+#ifndef TEST_CRYPTODEV_MOD_TEST_VECTORS_H_
+#define TEST_CRYPTODEV_MOD_TEST_VECTORS_H_
+
+/* modular operation test data */
+uint8_t base[] = {
+	0xF8, 0xBA, 0x1A, 0x55, 0xD0, 0x2F, 0x85,
+	0xAE, 0x96, 0x7B, 0xB6, 0x2F, 0xB6, 0xCD,
+	0xA8, 0xEB, 0x7E, 0x78, 0xA0, 0x50
+};
+
+uint8_t mod_p[] = {
+	0x00, 0xb3, 0xa1, 0xaf, 0xb7, 0x13, 0x08, 0x00,
+	0x0a, 0x35, 0xdc, 0x2b, 0x20, 0x8d, 0xa1, 0xb5,
+	0xce, 0x47, 0x8a, 0xc3, 0x80, 0xf4, 0x7d, 0x4a,
+	0xa2, 0x62, 0xfd, 0x61, 0x7f, 0xb5, 0xa8, 0xde,
+	0x0a, 0x17, 0x97, 0xa0, 0xbf, 0xdf, 0x56, 0x5a,
+	0x3d, 0x51, 0x56, 0x4f, 0x70, 0x70, 0x3f, 0x63,
+	0x6a, 0x44, 0x5b, 0xad, 0x84, 0x0d, 0x3f, 0x27,
+	0x6e, 0x3b, 0x34, 0x91, 0x60, 0x14, 0xb9, 0xaa,
+	0x72, 0xfd, 0xa3, 0x64, 0xd2, 0x03, 0xa7, 0x53,
+	0x87, 0x9e, 0x88, 0x0b, 0xc1, 0x14, 0x93, 0x1a,
+	0x62, 0xff, 0xb1, 0x5d, 0x74, 0xcd, 0x59, 0x63,
+	0x18, 0x11, 0x3d, 0x4f, 0xba, 0x75, 0xd4, 0x33,
+	0x4e, 0x23, 0x6b, 0x7b, 0x57, 0x44, 0xe1, 0xd3,
+	0x03, 0x13, 0xa6, 0xf0, 0x8b, 0x60, 0xb0, 0x9e,
+	0xee, 0x75, 0x08, 0x9d, 0x71, 0x63, 0x13, 0xcb,
+	0xa6, 0x81, 0x92, 0x14, 0x03, 0x22, 0x2d, 0xde,
+	0x55
+};
+
+uint8_t mod_e[] = {0x01, 0x00, 0x01};
+
+/* Precomputed modular exponentiation for verification */
+uint8_t mod_exp[] = {
+	0x2C, 0x60, 0x75, 0x45, 0x98, 0x9D, 0xE0, 0x72,
+	0xA0, 0x9D, 0x3A, 0x9E, 0x03, 0x38, 0x73, 0x3C,
+	0x31, 0x83, 0x04, 0xFE, 0x75, 0x43, 0xE6, 0x17,
+	0x5C, 0x01, 0x29, 0x51, 0x69, 0x33, 0x62, 0x2D,
+	0x78, 0xBE, 0xAE, 0xC4, 0xBC, 0xDE, 0x7E, 0x2C,
+	0x77, 0x84, 0xF2, 0xC5, 0x14, 0xB5, 0x2F, 0xF7,
+	0xC5, 0x94, 0xEF, 0x86, 0x75, 0x75, 0xB5, 0x11,
+	0xE5, 0x0E, 0x0A, 0x29, 0x76, 0xE2, 0xEA, 0x32,
+	0x0E, 0x43, 0x77, 0x7E, 0x2C, 0x27, 0xAC, 0x3B,
+	0x86, 0xA5, 0xDB, 0xC9, 0x48, 0x40, 0xE8, 0x99,
+	0x9A, 0x0A, 0x3D, 0xD6, 0x74, 0xFA, 0x2E, 0x2E,
+	0x5B, 0xAF, 0x8C, 0x99, 0x44, 0x2A, 0x67, 0x38,
+	0x27, 0x41, 0x59, 0x9D, 0xB8, 0x51, 0xC9, 0xF7,
+	0x43, 0x61, 0x31, 0x6E, 0xF1, 0x25, 0x38, 0x7F,
+	0xAE, 0xC6, 0xD0, 0xBB, 0x29, 0x76, 0x3F, 0x46,
+	0x2E, 0x1B, 0xE4, 0x67, 0x71, 0xE3, 0x87, 0x5A
+};
+
+/* Precomputed modular inverse for verification */
+uint8_t mod_inv[] = {
+	0x52, 0xb1, 0xa3, 0x8c, 0xc5, 0x8a, 0xb9, 0x1f,
+	0xb6, 0x82, 0xf5, 0x6a, 0x9a, 0xde, 0x8d, 0x2e,
+	0x62, 0x4b, 0xac, 0x49, 0x21, 0x1d, 0x30, 0x4d,
+	0x32, 0xac, 0x1f, 0x40, 0x6d, 0x52, 0xc7, 0x9b,
+	0x6c, 0x0a, 0x82, 0x3a, 0x2c, 0xaf, 0x6b, 0x6d,
+	0x17, 0xbe, 0x43, 0xed, 0x97, 0x78, 0xeb, 0x4c,
+	0x92, 0x6f, 0xcf, 0xed, 0xb1, 0x09, 0xcb, 0x27,
+	0xc2, 0xde, 0x62, 0xfd, 0x21, 0xe6, 0xbd, 0x4f,
+	0xfe, 0x7a, 0x1b, 0x50, 0xfe, 0x10, 0x4a, 0xb0,
+	0xb7, 0xcf, 0xdb, 0x7d, 0xca, 0xc2, 0xf0, 0x1c,
+	0x39, 0x48, 0x6a, 0xb5, 0x4d, 0x8c, 0xfe, 0x63,
+	0x91, 0x9c, 0x21, 0xc3, 0x0e, 0x76, 0xad, 0x44,
+	0x8d, 0x54, 0x33, 0x99, 0xe1, 0x80, 0x19, 0xba,
+	0xb5, 0xac, 0x7d, 0x9c, 0xce, 0x91, 0x2a, 0xd9,
+	0x2c, 0xe1, 0x16, 0xd6, 0xd7, 0xcf, 0x9d, 0x05,
+	0x9a, 0x66, 0x9a, 0x3a, 0xc1, 0xb8, 0x4b, 0xc3
+};
+
+struct rte_crypto_asym_xform modex_xform = {
+	.next = NULL,
+	.xform_type = RTE_CRYPTO_ASYM_XFORM_MODEX,
+	.modex = {
+		.modulus = {
+			.data = mod_p,
+			.length = sizeof(mod_p)
+		},
+		.exponent = {
+			.data = mod_e,
+			.length = sizeof(mod_e)
+		}
+	}
+};
+
+struct rte_crypto_asym_xform modinv_xform = {
+	.next = NULL,
+	.xform_type = RTE_CRYPTO_ASYM_XFORM_MODINV,
+	.modinv = {
+		.modulus = {
+			.data = mod_p,
+			.length = sizeof(mod_p)
+		}
+	}
+};
+
+#endif /* TEST_CRYPTODEV_MOD_TEST_VECTORS_H__ */
diff --git a/test/test/test_cryptodev_rsa_test_vectors.h b/test/test/test_cryptodev_rsa_test_vectors.h
new file mode 100644
index 00000000..3f8c41a6
--- /dev/null
+++ b/test/test/test_cryptodev_rsa_test_vectors.h
@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium Networks
+ */
+
+#ifndef TEST_CRYPTODEV_RSA_TEST_VECTORS_H__
+#define TEST_CRYPTODEV_RSA_TEST_VECTORS_H__
+
+#include "rte_crypto_asym.h"
+
+#define TEST_DATA_SIZE 4096
+
+struct rsa_test_data {
+	uint8_t data[TEST_DATA_SIZE];
+	unsigned int len;
+};
+
+struct rsa_test_data rsaplaintext = {
+	.data = {
+		0xf8, 0xba, 0x1a, 0x55, 0xd0, 0x2f, 0x85, 0xae,
+		0x96, 0x7b, 0xb6, 0x2f, 0xb6, 0xcd, 0xa8, 0xeb,
+		0x7e, 0x78, 0xa0, 0x50
+	},
+	.len = 20
+};
+
+uint8_t rsa_n[] = {
+	0xb3, 0xa1, 0xaf, 0xb7, 0x13, 0x08, 0x00,
+	0x0a, 0x35, 0xdc, 0x2b, 0x20, 0x8d, 0xa1, 0xb5,
+	0xce, 0x47, 0x8a, 0xc3, 0x80, 0xf4, 0x7d, 0x4a,
+	0xa2, 0x62, 0xfd, 0x61, 0x7f, 0xb5, 0xa8, 0xde,
+	0x0a, 0x17, 0x97, 0xa0, 0xbf, 0xdf, 0x56, 0x5a,
+	0x3d, 0x51, 0x56, 0x4f, 0x70, 0x70, 0x3f, 0x63,
+	0x6a, 0x44, 0x5b, 0xad, 0x84, 0x0d, 0x3f, 0x27,
+	0x6e, 0x3b, 0x34, 0x91, 0x60, 0x14, 0xb9, 0xaa,
+	0x72, 0xfd, 0xa3, 0x64, 0xd2, 0x03, 0xa7, 0x53,
+	0x87, 0x9e, 0x88, 0x0b, 0xc1, 0x14, 0x93, 0x1a,
+	0x62, 0xff, 0xb1, 0x5d, 0x74, 0xcd, 0x59, 0x63,
+	0x18, 0x11, 0x3d, 0x4f, 0xba, 0x75, 0xd4, 0x33,
+	0x4e, 0x23, 0x6b, 0x7b, 0x57, 0x44, 0xe1, 0xd3,
+	0x03, 0x13, 0xa6, 0xf0, 0x8b, 0x60, 0xb0, 0x9e,
+	0xee, 0x75, 0x08, 0x9d, 0x71, 0x63, 0x13, 0xcb,
+	0xa6, 0x81, 0x92, 0x14, 0x03, 0x22, 0x2d, 0xde,
+	0x55
+};
+
+uint8_t rsa_d[] = {
+	0x24, 0xd7, 0xea, 0xf4, 0x7f, 0xe0, 0xca, 0x31,
+	0x4d, 0xee, 0xc4, 0xa1, 0xbe, 0xab, 0x06, 0x61,
+	0x32, 0xe7, 0x51, 0x46, 0x27, 0xdf, 0x72, 0xe9,
+	0x6f, 0xa8, 0x4c, 0xd1, 0x26, 0xef, 0x65, 0xeb,
+	0x67, 0xff, 0x5f, 0xa7, 0x3b, 0x25, 0xb9, 0x08,
+	0x8e, 0xa0, 0x47, 0x56, 0xe6, 0x8e, 0xf9, 0xd3,
+	0x18, 0x06, 0x3d, 0xc6, 0xb1, 0xf8, 0xdc, 0x1b,
+	0x8d, 0xe5, 0x30, 0x54, 0x26, 0xac, 0x16, 0x3b,
+	0x7b, 0xad, 0x46, 0x9e, 0x21, 0x6a, 0x57, 0xe6,
+	0x81, 0x56, 0x1d, 0x2a, 0xc4, 0x39, 0x63, 0x67,
+	0x81, 0x2c, 0xca, 0xcc, 0xf8, 0x42, 0x04, 0xbe,
+	0xcf, 0x8f, 0x6c, 0x5b, 0x81, 0x46, 0xb9, 0xc7,
+	0x62, 0x90, 0x87, 0x35, 0x03, 0x9b, 0x89, 0xcb,
+	0x37, 0xbd, 0xf1, 0x1b, 0x99, 0xa1, 0x9a, 0x78,
+	0xd5, 0x4c, 0xdd, 0x3f, 0x41, 0x0c, 0xb7, 0x1a,
+	0xd9, 0x7b, 0x87, 0x5f, 0xbe, 0xb1, 0x83, 0x41
+};
+
+uint8_t rsa_e[] = {0x01, 0x00, 0x01};
+
+/** rsa xform using exponent key */
+struct rte_crypto_asym_xform rsa_xform = {
+	.next = NULL,
+	.xform_type = RTE_CRYPTO_ASYM_XFORM_RSA,
+	.rsa = {
+		.n = {
+			.data = rsa_n,
+			.length = sizeof(rsa_n)
+		},
+		.e = {
+			.data = rsa_e,
+			.length = sizeof(rsa_e)
+		},
+		.key_type = RTE_RSA_KEY_TYPE_EXP,
+		.d = {
+			.data = rsa_d,
+			.length = sizeof(rsa_d)
+		}
+	}
+};
+
+#endif /* TEST_CRYPTODEV_RSA_TEST_VECTORS_H__ */
diff --git a/test/test/test_eal_flags.c b/test/test/test_eal_flags.c
index f840ca50..3a990766 100644
--- a/test/test/test_eal_flags.c
+++ b/test/test/test_eal_flags.c
@@ -221,30 +221,6 @@ get_number_of_sockets(void)
 }
 #endif
 
-static char*
-get_current_prefix(char * prefix, int size)
-{
-	char path[PATH_MAX] = {0};
-	char buf[PATH_MAX] = {0};
-
-	/* get file for config (fd is always 3) */
-	snprintf(path, sizeof(path), "/proc/self/fd/%d", 3);
-
-	/* return NULL on error */
-	if (readlink(path, buf, sizeof(buf)) == -1)
-		return NULL;
-
-	/* get the basename */
-	snprintf(buf, sizeof(buf), "%s", basename(buf));
-
-	/* copy string all the way from second char up to start of _config */
-	snprintf(prefix, size, "%.*s",
-			(int)(strnlen(buf, sizeof(buf)) - sizeof("_config")),
-			&buf[1]);
-
-	return prefix;
-}
-
 /*
  * Test that the app doesn't run with invalid whitelist option.
  * Final tests ensures it does run with valid options as sanity check (one
@@ -376,17 +352,17 @@ test_invalid_vdev_flag(void)
 #endif
 
 	/* Test with invalid vdev option */
-	const char *vdevinval[] = {prgname, prefix, "-n", "1",
+	const char *vdevinval[] = {prgname, prefix, no_huge, "-n", "1",
 				"-c", "1", vdev, "eth_dummy"};
 
 	/* Test with valid vdev option */
-	const char *vdevval1[] = {prgname, prefix, "-n", "1",
+	const char *vdevval1[] = {prgname, prefix, no_huge, "-n", "1",
 	"-c", "1", vdev, "net_ring0"};
 
-	const char *vdevval2[] = {prgname, prefix, "-n", "1",
+	const char *vdevval2[] = {prgname, prefix, no_huge, "-n", "1",
 	"-c", "1", vdev, "net_ring0,args=test"};
 
-	const char *vdevval3[] = {prgname, prefix, "-n", "1",
+	const char *vdevval3[] = {prgname, prefix, no_huge, "-n", "1",
 	"-c", "1", vdev, "net_ring0,nodeaction=r1:0:CREATE"};
 
 	if (launch_proc(vdevinval) == 0) {
@@ -697,16 +673,18 @@ test_invalid_n_flag(void)
 static int
 test_no_hpet_flag(void)
 {
-	char prefix[PATH_MAX], tmp[PATH_MAX];
+	char prefix[PATH_MAX] = "";
 
 #ifdef RTE_EXEC_ENV_BSDAPP
 	return 0;
-#endif
+#else
+	char tmp[PATH_MAX];
 	if (get_current_prefix(tmp, sizeof(tmp)) == NULL) {
 		printf("Error - unable to get current prefix!\n");
 		return -1;
 	}
 	snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp);
+#endif
 
 	/* With --no-hpet */
 	const char *argv1[] = {prgname, prefix, mp_flag, no_hpet, "-c", "1", "-n", "2"};
@@ -849,13 +827,10 @@ test_misc_flags(void)
 	const char *argv4[] = {prgname, prefix, mp_flag, "-c", "1", "--syslog"};
 	/* With invalid --syslog */
 	const char *argv5[] = {prgname, prefix, mp_flag, "-c", "1", "--syslog", "error"};
-	/* With no-sh-conf */
+	/* With no-sh-conf, also use no-huge to ensure this test runs on BSD */
 	const char *argv6[] = {prgname, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE,
-			no_shconf, nosh_prefix };
+			no_shconf, nosh_prefix, no_huge};
 
-#ifdef RTE_EXEC_ENV_BSDAPP
-	return 0;
-#endif
 	/* With --huge-dir */
 	const char *argv7[] = {prgname, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE,
 			"--file-prefix=hugedir", "--huge-dir", hugepath};
@@ -889,6 +864,7 @@ test_misc_flags(void)
 	const char *argv15[] = {prgname, "--file-prefix=intr",
 			"-c", "1", "-n", "2", "--vfio-intr=invalid"};
 
+	/* run all tests also applicable to FreeBSD first */
 
 	if (launch_proc(argv0) == 0) {
 		printf("Error - process ran ok with invalid flag\n");
@@ -902,6 +878,16 @@ test_misc_flags(void)
 		printf("Error - process did not run ok with -v flag\n");
 		return -1;
 	}
+	if (launch_proc(argv6) != 0) {
+		printf("Error - process did not run ok with --no-shconf flag\n");
+		return -1;
+	}
+
+#ifdef RTE_EXEC_ENV_BSDAPP
+	/* no more tests to be done on FreeBSD */
+	return 0;
+#endif
+
 	if (launch_proc(argv3) != 0) {
 		printf("Error - process did not run ok with --syslog flag\n");
 		return -1;
@@ -914,13 +900,6 @@ test_misc_flags(void)
 		printf("Error - process run ok with invalid --syslog flag\n");
 		return -1;
 	}
-	if (launch_proc(argv6) != 0) {
-		printf("Error - process did not run ok with --no-shconf flag\n");
-		return -1;
-	}
-#ifdef RTE_EXEC_ENV_BSDAPP
-	return 0;
-#endif
 	if (launch_proc(argv7) != 0) {
 		printf("Error - process did not run ok with --huge-dir flag\n");
 		return -1;
@@ -977,9 +956,15 @@ test_file_prefix(void)
 	 * 6. run a primary process with memtest2 prefix
 	 * 7. check that only memtest2 hugefiles are present in the hugedir
 	 */
+	char prefix[PATH_MAX] = "";
 
 #ifdef RTE_EXEC_ENV_BSDAPP
 	return 0;
+#else
+	if (get_current_prefix(prefix, sizeof(prefix)) == NULL) {
+		printf("Error - unable to get current prefix!\n");
+		return -1;
+	}
 #endif
 
 	/* this should fail unless the test itself is run with "memtest" prefix */
@@ -994,12 +979,6 @@ test_file_prefix(void)
 	const char *argv2[] = {prgname, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE,
 				"--file-prefix=" memtest2 };
 
-	char prefix[32];
-	if (get_current_prefix(prefix, sizeof(prefix)) == NULL) {
-		printf("Error - unable to get current prefix!\n");
-		return -1;
-	}
-
 	/* check if files for current prefix are present */
 	if (process_hugefiles(prefix, HUGEPAGE_CHECK_EXISTS) != 1) {
 		printf("Error - hugepage files for %s were not created!\n", prefix);
diff --git a/test/test/test_event_crypto_adapter.c b/test/test/test_event_crypto_adapter.c
index 066b0ade..de258c34 100644
--- a/test/test/test_event_crypto_adapter.c
+++ b/test/test/test_event_crypto_adapter.c
@@ -205,12 +205,12 @@ test_op_forward_mode(uint8_t session_less)
 		TEST_ASSERT_SUCCESS(ret, "Failed to get adapter capabilities\n");
 
 		if (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA) {
-			/* Fill in private date information */
+			/* Fill in private user data information */
 			rte_memcpy(&m_data.response_info, &response_info,
 				sizeof(response_info));
 			rte_memcpy(&m_data.request_info, &request_info,
 				sizeof(request_info));
-			rte_cryptodev_sym_session_set_private_data(sess,
+			rte_cryptodev_sym_session_set_user_data(sess,
 						&m_data, sizeof(m_data));
 		}
 
@@ -389,10 +389,10 @@ test_op_new_mode(uint8_t session_less)
 		TEST_ASSERT_SUCCESS(ret, "Failed to get adapter capabilities\n");
 
 		if (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA) {
-			/* Fill in private data information */
+			/* Fill in private user data information */
 			rte_memcpy(&m_data.response_info, &response_info,
 				   sizeof(m_data));
-			rte_cryptodev_sym_session_set_private_data(sess,
+			rte_cryptodev_sym_session_set_user_data(sess,
 						&m_data, sizeof(m_data));
 		}
 		rte_cryptodev_sym_session_init(TEST_CDEV_ID, sess,
diff --git a/test/test/test_event_eth_rx_adapter.c b/test/test/test_event_eth_rx_adapter.c
index dee632bf..2337e543 100644
--- a/test/test/test_event_eth_rx_adapter.c
+++ b/test/test/test_event_eth_rx_adapter.c
@@ -25,28 +25,17 @@ struct event_eth_rx_adapter_test_params {
 	struct rte_mempool *mp;
 	uint16_t rx_rings, tx_rings;
 	uint32_t caps;
+	int rx_intr_port_inited;
+	uint16_t rx_intr_port;
 };
 
 static struct event_eth_rx_adapter_test_params default_params;
 
 static inline int
-port_init(uint8_t port, struct rte_mempool *mp)
+port_init_common(uint8_t port, const struct rte_eth_conf *port_conf,
+		struct rte_mempool *mp)
 {
-	static const struct rte_eth_conf port_conf_default = {
-		.rxmode = {
-			.mq_mode = ETH_MQ_RX_RSS,
-			.max_rx_pkt_len = ETHER_MAX_LEN
-		},
-		.rx_adv_conf = {
-			.rss_conf = {
-				.rss_hf = ETH_RSS_IP |
-					  ETH_RSS_TCP |
-					  ETH_RSS_UDP,
-			}
-		}
-	};
 	const uint16_t rx_ring_size = 512, tx_ring_size = 512;
-	struct rte_eth_conf port_conf = port_conf_default;
 	int retval;
 	uint16_t q;
 	struct rte_eth_dev_info dev_info;
@@ -54,7 +43,7 @@ port_init(uint8_t port, struct rte_mempool *mp)
 	if (!rte_eth_dev_is_valid_port(port))
 		return -1;
 
-	retval = rte_eth_dev_configure(port, 0, 0, &port_conf);
+	retval = rte_eth_dev_configure(port, 0, 0, port_conf);
 
 	rte_eth_dev_info_get(port, &dev_info);
 
@@ -64,7 +53,7 @@ port_init(uint8_t port, struct rte_mempool *mp)
 
 	/* Configure the Ethernet device. */
 	retval = rte_eth_dev_configure(port, default_params.rx_rings,
-				default_params.tx_rings, &port_conf);
+				default_params.tx_rings, port_conf);
 	if (retval != 0)
 		return retval;
 
@@ -104,18 +93,95 @@ port_init(uint8_t port, struct rte_mempool *mp)
 	return 0;
 }
 
+static inline int
+port_init_rx_intr(uint8_t port, struct rte_mempool *mp)
+{
+	static const struct rte_eth_conf port_conf_default = {
+		.rxmode = {
+			.mq_mode = ETH_MQ_RX_RSS,
+			.max_rx_pkt_len = ETHER_MAX_LEN
+		},
+		.intr_conf = {
+			.rxq = 1,
+		},
+	};
+
+	return port_init_common(port, &port_conf_default, mp);
+}
+
+static inline int
+port_init(uint8_t port, struct rte_mempool *mp)
+{
+	static const struct rte_eth_conf port_conf_default = {
+		.rxmode = {
+			.mq_mode = ETH_MQ_RX_RSS,
+			.max_rx_pkt_len = ETHER_MAX_LEN
+		},
+		.rx_adv_conf = {
+			.rss_conf = {
+				.rss_hf = ETH_RSS_IP |
+					ETH_RSS_TCP |
+					ETH_RSS_UDP,
+			}
+		}
+	};
+
+	return port_init_common(port, &port_conf_default, mp);
+}
+
+static int
+init_port_rx_intr(int num_ports)
+{
+	int retval;
+	uint16_t portid;
+	int err;
+
+	default_params.mp = rte_pktmbuf_pool_create("packet_pool",
+						   NB_MBUFS,
+						   MBUF_CACHE_SIZE,
+						   MBUF_PRIV_SIZE,
+						   RTE_MBUF_DEFAULT_BUF_SIZE,
+						   rte_socket_id());
+	if (!default_params.mp)
+		return -ENOMEM;
+
+	RTE_ETH_FOREACH_DEV(portid) {
+		retval = port_init_rx_intr(portid, default_params.mp);
+		if (retval)
+			continue;
+		err = rte_event_eth_rx_adapter_caps_get(TEST_DEV_ID, portid,
+							&default_params.caps);
+		if (err)
+			continue;
+		if (!(default_params.caps &
+			RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) {
+			default_params.rx_intr_port_inited = 1;
+			default_params.rx_intr_port = portid;
+			return 0;
+		}
+		rte_eth_dev_stop(portid);
+	}
+	return 0;
+}
+
 static int
 init_ports(int num_ports)
 {
 	uint16_t portid;
 	int retval;
 
-	default_params.mp = rte_pktmbuf_pool_create("packet_pool",
+	struct rte_mempool *ptr = rte_mempool_lookup("packet_pool");
+
+	if (ptr == NULL)
+		default_params.mp = rte_pktmbuf_pool_create("packet_pool",
 						NB_MBUFS,
 						MBUF_CACHE_SIZE,
 						MBUF_PRIV_SIZE,
 						RTE_MBUF_DEFAULT_BUF_SIZE,
 						rte_socket_id());
+	else
+		default_params.mp = ptr;
+
 	if (!default_params.mp)
 		return -ENOMEM;
 
@@ -175,6 +241,57 @@ testsuite_setup(void)
 	return err;
 }
 
+static int
+testsuite_setup_rx_intr(void)
+{
+	int err;
+	uint8_t count;
+	struct rte_event_dev_info dev_info;
+
+	count = rte_event_dev_count();
+	if (!count) {
+		printf("Failed to find a valid event device,"
+			" testing with event_skeleton device\n");
+		rte_vdev_init("event_skeleton", NULL);
+	}
+
+	struct rte_event_dev_config config = {
+		.nb_event_queues = 1,
+		.nb_event_ports = 1,
+	};
+
+	err = rte_event_dev_info_get(TEST_DEV_ID, &dev_info);
+	config.nb_event_queue_flows = dev_info.max_event_queue_flows;
+	config.nb_event_port_dequeue_depth =
+			dev_info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth =
+			dev_info.max_event_port_enqueue_depth;
+	config.nb_events_limit =
+			dev_info.max_num_events;
+
+	err = rte_event_dev_configure(TEST_DEV_ID, &config);
+	TEST_ASSERT(err == 0, "Event device initialization failed err %d\n",
+			err);
+
+	/*
+	 * eth devices like octeontx use event device to receive packets
+	 * so rte_eth_dev_start invokes rte_event_dev_start internally, so
+	 * call init_ports after rte_event_dev_configure
+	 */
+	err = init_port_rx_intr(rte_eth_dev_count_total());
+	TEST_ASSERT(err == 0, "Port initialization failed err %d\n", err);
+
+	if (!default_params.rx_intr_port_inited)
+		return 0;
+
+	err = rte_event_eth_rx_adapter_caps_get(TEST_DEV_ID,
+						default_params.rx_intr_port,
+						&default_params.caps);
+	TEST_ASSERT(err == 0, "Failed to get adapter cap err %d\n", err);
+
+	return err;
+}
+
 static void
 testsuite_teardown(void)
 {
@@ -185,6 +302,16 @@ testsuite_teardown(void)
 	rte_mempool_free(default_params.mp);
 }
 
+static void
+testsuite_teardown_rx_intr(void)
+{
+	if (!default_params.rx_intr_port_inited)
+		return;
+
+	rte_eth_dev_stop(default_params.rx_intr_port);
+	rte_mempool_free(default_params.mp);
+}
+
 static int
 adapter_create(void)
 {
@@ -333,6 +460,151 @@ adapter_queue_add_del(void)
 }
 
 static int
+adapter_multi_eth_add_del(void)
+{
+	int err;
+	struct rte_event ev;
+
+	uint16_t port_index, drv_id = 0;
+	char driver_name[50];
+
+	struct rte_event_eth_rx_adapter_queue_conf queue_config;
+
+	ev.queue_id = 0;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.priority = 0;
+
+	queue_config.rx_queue_flags = 0;
+	queue_config.ev = ev;
+	queue_config.servicing_weight = 1;
+
+	/* stop eth devices for existing */
+	port_index = 0;
+	for (; port_index < rte_eth_dev_count_total(); port_index += 1)
+		rte_eth_dev_stop(port_index);
+
+	/* add the max port for rx_adapter */
+	port_index = rte_eth_dev_count_total();
+	for (; port_index < RTE_MAX_ETHPORTS; port_index += 1) {
+		sprintf(driver_name, "%s%u", "net_null", drv_id);
+		err = rte_vdev_init(driver_name, NULL);
+		TEST_ASSERT(err == 0, "Failed driver %s got %d",
+		driver_name, err);
+		drv_id += 1;
+	}
+
+	err = init_ports(rte_eth_dev_count_total());
+	TEST_ASSERT(err == 0, "Port initialization failed err %d\n", err);
+
+	/* creating new instance for all newly added eth devices */
+	adapter_create();
+
+	/* eth_rx_adapter_queue_add for n ports */
+	port_index = 0;
+	for (; port_index < rte_eth_dev_count_total(); port_index += 1) {
+		err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID,
+				port_index, 0,
+				&queue_config);
+		TEST_ASSERT(err == 0, "Expected 0 got %d", err);
+	}
+
+	/* eth_rx_adapter_queue_del n ports */
+	port_index = 0;
+	for (; port_index < rte_eth_dev_count_total(); port_index += 1) {
+		err = rte_event_eth_rx_adapter_queue_del(TEST_INST_ID,
+				port_index, 0);
+		TEST_ASSERT(err == 0, "Expected 0 got %d", err);
+	}
+
+	adapter_free();
+
+	return TEST_SUCCESS;
+}
+
+static int
+adapter_intr_queue_add_del(void)
+{
+	int err;
+	struct rte_event ev;
+	uint32_t cap;
+	uint16_t eth_port;
+	struct rte_event_eth_rx_adapter_queue_conf queue_config;
+
+	if (!default_params.rx_intr_port_inited)
+		return 0;
+
+	eth_port = default_params.rx_intr_port;
+	err = rte_event_eth_rx_adapter_caps_get(TEST_DEV_ID, eth_port, &cap);
+	TEST_ASSERT(err == 0, "Expected 0 got %d", err);
+
+	ev.queue_id = 0;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.priority = 0;
+
+	queue_config.rx_queue_flags = 0;
+	queue_config.ev = ev;
+
+	/* weight = 0 => interrupt mode */
+	queue_config.servicing_weight = 0;
+
+	/* add queue 0 */
+	err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID,
+						TEST_ETHDEV_ID, 0,
+						&queue_config);
+	TEST_ASSERT(err == 0, "Expected 0 got %d", err);
+
+	/* add all queues */
+	queue_config.servicing_weight = 0;
+	err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID,
+						TEST_ETHDEV_ID,
+						-1,
+						&queue_config);
+	TEST_ASSERT(err == 0, "Expected 0 got %d", err);
+
+	/* del queue 0 */
+	err = rte_event_eth_rx_adapter_queue_del(TEST_INST_ID,
+						TEST_ETHDEV_ID,
+						0);
+	TEST_ASSERT(err == 0, "Expected 0 got %d", err);
+
+	/* del remaining queues */
+	err = rte_event_eth_rx_adapter_queue_del(TEST_INST_ID,
+						TEST_ETHDEV_ID,
+						-1);
+	TEST_ASSERT(err == 0, "Expected 0 got %d", err);
+
+	/* add all queues */
+	queue_config.servicing_weight = 0;
+	err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID,
+						TEST_ETHDEV_ID,
+						-1,
+						&queue_config);
+	TEST_ASSERT(err == 0, "Expected 0 got %d", err);
+
+	/* intr -> poll mode queue */
+	queue_config.servicing_weight = 1;
+	err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID,
+						TEST_ETHDEV_ID,
+						0,
+						&queue_config);
+	TEST_ASSERT(err == 0, "Expected 0 got %d", err);
+
+	err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID,
+						TEST_ETHDEV_ID,
+						-1,
+						 &queue_config);
+	TEST_ASSERT(err == 0, "Expected 0 got %d", err);
+
+	/* del queues */
+	err = rte_event_eth_rx_adapter_queue_del(TEST_INST_ID,
+						TEST_ETHDEV_ID,
+						-1);
+	TEST_ASSERT(err == 0, "Expected 0 got %d", err);
+
+	return TEST_SUCCESS;
+}
+
+static int
 adapter_start_stop(void)
 {
 	int err;
@@ -402,7 +674,7 @@ adapter_stats(void)
 	return TEST_SUCCESS;
 }
 
-static struct unit_test_suite service_tests  = {
+static struct unit_test_suite event_eth_rx_tests = {
 	.suite_name = "rx event eth adapter test suite",
 	.setup = testsuite_setup,
 	.teardown = testsuite_teardown,
@@ -410,17 +682,37 @@ static struct unit_test_suite service_tests  = {
 		TEST_CASE_ST(NULL, NULL, adapter_create_free),
 		TEST_CASE_ST(adapter_create, adapter_free,
 					adapter_queue_add_del),
+		TEST_CASE_ST(NULL, NULL, adapter_multi_eth_add_del),
 		TEST_CASE_ST(adapter_create, adapter_free, adapter_start_stop),
 		TEST_CASE_ST(adapter_create, adapter_free, adapter_stats),
 		TEST_CASES_END() /**< NULL terminate unit test array */
 	}
 };
 
+static struct unit_test_suite event_eth_rx_intr_tests = {
+	.suite_name = "rx event eth adapter test suite",
+	.setup = testsuite_setup_rx_intr,
+	.teardown = testsuite_teardown_rx_intr,
+	.unit_test_cases = {
+		TEST_CASE_ST(adapter_create, adapter_free,
+			adapter_intr_queue_add_del),
+		TEST_CASES_END() /**< NULL terminate unit test array */
+	}
+};
+
 static int
 test_event_eth_rx_adapter_common(void)
 {
-	return unit_test_suite_runner(&service_tests);
+	return unit_test_suite_runner(&event_eth_rx_tests);
+}
+
+static int
+test_event_eth_rx_intr_adapter_common(void)
+{
+	return unit_test_suite_runner(&event_eth_rx_intr_tests);
 }
 
 REGISTER_TEST_COMMAND(event_eth_rx_adapter_autotest,
 		test_event_eth_rx_adapter_common);
+REGISTER_TEST_COMMAND(event_eth_rx_intr_adapter_autotest,
+		test_event_eth_rx_intr_adapter_common);
diff --git a/test/test/test_fbarray.c b/test/test/test_fbarray.c
new file mode 100644
index 00000000..8c44e2c7
--- /dev/null
+++ b/test/test/test_fbarray.c
@@ -0,0 +1,576 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <limits.h>
+
+#include <rte_common.h>
+#include <rte_debug.h>
+#include <rte_errno.h>
+#include <rte_fbarray.h>
+
+#include "test.h"
+
+struct fbarray_testsuite_params {
+	struct rte_fbarray arr;
+	int start;
+	int end;
+};
+
+static struct fbarray_testsuite_params param;
+
+#define FBARRAY_TEST_ARR_NAME "fbarray_autotest"
+#define FBARRAY_TEST_LEN 256
+#define FBARRAY_TEST_ELT_SZ (sizeof(int))
+
+static int autotest_setup(void)
+{
+	return rte_fbarray_init(&param.arr, FBARRAY_TEST_ARR_NAME,
+			FBARRAY_TEST_LEN, FBARRAY_TEST_ELT_SZ);
+}
+
+static void autotest_teardown(void)
+{
+	rte_fbarray_destroy(&param.arr);
+}
+
+static int init_array(void)
+{
+	int i;
+	for (i = param.start; i <= param.end; i++) {
+		if (rte_fbarray_set_used(&param.arr, i))
+			return -1;
+	}
+	return 0;
+}
+
+static void reset_array(void)
+{
+	int i;
+	for (i = 0; i < FBARRAY_TEST_LEN; i++)
+		rte_fbarray_set_free(&param.arr, i);
+}
+
+static int first_msk_test_setup(void)
+{
+	/* put all within first mask */
+	param.start = 3;
+	param.end = 10;
+	return init_array();
+}
+
+static int cross_msk_test_setup(void)
+{
+	/* put all within second and third mask */
+	param.start = 70;
+	param.end = 160;
+	return init_array();
+}
+
+static int multi_msk_test_setup(void)
+{
+	/* put all within first and last mask */
+	param.start = 3;
+	param.end = FBARRAY_TEST_LEN - 20;
+	return init_array();
+}
+
+static int last_msk_test_setup(void)
+{
+	/* put all within last mask */
+	param.start = FBARRAY_TEST_LEN - 20;
+	param.end = FBARRAY_TEST_LEN - 1;
+	return init_array();
+}
+
+static int full_msk_test_setup(void)
+{
+	/* fill entire mask */
+	param.start = 0;
+	param.end = FBARRAY_TEST_LEN - 1;
+	return init_array();
+}
+
+static int empty_msk_test_setup(void)
+{
+	/* do not fill anything in */
+	reset_array();
+	return 0;
+}
+
+static int test_invalid(void)
+{
+	struct rte_fbarray dummy;
+
+	/* invalid parameters */
+	TEST_ASSERT_FAIL(rte_fbarray_attach(NULL),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT_FAIL(rte_fbarray_detach(NULL),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT_FAIL(rte_fbarray_destroy(NULL),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno valuey\n");
+	TEST_ASSERT_FAIL(rte_fbarray_init(NULL, "fail", 16, 16),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT_FAIL(rte_fbarray_init(&dummy, NULL, 16, 16),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT_FAIL(rte_fbarray_init(&dummy, "fail", 0, 16),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT_FAIL(rte_fbarray_init(&dummy, "fail", 16, 0),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	/* len must not be greater than INT_MAX */
+	TEST_ASSERT_FAIL(rte_fbarray_init(&dummy, "fail", INT_MAX + 1U, 16),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT_NULL(rte_fbarray_get(NULL, 0),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_idx(NULL, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_set_free(NULL, 0),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_set_used(NULL, 0),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_contig_free(NULL, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_contig_used(NULL, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_rev_contig_free(NULL, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_rev_contig_used(NULL, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_next_free(NULL, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_next_used(NULL, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_prev_free(NULL, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_prev_used(NULL, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_next_n_free(NULL, 0, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_next_n_used(NULL, 0, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_prev_n_free(NULL, 0, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_prev_n_used(NULL, 0, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_is_used(NULL, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT_SUCCESS(rte_fbarray_init(&dummy, "success",
+			FBARRAY_TEST_LEN, 8),
+			"Failed to initialize valid fbarray\n");
+
+	/* test API for handling invalid parameters with a valid fbarray */
+	TEST_ASSERT_NULL(rte_fbarray_get(&dummy, FBARRAY_TEST_LEN),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_idx(&dummy, NULL) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_set_free(&dummy, FBARRAY_TEST_LEN),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_set_used(&dummy, FBARRAY_TEST_LEN),
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_contig_free(&dummy, FBARRAY_TEST_LEN) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_contig_used(&dummy, FBARRAY_TEST_LEN) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_rev_contig_free(&dummy,
+			FBARRAY_TEST_LEN) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_rev_contig_used(&dummy,
+			FBARRAY_TEST_LEN) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_next_free(&dummy, FBARRAY_TEST_LEN) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_next_used(&dummy, FBARRAY_TEST_LEN) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_prev_free(&dummy, FBARRAY_TEST_LEN) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_prev_used(&dummy, FBARRAY_TEST_LEN) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_next_n_free(&dummy,
+			FBARRAY_TEST_LEN, 1) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_next_n_free(&dummy, 0,
+			FBARRAY_TEST_LEN + 1) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_next_n_free(&dummy, 0, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_next_n_used(&dummy,
+			FBARRAY_TEST_LEN, 1) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_next_n_used(&dummy, 0,
+			FBARRAY_TEST_LEN + 1) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_next_n_used(&dummy, 0, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_prev_n_free(&dummy,
+			FBARRAY_TEST_LEN, 1) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_prev_n_free(&dummy, 0,
+			FBARRAY_TEST_LEN + 1) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_prev_n_free(&dummy, 0, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_find_prev_n_used(&dummy,
+			FBARRAY_TEST_LEN, 1) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_prev_n_used(&dummy, 0,
+			FBARRAY_TEST_LEN + 1) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_prev_n_used(&dummy, 0, 0) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT(rte_fbarray_is_used(&dummy, FBARRAY_TEST_LEN) < 0,
+			"Call succeeded with invalid parameters\n");
+	TEST_ASSERT_EQUAL(rte_errno, EINVAL, "Wrong errno value\n");
+
+	TEST_ASSERT_SUCCESS(rte_fbarray_destroy(&dummy),
+			"Failed to destroy valid fbarray\n");
+
+	return TEST_SUCCESS;
+}
+
+static int check_free(void)
+{
+	const int idx = 0;
+	const int last_idx = FBARRAY_TEST_LEN - 1;
+
+	/* ensure we can find a free spot */
+	TEST_ASSERT_EQUAL(rte_fbarray_find_next_free(&param.arr, idx), idx,
+			"Free space not found where expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_next_n_free(&param.arr, idx, 1), idx,
+			"Free space not found where expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_contig_free(&param.arr, idx),
+			FBARRAY_TEST_LEN,
+			"Free space not found where expected\n");
+
+	TEST_ASSERT_EQUAL(rte_fbarray_find_prev_free(&param.arr, idx), idx,
+			"Free space not found where expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_prev_n_free(&param.arr, idx, 1), idx,
+			"Free space not found where expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_rev_contig_free(&param.arr, idx), 1,
+			"Free space not found where expected\n");
+
+	TEST_ASSERT_EQUAL(rte_fbarray_find_prev_free(&param.arr, last_idx),
+			last_idx, "Free space not found where expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_prev_n_free(&param.arr, last_idx, 1),
+			last_idx, "Free space not found where expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_rev_contig_free(&param.arr,
+			last_idx), FBARRAY_TEST_LEN,
+			"Free space not found where expected\n");
+
+	/* ensure we can't find any used spots */
+	TEST_ASSERT(rte_fbarray_find_next_used(&param.arr, idx) < 0,
+			"Used space found where none was expected\n");
+	TEST_ASSERT_EQUAL(rte_errno, ENOENT, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_next_n_used(&param.arr, idx, 1) < 0,
+			"Used space found where none was expected\n");
+	TEST_ASSERT_EQUAL(rte_errno, ENOENT, "Wrong errno value\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_contig_used(&param.arr, idx), 0,
+			"Used space found where none was expected\n");
+
+	TEST_ASSERT(rte_fbarray_find_prev_used(&param.arr, last_idx) < 0,
+			"Used space found where none was expected\n");
+	TEST_ASSERT_EQUAL(rte_errno, ENOENT, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_prev_n_used(&param.arr, last_idx, 1) < 0,
+			"Used space found where none was expected\n");
+	TEST_ASSERT_EQUAL(rte_errno, ENOENT, "Wrong errno value\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_rev_contig_used(&param.arr,
+			last_idx), 0,
+			"Used space found where none was expected\n");
+
+	return 0;
+}
+
+static int check_used_one(void)
+{
+	const int idx = 0;
+	const int last_idx = FBARRAY_TEST_LEN - 1;
+
+	/* check that we can find used spots now */
+	TEST_ASSERT_EQUAL(rte_fbarray_find_next_used(&param.arr, idx), idx,
+			"Used space not found where expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_next_n_used(&param.arr, idx, 1), idx,
+			"Used space not found where expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_contig_used(&param.arr, idx), 1,
+			"Used space not found where expected\n");
+
+	TEST_ASSERT_EQUAL(rte_fbarray_find_prev_used(&param.arr, last_idx), idx,
+			"Used space not found where expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_prev_n_used(&param.arr, last_idx, 1),
+			idx, "Used space not found where expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_rev_contig_used(&param.arr, idx), 1,
+			"Used space not found where expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_rev_contig_used(&param.arr,
+			last_idx), idx,
+			"Used space not found where expected\n");
+
+	/* check if further indices are still free */
+	TEST_ASSERT(rte_fbarray_find_next_used(&param.arr, idx + 1) < 0,
+			"Used space not found where none was expected\n");
+	TEST_ASSERT_EQUAL(rte_errno, ENOENT, "Wrong errno value\n");
+	TEST_ASSERT(rte_fbarray_find_next_n_used(&param.arr, idx + 1, 1) < 0,
+			"Used space not found where none was expected\n");
+	TEST_ASSERT_EQUAL(rte_errno, ENOENT, "Wrong errno value\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_contig_used(&param.arr, idx + 1), 0,
+			"Used space not found where none was expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_contig_free(&param.arr, idx + 1),
+			FBARRAY_TEST_LEN - 1,
+			"Used space not found where none was expected\n");
+
+	TEST_ASSERT_EQUAL(rte_fbarray_find_prev_used(&param.arr, last_idx), 0,
+			"Used space not found where none was expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_prev_n_used(&param.arr, last_idx, 1),
+			0, "Used space not found where none was expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_rev_contig_used(&param.arr,
+			last_idx), 0,
+			"Used space not found where none was expected\n");
+	TEST_ASSERT_EQUAL(rte_fbarray_find_rev_contig_free(&param.arr,
+			last_idx), FBARRAY_TEST_LEN - 1,
+			"Used space not found where none was expected\n");
+
+	return 0;
+}
+
+static int test_basic(void)
+{
+	const int idx = 0;
+	int i;
+
+	/* check array count */
+	TEST_ASSERT_EQUAL(param.arr.count, 0, "Wrong element count\n");
+
+	/* ensure we can find a free spot */
+	if (check_free())
+		return TEST_FAILED;
+
+	/* check if used */
+	TEST_ASSERT_EQUAL(rte_fbarray_is_used(&param.arr, idx), 0,
+			"Used space found where not expected\n");
+
+	/* mark as used */
+	TEST_ASSERT_SUCCESS(rte_fbarray_set_used(&param.arr, idx),
+			"Failed to set as used\n");
+
+	/* check if used again */
+	TEST_ASSERT_NOT_EQUAL(rte_fbarray_is_used(&param.arr, idx), 0,
+			"Used space not found where expected\n");
+
+	if (check_used_one())
+		return TEST_FAILED;
+
+	/* check array count */
+	TEST_ASSERT_EQUAL(param.arr.count, 1, "Wrong element count\n");
+
+	/* check if getting pointers works for every element */
+	for (i = 0; i < FBARRAY_TEST_LEN; i++) {
+		void *td = rte_fbarray_get(&param.arr, i);
+		TEST_ASSERT_NOT_NULL(td, "Invalid pointer returned\n");
+		TEST_ASSERT_EQUAL(rte_fbarray_find_idx(&param.arr, td), i,
+				"Wrong index returned\n");
+	}
+
+	/* mark as free */
+	TEST_ASSERT_SUCCESS(rte_fbarray_set_free(&param.arr, idx),
+			"Failed to set as free\n");
+
+	/* check array count */
+	TEST_ASSERT_EQUAL(param.arr.count, 0, "Wrong element count\n");
+
+	/* check if used */
+	TEST_ASSERT_EQUAL(rte_fbarray_is_used(&param.arr, idx), 0,
+			"Used space found where not expected\n");
+
+	if (check_free())
+		return TEST_FAILED;
+
+	reset_array();
+
+	return TEST_SUCCESS;
+}
+
+static int ensure_correct(struct rte_fbarray *arr, int first, int last,
+		bool used)
+{
+	int i, len = last - first + 1;
+	for (i = 0; i < len; i++) {
+		int cur = first + i;
+		int cur_len = len - i;
+
+		if (used) {
+			TEST_ASSERT_EQUAL(rte_fbarray_find_contig_used(arr,
+					cur), cur_len,
+					"Used space length is wrong\n");
+			TEST_ASSERT_EQUAL(rte_fbarray_find_rev_contig_used(arr,
+					last), len,
+					"Used space length is wrong\n");
+			TEST_ASSERT_EQUAL(rte_fbarray_find_rev_contig_used(arr,
+					cur), i + 1,
+					"Used space length is wrong\n");
+
+			TEST_ASSERT_EQUAL(rte_fbarray_find_next_used(arr, cur),
+					cur,
+					"Used space not found where expected\n");
+			TEST_ASSERT_EQUAL(rte_fbarray_find_next_n_used(arr,
+					cur, 1), cur,
+					"Used space not found where expected\n");
+			TEST_ASSERT_EQUAL(rte_fbarray_find_next_n_used(arr, cur,
+					cur_len), cur,
+					"Used space not found where expected\n");
+
+			TEST_ASSERT_EQUAL(rte_fbarray_find_prev_used(arr, cur),
+					cur,
+					"Used space not found where expected\n");
+			TEST_ASSERT_EQUAL(rte_fbarray_find_prev_n_used(arr,
+					last, cur_len), cur,
+					"Used space not found where expected\n");
+		} else {
+			TEST_ASSERT_EQUAL(rte_fbarray_find_contig_free(arr,
+					cur), cur_len,
+					"Free space length is wrong\n");
+			TEST_ASSERT_EQUAL(rte_fbarray_find_rev_contig_free(arr,
+					last), len,
+					"Free space length is wrong\n");
+			TEST_ASSERT_EQUAL(rte_fbarray_find_rev_contig_free(arr,
+					cur), i + 1,
+					"Free space length is wrong\n");
+
+			TEST_ASSERT_EQUAL(rte_fbarray_find_next_free(arr, cur),
+					cur,
+					"Free space not found where expected\n");
+			TEST_ASSERT_EQUAL(rte_fbarray_find_next_n_free(arr, cur,
+					1), cur,
+					"Free space not found where expected\n");
+			TEST_ASSERT_EQUAL(rte_fbarray_find_next_n_free(arr, cur,
+					cur_len), cur,
+					"Free space not found where expected\n");
+
+			TEST_ASSERT_EQUAL(rte_fbarray_find_prev_free(arr, cur),
+					cur,
+					"Free space not found where expected\n");
+			TEST_ASSERT_EQUAL(rte_fbarray_find_prev_n_free(arr,
+					last, cur_len), cur,
+					"Free space not found where expected\n");
+		}
+	}
+	return 0;
+}
+
+static int test_find(void)
+{
+	TEST_ASSERT_EQUAL((int)param.arr.count, param.end - param.start + 1,
+			"Wrong element count\n");
+	/* ensure space is free before start */
+	if (ensure_correct(&param.arr, 0, param.start - 1, false))
+		return TEST_FAILED;
+	/* ensure space is occupied where it's supposed to be */
+	if (ensure_correct(&param.arr, param.start, param.end, true))
+		return TEST_FAILED;
+	/* ensure space after end is free as well */
+	if (ensure_correct(&param.arr, param.end + 1, FBARRAY_TEST_LEN - 1,
+			false))
+		return TEST_FAILED;
+	return TEST_SUCCESS;
+}
+
+static int test_empty(void)
+{
+	TEST_ASSERT_EQUAL((int)param.arr.count, 0, "Wrong element count\n");
+	/* ensure space is free */
+	if (ensure_correct(&param.arr, 0, FBARRAY_TEST_LEN - 1, false))
+		return TEST_FAILED;
+	return TEST_SUCCESS;
+}
+
+
+static struct unit_test_suite fbarray_test_suite = {
+	.suite_name = "fbarray autotest",
+	.setup = autotest_setup,
+	.teardown = autotest_teardown,
+	.unit_test_cases = {
+		TEST_CASE(test_invalid),
+		TEST_CASE(test_basic),
+		TEST_CASE_ST(first_msk_test_setup, reset_array, test_find),
+		TEST_CASE_ST(cross_msk_test_setup, reset_array, test_find),
+		TEST_CASE_ST(multi_msk_test_setup, reset_array, test_find),
+		TEST_CASE_ST(last_msk_test_setup, reset_array, test_find),
+		TEST_CASE_ST(full_msk_test_setup, reset_array, test_find),
+		TEST_CASE_ST(empty_msk_test_setup, reset_array, test_empty),
+		TEST_CASES_END()
+	}
+};
+
+static int
+test_fbarray(void)
+{
+	return unit_test_suite_runner(&fbarray_test_suite);
+}
+
+REGISTER_TEST_COMMAND(fbarray_autotest, test_fbarray);
diff --git a/test/test/test_flow_classify.c b/test/test/test_flow_classify.c
index fc83b69a..5f5beeee 100644
--- a/test/test/test_flow_classify.c
+++ b/test/test/test_flow_classify.c
@@ -871,32 +871,32 @@ test_flow_classify(void)
 		printf("Line %i: f_create has failed!\n", __LINE__);
 		rte_flow_classifier_free(cls->cls);
 		rte_free(cls);
-		return -1;
+		return TEST_FAILED;
 	}
 	printf("Created table_acl for for IPv4 five tuple packets\n");
 
 	ret = init_mbufpool();
 	if (ret) {
 		printf("Line %i: init_mbufpool has failed!\n", __LINE__);
-		return -1;
+		return TEST_FAILED;
 	}
 
 	if (test_invalid_parameters() < 0)
-		return -1;
+		return TEST_FAILED;
 	if (test_valid_parameters() < 0)
-		return -1;
+		return TEST_FAILED;
 	if (test_invalid_patterns() < 0)
-		return -1;
+		return TEST_FAILED;
 	if (test_invalid_actions() < 0)
-		return -1;
+		return TEST_FAILED;
 	if (test_query_udp() < 0)
-		return -1;
+		return TEST_FAILED;
 	if (test_query_tcp() < 0)
-		return -1;
+		return TEST_FAILED;
 	if (test_query_sctp() < 0)
-		return -1;
+		return TEST_FAILED;
 
-	return 0;
+	return TEST_SUCCESS;
 }
 
 REGISTER_TEST_COMMAND(flow_classify_autotest, test_flow_classify);
diff --git a/test/test/test_hash.c b/test/test/test_hash.c
index edf41f5a..b3db9fd1 100644
--- a/test/test/test_hash.c
+++ b/test/test/test_hash.c
@@ -1103,6 +1103,7 @@ static int test_average_table_utilization(void)
 	unsigned i, j;
 	unsigned added_keys, average_keys_added = 0;
 	int ret;
+	unsigned int cnt;
 
 	printf("\n# Running test to determine average utilization"
 	       "\n  before adding elements begins to fail\n");
@@ -1121,13 +1122,24 @@ static int test_average_table_utilization(void)
 			for (i = 0; i < ut_params.key_len; i++)
 				simple_key[i] = rte_rand() % 255;
 			ret = rte_hash_add_key(handle, simple_key);
+			if (ret < 0)
+				break;
 		}
+
 		if (ret != -ENOSPC) {
 			printf("Unexpected error when adding keys\n");
 			rte_hash_free(handle);
 			return -1;
 		}
 
+		cnt = rte_hash_count(handle);
+		if (cnt != added_keys) {
+			printf("rte_hash_count returned wrong value %u, %u,"
+					"%u\n", j, added_keys, cnt);
+			rte_hash_free(handle);
+			return -1;
+		}
+
 		average_keys_added += added_keys;
 
 		/* Reset the table */
diff --git a/test/test/test_hash_multiwriter.c b/test/test/test_hash_multiwriter.c
index ef5fce3d..6a3eb10b 100644
--- a/test/test/test_hash_multiwriter.c
+++ b/test/test/test_hash_multiwriter.c
@@ -48,18 +48,29 @@ static rte_atomic64_t ginsertions;
 static int use_htm;
 
 static int
-test_hash_multiwriter_worker(__attribute__((unused)) void *arg)
+test_hash_multiwriter_worker(void *arg)
 {
 	uint64_t i, offset;
+	uint16_t pos_core;
 	uint32_t lcore_id = rte_lcore_id();
 	uint64_t begin, cycles;
+	uint16_t *enabled_core_ids = (uint16_t *)arg;
 
-	offset = (lcore_id - rte_get_master_lcore())
-		* tbl_multiwriter_test_params.nb_tsx_insertion;
+	for (pos_core = 0; pos_core < rte_lcore_count(); pos_core++) {
+		if (enabled_core_ids[pos_core] == lcore_id)
+			break;
+	}
+
+	/*
+	 * Calculate offset for entries based on the position of the
+	 * logical core, from the master core (not counting not enabled cores)
+	 */
+	offset = pos_core * tbl_multiwriter_test_params.nb_tsx_insertion;
 
 	printf("Core #%d inserting %d: %'"PRId64" - %'"PRId64"\n",
 	       lcore_id, tbl_multiwriter_test_params.nb_tsx_insertion,
-	       offset, offset + tbl_multiwriter_test_params.nb_tsx_insertion);
+	       offset,
+	       offset + tbl_multiwriter_test_params.nb_tsx_insertion - 1);
 
 	begin = rte_rdtsc_precise();
 
@@ -88,6 +99,8 @@ test_hash_multiwriter(void)
 {
 	unsigned int i, rounded_nb_total_tsx_insertion;
 	static unsigned calledCount = 1;
+	uint16_t enabled_core_ids[RTE_MAX_LCORE];
+	uint16_t core_id;
 
 	uint32_t *keys;
 	uint32_t *found;
@@ -116,6 +129,7 @@ test_hash_multiwriter(void)
 
 	uint32_t duplicated_keys = 0;
 	uint32_t lost_keys = 0;
+	uint32_t count;
 
 	snprintf(name, 32, "test%u", calledCount++);
 	hash_params.name = name;
@@ -140,16 +154,17 @@ test_hash_multiwriter(void)
 		goto err1;
 	}
 
+	for (i = 0; i < nb_entries; i++)
+		keys[i] = i;
+
+	tbl_multiwriter_test_params.keys = keys;
+
 	found = rte_zmalloc(NULL, sizeof(uint32_t) * nb_entries, 0);
 	if (found == NULL) {
 		printf("RTE_ZMALLOC failed\n");
 		goto err2;
 	}
 
-	for (i = 0; i < nb_entries; i++)
-		keys[i] = i;
-
-	tbl_multiwriter_test_params.keys = keys;
 	tbl_multiwriter_test_params.found = found;
 
 	rte_atomic64_init(&gcycles);
@@ -158,11 +173,36 @@ test_hash_multiwriter(void)
 	rte_atomic64_init(&ginsertions);
 	rte_atomic64_clear(&ginsertions);
 
+	/* Get list of enabled cores */
+	i = 0;
+	for (core_id = 0; core_id < RTE_MAX_LCORE; core_id++) {
+		if (i == rte_lcore_count())
+			break;
+
+		if (rte_lcore_is_enabled(core_id)) {
+			enabled_core_ids[i] = core_id;
+			i++;
+		}
+	}
+
+	if (i != rte_lcore_count()) {
+		printf("Number of enabled cores in list is different from "
+				"number given by rte_lcore_count()\n");
+		goto err3;
+	}
+
 	/* Fire all threads. */
 	rte_eal_mp_remote_launch(test_hash_multiwriter_worker,
-				 NULL, CALL_MASTER);
+				 enabled_core_ids, CALL_MASTER);
 	rte_eal_mp_wait_lcore();
 
+	count = rte_hash_count(handle);
+	if (count != rounded_nb_total_tsx_insertion) {
+		printf("rte_hash_count returned wrong value %u, %d\n",
+				rounded_nb_total_tsx_insertion, count);
+		goto err3;
+	}
+
 	while (rte_hash_iterate(handle, &next_key, &next_data, &iter) >= 0) {
 		/* Search for the key in the list of keys added .*/
 		i = *(const uint32_t *)next_key;
diff --git a/test/test/test_hash_perf.c b/test/test/test_hash_perf.c
index a81d0c79..33dcb9fc 100644
--- a/test/test/test_hash_perf.c
+++ b/test/test/test_hash_perf.c
@@ -76,7 +76,8 @@ static struct rte_hash_parameters ut_params = {
 };
 
 static int
-create_table(unsigned with_data, unsigned table_index)
+create_table(unsigned int with_data, unsigned int table_index,
+		unsigned int with_locks)
 {
 	char name[RTE_HASH_NAMESIZE];
 
@@ -86,6 +87,14 @@ create_table(unsigned with_data, unsigned table_index)
 	else
 		sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
 
+
+	if (with_locks)
+		ut_params.extra_flag =
+			RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT
+				| RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY;
+	else
+		ut_params.extra_flag = 0;
+
 	ut_params.name = name;
 	ut_params.key_len = hashtest_key_lens[table_index];
 	ut_params.socket_id = rte_socket_id();
@@ -459,7 +468,7 @@ reset_table(unsigned table_index)
 }
 
 static int
-run_all_tbl_perf_tests(unsigned with_pushes)
+run_all_tbl_perf_tests(unsigned int with_pushes, unsigned int with_locks)
 {
 	unsigned i, j, with_data, with_hash;
 
@@ -468,7 +477,7 @@ run_all_tbl_perf_tests(unsigned with_pushes)
 
 	for (with_data = 0; with_data <= 1; with_data++) {
 		for (i = 0; i < NUM_KEYSIZES; i++) {
-			if (create_table(with_data, i) < 0)
+			if (create_table(with_data, i, with_locks) < 0)
 				return -1;
 
 			if (get_input_keys(with_pushes, i) < 0)
@@ -611,15 +620,20 @@ fbk_hash_perf_test(void)
 static int
 test_hash_perf(void)
 {
-	unsigned with_pushes;
-
-	for (with_pushes = 0; with_pushes <= 1; with_pushes++) {
-		if (with_pushes == 0)
-			printf("\nALL ELEMENTS IN PRIMARY LOCATION\n");
+	unsigned int with_pushes, with_locks;
+	for (with_locks = 0; with_locks <= 1; with_locks++) {
+		if (with_locks)
+			printf("\nWith locks in the code\n");
 		else
-			printf("\nELEMENTS IN PRIMARY OR SECONDARY LOCATION\n");
-		if (run_all_tbl_perf_tests(with_pushes) < 0)
-			return -1;
+			printf("\nWithout locks in the code\n");
+		for (with_pushes = 0; with_pushes <= 1; with_pushes++) {
+			if (with_pushes == 0)
+				printf("\nALL ELEMENTS IN PRIMARY LOCATION\n");
+			else
+				printf("\nELEMENTS IN PRIMARY OR SECONDARY LOCATION\n");
+			if (run_all_tbl_perf_tests(with_pushes, with_locks) < 0)
+				return -1;
+		}
 	}
 	if (fbk_hash_perf_test() < 0)
 		return -1;
diff --git a/test/test/test_hash_readwrite.c b/test/test/test_hash_readwrite.c
new file mode 100644
index 00000000..55ae33d8
--- /dev/null
+++ b/test/test/test_hash_readwrite.c
@@ -0,0 +1,637 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <inttypes.h>
+#include <locale.h>
+
+#include <rte_cycles.h>
+#include <rte_hash.h>
+#include <rte_hash_crc.h>
+#include <rte_jhash.h>
+#include <rte_launch.h>
+#include <rte_malloc.h>
+#include <rte_random.h>
+#include <rte_spinlock.h>
+
+#include "test.h"
+
+#define RTE_RWTEST_FAIL 0
+
+#define TOTAL_ENTRY (16*1024*1024)
+#define TOTAL_INSERT (15*1024*1024)
+
+#define NUM_TEST 3
+unsigned int core_cnt[NUM_TEST] = {2, 4, 8};
+
+struct perf {
+	uint32_t single_read;
+	uint32_t single_write;
+	uint32_t read_only[NUM_TEST];
+	uint32_t write_only[NUM_TEST];
+	uint32_t read_write_r[NUM_TEST];
+	uint32_t read_write_w[NUM_TEST];
+};
+
+static struct perf htm_results, non_htm_results;
+
+struct {
+	uint32_t *keys;
+	uint32_t *found;
+	uint32_t num_insert;
+	uint32_t rounded_tot_insert;
+	struct rte_hash *h;
+} tbl_rw_test_param;
+
+static rte_atomic64_t gcycles;
+static rte_atomic64_t ginsertions;
+
+static rte_atomic64_t gread_cycles;
+static rte_atomic64_t gwrite_cycles;
+
+static rte_atomic64_t greads;
+static rte_atomic64_t gwrites;
+
+static int
+test_hash_readwrite_worker(__attribute__((unused)) void *arg)
+{
+	uint64_t i, offset;
+	uint32_t lcore_id = rte_lcore_id();
+	uint64_t begin, cycles;
+	int ret;
+
+	offset = (lcore_id - rte_get_master_lcore())
+			* tbl_rw_test_param.num_insert;
+
+	printf("Core #%d inserting and reading %d: %'"PRId64" - %'"PRId64"\n",
+	       lcore_id, tbl_rw_test_param.num_insert,
+	       offset, offset + tbl_rw_test_param.num_insert);
+
+	begin = rte_rdtsc_precise();
+
+	for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
+
+		if (rte_hash_lookup(tbl_rw_test_param.h,
+				tbl_rw_test_param.keys + i) > 0)
+			break;
+
+		ret = rte_hash_add_key(tbl_rw_test_param.h,
+				     tbl_rw_test_param.keys + i);
+		if (ret < 0)
+			break;
+
+		if (rte_hash_lookup(tbl_rw_test_param.h,
+				tbl_rw_test_param.keys + i) != ret)
+			break;
+	}
+
+	cycles = rte_rdtsc_precise() - begin;
+	rte_atomic64_add(&gcycles, cycles);
+	rte_atomic64_add(&ginsertions, i - offset);
+
+	for (; i < offset + tbl_rw_test_param.num_insert; i++)
+		tbl_rw_test_param.keys[i] = RTE_RWTEST_FAIL;
+
+	return 0;
+}
+
+static int
+init_params(int use_htm, int use_jhash)
+{
+	unsigned int i;
+
+	uint32_t *keys = NULL;
+	uint32_t *found = NULL;
+	struct rte_hash *handle;
+
+	struct rte_hash_parameters hash_params = {
+		.entries = TOTAL_ENTRY,
+		.key_len = sizeof(uint32_t),
+		.hash_func_init_val = 0,
+		.socket_id = rte_socket_id(),
+	};
+	if (use_jhash)
+		hash_params.hash_func = rte_jhash;
+	else
+		hash_params.hash_func = rte_hash_crc;
+
+	if (use_htm)
+		hash_params.extra_flag =
+			RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
+			RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY;
+	else
+		hash_params.extra_flag =
+			RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY;
+
+	hash_params.name = "tests";
+
+	handle = rte_hash_create(&hash_params);
+	if (handle == NULL) {
+		printf("hash creation failed");
+		return -1;
+	}
+
+	tbl_rw_test_param.h = handle;
+	keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
+
+	if (keys == NULL) {
+		printf("RTE_MALLOC failed\n");
+		goto err;
+	}
+
+	found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
+	if (found == NULL) {
+		printf("RTE_ZMALLOC failed\n");
+		goto err;
+	}
+
+	tbl_rw_test_param.keys = keys;
+	tbl_rw_test_param.found = found;
+
+	for (i = 0; i < TOTAL_ENTRY; i++)
+		keys[i] = i;
+
+	return 0;
+
+err:
+	rte_free(keys);
+	rte_hash_free(handle);
+
+	return -1;
+}
+
+static int
+test_hash_readwrite_functional(int use_htm)
+{
+	unsigned int i;
+	const void *next_key;
+	void *next_data;
+	uint32_t iter = 0;
+
+	uint32_t duplicated_keys = 0;
+	uint32_t lost_keys = 0;
+	int use_jhash = 1;
+
+	rte_atomic64_init(&gcycles);
+	rte_atomic64_clear(&gcycles);
+
+	rte_atomic64_init(&ginsertions);
+	rte_atomic64_clear(&ginsertions);
+
+	if (init_params(use_htm, use_jhash) != 0)
+		goto err;
+
+	tbl_rw_test_param.num_insert =
+		TOTAL_INSERT / rte_lcore_count();
+
+	tbl_rw_test_param.rounded_tot_insert =
+		tbl_rw_test_param.num_insert
+		* rte_lcore_count();
+
+	printf("++++++++Start function tests:+++++++++\n");
+
+	/* Fire all threads. */
+	rte_eal_mp_remote_launch(test_hash_readwrite_worker,
+				 NULL, CALL_MASTER);
+	rte_eal_mp_wait_lcore();
+
+	while (rte_hash_iterate(tbl_rw_test_param.h, &next_key,
+			&next_data, &iter) >= 0) {
+		/* Search for the key in the list of keys added .*/
+		i = *(const uint32_t *)next_key;
+		tbl_rw_test_param.found[i]++;
+	}
+
+	for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
+		if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
+			if (tbl_rw_test_param.found[i] > 1) {
+				duplicated_keys++;
+				break;
+			}
+			if (tbl_rw_test_param.found[i] == 0) {
+				lost_keys++;
+				printf("key %d is lost\n", i);
+				break;
+			}
+		}
+	}
+
+	if (duplicated_keys > 0) {
+		printf("%d key duplicated\n", duplicated_keys);
+		goto err_free;
+	}
+
+	if (lost_keys > 0) {
+		printf("%d key lost\n", lost_keys);
+		goto err_free;
+	}
+
+	printf("No key corrupted during read-write test.\n");
+
+	unsigned long long int cycles_per_insertion =
+		rte_atomic64_read(&gcycles) /
+		rte_atomic64_read(&ginsertions);
+
+	printf("cycles per insertion and lookup: %llu\n", cycles_per_insertion);
+
+	rte_free(tbl_rw_test_param.found);
+	rte_free(tbl_rw_test_param.keys);
+	rte_hash_free(tbl_rw_test_param.h);
+	printf("+++++++++Complete function tests+++++++++\n");
+	return 0;
+
+err_free:
+	rte_free(tbl_rw_test_param.found);
+	rte_free(tbl_rw_test_param.keys);
+	rte_hash_free(tbl_rw_test_param.h);
+err:
+	return -1;
+}
+
+static int
+test_rw_reader(__attribute__((unused)) void *arg)
+{
+	uint64_t i;
+	uint64_t begin, cycles;
+	uint64_t read_cnt = (uint64_t)((uintptr_t)arg);
+
+	begin = rte_rdtsc_precise();
+	for (i = 0; i < read_cnt; i++) {
+		void *data;
+		rte_hash_lookup_data(tbl_rw_test_param.h,
+				tbl_rw_test_param.keys + i,
+				&data);
+		if (i != (uint64_t)(uintptr_t)data) {
+			printf("lookup find wrong value %"PRIu64","
+				"%"PRIu64"\n", i,
+				(uint64_t)(uintptr_t)data);
+			break;
+		}
+	}
+
+	cycles = rte_rdtsc_precise() - begin;
+	rte_atomic64_add(&gread_cycles, cycles);
+	rte_atomic64_add(&greads, i);
+	return 0;
+}
+
+static int
+test_rw_writer(__attribute__((unused)) void *arg)
+{
+	uint64_t i;
+	uint32_t lcore_id = rte_lcore_id();
+	uint64_t begin, cycles;
+	int ret;
+	uint64_t start_coreid = (uint64_t)(uintptr_t)arg;
+	uint64_t offset;
+
+	offset = TOTAL_INSERT / 2 + (lcore_id - start_coreid)
+					* tbl_rw_test_param.num_insert;
+	begin = rte_rdtsc_precise();
+	for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
+		ret = rte_hash_add_key_data(tbl_rw_test_param.h,
+				tbl_rw_test_param.keys + i,
+				(void *)((uintptr_t)i));
+		if (ret < 0) {
+			printf("writer failed %"PRIu64"\n", i);
+			break;
+		}
+	}
+
+	cycles = rte_rdtsc_precise() - begin;
+	rte_atomic64_add(&gwrite_cycles, cycles);
+	rte_atomic64_add(&gwrites, tbl_rw_test_param.num_insert);
+	return 0;
+}
+
+static int
+test_hash_readwrite_perf(struct perf *perf_results, int use_htm,
+							int reader_faster)
+{
+	unsigned int n;
+	int ret;
+	int start_coreid;
+	uint64_t i, read_cnt;
+
+	const void *next_key;
+	void *next_data;
+	uint32_t iter = 0;
+	int use_jhash = 0;
+
+	uint32_t duplicated_keys = 0;
+	uint32_t lost_keys = 0;
+
+	uint64_t start = 0, end = 0;
+
+	rte_atomic64_init(&greads);
+	rte_atomic64_init(&gwrites);
+	rte_atomic64_clear(&gwrites);
+	rte_atomic64_clear(&greads);
+
+	rte_atomic64_init(&gread_cycles);
+	rte_atomic64_clear(&gread_cycles);
+	rte_atomic64_init(&gwrite_cycles);
+	rte_atomic64_clear(&gwrite_cycles);
+
+	if (init_params(use_htm, use_jhash) != 0)
+		goto err;
+
+	/*
+	 * Do a readers finish faster or writers finish faster test.
+	 * When readers finish faster, we timing the readers, and when writers
+	 * finish faster, we timing the writers.
+	 * Divided by 10 or 2 is just experimental values to vary the workload
+	 * of readers.
+	 */
+	if (reader_faster) {
+		printf("++++++Start perf test: reader++++++++\n");
+		read_cnt = TOTAL_INSERT / 10;
+	} else {
+		printf("++++++Start perf test: writer++++++++\n");
+		read_cnt = TOTAL_INSERT / 2;
+	}
+
+	/* We first test single thread performance */
+	start = rte_rdtsc_precise();
+	/* Insert half of the keys */
+	for (i = 0; i < TOTAL_INSERT / 2; i++) {
+		ret = rte_hash_add_key_data(tbl_rw_test_param.h,
+				     tbl_rw_test_param.keys + i,
+					(void *)((uintptr_t)i));
+		if (ret < 0) {
+			printf("Failed to insert half of keys\n");
+			goto err_free;
+		}
+	}
+	end = rte_rdtsc_precise() - start;
+	perf_results->single_write = end / i;
+
+	start = rte_rdtsc_precise();
+
+	for (i = 0; i < read_cnt; i++) {
+		void *data;
+		rte_hash_lookup_data(tbl_rw_test_param.h,
+				tbl_rw_test_param.keys + i,
+				&data);
+		if (i != (uint64_t)(uintptr_t)data) {
+			printf("lookup find wrong value"
+					" %"PRIu64",%"PRIu64"\n", i,
+					(uint64_t)(uintptr_t)data);
+			break;
+		}
+	}
+	end = rte_rdtsc_precise() - start;
+	perf_results->single_read = end / i;
+
+	for (n = 0; n < NUM_TEST; n++) {
+		unsigned int tot_lcore = rte_lcore_count();
+		if (tot_lcore < core_cnt[n] * 2 + 1)
+			goto finish;
+
+		rte_atomic64_clear(&greads);
+		rte_atomic64_clear(&gread_cycles);
+		rte_atomic64_clear(&gwrites);
+		rte_atomic64_clear(&gwrite_cycles);
+
+		rte_hash_reset(tbl_rw_test_param.h);
+
+		tbl_rw_test_param.num_insert = TOTAL_INSERT / 2 / core_cnt[n];
+		tbl_rw_test_param.rounded_tot_insert = TOTAL_INSERT / 2 +
+						tbl_rw_test_param.num_insert *
+						core_cnt[n];
+
+		for (i = 0; i < TOTAL_INSERT / 2; i++) {
+			ret = rte_hash_add_key_data(tbl_rw_test_param.h,
+					tbl_rw_test_param.keys + i,
+					(void *)((uintptr_t)i));
+			if (ret < 0) {
+				printf("Failed to insert half of keys\n");
+				goto err_free;
+			}
+		}
+
+		/* Then test multiple thread case but only all reads or
+		 * all writes
+		 */
+
+		/* Test only reader cases */
+		for (i = 1; i <= core_cnt[n]; i++)
+			rte_eal_remote_launch(test_rw_reader,
+					(void *)(uintptr_t)read_cnt, i);
+
+		rte_eal_mp_wait_lcore();
+
+		start_coreid = i;
+		/* Test only writer cases */
+		for (; i <= core_cnt[n] * 2; i++)
+			rte_eal_remote_launch(test_rw_writer,
+					(void *)((uintptr_t)start_coreid), i);
+
+		rte_eal_mp_wait_lcore();
+
+		if (reader_faster) {
+			unsigned long long int cycles_per_insertion =
+				rte_atomic64_read(&gread_cycles) /
+				rte_atomic64_read(&greads);
+			perf_results->read_only[n] = cycles_per_insertion;
+			printf("Reader only: cycles per lookup: %llu\n",
+							cycles_per_insertion);
+		}
+
+		else {
+			unsigned long long int cycles_per_insertion =
+				rte_atomic64_read(&gwrite_cycles) /
+				rte_atomic64_read(&gwrites);
+			perf_results->write_only[n] = cycles_per_insertion;
+			printf("Writer only: cycles per writes: %llu\n",
+							cycles_per_insertion);
+		}
+
+		rte_atomic64_clear(&greads);
+		rte_atomic64_clear(&gread_cycles);
+		rte_atomic64_clear(&gwrites);
+		rte_atomic64_clear(&gwrite_cycles);
+
+		rte_hash_reset(tbl_rw_test_param.h);
+
+		for (i = 0; i < TOTAL_INSERT / 2; i++) {
+			ret = rte_hash_add_key_data(tbl_rw_test_param.h,
+					tbl_rw_test_param.keys + i,
+					(void *)((uintptr_t)i));
+			if (ret < 0) {
+				printf("Failed to insert half of keys\n");
+				goto err_free;
+			}
+		}
+
+		start_coreid = core_cnt[n] + 1;
+
+		if (reader_faster) {
+			for (i = core_cnt[n] + 1; i <= core_cnt[n] * 2; i++)
+				rte_eal_remote_launch(test_rw_writer,
+					(void *)((uintptr_t)start_coreid), i);
+			for (i = 1; i <= core_cnt[n]; i++)
+				rte_eal_remote_launch(test_rw_reader,
+					(void *)(uintptr_t)read_cnt, i);
+		} else {
+			for (i = 1; i <= core_cnt[n]; i++)
+				rte_eal_remote_launch(test_rw_reader,
+					(void *)(uintptr_t)read_cnt, i);
+			for (; i <= core_cnt[n] * 2; i++)
+				rte_eal_remote_launch(test_rw_writer,
+					(void *)((uintptr_t)start_coreid), i);
+		}
+
+		rte_eal_mp_wait_lcore();
+
+		while (rte_hash_iterate(tbl_rw_test_param.h,
+				&next_key, &next_data, &iter) >= 0) {
+			/* Search for the key in the list of keys added .*/
+			i = *(const uint32_t *)next_key;
+			tbl_rw_test_param.found[i]++;
+		}
+
+		for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
+			if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
+				if (tbl_rw_test_param.found[i] > 1) {
+					duplicated_keys++;
+					break;
+				}
+				if (tbl_rw_test_param.found[i] == 0) {
+					lost_keys++;
+					printf("key %"PRIu64" is lost\n", i);
+					break;
+				}
+			}
+		}
+
+		if (duplicated_keys > 0) {
+			printf("%d key duplicated\n", duplicated_keys);
+			goto err_free;
+		}
+
+		if (lost_keys > 0) {
+			printf("%d key lost\n", lost_keys);
+			goto err_free;
+		}
+
+		printf("No key corrupted during read-write test.\n");
+
+		if (reader_faster) {
+			unsigned long long int cycles_per_insertion =
+				rte_atomic64_read(&gread_cycles) /
+				rte_atomic64_read(&greads);
+			perf_results->read_write_r[n] = cycles_per_insertion;
+			printf("Read-write cycles per lookup: %llu\n",
+							cycles_per_insertion);
+		}
+
+		else {
+			unsigned long long int cycles_per_insertion =
+				rte_atomic64_read(&gwrite_cycles) /
+				rte_atomic64_read(&gwrites);
+			perf_results->read_write_w[n] = cycles_per_insertion;
+			printf("Read-write cycles per writes: %llu\n",
+							cycles_per_insertion);
+		}
+	}
+
+finish:
+	rte_free(tbl_rw_test_param.found);
+	rte_free(tbl_rw_test_param.keys);
+	rte_hash_free(tbl_rw_test_param.h);
+	return 0;
+
+err_free:
+	rte_free(tbl_rw_test_param.found);
+	rte_free(tbl_rw_test_param.keys);
+	rte_hash_free(tbl_rw_test_param.h);
+
+err:
+	return -1;
+}
+
+static int
+test_hash_readwrite_main(void)
+{
+	/*
+	 * Variables used to choose different tests.
+	 * use_htm indicates if hardware transactional memory should be used.
+	 * reader_faster indicates if the reader threads should finish earlier
+	 * than writer threads. This is to timing either reader threads or
+	 * writer threads for performance numbers.
+	 */
+	int use_htm, reader_faster;
+
+	if (rte_lcore_count() == 1) {
+		printf("More than one lcore is required "
+			"to do read write test\n");
+		return 0;
+	}
+
+	setlocale(LC_NUMERIC, "");
+
+	if (rte_tm_supported()) {
+		printf("Hardware transactional memory (lock elision) "
+			"is supported\n");
+
+		printf("Test read-write with Hardware transactional memory\n");
+
+		use_htm = 1;
+		if (test_hash_readwrite_functional(use_htm) < 0)
+			return -1;
+
+		reader_faster = 1;
+		if (test_hash_readwrite_perf(&htm_results, use_htm,
+							reader_faster) < 0)
+			return -1;
+
+		reader_faster = 0;
+		if (test_hash_readwrite_perf(&htm_results, use_htm,
+							reader_faster) < 0)
+			return -1;
+	} else {
+		printf("Hardware transactional memory (lock elision) "
+			"is NOT supported\n");
+	}
+
+	printf("Test read-write without Hardware transactional memory\n");
+	use_htm = 0;
+	if (test_hash_readwrite_functional(use_htm) < 0)
+		return -1;
+	reader_faster = 1;
+	if (test_hash_readwrite_perf(&non_htm_results, use_htm,
+							reader_faster) < 0)
+		return -1;
+	reader_faster = 0;
+	if (test_hash_readwrite_perf(&non_htm_results, use_htm,
+							reader_faster) < 0)
+		return -1;
+
+	printf("Results summary:\n");
+
+	int i;
+
+	printf("single read: %u\n", htm_results.single_read);
+	printf("single write: %u\n", htm_results.single_write);
+	for (i = 0; i < NUM_TEST; i++) {
+		printf("core_cnt: %u\n", core_cnt[i]);
+		printf("HTM:\n");
+		printf("read only: %u\n", htm_results.read_only[i]);
+		printf("write only: %u\n", htm_results.write_only[i]);
+		printf("read-write read: %u\n", htm_results.read_write_r[i]);
+		printf("read-write write: %u\n", htm_results.read_write_w[i]);
+
+		printf("non HTM:\n");
+		printf("read only: %u\n", non_htm_results.read_only[i]);
+		printf("write only: %u\n", non_htm_results.write_only[i]);
+		printf("read-write read: %u\n",
+			non_htm_results.read_write_r[i]);
+		printf("read-write write: %u\n",
+			non_htm_results.read_write_w[i]);
+	}
+
+	return 0;
+}
+
+REGISTER_TEST_COMMAND(hash_readwrite_autotest, test_hash_readwrite_main);
diff --git a/test/test/test_kni.c b/test/test/test_kni.c
index 56773c8a..1b876719 100644
--- a/test/test/test_kni.c
+++ b/test/test/test_kni.c
@@ -71,11 +71,7 @@ static const struct rte_eth_txconf tx_conf = {
 
 static const struct rte_eth_conf port_conf = {
 	.rxmode = {
-		.header_split = 0,
-		.hw_ip_checksum = 0,
-		.hw_vlan_filter = 0,
-		.jumbo_frame = 0,
-		.hw_strip_crc = 1,
+		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_DCB_NONE,
diff --git a/test/test/test_link_bonding.c b/test/test/test_link_bonding.c
index 0ffd6509..0fe1d78e 100644
--- a/test/test/test_link_bonding.c
+++ b/test/test/test_link_bonding.c
@@ -43,9 +43,6 @@
 #define TX_HTHRESH 0
 #define TX_WTHRESH 0
 #define TX_RSBIT_THRESH 32
-#define TX_Q_FLAGS (ETH_TXQ_FLAGS_NOMULTSEGS | ETH_TXQ_FLAGS_NOVLANOFFL |\
-	ETH_TXQ_FLAGS_NOXSUMSCTP | ETH_TXQ_FLAGS_NOXSUMUDP | \
-	ETH_TXQ_FLAGS_NOXSUMTCP)
 
 #define MBUF_CACHE_SIZE (250)
 #define BURST_SIZE (32)
@@ -135,35 +132,11 @@ static uint16_t dst_port_1 = 2024;
 
 static uint16_t vlan_id = 0x100;
 
-struct rte_eth_rxmode rx_mode = {
-	.max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */
-	.split_hdr_size = 0,
-	.header_split   = 0, /**< Header Split disabled. */
-	.hw_ip_checksum = 0, /**< IP checksum offload disabled. */
-	.hw_vlan_filter = 1, /**< VLAN filtering enabled. */
-	.hw_vlan_strip  = 1, /**< VLAN strip enabled. */
-	.hw_vlan_extend = 0, /**< Extended VLAN disabled. */
-	.jumbo_frame    = 0, /**< Jumbo Frame Support disabled. */
-	.hw_strip_crc   = 1, /**< CRC stripping by hardware enabled. */
-};
-
-struct rte_fdir_conf fdir_conf = {
-	.mode = RTE_FDIR_MODE_NONE,
-	.pballoc = RTE_FDIR_PBALLOC_64K,
-	.status = RTE_FDIR_REPORT_STATUS,
-	.drop_queue = 127,
-};
-
 static struct rte_eth_conf default_pmd_conf = {
 	.rxmode = {
 		.mq_mode = ETH_MQ_RX_NONE,
-		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.header_split   = 0, /**< Header Split disabled */
-		.hw_ip_checksum = 0, /**< IP checksum offload enabled */
-		.hw_vlan_filter = 0, /**< VLAN filtering disabled */
-		.jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
-		.hw_strip_crc   = 1, /**< CRC stripped by hardware */
+		.max_rx_pkt_len = ETHER_MAX_LEN,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
@@ -189,8 +162,6 @@ static struct rte_eth_txconf tx_conf_default = {
 	},
 	.tx_free_thresh = TX_FREE_THRESH,
 	.tx_rs_thresh = TX_RSBIT_THRESH,
-	.txq_flags = TX_Q_FLAGS
-
 };
 
 static void free_virtualpmd_tx_queue(void);
diff --git a/test/test/test_link_bonding_mode4.c b/test/test/test_link_bonding_mode4.c
index 33c1fab0..9163f631 100644
--- a/test/test/test_link_bonding_mode4.c
+++ b/test/test/test_link_bonding_mode4.c
@@ -110,11 +110,7 @@ static struct rte_eth_conf default_pmd_conf = {
 		.mq_mode = ETH_MQ_RX_NONE,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.header_split   = 0, /**< Header Split disabled */
-		.hw_ip_checksum = 0, /**< IP checksum offload enabled */
-		.hw_vlan_filter = 0, /**< VLAN filtering disabled */
-		.jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
-		.hw_strip_crc   = 1, /**< CRC stripped by hardware */
+		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/test/test/test_link_bonding_rssconf.c b/test/test/test_link_bonding_rssconf.c
index 6a1a28d2..d82de2ce 100644
--- a/test/test/test_link_bonding_rssconf.c
+++ b/test/test/test_link_bonding_rssconf.c
@@ -83,11 +83,6 @@ static struct rte_eth_conf default_pmd_conf = {
 		.mq_mode = ETH_MQ_RX_NONE,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.header_split   = 0, /**< Header Split disabled */
-		.hw_ip_checksum = 0, /**< IP checksum offload enabled */
-		.hw_vlan_filter = 0, /**< VLAN filtering disabled */
-		.jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
-		.hw_strip_crc   = 1, /**< CRC stripped by hardware */
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
@@ -100,11 +95,6 @@ static struct rte_eth_conf rss_pmd_conf = {
 		.mq_mode = ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.header_split   = 0, /**< Header Split disabled */
-		.hw_ip_checksum = 0, /**< IP checksum offload enabled */
-		.hw_vlan_filter = 0, /**< VLAN filtering disabled */
-		.jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
-		.hw_strip_crc   = 1, /**< CRC stripped by hardware */
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/test/test/test_memzone.c b/test/test/test_memzone.c
index efcf7327..452d7cc5 100644
--- a/test/test/test_memzone.c
+++ b/test/test/test_memzone.c
@@ -467,61 +467,69 @@ test_memzone_reserve_flags(void)
 
 /* Find the heap with the greatest free block size */
 static size_t
-find_max_block_free_size(const unsigned _align)
+find_max_block_free_size(unsigned int align, unsigned int socket_id)
 {
 	struct rte_malloc_socket_stats stats;
-	unsigned i, align = _align;
-	size_t len = 0;
+	size_t len, overhead;
 
-	for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
-		rte_malloc_get_socket_stats(i, &stats);
-		if (stats.greatest_free_size > len)
-			len = stats.greatest_free_size;
-	}
+	rte_malloc_get_socket_stats(socket_id, &stats);
+
+	len = stats.greatest_free_size;
+	overhead = MALLOC_ELEM_OVERHEAD;
+
+	if (len == 0)
+		return 0;
 
-	if (align < RTE_CACHE_LINE_SIZE)
-		align = RTE_CACHE_LINE_ROUNDUP(align+1);
+	align = RTE_CACHE_LINE_ROUNDUP(align);
+	overhead += align;
 
-	if (len <= MALLOC_ELEM_OVERHEAD + align)
+	if (len < overhead)
 		return 0;
 
-	return len - MALLOC_ELEM_OVERHEAD - align;
+	return len - overhead;
 }
 
 static int
 test_memzone_reserve_max(void)
 {
-	const struct rte_memzone *mz;
-	size_t maxlen;
+	unsigned int i;
 
-	maxlen = find_max_block_free_size(0);
+	for (i = 0; i < rte_socket_count(); i++) {
+		const struct rte_memzone *mz;
+		size_t maxlen;
+		int socket;
 
-	if (maxlen == 0) {
-		printf("There is no space left!\n");
-		return 0;
-	}
+		socket = rte_socket_id_by_idx(i);
+		maxlen = find_max_block_free_size(0, socket);
 
-	mz = rte_memzone_reserve(TEST_MEMZONE_NAME("max_zone"), 0,
-			SOCKET_ID_ANY, 0);
-	if (mz == NULL){
-		printf("Failed to reserve a big chunk of memory - %s\n",
-				rte_strerror(rte_errno));
-		rte_dump_physmem_layout(stdout);
-		rte_memzone_dump(stdout);
-		return -1;
-	}
+		if (maxlen == 0) {
+			printf("There is no space left!\n");
+			return 0;
+		}
 
-	if (mz->len != maxlen) {
-		printf("Memzone reserve with 0 size did not return bigest block\n");
-		printf("Expected size = %zu, actual size = %zu\n", maxlen, mz->len);
-		rte_dump_physmem_layout(stdout);
-		rte_memzone_dump(stdout);
-		return -1;
-	}
+		mz = rte_memzone_reserve(TEST_MEMZONE_NAME("max_zone"), 0,
+				socket, 0);
+		if (mz == NULL) {
+			printf("Failed to reserve a big chunk of memory - %s\n",
+					rte_strerror(rte_errno));
+			rte_dump_physmem_layout(stdout);
+			rte_memzone_dump(stdout);
+			return -1;
+		}
 
-	if (rte_memzone_free(mz)) {
-		printf("Fail memzone free\n");
-		return -1;
+		if (mz->len != maxlen) {
+			printf("Memzone reserve with 0 size did not return bigest block\n");
+			printf("Expected size = %zu, actual size = %zu\n",
+					maxlen, mz->len);
+			rte_dump_physmem_layout(stdout);
+			rte_memzone_dump(stdout);
+			return -1;
+		}
+
+		if (rte_memzone_free(mz)) {
+			printf("Fail memzone free\n");
+			return -1;
+		}
 	}
 
 	return 0;
@@ -530,45 +538,62 @@ test_memzone_reserve_max(void)
 static int
 test_memzone_reserve_max_aligned(void)
 {
-	const struct rte_memzone *mz;
-	size_t maxlen = 0;
+	unsigned int i;
 
-	/* random alignment */
-	rte_srand((unsigned)rte_rdtsc());
-	const unsigned align = 1 << ((rte_rand() % 8) + 5); /* from 128 up to 4k alignment */
+	for (i = 0; i < rte_socket_count(); i++) {
+		const struct rte_memzone *mz;
+		size_t maxlen, minlen = 0;
+		int socket;
 
-	maxlen = find_max_block_free_size(align);
+		socket = rte_socket_id_by_idx(i);
 
-	if (maxlen == 0) {
-		printf("There is no space left for biggest %u-aligned memzone!\n", align);
-		return 0;
-	}
+		/* random alignment */
+		rte_srand((unsigned int)rte_rdtsc());
+		const unsigned int align = 1 << ((rte_rand() % 8) + 5); /* from 128 up to 4k alignment */
 
-	mz = rte_memzone_reserve_aligned(TEST_MEMZONE_NAME("max_zone_aligned"),
-			0, SOCKET_ID_ANY, 0, align);
-	if (mz == NULL){
-		printf("Failed to reserve a big chunk of memory - %s\n",
-				rte_strerror(rte_errno));
-		rte_dump_physmem_layout(stdout);
-		rte_memzone_dump(stdout);
-		return -1;
-	}
+		/* memzone size may be between size and size - align */
+		minlen = find_max_block_free_size(align, socket);
+		maxlen = find_max_block_free_size(0, socket);
 
-	if (mz->len != maxlen) {
-		printf("Memzone reserve with 0 size and alignment %u did not return"
-				" bigest block\n", align);
-		printf("Expected size = %zu, actual size = %zu\n",
-				maxlen, mz->len);
-		rte_dump_physmem_layout(stdout);
-		rte_memzone_dump(stdout);
-		return -1;
-	}
+		if (minlen == 0 || maxlen == 0) {
+			printf("There is no space left for biggest %u-aligned memzone!\n",
+					align);
+			return 0;
+		}
 
-	if (rte_memzone_free(mz)) {
-		printf("Fail memzone free\n");
-		return -1;
-	}
+		mz = rte_memzone_reserve_aligned(
+				TEST_MEMZONE_NAME("max_zone_aligned"),
+				0, socket, 0, align);
+		if (mz == NULL) {
+			printf("Failed to reserve a big chunk of memory - %s\n",
+					rte_strerror(rte_errno));
+			rte_dump_physmem_layout(stdout);
+			rte_memzone_dump(stdout);
+			return -1;
+		}
+		if (mz->addr != RTE_PTR_ALIGN(mz->addr, align)) {
+			printf("Memzone reserve with 0 size and alignment %u did not return aligned block\n",
+					align);
+			rte_dump_physmem_layout(stdout);
+			rte_memzone_dump(stdout);
+			return -1;
+		}
 
+		if (mz->len < minlen || mz->len > maxlen) {
+			printf("Memzone reserve with 0 size and alignment %u did not return"
+					" bigest block\n", align);
+			printf("Expected size = %zu-%zu, actual size = %zu\n",
+					minlen, maxlen, mz->len);
+			rte_dump_physmem_layout(stdout);
+			rte_memzone_dump(stdout);
+			return -1;
+		}
+
+		if (rte_memzone_free(mz)) {
+			printf("Fail memzone free\n");
+			return -1;
+		}
+	}
 	return 0;
 }
 
diff --git a/test/test/test_mp_secondary.c b/test/test/test_mp_secondary.c
index cc46cf4d..b597dfcd 100644
--- a/test/test/test_mp_secondary.c
+++ b/test/test/test_mp_secondary.c
@@ -50,32 +50,6 @@
 #define launch_proc(ARGV) process_dup(ARGV, \
 		sizeof(ARGV)/(sizeof(ARGV[0])), __func__)
 
-#ifdef RTE_EXEC_ENV_LINUXAPP
-static char*
-get_current_prefix(char * prefix, int size)
-{
-	char path[PATH_MAX] = {0};
-	char buf[PATH_MAX] = {0};
-
-	/* get file for config (fd is always 3) */
-	snprintf(path, sizeof(path), "/proc/self/fd/%d", 3);
-
-	/* return NULL on error */
-	if (readlink(path, buf, sizeof(buf)) == -1)
-		return NULL;
-
-	/* get the basename */
-	snprintf(buf, sizeof(buf), "%s", basename(buf));
-
-	/* copy string all the way from second char up to start of _config */
-	snprintf(prefix, size, "%.*s",
-			(int)(strnlen(buf, sizeof(buf)) - sizeof("_config")),
-			&buf[1]);
-
-	return prefix;
-}
-#endif
-
 /*
  * This function is called in the primary i.e. main test, to spawn off secondary
  * processes to run actual mp tests. Uses fork() and exec pair
diff --git a/test/test/test_pmd_perf.c b/test/test/test_pmd_perf.c
index 54bc4f6b..4549965f 100644
--- a/test/test/test_pmd_perf.c
+++ b/test/test/test_pmd_perf.c
@@ -65,14 +65,7 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode = ETH_MQ_RX_NONE,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.header_split   = 0, /**< Header Split disabled */
-		.hw_ip_checksum = 0, /**< IP checksum offload enabled */
-		.hw_vlan_filter = 0, /**< VLAN filtering disabled */
-		.hw_vlan_strip  = 0, /**< VLAN strip enabled. */
-		.hw_vlan_extend = 0, /**< Extended VLAN disabled. */
-		.jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
-		.hw_strip_crc   = 1, /**< CRC stripped by hardware */
-		.enable_scatter = 0, /**< scatter rx disabled */
+		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
@@ -97,11 +90,6 @@ static struct rte_eth_txconf tx_conf = {
 	},
 	.tx_free_thresh = 32, /* Use PMD default values */
 	.tx_rs_thresh = 32, /* Use PMD default values */
-	.txq_flags = (ETH_TXQ_FLAGS_NOMULTSEGS |
-		      ETH_TXQ_FLAGS_NOVLANOFFL |
-		      ETH_TXQ_FLAGS_NOXSUMSCTP |
-		      ETH_TXQ_FLAGS_NOXSUMUDP |
-		      ETH_TXQ_FLAGS_NOXSUMTCP)
 };
 
 enum {
@@ -808,38 +796,29 @@ test_set_rxtx_conf(cmdline_fixed_string_t mode)
 
 	if (!strcmp(mode, "vector")) {
 		/* vector rx, tx */
-		tx_conf.txq_flags = 0xf01;
 		tx_conf.tx_rs_thresh = 32;
 		tx_conf.tx_free_thresh = 32;
-		port_conf.rxmode.hw_ip_checksum = 0;
-		port_conf.rxmode.enable_scatter = 0;
 		return 0;
 	} else if (!strcmp(mode, "scalar")) {
 		/* bulk alloc rx, full-featured tx */
-		tx_conf.txq_flags = 0;
 		tx_conf.tx_rs_thresh = 32;
 		tx_conf.tx_free_thresh = 32;
-		port_conf.rxmode.hw_ip_checksum = 1;
-		port_conf.rxmode.enable_scatter = 0;
+		port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM;
 		return 0;
 	} else if (!strcmp(mode, "hybrid")) {
 		/* bulk alloc rx, vector tx
 		 * when vec macro not define,
 		 * using the same rx/tx as scalar
 		 */
-		tx_conf.txq_flags = 0xf01;
 		tx_conf.tx_rs_thresh = 32;
 		tx_conf.tx_free_thresh = 32;
-		port_conf.rxmode.hw_ip_checksum = 1;
-		port_conf.rxmode.enable_scatter = 0;
+		port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM;
 		return 0;
 	} else if (!strcmp(mode, "full")) {
 		/* full feature rx,tx pair */
-		tx_conf.txq_flags = 0x0;   /* must condition */
 		tx_conf.tx_rs_thresh = 32;
 		tx_conf.tx_free_thresh = 32;
-		port_conf.rxmode.hw_ip_checksum = 0;
-		port_conf.rxmode.enable_scatter = 1; /* must condition */
+		port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SCATTER;
 		return 0;
 	}
 
diff --git a/test/test/test_pmd_ring.c b/test/test/test_pmd_ring.c
index 21962012..19d7d20a 100644
--- a/test/test/test_pmd_ring.c
+++ b/test/test/test_pmd_ring.c
@@ -218,6 +218,8 @@ test_pmd_ring_pair_create_attach(int portd, int porte)
 	struct rte_mbuf buf, *pbuf = &buf;
 	struct rte_eth_conf null_conf;
 
+	memset(&null_conf, 0, sizeof(struct rte_eth_conf));
+
 	if ((rte_eth_dev_configure(portd, 1, 1, &null_conf) < 0)
 		|| (rte_eth_dev_configure(porte, 1, 1, &null_conf) < 0)) {
 		printf("Configure failed for port\n");
diff --git a/test/test/test_power_acpi_cpufreq.c b/test/test/test_power_acpi_cpufreq.c
index 8da2dcc1..22e541d6 100644
--- a/test/test/test_power_acpi_cpufreq.c
+++ b/test/test/test_power_acpi_cpufreq.c
@@ -7,6 +7,7 @@
 #include <unistd.h>
 #include <limits.h>
 #include <string.h>
+#include <inttypes.h>
 
 #include "test.h"
 
@@ -19,6 +20,13 @@ test_power_acpi_cpufreq(void)
 	return TEST_SKIPPED;
 }
 
+static int
+test_power_acpi_caps(void)
+{
+	printf("Power management library not supported, skipping test\n");
+	return TEST_SKIPPED;
+}
+
 #else
 #include <rte_power.h>
 
@@ -517,6 +525,42 @@ fail_all:
 	rte_power_unset_env();
 	return -1;
 }
+
+static int
+test_power_acpi_caps(void)
+{
+	struct rte_power_core_capabilities caps;
+	int ret;
+
+	ret = rte_power_set_env(PM_ENV_ACPI_CPUFREQ);
+	if (ret) {
+		printf("Error setting ACPI environment\n");
+		return -1;
+	}
+
+	ret = rte_power_init(TEST_POWER_LCORE_ID);
+	if (ret < 0) {
+		printf("Cannot initialise power management for lcore %u, this "
+			"may occur if environment is not configured "
+			"correctly(APCI cpufreq) or operating in another valid "
+			"Power management environment\n", TEST_POWER_LCORE_ID);
+		rte_power_unset_env();
+		return -1;
+	}
+
+	ret = rte_power_get_capabilities(TEST_POWER_LCORE_ID, &caps);
+	if (ret) {
+		printf("ACPI: Error getting capabilities\n");
+		return -1;
+	}
+
+	printf("ACPI: Capabilities %"PRIx64"\n", caps.capabilities);
+
+	rte_power_unset_env();
+	return 0;
+}
+
 #endif
 
 REGISTER_TEST_COMMAND(power_acpi_cpufreq_autotest, test_power_acpi_cpufreq);
+REGISTER_TEST_COMMAND(power_acpi_caps_autotest, test_power_acpi_caps);
diff --git a/test/test/test_service_cores.c b/test/test/test_service_cores.c
index 86b4073d..ec31882d 100644
--- a/test/test/test_service_cores.c
+++ b/test/test/test_service_cores.c
@@ -325,6 +325,78 @@ service_attr_get(void)
 	return unregister_all();
 }
 
+/* verify service lcore attr get */
+static int
+service_lcore_attr_get(void)
+{
+	/* ensure all services unregistered so cycle counts are zero */
+	unregister_all();
+
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_cb;
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	service.capabilities |= RTE_SERVICE_CAP_MT_SAFE;
+	uint32_t id;
+	TEST_ASSERT_EQUAL(0, rte_service_component_register(&service, &id),
+			"Register of  service failed");
+	rte_service_component_runstate_set(id, 1);
+	TEST_ASSERT_EQUAL(0, rte_service_runstate_set(id, 1),
+			"Error: Service start returned non-zero");
+	rte_service_set_stats_enable(id, 1);
+
+	uint64_t lcore_attr_value = 0xdead;
+	uint32_t lcore_attr_id = UINT32_MAX;
+
+	/* check error return values */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_attr_get(UINT32_MAX,
+			lcore_attr_id, &lcore_attr_value),
+			"Invalid lcore_id didn't return -EINVAL");
+	TEST_ASSERT_EQUAL(-ENOTSUP, rte_service_lcore_attr_get(rte_lcore_id(),
+			lcore_attr_id, &lcore_attr_value),
+			"Non-service core didn't return -ENOTSUP");
+
+	/* Start service core to increment loop count */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_map_lcore_set(id, slcore_id, 1),
+			"Enabling valid service and core failed");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_start(slcore_id),
+			"Starting service core failed");
+
+	/* wait for the service lcore to run */
+	rte_delay_ms(200);
+
+	lcore_attr_id = RTE_SERVICE_LCORE_ATTR_LOOPS;
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_attr_get(slcore_id,
+			lcore_attr_id, &lcore_attr_value),
+			"Valid lcore_attr_get() call didn't return success");
+	int loops_gt_zero = lcore_attr_value > 0;
+	TEST_ASSERT_EQUAL(1, loops_gt_zero,
+			"lcore_attr_get() failed to get loops "
+			"(expected > zero)");
+
+	lcore_attr_id++;  // invalid lcore attr id
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_attr_get(slcore_id,
+			lcore_attr_id, &lcore_attr_value),
+			"Invalid lcore attr didn't return -EINVAL");
+
+	rte_service_lcore_stop(slcore_id);
+
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_attr_reset_all(slcore_id),
+			  "Valid lcore_attr_reset_all() didn't return success");
+
+	lcore_attr_id = RTE_SERVICE_LCORE_ATTR_LOOPS;
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_attr_get(slcore_id,
+			lcore_attr_id, &lcore_attr_value),
+			"Valid lcore_attr_get() call didn't return success");
+	TEST_ASSERT_EQUAL(0, lcore_attr_value,
+			"lcore_attr_get() didn't get correct loop count "
+			"(zero)");
+
+	return unregister_all();
+}
+
 /* verify service dump */
 static int
 service_dump(void)
@@ -771,6 +843,48 @@ service_lcore_start_stop(void)
 	return unregister_all();
 }
 
+/* stop a service and wait for it to become inactive */
+static int
+service_may_be_active(void)
+{
+	const uint32_t sid = 0;
+	int i;
+
+	/* expected failure cases */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_may_be_active(10000),
+			"Invalid service may be active check did not fail");
+
+	/* start the service */
+	TEST_ASSERT_EQUAL(0, rte_service_runstate_set(sid, 1),
+			"Starting valid service failed");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Add service core failed when not in use before");
+	TEST_ASSERT_EQUAL(0, rte_service_map_lcore_set(sid, slcore_id, 1),
+			"Enabling valid service on valid core failed");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_start(slcore_id),
+			"Service core start after add failed");
+
+	/* ensures core really is running the service function */
+	TEST_ASSERT_EQUAL(1, service_lcore_running_check(),
+			"Service core expected to poll service but it didn't");
+
+	/* stop the service */
+	TEST_ASSERT_EQUAL(0, rte_service_runstate_set(sid, 0),
+			"Error: Service stop returned non-zero");
+
+	/* give the service 100ms to stop running */
+	for (i = 0; i < 100; i++) {
+		if (!rte_service_may_be_active(sid))
+			break;
+		rte_delay_ms(SERVICE_DELAY);
+	}
+
+	TEST_ASSERT_EQUAL(0, rte_service_may_be_active(sid),
+			  "Error: Service not stopped after 100ms");
+
+	return unregister_all();
+}
+
 static struct unit_test_suite service_tests  = {
 	.suite_name = "service core test suite",
 	.setup = testsuite_setup,
@@ -781,6 +895,7 @@ static struct unit_test_suite service_tests  = {
 		TEST_CASE_ST(dummy_register, NULL, service_get_by_name),
 		TEST_CASE_ST(dummy_register, NULL, service_dump),
 		TEST_CASE_ST(dummy_register, NULL, service_attr_get),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_attr_get),
 		TEST_CASE_ST(dummy_register, NULL, service_probe_capability),
 		TEST_CASE_ST(dummy_register, NULL, service_start_stop),
 		TEST_CASE_ST(dummy_register, NULL, service_lcore_add_del),
@@ -790,6 +905,7 @@ static struct unit_test_suite service_tests  = {
 		TEST_CASE_ST(dummy_register, NULL, service_mt_safe_poll),
 		TEST_CASE_ST(dummy_register, NULL, service_app_lcore_mt_safe),
 		TEST_CASE_ST(dummy_register, NULL, service_app_lcore_mt_unsafe),
+		TEST_CASE_ST(dummy_register, NULL, service_may_be_active),
 		TEST_CASES_END() /**< NULL terminate unit test array */
 	}
 };
diff --git a/test/test/virtual_pmd.c b/test/test/virtual_pmd.c
index f8ddc2db..591b3098 100644
--- a/test/test/virtual_pmd.c
+++ b/test/test/virtual_pmd.c
@@ -91,6 +91,7 @@ virtual_ethdev_info_get(struct rte_eth_dev *dev __rte_unused,
 	dev_info->max_tx_queues = (uint16_t)512;
 
 	dev_info->min_rx_bufsize = 0;
+	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_CRC_STRIP;
 }
 
 static int
diff --git a/usertools/dpdk-devbind.py b/usertools/dpdk-devbind.py
index d0c42090..7d564634 100755
--- a/usertools/dpdk-devbind.py
+++ b/usertools/dpdk-devbind.py
@@ -24,6 +24,8 @@ cavium_pkx = {'Class': '08', 'Vendor': '177d', 'Device': 'a0dd,a049',
               'SVendor': None, 'SDevice': None}
 cavium_tim = {'Class': '08', 'Vendor': '177d', 'Device': 'a051',
               'SVendor': None, 'SDevice': None}
+cavium_zip = {'Class': '12', 'Vendor': '177d', 'Device': 'a037',
+              'SVendor': None, 'SDevice': None}
 avp_vnic = {'Class': '05', 'Vendor': '1af4', 'Device': '1110',
               'SVendor': None, 'SDevice': None}
 
@@ -31,6 +33,7 @@ network_devices = [network_class, cavium_pkx, avp_vnic]
 crypto_devices = [encryption_class, intel_processor_class]
 eventdev_devices = [cavium_sso, cavium_tim]
 mempool_devices = [cavium_fpa]
+compress_devices = [cavium_zip]
 
 # global dict ethernet devices present. Dictionary indexed by PCI address.
 # Each device within this is itself a dictionary of device properties
@@ -79,7 +82,7 @@ Options:
 
     --status-dev:
         Print the status of given device group. Supported device groups are:
-        "net", "crypto", "event" and "mempool"
+        "net", "crypto", "event", "mempool" and "compress"
 
     -b driver, --bind=driver:
         Select the driver to use or \"none\" to unbind the device
@@ -569,6 +572,10 @@ def show_status():
     if status_dev == "mempool" or status_dev == "all":
         show_device_status(mempool_devices, "Mempool")
 
+    if status_dev == "compress" or status_dev == "all":
+        show_device_status(compress_devices , "Compress")
+
+
 def parse_args():
     '''Parses the command-line arguments given by the user and takes the
     appropriate action for each'''
@@ -642,6 +649,7 @@ def do_arg_actions():
             get_device_details(crypto_devices)
             get_device_details(eventdev_devices)
             get_device_details(mempool_devices)
+            get_device_details(compress_devices)
         show_status()
 
 
@@ -654,6 +662,7 @@ def main():
     get_device_details(crypto_devices)
     get_device_details(eventdev_devices)
     get_device_details(mempool_devices)
+    get_device_details(compress_devices)
     do_arg_actions()
 
 if __name__ == "__main__":