New upstream version 18.11-rc1upstream/18.11-rc1

Change-Id: Iaa71986dd6332e878d8f4bf493101b2bbc6313bb
Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>

1 files changed, 391 insertions, 119 deletions

diff --git a/drivers/net/mvpp2/mrvl_ethdev.c b/drivers/net/mvpp2/mrvl_ethdev.c
index a2d0576e..ab4c14e5 100644
--- a/drivers/net/mvpp2/mrvl_ethdev.c
+++ b/drivers/net/mvpp2/mrvl_ethdev.c
@@ -10,15 +10,6 @@
 #include <rte_malloc.h>
 #include <rte_bus_vdev.h>
 
-/* Unluckily, container_of is defined by both DPDK and MUSDK,
- * we'll declare only one version.
- *
- * Note that it is not used in this PMD anyway.
- */
-#ifdef container_of
-#undef container_of
-#endif
-
 #include <fcntl.h>
 #include <linux/ethtool.h>
 #include <linux/sockios.h>
@@ -29,8 +20,12 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 
+#include <rte_mvep_common.h>
 #include "mrvl_ethdev.h"
 #include "mrvl_qos.h"
+#include "mrvl_flow.h"
+#include "mrvl_mtr.h"
+#include "mrvl_tm.h"
 
 /* bitmask with reserved hifs */
 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
@@ -59,23 +54,18 @@
 #define MRVL_ARP_LENGTH 28
 
 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
-
-#define MRVL_COOKIE_HIGH_ADDR_SHIFT	(sizeof(pp2_cookie_t) * 8)
-#define MRVL_COOKIE_HIGH_ADDR_MASK	(~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
-
-/* Memory size (in bytes) for MUSDK dma buffers */
-#define MRVL_MUSDK_DMA_MEMSIZE 41943040
+#define MRVL_COOKIE_HIGH_ADDR_MASK 0xffffff0000000000
 
 /** Port Rx offload capabilities */
 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
 			  DEV_RX_OFFLOAD_JUMBO_FRAME | \
-			  DEV_RX_OFFLOAD_CRC_STRIP | \
 			  DEV_RX_OFFLOAD_CHECKSUM)
 
 /** Port Tx offloads capabilities */
 #define MRVL_TX_OFFLOADS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
 			  DEV_TX_OFFLOAD_UDP_CKSUM | \
-			  DEV_TX_OFFLOAD_TCP_CKSUM)
+			  DEV_TX_OFFLOAD_TCP_CKSUM | \
+			  DEV_TX_OFFLOAD_MULTI_SEGS)
 
 static const char * const valid_args[] = {
 	MRVL_IFACE_NAME_ARG,
@@ -86,13 +76,12 @@ static const char * const valid_args[] = {
 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 static struct pp2_hif *hifs[RTE_MAX_LCORE];
 static int used_bpools[PP2_NUM_PKT_PROC] = {
-	MRVL_MUSDK_BPOOLS_RESERVED,
-	MRVL_MUSDK_BPOOLS_RESERVED
+	[0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
 };
 
-struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
-int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
-uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
+static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
+static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
+static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 
 int mrvl_logtype;
 
@@ -116,7 +105,9 @@ struct mrvl_shadow_txq {
 	int head;           /* write index - used when sending buffers */
 	int tail;           /* read index - used when releasing buffers */
 	u16 size;           /* queue occupied size */
-	u16 num_to_release; /* number of buffers sent, that can be released */
+	u16 num_to_release; /* number of descriptors sent, that can be
+			     * released
+			     */
 	struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 };
 
@@ -148,6 +139,12 @@ static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 			struct pp2_hif *hif, unsigned int core_id,
 			struct mrvl_shadow_txq *sq, int qid, int force);
 
+static uint16_t mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
+				  uint16_t nb_pkts);
+static uint16_t mrvl_tx_sg_pkt_burst(void *txq,	struct rte_mbuf **tx_pkts,
+				     uint16_t nb_pkts);
+
+
 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 	#name, offsetof(struct pp2_ppio_statistics, name),	\
 	sizeof(((struct pp2_ppio_statistics *)0)->name)		\
@@ -174,6 +171,31 @@ static struct {
 	MRVL_XSTATS_TBL_ENTRY(tx_errors)
 };
 
+static inline void
+mrvl_fill_shadowq(struct mrvl_shadow_txq *sq, struct rte_mbuf *buf)
+{
+	sq->ent[sq->head].buff.cookie = (uint64_t)buf;
+	sq->ent[sq->head].buff.addr = buf ?
+		rte_mbuf_data_iova_default(buf) : 0;
+
+	sq->ent[sq->head].bpool =
+		(unlikely(!buf || buf->port >= RTE_MAX_ETHPORTS ||
+		 buf->refcnt > 1)) ? NULL :
+		 mrvl_port_to_bpool_lookup[buf->port];
+
+	sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
+	sq->size++;
+}
+
+static inline void
+mrvl_fill_desc(struct pp2_ppio_desc *desc, struct rte_mbuf *buf)
+{
+	pp2_ppio_outq_desc_reset(desc);
+	pp2_ppio_outq_desc_set_phys_addr(desc, rte_pktmbuf_iova(buf));
+	pp2_ppio_outq_desc_set_pkt_offset(desc, 0);
+	pp2_ppio_outq_desc_set_pkt_len(desc, rte_pktmbuf_data_len(buf));
+}
+
 static inline int
 mrvl_get_bpool_size(int pp2_id, int pool_id)
 {
@@ -253,6 +275,27 @@ out:
 }
 
 /**
+ * Set tx burst function according to offload flag
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ */
+static void
+mrvl_set_tx_function(struct rte_eth_dev *dev)
+{
+	struct mrvl_priv *priv = dev->data->dev_private;
+
+	/* Use a simple Tx queue (no offloads, no multi segs) if possible */
+	if (priv->multiseg) {
+		RTE_LOG(INFO, PMD, "Using multi-segment tx callback\n");
+		dev->tx_pkt_burst = mrvl_tx_sg_pkt_burst;
+	} else {
+		RTE_LOG(INFO, PMD, "Using single-segment tx callback\n");
+		dev->tx_pkt_burst = mrvl_tx_pkt_burst;
+	}
+}
+
+/**
  * Configure rss based on dpdk rss configuration.
  *
  * @param priv
@@ -307,6 +350,11 @@ mrvl_dev_configure(struct rte_eth_dev *dev)
 	struct mrvl_priv *priv = dev->data->dev_private;
 	int ret;
 
+	if (priv->ppio) {
+		MRVL_LOG(INFO, "Device reconfiguration is not supported");
+		return -EINVAL;
+	}
+
 	if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 	    dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 		MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
@@ -314,14 +362,6 @@ mrvl_dev_configure(struct rte_eth_dev *dev)
 		return -EINVAL;
 	}
 
-	/* KEEP_CRC offload flag is not supported by PMD
-	 * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
-	 */
-	if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads)) {
-		MRVL_LOG(INFO, "L2 CRC stripping is always enabled in hw");
-		dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CRC_STRIP;
-	}
-
 	if (dev->data->dev_conf.rxmode.split_hdr_size) {
 		MRVL_LOG(INFO, "Split headers not supported");
 		return -EINVAL;
@@ -329,7 +369,10 @@ mrvl_dev_configure(struct rte_eth_dev *dev)
 
 	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 		dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
-				 ETHER_HDR_LEN - ETHER_CRC_LEN;
+				 MRVL_PP2_ETH_HDRS_LEN;
+
+	if (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
+		priv->multiseg = 1;
 
 	ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 				  dev->data->nb_rx_queues);
@@ -345,6 +388,10 @@ mrvl_dev_configure(struct rte_eth_dev *dev)
 	priv->ppio_params.maintain_stats = 1;
 	priv->nb_rx_queues = dev->data->nb_rx_queues;
 
+	ret = mrvl_tm_init(dev);
+	if (ret < 0)
+		return ret;
+
 	if (dev->data->nb_rx_queues == 1 &&
 	    dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 		MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
@@ -375,21 +422,55 @@ static int
 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 {
 	struct mrvl_priv *priv = dev->data->dev_private;
-	/* extra MV_MH_SIZE bytes are required for Marvell tag */
-	uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
+	uint16_t mru;
+	uint16_t mbuf_data_size = 0; /* SW buffer size */
 	int ret;
 
-	if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
+	mru = MRVL_PP2_MTU_TO_MRU(mtu);
+	/*
+	 * min_rx_buf_size is equal to mbuf data size
+	 * if pmd didn't set it differently
+	 */
+	mbuf_data_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
+	/* Prevent PMD from:
+	 * - setting mru greater than the mbuf size resulting in
+	 * hw and sw buffer size mismatch
+	 * - setting mtu that requires the support of scattered packets
+	 * when this feature has not been enabled/supported so far
+	 * (TODO check scattered_rx flag here once scattered RX is supported).
+	 */
+	if (mru + MRVL_PKT_OFFS > mbuf_data_size) {
+		mru = mbuf_data_size - MRVL_PKT_OFFS;
+		mtu = MRVL_PP2_MRU_TO_MTU(mru);
+		MRVL_LOG(WARNING, "MTU too big, max MTU possible limitted "
+			"by current mbuf size: %u. Set MTU to %u, MRU to %u",
+			mbuf_data_size, mtu, mru);
+	}
+
+	if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX) {
+		MRVL_LOG(ERR, "Invalid MTU [%u] or MRU [%u]", mtu, mru);
 		return -EINVAL;
+	}
+
+	dev->data->mtu = mtu;
+	dev->data->dev_conf.rxmode.max_rx_pkt_len = mru - MV_MH_SIZE;
 
 	if (!priv->ppio)
 		return 0;
 
 	ret = pp2_ppio_set_mru(priv->ppio, mru);
-	if (ret)
+	if (ret) {
+		MRVL_LOG(ERR, "Failed to change MRU");
 		return ret;
+	}
 
-	return pp2_ppio_set_mtu(priv->ppio, mtu);
+	ret = pp2_ppio_set_mtu(priv->ppio, mtu);
+	if (ret) {
+		MRVL_LOG(ERR, "Failed to change MTU");
+		return ret;
+	}
+
+	return 0;
 }
 
 /**
@@ -528,6 +609,9 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 	char match[MRVL_MATCH_LEN];
 	int ret = 0, i, def_init_size;
 
+	if (priv->ppio)
+		return mrvl_dev_set_link_up(dev);
+
 	snprintf(match, sizeof(match), "ppio-%d:%d",
 		 priv->pp_id, priv->ppio_id);
 	priv->ppio_params.match = match;
@@ -597,9 +681,13 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 		}
 		priv->vlan_flushed = 1;
 	}
+	ret = mrvl_mtu_set(dev, dev->data->mtu);
+	if (ret)
+		MRVL_LOG(ERR, "Failed to set MTU to %d", dev->data->mtu);
 
 	/* For default QoS config, don't start classifier. */
-	if (mrvl_qos_cfg) {
+	if (mrvl_qos_cfg  &&
+	    mrvl_qos_cfg->port[dev->data->port_id].use_global_defaults == 0) {
 		ret = mrvl_start_qos_mapping(priv);
 		if (ret) {
 			MRVL_LOG(ERR, "Failed to setup QoS mapping");
@@ -631,6 +719,10 @@ mrvl_dev_start(struct rte_eth_dev *dev)
 			goto out;
 	}
 
+	mrvl_flow_init(dev);
+	mrvl_mtr_init(dev);
+	mrvl_set_tx_function(dev);
+
 	return 0;
 out:
 	MRVL_LOG(ERR, "Failed to start device");
@@ -752,28 +844,7 @@ mrvl_flush_bpool(struct rte_eth_dev *dev)
 static void
 mrvl_dev_stop(struct rte_eth_dev *dev)
 {
-	struct mrvl_priv *priv = dev->data->dev_private;
-
 	mrvl_dev_set_link_down(dev);
-	mrvl_flush_rx_queues(dev);
-	mrvl_flush_tx_shadow_queues(dev);
-	if (priv->cls_tbl) {
-		pp2_cls_tbl_deinit(priv->cls_tbl);
-		priv->cls_tbl = NULL;
-	}
-	if (priv->qos_tbl) {
-		pp2_cls_qos_tbl_deinit(priv->qos_tbl);
-		priv->qos_tbl = NULL;
-	}
-	if (priv->ppio)
-		pp2_ppio_deinit(priv->ppio);
-	priv->ppio = NULL;
-
-	/* policer must be released after ppio deinitialization */
-	if (priv->policer) {
-		pp2_cls_plcr_deinit(priv->policer);
-		priv->policer = NULL;
-	}
 }
 
 /**
@@ -788,6 +859,11 @@ mrvl_dev_close(struct rte_eth_dev *dev)
 	struct mrvl_priv *priv = dev->data->dev_private;
 	size_t i;
 
+	mrvl_flush_rx_queues(dev);
+	mrvl_flush_tx_shadow_queues(dev);
+	mrvl_flow_deinit(dev);
+	mrvl_mtr_deinit(dev);
+
 	for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 		struct pp2_ppio_tc_params *tc_params =
 			&priv->ppio_params.inqs_params.tcs_params[i];
@@ -798,7 +874,29 @@ mrvl_dev_close(struct rte_eth_dev *dev)
 		}
 	}
 
+	if (priv->cls_tbl) {
+		pp2_cls_tbl_deinit(priv->cls_tbl);
+		priv->cls_tbl = NULL;
+	}
+
+	if (priv->qos_tbl) {
+		pp2_cls_qos_tbl_deinit(priv->qos_tbl);
+		priv->qos_tbl = NULL;
+	}
+
 	mrvl_flush_bpool(dev);
+	mrvl_tm_deinit(dev);
+
+	if (priv->ppio) {
+		pp2_ppio_deinit(priv->ppio);
+		priv->ppio = NULL;
+	}
+
+	/* policer must be released after ppio deinitialization */
+	if (priv->default_policer) {
+		pp2_cls_plcr_deinit(priv->default_policer);
+		priv->default_policer = NULL;
+	}
 }
 
 /**
@@ -1337,7 +1435,6 @@ mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
 
 	/* By default packets are dropped if no descriptors are available */
 	info->default_rxconf.rx_drop_en = 1;
-	info->default_rxconf.offloads = DEV_RX_OFFLOAD_CRC_STRIP;
 
 	info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
 }
@@ -1356,6 +1453,8 @@ mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
 {
 	static const uint32_t ptypes[] = {
 		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_L2_ETHER_VLAN,
+		RTE_PTYPE_L2_ETHER_QINQ,
 		RTE_PTYPE_L3_IPV4,
 		RTE_PTYPE_L3_IPV4_EXT,
 		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
@@ -1492,7 +1591,7 @@ mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
 
 		entries[i].buff.addr =
 			rte_mbuf_data_iova_default(mbufs[i]);
-		entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
+		entries[i].buff.cookie = (uint64_t)mbufs[i];
 		entries[i].bpool = bpool;
 	}
 
@@ -1537,8 +1636,8 @@ mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 {
 	struct mrvl_priv *priv = dev->data->dev_private;
 	struct mrvl_rxq *rxq;
-	uint32_t min_size,
-		 max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
+	uint32_t frame_size, buf_size = rte_pktmbuf_data_room_size(mp);
+	uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 	int ret, tc, inq;
 	uint64_t offloads;
 
@@ -1553,15 +1652,16 @@ mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		return -EFAULT;
 	}
 
-	min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
-		   MRVL_PKT_EFFEC_OFFS;
-	if (min_size < max_rx_pkt_len) {
-		MRVL_LOG(ERR,
-			"Mbuf size must be increased to %u bytes to hold up to %u bytes of data.",
-			max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
-			MRVL_PKT_EFFEC_OFFS,
+	frame_size = buf_size - RTE_PKTMBUF_HEADROOM - MRVL_PKT_EFFEC_OFFS;
+	if (frame_size < max_rx_pkt_len) {
+		MRVL_LOG(WARNING,
+			"Mbuf size must be increased to %u bytes to hold up "
+			"to %u bytes of data.",
+			buf_size + max_rx_pkt_len - frame_size,
 			max_rx_pkt_len);
-		return -EINVAL;
+		dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
+		MRVL_LOG(INFO, "Setting max rx pkt len to %u",
+			dev->data->dev_conf.rxmode.max_rx_pkt_len);
 	}
 
 	if (dev->data->rx_queues[idx]) {
@@ -1867,6 +1967,44 @@ mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
 	}
 }
 
+/**
+ * DPDK callback to get rte_mtr callbacks.
+ *
+ * @param dev
+ *   Pointer to the device structure.
+ * @param ops
+ *   Pointer to pass the mtr ops.
+ *
+ * @return
+ *   Always 0.
+ */
+static int
+mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
+{
+	*(const void **)ops = &mrvl_mtr_ops;
+
+	return 0;
+}
+
+/**
+ * DPDK callback to get rte_tm callbacks.
+ *
+ * @param dev
+ *   Pointer to the device structure.
+ * @param ops
+ *   Pointer to pass the tm ops.
+ *
+ * @return
+ *   Always 0.
+ */
+static int
+mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
+{
+	*(const void **)ops = &mrvl_tm_ops;
+
+	return 0;
+}
+
 static const struct eth_dev_ops mrvl_ops = {
 	.dev_configure = mrvl_dev_configure,
 	.dev_start = mrvl_dev_start,
@@ -1904,6 +2042,8 @@ static const struct eth_dev_ops mrvl_ops = {
 	.rss_hash_update = mrvl_rss_hash_update,
 	.rss_hash_conf_get = mrvl_rss_hash_conf_get,
 	.filter_ctrl = mrvl_eth_filter_ctrl,
+	.mtr_ops_get = mrvl_mtr_ops_get,
+	.tm_ops_get = mrvl_tm_ops_get,
 };
 
 /**
@@ -1925,13 +2065,27 @@ mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
 {
 	enum pp2_inq_l3_type l3_type;
 	enum pp2_inq_l4_type l4_type;
+	enum pp2_inq_vlan_tag vlan_tag;
 	uint64_t packet_type;
 
 	pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
 	pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
+	pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
 
 	packet_type = RTE_PTYPE_L2_ETHER;
 
+	switch (vlan_tag) {
+	case PP2_INQ_VLAN_TAG_SINGLE:
+		packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
+		break;
+	case PP2_INQ_VLAN_TAG_DOUBLE:
+	case PP2_INQ_VLAN_TAG_TRIPLE:
+		packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
+		break;
+	default:
+		break;
+	}
+
 	switch (l3_type) {
 	case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
 		packet_type |= RTE_PTYPE_L3_IPV4;
@@ -2073,7 +2227,7 @@ mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		if (unlikely(status != PP2_DESC_ERR_OK)) {
 			struct pp2_buff_inf binf = {
 				.addr = rte_mbuf_data_iova_default(mbuf),
-				.cookie = (pp2_cookie_t)(uint64_t)mbuf,
+				.cookie = (uint64_t)mbuf,
 			};
 
 			pp2_bpool_put_buff(hif, bpool, &binf);
@@ -2334,22 +2488,8 @@ mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			rte_mbuf_prefetch_part2(pref_pkt_hdr);
 		}
 
-		sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
-		sq->ent[sq->head].buff.addr =
-			rte_mbuf_data_iova_default(mbuf);
-		sq->ent[sq->head].bpool =
-			(unlikely(mbuf->port >= RTE_MAX_ETHPORTS ||
-			 mbuf->refcnt > 1)) ? NULL :
-			 mrvl_port_to_bpool_lookup[mbuf->port];
-		sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
-		sq->size++;
-
-		pp2_ppio_outq_desc_reset(&descs[i]);
-		pp2_ppio_outq_desc_set_phys_addr(&descs[i],
-						 rte_pktmbuf_iova(mbuf));
-		pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
-		pp2_ppio_outq_desc_set_pkt_len(&descs[i],
-					       rte_pktmbuf_pkt_len(mbuf));
+		mrvl_fill_shadowq(sq, mbuf);
+		mrvl_fill_desc(&descs[i], mbuf);
 
 		bytes_sent += rte_pktmbuf_pkt_len(mbuf);
 		/*
@@ -2387,6 +2527,152 @@ mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	return nb_pkts;
 }
 
+/** DPDK callback for S/G transmit.
+ *
+ * @param txq
+ *   Generic pointer transmit queue.
+ * @param tx_pkts
+ *   Packets to transmit.
+ * @param nb_pkts
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted.
+ */
+static uint16_t
+mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
+		     uint16_t nb_pkts)
+{
+	struct mrvl_txq *q = txq;
+	struct mrvl_shadow_txq *sq;
+	struct pp2_hif *hif;
+	struct pp2_ppio_desc descs[nb_pkts * PP2_PPIO_DESC_NUM_FRAGS];
+	struct pp2_ppio_sg_pkts pkts;
+	uint8_t frags[nb_pkts];
+	unsigned int core_id = rte_lcore_id();
+	int i, j, ret, bytes_sent = 0;
+	int tail, tail_first;
+	uint16_t num, sq_free_size;
+	uint16_t nb_segs, total_descs = 0;
+	uint64_t addr;
+
+	hif = mrvl_get_hif(q->priv, core_id);
+	sq = &q->shadow_txqs[core_id];
+	pkts.frags = frags;
+	pkts.num = 0;
+
+	if (unlikely(!q->priv->ppio || !hif))
+		return 0;
+
+	if (sq->size)
+		mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
+				       sq, q->queue_id, 0);
+
+	/* Save shadow queue free size */
+	sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
+
+	tail = 0;
+	for (i = 0; i < nb_pkts; i++) {
+		struct rte_mbuf *mbuf = tx_pkts[i];
+		struct rte_mbuf *seg = NULL;
+		int gen_l3_cksum, gen_l4_cksum;
+		enum pp2_outq_l3_type l3_type;
+		enum pp2_outq_l4_type l4_type;
+
+		nb_segs = mbuf->nb_segs;
+		tail_first = tail;
+		total_descs += nb_segs;
+
+		/*
+		 * Check if total_descs does not exceed
+		 * shadow queue free size
+		 */
+		if (unlikely(total_descs > sq_free_size)) {
+			total_descs -= nb_segs;
+			RTE_LOG(DEBUG, PMD,
+				"No room in shadow queue for %d packets! "
+				"%d packets will be sent.\n",
+				nb_pkts, i);
+			break;
+		}
+
+		/* Check if nb_segs does not exceed the max nb of desc per
+		 * fragmented packet
+		 */
+		if (nb_segs > PP2_PPIO_DESC_NUM_FRAGS) {
+			total_descs -= nb_segs;
+			RTE_LOG(ERR, PMD,
+				"Too many segments. Packet won't be sent.\n");
+			break;
+		}
+
+		if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
+			struct rte_mbuf *pref_pkt_hdr;
+
+			pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
+			rte_mbuf_prefetch_part1(pref_pkt_hdr);
+			rte_mbuf_prefetch_part2(pref_pkt_hdr);
+		}
+
+		pkts.frags[pkts.num] = nb_segs;
+		pkts.num++;
+
+		seg = mbuf;
+		for (j = 0; j < nb_segs - 1; j++) {
+			/* For the subsequent segments, set shadow queue
+			 * buffer to NULL
+			 */
+			mrvl_fill_shadowq(sq, NULL);
+			mrvl_fill_desc(&descs[tail], seg);
+
+			tail++;
+			seg = seg->next;
+		}
+		/* Put first mbuf info in last shadow queue entry */
+		mrvl_fill_shadowq(sq, mbuf);
+		/* Update descriptor with last segment */
+		mrvl_fill_desc(&descs[tail++], seg);
+
+		bytes_sent += rte_pktmbuf_pkt_len(mbuf);
+		/* In case unsupported ol_flags were passed
+		 * do not update descriptor offload information
+		 */
+		ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
+					      &l3_type, &l4_type, &gen_l3_cksum,
+					      &gen_l4_cksum);
+		if (unlikely(ret))
+			continue;
+
+		pp2_ppio_outq_desc_set_proto_info(&descs[tail_first], l3_type,
+						  l4_type, mbuf->l2_len,
+						  mbuf->l2_len + mbuf->l3_len,
+						  gen_l3_cksum, gen_l4_cksum);
+	}
+
+	num = total_descs;
+	pp2_ppio_send_sg(q->priv->ppio, hif, q->queue_id, descs,
+			 &total_descs, &pkts);
+	/* number of packets that were not sent */
+	if (unlikely(num > total_descs)) {
+		for (i = total_descs; i < num; i++) {
+			sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
+				MRVL_PP2_TX_SHADOWQ_MASK;
+
+			addr = sq->ent[sq->head].buff.cookie;
+			if (addr)
+				bytes_sent -=
+					rte_pktmbuf_pkt_len((struct rte_mbuf *)
+						(cookie_addr_high | addr));
+		}
+		sq->size -= num - total_descs;
+		nb_pkts = pkts.num;
+	}
+
+	q->bytes_sent += bytes_sent;
+
+	return nb_pkts;
+}
+
 /**
  * Initialize packet processor.
  *
@@ -2494,8 +2780,9 @@ mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
 	priv = mrvl_priv_create(name);
 	if (!priv) {
 		ret = -ENOMEM;
-		goto out_free_dev;
+		goto out_free;
 	}
+	eth_dev->data->dev_private = priv;
 
 	eth_dev->data->mac_addrs =
 		rte_zmalloc("mac_addrs",
@@ -2503,33 +2790,28 @@ mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
 	if (!eth_dev->data->mac_addrs) {
 		MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
 		ret = -ENOMEM;
-		goto out_free_priv;
+		goto out_free;
 	}
 
 	memset(&req, 0, sizeof(req));
 	strcpy(req.ifr_name, name);
 	ret = ioctl(fd, SIOCGIFHWADDR, &req);
 	if (ret)
-		goto out_free_mac;
+		goto out_free;
 
 	memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
 	       req.ifr_addr.sa_data, ETHER_ADDR_LEN);
 
-	eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
-	eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
 	eth_dev->data->kdrv = RTE_KDRV_NONE;
-	eth_dev->data->dev_private = priv;
 	eth_dev->device = &vdev->device;
+	eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
+	mrvl_set_tx_function(eth_dev);
 	eth_dev->dev_ops = &mrvl_ops;
 
 	rte_eth_dev_probing_finish(eth_dev);
 	return 0;
-out_free_mac:
-	rte_free(eth_dev->data->mac_addrs);
-out_free_dev:
+out_free:
 	rte_eth_dev_release_port(eth_dev);
-out_free_priv:
-	rte_free(priv);
 
 	return ret;
 }
@@ -2553,8 +2835,6 @@ mrvl_eth_dev_destroy(const char *name)
 	priv = eth_dev->data->dev_private;
 	pp2_bpool_deinit(priv->bpool);
 	used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
-	rte_free(priv);
-	rte_free(eth_dev->data->mac_addrs);
 	rte_eth_dev_release_port(eth_dev);
 }
 
@@ -2654,23 +2934,16 @@ rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
 		goto init_devices;
 
 	MRVL_LOG(INFO, "Perform MUSDK initializations");
-	/*
-	 * ret == -EEXIST is correct, it means DMA
-	 * has been already initialized (by another PMD).
-	 */
-	ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE);
-	if (ret < 0) {
-		if (ret != -EEXIST)
-			goto out_free_kvlist;
-		else
-			MRVL_LOG(INFO,
-				"DMA memory has been already initialized by a different driver.");
-	}
+
+	ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
+	if (ret)
+		goto out_free_kvlist;
 
 	ret = mrvl_init_pp2();
 	if (ret) {
 		MRVL_LOG(ERR, "Failed to init PP!");
-		goto out_deinit_dma;
+		rte_mvep_deinit(MVEP_MOD_T_PP2);
+		goto out_free_kvlist;
 	}
 
 	memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
@@ -2695,11 +2968,10 @@ out_cleanup:
 	for (; i > 0; i--)
 		mrvl_eth_dev_destroy(ifnames.names[i]);
 
-	if (mrvl_dev_num == 0)
+	if (mrvl_dev_num == 0) {
 		mrvl_deinit_pp2();
-out_deinit_dma:
-	if (mrvl_dev_num == 0)
-		mv_sys_dma_mem_destroy();
+		rte_mvep_deinit(MVEP_MOD_T_PP2);
+	}
 out_free_kvlist:
 	rte_kvargs_free(kvlist);
 
@@ -2739,7 +3011,7 @@ rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
 		MRVL_LOG(INFO, "Perform MUSDK deinit");
 		mrvl_deinit_hifs();
 		mrvl_deinit_pp2();
-		mv_sys_dma_mem_destroy();
+		rte_mvep_deinit(MVEP_MOD_T_PP2);
 	}
 
 	return 0;