New upstream version 17.08

Change-Id: I288b50990f52646089d6b1f3aaa6ba2f091a51d7
Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>

5 files changed, 808 insertions, 157 deletions

diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index e873fb48..c045bd79 100644
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -40,7 +40,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_flow.c
 
 # Basic CFLAGS.
 CFLAGS += -O3
-CFLAGS += -std=gnu99 -Wall -Wextra
+CFLAGS += -std=c11 -Wall -Wextra
 CFLAGS += -g
 CFLAGS += -I.
 CFLAGS += -D_BSD_SOURCE
@@ -84,6 +84,10 @@ ifdef CONFIG_RTE_LIBRTE_MLX4_SOFT_COUNTERS
 CFLAGS += -DMLX4_PMD_SOFT_COUNTERS=$(CONFIG_RTE_LIBRTE_MLX4_SOFT_COUNTERS)
 endif
 
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DEBUG_BROKEN_VERBS),y)
+CFLAGS += -DMLX4_PMD_DEBUG_BROKEN_VERBS
+endif
+
 include $(RTE_SDK)/mk/rte.lib.mk
 
 # Generate and clean-up mlx4_autoconf.h.
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index ec4419a8..055de49a 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -75,6 +75,7 @@
 #include <rte_memory.h>
 #include <rte_flow.h>
 #include <rte_kvargs.h>
+#include <rte_interrupts.h>
 
 /* Generated configuration header. */
 #include "mlx4_autoconf.h"
@@ -127,6 +128,18 @@ const char *pmd_mlx4_init_params[] = {
 	NULL,
 };
 
+static int
+mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx);
+
+static int
+mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx);
+
+static int
+priv_rx_intr_vec_enable(struct priv *priv);
+
+static void
+priv_rx_intr_vec_disable(struct priv *priv);
+
 /**
  * Check if running as a secondary process.
  *
@@ -533,13 +546,93 @@ txq_cleanup(struct txq *txq);
 
 static int
 rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
-	  unsigned int socket, int inactive, const struct rte_eth_rxconf *conf,
-	  struct rte_mempool *mp);
+	  unsigned int socket, int inactive,
+	  const struct rte_eth_rxconf *conf,
+	  struct rte_mempool *mp, int children_n,
+	  struct rxq *rxq_parent);
 
 static void
 rxq_cleanup(struct rxq *rxq);
 
 /**
+ * Create RSS parent queue.
+ *
+ * The new parent is inserted in front of the list in the private structure.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param queues
+ *   Queues indices array, if NULL use all Rx queues.
+ * @param children_n
+ *   The number of entries in queues[].
+ *
+ * @return
+ *   Pointer to a parent rxq structure, NULL on failure.
+ */
+struct rxq *
+priv_parent_create(struct priv *priv,
+		   uint16_t queues[],
+		   uint16_t children_n)
+{
+	int ret;
+	uint16_t i;
+	struct rxq *parent;
+
+	parent = rte_zmalloc("parent queue",
+			     sizeof(*parent),
+			     RTE_CACHE_LINE_SIZE);
+	if (!parent) {
+		ERROR("cannot allocate memory for RSS parent queue");
+		return NULL;
+	}
+	ret = rxq_setup(priv->dev, parent, 0, 0, 0,
+			NULL, NULL, children_n, NULL);
+	if (ret) {
+		rte_free(parent);
+		return NULL;
+	}
+	parent->rss.queues_n = children_n;
+	if (queues) {
+		for (i = 0; i < children_n; ++i)
+			parent->rss.queues[i] = queues[i];
+	} else {
+		/* the default RSS ring case */
+		assert(priv->rxqs_n == children_n);
+		for (i = 0; i < priv->rxqs_n; ++i)
+			parent->rss.queues[i] = i;
+	}
+	LIST_INSERT_HEAD(&priv->parents, parent, next);
+	return parent;
+}
+
+/**
+ * Clean up RX queue parent structure.
+ *
+ * @param parent
+ *   RX queue parent structure.
+ */
+void
+rxq_parent_cleanup(struct rxq *parent)
+{
+	LIST_REMOVE(parent, next);
+	rxq_cleanup(parent);
+	rte_free(parent);
+}
+
+/**
+ * Clean up parent structures from the parent list.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ */
+static void
+priv_parent_list_cleanup(struct priv *priv)
+{
+	while (!LIST_EMPTY(&priv->parents))
+		rxq_parent_cleanup(LIST_FIRST(&priv->parents));
+}
+
+/**
  * Ethernet device configuration.
  *
  * Prepare the driver for a given number of TX and RX queues.
@@ -558,7 +651,6 @@ dev_configure(struct rte_eth_dev *dev)
 	unsigned int rxqs_n = dev->data->nb_rx_queues;
 	unsigned int txqs_n = dev->data->nb_tx_queues;
 	unsigned int tmp;
-	int ret;
 
 	priv->rxqs = (void *)dev->data->rx_queues;
 	priv->txqs = (void *)dev->data->tx_queues;
@@ -569,7 +661,7 @@ dev_configure(struct rte_eth_dev *dev)
 	}
 	if (rxqs_n == priv->rxqs_n)
 		return 0;
-	if (!rte_is_power_of_2(rxqs_n)) {
+	if (!rte_is_power_of_2(rxqs_n) && !priv->isolated) {
 		unsigned n_active;
 
 		n_active = rte_align32pow2(rxqs_n + 1) >> 1;
@@ -588,7 +680,7 @@ dev_configure(struct rte_eth_dev *dev)
 		for (i = 0; (i != priv->rxqs_n); ++i)
 			if ((*priv->rxqs)[i] != NULL)
 				return EINVAL;
-		rxq_cleanup(&priv->rxq_parent);
+		priv_parent_list_cleanup(priv);
 		priv->rss = 0;
 		priv->rxqs_n = 0;
 	}
@@ -613,14 +705,14 @@ dev_configure(struct rte_eth_dev *dev)
 	priv->rss = 1;
 	tmp = priv->rxqs_n;
 	priv->rxqs_n = rxqs_n;
-	ret = rxq_setup(dev, &priv->rxq_parent, 0, 0, 0, NULL, NULL);
-	if (!ret)
+	if (priv->isolated)
+		return 0;
+	if (priv_parent_create(priv, NULL, priv->rxqs_n))
 		return 0;
 	/* Failure, rollback. */
 	priv->rss = 0;
 	priv->rxqs_n = tmp;
-	assert(ret > 0);
-	return ret;
+	return ENOMEM;
 }
 
 /**
@@ -1098,7 +1190,7 @@ static int mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start,
 
 /* For best performance, this function should not be inlined. */
 static struct ibv_mr *mlx4_mp2mr(struct ibv_pd *, struct rte_mempool *)
-	__attribute__((noinline));
+	__rte_noinline;
 
 /**
  * Register mempool as a memory region.
@@ -2499,11 +2591,12 @@ priv_mac_addr_del(struct priv *priv, unsigned int mac_index)
 {
 	unsigned int i;
 
+	assert(!priv->isolated);
 	assert(mac_index < elemof(priv->mac));
 	if (!BITFIELD_ISSET(priv->mac_configured, mac_index))
 		return;
 	if (priv->rss) {
-		rxq_mac_addr_del(&priv->rxq_parent, mac_index);
+		rxq_mac_addr_del(LIST_FIRST(&priv->parents), mac_index);
 		goto end;
 	}
 	for (i = 0; (i != priv->dev->data->nb_rx_queues); ++i)
@@ -2570,7 +2663,7 @@ priv_mac_addr_add(struct priv *priv, unsigned int mac_index,
 		goto end;
 	}
 	if (priv->rss) {
-		ret = rxq_mac_addr_add(&priv->rxq_parent, mac_index);
+		ret = rxq_mac_addr_add(LIST_FIRST(&priv->parents), mac_index);
 		if (ret)
 			return ret;
 		goto end;
@@ -2748,14 +2841,17 @@ rxq_cleanup(struct rxq *rxq)
 						rxq->if_cq,
 						&params));
 	}
-	if (rxq->qp != NULL) {
+	if (rxq->qp != NULL && !rxq->priv->isolated) {
 		rxq_promiscuous_disable(rxq);
 		rxq_allmulticast_disable(rxq);
 		rxq_mac_addrs_del(rxq);
-		claim_zero(ibv_destroy_qp(rxq->qp));
 	}
+	if (rxq->qp != NULL)
+		claim_zero(ibv_destroy_qp(rxq->qp));
 	if (rxq->cq != NULL)
 		claim_zero(ibv_destroy_cq(rxq->cq));
+	if (rxq->channel != NULL)
+		claim_zero(ibv_destroy_comp_channel(rxq->channel));
 	if (rxq->rd != NULL) {
 		struct ibv_exp_destroy_res_domain_attr attr = {
 			.comp_mask = 0,
@@ -2987,6 +3083,13 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			NB_SEGS(rep) = 0x2a;
 			PORT(rep) = 0x2a;
 			rep->ol_flags = -1;
+			/*
+			 * Clear special flags in mbuf to avoid
+			 * crashing while freeing.
+			 */
+			rep->ol_flags &=
+				~(uint64_t)(IND_ATTACHED_MBUF |
+					    CTRL_MBUF_FLAG);
 #endif
 			assert(rep->buf_len == seg->buf_len);
 			/* Reconfigure sge to use rep instead of seg. */
@@ -3330,15 +3433,18 @@ rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
  *   Completion queue to associate with QP.
  * @param desc
  *   Number of descriptors in QP (hint only).
- * @param parent
- *   If nonzero, create a parent QP, otherwise a child.
+ * @param children_n
+ *   If nonzero, a number of children for parent QP and zero for a child.
+ * @param rxq_parent
+ *   Pointer for a parent in a child case, NULL otherwise.
  *
  * @return
  *   QP pointer or NULL in case of error.
  */
 static struct ibv_qp *
 rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
-		 int parent, struct ibv_exp_res_domain *rd)
+		 int children_n, struct ibv_exp_res_domain *rd,
+		 struct rxq *rxq_parent)
 {
 	struct ibv_exp_qp_init_attr attr = {
 		/* CQ to be associated with the send queue. */
@@ -3368,16 +3474,16 @@ rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
 	attr.max_inl_recv = priv->inl_recv_size,
 	attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_INL_RECV;
 #endif
-	if (parent) {
+	if (children_n > 0) {
 		attr.qpg.qpg_type = IBV_EXP_QPG_PARENT;
 		/* TSS isn't necessary. */
 		attr.qpg.parent_attrib.tss_child_count = 0;
 		attr.qpg.parent_attrib.rss_child_count =
-			rte_align32pow2(priv->rxqs_n + 1) >> 1;
+			rte_align32pow2(children_n + 1) >> 1;
 		DEBUG("initializing parent RSS queue");
 	} else {
 		attr.qpg.qpg_type = IBV_EXP_QPG_CHILD_RX;
-		attr.qpg.qpg_parent = priv->rxq_parent.qp;
+		attr.qpg.qpg_parent = rxq_parent->qp;
 		DEBUG("initializing child RSS queue");
 	}
 	return ibv_exp_create_qp(priv->ctx, &attr);
@@ -3413,13 +3519,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 	struct ibv_recv_wr *bad_wr;
 	unsigned int mb_len;
 	int err;
-	int parent = (rxq == &priv->rxq_parent);
 
-	if (parent) {
-		ERROR("%p: cannot rehash parent queue %p",
-		      (void *)dev, (void *)rxq);
-		return EINVAL;
-	}
 	mb_len = rte_pktmbuf_data_room_size(rxq->mp);
 	DEBUG("%p: rehashing queue %p", (void *)dev, (void *)rxq);
 	/* Number of descriptors and mbufs currently allocated. */
@@ -3451,7 +3551,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 		return 0;
 	}
 	/* Remove attached flows if RSS is disabled (no parent queue). */
-	if (!priv->rss) {
+	if (!priv->rss && !priv->isolated) {
 		rxq_allmulticast_disable(&tmpl);
 		rxq_promiscuous_disable(&tmpl);
 		rxq_mac_addrs_del(&tmpl);
@@ -3464,6 +3564,8 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 	}
 	/* From now on, any failure will render the queue unusable.
 	 * Reinitialize QP. */
+	if (!tmpl.qp)
+		goto skip_init;
 	mod = (struct ibv_exp_qp_attr){ .qp_state = IBV_QPS_RESET };
 	err = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE);
 	if (err) {
@@ -3471,12 +3573,6 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 		assert(err > 0);
 		return err;
 	}
-	err = ibv_resize_cq(tmpl.cq, desc_n);
-	if (err) {
-		ERROR("%p: cannot resize CQ: %s", (void *)dev, strerror(err));
-		assert(err > 0);
-		return err;
-	}
 	mod = (struct ibv_exp_qp_attr){
 		/* Move the QP to this state. */
 		.qp_state = IBV_QPS_INIT,
@@ -3485,9 +3581,6 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 	};
 	err = ibv_exp_modify_qp(tmpl.qp, &mod,
 				(IBV_EXP_QP_STATE |
-#ifdef RSS_SUPPORT
-				 (parent ? IBV_EXP_QP_GROUP_RSS : 0) |
-#endif /* RSS_SUPPORT */
 				 IBV_EXP_QP_PORT));
 	if (err) {
 		ERROR("%p: QP state to IBV_QPS_INIT failed: %s",
@@ -3495,8 +3588,15 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 		assert(err > 0);
 		return err;
 	};
+skip_init:
+	err = ibv_resize_cq(tmpl.cq, desc_n);
+	if (err) {
+		ERROR("%p: cannot resize CQ: %s", (void *)dev, strerror(err));
+		assert(err > 0);
+		return err;
+	}
 	/* Reconfigure flows. Do not care for errors. */
-	if (!priv->rss) {
+	if (!priv->rss && !priv->isolated) {
 		rxq_mac_addrs_add(&tmpl);
 		if (priv->promisc)
 			rxq_promiscuous_enable(&tmpl);
@@ -3562,6 +3662,8 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 	rxq->elts_n = 0;
 	rte_free(rxq->elts.sp);
 	rxq->elts.sp = NULL;
+	if (!tmpl.qp)
+		goto skip_rtr;
 	/* Post WRs. */
 	err = ibv_post_recv(tmpl.qp,
 			    (tmpl.sp ?
@@ -3589,6 +3691,116 @@ skip_rtr:
 }
 
 /**
+ * Create verbs QP resources associated with a rxq.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param desc
+ *   Number of descriptors to configure in queue.
+ * @param inactive
+ *   If true, the queue is disabled because its index is higher or
+ *   equal to the real number of queues, which must be a power of 2.
+ * @param children_n
+ *   The number of children in a parent case, zero for a child.
+ * @param rxq_parent
+ *   The pointer to a parent RX structure for a child in RSS case,
+ *   NULL for parent.
+ *
+ * @return
+ *   0 on success, errno value on failure.
+ */
+int
+rxq_create_qp(struct rxq *rxq,
+	      uint16_t desc,
+	      int inactive,
+	      int children_n,
+	      struct rxq *rxq_parent)
+{
+	int ret;
+	struct ibv_exp_qp_attr mod;
+	struct ibv_exp_query_intf_params params;
+	enum ibv_exp_query_intf_status status;
+	struct ibv_recv_wr *bad_wr;
+	int parent = (children_n > 0);
+	struct priv *priv = rxq->priv;
+
+#ifdef RSS_SUPPORT
+	if (priv->rss && !inactive && (rxq_parent || parent))
+		rxq->qp = rxq_setup_qp_rss(priv, rxq->cq, desc,
+					   children_n, rxq->rd,
+					   rxq_parent);
+	else
+#endif /* RSS_SUPPORT */
+		rxq->qp = rxq_setup_qp(priv, rxq->cq, desc, rxq->rd);
+	if (rxq->qp == NULL) {
+		ret = (errno ? errno : EINVAL);
+		ERROR("QP creation failure: %s",
+		      strerror(ret));
+		return ret;
+	}
+	mod = (struct ibv_exp_qp_attr){
+		/* Move the QP to this state. */
+		.qp_state = IBV_QPS_INIT,
+		/* Primary port number. */
+		.port_num = priv->port
+	};
+	ret = ibv_exp_modify_qp(rxq->qp, &mod,
+				(IBV_EXP_QP_STATE |
+#ifdef RSS_SUPPORT
+				 (parent ? IBV_EXP_QP_GROUP_RSS : 0) |
+#endif /* RSS_SUPPORT */
+				 IBV_EXP_QP_PORT));
+	if (ret) {
+		ERROR("QP state to IBV_QPS_INIT failed: %s",
+		      strerror(ret));
+		return ret;
+	}
+	if (!priv->isolated && (parent || !priv->rss)) {
+		/* Configure MAC and broadcast addresses. */
+		ret = rxq_mac_addrs_add(rxq);
+		if (ret) {
+			ERROR("QP flow attachment failed: %s",
+			      strerror(ret));
+			return ret;
+		}
+	}
+	if (!parent) {
+		ret = ibv_post_recv(rxq->qp,
+				    (rxq->sp ?
+				     &(*rxq->elts.sp)[0].wr :
+				     &(*rxq->elts.no_sp)[0].wr),
+				    &bad_wr);
+		if (ret) {
+			ERROR("ibv_post_recv() failed for WR %p: %s",
+			      (void *)bad_wr,
+			      strerror(ret));
+			return ret;
+		}
+	}
+	mod = (struct ibv_exp_qp_attr){
+		.qp_state = IBV_QPS_RTR
+	};
+	ret = ibv_exp_modify_qp(rxq->qp, &mod, IBV_EXP_QP_STATE);
+	if (ret) {
+		ERROR("QP state to IBV_QPS_RTR failed: %s",
+		      strerror(ret));
+		return ret;
+	}
+	params = (struct ibv_exp_query_intf_params){
+		.intf_scope = IBV_EXP_INTF_GLOBAL,
+		.intf = IBV_EXP_INTF_QP_BURST,
+		.obj = rxq->qp,
+	};
+	rxq->if_qp = ibv_exp_query_intf(priv->ctx, &params, &status);
+	if (rxq->if_qp == NULL) {
+		ERROR("QP interface family query failed with status %d",
+		      status);
+		return errno;
+	}
+	return 0;
+}
+
+/**
  * Configure a RX queue.
  *
  * @param dev
@@ -3606,14 +3818,21 @@ skip_rtr:
  *   Thresholds parameters.
  * @param mp
  *   Memory pool for buffer allocations.
+ * @param children_n
+ *   The number of children in a parent case, zero for a child.
+ * @param rxq_parent
+ *   The pointer to a parent RX structure (or NULL) in a child case,
+ *   NULL for parent.
  *
  * @return
  *   0 on success, errno value on failure.
  */
 static int
 rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
-	  unsigned int socket, int inactive, const struct rte_eth_rxconf *conf,
-	  struct rte_mempool *mp)
+	  unsigned int socket, int inactive,
+	  const struct rte_eth_rxconf *conf,
+	  struct rte_mempool *mp, int children_n,
+	  struct rxq *rxq_parent)
 {
 	struct priv *priv = dev->data->dev_private;
 	struct rxq tmpl = {
@@ -3621,17 +3840,15 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		.mp = mp,
 		.socket = socket
 	};
-	struct ibv_exp_qp_attr mod;
 	union {
 		struct ibv_exp_query_intf_params params;
 		struct ibv_exp_cq_init_attr cq;
 		struct ibv_exp_res_domain_init_attr rd;
 	} attr;
 	enum ibv_exp_query_intf_status status;
-	struct ibv_recv_wr *bad_wr;
 	unsigned int mb_len;
 	int ret = 0;
-	int parent = (rxq == &priv->rxq_parent);
+	int parent = (children_n > 0);
 
 	(void)conf; /* Thresholds configuration (ignored). */
 	/*
@@ -3696,11 +3913,22 @@ skip_mr:
 		      (void *)dev, strerror(ret));
 		goto error;
 	}
+	if (dev->data->dev_conf.intr_conf.rxq) {
+		tmpl.channel = ibv_create_comp_channel(priv->ctx);
+		if (tmpl.channel == NULL) {
+			ret = ENOMEM;
+			ERROR("%p: Rx interrupt completion channel creation"
+			      " failure: %s",
+			      (void *)dev, strerror(ret));
+			goto error;
+		}
+	}
 	attr.cq = (struct ibv_exp_cq_init_attr){
 		.comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN,
 		.res_domain = tmpl.rd,
 	};
-	tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq);
+	tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, tmpl.channel, 0,
+				    &attr.cq);
 	if (tmpl.cq == NULL) {
 		ret = ENOMEM;
 		ERROR("%p: CQ creation failure: %s",
@@ -3711,45 +3939,6 @@ skip_mr:
 	      priv->device_attr.max_qp_wr);
 	DEBUG("priv->device_attr.max_sge is %d",
 	      priv->device_attr.max_sge);
-#ifdef RSS_SUPPORT
-	if (priv->rss && !inactive)
-		tmpl.qp = rxq_setup_qp_rss(priv, tmpl.cq, desc, parent,
-					   tmpl.rd);
-	else
-#endif /* RSS_SUPPORT */
-		tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc, tmpl.rd);
-	if (tmpl.qp == NULL) {
-		ret = (errno ? errno : EINVAL);
-		ERROR("%p: QP creation failure: %s",
-		      (void *)dev, strerror(ret));
-		goto error;
-	}
-	mod = (struct ibv_exp_qp_attr){
-		/* Move the QP to this state. */
-		.qp_state = IBV_QPS_INIT,
-		/* Primary port number. */
-		.port_num = priv->port
-	};
-	ret = ibv_exp_modify_qp(tmpl.qp, &mod,
-				(IBV_EXP_QP_STATE |
-#ifdef RSS_SUPPORT
-				 (parent ? IBV_EXP_QP_GROUP_RSS : 0) |
-#endif /* RSS_SUPPORT */
-				 IBV_EXP_QP_PORT));
-	if (ret) {
-		ERROR("%p: QP state to IBV_QPS_INIT failed: %s",
-		      (void *)dev, strerror(ret));
-		goto error;
-	}
-	if ((parent) || (!priv->rss))  {
-		/* Configure MAC and broadcast addresses. */
-		ret = rxq_mac_addrs_add(&tmpl);
-		if (ret) {
-			ERROR("%p: QP flow attachment failed: %s",
-			      (void *)dev, strerror(ret));
-			goto error;
-		}
-	}
 	/* Allocate descriptors for RX queues, except for the RSS parent. */
 	if (parent)
 		goto skip_alloc;
@@ -3760,29 +3949,14 @@ skip_mr:
 	if (ret) {
 		ERROR("%p: RXQ allocation failed: %s",
 		      (void *)dev, strerror(ret));
-		goto error;
-	}
-	ret = ibv_post_recv(tmpl.qp,
-			    (tmpl.sp ?
-			     &(*tmpl.elts.sp)[0].wr :
-			     &(*tmpl.elts.no_sp)[0].wr),
-			    &bad_wr);
-	if (ret) {
-		ERROR("%p: ibv_post_recv() failed for WR %p: %s",
-		      (void *)dev,
-		      (void *)bad_wr,
-		      strerror(ret));
-		goto error;
+		return ret;
 	}
 skip_alloc:
-	mod = (struct ibv_exp_qp_attr){
-		.qp_state = IBV_QPS_RTR
-	};
-	ret = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE);
-	if (ret) {
-		ERROR("%p: QP state to IBV_QPS_RTR failed: %s",
-		      (void *)dev, strerror(ret));
-		goto error;
+	if (parent || rxq_parent || !priv->rss) {
+		ret = rxq_create_qp(&tmpl, desc, inactive,
+				    children_n, rxq_parent);
+		if (ret)
+			goto error;
 	}
 	/* Save port ID. */
 	tmpl.port_id = dev->data->port_id;
@@ -3794,21 +3968,11 @@ skip_alloc:
 	};
 	tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status);
 	if (tmpl.if_cq == NULL) {
+		ret = EINVAL;
 		ERROR("%p: CQ interface family query failed with status %d",
 		      (void *)dev, status);
 		goto error;
 	}
-	attr.params = (struct ibv_exp_query_intf_params){
-		.intf_scope = IBV_EXP_INTF_GLOBAL,
-		.intf = IBV_EXP_INTF_QP_BURST,
-		.obj = tmpl.qp,
-	};
-	tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status);
-	if (tmpl.if_qp == NULL) {
-		ERROR("%p: QP interface family query failed with status %d",
-		      (void *)dev, status);
-		goto error;
-	}
 	/* Clean up rxq in case we're reinitializing it. */
 	DEBUG("%p: cleaning-up old rxq just in case", (void *)rxq);
 	rxq_cleanup(rxq);
@@ -3846,6 +4010,7 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		    unsigned int socket, const struct rte_eth_rxconf *conf,
 		    struct rte_mempool *mp)
 {
+	struct rxq *parent;
 	struct priv *priv = dev->data->dev_private;
 	struct rxq *rxq = (*priv->rxqs)[idx];
 	int inactive = 0;
@@ -3880,9 +4045,16 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 			return -ENOMEM;
 		}
 	}
-	if (idx >= rte_align32pow2(priv->rxqs_n + 1) >> 1)
-		inactive = 1;
-	ret = rxq_setup(dev, rxq, desc, socket, inactive, conf, mp);
+	if (priv->rss && !priv->isolated) {
+		/* The list consists of the single default one. */
+		parent = LIST_FIRST(&priv->parents);
+		if (idx >= rte_align32pow2(priv->rxqs_n + 1) >> 1)
+			inactive = 1;
+	} else {
+		parent = NULL;
+	}
+	ret = rxq_setup(dev, rxq, desc, socket,
+			inactive, conf, mp, 0, parent);
 	if (ret)
 		rte_free(rxq);
 	else {
@@ -3919,7 +4091,6 @@ mlx4_rx_queue_release(void *dpdk_rxq)
 		return;
 	priv = rxq->priv;
 	priv_lock(priv);
-	assert(rxq != &priv->rxq_parent);
 	for (i = 0; (i != priv->rxqs_n); ++i)
 		if ((*priv->rxqs)[i] == rxq) {
 			DEBUG("%p: removing RX queue %p from list",
@@ -3970,8 +4141,11 @@ mlx4_dev_start(struct rte_eth_dev *dev)
 	}
 	DEBUG("%p: attaching configured flows to all RX queues", (void *)dev);
 	priv->started = 1;
-	if (priv->rss) {
-		rxq = &priv->rxq_parent;
+	if (priv->isolated) {
+		rxq = NULL;
+		r = 1;
+	} else if (priv->rss) {
+		rxq = LIST_FIRST(&priv->parents);
 		r = 1;
 	} else {
 		rxq = (*priv->rxqs)[0];
@@ -4005,6 +4179,12 @@ mlx4_dev_start(struct rte_eth_dev *dev)
 		     (void *)dev);
 		goto err;
 	}
+	ret = priv_rx_intr_vec_enable(priv);
+	if (ret) {
+		ERROR("%p: Rx interrupt vector creation failed",
+		      (void *)dev);
+		goto err;
+	}
 	ret = mlx4_priv_flow_start(priv);
 	if (ret) {
 		ERROR("%p: flow start failed: %s",
@@ -4053,8 +4233,11 @@ mlx4_dev_stop(struct rte_eth_dev *dev)
 	}
 	DEBUG("%p: detaching flows from all RX queues", (void *)dev);
 	priv->started = 0;
-	if (priv->rss) {
-		rxq = &priv->rxq_parent;
+	if (priv->isolated) {
+		rxq = NULL;
+		r = 1;
+	} else if (priv->rss) {
+		rxq = LIST_FIRST(&priv->parents);
 		r = 1;
 	} else {
 		rxq = (*priv->rxqs)[0];
@@ -4188,7 +4371,7 @@ mlx4_dev_close(struct rte_eth_dev *dev)
 		priv->txqs = NULL;
 	}
 	if (priv->rss)
-		rxq_cleanup(&priv->rxq_parent);
+		priv_parent_list_cleanup(priv);
 	if (priv->pd != NULL) {
 		assert(priv->ctx != NULL);
 		claim_zero(ibv_dealloc_pd(priv->pd));
@@ -4197,6 +4380,7 @@ mlx4_dev_close(struct rte_eth_dev *dev)
 		assert(priv->ctx == NULL);
 	priv_dev_removal_interrupt_handler_uninstall(priv, dev);
 	priv_dev_link_interrupt_handler_uninstall(priv, dev);
+	priv_rx_intr_vec_disable(priv);
 	priv_unlock(priv);
 	memset(priv, 0, sizeof(*priv));
 }
@@ -4300,7 +4484,7 @@ mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 	unsigned int max;
 	char ifname[IF_NAMESIZE];
 
-	info->pci_dev = RTE_DEV_TO_PCI(dev->device);
+	info->pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 
 	if (priv == NULL)
 		return;
@@ -4481,6 +4665,8 @@ mlx4_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 	if (mlx4_is_secondary())
 		return;
 	priv_lock(priv);
+	if (priv->isolated)
+		goto end;
 	DEBUG("%p: removing MAC address from index %" PRIu32,
 	      (void *)dev, index);
 	/* Last array entry is reserved for broadcast. */
@@ -4514,6 +4700,12 @@ mlx4_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 		return -ENOTSUP;
 	(void)vmdq;
 	priv_lock(priv);
+	if (priv->isolated) {
+		DEBUG("%p: cannot add MAC address, "
+		      "device is in isolated mode", (void *)dev);
+		re = EPERM;
+		goto end;
+	}
 	DEBUG("%p: adding MAC address at index %" PRIu32,
 	      (void *)dev, index);
 	/* Last array entry is reserved for broadcast. */
@@ -4561,6 +4753,12 @@ mlx4_promiscuous_enable(struct rte_eth_dev *dev)
 	if (mlx4_is_secondary())
 		return;
 	priv_lock(priv);
+	if (priv->isolated) {
+		DEBUG("%p: cannot enable promiscuous, "
+		      "device is in isolated mode", (void *)dev);
+		priv_unlock(priv);
+		return;
+	}
 	if (priv->promisc) {
 		priv_unlock(priv);
 		return;
@@ -4569,7 +4767,7 @@ mlx4_promiscuous_enable(struct rte_eth_dev *dev)
 	if (!priv->started)
 		goto end;
 	if (priv->rss) {
-		ret = rxq_promiscuous_enable(&priv->rxq_parent);
+		ret = rxq_promiscuous_enable(LIST_FIRST(&priv->parents));
 		if (ret) {
 			priv_unlock(priv);
 			return;
@@ -4609,12 +4807,12 @@ mlx4_promiscuous_disable(struct rte_eth_dev *dev)
 	if (mlx4_is_secondary())
 		return;
 	priv_lock(priv);
-	if (!priv->promisc) {
+	if (!priv->promisc || priv->isolated) {
 		priv_unlock(priv);
 		return;
 	}
 	if (priv->rss) {
-		rxq_promiscuous_disable(&priv->rxq_parent);
+		rxq_promiscuous_disable(LIST_FIRST(&priv->parents));
 		goto end;
 	}
 	for (i = 0; (i != priv->rxqs_n); ++i)
@@ -4641,6 +4839,12 @@ mlx4_allmulticast_enable(struct rte_eth_dev *dev)
 	if (mlx4_is_secondary())
 		return;
 	priv_lock(priv);
+	if (priv->isolated) {
+		DEBUG("%p: cannot enable allmulticast, "
+		      "device is in isolated mode", (void *)dev);
+		priv_unlock(priv);
+		return;
+	}
 	if (priv->allmulti) {
 		priv_unlock(priv);
 		return;
@@ -4649,7 +4853,7 @@ mlx4_allmulticast_enable(struct rte_eth_dev *dev)
 	if (!priv->started)
 		goto end;
 	if (priv->rss) {
-		ret = rxq_allmulticast_enable(&priv->rxq_parent);
+		ret = rxq_allmulticast_enable(LIST_FIRST(&priv->parents));
 		if (ret) {
 			priv_unlock(priv);
 			return;
@@ -4689,12 +4893,12 @@ mlx4_allmulticast_disable(struct rte_eth_dev *dev)
 	if (mlx4_is_secondary())
 		return;
 	priv_lock(priv);
-	if (!priv->allmulti) {
+	if (!priv->allmulti || priv->isolated) {
 		priv_unlock(priv);
 		return;
 	}
 	if (priv->rss) {
-		rxq_allmulticast_disable(&priv->rxq_parent);
+		rxq_allmulticast_disable(LIST_FIRST(&priv->parents));
 		goto end;
 	}
 	for (i = 0; (i != priv->rxqs_n); ++i)
@@ -4832,7 +5036,7 @@ mlx4_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 		}
 		/* Reenable non-RSS queue attributes. No need to check
 		 * for errors at this stage. */
-		if (!priv->rss) {
+		if (!priv->rss && !priv->isolated) {
 			rxq_mac_addrs_add(rxq);
 			if (priv->promisc)
 				rxq_promiscuous_enable(rxq);
@@ -5003,7 +5207,7 @@ vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 		 * Rehashing flows in all RX queues is necessary.
 		 */
 		if (priv->rss)
-			rxq_mac_addrs_del(&priv->rxq_parent);
+			rxq_mac_addrs_del(LIST_FIRST(&priv->parents));
 		else
 			for (i = 0; (i != priv->rxqs_n); ++i)
 				if ((*priv->rxqs)[i] != NULL)
@@ -5011,7 +5215,7 @@ vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 		priv->vlan_filter[j].enabled = 1;
 		if (priv->started) {
 			if (priv->rss)
-				rxq_mac_addrs_add(&priv->rxq_parent);
+				rxq_mac_addrs_add(LIST_FIRST(&priv->parents));
 			else
 				for (i = 0; (i != priv->rxqs_n); ++i) {
 					if ((*priv->rxqs)[i] == NULL)
@@ -5025,7 +5229,7 @@ vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 		 * Rehashing flows in all RX queues is necessary.
 		 */
 		if (priv->rss)
-			rxq_mac_addrs_del(&priv->rxq_parent);
+			rxq_mac_addrs_del(LIST_FIRST(&priv->parents));
 		else
 			for (i = 0; (i != priv->rxqs_n); ++i)
 				if ((*priv->rxqs)[i] != NULL)
@@ -5033,7 +5237,7 @@ vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 		priv->vlan_filter[j].enabled = 0;
 		if (priv->started) {
 			if (priv->rss)
-				rxq_mac_addrs_add(&priv->rxq_parent);
+				rxq_mac_addrs_add(LIST_FIRST(&priv->parents));
 			else
 				for (i = 0; (i != priv->rxqs_n); ++i) {
 					if ((*priv->rxqs)[i] == NULL)
@@ -5067,6 +5271,12 @@ mlx4_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 	if (mlx4_is_secondary())
 		return -E_RTE_SECONDARY;
 	priv_lock(priv);
+	if (priv->isolated) {
+		DEBUG("%p: cannot set vlan filter, "
+		      "device is in isolated mode", (void *)dev);
+		priv_unlock(priv);
+		return -EINVAL;
+	}
 	ret = vlan_filter_set(dev, vlan_id, on);
 	priv_unlock(priv);
 	assert(ret >= 0);
@@ -5079,6 +5289,7 @@ const struct rte_flow_ops mlx4_flow_ops = {
 	.destroy = mlx4_flow_destroy,
 	.flush = mlx4_flow_flush,
 	.query = NULL,
+	.isolate = mlx4_flow_isolate,
 };
 
 /**
@@ -5157,6 +5368,8 @@ static const struct eth_dev_ops mlx4_dev_ops = {
 	.mac_addr_set = mlx4_mac_addr_set,
 	.mtu_set = mlx4_dev_set_mtu,
 	.filter_ctrl = mlx4_dev_filter_ctrl,
+	.rx_queue_intr_enable = mlx4_rx_intr_enable,
+	.rx_queue_intr_disable = mlx4_rx_intr_disable,
 };
 
 /**
@@ -5196,7 +5409,7 @@ mlx4_ibv_device_to_pci_addr(const struct ibv_device *device,
 		/* Extract information. */
 		if (sscanf(line,
 			   "PCI_SLOT_NAME="
-			   "%" SCNx16 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n",
+			   "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n",
 			   &pci_addr->domain,
 			   &pci_addr->bus,
 			   &pci_addr->devid,
@@ -5308,6 +5521,7 @@ priv_dev_status_handler(struct priv *priv, struct rte_eth_dev *dev,
 {
 	struct ibv_async_event event;
 	int port_change = 0;
+	struct rte_eth_link *link = &dev->data->dev_link;
 	int ret = 0;
 
 	*events = 0;
@@ -5329,22 +5543,20 @@ priv_dev_status_handler(struct priv *priv, struct rte_eth_dev *dev,
 			      event.event_type, event.element.port_num);
 		ibv_ack_async_event(&event);
 	}
-
-	if (port_change ^ priv->pending_alarm) {
-		struct rte_eth_link *link = &dev->data->dev_link;
-
-		priv->pending_alarm = 0;
-		mlx4_link_update(dev, 0);
-		if (((link->link_speed == 0) && link->link_status) ||
-		    ((link->link_speed != 0) && !link->link_status)) {
+	if (!port_change)
+		return ret;
+	mlx4_link_update(dev, 0);
+	if (((link->link_speed == 0) && link->link_status) ||
+	    ((link->link_speed != 0) && !link->link_status)) {
+		if (!priv->pending_alarm) {
 			/* Inconsistent status, check again later. */
 			priv->pending_alarm = 1;
 			rte_eal_alarm_set(MLX4_ALARM_TIMEOUT_US,
 					  mlx4_dev_link_status_handler,
 					  dev);
-		} else {
-			*events |= (1 << RTE_ETH_EVENT_INTR_LSC);
 		}
+	} else {
+		*events |= (1 << RTE_ETH_EVENT_INTR_LSC);
 	}
 	return ret;
 }
@@ -5365,10 +5577,12 @@ mlx4_dev_link_status_handler(void *arg)
 
 	priv_lock(priv);
 	assert(priv->pending_alarm == 1);
+	priv->pending_alarm = 0;
 	ret = priv_dev_status_handler(priv, dev, &events);
 	priv_unlock(priv);
 	if (ret > 0 && events & (1 << RTE_ETH_EVENT_INTR_LSC))
-		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL,
+					      NULL);
 }
 
 /**
@@ -5397,7 +5611,8 @@ mlx4_dev_interrupt_handler(void *cb_arg)
 		     i++) {
 			if (ev & (1 << i)) {
 				ev &= ~(1 << i);
-				_rte_eth_dev_callback_process(dev, i, NULL);
+				_rte_eth_dev_callback_process(dev, i, NULL,
+							      NULL);
 				ret--;
 			}
 		}
@@ -5592,6 +5807,154 @@ priv_dev_removal_interrupt_handler_install(struct priv *priv,
 }
 
 /**
+ * Allocate queue vector and fill epoll fd list for Rx interrupts.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   0 on success, negative on failure.
+ */
+static int
+priv_rx_intr_vec_enable(struct priv *priv)
+{
+	unsigned int i;
+	unsigned int rxqs_n = priv->rxqs_n;
+	unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
+	unsigned int count = 0;
+	struct rte_intr_handle *intr_handle = priv->dev->intr_handle;
+
+	if (!priv->dev->data->dev_conf.intr_conf.rxq)
+		return 0;
+	priv_rx_intr_vec_disable(priv);
+	intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n]));
+	if (intr_handle->intr_vec == NULL) {
+		ERROR("failed to allocate memory for interrupt vector,"
+		      " Rx interrupts will not be supported");
+		return -ENOMEM;
+	}
+	intr_handle->type = RTE_INTR_HANDLE_EXT;
+	for (i = 0; i != n; ++i) {
+		struct rxq *rxq = (*priv->rxqs)[i];
+		int fd;
+		int flags;
+		int rc;
+
+		/* Skip queues that cannot request interrupts. */
+		if (!rxq || !rxq->channel) {
+			/* Use invalid intr_vec[] index to disable entry. */
+			intr_handle->intr_vec[i] =
+				RTE_INTR_VEC_RXTX_OFFSET +
+				RTE_MAX_RXTX_INTR_VEC_ID;
+			continue;
+		}
+		if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
+			ERROR("too many Rx queues for interrupt vector size"
+			      " (%d), Rx interrupts cannot be enabled",
+			      RTE_MAX_RXTX_INTR_VEC_ID);
+			priv_rx_intr_vec_disable(priv);
+			return -1;
+		}
+		fd = rxq->channel->fd;
+		flags = fcntl(fd, F_GETFL);
+		rc = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
+		if (rc < 0) {
+			ERROR("failed to make Rx interrupt file descriptor"
+			      " %d non-blocking for queue index %d", fd, i);
+			priv_rx_intr_vec_disable(priv);
+			return rc;
+		}
+		intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
+		intr_handle->efds[count] = fd;
+		count++;
+	}
+	if (!count)
+		priv_rx_intr_vec_disable(priv);
+	else
+		intr_handle->nb_efd = count;
+	return 0;
+}
+
+/**
+ * Clean up Rx interrupts handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ */
+static void
+priv_rx_intr_vec_disable(struct priv *priv)
+{
+	struct rte_intr_handle *intr_handle = priv->dev->intr_handle;
+
+	rte_intr_free_epoll_fd(intr_handle);
+	free(intr_handle->intr_vec);
+	intr_handle->nb_efd = 0;
+	intr_handle->intr_vec = NULL;
+}
+
+/**
+ * DPDK callback for Rx queue interrupt enable.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param idx
+ *   Rx queue index.
+ *
+ * @return
+ *   0 on success, negative on failure.
+ */
+static int
+mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rxq *rxq = (*priv->rxqs)[idx];
+	int ret;
+
+	if (!rxq || !rxq->channel)
+		ret = EINVAL;
+	else
+		ret = ibv_req_notify_cq(rxq->cq, 0);
+	if (ret)
+		WARN("unable to arm interrupt on rx queue %d", idx);
+	return -ret;
+}
+
+/**
+ * DPDK callback for Rx queue interrupt disable.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param idx
+ *   Rx queue index.
+ *
+ * @return
+ *   0 on success, negative on failure.
+ */
+static int
+mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rxq *rxq = (*priv->rxqs)[idx];
+	struct ibv_cq *ev_cq;
+	void *ev_ctx;
+	int ret;
+
+	if (!rxq || !rxq->channel) {
+		ret = EINVAL;
+	} else {
+		ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
+		if (ret || ev_cq != rxq->cq)
+			ret = EINVAL;
+	}
+	if (ret)
+		WARN("unable to disable interrupt on rx queue %d",
+		     idx);
+	else
+		ibv_ack_cq_events(rxq->cq, 1);
+	return -ret;
+}
+
+/**
  * Verify and store value for device argument.
  *
  * @param[in] key
@@ -5760,12 +6123,15 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 	ibv_dev = list[i];
 
 	DEBUG("device opened");
-	if (ibv_query_device(attr_ctx, &device_attr))
+	if (ibv_query_device(attr_ctx, &device_attr)) {
+		err = ENODEV;
 		goto error;
+	}
 	INFO("%u port(s) detected", device_attr.phys_port_cnt);
 
 	if (mlx4_args(pci_dev->device.devargs, &conf)) {
 		ERROR("failed to process device arguments");
+		err = EINVAL;
 		goto error;
 	}
 	/* Use all ports when none are defined */
@@ -5799,19 +6165,23 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		DEBUG("using port %u (%08" PRIx32 ")", port, test);
 
 		ctx = ibv_open_device(ibv_dev);
-		if (ctx == NULL)
+		if (ctx == NULL) {
+			err = ENODEV;
 			goto port_error;
+		}
 
 		/* Check port status. */
 		err = ibv_query_port(ctx, port, &port_attr);
 		if (err) {
 			ERROR("port query failed: %s", strerror(err));
+			err = ENODEV;
 			goto port_error;
 		}
 
 		if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
 			ERROR("port %d is not configured in Ethernet mode",
 			      port);
+			err = EINVAL;
 			goto port_error;
 		}
 
@@ -5848,6 +6218,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 #ifdef HAVE_EXP_QUERY_DEVICE
 		if (ibv_exp_query_device(ctx, &exp_device_attr)) {
 			ERROR("ibv_exp_query_device() failed");
+			err = ENODEV;
 			goto port_error;
 		}
 #ifdef RSS_SUPPORT
@@ -5923,6 +6294,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		if (priv_get_mac(priv, &mac.addr_bytes)) {
 			ERROR("cannot get MAC address, is mlx4_en loaded?"
 			      " (errno: %s)", strerror(errno));
+			err = ENODEV;
 			goto port_error;
 		}
 		INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
@@ -6002,8 +6374,18 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 
 		eth_dev->device->driver = &mlx4_driver.driver;
 
+		/*
+		 * Copy and override interrupt handle to prevent it from
+		 * being shared between all ethdev instances of a given PCI
+		 * device. This is required to properly handle Rx interrupts
+		 * on all ports.
+		 */
+		priv->intr_handle_dev = *eth_dev->intr_handle;
+		eth_dev->intr_handle = &priv->intr_handle_dev;
+
 		priv->dev = eth_dev;
 		eth_dev->dev_ops = &mlx4_dev_ops;
+		eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
 
 		/* Bring Ethernet device up. */
 		DEBUG("forcing Ethernet interface up");
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 9a3bae90..c0ade4f1 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -182,7 +182,13 @@ enum {
 		(DEBUG__(__VA_ARGS__), 0)	\
 	})[0])
 #define DEBUG(...) DEBUG_(__VA_ARGS__, '\n')
+#ifndef MLX4_PMD_DEBUG_BROKEN_VERBS
 #define claim_zero(...) assert((__VA_ARGS__) == 0)
+#else /* MLX4_PMD_DEBUG_BROKEN_VERBS */
+#define claim_zero(...) \
+	(void)(((__VA_ARGS__) == 0) || \
+		DEBUG("Assertion `(" # __VA_ARGS__ ") == 0' failed (IGNORED)."))
+#endif /* MLX4_PMD_DEBUG_BROKEN_VERBS */
 #define claim_nonzero(...) assert((__VA_ARGS__) != 0)
 #define claim_positive(...) assert((__VA_ARGS__) >= 0)
 #else /* NDEBUG */
@@ -219,6 +225,7 @@ struct rxq_elt {
 
 /* RX queue descriptor. */
 struct rxq {
+	LIST_ENTRY(rxq) next; /* Used by parent queue only */
 	struct priv *priv; /* Back pointer to private data. */
 	struct rte_mempool *mp; /* Memory Pool for allocations. */
 	struct ibv_mr *mr; /* Memory Region (for mp). */
@@ -226,6 +233,7 @@ struct rxq {
 	struct ibv_qp *qp; /* Queue Pair. */
 	struct ibv_exp_qp_burst_family *if_qp; /* QP burst interface. */
 	struct ibv_exp_cq_family *if_cq; /* CQ interface. */
+	struct ibv_comp_channel *channel;
 	/*
 	 * Each VLAN ID requires a separate flow steering rule.
 	 */
@@ -246,6 +254,10 @@ struct rxq {
 	struct mlx4_rxq_stats stats; /* RX queue counters. */
 	unsigned int socket; /* CPU socket ID for allocations. */
 	struct ibv_exp_res_domain *rd; /* Resource Domain. */
+	struct {
+		uint16_t queues_n;
+		uint16_t queues[RTE_MAX_QUEUES_PER_PORT];
+	} rss;
 };
 
 /* TX element. */
@@ -334,24 +346,41 @@ struct priv {
 	unsigned int rss:1; /* RSS is enabled. */
 	unsigned int vf:1; /* This is a VF device. */
 	unsigned int pending_alarm:1; /* An alarm is pending. */
+	unsigned int isolated:1; /* Toggle isolated mode. */
 #ifdef INLINE_RECV
 	unsigned int inl_recv_size; /* Inline recv size */
 #endif
 	unsigned int max_rss_tbl_sz; /* Maximum number of RSS queues. */
 	/* RX/TX queues. */
-	struct rxq rxq_parent; /* Parent queue when RSS is enabled. */
 	unsigned int rxqs_n; /* RX queues array size. */
 	unsigned int txqs_n; /* TX queues array size. */
 	struct rxq *(*rxqs)[]; /* RX queues. */
 	struct txq *(*txqs)[]; /* TX queues. */
+	struct rte_intr_handle intr_handle_dev; /* Device interrupt handler. */
 	struct rte_intr_handle intr_handle; /* Interrupt handler. */
 	struct rte_flow_drop *flow_drop_queue; /* Flow drop queue. */
 	LIST_HEAD(mlx4_flows, rte_flow) flows;
 	struct rte_intr_conf intr_conf; /* Active interrupt configuration. */
+	LIST_HEAD(mlx4_parents, rxq) parents;
 	rte_spinlock_t lock; /* Lock for control functions. */
 };
 
 void priv_lock(struct priv *priv);
 void priv_unlock(struct priv *priv);
 
+int
+rxq_create_qp(struct rxq *rxq,
+	      uint16_t desc,
+	      int inactive,
+	      int children_n,
+	      struct rxq *rxq_parent);
+
+void
+rxq_parent_cleanup(struct rxq *parent);
+
+struct rxq *
+priv_parent_create(struct priv *priv,
+		   uint16_t queues[],
+		   uint16_t children_n);
+
 #endif /* RTE_PMD_MLX4_H_ */
diff --git a/drivers/net/mlx4/mlx4_flow.c b/drivers/net/mlx4/mlx4_flow.c
index edfac038..925c89c5 100644
--- a/drivers/net/mlx4/mlx4_flow.c
+++ b/drivers/net/mlx4/mlx4_flow.c
@@ -112,6 +112,7 @@ struct rte_flow_drop {
 static const enum rte_flow_action_type valid_actions[] = {
 	RTE_FLOW_ACTION_TYPE_DROP,
 	RTE_FLOW_ACTION_TYPE_QUEUE,
+	RTE_FLOW_ACTION_TYPE_RSS,
 	RTE_FLOW_ACTION_TYPE_END,
 };
 
@@ -672,6 +673,76 @@ priv_flow_validate(struct priv *priv,
 			if (!queue || (queue->index > (priv->rxqs_n - 1)))
 				goto exit_action_not_supported;
 			action.queue = 1;
+			action.queues_n = 1;
+			action.queues[0] = queue->index;
+		} else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
+			int i;
+			int ierr;
+			const struct rte_flow_action_rss *rss =
+				(const struct rte_flow_action_rss *)
+				actions->conf;
+
+			if (!priv->hw_rss) {
+				rte_flow_error_set(error, ENOTSUP,
+					   RTE_FLOW_ERROR_TYPE_ACTION,
+					   actions,
+					   "RSS cannot be used with "
+					   "the current configuration");
+				return -rte_errno;
+			}
+			if (!priv->isolated) {
+				rte_flow_error_set(error, ENOTSUP,
+					   RTE_FLOW_ERROR_TYPE_ACTION,
+					   actions,
+					   "RSS cannot be used without "
+					   "isolated mode");
+				return -rte_errno;
+			}
+			if (!rte_is_power_of_2(rss->num)) {
+				rte_flow_error_set(error, ENOTSUP,
+					   RTE_FLOW_ERROR_TYPE_ACTION,
+					   actions,
+					   "the number of queues "
+					   "should be power of two");
+				return -rte_errno;
+			}
+			if (priv->max_rss_tbl_sz < rss->num) {
+				rte_flow_error_set(error, ENOTSUP,
+					   RTE_FLOW_ERROR_TYPE_ACTION,
+					   actions,
+					   "the number of queues "
+					   "is too large");
+				return -rte_errno;
+			}
+			/* checking indexes array */
+			ierr = 0;
+			for (i = 0; i < rss->num; ++i) {
+				int j;
+				if (rss->queue[i] >= priv->rxqs_n)
+					ierr = 1;
+				/*
+				 * Prevent the user from specifying
+				 * the same queue twice in the RSS array.
+				 */
+				for (j = i + 1; j < rss->num && !ierr; ++j)
+					if (rss->queue[j] == rss->queue[i])
+						ierr = 1;
+				if (ierr) {
+					rte_flow_error_set(
+						error,
+						ENOTSUP,
+						RTE_FLOW_ERROR_TYPE_HANDLE,
+						NULL,
+						"RSS action only supports "
+						"unique queue indices "
+						"in a list");
+					return -rte_errno;
+				}
+			}
+			action.queue = 1;
+			action.queues_n = rss->num;
+			for (i = 0; i < rss->num; ++i)
+				action.queues[i] = rss->queue[i];
 		} else {
 			goto exit_action_not_supported;
 		}
@@ -797,6 +868,82 @@ err:
 }
 
 /**
+ * Get RSS parent rxq structure for given queues.
+ *
+ * Creates a new or returns an existed one.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param queues
+ *   queues indices array, NULL in default RSS case.
+ * @param children_n
+ *   the size of queues array.
+ *
+ * @return
+ *   Pointer to a parent rxq structure, NULL on failure.
+ */
+static struct rxq *
+priv_parent_get(struct priv *priv,
+		uint16_t queues[],
+		uint16_t children_n,
+		struct rte_flow_error *error)
+{
+	unsigned int i;
+	struct rxq *parent;
+
+	for (parent = LIST_FIRST(&priv->parents);
+	     parent;
+	     parent = LIST_NEXT(parent, next)) {
+		unsigned int same = 0;
+		unsigned int overlap = 0;
+
+		/*
+		 * Find out whether an appropriate parent queue already exists
+		 * and can be reused, otherwise make sure there are no overlaps.
+		 */
+		for (i = 0; i < children_n; ++i) {
+			unsigned int j;
+
+			for (j = 0; j < parent->rss.queues_n; ++j) {
+				if (parent->rss.queues[j] != queues[i])
+					continue;
+				++overlap;
+				if (i == j)
+					++same;
+			}
+		}
+		if (same == children_n &&
+			children_n == parent->rss.queues_n)
+			return parent;
+		else if (overlap)
+			goto error;
+	}
+	/* Exclude the cases when some QPs were created without RSS */
+	for (i = 0; i < children_n; ++i) {
+		struct rxq *rxq = (*priv->rxqs)[queues[i]];
+		if (rxq->qp)
+			goto error;
+	}
+	parent = priv_parent_create(priv, queues, children_n);
+	if (!parent) {
+		rte_flow_error_set(error,
+				   ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL, "flow rule creation failure");
+		return NULL;
+	}
+	return parent;
+
+error:
+	rte_flow_error_set(error,
+			   EEXIST,
+			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			   NULL,
+			   "sharing a queue between several"
+			   " RSS groups is not supported");
+	return NULL;
+}
+
+/**
  * Complete flow rule creation.
  *
  * @param priv
@@ -819,6 +966,7 @@ priv_flow_create_action_queue(struct priv *priv,
 {
 	struct ibv_qp *qp;
 	struct rte_flow *rte_flow;
+	struct rxq *rxq_parent = NULL;
 
 	assert(priv->pd);
 	assert(priv->ctx);
@@ -829,14 +977,46 @@ priv_flow_create_action_queue(struct priv *priv,
 		return NULL;
 	}
 	if (action->drop) {
-		qp = priv->flow_drop_queue->qp;
+		qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
 	} else {
-		struct rxq *rxq = (*priv->rxqs)[action->queue_id];
+		int ret;
+		unsigned int i;
+		struct rxq *rxq = NULL;
 
-		qp = rxq->qp;
+		if (action->queues_n > 1) {
+			rxq_parent = priv_parent_get(priv, action->queues,
+						     action->queues_n, error);
+			if (!rxq_parent)
+				goto error;
+		}
+		for (i = 0; i < action->queues_n; ++i) {
+			rxq = (*priv->rxqs)[action->queues[i]];
+			/*
+			 * In case of isolated mode we postpone
+			 * ibv receive queue creation till the first
+			 * rte_flow rule will be applied on that queue.
+			 */
+			if (!rxq->qp) {
+				assert(priv->isolated);
+				ret = rxq_create_qp(rxq, rxq->elts_n,
+						    0, 0, rxq_parent);
+				if (ret) {
+					rte_flow_error_set(
+						error,
+						ENOMEM,
+						RTE_FLOW_ERROR_TYPE_HANDLE,
+						NULL,
+						"flow rule creation failure");
+					goto error;
+				}
+			}
+		}
+		qp = action->queues_n > 1 ? rxq_parent->qp : rxq->qp;
 		rte_flow->qp = qp;
 	}
 	rte_flow->ibv_attr = ibv_attr;
+	if (!priv->started)
+		return rte_flow;
 	rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
 	if (!rte_flow->ibv_flow) {
 		rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
@@ -846,6 +1026,8 @@ priv_flow_create_action_queue(struct priv *priv,
 	return rte_flow;
 
 error:
+	if (rxq_parent)
+		rxq_parent_cleanup(rxq_parent);
 	rte_free(rte_flow);
 	return NULL;
 }
@@ -909,11 +1091,22 @@ priv_flow_create(struct priv *priv,
 			continue;
 		} else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 			action.queue = 1;
-			action.queue_id =
+			action.queues_n = 1;
+			action.queues[0] =
 				((const struct rte_flow_action_queue *)
 				 actions->conf)->index;
 		} else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
 			action.drop = 1;
+		} else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
+			unsigned int i;
+			const struct rte_flow_action_rss *rss =
+				(const struct rte_flow_action_rss *)
+				 actions->conf;
+
+			action.queue = 1;
+			action.queues_n = rss->num;
+			for (i = 0; i < rss->num; ++i)
+				action.queues[i] = rss->queue[i];
 		} else {
 			rte_flow_error_set(error, ENOTSUP,
 					   RTE_FLOW_ERROR_TYPE_ACTION,
@@ -957,6 +1150,43 @@ mlx4_flow_create(struct rte_eth_dev *dev,
 }
 
 /**
+ * @see rte_flow_isolate()
+ *
+ * Must be done before calling dev_configure().
+ *
+ * @param dev
+ *   Pointer to the ethernet device structure.
+ * @param enable
+ *   Nonzero to enter isolated mode, attempt to leave it otherwise.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL. PMDs initialize this
+ *   structure in case of error only.
+ *
+ * @return
+ *   0 on success, a negative value on error.
+ */
+int
+mlx4_flow_isolate(struct rte_eth_dev *dev,
+		  int enable,
+		  struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	priv_lock(priv);
+	if (priv->rxqs) {
+		rte_flow_error_set(error, ENOTSUP,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL, "isolated mode must be set"
+				   " before configuring the device");
+		priv_unlock(priv);
+		return -rte_errno;
+	}
+	priv->isolated = !!enable;
+	priv_unlock(priv);
+	return 0;
+}
+
+/**
  * Destroy a flow.
  *
  * @param priv
diff --git a/drivers/net/mlx4/mlx4_flow.h b/drivers/net/mlx4/mlx4_flow.h
index 12a293e4..beabcf2d 100644
--- a/drivers/net/mlx4/mlx4_flow.h
+++ b/drivers/net/mlx4/mlx4_flow.h
@@ -90,10 +90,16 @@ struct mlx4_flow {
 	unsigned int offset; /**< Offset in bytes in the ibv_attr buffer. */
 };
 
+int
+mlx4_flow_isolate(struct rte_eth_dev *dev,
+		  int enable,
+		  struct rte_flow_error *error);
+
 struct mlx4_flow_action {
 	uint32_t drop:1; /**< Target is a drop queue. */
 	uint32_t queue:1; /**< Target is a receive queue. */
-	uint32_t queue_id; /**< Identifier of the queue. */
+	uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queue indices to use. */
+	uint16_t queues_n; /**< Number of entries in queue[] */
 };
 
 int mlx4_priv_flow_start(struct priv *priv);